Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(qwen4_exp): take the expert quant kind from the checkpoint QuantConfig (#426)

Xiaoze Fanfa814ab594fe
diff --git a/docs/models.md b/docs/models.md
index ca3c1853..10edbfae 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -9,7 +9,7 @@ for them; other checkpoints of the same architectures work too.
| GLM-5.3-Flash | [RedHatAI/GLM-5.3-Flash-NVFP4](https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4) |
| GLM-5.2 | [nvidia/GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4) |
| GLM-4.7 | [nvidia/GLM-4.7-NVFP4](https://huggingface.co/nvidia/GLM-4.7-NVFP4) |
-| Qwen3.8-Flash-Next | [Qwen/Qwen3.8-Flash-Next-FP8](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8), [RadixArk/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4) |
+| Qwen3.8-Flash-Next | [Qwen/Qwen3.8-Flash-Next-FP8](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8), [RadixArk/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4), [nvidia/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/nvidia/Qwen3.8-Flash-Next-NVFP4) |
| Qwen3.6 / Qwen3.5 MoE | [Qwen/Qwen3.6-35B-A3B](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8)), [nvidia/Qwen3.6-35B-A3B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4), [Qwen/Qwen3.5-35B-A3B](https://huggingface.co/Qwen/Qwen3.5-35B-A3B) ([-FP8](https://huggingface.co/Qwen/Qwen3.5-35B-A3B-FP8)) |
| Qwen3.8 / Qwen3.6 dense | [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) ([-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8)), [RadixArk/Qwen3.8-27B-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4), [Qwen/Qwen3.6-27B](https://huggingface.co/Qwen/Qwen3.6-27B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-27B-FP8)), [nvidia/Qwen3.6-27B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4) |
| Qwen3-MoE | [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) |
diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.py
index bb5d1dff..4bcd28de 100644
--- a/python/freetoken/models/qwen4_exp/config.py
+++ b/python/freetoken/models/qwen4_exp/config.py
@@ -1,11 +1,11 @@
from __future__ import annotations
from dataclasses import dataclass
-from fnmatch import fnmatch
from typing import Any, Tuple
import torch
+from freetoken.layers.quantization import QuantConfig
from freetoken.models.config import (
FullAttentionGroupConfig,
LinearGatedDeltaGroupConfig,
@@ -93,17 +93,6 @@ def ple_slot_states(args: Qwen4ExpArgs) -> Tuple[SlotStateSpec, ...]:
)
-def _quant_get(hf_config: Any):
- quant = getattr(hf_config, "quantization_config", None)
- if quant is None:
- return None
- return quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
-
-
-def _ignored(patterns, module_name: str) -> bool:
- return any(fnmatch(module_name, pat) for pat in patterns)
-
-
def _layer_types(text: Any) -> list[str]:
layer_types = getattr(text, "layer_types", None)
if layer_types is not None:
@@ -149,40 +138,14 @@ def parse_config(hf_config: Any) -> ModelConfig:
else {k: v for k, v in rope_params.items() if not isinstance(v, (list, dict))}
)
- get = _quant_get(hf_config)
- if get is None:
- expert_quant = attn_quant = dense_quant = lm_head_quant = "none"
- else:
- algo = str(get("quant_algo") or get("quant_method") or "").lower()
- block = get("weight_block_size")
- if algo == "fp8" and block:
- # Official FP8 build (DeepSeek-V3-style block-fp8): only the routed experts
- # are quantized (fp8-e4m3 weights + per-block weight_scale_inv); attention,
- # GDN, the shared expert, HC, PLE and lm_head stay bf16.
- bs = tuple(int(x) for x in block)
- assert bs == (128, 128), f"only 128x128 block-fp8 is supported, got {bs}"
- expert_quant = "fp8_block"
- attn_quant = dense_quant = lm_head_quant = "none"
- else:
- is_fp4 = "fp4" in algo
- ignore = list(get("ignore") or [])
-
- # The RadixArk NVFP4 build quantizes only the routed experts; attention/GDN,
- # the shared expert, HC, PLE and lm_head all sit in the modelopt ignore list
- # and stay bf16. Derive every flag from that list instead of assuming the split.
- def _quant(probe: str) -> str:
- return "nvfp4" if is_fp4 and not _ignored(ignore, probe) else "none"
-
- prefix = "model.language_model.layers.0"
- expert_quant = _quant(f"{prefix}.mlp.experts.0.gate_proj")
- dense_quant = _quant(f"{prefix}.mlp.shared_expert.gate_proj")
- attn_quant = _quant(f"{prefix}.self_attn.q_proj")
- lm_head_quant = _quant("lm_head")
-
layer_types = _layer_types(text)
full_ids = tuple(i for i, t in enumerate(layer_types) if t == "full_attention")
linear_ids = tuple(i for i, t in enumerate(layer_types) if t == "linear_attention")
+ # the engine reads this flag for its MoE strategy decisions; every module takes its own scheme from the QuantConfig when it is built
+ expert_scheme = QuantConfig.from_hf(hf_config).scheme_for_name("model.language_model.layers.0.mlp.experts.0.gate_proj")
+ expert_quant = "none" if expert_scheme is None else str(expert_scheme.kind)
+
# HF stores ple_layer_ids one-indexed (validated upstream as [1, num_layers]).
ple_layer_ids = tuple(int(i) - 1 for i in (getattr(text, "ple_layer_ids", None) or ()))
for lid in ple_layer_ids:
@@ -282,9 +245,6 @@ def parse_config(hf_config: Any) -> ModelConfig:
image_token_id=getattr(hf_config, "image_token_id", None),
attention_groups=groups,
expert_quant=expert_quant,
- attn_quant=attn_quant,
- dense_quant=dense_quant,
- lm_head_quant=lm_head_quant,
qwen4_args=qwen4_args,
slot_states=ple_slot_states(qwen4_args),
)
diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py
index 8fc2e550..77135630 100644
--- a/tests/models/qwen4_exp/test_config.py
+++ b/tests/models/qwen4_exp/test_config.py
@@ -1,4 +1,4 @@
-"""qwen4_exp.parse_config against a synthetic config shaped like the RadixArk NVFP4 checkpoint."""
+"""qwen4_exp.parse_config against synthetic configs shaped like the released checkpoints."""
from types import SimpleNamespace
@@ -61,31 +61,68 @@ def _text_config():
)
-def _hf_config():
+# quantization_config of each released checkpoint, trimmed to the entries parse_config looks at
+
+# RadixArk/Qwen3.8-Flash-Next-NVFP4: modelopt NVFP4 everywhere except the ignore list
+RADIXARK_NVFP4 = {
+ "quant_algo": "NVFP4",
+ "quant_method": "modelopt",
+ "ignore": [
+ "model.embed_tokens",
+ "mtp.*",
+ "model.mtp.*",
+ "*.self_attn.*",
+ "*.linear_attn.*",
+ "*.mlp.gate*",
+ "*.mlp.shared_expert.*",
+ "*.mlp.shared_expert_gate*",
+ "*hyper_connection*",
+ "*.ple.*",
+ "model.visual.*",
+ "model.language_model.embed_tokens",
+ "lm_head",
+ ],
+}
+
+# nvidia/Qwen3.8-Flash-Next-NVFP4: modelopt MIXED_PRECISION, the per-module algo sits in quantized_layers
+NVIDIA_NVFP4 = {
+ "quant_algo": "MIXED_PRECISION",
+ "quant_method": "modelopt",
+ "quantized_layers": {
+ **{f"model.language_model.layers.{i}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for i in range(48)},
+ "model.language_model.layers.1.ple.ple_embedding.ngram_embedding": {"quant_algo": "FP8"},
+ "mtp.layers.0.mlp.experts": {"quant_algo": "FP8_PB_WO", "group_size": 128},
+ },
+ "ignore": ["lm_head", "model.language_model.embed_tokens", "model.language_model.layers.0.mlp.shared_expert*", "model.visual*"],
+}
+
+# Qwen/Qwen3.8-Flash-Next-FP8: 128x128 block-fp8 experts, everything else listed in modules_to_not_convert
+QWEN_FP8 = {
+ "quant_method": "fp8",
+ "activation_scheme": "dynamic",
+ "weight_per_tensor": False,
+ "act_per_tensor": False,
+ "weight_block_size": [128, 128],
+ "modules_to_not_convert": [
+ "lm_head",
+ "model.language_model.embed_tokens",
+ "model.language_model.hyper_connection_mixer.input_mix_weight_up",
+ "model.language_model.layers.0.linear_attn.in_proj_qkv",
+ "model.language_model.layers.3.self_attn.q_proj",
+ "model.language_model.layers.3.mlp.gate",
+ "model.language_model.layers.3.mlp.shared_expert.gate_proj",
+ ],
+ "modules_to_convert": ["ple.ple_embedding.ngram_embedding"],
+}
+
+
+def _hf_config(quantization_config=RADIXARK_NVFP4):
return SimpleNamespace(
model_type="qwen4_exp",
architectures=["Qwen4ExpForConditionalGeneration"],
image_token_id=248056,
text_config=_text_config(),
- quantization_config={
- "quant_algo": "NVFP4",
- "quant_method": "modelopt",
- "ignore": [
- "model.embed_tokens",
- "mtp.*",
- "model.mtp.*",
- "*.self_attn.*",
- "*.linear_attn.*",
- "*.mlp.gate*",
- "*.mlp.shared_expert.*",
- "*.mlp.shared_expert_gate*",
- "*hyper_connection*",
- "*.ple.*",
- "model.visual.*",
- "model.language_model.embed_tokens",
- "lm_head",
- ],
- },
+ quantization_config=quantization_config,
)
@@ -115,22 +152,25 @@ def test_kv_specs_resolve_qsa():
assert cfg.has_linear_attention
-def test_moe_and_quant_flags():
+def test_moe_dims():
cfg = parse_config(_hf_config())
assert cfg.num_experts == 512
assert cfg.num_experts_per_tok == 10
assert cfg.norm_topk_prob is True
assert cfg.moe_enabled
- assert cfg.expert_quant == "nvfp4"
- assert cfg.dense_quant == "none"
- assert cfg.attn_quant == "none"
- assert cfg.lm_head_quant == "none"
-def test_unquantized_config_parses():
- hf = _hf_config()
- hf.quantization_config = None
- assert parse_config(hf).expert_quant == "none"
+ "quantization_config, expert_quant",
+ [
+ pytest.param(RADIXARK_NVFP4, "nvfp4", id="RadixArk/Qwen3.8-Flash-Next-NVFP4"),
+ pytest.param(NVIDIA_NVFP4, "nvfp4", id="nvidia/Qwen3.8-Flash-Next-NVFP4"),
+ pytest.param(QWEN_FP8, "fp8_block", id="Qwen/Qwen3.8-Flash-Next-FP8"),
+ pytest.param(None, "none", id="Qwen/Qwen3.8-Flash-Next"),
+ ],
+)
+def test_released_checkpoints_expert_quant(quantization_config, expert_quant):
+ assert parse_config(_hf_config(quantization_config)).expert_quant == expert_quant
def test_qwen4_args_payload():