FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(qwen4_exp): take the expert quant kind from the checkpoint QuantConfig (#426)
Xiaoze Fanfa814ab594fe
diff --git a/docs/models.md b/docs/models.mdindex ca3c1853..10edbfae 100644--- a/docs/models.md+++ b/docs/models.md@@ -9,7 +9,7 @@ for them; other checkpoints of the same architectures work too.| GLM-5.3-Flash | [RedHatAI/GLM-5.3-Flash-NVFP4](https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4) || GLM-5.2 | [nvidia/GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4) || GLM-4.7 | [nvidia/GLM-4.7-NVFP4](https://huggingface.co/nvidia/GLM-4.7-NVFP4) |-| Qwen3.8-Flash-Next | [Qwen/Qwen3.8-Flash-Next-FP8](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8), [RadixArk/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4) |+| Qwen3.8-Flash-Next | [Qwen/Qwen3.8-Flash-Next-FP8](https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8), [RadixArk/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-Flash-Next-NVFP4), [nvidia/Qwen3.8-Flash-Next-NVFP4](https://huggingface.co/nvidia/Qwen3.8-Flash-Next-NVFP4) || Qwen3.6 / Qwen3.5 MoE | [Qwen/Qwen3.6-35B-A3B](https://huggingface.co/Qwen/Qwen3.6-35B-A3B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-35B-A3B-FP8)), [nvidia/Qwen3.6-35B-A3B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-35B-A3B-NVFP4), [Qwen/Qwen3.5-35B-A3B](https://huggingface.co/Qwen/Qwen3.5-35B-A3B) ([-FP8](https://huggingface.co/Qwen/Qwen3.5-35B-A3B-FP8)) || Qwen3.8 / Qwen3.6 dense | [Qwen/Qwen3.8-27B](https://huggingface.co/Qwen/Qwen3.8-27B) ([-FP8](https://huggingface.co/Qwen/Qwen3.8-27B-FP8)), [RadixArk/Qwen3.8-27B-NVFP4](https://huggingface.co/RadixArk/Qwen3.8-27B-NVFP4), [Qwen/Qwen3.6-27B](https://huggingface.co/Qwen/Qwen3.6-27B) ([-FP8](https://huggingface.co/Qwen/Qwen3.6-27B-FP8)), [nvidia/Qwen3.6-27B-NVFP4](https://huggingface.co/nvidia/Qwen3.6-27B-NVFP4) || Qwen3-MoE | [Qwen/Qwen3-30B-A3B](https://huggingface.co/Qwen/Qwen3-30B-A3B) |diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.pyindex bb5d1dff..4bcd28de 100644--- a/python/freetoken/models/qwen4_exp/config.py+++ b/python/freetoken/models/qwen4_exp/config.py@@ -1,11 +1,11 @@from __future__ import annotationsfrom dataclasses import dataclass-from fnmatch import fnmatchfrom typing import Any, Tupleimport torch+from freetoken.layers.quantization import QuantConfigfrom freetoken.models.config import (FullAttentionGroupConfig,LinearGatedDeltaGroupConfig,@@ -93,17 +93,6 @@ def ple_slot_states(args: Qwen4ExpArgs) -> Tuple[SlotStateSpec, ...]:)-def _quant_get(hf_config: Any):- quant = getattr(hf_config, "quantization_config", None)- if quant is None:- return None- return quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))---def _ignored(patterns, module_name: str) -> bool:- return any(fnmatch(module_name, pat) for pat in patterns)--def _layer_types(text: Any) -> list[str]:layer_types = getattr(text, "layer_types", None)if layer_types is not None:@@ -149,40 +138,14 @@ def parse_config(hf_config: Any) -> ModelConfig:else {k: v for k, v in rope_params.items() if not isinstance(v, (list, dict))})- get = _quant_get(hf_config)- if get is None:- expert_quant = attn_quant = dense_quant = lm_head_quant = "none"- else:- algo = str(get("quant_algo") or get("quant_method") or "").lower()- block = get("weight_block_size")- if algo == "fp8" and block:- # Official FP8 build (DeepSeek-V3-style block-fp8): only the routed experts- # are quantized (fp8-e4m3 weights + per-block weight_scale_inv); attention,- # GDN, the shared expert, HC, PLE and lm_head stay bf16.- bs = tuple(int(x) for x in block)- assert bs == (128, 128), f"only 128x128 block-fp8 is supported, got {bs}"- expert_quant = "fp8_block"- attn_quant = dense_quant = lm_head_quant = "none"- else:- is_fp4 = "fp4" in algo- ignore = list(get("ignore") or [])-- # The RadixArk NVFP4 build quantizes only the routed experts; attention/GDN,- # the shared expert, HC, PLE and lm_head all sit in the modelopt ignore list- # and stay bf16. Derive every flag from that list instead of assuming the split.- def _quant(probe: str) -> str:- return "nvfp4" if is_fp4 and not _ignored(ignore, probe) else "none"-- prefix = "model.language_model.layers.0"- expert_quant = _quant(f"{prefix}.mlp.experts.0.gate_proj")- dense_quant = _quant(f"{prefix}.mlp.shared_expert.gate_proj")- attn_quant = _quant(f"{prefix}.self_attn.q_proj")- lm_head_quant = _quant("lm_head")-layer_types = _layer_types(text)full_ids = tuple(i for i, t in enumerate(layer_types) if t == "full_attention")linear_ids = tuple(i for i, t in enumerate(layer_types) if t == "linear_attention")+ # the engine reads this flag for its MoE strategy decisions; every module takes its own scheme from the QuantConfig when it is built+ expert_scheme = QuantConfig.from_hf(hf_config).scheme_for_name("model.language_model.layers.0.mlp.experts.0.gate_proj")+ expert_quant = "none" if expert_scheme is None else str(expert_scheme.kind)+# HF stores ple_layer_ids one-indexed (validated upstream as [1, num_layers]).ple_layer_ids = tuple(int(i) - 1 for i in (getattr(text, "ple_layer_ids", None) or ()))for lid in ple_layer_ids:@@ -282,9 +245,6 @@ def parse_config(hf_config: Any) -> ModelConfig:image_token_id=getattr(hf_config, "image_token_id", None),attention_groups=groups,expert_quant=expert_quant,- attn_quant=attn_quant,- dense_quant=dense_quant,- lm_head_quant=lm_head_quant,qwen4_args=qwen4_args,slot_states=ple_slot_states(qwen4_args),)diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.pyindex 8fc2e550..77135630 100644--- a/tests/models/qwen4_exp/test_config.py+++ b/tests/models/qwen4_exp/test_config.py@@ -1,4 +1,4 @@-"""qwen4_exp.parse_config against a synthetic config shaped like the RadixArk NVFP4 checkpoint."""+"""qwen4_exp.parse_config against synthetic configs shaped like the released checkpoints."""from types import SimpleNamespace@@ -61,31 +61,68 @@ def _text_config():)-def _hf_config():+# quantization_config of each released checkpoint, trimmed to the entries parse_config looks at++# RadixArk/Qwen3.8-Flash-Next-NVFP4: modelopt NVFP4 everywhere except the ignore list+RADIXARK_NVFP4 = {+ "quant_algo": "NVFP4",+ "quant_method": "modelopt",+ "ignore": [+ "model.embed_tokens",+ "mtp.*",+ "model.mtp.*",+ "*.self_attn.*",+ "*.linear_attn.*",+ "*.mlp.gate*",+ "*.mlp.shared_expert.*",+ "*.mlp.shared_expert_gate*",+ "*hyper_connection*",+ "*.ple.*",+ "model.visual.*",+ "model.language_model.embed_tokens",+ "lm_head",+ ],+}++# nvidia/Qwen3.8-Flash-Next-NVFP4: modelopt MIXED_PRECISION, the per-module algo sits in quantized_layers+NVIDIA_NVFP4 = {+ "quant_algo": "MIXED_PRECISION",+ "quant_method": "modelopt",+ "quantized_layers": {+ **{f"model.language_model.layers.{i}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for i in range(48)},+ "model.language_model.layers.1.ple.ple_embedding.ngram_embedding": {"quant_algo": "FP8"},+ "mtp.layers.0.mlp.experts": {"quant_algo": "FP8_PB_WO", "group_size": 128},+ },+ "ignore": ["lm_head", "model.language_model.embed_tokens", "model.language_model.layers.0.mlp.shared_expert*", "model.visual*"],+}++# Qwen/Qwen3.8-Flash-Next-FP8: 128x128 block-fp8 experts, everything else listed in modules_to_not_convert+QWEN_FP8 = {+ "quant_method": "fp8",+ "activation_scheme": "dynamic",+ "weight_per_tensor": False,+ "act_per_tensor": False,+ "weight_block_size": [128, 128],+ "modules_to_not_convert": [+ "lm_head",+ "model.language_model.embed_tokens",+ "model.language_model.hyper_connection_mixer.input_mix_weight_up",+ "model.language_model.layers.0.linear_attn.in_proj_qkv",+ "model.language_model.layers.3.self_attn.q_proj",+ "model.language_model.layers.3.mlp.gate",+ "model.language_model.layers.3.mlp.shared_expert.gate_proj",+ ],+ "modules_to_convert": ["ple.ple_embedding.ngram_embedding"],+}+++def _hf_config(quantization_config=RADIXARK_NVFP4):return SimpleNamespace(model_type="qwen4_exp",architectures=["Qwen4ExpForConditionalGeneration"],image_token_id=248056,text_config=_text_config(),- quantization_config={- "quant_algo": "NVFP4",- "quant_method": "modelopt",- "ignore": [- "model.embed_tokens",- "mtp.*",- "model.mtp.*",- "*.self_attn.*",- "*.linear_attn.*",- "*.mlp.gate*",- "*.mlp.shared_expert.*",- "*.mlp.shared_expert_gate*",- "*hyper_connection*",- "*.ple.*",- "model.visual.*",- "model.language_model.embed_tokens",- "lm_head",- ],- },+ quantization_config=quantization_config,)@@ -115,22 +152,25 @@ def test_kv_specs_resolve_qsa():assert cfg.has_linear_attention-def test_moe_and_quant_flags():+def test_moe_dims():cfg = parse_config(_hf_config())assert cfg.num_experts == 512assert cfg.num_experts_per_tok == 10assert cfg.norm_topk_prob is Trueassert cfg.moe_enabled- assert cfg.expert_quant == "nvfp4"- assert cfg.dense_quant == "none"- assert cfg.attn_quant == "none"- assert cfg.lm_head_quant == "none"-def test_unquantized_config_parses():- hf = _hf_config()- hf.quantization_config = None- assert parse_config(hf).expert_quant == "none"+ "quantization_config, expert_quant",+ [+ pytest.param(RADIXARK_NVFP4, "nvfp4", id="RadixArk/Qwen3.8-Flash-Next-NVFP4"),+ pytest.param(NVIDIA_NVFP4, "nvfp4", id="nvidia/Qwen3.8-Flash-Next-NVFP4"),+ pytest.param(QWEN_FP8, "fp8_block", id="Qwen/Qwen3.8-Flash-Next-FP8"),+ pytest.param(None, "none", id="Qwen/Qwen3.8-Flash-Next"),+ ],+)+def test_released_checkpoints_expert_quant(quantization_config, expert_quant):+ assert parse_config(_hf_config(quantization_config)).expert_quant == expert_quantdef test_qwen4_args_payload():