FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(models): detect nvfp4 experts behind a mixed-precision compressed-tensors format (#343)
Xiaoze Fan6eca2d7d2b85
diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.pyindex 8ce69d65..1bce039c 100644--- a/python/freetoken/models/config.py+++ b/python/freetoken/models/config.py@@ -21,7 +21,8 @@ def vision_load_enabled() -> bool:def detect_expert_quant(hf_config: Any) -> str:"""Routed-expert quantization from a checkpoint's ``quantization_config``: ``"nvfp4"`` fora ModelOpt FP4 build (``quant_algo: NVFP4``) OR an llm-compressor NVFP4 export- (``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, e.g.+ (``quant_method: compressed-tensors`` + ``format: nvfp4-pack-quantized``, or+ ``format: mixed-precision`` with an nvfp4 config group, e.g.RedHatAI/GLM-5.3-Flash-NVFP4), else the lowercased algo string (``"none"`` whenunquantized). Models with mixed-precision configs (e.g. qwen3_5_moe) need theirown detector."""@@ -34,9 +35,19 @@ def detect_expert_quant(hf_config: Any) -> str:return "none"if "fp4" in str(algo).lower():return "nvfp4"+ fmt = str(get("format") or "").lower()# exact "nvfp4" (not the "fp4" substring) so MXFP4 exports don't misroute- if "nvfp4" in str(get("format") or "").lower():+ if "nvfp4" in fmt:return "nvfp4"+ # llm-compressor writes "mixed-precision" at the top when the groups differ (GLM-5.3-Flash: nvfp4 routed experts, fp8 MTP experts); the real format then sits in each group+ if fmt == "mixed-precision":+ groups = get("config_groups") or {}+ groups = [g or {} for g in (groups.values() if isinstance(groups, dict) else [])]+ # groups that target the experts decide; only a generic ["Linear"] group falls back to all of them+ expert_groups = [g for g in groups if any("experts" in str(t) for t in (g.get("targets") or []))]+ for g in expert_groups or groups:+ if "nvfp4" in str(g.get("format") or "").lower():+ return "nvfp4"return str(algo).lower()diff --git a/tests/models/test_glm5_next_config.py b/tests/models/test_glm5_next_config.pyindex ae4daf5b..8f0bc518 100644--- a/tests/models/test_glm5_next_config.py+++ b/tests/models/test_glm5_next_config.py@@ -121,6 +121,24 @@ _CT_NVFP4_QUANT = {},}+_CT_MIXED_QUANT = {+ # From RedHatAI/GLM-5.3-Flash-NVFP4 as published: nvfp4 routed experts plus fp8 experts on the MTP layer, so the top-level format is "mixed-precision".+ "quant_method": "compressed-tensors",+ "format": "mixed-precision",+ "config_groups": {+ "group_0": {+ "targets": ["re:.*\\.layers\\.(?:[3-9]|[1-3][0-9]|4[0-4])\\.mlp\\.experts\\..*(gate|up|down)_proj$"],+ "weights": {"num_bits": 4, "type": "float", "group_size": 16, "strategy": "tensor_group"},+ "format": "nvfp4-pack-quantized",+ },+ "group_1": {+ "targets": ["re:.*\\.layers\\.45\\.mlp\\.experts\\.\\d+\\.(gate_proj|up_proj|down_proj)$"],+ "weights": {"num_bits": 8, "type": "float", "strategy": "block"},+ "format": "float-quantized",+ },+ },+}+_NVFP4_QUANT = {# From LibertAIDAI/GLM-5.3-Flash-NVFP4 (ModelOpt weight-only NVFP4)."quant_algo": "NVFP4",@@ -246,6 +264,37 @@ def test_compressed_tensors_nvfp4_detected():assert cfg.expert_quant == "nvfp4"+def test_compressed_tensors_mixed_precision_detected():+ """The published RedHatAI export says ``format: mixed-precision`` at the top and+ ``nvfp4-pack-quantized`` only inside the routed-expert group; expert_quant must+ still resolve to nvfp4, not to the raw quant_method."""+ cfg = parse_config(_hf_config(_CT_MIXED_QUANT))+ assert cfg.expert_quant == "nvfp4"+++def test_compressed_tensors_mixed_precision_reads_the_expert_group():+ """A mixed export with nvfp4 dense layers but fp8 experts must not report nvfp4+ experts: the group that targets the experts decides."""+ quant = {+ "quant_method": "compressed-tensors",+ "format": "mixed-precision",+ "config_groups": {+ "group_0": {+ "targets": ["re:.*self_attn.*_proj$"],+ "weights": {"num_bits": 4, "type": "float", "group_size": 16, "strategy": "tensor_group"},+ "format": "nvfp4-pack-quantized",+ },+ "group_1": {+ "targets": ["re:.*mlp\\.experts\\..*(gate|up|down)_proj$"],+ "weights": {"num_bits": 8, "type": "float", "strategy": "block"},+ "format": "float-quantized",+ },+ },+ }+ cfg = parse_config(_hf_config(quant))+ assert cfg.expert_quant == "compressed-tensors"++def test_expert_source_spec_selection():"""quant_method picks the bank source spec: compressed-tensors mapsweight_packed/weight_global_scale onto the canonical kinds with a reciprocal