FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(qwen4_exp): load block-fp8 dense projections natively (#428)
Xiaoze Fanddd2e3abbece
diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.pyindex 2ce7462b..4a2a5b61 100644--- a/python/freetoken/models/qwen4_exp/weight.py+++ b/python/freetoken/models/qwen4_exp/weight.py@@ -2,7 +2,7 @@Three separate paths, because the checkpoint's three weight classes live in different places:-* :func:`iter_weights` -- every dense (non-expert) tensor, with the ``model.language_model.`` prefix stripped and fused where the model expects one buffer. See ``_FUSIONS``.+* :func:`iter_weights` -- every dense (non-expert) tensor, with the ``model.language_model.`` prefix stripped and fused where the model expects one buffer. See ``_DenseFuser``.* :func:`load_ple_table` -- the 47.7 GiB FP8 n-gram table, 128 checkpoint shards concatenated into one pinned :class:`HostBank`.* :func:`nvfp4_expert_spec` -- how the routed NVFP4 experts are named, for the offload cache's expert reader.@@ -25,8 +25,10 @@ from freetoken.models.loader import drop_page_cache, iter_weight_filesfrom freetoken.models.nvfp4_banks import (Nvfp4ExpertSourceSpec,)+from freetoken.layers.quantization import get_quant_config+from freetoken.models.register import get_model_specfrom freetoken.moe.host_banks import HostBank, read_range_into-from freetoken.utils import download_hf_weight+from freetoken.utils import cached_load_hf_config, download_hf_weightfrom freetoken.utils.progress import byte_barfrom tqdm import tqdm@@ -70,32 +72,13 @@ _ZERO_CENTERED_NORM_SUFFIXES = (".self_attn.indexer.k_layernorm.weight",)-# Fused projections: concat the checkpoint parts along dim 0 in this exact order. A nonzero pad-# rounds the merged row count up; the model splits the result back with the same sizes.-_FUSIONS: dict[str, tuple[tuple[str, ...], int]] = {- # q carries the output gate, so its half is twice the attention width: [2*qo | kv | kv].- ".self_attn.qkv_proj.weight": ((- ".self_attn.q_proj.weight", ".self_attn.k_proj.weight", ".self_attn.v_proj.weight",- ), 0),- ".linear_attn.in_proj.weight": ((- ".linear_attn.in_proj_qkv.weight", ".linear_attn.in_proj_z.weight",- ".linear_attn.in_proj_b.weight", ".linear_attn.in_proj_a.weight",- ), 0),- ".mlp.shared_expert.gate_up_proj.weight": ((- ".mlp.shared_expert.gate_proj.weight", ".mlp.shared_expert.up_proj.weight",- ), 0),- # HC mix reads the low-rank down projection and the injection logits from one GEMM; vLLM- # pads the merged output to a multiple of 16 rows for cuBLAS (hyperconnection.py pad_size).- # The top-level hyper_connection_mixer has no injection and so never fuses.- ".attn_hyper_connection.input_mix_weight_down_block_inject.weight": ((- ".attn_hyper_connection.input_mix_weight_down.weight",- ".attn_hyper_connection.block_inject_weight.weight",- ), 16),- ".mlp_hyper_connection.input_mix_weight_down_block_inject.weight": ((- ".mlp_hyper_connection.input_mix_weight_down.weight",- ".mlp_hyper_connection.block_inject_weight.weight",- ), 16),-}+# The per-layer HC mix reads the low-rank down projection and the injection logits from one GEMM; vLLM pads the merged rows to a multiple of 16 for cuBLAS (hyperconnection.py pad_size).+# The top-level hyper_connection_mixer has no injection and never fuses.+_PAD_TO = {"input_mix_weight_down_block_inject": 16}+_HC_WITH_INJECT = (".attn_hyper_connection", ".mlp_hyper_connection")+_KIND_SUFFIXES = (".weight_scale_inv", ".weight")+_FP8_DTYPES = (torch.float8_e4m3fn, torch.float8_e5m2)+_ELEM_DTYPES = {"e4m3": torch.float8_e4m3fn}def _rename(raw_name: str) -> str | None:@@ -115,26 +98,95 @@ def _rename(raw_name: str) -> str | None:return raw_name-def _try_fuse(- name: str, tensor: torch.Tensor, buf: dict[str, dict[int, torch.Tensor]]-) -> tuple[str, torch.Tensor] | tuple[()] | None:- """Buffer a fusion part; return the merged ``(name, tensor)`` once all parts arrive, ``()`` while incomplete, ``None`` if ``name`` is not a fusion part."""- for fused_suffix, (parts, pad_to) in _FUSIONS.items():- for idx, part in enumerate(parts):- if not name.endswith(part):- continue- key = name[: -len(part)] + fused_suffix- slots = buf.setdefault(key, {})- slots[idx] = tensor- if len(slots) < len(parts):- return ()- del buf[key]- rows = [slots[i] for i in range(len(parts))]- pad = (-sum(t.shape[0] for t in rows)) % pad_to if pad_to else 0- if pad:- rows.append(torch.zeros(pad, *rows[0].shape[1:], dtype=rows[0].dtype, device=rows[0].device))- return key, torch.cat(rows, dim=0)- return None+def _split_kind(name: str) -> tuple[str, str]:+ """``name`` -> ``(module, kind)``; kind is "" for tensors that are neither a weight nor a block scale."""+ for suffix in _KIND_SUFFIXES:+ if name.endswith(suffix):+ return name[: -len(suffix)], suffix+ return name, ""+++class _DenseFuser:+ """Concatenates checkpoint projection parts into the model's merged buffers, per kind (weight / block scale).++ The part table is the family's packed_modules_mapping. The QuantConfig picks the GDN in_proj layout and validates each part against the scheme the model built its buffer from.+ """++ def __init__(self, quant, packed: tuple[tuple[str, tuple[str, ...]], ...]) -> None:+ self.quant = quant+ self.groups = {fused: parts for fused, parts in packed if fused != "experts"} # experts: bank reader+ self.by_part: dict[str, list[tuple[str, int]]] = {}+ for fused, parts in self.groups.items():+ for idx, part in enumerate(parts):+ self.by_part.setdefault(part, []).append((fused, idx))+ self.buf: dict[tuple[str, str], dict[int, torch.Tensor]] = {}++ def scheme(self, module: str):+ return None if self.quant is None else self.quant.scheme_for(module)++ def _target(self, parent: str, leaf: str) -> tuple[str, int] | None:+ candidates = self.by_part.get(leaf)+ if not candidates:+ return None+ if len(candidates) > 1:+ # GDN: quantized checkpoints split qkv|z from the bf16 b|a; same test as gdn.py+ split = self.scheme(f"{parent}.in_proj_qkvz") is not None+ keep = {"in_proj_qkvz", "in_proj_ba"} if split else {"in_proj"}+ candidates = [c for c in candidates if c[0] in keep]+ if not candidates:+ raise ValueError(f"{parent}.{leaf}: no merged projection for the {'split' if split else 'fused'} GDN layout")+ fused, idx = candidates[0]+ if fused in _PAD_TO and not parent.endswith(_HC_WITH_INJECT):+ return None+ return f"{parent}.{fused}", idx++ def check(self, module: str, name: str, tensor: torch.Tensor) -> None:+ """``tensor`` (checkpoint key ``name``) must match the scheme the model built ``module`` from."""+ scheme = self.scheme(module)+ if name.endswith(".weight_scale_inv"):+ if scheme is None or not scheme.has("weight_scale_inv"):+ raise ValueError(f"{name}: {module} has no block scale in the checkpoint's quant config ({scheme})")+ return+ is_fp8 = tensor.dtype in _FP8_DTYPES+ if scheme is None:+ if is_fp8:+ raise ValueError(f"{name} is {tensor.dtype} but the checkpoint's quant config declares {module} unquantized")+ return+ expected = _ELEM_DTYPES.get(scheme.weight.elem)+ if expected is not None and tensor.dtype is not expected:+ raise ValueError(f"{name} is {tensor.dtype} but the checkpoint's quant config declares {module} {scheme}")+ rows, cols = (scheme.weight.group or (1, 1))+ if rows > 1 and tensor.shape[0] % rows or cols > 1 and tensor.shape[1] % cols:+ raise ValueError(f"{name}: {tuple(tensor.shape)} is not a multiple of the {rows}x{cols} scale block of {module}")++ def check_unfused(self, name: str, tensor: torch.Tensor) -> None:+ module, kind = _split_kind(name)+ if kind == ".weight_scale_inv" or (kind == ".weight" and tensor.dtype in _FP8_DTYPES):+ self.check(module, name, tensor)++ def fuse(self, name: str, tensor: torch.Tensor) -> list[tuple[str, torch.Tensor]] | None:+ """Buffer a part; return the merged ``[(name, tensor)]`` once its kind is complete, ``[]`` while incomplete, ``None`` if ``name`` is not a part."""+ module, kind = _split_kind(name)+ if not kind:+ return None+ parent, _, leaf = module.rpartition(".")+ hit = self._target(parent, leaf)+ if hit is None:+ return None+ fused, idx = hit+ self.check(fused, name, tensor)+ slots = self.buf.setdefault((fused, kind), {})+ slots[idx] = tensor+ parts = self.groups[fused.rpartition(".")[2]]+ if len(slots) < len(parts):+ return []+ del self.buf[(fused, kind)]+ rows = [slots[i] for i in range(len(parts))]+ pad_to = _PAD_TO.get(fused.rpartition(".")[2], 0) if kind == ".weight" else 0+ pad = (-sum(t.shape[0] for t in rows)) % pad_to if pad_to else 0+ if pad:+ rows.append(torch.zeros(pad, *rows[0].shape[1:], dtype=rows[0].dtype, device=rows[0].device))+ return [(fused + kind, torch.cat(rows, dim=0))]def iter_weights(@@ -146,24 +198,19 @@ def iter_weights() -> Iterator[tuple[str, torch.Tensor]]:"""Yield the dense (non-expert) weights, prefix-stripped and fused to the model's buffers.- Keys keep the checkpoint's module names below the stripped prefix, so the emitted set is the- model's state dict minus the routed experts. Nothing here is quantized: every release's skip- list (modelopt ``ignore``, fp8 ``modules_to_not_convert``) covers everything except those experts,- so attention, GDN, HC, PLE, the shared expert and lm_head are all plain bf16 (the n-gram hash- constants stay int64). Fusions:- attention q|k|v -> ``qkv_proj``, GDN ``in_proj_{qkv,z,b,a}`` -> ``in_proj``, shared-expert- gate|up -> ``gate_up_proj``, and each per-layer HC's ``input_mix_weight_down`` |- ``block_inject_weight`` -> a zero-padded ``input_mix_weight_down_block_inject``.-- ``include_moe_experts`` is accepted for the loader contract but never yields anything: the- routed experts are NVFP4 and always come from the offload cache's expert reader.+ Keys keep the checkpoint's module names below the stripped prefix, so the emitted set is the model's state dict minus the routed experts.+ A dense projection is bf16 or 128x128 block-fp8 (``.weight`` e4m3 + ``.weight_scale_inv``) as the checkpoint's QuantConfig says: the official releases skip everything but the routed experts, the community NVFP4-FP8 requants quantize the attention / GDN projections.+ Fusions, per kind: attention q|k|v -> ``qkv_proj``; GDN ``in_proj_{qkv,z,b,a}`` -> ``in_proj``, or ``in_proj_qkvz`` + bf16 ``in_proj_ba`` when qkv|z is quantized; shared-expert gate|up -> ``gate_up_proj``; each per-layer HC's ``input_mix_weight_down`` | ``block_inject_weight`` -> a zero-padded ``input_mix_weight_down_block_inject``.+ ``include_moe_experts`` is accepted for the loader contract but never yields anything: the routed experts are NVFP4 and always come from the offload cache's expert reader."""if get_tp_info().size > 1:raise NotImplementedError("qwen4_exp weight loading supports TP=1 only")if not include_non_moe:return- fuse_buf: dict[str, dict[int, torch.Tensor]] = {}+ hf_config = cached_load_hf_config(model_path)+ spec = get_model_spec(hf_config.architectures[0])+ fuser = _DenseFuser(get_quant_config(), spec.packed_modules_mapping)for file in tqdm(iter_weight_files(model_path),desc="Loading weights",@@ -175,14 +222,14 @@ def iter_weights(if name is None:continuetensor = f.get_tensor(raw_name)- fused = _try_fuse(name, tensor, fuse_buf)- if fused is not None:- if fused != (): # () means buffered, not yet complete- yield fused- continue- yield name, tensor-- assert not fuse_buf, f"Incomplete projection fusions: {sorted(fuse_buf)}"+ fused = fuser.fuse(name, tensor)+ if fused is None:+ fuser.check_unfused(name, tensor)+ yield name, tensor+ else:+ yield from fused++ assert not fuser.buf, f"Incomplete projection fusions: {sorted(k[0] + k[1] for k in fuser.buf)}"# ======================================================================================diff --git a/tests/models/qwen4_exp/common.py b/tests/models/qwen4_exp/common.pyindex 1f9c117b..aa6f2399 100644--- a/tests/models/qwen4_exp/common.py+++ b/tests/models/qwen4_exp/common.py@@ -255,3 +255,118 @@ def selection_spy(monkeypatch, backend) -> dict:monkeypatch.setattr(QSASparseAttnBackend, "_select", spy)return seen+++LM = "model.language_model"++# quantization_config of each released checkpoint, trimmed to the entries parse_config and the reader look at++# RadixArk/Qwen3.8-Flash-Next-NVFP4: modelopt NVFP4 everywhere except the ignore list+RADIXARK_NVFP4 = {+ "quant_algo": "NVFP4",+ "quant_method": "modelopt",+ "ignore": [+ "model.embed_tokens",+ "mtp.*",+ "model.mtp.*",+ "*.self_attn.*",+ "*.linear_attn.*",+ "*.mlp.gate*",+ "*.mlp.shared_expert.*",+ "*.mlp.shared_expert_gate*",+ "*hyper_connection*",+ "*.ple.*",+ "model.visual.*",+ "model.language_model.embed_tokens",+ "lm_head",+ ],+}++# nvidia/Qwen3.8-Flash-Next-NVFP4: modelopt MIXED_PRECISION, the per-module algo sits in quantized_layers+NVIDIA_NVFP4 = {+ "quant_algo": "MIXED_PRECISION",+ "quant_method": "modelopt",+ "quantized_layers": {+ **{f"model.language_model.layers.{i}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for i in range(48)},+ "model.language_model.layers.1.ple.ple_embedding.ngram_embedding": {"quant_algo": "FP8"},+ "mtp.layers.0.mlp.experts": {"quant_algo": "FP8_PB_WO", "group_size": 128},+ },+ "ignore": ["lm_head", "model.language_model.embed_tokens", "model.language_model.layers.0.mlp.shared_expert*", "model.visual*"],+}++# Qwen/Qwen3.8-Flash-Next-FP8: 128x128 block-fp8 experts, everything else listed in modules_to_not_convert+QWEN_FP8 = {+ "quant_method": "fp8",+ "activation_scheme": "dynamic",+ "weight_per_tensor": False,+ "act_per_tensor": False,+ "weight_block_size": [128, 128],+ "modules_to_not_convert": [+ "lm_head",+ "model.language_model.embed_tokens",+ "model.language_model.hyper_connection_mixer.input_mix_weight_up",+ "model.language_model.layers.0.linear_attn.in_proj_qkv",+ "model.language_model.layers.3.self_attn.q_proj",+ "model.language_model.layers.3.mlp.gate",+ "model.language_model.layers.3.mlp.shared_expert.gate_proj",+ ],+ "modules_to_convert": ["ple.ple_embedding.ngram_embedding"],+}+++def mixed_precision_quant(gdn_layers, attn_layers, moe_layers) -> dict:+ """modelopt MIXED_PRECISION with NVFP4 routed experts and FP8_PB_WO attention / GDN projections, ignore list as in lovedheart/Qwen3.8-Flash-Next-NVFP4-FP8."""+ return {+ "quant_method": "modelopt",+ "quant_algo": "MIXED_PRECISION",+ "quantized_layers": {+ **{f"{LM}.layers.{i}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for i in moe_layers},+ **{f"{LM}.layers.{i}.linear_attn.{p}": {"quant_algo": "FP8_PB_WO", "group_size": 128}+ for i in gdn_layers for p in ("in_proj_qkv", "in_proj_z", "out_proj")},+ **{f"{LM}.layers.{i}.self_attn.{p}_proj": {"quant_algo": "FP8_PB_WO", "group_size": 128}+ for i in attn_layers for p in "qkvo"},+ },+ "ignore": [+ "model.embed_tokens", "mtp.*", "model.mtp.*", "*.mlp.gate*", "*.mlp.shared_expert.*",+ "*.mlp.shared_expert_gate*", "*hyper_connection*", "*.ple.*", "model.visual.*",+ "model.language_model.embed_tokens", "lm_head", "*.self_attn.indexer*",+ ],+ }+++# lovedheart/Qwen3.8-Flash-Next-NVFP4-FP8, trimmed to layers 0 (GDN) and 3 (attention)+LOVEDHEART_NVFP4_FP8 = mixed_precision_quant(gdn_layers=(0,), attn_layers=(3,), moe_layers=(0, 3))+++def install_quant_config(model_path: str) -> None:+ """Install ``model_path``'s QuantConfig process-wide, as EngineConfig does before the reader runs."""+ from freetoken.layers.quantization import set_quant_config+ from freetoken.models.register import checkpoint_quant_config, get_model_spec+ from freetoken.utils import cached_load_hf_config++ hf = cached_load_hf_config(model_path)+ set_quant_config(checkpoint_quant_config(model_path, hf, get_model_spec(hf.architectures[0])))+++def meta_state_dict(model_path: str) -> dict[str, torch.Tensor]:+ """State dict of the model the engine builds for ``model_path`` (experts offloaded), on the meta device."""+ from freetoken.engine.config import EngineConfig+ from freetoken.engine.engine import _decode_target+ from freetoken.layers import rotary+ from freetoken.models import create_model+ from freetoken.utils.torch_utils import torch_dtype++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+ config = EngineConfig(model_path=model_path, tp_info=try_get_tp_info(), dtype=torch.bfloat16, moe_strategy="offload")+ object.__setattr__(config.model_config, "moe_strategy", "offload")+ object.__setattr__(config.model_config, "decode_target", _decode_target(config))+ saved = rotary._ROPE_DEVICE+ rotary.set_rope_device(torch.device("cpu")) # get_rope refuses to build on meta+ rotary.get_rope.cache_clear()+ try:+ with torch.device("meta"), torch_dtype(torch.bfloat16):+ return create_model(config.model_config).state_dict()+ finally:+ rotary.set_rope_device(saved)+ rotary.get_rope.cache_clear() # the cpu rope must not leak into the GPU tests' cachediff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.pyindex 77135630..b81e02f4 100644--- a/tests/models/qwen4_exp/test_config.py+++ b/tests/models/qwen4_exp/test_config.py@@ -8,6 +8,8 @@ from freetoken.attention import AttnTypefrom freetoken.models.config import FullAttentionGroupConfig, LinearGatedDeltaGroupConfigfrom freetoken.models.qwen4_exp.config import parse_config+from .common import LOVEDHEART_NVFP4_FP8, NVIDIA_NVFP4, QWEN_FP8, RADIXARK_NVFP4+def _text_config():return SimpleNamespace(@@ -61,61 +63,6 @@ def _text_config():)-# quantization_config of each released checkpoint, trimmed to the entries parse_config looks at--# RadixArk/Qwen3.8-Flash-Next-NVFP4: modelopt NVFP4 everywhere except the ignore list-RADIXARK_NVFP4 = {- "quant_algo": "NVFP4",- "quant_method": "modelopt",- "ignore": [- "model.embed_tokens",- "mtp.*",- "model.mtp.*",- "*.self_attn.*",- "*.linear_attn.*",- "*.mlp.gate*",- "*.mlp.shared_expert.*",- "*.mlp.shared_expert_gate*",- "*hyper_connection*",- "*.ple.*",- "model.visual.*",- "model.language_model.embed_tokens",- "lm_head",- ],-}--# nvidia/Qwen3.8-Flash-Next-NVFP4: modelopt MIXED_PRECISION, the per-module algo sits in quantized_layers-NVIDIA_NVFP4 = {- "quant_algo": "MIXED_PRECISION",- "quant_method": "modelopt",- "quantized_layers": {- **{f"model.language_model.layers.{i}.mlp.experts": {"quant_algo": "NVFP4", "group_size": 16} for i in range(48)},- "model.language_model.layers.1.ple.ple_embedding.ngram_embedding": {"quant_algo": "FP8"},- "mtp.layers.0.mlp.experts": {"quant_algo": "FP8_PB_WO", "group_size": 128},- },- "ignore": ["lm_head", "model.language_model.embed_tokens", "model.language_model.layers.0.mlp.shared_expert*", "model.visual*"],-}--# Qwen/Qwen3.8-Flash-Next-FP8: 128x128 block-fp8 experts, everything else listed in modules_to_not_convert-QWEN_FP8 = {- "quant_method": "fp8",- "activation_scheme": "dynamic",- "weight_per_tensor": False,- "act_per_tensor": False,- "weight_block_size": [128, 128],- "modules_to_not_convert": [- "lm_head",- "model.language_model.embed_tokens",- "model.language_model.hyper_connection_mixer.input_mix_weight_up",- "model.language_model.layers.0.linear_attn.in_proj_qkv",- "model.language_model.layers.3.self_attn.q_proj",- "model.language_model.layers.3.mlp.gate",- "model.language_model.layers.3.mlp.shared_expert.gate_proj",- ],- "modules_to_convert": ["ple.ple_embedding.ngram_embedding"],-}--def _hf_config(quantization_config=RADIXARK_NVFP4):return SimpleNamespace(model_type="qwen4_exp",@@ -208,3 +155,38 @@ def test_eos_token_id_list_uses_the_first_entry():hf = _hf_config()hf.text_config.eos_token_id = [base, base + 1]assert parse_config(hf).qwen4_args.ngram_boundary_token_id == base+++# the merged-projection prefixes the model asks the QuantConfig about (attention.py / gdn.py)+DENSE_PREFIXES = (+ "model.layers.3.self_attn.qkv_proj", "model.layers.3.self_attn.o_proj",+ "model.layers.0.linear_attn.in_proj_qkvz", "model.layers.0.linear_attn.out_proj",+)+BF16_PREFIXES = (+ "model.layers.0.linear_attn.in_proj_ba", "model.layers.0.mlp.shared_expert.gate_up_proj",+ "model.layers.3.self_attn.indexer.index_qk_proj", "lm_head",+)+++def _quant(hf, tmp_path):+ from freetoken.models.register import checkpoint_quant_config, get_model_spec++ return checkpoint_quant_config(str(tmp_path), hf, get_model_spec(hf.architectures[0]))+++def test_block_fp8_dense_schemes(tmp_path):+ quant = _quant(_hf_config(LOVEDHEART_NVFP4_FP8), tmp_path)+ for prefix in DENSE_PREFIXES:+ scheme = quant.scheme_for(prefix)+ assert str(scheme.kind) == "fp8_block" and scheme.has("weight_scale_inv"), prefix+ for prefix in BF16_PREFIXES:+ assert quant.scheme_for(prefix) is None, prefix+ assert str(quant.scheme_for("model.layers.0.mlp.experts").kind) == "nvfp4"+ assert parse_config(_hf_config(LOVEDHEART_NVFP4_FP8)).expert_quant == "nvfp4"++[email protected]("quantization_config", [RADIXARK_NVFP4, NVIDIA_NVFP4, None], ids=["RadixArk", "nvidia", "bf16"])+def test_released_checkpoints_keep_the_dense_projections_bf16(quantization_config, tmp_path):+ quant = _quant(_hf_config(quantization_config), tmp_path)+ for prefix in DENSE_PREFIXES + BF16_PREFIXES:+ assert quant.scheme_for(prefix) is None, prefixdiff --git a/tests/models/qwen4_exp/test_weight.py b/tests/models/qwen4_exp/test_weight.pyindex b3f1f851..1ddbf425 100644--- a/tests/models/qwen4_exp/test_weight.py+++ b/tests/models/qwen4_exp/test_weight.py@@ -1,11 +1,12 @@-"""qwen4_exp weight loading against a synthetic checkpoint shaped like the RadixArk NVFP4 one.+"""qwen4_exp weight loading against synthetic checkpoints shaped like the released ones.The tensors are tiny but the key names, dtypes and the fusion geometry that matters-(hc_lowrank=320 + hc_count=4 -> a 12-row zero pad) are the real ones.+(hc_lowrank=320 + hc_count=4 -> a 12-row zero pad; 128-row block scales) are the real ones."""from __future__ import annotations+import jsonimport randomfrom types import SimpleNamespace@@ -17,18 +18,23 @@ from freetoken.distributed import set_tp_info, try_get_tp_infofrom freetoken.kernel.aot_models import SUPPORTED_MODELS, expert_bank_row_bytesfrom freetoken.models.qwen4_exp.weight import (_ZERO_CENTERED_NORM_SUFFIXES,+ _DenseFuser,iter_weights,load_ple_table,)+from freetoken.models.register import get_model_specfrom freetoken.moe.host_banks import HostBank, read_range_into-H = 32 # hidden_size+from .common import LM, RADIXARK_NVFP4, hf_config, install_quant_config, meta_state_dict, mixed_precision_quant++H = 128 # hidden_size; every block-fp8 projection needs in/out multiples of 128HC = 4 # hc_countLR = 320 # hc_lowrank; kept real so the merged HC pad is the real (-(320+4)) % 16 = 12HCH = HC * H # hyper-connection stream width-KH, VH, HD = 2, 6, 8 # GDN key / value heads, head dim-QH, KVH, AHD = 4, 2, 16 # QSA q / kv heads, head dim-IHD = 8 # indexer head dim+KH, VH, HD = 2, 4, 32 # GDN key / value heads, head dim: qkv rows 256, z rows 128+QH, KVH, AHD = 4, 2, 64 # QSA q / kv heads, head dim: q rows 512, k / v rows 128+IHD = 64 # indexer head dim+BLOCK = 128E, I = 3, 6 # routed experts, moe_intermediate_sizeNGRAM_DIM, NGRAM_ROWS, NGRAM_SHARDS = 4, 7, 4@@ -54,8 +60,15 @@ def _hc_weights(prefix: str, inject: bool) -> dict[str, torch.Tensor]:return w-def _raw_checkpoint() -> dict[str, torch.Tensor]:- """Layer 0 = GDN + PLE, layer 1 = QSA; plus the mtp / visual / routed-expert noise."""+def _fp8_scale(weight: torch.Tensor) -> torch.Tensor:+ return torch.rand(weight.shape[0] // BLOCK, weight.shape[1] // BLOCK) + 0.5+++def _raw_checkpoint(dense_fp8: bool = False) -> dict[str, torch.Tensor]:+ """Layer 0 = GDN + PLE, layer 1 = QSA; plus the mtp / visual / routed-expert noise.++ ``dense_fp8`` stores the attention and GDN qkv|z / out projections as 128x128 block-fp8 (e4m3 ``.weight`` + fp32 ``.weight_scale_inv``) like the community NVFP4-FP8 requants.+ """lm = "model.language_model"raw: dict[str, torch.Tensor] = {f"{lm}.embed_tokens.weight": _bf16(11, H),@@ -127,9 +140,31 @@ def _raw_checkpoint() -> dict[str, torch.Tensor]:"model.visual.blocks.0.attn.qkv.weight": _bf16(3 * H, H),"model.visual.merger.norm.weight": _bf16(H),})+ if dense_fp8:+ for module in (f"{gdn}.in_proj_qkv", f"{gdn}.in_proj_z", f"{gdn}.out_proj",+ *(f"{attn}.{p}_proj" for p in "qkvo")):+ weight = raw[f"{module}.weight"]+ raw[f"{module}.weight"] = weight.to(torch.float8_e4m3fn)+ raw[f"{module}.weight_scale_inv"] = _fp8_scale(weight)return raw+FP8_DENSE_QUANT = mixed_precision_quant(gdn_layers=(0,), attn_layers=(1,), moe_layers=(0, 1))+++def _config_json(quantization_config) -> dict:+ cfg = hf_config(+ num_layers=2, head_dim=AHD, num_q=QH, num_kv=KVH, index_head_dim=IHD, index_heads=2,+ budget=16, hidden=H, max_position=4096, rope_theta=10000.0,+ layer_types=["linear_attention", "full_attention"],+ linear_num_key_heads=KH, linear_num_value_heads=VH,+ linear_key_head_dim=HD, linear_value_head_dim=HD,+ hc_lowrank=LR, ple_layer_ids=[1],+ num_experts=E, moe_intermediate_size=I, shared_expert_intermediate_size=I,+ )+ return {**vars(cfg), "text_config": vars(cfg.text_config), "quantization_config": quantization_config}++def _ngram_table() -> tuple[dict[str, torch.Tensor], torch.Tensor]:prefix = "model.language_model.layers.0.ple.ple_embedding.ngram_embedding"shards = {@@ -144,11 +179,7 @@ def _ngram_table() -> tuple[dict[str, torch.Tensor], torch.Tensor]:return shards, scale[email protected](scope="module")-def checkpoint(tmp_path_factory) -> tuple[str, dict[str, torch.Tensor]]:- torch.manual_seed(0)- folder = tmp_path_factory.mktemp("qwen4_exp_ckpt")- raw = _raw_checkpoint()+def _write_checkpoint(folder, raw: dict[str, torch.Tensor], quantization_config) -> tuple[str, dict[str, torch.Tensor]]:table, _scale = _ngram_table()# Spread the dense tensors over two shards so the fusion buffer has to survive a file# boundary, and put the n-gram table in its own shards like the real checkpoint does.@@ -158,12 +189,12 @@ def checkpoint(tmp_path_factory) -> tuple[str, dict[str, torch.Tensor]]:shard_names = sorted(table)save_file({n: table[n] for n in shard_names[:2]}, str(folder / "model-plefp8-00000.safetensors"))save_file({n: table[n] for n in shard_names[2:]}, str(folder / "model-plefp8-00001.safetensors"))+ (folder / "config.json").write_text(json.dumps(_config_json(quantization_config)))return str(folder), {**raw, **table}[email protected](scope="module")-def loaded(checkpoint) -> dict[str, torch.Tensor]:- folder, _raw = checkpoint+def _load(folder: str) -> dict[str, torch.Tensor]:+ install_quant_config(folder)return {name: tensor.clone()for name, tensor in iter_weights(@@ -172,6 +203,30 @@ def loaded(checkpoint) -> dict[str, torch.Tensor]:}[email protected](scope="module")+def checkpoint(tmp_path_factory) -> tuple[str, dict[str, torch.Tensor]]:+ torch.manual_seed(0)+ return _write_checkpoint(tmp_path_factory.mktemp("qwen4_exp_ckpt"), _raw_checkpoint(), None)++[email protected](scope="module")+def loaded(checkpoint) -> dict[str, torch.Tensor]:+ return _load(checkpoint[0])++[email protected](scope="module")+def checkpoint_fp8(tmp_path_factory) -> tuple[str, dict[str, torch.Tensor]]:+ torch.manual_seed(1)+ return _write_checkpoint(+ tmp_path_factory.mktemp("qwen4_exp_fp8_ckpt"), _raw_checkpoint(dense_fp8=True), FP8_DENSE_QUANT+ )++[email protected](scope="module")+def loaded_fp8(checkpoint_fp8) -> dict[str, torch.Tensor]:+ return _load(checkpoint_fp8[0])++def _expected_names() -> set[str]:names = {"model.embed_tokens.weight", "lm_head.weight"}names |= {f"model.hyper_connection_mixer.{leaf}" for leaf in@@ -206,7 +261,7 @@ def test_mtp_visual_experts_and_table_never_loaded(loaded):assert not name.startswith(("mtp.", "model.visual."))assert ".mlp.experts." not in nameassert "ngram_embedding" not in name- assert not name.endswith((".weight_scale", ".weight_scale_2", ".input_scale"))+ assert not name.endswith((".weight_scale", ".weight_scale_2", ".input_scale", ".weight_scale_inv"))def test_hc_merge_is_down_then_inject_then_zero_pad(loaded, checkpoint):@@ -399,12 +454,92 @@ def test_every_registry_architecture_is_claimed_by_an_aot_entry():@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs cuda")def test_fusion_pad_rides_the_tensor_device():"""safetensors loads straight to cuda; a cpu-allocated pad row would break torch.cat."""- from freetoken.models.qwen4_exp.weight import _try_fuse-- buf = {}+ fuser = _DenseFuser(None, get_model_spec("Qwen4ExpForConditionalGeneration").packed_modules_mapping)down = torch.randn(320, 64, device="cuda", dtype=torch.bfloat16)inject = torch.randn(4, 64, device="cuda", dtype=torch.bfloat16)- assert _try_fuse("model.layers.0.attn_hyper_connection.input_mix_weight_down.weight", down, buf) == ()- key, fused = _try_fuse("model.layers.0.attn_hyper_connection.block_inject_weight.weight", inject, buf)+ assert fuser.fuse("model.layers.0.attn_hyper_connection.input_mix_weight_down.weight", down) == []+ [(key, fused)] = fuser.fuse("model.layers.0.attn_hyper_connection.block_inject_weight.weight", inject)+ assert key == "model.layers.0.attn_hyper_connection.input_mix_weight_down_block_inject.weight"assert fused.device.type == "cuda" and fused.shape[0] == 336assert torch.equal(fused[324:], torch.zeros(12, 64, device="cuda", dtype=torch.bfloat16))+++# ======================================================================================+# the reader against the model the engine builds, for each released quant layout+# ======================================================================================++[email protected](scope="module")+def checkpoint_nvfp4(tmp_path_factory) -> tuple[str, dict[str, torch.Tensor]]:+ """bf16 dense tensors under a real ModelOptConfig whose ignore list covers them (RadixArk)."""+ torch.manual_seed(2)+ return _write_checkpoint(tmp_path_factory.mktemp("qwen4_exp_nvfp4_ckpt"), _raw_checkpoint(), RADIXARK_NVFP4)+++FP8_MODULES = (+ "model.layers.1.self_attn.qkv_proj", "model.layers.1.self_attn.o_proj",+ "model.layers.0.linear_attn.in_proj_qkvz", "model.layers.0.linear_attn.out_proj",+)++[email protected]("fixture", ["checkpoint", "checkpoint_nvfp4", "checkpoint_fp8"])+def test_emitted_keys_are_the_model_state_dict(fixture, request):+ """The reader fills exactly the buffers the engine builds from the same config, block-fp8 ones with the stored dtypes."""+ folder, _raw = request.getfixturevalue(fixture)+ loaded, state = _load(folder), meta_state_dict(folder)+ assert set(loaded) == set(state)+ if fixture != "checkpoint_fp8":+ assert loaded["model.layers.0.linear_attn.in_proj.weight"].dtype is torch.bfloat16+ return+ for module in FP8_MODULES:+ for kind in (".weight", ".weight_scale_inv"):+ assert loaded[module + kind].shape == state[module + kind].shape, module + kind+ assert loaded[module + ".weight"].dtype is state[module + ".weight"].dtype is torch.float8_e4m3fn+ assert loaded[module + ".weight_scale_inv"].dtype is torch.float32 # the engine casts it to the bf16 buffer at load+++def _assert_fused_per_kind(loaded, raw, fused: str, parts: list[str]) -> None:+ for kind in (".weight", ".weight_scale_inv"):+ sources = [raw[f"{p}{kind}"].view(torch.uint8) for p in parts]+ merged = loaded[fused + kind]+ assert merged.dtype is raw[f"{parts[0]}{kind}"].dtype+ for source, back in zip(sources, torch.split(merged.view(torch.uint8), [s.shape[0] for s in sources], dim=0)):+ assert torch.equal(source, back)+++def test_fp8_projections_fuse_per_kind(loaded_fp8, checkpoint_fp8):+ _folder, raw = checkpoint_fp8+ attn, gdn = f"{LM}.layers.1.self_attn", f"{LM}.layers.0.linear_attn"+ _assert_fused_per_kind(loaded_fp8, raw, "model.layers.1.self_attn.qkv_proj", [f"{attn}.{p}_proj" for p in "qkv"])+ _assert_fused_per_kind(loaded_fp8, raw, "model.layers.0.linear_attn.in_proj_qkvz", [f"{gdn}.in_proj_qkv", f"{gdn}.in_proj_z"])+ assert torch.equal(loaded_fp8["model.layers.0.linear_attn.out_proj.weight_scale_inv"], raw[f"{gdn}.out_proj.weight_scale_inv"])+ assert torch.equal(+ loaded_fp8["model.layers.0.linear_attn.in_proj_ba.weight"],+ torch.cat([raw[f"{gdn}.in_proj_b.weight"], raw[f"{gdn}.in_proj_a.weight"]], dim=0),+ )+ for name in ("model.layers.0.linear_attn.in_proj_ba.weight", "model.layers.1.mlp.shared_expert.gate_up_proj.weight",+ "model.layers.1.self_attn.indexer.index_qk_proj.weight", "lm_head.weight",+ "model.hyper_connection_mixer.input_mix_weight_down.weight",+ "model.layers.0.attn_hyper_connection.input_mix_weight_down_block_inject.weight"):+ assert loaded_fp8[name].dtype is torch.bfloat16+++ATTN = f"{LM}.layers.1.self_attn"+REJECTED = [+ pytest.param(None, lambda w: {f"{ATTN}.q_proj.weight": w, f"{ATTN}.q_proj.weight_scale_inv": _fp8_scale(w)},+ r"q_proj\.weight_scale_inv", id="scale the config does not declare"),+ pytest.param(None, lambda w: {f"{ATTN}.o_proj.weight": w.to(torch.float8_e4m3fn)},+ r"o_proj\.weight is torch\.float8", id="fp8 weight the config declares bf16"),+ pytest.param(FP8_DENSE_QUANT, lambda w: {f"{ATTN}.{p}_proj.weight": w.clone() for p in "qkv"},+ r"[qkv]_proj\.weight is torch\.bfloat16", id="bf16 weight the config declares fp8"),+ pytest.param(FP8_DENSE_QUANT, lambda w: {f"{ATTN}.q_proj.weight": w[:-64].to(torch.float8_e4m3fn), f"{ATTN}.q_proj.weight_scale_inv": _fp8_scale(w)},+ "128x128", id="part that is not a 128-row multiple"),+]++[email protected]("quantization_config, tensors, match", REJECTED)+def test_checkpoint_disagreeing_with_its_quant_config_is_rejected(tmp_path, quantization_config, tensors, match):+ save_file(tensors(_bf16(2 * QH * AHD, H)), str(tmp_path / "model.safetensors"))+ (tmp_path / "config.json").write_text(json.dumps(_config_json(quantization_config)))+ with pytest.raises(ValueError, match=match):+ _load(str(tmp_path))diff --git a/tests/models/qwen4_exp/test_weight_ckpt.py b/tests/models/qwen4_exp/test_weight_ckpt.pyindex 82f9a01d..605c40f1 100644--- a/tests/models/qwen4_exp/test_weight_ckpt.py+++ b/tests/models/qwen4_exp/test_weight_ckpt.py@@ -31,6 +31,8 @@ from freetoken.moe.expert_banks import build_expert_banksfrom freetoken.moe.host_banks import HostResidencyfrom freetoken.utils import cached_load_hf_config+from .common import install_quant_config, meta_state_dict+MODEL_PATH = os.environ.get("FREETOKEN_QWEN4EXP_MODEL")pytestmark = [pytest.mark.needs_weights,@@ -166,6 +168,7 @@ def dense_pass() -> tuple[list[str], dict[str, torch.Tensor]]:wanted = {name for name, _raw, _mode in SAMPLES}names: list[str] = []sampled: dict[str, torch.Tensor] = {}+ install_quant_config(MODEL_PATH)for name, tensor in iter_weights(MODEL_PATH, torch.device("cpu"), include_moe_experts=True, include_non_moe=True):@@ -190,27 +193,12 @@ def test_emitted_names_are_unique_and_complete(dense_pass):@pytest.fixture(scope="module")def model_state_dict_keys() -> set[str]:"""Keys ``Qwen4ExpForCausalLM`` declares -- the authoritative target the loader must fill."""- from freetoken.layers import rotary- from freetoken.models.qwen4_exp.model import Qwen4ExpForCausalLM-- config = parse_config(cached_load_hf_config(MODEL_PATH))- saved = rotary._ROPE_DEVICE- rotary.set_rope_device(torch.device("cpu")) # get_rope refuses to build on meta- rotary.get_rope.cache_clear()- try:- with torch.device("meta"):- return set(Qwen4ExpForCausalLM(config).state_dict())- finally:- rotary.set_rope_device(saved)- rotary.get_rope.cache_clear()+ return set(meta_state_dict(MODEL_PATH))def test_emitted_names_are_the_model_state_dict(dense_pass, model_state_dict_keys):names, _sampled = dense_pass- # The routed NVFP4 experts come from the offload source banks, never from the dense pass.- expected = {k for k in model_state_dict_keys- if not k.endswith((".mlp.experts.gate_up_proj", ".mlp.experts.down_proj"))}- assert set(names) == expected+ assert set(names) == model_state_dict_keysdef test_every_zero_centered_norm_is_present_and_raw(dense_pass, reader):