Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(qwen3_5_moe)!: apply the zero-centred norm weights' +1 in fp32 at runtime (#755)

Xiaoze Fan05e7cb717795
diff --git a/docs/ftw-hotfix.md b/docs/ftw-hotfix.md
index a88c0066..0c6b6b9f 100644
--- a/docs/ftw-hotfix.md
+++ b/docs/ftw-hotfix.md
@@ -33,6 +33,7 @@ byte range, never the whole checkpoint; `--revision` pins the Hub revision to re
| `RuntimeError: Unexpected keys ... .weight_scale` | nvidia/GLM-5.2-NVFP4 | dequantize the old runtime-fp8 weights back to bf16 | none |
| `PLE shard indices are not contiguous 0..N-1: []` | Qwen3.8-Flash-Next (FTWs converted before #420) | write the PLE table as `ple-table-*.safetensors` | 47.7 GiB |
| `ValueError: ... stores NVFP4 global scales as fp16` | NVFP4 FTWs converted by older builds | none: the fp16 rounding cannot be undone, so the script refuses; reconvert with `ft checkpoint` | - |
+| none: loads, but the output is wrong | Qwen3.5 / Qwen3.6 / Qwen3.8-27B FTWs converted by older builds (norm weights stored as bf16(1 + w)) | none: the script does not detect it; reconvert with `ft checkpoint` | - |
| `ValueError: ... holds no vision encoder tensors` | image-input families (Qwen3-VL, Qwen3.6, Qwen3.8-Flash-Next, Gemma-4, GLM-5.3-Flash, Muse-Glimmer, MiniMax-M3) converted before the family served images | add the vision encoder: the tower's checkpoint tensors are fetched and the family's own weight reader emits them as the converter would | 1 to 4 GiB |
The script decides by itself: it builds the current model from the FTW's `config.json`, compares
diff --git a/docs/models.md b/docs/models.md
index 9c35ece2..18f73fac 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -88,6 +88,9 @@ unset on PCIe GPUs: wider grids add no bandwidth there.
when the gate was fused into `qkvg_proj` no longer loads, reconvert it with `ft checkpoint`.
- NVFP4 global scales are kept in fp32. An NVFP4 FTW that stores them as fp16 (converted by an
older build) is refused at load; reconvert it with `ft checkpoint`.
+- Qwen3.5 / Qwen3.6 / Qwen3.8-27B load their zero-centred norm weights as stored and add the 1 in
+ fp32 at runtime. An FTW of these models converted by an older build stores the norms with the 1
+ already added: it loads without an error but gives wrong output; reconvert it with `ft checkpoint`.
- An FTW converted before its family served images holds no vision encoder: `ft serve`
refuses it unless started with `--text-model-only` (or `--mm-disable vision`); reconvert it
with `ft checkpoint`, or add the encoder in place with [scripts/ftw_hotfix.py](ftw-hotfix.md).
diff --git a/python/freetoken/models/qwen3_5_moe/attention.py b/python/freetoken/models/qwen3_5_moe/attention.py
index 7a9e6a33..0a736a10 100644
--- a/python/freetoken/models/qwen3_5_moe/attention.py
+++ b/python/freetoken/models/qwen3_5_moe/attention.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING
import torch
from freetoken.core import get_global_ctx
-from freetoken.layers import BaseOP, GemmaRMSNorm, LinearColParallelMerged, LinearReplicated
+from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm, LinearColParallelMerged, LinearReplicated
from freetoken.layers.rotary import get_rope
from freetoken.utils import nvtx_annotate
@@ -41,10 +41,8 @@ class Qwen3_5Attention(BaseOP):
config.hidden_size, self._qkv_split, has_bias=False,
quant_config=config.quant, prefix=f"{prefix}.qkv_proj",
)
- # Qwen3.5 uses Gemma-style (1+weight) RMSNorm; the weight loader bakes the +1
- # into the stored weight (GemmaRMSNorm scales by the raw weight).
- self.q_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps)
- self.k_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps)
+ self.q_norm = GemmaPlusOneRMSNorm(head_dim, eps=config.rms_norm_eps)
+ self.k_norm = GemmaPlusOneRMSNorm(head_dim, eps=config.rms_norm_eps)
self.rotary = get_rope(
head_dim=head_dim,
rotary_dim=config.rotary_config.rotary_dim,
diff --git a/python/freetoken/models/qwen3_5_moe/gguf.py b/python/freetoken/models/qwen3_5_moe/gguf.py
index 7f72061e..caf12680 100644
--- a/python/freetoken/models/qwen3_5_moe/gguf.py
+++ b/python/freetoken/models/qwen3_5_moe/gguf.py
@@ -1,6 +1,6 @@
"""Qwen3.5 / 3.6 / 3.8 dense and MoE from a llama.cpp ``qwen35`` / ``qwen35moe`` GGUF: config from the GGUF metadata, tensor names, and the converter's transforms undone.
-The llama.cpp converter (conversion/qwen.py) stores ``A_log`` as ``-exp(A_log)``, the squeezed conv kernels, and the GDN value heads in tiled order (k-head minor). Its zero-centred norms are stored as ``w + 1``, which is what this family loads. Row reorders are undone on the packed bytes; ``ssm_out`` keeps its tiled columns and the GDN layer permutes its input instead.
+The llama.cpp converter (conversion/qwen.py) stores ``A_log`` as ``-exp(A_log)``, the squeezed conv kernels, and the GDN value heads in tiled order (k-head minor). Its zero-centred norms are stored as ``w + 1``; the loader subtracts the 1 in fp32 and the norms add it back in fp32 at runtime. Row reorders are undone on the packed bytes; ``ssm_out`` keeps its tiled columns and the GDN layer permutes its input instead.
"""
from __future__ import annotations
@@ -162,15 +162,18 @@ def iter_gguf_weights(
def t(name: str) -> GgufTensorInfo:
return index[name if name in index else f"{name}.weight"]
+ def norm(name: str) -> torch.Tensor:
+ return float_tensor(t(name)).float() - 1.0
+
yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
if not config.tie_word_embeddings:
yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
- yield "model.norm.weight", float_tensor(t("output_norm"))
+ yield "model.norm.weight", norm("output_norm")
for layer in range(config.num_layers):
blk, pre = f"blk.{layer}", f"model.layers.{layer}"
- yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
- yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.post_attention_norm"))
+ yield f"{pre}.input_layernorm.weight", norm(f"{blk}.attn_norm")
+ yield f"{pre}.post_attention_layernorm.weight", norm(f"{blk}.post_attention_norm")
if config.is_linear_layer(layer):
la = f"{pre}.linear_attn"
nk, dk = g.num_key_heads, g.key_head_dim
@@ -199,8 +202,8 @@ def iter_gguf_weights(
yield from fused(f"{sa}.qkv_proj", parts, [matrix(p) for p in parts], hidden)
out = t(f"{blk}.attn_output")
yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
- yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))
- yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))
+ yield f"{sa}.q_norm.weight", norm(f"{blk}.attn_q_norm")
+ yield f"{sa}.k_norm.weight", norm(f"{blk}.attn_k_norm")
mlp = f"{pre}.mlp"
if config.moe_enabled:
diff --git a/python/freetoken/models/qwen3_5_moe/model.py b/python/freetoken/models/qwen3_5_moe/model.py
index 3fe373ca..88c6ae13 100644
--- a/python/freetoken/models/qwen3_5_moe/model.py
+++ b/python/freetoken/models/qwen3_5_moe/model.py
@@ -6,7 +6,7 @@ import torch
from freetoken.core import get_global_ctx
from freetoken.layers import (
BaseOP,
- GemmaRMSNorm,
+ GemmaPlusOneRMSNormFused,
OPList,
ParallelLMHead,
VocabParallelEmbedding,
@@ -58,20 +58,15 @@ class Qwen3_5DecoderLayer(BaseOP):
if config.moe_enabled
else Qwen3_5DenseMLP(config, prefix=f"{prefix}.mlp")
)
- self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
- self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.input_layernorm = GemmaPlusOneRMSNormFused(config.hidden_size, eps=config.rms_norm_eps)
+ self.post_attention_layernorm = GemmaPlusOneRMSNormFused(config.hidden_size, eps=config.rms_norm_eps)
@nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
def forward(self, hidden: torch.Tensor, residual: torch.Tensor | None):
- # Residual-stream form: fuse each residual-add into the next RMSNorm
- # (GemmaRMSNorm.forward_add_residual) so add + norm are one kernel per sublayer.
- if residual is None:
- residual = hidden
- hidden = self.input_layernorm.forward(hidden)
- else:
- hidden, residual = self.input_layernorm.forward_add_residual(hidden, residual)
+ # Residual-stream form: fuse each residual-add into the next RMSNorm so add + norm are one kernel per sublayer.
+ hidden, residual = self.input_layernorm.forward(hidden, residual)
hidden = self.linear_attn.forward(hidden) if self._is_linear else self.self_attn.forward(hidden)
- hidden, residual = self.post_attention_layernorm.forward_add_residual(hidden, residual)
+ hidden, residual = self.post_attention_layernorm.forward(hidden, residual)
hidden = self.mlp.forward(hidden)
return hidden, residual
@@ -90,14 +85,14 @@ class Qwen3_5Model(BaseOP):
for layer_id in range(config.num_layers)
]
)
- self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+ self.norm = GemmaPlusOneRMSNormFused(config.hidden_size, eps=config.rms_norm_eps)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
x = embed_input_ids(self.embed_tokens, input_ids, get_global_ctx().batch)
residual: torch.Tensor | None = None
for layer in self.layers.op_list:
x, residual = layer.forward(x, residual)
- x, _ = self.norm.forward_add_residual(x, residual)
+ x, _ = self.norm.forward(x, residual)
return x
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index e7c4676d..afd32eb2 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -35,13 +35,6 @@ _EXPERT_KEY_RE = (
# role -> the expert bank reader's canonical (ModelOpt) tensor kind
_BANK_KINDS = {"weight": "weight", "weight_scale": "weight_scale", "weight_global": "weight_scale_2"}
-# Gemma-style (1+weight) RMSNorm weights; the GDN gated norm (linear_attn.norm) is a plain weight*x norm
-_GEMMA_NORM_SUFFIXES = (
- ".input_layernorm.weight",
- ".post_attention_layernorm.weight",
- ".self_attn.q_norm.weight",
- ".self_attn.k_norm.weight",
-)
# leaves the model builds as Linear layers: only their tensors are read under the QuantConfig, the rest passes through as stored
_LINEAR_LEAVES = frozenset({
"q_proj", "k_proj", "v_proj", "o_proj", "in_proj_qkv", "in_proj_z", "in_proj_b", "in_proj_a", "out_proj",
@@ -63,10 +56,6 @@ def _rename(raw_name: str) -> str | None:
return rename_vl_prefix(raw_name)
-def _is_gemma_norm(name: str) -> bool:
- return name == "model.norm.weight" or name.endswith(_GEMMA_NORM_SUFFIXES)
-
-
def _per_row_scale(scale: torch.Tensor, rows: int) -> torch.Tensor:
"""Per-tensor scalar or per-channel ``[rows, 1]`` fp8 scale -> fp32 ``[rows]``; any other count is refused rather than broadcast onto the wrong rows."""
flat = scale.reshape(-1).to(torch.float32)
@@ -274,8 +263,6 @@ def _iter_shards(model_path: str, device: torch.device, reader: _DenseReader | N
emitted = reader.add(name, tensor)
if emitted is not None:
yield from emitted
- elif _is_gemma_norm(name):
- yield name, tensor + 1.0 # (1 + weight) baked into the stored norm weight
else:
yield name, tensor
if reader is not None and reader.pending:
diff --git a/tests/models/test_qwen3_5_moe_weight.py b/tests/models/test_qwen3_5_moe_weight.py
index c502067e..958d6daf 100644
--- a/tests/models/test_qwen3_5_moe_weight.py
+++ b/tests/models/test_qwen3_5_moe_weight.py
@@ -358,7 +358,7 @@ def _same(a: torch.Tensor, b: torch.Tensor) -> bool:
return a.dtype is b.dtype and torch.equal(a.view(torch.uint8), b.view(torch.uint8))
-def test_bf16_fusions_slice_back_and_norms_get_plus_one(checkpoint):
+def test_bf16_fusions_slice_back_and_norms_load_as_stored(checkpoint):
name, folder, raw = checkpoint
if name != "bf16":
pytest.skip("bf16 layout only")
@@ -371,8 +371,8 @@ def test_bf16_fusions_slice_back_and_norms_get_plus_one(checkpoint):
merged = loaded["model.layers.1.mlp.shared_expert.gate_up_proj.weight"]
assert _same(merged[:I], raw[f"{LM}.layers.1.mlp.shared_expert.gate_proj.weight"])
assert _same(merged[I:], raw[f"{LM}.layers.1.mlp.shared_expert.up_proj.weight"])
- assert torch.equal(loaded["model.norm.weight"], raw[f"{LM}.norm.weight"] + 1.0)
- assert torch.equal(loaded["model.layers.1.self_attn.q_norm.weight"], raw[f"{attn}.q_norm.weight"] + 1.0)
+ assert torch.equal(loaded["model.norm.weight"], raw[f"{LM}.norm.weight"])
+ assert torch.equal(loaded["model.layers.1.self_attn.q_norm.weight"], raw[f"{attn}.q_norm.weight"])
assert torch.equal(loaded["model.layers.0.linear_attn.norm.weight"], raw[f"{gdn}.norm.weight"])
assert loaded["model.layers.0.linear_attn.A_log"].dtype is torch.float32