FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(models): preserve glm4 router correction bias in fp32 (#469)
cherry77-cloudcab110ecb74a
diff --git a/python/freetoken/models/glm4_moe/moe.py b/python/freetoken/models/glm4_moe/moe.pyindex 8e595ee7..648eb9c3 100644--- a/python/freetoken/models/glm4_moe/moe.py+++ b/python/freetoken/models/glm4_moe/moe.py@@ -32,10 +32,8 @@ class Glm4MoeSparseBlock(BaseOP):self.topk_group = config.topk_groupself.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)- # DeepSeek-style selection bias; a registered buffer in HF (kept fp32 there). We- # store it in the model's bf16 dtype and upcast at use, which is exact enough for- # the argmax-style top-k selection.- self.e_score_correction_bias = torch.empty(config.num_experts)+ # Keep selection bias in fp32: rounding can change the selected experts.+ self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32)# The offload cache indexes experts by *MoE* layer (global layer minus# first_k_dense_replace), matching how the loader packs the expert banks. Thediff --git a/python/freetoken/models/glm4_moe/weight.py b/python/freetoken/models/glm4_moe/weight.pyindex 38b17a05..aa6538dc 100644--- a/python/freetoken/models/glm4_moe/weight.py+++ b/python/freetoken/models/glm4_moe/weight.py@@ -168,11 +168,11 @@ def _iter_resident_weights(reader, config, primary) -> Iterator[tuple[str, torchfor proj in ("gate_proj", "up_proj", "down_proj"):yield from _iter_nvfp4_resident(reader, f"{m}.{proj}", f"{m}.{proj}")else:- # router (bf16 gate + fp32 selection bias -> bf16) and shared expert.+ # router (bf16 gate + fp32 selection bias) and shared expert.yield f"{m}.gate.weight", reader.get(f"{m}.gate.weight")yield (f"{m}.e_score_correction_bias",- reader.get(f"{m}.gate.e_score_correction_bias").to(torch.bfloat16),+ reader.get(f"{m}.gate.e_score_correction_bias").to(torch.float32),)s = f"{m}.shared_experts"for proj in ("gate_proj", "up_proj", "down_proj"):