FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(qwen4_exp): load gguf files with a quantized indexer (#760)
Xiaoze Fan64b85c344c7a
diff --git a/python/freetoken/models/qwen4_exp/attention.py b/python/freetoken/models/qwen4_exp/attention.pyindex 8f6b9388..0ac45ef1 100644--- a/python/freetoken/models/qwen4_exp/attention.py+++ b/python/freetoken/models/qwen4_exp/attention.py@@ -85,7 +85,7 @@ class Qwen4ExpIndexer(BaseOP):self.eps = config.rms_norm_epsself._split = [self.num_heads * self.head_dim, self.num_kv_heads * self.head_dim]self.index_qk_proj = LinearReplicated(- args.hidden_size, sum(self._split), has_bias=False,+ args.hidden_size, sum(self._split), has_bias=False, output_sizes=self._split,quant_config=config.quant, prefix=f"{prefix}.index_qk_proj",)self.q_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)diff --git a/python/freetoken/models/qwen4_exp/gguf.py b/python/freetoken/models/qwen4_exp/gguf.pyindex 0c1bf980..f29f6533 100644--- a/python/freetoken/models/qwen4_exp/gguf.py+++ b/python/freetoken/models/qwen4_exp/gguf.py@@ -45,7 +45,8 @@ _LAYER_LEAVES: dict[str, tuple[str, ...]] = {"self_attn.k_proj": ("attn_k",),"self_attn.v_proj": ("attn_v",),"self_attn.o_proj": ("attn_output",),- "self_attn.indexer.index_qk_proj": ("indexer.q_proj", "indexer.k_proj"),+ "self_attn.indexer.index_q_proj": ("indexer.q_proj",),+ "self_attn.indexer.index_k_proj": ("indexer.k_proj",),"mlp.gate": ("ffn_gate_inp",),"mlp.shared_expert_gate": ("ffn_gate_inp_shexp",),"mlp.shared_expert.gate_proj": ("ffn_gate_shexp",),@@ -85,6 +86,10 @@ def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:# NVFP4 weight scales are unvalidated on this arch, so refuse them instead of risking wrong outputcheck_scales(shim.model_path, nvfp4=False)+ # a metadata-only GGUF (an FTW dir's source_metadata.gguf) has no tensor table to check+ index = gguf_tensor_index(shim.model_path)+ if index and "blk.0.hc_attn_down.weight" not in index:+ raise NotImplementedError(f"{shim.model_path}: no decoder layer tensors (blk.0.hc_attn_down), e.g. an MTP-only draft GGUF")md = shim.metadatadef kv(key: str):diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 42c6f3f5..80b732cd 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -63,6 +63,8 @@ _QWEN3_5_UNQUANTIZED = ("*.mlp.gate", "*.mlp.shared_expert_gate")_QWEN4_EXP_PACKED = _QWEN3_5_PACKED + (("input_mix_weight_down_block_inject", ("input_mix_weight_down", "block_inject_weight")),)+# the llama.cpp converter splits the indexer projection into q and k+_QWEN4_EXP_GGUF_PACKED = _QWEN4_EXP_PACKED + (("index_qk_proj", ("index_q_proj", "index_k_proj")),)# GLM-5.3-Flash's KDA fuses its six input projections._GLM5_NEXT_PACKED = _EXPERTS_PACKED + (("in_proj", ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")),@@ -288,7 +290,7 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {parse_config="parse_gguf_config",iter_weights="iter_gguf_weights",checkpoint_roots=_LANGUAGE_MODEL_ROOT,- packed_modules_mapping=_QWEN4_EXP_PACKED,+ packed_modules_mapping=_QWEN4_EXP_GGUF_PACKED,),# Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the# qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the