Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(qwen4_exp): load gguf files with a quantized indexer (#760)

Xiaoze Fan64b85c344c7a
diff --git a/python/freetoken/models/qwen4_exp/attention.py b/python/freetoken/models/qwen4_exp/attention.py
index 8f6b9388..0ac45ef1 100644
--- a/python/freetoken/models/qwen4_exp/attention.py
+++ b/python/freetoken/models/qwen4_exp/attention.py
@@ -85,7 +85,7 @@ class Qwen4ExpIndexer(BaseOP):
self.eps = config.rms_norm_eps
self._split = [self.num_heads * self.head_dim, self.num_kv_heads * self.head_dim]
self.index_qk_proj = LinearReplicated(
- args.hidden_size, sum(self._split), has_bias=False,
+ args.hidden_size, sum(self._split), has_bias=False, output_sizes=self._split,
quant_config=config.quant, prefix=f"{prefix}.index_qk_proj",
)
self.q_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)
diff --git a/python/freetoken/models/qwen4_exp/gguf.py b/python/freetoken/models/qwen4_exp/gguf.py
index 0c1bf980..f29f6533 100644
--- a/python/freetoken/models/qwen4_exp/gguf.py
+++ b/python/freetoken/models/qwen4_exp/gguf.py
@@ -45,7 +45,8 @@ _LAYER_LEAVES: dict[str, tuple[str, ...]] = {
"self_attn.k_proj": ("attn_k",),
"self_attn.v_proj": ("attn_v",),
"self_attn.o_proj": ("attn_output",),
- "self_attn.indexer.index_qk_proj": ("indexer.q_proj", "indexer.k_proj"),
+ "self_attn.indexer.index_q_proj": ("indexer.q_proj",),
+ "self_attn.indexer.index_k_proj": ("indexer.k_proj",),
"mlp.gate": ("ffn_gate_inp",),
"mlp.shared_expert_gate": ("ffn_gate_inp_shexp",),
"mlp.shared_expert.gate_proj": ("ffn_gate_shexp",),
@@ -85,6 +86,10 @@ def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
# NVFP4 weight scales are unvalidated on this arch, so refuse them instead of risking wrong output
check_scales(shim.model_path, nvfp4=False)
+ # a metadata-only GGUF (an FTW dir's source_metadata.gguf) has no tensor table to check
+ index = gguf_tensor_index(shim.model_path)
+ if index and "blk.0.hc_attn_down.weight" not in index:
+ raise NotImplementedError(f"{shim.model_path}: no decoder layer tensors (blk.0.hc_attn_down), e.g. an MTP-only draft GGUF")
md = shim.metadata
def kv(key: str):
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 42c6f3f5..80b732cd 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -63,6 +63,8 @@ _QWEN3_5_UNQUANTIZED = ("*.mlp.gate", "*.mlp.shared_expert_gate")
_QWEN4_EXP_PACKED = _QWEN3_5_PACKED + (
("input_mix_weight_down_block_inject", ("input_mix_weight_down", "block_inject_weight")),
)
+# the llama.cpp converter splits the indexer projection into q and k
+_QWEN4_EXP_GGUF_PACKED = _QWEN4_EXP_PACKED + (("index_qk_proj", ("index_q_proj", "index_k_proj")),)
# GLM-5.3-Flash's KDA fuses its six input projections.
_GLM5_NEXT_PACKED = _EXPERTS_PACKED + (
("in_proj", ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")),
@@ -288,7 +290,7 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
parse_config="parse_gguf_config",
iter_weights="iter_gguf_weights",
checkpoint_roots=_LANGUAGE_MODEL_ROOT,
- packed_modules_mapping=_QWEN4_EXP_PACKED,
+ packed_modules_mapping=_QWEN4_EXP_GGUF_PACKED,
),
# Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the
# qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the