Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): serve Qwen3.5 dense and MoE GGUF checkpoints (#701)

Xiaoze Fan18448d29a764
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index 4eb2a999..f9210938 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -18,6 +18,8 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names
# reuses the model classes but a GGUF parse_config / iter_weights).
GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
"gemma4": "Gemma4GGUFForCausalLM",
+ "qwen35": "Qwen3_5GGUFForCausalLM",
+ "qwen35moe": "Qwen3_5MoeGGUFForCausalLM",
"qwen4exp": "Qwen4ExpGGUFForCausalLM",
}
diff --git a/python/freetoken/models/qwen3_5_moe/__init__.py b/python/freetoken/models/qwen3_5_moe/__init__.py
index 018b1c1e..abe8fd1c 100644
--- a/python/freetoken/models/qwen3_5_moe/__init__.py
+++ b/python/freetoken/models/qwen3_5_moe/__init__.py
@@ -1,4 +1,5 @@
from .config import parse_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import (
Qwen3_5ForCausalLM,
Qwen3_5ForConditionalGeneration,
@@ -18,4 +19,7 @@ __all__ = [
"iter_weights_parallel",
"iter_expert_pieces",
"nvfp4_expert_spec",
+ "iter_gguf_weights",
+ "parse_gguf_config",
+ "to_gguf",
]
diff --git a/python/freetoken/models/qwen3_5_moe/gdn.py b/python/freetoken/models/qwen3_5_moe/gdn.py
index d5fe2aa8..0785ad49 100644
--- a/python/freetoken/models/qwen3_5_moe/gdn.py
+++ b/python/freetoken/models/qwen3_5_moe/gdn.py
@@ -30,9 +30,10 @@ class Qwen3_5GatedDeltaNet(BaseOP):
def __init__(
self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,
conv_kernel_size, rms_norm_eps, layer_id, *, quant_config: QuantConfig | None = None,
- prefix: str = "",
+ prefix: str = "", out_heads_tiled: bool = False,
):
self.layer_id = layer_id
+ self._out_heads_tiled = out_heads_tiled
# The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as
# [V, K] while the LinearStatePool declares it [K, V]; these coincide (and the
# hybrid-radix snapshot scatter h[h_row]->slot is a plain copy) only when the two head
@@ -195,6 +196,10 @@ class Qwen3_5GatedDeltaNet(BaseOP):
core_out = core_out.reshape(-1, self.head_v_dim)
z = z.reshape(-1, self.head_v_dim)
out = self.norm.forward(core_out, z).reshape(total, -1)
+ if self._out_heads_tiled:
+ # grouped value heads (k-head major) -> tiled (k-head minor), the column order of a GGUF ssm_out
+ r = self.num_v_heads // self.num_k_heads
+ out = out.view(total, self.num_k_heads, r, self.head_v_dim).transpose(1, 2).reshape(total, -1)
return self.out_proj.forward(out)
diff --git a/python/freetoken/models/qwen3_5_moe/gguf.py b/python/freetoken/models/qwen3_5_moe/gguf.py
new file mode 100644
index 00000000..85af8ab8
--- /dev/null
+++ b/python/freetoken/models/qwen3_5_moe/gguf.py
@@ -0,0 +1,220 @@
+"""Qwen3.5 / 3.6 / 3.8 dense and MoE from a llama.cpp ``qwen35`` / ``qwen35moe`` GGUF: config from the GGUF metadata, tensor names, and the converter's transforms undone.
+
+The llama.cpp converter (conversion/qwen.py) stores ``A_log`` as ``-exp(A_log)``, the squeezed conv kernels, and the GDN value heads in tiled order (k-head minor). Its zero-centred norms are stored as ``w + 1``, which is what this family loads. Row reorders are undone on the packed bytes; ``ssm_out`` keeps its tiled columns and the GDN layer permutes its input instead.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Iterator
+
+import torch
+
+from freetoken.layers.quantization.ggml_types import is_quantized
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index
+from freetoken.models.gguf.tensors import float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+ from freetoken.models.gguf.config import GgufConfigShim
+
+_LAYER_LEAVES: dict[str, tuple[str, ...]] = {
+ "linear_attn.in_proj_qkv": ("attn_qkv",),
+ "linear_attn.in_proj_z": ("attn_gate",),
+ "linear_attn.in_proj_b": ("ssm_beta",),
+ "linear_attn.in_proj_a": ("ssm_alpha",),
+ "linear_attn.out_proj": ("ssm_out",),
+ "self_attn.q_proj": ("attn_q",),
+ "self_attn.k_proj": ("attn_k",),
+ "self_attn.v_proj": ("attn_v",),
+ "self_attn.o_proj": ("attn_output",),
+ "mlp.gate_proj": ("ffn_gate",),
+ "mlp.up_proj": ("ffn_up",),
+ "mlp.down_proj": ("ffn_down",),
+ "mlp.gate": ("ffn_gate_inp",),
+ "mlp.shared_expert_gate": ("ffn_gate_inp_shexp",),
+ "mlp.shared_expert.gate_proj": ("ffn_gate_shexp",),
+ "mlp.shared_expert.up_proj": ("ffn_up_shexp",),
+ "mlp.shared_expert.down_proj": ("ffn_down_shexp",),
+ "mlp.experts.0.gate_proj": ("ffn_gate_exps",),
+ "mlp.experts.0.up_proj": ("ffn_up_exps",),
+ "mlp.experts.0.down_proj": ("ffn_down_exps",),
+}
+_TOP_LEAVES: dict[str, tuple[str, ...]] = {
+ "model.language_model.embed_tokens": ("token_embd",),
+ "lm_head": ("output",),
+}
+_LAYER_RE = re.compile(r"^model\.language_model\.layers\.(\d+)\.(.+)$")
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name in _TOP_LEAVES:
+ return _TOP_LEAVES[name]
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ blk, leaf = f"blk.{m.group(1)}", m.group(2)
+ if leaf in ("mlp.experts.0.gate_proj", "mlp.experts.0.up_proj") and f"{blk}.ffn_gate_up_exps.weight" in tensors:
+ return (f"{blk}.ffn_gate_up_exps",)
+ return tuple(f"{blk}.{t}" for t in _LAYER_LEAVES.get(leaf, ()))
+
+
+def _check_supported(shim: GgufConfigShim) -> None:
+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):
+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
+ scaled = sorted(n for n in gguf_tensor_index(shim.model_path) if n.endswith((".scale", ".input_scale")))
+ if scaled:
+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ md = shim.metadata
+ arch = shim.model_type
+
+ def kv(key: str):
+ return md[f"{arch}.{key}"]
+
+ _check_supported(shim)
+ head_dim = int(kv("attention.key_length"))
+ n_v = int(kv("ssm.time_step_rank"))
+ # block_count also counts the MTP layers a GGUF may carry after the decoder
+ num_layers = int(kv("block_count")) - int(md.get(f"{arch}.nextn_predict_layers", 0))
+ recurrent = md.get(f"{arch}.attention.recurrent_layers")
+ if recurrent is not None:
+ layer_types = ["linear_attention" if r else "full_attention" for r in list(recurrent)[:num_layers]]
+ else:
+ interval = int(kv("full_attention_interval"))
+ layer_types = ["full_attention" if (i + 1) % interval == 0 else "linear_attention" for i in range(num_layers)]
+ moe = arch == "qwen35moe"
+ text = SimpleNamespace(
+ hidden_act="silu",
+ hidden_size=int(kv("embedding_length")),
+ num_hidden_layers=num_layers,
+ layer_types=layer_types,
+ num_attention_heads=int(kv("attention.head_count")),
+ num_key_value_heads=int(kv("attention.head_count_kv")),
+ head_dim=head_dim,
+ rope_parameters={
+ "rope_type": "default",
+ "rope_theta": float(kv("rope.freq_base")),
+ "partial_rotary_factor": int(kv("rope.dimension_count")) / head_dim,
+ "mrope_section": [int(s) for s in kv("rope.dimension_sections")][:3],
+ "mrope_interleaved": True,
+ },
+ max_position_embeddings=int(kv("context_length")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ vocab_size=int(shim.vocab_size),
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ intermediate_size=0 if moe else int(kv("feed_forward_length")),
+ num_experts=int(kv("expert_count")) if moe else 0,
+ num_experts_per_tok=int(kv("expert_used_count")) if moe else 0,
+ moe_intermediate_size=int(kv("expert_feed_forward_length")) if moe else 0,
+ shared_expert_intermediate_size=int(kv("expert_shared_feed_forward_length")) if moe else 0,
+ linear_num_key_heads=int(kv("ssm.group_count")),
+ linear_num_value_heads=n_v,
+ linear_key_head_dim=int(kv("ssm.state_size")),
+ linear_value_head_dim=int(kv("ssm.inner_size")) // n_v,
+ linear_conv_kernel_dim=int(kv("ssm.conv_kernel")),
+ )
+ hf_like = SimpleNamespace(
+ text_config=text,
+ model_type="qwen3_5_moe" if moe else "qwen3_5",
+ architectures=list(shim.architectures),
+ vision_config=None,
+ )
+ config = parse_config(hf_like)
+ return replace(config, expert_quant="gguf") if moe else config
+
+
+# ----------------------------------------------------------------------------------------------
+# Weights
+# ----------------------------------------------------------------------------------------------
+
+def _untile(t: torch.Tensor, num_k_heads: int, head_dim: int, dim: int = 0) -> torch.Tensor:
+ """Tiled value heads (k-head minor) back to grouped (k-head major) along ``dim``."""
+ n = t.shape[dim]
+ r = n // (num_k_heads * head_dim)
+ idx = torch.arange(n).view(r, num_k_heads, head_dim).transpose(0, 1).reshape(n)
+ return t.index_select(dim, idx)
+
+
+def iter_gguf_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+ include_vision: bool = True,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """The dense state dict from the GGUF; the routed experts load as offload banks (``freetoken.models.gguf.experts``)."""
+ if not include_non_moe:
+ return
+ from freetoken.utils import cached_load_hf_config
+
+ config = parse_gguf_config(cached_load_hf_config(model_path))
+ index = gguf_tensor_index(model_path)
+ g = config.linear_attention_group()
+ hidden = config.hidden_size
+
+ def t(name: str) -> GgufTensorInfo:
+ return index[name if name in index else f"{name}.weight"]
+
+ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
+ if not config.tie_word_embeddings:
+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
+ yield "model.norm.weight", float_tensor(t("output_norm"))
+
+ for layer in range(config.num_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.post_attention_norm"))
+ if config.is_linear_layer(layer):
+ la = f"{pre}.linear_attn"
+ nk, dk = g.num_key_heads, g.key_head_dim
+ qkv, gate = t(f"{blk}.attn_qkv"), t(f"{blk}.attn_gate")
+ qkv_rows = matrix(qkv)
+ v_start = 2 * nk * dk
+ qkv_rows = torch.cat([qkv_rows[:v_start], _untile(qkv_rows[v_start:], nk, g.value_head_dim)])
+ z_rows = _untile(matrix(gate), nk, g.value_head_dim)
+ beta_info, alpha_info = t(f"{blk}.ssm_beta"), t(f"{blk}.ssm_alpha")
+ beta, alpha = (_untile(matrix(i), nk, 1) for i in (beta_info, alpha_info))
+ if is_quantized(qkv.ggml_type) or is_quantized(gate.ggml_type):
+ yield from fused(f"{la}.in_proj_qkvz", [qkv, gate], [qkv_rows, z_rows], hidden)
+ yield from fused(f"{la}.in_proj_ba", [beta_info, alpha_info], [beta, alpha], hidden)
+ else:
+ yield from fused(f"{la}.in_proj", [qkv, gate, beta_info, alpha_info], [qkv_rows, z_rows, beta, alpha], hidden)
+ conv = float_tensor(t(f"{blk}.ssm_conv1d")).float()
+ conv = torch.cat([conv[:v_start], _untile(conv[v_start:], nk, g.value_head_dim)])
+ yield f"{la}.conv1d.weight", conv.unsqueeze(1)
+ yield f"{la}.dt_bias", _untile(float_tensor(t(f"{blk}.ssm_dt.bias")).float(), nk, 1)
+ yield f"{la}.A_log", _untile(torch.log(-float_tensor(t(f"{blk}.ssm_a")).float()), nk, 1)
+ yield f"{la}.norm.weight", float_tensor(t(f"{blk}.ssm_norm"))
+ yield from fused(f"{la}.out_proj", [t(f"{blk}.ssm_out")], [matrix(t(f"{blk}.ssm_out"))], g.num_value_heads * g.value_head_dim)
+ else:
+ sa = f"{pre}.self_attn"
+ parts = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
+ yield from fused(f"{sa}.qkv_proj", parts, [matrix(p) for p in parts], hidden)
+ out = t(f"{blk}.attn_output")
+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
+ yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))
+ yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))
+
+ mlp = f"{pre}.mlp"
+ if config.moe_enabled:
+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))
+ sg, su, sd = (t(f"{blk}.ffn_{p}_shexp") for p in ("gate", "up", "down"))
+ yield from fused(f"{mlp}.shared_expert.gate_up_proj", [sg, su], [matrix(sg), matrix(su)], hidden)
+ yield from fused(f"{mlp}.shared_expert.down_proj", [sd], [matrix(sd)], config.shared_expert_intermediate_size)
+ yield f"{mlp}.shared_expert_gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp_shexp")).reshape(1, -1)
+ else:
+ fg, fu, fd = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))
+ yield from fused(f"{mlp}.gate_up_proj", [fg, fu], [matrix(fg), matrix(fu)], hidden)
+ yield from fused(f"{mlp}.down_proj", [fd], [matrix(fd)], config.intermediate_size)
+
+
+__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/qwen3_5_moe/model.py b/python/freetoken/models/qwen3_5_moe/model.py
index 8cdd41ae..3fe373ca 100644
--- a/python/freetoken/models/qwen3_5_moe/model.py
+++ b/python/freetoken/models/qwen3_5_moe/model.py
@@ -46,6 +46,8 @@ class Qwen3_5DecoderLayer(BaseOP):
layer_id=layer_id,
quant_config=config.quant,
prefix=f"{prefix}.linear_attn",
+ # llama.cpp stores every qwen35 ssm_out with tiled value-head columns
+ out_heads_tiled=config.quant is not None and config.quant.dialect == "gguf",
)
else:
self.self_attn = Qwen3_5Attention(config, layer_id, prefix=f"{prefix}.self_attn")
@@ -79,6 +81,8 @@ class Qwen3_5Model(BaseOP):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
+ quant_config=config.quant,
+ prefix=f"{prefix}.embed_tokens",
)
self.layers = OPList(
[
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 4b0cea25..86e3a734 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -201,6 +201,25 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
packed_modules_mapping=_QWEN3_5_PACKED,
unquantized_modules=_QWEN3_5_UNQUANTIZED,
),
+ # llama.cpp qwen35 / qwen35moe GGUF (text only): the same model classes with every tensor in its packed ggml type
+ "Qwen3_5GGUFForCausalLM": ModelSpec(
+ "freetoken.models.qwen3_5_moe",
+ "Qwen3_5ForCausalLM",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ packed_modules_mapping=_QWEN3_5_PACKED,
+ unquantized_modules=_QWEN3_5_UNQUANTIZED,
+ ),
+ "Qwen3_5MoeGGUFForCausalLM": ModelSpec(
+ "freetoken.models.qwen3_5_moe",
+ "Qwen3_5MoeForCausalLM",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,
+ packed_modules_mapping=_QWEN3_5_PACKED,
+ unquantized_modules=_QWEN3_5_UNQUANTIZED,
+ ),
# Qwen3.8-Flash-Next (model_type qwen4_exp): multimodal wrapper config (text tower in
# text_config, weights under model.language_model.). 36 GDN + 12 QSA
# compressed-sparse attention layers on 4 hyper-connection residual streams, a PLE