FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(gguf): serve Qwen3.5 dense and MoE GGUF checkpoints (#701)
Xiaoze Fan18448d29a764
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.pyindex 4eb2a999..f9210938 100644--- a/python/freetoken/models/gguf/config.py+++ b/python/freetoken/models/gguf/config.py@@ -18,6 +18,8 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names# reuses the model classes but a GGUF parse_config / iter_weights).GGUF_ARCH_TO_REGISTRY: dict[str, str] = {"gemma4": "Gemma4GGUFForCausalLM",+ "qwen35": "Qwen3_5GGUFForCausalLM",+ "qwen35moe": "Qwen3_5MoeGGUFForCausalLM","qwen4exp": "Qwen4ExpGGUFForCausalLM",}diff --git a/python/freetoken/models/qwen3_5_moe/__init__.py b/python/freetoken/models/qwen3_5_moe/__init__.pyindex 018b1c1e..abe8fd1c 100644--- a/python/freetoken/models/qwen3_5_moe/__init__.py+++ b/python/freetoken/models/qwen3_5_moe/__init__.py@@ -1,4 +1,5 @@from .config import parse_config+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguffrom .model import (Qwen3_5ForCausalLM,Qwen3_5ForConditionalGeneration,@@ -18,4 +19,7 @@ __all__ = ["iter_weights_parallel","iter_expert_pieces","nvfp4_expert_spec",+ "iter_gguf_weights",+ "parse_gguf_config",+ "to_gguf",]diff --git a/python/freetoken/models/qwen3_5_moe/gdn.py b/python/freetoken/models/qwen3_5_moe/gdn.pyindex d5fe2aa8..0785ad49 100644--- a/python/freetoken/models/qwen3_5_moe/gdn.py+++ b/python/freetoken/models/qwen3_5_moe/gdn.py@@ -30,9 +30,10 @@ class Qwen3_5GatedDeltaNet(BaseOP):def __init__(self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,conv_kernel_size, rms_norm_eps, layer_id, *, quant_config: QuantConfig | None = None,- prefix: str = "",+ prefix: str = "", out_heads_tiled: bool = False,):self.layer_id = layer_id+ self._out_heads_tiled = out_heads_tiled# The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as# [V, K] while the LinearStatePool declares it [K, V]; these coincide (and the# hybrid-radix snapshot scatter h[h_row]->slot is a plain copy) only when the two head@@ -195,6 +196,10 @@ class Qwen3_5GatedDeltaNet(BaseOP):core_out = core_out.reshape(-1, self.head_v_dim)z = z.reshape(-1, self.head_v_dim)out = self.norm.forward(core_out, z).reshape(total, -1)+ if self._out_heads_tiled:+ # grouped value heads (k-head major) -> tiled (k-head minor), the column order of a GGUF ssm_out+ r = self.num_v_heads // self.num_k_heads+ out = out.view(total, self.num_k_heads, r, self.head_v_dim).transpose(1, 2).reshape(total, -1)return self.out_proj.forward(out)diff --git a/python/freetoken/models/qwen3_5_moe/gguf.py b/python/freetoken/models/qwen3_5_moe/gguf.pynew file mode 100644index 00000000..85af8ab8--- /dev/null+++ b/python/freetoken/models/qwen3_5_moe/gguf.py@@ -0,0 +1,220 @@+"""Qwen3.5 / 3.6 / 3.8 dense and MoE from a llama.cpp ``qwen35`` / ``qwen35moe`` GGUF: config from the GGUF metadata, tensor names, and the converter's transforms undone.++The llama.cpp converter (conversion/qwen.py) stores ``A_log`` as ``-exp(A_log)``, the squeezed conv kernels, and the GDN value heads in tiled order (k-head minor). Its zero-centred norms are stored as ``w + 1``, which is what this family loads. Row reorders are undone on the packed bytes; ``ssm_out`` keeps its tiled columns and the GDN layer permutes its input instead.+"""++from __future__ import annotations++import re+from dataclasses import replace+from types import SimpleNamespace+from typing import TYPE_CHECKING, Iterator++import torch++from freetoken.layers.quantization.ggml_types import is_quantized+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index+from freetoken.models.gguf.tensors import float_tensor, fused, matrix++from .config import parse_config++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+ from freetoken.models.gguf.config import GgufConfigShim++_LAYER_LEAVES: dict[str, tuple[str, ...]] = {+ "linear_attn.in_proj_qkv": ("attn_qkv",),+ "linear_attn.in_proj_z": ("attn_gate",),+ "linear_attn.in_proj_b": ("ssm_beta",),+ "linear_attn.in_proj_a": ("ssm_alpha",),+ "linear_attn.out_proj": ("ssm_out",),+ "self_attn.q_proj": ("attn_q",),+ "self_attn.k_proj": ("attn_k",),+ "self_attn.v_proj": ("attn_v",),+ "self_attn.o_proj": ("attn_output",),+ "mlp.gate_proj": ("ffn_gate",),+ "mlp.up_proj": ("ffn_up",),+ "mlp.down_proj": ("ffn_down",),+ "mlp.gate": ("ffn_gate_inp",),+ "mlp.shared_expert_gate": ("ffn_gate_inp_shexp",),+ "mlp.shared_expert.gate_proj": ("ffn_gate_shexp",),+ "mlp.shared_expert.up_proj": ("ffn_up_shexp",),+ "mlp.shared_expert.down_proj": ("ffn_down_shexp",),+ "mlp.experts.0.gate_proj": ("ffn_gate_exps",),+ "mlp.experts.0.up_proj": ("ffn_up_exps",),+ "mlp.experts.0.down_proj": ("ffn_down_exps",),+}+_TOP_LEAVES: dict[str, tuple[str, ...]] = {+ "model.language_model.embed_tokens": ("token_embd",),+ "lm_head": ("output",),+}+_LAYER_RE = re.compile(r"^model\.language_model\.layers\.(\d+)\.(.+)$")+++def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""+ if name in _TOP_LEAVES:+ return _TOP_LEAVES[name]+ m = _LAYER_RE.match(name)+ if m is None:+ return ()+ blk, leaf = f"blk.{m.group(1)}", m.group(2)+ if leaf in ("mlp.experts.0.gate_proj", "mlp.experts.0.up_proj") and f"{blk}.ffn_gate_up_exps.weight" in tensors:+ return (f"{blk}.ffn_gate_up_exps",)+ return tuple(f"{blk}.{t}" for t in _LAYER_LEAVES.get(leaf, ()))+++def _check_supported(shim: GgufConfigShim) -> None:+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")+ scaled = sorted(n for n in gguf_tensor_index(shim.model_path) if n.endswith((".scale", ".input_scale")))+ if scaled:+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")+++def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:+ md = shim.metadata+ arch = shim.model_type++ def kv(key: str):+ return md[f"{arch}.{key}"]++ _check_supported(shim)+ head_dim = int(kv("attention.key_length"))+ n_v = int(kv("ssm.time_step_rank"))+ # block_count also counts the MTP layers a GGUF may carry after the decoder+ num_layers = int(kv("block_count")) - int(md.get(f"{arch}.nextn_predict_layers", 0))+ recurrent = md.get(f"{arch}.attention.recurrent_layers")+ if recurrent is not None:+ layer_types = ["linear_attention" if r else "full_attention" for r in list(recurrent)[:num_layers]]+ else:+ interval = int(kv("full_attention_interval"))+ layer_types = ["full_attention" if (i + 1) % interval == 0 else "linear_attention" for i in range(num_layers)]+ moe = arch == "qwen35moe"+ text = SimpleNamespace(+ hidden_act="silu",+ hidden_size=int(kv("embedding_length")),+ num_hidden_layers=num_layers,+ layer_types=layer_types,+ num_attention_heads=int(kv("attention.head_count")),+ num_key_value_heads=int(kv("attention.head_count_kv")),+ head_dim=head_dim,+ rope_parameters={+ "rope_type": "default",+ "rope_theta": float(kv("rope.freq_base")),+ "partial_rotary_factor": int(kv("rope.dimension_count")) / head_dim,+ "mrope_section": [int(s) for s in kv("rope.dimension_sections")][:3],+ "mrope_interleaved": True,+ },+ max_position_embeddings=int(kv("context_length")),+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),+ vocab_size=int(shim.vocab_size),+ tie_word_embeddings=bool(shim.tie_word_embeddings),+ intermediate_size=0 if moe else int(kv("feed_forward_length")),+ num_experts=int(kv("expert_count")) if moe else 0,+ num_experts_per_tok=int(kv("expert_used_count")) if moe else 0,+ moe_intermediate_size=int(kv("expert_feed_forward_length")) if moe else 0,+ shared_expert_intermediate_size=int(kv("expert_shared_feed_forward_length")) if moe else 0,+ linear_num_key_heads=int(kv("ssm.group_count")),+ linear_num_value_heads=n_v,+ linear_key_head_dim=int(kv("ssm.state_size")),+ linear_value_head_dim=int(kv("ssm.inner_size")) // n_v,+ linear_conv_kernel_dim=int(kv("ssm.conv_kernel")),+ )+ hf_like = SimpleNamespace(+ text_config=text,+ model_type="qwen3_5_moe" if moe else "qwen3_5",+ architectures=list(shim.architectures),+ vision_config=None,+ )+ config = parse_config(hf_like)+ return replace(config, expert_quant="gguf") if moe else config+++# ----------------------------------------------------------------------------------------------+# Weights+# ----------------------------------------------------------------------------------------------++def _untile(t: torch.Tensor, num_k_heads: int, head_dim: int, dim: int = 0) -> torch.Tensor:+ """Tiled value heads (k-head minor) back to grouped (k-head major) along ``dim``."""+ n = t.shape[dim]+ r = n // (num_k_heads * head_dim)+ idx = torch.arange(n).view(r, num_k_heads, head_dim).transpose(0, 1).reshape(n)+ return t.index_select(dim, idx)+++def iter_gguf_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+ include_vision: bool = True,+) -> Iterator[tuple[str, torch.Tensor]]:+ """The dense state dict from the GGUF; the routed experts load as offload banks (``freetoken.models.gguf.experts``)."""+ if not include_non_moe:+ return+ from freetoken.utils import cached_load_hf_config++ config = parse_gguf_config(cached_load_hf_config(model_path))+ index = gguf_tensor_index(model_path)+ g = config.linear_attention_group()+ hidden = config.hidden_size++ def t(name: str) -> GgufTensorInfo:+ return index[name if name in index else f"{name}.weight"]++ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)+ if not config.tie_word_embeddings:+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)+ yield "model.norm.weight", float_tensor(t("output_norm"))++ for layer in range(config.num_layers):+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.post_attention_norm"))+ if config.is_linear_layer(layer):+ la = f"{pre}.linear_attn"+ nk, dk = g.num_key_heads, g.key_head_dim+ qkv, gate = t(f"{blk}.attn_qkv"), t(f"{blk}.attn_gate")+ qkv_rows = matrix(qkv)+ v_start = 2 * nk * dk+ qkv_rows = torch.cat([qkv_rows[:v_start], _untile(qkv_rows[v_start:], nk, g.value_head_dim)])+ z_rows = _untile(matrix(gate), nk, g.value_head_dim)+ beta_info, alpha_info = t(f"{blk}.ssm_beta"), t(f"{blk}.ssm_alpha")+ beta, alpha = (_untile(matrix(i), nk, 1) for i in (beta_info, alpha_info))+ if is_quantized(qkv.ggml_type) or is_quantized(gate.ggml_type):+ yield from fused(f"{la}.in_proj_qkvz", [qkv, gate], [qkv_rows, z_rows], hidden)+ yield from fused(f"{la}.in_proj_ba", [beta_info, alpha_info], [beta, alpha], hidden)+ else:+ yield from fused(f"{la}.in_proj", [qkv, gate, beta_info, alpha_info], [qkv_rows, z_rows, beta, alpha], hidden)+ conv = float_tensor(t(f"{blk}.ssm_conv1d")).float()+ conv = torch.cat([conv[:v_start], _untile(conv[v_start:], nk, g.value_head_dim)])+ yield f"{la}.conv1d.weight", conv.unsqueeze(1)+ yield f"{la}.dt_bias", _untile(float_tensor(t(f"{blk}.ssm_dt.bias")).float(), nk, 1)+ yield f"{la}.A_log", _untile(torch.log(-float_tensor(t(f"{blk}.ssm_a")).float()), nk, 1)+ yield f"{la}.norm.weight", float_tensor(t(f"{blk}.ssm_norm"))+ yield from fused(f"{la}.out_proj", [t(f"{blk}.ssm_out")], [matrix(t(f"{blk}.ssm_out"))], g.num_value_heads * g.value_head_dim)+ else:+ sa = f"{pre}.self_attn"+ parts = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]+ yield from fused(f"{sa}.qkv_proj", parts, [matrix(p) for p in parts], hidden)+ out = t(f"{blk}.attn_output")+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))+ yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))+ yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))++ mlp = f"{pre}.mlp"+ if config.moe_enabled:+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))+ sg, su, sd = (t(f"{blk}.ffn_{p}_shexp") for p in ("gate", "up", "down"))+ yield from fused(f"{mlp}.shared_expert.gate_up_proj", [sg, su], [matrix(sg), matrix(su)], hidden)+ yield from fused(f"{mlp}.shared_expert.down_proj", [sd], [matrix(sd)], config.shared_expert_intermediate_size)+ yield f"{mlp}.shared_expert_gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp_shexp")).reshape(1, -1)+ else:+ fg, fu, fd = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))+ yield from fused(f"{mlp}.gate_up_proj", [fg, fu], [matrix(fg), matrix(fu)], hidden)+ yield from fused(f"{mlp}.down_proj", [fd], [matrix(fd)], config.intermediate_size)+++__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]diff --git a/python/freetoken/models/qwen3_5_moe/model.py b/python/freetoken/models/qwen3_5_moe/model.pyindex 8cdd41ae..3fe373ca 100644--- a/python/freetoken/models/qwen3_5_moe/model.py+++ b/python/freetoken/models/qwen3_5_moe/model.py@@ -46,6 +46,8 @@ class Qwen3_5DecoderLayer(BaseOP):layer_id=layer_id,quant_config=config.quant,prefix=f"{prefix}.linear_attn",+ # llama.cpp stores every qwen35 ssm_out with tiled value-head columns+ out_heads_tiled=config.quant is not None and config.quant.dialect == "gguf",)else:self.self_attn = Qwen3_5Attention(config, layer_id, prefix=f"{prefix}.self_attn")@@ -79,6 +81,8 @@ class Qwen3_5Model(BaseOP):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,+ quant_config=config.quant,+ prefix=f"{prefix}.embed_tokens",)self.layers = OPList([diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 4b0cea25..86e3a734 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -201,6 +201,25 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {packed_modules_mapping=_QWEN3_5_PACKED,unquantized_modules=_QWEN3_5_UNQUANTIZED,),+ # llama.cpp qwen35 / qwen35moe GGUF (text only): the same model classes with every tensor in its packed ggml type+ "Qwen3_5GGUFForCausalLM": ModelSpec(+ "freetoken.models.qwen3_5_moe",+ "Qwen3_5ForCausalLM",+ parse_config="parse_gguf_config",+ iter_weights="iter_gguf_weights",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ packed_modules_mapping=_QWEN3_5_PACKED,+ unquantized_modules=_QWEN3_5_UNQUANTIZED,+ ),+ "Qwen3_5MoeGGUFForCausalLM": ModelSpec(+ "freetoken.models.qwen3_5_moe",+ "Qwen3_5MoeForCausalLM",+ parse_config="parse_gguf_config",+ iter_weights="iter_gguf_weights",+ checkpoint_roots=_LANGUAGE_MODEL_ROOT,+ packed_modules_mapping=_QWEN3_5_PACKED,+ unquantized_modules=_QWEN3_5_UNQUANTIZED,+ ),# Qwen3.8-Flash-Next (model_type qwen4_exp): multimodal wrapper config (text tower in# text_config, weights under model.language_model.). 36 GDN + 12 QSA# compressed-sparse attention layers on 4 hyper-connection residual streams, a PLE