FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(gguf): serve GLM-4.5, GLM-4.6 and GLM-4.7 GGUF checkpoints (#731)
Xiaoze Fan6b8f2004547d
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex f0b70e20..1c367aa4 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -253,6 +253,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (top_k=8,moe_intermediate_size=1536,expert_formats=("bf16",),+ arch_aliases=("Glm4MoeGGUFForCausalLM",),),AotModel(name="nvidia/GLM-4.7-NVFP4",diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.pyindex 677adff6..5b780fe0 100644--- a/python/freetoken/models/gguf/config.py+++ b/python/freetoken/models/gguf/config.py@@ -18,6 +18,7 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names# reuses the model classes but a GGUF parse_config / iter_weights).GGUF_ARCH_TO_REGISTRY: dict[str, str] = {"gemma4": "Gemma4GGUFForCausalLM",+ "glm4moe": "Glm4MoeGGUFForCausalLM","llama": "LlamaGGUFForCausalLM","minimax-m2": "MiniMaxM2GGUFForCausalLM","minimax-m3": "MiniMaxM3GGUFForCausalLM",diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.pyindex 7f0bf29a..3945c05d 100644--- a/python/freetoken/models/gguf/tokenizer.py+++ b/python/freetoken/models/gguf/tokenizer.py@@ -22,8 +22,8 @@ _BOS_PRE = frozenset(("llama3", "llama-v3", "llama-bpe", "falcon3", "falcon-h1", "pixtral", "midm-2.0", "lfm2", "jina-v5-nano", "tekken", "chameleon",))-# Llama 3 pre-tokenizers, whose HF tokenizer and llama.cpp take a word that is one vocabulary token as is instead of merging it-_IGNORE_MERGES_PRE = frozenset(("llama3", "llama-v3", "llama-bpe"))+# Llama 3 and GLM pre-tokenizers, whose HF tokenizer and llama.cpp take a word that is one vocabulary token as is instead of merging it+_IGNORE_MERGES_PRE = frozenset(("llama3", "llama-v3", "llama-bpe", "glm4"))def _add_bos(meta: dict) -> bool:@@ -80,7 +80,17 @@ def load_gguf_tokenizer(model_path: str):fast, _ = convert_gguf_tokenizer(arch, tok_dict)meta = load_gguf_metadata(model_path)fast = _fix_added_tokens(fast, meta)- if meta.get("tokenizer.ggml.model") == "gpt2" and meta.get("tokenizer.ggml.pre") in _IGNORE_MERGES_PRE:+ pre = meta.get("tokenizer.ggml.pre") if meta.get("tokenizer.ggml.model") == "gpt2" else None+ if pre == "glm4":+ from tokenizers import Regex, pre_tokenizers+ from transformers.integrations.gguf.gguf_tokenizer_mapping import GGUF_PRE_TOKENIZER_SPLITS++ # transformers maps no split for glm4, which the HF tokenizer and llama.cpp split like Llama 3+ fast.pre_tokenizer = pre_tokenizers.Sequence([+ pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"),+ pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),+ ])+ if pre in _IGNORE_MERGES_PRE:fast.model.ignore_merges = True# special: bos/eos/unk/pad as the file names them (None when absent) and the chat templatereturn PreTrainedTokenizerFast(diff --git a/python/freetoken/models/glm4_moe/__init__.py b/python/freetoken/models/glm4_moe/__init__.pyindex b8748b1d..7f093c67 100644--- a/python/freetoken/models/glm4_moe/__init__.py+++ b/python/freetoken/models/glm4_moe/__init__.py@@ -1,4 +1,5 @@from .config import parse_config+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguffrom .model import Glm4MoeForCausalLMfrom .weight import iter_weights, nvfp4_expert_spec@@ -7,4 +8,7 @@ __all__ = ["Glm4MoeForCausalLM","parse_config","iter_weights",+ "iter_gguf_weights",+ "parse_gguf_config",+ "to_gguf",]diff --git a/python/freetoken/models/glm4_moe/attention.py b/python/freetoken/models/glm4_moe/attention.pyindex 7a4b335e..65b70bf1 100644--- a/python/freetoken/models/glm4_moe/attention.py+++ b/python/freetoken/models/glm4_moe/attention.py@@ -3,7 +3,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGfrom freetoken.core import get_global_ctx-from freetoken.layers import BaseOP, RMSNorm+from freetoken.layers import BaseOP, LinearReplicated, RMSNormfrom freetoken.layers.rotary import get_ropefrom freetoken.utils import nvtx_annotate@@ -27,9 +27,10 @@ class Glm4MoeAttention(BaseOP):every ``self_attn`` out of quantization (qkvo stay bf16), so we reproduce that exactly bystoring the bf16 weights compressed (~10.7 bits/weight) and decoding them bit-for-bit inthe forward. This fits a 32 GB VRAM target without the accuracy loss fp8 attention would add.+ From a GGUF file the projections are the file's packed ggml linears instead."""- def __init__(self, config: ModelConfig, layer_id: int):+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):head_dim = config.head_dimself.layer_id = layer_idself.num_qo_heads = config.num_qo_heads@@ -38,10 +39,17 @@ class Glm4MoeAttention(BaseOP):self.kv_attn_dim = self.num_kv_heads * head_dimself.head_dim = head_dim- self.q_proj = LinearDF11(config.hidden_size, self.qo_attn_dim, has_bias=config.has_attn_bias)- self.k_proj = LinearDF11(config.hidden_size, self.kv_attn_dim, has_bias=config.has_attn_bias)- self.v_proj = LinearDF11(config.hidden_size, self.kv_attn_dim, has_bias=config.has_attn_bias)- self.o_proj = LinearDF11(self.qo_attn_dim, config.hidden_size, has_bias=False)+ gguf = config.quant is not None and config.quant.dialect == "gguf"++ def linear(in_features: int, out_features: int, has_bias: bool, name: str) -> BaseOP:+ if gguf:+ return LinearReplicated(in_features, out_features, has_bias, quant_config=config.quant, prefix=f"{prefix}.{name}")+ return LinearDF11(in_features, out_features, has_bias=has_bias)++ self.q_proj = linear(config.hidden_size, self.qo_attn_dim, config.has_attn_bias, "q_proj")+ self.k_proj = linear(config.hidden_size, self.kv_attn_dim, config.has_attn_bias, "k_proj")+ self.v_proj = linear(config.hidden_size, self.kv_attn_dim, config.has_attn_bias, "v_proj")+ self.o_proj = linear(self.qo_attn_dim, config.hidden_size, False, "o_proj")if config.use_qk_norm:self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)diff --git a/python/freetoken/models/glm4_moe/gguf.py b/python/freetoken/models/glm4_moe/gguf.pynew file mode 100644index 00000000..a477706b--- /dev/null+++ b/python/freetoken/models/glm4_moe/gguf.py@@ -0,0 +1,179 @@+"""GLM-4.5 / 4.6 / 4.7 from a llama.cpp ``glm4moe`` GGUF: config from the GGUF metadata, tensors by name; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``).++The llama.cpp converter (conversion/glm.py) only renames the tensors and stacks the experts: both sides run NeoX rope on the first ``rope.dimension_count`` dims of a head, so q/k carry no rope permute.+Files that keep the MTP layer append it as the last block, which is not loaded.+"""++from __future__ import annotations++import re+from dataclasses import replace+from types import SimpleNamespace+from typing import TYPE_CHECKING, Iterator++import torch++from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index+from freetoken.models.gguf.tensors import float_tensor, fused, matrix++from .config import parse_config++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+ from freetoken.models.gguf.config import GgufConfigShim++_ARCH = "glm4moe"+_LAYER_LEAVES = {+ "self_attn.q_proj": "attn_q",+ "self_attn.k_proj": "attn_k",+ "self_attn.v_proj": "attn_v",+ "self_attn.o_proj": "attn_output",+ "mlp.gate_proj": "ffn_gate",+ "mlp.up_proj": "ffn_up",+ "mlp.down_proj": "ffn_down",+ "mlp.gate": "ffn_gate_inp",+ "mlp.shared_experts.gate_proj": "ffn_gate_shexp",+ "mlp.shared_experts.up_proj": "ffn_up_shexp",+ "mlp.shared_experts.down_proj": "ffn_down_shexp",+ "mlp.experts.0.gate_proj": "ffn_gate_exps",+ "mlp.experts.0.up_proj": "ffn_up_exps",+ "mlp.experts.0.down_proj": "ffn_down_exps",+}+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")+# llama.cpp's LLAMA_EXPERT_GATING_FUNC_TYPE_*: 0 (unset) means sigmoid for this arch+_SIGMOID_GATING = (0, 2)+++def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""+ if name in _TOP_LEAVES:+ return (_TOP_LEAVES[name],)+ m = _LAYER_RE.match(name)+ if m is None:+ return ()+ tensor = _LAYER_LEAVES.get(m.group(2))+ return (f"blk.{m.group(1)}.{tensor}",) if tensor is not None else ()+++def _check_supported(shim: GgufConfigShim) -> None:+ md = shim.metadata+ if any(k.startswith("prism.hadamard.") for k in md):+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")+ names = sorted(gguf_tensor_index(shim.model_path))+ if not names:+ raise NotImplementedError(f"{shim.model_path}: a metadata-only glm4moe GGUF has no tensor table to tell whether the attention has q/k norms and biases")+ if "blk.0.attn_q.weight" not in names:+ raise NotImplementedError(f"{shim.model_path}: no decoder layer tensors (blk.0.attn_q), e.g. an MTP-only draft GGUF")+ scaled = [n for n in names if n.endswith((".scale", ".input_scale"))]+ if scaled:+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")+ fused_qkv = [n for n in names if ".attn_qkv." in n]+ if fused_qkv:+ raise NotImplementedError(f"{shim.model_path}: a fused attn_qkv glm4moe GGUF is not supported (e.g. {fused_qkv[0]})")+ gating = int(md.get(f"{_ARCH}.expert_gating_func", 0))+ if gating not in _SIGMOID_GATING:+ raise NotImplementedError(f"{shim.model_path}: expert gating function {gating} is not supported, GLM routes with sigmoid")+ scaling = md.get(f"{_ARCH}.rope.scaling.type", "none")+ if scaling != "none":+ raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported")+++def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:+ """The HF config fields ``parse_config`` reads, from the GGUF metadata."""+ md = shim.metadata++ def kv(key: str):+ return md[f"{_ARCH}.{key}"]++ _check_supported(shim)+ index = gguf_tensor_index(shim.model_path)+ head_dim = int(kv("attention.key_length"))+ return SimpleNamespace(+ model_type="glm4_moe",+ architectures=list(shim.architectures),+ hidden_act="silu",+ hidden_size=int(kv("embedding_length")),+ # block_count also counts the MTP layers a GGUF may carry after the decoder+ num_hidden_layers=int(kv("block_count")) - int(md.get(f"{_ARCH}.nextn_predict_layers", 0)),+ num_attention_heads=int(kv("attention.head_count")),+ num_key_value_heads=int(kv("attention.head_count_kv")),+ head_dim=head_dim,+ rope_theta=float(kv("rope.freq_base")),+ partial_rotary_factor=int(kv("rope.dimension_count")) / head_dim,+ max_position_embeddings=int(kv("context_length")),+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),+ vocab_size=int(shim.vocab_size),+ tie_word_embeddings=bool(shim.tie_word_embeddings),+ intermediate_size=int(kv("feed_forward_length")),+ n_routed_experts=int(kv("expert_count")),+ num_experts_per_tok=int(kv("expert_used_count")),+ moe_intermediate_size=int(kv("expert_feed_forward_length")),+ n_shared_experts=int(kv("expert_shared_count")),+ first_k_dense_replace=int(md.get(f"{_ARCH}.leading_dense_block_count", 0)),+ # llama.cpp neither renormalizes nor scales the router weights when the keys are absent+ norm_topk_prob=bool(md.get(f"{_ARCH}.expert_weights_norm", False)),+ routed_scaling_factor=float(md.get(f"{_ARCH}.expert_weights_scale", 0.0)) or 1.0,+ n_group=max(1, int(md.get(f"{_ARCH}.expert_group_count", 1))),+ topk_group=max(1, int(md.get(f"{_ARCH}.expert_group_used_count", 1))),+ # GLM-4.5-Air has no q/k norms; the converter writes no flag for them or for the qkv biases+ use_qk_norm="blk.0.attn_q_norm.weight" in index,+ attention_bias="blk.0.attn_q.bias" in index,+ )+++def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:+ return replace(parse_config(_hf_config(shim)), expert_quant="gguf")+++def iter_gguf_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+) -> Iterator[tuple[str, torch.Tensor]]:+ """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""+ if not include_non_moe:+ return+ from freetoken.utils import cached_load_hf_config++ hf = _hf_config(cached_load_hf_config(model_path))+ index = gguf_tensor_index(model_path)++ def t(name: str) -> GgufTensorInfo:+ return index[name if name in index else f"{name}.weight"]++ def linear(module: str, tensor: str) -> Iterator[tuple[str, torch.Tensor]]:+ info = t(tensor)+ yield from fused(module, [info], [matrix(info)], int(info.ne[0]))+ if f"{tensor}.bias" in index:+ yield f"{module}.bias", float_tensor(index[f"{tensor}.bias"])++ yield from linear("model.embed_tokens", "token_embd")+ if not hf.tie_word_embeddings:+ yield from linear("lm_head", "output")+ yield "model.norm.weight", float_tensor(t("output_norm"))++ for layer in range(hf.num_hidden_layers):+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.post_attention_norm"))+ sa = f"{pre}.self_attn"+ for proj, tensor in (("q_proj", "attn_q"), ("k_proj", "attn_k"), ("v_proj", "attn_v"), ("o_proj", "attn_output")):+ yield from linear(f"{sa}.{proj}", f"{blk}.{tensor}")+ if hf.use_qk_norm:+ yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))+ yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))+ mlp = f"{pre}.mlp"+ if layer < hf.first_k_dense_replace:+ for proj in ("gate", "up", "down"):+ yield from linear(f"{mlp}.{proj}_proj", f"{blk}.ffn_{proj}")+ continue+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))+ yield f"{mlp}.e_score_correction_bias", float_tensor(t(f"{blk}.exp_probs_b.bias"))+ for proj in ("gate", "up", "down"):+ yield from linear(f"{mlp}.shared_experts.{proj}_proj", f"{blk}.ffn_{proj}_shexp")+++__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]diff --git a/python/freetoken/models/glm4_moe/model.py b/python/freetoken/models/glm4_moe/model.pyindex b5c647e1..8824dfb5 100644--- a/python/freetoken/models/glm4_moe/model.py+++ b/python/freetoken/models/glm4_moe/model.py@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Tupleimport torchfrom freetoken.core import get_global_ctx-from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNormFused+from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNormFused, VocabParallelEmbeddingfrom freetoken.models.blocks import BaseLLMModelfrom freetoken.utils import nvtx_annotate@@ -19,7 +19,7 @@ if TYPE_CHECKING:class Glm4MoeDecoderLayer(BaseOP):def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):- self.self_attn = Glm4MoeAttention(config, layer_id)+ self.self_attn = Glm4MoeAttention(config, layer_id, prefix=f"{prefix}.self_attn")if layer_id >= config.first_k_dense_replace:self.mlp: BaseOP = Glm4MoeSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")else:@@ -45,10 +45,15 @@ class Glm4MoeDecoderLayer(BaseOP):class Glm4MoeModel(BaseOP):def __init__(self, config: ModelConfig, *, prefix: str = "model"):- self.embed_tokens = EmbeddingDF11(- num_embeddings=config.vocab_size,- embedding_dim=config.hidden_size,- )+ if config.quant is not None and config.quant.dialect == "gguf":+ self.embed_tokens: BaseOP = VocabParallelEmbedding(+ config.vocab_size, config.hidden_size, quant_config=config.quant, prefix=f"{prefix}.embed_tokens"+ )+ else:+ self.embed_tokens = EmbeddingDF11(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ )self.layers = OPList([Glm4MoeDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex bfa0a1d6..858e17db 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -387,6 +387,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"Glm4MoeForCausalLM",packed_modules_mapping=_EXPERTS_PACKED,),+ # llama.cpp glm4moe GGUF (GLM-4.5, 4.5-Air, 4.6, 4.7): the same model classes with every tensor in its packed ggml type+ "Glm4MoeGGUFForCausalLM": ModelSpec(+ "freetoken.models.glm4_moe",+ "Glm4MoeForCausalLM",+ parse_config="parse_gguf_config",+ iter_weights="iter_gguf_weights",+ packed_modules_mapping=_EXPERTS_PACKED,+ ),# GLM-5.2 (model_type glm_moe_dsa): DeepSeek-V3.2-class MLA + DSA sparse attention# with GLM-4-style sigmoid/noaux_tc MoE routing; NVFP4 routed experts served from# the offload cache.