Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): serve GLM-4.5, GLM-4.6 and GLM-4.7 GGUF checkpoints (#731)

Xiaoze Fan6b8f2004547d
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index f0b70e20..1c367aa4 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -253,6 +253,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
top_k=8,
moe_intermediate_size=1536,
expert_formats=("bf16",),
+ arch_aliases=("Glm4MoeGGUFForCausalLM",),
),
AotModel(
name="nvidia/GLM-4.7-NVFP4",
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index 677adff6..5b780fe0 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -18,6 +18,7 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names
# reuses the model classes but a GGUF parse_config / iter_weights).
GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
"gemma4": "Gemma4GGUFForCausalLM",
+ "glm4moe": "Glm4MoeGGUFForCausalLM",
"llama": "LlamaGGUFForCausalLM",
"minimax-m2": "MiniMaxM2GGUFForCausalLM",
"minimax-m3": "MiniMaxM3GGUFForCausalLM",
diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py
index 7f0bf29a..3945c05d 100644
--- a/python/freetoken/models/gguf/tokenizer.py
+++ b/python/freetoken/models/gguf/tokenizer.py
@@ -22,8 +22,8 @@ _BOS_PRE = frozenset((
"llama3", "llama-v3", "llama-bpe", "falcon3", "falcon-h1", "pixtral", "midm-2.0", "lfm2", "jina-v5-nano", "tekken", "chameleon",
))
-# Llama 3 pre-tokenizers, whose HF tokenizer and llama.cpp take a word that is one vocabulary token as is instead of merging it
-_IGNORE_MERGES_PRE = frozenset(("llama3", "llama-v3", "llama-bpe"))
+# Llama 3 and GLM pre-tokenizers, whose HF tokenizer and llama.cpp take a word that is one vocabulary token as is instead of merging it
+_IGNORE_MERGES_PRE = frozenset(("llama3", "llama-v3", "llama-bpe", "glm4"))
def _add_bos(meta: dict) -> bool:
@@ -80,7 +80,17 @@ def load_gguf_tokenizer(model_path: str):
fast, _ = convert_gguf_tokenizer(arch, tok_dict)
meta = load_gguf_metadata(model_path)
fast = _fix_added_tokens(fast, meta)
- if meta.get("tokenizer.ggml.model") == "gpt2" and meta.get("tokenizer.ggml.pre") in _IGNORE_MERGES_PRE:
+ pre = meta.get("tokenizer.ggml.pre") if meta.get("tokenizer.ggml.model") == "gpt2" else None
+ if pre == "glm4":
+ from tokenizers import Regex, pre_tokenizers
+ from transformers.integrations.gguf.gguf_tokenizer_mapping import GGUF_PRE_TOKENIZER_SPLITS
+
+ # transformers maps no split for glm4, which the HF tokenizer and llama.cpp split like Llama 3
+ fast.pre_tokenizer = pre_tokenizers.Sequence([
+ pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"),
+ pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
+ ])
+ if pre in _IGNORE_MERGES_PRE:
fast.model.ignore_merges = True
# special: bos/eos/unk/pad as the file names them (None when absent) and the chat template
return PreTrainedTokenizerFast(
diff --git a/python/freetoken/models/glm4_moe/__init__.py b/python/freetoken/models/glm4_moe/__init__.py
index b8748b1d..7f093c67 100644
--- a/python/freetoken/models/glm4_moe/__init__.py
+++ b/python/freetoken/models/glm4_moe/__init__.py
@@ -1,4 +1,5 @@
from .config import parse_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import Glm4MoeForCausalLM
from .weight import iter_weights, nvfp4_expert_spec
@@ -7,4 +8,7 @@ __all__ = [
"Glm4MoeForCausalLM",
"parse_config",
"iter_weights",
+ "iter_gguf_weights",
+ "parse_gguf_config",
+ "to_gguf",
]
diff --git a/python/freetoken/models/glm4_moe/attention.py b/python/freetoken/models/glm4_moe/attention.py
index 7a4b335e..65b70bf1 100644
--- a/python/freetoken/models/glm4_moe/attention.py
+++ b/python/freetoken/models/glm4_moe/attention.py
@@ -3,7 +3,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from freetoken.core import get_global_ctx
-from freetoken.layers import BaseOP, RMSNorm
+from freetoken.layers import BaseOP, LinearReplicated, RMSNorm
from freetoken.layers.rotary import get_rope
from freetoken.utils import nvtx_annotate
@@ -27,9 +27,10 @@ class Glm4MoeAttention(BaseOP):
every ``self_attn`` out of quantization (qkvo stay bf16), so we reproduce that exactly by
storing the bf16 weights compressed (~10.7 bits/weight) and decoding them bit-for-bit in
the forward. This fits a 32 GB VRAM target without the accuracy loss fp8 attention would add.
+ From a GGUF file the projections are the file's packed ggml linears instead.
"""
- def __init__(self, config: ModelConfig, layer_id: int):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
head_dim = config.head_dim
self.layer_id = layer_id
self.num_qo_heads = config.num_qo_heads
@@ -38,10 +39,17 @@ class Glm4MoeAttention(BaseOP):
self.kv_attn_dim = self.num_kv_heads * head_dim
self.head_dim = head_dim
- self.q_proj = LinearDF11(config.hidden_size, self.qo_attn_dim, has_bias=config.has_attn_bias)
- self.k_proj = LinearDF11(config.hidden_size, self.kv_attn_dim, has_bias=config.has_attn_bias)
- self.v_proj = LinearDF11(config.hidden_size, self.kv_attn_dim, has_bias=config.has_attn_bias)
- self.o_proj = LinearDF11(self.qo_attn_dim, config.hidden_size, has_bias=False)
+ gguf = config.quant is not None and config.quant.dialect == "gguf"
+
+ def linear(in_features: int, out_features: int, has_bias: bool, name: str) -> BaseOP:
+ if gguf:
+ return LinearReplicated(in_features, out_features, has_bias, quant_config=config.quant, prefix=f"{prefix}.{name}")
+ return LinearDF11(in_features, out_features, has_bias=has_bias)
+
+ self.q_proj = linear(config.hidden_size, self.qo_attn_dim, config.has_attn_bias, "q_proj")
+ self.k_proj = linear(config.hidden_size, self.kv_attn_dim, config.has_attn_bias, "k_proj")
+ self.v_proj = linear(config.hidden_size, self.kv_attn_dim, config.has_attn_bias, "v_proj")
+ self.o_proj = linear(self.qo_attn_dim, config.hidden_size, False, "o_proj")
if config.use_qk_norm:
self.q_norm = RMSNorm(head_dim, eps=config.rms_norm_eps)
diff --git a/python/freetoken/models/glm4_moe/gguf.py b/python/freetoken/models/glm4_moe/gguf.py
new file mode 100644
index 00000000..a477706b
--- /dev/null
+++ b/python/freetoken/models/glm4_moe/gguf.py
@@ -0,0 +1,179 @@
+"""GLM-4.5 / 4.6 / 4.7 from a llama.cpp ``glm4moe`` GGUF: config from the GGUF metadata, tensors by name; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``).
+
+The llama.cpp converter (conversion/glm.py) only renames the tensors and stacks the experts: both sides run NeoX rope on the first ``rope.dimension_count`` dims of a head, so q/k carry no rope permute.
+Files that keep the MTP layer append it as the last block, which is not loaded.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Iterator
+
+import torch
+
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index
+from freetoken.models.gguf.tensors import float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+ from freetoken.models.gguf.config import GgufConfigShim
+
+_ARCH = "glm4moe"
+_LAYER_LEAVES = {
+ "self_attn.q_proj": "attn_q",
+ "self_attn.k_proj": "attn_k",
+ "self_attn.v_proj": "attn_v",
+ "self_attn.o_proj": "attn_output",
+ "mlp.gate_proj": "ffn_gate",
+ "mlp.up_proj": "ffn_up",
+ "mlp.down_proj": "ffn_down",
+ "mlp.gate": "ffn_gate_inp",
+ "mlp.shared_experts.gate_proj": "ffn_gate_shexp",
+ "mlp.shared_experts.up_proj": "ffn_up_shexp",
+ "mlp.shared_experts.down_proj": "ffn_down_shexp",
+ "mlp.experts.0.gate_proj": "ffn_gate_exps",
+ "mlp.experts.0.up_proj": "ffn_up_exps",
+ "mlp.experts.0.down_proj": "ffn_down_exps",
+}
+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}
+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+# llama.cpp's LLAMA_EXPERT_GATING_FUNC_TYPE_*: 0 (unset) means sigmoid for this arch
+_SIGMOID_GATING = (0, 2)
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name in _TOP_LEAVES:
+ return (_TOP_LEAVES[name],)
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ tensor = _LAYER_LEAVES.get(m.group(2))
+ return (f"blk.{m.group(1)}.{tensor}",) if tensor is not None else ()
+
+
+def _check_supported(shim: GgufConfigShim) -> None:
+ md = shim.metadata
+ if any(k.startswith("prism.hadamard.") for k in md):
+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
+ names = sorted(gguf_tensor_index(shim.model_path))
+ if not names:
+ raise NotImplementedError(f"{shim.model_path}: a metadata-only glm4moe GGUF has no tensor table to tell whether the attention has q/k norms and biases")
+ if "blk.0.attn_q.weight" not in names:
+ raise NotImplementedError(f"{shim.model_path}: no decoder layer tensors (blk.0.attn_q), e.g. an MTP-only draft GGUF")
+ scaled = [n for n in names if n.endswith((".scale", ".input_scale"))]
+ if scaled:
+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")
+ fused_qkv = [n for n in names if ".attn_qkv." in n]
+ if fused_qkv:
+ raise NotImplementedError(f"{shim.model_path}: a fused attn_qkv glm4moe GGUF is not supported (e.g. {fused_qkv[0]})")
+ gating = int(md.get(f"{_ARCH}.expert_gating_func", 0))
+ if gating not in _SIGMOID_GATING:
+ raise NotImplementedError(f"{shim.model_path}: expert gating function {gating} is not supported, GLM routes with sigmoid")
+ scaling = md.get(f"{_ARCH}.rope.scaling.type", "none")
+ if scaling != "none":
+ raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported")
+
+
+def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
+ """The HF config fields ``parse_config`` reads, from the GGUF metadata."""
+ md = shim.metadata
+
+ def kv(key: str):
+ return md[f"{_ARCH}.{key}"]
+
+ _check_supported(shim)
+ index = gguf_tensor_index(shim.model_path)
+ head_dim = int(kv("attention.key_length"))
+ return SimpleNamespace(
+ model_type="glm4_moe",
+ architectures=list(shim.architectures),
+ hidden_act="silu",
+ hidden_size=int(kv("embedding_length")),
+ # block_count also counts the MTP layers a GGUF may carry after the decoder
+ num_hidden_layers=int(kv("block_count")) - int(md.get(f"{_ARCH}.nextn_predict_layers", 0)),
+ num_attention_heads=int(kv("attention.head_count")),
+ num_key_value_heads=int(kv("attention.head_count_kv")),
+ head_dim=head_dim,
+ rope_theta=float(kv("rope.freq_base")),
+ partial_rotary_factor=int(kv("rope.dimension_count")) / head_dim,
+ max_position_embeddings=int(kv("context_length")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ vocab_size=int(shim.vocab_size),
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ intermediate_size=int(kv("feed_forward_length")),
+ n_routed_experts=int(kv("expert_count")),
+ num_experts_per_tok=int(kv("expert_used_count")),
+ moe_intermediate_size=int(kv("expert_feed_forward_length")),
+ n_shared_experts=int(kv("expert_shared_count")),
+ first_k_dense_replace=int(md.get(f"{_ARCH}.leading_dense_block_count", 0)),
+ # llama.cpp neither renormalizes nor scales the router weights when the keys are absent
+ norm_topk_prob=bool(md.get(f"{_ARCH}.expert_weights_norm", False)),
+ routed_scaling_factor=float(md.get(f"{_ARCH}.expert_weights_scale", 0.0)) or 1.0,
+ n_group=max(1, int(md.get(f"{_ARCH}.expert_group_count", 1))),
+ topk_group=max(1, int(md.get(f"{_ARCH}.expert_group_used_count", 1))),
+ # GLM-4.5-Air has no q/k norms; the converter writes no flag for them or for the qkv biases
+ use_qk_norm="blk.0.attn_q_norm.weight" in index,
+ attention_bias="blk.0.attn_q.bias" in index,
+ )
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ return replace(parse_config(_hf_config(shim)), expert_quant="gguf")
+
+
+def iter_gguf_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""
+ if not include_non_moe:
+ return
+ from freetoken.utils import cached_load_hf_config
+
+ hf = _hf_config(cached_load_hf_config(model_path))
+ index = gguf_tensor_index(model_path)
+
+ def t(name: str) -> GgufTensorInfo:
+ return index[name if name in index else f"{name}.weight"]
+
+ def linear(module: str, tensor: str) -> Iterator[tuple[str, torch.Tensor]]:
+ info = t(tensor)
+ yield from fused(module, [info], [matrix(info)], int(info.ne[0]))
+ if f"{tensor}.bias" in index:
+ yield f"{module}.bias", float_tensor(index[f"{tensor}.bias"])
+
+ yield from linear("model.embed_tokens", "token_embd")
+ if not hf.tie_word_embeddings:
+ yield from linear("lm_head", "output")
+ yield "model.norm.weight", float_tensor(t("output_norm"))
+
+ for layer in range(hf.num_hidden_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.post_attention_norm"))
+ sa = f"{pre}.self_attn"
+ for proj, tensor in (("q_proj", "attn_q"), ("k_proj", "attn_k"), ("v_proj", "attn_v"), ("o_proj", "attn_output")):
+ yield from linear(f"{sa}.{proj}", f"{blk}.{tensor}")
+ if hf.use_qk_norm:
+ yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))
+ yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))
+ mlp = f"{pre}.mlp"
+ if layer < hf.first_k_dense_replace:
+ for proj in ("gate", "up", "down"):
+ yield from linear(f"{mlp}.{proj}_proj", f"{blk}.ffn_{proj}")
+ continue
+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))
+ yield f"{mlp}.e_score_correction_bias", float_tensor(t(f"{blk}.exp_probs_b.bias"))
+ for proj in ("gate", "up", "down"):
+ yield from linear(f"{mlp}.shared_experts.{proj}_proj", f"{blk}.ffn_{proj}_shexp")
+
+
+__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/glm4_moe/model.py b/python/freetoken/models/glm4_moe/model.py
index b5c647e1..8824dfb5 100644
--- a/python/freetoken/models/glm4_moe/model.py
+++ b/python/freetoken/models/glm4_moe/model.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Tuple
import torch
from freetoken.core import get_global_ctx
-from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNormFused
+from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNormFused, VocabParallelEmbedding
from freetoken.models.blocks import BaseLLMModel
from freetoken.utils import nvtx_annotate
@@ -19,7 +19,7 @@ if TYPE_CHECKING:
class Glm4MoeDecoderLayer(BaseOP):
def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
- self.self_attn = Glm4MoeAttention(config, layer_id)
+ self.self_attn = Glm4MoeAttention(config, layer_id, prefix=f"{prefix}.self_attn")
if layer_id >= config.first_k_dense_replace:
self.mlp: BaseOP = Glm4MoeSparseBlock(config, layer_id, prefix=f"{prefix}.mlp")
else:
@@ -45,10 +45,15 @@ class Glm4MoeDecoderLayer(BaseOP):
class Glm4MoeModel(BaseOP):
def __init__(self, config: ModelConfig, *, prefix: str = "model"):
- self.embed_tokens = EmbeddingDF11(
- num_embeddings=config.vocab_size,
- embedding_dim=config.hidden_size,
- )
+ if config.quant is not None and config.quant.dialect == "gguf":
+ self.embed_tokens: BaseOP = VocabParallelEmbedding(
+ config.vocab_size, config.hidden_size, quant_config=config.quant, prefix=f"{prefix}.embed_tokens"
+ )
+ else:
+ self.embed_tokens = EmbeddingDF11(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ )
self.layers = OPList(
[
Glm4MoeDecoderLayer(config, layer_id, prefix=f"{prefix}.layers.{layer_id}")
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index bfa0a1d6..858e17db 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -387,6 +387,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"Glm4MoeForCausalLM",
packed_modules_mapping=_EXPERTS_PACKED,
),
+ # llama.cpp glm4moe GGUF (GLM-4.5, 4.5-Air, 4.6, 4.7): the same model classes with every tensor in its packed ggml type
+ "Glm4MoeGGUFForCausalLM": ModelSpec(
+ "freetoken.models.glm4_moe",
+ "Glm4MoeForCausalLM",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ packed_modules_mapping=_EXPERTS_PACKED,
+ ),
# GLM-5.2 (model_type glm_moe_dsa): DeepSeek-V3.2-class MLA + DSA sparse attention
# with GLM-4-style sigmoid/noaux_tc MoE routing; NVFP4 routed experts served from
# the offload cache.