Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): serve Muse-Glimmer GGUF checkpoints (#724)

Xiaoze Fan5d39d6408a23
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index 040236cb..eca18d00 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -369,6 +369,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
hidden_size=6656,
kv_groups=((2, 128),), # sliding and full (NoPE) layers share the same geometry
aliases=("RedHatAI/Muse-Glimmer-30B-NVFP4",),
+ arch_aliases=("MuseGlimmerGGUFForCausalLM",),
),
AotModel(
name="meta-llama/Llama-3.1-8B-Instruct",
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index 9f860e66..6e31b8ff 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -19,6 +19,7 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names
GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
"gemma4": "Gemma4GGUFForCausalLM",
"llama": "LlamaGGUFForCausalLM",
+ "muse-glimmer": "MuseGlimmerGGUFForCausalLM",
"qwen2": "Qwen2GGUFForCausalLM",
"qwen3": "Qwen3GGUFForCausalLM",
"qwen3moe": "Qwen3MoeGGUFForCausalLM",
diff --git a/python/freetoken/models/muse_glimmer/__init__.py b/python/freetoken/models/muse_glimmer/__init__.py
index 4ee14808..ff8d3d17 100644
--- a/python/freetoken/models/muse_glimmer/__init__.py
+++ b/python/freetoken/models/muse_glimmer/__init__.py
@@ -1,5 +1,6 @@
from .attention import MuseGlimmerAttention
from .config import VisionConfig, parse_config, parse_vision_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import MuseGlimmerForCausalLM, MuseGlimmerForConditionalGeneration
from .vision import MuseGlimmerVisionModel
from .weight import iter_vision_weights, iter_weights
@@ -14,4 +15,7 @@ __all__ = [
"parse_vision_config",
"iter_vision_weights",
"iter_weights",
+ "iter_gguf_weights",
+ "parse_gguf_config",
+ "to_gguf",
]
diff --git a/python/freetoken/models/muse_glimmer/gguf.py b/python/freetoken/models/muse_glimmer/gguf.py
new file mode 100644
index 00000000..7d5cee3b
--- /dev/null
+++ b/python/freetoken/models/muse_glimmer/gguf.py
@@ -0,0 +1,184 @@
+"""Muse Glimmer text from a llama.cpp ``muse-glimmer`` GGUF: config from the GGUF metadata, tensors by name.
+
+The converter (conversion/muse_glimmer.py) adds 1 to the four norms of each block, permutes the rows of each q and k head for llama.cpp's interleaved rope, and writes constant q/k norms that carry ``qk_scale_factor``.
+The loader subtracts the 1, undoes the permute, and reads ``qk_scale_factor`` back from the q norm.
+"""
+
+from __future__ import annotations
+
+import re
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Iterator
+
+import torch
+
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index
+from freetoken.models.gguf.tensors import float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+ from freetoken.models.gguf.config import GgufConfigShim
+
+_ARCH = "muse-glimmer"
+_LAYER_LEAVES = {
+ "self_attn.q_proj": "attn_q",
+ "self_attn.k_proj": "attn_k",
+ "self_attn.v_proj": "attn_v",
+ "self_attn.gate_proj": "attn_gate",
+ "self_attn.o_proj": "attn_output",
+ "mlp.gate_proj": "ffn_gate",
+ "mlp.up_proj": "ffn_up",
+ "mlp.down_proj": "ffn_down",
+}
+_QKV_LEAVES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj")
+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}
+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+# checkpoint norm -> GGUF norm; the GGUF stores each as weight + 1
+_SHIFTED_NORMS = {
+ "input_layernorm": "attn_norm",
+ "post_attention_layernorm": "post_attention_norm",
+ "pre_feedforward_layernorm": "ffn_norm",
+ "post_feedforward_layernorm": "post_ffw_norm",
+}
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name in _TOP_LEAVES:
+ return (_TOP_LEAVES[name],)
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ blk, leaf = f"blk.{m.group(1)}", m.group(2)
+ if leaf in _QKV_LEAVES and f"{blk}.attn_qkv.weight" in tensors:
+ return (f"{blk}.attn_qkv",)
+ tensor = _LAYER_LEAVES.get(leaf)
+ return (f"{blk}.{tensor}",) if tensor is not None else ()
+
+
+def _check_supported(shim: GgufConfigShim) -> None:
+ index = gguf_tensor_index(shim.model_path)
+ if not index:
+ raise NotImplementedError(f"{shim.model_path}: a metadata-only muse-glimmer GGUF has no q norm to read qk_scale_factor from")
+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):
+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
+ scaled = sorted(n for n in index if n.endswith((".scale", ".input_scale")))
+ if scaled:
+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")
+
+
+def _qk_scale_factor(model_path: str, num_layers: int) -> float:
+ """The constant the converter wrote into every q norm; the k norms are all ones."""
+ index = gguf_tensor_index(model_path)
+ scale = float(float_tensor(index["blk.0.attn_q_norm.weight"])[0])
+ for layer in range(num_layers):
+ q, k = (float_tensor(index[f"blk.{layer}.attn_{p}_norm.weight"]).float() for p in ("q", "k"))
+ if not (torch.all(q == scale) and torch.all(k == 1.0)):
+ raise NotImplementedError(f"{model_path}: the q/k norms of layer {layer} are not the converter's constant qk_scale_factor / ones")
+ return scale
+
+
+def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
+ """The HF text config fields ``parse_config`` reads, from the GGUF metadata."""
+ md = shim.metadata
+
+ def kv(key: str):
+ return md[f"{_ARCH}.{key}"]
+
+ _check_supported(shim)
+ num_layers = int(kv("block_count"))
+ # a bool per layer, or llama.cpp's period: every n-th layer is full attention, and 0 means none is
+ pattern = md.get(f"{_ARCH}.attention.sliding_window_pattern", 4)
+ if isinstance(pattern, list):
+ swa = [bool(p) for p in pattern]
+ if len(swa) != num_layers:
+ raise ValueError(f"{shim.model_path}: sliding_window_pattern has {len(swa)} entries for {num_layers} layers")
+ else:
+ swa = [int(pattern) == 0 or i % int(pattern) < int(pattern) - 1 for i in range(num_layers)]
+ base = float(kv("rope.freq_base"))
+ swa_base = float(md.get(f"{_ARCH}.rope.freq_base_swa", base))
+ return SimpleNamespace(
+ model_type="muse_glimmer",
+ architectures=list(shim.architectures),
+ hidden_activation="silu",
+ hidden_size=int(kv("embedding_length")),
+ num_hidden_layers=num_layers,
+ num_attention_heads=int(kv("attention.head_count")),
+ num_key_value_heads=int(kv("attention.head_count_kv")),
+ head_dim=int(kv("attention.key_length")),
+ intermediate_size=int(kv("feed_forward_length")),
+ max_position_embeddings=int(kv("context_length")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ # llama.cpp hard-codes the post-norm eps and NoPE on the full layers (src/models/muse-glimmer.cpp)
+ post_norm_eps=1e-8,
+ layer_types=["sliding_attention" if s else "full_attention" for s in swa],
+ layer_rope_theta=[swa_base if s else 0.0 for s in swa],
+ sliding_window=int(kv("attention.sliding_window")),
+ qk_scale_factor=_qk_scale_factor(shim.model_path, num_layers),
+ # llama.cpp's default when the key is absent; 0 turns the cap off
+ final_logit_softcapping=float(md.get(f"{_ARCH}.final_logit_softcapping", 30.0)) or None,
+ output_multiplier=float(kv("logit_scale")),
+ vocab_size=int(shim.vocab_size),
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ )
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ return parse_config(_hf_config(shim))
+
+
+def _unpermute(rows: torch.Tensor, heads: int) -> torch.Tensor:
+ """Undo the converter's rope permute of one q or k matrix (``_unpermute_for_rope``); it moves whole rows, so packed ggml rows work too."""
+ return rows.reshape(heads, rows.shape[0] // heads // 2, 2, *rows.shape[1:]).swapaxes(1, 2).reshape(rows.shape)
+
+
+def iter_gguf_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """The text state dict from the GGUF."""
+ if not include_non_moe:
+ return
+ from freetoken.utils import cached_load_hf_config
+
+ hf = _hf_config(cached_load_hf_config(model_path))
+ index = gguf_tensor_index(model_path)
+ hidden, heads, kv_heads = hf.hidden_size, hf.num_attention_heads, hf.num_key_value_heads
+ q_rows, kv_rows = heads * hf.head_dim, kv_heads * hf.head_dim
+
+ def t(name: str) -> GgufTensorInfo:
+ return index[name if name in index else f"{name}.weight"]
+
+ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
+ if not hf.tie_word_embeddings:
+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
+ yield "model.norm.weight", float_tensor(t("output_norm"))
+
+ for layer in range(hf.num_hidden_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ for norm, tensor in _SHIFTED_NORMS.items():
+ yield f"{pre}.{norm}.weight", float_tensor(t(f"{blk}.{tensor}")).float() - 1.0
+ sa = f"{pre}.self_attn"
+ # convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrix
+ if f"{blk}.attn_qkv.weight" in index:
+ qkv = [t(f"{blk}.attn_qkv")] * 3
+ q, k, v = matrix(qkv[0]).split([q_rows, kv_rows, kv_rows])
+ else:
+ qkv = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
+ q, k, v = (matrix(p) for p in qkv)
+ gate = t(f"{blk}.attn_gate")
+ yield from fused(f"{sa}.qkvg_proj", [*qkv, gate], [_unpermute(q, heads), _unpermute(k, kv_heads), v, matrix(gate)], hidden)
+ out = t(f"{blk}.attn_output")
+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
+ mlp = f"{pre}.mlp"
+ gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))
+ yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)
+ yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))
+
+
+__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/muse_glimmer/model.py b/python/freetoken/models/muse_glimmer/model.py
index 50bd4506..d51c2117 100644
--- a/python/freetoken/models/muse_glimmer/model.py
+++ b/python/freetoken/models/muse_glimmer/model.py
@@ -105,6 +105,8 @@ class MuseGlimmerModel(BaseOP):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
+ quant_config=config.quant,
+ prefix=f"{prefix}.embed_tokens",
)
# NormedEmbedding: a weightless RMSNorm on top of the embeddings. Kept separate
# from the embedding matrix (the reference cannot fold it either -- the DFlash
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 057e2ae6..08b70e83 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -78,6 +78,10 @@ _GEMMA4_PROCESSOR = "freetoken.mm.processors.gemma4:Gemma4MMProcessor"
_GEMMA4_UNIFIED_PROCESSOR = "freetoken.mm.processors.gemma4:Gemma4UnifiedMMProcessor"
_GEMMA4_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)
_MUSE_GLIMMER_PROCESSOR = "freetoken.mm.processors.muse_glimmer:MuseGlimmerMMProcessor"
+_MUSE_GLIMMER_PACKED = (
+ ("qkvg_proj", ("q_proj", "k_proj", "v_proj", "gate_proj")),
+ ("gate_up_proj", ("gate_proj", "up_proj")),
+)
_MUSE_GLIMMER_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)
_MINIMAX_M3_PACKED = _DENSE_PACKED + (
("index_qk_proj", ("index_q_proj", "index_k_proj")),
@@ -293,13 +297,18 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"freetoken.models.muse_glimmer",
"MuseGlimmerForConditionalGeneration",
checkpoint_roots=_LANGUAGE_MODEL_ROOT,
- packed_modules_mapping=(
- ("qkvg_proj", ("q_proj", "k_proj", "v_proj", "gate_proj")),
- ("gate_up_proj", ("gate_proj", "up_proj")),
- ),
+ packed_modules_mapping=_MUSE_GLIMMER_PACKED,
mm_processor=_MUSE_GLIMMER_PROCESSOR,
encoders=_MUSE_GLIMMER_ENCODERS,
),
+ # llama.cpp muse-glimmer GGUF (text only): the same model classes with every tensor in its packed ggml type
+ "MuseGlimmerGGUFForCausalLM": ModelSpec(
+ "freetoken.models.muse_glimmer",
+ "MuseGlimmerForCausalLM",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ packed_modules_mapping=_MUSE_GLIMMER_PACKED,
+ ),
"MistralForCausalLM": ModelSpec(
"freetoken.models.mistral",
"MistralForCausalLM",