Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): serve qwen3vl and qwen3vlmoe gguf checkpoints text only (#807)

Mukkesh Ganeshed359eee0af1
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index 3773d86d..bf7e75b6 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -142,7 +142,8 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
moe_intermediate_size=768,
expert_formats=("bf16",),
aliases=("Qwen/Qwen3-30B-A3B-Thinking-2507",),
- arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM"), # Qwen3-VL-30B-A3B: same text tower
+ # Qwen3-VL-30B-A3B: same text tower
+ arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM", "Qwen3VLMoeGGUFForCausalLM"),
),
AotModel(
name="Qwen/Qwen3.5-35B-A3B",
@@ -395,7 +396,8 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
architecture="Qwen3ForCausalLM",
hidden_size=4096,
kv_groups=((8, 128),),
- arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM"), # Qwen3-VL-8B: same text tower
+ # Qwen3-VL: same text tower
+ arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM", "Qwen3VLGGUFForCausalLM"),
),
AotModel(
name="mistralai/Mistral-7B-Instruct-v0.3",
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index eb859abe..61198ead 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -35,6 +35,8 @@ GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
"qwen3moe": "Qwen3MoeGGUFForCausalLM",
"qwen35": "Qwen3_5GGUFForCausalLM",
"qwen35moe": "Qwen3_5MoeGGUFForCausalLM",
+ "qwen3vl": "Qwen3VLGGUFForCausalLM",
+ "qwen3vlmoe": "Qwen3VLMoeGGUFForCausalLM",
"qwen4exp": "Qwen4ExpGGUFForCausalLM",
}
# (GGUF architecture, mmproj projector) -> registry key, for text GGUFs whose release is told apart only by its mmproj
diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py
index 5d0ff489..a9c77d79 100644
--- a/python/freetoken/models/gguf/tokenizer.py
+++ b/python/freetoken/models/gguf/tokenizer.py
@@ -99,6 +99,12 @@ def load_gguf_tokenizer(model_path: str):
pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"),
pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
])
+ elif pre in ("qwen2", "qwen35"):
+ # every Qwen release's tokenizer.json applies NFC (Unicode Normalization Form C) before the
+ # byte-level BPE, but a GGUF carries no normalizer; without it decomposed text differs
+ from tokenizers import normalizers
+
+ fast.normalizer = normalizers.NFC()
elif pre == "joyai-llm":
from tokenizers import Regex, pre_tokenizers
diff --git a/python/freetoken/models/qwen3_vl/__init__.py b/python/freetoken/models/qwen3_vl/__init__.py
index 62640d43..2b311ba5 100644
--- a/python/freetoken/models/qwen3_vl/__init__.py
+++ b/python/freetoken/models/qwen3_vl/__init__.py
@@ -1,4 +1,5 @@
from .config import VisionConfig, parse_config, parse_vision_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import (
Qwen3VLForConditionalGeneration,
Qwen3VLMoeForConditionalGeneration,
@@ -13,10 +14,13 @@ __all__ = [
"Qwen3VLVisionModel",
"VisionConfig",
"deepstack_add",
+ "iter_gguf_weights",
"iter_vision_weights",
"iter_weights",
"iter_weights_parallel",
"parse_config",
+ "parse_gguf_config",
"parse_vision_config",
"rename_vl_prefix",
+ "to_gguf",
]
diff --git a/python/freetoken/models/qwen3_vl/gguf.py b/python/freetoken/models/qwen3_vl/gguf.py
index 42c40c2e..52f1edd7 100644
--- a/python/freetoken/models/qwen3_vl/gguf.py
+++ b/python/freetoken/models/qwen3_vl/gguf.py
@@ -1,4 +1,6 @@
-"""The Qwen VL vision tower from a llama.cpp ``qwen3vl_merger`` mmproj: the HF config keys, the image processor config, and the tower weights.
+"""Qwen3-VL GGUFs: the text trunks of llama.cpp ``qwen3vl`` / ``qwen3vlmoe`` files, and the Qwen VL vision tower from a ``qwen3vl_merger`` mmproj (the HF config keys, the image processor config, and the tower weights).
+
+The text trunk runs without its vision side: no tower and plain rope instead of interleaved mrope. llama.cpp's IMROPE keeps the standard full-spectrum theta (ggml.h: the dim index is not reset per section), so equal (t, h, w) text positions reduce to NeoX rope and q/k carry no permute, like the ``qwen3`` / ``qwen3moe`` GGUFs. The routed experts stack as ``ffn_*_exps`` because the converter splits HF's packed ``gate_up_proj`` / ``down_proj`` experts (conversion/qwen3vl.py).
llama.cpp splits the Conv3d patch embedding into one tensor per temporal slice (``v.patch_embd.weight`` and ``.weight.1``); everything else is stored in HF layout.
"""
@@ -6,12 +8,25 @@ llama.cpp splits the Conv3d patch embedding into one tensor per temporal slice (
from __future__ import annotations
import re
-from typing import Any, Iterator
+from dataclasses import replace
+from types import SimpleNamespace
+from typing import Any, TYPE_CHECKING, Iterator
import torch
from freetoken.models.gguf.mmproj import Mmproj, mmproj_tensor
-from freetoken.models.gguf.reader import gguf_tensor_index, load_gguf_metadata
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, load_gguf_metadata
+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+ from freetoken.models.gguf.config import GgufConfigShim
+
+# ----------------------------------------------------------------------------------------------
+# Vision tower (a qwen3vl_merger mmproj)
+# ----------------------------------------------------------------------------------------------
_PROJECTOR = "qwen3vl_merger"
_IMAGE_TOKEN = "<|image_pad|>"
@@ -111,4 +126,155 @@ def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:
yield f"visual.{_name(gguf_name)}", mmproj_tensor(info)
-__all__ = ["gguf_mm_config", "iter_mmproj_weights"]
+# ----------------------------------------------------------------------------------------------
+# Text trunk (a qwen3vl / qwen3vlmoe GGUF)
+# ----------------------------------------------------------------------------------------------
+
+_LAYER_LEAVES = {
+ "self_attn.q_proj": "attn_q",
+ "self_attn.k_proj": "attn_k",
+ "self_attn.v_proj": "attn_v",
+ "self_attn.o_proj": "attn_output",
+ "mlp.gate_proj": "ffn_gate",
+ "mlp.up_proj": "ffn_up",
+ "mlp.down_proj": "ffn_down",
+ "mlp.gate": "ffn_gate_inp",
+ "mlp.experts.0.gate_proj": "ffn_gate_exps",
+ "mlp.experts.0.up_proj": "ffn_up_exps",
+ "mlp.experts.0.down_proj": "ffn_down_exps",
+}
+_QKV_LEAVES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj")
+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}
+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name in _TOP_LEAVES:
+ return (_TOP_LEAVES[name],)
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ blk, leaf = f"blk.{m.group(1)}", m.group(2)
+ if leaf in _QKV_LEAVES and f"{blk}.attn_qkv.weight" in tensors:
+ return (f"{blk}.attn_qkv",)
+ if leaf in ("mlp.experts.0.gate_proj", "mlp.experts.0.up_proj") and f"{blk}.ffn_gate_up_exps.weight" in tensors:
+ return (f"{blk}.ffn_gate_up_exps",)
+ tensor = _LAYER_LEAVES.get(leaf)
+ return (f"{blk}.{tensor}",) if tensor is not None else ()
+
+
+def _check_supported(shim: GgufConfigShim) -> None:
+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):
+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
+ check_scales(shim.model_path)
+
+
+def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
+ """The HF config fields ``parse_config`` reads, from the GGUF metadata."""
+ md = shim.metadata
+ arch = shim.model_type # the GGUF architecture: qwen3vl or qwen3vlmoe
+ moe = arch.endswith("moe")
+
+ def kv(key: str):
+ return md[f"{arch}.{key}"]
+
+ _check_supported(shim)
+ rope = {"rope_type": "default", "rope_theta": float(kv("rope.freq_base"))}
+ scaling = md.get(f"{arch}.rope.scaling.type", "none")
+ if scaling == "yarn":
+ rope.update(
+ rope_type="yarn",
+ factor=float(kv("rope.scaling.factor")),
+ original_max_position_embeddings=int(kv("rope.scaling.original_context_length")),
+ )
+ elif scaling != "none":
+ raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported")
+ key_length = md.get(f"{arch}.attention.key_length")
+ text = SimpleNamespace(
+ model_type="qwen3_moe" if moe else "qwen3",
+ architectures=list(shim.architectures),
+ hidden_act="silu",
+ hidden_size=int(kv("embedding_length")),
+ num_hidden_layers=int(kv("block_count")),
+ num_attention_heads=int(kv("attention.head_count")),
+ num_key_value_heads=int(kv("attention.head_count_kv")),
+ head_dim=int(key_length) if key_length is not None else None,
+ rope_scaling=None if scaling == "none" else rope,
+ rope_parameters=rope,
+ max_position_embeddings=int(kv("context_length")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ vocab_size=int(shim.vocab_size),
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ intermediate_size=int(kv("feed_forward_length")),
+ )
+ if moe:
+ text.num_experts = int(kv("expert_count"))
+ text.num_experts_per_tok = int(kv("expert_used_count"))
+ text.moe_intermediate_size = int(kv("expert_feed_forward_length"))
+ # llama.cpp always renormalizes the top-k weights of a qwen3moe-style router
+ text.norm_topk_prob = True
+ else:
+ text.num_experts = 0
+ # text only: no vision_config, so the wrapper builds no tower and keeps 1-D rope
+ return SimpleNamespace(
+ text_config=text,
+ model_type="qwen3_vl",
+ architectures=list(shim.architectures),
+ image_token_id=None,
+ vision_config=None,
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ )
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ config = parse_config(_hf_config(shim))
+ return replace(config, expert_quant="gguf") if config.num_experts else config
+
+
+def iter_gguf_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""
+ if not include_non_moe:
+ return
+ from freetoken.utils import cached_load_hf_config
+
+ text = _hf_config(cached_load_hf_config(model_path)).text_config
+ index = gguf_tensor_index(model_path)
+ hidden = text.hidden_size
+
+ def t(name: str) -> GgufTensorInfo:
+ return index[name if name in index else f"{name}.weight"]
+
+ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
+ if not text.tie_word_embeddings:
+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
+ yield "model.norm.weight", float_tensor(t("output_norm"))
+
+ for layer in range(text.num_hidden_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))
+ sa = f"{pre}.self_attn"
+ # convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrix
+ qkv = [t(f"{blk}.attn_qkv")] if f"{blk}.attn_qkv.weight" in index else [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
+ yield from fused(f"{sa}.qkv_proj", qkv, [matrix(p) for p in qkv], hidden)
+ yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))
+ yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))
+ out = t(f"{blk}.attn_output")
+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
+ mlp = f"{pre}.mlp"
+ if text.num_experts:
+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))
+ else:
+ gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))
+ yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)
+ yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))
+
+
+__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index bbe34f06..ad8ade86 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -162,6 +162,21 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
encoders=_QWEN_VL_ENCODERS,
packed_modules_mapping=_DENSE_PACKED + _STACKED_EXPERTS_PACKED,
),
+ # llama.cpp qwen3vl / qwen3vlmoe GGUF (text only): the same model classes with every tensor in its packed ggml type
+ "Qwen3VLGGUFForCausalLM": ModelSpec(
+ "freetoken.models.qwen3_vl",
+ "Qwen3VLForConditionalGeneration",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ packed_modules_mapping=_DENSE_PACKED,
+ ),
+ "Qwen3VLMoeGGUFForCausalLM": ModelSpec(
+ "freetoken.models.qwen3_vl",
+ "Qwen3VLMoeForConditionalGeneration",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED,
+ ),
"MiniMaxM2ForCausalLM": ModelSpec(
"freetoken.models.minimax_m2",
"MiniMaxM2ForCausalLM",