FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(gguf): serve qwen3vl and qwen3vlmoe gguf checkpoints text only (#807)
Mukkesh Ganeshed359eee0af1
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex 3773d86d..bf7e75b6 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -142,7 +142,8 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (moe_intermediate_size=768,expert_formats=("bf16",),aliases=("Qwen/Qwen3-30B-A3B-Thinking-2507",),- arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM"), # Qwen3-VL-30B-A3B: same text tower+ # Qwen3-VL-30B-A3B: same text tower+ arch_aliases=("Qwen3VLMoeForConditionalGeneration", "Qwen3MoeGGUFForCausalLM", "Qwen3VLMoeGGUFForCausalLM"),),AotModel(name="Qwen/Qwen3.5-35B-A3B",@@ -395,7 +396,8 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (architecture="Qwen3ForCausalLM",hidden_size=4096,kv_groups=((8, 128),),- arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM"), # Qwen3-VL-8B: same text tower+ # Qwen3-VL: same text tower+ arch_aliases=("Qwen3VLForConditionalGeneration", "Qwen3GGUFForCausalLM", "Qwen3VLGGUFForCausalLM"),),AotModel(name="mistralai/Mistral-7B-Instruct-v0.3",diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.pyindex eb859abe..61198ead 100644--- a/python/freetoken/models/gguf/config.py+++ b/python/freetoken/models/gguf/config.py@@ -35,6 +35,8 @@ GGUF_ARCH_TO_REGISTRY: dict[str, str] = {"qwen3moe": "Qwen3MoeGGUFForCausalLM","qwen35": "Qwen3_5GGUFForCausalLM","qwen35moe": "Qwen3_5MoeGGUFForCausalLM",+ "qwen3vl": "Qwen3VLGGUFForCausalLM",+ "qwen3vlmoe": "Qwen3VLMoeGGUFForCausalLM","qwen4exp": "Qwen4ExpGGUFForCausalLM",}# (GGUF architecture, mmproj projector) -> registry key, for text GGUFs whose release is told apart only by its mmprojdiff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.pyindex 5d0ff489..a9c77d79 100644--- a/python/freetoken/models/gguf/tokenizer.py+++ b/python/freetoken/models/gguf/tokenizer.py@@ -99,6 +99,12 @@ def load_gguf_tokenizer(model_path: str):pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"),pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),])+ elif pre in ("qwen2", "qwen35"):+ # every Qwen release's tokenizer.json applies NFC (Unicode Normalization Form C) before the+ # byte-level BPE, but a GGUF carries no normalizer; without it decomposed text differs+ from tokenizers import normalizers++ fast.normalizer = normalizers.NFC()elif pre == "joyai-llm":from tokenizers import Regex, pre_tokenizersdiff --git a/python/freetoken/models/qwen3_vl/__init__.py b/python/freetoken/models/qwen3_vl/__init__.pyindex 62640d43..2b311ba5 100644--- a/python/freetoken/models/qwen3_vl/__init__.py+++ b/python/freetoken/models/qwen3_vl/__init__.py@@ -1,4 +1,5 @@from .config import VisionConfig, parse_config, parse_vision_config+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguffrom .model import (Qwen3VLForConditionalGeneration,Qwen3VLMoeForConditionalGeneration,@@ -13,10 +14,13 @@ __all__ = ["Qwen3VLVisionModel","VisionConfig","deepstack_add",+ "iter_gguf_weights","iter_vision_weights","iter_weights","iter_weights_parallel","parse_config",+ "parse_gguf_config","parse_vision_config","rename_vl_prefix",+ "to_gguf",]diff --git a/python/freetoken/models/qwen3_vl/gguf.py b/python/freetoken/models/qwen3_vl/gguf.pyindex 42c40c2e..52f1edd7 100644--- a/python/freetoken/models/qwen3_vl/gguf.py+++ b/python/freetoken/models/qwen3_vl/gguf.py@@ -1,4 +1,6 @@-"""The Qwen VL vision tower from a llama.cpp ``qwen3vl_merger`` mmproj: the HF config keys, the image processor config, and the tower weights.+"""Qwen3-VL GGUFs: the text trunks of llama.cpp ``qwen3vl`` / ``qwen3vlmoe`` files, and the Qwen VL vision tower from a ``qwen3vl_merger`` mmproj (the HF config keys, the image processor config, and the tower weights).++The text trunk runs without its vision side: no tower and plain rope instead of interleaved mrope. llama.cpp's IMROPE keeps the standard full-spectrum theta (ggml.h: the dim index is not reset per section), so equal (t, h, w) text positions reduce to NeoX rope and q/k carry no permute, like the ``qwen3`` / ``qwen3moe`` GGUFs. The routed experts stack as ``ffn_*_exps`` because the converter splits HF's packed ``gate_up_proj`` / ``down_proj`` experts (conversion/qwen3vl.py).llama.cpp splits the Conv3d patch embedding into one tensor per temporal slice (``v.patch_embd.weight`` and ``.weight.1``); everything else is stored in HF layout."""@@ -6,12 +8,25 @@ llama.cpp splits the Conv3d patch embedding into one tensor per temporal slice (from __future__ import annotationsimport re-from typing import Any, Iterator+from dataclasses import replace+from types import SimpleNamespace+from typing import Any, TYPE_CHECKING, Iteratorimport torchfrom freetoken.models.gguf.mmproj import Mmproj, mmproj_tensor-from freetoken.models.gguf.reader import gguf_tensor_index, load_gguf_metadata+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, load_gguf_metadata+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrix++from .config import parse_config++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+ from freetoken.models.gguf.config import GgufConfigShim++# ----------------------------------------------------------------------------------------------+# Vision tower (a qwen3vl_merger mmproj)+# ----------------------------------------------------------------------------------------------_PROJECTOR = "qwen3vl_merger"_IMAGE_TOKEN = "<|image_pad|>"@@ -111,4 +126,155 @@ def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:yield f"visual.{_name(gguf_name)}", mmproj_tensor(info)-__all__ = ["gguf_mm_config", "iter_mmproj_weights"]+# ----------------------------------------------------------------------------------------------+# Text trunk (a qwen3vl / qwen3vlmoe GGUF)+# ----------------------------------------------------------------------------------------------++_LAYER_LEAVES = {+ "self_attn.q_proj": "attn_q",+ "self_attn.k_proj": "attn_k",+ "self_attn.v_proj": "attn_v",+ "self_attn.o_proj": "attn_output",+ "mlp.gate_proj": "ffn_gate",+ "mlp.up_proj": "ffn_up",+ "mlp.down_proj": "ffn_down",+ "mlp.gate": "ffn_gate_inp",+ "mlp.experts.0.gate_proj": "ffn_gate_exps",+ "mlp.experts.0.up_proj": "ffn_up_exps",+ "mlp.experts.0.down_proj": "ffn_down_exps",+}+_QKV_LEAVES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj")+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")+++def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""+ if name in _TOP_LEAVES:+ return (_TOP_LEAVES[name],)+ m = _LAYER_RE.match(name)+ if m is None:+ return ()+ blk, leaf = f"blk.{m.group(1)}", m.group(2)+ if leaf in _QKV_LEAVES and f"{blk}.attn_qkv.weight" in tensors:+ return (f"{blk}.attn_qkv",)+ if leaf in ("mlp.experts.0.gate_proj", "mlp.experts.0.up_proj") and f"{blk}.ffn_gate_up_exps.weight" in tensors:+ return (f"{blk}.ffn_gate_up_exps",)+ tensor = _LAYER_LEAVES.get(leaf)+ return (f"{blk}.{tensor}",) if tensor is not None else ()+++def _check_supported(shim: GgufConfigShim) -> None:+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")+ check_scales(shim.model_path)+++def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:+ """The HF config fields ``parse_config`` reads, from the GGUF metadata."""+ md = shim.metadata+ arch = shim.model_type # the GGUF architecture: qwen3vl or qwen3vlmoe+ moe = arch.endswith("moe")++ def kv(key: str):+ return md[f"{arch}.{key}"]++ _check_supported(shim)+ rope = {"rope_type": "default", "rope_theta": float(kv("rope.freq_base"))}+ scaling = md.get(f"{arch}.rope.scaling.type", "none")+ if scaling == "yarn":+ rope.update(+ rope_type="yarn",+ factor=float(kv("rope.scaling.factor")),+ original_max_position_embeddings=int(kv("rope.scaling.original_context_length")),+ )+ elif scaling != "none":+ raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} is not supported")+ key_length = md.get(f"{arch}.attention.key_length")+ text = SimpleNamespace(+ model_type="qwen3_moe" if moe else "qwen3",+ architectures=list(shim.architectures),+ hidden_act="silu",+ hidden_size=int(kv("embedding_length")),+ num_hidden_layers=int(kv("block_count")),+ num_attention_heads=int(kv("attention.head_count")),+ num_key_value_heads=int(kv("attention.head_count_kv")),+ head_dim=int(key_length) if key_length is not None else None,+ rope_scaling=None if scaling == "none" else rope,+ rope_parameters=rope,+ max_position_embeddings=int(kv("context_length")),+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),+ vocab_size=int(shim.vocab_size),+ tie_word_embeddings=bool(shim.tie_word_embeddings),+ intermediate_size=int(kv("feed_forward_length")),+ )+ if moe:+ text.num_experts = int(kv("expert_count"))+ text.num_experts_per_tok = int(kv("expert_used_count"))+ text.moe_intermediate_size = int(kv("expert_feed_forward_length"))+ # llama.cpp always renormalizes the top-k weights of a qwen3moe-style router+ text.norm_topk_prob = True+ else:+ text.num_experts = 0+ # text only: no vision_config, so the wrapper builds no tower and keeps 1-D rope+ return SimpleNamespace(+ text_config=text,+ model_type="qwen3_vl",+ architectures=list(shim.architectures),+ image_token_id=None,+ vision_config=None,+ tie_word_embeddings=bool(shim.tie_word_embeddings),+ )+++def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:+ config = parse_config(_hf_config(shim))+ return replace(config, expert_quant="gguf") if config.num_experts else config+++def iter_gguf_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+) -> Iterator[tuple[str, torch.Tensor]]:+ """The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""+ if not include_non_moe:+ return+ from freetoken.utils import cached_load_hf_config++ text = _hf_config(cached_load_hf_config(model_path)).text_config+ index = gguf_tensor_index(model_path)+ hidden = text.hidden_size++ def t(name: str) -> GgufTensorInfo:+ return index[name if name in index else f"{name}.weight"]++ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)+ if not text.tie_word_embeddings:+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)+ yield "model.norm.weight", float_tensor(t("output_norm"))++ for layer in range(text.num_hidden_layers):+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))+ sa = f"{pre}.self_attn"+ # convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrix+ qkv = [t(f"{blk}.attn_qkv")] if f"{blk}.attn_qkv.weight" in index else [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]+ yield from fused(f"{sa}.qkv_proj", qkv, [matrix(p) for p in qkv], hidden)+ yield f"{sa}.q_norm.weight", float_tensor(t(f"{blk}.attn_q_norm"))+ yield f"{sa}.k_norm.weight", float_tensor(t(f"{blk}.attn_k_norm"))+ out = t(f"{blk}.attn_output")+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))+ mlp = f"{pre}.mlp"+ if text.num_experts:+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))+ else:+ gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))+ yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)+ yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))+++__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"]diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex bbe34f06..ad8ade86 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -162,6 +162,21 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {encoders=_QWEN_VL_ENCODERS,packed_modules_mapping=_DENSE_PACKED + _STACKED_EXPERTS_PACKED,),+ # llama.cpp qwen3vl / qwen3vlmoe GGUF (text only): the same model classes with every tensor in its packed ggml type+ "Qwen3VLGGUFForCausalLM": ModelSpec(+ "freetoken.models.qwen3_vl",+ "Qwen3VLForConditionalGeneration",+ parse_config="parse_gguf_config",+ iter_weights="iter_gguf_weights",+ packed_modules_mapping=_DENSE_PACKED,+ ),+ "Qwen3VLMoeGGUFForCausalLM": ModelSpec(+ "freetoken.models.qwen3_vl",+ "Qwen3VLMoeForConditionalGeneration",+ parse_config="parse_gguf_config",+ iter_weights="iter_gguf_weights",+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED,+ ),"MiniMaxM2ForCausalLM": ModelSpec("freetoken.models.minimax_m2","MiniMaxM2ForCausalLM",