FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(gguf): serve image input on qwen3vl ggufs from a deepstack qwen3vl_merger mmproj (#827)
Mukkesh Ganesh380693af9c43
diff --git a/python/freetoken/models/qwen3_vl/__init__.py b/python/freetoken/models/qwen3_vl/__init__.pyindex 2b311ba5..8de9eec8 100644--- a/python/freetoken/models/qwen3_vl/__init__.py+++ b/python/freetoken/models/qwen3_vl/__init__.py@@ -1,5 +1,5 @@from .config import VisionConfig, parse_config, parse_vision_config-from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf+from .gguf import gguf_mm_config, iter_gguf_weights, parse_gguf_config, to_gguffrom .model import (Qwen3VLForConditionalGeneration,Qwen3VLMoeForConditionalGeneration,@@ -14,6 +14,7 @@ __all__ = ["Qwen3VLVisionModel","VisionConfig","deepstack_add",+ "gguf_mm_config","iter_gguf_weights","iter_vision_weights","iter_weights",diff --git a/python/freetoken/models/qwen3_vl/gguf.py b/python/freetoken/models/qwen3_vl/gguf.pyindex 6fa124a3..ecb54c89 100644--- a/python/freetoken/models/qwen3_vl/gguf.py+++ b/python/freetoken/models/qwen3_vl/gguf.py@@ -1,8 +1,8 @@"""Qwen3-VL GGUFs: the text trunks of llama.cpp ``qwen3vl`` / ``qwen3vlmoe`` files, and the Qwen VL vision tower from a ``qwen3vl_merger`` mmproj (the HF config keys, the image processor config, and the tower weights).-The text trunk runs without its vision side: no tower and plain rope instead of interleaved mrope. llama.cpp's IMROPE keeps the standard full-spectrum theta (ggml.h: the dim index is not reset per section), so equal (t, h, w) text positions reduce to NeoX rope and q/k carry no permute, like the ``qwen3`` / ``qwen3moe`` GGUFs. The routed experts stack as ``ffn_*_exps`` because the converter splits HF's packed ``gate_up_proj`` / ``down_proj`` experts (conversion/qwen3vl.py).+Without an mmproj the text trunk runs without its vision side: no tower and plain rope instead of interleaved mrope. llama.cpp's IMROPE keeps the standard full-spectrum theta (ggml.h: the dim index is not reset per section), so equal (t, h, w) text positions reduce to NeoX rope and q/k carry no permute, like the ``qwen3`` / ``qwen3moe`` GGUFs. The routed experts stack as ``ffn_*_exps`` because the converter splits HF's packed ``gate_up_proj`` / ``down_proj`` experts (conversion/qwen3vl.py).-llama.cpp splits the Conv3d patch embedding into one tensor per temporal slice (``v.patch_embd.weight`` and ``.weight.1``); everything else is stored in HF layout.+llama.cpp splits the Conv3d patch embedding into one tensor per temporal slice (``v.patch_embd.weight`` and ``.weight.1``); everything else is stored in HF layout. A Qwen3-VL mmproj also carries one DeepStack merger per tapped block as ``v.deepstack.{block}.{norm,fc1,fc2}``; the merger index is the tap's rank, as the converter writes it (conversion/qwen3vl.py maps ``deepstack_merger_list.{k}`` through ``deepstack_visual_indexes[k]``)."""from __future__ import annotations@@ -14,7 +14,7 @@ from typing import Any, TYPE_CHECKING, Iteratorimport torch-from freetoken.models.gguf.mmproj import Mmproj, mmproj_tensor+from freetoken.models.gguf.mmproj import Mmproj, find_mmproj, log_mmproj, mmproj_tensorfrom freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, load_gguf_metadatafrom freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrix@@ -53,6 +53,13 @@ _BLOCK = {}_BLOCK_RE = re.compile(r"^v\.blk\.(\d+)\.(\w+)\.(weight|bias)$")_PATCH = ("v.patch_embd.weight", "v.patch_embd.weight.1")+_DS_RE = re.compile(r"^v\.deepstack\.(\d+)\.(norm|fc1|fc2)\.(weight|bias)$")+_DS_LEAF = {"norm": "norm", "fc1": "linear_fc1", "fc2": "linear_fc2"}+++def _deepstack_taps(index: dict[str, GgufTensorInfo]) -> tuple[int, ...]:+ """The ViT blocks a DeepStack merger taps, ascending; ``deepstack_visual_indexes`` and the merger order both follow it."""+ return tuple(sorted({int(m[1]) for n in index if (m := _DS_RE.match(n))}))def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]:@@ -60,13 +67,31 @@ def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *,if mmproj.projector != _PROJECTOR:raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; Qwen VL needs {_PROJECTOR!r}")md, index = load_gguf_metadata(mmproj.path), gguf_tensor_index(mmproj.path)- if any(md.get("clip.vision.is_deepstack_layers") or []):+ arch = text_md["general.architecture"]+ taps = _deepstack_taps(index)+ if taps and arch not in ("qwen3vl", "qwen3vlmoe"):raise ValueError(f"mmproj {mmproj.path} is a Qwen3-VL tower with DeepStack layers; Qwen3.5 and later read none")+ declared = [i for i, on in enumerate(md.get("clip.vision.is_deepstack_layers") or []) if on]+ if declared and tuple(declared) != taps:+ raise ValueError(f"mmproj {mmproj.path} declares DeepStack layers {declared} but carries tensors for blocks {list(taps)}")+ text_n = text_md.get(f"{arch}.n_deepstack_layers")+ if text_n is not None and int(text_n) != len(taps):+ raise ValueError(f"mmproj {mmproj.path} taps {len(taps)} DeepStack layers; the text GGUF declares {int(text_n)}")+ if taps:+ if max(taps) >= int(md["clip.vision.block_count"]):+ raise ValueError(f"mmproj {mmproj.path} taps DeepStack mergers at blocks {list(taps)} past its {int(md['clip.vision.block_count'])} tower blocks")+ missing = [+ f"v.deepstack.{i}.{leaf}"+ for i in taps+ for leaf in ("norm.weight", "norm.bias", "fc1.weight", "fc1.bias", "fc2.weight", "fc2.bias")+ if f"v.deepstack.{i}.{leaf}" not in index+ ]+ if missing:+ raise ValueError(f"mmproj {mmproj.path} carries incomplete DeepStack mergers (no {missing[0]})")def kv(key: str):return md[f"clip.vision.{key}"]- arch = text_md["general.architecture"]text_hidden = int(text_md[f"{arch}.embedding_length"])if int(kv("projection_dim")) != text_hidden:raise ValueError(f"mmproj {mmproj.path} projects to {int(kv('projection_dim'))} but the text model is {text_hidden} wide")@@ -86,7 +111,7 @@ def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *,"out_hidden_size": text_hidden,# ggml order [kw, kh, in, out]"in_channels": int(index[_PATCH[0]].ne[2]),- "deepstack_visual_indexes": [],+ "deepstack_visual_indexes": list(taps),}hf_fields = {"vision_config": vision_config,@@ -108,22 +133,26 @@ def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *,return hf_fields, preprocessor-def _name(gguf_name: str) -> str:+def _name(gguf_name: str, taps: tuple[int, ...]) -> str:if gguf_name in _TOP:return _TOP[gguf_name]+ if m := _DS_RE.match(gguf_name):+ # gguf_mm_config refuses a tap past the tower depth or an incomplete merger before any weight loads+ return f"deepstack_merger_list.{taps.index(int(m[1]))}.{_DS_LEAF[m[2]]}.{m[3]}"if (m := _BLOCK_RE.match(gguf_name)) and m[2] in _BLOCK:return f"blocks.{m[1]}.{_BLOCK[m[2]]}.{m[3]}"raise ValueError(f"unexpected tensor {gguf_name!r} in a {_PROJECTOR} mmproj")def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:- """The tower state dict under ``visual.``, named as the HF checkpoints load."""+ """The tower state dict under ``visual.``, named as the HF checkpoints load; merger ``k`` is the tap's rank."""index = gguf_tensor_index(mmproj.path)+ taps = _deepstack_taps(index)slices = [mmproj_tensor(index[n]) for n in _PATCH if n in index]yield "visual.patch_embed.proj.weight", torch.stack(slices, dim=2)for gguf_name, info in index.items():if gguf_name not in _PATCH:- yield f"visual.{_name(gguf_name)}", mmproj_tensor(info)+ yield f"visual.{_name(gguf_name, taps)}", mmproj_tensor(info)# ----------------------------------------------------------------------------------------------@@ -180,7 +209,12 @@ def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:return md[f"{arch}.{key}"]_check_supported(shim)+ vision_config = getattr(shim, "vision_config", None)rope = {"rope_type": "default", "rope_theta": float(kv("rope.freq_base"))}+ if vision_config is not None:+ # every Qwen3-VL release interleaves its [t | h | w] sections; without a tower the text keeps 1-D rope+ rope["mrope_section"] = [int(s) for s in kv("rope.dimension_sections")][:3]+ rope["mrope_interleaved"] = Truescaling = md.get(f"{arch}.rope.scaling.type", "none")if scaling == "yarn":rope.update(@@ -216,13 +250,12 @@ def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:text.norm_topk_prob = Trueelse:text.num_experts = 0- # text only: no vision_config, so the wrapper builds no tower and keeps 1-D ropereturn SimpleNamespace(text_config=text,model_type="qwen3_vl",architectures=list(shim.architectures),- image_token_id=None,- vision_config=None,+ image_token_id=getattr(shim, "image_token_id", None),+ vision_config=vision_config,tie_word_embeddings=bool(shim.tie_word_embeddings),)@@ -238,25 +271,28 @@ def iter_gguf_weights(*,include_moe_experts: bool,include_non_moe: bool,+ include_vision: bool = True,) -> Iterator[tuple[str, torch.Tensor]]:"""The dense state dict from the GGUF; the routed experts load as GGUF expert banks (``freetoken.models.gguf.experts``)."""if not include_non_moe:returnfrom freetoken.utils import cached_load_hf_config- text = _hf_config(cached_load_hf_config(model_path)).text_config+ config = parse_gguf_config(cached_load_hf_config(model_path))+ vision = include_vision and config.is_multimodal+ log_mmproj(model_path, vision)index = gguf_tensor_index(model_path)- hidden = text.hidden_size+ hidden = config.hidden_sizedef t(name: str) -> GgufTensorInfo:return index[name if name in index else f"{name}.weight"]yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)- if not text.tie_word_embeddings:+ if not config.tie_word_embeddings:yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)yield "model.norm.weight", float_tensor(t("output_norm"))- for layer in range(text.num_hidden_layers):+ for layer in range(config.num_layers):blk, pre = f"blk.{layer}", f"model.layers.{layer}"yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))@@ -269,12 +305,15 @@ def iter_gguf_weights(out = t(f"{blk}.attn_output")yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))mlp = f"{pre}.mlp"- if text.num_experts:+ if config.moe_enabled:yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))else:gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))+ if vision:+ yield from iter_mmproj_weights(find_mmproj(model_path))+__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"]diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 4134dfa3..20f5ea39 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -173,13 +173,16 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {encoders=_QWEN_VL_ENCODERS,packed_modules_mapping=_DENSE_PACKED + _STACKED_EXPERTS_PACKED,),- # llama.cpp qwen3vl / qwen3vlmoe GGUF (text only): the same model classes with every tensor in its packed ggml type+ # llama.cpp qwen3vl / qwen3vlmoe GGUF: the same model classes with every tensor in+ # its packed ggml type; a qwen3vl_merger mmproj adds the tower with its DeepStack mergers"Qwen3VLGGUFForCausalLM": ModelSpec("freetoken.models.qwen3_vl","Qwen3VLForConditionalGeneration",parse_config="parse_gguf_config",iter_weights="iter_gguf_weights",packed_modules_mapping=_DENSE_PACKED,+ mm_processor=_QWEN_VL_PROCESSOR,+ encoders=_QWEN_VL_ENCODERS,),"Qwen3VLMoeGGUFForCausalLM": ModelSpec("freetoken.models.qwen3_vl",@@ -187,6 +190,8 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {parse_config="parse_gguf_config",iter_weights="iter_gguf_weights",packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED,+ mm_processor=_QWEN_VL_PROCESSOR,+ encoders=_QWEN_VL_ENCODERS,),"MiniMaxM2ForCausalLM": ModelSpec("freetoken.models.minimax_m2",