Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): serve image input on deepseek4 ggufs from a deepseek4v mmproj (#816)

Mukkesh Ganesh271ff07b30cf
diff --git a/docs/models.md b/docs/models.md
index 9807df80..912b624c 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -125,6 +125,10 @@ unset on PCIe GPUs: wider grids add no bandwidth there.
with `ft checkpoint`, or add the encoder in place with [scripts/ftw_hotfix.py](ftw-hotfix.md).
- DeepSeek-V4 checkpoints must keep the `inference/config.json` subdir — the
authoritative model args are read from there.
+- A DeepSeek-V4 GGUF takes image input from the Vision-Exp text file paired with its
+ `deepseek4v` mmproj; the text-only 0731 release has no image token and refuses the
+ mmproj. ggml-org's Vision-Exp text GGUF ships a chat template that renders no image
+ placeholder; serve unsloth's release for image requests.
- DeepSeek-V4.1-Flash serves text and images from the HF checkpoint as shipped (fp8 block-32
dense weights, fp4 experts on the offload cache, 890 B/token global KV in packed
fp4/fp8 pools). Its two 98 GiB Engram tables stream from the checkpoint shards on
diff --git a/python/freetoken/mm/processors/deepseek_v4.py b/python/freetoken/mm/processors/deepseek_v4.py
index 5cea9919..96528fd4 100644
--- a/python/freetoken/mm/processors/deepseek_v4.py
+++ b/python/freetoken/mm/processors/deepseek_v4.py
@@ -103,6 +103,11 @@ class DSV4MMProcessor(MMProcessor):
positions, delta = self.positions(len(out), items) or (None, 0)
return MMResult(new_ids, items, positions, delta)
+ def _image_processor(self) -> None:
+ """None: this family's preprocessing is FreeToken-local (:func:`load_image`); no
+ transformers image-processor class exists to build from a preprocessor_config."""
+ return None
+
def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
# The smallest block the tower can run: one 2x2 merge grid out of a 6x6 patch grid.
r, p = self.vc.vision_downsample_ratio, self.vc.vision_patch_size
diff --git a/python/freetoken/models/deepseek_v4/__init__.py b/python/freetoken/models/deepseek_v4/__init__.py
index 31931ede..65d0e5f8 100644
--- a/python/freetoken/models/deepseek_v4/__init__.py
+++ b/python/freetoken/models/deepseek_v4/__init__.py
@@ -15,7 +15,7 @@ addressed by page tables, and sparse attention is a physical-slot gather (see
from .args import DeepseekV4Args, load_args
from .config import DSV4VisionConfig, model_config_from_args, parse_config, parse_vision_config
-from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
+from .gguf import gguf_mm_config, iter_gguf_weights, iter_mmproj_weights, parse_gguf_config, to_gguf
from .model import DeepseekV4ForCausalLM
from .vision import DSV4Aligner, DSV4Vision, DSV4VisionTower
from .weight import iter_expert_pieces, iter_vision_weights, iter_weights
@@ -31,8 +31,10 @@ __all__ = [
"parse_config",
"parse_gguf_config",
"parse_vision_config",
+ "gguf_mm_config",
"DeepseekV4ForCausalLM",
"iter_gguf_weights",
+ "iter_mmproj_weights",
"iter_weights",
"iter_expert_pieces",
"iter_vision_weights",
diff --git a/python/freetoken/models/deepseek_v4/gguf.py b/python/freetoken/models/deepseek_v4/gguf.py
index 4f90137d..29055e51 100644
--- a/python/freetoken/models/deepseek_v4/gguf.py
+++ b/python/freetoken/models/deepseek_v4/gguf.py
@@ -1,5 +1,5 @@
-"""DeepSeek-V4-Flash from a llama.cpp ``deepseek4`` GGUF (unsloth ``DeepSeek-V4-Flash-0731-GGUF``,
-text only): config from the GGUF metadata, tensor names, and the dense state dict.
+"""DeepSeek-V4-Flash from a llama.cpp ``deepseek4`` GGUF (unsloth ``DeepSeek-V4-Flash-0731-GGUF``
+and the Vision-Exp release): config from the GGUF metadata, tensor names, and the dense state dict.
The GGUF geometry matches the HF checkpoint (43 MLA blocks with sliding-window sparse
attention, per-layer KV compressors and a Lightning Indexer on the ratio-4 blocks, mHC x4
@@ -13,10 +13,17 @@ placeholder 0, so the per-layer compressor layout is recovered from the tensor t
``indexer.*`` tensor means ratio 4, a compressor without an indexer means 128, unless its
``wkv`` carries the CSA overlap's double rows (also ratio 4). Routed experts serve from
the GGUF expert banks (``freetoken.models.gguf.experts``) like every GGUF family.
+
+A ``deepseek4v`` mmproj beside the Vision-Exp text file adds the vision tower under
+``visual.``: the converter splits the fused SwiGLU ``w1`` into ``ffn_gate`` / ``ffn_up``
+rows and reshapes the patch embedding into the conv layout (both undone at load); ggml-org
+writes the attention block fused and unsloth split. A vision trunk also carries the
+``exp_probs_b_vl`` router biases and drops the hash layers' never-applied text bias.
"""
from __future__ import annotations
+import os
import re
from dataclasses import replace
from typing import TYPE_CHECKING, Iterator
@@ -25,12 +32,13 @@ import numpy as np
import torch
from freetoken.layers.quantization.ggml_types import is_quantized
-from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, gguf_tensor_names
+from freetoken.models.gguf.mmproj import Mmproj, find_mmproj, log_mmproj, mmproj_tensor
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, gguf_tensor_names, load_gguf_metadata
from freetoken.models.gguf.tensors import check_scales, fused, matrix
from freetoken.utils import init_logger
from .args import DeepseekV4Args
-from .config import model_config_from_args
+from .config import IMAGE_TOKEN_ID, model_config_from_args, parse_vision_config
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
@@ -111,9 +119,14 @@ def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:
names = gguf_tensor_names(shim.model_path)
if any(n.startswith(("vision.", "aligner.")) for n in names):
raise NotImplementedError(
- f"{shim.model_path}: deepseek4 GGUF with a vision tower is not served (the text-only 0731 release has none)"
+ f"{shim.model_path}: deepseek4 GGUF with an inline vision tower is not served "
+ "(llama.cpp ships the tower as a separate mmproj file)"
)
+ # a deepseek4v mmproj beside the file fills the shim's flat vision fields; a text-only
+ # process nulls vision_n_layers on its copy, and None here is the same answer
+ vc = parse_vision_config(shim)
+
# the clamp is per-layer in the GGUF (and the shared expert carries its own array); the model
# takes one value, so anything non-uniform is refused instead of silently taking the first
clamp = need("swiglu_clamp_exp")
@@ -209,12 +222,178 @@ def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:
hc_sinkhorn_iters=int(need("hyper_connection.sinkhorn_iterations")),
hc_eps=float(need("hyper_connection.epsilon")),
n_mtp_layers=0, # the converter drops the MTP blocks
- vision_n_layers=0,
+ # vision fields only when a mmproj filled them (the defaults are the text model's)
+ **({} if vc is None else dict(
+ vision_n_layers=vc.vision_n_layers,
+ vision_dim=vc.vision_dim,
+ vision_n_heads=vc.vision_n_heads,
+ vision_inter_dim=vc.vision_inter_dim,
+ vision_patch_size=vc.vision_patch_size,
+ vision_rope_theta=vc.vision_rope_theta,
+ vision_downsample_ratio=vc.vision_downsample_ratio,
+ vision_max_n_token=vc.vision_max_n_token,
+ vision_min_pixels=vc.vision_min_pixels,
+ vision_max_wh_ratio=vc.vision_max_wh_ratio,
+ )),
)
- config = model_config_from_args(args, max_position=int(need("context_length")))
+ config = model_config_from_args(args, max_position=int(need("context_length")), vision_config=vc)
return replace(config, expert_quant="gguf")
+# ----------------------------------------------------------------------------------------------
+# Vision tower (a deepseek4v mmproj)
+# ----------------------------------------------------------------------------------------------
+
+_PROJECTOR = "deepseek4v"
+_IMAGE_TOKEN = "<|deepseek_image|>"
+# no mmproj key carries these; llama.cpp hardcodes them and its converter asserts them
+_ROPE_THETA = 10000.0
+_MAX_N_TOKEN = 384
+_MAX_WH_RATIO = 8
+
+# top-level mmproj tensor -> the tower name under ``visual.``
+_MM_TOP: dict[str, str] = {
+ "v.patch_embd.bias": "vision.patch_embed.proj.bias",
+ "v.post_ln.weight": "vision.norm.weight",
+ "mm.1.weight": "aligner.w1.weight",
+ "mm.1.bias": "aligner.w1.bias",
+ "mm.2.weight": "aligner.w2.weight",
+ "mm.2.bias": "aligner.w2.bias",
+ "v.token_embd.img_start": "image_start",
+ "v.token_embd.img_end": "image_end",
+ "v.image_newline": "image_newline",
+ "v.token_embd.img_pad": "image_pad",
+}
+
+
+def gguf_mm_config(text_md: dict, mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict, dict]:
+ """The flat HF config keys (the vision dims sit at the top level on this family, like the
+ checkpoint's inference config) and the preprocessor dict of the tower in ``mmproj``, for
+ the text GGUF whose metadata is ``text_md``."""
+ if mmproj.projector != _PROJECTOR:
+ raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; DeepSeek-V4 needs {_PROJECTOR!r}")
+ md, index = load_gguf_metadata(mmproj.path), gguf_tensor_index(mmproj.path)
+ # the converter writes use_silu=True only for the SwiGLU tower; the module has no gelu MLP
+ if md.get("clip.use_silu") is not True:
+ raise NotImplementedError(f"mmproj {mmproj.path}: only the SwiGLU tower is served")
+
+ def kv(key: str):
+ return md[f"clip.vision.{key}"]
+
+ arch = text_md["general.architecture"]
+ text_hidden = int(text_md[f"{arch}.embedding_length"])
+ if int(kv("projection_dim")) != text_hidden:
+ raise ValueError(f"mmproj {mmproj.path} projects to {int(kv('projection_dim'))} but the text model is {text_hidden} wide")
+ tokens = text_md["tokenizer.ggml.tokens"]
+ if _IMAGE_TOKEN not in tokens:
+ raise ValueError(
+ f"the text GGUF tokenizer has no {_IMAGE_TOKEN} token (the 0731 text-only release "
+ "uses another dialect; pair the mmproj with a Vision-Exp text GGUF)"
+ )
+ # the model config and the processor hardcode the family id; a file with the token at
+ # another id would fail per request instead of here
+ if tokens.index(_IMAGE_TOKEN) != IMAGE_TOKEN_ID:
+ raise ValueError(
+ f"the text GGUF carries {_IMAGE_TOKEN} at id {tokens.index(_IMAGE_TOKEN)}, not the family's {IMAGE_TOKEN_ID}"
+ )
+ # a ValueError so auto-discovery skips this mmproj (text only); only --mmproj fails
+ if not any(n.endswith(".exp_probs_b_vl.bias") for n in gguf_tensor_names(model_path)):
+ raise ValueError(
+ f"the text trunk {os.path.basename(model_path)} predates vision routing "
+ "(no exp_probs_b_vl; converted before llama.cpp #28154) -- pair a Vision-Exp "
+ "text GGUF reconverted after 2026-09-02"
+ )
+ for sentinel in ("v.token_embd.img_start", "v.token_embd.img_end", "v.image_newline", "v.token_embd.img_pad"):
+ if sentinel not in index:
+ raise ValueError(f"mmproj {mmproj.path}: sentinel tensor {sentinel!r} is missing")
+ # the tower hardcodes eps 1e-6 (vision.py); the converter warns it is NOT the text 1e-20
+ if abs(float(kv("attention.layer_norm_epsilon")) - 1e-6) > 1e-9:
+ raise NotImplementedError(f"mmproj {mmproj.path}: vision layer_norm_epsilon {kv('attention.layer_norm_epsilon')} is not the served 1e-6")
+ # qkv dialect: ggml-org writes the fused attn_qkv, unsloth the split attn_q/k/v (clip.cpp loads both)
+ split = all(f"v.blk.0.attn_{q}.{part}" in index for q in "qkv" for part in ("weight", "bias"))
+ if split == ("v.blk.0.attn_qkv.weight" in index):
+ raise ValueError(f"mmproj {mmproj.path}: the attn tensors are neither the fused attn_qkv nor the split attn_q/k/v layout (or carry both)")
+ patch, dim = int(kv("patch_size")), int(kv("embedding_length"))
+ # the converter reshapes the Linear's [dim, 3*p*p] kernel into the conv [p, p, 3, dim]
+ if tuple(index["v.patch_embd.weight"].ne) != (patch, patch, 3, dim):
+ raise ValueError(f"mmproj {mmproj.path}: v.patch_embd.weight {tuple(index['v.patch_embd.weight'].ne)} is not the [p, p, 3, dim] conv layout")
+ # clip.vision.image_size (672) is deliberately unread: a converter compat/warmup value,
+ # the tower is native-resolution
+
+ hf_fields = {
+ "vision_n_layers": int(kv("block_count")),
+ "vision_dim": dim,
+ "vision_n_heads": int(kv("attention.head_count")),
+ "vision_inter_dim": int(kv("feed_forward_length")),
+ "vision_patch_size": patch,
+ "vision_rope_theta": _ROPE_THETA,
+ "vision_downsample_ratio": int(kv("projector.scale_factor")),
+ "vision_max_n_token": _MAX_N_TOKEN,
+ "vision_min_pixels": int(kv("image_min_pixels")),
+ "vision_max_wh_ratio": _MAX_WH_RATIO,
+ # parse_vision_config reads the aligner's target width off hidden_size
+ "hidden_size": text_hidden,
+ "image_token_id": tokens.index(_IMAGE_TOKEN),
+ }
+ preprocessor = {
+ # no transformers class exists; DSV4MMProcessor._image_processor returns None
+ "image_processor_type": "DeepseekV4ImageProcessor",
+ "patch_size": patch,
+ "temporal_patch_size": 1,
+ "merge_size": int(kv("projector.scale_factor")),
+ "image_mean": [float(v) for v in kv("image_mean")],
+ "image_std": [float(v) for v in kv("image_std")],
+ "min_pixels": int(kv("image_min_pixels")),
+ "max_n_token": _MAX_N_TOKEN,
+ }
+ return hf_fields, preprocessor
+
+
+def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:
+ """The tower state dict under ``visual.``, named as the HF checkpoints load."""
+ index = gguf_tensor_index(mmproj.path)
+ fused = "v.blk.0.attn_qkv.weight" in index
+ n_blocks = max(int(n.split(".")[2]) for n in index if n.startswith("v.blk.")) + 1
+
+ pe = index["v.patch_embd.weight"]
+ # ggml order [p, p, 3, out] -> torch [out, 3, p, p] -> the module's flat [out, 3*p*p]
+ yield "visual.vision.patch_embed.proj.weight", mmproj_tensor(pe).reshape(pe.ne[3], 3 * pe.ne[0] * pe.ne[0])
+ for gguf_name, ft_name in _MM_TOP.items():
+ yield f"visual.{ft_name}", mmproj_tensor(index[gguf_name])
+
+ consumed = {"v.patch_embd.weight", *_MM_TOP}
+ for layer in range(n_blocks):
+ blk, dst = f"v.blk.{layer}", f"visual.vision.blocks.{layer}"
+ if fused:
+ for part in ("weight", "bias"):
+ yield f"{dst}.attn.wqkv.{part}", mmproj_tensor(index[f"{blk}.attn_qkv.{part}"])
+ consumed.update((f"{blk}.attn_qkv.weight", f"{blk}.attn_qkv.bias"))
+ else:
+ for part in ("weight", "bias"):
+ yield f"{dst}.attn.wqkv.{part}", torch.cat(
+ [mmproj_tensor(index[f"{blk}.attn_{q}.{part}"]) for q in "qkv"], dim=0
+ )
+ consumed.update(f"{blk}.attn_{q}.{part}" for q in "qkv" for part in ("weight", "bias"))
+ for part in ("weight", "bias"):
+ yield f"{dst}.attn.wo.{part}", mmproj_tensor(index[f"{blk}.attn_out.{part}"])
+ # the converter chunks the fused SwiGLU w1 rows into gate / up; cat them back
+ yield f"{dst}.mlp.w1.weight", torch.cat(
+ [mmproj_tensor(index[f"{blk}.ffn_gate.weight"]), mmproj_tensor(index[f"{blk}.ffn_up.weight"])], dim=0
+ )
+ yield f"{dst}.mlp.w2.weight", mmproj_tensor(index[f"{blk}.ffn_down.weight"])
+ yield f"{dst}.norm1.weight", mmproj_tensor(index[f"{blk}.ln1.weight"])
+ yield f"{dst}.norm2.weight", mmproj_tensor(index[f"{blk}.ln2.weight"])
+ consumed.update(
+ f"{blk}.attn_out.{part}" for part in ("weight", "bias")
+ )
+ consumed.update(f"{blk}.ffn_{m}.weight" for m in ("gate", "up", "down"))
+ consumed.update((f"{blk}.ln1.weight", f"{blk}.ln2.weight"))
+
+ leftovers = set(index) - consumed
+ if leftovers:
+ raise ValueError(f"mmproj {mmproj.path}: unexpected tensors {sorted(leftovers)[:5]}")
+
+
# ----------------------------------------------------------------------------------------------
# Weights
# ----------------------------------------------------------------------------------------------
@@ -267,6 +446,8 @@ def iter_gguf_weights(
from freetoken.utils import cached_load_hf_config
config = parse_gguf_config(cached_load_hf_config(model_path))
+ vision = include_vision and config.is_multimodal
+ log_mmproj(model_path, vision)
args = config.dsv4_args
index = gguf_tensor_index(model_path)
hidden = config.hidden_size
@@ -331,14 +512,22 @@ def iter_gguf_weights(
g = f"{pre}.ffn.gate"
yield f"{g}.weight", bf16(f"{blk}.ffn_gate_inp")
if layer < args.n_hash_layers:
- # a vision checkpoint also gives its hash layers a text bias; the text model has no slot for it
yield f"{g}.tid2eid", _tid2eid(t(f"{blk}.ffn_gate_tid2eid"))
+ if vision:
+ # the converter drops the hash layers' text bias (the reference never applies
+ # it); the module builds the slot on a vision build, so it zero-fills
+ yield f"{g}.bias", torch.zeros(args.n_routed_experts, dtype=torch.float32)
else:
yield f"{g}.bias", _float(t(f"{blk}.exp_probs_b.bias"))
+ if vision:
+ yield f"{g}.bias_vl", _float(t(f"{blk}.exp_probs_b_vl.bias"))
for proj, src in (("w1", "ffn_gate_shexp"), ("w3", "ffn_up_shexp"), ("w2", "ffn_down_shexp")):
info = t(f"{blk}.{src}")
k = hidden if proj != "w2" else args.moe_inter_dim
yield from fused(f"{pre}.ffn.shared_experts.{proj}", [info], [matrix(info)], k)
+ if vision:
+ yield from iter_mmproj_weights(find_mmproj(model_path))
+
-__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]
+__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/gemma4/gguf.py b/python/freetoken/models/gemma4/gguf.py
index 047d089d..ce806ba8 100644
--- a/python/freetoken/models/gemma4/gguf.py
+++ b/python/freetoken/models/gemma4/gguf.py
@@ -105,7 +105,7 @@ def _f32_value(v: float) -> float:
return float(str(np.float32(v)))
-def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any]]:
+def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]:
"""The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``."""
if mmproj.projector not in (_VIT, _UNIFIED):
raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; Gemma 4 needs {_VIT!r} or {_UNIFIED!r}")
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index e718809f..1f234b1c 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -96,7 +96,9 @@ def _vocab_size(model_path: str) -> int:
@functools.cache
def _mm_fields(module: str, model_path: str, mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any] | None]:
- return importlib.import_module(module).gguf_mm_config(load_gguf_metadata(model_path), mmproj, vocab_size)
+ return importlib.import_module(module).gguf_mm_config(
+ load_gguf_metadata(model_path), mmproj, vocab_size, model_path=model_path
+ )
def mm_fields(model_path: str, mmproj: Mmproj) -> tuple[dict[str, Any], dict[str, Any] | None]:
diff --git a/python/freetoken/models/glm5_next/gguf.py b/python/freetoken/models/glm5_next/gguf.py
index c9182253..97c5703d 100644
--- a/python/freetoken/models/glm5_next/gguf.py
+++ b/python/freetoken/models/glm5_next/gguf.py
@@ -281,7 +281,7 @@ _BLOCK = {
_BLOCK_RE = re.compile(r"^v\.blk\.(\d+)\.(\w+)\.(weight|bias)$")
-def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any]]:
+def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]:
"""The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``."""
if mmproj.projector != _PROJECTOR:
raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; GLM-5.3-Flash needs {_PROJECTOR!r}")
diff --git a/python/freetoken/models/qwen3_vl/gguf.py b/python/freetoken/models/qwen3_vl/gguf.py
index 52f1edd7..6fa124a3 100644
--- a/python/freetoken/models/qwen3_vl/gguf.py
+++ b/python/freetoken/models/qwen3_vl/gguf.py
@@ -55,7 +55,7 @@ _BLOCK_RE = re.compile(r"^v\.blk\.(\d+)\.(\w+)\.(weight|bias)$")
_PATCH = ("v.patch_embd.weight", "v.patch_embd.weight.1")
-def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any]]:
+def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]:
"""The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``."""
if mmproj.projector != _PROJECTOR:
raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; Qwen VL needs {_PROJECTOR!r}")
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index efea3d30..4134dfa3 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -243,8 +243,9 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
mm_processor=_DSV4_PROCESSOR,
encoders=_DSV4_ENCODERS,
),
- # llama.cpp deepseek4 GGUF (unsloth DeepSeek-V4-Flash-0731-GGUF; text only): the same model
- # classes with every tensor in its packed ggml type
+ # llama.cpp deepseek4 GGUF (unsloth DeepSeek-V4-Flash-0731-GGUF and the Vision-Exp
+ # release + its deepseek4v mmproj): the same model classes with every tensor in its
+ # packed ggml type
"DeepseekV4GGUFForCausalLM": ModelSpec(
"freetoken.models.deepseek_v4",
"DeepseekV4ForCausalLM",
@@ -255,6 +256,8 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
# the KV compressors read their weights through a dense bf16 einsum, so the reader
# dequantizes them instead of serving the packed ggml rows
unquantized_modules=("*.compressor.wkv", "*.compressor.wgate"),
+ mm_processor=_DSV4_PROCESSOR,
+ encoders=_DSV4_ENCODERS,
),
# DeepSeek-V4.1-Flash (model_type deepseek_v41): multimodal wrapper config (text tower in
# text_config). The checkpoint has no ``model.`` root; the compressors, the indexer's key projection /