Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(deepseek_v4): serve image input on the V4-Flash-Vision release (#502)

Mukkesh Ganesh23fc922bf8df
diff --git a/docs/models.md b/docs/models.md
index b999d29e..cdadc2dc 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -6,7 +6,7 @@ for them; other checkpoints of the same architectures work too.
| Model | HF checkpoints |
|---|---|
| DeepSeek-V4.1-Flash | [deepseek-ai/DeepSeek-V4.1-Flash](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash) |
-| DeepSeek-V4 | [deepseek-ai/DeepSeek-V4-Flash-0731](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731) |
+| DeepSeek-V4 | [deepseek-ai/DeepSeek-V4-Flash-0731](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731), [deepseek-ai/DeepSeek-V4-Flash-Vision-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp) |
| GLM-5.3-Flash | [RedHatAI/GLM-5.3-Flash-NVFP4](https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4) |
| GLM-5.2 | [nvidia/GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4) |
| GLM-4.7 | [nvidia/GLM-4.7-NVFP4](https://huggingface.co/nvidia/GLM-4.7-NVFP4) |
@@ -35,6 +35,7 @@ These families accept image input by default; pass `--text-model-only` to skip t
| GLM-5.3-Flash (`glm5_next`: ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution on a canvas zero-padded to a 28-multiple | token counts, passed through as the processor's `min_image_tokens` / `max_image_tokens`; checkpoint defaults 16 to 8000 tokens | `{"max_image_tokens": 2048}` |
| Muse-Glimmer-30B (windowed ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, aspect ratio kept under a token cap; checkpoint default 4096 tokens | the maximum is the cap (`max_image_tokens`); the minimum has no effect | `{"max_image_tokens": 1024}` |
| MiniMax-M3 (`minimax_m3`: CLIP-style ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution | pixel areas in `size.shortest_edge` / `longest_edge`; checkpoint defaults 4 to 576 tokens | `{"size": {"longest_edge": 1048576}}` |
+| DeepSeek-V4-Flash-Vision (`deepseek_v4`: native-resolution ViT tower, streamed under `--mm-encoder-weights host`) | one token per 3x3 grid of 14x14 patches, against a per-image cap; the block carries sentinel tokens and lead pads that align it to a 4-token compression stride, so its length depends on where it lands in the prompt | the maximum is a cap on the block (`vision_max_n_token`, checkpoint default 384) and lowers it, the minimum is read as the checkpoint's upscale floor and has no effect | not read |
## MoE strategies
diff --git a/python/freetoken/mm/processor.py b/python/freetoken/mm/processor.py
index 48211a5b..84463bc2 100644
--- a/python/freetoken/mm/processor.py
+++ b/python/freetoken/mm/processor.py
@@ -200,7 +200,8 @@ def get_mm_processor(model_path: str, mm: MultimodalConfig | None = None) -> MMP
served = [e for e in spec.encoders if getattr(config, e.config_key, None) is not None and e.kind not in mm.disabled_encoders]
if spec.mm_processor is None or not served:
return None
- check_mm_pad_shift(config.text_config.vocab_size)
+ # families with a flat checkpoint config (deepseek_v4) carry vocab_size at the top level
+ check_mm_pad_shift(getattr(config, "text_config", config).vocab_size)
module, _, cls = spec.mm_processor.partition(":")
return getattr(importlib.import_module(module), cls)(config, model_path, mm)
diff --git a/python/freetoken/mm/processors/deepseek_v4.py b/python/freetoken/mm/processors/deepseek_v4.py
new file mode 100644
index 00000000..5cea9919
--- /dev/null
+++ b/python/freetoken/mm/processors/deepseek_v4.py
@@ -0,0 +1,129 @@
+"""DeepSeek-V4 image processor: each ``<|deepseek_image|>`` expands into the image's block of tokens.
+
+Where the reference streams ``vocab_size + type`` pseudo ids, the whole span carries one
+content pad id and the block's embeddings -- sentinels included -- come from the encoder.
+"""
+
+from __future__ import annotations
+
+import io
+import struct
+from typing import Any
+
+import torch
+
+from freetoken.message import MMItem
+from freetoken.mm import mm_pad_value
+from freetoken.mm.config import MultimodalConfig
+from freetoken.mm.processor import MMResult, MMProcessor, PromptReplacement, _find_all, content_hash
+from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID, DSV4VisionConfig, parse_vision_config
+from freetoken.models.deepseek_v4.image_processor import load_image
+from freetoken.models.deepseek_v4.vision import COMPRESS_PAD_TO, build_image_block
+
+
+class DSV4MMProcessor(MMProcessor):
+ def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:
+ super().__init__(model_path, mm)
+ vc = parse_vision_config(hf_config)
+ if vc is None:
+ raise ValueError(f"{model_path} carries no vision tower (vision_n_layers <= 0)")
+ self.vc: DSV4VisionConfig = vc
+ self.image_token_id = IMAGE_TOKEN_ID
+ self.placeholder = [self.image_token_id]
+ # --image-max-tokens lowers the checkpoint's per-image block cap
+ self.max_n_token = min(vc.vision_max_n_token, mm.image_max_tokens) if mm.image_max_tokens else vc.vision_max_n_token
+
+ def process(self, images: list[Any]) -> list[MMItem]:
+ items: list[MMItem] = []
+ for image in images:
+ patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = load_image(image, self.vc, self.max_n_token)
+ # layout, hash and token count are only known at the insertion offset
+ items.append(
+ MMItem(
+ modality="image",
+ hash=0,
+ pad_value=0,
+ offsets=[],
+ feature=patches,
+ model_specific_data={
+ "n_vit_h": n_vit_h,
+ "n_vit_w": n_vit_w,
+ "n_llm_h": n_llm_h,
+ "n_llm_w": n_llm_w,
+ },
+ )
+ )
+ return items
+
+ def prompt_replacement(self, item: MMItem, start: int) -> PromptReplacement:
+ """The DSV4-only ``start`` is where the block lands in input_ids; only our apply below calls this."""
+ n_llm_h, n_llm_w = item.n_llm_h, item.n_llm_w
+ types, _perm = build_image_block(n_llm_h, n_llm_w, start)
+ # the lead pads align the block, so the same image at another offset is another
+ # sequence: the layout belongs in the hash or the two would share a radix key
+ compress_pad = COMPRESS_PAD_TO - 1 - start % COMPRESS_PAD_TO
+ item.hash = content_hash(item.feature, struct.pack("<3i", n_llm_h, n_llm_w, compress_pad))
+ item.pad_value = mm_pad_value(item.hash)
+ item.model_specific_data["start"] = start
+ return PromptReplacement([self.image_token_id] * types.numel())
+
+ def apply(self, input_ids: torch.Tensor, images: list[bytes]) -> MMResult:
+ """MMProcessor.apply, save that each replacement is built knowing the offset it lands at."""
+ from PIL import Image
+
+ ids = input_ids.tolist()
+ target = self.placeholder
+ slots = _find_all(ids, target)
+ if len(slots) != len(images):
+ raise ValueError(
+ f"prompt renders {len(slots)} image placeholders but the request "
+ f"carries {len(images)} images"
+ )
+
+ pils = [Image.open(io.BytesIO(raw)).convert("RGB") for raw in images]
+ items = self.process(pils)
+ out: list[int] = []
+ cursor = 0
+ for slot, item in zip(slots, items):
+ out.extend(ids[cursor:slot])
+ base = len(out)
+ repl = self.prompt_replacement(item, base)
+ full = list(repl.full)
+ spans = repl.embed_spans()
+ # embedding slots carry the content pad id so radix keys and the model's scatter mask see the image
+ for lo, hi in spans:
+ full[lo:hi] = [item.pad_value] * (hi - lo)
+ out.extend(full)
+ item.offsets = [[base + lo, base + hi] for lo, hi in spans]
+ item.validate()
+ cursor = slot + len(target)
+ out.extend(ids[cursor:])
+ new_ids = torch.tensor(out, dtype=input_ids.dtype)
+
+ positions, delta = self.positions(len(out), items) or (None, 0)
+ return MMResult(new_ids, items, positions, delta)
+
+ def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
+ # The smallest block the tower can run: one 2x2 merge grid out of a 6x6 patch grid.
+ r, p = self.vc.vision_downsample_ratio, self.vc.vision_patch_size
+ n_vit, n_llm = 2 * r, 2
+ types, _perm = build_image_block(n_llm, n_llm, 0)
+ return [
+ MMItem(
+ modality="image",
+ hash=0,
+ pad_value=0,
+ offsets=[[0, types.numel()]],
+ feature=torch.zeros(n_vit * n_vit, 3, p, p, dtype=dtype, device=device),
+ model_specific_data={
+ "n_vit_h": n_vit,
+ "n_vit_w": n_vit,
+ "n_llm_h": n_llm,
+ "n_llm_w": n_llm,
+ "start": 0,
+ },
+ )
+ ]
+
+
+__all__ = ["DSV4MMProcessor"]
diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py
index 55b90bb5..25bc73f5 100644
--- a/python/freetoken/models/config.py
+++ b/python/freetoken/models/config.py
@@ -216,6 +216,9 @@ class DSV4AttentionGroupConfig(BaseAttentionGroupConfig):
sliding_window: int # the P-token window page
# "v41": DeepSeek-V4.1's cross-layer-shared packed tiers (AttnType.DSV41, its own pool and backend)
variant: Literal["v4", "v41"] = "v4"
+ # image token spans attend to each other in both directions, so prefill chunks end
+ # before a block instead of splitting it (the scheduler's keep_images_whole gate)
+ bidirectional_mm_blocks: bool = False
AttentionGroupConfig: TypeAlias = (
diff --git a/python/freetoken/models/deepseek_v4/__init__.py b/python/freetoken/models/deepseek_v4/__init__.py
index e19438d1..5bbb9b8e 100644
--- a/python/freetoken/models/deepseek_v4/__init__.py
+++ b/python/freetoken/models/deepseek_v4/__init__.py
@@ -14,15 +14,22 @@ addressed by page tables, and sparse attention is a physical-slot gather (see
"""
from .args import DeepseekV4Args, load_args
-from .config import parse_config
+from .config import DSV4VisionConfig, parse_config, parse_vision_config
from .model import DeepseekV4ForCausalLM
-from .weight import iter_expert_pieces, iter_weights
+from .vision import DSV4Aligner, DSV4Vision, DSV4VisionTower
+from .weight import iter_expert_pieces, iter_vision_weights, iter_weights
__all__ = [
"DeepseekV4Args",
+ "DSV4Aligner",
+ "DSV4Vision",
+ "DSV4VisionConfig",
+ "DSV4VisionTower",
"load_args",
"parse_config",
+ "parse_vision_config",
"DeepseekV4ForCausalLM",
"iter_weights",
"iter_expert_pieces",
+ "iter_vision_weights",
]
diff --git a/python/freetoken/models/deepseek_v4/args.py b/python/freetoken/models/deepseek_v4/args.py
index b75d2800..d2646a2b 100644
--- a/python/freetoken/models/deepseek_v4/args.py
+++ b/python/freetoken/models/deepseek_v4/args.py
@@ -69,6 +69,18 @@ class DeepseekV4Args:
hc_sinkhorn_iters: int = 20
hc_eps: float = 1e-6
+ # ----- vision tower (DeepSeek-V4-Flash-Vision-Exp; 0 on the text-only releases) -----
+ vision_n_layers: int = 0
+ vision_dim: int = 1024
+ vision_n_heads: int = 16
+ vision_inter_dim: int = 2816
+ vision_patch_size: int = 14
+ vision_rope_theta: float = 10000.0
+ vision_downsample_ratio: int = 3
+ vision_max_n_token: int = 384
+ vision_min_pixels: int = 147456
+ vision_max_wh_ratio: int = 8
+
def __post_init__(self) -> None:
# JSON lists -> tuple so the dataclass stays hashable / immutable-ish.
if isinstance(self.compress_ratios, list):
diff --git a/python/freetoken/models/deepseek_v4/attention.py b/python/freetoken/models/deepseek_v4/attention.py
index cb0dd315..91bc8ac0 100644
--- a/python/freetoken/models/deepseek_v4/attention.py
+++ b/python/freetoken/models/deepseek_v4/attention.py
@@ -38,6 +38,7 @@ class Attention(BaseOP):
self.window_size = args.window_size
self.compress_ratio = args.compress_ratios[layer_id]
self.eps = args.norm_eps
+ self.max_image_tokens = args.vision_max_n_token
self.attn_sink = torch.empty(self.n_heads, dtype=torch.float32)
# the latent projections are replicated; wq_b shards over heads, wo_b over the output groups
@@ -104,7 +105,53 @@ class Attention(BaseOP):
o = torch.einsum("bsgd,grd->bsgr", o, wo_a).flatten(2)
return self.wo_b.forward(o)
- def _prefill_segment(self, x_seg, qr_seg, kv_seg, ti: int, start_pos: int, n: int):
+ @staticmethod
+ def visible_window_cols(
+ spans, start_pos: int, n: int, win: int, max_img: int, device: torch.device,
+ addr_lo: int = 0, width: int | None = None,
+ ) -> torch.Tensor:
+ """Window candidate columns (``-1`` padded) for a segment containing image spans.
+
+ Port of the reference ``get_image_visible`` + ``get_window_topk_idxs_visible``
+ pair: span tokens see their whole span bidirectionally (capped at ``max_img``)
+ instead of the 128-window, while text tokens -- and span tokens near the span
+ start -- keep the ordinary window into history. Returns ``[n, width]`` in
+ request-absolute coordinates with live entries in ``[addr_lo, span_end)``; the
+ caller translates to slot-lut indices (cold: as-is; extend: minus ``w_lo``).
+ """
+ if width is None:
+ width = win + max_img
+ left, right = Attention._span_left_right(spans, start_pos, n, max_img, device)
+ idx = start_pos + torch.arange(n, device=device)
+ left_add = (left - (win - 1)).clamp(min=0)
+ starts = (idx - (win - 1) - left_add).clamp(min=addr_lo)
+ cols = starts.unsqueeze(1) + torch.arange(width, device=device)
+ return torch.where(cols > (idx + right).unsqueeze(1), -1, cols)
+
+ @staticmethod
+ def _span_left_right(
+ spans, start_pos: int, n: int, max_img: int, device: torch.device
+ ) -> tuple[torch.Tensor, torch.Tensor]:
+ """Per-token visible counts to the left/right within each image span.
+
+ Metadata twin of the reference ``get_image_visible`` (which derives spans from
+ virtual token ids): ``left[q] = q - span_start`` clamped to ``max_img - 1``,
+ ``right[q] = span_end - q`` clamped to ``max_img``, zero outside spans. ``spans``
+ are request-absolute positions inside ``[start_pos, start_pos + n)``.
+ """
+ left = torch.zeros(n, dtype=torch.int64, device=device)
+ right = torch.zeros(n, dtype=torch.int64, device=device)
+ for s, e in spans:
+ lo, hi = s - start_pos, e - start_pos
+ idx = torch.arange(lo, hi, device=device)
+ left[lo:hi] = (idx - lo).clamp(max=max_img - 1)
+ # the END sentinel sits at e-1: visibility reaches the END TOKEN, not past it
+ right[lo:hi] = ((hi - 1) - idx).clamp(max=max_img)
+ return left, right
+
+ def _prefill_segment(
+ self, x_seg, qr_seg, kv_seg, ti: int, start_pos: int, n: int, spans=None
+ ):
"""One request's prefill work inside a (possibly ragged) batch: persist its window KV,
advance its compressor/indexer carry, and return its per-query candidate lists.
@@ -113,6 +160,16 @@ class Attention(BaseOP):
``(win_global [1, n, w], cmp_global [1, n, c] | None)`` at natural widths (cold: w =
min(n, win); extend: w = win); the caller pads BOTH halves to the batch-uniform widths
and concatenates.
+
+ ``spans`` (VL requests) are the request's image-block ranges ``[(start, end)]`` inside
+ ``[start_pos, start_pos + n)``. Vision checkpoints set ``bidirectional_mm_blocks`` on the
+ attention group, which makes prefill end a chunk before a block instead of splitting it;
+ only a block longer than one chunk budget is still cut. Inside a span, every token
+ sees the whole span (bidirectional, capped at ``vision_max_n_token``) instead of only its
+ 128-window, matching the reference's ``get_window_topk_idxs_visible``. All span-visible
+ candidates are intra-segment, so the live full-page map covers them (the 128-slot ring
+ never needs to serve >128-back reads). ``spans=None`` keeps the historical path
+ bit-identical.
"""
win, ratio, device = self.window_size, self.compress_ratio, x_seg.device
end = start_pos + n
@@ -120,23 +177,38 @@ class Attention(BaseOP):
self.attn.store_window(kv_seg, self.layer_id, slots)
if start_pos == 0:
- win_cols = get_window_topk_idxs(win, 1, n, 0).to(device)
- # natural width min(n, win); the caller pads to the batch-uniform width
+ if spans:
+ max_img = self.max_image_tokens
+ cols = Attention.visible_window_cols(
+ spans, 0, n, win, max_img, device, addr_lo=0,
+ width=min(n, win + max_img),
+ )
+ win_cols = cols.unsqueeze(0)
+ else:
+ win_cols = get_window_topk_idxs(win, 1, n, 0).to(device)
+ # natural width min(n, win) -- win+max_img for span segments; the caller pads
win_global = self.attn.win_cols_to_global(win_cols, slots)
else:
# Candidates span [w_lo, end): each new query's 128-sliding window over the retained
# prefix plus the new tokens, causal-masked to -1 past its own position.
w_lo = max(0, start_pos - win + 1)
ws_pool = self.attn.window_slots_of(ti, w_lo, end)
- abs_p = start_pos + torch.arange(n, device=device).unsqueeze(1)
- cand = (abs_p - win + 1).clamp(min=w_lo) + torch.arange(win, device=device)
- win_cols = torch.where(cand > abs_p, -1, cand - w_lo).unsqueeze(0)
+ if spans:
+ max_img = self.max_image_tokens
+ cols = Attention.visible_window_cols(
+ spans, start_pos, n, win, max_img, device, addr_lo=w_lo,
+ )
+ win_cols = torch.where(cols < 0, cols, cols - w_lo).unsqueeze(0)
+ else:
+ abs_p = start_pos + torch.arange(n, device=device).unsqueeze(1)
+ cand = (abs_p - win + 1).clamp(min=w_lo) + torch.arange(win, device=device)
+ win_cols = torch.where(cand > abs_p, -1, cand - w_lo).unsqueeze(0)
win_global = self.attn.win_cols_to_global(win_cols, ws_pool)
if not ratio:
return win_global, None
- # Only the compressor/indexer read the matched tail page's slot; resolving it costs a
- # host sync (.item()), so do it after the ratio-0 early-out.
+ # Only the compressor/indexer read the matched tail page's slot; resolving it costs
+ # a host sync (.item()), so do it after the ratio-0 early-out.
tail_ws = (
int(self.attn.window_slots_of(ti, start_pos - 1, start_pos).item())
if start_pos > 0 else None
@@ -157,7 +229,7 @@ class Attention(BaseOP):
self.compressor.forward(x_seg, start_pos, slots, tail_window_slot=tail_ws, ti=ti)
return win_global, self.attn.blocks_to_global(blocks, ratio, ti=ti)
- def forward_ragged(self, x, segments, flat_positions):
+ def forward_ragged(self, x, segments, flat_positions, spans_by_ti=None):
"""Ragged batched prefill (cu_seqlens). ``x`` is [1, T, dim] -- the requests' NEW token
streams concatenated (NO padding); ``segments`` is [(offset, n, table_idx, start_pos)]
tiling [0, T); ``flat_positions`` [T] is each token's ABSOLUTE position, so a radix-hit
@@ -197,9 +269,18 @@ class Attention(BaseOP):
win_parts: list[torch.Tensor] = []
cmp_parts: list[torch.Tensor | None] = []
max_c = 0
+ any_spans = False
for off, n, ti, start_pos in segments:
+ spans = None
+ if spans_by_ti is not None:
+ req_spans = spans_by_ti.get(ti)
+ if req_spans:
+ end = start_pos + n
+ spans = [(s, e) for (s, e) in req_spans if s >= start_pos and e <= end]
+ any_spans = any_spans or bool(spans)
win_global, cmp_global = self._prefill_segment(
- x[:, off:off + n], qr[:, off:off + n], kv[0, off:off + n], ti, start_pos, n
+ x[:, off:off + n], qr[:, off:off + n], kv[0, off:off + n], ti, start_pos, n,
+ spans=spans,
)
win_parts.append(win_global)
cmp_parts.append(cmp_global)
@@ -213,6 +294,11 @@ class Attention(BaseOP):
# is enough to flip a greedy near-tie. bs>1 pads every part to win (the historical
# batched behavior).
n_window = win if len(segments) > 1 else win_parts[0].shape[-1]
+ if any_spans:
+ # Span segments carry win+max_img-wide candidates; pad every part to the widest
+ # so the single flat launch stays rectangular. Text-only batches (any_spans
+ # False) keep the historical widths bit-identically.
+ n_window = max(n_window, *(w.shape[-1] for w in win_parts))
flat = []
for i, (off, n, ti, _start) in enumerate(segments):
wg = win_parts[i]
diff --git a/python/freetoken/models/deepseek_v4/config.py b/python/freetoken/models/deepseek_v4/config.py
index 1005b8a3..9671c682 100644
--- a/python/freetoken/models/deepseek_v4/config.py
+++ b/python/freetoken/models/deepseek_v4/config.py
@@ -13,12 +13,38 @@ The transformers ``AutoConfig`` for ``deepseek_v4`` drops a few keys (e.g.
from __future__ import annotations
+from dataclasses import dataclass
from typing import Any
from freetoken.models.config import DSV4AttentionGroupConfig, ModelConfig, RotaryConfig
from .args import load_args
+# ``<|deepseek_image|>``: the only vocab-level image token. The reference expands each
+# occurrence into a block of ``vocab_size + type`` pseudo ids; here the block keeps this
+# one id and the image embeddings are scattered over the span it occupies.
+IMAGE_TOKEN_ID = 129264
+
+
+@dataclass(frozen=True)
+class DSV4VisionConfig:
+ """The tower's dims, split out of the checkpoint's flat inference config so the engine's
+ section nulling and the loader treat DSV4 like every other family with a vision tower.
+ Field names follow the reference (``inference/model.py`` ModelArgs) one to one."""
+
+ vision_n_layers: int
+ vision_dim: int
+ vision_n_heads: int
+ vision_inter_dim: int
+ vision_patch_size: int
+ vision_rope_theta: float
+ vision_downsample_ratio: int
+ vision_max_n_token: int
+ vision_min_pixels: int
+ vision_max_wh_ratio: int
+ # the language model's hidden size the aligner projects into (ModelArgs.dim)
+ text_dim: int
+
def parse_config(hf_config: Any) -> ModelConfig:
model_path = getattr(hf_config, "_name_or_path", None) or getattr(
@@ -46,6 +72,8 @@ def parse_config(hf_config: Any) -> ModelConfig:
if max_position <= 0:
max_position = int(args.original_seq_len * args.rope_factor)
+ vision_config = parse_vision_config(hf_config)
+
return ModelConfig(
num_layers=args.n_layers,
num_qo_heads=args.n_heads,
@@ -88,9 +116,38 @@ def parse_config(hf_config: Any) -> ModelConfig:
num_kv_heads=1, # MLA-style shared latent (K == V)
head_dim=args.head_dim,
sliding_window=args.window_size,
+ bidirectional_mm_blocks=vision_config is not None,
),
),
+ vision_config=vision_config,
+ image_token_id=IMAGE_TOKEN_ID if vision_config is not None else None,
+ )
+
+
+def parse_vision_config(hf_config: Any) -> DSV4VisionConfig | None:
+ """The tower's dims from the checkpoint config, or None when it carries no vision stack.
+
+ The vision dims sit at the top level of the checkpoint config, next to the language
+ model's, and the engine nulls ``vision_n_layers`` on the copy it hands the parser when
+ this process serves text-only (--text-model-only / --mm-disable). That null is what
+ decides, so the built tower always matches the weights the loader is told to read.
+ """
+ n_layers = int(getattr(hf_config, "vision_n_layers", 0) or 0)
+ if n_layers <= 0:
+ return None
+ return DSV4VisionConfig(
+ vision_n_layers=n_layers,
+ vision_dim=int(hf_config.vision_dim),
+ vision_n_heads=int(hf_config.vision_n_heads),
+ vision_inter_dim=int(hf_config.vision_inter_dim),
+ vision_patch_size=int(hf_config.vision_patch_size),
+ vision_rope_theta=float(hf_config.vision_rope_theta),
+ vision_downsample_ratio=int(hf_config.vision_downsample_ratio),
+ vision_max_n_token=int(hf_config.vision_max_n_token),
+ vision_min_pixels=int(hf_config.vision_min_pixels),
+ vision_max_wh_ratio=int(hf_config.vision_max_wh_ratio),
+ text_dim=int(hf_config.hidden_size),
)
-__all__ = ["parse_config"]
+__all__ = ["IMAGE_TOKEN_ID", "DSV4VisionConfig", "parse_config", "parse_vision_config"]
diff --git a/python/freetoken/models/deepseek_v4/image_processor.py b/python/freetoken/models/deepseek_v4/image_processor.py
new file mode 100644
index 00000000..16742b1e
--- /dev/null
+++ b/python/freetoken/models/deepseek_v4/image_processor.py
@@ -0,0 +1,50 @@
+"""DeepSeek-V4 image preprocessing: one image -> ViT patches plus its grid metadata.
+
+Port of the reference ``inference/image_processor.py:load_image``. The PIL work stays on the
+CPU and must be bit-identical across runs: an image's radix key rides on these patches.
+"""
+
+from __future__ import annotations
+
+import math
+
+import numpy as np
+import torch
+from PIL import Image, ImageOps
+
+from .config import DSV4VisionConfig
+from .vision import safe_resize
+
+
+def load_image(image: Image.Image, vc: DSV4VisionConfig, max_n_token: int | None = None) -> tuple[torch.Tensor, int, int, int, int]:
+ """One image -> ``(patches bf16 [n_vit_h*n_vit_w, 3, p, p], n_vit_h, n_vit_w, n_llm_h, n_llm_w)``.
+
+ ``max_n_token`` lowers the checkpoint's per-image block cap (``--image-max-tokens``).
+ """
+ image = image.convert("RGB")
+ p = vc.vision_patch_size
+ width, height = image.size
+ if vc.vision_max_wh_ratio is not None and width > height * vc.vision_max_wh_ratio:
+ width = height * vc.vision_max_wh_ratio
+ if 0 < width * height < vc.vision_min_pixels:
+ ratio = (vc.vision_min_pixels / (width * height)) ** 0.5
+ width = int(width * ratio)
+ height = int(height * ratio)
+ best_width = math.ceil(width / p) * p
+ best_height = math.ceil(height / p) * p
+ n_llm_h, n_llm_w, best_height, best_width = safe_resize(
+ height, width, best_height, best_width, p, vc.vision_downsample_ratio,
+ vc.vision_max_n_token if max_n_token is None else min(vc.vision_max_n_token, max_n_token),
+ )
+ n_vit_h, n_vit_w = best_height // p, best_width // p
+ if vc.vision_max_wh_ratio is not None and image.width >= vc.vision_max_wh_ratio * image.height:
+ image = image.resize((best_width, best_height))
+ else:
+ image = ImageOps.pad(image, (best_width, best_height), color=(127, 127, 127))
+ x = torch.from_numpy(np.asarray(image, dtype=np.float32)).permute(2, 0, 1) / 255
+ x = ((x - 0.5) / 0.5).to(torch.bfloat16)
+ patches = x.reshape(3, n_vit_h, p, n_vit_w, p).permute(1, 3, 0, 2, 4).reshape(n_vit_h * n_vit_w, 3, p, p)
+ return patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w
+
+
+__all__ = ["load_image"]
diff --git a/python/freetoken/models/deepseek_v4/model.py b/python/freetoken/models/deepseek_v4/model.py
index 9faefb83..0c3441b9 100644
--- a/python/freetoken/models/deepseek_v4/model.py
+++ b/python/freetoken/models/deepseek_v4/model.py
@@ -29,11 +29,12 @@ from freetoken.core import get_global_ctx
from freetoken.kernel.triton.dsv4.hc import hc_post_combine, hc_pre_combine
from freetoken.kernel.triton.dsv4.sinkhorn import hc_split_sinkhorn
from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNorm, VocabParallelEmbedding
-from freetoken.models.blocks import BaseLLMModel
+from freetoken.models.blocks import BaseLLMModel, embed_input_ids
from .args import DeepseekV4Args
from .attention import Attention
from .moe import MoE
+from .vision import COMPRESS_PAD_TO, DSV4Vision
# Re-exports: keep every class/helper previously defined here importable from .model
# (external import stability; the moved definitions live in their own modules).
@@ -85,7 +86,7 @@ class Block(BaseOP):
)
return y.view(shape)
- def prefill_batched(self, x, input_ids, segments, flat_positions):
+ def prefill_batched(self, x, input_ids, segments, flat_positions, spans_by_ti=None):
# Ragged batched prefill (cu_seqlens, no padding; bs >= 1, cold and radix-hit segments
# mixed freely). ``x`` is [1, T, hc_mult, dim] -- the requests' token streams
# concatenated. Per-token ops (HC / norm / MoE) run batched over ALL T tokens (the
@@ -93,10 +94,12 @@ class Block(BaseOP):
# T queries (Attention.forward_ragged), with the stateful compressor/indexer looped per
# request. ``segments`` = [(offset, extend_len, table_idx, start_pos)] off the attention
# metadata; ``flat_positions`` [T] = per-token ABSOLUTE position (batch.positions).
+ # ``spans_by_ti`` maps a page-table row to its image spans: the sparse attention needs
+ # the pairs, the block-tiled backends read the flat ends (batch.mm_block_ends) instead.
residual = x
x, post, comb = self.hc_pre(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)
x = self.attn_norm.forward(x)
- x = self.attn.forward_ragged(x, segments, flat_positions)
+ x = self.attn.forward_ragged(x, segments, flat_positions, spans_by_ti=spans_by_ti)
x = self.hc_post(x, residual, post, comb)
residual = x
@@ -152,6 +155,7 @@ class Transformer(BaseOP):
def prefill_batched(
self, input_ids: torch.Tensor, segments, flat_positions: torch.Tensor,
+ spans_by_ti: dict | None = None,
) -> torch.Tensor:
# Ragged batched prefill (bs >= 1). ``input_ids`` is [1, T] -- the requests' NEW tokens
# concatenated (cu_seqlens, no padding); each request starts at its own cached_len
@@ -163,10 +167,13 @@ class Transformer(BaseOP):
# metadata; ``flat_positions`` [T] is the scheduler-staged batch.positions (per-token
# ABSOLUTE position); the head picks each request's final token off the attention
# metadata -> its next-token logits row.
- h = self.embed.forward(input_ids.view(-1)).view(1, -1, self.args.dim)
+ batch = get_global_ctx().batch
+ # on a chunk that carries image rows, batch.mm_embeds' leading columns replace the
+ # embedding of every row in batch.mm_rows -- the whole image span, sentinels included
+ h = embed_input_ids(self.embed, input_ids.view(-1), batch).view(1, -1, self.args.dim)
h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1)
for layer in self.layers.op_list:
- h = layer.prefill_batched(h, input_ids, segments, flat_positions)
+ h = layer.prefill_batched(h, input_ids, segments, flat_positions, spans_by_ti)
h = self.hc_head(h)
h = self.norm.forward(h)
return self.head.forward(h[0]) # [B, vocab]
@@ -200,6 +207,26 @@ class Transformer(BaseOP):
return self.head.forward(h[:, -1])
+def _image_spans_by_ti(reqs) -> dict | None:
+ """The batch's image spans as ``(IMAGE_START position, block end)`` pairs per page-table row.
+
+ Only this model reads the pairs (built here, not staged on the batch): its sparse
+ attention builds candidate lists per span. The reference ``get_image_visible`` counts
+ visibility within [IMAGE_START, IMAGE_END], so the span starts at the START sentinel
+ (after ``build_image_block``'s alignment lead pads), not at the block's first token.
+ """
+ spans: dict[int, list[tuple[int, int]]] = {}
+ for req in reqs:
+ req_spans = [
+ (lo + COMPRESS_PAD_TO - 1 - lo % COMPRESS_PAD_TO, hi)
+ for item in req.mm_items or ()
+ for lo, hi in item.offsets
+ ]
+ if req_spans:
+ spans[req.table_idx] = req_spans
+ return spans or None
+
+
class DeepseekV4ForCausalLM(BaseLLMModel):
"""Engine adapter: a registered :class:`BaseLLMModel` wrapping the DSV4 transformer.
@@ -211,6 +238,12 @@ class DeepseekV4ForCausalLM(BaseLLMModel):
self._args: DeepseekV4Args = config.dsv4_args
self.model = Transformer(self._args, config.quant, strategy=config.moe_strategy, decode_target=config.decode_target, prefix="model")
self._bound = False
+ # Vision (DeepSeek-V4-Flash-Vision-Exp). config.is_multimodal is the engine's
+ # resolved gate, so a text-only process never builds weights the loader skips.
+ # The whole stack sits under one ``visual.`` mount, so the shared vision-key
+ # filter (VISION_KEY_PREFIXES) covers every image-only parameter.
+ if config.is_multimodal:
+ self.visual = DSV4Vision(config.vision_config, config.hidden_size)
def _ensure_bound(self) -> None:
if self._bound:
@@ -226,6 +259,12 @@ class DeepseekV4ForCausalLM(BaseLLMModel):
block count, freqs) depends on the new pool's geometry, so re-derive it via _ensure_bound."""
self._bound = False
+ def place_encoder_weights(self, mode: str) -> None:
+ self.visual.place_weights(mode)
+
+ def encode(self, item):
+ return self.visual.forward(item)
+
def forward(self) -> torch.Tensor:
self._ensure_bound()
batch = get_global_ctx().batch
@@ -240,6 +279,7 @@ class DeepseekV4ForCausalLM(BaseLLMModel):
# cross requests.
return self.model.prefill_batched(
input_ids.view(1, -1), md.segments, batch.positions.long(),
+ spans_by_ti=_image_spans_by_ti(batch.reqs),
)
# DECODE (bs>=1): per-row position (GPU int tensor -> no host syncs / graph safe). The
# compressed staging cap is the max position any row reaches (eager); a static max_seq-1
diff --git a/python/freetoken/models/deepseek_v4/moe.py b/python/freetoken/models/deepseek_v4/moe.py
index 6b624567..35a316f1 100644
--- a/python/freetoken/models/deepseek_v4/moe.py
+++ b/python/freetoken/models/deepseek_v4/moe.py
@@ -9,24 +9,34 @@ import torch.nn.functional as F
from freetoken.kernel.triton.dsv4.bf16_linear import bf16_linear_fp32
from freetoken.kernel.triton.dsv4.swiglu import fused_swiglu
from freetoken.layers import BaseOP, LinearColParallelMerged, LinearRowParallel, OffloadMoELayer
+from freetoken.mm import MM_PAD_SHIFT_VALUE
from .args import DeepseekV4Args
class Gate(BaseOP):
- """MoE router: sqrtsoftplus scoring + hash routing (first ``n_hash_layers``)."""
+ """MoE router: sqrtsoftplus scoring + hash routing (first ``n_hash_layers``).
+
+ A vision checkpoint gives every layer a ``bias_vl`` too: the expert-selection bias for
+ image rows, which never touches the routing weights. Hash layers keep routing text rows
+ through ``tid2eid`` and pick image rows by ``(scores + bias_vl).topk``, mirroring the
+ reference ``Gate.forward``.
+ """
def __init__(self, layer_id: int, args: DeepseekV4Args):
self.topk = args.n_activated_experts
self.score_func = args.score_func
self.route_scale = args.route_scale
self.hash = layer_id < args.n_hash_layers
+ vl = args.vision_n_layers > 0
self.weight = torch.empty(args.n_routed_experts, args.dim, dtype=torch.bfloat16)
if self.hash:
self.tid2eid = torch.empty(args.vocab_size, args.n_activated_experts, dtype=torch.int64)
- self.bias = None
+ # present on hash layers of a vision checkpoint, loaded but never applied
+ self.bias = torch.empty(args.n_routed_experts, dtype=torch.float32) if vl else None
else:
self.bias = torch.empty(args.n_routed_experts, dtype=torch.float32)
+ self.bias_vl = torch.empty(args.n_routed_experts, dtype=torch.float32) if vl else None
def forward(self, x: torch.Tensor, input_ids: torch.Tensor):
scores = bf16_linear_fp32(x, self.weight)
@@ -37,11 +47,20 @@ class Gate(BaseOP):
else:
scores = F.softplus(scores).sqrt()
original_scores = scores
- if self.bias is not None:
- scores = scores + self.bias
+ # image rows are the content pad ids of an expanded image span
+ image_mask = input_ids >= MM_PAD_SHIFT_VALUE if self.bias_vl is not None else None
if self.hash:
- indices = self.tid2eid[input_ids]
+ if image_mask is None:
+ indices = self.tid2eid[input_ids]
+ else:
+ indices = self.tid2eid[torch.where(image_mask, 0, input_ids)]
+ vl_indices = (scores + self.bias_vl).topk(self.topk, dim=-1)[1]
+ indices = torch.where(image_mask.unsqueeze(-1), vl_indices.to(indices.dtype), indices)
else:
+ if image_mask is None:
+ scores = scores + self.bias
+ else:
+ scores = scores + torch.where(image_mask.unsqueeze(-1), self.bias_vl, self.bias)
indices = scores.topk(self.topk, dim=-1)[1]
weights = original_scores.gather(1, indices)
if self.score_func != "softmax":
diff --git a/python/freetoken/models/deepseek_v4/vision.py b/python/freetoken/models/deepseek_v4/vision.py
new file mode 100644
index 00000000..44549460
--- /dev/null
+++ b/python/freetoken/models/deepseek_v4/vision.py
@@ -0,0 +1,336 @@
+"""DeepSeek-V4 vision tower, aligner and image-block layout.
+
+Port of the reference ``inference/vision.py`` / ``inference/image_processor.py`` from
+deepseek-ai/DeepSeek-V4-Flash-Vision-Exp. The tower, the aligner and the four sentinel
+vectors hang under one :class:`DSV4Vision` container the wrapper mounts as ``visual``,
+so their state-dict keys sit behind the ``visual.`` prefix every VLM family filters on
+(the checkpoint's own unprefixed names, ``vision.patch_embed.proj.weight``,
+``aligner.w1.weight``, ``image_start``, ..., are renamed on load); every tensor is bf16
+exactly as stored.
+
+Image tokens occupy plain sequential positions: the 2-D structure lives inside the
+tower (2-D RoPE over the patch grid) and in the N-layout of the token block the
+:func:`build_image_block` / :func:`assemble_block` pair renders. All block embeddings,
+sentinels included, come from the vision side, so the text embedding row of the
+placeholder id is never read -- the span is overwritten before the first layer.
+"""
+
+from __future__ import annotations
+
+import math
+from functools import lru_cache
+
+import torch
+import torch.nn.functional as F
+
+from freetoken.layers import BaseOP, OPList
+from freetoken.models.weight_stream import BlockWeightStreamer
+
+from .config import DSV4VisionConfig
+
+# Sentinel token types inside an image block (reference inference/image_processor.py).
+IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5)
+# An image block is padded so its first row token lands on a compressor stride boundary.
+COMPRESS_PAD_TO = 4
+
+
+@lru_cache(8)
+def get_vision_cos_sin(n_h: int, n_w: int, dim: int, theta: float):
+ inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
+ hpos = torch.arange(n_h).unsqueeze(1).expand(n_h, n_w)
+ wpos = torch.arange(n_w).unsqueeze(0).expand(n_h, n_w)
+ freqs = torch.stack([hpos, wpos], dim=-1).reshape(-1, 2, 1).float() * inv_freq
+ freqs = freqs.flatten(1)
+ return freqs.cos().unsqueeze(1), freqs.sin().unsqueeze(1)
+
+
+def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+ dtype = x.dtype
+ x1, x2 = x.float().chunk(2, dim=-1)
+ return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(dtype)
+
+
+class _Linear(BaseOP):
+ """Linear in checkpoint-native layout: ``weight`` [out, in] plus an optional ``bias``."""
+
+ def __init__(self, in_features: int, out_features: int, bias: bool = True):
+ self.weight = torch.empty(out_features, in_features, dtype=torch.bfloat16)
+ self.bias = torch.empty(out_features, dtype=torch.bfloat16) if bias else None
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return F.linear(x, self.weight, self.bias)
+
+
+class RMSNorm(BaseOP):
+ def __init__(self, dim: int, eps: float = 1e-6):
+ self.eps = eps
+ self.weight = torch.empty(dim, dtype=torch.bfloat16)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ dtype = x.dtype
+ x = x.float()
+ x = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.eps)
+ return (self.weight * x).to(dtype)
+
+
+class PatchEmbed(BaseOP):
+ def __init__(self, patch_size: int, dim: int):
+ self.proj = _Linear(3 * patch_size**2, dim)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return self.proj.forward(x.flatten(1))
+
+
+class VisionAttention(BaseOP):
+ def __init__(self, dim: int, n_heads: int):
+ self.n_heads = n_heads
+ self.head_dim = dim // n_heads
+ self.wqkv = _Linear(dim, 3 * dim)
+ self.wo = _Linear(dim, dim)
+
+ def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+ n = x.size(0)
+ q, k, v = (t.view(n, self.n_heads, self.head_dim) for t in self.wqkv.forward(x).chunk(3, dim=-1))
+ q = apply_rotary(q, cos, sin)
+ k = apply_rotary(k, cos, sin)
+ o = F.scaled_dot_product_attention(q.transpose(0, 1), k.transpose(0, 1), v.transpose(0, 1))
+ return self.wo.forward(o.transpose(0, 1).reshape(n, -1))
+
+
+class VisionMLP(BaseOP):
+ def __init__(self, dim: int, inter_dim: int):
+ self.w1 = _Linear(dim, 2 * inter_dim, bias=False)
+ self.w2 = _Linear(inter_dim, dim, bias=False)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ gate, up = self.w1.forward(x).chunk(2, dim=-1)
+ return self.w2.forward(F.silu(gate) * up)
+
+
+class VisionBlock(BaseOP):
+ def __init__(self, dim: int, n_heads: int, inter_dim: int, eps: float):
+ self.norm1 = RMSNorm(dim, eps)
+ self.attn = VisionAttention(dim, n_heads)
+ self.norm2 = RMSNorm(dim, eps)
+ self.mlp = VisionMLP(dim, inter_dim)
+
+ def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
+ x = x + self.attn.forward(self.norm1.forward(x), cos, sin)
+ return x + self.mlp.forward(self.norm2.forward(x))
+
+
+class DSV4VisionTower(BaseOP):
+ """Native-resolution ViT: bidirectional attention inside one image, 2-D RoPE over its patch grid."""
+
+ def __init__(self, vc: DSV4VisionConfig):
+ self.rope_dim = vc.vision_dim // vc.vision_n_heads // 2
+ self.rope_theta = vc.vision_rope_theta
+ self.patch_embed = PatchEmbed(vc.vision_patch_size, vc.vision_dim)
+ self.blocks = OPList(
+ [
+ VisionBlock(vc.vision_dim, vc.vision_n_heads, vc.vision_inter_dim, eps=1e-6)
+ for _ in range(vc.vision_n_layers)
+ ]
+ )
+ self.norm = RMSNorm(vc.vision_dim, 1e-6)
+ self._streamer: BlockWeightStreamer | None = None
+
+ def place_weights(self, mode: str) -> None:
+ """gpu: every tensor resident; host: the block stack in pinned banks, two blocks streamed at a time (patch embed and final norm stay resident)."""
+ if mode == "host" and self._streamer is None:
+ self._streamer = BlockWeightStreamer(self.blocks.op_list, self.patch_embed.proj.weight.device)
+ elif mode == "gpu" and self._streamer is not None:
+ self._streamer.unstream()
+ self._streamer = None
+ elif mode not in ("gpu", "host"):
+ raise ValueError(f"unknown vision weight placement {mode!r}")
+
+ def _blocks(self):
+ if self._streamer is None:
+ return enumerate(self.blocks.op_list)
+ return self._streamer.blocks(self.blocks.op_list)
+
+ def forward(self, patches: torch.Tensor, n_h: int, n_w: int) -> torch.Tensor:
+ patches = patches.to(self.patch_embed.proj.weight.dtype)
+ x = self.patch_embed.forward(patches)
+ cos, sin = get_vision_cos_sin(n_h, n_w, self.rope_dim, self.rope_theta)
+ cos, sin = cos.to(x.device), sin.to(x.device)
+ for _, block in self._blocks():
+ x = block.forward(x, cos, sin)
+ return self.norm.forward(x)
+
+
+class DSV4Aligner(BaseOP):
+ """3x3/stride-3 merge of every 3x3 patch grid into one token, then a 2-layer erf-GELU MLP."""
+
+ def __init__(self, vc: DSV4VisionConfig):
+ self.downsample_ratio = vc.vision_downsample_ratio
+ in_dim = vc.vision_dim * self.downsample_ratio**2
+ self.w1 = _Linear(in_dim, vc.text_dim)
+ self.w2 = _Linear(vc.text_dim, vc.text_dim)
+
+ def forward(self, x: torch.Tensor, n_h: int, n_w: int) -> torch.Tensor:
+ dtype = self.w1.weight.dtype
+ x = x.to(dtype)
+ r = self.downsample_ratio
+ x = x.view(n_h, n_w, -1).permute(2, 0, 1)
+ x = F.pad(x, (0, -n_w % r, 0, -n_h % r))
+ x = F.unfold(x.unsqueeze(0), r, stride=r).squeeze(0).transpose(0, 1)
+ return self.w2.forward(F.gelu(self.w1.forward(x)))
+
+
+class DSV4Vision(BaseOP):
+ """The wrapper's whole vision stack under one ``visual.`` mount, like the V4.1 sibling."""
+
+ def __init__(self, vc: DSV4VisionConfig, text_dim: int):
+ self.vision = DSV4VisionTower(vc)
+ self.aligner = DSV4Aligner(vc)
+ for name in ("image_start", "image_end", "image_newline", "image_pad"):
+ setattr(self, name, torch.empty(text_dim, dtype=torch.bfloat16))
+
+ def place_weights(self, mode: str) -> None:
+ self.vision.place_weights(mode)
+
+ @torch.inference_mode()
+ def forward(self, item):
+ """One image -> the embeddings of its whole token block, sentinels included.
+
+ The block's layout is a function of where it lands in the prompt, so the item
+ carries the grid and the offset; ``build_image_block`` re-derives the same
+ ``(types, perm)`` the processor tokenized with.
+ """
+ device = self.vision.patch_embed.proj.weight.device
+ patches = item.feature.to(device, non_blocking=True)
+ aligned = self.aligner.forward(
+ self.vision.forward(patches, item.n_vit_h, item.n_vit_w), item.n_vit_h, item.n_vit_w
+ )
+ types, perm = build_image_block(item.n_llm_h, item.n_llm_w, item.start)
+ sentinels = torch.stack(
+ [self.image_start, self.image_pad, self.image_pad, self.image_newline, self.image_end]
+ )
+ return assemble_block(aligned, types, perm, sentinels)
+
+
+def grid_tokens(best_height: int, best_width: int, patch_size: int, downsample_ratio: int):
+ """LLM-token footprint of a resized image: the N-layout row/align padding included."""
+ n_llm_h = math.ceil((best_height // patch_size) / downsample_ratio)
+ n_llm_w = math.ceil((best_width // patch_size) / downsample_ratio)
+ num_tokens = n_llm_h * (n_llm_w + 1) + 2
+ if n_llm_h % 2 == 1:
+ num_tokens += n_llm_w + 1
+ num_tokens += (n_llm_h + 1) // 2 * (n_llm_w + 1) % 2 * 2
+ return n_llm_h, n_llm_w, num_tokens
+
+
+def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):
+ r = height / width
+ max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5
+ max_h_float = max_w_float * r
+ if max_w_float < 1.0:
+ max_w = 1
+ max_h = (max_n_token - 2) // (max_w + 1)
+ if max_h % 2 == 1:
+ max_h -= 1
+ best_width = max_w * patch_size * downsample_ratio
+ best_height = max_h * patch_size * downsample_ratio
+ elif max_h_float < 2.0:
+ max_h = 2
+ max_w = ((max_n_token - 2) // max_h) - 1
+ assert max_w > 1
+ best_width = max_w * patch_size * downsample_ratio
+ best_height = max_h * patch_size * downsample_ratio
+ else:
+ max_w = math.floor(max_w_float)
+ max_h = math.floor(max_h_float)
+ if max_h % 2 == 1:
+ max_h -= 1
+ beta = min(max_w * patch_size * downsample_ratio / width, max_h * patch_size * downsample_ratio / height)
+ best_width = math.floor(width * beta / patch_size) * patch_size
+ best_height = math.floor(height * beta / patch_size) * patch_size
+ n_llm_h, n_llm_w, num_tokens = grid_tokens(best_height, best_width, patch_size, downsample_ratio)
+ return n_llm_h, n_llm_w, best_height, best_width, num_tokens
+
+
+def safe_resize(height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token):
+ """Largest grid whose block still fits ``max_n_token`` once the alignment pads are added."""
+ max_n_token -= COMPRESS_PAD_TO - 1
+ n_llm_h, n_llm_w, num_tokens = grid_tokens(best_height, best_width, patch_size, downsample_ratio)
+ budget = max_n_token
+ while num_tokens > max_n_token:
+ n_llm_h, n_llm_w, best_height, best_width, num_tokens = solve_resize_ratio(
+ height, width, patch_size, downsample_ratio, budget
+ )
+ budget -= 1
+ return n_llm_h, n_llm_w, best_height, best_width
+
+
+def build_image_block(n_llm_h: int, n_llm_w: int, start_pos: int):
+ """The block's token types in their final order, plus the aligner-row order of its IMAGE slots.
+
+ ``types`` [n_out] places every token of the block (lead pads, START, the row-interleaved
+ IMAGE tokens, trailing pads, END). The reference streams them as ``vocab_size + types``;
+ here the span carries the image's content pad id and the assembled embeddings are
+ scattered over it. ``perm`` [G] maps aligner outputs (row-major grid) into the IMAGE slots.
+
+ ``start_pos`` is the block's absolute position in the prompt: the lead-pad count aligns
+ the first row token to a compressor stride boundary, so the layout -- and the block's
+ length -- depend on where the image lands.
+ """
+ compress_pad = COMPRESS_PAD_TO - 1 - start_pos % COMPRESS_PAD_TO
+ pad_h = n_llm_h % 2
+ rows = n_llm_h + pad_h
+ row_len = n_llm_w + 1
+ pad_last = rows // 2 * row_len % 2 * 2
+ types = torch.tensor(
+ ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h + [IMAGE_PAD] * (row_len * pad_h),
+ dtype=torch.int64,
+ )
+ order = torch.arange(rows * row_len).view(rows // 2, 2, row_len).transpose(1, 2).reshape(-1)
+ image_idx = torch.full((rows * row_len,), -1, dtype=torch.int64)
+ image_idx.view(rows, row_len)[:n_llm_h, :n_llm_w] = torch.arange(n_llm_h * n_llm_w).view(n_llm_h, n_llm_w)
+ perm = image_idx[order]
+ perm = perm[perm >= 0]
+ types = torch.cat(
+ [
+ torch.full((compress_pad,), IMAGE_PAD, dtype=torch.int64),
+ torch.tensor([IMAGE_START]),
+ types[order],
+ torch.full((pad_last,), IMAGE_PAD, dtype=torch.int64),
+ torch.tensor([IMAGE_END]),
+ ]
+ )
+ return types, perm
+
+
+def assemble_block(
+ aligned: torch.Tensor,
+ types: torch.Tensor,
+ perm: torch.Tensor,
+ sentinels: torch.Tensor,
+) -> torch.Tensor:
+ """Block embeddings [n_out, dim] from the aligner rows and the learned sentinels.
+
+ ``sentinels`` [5, dim] is ``(start, pad, pad, newline, end)``, indexable by the type
+ enum exactly like the reference ``merge_image_embeddings``.
+ """
+ block = sentinels[types.to(sentinels.device)]
+ block[types == IMAGE] = aligned[perm.to(aligned.device)]
+ return block
+
+
+__all__ = [
+ "COMPRESS_PAD_TO",
+ "DSV4Aligner",
+ "DSV4Vision",
+ "DSV4VisionTower",
+ "IMAGE",
+ "IMAGE_END",
+ "IMAGE_NEW_LINE",
+ "IMAGE_PAD",
+ "IMAGE_START",
+ "assemble_block",
+ "build_image_block",
+ "grid_tokens",
+ "safe_resize",
+ "solve_resize_ratio",
+]
diff --git a/python/freetoken/models/deepseek_v4/weight.py b/python/freetoken/models/deepseek_v4/weight.py
index 4728f7f9..fbcddb1c 100644
--- a/python/freetoken/models/deepseek_v4/weight.py
+++ b/python/freetoken/models/deepseek_v4/weight.py
@@ -78,12 +78,16 @@ def iter_weights(
*,
include_moe_experts: bool = True,
include_non_moe: bool = True,
+ include_vision: bool = True,
):
"""Stream resident (non-expert) weights as ``(name, tensor)`` keyed to engine params.
Routed MXFP4 experts come from the offload cache, so ``include_moe_experts`` must be
False (DeepSeek-V4 only runs ``--moe-strategy offload``). Tensors yielded in checkpoint
dtype (fp8 + e8m0 preserved); ``wo_a`` dequantized to bf16 to match the reference einsum.
+ ``include_vision`` (the engine's active-encoder set) drops the ``visual.*`` tower
+ tensors; the router biases stay in either way, since the gate builds
+ them on any vision checkpoint.
"""
if include_moe_experts:
raise ValueError(
@@ -112,6 +116,9 @@ def iter_weights(
for nm in ("hc_head_fn", "hc_head_base", "hc_head_scale"):
yield f"model.{nm}", get(nm)
+ if include_vision:
+ yield from _iter_vision(reader, args)
+
for L in range(args.n_layers):
a = f"layers.{L}.attn"
m = f"model.{a}"
@@ -147,8 +154,13 @@ def iter_weights(
yield f"model.{g}.weight", get(f"{g}.weight")
if L < args.n_hash_layers:
yield f"model.{g}.tid2eid", get(f"{g}.tid2eid")
+ if args.vision_n_layers > 0:
+ # a vision checkpoint gives its hash layers a text bias too
+ yield f"model.{g}.bias", get(f"{g}.bias")
else:
yield f"model.{g}.bias", get(f"{g}.bias")
+ if args.vision_n_layers > 0:
+ yield f"model.{g}.bias_vl", get(f"{g}.bias_vl")
for proj in ("w1", "w2", "w3"):
src = f"layers.{L}.ffn.shared_experts.{proj}"
yield from linear(src, f"model.{src}")
@@ -162,6 +174,58 @@ def iter_weights(
reader.close()
+# --------------------------------------------------------------------------------------
+# Vision tower.
+# --------------------------------------------------------------------------------------
+# The wrapper mounts the tower, the aligner and the sentinels under one ``visual``
+# container, so every engine-side name carries the ``visual.`` prefix the shared
+# VISION_KEY_PREFIXES filter matches; the checkpoint's unprefixed names
+# (``vision.*``, ``aligner.*``, ``image_*``) are renamed here on load.
+_VISION_BLOCK_SUFFIXES = (
+ "norm1.weight",
+ "attn.wqkv.weight",
+ "attn.wqkv.bias",
+ "attn.wo.weight",
+ "attn.wo.bias",
+ "norm2.weight",
+ "mlp.w1.weight",
+ "mlp.w2.weight",
+)
+_VISION_TAIL = (
+ "vision.patch_embed.proj.weight",
+ "vision.patch_embed.proj.bias",
+ "vision.norm.weight",
+ "aligner.w1.weight",
+ "aligner.w1.bias",
+ "aligner.w2.weight",
+ "aligner.w2.bias",
+ "image_start",
+ "image_end",
+ "image_newline",
+ "image_pad",
+)
+
+
+def _iter_vision(reader, args: DeepseekV4Args):
+ if args.vision_n_layers <= 0:
+ return
+ for L in range(args.vision_n_layers):
+ for suffix in _VISION_BLOCK_SUFFIXES:
+ yield f"visual.vision.blocks.{L}.{suffix}", reader.get(f"vision.blocks.{L}.{suffix}")
+ for name in _VISION_TAIL:
+ yield f"visual.{name}", reader.get(name)
+
+
+def iter_vision_weights(model_path: str, device):
+ """The vision tower alone, for the engine's encoder-only load and the FTW converter."""
+ args = load_args(model_path, max_batch_size=1)
+ reader = _ShardReader(model_path, _weight_map(model_path), device)
+ try:
+ yield from _iter_vision(reader, args)
+ finally:
+ reader.close()
+
+
# --------------------------------------------------------------------------------------
# Routed MXFP4 expert pieces.
# --------------------------------------------------------------------------------------
@@ -212,4 +276,4 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: bo
return per_expert_pieces(_serial(), locate, tensors_per_expert=6)
-__all__ = ["iter_weights", "iter_expert_pieces"]
+__all__ = ["iter_weights", "iter_expert_pieces", "iter_vision_weights"]
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index fe150ddc..678424ff 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -84,6 +84,10 @@ _MINIMAX_M3_PACKED = _DENSE_PACKED + (
) + _EXPERTS_W123_PACKED
_MINIMAX_M3_PROCESSOR = "freetoken.mm.processors.minimax_m3:MiniMaxM3MMProcessor"
_MINIMAX_M3_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)
+# DeepSeek-V4's vision dims sit at the top level of the checkpoint config, next to the
+# language model's rather than under a section of their own.
+_DSV4_PROCESSOR = "freetoken.mm.processors.deepseek_v4:DSV4MMProcessor"
+_DSV4_ENCODERS = (EncoderSpec("vision", "vision_n_layers", ("image",)),)
_MODEL_REGISTRY: dict[str, ModelSpec] = {
"LlamaForCausalLM": ModelSpec(
@@ -153,8 +157,13 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
# the checkpoint has no ``model.`` root
checkpoint_roots=(("model.layers", "layers"), ("model.head", "head")),
packed_modules_mapping=_EXPERTS_W123_PACKED,
- # the head, the KV compressors and the indexer's scorer ship bf16; the fp8 config has no modules_to_not_convert
- unquantized_modules=("head", "*.compressor.wkv", "*.compressor.wgate", "*.indexer.weights_proj"),
+ # the head, the KV compressors, the indexer's scorer and the vision tower + aligner ship bf16;
+ # the fp8 config has no modules_to_not_convert (patterns match checkpoint names, like V4.1)
+ unquantized_modules=(
+ "head", "*.compressor.wkv", "*.compressor.wgate", "*.indexer.weights_proj", "vision.*", "aligner.*",
+ ),
+ mm_processor=_DSV4_PROCESSOR,
+ encoders=_DSV4_ENCODERS,
),
# DeepSeek-V4.1-Flash (model_type deepseek_v41): multimodal wrapper config (text tower in
# text_config). The checkpoint has no ``model.`` root; the compressors, the indexer's key projection /
diff --git a/scripts/ftw_hotfix.py b/scripts/ftw_hotfix.py
index 9640c7b7..12574651 100644
--- a/scripts/ftw_hotfix.py
+++ b/scripts/ftw_hotfix.py
@@ -264,7 +264,7 @@ def is_checkpoint_tower_name(name: str) -> bool:
if head[0] == "model" and len(head) > 1:
head = head[1:]
return ("vision" in head[0] or "visual" in head[0] or head[0] in _TOWER_SEGMENTS
- or head[0] in ("aligner", "image_start", "image_end", "image_newline")
+ or head[0] in ("aligner", "image_start", "image_end", "image_newline", "image_pad")
or name.endswith(".ffn.gate.bias_vl"))
diff --git a/tests/README.md b/tests/README.md
index 76b2b1fb..adc87035 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -63,6 +63,8 @@ checkpoint is set:
| `FREETOKEN_REBUILD_TEST_MODEL` | `e2e/test_cache_rebuild.py` — a SMALL local model dir; boots a real server (falls back to `FREETOKEN_TEST_MODEL`) |
| `FREETOKEN_GEMMA4_GGUF_GLOB` | `models/test_gemma4_gguf_rope.py` — glob matching a local gemma-4 GGUF file |
| `FREETOKEN_TEST_MODELS_ROOT` | `models/test_quant_config.py` — directory whose checkpoints the stored-tensor scheme scan checks (default `/mnt/nvme/models`) |
+| `FREETOKEN_DSV4_VL_MODEL` | `tokenizer/test_mm_tokenize.py` - local DeepSeek-V4 vision checkpoint |
+| `FREETOKEN_DSV4_VISION_CKPT` | `models/test_dsv4_vision.py` - the same checkpoint, for the parity tests against its `inference/` reference code |
`test_aime.py` takes its sampling protocol from the checkpoint's own
`generation_config.json` (pass@N at the recommended temperature, or a single greedy
diff --git a/tests/checkpoint/test_ftw_hotfix_vision.py b/tests/checkpoint/test_ftw_hotfix_vision.py
index eb0c592f..ef9225a9 100644
--- a/tests/checkpoint/test_ftw_hotfix_vision.py
+++ b/tests/checkpoint/test_ftw_hotfix_vision.py
@@ -37,6 +37,7 @@ def hotfix():
("vision.blocks.0.norm1.weight", True),
("aligner.w1.weight", True),
("image_newline", True),
+ ("image_pad", True),
("layers.3.ffn.gate.bias_vl", True),
("model.embed_audio.embedding_projection.weight", False),
("model.language_model.layers.0.self_attn.q_proj.weight", False),
diff --git a/tests/mm/test_processor.py b/tests/mm/test_processor.py
index a9a50e3c..9562b366 100644
--- a/tests/mm/test_processor.py
+++ b/tests/mm/test_processor.py
@@ -9,7 +9,7 @@ import pytest
import torch
from freetoken.message import MMItem
-from freetoken.mm import MM_PAD_SHIFT_VALUE
+from freetoken.mm import MM_PAD_SHIFT_VALUE, mm_pad_value
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import MMProcessor, PromptReplacement, image_positions
from freetoken.mm.processors.gemma4 import Gemma4MMProcessor, Gemma4UnifiedMMProcessor
@@ -17,6 +17,7 @@ from freetoken.mm.processors.glm5_next import Glm5NextMMProcessor
from freetoken.mm.processors.muse_glimmer import MuseGlimmerMMProcessor
from freetoken.mm.processors.minimax_m3 import IMAGE_END_ID, IMAGE_START_ID, MiniMaxM3MMProcessor
from freetoken.mm.processors.qwen_vl import QwenVLMMProcessor
+from freetoken.mm.processors.deepseek_v4 import DSV4MMProcessor
PLACEHOLDER = 7
BOI, SOFT, EOI = 90, 91, 92
@@ -415,3 +416,103 @@ def test_registry_resolves_by_architecture(monkeypatch):
assert get_mm_processor("/text-only") is None
assert get_mm_processor("/unknown-vlm") is None
assert get_mm_processor("/missing") is None # a config that fails to load means no vision
+
+
+def _dsv4_config(**over):
+ """The vision dims sit at the top level of a deepseek_v4 checkpoint config."""
+ base = dict(
+ vision_n_layers=32,
+ vision_dim=1024,
+ vision_n_heads=16,
+ vision_inter_dim=2816,
+ vision_patch_size=14,
+ vision_rope_theta=10000.0,
+ vision_downsample_ratio=3,
+ vision_max_n_token=384,
+ vision_min_pixels=147456,
+ vision_max_wh_ratio=8,
+ hidden_size=4096,
+ )
+ return SimpleNamespace(**{**base, **over})
+
+
+def _dsv4_image(width, height):
+ from PIL import Image
+
+ return Image.new("RGB", (width, height))
+
+
+def test_dsv4_block_length_and_hash_follow_the_insertion_offset():
+ from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID
+ from freetoken.models.deepseek_v4.vision import build_image_block
+
+ proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())
+ assert proc.placeholder == [IMAGE_TOKEN_ID]
+ (item,) = proc.process([_dsv4_image(640, 480)])
+ assert (item.n_vit_h, item.n_vit_w, item.n_llm_h, item.n_llm_w) == (35, 46, 12, 16)
+
+ repl = proc.prompt_replacement(item, 0)
+ at_zero, hash_zero = len(repl.full), item.hash
+ assert at_zero == len(build_image_block(12, 16, 0)[0]) == 209
+ assert repl.full == [IMAGE_TOKEN_ID] * at_zero
+ # the block's sentinels are embeddings too, so the whole span takes image rows
+ assert repl.embed_spans() == [[0, at_zero]]
+ assert item.pad_value == mm_pad_value(item.hash) and item.pad_value >= MM_PAD_SHIFT_VALUE
+
+ # the lead pads align the block to a compression stride, so a later offset is a shorter
+ # block AND a different content hash: one alignment's rows must never serve another's
+ repl = proc.prompt_replacement(item, 2)
+ assert len(repl.full) == at_zero - 2 and item.hash != hash_zero
+ assert item.pad_value == mm_pad_value(item.hash) >= MM_PAD_SHIFT_VALUE
+
+
+def test_dsv4_dummy_item_is_the_smallest_encodable_block():
+ from freetoken.models.deepseek_v4.vision import build_image_block
+
+ proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())
+ (dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
+ dummy.validate()
+ assert dummy.feature.shape == (36, 3, 14, 14) # a 6x6 patch grid, one 2x2 merge grid
+ assert dummy.num_tokens == len(build_image_block(2, 2, 0)[0]) == 13
+
+
+def test_a_text_only_checkpoint_config_has_no_dsv4_processor():
+ proc = DSV4MMProcessor
+ with pytest.raises(ValueError):
+ proc(_dsv4_config(vision_n_layers=0), "/nonexistent", MultimodalConfig())
+
+
+def test_dsv4_apply_passes_each_block_the_offset_it_lands_at():
+ """Only the DSV4 block depends on its position, so its own apply computes the offset."""
+ from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID
+ from freetoken.models.deepseek_v4.vision import build_image_block
+
+ proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())
+ buf = io.BytesIO()
+ _dsv4_image(640, 480).save(buf, format="PNG")
+ raw = buf.getvalue()
+ ids = torch.tensor([1, IMAGE_TOKEN_ID, 2, IMAGE_TOKEN_ID, 3], dtype=torch.int32)
+ out = proc.apply(ids, [raw, raw])
+ first, second = out.mm_items
+ assert (first.n_llm_h, first.n_llm_w) == (12, 16)
+ # the first placeholder sits at 1, the second after the text between plus the first block
+ assert first.start == first.offsets[0][0] == 1
+ assert second.start == second.offsets[0][0]
+ assert first.num_tokens == len(build_image_block(12, 16, first.start)[0])
+ assert second.num_tokens == len(build_image_block(12, 16, second.start)[0])
+ # apply fills the embedding slots with each block's content pad id, not the placeholder
+ for item in (first, second):
+ lo, hi = item.offsets[0]
+ assert set(out.input_ids[lo:hi].tolist()) == {item.pad_value}
+ assert IMAGE_TOKEN_ID not in out.input_ids.tolist()
+
+
+def test_dsv4_image_max_tokens_lowers_the_checkpoints_block_cap():
+ from freetoken.models.deepseek_v4.vision import build_image_block
+
+ big = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())
+ capped = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig(image_max_tokens=64))
+ (a,) = big.process([_dsv4_image(1600, 1200)])
+ (b,) = capped.process([_dsv4_image(1600, 1200)])
+ assert b.n_llm_h * b.n_llm_w < a.n_llm_h * a.n_llm_w
+ assert len(build_image_block(b.n_llm_h, b.n_llm_w, 0)[0]) <= 64
diff --git a/tests/models/test_dsv4_vision.py b/tests/models/test_dsv4_vision.py
new file mode 100644
index 00000000..68a3f0ab
--- /dev/null
+++ b/tests/models/test_dsv4_vision.py
@@ -0,0 +1,335 @@
+"""DeepSeek-V4 image preprocessing, block layout and the tiny vision tower (no checkpoint).
+
+The block layout is what makes an image's token count depend on where it lands in the
+prompt, so it is pinned here against hand-written expectations; the checkpoint-gated test
+at the bottom checks the same functions against the reference ``inference/`` code.
+"""
+
+from __future__ import annotations
+
+import io
+import os
+
+import numpy as np
+import pytest
+import torch
+from PIL import Image
+
+from freetoken.models.deepseek_v4.config import DSV4VisionConfig, parse_config, parse_vision_config
+from freetoken.models.deepseek_v4.image_processor import load_image
+from freetoken.models.deepseek_v4.vision import (
+ COMPRESS_PAD_TO,
+ DSV4Aligner,
+ DSV4VisionTower,
+ IMAGE,
+ IMAGE_END,
+ IMAGE_NEW_LINE,
+ IMAGE_PAD,
+ IMAGE_START,
+ assemble_block,
+ build_image_block,
+)
+
+checkpoint_path = os.environ.get("FREETOKEN_DSV4_VISION_CKPT")
+needs_checkpoint = pytest.mark.skipif(
+ not checkpoint_path, reason="needs FREETOKEN_DSV4_VISION_CKPT pointing at the vision checkpoint"
+)
+
+_HF_CONFIG = {
+ "vision_n_layers": 32,
+ "vision_dim": 1024,
+ "vision_n_heads": 16,
+ "vision_inter_dim": 2816,
+ "vision_patch_size": 14,
+ "vision_rope_theta": 10000.0,
+ "vision_downsample_ratio": 3,
+ "vision_max_n_token": 384,
+ "vision_min_pixels": 147456,
+ "vision_max_wh_ratio": 8,
+ "hidden_size": 4096,
+}
+
+
+def _hf_config(**over):
+ from types import SimpleNamespace
+
+ return SimpleNamespace(**{**_HF_CONFIG, **over})
+
+
+def _vc(**over) -> DSV4VisionConfig:
+ """A tower small enough to run: 2 patch-2 blocks over a 2x2 merge grid."""
+ base = dict(
+ vision_n_layers=2,
+ vision_dim=16,
+ vision_n_heads=2,
+ vision_inter_dim=32,
+ vision_patch_size=4,
+ vision_rope_theta=10000.0,
+ vision_downsample_ratio=2,
+ vision_max_n_token=384,
+ vision_min_pixels=147456,
+ vision_max_wh_ratio=8,
+ text_dim=64,
+ )
+ return DSV4VisionConfig(**{**base, **over})
+
+
+def _png(width, height, seed=0):
+ rng = np.random.default_rng(seed)
+ array = (rng.random((height, width, 3)) * 255).astype(np.uint8)
+ buf = io.BytesIO()
+ Image.fromarray(array, "RGB").save(buf, format="PNG")
+ return buf.getvalue()
+
+
+def test_a_vision_checkpoint_config_becomes_the_tower_dims_and_a_text_one_does_not():
+ vc = parse_vision_config(_hf_config())
+ assert (vc.vision_n_layers, vc.vision_dim, vc.text_dim) == (32, 1024, 4096)
+ # the engine nulls the section on the copy it hands the parser when it serves text-only
+ assert parse_vision_config(_hf_config(vision_n_layers=None)) is None
+ assert parse_vision_config(_hf_config(vision_n_layers=0)) is None
+
+
+def test_block_layout_interleaves_rows_and_pads_to_the_compression_stride():
+ types, perm = build_image_block(2, 2, 0)
+ # 3 lead pads put the first row token on a COMPRESS_PAD_TO (4) boundary, then START,
+ # the N-layout rows (column pairs: 2 rows of 2 IMAGE tokens plus their row markers),
+ # the trailing parity pad, END.
+ assert types.tolist() == (
+ [IMAGE_PAD] * 3
+ + [IMAGE_START]
+ + [IMAGE, IMAGE, IMAGE, IMAGE, IMAGE_NEW_LINE, IMAGE_NEW_LINE]
+ + [IMAGE_PAD] * 2
+ + [IMAGE_END]
+ )
+ # row-major aligner outputs 0..3 land in the IMAGE slots in N-layout order
+ assert perm.tolist() == [0, 2, 1, 3]
+
+
+def test_the_block_shrinks_by_one_token_per_lead_pad_the_offset_absorbs():
+ # the lead pads are what aligns the block, so the same image at a later offset is a
+ # SHORTER span: the count cannot be a property of the image alone
+ lengths = [len(build_image_block(2, 2, start)[0]) for start in range(4)]
+ assert lengths == [13, 12, 11, 10]
+ # an odd row count adds a marker row, which is not pad bytes
+ assert [len(build_image_block(3, 2, start)[0]) for start in range(4)] == [17, 16, 15, 14]
+
+
+def test_assemble_block_puts_the_sentinels_and_the_aligner_rows_where_the_types_say():
+ types, perm = build_image_block(2, 2, 0)
+ sentinels = torch.stack([torch.full((4,), float(i)) for i in range(5)])
+ aligned = torch.arange(4 * 4, dtype=torch.float32).view(4, 4) * 10
+ block = assemble_block(aligned, types, perm, sentinels)
+ assert block.shape == (len(types), 4)
+ # sentinels index the type enum, so START/PAD/END rows carry 0/1/4 and the IMAGE rows
+ # carry the aligner output perm says they take
+ assert block[0].tolist() == [1.0] * 4 # lead pad
+ assert block[3].tolist() == [0.0] * 4 # IMAGE_START
+ assert block[-1].tolist() == [4.0] * 4 # IMAGE_END
+ image_rows = block[types == IMAGE]
+ assert image_rows.tolist() == aligned[perm].tolist()
+
+
+def test_preprocessing_resizes_to_the_merge_grid_and_normalizes_like_the_reference():
+ # min_pixels 0 keeps the input size: the reference upscales below it
+ vc = _vc(vision_min_pixels=0)
+ patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = load_image(
+ Image.open(io.BytesIO(_png(80, 64))), vc
+ )
+ assert (n_vit_h, n_vit_w, n_llm_h, n_llm_w) == (16, 20, 8, 10)
+ assert patches.shape == (n_vit_h * n_vit_w, 3, 4, 4)
+ assert patches.dtype == torch.bfloat16
+ assert float(patches.min()) >= -1.0 and float(patches.max()) <= 1.0
+ # identical bytes, identical patches: the content hash and the radix key ride on these
+ again, *_ = load_image(Image.open(io.BytesIO(_png(80, 64))), vc)
+ assert torch.equal(patches, again)
+
+
+def test_the_token_budget_caps_a_tall_image():
+ vc = _vc(vision_max_n_token=64, vision_min_pixels=0)
+ _patches, _n_vit_h, _n_vit_w, n_llm_h, n_llm_w = load_image(
+ Image.open(io.BytesIO(_png(64, 1024))), vc
+ )
+ types, _perm = build_image_block(n_llm_h, n_llm_w, 0)
+ assert len(types) <= vc.vision_max_n_token
+
+
+def test_tiny_tower_runs_and_its_aligner_merges_every_merge_grid():
+ vc = _vc()
+ tower = DSV4VisionTower(vc)
+ aligner = DSV4Aligner(vc)
+ patches = torch.zeros(20 * 24, 3, 4, 4, dtype=torch.bfloat16)
+ out = tower.forward(patches, 20, 24)
+ assert out.shape == (20 * 24, vc.vision_dim)
+ assert aligner.forward(out, 20, 24).shape == (10 * 12, vc.text_dim)
+
+
+def _reference_modules(path):
+ """Import the checkpoint's own ``inference/image_processor.py`` (PIL + torch only)."""
+ import importlib.util
+
+ module_path = os.path.join(path, "inference", "image_processor.py")
+ if not os.path.exists(module_path):
+ pytest.skip(f"{module_path} not found")
+ spec = importlib.util.spec_from_file_location("_dsv4_reference_image_processor", module_path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+@needs_checkpoint
+def test_preprocessing_and_block_layout_match_the_reference():
+ from types import SimpleNamespace
+
+ reference = _reference_modules(checkpoint_path)
+ vc = parse_vision_config(_hf_config())
+ args = SimpleNamespace(
+ vision_patch_size=vc.vision_patch_size,
+ vision_downsample_ratio=vc.vision_downsample_ratio,
+ vision_max_n_token=vc.vision_max_n_token,
+ vision_min_pixels=vc.vision_min_pixels,
+ vision_max_wh_ratio=vc.vision_max_wh_ratio,
+ )
+ for width, height in ((640, 480), (100, 100), (3000, 400), (1600, 1200)):
+ raw = _png(width, height)
+ mine = load_image(Image.open(io.BytesIO(raw)), vc)
+ theirs = reference.load_image({"data": raw}, args)
+ assert torch.equal(mine[0], theirs[0])
+ assert mine[1:] == theirs[1:]
+ for start in (0, 1, 3, 7):
+ types, perm = build_image_block(mine[3], mine[4], start)
+ ref_types, ref_perm = reference.build_image_block(theirs[3], theirs[4], start)
+ assert torch.equal(types, ref_types) and torch.equal(perm, ref_perm)
+
+
+_WINDOW, _MAX_IMG = 128, 384
+
+
+def _ref_get_image_visible(input_ids: torch.Tensor, vocab_size: int, max_image_tokens: int):
+ """Verbatim port of the checkpoint's ``inference/model.py`` pair (not vendored in this
+ repo): ``get_image_visible`` derives spans from the virtual ids ``vocab + type``, so the
+ lead pads before IMAGE_START stay outside; only the START/END comparisons matter, our
+ type enum values index the same way."""
+ seqlen = input_ids.size(1)
+ idx = torch.arange(seqlen, dtype=torch.int32).unsqueeze(0)
+ is_start = input_ids == vocab_size + IMAGE_START
+ is_end = input_ids == vocab_size + IMAGE_END
+ valid = (is_start.cumsum(1) > is_end.cumsum(1)) | is_end
+ starts = torch.where(is_start, idx, 0).cummax(1)[0]
+ left = (idx - starts) * valid
+ ends = torch.where(is_end, idx, seqlen).flip(1).cummin(1)[0].flip(1)
+ right = (ends - idx) * valid
+ return left.clamp(max=max_image_tokens - 1), right.clamp(max=max_image_tokens)
+
+
+def _ref_get_window_topk_idxs_visible(window_size, seqlen, left, right, max_image_tokens):
+ width = min(seqlen, window_size + max_image_tokens)
+ idx = torch.arange(seqlen).unsqueeze(0)
+ left_add = (left - (window_size - 1)).clamp(min=0)
+ starts = (idx - (window_size - 1) - left_add).clamp(min=0)
+ matrix = starts.unsqueeze(-1) + torch.arange(width)
+ return torch.where(matrix > (idx + right).unsqueeze(-1), -1, matrix).int().contiguous()
+
+
+def _reference_candidates(prefix: int, n_llm_h: int, n_llm_w: int, vocab: int = 100_000):
+ """The reference pair's window candidates for a text prefix followed by one image block."""
+ types, _perm = build_image_block(n_llm_h, n_llm_w, prefix)
+ ids = torch.full((1, prefix + len(types)), vocab + 50, dtype=torch.int64)
+ ids[0, prefix:] = vocab + types
+ n = ids.size(1)
+ left, right = _ref_get_image_visible(ids, vocab, _MAX_IMG)
+ block = (prefix, prefix + len(types))
+ return block, types, _ref_get_window_topk_idxs_visible(_WINDOW, n, left, right, _MAX_IMG)[0]
+
+
+def test_model_spans_match_the_reference_visibility_pair():
+ from types import SimpleNamespace
+
+ from freetoken.models.deepseek_v4.attention import Attention
+ from freetoken.models.deepseek_v4.model import _image_spans_by_ti
+
+ for prefix, h, w in ((0, 3, 2), (1, 3, 2), (2, 3, 2), (3, 3, 2), (5, 4, 3), (201, 20, 16), (402, 2, 2)):
+ (lo, hi), types, ref = _reference_candidates(prefix, h, w)
+ req = SimpleNamespace(table_idx=7, mm_items=[SimpleNamespace(offsets=[[lo, hi]])])
+ spans = _image_spans_by_ti([req])
+ assert spans == {7: [(lo + types.tolist().index(IMAGE_START), hi)]}
+ cols = Attention.visible_window_cols(
+ spans[7], 0, hi, _WINDOW, _MAX_IMG, torch.device("cpu"),
+ addr_lo=0, width=min(hi, _WINDOW + _MAX_IMG),
+ )
+ assert torch.equal(cols, ref)
+ text = SimpleNamespace(table_idx=0, mm_items=None)
+ assert _image_spans_by_ti([text]) is None
+
+
+def test_a_span_started_at_the_first_lead_pad_breaks_reference_parity():
+ from freetoken.models.deepseek_v4.attention import Attention
+
+ # the maintainer's case: a 201-token prefix puts 2 lead pads in front of a 20x16 grid,
+ # and a span covering the whole block diverges from the reference on 218 rows
+ (lo, hi), _types, ref = _reference_candidates(201, 20, 16)
+ assert lo + COMPRESS_PAD_TO - 1 - lo % COMPRESS_PAD_TO - lo == 2 # the two lead pads
+ cols = Attention.visible_window_cols(
+ [(lo, hi)], 0, hi, _WINDOW, _MAX_IMG, torch.device("cpu"),
+ addr_lo=0, width=min(hi, _WINDOW + _MAX_IMG),
+ )
+ assert int((cols != ref).any(dim=1).sum()) == 218
+
+
+def test_a_vision_checkpoint_keeps_image_blocks_whole_across_chunk_boundaries(tmp_path):
+ """Image spans attend bidirectionally, so a vision checkpoint sets
+ bidirectional_mm_blocks on its attention group and the scheduler ends a chunk before a
+ block instead of splitting it; text-only defaults to False and keeps plain chunking."""
+ from types import SimpleNamespace
+
+ from freetoken.core import Context, SamplingParams, get_global_ctx, set_global_ctx
+ from freetoken.message import MMItem
+ from freetoken.scheduler.cache import CacheManager
+ from freetoken.scheduler.decode import DecodeManager
+ from freetoken.scheduler.mm import cut_image_spans
+ from freetoken.scheduler.prefill import PrefillManager
+ from freetoken.scheduler.table import TableManager
+ from freetoken.scheduler.utils import PendingReq
+
+ # DeepseekV4Args defaults fill an empty inference/config.json; the hf namespace carries the tower
+ (tmp_path / "inference").mkdir()
+ (tmp_path / "inference" / "config.json").write_text("{}")
+ vision = parse_config(SimpleNamespace(_name_or_path=str(tmp_path), **_HF_CONFIG))
+ text = parse_config(SimpleNamespace(_name_or_path=str(tmp_path), **{**_HF_CONFIG, "vision_n_layers": None}))
+ assert vision.attention_groups[0].bidirectional_mm_blocks is True
+ assert text.attention_groups[0].bidirectional_mm_blocks is False
+
+ prefix = 10
+ types, _perm = build_image_block(2, 2, prefix)
+ lo, hi = prefix, prefix + len(types)
+ image = MMItem(modality="image", hash=1, pad_value=1, offsets=[[lo, hi]], feature=torch.zeros(1))
+ assert lo < 15 < hi # a plain 15-token chunk budget ends inside the block
+
+ def chunk_lens(keep_images_whole):
+ try:
+ get_global_ctx()
+ except AssertionError:
+ set_global_ctx(Context(page_size=1))
+ pt = torch.zeros((5, 64), dtype=torch.int32)
+ cm = CacheManager(num_pages=64, page_size=1, page_table=pt, type="radix")
+ pm = PrefillManager(
+ cm, TableManager(max_running_reqs=4, page_table=pt), DecodeManager(1),
+ keep_images_whole=keep_images_whole,
+ )
+ pm.pending_list = [PendingReq(uid=7, input_ids=torch.arange(30, dtype=torch.int32),
+ sampling_params=SamplingParams(max_tokens=4), mm_items=[image])]
+ lens, cuts = [], []
+ while pm.runnable:
+ batch = pm.schedule_next_batch(15)
+ assert batch is not None
+ cm.allocate_paged(batch.reqs)
+ cuts.extend(cut_image_spans(batch.reqs))
+ for r in batch.reqs:
+ lens.append(r.extend_len)
+ r.complete_one()
+ return lens, cuts
+
+ lens, cuts = chunk_lens(vision.attention_groups[0].bidirectional_mm_blocks)
+ assert lens == [lo, 15, 30 - lo - 15] and cuts == [] # first chunk stops at the block, the next holds it whole
+ lens, cuts = chunk_lens(text.attention_groups[0].bidirectional_mm_blocks)
+ assert lens == [15, 15] and cuts == [(lo, hi)] # the block is split into two spans; the cut warning would fire
diff --git a/tests/scheduler/test_mm.py b/tests/scheduler/test_mm.py
index 13447634..72db7e3d 100644
--- a/tests/scheduler/test_mm.py
+++ b/tests/scheduler/test_mm.py
@@ -6,7 +6,13 @@ import torch
from freetoken.message import MMItem
from freetoken.mm.encoder_cache import EncoderCache
-from freetoken.scheduler.mm import cut_image_spans, mm_chunk_end, mm_rows_after, plan_mm_batch, plan_mm_chunk
+from freetoken.scheduler.mm import (
+ cut_image_spans,
+ mm_chunk_end,
+ mm_rows_after,
+ plan_mm_batch,
+ plan_mm_chunk,
+)
CPU = torch.device("cpu")
diff --git a/tests/tokenizer/test_mm_tokenize.py b/tests/tokenizer/test_mm_tokenize.py
index 8f2e0211..5adac830 100644
--- a/tests/tokenizer/test_mm_tokenize.py
+++ b/tests/tokenizer/test_mm_tokenize.py
@@ -1,4 +1,4 @@
-"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3, Muse-Glimmer and MiniMax-M3 checkpoints (skipped when absent)."""
+"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3, Muse-Glimmer, MiniMax-M3 and DeepSeek-V4-Vision checkpoints (skipped when absent)."""
from __future__ import annotations
@@ -20,12 +20,13 @@ GEMMA = os.environ.get("FREETOKEN_GEMMA4_MODEL", "")
GLM = os.environ.get("FREETOKEN_GLM53_MODEL", "")
MUSE = os.environ.get("FREETOKEN_MUSE_MODEL", "")
MINIMAX = os.environ.get("FREETOKEN_MINIMAX_M3_MODEL", "")
+DSV4 = os.environ.get("FREETOKEN_DSV4_VL_MODEL", "")
pytestmark = [
pytest.mark.needs_weights,
pytest.mark.skipif(
- not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE, MINIMAX)),
- reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL / FREETOKEN_MINIMAX_M3_MODEL checkpoint",
+ not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE, MINIMAX, DSV4)),
+ reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL / FREETOKEN_MINIMAX_M3_MODEL / FREETOKEN_DSV4_VL_MODEL checkpoint",
),
]
@@ -214,3 +215,51 @@ def test_minimax_wraps_the_pad_span_in_start_end_tokens():
assert bool((ids[start:end] == item.pad_value).all())
budgeted = TokenizeManager(tokenizer, get_mm_processor(MINIMAX, MultimodalConfig(image_max_tokens=64))).tokenize([_msg([_png(640, 400)])])[0]
assert budgeted.mm_items[0].num_tokens <= 64
+
+
+ not os.path.exists(os.path.join(DSV4, "config.json")),
+ reason="no FREETOKEN_DSV4_VL_MODEL checkpoint",
+)
+def test_dsv4_expands_the_placeholder_into_the_reference_block():
+ """The block carries one content pad id, and its length follows the reference's grid."""
+ from freetoken.mm.processor import get_mm_processor
+ from freetoken.tokenizer.tokenize import TokenizeManager
+ from freetoken.utils.hf import load_tokenizer
+
+ manager = TokenizeManager(load_tokenizer(DSV4), get_mm_processor(DSV4))
+ # 640x480 through the reference: a 12x16 merge grid, so 206 tokens of rows and markers
+ # plus the 3 lead pads that align the block at this offset
+ out = manager.tokenize([_msg([_png(640, 480)])])[0]
+ ids = out.input_ids.tolist()
+ (item,) = out.mm_items
+ assert (item.n_llm_h, item.n_llm_w) == (12, 16)
+ assert item.num_tokens == 209
+ lo, hi = item.offsets[0]
+ assert (lo, hi) == (item.start, item.start + 209)
+ assert set(ids[lo:hi]) == {item.pad_value}
+ assert item.pad_value >= MM_PAD_SHIFT_VALUE and item.pad_value == mm_pad_value(item.hash)
+ assert 129264 not in ids # the raw <|deepseek_image|> placeholder is gone
+ assert item.feature.shape == (35 * 46, 3, 14, 14) and item.feature.dtype == torch.bfloat16
+
+
+ not os.path.exists(os.path.join(DSV4, "config.json")),
+ reason="no FREETOKEN_DSV4_VL_MODEL checkpoint",
+)
+def test_dsv4_two_images_take_two_blocks_and_the_second_is_shorter():
+ """The second image sits at a different offset, so its block is a different length."""
+ from freetoken.mm.processor import get_mm_processor
+ from freetoken.tokenizer.tokenize import TokenizeManager
+ from freetoken.utils.hf import load_tokenizer
+
+ manager = TokenizeManager(load_tokenizer(DSV4), get_mm_processor(DSV4))
+ raw = _png(320, 240)
+ out = manager.tokenize([_msg([raw, raw], n_parts=2)])[0]
+ first, second = out.mm_items
+ assert (first.start, second.start) == (first.offsets[0][0], second.offsets[0][0])
+ # the lead pads take up to three tokens of slack, so the blocks nearly match
+ assert abs(second.num_tokens - first.num_tokens) <= 3
+ # two images share a pad id only when their blocks are laid out identically, which is
+ # exactly when their offsets are congruent: otherwise one must not serve the other's key
+ assert (first.pad_value == second.pad_value) == ((second.start - first.start) % 4 == 0)