FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(deepseek_v4): serve image input on the V4-Flash-Vision release (#502)
Mukkesh Ganesh23fc922bf8df
diff --git a/docs/models.md b/docs/models.mdindex b999d29e..cdadc2dc 100644--- a/docs/models.md+++ b/docs/models.md@@ -6,7 +6,7 @@ for them; other checkpoints of the same architectures work too.| Model | HF checkpoints ||---|---|| DeepSeek-V4.1-Flash | [deepseek-ai/DeepSeek-V4.1-Flash](https://huggingface.co/deepseek-ai/DeepSeek-V4.1-Flash) |-| DeepSeek-V4 | [deepseek-ai/DeepSeek-V4-Flash-0731](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731) |+| DeepSeek-V4 | [deepseek-ai/DeepSeek-V4-Flash-0731](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731), [deepseek-ai/DeepSeek-V4-Flash-Vision-Exp](https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-Vision-Exp) || GLM-5.3-Flash | [RedHatAI/GLM-5.3-Flash-NVFP4](https://huggingface.co/RedHatAI/GLM-5.3-Flash-NVFP4) || GLM-5.2 | [nvidia/GLM-5.2-NVFP4](https://huggingface.co/nvidia/GLM-5.2-NVFP4) || GLM-4.7 | [nvidia/GLM-4.7-NVFP4](https://huggingface.co/nvidia/GLM-4.7-NVFP4) |@@ -35,6 +35,7 @@ These families accept image input by default; pass `--text-model-only` to skip t| GLM-5.3-Flash (`glm5_next`: ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution on a canvas zero-padded to a 28-multiple | token counts, passed through as the processor's `min_image_tokens` / `max_image_tokens`; checkpoint defaults 16 to 8000 tokens | `{"max_image_tokens": 2048}` || Muse-Glimmer-30B (windowed ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, aspect ratio kept under a token cap; checkpoint default 4096 tokens | the maximum is the cap (`max_image_tokens`); the minimum has no effect | `{"max_image_tokens": 1024}` || MiniMax-M3 (`minimax_m3`: CLIP-style ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution | pixel areas in `size.shortest_edge` / `longest_edge`; checkpoint defaults 4 to 576 tokens | `{"size": {"longest_edge": 1048576}}` |+| DeepSeek-V4-Flash-Vision (`deepseek_v4`: native-resolution ViT tower, streamed under `--mm-encoder-weights host`) | one token per 3x3 grid of 14x14 patches, against a per-image cap; the block carries sentinel tokens and lead pads that align it to a 4-token compression stride, so its length depends on where it lands in the prompt | the maximum is a cap on the block (`vision_max_n_token`, checkpoint default 384) and lowers it, the minimum is read as the checkpoint's upscale floor and has no effect | not read |## MoE strategiesdiff --git a/python/freetoken/mm/processor.py b/python/freetoken/mm/processor.pyindex 48211a5b..84463bc2 100644--- a/python/freetoken/mm/processor.py+++ b/python/freetoken/mm/processor.py@@ -200,7 +200,8 @@ def get_mm_processor(model_path: str, mm: MultimodalConfig | None = None) -> MMPserved = [e for e in spec.encoders if getattr(config, e.config_key, None) is not None and e.kind not in mm.disabled_encoders]if spec.mm_processor is None or not served:return None- check_mm_pad_shift(config.text_config.vocab_size)+ # families with a flat checkpoint config (deepseek_v4) carry vocab_size at the top level+ check_mm_pad_shift(getattr(config, "text_config", config).vocab_size)module, _, cls = spec.mm_processor.partition(":")return getattr(importlib.import_module(module), cls)(config, model_path, mm)diff --git a/python/freetoken/mm/processors/deepseek_v4.py b/python/freetoken/mm/processors/deepseek_v4.pynew file mode 100644index 00000000..5cea9919--- /dev/null+++ b/python/freetoken/mm/processors/deepseek_v4.py@@ -0,0 +1,129 @@+"""DeepSeek-V4 image processor: each ``<|deepseek_image|>`` expands into the image's block of tokens.++Where the reference streams ``vocab_size + type`` pseudo ids, the whole span carries one+content pad id and the block's embeddings -- sentinels included -- come from the encoder.+"""++from __future__ import annotations++import io+import struct+from typing import Any++import torch++from freetoken.message import MMItem+from freetoken.mm import mm_pad_value+from freetoken.mm.config import MultimodalConfig+from freetoken.mm.processor import MMResult, MMProcessor, PromptReplacement, _find_all, content_hash+from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID, DSV4VisionConfig, parse_vision_config+from freetoken.models.deepseek_v4.image_processor import load_image+from freetoken.models.deepseek_v4.vision import COMPRESS_PAD_TO, build_image_block+++class DSV4MMProcessor(MMProcessor):+ def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:+ super().__init__(model_path, mm)+ vc = parse_vision_config(hf_config)+ if vc is None:+ raise ValueError(f"{model_path} carries no vision tower (vision_n_layers <= 0)")+ self.vc: DSV4VisionConfig = vc+ self.image_token_id = IMAGE_TOKEN_ID+ self.placeholder = [self.image_token_id]+ # --image-max-tokens lowers the checkpoint's per-image block cap+ self.max_n_token = min(vc.vision_max_n_token, mm.image_max_tokens) if mm.image_max_tokens else vc.vision_max_n_token++ def process(self, images: list[Any]) -> list[MMItem]:+ items: list[MMItem] = []+ for image in images:+ patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = load_image(image, self.vc, self.max_n_token)+ # layout, hash and token count are only known at the insertion offset+ items.append(+ MMItem(+ modality="image",+ hash=0,+ pad_value=0,+ offsets=[],+ feature=patches,+ model_specific_data={+ "n_vit_h": n_vit_h,+ "n_vit_w": n_vit_w,+ "n_llm_h": n_llm_h,+ "n_llm_w": n_llm_w,+ },+ )+ )+ return items++ def prompt_replacement(self, item: MMItem, start: int) -> PromptReplacement:+ """The DSV4-only ``start`` is where the block lands in input_ids; only our apply below calls this."""+ n_llm_h, n_llm_w = item.n_llm_h, item.n_llm_w+ types, _perm = build_image_block(n_llm_h, n_llm_w, start)+ # the lead pads align the block, so the same image at another offset is another+ # sequence: the layout belongs in the hash or the two would share a radix key+ compress_pad = COMPRESS_PAD_TO - 1 - start % COMPRESS_PAD_TO+ item.hash = content_hash(item.feature, struct.pack("<3i", n_llm_h, n_llm_w, compress_pad))+ item.pad_value = mm_pad_value(item.hash)+ item.model_specific_data["start"] = start+ return PromptReplacement([self.image_token_id] * types.numel())++ def apply(self, input_ids: torch.Tensor, images: list[bytes]) -> MMResult:+ """MMProcessor.apply, save that each replacement is built knowing the offset it lands at."""+ from PIL import Image++ ids = input_ids.tolist()+ target = self.placeholder+ slots = _find_all(ids, target)+ if len(slots) != len(images):+ raise ValueError(+ f"prompt renders {len(slots)} image placeholders but the request "+ f"carries {len(images)} images"+ )++ pils = [Image.open(io.BytesIO(raw)).convert("RGB") for raw in images]+ items = self.process(pils)+ out: list[int] = []+ cursor = 0+ for slot, item in zip(slots, items):+ out.extend(ids[cursor:slot])+ base = len(out)+ repl = self.prompt_replacement(item, base)+ full = list(repl.full)+ spans = repl.embed_spans()+ # embedding slots carry the content pad id so radix keys and the model's scatter mask see the image+ for lo, hi in spans:+ full[lo:hi] = [item.pad_value] * (hi - lo)+ out.extend(full)+ item.offsets = [[base + lo, base + hi] for lo, hi in spans]+ item.validate()+ cursor = slot + len(target)+ out.extend(ids[cursor:])+ new_ids = torch.tensor(out, dtype=input_ids.dtype)++ positions, delta = self.positions(len(out), items) or (None, 0)+ return MMResult(new_ids, items, positions, delta)++ def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:+ # The smallest block the tower can run: one 2x2 merge grid out of a 6x6 patch grid.+ r, p = self.vc.vision_downsample_ratio, self.vc.vision_patch_size+ n_vit, n_llm = 2 * r, 2+ types, _perm = build_image_block(n_llm, n_llm, 0)+ return [+ MMItem(+ modality="image",+ hash=0,+ pad_value=0,+ offsets=[[0, types.numel()]],+ feature=torch.zeros(n_vit * n_vit, 3, p, p, dtype=dtype, device=device),+ model_specific_data={+ "n_vit_h": n_vit,+ "n_vit_w": n_vit,+ "n_llm_h": n_llm,+ "n_llm_w": n_llm,+ "start": 0,+ },+ )+ ]+++__all__ = ["DSV4MMProcessor"]diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.pyindex 55b90bb5..25bc73f5 100644--- a/python/freetoken/models/config.py+++ b/python/freetoken/models/config.py@@ -216,6 +216,9 @@ class DSV4AttentionGroupConfig(BaseAttentionGroupConfig):sliding_window: int # the P-token window page# "v41": DeepSeek-V4.1's cross-layer-shared packed tiers (AttnType.DSV41, its own pool and backend)variant: Literal["v4", "v41"] = "v4"+ # image token spans attend to each other in both directions, so prefill chunks end+ # before a block instead of splitting it (the scheduler's keep_images_whole gate)+ bidirectional_mm_blocks: bool = FalseAttentionGroupConfig: TypeAlias = (diff --git a/python/freetoken/models/deepseek_v4/__init__.py b/python/freetoken/models/deepseek_v4/__init__.pyindex e19438d1..5bbb9b8e 100644--- a/python/freetoken/models/deepseek_v4/__init__.py+++ b/python/freetoken/models/deepseek_v4/__init__.py@@ -14,15 +14,22 @@ addressed by page tables, and sparse attention is a physical-slot gather (see"""from .args import DeepseekV4Args, load_args-from .config import parse_config+from .config import DSV4VisionConfig, parse_config, parse_vision_configfrom .model import DeepseekV4ForCausalLM-from .weight import iter_expert_pieces, iter_weights+from .vision import DSV4Aligner, DSV4Vision, DSV4VisionTower+from .weight import iter_expert_pieces, iter_vision_weights, iter_weights__all__ = ["DeepseekV4Args",+ "DSV4Aligner",+ "DSV4Vision",+ "DSV4VisionConfig",+ "DSV4VisionTower","load_args","parse_config",+ "parse_vision_config","DeepseekV4ForCausalLM","iter_weights","iter_expert_pieces",+ "iter_vision_weights",]diff --git a/python/freetoken/models/deepseek_v4/args.py b/python/freetoken/models/deepseek_v4/args.pyindex b75d2800..d2646a2b 100644--- a/python/freetoken/models/deepseek_v4/args.py+++ b/python/freetoken/models/deepseek_v4/args.py@@ -69,6 +69,18 @@ class DeepseekV4Args:hc_sinkhorn_iters: int = 20hc_eps: float = 1e-6+ # ----- vision tower (DeepSeek-V4-Flash-Vision-Exp; 0 on the text-only releases) -----+ vision_n_layers: int = 0+ vision_dim: int = 1024+ vision_n_heads: int = 16+ vision_inter_dim: int = 2816+ vision_patch_size: int = 14+ vision_rope_theta: float = 10000.0+ vision_downsample_ratio: int = 3+ vision_max_n_token: int = 384+ vision_min_pixels: int = 147456+ vision_max_wh_ratio: int = 8+def __post_init__(self) -> None:# JSON lists -> tuple so the dataclass stays hashable / immutable-ish.if isinstance(self.compress_ratios, list):diff --git a/python/freetoken/models/deepseek_v4/attention.py b/python/freetoken/models/deepseek_v4/attention.pyindex cb0dd315..91bc8ac0 100644--- a/python/freetoken/models/deepseek_v4/attention.py+++ b/python/freetoken/models/deepseek_v4/attention.py@@ -38,6 +38,7 @@ class Attention(BaseOP):self.window_size = args.window_sizeself.compress_ratio = args.compress_ratios[layer_id]self.eps = args.norm_eps+ self.max_image_tokens = args.vision_max_n_tokenself.attn_sink = torch.empty(self.n_heads, dtype=torch.float32)# the latent projections are replicated; wq_b shards over heads, wo_b over the output groups@@ -104,7 +105,53 @@ class Attention(BaseOP):o = torch.einsum("bsgd,grd->bsgr", o, wo_a).flatten(2)return self.wo_b.forward(o)- def _prefill_segment(self, x_seg, qr_seg, kv_seg, ti: int, start_pos: int, n: int):+ @staticmethod+ def visible_window_cols(+ spans, start_pos: int, n: int, win: int, max_img: int, device: torch.device,+ addr_lo: int = 0, width: int | None = None,+ ) -> torch.Tensor:+ """Window candidate columns (``-1`` padded) for a segment containing image spans.++ Port of the reference ``get_image_visible`` + ``get_window_topk_idxs_visible``+ pair: span tokens see their whole span bidirectionally (capped at ``max_img``)+ instead of the 128-window, while text tokens -- and span tokens near the span+ start -- keep the ordinary window into history. Returns ``[n, width]`` in+ request-absolute coordinates with live entries in ``[addr_lo, span_end)``; the+ caller translates to slot-lut indices (cold: as-is; extend: minus ``w_lo``).+ """+ if width is None:+ width = win + max_img+ left, right = Attention._span_left_right(spans, start_pos, n, max_img, device)+ idx = start_pos + torch.arange(n, device=device)+ left_add = (left - (win - 1)).clamp(min=0)+ starts = (idx - (win - 1) - left_add).clamp(min=addr_lo)+ cols = starts.unsqueeze(1) + torch.arange(width, device=device)+ return torch.where(cols > (idx + right).unsqueeze(1), -1, cols)++ @staticmethod+ def _span_left_right(+ spans, start_pos: int, n: int, max_img: int, device: torch.device+ ) -> tuple[torch.Tensor, torch.Tensor]:+ """Per-token visible counts to the left/right within each image span.++ Metadata twin of the reference ``get_image_visible`` (which derives spans from+ virtual token ids): ``left[q] = q - span_start`` clamped to ``max_img - 1``,+ ``right[q] = span_end - q`` clamped to ``max_img``, zero outside spans. ``spans``+ are request-absolute positions inside ``[start_pos, start_pos + n)``.+ """+ left = torch.zeros(n, dtype=torch.int64, device=device)+ right = torch.zeros(n, dtype=torch.int64, device=device)+ for s, e in spans:+ lo, hi = s - start_pos, e - start_pos+ idx = torch.arange(lo, hi, device=device)+ left[lo:hi] = (idx - lo).clamp(max=max_img - 1)+ # the END sentinel sits at e-1: visibility reaches the END TOKEN, not past it+ right[lo:hi] = ((hi - 1) - idx).clamp(max=max_img)+ return left, right++ def _prefill_segment(+ self, x_seg, qr_seg, kv_seg, ti: int, start_pos: int, n: int, spans=None+ ):"""One request's prefill work inside a (possibly ragged) batch: persist its window KV,advance its compressor/indexer carry, and return its per-query candidate lists.@@ -113,6 +160,16 @@ class Attention(BaseOP):``(win_global [1, n, w], cmp_global [1, n, c] | None)`` at natural widths (cold: w =min(n, win); extend: w = win); the caller pads BOTH halves to the batch-uniform widthsand concatenates.++ ``spans`` (VL requests) are the request's image-block ranges ``[(start, end)]`` inside+ ``[start_pos, start_pos + n)``. Vision checkpoints set ``bidirectional_mm_blocks`` on the+ attention group, which makes prefill end a chunk before a block instead of splitting it;+ only a block longer than one chunk budget is still cut. Inside a span, every token+ sees the whole span (bidirectional, capped at ``vision_max_n_token``) instead of only its+ 128-window, matching the reference's ``get_window_topk_idxs_visible``. All span-visible+ candidates are intra-segment, so the live full-page map covers them (the 128-slot ring+ never needs to serve >128-back reads). ``spans=None`` keeps the historical path+ bit-identical."""win, ratio, device = self.window_size, self.compress_ratio, x_seg.deviceend = start_pos + n@@ -120,23 +177,38 @@ class Attention(BaseOP):self.attn.store_window(kv_seg, self.layer_id, slots)if start_pos == 0:- win_cols = get_window_topk_idxs(win, 1, n, 0).to(device)- # natural width min(n, win); the caller pads to the batch-uniform width+ if spans:+ max_img = self.max_image_tokens+ cols = Attention.visible_window_cols(+ spans, 0, n, win, max_img, device, addr_lo=0,+ width=min(n, win + max_img),+ )+ win_cols = cols.unsqueeze(0)+ else:+ win_cols = get_window_topk_idxs(win, 1, n, 0).to(device)+ # natural width min(n, win) -- win+max_img for span segments; the caller padswin_global = self.attn.win_cols_to_global(win_cols, slots)else:# Candidates span [w_lo, end): each new query's 128-sliding window over the retained# prefix plus the new tokens, causal-masked to -1 past its own position.w_lo = max(0, start_pos - win + 1)ws_pool = self.attn.window_slots_of(ti, w_lo, end)- abs_p = start_pos + torch.arange(n, device=device).unsqueeze(1)- cand = (abs_p - win + 1).clamp(min=w_lo) + torch.arange(win, device=device)- win_cols = torch.where(cand > abs_p, -1, cand - w_lo).unsqueeze(0)+ if spans:+ max_img = self.max_image_tokens+ cols = Attention.visible_window_cols(+ spans, start_pos, n, win, max_img, device, addr_lo=w_lo,+ )+ win_cols = torch.where(cols < 0, cols, cols - w_lo).unsqueeze(0)+ else:+ abs_p = start_pos + torch.arange(n, device=device).unsqueeze(1)+ cand = (abs_p - win + 1).clamp(min=w_lo) + torch.arange(win, device=device)+ win_cols = torch.where(cand > abs_p, -1, cand - w_lo).unsqueeze(0)win_global = self.attn.win_cols_to_global(win_cols, ws_pool)if not ratio:return win_global, None- # Only the compressor/indexer read the matched tail page's slot; resolving it costs a- # host sync (.item()), so do it after the ratio-0 early-out.+ # Only the compressor/indexer read the matched tail page's slot; resolving it costs+ # a host sync (.item()), so do it after the ratio-0 early-out.tail_ws = (int(self.attn.window_slots_of(ti, start_pos - 1, start_pos).item())if start_pos > 0 else None@@ -157,7 +229,7 @@ class Attention(BaseOP):self.compressor.forward(x_seg, start_pos, slots, tail_window_slot=tail_ws, ti=ti)return win_global, self.attn.blocks_to_global(blocks, ratio, ti=ti)- def forward_ragged(self, x, segments, flat_positions):+ def forward_ragged(self, x, segments, flat_positions, spans_by_ti=None):"""Ragged batched prefill (cu_seqlens). ``x`` is [1, T, dim] -- the requests' NEW tokenstreams concatenated (NO padding); ``segments`` is [(offset, n, table_idx, start_pos)]tiling [0, T); ``flat_positions`` [T] is each token's ABSOLUTE position, so a radix-hit@@ -197,9 +269,18 @@ class Attention(BaseOP):win_parts: list[torch.Tensor] = []cmp_parts: list[torch.Tensor | None] = []max_c = 0+ any_spans = Falsefor off, n, ti, start_pos in segments:+ spans = None+ if spans_by_ti is not None:+ req_spans = spans_by_ti.get(ti)+ if req_spans:+ end = start_pos + n+ spans = [(s, e) for (s, e) in req_spans if s >= start_pos and e <= end]+ any_spans = any_spans or bool(spans)win_global, cmp_global = self._prefill_segment(- x[:, off:off + n], qr[:, off:off + n], kv[0, off:off + n], ti, start_pos, n+ x[:, off:off + n], qr[:, off:off + n], kv[0, off:off + n], ti, start_pos, n,+ spans=spans,)win_parts.append(win_global)cmp_parts.append(cmp_global)@@ -213,6 +294,11 @@ class Attention(BaseOP):# is enough to flip a greedy near-tie. bs>1 pads every part to win (the historical# batched behavior).n_window = win if len(segments) > 1 else win_parts[0].shape[-1]+ if any_spans:+ # Span segments carry win+max_img-wide candidates; pad every part to the widest+ # so the single flat launch stays rectangular. Text-only batches (any_spans+ # False) keep the historical widths bit-identically.+ n_window = max(n_window, *(w.shape[-1] for w in win_parts))flat = []for i, (off, n, ti, _start) in enumerate(segments):wg = win_parts[i]diff --git a/python/freetoken/models/deepseek_v4/config.py b/python/freetoken/models/deepseek_v4/config.pyindex 1005b8a3..9671c682 100644--- a/python/freetoken/models/deepseek_v4/config.py+++ b/python/freetoken/models/deepseek_v4/config.py@@ -13,12 +13,38 @@ The transformers ``AutoConfig`` for ``deepseek_v4`` drops a few keys (e.g.from __future__ import annotations+from dataclasses import dataclassfrom typing import Anyfrom freetoken.models.config import DSV4AttentionGroupConfig, ModelConfig, RotaryConfigfrom .args import load_args+# ``<|deepseek_image|>``: the only vocab-level image token. The reference expands each+# occurrence into a block of ``vocab_size + type`` pseudo ids; here the block keeps this+# one id and the image embeddings are scattered over the span it occupies.+IMAGE_TOKEN_ID = 129264+++@dataclass(frozen=True)+class DSV4VisionConfig:+ """The tower's dims, split out of the checkpoint's flat inference config so the engine's+ section nulling and the loader treat DSV4 like every other family with a vision tower.+ Field names follow the reference (``inference/model.py`` ModelArgs) one to one."""++ vision_n_layers: int+ vision_dim: int+ vision_n_heads: int+ vision_inter_dim: int+ vision_patch_size: int+ vision_rope_theta: float+ vision_downsample_ratio: int+ vision_max_n_token: int+ vision_min_pixels: int+ vision_max_wh_ratio: int+ # the language model's hidden size the aligner projects into (ModelArgs.dim)+ text_dim: int+def parse_config(hf_config: Any) -> ModelConfig:model_path = getattr(hf_config, "_name_or_path", None) or getattr(@@ -46,6 +72,8 @@ def parse_config(hf_config: Any) -> ModelConfig:if max_position <= 0:max_position = int(args.original_seq_len * args.rope_factor)+ vision_config = parse_vision_config(hf_config)+return ModelConfig(num_layers=args.n_layers,num_qo_heads=args.n_heads,@@ -88,9 +116,38 @@ def parse_config(hf_config: Any) -> ModelConfig:num_kv_heads=1, # MLA-style shared latent (K == V)head_dim=args.head_dim,sliding_window=args.window_size,+ bidirectional_mm_blocks=vision_config is not None,),),+ vision_config=vision_config,+ image_token_id=IMAGE_TOKEN_ID if vision_config is not None else None,+ )+++def parse_vision_config(hf_config: Any) -> DSV4VisionConfig | None:+ """The tower's dims from the checkpoint config, or None when it carries no vision stack.++ The vision dims sit at the top level of the checkpoint config, next to the language+ model's, and the engine nulls ``vision_n_layers`` on the copy it hands the parser when+ this process serves text-only (--text-model-only / --mm-disable). That null is what+ decides, so the built tower always matches the weights the loader is told to read.+ """+ n_layers = int(getattr(hf_config, "vision_n_layers", 0) or 0)+ if n_layers <= 0:+ return None+ return DSV4VisionConfig(+ vision_n_layers=n_layers,+ vision_dim=int(hf_config.vision_dim),+ vision_n_heads=int(hf_config.vision_n_heads),+ vision_inter_dim=int(hf_config.vision_inter_dim),+ vision_patch_size=int(hf_config.vision_patch_size),+ vision_rope_theta=float(hf_config.vision_rope_theta),+ vision_downsample_ratio=int(hf_config.vision_downsample_ratio),+ vision_max_n_token=int(hf_config.vision_max_n_token),+ vision_min_pixels=int(hf_config.vision_min_pixels),+ vision_max_wh_ratio=int(hf_config.vision_max_wh_ratio),+ text_dim=int(hf_config.hidden_size),)-__all__ = ["parse_config"]+__all__ = ["IMAGE_TOKEN_ID", "DSV4VisionConfig", "parse_config", "parse_vision_config"]diff --git a/python/freetoken/models/deepseek_v4/image_processor.py b/python/freetoken/models/deepseek_v4/image_processor.pynew file mode 100644index 00000000..16742b1e--- /dev/null+++ b/python/freetoken/models/deepseek_v4/image_processor.py@@ -0,0 +1,50 @@+"""DeepSeek-V4 image preprocessing: one image -> ViT patches plus its grid metadata.++Port of the reference ``inference/image_processor.py:load_image``. The PIL work stays on the+CPU and must be bit-identical across runs: an image's radix key rides on these patches.+"""++from __future__ import annotations++import math++import numpy as np+import torch+from PIL import Image, ImageOps++from .config import DSV4VisionConfig+from .vision import safe_resize+++def load_image(image: Image.Image, vc: DSV4VisionConfig, max_n_token: int | None = None) -> tuple[torch.Tensor, int, int, int, int]:+ """One image -> ``(patches bf16 [n_vit_h*n_vit_w, 3, p, p], n_vit_h, n_vit_w, n_llm_h, n_llm_w)``.++ ``max_n_token`` lowers the checkpoint's per-image block cap (``--image-max-tokens``).+ """+ image = image.convert("RGB")+ p = vc.vision_patch_size+ width, height = image.size+ if vc.vision_max_wh_ratio is not None and width > height * vc.vision_max_wh_ratio:+ width = height * vc.vision_max_wh_ratio+ if 0 < width * height < vc.vision_min_pixels:+ ratio = (vc.vision_min_pixels / (width * height)) ** 0.5+ width = int(width * ratio)+ height = int(height * ratio)+ best_width = math.ceil(width / p) * p+ best_height = math.ceil(height / p) * p+ n_llm_h, n_llm_w, best_height, best_width = safe_resize(+ height, width, best_height, best_width, p, vc.vision_downsample_ratio,+ vc.vision_max_n_token if max_n_token is None else min(vc.vision_max_n_token, max_n_token),+ )+ n_vit_h, n_vit_w = best_height // p, best_width // p+ if vc.vision_max_wh_ratio is not None and image.width >= vc.vision_max_wh_ratio * image.height:+ image = image.resize((best_width, best_height))+ else:+ image = ImageOps.pad(image, (best_width, best_height), color=(127, 127, 127))+ x = torch.from_numpy(np.asarray(image, dtype=np.float32)).permute(2, 0, 1) / 255+ x = ((x - 0.5) / 0.5).to(torch.bfloat16)+ patches = x.reshape(3, n_vit_h, p, n_vit_w, p).permute(1, 3, 0, 2, 4).reshape(n_vit_h * n_vit_w, 3, p, p)+ return patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w+++__all__ = ["load_image"]diff --git a/python/freetoken/models/deepseek_v4/model.py b/python/freetoken/models/deepseek_v4/model.pyindex 9faefb83..0c3441b9 100644--- a/python/freetoken/models/deepseek_v4/model.py+++ b/python/freetoken/models/deepseek_v4/model.py@@ -29,11 +29,12 @@ from freetoken.core import get_global_ctxfrom freetoken.kernel.triton.dsv4.hc import hc_post_combine, hc_pre_combinefrom freetoken.kernel.triton.dsv4.sinkhorn import hc_split_sinkhornfrom freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNorm, VocabParallelEmbedding-from freetoken.models.blocks import BaseLLMModel+from freetoken.models.blocks import BaseLLMModel, embed_input_idsfrom .args import DeepseekV4Argsfrom .attention import Attentionfrom .moe import MoE+from .vision import COMPRESS_PAD_TO, DSV4Vision# Re-exports: keep every class/helper previously defined here importable from .model# (external import stability; the moved definitions live in their own modules).@@ -85,7 +86,7 @@ class Block(BaseOP):)return y.view(shape)- def prefill_batched(self, x, input_ids, segments, flat_positions):+ def prefill_batched(self, x, input_ids, segments, flat_positions, spans_by_ti=None):# Ragged batched prefill (cu_seqlens, no padding; bs >= 1, cold and radix-hit segments# mixed freely). ``x`` is [1, T, hc_mult, dim] -- the requests' token streams# concatenated. Per-token ops (HC / norm / MoE) run batched over ALL T tokens (the@@ -93,10 +94,12 @@ class Block(BaseOP):# T queries (Attention.forward_ragged), with the stateful compressor/indexer looped per# request. ``segments`` = [(offset, extend_len, table_idx, start_pos)] off the attention# metadata; ``flat_positions`` [T] = per-token ABSOLUTE position (batch.positions).+ # ``spans_by_ti`` maps a page-table row to its image spans: the sparse attention needs+ # the pairs, the block-tiled backends read the flat ends (batch.mm_block_ends) instead.residual = xx, post, comb = self.hc_pre(x, self.hc_attn_fn, self.hc_attn_scale, self.hc_attn_base)x = self.attn_norm.forward(x)- x = self.attn.forward_ragged(x, segments, flat_positions)+ x = self.attn.forward_ragged(x, segments, flat_positions, spans_by_ti=spans_by_ti)x = self.hc_post(x, residual, post, comb)residual = x@@ -152,6 +155,7 @@ class Transformer(BaseOP):def prefill_batched(self, input_ids: torch.Tensor, segments, flat_positions: torch.Tensor,+ spans_by_ti: dict | None = None,) -> torch.Tensor:# Ragged batched prefill (bs >= 1). ``input_ids`` is [1, T] -- the requests' NEW tokens# concatenated (cu_seqlens, no padding); each request starts at its own cached_len@@ -163,10 +167,13 @@ class Transformer(BaseOP):# metadata; ``flat_positions`` [T] is the scheduler-staged batch.positions (per-token# ABSOLUTE position); the head picks each request's final token off the attention# metadata -> its next-token logits row.- h = self.embed.forward(input_ids.view(-1)).view(1, -1, self.args.dim)+ batch = get_global_ctx().batch+ # on a chunk that carries image rows, batch.mm_embeds' leading columns replace the+ # embedding of every row in batch.mm_rows -- the whole image span, sentinels included+ h = embed_input_ids(self.embed, input_ids.view(-1), batch).view(1, -1, self.args.dim)h = h.unsqueeze(2).repeat(1, 1, self.hc_mult, 1)for layer in self.layers.op_list:- h = layer.prefill_batched(h, input_ids, segments, flat_positions)+ h = layer.prefill_batched(h, input_ids, segments, flat_positions, spans_by_ti)h = self.hc_head(h)h = self.norm.forward(h)return self.head.forward(h[0]) # [B, vocab]@@ -200,6 +207,26 @@ class Transformer(BaseOP):return self.head.forward(h[:, -1])+def _image_spans_by_ti(reqs) -> dict | None:+ """The batch's image spans as ``(IMAGE_START position, block end)`` pairs per page-table row.++ Only this model reads the pairs (built here, not staged on the batch): its sparse+ attention builds candidate lists per span. The reference ``get_image_visible`` counts+ visibility within [IMAGE_START, IMAGE_END], so the span starts at the START sentinel+ (after ``build_image_block``'s alignment lead pads), not at the block's first token.+ """+ spans: dict[int, list[tuple[int, int]]] = {}+ for req in reqs:+ req_spans = [+ (lo + COMPRESS_PAD_TO - 1 - lo % COMPRESS_PAD_TO, hi)+ for item in req.mm_items or ()+ for lo, hi in item.offsets+ ]+ if req_spans:+ spans[req.table_idx] = req_spans+ return spans or None++class DeepseekV4ForCausalLM(BaseLLMModel):"""Engine adapter: a registered :class:`BaseLLMModel` wrapping the DSV4 transformer.@@ -211,6 +238,12 @@ class DeepseekV4ForCausalLM(BaseLLMModel):self._args: DeepseekV4Args = config.dsv4_argsself.model = Transformer(self._args, config.quant, strategy=config.moe_strategy, decode_target=config.decode_target, prefix="model")self._bound = False+ # Vision (DeepSeek-V4-Flash-Vision-Exp). config.is_multimodal is the engine's+ # resolved gate, so a text-only process never builds weights the loader skips.+ # The whole stack sits under one ``visual.`` mount, so the shared vision-key+ # filter (VISION_KEY_PREFIXES) covers every image-only parameter.+ if config.is_multimodal:+ self.visual = DSV4Vision(config.vision_config, config.hidden_size)def _ensure_bound(self) -> None:if self._bound:@@ -226,6 +259,12 @@ class DeepseekV4ForCausalLM(BaseLLMModel):block count, freqs) depends on the new pool's geometry, so re-derive it via _ensure_bound."""self._bound = False+ def place_encoder_weights(self, mode: str) -> None:+ self.visual.place_weights(mode)++ def encode(self, item):+ return self.visual.forward(item)+def forward(self) -> torch.Tensor:self._ensure_bound()batch = get_global_ctx().batch@@ -240,6 +279,7 @@ class DeepseekV4ForCausalLM(BaseLLMModel):# cross requests.return self.model.prefill_batched(input_ids.view(1, -1), md.segments, batch.positions.long(),+ spans_by_ti=_image_spans_by_ti(batch.reqs),)# DECODE (bs>=1): per-row position (GPU int tensor -> no host syncs / graph safe). The# compressed staging cap is the max position any row reaches (eager); a static max_seq-1diff --git a/python/freetoken/models/deepseek_v4/moe.py b/python/freetoken/models/deepseek_v4/moe.pyindex 6b624567..35a316f1 100644--- a/python/freetoken/models/deepseek_v4/moe.py+++ b/python/freetoken/models/deepseek_v4/moe.py@@ -9,24 +9,34 @@ import torch.nn.functional as Ffrom freetoken.kernel.triton.dsv4.bf16_linear import bf16_linear_fp32from freetoken.kernel.triton.dsv4.swiglu import fused_swiglufrom freetoken.layers import BaseOP, LinearColParallelMerged, LinearRowParallel, OffloadMoELayer+from freetoken.mm import MM_PAD_SHIFT_VALUEfrom .args import DeepseekV4Argsclass Gate(BaseOP):- """MoE router: sqrtsoftplus scoring + hash routing (first ``n_hash_layers``)."""+ """MoE router: sqrtsoftplus scoring + hash routing (first ``n_hash_layers``).++ A vision checkpoint gives every layer a ``bias_vl`` too: the expert-selection bias for+ image rows, which never touches the routing weights. Hash layers keep routing text rows+ through ``tid2eid`` and pick image rows by ``(scores + bias_vl).topk``, mirroring the+ reference ``Gate.forward``.+ """def __init__(self, layer_id: int, args: DeepseekV4Args):self.topk = args.n_activated_expertsself.score_func = args.score_funcself.route_scale = args.route_scaleself.hash = layer_id < args.n_hash_layers+ vl = args.vision_n_layers > 0self.weight = torch.empty(args.n_routed_experts, args.dim, dtype=torch.bfloat16)if self.hash:self.tid2eid = torch.empty(args.vocab_size, args.n_activated_experts, dtype=torch.int64)- self.bias = None+ # present on hash layers of a vision checkpoint, loaded but never applied+ self.bias = torch.empty(args.n_routed_experts, dtype=torch.float32) if vl else Noneelse:self.bias = torch.empty(args.n_routed_experts, dtype=torch.float32)+ self.bias_vl = torch.empty(args.n_routed_experts, dtype=torch.float32) if vl else Nonedef forward(self, x: torch.Tensor, input_ids: torch.Tensor):scores = bf16_linear_fp32(x, self.weight)@@ -37,11 +47,20 @@ class Gate(BaseOP):else:scores = F.softplus(scores).sqrt()original_scores = scores- if self.bias is not None:- scores = scores + self.bias+ # image rows are the content pad ids of an expanded image span+ image_mask = input_ids >= MM_PAD_SHIFT_VALUE if self.bias_vl is not None else Noneif self.hash:- indices = self.tid2eid[input_ids]+ if image_mask is None:+ indices = self.tid2eid[input_ids]+ else:+ indices = self.tid2eid[torch.where(image_mask, 0, input_ids)]+ vl_indices = (scores + self.bias_vl).topk(self.topk, dim=-1)[1]+ indices = torch.where(image_mask.unsqueeze(-1), vl_indices.to(indices.dtype), indices)else:+ if image_mask is None:+ scores = scores + self.bias+ else:+ scores = scores + torch.where(image_mask.unsqueeze(-1), self.bias_vl, self.bias)indices = scores.topk(self.topk, dim=-1)[1]weights = original_scores.gather(1, indices)if self.score_func != "softmax":diff --git a/python/freetoken/models/deepseek_v4/vision.py b/python/freetoken/models/deepseek_v4/vision.pynew file mode 100644index 00000000..44549460--- /dev/null+++ b/python/freetoken/models/deepseek_v4/vision.py@@ -0,0 +1,336 @@+"""DeepSeek-V4 vision tower, aligner and image-block layout.++Port of the reference ``inference/vision.py`` / ``inference/image_processor.py`` from+deepseek-ai/DeepSeek-V4-Flash-Vision-Exp. The tower, the aligner and the four sentinel+vectors hang under one :class:`DSV4Vision` container the wrapper mounts as ``visual``,+so their state-dict keys sit behind the ``visual.`` prefix every VLM family filters on+(the checkpoint's own unprefixed names, ``vision.patch_embed.proj.weight``,+``aligner.w1.weight``, ``image_start``, ..., are renamed on load); every tensor is bf16+exactly as stored.++Image tokens occupy plain sequential positions: the 2-D structure lives inside the+tower (2-D RoPE over the patch grid) and in the N-layout of the token block the+:func:`build_image_block` / :func:`assemble_block` pair renders. All block embeddings,+sentinels included, come from the vision side, so the text embedding row of the+placeholder id is never read -- the span is overwritten before the first layer.+"""++from __future__ import annotations++import math+from functools import lru_cache++import torch+import torch.nn.functional as F++from freetoken.layers import BaseOP, OPList+from freetoken.models.weight_stream import BlockWeightStreamer++from .config import DSV4VisionConfig++# Sentinel token types inside an image block (reference inference/image_processor.py).+IMAGE_START, IMAGE_PAD, IMAGE, IMAGE_NEW_LINE, IMAGE_END = range(5)+# An image block is padded so its first row token lands on a compressor stride boundary.+COMPRESS_PAD_TO = 4+++@lru_cache(8)+def get_vision_cos_sin(n_h: int, n_w: int, dim: int, theta: float):+ inv_freq = 1.0 / (theta ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))+ hpos = torch.arange(n_h).unsqueeze(1).expand(n_h, n_w)+ wpos = torch.arange(n_w).unsqueeze(0).expand(n_h, n_w)+ freqs = torch.stack([hpos, wpos], dim=-1).reshape(-1, 2, 1).float() * inv_freq+ freqs = freqs.flatten(1)+ return freqs.cos().unsqueeze(1), freqs.sin().unsqueeze(1)+++def apply_rotary(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:+ dtype = x.dtype+ x1, x2 = x.float().chunk(2, dim=-1)+ return torch.cat([x1 * cos - x2 * sin, x2 * cos + x1 * sin], dim=-1).to(dtype)+++class _Linear(BaseOP):+ """Linear in checkpoint-native layout: ``weight`` [out, in] plus an optional ``bias``."""++ def __init__(self, in_features: int, out_features: int, bias: bool = True):+ self.weight = torch.empty(out_features, in_features, dtype=torch.bfloat16)+ self.bias = torch.empty(out_features, dtype=torch.bfloat16) if bias else None++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return F.linear(x, self.weight, self.bias)+++class RMSNorm(BaseOP):+ def __init__(self, dim: int, eps: float = 1e-6):+ self.eps = eps+ self.weight = torch.empty(dim, dtype=torch.bfloat16)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ dtype = x.dtype+ x = x.float()+ x = x * torch.rsqrt(x.square().mean(-1, keepdim=True) + self.eps)+ return (self.weight * x).to(dtype)+++class PatchEmbed(BaseOP):+ def __init__(self, patch_size: int, dim: int):+ self.proj = _Linear(3 * patch_size**2, dim)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return self.proj.forward(x.flatten(1))+++class VisionAttention(BaseOP):+ def __init__(self, dim: int, n_heads: int):+ self.n_heads = n_heads+ self.head_dim = dim // n_heads+ self.wqkv = _Linear(dim, 3 * dim)+ self.wo = _Linear(dim, dim)++ def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:+ n = x.size(0)+ q, k, v = (t.view(n, self.n_heads, self.head_dim) for t in self.wqkv.forward(x).chunk(3, dim=-1))+ q = apply_rotary(q, cos, sin)+ k = apply_rotary(k, cos, sin)+ o = F.scaled_dot_product_attention(q.transpose(0, 1), k.transpose(0, 1), v.transpose(0, 1))+ return self.wo.forward(o.transpose(0, 1).reshape(n, -1))+++class VisionMLP(BaseOP):+ def __init__(self, dim: int, inter_dim: int):+ self.w1 = _Linear(dim, 2 * inter_dim, bias=False)+ self.w2 = _Linear(inter_dim, dim, bias=False)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ gate, up = self.w1.forward(x).chunk(2, dim=-1)+ return self.w2.forward(F.silu(gate) * up)+++class VisionBlock(BaseOP):+ def __init__(self, dim: int, n_heads: int, inter_dim: int, eps: float):+ self.norm1 = RMSNorm(dim, eps)+ self.attn = VisionAttention(dim, n_heads)+ self.norm2 = RMSNorm(dim, eps)+ self.mlp = VisionMLP(dim, inter_dim)++ def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:+ x = x + self.attn.forward(self.norm1.forward(x), cos, sin)+ return x + self.mlp.forward(self.norm2.forward(x))+++class DSV4VisionTower(BaseOP):+ """Native-resolution ViT: bidirectional attention inside one image, 2-D RoPE over its patch grid."""++ def __init__(self, vc: DSV4VisionConfig):+ self.rope_dim = vc.vision_dim // vc.vision_n_heads // 2+ self.rope_theta = vc.vision_rope_theta+ self.patch_embed = PatchEmbed(vc.vision_patch_size, vc.vision_dim)+ self.blocks = OPList(+ [+ VisionBlock(vc.vision_dim, vc.vision_n_heads, vc.vision_inter_dim, eps=1e-6)+ for _ in range(vc.vision_n_layers)+ ]+ )+ self.norm = RMSNorm(vc.vision_dim, 1e-6)+ self._streamer: BlockWeightStreamer | None = None++ def place_weights(self, mode: str) -> None:+ """gpu: every tensor resident; host: the block stack in pinned banks, two blocks streamed at a time (patch embed and final norm stay resident)."""+ if mode == "host" and self._streamer is None:+ self._streamer = BlockWeightStreamer(self.blocks.op_list, self.patch_embed.proj.weight.device)+ elif mode == "gpu" and self._streamer is not None:+ self._streamer.unstream()+ self._streamer = None+ elif mode not in ("gpu", "host"):+ raise ValueError(f"unknown vision weight placement {mode!r}")++ def _blocks(self):+ if self._streamer is None:+ return enumerate(self.blocks.op_list)+ return self._streamer.blocks(self.blocks.op_list)++ def forward(self, patches: torch.Tensor, n_h: int, n_w: int) -> torch.Tensor:+ patches = patches.to(self.patch_embed.proj.weight.dtype)+ x = self.patch_embed.forward(patches)+ cos, sin = get_vision_cos_sin(n_h, n_w, self.rope_dim, self.rope_theta)+ cos, sin = cos.to(x.device), sin.to(x.device)+ for _, block in self._blocks():+ x = block.forward(x, cos, sin)+ return self.norm.forward(x)+++class DSV4Aligner(BaseOP):+ """3x3/stride-3 merge of every 3x3 patch grid into one token, then a 2-layer erf-GELU MLP."""++ def __init__(self, vc: DSV4VisionConfig):+ self.downsample_ratio = vc.vision_downsample_ratio+ in_dim = vc.vision_dim * self.downsample_ratio**2+ self.w1 = _Linear(in_dim, vc.text_dim)+ self.w2 = _Linear(vc.text_dim, vc.text_dim)++ def forward(self, x: torch.Tensor, n_h: int, n_w: int) -> torch.Tensor:+ dtype = self.w1.weight.dtype+ x = x.to(dtype)+ r = self.downsample_ratio+ x = x.view(n_h, n_w, -1).permute(2, 0, 1)+ x = F.pad(x, (0, -n_w % r, 0, -n_h % r))+ x = F.unfold(x.unsqueeze(0), r, stride=r).squeeze(0).transpose(0, 1)+ return self.w2.forward(F.gelu(self.w1.forward(x)))+++class DSV4Vision(BaseOP):+ """The wrapper's whole vision stack under one ``visual.`` mount, like the V4.1 sibling."""++ def __init__(self, vc: DSV4VisionConfig, text_dim: int):+ self.vision = DSV4VisionTower(vc)+ self.aligner = DSV4Aligner(vc)+ for name in ("image_start", "image_end", "image_newline", "image_pad"):+ setattr(self, name, torch.empty(text_dim, dtype=torch.bfloat16))++ def place_weights(self, mode: str) -> None:+ self.vision.place_weights(mode)++ @torch.inference_mode()+ def forward(self, item):+ """One image -> the embeddings of its whole token block, sentinels included.++ The block's layout is a function of where it lands in the prompt, so the item+ carries the grid and the offset; ``build_image_block`` re-derives the same+ ``(types, perm)`` the processor tokenized with.+ """+ device = self.vision.patch_embed.proj.weight.device+ patches = item.feature.to(device, non_blocking=True)+ aligned = self.aligner.forward(+ self.vision.forward(patches, item.n_vit_h, item.n_vit_w), item.n_vit_h, item.n_vit_w+ )+ types, perm = build_image_block(item.n_llm_h, item.n_llm_w, item.start)+ sentinels = torch.stack(+ [self.image_start, self.image_pad, self.image_pad, self.image_newline, self.image_end]+ )+ return assemble_block(aligned, types, perm, sentinels)+++def grid_tokens(best_height: int, best_width: int, patch_size: int, downsample_ratio: int):+ """LLM-token footprint of a resized image: the N-layout row/align padding included."""+ n_llm_h = math.ceil((best_height // patch_size) / downsample_ratio)+ n_llm_w = math.ceil((best_width // patch_size) / downsample_ratio)+ num_tokens = n_llm_h * (n_llm_w + 1) + 2+ if n_llm_h % 2 == 1:+ num_tokens += n_llm_w + 1+ num_tokens += (n_llm_h + 1) // 2 * (n_llm_w + 1) % 2 * 2+ return n_llm_h, n_llm_w, num_tokens+++def solve_resize_ratio(height, width, patch_size, downsample_ratio, max_n_token):+ r = height / width+ max_w_float = math.sqrt((max_n_token - 2) / r + 0.25) - 0.5+ max_h_float = max_w_float * r+ if max_w_float < 1.0:+ max_w = 1+ max_h = (max_n_token - 2) // (max_w + 1)+ if max_h % 2 == 1:+ max_h -= 1+ best_width = max_w * patch_size * downsample_ratio+ best_height = max_h * patch_size * downsample_ratio+ elif max_h_float < 2.0:+ max_h = 2+ max_w = ((max_n_token - 2) // max_h) - 1+ assert max_w > 1+ best_width = max_w * patch_size * downsample_ratio+ best_height = max_h * patch_size * downsample_ratio+ else:+ max_w = math.floor(max_w_float)+ max_h = math.floor(max_h_float)+ if max_h % 2 == 1:+ max_h -= 1+ beta = min(max_w * patch_size * downsample_ratio / width, max_h * patch_size * downsample_ratio / height)+ best_width = math.floor(width * beta / patch_size) * patch_size+ best_height = math.floor(height * beta / patch_size) * patch_size+ n_llm_h, n_llm_w, num_tokens = grid_tokens(best_height, best_width, patch_size, downsample_ratio)+ return n_llm_h, n_llm_w, best_height, best_width, num_tokens+++def safe_resize(height, width, best_height, best_width, patch_size, downsample_ratio, max_n_token):+ """Largest grid whose block still fits ``max_n_token`` once the alignment pads are added."""+ max_n_token -= COMPRESS_PAD_TO - 1+ n_llm_h, n_llm_w, num_tokens = grid_tokens(best_height, best_width, patch_size, downsample_ratio)+ budget = max_n_token+ while num_tokens > max_n_token:+ n_llm_h, n_llm_w, best_height, best_width, num_tokens = solve_resize_ratio(+ height, width, patch_size, downsample_ratio, budget+ )+ budget -= 1+ return n_llm_h, n_llm_w, best_height, best_width+++def build_image_block(n_llm_h: int, n_llm_w: int, start_pos: int):+ """The block's token types in their final order, plus the aligner-row order of its IMAGE slots.++ ``types`` [n_out] places every token of the block (lead pads, START, the row-interleaved+ IMAGE tokens, trailing pads, END). The reference streams them as ``vocab_size + types``;+ here the span carries the image's content pad id and the assembled embeddings are+ scattered over it. ``perm`` [G] maps aligner outputs (row-major grid) into the IMAGE slots.++ ``start_pos`` is the block's absolute position in the prompt: the lead-pad count aligns+ the first row token to a compressor stride boundary, so the layout -- and the block's+ length -- depend on where the image lands.+ """+ compress_pad = COMPRESS_PAD_TO - 1 - start_pos % COMPRESS_PAD_TO+ pad_h = n_llm_h % 2+ rows = n_llm_h + pad_h+ row_len = n_llm_w + 1+ pad_last = rows // 2 * row_len % 2 * 2+ types = torch.tensor(+ ([IMAGE] * n_llm_w + [IMAGE_NEW_LINE]) * n_llm_h + [IMAGE_PAD] * (row_len * pad_h),+ dtype=torch.int64,+ )+ order = torch.arange(rows * row_len).view(rows // 2, 2, row_len).transpose(1, 2).reshape(-1)+ image_idx = torch.full((rows * row_len,), -1, dtype=torch.int64)+ image_idx.view(rows, row_len)[:n_llm_h, :n_llm_w] = torch.arange(n_llm_h * n_llm_w).view(n_llm_h, n_llm_w)+ perm = image_idx[order]+ perm = perm[perm >= 0]+ types = torch.cat(+ [+ torch.full((compress_pad,), IMAGE_PAD, dtype=torch.int64),+ torch.tensor([IMAGE_START]),+ types[order],+ torch.full((pad_last,), IMAGE_PAD, dtype=torch.int64),+ torch.tensor([IMAGE_END]),+ ]+ )+ return types, perm+++def assemble_block(+ aligned: torch.Tensor,+ types: torch.Tensor,+ perm: torch.Tensor,+ sentinels: torch.Tensor,+) -> torch.Tensor:+ """Block embeddings [n_out, dim] from the aligner rows and the learned sentinels.++ ``sentinels`` [5, dim] is ``(start, pad, pad, newline, end)``, indexable by the type+ enum exactly like the reference ``merge_image_embeddings``.+ """+ block = sentinels[types.to(sentinels.device)]+ block[types == IMAGE] = aligned[perm.to(aligned.device)]+ return block+++__all__ = [+ "COMPRESS_PAD_TO",+ "DSV4Aligner",+ "DSV4Vision",+ "DSV4VisionTower",+ "IMAGE",+ "IMAGE_END",+ "IMAGE_NEW_LINE",+ "IMAGE_PAD",+ "IMAGE_START",+ "assemble_block",+ "build_image_block",+ "grid_tokens",+ "safe_resize",+ "solve_resize_ratio",+]diff --git a/python/freetoken/models/deepseek_v4/weight.py b/python/freetoken/models/deepseek_v4/weight.pyindex 4728f7f9..fbcddb1c 100644--- a/python/freetoken/models/deepseek_v4/weight.py+++ b/python/freetoken/models/deepseek_v4/weight.py@@ -78,12 +78,16 @@ def iter_weights(*,include_moe_experts: bool = True,include_non_moe: bool = True,+ include_vision: bool = True,):"""Stream resident (non-expert) weights as ``(name, tensor)`` keyed to engine params.Routed MXFP4 experts come from the offload cache, so ``include_moe_experts`` must beFalse (DeepSeek-V4 only runs ``--moe-strategy offload``). Tensors yielded in checkpointdtype (fp8 + e8m0 preserved); ``wo_a`` dequantized to bf16 to match the reference einsum.+ ``include_vision`` (the engine's active-encoder set) drops the ``visual.*`` tower+ tensors; the router biases stay in either way, since the gate builds+ them on any vision checkpoint."""if include_moe_experts:raise ValueError(@@ -112,6 +116,9 @@ def iter_weights(for nm in ("hc_head_fn", "hc_head_base", "hc_head_scale"):yield f"model.{nm}", get(nm)+ if include_vision:+ yield from _iter_vision(reader, args)+for L in range(args.n_layers):a = f"layers.{L}.attn"m = f"model.{a}"@@ -147,8 +154,13 @@ def iter_weights(yield f"model.{g}.weight", get(f"{g}.weight")if L < args.n_hash_layers:yield f"model.{g}.tid2eid", get(f"{g}.tid2eid")+ if args.vision_n_layers > 0:+ # a vision checkpoint gives its hash layers a text bias too+ yield f"model.{g}.bias", get(f"{g}.bias")else:yield f"model.{g}.bias", get(f"{g}.bias")+ if args.vision_n_layers > 0:+ yield f"model.{g}.bias_vl", get(f"{g}.bias_vl")for proj in ("w1", "w2", "w3"):src = f"layers.{L}.ffn.shared_experts.{proj}"yield from linear(src, f"model.{src}")@@ -162,6 +174,58 @@ def iter_weights(reader.close()+# --------------------------------------------------------------------------------------+# Vision tower.+# --------------------------------------------------------------------------------------+# The wrapper mounts the tower, the aligner and the sentinels under one ``visual``+# container, so every engine-side name carries the ``visual.`` prefix the shared+# VISION_KEY_PREFIXES filter matches; the checkpoint's unprefixed names+# (``vision.*``, ``aligner.*``, ``image_*``) are renamed here on load.+_VISION_BLOCK_SUFFIXES = (+ "norm1.weight",+ "attn.wqkv.weight",+ "attn.wqkv.bias",+ "attn.wo.weight",+ "attn.wo.bias",+ "norm2.weight",+ "mlp.w1.weight",+ "mlp.w2.weight",+)+_VISION_TAIL = (+ "vision.patch_embed.proj.weight",+ "vision.patch_embed.proj.bias",+ "vision.norm.weight",+ "aligner.w1.weight",+ "aligner.w1.bias",+ "aligner.w2.weight",+ "aligner.w2.bias",+ "image_start",+ "image_end",+ "image_newline",+ "image_pad",+)+++def _iter_vision(reader, args: DeepseekV4Args):+ if args.vision_n_layers <= 0:+ return+ for L in range(args.vision_n_layers):+ for suffix in _VISION_BLOCK_SUFFIXES:+ yield f"visual.vision.blocks.{L}.{suffix}", reader.get(f"vision.blocks.{L}.{suffix}")+ for name in _VISION_TAIL:+ yield f"visual.{name}", reader.get(name)+++def iter_vision_weights(model_path: str, device):+ """The vision tower alone, for the engine's encoder-only load and the FTW converter."""+ args = load_args(model_path, max_batch_size=1)+ reader = _ShardReader(model_path, _weight_map(model_path), device)+ try:+ yield from _iter_vision(reader, args)+ finally:+ reader.close()++# --------------------------------------------------------------------------------------# Routed MXFP4 expert pieces.# --------------------------------------------------------------------------------------@@ -212,4 +276,4 @@ def iter_expert_pieces(model_path: str, config, kind: QuantKind, *, parallel: boreturn per_expert_pieces(_serial(), locate, tensors_per_expert=6)-__all__ = ["iter_weights", "iter_expert_pieces"]+__all__ = ["iter_weights", "iter_expert_pieces", "iter_vision_weights"]diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex fe150ddc..678424ff 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -84,6 +84,10 @@ _MINIMAX_M3_PACKED = _DENSE_PACKED + () + _EXPERTS_W123_PACKED_MINIMAX_M3_PROCESSOR = "freetoken.mm.processors.minimax_m3:MiniMaxM3MMProcessor"_MINIMAX_M3_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)+# DeepSeek-V4's vision dims sit at the top level of the checkpoint config, next to the+# language model's rather than under a section of their own.+_DSV4_PROCESSOR = "freetoken.mm.processors.deepseek_v4:DSV4MMProcessor"+_DSV4_ENCODERS = (EncoderSpec("vision", "vision_n_layers", ("image",)),)_MODEL_REGISTRY: dict[str, ModelSpec] = {"LlamaForCausalLM": ModelSpec(@@ -153,8 +157,13 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {# the checkpoint has no ``model.`` rootcheckpoint_roots=(("model.layers", "layers"), ("model.head", "head")),packed_modules_mapping=_EXPERTS_W123_PACKED,- # the head, the KV compressors and the indexer's scorer ship bf16; the fp8 config has no modules_to_not_convert- unquantized_modules=("head", "*.compressor.wkv", "*.compressor.wgate", "*.indexer.weights_proj"),+ # the head, the KV compressors, the indexer's scorer and the vision tower + aligner ship bf16;+ # the fp8 config has no modules_to_not_convert (patterns match checkpoint names, like V4.1)+ unquantized_modules=(+ "head", "*.compressor.wkv", "*.compressor.wgate", "*.indexer.weights_proj", "vision.*", "aligner.*",+ ),+ mm_processor=_DSV4_PROCESSOR,+ encoders=_DSV4_ENCODERS,),# DeepSeek-V4.1-Flash (model_type deepseek_v41): multimodal wrapper config (text tower in# text_config). The checkpoint has no ``model.`` root; the compressors, the indexer's key projection /diff --git a/scripts/ftw_hotfix.py b/scripts/ftw_hotfix.pyindex 9640c7b7..12574651 100644--- a/scripts/ftw_hotfix.py+++ b/scripts/ftw_hotfix.py@@ -264,7 +264,7 @@ def is_checkpoint_tower_name(name: str) -> bool:if head[0] == "model" and len(head) > 1:head = head[1:]return ("vision" in head[0] or "visual" in head[0] or head[0] in _TOWER_SEGMENTS- or head[0] in ("aligner", "image_start", "image_end", "image_newline")+ or head[0] in ("aligner", "image_start", "image_end", "image_newline", "image_pad")or name.endswith(".ffn.gate.bias_vl"))diff --git a/tests/README.md b/tests/README.mdindex 76b2b1fb..adc87035 100644--- a/tests/README.md+++ b/tests/README.md@@ -63,6 +63,8 @@ checkpoint is set:| `FREETOKEN_REBUILD_TEST_MODEL` | `e2e/test_cache_rebuild.py` — a SMALL local model dir; boots a real server (falls back to `FREETOKEN_TEST_MODEL`) || `FREETOKEN_GEMMA4_GGUF_GLOB` | `models/test_gemma4_gguf_rope.py` — glob matching a local gemma-4 GGUF file || `FREETOKEN_TEST_MODELS_ROOT` | `models/test_quant_config.py` — directory whose checkpoints the stored-tensor scheme scan checks (default `/mnt/nvme/models`) |+| `FREETOKEN_DSV4_VL_MODEL` | `tokenizer/test_mm_tokenize.py` - local DeepSeek-V4 vision checkpoint |+| `FREETOKEN_DSV4_VISION_CKPT` | `models/test_dsv4_vision.py` - the same checkpoint, for the parity tests against its `inference/` reference code |`test_aime.py` takes its sampling protocol from the checkpoint's own`generation_config.json` (pass@N at the recommended temperature, or a single greedydiff --git a/tests/checkpoint/test_ftw_hotfix_vision.py b/tests/checkpoint/test_ftw_hotfix_vision.pyindex eb0c592f..ef9225a9 100644--- a/tests/checkpoint/test_ftw_hotfix_vision.py+++ b/tests/checkpoint/test_ftw_hotfix_vision.py@@ -37,6 +37,7 @@ def hotfix():("vision.blocks.0.norm1.weight", True),("aligner.w1.weight", True),("image_newline", True),+ ("image_pad", True),("layers.3.ffn.gate.bias_vl", True),("model.embed_audio.embedding_projection.weight", False),("model.language_model.layers.0.self_attn.q_proj.weight", False),diff --git a/tests/mm/test_processor.py b/tests/mm/test_processor.pyindex a9a50e3c..9562b366 100644--- a/tests/mm/test_processor.py+++ b/tests/mm/test_processor.py@@ -9,7 +9,7 @@ import pytestimport torchfrom freetoken.message import MMItem-from freetoken.mm import MM_PAD_SHIFT_VALUE+from freetoken.mm import MM_PAD_SHIFT_VALUE, mm_pad_valuefrom freetoken.mm.config import MultimodalConfigfrom freetoken.mm.processor import MMProcessor, PromptReplacement, image_positionsfrom freetoken.mm.processors.gemma4 import Gemma4MMProcessor, Gemma4UnifiedMMProcessor@@ -17,6 +17,7 @@ from freetoken.mm.processors.glm5_next import Glm5NextMMProcessorfrom freetoken.mm.processors.muse_glimmer import MuseGlimmerMMProcessorfrom freetoken.mm.processors.minimax_m3 import IMAGE_END_ID, IMAGE_START_ID, MiniMaxM3MMProcessorfrom freetoken.mm.processors.qwen_vl import QwenVLMMProcessor+from freetoken.mm.processors.deepseek_v4 import DSV4MMProcessorPLACEHOLDER = 7BOI, SOFT, EOI = 90, 91, 92@@ -415,3 +416,103 @@ def test_registry_resolves_by_architecture(monkeypatch):assert get_mm_processor("/text-only") is Noneassert get_mm_processor("/unknown-vlm") is Noneassert get_mm_processor("/missing") is None # a config that fails to load means no vision+++def _dsv4_config(**over):+ """The vision dims sit at the top level of a deepseek_v4 checkpoint config."""+ base = dict(+ vision_n_layers=32,+ vision_dim=1024,+ vision_n_heads=16,+ vision_inter_dim=2816,+ vision_patch_size=14,+ vision_rope_theta=10000.0,+ vision_downsample_ratio=3,+ vision_max_n_token=384,+ vision_min_pixels=147456,+ vision_max_wh_ratio=8,+ hidden_size=4096,+ )+ return SimpleNamespace(**{**base, **over})+++def _dsv4_image(width, height):+ from PIL import Image++ return Image.new("RGB", (width, height))+++def test_dsv4_block_length_and_hash_follow_the_insertion_offset():+ from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID+ from freetoken.models.deepseek_v4.vision import build_image_block++ proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())+ assert proc.placeholder == [IMAGE_TOKEN_ID]+ (item,) = proc.process([_dsv4_image(640, 480)])+ assert (item.n_vit_h, item.n_vit_w, item.n_llm_h, item.n_llm_w) == (35, 46, 12, 16)++ repl = proc.prompt_replacement(item, 0)+ at_zero, hash_zero = len(repl.full), item.hash+ assert at_zero == len(build_image_block(12, 16, 0)[0]) == 209+ assert repl.full == [IMAGE_TOKEN_ID] * at_zero+ # the block's sentinels are embeddings too, so the whole span takes image rows+ assert repl.embed_spans() == [[0, at_zero]]+ assert item.pad_value == mm_pad_value(item.hash) and item.pad_value >= MM_PAD_SHIFT_VALUE++ # the lead pads align the block to a compression stride, so a later offset is a shorter+ # block AND a different content hash: one alignment's rows must never serve another's+ repl = proc.prompt_replacement(item, 2)+ assert len(repl.full) == at_zero - 2 and item.hash != hash_zero+ assert item.pad_value == mm_pad_value(item.hash) >= MM_PAD_SHIFT_VALUE+++def test_dsv4_dummy_item_is_the_smallest_encodable_block():+ from freetoken.models.deepseek_v4.vision import build_image_block++ proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())+ (dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))+ dummy.validate()+ assert dummy.feature.shape == (36, 3, 14, 14) # a 6x6 patch grid, one 2x2 merge grid+ assert dummy.num_tokens == len(build_image_block(2, 2, 0)[0]) == 13+++def test_a_text_only_checkpoint_config_has_no_dsv4_processor():+ proc = DSV4MMProcessor+ with pytest.raises(ValueError):+ proc(_dsv4_config(vision_n_layers=0), "/nonexistent", MultimodalConfig())+++def test_dsv4_apply_passes_each_block_the_offset_it_lands_at():+ """Only the DSV4 block depends on its position, so its own apply computes the offset."""+ from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID+ from freetoken.models.deepseek_v4.vision import build_image_block++ proc = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())+ buf = io.BytesIO()+ _dsv4_image(640, 480).save(buf, format="PNG")+ raw = buf.getvalue()+ ids = torch.tensor([1, IMAGE_TOKEN_ID, 2, IMAGE_TOKEN_ID, 3], dtype=torch.int32)+ out = proc.apply(ids, [raw, raw])+ first, second = out.mm_items+ assert (first.n_llm_h, first.n_llm_w) == (12, 16)+ # the first placeholder sits at 1, the second after the text between plus the first block+ assert first.start == first.offsets[0][0] == 1+ assert second.start == second.offsets[0][0]+ assert first.num_tokens == len(build_image_block(12, 16, first.start)[0])+ assert second.num_tokens == len(build_image_block(12, 16, second.start)[0])+ # apply fills the embedding slots with each block's content pad id, not the placeholder+ for item in (first, second):+ lo, hi = item.offsets[0]+ assert set(out.input_ids[lo:hi].tolist()) == {item.pad_value}+ assert IMAGE_TOKEN_ID not in out.input_ids.tolist()+++def test_dsv4_image_max_tokens_lowers_the_checkpoints_block_cap():+ from freetoken.models.deepseek_v4.vision import build_image_block++ big = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig())+ capped = DSV4MMProcessor(_dsv4_config(), "/nonexistent", MultimodalConfig(image_max_tokens=64))+ (a,) = big.process([_dsv4_image(1600, 1200)])+ (b,) = capped.process([_dsv4_image(1600, 1200)])+ assert b.n_llm_h * b.n_llm_w < a.n_llm_h * a.n_llm_w+ assert len(build_image_block(b.n_llm_h, b.n_llm_w, 0)[0]) <= 64diff --git a/tests/models/test_dsv4_vision.py b/tests/models/test_dsv4_vision.pynew file mode 100644index 00000000..68a3f0ab--- /dev/null+++ b/tests/models/test_dsv4_vision.py@@ -0,0 +1,335 @@+"""DeepSeek-V4 image preprocessing, block layout and the tiny vision tower (no checkpoint).++The block layout is what makes an image's token count depend on where it lands in the+prompt, so it is pinned here against hand-written expectations; the checkpoint-gated test+at the bottom checks the same functions against the reference ``inference/`` code.+"""++from __future__ import annotations++import io+import os++import numpy as np+import pytest+import torch+from PIL import Image++from freetoken.models.deepseek_v4.config import DSV4VisionConfig, parse_config, parse_vision_config+from freetoken.models.deepseek_v4.image_processor import load_image+from freetoken.models.deepseek_v4.vision import (+ COMPRESS_PAD_TO,+ DSV4Aligner,+ DSV4VisionTower,+ IMAGE,+ IMAGE_END,+ IMAGE_NEW_LINE,+ IMAGE_PAD,+ IMAGE_START,+ assemble_block,+ build_image_block,+)++checkpoint_path = os.environ.get("FREETOKEN_DSV4_VISION_CKPT")+needs_checkpoint = pytest.mark.skipif(+ not checkpoint_path, reason="needs FREETOKEN_DSV4_VISION_CKPT pointing at the vision checkpoint"+)++_HF_CONFIG = {+ "vision_n_layers": 32,+ "vision_dim": 1024,+ "vision_n_heads": 16,+ "vision_inter_dim": 2816,+ "vision_patch_size": 14,+ "vision_rope_theta": 10000.0,+ "vision_downsample_ratio": 3,+ "vision_max_n_token": 384,+ "vision_min_pixels": 147456,+ "vision_max_wh_ratio": 8,+ "hidden_size": 4096,+}+++def _hf_config(**over):+ from types import SimpleNamespace++ return SimpleNamespace(**{**_HF_CONFIG, **over})+++def _vc(**over) -> DSV4VisionConfig:+ """A tower small enough to run: 2 patch-2 blocks over a 2x2 merge grid."""+ base = dict(+ vision_n_layers=2,+ vision_dim=16,+ vision_n_heads=2,+ vision_inter_dim=32,+ vision_patch_size=4,+ vision_rope_theta=10000.0,+ vision_downsample_ratio=2,+ vision_max_n_token=384,+ vision_min_pixels=147456,+ vision_max_wh_ratio=8,+ text_dim=64,+ )+ return DSV4VisionConfig(**{**base, **over})+++def _png(width, height, seed=0):+ rng = np.random.default_rng(seed)+ array = (rng.random((height, width, 3)) * 255).astype(np.uint8)+ buf = io.BytesIO()+ Image.fromarray(array, "RGB").save(buf, format="PNG")+ return buf.getvalue()+++def test_a_vision_checkpoint_config_becomes_the_tower_dims_and_a_text_one_does_not():+ vc = parse_vision_config(_hf_config())+ assert (vc.vision_n_layers, vc.vision_dim, vc.text_dim) == (32, 1024, 4096)+ # the engine nulls the section on the copy it hands the parser when it serves text-only+ assert parse_vision_config(_hf_config(vision_n_layers=None)) is None+ assert parse_vision_config(_hf_config(vision_n_layers=0)) is None+++def test_block_layout_interleaves_rows_and_pads_to_the_compression_stride():+ types, perm = build_image_block(2, 2, 0)+ # 3 lead pads put the first row token on a COMPRESS_PAD_TO (4) boundary, then START,+ # the N-layout rows (column pairs: 2 rows of 2 IMAGE tokens plus their row markers),+ # the trailing parity pad, END.+ assert types.tolist() == (+ [IMAGE_PAD] * 3+ + [IMAGE_START]+ + [IMAGE, IMAGE, IMAGE, IMAGE, IMAGE_NEW_LINE, IMAGE_NEW_LINE]+ + [IMAGE_PAD] * 2+ + [IMAGE_END]+ )+ # row-major aligner outputs 0..3 land in the IMAGE slots in N-layout order+ assert perm.tolist() == [0, 2, 1, 3]+++def test_the_block_shrinks_by_one_token_per_lead_pad_the_offset_absorbs():+ # the lead pads are what aligns the block, so the same image at a later offset is a+ # SHORTER span: the count cannot be a property of the image alone+ lengths = [len(build_image_block(2, 2, start)[0]) for start in range(4)]+ assert lengths == [13, 12, 11, 10]+ # an odd row count adds a marker row, which is not pad bytes+ assert [len(build_image_block(3, 2, start)[0]) for start in range(4)] == [17, 16, 15, 14]+++def test_assemble_block_puts_the_sentinels_and_the_aligner_rows_where_the_types_say():+ types, perm = build_image_block(2, 2, 0)+ sentinels = torch.stack([torch.full((4,), float(i)) for i in range(5)])+ aligned = torch.arange(4 * 4, dtype=torch.float32).view(4, 4) * 10+ block = assemble_block(aligned, types, perm, sentinels)+ assert block.shape == (len(types), 4)+ # sentinels index the type enum, so START/PAD/END rows carry 0/1/4 and the IMAGE rows+ # carry the aligner output perm says they take+ assert block[0].tolist() == [1.0] * 4 # lead pad+ assert block[3].tolist() == [0.0] * 4 # IMAGE_START+ assert block[-1].tolist() == [4.0] * 4 # IMAGE_END+ image_rows = block[types == IMAGE]+ assert image_rows.tolist() == aligned[perm].tolist()+++def test_preprocessing_resizes_to_the_merge_grid_and_normalizes_like_the_reference():+ # min_pixels 0 keeps the input size: the reference upscales below it+ vc = _vc(vision_min_pixels=0)+ patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = load_image(+ Image.open(io.BytesIO(_png(80, 64))), vc+ )+ assert (n_vit_h, n_vit_w, n_llm_h, n_llm_w) == (16, 20, 8, 10)+ assert patches.shape == (n_vit_h * n_vit_w, 3, 4, 4)+ assert patches.dtype == torch.bfloat16+ assert float(patches.min()) >= -1.0 and float(patches.max()) <= 1.0+ # identical bytes, identical patches: the content hash and the radix key ride on these+ again, *_ = load_image(Image.open(io.BytesIO(_png(80, 64))), vc)+ assert torch.equal(patches, again)+++def test_the_token_budget_caps_a_tall_image():+ vc = _vc(vision_max_n_token=64, vision_min_pixels=0)+ _patches, _n_vit_h, _n_vit_w, n_llm_h, n_llm_w = load_image(+ Image.open(io.BytesIO(_png(64, 1024))), vc+ )+ types, _perm = build_image_block(n_llm_h, n_llm_w, 0)+ assert len(types) <= vc.vision_max_n_token+++def test_tiny_tower_runs_and_its_aligner_merges_every_merge_grid():+ vc = _vc()+ tower = DSV4VisionTower(vc)+ aligner = DSV4Aligner(vc)+ patches = torch.zeros(20 * 24, 3, 4, 4, dtype=torch.bfloat16)+ out = tower.forward(patches, 20, 24)+ assert out.shape == (20 * 24, vc.vision_dim)+ assert aligner.forward(out, 20, 24).shape == (10 * 12, vc.text_dim)+++def _reference_modules(path):+ """Import the checkpoint's own ``inference/image_processor.py`` (PIL + torch only)."""+ import importlib.util++ module_path = os.path.join(path, "inference", "image_processor.py")+ if not os.path.exists(module_path):+ pytest.skip(f"{module_path} not found")+ spec = importlib.util.spec_from_file_location("_dsv4_reference_image_processor", module_path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++@needs_checkpoint+def test_preprocessing_and_block_layout_match_the_reference():+ from types import SimpleNamespace++ reference = _reference_modules(checkpoint_path)+ vc = parse_vision_config(_hf_config())+ args = SimpleNamespace(+ vision_patch_size=vc.vision_patch_size,+ vision_downsample_ratio=vc.vision_downsample_ratio,+ vision_max_n_token=vc.vision_max_n_token,+ vision_min_pixels=vc.vision_min_pixels,+ vision_max_wh_ratio=vc.vision_max_wh_ratio,+ )+ for width, height in ((640, 480), (100, 100), (3000, 400), (1600, 1200)):+ raw = _png(width, height)+ mine = load_image(Image.open(io.BytesIO(raw)), vc)+ theirs = reference.load_image({"data": raw}, args)+ assert torch.equal(mine[0], theirs[0])+ assert mine[1:] == theirs[1:]+ for start in (0, 1, 3, 7):+ types, perm = build_image_block(mine[3], mine[4], start)+ ref_types, ref_perm = reference.build_image_block(theirs[3], theirs[4], start)+ assert torch.equal(types, ref_types) and torch.equal(perm, ref_perm)+++_WINDOW, _MAX_IMG = 128, 384+++def _ref_get_image_visible(input_ids: torch.Tensor, vocab_size: int, max_image_tokens: int):+ """Verbatim port of the checkpoint's ``inference/model.py`` pair (not vendored in this+ repo): ``get_image_visible`` derives spans from the virtual ids ``vocab + type``, so the+ lead pads before IMAGE_START stay outside; only the START/END comparisons matter, our+ type enum values index the same way."""+ seqlen = input_ids.size(1)+ idx = torch.arange(seqlen, dtype=torch.int32).unsqueeze(0)+ is_start = input_ids == vocab_size + IMAGE_START+ is_end = input_ids == vocab_size + IMAGE_END+ valid = (is_start.cumsum(1) > is_end.cumsum(1)) | is_end+ starts = torch.where(is_start, idx, 0).cummax(1)[0]+ left = (idx - starts) * valid+ ends = torch.where(is_end, idx, seqlen).flip(1).cummin(1)[0].flip(1)+ right = (ends - idx) * valid+ return left.clamp(max=max_image_tokens - 1), right.clamp(max=max_image_tokens)+++def _ref_get_window_topk_idxs_visible(window_size, seqlen, left, right, max_image_tokens):+ width = min(seqlen, window_size + max_image_tokens)+ idx = torch.arange(seqlen).unsqueeze(0)+ left_add = (left - (window_size - 1)).clamp(min=0)+ starts = (idx - (window_size - 1) - left_add).clamp(min=0)+ matrix = starts.unsqueeze(-1) + torch.arange(width)+ return torch.where(matrix > (idx + right).unsqueeze(-1), -1, matrix).int().contiguous()+++def _reference_candidates(prefix: int, n_llm_h: int, n_llm_w: int, vocab: int = 100_000):+ """The reference pair's window candidates for a text prefix followed by one image block."""+ types, _perm = build_image_block(n_llm_h, n_llm_w, prefix)+ ids = torch.full((1, prefix + len(types)), vocab + 50, dtype=torch.int64)+ ids[0, prefix:] = vocab + types+ n = ids.size(1)+ left, right = _ref_get_image_visible(ids, vocab, _MAX_IMG)+ block = (prefix, prefix + len(types))+ return block, types, _ref_get_window_topk_idxs_visible(_WINDOW, n, left, right, _MAX_IMG)[0]+++def test_model_spans_match_the_reference_visibility_pair():+ from types import SimpleNamespace++ from freetoken.models.deepseek_v4.attention import Attention+ from freetoken.models.deepseek_v4.model import _image_spans_by_ti++ for prefix, h, w in ((0, 3, 2), (1, 3, 2), (2, 3, 2), (3, 3, 2), (5, 4, 3), (201, 20, 16), (402, 2, 2)):+ (lo, hi), types, ref = _reference_candidates(prefix, h, w)+ req = SimpleNamespace(table_idx=7, mm_items=[SimpleNamespace(offsets=[[lo, hi]])])+ spans = _image_spans_by_ti([req])+ assert spans == {7: [(lo + types.tolist().index(IMAGE_START), hi)]}+ cols = Attention.visible_window_cols(+ spans[7], 0, hi, _WINDOW, _MAX_IMG, torch.device("cpu"),+ addr_lo=0, width=min(hi, _WINDOW + _MAX_IMG),+ )+ assert torch.equal(cols, ref)+ text = SimpleNamespace(table_idx=0, mm_items=None)+ assert _image_spans_by_ti([text]) is None+++def test_a_span_started_at_the_first_lead_pad_breaks_reference_parity():+ from freetoken.models.deepseek_v4.attention import Attention++ # the maintainer's case: a 201-token prefix puts 2 lead pads in front of a 20x16 grid,+ # and a span covering the whole block diverges from the reference on 218 rows+ (lo, hi), _types, ref = _reference_candidates(201, 20, 16)+ assert lo + COMPRESS_PAD_TO - 1 - lo % COMPRESS_PAD_TO - lo == 2 # the two lead pads+ cols = Attention.visible_window_cols(+ [(lo, hi)], 0, hi, _WINDOW, _MAX_IMG, torch.device("cpu"),+ addr_lo=0, width=min(hi, _WINDOW + _MAX_IMG),+ )+ assert int((cols != ref).any(dim=1).sum()) == 218+++def test_a_vision_checkpoint_keeps_image_blocks_whole_across_chunk_boundaries(tmp_path):+ """Image spans attend bidirectionally, so a vision checkpoint sets+ bidirectional_mm_blocks on its attention group and the scheduler ends a chunk before a+ block instead of splitting it; text-only defaults to False and keeps plain chunking."""+ from types import SimpleNamespace++ from freetoken.core import Context, SamplingParams, get_global_ctx, set_global_ctx+ from freetoken.message import MMItem+ from freetoken.scheduler.cache import CacheManager+ from freetoken.scheduler.decode import DecodeManager+ from freetoken.scheduler.mm import cut_image_spans+ from freetoken.scheduler.prefill import PrefillManager+ from freetoken.scheduler.table import TableManager+ from freetoken.scheduler.utils import PendingReq++ # DeepseekV4Args defaults fill an empty inference/config.json; the hf namespace carries the tower+ (tmp_path / "inference").mkdir()+ (tmp_path / "inference" / "config.json").write_text("{}")+ vision = parse_config(SimpleNamespace(_name_or_path=str(tmp_path), **_HF_CONFIG))+ text = parse_config(SimpleNamespace(_name_or_path=str(tmp_path), **{**_HF_CONFIG, "vision_n_layers": None}))+ assert vision.attention_groups[0].bidirectional_mm_blocks is True+ assert text.attention_groups[0].bidirectional_mm_blocks is False++ prefix = 10+ types, _perm = build_image_block(2, 2, prefix)+ lo, hi = prefix, prefix + len(types)+ image = MMItem(modality="image", hash=1, pad_value=1, offsets=[[lo, hi]], feature=torch.zeros(1))+ assert lo < 15 < hi # a plain 15-token chunk budget ends inside the block++ def chunk_lens(keep_images_whole):+ try:+ get_global_ctx()+ except AssertionError:+ set_global_ctx(Context(page_size=1))+ pt = torch.zeros((5, 64), dtype=torch.int32)+ cm = CacheManager(num_pages=64, page_size=1, page_table=pt, type="radix")+ pm = PrefillManager(+ cm, TableManager(max_running_reqs=4, page_table=pt), DecodeManager(1),+ keep_images_whole=keep_images_whole,+ )+ pm.pending_list = [PendingReq(uid=7, input_ids=torch.arange(30, dtype=torch.int32),+ sampling_params=SamplingParams(max_tokens=4), mm_items=[image])]+ lens, cuts = [], []+ while pm.runnable:+ batch = pm.schedule_next_batch(15)+ assert batch is not None+ cm.allocate_paged(batch.reqs)+ cuts.extend(cut_image_spans(batch.reqs))+ for r in batch.reqs:+ lens.append(r.extend_len)+ r.complete_one()+ return lens, cuts++ lens, cuts = chunk_lens(vision.attention_groups[0].bidirectional_mm_blocks)+ assert lens == [lo, 15, 30 - lo - 15] and cuts == [] # first chunk stops at the block, the next holds it whole+ lens, cuts = chunk_lens(text.attention_groups[0].bidirectional_mm_blocks)+ assert lens == [15, 15] and cuts == [(lo, hi)] # the block is split into two spans; the cut warning would firediff --git a/tests/scheduler/test_mm.py b/tests/scheduler/test_mm.pyindex 13447634..72db7e3d 100644--- a/tests/scheduler/test_mm.py+++ b/tests/scheduler/test_mm.py@@ -6,7 +6,13 @@ import torchfrom freetoken.message import MMItemfrom freetoken.mm.encoder_cache import EncoderCache-from freetoken.scheduler.mm import cut_image_spans, mm_chunk_end, mm_rows_after, plan_mm_batch, plan_mm_chunk+from freetoken.scheduler.mm import (+ cut_image_spans,+ mm_chunk_end,+ mm_rows_after,+ plan_mm_batch,+ plan_mm_chunk,+)CPU = torch.device("cpu")diff --git a/tests/tokenizer/test_mm_tokenize.py b/tests/tokenizer/test_mm_tokenize.pyindex 8f2e0211..5adac830 100644--- a/tests/tokenizer/test_mm_tokenize.py+++ b/tests/tokenizer/test_mm_tokenize.py@@ -1,4 +1,4 @@-"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3, Muse-Glimmer and MiniMax-M3 checkpoints (skipped when absent)."""+"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3, Muse-Glimmer, MiniMax-M3 and DeepSeek-V4-Vision checkpoints (skipped when absent)."""from __future__ import annotations@@ -20,12 +20,13 @@ GEMMA = os.environ.get("FREETOKEN_GEMMA4_MODEL", "")GLM = os.environ.get("FREETOKEN_GLM53_MODEL", "")MUSE = os.environ.get("FREETOKEN_MUSE_MODEL", "")MINIMAX = os.environ.get("FREETOKEN_MINIMAX_M3_MODEL", "")+DSV4 = os.environ.get("FREETOKEN_DSV4_VL_MODEL", "")pytestmark = [pytest.mark.needs_weights,pytest.mark.skipif(- not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE, MINIMAX)),- reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL / FREETOKEN_MINIMAX_M3_MODEL checkpoint",+ not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE, MINIMAX, DSV4)),+ reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL / FREETOKEN_MINIMAX_M3_MODEL / FREETOKEN_DSV4_VL_MODEL checkpoint",),]@@ -214,3 +215,51 @@ def test_minimax_wraps_the_pad_span_in_start_end_tokens():assert bool((ids[start:end] == item.pad_value).all())budgeted = TokenizeManager(tokenizer, get_mm_processor(MINIMAX, MultimodalConfig(image_max_tokens=64))).tokenize([_msg([_png(640, 400)])])[0]assert budgeted.mm_items[0].num_tokens <= 64+++ not os.path.exists(os.path.join(DSV4, "config.json")),+ reason="no FREETOKEN_DSV4_VL_MODEL checkpoint",+)+def test_dsv4_expands_the_placeholder_into_the_reference_block():+ """The block carries one content pad id, and its length follows the reference's grid."""+ from freetoken.mm.processor import get_mm_processor+ from freetoken.tokenizer.tokenize import TokenizeManager+ from freetoken.utils.hf import load_tokenizer++ manager = TokenizeManager(load_tokenizer(DSV4), get_mm_processor(DSV4))+ # 640x480 through the reference: a 12x16 merge grid, so 206 tokens of rows and markers+ # plus the 3 lead pads that align the block at this offset+ out = manager.tokenize([_msg([_png(640, 480)])])[0]+ ids = out.input_ids.tolist()+ (item,) = out.mm_items+ assert (item.n_llm_h, item.n_llm_w) == (12, 16)+ assert item.num_tokens == 209+ lo, hi = item.offsets[0]+ assert (lo, hi) == (item.start, item.start + 209)+ assert set(ids[lo:hi]) == {item.pad_value}+ assert item.pad_value >= MM_PAD_SHIFT_VALUE and item.pad_value == mm_pad_value(item.hash)+ assert 129264 not in ids # the raw <|deepseek_image|> placeholder is gone+ assert item.feature.shape == (35 * 46, 3, 14, 14) and item.feature.dtype == torch.bfloat16+++ not os.path.exists(os.path.join(DSV4, "config.json")),+ reason="no FREETOKEN_DSV4_VL_MODEL checkpoint",+)+def test_dsv4_two_images_take_two_blocks_and_the_second_is_shorter():+ """The second image sits at a different offset, so its block is a different length."""+ from freetoken.mm.processor import get_mm_processor+ from freetoken.tokenizer.tokenize import TokenizeManager+ from freetoken.utils.hf import load_tokenizer++ manager = TokenizeManager(load_tokenizer(DSV4), get_mm_processor(DSV4))+ raw = _png(320, 240)+ out = manager.tokenize([_msg([raw, raw], n_parts=2)])[0]+ first, second = out.mm_items+ assert (first.start, second.start) == (first.offsets[0][0], second.offsets[0][0])+ # the lead pads take up to three tokens of slack, so the blocks nearly match+ assert abs(second.num_tokens - first.num_tokens) <= 3+ # two images share a pad id only when their blocks are laid out identically, which is+ # exactly when their offsets are congruent: otherwise one must not serve the other's key+ assert (first.pad_value == second.pad_value) == ((second.start - first.start) % 4 == 0)