Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

5.7 KB
Raw Blame
"""DeepSeek-V4 image processor: each ``<|deepseek_image|>`` expands into the image's block of tokens.

Where the reference streams ``vocab_size + type`` pseudo ids, the whole span carries one
content pad id and the block's embeddings -- sentinels included -- come from the encoder.
"""

from __future__ import annotations

import io
import struct
from typing import Any

import torch

from freetoken.message import MMItem
from freetoken.mm import mm_pad_value
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import MMResult, MMProcessor, PromptReplacement, _find_all, content_hash
from freetoken.models.deepseek_v4.config import IMAGE_TOKEN_ID, DSV4VisionConfig, parse_vision_config
from freetoken.models.deepseek_v4.image_processor import load_image
from freetoken.models.deepseek_v4.vision import COMPRESS_PAD_TO, build_image_block


class DSV4MMProcessor(MMProcessor):
    def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:
        super().__init__(model_path, mm)
        vc = parse_vision_config(hf_config)
        if vc is None:
            raise ValueError(f"{model_path} carries no vision tower (vision_n_layers <= 0)")
        self.vc: DSV4VisionConfig = vc
        self.image_token_id = IMAGE_TOKEN_ID
        self.placeholder = [self.image_token_id]
        # --image-max-tokens lowers the checkpoint's per-image block cap
        self.max_n_token = min(vc.vision_max_n_token, mm.image_max_tokens) if mm.image_max_tokens else vc.vision_max_n_token

    def process(self, images: list[Any]) -> list[MMItem]:
        items: list[MMItem] = []
        for image in images:
            patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = load_image(image, self.vc, self.max_n_token)
            # layout, hash and token count are only known at the insertion offset
            items.append(
                MMItem(
                    modality="image",
                    hash=0,
                    pad_value=0,
                    offsets=[],
                    feature=patches,
                    model_specific_data={
                        "n_vit_h": n_vit_h,
                        "n_vit_w": n_vit_w,
                        "n_llm_h": n_llm_h,
                        "n_llm_w": n_llm_w,
                    },
                )
            )
        return items

    def prompt_replacement(self, item: MMItem, start: int) -> PromptReplacement:
        """The DSV4-only ``start`` is where the block lands in input_ids; only our apply below calls this."""
        n_llm_h, n_llm_w = item.n_llm_h, item.n_llm_w
        types, _perm = build_image_block(n_llm_h, n_llm_w, start)
        # the lead pads align the block, so the same image at another offset is another
        # sequence: the layout belongs in the hash or the two would share a radix key
        compress_pad = COMPRESS_PAD_TO - 1 - start % COMPRESS_PAD_TO
        item.hash = content_hash(item.feature, struct.pack("<3i", n_llm_h, n_llm_w, compress_pad))
        item.pad_value = mm_pad_value(item.hash)
        item.model_specific_data["start"] = start
        return PromptReplacement([self.image_token_id] * types.numel())

    def apply(self, input_ids: torch.Tensor, images: list[bytes]) -> MMResult:
        """MMProcessor.apply, save that each replacement is built knowing the offset it lands at."""
        from PIL import Image

        ids = input_ids.tolist()
        target = self.placeholder
        slots = _find_all(ids, target)
        if len(slots) != len(images):
            raise ValueError(
                f"prompt renders {len(slots)} image placeholders but the request "
                f"carries {len(images)} images"
            )

        pils = [Image.open(io.BytesIO(raw)).convert("RGB") for raw in images]
        items = self.process(pils)
        out: list[int] = []
        cursor = 0
        for slot, item in zip(slots, items):
            out.extend(ids[cursor:slot])
            base = len(out)
            repl = self.prompt_replacement(item, base)
            full = list(repl.full)
            spans = repl.embed_spans()
            # embedding slots carry the content pad id so radix keys and the model's scatter mask see the image
            for lo, hi in spans:
                full[lo:hi] = [item.pad_value] * (hi - lo)
            out.extend(full)
            item.offsets = [[base + lo, base + hi] for lo, hi in spans]
            item.validate()
            cursor = slot + len(target)
        out.extend(ids[cursor:])
        new_ids = torch.tensor(out, dtype=input_ids.dtype)

        positions, delta = self.positions(len(out), items) or (None, 0)
        return MMResult(new_ids, items, positions, delta)

    def _image_processor(self) -> None:
        """None: this family's preprocessing is FreeToken-local (:func:`load_image`); no
        transformers image-processor class exists to build from a preprocessor_config."""
        return None

    def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
        # The smallest block the tower can run: one 2x2 merge grid out of a 6x6 patch grid.
        r, p = self.vc.vision_downsample_ratio, self.vc.vision_patch_size
        n_vit, n_llm = 2 * r, 2
        types, _perm = build_image_block(n_llm, n_llm, 0)
        return [
            MMItem(
                modality="image",
                hash=0,
                pad_value=0,
                offsets=[[0, types.numel()]],
                feature=torch.zeros(n_vit * n_vit, 3, p, p, dtype=dtype, device=device),
                model_specific_data={
                    "n_vit_h": n_vit,
                    "n_vit_w": n_vit,
                    "n_llm_h": n_llm,
                    "n_llm_w": n_llm,
                    "start": 0,
                },
            )
        ]


__all__ = ["DSV4MMProcessor"]