Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

8.6 KB
Raw Blame
"""Per-family multimodal processors and the architecture registry; MMProcessor.apply is the image half of tokenization."""

from __future__ import annotations

from abc import ABC, abstractmethod
import hashlib
import importlib
import io
import threading
from dataclasses import dataclass
from typing import Any

import torch

from freetoken.message import MMItem
from freetoken.mm.config import MultimodalConfig
from freetoken.mm import check_mm_pad_shift


@dataclass
class PromptReplacement:
    """The token sequence that replaces one placeholder; is_embed marks the positions that take image embeddings (None = all of them)."""

    full: list[int]
    is_embed: list[bool] | None = None

    @classmethod
    def select_token_id(cls, full: list[int], embed_token_id: int) -> PromptReplacement:
        return cls(full, [t == embed_token_id for t in full])

    def embed_spans(self) -> list[list[int]]:
        """Half-open [lo, hi) runs of embedding positions, relative to the start of full."""
        mask = self.is_embed if self.is_embed is not None else [True] * len(self.full)
        spans: list[list[int]] = []
        lo: int | None = None
        for i, flag in enumerate([*mask, False]):
            if flag and lo is None:
                lo = i
            elif not flag and lo is not None:
                spans.append([lo, i])
                lo = None
        return spans


@dataclass
class MMResult:
    input_ids: torch.Tensor
    mm_items: list[MMItem]
    mrope_positions: torch.Tensor | None
    mrope_delta: int


def _find_all(ids: list[int], target: list[int]) -> list[int]:
    """Start index of every non-overlapping occurrence of target."""
    n = len(target)
    found: list[int] = []
    i = 0
    while i <= len(ids) - n:
        if ids[i : i + n] == target:
            found.append(i)
            i += n
        else:
            i += 1
    return found


class MMProcessor(ABC):
    """How a checkpoint's images become items, tokens and positions; apply is the family-agnostic driver, subclasses fill the hooks.

    Built from the checkpoint config alone and import-light: tokenizer workers call apply without the model layer, the engine calls dummy_items.
    Image preprocessing must stay on the CPU with one fixed implementation: radix keys and the embedding cache are content hashes of pixel_values, and resize results differ across backends/devices.
    """

    # the token sequence the chat template renders once per image
    placeholder: list[int]

    def __init__(self, model_path: str, mm: MultimodalConfig) -> None:
        self.model_path = model_path
        self.mm = mm
        self._image_processor_instance: Any = None
        self._image_processor_lock = threading.Lock()

    def _image_processor(self) -> Any:
        """The checkpoint's image processor, loaded on first use (tokenizer workers share the instance across threads)."""
        with self._image_processor_lock:
            if self._image_processor_instance is None:
                from freetoken.utils import load_image_processor

                self._image_processor_instance = load_image_processor(self.model_path)
            return self._image_processor_instance

    def get_mm_processor_kwargs(self, mm: MultimodalConfig) -> dict[str, Any]:
        """Keyword arguments for the checkpoint processor call: the family's reading of the mm knobs, then mm.processor_kwargs on top; families with a token budget override this."""
        return dict(mm.processor_kwargs)

    @abstractmethod
    def process(self, images: list[Any]) -> list[MMItem]:
        """One MMItem per image with feature and hash; offsets are assigned by apply."""

    @abstractmethod
    def prompt_replacement(self, item: MMItem) -> PromptReplacement:
        """The token sequence that stands in for this item's placeholder in the prompt."""

    def positions(self, length: int, items: list[MMItem]) -> tuple[torch.Tensor, int] | None:
        """[3, length] t/h/w rope positions and the decode delta; None for 1-D rope families."""
        return None

    @abstractmethod
    def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
        """The smallest encodable item of each modality the family serves, for warming the encoders."""

    def apply(self, input_ids: torch.Tensor, images: list[bytes]) -> MMResult:
        """Swap each image placeholder for its replacement sequence and precompute rope positions."""
        from PIL import Image

        ids = input_ids.tolist()
        target = self.placeholder
        slots = _find_all(ids, target)
        if len(slots) != len(images):
            raise ValueError(
                f"prompt renders {len(slots)} image placeholders but the request "
                f"carries {len(images)} images"
            )

        pils = [Image.open(io.BytesIO(raw)).convert("RGB") for raw in images]
        items = self.process(pils)
        out: list[int] = []
        cursor = 0
        for slot, item in zip(slots, items):
            repl = self.prompt_replacement(item)
            out.extend(ids[cursor:slot])
            base = len(out)
            full = list(repl.full)
            spans = repl.embed_spans()
            # embedding slots carry the content pad id so radix keys and the model's scatter mask see the image
            for lo, hi in spans:
                full[lo:hi] = [item.pad_value] * (hi - lo)
            out.extend(full)
            item.offsets = [[base + lo, base + hi] for lo, hi in spans]
            item.validate()
            cursor = slot + len(target)
        out.extend(ids[cursor:])
        new_ids = torch.tensor(out, dtype=input_ids.dtype)

        positions, delta = self.positions(len(out), items) or (None, 0)
        return MMResult(new_ids, items, positions, delta)


def image_positions(
    length: int, blocks: list[tuple[int, int, int, int]]
) -> tuple[torch.Tensor, int]:
    """t/h/w positions: text advances all three rows, an image freezes t and spreads h/w over its grid, then the running position jumps by max(h, w).

    blocks are (offset, n_tokens, grid_h, grid_w) in prompt order; delta shifts decode positions (sequence index + delta)."""
    pos = torch.empty((3, length), dtype=torch.int32)
    st = 0
    cursor = 0
    for offset, n_tokens, h_m, w_m in blocks:
        run = offset - cursor
        if run:
            text = torch.arange(st, st + run, dtype=torch.int32)
            pos[:, cursor:offset] = text
            st += run
            cursor = offset
        idx = torch.arange(n_tokens, dtype=torch.int32)
        pos[0, cursor : cursor + n_tokens] = st
        pos[1, cursor : cursor + n_tokens] = st + idx // w_m
        pos[2, cursor : cursor + n_tokens] = st + idx % w_m
        st += max(h_m, w_m)
        cursor += n_tokens
    run = length - cursor
    if run:
        pos[:, cursor:] = torch.arange(st, st + run, dtype=torch.int32)
        st += run
    delta = int(pos.max().item()) + 1 - length
    return pos, delta


def content_hash(feature: torch.Tensor, extra: bytes = b"") -> int:
    """62-bit hash of the wire bytes plus family extras (e.g. the grid, so transposed images differ)."""
    buf = feature.contiguous()
    if buf.dtype == torch.bfloat16:
        buf = buf.view(torch.uint16)
    hasher = hashlib.sha256(memoryview(buf.numpy()))
    hasher.update(extra)
    return int.from_bytes(hasher.digest()[:8], "little") % (1 << 62)


def get_mm_processor(model_path: str, mm: MultimodalConfig | None = None) -> MMProcessor | None:
    """A processor for the checkpoint under the mm knobs, or None when no encoder is left to serve: the family registers none, the checkpoint ships none, or mm.disabled_encoders names them all; callers keep the instance, it holds the loaded image processor."""
    from freetoken.models.register import get_model_spec
    from freetoken.utils import cached_load_hf_config

    try:
        config = cached_load_hf_config(model_path)
        spec = get_model_spec((config.architectures or [None])[0])
    except Exception:  # noqa: BLE001 -- missing/foreign config or unknown architecture: no multimodal input
        return None
    mm = mm or MultimodalConfig()
    served = [e for e in spec.encoders if getattr(config, e.config_key, None) is not None and e.kind not in mm.disabled_encoders]
    if spec.mm_processor is None or not served:
        return None
    # families with a flat checkpoint config (deepseek_v4) carry vocab_size at the top level
    check_mm_pad_shift(getattr(config, "text_config", config).vocab_size)
    module, _, cls = spec.mm_processor.partition(":")
    return getattr(importlib.import_module(module), cls)(config, model_path, mm)


__all__ = [
    "MMProcessor",
    "MMResult",
    "PromptReplacement",
    "content_hash",
    "get_mm_processor",
    "image_positions",
]