FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Per-family multimodal processors and the architecture registry; MMProcessor.apply is the image half of tokenization."""
from __future__ import annotations
from abc import ABC, abstractmethod
import hashlib
import importlib
import io
import threading
from dataclasses import dataclass
from typing import Any
import torch
from freetoken.message import MMItem
from freetoken.mm.config import MultimodalConfig
from freetoken.mm import check_mm_pad_shift
@dataclass
class PromptReplacement:
"""The token sequence that replaces one placeholder; is_embed marks the positions that take image embeddings (None = all of them)."""
full: list[int]
is_embed: list[bool] | None = None
@classmethod
def select_token_id(cls, full: list[int], embed_token_id: int) -> PromptReplacement:
return cls(full, [t == embed_token_id for t in full])
def embed_spans(self) -> list[list[int]]:
"""Half-open [lo, hi) runs of embedding positions, relative to the start of full."""
mask = self.is_embed if self.is_embed is not None else [True] * len(self.full)
spans: list[list[int]] = []
lo: int | None = None
for i, flag in enumerate([*mask, False]):
if flag and lo is None:
lo = i
elif not flag and lo is not None:
spans.append([lo, i])
lo = None
return spans
@dataclass
class MMResult:
input_ids: torch.Tensor
mm_items: list[MMItem]
mrope_positions: torch.Tensor | None
mrope_delta: int
def _find_all(ids: list[int], target: list[int]) -> list[int]:
"""Start index of every non-overlapping occurrence of target."""
n = len(target)
found: list[int] = []
i = 0
while i <= len(ids) - n:
if ids[i : i + n] == target:
found.append(i)
i += n
else:
i += 1
return found
class MMProcessor(ABC):
"""How a checkpoint's images become items, tokens and positions; apply is the family-agnostic driver, subclasses fill the hooks.
Built from the checkpoint config alone and import-light: tokenizer workers call apply without the model layer, the engine calls dummy_items.
Image preprocessing must stay on the CPU with one fixed implementation: radix keys and the embedding cache are content hashes of pixel_values, and resize results differ across backends/devices.
"""
# the token sequence the chat template renders once per image
placeholder: list[int]
def __init__(self, model_path: str, mm: MultimodalConfig) -> None:
self.model_path = model_path
self.mm = mm
self._image_processor_instance: Any = None
self._image_processor_lock = threading.Lock()
def _image_processor(self) -> Any:
"""The checkpoint's image processor, loaded on first use (tokenizer workers share the instance across threads)."""
with self._image_processor_lock:
if self._image_processor_instance is None:
from freetoken.utils import load_image_processor
self._image_processor_instance = load_image_processor(self.model_path)
return self._image_processor_instance
def get_mm_processor_kwargs(self, mm: MultimodalConfig) -> dict[str, Any]:
"""Keyword arguments for the checkpoint processor call: the family's reading of the mm knobs, then mm.processor_kwargs on top; families with a token budget override this."""
return dict(mm.processor_kwargs)
@abstractmethod
def process(self, images: list[Any]) -> list[MMItem]:
"""One MMItem per image with feature and hash; offsets are assigned by apply."""
@abstractmethod
def prompt_replacement(self, item: MMItem) -> PromptReplacement:
"""The token sequence that stands in for this item's placeholder in the prompt."""
def positions(self, length: int, items: list[MMItem]) -> tuple[torch.Tensor, int] | None:
"""[3, length] t/h/w rope positions and the decode delta; None for 1-D rope families."""
return None
@abstractmethod
def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
"""The smallest encodable item of each modality the family serves, for warming the encoders."""
def apply(self, input_ids: torch.Tensor, images: list[bytes]) -> MMResult:
"""Swap each image placeholder for its replacement sequence and precompute rope positions."""
from PIL import Image
ids = input_ids.tolist()
target = self.placeholder
slots = _find_all(ids, target)
if len(slots) != len(images):
raise ValueError(
f"prompt renders {len(slots)} image placeholders but the request "
f"carries {len(images)} images"
)
pils = [Image.open(io.BytesIO(raw)).convert("RGB") for raw in images]
items = self.process(pils)
out: list[int] = []
cursor = 0
for slot, item in zip(slots, items):
repl = self.prompt_replacement(item)
out.extend(ids[cursor:slot])
base = len(out)
full = list(repl.full)
spans = repl.embed_spans()
# embedding slots carry the content pad id so radix keys and the model's scatter mask see the image
for lo, hi in spans:
full[lo:hi] = [item.pad_value] * (hi - lo)
out.extend(full)
item.offsets = [[base + lo, base + hi] for lo, hi in spans]
item.validate()
cursor = slot + len(target)
out.extend(ids[cursor:])
new_ids = torch.tensor(out, dtype=input_ids.dtype)
positions, delta = self.positions(len(out), items) or (None, 0)
return MMResult(new_ids, items, positions, delta)
def image_positions(
length: int, blocks: list[tuple[int, int, int, int]]
) -> tuple[torch.Tensor, int]:
"""t/h/w positions: text advances all three rows, an image freezes t and spreads h/w over its grid, then the running position jumps by max(h, w).
blocks are (offset, n_tokens, grid_h, grid_w) in prompt order; delta shifts decode positions (sequence index + delta)."""
pos = torch.empty((3, length), dtype=torch.int32)
st = 0
cursor = 0
for offset, n_tokens, h_m, w_m in blocks:
run = offset - cursor
if run:
text = torch.arange(st, st + run, dtype=torch.int32)
pos[:, cursor:offset] = text
st += run
cursor = offset
idx = torch.arange(n_tokens, dtype=torch.int32)
pos[0, cursor : cursor + n_tokens] = st
pos[1, cursor : cursor + n_tokens] = st + idx // w_m
pos[2, cursor : cursor + n_tokens] = st + idx % w_m
st += max(h_m, w_m)
cursor += n_tokens
run = length - cursor
if run:
pos[:, cursor:] = torch.arange(st, st + run, dtype=torch.int32)
st += run
delta = int(pos.max().item()) + 1 - length
return pos, delta
def content_hash(feature: torch.Tensor, extra: bytes = b"") -> int:
"""62-bit hash of the wire bytes plus family extras (e.g. the grid, so transposed images differ)."""
buf = feature.contiguous()
if buf.dtype == torch.bfloat16:
buf = buf.view(torch.uint16)
hasher = hashlib.sha256(memoryview(buf.numpy()))
hasher.update(extra)
return int.from_bytes(hasher.digest()[:8], "little") % (1 << 62)
def get_mm_processor(model_path: str, mm: MultimodalConfig | None = None) -> MMProcessor | None:
"""A processor for the checkpoint under the mm knobs, or None when no encoder is left to serve: the family registers none, the checkpoint ships none, or mm.disabled_encoders names them all; callers keep the instance, it holds the loaded image processor."""
from freetoken.models.register import get_model_spec
from freetoken.utils import cached_load_hf_config
try:
config = cached_load_hf_config(model_path)
spec = get_model_spec((config.architectures or [None])[0])
except Exception: # noqa: BLE001 -- missing/foreign config or unknown architecture: no multimodal input
return None
mm = mm or MultimodalConfig()
served = [e for e in spec.encoders if getattr(config, e.config_key, None) is not None and e.kind not in mm.disabled_encoders]
if spec.mm_processor is None or not served:
return None
# families with a flat checkpoint config (deepseek_v4) carry vocab_size at the top level
check_mm_pad_shift(getattr(config, "text_config", config).vocab_size)
module, _, cls = spec.mm_processor.partition(":")
return getattr(importlib.import_module(module), cls)(config, model_path, mm)
__all__ = [
"MMProcessor",
"MMResult",
"PromptReplacement",
"content_hash",
"get_mm_processor",
"image_positions",
]