FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Gemma 4 image processor: aspect-ratio-preserving 16x16 patches, pooled 3x3 into soft tokens, 1-D rope."""
from __future__ import annotations
import struct
from typing import Any
import torch
from freetoken.message import MMItem
from freetoken.mm import mm_pad_value
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import MMProcessor, PromptReplacement, content_hash
# the only soft-token budgets the image processor accepts
_SOFT_TOKEN_BUDGETS = (70, 140, 280, 560, 1120)
def _soft_token_budget(max_tokens: int) -> int:
"""Largest accepted budget within max_tokens."""
return max(b for b in _SOFT_TOKEN_BUDGETS if b <= max_tokens)
class Gemma4MMProcessor(MMProcessor):
def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:
super().__init__(model_path, mm)
if mm.image_max_tokens is not None and mm.image_max_tokens < _SOFT_TOKEN_BUDGETS[0]:
raise ValueError(f"--image-max-tokens {mm.image_max_tokens} is below the smallest soft-token budget this image processor accepts ({_SOFT_TOKEN_BUDGETS[0]})")
vc = hf_config.vision_config
self.image_token_id = hf_config.image_token_id
self.boi_token_id = hf_config.boi_token_id
self.eoi_token_id = hf_config.eoi_token_id
self.placeholder = [self.image_token_id]
self.pooling_kernel_size = vc.pooling_kernel_size
self.patch_dim = 3 * vc.patch_size**2
def get_mm_processor_kwargs(self, mm: MultimodalConfig) -> dict[str, Any]:
kwargs: dict[str, Any] = {"return_tensors": "pt"}
if mm.image_max_tokens is not None:
# the processor scales every image to its soft-token budget as far as the aspect ratio allows, so a lower bound has no effect
kwargs["max_soft_tokens"] = _soft_token_budget(mm.image_max_tokens)
return {**kwargs, **mm.processor_kwargs}
def process(self, images: list[Any]) -> list[MMItem]:
processor = self._image_processor()
kwargs = self.get_mm_processor_kwargs(self.mm)
items: list[MMItem] = []
for pil in images:
out = processor(images=pil, **kwargs)
positions = out["image_position_ids"][0]
valid = (positions != -1).all(dim=-1)
positions = positions[valid].to(torch.int32)
feature = out["pixel_values"][0][valid]
n_soft = int(out["num_soft_tokens_per_image"][0])
item_hash = content_hash(feature, positions.numpy().tobytes() + struct.pack("<i", n_soft))
items.append(
MMItem(
modality="image",
hash=item_hash,
pad_value=mm_pad_value(item_hash),
offsets=[],
feature=feature,
model_specific_data={"position_ids": positions, "num_soft_tokens": n_soft},
)
)
return items
def prompt_replacement(self, item: MMItem) -> PromptReplacement:
full = [self.boi_token_id] + [self.image_token_id] * item.num_soft_tokens + [self.eoi_token_id]
return PromptReplacement.select_token_id(full, self.image_token_id)
def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
k = self.pooling_kernel_size
ys, xs = torch.meshgrid(torch.arange(k), torch.arange(k), indexing="ij")
positions = torch.stack([xs.reshape(-1), ys.reshape(-1)], dim=-1).to(torch.int32)
return [MMItem(
modality="image",
hash=0,
pad_value=0,
offsets=[[0, 1]],
feature=torch.zeros(k * k, self.patch_dim, dtype=dtype, device=device),
model_specific_data={"position_ids": positions.to(device), "num_soft_tokens": 1},
)]
class Gemma4UnifiedMMProcessor(Gemma4MMProcessor):
"""The gemma4_unified release: one 48x48 super-patch per soft token and no pooling; the processor call and its budgets are the same."""
def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:
super().__init__(hf_config, model_path, mm)
self.patch_dim = 3 * hf_config.vision_config.model_patch_size**2
def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
return [MMItem(
modality="image",
hash=0,
pad_value=0,
offsets=[[0, 1]],
feature=torch.zeros(1, self.patch_dim, dtype=dtype, device=device),
model_specific_data={"position_ids": torch.zeros(1, 2, dtype=torch.int32, device=device), "num_soft_tokens": 1},
)]
__all__ = ["Gemma4MMProcessor", "Gemma4UnifiedMMProcessor"]