FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Multimodal input handling: pad-id scheme, per-family processors, media fetch, embedding cache."""
from __future__ import annotations
import torch
# image spans become content pseudo ids above every real token id: the radix cache keys them by content and the model finds image rows by id >= MM_PAD_SHIFT_VALUE
# they are clamped back into the vocab before the embedding lookup; those rows are overwritten anyway
MM_PAD_SHIFT_VALUE = 1_000_000
def restore_placeholder(input_ids: torch.Tensor, token_id: int) -> torch.Tensor:
"""input_ids with every content pad id put back to the family's placeholder token."""
return input_ids.masked_fill(input_ids >= MM_PAD_SHIFT_VALUE, token_id)
def mm_pad_value(item_hash: int) -> int:
return MM_PAD_SHIFT_VALUE + item_hash % (1 << 30)
def check_mm_pad_shift(vocab_size: int) -> None:
if vocab_size > MM_PAD_SHIFT_VALUE:
raise ValueError(
f"vocab_size ({vocab_size}) exceeds MM_PAD_SHIFT_VALUE ({MM_PAD_SHIFT_VALUE}); "
"image pseudo-ids would alias real token ids"
)
__all__ = [
"MM_PAD_SHIFT_VALUE",
"check_mm_pad_shift",
"mm_pad_value",
]