Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

1.1 KB
Raw Blame
"""Multimodal input handling: pad-id scheme, per-family processors, media fetch, embedding cache."""

from __future__ import annotations

import torch

# image spans become content pseudo ids above every real token id: the radix cache keys them by content and the model finds image rows by id >= MM_PAD_SHIFT_VALUE
# they are clamped back into the vocab before the embedding lookup; those rows are overwritten anyway
MM_PAD_SHIFT_VALUE = 1_000_000


def restore_placeholder(input_ids: torch.Tensor, token_id: int) -> torch.Tensor:
    """input_ids with every content pad id put back to the family's placeholder token."""
    return input_ids.masked_fill(input_ids >= MM_PAD_SHIFT_VALUE, token_id)


def mm_pad_value(item_hash: int) -> int:
    return MM_PAD_SHIFT_VALUE + item_hash % (1 << 30)


def check_mm_pad_shift(vocab_size: int) -> None:
    if vocab_size > MM_PAD_SHIFT_VALUE:
        raise ValueError(
            f"vocab_size ({vocab_size}) exceeds MM_PAD_SHIFT_VALUE ({MM_PAD_SHIFT_VALUE}); "
            "image pseudo-ids would alias real token ids"
        )


__all__ = [
    "MM_PAD_SHIFT_VALUE",
    "check_mm_pad_shift",
    "mm_pad_value",
]