FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Image tokenization against real Qwen VL, Gemma-4, GLM-5.3, Muse-Glimmer, MiniMax-M3 and DeepSeek-V4-Vision checkpoints (skipped when absent)."""
from __future__ import annotations
import io
import os
import pytest
import torch
from freetoken.mm import MM_PAD_SHIFT_VALUE, mm_pad_value
MODELS = [
p
for p in (os.environ.get("FREETOKEN_QWEN36_MODEL", ""), os.environ.get("FREETOKEN_QWEN3VL_MODEL", ""))
if os.path.exists(os.path.join(p, "config.json"))
]
GEMMA = os.environ.get("FREETOKEN_GEMMA4_MODEL", "")
GLM = os.environ.get("FREETOKEN_GLM53_MODEL", "")
MUSE = os.environ.get("FREETOKEN_MUSE_MODEL", "")
MINIMAX = os.environ.get("FREETOKEN_MINIMAX_M3_MODEL", "")
DSV4 = os.environ.get("FREETOKEN_DSV4_VL_MODEL", "")
pytestmark = [
pytest.mark.needs_weights,
pytest.mark.skipif(
not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM, MUSE, MINIMAX, DSV4)),
reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL / FREETOKEN_MUSE_MODEL / FREETOKEN_MINIMAX_M3_MODEL / FREETOKEN_DSV4_VL_MODEL checkpoint",
),
]
def _png(w, h):
from PIL import Image
buf = io.BytesIO()
Image.new("RGB", (w, h), (120, 30, 200)).save(buf, format="PNG")
return buf.getvalue()
@pytest.fixture(scope="module", params=MODELS, ids=os.path.basename)
def manager(request):
from freetoken.mm.processor import get_mm_processor
from freetoken.tokenizer.tokenize import TokenizeManager
from freetoken.utils.hf import load_tokenizer
return TokenizeManager(load_tokenizer(request.param), get_mm_processor(request.param))
def _msg(images, n_parts=1):
from freetoken.core import SamplingParams
from freetoken.message import TokenizeMsg
content = [{"type": "text", "text": "描述"}] + [{"type": "image"}] * n_parts
return TokenizeMsg(
uid=1, text=[{"role": "user", "content": content}],
sampling_params=SamplingParams(), images=images,
)
def test_expansion_mrope_and_radix_ids(manager):
r = manager.tokenize([_msg([_png(512, 512)])])[0]
item = r.mm_items[0]
L = r.input_ids.numel()
assert item.num_tokens == 256 and item.grid_thw == [1, 32, 32]
assert r.mrope_positions.shape == (3, L)
assert int(r.mrope_positions.max()) + 1 - L == r.mrope_delta
# the image span carries one content pad id above the vocab; text is untouched
(start, end), = item.offsets
span = r.input_ids[start:end]
assert span.unique().numel() == 1
assert span[0].item() == item.pad_value == mm_pad_value(item.hash) >= MM_PAD_SHIFT_VALUE
assert bool((r.input_ids[:start] < MM_PAD_SHIFT_VALUE).all())
def test_same_image_same_hash_and_pad(manager):
a = manager.tokenize([_msg([_png(512, 512)])])[0]
b = manager.tokenize([_msg([_png(512, 512)])])[0]
assert a.mm_items[0].hash == b.mm_items[0].hash
assert torch.equal(a.input_ids, b.input_ids)
c = manager.tokenize([_msg([_png(256, 256)])])[0]
assert c.mm_items[0].hash != a.mm_items[0].hash
def test_image_max_tokens_clamps(manager):
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import get_mm_processor
from freetoken.tokenizer.tokenize import TokenizeManager
path = manager.tokenizer.name_or_path
budgeted = TokenizeManager(manager.tokenizer, get_mm_processor(path, MultimodalConfig(image_max_tokens=2048)))
assert manager.tokenize([_msg([_png(3840, 2160)])])[0].mm_items[0].num_tokens > 2048
assert budgeted.tokenize([_msg([_png(3840, 2160)])])[0].mm_items[0].num_tokens <= 2048
def test_count_prompt_tokens_counts_the_expanded_image(manager):
import asyncio
import base64
from types import SimpleNamespace
from freetoken.server.generation import count_prompt_tokens
png = _png(512, 512)
messages = [{"role": "user", "content": [
{"type": "image", "freetoken_ref": {"kind": "b64", "data": base64.b64encode(png).decode()}},
{"type": "text", "text": "描述"},
]}]
state = SimpleNamespace(
frontend_tokenizer=lambda: manager,
config=SimpleNamespace(served_modalities=frozenset({"image"}), mm=SimpleNamespace(text_model_only=False, disabled_encoders=frozenset())),
)
counted = asyncio.run(count_prompt_tokens(messages, None, {}, state))
expanded = manager.tokenize([_msg([png])])[0].input_ids.numel()
assert counted == expanded and counted > 256
@pytest.mark.skipif(not os.path.exists(os.path.join(GEMMA, "config.json")), reason="FREETOKEN_GEMMA4_MODEL not set")
def test_gemma_wraps_the_pad_span_in_boi_eoi():
from freetoken.mm.processor import get_mm_processor
from freetoken.tokenizer.tokenize import TokenizeManager
from freetoken.utils import cached_load_hf_config
from freetoken.utils.hf import load_tokenizer
hf = cached_load_hf_config(GEMMA)
r = TokenizeManager(load_tokenizer(GEMMA), get_mm_processor(GEMMA)).tokenize([_msg([_png(640, 400)])])[0]
item = r.mm_items[0]
((start, end),) = item.offsets
ids = r.input_ids
assert ids[start - 1].item() == hf.boi_token_id and ids[end].item() == hf.eoi_token_id
assert end - start == item.num_soft_tokens and r.mrope_positions is None
assert bool((ids[start:end] == item.pad_value).all())
@pytest.mark.skipif(not os.path.exists(os.path.join(GEMMA, "config.json")), reason="FREETOKEN_GEMMA4_MODEL not set")
def test_gemma_image_max_tokens_picks_a_smaller_soft_token_budget():
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import get_mm_processor
from freetoken.tokenizer.tokenize import TokenizeManager
from freetoken.utils.hf import load_tokenizer
tokenizer = load_tokenizer(GEMMA)
default = TokenizeManager(tokenizer, get_mm_processor(GEMMA)).tokenize([_msg([_png(640, 400)])])[0]
budgeted = TokenizeManager(tokenizer, get_mm_processor(GEMMA, MultimodalConfig(image_max_tokens=100))).tokenize([_msg([_png(640, 400)])])[0]
# the processor scales the image to the budget as far as its aspect ratio allows: 10x6 pooled cells for 640x400
assert budgeted.mm_items[0].num_soft_tokens == 60 < default.mm_items[0].num_soft_tokens
@pytest.mark.skipif(not os.path.exists(os.path.join(GLM, "config.json")), reason="FREETOKEN_GLM53_MODEL not set")
def test_glm_expands_the_image_token_between_the_template_wrappers():
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import get_mm_processor
from freetoken.tokenizer.tokenize import TokenizeManager
from freetoken.utils import cached_load_hf_config
from freetoken.utils.hf import load_tokenizer
hf = cached_load_hf_config(GLM)
tokenizer = load_tokenizer(GLM)
r = TokenizeManager(tokenizer, get_mm_processor(GLM)).tokenize([_msg([_png(640, 400)])])[0]
item = r.mm_items[0]
((start, end),) = item.offsets
ids = r.input_ids
# 640x400 is zero-padded to a 644x420 canvas: 46x30 patches, one token per 2x2 of them
assert item.grid_thw == [1, 30, 46] and end - start == item.num_tokens == 345
assert ids[start - 1].item() == hf.image_start_token_id and ids[end].item() == hf.image_end_token_id
assert bool((ids[start:end] == item.pad_value).all()) and r.mrope_positions is None
budgeted = TokenizeManager(tokenizer, get_mm_processor(GLM, MultimodalConfig(image_max_tokens=100))).tokenize([_msg([_png(640, 400)])])[0]
assert budgeted.mm_items[0].num_tokens <= 100
@pytest.mark.skipif(not os.path.exists(os.path.join(MUSE, "config.json")), reason="FREETOKEN_MUSE_MODEL not set")
def test_muse_wraps_the_pad_span_in_image_start_end_and_clamps_to_a_token_maximum():
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import get_mm_processor
from freetoken.tokenizer.tokenize import TokenizeManager
from freetoken.utils.hf import load_tokenizer
tokenizer = load_tokenizer(MUSE)
r = TokenizeManager(tokenizer, get_mm_processor(MUSE)).tokenize([_msg([_png(640, 400)])])[0]
item = r.mm_items[0]
((start, end),) = item.offsets
ids = r.input_ids
assert ids[start - 1].item() == tokenizer.convert_tokens_to_ids("<|image_start|>")
assert ids[end].item() == tokenizer.convert_tokens_to_ids("<|image_end|>")
# 640x400 resizes to a 44x28 patch grid; the single <|patch|> the template renders is gone
assert item.grid_thw == [1, 28, 44] and end - start == item.num_tokens == 308 and r.mrope_positions is None
assert bool((ids[start:end] == item.pad_value).all()) and not bool((ids == 200092).any())
budgeted = TokenizeManager(tokenizer, get_mm_processor(MUSE, MultimodalConfig(image_max_tokens=1024)))
assert budgeted.tokenize([_msg([_png(3840, 2160)])])[0].mm_items[0].num_tokens <= 1024 < item.num_tokens * 4
# an explicit processor kwarg replaces the checkpoint's 4096 cap: 640x400 under 256 tokens is a 24x40 grid
override = TokenizeManager(tokenizer, get_mm_processor(MUSE, MultimodalConfig(processor_kwargs={"max_image_tokens": 256})))
small = override.tokenize([_msg([_png(640, 400)])])[0].mm_items[0]
assert small.grid_thw == [1, 24, 40] and small.num_tokens == 240
@pytest.mark.skipif(not os.path.exists(os.path.join(MINIMAX, "config.json")), reason="FREETOKEN_MINIMAX_M3_MODEL not set")
def test_minimax_wraps_the_pad_span_in_start_end_tokens():
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import get_mm_processor
from freetoken.mm.processors.minimax_m3 import IMAGE_END_ID, IMAGE_START_ID
from freetoken.tokenizer.tokenize import TokenizeManager
from freetoken.utils.hf import load_tokenizer
tokenizer = load_tokenizer(MINIMAX)
# the processor hardcodes the wrapper ids the checkpoint config does not name
assert tokenizer.convert_tokens_to_ids("]<]start of image[>[") == IMAGE_START_ID
assert tokenizer.convert_tokens_to_ids("]<]end of image[>[") == IMAGE_END_ID
r = TokenizeManager(tokenizer, get_mm_processor(MINIMAX)).tokenize([_msg([_png(640, 400)])])[0]
item = r.mm_items[0]
((start, end),) = item.offsets
ids = r.input_ids
_, h, w = item.grid_thw
assert ids[start - 1].item() == IMAGE_START_ID and ids[end].item() == IMAGE_END_ID
assert end - start == h * w // 4 == 322 and r.mrope_positions is None
assert bool((ids[start:end] == item.pad_value).all())
budgeted = TokenizeManager(tokenizer, get_mm_processor(MINIMAX, MultimodalConfig(image_max_tokens=64))).tokenize([_msg([_png(640, 400)])])[0]
assert budgeted.mm_items[0].num_tokens <= 64
@pytest.mark.skipif(
not os.path.exists(os.path.join(DSV4, "config.json")),
reason="no FREETOKEN_DSV4_VL_MODEL checkpoint",
)
def test_dsv4_expands_the_placeholder_into_the_reference_block():
"""The block carries one content pad id, and its length follows the reference's grid."""
from freetoken.mm.processor import get_mm_processor
from freetoken.tokenizer.tokenize import TokenizeManager
from freetoken.utils.hf import load_tokenizer
manager = TokenizeManager(load_tokenizer(DSV4), get_mm_processor(DSV4))
# 640x480 through the reference: a 12x16 merge grid, so 206 tokens of rows and markers
# plus the 3 lead pads that align the block at this offset
out = manager.tokenize([_msg([_png(640, 480)])])[0]
ids = out.input_ids.tolist()
(item,) = out.mm_items
assert (item.n_llm_h, item.n_llm_w) == (12, 16)
assert item.num_tokens == 209
lo, hi = item.offsets[0]
assert (lo, hi) == (item.start, item.start + 209)
assert set(ids[lo:hi]) == {item.pad_value}
assert item.pad_value >= MM_PAD_SHIFT_VALUE and item.pad_value == mm_pad_value(item.hash)
assert 129264 not in ids # the raw <|deepseek_image|> placeholder is gone
assert item.feature.shape == (35 * 46, 3, 14, 14) and item.feature.dtype == torch.bfloat16
@pytest.mark.skipif(
not os.path.exists(os.path.join(DSV4, "config.json")),
reason="no FREETOKEN_DSV4_VL_MODEL checkpoint",
)
def test_dsv4_two_images_take_two_blocks_and_the_second_is_shorter():
"""The second image sits at a different offset, so its block is a different length."""
from freetoken.mm.processor import get_mm_processor
from freetoken.tokenizer.tokenize import TokenizeManager
from freetoken.utils.hf import load_tokenizer
manager = TokenizeManager(load_tokenizer(DSV4), get_mm_processor(DSV4))
raw = _png(320, 240)
out = manager.tokenize([_msg([raw, raw], n_parts=2)])[0]
first, second = out.mm_items
assert (first.start, second.start) == (first.offsets[0][0], second.offsets[0][0])
# the lead pads take up to three tokens of slack, so the blocks nearly match
assert abs(second.num_tokens - first.num_tokens) <= 3
# two images share a pad id only when their blocks are laid out identically, which is
# exactly when their offsets are congruent: otherwise one must not serve the other's key
assert (first.pad_value == second.pad_value) == ((second.start - first.start) % 4 == 0)