Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

14.2 KB
Raw Blame
"""DeepSeek-V4 image preprocessing, block layout and the tiny vision tower (no checkpoint).

The block layout is what makes an image's token count depend on where it lands in the
prompt, so it is pinned here against hand-written expectations; the checkpoint-gated test
at the bottom checks the same functions against the reference ``inference/`` code.
"""

from __future__ import annotations

import io
import os

import numpy as np
import pytest
import torch
from PIL import Image

from freetoken.models.deepseek_v4.config import DSV4VisionConfig, parse_config, parse_vision_config
from freetoken.models.deepseek_v4.image_processor import load_image
from freetoken.models.deepseek_v4.vision import (
    COMPRESS_PAD_TO,
    DSV4Aligner,
    DSV4VisionTower,
    IMAGE,
    IMAGE_END,
    IMAGE_NEW_LINE,
    IMAGE_PAD,
    IMAGE_START,
    assemble_block,
    build_image_block,
)

checkpoint_path = os.environ.get("FREETOKEN_DSV4_VISION_CKPT")
needs_checkpoint = pytest.mark.skipif(
    not checkpoint_path, reason="needs FREETOKEN_DSV4_VISION_CKPT pointing at the vision checkpoint"
)

_HF_CONFIG = {
    "vision_n_layers": 32,
    "vision_dim": 1024,
    "vision_n_heads": 16,
    "vision_inter_dim": 2816,
    "vision_patch_size": 14,
    "vision_rope_theta": 10000.0,
    "vision_downsample_ratio": 3,
    "vision_max_n_token": 384,
    "vision_min_pixels": 147456,
    "vision_max_wh_ratio": 8,
    "hidden_size": 4096,
}


def _hf_config(**over):
    from types import SimpleNamespace

    return SimpleNamespace(**{**_HF_CONFIG, **over})


def _vc(**over) -> DSV4VisionConfig:
    """A tower small enough to run: 2 patch-2 blocks over a 2x2 merge grid."""
    base = dict(
        vision_n_layers=2,
        vision_dim=16,
        vision_n_heads=2,
        vision_inter_dim=32,
        vision_patch_size=4,
        vision_rope_theta=10000.0,
        vision_downsample_ratio=2,
        vision_max_n_token=384,
        vision_min_pixels=147456,
        vision_max_wh_ratio=8,
        text_dim=64,
    )
    return DSV4VisionConfig(**{**base, **over})


def _png(width, height, seed=0):
    rng = np.random.default_rng(seed)
    array = (rng.random((height, width, 3)) * 255).astype(np.uint8)
    buf = io.BytesIO()
    Image.fromarray(array, "RGB").save(buf, format="PNG")
    return buf.getvalue()


def test_a_vision_checkpoint_config_becomes_the_tower_dims_and_a_text_one_does_not():
    vc = parse_vision_config(_hf_config())
    assert (vc.vision_n_layers, vc.vision_dim, vc.text_dim) == (32, 1024, 4096)
    # the engine nulls the section on the copy it hands the parser when it serves text-only
    assert parse_vision_config(_hf_config(vision_n_layers=None)) is None
    assert parse_vision_config(_hf_config(vision_n_layers=0)) is None


def test_block_layout_interleaves_rows_and_pads_to_the_compression_stride():
    types, perm = build_image_block(2, 2, 0)
    # 3 lead pads put the first row token on a COMPRESS_PAD_TO (4) boundary, then START,
    # the N-layout rows (column pairs: 2 rows of 2 IMAGE tokens plus their row markers),
    # the trailing parity pad, END.
    assert types.tolist() == (
        [IMAGE_PAD] * 3
        + [IMAGE_START]
        + [IMAGE, IMAGE, IMAGE, IMAGE, IMAGE_NEW_LINE, IMAGE_NEW_LINE]
        + [IMAGE_PAD] * 2
        + [IMAGE_END]
    )
    # row-major aligner outputs 0..3 land in the IMAGE slots in N-layout order
    assert perm.tolist() == [0, 2, 1, 3]


def test_the_block_shrinks_by_one_token_per_lead_pad_the_offset_absorbs():
    # the lead pads are what aligns the block, so the same image at a later offset is a
    # SHORTER span: the count cannot be a property of the image alone
    lengths = [len(build_image_block(2, 2, start)[0]) for start in range(4)]
    assert lengths == [13, 12, 11, 10]
    # an odd row count adds a marker row, which is not pad bytes
    assert [len(build_image_block(3, 2, start)[0]) for start in range(4)] == [17, 16, 15, 14]


def test_assemble_block_puts_the_sentinels_and_the_aligner_rows_where_the_types_say():
    types, perm = build_image_block(2, 2, 0)
    sentinels = torch.stack([torch.full((4,), float(i)) for i in range(5)])
    aligned = torch.arange(4 * 4, dtype=torch.float32).view(4, 4) * 10
    block = assemble_block(aligned, types, perm, sentinels)
    assert block.shape == (len(types), 4)
    # sentinels index the type enum, so START/PAD/END rows carry 0/1/4 and the IMAGE rows
    # carry the aligner output perm says they take
    assert block[0].tolist() == [1.0] * 4  # lead pad
    assert block[3].tolist() == [0.0] * 4  # IMAGE_START
    assert block[-1].tolist() == [4.0] * 4  # IMAGE_END
    image_rows = block[types == IMAGE]
    assert image_rows.tolist() == aligned[perm].tolist()


def test_preprocessing_resizes_to_the_merge_grid_and_normalizes_like_the_reference():
    # min_pixels 0 keeps the input size: the reference upscales below it
    vc = _vc(vision_min_pixels=0)
    patches, n_vit_h, n_vit_w, n_llm_h, n_llm_w = load_image(
        Image.open(io.BytesIO(_png(80, 64))), vc
    )
    assert (n_vit_h, n_vit_w, n_llm_h, n_llm_w) == (16, 20, 8, 10)
    assert patches.shape == (n_vit_h * n_vit_w, 3, 4, 4)
    assert patches.dtype == torch.bfloat16
    assert float(patches.min()) >= -1.0 and float(patches.max()) <= 1.0
    # identical bytes, identical patches: the content hash and the radix key ride on these
    again, *_ = load_image(Image.open(io.BytesIO(_png(80, 64))), vc)
    assert torch.equal(patches, again)


def test_the_token_budget_caps_a_tall_image():
    vc = _vc(vision_max_n_token=64, vision_min_pixels=0)
    _patches, _n_vit_h, _n_vit_w, n_llm_h, n_llm_w = load_image(
        Image.open(io.BytesIO(_png(64, 1024))), vc
    )
    types, _perm = build_image_block(n_llm_h, n_llm_w, 0)
    assert len(types) <= vc.vision_max_n_token


def test_tiny_tower_runs_and_its_aligner_merges_every_merge_grid():
    vc = _vc()
    tower = DSV4VisionTower(vc)
    aligner = DSV4Aligner(vc)
    patches = torch.zeros(20 * 24, 3, 4, 4, dtype=torch.bfloat16)
    out = tower.forward(patches, 20, 24)
    assert out.shape == (20 * 24, vc.vision_dim)
    assert aligner.forward(out, 20, 24).shape == (10 * 12, vc.text_dim)


def _reference_modules(path):
    """Import the checkpoint's own ``inference/image_processor.py`` (PIL + torch only)."""
    import importlib.util

    module_path = os.path.join(path, "inference", "image_processor.py")
    if not os.path.exists(module_path):
        pytest.skip(f"{module_path} not found")
    spec = importlib.util.spec_from_file_location("_dsv4_reference_image_processor", module_path)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


@needs_checkpoint
def test_preprocessing_and_block_layout_match_the_reference():
    from types import SimpleNamespace

    reference = _reference_modules(checkpoint_path)
    vc = parse_vision_config(_hf_config())
    args = SimpleNamespace(
        vision_patch_size=vc.vision_patch_size,
        vision_downsample_ratio=vc.vision_downsample_ratio,
        vision_max_n_token=vc.vision_max_n_token,
        vision_min_pixels=vc.vision_min_pixels,
        vision_max_wh_ratio=vc.vision_max_wh_ratio,
    )
    for width, height in ((640, 480), (100, 100), (3000, 400), (1600, 1200)):
        raw = _png(width, height)
        mine = load_image(Image.open(io.BytesIO(raw)), vc)
        theirs = reference.load_image({"data": raw}, args)
        assert torch.equal(mine[0], theirs[0])
        assert mine[1:] == theirs[1:]
        for start in (0, 1, 3, 7):
            types, perm = build_image_block(mine[3], mine[4], start)
            ref_types, ref_perm = reference.build_image_block(theirs[3], theirs[4], start)
            assert torch.equal(types, ref_types) and torch.equal(perm, ref_perm)


_WINDOW, _MAX_IMG = 128, 384


def _ref_get_image_visible(input_ids: torch.Tensor, vocab_size: int, max_image_tokens: int):
    """Verbatim port of the checkpoint's ``inference/model.py`` pair (not vendored in this
    repo): ``get_image_visible`` derives spans from the virtual ids ``vocab + type``, so the
    lead pads before IMAGE_START stay outside; only the START/END comparisons matter, our
    type enum values index the same way."""
    seqlen = input_ids.size(1)
    idx = torch.arange(seqlen, dtype=torch.int32).unsqueeze(0)
    is_start = input_ids == vocab_size + IMAGE_START
    is_end = input_ids == vocab_size + IMAGE_END
    valid = (is_start.cumsum(1) > is_end.cumsum(1)) | is_end
    starts = torch.where(is_start, idx, 0).cummax(1)[0]
    left = (idx - starts) * valid
    ends = torch.where(is_end, idx, seqlen).flip(1).cummin(1)[0].flip(1)
    right = (ends - idx) * valid
    return left.clamp(max=max_image_tokens - 1), right.clamp(max=max_image_tokens)


def _ref_get_window_topk_idxs_visible(window_size, seqlen, left, right, max_image_tokens):
    width = min(seqlen, window_size + max_image_tokens)
    idx = torch.arange(seqlen).unsqueeze(0)
    left_add = (left - (window_size - 1)).clamp(min=0)
    starts = (idx - (window_size - 1) - left_add).clamp(min=0)
    matrix = starts.unsqueeze(-1) + torch.arange(width)
    return torch.where(matrix > (idx + right).unsqueeze(-1), -1, matrix).int().contiguous()


def _reference_candidates(prefix: int, n_llm_h: int, n_llm_w: int, vocab: int = 100_000):
    """The reference pair's window candidates for a text prefix followed by one image block."""
    types, _perm = build_image_block(n_llm_h, n_llm_w, prefix)
    ids = torch.full((1, prefix + len(types)), vocab + 50, dtype=torch.int64)
    ids[0, prefix:] = vocab + types
    n = ids.size(1)
    left, right = _ref_get_image_visible(ids, vocab, _MAX_IMG)
    block = (prefix, prefix + len(types))
    return block, types, _ref_get_window_topk_idxs_visible(_WINDOW, n, left, right, _MAX_IMG)[0]


def test_model_spans_match_the_reference_visibility_pair():
    from types import SimpleNamespace

    from freetoken.models.deepseek_v4.attention import Attention
    from freetoken.models.deepseek_v4.model import _image_spans_by_ti

    for prefix, h, w in ((0, 3, 2), (1, 3, 2), (2, 3, 2), (3, 3, 2), (5, 4, 3), (201, 20, 16), (402, 2, 2)):
        (lo, hi), types, ref = _reference_candidates(prefix, h, w)
        req = SimpleNamespace(table_idx=7, mm_items=[SimpleNamespace(offsets=[[lo, hi]])])
        spans = _image_spans_by_ti([req])
        assert spans == {7: [(lo + types.tolist().index(IMAGE_START), hi)]}
        cols = Attention.visible_window_cols(
            spans[7], 0, hi, _WINDOW, _MAX_IMG, torch.device("cpu"),
            addr_lo=0, width=min(hi, _WINDOW + _MAX_IMG),
        )
        assert torch.equal(cols, ref)
    text = SimpleNamespace(table_idx=0, mm_items=None)
    assert _image_spans_by_ti([text]) is None


def test_a_span_started_at_the_first_lead_pad_breaks_reference_parity():
    from freetoken.models.deepseek_v4.attention import Attention

    # the maintainer's case: a 201-token prefix puts 2 lead pads in front of a 20x16 grid,
    # and a span covering the whole block diverges from the reference on 218 rows
    (lo, hi), _types, ref = _reference_candidates(201, 20, 16)
    assert lo + COMPRESS_PAD_TO - 1 - lo % COMPRESS_PAD_TO - lo == 2  # the two lead pads
    cols = Attention.visible_window_cols(
        [(lo, hi)], 0, hi, _WINDOW, _MAX_IMG, torch.device("cpu"),
        addr_lo=0, width=min(hi, _WINDOW + _MAX_IMG),
    )
    assert int((cols != ref).any(dim=1).sum()) == 218


def test_a_vision_checkpoint_keeps_image_blocks_whole_across_chunk_boundaries(tmp_path):
    """Image spans attend bidirectionally, so a vision checkpoint sets
    bidirectional_mm_blocks on its attention group and the scheduler ends a chunk before a
    block instead of splitting it; text-only defaults to False and keeps plain chunking."""
    from types import SimpleNamespace

    from freetoken.core import Context, SamplingParams, get_global_ctx, set_global_ctx
    from freetoken.message import MMItem
    from freetoken.scheduler.cache import CacheManager
    from freetoken.scheduler.decode import DecodeManager
    from freetoken.scheduler.mm import cut_image_spans
    from freetoken.scheduler.prefill import PrefillManager
    from freetoken.scheduler.table import TableManager
    from freetoken.scheduler.utils import PendingReq

    # DeepseekV4Args defaults fill an empty inference/config.json; the hf namespace carries the tower
    (tmp_path / "inference").mkdir()
    (tmp_path / "inference" / "config.json").write_text("{}")
    vision = parse_config(SimpleNamespace(_name_or_path=str(tmp_path), **_HF_CONFIG))
    text = parse_config(SimpleNamespace(_name_or_path=str(tmp_path), **{**_HF_CONFIG, "vision_n_layers": None}))
    assert vision.attention_groups[0].bidirectional_mm_blocks is True
    assert text.attention_groups[0].bidirectional_mm_blocks is False

    prefix = 10
    types, _perm = build_image_block(2, 2, prefix)
    lo, hi = prefix, prefix + len(types)
    image = MMItem(modality="image", hash=1, pad_value=1, offsets=[[lo, hi]], feature=torch.zeros(1))
    assert lo < 15 < hi  # a plain 15-token chunk budget ends inside the block

    def chunk_lens(keep_images_whole):
        try:
            get_global_ctx()
        except AssertionError:
            set_global_ctx(Context(page_size=1))
        pt = torch.zeros((5, 64), dtype=torch.int32)
        cm = CacheManager(num_pages=64, page_size=1, page_table=pt, type="radix")
        pm = PrefillManager(
            cm, TableManager(max_running_reqs=4, page_table=pt), DecodeManager(1),
            keep_images_whole=keep_images_whole,
        )
        pm.pending_list = [PendingReq(uid=7, input_ids=torch.arange(30, dtype=torch.int32),
                                      sampling_params=SamplingParams(max_tokens=4), mm_items=[image])]
        lens, cuts = [], []
        while pm.runnable:
            batch = pm.schedule_next_batch(15)
            assert batch is not None
            cm.allocate_paged(batch.reqs)
            cuts.extend(cut_image_spans(batch.reqs))
            for r in batch.reqs:
                lens.append(r.extend_len)
                r.complete_one()
        return lens, cuts

    lens, cuts = chunk_lens(vision.attention_groups[0].bidirectional_mm_blocks)
    assert lens == [lo, 15, 30 - lo - 15] and cuts == []  # first chunk stops at the block, the next holds it whole
    lens, cuts = chunk_lens(text.attention_groups[0].bidirectional_mm_blocks)
    assert lens == [15, 15] and cuts == [(lo, hi)]  # the block is split into two spans; the cut warning would fire