Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

3.3 KB
Raw Blame
"""Engine._run_mm_encoder against a fake model: chunked gathers, a shared image, precomputed embeddings, orphan jobs."""

from __future__ import annotations

from types import SimpleNamespace

import pytest
import torch

from freetoken.engine.engine import Engine
from freetoken.message.backend import MMItem
from freetoken.mm.encoder_cache import EncoderCache

H = 8


class _FakeVision:
    def __init__(self):
        self.calls = 0

    def encode(self, item: MMItem) -> torch.Tensor:
        self.calls += 1
        return torch.full((item.num_tokens, H), float(item.hash))


def _engine(cache: EncoderCache, model=None) -> SimpleNamespace:
    return SimpleNamespace(
        encoder_cache=cache, device=torch.device("cpu"), dtype=torch.float32,
        model=model or _FakeVision(),
    )


def _item(h: int, n_tokens: int, precomputed: torch.Tensor | None = None) -> MMItem:
    feature = None if precomputed is not None else torch.zeros(1)
    return MMItem(
        modality="image", hash=h, pad_value=0, offsets=[[0, n_tokens]],
        feature=feature, precomputed_embeddings=precomputed,
    )


def _batch(jobs, plan) -> SimpleNamespace:
    return SimpleNamespace(mm_encoder_jobs=jobs, mm_gather_plan=plan, mm_embeds=None, mm_rows=None)


def test_entry_lives_until_its_consumer_gathers_the_last_row():
    cache = EncoderCache(storage="cpu")
    eng = _engine(cache)
    item = _item(h=3, n_tokens=4)
    cache.register(3, 7, 4)  # admission claims the whole image
    # chunk 1 consumes rows [0, 2) of a 4-token image
    Engine._run_mm_encoder(eng, _batch([item], [(7, 3, 0, 2, 4, 0)]))
    assert cache._entries[3].remaining == {7: 2}
    assert item.feature is None
    # chunk 2 finishes the image; the entry dies with its last claim
    Engine._run_mm_encoder(eng, _batch([], [(7, 3, 2, 4, 4, 0)]))
    assert not cache.has(3)
    assert eng.model.calls == 1


def test_shared_image_is_encoded_once_and_sliced_per_request():
    cache = EncoderCache(storage="cpu")
    eng = _engine(cache)
    a, b = _item(h=5, n_tokens=4), _item(h=5, n_tokens=4)
    cache.register(5, 1, 4)
    cache.register(5, 2, 4)
    batch = _batch([a, b], [(1, 5, 0, 4, 4, 0), (2, 5, 0, 2, 4, 4)])
    Engine._run_mm_encoder(eng, batch)
    assert eng.model.calls == 1
    assert batch.mm_embeds.shape == (6, H)
    assert torch.equal(batch.mm_embeds, torch.full((6, H), 5.0))
    # request 1 consumed its image; request 2 still has rows to gather in its next chunk
    assert cache._entries[5].remaining == {2: 2}


def test_precomputed_embeddings_bypass_the_encoder():
    cache = EncoderCache(storage="cpu")

    class _NoVision:
        def encode(self, item):
            raise AssertionError("encoder must not run for precomputed embeddings")

    eng = _engine(cache, model=_NoVision())
    emb = torch.arange(2 * H, dtype=torch.float32).view(2, H)
    item = _item(h=4, n_tokens=2, precomputed=emb)
    cache.register(4, 1, 2)
    batch = _batch([item], [(1, 4, 0, 2, 2, 0)])
    Engine._run_mm_encoder(eng, batch)
    assert torch.equal(batch.mm_embeds, emb)
    assert item.precomputed_embeddings is None
    assert not cache.has(4)


def test_job_without_gather_row_fails_loudly():
    eng = _engine(EncoderCache(storage="cpu"))
    with pytest.raises(AssertionError, match="encoder jobs without gather rows"):
        Engine._run_mm_encoder(eng, _batch([_item(h=1, n_tokens=2)], []))