Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

13.7 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""The MLX reply snapshots decode only the ids since the last safe split, and still equal a full
re-decode at every step. Built on small in-memory tokenizers, so no MLX and no download."""

import random
import sys
import types
from contextlib import contextmanager
from types import SimpleNamespace

import pytest

tokenizers = pytest.importorskip("tokenizers")
transformers = pytest.importorskip("transformers")

from tokenizers import Tokenizer, decoders, models, normalizers, pre_tokenizers, trainers
from transformers import PreTrainedTokenizerFast

from core.inference.incremental_decode import IncrementalDecode, split_profile
from core.inference.native_tool_tokens import NativeToolTokenDecoder

_CORPUS = [
    "The quick brown fox isn't here. It's gone , they 've said ! Don 't ' quote ' me ? Yes .",
    "def f(x):\n    return {'a': [1, 2, 3]}  # comment\n\n\tif x  >  0 :\n\t\tpass\r\n",
    "😀🎉👩‍👩‍👧‍👦 🇯🇵 🏳️‍🌈 café naïve résumé 𝔘𝔫𝔦𝔠𝔬𝔡𝔢",
    "日本語のテキストと中文混合,한국어 문장도。龘靐齉",
    "a    b\t\t\tc\n\n\n   \n  d     e" + " " * 40 + "f",
    '<think>\nreasoning</think>\n\nAnswer: 42 <tool_call>{"name": "f"}</tool_call>',
]
_ADDED = ["<think>", "</think>", "<tool_call>", "</tool_call>"]


def _fast(backend, cleanup, **specials):
    fast = PreTrainedTokenizerFast(
        tokenizer_object = backend,
        clean_up_tokenization_spaces = cleanup,
        **specials,
    )
    fast.add_tokens(_ADDED)
    return fast


def _byte_level(cleanup = False):
    backend = Tokenizer(models.BPE())
    backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space = False)
    backend.decoder = decoders.ByteLevel()
    trainer = trainers.BpeTrainer(
        vocab_size = 700,
        initial_alphabet = pre_tokenizers.ByteLevel.alphabet(),
        special_tokens = ["<|eot|>"],
    )
    backend.train_from_iterator(_CORPUS * 4, trainer)
    return _fast(backend, cleanup, eos_token = "<|eot|>")


def _sentencepiece(decoder, cleanup = False):
    """A Llama-style piece vocab with <0xNN> byte fallback, so pieces outside it arrive as bytes."""
    pieces = ["<unk>", "<s>", "</s>"] + [f"<0x{value:02X}>" for value in range(256)]
    pieces += sorted({ch for line in _CORPUS[:2] for ch in line.replace(" ", "▁")})
    pieces += ["▁", "▁the", "▁a", "n't", "'s", "▁.", "▁,", "▁'", "▁n", "ve", "re", "▁▁", "日本"]
    vocab = {piece: index for index, piece in enumerate(dict.fromkeys(pieces))}
    backend = Tokenizer(models.BPE(vocab = vocab, merges = [], unk_token = "<unk>", byte_fallback = True))
    backend.normalizer = normalizers.Sequence(
        [normalizers.Prepend("▁"), normalizers.Replace(" ", "▁")]
    )
    backend.decoder = decoder
    return _fast(backend, cleanup, bos_token = "<s>", eos_token = "</s>", unk_token = "<unk>")


def _llama_spm_decoder():
    return decoders.Sequence(
        [
            decoders.Replace("▁", " "),
            decoders.ByteFallback(),
            decoders.Fuse(),
            decoders.Strip(" ", 1, 0),
        ]
    )


def _gemma_spm_decoder():
    return decoders.Sequence([decoders.Replace("▁", " "), decoders.ByteFallback(), decoders.Fuse()])


_TOKENIZERS = {
    "bytelevel": lambda: _byte_level(),
    "bytelevel-cleanup": lambda: _byte_level(cleanup = True),
    "llama-spm": lambda: _sentencepiece(_llama_spm_decoder()),
    "llama-spm-cleanup": lambda: _sentencepiece(_llama_spm_decoder(), cleanup = True),
    "gemma-spm": lambda: _sentencepiece(_gemma_spm_decoder()),
    "metaspace": lambda: _sentencepiece(
        decoders.Metaspace(replacement = "▁", prepend_scheme = "always")
    ),
}
_DECODE_KWARGS = [
    {"skip_special_tokens": True},
    {"skip_special_tokens": True, "clean_up_tokenization_spaces": False},
    {"skip_special_tokens": True, "clean_up_tokenization_spaces": True},
    {"skip_special_tokens": False, "spaces_between_special_tokens": False},
]


@pytest.fixture(scope = "module", params = sorted(_TOKENIZERS))
def tokenizer(request):
    return _TOKENIZERS[request.param]()


class _Wrapper:
    """mlx-lm's TokenizerWrapper forwards decode to the Hugging Face tokenizer."""

    def __init__(self, tokenizer):
        self._tokenizer = tokenizer

    def __getattr__(self, name):
        return getattr(self._tokenizer, name)


def _streams(tokenizer, seed = 0):
    rng = random.Random(seed)
    size = len(tokenizer)
    text = "\n".join(_CORPUS)
    encoded = tokenizer.encode(text * 3, add_special_tokens = False)
    special = sorted(tokenizer.added_tokens_decoder)
    byte_ids = [
        i for i in range(size) if (tokenizer.convert_ids_to_tokens(i) or "").startswith("<0x")
    ]
    tricky = tokenizer.encode(" n't 's 've 're ' . , ? ! \n 😀 日本", add_special_tokens = False)
    pool = byte_ids + special + tricky or list(range(size))
    streams = {
        "text": encoded,
        "text-from-mid-character": encoded[1:],
        "emoji": tokenizer.encode(_CORPUS[2] * 6, add_special_tokens = False),
    }
    for index in range(4):
        # Ids past the vocabulary too: a padded embedding can sample one, and decode drops it.
        streams[f"random-{index}"] = [rng.randrange(size + 8) for _ in range(300)]
        streams[f"adversarial-{index}"] = [
            rng.choice(pool) if rng.random() < 0.6 else rng.randrange(size) for _ in range(300)
        ]
    return streams


@pytest.mark.parametrize(
    "decode_kwargs", _DECODE_KWARGS, ids = lambda kw: ",".join(f"{k}={v}" for k, v in kw.items())
)
def test_every_snapshot_equals_a_full_redecode(tokenizer, decode_kwargs):
    assert split_profile(tokenizer) is not None
    for name, ids in _streams(tokenizer).items():
        for source in (tokenizer, _Wrapper(tokenizer)):
            incremental = IncrementalDecode(source, decode_kwargs)
            for count in range(1, len(ids) + 1):
                expected = tokenizer.decode(ids[:count], **decode_kwargs)
                assert incremental.feed(ids[count - 1]) == expected, (name, count)
            # The split path stayed on: a fallback to full decodes would also match.
            assert incremental._profile is not None, name


def test_native_decoder_snapshots_equal_its_full_decode(tokenizer):
    decoder = NativeToolTokenDecoder(tokenizer, preserved_tokens = {"<think>", "</think>"})
    for name, ids in _streams(tokenizer, seed = 1).items():
        incremental = decoder.incremental()
        for count in range(1, len(ids) + 1):
            assert incremental.feed(ids[count - 1]) == decoder.decode(ids[:count]), (name, count)


class _CountingBackend:
    """Counts the ids each Rust decode receives, the cost a full re-decode pays per token."""

    def __init__(self, inner):
        self._inner = inner
        self.decoded = 0

    def decode(self, ids, *args, **kwargs):
        self.decoded += len(ids)
        return self._inner.decode(ids, *args, **kwargs)

    def __getattr__(self, name):
        return getattr(self._inner, name)


def _counting(tokenizer):
    counter = _CountingBackend(tokenizer._tokenizer)
    tokenizer._tokenizer = counter
    return counter


def test_work_per_token_stays_bounded():
    tokenizer = _byte_level(cleanup = True)
    counter = _counting(tokenizer)
    ids = tokenizer.encode("\n".join(_CORPUS[:2]) * 60, add_special_tokens = False)
    incremental = IncrementalDecode(tokenizer, {"skip_special_tokens": True})
    for token_id in ids:
        incremental.feed(token_id)
    counter.decoded = 0
    for token_id in ids[:200]:
        incremental.feed(token_id)
    # A full re-decode would pass every id so far, len(ids) + 100 on average, each step.
    assert len(ids) > 2000
    assert counter.decoded <= 200 * 32


@pytest.mark.parametrize(
    "backend",
    [
        decoders.WordPiece(),
        decoders.Sequence([decoders.Fuse(), decoders.Replace("ab", "x")]),
        decoders.Sequence([decoders.Fuse(), decoders.Strip(" ", 2, 0)]),
        decoders.Sequence([decoders.Fuse(), decoders.ByteFallback()]),
        decoders.Sequence([decoders.Sequence([decoders.Fuse()]), decoders.ByteFallback()]),
        decoders.Sequence([decoders.ByteLevel(), decoders.ByteFallback()]),
    ],
    ids = [
        "wordpiece",
        "multi-character-replace",
        "strip-two",
        "fuse-then-bytes",
        "nested-fuse",
        "bytelevel-then-bytes",
    ],
)
def test_an_unknown_decoder_keeps_the_full_decode(backend):
    tokenizer = _sentencepiece(backend)
    assert split_profile(tokenizer) is None
    ids = _streams(tokenizer)["adversarial-0"]
    incremental = IncrementalDecode(tokenizer, {"skip_special_tokens": True})
    for count in range(1, len(ids) + 1):
        assert incremental.feed(ids[count - 1]) == tokenizer.decode(
            ids[:count], skip_special_tokens = True
        )


@pytest.mark.parametrize("scheme", ["always", "first"])
def test_a_metaspace_piece_ending_in_its_replacement_never_splits_a_cleanup_pattern(scheme):
    # Metaspace drops the replacement char of a run's first id only: "a▁" decodes alone
    # to "a" but to "a " mid-reply, so cleanup's " ." spans a split taken after it.
    vocab = {"<unk>": 0, "x": 1, "a▁": 2, ".": 3, "'s": 4, "▁b": 5}
    backend = Tokenizer(models.WordLevel(vocab, unk_token = "<unk>"))
    backend.decoder = decoders.Metaspace(prepend_scheme = scheme)
    tokenizer = PreTrainedTokenizerFast(
        tokenizer_object = backend, unk_token = "<unk>", clean_up_tokenization_spaces = True
    )
    assert split_profile(tokenizer) is not None
    for ids in ([1, 2, 2, 3, 1], [5, 2, 2, 4, 2, 3, 5], [1, 2, 3, 1, 2, 2, 3]):
        for kwargs in ({}, {"skip_special_tokens": True}, {"clean_up_tokenization_spaces": True}):
            incremental = IncrementalDecode(_Wrapper(tokenizer), kwargs)
            for count in range(1, len(ids) + 1):
                assert incremental.feed(ids[count - 1]) == tokenizer.decode(ids[:count], **kwargs)


def test_an_overridden_decode_keeps_the_full_decode():
    tokenizer = _byte_level()

    class _Override(type(tokenizer)):
        def _decode(self, token_ids, **kwargs):
            return super()._decode(token_ids, **kwargs).upper()

    tokenizer.__class__ = _Override
    assert split_profile(tokenizer) is None
    assert split_profile(SimpleNamespace(decode = lambda ids, **_k: "")) is None


# End to end through the MLX text path, with mlx_lm faked and a real tokenizer.


class _Resp:
    def __init__(self, text, token):
        self.text = text
        self.token = token


def _text_backend(monkeypatch, tokenizer, ids):
    from core.inference import mlx_inference

    mlx_lm = types.ModuleType("mlx_lm")
    sample_utils = types.ModuleType("mlx_lm.sample_utils")
    sample_utils.make_sampler = lambda **_k: object()
    mlx_lm.sample_utils = sample_utils
    mlx_lm.stream_generate = lambda *_a, **_k: iter([_Resp("", token) for token in ids])
    monkeypatch.setitem(sys.modules, "mlx_lm", mlx_lm)
    monkeypatch.setitem(sys.modules, "mlx_lm.sample_utils", sample_utils)
    monkeypatch.setattr(mlx_inference, "_mlx_sampling_processors", lambda **_k: None)

    @contextmanager
    def _adapter_state(_model, _state):
        yield

    monkeypatch.setattr(mlx_inference, "_temporary_mlx_adapter_state", _adapter_state)
    backend = mlx_inference.MLXInferenceBackend()
    backend._model = SimpleNamespace(config = {"model_type": "llama"})
    backend._tokenizer = _Wrapper(tokenizer)
    backend._served_context = 1 << 20
    monkeypatch.setattr(
        backend,
        "_render_text_prompt",
        lambda *_a, **_k: SimpleNamespace(prompt = "P", reasoning_channel_markers = None),
        raising = False,
    )
    monkeypatch.setattr(
        backend, "_prepare_prompt_cache", lambda _p, _a: ([1, 2], None, None, [1, 2], 0)
    )
    return backend


@pytest.mark.parametrize(
    "tools", [None, [{"type": "function", "function": {"name": "f"}}]], ids = ["plain", "tools"]
)
def test_the_mlx_text_stream_yields_full_redecode_snapshots_with_bounded_work(monkeypatch, tools):
    tokenizer = _byte_level(cleanup = True)
    ids = tokenizer.encode("\n".join(_CORPUS) * 30, add_special_tokens = False)
    expected = []
    if tools:
        decoder = NativeToolTokenDecoder(tokenizer)
        full = lambda count: decoder.decode(ids[:count])
    else:
        full = lambda count: tokenizer.decode(ids[:count], skip_special_tokens = True)
    for count in range(1, len(ids) + 1):
        expected.append(full(count).rstrip("\ufffd"))
    counter = _counting(tokenizer)
    backend = _text_backend(monkeypatch, tokenizer, ids)
    snapshots = list(
        backend._generate_text(
            [{"role": "user", "content": "hi"}], 0.0, 1.0, 0, 0.0, len(ids), 1.0, None, tools = tools
        )
    )
    assert snapshots[: len(expected)] == expected
    # Re-decoding every id each step would pass about len(ids) ** 2 / 2 ids.
    assert len(ids) > 1500
    assert counter.decoded <= 32 * len(ids)


def test_a_row_stream_snapshots_equal_a_full_redecode():
    from core.inference import mlx_inference

    tokenizer = _sentencepiece(_llama_spm_decoder(), cleanup = True)
    ids = _streams(tokenizer)["text"]
    row = mlx_inference._RowStream(
        decode = mlx_inference._ReplyText(tokenizer, {"skip_special_tokens": True})
    )
    for count in range(1, len(ids) + 1):
        expected = tokenizer.decode(ids[:count], skip_special_tokens = True).rstrip("\ufffd")
        assert row.feed(token = ids[count - 1]) == [expected]


def test_a_cached_profile_does_not_keep_its_tokenizer_alive():
    import gc
    import weakref

    tokenizer = _byte_level()
    profile = split_profile(tokenizer)
    assert profile is not None
    profile.splits(5, 6, True)
    ref = weakref.ref(tokenizer)
    del tokenizer, profile
    gc.collect()
    assert ref() is None