Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

13.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import json
import os
from types import SimpleNamespace

import pytest

from core.inference import mlx_speculative as spec

_DFLASH = {"architectures": ["DFlashDraftModel"], "dflash_config": {}, "vocab_size": 10}
_EAGLE3 = {"architectures": ["Eagle3Speculator"], "speculators_model_type": "eagle3"}
_ASSISTANT = {"architectures": ["Gemma4AssistantForCausalLM"], "model_type": "gemma4_assistant"}


_TARGET = "org/Qwen3.5-4B-4bit"


@pytest.fixture
def cache(tmp_path, monkeypatch):
    """Fills a Hugging Face cache with ``{repo: config}`` beside the target, and returns a resolver
    for the target."""
    snapshots = {}

    def fill(repos, builtin = False):
        for repo, config in {_TARGET: {"vocab_size": 10}, **repos}.items():
            snapshots[repo] = tmp_path / repo.replace("/", "--")
            snapshots[repo].mkdir(parents = True, exist_ok = True)
            (snapshots[repo] / "config.json").write_text(json.dumps(config))
            (snapshots[repo] / "model.safetensors").touch()
        monkeypatch.setattr(spec, "has_builtin_head", lambda model_dir: builtin)
        return lambda mode, named = None: spec.resolve_speculation(
            mode, named, model_dir = str(snapshots[_TARGET]), target_name = _TARGET
        )

    monkeypatch.setattr(spec, "_cached_repos", lambda: iter(list(snapshots)))
    monkeypatch.setattr(
        "utils.utils.hf_cache_snapshot_dir_for_repo", lambda repo: snapshots.get(repo)
    )
    return fill


def test_an_explicit_kind_tries_the_named_drafter_then_the_head_then_cached_companions(cache):
    _resolve = cache(
        {
            "a/Qwen3.5-4B-DFlash": _DFLASH,
            "b/Qwen3.5-4B-Eagle3": _EAGLE3,
            "e/Qwen3.5-4B-Eagle3": {
                "architectures": ["LlamaForCausalLMEagle3"],
                "model_type": "llama",
            },
            "c/Qwen3.5-9B-DFlash": _DFLASH,  # another model
            "d/Qwen3.5-4B-DFlash-bf16": {**_DFLASH, "vocab_size": 11},  # another vocabulary
            "g/Qwen3.5-4B-assistant": _ASSISTANT,
        },
        builtin = True,
    )
    assert [os.path.basename(s.path) for s in _resolve("dflash").sources] == [
        "a--Qwen3.5-4B-DFlash"
    ]
    assert _resolve("dflash").copies and spec.mlx_spec_mode("mtp+ngram") == "mtp"
    assert spec.mlx_spec_mode("eagle3+ngram") == "auto"
    assert [os.path.basename(s.path) for s in _resolve("eagle3").sources] == [
        "b--Qwen3.5-4B-Eagle3"
    ]
    named = _resolve("eagle3", "b/Qwen3.5-4B-Eagle3")
    assert [s.kind for s in named.sources] == ["eagle3"] and named.copies and named.speculative
    mtp = [(os.path.basename(s.path), s.builtin) for s in _resolve("draft-mtp").sources]
    assert mtp == [("org--Qwen3.5-4B-4bit", True), ("g--Qwen3.5-4B-assistant", False)]
    assert _resolve("mtp", "g/Qwen3.5-4B-assistant").sources[0].builtin is False
    # A named drafter of another kind is refused with a reason, never substituted silently.
    assert _resolve("dflash", "b/Qwen3.5-4B-Eagle3").reason == spec.DRAFTER_INCOMPATIBLE
    assert _resolve("dspark").reason == spec.DRAFTER_NOT_FOUND
    assert _resolve("dspark").speculative


def test_auto_takes_every_cached_kind_in_preference_order(cache):
    _resolve = cache({"b/Qwen3.5-4B-Eagle3": _EAGLE3, "a/Qwen3.5-4B-DFlash": _DFLASH}, True)
    assert [s.kind for s in _resolve("auto").sources] == ["mtp", "dflash", "eagle3"] and not (
        _resolve("auto").reason or _resolve("off").speculative
    )
    assert [spec.speculates_on_route("auto", vision) for vision in (False, True)] == [False, True]
    listed = [(repo, s.kind) for repo, s, _ in spec.cached_drafters(_TARGET, {"vocab_size": 10})]
    assert listed == [("a/Qwen3.5-4B-DFlash", "dflash"), ("b/Qwen3.5-4B-Eagle3", "eagle3")]
    mtp_head = {"h/Qwen3.5-4B-MTP-bf16": {"model_type": "qwen3_5_mtp"}}
    _resolve = cache({"g/Qwen3.5-4B-assistant": _ASSISTANT, **mtp_head}, True)
    for mode in (None, "bogus", "auto"):
        assert [(s.builtin, s.kind) for s in _resolve(mode).sources] == [(True, "mtp")] + [
            (False, "mtp")
        ] * 2 + [
            (False, "dflash"),
            (False, "eagle3"),
        ]
    assert _resolve("ngram-mod").speculative


def test_an_auto_offload_plan_makes_room_for_a_modes_first_drafter(cache, tmp_path):
    cache({"a/Qwen3.5-4B-DFlash": _DFLASH, "g/Qwen3.5-4B-assistant": _ASSISTANT}, True)
    target = str(tmp_path / "org--Qwen3.5-4B-4bit")
    drafter = lambda mode, named = None: spec.offload_drafter(
        mode, named, model_dir = target, target_name = _TARGET
    )
    assert drafter("mtp") == (target, True)  # the head, before the cached assistant
    assert drafter("mtp", "g/Qwen3.5-4B-assistant") == (
        str(tmp_path / "g--Qwen3.5-4B-assistant"),
        False,
    )
    assert drafter("dflash") == (str(tmp_path / "a--Qwen3.5-4B-DFlash"), False)
    assert drafter("auto") == (target, True)
    assert [drafter(mode) for mode in ("off", "ngram", "dspark")] == [None] * 3


def test_cached_drafters_are_matched_by_shape_and_weights_then_marked_by_name(cache, tmp_path):
    shaped = {**_DFLASH, "hidden_size": 8, "num_target_layers": 4}
    narrow = {"target_hidden_size": 8, "transformer_layer_config": {"hidden_size": 2}}
    cache(
        {
            "a/Qwen3.5-4B-DFlash": shaped,
            "n/Other-DFlash": shaped,  # fits, but is not named for the target
            "h/Qwen3.5-4B-DFlash": {**shaped, "hidden_size": 9},
            "t/Qwen3.5-4B-DFlash": {**shaped, "num_target_layers": 5},
            "l/Qwen3.5-4B-DFlash": {**shaped, "dflash_config": {"target_layer_ids": [1, 5]}},
            "w/Qwen3.5-4B-DFlash": shaped,
            "b/Qwen3.5-4B-assistant": {**_ASSISTANT, "backbone_hidden_size": 9},
            "m/Qwen3.5-4B-MTP": {"model_type": "x_mtp", "text_config": {"num_hidden_layers": 5}},
            "e/Qwen3.5-4B-Eagle3": {
                **_EAGLE3,
                **narrow,
                "eagle_aux_hidden_state_layer_ids": [1, 4],
            },
        }
    )
    (tmp_path / "w--Qwen3.5-4B-DFlash" / "model.safetensors").unlink()
    target = {"vocab_size": 10, "text_config": {"hidden_size": 8, "num_hidden_layers": 4}}
    found = spec.cached_drafters("unsloth/Qwen3.5-4B-UD-MLX-4bit", target)
    assert [(repo, named) for repo, _, named in found] == [
        ("a/Qwen3.5-4B-DFlash", True),
        ("e/Qwen3.5-4B-Eagle3", True),
        ("n/Other-DFlash", False),
    ]
    assert len(spec.discover_companions("unsloth/Qwen3.5-4B-UD-MLX-4bit", target)) == 2
    assert len(spec.discover_companions("unsloth/Qwen3.5-4B-UD-MLX-4bit", target, found[1])) == 1
    assert spec._stem("g/gemma-4-31B-it-qat-q4_0-unquantized-assistant") == "gemma431bit"


@pytest.mark.parametrize("decision", [None, False, True])
def test_a_drafter_build_hears_fast_verify_only_from_a_zoo_that_takes_it(monkeypatch, decision):
    drafters = pytest.importorskip("unsloth_zoo.mlx.speculative")
    told = []
    factory = lambda path, target, **kw: told.append(kw) or SimpleNamespace(max_depth = 8)
    monkeypatch.setattr(drafters, "companion_drafter", factory)
    monkeypatch.setattr(drafters, "native_mtp_drafter", factory)
    sources = (spec.DrafterSource("dflash", "a", False), spec.DrafterSource("mtp", "b", True))
    for source in sources:
        resolution = spec.SpecResolution("dflash", (source,))
        spec.build_draft(None, resolution, fits = lambda _: (True, None), fast_verify = decision)
    assert told == [{} if decision is None else {"fast_verify": decision}] * 2


def test_the_installed_zoo_refuses_a_fast_verify_flip_or_reports_no_support():
    import inspect

    from core.inference import mlx_inference as mi

    zoo = pytest.importorskip("unsloth_zoo.mlx.inference")
    if "fast_verify" not in inspect.signature(zoo.inference_scopes).parameters:
        assert mi._fast_verify_status(True, object())["reason"] == "unsupported_zoo"
        assert mi.mlx_fast_verify_checkpoint_status("/none") == (False, "unsupported_zoo")
        return
    assert mi._fast_verify_zoo() is zoo
    target = SimpleNamespace()
    zoo.pin_fast_verify(target, True)
    with pytest.raises(ValueError):
        zoo.pin_fast_verify(target, False)


def test_a_drafter_passed_over_keeps_its_reason_on_the_one_that_attaches(monkeypatch):
    drafters = pytest.importorskip("unsloth_zoo.mlx.speculative")

    built = []

    def factory(path, target):
        if path == "broken":
            raise ValueError("mismatched vocabulary")
        built.append(path)
        return SimpleNamespace(kind = "dflash", max_depth = 15)

    monkeypatch.setattr(drafters, "companion_drafter", factory)
    monkeypatch.setattr(drafters, "native_mtp_drafter", factory)
    sources = tuple(spec.DrafterSource("dflash", path, False) for path in ("big", "broken", "ok"))
    fits = lambda source: (source.path != "big", 4096)

    draft, kind, reason, context = spec.build_draft(
        None, spec.SpecResolution("dflash", sources), fits = fits, draft_n_max = 6
    )
    assert (kind, reason, context, built) == ("dflash", spec.DRAFTER_INCOMPATIBLE, 4096, ["ok"])
    assert (draft.controller.max_depth, draft.controller.max_copy) == (6, 0)
    for d, n, e in ((object(), 3, True), (object(), 4, False), (draft.drafter, 15, True)):
        assert spec._draft(d, False, n).controller.fixed_depth is e

    def chosen(
        picked,
        fit,
        stretch = lambda _: (None, None),
    ):
        built.clear()
        result = spec.build_draft(
            None, spec.SpecResolution("dflash", picked, copies = True), fits = fit, stretch = stretch
        )
        return result[1:] + (built[:],)

    assert chosen(sources[::2], fits) == ("dflash", spec.DRAFTER_NO_MEMORY, 4096, ["ok"])
    assert chosen(sources[:1], fits) == ("dflash", None, None, ["big"])
    assert chosen(sources[:1], lambda _: (None, None)) == ("dflash", None, None, ["big"])
    assert chosen(sources[:1], fits, lambda _: (True, 2048)) == ("dflash", None, 2048, ["big"])
    # Past MLX's own cap a chosen drafter is still passed over, as is one nothing can stretch.
    for stretch in (lambda _: (False, None), None):
        assert chosen(sources[:1], fits, stretch) == ("ngram", spec.DRAFTER_NO_MEMORY, None, [])
    assert chosen(sources[:2], fits) == ("dflash", spec.DRAFTER_INCOMPATIBLE, None, ["big"])
    assert chosen(sources[1:2], lambda _: (False, 4096)) == (
        "ngram",
        spec.DRAFTER_INCOMPATIBLE,
        None,
        [],
    )
    draft, *_ = spec.build_draft(
        None, spec.SpecResolution("dflash", sources[1:2], copies = True), fits = lambda _: (False, 4096)
    )
    assert (draft.drafter, draft.controller.max_depth, draft.controller.max_copy) == (None, 0, 16)
    auto = spec.SpecResolution("auto", sources[:1], copies = True)
    unfit = [spec.build_draft(None, auto, fits = lambda _: (ok, None)) for ok in (False, None)]
    assert unfit == [(None, None, spec.AUTO_CONTEXT_COST, None), (None,) * 4]
    capped = spec.build_draft(
        None, spec.SpecResolution("ngram", copies = True), fits = fits, draft_n_max = 5
    )
    assert capped[0].controller.max_copy == 5

    def encoder_decoder(inputs, encoder_outputs = None):
        pass

    def cross_attending(inputs, cross_attention_states = None):
        pass

    for target in (encoder_decoder, SimpleNamespace(language_model = cross_attending)):
        refused = spec.build_draft(
            target, spec.SpecResolution("dflash", sources[2:], copies = True), fits = fits
        )
        assert refused == (None, None, spec.RUNTIME_ERROR, None)


def test_a_drafter_carrying_its_own_model_module_is_never_found_or_attached(cache):
    # mlx-vlm imports a config's model_file on load, so such a checkpoint would run its own code.
    custom = {**_DFLASH, "model_file": "model.py"}
    _resolve = cache({"a/Qwen3.5-4B-DFlash": custom})
    assert spec.cached_drafters(_TARGET, {"vocab_size": 10}) == []
    assert not _resolve("auto").sources and not _resolve("dflash").sources
    assert _resolve("dflash", "a/Qwen3.5-4B-DFlash").reason == spec.DRAFTER_NOT_FOUND


def test_the_built_in_head_is_found_by_the_zoos_detection_or_mlx_vlms_without_it(
    monkeypatch, tmp_path
):
    import sys

    found = lambda name: lambda path: str(path) == str(tmp_path / name)
    zoo = SimpleNamespace(has_native_mtp_head = found("zoo"))
    vlm = SimpleNamespace(detect_mtp_splitter = lambda path: found("vlm")(path) or None)
    monkeypatch.setitem(sys.modules, "mlx_vlm.speculative.drafters.mtp_split", vlm)
    # An unsloth-zoo without has_native_mtp_head leaves detection to mlx-vlm.
    for module, finder in ((zoo, "zoo"), (SimpleNamespace(), "vlm")):
        monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.speculative", module)
        heads = [spec.has_builtin_head(str(tmp_path / name)) for name in ("zoo", "vlm")]
        assert heads == [finder == "zoo", finder == "vlm"]
    monkeypatch.setitem(sys.modules, "unsloth_zoo.mlx.speculative", zoo)
    assert not spec.has_builtin_head(None)
    monkeypatch.setattr(zoo, "has_native_mtp_head", lambda path: 1 / 0)
    assert not spec.has_builtin_head(str(tmp_path / "zoo"))