Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

16.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Unset generation limit resolves to the context left free; no flat default fits all prompts."""

import sys
import threading
from pathlib import Path
from types import SimpleNamespace

import pytest

_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
    sys.path.insert(0, _BACKEND_DIR)

from core.inference.runtime_context import (
    UNSET_GENERATION_BUDGET,
    generation_budget_within_context,
)

_WINDOW = 2048
_PROMPT_LEN = 37


def _model(window = _WINDOW):
    return SimpleNamespace(config = SimpleNamespace(max_position_embeddings = window))


def test_an_unset_budget_becomes_the_free_context():
    assert generation_budget_within_context(_model(), _PROMPT_LEN, None) == _WINDOW - _PROMPT_LEN


def test_an_explicit_budget_is_never_reduced():
    assert generation_budget_within_context(_model(), _PROMPT_LEN, 512) == 512
    assert generation_budget_within_context(_model(), _PROMPT_LEN, _WINDOW) == _WINDOW
    wide = SimpleNamespace(
        max_seq_length = 1024, config = SimpleNamespace(max_position_embeddings = 32768)
    )
    assert generation_budget_within_context(wide, _PROMPT_LEN, 4096) == 4096


def test_a_prompt_with_no_room_left_gets_the_default_not_a_token():
    assert generation_budget_within_context(_model(window = 32), 64, None) == UNSET_GENERATION_BUDGET
    narrow = SimpleNamespace(
        max_seq_length = 1024, config = SimpleNamespace(max_position_embeddings = 32768)
    )
    assert generation_budget_within_context(narrow, 1100, None) == UNSET_GENERATION_BUDGET
    assert generation_budget_within_context(_model(window = 32), 31, None) == 1


def test_the_selected_window_wins_over_a_wider_checkpoint():
    # from_pretrained sets max_position_embeddings to max(requested, native), so config overstates it.
    model = SimpleNamespace(
        max_seq_length = 1024,
        config = SimpleNamespace(max_position_embeddings = 32768),
    )
    assert generation_budget_within_context(model, _PROMPT_LEN, None) == 1024 - _PROMPT_LEN


def test_a_model_declaring_no_window_falls_back():
    assert (
        generation_budget_within_context(SimpleNamespace(), _PROMPT_LEN, None)
        == UNSET_GENERATION_BUDGET
    )
    for unusable in (None, "n/a", True):
        assert (
            generation_budget_within_context(_model(window = unusable), 1, None)
            == UNSET_GENERATION_BUDGET
        )
    assert generation_budget_within_context(SimpleNamespace(), _PROMPT_LEN, 256) == 256


def test_a_zero_budget_is_a_value_not_an_absence():
    assert generation_budget_within_context(_model(), _PROMPT_LEN, 0) == 0


# Call-site tests import unsloth/unsloth_zoo, absent on CPU CI.


class _FakeTensor:
    def __init__(self, length):
        self.shape = (1, length)

    def to(self, _device):
        return self

    def __getitem__(self, _idx):
        return 2


class _FakeEncoding(dict):
    def to(self, _device):
        return self


class _FakeTokenizer:
    eos_token_id = 2
    pad_token_id = 2
    all_special_tokens: list = []

    def __call__(
        self,
        _prompt,
        return_tensors = None,
        add_special_tokens = True,
    ):
        return _FakeEncoding(input_ids = _FakeTensor(_PROMPT_LEN))


class _FastDecoder:
    def forward(self):
        pass


_FastDecoder.forward.__name__ = "LlamaModel_fast_forward"


class _StockDecoder:
    def forward(self):
        pass


class _FakeModel:
    """Carries the one guard unsloth_fast_generate applies before generating."""

    def __init__(
        self,
        window,
        decoder = _FastDecoder,
        native = None,
    ):
        self.max_seq_length = window
        self.config = SimpleNamespace(max_position_embeddings = native or window)
        self.device = "cpu"
        self.calls = []
        self._decoder = decoder()

    def modules(self):
        return iter((self, self._decoder))

    def generate(self, **kwargs):
        ids, budget = kwargs.get("input_ids"), kwargs.get("max_new_tokens")
        if ids is not None and budget is not None:
            if ids.shape[-1] + budget > self.config.max_position_embeddings:
                raise ValueError(
                    f"Unsloth: input length {ids.shape[-1]} + max_new_tokens {budget} "
                    "exceeds the maximum sequence length of "
                    f"{self.config.max_position_embeddings}!"
                )
        self.calls.append(kwargs)
        return _FakeTensor(ids.shape[-1] + 1)


class _EmptyStreamer:
    def __iter__(self):
        return self

    def __next__(self):
        raise StopIteration

    def end(self):
        pass


def _streaming_backend(
    monkeypatch,
    window = _WINDOW,
    **model_kwargs,
):
    try:
        from core.inference.inference import InferenceBackend
    except (ImportError, RuntimeError) as exc:  # pragma: no cover - env-dependent
        # CPU job has no unsloth; the helper tests above still cover the bound.
        pytest.skip(f"full inference backend unavailable ({type(exc).__name__}: {exc})")

    backend = InferenceBackend.__new__(InferenceBackend)
    backend.active_model_name = "unsloth/tinyllama-chat-bnb-4bit"
    backend.last_generation_stats = None
    backend._generation_lock = threading.Lock()
    model = _FakeModel(window, **model_kwargs)
    backend.models = {
        backend.active_model_name: {
            "model": model,
            "tokenizer": _FakeTokenizer(),
            "is_vision": False,
            "chat_turn_end_eos_ids": [2],
        }
    }
    monkeypatch.setattr(
        backend, "_make_text_streamer", lambda *a, **k: _EmptyStreamer(), raising = False
    )
    return backend, model


def _run(backend, max_new_tokens):
    return list(
        backend.generate_stream(
            "PROMPT",
            temperature = 0.0,
            max_new_tokens = max_new_tokens,
            repetition_penalty = 1.0,
        )
    )


def test_generate_stream_resolves_an_unset_budget(monkeypatch):
    backend, model = _streaming_backend(monkeypatch)

    _run(backend, None)

    assert model.calls, "generate was never reached"
    assert model.calls[0]["max_new_tokens"] == _WINDOW - _PROMPT_LEN


def test_generate_stream_passes_a_fitting_budget_through(monkeypatch):
    backend, model = _streaming_backend(monkeypatch)

    _run(backend, 512)

    assert model.calls[0]["max_new_tokens"] == 512


def test_generate_stream_refuses_a_prompt_longer_than_the_window(monkeypatch):
    from core.inference.context_refusal import ContextBudgetExceeded

    backend, model = _streaming_backend(monkeypatch, window = _PROMPT_LEN - 1)

    with pytest.raises(ContextBudgetExceeded) as refused:
        _run(backend, 64)

    assert (refused.value.request_tokens, refused.value.context_tokens) == (
        _PROMPT_LEN,
        _PROMPT_LEN - 1,
    )
    assert not model.calls


def test_generate_stream_lets_a_stock_forward_model_run_past_the_window(monkeypatch):
    backend, model = _streaming_backend(
        monkeypatch, window = _PROMPT_LEN - 1, decoder = _StockDecoder, native = 8192
    )

    _run(backend, 64)

    assert model.calls[0]["max_new_tokens"] == 64


def _vision_backend(monkeypatch, window = _WINDOW):
    try:
        from core.inference.inference import InferenceBackend
    except (ImportError, RuntimeError) as exc:  # pragma: no cover - env-dependent
        pytest.skip(f"full inference backend unavailable ({type(exc).__name__}: {exc})")

    backend = InferenceBackend.__new__(InferenceBackend)
    backend.active_model_name = "vision-model"
    backend.last_generation_stats = None
    backend._generation_lock = threading.Lock()
    model = _FakeModel(window)
    tokenizer = _FakeTokenizer()
    backend.models = {
        backend.active_model_name: {
            "model": model,
            "tokenizer": tokenizer,
            "processor": tokenizer,
            "is_vision": True,
            "chat_turn_end_eos_ids": [2],
        }
    }
    monkeypatch.setattr(
        backend, "_make_text_streamer", lambda *a, **k: _EmptyStreamer(), raising = False
    )
    monkeypatch.setattr(backend, "format_chat_prompt", lambda *a, **k: "PROMPT", raising = False)
    return backend, model


def test_the_vision_path_resolves_an_unset_budget(monkeypatch):
    """Without this, a VLM turn reaches generate() with max_new_tokens=None, which
    transformers reads as its own tiny max_length default rather than no limit."""
    backend, model = _vision_backend(monkeypatch)

    list(
        backend._generate_vision_response(
            [{"role": "user", "content": "hi"}],
            "",
            None,
            0.0,
            1.0,
            0,
            0.0,
            None,
            1.0,
        )
    )

    assert model.calls, "generate was never reached"
    assert model.calls[0]["max_new_tokens"] == _WINDOW - _PROMPT_LEN


def _audio_backend(monkeypatch, window = _WINDOW):
    try:
        from core.inference.inference import InferenceBackend
    except (ImportError, RuntimeError) as exc:  # pragma: no cover - env-dependent
        pytest.skip(f"full inference backend unavailable ({type(exc).__name__}: {exc})")

    backend = InferenceBackend.__new__(InferenceBackend)
    backend.active_model_name = "audio-model"
    backend.last_generation_stats = None
    backend._generation_lock = threading.Lock()
    model = _FakeModel(window)

    class _AudioProcessor(_FakeTokenizer):
        def apply_chat_template(self, *_a, **_k):
            return _FakeEncoding(input_ids = _FakeTensor(_PROMPT_LEN))

    processor = _AudioProcessor()
    backend.models = {
        backend.active_model_name: {
            "model": model,
            "tokenizer": processor,
            "processor": processor,
            "audio_type": "audio_vlm",
            "chat_turn_end_eos_ids": [2],
        }
    }
    monkeypatch.setattr(
        backend, "_make_text_streamer", lambda *a, **k: _EmptyStreamer(), raising = False
    )
    return backend, model


def _run_audio(backend, max_new_tokens):
    list(
        backend.generate_audio_input_response(
            [{"role": "user", "content": "hi"}],
            "",
            object(),
            0.0,
            1.0,
            0,
            0.0,
            max_new_tokens,
            1.0,
        )
    )


def test_the_audio_input_path_resolves_an_unset_budget(monkeypatch):
    """The OpenAI route sends no cap when the client sent none, on the audio branch as on
    the text one. Reaching generate() with None would take transformers' own tiny default."""
    backend, model = _audio_backend(monkeypatch)

    _run_audio(backend, None)

    assert model.calls, "generate was never reached"
    assert model.calls[0]["max_new_tokens"] == _WINDOW - _PROMPT_LEN


def test_the_audio_input_path_stops_when_cancelled(monkeypatch):
    import threading

    backend, model = _audio_backend(monkeypatch)
    cancel = threading.Event()
    list(
        backend.generate_audio_input_response(
            [{"role": "user", "content": "hi"}],
            "",
            object(),
            0.0,
            1.0,
            0,
            0.0,
            8,
            1.0,
            cancel_event = cancel,
        )
    )

    # The stream's own finally sets the event, so clear it to see the criteria follow it.
    criteria = model.calls[0]["stopping_criteria"]
    cancel.clear()
    assert not any(c(None, None) for c in criteria)
    cancel.set()
    assert any(c(None, None) for c in criteria)


def _rendered_audio_messages(
    monkeypatch,
    messages,
    audio = None,
):
    backend, _model = _audio_backend(monkeypatch)
    processor = backend.models[backend.active_model_name]["processor"]
    render = processor.apply_chat_template
    seen = {}

    def _capture(messages, *args, **kwargs):
        seen["messages"] = messages
        return render(messages, *args, **kwargs)

    monkeypatch.setattr(processor, "apply_chat_template", _capture, raising = False)
    list(
        backend.generate_audio_input_response(
            messages, "be brief", audio or object(), 0.0, 1.0, 0, 0.0, 8, 1.0
        )
    )
    return seen["messages"]


def _audio_turns(monkeypatch, messages):
    return [(m["role"], m.get("name")) for m in _rendered_audio_messages(monkeypatch, messages)]


def test_the_audio_input_turns_keep_participant_names(monkeypatch):
    assert _audio_turns(monkeypatch, [{"role": "user", "name": "alice", "content": "hi"}]) == [
        ("system", None),
        ("user", "alice"),
    ]


def test_the_audio_turn_keeps_its_text_and_ends_the_prompt(monkeypatch):
    audio = object()
    rendered = _rendered_audio_messages(
        monkeypatch,
        [
            {"role": "user", "content": "Summarize this"},
            {"role": "assistant", "content": "partial"},
        ],
        audio,
    )

    assert rendered[1:] == [
        {
            "role": "user",
            "content": [
                {"type": "audio", "audio": audio},
                {"type": "text", "text": "Summarize this"},
            ],
        }
    ]


def test_the_audio_turn_is_named_for_whoever_recorded_it(monkeypatch):
    assert _audio_turns(
        monkeypatch,
        [
            {"role": "user", "name": "alice", "content": "hi"},
            {"role": "assistant", "content": "hello"},
            {"role": "user", "name": "bob", "content": ""},
        ],
    ) == [("system", None), ("user", "alice"), ("assistant", None), ("user", "bob")]


def test_an_audio_only_turn_keeps_the_conversation_before_it(monkeypatch):
    audio = object()
    rendered = _rendered_audio_messages(
        monkeypatch,
        [
            {"role": "user", "content": "Write a haiku about the sea"},
            {"role": "assistant", "content": "Waves fold into foam"},
            {"role": "user", "content": ""},
        ],
        audio,
    )

    assert rendered[1:] == [
        {"role": "user", "content": [{"type": "text", "text": "Write a haiku about the sea"}]},
        {"role": "assistant", "content": [{"type": "text", "text": "Waves fold into foam"}]},
        {
            "role": "user",
            "content": [
                {"type": "audio", "audio": audio},
                {"type": "text", "text": "Please transcribe this audio."},
            ],
        },
    ]


def test_an_earlier_audio_turn_keeps_its_reply(monkeypatch):
    audio = object()
    rendered = _rendered_audio_messages(
        monkeypatch,
        [
            {"role": "user", "content": "Write a haiku about the sea"},
            {"role": "assistant", "content": "Waves fold into foam"},
            {"role": "user", "content": ""},
            {"role": "assistant", "content": "Please count from one to five."},
            {"role": "user", "content": ""},
        ],
        audio,
    )

    assert rendered[1:] == [
        {"role": "user", "content": [{"type": "text", "text": "Write a haiku about the sea"}]},
        {"role": "assistant", "content": [{"type": "text", "text": "Waves fold into foam"}]},
        {"role": "user", "content": []},
        {
            "role": "assistant",
            "content": [{"type": "text", "text": "Please count from one to five."}],
        },
        {
            "role": "user",
            "content": [
                {"type": "audio", "audio": audio},
                {"type": "text", "text": "Please transcribe this audio."},
            ],
        },
    ]


@pytest.mark.parametrize(
    "history, roles",
    [
        ([{"role": "user", "content": "My name is Nilay"}], ["system", "user"]),
        ([{"role": "assistant", "content": "Hi, how can I help?"}], ["system", "user"]),
        (
            [{"role": "user", "content": "My name is Nilay"}, {"role": "assistant", "content": ""}],
            ["system", "user"],
        ),
        (
            [
                {"role": "user", "content": "Weather in Paris?"},
                {
                    "role": "assistant",
                    "content": None,
                    "tool_calls": [
                        {
                            "id": "c1",
                            "type": "function",
                            "function": {"name": "weather", "arguments": "{}"},
                        }
                    ],
                },
                {"role": "tool", "tool_call_id": "c1", "content": "sunny"},
                {"role": "assistant", "content": "It is sunny."},
            ],
            ["system", "user", "assistant", "user"],
        ),
    ],
)
def test_an_audio_turn_after_an_unanswered_or_tool_turn_still_alternates(
    monkeypatch, history, roles
):
    audio = object()
    rendered = _rendered_audio_messages(
        monkeypatch, [*history, {"role": "user", "content": ""}], audio
    )

    assert [m["role"] for m in rendered] == roles
    assert rendered[-1]["content"] == [
        {"type": "audio", "audio": audio},
        {"type": "text", "text": "Please transcribe this audio."},
    ]


def test_an_explicit_audio_budget_is_untouched(monkeypatch):
    backend, model = _audio_backend(monkeypatch)

    _run_audio(backend, 128)

    assert model.calls[0]["max_new_tokens"] == 128