Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Unset generation limit resolves to the context left free; no flat default fits all prompts."""
import sys
import threading
from pathlib import Path
from types import SimpleNamespace
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
from core.inference.runtime_context import (
UNSET_GENERATION_BUDGET,
generation_budget_within_context,
)
_WINDOW = 2048
_PROMPT_LEN = 37
def _model(window = _WINDOW):
return SimpleNamespace(config = SimpleNamespace(max_position_embeddings = window))
def test_an_unset_budget_becomes_the_free_context():
assert generation_budget_within_context(_model(), _PROMPT_LEN, None) == _WINDOW - _PROMPT_LEN
def test_an_explicit_budget_is_never_reduced():
assert generation_budget_within_context(_model(), _PROMPT_LEN, 512) == 512
assert generation_budget_within_context(_model(), _PROMPT_LEN, _WINDOW) == _WINDOW
wide = SimpleNamespace(
max_seq_length = 1024, config = SimpleNamespace(max_position_embeddings = 32768)
)
assert generation_budget_within_context(wide, _PROMPT_LEN, 4096) == 4096
def test_a_prompt_with_no_room_left_gets_the_default_not_a_token():
assert generation_budget_within_context(_model(window = 32), 64, None) == UNSET_GENERATION_BUDGET
narrow = SimpleNamespace(
max_seq_length = 1024, config = SimpleNamespace(max_position_embeddings = 32768)
)
assert generation_budget_within_context(narrow, 1100, None) == UNSET_GENERATION_BUDGET
assert generation_budget_within_context(_model(window = 32), 31, None) == 1
def test_the_selected_window_wins_over_a_wider_checkpoint():
# from_pretrained sets max_position_embeddings to max(requested, native), so config overstates it.
model = SimpleNamespace(
max_seq_length = 1024,
config = SimpleNamespace(max_position_embeddings = 32768),
)
assert generation_budget_within_context(model, _PROMPT_LEN, None) == 1024 - _PROMPT_LEN
def test_a_model_declaring_no_window_falls_back():
assert (
generation_budget_within_context(SimpleNamespace(), _PROMPT_LEN, None)
== UNSET_GENERATION_BUDGET
)
for unusable in (None, "n/a", True):
assert (
generation_budget_within_context(_model(window = unusable), 1, None)
== UNSET_GENERATION_BUDGET
)
assert generation_budget_within_context(SimpleNamespace(), _PROMPT_LEN, 256) == 256
def test_a_zero_budget_is_a_value_not_an_absence():
assert generation_budget_within_context(_model(), _PROMPT_LEN, 0) == 0
# Call-site tests import unsloth/unsloth_zoo, absent on CPU CI.
class _FakeTensor:
def __init__(self, length):
self.shape = (1, length)
def to(self, _device):
return self
def __getitem__(self, _idx):
return 2
class _FakeEncoding(dict):
def to(self, _device):
return self
class _FakeTokenizer:
eos_token_id = 2
pad_token_id = 2
all_special_tokens: list = []
def __call__(
self,
_prompt,
return_tensors = None,
add_special_tokens = True,
):
return _FakeEncoding(input_ids = _FakeTensor(_PROMPT_LEN))
class _FastDecoder:
def forward(self):
pass
_FastDecoder.forward.__name__ = "LlamaModel_fast_forward"
class _StockDecoder:
def forward(self):
pass
class _FakeModel:
"""Carries the one guard unsloth_fast_generate applies before generating."""
def __init__(
self,
window,
decoder = _FastDecoder,
native = None,
):
self.max_seq_length = window
self.config = SimpleNamespace(max_position_embeddings = native or window)
self.device = "cpu"
self.calls = []
self._decoder = decoder()
def modules(self):
return iter((self, self._decoder))
def generate(self, **kwargs):
ids, budget = kwargs.get("input_ids"), kwargs.get("max_new_tokens")
if ids is not None and budget is not None:
if ids.shape[-1] + budget > self.config.max_position_embeddings:
raise ValueError(
f"Unsloth: input length {ids.shape[-1]} + max_new_tokens {budget} "
"exceeds the maximum sequence length of "
f"{self.config.max_position_embeddings}!"
)
self.calls.append(kwargs)
return _FakeTensor(ids.shape[-1] + 1)
class _EmptyStreamer:
def __iter__(self):
return self
def __next__(self):
raise StopIteration
def end(self):
pass
def _streaming_backend(
monkeypatch,
window = _WINDOW,
**model_kwargs,
):
try:
from core.inference.inference import InferenceBackend
except (ImportError, RuntimeError) as exc: # pragma: no cover - env-dependent
# CPU job has no unsloth; the helper tests above still cover the bound.
pytest.skip(f"full inference backend unavailable ({type(exc).__name__}: {exc})")
backend = InferenceBackend.__new__(InferenceBackend)
backend.active_model_name = "unsloth/tinyllama-chat-bnb-4bit"
backend.last_generation_stats = None
backend._generation_lock = threading.Lock()
model = _FakeModel(window, **model_kwargs)
backend.models = {
backend.active_model_name: {
"model": model,
"tokenizer": _FakeTokenizer(),
"is_vision": False,
"chat_turn_end_eos_ids": [2],
}
}
monkeypatch.setattr(
backend, "_make_text_streamer", lambda *a, **k: _EmptyStreamer(), raising = False
)
return backend, model
def _run(backend, max_new_tokens):
return list(
backend.generate_stream(
"PROMPT",
temperature = 0.0,
max_new_tokens = max_new_tokens,
repetition_penalty = 1.0,
)
)
def test_generate_stream_resolves_an_unset_budget(monkeypatch):
backend, model = _streaming_backend(monkeypatch)
_run(backend, None)
assert model.calls, "generate was never reached"
assert model.calls[0]["max_new_tokens"] == _WINDOW - _PROMPT_LEN
def test_generate_stream_passes_a_fitting_budget_through(monkeypatch):
backend, model = _streaming_backend(monkeypatch)
_run(backend, 512)
assert model.calls[0]["max_new_tokens"] == 512
def test_generate_stream_refuses_a_prompt_longer_than_the_window(monkeypatch):
from core.inference.context_refusal import ContextBudgetExceeded
backend, model = _streaming_backend(monkeypatch, window = _PROMPT_LEN - 1)
with pytest.raises(ContextBudgetExceeded) as refused:
_run(backend, 64)
assert (refused.value.request_tokens, refused.value.context_tokens) == (
_PROMPT_LEN,
_PROMPT_LEN - 1,
)
assert not model.calls
def test_generate_stream_lets_a_stock_forward_model_run_past_the_window(monkeypatch):
backend, model = _streaming_backend(
monkeypatch, window = _PROMPT_LEN - 1, decoder = _StockDecoder, native = 8192
)
_run(backend, 64)
assert model.calls[0]["max_new_tokens"] == 64
def _vision_backend(monkeypatch, window = _WINDOW):
try:
from core.inference.inference import InferenceBackend
except (ImportError, RuntimeError) as exc: # pragma: no cover - env-dependent
pytest.skip(f"full inference backend unavailable ({type(exc).__name__}: {exc})")
backend = InferenceBackend.__new__(InferenceBackend)
backend.active_model_name = "vision-model"
backend.last_generation_stats = None
backend._generation_lock = threading.Lock()
model = _FakeModel(window)
tokenizer = _FakeTokenizer()
backend.models = {
backend.active_model_name: {
"model": model,
"tokenizer": tokenizer,
"processor": tokenizer,
"is_vision": True,
"chat_turn_end_eos_ids": [2],
}
}
monkeypatch.setattr(
backend, "_make_text_streamer", lambda *a, **k: _EmptyStreamer(), raising = False
)
monkeypatch.setattr(backend, "format_chat_prompt", lambda *a, **k: "PROMPT", raising = False)
return backend, model
def test_the_vision_path_resolves_an_unset_budget(monkeypatch):
"""Without this, a VLM turn reaches generate() with max_new_tokens=None, which
transformers reads as its own tiny max_length default rather than no limit."""
backend, model = _vision_backend(monkeypatch)
list(
backend._generate_vision_response(
[{"role": "user", "content": "hi"}],
"",
None,
0.0,
1.0,
0,
0.0,
None,
1.0,
)
)
assert model.calls, "generate was never reached"
assert model.calls[0]["max_new_tokens"] == _WINDOW - _PROMPT_LEN
def _audio_backend(monkeypatch, window = _WINDOW):
try:
from core.inference.inference import InferenceBackend
except (ImportError, RuntimeError) as exc: # pragma: no cover - env-dependent
pytest.skip(f"full inference backend unavailable ({type(exc).__name__}: {exc})")
backend = InferenceBackend.__new__(InferenceBackend)
backend.active_model_name = "audio-model"
backend.last_generation_stats = None
backend._generation_lock = threading.Lock()
model = _FakeModel(window)
class _AudioProcessor(_FakeTokenizer):
def apply_chat_template(self, *_a, **_k):
return _FakeEncoding(input_ids = _FakeTensor(_PROMPT_LEN))
processor = _AudioProcessor()
backend.models = {
backend.active_model_name: {
"model": model,
"tokenizer": processor,
"processor": processor,
"audio_type": "audio_vlm",
"chat_turn_end_eos_ids": [2],
}
}
monkeypatch.setattr(
backend, "_make_text_streamer", lambda *a, **k: _EmptyStreamer(), raising = False
)
return backend, model
def _run_audio(backend, max_new_tokens):
list(
backend.generate_audio_input_response(
[{"role": "user", "content": "hi"}],
"",
object(),
0.0,
1.0,
0,
0.0,
max_new_tokens,
1.0,
)
)
def test_the_audio_input_path_resolves_an_unset_budget(monkeypatch):
"""The OpenAI route sends no cap when the client sent none, on the audio branch as on
the text one. Reaching generate() with None would take transformers' own tiny default."""
backend, model = _audio_backend(monkeypatch)
_run_audio(backend, None)
assert model.calls, "generate was never reached"
assert model.calls[0]["max_new_tokens"] == _WINDOW - _PROMPT_LEN
def test_the_audio_input_path_stops_when_cancelled(monkeypatch):
import threading
backend, model = _audio_backend(monkeypatch)
cancel = threading.Event()
list(
backend.generate_audio_input_response(
[{"role": "user", "content": "hi"}],
"",
object(),
0.0,
1.0,
0,
0.0,
8,
1.0,
cancel_event = cancel,
)
)
# The stream's own finally sets the event, so clear it to see the criteria follow it.
criteria = model.calls[0]["stopping_criteria"]
cancel.clear()
assert not any(c(None, None) for c in criteria)
cancel.set()
assert any(c(None, None) for c in criteria)
def _rendered_audio_messages(
monkeypatch,
messages,
audio = None,
):
backend, _model = _audio_backend(monkeypatch)
processor = backend.models[backend.active_model_name]["processor"]
render = processor.apply_chat_template
seen = {}
def _capture(messages, *args, **kwargs):
seen["messages"] = messages
return render(messages, *args, **kwargs)
monkeypatch.setattr(processor, "apply_chat_template", _capture, raising = False)
list(
backend.generate_audio_input_response(
messages, "be brief", audio or object(), 0.0, 1.0, 0, 0.0, 8, 1.0
)
)
return seen["messages"]
def _audio_turns(monkeypatch, messages):
return [(m["role"], m.get("name")) for m in _rendered_audio_messages(monkeypatch, messages)]
def test_the_audio_input_turns_keep_participant_names(monkeypatch):
assert _audio_turns(monkeypatch, [{"role": "user", "name": "alice", "content": "hi"}]) == [
("system", None),
("user", "alice"),
]
def test_the_audio_turn_keeps_its_text_and_ends_the_prompt(monkeypatch):
audio = object()
rendered = _rendered_audio_messages(
monkeypatch,
[
{"role": "user", "content": "Summarize this"},
{"role": "assistant", "content": "partial"},
],
audio,
)
assert rendered[1:] == [
{
"role": "user",
"content": [
{"type": "audio", "audio": audio},
{"type": "text", "text": "Summarize this"},
],
}
]
def test_the_audio_turn_is_named_for_whoever_recorded_it(monkeypatch):
assert _audio_turns(
monkeypatch,
[
{"role": "user", "name": "alice", "content": "hi"},
{"role": "assistant", "content": "hello"},
{"role": "user", "name": "bob", "content": ""},
],
) == [("system", None), ("user", "alice"), ("assistant", None), ("user", "bob")]
def test_an_audio_only_turn_keeps_the_conversation_before_it(monkeypatch):
audio = object()
rendered = _rendered_audio_messages(
monkeypatch,
[
{"role": "user", "content": "Write a haiku about the sea"},
{"role": "assistant", "content": "Waves fold into foam"},
{"role": "user", "content": ""},
],
audio,
)
assert rendered[1:] == [
{"role": "user", "content": [{"type": "text", "text": "Write a haiku about the sea"}]},
{"role": "assistant", "content": [{"type": "text", "text": "Waves fold into foam"}]},
{
"role": "user",
"content": [
{"type": "audio", "audio": audio},
{"type": "text", "text": "Please transcribe this audio."},
],
},
]
def test_an_earlier_audio_turn_keeps_its_reply(monkeypatch):
audio = object()
rendered = _rendered_audio_messages(
monkeypatch,
[
{"role": "user", "content": "Write a haiku about the sea"},
{"role": "assistant", "content": "Waves fold into foam"},
{"role": "user", "content": ""},
{"role": "assistant", "content": "Please count from one to five."},
{"role": "user", "content": ""},
],
audio,
)
assert rendered[1:] == [
{"role": "user", "content": [{"type": "text", "text": "Write a haiku about the sea"}]},
{"role": "assistant", "content": [{"type": "text", "text": "Waves fold into foam"}]},
{"role": "user", "content": []},
{
"role": "assistant",
"content": [{"type": "text", "text": "Please count from one to five."}],
},
{
"role": "user",
"content": [
{"type": "audio", "audio": audio},
{"type": "text", "text": "Please transcribe this audio."},
],
},
]
@pytest.mark.parametrize(
"history, roles",
[
([{"role": "user", "content": "My name is Nilay"}], ["system", "user"]),
([{"role": "assistant", "content": "Hi, how can I help?"}], ["system", "user"]),
(
[{"role": "user", "content": "My name is Nilay"}, {"role": "assistant", "content": ""}],
["system", "user"],
),
(
[
{"role": "user", "content": "Weather in Paris?"},
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "c1",
"type": "function",
"function": {"name": "weather", "arguments": "{}"},
}
],
},
{"role": "tool", "tool_call_id": "c1", "content": "sunny"},
{"role": "assistant", "content": "It is sunny."},
],
["system", "user", "assistant", "user"],
),
],
)
def test_an_audio_turn_after_an_unanswered_or_tool_turn_still_alternates(
monkeypatch, history, roles
):
audio = object()
rendered = _rendered_audio_messages(
monkeypatch, [*history, {"role": "user", "content": ""}], audio
)
assert [m["role"] for m in rendered] == roles
assert rendered[-1]["content"] == [
{"type": "audio", "audio": audio},
{"type": "text", "text": "Please transcribe this audio."},
]
def test_an_explicit_audio_budget_is_untouched(monkeypatch):
backend, model = _audio_backend(monkeypatch)
_run_audio(backend, 128)
assert model.calls[0]["max_new_tokens"] == 128