Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import asyncio
import math
import threading
import time
from types import SimpleNamespace
import httpx
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from core.inference.llama_cpp import LlamaCppBackend
from routes import inference, settings
from storage import studio_db
from utils import first_token_timeout as limit
from utils.account_context import OWNER, AccountContext, bind_account, reset_account
@pytest.fixture(autouse = True)
def isolated(monkeypatch, tmp_path):
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
monkeypatch.setattr(studio_db, "_schema_ready", set())
monkeypatch.delenv(inference._OPENAI_COMPAT_FIRST_TOKEN_TIMEOUT_ENV, raising = False)
def client(account = OWNER):
app = FastAPI()
app.include_router(settings.router)
async def subject():
token = bind_account(account)
try:
yield account.username
finally:
reset_account(token)
app.dependency_overrides[settings.get_current_subject] = subject
return TestClient(app)
def test_settings_persist_and_both_inference_paths_use_them():
with client() as api:
assert api.get("/first-token-timeout").json()["minutes"] == 20
for minutes, seconds in [(75, 4500.0), (0, math.inf), (20, 1200.0)]:
assert api.put("/first-token-timeout", json = {"minutes": minutes}).status_code == 200
assert limit.timeout_seconds() == seconds
assert inference._first_token_timeout_s() == seconds
assert inference._llama_streaming_generation_timeout().read == limit.socket_timeout(
seconds
)
assert api.get("/first-token-timeout").json()["minutes"] == 20
@pytest.mark.parametrize("value", [-1, 10081, True, "60", 1.5, None])
def test_invalid_settings_are_rejected(value):
with client() as api:
assert api.put("/first-token-timeout", json = {"minutes": value}).status_code == 422
assert limit.get_minutes() == 20
def test_managed_account_reads_owner_setting_but_cannot_change_it():
limit.set_minutes(90)
with client(AccountContext("a" * 32, "alice")) as api:
assert api.get("/first-token-timeout").json()["minutes"] == 90
assert api.put("/first-token-timeout", json = {"minutes": 0}).status_code == 403
assert limit.get_minutes() == 90
@pytest.mark.parametrize("stored", [-1, 10081, True, "60", 1.5, None, {"minutes": 60}])
def test_unusable_stored_value_falls_back_to_default(stored):
studio_db.upsert_app_settings({limit.SETTING_KEY: stored})
assert limit.get_minutes() == limit.DEFAULT_MINUTES
def test_explicit_api_env_override_is_preserved(monkeypatch):
limit.set_minutes(0)
monkeypatch.setenv(inference._OPENAI_COMPAT_FIRST_TOKEN_TIMEOUT_ENV, "42")
assert inference._first_token_timeout_s() == 42
assert limit.timeout_seconds() == math.inf
assert "UNSLOTH_OPENAI_COMPAT" in inference._friendly_error(httpx.ReadTimeout("x"))
@pytest.mark.parametrize("minutes,read", [(1, 1200.0), (90, 5400.0), (0, None)])
def test_shortened_limit_does_not_cut_non_streamed_replies(minutes, read):
from core.inference.engine_transport import engine_request_timeout
limit.set_minutes(minutes)
assert inference._llama_non_streaming_generation_timeout().read == read
assert engine_request_timeout().read == 1200.0 # Managed engines ignore the GGUF limit.
assert inference._llama_streaming_generation_timeout().read == (minutes * 60 or None)
def test_sync_custom_wait_survives_original_limit_and_then_expires(monkeypatch):
limit.set_minutes(60)
clock = [0.0]
monkeypatch.setattr(limit.time, "monotonic", lambda: clock[0])
class Response:
request = SimpleNamespace(extensions = {"timeout": {"read": 1200}})
def iter_text(self):
for moment in (1300, 3601):
clock[0] = moment
yield 'data: {"choices":[{"delta":{"role":"assistant"}}]}\n\n'
iterator = LlamaCppBackend._iter_text_cancellable(Response())
with limit.track_first_token_wait() as wait:
next(iterator)
assert wait.active() # Past the original 20-minute limit.
next(iterator)
with pytest.raises(httpx.ReadTimeout):
next(iterator)
assert not wait.active()
def test_sync_unlimited_prefill_still_switches_to_decode_stall(monkeypatch):
limit.set_minutes(0)
clock = [0.0]
monkeypatch.setattr(limit.time, "monotonic", lambda: clock[0])
class Response:
request = SimpleNamespace(extensions = {"timeout": {"read": 1200}})
def iter_text(self):
clock[0] = 86400.0 # A day of simulated prefill.
yield 'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
clock[0] += 121
raise httpx.ReadTimeout("silent decode")
response = Response()
with limit.track_first_token_wait() as wait:
iterator = LlamaCppBackend._iter_text_cancellable(response)
assert "hello" in next(iterator)
assert not wait.active()
assert response.request.extensions["timeout"]["read"] == 120
with pytest.raises(httpx.ReadTimeout, match = "stopped producing tokens"):
next(iterator)
def test_windows_bounds_the_header_wait_while_unlimited_body_reads_stay_unbounded(monkeypatch):
import httpcore
from contextlib import contextmanager
limit.set_minutes(0)
attempts = iter([httpcore.ReadTimeout("slice"), httpcore.ReadTimeout("slice"), b"x"])
def read(_max_bytes, timeout = None):
time.sleep(0.002)
if isinstance(item := next(attempts), Exception):
raise item
return item
network = SimpleNamespace(read = read)
pool = SimpleNamespace(
_connections = [SimpleNamespace(_connection = SimpleNamespace(_network_stream = network))]
)
response = SimpleNamespace(request = SimpleNamespace(extensions = {"timeout": {"read": None}}))
seen = {}
@contextmanager
def stream(*_args, timeout, **_kwargs):
seen["read"] = timeout.read
yield response
client = SimpleNamespace(stream = stream, _transport = SimpleNamespace(_pool = pool))
monkeypatch.setattr("core.inference.llama_cpp.sys.platform", "win32")
with LlamaCppBackend._stream_with_retry(
client, "url", {}
): # No cancel event, as the CLI calls it.
assert seen["read"] == 1200.0
assert network.read(1, timeout = 0.001) == b"x"
@pytest.mark.asyncio
async def test_async_pump_arms_saved_limit_until_output_and_stays_cancellable():
limit.set_minutes(60)
cancelled = threading.Event()
async def upstream():
yield b'data: {"choices":[{"delta":{"role":"assistant"}}]}\n\n'
yield b'data: {"prompt_progress":{"processed":8}}\n\n'
yield b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n'
await asyncio.Future()
with limit.track_first_token_wait() as wait:
pump = inference._aiter_llama_stream_items(
upstream(),
cancel_event = cancelled,
track_prefill_progress = True,
keepalive_interval_s = 0.01,
)
await anext(pump)
assert 3500 < wait.deadline - time.monotonic() <= 3600
limit.set_minutes(90) # Advancing prefill re-arms with the limit then saved.
await anext(pump)
assert 5300 < wait.deadline - time.monotonic() <= 5400
assert b"hello" in await anext(pump)
assert not wait.active()
assert await anext(pump) is inference._LLAMA_STREAM_KEEPALIVE
cancelled.set()
with pytest.raises(StopAsyncIteration):
await asyncio.wait_for(anext(pump), 1)