Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
import asyncio
import json
import sys
from pathlib import Path
from types import SimpleNamespace
import pytest
sys.path.insert(0, str(Path(__file__).parent))
import test_sf_client_tools_passthrough as route_test
from core.inference.engine_adapters import ADAPTERS, tool_parser_for_template
from core.inference.managed_engine import validate_model
from routes import inference as api
@pytest.mark.parametrize("engine,expected", [("vllm", "hermes"), ("sglang", "qwen")])
def test_tool_parser_uses_template_not_model_family(tmp_path, engine, expected):
(tmp_path / "config.json").write_text(json.dumps({"model_type": "unrelated_finetune"}))
(tmp_path / "tokenizer_config.json").write_text(
json.dumps(
{"chat_template": '{% if tools %}<tool_call>{"name": "x", "arguments": {}}{% endif %}'}
)
)
config = SimpleNamespace(is_local = True, path = str(tmp_path))
options = validate_model(config, engine = engine)
assert options["tool_parser"] == expected
args = ADAPTERS[engine].command("python", "model", 40000, "key", 4096, 0.5, options = options)
assert args[args.index("--tool-call-parser") + 1] == expected
assert ("--enable-auto-tool-choice" in args) == (engine == "vllm")
(tmp_path / "chat_template.jinja").write_text("{{ tools }}<tool_call><function=example>")
assert validate_model(config, engine = engine)["tool_parser"] == "qwen3_coder"
(tmp_path / "chat_template.jinja").write_text("{{ messages }}")
assert validate_model(config, engine = engine)["tool_parser"] is None
def test_named_tool_template_and_unknown_syntax(tmp_path):
(tmp_path / "config.json").write_text("{}")
(tmp_path / "tokenizer_config.json").write_text(
json.dumps(
{
"chat_template": [
{"name": "default", "template": "{{ messages }}"},
{"name": "tool_use", "template": "{{ tools }}[TOOL_CALLS]"},
]
}
)
)
assert (
validate_model(SimpleNamespace(is_local = True, path = str(tmp_path)))["tool_parser"]
== "mistral"
)
assert tool_parser_for_template("{{ tools }}CUSTOM", "vllm") is None
@pytest.fixture
def native(monkeypatch):
from routes import managed_engine_chat
# Use the route's own httpx: another test module re-imports httpx at collection.
httpx = managed_engine_chat.httpx
backend = route_test._ScriptedBackend(route_test._fixed("plain"))
backend.models["sf-model"].update(engine = "vllm", supports_tools = True)
from core.inference.managed_engine import ManagedEngine
real = ManagedEngine("vllm")
backend._managed_engine = SimpleNamespace(
model = "sf-model",
context = 4096,
base_url = "http://native.test",
headers = {"Authorization": "Bearer private"},
engine = "vllm",
reasoning_parser = None,
reasoning_fields = real.reasoning_fields,
replay_reasoning = real.replay_reasoning,
)
route_test._install(monkeypatch, backend)
requests = []
call = {
"id": "native-call",
"type": "function",
"function": {"name": "lookup", "arguments": '{"q":"cats"}'},
}
async def handle(request):
body = json.loads(request.content)
requests.append(body)
assert request.headers["Authorization"] == "Bearer private"
name = body.get("tools", [{}])[0].get("function", {}).get("name", "lookup")
tc = {
**call,
"function": {
"name": name,
"arguments": '{"code":"print(3973)"}' if name == "python" else '{"q":"cats"}',
},
}
finished = (
any(m.get("role") == "tool" for m in body["messages"])
or body.get("tool_choice") == "none"
)
message = {"role": "assistant", "content": "3973" if finished else None}
if not finished:
message["tool_calls"] = [tc]
finish = "stop" if finished else "tool_calls"
usage = {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13}
if not body.get("stream"):
return httpx.Response(
200,
json = {
"id": "native-id",
"object": "chat.completion",
"model": "sf-model",
"choices": [{"index": 0, "message": message, "finish_reason": finish}],
"usage": usage,
},
)
chunks = [
{
"id": "native-id",
"model": "sf-model",
"choices": [
{"index": 0, "delta": {k: v for k, v in message.items() if k != "tool_calls"}}
],
}
]
if not finished:
chunks += [
{
"id": "native-id",
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{
**tc,
"index": 0,
"function": {
"name": name,
"arguments": tc["function"]["arguments"][:6],
},
}
]
},
}
],
},
{
"id": "native-id",
"choices": [
{
"index": 0,
"delta": {
"tool_calls": [
{
"index": 0,
"function": {"arguments": tc["function"]["arguments"][6:]},
}
]
},
}
],
},
]
chunks += [
{"id": "native-id", "choices": [{"index": 0, "delta": {}, "finish_reason": finish}]},
{"id": "native-id", "choices": [], "usage": usage},
]
return httpx.Response(
200,
text = "".join("data: " + json.dumps(e) + "\n\n" for e in chunks) + "data: [DONE]\n\n",
)
client = httpx.AsyncClient
monkeypatch.setattr(
httpx,
"AsyncClient",
lambda **kwargs: client(transport = httpx.MockTransport(handle), **kwargs),
)
return backend, requests
def run(payload):
async def request():
response = await api.openai_chat_completions(payload, route_test._Request(), "u")
if payload.stream:
return [
json.loads(line[5:])
async for line in response.body_iterator
if line.startswith("data:") and line[5:].strip() != "[DONE]"
]
return json.loads(response.body)
return asyncio.run(request())
@pytest.mark.parametrize("stream", [False, True])
def test_client_tools_and_history_reach_native_engine(native, stream):
backend, requests = native
result = run(
route_test._request(tools = [route_test.LOOKUP_TOOL], stream = stream, enable_tools = False)
)
assert requests[0]["tools"] == [route_test.LOOKUP_TOOL]
assert "enable_tools" not in requests[0]
if stream:
calls = [
tc
for e in result
for choice in e.get("choices", [])
for tc in choice.get("delta", {}).get("tool_calls", [])
]
assert calls[0]["id"] == "native-call"
assert "".join(tc["function"].get("arguments", "") for tc in calls) == '{"q":"cats"}'
else:
assert result["id"] == "native-id"
assert result["choices"][0]["finish_reason"] == "tool_calls"
history = [
{
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "native-call",
"type": "function",
"function": {"name": "lookup", "arguments": '{"q":"cats"}'},
}
],
},
{"role": "tool", "tool_call_id": "native-call", "content": "3973"},
]
result = run(
route_test._request(
messages = history, tools = [route_test.LOOKUP_TOOL], tool_choice = "none", enable_tools = False
)
)
assert result["choices"][0]["message"]["content"] == "3973"
assert requests[-1]["messages"][-1]["tool_call_id"] == "native-call"
assert requests[-1]["tool_choice"] == "none"
assert api.active_generations.count() == 0
@pytest.mark.parametrize("stream", [False, True])
def test_a_tool_request_cap_is_fitted_to_the_window(native, monkeypatch, stream):
from core.inference import managed_engine
backend, requests = native
counted = []
class _Client:
def __init__(self, **kwargs):
pass
def __enter__(self):
return self
def __exit__(self, *exc):
return False
def post(
self,
url,
headers = None,
json = None,
):
counted.append(json)
return SimpleNamespace(raise_for_status = lambda: None, json = lambda: {"count": 3000})
monkeypatch.setattr(managed_engine.httpx, "Client", _Client)
engine = managed_engine.ManagedEngine("vllm")
engine.model, engine.context, engine.key = "sf-model", 4096, "private"
engine.base_url = "http://native.test"
backend._managed_engine = engine
request = dict(tools = [route_test.LOOKUP_TOOL], stream = stream, enable_tools = False)
run(route_test._request(max_tokens = 2000, **request))
assert requests[-1]["max_tokens"] == 1096
assert counted[-1]["tools"] == [route_test.LOOKUP_TOOL]
run(route_test._request(max_tokens = 64, **request))
assert requests[-1]["max_tokens"] == 64
counts = len(counted)
backend.models["sf-model"]["is_vision"] = True
image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}
parts = [{"type": "text", "text": "hi"}, image]
looking = [route_test.ChatMessage(role = "user", content = parts)]
run(route_test._request(max_tokens = 2000, messages = looking, **request))
assert requests[-1]["max_tokens"] == 2000 and len(counted) == counts
def test_studio_loop_executes_and_replays_native_tool_calls(native, monkeypatch):
from core.inference import studio_tool_loop
executed = []
def execute(name, arguments, **kwargs):
executed.append((name, arguments))
return "3973"
monkeypatch.setattr(studio_tool_loop, "execute_tool", execute)
result = run(
route_test._request(
enable_tools = True,
enabled_tools = ["python"],
permission_mode = "off",
max_tool_calls_per_message = 1,
)
)
assert executed == [("python", {"code": "print(3973)"})]
assert result["choices"][0]["message"]["content"] == "3973"
assert result["usage"]["prompt_tokens"] == 20
assert result["usage"]["context_tokens"] == 13
assert "tool_calls" not in result["choices"][0]["message"]
assert any(
m.get("role") == "tool" and m.get("content") == "3973" for m in native[1][-1]["messages"]
)
assert api.active_generations.count() == 0
def test_run_automatically_streams_arm_the_no_sandbox_gate(native, monkeypatch):
from core.inference import studio_tool_loop
from routes import managed_engine_chat
seen = []
original = managed_engine_chat.stream_with_studio_tools
def capture(*args, **kwargs):
seen.append(kwargs["policy"].confirm_calls)
return original(*args, **kwargs)
monkeypatch.setattr(managed_engine_chat, "stream_with_studio_tools", capture)
monkeypatch.setattr(studio_tool_loop, "execute_tool", lambda *a, **k: "3973")
run(
route_test._request(
enable_tools = True,
enabled_tools = ["python"],
permission_mode = "off",
max_tool_calls_per_message = 1,
stream = True,
)
)
assert seen == [True]
@pytest.mark.parametrize("dedup", [None, False, True])
def test_deduplicate_tool_calls_reaches_the_managed_loop(native, monkeypatch, dedup):
from core.inference import studio_tool_loop
from routes import managed_engine_chat
seen = []
original = managed_engine_chat.stream_with_studio_tools
def capture(*args, **kwargs):
seen.append(kwargs["policy"].deduplicate_tool_calls is not False)
return original(*args, **kwargs)
monkeypatch.setattr(managed_engine_chat, "stream_with_studio_tools", capture)
monkeypatch.setattr(studio_tool_loop, "execute_tool", lambda *a, **k: "3973")
extra = {} if dedup is None else {"deduplicate_tool_calls": dedup}
run(
route_test._request(
enable_tools = True,
enabled_tools = ["python"],
permission_mode = "off",
max_tool_calls_per_message = 1,
**extra,
)
)
assert seen == [dedup is not False]
@pytest.mark.parametrize(
"policy", [None, {"allowedDomains": ["docs.python.org"], "blockedDomains": []}]
)
def test_web_search_domain_limits_reach_the_managed_loop(native, monkeypatch, policy):
from core.inference import studio_tool_loop
from routes import managed_engine_chat
seen = []
original = managed_engine_chat.stream_with_studio_tools
def capture(*args, **kwargs):
seen.append(kwargs["policy"].website_policy)
return original(*args, **kwargs)
monkeypatch.setattr(managed_engine_chat, "stream_with_studio_tools", capture)
monkeypatch.setattr(studio_tool_loop, "execute_tool", lambda *a, **k: "ok")
payload = route_test._request(
enable_tools = True,
enabled_tools = ["web_search"],
permission_mode = "off",
max_tool_calls_per_message = 1,
)
payload._website_policy = policy
run(payload)
assert seen == [policy]
@pytest.mark.parametrize(
"choice", ["auto", "required", "none", {"type": "function", "function": {"name": "lookup"}}]
)
def test_client_tool_choice_is_forwarded(native, choice):
run(route_test._request(tools = [route_test.LOOKUP_TOOL], tool_choice = choice, enable_tools = False))
assert native[1][0]["tool_choice"] == choice
@pytest.mark.parametrize(
"kwargs,match",
[
(
{
"tools": [route_test.LOOKUP_TOOL],
"tool_choice": {"type": "function", "function": {"name": "missing"}},
},
"not enabled",
),
(
{"enable_tools": True, "enabled_tools": ["python"], "permission_mode": "confirm"},
"confirmation",
),
],
)
def test_rejection_does_not_start_native_request(native, kwargs, match):
from fastapi import HTTPException
with pytest.raises(HTTPException) as error:
run(route_test._request(**kwargs))
assert match in str(error.value.detail).lower()
assert not native[1]
assert api.active_generations.count() == 0
def test_unknown_template_rejects_before_native_request(native):
from fastapi import HTTPException
native[0].models["sf-model"]["supports_tools"] = False
with pytest.raises(HTTPException) as error:
run(route_test._request(tools = [route_test.LOOKUP_TOOL]))
assert "chat template" in str(error.value.detail)
assert not native[1]
def test_unstarted_stream_releases_tracking(native):
async def request():
response = await api.openai_chat_completions(
route_test._request(tools = [route_test.LOOKUP_TOOL], stream = True),
route_test._Request(),
"u",
)
assert api.active_generations.count() == 1
await response._unstarted_cleanup()
assert api.active_generations.count() == 0
asyncio.run(request())
assert not native[1]
def test_stream_close_releases_tracking(native):
async def request():
response = await api.openai_chat_completions(
route_test._request(tools = [route_test.LOOKUP_TOOL], stream = True),
route_test._Request(),
"u",
)
await anext(response.body_iterator)
await response.body_iterator.aclose()
assert api.active_generations.count() == 0
asyncio.run(request())
def test_native_rejection_keeps_client_error_status(native):
from core.inference.engine_transport import EngineHTTPError
error = EngineHTTPError(400, '{"error":{"message":"Maximum context length exceeded"}}')
chunk = api._openai_stream_error_chunk(error)
assert chunk["error"]["code"] == "context_length_exceeded"
assert chunk["error"]["type"] == "invalid_request_error"
def test_plain_chat_native_rejection_returns_400(native):
from core.inference.engine_transport import EngineHTTPError
from fastapi import HTTPException
def reject(messages, tools):
raise EngineHTTPError(400, '{"error":{"message":"Maximum context length exceeded"}}')
native[0]._responder = reject
with pytest.raises(HTTPException) as error:
run(route_test._request(enable_tools = False))
assert error.value.status_code == 400
assert error.value.detail["error"]["code"] == "context_length_exceeded"
assert api.active_generations.count() == 0
def test_orchestrator_preserves_native_http_error():
from core.inference.engine_transport import EngineHTTPError
from core.inference.orchestrator import InferenceOrchestrator
def reject(**kwargs):
raise EngineHTTPError(400, "Maximum context length exceeded")
backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
backend._managed_engine = SimpleNamespace(generate = reject)
with pytest.raises(EngineHTTPError):
list(backend._generate_inner(messages = []))
@pytest.mark.parametrize("started", [False, True])
def test_monitor_failure_cannot_leak_active_request(native, monkeypatch, started):
def fail(*args, **kwargs):
raise RuntimeError("monitor unavailable")
async def request():
response = await api.openai_chat_completions(
route_test._request(tools = [route_test.LOOKUP_TOOL], stream = True),
route_test._Request(),
"u",
)
if started:
await anext(response.body_iterator)
monkeypatch.setattr(api.api_monitor, "finish", fail)
with pytest.raises(RuntimeError, match = "monitor unavailable"):
if started:
await response.body_iterator.aclose()
else:
await response._unstarted_cleanup()
assert api.active_generations.count() == 0
asyncio.run(request())
@pytest.mark.parametrize("vision", [False, True])
def test_managed_tool_history_promotes_images_for_vision_only(native, vision):
import base64
import io
from PIL import Image
from core.inference.mcp_images import SENTINEL
image = io.BytesIO()
Image.new("RGB", (8, 8), "red").save(image, format = "PNG")
envelope = (
"Screenshot\n"
+ SENTINEL
+ json.dumps(
[{"data": base64.b64encode(image.getvalue()).decode(), "mimeType": "image/png"}]
)
)
native[0].models["sf-model"]["is_vision"] = vision
payload = route_test._request(enable_tools = False, tools = [route_test.LOOKUP_TOOL])
payload.messages = [
route_test.ChatMessage(role = "user", content = "Look"),
route_test.ChatMessage(
role = "assistant",
content = "",
tool_calls = [
{
"id": "prior",
"type": "function",
"function": {"name": "mcp__test__image", "arguments": "{}"},
}
],
),
route_test.ChatMessage(
role = "tool", tool_call_id = "prior", name = "mcp__test__image", content = envelope
),
route_test.ChatMessage(role = "user", content = "Describe it"),
]
run(payload)
sent = native[1][0]["messages"]
assert SENTINEL not in json.dumps(sent)
parts = [
part
for message in sent
if isinstance(message.get("content"), list)
for part in message["content"]
]
assert any(part.get("type") == "image_url" for part in parts) is vision
def test_managed_route_never_forwards_a_bare_image_path(native):
native[0].models["sf-model"]["is_vision"] = True
payload = route_test._request(enable_tools = False, tools = [route_test.LOOKUP_TOOL])
path = "/" * 64 + "etc/hostname"
payload.messages = [
route_test.ChatMessage(
role = "user",
content = [
{"type": "text", "text": "Describe"},
{"type": "image_url", "image_url": {"url": path}},
],
)
]
run(payload)
urls = [
part["image_url"]["url"]
for body in native[1]
for message in body["messages"]
if isinstance(message.get("content"), list)
for part in message["content"]
if part.get("type") == "image_url"
]
assert urls and all(url.startswith("data:") for url in urls)
def test_managed_engine_serves_multiple_choices_one_at_a_time():
# A managed engine yields plain text, not the (row, text) events batching expects.
from core.inference.orchestrator import InferenceOrchestrator
backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
backend.models = {"m": {"parallel_slots": 4}}
backend.active_model_name = "m"
backend._managed_engine = None
assert backend.effective_parallel_slots == 4
backend._managed_engine = SimpleNamespace(model = "m")
assert backend.effective_parallel_slots == 1
def test_engine_left_by_a_cancelled_load_is_reaped_once_it_dies():
import threading
from core.inference.orchestrator import InferenceOrchestrator
backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
backend._subprocess_shutdown_lock = threading.RLock()
backend._managed_engine = SimpleNamespace(alive = lambda: False)
backend.active_model_name = None
reaped = []
backend._shutdown_subprocess = lambda *a, **k: reaped.append(True)
backend.loading_models = {"m"}
backend.reap_dead_managed_engine()
assert reaped == []
backend.loading_models = set()
def shutdown(*a, **k):
reaped.append(True)
backend._managed_engine = None
backend._shutdown_subprocess = shutdown
assert backend.reap_dead_managed_engine() is True
assert reaped == [True]
def test_reaping_a_crashed_engine_drops_its_claim_and_sharers(monkeypatch):
from core.inference import gpu_arbiter
from routes import inference as api
calls = []
monkeypatch.setattr(api, "release_chat_gpu_claim", lambda: calls.append("release") or True)
monkeypatch.setattr(api.account_access, "clear_resident", calls.append)
api.reap_dead_managed_engine(SimpleNamespace(reap_dead_managed_engine = lambda: True))
assert calls == ["release", "chat"]
calls.clear()
api.reap_dead_managed_engine(SimpleNamespace(reap_dead_managed_engine = lambda: False))
assert calls == []
monkeypatch.setattr(api, "release_chat_gpu_claim", lambda: False)
monkeypatch.setattr(gpu_arbiter, "current_owner", lambda: gpu_arbiter.CHAT)
api.reap_dead_managed_engine(SimpleNamespace(reap_dead_managed_engine = lambda: True))
assert calls == []
@pytest.mark.parametrize("stream", [False, True])
def test_tool_responses_report_the_public_model_id(native, stream):
backend, requests = native
backend._openai_advertised_id = "org/public"
result = run(
route_test._request(tools = [route_test.LOOKUP_TOOL], stream = stream, enable_tools = False)
)
assert requests[0]["model"] == "sf-model"
events = result if stream else [result]
models = {e["model"] for e in events if "model" in e}
assert models == {"org/public"}
def test_a_catalog_with_tool_choice_none_is_plain_chat(native):
backend, requests = native
backend.models["sf-model"]["supports_tools"] = False
result = run(
route_test._request(tools = [route_test.LOOKUP_TOOL], tool_choice = "none", enable_tools = False)
)
assert result["choices"][0]["message"]["content"]
assert requests == []
@pytest.mark.parametrize("tools", [False, True])
def test_managed_messages_get_no_unrequested_date(native, monkeypatch, tools):
from core.inference import orchestrator
backend, requests = native
monkeypatch.setattr(orchestrator, "peek_inference_backend", lambda: backend)
monkeypatch.setattr(api, "_date_gate_blocks", lambda *a: False)
monkeypatch.setattr(
api, "current_date_prompt_line", lambda **_k: "The current date is 2026-08-15."
)
seen = []
plain = backend._responder
def responder(messages, tools):
seen.append(messages)
return plain(messages, tools)
backend._responder = responder
run(
route_test._request(
enable_tools = False, **({"tools": [route_test.LOOKUP_TOOL]} if tools else {})
)
)
if tools:
sent = requests[-1]["messages"]
else:
sent = [*seen[-1], {"role": "system", "content": backend.calls[-1]["system_prompt"]}]
# vLLM/SGLang render a template Studio never sees, so no system turn is made up.
assert "2026-08-15" not in json.dumps(sent)
@pytest.mark.parametrize("vision", [False, True])
def test_managed_engine_offers_image_viewer_to_vision_models_only(native, monkeypatch, vision):
from core.inference import studio_tool_loop
native[0].models["sf-model"]["is_vision"] = vision
monkeypatch.setattr(studio_tool_loop, "execute_tool", lambda *a, **kw: "3973")
run(
route_test._request(
enable_tools = True,
enabled_tools = ["python", "view_image"],
permission_mode = "off",
max_tool_calls_per_message = 1,
)
)
names = [t["function"]["name"] for t in native[1][0]["tools"]]
assert "python" in names
assert ("view_image" in names) == vision