Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

26.0 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

import asyncio
import json
import sys
from pathlib import Path
from types import SimpleNamespace

import pytest

sys.path.insert(0, str(Path(__file__).parent))
import test_sf_client_tools_passthrough as route_test
from core.inference.engine_adapters import ADAPTERS, tool_parser_for_template
from core.inference.managed_engine import validate_model
from routes import inference as api


@pytest.mark.parametrize("engine,expected", [("vllm", "hermes"), ("sglang", "qwen")])
def test_tool_parser_uses_template_not_model_family(tmp_path, engine, expected):
    (tmp_path / "config.json").write_text(json.dumps({"model_type": "unrelated_finetune"}))
    (tmp_path / "tokenizer_config.json").write_text(
        json.dumps(
            {"chat_template": '{% if tools %}<tool_call>{"name": "x", "arguments": {}}{% endif %}'}
        )
    )
    config = SimpleNamespace(is_local = True, path = str(tmp_path))
    options = validate_model(config, engine = engine)
    assert options["tool_parser"] == expected
    args = ADAPTERS[engine].command("python", "model", 40000, "key", 4096, 0.5, options = options)
    assert args[args.index("--tool-call-parser") + 1] == expected
    assert ("--enable-auto-tool-choice" in args) == (engine == "vllm")
    (tmp_path / "chat_template.jinja").write_text("{{ tools }}<tool_call><function=example>")
    assert validate_model(config, engine = engine)["tool_parser"] == "qwen3_coder"
    (tmp_path / "chat_template.jinja").write_text("{{ messages }}")
    assert validate_model(config, engine = engine)["tool_parser"] is None


def test_named_tool_template_and_unknown_syntax(tmp_path):
    (tmp_path / "config.json").write_text("{}")
    (tmp_path / "tokenizer_config.json").write_text(
        json.dumps(
            {
                "chat_template": [
                    {"name": "default", "template": "{{ messages }}"},
                    {"name": "tool_use", "template": "{{ tools }}[TOOL_CALLS]"},
                ]
            }
        )
    )
    assert (
        validate_model(SimpleNamespace(is_local = True, path = str(tmp_path)))["tool_parser"]
        == "mistral"
    )
    assert tool_parser_for_template("{{ tools }}CUSTOM", "vllm") is None


@pytest.fixture
def native(monkeypatch):
    from routes import managed_engine_chat

    # Use the route's own httpx: another test module re-imports httpx at collection.
    httpx = managed_engine_chat.httpx
    backend = route_test._ScriptedBackend(route_test._fixed("plain"))
    backend.models["sf-model"].update(engine = "vllm", supports_tools = True)
    from core.inference.managed_engine import ManagedEngine

    real = ManagedEngine("vllm")
    backend._managed_engine = SimpleNamespace(
        model = "sf-model",
        context = 4096,
        base_url = "http://native.test",
        headers = {"Authorization": "Bearer private"},
        engine = "vllm",
        reasoning_parser = None,
        reasoning_fields = real.reasoning_fields,
        replay_reasoning = real.replay_reasoning,
    )
    route_test._install(monkeypatch, backend)
    requests = []
    call = {
        "id": "native-call",
        "type": "function",
        "function": {"name": "lookup", "arguments": '{"q":"cats"}'},
    }

    async def handle(request):
        body = json.loads(request.content)
        requests.append(body)
        assert request.headers["Authorization"] == "Bearer private"
        name = body.get("tools", [{}])[0].get("function", {}).get("name", "lookup")
        tc = {
            **call,
            "function": {
                "name": name,
                "arguments": '{"code":"print(3973)"}' if name == "python" else '{"q":"cats"}',
            },
        }
        finished = (
            any(m.get("role") == "tool" for m in body["messages"])
            or body.get("tool_choice") == "none"
        )
        message = {"role": "assistant", "content": "3973" if finished else None}
        if not finished:
            message["tool_calls"] = [tc]
        finish = "stop" if finished else "tool_calls"
        usage = {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13}
        if not body.get("stream"):
            return httpx.Response(
                200,
                json = {
                    "id": "native-id",
                    "object": "chat.completion",
                    "model": "sf-model",
                    "choices": [{"index": 0, "message": message, "finish_reason": finish}],
                    "usage": usage,
                },
            )
        chunks = [
            {
                "id": "native-id",
                "model": "sf-model",
                "choices": [
                    {"index": 0, "delta": {k: v for k, v in message.items() if k != "tool_calls"}}
                ],
            }
        ]
        if not finished:
            chunks += [
                {
                    "id": "native-id",
                    "choices": [
                        {
                            "index": 0,
                            "delta": {
                                "tool_calls": [
                                    {
                                        **tc,
                                        "index": 0,
                                        "function": {
                                            "name": name,
                                            "arguments": tc["function"]["arguments"][:6],
                                        },
                                    }
                                ]
                            },
                        }
                    ],
                },
                {
                    "id": "native-id",
                    "choices": [
                        {
                            "index": 0,
                            "delta": {
                                "tool_calls": [
                                    {
                                        "index": 0,
                                        "function": {"arguments": tc["function"]["arguments"][6:]},
                                    }
                                ]
                            },
                        }
                    ],
                },
            ]
        chunks += [
            {"id": "native-id", "choices": [{"index": 0, "delta": {}, "finish_reason": finish}]},
            {"id": "native-id", "choices": [], "usage": usage},
        ]
        return httpx.Response(
            200,
            text = "".join("data: " + json.dumps(e) + "\n\n" for e in chunks) + "data: [DONE]\n\n",
        )

    client = httpx.AsyncClient
    monkeypatch.setattr(
        httpx,
        "AsyncClient",
        lambda **kwargs: client(transport = httpx.MockTransport(handle), **kwargs),
    )
    return backend, requests


def run(payload):
    async def request():
        response = await api.openai_chat_completions(payload, route_test._Request(), "u")
        if payload.stream:
            return [
                json.loads(line[5:])
                async for line in response.body_iterator
                if line.startswith("data:") and line[5:].strip() != "[DONE]"
            ]
        return json.loads(response.body)

    return asyncio.run(request())


@pytest.mark.parametrize("stream", [False, True])
def test_client_tools_and_history_reach_native_engine(native, stream):
    backend, requests = native
    result = run(
        route_test._request(tools = [route_test.LOOKUP_TOOL], stream = stream, enable_tools = False)
    )
    assert requests[0]["tools"] == [route_test.LOOKUP_TOOL]
    assert "enable_tools" not in requests[0]
    if stream:
        calls = [
            tc
            for e in result
            for choice in e.get("choices", [])
            for tc in choice.get("delta", {}).get("tool_calls", [])
        ]
        assert calls[0]["id"] == "native-call"
        assert "".join(tc["function"].get("arguments", "") for tc in calls) == '{"q":"cats"}'
    else:
        assert result["id"] == "native-id"
        assert result["choices"][0]["finish_reason"] == "tool_calls"
    history = [
        {
            "role": "assistant",
            "content": None,
            "tool_calls": [
                {
                    "id": "native-call",
                    "type": "function",
                    "function": {"name": "lookup", "arguments": '{"q":"cats"}'},
                }
            ],
        },
        {"role": "tool", "tool_call_id": "native-call", "content": "3973"},
    ]
    result = run(
        route_test._request(
            messages = history, tools = [route_test.LOOKUP_TOOL], tool_choice = "none", enable_tools = False
        )
    )
    assert result["choices"][0]["message"]["content"] == "3973"
    assert requests[-1]["messages"][-1]["tool_call_id"] == "native-call"
    assert requests[-1]["tool_choice"] == "none"
    assert api.active_generations.count() == 0


@pytest.mark.parametrize("stream", [False, True])
def test_a_tool_request_cap_is_fitted_to_the_window(native, monkeypatch, stream):
    from core.inference import managed_engine

    backend, requests = native
    counted = []

    class _Client:
        def __init__(self, **kwargs):
            pass

        def __enter__(self):
            return self

        def __exit__(self, *exc):
            return False

        def post(
            self,
            url,
            headers = None,
            json = None,
        ):
            counted.append(json)
            return SimpleNamespace(raise_for_status = lambda: None, json = lambda: {"count": 3000})

    monkeypatch.setattr(managed_engine.httpx, "Client", _Client)
    engine = managed_engine.ManagedEngine("vllm")
    engine.model, engine.context, engine.key = "sf-model", 4096, "private"
    engine.base_url = "http://native.test"
    backend._managed_engine = engine
    request = dict(tools = [route_test.LOOKUP_TOOL], stream = stream, enable_tools = False)
    run(route_test._request(max_tokens = 2000, **request))
    assert requests[-1]["max_tokens"] == 1096
    assert counted[-1]["tools"] == [route_test.LOOKUP_TOOL]
    run(route_test._request(max_tokens = 64, **request))
    assert requests[-1]["max_tokens"] == 64
    counts = len(counted)
    backend.models["sf-model"]["is_vision"] = True
    image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}
    parts = [{"type": "text", "text": "hi"}, image]
    looking = [route_test.ChatMessage(role = "user", content = parts)]
    run(route_test._request(max_tokens = 2000, messages = looking, **request))
    assert requests[-1]["max_tokens"] == 2000 and len(counted) == counts


def test_studio_loop_executes_and_replays_native_tool_calls(native, monkeypatch):
    from core.inference import studio_tool_loop

    executed = []

    def execute(name, arguments, **kwargs):
        executed.append((name, arguments))
        return "3973"

    monkeypatch.setattr(studio_tool_loop, "execute_tool", execute)
    result = run(
        route_test._request(
            enable_tools = True,
            enabled_tools = ["python"],
            permission_mode = "off",
            max_tool_calls_per_message = 1,
        )
    )
    assert executed == [("python", {"code": "print(3973)"})]
    assert result["choices"][0]["message"]["content"] == "3973"
    assert result["usage"]["prompt_tokens"] == 20
    assert result["usage"]["context_tokens"] == 13
    assert "tool_calls" not in result["choices"][0]["message"]
    assert any(
        m.get("role") == "tool" and m.get("content") == "3973" for m in native[1][-1]["messages"]
    )
    assert api.active_generations.count() == 0


def test_run_automatically_streams_arm_the_no_sandbox_gate(native, monkeypatch):
    from core.inference import studio_tool_loop
    from routes import managed_engine_chat

    seen = []
    original = managed_engine_chat.stream_with_studio_tools

    def capture(*args, **kwargs):
        seen.append(kwargs["policy"].confirm_calls)
        return original(*args, **kwargs)

    monkeypatch.setattr(managed_engine_chat, "stream_with_studio_tools", capture)
    monkeypatch.setattr(studio_tool_loop, "execute_tool", lambda *a, **k: "3973")
    run(
        route_test._request(
            enable_tools = True,
            enabled_tools = ["python"],
            permission_mode = "off",
            max_tool_calls_per_message = 1,
            stream = True,
        )
    )
    assert seen == [True]


@pytest.mark.parametrize("dedup", [None, False, True])
def test_deduplicate_tool_calls_reaches_the_managed_loop(native, monkeypatch, dedup):
    from core.inference import studio_tool_loop
    from routes import managed_engine_chat

    seen = []
    original = managed_engine_chat.stream_with_studio_tools

    def capture(*args, **kwargs):
        seen.append(kwargs["policy"].deduplicate_tool_calls is not False)
        return original(*args, **kwargs)

    monkeypatch.setattr(managed_engine_chat, "stream_with_studio_tools", capture)
    monkeypatch.setattr(studio_tool_loop, "execute_tool", lambda *a, **k: "3973")
    extra = {} if dedup is None else {"deduplicate_tool_calls": dedup}
    run(
        route_test._request(
            enable_tools = True,
            enabled_tools = ["python"],
            permission_mode = "off",
            max_tool_calls_per_message = 1,
            **extra,
        )
    )
    assert seen == [dedup is not False]


@pytest.mark.parametrize(
    "policy", [None, {"allowedDomains": ["docs.python.org"], "blockedDomains": []}]
)
def test_web_search_domain_limits_reach_the_managed_loop(native, monkeypatch, policy):
    from core.inference import studio_tool_loop
    from routes import managed_engine_chat

    seen = []
    original = managed_engine_chat.stream_with_studio_tools

    def capture(*args, **kwargs):
        seen.append(kwargs["policy"].website_policy)
        return original(*args, **kwargs)

    monkeypatch.setattr(managed_engine_chat, "stream_with_studio_tools", capture)
    monkeypatch.setattr(studio_tool_loop, "execute_tool", lambda *a, **k: "ok")
    payload = route_test._request(
        enable_tools = True,
        enabled_tools = ["web_search"],
        permission_mode = "off",
        max_tool_calls_per_message = 1,
    )
    payload._website_policy = policy
    run(payload)
    assert seen == [policy]


@pytest.mark.parametrize(
    "choice", ["auto", "required", "none", {"type": "function", "function": {"name": "lookup"}}]
)
def test_client_tool_choice_is_forwarded(native, choice):
    run(route_test._request(tools = [route_test.LOOKUP_TOOL], tool_choice = choice, enable_tools = False))
    assert native[1][0]["tool_choice"] == choice


@pytest.mark.parametrize(
    "kwargs,match",
    [
        (
            {
                "tools": [route_test.LOOKUP_TOOL],
                "tool_choice": {"type": "function", "function": {"name": "missing"}},
            },
            "not enabled",
        ),
        (
            {"enable_tools": True, "enabled_tools": ["python"], "permission_mode": "confirm"},
            "confirmation",
        ),
    ],
)
def test_rejection_does_not_start_native_request(native, kwargs, match):
    from fastapi import HTTPException

    with pytest.raises(HTTPException) as error:
        run(route_test._request(**kwargs))
    assert match in str(error.value.detail).lower()
    assert not native[1]
    assert api.active_generations.count() == 0


def test_unknown_template_rejects_before_native_request(native):
    from fastapi import HTTPException

    native[0].models["sf-model"]["supports_tools"] = False
    with pytest.raises(HTTPException) as error:
        run(route_test._request(tools = [route_test.LOOKUP_TOOL]))
    assert "chat template" in str(error.value.detail)
    assert not native[1]


def test_unstarted_stream_releases_tracking(native):
    async def request():
        response = await api.openai_chat_completions(
            route_test._request(tools = [route_test.LOOKUP_TOOL], stream = True),
            route_test._Request(),
            "u",
        )
        assert api.active_generations.count() == 1
        await response._unstarted_cleanup()
        assert api.active_generations.count() == 0

    asyncio.run(request())
    assert not native[1]


def test_stream_close_releases_tracking(native):
    async def request():
        response = await api.openai_chat_completions(
            route_test._request(tools = [route_test.LOOKUP_TOOL], stream = True),
            route_test._Request(),
            "u",
        )
        await anext(response.body_iterator)
        await response.body_iterator.aclose()
        assert api.active_generations.count() == 0

    asyncio.run(request())


def test_native_rejection_keeps_client_error_status(native):
    from core.inference.engine_transport import EngineHTTPError

    error = EngineHTTPError(400, '{"error":{"message":"Maximum context length exceeded"}}')
    chunk = api._openai_stream_error_chunk(error)
    assert chunk["error"]["code"] == "context_length_exceeded"
    assert chunk["error"]["type"] == "invalid_request_error"


def test_plain_chat_native_rejection_returns_400(native):
    from core.inference.engine_transport import EngineHTTPError
    from fastapi import HTTPException

    def reject(messages, tools):
        raise EngineHTTPError(400, '{"error":{"message":"Maximum context length exceeded"}}')

    native[0]._responder = reject
    with pytest.raises(HTTPException) as error:
        run(route_test._request(enable_tools = False))
    assert error.value.status_code == 400
    assert error.value.detail["error"]["code"] == "context_length_exceeded"
    assert api.active_generations.count() == 0


def test_orchestrator_preserves_native_http_error():
    from core.inference.engine_transport import EngineHTTPError
    from core.inference.orchestrator import InferenceOrchestrator

    def reject(**kwargs):
        raise EngineHTTPError(400, "Maximum context length exceeded")

    backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
    backend._managed_engine = SimpleNamespace(generate = reject)
    with pytest.raises(EngineHTTPError):
        list(backend._generate_inner(messages = []))


@pytest.mark.parametrize("started", [False, True])
def test_monitor_failure_cannot_leak_active_request(native, monkeypatch, started):
    def fail(*args, **kwargs):
        raise RuntimeError("monitor unavailable")

    async def request():
        response = await api.openai_chat_completions(
            route_test._request(tools = [route_test.LOOKUP_TOOL], stream = True),
            route_test._Request(),
            "u",
        )
        if started:
            await anext(response.body_iterator)
        monkeypatch.setattr(api.api_monitor, "finish", fail)
        with pytest.raises(RuntimeError, match = "monitor unavailable"):
            if started:
                await response.body_iterator.aclose()
            else:
                await response._unstarted_cleanup()
        assert api.active_generations.count() == 0

    asyncio.run(request())


@pytest.mark.parametrize("vision", [False, True])
def test_managed_tool_history_promotes_images_for_vision_only(native, vision):
    import base64
    import io
    from PIL import Image
    from core.inference.mcp_images import SENTINEL

    image = io.BytesIO()
    Image.new("RGB", (8, 8), "red").save(image, format = "PNG")
    envelope = (
        "Screenshot\n"
        + SENTINEL
        + json.dumps(
            [{"data": base64.b64encode(image.getvalue()).decode(), "mimeType": "image/png"}]
        )
    )
    native[0].models["sf-model"]["is_vision"] = vision
    payload = route_test._request(enable_tools = False, tools = [route_test.LOOKUP_TOOL])
    payload.messages = [
        route_test.ChatMessage(role = "user", content = "Look"),
        route_test.ChatMessage(
            role = "assistant",
            content = "",
            tool_calls = [
                {
                    "id": "prior",
                    "type": "function",
                    "function": {"name": "mcp__test__image", "arguments": "{}"},
                }
            ],
        ),
        route_test.ChatMessage(
            role = "tool", tool_call_id = "prior", name = "mcp__test__image", content = envelope
        ),
        route_test.ChatMessage(role = "user", content = "Describe it"),
    ]
    run(payload)
    sent = native[1][0]["messages"]
    assert SENTINEL not in json.dumps(sent)
    parts = [
        part
        for message in sent
        if isinstance(message.get("content"), list)
        for part in message["content"]
    ]
    assert any(part.get("type") == "image_url" for part in parts) is vision


def test_managed_route_never_forwards_a_bare_image_path(native):
    native[0].models["sf-model"]["is_vision"] = True
    payload = route_test._request(enable_tools = False, tools = [route_test.LOOKUP_TOOL])
    path = "/" * 64 + "etc/hostname"
    payload.messages = [
        route_test.ChatMessage(
            role = "user",
            content = [
                {"type": "text", "text": "Describe"},
                {"type": "image_url", "image_url": {"url": path}},
            ],
        )
    ]
    run(payload)
    urls = [
        part["image_url"]["url"]
        for body in native[1]
        for message in body["messages"]
        if isinstance(message.get("content"), list)
        for part in message["content"]
        if part.get("type") == "image_url"
    ]
    assert urls and all(url.startswith("data:") for url in urls)


def test_managed_engine_serves_multiple_choices_one_at_a_time():
    # A managed engine yields plain text, not the (row, text) events batching expects.
    from core.inference.orchestrator import InferenceOrchestrator

    backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
    backend.models = {"m": {"parallel_slots": 4}}
    backend.active_model_name = "m"
    backend._managed_engine = None
    assert backend.effective_parallel_slots == 4
    backend._managed_engine = SimpleNamespace(model = "m")
    assert backend.effective_parallel_slots == 1


def test_engine_left_by_a_cancelled_load_is_reaped_once_it_dies():
    import threading

    from core.inference.orchestrator import InferenceOrchestrator

    backend = InferenceOrchestrator.__new__(InferenceOrchestrator)
    backend._subprocess_shutdown_lock = threading.RLock()
    backend._managed_engine = SimpleNamespace(alive = lambda: False)
    backend.active_model_name = None
    reaped = []
    backend._shutdown_subprocess = lambda *a, **k: reaped.append(True)
    backend.loading_models = {"m"}
    backend.reap_dead_managed_engine()
    assert reaped == []
    backend.loading_models = set()

    def shutdown(*a, **k):
        reaped.append(True)
        backend._managed_engine = None

    backend._shutdown_subprocess = shutdown
    assert backend.reap_dead_managed_engine() is True
    assert reaped == [True]


def test_reaping_a_crashed_engine_drops_its_claim_and_sharers(monkeypatch):
    from core.inference import gpu_arbiter
    from routes import inference as api

    calls = []
    monkeypatch.setattr(api, "release_chat_gpu_claim", lambda: calls.append("release") or True)
    monkeypatch.setattr(api.account_access, "clear_resident", calls.append)
    api.reap_dead_managed_engine(SimpleNamespace(reap_dead_managed_engine = lambda: True))
    assert calls == ["release", "chat"]

    calls.clear()
    api.reap_dead_managed_engine(SimpleNamespace(reap_dead_managed_engine = lambda: False))
    assert calls == []

    monkeypatch.setattr(api, "release_chat_gpu_claim", lambda: False)
    monkeypatch.setattr(gpu_arbiter, "current_owner", lambda: gpu_arbiter.CHAT)
    api.reap_dead_managed_engine(SimpleNamespace(reap_dead_managed_engine = lambda: True))
    assert calls == []


@pytest.mark.parametrize("stream", [False, True])
def test_tool_responses_report_the_public_model_id(native, stream):
    backend, requests = native
    backend._openai_advertised_id = "org/public"
    result = run(
        route_test._request(tools = [route_test.LOOKUP_TOOL], stream = stream, enable_tools = False)
    )
    assert requests[0]["model"] == "sf-model"
    events = result if stream else [result]
    models = {e["model"] for e in events if "model" in e}
    assert models == {"org/public"}


def test_a_catalog_with_tool_choice_none_is_plain_chat(native):
    backend, requests = native
    backend.models["sf-model"]["supports_tools"] = False
    result = run(
        route_test._request(tools = [route_test.LOOKUP_TOOL], tool_choice = "none", enable_tools = False)
    )
    assert result["choices"][0]["message"]["content"]
    assert requests == []


@pytest.mark.parametrize("tools", [False, True])
def test_managed_messages_get_no_unrequested_date(native, monkeypatch, tools):
    from core.inference import orchestrator

    backend, requests = native
    monkeypatch.setattr(orchestrator, "peek_inference_backend", lambda: backend)
    monkeypatch.setattr(api, "_date_gate_blocks", lambda *a: False)
    monkeypatch.setattr(
        api, "current_date_prompt_line", lambda **_k: "The current date is 2026-08-15."
    )
    seen = []
    plain = backend._responder

    def responder(messages, tools):
        seen.append(messages)
        return plain(messages, tools)

    backend._responder = responder
    run(
        route_test._request(
            enable_tools = False, **({"tools": [route_test.LOOKUP_TOOL]} if tools else {})
        )
    )
    if tools:
        sent = requests[-1]["messages"]
    else:
        sent = [*seen[-1], {"role": "system", "content": backend.calls[-1]["system_prompt"]}]
    # vLLM/SGLang render a template Studio never sees, so no system turn is made up.
    assert "2026-08-15" not in json.dumps(sent)


@pytest.mark.parametrize("vision", [False, True])
def test_managed_engine_offers_image_viewer_to_vision_models_only(native, monkeypatch, vision):
    from core.inference import studio_tool_loop

    native[0].models["sf-model"]["is_vision"] = vision
    monkeypatch.setattr(studio_tool_loop, "execute_tool", lambda *a, **kw: "3973")
    run(
        route_test._request(
            enable_tools = True,
            enabled_tools = ["python", "view_image"],
            permission_mode = "off",
            max_tool_calls_per_message = 1,
        )
    )
    names = [t["function"]["name"] for t in native[1][0]["tools"]]
    assert "python" in names
    assert ("view_image" in names) == vision