Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

7.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Refuse calls under tool_choice none, and on length or content_filter finishes; stop is allowed."""

from __future__ import annotations

import asyncio
import json
import threading

import pytest

from core.inference import studio_tool_loop as loop_mod
from core.inference.studio_tool_loop import (
    ToolLoopPolicy,
    ToolLoopRun,
    stream_with_studio_tools,
)


_DONE = "data: [DONE]"

WEB = {
    "type": "function",
    "function": {
        "name": "web_search",
        "description": "",
        "parameters": {
            "type": "object",
            "properties": {"query": {"type": "string"}},
            "required": ["query"],
        },
    },
}


def _call_line() -> str:
    return "data: " + json.dumps(
        {
            "choices": [
                {
                    "index": 0,
                    "delta": {
                        "tool_calls": [
                            {
                                "index": 0,
                                "id": "c1",
                                "type": "function",
                                "function": {
                                    "name": "web_search",
                                    "arguments": '{"query": "x"}',
                                },
                            }
                        ]
                    },
                }
            ]
        }
    )


def _finish(reason: str) -> str:
    return "data: " + json.dumps({"choices": [{"index": 0, "delta": {}, "finish_reason": reason}]})


class FakeTransport:
    heals_text_tool_calls = False

    def __init__(
        self,
        turns,
        *,
        max_turns = 20,
    ):
        self.turns = [list(turn) for turn in turns]
        self.requests: list[dict] = []
        self.max_turns = max_turns

    def stream(self, *, messages, tools, tool_choice, cancel_event):
        self.requests.append({"tools": tools, "tool_choice": tool_choice})
        assert len(self.requests) <= self.max_turns, "loop never terminated"
        lines = self.turns.pop(0) if self.turns else [_DONE]

        async def _gen():
            for line in lines:
                yield line

        return _gen()


@pytest.fixture
def executed(monkeypatch):
    calls: list[str] = []

    def _execute(name, arguments, **kwargs):
        calls.append(name)
        return f"RESULT<{name}>"

    monkeypatch.setattr(loop_mod, "execute_tool", _execute)
    monkeypatch.setattr(loop_mod, "build_rag_autoinject", lambda *a, **k: None)
    monkeypatch.setattr(loop_mod, "is_high_risk_tool_call", lambda name, args: False)
    return calls


def _run(transport, *, tool_choice = None):
    async def _collect():
        out: list[str] = []
        agen = stream_with_studio_tools(
            transport,
            run = ToolLoopRun(
                messages = [{"role": "user", "content": "hi"}],
                session_id = "s1",
                thread_id = "t1",
                tool_choice = tool_choice,
            ),
            policy = ToolLoopPolicy(
                tools = [WEB],
                max_calls = 25,
                timeout = 300,
                permission_mode = "off",
                confirm_calls = False,
                bypass_permissions = False,
                rag_scope = None,
            ),
            cancel_event = threading.Event(),
        )
        async for line in agen:
            out.append(line)
        return out

    return asyncio.run(asyncio.wait_for(_collect(), timeout = 30))


def _events(lines, kind):
    events = []
    for line in lines:
        if not line.startswith("data: "):
            continue
        try:
            payload = json.loads(line[6:])
        except ValueError:
            continue
        if isinstance(payload, dict) and payload.get("type") == kind:
            events.append(payload)
    return events


# ── tool_choice: "none" is enforced, not just advertised ─────────────


def test_tool_choice_none_refuses_a_call_the_provider_sent_anyway(executed):
    """Deep Research hops carry scraped text, so a tool_choice none call must never execute."""
    transport = FakeTransport([[_call_line(), _finish("tool_calls")], [_DONE]])
    lines = _run(transport, tool_choice = "none")

    assert executed == []

    starts = _events(lines, "tool_start")
    ends = _events(lines, "tool_end")

    assert [event["tool_call_id"] for event in starts] == ["c1"]
    assert [event["tool_call_id"] for event in ends] == ["c1"]
    assert starts[0]["arguments"] == {"query": "x"}
    assert ends[0]["tool_name"] == "web_search"
    assert ends[0]["result"] == loop_mod._TOOL_CHOICE_NONE


def test_a_truncated_call_under_tool_choice_none_closes_once_as_truncated(executed):
    """A turn that is both keeps the truncation reason and closes its call a single time."""
    transport = FakeTransport([[_call_line(), _finish("length")], [_DONE]])
    lines = _run(transport, tool_choice = "none")

    assert executed == []

    starts = _events(lines, "tool_start")
    ends = _events(lines, "tool_end")
    assert [event["tool_call_id"] for event in starts] == ["c1"]
    assert [event["tool_call_id"] for event in ends] == ["c1"]
    assert starts[0]["arguments"] == {}
    assert ends[0]["result"] == loop_mod._TOOL_TRUNCATED


def test_tool_choice_none_closes_every_refused_card_by_the_id_the_client_drew(executed):
    """Each refused call closes its own card, an id-less one under the minted ``tool_call_<index>``."""
    named = json.loads(_call_line()[6:])
    idless = json.loads(_call_line()[6:])
    idless["choices"][0]["delta"]["tool_calls"][0].update(index = 1)
    del idless["choices"][0]["delta"]["tool_calls"][0]["id"]
    transport = FakeTransport(
        [["data: " + json.dumps(named), "data: " + json.dumps(idless), _DONE], [_DONE]]
    )
    lines = _run(transport, tool_choice = "none")

    assert executed == []
    assert len(transport.requests) == 1

    starts = _events(lines, "tool_start")
    ends = _events(lines, "tool_end")
    assert [event["tool_call_id"] for event in starts] == ["c1", "tool_call_1"]
    assert [event["tool_call_id"] for event in ends] == ["c1", "tool_call_1"]
    assert all(event["arguments"] == {"query": "x"} for event in starts)

    # Closed before the empty badge the client ends the provider turn on.
    frames = [json.loads(line[6:]) for line in lines if line.startswith("data: {")]
    last_end = max(at for at, frame in enumerate(frames) if frame.get("type") == "tool_end")
    assert {"type": "tool_status", "content": ""} in frames[last_end:]


def test_tool_choice_none_still_withdraws_the_catalog(executed):
    """The outbound half of the same contract must not have regressed."""
    transport = FakeTransport([[_call_line(), _finish("tool_calls")], [_DONE]])
    _run(transport, tool_choice = "none")
    assert transport.requests[0]["tool_choice"] == "none"


def test_tool_choice_auto_still_executes(executed):
    """The refusal must be specific to "none"."""
    transport = FakeTransport([[_call_line(), _finish("tool_calls")], [_DONE]])
    _run(transport, tool_choice = "auto")
    assert executed == ["web_search"]


@pytest.mark.parametrize("reason", ["length", "content_filter"])
def test_a_turn_cut_short_does_not_execute_its_call(executed, reason):
    """Both endings mean the model never finished saying what it wanted."""
    transport = FakeTransport([[_call_line(), _finish(reason)], [_DONE]])
    _run(transport, tool_choice = "auto")
    assert executed == []


@pytest.mark.parametrize("reason", ["tool_calls", "stop"])
def test_a_completed_turn_still_executes(executed, reason):
    """Turns ending in stop must execute: llama.cpp and vLLM commonly finish good tool calls that way."""
    transport = FakeTransport([[_call_line(), _finish(reason)], [_DONE]])
    _run(transport, tool_choice = "auto")
    assert executed == ["web_search"]