Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let JSON-reply API requests still call their tools on MLX models (#13658)
Nilay42625f0bef29
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 1303a5f1483..2179d0f8638 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -30939,6 +30939,12 @@ async def produce_openai_chat_completions("route: a waveform admits no grammar, and transcription and audio-input ""answers are decoded without one. Send the request to a text model.",)+ # Callable client tools set the format aside (_response_format_beside_tools), so nothing+ # below applies, unless the request asks for Unsloth's own loop, which keeps the format.+ if _response_format_constrains_decoding(payload) and (+ _explicit_studio_tool_loop_requested(payload)+ or not _tools_outrank_response_format(payload, payload.tools)+ ):if not model_info.get("is_mlx"):_raise_unsupported_openai_parameter("response_format",@@ -34549,6 +34555,11 @@ async def produce_openai_chat_completions(and _sf_supports_toolsand ((payload.tools and len(payload.tools) > 0) or _sf_has_tool_msgs))+ _sf_response_format = _response_format_beside_tools(+ payload, payload.tools if _sf_client_tools else None+ )+ gen_kwargs["response_format"] = _sf_response_format+ _sf_single_block = _sf_single_block and _sf_response_format is not None# apply_chat_template sanitizes the catalog it renders, so a tool dropped for unsafe# markup never reached the prompt. Gating the healer on the caller's list instead would# promote a dropped tool with a clean NAME out of text-form output, handing the client a@@ -34600,7 +34611,7 @@ async def produce_openai_chat_completions(payload.auto_heal_tool_calls,_sf_healing_tools,payload.tool_choice,- response_format = _extract_response_format(payload),+ response_format = _sf_response_format,)if _sf_client_toolselse None@@ -35247,7 +35258,7 @@ async def produce_openai_chat_completions(_finish = "tool_calls"elif nudge_enabled(payload.nudge_tool_calls,- response_format = _extract_response_format(payload),+ response_format = _sf_response_format,):_data = {"choices": [@@ -44892,6 +44903,21 @@ def _reject_unhonorable_response_format(payload) -> None:_raise_unsupported_openai_parameter("response_format", str(exc))+def _tools_outrank_response_format(payload, tools) -> bool:+ return bool(tools) and payload.tool_choice != "none"+++def _response_format_beside_tools(payload, tools):+ """The request's ``response_format``, or None while ``tools`` can still be called:+ decoding locked to the document would never emit the call."""+ if _tools_outrank_response_format(payload, tools) and _response_format_constrains_decoding(+ payload+ ):+ logger.warning("Ignoring response_format: callable tools cannot run under a schema")+ return None+ return _extract_response_format(payload)++def _build_openai_passthrough_body(payload,backend_ctx = None,@@ -44924,10 +44950,9 @@ def _build_openai_passthrough_body(if llama_backend is not Noneelse None)- response_format = _response_format_for_llama_server(_extract_response_format(payload))- if tools and tool_choice != "none" and _response_format_constrains_decoding(payload):- logger.warning("Ignoring response_format: callable tools cannot run under a schema")- response_format = None+ response_format = _response_format_for_llama_server(+ _response_format_beside_tools(payload, tools)+ )body = _build_passthrough_payload(messages,tools,diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.pyindex 02493c11a46..5cc22ed749b 100644--- a/studio/backend/tests/test_openai_tool_passthrough.py+++ b/studio/backend/tests/test_openai_tool_passthrough.py@@ -3029,6 +3029,25 @@ class TestBuildPassthroughPayloadToolChoice:{"role": "user", "content": "hi"},]+ @pytest.mark.parametrize(+ "tool_choice, kept",+ [("auto", False), ("required", False), ("none", True)],+ )+ def test_passthrough_body_sets_response_format_aside_for_callable_tools(+ self, tool_choice, kept+ ):+ payload = ChatCompletionRequest(+ model = "default",+ messages = [{"role": "user", "content": "hi"}],+ tools = self._args()["openai_tools"],+ tool_choice = tool_choice,+ response_format = {"type": "json_object"},+ )++ body = _build_openai_passthrough_body(payload, backend_ctx = 4096)++ assert ("response_format" in body) is kept+class TestOpenAIPassthroughSSETerminalState:def test_done_sentinel(self):diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.pyindex cc4c6251c54..cd55be25fab 100644--- a/studio/backend/tests/test_passthrough_healing.py+++ b/studio/backend/tests/test_passthrough_healing.py@@ -1697,12 +1697,25 @@ def test_every_post_decode_reinterpreter_is_handed_the_request_s_own_contract():from pathlib import Pathnames = ("heal_gate", "nudge_enabled")- forwards = ('body.get("response_format")', "_extract_response_format(payload)")+ forwards = (+ 'body.get("response_format")',+ "_extract_response_format(payload)",+ "_sf_response_format",+ )source = (Path(__file__).resolve().parents[1] / "routes/inference.py").read_text(encoding = "utf-8")+ tree = ast.parse(source)+ # The MLX name may hold None only where _response_format_beside_tools sets the contract aside.+ sources = [+ ast.get_source_segment(source, getattr(node.value, "func", node.value))+ for node in ast.walk(tree)+ if isinstance(node, ast.Assign)+ and any(getattr(t, "id", None) == "_sf_response_format" for t in node.targets)+ ]+ assert sources == ["_response_format_beside_tools"], sourcescalls, missing = [], []- for node in ast.walk(ast.parse(source)):+ for node in ast.walk(tree):if not isinstance(node, ast.Call):continueif (getattr(node.func, "attr", None) or getattr(node.func, "id", None)) not in names:diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.pyindex 01eac85a6db..572a1bdea85 100644--- a/studio/backend/tests/test_sf_client_tools_passthrough.py+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py@@ -1536,6 +1536,183 @@ def test_the_contract_reaches_the_backend_and_its_reply_comes_back_whole(monkeypassert not message.get("reasoning_content")+def _mlx_format_call(+ monkeypatch,+ backend,+ caps = None,+ **fields,+):+ import routes.inference as inf++ monkeypatch.setattr(inf, "_reject_unhonorable_response_format", lambda payload: None)+ backend.models["sf-model"]["is_mlx"] = True+ return _call(+ _request(**{"tools": [LOOKUP_TOOL], **fields}), monkeypatch, backend, **(caps or {})+ )++[email protected]("response_format", [_RF_FORMAT, {"type": "json_object"}])+ "tool_choice",+ [None, "auto", "required", {"type": "function", "function": {"name": "lookup"}}],+)+def test_a_response_format_does_not_lock_out_callable_tools(+ monkeypatch, response_format, tool_choice+):+ backend = _ScriptedBackend(_fixed(_CALL_XML))+ reply = _mlx_format_call(+ monkeypatch,+ backend,+ tool_choice = tool_choice,+ response_format = response_format,+ stream = False,+ )+ choice = _json_body(reply)["choices"][0]+ assert backend.calls[0]["tools"] == [LOOKUP_TOOL]+ assert backend.calls[0]["response_format"] is None+ assert choice["finish_reason"] == "tool_calls"+ assert choice["message"]["tool_calls"][0]["function"]["name"] == "lookup"+++def test_a_streamed_response_format_does_not_lock_out_callable_tools(monkeypatch):+ backend = _ScriptedBackend(_fixed(_CALL_XML))+ reply = _mlx_format_call(monkeypatch, backend, response_format = _RF_FORMAT, stream = True)+ finishes = [+ c.get("finish_reason") for o in _sse_objects(_collect_sse(reply)) for c in o["choices"]+ ]+ assert backend.calls[0]["response_format"] is None+ assert "tool_calls" in finishes+++def test_a_set_aside_response_format_keeps_reasoning_out_of_the_call(monkeypatch):+ import routes.inference as inf++ monkeypatch.setattr(inf, "_sf_reasoning_prefill_mode", lambda *a, **k: True)+ backend = _ScriptedBackend(_fixed("weighing</think>" + _CALL_XML))+ reply = _mlx_format_call(+ monkeypatch,+ backend,+ caps = {"supports_reasoning": True},+ response_format = _RF_FORMAT,+ stream = False,+ )+ message = _json_body(reply)["choices"][0]["message"]+ assert message["reasoning_content"] == "weighing"+ assert not message["content"]+ assert message["tool_calls"][0]["function"]["name"] == "lookup"+++def test_a_set_aside_response_format_still_allows_the_nudge(monkeypatch):+ truncated = '<tool_call>{"name": "lookup"'+ backend = _ScriptedBackend(+ lambda messages, tools: [_CALL_XML]+ if any("native tool-call format" in (m.get("content") or "") for m in messages)+ else [truncated]+ )+ reply = _mlx_format_call(+ monkeypatch, backend, response_format = _RF_FORMAT, nudge_tool_calls = True, stream = False+ )+ assert len(backend.calls) == 2+ assert _json_body(reply)["choices"][0]["finish_reason"] == "tool_calls"+++_RF_TOOL_HISTORY = [+ ChatMessage(role = "user", content = "hi"),+ ChatMessage(+ role = "assistant",+ content = None,+ tool_calls = [+ {+ "id": "call_1",+ "type": "function",+ "function": {"name": "lookup", "arguments": '{"q": "cats"}'},+ }+ ],+ ),+ ChatMessage(role = "tool", tool_call_id = "call_1", content = "Paris"),+]+++ "fields",+ [{"tool_choice": "none"}, {"tools": None, "messages": _RF_TOOL_HISTORY}],+ ids = ["tool_choice_none", "history_without_tools"],+)+def test_a_response_format_still_applies_when_tools_cannot_be_called(monkeypatch, fields):+ backend = _ScriptedBackend(_fixed('{"city":"Paris"}'))+ reply = _mlx_format_call(+ monkeypatch, backend, response_format = _RF_FORMAT, stream = False, **fields+ )+ assert backend.calls[0]["response_format"] == _RF_FORMAT+ assert _json_body(reply)["choices"][0]["message"]["content"] == '{"city":"Paris"}'+++ "tool_choice", ["required", {"type": "function", "function": {"name": "x"}}]+)+def test_a_forced_tool_choice_beside_a_response_format_needs_tools(monkeypatch, tool_choice):+ from fastapi import HTTPException++ backend = _ScriptedBackend(_fixed('{"city":"Paris"}'))+ with pytest.raises(HTTPException) as exc:+ _mlx_format_call(+ monkeypatch,+ backend,+ tools = None,+ tool_choice = tool_choice,+ response_format = _RF_FORMAT,+ stream = False,+ )+ assert exc.value.status_code == 400+ assert not backend.calls++[email protected]("is_mlx", [False, True])+def test_a_requested_tool_loop_keeps_the_format_refusals(monkeypatch, is_mlx):+ from fastapi import HTTPException++ backend = _ScriptedBackend(_fixed(_CALL_XML))+ backend.models["sf-model"]["is_mlx"] = is_mlx+ payload = _request(+ tools = [LOOKUP_TOOL],+ enable_tools = True,+ max_tool_calls_per_message = 0,+ response_format = {"type": "json_schema"},+ stream = False,+ )+ with pytest.raises(HTTPException) as exc:+ _call(payload, monkeypatch, backend)+ assert exc.value.status_code == 400+ assert not backend.calls+++ "is_mlx, fields",+ [+ (False, {"response_format": _RF_FORMAT}),+ (True, {"response_format": {"type": "json_schema"}}),+ (+ True,+ {+ "response_format": _RF_FORMAT,+ "continue_final_message": True,+ "messages": [+ ChatMessage(role = "user", content = "hi"),+ ChatMessage(role = "assistant", content = "<tool_call>"),+ ],+ },+ ),+ ],+ ids = ["transformers", "unhonorable_schema", "continuation"],+)+def test_callable_tools_lift_the_refusals_of_a_format_they_set_aside(monkeypatch, is_mlx, fields):+ backend = _ScriptedBackend(_fixed(_CALL_XML))+ backend.models["sf-model"]["is_mlx"] = is_mlx+ reply = _call(_request(tools = [LOOKUP_TOOL], stream = False, **fields), monkeypatch, backend)+ assert backend.calls[0]["response_format"] is None+ assert _json_body(reply)["choices"][0]["finish_reason"] == "tool_calls"++class _FittedToolLoopBackend(_ToolLoopBackend):def __init__(self, responder, **kwargs):super().__init__(responder, **kwargs)