Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: let JSON-reply API requests still call their tools on MLX models (#13658)

Nilay42625f0bef29
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 1303a5f1483..2179d0f8638 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -30939,6 +30939,12 @@ async def produce_openai_chat_completions(
"route: a waveform admits no grammar, and transcription and audio-input "
"answers are decoded without one. Send the request to a text model.",
)
+ # Callable client tools set the format aside (_response_format_beside_tools), so nothing
+ # below applies, unless the request asks for Unsloth's own loop, which keeps the format.
+ if _response_format_constrains_decoding(payload) and (
+ _explicit_studio_tool_loop_requested(payload)
+ or not _tools_outrank_response_format(payload, payload.tools)
+ ):
if not model_info.get("is_mlx"):
_raise_unsupported_openai_parameter(
"response_format",
@@ -34549,6 +34555,11 @@ async def produce_openai_chat_completions(
and _sf_supports_tools
and ((payload.tools and len(payload.tools) > 0) or _sf_has_tool_msgs)
)
+ _sf_response_format = _response_format_beside_tools(
+ payload, payload.tools if _sf_client_tools else None
+ )
+ gen_kwargs["response_format"] = _sf_response_format
+ _sf_single_block = _sf_single_block and _sf_response_format is not None
# apply_chat_template sanitizes the catalog it renders, so a tool dropped for unsafe
# markup never reached the prompt. Gating the healer on the caller's list instead would
# promote a dropped tool with a clean NAME out of text-form output, handing the client a
@@ -34600,7 +34611,7 @@ async def produce_openai_chat_completions(
payload.auto_heal_tool_calls,
_sf_healing_tools,
payload.tool_choice,
- response_format = _extract_response_format(payload),
+ response_format = _sf_response_format,
)
if _sf_client_tools
else None
@@ -35247,7 +35258,7 @@ async def produce_openai_chat_completions(
_finish = "tool_calls"
elif nudge_enabled(
payload.nudge_tool_calls,
- response_format = _extract_response_format(payload),
+ response_format = _sf_response_format,
):
_data = {
"choices": [
@@ -44892,6 +44903,21 @@ def _reject_unhonorable_response_format(payload) -> None:
_raise_unsupported_openai_parameter("response_format", str(exc))
+def _tools_outrank_response_format(payload, tools) -> bool:
+ return bool(tools) and payload.tool_choice != "none"
+
+
+def _response_format_beside_tools(payload, tools):
+ """The request's ``response_format``, or None while ``tools`` can still be called:
+ decoding locked to the document would never emit the call."""
+ if _tools_outrank_response_format(payload, tools) and _response_format_constrains_decoding(
+ payload
+ ):
+ logger.warning("Ignoring response_format: callable tools cannot run under a schema")
+ return None
+ return _extract_response_format(payload)
+
+
def _build_openai_passthrough_body(
payload,
backend_ctx = None,
@@ -44924,10 +44950,9 @@ def _build_openai_passthrough_body(
if llama_backend is not None
else None
)
- response_format = _response_format_for_llama_server(_extract_response_format(payload))
- if tools and tool_choice != "none" and _response_format_constrains_decoding(payload):
- logger.warning("Ignoring response_format: callable tools cannot run under a schema")
- response_format = None
+ response_format = _response_format_for_llama_server(
+ _response_format_beside_tools(payload, tools)
+ )
body = _build_passthrough_payload(
messages,
tools,
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index 02493c11a46..5cc22ed749b 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -3029,6 +3029,25 @@ class TestBuildPassthroughPayloadToolChoice:
{"role": "user", "content": "hi"},
]
+ @pytest.mark.parametrize(
+ "tool_choice, kept",
+ [("auto", False), ("required", False), ("none", True)],
+ )
+ def test_passthrough_body_sets_response_format_aside_for_callable_tools(
+ self, tool_choice, kept
+ ):
+ payload = ChatCompletionRequest(
+ model = "default",
+ messages = [{"role": "user", "content": "hi"}],
+ tools = self._args()["openai_tools"],
+ tool_choice = tool_choice,
+ response_format = {"type": "json_object"},
+ )
+
+ body = _build_openai_passthrough_body(payload, backend_ctx = 4096)
+
+ assert ("response_format" in body) is kept
+
class TestOpenAIPassthroughSSETerminalState:
def test_done_sentinel(self):
diff --git a/studio/backend/tests/test_passthrough_healing.py b/studio/backend/tests/test_passthrough_healing.py
index cc4c6251c54..cd55be25fab 100644
--- a/studio/backend/tests/test_passthrough_healing.py
+++ b/studio/backend/tests/test_passthrough_healing.py
@@ -1697,12 +1697,25 @@ def test_every_post_decode_reinterpreter_is_handed_the_request_s_own_contract():
from pathlib import Path
names = ("heal_gate", "nudge_enabled")
- forwards = ('body.get("response_format")', "_extract_response_format(payload)")
+ forwards = (
+ 'body.get("response_format")',
+ "_extract_response_format(payload)",
+ "_sf_response_format",
+ )
source = (Path(__file__).resolve().parents[1] / "routes/inference.py").read_text(
encoding = "utf-8"
)
+ tree = ast.parse(source)
+ # The MLX name may hold None only where _response_format_beside_tools sets the contract aside.
+ sources = [
+ ast.get_source_segment(source, getattr(node.value, "func", node.value))
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Assign)
+ and any(getattr(t, "id", None) == "_sf_response_format" for t in node.targets)
+ ]
+ assert sources == ["_response_format_beside_tools"], sources
calls, missing = [], []
- for node in ast.walk(ast.parse(source)):
+ for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
if (getattr(node.func, "attr", None) or getattr(node.func, "id", None)) not in names:
diff --git a/studio/backend/tests/test_sf_client_tools_passthrough.py b/studio/backend/tests/test_sf_client_tools_passthrough.py
index 01eac85a6db..572a1bdea85 100644
--- a/studio/backend/tests/test_sf_client_tools_passthrough.py
+++ b/studio/backend/tests/test_sf_client_tools_passthrough.py
@@ -1536,6 +1536,183 @@ def test_the_contract_reaches_the_backend_and_its_reply_comes_back_whole(monkeyp
assert not message.get("reasoning_content")
+def _mlx_format_call(
+ monkeypatch,
+ backend,
+ caps = None,
+ **fields,
+):
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "_reject_unhonorable_response_format", lambda payload: None)
+ backend.models["sf-model"]["is_mlx"] = True
+ return _call(
+ _request(**{"tools": [LOOKUP_TOOL], **fields}), monkeypatch, backend, **(caps or {})
+ )
+
+
[email protected]("response_format", [_RF_FORMAT, {"type": "json_object"}])
+ "tool_choice",
+ [None, "auto", "required", {"type": "function", "function": {"name": "lookup"}}],
+)
+def test_a_response_format_does_not_lock_out_callable_tools(
+ monkeypatch, response_format, tool_choice
+):
+ backend = _ScriptedBackend(_fixed(_CALL_XML))
+ reply = _mlx_format_call(
+ monkeypatch,
+ backend,
+ tool_choice = tool_choice,
+ response_format = response_format,
+ stream = False,
+ )
+ choice = _json_body(reply)["choices"][0]
+ assert backend.calls[0]["tools"] == [LOOKUP_TOOL]
+ assert backend.calls[0]["response_format"] is None
+ assert choice["finish_reason"] == "tool_calls"
+ assert choice["message"]["tool_calls"][0]["function"]["name"] == "lookup"
+
+
+def test_a_streamed_response_format_does_not_lock_out_callable_tools(monkeypatch):
+ backend = _ScriptedBackend(_fixed(_CALL_XML))
+ reply = _mlx_format_call(monkeypatch, backend, response_format = _RF_FORMAT, stream = True)
+ finishes = [
+ c.get("finish_reason") for o in _sse_objects(_collect_sse(reply)) for c in o["choices"]
+ ]
+ assert backend.calls[0]["response_format"] is None
+ assert "tool_calls" in finishes
+
+
+def test_a_set_aside_response_format_keeps_reasoning_out_of_the_call(monkeypatch):
+ import routes.inference as inf
+
+ monkeypatch.setattr(inf, "_sf_reasoning_prefill_mode", lambda *a, **k: True)
+ backend = _ScriptedBackend(_fixed("weighing</think>" + _CALL_XML))
+ reply = _mlx_format_call(
+ monkeypatch,
+ backend,
+ caps = {"supports_reasoning": True},
+ response_format = _RF_FORMAT,
+ stream = False,
+ )
+ message = _json_body(reply)["choices"][0]["message"]
+ assert message["reasoning_content"] == "weighing"
+ assert not message["content"]
+ assert message["tool_calls"][0]["function"]["name"] == "lookup"
+
+
+def test_a_set_aside_response_format_still_allows_the_nudge(monkeypatch):
+ truncated = '<tool_call>{"name": "lookup"'
+ backend = _ScriptedBackend(
+ lambda messages, tools: [_CALL_XML]
+ if any("native tool-call format" in (m.get("content") or "") for m in messages)
+ else [truncated]
+ )
+ reply = _mlx_format_call(
+ monkeypatch, backend, response_format = _RF_FORMAT, nudge_tool_calls = True, stream = False
+ )
+ assert len(backend.calls) == 2
+ assert _json_body(reply)["choices"][0]["finish_reason"] == "tool_calls"
+
+
+_RF_TOOL_HISTORY = [
+ ChatMessage(role = "user", content = "hi"),
+ ChatMessage(
+ role = "assistant",
+ content = None,
+ tool_calls = [
+ {
+ "id": "call_1",
+ "type": "function",
+ "function": {"name": "lookup", "arguments": '{"q": "cats"}'},
+ }
+ ],
+ ),
+ ChatMessage(role = "tool", tool_call_id = "call_1", content = "Paris"),
+]
+
+
+ "fields",
+ [{"tool_choice": "none"}, {"tools": None, "messages": _RF_TOOL_HISTORY}],
+ ids = ["tool_choice_none", "history_without_tools"],
+)
+def test_a_response_format_still_applies_when_tools_cannot_be_called(monkeypatch, fields):
+ backend = _ScriptedBackend(_fixed('{"city":"Paris"}'))
+ reply = _mlx_format_call(
+ monkeypatch, backend, response_format = _RF_FORMAT, stream = False, **fields
+ )
+ assert backend.calls[0]["response_format"] == _RF_FORMAT
+ assert _json_body(reply)["choices"][0]["message"]["content"] == '{"city":"Paris"}'
+
+
+ "tool_choice", ["required", {"type": "function", "function": {"name": "x"}}]
+)
+def test_a_forced_tool_choice_beside_a_response_format_needs_tools(monkeypatch, tool_choice):
+ from fastapi import HTTPException
+
+ backend = _ScriptedBackend(_fixed('{"city":"Paris"}'))
+ with pytest.raises(HTTPException) as exc:
+ _mlx_format_call(
+ monkeypatch,
+ backend,
+ tools = None,
+ tool_choice = tool_choice,
+ response_format = _RF_FORMAT,
+ stream = False,
+ )
+ assert exc.value.status_code == 400
+ assert not backend.calls
+
+
[email protected]("is_mlx", [False, True])
+def test_a_requested_tool_loop_keeps_the_format_refusals(monkeypatch, is_mlx):
+ from fastapi import HTTPException
+
+ backend = _ScriptedBackend(_fixed(_CALL_XML))
+ backend.models["sf-model"]["is_mlx"] = is_mlx
+ payload = _request(
+ tools = [LOOKUP_TOOL],
+ enable_tools = True,
+ max_tool_calls_per_message = 0,
+ response_format = {"type": "json_schema"},
+ stream = False,
+ )
+ with pytest.raises(HTTPException) as exc:
+ _call(payload, monkeypatch, backend)
+ assert exc.value.status_code == 400
+ assert not backend.calls
+
+
+ "is_mlx, fields",
+ [
+ (False, {"response_format": _RF_FORMAT}),
+ (True, {"response_format": {"type": "json_schema"}}),
+ (
+ True,
+ {
+ "response_format": _RF_FORMAT,
+ "continue_final_message": True,
+ "messages": [
+ ChatMessage(role = "user", content = "hi"),
+ ChatMessage(role = "assistant", content = "<tool_call>"),
+ ],
+ },
+ ),
+ ],
+ ids = ["transformers", "unhonorable_schema", "continuation"],
+)
+def test_callable_tools_lift_the_refusals_of_a_format_they_set_aside(monkeypatch, is_mlx, fields):
+ backend = _ScriptedBackend(_fixed(_CALL_XML))
+ backend.models["sf-model"]["is_mlx"] = is_mlx
+ reply = _call(_request(tools = [LOOKUP_TOOL], stream = False, **fields), monkeypatch, backend)
+ assert backend.calls[0]["response_format"] is None
+ assert _json_body(reply)["choices"][0]["finish_reason"] == "tool_calls"
+
+
class _FittedToolLoopBackend(_ToolLoopBackend):
def __init__(self, responder, **kwargs):
super().__init__(responder, **kwargs)