Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Let API requests that ask for a JSON reply still call their tools (#12578)

Nilaya0136ac303c8
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 7b0deb444d1..a676785f8a7 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -40924,6 +40924,10 @@ def _build_openai_passthrough_body(
if llama_backend is not None
else None
)
+ response_format = _response_format_for_llama_server(_extract_response_format(payload))
+ if tools and tool_choice != "none" and _response_format_constrains_decoding(payload):
+ logger.warning("Ignoring response_format: callable tools cannot run under a schema")
+ response_format = None
body = _build_passthrough_payload(
messages,
tools,
@@ -40940,7 +40944,7 @@ def _build_openai_passthrough_body(
frequency_penalty = payload.frequency_penalty,
logit_bias = payload.logit_bias,
tool_choice = tool_choice,
- response_format = _response_format_for_llama_server(_extract_response_format(payload)),
+ response_format = response_format,
chat_template_kwargs = tpl_kwargs,
backend_ctx = backend_ctx,
seed = payload.seed,
diff --git a/studio/backend/tests/test_openai_tool_passthrough.py b/studio/backend/tests/test_openai_tool_passthrough.py
index d5a0259e53d..ae03995a962 100644
--- a/studio/backend/tests/test_openai_tool_passthrough.py
+++ b/studio/backend/tests/test_openai_tool_passthrough.py
@@ -10849,6 +10849,58 @@ def test_a_lenient_schema_reaches_llama_server_where_it_reads_one():
assert _build_openai_passthrough_body(request)["response_format"] == wrapped
+_WEATHER_TOOL = {
+ "type": "function",
+ "function": {
+ "name": "get_weather",
+ "parameters": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}},
+ "required": ["city"],
+ },
+ },
+}
+_WEATHER_FORMAT = {
+ "type": "json_schema",
+ "json_schema": {
+ "name": "final_output",
+ "strict": True,
+ "schema": {
+ "type": "object",
+ "properties": {"city": {"type": "string"}, "temp_c": {"type": "number"}},
+ "required": ["city", "temp_c"],
+ "additionalProperties": False,
+ },
+ },
+}
+
+
+def _weather_body(**fields):
+ request = ChatCompletionRequest(
+ model = "m", messages = [{"role": "user", "content": "Weather in Paris?"}], **fields
+ )
+ return _build_openai_passthrough_body(request)
+
+
[email protected]("response_format", [_WEATHER_FORMAT, {"type": "json_object"}])
+ "tool_choice",
+ [None, "auto", "required", {"type": "function", "function": {"name": "get_weather"}}],
+)
+def test_a_response_format_does_not_lock_out_callable_tools(tool_choice, response_format):
+ body = _weather_body(
+ tools = [_WEATHER_TOOL], tool_choice = tool_choice, response_format = response_format
+ )
+ assert [tool["function"]["name"] for tool in body["tools"]] == ["get_weather"]
+ assert "response_format" not in body
+
+
+def test_a_response_format_still_applies_when_tools_cannot_be_called():
+ body = _weather_body(tools = [_WEATHER_TOOL], tool_choice = "none", response_format = _WEATHER_FORMAT)
+ assert body["response_format"] == _WEATHER_FORMAT
+ assert _weather_body(response_format = _WEATHER_FORMAT)["response_format"] == _WEATHER_FORMAT
+
+
class TestPassthroughImageNormalization:
@staticmethod
def _data_url(fmt: str) -> str:
diff --git a/studio/backend/tests/test_responses_tool_passthrough.py b/studio/backend/tests/test_responses_tool_passthrough.py
index db68d3ccb2e..9f10dafec6d 100644
--- a/studio/backend/tests/test_responses_tool_passthrough.py
+++ b/studio/backend/tests/test_responses_tool_passthrough.py
@@ -489,6 +489,33 @@ class TestBuildChatRequest:
"json_schema": {"name": "Person", "schema": schema, "strict": True},
}
+ def test_text_format_does_not_lock_out_callable_tools(self):
+ payload = ResponsesRequest(
+ input = "Weather in Paris?",
+ tools = [
+ {
+ "type": "function",
+ "name": "get_weather",
+ "parameters": {"type": "object", "properties": {"city": {"type": "string"}}},
+ }
+ ],
+ text = {
+ "format": {
+ "type": "json_schema",
+ "name": "final_output",
+ "schema": {"type": "object", "properties": {"temp_c": {"type": "number"}}},
+ "strict": True,
+ }
+ },
+ )
+ messages = [ChatMessage(role = "user", content = "Weather in Paris?")]
+
+ chat_req = _build_chat_request(payload, messages, stream = True)
+ body = _build_openai_passthrough_body(chat_req, backend_ctx = 4096)
+
+ assert [tool["function"]["name"] for tool in body["tools"]] == ["get_weather"]
+ assert "response_format" not in body
+
def test_text_format_json_object_becomes_response_format(self):
payload = ResponsesRequest(input = "hi", text = {"format": {"type": "json_object"}})
messages = [ChatMessage(role = "user", content = "hi")]