Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: retry json_object as json_schema when a custom provider (LM Studio) refuses it (#11813)
Saravana Priyand6641ec3bd83
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.pyindex 0f6450e5f95..36d3afe3266 100644--- a/studio/backend/core/inference/external_provider.py+++ b/studio/backend/core/inference/external_provider.py@@ -49,6 +49,29 @@ _USAGE_STREAM_OPTION_PROVIDERS = frozenset({"vllm", "llama_cpp", "ollama", "openrouter", "kimi", "lemonade", "qwen", "qwencloud"})++# LM Studio 400s on {"type": "json_object"} ("must be 'json_schema' or 'text'"); a json_schema of the same object is the+# same constraint. A retry, not the default: some OpenAI-compatible APIs accept json_object but not json_schema.+def _json_object_as_schema(response_format: dict[str, Any]) -> dict[str, Any]:+ """Keep the llama.cpp-style `schema` member; absent or empty means any object, as in grammar_constraint."""+ schema = response_format.get("schema")+ if not isinstance(schema, dict) or not schema:+ schema = {"type": "object"}+ return {"type": "json_schema", "json_schema": {"name": "response", "schema": schema}}+++def _json_object_refused_for_schema(+ status_code: int, error_text: str, response_format: Optional[dict[str, Any]]+) -> bool:+ """True when the server refused JSON mode but names json_schema as the accepted spelling."""+ if status_code != 400 or not isinstance(response_format, dict):+ return False+ if response_format.get("type") != "json_object":+ return False+ text = error_text.lower()+ return "response_format" in text and "json_schema" in text++# launch-time windows are absent from catalogues; custom covers unregistered self-hosted servers._SERVED_WINDOW_PROVIDERS = frozenset({"vllm", "llama_cpp", "custom"})# refresh within one or two turns to detect server restarts.@@ -1141,25 +1164,26 @@ def _with_max_completion_tokens(body: dict[str, Any]) -> dict[str, Any]:@contextlib.asynccontextmanager-async def _stream_post_retrying_max_tokens(+async def _stream_post_with_compat_fallbacks(http: httpx.AsyncClient, url: str, body: dict[str, Any], **kwargs):- """`http.stream("POST", ...)` that resends once with `max_completion_tokens` if the upstream rejects `max_tokens`.- Nothing has been yielded to the caller at the status check, so the retry is invisible."""- async with http.stream("POST", url, json = body, **kwargs) as response:- retry = (- response.status_code == 400- and "max_tokens" in body- and _rejects_max_tokens(400, (await response.aread()).decode("utf-8", errors = "replace"))- )- if not retry:- yield response- return- logger.info("Upstream rejected max_tokens; retrying with max_completion_tokens")- async with http.stream(- "POST", url, json = _with_max_completion_tokens(body), **kwargs- ) as response:- yield response+ """Resend on a refused `max_tokens` or `json_object` before yielding; each rewrite removes its own trigger,+ so each field changes at most once."""+ while True:+ async with http.stream("POST", url, json = body, **kwargs) as response:+ if response.status_code != 400:+ yield response+ return+ error_text = (await response.aread()).decode("utf-8", errors = "replace")+ if "max_tokens" in body and _rejects_max_tokens(400, error_text):+ body = _with_max_completion_tokens(body)+ logger.info("Upstream rejected max_tokens; retrying with max_completion_tokens")+ elif _json_object_refused_for_schema(400, error_text, body.get("response_format")):+ body = {**body, "response_format": _json_object_as_schema(body["response_format"])}+ logger.info("Upstream refused json_object; retrying as json_schema")+ else:+ yield response+ returndef _client() -> httpx.AsyncClient:@@ -1963,7 +1987,7 @@ class ExternalProviderClient:)try:- async with _stream_post_retrying_max_tokens(+ async with _stream_post_with_compat_fallbacks(_loopback_http_client if getattr(self, "_managed_loopback", False) else _client(),url,body,diff --git a/studio/backend/tests/test_custom_provider_json_schema_fallback.py b/studio/backend/tests/test_custom_provider_json_schema_fallback.pynew file mode 100644index 00000000000..b4d0a83fdf2--- /dev/null+++ b/studio/backend/tests/test_custom_provider_json_schema_fallback.py@@ -0,0 +1,186 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""JSON mode against an OpenAI-compatible server that only takes ``json_schema`` (LM Studio)."""++from __future__ import annotations++import asyncio+import json++import httpx+import pytest++from core.inference import external_provider as ep_mod+from core.inference.external_provider import ExternalProviderClient++_OK_SSE = 'data: {"choices":[{"index":0,"delta":{"content":"{}"}}]}\n\n' "data: [DONE]\n\n"+_LMSTUDIO_REFUSAL = {"error": "'response_format.type' must be 'json_schema' or 'text'"}+++def _run(+ handler,+ response_format,+ *,+ provider_type = "custom",+ **kwargs,+) -> list[str]:+ mock_client = httpx.AsyncClient(transport = httpx.MockTransport(handler))+ client = ExternalProviderClient(+ provider_type = provider_type,+ base_url = "http://127.0.0.1:1234/v1",+ api_key = "",+ )+ lines: list[str] = []++ async def run() -> None:+ try:+ async for line in client.stream_chat_completion(+ messages = [{"role": "user", "content": "plan"}],+ model = "some-local-model",+ response_format = response_format,+ **kwargs,+ ):+ lines.append(line)+ finally:+ await mock_client.aclose()++ event_loop = asyncio.new_event_loop()+ previous_client = ep_mod._http_client+ ep_mod._http_client = mock_client+ try:+ event_loop.run_until_complete(run())+ finally:+ ep_mod._http_client = previous_client+ event_loop.close()+ return lines+++def _schema_only_server(bodies: list[dict]):+ def handler(request: httpx.Request) -> httpx.Response:+ body = json.loads(request.content.decode())+ bodies.append(body)+ if (body.get("response_format") or {}).get("type") == "json_object":+ return httpx.Response(400, json = _LMSTUDIO_REFUSAL)+ return httpx.Response(200, content = _OK_SSE, headers = {"content-type": "text/event-stream"})++ return handler+++def test_json_object_refusal_is_retried_once_as_json_schema():+ bodies: list[dict] = []+ lines = _run(_schema_only_server(bodies), {"type": "json_object"})++ assert [b["response_format"]["type"] for b in bodies] == ["json_object", "json_schema"]+ assert bodies[1]["response_format"]["json_schema"]["schema"] == {"type": "object"}+ assert bodies[0]["messages"] == bodies[1]["messages"]+ assert bodies[0]["model"] == bodies[1]["model"]+ assert any('"content": "{}"' in line or '"content":"{}"' in line for line in lines)+ assert not any('"error"' in line for line in lines)+++def test_schema_retry_keeps_a_supplied_json_object_schema():+ schema = {"type": "object", "properties": {"a": {"type": "string"}}, "required": ["a"]}+ bodies: list[dict] = []+ _run(_schema_only_server(bodies), {"type": "json_object", "schema": schema})++ assert [b["response_format"]["type"] for b in bodies] == ["json_object", "json_schema"]+ assert bodies[1]["response_format"]["json_schema"]["schema"] == schema+++def test_servers_that_accept_json_object_are_sent_it_unchanged():+ bodies: list[dict] = []++ def handler(request: httpx.Request) -> httpx.Response:+ bodies.append(json.loads(request.content.decode()))+ return httpx.Response(200, content = _OK_SSE, headers = {"content-type": "text/event-stream"})++ _run(handler, {"type": "json_object"})+ assert [b["response_format"] for b in bodies] == [{"type": "json_object"}]+++def test_unrelated_400s_are_not_retried():+ bodies: list[dict] = []++ def handler(request: httpx.Request) -> httpx.Response:+ bodies.append(json.loads(request.content.decode()))+ return httpx.Response(400, json = {"error": "context length exceeded"})++ lines = _run(handler, {"type": "json_object"})+ assert len(bodies) == 1+ assert any("error" in line for line in lines)+++def test_a_json_schema_refusal_is_not_retried_again():+ # No loop: the retried request already carries json_schema.+ bodies: list[dict] = []++ def handler(request: httpx.Request) -> httpx.Response:+ bodies.append(json.loads(request.content.decode()))+ return httpx.Response(400, json = _LMSTUDIO_REFUSAL)++ _run(handler, {"type": "json_object"})+ assert len(bodies) == 2+++def test_schema_retry_preserves_openrouter_session_and_request_options():+ bodies: list[dict] = []+ lines = _run(+ _schema_only_server(bodies),+ {"type": "json_object"},+ provider_type = "openrouter",+ thread_id = "research-thread",+ temperature = 0.2,+ top_p = None,+ max_tokens = 128,+ presence_penalty = 0.4,+ top_k = 30,+ min_p = 0.1,+ repetition_penalty = 1.1,+ reasoning_effort = "high",+ tools = [+ {"type": "function", "function": {"name": "search", "parameters": {"type": "object"}}}+ ],+ tool_choice = "none",+ )+ assert len(bodies) == 2+ assert bodies[0]["session_id"] == "research-thread"+ assert bodies[1] == {**bodies[0], "response_format": bodies[1]["response_format"]}+ assert not any('"error"' in line for line in lines)++[email protected]("first_refusal", ["max_tokens", "response_format"])[email protected]("schema_accepted", [True, False])+def test_schema_and_token_limit_retries_preserve_each_others_changes(+ first_refusal, schema_accepted+):+ bodies: list[dict] = []+ token_refusal = {"error": "Unsupported parameter: 'max_tokens'. Use 'max_completion_tokens'."}++ def handler(request: httpx.Request) -> httpx.Response:+ body = json.loads(request.content.decode())+ bodies.append(body)+ refusals = {+ "max_tokens": token_refusal if "max_tokens" in body else None,+ "response_format": _LMSTUDIO_REFUSAL+ if body["response_format"]["type"] == "json_object"+ else None,+ }+ error = refusals[first_refusal] or next((r for r in refusals.values() if r), None)+ if error is not None or not schema_accepted:+ return httpx.Response(400, json = error or _LMSTUDIO_REFUSAL)+ return httpx.Response(200, content = _OK_SSE, headers = {"content-type": "text/event-stream"})++ lines = _run(handler, {"type": "json_object"}, max_tokens = 128)++ assert len(bodies) == 3+ assert bodies[-1]["max_completion_tokens"] == 128+ assert "max_tokens" not in bodies[-1]+ assert bodies[-1]["response_format"]["json_schema"]["schema"] == {"type": "object"}+ for previous, current in zip(bodies, bodies[1:]):+ if "max_completion_tokens" in previous:+ assert current["max_completion_tokens"] == 128+ assert "max_tokens" not in current+ if previous["response_format"]["type"] == "json_schema":+ assert current["response_format"] == previous["response_format"]+ assert any('"error"' in line for line in lines) is not schema_accepted