Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: retry custom gateways with max_completion_tokens after a max_tokens 400 (#10837)
Sourav Rajvi5c239a233d00
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.pyindex 0efc50bbf12..c6b932c0001 100644--- a/studio/backend/core/inference/external_provider.py+++ b/studio/backend/core/inference/external_provider.py@@ -6,6 +6,7 @@import asyncioimport base64+import contextlibimport ioimport json as _jsonimport mimetypes@@ -1020,6 +1021,49 @@ def restore_account_clients(account_id: str) -> None:_retired_accounts.discard(account_id)+def _rejects_max_tokens(status_code: int, error_text: str) -> bool:+ """400 from an upstream that wants `max_completion_tokens` (Azure gpt-5.x / o-series behind custom gateways, #10787)."""+ if status_code != 400:+ return False+ try:+ err = _json.loads(error_text).get("error")+ except Exception:+ err = None+ if isinstance(err, dict) and err.get("param") == "max_tokens":+ return err.get("code") == "unsupported_parameter" or "max_completion_tokens" in str(+ err.get("message", "")+ )+ return "max_tokens" in error_text and "max_completion_tokens" in error_text+++def _with_max_completion_tokens(body: dict[str, Any]) -> dict[str, Any]:+ body = dict(body)+ body["max_completion_tokens"] = body.pop("max_tokens")+ return body+++async def _stream_post_retrying_max_tokens(+ http: httpx.AsyncClient, url: str, body: dict[str, Any], **kwargs+):+ """`http.stream("POST", ...)` that resends once with `max_completion_tokens` if the upstream rejects `max_tokens`.+ Nothing has been yielded to the caller at the status check, so the retry is invisible."""+ async with http.stream("POST", url, json = body, **kwargs) as response:+ retry = (+ response.status_code == 400+ and "max_tokens" in body+ and _rejects_max_tokens(400, (await response.aread()).decode("utf-8", errors = "replace"))+ )+ if not retry:+ yield response+ return+ logger.info("Upstream rejected max_tokens; retrying with max_completion_tokens")+ async with http.stream(+ "POST", url, json = _with_max_completion_tokens(body), **kwargs+ ) as response:+ yield response++def _client() -> httpx.AsyncClient:"""The shared client for the owner; a screening client of its own for each managed account."""from utils.account_context import current_account_id, is_owner_context@@ -1762,12 +1806,10 @@ class ExternalProviderClient:)try:- async with (- _loopback_http_client if getattr(self, "_managed_loopback", False) else _client()- ).stream(- "POST",+ async with _stream_post_retrying_max_tokens(+ _loopback_http_client if getattr(self, "_managed_loopback", False) else _client(),url,- json = body,+ body,headers = self._auth_headers(),timeout = self._stream_timeout,) as response:@@ -6799,12 +6841,20 @@ class ExternalProviderClient:else:body["max_tokens"] = max_tokens+ url = f"{self.base_url}/chat/completions"response = await _client().post(- f"{self.base_url}/chat/completions",+ url,json = body,headers = self._auth_headers(),timeout = self._timeout,)+ if "max_tokens" in body and _rejects_max_tokens(response.status_code, response.text):+ response = await _client().post(+ url,+ json = _with_max_completion_tokens(body),+ headers = self._auth_headers(),+ timeout = self._timeout,+ )response.raise_for_status()return response.json()diff --git a/studio/backend/tests/test_external_provider_max_tokens_field.py b/studio/backend/tests/test_external_provider_max_tokens_field.pynew file mode 100644index 00000000000..eea777345b2--- /dev/null+++ b/studio/backend/tests/test_external_provider_max_tokens_field.py@@ -0,0 +1,147 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Regression for #10787: custom gateways that reject max_tokens."""++from __future__ import annotations++import asyncio+import json+import threading+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer++import httpx+import pytest++from core.inference import external_provider as ep_mod+from core.inference.external_provider import ExternalProviderClient, _rejects_max_tokens++_AZURE_ERROR = json.dumps(+ {+ "error": {+ "message": (+ "Unsupported parameter: 'max_tokens' is not supported with this model. "+ "Use 'max_completion_tokens' instead."+ ),+ "type": "invalid_request_error",+ "param": "max_tokens",+ "code": "unsupported_parameter",+ }+ }+)+++ ("status", "text", "expected"),+ [+ (400, _AZURE_ERROR, True),+ (400, "Unsupported parameter: 'max_tokens'. Use 'max_completion_tokens'.", True),+ (500, _AZURE_ERROR, False),+ (+ 400,+ json.dumps({"error": {"message": "max_tokens too large", "param": "max_tokens"}}),+ False,+ ),+ (400, json.dumps({"error": {"message": "bad temperature", "param": "temperature"}}), False),+ ],+)+def test_rejects_max_tokens(status, text, expected):+ assert _rejects_max_tokens(status, text) is expected+++class _Handler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def log_message(self, *args, **kwargs) -> None:+ return++ def _send(self, status: int, payload: bytes, content_type: str) -> None:+ self.send_response(status)+ self.send_header("Content-Type", content_type)+ self.send_header("Content-Length", str(len(payload)))+ self.end_headers()+ self.wfile.write(payload)++ def do_POST(self) -> None: # noqa: N802+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))+ self.server.recorded.append(body) # type: ignore[attr-defined]+ if "max_tokens" in body:+ self._send(400, self.server.error.encode(), "application/json") # type: ignore[attr-defined]+ elif body.get("stream"):+ sse = 'data: {"choices":[{"index":0,"delta":{"content":"ok"}}]}\n\n' "data: [DONE]\n\n"+ self._send(200, sse.encode(), "text/event-stream")+ else:+ reply = {"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}}]}+ self._send(200, json.dumps(reply).encode(), "application/json")+++def gateway(monkeypatch):+ httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)+ httpd.recorded = [] # type: ignore[attr-defined]+ httpd.error = _AZURE_ERROR # type: ignore[attr-defined]+ thread = threading.Thread(target = httpd.serve_forever, daemon = True)+ thread.start()+ try:+ yield httpd+ finally:+ httpd.shutdown()+ httpd.server_close()+ thread.join(timeout = 10)+++def _run(coro_fn):+ async def wrapper():+ async with httpx.AsyncClient(trust_env = False) as http:+ ep_mod._http_client = http+ return await coro_fn()++ previous = ep_mod._http_client+ try:+ return asyncio.run(wrapper())+ finally:+ ep_mod._http_client = previous+++def _client(httpd) -> ExternalProviderClient:+ base_url = f"http://127.0.0.1:{httpd.server_address[1]}/v1"+ return ExternalProviderClient(provider_type = "custom", base_url = base_url, api_key = "")+++def _stream(client) -> str:+ async def go():+ out = []+ async for line in client.stream_chat_completion(+ messages = [{"role": "user", "content": "hi"}],+ model = "GPT5-Mitarbeitende",+ max_tokens = 128,+ ):+ out.append(line)+ return "\n".join(out)++ return _run(go)+++def test_stream_retries_with_max_completion_tokens(gateway):+ out = _stream(_client(gateway))+ assert [b.get("max_tokens") for b in gateway.recorded] == [128, None]+ assert gateway.recorded[1]["max_completion_tokens"] == 128+ assert '"content":"ok"' in out and "unsupported_parameter" not in out+++def test_stream_other_400_is_not_retried(gateway):+ gateway.error = json.dumps({"error": {"message": "bad request", "param": "messages"}})+ out = _stream(_client(gateway))+ assert len(gateway.recorded) == 1+ assert "bad request" in out+++def test_non_stream_retries_with_max_completion_tokens(gateway):+ client = _client(gateway)+ reply = _run(+ lambda: client.chat_completion(+ messages = [{"role": "user", "content": "hi"}], model = "gpt-5", max_tokens = 64+ )+ )+ assert reply["choices"][0]["message"]["content"] == "ok"+ assert [b.get("max_completion_tokens") for b in gateway.recorded] == [None, 64]