Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: retry custom gateways with max_completion_tokens after a max_tokens 400 (#10837)

Sourav Rajvi5c239a233d00
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index 0efc50bbf12..c6b932c0001 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -6,6 +6,7 @@
import asyncio
import base64
+import contextlib
import io
import json as _json
import mimetypes
@@ -1020,6 +1021,49 @@ def restore_account_clients(account_id: str) -> None:
_retired_accounts.discard(account_id)
+def _rejects_max_tokens(status_code: int, error_text: str) -> bool:
+ """400 from an upstream that wants `max_completion_tokens` (Azure gpt-5.x / o-series behind custom gateways, #10787)."""
+ if status_code != 400:
+ return False
+ try:
+ err = _json.loads(error_text).get("error")
+ except Exception:
+ err = None
+ if isinstance(err, dict) and err.get("param") == "max_tokens":
+ return err.get("code") == "unsupported_parameter" or "max_completion_tokens" in str(
+ err.get("message", "")
+ )
+ return "max_tokens" in error_text and "max_completion_tokens" in error_text
+
+
+def _with_max_completion_tokens(body: dict[str, Any]) -> dict[str, Any]:
+ body = dict(body)
+ body["max_completion_tokens"] = body.pop("max_tokens")
+ return body
+
+
+async def _stream_post_retrying_max_tokens(
+ http: httpx.AsyncClient, url: str, body: dict[str, Any], **kwargs
+):
+ """`http.stream("POST", ...)` that resends once with `max_completion_tokens` if the upstream rejects `max_tokens`.
+ Nothing has been yielded to the caller at the status check, so the retry is invisible."""
+ async with http.stream("POST", url, json = body, **kwargs) as response:
+ retry = (
+ response.status_code == 400
+ and "max_tokens" in body
+ and _rejects_max_tokens(400, (await response.aread()).decode("utf-8", errors = "replace"))
+ )
+ if not retry:
+ yield response
+ return
+ logger.info("Upstream rejected max_tokens; retrying with max_completion_tokens")
+ async with http.stream(
+ "POST", url, json = _with_max_completion_tokens(body), **kwargs
+ ) as response:
+ yield response
+
+
def _client() -> httpx.AsyncClient:
"""The shared client for the owner; a screening client of its own for each managed account."""
from utils.account_context import current_account_id, is_owner_context
@@ -1762,12 +1806,10 @@ class ExternalProviderClient:
)
try:
- async with (
- _loopback_http_client if getattr(self, "_managed_loopback", False) else _client()
- ).stream(
- "POST",
+ async with _stream_post_retrying_max_tokens(
+ _loopback_http_client if getattr(self, "_managed_loopback", False) else _client(),
url,
- json = body,
+ body,
headers = self._auth_headers(),
timeout = self._stream_timeout,
) as response:
@@ -6799,12 +6841,20 @@ class ExternalProviderClient:
else:
body["max_tokens"] = max_tokens
+ url = f"{self.base_url}/chat/completions"
response = await _client().post(
- f"{self.base_url}/chat/completions",
+ url,
json = body,
headers = self._auth_headers(),
timeout = self._timeout,
)
+ if "max_tokens" in body and _rejects_max_tokens(response.status_code, response.text):
+ response = await _client().post(
+ url,
+ json = _with_max_completion_tokens(body),
+ headers = self._auth_headers(),
+ timeout = self._timeout,
+ )
response.raise_for_status()
return response.json()
diff --git a/studio/backend/tests/test_external_provider_max_tokens_field.py b/studio/backend/tests/test_external_provider_max_tokens_field.py
new file mode 100644
index 00000000000..eea777345b2
--- /dev/null
+++ b/studio/backend/tests/test_external_provider_max_tokens_field.py
@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Regression for #10787: custom gateways that reject max_tokens."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+import httpx
+import pytest
+
+from core.inference import external_provider as ep_mod
+from core.inference.external_provider import ExternalProviderClient, _rejects_max_tokens
+
+_AZURE_ERROR = json.dumps(
+ {
+ "error": {
+ "message": (
+ "Unsupported parameter: 'max_tokens' is not supported with this model. "
+ "Use 'max_completion_tokens' instead."
+ ),
+ "type": "invalid_request_error",
+ "param": "max_tokens",
+ "code": "unsupported_parameter",
+ }
+ }
+)
+
+
+ ("status", "text", "expected"),
+ [
+ (400, _AZURE_ERROR, True),
+ (400, "Unsupported parameter: 'max_tokens'. Use 'max_completion_tokens'.", True),
+ (500, _AZURE_ERROR, False),
+ (
+ 400,
+ json.dumps({"error": {"message": "max_tokens too large", "param": "max_tokens"}}),
+ False,
+ ),
+ (400, json.dumps({"error": {"message": "bad temperature", "param": "temperature"}}), False),
+ ],
+)
+def test_rejects_max_tokens(status, text, expected):
+ assert _rejects_max_tokens(status, text) is expected
+
+
+class _Handler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def log_message(self, *args, **kwargs) -> None:
+ return
+
+ def _send(self, status: int, payload: bytes, content_type: str) -> None:
+ self.send_response(status)
+ self.send_header("Content-Type", content_type)
+ self.send_header("Content-Length", str(len(payload)))
+ self.end_headers()
+ self.wfile.write(payload)
+
+ def do_POST(self) -> None: # noqa: N802
+ body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
+ self.server.recorded.append(body) # type: ignore[attr-defined]
+ if "max_tokens" in body:
+ self._send(400, self.server.error.encode(), "application/json") # type: ignore[attr-defined]
+ elif body.get("stream"):
+ sse = 'data: {"choices":[{"index":0,"delta":{"content":"ok"}}]}\n\n' "data: [DONE]\n\n"
+ self._send(200, sse.encode(), "text/event-stream")
+ else:
+ reply = {"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}}]}
+ self._send(200, json.dumps(reply).encode(), "application/json")
+
+
+def gateway(monkeypatch):
+ httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
+ httpd.recorded = [] # type: ignore[attr-defined]
+ httpd.error = _AZURE_ERROR # type: ignore[attr-defined]
+ thread = threading.Thread(target = httpd.serve_forever, daemon = True)
+ thread.start()
+ try:
+ yield httpd
+ finally:
+ httpd.shutdown()
+ httpd.server_close()
+ thread.join(timeout = 10)
+
+
+def _run(coro_fn):
+ async def wrapper():
+ async with httpx.AsyncClient(trust_env = False) as http:
+ ep_mod._http_client = http
+ return await coro_fn()
+
+ previous = ep_mod._http_client
+ try:
+ return asyncio.run(wrapper())
+ finally:
+ ep_mod._http_client = previous
+
+
+def _client(httpd) -> ExternalProviderClient:
+ base_url = f"http://127.0.0.1:{httpd.server_address[1]}/v1"
+ return ExternalProviderClient(provider_type = "custom", base_url = base_url, api_key = "")
+
+
+def _stream(client) -> str:
+ async def go():
+ out = []
+ async for line in client.stream_chat_completion(
+ messages = [{"role": "user", "content": "hi"}],
+ model = "GPT5-Mitarbeitende",
+ max_tokens = 128,
+ ):
+ out.append(line)
+ return "\n".join(out)
+
+ return _run(go)
+
+
+def test_stream_retries_with_max_completion_tokens(gateway):
+ out = _stream(_client(gateway))
+ assert [b.get("max_tokens") for b in gateway.recorded] == [128, None]
+ assert gateway.recorded[1]["max_completion_tokens"] == 128
+ assert '"content":"ok"' in out and "unsupported_parameter" not in out
+
+
+def test_stream_other_400_is_not_retried(gateway):
+ gateway.error = json.dumps({"error": {"message": "bad request", "param": "messages"}})
+ out = _stream(_client(gateway))
+ assert len(gateway.recorded) == 1
+ assert "bad request" in out
+
+
+def test_non_stream_retries_with_max_completion_tokens(gateway):
+ client = _client(gateway)
+ reply = _run(
+ lambda: client.chat_completion(
+ messages = [{"role": "user", "content": "hi"}], model = "gpt-5", max_tokens = 64
+ )
+ )
+ assert reply["choices"][0]["message"]["content"] == "ok"
+ assert [b.get("max_completion_tokens") for b in gateway.recorded] == [None, 64]