Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop cutting Claude answers at 1,024 tokens without a limit (#16953)

Nilay183f507ddb37
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.py
index 586bcd973fe..256dfeffc2f 100644
--- a/studio/backend/core/inference/external_provider.py
+++ b/studio/backend/core/inference/external_provider.py
@@ -19,7 +19,7 @@ import uuid
import weakref
import wave
from typing import Any, AsyncGenerator, Awaitable, Callable, Literal, NamedTuple, Optional, Union
-from urllib.parse import urlparse, urlsplit, urlunsplit
+from urllib.parse import quote, urlparse, urlsplit, urlunsplit
import httpx
import structlog
@@ -1156,6 +1156,8 @@ _OPENROUTER_CATALOG_RETRY_S = 60.0
# cache key -> (expires_at, models)
_openrouter_catalog: dict[str, tuple[float, list[dict[str, Any]]]] = {}
_openrouter_catalog_locks: dict[str, asyncio.Lock] = {}
+_anthropic_output_caps: dict[tuple[str, str, str], Optional[int]] = {}
+_ANTHROPIC_DEFAULT_MAX_TOKENS = 32000
_OPENROUTER_MODEL_KEYS = (
"id",
"context_length",
@@ -2559,6 +2561,30 @@ class ExternalProviderClient:
self.provider_type,
)
+ async def _anthropic_output_cap(self, model: str) -> Optional[int]:
+ key = (self.base_url, self._credential_digest(), model)
+ if key not in _anthropic_output_caps:
+ try:
+ response = await _client().get(
+ _append_provider_path(self.base_url, f"/models/{quote(model, safe = '')}"),
+ headers = self._auth_headers(),
+ timeout = 10,
+ )
+ if response.status_code == 429 or response.status_code >= 500:
+ return None
+ cap = response.json().get("max_tokens") if response.is_success else None
+ except httpx.HTTPError:
+ return None
+ except (ValueError, AttributeError):
+ cap = None
+ # The full cap (up to 128k) plus a long prompt can exceed the context window, which Claude rejects.
+ _anthropic_output_caps[key] = (
+ min(cap, _ANTHROPIC_DEFAULT_MAX_TOKENS)
+ if isinstance(cap, int) and not isinstance(cap, bool) and cap > 0
+ else None
+ )
+ return _anthropic_output_caps[key]
+
async def _stream_anthropic(
self,
messages: list[dict[str, Any]],
@@ -2884,7 +2910,7 @@ class ExternalProviderClient:
body: dict[str, Any] = {
"model": model,
"messages": filtered,
- "max_tokens": max_tokens or 1024, # required by Anthropic
+ "max_tokens": max_tokens or await self._anthropic_output_cap(model) or 1024,
"stream": True,
}
if not sampling_removed:
diff --git a/studio/backend/tests/test_anthropic_default_max_tokens.py b/studio/backend/tests/test_anthropic_default_max_tokens.py
new file mode 100644
index 00000000000..e980dd89d3e
--- /dev/null
+++ b/studio/backend/tests/test_anthropic_default_max_tokens.py
@@ -0,0 +1,141 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import asyncio
+import json
+
+import httpx
+
+from core.inference import external_provider as ep_mod
+from core.inference.external_provider import ExternalProviderClient
+
+_STOP = b'event: message_stop\ndata: {"type": "message_stop"}\n\n'
+
+
+def _sent_max_tokens(
+ monkeypatch,
+ models_response,
+ max_tokens = None,
+ model = "claude-haiku-4-5",
+):
+ sent: list = []
+ model_lookups: list = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ if request.url.path.startswith("/v1/models/"):
+ model_lookups.append(request.url.raw_path.decode())
+ return models_response
+ sent.append(json.loads(request.content)["max_tokens"])
+ return httpx.Response(200, content = _STOP, headers = {"content-type": "text/event-stream"})
+
+ monkeypatch.setattr(ep_mod, "_anthropic_output_caps", {}, raising = False)
+ monkeypatch.setattr(
+ ep_mod, "_http_client", httpx.AsyncClient(transport = httpx.MockTransport(handler))
+ )
+
+ async def run():
+ client = ExternalProviderClient(
+ provider_type = "anthropic",
+ base_url = "https://api.anthropic.com/v1",
+ api_key = "sk-ant-test",
+ )
+ for _ in range(2):
+ async for _ in client.stream_chat_completion(
+ messages = [{"role": "user", "content": "hi"}],
+ model = model,
+ max_tokens = max_tokens,
+ ):
+ pass
+ await client.close()
+
+ asyncio.new_event_loop().run_until_complete(run())
+ return sent, model_lookups
+
+
+def test_omitted_max_tokens_uses_the_models_output_limit(monkeypatch):
+ sent, lookups = _sent_max_tokens(
+ monkeypatch,
+ httpx.Response(200, json = {"id": "claude-haiku-4-5", "max_tokens": 16000}),
+ )
+ assert sent == [16000, 16000]
+ assert lookups == ["/v1/models/claude-haiku-4-5"]
+
+
+def test_large_output_limit_is_capped_to_leave_room_for_the_prompt(monkeypatch):
+ sent, _ = _sent_max_tokens(
+ monkeypatch,
+ httpx.Response(200, json = {"id": "claude-opus-4-7", "max_tokens": 128000}),
+ model = "claude-opus-4-7",
+ )
+ assert sent == [32000, 32000]
+
+
+def test_explicit_max_tokens_is_sent_unchanged(monkeypatch):
+ sent, lookups = _sent_max_tokens(
+ monkeypatch,
+ httpx.Response(200, json = {"id": "claude-haiku-4-5", "max_tokens": 64000}),
+ max_tokens = 300,
+ )
+ assert sent == [300, 300]
+ assert lookups == []
+
+
+def test_endpoint_without_model_lookup_is_asked_once(monkeypatch):
+ sent, lookups = _sent_max_tokens(monkeypatch, httpx.Response(404, json = {"type": "error"}))
+ assert sent == [1024, 1024]
+ assert len(lookups) == 1
+
+
+def test_server_error_on_lookup_is_retried_next_time(monkeypatch):
+ sent, lookups = _sent_max_tokens(monkeypatch, httpx.Response(503))
+ assert sent == [1024, 1024]
+ assert len(lookups) == 2
+
+
+def test_rate_limited_lookup_is_retried_next_time(monkeypatch):
+ sent, lookups = _sent_max_tokens(monkeypatch, httpx.Response(429))
+ assert sent == [1024, 1024]
+ assert len(lookups) == 2
+
+
+def test_model_id_is_url_encoded_in_the_lookup(monkeypatch):
+ _, lookups = _sent_max_tokens(
+ monkeypatch,
+ httpx.Response(200, json = {"max_tokens": 8192}),
+ model = "proxy/claude-haiku-4-5",
+ )
+ assert lookups == ["/v1/models/proxy%2Fclaude-haiku-4-5"]
+
+
+def test_output_limit_cache_is_scoped_to_credentials(monkeypatch):
+ caps = {"sk-a": 16000, "sk-b": 8192}
+ sent: list = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ key = request.headers["x-api-key"]
+ if request.url.path.startswith("/v1/models/"):
+ return httpx.Response(200, json = {"max_tokens": caps[key]})
+ sent.append((key, json.loads(request.content)["max_tokens"]))
+ return httpx.Response(200, content = _STOP, headers = {"content-type": "text/event-stream"})
+
+ monkeypatch.setattr(ep_mod, "_anthropic_output_caps", {}, raising = False)
+ monkeypatch.setattr(
+ ep_mod, "_http_client", httpx.AsyncClient(transport = httpx.MockTransport(handler))
+ )
+
+ async def run():
+ for key in ("sk-a", "sk-b"):
+ client = ExternalProviderClient(
+ provider_type = "anthropic",
+ base_url = "https://gateway.example/v1",
+ api_key = key,
+ )
+ async for _ in client.stream_chat_completion(
+ messages = [{"role": "user", "content": "hi"}],
+ model = "claude-haiku-4-5",
+ ):
+ pass
+ await client.close()
+
+ asyncio.new_event_loop().run_until_complete(run())
+ assert sent == [("sk-a", 16000), ("sk-b", 8192)]