Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop cutting Claude answers at 1,024 tokens without a limit (#16953)
Nilay183f507ddb37
diff --git a/studio/backend/core/inference/external_provider.py b/studio/backend/core/inference/external_provider.pyindex 586bcd973fe..256dfeffc2f 100644--- a/studio/backend/core/inference/external_provider.py+++ b/studio/backend/core/inference/external_provider.py@@ -19,7 +19,7 @@ import uuidimport weakrefimport wavefrom typing import Any, AsyncGenerator, Awaitable, Callable, Literal, NamedTuple, Optional, Union-from urllib.parse import urlparse, urlsplit, urlunsplit+from urllib.parse import quote, urlparse, urlsplit, urlunsplitimport httpximport structlog@@ -1156,6 +1156,8 @@ _OPENROUTER_CATALOG_RETRY_S = 60.0# cache key -> (expires_at, models)_openrouter_catalog: dict[str, tuple[float, list[dict[str, Any]]]] = {}_openrouter_catalog_locks: dict[str, asyncio.Lock] = {}+_anthropic_output_caps: dict[tuple[str, str, str], Optional[int]] = {}+_ANTHROPIC_DEFAULT_MAX_TOKENS = 32000_OPENROUTER_MODEL_KEYS = ("id","context_length",@@ -2559,6 +2561,30 @@ class ExternalProviderClient:self.provider_type,)+ async def _anthropic_output_cap(self, model: str) -> Optional[int]:+ key = (self.base_url, self._credential_digest(), model)+ if key not in _anthropic_output_caps:+ try:+ response = await _client().get(+ _append_provider_path(self.base_url, f"/models/{quote(model, safe = '')}"),+ headers = self._auth_headers(),+ timeout = 10,+ )+ if response.status_code == 429 or response.status_code >= 500:+ return None+ cap = response.json().get("max_tokens") if response.is_success else None+ except httpx.HTTPError:+ return None+ except (ValueError, AttributeError):+ cap = None+ # The full cap (up to 128k) plus a long prompt can exceed the context window, which Claude rejects.+ _anthropic_output_caps[key] = (+ min(cap, _ANTHROPIC_DEFAULT_MAX_TOKENS)+ if isinstance(cap, int) and not isinstance(cap, bool) and cap > 0+ else None+ )+ return _anthropic_output_caps[key]+async def _stream_anthropic(self,messages: list[dict[str, Any]],@@ -2884,7 +2910,7 @@ class ExternalProviderClient:body: dict[str, Any] = {"model": model,"messages": filtered,- "max_tokens": max_tokens or 1024, # required by Anthropic+ "max_tokens": max_tokens or await self._anthropic_output_cap(model) or 1024,"stream": True,}if not sampling_removed:diff --git a/studio/backend/tests/test_anthropic_default_max_tokens.py b/studio/backend/tests/test_anthropic_default_max_tokens.pynew file mode 100644index 00000000000..e980dd89d3e--- /dev/null+++ b/studio/backend/tests/test_anthropic_default_max_tokens.py@@ -0,0 +1,141 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++import asyncio+import json++import httpx++from core.inference import external_provider as ep_mod+from core.inference.external_provider import ExternalProviderClient++_STOP = b'event: message_stop\ndata: {"type": "message_stop"}\n\n'+++def _sent_max_tokens(+ monkeypatch,+ models_response,+ max_tokens = None,+ model = "claude-haiku-4-5",+):+ sent: list = []+ model_lookups: list = []++ def handler(request: httpx.Request) -> httpx.Response:+ if request.url.path.startswith("/v1/models/"):+ model_lookups.append(request.url.raw_path.decode())+ return models_response+ sent.append(json.loads(request.content)["max_tokens"])+ return httpx.Response(200, content = _STOP, headers = {"content-type": "text/event-stream"})++ monkeypatch.setattr(ep_mod, "_anthropic_output_caps", {}, raising = False)+ monkeypatch.setattr(+ ep_mod, "_http_client", httpx.AsyncClient(transport = httpx.MockTransport(handler))+ )++ async def run():+ client = ExternalProviderClient(+ provider_type = "anthropic",+ base_url = "https://api.anthropic.com/v1",+ api_key = "sk-ant-test",+ )+ for _ in range(2):+ async for _ in client.stream_chat_completion(+ messages = [{"role": "user", "content": "hi"}],+ model = model,+ max_tokens = max_tokens,+ ):+ pass+ await client.close()++ asyncio.new_event_loop().run_until_complete(run())+ return sent, model_lookups+++def test_omitted_max_tokens_uses_the_models_output_limit(monkeypatch):+ sent, lookups = _sent_max_tokens(+ monkeypatch,+ httpx.Response(200, json = {"id": "claude-haiku-4-5", "max_tokens": 16000}),+ )+ assert sent == [16000, 16000]+ assert lookups == ["/v1/models/claude-haiku-4-5"]+++def test_large_output_limit_is_capped_to_leave_room_for_the_prompt(monkeypatch):+ sent, _ = _sent_max_tokens(+ monkeypatch,+ httpx.Response(200, json = {"id": "claude-opus-4-7", "max_tokens": 128000}),+ model = "claude-opus-4-7",+ )+ assert sent == [32000, 32000]+++def test_explicit_max_tokens_is_sent_unchanged(monkeypatch):+ sent, lookups = _sent_max_tokens(+ monkeypatch,+ httpx.Response(200, json = {"id": "claude-haiku-4-5", "max_tokens": 64000}),+ max_tokens = 300,+ )+ assert sent == [300, 300]+ assert lookups == []+++def test_endpoint_without_model_lookup_is_asked_once(monkeypatch):+ sent, lookups = _sent_max_tokens(monkeypatch, httpx.Response(404, json = {"type": "error"}))+ assert sent == [1024, 1024]+ assert len(lookups) == 1+++def test_server_error_on_lookup_is_retried_next_time(monkeypatch):+ sent, lookups = _sent_max_tokens(monkeypatch, httpx.Response(503))+ assert sent == [1024, 1024]+ assert len(lookups) == 2+++def test_rate_limited_lookup_is_retried_next_time(monkeypatch):+ sent, lookups = _sent_max_tokens(monkeypatch, httpx.Response(429))+ assert sent == [1024, 1024]+ assert len(lookups) == 2+++def test_model_id_is_url_encoded_in_the_lookup(monkeypatch):+ _, lookups = _sent_max_tokens(+ monkeypatch,+ httpx.Response(200, json = {"max_tokens": 8192}),+ model = "proxy/claude-haiku-4-5",+ )+ assert lookups == ["/v1/models/proxy%2Fclaude-haiku-4-5"]+++def test_output_limit_cache_is_scoped_to_credentials(monkeypatch):+ caps = {"sk-a": 16000, "sk-b": 8192}+ sent: list = []++ def handler(request: httpx.Request) -> httpx.Response:+ key = request.headers["x-api-key"]+ if request.url.path.startswith("/v1/models/"):+ return httpx.Response(200, json = {"max_tokens": caps[key]})+ sent.append((key, json.loads(request.content)["max_tokens"]))+ return httpx.Response(200, content = _STOP, headers = {"content-type": "text/event-stream"})++ monkeypatch.setattr(ep_mod, "_anthropic_output_caps", {}, raising = False)+ monkeypatch.setattr(+ ep_mod, "_http_client", httpx.AsyncClient(transport = httpx.MockTransport(handler))+ )++ async def run():+ for key in ("sk-a", "sk-b"):+ client = ExternalProviderClient(+ provider_type = "anthropic",+ base_url = "https://gateway.example/v1",+ api_key = key,+ )+ async for _ in client.stream_chat_completion(+ messages = [{"role": "user", "content": "hi"}],+ model = "claude-haiku-4-5",+ ):+ pass+ await client.close()++ asyncio.new_event_loop().run_until_complete(run())+ assert sent == [("sk-a", 16000), ("sk-b", 8192)]