Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Fix HTTP 400 when max_(completion_)tokens exceeds a model's cap (#955) (#991)
newmemories360418f4963e5cf
diff --git a/providers/transports/openai_chat/output_cap.py b/providers/transports/openai_chat/output_cap.pynew file mode 100644index 00000000..c5a13ef9--- /dev/null+++ b/providers/transports/openai_chat/output_cap.py@@ -0,0 +1,83 @@+"""Recover from upstream ``max_(completion_)tokens`` too-large 400 rejections.++Some OpenAI-compatible providers (Groq, NVIDIA NIM, ...) cap the per-request+output token count below what Claude Code asks for and reject the whole request+with an HTTP 400 that names the allowed maximum, e.g.::++ max_completion_tokens must be less than or equal to 40960, ...++This module parses that maximum and clamps the request body so the transport can+retry once and succeed. The transport also remembers the learned cap per model+so later requests clamp proactively instead of paying the 400 every time.+"""++import json+import re+from typing import Any++import openai++# Body keys that carry the output-token budget across OpenAI-compatible policies.+_OUTPUT_TOKEN_FIELDS = ("max_completion_tokens", "max_tokens")++# Only treat a 400 as an output-cap rejection when it names one of these fields.+_OUTPUT_TOKEN_KEYWORDS = ("max_completion_tokens", "max_tokens")++# Comparator phrases that precede the allowed maximum in provider error text.+_CAP_PATTERNS: tuple[re.Pattern[str], ...] = (+ re.compile(r"less than or equal to\s+(\d+)"),+ re.compile(r"smaller than or equal to\s+(\d+)"),+ re.compile(r"<=\s*(\d+)"),+ re.compile(r"at most\s+(\d+)"),+ re.compile(r"must not exceed\s+(\d+)"),+ re.compile(r"maximum(?:\s+value)?(?:\s+for\s+\S+)?\s+is\s+(\d+)"),+ re.compile(r"maximum(?:\s+allowed)?(?:\s+value)?\s+of\s+(\d+)"),+)+++def _is_bad_request(error: Exception) -> bool:+ return isinstance(error, openai.BadRequestError) or (+ getattr(error, "status_code", None) == 400+ )+++def _error_text(error: Exception) -> str:+ text = str(error)+ body = getattr(error, "body", None)+ if body is not None:+ text = f"{text} {json.dumps(body, default=str)}"+ return text.lower()+++def parse_output_token_cap(error: Exception) -> int | None:+ """Return the allowed output-token maximum named in a 400 rejection, if any."""+ if not _is_bad_request(error):+ return None++ text = _error_text(error)+ if not any(keyword in text for keyword in _OUTPUT_TOKEN_KEYWORDS):+ return None++ for pattern in _CAP_PATTERNS:+ match = pattern.search(text)+ if match:+ cap = int(match.group(1))+ if cap > 0:+ return cap+ return None+++def clamp_output_tokens(body: dict[str, Any], cap: int) -> dict[str, Any] | None:+ """Return a shallow clone with output-token fields clamped to ``cap``.++ Returns ``None`` when nothing needs clamping (no output field exceeds the+ cap), so callers can avoid a pointless identical retry.+ """+ clamped: dict[str, Any] | None = None+ for field in _OUTPUT_TOKEN_FIELDS:+ value = body.get(field)+ if isinstance(value, int) and not isinstance(value, bool) and value > cap:+ if clamped is None:+ clamped = dict(body)+ clamped[field] = cap+ return clampeddiff --git a/providers/transports/openai_chat/transport.py b/providers/transports/openai_chat/transport.pyindex f40665a1..e3726c17 100644--- a/providers/transports/openai_chat/transport.py+++ b/providers/transports/openai_chat/transport.py@@ -5,6 +5,7 @@ from collections.abc import AsyncIterator, Iterator, Mappingfrom typing import Anyimport httpx+from loguru import loggerfrom openai import AsyncOpenAIfrom core.anthropic.streaming import AnthropicStreamLedger@@ -17,6 +18,7 @@ from providers.error_mapping import (from providers.model_listing import extract_openai_model_idsfrom providers.rate_limit import GlobalRateLimiter+from .output_cap import clamp_output_tokens, parse_output_token_capfrom .stream import OpenAIChatStreamAdapter@@ -36,6 +38,9 @@ class OpenAIChatTransport(BaseProvider):self._provider_name = provider_nameself._api_key = api_keyself._base_url = base_url.rstrip("/")+ # Learned per-model output-token caps from upstream 400 rejections, so+ # later requests clamp proactively instead of paying the 400 each time.+ self._model_output_caps: dict[str, int] = {}self._global_rate_limiter = GlobalRateLimiter.get_scoped_instance(provider_name.lower(),rate_limit=config.rate_limit,@@ -113,6 +118,7 @@ class OpenAIChatTransport(BaseProvider):async def _create_stream(self, body: dict) -> tuple[Any, dict]:"""Create a streaming chat completion, optionally retrying once."""+ body = self._apply_learned_output_cap(body)try:create_body = self._prepare_create_body(body)stream = await self._global_rate_limiter.execute_with_retry(@@ -120,7 +126,9 @@ class OpenAIChatTransport(BaseProvider):)return stream, bodyexcept Exception as error:- retry_body = self._get_retry_request_body(error, body)+ retry_body = self._retry_body_for_output_cap(error, body)+ if retry_body is None:+ retry_body = self._get_retry_request_body(error, body)if retry_body is None:raise@@ -130,6 +138,37 @@ class OpenAIChatTransport(BaseProvider):)return stream, retry_body+ def _apply_learned_output_cap(self, body: dict) -> dict:+ """Clamp output tokens to a previously learned cap for this model."""+ model = body.get("model")+ if not isinstance(model, str):+ return body+ cap = self._model_output_caps.get(model)+ if cap is None:+ return body+ clamped = clamp_output_tokens(body, cap)+ return clamped if clamped is not None else body++ def _retry_body_for_output_cap(self, error: Exception, body: dict) -> dict | None:+ """Learn an upstream output-token cap from a 400 and clamp for one retry."""+ cap = parse_output_token_cap(error)+ if cap is None:+ return None+ model = body.get("model")+ if isinstance(model, str):+ previous = self._model_output_caps.get(model)+ cap = cap if previous is None else min(previous, cap)+ self._model_output_caps[model] = cap+ clamped = clamp_output_tokens(body, cap)+ if clamped is None:+ return None+ logger.warning(+ "{}_STREAM: clamping output tokens to {} after upstream cap rejection",+ self._provider_name,+ cap,+ )+ return clamped+def _openai_error_message(self, error: Exception, request_id: str | None) -> str:mapped_error = map_error(error, rate_limiter=self._global_rate_limiter)return user_visible_message_for_mapped_provider_error(diff --git a/pyproject.toml b/pyproject.tomlindex 2de42e1b..999ff2fa 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "hatchling.build"[project]name = "free-claude-code"-version = "3.2.0"+version = "3.2.1"description = "Middleware between Claude Code CLI (Anthropic API) and NVIDIA NIM"readme = "README.md"requires-python = ">=3.14.0"diff --git a/tests/providers/test_openai_chat_output_cap.py b/tests/providers/test_openai_chat_output_cap.pynew file mode 100644index 00000000..fdab5e54--- /dev/null+++ b/tests/providers/test_openai_chat_output_cap.py@@ -0,0 +1,207 @@+"""Tests for OpenAI-compatible output-token cap recovery (issue #955).++Covers the pure parse/clamp helpers and the transport behavior that clamps+``max_completion_tokens``/``max_tokens`` to the upstream maximum, retries once,+and learns the cap so later requests clamp proactively.+"""++from contextlib import asynccontextmanager+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from providers.base import ProviderConfig+from providers.groq import GROQ_DEFAULT_BASE, GroqProvider+from providers.transports.openai_chat.output_cap import (+ clamp_output_tokens,+ parse_output_token_cap,+)+++class _BadRequest(Exception):+ """Stand-in for openai.BadRequestError (status_code + optional JSON body)."""++ def __init__(self, message: str, body: object | None = None):+ super().__init__(message)+ self.status_code = 400+ self.body = body+++# --------------------------------------------------------------------------- #+# Pure helpers+# --------------------------------------------------------------------------- #+++def test_parse_cap_from_groq_message():+ error = _BadRequest(+ "max_completion_tokens must be less than or equal to 40960, the maximum "+ "value for max_completion_tokens is less than the context_window for this model"+ )+ assert parse_output_token_cap(error) == 40960+++ "message,expected",+ [+ ("max_tokens: maximum value is 8192", 8192),+ ("max_tokens must not exceed 16000", 16000),+ ("`max_completion_tokens` <= 4096 required", 4096),+ ("max_tokens at most 2048 allowed", 2048),+ ("maximum allowed value of 32768 for max_tokens", 32768),+ ],+)+def test_parse_cap_various_phrasings(message, expected):+ assert parse_output_token_cap(_BadRequest(message)) == expected+++def test_parse_cap_reads_json_body():+ error = _BadRequest(+ "invalid request",+ body={"error": {"param": "max_completion_tokens", "message": "<= 12000"}},+ )+ assert parse_output_token_cap(error) == 12000+++def test_parse_cap_ignores_non_400():+ error = _BadRequest("max_tokens must be less than or equal to 40960")+ error.status_code = 500+ assert parse_output_token_cap(error) is None+++def test_parse_cap_ignores_unrelated_400():+ assert parse_output_token_cap(_BadRequest("temperature must be <= 2")) is None+++def test_parse_cap_returns_none_without_number():+ assert (+ parse_output_token_cap(_BadRequest("max_tokens is larger than allowed")) is None+ )+++def test_clamp_reduces_max_completion_tokens():+ assert clamp_output_tokens({"max_completion_tokens": 64000}, 40960) == {+ "max_completion_tokens": 40960+ }+++def test_clamp_reduces_max_tokens():+ assert clamp_output_tokens({"max_tokens": 100000}, 8192) == {"max_tokens": 8192}+++def test_clamp_noop_when_within_cap_returns_none():+ assert clamp_output_tokens({"max_completion_tokens": 1000}, 40960) is None+++def test_clamp_does_not_mutate_input():+ body = {"max_tokens": 99999, "model": "m"}+ clamped = clamp_output_tokens(body, 8192)+ assert body["max_tokens"] == 99999+ assert clamped is not None+ assert clamped["max_tokens"] == 8192+++def test_clamp_ignores_bool_values():+ assert clamp_output_tokens({"max_tokens": True}, 8192) is None+++# --------------------------------------------------------------------------- #+# Transport integration (via GroqProvider, which uses max_completion_tokens)+# --------------------------------------------------------------------------- #+++class MockMessage:+ def __init__(self, role, content):+ self.role = role+ self.content = content+++class MockRequest:+ def __init__(self, max_tokens=64000):+ self.model = "llama-3.3-70b-versatile"+ self.messages = [MockMessage("user", "Hello")]+ self.max_tokens = max_tokens+ self.temperature = 0.5+ self.top_p = 0.9+ self.system = "System prompt"+ self.stop_sequences = None+ self.tools = []+ self.thinking = MagicMock()+ self.thinking.enabled = False++[email protected](autouse=True)+def mock_rate_limiter():+ @asynccontextmanager+ async def _slot():+ yield++ with patch("providers.transports.openai_chat.transport.GlobalRateLimiter") as mock:+ instance = mock.get_scoped_instance.return_value++ async def _passthrough(fn, *args, **kwargs):+ return await fn(*args, **kwargs)++ instance.execute_with_retry = AsyncMock(side_effect=_passthrough)+ instance.concurrency_slot.side_effect = _slot+ yield instance+++def groq_provider():+ return GroqProvider(+ ProviderConfig(+ api_key="test_groq_key",+ base_url=GROQ_DEFAULT_BASE,+ rate_limit=10,+ rate_window=60,+ enable_thinking=False,+ )+ )+++async def test_create_stream_clamps_and_learns_on_cap_rejection(groq_provider):+ body = groq_provider._build_request_body(MockRequest(max_tokens=64000))+ assert body["max_completion_tokens"] == 64000+ model = body["model"]++ error = _BadRequest("max_completion_tokens must be less than or equal to 40960")+ create = AsyncMock(side_effect=[error, object()])++ with patch.object(groq_provider._client.chat.completions, "create", create):+ _stream, used_body = await groq_provider._create_stream(body)++ assert create.call_count == 2+ assert create.call_args_list[1].kwargs["max_completion_tokens"] == 40960+ assert used_body["max_completion_tokens"] == 40960+ assert groq_provider._model_output_caps[model] == 40960+++async def test_learned_cap_clamps_next_request_without_a_400(groq_provider):+ body = groq_provider._build_request_body(MockRequest(max_tokens=64000))+ model = body["model"]+ groq_provider._model_output_caps[model] = 40960++ create = AsyncMock(return_value=object())+ with patch.object(groq_provider._client.chat.completions, "create", create):+ _stream, used_body = await groq_provider._create_stream(body)++ assert create.call_count == 1+ assert create.call_args.kwargs["max_completion_tokens"] == 40960+ assert used_body["max_completion_tokens"] == 40960+++async def test_unrelated_400_is_not_clamped_and_propagates(groq_provider):+ body = groq_provider._build_request_body(MockRequest(max_tokens=100))+ create = AsyncMock(side_effect=_BadRequest("messages: invalid role 'wizard'"))++ with (+ patch.object(groq_provider._client.chat.completions, "create", create),+ pytest.raises(Exception, match="wizard"),+ ):+ await groq_provider._create_stream(body)++ assert create.call_count == 1+ assert groq_provider._model_output_caps == {}diff --git a/uv.lock b/uv.lockindex fa006ca7..1358a9a9 100644--- a/uv.lock+++ b/uv.lock@@ -561,7 +561,7 @@ wheels = [[[package]]name = "free-claude-code"-version = "3.2.0"+version = "3.2.1"source = { editable = "." }dependencies = [{ name = "aiohttp" },