Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Fix NIM reasoning budget fallback (#968)

Ali Khokhar4601b80a3666
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index aedff70b..26dc0373 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -356,6 +356,9 @@ DeepSeek attachment/tool/thinking compatibility. DeepSeek intentionally uses its
OpenAI-compatible Chat Completions endpoint because that is the endpoint that
reports prompt-cache hit/miss counters; the provider maps those counters back
into Anthropic usage fields for Claude-compatible clients.
+NIM reasoning budget control is also treated as a provider-owned best-effort
+downgrade: if an upstream NIM deployment rejects explicit budget control, FCC
+retries without the budget while preserving thinking enablement.
Shared provider responsibilities include upstream rate limiting, model listing,
safe error mapping, transport cleanup, thinking/tool handling, retry or recovery
diff --git a/providers/nvidia_nim/client.py b/providers/nvidia_nim/client.py
index 5650c6aa..d7a9f9e1 100644
--- a/providers/nvidia_nim/client.py
+++ b/providers/nvidia_nim/client.py
@@ -56,8 +56,9 @@ class NvidiaNimProvider(OpenAIChatTransport):
def _get_retry_request_body(self, error: Exception, body: dict) -> dict | None:
"""Retry once with a downgraded body when NIM rejects a known field."""
status_code = getattr(error, "status_code", None)
- if not isinstance(error, openai.BadRequestError) and status_code != 400:
- return None
+ bad_request_like = isinstance(error, openai.BadRequestError) or (
+ status_code == 400
+ )
error_text = str(error)
error_body = getattr(error, "body", None)
@@ -65,15 +66,20 @@ class NvidiaNimProvider(OpenAIChatTransport):
error_text = f"{error_text} {json.dumps(error_body, default=str)}"
error_text = error_text.lower()
- if "reasoning_budget" in error_text:
+ if _is_reasoning_budget_rejection(error_text) and (
+ bad_request_like or status_code == 500
+ ):
retry_body = clone_body_without_reasoning_budget(body)
if retry_body is None:
return None
logger.warning(
- "NIM_STREAM: retrying without reasoning_budget after 400 error"
+ "NIM_STREAM: retrying without reasoning budget after upstream rejection"
)
return retry_body
+ if not bad_request_like:
+ return None
+
if "chat_template" in error_text:
retry_body = clone_body_without_chat_template(body)
if retry_body is None:
@@ -91,3 +97,10 @@ class NvidiaNimProvider(OpenAIChatTransport):
return retry_body
return None
+
+
+def _is_reasoning_budget_rejection(error_text: str) -> bool:
+ """Return whether NIM rejected optional thinking budget control."""
+ if "reasoning_budget" in error_text:
+ return True
+ return "thinking_token_budget" in error_text and "reasoning_config" in error_text
diff --git a/pyproject.toml b/pyproject.toml
index 44ffd87b..ff10a6d1 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "free-claude-code"
-version = "2.4.2"
+version = "2.4.3"
description = "Middleware between Claude Code CLI (Anthropic API) and NVIDIA NIM"
readme = "README.md"
requires-python = ">=3.14.0"
diff --git a/tests/providers/test_nvidia_nim.py b/tests/providers/test_nvidia_nim.py
index 2efa975c..83467f8d 100644
--- a/tests/providers/test_nvidia_nim.py
+++ b/tests/providers/test_nvidia_nim.py
@@ -97,6 +97,21 @@ def _make_bad_request_error(message: str) -> openai.BadRequestError:
return openai.BadRequestError(message, response=response, body=body)
+def _make_internal_server_error(message: str) -> openai.InternalServerError:
+ response = Response(
+ status_code=500,
+ request=Request("POST", f"{NVIDIA_NIM_DEFAULT_BASE}/chat/completions"),
+ )
+ body = {
+ "error": {
+ "message": message,
+ "type": "internal_server_error",
+ "code": 500,
+ }
+ }
+ return openai.InternalServerError(message, response=response, body=body)
+
+
@pytest.fixture(autouse=True)
def mock_rate_limiter():
"""Mock the global rate limiter to prevent waiting."""
@@ -712,6 +727,51 @@ async def test_stream_response_retries_without_reasoning_budget(nim_provider):
assert any("message_stop" in event for event in events)
+async def test_stream_response_retries_without_budget_for_thinking_token_error(
+ nim_provider,
+):
+ req = MockRequest(model="meta/llama-3.3-70b-instruct")
+
+ mock_chunk = MagicMock()
+ mock_chunk.choices = [
+ MagicMock(
+ delta=MagicMock(content="Recovered", reasoning_content=""),
+ finish_reason="stop",
+ )
+ ]
+ mock_chunk.usage = MagicMock(completion_tokens=5)
+
+ async def mock_stream():
+ yield mock_chunk
+
+ error = _make_internal_server_error(
+ "ValueError: thinking_token_budget is set but reasoning_config is not "
+ "configured. Please set --reasoning-config to use thinking_token_budget."
+ )
+
+ with patch.object(
+ nim_provider._client.chat.completions, "create", new_callable=AsyncMock
+ ) as mock_create:
+ mock_create.side_effect = [error, mock_stream()]
+
+ events = [e async for e in nim_provider.stream_response(req)]
+
+ assert mock_create.await_count == 2
+ first_call = mock_create.await_args_list[0].kwargs
+ second_call = mock_create.await_args_list[1].kwargs
+ assert (
+ first_call["extra_body"]["chat_template_kwargs"]["reasoning_budget"]
+ == first_call["max_tokens"]
+ )
+ assert "reasoning_budget" not in second_call["extra_body"]
+ assert "reasoning_budget" not in second_call["extra_body"]["chat_template_kwargs"]
+ assert second_call["extra_body"]["chat_template_kwargs"]["thinking"] is True
+ assert second_call["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True
+ assert any("Recovered" in event for event in events)
+ assert any("message_stop" in event for event in events)
+
+
@pytest.mark.asyncio
async def test_stream_response_retries_without_reasoning_content(nim_provider):
req = MockRequest(
@@ -780,3 +840,43 @@ async def test_stream_response_bad_request_without_reasoning_budget_does_not_ret
assert mock_create.await_count == 1
assert any("Invalid request sent to provider" in event for event in events)
assert any("message_stop" in event for event in events)
+
+
+async def test_stream_response_unrelated_internal_error_does_not_downgrade(
+ nim_provider,
+):
+ req = MockRequest()
+ error = _make_internal_server_error("unrelated internal provider failure")
+
+ with patch.object(
+ nim_provider._client.chat.completions, "create", new_callable=AsyncMock
+ ) as mock_create:
+ mock_create.side_effect = error
+
+ events = [e async for e in nim_provider.stream_response(req)]
+
+ assert mock_create.await_count == 1
+ assert any("Provider API request failed" in event for event in events)
+ assert any("message_stop" in event for event in events)
+
+
+async def test_stream_response_internal_reasoning_content_error_does_not_downgrade(
+ nim_provider,
+):
+ req = MockRequest()
+ error = _make_internal_server_error(
+ "reasoning_content could not be processed by the upstream model"
+ )
+
+ with patch.object(
+ nim_provider._client.chat.completions, "create", new_callable=AsyncMock
+ ) as mock_create:
+ mock_create.side_effect = error
+
+ events = [e async for e in nim_provider.stream_response(req)]
+
+ assert mock_create.await_count == 1
+ assert any("Provider API request failed" in event for event in events)
+ assert any("message_stop" in event for event in events)
diff --git a/uv.lock b/uv.lock
index 5df3ea24..01823ba3 100644
--- a/uv.lock
+++ b/uv.lock
@@ -561,7 +561,7 @@ wheels = [
[[package]]
name = "free-claude-code"
-version = "2.4.2"
+version = "2.4.3"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },