Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

patch: Use named reasoning effort for NVIDIA NIM (#1914)

Ali Khokharb8d6e6d935d3
diff --git a/src/free_claude_code/providers/nvidia_nim/client.py b/src/free_claude_code/providers/nvidia_nim/client.py
index 441331f9..8afb4ed3 100644
--- a/src/free_claude_code/providers/nvidia_nim/client.py
+++ b/src/free_claude_code/providers/nvidia_nim/client.py
@@ -33,7 +33,6 @@ from .request_options import (
)
from .retry import (
clone_body_without_chat_template,
- clone_body_without_reasoning_budget,
clone_body_without_reasoning_content,
)
from .tool_schema import (
@@ -61,10 +60,7 @@ class NvidiaNimChatBehavior(OpenAIChatBehavior):
@property
def reasoning_off_fields(self) -> tuple[tuple[str, ...], ...]:
- return (
- ("extra_body", "chat_template_kwargs", "thinking"),
- ("extra_body", "chat_template_kwargs", "enable_thinking"),
- )
+ return (("reasoning_effort",),)
@property
def normal_max_tokens(self) -> int | None:
@@ -118,17 +114,6 @@ class NvidiaNimChatBehavior(OpenAIChatBehavior):
error_text = f"{error_text} {json.dumps(error_body, default=str)}"
error_text = error_text.lower()
- if _is_reasoning_budget_rejection(error_text) and (
- bad_request_like or status_code == 500
- ):
- retry_body = clone_body_without_reasoning_budget(body)
- if retry_body is None:
- return None
- logger.warning(
- "NIM_STREAM: retrying without reasoning budget after upstream rejection"
- )
- return retry_body
-
if not bad_request_like:
return None
@@ -215,10 +200,3 @@ def _is_degraded_function(body: Mapping[str, Any]) -> bool:
and function_id
and state.strip() == _DEGRADED_FUNCTION_STATE
)
-
-
-def _is_reasoning_budget_rejection(error_text: str) -> bool:
- """Return whether NIM rejected optional thinking budget control."""
- if "reasoning_budget" in error_text:
- return True
- return "thinking_token_budget" in error_text and "reasoning_config" in error_text
diff --git a/src/free_claude_code/providers/nvidia_nim/request_options.py b/src/free_claude_code/providers/nvidia_nim/request_options.py
index ea43cf79..8c9f045b 100644
--- a/src/free_claude_code/providers/nvidia_nim/request_options.py
+++ b/src/free_claude_code/providers/nvidia_nim/request_options.py
@@ -6,14 +6,21 @@ from typing import Any
from free_claude_code.config.nim import NimSettings
from free_claude_code.core.anthropic import ReasoningReplayMode, set_if_not_none
from free_claude_code.core.anthropic.models import MessagesRequest
-from free_claude_code.core.reasoning import ReasoningControl, ReasoningPolicy
+from free_claude_code.core.reasoning import ReasoningEffort, ReasoningPolicy
from free_claude_code.providers.openai_chat import (
+ NamedEffortReasoning,
OpenAIChatRequestPolicy,
build_openai_chat_request_body,
)
from .tool_schema import sanitize_nim_tool_schemas
+NIM_REASONING = NamedEffortReasoning(
+ tuple((effort, effort.value) for effort in ReasoningEffort),
+ disabled_value="none",
+ enabled_value="high",
+)
+
NIM_REQUEST_POLICY = OpenAIChatRequestPolicy(
provider_name="NIM",
reasoning_replay=ReasoningReplayMode.REASONING_CONTENT,
@@ -106,14 +113,7 @@ def apply_nim_request_options(
if not request_template_kwargs:
extra_body.pop("chat_template_kwargs", None)
- if reasoning.control is ReasoningControl.OFF or reasoning.requests_reasoning:
- chat_template_kwargs = extra_body.setdefault("chat_template_kwargs", {})
- if isinstance(chat_template_kwargs, dict):
- enabled = reasoning.control is not ReasoningControl.OFF
- chat_template_kwargs["thinking"] = enabled
- chat_template_kwargs["enable_thinking"] = enabled
- if enabled and (budget := reasoning.numeric_budget_tokens) is not None:
- chat_template_kwargs["reasoning_budget"] = budget
+ NIM_REASONING.encode(body, reasoning)
_set_extra(extra_body, "top_k", nim.top_k, ignore_value=-1)
_set_extra(extra_body, "min_p", nim.min_p, ignore_value=0.0)
diff --git a/src/free_claude_code/providers/nvidia_nim/retry.py b/src/free_claude_code/providers/nvidia_nim/retry.py
index 60d46b39..08b10dd8 100644
--- a/src/free_claude_code/providers/nvidia_nim/retry.py
+++ b/src/free_claude_code/providers/nvidia_nim/retry.py
@@ -7,11 +7,6 @@ from typing import Any
from free_claude_code.core.history_replay import reasoning_context
-def clone_body_without_reasoning_budget(body: dict[str, Any]) -> dict[str, Any] | None:
- """Clone a request body and strip only reasoning_budget fields."""
- return _clone_strip_extra_body(body, _strip_reasoning_budget_fields)
-
-
def clone_body_without_chat_template(body: dict[str, Any]) -> dict[str, Any] | None:
"""Clone a request body and strip NIM chat-template control fields."""
return _clone_strip_extra_body(body, _strip_chat_template_fields)
@@ -42,17 +37,6 @@ def _clone_strip_extra_body(
return cloned_body
-def _strip_reasoning_budget_fields(extra_body: dict[str, Any]) -> bool:
- removed = extra_body.pop("reasoning_budget", None) is not None
- chat_template_kwargs = extra_body.get("chat_template_kwargs")
- if (
- isinstance(chat_template_kwargs, dict)
- and chat_template_kwargs.pop("reasoning_budget", None) is not None
- ):
- removed = True
- return removed
-
-
def _strip_chat_template_fields(extra_body: dict[str, Any]) -> bool:
removed = extra_body.pop("chat_template", None) is not None
if extra_body.pop("chat_template_kwargs", None) is not None:
diff --git a/tests/providers/test_nim_request_clone.py b/tests/providers/test_nim_request_clone.py
deleted file mode 100644
index 0e825094..00000000
--- a/tests/providers/test_nim_request_clone.py
+++ /dev/null
@@ -1,42 +0,0 @@
-"""Tests for NVIDIA NIM request body cloning helpers."""
-
-from copy import deepcopy
-
-from free_claude_code.providers.nvidia_nim.retry import (
- clone_body_without_reasoning_budget,
-)
-
-
-def test_clone_body_without_reasoning_budget_strips_top_level_and_nested():
- body: dict = {
- "model": "x",
- "extra_body": {
- "reasoning_budget": 99,
- "chat_template_kwargs": {"reasoning_budget": 42, "thinking": True},
- "top_k": 1,
- },
- }
- original_extra = deepcopy(body["extra_body"])
- out = clone_body_without_reasoning_budget(body)
-
- assert out is not None
- assert out["extra_body"]["chat_template_kwargs"] == {"thinking": True}
- assert "reasoning_budget" not in out["extra_body"]
- assert body["extra_body"] == original_extra
-
-
-def test_clone_body_without_reasoning_budget_returns_none_when_unchanged():
- body = {"model": "x", "extra_body": {"top_k": 3}}
- assert clone_body_without_reasoning_budget(body) is None
-
-
-def test_clone_body_without_reasoning_budget_returns_none_without_extra_body():
- assert clone_body_without_reasoning_budget({"model": "y"}) is None
-
-
-def test_clone_body_drops_empty_extra_body_after_strip():
- body = {"model": "z", "extra_body": {"reasoning_budget": 7}}
- out = clone_body_without_reasoning_budget(body)
- assert out is not None
- assert "extra_body" not in out
- assert "extra_body" in body
diff --git a/tests/providers/test_nvidia_nim.py b/tests/providers/test_nvidia_nim.py
index 7a8df114..fe91fe1b 100644
--- a/tests/providers/test_nvidia_nim.py
+++ b/tests/providers/test_nvidia_nim.py
@@ -16,7 +16,7 @@ from free_claude_code.core.history_replay import (
encode_replay,
)
from free_claude_code.core.openai_responses.models import OpenAIResponsesRequest
-from free_claude_code.core.reasoning import ReasoningEffort, ReasoningPolicy
+from free_claude_code.core.reasoning import ReasoningPolicy
from free_claude_code.providers.admission import UPSTREAM_TRANSIENT_TOTAL_ATTEMPTS
from free_claude_code.providers.nvidia_nim import NvidiaNimProvider
from free_claude_code.providers.nvidia_nim.client import _PROFILE as NIM_PROFILE
@@ -165,7 +165,6 @@ def _alias_events():
("responses", None),
("messages", "chat_template"),
("responses", "chat_template"),
- ("messages", "reasoning_budget"),
("messages", "reasoning_content"),
],
)
@@ -398,10 +397,8 @@ async def test_build_request_body(provider_config):
assert body["messages"][0]["content"] == "System prompt"
assert "extra_body" in body
- ctk = body["extra_body"]["chat_template_kwargs"]
- assert ctk["thinking"] is True
- assert ctk["enable_thinking"] is True
- assert "reasoning_budget" not in ctk
+ assert body["reasoning_effort"] == "high"
+ assert "chat_template_kwargs" not in body["extra_body"]
assert "reasoning_budget" not in body["extra_body"]
@@ -454,10 +451,8 @@ def test_responses_request_uses_nim_chat_policy(provider_config):
assert parameters["properties"] == {"_fcc_arg_type": {"type": "string"}}
extra_body = body["extra_body"]
assert isinstance(extra_body, dict)
- assert extra_body["chat_template_kwargs"] == {
- "thinking": True,
- "enable_thinking": True,
- }
+ assert body["reasoning_effort"] == "high"
+ assert "chat_template_kwargs" not in extra_body
@pytest.mark.asyncio
@@ -473,10 +468,8 @@ async def test_build_request_body_encodes_explicit_reasoning_off(
body = provider._chat._build_request_body(req, reasoning=REASONING_OFF)
extra = body.get("extra_body", {})
- assert extra["chat_template_kwargs"] == {
- "thinking": False,
- "enable_thinking": False,
- }
+ assert body["reasoning_effort"] == "none"
+ assert "chat_template_kwargs" not in extra
assert "reasoning_budget" not in extra
@@ -713,10 +706,7 @@ async def test_stream_messages_retries_without_chat_template(provider_config):
second_extra = mock_create.call_args_list[1].kwargs["extra_body"]
assert first_extra["chat_template"] == "custom_template"
- assert first_extra["chat_template_kwargs"] == {
- "thinking": True,
- "enable_thinking": True,
- }
+ assert "chat_template_kwargs" not in first_extra
assert "reasoning_budget" not in first_extra
assert "chat_template" not in second_extra
@@ -737,7 +727,10 @@ async def test_stream_messages_retries_without_chat_template_kwargs_issue_993(
nim_settings=NimSettings(),
admission=immediate_admission(),
)
- req = make_request(model="mistralai/mistral-small-4-119b-2603")
+ req = make_request(
+ model="mistralai/mistral-small-4-119b-2603",
+ extra_body={"chat_template_kwargs": {"custom": "value"}},
+ )
mock_chunk = MagicMock()
mock_chunk.choices = [
@@ -770,10 +763,7 @@ async def test_stream_messages_retries_without_chat_template_kwargs_issue_993(
second_kwargs = mock_create.call_args_list[1].kwargs
assert "chat_template" not in first_extra
- assert first_extra["chat_template_kwargs"] == {
- "thinking": True,
- "enable_thinking": True,
- }
+ assert first_extra["chat_template_kwargs"] == {"custom": "value"}
second_extra = second_kwargs.get("extra_body") or {}
assert "chat_template" not in second_extra
assert "chat_template_kwargs" not in second_extra
@@ -1330,95 +1320,6 @@ async def test_stream_messages_task_tool_still_forces_background_false(nim_provi
assert json.loads(deltas[0])["run_in_background"] is False
-async def test_stream_messages_retries_without_reasoning_budget(nim_provider):
- req = make_request()
-
- mock_chunk = MagicMock()
- mock_chunk.choices = [
- MagicMock(
- delta=MagicMock(content="Recovered", reasoning_content=""),
- finish_reason="stop",
- )
- ]
- mock_chunk.usage = MagicMock(completion_tokens=5)
-
- async def mock_stream():
- yield mock_chunk
-
- error = _make_bad_request_error("Unsupported field: reasoning_budget")
-
- with patch.object(
- nim_provider._client.chat.completions, "create", new_callable=AsyncMock
- ) as mock_create:
- mock_create.side_effect = [error, SDKStreamDouble(mock_stream())]
-
- events = [
- e
- async for e in nim_provider.stream_messages(
- req,
- reasoning=ReasoningPolicy.on(effort=ReasoningEffort.XHIGH),
- )
- ]
-
- assert mock_create.await_count == 2
- first_call = mock_create.await_args_list[0].kwargs
- second_call = mock_create.await_args_list[1].kwargs
- assert first_call["extra_body"]["chat_template_kwargs"]["reasoning_budget"] == 4096
- assert "reasoning_budget" not in second_call["extra_body"]
- assert "reasoning_budget" not in second_call["extra_body"]["chat_template_kwargs"]
- assert second_call["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True
- assert any("Recovered" in event for event in events)
- assert any("message_stop" in event for event in events)
-
-
-async def test_stream_messages_retries_without_budget_for_thinking_token_error(
- nim_provider,
-):
- req = make_request(model="meta/llama-3.3-70b-instruct")
-
- mock_chunk = MagicMock()
- mock_chunk.choices = [
- MagicMock(
- delta=MagicMock(content="Recovered", reasoning_content=""),
- finish_reason="stop",
- )
- ]
- mock_chunk.usage = MagicMock(completion_tokens=5)
-
- async def mock_stream():
- yield mock_chunk
-
- error = _make_internal_server_error(
- "ValueError: thinking_token_budget is set but reasoning_config is not "
- "configured. Please set --reasoning-config to use thinking_token_budget."
- )
-
- with patch.object(
- nim_provider._client.chat.completions, "create", new_callable=AsyncMock
- ) as mock_create:
- mock_create.side_effect = [error, SDKStreamDouble(mock_stream())]
-
- events = [
- e
- async for e in nim_provider.stream_messages(
- req, reasoning=ReasoningPolicy.on(budget_tokens=77)
- )
- ]
-
- assert mock_create.await_count == 2
- first_call = mock_create.await_args_list[0].kwargs
- second_call = mock_create.await_args_list[1].kwargs
- assert first_call["extra_body"]["chat_template_kwargs"]["reasoning_budget"] == 77
- assert "reasoning_budget" not in second_call["extra_body"]
- assert "reasoning_budget" not in second_call["extra_body"]["chat_template_kwargs"]
- assert second_call["extra_body"]["chat_template_kwargs"]["thinking"] is True
- assert second_call["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True
- assert any("Recovered" in event for event in events)
- assert any("message_stop" in event for event in events)
-
-
@pytest.mark.asyncio
async def test_stream_messages_retries_without_reasoning_content(nim_provider):
req = make_request(
diff --git a/tests/providers/test_nvidia_nim_reasoning.py b/tests/providers/test_nvidia_nim_reasoning.py
new file mode 100644
index 00000000..3bba07bd
--- /dev/null
+++ b/tests/providers/test_nvidia_nim_reasoning.py
@@ -0,0 +1,85 @@
+"""NIM named reasoning across both supported ingress protocols."""
+
+import pytest
+
+from free_claude_code.core.failures import ExecutionFailure
+from free_claude_code.core.reasoning import ReasoningEffort, ReasoningPolicy
+from tests.providers.test_history_transports import _harness, _saved_reply
+from tests.providers.test_nvidia_nim import _alias_provider, _alias_request
+
+
[email protected]("wire", ["messages", "responses"])
+ "policy,expected",
+ [
+ *((ReasoningPolicy(effort=effort), effort.value) for effort in ReasoningEffort),
+ (ReasoningPolicy.off(), "none"),
+ (ReasoningPolicy.on(), "high"),
+ (ReasoningPolicy.on(budget_tokens=2048), "high"),
+ (ReasoningPolicy.provider_default(), None),
+ ],
+)
+async def test_named_effort_reaches_nim_without_numeric_budget(wire, policy, expected):
+ def responder(bodies):
+ body = bodies[-1]
+ assert body.get("reasoning_effort") == expected
+ assert "reasoning_budget" not in body
+ assert "chat_template_kwargs" not in body
+ return 200, [
+ {
+ "id": "response",
+ "object": "chat.completion.chunk",
+ "model": "test-model",
+ "choices": [
+ {
+ "index": 0,
+ "delta": {"content": "Hello"},
+ "finish_reason": "stop",
+ }
+ ],
+ }
+ ]
+
+ async with _harness("chat", responder, chat_provider_factory=_alias_provider) as (
+ _,
+ bodies,
+ provider,
+ ):
+ stream = (
+ provider.stream_messages
+ if wire == "messages"
+ else provider.stream_responses
+ )
+ reply = await _saved_reply(stream(_alias_request(wire), reasoning=policy), wire)
+
+ assert reply
+ assert len(bodies) == 1
+
+
[email protected]("wire", ["messages", "responses"])
+ "policy", [ReasoningPolicy.off(), ReasoningPolicy(effort=ReasoningEffort.MAX)]
+)
+async def test_unsupported_named_effort_is_not_silently_remapped(wire, policy):
+ def responder(bodies):
+ return 400, {
+ "message": "reasoning_effort must be one of low, medium, high",
+ "param": "reasoning_effort",
+ }
+
+ async with _harness("chat", responder, chat_provider_factory=_alias_provider) as (
+ _,
+ bodies,
+ provider,
+ ):
+ stream = (
+ provider.stream_messages
+ if wire == "messages"
+ else provider.stream_responses
+ )
+ with pytest.raises(ExecutionFailure, match="reasoning_effort"):
+ await _saved_reply(stream(_alias_request(wire), reasoning=policy), wire)
+
+ assert len(bodies) == 1
diff --git a/tests/providers/test_nvidia_nim_request.py b/tests/providers/test_nvidia_nim_request.py
index 84b22b87..cf022ceb 100644
--- a/tests/providers/test_nvidia_nim_request.py
+++ b/tests/providers/test_nvidia_nim_request.py
@@ -102,32 +102,13 @@ class TestSetExtra:
class TestBuildRequestBody:
- @pytest.mark.parametrize(
- ("effort", "expected_budget"),
- (
- (ReasoningEffort.MINIMAL, 512),
- (ReasoningEffort.LOW, 512),
- (ReasoningEffort.MEDIUM, 1_024),
- (ReasoningEffort.HIGH, 2_048),
- (ReasoningEffort.XHIGH, 4_096),
- (ReasoningEffort.MAX, 8_192),
- ),
- )
- def test_named_effort_enables_thinking_with_numeric_budget(
- self,
- req,
- effort: ReasoningEffort,
- expected_budget: int,
- ):
- policy = ReasoningPolicy(effort=effort)
-
- body = build_request_body(req, NimSettings(), reasoning=policy)
-
- assert body["extra_body"]["chat_template_kwargs"] == {
- "thinking": True,
- "enable_thinking": True,
- "reasoning_budget": expected_budget,
- }
+ @pytest.mark.parametrize("effort", list(ReasoningEffort))
+ def test_named_effort_passes_through(self, req, effort):
+ body = build_request_body(
+ req, NimSettings(), reasoning=ReasoningPolicy(effort=effort)
+ )
+ assert body["reasoning_effort"] == effort.value
+ assert "chat_template_kwargs" not in body["extra_body"]
def test_named_effort_replaces_client_reasoning_budgets(self):
req = make_messages_request(
@@ -150,12 +131,8 @@ class TestBuildRequestBody:
extra_body = body["extra_body"]
assert "reasoning_budget" not in extra_body
- assert extra_body["chat_template_kwargs"] == {
- "custom": "value",
- "thinking": True,
- "enable_thinking": True,
- "reasoning_budget": 2048,
- }
+ assert body["reasoning_effort"] == "high"
+ assert extra_body["chat_template_kwargs"] == {"custom": "value"}
def test_max_tokens_capped_by_nim(self, req):
req.max_tokens = 100000
@@ -377,7 +354,7 @@ class TestBuildRequestBody:
"Grep": {"_fcc_arg_A": "-A"}
}
- def test_reasoning_params_in_extra_body(self):
+ def test_reasoning_uses_named_effort(self):
req = make_messages_request(
model="test",
messages=[{"role": "user", "content": "hi"}],
@@ -396,10 +373,8 @@ class TestBuildRequestBody:
nim = NimSettings()
body = build_request_body(req, nim, reasoning=REASONING_ON)
extra = body["extra_body"]
- assert extra["chat_template_kwargs"] == {
- "thinking": True,
- "enable_thinking": True,
- }
+ assert body["reasoning_effort"] == "high"
+ assert "chat_template_kwargs" not in extra
assert "reasoning_budget" not in extra
def test_canonicalization_removes_empty_client_reasoning_envelope(self):
@@ -493,13 +468,11 @@ class TestBuildRequestBody:
nim = NimSettings()
body = build_request_body(req, nim, reasoning=REASONING_OFF)
extra = body.get("extra_body", {})
- assert extra["chat_template_kwargs"] == {
- "thinking": False,
- "enable_thinking": False,
- }
+ assert body["reasoning_effort"] == "none"
+ assert "chat_template_kwargs" not in extra
assert "reasoning_budget" not in extra
- def test_reasoning_budget_respects_existing_chat_template_kwargs(self):
+ def test_named_effort_preserves_unrelated_chat_template_kwargs(self):
req = make_messages_request(
model="test",
messages=[{"role": "user", "content": "hi"}],
@@ -521,11 +494,8 @@ class TestBuildRequestBody:
)
body = build_request_body(req, NimSettings(), reasoning=REASONING_ON)
- assert body["extra_body"]["chat_template_kwargs"] == {
- "enable_thinking": True,
- "custom": "value",
- "thinking": True,
- }
+ assert body["reasoning_effort"] == "high"
+ assert body["extra_body"]["chat_template_kwargs"] == {"custom": "value"}
def test_chat_template_fields_are_provider_wide(self):
req = make_messages_request(
@@ -546,10 +516,8 @@ class TestBuildRequestBody:
nim = NimSettings(chat_template="custom_template")
body = build_request_body(req, nim, reasoning=REASONING_ON)
extra = body.get("extra_body", {})
- assert extra["chat_template_kwargs"] == {
- "thinking": True,
- "enable_thinking": True,
- }
+ assert body["reasoning_effort"] == "high"
+ assert "chat_template_kwargs" not in extra
assert extra["chat_template"] == "custom_template"
def test_no_reasoning_params_in_extra_body(self):
@@ -579,12 +547,10 @@ class TestBuildRequestBody:
"reasoning_effort",
):
assert param not in extra
- assert extra["chat_template_kwargs"] == {
- "thinking": False,
- "enable_thinking": False,
- }
+ assert body["reasoning_effort"] == "none"
+ assert "chat_template_kwargs" not in extra
- def test_explicit_reasoning_budget_is_preserved_exactly(self):
+ def test_budget_only_request_uses_enabled_named_effort(self):
req = make_messages_request(model="test", thinking=None)
body = build_request_body(
@@ -593,11 +559,9 @@ class TestBuildRequestBody:
reasoning=ReasoningPolicy.on(budget_tokens=321),
)
- assert body["extra_body"]["chat_template_kwargs"] == {
- "thinking": True,
- "enable_thinking": True,
- "reasoning_budget": 321,
- }
+ assert body["reasoning_effort"] == "high"
+ assert "chat_template_kwargs" not in body["extra_body"]
+ assert "reasoning_budget" not in body["extra_body"]
def test_assistant_thinking_blocks_removed_when_disabled(self):
req = make_messages_request(