Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
patch: Use named reasoning effort for NVIDIA NIM (#1914)
Ali Khokharb8d6e6d935d3
diff --git a/src/free_claude_code/providers/nvidia_nim/client.py b/src/free_claude_code/providers/nvidia_nim/client.pyindex 441331f9..8afb4ed3 100644--- a/src/free_claude_code/providers/nvidia_nim/client.py+++ b/src/free_claude_code/providers/nvidia_nim/client.py@@ -33,7 +33,6 @@ from .request_options import ()from .retry import (clone_body_without_chat_template,- clone_body_without_reasoning_budget,clone_body_without_reasoning_content,)from .tool_schema import (@@ -61,10 +60,7 @@ class NvidiaNimChatBehavior(OpenAIChatBehavior):@propertydef reasoning_off_fields(self) -> tuple[tuple[str, ...], ...]:- return (- ("extra_body", "chat_template_kwargs", "thinking"),- ("extra_body", "chat_template_kwargs", "enable_thinking"),- )+ return (("reasoning_effort",),)@propertydef normal_max_tokens(self) -> int | None:@@ -118,17 +114,6 @@ class NvidiaNimChatBehavior(OpenAIChatBehavior):error_text = f"{error_text} {json.dumps(error_body, default=str)}"error_text = error_text.lower()- if _is_reasoning_budget_rejection(error_text) and (- bad_request_like or status_code == 500- ):- retry_body = clone_body_without_reasoning_budget(body)- if retry_body is None:- return None- logger.warning(- "NIM_STREAM: retrying without reasoning budget after upstream rejection"- )- return retry_body-if not bad_request_like:return None@@ -215,10 +200,3 @@ def _is_degraded_function(body: Mapping[str, Any]) -> bool:and function_idand state.strip() == _DEGRADED_FUNCTION_STATE)---def _is_reasoning_budget_rejection(error_text: str) -> bool:- """Return whether NIM rejected optional thinking budget control."""- if "reasoning_budget" in error_text:- return True- return "thinking_token_budget" in error_text and "reasoning_config" in error_textdiff --git a/src/free_claude_code/providers/nvidia_nim/request_options.py b/src/free_claude_code/providers/nvidia_nim/request_options.pyindex ea43cf79..8c9f045b 100644--- a/src/free_claude_code/providers/nvidia_nim/request_options.py+++ b/src/free_claude_code/providers/nvidia_nim/request_options.py@@ -6,14 +6,21 @@ from typing import Anyfrom free_claude_code.config.nim import NimSettingsfrom free_claude_code.core.anthropic import ReasoningReplayMode, set_if_not_nonefrom free_claude_code.core.anthropic.models import MessagesRequest-from free_claude_code.core.reasoning import ReasoningControl, ReasoningPolicy+from free_claude_code.core.reasoning import ReasoningEffort, ReasoningPolicyfrom free_claude_code.providers.openai_chat import (+ NamedEffortReasoning,OpenAIChatRequestPolicy,build_openai_chat_request_body,)from .tool_schema import sanitize_nim_tool_schemas+NIM_REASONING = NamedEffortReasoning(+ tuple((effort, effort.value) for effort in ReasoningEffort),+ disabled_value="none",+ enabled_value="high",+)+NIM_REQUEST_POLICY = OpenAIChatRequestPolicy(provider_name="NIM",reasoning_replay=ReasoningReplayMode.REASONING_CONTENT,@@ -106,14 +113,7 @@ def apply_nim_request_options(if not request_template_kwargs:extra_body.pop("chat_template_kwargs", None)- if reasoning.control is ReasoningControl.OFF or reasoning.requests_reasoning:- chat_template_kwargs = extra_body.setdefault("chat_template_kwargs", {})- if isinstance(chat_template_kwargs, dict):- enabled = reasoning.control is not ReasoningControl.OFF- chat_template_kwargs["thinking"] = enabled- chat_template_kwargs["enable_thinking"] = enabled- if enabled and (budget := reasoning.numeric_budget_tokens) is not None:- chat_template_kwargs["reasoning_budget"] = budget+ NIM_REASONING.encode(body, reasoning)_set_extra(extra_body, "top_k", nim.top_k, ignore_value=-1)_set_extra(extra_body, "min_p", nim.min_p, ignore_value=0.0)diff --git a/src/free_claude_code/providers/nvidia_nim/retry.py b/src/free_claude_code/providers/nvidia_nim/retry.pyindex 60d46b39..08b10dd8 100644--- a/src/free_claude_code/providers/nvidia_nim/retry.py+++ b/src/free_claude_code/providers/nvidia_nim/retry.py@@ -7,11 +7,6 @@ from typing import Anyfrom free_claude_code.core.history_replay import reasoning_context-def clone_body_without_reasoning_budget(body: dict[str, Any]) -> dict[str, Any] | None:- """Clone a request body and strip only reasoning_budget fields."""- return _clone_strip_extra_body(body, _strip_reasoning_budget_fields)--def clone_body_without_chat_template(body: dict[str, Any]) -> dict[str, Any] | None:"""Clone a request body and strip NIM chat-template control fields."""return _clone_strip_extra_body(body, _strip_chat_template_fields)@@ -42,17 +37,6 @@ def _clone_strip_extra_body(return cloned_body-def _strip_reasoning_budget_fields(extra_body: dict[str, Any]) -> bool:- removed = extra_body.pop("reasoning_budget", None) is not None- chat_template_kwargs = extra_body.get("chat_template_kwargs")- if (- isinstance(chat_template_kwargs, dict)- and chat_template_kwargs.pop("reasoning_budget", None) is not None- ):- removed = True- return removed--def _strip_chat_template_fields(extra_body: dict[str, Any]) -> bool:removed = extra_body.pop("chat_template", None) is not Noneif extra_body.pop("chat_template_kwargs", None) is not None:diff --git a/tests/providers/test_nim_request_clone.py b/tests/providers/test_nim_request_clone.pydeleted file mode 100644index 0e825094..00000000--- a/tests/providers/test_nim_request_clone.py+++ /dev/null@@ -1,42 +0,0 @@-"""Tests for NVIDIA NIM request body cloning helpers."""--from copy import deepcopy--from free_claude_code.providers.nvidia_nim.retry import (- clone_body_without_reasoning_budget,-)---def test_clone_body_without_reasoning_budget_strips_top_level_and_nested():- body: dict = {- "model": "x",- "extra_body": {- "reasoning_budget": 99,- "chat_template_kwargs": {"reasoning_budget": 42, "thinking": True},- "top_k": 1,- },- }- original_extra = deepcopy(body["extra_body"])- out = clone_body_without_reasoning_budget(body)-- assert out is not None- assert out["extra_body"]["chat_template_kwargs"] == {"thinking": True}- assert "reasoning_budget" not in out["extra_body"]- assert body["extra_body"] == original_extra---def test_clone_body_without_reasoning_budget_returns_none_when_unchanged():- body = {"model": "x", "extra_body": {"top_k": 3}}- assert clone_body_without_reasoning_budget(body) is None---def test_clone_body_without_reasoning_budget_returns_none_without_extra_body():- assert clone_body_without_reasoning_budget({"model": "y"}) is None---def test_clone_body_drops_empty_extra_body_after_strip():- body = {"model": "z", "extra_body": {"reasoning_budget": 7}}- out = clone_body_without_reasoning_budget(body)- assert out is not None- assert "extra_body" not in out- assert "extra_body" in bodydiff --git a/tests/providers/test_nvidia_nim.py b/tests/providers/test_nvidia_nim.pyindex 7a8df114..fe91fe1b 100644--- a/tests/providers/test_nvidia_nim.py+++ b/tests/providers/test_nvidia_nim.py@@ -16,7 +16,7 @@ from free_claude_code.core.history_replay import (encode_replay,)from free_claude_code.core.openai_responses.models import OpenAIResponsesRequest-from free_claude_code.core.reasoning import ReasoningEffort, ReasoningPolicy+from free_claude_code.core.reasoning import ReasoningPolicyfrom free_claude_code.providers.admission import UPSTREAM_TRANSIENT_TOTAL_ATTEMPTSfrom free_claude_code.providers.nvidia_nim import NvidiaNimProviderfrom free_claude_code.providers.nvidia_nim.client import _PROFILE as NIM_PROFILE@@ -165,7 +165,6 @@ def _alias_events():("responses", None),("messages", "chat_template"),("responses", "chat_template"),- ("messages", "reasoning_budget"),("messages", "reasoning_content"),],)@@ -398,10 +397,8 @@ async def test_build_request_body(provider_config):assert body["messages"][0]["content"] == "System prompt"assert "extra_body" in body- ctk = body["extra_body"]["chat_template_kwargs"]- assert ctk["thinking"] is True- assert ctk["enable_thinking"] is True- assert "reasoning_budget" not in ctk+ assert body["reasoning_effort"] == "high"+ assert "chat_template_kwargs" not in body["extra_body"]assert "reasoning_budget" not in body["extra_body"]@@ -454,10 +451,8 @@ def test_responses_request_uses_nim_chat_policy(provider_config):assert parameters["properties"] == {"_fcc_arg_type": {"type": "string"}}extra_body = body["extra_body"]assert isinstance(extra_body, dict)- assert extra_body["chat_template_kwargs"] == {- "thinking": True,- "enable_thinking": True,- }+ assert body["reasoning_effort"] == "high"+ assert "chat_template_kwargs" not in extra_body@pytest.mark.asyncio@@ -473,10 +468,8 @@ async def test_build_request_body_encodes_explicit_reasoning_off(body = provider._chat._build_request_body(req, reasoning=REASONING_OFF)extra = body.get("extra_body", {})- assert extra["chat_template_kwargs"] == {- "thinking": False,- "enable_thinking": False,- }+ assert body["reasoning_effort"] == "none"+ assert "chat_template_kwargs" not in extraassert "reasoning_budget" not in extra@@ -713,10 +706,7 @@ async def test_stream_messages_retries_without_chat_template(provider_config):second_extra = mock_create.call_args_list[1].kwargs["extra_body"]assert first_extra["chat_template"] == "custom_template"- assert first_extra["chat_template_kwargs"] == {- "thinking": True,- "enable_thinking": True,- }+ assert "chat_template_kwargs" not in first_extraassert "reasoning_budget" not in first_extraassert "chat_template" not in second_extra@@ -737,7 +727,10 @@ async def test_stream_messages_retries_without_chat_template_kwargs_issue_993(nim_settings=NimSettings(),admission=immediate_admission(),)- req = make_request(model="mistralai/mistral-small-4-119b-2603")+ req = make_request(+ model="mistralai/mistral-small-4-119b-2603",+ extra_body={"chat_template_kwargs": {"custom": "value"}},+ )mock_chunk = MagicMock()mock_chunk.choices = [@@ -770,10 +763,7 @@ async def test_stream_messages_retries_without_chat_template_kwargs_issue_993(second_kwargs = mock_create.call_args_list[1].kwargsassert "chat_template" not in first_extra- assert first_extra["chat_template_kwargs"] == {- "thinking": True,- "enable_thinking": True,- }+ assert first_extra["chat_template_kwargs"] == {"custom": "value"}second_extra = second_kwargs.get("extra_body") or {}assert "chat_template" not in second_extraassert "chat_template_kwargs" not in second_extra@@ -1330,95 +1320,6 @@ async def test_stream_messages_task_tool_still_forces_background_false(nim_proviassert json.loads(deltas[0])["run_in_background"] is False-async def test_stream_messages_retries_without_reasoning_budget(nim_provider):- req = make_request()-- mock_chunk = MagicMock()- mock_chunk.choices = [- MagicMock(- delta=MagicMock(content="Recovered", reasoning_content=""),- finish_reason="stop",- )- ]- mock_chunk.usage = MagicMock(completion_tokens=5)-- async def mock_stream():- yield mock_chunk-- error = _make_bad_request_error("Unsupported field: reasoning_budget")-- with patch.object(- nim_provider._client.chat.completions, "create", new_callable=AsyncMock- ) as mock_create:- mock_create.side_effect = [error, SDKStreamDouble(mock_stream())]-- events = [- e- async for e in nim_provider.stream_messages(- req,- reasoning=ReasoningPolicy.on(effort=ReasoningEffort.XHIGH),- )- ]-- assert mock_create.await_count == 2- first_call = mock_create.await_args_list[0].kwargs- second_call = mock_create.await_args_list[1].kwargs- assert first_call["extra_body"]["chat_template_kwargs"]["reasoning_budget"] == 4096- assert "reasoning_budget" not in second_call["extra_body"]- assert "reasoning_budget" not in second_call["extra_body"]["chat_template_kwargs"]- assert second_call["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True- assert any("Recovered" in event for event in events)- assert any("message_stop" in event for event in events)---async def test_stream_messages_retries_without_budget_for_thinking_token_error(- nim_provider,-):- req = make_request(model="meta/llama-3.3-70b-instruct")-- mock_chunk = MagicMock()- mock_chunk.choices = [- MagicMock(- delta=MagicMock(content="Recovered", reasoning_content=""),- finish_reason="stop",- )- ]- mock_chunk.usage = MagicMock(completion_tokens=5)-- async def mock_stream():- yield mock_chunk-- error = _make_internal_server_error(- "ValueError: thinking_token_budget is set but reasoning_config is not "- "configured. Please set --reasoning-config to use thinking_token_budget."- )-- with patch.object(- nim_provider._client.chat.completions, "create", new_callable=AsyncMock- ) as mock_create:- mock_create.side_effect = [error, SDKStreamDouble(mock_stream())]-- events = [- e- async for e in nim_provider.stream_messages(- req, reasoning=ReasoningPolicy.on(budget_tokens=77)- )- ]-- assert mock_create.await_count == 2- first_call = mock_create.await_args_list[0].kwargs- second_call = mock_create.await_args_list[1].kwargs- assert first_call["extra_body"]["chat_template_kwargs"]["reasoning_budget"] == 77- assert "reasoning_budget" not in second_call["extra_body"]- assert "reasoning_budget" not in second_call["extra_body"]["chat_template_kwargs"]- assert second_call["extra_body"]["chat_template_kwargs"]["thinking"] is True- assert second_call["extra_body"]["chat_template_kwargs"]["enable_thinking"] is True- assert any("Recovered" in event for event in events)- assert any("message_stop" in event for event in events)--@pytest.mark.asyncioasync def test_stream_messages_retries_without_reasoning_content(nim_provider):req = make_request(diff --git a/tests/providers/test_nvidia_nim_reasoning.py b/tests/providers/test_nvidia_nim_reasoning.pynew file mode 100644index 00000000..3bba07bd--- /dev/null+++ b/tests/providers/test_nvidia_nim_reasoning.py@@ -0,0 +1,85 @@+"""NIM named reasoning across both supported ingress protocols."""++import pytest++from free_claude_code.core.failures import ExecutionFailure+from free_claude_code.core.reasoning import ReasoningEffort, ReasoningPolicy+from tests.providers.test_history_transports import _harness, _saved_reply+from tests.providers.test_nvidia_nim import _alias_provider, _alias_request++[email protected]("wire", ["messages", "responses"])+ "policy,expected",+ [+ *((ReasoningPolicy(effort=effort), effort.value) for effort in ReasoningEffort),+ (ReasoningPolicy.off(), "none"),+ (ReasoningPolicy.on(), "high"),+ (ReasoningPolicy.on(budget_tokens=2048), "high"),+ (ReasoningPolicy.provider_default(), None),+ ],+)+async def test_named_effort_reaches_nim_without_numeric_budget(wire, policy, expected):+ def responder(bodies):+ body = bodies[-1]+ assert body.get("reasoning_effort") == expected+ assert "reasoning_budget" not in body+ assert "chat_template_kwargs" not in body+ return 200, [+ {+ "id": "response",+ "object": "chat.completion.chunk",+ "model": "test-model",+ "choices": [+ {+ "index": 0,+ "delta": {"content": "Hello"},+ "finish_reason": "stop",+ }+ ],+ }+ ]++ async with _harness("chat", responder, chat_provider_factory=_alias_provider) as (+ _,+ bodies,+ provider,+ ):+ stream = (+ provider.stream_messages+ if wire == "messages"+ else provider.stream_responses+ )+ reply = await _saved_reply(stream(_alias_request(wire), reasoning=policy), wire)++ assert reply+ assert len(bodies) == 1++[email protected]("wire", ["messages", "responses"])+ "policy", [ReasoningPolicy.off(), ReasoningPolicy(effort=ReasoningEffort.MAX)]+)+async def test_unsupported_named_effort_is_not_silently_remapped(wire, policy):+ def responder(bodies):+ return 400, {+ "message": "reasoning_effort must be one of low, medium, high",+ "param": "reasoning_effort",+ }++ async with _harness("chat", responder, chat_provider_factory=_alias_provider) as (+ _,+ bodies,+ provider,+ ):+ stream = (+ provider.stream_messages+ if wire == "messages"+ else provider.stream_responses+ )+ with pytest.raises(ExecutionFailure, match="reasoning_effort"):+ await _saved_reply(stream(_alias_request(wire), reasoning=policy), wire)++ assert len(bodies) == 1diff --git a/tests/providers/test_nvidia_nim_request.py b/tests/providers/test_nvidia_nim_request.pyindex 84b22b87..cf022ceb 100644--- a/tests/providers/test_nvidia_nim_request.py+++ b/tests/providers/test_nvidia_nim_request.py@@ -102,32 +102,13 @@ class TestSetExtra:class TestBuildRequestBody:- @pytest.mark.parametrize(- ("effort", "expected_budget"),- (- (ReasoningEffort.MINIMAL, 512),- (ReasoningEffort.LOW, 512),- (ReasoningEffort.MEDIUM, 1_024),- (ReasoningEffort.HIGH, 2_048),- (ReasoningEffort.XHIGH, 4_096),- (ReasoningEffort.MAX, 8_192),- ),- )- def test_named_effort_enables_thinking_with_numeric_budget(- self,- req,- effort: ReasoningEffort,- expected_budget: int,- ):- policy = ReasoningPolicy(effort=effort)-- body = build_request_body(req, NimSettings(), reasoning=policy)-- assert body["extra_body"]["chat_template_kwargs"] == {- "thinking": True,- "enable_thinking": True,- "reasoning_budget": expected_budget,- }+ @pytest.mark.parametrize("effort", list(ReasoningEffort))+ def test_named_effort_passes_through(self, req, effort):+ body = build_request_body(+ req, NimSettings(), reasoning=ReasoningPolicy(effort=effort)+ )+ assert body["reasoning_effort"] == effort.value+ assert "chat_template_kwargs" not in body["extra_body"]def test_named_effort_replaces_client_reasoning_budgets(self):req = make_messages_request(@@ -150,12 +131,8 @@ class TestBuildRequestBody:extra_body = body["extra_body"]assert "reasoning_budget" not in extra_body- assert extra_body["chat_template_kwargs"] == {- "custom": "value",- "thinking": True,- "enable_thinking": True,- "reasoning_budget": 2048,- }+ assert body["reasoning_effort"] == "high"+ assert extra_body["chat_template_kwargs"] == {"custom": "value"}def test_max_tokens_capped_by_nim(self, req):req.max_tokens = 100000@@ -377,7 +354,7 @@ class TestBuildRequestBody:"Grep": {"_fcc_arg_A": "-A"}}- def test_reasoning_params_in_extra_body(self):+ def test_reasoning_uses_named_effort(self):req = make_messages_request(model="test",messages=[{"role": "user", "content": "hi"}],@@ -396,10 +373,8 @@ class TestBuildRequestBody:nim = NimSettings()body = build_request_body(req, nim, reasoning=REASONING_ON)extra = body["extra_body"]- assert extra["chat_template_kwargs"] == {- "thinking": True,- "enable_thinking": True,- }+ assert body["reasoning_effort"] == "high"+ assert "chat_template_kwargs" not in extraassert "reasoning_budget" not in extradef test_canonicalization_removes_empty_client_reasoning_envelope(self):@@ -493,13 +468,11 @@ class TestBuildRequestBody:nim = NimSettings()body = build_request_body(req, nim, reasoning=REASONING_OFF)extra = body.get("extra_body", {})- assert extra["chat_template_kwargs"] == {- "thinking": False,- "enable_thinking": False,- }+ assert body["reasoning_effort"] == "none"+ assert "chat_template_kwargs" not in extraassert "reasoning_budget" not in extra- def test_reasoning_budget_respects_existing_chat_template_kwargs(self):+ def test_named_effort_preserves_unrelated_chat_template_kwargs(self):req = make_messages_request(model="test",messages=[{"role": "user", "content": "hi"}],@@ -521,11 +494,8 @@ class TestBuildRequestBody:)body = build_request_body(req, NimSettings(), reasoning=REASONING_ON)- assert body["extra_body"]["chat_template_kwargs"] == {- "enable_thinking": True,- "custom": "value",- "thinking": True,- }+ assert body["reasoning_effort"] == "high"+ assert body["extra_body"]["chat_template_kwargs"] == {"custom": "value"}def test_chat_template_fields_are_provider_wide(self):req = make_messages_request(@@ -546,10 +516,8 @@ class TestBuildRequestBody:nim = NimSettings(chat_template="custom_template")body = build_request_body(req, nim, reasoning=REASONING_ON)extra = body.get("extra_body", {})- assert extra["chat_template_kwargs"] == {- "thinking": True,- "enable_thinking": True,- }+ assert body["reasoning_effort"] == "high"+ assert "chat_template_kwargs" not in extraassert extra["chat_template"] == "custom_template"def test_no_reasoning_params_in_extra_body(self):@@ -579,12 +547,10 @@ class TestBuildRequestBody:"reasoning_effort",):assert param not in extra- assert extra["chat_template_kwargs"] == {- "thinking": False,- "enable_thinking": False,- }+ assert body["reasoning_effort"] == "none"+ assert "chat_template_kwargs" not in extra- def test_explicit_reasoning_budget_is_preserved_exactly(self):+ def test_budget_only_request_uses_enabled_named_effort(self):req = make_messages_request(model="test", thinking=None)body = build_request_body(@@ -593,11 +559,9 @@ class TestBuildRequestBody:reasoning=ReasoningPolicy.on(budget_tokens=321),)- assert body["extra_body"]["chat_template_kwargs"] == {- "thinking": True,- "enable_thinking": True,- "reasoning_budget": 321,- }+ assert body["reasoning_effort"] == "high"+ assert "chat_template_kwargs" not in body["extra_body"]+ assert "reasoning_budget" not in body["extra_body"]def test_assistant_thinking_blocks_removed_when_disabled(self):req = make_messages_request(