Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Fix: Exclude chat_template for Mistral tokenizers in NVIDIA NIM (#130) (#131)
Anuj Nitin Bharambe4fdf7e8b7e8c
diff --git a/providers/nvidia_nim/client.py b/providers/nvidia_nim/client.pyindex 6adba758..78acc7a1 100644--- a/providers/nvidia_nim/client.py+++ b/providers/nvidia_nim/client.py@@ -10,7 +10,11 @@ from config.nim import NimSettingsfrom providers.base import ProviderConfigfrom providers.openai_compat import OpenAICompatibleProvider-from .request import build_request_body, clone_body_without_reasoning_budget+from .request import (+ build_request_body,+ clone_body_without_chat_template,+ clone_body_without_reasoning_budget,+)NVIDIA_NIM_BASE_URL = "https://integrate.api.nvidia.com/v1"@@ -36,7 +40,7 @@ class NvidiaNimProvider(OpenAICompatibleProvider):)def _get_retry_request_body(self, error: Exception, body: dict) -> dict | None:- """Retry once without reasoning_budget when NIM rejects that field."""+ """Retry once with a downgraded body when NIM rejects a known field."""status_code = getattr(error, "status_code", None)if not isinstance(error, openai.BadRequestError) and status_code != 400:return None@@ -45,12 +49,22 @@ class NvidiaNimProvider(OpenAICompatibleProvider):error_body = getattr(error, "body", None)if error_body is not None:error_text = f"{error_text} {json.dumps(error_body, default=str)}"- if "reasoning_budget" not in error_text.lower():- return None+ error_text = error_text.lower()- retry_body = clone_body_without_reasoning_budget(body)- if retry_body is None:- return None+ if "reasoning_budget" in error_text:+ retry_body = clone_body_without_reasoning_budget(body)+ if retry_body is None:+ return None+ logger.warning(+ "NIM_STREAM: retrying without reasoning_budget after 400 error"+ )+ return retry_body++ if "chat_template" in error_text:+ retry_body = clone_body_without_chat_template(body)+ if retry_body is None:+ return None+ logger.warning("NIM_STREAM: retrying without chat_template after 400 error")+ return retry_body- logger.warning("NIM_STREAM: retrying without reasoning_budget after 400 error")- return retry_body+ return Nonediff --git a/providers/nvidia_nim/request.py b/providers/nvidia_nim/request.pyindex 41d480ec..0c2f26d0 100644--- a/providers/nvidia_nim/request.py+++ b/providers/nvidia_nim/request.py@@ -47,6 +47,22 @@ def clone_body_without_reasoning_budget(body: dict[str, Any]) -> dict[str, Any]return cloned_body+def clone_body_without_chat_template(body: dict[str, Any]) -> dict[str, Any] | None:+ """Clone a request body and strip only chat_template."""+ cloned_body = deepcopy(body)+ extra_body = cloned_body.get("extra_body")+ if not isinstance(extra_body, dict):+ return None++ if extra_body.pop("chat_template", None) is None:+ return None++ if not extra_body:+ cloned_body.pop("extra_body", None)++ return cloned_body++def build_request_body(request_data: Any, nim: NimSettings, *, thinking_enabled: bool) -> dict:diff --git a/tests/providers/test_nvidia_nim.py b/tests/providers/test_nvidia_nim.pyindex d8875ab2..41d1e38d 100644--- a/tests/providers/test_nvidia_nim.py+++ b/tests/providers/test_nvidia_nim.py@@ -39,6 +39,15 @@ class MockRequest:setattr(self, k, v)+def _make_bad_request_error(message: str) -> openai.BadRequestError:+ response = Response(+ status_code=400,+ request=Request("POST", "https://integrate.api.nvidia.com/v1/chat/completions"),+ )+ body = {"error": {"message": message, "type": "BadRequestError", "code": 400}}+ return openai.BadRequestError(message, response=response, body=body)++@pytest.fixture(autouse=True)def mock_rate_limiter():"""Mock the global rate limiter to prevent waiting."""@@ -268,10 +277,86 @@ async def test_stream_response_suppresses_thinking_when_disabled(provider_configassert "Answer" in event_text-def _make_bad_request_error(message: str) -> openai.BadRequestError:- response = Response(status_code=400, request=Request("POST", "http://test"))- body = {"error": {"message": message}}- return openai.BadRequestError(message, response=response, body=body)+async def test_stream_response_retries_without_chat_template(provider_config):+ from config.nim import NimSettings++ provider = NvidiaNimProvider(+ provider_config,+ nim_settings=NimSettings(chat_template="custom_template"),+ )+ req = MockRequest(model="mistralai/mixtral-8x7b-instruct-v0.1")++ mock_chunk = MagicMock()+ mock_chunk.choices = [+ MagicMock(+ delta=MagicMock(content="OK", reasoning_content=""),+ finish_reason="stop",+ )+ ]+ mock_chunk.usage = MagicMock(completion_tokens=2)++ async def mock_stream():+ yield mock_chunk++ first_error = _make_bad_request_error(+ "chat_template is not supported for Mistral tokenizers."+ )++ with patch.object(+ provider._client.chat.completions, "create", new_callable=AsyncMock+ ) as mock_create:+ mock_create.side_effect = [first_error, mock_stream()]++ events = [e async for e in provider.stream_response(req)]++ assert mock_create.await_count == 2++ first_extra = mock_create.call_args_list[0].kwargs["extra_body"]+ second_extra = mock_create.call_args_list[1].kwargs["extra_body"]++ assert first_extra["chat_template"] == "custom_template"+ assert first_extra["chat_template_kwargs"] == {+ "thinking": True,+ "enable_thinking": True,+ "reasoning_budget": 100,+ }+ assert "reasoning_budget" not in first_extra++ assert "chat_template" not in second_extra+ assert second_extra["chat_template_kwargs"] == {+ "thinking": True,+ "enable_thinking": True,+ "reasoning_budget": 100,+ }+ assert "reasoning_budget" not in second_extra++ event_text = "".join(events)+ assert "event: error" not in event_text+ assert "OK" in event_text+++async def test_stream_response_does_not_retry_unrelated_bad_request(provider_config):+ from config.nim import NimSettings++ provider = NvidiaNimProvider(+ provider_config,+ nim_settings=NimSettings(chat_template="custom_template"),+ )+ req = MockRequest(model="mistralai/mixtral-8x7b-instruct-v0.1")++ with patch.object(+ provider._client.chat.completions, "create", new_callable=AsyncMock+ ) as mock_create:+ mock_create.side_effect = _make_bad_request_error("unrelated bad request")++ events = [e async for e in provider.stream_response(req)]++ assert mock_create.await_count == 1+ event_text = "".join(events)+ assert "unrelated bad request" in event_text+ assert "event: message_stop" in event_text@pytest.mark.asynciodiff --git a/tests/providers/test_nvidia_nim_request.py b/tests/providers/test_nvidia_nim_request.pyindex 2dd040f3..e4b9953e 100644--- a/tests/providers/test_nvidia_nim_request.py+++ b/tests/providers/test_nvidia_nim_request.py@@ -6,7 +6,11 @@ import pytestfrom config.nim import NimSettingsfrom providers.common.utils import set_if_not_none-from providers.nvidia_nim.request import _set_extra, build_request_body+from providers.nvidia_nim.request import (+ _set_extra,+ build_request_body,+ clone_body_without_chat_template,+)@pytest.fixture@@ -105,6 +109,32 @@ class TestBuildRequestBody:}assert "reasoning_budget" not in extra+ def test_clone_body_without_chat_template(self):+ body = {+ "model": "test",+ "extra_body": {+ "chat_template": "custom_template",+ "chat_template_kwargs": {+ "thinking": True,+ "enable_thinking": True,+ "reasoning_budget": 100,+ },+ "ignore_eos": False,+ },+ }++ cloned = clone_body_without_chat_template(body)++ assert cloned is not None+ assert "chat_template" not in cloned["extra_body"]+ assert cloned["extra_body"]["chat_template_kwargs"] == {+ "thinking": True,+ "enable_thinking": True,+ "reasoning_budget": 100,+ }+ assert cloned["extra_body"]["ignore_eos"] is False+ assert body["extra_body"]["chat_template"] == "custom_template"+def test_no_chat_template_kwargs_when_thinking_disabled(self):req = MagicMock()req.model = "test"@@ -148,6 +178,30 @@ class TestBuildRequestBody:"reasoning_budget": body["max_tokens"],}+ def test_chat_template_fields_present_for_mistral_model(self):+ req = MagicMock()+ req.model = "mistralai/mixtral-8x7b-instruct-v0.1"+ req.messages = [MagicMock(role="user", content="hi")]+ req.max_tokens = 100+ req.system = None+ req.temperature = None+ req.top_p = None+ req.stop_sequences = None+ req.tools = None+ req.tool_choice = None+ req.extra_body = None+ req.top_k = None++ nim = NimSettings(chat_template="custom_template")+ body = build_request_body(req, nim, thinking_enabled=True)+ extra = body.get("extra_body", {})+ assert extra["chat_template_kwargs"] == {+ "thinking": True,+ "enable_thinking": True,+ "reasoning_budget": body["max_tokens"],+ }+ assert extra["chat_template"] == "custom_template"+def test_no_reasoning_params_in_extra_body(self):req = MagicMock()req.model = "test"