Personal AI, On Personal Devices
Python
fix(engine): retry without temperature in streaming and OpenRouter cloud paths (#1047)
Arush Wadhawanbd766e4bfc7f
diff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.pyindex 5dae86c9..17bff9d9 100644--- a/src/openjarvis/engine/cloud.py+++ b/src/openjarvis/engine/cloud.py@@ -249,6 +249,20 @@ def _is_unsupported_temperature_error(exc: Exception) -> bool:)+def _chat_completion_with_temperature_retry(+ client: Any,+ create_kwargs: Dict[str, Any],+) -> Any:+ """Invoke client.chat.completions.create, retrying without temperature on 400."""+ try:+ return client.chat.completions.create(**create_kwargs)+ except Exception as exc:+ if "temperature" in create_kwargs and _is_unsupported_temperature_error(exc):+ create_kwargs.pop("temperature", None)+ return client.chat.completions.create(**create_kwargs)+ raise++def estimate_cost(model: str, prompt_tokens: int, completion_tokens: int) -> float:"""Estimate USD cost based on the hardcoded pricing table."""# Try exact match first, then prefix match@@ -690,19 +704,9 @@ class CloudEngine(InferenceEngine):create_kwargs["response_format"] = response_formatt0 = time.monotonic()- try:- resp = self._openai_client.chat.completions.create(**create_kwargs)- except Exception as exc:- # Some models reject a non-default temperature with a 400- # unsupported_value (see #426). Retry once without it rather- # than failing the user's first prompt.- if "temperature" in create_kwargs and _is_unsupported_temperature_error(- exc- ):- create_kwargs.pop("temperature", None)- resp = self._openai_client.chat.completions.create(**create_kwargs)- else:- raise+ resp = _chat_completion_with_temperature_retry(+ self._openai_client, create_kwargs+ )elapsed = time.monotonic() - t0choice = _first_choice_or_raise(resp, provider="OpenAI", model=model)usage = resp.usage@@ -1045,7 +1049,9 @@ class CloudEngine(InferenceEngine):if tool_choice is not None:create_kwargs["tool_choice"] = tool_choicet0 = time.monotonic()- resp = self._openrouter_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._openrouter_client, create_kwargs+ )elapsed = time.monotonic() - t0choice = _first_choice_or_raise(resp, provider="OpenRouter", model=actual_model)usage = resp.usage@@ -1109,7 +1115,9 @@ class CloudEngine(InferenceEngine):if tool_choice is not None:create_kwargs["tool_choice"] = tool_choicet0 = time.monotonic()- resp = self._atlascloud_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._atlascloud_client, create_kwargs+ )elapsed = time.monotonic() - t0choice = resp.choices[0]usage = resp.usage@@ -1167,7 +1175,9 @@ class CloudEngine(InferenceEngine):"temperature": temperature,}t0 = time.monotonic()- resp = self._minimax_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._minimax_client, create_kwargs+ )elapsed = time.monotonic() - t0choice = _first_choice_or_raise(resp, provider="MiniMax", model=model)usage = resp.usage@@ -1217,7 +1227,9 @@ class CloudEngine(InferenceEngine):"temperature": temperature,}t0 = time.monotonic()- resp = self._deepseek_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._deepseek_client, create_kwargs+ )elapsed = time.monotonic() - t0choice = _first_choice_or_raise(resp, provider="DeepSeek", model=model)usage = resp.usage@@ -1393,7 +1405,9 @@ class CloudEngine(InferenceEngine):}if not _is_openai_reasoning_model(model):create_kwargs["temperature"] = temperature- resp = self._openai_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._openai_client, create_kwargs+ )for chunk in resp:delta = chunk.choices[0].delta if chunk.choices else Noneif delta and delta.content:@@ -1647,7 +1661,9 @@ class CloudEngine(InferenceEngine):tool_choice = kwargs.pop("tool_choice", None)if tool_choice is not None:create_kwargs["tool_choice"] = tool_choice- resp = self._openrouter_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._openrouter_client, create_kwargs+ )for chunk in resp:delta = chunk.choices[0].delta if chunk.choices else Noneif delta and delta.content:@@ -1679,7 +1695,9 @@ class CloudEngine(InferenceEngine):tool_choice = kwargs.pop("tool_choice", None)if tool_choice is not None:create_kwargs["tool_choice"] = tool_choice- resp = self._atlascloud_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._atlascloud_client, create_kwargs+ )for chunk in resp:delta = chunk.choices[0].delta if chunk.choices else Noneif delta and delta.content:@@ -1705,7 +1723,9 @@ class CloudEngine(InferenceEngine):"temperature": temperature,"stream": True,}- resp = self._minimax_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._minimax_client, create_kwargs+ )for chunk in resp:delta = chunk.choices[0].delta if chunk.choices else Noneif delta and delta.content:@@ -1729,7 +1749,9 @@ class CloudEngine(InferenceEngine):"temperature": temperature,"stream": True,}- resp = self._deepseek_client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(+ self._deepseek_client, create_kwargs+ )for chunk in resp:delta = chunk.choices[0].delta if chunk.choices else Noneif delta and delta.content:@@ -1825,7 +1847,7 @@ class CloudEngine(InferenceEngine):}if not _is_openai_reasoning_model(model):create_kwargs["temperature"] = temperature- resp = client.chat.completions.create(**create_kwargs)+ resp = _chat_completion_with_temperature_retry(client, create_kwargs)for chunk in resp:choice = chunk.choices[0] if chunk.choices else Noneif not choice:diff --git a/tests/engine/test_cloud.py b/tests/engine/test_cloud.pyindex 5e7a82e6..2d3a624f 100644--- a/tests/engine/test_cloud.py+++ b/tests/engine/test_cloud.py@@ -202,6 +202,18 @@ class TestOpenAIUnsupportedTemperatureRetry:model="gpt-5",)+ def _fake_stream_resp(self):+ return [+ SimpleNamespace(+ choices=[+ SimpleNamespace(+ delta=SimpleNamespace(content="stream-ok", tool_calls=None),+ finish_reason="stop",+ )+ ]+ )+ ]+def test_retries_without_temperature_on_unsupported_value(self, monkeypatch: pytest.MonkeyPatch) -> None:@@ -338,6 +350,160 @@ class TestOpenAIUnsupportedTemperatureRetry:# No temperature-retry for an unrelated 400 — exactly one attempt.assert len(calls) == 1+ @pytest.mark.asyncio+ async def test_stream_openai_retries_without_temperature(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)++ calls: list[dict] = []+ err = Exception(+ "Error code: 400 - Unsupported value: 'temperature' does not support 0.7"+ )++ def create(**kwargs):+ calls.append(kwargs)+ if "temperature" in kwargs:+ raise err+ return self._fake_stream_resp()++ fake_client = mock.MagicMock()+ fake_client.chat.completions.create.side_effect = create++ EngineRegistry.register_value("cloud", CloudEngine)+ engine = CloudEngine()+ engine._openai_client = fake_client++ tokens = [+ t+ async for t in engine.stream(+ [Message(role=Role.USER, content="Hi")],+ model="gpt-5",+ temperature=0.7,+ )+ ]+ assert tokens == ["stream-ok"]+ assert len(calls) == 2+ assert "temperature" in calls[0]+ assert "temperature" not in calls[1]++ @pytest.mark.asyncio+ async def test_stream_full_openai_retries_without_temperature(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)++ calls: list[dict] = []+ err = Exception(+ "Error code: 400 - Unsupported value: 'temperature' does not support 0.7"+ )++ def create(**kwargs):+ calls.append(kwargs)+ if "temperature" in kwargs:+ raise err+ return self._fake_stream_resp()++ fake_client = mock.MagicMock()+ fake_client.chat.completions.create.side_effect = create++ EngineRegistry.register_value("cloud", CloudEngine)+ engine = CloudEngine()+ engine._openai_client = fake_client++ chunks = [+ c+ async for c in engine.stream_full(+ [Message(role=Role.USER, content="Hi")],+ model="gpt-5",+ temperature=0.7,+ )+ ]+ assert len(chunks) == 1+ assert chunks[0].content == "stream-ok"+ assert chunks[0].finish_reason == "stop"+ assert len(calls) == 2+ assert "temperature" in calls[0]+ assert "temperature" not in calls[1]++ def test_generate_openrouter_retries_without_temperature(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test")+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)++ calls: list[dict] = []+ err = Exception(+ "Error code: 400 - Unsupported parameter: "+ "'temperature' is not supported with this model."+ )++ def create(**kwargs):+ calls.append(kwargs)+ if "temperature" in kwargs:+ raise err+ return self._fake_resp()++ fake_client = mock.MagicMock()+ fake_client.chat.completions.create.side_effect = create++ EngineRegistry.register_value("cloud", CloudEngine)+ engine = CloudEngine()+ engine._openrouter_client = fake_client++ result = engine.generate(+ [Message(role=Role.USER, content="Hi")],+ model="openrouter/openai/o3-mini",+ temperature=0.7,+ )+ assert result["content"] == "ok"+ assert len(calls) == 2+ assert "temperature" in calls[0]+ assert "temperature" not in calls[1]++ @pytest.mark.asyncio+ async def test_stream_openrouter_retries_without_temperature(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ monkeypatch.setenv("OPENROUTER_API_KEY", "sk-or-test")+ monkeypatch.delenv("OPENAI_API_KEY", raising=False)+ monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)++ calls: list[dict] = []+ err = Exception(+ "Error code: 400 - Unsupported parameter: "+ "'temperature' is not supported with this model."+ )++ def create(**kwargs):+ calls.append(kwargs)+ if "temperature" in kwargs:+ raise err+ return self._fake_stream_resp()++ fake_client = mock.MagicMock()+ fake_client.chat.completions.create.side_effect = create++ EngineRegistry.register_value("cloud", CloudEngine)+ engine = CloudEngine()+ engine._openrouter_client = fake_client++ tokens = [+ t+ async for t in engine.stream(+ [Message(role=Role.USER, content="Hi")],+ model="openrouter/openai/o3-mini",+ temperature=0.7,+ )+ ]+ assert tokens == ["stream-ok"]+ assert len(calls) == 2+ assert "temperature" in calls[0]+ assert "temperature" not in calls[1]+# ---------------------------------------------------------------------------# Codex provider support (OpenAI Responses API)diff --git a/tests/engine/test_cloud_atlascloud.py b/tests/engine/test_cloud_atlascloud.pyindex f3f6a25f..a9fbd40d 100644--- a/tests/engine/test_cloud_atlascloud.py+++ b/tests/engine/test_cloud_atlascloud.py@@ -209,6 +209,37 @@ class TestAtlasCloudGenerate:assert result["usage"]["completion_tokens"] == 5assert result["finish_reason"] == "stop"+ def test_generate_retries_without_unsupported_temperature(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ engine = _make_cloud_engine(monkeypatch)+ client = mock.MagicMock()+ calls: list[dict] = []++ def create(**kwargs):+ calls.append(kwargs)+ if "temperature" in kwargs:+ raise Exception(+ "Error code: 400 - Unsupported value: 'temperature' "+ "does not support 0.7"+ )+ return _fake_response(model="openai/gpt-5.4-mini")++ client.chat.completions.create.side_effect = create+ engine._atlascloud_client = client++ result = engine.generate(+ [Message(role=Role.USER, content="Hi")],+ model="atlascloud/openai/gpt-5.4-mini",+ temperature=0.7,+ )++ assert result["content"] == "Hello from Atlas Cloud!"+ assert len(calls) == 2+ assert calls[0]["model"] == calls[1]["model"] == "openai/gpt-5.4-mini"+ assert "temperature" in calls[0]+ assert "temperature" not in calls[1]+@pytest.mark.parametrize("model", _ATLASCLOUD_POPULAR)def test_every_listed_model_routes_to_the_atlas_client(self, monkeypatch: pytest.MonkeyPatch, model: str@@ -326,6 +357,46 @@ class TestAtlasCloudStream:assert sent["model"] == "openai/gpt-4.1-mini"assert sent["stream"] is True+ @pytest.mark.asyncio+ async def test_stream_retries_without_unsupported_temperature(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ engine = _make_cloud_engine(monkeypatch)+ client = mock.MagicMock()+ calls: list[dict] = []++ def create(**kwargs):+ calls.append(kwargs)+ if "temperature" in kwargs:+ raise Exception(+ "Error code: 400 - Unsupported value: 'temperature' "+ "does not support 0.7"+ )+ return [+ SimpleNamespace(+ choices=[SimpleNamespace(delta=SimpleNamespace(content="ok"))]+ )+ ]++ client.chat.completions.create.side_effect = create+ engine._atlascloud_client = client++ tokens = [+ token+ async for token in engine.stream(+ [Message(role=Role.USER, content="Hi")],+ model="atlascloud/openai/gpt-5.4-mini",+ temperature=0.7,+ )+ ]++ assert tokens == ["ok"]+ assert len(calls) == 2+ assert calls[0]["model"] == calls[1]["model"] == "openai/gpt-5.4-mini"+ assert calls[0]["stream"] is calls[1]["stream"] is True+ assert "temperature" in calls[0]+ assert "temperature" not in calls[1]+@pytest.mark.asyncioasync def test_stream_without_a_client_raises(self, monkeypatch: pytest.MonkeyPatch