Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix(llms): send reasoning_effort to every openai reasoning model (#7187)
João Moura66ef97c73e54
diff --git a/lib/crewai/src/crewai/llm.py b/lib/crewai/src/crewai/llm.pyindex 6c9c60911..4dda8a77b 100644--- a/lib/crewai/src/crewai/llm.py+++ b/lib/crewai/src/crewai/llm.py@@ -386,7 +386,9 @@ class LLM(BaseLLM):api_base: str | None = Noneapi_version: str | None = Nonecallbacks: list[Any] | None = None- reasoning_effort: Literal["none", "low", "medium", "high"] | None = None+ reasoning_effort: (+ Literal["none", "minimal", "low", "medium", "high", "xhigh"] | None+ ) = Nonestream: bool = Falseinterceptor: Any = Nonethinking: Any = Nonediff --git a/lib/crewai/src/crewai/llms/providers/openai/completion.py b/lib/crewai/src/crewai/llms/providers/openai/completion.pyindex 75e7fb7a7..88164ba64 100644--- a/lib/crewai/src/crewai/llms/providers/openai/completion.py+++ b/lib/crewai/src/crewai/llms/providers/openai/completion.py@@ -5,7 +5,9 @@ from dataclasses import dataclass, fieldimport jsonimport loggingimport os+import refrom typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, TypedDict+from urllib.parse import urlsplitimport httpxfrom openai import (@@ -78,6 +80,12 @@ if TYPE_CHECKING:# per process rather than on every call._LEARNED_RESPONSES_ONLY_MODELS: set[str] = set()+# (endpoint, model) pairs a 400 has shown to reject `reasoning_effort` in this+# process, so the rejected call is paid once rather than on every request. Keyed+# by endpoint as well as name because the same name on a gateway and on OpenAI+# itself are different models, and one must not silence the other.+_LEARNED_NO_REASONING_EFFORT_MODELS: set[tuple[str, str]] = set()+# Upstream status codes carried inside a 200 body, mapped to the exception the SDK# raises when the same code arrives as a real HTTP status. Keeping the classes# identical means a gateway-masked failure is catchable by whatever already handles@@ -168,6 +176,48 @@ def _raise_for_upstream_error(raise error_cls(message, response=http_response, body=error)+# `reasoning_effort` is accepted by the o-series and by GPT generation 5 onwards.+# Matched by shape rather than by a list of names so a new member of an existing+# family works without a release here; `gpt-4o` and `gpt-4.1` parse to generation+# 4 and are excluded, and a fine-tune is judged by its base model. An over-match+# is recovered rather than fatal -- the call is retried without the parameter+# when the API rejects it.+_O_SERIES_MODEL: Final = re.compile(r"^o\d")+_GPT_GENERATION: Final = re.compile(r"^gpt-(\d+)")+_MIN_REASONING_GPT_GENERATION: Final[int] = 5+# ft:<base model>:<org>:<suffix>:<id>+_FINE_TUNE_PREFIX: Final = "ft:"+_OPENAI_API_HOST: Final = "api.openai.com"+# How servers say a request field is not one they know: OpenAI's own two shapes+# alongside the wordings of compatible servers (pydantic, serde, Go). A 400 that+# names the parameter without one of these -- a bad value, the tools case -- is+# not a rejection of the parameter and must surface.+_UNKNOWN_PARAMETER_PHRASES: Final = (+ "unsupported parameter",+ "unrecognized request argument",+ "unknown field",+ "unknown parameter",+ "unknown argument",+ "unexpected field",+ "unexpected parameter",+ "extra inputs",+ "not permitted",+ "additional properties",+)+++def _supports_reasoning_effort(model: str) -> bool:+ """Whether the model accepts `reasoning_effort` on /v1/chat/completions."""+ name = model.rsplit("/", 1)[-1].lower().removeprefix(_FINE_TUNE_PREFIX)+ if _O_SERIES_MODEL.match(name):+ return True+ generation = _GPT_GENERATION.match(name)+ return (+ generation is not None+ and int(generation.group(1)) >= _MIN_REASONING_GPT_GENERATION+ )++class WebSearchResult(TypedDict, total=False):"""Result from web search built-in tool."""@@ -649,6 +699,13 @@ class OpenAICompletion(BaseLLM):)return dispatch(retry_params)+ if self._rejects_reasoning_effort_as_unsupported(cause):+ retry_params = self._without_reasoning_effort(completion_params)+ if retry_params is not None:+ result = dispatch(retry_params)+ self._remember_no_reasoning_effort_model()+ return result+if self.custom_openai or not self._is_responses_only_error(cause):raiseself._remember_responses_only_model()@@ -780,6 +837,13 @@ class OpenAICompletion(BaseLLM):if retry_params is not None:return await dispatch(retry_params)+ if self._rejects_reasoning_effort_as_unsupported(cause):+ retry_params = self._without_reasoning_effort(completion_params)+ if retry_params is not None:+ result = await dispatch(retry_params)+ self._remember_no_reasoning_effort_model()+ return result+if self.custom_openai or not self._is_responses_only_error(cause):raiseself._remember_responses_only_model()@@ -1862,6 +1926,118 @@ class OpenAICompletion(BaseLLM):message = str(source.get("message") or "").lower()return "function tools" in message and "reasoning_effort" in message+ def _sends_reasoning_effort(self) -> bool:+ """Whether `reasoning_effort` goes on the wire for this model.++ OpenAI's own models are matched by shape. A compatible server names its+ models in its own namespace, so the name says nothing about support+ there and the explicit setting is honoured. Either way, a model that+ has rejected the parameter on this endpoint in this process is not+ sent it again.+ """+ if self._reasoning_effort_key() in _LEARNED_NO_REASONING_EFFORT_MODELS:+ return False+ return self._on_compatible_server() or _supports_reasoning_effort(self.model)++ def _effective_base_url(self) -> str | None:+ """The base URL the client is built with; None means OpenAI's own API.++ Same precedence as `_get_client_params`, where `client_params` win. The+ SDK accepts ``str | httpx.URL`` there and `_get_client_params` forwards+ either, so both are normalised to the string the client will call.+ """+ override = (self.client_params or {}).get("base_url")+ if isinstance(override, (str, httpx.URL)) and str(override):+ return str(override)+ return (+ self.base_url+ or self.api_base+ or os.getenv("OPENAI_BASE_URL")+ or os.getenv("OPENAI_API_BASE")+ )++ def _on_compatible_server(self) -> bool:+ """Whether requests go to an OpenAI-compatible server rather than OpenAI.++ Decided by where the client actually points -- an explicit ``base_url``+ or ``api_base``, the ``OPENAI_BASE_URL``/``OPENAI_API_BASE`` env, or the+ URL a provider subclass resolves -- because ``custom_openai`` is only+ set on some of those paths.+ """+ if self.custom_openai:+ return True+ url = self._effective_base_url()+ if not url:+ return False+ return urlsplit(url).hostname != _OPENAI_API_HOST++ def _reasoning_effort_key(self) -> tuple[str, str]:+ return (self._effective_base_url() or _OPENAI_API_HOST, self.model)++ def _remember_no_reasoning_effort_model(self) -> None:+ """Record that this model rejected `reasoning_effort`.++ Called once the retry without the parameter has succeeded, so a retry+ that fails for an unrelated reason leaves the setting in place for the+ next call instead of silently dropping it for the rest of the process.+ """+ logging.warning(+ "%r rejected reasoning_effort=%r; the call succeeded without it, and "+ "it will not be sent to this model again in this process.",+ self.model,+ self.reasoning_effort,+ )+ _LEARNED_NO_REASONING_EFFORT_MODELS.add(self._reasoning_effort_key())++ def _rejects_reasoning_effort_as_unsupported(self, error: BaseException) -> bool:+ """Whether a 400 is the server refusing `reasoning_effort` for this model.++ Non-reasoning models reject the parameter itself, in one of two shapes:++ {"code": "unsupported_parameter", "param": "reasoning_effort",+ "message": "Unsupported parameter: 'reasoning_effort' is not+ supported with this model."}++ {"param": null, "message": "Unrecognized request argument+ supplied: reasoning_effort"}++ Distinct from `_rejects_reasoning_effort_with_tools`: that is a+ reasoning model refusing the parameter only alongside function tools,+ and it recovers by sending "none" rather than by dropping the key. Also+ deliberately does not match the "Unsupported value" 400 that o1/o3+ return for a bad *value* -- the model does support the parameter, so+ silently dropping it would restore the very bug this recovers from.++ A compatible server words the same rejection its own way -- "unknown+ field", "Extra inputs are not permitted" -- so the message is matched+ against the common phrasings, read from the body's `message` or from+ the error text when the body is not the OpenAI shape. A 400 (or 422)+ that names the parameter without saying it is unknown is a bad value,+ or the tools case, and surfaces.+ """+ if not isinstance(error, (BadRequestError, UnprocessableEntityError)):+ return False+ body = getattr(error, "body", None)+ source: dict[str, Any] = {}+ if isinstance(body, dict):+ inner = body.get("error")+ source = inner if isinstance(inner, dict) else body+ message = str(+ source.get("message") or getattr(error, "message", "") or error+ ).lower()+ if "reasoning_effort" not in message:+ return False+ if source.get("code") == "unsupported_parameter":+ return True+ return any(phrase in message for phrase in _UNKNOWN_PARAMETER_PHRASES)++ @staticmethod+ def _without_reasoning_effort(params: dict[str, Any]) -> dict[str, Any] | None:+ """Params with `reasoning_effort` removed, or None if it was not set."""+ if "reasoning_effort" not in params:+ return None+ return {k: v for k, v in params.items() if k != "reasoning_effort"}+def _reasoning_effort_none_params(self, params: dict[str, Any]) -> dict[str, Any] | None:@@ -1926,8 +2102,11 @@ class OpenAICompletion(BaseLLM):if self.top_logprobs is not None:params["top_logprobs"] = self.top_logprobs- # Handle o1 model specific parameters- if self.is_o1_model and self.reasoning_effort:+ # Not gated on `is_o1_model`: that is a literal "o1" substring test, so+ # gpt-5, o3 and o4-mini failed it and silently thought at the server+ # default. It also drives tool support and message rewriting, so it+ # cannot be widened to mean "is a reasoning model".+ if self.reasoning_effort and self._sends_reasoning_effort():params["reasoning_effort"] = self.reasoning_effortif self.response_format is not None:@@ -2154,9 +2333,11 @@ class OpenAICompletion(BaseLLM):logging.error(f"Context window exceeded: {e}")raise LLMContextLengthExceededError(str(e)) from e- # `_call_completions` retries this one, so reporting a failed call+ # `_call_completions` retries these, so reporting a failed call# here would surface an error the caller never experiences.- if self._rejects_reasoning_effort_with_tools(e):+ if self._rejects_reasoning_effort_with_tools(+ e+ ) or self._rejects_reasoning_effort_as_unsupported(e):raiseerror_msg = f"OpenAI API call failed: {e!s}"@@ -2592,9 +2773,11 @@ class OpenAICompletion(BaseLLM):logging.error(f"Context window exceeded: {e}")raise LLMContextLengthExceededError(str(e)) from e- # `_call_completions` retries this one, so reporting a failed call+ # `_call_completions` retries these, so reporting a failed call# here would surface an error the caller never experiences.- if self._rejects_reasoning_effort_with_tools(e):+ if self._rejects_reasoning_effort_with_tools(+ e+ ) or self._rejects_reasoning_effort_as_unsupported(e):raiseerror_msg = f"OpenAI API call failed: {e!s}"diff --git a/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py b/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.pynew file mode 100644index 000000000..fbdbea4a5--- /dev/null+++ b/lib/crewai/tests/llms/openai/test_reasoning_effort_forwarding.py@@ -0,0 +1,711 @@+"""`reasoning_effort` reaches every reasoning model, not just o1.++The completions path used to gate the parameter behind+``is_o1_model = "o1" in model.lower()``, a literal substring test. gpt-5, o3 and+o4-mini contain no "o1", so an explicitly configured effort was dropped and the+model thought at the server default -- silently, since the request still+succeeded.++The gate could not simply be widened: ``is_o1_model`` also drives+``supports_function_calling``, ``supports_stop_words`` and the system->user+message rewrite, so marking gpt-5 as an o1 model would report that it cannot+call tools. ``_supports_reasoning_effort`` is a separate predicate matched on+model *shape* -- the o-series, and GPT generation 5 onwards -- so a new member of+an existing family needs no release, and a non-reasoning model never pays a+wasted round trip. If the shape match is ever wrong for a future family, the+400 is retried without the key rather than surfacing.+"""++from __future__ import annotations++from typing import Any++import httpx+import pytest+from openai import APIConnectionError, BadRequestError, UnprocessableEntityError++from crewai.llm import LLM+from crewai.llms.providers.openai import completion as completion_module+from crewai.llms.providers.openai.completion import (+ OpenAICompletion,+ _supports_reasoning_effort,+)+from crewai.llms.providers.openai_compatible.completion import (+ OpenAICompatibleCompletion,+)+++MESSAGES = [{"role": "user", "content": "hi"}]++REASONING_MODELS = ["gpt-5", "gpt-5-mini", "o3", "o3-mini", "o4-mini", "o1"]+++def build(model: str = "gpt-5", **kwargs: Any) -> OpenAICompletion:+ return OpenAICompletion(model=model, api_key="sk-test", **kwargs)+++GATEWAY = "https://gateway.example/v1"+++def gateway(model: str = "gpt-4o", **kwargs: Any) -> OpenAICompletion:+ """An OpenAI-compatible server fronting whatever it calls `model`."""+ return build(model, base_url=GATEWAY, **kwargs)+++def _status_error(cls: type, status: int, body: Any) -> Any:+ """A rejection whose body is not the OpenAI shape, as the SDK raises it."""+ message = f"Error code: {status} - {body}"+ return cls(+ message,+ response=httpx.Response(+ status, json=body, request=httpx.Request("POST", GATEWAY)+ ),+ body=body,+ )++[email protected](autouse=True)+def forget_rejecting_models():+ completion_module._LEARNED_NO_REASONING_EFFORT_MODELS.clear()+ yield+ completion_module._LEARNED_NO_REASONING_EFFORT_MODELS.clear()+++def _bad_request(message: str, **source: Any) -> BadRequestError:+ body = {"error": {"message": message, "type": "invalid_request_error", **source}}+ return BadRequestError(+ message,+ response=httpx.Response(+ 400,+ json=body,+ request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions"),+ ),+ body=body,+ )+++def unsupported_parameter_error() -> BadRequestError:+ """What a non-reasoning model returns for the parameter itself."""+ return _bad_request(+ "Unsupported parameter: 'reasoning_effort' is not supported with this model.",+ param="reasoning_effort",+ code="unsupported_parameter",+ )+++def unrecognized_argument_error() -> BadRequestError:+ """The other shape, where `param` is null."""+ return _bad_request(+ "Unrecognized request argument supplied: reasoning_effort", param=None+ )+++def unsupported_value_error() -> BadRequestError:+ """A bad *value* -- the model does support the parameter."""+ return _bad_request(+ "Unsupported value: 'reasoning_effort' does not support 'none'.",+ param="reasoning_effort",+ )+++class TestParameterReachesTheModel:+ @pytest.mark.parametrize("model", REASONING_MODELS)+ def test_forwarded_for_every_reasoning_model(self, model):+ params = build(model, reasoning_effort="high")._prepare_completion_params(+ MESSAGES+ )++ assert params["reasoning_effort"] == "high"++ def test_minimal_is_forwarded(self):+ """gpt-5's cheapest setting, and the one extraction workloads want."""+ params = build("gpt-5", reasoning_effort="minimal")._prepare_completion_params(+ MESSAGES+ )++ assert params["reasoning_effort"] == "minimal"++ @pytest.mark.parametrize("model", ["gpt-4o", "gpt-5", "o3"])+ def test_absent_when_not_configured(self, model):+ """Unset must stay off the wire, whatever the model."""+ assert "reasoning_effort" not in build(model)._prepare_completion_params(+ MESSAGES+ )++ @pytest.mark.parametrize("model", ["gpt-4o", "gpt-4.1", "gpt-3.5-turbo"])+ def test_not_sent_to_non_reasoning_models(self, model):+ """No wasted round trip: these never reach the API with the parameter."""+ params = build(model, reasoning_effort="high")._prepare_completion_params(+ MESSAGES+ )++ assert "reasoning_effort" not in params+++class TestSupportedModelShape:+ """Matched on shape so a new family member needs no release."""++ @pytest.mark.parametrize(+ "model",+ [+ "o1",+ "o1-mini",+ "o1-preview",+ "o3",+ "o3-mini",+ "o4-mini",+ "gpt-5",+ "gpt-5-mini",+ "gpt-5.1",+ "gpt-5.6-sol",+ "openai/gpt-5",+ "ft:o4-mini-2025-04-16:acme::abc123",+ ],+ )+ def test_supported(self, model):+ assert _supports_reasoning_effort(model) is True++ @pytest.mark.parametrize(+ "model",+ [+ "gpt-4o",+ "gpt-4o-mini",+ "gpt-4.1",+ "gpt-4-turbo",+ "gpt-3.5-turbo",+ "chatgpt-4o-latest",+ "omni-moderation-latest",+ "text-embedding-3-small",+ "ft:gpt-4o-mini-2024-07-18:acme::abc123",+ ],+ )+ def test_unsupported(self, model):+ assert _supports_reasoning_effort(model) is False++ @pytest.mark.parametrize("model", ["gpt-6", "gpt-7-turbo", "o5", "o9-mini"])+ def test_future_family_members_need_no_release(self, model):+ """The staleness this replaced: an unreleased family must still match."""+ assert _supports_reasoning_effort(model) is True+++class TestO1FlagUntouched:+ """The gate was shared; widening it would have broken these."""++ @pytest.mark.parametrize("model", ["gpt-5", "o3", "o4-mini"])+ def test_reasoning_models_still_support_tools(self, model):+ assert build(model).supports_function_calling() is True++ @pytest.mark.parametrize("model", ["gpt-5", "o3"])+ def test_is_o1_model_still_only_matches_o1(self, model):+ assert build(model).is_o1_model is False+ assert build("o1").is_o1_model is True++ def test_system_messages_are_not_rewritten_for_gpt5(self):+ formatted = build("gpt-5")._format_messages(+ [{"role": "system", "content": "be terse"}]+ )++ assert formatted[0]["role"] == "system"+++class TestErrorDetection:+ def test_matches_unsupported_parameter(self):+ assert build()._rejects_reasoning_effort_as_unsupported(+ unsupported_parameter_error()+ )++ def test_matches_unrecognized_argument(self):+ assert build()._rejects_reasoning_effort_as_unsupported(+ unrecognized_argument_error()+ )++ def test_ignores_an_unsupported_value(self):+ """Dropping the key here would silently restore the original bug."""+ assert not build()._rejects_reasoning_effort_as_unsupported(+ unsupported_value_error()+ )++ def test_ignores_a_400_about_another_parameter(self):+ assert not build()._rejects_reasoning_effort_as_unsupported(+ _bad_request(+ "Unsupported parameter: 'temperature' is not supported.",+ param="temperature",+ code="unsupported_parameter",+ )+ )++ def test_ignores_unrelated_exceptions(self):+ assert not build()._rejects_reasoning_effort_as_unsupported(+ RuntimeError("boom")+ )+++class TestRetryParams:+ def test_removes_the_key(self):+ params = OpenAICompletion._without_reasoning_effort(+ {"model": "gpt-4o", "reasoning_effort": "high"}+ )++ assert params == {"model": "gpt-4o"}++ def test_returns_none_when_absent(self):+ """Nothing left to drop -- the retry must not loop."""+ assert OpenAICompletion._without_reasoning_effort({"model": "gpt-4o"}) is None+++class TestRetryBehaviour:+ def test_retries_without_the_key_and_succeeds(self, monkeypatch):+ llm = build("gpt-6-future", reasoning_effort="high")+ seen: list[dict] = []++ def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise unsupported_parameter_error()+ return "ok"++ monkeypatch.setattr(llm, "_handle_completion", fake_handle)++ assert llm._call_completions(MESSAGES) == "ok"+ assert len(seen) == 2, "expected one rejected call and one retry"+ assert seen[0]["reasoning_effort"] == "high"+ assert "reasoning_effort" not in seen[1]++ @pytest.mark.asyncio+ async def test_retries_on_the_async_path(self, monkeypatch):+ llm = build("gpt-6-future", reasoning_effort="high")+ seen: list[dict] = []++ async def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise unsupported_parameter_error()+ return "ok"++ monkeypatch.setattr(llm, "_ahandle_completion", fake_handle)++ assert await llm._acall_completions(MESSAGES) == "ok"+ assert len(seen) == 2+ assert "reasoning_effort" not in seen[1]++ def test_retries_on_the_streaming_path(self, monkeypatch):+ llm = build("gpt-6-future", reasoning_effort="high", stream=True)+ seen: list[dict] = []++ def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise unsupported_parameter_error()+ return "ok"++ monkeypatch.setattr(llm, "_handle_streaming_completion", fake_handle)++ assert llm._call_completions(MESSAGES) == "ok"+ assert len(seen) == 2+ assert "reasoning_effort" not in seen[1]++ @pytest.mark.asyncio+ async def test_retries_on_the_async_streaming_path(self, monkeypatch):+ llm = build("gpt-6-future", reasoning_effort="high", stream=True)+ seen: list[dict] = []++ async def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise unsupported_parameter_error()+ return "ok"++ monkeypatch.setattr(llm, "_ahandle_streaming_completion", fake_handle)++ assert await llm._acall_completions(MESSAGES) == "ok"+ assert len(seen) == 2+ assert "reasoning_effort" not in seen[1]++ def test_does_not_retry_forever(self, monkeypatch):+ llm = build("gpt-6-future", reasoning_effort="high")+ calls: list[dict] = []++ def always_fail(params, **kwargs):+ calls.append(params)+ raise unsupported_parameter_error()++ monkeypatch.setattr(llm, "_handle_completion", always_fail)++ with pytest.raises(BadRequestError, match="not supported with this model"):+ llm._call_completions(MESSAGES)++ assert len(calls) == 2, "one original call plus exactly one retry"++ def test_an_unsupported_value_surfaces(self, monkeypatch):+ """A bad value is the caller's mistake and must not be papered over."""+ llm = build("o3", reasoning_effort="none")+ calls: list[dict] = []++ def always_fail(params, **kwargs):+ calls.append(params)+ raise unsupported_value_error()++ monkeypatch.setattr(llm, "_handle_completion", always_fail)++ with pytest.raises(BadRequestError, match="does not support 'none'"):+ llm._call_completions(MESSAGES)++ assert len(calls) == 1, "a bad value must not be retried"++ def test_a_model_that_rejected_the_parameter_is_not_sent_it_again(+ self, monkeypatch+ ):+ """The rejected call is paid once per process, not on every request."""+ llm = build("gpt-6-future", reasoning_effort="high")+ seen: list[dict] = []++ def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise unsupported_parameter_error()+ return "ok"++ monkeypatch.setattr(llm, "_handle_completion", fake_handle)++ llm._call_completions(MESSAGES)+ llm._call_completions(MESSAGES)++ assert ["reasoning_effort" in params for params in seen] == [+ True,+ False,+ False,+ ]++ def test_a_rejection_is_remembered_only_once_the_retry_succeeds(+ self, monkeypatch+ ):+ """A retry that dies for another reason has not established the fallback.++ Remembering before the retry would silently stop sending a configured+ effort for the rest of the process on the strength of one transient+ failure; the setting must survive until a call without it succeeds.+ """+ llm = build("gpt-6-future", reasoning_effort="high")+ seen: list[dict] = []+ retry_outcome: list[Exception | None] = [+ APIConnectionError(request=httpx.Request("POST", GATEWAY))+ ]++ def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise unsupported_parameter_error()+ if retry_outcome and (failure := retry_outcome.pop()):+ raise failure+ return "ok"++ monkeypatch.setattr(llm, "_handle_completion", fake_handle)++ with pytest.raises(APIConnectionError):+ llm._call_completions(MESSAGES)+ assert not completion_module._LEARNED_NO_REASONING_EFFORT_MODELS++ assert llm._call_completions(MESSAGES) == "ok"+ assert llm._reasoning_effort_key() in (+ completion_module._LEARNED_NO_REASONING_EFFORT_MODELS+ )+ assert ["reasoning_effort" in params for params in seen] == [+ True,+ False,+ True,+ False,+ ], "the effort is sent again after a failed retry, then dropped for good"++ @pytest.mark.asyncio+ async def test_a_rejection_is_remembered_only_once_the_async_retry_succeeds(+ self, monkeypatch+ ):+ llm = build("gpt-6-future", reasoning_effort="high")+ seen: list[dict] = []+ retry_outcome: list[Exception | None] = [+ APIConnectionError(request=httpx.Request("POST", GATEWAY))+ ]++ async def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise unsupported_parameter_error()+ if retry_outcome and (failure := retry_outcome.pop()):+ raise failure+ return "ok"++ monkeypatch.setattr(llm, "_ahandle_completion", fake_handle)++ with pytest.raises(APIConnectionError):+ await llm._acall_completions(MESSAGES)+ assert not completion_module._LEARNED_NO_REASONING_EFFORT_MODELS++ assert await llm._acall_completions(MESSAGES) == "ok"+ assert ["reasoning_effort" in params for params in seen] == [+ True,+ False,+ True,+ False,+ ]+++class TestCompatibleServers:+ """The model name is the server's namespace, so it says nothing about support."""++ @pytest.mark.parametrize(+ "model", ["gpt-4o", "gpt-oss-120b", "qwen3-235b", "deepseek-r1"]+ )+ def test_explicit_setting_is_forwarded_whatever_the_name(self, model):+ params = gateway(model, reasoning_effort="low")._prepare_completion_params(+ MESSAGES+ )++ assert params["reasoning_effort"] == "low"++ def test_the_flag_marks_a_server_compatible(self):+ params = gateway(+ "gpt-4o", custom_openai=True, reasoning_effort="low"+ )._prepare_completion_params(MESSAGES)++ assert params["reasoning_effort"] == "low"++ def test_a_client_params_base_url_marks_a_server_compatible(self):+ """`client_params` win over the standard fields when the client is built."""+ llm = build("gpt-4o", client_params={"base_url": GATEWAY}, reasoning_effort="low")++ assert llm._prepare_completion_params(MESSAGES)["reasoning_effort"] == "low"++ def test_an_httpx_url_override_is_the_endpoint_the_client_calls(self):+ """The SDK takes `str | httpx.URL`; detection and memory must see both alike."""+ llm = build(+ "gpt-4o",+ client_params={"base_url": httpx.URL(GATEWAY)},+ reasoning_effort="low",+ )++ assert llm._prepare_completion_params(MESSAGES)["reasoning_effort"] == "low"+ assert llm._reasoning_effort_key() == (GATEWAY, "gpt-4o")++ def test_an_env_base_url_marks_a_server_compatible(self, monkeypatch):+ monkeypatch.setenv("OPENAI_BASE_URL", "http://localhost:11434/v1")++ params = build("qwen3", reasoning_effort="low")._prepare_completion_params(+ MESSAGES+ )++ assert params["reasoning_effort"] == "low"++ def test_openais_own_url_is_not_a_compatible_server(self):+ llm = build("gpt-4o", base_url="https://api.openai.com/v1", reasoning_effort="low")++ assert "reasoning_effort" not in llm._prepare_completion_params(MESSAGES)++ def test_reaches_the_wire_through_the_llm_factory(self):+ """gpt-oss looks like an OpenAI name to the factory, so it never flags it."""+ llm = LLM(+ model="gpt-oss-120b",+ base_url="http://localhost:8000/v1",+ api_key="sk-test",+ reasoning_effort="high",+ )++ assert llm._prepare_completion_params(MESSAGES)["reasoning_effort"] == "high"++ def test_forwarded_through_a_provider_subclass(self):+ llm = OpenAICompatibleCompletion(+ model="openai/gpt-oss-120b",+ provider="openrouter",+ api_key="sk-test",+ reasoning_effort="low",+ )++ assert llm._prepare_completion_params(MESSAGES)["reasoning_effort"] == "low"++ def test_unset_stays_off_the_wire(self):+ params = gateway("gpt-oss-120b")._prepare_completion_params(MESSAGES)++ assert "reasoning_effort" not in params++ def test_a_rejection_in_the_servers_own_words_is_recovered(self, monkeypatch):+ llm = gateway("qwen3-235b", reasoning_effort="low")+ seen: list[dict] = []++ def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise _bad_request(+ "[{'loc': ('body', 'reasoning_effort'), "+ "'msg': 'Extra inputs are not permitted'}]",+ param=None,+ )+ return "ok"++ monkeypatch.setattr(llm, "_handle_completion", fake_handle)++ assert llm._call_completions(MESSAGES) == "ok"+ assert len(seen) == 2+ assert "reasoning_effort" not in seen[1]++ @pytest.mark.parametrize(+ "error",+ [+ _status_error(+ BadRequestError, 400, {"error": "unknown field `reasoning_effort`"}+ ),+ _status_error(+ UnprocessableEntityError,+ 422,+ {+ "detail": [+ {+ "loc": ["body", "reasoning_effort"],+ "msg": "Extra inputs are not permitted",+ }+ ]+ },+ ),+ ],+ ids=["string-body", "fastapi-422"],+ )+ def test_a_rejection_outside_the_openai_shape_is_recovered(+ self, monkeypatch, error+ ):+ llm = gateway("qwen3-235b", reasoning_effort="low")+ seen: list[dict] = []++ def fake_handle(params, **kwargs):+ seen.append(params)+ if "reasoning_effort" in params:+ raise error+ return "ok"++ monkeypatch.setattr(llm, "_handle_completion", fake_handle)++ assert llm._call_completions(MESSAGES) == "ok"+ assert "reasoning_effort" not in seen[1]++ def test_a_complaint_about_the_value_surfaces_and_is_not_remembered(+ self, monkeypatch+ ):+ """The server takes the parameter; the value is the caller's mistake."""+ llm = gateway("o3", reasoning_effort="minimal")+ calls: list[dict] = []++ def always_fail(params, **kwargs):+ calls.append(params)+ raise _bad_request(+ "Invalid value 'minimal' for reasoning_effort; "+ "must be one of low, medium, high.",+ param="reasoning_effort",+ )++ monkeypatch.setattr(llm, "_handle_completion", always_fail)++ with pytest.raises(BadRequestError, match="Invalid value"):+ llm._call_completions(MESSAGES)++ assert len(calls) == 1+ assert not completion_module._LEARNED_NO_REASONING_EFFORT_MODELS++ @pytest.mark.parametrize(+ "error",+ [+ _status_error(+ UnprocessableEntityError,+ 422,+ {+ "detail": [+ {+ "loc": ["body", "reasoning_effort"],+ "msg": "Input should be 'low', 'medium' or 'high'",+ "type": "literal_error",+ }+ ]+ },+ ),+ _status_error(+ BadRequestError, 400, {"error": "reasoning_effort: something odd"}+ ),+ ],+ ids=["pydantic-enum", "ambiguous"],+ )+ def test_without_evidence_the_parameter_is_unknown_the_error_surfaces(+ self, monkeypatch, error+ ):+ """Only a rejection of the parameter itself is recovered; anything else+ naming it is the caller's to see."""+ llm = gateway("qwen3-235b", reasoning_effort="minimal")+ calls: list[dict] = []++ def always_fail(params, **kwargs):+ calls.append(params)+ raise error++ monkeypatch.setattr(llm, "_handle_completion", always_fail)++ with pytest.raises(type(error)):+ llm._call_completions(MESSAGES)++ assert len(calls) == 1+ assert not completion_module._LEARNED_NO_REASONING_EFFORT_MODELS++ @pytest.mark.parametrize(+ "gateway_config",+ [+ pytest.param({"base_url": GATEWAY}, id="base_url"),+ pytest.param(+ {"client_params": {"base_url": httpx.URL(GATEWAY)}}, id="httpx.URL"+ ),+ ],+ )+ def test_a_gateway_rejection_does_not_silence_the_model_on_openai(+ self, monkeypatch, gateway_config+ ):+ """However the gateway is configured, its rejection is keyed to it, not to OpenAI."""+ llm = build("gpt-5", reasoning_effort="high", **gateway_config)++ def reject(params, **kwargs):+ if "reasoning_effort" in params:+ raise unsupported_parameter_error()+ return "ok"++ monkeypatch.setattr(llm, "_handle_completion", reject)+ llm._call_completions(MESSAGES)++ params = build("gpt-5", reasoning_effort="high")._prepare_completion_params(+ MESSAGES+ )+ assert params["reasoning_effort"] == "high"++ def test_openai_itself_is_held_to_the_known_shapes(self, monkeypatch):+ """The lenient match is for other servers; OpenAI's 400s stay precise."""+ llm = build("gpt-5", reasoning_effort="low")+ calls: list[dict] = []++ def always_fail(params, **kwargs):+ calls.append(params)+ raise _bad_request("Something else about reasoning_effort.", param=None)++ monkeypatch.setattr(llm, "_handle_completion", always_fail)++ with pytest.raises(BadRequestError, match="Something else"):+ llm._call_completions(MESSAGES)++ assert len(calls) == 1+++class TestLLMSurface:+ @pytest.mark.parametrize(+ "effort", ["none", "minimal", "low", "medium", "high", "xhigh"]+ )+ def test_llm_accepts_every_documented_effort(self, effort):+ llm = LLM(model="gpt-5", reasoning_effort=effort, is_litellm=True)++ assert llm.reasoning_effort == effort++ def test_reaches_the_wire_through_the_llm_factory(self):+ """End to end: LLM(...) -> native provider -> request params."""+ llm = LLM(model="gpt-5", reasoning_effort="minimal", is_litellm=False)+ params = llm._prepare_completion_params(MESSAGES)++ assert params["reasoning_effort"] == "minimal"