The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(langchain): handle malformed structured-output responses (#39245)
Nishitha M1bdaf0c133de
diff --git a/libs/langchain_v1/langchain/agents/middleware/tool_selection.py b/libs/langchain_v1/langchain/agents/middleware/tool_selection.pyindex 98cf01e4b5..e55e267f10 100644--- a/libs/langchain_v1/langchain/agents/middleware/tool_selection.py+++ b/libs/langchain_v1/langchain/agents/middleware/tool_selection.py@@ -3,11 +3,12 @@from __future__ import annotationsimport logging+from collections.abc import Callablefrom dataclasses import dataclass-from typing import TYPE_CHECKING, Annotated, Any, Literal, Union+from typing import TYPE_CHECKING, Annotated, Any, Literal, TypeGuard, Unionfrom langchain_core.language_models.chat_models import BaseChatModel-from langchain_core.messages import AIMessage, HumanMessage+from langchain_core.messages import AIMessage, BaseMessage, HumanMessagefrom pydantic import Field, TypeAdapterfrom typing_extensions import TypedDict@@ -22,7 +23,9 @@ from langchain.agents.middleware.types import (from langchain.chat_models.base import init_chat_modelif TYPE_CHECKING:- from collections.abc import Awaitable, Callable+ from collections.abc import Awaitable++ from langchain_core.runnables import RunnableConfigfrom langchain.tools import BaseTool@@ -32,6 +35,17 @@ DEFAULT_SYSTEM_PROMPT = ("Your goal is to select the most relevant tools for answering the user's query.")+OnParsingFailure = Literal["error", "none", "all"] | list[str] | Callable[[Any], list[str]]+"""Behavior when the selection model keeps returning a malformed response.++Can be either:+- `'error'`: Raise a `ValueError` (the default).+- `'none'`: Select no tools.+- `'all'`: Select every available tool.+- A `list[str]` of tool names to fall back to.+- A callable that takes the last (malformed) response and returns tool names to use.+"""+@dataclassclass _SelectionRequest:@@ -78,6 +92,18 @@ def _create_tool_selection_response(tools: list[BaseTool]) -> TypeAdapter[Any]:return TypeAdapter(ToolSelectionResponse)+def _is_valid_selection_response(response: Any) -> TypeGuard[dict[str, Any]]:+ """Check whether a structured-output response is a well-formed tool selection.++ Args:+ response: Raw response from the structured-output model call.++ Returns:+ `True` if `response` is a dict with a `tools` list, `False` otherwise.+ """+ return isinstance(response, dict) and isinstance(response.get("tools"), list)++def _render_tool_list(tools: list[BaseTool]) -> str:"""Format tools as markdown list.@@ -126,6 +152,8 @@ class LLMToolSelectorMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT,system_prompt: str = DEFAULT_SYSTEM_PROMPT,max_tools: int | None = None,always_include: list[str] | None = None,+ max_retries: int = 1,+ on_parsing_failure: OnParsingFailure = "error",) -> None:"""Initialize the tool selector.@@ -144,11 +172,39 @@ class LLMToolSelectorMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT,always_include: Tool names to always include regardless of selection.These do not count against the `max_tools` limit.+ max_retries: Maximum number of retry attempts after the initial call if+ the selection model returns a malformed response (not a dict with a+ `tools` list).++ Must be `>= 0`.+ on_parsing_failure: Behavior once `max_retries` is exhausted and the+ response is still malformed.++ Options:++ - `'error'` (default): Raise a `ValueError`.+ - `'none'`: Select no tools.+ - `'all'`: Select every available tool.+ - A `list[str]` of tool names to fall back to.+ - A callable that takes the last (malformed) response and returns+ the tool names to use.++ Unlike a normal model selection, the fallback tools are not capped+ by `max_tools` -- it's an already-deliberate choice, not raw model+ output that needs bounding.++ Raises:+ ValueError: If `max_retries < 0`."""super().__init__()+ if max_retries < 0:+ msg = "max_retries must be >= 0"+ raise ValueError(msg)self.system_prompt = system_promptself.max_tools = max_toolsself.always_include = always_include or []+ self.max_retries = max_retries+ self.on_parsing_failure = on_parsing_failureif isinstance(model, (BaseChatModel, type(None))):self.model: BaseChatModel | None = model@@ -234,19 +290,34 @@ class LLMToolSelectorMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT,available_tools: list[BaseTool],valid_tool_names: list[str],request: ModelRequest[ContextT],+ *,+ apply_max_tools: bool = True,) -> ModelRequest[ContextT]:- """Process the selection response and return filtered `ModelRequest`."""+ """Process the selection response and return filtered `ModelRequest`.++ Args:+ response: Selection response, expected to have a `tools` list.+ available_tools: Tools eligible for selection.+ valid_tool_names: Names of `available_tools`.+ request: Original model request to override.+ apply_max_tools: Whether to cap the selection at `max_tools`.++ Set to `False` for an already-deliberate fallback selection (e.g.+ `on_parsing_failure="all"`), where truncating would contradict the+ fallback's own semantics.+ """selected_tool_names: list[str] = []invalid_tool_selections = []+ max_tools = self.max_tools if apply_max_tools else None- for tool_name in response["tools"]:+ for tool_name in response.get("tools", []):if tool_name not in valid_tool_names:invalid_tool_selections.append(tool_name)continue# Only add if not already selected and within max_tools limitif tool_name not in selected_tool_names and (- self.max_tools is None or len(selected_tool_names) < self.max_tools+ max_tools is None or len(selected_tool_names) < max_tools):selected_tool_names.append(tool_name)@@ -270,6 +341,34 @@ class LLMToolSelectorMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT,return request.override(tools=[*selected_tools, *provider_tools])+ def _resolve_parsing_failure(self, response: Any, valid_tool_names: list[str]) -> list[str]:+ """Determine which tool names to use once `max_retries` is exhausted.++ Args:+ response: The last (still malformed) response from the selection model.+ valid_tool_names: Tool names available for selection.++ Returns:+ Tool names to select, per `on_parsing_failure`.++ Raises:+ ValueError: If `on_parsing_failure == "error"` (the default).+ """+ if self.on_parsing_failure == "error":+ msg = (+ "LLMToolSelectorMiddleware: selection model returned a malformed "+ f"response after {self.max_retries} retries (expected a dict with a "+ f"'tools' list): {response!r}"+ )+ raise ValueError(msg)+ if self.on_parsing_failure == "none":+ return []+ if self.on_parsing_failure == "all":+ return valid_tool_names+ if callable(self.on_parsing_failure):+ return self.on_parsing_failure(response)+ return list(self.on_parsing_failure)+def wrap_model_call(self,request: ModelRequest[ContextT],@@ -285,7 +384,9 @@ class LLMToolSelectorMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT,The model call result.Raises:- AssertionError: If the selection model response is not a dict.+ ValueError: If `on_parsing_failure == "error"` (the default) and the+ selection model returns a malformed response after `max_retries`+ retries."""selection_request = self._prepare_selection_request(request)if selection_request is None:@@ -296,20 +397,41 @@ class LLMToolSelectorMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT,schema = type_adapter.json_schema()structured_model = selection_request.model.with_structured_output(schema)- response = structured_model.invoke(- [- {"role": "system", "content": selection_request.system_message},- selection_request.last_user_message,- ]- )-- # Response should be a dict since we're passing a schema (not a Pydantic model class)- if not isinstance(response, dict):- msg = f"Expected dict response, got {type(response)}"- raise AssertionError(msg) # noqa: TRY004- modified_request = self._process_selection_response(- response, selection_request.available_tools, selection_request.valid_tool_names, request- )+ messages: list[BaseMessage | dict[str, Any]] = [+ {"role": "system", "content": selection_request.system_message},+ selection_request.last_user_message,+ ]+ config: RunnableConfig = {"metadata": {"lc_source": "tool_selection"}}++ response = structured_model.invoke(messages, config=config)+ attempts = 0+ while not _is_valid_selection_response(response) and attempts < self.max_retries:+ # Malformed structured output is usually transient, retry before giving+ # up, rather than silently degrading to no/all tools selected.+ response = structured_model.invoke(messages, config=config)+ attempts += 1++ if _is_valid_selection_response(response):+ modified_request = self._process_selection_response(+ response,+ selection_request.available_tools,+ selection_request.valid_tool_names,+ request,+ )+ else:+ fallback_tool_names = self._resolve_parsing_failure(+ response, selection_request.valid_tool_names+ )+ # Bypass max_tools: the fallback is already a deliberate selection+ # (e.g. `on_parsing_failure="all"`), not raw model output that needs+ # to be capped.+ modified_request = self._process_selection_response(+ {"tools": fallback_tool_names},+ selection_request.available_tools,+ selection_request.valid_tool_names,+ request,+ apply_max_tools=False,+ )return handler(modified_request)async def awrap_model_call(@@ -327,7 +449,9 @@ class LLMToolSelectorMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT,The model call result.Raises:- AssertionError: If the selection model response is not a dict.+ ValueError: If `on_parsing_failure == "error"` (the default) and the+ selection model returns a malformed response after `max_retries`+ retries."""selection_request = self._prepare_selection_request(request)if selection_request is None:@@ -338,18 +462,39 @@ class LLMToolSelectorMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT,schema = type_adapter.json_schema()structured_model = selection_request.model.with_structured_output(schema)- response = await structured_model.ainvoke(- [- {"role": "system", "content": selection_request.system_message},- selection_request.last_user_message,- ]- )-- # Response should be a dict since we're passing a schema (not a Pydantic model class)- if not isinstance(response, dict):- msg = f"Expected dict response, got {type(response)}"- raise AssertionError(msg) # noqa: TRY004- modified_request = self._process_selection_response(- response, selection_request.available_tools, selection_request.valid_tool_names, request- )+ messages: list[BaseMessage | dict[str, Any]] = [+ {"role": "system", "content": selection_request.system_message},+ selection_request.last_user_message,+ ]+ config: RunnableConfig = {"metadata": {"lc_source": "tool_selection"}}++ response = await structured_model.ainvoke(messages, config=config)+ attempts = 0+ while not _is_valid_selection_response(response) and attempts < self.max_retries:+ # Malformed structured output is usually transient, retry before giving+ # up, rather than silently degrading to no/all tools selected.+ response = await structured_model.ainvoke(messages, config=config)+ attempts += 1++ if _is_valid_selection_response(response):+ modified_request = self._process_selection_response(+ response,+ selection_request.available_tools,+ selection_request.valid_tool_names,+ request,+ )+ else:+ fallback_tool_names = self._resolve_parsing_failure(+ response, selection_request.valid_tool_names+ )+ # Bypass max_tools: the fallback is already a deliberate selection+ # (e.g. `on_parsing_failure="all"`), not raw model output that needs+ # to be capped.+ modified_request = self._process_selection_response(+ {"tools": fallback_tool_names},+ selection_request.available_tools,+ selection_request.valid_tool_names,+ request,+ apply_max_tools=False,+ )return await handler(modified_request)diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_selection.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_selection.pyindex 231583ce2b..a2eecf82e2 100644--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_selection.py+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_selection.py@@ -641,3 +641,224 @@ class TestEdgeCases:"""Test that empty tools list raises an error in schema creation."""with pytest.raises(AssertionError, match="tools must be non-empty"):_create_tool_selection_response([])+++def _malformed_response_model(times: int) -> FakeModel:+ """A selection model that returns a malformed response `times` times in a row."""+ malformed = AIMessage(+ content="", tool_calls=[{"name": "ToolSelectionResponse", "id": "1", "args": {}}]+ )+ valid = AIMessage(+ content="",+ tool_calls=[+ {"name": "ToolSelectionResponse", "id": "2", "args": {"tools": ["get_weather"]}}+ ],+ )+ return FakeModel(messages=iter([malformed] * times + [valid]))+++class TestMalformedSelectionResponse:+ """Test retry and fallback behavior when the selection model returns malformed output."""++ def test_recovers_within_max_retries(self) -> None:+ """Test that a malformed response is retried and recovers within `max_retries`."""+ tool_selection_model = _malformed_response_model(times=1)+ model = FakeModel(messages=iter([AIMessage(content="Done")]))+ tool_selector = LLMToolSelectorMiddleware(max_tools=1, model=tool_selection_model)++ agent = create_agent(+ model=model, tools=[get_weather, search_web], middleware=[tool_selector]+ )++ response = agent.invoke({"messages": [HumanMessage("test")]})++ assert isinstance(response["messages"][-1], AIMessage)++ def test_default_raises_after_max_retries_exhausted(self) -> None:+ """Test that the default `on_parsing_failure='error'` raises a clear `ValueError`."""+ tool_selection_model = _malformed_response_model(times=100)+ model = FakeModel(messages=iter([AIMessage(content="Done")]))+ tool_selector = LLMToolSelectorMiddleware(model=tool_selection_model)++ agent = create_agent(+ model=model, tools=[get_weather, search_web], middleware=[tool_selector]+ )++ with pytest.raises(ValueError, match="malformed"):+ agent.invoke({"messages": [HumanMessage("test")]})++ async def test_async_default_raises_after_max_retries_exhausted(self) -> None:+ """Async counterpart: the default behavior also raises via `awrap_model_call`."""+ tool_selection_model = _malformed_response_model(times=100)+ model = FakeModel(messages=iter([AIMessage(content="Done")]))+ tool_selector = LLMToolSelectorMiddleware(model=tool_selection_model)++ agent = create_agent(+ model=model, tools=[get_weather, search_web], middleware=[tool_selector]+ )++ with pytest.raises(ValueError, match="malformed"):+ await agent.ainvoke({"messages": [HumanMessage("test")]})++ def test_max_retries_zero_gives_up_immediately(self) -> None:+ """Test that `max_retries=0` doesn't retry before applying `on_parsing_failure`."""+ tool_selection_model = _malformed_response_model(times=1)+ model = FakeModel(messages=iter([AIMessage(content="Done")]))+ tool_selector = LLMToolSelectorMiddleware(max_retries=0, model=tool_selection_model)++ agent = create_agent(+ model=model, tools=[get_weather, search_web], middleware=[tool_selector]+ )++ with pytest.raises(ValueError, match="malformed"):+ agent.invoke({"messages": [HumanMessage("test")]})++ def test_max_retries_negative_rejected_at_construction(self) -> None:+ """Test that a negative `max_retries` is rejected eagerly."""+ with pytest.raises(ValueError, match="max_retries must be >= 0"):+ LLMToolSelectorMiddleware(max_retries=-1)++ def test_on_parsing_failure_none_selects_no_tools(self) -> None:+ """Test that `on_parsing_failure='none'` selects no tools once retries are exhausted."""+ model_requests: list[ModelRequest] = []++ @wrap_model_call+ def trace_model_requests(+ request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]+ ) -> ModelResponse:+ model_requests.append(request)+ return handler(request)++ tool_selection_model = _malformed_response_model(times=100)+ model = FakeModel(messages=iter([AIMessage(content="Done")]))+ tool_selector = LLMToolSelectorMiddleware(+ on_parsing_failure="none", model=tool_selection_model+ )++ agent = create_agent(+ model=model,+ tools=[get_weather, search_web],+ middleware=[tool_selector, trace_model_requests],+ )+ agent.invoke({"messages": [HumanMessage("test")]})++ assert len(model_requests) > 0+ for request in model_requests:+ assert request.tools == []++ def test_on_parsing_failure_all_ignores_max_tools(self) -> None:+ """Test that `on_parsing_failure='all'` selects every tool, bypassing `max_tools`."""+ model_requests: list[ModelRequest] = []++ @wrap_model_call+ def trace_model_requests(+ request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]+ ) -> ModelResponse:+ model_requests.append(request)+ return handler(request)++ tool_selection_model = _malformed_response_model(times=100)+ model = FakeModel(messages=iter([AIMessage(content="Done")]))+ # max_tools=1 would normally cap a real selection at one tool; the "all"+ # fallback must not be truncated by it.+ tool_selector = LLMToolSelectorMiddleware(+ max_tools=1, on_parsing_failure="all", model=tool_selection_model+ )++ agent = create_agent(+ model=model,+ tools=[get_weather, search_web, calculate],+ middleware=[tool_selector, trace_model_requests],+ )+ agent.invoke({"messages": [HumanMessage("test")]})++ assert len(model_requests) > 0+ for request in model_requests:+ tool_names = set()+ for tool_ in request.tools:+ assert isinstance(tool_, BaseTool)+ tool_names.add(tool_.name)+ assert tool_names == {"get_weather", "search_web", "calculate"}++ def test_on_parsing_failure_list_falls_back_to_subset(self) -> None:+ """Test that a `list[str]` fallback selects exactly that subset."""+ model_requests: list[ModelRequest] = []++ @wrap_model_call+ def trace_model_requests(+ request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]+ ) -> ModelResponse:+ model_requests.append(request)+ return handler(request)++ tool_selection_model = _malformed_response_model(times=100)+ model = FakeModel(messages=iter([AIMessage(content="Done")]))+ tool_selector = LLMToolSelectorMiddleware(+ on_parsing_failure=["calculate"], model=tool_selection_model+ )++ agent = create_agent(+ model=model,+ tools=[get_weather, search_web, calculate],+ middleware=[tool_selector, trace_model_requests],+ )+ agent.invoke({"messages": [HumanMessage("test")]})++ assert len(model_requests) > 0+ for request in model_requests:+ tool_names = []+ for tool_ in request.tools:+ assert isinstance(tool_, BaseTool)+ tool_names.append(tool_.name)+ assert tool_names == ["calculate"]++ def test_on_parsing_failure_callable_receives_response(self) -> None:+ """Test that a callable fallback receives the last malformed response."""+ received: list[Any] = []++ def fallback(response: Any) -> list[str]:+ received.append(response)+ return ["search_web"]++ model_requests: list[ModelRequest] = []++ @wrap_model_call+ def trace_model_requests(+ request: ModelRequest, handler: Callable[[ModelRequest], ModelResponse]+ ) -> ModelResponse:+ model_requests.append(request)+ return handler(request)++ tool_selection_model = FakeModel(+ messages=cycle(+ [+ AIMessage(+ content="",+ tool_calls=[+ {"name": "ToolSelectionResponse", "id": "1", "args": {"oops": True}}+ ],+ )+ ]+ )+ )+ model = FakeModel(messages=iter([AIMessage(content="Done")]))+ tool_selector = LLMToolSelectorMiddleware(+ on_parsing_failure=fallback, model=tool_selection_model+ )++ agent = create_agent(+ model=model,+ tools=[get_weather, search_web],+ middleware=[tool_selector, trace_model_requests],+ )+ agent.invoke({"messages": [HumanMessage("test")]})++ assert len(model_requests) > 0+ for request in model_requests:+ tool_names = []+ for tool_ in request.tools:+ assert isinstance(tool_, BaseTool)+ tool_names.append(tool_.name)+ assert tool_names == ["search_web"]+ assert len(received) == 1+ assert received[0] == {"oops": True}