The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(langchain_v1): add async implementations to wrap_model_call (#33467)
Eugene Yurtsevd38729fbac4b
diff --git a/libs/langchain_v1/langchain/agents/middleware/context_editing.py b/libs/langchain_v1/langchain/agents/middleware/context_editing.pyindex a0d9084572..746fc120cb 100644--- a/libs/langchain_v1/langchain/agents/middleware/context_editing.py+++ b/libs/langchain_v1/langchain/agents/middleware/context_editing.py@@ -8,7 +8,7 @@ with any LangChain chat model.from __future__ import annotations-from collections.abc import Callable, Iterable, Sequence+from collections.abc import Awaitable, Callable, Iterable, Sequencefrom dataclasses import dataclassfrom typing import Literal@@ -239,6 +239,34 @@ class ContextEditingMiddleware(AgentMiddleware):return handler(request)+ async def awrap_model_call(+ self,+ request: ModelRequest,+ handler: Callable[[ModelRequest], Awaitable[ModelResponse]],+ ) -> ModelCallResult:+ """Apply context edits before invoking the model via handler (async version)."""+ if not request.messages:+ return await handler(request)++ if self.token_count_method == "approximate": # noqa: S105++ def count_tokens(messages: Sequence[BaseMessage]) -> int:+ return count_tokens_approximately(messages)+ else:+ system_msg = (+ [SystemMessage(content=request.system_prompt)] if request.system_prompt else []+ )++ def count_tokens(messages: Sequence[BaseMessage]) -> int:+ return request.model.get_num_tokens_from_messages(+ system_msg + list(messages), request.tools+ )++ for edit in self.edits:+ edit.apply(request.messages, count_tokens=count_tokens)++ return await handler(request)+__all__ = ["ClearToolUsesEdit",diff --git a/libs/langchain_v1/langchain/agents/middleware/model_fallback.py b/libs/langchain_v1/langchain/agents/middleware/model_fallback.pyindex c4d4f6b9cb..8c88f4953f 100644--- a/libs/langchain_v1/langchain/agents/middleware/model_fallback.py+++ b/libs/langchain_v1/langchain/agents/middleware/model_fallback.py@@ -13,7 +13,7 @@ from langchain.agents.middleware.types import (from langchain.chat_models import init_chat_modelif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Awaitable, Callablefrom langchain_core.language_models.chat_models import BaseChatModel@@ -102,3 +102,38 @@ class ModelFallbackMiddleware(AgentMiddleware):continueraise last_exception++ async def awrap_model_call(+ self,+ request: ModelRequest,+ handler: Callable[[ModelRequest], Awaitable[ModelResponse]],+ ) -> ModelCallResult:+ """Try fallback models in sequence on errors (async version).++ Args:+ request: Initial model request.+ handler: Async callback to execute the model.++ Returns:+ AIMessage from successful model call.++ Raises:+ Exception: If all models fail, re-raises last exception.+ """+ # Try primary model first+ last_exception: Exception+ try:+ return await handler(request)+ except Exception as e: # noqa: BLE001+ last_exception = e++ # Try fallback models+ for fallback_model in self.models:+ request.model = fallback_model+ try:+ return await handler(request)+ except Exception as e: # noqa: BLE001+ last_exception = e+ continue++ raise last_exceptiondiff --git a/libs/langchain_v1/langchain/agents/middleware/todo.py b/libs/langchain_v1/langchain/agents/middleware/todo.pyindex 4ae206584a..beeee6350f 100644--- a/libs/langchain_v1/langchain/agents/middleware/todo.py+++ b/libs/langchain_v1/langchain/agents/middleware/todo.py@@ -6,7 +6,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING, Annotated, Literalif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Awaitable, Callablefrom langchain_core.messages import ToolMessagefrom langchain_core.tools import tool@@ -204,3 +204,16 @@ class TodoListMiddleware(AgentMiddleware):else self.system_prompt)return handler(request)++ async def awrap_model_call(+ self,+ request: ModelRequest,+ handler: Callable[[ModelRequest], Awaitable[ModelResponse]],+ ) -> ModelCallResult:+ """Update the system prompt to include the todo system prompt (async version)."""+ request.system_prompt = (+ request.system_prompt + "\n\n" + self.system_prompt+ if request.system_prompt+ else self.system_prompt+ )+ return await handler(request)diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_context_editing_middleware.py b/libs/langchain_v1/tests/unit_tests/agents/test_context_editing_middleware.pyindex 3ba8bb285b..7a9d901b95 100644--- a/libs/langchain_v1/tests/unit_tests/agents/test_context_editing_middleware.py+++ b/libs/langchain_v1/tests/unit_tests/agents/test_context_editing_middleware.py@@ -233,3 +233,169 @@ def test_exclude_tools_prevents_clearing() -> None:def _fake_runtime() -> Runtime:return cast(Runtime, object())+++async def test_no_edit_when_below_trigger_async() -> None:+ """Test async version of context editing with no edit when below trigger."""+ tool_call_id = "call-1"+ ai_message = AIMessage(+ content="",+ tool_calls=[{"id": tool_call_id, "name": "search", "args": {}}],+ )+ tool_message = ToolMessage(content="12345", tool_call_id=tool_call_id)++ state, request = _make_state_and_request([ai_message, tool_message])+ middleware = ContextEditingMiddleware(+ edits=[ClearToolUsesEdit(trigger=50)],+ )++ async def mock_handler(req: ModelRequest) -> AIMessage:+ return AIMessage(content="mock response")++ # Call awrap_model_call which modifies the request+ await middleware.awrap_model_call(request, mock_handler)++ # The request should have been modified in place+ assert request.messages[0].content == ""+ assert request.messages[1].content == "12345"+ assert state["messages"] == request.messages+++async def test_clear_tool_outputs_and_inputs_async() -> None:+ """Test async version of clearing tool outputs and inputs."""+ tool_call_id = "call-2"+ ai_message = AIMessage(+ content=[+ {"type": "tool_call", "id": tool_call_id, "name": "search", "args": {"query": "foo"}}+ ],+ tool_calls=[{"id": tool_call_id, "name": "search", "args": {"query": "foo"}}],+ )+ tool_message = ToolMessage(content="x" * 200, tool_call_id=tool_call_id)++ state, request = _make_state_and_request([ai_message, tool_message])++ edit = ClearToolUsesEdit(+ trigger=50,+ clear_at_least=10,+ clear_tool_inputs=True,+ keep=0,+ placeholder="[cleared output]",+ )+ middleware = ContextEditingMiddleware(edits=[edit])++ async def mock_handler(req: ModelRequest) -> AIMessage:+ return AIMessage(content="mock response")++ # Call awrap_model_call which modifies the request+ await middleware.awrap_model_call(request, mock_handler)++ cleared_ai = request.messages[0]+ cleared_tool = request.messages[1]++ assert isinstance(cleared_tool, ToolMessage)+ assert cleared_tool.content == "[cleared output]"+ assert cleared_tool.response_metadata["context_editing"]["cleared"] is True++ assert isinstance(cleared_ai, AIMessage)+ assert cleared_ai.tool_calls[0]["args"] == {}+ context_meta = cleared_ai.response_metadata.get("context_editing")+ assert context_meta is not None+ assert context_meta["cleared_tool_inputs"] == [tool_call_id]++ assert state["messages"] == request.messages+++async def test_respects_keep_last_tool_results_async() -> None:+ """Test async version respects keep parameter for last tool results."""+ conversation: list[AIMessage | ToolMessage] = []+ edits = [+ ("call-a", "tool-output-a" * 5),+ ("call-b", "tool-output-b" * 5),+ ("call-c", "tool-output-c" * 5),+ ]++ for call_id, text in edits:+ conversation.append(+ AIMessage(+ content="",+ tool_calls=[{"id": call_id, "name": "tool", "args": {"input": call_id}}],+ )+ )+ conversation.append(ToolMessage(content=text, tool_call_id=call_id))++ state, request = _make_state_and_request(conversation)++ middleware = ContextEditingMiddleware(+ edits=[+ ClearToolUsesEdit(+ trigger=50,+ keep=1,+ placeholder="[cleared]",+ )+ ],+ token_count_method="model",+ )++ async def mock_handler(req: ModelRequest) -> AIMessage:+ return AIMessage(content="mock response")++ # Call awrap_model_call which modifies the request+ await middleware.awrap_model_call(request, mock_handler)++ cleared_messages = [+ msg+ for msg in request.messages+ if isinstance(msg, ToolMessage) and msg.content == "[cleared]"+ ]++ assert len(cleared_messages) == 2+ assert isinstance(request.messages[-1], ToolMessage)+ assert request.messages[-1].content != "[cleared]"+++async def test_exclude_tools_prevents_clearing_async() -> None:+ """Test async version of excluding tools from clearing."""+ search_call = "call-search"+ calc_call = "call-calc"++ state, request = _make_state_and_request(+ [+ AIMessage(+ content="",+ tool_calls=[{"id": search_call, "name": "search", "args": {"query": "foo"}}],+ ),+ ToolMessage(content="search-results" * 20, tool_call_id=search_call),+ AIMessage(+ content="",+ tool_calls=[{"id": calc_call, "name": "calculator", "args": {"a": 1, "b": 2}}],+ ),+ ToolMessage(content="42", tool_call_id=calc_call),+ ]+ )++ middleware = ContextEditingMiddleware(+ edits=[+ ClearToolUsesEdit(+ trigger=50,+ clear_at_least=10,+ keep=0,+ exclude_tools=("search",),+ placeholder="[cleared]",+ )+ ],+ )++ async def mock_handler(req: ModelRequest) -> AIMessage:+ return AIMessage(content="mock response")++ # Call awrap_model_call which modifies the request+ await middleware.awrap_model_call(request, mock_handler)++ search_tool = request.messages[1]+ calc_tool = request.messages[3]++ assert isinstance(search_tool, ToolMessage)+ assert search_tool.content == "search-results" * 20++ assert isinstance(calc_tool, ToolMessage)+ assert calc_tool.content == "[cleared]"diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_model_fallback_middleware.py b/libs/langchain_v1/tests/unit_tests/agents/test_model_fallback_middleware.pynew file mode 100644index 0000000000..2c24296fc1--- /dev/null+++ b/libs/langchain_v1/tests/unit_tests/agents/test_model_fallback_middleware.py@@ -0,0 +1,215 @@+"""Unit tests for ModelFallbackMiddleware."""++from __future__ import annotations++from typing import cast++import pytest+from langchain_core.language_models.fake_chat_models import GenericFakeChatModel+from langchain_core.messages import AIMessage++from langchain.agents.middleware.model_fallback import ModelFallbackMiddleware+from langchain.agents.middleware.types import ModelRequest, ModelResponse+from langgraph.runtime import Runtime+++def _fake_runtime() -> Runtime:+ return cast(Runtime, object())+++def _make_request() -> ModelRequest:+ """Create a minimal ModelRequest for testing."""+ model = GenericFakeChatModel(messages=iter([AIMessage(content="primary")]))+ return ModelRequest(+ model=model,+ system_prompt=None,+ messages=[],+ tool_choice=None,+ tools=[],+ response_format=None,+ state=cast("AgentState", {}), # type: ignore[name-defined]+ runtime=_fake_runtime(),+ model_settings={},+ )+++def test_primary_model_succeeds() -> None:+ """Test that primary model is used when it succeeds."""+ primary_model = GenericFakeChatModel(messages=iter([AIMessage(content="primary response")]))+ fallback_model = GenericFakeChatModel(messages=iter([AIMessage(content="fallback response")]))++ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request()+ request.model = primary_model++ def mock_handler(req: ModelRequest) -> ModelResponse:+ # Simulate successful model call+ result = req.model.invoke([])+ return ModelResponse(result=[result])++ response = middleware.wrap_model_call(request, mock_handler)++ assert isinstance(response, ModelResponse)+ assert response.result[0].content == "primary response"+++def test_fallback_on_primary_failure() -> None:+ """Test that fallback model is used when primary fails."""++ class FailingPrimaryModel(GenericFakeChatModel):+ def _generate(self, messages, **kwargs):+ raise ValueError("Primary model failed")++ primary_model = FailingPrimaryModel(messages=iter([AIMessage(content="should not see")]))+ fallback_model = GenericFakeChatModel(messages=iter([AIMessage(content="fallback response")]))++ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request()+ request.model = primary_model++ def mock_handler(req: ModelRequest) -> ModelResponse:+ result = req.model.invoke([])+ return ModelResponse(result=[result])++ response = middleware.wrap_model_call(request, mock_handler)++ assert isinstance(response, ModelResponse)+ assert response.result[0].content == "fallback response"+++def test_multiple_fallbacks() -> None:+ """Test that multiple fallback models are tried in sequence."""++ class FailingModel(GenericFakeChatModel):+ def _generate(self, messages, **kwargs):+ raise ValueError("Model failed")++ primary_model = FailingModel(messages=iter([AIMessage(content="should not see")]))+ fallback1 = FailingModel(messages=iter([AIMessage(content="fallback1")]))+ fallback2 = GenericFakeChatModel(messages=iter([AIMessage(content="fallback2")]))++ middleware = ModelFallbackMiddleware(fallback1, fallback2)+ request = _make_request()+ request.model = primary_model++ def mock_handler(req: ModelRequest) -> ModelResponse:+ result = req.model.invoke([])+ return ModelResponse(result=[result])++ response = middleware.wrap_model_call(request, mock_handler)++ assert isinstance(response, ModelResponse)+ assert response.result[0].content == "fallback2"+++def test_all_models_fail() -> None:+ """Test that exception is raised when all models fail."""++ class AlwaysFailingModel(GenericFakeChatModel):+ def _generate(self, messages, **kwargs):+ raise ValueError("Model failed")++ primary_model = AlwaysFailingModel(messages=iter([]))+ fallback_model = AlwaysFailingModel(messages=iter([]))++ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request()+ request.model = primary_model++ def mock_handler(req: ModelRequest) -> ModelResponse:+ result = req.model.invoke([])+ return ModelResponse(result=[result])++ with pytest.raises(ValueError, match="Model failed"):+ middleware.wrap_model_call(request, mock_handler)+++async def test_primary_model_succeeds_async() -> None:+ """Test async version - primary model is used when it succeeds."""+ primary_model = GenericFakeChatModel(messages=iter([AIMessage(content="primary response")]))+ fallback_model = GenericFakeChatModel(messages=iter([AIMessage(content="fallback response")]))++ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request()+ request.model = primary_model++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ # Simulate successful async model call+ result = await req.model.ainvoke([])+ return ModelResponse(result=[result])++ response = await middleware.awrap_model_call(request, mock_handler)++ assert isinstance(response, ModelResponse)+ assert response.result[0].content == "primary response"+++async def test_fallback_on_primary_failure_async() -> None:+ """Test async version - fallback model is used when primary fails."""++ class AsyncFailingPrimaryModel(GenericFakeChatModel):+ async def _agenerate(self, messages, **kwargs):+ raise ValueError("Primary model failed")++ primary_model = AsyncFailingPrimaryModel(messages=iter([AIMessage(content="should not see")]))+ fallback_model = GenericFakeChatModel(messages=iter([AIMessage(content="fallback response")]))++ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request()+ request.model = primary_model++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ result = await req.model.ainvoke([])+ return ModelResponse(result=[result])++ response = await middleware.awrap_model_call(request, mock_handler)++ assert isinstance(response, ModelResponse)+ assert response.result[0].content == "fallback response"+++async def test_multiple_fallbacks_async() -> None:+ """Test async version - multiple fallback models are tried in sequence."""++ class AsyncFailingModel(GenericFakeChatModel):+ async def _agenerate(self, messages, **kwargs):+ raise ValueError("Model failed")++ primary_model = AsyncFailingModel(messages=iter([AIMessage(content="should not see")]))+ fallback1 = AsyncFailingModel(messages=iter([AIMessage(content="fallback1")]))+ fallback2 = GenericFakeChatModel(messages=iter([AIMessage(content="fallback2")]))++ middleware = ModelFallbackMiddleware(fallback1, fallback2)+ request = _make_request()+ request.model = primary_model++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ result = await req.model.ainvoke([])+ return ModelResponse(result=[result])++ response = await middleware.awrap_model_call(request, mock_handler)++ assert isinstance(response, ModelResponse)+ assert response.result[0].content == "fallback2"+++async def test_all_models_fail_async() -> None:+ """Test async version - exception is raised when all models fail."""++ class AsyncAlwaysFailingModel(GenericFakeChatModel):+ async def _agenerate(self, messages, **kwargs):+ raise ValueError("Model failed")++ primary_model = AsyncAlwaysFailingModel(messages=iter([]))+ fallback_model = AsyncAlwaysFailingModel(messages=iter([]))++ middleware = ModelFallbackMiddleware(fallback_model)+ request = _make_request()+ request.model = primary_model++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ result = await req.model.ainvoke([])+ return ModelResponse(result=[result])++ with pytest.raises(ValueError, match="Model failed"):+ await middleware.awrap_model_call(request, mock_handler)diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_todo_middleware.py b/libs/langchain_v1/tests/unit_tests/agents/test_todo_middleware.pynew file mode 100644index 0000000000..5f96855d4d--- /dev/null+++ b/libs/langchain_v1/tests/unit_tests/agents/test_todo_middleware.py@@ -0,0 +1,172 @@+"""Unit tests for TodoListMiddleware."""++from __future__ import annotations++from typing import cast++from langchain_core.language_models.fake_chat_models import GenericFakeChatModel+from langchain_core.messages import AIMessage++from langchain.agents.middleware.todo import TodoListMiddleware+from langchain.agents.middleware.types import ModelRequest, ModelResponse+from langgraph.runtime import Runtime+++def _fake_runtime() -> Runtime:+ return cast(Runtime, object())+++def _make_request(system_prompt: str | None = None) -> ModelRequest:+ """Create a minimal ModelRequest for testing."""+ model = GenericFakeChatModel(messages=iter([AIMessage(content="response")]))+ return ModelRequest(+ model=model,+ system_prompt=system_prompt,+ messages=[],+ tool_choice=None,+ tools=[],+ response_format=None,+ state=cast("AgentState", {}), # type: ignore[name-defined]+ runtime=_fake_runtime(),+ model_settings={},+ )+++def test_adds_system_prompt_when_none_exists() -> None:+ """Test that middleware adds system prompt when request has none."""+ middleware = TodoListMiddleware()+ request = _make_request(system_prompt=None)++ def mock_handler(req: ModelRequest) -> ModelResponse:+ return ModelResponse(result=[AIMessage(content="response")])++ middleware.wrap_model_call(request, mock_handler)++ # System prompt should be set+ assert request.system_prompt is not None+ assert "write_todos" in request.system_prompt+++def test_appends_to_existing_system_prompt() -> None:+ """Test that middleware appends to existing system prompt."""+ existing_prompt = "You are a helpful assistant."+ middleware = TodoListMiddleware()+ request = _make_request(system_prompt=existing_prompt)++ def mock_handler(req: ModelRequest) -> ModelResponse:+ return ModelResponse(result=[AIMessage(content="response")])++ middleware.wrap_model_call(request, mock_handler)++ # System prompt should contain both+ assert request.system_prompt is not None+ assert existing_prompt in request.system_prompt+ assert "write_todos" in request.system_prompt+ assert request.system_prompt.startswith(existing_prompt)+++def test_custom_system_prompt() -> None:+ """Test that middleware uses custom system prompt."""+ custom_prompt = "Custom planning instructions"+ middleware = TodoListMiddleware(system_prompt=custom_prompt)+ request = _make_request(system_prompt=None)++ def mock_handler(req: ModelRequest) -> ModelResponse:+ return ModelResponse(result=[AIMessage(content="response")])++ middleware.wrap_model_call(request, mock_handler)++ # Should use custom prompt+ assert request.system_prompt == custom_prompt+++def test_has_write_todos_tool() -> None:+ """Test that middleware registers the write_todos tool."""+ middleware = TodoListMiddleware()++ # Should have one tool registered+ assert len(middleware.tools) == 1+ assert middleware.tools[0].name == "write_todos"+++def test_custom_tool_description() -> None:+ """Test that middleware uses custom tool description."""+ custom_description = "Custom todo tool description"+ middleware = TodoListMiddleware(tool_description=custom_description)++ # Tool should use custom description+ assert len(middleware.tools) == 1+ assert middleware.tools[0].description == custom_description+++# ==============================================================================+# Async Tests+# ==============================================================================+++async def test_adds_system_prompt_when_none_exists_async() -> None:+ """Test async version - middleware adds system prompt when request has none."""+ middleware = TodoListMiddleware()+ request = _make_request(system_prompt=None)++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ return ModelResponse(result=[AIMessage(content="response")])++ await middleware.awrap_model_call(request, mock_handler)++ # System prompt should be set+ assert request.system_prompt is not None+ assert "write_todos" in request.system_prompt+++async def test_appends_to_existing_system_prompt_async() -> None:+ """Test async version - middleware appends to existing system prompt."""+ existing_prompt = "You are a helpful assistant."+ middleware = TodoListMiddleware()+ request = _make_request(system_prompt=existing_prompt)++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ return ModelResponse(result=[AIMessage(content="response")])++ await middleware.awrap_model_call(request, mock_handler)++ # System prompt should contain both+ assert request.system_prompt is not None+ assert existing_prompt in request.system_prompt+ assert "write_todos" in request.system_prompt+ assert request.system_prompt.startswith(existing_prompt)+++async def test_custom_system_prompt_async() -> None:+ """Test async version - middleware uses custom system prompt."""+ custom_prompt = "Custom planning instructions"+ middleware = TodoListMiddleware(system_prompt=custom_prompt)+ request = _make_request(system_prompt=None)++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ return ModelResponse(result=[AIMessage(content="response")])++ await middleware.awrap_model_call(request, mock_handler)++ # Should use custom prompt+ assert request.system_prompt == custom_prompt+++async def test_handler_called_with_modified_request_async() -> None:+ """Test async version - handler receives the modified request."""+ middleware = TodoListMiddleware()+ request = _make_request(system_prompt="Original")+ handler_called = {"value": False}+ received_prompt = {"value": None}++ async def mock_handler(req: ModelRequest) -> ModelResponse:+ handler_called["value"] = True+ received_prompt["value"] = req.system_prompt+ return ModelResponse(result=[AIMessage(content="response")])++ await middleware.awrap_model_call(request, mock_handler)++ assert handler_called["value"]+ assert received_prompt["value"] is not None+ assert "Original" in received_prompt["value"]+ assert "write_todos" in received_prompt["value"]