The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(langchain_v1): Add retry_model_request middleware hook, add ModelFallbackMiddleware (#33275)
Nuno Camposa9aa3f232dce
diff --git a/.claude/settings.local.json b/.claude/settings.local.jsonindex 8bd1383607..630c4d8fd1 100644--- a/.claude/settings.local.json+++ b/.claude/settings.local.json@@ -2,7 +2,7 @@"permissions": {"allow": ["Bash(uv run:*)",- "Bash(make:*)"+ "Bash(make:*)","WebSearch","WebFetch(domain:ai.pydantic.dev)","WebFetch(domain:openai.github.io)",@@ -12,4 +12,4 @@"deny": [],"ask": []}-}\ No newline at end of file+}diff --git a/libs/langchain_v1/langchain/agents/middleware/__init__.py b/libs/langchain_v1/langchain/agents/middleware/__init__.pyindex 823bfeaf35..5915b75164 100644--- a/libs/langchain_v1/langchain/agents/middleware/__init__.py+++ b/libs/langchain_v1/langchain/agents/middleware/__init__.py@@ -1,6 +1,7 @@"""Middleware plugins for agents."""from .human_in_the_loop import HumanInTheLoopMiddleware+from .model_fallback import ModelFallbackMiddlewarefrom .pii import PIIDetectionError, PIIMiddlewarefrom .planning import PlanningMiddlewarefrom .prompt_caching import AnthropicPromptCachingMiddleware@@ -25,6 +26,7 @@ __all__ = ["AnthropicPromptCachingMiddleware","HumanInTheLoopMiddleware","LLMToolSelectorMiddleware",+ "ModelFallbackMiddleware","ModelRequest","PIIDetectionError","PIIMiddleware",diff --git a/libs/langchain_v1/langchain/agents/middleware/model_fallback.py b/libs/langchain_v1/langchain/agents/middleware/model_fallback.pynew file mode 100644index 0000000000..c3aac7b229--- /dev/null+++ b/libs/langchain_v1/langchain/agents/middleware/model_fallback.py@@ -0,0 +1,94 @@+"""Model fallback middleware for agents."""++from __future__ import annotations++from typing import TYPE_CHECKING++from langchain.agents.middleware.types import AgentMiddleware, AgentState, ModelRequest+from langchain.chat_models import init_chat_model++if TYPE_CHECKING:+ from langchain_core.language_models.chat_models import BaseChatModel+ from langgraph.runtime import Runtime+++class ModelFallbackMiddleware(AgentMiddleware):+ """Middleware that provides automatic model fallback on errors.++ This middleware attempts to retry failed model calls with alternative models+ in sequence. When a model call fails, it tries the next model in the fallback+ list until either a call succeeds or all models have been exhausted.++ Example:+ ```python+ from langchain.agents.middleware.model_fallback import ModelFallbackMiddleware+ from langchain.agents import create_agent++ # Create middleware with fallback models (not including primary)+ fallback = ModelFallbackMiddleware(+ "openai:gpt-4o-mini", # First fallback+ "anthropic:claude-3-5-sonnet-20241022", # Second fallback+ )++ agent = create_agent(+ model="openai:gpt-4o", # Primary model+ middleware=[fallback],+ )++ # If gpt-4o fails, automatically tries gpt-4o-mini, then claude+ result = await agent.invoke({"messages": [HumanMessage("Hello")]})+ ```+ """++ def __init__(+ self,+ first_model: str | BaseChatModel,+ *additional_models: str | BaseChatModel,+ ) -> None:+ """Initialize the model fallback middleware.++ Args:+ first_model: The first fallback model to try when the primary model fails.+ Can be a model name string or BaseChatModel instance.+ *additional_models: Additional fallback models to try, in order.+ Can be model name strings or BaseChatModel instances.+ """+ super().__init__()++ # Initialize all fallback models+ all_models = (first_model, *additional_models)+ self.models: list[BaseChatModel] = []+ for model in all_models:+ if isinstance(model, str):+ self.models.append(init_chat_model(model))+ else:+ self.models.append(model)++ def retry_model_request(+ self,+ error: Exception, # noqa: ARG002+ request: ModelRequest,+ state: AgentState, # noqa: ARG002+ runtime: Runtime, # noqa: ARG002+ attempt: int,+ ) -> ModelRequest | None:+ """Retry with the next fallback model.++ Args:+ error: The exception that occurred during model invocation.+ request: The original model request that failed.+ state: The current agent state.+ runtime: The langgraph runtime.+ attempt: The current attempt number (1-indexed).++ Returns:+ ModelRequest with the next fallback model, or None if all models exhausted.+ """+ # attempt 1 = primary model failed, try models[0] (first fallback)+ fallback_index = attempt - 1+ # All fallback models exhausted+ if fallback_index >= len(self.models):+ return None+ # Try next fallback model+ request.model = self.models[fallback_index]+ return requestdiff --git a/libs/langchain_v1/langchain/agents/middleware/types.py b/libs/langchain_v1/langchain/agents/middleware/types.pyindex db5174839b..6aea90ab9f 100644--- a/libs/langchain_v1/langchain/agents/middleware/types.py+++ b/libs/langchain_v1/langchain/agents/middleware/types.py@@ -167,6 +167,54 @@ class AgentMiddleware(Generic[StateT, ContextT]):) -> dict[str, Any] | None:"""Async logic to run after the model is called."""+ def retry_model_request(+ self,+ error: Exception, # noqa: ARG002+ request: ModelRequest, # noqa: ARG002+ state: StateT, # noqa: ARG002+ runtime: Runtime[ContextT], # noqa: ARG002+ attempt: int, # noqa: ARG002+ ) -> ModelRequest | None:+ """Logic to handle model invocation errors and optionally retry.++ Args:+ error: The exception that occurred during model invocation.+ request: The original model request that failed.+ state: The current agent state.+ runtime: The langgraph runtime.+ attempt: The current attempt number (1-indexed).++ Returns:+ ModelRequest: Modified request to retry with.+ None: Propagate the error (re-raise).+ """+ return None++ async def aretry_model_request(+ self,+ error: Exception,+ request: ModelRequest,+ state: StateT,+ runtime: Runtime[ContextT],+ attempt: int,+ ) -> ModelRequest | None:+ """Async logic to handle model invocation errors and optionally retry.++ Args:+ error: The exception that occurred during model invocation.+ request: The original model request that failed.+ state: The current agent state.+ runtime: The langgraph runtime.+ attempt: The current attempt number (1-indexed).++ Returns:+ ModelRequest: Modified request to retry with.+ None: Propagate the error (re-raise).+ """+ return await run_in_executor(+ None, self.retry_model_request, error, request, state, runtime, attempt+ )+class _CallableWithStateAndRuntime(Protocol[StateT_contra, ContextT]):"""Callable with AgentState and Runtime as arguments."""diff --git a/libs/langchain_v1/langchain/agents/middleware_agent.py b/libs/langchain_v1/langchain/agents/middleware_agent.pyindex c2e5159932..b4f6370446 100644--- a/libs/langchain_v1/langchain/agents/middleware_agent.py+++ b/libs/langchain_v1/langchain/agents/middleware_agent.py@@ -278,6 +278,12 @@ def create_agent( # noqa: PLR0915if m.__class__.after_model is not AgentMiddleware.after_modelor m.__class__.aafter_model is not AgentMiddleware.aafter_model]+ middleware_w_retry = [+ m+ for m in middleware+ if m.__class__.retry_model_request is not AgentMiddleware.retry_model_request+ or m.__class__.aretry_model_request is not AgentMiddleware.aretry_model_request+ ]state_schemas = {m.state_schema for m in middleware}state_schemas.add(AgentState)@@ -526,18 +532,47 @@ def create_agent( # noqa: PLR0915)raise TypeError(msg)- # Get the bound model (with auto-detection if needed)- model_, effective_response_format = _get_bound_model(request)- messages = request.messages- if request.system_prompt:- messages = [SystemMessage(request.system_prompt), *messages]+ # Retry loop for model invocation with error handling+ # Hard limit of 100 attempts to prevent infinite loops from buggy middleware+ max_attempts = 100+ for attempt in range(1, max_attempts + 1):+ try:+ # Get the bound model (with auto-detection if needed)+ model_, effective_response_format = _get_bound_model(request)+ messages = request.messages+ if request.system_prompt:+ messages = [SystemMessage(request.system_prompt), *messages]++ output = model_.invoke(messages)+ return {+ "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,+ "run_model_call_count": state.get("run_model_call_count", 0) + 1,+ **_handle_model_output(output, effective_response_format),+ }+ except Exception as error:+ # Try retry_model_request on each middleware+ for m in middleware_w_retry:+ if m.__class__.retry_model_request is not AgentMiddleware.retry_model_request:+ if retry_request := m.retry_model_request(+ error, request, state, runtime, attempt+ ):+ # Break on first middleware that wants to retry+ request = retry_request+ break+ else:+ msg = (+ f"No synchronous function provided for "+ f'{m.__class__.__name__}.aretry_model_request".'+ "\nEither initialize with a synchronous function or invoke"+ " via the async API (ainvoke, astream, etc.)"+ )+ raise TypeError(msg)+ else:+ raise- output = model_.invoke(messages)- return {- "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,- "run_model_call_count": state.get("run_model_call_count", 0) + 1,- **_handle_model_output(output, effective_response_format),- }+ # If we exit the loop, max attempts exceeded+ msg = f"Maximum retry attempts ({max_attempts}) exceeded"+ raise RuntimeError(msg)async def amodel_request(state: AgentState, runtime: Runtime[ContextT]) -> dict[str, Any]:"""Async model request handler with sequential middleware processing."""@@ -554,18 +589,39 @@ def create_agent( # noqa: PLR0915for m in middleware_w_modify_model_request:await m.amodify_model_request(request, state, runtime)- # Get the bound model (with auto-detection if needed)- model_, effective_response_format = _get_bound_model(request)- messages = request.messages- if request.system_prompt:- messages = [SystemMessage(request.system_prompt), *messages]-- output = await model_.ainvoke(messages)- return {- "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,- "run_model_call_count": state.get("run_model_call_count", 0) + 1,- **_handle_model_output(output, effective_response_format),- }+ # Retry loop for model invocation with error handling+ # Hard limit of 100 attempts to prevent infinite loops from buggy middleware+ max_attempts = 100+ for attempt in range(1, max_attempts + 1):+ try:+ # Get the bound model (with auto-detection if needed)+ model_, effective_response_format = _get_bound_model(request)+ messages = request.messages+ if request.system_prompt:+ messages = [SystemMessage(request.system_prompt), *messages]++ output = await model_.ainvoke(messages)+ return {+ "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,+ "run_model_call_count": state.get("run_model_call_count", 0) + 1,+ **_handle_model_output(output, effective_response_format),+ }+ except Exception as error:+ # Try retry_model_request on each middleware+ for m in middleware_w_retry:+ if retry_request := await m.aretry_model_request(+ error, request, state, runtime, attempt+ ):+ # Break on first middleware that wants to retry+ request = retry_request+ break+ else:+ # If no middleware wants to retry, re-raise the error+ raise++ # If we exit the loop, max attempts exceeded+ msg = f"Maximum retry attempts ({max_attempts}) exceeded"+ raise RuntimeError(msg)# Use sync or async based on model capabilitiesfrom langgraph._internal._runnable import RunnableCallablediff --git a/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py b/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.pyindex 2716d5c473..ee34a47d7d 100644--- a/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py+++ b/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py@@ -40,6 +40,7 @@ from langchain.agents.middleware.call_tracking import (ModelCallLimitMiddleware,ModelCallLimitExceededError,)+from langchain.agents.middleware.model_fallback import ModelFallbackMiddlewarefrom langchain.agents.middleware.prompt_caching import AnthropicPromptCachingMiddlewarefrom langchain.agents.middleware.summarization import SummarizationMiddlewarefrom langchain.agents.middleware.types import (@@ -2109,6 +2110,278 @@ async def test_create_agent_mixed_sync_async_middleware() -> None:]+# Tests for retry_model_request hook+def test_retry_model_request_hook() -> None:+ """Test that retry_model_request hook is called on model errors."""+ call_count = {"value": 0}++ class FailingModel(BaseChatModel):+ """Model that fails on first call, succeeds on second."""++ def _generate(self, messages, **kwargs):+ call_count["value"] += 1+ if call_count["value"] == 1:+ raise ValueError("First call fails")+ return ChatResult(+ generations=[ChatGeneration(message=AIMessage(content="Success on retry"))]+ )++ @property+ def _llm_type(self):+ return "failing"++ class RetryMiddleware(AgentMiddleware):+ def __init__(self):+ super().__init__()+ self.retry_count = 0++ def retry_model_request(self, error, request, state, runtime, attempt):+ self.retry_count += 1+ # Return the same request to retry+ return request++ failing_model = FailingModel()+ retry_middleware = RetryMiddleware()++ agent = create_agent(model=failing_model, middleware=[retry_middleware]).compile()++ result = agent.invoke({"messages": [HumanMessage("Test")]})++ # Should have retried once+ assert retry_middleware.retry_count == 1+ # Should have succeeded on second attempt+ assert len(result["messages"]) == 2+ assert result["messages"][1].content == "Success on retry"+++def test_retry_model_request_attempt_number() -> None:+ """Test that attempt number is correctly passed to retry_model_request."""++ class AlwaysFailingModel(BaseChatModel):+ """Model that always fails."""++ def _generate(self, messages, **kwargs):+ raise ValueError("Always fails")++ @property+ def _llm_type(self):+ return "always_failing"++ class AttemptTrackingMiddleware(AgentMiddleware):+ def __init__(self):+ super().__init__()+ self.attempts = []++ def retry_model_request(self, error, request, state, runtime, attempt):+ self.attempts.append(attempt)+ if attempt < 3: # noqa: PLR2004+ return request # Retry+ return None # Stop after 3 attempts++ model = AlwaysFailingModel()+ tracker = AttemptTrackingMiddleware()++ agent = create_agent(model=model, middleware=[tracker]).compile()++ with pytest.raises(ValueError, match="Always fails"):+ agent.invoke({"messages": [HumanMessage("Test")]})++ # Should have been called with attempts 1, 2, 3+ assert tracker.attempts == [1, 2, 3]+++def test_retry_model_request_no_retry() -> None:+ """Test that error is propagated when no middleware wants to retry."""++ class FailingModel(BaseChatModel):+ """Model that always fails."""++ def _generate(self, messages, **kwargs):+ raise ValueError("Model error")++ @property+ def _llm_type(self):+ return "failing"++ class NoRetryMiddleware(AgentMiddleware):+ def retry_model_request(self, error, request, state, runtime, attempt):+ # Always return None to not retry+ return None++ agent = create_agent(model=FailingModel(), middleware=[NoRetryMiddleware()]).compile()++ with pytest.raises(ValueError, match="Model error"):+ agent.invoke({"messages": [HumanMessage("Test")]})+++def test_model_fallback_middleware() -> None:+ """Test ModelFallbackMiddleware with fallback models only."""++ class FailingModel(BaseChatModel):+ """Model that always fails."""++ def _generate(self, messages, **kwargs):+ raise ValueError("Primary model failed")++ @property+ def _llm_type(self):+ return "failing"++ class SuccessModel(BaseChatModel):+ """Model that succeeds."""++ def _generate(self, messages, **kwargs):+ return ChatResult(+ generations=[ChatGeneration(message=AIMessage(content="Fallback success"))]+ )++ @property+ def _llm_type(self):+ return "success"++ primary = FailingModel()+ fallback = SuccessModel()++ # Only pass fallback models to middleware (not the primary)+ fallback_middleware = ModelFallbackMiddleware(fallback)++ agent = create_agent(model=primary, middleware=[fallback_middleware]).compile()++ result = agent.invoke({"messages": [HumanMessage("Test")]})++ # Should have succeeded with fallback model+ assert len(result["messages"]) == 2+ assert result["messages"][1].content == "Fallback success"+++def test_model_fallback_middleware_exhausted() -> None:+ """Test ModelFallbackMiddleware when all models fail."""++ class AlwaysFailingModel(BaseChatModel):+ """Model that always fails."""++ def __init__(self, name: str):+ super().__init__()+ self.name = name++ def _generate(self, messages, **kwargs):+ raise ValueError(f"{self.name} failed")++ @property+ def _llm_type(self):+ return self.name++ primary = AlwaysFailingModel("primary")+ fallback1 = AlwaysFailingModel("fallback1")+ fallback2 = AlwaysFailingModel("fallback2")++ # Primary fails (attempt 1), then fallback1 (attempt 2), then fallback2 (attempt 3)+ fallback_middleware = ModelFallbackMiddleware(fallback1, fallback2)++ agent = create_agent(model=primary, middleware=[fallback_middleware]).compile()++ # Should fail with the last fallback's error+ with pytest.raises(ValueError, match="fallback2 failed"):+ agent.invoke({"messages": [HumanMessage("Test")]})+++def test_model_fallback_middleware_initialization() -> None:+ """Test ModelFallbackMiddleware initialization."""++ # Test with no models - now a TypeError (missing required argument)+ with pytest.raises(TypeError):+ ModelFallbackMiddleware() # type: ignore[call-arg]++ # Test with one fallback model (valid)+ middleware = ModelFallbackMiddleware(FakeToolCallingModel())+ assert len(middleware.models) == 1++ # Test with multiple fallback models+ middleware = ModelFallbackMiddleware(FakeToolCallingModel(), FakeToolCallingModel())+ assert len(middleware.models) == 2+++def test_retry_model_request_max_attempts() -> None:+ """Test that retry stops after maximum attempts."""++ class AlwaysFailingModel(BaseChatModel):+ """Model that always fails."""++ def _generate(self, messages, **kwargs):+ raise ValueError("Always fails")++ @property+ def _llm_type(self):+ return "always_failing"++ class InfiniteRetryMiddleware(AgentMiddleware):+ """Middleware that always wants to retry (buggy behavior)."""++ def __init__(self):+ super().__init__()+ self.attempt_count = 0++ def retry_model_request(self, error, request, state, runtime, attempt):+ self.attempt_count = attempt+ return request # Always retry (infinite loop without limit)++ model = AlwaysFailingModel()+ middleware = InfiniteRetryMiddleware()++ agent = create_agent(model=model, middleware=[middleware]).compile()++ # Should fail with max attempts error, not infinite loop+ with pytest.raises(RuntimeError, match="Maximum retry attempts \\(100\\) exceeded"):+ agent.invoke({"messages": [HumanMessage("Test")]})++ # Should have attempted 100 times+ assert middleware.attempt_count == 100+++async def test_retry_model_request_async() -> None:+ """Test async retry_model_request hook."""+ call_count = {"value": 0}++ class AsyncFailingModel(BaseChatModel):+ """Model that fails on first async call, succeeds on second."""++ def _generate(self, messages, **kwargs):+ return ChatResult(generations=[ChatGeneration(message=AIMessage(content="sync"))])++ async def _agenerate(self, messages, **kwargs):+ call_count["value"] += 1+ if call_count["value"] == 1:+ raise ValueError("First async call fails")+ return ChatResult(+ generations=[ChatGeneration(message=AIMessage(content="Async retry success"))]+ )++ @property+ def _llm_type(self):+ return "async_failing"++ class AsyncRetryMiddleware(AgentMiddleware):+ def __init__(self):+ super().__init__()+ self.retry_count = 0++ async def aretry_model_request(self, error, request, state, runtime, attempt):+ self.retry_count += 1+ return request # Retry with same request++ failing_model = AsyncFailingModel()+ retry_middleware = AsyncRetryMiddleware()++ agent = create_agent(model=failing_model, middleware=[retry_middleware]).compile()++ result = await agent.ainvoke({"messages": [HumanMessage("Test")]})++ # Should have retried once+ assert retry_middleware.retry_count == 1+ # Should have succeeded on second attempt+ assert result["messages"][1].content == "Async retry success"++def test_create_agent_sync_invoke_with_only_async_middleware_raises_error() -> None:"""Test that sync invoke with only async middleware works via run_in_executor."""