Skip to content

The agent engineering platform.

feat(langchain_v1): Add retry_model_request middleware hook, add ModelFallbackMiddleware (#33275)

Nuno Camposa9aa3f232dce
diff --git a/.claude/settings.local.json b/.claude/settings.local.json
index 8bd1383607..630c4d8fd1 100644
--- a/.claude/settings.local.json
+++ b/.claude/settings.local.json
@@ -2,7 +2,7 @@
"permissions": {
"allow": [
"Bash(uv run:*)",
- "Bash(make:*)"
+ "Bash(make:*)",
"WebSearch",
"WebFetch(domain:ai.pydantic.dev)",
"WebFetch(domain:openai.github.io)",
@@ -12,4 +12,4 @@
"deny": [],
"ask": []
}
-}
\ No newline at end of file
+}
diff --git a/libs/langchain_v1/langchain/agents/middleware/__init__.py b/libs/langchain_v1/langchain/agents/middleware/__init__.py
index 823bfeaf35..5915b75164 100644
--- a/libs/langchain_v1/langchain/agents/middleware/__init__.py
+++ b/libs/langchain_v1/langchain/agents/middleware/__init__.py
@@ -1,6 +1,7 @@
"""Middleware plugins for agents."""
from .human_in_the_loop import HumanInTheLoopMiddleware
+from .model_fallback import ModelFallbackMiddleware
from .pii import PIIDetectionError, PIIMiddleware
from .planning import PlanningMiddleware
from .prompt_caching import AnthropicPromptCachingMiddleware
@@ -25,6 +26,7 @@ __all__ = [
"AnthropicPromptCachingMiddleware",
"HumanInTheLoopMiddleware",
"LLMToolSelectorMiddleware",
+ "ModelFallbackMiddleware",
"ModelRequest",
"PIIDetectionError",
"PIIMiddleware",
diff --git a/libs/langchain_v1/langchain/agents/middleware/model_fallback.py b/libs/langchain_v1/langchain/agents/middleware/model_fallback.py
new file mode 100644
index 0000000000..c3aac7b229
--- /dev/null
+++ b/libs/langchain_v1/langchain/agents/middleware/model_fallback.py
@@ -0,0 +1,94 @@
+"""Model fallback middleware for agents."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from langchain.agents.middleware.types import AgentMiddleware, AgentState, ModelRequest
+from langchain.chat_models import init_chat_model
+
+if TYPE_CHECKING:
+ from langchain_core.language_models.chat_models import BaseChatModel
+ from langgraph.runtime import Runtime
+
+
+class ModelFallbackMiddleware(AgentMiddleware):
+ """Middleware that provides automatic model fallback on errors.
+
+ This middleware attempts to retry failed model calls with alternative models
+ in sequence. When a model call fails, it tries the next model in the fallback
+ list until either a call succeeds or all models have been exhausted.
+
+ Example:
+ ```python
+ from langchain.agents.middleware.model_fallback import ModelFallbackMiddleware
+ from langchain.agents import create_agent
+
+ # Create middleware with fallback models (not including primary)
+ fallback = ModelFallbackMiddleware(
+ "openai:gpt-4o-mini", # First fallback
+ "anthropic:claude-3-5-sonnet-20241022", # Second fallback
+ )
+
+ agent = create_agent(
+ model="openai:gpt-4o", # Primary model
+ middleware=[fallback],
+ )
+
+ # If gpt-4o fails, automatically tries gpt-4o-mini, then claude
+ result = await agent.invoke({"messages": [HumanMessage("Hello")]})
+ ```
+ """
+
+ def __init__(
+ self,
+ first_model: str | BaseChatModel,
+ *additional_models: str | BaseChatModel,
+ ) -> None:
+ """Initialize the model fallback middleware.
+
+ Args:
+ first_model: The first fallback model to try when the primary model fails.
+ Can be a model name string or BaseChatModel instance.
+ *additional_models: Additional fallback models to try, in order.
+ Can be model name strings or BaseChatModel instances.
+ """
+ super().__init__()
+
+ # Initialize all fallback models
+ all_models = (first_model, *additional_models)
+ self.models: list[BaseChatModel] = []
+ for model in all_models:
+ if isinstance(model, str):
+ self.models.append(init_chat_model(model))
+ else:
+ self.models.append(model)
+
+ def retry_model_request(
+ self,
+ error: Exception, # noqa: ARG002
+ request: ModelRequest,
+ state: AgentState, # noqa: ARG002
+ runtime: Runtime, # noqa: ARG002
+ attempt: int,
+ ) -> ModelRequest | None:
+ """Retry with the next fallback model.
+
+ Args:
+ error: The exception that occurred during model invocation.
+ request: The original model request that failed.
+ state: The current agent state.
+ runtime: The langgraph runtime.
+ attempt: The current attempt number (1-indexed).
+
+ Returns:
+ ModelRequest with the next fallback model, or None if all models exhausted.
+ """
+ # attempt 1 = primary model failed, try models[0] (first fallback)
+ fallback_index = attempt - 1
+ # All fallback models exhausted
+ if fallback_index >= len(self.models):
+ return None
+ # Try next fallback model
+ request.model = self.models[fallback_index]
+ return request
diff --git a/libs/langchain_v1/langchain/agents/middleware/types.py b/libs/langchain_v1/langchain/agents/middleware/types.py
index db5174839b..6aea90ab9f 100644
--- a/libs/langchain_v1/langchain/agents/middleware/types.py
+++ b/libs/langchain_v1/langchain/agents/middleware/types.py
@@ -167,6 +167,54 @@ class AgentMiddleware(Generic[StateT, ContextT]):
) -> dict[str, Any] | None:
"""Async logic to run after the model is called."""
+ def retry_model_request(
+ self,
+ error: Exception, # noqa: ARG002
+ request: ModelRequest, # noqa: ARG002
+ state: StateT, # noqa: ARG002
+ runtime: Runtime[ContextT], # noqa: ARG002
+ attempt: int, # noqa: ARG002
+ ) -> ModelRequest | None:
+ """Logic to handle model invocation errors and optionally retry.
+
+ Args:
+ error: The exception that occurred during model invocation.
+ request: The original model request that failed.
+ state: The current agent state.
+ runtime: The langgraph runtime.
+ attempt: The current attempt number (1-indexed).
+
+ Returns:
+ ModelRequest: Modified request to retry with.
+ None: Propagate the error (re-raise).
+ """
+ return None
+
+ async def aretry_model_request(
+ self,
+ error: Exception,
+ request: ModelRequest,
+ state: StateT,
+ runtime: Runtime[ContextT],
+ attempt: int,
+ ) -> ModelRequest | None:
+ """Async logic to handle model invocation errors and optionally retry.
+
+ Args:
+ error: The exception that occurred during model invocation.
+ request: The original model request that failed.
+ state: The current agent state.
+ runtime: The langgraph runtime.
+ attempt: The current attempt number (1-indexed).
+
+ Returns:
+ ModelRequest: Modified request to retry with.
+ None: Propagate the error (re-raise).
+ """
+ return await run_in_executor(
+ None, self.retry_model_request, error, request, state, runtime, attempt
+ )
+
class _CallableWithStateAndRuntime(Protocol[StateT_contra, ContextT]):
"""Callable with AgentState and Runtime as arguments."""
diff --git a/libs/langchain_v1/langchain/agents/middleware_agent.py b/libs/langchain_v1/langchain/agents/middleware_agent.py
index c2e5159932..b4f6370446 100644
--- a/libs/langchain_v1/langchain/agents/middleware_agent.py
+++ b/libs/langchain_v1/langchain/agents/middleware_agent.py
@@ -278,6 +278,12 @@ def create_agent( # noqa: PLR0915
if m.__class__.after_model is not AgentMiddleware.after_model
or m.__class__.aafter_model is not AgentMiddleware.aafter_model
]
+ middleware_w_retry = [
+ m
+ for m in middleware
+ if m.__class__.retry_model_request is not AgentMiddleware.retry_model_request
+ or m.__class__.aretry_model_request is not AgentMiddleware.aretry_model_request
+ ]
state_schemas = {m.state_schema for m in middleware}
state_schemas.add(AgentState)
@@ -526,18 +532,47 @@ def create_agent( # noqa: PLR0915
)
raise TypeError(msg)
- # Get the bound model (with auto-detection if needed)
- model_, effective_response_format = _get_bound_model(request)
- messages = request.messages
- if request.system_prompt:
- messages = [SystemMessage(request.system_prompt), *messages]
+ # Retry loop for model invocation with error handling
+ # Hard limit of 100 attempts to prevent infinite loops from buggy middleware
+ max_attempts = 100
+ for attempt in range(1, max_attempts + 1):
+ try:
+ # Get the bound model (with auto-detection if needed)
+ model_, effective_response_format = _get_bound_model(request)
+ messages = request.messages
+ if request.system_prompt:
+ messages = [SystemMessage(request.system_prompt), *messages]
+
+ output = model_.invoke(messages)
+ return {
+ "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,
+ "run_model_call_count": state.get("run_model_call_count", 0) + 1,
+ **_handle_model_output(output, effective_response_format),
+ }
+ except Exception as error:
+ # Try retry_model_request on each middleware
+ for m in middleware_w_retry:
+ if m.__class__.retry_model_request is not AgentMiddleware.retry_model_request:
+ if retry_request := m.retry_model_request(
+ error, request, state, runtime, attempt
+ ):
+ # Break on first middleware that wants to retry
+ request = retry_request
+ break
+ else:
+ msg = (
+ f"No synchronous function provided for "
+ f'{m.__class__.__name__}.aretry_model_request".'
+ "\nEither initialize with a synchronous function or invoke"
+ " via the async API (ainvoke, astream, etc.)"
+ )
+ raise TypeError(msg)
+ else:
+ raise
- output = model_.invoke(messages)
- return {
- "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,
- "run_model_call_count": state.get("run_model_call_count", 0) + 1,
- **_handle_model_output(output, effective_response_format),
- }
+ # If we exit the loop, max attempts exceeded
+ msg = f"Maximum retry attempts ({max_attempts}) exceeded"
+ raise RuntimeError(msg)
async def amodel_request(state: AgentState, runtime: Runtime[ContextT]) -> dict[str, Any]:
"""Async model request handler with sequential middleware processing."""
@@ -554,18 +589,39 @@ def create_agent( # noqa: PLR0915
for m in middleware_w_modify_model_request:
await m.amodify_model_request(request, state, runtime)
- # Get the bound model (with auto-detection if needed)
- model_, effective_response_format = _get_bound_model(request)
- messages = request.messages
- if request.system_prompt:
- messages = [SystemMessage(request.system_prompt), *messages]
-
- output = await model_.ainvoke(messages)
- return {
- "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,
- "run_model_call_count": state.get("run_model_call_count", 0) + 1,
- **_handle_model_output(output, effective_response_format),
- }
+ # Retry loop for model invocation with error handling
+ # Hard limit of 100 attempts to prevent infinite loops from buggy middleware
+ max_attempts = 100
+ for attempt in range(1, max_attempts + 1):
+ try:
+ # Get the bound model (with auto-detection if needed)
+ model_, effective_response_format = _get_bound_model(request)
+ messages = request.messages
+ if request.system_prompt:
+ messages = [SystemMessage(request.system_prompt), *messages]
+
+ output = await model_.ainvoke(messages)
+ return {
+ "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,
+ "run_model_call_count": state.get("run_model_call_count", 0) + 1,
+ **_handle_model_output(output, effective_response_format),
+ }
+ except Exception as error:
+ # Try retry_model_request on each middleware
+ for m in middleware_w_retry:
+ if retry_request := await m.aretry_model_request(
+ error, request, state, runtime, attempt
+ ):
+ # Break on first middleware that wants to retry
+ request = retry_request
+ break
+ else:
+ # If no middleware wants to retry, re-raise the error
+ raise
+
+ # If we exit the loop, max attempts exceeded
+ msg = f"Maximum retry attempts ({max_attempts}) exceeded"
+ raise RuntimeError(msg)
# Use sync or async based on model capabilities
from langgraph._internal._runnable import RunnableCallable
diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py b/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py
index 2716d5c473..ee34a47d7d 100644
--- a/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py
+++ b/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py
@@ -40,6 +40,7 @@ from langchain.agents.middleware.call_tracking import (
ModelCallLimitMiddleware,
ModelCallLimitExceededError,
)
+from langchain.agents.middleware.model_fallback import ModelFallbackMiddleware
from langchain.agents.middleware.prompt_caching import AnthropicPromptCachingMiddleware
from langchain.agents.middleware.summarization import SummarizationMiddleware
from langchain.agents.middleware.types import (
@@ -2109,6 +2110,278 @@ async def test_create_agent_mixed_sync_async_middleware() -> None:
]
+# Tests for retry_model_request hook
+def test_retry_model_request_hook() -> None:
+ """Test that retry_model_request hook is called on model errors."""
+ call_count = {"value": 0}
+
+ class FailingModel(BaseChatModel):
+ """Model that fails on first call, succeeds on second."""
+
+ def _generate(self, messages, **kwargs):
+ call_count["value"] += 1
+ if call_count["value"] == 1:
+ raise ValueError("First call fails")
+ return ChatResult(
+ generations=[ChatGeneration(message=AIMessage(content="Success on retry"))]
+ )
+
+ @property
+ def _llm_type(self):
+ return "failing"
+
+ class RetryMiddleware(AgentMiddleware):
+ def __init__(self):
+ super().__init__()
+ self.retry_count = 0
+
+ def retry_model_request(self, error, request, state, runtime, attempt):
+ self.retry_count += 1
+ # Return the same request to retry
+ return request
+
+ failing_model = FailingModel()
+ retry_middleware = RetryMiddleware()
+
+ agent = create_agent(model=failing_model, middleware=[retry_middleware]).compile()
+
+ result = agent.invoke({"messages": [HumanMessage("Test")]})
+
+ # Should have retried once
+ assert retry_middleware.retry_count == 1
+ # Should have succeeded on second attempt
+ assert len(result["messages"]) == 2
+ assert result["messages"][1].content == "Success on retry"
+
+
+def test_retry_model_request_attempt_number() -> None:
+ """Test that attempt number is correctly passed to retry_model_request."""
+
+ class AlwaysFailingModel(BaseChatModel):
+ """Model that always fails."""
+
+ def _generate(self, messages, **kwargs):
+ raise ValueError("Always fails")
+
+ @property
+ def _llm_type(self):
+ return "always_failing"
+
+ class AttemptTrackingMiddleware(AgentMiddleware):
+ def __init__(self):
+ super().__init__()
+ self.attempts = []
+
+ def retry_model_request(self, error, request, state, runtime, attempt):
+ self.attempts.append(attempt)
+ if attempt < 3: # noqa: PLR2004
+ return request # Retry
+ return None # Stop after 3 attempts
+
+ model = AlwaysFailingModel()
+ tracker = AttemptTrackingMiddleware()
+
+ agent = create_agent(model=model, middleware=[tracker]).compile()
+
+ with pytest.raises(ValueError, match="Always fails"):
+ agent.invoke({"messages": [HumanMessage("Test")]})
+
+ # Should have been called with attempts 1, 2, 3
+ assert tracker.attempts == [1, 2, 3]
+
+
+def test_retry_model_request_no_retry() -> None:
+ """Test that error is propagated when no middleware wants to retry."""
+
+ class FailingModel(BaseChatModel):
+ """Model that always fails."""
+
+ def _generate(self, messages, **kwargs):
+ raise ValueError("Model error")
+
+ @property
+ def _llm_type(self):
+ return "failing"
+
+ class NoRetryMiddleware(AgentMiddleware):
+ def retry_model_request(self, error, request, state, runtime, attempt):
+ # Always return None to not retry
+ return None
+
+ agent = create_agent(model=FailingModel(), middleware=[NoRetryMiddleware()]).compile()
+
+ with pytest.raises(ValueError, match="Model error"):
+ agent.invoke({"messages": [HumanMessage("Test")]})
+
+
+def test_model_fallback_middleware() -> None:
+ """Test ModelFallbackMiddleware with fallback models only."""
+
+ class FailingModel(BaseChatModel):
+ """Model that always fails."""
+
+ def _generate(self, messages, **kwargs):
+ raise ValueError("Primary model failed")
+
+ @property
+ def _llm_type(self):
+ return "failing"
+
+ class SuccessModel(BaseChatModel):
+ """Model that succeeds."""
+
+ def _generate(self, messages, **kwargs):
+ return ChatResult(
+ generations=[ChatGeneration(message=AIMessage(content="Fallback success"))]
+ )
+
+ @property
+ def _llm_type(self):
+ return "success"
+
+ primary = FailingModel()
+ fallback = SuccessModel()
+
+ # Only pass fallback models to middleware (not the primary)
+ fallback_middleware = ModelFallbackMiddleware(fallback)
+
+ agent = create_agent(model=primary, middleware=[fallback_middleware]).compile()
+
+ result = agent.invoke({"messages": [HumanMessage("Test")]})
+
+ # Should have succeeded with fallback model
+ assert len(result["messages"]) == 2
+ assert result["messages"][1].content == "Fallback success"
+
+
+def test_model_fallback_middleware_exhausted() -> None:
+ """Test ModelFallbackMiddleware when all models fail."""
+
+ class AlwaysFailingModel(BaseChatModel):
+ """Model that always fails."""
+
+ def __init__(self, name: str):
+ super().__init__()
+ self.name = name
+
+ def _generate(self, messages, **kwargs):
+ raise ValueError(f"{self.name} failed")
+
+ @property
+ def _llm_type(self):
+ return self.name
+
+ primary = AlwaysFailingModel("primary")
+ fallback1 = AlwaysFailingModel("fallback1")
+ fallback2 = AlwaysFailingModel("fallback2")
+
+ # Primary fails (attempt 1), then fallback1 (attempt 2), then fallback2 (attempt 3)
+ fallback_middleware = ModelFallbackMiddleware(fallback1, fallback2)
+
+ agent = create_agent(model=primary, middleware=[fallback_middleware]).compile()
+
+ # Should fail with the last fallback's error
+ with pytest.raises(ValueError, match="fallback2 failed"):
+ agent.invoke({"messages": [HumanMessage("Test")]})
+
+
+def test_model_fallback_middleware_initialization() -> None:
+ """Test ModelFallbackMiddleware initialization."""
+
+ # Test with no models - now a TypeError (missing required argument)
+ with pytest.raises(TypeError):
+ ModelFallbackMiddleware() # type: ignore[call-arg]
+
+ # Test with one fallback model (valid)
+ middleware = ModelFallbackMiddleware(FakeToolCallingModel())
+ assert len(middleware.models) == 1
+
+ # Test with multiple fallback models
+ middleware = ModelFallbackMiddleware(FakeToolCallingModel(), FakeToolCallingModel())
+ assert len(middleware.models) == 2
+
+
+def test_retry_model_request_max_attempts() -> None:
+ """Test that retry stops after maximum attempts."""
+
+ class AlwaysFailingModel(BaseChatModel):
+ """Model that always fails."""
+
+ def _generate(self, messages, **kwargs):
+ raise ValueError("Always fails")
+
+ @property
+ def _llm_type(self):
+ return "always_failing"
+
+ class InfiniteRetryMiddleware(AgentMiddleware):
+ """Middleware that always wants to retry (buggy behavior)."""
+
+ def __init__(self):
+ super().__init__()
+ self.attempt_count = 0
+
+ def retry_model_request(self, error, request, state, runtime, attempt):
+ self.attempt_count = attempt
+ return request # Always retry (infinite loop without limit)
+
+ model = AlwaysFailingModel()
+ middleware = InfiniteRetryMiddleware()
+
+ agent = create_agent(model=model, middleware=[middleware]).compile()
+
+ # Should fail with max attempts error, not infinite loop
+ with pytest.raises(RuntimeError, match="Maximum retry attempts \\(100\\) exceeded"):
+ agent.invoke({"messages": [HumanMessage("Test")]})
+
+ # Should have attempted 100 times
+ assert middleware.attempt_count == 100
+
+
+async def test_retry_model_request_async() -> None:
+ """Test async retry_model_request hook."""
+ call_count = {"value": 0}
+
+ class AsyncFailingModel(BaseChatModel):
+ """Model that fails on first async call, succeeds on second."""
+
+ def _generate(self, messages, **kwargs):
+ return ChatResult(generations=[ChatGeneration(message=AIMessage(content="sync"))])
+
+ async def _agenerate(self, messages, **kwargs):
+ call_count["value"] += 1
+ if call_count["value"] == 1:
+ raise ValueError("First async call fails")
+ return ChatResult(
+ generations=[ChatGeneration(message=AIMessage(content="Async retry success"))]
+ )
+
+ @property
+ def _llm_type(self):
+ return "async_failing"
+
+ class AsyncRetryMiddleware(AgentMiddleware):
+ def __init__(self):
+ super().__init__()
+ self.retry_count = 0
+
+ async def aretry_model_request(self, error, request, state, runtime, attempt):
+ self.retry_count += 1
+ return request # Retry with same request
+
+ failing_model = AsyncFailingModel()
+ retry_middleware = AsyncRetryMiddleware()
+
+ agent = create_agent(model=failing_model, middleware=[retry_middleware]).compile()
+
+ result = await agent.ainvoke({"messages": [HumanMessage("Test")]})
+
+ # Should have retried once
+ assert retry_middleware.retry_count == 1
+ # Should have succeeded on second attempt
+ assert result["messages"][1].content == "Async retry success"
+
+
def test_create_agent_sync_invoke_with_only_async_middleware_raises_error() -> None:
"""Test that sync invoke with only async middleware works via run_in_executor."""