The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(langchain): model call limits (#33178)
Sydney Runkle3d2f13a2f174
diff --git a/libs/langchain_v1/langchain/agents/middleware/call_tracking.py b/libs/langchain_v1/langchain/agents/middleware/call_tracking.pynew file mode 100644index 0000000000..b8eb425adf--- /dev/null+++ b/libs/langchain_v1/langchain/agents/middleware/call_tracking.py@@ -0,0 +1,177 @@+"""Call tracking middleware for agents."""++from __future__ import annotations++from typing import TYPE_CHECKING, Any, Literal++from langchain_core.messages import AIMessage++from langchain.agents.middleware.types import AgentMiddleware, AgentState, hook_config++if TYPE_CHECKING:+ from langgraph.runtime import Runtime+++def _build_limit_exceeded_message(+ thread_count: int,+ run_count: int,+ thread_limit: int | None,+ run_limit: int | None,+) -> str:+ """Build a message indicating which limits were exceeded.++ Args:+ thread_count: Current thread model call count.+ run_count: Current run model call count.+ thread_limit: Thread model call limit (if set).+ run_limit: Run model call limit (if set).++ Returns:+ A formatted message describing which limits were exceeded.+ """+ exceeded_limits = []+ if thread_limit is not None and thread_count >= thread_limit:+ exceeded_limits.append(f"thread limit ({thread_count}/{thread_limit})")+ if run_limit is not None and run_count >= run_limit:+ exceeded_limits.append(f"run limit ({run_count}/{run_limit})")++ return f"Model call limits exceeded: {', '.join(exceeded_limits)}"+++class ModelCallLimitExceededError(Exception):+ """Exception raised when model call limits are exceeded.++ This exception is raised when the configured exit behavior is 'error'+ and either the thread or run model call limit has been exceeded.+ """++ def __init__(+ self,+ thread_count: int,+ run_count: int,+ thread_limit: int | None,+ run_limit: int | None,+ ) -> None:+ """Initialize the exception with call count information.++ Args:+ thread_count: Current thread model call count.+ run_count: Current run model call count.+ thread_limit: Thread model call limit (if set).+ run_limit: Run model call limit (if set).+ """+ self.thread_count = thread_count+ self.run_count = run_count+ self.thread_limit = thread_limit+ self.run_limit = run_limit++ msg = _build_limit_exceeded_message(thread_count, run_count, thread_limit, run_limit)+ super().__init__(msg)+++class ModelCallLimitMiddleware(AgentMiddleware):+ """Middleware that tracks model call counts and enforces limits.++ This middleware monitors the number of model calls made during agent execution+ and can terminate the agent when specified limits are reached. It supports+ both thread-level and run-level call counting with configurable exit behaviors.++ Thread-level: The middleware tracks the number of model calls and persists+ call count across multiple runs (invocations) of the agent.++ Run-level: The middleware tracks the number of model calls made during a single+ run (invocation) of the agent.++ Example:+ ```python+ from langchain.agents.middleware.call_tracking import ModelCallLimitMiddleware+ from langchain.agents import create_agent++ # Create middleware with limits+ call_tracker = ModelCallLimitMiddleware(thread_limit=10, run_limit=5, exit_behavior="end")++ agent = create_agent("openai:gpt-4o", middleware=[call_tracker])++ # Agent will automatically jump to end when limits are exceeded+ result = await agent.invoke({"messages": [HumanMessage("Help me with a task")]})+ ```+ """++ def __init__(+ self,+ *,+ thread_limit: int | None = None,+ run_limit: int | None = None,+ exit_behavior: Literal["end", "error"] = "end",+ ) -> None:+ """Initialize the call tracking middleware.++ Args:+ thread_limit: Maximum number of model calls allowed per thread.+ None means no limit. Defaults to None.+ run_limit: Maximum number of model calls allowed per run.+ None means no limit. Defaults to None.+ exit_behavior: What to do when limits are exceeded.+ - "end": Jump to the end of the agent execution and+ inject an artificial AI message indicating that the limit was exceeded.+ - "error": Raise a ModelCallLimitExceededError+ Defaults to "end".++ Raises:+ ValueError: If both limits are None or if exit_behavior is invalid.+ """+ super().__init__()++ if thread_limit is None and run_limit is None:+ msg = "At least one limit must be specified (thread_limit or run_limit)"+ raise ValueError(msg)++ if exit_behavior not in ("end", "error"):+ msg = f"Invalid exit_behavior: {exit_behavior}. Must be 'end' or 'error'"+ raise ValueError(msg)++ self.thread_limit = thread_limit+ self.run_limit = run_limit+ self.exit_behavior = exit_behavior++ @hook_config(can_jump_to=["end"])+ def before_model(self, state: AgentState, runtime: Runtime) -> dict[str, Any] | None: # noqa: ARG002+ """Check model call limits before making a model call.++ Args:+ state: The current agent state containing call counts.+ runtime: The langgraph runtime.++ Returns:+ If limits are exceeded and exit_behavior is "end", returns+ a Command to jump to the end with a limit exceeded message. Otherwise returns None.++ Raises:+ ModelCallLimitExceededError: If limits are exceeded and exit_behavior+ is "error".+ """+ thread_count = state.get("thread_model_call_count", 0)+ run_count = state.get("run_model_call_count", 0)++ # Check if any limits will be exceeded after the next call+ thread_limit_exceeded = self.thread_limit is not None and thread_count >= self.thread_limit+ run_limit_exceeded = self.run_limit is not None and run_count >= self.run_limit++ if thread_limit_exceeded or run_limit_exceeded:+ if self.exit_behavior == "error":+ raise ModelCallLimitExceededError(+ thread_count=thread_count,+ run_count=run_count,+ thread_limit=self.thread_limit,+ run_limit=self.run_limit,+ )+ if self.exit_behavior == "end":+ # Create a message indicating the limit was exceeded+ limit_message = _build_limit_exceeded_message(+ thread_count, run_count, self.thread_limit, self.run_limit+ )+ limit_ai_message = AIMessage(content=limit_message)++ return {"jump_to": "end", "messages": [limit_ai_message]}++ return Nonediff --git a/libs/langchain_v1/langchain/agents/middleware/types.py b/libs/langchain_v1/langchain/agents/middleware/types.pyindex 48b38c6d2b..d0328e1191 100644--- a/libs/langchain_v1/langchain/agents/middleware/types.py+++ b/libs/langchain_v1/langchain/agents/middleware/types.py@@ -24,6 +24,7 @@ if TYPE_CHECKING:# needed as top level import for pydantic schema generation on AgentStatefrom langchain_core.messages import AnyMessage # noqa: TC002from langgraph.channels.ephemeral_value import EphemeralValue+from langgraph.channels.untracked_value import UntrackedValuefrom langgraph.graph.message import add_messagesfrom langgraph.runtime import Runtimefrom langgraph.typing import ContextT@@ -93,6 +94,8 @@ class AgentState(TypedDict, Generic[ResponseT]):messages: Required[Annotated[list[AnyMessage], add_messages]]jump_to: NotRequired[Annotated[JumpTo | None, EphemeralValue, PrivateStateAttr]]structured_response: NotRequired[ResponseT]+ thread_model_call_count: NotRequired[Annotated[int, PrivateStateAttr]]+ run_model_call_count: NotRequired[Annotated[int, UntrackedValue, PrivateStateAttr]]class PublicAgentState(TypedDict, Generic[ResponseT]):diff --git a/libs/langchain_v1/langchain/agents/middleware_agent.py b/libs/langchain_v1/langchain/agents/middleware_agent.pyindex ce752aefb6..17c2825d0e 100644--- a/libs/langchain_v1/langchain/agents/middleware_agent.py+++ b/libs/langchain_v1/langchain/agents/middleware_agent.py@@ -438,7 +438,11 @@ def create_agent( # noqa: PLR0915messages = [SystemMessage(request.system_prompt), *messages]output = model_.invoke(messages)- return _handle_model_output(output)+ return {+ "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,+ "run_model_call_count": state.get("run_model_call_count", 0) + 1,+ **_handle_model_output(output),+ }async def amodel_request(state: AgentState, runtime: Runtime[ContextT]) -> dict[str, Any]:"""Async model request handler with sequential middleware processing."""@@ -462,7 +466,11 @@ def create_agent( # noqa: PLR0915messages = [SystemMessage(request.system_prompt), *messages]output = await model_.ainvoke(messages)- return _handle_model_output(output)+ return {+ "thread_model_call_count": state.get("thread_model_call_count", 0) + 1,+ "run_model_call_count": state.get("run_model_call_count", 0) + 1,+ **_handle_model_output(output),+ }# Use sync or async based on model capabilitiesfrom langgraph._internal._runnable import RunnableCallablediff --git a/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py b/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.pyindex 451849b977..67aa085e54 100644--- a/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py+++ b/libs/langchain_v1/tests/unit_tests/agents/test_middleware_agent.py@@ -36,6 +36,10 @@ from langchain.agents.middleware.planning import (write_todos,WRITE_TODOS_TOOL_DESCRIPTION,)+from langchain.agents.middleware.call_tracking import (+ ModelCallLimitMiddleware,+ ModelCallLimitExceededError,+)from langchain.agents.middleware.prompt_caching import AnthropicPromptCachingMiddlewarefrom langchain.agents.middleware.summarization import SummarizationMiddlewarefrom langchain.agents.middleware.types import (@@ -1665,6 +1669,217 @@ def test_planning_middleware_custom_system_prompt() -> None:assert "call the write_todos tool" in result["messages"][1].content+@tool+def simple_tool(input: str) -> str:+ """A simple tool"""+ return input+++def test_middleware_unit_functionality():+ """Test that the middleware works as expected in isolation."""+ # Test with end behavior+ middleware = ModelCallLimitMiddleware(thread_limit=2, run_limit=1)++ # Mock runtime (not used in current implementation)+ runtime = None++ # Test when limits are not exceeded+ state = {"thread_model_call_count": 0, "run_model_call_count": 0}+ result = middleware.before_model(state, runtime)+ assert result is None++ # Test when thread limit is exceeded+ state = {"thread_model_call_count": 2, "run_model_call_count": 0}+ result = middleware.before_model(state, runtime)+ assert result is not None+ assert result["jump_to"] == "end"+ assert "messages" in result+ assert len(result["messages"]) == 1+ assert "thread limit (2/2)" in result["messages"][0].content++ # Test when run limit is exceeded+ state = {"thread_model_call_count": 1, "run_model_call_count": 1}+ result = middleware.before_model(state, runtime)+ assert result is not None+ assert result["jump_to"] == "end"+ assert "messages" in result+ assert len(result["messages"]) == 1+ assert "run limit (1/1)" in result["messages"][0].content++ # Test with error behavior+ middleware_exception = ModelCallLimitMiddleware(+ thread_limit=2, run_limit=1, exit_behavior="error"+ )++ # Test exception when thread limit exceeded+ state = {"thread_model_call_count": 2, "run_model_call_count": 0}+ with pytest.raises(ModelCallLimitExceededError) as exc_info:+ middleware_exception.before_model(state, runtime)++ assert "thread limit (2/2)" in str(exc_info.value)++ # Test exception when run limit exceeded+ state = {"thread_model_call_count": 1, "run_model_call_count": 1}+ with pytest.raises(ModelCallLimitExceededError) as exc_info:+ middleware_exception.before_model(state, runtime)++ assert "run limit (1/1)" in str(exc_info.value)+++def test_thread_limit_with_create_agent():+ """Test that thread limits work correctly with create_agent."""+ model = FakeToolCallingModel()++ # Set thread limit to 1 (should be exceeded after 1 call)+ agent = create_agent(+ model=model,+ tools=[simple_tool],+ middleware=[ModelCallLimitMiddleware(thread_limit=1)],+ ).compile(checkpointer=InMemorySaver())++ # First invocation should work - 1 model call, within thread limit+ result = agent.invoke(+ {"messages": [HumanMessage("Hello")]}, {"configurable": {"thread_id": "thread1"}}+ )++ # Should complete successfully with 1 model call+ assert "messages" in result+ assert len(result["messages"]) == 2 # Human + AI messages++ # Second invocation in same thread should hit thread limit+ # The agent should jump to end after detecting the limit+ result2 = agent.invoke(+ {"messages": [HumanMessage("Hello again")]}, {"configurable": {"thread_id": "thread1"}}+ )++ assert "messages" in result2+ # The agent should have detected the limit and jumped to end with a limit exceeded message+ # So we should have: previous messages + new human message + limit exceeded AI message+ assert len(result2["messages"]) == 4 # Previous Human + AI + New Human + Limit AI+ assert isinstance(result2["messages"][0], HumanMessage) # First human+ assert isinstance(result2["messages"][1], AIMessage) # First AI response+ assert isinstance(result2["messages"][2], HumanMessage) # Second human+ assert isinstance(result2["messages"][3], AIMessage) # Limit exceeded message+ assert "thread limit" in result2["messages"][3].content+++def test_run_limit_with_create_agent():+ """Test that run limits work correctly with create_agent."""+ # Create a model that will make 2 calls+ model = FakeToolCallingModel(+ tool_calls=[+ [{"name": "simple_tool", "args": {"input": "test"}, "id": "1"}],+ [], # No tool calls on second call+ ]+ )++ # Set run limit to 1 (should be exceeded after 1 call)+ agent = create_agent(+ model=model,+ tools=[simple_tool],+ middleware=[ModelCallLimitMiddleware(run_limit=1)],+ ).compile(checkpointer=InMemorySaver())++ # This should hit the run limit after the first model call+ result = agent.invoke(+ {"messages": [HumanMessage("Hello")]}, {"configurable": {"thread_id": "thread1"}}+ )++ assert "messages" in result+ # The agent should have made 1 model call then jumped to end with limit exceeded message+ # So we should have: Human + AI + Tool + Limit exceeded AI message+ assert len(result["messages"]) == 4 # Human + AI + Tool + Limit AI+ assert isinstance(result["messages"][0], HumanMessage)+ assert isinstance(result["messages"][1], AIMessage)+ assert isinstance(result["messages"][2], ToolMessage)+ assert isinstance(result["messages"][3], AIMessage) # Limit exceeded message+ assert "run limit" in result["messages"][3].content+++def test_middleware_initialization_validation():+ """Test that middleware initialization validates parameters correctly."""+ # Test that at least one limit must be specified+ with pytest.raises(ValueError, match="At least one limit must be specified"):+ ModelCallLimitMiddleware()++ # Test invalid exit behavior+ with pytest.raises(ValueError, match="Invalid exit_behavior"):+ ModelCallLimitMiddleware(thread_limit=5, exit_behavior="invalid")++ # Test valid initialization+ middleware = ModelCallLimitMiddleware(thread_limit=5, run_limit=3)+ assert middleware.thread_limit == 5+ assert middleware.run_limit == 3+ assert middleware.exit_behavior == "end"++ # Test with only thread limit+ middleware = ModelCallLimitMiddleware(thread_limit=5)+ assert middleware.thread_limit == 5+ assert middleware.run_limit is None++ # Test with only run limit+ middleware = ModelCallLimitMiddleware(run_limit=3)+ assert middleware.thread_limit is None+ assert middleware.run_limit == 3+++def test_exception_error_message():+ """Test that the exception provides clear error messages."""+ middleware = ModelCallLimitMiddleware(thread_limit=2, run_limit=1, exit_behavior="error")++ # Test thread limit exceeded+ state = {"thread_model_call_count": 2, "run_model_call_count": 0}+ with pytest.raises(ModelCallLimitExceededError) as exc_info:+ middleware.before_model(state, None)++ error_msg = str(exc_info.value)+ assert "Model call limits exceeded" in error_msg+ assert "thread limit (2/2)" in error_msg++ # Test run limit exceeded+ state = {"thread_model_call_count": 0, "run_model_call_count": 1}+ with pytest.raises(ModelCallLimitExceededError) as exc_info:+ middleware.before_model(state, None)++ error_msg = str(exc_info.value)+ assert "Model call limits exceeded" in error_msg+ assert "run limit (1/1)" in error_msg++ # Test both limits exceeded+ state = {"thread_model_call_count": 2, "run_model_call_count": 1}+ with pytest.raises(ModelCallLimitExceededError) as exc_info:+ middleware.before_model(state, None)++ error_msg = str(exc_info.value)+ assert "Model call limits exceeded" in error_msg+ assert "thread limit (2/2)" in error_msg+ assert "run limit (1/1)" in error_msg+++def test_run_limit_resets_between_invocations() -> None:+ """Test that run_model_call_count resets between invocations, but thread_model_call_count accumulates."""++ # First: No tool calls per invocation, so model does not increment call counts internally+ middleware = ModelCallLimitMiddleware(thread_limit=3, run_limit=1, exit_behavior="error")+ model = FakeToolCallingModel(+ tool_calls=[[], [], [], []]+ ) # No tool calls, so only model call per run++ agent = create_agent(model=model, middleware=[middleware])+ agent = agent.compile(checkpointer=InMemorySaver())++ thread_config = {"configurable": {"thread_id": "test_thread"}}+ agent.invoke({"messages": [HumanMessage("Hello")]}, thread_config)+ agent.invoke({"messages": [HumanMessage("Hello again")]}, thread_config)+ agent.invoke({"messages": [HumanMessage("Hello third")]}, thread_config)++ # Fourth run: should raise, thread_model_call_count == 3 (limit)+ with pytest.raises(ModelCallLimitExceededError) as exc_info:+ agent.invoke({"messages": [HumanMessage("Hello fourth")]}, thread_config)+ error_msg = str(exc_info.value)+ assert "thread limit (3/3)" in error_msg++# Async Middleware Testsasync def test_create_agent_async_invoke() -> None:"""Test async invoke with async middleware hooks."""