The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
test(langchain): types in `test_tool_call_limit` and `test_model_retry` (#34629)
Christophe Bornetd181a59ebef9
diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_retry.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_retry.pyindex 6f0eb19a23..f38abe3f21 100644--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_retry.py+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_retry.py@@ -1,6 +1,7 @@"""Tests for ModelRetryMiddleware functionality."""import time+from collections.abc import Callablefrom typing import Anyimport pytest@@ -13,7 +14,12 @@ from pydantic import Fieldfrom langchain.agents.factory import create_agentfrom langchain.agents.middleware._retry import calculate_delayfrom langchain.agents.middleware.model_retry import ModelRetryMiddleware-from langchain.agents.middleware.types import wrap_model_call+from langchain.agents.middleware.types import (+ ModelCallResult,+ ModelRequest,+ ModelResponse,+ wrap_model_call,+)from tests.unit_tests.agents.model import FakeToolCallingModel@@ -659,7 +665,10 @@ def test_model_retry_multiple_middleware_composition() -> None:# Custom middleware that logs calls@wrap_model_call- def logging_middleware(request, handler):+ def logging_middleware(+ request: ModelRequest,+ handler: Callable[[ModelRequest], ModelResponse],+ ) -> ModelCallResult:call_log.append("before_model")response = handler(request)call_log.append("after_model")diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_call_limit.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_call_limit.pyindex ec82265a61..ade644a378 100644--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_call_limit.py+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_call_limit.py@@ -9,11 +9,12 @@ from langchain.agents.factory import create_agentfrom langchain.agents.middleware.tool_call_limit import (ToolCallLimitExceededError,ToolCallLimitMiddleware,+ ToolCallLimitState,)from tests.unit_tests.agents.model import FakeToolCallingModel-def test_middleware_initialization_validation():+def test_middleware_initialization_validation() -> None:"""Test that middleware initialization validates parameters correctly."""# Test that at least one limit must be specifiedwith pytest.raises(ValueError, match="At least one limit must be specified"):@@ -59,7 +60,7 @@ def test_middleware_initialization_validation():assert middleware.run_limit == 3-def test_middleware_name_property():+def test_middleware_name_property() -> None:"""Test that the name property includes tool name when specified."""# Test without tool namemiddleware = ToolCallLimitMiddleware(thread_limit=5)@@ -77,7 +78,7 @@ def test_middleware_name_property():assert middleware2.name == "ToolCallLimitMiddleware[calculator]"-def test_middleware_unit_functionality():+def test_middleware_unit_functionality() -> None:"""Test that the middleware works as expected in isolation.Tests basic count tracking, thread limit, run limit, and limit-not-exceeded cases.@@ -86,14 +87,14 @@ def test_middleware_unit_functionality():runtime = None# Test when limits are not exceeded - counts should increment normally- state = {- "messages": [+ state = ToolCallLimitState(+ messages=[HumanMessage("Question"),AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}]),],- "thread_tool_call_count": {},- "run_tool_call_count": {},- }+ thread_tool_call_count={},+ run_tool_call_count={},+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is not Noneassert result["thread_tool_call_count"] == {"__all__": 1}@@ -101,14 +102,14 @@ def test_middleware_unit_functionality():assert "jump_to" not in result# Test thread limit exceeded (start at thread_limit so next call will exceed)- state = {- "messages": [+ state = ToolCallLimitState(+ messages=[HumanMessage("Question 2"),AIMessage("Response 2", tool_calls=[{"name": "search", "args": {}, "id": "3"}]),],- "thread_tool_call_count": {"__all__": 3}, # Already exceeds thread_limit=3- "run_tool_call_count": {"__all__": 0}, # No calls yet- }+ thread_tool_call_count={"__all__": 3}, # Already exceeds thread_limit=3+ run_tool_call_count={"__all__": 0}, # No calls yet+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is not Noneassert result["jump_to"] == "end"@@ -124,6 +125,7 @@ def test_middleware_unit_functionality():# Check the final AI message (displayed to user - includes thread/run details)final_ai_msg = result["messages"][-1]assert isinstance(final_ai_msg, AIMessage)+ assert isinstance(final_ai_msg.content, str)assert "limit" in final_ai_msg.content.lower()assert "thread limit exceeded" in final_ai_msg.content.lower()# Thread count stays at 3 (blocked call not counted)@@ -132,14 +134,14 @@ def test_middleware_unit_functionality():assert result["run_tool_call_count"] == {"__all__": 1}# Test run limit exceeded (thread count must be >= run count)- state = {- "messages": [+ state = ToolCallLimitState(+ messages=[HumanMessage("Question"),AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}]),],- "thread_tool_call_count": {"__all__": 2},- "run_tool_call_count": {"__all__": 2},- }+ thread_tool_call_count={"__all__": 2},+ run_tool_call_count={"__all__": 2},+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is not Noneassert result["jump_to"] == "end"@@ -156,7 +158,7 @@ def test_middleware_unit_functionality():)-def test_middleware_end_behavior_with_unrelated_parallel_tool_calls():+def test_middleware_end_behavior_with_unrelated_parallel_tool_calls() -> None:"""Test middleware 'end' behavior with unrelated parallel tool calls.Test that 'end' behavior raises NotImplementedError when there are parallel calls@@ -171,8 +173,8 @@ def test_middleware_end_behavior_with_unrelated_parallel_tool_calls():runtime = None# Test with search + calculator calls when search exceeds limit- state = {- "messages": [+ state = ToolCallLimitState(+ messages=[AIMessage("Response",tool_calls=[@@ -181,9 +183,9 @@ def test_middleware_end_behavior_with_unrelated_parallel_tool_calls():],),],- "thread_tool_call_count": {"search": 1},- "run_tool_call_count": {"search": 1},- }+ thread_tool_call_count={"search": 1},+ run_tool_call_count={"search": 1},+ )with pytest.raises(NotImplementedError, match="Cannot end execution with other tool calls pending"@@ -191,7 +193,7 @@ def test_middleware_end_behavior_with_unrelated_parallel_tool_calls():middleware.after_model(state, runtime) # type: ignore[arg-type]-def test_middleware_with_specific_tool():+def test_middleware_with_specific_tool() -> None:"""Test middleware that limits a specific tool while ignoring others."""middleware = ToolCallLimitMiddleware(tool_name="search", thread_limit=2, run_limit=1, exit_behavior="end"@@ -199,31 +201,31 @@ def test_middleware_with_specific_tool():runtime = None# Test search tool exceeding run limit- state = {- "messages": [+ state = ToolCallLimitState(+ messages=[AIMessage("Response 2", tool_calls=[{"name": "search", "args": {}, "id": "3"}]),],- "thread_tool_call_count": {"search": 1},- "run_tool_call_count": {"search": 1},- }+ thread_tool_call_count={"search": 1},+ run_tool_call_count={"search": 1},+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is not Noneassert result["jump_to"] == "end"assert "search" in result["messages"][0].content.lower()# Test calculator tool - should be ignored by search-specific middleware- state = {- "messages": [+ state = ToolCallLimitState(+ messages=[AIMessage("Response", tool_calls=[{"name": "calculator", "args": {}, "id": "1"}] * 10),],- "thread_tool_call_count": {},- "run_tool_call_count": {},- }+ thread_tool_call_count={},+ run_tool_call_count={},+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is None, "Calculator calls shouldn't be counted by search-specific middleware"-def test_middleware_error_behavior():+def test_middleware_error_behavior() -> None:"""Test middleware error behavior.Test that middleware raises ToolCallLimitExceededError when configured with@@ -232,11 +234,11 @@ def test_middleware_error_behavior():middleware = ToolCallLimitMiddleware(thread_limit=2, exit_behavior="error")runtime = None- state = {- "messages": [AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],- "thread_tool_call_count": {"__all__": 2},- "run_tool_call_count": {"__all__": 2},- }+ state = ToolCallLimitState(+ messages=[AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],+ thread_tool_call_count={"__all__": 2},+ run_tool_call_count={"__all__": 2},+ )with pytest.raises(ToolCallLimitExceededError) as exc_info:middleware.after_model(state, runtime) # type: ignore[arg-type]@@ -250,7 +252,7 @@ def test_middleware_error_behavior():assert error.tool_name is None-def test_multiple_middleware_instances():+def test_multiple_middleware_instances() -> None:"""Test that multiple middleware instances can coexist and track independently."""@tool@@ -302,15 +304,17 @@ def test_multiple_middleware_instances():# The agent should stop after the second iteration# because search will hit its limit (3 calls > 2 limit)- ai_limit_messages = [- msg- for msg in result["messages"]- if isinstance(msg, AIMessage) and "limit" in msg.content.lower()- ]+ ai_limit_messages = []+ for msg in result["messages"]:+ if not isinstance(msg, AIMessage):+ continue+ assert isinstance(msg.content, str)+ if "limit" in msg.content.lower():+ ai_limit_messages.append(msg)assert len(ai_limit_messages) > 0, "Should have AI message explaining limit was exceeded"-def test_run_limit_with_multiple_human_messages():+def test_run_limit_with_multiple_human_messages() -> None:"""Test that run limits reset between invocations.Verifies that when using run_limit, the count resets for each new user message,@@ -343,11 +347,13 @@ def test_run_limit_with_multiple_human_messages():tool_messages = [msg for msg in result1["messages"] if isinstance(msg, ToolMessage)]successful_tool_msgs = [msg for msg in tool_messages if msg.status != "error"]error_tool_msgs = [msg for msg in tool_messages if msg.status == "error"]- ai_limit_msgs = [- msg- for msg in result1["messages"]- if isinstance(msg, AIMessage) and "limit" in msg.content.lower() and not msg.tool_calls- ]+ ai_limit_msgs = []+ for msg in result1["messages"]:+ if not isinstance(msg, AIMessage):+ continue+ assert isinstance(msg.content, str)+ if "limit" in msg.content.lower() and not msg.tool_calls:+ ai_limit_msgs.append(msg)assert len(successful_tool_msgs) == 1, "Should have 1 successful tool execution (test1)"assert len(error_tool_msgs) == 1, "Should have 1 artificial error ToolMessage (test2)"@@ -364,7 +370,7 @@ def test_run_limit_with_multiple_human_messages():)-def test_exception_error_messages():+def test_exception_error_messages() -> None:"""Test that error messages include expected information."""# Test for specific toolwith pytest.raises(ToolCallLimitExceededError) as exc_info:@@ -384,35 +390,35 @@ def test_exception_error_messages():assert "10/8" in msg or "thread" in msg.lower()-def test_limit_reached_but_not_exceeded():+def test_limit_reached_but_not_exceeded() -> None:"""Test that limits are only triggered when exceeded (>), not when reached (==)."""middleware = ToolCallLimitMiddleware(thread_limit=3, run_limit=2, exit_behavior="end")runtime = None# Test when limit is reached exactly (count = limit) - should not trigger- state = {- "messages": [AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],- "thread_tool_call_count": {"__all__": 2}, # After +1 will be exactly 3- "run_tool_call_count": {"__all__": 1},- }+ state = ToolCallLimitState(+ messages=[AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],+ thread_tool_call_count={"__all__": 2}, # After +1 will be exactly 3+ run_tool_call_count={"__all__": 1},+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is not Noneassert "jump_to" not in resultassert result["thread_tool_call_count"]["__all__"] == 3# Test when limit is exceeded (count > limit) - should trigger- state = {- "messages": [AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],- "thread_tool_call_count": {"__all__": 3}, # After +1 will be 4 > 3- "run_tool_call_count": {"__all__": 1},- }+ state = ToolCallLimitState(+ messages=[AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],+ thread_tool_call_count={"__all__": 3}, # After +1 will be 4 > 3+ run_tool_call_count={"__all__": 1},+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is not Noneassert "jump_to" in resultassert result["jump_to"] == "end"-def test_exit_behavior_continue():+def test_exit_behavior_continue() -> None:"""Test that exit_behavior='continue' blocks only the exceeded tool, not others.Verifies that when a specific tool hits its limit, it gets blocked with error messages@@ -468,11 +474,11 @@ def test_exit_behavior_continue():# Verify search has 2 successful + 1 blocked, calculator has all 3 successfulsuccessful_search_msgs = [msg for msg in tool_messages if "Search:" in msg.content]- blocked_search_msgs = [- msg- for msg in tool_messages- if "limit" in msg.content.lower() and "search" in msg.content.lower()- ]+ blocked_search_msgs = []+ for msg in tool_messages:+ assert isinstance(msg.content, str)+ if "limit" in msg.content.lower() and "search" in msg.content.lower():+ blocked_search_msgs.append(msg)successful_calc_msgs = [msg for msg in tool_messages if "Calc:" in msg.content]assert len(successful_search_msgs) == 2, "Should have 2 successful search calls"@@ -480,7 +486,7 @@ def test_exit_behavior_continue():assert len(successful_calc_msgs) == 3, "All calculator calls should succeed"-def test_thread_count_excludes_blocked_run_calls():+def test_thread_count_excludes_blocked_run_calls() -> None:"""Test that thread count only includes allowed calls, not blocked run-scoped calls.When run_limit is lower than thread_limit and multiple parallel calls are made,@@ -494,8 +500,8 @@ def test_thread_count_excludes_blocked_run_calls():runtime = None# Make 3 parallel tool calls - only 1 should be allowed by run_limit- state = {- "messages": [+ state = ToolCallLimitState(+ messages=[AIMessage("Response",tool_calls=[@@ -505,9 +511,9 @@ def test_thread_count_excludes_blocked_run_calls():],)],- "thread_tool_call_count": {},- "run_tool_call_count": {},- }+ thread_tool_call_count={},+ run_tool_call_count={},+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is not None@@ -526,7 +532,7 @@ def test_thread_count_excludes_blocked_run_calls():assert len(error_messages) == 2, "Should have 2 error messages for the 2 blocked calls"-def test_unified_error_messages():+def test_unified_error_messages() -> None:"""Test that error messages instruct model not to call again for both run and thread limits.Previously, only thread limit messages included 'Do not' instruction.@@ -536,11 +542,11 @@ def test_unified_error_messages():runtime = None# Test with run limit exceeded (thread limit not exceeded)- state = {- "messages": [AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "2"}])],- "thread_tool_call_count": {"__all__": 1}, # Under thread limit- "run_tool_call_count": {"__all__": 1}, # At run limit, next call will exceed- }+ state = ToolCallLimitState(+ messages=[AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "2"}])],+ thread_tool_call_count={"__all__": 1}, # Under thread limit+ run_tool_call_count={"__all__": 1}, # At run limit, next call will exceed+ )result = middleware.after_model(state, runtime) # type: ignore[arg-type]assert result is not None@@ -553,7 +559,7 @@ def test_unified_error_messages():)-def test_end_behavior_creates_artificial_messages():+def test_end_behavior_creates_artificial_messages() -> None:"""Test that 'end' behavior creates an AI message explaining why execution stopped.Verifies that when limit is exceeded with exit_behavior='end', the middleware:@@ -586,17 +592,20 @@ def test_end_behavior_creates_artificial_messages():)# Verify AI message explaining the limit (displayed to user - includes thread/run details)- ai_limit_messages = [- msg- for msg in result["messages"]- if isinstance(msg, AIMessage) and "limit" in msg.content.lower() and not msg.tool_calls- ]+ ai_limit_messages = []+ for msg in result["messages"]:+ if not isinstance(msg, AIMessage):+ continue+ assert isinstance(msg.content, str)+ if "limit" in msg.content.lower() and not msg.tool_calls:+ ai_limit_messages.append(msg)assert len(ai_limit_messages) == 1, "Should have exactly one AI message explaining the limit"- ai_msg_content = ai_limit_messages[0].content.lower()- assert "thread limit exceeded" in ai_msg_content or "run limit exceeded" in ai_msg_content, (- "AI message should include thread/run limit details for the user"- )+ ai_msg_content = ai_limit_messages[0].content+ assert isinstance(ai_msg_content, str)+ assert "thread limit exceeded" in ai_msg_content.lower() or (+ "run limit exceeded" in ai_msg_content.lower()+ ), "AI message should include thread/run limit details for the user"# Verify tool message countstool_messages = [msg for msg in result["messages"] if isinstance(msg, ToolMessage)]@@ -614,7 +623,7 @@ def test_end_behavior_creates_artificial_messages():)-def test_parallel_tool_calls_with_limit_continue_mode():+def test_parallel_tool_calls_with_limit_continue_mode() -> None:"""Test parallel tool calls with a limit of 1 in 'continue' mode.When the model proposes 3 tool calls with a limit of 1:@@ -663,6 +672,7 @@ def test_parallel_tool_calls_with_limit_continue_mode():# Verify error messages explain the limitfor error_msg in error_tool_messages:+ assert isinstance(error_msg.content, str)assert "limit" in error_msg.content.lower()# Verify execution continued (no early termination)@@ -671,7 +681,7 @@ def test_parallel_tool_calls_with_limit_continue_mode():assert len(ai_messages) >= 2-def test_parallel_tool_calls_with_limit_end_mode():+def test_parallel_tool_calls_with_limit_end_mode() -> None:"""Test parallel tool calls with a limit of 1 in 'end' mode.When the model proposes 3 tool calls with a limit of 1:@@ -725,20 +735,23 @@ def test_parallel_tool_calls_with_limit_end_mode():# Verify AI message explaining why execution stopped# (displayed to user - includes thread/run details)- ai_limit_messages = [- msg- for msg in messages- if isinstance(msg, AIMessage) and "limit" in msg.content.lower() and not msg.tool_calls- ]+ ai_limit_messages = []+ for msg in messages:+ if not isinstance(msg, AIMessage):+ continue+ assert isinstance(msg.content, str)+ if "limit" in msg.content.lower() and not msg.tool_calls:+ ai_limit_messages.append(msg)assert len(ai_limit_messages) == 1, "Should have exactly one AI message explaining the limit"- ai_msg_content = ai_limit_messages[0].content.lower()- assert "thread limit exceeded" in ai_msg_content or "run limit exceeded" in ai_msg_content, (- "AI message should include thread/run limit details for the user"- )+ ai_msg_content = ai_limit_messages[0].content+ assert isinstance(ai_msg_content, str)+ assert "thread limit exceeded" in ai_msg_content.lower() or (+ "run limit exceeded" in ai_msg_content.lower()+ ), "AI message should include thread/run limit details for the user"-def test_parallel_mixed_tool_calls_with_specific_tool_limit():+def test_parallel_mixed_tool_calls_with_specific_tool_limit() -> None:"""Test parallel calls to different tools when limiting a specific tool.When limiting 'search' to 1 call, and model proposes 3 search + 2 calculator calls:@@ -785,12 +798,19 @@ def test_parallel_mixed_tool_calls_with_specific_tool_limit():)messages = result["messages"]- tool_messages = [msg for msg in messages if isinstance(msg, ToolMessage)]- search_success = [m for m in tool_messages if "Search:" in m.content]- search_blocked = [- m for m in tool_messages if "limit" in m.content.lower() and "search" in m.content.lower()- ]- calc_success = [m for m in tool_messages if "Calc:" in m.content]+ search_success = []+ search_blocked = []+ calc_success = []+ for m in messages:+ if not isinstance(m, ToolMessage):+ continue+ assert isinstance(m.content, str)+ if "Search:" in m.content:+ search_success.append(m)+ if "limit" in m.content.lower() and "search" in m.content.lower():+ search_blocked.append(m)+ if "Calc:" in m.content:+ calc_success.append(m)assert len(search_success) == 1, "Should have 1 successful search call"assert len(search_blocked) == 2, "Should have 2 blocked search calls"