Skip to content

The agent engineering platform.

test(langchain): types in `test_tool_call_limit` and `test_model_retry` (#34629)

Christophe Bornetd181a59ebef9
diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_retry.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_retry.py
index 6f0eb19a23..f38abe3f21 100644
--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_retry.py
+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_model_retry.py
@@ -1,6 +1,7 @@
"""Tests for ModelRetryMiddleware functionality."""
import time
+from collections.abc import Callable
from typing import Any
import pytest
@@ -13,7 +14,12 @@ from pydantic import Field
from langchain.agents.factory import create_agent
from langchain.agents.middleware._retry import calculate_delay
from langchain.agents.middleware.model_retry import ModelRetryMiddleware
-from langchain.agents.middleware.types import wrap_model_call
+from langchain.agents.middleware.types import (
+ ModelCallResult,
+ ModelRequest,
+ ModelResponse,
+ wrap_model_call,
+)
from tests.unit_tests.agents.model import FakeToolCallingModel
@@ -659,7 +665,10 @@ def test_model_retry_multiple_middleware_composition() -> None:
# Custom middleware that logs calls
@wrap_model_call
- def logging_middleware(request, handler):
+ def logging_middleware(
+ request: ModelRequest,
+ handler: Callable[[ModelRequest], ModelResponse],
+ ) -> ModelCallResult:
call_log.append("before_model")
response = handler(request)
call_log.append("after_model")
diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_call_limit.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_call_limit.py
index ec82265a61..ade644a378 100644
--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_call_limit.py
+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_tool_call_limit.py
@@ -9,11 +9,12 @@ from langchain.agents.factory import create_agent
from langchain.agents.middleware.tool_call_limit import (
ToolCallLimitExceededError,
ToolCallLimitMiddleware,
+ ToolCallLimitState,
)
from tests.unit_tests.agents.model import FakeToolCallingModel
-def test_middleware_initialization_validation():
+def test_middleware_initialization_validation() -> None:
"""Test that middleware initialization validates parameters correctly."""
# Test that at least one limit must be specified
with pytest.raises(ValueError, match="At least one limit must be specified"):
@@ -59,7 +60,7 @@ def test_middleware_initialization_validation():
assert middleware.run_limit == 3
-def test_middleware_name_property():
+def test_middleware_name_property() -> None:
"""Test that the name property includes tool name when specified."""
# Test without tool name
middleware = ToolCallLimitMiddleware(thread_limit=5)
@@ -77,7 +78,7 @@ def test_middleware_name_property():
assert middleware2.name == "ToolCallLimitMiddleware[calculator]"
-def test_middleware_unit_functionality():
+def test_middleware_unit_functionality() -> None:
"""Test that the middleware works as expected in isolation.
Tests basic count tracking, thread limit, run limit, and limit-not-exceeded cases.
@@ -86,14 +87,14 @@ def test_middleware_unit_functionality():
runtime = None
# Test when limits are not exceeded - counts should increment normally
- state = {
- "messages": [
+ state = ToolCallLimitState(
+ messages=[
HumanMessage("Question"),
AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}]),
],
- "thread_tool_call_count": {},
- "run_tool_call_count": {},
- }
+ thread_tool_call_count={},
+ run_tool_call_count={},
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is not None
assert result["thread_tool_call_count"] == {"__all__": 1}
@@ -101,14 +102,14 @@ def test_middleware_unit_functionality():
assert "jump_to" not in result
# Test thread limit exceeded (start at thread_limit so next call will exceed)
- state = {
- "messages": [
+ state = ToolCallLimitState(
+ messages=[
HumanMessage("Question 2"),
AIMessage("Response 2", tool_calls=[{"name": "search", "args": {}, "id": "3"}]),
],
- "thread_tool_call_count": {"__all__": 3}, # Already exceeds thread_limit=3
- "run_tool_call_count": {"__all__": 0}, # No calls yet
- }
+ thread_tool_call_count={"__all__": 3}, # Already exceeds thread_limit=3
+ run_tool_call_count={"__all__": 0}, # No calls yet
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is not None
assert result["jump_to"] == "end"
@@ -124,6 +125,7 @@ def test_middleware_unit_functionality():
# Check the final AI message (displayed to user - includes thread/run details)
final_ai_msg = result["messages"][-1]
assert isinstance(final_ai_msg, AIMessage)
+ assert isinstance(final_ai_msg.content, str)
assert "limit" in final_ai_msg.content.lower()
assert "thread limit exceeded" in final_ai_msg.content.lower()
# Thread count stays at 3 (blocked call not counted)
@@ -132,14 +134,14 @@ def test_middleware_unit_functionality():
assert result["run_tool_call_count"] == {"__all__": 1}
# Test run limit exceeded (thread count must be >= run count)
- state = {
- "messages": [
+ state = ToolCallLimitState(
+ messages=[
HumanMessage("Question"),
AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}]),
],
- "thread_tool_call_count": {"__all__": 2},
- "run_tool_call_count": {"__all__": 2},
- }
+ thread_tool_call_count={"__all__": 2},
+ run_tool_call_count={"__all__": 2},
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is not None
assert result["jump_to"] == "end"
@@ -156,7 +158,7 @@ def test_middleware_unit_functionality():
)
-def test_middleware_end_behavior_with_unrelated_parallel_tool_calls():
+def test_middleware_end_behavior_with_unrelated_parallel_tool_calls() -> None:
"""Test middleware 'end' behavior with unrelated parallel tool calls.
Test that 'end' behavior raises NotImplementedError when there are parallel calls
@@ -171,8 +173,8 @@ def test_middleware_end_behavior_with_unrelated_parallel_tool_calls():
runtime = None
# Test with search + calculator calls when search exceeds limit
- state = {
- "messages": [
+ state = ToolCallLimitState(
+ messages=[
AIMessage(
"Response",
tool_calls=[
@@ -181,9 +183,9 @@ def test_middleware_end_behavior_with_unrelated_parallel_tool_calls():
],
),
],
- "thread_tool_call_count": {"search": 1},
- "run_tool_call_count": {"search": 1},
- }
+ thread_tool_call_count={"search": 1},
+ run_tool_call_count={"search": 1},
+ )
with pytest.raises(
NotImplementedError, match="Cannot end execution with other tool calls pending"
@@ -191,7 +193,7 @@ def test_middleware_end_behavior_with_unrelated_parallel_tool_calls():
middleware.after_model(state, runtime) # type: ignore[arg-type]
-def test_middleware_with_specific_tool():
+def test_middleware_with_specific_tool() -> None:
"""Test middleware that limits a specific tool while ignoring others."""
middleware = ToolCallLimitMiddleware(
tool_name="search", thread_limit=2, run_limit=1, exit_behavior="end"
@@ -199,31 +201,31 @@ def test_middleware_with_specific_tool():
runtime = None
# Test search tool exceeding run limit
- state = {
- "messages": [
+ state = ToolCallLimitState(
+ messages=[
AIMessage("Response 2", tool_calls=[{"name": "search", "args": {}, "id": "3"}]),
],
- "thread_tool_call_count": {"search": 1},
- "run_tool_call_count": {"search": 1},
- }
+ thread_tool_call_count={"search": 1},
+ run_tool_call_count={"search": 1},
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is not None
assert result["jump_to"] == "end"
assert "search" in result["messages"][0].content.lower()
# Test calculator tool - should be ignored by search-specific middleware
- state = {
- "messages": [
+ state = ToolCallLimitState(
+ messages=[
AIMessage("Response", tool_calls=[{"name": "calculator", "args": {}, "id": "1"}] * 10),
],
- "thread_tool_call_count": {},
- "run_tool_call_count": {},
- }
+ thread_tool_call_count={},
+ run_tool_call_count={},
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is None, "Calculator calls shouldn't be counted by search-specific middleware"
-def test_middleware_error_behavior():
+def test_middleware_error_behavior() -> None:
"""Test middleware error behavior.
Test that middleware raises ToolCallLimitExceededError when configured with
@@ -232,11 +234,11 @@ def test_middleware_error_behavior():
middleware = ToolCallLimitMiddleware(thread_limit=2, exit_behavior="error")
runtime = None
- state = {
- "messages": [AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],
- "thread_tool_call_count": {"__all__": 2},
- "run_tool_call_count": {"__all__": 2},
- }
+ state = ToolCallLimitState(
+ messages=[AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],
+ thread_tool_call_count={"__all__": 2},
+ run_tool_call_count={"__all__": 2},
+ )
with pytest.raises(ToolCallLimitExceededError) as exc_info:
middleware.after_model(state, runtime) # type: ignore[arg-type]
@@ -250,7 +252,7 @@ def test_middleware_error_behavior():
assert error.tool_name is None
-def test_multiple_middleware_instances():
+def test_multiple_middleware_instances() -> None:
"""Test that multiple middleware instances can coexist and track independently."""
@tool
@@ -302,15 +304,17 @@ def test_multiple_middleware_instances():
# The agent should stop after the second iteration
# because search will hit its limit (3 calls > 2 limit)
- ai_limit_messages = [
- msg
- for msg in result["messages"]
- if isinstance(msg, AIMessage) and "limit" in msg.content.lower()
- ]
+ ai_limit_messages = []
+ for msg in result["messages"]:
+ if not isinstance(msg, AIMessage):
+ continue
+ assert isinstance(msg.content, str)
+ if "limit" in msg.content.lower():
+ ai_limit_messages.append(msg)
assert len(ai_limit_messages) > 0, "Should have AI message explaining limit was exceeded"
-def test_run_limit_with_multiple_human_messages():
+def test_run_limit_with_multiple_human_messages() -> None:
"""Test that run limits reset between invocations.
Verifies that when using run_limit, the count resets for each new user message,
@@ -343,11 +347,13 @@ def test_run_limit_with_multiple_human_messages():
tool_messages = [msg for msg in result1["messages"] if isinstance(msg, ToolMessage)]
successful_tool_msgs = [msg for msg in tool_messages if msg.status != "error"]
error_tool_msgs = [msg for msg in tool_messages if msg.status == "error"]
- ai_limit_msgs = [
- msg
- for msg in result1["messages"]
- if isinstance(msg, AIMessage) and "limit" in msg.content.lower() and not msg.tool_calls
- ]
+ ai_limit_msgs = []
+ for msg in result1["messages"]:
+ if not isinstance(msg, AIMessage):
+ continue
+ assert isinstance(msg.content, str)
+ if "limit" in msg.content.lower() and not msg.tool_calls:
+ ai_limit_msgs.append(msg)
assert len(successful_tool_msgs) == 1, "Should have 1 successful tool execution (test1)"
assert len(error_tool_msgs) == 1, "Should have 1 artificial error ToolMessage (test2)"
@@ -364,7 +370,7 @@ def test_run_limit_with_multiple_human_messages():
)
-def test_exception_error_messages():
+def test_exception_error_messages() -> None:
"""Test that error messages include expected information."""
# Test for specific tool
with pytest.raises(ToolCallLimitExceededError) as exc_info:
@@ -384,35 +390,35 @@ def test_exception_error_messages():
assert "10/8" in msg or "thread" in msg.lower()
-def test_limit_reached_but_not_exceeded():
+def test_limit_reached_but_not_exceeded() -> None:
"""Test that limits are only triggered when exceeded (>), not when reached (==)."""
middleware = ToolCallLimitMiddleware(thread_limit=3, run_limit=2, exit_behavior="end")
runtime = None
# Test when limit is reached exactly (count = limit) - should not trigger
- state = {
- "messages": [AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],
- "thread_tool_call_count": {"__all__": 2}, # After +1 will be exactly 3
- "run_tool_call_count": {"__all__": 1},
- }
+ state = ToolCallLimitState(
+ messages=[AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],
+ thread_tool_call_count={"__all__": 2}, # After +1 will be exactly 3
+ run_tool_call_count={"__all__": 1},
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is not None
assert "jump_to" not in result
assert result["thread_tool_call_count"]["__all__"] == 3
# Test when limit is exceeded (count > limit) - should trigger
- state = {
- "messages": [AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],
- "thread_tool_call_count": {"__all__": 3}, # After +1 will be 4 > 3
- "run_tool_call_count": {"__all__": 1},
- }
+ state = ToolCallLimitState(
+ messages=[AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "1"}])],
+ thread_tool_call_count={"__all__": 3}, # After +1 will be 4 > 3
+ run_tool_call_count={"__all__": 1},
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is not None
assert "jump_to" in result
assert result["jump_to"] == "end"
-def test_exit_behavior_continue():
+def test_exit_behavior_continue() -> None:
"""Test that exit_behavior='continue' blocks only the exceeded tool, not others.
Verifies that when a specific tool hits its limit, it gets blocked with error messages
@@ -468,11 +474,11 @@ def test_exit_behavior_continue():
# Verify search has 2 successful + 1 blocked, calculator has all 3 successful
successful_search_msgs = [msg for msg in tool_messages if "Search:" in msg.content]
- blocked_search_msgs = [
- msg
- for msg in tool_messages
- if "limit" in msg.content.lower() and "search" in msg.content.lower()
- ]
+ blocked_search_msgs = []
+ for msg in tool_messages:
+ assert isinstance(msg.content, str)
+ if "limit" in msg.content.lower() and "search" in msg.content.lower():
+ blocked_search_msgs.append(msg)
successful_calc_msgs = [msg for msg in tool_messages if "Calc:" in msg.content]
assert len(successful_search_msgs) == 2, "Should have 2 successful search calls"
@@ -480,7 +486,7 @@ def test_exit_behavior_continue():
assert len(successful_calc_msgs) == 3, "All calculator calls should succeed"
-def test_thread_count_excludes_blocked_run_calls():
+def test_thread_count_excludes_blocked_run_calls() -> None:
"""Test that thread count only includes allowed calls, not blocked run-scoped calls.
When run_limit is lower than thread_limit and multiple parallel calls are made,
@@ -494,8 +500,8 @@ def test_thread_count_excludes_blocked_run_calls():
runtime = None
# Make 3 parallel tool calls - only 1 should be allowed by run_limit
- state = {
- "messages": [
+ state = ToolCallLimitState(
+ messages=[
AIMessage(
"Response",
tool_calls=[
@@ -505,9 +511,9 @@ def test_thread_count_excludes_blocked_run_calls():
],
)
],
- "thread_tool_call_count": {},
- "run_tool_call_count": {},
- }
+ thread_tool_call_count={},
+ run_tool_call_count={},
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is not None
@@ -526,7 +532,7 @@ def test_thread_count_excludes_blocked_run_calls():
assert len(error_messages) == 2, "Should have 2 error messages for the 2 blocked calls"
-def test_unified_error_messages():
+def test_unified_error_messages() -> None:
"""Test that error messages instruct model not to call again for both run and thread limits.
Previously, only thread limit messages included 'Do not' instruction.
@@ -536,11 +542,11 @@ def test_unified_error_messages():
runtime = None
# Test with run limit exceeded (thread limit not exceeded)
- state = {
- "messages": [AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "2"}])],
- "thread_tool_call_count": {"__all__": 1}, # Under thread limit
- "run_tool_call_count": {"__all__": 1}, # At run limit, next call will exceed
- }
+ state = ToolCallLimitState(
+ messages=[AIMessage("Response", tool_calls=[{"name": "search", "args": {}, "id": "2"}])],
+ thread_tool_call_count={"__all__": 1}, # Under thread limit
+ run_tool_call_count={"__all__": 1}, # At run limit, next call will exceed
+ )
result = middleware.after_model(state, runtime) # type: ignore[arg-type]
assert result is not None
@@ -553,7 +559,7 @@ def test_unified_error_messages():
)
-def test_end_behavior_creates_artificial_messages():
+def test_end_behavior_creates_artificial_messages() -> None:
"""Test that 'end' behavior creates an AI message explaining why execution stopped.
Verifies that when limit is exceeded with exit_behavior='end', the middleware:
@@ -586,17 +592,20 @@ def test_end_behavior_creates_artificial_messages():
)
# Verify AI message explaining the limit (displayed to user - includes thread/run details)
- ai_limit_messages = [
- msg
- for msg in result["messages"]
- if isinstance(msg, AIMessage) and "limit" in msg.content.lower() and not msg.tool_calls
- ]
+ ai_limit_messages = []
+ for msg in result["messages"]:
+ if not isinstance(msg, AIMessage):
+ continue
+ assert isinstance(msg.content, str)
+ if "limit" in msg.content.lower() and not msg.tool_calls:
+ ai_limit_messages.append(msg)
assert len(ai_limit_messages) == 1, "Should have exactly one AI message explaining the limit"
- ai_msg_content = ai_limit_messages[0].content.lower()
- assert "thread limit exceeded" in ai_msg_content or "run limit exceeded" in ai_msg_content, (
- "AI message should include thread/run limit details for the user"
- )
+ ai_msg_content = ai_limit_messages[0].content
+ assert isinstance(ai_msg_content, str)
+ assert "thread limit exceeded" in ai_msg_content.lower() or (
+ "run limit exceeded" in ai_msg_content.lower()
+ ), "AI message should include thread/run limit details for the user"
# Verify tool message counts
tool_messages = [msg for msg in result["messages"] if isinstance(msg, ToolMessage)]
@@ -614,7 +623,7 @@ def test_end_behavior_creates_artificial_messages():
)
-def test_parallel_tool_calls_with_limit_continue_mode():
+def test_parallel_tool_calls_with_limit_continue_mode() -> None:
"""Test parallel tool calls with a limit of 1 in 'continue' mode.
When the model proposes 3 tool calls with a limit of 1:
@@ -663,6 +672,7 @@ def test_parallel_tool_calls_with_limit_continue_mode():
# Verify error messages explain the limit
for error_msg in error_tool_messages:
+ assert isinstance(error_msg.content, str)
assert "limit" in error_msg.content.lower()
# Verify execution continued (no early termination)
@@ -671,7 +681,7 @@ def test_parallel_tool_calls_with_limit_continue_mode():
assert len(ai_messages) >= 2
-def test_parallel_tool_calls_with_limit_end_mode():
+def test_parallel_tool_calls_with_limit_end_mode() -> None:
"""Test parallel tool calls with a limit of 1 in 'end' mode.
When the model proposes 3 tool calls with a limit of 1:
@@ -725,20 +735,23 @@ def test_parallel_tool_calls_with_limit_end_mode():
# Verify AI message explaining why execution stopped
# (displayed to user - includes thread/run details)
- ai_limit_messages = [
- msg
- for msg in messages
- if isinstance(msg, AIMessage) and "limit" in msg.content.lower() and not msg.tool_calls
- ]
+ ai_limit_messages = []
+ for msg in messages:
+ if not isinstance(msg, AIMessage):
+ continue
+ assert isinstance(msg.content, str)
+ if "limit" in msg.content.lower() and not msg.tool_calls:
+ ai_limit_messages.append(msg)
assert len(ai_limit_messages) == 1, "Should have exactly one AI message explaining the limit"
- ai_msg_content = ai_limit_messages[0].content.lower()
- assert "thread limit exceeded" in ai_msg_content or "run limit exceeded" in ai_msg_content, (
- "AI message should include thread/run limit details for the user"
- )
+ ai_msg_content = ai_limit_messages[0].content
+ assert isinstance(ai_msg_content, str)
+ assert "thread limit exceeded" in ai_msg_content.lower() or (
+ "run limit exceeded" in ai_msg_content.lower()
+ ), "AI message should include thread/run limit details for the user"
-def test_parallel_mixed_tool_calls_with_specific_tool_limit():
+def test_parallel_mixed_tool_calls_with_specific_tool_limit() -> None:
"""Test parallel calls to different tools when limiting a specific tool.
When limiting 'search' to 1 call, and model proposes 3 search + 2 calculator calls:
@@ -785,12 +798,19 @@ def test_parallel_mixed_tool_calls_with_specific_tool_limit():
)
messages = result["messages"]
- tool_messages = [msg for msg in messages if isinstance(msg, ToolMessage)]
- search_success = [m for m in tool_messages if "Search:" in m.content]
- search_blocked = [
- m for m in tool_messages if "limit" in m.content.lower() and "search" in m.content.lower()
- ]
- calc_success = [m for m in tool_messages if "Calc:" in m.content]
+ search_success = []
+ search_blocked = []
+ calc_success = []
+ for m in messages:
+ if not isinstance(m, ToolMessage):
+ continue
+ assert isinstance(m.content, str)
+ if "Search:" in m.content:
+ search_success.append(m)
+ if "limit" in m.content.lower() and "search" in m.content.lower():
+ search_blocked.append(m)
+ if "Calc:" in m.content:
+ calc_success.append(m)
assert len(search_success) == 1, "Should have 1 successful search call"
assert len(search_blocked) == 2, "Should have 2 blocked search calls"