Skip to content

Build and run agents you can see, understand and trust.

fix(middleware): don't block structured output when the budget is spent (#3027)

Mahmoud Zaher24aac5d14dd9
diff --git a/src/agentscope/middleware/_budget.py b/src/agentscope/middleware/_budget.py
index b97429a7b..a34ae5049 100644
--- a/src/agentscope/middleware/_budget.py
+++ b/src/agentscope/middleware/_budget.py
@@ -2,6 +2,7 @@
"""Budget control middleware for AgentScope agents."""
from typing import AsyncGenerator, Callable, TYPE_CHECKING
+from ..agent._structured_output_tool import _GenerateStructuredOutput
from ..event import ModelCallEndEvent, ReplyStartEvent
from ..message import AssistantMsg, HintBlock
from ..tool import ToolChoice
@@ -17,6 +18,13 @@ _DEFAULT_HINT_MESSAGE = (
"</system-reminder>"
)
+_STRUCTURED_OUTPUT_HINT_MESSAGE = (
+ "<system-reminder>You have reached the maximum token budget set by the "
+ "user. Now you MUST wrap up immediately by calling the '{tool_name}' "
+ "tool with the final structured output, without invoking any other "
+ "tools.</system-reminder>"
+)
+
class ReplyBudgetControlMiddleware(MiddlewareBase):
"""Middleware that enforces a weighted token budget per reply.
@@ -30,7 +38,8 @@ class ReplyBudgetControlMiddleware(MiddlewareBase):
Once the accumulated cost reaches ``token_budget``, a hint message is
injected into the agent's context before the next reasoning step, and
``tool_choice`` is forced to ``"none"`` so the agent wraps up without
- invoking any further tools.
+ invoking any further tools. A reply that still owes a structured output
+ is offered only the structured output tool instead.
Budget state is stored in
:attr:`~agentscope.agent.AgentState.middle_context`
@@ -153,7 +162,9 @@ class ReplyBudgetControlMiddleware(MiddlewareBase):
:class:`~agentscope.message.HintBlock` to the last assistant message
in context (or creates a new
:class:`~agentscope.message.AssistantMsg`) and overrides
- ``tool_choice`` to ``ToolChoice(mode="none")``.
+ ``tool_choice`` to ``ToolChoice(mode="none")``, or, while a
+ structured output is still owed, restricts it to the structured
+ output tool.
Args:
agent (`Agent`):
@@ -177,7 +188,20 @@ class ReplyBudgetControlMiddleware(MiddlewareBase):
# Insert hint block if exceeded budget
if used >= self.token_budget:
- hint_block = HintBlock(hint=self.hint_message)
+ # A structured reply can only end by calling this tool
+ tool_name = _GenerateStructuredOutput.name
+ reply_context = agent.state.reply_context
+ structured_output_pending = (
+ reply_context.structured_schema is not None
+ and reply_context.structured_output is None
+ )
+ hint_block = HintBlock(
+ hint=(
+ _STRUCTURED_OUTPUT_HINT_MESSAGE.format(tool_name=tool_name)
+ if structured_output_pending
+ else self.hint_message
+ ),
+ )
if (
len(agent.state.context) > 0
and agent.state.context[-1].role == "assistant"
@@ -193,7 +217,17 @@ class ReplyBudgetControlMiddleware(MiddlewareBase):
content=[hint_block],
),
)
- input_kwargs["tool_choice"] = ToolChoice(mode="none")
+ if structured_output_pending:
+ # Don't force it unless the agent did: providers reject a
+ # forced tool_choice in thinking mode
+ tool_choice = input_kwargs.get("tool_choice")
+ forced = getattr(tool_choice, "mode", None) == tool_name
+ input_kwargs["tool_choice"] = ToolChoice(
+ mode=tool_name if forced else "auto",
+ tools=[tool_name],
+ )
+ else:
+ input_kwargs["tool_choice"] = ToolChoice(mode="none")
async for event in next_handler(**input_kwargs):
yield event
diff --git a/tests/middleware_budget_test.py b/tests/middleware_budget_test.py
index 4dc0cc193..5f1d470f6 100644
--- a/tests/middleware_budget_test.py
+++ b/tests/middleware_budget_test.py
@@ -3,8 +3,10 @@
from typing import Any, AsyncGenerator, Callable
from unittest.async_case import IsolatedAsyncioTestCase
+from pydantic import BaseModel
+
from utils import MockModel
-from agentscope.agent import Agent
+from agentscope.agent import Agent, ReActConfig
from agentscope.message import UserMsg, TextBlock, ToolCallBlock, HintBlock
from agentscope.middleware import MiddlewareBase, ReplyBudgetControlMiddleware
from agentscope.model import ChatResponse, ChatUsage
@@ -18,7 +20,7 @@ from agentscope.event import (
ReplyEndEvent,
UserConfirmResultEvent,
)
-from agentscope.tool import ToolBase, Toolkit, ToolChunk
+from agentscope.tool import ToolBase, Toolkit, ToolChoice, ToolChunk
def _response(
@@ -94,6 +96,44 @@ class ConfirmRequiredTool(ToolBase):
return ToolChunk(content=[TextBlock(text="confirmed result")])
+class Answer(BaseModel):
+ """The structured output schema used in the tests."""
+
+ city: str
+
+
+class ToolChoiceModel(MockModel):
+ """Model that follows ``tool_choice`` as a provider does: text only
+ under ``"none"``. The first call runs the dummy tool for 300 tokens."""
+
+ def __init__(self, calls_offered_tool: bool) -> None:
+ """Initialize the recorded tool choices."""
+ super().__init__()
+ self.stream = False
+ self.calls_offered_tool = calls_offered_tool
+ self.tool_choices: list = []
+
+ async def _call_api(self, *args: Any, **kwargs: Any) -> ChatResponse:
+ """Record tool_choice and answer by it."""
+ tool_choice = kwargs.get("tool_choice")
+ self.tool_choices.append(tool_choice)
+ mode = getattr(tool_choice, "mode", None)
+ usage = ChatUsage(input_tokens=200, output_tokens=100, time=0.0)
+ if len(self.tool_choices) == 1:
+ block = ToolCallBlock(id="tc_1", name="dummy", input="{}")
+ elif mode == "GenerateStructuredOutput" or (
+ mode != "none" and self.calls_offered_tool
+ ):
+ block = ToolCallBlock(
+ id=f"tc_{len(self.tool_choices)}",
+ name="GenerateStructuredOutput",
+ input='{"city": "Paris"}',
+ )
+ else:
+ block = TextBlock(text="Paris")
+ return ChatResponse(content=[block], is_last=True, usage=usage)
+
+
def _has_hint_block(msg: Any, hint_message: str) -> bool:
"""Return True if *msg* contains a HintBlock with *hint_message*."""
content = getattr(msg, "content", None)
@@ -535,3 +575,65 @@ class TestBudgetControlMiddleware(IsolatedAsyncioTestCase):
agent.state.middle_context[middleware_key],
{reply_id: 370},
)
+
+ async def test_budget_exceeded_keeps_structured_output_tool(self) -> None:
+ """A spent budget still lets a structured reply end."""
+ model = ToolChoiceModel(calls_offered_tool=True)
+ agent = Agent(
+ name="test_agent",
+ system_prompt="you are helpful",
+ model=model,
+ toolkit=Toolkit(tools=[DummyTool()]),
+ middlewares=[ReplyBudgetControlMiddleware(token_budget=300)],
+ )
+
+ res = await agent.reply(
+ UserMsg("user", "Capital of France?"),
+ structured_schema=Answer,
+ )
+
+ self.assertEqual(res.structured_output, {"city": "Paris"})
+ self.assertListEqual(
+ model.tool_choices,
+ [
+ None,
+ ToolChoice(
+ mode="auto",
+ tools=["GenerateStructuredOutput"],
+ ),
+ ],
+ )
+
+ async def test_budget_exceeded_keeps_forced_structured_output(
+ self,
+ ) -> None:
+ """The agent's forced structured output call is kept."""
+ model = ToolChoiceModel(calls_offered_tool=False)
+ agent = Agent(
+ name="test_agent",
+ system_prompt="you are helpful",
+ model=model,
+ toolkit=Toolkit(tools=[DummyTool()]),
+ middlewares=[ReplyBudgetControlMiddleware(token_budget=300)],
+ react_config=ReActConfig(max_iters=3),
+ )
+
+ res = await agent.reply(
+ UserMsg("user", "Capital of France?"),
+ structured_schema=Answer,
+ )
+
+ self.assertEqual(res.structured_output, {"city": "Paris"})
+ offered = ToolChoice(mode="auto", tools=["GenerateStructuredOutput"])
+ self.assertListEqual(
+ model.tool_choices,
+ [
+ None,
+ offered,
+ offered,
+ ToolChoice(
+ mode="GenerateStructuredOutput",
+ tools=["GenerateStructuredOutput"],
+ ),
+ ],
+ )