Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
feat(middleware): add BudgetControlMiddleware for token budget enforcement (#1738)
Parman Mohammadalizadehc13c3effcb56
diff --git a/src/agentscope/middleware/__init__.py b/src/agentscope/middleware/__init__.pyindex 171d87591..f3c51e568 100644--- a/src/agentscope/middleware/__init__.py+++ b/src/agentscope/middleware/__init__.py@@ -2,6 +2,7 @@"""Middleware system for AgentScope agents."""from ._base import MiddlewareBase+from ._budget import ReplyBudgetControlMiddlewarefrom ._longterm_memory import Mem0Middlewarefrom ._tracing import TracingMiddlewarefrom ._tts_middleware import TTSMiddleware@@ -10,5 +11,6 @@ __all__ = ["Mem0Middleware","MiddlewareBase","TracingMiddleware",+ "ReplyBudgetControlMiddleware","TTSMiddleware",]diff --git a/src/agentscope/middleware/_base.py b/src/agentscope/middleware/_base.pyindex 5d945b593..1b815dc86 100644--- a/src/agentscope/middleware/_base.py+++ b/src/agentscope/middleware/_base.py@@ -238,3 +238,12 @@ class MiddlewareBase: # pylint: disable=unused-argumentA list of tools provided by this middleware."""return []++ async def get_middleware_key(self) -> str:+ """Get the unique key for this middleware, used to save middleware+ states in `AgentState` instances.++ Optionally, middleware classes can override this method to+ provide their own unique key.+ """+ return self.__class__.__name__diff --git a/src/agentscope/middleware/_budget.py b/src/agentscope/middleware/_budget.pynew file mode 100644index 000000000..53c56b33a--- /dev/null+++ b/src/agentscope/middleware/_budget.py@@ -0,0 +1,207 @@+# -*- coding: utf-8 -*-+"""Budget control middleware for AgentScope agents."""+from typing import AsyncGenerator, Callable, TYPE_CHECKING++from ..event import ModelCallEndEvent, ReplyStartEvent, ReplyEndEvent+from ..message import AssistantMsg, HintBlock+from ..tool import ToolChoice+from ._base import MiddlewareBase++if TYPE_CHECKING:+ from ..agent import Agent++_DEFAULT_HINT_MESSAGE = (+ "<system-reminder>You have reached the maximum token budget set by the "+ "user. Now you MUST wrap up immediately and provide a final "+ "concluding response without invoking any tools."+ "</system-reminder>"+)+++class ReplyBudgetControlMiddleware(MiddlewareBase):+ """Middleware that enforces a weighted token budget per reply.++ Tracks cumulative weighted token usage across all reasoning steps within+ a single reply. The weighted cost is computed as::++ cost = input_token_weight * input_tokens + output_token_weight * \+ output_tokens++ Once the accumulated cost reaches ``token_budget``, a hint message is+ injected into the agent's context before the next reasoning step, and+ ``tool_choice`` is forced to ``"none"`` so the agent wraps up without+ invoking any further tools.++ Budget state is stored in+ :attr:`~agentscope.agent.AgentState.middle_context`+ keyed by the middleware key, so it persists across human-in-the-loop (HITL)+ interruptions and resumptions. State is automatically cleaned up when the+ reply ends via a :class:`~agentscope.event.ReplyEndEvent`.++ .. note::+ The middleware is stateless on the instance itself — all runtime state+ lives in ``agent.state.middle_context``. This means the same middleware+ instance can safely be shared across multiple agents.++ Example::++ from agentscope.middleware import BudgetControlMiddleware++ agent = Agent(+ ...,+ middlewares=[+ BudgetControlMiddleware(+ token_budget=10000,+ input_token_weight=1.0,+ output_token_weight=2.0,+ )+ ],+ )++ """++ def __init__(+ self,+ token_budget: float,+ input_token_weight: float = 1,+ output_token_weight: float = 1,+ hint_message: str = _DEFAULT_HINT_MESSAGE,+ ) -> None:+ """Initialize the budget control middleware.++ Args:+ token_budget (`float`):+ Maximum weighted token cost allowed per reply. The cost for+ each model call is computed as+ ``input_token_weight * input_tokens + output_token_weight *+ output_tokens``. Once the accumulated cost reaches this+ threshold, the agent is instructed to wrap up without calling+ any more tools.+ input_token_weight (`float`, optional):+ Multiplier applied to input tokens when computing the+ weighted cost. Defaults to ``1``.+ output_token_weight (`float`, optional):+ Multiplier applied to output tokens when computing the+ weighted cost. Defaults to ``1``. Set this higher than+ ``input_token_weight`` to reflect that output tokens are+ typically more expensive.+ hint_message (`str`, optional):+ The message injected into the agent's context when the budget+ is exceeded. Defaults to a built-in wrap-up prompt.+ """+ self.token_budget = token_budget+ self.input_token_weight = input_token_weight+ self.output_token_weight = output_token_weight++ self.hint_message = hint_message++ async def on_reply(+ self,+ agent: "Agent",+ input_kwargs: dict,+ next_handler: Callable[..., AsyncGenerator],+ ) -> AsyncGenerator:+ """Manage per-reply budget state in ``agent.state.middle_context``.++ Initializes the weighted cost counter for the reply on+ :class:`~agentscope.event.ReplyStartEvent`, accumulates cost on each+ :class:`~agentscope.event.ModelCallEndEvent`, and removes the entry on+ :class:`~agentscope.event.ReplyEndEvent`.++ Args:+ agent (`Agent`):+ The agent instance executing this middleware.+ input_kwargs (`dict`):+ Reply input kwargs (passed through unchanged).+ next_handler (`Callable[..., AsyncGenerator]`):+ Callable that executes the next middleware or ``_reply``.++ Yields:+ Events from the reply process.+ """++ middleware_key = await self.get_middleware_key()++ async for event in next_handler(**input_kwargs):+ if isinstance(event, ReplyStartEvent):+ # Initialize the token counting number+ if middleware_key not in agent.state.middle_context:+ agent.state.middle_context[middleware_key] = {}+ agent.state.middle_context[middleware_key][event.reply_id] = 0++ elif isinstance(event, ReplyEndEvent):+ # Clean up the token counting number+ agent.state.middle_context[middleware_key].pop(+ event.reply_id,+ None,+ )++ elif isinstance(event, ModelCallEndEvent):+ # Update the used tokens+ if middleware_key not in agent.state.middle_context:+ agent.state.middle_context[middleware_key] = {}+ agent.state.middle_context[middleware_key][event.reply_id] += (+ self.input_token_weight * event.input_tokens+ + self.output_token_weight * event.output_tokens+ )++ yield event++ async def on_reasoning(+ self,+ agent: "Agent",+ input_kwargs: dict,+ next_handler: Callable[..., AsyncGenerator],+ ) -> AsyncGenerator:+ """Intercept each reasoning step to enforce the token budget.++ Before forwarding to the next handler, reads the accumulated weighted+ cost for the current reply from ``agent.state.middle_context``. If the+ budget is exhausted, appends a+ :class:`~agentscope.message.HintBlock` to the last assistant message+ in context (or creates a new+ :class:`~agentscope.message.AssistantMsg`) and overrides+ ``tool_choice`` to ``ToolChoice(mode="none")``.++ Args:+ agent (`Agent`):+ The agent instance executing this middleware.+ input_kwargs (`dict`):+ Dictionary containing ``tool_choice`` and other reasoning+ kwargs forwarded to the next handler.+ next_handler (`Callable[..., AsyncGenerator]`):+ Callable that executes the next middleware or+ ``_reasoning_impl``.++ Yields:+ Events from the reasoning process.+ """+ reply_id = agent.state.reply_id+ middleware_key = await self.get_middleware_key()+ used = agent.state.middle_context.get(+ middleware_key,+ {},+ ).get(reply_id, 0)++ # Insert hint block if exceeded budget+ if used >= self.token_budget:+ hint_block = HintBlock(hint=self.hint_message)+ if (+ len(agent.state.context) > 0+ and agent.state.context[-1].role == "assistant"+ and agent.state.context[-1].name == agent.name+ ):+ agent.state.context[-1].content.append(hint_block)++ else:+ agent.state.context.append(+ AssistantMsg(+ id=agent.state.reply_id,+ name=agent.name,+ content=[hint_block],+ ),+ )+ input_kwargs["tool_choice"] = ToolChoice(mode="none")++ async for event in next_handler(**input_kwargs):+ yield eventdiff --git a/src/agentscope/state/_state.py b/src/agentscope/state/_state.pyindex 44de5fa55..54185ad09 100644--- a/src/agentscope/state/_state.py+++ b/src/agentscope/state/_state.py@@ -1,6 +1,7 @@# -*- coding: utf-8 -*-"""The agent state class."""import uuid+from typing import Anyfrom pydantic import BaseModel, Field@@ -174,3 +175,10 @@ class AgentState(BaseModel):# =================================================================tasks_context: TaskContext = Field(default_factory=TaskContext)"""The task context that records the agent tasks."""++ # =================================================================+ # The middleware context+ # =================================================================+ middle_context: dict[str, Any] = Field(default_factory=dict)+ """The context that allow the middlewares to store/get data across+ different replies."""diff --git a/tests/builtin_glob_test.py b/tests/builtin_glob_test.pyindex 8a5d109a8..24da9ae5a 100644--- a/tests/builtin_glob_test.py+++ b/tests/builtin_glob_test.py@@ -4,6 +4,7 @@ import osimport tempfilefrom unittest.async_case import IsolatedAsyncioTestCase+from utils import AnyStringfrom agentscope.tool import Globfrom agentscope.permission import (PermissionContext,@@ -11,8 +12,6 @@ from agentscope.permission import (PermissionRule,)-from tests.utils import AnyString-class GlobToolTest(IsolatedAsyncioTestCase):"""The glob tool test case."""diff --git a/tests/middleware_budget_test.py b/tests/middleware_budget_test.pynew file mode 100644index 000000000..c6a8c5438--- /dev/null+++ b/tests/middleware_budget_test.py@@ -0,0 +1,476 @@+# -*- coding: utf-8 -*-+"""Unit tests for BudgetControlMiddleware."""+from typing import Any+from unittest.async_case import IsolatedAsyncioTestCase++from utils import MockModel+from agentscope.agent import Agent+from agentscope.message import UserMsg, TextBlock, ToolCallBlock, HintBlock+from agentscope.middleware import ReplyBudgetControlMiddleware+from agentscope.model import ChatResponse, ChatUsage+from agentscope.permission import (+ PermissionBehavior,+ PermissionContext,+ PermissionDecision,+)+from agentscope.event import UserConfirmResultEvent, ConfirmResult+from agentscope.tool import ToolBase, Toolkit, ToolChunk+++def _response(+ text: str,+ input_tokens: int,+ output_tokens: int,+) -> ChatResponse:+ """Build a non-streaming ChatResponse with usage."""+ return ChatResponse(+ content=[TextBlock(text=text)],+ is_last=True,+ usage=ChatUsage(+ input_tokens=input_tokens,+ output_tokens=output_tokens,+ time=0.0,+ ),+ )+++class DummyTool(ToolBase):+ """Minimal tool that always allows and returns a fixed result."""++ name: str = "dummy"+ description: str = "A dummy tool for testing"+ input_schema: dict[str, Any] = {"type": "object", "properties": {}}+ is_concurrency_safe: bool = True+ is_read_only: bool = True+ is_external_tool: bool = False+ is_mcp: bool = False++ async def check_permissions(+ self,+ tool_input: dict[str, Any],+ context: PermissionContext,+ ) -> PermissionDecision:+ """Always allow."""+ return PermissionDecision(+ behavior=PermissionBehavior.ALLOW,+ decision_reason="Dummy tool always allows",+ message="Dummy tool always allows",+ )++ async def __call__(self, **kwargs: Any) -> ToolChunk:+ """Return a fixed result."""+ return ToolChunk(content=[TextBlock(text="ok")])+++class ConfirmRequiredTool(ToolBase):+ """Minimal tool that always requires user confirmation before running."""++ name: str = "confirm_required"+ description: str = "A tool that requires user confirmation"+ input_schema: dict[str, Any] = {"type": "object", "properties": {}}+ is_concurrency_safe: bool = False+ is_read_only: bool = False+ is_external_tool: bool = False+ is_mcp: bool = False++ async def check_permissions(+ self,+ tool_input: dict[str, Any],+ context: PermissionContext,+ ) -> PermissionDecision:+ """Always require user confirmation."""+ return PermissionDecision(+ behavior=PermissionBehavior.ASK,+ decision_reason="Confirmation required",+ message="Confirmation required",+ )++ async def __call__(self, **kwargs: Any) -> ToolChunk:+ """Return a fixed result."""+ return ToolChunk(content=[TextBlock(text="confirmed result")])+++def _has_hint_block(msg: Any, hint_message: str) -> bool:+ """Return True if *msg* contains a HintBlock with *hint_message*."""+ content = getattr(msg, "content", None)+ if not isinstance(content, list):+ return False+ return any(+ isinstance(b, HintBlock) and hint_message in b.hint for b in content+ )+++class TestBudgetControlMiddleware(IsolatedAsyncioTestCase):+ """Test cases for BudgetControlMiddleware."""++ async def asyncSetUp(self) -> None:+ """Set up shared fixtures."""+ self.toolkit = Toolkit()++ async def test_under_budget_no_hint_injected(self) -> None:+ """When token usage stays below the budget, no hint is injected."""+ model = MockModel()+ model.set_responses(+ [_response("done", input_tokens=10, output_tokens=5)],+ )++ middleware = ReplyBudgetControlMiddleware(token_budget=1000)+ agent = Agent(+ name="test_agent",+ system_prompt="you are helpful",+ model=model,+ toolkit=self.toolkit,+ middlewares=[middleware],+ )++ context_before = len(agent.state.context)+ await agent.reply(UserMsg("user", "hello"))++ # No HintBlock should have been added to context+ hint_msgs = [+ m+ for m in agent.state.context[context_before:]+ if _has_hint_block(m, middleware.hint_message)+ ]+ self.assertEqual(len(hint_msgs), 0)++ async def test_budget_exceeded_injects_hint(self) -> None:+ """When the budget is exceeded, the hint block is injected.++ Uses token_budget=0 so the budget condition fires on the very first+ reasoning call (0 used >= 0 max).+ """+ model = MockModel()+ model.set_responses(+ [_response("wrap up", input_tokens=10, output_tokens=5)],+ )++ middleware = ReplyBudgetControlMiddleware(token_budget=0)+ agent = Agent(+ name="test_agent",+ system_prompt="you are helpful",+ model=model,+ toolkit=self.toolkit,+ middlewares=[middleware],+ )++ context_before = len(agent.state.context)+ await agent.reply(UserMsg("user", "hello"))++ hint_msgs = [+ m+ for m in agent.state.context[context_before:]+ if _has_hint_block(m, middleware.hint_message)+ ]+ self.assertGreater(len(hint_msgs), 0)++ async def test_budget_exceeded_forces_tool_choice_none(self) -> None:+ """When budget is exceeded, tool_choice forwarded to model is none.++ Uses token_budget=0 so the override fires on the first reasoning call.+ """+ received_tool_choices: list = []++ class TrackingModel(MockModel):+ """Model that records tool_choice on every call."""++ async def _call_api(+ self,+ *args: Any,+ **kwargs: Any,+ ) -> ChatResponse:+ """Record tool_choice and delegate to mock."""+ received_tool_choices.append(kwargs.get("tool_choice"))+ return await super()._call_api(*args, **kwargs)++ model = TrackingModel()+ model.set_responses(+ [_response("wrap up", input_tokens=10, output_tokens=5)],+ )++ middleware = ReplyBudgetControlMiddleware(token_budget=0)+ agent = Agent(+ name="test_agent",+ system_prompt="you are helpful",+ model=model,+ toolkit=self.toolkit,+ middlewares=[middleware],+ )++ await agent.reply(UserMsg("user", "hello"))++ # At least one call must have received tool_choice with mode="none"+ self.assertTrue(+ any(+ getattr(tc, "mode", None) == "none"+ for tc in received_tool_choices+ if tc is not None+ ),+ )++ async def test_token_accumulation_across_steps(self) -> None:+ """Tokens accumulate across steps and trigger enforcement correctly.++ Step 1: tool call costs 200+100=300 tokens (token_budget=300 so+ step 2 sees used >= max and injects the hint).+ """+ toolkit = Toolkit(tools=[DummyTool()])++ model = MockModel()+ model.set_responses(+ [+ [+ ChatResponse(+ content=[+ ToolCallBlock(+ id="tc_1",+ name="dummy",+ input="{}",+ ),+ ],+ is_last=True,+ usage=ChatUsage(+ input_tokens=200,+ output_tokens=100,+ time=0.0,+ ),+ ),+ ],+ [+ ChatResponse(+ content=[TextBlock(text="done")],+ is_last=True,+ usage=ChatUsage(+ input_tokens=150,+ output_tokens=50,+ time=0.0,+ ),+ ),+ ],+ ],+ )++ middleware = ReplyBudgetControlMiddleware(token_budget=300)+ agent = Agent(+ name="test_agent",+ system_prompt="you are helpful",+ model=model,+ toolkit=toolkit,+ middlewares=[middleware],+ )++ context_before = len(agent.state.context)+ await agent.reply(UserMsg("user", "hello"))++ hint_msgs = [+ m+ for m in agent.state.context[context_before:]+ if _has_hint_block(m, middleware.hint_message)+ ]+ self.assertGreater(len(hint_msgs), 0)++ async def test_weighted_token_calculation(self) -> None:+ """output_token_weight scales output tokens in the budget calculation.++ With input_token_weight=1, output_token_weight=3, token_budget=200:+ - Step 1: 50 input * 1 + 50 output * 3 = 200 → budget hit exactly+ - Step 2: hint should be injected before the model call+ """+ toolkit = Toolkit(tools=[DummyTool()])++ model = MockModel()+ model.set_responses(+ [+ [+ ChatResponse(+ content=[+ ToolCallBlock(+ id="tc_1",+ name="dummy",+ input="{}",+ ),+ ],+ is_last=True,+ usage=ChatUsage(+ input_tokens=50,+ output_tokens=50,+ time=0.0,+ ),+ ),+ ],+ [+ ChatResponse(+ content=[TextBlock(text="done")],+ is_last=True,+ usage=ChatUsage(+ input_tokens=30,+ output_tokens=10,+ time=0.0,+ ),+ ),+ ],+ ],+ )++ # 50*1 + 50*3 = 200 == token_budget → step 2 triggers enforcement+ middleware = ReplyBudgetControlMiddleware(+ token_budget=200,+ input_token_weight=1,+ output_token_weight=3,+ )+ agent = Agent(+ name="test_agent",+ system_prompt="you are helpful",+ model=model,+ toolkit=toolkit,+ middlewares=[middleware],+ )++ context_before = len(agent.state.context)+ await agent.reply(UserMsg("user", "hello"))++ hint_msgs = [+ m+ for m in agent.state.context[context_before:]+ if _has_hint_block(m, middleware.hint_message)+ ]+ self.assertGreater(len(hint_msgs), 0)++ async def test_state_cleanup_after_reply(self) -> None:+ """middle_context entry for the reply is removed after reply ends."""+ model = MockModel()+ model.set_responses(+ [_response("done", input_tokens=10, output_tokens=5)],+ )++ middleware = ReplyBudgetControlMiddleware(token_budget=1000)+ agent = Agent(+ name="test_agent",+ system_prompt="you are helpful",+ model=model,+ toolkit=self.toolkit,+ middlewares=[middleware],+ )++ await agent.reply(UserMsg("user", "hello"))++ middleware_key = await middleware.get_middleware_key()+ bucket = agent.state.middle_context.get(middleware_key, {})+ # All per-reply entries must have been cleaned up+ self.assertEqual(len(bucket), 0)++ async def test_token_accumulation_persists_across_hitl(self) -> None:+ """Token accumulation in middle_context persists across HITL boundary.++ Scenario:+ - token_budget=300, both weights default to 1.+ - First reply_stream call: model call costs 200 input + 100 output+ = 300 tokens, then pauses at REQUIRE_USER_CONFIRM (no ReplyEndEvent+ is emitted). The 300-token count is stored in middle_context.+ - Second reply_stream call with UserConfirmResultEvent: the same+ reply_id resumes. on_reasoning reads 300 >= 300 from middle_context+ and injects the hint + forces tool_choice=none before the final+ model call, proving budget state survived the HITL round-trip.+ """+ tool_call_id = "tc_hitl"+ tool_input = "{}"+ toolkit = Toolkit(tools=[ConfirmRequiredTool()])++ model = MockModel()+ model.set_responses(+ [+ # Step 1: model produces a tool call that requires confirmation+ [+ ChatResponse(+ content=[+ ToolCallBlock(+ id=tool_call_id,+ name="confirm_required",+ input=tool_input,+ ),+ ],+ is_last=True,+ usage=ChatUsage(+ input_tokens=200,+ output_tokens=100,+ time=0.0,+ ),+ ),+ ],+ # Step 2 (after confirmation): final wrap-up text+ [+ ChatResponse(+ content=[TextBlock(text="wrap up")],+ is_last=True,+ usage=ChatUsage(+ input_tokens=50,+ output_tokens=20,+ time=0.0,+ ),+ ),+ ],+ ],+ )++ # 200*1 + 100*1 = 300 == token_budget → reasoning after resume+ # injects hint+ middleware = ReplyBudgetControlMiddleware(token_budget=300)+ agent = Agent(+ name="test_agent",+ system_prompt="you are helpful",+ model=model,+ toolkit=toolkit,+ middlewares=[middleware],+ )++ # --- First call: pauses at REQUIRE_USER_CONFIRM ---+ events = []+ async for event in agent.reply_stream(UserMsg("user", "hello")):+ events.append(event)++ event_types = [e.type for e in events]+ self.assertIn("REQUIRE_USER_CONFIRM", event_types)+ self.assertNotIn("REPLY_END", event_types)++ reply_id = agent.state.reply_id++ # Token count must be stored in middle_context (survived the pause)+ middleware_key = await middleware.get_middleware_key()+ stored = agent.state.middle_context.get(middleware_key, {})+ self.assertAlmostEqual(stored.get(reply_id, 0), 300.0)++ # --- Second call: resume with user confirmation ---+ user_confirm_event = UserConfirmResultEvent(+ reply_id=reply_id,+ confirm_results=[+ ConfirmResult(+ confirmed=True,+ tool_call=ToolCallBlock(+ id=tool_call_id,+ name="confirm_required",+ input=tool_input,+ ),+ ),+ ],+ )++ resume_events = []+ async for event in agent.reply_stream(inputs=user_confirm_event):+ resume_events.append(event)++ resume_event_types = [e.type for e in resume_events]+ self.assertIn("REPLY_END", resume_event_types)++ # Hint is appended to the existing assistant message (which was created+ # in the first call), so we search the full context rather than a+ # slice.+ hint_msgs = [+ m+ for m in agent.state.context+ if _has_hint_block(m, middleware.hint_message)+ ]+ self.assertGreater(len(hint_msgs), 0)++ # middle_context must be cleaned up after reply ends+ bucket = agent.state.middle_context.get(middleware_key, {})+ self.assertNotIn(reply_id, bucket)