Skip to content

Build and run agents you can see, understand and trust.

fix(middleware): release queued output after tool offload (#3887)

qkycir22c91abc94bc
diff --git a/src/agentscope/app/middleware/_tool_offload_middleware.py b/src/agentscope/app/middleware/_tool_offload_middleware.py
index 2bc1aedb0..5811ccaa7 100644
--- a/src/agentscope/app/middleware/_tool_offload_middleware.py
+++ b/src/agentscope/app/middleware/_tool_offload_middleware.py
@@ -228,7 +228,7 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
# ----------------------------------------------------------------
# Wrap next_handler in a Task, draining output into a Queue
# ----------------------------------------------------------------
- queue: asyncio.Queue = asyncio.Queue()
+ queue: asyncio.Queue | None = asyncio.Queue()
async def _drain_to_queue() -> ToolResponse | None:
"""Drain output and return the terminal tool outcome."""
@@ -237,16 +237,20 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
async for item in next_handler(**input_kwargs):
if isinstance(item, ToolResponse):
response = item
- await queue.put(item)
+ if queue is not None:
+ queue.put_nowait(item)
+ del item
except Exception as exc: # pylint: disable=broad-except
- await queue.put(exc)
+ if queue is not None:
+ queue.put_nowait(exc)
response = ToolResponse(
id=tool_call.id,
state=ToolResultState.ERROR,
content=[TextBlock(text=str(exc))],
)
finally:
- await queue.put(_QUEUE_SENTINEL)
+ if queue is not None:
+ queue.put_nowait(_QUEUE_SENTINEL)
return response
execution_started_at = time.time()
@@ -294,6 +298,14 @@ class ToolOffloadMiddleware(MiddlewareBase): # pylint: disable=abstract-method
drain_task.cancel()
return
+ # The deliverer awaits the final response directly. Drop the queue
+ # and foreground references before yielding: nobody consumes further
+ # stream events, and a bounded queue here would deadlock the producer.
+ # No await separates this transition from the producer's queue check.
+ queue = None
+ pre_collected.clear()
+ item = None
+
# ----------------------------------------------------------------
# Timeout path: spawn deliverer, register drain_task, yield synthetic
# ----------------------------------------------------------------
diff --git a/tests/tool_offload_middleware_test.py b/tests/tool_offload_middleware_test.py
index 7854434db..3a0aa2242 100644
--- a/tests/tool_offload_middleware_test.py
+++ b/tests/tool_offload_middleware_test.py
@@ -2,7 +2,9 @@
# pylint: disable=protected-access
"""Unit tests for ToolOffloadMiddleware."""
import asyncio
+import gc
import json
+import weakref
from typing import Any, AsyncGenerator
from unittest.async_case import IsolatedAsyncioTestCase
from unittest.mock import AsyncMock, MagicMock, patch
@@ -235,6 +237,101 @@ class ToolOffloadMiddlewareTest(IsolatedAsyncioTestCase):
# No background tasks registered
self.assertEqual(len(self.bg_manager.tasks), 0)
+ async def test_offloaded_stream_releases_intermediate_chunks(self) -> None:
+ """Detaching the foreground releases old and future stream events."""
+ for outcome in ("success", "error", "cancel"):
+ with self.subTest(outcome=outcome):
+ await self._check_stream_retention(outcome)
+
+ async def _check_stream_retention(self, outcome: str) -> None:
+ """Run one isolated producer/consumer handoff."""
+ agent, middleware = self._make_agent(
+ Toolkit(tools=[SlowTool()]),
+ timeout_secs=0.01,
+ )
+ start = asyncio.Event()
+ streamed = asyncio.Event()
+ finish = asyncio.Event()
+ delivered = asyncio.Event()
+ refs: list[weakref.ReferenceType] = []
+
+ async def handler(**_kwargs: Any) -> AsyncGenerator:
+ """Emit a foreground chunk and a background burst."""
+ item = ToolChunk(content=[TextBlock(text="foreground")])
+ refs.append(weakref.ref(item))
+ yield item
+ del item
+ await start.wait()
+ for index in range(32):
+ item = ToolChunk(
+ content=[
+ TextBlock(
+ text=f"{index}:" + "x" * 65536,
+ ),
+ ],
+ )
+ refs.append(weakref.ref(item))
+ yield item
+ del item
+ streamed.set()
+ await finish.wait()
+ if outcome == "error":
+ raise ValueError("stream failed")
+ yield ToolResponse(
+ id="stream-retention",
+ content=[TextBlock(text="done")],
+ )
+
+ async def push(key: str, *_args: Any, **_kwargs: Any) -> None:
+ """Observe real notification delivery without sleeps."""
+ if key == MessageBusKeys.inbox(agent.state.session_id):
+ delivered.set()
+
+ middleware._message_bus.queue_push.side_effect = push
+ stream = middleware.on_acting(
+ agent,
+ {
+ "tool_call": ToolCallBlock(
+ id="stream-retention",
+ name="slow_tool",
+ input="{}",
+ ),
+ },
+ handler,
+ )
+ await asyncio.wait_for(anext(stream), 2)
+ task = next(iter(self.bg_manager.tasks.values())).asyncio_task
+ try:
+ # Keep on_acting suspended at its synthetic response too.
+ start.set()
+ await asyncio.wait_for(streamed.wait(), 2)
+ gc.collect()
+ self.assertEqual(sum(ref() is not None for ref in refs), 0)
+ if outcome == "cancel":
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ self.assertFalse(delivered.is_set())
+ else:
+ finish.set()
+ await asyncio.wait_for(delivered.wait(), 2)
+ push_calls = middleware._message_bus.queue_push.call_args_list
+ calls = [
+ call
+ for call in push_calls
+ if call.args[0]
+ == MessageBusKeys.inbox(agent.state.session_id)
+ ]
+ self.assertEqual(len(calls), 1)
+ self.assertIn(
+ "stream failed" if outcome == "error" else "done",
+ str(calls[0]),
+ )
+ finally:
+ start.set()
+ finish.set()
+ await asyncio.gather(task, return_exceptions=True)
+ await stream.aclose()
+
async def test_slow_tool_offloaded_to_background(self) -> None:
"""A tool that exceeds timeout returns a synthetic result."""