Skip to content

Build and run agents you can see, understand and trust.

fix(app): drain the wake-up queue atomically (#2476)

DavdGao215289d0b25e
diff --git a/pyproject.toml b/pyproject.toml
index 4c0327202..e9ab4da2d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -178,7 +178,8 @@ dev = [
"pytest-forked",
"myst_parser",
"matplotlib",
- "fakeredis",
+ # ``[lua]`` pulls in lupa so EVAL-based code paths are testable.
+ "fakeredis[lua]",
# SQL backend tests — SQLite via aiosqlite so no server is needed.
"aiosqlite",
# S3-backend testing — moto[server] spins up an in-process S3
diff --git a/src/agentscope/app/message_bus/_base.py b/src/agentscope/app/message_bus/_base.py
index aaf382d9f..5ed20aa5a 100644
--- a/src/agentscope/app/message_bus/_base.py
+++ b/src/agentscope/app/message_bus/_base.py
@@ -89,7 +89,7 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methods
"""Release underlying transport resources. Default is a no-op."""
# ------------------------------------------------------------------
- # Mode A — drain queue (single consumer, ack-on-read)
+ # Mode A — drain queue (competing consumers, ack-on-read)
# ------------------------------------------------------------------
@abstractmethod
@@ -102,10 +102,10 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methods
) -> str:
"""Append ``payload`` to the drain queue at ``key``.
- Drain queues are single-consumer, ack-on-read: a subsequent
- :meth:`queue_drain` returns each entry exactly once and then
- deletes it. ``ttl_secs`` bounds the queue's lifetime so a key
- whose consumer disappears does not accumulate entries forever.
+ Drain queues are ack-on-read: :meth:`queue_drain` returns each
+ entry to exactly one caller and then deletes it. ``ttl_secs``
+ bounds the queue's lifetime so a key whose consumer disappears
+ does not accumulate entries forever.
Args:
key (`str`):
@@ -135,10 +135,11 @@ class MessageBus(ABC): # pylint: disable=too-many-public-methods
) -> list[tuple[str, dict]]:
"""Drain up to ``max_count`` entries from the queue at ``key``.
- Returned entries are removed from the queue in the same
- operation, so a subsequent call returns only entries that
- arrived after this one. Safe under the single-consumer-per-key
- invariant.
+ Reading and removing are one atomic operation, so concurrent
+ callers on the same key get disjoint entries and each entry is
+ delivered once. Delivery is at-most-once: an entry is gone from
+ the queue as soon as it is returned, so a caller that dies
+ before acting on it loses it.
Args:
key (`str`):
diff --git a/src/agentscope/app/message_bus/_redis_message_bus.py b/src/agentscope/app/message_bus/_redis_message_bus.py
index af07e13ef..b5f824922 100644
--- a/src/agentscope/app/message_bus/_redis_message_bus.py
+++ b/src/agentscope/app/message_bus/_redis_message_bus.py
@@ -15,6 +15,17 @@ else:
ConnectionPool = Any
Redis = Any
+# Reads and removes a batch in one atomic step. Doing so in two
+# round-trips lets competing consumers on the same key both read an
+# entry before either deletes it, and each dispatch it (#1868).
+_QUEUE_DRAIN_LUA = """
+local entries = redis.call('XRANGE', KEYS[1], '-', '+', 'COUNT', ARGV[1])
+for _, entry in ipairs(entries) do
+ redis.call('XDEL', KEYS[1], entry[1])
+end
+return entries
+"""
+
class RedisMessageBus(MessageBus):
"""Redis-backed implementation of :class:`MessageBus`.
@@ -23,9 +34,9 @@ class RedisMessageBus(MessageBus):
- **Mode A (drain queue)** uses a Redis Stream per key. ``XADD``
appends a payload whose single field ``payload`` carries the
- JSON-serialised dict. ``queue_drain`` performs ``XRANGE`` followed
- by per-id ``XDEL`` so the read is destructive and idempotent
- under the single-consumer-per-key invariant. ``ttl_secs`` is
+ JSON-serialised dict. ``queue_drain`` runs ``XRANGE`` and ``XDEL``
+ in one Lua script, so an entry is handed to exactly one consumer
+ however many drain the key concurrently. ``ttl_secs`` is
enforced via ``EXPIRE`` after each push (sliding TTL).
- **Mode C (replay log)** also uses a Redis Stream, but never
``XDEL``s on read. Trimming happens via ``XADD … MAXLEN ~N``
@@ -232,10 +243,10 @@ class RedisMessageBus(MessageBus):
) -> list[tuple[str, dict]]:
"""Drain up to ``max_count`` entries from the queue at ``key``.
- Implementation: ``XRANGE`` followed by ``XDEL`` of the
- returned ids, so the operation is destructive in a single
- round-trip pair. Entries that arrive between ``XRANGE`` and
- ``XDEL`` are not affected.
+ Implementation: ``XRANGE`` and ``XDEL`` of the returned ids run
+ as one Lua script, so nothing else observes the entries between
+ the read and their removal. Entries that arrive mid-script are
+ not affected.
Args:
key (`str`):
@@ -248,22 +259,17 @@ class RedisMessageBus(MessageBus):
``(entry_id, payload)`` pairs in arrival order. Empty
list when the queue is empty or absent.
"""
- entries = await self._client.xrange(key, count=max_count)
- if not entries:
- return []
+ entries = await self._client.eval(_QUEUE_DRAIN_LUA, 1, key, max_count)
results: list[tuple[str, dict]] = []
- ids_to_delete: list[str] = []
for entry_id, fields in entries:
- ids_to_delete.append(entry_id)
- raw = fields.get("payload")
+ # Lua flattens an entry's field map into ``[name, value]``;
+ # ``queue_push`` only ever writes the ``payload`` field.
+ raw = dict(zip(fields[::2], fields[1::2])).get("payload")
if raw is None:
continue
results.append((entry_id, json.loads(raw)))
- if ids_to_delete:
- await self._client.xdel(key, *ids_to_delete)
-
return results
async def queue_delete(self, key: str) -> None:
diff --git a/tests/service_message_bus_test.py b/tests/service_message_bus_test.py
index bbf6455dc..86a78da47 100644
--- a/tests/service_message_bus_test.py
+++ b/tests/service_message_bus_test.py
@@ -10,9 +10,11 @@ that are layered on top.
"""
import asyncio
from contextlib import AsyncExitStack
+from typing import Any
from unittest import IsolatedAsyncioTestCase
import fakeredis.aioredis
+from utils import AnyString
from agentscope.app.message_bus import MessageBus, RedisMessageBus
@@ -83,6 +85,29 @@ class TestQueuePrimitive(IsolatedAsyncioTestCase):
self.assertEqual([p["i"] for _id, p in first], [0, 1, 2])
self.assertEqual([p["i"] for _id, p in rest], [3, 4])
+ async def test_drain_reads_and_deletes_in_one_command(self) -> None:
+ """Reading and deleting as two commands lets a competing
+ consumer land in between, read the same entry and dispatch it a
+ second time (#1868)."""
+ await self.bus.queue_push("k", {"x": 1})
+
+ execute_command = self.fr.execute_command
+ issued: list[str] = []
+
+ async def recording_execute_command(*args: Any, **kwargs: Any) -> Any:
+ issued.append(args[0])
+ return await execute_command(*args, **kwargs)
+
+ self.fr.execute_command = recording_execute_command
+
+ self.assertListEqual(
+ await self.bus.queue_drain("k"),
+ [(AnyString(), {"x": 1})],
+ )
+ # One command, whatever it is — the contract is that the read
+ # and the delete cannot be observed apart.
+ self.assertEqual(len(issued), 1)
+
class TestLogPrimitive(IsolatedAsyncioTestCase):
"""Mode C — replay log: append / read with cursor / trim."""