Skip to content

The agent that grows with you

fix(gateway): let a serial reader return once a busy burst input is held

David Dudok de Witde526521aba6
diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py
index 73270e9bd4a..b1ffb8dd522 100644
--- a/gateway/platforms/base.py
+++ b/gateway/platforms/base.py
@@ -4019,10 +4019,12 @@ class BasePlatformAdapter(ABC):
logger.warning("Dropping internally routed event: expected session=%s derived=%s",
expected_session_key, session_key)
return
+ from gateway.session_ingress_burst import burst_gate
+ behind_burst = await burst_gate(self, event, session_key) # a serial reader's command waits too
# On-entry self-heal: clear a guard whose owner task already exited.
if session_key in self._active_sessions:
self._heal_stale_session_lock(session_key)
- if session_key in self._active_sessions or self._shared_burst_pending(event, session_key):
+ if session_key in self._active_sessions or behind_burst:
await self._handle_message_while_active(event, session_key)
return
# Guard installed synchronously BEFORE the task spawns so a second message can't race in.
@@ -4033,10 +4035,6 @@ class BasePlatformAdapter(ABC):
runner = getattr(self._message_handler, '__self__', None)
return getattr(runner, 'session_authority', None) is not None and not event.get_command()
- def _shared_burst_pending(self, event: MessageEvent, session_key: str) -> bool:
- from gateway.session_ingress_burst import holds_busy_burst
- return self._shared_runtime_owns_input(event) and holds_busy_burst(self, session_key)
-
def _pending_recovery_owners(self, event: MessageEvent, session_key: str) -> list:
"""Adapters whose pending slot may hold *event* after busy admission: this one, plus the
runner's current delivery adapter for its source when that replacement has no live guard."""
diff --git a/gateway/session_ingress_burst.py b/gateway/session_ingress_burst.py
index 96ff54ede22..3a89932aaaa 100644
--- a/gateway/session_ingress_burst.py
+++ b/gateway/session_ingress_burst.py
@@ -5,15 +5,23 @@ Main merged a busy photo burst into the pending slot and debounced queue-mode te
turn. The authority FIFO admits each input as it arrives, so the burst is held here for main's
bounded window first, then committed once through ``dispatch_shared_busy``. Every absorbed caller
returns only after that commit, with its acceptance copied from it (committed before ACK), and
-input that arrives later never commits before the burst.
+input that arrives later never commits before the burst. An adapter declaring
+``serial_receive_loop`` (Signal, IRC, ...) delivers a chat's next input only after this returns, and
+returning triggers no acknowledgement, so its caller returns once the input is held, as main
+returned once merged. Its reader still never runs ahead of what it returned on: a chat holds at
+most one uncommitted burst (the next one waits for that commit, main's backpressure), and a
+command first waits until every earlier input of that reader has committed (``burst_gate``).
"""
import asyncio
from dataclasses import dataclass, field
from functools import partial
+import logging
import time
from gateway.platforms.event import MessageEvent, MessageType
+logger = logging.getLogger(__name__)
+
@dataclass(eq=False)
class _Burst:
@@ -36,11 +44,34 @@ def _state(adapter, name):
return adapter.__dict__.setdefault(name, {})
+def _serial(adapter):
+ return getattr(adapter, 'serial_receive_loop', False)
+
+
def holds_busy_burst(adapter, session_key):
"""A burst is held or committing: later input for the route must queue behind it."""
return any(session_key in adapter.__dict__.get(name, ()) for name in ('_shared_bursts', '_shared_burst_commits'))
+async def burst_gate(adapter, event, session_key):
+ """``handle_message``'s entry check. True: a held or committing burst owns the route, so this
+ shared-runtime input must queue behind it. A command on a serial reader first waits for every
+ input that reader already returned on, as main's reader reached it only after their commits:
+ /stop, /new and /reset drop their own chat's held burst (main dropped the debounce buffer),
+ every other held burst commits now, and every pending commit lands before the command runs."""
+ if adapter._shared_runtime_owns_input(event):
+ return holds_busy_burst(adapter, session_key)
+ if (cmd := event.get_command()) and _serial(adapter):
+ from hermes_cli.commands import is_interrupt_then_dispatch
+ if is_interrupt_then_dispatch(cmd):
+ discard_busy_burst(adapter, session_key)
+ for key, burst in list(adapter.__dict__.get('_shared_bursts', {}).items()):
+ _flush(adapter, key, burst)
+ if pending := list(adapter.__dict__.get('_shared_burst_commits', {}).values()):
+ await asyncio.wait(pending)
+ return False
+
+
async def dispatch_busy_burst(adapter, event, session_key):
"""``dispatch_shared_busy`` for one busy input, merging a same-sender PHOTO+PHOTO or
queue-mode TEXT+TEXT burst first."""
@@ -54,9 +85,12 @@ async def dispatch_busy_burst(adapter, event, session_key):
burst.absorbed.append(event)
burst.last_ts = time.monotonic()
_arm(adapter, session_key, burst)
- return await _accepted(burst, event)
+ return await _accepted(adapter, burst, event)
await asyncio.shield(_flush(adapter, session_key, burst)) # a different sender or kind: in order
previous = _state(adapter, '_shared_burst_commits').get(session_key)
+ if kind is not None and previous is not None and not previous.done() and _serial(adapter):
+ await asyncio.wait({previous}) # one uncommitted burst per chat: the reader waits, as on main
+ return await dispatch_busy_burst(adapter, event, session_key) # a burst may have started meanwhile
if kind is None:
if previous is not None:
await asyncio.wait({previous})
@@ -64,10 +98,12 @@ async def dispatch_busy_burst(adapter, event, session_key):
now = time.monotonic()
burst = bursts[session_key] = _Burst(event, asyncio.get_running_loop().create_future(), now, now)
_arm(adapter, session_key, burst)
- return await _accepted(burst, event)
+ return await _accepted(adapter, burst, event)
-async def _accepted(burst, event):
+async def _accepted(adapter, burst, event):
+ if _serial(adapter):
+ return # waiting out the hold window here would keep the burst's next input unread
await asyncio.shield(burst.done) # raises the merged admission's own refusal
event._gateway_accepted = burst.head._gateway_accepted
@@ -109,7 +145,13 @@ async def _commit(adapter, session_key, burst, previous):
from gateway.session_ingress import dispatch_shared_busy
if previous is not None:
await asyncio.wait({previous})
- await dispatch_shared_busy(adapter, burst.head, session_key)
+ try:
+ await dispatch_shared_busy(adapter, burst.head, session_key)
+ except Exception:
+ if not _serial(adapter):
+ raise # every absorbed caller sees it (``_settle``)
+ # No caller waits on a serial reader's burst: log it as its receive loop would have.
+ logger.exception('[%s] Busy burst for %s was not queued', adapter.name, session_key)
def _settle(burst, task):
diff --git a/plugins/platforms/email/adapter.py b/plugins/platforms/email/adapter.py
index d1090b64544..49abaf5f6af 100644
--- a/plugins/platforms/email/adapter.py
+++ b/plugins/platforms/email/adapter.py
@@ -449,6 +449,7 @@ def _attach_file(msg: MIMEMultipart, path: Path, filename: str) -> None:
class EmailAdapter(BasePlatformAdapter):
"""Email gateway adapter using IMAP (receive) and SMTP (send)."""
+ serial_receive_loop = True # the IMAP poll loop awaits each message before dispatching the next
# One email carries the whole body, so cron delivery hands over the full payload untruncated.
splits_long_messages = True
diff --git a/plugins/platforms/feishu/adapter.py b/plugins/platforms/feishu/adapter.py
index 2aa863a69c3..46fbeae062c 100644
--- a/plugins/platforms/feishu/adapter.py
+++ b/plugins/platforms/feishu/adapter.py
@@ -1291,7 +1291,7 @@ class FeishuAdapter(BasePlatformAdapter):
supports_code_blocks = True # Feishu renders fenced code blocks
splits_long_messages = True # send() chunks via truncate_message(MAX_MESSAGE_LENGTH)
-
+ serial_receive_loop = True # a chat's next message waits on its lock (_handle_message_with_guards)
MAX_MESSAGE_LENGTH = 8000
CHAT_LOCK_MAX_SIZE: int = 1000 # distinct chat IDs kept in _chat_locks before LRU eviction
_SPLIT_THRESHOLD = 4000 # chunk near Feishu's ~4096-char client split → continuation almost certain
diff --git a/plugins/platforms/irc/adapter.py b/plugins/platforms/irc/adapter.py
index e237ab84c06..f807682ccfb 100644
--- a/plugins/platforms/irc/adapter.py
+++ b/plugins/platforms/irc/adapter.py
@@ -122,6 +122,8 @@ def _ssl_ctx(use_tls: bool) -> Optional[ssl.SSLContext]:
class IRCAdapter(BasePlatformAdapter):
"""Async IRC adapter implementing the BasePlatformAdapter interface."""
+ serial_receive_loop = True # the read loop awaits each PRIVMSG before reading the next line
+
def __init__(self, config, **kwargs):
super().__init__(config=config, platform=Platform("irc"))
extra = getattr(config, "extra", {}) or {}
diff --git a/plugins/platforms/matrix/adapter.py b/plugins/platforms/matrix/adapter.py
index 58f5c78fb97..b3f3c0eeda6 100644
--- a/plugins/platforms/matrix/adapter.py
+++ b/plugins/platforms/matrix/adapter.py
@@ -793,7 +793,7 @@ def ensure_matrix_deps() -> bool:
class MatrixAdapter(BasePlatformAdapter):
"""Gateway adapter for Matrix (any homeserver)."""
-
+ serial_receive_loop = True # _sync_loop reads the next sync only after this sync's handlers return
supports_code_blocks = True # Matrix renders fenced code blocks (HTML/markdown)
splits_long_messages = True # send() chunks via truncate_message(max_message_length)
typed_command_prefix = "!" # clients reserve typed "/" for local commands; "!command" always reaches Hermes
diff --git a/plugins/platforms/mattermost/adapter.py b/plugins/platforms/mattermost/adapter.py
index 611c13185f3..253d65dd6f8 100644
--- a/plugins/platforms/mattermost/adapter.py
+++ b/plugins/platforms/mattermost/adapter.py
@@ -105,6 +105,7 @@ class MattermostAdapter(BasePlatformAdapter):
"""Gateway adapter for Mattermost (self-hosted or cloud)."""
splits_long_messages = True # send() chunks via truncate_message(MAX_POST_LENGTH)
+ serial_receive_loop = True # the WebSocket loop awaits each event before reading the next
def __init__(self, config: PlatformConfig):
super().__init__(config, Platform.MATTERMOST)
diff --git a/plugins/platforms/ntfy/adapter.py b/plugins/platforms/ntfy/adapter.py
index df3ed2d513c..07c6ad248c4 100644
--- a/plugins/platforms/ntfy/adapter.py
+++ b/plugins/platforms/ntfy/adapter.py
@@ -119,6 +119,7 @@ class NtfyAdapter(BasePlatformAdapter):
"""ntfy adapter: HTTP-streaming subscription in, HTTP POST publish out."""
MAX_MESSAGE_LENGTH = MAX_MESSAGE_LENGTH
+ serial_receive_loop = True # the stream loop awaits each message before reading the next line
def __init__(self, config: PlatformConfig):
super().__init__(config=config, platform=Platform("ntfy"))
diff --git a/plugins/platforms/photon/adapter.py b/plugins/platforms/photon/adapter.py
index 9dfe3d98d06..5771aa45eae 100644
--- a/plugins/platforms/photon/adapter.py
+++ b/plugins/platforms/photon/adapter.py
@@ -561,6 +561,7 @@ class PhotonAdapter(BasePlatformAdapter):
MAX_MESSAGE_LENGTH = _MAX_MESSAGE_LENGTH
SUPPORTS_MESSAGE_EDITING = False # no edit API: streaming must not leave a stale cursor (▉)
+ serial_receive_loop = True # the /inbound NDJSON loop awaits each line before reading the next
def __init__(self, config: PlatformConfig):
super().__init__(config, Platform("photon"))
diff --git a/plugins/platforms/signal/adapter.py b/plugins/platforms/signal/adapter.py
index 93477c3378b..8f21db746e7 100644
--- a/plugins/platforms/signal/adapter.py
+++ b/plugins/platforms/signal/adapter.py
@@ -180,6 +180,7 @@ class SignalAdapter(BasePlatformAdapter):
platform = Platform.SIGNAL
MAX_MESSAGE_LENGTH = MAX_MESSAGE_LENGTH
splits_long_messages = True # send() chunks after markdown → Signal formatting conversion
+ serial_receive_loop = True # _sse_listener awaits each envelope before reading the next
# No real edit API; declaring it lets streaming suppress the cursor instead of a stale tofu square.
SUPPORTS_MESSAGE_EDITING = False
diff --git a/plugins/platforms/simplex/adapter.py b/plugins/platforms/simplex/adapter.py
index 9b7b88c2034..c1f50086a7c 100644
--- a/plugins/platforms/simplex/adapter.py
+++ b/plugins/platforms/simplex/adapter.py
@@ -116,6 +116,7 @@ class SimplexAdapter(BasePlatformAdapter):
"""SimpleX Chat adapter using the simplex-chat daemon WebSocket API."""
MAX_MESSAGE_LENGTH = MAX_MESSAGE_LENGTH
+ serial_receive_loop = True # the WebSocket loop awaits each event before reading the next
def __init__(self, config: PlatformConfig, **kwargs):
super().__init__(config=config, platform=Platform("simplex"))
diff --git a/plugins/platforms/wecom/adapter.py b/plugins/platforms/wecom/adapter.py
index 9f967241316..24c109f33b3 100644
--- a/plugins/platforms/wecom/adapter.py
+++ b/plugins/platforms/wecom/adapter.py
@@ -110,6 +110,7 @@ class WeComAdapter(WeComStreamMixin, WeComMediaMixin, ChatSendQueueMixin, OwnAcc
MAX_STREAM_CONTENT_LENGTH = MAX_STREAM_CONTENT_LENGTH
splits_long_messages = True # send() chunks via truncate_message(MAX_MESSAGE_LENGTH)
_SPLIT_THRESHOLD = 3900 # chunks near the 4000-char client split are almost certainly continued
+ serial_receive_loop = True # _read_events awaits each frame before reading the next
def __init__(self, config: PlatformConfig):
super().__init__(config, Platform.WECOM)
diff --git a/plugins/platforms/whatsapp/adapter.py b/plugins/platforms/whatsapp/adapter.py
index 2279c8ef268..f40f8e6375d 100644
--- a/plugins/platforms/whatsapp/adapter.py
+++ b/plugins/platforms/whatsapp/adapter.py
@@ -350,6 +350,7 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter):
_DEFAULT_BRIDGE_DIR = None # resolved in __init__
splits_long_messages = True # send() chunks via truncate_message()
+ serial_receive_loop = True # the bridge poll loop awaits each media message before the next
def __init__(self, config: PlatformConfig):
super().__init__(config, Platform.WHATSAPP)
diff --git a/tests/gateway/test_busy_burst_serial_reader.py b/tests/gateway/test_busy_burst_serial_reader.py
new file mode 100644
index 00000000000..4f5797cdc5b
--- /dev/null
+++ b/tests/gateway/test_busy_burst_serial_reader.py
@@ -0,0 +1,176 @@
+"""A serial receive loop's busy burst still merges, and the loop never waits out the hold window.
+
+Signal (``_sse_listener``) and IRC (``_receive_loop``) await ``handle_message`` before reading their
+next input. A busy input held for its burst window must therefore return at once; otherwise the
+burst's next input arrives only after the burst committed alone (main returned once the input was
+merged into the pending slot). The reader still never runs ahead of what it returned on: a command
+waits for those inputs to commit, and a chat holds at most one uncommitted burst. Real
+GatewayRunner, adapter, SessionAuthority and drain over SQLite; the running turn is a stand-in.
+"""
+import asyncio
+import threading
+from types import SimpleNamespace
+
+import pytest
+
+from gateway.config import GatewayConfig, PlatformConfig
+from gateway.platforms.base import SendResult
+from gateway.platforms.event import MessageEvent, MessageType
+from gateway.run import GatewayRunner
+from gateway.session_authority import initialize_session_authority
+from gateway.session_ingress_burst import holds_busy_burst
+from gateway.session_results import execution_result
+from hermes_state_runtime import list_session_admissions
+from tests.waiting import async_wait_until
+
+
+def _rows(authority):
+ return [row for sid in list(authority.sessions)
+ for row in list_session_admissions(authority.db, session_id=sid, pending_only=False)]
+
+
+async def _busy_owner(monkeypatch, adapter, source, **wiring):
+ """Wire ``adapter`` to a real runner and authority, then start turn 'A' and hold it running."""
+ runner = GatewayRunner(GatewayConfig())
+
+ async def send(chat_id, content, reply_to=None, metadata=None):
+ return SendResult(success=True, message_id='sent')
+ monkeypatch.setattr(adapter, 'send', send)
+ monkeypatch.setattr(adapter, 'send_typing', lambda *a, **k: asyncio.sleep(0))
+ adapter.gateway_runner = runner
+ runner.adapters = {adapter.platform: adapter}
+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='owner')
+ runner._wire_adapter_handlers(adapter, **wiring)
+ running, finish, executed = asyncio.Event(), asyncio.Event(), []
+
+ async def turn(event, source, quick_key, run_generation):
+ executed.append((event.text, len(event.media_urls)))
+ if event.text == 'A':
+ running.set()
+ await finish.wait()
+ execution_result.get()['result'] = {'final_response': 'ok', 'messages': []}
+ return 'ok'
+ monkeypatch.setattr(runner, '_handle_message_with_agent', turn)
+ await adapter.handle_message(MessageEvent(text='A', source=source, message_id='A'))
+ await asyncio.wait_for(running.wait(), 10)
+ key = adapter._event_session_key(MessageEvent(text='', source=source))
+
+ async def settle(turns):
+ finish.set()
+ await async_wait_until(lambda: not holds_busy_burst(adapter, key) and len(executed) >= turns and all(
+ row['status'] == 'terminal' for row in _rows(authority)), timeout=15, what='the follow-ups to settle')
+ await asyncio.to_thread(authority.db.close)
+ return executed[1:]
+ return SimpleNamespace(runner=runner, authority=authority, key=key, settle=settle)
+
+
+def _signal(monkeypatch):
+ from plugins.platforms.signal.adapter import SignalAdapter
+ monkeypatch.setenv('SIGNAL_ALLOWED_USERS', '+15550001')
+ adapter = SignalAdapter(PlatformConfig(enabled=True, extra={'account': '+15559999'}))
+ return adapter, adapter.build_source(chat_id='+15550001', chat_type='dm', user_id='+15550001')
+
+
+def _photo(source, tmp_path, n):
+ path = tmp_path / f'p{n}.png'
+ path.write_bytes(b'\x89PNG\r\n\x1a\n' + bytes(24) + bytes([n]))
+ return MessageEvent(text='', message_type=MessageType.PHOTO, source=source, message_id=f'P{n}',
+ media_urls=[str(path)], media_types=['image/png'])
+
+
+def _hold_capture(monkeypatch):
+ """The real media capture, held on an Event: a slow disk under the burst's commit."""
+ import gateway.session_envelope as envelope
+ entered, release, real = threading.Event(), threading.Event(), envelope.capture_native_media
+
+ def held(paths):
+ entered.set()
+ release.wait(10)
+ return real(paths)
+ monkeypatch.setattr(envelope, 'capture_native_media', held)
+ return entered, release
+
+
+async def test_signal_busy_photos_merge_without_holding_the_reader(monkeypatch, tmp_path):
+ adapter, source = _signal(monkeypatch)
+ owner = await _busy_owner(monkeypatch, adapter, source)
+ for n in (1, 2): # as _sse_listener does: the next envelope only after handle_message returned
+ await adapter.handle_message(_photo(source, tmp_path, n))
+ assert owner.key in adapter._shared_bursts, 'the reader waited out the hold window'
+ assert await owner.settle(2) == [('', 2)]
+
+
+async def test_irc_split_text_merges_through_the_receive_loop(monkeypatch):
+ from gateway import platform_registry
+ from gateway.platform_registry import PlatformEntry, PlatformRegistry
+ from tests.gateway._plugin_adapter_loader import load_plugin_adapter
+ monkeypatch.setenv('IRC_ALLOWED_USERS', 'sender')
+ module = load_plugin_adapter('irc')
+ registry = PlatformRegistry()
+ monkeypatch.setattr(platform_registry, 'platform_registry', registry)
+ module.register(SimpleNamespace(register_platform=lambda **kw: registry.register(PlatformEntry(**kw))))
+ adapter = module.IRCAdapter(PlatformConfig(enabled=True, typing_indicator=False, extra={
+ 'server': 'irc.example.invalid', 'nickname': 'testbot', 'allowed_users': ['sender']}))
+ source = adapter.build_source(chat_id='sender', chat_name='sender', chat_type='dm',
+ user_id='sender', user_name='sender')
+ owner = await _busy_owner(monkeypatch, adapter, source, busy_text_mode='queue')
+ adapter._reader = asyncio.StreamReader()
+ adapter._reader.feed_data(b':sender!u@h PRIVMSG testbot :first half\r\n'
+ b':sender!u@h PRIVMSG testbot :second half\r\n')
+ adapter._reader.feed_eof()
+ await asyncio.wait_for(adapter._receive_loop(), 10)
+ assert await owner.settle(2) == [('first half\nsecond half', 0)]
+
+
+async def test_restart_after_a_held_text_waits_for_its_commit(monkeypatch):
+ """A text, then an authorized /restart inside the burst window: the text is queued before
+ admission closes, so the next owner runs it."""
+ adapter, source = _signal(monkeypatch)
+ owner = await _busy_owner(monkeypatch, adapter, source, busy_text_mode='queue')
+ monkeypatch.setattr(owner.runner, 'request_restart', lambda **kw: setattr(owner.runner, '_draining', True))
+ await adapter.handle_message(MessageEvent(text='B', source=source, message_id='B'))
+ await adapter.handle_message(MessageEvent(text='/restart', source=source, message_id='R'))
+ await async_wait_until(lambda: not holds_busy_burst(adapter, owner.key), timeout=5, what='the burst to land')
+ assert [row['request_id'] for row in _rows(owner.authority)] == ['A', 'B']
+ owner.runner._draining = False # the stand-in for the next owner
+ assert await owner.settle(2) == [('B', 0)]
+
+
+async def test_new_waits_for_a_burst_still_committing(monkeypatch, tmp_path):
+ """A photo still capturing when /new arrives commits first, so /new cancels it with the old
+ conversation instead of it running in the new one."""
+ adapter, source = _signal(monkeypatch)
+ entered, release = _hold_capture(monkeypatch)
+ owner = await _busy_owner(monkeypatch, adapter, source)
+ try:
+ await adapter.handle_message(_photo(source, tmp_path, 1))
+ assert await asyncio.to_thread(entered.wait, 5)
+ new = asyncio.create_task(adapter.handle_message(MessageEvent(text='/new', source=source, message_id='N')))
+ await asyncio.wait({new}, timeout=0.5)
+ finally:
+ release.set()
+ await asyncio.wait_for(new, 10)
+ assert await owner.settle(1) == []
+
+
+async def test_next_burst_waits_while_the_chats_last_one_commits(monkeypatch, tmp_path):
+ """The chat's first burst is still committing: the reader waits before holding another one,
+ as main's reader waited at the first commit, so uncommitted input stays bounded."""
+ adapter, source = _signal(monkeypatch)
+ entered, release = _hold_capture(monkeypatch)
+ owner = await _busy_owner(monkeypatch, adapter, source)
+ try:
+ await adapter.handle_message(_photo(source, tmp_path, 1))
+ assert await asyncio.to_thread(entered.wait, 5)
+ second = asyncio.create_task(adapter.handle_message(_photo(source, tmp_path, 2)))
+ done, _ = await asyncio.wait({second}, timeout=0.5)
+ assert not done, 'the reader went on while the chat still had a burst committing'
+ finally:
+ release.set()
+ await asyncio.wait_for(second, 10)
+ assert await owner.settle(3) == [('', 1), ('', 1)]