Skip to content

The agent that grows with you

fix(gateway): /retry answers the authority's drain and queue-full refusals before rewriting

Tekniumfb322986a62e
diff --git a/gateway/session_ingress.py b/gateway/session_ingress.py
index 6a35f30fdb6..2a0e12cd142 100644
--- a/gateway/session_ingress.py
+++ b/gateway/session_ingress.py
@@ -23,6 +23,10 @@ _CONTENT_CLAUSES = frozenset({
'media_urls', 'media_types', 'media_text_inlined', 'media_types_length', 'media_text_inlined_length',
})
+# Nothing was committed (main's busy-queue cap); /retry answers it before rewriting (#139257).
+QUEUE_FULL_REPLY = ('⏳ Too many messages are already waiting in this conversation; '
+ 'send this again after the current reply.')
+
async def admit_message(authority, event):
accepted = busy_acceptance.get()
@@ -40,7 +44,7 @@ async def admit_message(authority, event):
if exc.reason != 'queue_full':
raise
# Nothing was committed (main's busy-queue cap): say so instead of a generic turn error.
- return '⏳ Too many messages are already waiting in this conversation; send this again after the current reply.'
+ return QUEUE_FULL_REPLY
if receipt.status == 'terminal':
return None
# Only the delivery waiter is process-local; execution reads the committed snapshot.
diff --git a/gateway/slash_commands_session.py b/gateway/slash_commands_session.py
index fab0d337a4e..06d88d88cd9 100644
--- a/gateway/slash_commands_session.py
+++ b/gateway/slash_commands_session.py
@@ -458,8 +458,35 @@ class GatewaySessionCommandsMixin:
return paused_notice
if getattr(self, "_external_drain_active", False):
return t("gateway.busy.draining_maintenance")
+ authority_reply = self._retry_authority_refusal(source)
+ if authority_reply is not None:
+ return authority_reply
return self._active_session_limit_message(self._session_key_for_source(source))
+ def _retry_authority_refusal(self, source: SessionSource) -> Optional[str]:
+ """The routed authority's own admission refusals for the re-send (#139257): a draining or
+ retiring runtime (``runtime_draining``) and a full FIFO (``queue_full``), the gates
+ ``admit_native`` applies. Same not-race-free caveat as the caller."""
+ from gateway.session_authorities import active_authority
+ authority = active_authority(self)
+ if authority is None:
+ return None
+ import hermes_state_runtime
+ from hermes_state_runtime import RuntimeStoreError
+ try:
+ authority._require_admission_open()
+ except RuntimeStoreError:
+ return t("gateway.busy.drain_rejected", action=self._status_action_gerund())
+ entry = self.session_store.lookup_by_session_key(self._session_key_for_source(source))
+ if entry is None:
+ return None
+ rows = hermes_state_runtime.list_session_admissions(
+ authority.db, session_id=authority.logical_owner(entry.session_id))
+ if sum(row["status"] == "queued" for row in rows) >= hermes_state_runtime.ADMISSION_QUEUE_MAX:
+ from gateway.session_ingress import QUEUE_FULL_REPLY
+ return QUEUE_FULL_REPLY
+ return None
+
def _record_model_friction(self, signal: str, source, session_id: str, turns: int = 1) -> None:
"""Slash dispatch does not install the routed profile's scope, so a multiplexed runner
names the owning home explicitly."""
diff --git a/tests/gateway/test_retry_authority_refusal.py b/tests/gateway/test_retry_authority_refusal.py
new file mode 100644
index 00000000000..83b6edb9d76
--- /dev/null
+++ b/tests/gateway/test_retry_authority_refusal.py
@@ -0,0 +1,115 @@
+"""Messaging /retry under a session authority answers the authority's admission refusals BEFORE
+it rewrites the transcript (#139257).
+
+The re-sent turn is admitted by the routed profile's authority (``admit_native``). A draining or
+retiring runtime refuses it with ``runtime_draining`` and a full FIFO with ``queue_full``; both
+landed after ``rewrite_transcript`` had already dropped the last exchange, so every refused /retry
+lost a turn (and the next /retry ate the one before it). Real authority, real SQLite; only the
+turn handler is replaced by a recorder.
+"""
+import weakref
+from pathlib import Path
+
+import pytest
+
+from gateway.config import GatewayConfig, Platform
+from gateway.platforms.event import MessageEvent, MessageType
+from gateway.run import GatewayRunner
+from gateway.run_runtime import initialize_gateway_runtime
+from gateway.runtime_ownership import process_ownership
+from gateway.session import SessionSource
+from gateway.session_ingress_context import native_callback, register_transport_home
+
+
+class _FakeAdapter:
+ pass
+
+
+def served(tmp_path, monkeypatch):
+ root = tmp_path / '.hermes'
+ root.mkdir(mode=0o700)
+ monkeypatch.setattr(Path, 'home', lambda: tmp_path)
+ monkeypatch.setenv('HERMES_HOME', str(root))
+ import hermes_state
+ monkeypatch.setattr(hermes_state, 'DEFAULT_DB_PATH', hermes_state._IMPORT_DEFAULT_DB_PATH)
+ homes = [('default', root)]
+ config = GatewayConfig()
+ config._runtime_profile_homes = tuple(homes)
+ runner = GatewayRunner(config)
+ process_ownership.reserve([home for _, home in homes])
+ yield runner, root
+ for authority in list(getattr(runner, 'session_authorities', []) or []):
+ authority.db.close()
+ for _, home in homes:
+ process_ownership.release(home)
+
+
+async def _exchange(runner, root):
+ await initialize_gateway_runtime(runner)
+ register_transport_home(runner, None, root)
+ adapter = _FakeAdapter()
+ runner.adapters = {Platform.DISCORD: adapter}
+ runner._is_user_authorized_for_source = lambda *a, **k: True
+ source = SessionSource(platform=Platform.DISCORD, chat_id='c1', chat_type='dm',
+ user_id='u1', user_name='probe')
+ source._transport_adapter_ref = weakref.ref(adapter)
+ entry = await runner.async_session_store.get_or_create_session(source)
+ runner.session_store.append_to_transcript(entry.session_id, {'role': 'user', 'content': 'retry me'})
+ runner.session_store.append_to_transcript(entry.session_id, {'role': 'assistant', 'content': 'old answer'})
+ sent = []
+
+ async def recorder(event):
+ sent.append(event.text)
+ return 'resent'
+ runner._handle_message = recorder
+ return source, entry.session_id, sent
+
+
+async def _retry(runner, root, source):
+ event = MessageEvent(text='/retry', message_type=MessageType.TEXT, source=source, message_id='m-retry')
+ with native_callback(runner, event, root, None):
+ return await runner._handle_retry_command(event)
+
+
+def _contents(runner, sid):
+ return [m.get('content') for m in runner.session_store.load_transcript(sid)]
+
+
+async def test_retry_into_a_draining_authority_keeps_the_exchange(served):
+ runner, root = served
+ source, sid, sent = await _exchange(runner, root)
+ runner._draining = True
+
+ reply = await _retry(runner, root, source)
+
+ assert sent == [], 'the re-send ran into a draining authority'
+ assert reply and '⏳' in reply
+ assert _contents(runner, sid) == ['retry me', 'old answer']
+
+
+async def test_retry_into_a_full_fifo_keeps_the_exchange(served, monkeypatch):
+ runner, root = served
+ source, sid, sent = await _exchange(runner, root)
+ import hermes_state_runtime
+ monkeypatch.setattr(hermes_state_runtime, 'ADMISSION_QUEUE_MAX', 0)
+
+ reply = await _retry(runner, root, source)
+
+ assert sent == []
+ assert reply and 'waiting' in reply
+ assert _contents(runner, sid) == ['retry me', 'old answer']
+
+
+async def test_an_admissible_retry_still_replaces_the_exchange_once(served):
+ runner, root = served
+ source, sid, sent = await _exchange(runner, root)
+
+ reply = await _retry(runner, root, source)
+
+ assert reply == 'resent'
+ assert sent == ['retry me']
+ assert _contents(runner, sid) == []