Skip to content

The agent that grows with you

fix(gateway): keep a chat's media and quick text in arrival order

David Dudok de Wit4aa7fce7ea19
diff --git a/gateway/run_inbound.py b/gateway/run_inbound.py
index f91f4305db6..3870749dae6 100644
--- a/gateway/run_inbound.py
+++ b/gateway/run_inbound.py
@@ -618,6 +618,8 @@ class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayPluginInjectionMixin)
authority = active_authority(self)
sid = authority.logical_owner(self.session_store.peek_session_id(_quick_key)) if authority else None
if not sid or sid not in authority.sessions:
+ if authority is not None:
+ authority.fence_route(_quick_key) # the chat's first input, not committed yet
return
from gateway.session_contract import Principal, SessionRef
from hermes_state_runtime import RuntimeStoreError, list_session_admissions
@@ -628,6 +630,11 @@ class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayPluginInjectionMixin)
# A FIFO paused on a lost (``unknown``) turn is the operator's to resume: its pause notice
# promised the queued followers stay saved, so a /new from the chat leaves them queued.
paused = any(row['status'] == 'unknown' for row in rows)
+ if not paused:
+ # Input that arrived before this and is not committed yet never commits after it; one
+ # that committed before the fence is in the second listing and cancelled below.
+ authority.fence_route(_quick_key)
+ rows = await asyncio.to_thread(list_session_admissions, authority.db, session_id=sid)
for row in [] if paused else rows:
if row['status'] == 'queued' and row['principal_id'].startswith('messaging:'):
# Claimed meanwhile (stale_generation): it runs as main's post-claim race does.
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index a3b7011b029..91a635057c6 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -32,6 +32,11 @@ _MARKER_RELEASE_WAIT_S = 60.0
# True in the admitting task once ``admit_native``'s shielded ledger write began: a caller cancelled
# after that may still see the row commit, so it must not release the media the write commits.
native_write_started = ContextVar('native_write_started', default=False)
+# Set by the messaging ingress around ``admit_native``: True, or the busy caller's acceptance future.
+# Such an input is fenced by a later /stop or /new of its route (``fence_route``), and the future is
+# released once the input queues behind an earlier one of its chat, so a serial reader moves on
+# while acceptance itself still waits for the commit.
+messaging_admission = ContextVar('messaging_admission', default=None)
@dataclass
@@ -127,6 +132,8 @@ class SessionAuthority:
# Logical owner rotated away by the exhaustion auto-reset -> its session key: an admission
# that reserved it before the rotation commits on that key's current route instead.
self.rotated_owners = {}
+ # Route key -> [lock, holder + waiters, stop fence] of the native admissions in flight (``_route_turn``).
+ self._route_turns = {}
def authorize(self, actor, ref, capability):
"""Every handler calls this first, so a later ``self.sessions[ref.session_id]`` is
@@ -396,10 +403,52 @@ class SessionAuthority:
async def admit_native(self, event):
"""Await current connector policy, then commit before ACK or scheduling execution."""
from gateway.session_ingress_media import capture_lease
- with capture_lease(): # captured bytes are held from capture until commit or refusal
- return await self._admit_native(event)
-
- async def _admit_native(self, event):
+ async with self._route_turn(event) as fence:
+ with capture_lease(): # captured bytes are held from capture until commit or refusal
+ return await self._admit_native(event, fence)
+
+ @contextlib.asynccontextmanager
+ async def _route_turn(self, event):
+ """One route's inputs commit in the order they entered ``admit_native``: a photo's off-loop
+ capture must not let the chat's next text take the earlier FIFO position. asyncio's Lock is
+ FIFO and a cancelled waiter gives up only its own place, so a later input never overtakes an
+ earlier one still in flight. Other routes never wait; an unroutable event (refused by
+ ``prepare_native``) is not held. Yields the messaging input's fence check (else None)."""
+ ingress = messaging_admission.get()
+ messaging_admission.set(None) # not inherited by the drain this admission schedules
+ try:
+ key = self.runner.session_store._generate_session_key(event.source)
+ except AttributeError: # no source, or a duck-typed one without routing fields
+ key = None
+ if key is None:
+ yield None
+ return
+ entry = self._route_turns.setdefault(key, [asyncio.Lock(), 0, 0])
+ entry[1] += 1
+ arrived = entry[2]
+
+ def fence(_conn=None):
+ if entry[2] != arrived:
+ raise RuntimeStoreError('stale_generation', field='route',
+ detail='a /stop or /new of this chat came after this message')
+ if isinstance(ingress, asyncio.Future) and entry[1] > 1 and not ingress.done():
+ ingress.set_result(None) # its FIFO position is fixed behind the earlier input
+ try:
+ async with entry[0]:
+ if ingress is not None:
+ fence()
+ yield fence if ingress is not None else None
+ finally:
+ entry[1] -= 1
+ if not entry[1]:
+ del self._route_turns[key]
+
+ def fence_route(self, route):
+ """/stop and /new: the route's messaging input still capturing or waiting never commits."""
+ if (entry := self._route_turns.get(route)) is not None:
+ entry[2] += 1
+
+ async def _admit_native(self, event, fence=None):
import json
from gateway.session_envelope import prepare_native, restore_native_source
self._require_admission_open()
@@ -427,7 +476,7 @@ class SessionAuthority:
return await tracked_write(self, partial(
self._admit_native_write, principal_id='messaging:' + identity, session_id=ref.session_id,
request_id=request_id, payload=payload,
- authorize=session_retry_admission.compose(rewrite, self._admission_gate()),
+ authorize=session_retry_admission.compose(rewrite, self._admission_gate(fence)),
raw=getattr(event, 'raw_message', None)),
then=admitted, ordered=True, after=lambda _receipt: self._schedule_admitted(committed[0]))
diff --git a/gateway/session_ingress.py b/gateway/session_ingress.py
index 2a0e12cd142..ba2a6fc1ad1 100644
--- a/gateway/session_ingress.py
+++ b/gateway/session_ingress.py
@@ -29,11 +29,15 @@ QUEUE_FULL_REPLY = ('⏳ Too many messages are already waiting in this conversat
async def admit_message(authority, event):
+ from gateway.session_authority import messaging_admission
accepted = busy_acceptance.get()
busy_acceptance.set(None) # the drain task this schedules must not inherit the busy caller's future
+ ingress = messaging_admission.set(True if accepted is None else accepted)
try:
receipt = await authority.admit_native(event)
except RuntimeStoreError as exc:
+ if exc.reason == 'stale_generation' and exc.field == 'route':
+ return None # a later /stop or /new of the chat fenced it, as main cancels a queued row
if exc.reason == 'invalid_params' and exc.field:
# Refused before commit (_validate_native): a refusal notice instead of a generic turn error.
# The reply reaches the whole chat, so only content clauses are named; trust and provenance
@@ -45,6 +49,8 @@ async def admit_message(authority, event):
raise
# Nothing was committed (main's busy-queue cap): say so instead of a generic turn error.
return QUEUE_FULL_REPLY
+ finally:
+ messaging_admission.reset(ingress)
if receipt.status == 'terminal':
return None
# Only the delivery waiter is process-local; execution reads the committed snapshot.
@@ -384,7 +390,9 @@ async def deliver_response(adapter, event, session_key, response):
async def dispatch_shared_busy(adapter, event, session_key):
"""Serial receive loops (IRC, Signal) await this before reading their next frame, so return
- once the follow-up is committed to the FIFO; a tracked observer delivers its one reply."""
+ once the follow-up is committed to the FIFO, or queued behind an earlier input of its chat still
+ capturing (its position is then fixed; acceptance still waits for the commit); a tracked
+ observer delivers its one reply."""
delivery_event = replace(event, source=replace(event.source))
accepted = asyncio.get_running_loop().create_future()
diff --git a/tests/gateway/test_native_admission_arrival_order.py b/tests/gateway/test_native_admission_arrival_order.py
new file mode 100644
index 00000000000..7a97ec5ed4e
--- /dev/null
+++ b/tests/gateway/test_native_admission_arrival_order.py
@@ -0,0 +1,195 @@
+"""A chat's inputs reach its FIFO in arrival order, whatever off-loop media capture each needs.
+
+The capture (copy + SHA-256 + fsyncs) runs in a worker thread before the ordered admission write,
+so a text sent right after a photo or video must wait for it instead of taking the earlier slot
+and running first without the media. Other chats never wait.
+"""
+import asyncio
+import threading
+
+import pytest
+
+from gateway.config import GatewayConfig, Platform, PlatformConfig
+from gateway.platforms.event import MessageEvent, MessageType
+from gateway.run import GatewayRunner
+from gateway.session_authority import SessionAuthority, initialize_session_authority
+from hermes_state_runtime import list_session_admissions
+from tests.gateway.test_shared_busy_receive_loop import Adapter
+from tests.waiting import async_wait_until
+
+
+
+def _order(authority):
+ rows = (row for sid in authority.sessions
+ for row in list_session_admissions(authority.db, session_id=sid, pending_only=False))
+ return [row['request_id'] for row in sorted(rows, key=lambda row: row['seq'])]
+
+
+def _hold_capture(monkeypatch):
+ """The real capture, held on an Event: a large video on a slow disk."""
+ import gateway.session_envelope as envelope
+ import gateway.session_ingress_media as media
+ entered, release = threading.Event(), threading.Event()
+ real_capture = media.capture_native_media
+
+ def held(paths):
+ entered.set()
+ release.wait(5.0)
+ return real_capture(paths)
+ monkeypatch.setattr(envelope, 'capture_native_media', held)
+ return entered, release
+
+
+def _video(source, mid, clip):
+ return MessageEvent(text='', message_type=MessageType.VIDEO, source=source, message_id=mid,
+ media_urls=[str(clip)], media_types=['video/mp4'])
+
+
[email protected]('busy', [False, True], ids=['idle', 'busy'])
+async def test_media_then_quick_text_keep_arrival_order(tmp_path, monkeypatch, busy):
+ """The adapter path: an idle video then a busy text, or both while a turn runs."""
+ import gateway.session_finite as finite
+ monkeypatch.setenv('TELEGRAM_ALLOWED_USERS', 'fixture-user')
+ gate = asyncio.Event()
+
+ async def produce(authority, ref, row):
+ await gate.wait()
+ return 'ok'
+ monkeypatch.setattr(finite, 'execute_finite_admission', produce)
+ entered, release = _hold_capture(monkeypatch)
+ runner = GatewayRunner(GatewayConfig())
+ adapter = Adapter()
+ runner.adapters = {Platform.TELEGRAM: adapter}
+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='order')
+ runner._wire_adapter_handlers(adapter)
+ source = adapter.build_source(chat_id='order-chat', chat_type='dm', user_id='fixture-user')
+ clip = tmp_path / 'clip.mp4'
+ clip.write_bytes(b'video-bytes' * 1000)
+ expected = ['v1', 't1']
+ try:
+ if busy:
+ await adapter.handle_message(MessageEvent(text='hello', source=source, message_id='h0'))
+ await async_wait_until(lambda: _order(authority) == ['h0'], what='the running turn admitted')
+ expected.insert(0, 'h0')
+ # One task per delivery, as Discord and Slack dispatch them.
+ first = asyncio.create_task(adapter.handle_message(_video(source, 'v1', clip)))
+ await asyncio.to_thread(entered.wait, 5)
+ text = MessageEvent(text='what is this?', source=source, message_id='t1')
+ second = asyncio.create_task(adapter.handle_message(text))
+ # A serial reader (Signal, IRC) awaits this: it returns once the text queues behind the
+ # video, without acknowledging it (main commits the text here, ahead of the video).
+ assert second in (await asyncio.wait({second}, timeout=2.0))[0], 'the reader waited for the capture'
+ assert 't1' not in _order(authority) and text._gateway_accepted is False
+ release.set()
+ await asyncio.gather(first, second)
+ await async_wait_until(lambda: len(_order(authority)) == len(expected), what='both admitted')
+ assert _order(authority) == expected
+ finally:
+ release.set()
+ gate.set()
+ await async_wait_until(lambda: not adapter._background_tasks, what='adapter tasks done')
+ authority.db.close()
+
+
+async def _held_owner(tmp_path, monkeypatch):
+ """Real authority over SQLite, WhatsApp sources, the capture held on an Event; no drain."""
+ from plugins.platforms.whatsapp.adapter import WhatsAppAdapter
+ monkeypatch.setenv('WHATSAPP_ALLOWED_USERS', '15551234567,15557654321')
+ monkeypatch.setattr(SessionAuthority, '_schedule', lambda self, ref: None)
+ entered, release = _hold_capture(monkeypatch)
+ runner = GatewayRunner(GatewayConfig(sessions_dir=tmp_path / 'sessions'))
+ adapter = WhatsAppAdapter(PlatformConfig(enabled=True, extra={'session_path': str(tmp_path / 'wa')}))
+ runner.adapters = {Platform.WHATSAPP: adapter}
+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='held')
+ clip = tmp_path / 'clip.mp4'
+ clip.write_bytes(b'video-bytes' * 1000)
+
+ def admit(chat, mid, media=False, ingress=False):
+ """``ingress``: through the messaging ingress (``admit_message``), else ``admit_native``."""
+ from gateway.session_ingress import admit_message
+ source = adapter.build_source(chat_id=chat, user_id=chat, chat_type='dm')
+ event = _video(source, mid, clip) if media else MessageEvent(text='text ' + mid, source=source,
+ message_id=mid)
+ return asyncio.create_task(admit_message(authority, event) if ingress else authority.admit_native(event))
+ route = lambda chat: runner._session_key_for_source(adapter.build_source(chat_id=chat, user_id=chat, chat_type='dm'))
+ return authority, entered, release, admit, route
+
+
+async def test_a_cancelled_middle_input_does_not_let_the_next_overtake(tmp_path, monkeypatch):
+ authority, entered, release, admit, _ = await _held_owner(tmp_path, monkeypatch)
+ try:
+ first = admit(X, 'x1-video', media=True)
+ await asyncio.to_thread(entered.wait, 5)
+ second = admit(X, 'x2-text')
+ await asyncio.sleep(0)
+ third = admit(X, 'x3-text')
+ await asyncio.sleep(0)
+ second.cancel() # e.g. the busy caller's task is cancelled
+ await asyncio.gather(second, return_exceptions=True)
+ await asyncio.wait({third}, timeout=0.3)
+ release.set()
+ await asyncio.gather(first, third)
+ assert _order(authority) == ['x1-video', 'x3-text']
+ assert authority._route_turns == {} # the per-chat entries go once idle
+ finally:
+ release.set()
+ authority.db.close()
+
+
+async def test_a_held_capture_never_delays_another_chat_nor_outlives_its_cancel(tmp_path, monkeypatch):
+ authority, entered, release, admit, _ = await _held_owner(tmp_path, monkeypatch)
+ try:
+ first = admit(X, 'x1-video', media=True)
+ await asyncio.to_thread(entered.wait, 5)
+ other = admit(Y, 'y-text')
+ assert other in (await asyncio.wait({other}, timeout=1.0))[0], 'chat Y waited on chat X'
+ follower = admit(X, 'x2-text')
+ await asyncio.sleep(0)
+ first.cancel() # the earlier input is abandoned mid-capture: its successor goes on
+ await asyncio.gather(first, return_exceptions=True)
+ assert follower in (await asyncio.wait({follower}, timeout=1.0))[0], 'a cancelled input blocked the next'
+ finally:
+ release.set()
+ authority.db.close()
+
+
[email protected]('chat', ['existing', 'first-message', 'paused'])
+async def test_stop_fences_inputs_still_capturing_or_waiting(tmp_path, monkeypatch, chat):
+ """/stop (and /new, /reset: the same hook) reaches the chat's inputs that arrived before it but
+ have not committed yet, also when the chat has no session yet. A FIFO paused on a lost turn
+ keeps its queued followers through /stop and /new, and so these too."""
+ authority, entered, release, admit, route = await _held_owner(tmp_path, monkeypatch)
+ queued = lambda: [row['request_id'] for sid in authority.sessions for row in
+ list_session_admissions(authority.db, session_id=sid) if row['status'] == 'queued']
+ first = second = None
+ try:
+ if chat != 'first-message':
+ await admit(X, 'x0-text')
+ if chat == 'paused': # x0 was running when the previous owner died
+ authority.db._execute_write(lambda conn: conn.execute(
+ "UPDATE session_admissions SET status='unknown' WHERE request_id='x0-text'"))
+ first = admit(X, 'x1-video', media=True, ingress=True)
+ await asyncio.to_thread(entered.wait, 5)
+ second = admit(X, 'x2-text', ingress=True)
+ await asyncio.wait({second}, timeout=0.3) # main commits it here
+ await authority.runner._hm_stop_owner_turn(route(X))
+ release.set()
+ if chat == 'paused':
+ await async_wait_until(lambda: len(queued()) == 2, timeout=3.0, what='both kept')
+ assert queued() == ['x1-video', 'x2-text']
+ return
+ done, _ = await asyncio.wait({first, second}, timeout=3.0)
+ assert queued() == [], queued()
+ assert done == {first, second} and first.result() is second.result() is None # dropped silently
+ assert authority._route_turns == {}
+ finally:
+ release.set()
+ for task in (first, second):
+ if task is not None:
+ task.cancel()
+ authority.db.close()
diff --git a/tests/gateway/test_native_fifo_context_exhaustion.py b/tests/gateway/test_native_fifo_context_exhaustion.py
index 00414161b99..f8ec0808203 100644
--- a/tests/gateway/test_native_fifo_context_exhaustion.py
+++ b/tests/gateway/test_native_fifo_context_exhaustion.py
@@ -211,8 +211,9 @@ async def test_no_scheduler_starts_the_fresh_fifo_before_reset_cleanup_finishes(
@pytest.mark.asyncio
async def test_admission_reserved_on_the_old_route_commits_on_the_fresh_session(native, monkeypatch):
- """A retry of queued B and a new D that reserved the old route before the reset commit after
- the re-home: B dedupes onto its moved row (same admission id), D queues behind it there."""
+ """A retry of queued B that reserved the old route before the reset commits after the re-home,
+ deduped onto its moved row (same admission id); D, the chat's next input, waits for it at the
+ route's admission turn and queues behind it there."""
import threading
from gateway.session_results import execution_result
runner, authority, db = native.runner, native.authority, native.authority.db
@@ -230,10 +231,11 @@ async def test_admission_reserved_on_the_old_route_commits_on_the_fresh_session(
monkeypatch.setattr(authority, "_reserve_route", held_reserve)
late = [asyncio.create_task(_admit(native, "B", admitted[1][0])), asyncio.create_task(_admit(native, "D"))]
for _ in range(250):
- if len(held) == 2:
+ if held:
break
await asyncio.sleep(0.02)
- assert [sid for sid, _ in held] == [old.session_id] * 2
+ await asyncio.wait({late[1]}, timeout=0.2)
+ assert [sid for sid, _ in held] == [old.session_id] # D waits behind B's admission
executed, fresh = [], {}
async def turn(event, source, session_key, run_generation):