The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): keep a chat's media and quick text in arrival order
David Dudok de Wit4aa7fce7ea19
diff --git a/gateway/run_inbound.py b/gateway/run_inbound.pyindex f91f4305db6..3870749dae6 100644--- a/gateway/run_inbound.py+++ b/gateway/run_inbound.py@@ -618,6 +618,8 @@ class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayPluginInjectionMixin)authority = active_authority(self)sid = authority.logical_owner(self.session_store.peek_session_id(_quick_key)) if authority else Noneif not sid or sid not in authority.sessions:+ if authority is not None:+ authority.fence_route(_quick_key) # the chat's first input, not committed yetreturnfrom gateway.session_contract import Principal, SessionReffrom hermes_state_runtime import RuntimeStoreError, list_session_admissions@@ -628,6 +630,11 @@ class GatewayInboundMixin(GatewayInboundHooksMixin, GatewayPluginInjectionMixin)# A FIFO paused on a lost (``unknown``) turn is the operator's to resume: its pause notice# promised the queued followers stay saved, so a /new from the chat leaves them queued.paused = any(row['status'] == 'unknown' for row in rows)+ if not paused:+ # Input that arrived before this and is not committed yet never commits after it; one+ # that committed before the fence is in the second listing and cancelled below.+ authority.fence_route(_quick_key)+ rows = await asyncio.to_thread(list_session_admissions, authority.db, session_id=sid)for row in [] if paused else rows:if row['status'] == 'queued' and row['principal_id'].startswith('messaging:'):# Claimed meanwhile (stale_generation): it runs as main's post-claim race does.diff --git a/gateway/session_authority.py b/gateway/session_authority.pyindex a3b7011b029..91a635057c6 100644--- a/gateway/session_authority.py+++ b/gateway/session_authority.py@@ -32,6 +32,11 @@ _MARKER_RELEASE_WAIT_S = 60.0# True in the admitting task once ``admit_native``'s shielded ledger write began: a caller cancelled# after that may still see the row commit, so it must not release the media the write commits.native_write_started = ContextVar('native_write_started', default=False)+# Set by the messaging ingress around ``admit_native``: True, or the busy caller's acceptance future.+# Such an input is fenced by a later /stop or /new of its route (``fence_route``), and the future is+# released once the input queues behind an earlier one of its chat, so a serial reader moves on+# while acceptance itself still waits for the commit.+messaging_admission = ContextVar('messaging_admission', default=None)@dataclass@@ -127,6 +132,8 @@ class SessionAuthority:# Logical owner rotated away by the exhaustion auto-reset -> its session key: an admission# that reserved it before the rotation commits on that key's current route instead.self.rotated_owners = {}+ # Route key -> [lock, holder + waiters, stop fence] of the native admissions in flight (``_route_turn``).+ self._route_turns = {}def authorize(self, actor, ref, capability):"""Every handler calls this first, so a later ``self.sessions[ref.session_id]`` is@@ -396,10 +403,52 @@ class SessionAuthority:async def admit_native(self, event):"""Await current connector policy, then commit before ACK or scheduling execution."""from gateway.session_ingress_media import capture_lease- with capture_lease(): # captured bytes are held from capture until commit or refusal- return await self._admit_native(event)-- async def _admit_native(self, event):+ async with self._route_turn(event) as fence:+ with capture_lease(): # captured bytes are held from capture until commit or refusal+ return await self._admit_native(event, fence)++ @contextlib.asynccontextmanager+ async def _route_turn(self, event):+ """One route's inputs commit in the order they entered ``admit_native``: a photo's off-loop+ capture must not let the chat's next text take the earlier FIFO position. asyncio's Lock is+ FIFO and a cancelled waiter gives up only its own place, so a later input never overtakes an+ earlier one still in flight. Other routes never wait; an unroutable event (refused by+ ``prepare_native``) is not held. Yields the messaging input's fence check (else None)."""+ ingress = messaging_admission.get()+ messaging_admission.set(None) # not inherited by the drain this admission schedules+ try:+ key = self.runner.session_store._generate_session_key(event.source)+ except AttributeError: # no source, or a duck-typed one without routing fields+ key = None+ if key is None:+ yield None+ return+ entry = self._route_turns.setdefault(key, [asyncio.Lock(), 0, 0])+ entry[1] += 1+ arrived = entry[2]++ def fence(_conn=None):+ if entry[2] != arrived:+ raise RuntimeStoreError('stale_generation', field='route',+ detail='a /stop or /new of this chat came after this message')+ if isinstance(ingress, asyncio.Future) and entry[1] > 1 and not ingress.done():+ ingress.set_result(None) # its FIFO position is fixed behind the earlier input+ try:+ async with entry[0]:+ if ingress is not None:+ fence()+ yield fence if ingress is not None else None+ finally:+ entry[1] -= 1+ if not entry[1]:+ del self._route_turns[key]++ def fence_route(self, route):+ """/stop and /new: the route's messaging input still capturing or waiting never commits."""+ if (entry := self._route_turns.get(route)) is not None:+ entry[2] += 1++ async def _admit_native(self, event, fence=None):import jsonfrom gateway.session_envelope import prepare_native, restore_native_sourceself._require_admission_open()@@ -427,7 +476,7 @@ class SessionAuthority:return await tracked_write(self, partial(self._admit_native_write, principal_id='messaging:' + identity, session_id=ref.session_id,request_id=request_id, payload=payload,- authorize=session_retry_admission.compose(rewrite, self._admission_gate()),+ authorize=session_retry_admission.compose(rewrite, self._admission_gate(fence)),raw=getattr(event, 'raw_message', None)),then=admitted, ordered=True, after=lambda _receipt: self._schedule_admitted(committed[0]))diff --git a/gateway/session_ingress.py b/gateway/session_ingress.pyindex 2a0e12cd142..ba2a6fc1ad1 100644--- a/gateway/session_ingress.py+++ b/gateway/session_ingress.py@@ -29,11 +29,15 @@ QUEUE_FULL_REPLY = ('⏳ Too many messages are already waiting in this conversatasync def admit_message(authority, event):+ from gateway.session_authority import messaging_admissionaccepted = busy_acceptance.get()busy_acceptance.set(None) # the drain task this schedules must not inherit the busy caller's future+ ingress = messaging_admission.set(True if accepted is None else accepted)try:receipt = await authority.admit_native(event)except RuntimeStoreError as exc:+ if exc.reason == 'stale_generation' and exc.field == 'route':+ return None # a later /stop or /new of the chat fenced it, as main cancels a queued rowif exc.reason == 'invalid_params' and exc.field:# Refused before commit (_validate_native): a refusal notice instead of a generic turn error.# The reply reaches the whole chat, so only content clauses are named; trust and provenance@@ -45,6 +49,8 @@ async def admit_message(authority, event):raise# Nothing was committed (main's busy-queue cap): say so instead of a generic turn error.return QUEUE_FULL_REPLY+ finally:+ messaging_admission.reset(ingress)if receipt.status == 'terminal':return None# Only the delivery waiter is process-local; execution reads the committed snapshot.@@ -384,7 +390,9 @@ async def deliver_response(adapter, event, session_key, response):async def dispatch_shared_busy(adapter, event, session_key):"""Serial receive loops (IRC, Signal) await this before reading their next frame, so return- once the follow-up is committed to the FIFO; a tracked observer delivers its one reply."""+ once the follow-up is committed to the FIFO, or queued behind an earlier input of its chat still+ capturing (its position is then fixed; acceptance still waits for the commit); a tracked+ observer delivers its one reply."""delivery_event = replace(event, source=replace(event.source))accepted = asyncio.get_running_loop().create_future()diff --git a/tests/gateway/test_native_admission_arrival_order.py b/tests/gateway/test_native_admission_arrival_order.pynew file mode 100644index 00000000000..7a97ec5ed4e--- /dev/null+++ b/tests/gateway/test_native_admission_arrival_order.py@@ -0,0 +1,195 @@+"""A chat's inputs reach its FIFO in arrival order, whatever off-loop media capture each needs.++The capture (copy + SHA-256 + fsyncs) runs in a worker thread before the ordered admission write,+so a text sent right after a photo or video must wait for it instead of taking the earlier slot+and running first without the media. Other chats never wait.+"""+import asyncio+import threading++import pytest++from gateway.config import GatewayConfig, Platform, PlatformConfig+from gateway.platforms.event import MessageEvent, MessageType+from gateway.run import GatewayRunner+from gateway.session_authority import SessionAuthority, initialize_session_authority+from hermes_state_runtime import list_session_admissions+from tests.gateway.test_shared_busy_receive_loop import Adapter+from tests.waiting import async_wait_until++X, Y = '[email protected]', '[email protected]'+++def _order(authority):+ rows = (row for sid in authority.sessions+ for row in list_session_admissions(authority.db, session_id=sid, pending_only=False))+ return [row['request_id'] for row in sorted(rows, key=lambda row: row['seq'])]+++def _hold_capture(monkeypatch):+ """The real capture, held on an Event: a large video on a slow disk."""+ import gateway.session_envelope as envelope+ import gateway.session_ingress_media as media+ entered, release = threading.Event(), threading.Event()+ real_capture = media.capture_native_media++ def held(paths):+ entered.set()+ release.wait(5.0)+ return real_capture(paths)+ monkeypatch.setattr(envelope, 'capture_native_media', held)+ return entered, release+++def _video(source, mid, clip):+ return MessageEvent(text='', message_type=MessageType.VIDEO, source=source, message_id=mid,+ media_urls=[str(clip)], media_types=['video/mp4'])++[email protected]('busy', [False, True], ids=['idle', 'busy'])+async def test_media_then_quick_text_keep_arrival_order(tmp_path, monkeypatch, busy):+ """The adapter path: an idle video then a busy text, or both while a turn runs."""+ import gateway.session_finite as finite+ monkeypatch.setenv('TELEGRAM_ALLOWED_USERS', 'fixture-user')+ gate = asyncio.Event()++ async def produce(authority, ref, row):+ await gate.wait()+ return 'ok'+ monkeypatch.setattr(finite, 'execute_finite_admission', produce)+ entered, release = _hold_capture(monkeypatch)+ runner = GatewayRunner(GatewayConfig())+ adapter = Adapter()+ runner.adapters = {Platform.TELEGRAM: adapter}+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='order')+ runner._wire_adapter_handlers(adapter)+ source = adapter.build_source(chat_id='order-chat', chat_type='dm', user_id='fixture-user')+ clip = tmp_path / 'clip.mp4'+ clip.write_bytes(b'video-bytes' * 1000)+ expected = ['v1', 't1']+ try:+ if busy:+ await adapter.handle_message(MessageEvent(text='hello', source=source, message_id='h0'))+ await async_wait_until(lambda: _order(authority) == ['h0'], what='the running turn admitted')+ expected.insert(0, 'h0')+ # One task per delivery, as Discord and Slack dispatch them.+ first = asyncio.create_task(adapter.handle_message(_video(source, 'v1', clip)))+ await asyncio.to_thread(entered.wait, 5)+ text = MessageEvent(text='what is this?', source=source, message_id='t1')+ second = asyncio.create_task(adapter.handle_message(text))+ # A serial reader (Signal, IRC) awaits this: it returns once the text queues behind the+ # video, without acknowledging it (main commits the text here, ahead of the video).+ assert second in (await asyncio.wait({second}, timeout=2.0))[0], 'the reader waited for the capture'+ assert 't1' not in _order(authority) and text._gateway_accepted is False+ release.set()+ await asyncio.gather(first, second)+ await async_wait_until(lambda: len(_order(authority)) == len(expected), what='both admitted')+ assert _order(authority) == expected+ finally:+ release.set()+ gate.set()+ await async_wait_until(lambda: not adapter._background_tasks, what='adapter tasks done')+ authority.db.close()+++async def _held_owner(tmp_path, monkeypatch):+ """Real authority over SQLite, WhatsApp sources, the capture held on an Event; no drain."""+ from plugins.platforms.whatsapp.adapter import WhatsAppAdapter+ monkeypatch.setenv('WHATSAPP_ALLOWED_USERS', '15551234567,15557654321')+ monkeypatch.setattr(SessionAuthority, '_schedule', lambda self, ref: None)+ entered, release = _hold_capture(monkeypatch)+ runner = GatewayRunner(GatewayConfig(sessions_dir=tmp_path / 'sessions'))+ adapter = WhatsAppAdapter(PlatformConfig(enabled=True, extra={'session_path': str(tmp_path / 'wa')}))+ runner.adapters = {Platform.WHATSAPP: adapter}+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='held')+ clip = tmp_path / 'clip.mp4'+ clip.write_bytes(b'video-bytes' * 1000)++ def admit(chat, mid, media=False, ingress=False):+ """``ingress``: through the messaging ingress (``admit_message``), else ``admit_native``."""+ from gateway.session_ingress import admit_message+ source = adapter.build_source(chat_id=chat, user_id=chat, chat_type='dm')+ event = _video(source, mid, clip) if media else MessageEvent(text='text ' + mid, source=source,+ message_id=mid)+ return asyncio.create_task(admit_message(authority, event) if ingress else authority.admit_native(event))+ route = lambda chat: runner._session_key_for_source(adapter.build_source(chat_id=chat, user_id=chat, chat_type='dm'))+ return authority, entered, release, admit, route+++async def test_a_cancelled_middle_input_does_not_let_the_next_overtake(tmp_path, monkeypatch):+ authority, entered, release, admit, _ = await _held_owner(tmp_path, monkeypatch)+ try:+ first = admit(X, 'x1-video', media=True)+ await asyncio.to_thread(entered.wait, 5)+ second = admit(X, 'x2-text')+ await asyncio.sleep(0)+ third = admit(X, 'x3-text')+ await asyncio.sleep(0)+ second.cancel() # e.g. the busy caller's task is cancelled+ await asyncio.gather(second, return_exceptions=True)+ await asyncio.wait({third}, timeout=0.3)+ release.set()+ await asyncio.gather(first, third)+ assert _order(authority) == ['x1-video', 'x3-text']+ assert authority._route_turns == {} # the per-chat entries go once idle+ finally:+ release.set()+ authority.db.close()+++async def test_a_held_capture_never_delays_another_chat_nor_outlives_its_cancel(tmp_path, monkeypatch):+ authority, entered, release, admit, _ = await _held_owner(tmp_path, monkeypatch)+ try:+ first = admit(X, 'x1-video', media=True)+ await asyncio.to_thread(entered.wait, 5)+ other = admit(Y, 'y-text')+ assert other in (await asyncio.wait({other}, timeout=1.0))[0], 'chat Y waited on chat X'+ follower = admit(X, 'x2-text')+ await asyncio.sleep(0)+ first.cancel() # the earlier input is abandoned mid-capture: its successor goes on+ await asyncio.gather(first, return_exceptions=True)+ assert follower in (await asyncio.wait({follower}, timeout=1.0))[0], 'a cancelled input blocked the next'+ finally:+ release.set()+ authority.db.close()++[email protected]('chat', ['existing', 'first-message', 'paused'])+async def test_stop_fences_inputs_still_capturing_or_waiting(tmp_path, monkeypatch, chat):+ """/stop (and /new, /reset: the same hook) reaches the chat's inputs that arrived before it but+ have not committed yet, also when the chat has no session yet. A FIFO paused on a lost turn+ keeps its queued followers through /stop and /new, and so these too."""+ authority, entered, release, admit, route = await _held_owner(tmp_path, monkeypatch)+ queued = lambda: [row['request_id'] for sid in authority.sessions for row in+ list_session_admissions(authority.db, session_id=sid) if row['status'] == 'queued']+ first = second = None+ try:+ if chat != 'first-message':+ await admit(X, 'x0-text')+ if chat == 'paused': # x0 was running when the previous owner died+ authority.db._execute_write(lambda conn: conn.execute(+ "UPDATE session_admissions SET status='unknown' WHERE request_id='x0-text'"))+ first = admit(X, 'x1-video', media=True, ingress=True)+ await asyncio.to_thread(entered.wait, 5)+ second = admit(X, 'x2-text', ingress=True)+ await asyncio.wait({second}, timeout=0.3) # main commits it here+ await authority.runner._hm_stop_owner_turn(route(X))+ release.set()+ if chat == 'paused':+ await async_wait_until(lambda: len(queued()) == 2, timeout=3.0, what='both kept')+ assert queued() == ['x1-video', 'x2-text']+ return+ done, _ = await asyncio.wait({first, second}, timeout=3.0)+ assert queued() == [], queued()+ assert done == {first, second} and first.result() is second.result() is None # dropped silently+ assert authority._route_turns == {}+ finally:+ release.set()+ for task in (first, second):+ if task is not None:+ task.cancel()+ authority.db.close()diff --git a/tests/gateway/test_native_fifo_context_exhaustion.py b/tests/gateway/test_native_fifo_context_exhaustion.pyindex 00414161b99..f8ec0808203 100644--- a/tests/gateway/test_native_fifo_context_exhaustion.py+++ b/tests/gateway/test_native_fifo_context_exhaustion.py@@ -211,8 +211,9 @@ async def test_no_scheduler_starts_the_fresh_fifo_before_reset_cleanup_finishes(@pytest.mark.asyncioasync def test_admission_reserved_on_the_old_route_commits_on_the_fresh_session(native, monkeypatch):- """A retry of queued B and a new D that reserved the old route before the reset commit after- the re-home: B dedupes onto its moved row (same admission id), D queues behind it there."""+ """A retry of queued B that reserved the old route before the reset commits after the re-home,+ deduped onto its moved row (same admission id); D, the chat's next input, waits for it at the+ route's admission turn and queues behind it there."""import threadingfrom gateway.session_results import execution_resultrunner, authority, db = native.runner, native.authority, native.authority.db@@ -230,10 +231,11 @@ async def test_admission_reserved_on_the_old_route_commits_on_the_fresh_session(monkeypatch.setattr(authority, "_reserve_route", held_reserve)late = [asyncio.create_task(_admit(native, "B", admitted[1][0])), asyncio.create_task(_admit(native, "D"))]for _ in range(250):- if len(held) == 2:+ if held:breakawait asyncio.sleep(0.02)- assert [sid for sid, _ in held] == [old.session_id] * 2+ await asyncio.wait({late[1]}, timeout=0.2)+ assert [sid for sid, _ in held] == [old.session_id] # D waits behind B's admissionexecuted, fresh = [], {}async def turn(event, source, session_key, run_generation):