The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a tick admitted while its completion lands is withdrawn, never left behind it
Teknium58fef698c8cc
diff --git a/gateway/run_notifications.py b/gateway/run_notifications.pyindex 989b92a3a10..312f0d27098 100644--- a/gateway/run_notifications.py+++ b/gateway/run_notifications.py@@ -717,25 +717,18 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotificaif self._load_background_notifications_mode() == "off":continueif evt.get("type") == "heartbeat":- from gateway.session_automation import prepare_heartbeat+ from gateway.session_automation import deliver_heartbeat+ delivered = await deliver_heartbeat(self, evt)+ else:+ synth_text = _format_gateway_process_notification(evt)+ if not synth_text:+ continuetry:- prepared = await prepare_heartbeat(self, evt)+ delivered = await self._inject_watch_notification(synth_text, evt)except Exception:- # One tick's failure must not drop the rest of the drained events.- logger.exception("Heartbeat preparation error; the tick is requeued")- completion_queue.put(evt)- continue- if prepared is None:- continue- evt = prepared- synth_text = _format_gateway_process_notification(evt)- if not synth_text:- continue- try:- delivered = await self._inject_watch_notification(synth_text, evt)- except Exception:- logger.exception("Watch notification injection error")- delivered = False+ logger.exception("Watch notification injection error")+ delivered = False+ # The original event: a requeued tick folds again from what is still queued.if delivered is False:completion_queue.put(evt)@@ -904,15 +897,10 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotificafrom gateway.session_authorities import active_authorityauthority = active_authority(self)if authority is not None:- from gateway.session_automation import producer_identity, remember_heartbeat, withdraw_process_ticks+ from gateway.session_automation import admit_process_noticefrom hermes_state_runtime import RuntimeStoreErrortry:- identity = producer_identity(self, evt)- await authority.admit_automation(adapter, synth_event, identity)- if evt.get("type") == "heartbeat":- remember_heartbeat(self, identity, evt)- elif evt.get("type") == "completion":- await withdraw_process_ticks(self, evt)+ await admit_process_notice(self, authority, adapter, synth_event, evt)except RuntimeStoreError as exc:raise WakeNotAccepted(str(exc)) from excelse:diff --git a/gateway/session_automation.py b/gateway/session_automation.pyindex af9333cdf08..1ec016c0425 100644--- a/gateway/session_automation.py+++ b/gateway/session_automation.py@@ -53,9 +53,48 @@ def remember_heartbeat(runner, identity, event):ticks.popitem(last=False)+async def deliver_heartbeat(runner, event):+ """Prepare, format and admit one liveness tick as one critical section with a completion's+ admit-and-withdraw (``admit_process_notice``): a completion that lands while this tick is on its+ way in withdraws it once it is in, and a tick checked after it sees it. True when admitted, None+ to drop it, False to requeue the original event."""+ from gateway.run import _format_gateway_process_notification+ async with _heartbeat_lock(runner):+ try:+ prepared = await prepare_heartbeat(runner, event)+ except Exception:+ # One tick's failure must not drop the rest of the drained events.+ logger.exception("Heartbeat preparation error; the tick is requeued")+ return False+ text = prepared and _format_gateway_process_notification(prepared)+ if not text:+ return None+ try:+ return await runner._inject_watch_notification(text, prepared)+ except Exception:+ logger.exception("Watch notification injection error")+ return False+++async def admit_process_notice(runner, authority, adapter, event, evt):+ """Admit a process notice. A completion and the withdrawal of its process's waiting ticks hold+ the heartbeat lock together; a tick already holds it (``deliver_heartbeat``)."""+ identity = producer_identity(runner, evt)+ if evt.get('type') != 'completion':+ receipt = await authority.admit_automation(adapter, event, identity)+ if evt.get('type') == 'heartbeat':+ remember_heartbeat(runner, identity, evt)+ return receipt+ async with _heartbeat_lock(runner):+ receipt = await authority.admit_automation(adapter, event, identity)+ await withdraw_process_ticks(runner, evt)+ return receipt++async def prepare_heartbeat(runner, event):"""The liveness tick to admit for ``event``, or None to drop it because its process's completion- is already in the session: a later tick would be a stale "still running" turn.+ is already in the session: a later tick would be a stale "still running" turn. The caller holds+ the heartbeat lock.Otherwise a tick of the same process still queued is cancelled and its output carried into thisone, since a tick's output is never sent again: a busy session gets one waiting turn holding@@ -69,16 +108,15 @@ async def prepare_heartbeat(runner, event):if entry is None:return eventowner, process = _route_owner(authority, entry), str(event.get('session_id') or '')- async with _heartbeat_lock(runner):- if await asyncio.to_thread(_process_completed, runner, authority, owner, event):- logger.debug("Dropped heartbeat %s of process %s: its completion is already admitted",- event.get('event_id'), process)- return None- pending = await asyncio.to_thread(list_session_admissions, authority.db, session_id=owner)- waiting = next((row for row in pending if _queued_tick(row, process, entry.session_key)), None)- earlier = waiting and getattr(runner, '_admitted_heartbeat_ticks', {}).get(waiting['request_id'])- if earlier is None or not await _withdraw(authority, waiting):- return event+ if await asyncio.to_thread(_process_completed, runner, authority, owner, event):+ logger.debug("Dropped heartbeat %s of process %s: its completion is already admitted",+ event.get('event_id'), process)+ return None+ pending = await asyncio.to_thread(list_session_admissions, authority.db, session_id=owner)+ waiting = next((row for row in pending if _queued_tick(row, process, entry.session_key)), None)+ earlier = waiting and getattr(runner, '_admitted_heartbeat_ticks', {}).get(waiting['request_id'])+ if earlier is None or not await _withdraw(authority, waiting):+ return eventfrom tools.process_registry_notifications import join_heartbeat_outputsreturn {**event, **join_heartbeat_outputs(earlier, event)}@@ -87,7 +125,8 @@ async def withdraw_process_ticks(runner, event):"""A process's completion is in the session: its ticks still queued there would run after it asstale "still running" turns, and the completion carries the final output, so they are cancelled.A tick that already started is left alone. Never raises: the completion is already admitted, and- a retry of it takes the lost-acknowledgement path, not this one."""+ a retry of it takes the lost-acknowledgement path, not this one. The caller holds the heartbeat+ lock (``admit_process_notice``)."""import asynciofrom gateway.session_authorities import active_authoritytry:@@ -98,19 +137,19 @@ async def withdraw_process_ticks(runner, event):finished = {identity[1] for identity in _identities([producer_identity(runner, event), *(event.get('_automation_identities') or ())])if identity[0] == 'completion' and len(identity) > 1}- async with _heartbeat_lock(runner):- pending = await asyncio.to_thread(- list_session_admissions, authority.db, session_id=_route_owner(authority, entry))- for row in pending:- if any(_queued_tick(row, process, entry.session_key) for process in finished):- await _withdraw(authority, row)+ pending = await asyncio.to_thread(+ list_session_admissions, authority.db, session_id=_route_owner(authority, entry))+ for row in pending:+ if any(_queued_tick(row, process, entry.session_key) for process in finished):+ await _withdraw(authority, row)except Exception:logger.warning("Could not withdraw the waiting heartbeats of finished process %s",event.get('session_id'), exc_info=True)def _heartbeat_lock(runner):- # Two drains withdrawing the same queued tick would both carry its output.+ # A tick's check-and-admit and a completion's admit-and-withdraw: interleaved, a tick checked+ # before the completion landed is admitted after its withdrawal and runs as a stale turn.import asynciolock = runner.__dict__.get('_heartbeat_lock')if lock is None:diff --git a/tests/gateway/test_local_producer_admission.py b/tests/gateway/test_local_producer_admission.pyindex 9267901f155..b6896c627bd 100644--- a/tests/gateway/test_local_producer_admission.py+++ b/tests/gateway/test_local_producer_admission.py@@ -278,6 +278,32 @@ def _automation_rows(messaging):return {row['request_id']: row for row in rows if row['principal_id'] == 'automation:' + messaging.route}+async def test_a_tick_admitted_while_its_completion_lands_never_queues_behind_it(messaging, monkeypatch):+ """The tick's check-and-admit and the completion's admit-and-withdraw are one critical section: a+ completion that lands while a checked tick is still on its way in withdraws it once it is in."""+ authority, gate, entered = messaging.authority, asyncio.Event(), asyncio.Event()+ admit = authority.admit_automation++ async def held(adapter, event, identity, **kwargs):+ if identity.startswith('["heartbeat"'):+ entered.set()+ await gate.wait()+ return await admit(adapter, event, identity, **kwargs)+ monkeypatch.setattr(authority, 'admit_automation', held)++ tick = asyncio.ensure_future(_drain_one(messaging.runner, messaging.event('heartbeat', **_tick_fields('proc_job', 1))))+ await asyncio.wait_for(entered.wait(), 10)+ exited = messaging.event('completion', exit_code=0, output='done')+ completion = asyncio.ensure_future(messaging.runner._inject_watch_notification('[proc_job finished]', exited))+ # Unserialized, the completion admits and withdraws (nothing yet) while the tick is held.+ await asyncio.wait({completion}, timeout=1)+ gate.set()+ assert (await tick, await completion) == (0, True)+ assert {key: row['outcome'] for key, row in _automation_rows(messaging).items()} == {+ '["heartbeat","proc_job","proc_job-tick-1"]': 'cancelled', '["completion","proc_job",1.0]': None}++@pytest.mark.asyncioasync def test_job_output_that_looks_like_a_cut_marker_keeps_the_earlier_output(messaging):"""The omitted count rides the tick as its own field; text the job printed is never read as one."""