Skip to content

The agent that grows with you

fix(gateway): a tick admitted while its completion lands is withdrawn, never left behind it

Teknium58fef698c8cc
diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py
index 989b92a3a10..312f0d27098 100644
--- a/gateway/run_notifications.py
+++ b/gateway/run_notifications.py
@@ -717,25 +717,18 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
if self._load_background_notifications_mode() == "off":
continue
if evt.get("type") == "heartbeat":
- from gateway.session_automation import prepare_heartbeat
+ from gateway.session_automation import deliver_heartbeat
+ delivered = await deliver_heartbeat(self, evt)
+ else:
+ synth_text = _format_gateway_process_notification(evt)
+ if not synth_text:
+ continue
try:
- prepared = await prepare_heartbeat(self, evt)
+ delivered = await self._inject_watch_notification(synth_text, evt)
except Exception:
- # One tick's failure must not drop the rest of the drained events.
- logger.exception("Heartbeat preparation error; the tick is requeued")
- completion_queue.put(evt)
- continue
- if prepared is None:
- continue
- evt = prepared
- synth_text = _format_gateway_process_notification(evt)
- if not synth_text:
- continue
- try:
- delivered = await self._inject_watch_notification(synth_text, evt)
- except Exception:
- logger.exception("Watch notification injection error")
- delivered = False
+ logger.exception("Watch notification injection error")
+ delivered = False
+ # The original event: a requeued tick folds again from what is still queued.
if delivered is False:
completion_queue.put(evt)
@@ -904,15 +897,10 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
from gateway.session_authorities import active_authority
authority = active_authority(self)
if authority is not None:
- from gateway.session_automation import producer_identity, remember_heartbeat, withdraw_process_ticks
+ from gateway.session_automation import admit_process_notice
from hermes_state_runtime import RuntimeStoreError
try:
- identity = producer_identity(self, evt)
- await authority.admit_automation(adapter, synth_event, identity)
- if evt.get("type") == "heartbeat":
- remember_heartbeat(self, identity, evt)
- elif evt.get("type") == "completion":
- await withdraw_process_ticks(self, evt)
+ await admit_process_notice(self, authority, adapter, synth_event, evt)
except RuntimeStoreError as exc:
raise WakeNotAccepted(str(exc)) from exc
else:
diff --git a/gateway/session_automation.py b/gateway/session_automation.py
index af9333cdf08..1ec016c0425 100644
--- a/gateway/session_automation.py
+++ b/gateway/session_automation.py
@@ -53,9 +53,48 @@ def remember_heartbeat(runner, identity, event):
ticks.popitem(last=False)
+async def deliver_heartbeat(runner, event):
+ """Prepare, format and admit one liveness tick as one critical section with a completion's
+ admit-and-withdraw (``admit_process_notice``): a completion that lands while this tick is on its
+ way in withdraws it once it is in, and a tick checked after it sees it. True when admitted, None
+ to drop it, False to requeue the original event."""
+ from gateway.run import _format_gateway_process_notification
+ async with _heartbeat_lock(runner):
+ try:
+ prepared = await prepare_heartbeat(runner, event)
+ except Exception:
+ # One tick's failure must not drop the rest of the drained events.
+ logger.exception("Heartbeat preparation error; the tick is requeued")
+ return False
+ text = prepared and _format_gateway_process_notification(prepared)
+ if not text:
+ return None
+ try:
+ return await runner._inject_watch_notification(text, prepared)
+ except Exception:
+ logger.exception("Watch notification injection error")
+ return False
+
+
+async def admit_process_notice(runner, authority, adapter, event, evt):
+ """Admit a process notice. A completion and the withdrawal of its process's waiting ticks hold
+ the heartbeat lock together; a tick already holds it (``deliver_heartbeat``)."""
+ identity = producer_identity(runner, evt)
+ if evt.get('type') != 'completion':
+ receipt = await authority.admit_automation(adapter, event, identity)
+ if evt.get('type') == 'heartbeat':
+ remember_heartbeat(runner, identity, evt)
+ return receipt
+ async with _heartbeat_lock(runner):
+ receipt = await authority.admit_automation(adapter, event, identity)
+ await withdraw_process_ticks(runner, evt)
+ return receipt
+
+
async def prepare_heartbeat(runner, event):
"""The liveness tick to admit for ``event``, or None to drop it because its process's completion
- is already in the session: a later tick would be a stale "still running" turn.
+ is already in the session: a later tick would be a stale "still running" turn. The caller holds
+ the heartbeat lock.
Otherwise a tick of the same process still queued is cancelled and its output carried into this
one, since a tick's output is never sent again: a busy session gets one waiting turn holding
@@ -69,16 +108,15 @@ async def prepare_heartbeat(runner, event):
if entry is None:
return event
owner, process = _route_owner(authority, entry), str(event.get('session_id') or '')
- async with _heartbeat_lock(runner):
- if await asyncio.to_thread(_process_completed, runner, authority, owner, event):
- logger.debug("Dropped heartbeat %s of process %s: its completion is already admitted",
- event.get('event_id'), process)
- return None
- pending = await asyncio.to_thread(list_session_admissions, authority.db, session_id=owner)
- waiting = next((row for row in pending if _queued_tick(row, process, entry.session_key)), None)
- earlier = waiting and getattr(runner, '_admitted_heartbeat_ticks', {}).get(waiting['request_id'])
- if earlier is None or not await _withdraw(authority, waiting):
- return event
+ if await asyncio.to_thread(_process_completed, runner, authority, owner, event):
+ logger.debug("Dropped heartbeat %s of process %s: its completion is already admitted",
+ event.get('event_id'), process)
+ return None
+ pending = await asyncio.to_thread(list_session_admissions, authority.db, session_id=owner)
+ waiting = next((row for row in pending if _queued_tick(row, process, entry.session_key)), None)
+ earlier = waiting and getattr(runner, '_admitted_heartbeat_ticks', {}).get(waiting['request_id'])
+ if earlier is None or not await _withdraw(authority, waiting):
+ return event
from tools.process_registry_notifications import join_heartbeat_outputs
return {**event, **join_heartbeat_outputs(earlier, event)}
@@ -87,7 +125,8 @@ async def withdraw_process_ticks(runner, event):
"""A process's completion is in the session: its ticks still queued there would run after it as
stale "still running" turns, and the completion carries the final output, so they are cancelled.
A tick that already started is left alone. Never raises: the completion is already admitted, and
- a retry of it takes the lost-acknowledgement path, not this one."""
+ a retry of it takes the lost-acknowledgement path, not this one. The caller holds the heartbeat
+ lock (``admit_process_notice``)."""
import asyncio
from gateway.session_authorities import active_authority
try:
@@ -98,19 +137,19 @@ async def withdraw_process_ticks(runner, event):
finished = {identity[1] for identity in _identities(
[producer_identity(runner, event), *(event.get('_automation_identities') or ())])
if identity[0] == 'completion' and len(identity) > 1}
- async with _heartbeat_lock(runner):
- pending = await asyncio.to_thread(
- list_session_admissions, authority.db, session_id=_route_owner(authority, entry))
- for row in pending:
- if any(_queued_tick(row, process, entry.session_key) for process in finished):
- await _withdraw(authority, row)
+ pending = await asyncio.to_thread(
+ list_session_admissions, authority.db, session_id=_route_owner(authority, entry))
+ for row in pending:
+ if any(_queued_tick(row, process, entry.session_key) for process in finished):
+ await _withdraw(authority, row)
except Exception:
logger.warning("Could not withdraw the waiting heartbeats of finished process %s",
event.get('session_id'), exc_info=True)
def _heartbeat_lock(runner):
- # Two drains withdrawing the same queued tick would both carry its output.
+ # A tick's check-and-admit and a completion's admit-and-withdraw: interleaved, a tick checked
+ # before the completion landed is admitted after its withdrawal and runs as a stale turn.
import asyncio
lock = runner.__dict__.get('_heartbeat_lock')
if lock is None:
diff --git a/tests/gateway/test_local_producer_admission.py b/tests/gateway/test_local_producer_admission.py
index 9267901f155..b6896c627bd 100644
--- a/tests/gateway/test_local_producer_admission.py
+++ b/tests/gateway/test_local_producer_admission.py
@@ -278,6 +278,32 @@ def _automation_rows(messaging):
return {row['request_id']: row for row in rows if row['principal_id'] == 'automation:' + messaging.route}
+async def test_a_tick_admitted_while_its_completion_lands_never_queues_behind_it(messaging, monkeypatch):
+ """The tick's check-and-admit and the completion's admit-and-withdraw are one critical section: a
+ completion that lands while a checked tick is still on its way in withdraws it once it is in."""
+ authority, gate, entered = messaging.authority, asyncio.Event(), asyncio.Event()
+ admit = authority.admit_automation
+
+ async def held(adapter, event, identity, **kwargs):
+ if identity.startswith('["heartbeat"'):
+ entered.set()
+ await gate.wait()
+ return await admit(adapter, event, identity, **kwargs)
+ monkeypatch.setattr(authority, 'admit_automation', held)
+
+ tick = asyncio.ensure_future(_drain_one(messaging.runner, messaging.event('heartbeat', **_tick_fields('proc_job', 1))))
+ await asyncio.wait_for(entered.wait(), 10)
+ exited = messaging.event('completion', exit_code=0, output='done')
+ completion = asyncio.ensure_future(messaging.runner._inject_watch_notification('[proc_job finished]', exited))
+ # Unserialized, the completion admits and withdraws (nothing yet) while the tick is held.
+ await asyncio.wait({completion}, timeout=1)
+ gate.set()
+ assert (await tick, await completion) == (0, True)
+ assert {key: row['outcome'] for key, row in _automation_rows(messaging).items()} == {
+ '["heartbeat","proc_job","proc_job-tick-1"]': 'cancelled', '["completion","proc_job",1.0]': None}
+
+
@pytest.mark.asyncio
async def test_job_output_that_looks_like_a_cut_marker_keeps_the_earlier_output(messaging):
"""The omitted count rides the tick as its own field; text the job printed is never read as one."""