Skip to content

The agent that grows with you

fix(gateway): background completions that pile up behind a busy turn run as one turn

Teknium343a5f7f58d8
diff --git a/gateway/session_automation.py b/gateway/session_automation.py
index f19db6fcc84..7621c64380a 100644
--- a/gateway/session_automation.py
+++ b/gateway/session_automation.py
@@ -7,6 +7,7 @@ from copy import deepcopy
from datetime import UTC, datetime
import json
import logging
+import re
from gateway.config import Platform
from gateway.platforms.event import MessageEvent, MessageType
@@ -94,11 +95,76 @@ async def admit_process_notice(runner, authority, adapter, event, evt):
if evt.get('type') != 'completion':
return await authority.admit_automation(adapter, event, identity)
async with _heartbeat_lock(runner):
- receipt = await authority.admit_automation(adapter, event, identity)
+ receipt = await _admit_folded_completion(authority, adapter, event, evt, identity)
await withdraw_process_ticks(runner, evt)
return receipt
+async def _admit_folded_completion(authority, adapter, event, evt, identity):
+ """A completion landing behind another completion still waiting at the FIFO tail joins it, so a
+ busy session runs ONE turn for every result that piled up instead of one turn each. The waiting
+ row is cancelled in this admission's own transaction (``replaces``); when it was claimed first,
+ this completion queues alone."""
+ import asyncio
+ waiting = await asyncio.to_thread(_waiting_completion, authority, evt, event)
+ if waiting is not None:
+ unfolded = event.text, dict(event.metadata)
+ _fold_completion(event, waiting, identity)
+ try:
+ return await authority.admit_automation(adapter, event, identity, replaces=waiting['admission_id'])
+ except RuntimeStoreError as exc:
+ if exc.reason != 'stale_generation':
+ raise
+ event.text, event.metadata = unfolded
+ return await authority.admit_automation(adapter, event, identity)
+
+
+def _waiting_completion(authority, evt, event):
+ """The route's own completion row that is the LAST queued input of the session, or None. Only the
+ tail folds: a completion never jumps ahead of, or behind, a prompt queued after it."""
+ from gateway.session_detached import route_entry
+ entry = route_entry(authority, str(evt.get('session_key') or ''))
+ if entry is None:
+ return None
+ queued = [row for row in list_session_admissions(authority.db, session_id=_route_owner(authority, entry))
+ if row['status'] == 'queued']
+ tail = queued[-1] if queued else None
+ if tail is None or tail['principal_id'] != 'automation:' + entry.session_key:
+ return None
+ if [identity[0] for identity in _identities([tail['request_id']])] != ['completion']:
+ return None
+ descriptor = _descriptor(tail['payload'])
+ if (descriptor.get('notification_category', 'result') != event.metadata.get('notification_category', 'result')
+ or descriptor.get('display_kind') != event.metadata.get('display_kind')):
+ return None
+ return tail
+
+
+_FOLD_HEADER = ('[IMPORTANT: {count} background process results arrived while this session was busy. '
+ 'Treat them as one batch and give one consolidated response; preserve failures and '
+ 'actionable results.]')
+_FOLD_HEADER_RE = re.compile(r'\[IMPORTANT: \d+ background process results arrived while this session was busy\.'
+ r'[^\]]*\]\n\n')
+
+
+def _fold_completion(event, waiting, identity):
+ from gateway.run_notifications import INTERNAL_NOTIFICATION_FOOTER, _mark_internal_notification
+ from tools.process_registry_notifications import process_completion_display_text
+ descriptor = _descriptor(waiting['payload'])
+ identities = sorted({*(descriptor.get('identities') or [waiting['request_id']]),
+ *(event.metadata.get('automation_identities') or [identity])})
+
+ def body(text):
+ text = text.rstrip().removesuffix(INTERNAL_NOTIFICATION_FOOTER).rstrip()
+ folded = _FOLD_HEADER_RE.match(text)
+ return text[folded.end():] if folded else text
+ event.text = _mark_internal_notification('\n\n'.join((
+ _FOLD_HEADER.format(count=len(identities)), body(waiting['payload']['text']), body(event.text))))
+ event.metadata['automation_identities'] = identities
+ if 'display_text' in event.metadata:
+ event.metadata['display_text'] = process_completion_display_text([{}] * len(identities))
+
+
async def prepare_heartbeat(runner, event):
"""The liveness tick to admit for ``event``, or None to drop it because its process's completion
is already in the session: a later tick would be a stale "still running" turn. The caller holds
diff --git a/tests/gateway/test_completion_fold.py b/tests/gateway/test_completion_fold.py
new file mode 100644
index 00000000000..599dec6f434
--- /dev/null
+++ b/tests/gateway/test_completion_fold.py
@@ -0,0 +1,77 @@
+"""Process completions that pile up behind a busy turn run as ONE queued turn, not one turn each."""
+import pytest
+
+from gateway.config import GatewayConfig, Platform, PlatformConfig
+from gateway.platforms.event import MessageEvent
+from gateway.run import GatewayRunner
+from gateway.session_authority import SessionAuthority, initialize_session_authority
+from hermes_state_runtime import list_session_admissions
+
+
+async def _discord_session(tmp_path, monkeypatch):
+ from gateway.session_ingress_context import native_callback, register_transport_home
+ from hermes_state import SessionDB
+ from plugins.platforms.discord.adapter import DiscordAdapter
+ monkeypatch.setenv('HERMES_HOME', str(tmp_path))
+ monkeypatch.setenv('DISCORD_ALLOWED_USERS', '42')
+ monkeypatch.setattr(SessionAuthority, '_schedule', lambda self, ref: None) # nothing claims: all stay queued
+ runner = GatewayRunner(GatewayConfig(sessions_dir=tmp_path / 'sessions'))
+ db = SessionDB(tmp_path / 'state.db')
+ runner.session_store._db = runner._session_db = db
+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='fold', db=db)
+ adapter = DiscordAdapter(PlatformConfig(enabled=True, token='fixture-token', typing_indicator=False))
+ runner.adapters = {Platform.DISCORD: adapter}
+ runner._wire_adapter_handlers(adapter)
+ register_transport_home(runner, None, tmp_path)
+ source = adapter.build_source(chat_id='42', chat_type='dm', user_id='42')
+
+ async def human(text):
+ event = MessageEvent(text=text, source=source, message_id=text)
+ with native_callback(runner, event, tmp_path):
+ await authority.admit_native(event)
+ await human('running turn')
+ route = runner.session_store._generate_session_key(source)
+ sid = runner.session_store.peek_session_id(route)
+ owner = authority.logical_owner(sid)
+
+ async def complete(name):
+ evt = {'type': 'completion', 'session_id': name, 'started_at': 1.0, 'session_key': route, 'parent_session_id': sid,
+ 'platform': 'discord', 'chat_type': 'dm', 'chat_id': '42', 'user_id': '42',
+ 'exit_code': 0, 'output': f'{name} output'}
+ assert await runner._inject_watch_notification(f'[IMPORTANT: {name} finished]', evt) is True
+
+ def queued():
+ return [row for row in list_session_admissions(authority.db, session_id=owner) if row['status'] == 'queued']
+ return authority, human, complete, queued
+
+
+async def test_completions_queued_behind_a_turn_fold_into_one_turn(tmp_path, monkeypatch):
+ authority, _human, complete, queued = await _discord_session(tmp_path, monkeypatch)
+ try:
+ for name in ('proc_a', 'proc_b', 'proc_c'):
+ await complete(name)
+ rows = queued()
+ assert [row['principal_id'].split(':')[0] for row in rows] == ['messaging', 'automation']
+ text = rows[-1]['payload']['text']
+ assert text.startswith('[IMPORTANT: 3 background process results arrived')
+ assert text.count('background process results arrived') == 1 # re-folding replaces the header
+ assert all(f'{name} finished' in text for name in ('proc_a', 'proc_b', 'proc_c'))
+ assert text.count('INTERNAL NOTIFICATION') == 1
+ assert len(rows[-1]['payload']['native_text_v1']['automation']['identities']) == 3
+ finally:
+ authority.db.close()
+
+
+async def test_a_prompt_queued_between_completions_keeps_them_apart(tmp_path, monkeypatch):
+ authority, human, complete, queued = await _discord_session(tmp_path, monkeypatch)
+ try:
+ await complete('proc_a')
+ await human('queued follow-up')
+ await complete('proc_b')
+ rows = queued()
+ assert [row['payload']['text'].split('\n')[0] for row in rows] == [
+ 'running turn', '[IMPORTANT: proc_a finished]', 'queued follow-up', '[IMPORTANT: proc_b finished]']
+ finally:
+ authority.db.close()
diff --git a/tests/gateway/test_unviewed_completion_notes.py b/tests/gateway/test_unviewed_completion_notes.py
index 68ab248d940..5561c249745 100644
--- a/tests/gateway/test_unviewed_completion_notes.py
+++ b/tests/gateway/test_unviewed_completion_notes.py
@@ -63,14 +63,17 @@ async def test_busy_session_keeps_the_completion_durably_until_its_turn(tmp_path
await _deliver(runner, authority, ref, _PROCESS)
evt = _process_event(authority, ref, target)
assert await runner._deliver_completion_notification(f'[finished] RESULT_{evt["session_id"]}', evt) is True
- assert [r['status'] for r in _admissions(authority, ref)] == ['queued', 'queued']
+ # The second completion folds into the one still waiting at the FIFO tail: one turn for both.
+ assert [r['status'] for r in _admissions(authority, ref)] == ['terminal', 'queued']
assert _notes(authority, target) == []
seen = _turns(runner, authority, target)
await authority._drain(ref)
assert seen == [('FIRST', 0)], 'the human input runs first and sees no note yet'
- assert len(_notes(authority, target)) == 2
- assert [r['outcome'] for r in _admissions(authority, ref)] == ['completed', 'completed']
+ notes = _notes(authority, target)
+ assert len(notes) == 1 and all(f'RESULT_{name}' in str(notes[0]['content'])
+ for name in (_PROCESS[1], evt['session_id']))
+ assert [r['outcome'] for r in _admissions(authority, ref)] == ['cancelled', 'completed']
finally:
authority.db.close()
diff --git a/website/docs/developer-guide/gateway-internals.md b/website/docs/developer-guide/gateway-internals.md
index 99b18a350fe..4d8af44658b 100644
--- a/website/docs/developer-guide/gateway-internals.md
+++ b/website/docs/developer-guide/gateway-internals.md
@@ -709,6 +709,13 @@ acceptance. Refused admission refunds every claimed batch sibling without spendi
actual delivery errors keep their bounded retry policy. Recognized raw API routes resolve after
persisted messaging origins and defer quietly when unavailable; malformed routes still warn.
+Completions that land while the session is busy fold into ONE queued turn
+(`session_automation.py::_admit_folded_completion`): a completion whose route already has a
+completion waiting at the very end of the FIFO replaces it with a combined admission (both texts,
+every producer identity, a "N background process results" header), cancelling the waiting row in
+the same transaction. Only the tail folds, so a prompt queued after a completion keeps its place
+between them; a waiting row claimed first leaves the new completion to queue alone.
+
Cron execution has its own session. Eligible continuable deliveries may mirror or seed the
reply-facing conversation: origin, origin-less home fallback, user-written bare-platform home,
or opted-in explicit targets. `all` expansions do not gain home mirror eligibility. Mirrored