Skip to content

The agent that grows with you

fix(gateway): deliver background-process heartbeats again, in every chat

David Dudok de Wit6232d873f558
diff --git a/gateway/response_filters.py b/gateway/response_filters.py
index 7b97674e7c5..d4a00defa2d 100644
--- a/gateway/response_filters.py
+++ b/gateway/response_filters.py
@@ -31,10 +31,14 @@ _BRACKETED_SILENCE_MARKERS = tuple(
# The persisted user-row kind of a self-injected MessageEvent(internal=True) turn; only these
# machinery kinds may vanish on a bare silence marker.
INTERNAL_NOTIFICATION_DISPLAY_KIND = "internal_notification"
-# Async-result cards a gateway producer may name on its own internal event (with a display_text),
-# the same rows the in-process TUI persisted. A closed set: never a client-chosen presentation.
+# Async-result cards the in-process TUI persisted: a producer's internal event naming one (with a
+# display_text) persists as that row kind instead of internal_notification.
PRODUCER_NOTICE_DISPLAY_KINDS = frozenset({"process_complete", "async_delegation_complete"})
MACHINERY_DISPLAY_KINDS = frozenset({INTERNAL_NOTIFICATION_DISPLAY_KIND, *PRODUCER_NOTICE_DISPLAY_KINDS})
+# What a gateway producer's card may name (a liveness tick names internal_notification for its CLI
+# receipt). Closed, and kept apart from the silence set above so a kind added there never becomes
+# nameable; never a client-chosen presentation such as ``hidden``.
+PRODUCER_CARD_KINDS = frozenset({INTERNAL_NOTIFICATION_DISPLAY_KIND, *PRODUCER_NOTICE_DISPLAY_KINDS})
# Longer than any marker could plausibly be, even with stray punctuation.
_MARKER_LENGTH_CAP = 64
diff --git a/gateway/run_notifications.py b/gateway/run_notifications.py
index d62859f1062..989b92a3a10 100644
--- a/gateway/run_notifications.py
+++ b/gateway/run_notifications.py
@@ -111,9 +111,10 @@ def _raw_process_event_session_id(evt: dict) -> str:
def _local_completion_card(evt: dict) -> dict[str, str]:
- """The async-result card main's in-process TUI wrote for a completion; the claim's
- ``message.start`` carries it, and an idle classic CLI only opens a turn that has one (a
- gateway-side ``/review`` reports as an async delegation). ``{}`` for a watch event."""
+ """The card of a local process notice: the async-result card the in-process TUI wrote for a
+ completion or async delegation (a gateway-side ``/review`` reports as one), and a liveness tick's
+ one-line receipt. The claim's ``message.start`` carries it, and an idle classic CLI only opens a
+ turn that has one. ``{}`` for a watch match."""
from tools.process_registry_notifications import (
PROCESS_COMPLETE_DISPLAY_KIND, async_delegation_display_text, process_completion_display_text)
evt_type = evt.get("type", "completion")
@@ -122,6 +123,12 @@ def _local_completion_card(evt: dict) -> dict[str, str]:
"display_text": evt.get("_display_text") or process_completion_display_text([evt])}
if evt_type == "async_delegation":
return {"display_kind": "async_delegation_complete", "display_text": async_delegation_display_text(evt)}
+ if evt_type == "heartbeat":
+ # The wake stays machinery (no bubble, and a quiet tick may end in silence); the receipt lets an
+ # idle CLI open the turn with one line and show the agent's reply.
+ from gateway.response_filters import INTERNAL_NOTIFICATION_DISPLAY_KIND
+ from tools.process_registry_notifications import heartbeat_display_text
+ return {"display_kind": INTERNAL_NOTIFICATION_DISPLAY_KIND, "display_text": heartbeat_display_text(evt)}
return {}
@@ -141,8 +148,8 @@ def _watch_notification_metadata(evt: dict, source) -> dict[str, Any]:
if evt.get('_automation_identities'):
metadata['automation_identities'] = evt['_automation_identities']
if source.platform == Platform.LOCAL:
- # A local completion persists as the async-result card the in-process TUI wrote;
- # messaging rows stay internal_notification, as on main.
+ # A local completion persists as the async-result card the in-process TUI wrote, and a tick
+ # carries its CLI receipt; messaging rows stay internal_notification, as on main.
metadata.update(_local_completion_card(evt))
proven_session_id = str(evt.get("proven_session_id") or "").strip()
if proven_session_id:
@@ -709,6 +716,18 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
continue
if self._load_background_notifications_mode() == "off":
continue
+ if evt.get("type") == "heartbeat":
+ from gateway.session_automation import prepare_heartbeat
+ try:
+ prepared = await prepare_heartbeat(self, evt)
+ except Exception:
+ # One tick's failure must not drop the rest of the drained events.
+ logger.exception("Heartbeat preparation error; the tick is requeued")
+ completion_queue.put(evt)
+ continue
+ if prepared is None:
+ continue
+ evt = prepared
synth_text = _format_gateway_process_notification(evt)
if not synth_text:
continue
@@ -885,11 +904,15 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
from gateway.session_authorities import active_authority
authority = active_authority(self)
if authority is not None:
- from gateway.session_automation import producer_identity
+ from gateway.session_automation import producer_identity, remember_heartbeat, withdraw_process_ticks
from hermes_state_runtime import RuntimeStoreError
try:
identity = producer_identity(self, evt)
await authority.admit_automation(adapter, synth_event, identity)
+ if evt.get("type") == "heartbeat":
+ remember_heartbeat(self, identity, evt)
+ elif evt.get("type") == "completion":
+ await withdraw_process_ticks(self, evt)
except RuntimeStoreError as exc:
raise WakeNotAccepted(str(exc)) from exc
else:
@@ -925,7 +948,9 @@ class GatewayNotificationsMixin(GatewayUpdateNotificationsMixin, GatewayNotifica
task_idx = ((evt.get("results") or [{}])[0] or {}).get("task_index", "")
return (evt_type, producer_id, f"task_failure:{task_idx}")
return (evt_type, producer_id, "")
- if evt_type in {"watch_match", "watch_disabled"} and evt.get("event_id"):
+ # A liveness tick is one watch event like a match: without an identity the session authority
+ # refuses it and the drain requeues it forever.
+ if evt_type in {"watch_match", "watch_disabled", "heartbeat"} and evt.get("event_id"):
return (evt_type, str(evt["session_id"]), str(evt["event_id"]))
if evt_type == "completion":
producer_id = str(evt.get("session_id") or "")
diff --git a/gateway/session_automation.py b/gateway/session_automation.py
index 7973b13a7f4..cff6e714d09 100644
--- a/gateway/session_automation.py
+++ b/gateway/session_automation.py
@@ -3,14 +3,18 @@
Reuse the private native route envelope and FIFO, retaining nonhuman turn semantics.
A producer ACK means committed input, not successful inference or outbound delivery.
"""
+from collections import OrderedDict
from copy import deepcopy
from datetime import UTC, datetime
import json
+import logging
from gateway.config import Platform
from gateway.platforms.event import MessageEvent, MessageType
from gateway.session_contract import SessionRef
-from hermes_state_runtime import RuntimeStoreError, admit_session_input
+from hermes_state_runtime import RuntimeStoreError, admit_session_input, list_session_admissions
+
+logger = logging.getLogger(__name__)
def producer_identity(runner, event):
@@ -30,16 +34,152 @@ def completion_admission(runner, event):
return None
from hermes_state_runtime import list_session_admissions
identity = producer_identity(runner, event)
- # Compression moves the route's transcript to a child; admissions stay on the logical root.
- sid = (entry.origin.chat_id if entry.origin.platform == Platform.LOCAL
- else authority.logical_owner(entry.session_id))
- for row in list_session_admissions(authority.db, session_id=sid, pending_only=False):
+ for row in list_session_admissions(authority.db, session_id=_route_owner(authority, entry), pending_only=False):
descriptor = row['payload'].get('local_automation_v1') or row['payload'].get('native_text_v1', {}).get('automation', {})
if identity in descriptor.get('identities', [descriptor.get('identity')]):
return row
return None
+# Ticks this runner admitted, by identity: a later tick of the same process folds in their output.
+# Bounded, since only a tick still queued is ever read back; a forgotten one only means no fold.
+_REMEMBERED_TICKS = 64
+
+
+def remember_heartbeat(runner, identity, event):
+ ticks = runner.__dict__.setdefault('_admitted_heartbeat_ticks', OrderedDict())
+ ticks[identity] = event
+ while len(ticks) > _REMEMBERED_TICKS:
+ ticks.popitem(last=False)
+
+
+async def prepare_heartbeat(runner, event):
+ """The liveness tick to admit for ``event``, or None to drop it because its process's completion
+ is already in the session: a later tick would be a stale "still running" turn.
+
+ Otherwise a tick of the same process still queued is cancelled and its output carried into this
+ one, since a tick's output is never sent again: a busy session gets one waiting turn holding
+ every tick's output instead of one turn per tick. A queued tick that started or settled first
+ ran with its own output, and this one queues behind it; so it does behind a tick this runner did
+ not admit (one from before a restart)."""
+ import asyncio
+ from gateway.session_authorities import active_authority
+ authority = active_authority(runner)
+ entry = runner.session_store.lookup_by_session_key(str(event.get('session_key') or '')) if authority else None
+ if entry is None:
+ return event
+ owner, process = _route_owner(authority, entry), str(event.get('session_id') or '')
+ async with _heartbeat_lock(runner):
+ if await asyncio.to_thread(_process_completed, runner, authority, owner, event):
+ logger.debug("Dropped heartbeat %s of process %s: its completion is already admitted",
+ event.get('event_id'), process)
+ return None
+ pending = await asyncio.to_thread(list_session_admissions, authority.db, session_id=owner)
+ waiting = next((row for row in pending if _queued_tick(row, process, entry.session_key)), None)
+ earlier = waiting and getattr(runner, '_admitted_heartbeat_ticks', {}).get(waiting['request_id'])
+ if earlier is None or not await _withdraw(authority, waiting):
+ return event
+ from tools.process_registry_notifications import join_heartbeat_outputs
+ return {**event, 'output': join_heartbeat_outputs(earlier.get('output', ''), event.get('output', ''))}
+
+
+async def withdraw_process_ticks(runner, event):
+ """A process's completion is in the session: its ticks still queued there would run after it as
+ stale "still running" turns, and the completion carries the final output, so they are cancelled.
+ A tick that already started is left alone. Never raises: the completion is already admitted, and
+ a retry of it takes the lost-acknowledgement path, not this one."""
+ import asyncio
+ from gateway.session_authorities import active_authority
+ try:
+ authority = active_authority(runner)
+ entry = runner.session_store.lookup_by_session_key(str(event.get('session_key') or '')) if authority else None
+ if entry is None:
+ return
+ finished = {identity[1] for identity in _identities(
+ [producer_identity(runner, event), *(event.get('_automation_identities') or ())])
+ if identity[0] == 'completion' and len(identity) > 1}
+ async with _heartbeat_lock(runner):
+ pending = await asyncio.to_thread(
+ list_session_admissions, authority.db, session_id=_route_owner(authority, entry))
+ for row in pending:
+ if any(_queued_tick(row, process, entry.session_key) for process in finished):
+ await _withdraw(authority, row)
+ except Exception:
+ logger.warning("Could not withdraw the waiting heartbeats of finished process %s",
+ event.get('session_id'), exc_info=True)
+
+
+def _heartbeat_lock(runner):
+ # Two drains withdrawing the same queued tick would both carry its output.
+ import asyncio
+ lock = runner.__dict__.get('_heartbeat_lock')
+ if lock is None:
+ lock = runner.__dict__['_heartbeat_lock'] = asyncio.Lock()
+ return lock
+
+
+def _process_completed(runner, authority, owner, event):
+ """Whether the completion of tick ``event``'s process is admitted into the session ``owner``: by
+ its exact identity (``started_at`` names the process incarnation), or as a sibling in a pending
+ batch."""
+ completion = runner._completion_delivery_identity(
+ {'type': 'completion', 'session_id': event.get('session_id'), 'started_at': event.get('started_at')})
+ if completion is None:
+ return False
+ identity = json.dumps(completion, separators=(',', ':'))
+ with authority.db._read_ctx() as conn:
+ if conn.execute('SELECT 1 FROM session_admissions WHERE target_session_id=? AND request_id=?',
+ (owner, identity)).fetchone():
+ return True
+ for row in list_session_admissions(authority.db, session_id=owner):
+ descriptor = row['payload'].get('local_automation_v1') or row['payload'].get('native_text_v1', {}).get('automation', {})
+ if identity in (descriptor.get('identities') or ()):
+ return True
+ return False
+
+
+def _identities(raws):
+ parsed = []
+ for raw in raws:
+ try:
+ identity = json.loads(raw)
+ except (TypeError, ValueError):
+ continue
+ if isinstance(identity, list) and identity:
+ parsed.append(identity)
+ return parsed
+
+
+def _queued_tick(row, process, route):
+ """Whether *row* is the gateway's own queued liveness tick of *process* on *route*: the route's
+ producer admitted ``["heartbeat", process, event_id]``. A client input whose id merely looks like
+ one, and the ``/heartbeat`` wake (four parts), never match."""
+ if row['status'] != 'queued' or row['principal_id'] != 'automation:' + route:
+ return False
+ return any(len(identity) == 3 and identity[:2] == ['heartbeat', process]
+ for identity in _identities([row['request_id']]))
+
+
+async def _withdraw(authority, row):
+ """Cancel the gateway's own queued automation *row*; False when it was claimed or settled first."""
+ from gateway.session_contract import Principal
+ # The row's own principal, with the operator access a LOCAL route requires of anyone but its viewer.
+ actor = Principal(row['principal_id'], authority.profile_id,
+ frozenset({'session:submit', 'session:control', 'session:operator'}), 'producer-withdraw')
+ try:
+ receipt = await authority.cancel_queued(
+ actor, SessionRef(authority.profile_id, row['target_session_id']), row['admission_id'])
+ except RuntimeStoreError:
+ return False
+ return receipt.outcome == 'cancelled'
+
+
+def _route_owner(authority, entry):
+ # Compression moves the route's transcript to a child; admissions stay on the logical root.
+ return (entry.origin.chat_id if entry.origin.platform == Platform.LOCAL
+ else authority.logical_owner(entry.session_id))
+
+
def _owner(runner, event, authority):
route = event.metadata.get('gateway_session_key') or runner.session_store._generate_session_key(event.source)
entry = runner.session_store.lookup_by_session_key(route)
@@ -69,12 +209,13 @@ _DISPLAY_TEXT_LIMIT = 2000
def automation_display_metadata(metadata):
- """Validate a producer's async-result card: both keys or neither, kind from the closed set."""
+ """Validate a producer's card (an async result, or a liveness tick's one-line receipt): both keys
+ or neither, kind from the closed producer set."""
if 'display_kind' not in metadata and 'display_text' not in metadata:
return {}
- from gateway.response_filters import PRODUCER_NOTICE_DISPLAY_KINDS
+ from gateway.response_filters import PRODUCER_CARD_KINDS
kind, text = metadata.get('display_kind'), metadata.get('display_text')
- if (kind not in PRODUCER_NOTICE_DISPLAY_KINDS or not isinstance(text, str) or not text.strip()
+ if (kind not in PRODUCER_CARD_KINDS or not isinstance(text, str) or not text.strip()
or len(text) > _DISPLAY_TEXT_LIMIT):
raise RuntimeStoreError('invalid_params')
return {'display_kind': kind, 'display_text': text}
diff --git a/tests/gateway/test_local_producer_admission.py b/tests/gateway/test_local_producer_admission.py
new file mode 100644
index 00000000000..70816f1bec4
--- /dev/null
+++ b/tests/gateway/test_local_producer_admission.py
@@ -0,0 +1,325 @@
+"""Background producers reach the chat that started them, through the real authorities, adapters,
+turn path and SQLite. Only the model call is stubbed; it records the frozen session policy each turn
+resolved. Regression coverage for #139377 and #139699.
+
+- A ``terminal(background=true, heartbeat=N)`` liveness tick is admitted into the local or messaging
+ chat that started the process. A tick that finds an earlier tick of its process still queued
+ replaces it, carrying both outputs, instead of stacking another full turn, and no tick runs after
+ its process's completion.
+- A served (non-launch) profile's Bot Chat takes every relayed DM with no client attached, minted by
+ the relay door itself, also after a restart over rows that store the launch profile's transport.
+- A served profile's Desktop session admits its own process's watch match and completion.
+"""
+import asyncio
+from contextlib import closing
+import json
+import queue
+import sqlite3
+from types import SimpleNamespace
+
+import pytest
+import pytest_asyncio
+
+from gateway.config import GatewayConfig
+from gateway.run import GatewayRunner
+from hermes_state_runtime import list_session_admissions
+from tests.waiting import async_wait_until
+
+AUTHOR = {'id': 'bot:alpha', 'name': 'alpha', 'is_bot': True}
+LEGACY_DESKTOP_ID = '20261015_112020_548787'
+
+
+class _Gateway:
+ """Two profiles on one multiplexed gateway, each with its own model; restartable over its homes."""
+
+ def __init__(self, tmp_path, monkeypatch):
+ from pathlib import Path
+ self.tmp_path, root = tmp_path, tmp_path / '.hermes'
+ self.homes = (('default', root), ('beta', root / 'profiles' / 'beta'))
+ for name, home in self.homes:
+ home.mkdir(parents=True, exist_ok=True, mode=0o700)
+ (home / 'config.yaml').write_text(json.dumps({'model': {
+ 'default': f'{name}-model', 'provider': 'custom', 'base_url': 'http://127.0.0.1:9/v1',
+ 'api_key': 'loopback-only'}}))
+ monkeypatch.setattr(Path, 'home', lambda: tmp_path)
+ monkeypatch.setenv('HERMES_HOME', str(root))
+ import hermes_state
+ monkeypatch.setattr(hermes_state, 'DEFAULT_DB_PATH', hermes_state._IMPORT_DEFAULT_DB_PATH)
+ self.turns, self.gate = [], None
+ self.runner, self.connections = None, []
+
+ async def model(runner, message, context_prompt, history, source, session_id, **_kwargs):
+ from gateway.session_policy import policy_for_source
+ policy = policy_for_source(runner, source)
+ self.turns.append(SimpleNamespace(profile=source.profile, chat=source.chat_id, text=message,
+ policy=policy and (policy.model, policy.cwd)))
+ if self.gate is not None:
+ await self.gate.wait()
+ return {'final_response': 'ack', 'messages': [], 'api_calls': 1, 'tools': [],
+ 'history_offset': len(history), 'session_id': session_id}
+
+ monkeypatch.setattr(GatewayRunner, '_run_agent', model)
+
+ async def start(self):
+ from gateway.run_runtime import initialize_gateway_runtime
+ from gateway.runtime_ownership import process_ownership
+ config = GatewayConfig()
+ config.multiplex_profiles = True
+ config._runtime_profile_homes = self.homes
+ self.runner = GatewayRunner(config)
+ process_ownership.reserve([home for _, home in self.homes])
+ await initialize_gateway_runtime(self.runner)
+ self.primary, self.secondary = (self.runner.session_authorities.for_home(home) for _, home in self.homes)
+ # The launch profile has a local session of its own, as on any install that used it, so the
+ # launch profile's local adapter exists next to the served profile's.
+ self.launch = self.create(self.primary, self.connect(self.primary), 'launch', 'cli')
+
+ async def stop(self):
+ from gateway.runtime_ownership import process_ownership
+ for connection in self.connections:
+ await connection.close()
+ self.connections = []
+ for authority in getattr(self.runner, 'session_authorities', None) or []:
+ authority.db.close()
+ for _, home in self.homes:
+ process_ownership.release(home)
+
+ def connect(self, authority, **identity):
+ from gateway.session_controls import AuthorityConnection
+ connection = AuthorityConnection(authority, object(), {'user_id': 'uid:1000', **identity})
+ self.connections.append(connection)
+ return connection
+
+ def create(self, authority, connection, request_id, source, **params):
+ from gateway.session_authorities import owner_scope
+ from gateway.session_local import create_local_session
+ with owner_scope(authority):
+ return create_local_session(authority, connection.actor, dict(
+ request_id=request_id, source=source, cwd=str(self.tmp_path), toolsets=[], **params))
+
+ async def watch(self, authority, session_id):
+ """Attach a viewer to the session, as an open CLI or Desktop pane is."""
+ viewer = self.connect(authority)
+ reply = await viewer.dispatch({'id': 'watch', 'method': 'session.resume', 'params': {'session_id': session_id}})
+ assert 'result' in reply, reply
+ return viewer
+
+ def rows(self, authority, session_id):
+ return list_session_admissions(authority.db, session_id=session_id, pending_only=False)
+
+ async def settled(self, authority, session_id, count):
+ def done():
+ rows = self.rows(authority, session_id)
+ return len(rows) >= count and all(row['status'] == 'terminal' for row in rows)
+ await async_wait_until(done, timeout=30, what=f'{count} settled admissions')
+
+ def process_event(self, authority, session_id, kind, process='proc_job', **fields):
+ live = authority.sessions[session_id]
+ target = self.runner.session_store.lookup_by_session_key(live.route).session_id
+ return {'type': kind, 'session_id': process, 'started_at': 1.0, 'session_key': live.route,
+ 'parent_session_id': target, 'command': 'make test', 'output': 'tick', **fields}
+
+ def tick(self, authority, session_id, seq, process='proc_job', output=None):
+ return self.process_event(authority, session_id, 'heartbeat', process=process,
+ **_tick_fields(process, seq, output))
+
+ async def drain(self, *events):
+ waiting = queue.Queue()
+ for event in events:
+ waiting.put(event)
+ await self.runner._drain_watch_notifications(waiting)
+ return [waiting.get_nowait() for _ in range(waiting.qsize())]
+
+
+@pytest_asyncio.fixture
+async def gateway(tmp_path, monkeypatch):
+ gateway = _Gateway(tmp_path, monkeypatch)
+ await gateway.start()
+ try:
+ yield gateway
+ finally:
+ if gateway.gate is not None:
+ gateway.gate.set()
+ await gateway.stop()
+
+
+def _tick_fields(process, seq, output=None):
+ return {'event_id': f'{process}-tick-{seq}', 'seq': seq, 'interval': 60, 'elapsed': 60 * seq,
+ 'output': f'{process}/{seq}\n' if output is None else output}
+
+
+def _tick_output(text):
+ return text.split('Output since last heartbeat:\n', 1)[1].split(']', 1)[0]
+
+
+async def test_a_liveness_tick_is_admitted_into_the_session_that_started_the_process(gateway):
+ from gateway.session_automation import admission_display
+ from hermes_cli.gateway_tools_notify import _card
+ sid = gateway.launch.session_id
+
+ assert await gateway.drain(gateway.tick(gateway.primary, sid, 1)) == [], 'the tick was refused and requeued'
+ await gateway.settled(gateway.primary, sid, 1)
+ [turn] = gateway.turns
+ assert (turn.chat, turn.text.startswith('[Background process proc_job heartbeat #1 ')) == (sid, True)
+ # The claim's message.start card: the wake stays machinery, and an idle CLI opens the turn with a
+ # one-line receipt.
+ [row] = gateway.rows(gateway.primary, sid)
+ card = _card({'payload': admission_display(row['payload'])})
+ assert card is not None and (card['display_kind'], card['display_text']) == (
+ 'internal_notification', 'Background Process Output after 1m: make test')
+
+
+async def test_waiting_ticks_fold_into_one_turn_and_none_outlives_its_process(gateway):
+ primary, sid = gateway.primary, gateway.launch.session_id
+ viewer = await gateway.watch(primary, sid)
+ gateway.gate = asyncio.Event()
+ await gateway.drain(gateway.tick(primary, sid, 1, process='proc_1'))
+ await async_wait_until(lambda: gateway.turns, timeout=30, what='the first tick running')
+
+ # While proc_1's first tick runs: proc_12 queues two ticks (they fold), the viewer queues inputs
+ # whose ids look like ticks, then proc_1 queues two ticks of its own (proc_1 is a prefix of
+ # proc_12), the second carrying both outputs past the 2,000-character tail.
+ for seq in (1, 2):
+ assert await gateway.drain(gateway.tick(primary, sid, seq, process='proc_12')) == []
+ for count, lookalike in enumerate(('["heartbeat","proc_12","typed"]', '["heartbeat" typed'), start=1):
+ submitted = await viewer.dispatch({'id': count, 'method': 'prompt.submit', 'params': {
+ 'session_id': sid, 'input_id': lookalike, 'text': 'typed'}})
+ assert 'result' in submitted, submitted
+ first_cut = '...(300 earlier characters omitted)\n' + 'a' * 2000
+ for seq, output in ((2, first_cut), (3, 'b' * 1000)):
+ assert await gateway.drain(gateway.tick(primary, sid, seq, process='proc_1', output=output)) == []
+ # proc_12 exits: its waiting tick is withdrawn, and a late tick is dropped.
+ exited = gateway.process_event(primary, sid, 'completion', process='proc_12', exit_code=0)
+ assert await gateway.runner._deliver_completion_notification('[proc_12 finished]', exited) is True
+ assert await gateway.drain(gateway.tick(primary, sid, 3, process='proc_12')) == []
+ gateway.gate.set()
+ await gateway.settled(primary, sid, 8)
+
+ assert {row['request_id']: row['outcome'] for row in gateway.rows(primary, sid)} == {
+ '["heartbeat","proc_1","proc_1-tick-1"]': 'completed',
+ '["heartbeat","proc_12","proc_12-tick-1"]': 'cancelled',
+ '["heartbeat","proc_12","proc_12-tick-2"]': 'cancelled',
+ '["heartbeat","proc_12","typed"]': 'completed', '["heartbeat" typed': 'completed',
+ '["heartbeat","proc_1","proc_1-tick-2"]': 'cancelled',
+ '["heartbeat","proc_1","proc_1-tick-3"]': 'completed',
+ '["completion","proc_12",1.0]': 'completed'}
+ assert [_tick_output(turn.text) for turn in gateway.turns if 'heartbeat #' in turn.text] == [
+ 'proc_1/1\n', '...(1300 earlier characters omitted)\n' + 'a' * 1000 + 'b' * 1000]
+
+
+@pytest_asyncio.fixture
+async def messaging(tmp_path, monkeypatch):
+ """A Discord DM under the session authority with one committed human message; inputs stay queued."""
+ from gateway.config import Platform, PlatformConfig
+ from gateway.platforms.event import MessageEvent
+ from gateway.session_authority import SessionAuthority, initialize_session_authority
+ from gateway.session_ingress_context import native_callback, register_transport_home
+ from hermes_state import SessionDB
+ from plugins.platforms.discord.adapter import DiscordAdapter
+ monkeypatch.setenv('HERMES_HOME', str(tmp_path))
+ monkeypatch.setenv('DISCORD_ALLOWED_USERS', '42')
+ monkeypatch.setattr(SessionAuthority, '_schedule', lambda self, ref: None)
+ runner = GatewayRunner(GatewayConfig(sessions_dir=tmp_path / 'sessions'))
+ db = SessionDB(tmp_path / 'state.db')
+ runner.session_store._db = runner._session_db = db
+ authority = await initialize_session_authority(runner, profile_id='default', instance_id='ticks', db=db)
+ adapter = DiscordAdapter(PlatformConfig(enabled=True, token='***', typing_indicator=False))
+ runner.adapters = {Platform.DISCORD: adapter}
+ runner._wire_adapter_handlers(adapter)
+ register_transport_home(runner, None, tmp_path)
+ chat = dict(chat_id='42', chat_type='dm', user_id='42', user_name='human')
+ human = MessageEvent(text='human', source=adapter.build_source(**chat, message_id='m1'), message_id='m1')
+ with native_callback(runner, human, tmp_path):
+ await authority.admit_native(human)
+ route = runner.session_store._generate_session_key(adapter.build_source(**chat))
+ owner = authority.logical_owner(runner.session_store.peek_session_id(route))
+ try:
+ yield SimpleNamespace(runner=runner, authority=authority, route=route, owner=owner, event=lambda kind, **fields: {
+ 'type': kind, 'session_id': 'proc_job', 'started_at': 1.0, 'session_key': route, 'parent_session_id': owner,
+ 'platform': 'discord', 'command': 'make test', **chat, **fields})
+ finally:
+ db.close()
+
+
+async def test_a_messaging_chat_gets_its_ticks_folded_and_none_after_the_completion(messaging):
+ async def drain(event):
+ waiting = queue.Queue()
+ waiting.put(event)
+ await messaging.runner._drain_watch_notifications(waiting)
+ return waiting.qsize()
+
+ for seq in (1, 2):
+ assert await drain(messaging.event('heartbeat', **_tick_fields('proc_job', seq))) == 0, 'refused and requeued'
+ exited = messaging.event('completion', exit_code=0, output='done')
+ assert await messaging.runner._inject_watch_notification('[proc_job finished]', exited) is True
+ assert await drain(messaging.event('heartbeat', **_tick_fields('proc_job', 3))) == 0
+
+ rows = list_session_admissions(messaging.authority.db, session_id=messaging.owner, pending_only=False)
+ automation = {row['request_id']: row for row in rows if row['principal_id'] == 'automation:' + messaging.route}
+ assert {key: row['outcome'] for key, row in automation.items()} == {
+ '["heartbeat","proc_job","proc_job-tick-1"]': 'cancelled',
+ '["heartbeat","proc_job","proc_job-tick-2"]': 'cancelled', '["completion","proc_job",1.0]': None}
+ folded = automation['["heartbeat","proc_job","proc_job-tick-2"]']['payload']['text']
+ assert _tick_output(folded) == 'proc_job/1\nproc_job/2\n'
+
+
+async def test_a_served_profiles_bot_chat_takes_every_dm_with_no_client_attached(gateway):
+ sender = gateway.connect(gateway.secondary)
+
+ async def deliver(count):
+ reply = await sender.dispatch({'id': count, 'method': 'bot_relay.deliver', 'params': {
+ 'id': f'{count:032x}', 'profile': 'beta', 'message': f'DM {count}', 'author': AUTHOR}})
+ assert 'result' in reply, reply
+ await gateway.settled(gateway.secondary, reply['result']['session_id'], count)
+ return reply['result']['session_id']
+
+ # The first DM mints the Bot Chat through the door; the second follows a turn that ran in it.
+ chat = await deliver(1)
+ assert await deliver(2) == chat
+
+ # A turn in a served profile's local chat records the launch profile as its transport; a restart
+ # reads the chat back over that row.
+ db = gateway.secondary.db.db_path
+ await gateway.stop()
+ with closing(sqlite3.connect(db)) as conn:
+ assert conn.execute('SELECT transport_profile FROM sessions WHERE id=?', (chat,)).fetchone() == ('default',)
+ await gateway.start()
+ sender = gateway.connect(gateway.secondary)
+ assert [await deliver(count) for count in (3, 4)] == [chat, chat]
+
+ frozen = ('beta-model', str(gateway.tmp_path))
+ assert [(turn.profile, turn.chat, turn.text, turn.policy) for turn in gateway.turns] == [
+ ('beta', chat, f'DM {count}', frozen) for count in (1, 2, 3, 4)]
+
+
[email protected]('origin', ['created', 'legacy'])
+async def test_a_served_profiles_desktop_session_admits_its_own_process_notices(gateway, origin):
+ secondary = gateway.secondary
+ if origin == 'created':
+ desktop = gateway.connect(secondary)
+ sid = gateway.create(secondary, desktop, 'desktop', 'gui', model='desk-model').session_id
+ else:
+ # A row written by a Desktop before the session authority existed; resuming it adopts it.
+ desktop = gateway.connect(secondary, provider='local', profile_id=secondary.profile_id,
+ instance_id=secondary.instance_id)
+ sid = secondary.db.create_session(LEGACY_DESKTOP_ID, source='desktop', model='desk-model',
+ cwd=str(gateway.tmp_path))
+ resumed = await desktop.dispatch({'id': 1, 'method': 'session.resume', 'params': {'session_id': sid}})
+ assert 'result' in resumed, resumed
+ submitted = await desktop.dispatch({'id': 2, 'method': 'prompt.submit', 'params': {
+ 'session_id': sid, 'input_id': 'start', 'text': 'run the job in the background'}})
+ assert 'result' in submitted, submitted
+ await gateway.settled(secondary, sid, 1)
+
+ match = gateway.process_event(secondary, sid, 'watch_match', event_id='match-1', pattern='READY')
+ assert await gateway.drain(match) == [], 'the watch match was refused and requeued'
+ completion = gateway.process_event(secondary, sid, 'completion', exit_code=0)
+ assert await gateway.runner._deliver_completion_notification('[job finished]', completion) is True
+ await gateway.settled(secondary, sid, 3)
+ assert [(turn.profile, turn.chat, turn.policy) for turn in gateway.turns] == [
+ ('beta', sid, ('desk-model', str(gateway.tmp_path)))] * 3
diff --git a/tools/process_registry.py b/tools/process_registry.py
index 6f915c15b4b..0673116efc9 100644
--- a/tools/process_registry.py
+++ b/tools/process_registry.py
@@ -32,7 +32,8 @@ from typing import Any, Dict, List, Literal, NamedTuple, Optional
from hermes_cli.config import get_hermes_home
-from tools.process_registry_notifications import format_process_notification, is_child_process_notification, should_surface_notification
+from tools.process_registry_notifications import (
+ format_process_notification, heartbeat_output_tail, is_child_process_notification, should_surface_notification)
from tools.process_registry_checkpoint import ProcessCheckpointMixin
from tools.process_registry_output_log import ProcessOutputLogMixin, open_output_log, record_exit_command, worker_output_log
from tools.process_registry_termination import ProcessTerminationMixin
@@ -84,7 +85,6 @@ WATCH_LIFETIME_MAX_HITS = 8
# per process) and needs no strike/lifetime breaker. The floor exists so a model cannot turn
# it into a 5-second poll; the output slice is capped like a completion notice.
HEARTBEAT_MIN_SECONDS = 60
-HEARTBEAT_OUTPUT_CHARS = 2000
HEARTBEAT_TICK_SECONDS = 5
# Reader final drain after a reconcile saw the direct child exit: the exited child's unread tail is at
# most one pipe buffer (Linux pipe-max-size default 1 MiB), and the reader publishes within one select
@@ -765,9 +765,7 @@ class ProcessRegistry(ProcessTerminationMixin, ProcessCheckpointMixin, ProcessOu
session._heartbeat_total_at_last = session.total_output_chars
if not output:
return
- if len(output) > HEARTBEAT_OUTPUT_CHARS:
- cut = len(output) - HEARTBEAT_OUTPUT_CHARS
- output = f"...({cut} earlier characters omitted)\n" + output[-HEARTBEAT_OUTPUT_CHARS:]
+ output = heartbeat_output_tail(output)
session._heartbeat_seq += 1
notification = {
**self._watch_event_base(session),
diff --git a/tools/process_registry_notifications.py b/tools/process_registry_notifications.py
index 8da8679edba..bdad4080af4 100644
--- a/tools/process_registry_notifications.py
+++ b/tools/process_registry_notifications.py
@@ -3,6 +3,7 @@ watch_match, watch_disabled, watch_overflow_*, async_delegation) into the
``[IMPORTANT: ...]`` / ``[ASYNC DELEGATION ...]`` text the CLI drain loop, gateway and
TUI inject into the agent conversation."""
+import re
import time
from dataclasses import dataclass
from contextlib import suppress
@@ -372,6 +373,34 @@ def process_completion_display_text(events: list) -> str:
HEARTBEAT_DISPLAY_KIND = "hidden" # a wake, not a message: no surface paints the row
+HEARTBEAT_OUTPUT_CHARS = 2000
+
+
+_OMITTED_PREFIX = re.compile(r"\.\.\.\((\d+) earlier characters omitted\)\n")
+
+
+def heartbeat_output_tail(output: str, omitted: int = 0) -> str:
+ """A tick's output, capped to its newest characters: a long job's latest lines are what matter.
+ ``omitted`` counts characters already cut before ``output``."""
+ if len(output) > HEARTBEAT_OUTPUT_CHARS:
+ omitted += len(output) - HEARTBEAT_OUTPUT_CHARS
+ output = output[-HEARTBEAT_OUTPUT_CHARS:]
+ return f"...({omitted} earlier characters omitted)\n{output}" if omitted else output
+
+
+def join_heartbeat_outputs(earlier: str, later: str) -> str:
+ """Two consecutive ticks' output as one tick's, capped the same way, counting every character
+ either cut. A later tick that was itself cut leaves none of the earlier output."""
+ earlier_cut, earlier_text = _omitted_prefix(earlier)
+ later_cut, later_text = _omitted_prefix(later)
+ if later_cut:
+ return heartbeat_output_tail(later_text, earlier_cut + len(earlier_text) + later_cut)
+ return heartbeat_output_tail(earlier_text + later_text, earlier_cut)
+
+
+def _omitted_prefix(output: str) -> tuple[int, str]:
+ match = _OMITTED_PREFIX.match(output)
+ return (int(match.group(1)), output[match.end():]) if match else (0, output)
def heartbeat_display_text(evt: dict) -> str: