The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a turn interrupted by restart names the restart; document restart_drain_timeout
Tekniumc71174441c12
diff --git a/gateway/session_settlement_recovery.py b/gateway/session_settlement_recovery.pyindex d5ae8120470..054ce30d307 100644--- a/gateway/session_settlement_recovery.py+++ b/gateway/session_settlement_recovery.py@@ -98,7 +98,31 @@ def _publish_completion(authority, live, ref, row, settled, response, captured):live.controls.snapshot(ref.session_id, None)except Exception:logger.exception('Terminal control cleanup failed for admission %s', row['admission_id'])- live.event_stream.publish(ref.session_id, completion_payload(row, settled, response, captured))+ complete = completion_payload(row, settled, response, captured)+ notice = owner_stop_notice(getattr(authority, 'runner', None), settled['outcome'])+ if notice and not complete.get('warning'):+ # ``warning`` is the slot every surface already renders on message.complete (Ink activity+ # line, Desktop toast), so a turn the owner's stop cut short says why (#134889 F1).+ complete['warning'] = notice+ live.event_stream.publish(ref.session_id, complete)+++RESTART_INTERRUPT_NOTICE = ('Gateway restarting: this turn was interrupted and will not be run again. '+ 'Send your message again once Hermes reconnects.')+SHUTDOWN_INTERRUPT_NOTICE = ('Gateway shutting down: this turn was interrupted and will not be run again. '+ 'Send your message again once Hermes is back.')+++def owner_stop_notice(runner, outcome):+ """The named notice for a turn the owner's restart/stop interrupted, or None. With the default+ ``agent.restart_drain_timeout: 0`` a restart interrupts in-flight turns at once; the admission+ settles ``interrupted`` and is never replayed, so the viewer is told the restart did it."""+ # ``_running`` drops only once stop() begins: a user Stop during a restart's after-turn wait+ # (draining, still running) is the user's own interrupt and gets no restart notice.+ if (outcome != 'interrupted' or runner is None or getattr(runner, '_draining', False) is not True+ or getattr(runner, '_running', True) is not False):+ return None+ return RESTART_INTERRUPT_NOTICE if getattr(runner, '_restart_requested', False) is True else SHUTDOWN_INTERRUPT_NOTICEdef _release_media(authority, row):diff --git a/hermes_cli/gateway_chat_view.py b/hermes_cli/gateway_chat_view.pyindex 0bd0ef6a367..f3e8e426e86 100644--- a/hermes_cli/gateway_chat_view.py+++ b/hermes_cli/gateway_chat_view.py@@ -281,6 +281,11 @@ class GatewayChatView:print(flush=True)else:print("\n" + text, flush=True)+ # A turn the owner's restart/stop interrupted names that cause (#134889 F1); other+ # message.complete warnings stay Ink/Desktop-only as before.+ warning = payload.get("warning")+ if payload.get("status") == "interrupted" and isinstance(warning, str) and warning.strip():+ print(warning.strip(), file=sys.stderr, flush=True)self.completions[admission] = payloaddef _request(self, admission, payload):diff --git a/tests/gateway/test_restart_interrupt_notice.py b/tests/gateway/test_restart_interrupt_notice.pynew file mode 100644index 00000000000..5bffaff9c91--- /dev/null+++ b/tests/gateway/test_restart_interrupt_notice.py@@ -0,0 +1,81 @@+"""#134889 F1: ``agent.restart_drain_timeout`` stays 0, and a turn a restart interrupts says so.++With the 0 s default a restart or stop interrupts in-flight turns at once. The admission settles+``interrupted`` (never replayed: ``tests/gateway/test_journey_new_session_queued_work.py`` restart+case, ``tests/gateway/test_prompt_queued_transfer.py`` owner-restart case), and its+``message.complete`` carries a named notice in ``warning``, the slot Ink and Desktop already render.+"""+from types import SimpleNamespace++import pytest++from gateway.session_settlement_recovery import (+ RESTART_INTERRUPT_NOTICE,+ SHUTDOWN_INTERRUPT_NOTICE,+ _publish_completion,+)+from tui_gateway.contracts.registry import EVENTS+++def test_restart_drain_timeout_default_stays_zero():+ from gateway.restart import DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT+ from hermes_cli.config import DEFAULT_CONFIG++ assert DEFAULT_CONFIG['agent']['restart_drain_timeout'] == 0+ assert DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT == 0.0+++class _Stream:+ def __init__(self):+ self.frames = []++ def publish(self, session_id, payload):+ self.frames.append(payload)+++def _publish(runner, outcome, captured=None):+ stream = _Stream()+ live = SimpleNamespace(controls=SimpleNamespace(snapshot=lambda *a: None), event_stream=stream)+ row = {'admission_id': 'adm-1', 'request_id': 'sub-1'}+ _publish_completion(SimpleNamespace(runner=runner), live, SimpleNamespace(session_id='s'), row,+ {'outcome': outcome}, 'partial reply', captured)+ (frame,) = stream.frames+ return frame+++def _runner(*, restart, stopping=True):+ return SimpleNamespace(_draining=True, _running=not stopping, _restart_requested=restart)++[email protected](('restart', 'notice'), [(True, RESTART_INTERRUPT_NOTICE), (False, SHUTDOWN_INTERRUPT_NOTICE)])+def test_turn_interrupted_by_the_owner_stop_carries_a_named_notice(restart, notice):+ frame = _publish(_runner(restart=restart), 'interrupted')+ assert frame['status'] == 'interrupted' and frame['warning'] == notice+ assert 'will not be run again' in notice+ EVENTS['message.complete'].payload.model_validate({k: frame[k] for k in ('text', 'status', 'warning')})+++def test_other_endings_get_no_restart_notice():+ # Completed during the drain, a user Stop in a restart's after-turn wait (still running),+ # and an ordinary interrupt with no restart all stay as they were.+ assert 'warning' not in _publish(_runner(restart=True), 'completed')+ assert 'warning' not in _publish(_runner(restart=True, stopping=False), 'interrupted')+ assert 'warning' not in _publish(SimpleNamespace(_draining=False, _running=True, _restart_requested=False),+ 'interrupted')+++def test_a_turn_warning_is_not_overwritten():+ captured = {'result': {'local_viewer': True, 'warning': 'own warning'}}+ assert _publish(_runner(restart=True), 'interrupted', captured)['warning'] == 'own warning'+++def test_classic_cli_view_prints_the_restart_notice_for_an_interrupted_turn(capsys):+ """The attached classic CLI (``hermes chat``) prints the notice on stderr; a completed turn's+ warning stays unprinted there, as before."""+ from hermes_cli.gateway_chat_view import GatewayChatView++ view = GatewayChatView(object(), {'stored_session_id': 'sid'})+ view._complete('a1', {'text': '', 'status': 'interrupted', 'warning': RESTART_INTERRUPT_NOTICE})+ view._complete('a2', {'text': 'done', 'status': 'complete', 'warning': 'other warning'})+ err = capsys.readouterr().err+ assert RESTART_INTERRUPT_NOTICE in err and 'other warning' not in errdiff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.mdindex 5a4858215d9..ca0e4c0aca4 100644--- a/website/docs/reference/environment-variables.md+++ b/website/docs/reference/environment-variables.md@@ -788,7 +788,7 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us| `HERMES_WECOM_TEXT_BATCH_DELAY_SECONDS` / `_SPLIT_DELAY_SECONDS` | WeCom batcher tuning. || `HERMES_VISION_DOWNLOAD_TIMEOUT` | Timeout in seconds for downloading an image before handing it to vision models (default: `30`). || `HERMES_VISION_MAX_CONCURRENCY` | Max concurrent image **encode/resize** bursts across the whole process (override for `auxiliary.vision.max_concurrency`; default: host CPU core count, no ceiling). Bounds only the CPU-bound encode step so a video-frame fan-out can't saturate every core and starve the event loop — the LLM calls stay fully concurrent. Values `< 1` are ignored. |-| `HERMES_RESTART_DRAIN_TIMEOUT` | Gateway: seconds to wait for active runs to drain on `/restart` before forcing the restart (default: `900`). |+| `HERMES_RESTART_DRAIN_TIMEOUT` | Gateway: overrides `agent.restart_drain_timeout`, the seconds in-flight chat turns may keep running once the gateway stops or restarts before they are interrupted (default: `0`, interrupt at once). See [Gateway Restart Drain](../user-guide/configuration.md#gateway-restart-drain). || `HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT` | Per-platform connect timeout during gateway startup and reconnect (seconds; `0`/negative waits indefinitely). Applies to the connect attempt *and* the Discord adapter's ready-wait, so accounts with many slash commands to sync don't get killed mid-startup. Bridged from `gateway.platform_connect_timeout` in `config.yaml` (default `30`); this env var is the manual override and wins if set explicitly. || `HERMES_GATEWAY_BUSY_INPUT_MODE` | Default gateway busy-input behavior: `queue`, `steer`, or `interrupt`. Can be overridden for the active profile with `/busy`. || `HERMES_GATEWAY_BUSY_ACK_ENABLED` | Whether the gateway sends an acknowledgment message (⚡/⏳/⏩) when a user sends input while the agent is busy (default: `true`). Set to `false` to suppress these messages entirely — the input is still queued/steered/interrupts as normal, only the chat reply is silenced. Bridged from `display.busy_ack_enabled` in `config.yaml`. |diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.mdindex fc5d1897c73..1d75272845b 100644--- a/website/docs/user-guide/configuration.md+++ b/website/docs/user-guide/configuration.md@@ -1146,6 +1146,43 @@ automatically requeue the message because doing so without durable ordering andidempotency could process it twice. Non-positive values use the 5-seconddefault.+## Gateway Restart Drain++When the gateway stops or restarts (SIGTERM, `hermes update`, `hermes gateway stop`, or the final+phase of `hermes gateway restart` / `/restart`), `agent.restart_drain_timeout` is how many seconds+it lets in-flight chat turns keep running before it interrupts them:++```yaml+agent:+ restart_drain_timeout: 0 # default: interrupt in-flight chat turns at once+ cron_drain_timeout: 30 # floor for cron jobs and api_server (/v1) runs+ restart_after_turn_timeout: 1800 # /restart and `gateway restart`: wait for turns BEFORE stopping+```++With the default `0`, a restart interrupts a running turn immediately. The turn settles as+interrupted and is never run again; the TUI and Desktop show a "Gateway restarting" (or "Gateway+shutting down") notice on that turn, and you send the message again once Hermes reconnects.+Queued messages behind it are kept and run in order on the new gateway.++Set a positive value if you would rather let turns finish:++```yaml+agent:+ restart_drain_timeout: 60+```++A longer drain makes every restart and update wait up to that long. `HERMES_RESTART_DRAIN_TIMEOUT`+overrides the config value.++:::caution systemd `TimeoutStopSec`+The generated systemd unit sizes `TimeoutStopSec` from this value: the larger of+`restart_drain_timeout` and `cron_drain_timeout` + 10 s, plus 30 s of headroom, and at least 60 s+(`gateway/restart.py`, `resolve_systemd_timeout_stop_sec`). If you raise `restart_drain_timeout`,+re-run `hermes gateway install` so the unit picks up the new budget. Otherwise systemd SIGKILLs the+gateway in the middle of its drain. launchd caps its stop timeout at 60 s, so on macOS the drain+is limited to what fits inside that.+:::+## Session Stall WatchdogThe gateway runs a notify-only stall watchdog (`agent.session_stall_timeout`, default `300` seconds, `0` = disabled). When a busy session has a **pending inbound follow-up** and the agent's shared activity clock has been idle for at least this long, the gateway logs a WARNING and sends the user a one-shot notification: