Skip to content

The agent that grows with you

fix(gateway): a turn interrupted by restart names the restart; document restart_drain_timeout

Tekniumc71174441c12
diff --git a/gateway/session_settlement_recovery.py b/gateway/session_settlement_recovery.py
index d5ae8120470..054ce30d307 100644
--- a/gateway/session_settlement_recovery.py
+++ b/gateway/session_settlement_recovery.py
@@ -98,7 +98,31 @@ def _publish_completion(authority, live, ref, row, settled, response, captured):
live.controls.snapshot(ref.session_id, None)
except Exception:
logger.exception('Terminal control cleanup failed for admission %s', row['admission_id'])
- live.event_stream.publish(ref.session_id, completion_payload(row, settled, response, captured))
+ complete = completion_payload(row, settled, response, captured)
+ notice = owner_stop_notice(getattr(authority, 'runner', None), settled['outcome'])
+ if notice and not complete.get('warning'):
+ # ``warning`` is the slot every surface already renders on message.complete (Ink activity
+ # line, Desktop toast), so a turn the owner's stop cut short says why (#134889 F1).
+ complete['warning'] = notice
+ live.event_stream.publish(ref.session_id, complete)
+
+
+RESTART_INTERRUPT_NOTICE = ('Gateway restarting: this turn was interrupted and will not be run again. '
+ 'Send your message again once Hermes reconnects.')
+SHUTDOWN_INTERRUPT_NOTICE = ('Gateway shutting down: this turn was interrupted and will not be run again. '
+ 'Send your message again once Hermes is back.')
+
+
+def owner_stop_notice(runner, outcome):
+ """The named notice for a turn the owner's restart/stop interrupted, or None. With the default
+ ``agent.restart_drain_timeout: 0`` a restart interrupts in-flight turns at once; the admission
+ settles ``interrupted`` and is never replayed, so the viewer is told the restart did it."""
+ # ``_running`` drops only once stop() begins: a user Stop during a restart's after-turn wait
+ # (draining, still running) is the user's own interrupt and gets no restart notice.
+ if (outcome != 'interrupted' or runner is None or getattr(runner, '_draining', False) is not True
+ or getattr(runner, '_running', True) is not False):
+ return None
+ return RESTART_INTERRUPT_NOTICE if getattr(runner, '_restart_requested', False) is True else SHUTDOWN_INTERRUPT_NOTICE
def _release_media(authority, row):
diff --git a/hermes_cli/gateway_chat_view.py b/hermes_cli/gateway_chat_view.py
index 0bd0ef6a367..f3e8e426e86 100644
--- a/hermes_cli/gateway_chat_view.py
+++ b/hermes_cli/gateway_chat_view.py
@@ -281,6 +281,11 @@ class GatewayChatView:
print(flush=True)
else:
print("\n" + text, flush=True)
+ # A turn the owner's restart/stop interrupted names that cause (#134889 F1); other
+ # message.complete warnings stay Ink/Desktop-only as before.
+ warning = payload.get("warning")
+ if payload.get("status") == "interrupted" and isinstance(warning, str) and warning.strip():
+ print(warning.strip(), file=sys.stderr, flush=True)
self.completions[admission] = payload
def _request(self, admission, payload):
diff --git a/tests/gateway/test_restart_interrupt_notice.py b/tests/gateway/test_restart_interrupt_notice.py
new file mode 100644
index 00000000000..5bffaff9c91
--- /dev/null
+++ b/tests/gateway/test_restart_interrupt_notice.py
@@ -0,0 +1,81 @@
+"""#134889 F1: ``agent.restart_drain_timeout`` stays 0, and a turn a restart interrupts says so.
+
+With the 0 s default a restart or stop interrupts in-flight turns at once. The admission settles
+``interrupted`` (never replayed: ``tests/gateway/test_journey_new_session_queued_work.py`` restart
+case, ``tests/gateway/test_prompt_queued_transfer.py`` owner-restart case), and its
+``message.complete`` carries a named notice in ``warning``, the slot Ink and Desktop already render.
+"""
+from types import SimpleNamespace
+
+import pytest
+
+from gateway.session_settlement_recovery import (
+ RESTART_INTERRUPT_NOTICE,
+ SHUTDOWN_INTERRUPT_NOTICE,
+ _publish_completion,
+)
+from tui_gateway.contracts.registry import EVENTS
+
+
+def test_restart_drain_timeout_default_stays_zero():
+ from gateway.restart import DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT
+ from hermes_cli.config import DEFAULT_CONFIG
+
+ assert DEFAULT_CONFIG['agent']['restart_drain_timeout'] == 0
+ assert DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT == 0.0
+
+
+class _Stream:
+ def __init__(self):
+ self.frames = []
+
+ def publish(self, session_id, payload):
+ self.frames.append(payload)
+
+
+def _publish(runner, outcome, captured=None):
+ stream = _Stream()
+ live = SimpleNamespace(controls=SimpleNamespace(snapshot=lambda *a: None), event_stream=stream)
+ row = {'admission_id': 'adm-1', 'request_id': 'sub-1'}
+ _publish_completion(SimpleNamespace(runner=runner), live, SimpleNamespace(session_id='s'), row,
+ {'outcome': outcome}, 'partial reply', captured)
+ (frame,) = stream.frames
+ return frame
+
+
+def _runner(*, restart, stopping=True):
+ return SimpleNamespace(_draining=True, _running=not stopping, _restart_requested=restart)
+
+
[email protected](('restart', 'notice'), [(True, RESTART_INTERRUPT_NOTICE), (False, SHUTDOWN_INTERRUPT_NOTICE)])
+def test_turn_interrupted_by_the_owner_stop_carries_a_named_notice(restart, notice):
+ frame = _publish(_runner(restart=restart), 'interrupted')
+ assert frame['status'] == 'interrupted' and frame['warning'] == notice
+ assert 'will not be run again' in notice
+ EVENTS['message.complete'].payload.model_validate({k: frame[k] for k in ('text', 'status', 'warning')})
+
+
+def test_other_endings_get_no_restart_notice():
+ # Completed during the drain, a user Stop in a restart's after-turn wait (still running),
+ # and an ordinary interrupt with no restart all stay as they were.
+ assert 'warning' not in _publish(_runner(restart=True), 'completed')
+ assert 'warning' not in _publish(_runner(restart=True, stopping=False), 'interrupted')
+ assert 'warning' not in _publish(SimpleNamespace(_draining=False, _running=True, _restart_requested=False),
+ 'interrupted')
+
+
+def test_a_turn_warning_is_not_overwritten():
+ captured = {'result': {'local_viewer': True, 'warning': 'own warning'}}
+ assert _publish(_runner(restart=True), 'interrupted', captured)['warning'] == 'own warning'
+
+
+def test_classic_cli_view_prints_the_restart_notice_for_an_interrupted_turn(capsys):
+ """The attached classic CLI (``hermes chat``) prints the notice on stderr; a completed turn's
+ warning stays unprinted there, as before."""
+ from hermes_cli.gateway_chat_view import GatewayChatView
+
+ view = GatewayChatView(object(), {'stored_session_id': 'sid'})
+ view._complete('a1', {'text': '', 'status': 'interrupted', 'warning': RESTART_INTERRUPT_NOTICE})
+ view._complete('a2', {'text': 'done', 'status': 'complete', 'warning': 'other warning'})
+ err = capsys.readouterr().err
+ assert RESTART_INTERRUPT_NOTICE in err and 'other warning' not in err
diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md
index 5a4858215d9..ca0e4c0aca4 100644
--- a/website/docs/reference/environment-variables.md
+++ b/website/docs/reference/environment-variables.md
@@ -788,7 +788,7 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us
| `HERMES_WECOM_TEXT_BATCH_DELAY_SECONDS` / `_SPLIT_DELAY_SECONDS` | WeCom batcher tuning. |
| `HERMES_VISION_DOWNLOAD_TIMEOUT` | Timeout in seconds for downloading an image before handing it to vision models (default: `30`). |
| `HERMES_VISION_MAX_CONCURRENCY` | Max concurrent image **encode/resize** bursts across the whole process (override for `auxiliary.vision.max_concurrency`; default: host CPU core count, no ceiling). Bounds only the CPU-bound encode step so a video-frame fan-out can't saturate every core and starve the event loop — the LLM calls stay fully concurrent. Values `< 1` are ignored. |
-| `HERMES_RESTART_DRAIN_TIMEOUT` | Gateway: seconds to wait for active runs to drain on `/restart` before forcing the restart (default: `900`). |
+| `HERMES_RESTART_DRAIN_TIMEOUT` | Gateway: overrides `agent.restart_drain_timeout`, the seconds in-flight chat turns may keep running once the gateway stops or restarts before they are interrupted (default: `0`, interrupt at once). See [Gateway Restart Drain](../user-guide/configuration.md#gateway-restart-drain). |
| `HERMES_GATEWAY_PLATFORM_CONNECT_TIMEOUT` | Per-platform connect timeout during gateway startup and reconnect (seconds; `0`/negative waits indefinitely). Applies to the connect attempt *and* the Discord adapter's ready-wait, so accounts with many slash commands to sync don't get killed mid-startup. Bridged from `gateway.platform_connect_timeout` in `config.yaml` (default `30`); this env var is the manual override and wins if set explicitly. |
| `HERMES_GATEWAY_BUSY_INPUT_MODE` | Default gateway busy-input behavior: `queue`, `steer`, or `interrupt`. Can be overridden for the active profile with `/busy`. |
| `HERMES_GATEWAY_BUSY_ACK_ENABLED` | Whether the gateway sends an acknowledgment message (⚡/⏳/⏩) when a user sends input while the agent is busy (default: `true`). Set to `false` to suppress these messages entirely — the input is still queued/steered/interrupts as normal, only the chat reply is silenced. Bridged from `display.busy_ack_enabled` in `config.yaml`. |
diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md
index fc5d1897c73..1d75272845b 100644
--- a/website/docs/user-guide/configuration.md
+++ b/website/docs/user-guide/configuration.md
@@ -1146,6 +1146,43 @@ automatically requeue the message because doing so without durable ordering and
idempotency could process it twice. Non-positive values use the 5-second
default.
+## Gateway Restart Drain
+
+When the gateway stops or restarts (SIGTERM, `hermes update`, `hermes gateway stop`, or the final
+phase of `hermes gateway restart` / `/restart`), `agent.restart_drain_timeout` is how many seconds
+it lets in-flight chat turns keep running before it interrupts them:
+
+```yaml
+agent:
+ restart_drain_timeout: 0 # default: interrupt in-flight chat turns at once
+ cron_drain_timeout: 30 # floor for cron jobs and api_server (/v1) runs
+ restart_after_turn_timeout: 1800 # /restart and `gateway restart`: wait for turns BEFORE stopping
+```
+
+With the default `0`, a restart interrupts a running turn immediately. The turn settles as
+interrupted and is never run again; the TUI and Desktop show a "Gateway restarting" (or "Gateway
+shutting down") notice on that turn, and you send the message again once Hermes reconnects.
+Queued messages behind it are kept and run in order on the new gateway.
+
+Set a positive value if you would rather let turns finish:
+
+```yaml
+agent:
+ restart_drain_timeout: 60
+```
+
+A longer drain makes every restart and update wait up to that long. `HERMES_RESTART_DRAIN_TIMEOUT`
+overrides the config value.
+
+:::caution systemd `TimeoutStopSec`
+The generated systemd unit sizes `TimeoutStopSec` from this value: the larger of
+`restart_drain_timeout` and `cron_drain_timeout` + 10 s, plus 30 s of headroom, and at least 60 s
+(`gateway/restart.py`, `resolve_systemd_timeout_stop_sec`). If you raise `restart_drain_timeout`,
+re-run `hermes gateway install` so the unit picks up the new budget. Otherwise systemd SIGKILLs the
+gateway in the middle of its drain. launchd caps its stop timeout at 60 s, so on macOS the drain
+is limited to what fits inside that.
+:::
+
## Session Stall Watchdog
The gateway runs a notify-only stall watchdog (`agent.session_stall_timeout`, default `300` seconds, `0` = disabled). When a busy session has a **pending inbound follow-up** and the agent's shared activity clock has been idle for at least this long, the gateway logs a WARNING and sends the user a one-shot notification: