Skip to content

The agent that grows with you

fix(gateway): managed-worker turns keep the restart notice the owner's stop caused

Teknium6c256e4f6c0f
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.py
index 2277426e0e6..ccf9cd87ddb 100644
--- a/gateway/run_runtime.py
+++ b/gateway/run_runtime.py
@@ -229,17 +229,20 @@ def managed_turn_count(runner):
return uncounted_runtime_work(runner)
-def stop_authority_turns(authority, *, in_process=False):
+def stop_authority_turns(authority, *, in_process=False, reason=None):
"""Cooperative Stop for every turn *authority* is executing: the control a user's Stop sends to a
managed worker (acknowledged, or terminated within STOP_ACK_SECONDS, and the admission settles
through its own fenced path). ``in_process`` also hard-interrupts the running agent of each
- executing in-process turn, or latches the generation ``adopt_agent`` consumes before it exists."""
+ executing in-process turn, or latches the generation ``adopt_agent`` consumes before it exists.
+ ``reason`` (a gateway restart/shutdown) is recorded on each worker it signals, so the turn settles
+ with that cause; a worker already stopped (the user's Stop came first) keeps none."""
from hermes_state_runtime import RuntimeStoreError
signalled = 0
workers = getattr(authority, '_managed_workers', {})
for worker in list(workers.values()):
if worker.closed.is_set() or (getattr(worker, 'stop', None) is not None and worker.stop.is_set()):
continue
+ worker.stop_cause = reason
try:
worker.control({'type': 'stop'})
except RuntimeStoreError as exc:
@@ -267,9 +270,9 @@ def stop_authority_turns(authority, *, in_process=False):
return signalled
-def stop_managed_turns(runner):
+def stop_managed_turns(runner, reason=None):
"""``stop_authority_turns`` across every served authority (whole-runtime shutdown)."""
- return sum(stop_authority_turns(authority) for authority in _authorities(runner))
+ return sum(stop_authority_turns(authority, reason=reason) for authority in _authorities(runner))
def interrupt_side_agents(runner, reason):
@@ -577,7 +580,7 @@ async def drain_gateway_runtime(runner):
adapter.set_message_handler(None)
-async def settle_gateway_runtime(runner, budget=None):
+async def settle_gateway_runtime(runner, budget=None, reason=None):
"""Keep authority tasks alive until their last durable settlement write, within a bound.
A managed turn still running here gets its Stop first (its worker acknowledges or is
@@ -587,7 +590,7 @@ async def settle_gateway_runtime(runner, budget=None):
the executor quiesce skips the DB close while a writer is live, and process exit releases
the ownership lock, so raising here would only skip teardown and disarm the watchdog."""
authorities = _authorities(runner)
- stop_managed_turns(runner)
+ stop_managed_turns(runner, reason)
from gateway.session_llm_oneshot import stop as stop_oneshots
for authority in authorities:
stop_oneshots(authority) # an answer is no durable settlement: never waited for
diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py
index e1f7285b914..f0277bca10e 100644
--- a/gateway/run_shutdown.py
+++ b/gateway/run_shutdown.py
@@ -860,7 +860,7 @@ class GatewayShutdownMixin(GatewaySessionEndMixin):
for count, what in (
(self._interrupt_api_server_runs(reason), "api_server run(s)"),
(self._interrupt_deferred_agent_workers(reason), "deferred agent worker(s)"),
- (stop_managed_turns(self), "managed-worker turn(s)"), (interrupt_side_agents(self, reason), "side agent(s)"),
+ (stop_managed_turns(self, reason), "managed-worker turn(s)"), (interrupt_side_agents(self, reason), "side agent(s)"),
):
if count:
logger.debug("Interrupted %d %s during shutdown", count, what)
@@ -2219,7 +2219,7 @@ class GatewayShutdownMixin(GatewaySessionEndMixin):
if ctx.timed_out:
await GatewayRunner._stop_interrupt_remaining_work(self, ctx)
from gateway.run_runtime import settle_gateway_runtime # within what the leash has left
- await settle_gateway_runtime(self, budget=_effective_watchdog_leash(self) - ctx.elapsed())
+ await settle_gateway_runtime(self, _effective_watchdog_leash(self) - ctx.elapsed(), GatewayRunner._shutdown_interrupt_reason(self))
await GatewayRunner._stop_finalize_agents_and_adapters(self, ctx)
await GatewayRunner._stop_release_runtime_state(self, ctx)
GatewayRunner._stop_quiesce_and_close_session_dbs(self, timeout, ctx)
diff --git a/gateway/session_managed_worker.py b/gateway/session_managed_worker.py
index fa7c88dc903..ca11678401e 100644
--- a/gateway/session_managed_worker.py
+++ b/gateway/session_managed_worker.py
@@ -158,6 +158,8 @@ class ManagedWorker:
# Monotonic instant of the FIRST Stop: one acknowledgment budget runs from here, never
# renewed by the child's later output or by a repeated Stop.
self.stopped_at = None
+ # The gateway restart/shutdown reason when that stop (not a user's) signalled this worker.
+ self.stop_cause = None
self.writer = threading.Thread(target=self._write_controls, name='managed-control-writer', daemon=True)
# One dedicated reader thread per worker (never the loop's shared default executor, which
# parked pipe reads would exhaust). It reads one frame per demand, so the pipe keeps its
@@ -445,12 +447,20 @@ def _launch_env_only_credentials(home, env):
return {k: v for k, v in missing.items() if k not in plain}
-def _interrupted_before_bootstrap(authority, row):
+def _interrupted_before_bootstrap(authority, row, cause=None):
authority.pending_results[row['admission_id']] = {
- 'result': {'final_response': '', 'interrupted': True}, 'usage': {}}
+ 'result': {'final_response': '', 'interrupted': True, **({'interrupt_message': cause} if cause else {})},
+ 'usage': {}}
return ''
+def _with_stop_cause(accepted, worker):
+ """The wire result carries no interruption cause; the owner recorded which stop it sent."""
+ if accepted.get('interrupted') and worker.stop_cause:
+ return {**accepted, 'interrupt_message': worker.stop_cause}
+ return accepted
+
+
def _settled_session_usage(authority, session_id, accepted, execution_id=None):
"""A managed turn's ``message.complete.usage``: the worker exits with its turn, so the session-cumulative counters come from the rows its usage writes committed. A worker that compresses mid-turn ends on a child of this reservation's session; the owner's execution row records that tip (compression moves it in the same transaction), so the path from the tip back to the reservation plus the reservation's own compression ancestors is summed, and stale siblings or the conversation a branch came from never count."""
import sqlite3
@@ -584,6 +594,7 @@ async def execute_managed(authority, ref, row, policy):
# A slow or non-zero interpreter exit after it is close()'s cleanup, not a loss.
with suppress(subprocess.TimeoutExpired):
await asyncio.to_thread(process.wait, 10)
+ accepted = _with_stop_cause(accepted, worker)
# Like in-process execution, settlement belongs to the drain's stream lock.
authority.pending_results[row['admission_id']] = {
'result': {**accepted, 'local_viewer': True}, 'usage': usage,
@@ -599,7 +610,7 @@ async def execute_managed(authority, ref, row, policy):
isinstance(exc, RuntimeStoreError) and exc.reason == 'managed_worker_stopped'):
# Stopped before the child ever received its bootstrap: nothing executed, so
# this settles like an ordinary interrupted turn (the finally kills the child).
- return _interrupted_before_bootstrap(authority, row)
+ return _interrupted_before_bootstrap(authority, row, worker.stop_cause)
raise
from gateway.session_worker_reservation import lose_admission_worker
from gateway.session_runtime_workers import tracked_write
diff --git a/tests/gateway/test_managed_turn_contract.py b/tests/gateway/test_managed_turn_contract.py
index 9d90ef3570c..c3fc1fa47e0 100644
--- a/tests/gateway/test_managed_turn_contract.py
+++ b/tests/gateway/test_managed_turn_contract.py
@@ -144,6 +144,7 @@ async def test_stop_during_environment_preparation_prevents_bootstrap(tmp_path,
class Worker:
def __init__(self, process):
self.stop = asyncio.Event()
+ self.stop_cause = None
def interrupt(self):
self.stop.set()
async def next_frame(self, *args, **kwargs):
diff --git a/tests/gateway/test_restart_interrupt_notice.py b/tests/gateway/test_restart_interrupt_notice.py
index 9cf701759cf..3dd2219f7d9 100644
--- a/tests/gateway/test_restart_interrupt_notice.py
+++ b/tests/gateway/test_restart_interrupt_notice.py
@@ -145,3 +145,84 @@ async def test_user_stop_is_not_relabelled_when_shutdown_starts_during_settlemen
assert len(complete) == 1
assert persisted['result']['interrupt_message'] == 'User requested stop'
assert 'warning' not in complete[0], complete[0]
+
+
+STOPPABLE_CHILD = """import json, os, sys, psutil
+me = psutil.Process()
+out = os.fdopen(os.dup(sys.stdout.fileno()), 'wb', buffering=0)
+def send(frame):
+ out.write((json.dumps(frame) + '\\n').encode())
+send({'type': 'hello', 'pid': me.pid, 'birth': me.create_time(), 'ancestors': [p.pid for p in me.parents()[:3]]})
+for line in sys.stdin.buffer:
+ frame = json.loads(line)
+ if frame == {'type': 'stop'}:
+ send({'type': 'result', 'result': {'final_response': '', 'interrupted': True, 'completed': False}})
+ elif frame == {'type': 'finish'}:
+ send({'type': 'finished'})
+ break
+"""
+
+
[email protected]('user_stopped_first', [False, True])
+def test_managed_worker_turn_settles_with_the_cause_of_the_stop_that_reached_it(tmp_path, monkeypatch,
+ user_stopped_first):
+ """A managed-worker turn (real child, real SQLite authority) that the gateway restart stops
+ settles with the restart notice; one the user had already stopped before shutdown began gets
+ none. The worker's wire result carries no cause: the owner records which stop it sent."""
+ import subprocess
+
+ from gateway import session_managed_worker as managed
+ from gateway.session_authority import LiveSession, SessionAuthority
+ from gateway.session_contract import Principal, SessionRef, Submission
+ from gateway.session_policy import build_policy
+ from gateway.session_settlement_recovery import commit_and_publish
+ from hermes_state import SessionDB
+ from hermes_state_runtime import begin_runtime_epoch, claim_session_input
+ from tests.gateway.restart_test_helpers import make_restart_runner
+ from tests.waiting import async_wait_until
+
+ monkeypatch.setenv('HERMES_HOME', str(tmp_path))
+ runner, _adapter = make_restart_runner()
+ runner._restart_requested = True
+ db = SessionDB(db_path=tmp_path / 'state.db')
+ db.create_session('s', source='test')
+ authority = SessionAuthority(runner, profile_id='owned', instance_id='i', db=db,
+ epoch=begin_runtime_epoch(db, instance_id='i'))
+ runner.session_authority = authority
+ live = authority.sessions['s'] = LiveSession(SimpleNamespace(platform=None, user_id='human', chat_id='c'), 'route')
+ actor = Principal('human', 'owned', frozenset({'session:submit', 'session:control'}), 'cli')
+ ref = SessionRef('owned', 's')
+ monkeypatch.setattr(authority, '_schedule', lambda ref: None)
+
+ class Child(subprocess.Popen):
+ def __init__(self, args, **kw):
+ super().__init__([sys.executable, '-c', STOPPABLE_CHILD], **kw)
+ monkeypatch.setattr(managed.subprocess, 'Popen', Child)
+ policy = build_policy({'cwd': str(tmp_path), 'model': 'm'}, {}, profile_terminal=False)
+
+ async def scenario():
+ await authority.submit(actor, Submission('turn', ref, {'text': 'go'}, 'queue'))
+ row = claim_session_input(db, epoch=authority.epoch, session_id='s')
+ authority._stamp_claim(ref, live, row)
+ task = asyncio.create_task(managed.execute_managed(authority, ref, row, policy))
+ await async_wait_until(lambda: (w := getattr(authority, '_managed_workers', {}).get('s')) is not None and w.writer.is_alive(),
+ timeout=30, what='the managed worker to receive its bootstrap')
+ if user_stopped_first:
+ assert managed.interrupt_managed(authority, actor, ref, row['generation'])
+ runner._interrupt_running_agents(runner._shutdown_interrupt_reason()) # stop()'s timed-out path
+ async with asyncio.timeout(30):
+ return row, await task
+
+ with db:
+ row, response = asyncio.run(scenario())
+ frames = []
+ live.event_stream.observers.add(frames.append)
+ commit_and_publish(authority, live, ref, row, response, 'interrupted',
+ authority.pending_results[row['admission_id']], {})
+ (complete,) = [f['params']['payload'] for f in frames if f['params']['type'] == 'message.complete']
+ assert complete['status'] == 'interrupted'
+ if user_stopped_first:
+ assert 'warning' not in complete, complete
+ else:
+ assert complete['warning'] == RESTART_INTERRUPT_NOTICE