Skip to content

The agent that grows with you

fix(gateway): a shutdown Stop before the managed worker spawns keeps its cause

Tekniumf959be49c16e
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.py
index ccf9cd87ddb..42e80bf5387 100644
--- a/gateway/run_runtime.py
+++ b/gateway/run_runtime.py
@@ -235,7 +235,8 @@ def stop_authority_turns(authority, *, in_process=False, reason=None):
through its own fenced path). ``in_process`` also hard-interrupts the running agent of each
executing in-process turn, or latches the generation ``adopt_agent`` consumes before it exists.
``reason`` (a gateway restart/shutdown) is recorded on each worker it signals, so the turn settles
- with that cause; a worker already stopped (the user's Stop came first) keeps none."""
+ with that cause; a worker already stopped (the user's Stop came first) keeps none. A worker
+ still being spawned gets the same Stop and cause through latches (``_stop_spawning_managed``)."""
from hermes_state_runtime import RuntimeStoreError
signalled = 0
workers = getattr(authority, '_managed_workers', {})
@@ -250,6 +251,7 @@ def stop_authority_turns(authority, *, in_process=False, reason=None):
# loop escalates on that latch alone (terminate after STOP_ACK_SECONDS).
logger.debug('managed Stop not queued (%s); the latch escalates it', exc.reason)
signalled += 1
+ signalled += _stop_spawning_managed(authority, workers, reason)
if not in_process:
return signalled
from agent.interrupt_compat import request_hard_interrupt
@@ -257,7 +259,7 @@ def stop_authority_turns(authority, *, in_process=False, reason=None):
running = getattr(authority.runner, '_running_agents', {})
for sid, live in list(authority.sessions.items()):
generation = live.event_stream.execution.get('execution_generation')
- if generation is None or sid in workers:
+ if generation is None or sid in workers or sid in getattr(authority, '_managed_spawning', {}):
continue
adopted_generation, agent = authority.adopted.get(sid, (None, None))
if adopted_generation != generation:
@@ -270,6 +272,26 @@ def stop_authority_turns(authority, *, in_process=False, reason=None):
return signalled
+def _stop_spawning_managed(authority, workers, reason):
+ """A managed turn whose worker is still being spawned has nothing to signal yet: latch the
+ Stop for its generation (``adopt_agent`` delivers it to the worker, which is never
+ bootstrapped) and, for a gateway restart/shutdown, the cause its pre-bootstrap interrupted
+ result carries (#134889 GW-22). A user's Stop latched first keeps no cause."""
+ stopped = 0
+ for sid, generation in list(getattr(authority, '_managed_spawning', {}).items()):
+ if sid in workers:
+ continue
+ stops = authority.pending_stops
+ if reason and stops.get(sid) != generation:
+ causes = getattr(authority, 'pending_stop_causes', None)
+ if causes is None:
+ causes = authority.pending_stop_causes = {}
+ causes[sid] = (generation, reason)
+ stops[sid] = generation
+ stopped += 1
+ return stopped
+
+
def stop_managed_turns(runner, reason=None):
"""``stop_authority_turns`` across every served authority (whole-runtime shutdown)."""
return sum(stop_authority_turns(authority, reason=reason) for authority in _authorities(runner))
diff --git a/gateway/session_authority.py b/gateway/session_authority.py
index 7c2216852e7..5c6a7e041a0 100644
--- a/gateway/session_authority.py
+++ b/gateway/session_authority.py
@@ -132,6 +132,9 @@ class SessionAuthority:
# Stops accepted for a running generation whose agent does not exist yet
# (first-turn construction); consumed by adopt_agent, keyed session -> generation.
self.pending_stops = {}
+ # session -> (generation, reason): a gateway restart/shutdown Stop latched before that
+ # generation's managed worker existed; its pre-bootstrap result carries the cause (GW-22).
+ self.pending_stop_causes = {}
# session -> (generation, [steer text]) accepted before that generation adopted an agent;
# adopt_agent hands it to that agent (never a later generation's), the drain drops the rest.
self.pending_steers = {}
@@ -1144,6 +1147,7 @@ class SessionAuthority:
with live.event_stream.lock:
live.event_stream.execution = {}
self.pending_stops.pop(ref.session_id, None)
+ self.pending_stop_causes.pop(ref.session_id, None)
self.pending_steers.pop(ref.session_id, None)
self.adopted.pop(ref.session_id, None)
# An automation turn nobody waits on settles here, at the exit that fenced it (GW-10),
@@ -1267,6 +1271,7 @@ class SessionAuthority:
with live.event_stream.lock:
live.event_stream.execution = {}
self.pending_stops.pop(ref.session_id, None)
+ self.pending_stop_causes.pop(ref.session_id, None)
self.pending_steers.pop(ref.session_id, None)
self.adopted.pop(ref.session_id, None)
from gateway.session_queued_controls import publish_transfers
diff --git a/gateway/session_managed_worker.py b/gateway/session_managed_worker.py
index ca11678401e..49a4a0dbda4 100644
--- a/gateway/session_managed_worker.py
+++ b/gateway/session_managed_worker.py
@@ -454,6 +454,13 @@ def _interrupted_before_bootstrap(authority, row, cause=None):
return ''
+def _latched_stop_cause(authority, session_id, generation):
+ """The gateway restart/shutdown cause latched for this generation before its worker existed
+ (``run_runtime._stop_spawning_managed``), consumed once; None for a user's Stop."""
+ latched = getattr(authority, 'pending_stop_causes', {}).pop(session_id, None)
+ return latched[1] if latched and latched[0] == generation else None
+
+
def _with_stop_cause(accepted, worker):
"""The wire result carries no interruption cause; the owner recorded which stop it sent."""
if accepted.get('interrupted') and worker.stop_cause:
@@ -514,14 +521,14 @@ def _compression_ancestors(db, session_id):
return ancestors[::-1]
-async def execute_managed(authority, ref, row, policy):
- from gateway.run_turn_progress import publish_worker_tool_event
- # This route never passes the inbound prepare step, so the worker would read ``@`` tokens as typed.
- from gateway.session_context_refs import managed_turn_text, refused_turn_result
- text, refusal = await managed_turn_text(authority, row, policy)
- if refusal is not None:
- authority.pending_results[row['admission_id']] = {'result': refused_turn_result(row, refusal), 'usage': {}}
- return refusal
+async def _spawn_registered_worker(authority, ref, row):
+ """Spawn the turn's child and register it for Stops: ``(worker, process)``, or None when the spawn was cancelled (the
+ interrupted result is already recorded). Until registration a gateway stop has nothing to
+ signal, so the generation is marked spawning and the stop latches instead (#134889 GW-22)."""
+ spawning = getattr(authority, '_managed_spawning', None)
+ if spawning is None:
+ spawning = authority._managed_spawning = {}
+ spawning[ref.session_id] = generation = row.get('generation')
try:
env = await asyncio.to_thread(_worker_env, authority)
cwd = (await asyncio.to_thread(Path(__file__).resolve)).parents[1]
@@ -533,7 +540,12 @@ async def execute_managed(authority, ref, row, policy):
stderr=subprocess.DEVNULL if log_path is None else subprocess.PIPE, **WORKER_POPEN)
except asyncio.CancelledError:
# acquire_process owns late-child cleanup; no bootstrap or reservation was sent.
- return _interrupted_before_bootstrap(authority, row)
+ spawning.pop(ref.session_id, None)
+ _interrupted_before_bootstrap(authority, row, _latched_stop_cause(authority, ref.session_id, generation))
+ return None
+ except BaseException:
+ spawning.pop(ref.session_id, None)
+ raise
worker = ManagedWorker(process)
if log_path is not None:
worker.drain_stderr(log_path)
@@ -541,6 +553,24 @@ async def execute_managed(authority, ref, row, policy):
if workers is None:
workers = authority._managed_workers = {}
workers[ref.session_id] = worker
+ spawning.pop(ref.session_id, None)
+ worker.stop_cause = _latched_stop_cause(authority, ref.session_id, generation)
+ return worker, process
+
+
+async def execute_managed(authority, ref, row, policy):
+ from gateway.run_turn_progress import publish_worker_tool_event
+ # This route never passes the inbound prepare step, so the worker would read ``@`` tokens as typed.
+ from gateway.session_context_refs import managed_turn_text, refused_turn_result
+ text, refusal = await managed_turn_text(authority, row, policy)
+ if refusal is not None:
+ authority.pending_results[row['admission_id']] = {'result': refused_turn_result(row, refusal), 'usage': {}}
+ return refusal
+ spawned = await _spawn_registered_worker(authority, ref, row)
+ if spawned is None:
+ return ''
+ worker, process = spawned
+ workers = authority._managed_workers
# A Stop acknowledged during the env/spawn awaits found no worker and was latched for this
# generation; consume it in the same step that makes the worker reachable to interrupt_managed.
authority.adopt_agent(ref.session_id, row['generation'], worker)
diff --git a/tests/gateway/test_restart_interrupt_notice.py b/tests/gateway/test_restart_interrupt_notice.py
index 33894928ab4..0e89008e304 100644
--- a/tests/gateway/test_restart_interrupt_notice.py
+++ b/tests/gateway/test_restart_interrupt_notice.py
@@ -395,3 +395,80 @@ def test_automation_turn_transcript_notice_is_the_neutral_copy(tmp_path):
db, *_ = _settle_on_real_store(tmp_path, 'Gateway restarting', principal='cron-owner')
with db:
assert _notices(db) == [AUTOMATION_INTERRUPTED_NOTICE]
+
+
[email protected]('ending', ['spawned', 'cancelled'])
+def test_shutdown_stop_before_the_managed_worker_spawned_carries_its_cause(tmp_path, monkeypatch, ending):
+ """#134889 GW-22: the gateway's stop lands while the managed worker is still being spawned
+ (no worker to signal yet). The authority latches the cause for that generation, so the
+ pre-bootstrap interrupted result carries it: the spawned child is stopped before it is ever
+ bootstrapped, or the cancelled spawn settles with the cause. Real child, real SQLite."""
+ import subprocess
+
+ from gateway import session_managed_worker as managed
+ from gateway.run_runtime import stop_authority_turns
+ from gateway.session_authority import LiveSession, SessionAuthority
+ from gateway.session_contract import Principal, SessionRef, Submission
+ from gateway.session_policy import build_policy
+ from hermes_state import SessionDB
+ from hermes_state_runtime import begin_runtime_epoch, claim_session_input
+ from tests.gateway.restart_test_helpers import make_restart_runner
+ from tests.waiting import async_wait_until
+
+ monkeypatch.setenv('HERMES_HOME', str(tmp_path))
+ runner, _adapter = make_restart_runner()
+ db = SessionDB(db_path=tmp_path / 'state.db')
+ db.create_session('s', source='test')
+ authority = SessionAuthority(runner, profile_id='owned', instance_id='i', db=db,
+ epoch=begin_runtime_epoch(db, instance_id='i'))
+ live = authority.sessions['s'] = LiveSession(SimpleNamespace(platform=None, user_id='human', chat_id='c'), 'route')
+ actor = Principal('human', 'owned', frozenset({'session:submit', 'session:control'}), 'cli')
+ ref = SessionRef('owned', 's')
+ monkeypatch.setattr(authority, '_schedule', lambda ref: None)
+ spawning, release, bootstrapped = threading.Event(), threading.Event(), []
+
+ class Child(subprocess.Popen):
+ def __init__(self, args, **kw):
+ spawning.set()
+ release.wait(30)
+ super().__init__([sys.executable, '-c', STOPPABLE_CHILD], **kw)
+ monkeypatch.setattr(managed.subprocess, 'Popen', Child)
+ monkeypatch.setattr(managed, '_bootstrap', lambda *a: bootstrapped.append(a) or pytest.fail('bootstrapped'))
+ policy = build_policy({'cwd': str(tmp_path), 'model': 'm'}, {}, profile_terminal=False)
+
+ async def scenario():
+ await authority.submit(actor, Submission('turn', ref, {'text': 'go'}, 'queue'))
+ row = claim_session_input(db, epoch=authority.epoch, session_id='s')
+ authority._stamp_claim(ref, live, row)
+ task = asyncio.create_task(managed.execute_managed(authority, ref, row, policy))
+ await asyncio.to_thread(spawning.wait, 30)
+ assert stop_authority_turns(authority, reason='Gateway restarting') == 1
+ if ending == 'cancelled':
+ task.cancel()
+ release.set()
+ async with asyncio.timeout(20):
+ await async_wait_until(task.done, timeout=20, what='the stopped turn to end')
+ return row, await task
+
+ with db:
+ row, response = asyncio.run(scenario())
+ result = authority.pending_results[row['admission_id']]['result']
+ assert response == '' and not bootstrapped
+ assert result['interrupted'] and result['interrupt_message'] == 'Gateway restarting'
+
+
+def test_a_user_stop_first_keeps_no_shutdown_cause_before_spawn():
+ """A user's Stop latched for the pending generation came first: shutdown records no cause."""
+ from gateway.run_runtime import stop_authority_turns
+ live = SimpleNamespace(event_stream=SimpleNamespace(execution={'execution_generation': 4}))
+ authority = SimpleNamespace(_managed_workers={}, _managed_spawning={'s': 4}, sessions={'s': live},
+ pending_stops={'s': 4}, pending_stop_causes={})
+ assert stop_authority_turns(authority, reason='Gateway restarting') == 1
+ assert authority.pending_stop_causes == {} and authority.pending_stops == {'s': 4}
+ authority.pending_stops = {}
+ stop_authority_turns(authority) # no reason (profile retire): latches the Stop, never a cause
+ assert authority.pending_stop_causes == {} and authority.pending_stops == {'s': 4}
+ authority.pending_stops = {}
+ assert stop_authority_turns(authority, reason='Gateway shutting down') == 1
+ assert authority.pending_stop_causes == {'s': (4, 'Gateway shutting down')}