The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): a shutdown Stop before the managed worker spawns keeps its cause
Tekniumf959be49c16e
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.pyindex ccf9cd87ddb..42e80bf5387 100644--- a/gateway/run_runtime.py+++ b/gateway/run_runtime.py@@ -235,7 +235,8 @@ def stop_authority_turns(authority, *, in_process=False, reason=None):through its own fenced path). ``in_process`` also hard-interrupts the running agent of eachexecuting in-process turn, or latches the generation ``adopt_agent`` consumes before it exists.``reason`` (a gateway restart/shutdown) is recorded on each worker it signals, so the turn settles- with that cause; a worker already stopped (the user's Stop came first) keeps none."""+ with that cause; a worker already stopped (the user's Stop came first) keeps none. A worker+ still being spawned gets the same Stop and cause through latches (``_stop_spawning_managed``)."""from hermes_state_runtime import RuntimeStoreErrorsignalled = 0workers = getattr(authority, '_managed_workers', {})@@ -250,6 +251,7 @@ def stop_authority_turns(authority, *, in_process=False, reason=None):# loop escalates on that latch alone (terminate after STOP_ACK_SECONDS).logger.debug('managed Stop not queued (%s); the latch escalates it', exc.reason)signalled += 1+ signalled += _stop_spawning_managed(authority, workers, reason)if not in_process:return signalledfrom agent.interrupt_compat import request_hard_interrupt@@ -257,7 +259,7 @@ def stop_authority_turns(authority, *, in_process=False, reason=None):running = getattr(authority.runner, '_running_agents', {})for sid, live in list(authority.sessions.items()):generation = live.event_stream.execution.get('execution_generation')- if generation is None or sid in workers:+ if generation is None or sid in workers or sid in getattr(authority, '_managed_spawning', {}):continueadopted_generation, agent = authority.adopted.get(sid, (None, None))if adopted_generation != generation:@@ -270,6 +272,26 @@ def stop_authority_turns(authority, *, in_process=False, reason=None):return signalled+def _stop_spawning_managed(authority, workers, reason):+ """A managed turn whose worker is still being spawned has nothing to signal yet: latch the+ Stop for its generation (``adopt_agent`` delivers it to the worker, which is never+ bootstrapped) and, for a gateway restart/shutdown, the cause its pre-bootstrap interrupted+ result carries (#134889 GW-22). A user's Stop latched first keeps no cause."""+ stopped = 0+ for sid, generation in list(getattr(authority, '_managed_spawning', {}).items()):+ if sid in workers:+ continue+ stops = authority.pending_stops+ if reason and stops.get(sid) != generation:+ causes = getattr(authority, 'pending_stop_causes', None)+ if causes is None:+ causes = authority.pending_stop_causes = {}+ causes[sid] = (generation, reason)+ stops[sid] = generation+ stopped += 1+ return stopped++def stop_managed_turns(runner, reason=None):"""``stop_authority_turns`` across every served authority (whole-runtime shutdown)."""return sum(stop_authority_turns(authority, reason=reason) for authority in _authorities(runner))diff --git a/gateway/session_authority.py b/gateway/session_authority.pyindex 7c2216852e7..5c6a7e041a0 100644--- a/gateway/session_authority.py+++ b/gateway/session_authority.py@@ -132,6 +132,9 @@ class SessionAuthority:# Stops accepted for a running generation whose agent does not exist yet# (first-turn construction); consumed by adopt_agent, keyed session -> generation.self.pending_stops = {}+ # session -> (generation, reason): a gateway restart/shutdown Stop latched before that+ # generation's managed worker existed; its pre-bootstrap result carries the cause (GW-22).+ self.pending_stop_causes = {}# session -> (generation, [steer text]) accepted before that generation adopted an agent;# adopt_agent hands it to that agent (never a later generation's), the drain drops the rest.self.pending_steers = {}@@ -1144,6 +1147,7 @@ class SessionAuthority:with live.event_stream.lock:live.event_stream.execution = {}self.pending_stops.pop(ref.session_id, None)+ self.pending_stop_causes.pop(ref.session_id, None)self.pending_steers.pop(ref.session_id, None)self.adopted.pop(ref.session_id, None)# An automation turn nobody waits on settles here, at the exit that fenced it (GW-10),@@ -1267,6 +1271,7 @@ class SessionAuthority:with live.event_stream.lock:live.event_stream.execution = {}self.pending_stops.pop(ref.session_id, None)+ self.pending_stop_causes.pop(ref.session_id, None)self.pending_steers.pop(ref.session_id, None)self.adopted.pop(ref.session_id, None)from gateway.session_queued_controls import publish_transfersdiff --git a/gateway/session_managed_worker.py b/gateway/session_managed_worker.pyindex ca11678401e..49a4a0dbda4 100644--- a/gateway/session_managed_worker.py+++ b/gateway/session_managed_worker.py@@ -454,6 +454,13 @@ def _interrupted_before_bootstrap(authority, row, cause=None):return ''+def _latched_stop_cause(authority, session_id, generation):+ """The gateway restart/shutdown cause latched for this generation before its worker existed+ (``run_runtime._stop_spawning_managed``), consumed once; None for a user's Stop."""+ latched = getattr(authority, 'pending_stop_causes', {}).pop(session_id, None)+ return latched[1] if latched and latched[0] == generation else None++def _with_stop_cause(accepted, worker):"""The wire result carries no interruption cause; the owner recorded which stop it sent."""if accepted.get('interrupted') and worker.stop_cause:@@ -514,14 +521,14 @@ def _compression_ancestors(db, session_id):return ancestors[::-1]-async def execute_managed(authority, ref, row, policy):- from gateway.run_turn_progress import publish_worker_tool_event- # This route never passes the inbound prepare step, so the worker would read ``@`` tokens as typed.- from gateway.session_context_refs import managed_turn_text, refused_turn_result- text, refusal = await managed_turn_text(authority, row, policy)- if refusal is not None:- authority.pending_results[row['admission_id']] = {'result': refused_turn_result(row, refusal), 'usage': {}}- return refusal+async def _spawn_registered_worker(authority, ref, row):+ """Spawn the turn's child and register it for Stops: ``(worker, process)``, or None when the spawn was cancelled (the+ interrupted result is already recorded). Until registration a gateway stop has nothing to+ signal, so the generation is marked spawning and the stop latches instead (#134889 GW-22)."""+ spawning = getattr(authority, '_managed_spawning', None)+ if spawning is None:+ spawning = authority._managed_spawning = {}+ spawning[ref.session_id] = generation = row.get('generation')try:env = await asyncio.to_thread(_worker_env, authority)cwd = (await asyncio.to_thread(Path(__file__).resolve)).parents[1]@@ -533,7 +540,12 @@ async def execute_managed(authority, ref, row, policy):stderr=subprocess.DEVNULL if log_path is None else subprocess.PIPE, **WORKER_POPEN)except asyncio.CancelledError:# acquire_process owns late-child cleanup; no bootstrap or reservation was sent.- return _interrupted_before_bootstrap(authority, row)+ spawning.pop(ref.session_id, None)+ _interrupted_before_bootstrap(authority, row, _latched_stop_cause(authority, ref.session_id, generation))+ return None+ except BaseException:+ spawning.pop(ref.session_id, None)+ raiseworker = ManagedWorker(process)if log_path is not None:worker.drain_stderr(log_path)@@ -541,6 +553,24 @@ async def execute_managed(authority, ref, row, policy):if workers is None:workers = authority._managed_workers = {}workers[ref.session_id] = worker+ spawning.pop(ref.session_id, None)+ worker.stop_cause = _latched_stop_cause(authority, ref.session_id, generation)+ return worker, process+++async def execute_managed(authority, ref, row, policy):+ from gateway.run_turn_progress import publish_worker_tool_event+ # This route never passes the inbound prepare step, so the worker would read ``@`` tokens as typed.+ from gateway.session_context_refs import managed_turn_text, refused_turn_result+ text, refusal = await managed_turn_text(authority, row, policy)+ if refusal is not None:+ authority.pending_results[row['admission_id']] = {'result': refused_turn_result(row, refusal), 'usage': {}}+ return refusal+ spawned = await _spawn_registered_worker(authority, ref, row)+ if spawned is None:+ return ''+ worker, process = spawned+ workers = authority._managed_workers# A Stop acknowledged during the env/spawn awaits found no worker and was latched for this# generation; consume it in the same step that makes the worker reachable to interrupt_managed.authority.adopt_agent(ref.session_id, row['generation'], worker)diff --git a/tests/gateway/test_restart_interrupt_notice.py b/tests/gateway/test_restart_interrupt_notice.pyindex 33894928ab4..0e89008e304 100644--- a/tests/gateway/test_restart_interrupt_notice.py+++ b/tests/gateway/test_restart_interrupt_notice.py@@ -395,3 +395,80 @@ def test_automation_turn_transcript_notice_is_the_neutral_copy(tmp_path):db, *_ = _settle_on_real_store(tmp_path, 'Gateway restarting', principal='cron-owner')with db:assert _notices(db) == [AUTOMATION_INTERRUPTED_NOTICE]++[email protected]("linux")[email protected]('ending', ['spawned', 'cancelled'])+def test_shutdown_stop_before_the_managed_worker_spawned_carries_its_cause(tmp_path, monkeypatch, ending):+ """#134889 GW-22: the gateway's stop lands while the managed worker is still being spawned+ (no worker to signal yet). The authority latches the cause for that generation, so the+ pre-bootstrap interrupted result carries it: the spawned child is stopped before it is ever+ bootstrapped, or the cancelled spawn settles with the cause. Real child, real SQLite."""+ import subprocess++ from gateway import session_managed_worker as managed+ from gateway.run_runtime import stop_authority_turns+ from gateway.session_authority import LiveSession, SessionAuthority+ from gateway.session_contract import Principal, SessionRef, Submission+ from gateway.session_policy import build_policy+ from hermes_state import SessionDB+ from hermes_state_runtime import begin_runtime_epoch, claim_session_input+ from tests.gateway.restart_test_helpers import make_restart_runner+ from tests.waiting import async_wait_until++ monkeypatch.setenv('HERMES_HOME', str(tmp_path))+ runner, _adapter = make_restart_runner()+ db = SessionDB(db_path=tmp_path / 'state.db')+ db.create_session('s', source='test')+ authority = SessionAuthority(runner, profile_id='owned', instance_id='i', db=db,+ epoch=begin_runtime_epoch(db, instance_id='i'))+ live = authority.sessions['s'] = LiveSession(SimpleNamespace(platform=None, user_id='human', chat_id='c'), 'route')+ actor = Principal('human', 'owned', frozenset({'session:submit', 'session:control'}), 'cli')+ ref = SessionRef('owned', 's')+ monkeypatch.setattr(authority, '_schedule', lambda ref: None)+ spawning, release, bootstrapped = threading.Event(), threading.Event(), []++ class Child(subprocess.Popen):+ def __init__(self, args, **kw):+ spawning.set()+ release.wait(30)+ super().__init__([sys.executable, '-c', STOPPABLE_CHILD], **kw)+ monkeypatch.setattr(managed.subprocess, 'Popen', Child)+ monkeypatch.setattr(managed, '_bootstrap', lambda *a: bootstrapped.append(a) or pytest.fail('bootstrapped'))+ policy = build_policy({'cwd': str(tmp_path), 'model': 'm'}, {}, profile_terminal=False)++ async def scenario():+ await authority.submit(actor, Submission('turn', ref, {'text': 'go'}, 'queue'))+ row = claim_session_input(db, epoch=authority.epoch, session_id='s')+ authority._stamp_claim(ref, live, row)+ task = asyncio.create_task(managed.execute_managed(authority, ref, row, policy))+ await asyncio.to_thread(spawning.wait, 30)+ assert stop_authority_turns(authority, reason='Gateway restarting') == 1+ if ending == 'cancelled':+ task.cancel()+ release.set()+ async with asyncio.timeout(20):+ await async_wait_until(task.done, timeout=20, what='the stopped turn to end')+ return row, await task++ with db:+ row, response = asyncio.run(scenario())+ result = authority.pending_results[row['admission_id']]['result']+ assert response == '' and not bootstrapped+ assert result['interrupted'] and result['interrupt_message'] == 'Gateway restarting'+++def test_a_user_stop_first_keeps_no_shutdown_cause_before_spawn():+ """A user's Stop latched for the pending generation came first: shutdown records no cause."""+ from gateway.run_runtime import stop_authority_turns+ live = SimpleNamespace(event_stream=SimpleNamespace(execution={'execution_generation': 4}))+ authority = SimpleNamespace(_managed_workers={}, _managed_spawning={'s': 4}, sessions={'s': live},+ pending_stops={'s': 4}, pending_stop_causes={})+ assert stop_authority_turns(authority, reason='Gateway restarting') == 1+ assert authority.pending_stop_causes == {} and authority.pending_stops == {'s': 4}+ authority.pending_stops = {}+ stop_authority_turns(authority) # no reason (profile retire): latches the Stop, never a cause+ assert authority.pending_stop_causes == {} and authority.pending_stops == {'s': 4}+ authority.pending_stops = {}+ assert stop_authority_turns(authority, reason='Gateway shutting down') == 1+ assert authority.pending_stop_causes == {'s': (4, 'Gateway shutting down')}