Skip to content

The agent that grows with you

fix(gateway): the timed-out stop drain interrupts counted side agents

Teknium08449d27b313
diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py
index 8d578cd8e94..f205fedce12 100644
--- a/gateway/run_shutdown.py
+++ b/gateway/run_shutdown.py
@@ -29,6 +29,7 @@ from gateway.restart import (
)
from gateway.run_common import _UNSET
from gateway.run_runtime import managed_turn_count, stop_managed_turns
+from gateway.session_runtime_workers import interrupt_side_agents
from gateway.run_shutdown_session_end import GatewaySessionEndMixin
from gateway.shutdown_watchdog import arm_shutdown_watchdog, resolve_shutdown_watchdog_delay
@@ -856,11 +857,13 @@ class GatewayShutdownMixin(GatewaySessionEndMixin):
with _log_suppressed(logging.DEBUG, "Failed interrupting agent during shutdown: %s"):
request_hard_interrupt(agent, reason, tool_reason=_INTERRUPT_TOOL_REASON_GATEWAY_SHUTDOWN)
logger.debug("Interrupted running agent for session %s during shutdown", session_key)
- # API-server / desk / managed-worker turns never enter _running_agents: the drain waited on them.
+ # API-server / desk / managed-worker turns and side agents never enter _running_agents: the
+ # drain waited on them.
for count, what in (
(self._interrupt_api_server_runs(reason), "api_server run(s)"),
(self._interrupt_deferred_agent_workers(reason), "deferred agent worker(s)"),
(stop_managed_turns(self), "managed-worker turn(s)"),
+ (interrupt_side_agents(self, reason), "side agent(s)"),
):
if count:
logger.debug("Interrupted %d %s during shutdown", count, what)
diff --git a/gateway/session_runtime_workers.py b/gateway/session_runtime_workers.py
index 841e6272610..a22f82587ed 100644
--- a/gateway/session_runtime_workers.py
+++ b/gateway/session_runtime_workers.py
@@ -32,6 +32,18 @@ def side_agents(authority):
if not _abandoned(holder)]
+def interrupt_side_agents(runner, reason):
+ """Hard-interrupt every counted side agent's holder across served authorities (the stop
+ drain waits for them, so its timed-out interrupt must reach them). Returns how many."""
+ from agent.interrupt_compat import request_hard_interrupt
+ count = 0
+ for authority in all_authorities(runner):
+ for _, holder in side_agents(authority):
+ if holder and holder[0] is not None:
+ count += bool(request_hard_interrupt(holder[0], reason))
+ return count
+
+
def persist_work_count(authority, _task=None):
"""Re-publish the runner's in-flight count when owner work it counts (a drain, a tracked
mutation, a mailbox operation) ends: the agent-slot release that last wrote
diff --git a/tests/gateway/test_runtime_worker_shutdown.py b/tests/gateway/test_runtime_worker_shutdown.py
index ad6ba3cd3f5..8a3e4e67a20 100644
--- a/tests/gateway/test_runtime_worker_shutdown.py
+++ b/tests/gateway/test_runtime_worker_shutdown.py
@@ -297,3 +297,19 @@ async def test_btw_side_question_does_not_hold_profile_retirement(monkeypatch, p
assert not thread.is_alive() and events == [] # a stopped /btw publishes no late answer
finally:
release.set()
+
+
+def test_shutdown_interrupt_reaches_counted_side_agents():
+ """The stop drain waits for counted side agents (/bg, /btw, preview restart), so the timed-out
+ drain's interrupt must reach them too, or a long /bg holds the whole drain and is then
+ abandoned mid-tool when the process exits."""
+ from gateway.run_shutdown import GatewayShutdownMixin
+ interrupts = []
+ agent = SimpleNamespace(hard_interrupt=lambda message=None, **_: interrupts.append(message))
+ authority = SimpleNamespace(sessions={}, pending_stops={}, adopted={},
+ _side_agents={object(): [agent], object(): [None], object(): None})
+ runner = SimpleNamespace(_running_agents={}, session_authority=authority,
+ _interrupt_api_server_runs=lambda reason: 0,
+ _interrupt_deferred_agent_workers=lambda reason: 0)
+ GatewayShutdownMixin._interrupt_running_agents(runner, 'Gateway shutting down')
+ assert interrupts == ['Gateway shutting down']