Skip to content

The agent that grows with you

fix(api_server): the API turn executor is retired at adapter disconnect

Teknium5f0182e61147
diff --git a/gateway/platforms/api_server_runs.py b/gateway/platforms/api_server_runs.py
index a7e0f7a576e..599812a1175 100644
--- a/gateway/platforms/api_server_runs.py
+++ b/gateway/platforms/api_server_runs.py
@@ -46,12 +46,35 @@ _API_WORKER_LIVE = 0
# shared with every ``asyncio.to_thread`` SessionDB read in the session handlers, so once live
# turns filled it each /api/sessions read queued behind a whole turn (#144362). Turn concurrency is
# bounded where it is refused: ``gateway.api_server.max_concurrent_runs`` (429 past the cap).
-_API_TURN_EXECUTOR = _UnboundedThreadExecutor(thread_name_prefix="hermes-api-turn")
+# Created on first use and retired at adapter disconnect (``_close_run_state``), so its parked
+# threads do not outlive the adapter that used them; the loop's default executor this replaced
+# went away with its loop the same way.
+_API_TURN_EXECUTOR: _UnboundedThreadExecutor | None = None
+_API_TURN_EXECUTOR_LOCK = threading.Lock()
def api_turn_executor() -> _UnboundedThreadExecutor:
"""The executor that runs API-server agent turns and cron fires (never the loop's default)."""
- return _API_TURN_EXECUTOR
+ global _API_TURN_EXECUTOR
+ with _API_TURN_EXECUTOR_LOCK:
+ if _API_TURN_EXECUTOR is None:
+ _API_TURN_EXECUTOR = _UnboundedThreadExecutor(thread_name_prefix="hermes-api-turn")
+ return _API_TURN_EXECUTOR
+
+
+def retire_api_turn_executor() -> _UnboundedThreadExecutor | None:
+ """Shut the API turn executor down without waiting and return it (None if never used).
+
+ Parked workers exit now; a worker still inside a turn finishes it, releases its
+ ``api_worker_live_count`` share as before, and exits instead of parking. A later submit gets
+ a fresh executor, so another adapter still serving keeps working.
+ """
+ global _API_TURN_EXECUTOR
+ with _API_TURN_EXECUTOR_LOCK:
+ executor, _API_TURN_EXECUTOR = _API_TURN_EXECUTOR, None
+ if executor is not None:
+ executor.shutdown(wait=False)
+ return executor
def api_worker_live_count() -> int:
@@ -63,7 +86,7 @@ def api_worker_live_count() -> int:
async def on_turn_executor(fn, /, *args, **kwargs):
"""``asyncio.to_thread`` (context copied the same way) on the API turn executor."""
call = functools.partial(contextvars.copy_context().run, fn, *args, **kwargs)
- return await asyncio.get_running_loop().run_in_executor(_API_TURN_EXECUTOR, call)
+ return await asyncio.get_running_loop().run_in_executor(api_turn_executor(), call)
def _submit_api_worker(loop, fn):
@@ -88,7 +111,7 @@ def _submit_api_worker(loop, fn):
_API_WORKER_LIVE -= 1
try:
- return loop.run_in_executor(_API_TURN_EXECUTOR, _counted)
+ return loop.run_in_executor(api_turn_executor(), _counted)
except BaseException:
with _API_WORKER_LOCK:
_API_WORKER_LIVE -= 1
@@ -277,6 +300,7 @@ def _idempotency_capabilities(self, *, store_type) -> dict[str, Any]:
def _close_run_state(self) -> None:
+ retire_api_turn_executor()
try:
if getattr(self, "_run_idempotency_store", None) is not None:
self._run_idempotency_store.close()
diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.py
index 27e37700ae5..6f187b62d14 100644
--- a/gateway/run_shutdown.py
+++ b/gateway/run_shutdown.py
@@ -2059,10 +2059,10 @@ class GatewayShutdownMixin(GatewaySessionEndMixin):
)
return
logger.info("Shutdown phase: executor quiesced at +%.2fs", ctx.elapsed())
- # Cron jobs (scheduler pool), API-server runs (their own bounded executor) and deferred hygiene
- # workers (the loop's default executor) never touch self._executor, so the join above cannot see them. A writer that
- # outlived the drain is mid-write for the same #101093 reasons; the drain already spent its
- # budget, so no second wait — leave the handles open (#102198). The API count is the snapshot
+ # Cron jobs (scheduler pool), API-server runs (their own unbounded executor, never joined at disconnect)
+ # and deferred hygiene workers (the loop's default executor) never touch self._executor, so the join above
+ # cannot see them. A writer that outlived the drain is mid-write for the same #101093 reasons; the drain
+ # spent its budget, so no second wait — leave the handles open (#102198). The API count is the snapshot
# taken before the adapters were released; a run whose handler task was cancelled at disconnect
# has already left it, so that term under-counts — the live worker-scoped count below covers
# the cancelled-handler case (#116535).
diff --git a/tests/gateway/conftest.py b/tests/gateway/conftest.py
index 09f3c2f942a..9f731fa5ca7 100644
--- a/tests/gateway/conftest.py
+++ b/tests/gateway/conftest.py
@@ -40,6 +40,22 @@ from unittest.mock import MagicMock
import pytest
[email protected](autouse=True)
+def _join_api_turn_workers():
+ """Join the API turn workers a test left running, as its loop's teardown used to.
+
+ API turns ran on the loop's default executor, which each test's loop teardown joined. They now
+ run on ``api_server_runs``' own executor, which only an adapter disconnect retires, and most
+ API-server tests never disconnect: a worker still finishing would raise the process-global
+ ``api_worker_live_count`` inside the next test (the shutdown close-gate tests read it).
+ """
+ yield
+ runs = sys.modules.get("gateway.platforms.api_server_runs")
+ executor = runs.retire_api_turn_executor() if runs is not None else None
+ for thread in list(getattr(executor, "_threads", ())):
+ thread.join(30)
+
+
@pytest.fixture
def private_db_probe_cleanup(tmp_path):
"""Release only this test's probe fds after its explicit SQLite-handle finalizers.
diff --git a/tests/gateway/test_api_server_turn_executor.py b/tests/gateway/test_api_server_turn_executor.py
index 8179130566e..bcae4675280 100644
--- a/tests/gateway/test_api_server_turn_executor.py
+++ b/tests/gateway/test_api_server_turn_executor.py
@@ -54,3 +54,18 @@ async def test_cron_fire_body_keeps_contextvars_off_the_default_pool():
out = await api_runs.on_turn_executor(_fire, "job-1", adapters={"a": 1}, loop=asyncio.get_running_loop())
assert out == "fired"
assert seen["scope"] == "profile-a" and seen["job"] == "job-1" and seen["adapters"] == {"a": 1}
+
+
+async def test_adapter_disconnect_leaves_no_api_turn_thread_behind():
+ from gateway.config import PlatformConfig
+ from gateway.platforms.api_server import APIServerAdapter
+
+ loop = asyncio.get_running_loop()
+ ran_on = []
+ await api_runs._submit_api_worker(loop, lambda: ran_on.append(threading.current_thread()))
+ adapter = APIServerAdapter(PlatformConfig(enabled=True))
+ await adapter.disconnect()
+ ran_on[0].join(5)
+ assert not ran_on[0].is_alive(), "an idle API turn thread outlived the adapter's disconnect"
+ assert await api_runs._submit_api_worker(loop, lambda: "ran") == "ran", "no API turn ran after it"