The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(api_server): the API turn executor is retired at adapter disconnect
Teknium5f0182e61147
diff --git a/gateway/platforms/api_server_runs.py b/gateway/platforms/api_server_runs.pyindex a7e0f7a576e..599812a1175 100644--- a/gateway/platforms/api_server_runs.py+++ b/gateway/platforms/api_server_runs.py@@ -46,12 +46,35 @@ _API_WORKER_LIVE = 0# shared with every ``asyncio.to_thread`` SessionDB read in the session handlers, so once live# turns filled it each /api/sessions read queued behind a whole turn (#144362). Turn concurrency is# bounded where it is refused: ``gateway.api_server.max_concurrent_runs`` (429 past the cap).-_API_TURN_EXECUTOR = _UnboundedThreadExecutor(thread_name_prefix="hermes-api-turn")+# Created on first use and retired at adapter disconnect (``_close_run_state``), so its parked+# threads do not outlive the adapter that used them; the loop's default executor this replaced+# went away with its loop the same way.+_API_TURN_EXECUTOR: _UnboundedThreadExecutor | None = None+_API_TURN_EXECUTOR_LOCK = threading.Lock()def api_turn_executor() -> _UnboundedThreadExecutor:"""The executor that runs API-server agent turns and cron fires (never the loop's default)."""- return _API_TURN_EXECUTOR+ global _API_TURN_EXECUTOR+ with _API_TURN_EXECUTOR_LOCK:+ if _API_TURN_EXECUTOR is None:+ _API_TURN_EXECUTOR = _UnboundedThreadExecutor(thread_name_prefix="hermes-api-turn")+ return _API_TURN_EXECUTOR+++def retire_api_turn_executor() -> _UnboundedThreadExecutor | None:+ """Shut the API turn executor down without waiting and return it (None if never used).++ Parked workers exit now; a worker still inside a turn finishes it, releases its+ ``api_worker_live_count`` share as before, and exits instead of parking. A later submit gets+ a fresh executor, so another adapter still serving keeps working.+ """+ global _API_TURN_EXECUTOR+ with _API_TURN_EXECUTOR_LOCK:+ executor, _API_TURN_EXECUTOR = _API_TURN_EXECUTOR, None+ if executor is not None:+ executor.shutdown(wait=False)+ return executordef api_worker_live_count() -> int:@@ -63,7 +86,7 @@ def api_worker_live_count() -> int:async def on_turn_executor(fn, /, *args, **kwargs):"""``asyncio.to_thread`` (context copied the same way) on the API turn executor."""call = functools.partial(contextvars.copy_context().run, fn, *args, **kwargs)- return await asyncio.get_running_loop().run_in_executor(_API_TURN_EXECUTOR, call)+ return await asyncio.get_running_loop().run_in_executor(api_turn_executor(), call)def _submit_api_worker(loop, fn):@@ -88,7 +111,7 @@ def _submit_api_worker(loop, fn):_API_WORKER_LIVE -= 1try:- return loop.run_in_executor(_API_TURN_EXECUTOR, _counted)+ return loop.run_in_executor(api_turn_executor(), _counted)except BaseException:with _API_WORKER_LOCK:_API_WORKER_LIVE -= 1@@ -277,6 +300,7 @@ def _idempotency_capabilities(self, *, store_type) -> dict[str, Any]:def _close_run_state(self) -> None:+ retire_api_turn_executor()try:if getattr(self, "_run_idempotency_store", None) is not None:self._run_idempotency_store.close()diff --git a/gateway/run_shutdown.py b/gateway/run_shutdown.pyindex 27e37700ae5..6f187b62d14 100644--- a/gateway/run_shutdown.py+++ b/gateway/run_shutdown.py@@ -2059,10 +2059,10 @@ class GatewayShutdownMixin(GatewaySessionEndMixin):)returnlogger.info("Shutdown phase: executor quiesced at +%.2fs", ctx.elapsed())- # Cron jobs (scheduler pool), API-server runs (their own bounded executor) and deferred hygiene- # workers (the loop's default executor) never touch self._executor, so the join above cannot see them. A writer that- # outlived the drain is mid-write for the same #101093 reasons; the drain already spent its- # budget, so no second wait — leave the handles open (#102198). The API count is the snapshot+ # Cron jobs (scheduler pool), API-server runs (their own unbounded executor, never joined at disconnect)+ # and deferred hygiene workers (the loop's default executor) never touch self._executor, so the join above+ # cannot see them. A writer that outlived the drain is mid-write for the same #101093 reasons; the drain+ # spent its budget, so no second wait — leave the handles open (#102198). The API count is the snapshot# taken before the adapters were released; a run whose handler task was cancelled at disconnect# has already left it, so that term under-counts — the live worker-scoped count below covers# the cancelled-handler case (#116535).diff --git a/tests/gateway/conftest.py b/tests/gateway/conftest.pyindex 09f3c2f942a..9f731fa5ca7 100644--- a/tests/gateway/conftest.py+++ b/tests/gateway/conftest.py@@ -40,6 +40,22 @@ from unittest.mock import MagicMockimport pytest[email protected](autouse=True)+def _join_api_turn_workers():+ """Join the API turn workers a test left running, as its loop's teardown used to.++ API turns ran on the loop's default executor, which each test's loop teardown joined. They now+ run on ``api_server_runs``' own executor, which only an adapter disconnect retires, and most+ API-server tests never disconnect: a worker still finishing would raise the process-global+ ``api_worker_live_count`` inside the next test (the shutdown close-gate tests read it).+ """+ yield+ runs = sys.modules.get("gateway.platforms.api_server_runs")+ executor = runs.retire_api_turn_executor() if runs is not None else None+ for thread in list(getattr(executor, "_threads", ())):+ thread.join(30)++@pytest.fixturedef private_db_probe_cleanup(tmp_path):"""Release only this test's probe fds after its explicit SQLite-handle finalizers.diff --git a/tests/gateway/test_api_server_turn_executor.py b/tests/gateway/test_api_server_turn_executor.pyindex 8179130566e..bcae4675280 100644--- a/tests/gateway/test_api_server_turn_executor.py+++ b/tests/gateway/test_api_server_turn_executor.py@@ -54,3 +54,18 @@ async def test_cron_fire_body_keeps_contextvars_off_the_default_pool():out = await api_runs.on_turn_executor(_fire, "job-1", adapters={"a": 1}, loop=asyncio.get_running_loop())assert out == "fired"assert seen["scope"] == "profile-a" and seen["job"] == "job-1" and seen["adapters"] == {"a": 1}+++async def test_adapter_disconnect_leaves_no_api_turn_thread_behind():+ from gateway.config import PlatformConfig+ from gateway.platforms.api_server import APIServerAdapter++ loop = asyncio.get_running_loop()+ ran_on = []+ await api_runs._submit_api_worker(loop, lambda: ran_on.append(threading.current_thread()))+ adapter = APIServerAdapter(PlatformConfig(enabled=True))+ await adapter.disconnect()+ ran_on[0].join(5)+ assert not ran_on[0].is_alive(), "an idle API turn thread outlived the adapter's disconnect"+ assert await api_runs._submit_api_worker(loop, lambda: "ran") == "ran", "no API turn ran after it"