Skip to content

The agent that grows with you

fix(gateway): wait for /cron run on the runner's turn pool, not the pool its job needs

kshitijk4poordb481bfadeb1
diff --git a/gateway/session_cron_commands.py b/gateway/session_cron_commands.py
index 360a3c12283..52ad0896839 100644
--- a/gateway/session_cron_commands.py
+++ b/gateway/session_cron_commands.py
@@ -22,4 +22,8 @@ async def run_cron(authority, arg) -> dict:
"""``/cron <arg>`` off the owner loop in the owning profile: the store is written here, never by a client."""
from gateway.session_info_reads import owner_lines
from hermes_cli.cli_cron_lines import cron_command_lines
- return await owner_lines(authority, lambda: cron_command_lines(arg))
+ # A /cron run waits for its job, and the job's admission and execution take threads from the loop's
+ # default pool (tracked_write, session_cron.execute). The wait therefore holds a thread of the runner's
+ # unbounded turn pool, never one of the pool the job needs, however many runs (or Ctrl+C'd waits) pile up.
+ return await owner_lines(authority, lambda: cron_command_lines(arg),
+ getattr(authority.runner, '_run_in_executor_with_context', None))
diff --git a/gateway/session_info_reads.py b/gateway/session_info_reads.py
index 787e974d531..3265fb65c8d 100644
--- a/gateway/session_info_reads.py
+++ b/gateway/session_info_reads.py
@@ -23,14 +23,15 @@ def _platforms(authority, policy):
READS = {'toolsets': _toolsets, 'platforms': _platforms}
-async def owner_lines(authority, lines) -> dict:
- """``slash.exec``'s reply for ``lines()``, built off the owner loop in the owning profile."""
+async def owner_lines(authority, lines, offload=None) -> dict:
+ """``slash.exec``'s reply for ``lines()``, built off the owner loop in the owning profile, on
+ ``offload`` (an awaitable executor hop) or else the loop's default pool."""
from gateway.session_authorities import owner_scope
def build():
with owner_scope(authority):
return '\n'.join(lines()).strip('\n')
- return {'type': 'exec', 'output': await asyncio.to_thread(build)}
+ return {'type': 'exec', 'output': await (offload or asyncio.to_thread)(build)}
async def run_info_read(authority, live, name) -> dict:
diff --git a/tests/gateway/test_tui_session_reads.py b/tests/gateway/test_tui_session_reads.py
index 6a2faabd8b1..7f380686cd8 100644
--- a/tests/gateway/test_tui_session_reads.py
+++ b/tests/gateway/test_tui_session_reads.py
@@ -153,6 +153,52 @@ async def test_cron_writes_stay_refused_on_a_viewer_connection_and_reads_list_al
assert 'State: paused' in await _exec(owner, sid, 'cron list --all') # nothing resumed it
+async def test_concurrent_cron_runs_never_wait_on_the_pool_their_jobs_need(owner, monkeypatch):
+ """A /cron run waits for its job, and the job's admission (tracked_write) and execution
+ (session_cron.execute) take threads from the loop's default pool. Waiting on that same pool, two
+ runs on a one-thread pool never answered: the job could not start, and the cron layer paused it as
+ unverified. The wait holds a thread of the runner's own unbounded pool instead."""
+ import asyncio
+ import concurrent.futures
+ from pathlib import Path
+ from types import MethodType
+ import cron.scheduler
+ from cron.jobs import create_job, get_job
+ from gateway.run import GatewayRunner
+ from gateway.session_controls import AuthorityConnection
+ from gateway.session_cron import current_execution
+ runner = owner.authority.runner
+ runner._get_executor = MethodType(GatewayRunner._get_executor, runner)
+ runner._run_in_executor_with_context = MethodType(GatewayRunner._run_in_executor_with_context, runner)
+ submit = cron.scheduler.run_job # outside the owner's execution: submit the fire and wait for it
+
+ def run_job(job, **kwargs):
+ return submit(job, **kwargs) if current_execution() is None else (True, '# out', 'ran', None)
+ monkeypatch.setattr(cron.scheduler, 'run_job', run_job)
+ # The fire is submitted to the owner bound to the home of the authority's store (bind_owner).
+ home = Path(owner.authority.db.db_path).resolve().parent
+ (home / 'config.yaml').write_text((owner.home / 'config.yaml').read_text())
+ monkeypatch.setenv('HERMES_HOME', str(home))
+ ids = [create_job('ping', 'every 2h', name=name)['id'] for name in ('one', 'two')]
+ sid = (await owner.call('session.create', request_id='r', source='cli', cwd=str(owner.workdir)))['session_id']
+ asyncio.get_running_loop().set_default_executor(concurrent.futures.ThreadPoolExecutor(1))
+
+ async def run(n, job_id):
+ socket = AuthorityConnection(owner.authority, object(), {'user_id': 'human'}) # the CLI's own
+ reply = await socket.dispatch({'id': n, 'method': 'slash.exec',
+ 'params': {'session_id': sid, 'command': f'cron run {job_id}'}})
+ assert 'result' in reply, reply
+ return reply['result']['output']
+ try:
+ outs = await asyncio.wait_for(asyncio.gather(*(run(n, i) for n, i in enumerate(ids))), 30)
+ finally:
+ if getattr(runner, '_executor', None) is not None:
+ runner._executor.shutdown(wait=False)
+ assert [out.splitlines()[-1].strip() for out in outs] == ['Ran now: succeeded.'] * 2, (
+ outs, [get_job(i)['last_error'] for i in ids])
+
+
@pytest.mark.asyncio
async def test_other_surfaces_keep_the_gateway_handlers(owner):
from gateway.session_tui_reads import tui_read