Skip to content

The agent that grows with you

fix(gateway): Desktop llm.oneshot borrows the chat's model on the shared gateway

unsupportedpastelsff3aec5ed26e
diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py
index 4403b1c4e9f..baf3cfc77c1 100644
--- a/agent/auxiliary_client.py
+++ b/agent/auxiliary_client.py
@@ -5111,11 +5111,11 @@ def _resolve_custom_branch(req: _ResolveRequest) -> _ResolveResult:
final_model = _normalize_resolved_model(
model or (main_runtime.get("model") if main_runtime else None) or "gpt-4o-mini", provider,
)
- extra = {}
_clean_base, _dq = _extract_url_query_params(custom_base)
- if _dq:
- extra["default_query"] = _dq
- _custom_headers = _endpoint_default_headers(custom_base, provider, is_vision=req.is_vision)
+ extra = {"default_query": _dq} if _dq else {}
+ from hermes_cli.config import get_custom_provider_extra_headers # this endpoint's entry, as agent_init
+ _custom_headers = {**(_endpoint_default_headers(custom_base, provider, is_vision=req.is_vision) or {}),
+ **get_custom_provider_extra_headers(custom_base)}
if _custom_headers:
extra["default_headers"] = _custom_headers
client = _create_openai_client(api_key=custom_key, base_url=_clean_base, **extra)
diff --git a/gateway/run_runtime.py b/gateway/run_runtime.py
index 78204639e12..ca8afbcf4ec 100644
--- a/gateway/run_runtime.py
+++ b/gateway/run_runtime.py
@@ -277,6 +277,8 @@ async def _retire_profile_authority(authority, timeout=None):
if getattr(authority.runner, 'session_ticket_store', None) is not None:
_publish_served_set(authority.runner) # discovery and tickets stop naming it now
stop_authority_work(authority)
+ from gateway.session_llm_oneshot import stop as stop_oneshots
+ stop_oneshots(authority)
service = getattr(authority, 'hosted_room_service', None)
rooms = None if service is None else asyncio.ensure_future(asyncio.to_thread(service.stop, timeout=timeout))
try:
@@ -568,6 +570,9 @@ async def settle_gateway_runtime(runner, budget=None):
the ownership lock, so raising here would only skip teardown and disarm the watchdog."""
authorities = _authorities(runner)
stop_managed_turns(runner)
+ from gateway.session_llm_oneshot import stop as stop_oneshots
+ for authority in authorities:
+ stop_oneshots(authority) # an answer is no durable settlement: never waited for
tasks = [task for authority in authorities for task in _authority_tasks(authority)]
wait = TURN_SETTLE_SECONDS if budget is None else max(0.0, min(TURN_SETTLE_SECONDS,
budget - STOP_TAIL_RESERVE_SECONDS))
diff --git a/gateway/session_controls.py b/gateway/session_controls.py
index 6f0af656650..a67bcff7282 100644
--- a/gateway/session_controls.py
+++ b/gateway/session_controls.py
@@ -193,6 +193,8 @@ class AuthorityConnection:
handlers.update(tui_ops_handlers(self))
from gateway.session_tui_side import handlers as side_handlers
handlers.update(side_handlers(self))
+ from gateway.session_llm_oneshot import handlers as oneshot_handlers
+ handlers.update(oneshot_handlers(self))
from gateway.session_images import attach_bytes
from functools import partial
handlers['image.attach_bytes'] = partial(attach_bytes, self)
@@ -290,6 +292,10 @@ class AuthorityConnection:
'data': {'reason': 'unknown_method'}}}
else:
result = await handlers[method](ref, params)
+ from gateway.session_llm_oneshot import DeferredReply
+ if isinstance(result, DeferredReply): # answered off the serial dispatch
+ check_params_accepted(contract, original_params)
+ return result.start(self, rid, partial(check_result, contract))
check_params_accepted(contract, original_params)
check_result(contract, result)
return {'jsonrpc': '2.0', 'id': rid, 'result': result}
@@ -757,6 +763,8 @@ class AuthorityConnection:
for sibling in self._siblings.values():
await sibling.close()
self._siblings.clear()
+ from gateway.session_llm_oneshot import forget
+ forget(self) # nobody is left to read its one-shot replies
# Deletion may already have evicted a subscribed session; its membership died
# with it, and one retired ID must not leave the others (or the transport) attached.
for session_id, subscription in list(self.subscriptions.items()):
diff --git a/gateway/session_llm_oneshot.py b/gateway/session_llm_oneshot.py
new file mode 100644
index 00000000000..c640bbf09d3
--- /dev/null
+++ b/gateway/session_llm_oneshot.py
@@ -0,0 +1,231 @@
+"""``llm.oneshot`` on the shared gateway: a session-bound one-shot borrows that session's model.
+
+Main's sidecar looked the session up in its own table and lent its live agent's route
+(``tui_gateway/session_registry._main_runtime_from_agent``); a session-less call (project ideas)
+ran on the ``task`` auxiliary backend. The sidecar's table never holds an owner session, so on the
+shared gateway every Desktop one-shot fell through to the task backend. The owner answers it here:
+the resident agent's route, else the session's own frozen launch route.
+
+The model call never runs on the connection's serial dispatch: like the sidecar's long-handler
+pool, the reply is written by a detached task, so a slow commit message does not hold up the next
+request on that socket. Nor does it run on the loop's default executor, which session resume,
+socket setup and disconnect cleanup share across every connection and profile: each owner has its
+own small pool, and a call past its bound is refused by name instead of queueing without limit.
+"""
+import asyncio
+from concurrent.futures import Future
+from contextvars import copy_context
+from functools import partial
+import logging
+import threading
+
+from gateway.session_busy_controls import authorize
+from hermes_state_runtime import RuntimeStoreError
+
+logger = logging.getLogger(__name__)
+
+_FIELDS = frozenset({'session_id', 'profile', 'template', 'instructions', 'input', 'variables', 'task',
+ 'temperature', 'max_tokens'})
+_TEXT = ('template', 'instructions', 'input', 'task')
+# Per owner: model calls running at once, and calls admitted in all (running + queued). A call holds
+# its slot until its thread returns, even after its socket closed.
+_WORKERS = 4
+_ADMITTED = 16
+
+
+def handlers(connection):
+ return {'llm.oneshot': partial(oneshot, connection)}
+
+
+class _Pool:
+ """One owner's one-shot threads, its admitted calls, and the answer task of each per connection.
+
+ Its own FIFO, not a ThreadPoolExecutor: an executor keeps a cancelled queued call (its params and
+ copied context) until a worker dequeues it, so with every worker parked on the model a reconnect
+ loop would pile up payloads past the bound. Here a cancelled call leaves the queue at once."""
+
+ def __init__(self):
+ self.lock = threading.Lock() # ``calls``, ``queue`` and ``threads`` are shared with the workers
+ self.calls = set() # admitted calls, queued or running: at most _ADMITTED
+ self.queue = {} # queued call -> its work, oldest first
+ self.threads = 0 # worker threads alive: at most _WORKERS
+ self.answers = {} # answer task -> connection
+ self.closed = False
+
+ def submit(self, work):
+ """Admit ``work`` (or refuse it: ``runtime_draining``, ``oneshot_busy``) and return its Future."""
+ call = Future()
+ with self.lock:
+ if self.closed:
+ raise RuntimeStoreError('runtime_draining')
+ if len(self.calls) >= _ADMITTED:
+ raise RuntimeStoreError('oneshot_busy', detail=f'{_ADMITTED} one-shot generations already in flight')
+ self.calls.add(call)
+ self.queue[call] = work
+ spawn = self.threads < _WORKERS
+ self.threads += spawn
+ call.add_done_callback(self.release)
+ if spawn:
+ # Daemon: a call stuck on its model must not hold up interpreter exit (it writes no state).
+ from agent.memory_provider import spawn_context_thread
+ spawn_context_thread(self.drain, name='hermes-oneshot').start()
+ return call
+
+ def release(self, call):
+ """The call finished, or was cancelled while queued: its slot and its queued work go now."""
+ with self.lock:
+ self.calls.discard(call)
+ self.queue.pop(call, None)
+
+ def drain(self):
+ while True:
+ with self.lock:
+ if not self.queue:
+ self.threads -= 1
+ return
+ call = next(iter(self.queue))
+ work = self.queue.pop(call)
+ if call.set_running_or_notify_cancel():
+ try:
+ call.set_result(work())
+ except BaseException as exc: # health: allow BLE001 -- the Future hands every failure to _answer, which logs it with exc_info
+ call.set_exception(exc)
+ del call, work # a finished call's payload is not kept while this thread waits for more
+
+
+def _pool(authority) -> _Pool:
+ pool = getattr(authority, '_oneshot_pool', None)
+ if not isinstance(pool, _Pool):
+ pool = authority._oneshot_pool = _Pool()
+ return pool
+
+
+def forget(connection):
+ """The socket closed: drop its queued calls and answers. A call already running in its thread
+ cannot be interrupted; it finishes, keeps its slot until then, and its reply is discarded."""
+ pool = getattr(connection.authority, '_oneshot_pool', None)
+ for task, owner in list(pool.answers.items()) if isinstance(pool, _Pool) else ():
+ if owner is connection:
+ task.cancel()
+
+
+def stop(authority):
+ """Owner shutdown (whole runtime or one profile's unserve): refuse new calls, cancel every answer
+ and queued call. Running calls finish on their own (no SessionDB writes, only the model reply)."""
+ pool = _pool(authority) # an owner that never served one still refuses from now on
+ with pool.lock:
+ pool.closed = True
+ queued = list(pool.queue)
+ for task in list(pool.answers):
+ task.cancel()
+ for call in queued:
+ call.cancel()
+
+
+class DeferredReply:
+ """A handler result answered later: ``dispatch`` hands over the request id and sends no frame."""
+
+ def __init__(self, work):
+ self.work = work # blocking callable -> result dict; runs on the owner's one-shot pool
+
+ def start(self, connection, rid, check=None):
+ """Admit the call (or refuse it: ``oneshot_busy``, ``runtime_draining``) and answer it later;
+ ``check`` is the contract's result check, run before the reply is written."""
+ pool = _pool(connection.authority)
+ # The handler's context (profile scope, secret scope) carries into the thread.
+ call = pool.submit(partial(copy_context().run, self.work))
+ task = asyncio.get_running_loop().create_task(_answer(connection.transport, rid, call, check))
+ pool.answers[task] = connection
+
+ def settled(done):
+ pool.answers.pop(done, None)
+ if done.cancelled():
+ call.cancel() # still queued: leaves the queue, never runs (also before its first step)
+ task.add_done_callback(settled)
+
+
+async def _answer(transport, rid, call, check):
+ try:
+ result = await asyncio.wrap_future(call)
+ if check is not None:
+ check(result)
+ frame = {'jsonrpc': '2.0', 'id': rid, 'result': result}
+ except Exception as exc: # the request's only answer: every failure must reach the client
+ # The sidecar's own failure code and text (tui_gateway/methods_session.py).
+ logger.warning('llm.oneshot failed: %s', exc, exc_info=True)
+ frame = {'jsonrpc': '2.0', 'id': rid,
+ 'error': {'code': 5030, 'message': f'one-shot generation failed: {exc}'}}
+ await transport.write_async(frame)
+
+
+def _validate(params):
+ from agent.oneshot import PROMPT_TEMPLATES
+ if (set(params) - _FIELDS or any(key in params and params[key] is not None and not isinstance(params[key], str)
+ for key in (*_TEXT, 'session_id'))
+ or not isinstance(params.get('variables') or {}, dict)
+ or any(key in params and params[key] is not None
+ and (isinstance(params[key], bool) or not isinstance(params[key], (int, float)))
+ for key in ('temperature', 'max_tokens'))):
+ raise RuntimeStoreError('invalid_params')
+ template = (params.get('template') or '').strip()
+ if template and template not in PROMPT_TEMPLATES:
+ raise RuntimeStoreError('invalid_params', field='template', detail='unknown one-shot template')
+ if not template and not (params.get('instructions') or '').strip() and not (params.get('input') or '').strip():
+ raise RuntimeStoreError('invalid_params', field='input', detail='needs a template or instructions/input')
+
+
+def _session_runtime(authority, ref):
+ """``main_runtime`` for the auxiliary client: the resident agent's live route (main's sidecar),
+ snapshotted here on the loop, else the session's own frozen launch route as its next turn
+ resolves it (read from config in the worker)."""
+ from tui_gateway.session_registry import _main_runtime_from_agent
+ agent = authority.agent(ref)
+ if agent is not None:
+ runtime = _main_runtime_from_agent(agent)
+ return lambda: runtime
+ from gateway.session_policy import policy_for_source
+ policy = policy_for_source(authority.runner, authority.sessions[ref.session_id].source)
+ if policy is None:
+ return lambda: None
+
+ def resolve():
+ from gateway.run_turn_prepare import _resolve_policy_agent_runtime
+ # fallback=False: only the session's primary route. If it cannot resolve (an AuthError), the
+ # one-shot fails with 5030 instead of walking the route's fallback chain, because that walk
+ # sets the runner-wide fallback notice the next turn would report.
+ model, kwargs = _resolve_policy_agent_runtime(authority.runner, policy, fallback=False)
+ runtime = {key: kwargs[key] for key in ('provider', 'base_url', 'api_key', 'api_mode') if kwargs.get(key)}
+ return {**runtime, 'model': model} if model else runtime or None
+ return resolve
+
+
+async def oneshot(connection, ref, params):
+ _validate(params)
+ authority = connection.authority
+ if ref.session_id:
+ authorize(connection, ref, params, 'session:read')
+ runtime = _session_runtime(authority, ref)
+ else:
+ from gateway.session_authorities import require_serving
+ require_serving(authority)
+ if 'session:read' not in connection.actor.capabilities:
+ raise RuntimeStoreError('permission_denied')
+ if params.get('profile'):
+ from pathlib import Path
+ from hermes_cli.profiles import profile_matches_home
+ if not profile_matches_home(params['profile'], Path(authority.profile_id)):
+ raise RuntimeStoreError('profile_mismatch')
+ runtime = lambda: None # the task backend
+
+ def work():
+ from agent.oneshot import run_oneshot
+ from gateway.session_authorities import owner_scope
+ with owner_scope(authority):
+ return {'text': run_oneshot(
+ instructions=params.get('instructions') or '', user_input=params.get('input') or '',
+ template=(params.get('template') or '').strip() or None, variables=params.get('variables') or {},
+ task=(params.get('task') or '').strip() or 'title_generation',
+ max_tokens=int(params.get('max_tokens') or 1024),
+ temperature=0.3 if params.get('temperature') is None else float(params['temperature']),
+ main_runtime=runtime())}
+ return DeferredReply(work)
diff --git a/tests/gateway/test_session_llm_oneshot.py b/tests/gateway/test_session_llm_oneshot.py
new file mode 100644
index 00000000000..31ec3f78817
--- /dev/null
+++ b/tests/gateway/test_session_llm_oneshot.py
@@ -0,0 +1,205 @@
+"""Desktop's ``llm.oneshot`` on the shared gateway borrows the chat's model, as main's sidecar did.
+
+Main's sidecar looked the session up in its own table and lent its live agent's route
+(``_main_runtime_from_agent``). On the shared gateway that table never holds an owner session, so a
+session-bound one-shot (commit message, approval summary) silently ran on the ``task`` auxiliary
+backend (``auto``: the profile's default model, not the chat's). Session-less one-shots (project
+ideas) keep the task backend."""
+import asyncio
+from contextlib import contextmanager
+import io
+import json
+from http.server import ThreadingHTTPServer
+from pathlib import Path
+import signal
+import threading
+import time
+
+import pytest
+
+from tests.gateway.fixtures.local_recovery_probe import Model, child_env, daemon, rpc, websocket
+from tests.waiting import async_wait_until
+
+
+async def _settled(ws, sid, marker):
+ deadline = time.monotonic() + 45
+ while True:
+ snap = (await rpc(ws, 'session.resume', session_id=sid))['result']
+ if not snap['running'] and not snap['pending'] and marker in json.dumps(snap['messages']):
+ return
+ assert time.monotonic() < deadline, ('turn never settled', snap)
+ await asyncio.sleep(.1)
+
+
+class _HeaderModel(Model):
+ """The loopback model, also recording each request's ``X-Route-Token`` with its body."""
+
+ def do_POST(self):
+ raw = self.rfile.read(int(self.headers['Content-Length']))
+ self.server.tokens.append((self.headers.get('X-Route-Token'), raw.decode()))
+ self.rfile = io.BytesIO(raw)
+ super().do_POST()
+
+
+@contextmanager
+def _gateway(tmp_path, named=False):
+ """A real gateway on a loopback model whose ``BLOCK_STARTED`` requests wait for ``release``;
+ ``named`` serves the model as a named custom provider that requires its ``extra_headers``."""
+ root = Path(__file__).resolve().parents[2]
+ home, user, work = tmp_path / 'state', tmp_path / 'user', tmp_path / 'work'
+ home.mkdir(mode=0o700)
+ user.mkdir()
+ work.mkdir()
+ peer = ThreadingHTTPServer(('127.0.0.1', 0), _HeaderModel)
+ peer.requests, peer.tokens = [], []
+ peer.blocked, peer.release = threading.Event(), threading.Event()
+ thread = threading.Thread(target=peer.serve_forever, daemon=True)
+ thread.start()
+ url = f'http://127.0.0.1:{peer.server_port}/v1'
+ model = {'provider': 'custom', 'default': 'chat-model', 'base_url': url}
+ if named:
+ model = {'provider': 'custom:edge', 'default': 'chat-model'}
+ (home / 'config.yaml').write_text(json.dumps({
+ 'gateway': {'multiplex_profiles': False},
+ 'model': model,
+ 'custom_providers': [{'name': 'edge', 'base_url': url, 'api_key': 'loopback-only',
+ 'extra_headers': {'X-Route-Token': 'edge-token'}}] if named else [],
+ # The task's default ``auto`` route: the borrowed session runtime, else the profile's model.
+ 'auxiliary': {'title_generation': {'enabled': False}}}))
+ env = child_env()
+ env.update(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home), PYTHONPATH=str(root),
+ OPENAI_API_KEY='loopback-only', OPENAI_BASE_URL=url, PYTHONUNBUFFERED='1')
+ try:
+ with daemon(root, home, env, barrier=False) as (proc, desc):
+ yield home, work, peer, desc
+ peer.release.set()
+ proc.send_signal(signal.SIGINT)
+ proc.wait(timeout=20)
+ finally:
+ peer.release.set()
+ peer.shutdown()
+ peer.server_close()
+ thread.join(timeout=5)
+
+
+def test_session_bound_oneshot_uses_the_chats_model_and_never_blocks_the_socket(tmp_path):
+ out = {}
+
+ async def probe(home, work, peer, desc):
+ def model_for(text):
+ return next(r['model'] for r in reversed(peer.requests)
+ if any(m.get('content') == text for m in r['messages']))
+
+ async with websocket(home, desc) as ws:
+ created = await rpc(ws, 'session.create', request_id='rt7', source='gui', toolsets=[], cwd=str(work),
+ model='session-model')
+ sid = created['result']['session_id']
+ # No turn yet: no resident agent, the session's own frozen route.
+ out['cold'] = await rpc(ws, 'llm.oneshot', session_id=sid, instructions='sys', input='ONESHOT_COLD')
+ out['cold_model'] = model_for('ONESHOT_COLD')
+ await rpc(ws, 'prompt.submit', session_id=sid, input_id='warm', text='WARM')
+ await _settled(ws, sid, 'RECOVERY_ACK_WARM')
+ out['turn_model'] = model_for('WARM')
+ out['warm'] = await rpc(ws, 'llm.oneshot', session_id=sid, instructions='sys', input='ONESHOT_WARM')
+ out['warm_model'] = model_for('ONESHOT_WARM')
+ out['free'] = await rpc(ws, 'llm.oneshot', instructions='sys', input='ONESHOT_FREE')
+ out['free_model'] = model_for('ONESHOT_FREE')
+ out['bad'] = await rpc(ws, 'llm.oneshot', session_id=sid, instructions='sys', input='x', bogus=1)
+ # A one-shot waiting on its model never holds up the next request on this socket.
+ await ws.send(json.dumps({'jsonrpc': '2.0', 'id': 'slow', 'method': 'llm.oneshot',
+ 'params': {'session_id': sid, 'input': 'BLOCK_STARTED'}}))
+ assert await asyncio.to_thread(peer.blocked.wait, 20)
+ out['ping'] = await rpc(ws, 'ping')
+ peer.release.set()
+ async with asyncio.timeout(20):
+ while (reply := json.loads(await ws.recv())).get('id') != 'slow':
+ pass
+ out['slow'] = reply
+ with _gateway(tmp_path) as served:
+ asyncio.run(probe(*served))
+ assert out['cold']['result']['text'] == 'RECOVERY_ACK_ONESHOT_COLD', out
+ assert out['turn_model'] == 'session-model', out
+ assert (out['cold_model'], out['warm_model']) == ('session-model', 'session-model'), out
+ assert out['warm']['result']['text'] == 'RECOVERY_ACK_ONESHOT_WARM', out
+ assert out['free_model'] == 'chat-model', out
+ assert out['free']['result']['text'] == 'RECOVERY_ACK_ONESHOT_FREE', out
+ assert out['bad']['error']['message'] == 'invalid_params', out
+ assert out['ping'].get('result') == {'pong': True}, out
+ assert out['slow']['result']['text'] == 'RECOVERY_ACK_BLOCK_STARTED', out
+
+
+def test_a_borrowed_named_provider_route_keeps_its_extra_headers(tmp_path):
+ """The borrowed route reaches the aux client as provider ``custom`` with the session's endpoint and
+ key; the named entry's required ``extra_headers`` still ride on it, cold and resident alike."""
+ out = {}
+
+ async def probe(home, work, peer, desc):
+ def token_for(text):
+ return next(token for token, raw in reversed(peer.tokens) if json.dumps(text) in raw)
+
+ async with websocket(home, desc) as ws:
+ created = await rpc(ws, 'session.create', request_id='hdr', source='gui', toolsets=[], cwd=str(work),
+ model='session-model')
+ sid = created['result']['session_id']
+ out['cold'] = await rpc(ws, 'llm.oneshot', session_id=sid, instructions='sys', input='ONESHOT_COLD')
+ out['cold_token'] = token_for('ONESHOT_COLD')
+ await rpc(ws, 'prompt.submit', session_id=sid, input_id='warm', text='WARM')
+ await _settled(ws, sid, 'RECOVERY_ACK_WARM')
+ out['turn_token'] = token_for('WARM')
+ out['warm'] = await rpc(ws, 'llm.oneshot', session_id=sid, instructions='sys', input='ONESHOT_WARM')
+ out['warm_token'] = token_for('ONESHOT_WARM')
+ out['warm_model'] = next(r['model'] for r in reversed(peer.requests)
+ if any(m.get('content') == 'ONESHOT_WARM' for m in r['messages']))
+ with _gateway(tmp_path, named=True) as served:
+ asyncio.run(probe(*served))
+ assert out['turn_token'] == 'edge-token', out # the chat itself always sent it
+ assert out['cold']['result']['text'] == 'RECOVERY_ACK_ONESHOT_COLD', out
+ assert out['warm']['result']['text'] == 'RECOVERY_ACK_ONESHOT_WARM', out
+ assert (out['cold_token'], out['warm_token']) == ('edge-token', 'edge-token'), out
+ assert out['warm_model'] == 'session-model', out
+
+
+def test_a_burst_of_slow_oneshots_never_starves_the_rest_of_the_gateway(tmp_path):
+ """More blocked one-shots than the loop's default executor has threads (``min(32, cpus + 4)``):
+ another socket's work on that executor (``tools.show``) and its ``ping`` still answer at once."""
+ burst, out = 40, {}
+
+ async def probe(home, work, peer, desc):
+ async with websocket(home, desc) as ws:
+ created = await rpc(ws, 'session.create', request_id='burst', source='gui', toolsets=[], cwd=str(work))
+ sid = created['result']['session_id']
+ for n in range(burst):
+ await ws.send(json.dumps({'jsonrpc': '2.0', 'id': f'slow{n}', 'method': 'llm.oneshot',
+ 'params': {'session_id': sid, 'input': 'BLOCK_STARTED'}}))
+ # Every one-shot thread is parked on the model: the owner's pool here, the default executor before.
+ await async_wait_until(lambda: sum(any(m.get('content') == 'BLOCK_STARTED' for m in r['messages'])
+ for r in list(peer.requests)) >= 4, timeout=20)
+ try:
+ async with asyncio.timeout(10), websocket(home, desc) as other:
+ mine = await rpc(other, 'session.create', request_id='other', source='gui', toolsets=[],
+ cwd=str(work))
+ out['tools'] = await rpc(other, 'tools.show', session_id=mine['result']['session_id'])
+ out['ping'] = await rpc(other, 'ping')
+ except TimeoutError:
+ out['starved'] = True
+ peer.release.set()
+ replies = {}
+ async with asyncio.timeout(60):
+ while len(replies) < burst:
+ reply = json.loads(await ws.recv())
+ if str(reply.get('id', '')).startswith('slow'):
+ replies[reply['id']] = reply
+ out['replies'] = replies
+ with _gateway(tmp_path) as served:
+ asyncio.run(probe(*served))
+ assert 'starved' not in out, 'one-shots starved the gateway: tools.show/ping never answered'
+ assert 'sections' in out['tools'].get('result', {}), out['tools']
+ assert out['ping'].get('result') == {'pong': True}, out['ping']
+ answers = [reply.get('result', {}).get('text') or reply.get('error', {}).get('message')
+ for reply in out['replies'].values()]
+ # Every call is answered: generated, or refused by name once the one-shot pool is full.
+ assert set(answers) <= {'RECOVERY_ACK_BLOCK_STARTED', 'oneshot_busy'}, answers
+ assert 'RECOVERY_ACK_BLOCK_STARTED' in answers, answers
diff --git a/tests/gateway/test_session_llm_oneshot_pool.py b/tests/gateway/test_session_llm_oneshot_pool.py
new file mode 100644
index 00000000000..d6ff068b33e
--- /dev/null
+++ b/tests/gateway/test_session_llm_oneshot_pool.py
@@ -0,0 +1,185 @@
+"""The owner's one-shot pool: bounded, scoped like the handler, and owned by connection and owner.
+
+A one-shot's model call is the one long blocking call the owner answers off the connection's serial
+dispatch. It runs on the owner's own small pool, never the loop's default executor, and its answer
+lives only as long as the socket that asked for it and the owner that serves it."""
+import asyncio
+import contextvars
+import gc
+import threading
+from types import SimpleNamespace
+import weakref
+
+import pytest
+
+from gateway import session_llm_oneshot as oneshot
+from hermes_state_runtime import RuntimeStoreError
+from tests.waiting import async_wait_until
+
+
+class _Transport:
+ def __init__(self):
+ self.frames = []
+
+ async def write_async(self, frame):
+ self.frames.append(frame)
+ return True
+
+
+def _connection(authority):
+ return SimpleNamespace(authority=authority, transport=_Transport())
+
+
+class _Calls:
+ """Model calls parked on one gate, counting how many ran at once."""
+
+ def __init__(self):
+ self.gate, self.lock = threading.Event(), threading.Lock()
+ self.ran, self.live, self.peak = [], 0, 0
+
+ def work(self, label):
+ def run():
+ with self.lock:
+ self.ran.append(label)
+ self.live += 1
+ self.peak = max(self.peak, self.live)
+ self.gate.wait(10)
+ with self.lock:
+ self.live -= 1
+ return {'text': label}
+ return run
+
+
+def test_a_full_pool_refuses_by_name_and_never_runs_past_its_workers():
+ calls = _Calls()
+
+ async def scenario():
+ connection = _connection(SimpleNamespace())
+ for n in range(oneshot._ADMITTED):
+ oneshot.DeferredReply(calls.work(f'call{n}')).start(connection, n)
+ with pytest.raises(RuntimeStoreError) as refused:
+ oneshot.DeferredReply(calls.work('extra')).start(connection, 'extra')
+ await async_wait_until(lambda: len(calls.ran) == oneshot._WORKERS)
+ # Unrelated default-executor work is untouched by the parked calls.
+ async with asyncio.timeout(5):
+ assert await asyncio.to_thread(lambda: 'free') == 'free'
+ calls.gate.set()
+ await async_wait_until(lambda: len(connection.transport.frames) == oneshot._ADMITTED)
+ return refused.value.reason, connection.transport.frames
+ reason, frames = asyncio.run(scenario())
+ assert reason == 'oneshot_busy'
+ assert calls.peak == oneshot._WORKERS
+ assert 'extra' not in calls.ran and len(calls.ran) == oneshot._ADMITTED
+ assert sorted(frame['id'] for frame in frames) == list(range(oneshot._ADMITTED))
+
+
+def test_a_closed_socket_drops_its_queued_calls_and_its_running_reply():
+ calls = _Calls()
+
+ async def scenario():
+ authority = SimpleNamespace()
+ gone, kept = _connection(authority), _connection(authority)
+ for n in range(oneshot._WORKERS): # fill every worker: the next calls queue
+ oneshot.DeferredReply(calls.work(f'running{n}')).start(gone, f'running{n}')
+ await async_wait_until(lambda: len(calls.ran) == oneshot._WORKERS)
+ oneshot.DeferredReply(calls.work('queued')).start(gone, 'queued')
+ oneshot.DeferredReply(calls.work('other')).start(kept, 'other')
+ pool = authority._oneshot_pool
+ oneshot.forget(gone)
+ await async_wait_until(lambda: all(owner is not gone for owner in pool.answers.values()))
+ # A running call can't be interrupted: it keeps its slot until its thread returns.
+ held = len(pool.calls)
+ calls.gate.set()
+ await async_wait_until(lambda: not pool.answers and not pool.calls)
+ return held, gone.transport.frames, kept.transport.frames
+ held, gone_frames, kept_frames = asyncio.run(scenario())
+ assert held == oneshot._WORKERS + 1 # the running calls and the other socket's queued one
+ assert 'queued' not in calls.ran # dropped before it started
+ assert gone_frames == [] # nobody reads a closed socket's replies
+ assert kept_frames == [{'jsonrpc': '2.0', 'id': 'other', 'result': {'text': 'other'}}]
+
+
+class _Payload:
+ """Stands in for a queued call's params and copied context."""
+
+
+def test_a_reconnect_loop_never_retains_more_queued_payloads_than_the_bound():
+ """Every worker parked on the model; sockets keep filling the queue and closing. A cancelled call
+ must leave the queue with its payload, not wait for a worker to dequeue it."""
+ calls, payloads = _Calls(), weakref.WeakSet()
+
+ def queued(label):
+ payload = _Payload()
+ payloads.add(payload)
+ return lambda: {'text': label, 'payload': repr(payload)}
+
+ async def scenario():
+ authority = SimpleNamespace()
+ busy = _connection(authority)
+ for n in range(oneshot._WORKERS):
+ oneshot.DeferredReply(calls.work(f'running{n}')).start(busy, f'running{n}')
+ await async_wait_until(lambda: len(calls.ran) == oneshot._WORKERS)
+ pool, retained = authority._oneshot_pool, []
+ for cycle in range(5):
+ gone = _connection(authority)
+ for n in range(oneshot._ADMITTED - oneshot._WORKERS):
+ oneshot.DeferredReply(queued(f'{cycle}.{n}')).start(gone, f'{cycle}.{n}')
+ oneshot.forget(gone)
+ await async_wait_until(lambda: len(pool.answers) == oneshot._WORKERS)
+ gc.collect()
+ retained.append(len(payloads))
+ calls.gate.set()
+ await async_wait_until(lambda: not pool.answers and not pool.calls)
+ return retained, busy.transport.frames
+ retained, frames = asyncio.run(scenario())
+ assert max(retained) <= oneshot._ADMITTED - oneshot._WORKERS, retained
+ assert retained == [0] * 5, retained # nothing queued survives its socket
+ assert len(calls.ran) == oneshot._WORKERS and len(frames) == oneshot._WORKERS
+
+
+def test_owner_shutdown_refuses_new_calls_and_cancels_the_queue():
+ calls = _Calls()
+
+ async def scenario():
+ authority = SimpleNamespace()
+ connection = _connection(authority)
+ idle = SimpleNamespace()
+ oneshot.stop(idle) # an owner that never served a one-shot refuses from its stop on too
+ with pytest.raises(RuntimeStoreError):
+ oneshot.DeferredReply(calls.work('idle')).start(_connection(idle), 'idle')
+ for n in range(oneshot._WORKERS + 2):
+ oneshot.DeferredReply(calls.work(f'call{n}')).start(connection, n)
+ await async_wait_until(lambda: len(calls.ran) == oneshot._WORKERS)
+ oneshot.stop(authority)
+ with pytest.raises(RuntimeStoreError) as refused:
+ oneshot.DeferredReply(calls.work('late')).start(connection, 'late')
+ pool = authority._oneshot_pool
+ calls.gate.set()
+ await async_wait_until(lambda: not pool.calls and not pool.answers)
+ return refused.value.reason, connection.transport.frames
+ reason, frames = asyncio.run(scenario())
+ assert reason == 'runtime_draining'
+ assert len(calls.ran) == oneshot._WORKERS # the queued two never started
+ assert frames == []
+
+
+def test_the_call_runs_in_the_handlers_context_and_its_result_is_checked_before_the_reply():
+ scope = contextvars.ContextVar('scope', default='ambient')
+
+ def bad_result(result):
+ raise ValueError(f'result violates its contract: {result}')
+
+ async def scenario():
+ connection = _connection(SimpleNamespace())
+ token = scope.set('secondary-profile')
+ try:
+ oneshot.DeferredReply(lambda: {'text': scope.get()}).start(connection, 'scoped', lambda result: None)
+ finally:
+ scope.reset(token)
+ oneshot.DeferredReply(lambda: {'text': 'x'}).start(connection, 'checked', bad_result)
+ await async_wait_until(lambda: len(connection.transport.frames) == 2)
+ return {frame['id']: frame for frame in connection.transport.frames}
+ frames = asyncio.run(scenario())
+ assert frames['scoped']['result'] == {'text': 'secondary-profile'}
+ assert frames['checked']['error']['code'] == 5030
+ assert 'violates its contract' in frames['checked']['error']['message']