Skip to content

The agent that grows with you

fix(gateway): log a session WebSocket upgrade the owner fails with HTTP 500

unsupportedpastelsb36d4465a3bb
diff --git a/gateway/run_api.py b/gateway/run_api.py
index 1866448d00a..aa58cabb741 100644
--- a/gateway/run_api.py
+++ b/gateway/run_api.py
@@ -255,6 +255,28 @@ class GatewayRuntimeAPI:
self.app, self.runner, self.web_app = app, runner, web_app
async def __call__(self, scope, receive, send):
+ if scope['type'] != 'websocket' or scope.get('path') != '/api/ws':
+ return await self._serve(scope, receive, send)
+ accepted = False
+
+ async def send_watched(message):
+ nonlocal accepted
+ accepted = accepted or message.get('type') == 'websocket.accept'
+ await send(message)
+
+ try:
+ return await self._serve(scope, receive, send_watched)
+ except Exception as exc:
+ # Uvicorn answers an upgrade that raised before accept with HTTP 500 and logs it only to
+ # its own logger, never gateway.log: the dialing cron worker or managed worker sees a bare
+ # InvalidStatus (#139922). Name the cause here; uvicorn still sends the 500.
+ if not accepted:
+ logger.error('Gateway refused a session WebSocket upgrade from %s with HTTP 500: %s: %s',
+ scope.get('hermes.gateway_socket_peer') or scope.get('client'),
+ type(exc).__name__, exc, exc_info=True)
+ raise
+
+ async def _serve(self, scope, receive, send):
descriptor = getattr(self.runner, 'session_runtime_descriptor', None)
if scope['type'] not in {'http', 'websocket'} or descriptor is None:
return await self.app(scope, receive, send)
diff --git a/tests/gateway/test_session_ws_upgrade_failure_logged.py b/tests/gateway/test_session_ws_upgrade_failure_logged.py
new file mode 100644
index 00000000000..12fff664fd2
--- /dev/null
+++ b/tests/gateway/test_session_ws_upgrade_failure_logged.py
@@ -0,0 +1,109 @@
+"""An owner that fails a session WebSocket upgrade names the cause in gateway.log (#139922).
+
+The reporter's owner could not import its session WebSocket handler, so every cron agent turn and
+managed-worker dial got ``InvalidStatus: ... HTTP 500`` while gateway.log stayed empty: uvicorn's own
+error logger is not a gateway logger. This boots a real owner (``python -m gateway.run``) in a
+throwaway home with that handler broken, then dials it the way an external cron worker and a managed
+worker do.
+"""
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+from tests.gateway.test_normal_runtime_boot import control
+from tests.waiting import wait_until
+
+# The owner, with ``tui_gateway.ws.handle_ws`` failing the way the unimportable onboarding module did.
+_BROKEN_OWNER = textwrap.dedent('''
+ import runpy, sys
+ import tui_gateway.ws as ws
+ del ws.handle_ws
+ def __getattr__(name):
+ if name == 'handle_ws':
+ raise NameError("name 'Path' is not defined")
+ raise AttributeError(name)
+ ws.__getattr__ = __getattr__
+ sys.argv = ['gateway.run']
+ runpy.run_module('gateway.run', run_name='__main__', alter_sys=True)
+''')
+
+# An external cron worker's agent turn (no owner in-process), then a managed worker's persistence call.
+_DIALS = textwrap.dedent('''
+ import json
+ from cron.scheduler_authority import run_canonical_job
+ from agent.runtime_session_store import WorkerRPC
+ from hermes_constants import get_hermes_home
+ success, _output, _answer, error = run_canonical_job({'id': 'cron4job', 'name': 'cron-4'},
+ execution_id='cron4-exec')
+ try:
+ WorkerRPC(get_hermes_home())('worker.heartbeat')
+ worker = None
+ except Exception as exc:
+ worker = f'{type(exc).__name__}: {exc}'
+ print(json.dumps({'cron': [success, error], 'worker': worker}))
+''')
+
+
+def test_a_failed_session_upgrade_is_logged_with_its_cause(tmp_path):
+ home = tmp_path / 'state'
+ home.mkdir(mode=0o700)
+ user = tmp_path / 'user'
+ user.mkdir()
+ (home / 'config.yaml').write_text(json.dumps({
+ 'gateway': {'multiplex_profiles': False},
+ 'model': {'provider': 'custom', 'default': 'local-wire-stub', 'base_url': 'http://127.0.0.1:9/v1'},
+ 'auxiliary': {'title_generation': {'enabled': False}},
+ }))
+ root = Path(__file__).resolve().parents[2]
+ env = {k: os.environ[k] for k in ('PATH', 'LANG', 'TZ', 'TIRITH_ENABLED') if k in os.environ}
+ env.update(HOME=str(user), USERPROFILE=str(user), HERMES_HOME=str(home),
+ PYTHONPATH=str(root), PYTHONUNBUFFERED='1', OPENAI_API_KEY='loopback-only')
+ with (tmp_path / 'owner.out').open('w+') as out:
+ owner = subprocess.Popen([sys.executable, '-c', _BROKEN_OWNER], cwd=root, env=env,
+ stdin=subprocess.DEVNULL, stdout=out, stderr=subprocess.STDOUT)
+ try:
+ descriptor = {}
+
+ def ready_or_exited():
+ if owner.poll() is not None:
+ return 'exited' # fail now, not after the deadline
+ try:
+ descriptor.update(control(home, 'identify'))
+ except (OSError, ValueError):
+ return None
+ return descriptor.get('state') == 'ready' and 'ready'
+
+ try:
+ outcome = wait_until(ready_or_exited, timeout=35, what='the owner to be ready')
+ except AssertionError as exc:
+ outcome = str(exc)
+ out.flush()
+ out.seek(0)
+ assert outcome == 'ready', (outcome, descriptor, out.read())
+ dialed = subprocess.run([sys.executable, '-c', _DIALS], cwd=root, env=env, capture_output=True,
+ text=True, timeout=60, check=False)
+ assert dialed.returncode == 0, dialed.stderr
+ result = json.loads(dialed.stdout.strip().splitlines()[-1])
+ # The failure itself is unchanged: the owner still refuses the upgrade with HTTP 500.
+ assert result['cron'][0] is False and 'HTTP 500' in result['cron'][1], result
+ assert result['worker'] is not None and 'HTTP 500' in result['worker'], result
+ finally:
+ if owner.poll() is None:
+ owner.send_signal(signal.SIGINT)
+ try:
+ owner.wait(timeout=20)
+ except subprocess.TimeoutExpired:
+ owner.kill()
+ owner.wait(timeout=5)
+ log = (home / 'logs' / 'gateway.log').read_text(encoding='utf-8')
+ refused = [line for line in log.splitlines() if 'session WebSocket upgrade' in line]
+ # One line per refused dial (the cron turn's and the worker's), each naming the cause.
+ assert len(refused) == 2, log
+ assert all("NameError: name 'Path' is not defined" in line for line in refused), refused