Skip to content

The agent that grows with you

fix(gateway): stop per-frame deflate and per-tool control reads from saturating the loop

kshitijk4poor1ae92cf59f21
diff --git a/gateway/local_viewer_events.py b/gateway/local_viewer_events.py
index c3353f94f06..958a3d46dad 100644
--- a/gateway/local_viewer_events.py
+++ b/gateway/local_viewer_events.py
@@ -173,14 +173,15 @@ def restore(agent, previous: dict | None) -> None:
def publish_control_if_changed(turn) -> None:
"""Agent thread, after each tool: a tool (a plugin adding goal criteria, parking the goal) can move
- automation state mid-turn, so viewers' cards repaint now instead of at turn end. The snapshot is read,
- compared and published on the gateway loop, in order with the slash commands' publishes (a snapshot
- read here could land after a newer one); revision-gated, so other tools publish nothing."""
- from gateway.session_control_actions import publish
+ automation state mid-turn, so viewers' cards repaint now instead of at turn end. Scheduled on the
+ gateway loop, which takes the publish ticket in order with the slash commands' publishes; the snapshot
+ (three state.db meta reads on the shared connection lock) is read in a worker thread, because one read
+ per tool across every running session saturated the loop. Revision-gated: other tools publish nothing."""
+ from gateway.session_control_actions import publish_async
authority, canonical_id, _generation = turn._approval_owner
async def republish():
- publish(authority, canonical_id, only_if_changed=True)
+ await publish_async(authority, canonical_id, only_if_changed=True)
turn._schedule(republish(), "session.control.update after a tool")
diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py
index 998fa932364..ff0fd298d7e 100644
--- a/hermes_cli/web_server.py
+++ b/hermes_cli/web_server.py
@@ -1064,6 +1064,9 @@ def _build_uvicorn_server(host: str, port: int, *, ssh_isolated: bool = False):
forwarded_allow_ips=_dashboard_forwarded_allow_ips(_dash_cfg),
ws_ping_interval=ping_interval,
ws_ping_timeout=ping_timeout,
+ # Per-frame zlib on the loop buys nothing for a local client (build_listener_server); a peer
+ # behind a tunnel (SSH-isolated backend, public gate) keeps it.
+ ws_per_message_deflate=ssh_isolated or not _is_loopback or bool(app.state.auth_required),
ws_max_size=_DESKTOP_ATTACHMENT_WS_MAX_BYTES,
# Desktop sends a single SIGTERM and escalates to SIGKILL ~5s later;
# uvicorn's default (None) waits on lingering ASGI tasks forever, so a
diff --git a/hermes_cli/web_server_lifecycle.py b/hermes_cli/web_server_lifecycle.py
index 4f1d7d14052..b5eca322825 100644
--- a/hermes_cli/web_server_lifecycle.py
+++ b/hermes_cli/web_server_lifecycle.py
@@ -581,6 +581,11 @@ def build_listener_server(served_app, host: str, port: int, *, auth_required: bo
except (TypeError, ValueError):
return default
+ # permessage-deflate runs zlib on the event loop for every outbound frame, and the gateway sends
+ # one frame per streamed token of every running session: under load it was the largest single
+ # cost on the loop. Loopback bytes are free, so only a listener remote peers reach (a non-loopback
+ # bind, or a loopback bind behind a public URL / tunnel, which engages the auth gate) compresses.
+ per_message_deflate = auth_required or host not in _LOOPBACK_HOST_VALUES
if ws_ping is None:
ws_ping = (None, None) if host in _LOOPBACK_HOST_VALUES else (
_ws_ping_setting("ws_ping_interval"), _ws_ping_setting("ws_ping_timeout"))
@@ -596,6 +601,7 @@ def build_listener_server(served_app, host: str, port: int, *, auth_required: bo
forwarded_allow_ips=_dashboard_forwarded_allow_ips(dash_cfg),
ws_ping_interval=ws_ping[0],
ws_ping_timeout=ws_ping[1],
+ ws_per_message_deflate=per_message_deflate,
ws_max_size=_DESKTOP_ATTACHMENT_WS_MAX_BYTES,
# Desktop sends a single SIGTERM and escalates to SIGKILL ~5s later;
# uvicorn's default (None) waits on lingering ASGI tasks forever, so a
diff --git a/tests/gateway/test_control_publish_off_loop.py b/tests/gateway/test_control_publish_off_loop.py
index dc5457b26bb..80f4732e93a 100644
--- a/tests/gateway/test_control_publish_off_loop.py
+++ b/tests/gateway/test_control_publish_off_loop.py
@@ -46,3 +46,33 @@ async def test_a_slow_snapshot_read_never_paints_over_a_newer_publish(monkeypatc
assert published == ['new']
assert live.control_revision == 'new'
assert loop_thread not in read_threads, 'the snapshot was read on the event loop'
+
+
+async def test_the_per_tool_republish_reads_its_snapshot_off_the_loop(monkeypatch):
+ """Every tool of every running session republishes the control card. Its snapshot read takes
+ state.db's connection lock, which the agents' own writes hold, so on the loop it was a large
+ share of loop time with many sessions running (gateway disconnects under load)."""
+ from gateway.local_viewer_events import publish_control_if_changed
+
+ published = []
+ authority, live = _session(published)
+ loop_thread = threading.get_ident()
+ read_threads = []
+
+ def read(_authority, _sid, *, scoped=False):
+ read_threads.append(threading.get_ident())
+ return {'revision': 'r1'}
+
+ monkeypatch.setattr(actions, '_read_control_snapshot', read)
+ scheduled = []
+ turn = SimpleNamespace(_approval_owner=(authority, 's1', 1),
+ _schedule=lambda coro, _message: scheduled.append(asyncio.ensure_future(coro)))
+ publish_control_if_changed(turn)
+ await asyncio.gather(*scheduled)
+ assert published == ['r1'] and live.control_revision == 'r1'
+ assert read_threads and loop_thread not in read_threads, 'the snapshot was read on the event loop'
+
+ publish_control_if_changed(turn) # nothing changed: revision-gated, nothing published
+ await asyncio.gather(*scheduled)
+ assert published == ['r1']
diff --git a/tests/gateway/test_session_viewer_automation.py b/tests/gateway/test_session_viewer_automation.py
index 42f4e11fa9f..ebe3e340ccc 100644
--- a/tests/gateway/test_session_viewer_automation.py
+++ b/tests/gateway/test_session_viewer_automation.py
@@ -161,13 +161,15 @@ async def test_a_tool_that_moves_the_goal_mid_turn_repaints_the_viewers_once(own
lambda frame: frame['params']['type'] == 'session.control.update' and updates.append(frame['params']['payload']))
turn = _turn(Platform.LOCAL, [])
turn._approval_owner = (owner.authority, owner.ref.session_id, 1)
- turn._schedule = lambda coro, _message: asyncio.ensure_future(coro)
+ scheduled = []
+ turn._schedule = lambda coro, _message: scheduled.append(asyncio.ensure_future(coro))
async def tool_completes(write=None):
if write:
GoalManager(session_id=owner.target).add_subgoal(write) # the plugin tool's own write
turn.combined_tool_complete_callback('call', 'probe', {}, 'ok')
- await asyncio.sleep(0)
+ await asyncio.gather(*scheduled) # the republish reads its snapshot in a worker thread
+ scheduled.clear()
await tool_completes()
assert updates == []
await tool_completes('tests pass')
diff --git a/tests/hermes_cli/test_web_server_idle_exit.py b/tests/hermes_cli/test_web_server_idle_exit.py
index 8eeb48c652d..14658a27cdf 100644
--- a/tests/hermes_cli/test_web_server_idle_exit.py
+++ b/tests/hermes_cli/test_web_server_idle_exit.py
@@ -72,9 +72,11 @@ def test_watchdog_sets_should_exit_and_only_arms_for_ssh_isolated_backends(monke
monkeypatch.setattr(ws_mod.app.state, "auth_required", False, raising=False)
plain, _ = ws_mod._build_uvicorn_server("127.0.0.1", 0)
assert plain.ws_ping_interval is None and getattr(ws_mod.app.state, "ssh_isolated_clients", None) is None
+ assert plain.ws_per_message_deflate is False # loopback: no per-frame zlib on the loop
try:
isolated, _ = ws_mod._build_uvicorn_server("127.0.0.1", 0, ssh_isolated=True)
assert isolated.ws_ping_interval and isolated.ws_ping_timeout > isolated.ws_ping_interval
+ assert isolated.ws_per_message_deflate is True # the tunnelled client is remote
assert isinstance(ws_mod.app.state.ssh_isolated_clients, IdleClientTracker)
finally:
ws_mod.app.state._state.pop("ssh_isolated_clients", None) # process-global app: never leak the tracker
diff --git a/tests/hermes_cli/test_web_server_ws_ping.py b/tests/hermes_cli/test_web_server_ws_ping.py
index a11e98d9b11..832a36f9ac1 100644
--- a/tests/hermes_cli/test_web_server_ws_ping.py
+++ b/tests/hermes_cli/test_web_server_ws_ping.py
@@ -119,6 +119,8 @@ def test_start_server_disables_ws_ping_on_loopback(monkeypatch):
assert captured["ws_ping_interval"] is None
assert captured["ws_ping_timeout"] is None
+ # Per-frame zlib on the loop buys nothing on loopback and was the largest loop cost under load.
+ assert captured["ws_per_message_deflate"] is False
def test_start_server_accepts_base64_desktop_attachments_above_preview_limit(monkeypatch):
@@ -158,6 +160,7 @@ def test_start_server_enables_ws_ping_for_half_open_detection(monkeypatch):
assert captured["ws_ping_interval"] and captured["ws_ping_interval"] > 0
assert captured["ws_ping_timeout"] and captured["ws_ping_timeout"] > 0
assert captured["ws_ping_timeout"] >= captured["ws_ping_interval"]
+ assert captured.get("ws_per_message_deflate", True) is True # a remote peer keeps compression (uvicorn default)
@pytest.mark.platforms("windows")
@@ -380,3 +383,19 @@ def test_shutdown_returns_when_lingering_task_never_completes(monkeypatch):
assert lingerer.cancelled(), "the lingering task must be cancelled, not leaked"
asyncio.run(scenario())
+
+
[email protected]("host, auth_required, deflate", [
+ ("127.0.0.1", False, False), # local gateway session listener: no compression
+ ("0.0.0.0", True, True), # LAN bind keeps it
+ ("127.0.0.1", True, True), # loopback behind a public URL / tunnel: remote clients keep it
+])
+def test_gateway_listener_compresses_only_for_remote_peers(host, auth_required, deflate):
+ """``build_listener_server`` (the gateway's session WebSocket) compresses frames only when the
+ client is remote. uvicorn runs permessage-deflate on the event loop for every frame, and the
+ gateway sends one frame per streamed token of every running session."""
+ from hermes_cli.web_server_lifecycle import build_listener_server
+
+ config, _server = build_listener_server(lambda scope, receive, send: None, host, 0,
+ auth_required=auth_required)
+ assert config.ws_per_message_deflate is deflate