The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(gateway): stop per-frame deflate and per-tool control reads from saturating the loop
kshitijk4poor1ae92cf59f21
diff --git a/gateway/local_viewer_events.py b/gateway/local_viewer_events.pyindex c3353f94f06..958a3d46dad 100644--- a/gateway/local_viewer_events.py+++ b/gateway/local_viewer_events.py@@ -173,14 +173,15 @@ def restore(agent, previous: dict | None) -> None:def publish_control_if_changed(turn) -> None:"""Agent thread, after each tool: a tool (a plugin adding goal criteria, parking the goal) can move- automation state mid-turn, so viewers' cards repaint now instead of at turn end. The snapshot is read,- compared and published on the gateway loop, in order with the slash commands' publishes (a snapshot- read here could land after a newer one); revision-gated, so other tools publish nothing."""- from gateway.session_control_actions import publish+ automation state mid-turn, so viewers' cards repaint now instead of at turn end. Scheduled on the+ gateway loop, which takes the publish ticket in order with the slash commands' publishes; the snapshot+ (three state.db meta reads on the shared connection lock) is read in a worker thread, because one read+ per tool across every running session saturated the loop. Revision-gated: other tools publish nothing."""+ from gateway.session_control_actions import publish_asyncauthority, canonical_id, _generation = turn._approval_ownerasync def republish():- publish(authority, canonical_id, only_if_changed=True)+ await publish_async(authority, canonical_id, only_if_changed=True)turn._schedule(republish(), "session.control.update after a tool")diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.pyindex 998fa932364..ff0fd298d7e 100644--- a/hermes_cli/web_server.py+++ b/hermes_cli/web_server.py@@ -1064,6 +1064,9 @@ def _build_uvicorn_server(host: str, port: int, *, ssh_isolated: bool = False):forwarded_allow_ips=_dashboard_forwarded_allow_ips(_dash_cfg),ws_ping_interval=ping_interval,ws_ping_timeout=ping_timeout,+ # Per-frame zlib on the loop buys nothing for a local client (build_listener_server); a peer+ # behind a tunnel (SSH-isolated backend, public gate) keeps it.+ ws_per_message_deflate=ssh_isolated or not _is_loopback or bool(app.state.auth_required),ws_max_size=_DESKTOP_ATTACHMENT_WS_MAX_BYTES,# Desktop sends a single SIGTERM and escalates to SIGKILL ~5s later;# uvicorn's default (None) waits on lingering ASGI tasks forever, so adiff --git a/hermes_cli/web_server_lifecycle.py b/hermes_cli/web_server_lifecycle.pyindex 4f1d7d14052..b5eca322825 100644--- a/hermes_cli/web_server_lifecycle.py+++ b/hermes_cli/web_server_lifecycle.py@@ -581,6 +581,11 @@ def build_listener_server(served_app, host: str, port: int, *, auth_required: boexcept (TypeError, ValueError):return default+ # permessage-deflate runs zlib on the event loop for every outbound frame, and the gateway sends+ # one frame per streamed token of every running session: under load it was the largest single+ # cost on the loop. Loopback bytes are free, so only a listener remote peers reach (a non-loopback+ # bind, or a loopback bind behind a public URL / tunnel, which engages the auth gate) compresses.+ per_message_deflate = auth_required or host not in _LOOPBACK_HOST_VALUESif ws_ping is None:ws_ping = (None, None) if host in _LOOPBACK_HOST_VALUES else (_ws_ping_setting("ws_ping_interval"), _ws_ping_setting("ws_ping_timeout"))@@ -596,6 +601,7 @@ def build_listener_server(served_app, host: str, port: int, *, auth_required: boforwarded_allow_ips=_dashboard_forwarded_allow_ips(dash_cfg),ws_ping_interval=ws_ping[0],ws_ping_timeout=ws_ping[1],+ ws_per_message_deflate=per_message_deflate,ws_max_size=_DESKTOP_ATTACHMENT_WS_MAX_BYTES,# Desktop sends a single SIGTERM and escalates to SIGKILL ~5s later;# uvicorn's default (None) waits on lingering ASGI tasks forever, so adiff --git a/tests/gateway/test_control_publish_off_loop.py b/tests/gateway/test_control_publish_off_loop.pyindex dc5457b26bb..80f4732e93a 100644--- a/tests/gateway/test_control_publish_off_loop.py+++ b/tests/gateway/test_control_publish_off_loop.py@@ -46,3 +46,33 @@ async def test_a_slow_snapshot_read_never_paints_over_a_newer_publish(monkeypatcassert published == ['new']assert live.control_revision == 'new'assert loop_thread not in read_threads, 'the snapshot was read on the event loop'+++async def test_the_per_tool_republish_reads_its_snapshot_off_the_loop(monkeypatch):+ """Every tool of every running session republishes the control card. Its snapshot read takes+ state.db's connection lock, which the agents' own writes hold, so on the loop it was a large+ share of loop time with many sessions running (gateway disconnects under load)."""+ from gateway.local_viewer_events import publish_control_if_changed++ published = []+ authority, live = _session(published)+ loop_thread = threading.get_ident()+ read_threads = []++ def read(_authority, _sid, *, scoped=False):+ read_threads.append(threading.get_ident())+ return {'revision': 'r1'}++ monkeypatch.setattr(actions, '_read_control_snapshot', read)+ scheduled = []+ turn = SimpleNamespace(_approval_owner=(authority, 's1', 1),+ _schedule=lambda coro, _message: scheduled.append(asyncio.ensure_future(coro)))+ publish_control_if_changed(turn)+ await asyncio.gather(*scheduled)+ assert published == ['r1'] and live.control_revision == 'r1'+ assert read_threads and loop_thread not in read_threads, 'the snapshot was read on the event loop'++ publish_control_if_changed(turn) # nothing changed: revision-gated, nothing published+ await asyncio.gather(*scheduled)+ assert published == ['r1']diff --git a/tests/gateway/test_session_viewer_automation.py b/tests/gateway/test_session_viewer_automation.pyindex 42f4e11fa9f..ebe3e340ccc 100644--- a/tests/gateway/test_session_viewer_automation.py+++ b/tests/gateway/test_session_viewer_automation.py@@ -161,13 +161,15 @@ async def test_a_tool_that_moves_the_goal_mid_turn_repaints_the_viewers_once(ownlambda frame: frame['params']['type'] == 'session.control.update' and updates.append(frame['params']['payload']))turn = _turn(Platform.LOCAL, [])turn._approval_owner = (owner.authority, owner.ref.session_id, 1)- turn._schedule = lambda coro, _message: asyncio.ensure_future(coro)+ scheduled = []+ turn._schedule = lambda coro, _message: scheduled.append(asyncio.ensure_future(coro))async def tool_completes(write=None):if write:GoalManager(session_id=owner.target).add_subgoal(write) # the plugin tool's own writeturn.combined_tool_complete_callback('call', 'probe', {}, 'ok')- await asyncio.sleep(0)+ await asyncio.gather(*scheduled) # the republish reads its snapshot in a worker thread+ scheduled.clear()await tool_completes()assert updates == []await tool_completes('tests pass')diff --git a/tests/hermes_cli/test_web_server_idle_exit.py b/tests/hermes_cli/test_web_server_idle_exit.pyindex 8eeb48c652d..14658a27cdf 100644--- a/tests/hermes_cli/test_web_server_idle_exit.py+++ b/tests/hermes_cli/test_web_server_idle_exit.py@@ -72,9 +72,11 @@ def test_watchdog_sets_should_exit_and_only_arms_for_ssh_isolated_backends(monkemonkeypatch.setattr(ws_mod.app.state, "auth_required", False, raising=False)plain, _ = ws_mod._build_uvicorn_server("127.0.0.1", 0)assert plain.ws_ping_interval is None and getattr(ws_mod.app.state, "ssh_isolated_clients", None) is None+ assert plain.ws_per_message_deflate is False # loopback: no per-frame zlib on the looptry:isolated, _ = ws_mod._build_uvicorn_server("127.0.0.1", 0, ssh_isolated=True)assert isolated.ws_ping_interval and isolated.ws_ping_timeout > isolated.ws_ping_interval+ assert isolated.ws_per_message_deflate is True # the tunnelled client is remoteassert isinstance(ws_mod.app.state.ssh_isolated_clients, IdleClientTracker)finally:ws_mod.app.state._state.pop("ssh_isolated_clients", None) # process-global app: never leak the trackerdiff --git a/tests/hermes_cli/test_web_server_ws_ping.py b/tests/hermes_cli/test_web_server_ws_ping.pyindex a11e98d9b11..832a36f9ac1 100644--- a/tests/hermes_cli/test_web_server_ws_ping.py+++ b/tests/hermes_cli/test_web_server_ws_ping.py@@ -119,6 +119,8 @@ def test_start_server_disables_ws_ping_on_loopback(monkeypatch):assert captured["ws_ping_interval"] is Noneassert captured["ws_ping_timeout"] is None+ # Per-frame zlib on the loop buys nothing on loopback and was the largest loop cost under load.+ assert captured["ws_per_message_deflate"] is Falsedef test_start_server_accepts_base64_desktop_attachments_above_preview_limit(monkeypatch):@@ -158,6 +160,7 @@ def test_start_server_enables_ws_ping_for_half_open_detection(monkeypatch):assert captured["ws_ping_interval"] and captured["ws_ping_interval"] > 0assert captured["ws_ping_timeout"] and captured["ws_ping_timeout"] > 0assert captured["ws_ping_timeout"] >= captured["ws_ping_interval"]+ assert captured.get("ws_per_message_deflate", True) is True # a remote peer keeps compression (uvicorn default)@pytest.mark.platforms("windows")@@ -380,3 +383,19 @@ def test_shutdown_returns_when_lingering_task_never_completes(monkeypatch):assert lingerer.cancelled(), "the lingering task must be cancelled, not leaked"asyncio.run(scenario())++[email protected]("host, auth_required, deflate", [+ ("127.0.0.1", False, False), # local gateway session listener: no compression+ ("0.0.0.0", True, True), # LAN bind keeps it+ ("127.0.0.1", True, True), # loopback behind a public URL / tunnel: remote clients keep it+])+def test_gateway_listener_compresses_only_for_remote_peers(host, auth_required, deflate):+ """``build_listener_server`` (the gateway's session WebSocket) compresses frames only when the+ client is remote. uvicorn runs permessage-deflate on the event loop for every frame, and the+ gateway sends one frame per streamed token of every running session."""+ from hermes_cli.web_server_lifecycle import build_listener_server++ config, _server = build_listener_server(lambda scope, receive, send: None, host, 0,+ auth_required=auth_required)+ assert config.ws_per_message_deflate is deflate