The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(webapp): keep reading a host terminal socket behind a waiting write, so keepalive pongs are seen
unsupportedpastels0cb2f60ac078
diff --git a/hermes_cli/web_host_terminal_sessions.py b/hermes_cli/web_host_terminal_sessions.pyindex 986110fc864..d3aaa6a47a4 100644--- a/hermes_cli/web_host_terminal_sessions.py+++ b/hermes_cli/web_host_terminal_sessions.py@@ -30,8 +30,11 @@ from hermes_constants import assert_named_profile_home_live, get_hermes_home# No survival across backend restart. Disconnects retain a bounded ANSI tail.RETENTION_SECONDS = 15 * 60_MAX_SHELLS = 16-# Inbound frames buffered while one batch is written (backpressure bound).-_INPUT_BATCH_FRAMES = 64+# Inbound input buffered while one batch is written (backpressure bound), in bytes; each frame also+# costs ``_FRAME_OVERHEAD_BYTES``. Bytes, not frames: until it is reached the reader keeps reading the+# socket, which is where the server's keepalive pongs arrive (``_InputQueue``).+_INPUT_QUEUE_BYTES = 1024 * 1024+_FRAME_OVERHEAD_BYTES = 256# How often a reader held up by that bound checks that the tab is still there (``_queue_frame``).# Once the connection is gone a check fails within about three: the first can still reach the# kernel before the peer's reset comes back.@@ -304,6 +307,40 @@ async def _admit(ws: WebSocket, registry: HostTerminalRegistry):return session, owner+def _frame_cost(item) -> int:+ """What a queued item counts against ``_INPUT_QUEUE_BYTES``: its payload and the per-frame overhead."""+ if not isinstance(item, dict): # the reader's last item: None or the error that ended reading+ return _FRAME_OVERHEAD_BYTES+ return len(item.get("bytes") or item.get("text") or b"") + _FRAME_OVERHEAD_BYTES+++class _InputQueue(asyncio.Queue):+ """The frames a tab sent while a batch is written, full at ``_INPUT_QUEUE_BYTES``.++ A full queue stops the reader, and with it the WebSocket library's own read of the socket once+ that buffers its few frames: keepalive pongs then go unread, and a bind that pings (any+ non-loopback Webapp) closes a live tab with 1011 after the ping timeout, dropping the input+ queued behind it. A frame count would fill on a few dozen keys typed while a paste waits on a+ program that is not reading; a byte bound leaves the reader reading through any typing.+ """++ def __init__(self):+ super().__init__()+ self.queued = 0++ def full(self) -> bool:+ return self.queued >= _INPUT_QUEUE_BYTES++ def _put(self, item):+ self.queued += _frame_cost(item)+ super()._put(item)++ def _get(self):+ item = super()._get()+ self.queued -= _frame_cost(item)+ return item++async def _queue_frame(ws: WebSocket, inbox: asyncio.Queue, ended: asyncio.Future, msg) -> None:"""``inbox.put(msg)``, checking that the tab is still there while the inbox stays full.@@ -440,7 +477,7 @@ async def _pump_input(ws: WebSocket, registry: HostTerminalRegistry, token: str,buffers input itself, so Windows writes land at once and keep the bridge's deadline."""from hermes_cli.web_server_chat import _RESIZE_RE- inbox: asyncio.Queue = asyncio.Queue(maxsize=_INPUT_BATCH_FRAMES)+ inbox: asyncio.Queue = _InputQueue()ended = asyncio.get_running_loop().create_future()reader = asyncio.create_task(_read_frames(ws, inbox, ended))undo = Nonediff --git a/tests/hermes_cli/test_host_terminal_viewer_ownership.py b/tests/hermes_cli/test_host_terminal_viewer_ownership.pyindex 6af731cf1cd..4d2780cde5a 100644--- a/tests/hermes_cli/test_host_terminal_viewer_ownership.py+++ b/tests/hermes_cli/test_host_terminal_viewer_ownership.py@@ -371,14 +371,24 @@ def unread_posix_pty(_monkeypatch):os.close(slave)+def keys_filling_the_input_queue(extra=0):+ """``(frames, filled)``: keystroke frames whose first ``filled`` fill the host terminal's input+ queue, then ``extra`` more. Each frame repeats one letter; the next frame has the next letter."""+ from hermes_cli import web_host_terminal_sessions as host_sessions++ size = host_sessions._INPUT_QUEUE_BYTES // 16+ filled = -(-host_sessions._INPUT_QUEUE_BYTES // (size + host_sessions._FRAME_OVERHEAD_BYTES))+ return [bytes([65 + n % 26]) * size for n in range(filled + extra)], filled++@pytest.mark.asyncio[email protected]("typed_after", [0, 64], ids=["quiet", "full-inbox"])[email protected]("fill_inbox", [False, True], ids=["quiet", "full-inbox"])@pytest.mark.parametrize("terminal", [pytest.param(unread_posix_pty, id="posix-pty", marks=pytest.mark.platforms("posix")),pytest.param(unread_input, id="unread-input", marks=pytest.mark.platforms("any")),])async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_shell(- tmp_path, monkeypatch, terminal, typed_after,+ tmp_path, monkeypatch, terminal, fill_inbox,):"""A paste into a program that is not reading (``sleep``, ``npm install``) waits for it with nodeadline. When the tab closes meanwhile, the wait is cancelled and the shell is detached, as on@@ -387,7 +397,6 @@ async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_shehide the close."""from hermes_cli import web_host_terminal_sessions as host_sessions- assert typed_after in (0, host_sessions._INPUT_BATCH_FRAMES)home = tmp_path / ".hermes"profile = home / "profiles" / "worker"profile.mkdir(parents=True)@@ -412,7 +421,7 @@ async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_shetry:handler = asyncio.create_task(host_sessions.host_terminal(tab))assert await asyncio.to_thread(waiting.wait, 5), "the paste never waited for the program"- tab.type(*[b"k"] * typed_after)+ tab.type(*(keys_filling_the_input_queue()[0] if fill_inbox else ()))tab.close_tab()async with asyncio.timeout(5):await handler@@ -467,12 +476,61 @@ class BrowserTab:self.sock.close()+class AnsweringTab(BrowserTab):+ """A tab whose browser answers the server's WebSocket pings as they arrive, on its own thread, as a+ browser does whatever the page is doing. ``pings`` counts the pings it answered."""++ def __init__(self, port):+ super().__init__(port)+ self.protocol.events_received() # the handshake response+ self.pings, self._lock, self._stop = 0, threading.Lock(), threading.Event()+ self.sock.settimeout(0.05)+ self._thread = threading.Thread(target=self._answer, daemon=True)+ self._thread.start()++ def _answer(self):+ from websockets.frames import Opcode++ while not self._stop.is_set():+ try:+ data = self.sock.recv(65536)+ except TimeoutError:+ continue+ except OSError:+ return+ with self._lock:+ if data:+ self.protocol.receive_data(data)+ else:+ self.protocol.receive_eof()+ self.pings += sum(getattr(event, "opcode", None) is Opcode.PING+ for event in self.protocol.events_received())+ with suppress(OSError):+ self._flush() # the pongs+ if not data:+ return++ def send(self, *frames):+ with self._lock:+ super().send(*frames)++ def close(self):+ from websockets.protocol import State++ self._stop.set()+ self._thread.join(5)+ if self.protocol.state is State.OPEN:+ super().close()+ else:+ self.sock.close()++@contextmanager-def served_host_terminal(monkeypatch, session, owner):+def served_host_terminal(monkeypatch, session, owner, ws_ping=(None, None)):"""``host_terminal`` for ``session`` behind real uvicorn and Starlette on a loopback port, with- uvicorn's default WebSocket protocol and the loopback Webapp's settings (no WebSocket pings).- Yields the port, the frames the route has taken from the socket, and an Event set once the- route returns."""+ uvicorn's default WebSocket protocol and the loopback Webapp's settings (no WebSocket pings), or+ a non-loopback bind's ``ws_ping`` (interval, timeout). Yields the port, the frames the route has+ taken from the socket, and an Event set once the route returns."""import uvicornfrom starlette.applications import Starlettefrom starlette.routing import WebSocketRoute@@ -508,7 +566,7 @@ def served_host_terminal(monkeypatch, session, owner):sock = socket.socket()sock.bind(("127.0.0.1", 0))server = uvicorn.Server(uvicorn.Config(- counting, lifespan="off", log_level="error", ws_ping_interval=None, ws_ping_timeout=None,+ counting, lifespan="off", log_level="error", ws_ping_interval=ws_ping[0], ws_ping_timeout=ws_ping[1],# A route that never returns (the bug) is cancelled instead of holding shutdown forever.timeout_graceful_shutdown=1,))@@ -535,8 +593,6 @@ def test_a_tab_closed_behind_more_keys_than_the_input_queue_holds_is_detached(tmthen some, so the tab's close is unread behind them when it comes, and the browser drops theconnection. Over a real socket the shell is still detached within a few seconds: it keepsrunning, retained for a reattach."""- from hermes_cli import web_host_terminal_sessions as host_sessions-home = tmp_path / ".hermes"profile = home / "profiles" / "worker"profile.mkdir(parents=True)@@ -545,14 +601,14 @@ def test_a_tab_closed_behind_more_keys_than_the_input_queue_holds_is_detached(tmowner = HostOwner(("loopback", "test"), profile, "sh", str(tmp_path))# 1 MiB: more than a pseudo-terminal buffers anywhere between its master and the program.paste = b"".join(b"%05d %s\n" % (n, b"x" * 57) for n in range(16 * 1024))- fill = host_sessions._INPUT_BATCH_FRAMES+ keys, fill = keys_filling_the_input_queue(17)with terminal(monkeypatch) as (bridge, waiting, cancelled):session = PtySession("terminal", bridge, buffer_cap=32, read_timeout=0.01)with served_host_terminal(monkeypatch, session, owner) as (port, taken, finished):tab = BrowserTab(port)tab.send(paste)assert waiting.wait(5), "the paste never waited for the program"- tab.send(*[b"k"] * (fill + 17))+ tab.send(*keys)# The paste, a full queue, and one more frame the reader holds with no room for it.wait_until(lambda: len(taken) >= fill + 2, timeout=5, what="the reader to fill the input queue")tab.close()@@ -562,6 +618,42 @@ def test_a_tab_closed_behind_more_keys_than_the_input_queue_holds_is_detached(tmassert not session.attached and session._ws is None+def test_a_live_tab_typing_behind_a_waiting_write_answers_keepalive_pings(tmp_path, monkeypatch):+ """A non-loopback Webapp pings its sockets (``web_server._build_uvicorn_server``) and closes one+ whose pong does not come back in time. Keys typed while a write waits on a slow program must not+ stop the server reading the socket, where the pongs arrive: the tab, answering every ping, stays+ attached through several ping timeouts, and every key reaches the program once it reads."""+ home = tmp_path / ".hermes"+ profile = home / "profiles" / "worker"+ profile.mkdir(parents=True)+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setenv("HERMES_HOME", str(home))+ owner = HostOwner(("loopback", "test"), profile, "sh", str(tmp_path))+ bridge = SlowAccept()+ session = PtySession("terminal", bridge, buffer_cap=32, read_timeout=0.01)+ # Far more frames than the WebSocket library buffers unread (32) and than 64 queued frames.+ typed = [bytes([65 + n % 26]) for n in range(1024)]+ with served_host_terminal(monkeypatch, session, owner, ws_ping=(0.2, 0.3)) as (port, taken, finished):+ tab = AnsweringTab(port)+ try:+ tab.send(b"A")+ assert bridge.entered.wait(5), "the first write never started"+ tab.send(*typed)+ # Several ping intervals and timeouts, all while the program holds its first write.+ wait_until(lambda: tab.pings >= 5 or finished.is_set(), timeout=10, what="five keepalive pings")+ assert not finished.is_set() and session.attached, "the keepalive closed a live tab"+ assert len(taken) == 1 + len(typed), "the reader stopped reading the socket"+ bridge.release.set()+ expected = b"A" + b"".join(typed)+ wait_until(lambda: bytes(bridge.written) == expected, timeout=5, what="every key to reach the program")+ assert session.attached and not finished.is_set()+ finally:+ bridge.release.set()+ tab.close()+ assert finished.wait(10), "the shell stayed attached after its tab closed"+ assert session.alive and not session.attached++class SlowAccept:"""A program that reads every byte, but not until ``release``: slow, not stopped. The first writetakes its bytes and then waits; later writes land at once."""@@ -618,8 +710,7 @@ async def test_a_live_tab_typing_past_a_full_input_queue_is_held_back_and_loses_monkeypatch.setattr(host_sessions, "get_host_terminals", lambda _app: registry)monkeypatch.setattr(host_sessions, "_admit", admit)- fill = host_sessions._INPUT_BATCH_FRAMES- typed = [bytes([65 + n % 26]) for n in range(16 * fill)]+ typed, fill = keys_filling_the_input_queue(3 * 16)# The first write holds b"A"; the reader queues ``fill`` frames and holds one more with no room.unread = len(typed) - fill - 1tab = ClosingTab(b"A")