Skip to content

The agent that grows with you

fix(webapp): keep reading a host terminal socket behind a waiting write, so keepalive pongs are seen

unsupportedpastels0cb2f60ac078
diff --git a/hermes_cli/web_host_terminal_sessions.py b/hermes_cli/web_host_terminal_sessions.py
index 986110fc864..d3aaa6a47a4 100644
--- a/hermes_cli/web_host_terminal_sessions.py
+++ b/hermes_cli/web_host_terminal_sessions.py
@@ -30,8 +30,11 @@ from hermes_constants import assert_named_profile_home_live, get_hermes_home
# No survival across backend restart. Disconnects retain a bounded ANSI tail.
RETENTION_SECONDS = 15 * 60
_MAX_SHELLS = 16
-# Inbound frames buffered while one batch is written (backpressure bound).
-_INPUT_BATCH_FRAMES = 64
+# Inbound input buffered while one batch is written (backpressure bound), in bytes; each frame also
+# costs ``_FRAME_OVERHEAD_BYTES``. Bytes, not frames: until it is reached the reader keeps reading the
+# socket, which is where the server's keepalive pongs arrive (``_InputQueue``).
+_INPUT_QUEUE_BYTES = 1024 * 1024
+_FRAME_OVERHEAD_BYTES = 256
# How often a reader held up by that bound checks that the tab is still there (``_queue_frame``).
# Once the connection is gone a check fails within about three: the first can still reach the
# kernel before the peer's reset comes back.
@@ -304,6 +307,40 @@ async def _admit(ws: WebSocket, registry: HostTerminalRegistry):
return session, owner
+def _frame_cost(item) -> int:
+ """What a queued item counts against ``_INPUT_QUEUE_BYTES``: its payload and the per-frame overhead."""
+ if not isinstance(item, dict): # the reader's last item: None or the error that ended reading
+ return _FRAME_OVERHEAD_BYTES
+ return len(item.get("bytes") or item.get("text") or b"") + _FRAME_OVERHEAD_BYTES
+
+
+class _InputQueue(asyncio.Queue):
+ """The frames a tab sent while a batch is written, full at ``_INPUT_QUEUE_BYTES``.
+
+ A full queue stops the reader, and with it the WebSocket library's own read of the socket once
+ that buffers its few frames: keepalive pongs then go unread, and a bind that pings (any
+ non-loopback Webapp) closes a live tab with 1011 after the ping timeout, dropping the input
+ queued behind it. A frame count would fill on a few dozen keys typed while a paste waits on a
+ program that is not reading; a byte bound leaves the reader reading through any typing.
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.queued = 0
+
+ def full(self) -> bool:
+ return self.queued >= _INPUT_QUEUE_BYTES
+
+ def _put(self, item):
+ self.queued += _frame_cost(item)
+ super()._put(item)
+
+ def _get(self):
+ item = super()._get()
+ self.queued -= _frame_cost(item)
+ return item
+
+
async def _queue_frame(ws: WebSocket, inbox: asyncio.Queue, ended: asyncio.Future, msg) -> None:
"""``inbox.put(msg)``, checking that the tab is still there while the inbox stays full.
@@ -440,7 +477,7 @@ async def _pump_input(ws: WebSocket, registry: HostTerminalRegistry, token: str,
buffers input itself, so Windows writes land at once and keep the bridge's deadline.
"""
from hermes_cli.web_server_chat import _RESIZE_RE
- inbox: asyncio.Queue = asyncio.Queue(maxsize=_INPUT_BATCH_FRAMES)
+ inbox: asyncio.Queue = _InputQueue()
ended = asyncio.get_running_loop().create_future()
reader = asyncio.create_task(_read_frames(ws, inbox, ended))
undo = None
diff --git a/tests/hermes_cli/test_host_terminal_viewer_ownership.py b/tests/hermes_cli/test_host_terminal_viewer_ownership.py
index 6af731cf1cd..4d2780cde5a 100644
--- a/tests/hermes_cli/test_host_terminal_viewer_ownership.py
+++ b/tests/hermes_cli/test_host_terminal_viewer_ownership.py
@@ -371,14 +371,24 @@ def unread_posix_pty(_monkeypatch):
os.close(slave)
+def keys_filling_the_input_queue(extra=0):
+ """``(frames, filled)``: keystroke frames whose first ``filled`` fill the host terminal's input
+ queue, then ``extra`` more. Each frame repeats one letter; the next frame has the next letter."""
+ from hermes_cli import web_host_terminal_sessions as host_sessions
+
+ size = host_sessions._INPUT_QUEUE_BYTES // 16
+ filled = -(-host_sessions._INPUT_QUEUE_BYTES // (size + host_sessions._FRAME_OVERHEAD_BYTES))
+ return [bytes([65 + n % 26]) * size for n in range(filled + extra)], filled
+
+
@pytest.mark.asyncio
[email protected]("typed_after", [0, 64], ids=["quiet", "full-inbox"])
[email protected]("fill_inbox", [False, True], ids=["quiet", "full-inbox"])
@pytest.mark.parametrize("terminal", [
pytest.param(unread_posix_pty, id="posix-pty", marks=pytest.mark.platforms("posix")),
pytest.param(unread_input, id="unread-input", marks=pytest.mark.platforms("any")),
])
async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_shell(
- tmp_path, monkeypatch, terminal, typed_after,
+ tmp_path, monkeypatch, terminal, fill_inbox,
):
"""A paste into a program that is not reading (``sleep``, ``npm install``) waits for it with no
deadline. When the tab closes meanwhile, the wait is cancelled and the shell is detached, as on
@@ -387,7 +397,6 @@ async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_she
hide the close."""
from hermes_cli import web_host_terminal_sessions as host_sessions
- assert typed_after in (0, host_sessions._INPUT_BATCH_FRAMES)
home = tmp_path / ".hermes"
profile = home / "profiles" / "worker"
profile.mkdir(parents=True)
@@ -412,7 +421,7 @@ async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_she
try:
handler = asyncio.create_task(host_sessions.host_terminal(tab))
assert await asyncio.to_thread(waiting.wait, 5), "the paste never waited for the program"
- tab.type(*[b"k"] * typed_after)
+ tab.type(*(keys_filling_the_input_queue()[0] if fill_inbox else ()))
tab.close_tab()
async with asyncio.timeout(5):
await handler
@@ -467,12 +476,61 @@ class BrowserTab:
self.sock.close()
+class AnsweringTab(BrowserTab):
+ """A tab whose browser answers the server's WebSocket pings as they arrive, on its own thread, as a
+ browser does whatever the page is doing. ``pings`` counts the pings it answered."""
+
+ def __init__(self, port):
+ super().__init__(port)
+ self.protocol.events_received() # the handshake response
+ self.pings, self._lock, self._stop = 0, threading.Lock(), threading.Event()
+ self.sock.settimeout(0.05)
+ self._thread = threading.Thread(target=self._answer, daemon=True)
+ self._thread.start()
+
+ def _answer(self):
+ from websockets.frames import Opcode
+
+ while not self._stop.is_set():
+ try:
+ data = self.sock.recv(65536)
+ except TimeoutError:
+ continue
+ except OSError:
+ return
+ with self._lock:
+ if data:
+ self.protocol.receive_data(data)
+ else:
+ self.protocol.receive_eof()
+ self.pings += sum(getattr(event, "opcode", None) is Opcode.PING
+ for event in self.protocol.events_received())
+ with suppress(OSError):
+ self._flush() # the pongs
+ if not data:
+ return
+
+ def send(self, *frames):
+ with self._lock:
+ super().send(*frames)
+
+ def close(self):
+ from websockets.protocol import State
+
+ self._stop.set()
+ self._thread.join(5)
+ if self.protocol.state is State.OPEN:
+ super().close()
+ else:
+ self.sock.close()
+
+
@contextmanager
-def served_host_terminal(monkeypatch, session, owner):
+def served_host_terminal(monkeypatch, session, owner, ws_ping=(None, None)):
"""``host_terminal`` for ``session`` behind real uvicorn and Starlette on a loopback port, with
- uvicorn's default WebSocket protocol and the loopback Webapp's settings (no WebSocket pings).
- Yields the port, the frames the route has taken from the socket, and an Event set once the
- route returns."""
+ uvicorn's default WebSocket protocol and the loopback Webapp's settings (no WebSocket pings), or
+ a non-loopback bind's ``ws_ping`` (interval, timeout). Yields the port, the frames the route has
+ taken from the socket, and an Event set once the route returns."""
import uvicorn
from starlette.applications import Starlette
from starlette.routing import WebSocketRoute
@@ -508,7 +566,7 @@ def served_host_terminal(monkeypatch, session, owner):
sock = socket.socket()
sock.bind(("127.0.0.1", 0))
server = uvicorn.Server(uvicorn.Config(
- counting, lifespan="off", log_level="error", ws_ping_interval=None, ws_ping_timeout=None,
+ counting, lifespan="off", log_level="error", ws_ping_interval=ws_ping[0], ws_ping_timeout=ws_ping[1],
# A route that never returns (the bug) is cancelled instead of holding shutdown forever.
timeout_graceful_shutdown=1,
))
@@ -535,8 +593,6 @@ def test_a_tab_closed_behind_more_keys_than_the_input_queue_holds_is_detached(tm
then some, so the tab's close is unread behind them when it comes, and the browser drops the
connection. Over a real socket the shell is still detached within a few seconds: it keeps
running, retained for a reattach."""
- from hermes_cli import web_host_terminal_sessions as host_sessions
-
home = tmp_path / ".hermes"
profile = home / "profiles" / "worker"
profile.mkdir(parents=True)
@@ -545,14 +601,14 @@ def test_a_tab_closed_behind_more_keys_than_the_input_queue_holds_is_detached(tm
owner = HostOwner(("loopback", "test"), profile, "sh", str(tmp_path))
# 1 MiB: more than a pseudo-terminal buffers anywhere between its master and the program.
paste = b"".join(b"%05d %s\n" % (n, b"x" * 57) for n in range(16 * 1024))
- fill = host_sessions._INPUT_BATCH_FRAMES
+ keys, fill = keys_filling_the_input_queue(17)
with terminal(monkeypatch) as (bridge, waiting, cancelled):
session = PtySession("terminal", bridge, buffer_cap=32, read_timeout=0.01)
with served_host_terminal(monkeypatch, session, owner) as (port, taken, finished):
tab = BrowserTab(port)
tab.send(paste)
assert waiting.wait(5), "the paste never waited for the program"
- tab.send(*[b"k"] * (fill + 17))
+ tab.send(*keys)
# The paste, a full queue, and one more frame the reader holds with no room for it.
wait_until(lambda: len(taken) >= fill + 2, timeout=5, what="the reader to fill the input queue")
tab.close()
@@ -562,6 +618,42 @@ def test_a_tab_closed_behind_more_keys_than_the_input_queue_holds_is_detached(tm
assert not session.attached and session._ws is None
+def test_a_live_tab_typing_behind_a_waiting_write_answers_keepalive_pings(tmp_path, monkeypatch):
+ """A non-loopback Webapp pings its sockets (``web_server._build_uvicorn_server``) and closes one
+ whose pong does not come back in time. Keys typed while a write waits on a slow program must not
+ stop the server reading the socket, where the pongs arrive: the tab, answering every ping, stays
+ attached through several ping timeouts, and every key reaches the program once it reads."""
+ home = tmp_path / ".hermes"
+ profile = home / "profiles" / "worker"
+ profile.mkdir(parents=True)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setenv("HERMES_HOME", str(home))
+ owner = HostOwner(("loopback", "test"), profile, "sh", str(tmp_path))
+ bridge = SlowAccept()
+ session = PtySession("terminal", bridge, buffer_cap=32, read_timeout=0.01)
+ # Far more frames than the WebSocket library buffers unread (32) and than 64 queued frames.
+ typed = [bytes([65 + n % 26]) for n in range(1024)]
+ with served_host_terminal(monkeypatch, session, owner, ws_ping=(0.2, 0.3)) as (port, taken, finished):
+ tab = AnsweringTab(port)
+ try:
+ tab.send(b"A")
+ assert bridge.entered.wait(5), "the first write never started"
+ tab.send(*typed)
+ # Several ping intervals and timeouts, all while the program holds its first write.
+ wait_until(lambda: tab.pings >= 5 or finished.is_set(), timeout=10, what="five keepalive pings")
+ assert not finished.is_set() and session.attached, "the keepalive closed a live tab"
+ assert len(taken) == 1 + len(typed), "the reader stopped reading the socket"
+ bridge.release.set()
+ expected = b"A" + b"".join(typed)
+ wait_until(lambda: bytes(bridge.written) == expected, timeout=5, what="every key to reach the program")
+ assert session.attached and not finished.is_set()
+ finally:
+ bridge.release.set()
+ tab.close()
+ assert finished.wait(10), "the shell stayed attached after its tab closed"
+ assert session.alive and not session.attached
+
+
class SlowAccept:
"""A program that reads every byte, but not until ``release``: slow, not stopped. The first write
takes its bytes and then waits; later writes land at once."""
@@ -618,8 +710,7 @@ async def test_a_live_tab_typing_past_a_full_input_queue_is_held_back_and_loses_
monkeypatch.setattr(host_sessions, "get_host_terminals", lambda _app: registry)
monkeypatch.setattr(host_sessions, "_admit", admit)
- fill = host_sessions._INPUT_BATCH_FRAMES
- typed = [bytes([65 + n % 26]) for n in range(16 * fill)]
+ typed, fill = keys_filling_the_input_queue(3 * 16)
# The first write holds b"A"; the reader queues ``fill`` frames and holds one more with no room.
unread = len(typed) - fill - 1
tab = ClosingTab(b"A")