Skip to content

The agent that grows with you

fix(webapp): detach a host terminal when close meets a full input queue

Adolaniumf4390cb20a4a
diff --git a/hermes_cli/web_host_terminal_sessions.py b/hermes_cli/web_host_terminal_sessions.py
index 61f2d110b99..d0c48b7d7a2 100644
--- a/hermes_cli/web_host_terminal_sessions.py
+++ b/hermes_cli/web_host_terminal_sessions.py
@@ -9,6 +9,7 @@ refused and the reaper closes it.
from __future__ import annotations
import asyncio
+from collections import deque
from concurrent.futures import ThreadPoolExecutor
from contextlib import asynccontextmanager, suppress
from dataclasses import dataclass
@@ -32,6 +33,10 @@ _MAX_SHELLS = 16
# Inbound frames buffered while one batch is written (backpressure bound).
_INPUT_BATCH_FRAMES = 64
TOKEN_RE = re.compile(r"^[A-Za-z0-9_-]{43}$")
+# Puts abandoned because this task is already cancelling. Awaiting them here is
+# cancelled again before they finish; holding them until the callback reads the
+# result keeps "exception was never retrieved" off the loop.
+_DROPPED_TASKS: set[asyncio.Task] = set()
class TerminalDenied(Exception):
@@ -259,22 +264,120 @@ async def _admit(ws: WebSocket, registry: HostTerminalRegistry):
return session, owner
+def _read_dropped_task(task: asyncio.Task) -> None:
+ _DROPPED_TASKS.discard(task)
+ if not task.cancelled():
+ task.exception()
+
+
+def _drop_task(task: asyncio.Task) -> None:
+ """Cancel ``task`` without awaiting it. Safe while this task is itself cancelling."""
+ if not task.done():
+ task.cancel()
+ _DROPPED_TASKS.add(task)
+ task.add_done_callback(_read_dropped_task)
+
+
+class _SocketTail:
+ """Socket reads that stay ahead of a full inbox.
+
+ The inbox bound is backpressure for a live viewer. A put that waits for room must
+ not be the only thing the reader is doing: the tab's close is the next socket
+ message, possibly behind further keystrokes, and it has to surface while that
+ put is blocked. Frames already pulled are kept, in order, so a program that is
+ still reading takes them. They are not dropped to make the queue move.
+ """
+
+ def __init__(self, ws: WebSocket):
+ self._ws = ws
+ self._ahead: deque = deque()
+ self._recv: asyncio.Task | None = asyncio.create_task(ws.receive())
+
+ def close(self) -> None:
+ recv, self._recv = self._recv, None
+ if recv is not None:
+ _drop_task(recv)
+
+ def pending_receive(self) -> asyncio.Task | None:
+ return self._recv
+
+ async def next_message(self):
+ """The next data frame, or None once the socket has disconnected."""
+ if self._ahead:
+ return self._ahead.popleft()
+ if self._recv is None:
+ return None
+ msg = await self._recv
+ if msg.get("type") == "websocket.disconnect":
+ self._recv = None
+ return None
+ self._recv = asyncio.create_task(self._ws.receive())
+ return msg
+
+ def saw_disconnect(self) -> bool:
+ """Consume a receive that has already finished. Data stays queued ahead."""
+ recv = self._recv
+ if recv is None or not recv.done():
+ return False
+ msg = recv.result()
+ if msg.get("type") == "websocket.disconnect":
+ self._recv = None
+ return True
+ self._ahead.append(msg)
+ self._recv = asyncio.create_task(self._ws.receive())
+ return False
+
+
+async def _put_frame(inbox: asyncio.Queue, tail: _SocketTail, ended: asyncio.Future, msg) -> None:
+ """Queue ``msg``. A full inbox still leaves the socket readable.
+
+ On disconnect, ``ended`` is set before this put finishes, so a write waiting on
+ the program is cancelled even though the frame itself stays queued for a pump
+ that is still delivering input.
+ """
+ if not inbox.full():
+ inbox.put_nowait(msg)
+ return
+ put = asyncio.create_task(inbox.put(msg))
+ try:
+ while not put.done():
+ pending = tail.pending_receive()
+ if pending is None:
+ break
+ await asyncio.wait((put, pending), return_when=asyncio.FIRST_COMPLETED)
+ if tail.saw_disconnect() and not ended.done():
+ ended.set_result(None)
+ await put
+ finally:
+ if not put.done():
+ _drop_task(put)
+
+
+async def _queue_until_close(tail: _SocketTail, inbox: asyncio.Queue, ended: asyncio.Future) -> None:
+ while (msg := await tail.next_message()) is not None:
+ await _put_frame(inbox, tail, ended, msg)
+
+
async def _read_frames(ws: WebSocket, inbox: asyncio.Queue, ended: asyncio.Future) -> None:
"""Queue inbound frames. The last item is None, or the error that ended reading
(re-raised by ``_pump_input`` so the route's handlers still see it). ``ended`` gets the same
item as soon as reading ends, before it is queued: a write waiting on the shell, or a full
inbox behind it, must not keep the pump from seeing that the socket is gone."""
end = None
+ tail = _SocketTail(ws)
try:
- while (msg := await ws.receive()).get("type") != "websocket.disconnect":
- await inbox.put(msg)
- except RuntimeError: # a superseding/drain task already closed us
- pass
- # health: allow BLE001 -- forwarded, not swallowed: _pump_input re-raises it, and a reader that died unqueued would hang the pump
- except Exception as exc:
- end = exc
- ended.set_result(end)
- await inbox.put(end)
+ try:
+ await _queue_until_close(tail, inbox, ended)
+ except RuntimeError: # a superseding/drain task already closed us
+ pass
+ # health: allow BLE001 -- forwarded, not swallowed: _pump_input re-raises it, and a reader that died unqueued would hang the pump
+ except Exception as exc:
+ end = exc
+ if not ended.done():
+ ended.set_result(end)
+ await inbox.put(end)
+ finally:
+ tail.close()
async def _write_while_open(session: PtySession, ws: WebSocket, data: bytes, fence,
diff --git a/tests/hermes_cli/test_host_terminal_viewer_ownership.py b/tests/hermes_cli/test_host_terminal_viewer_ownership.py
index 76b686b51c9..4f9375a0ac1 100644
--- a/tests/hermes_cli/test_host_terminal_viewer_ownership.py
+++ b/tests/hermes_cli/test_host_terminal_viewer_ownership.py
@@ -353,7 +353,11 @@ def unread_posix_pty(_monkeypatch):
@pytest.mark.asyncio
[email protected]("typed_after", [0, 64], ids=["quiet", "full-inbox"])
+ "typed_after",
+ ["quiet", "full", "overflow"],
+ ids=["quiet", "full-inbox", "close-behind-full-inbox"],
+)
@pytest.mark.parametrize("terminal", [
pytest.param(unread_posix_pty, id="posix-pty", marks=pytest.mark.platforms("posix")),
pytest.param(unread_input, id="unread-input", marks=pytest.mark.platforms("any")),
@@ -365,10 +369,13 @@ async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_she
deadline. When the tab closes meanwhile, the wait is cancelled and the shell is detached, as on
any disconnect: it keeps running, retained for a reattach, and the reaper retires it once its
retention runs out. A full input queue behind the paste (keys typed while it waited) does not
- hide the close."""
+ hide the close, including when the close sits behind more frames than the queue holds."""
from hermes_cli import web_host_terminal_sessions as host_sessions
- assert typed_after in (0, host_sessions._INPUT_BATCH_FRAMES)
+ fill = host_sessions._INPUT_BATCH_FRAMES
+ # Overflow is the put that blocks: the close is not the frame that fills the queue,
+ # it is behind further keystrokes the reader has not queued yet.
+ typed = {"quiet": 0, "full": fill, "overflow": fill + 17}[typed_after]
home = tmp_path / ".hermes"
profile = home / "profiles" / "worker"
profile.mkdir(parents=True)
@@ -393,7 +400,7 @@ async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_she
try:
handler = asyncio.create_task(host_sessions.host_terminal(tab))
assert await asyncio.to_thread(waiting.wait, 5), "the paste never waited for the program"
- tab.type(*[b"k"] * typed_after)
+ tab.type(*[b"k"] * typed)
tab.close_tab()
async with asyncio.timeout(5):
await handler
@@ -410,3 +417,96 @@ async def test_closing_the_tab_during_a_waiting_paste_detaches_and_keeps_the_she
registry._sessions.clear()
registry.owners.clear()
await registry.close_all()
+
+
+class SlowAccept:
+ """A program that reads every byte, but not until ``release``: slow, not stopped.
+
+ The first write accepts its buffer (it does not report a full queue) and then waits.
+ Later writes land immediately. Keystrokes that arrived while the first write waited
+ must still be delivered, including ones that did not fit in the input queue.
+ """
+
+ def __init__(self):
+ self._closed = False
+ self.written = bytearray()
+ self.entered, self.release = threading.Event(), threading.Event()
+ self._gated = True
+
+ async def write(self, data, *, timeout=10.0, fence=None):
+ assert timeout is None, "host terminal input has no deadline"
+ accepted = await asyncio.to_thread(fence, lambda: len(data))
+ assert accepted == len(data)
+ if self._gated:
+ self._gated = False
+ self.entered.set()
+ assert await asyncio.to_thread(self.release.wait, 5), "slow reader was not released"
+ self.written.extend(data)
+ return True
+
+ def resize(self, cols, rows):
+ pass
+
+ def close(self):
+ self._closed = True
+
+ def is_alive(self):
+ return not self._closed
+
+
+async def test_slow_program_keeps_keystrokes_typed_past_a_full_input_queue(tmp_path, monkeypatch):
+ """A live tab's input is not dropped when the program reads slowly. Frames past the
+ inbox bound stay in order and reach the shell once it takes them; the socket is read
+ while the inbox is full, so a later close is not stuck behind that put."""
+ from hermes_cli import web_host_terminal_sessions as host_sessions
+
+ home = tmp_path / ".hermes"
+ profile = home / "profiles" / "worker"
+ profile.mkdir(parents=True)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setenv("HERMES_HOME", str(home))
+ owner = HostOwner(("loopback", "test"), profile, "sh", str(tmp_path))
+ registry = HostTerminalRegistry()
+ bridge = SlowAccept()
+ session = PtySession("terminal", bridge, buffer_cap=32, read_timeout=0.01)
+ registry._sessions[session.key] = session
+ registry.owners[session.key] = owner
+
+ async def admit(_ws, _registry):
+ return session, owner
+
+ monkeypatch.setattr(host_sessions, "get_host_terminals", lambda _app: registry)
+ monkeypatch.setattr(host_sessions, "_admit", admit)
+ tab = ClosingTab(b"A")
+ handler = None
+ try:
+ handler = asyncio.create_task(host_sessions.host_terminal(tab))
+ assert await asyncio.to_thread(bridge.entered.wait, 5), "the first write never started"
+ extra = [bytes([65 + (n % 26)]) for n in range(host_sessions._INPUT_BATCH_FRAMES + 17)]
+ tab.type(*extra)
+ await async_wait_until(
+ tab.inbound.empty, timeout=5,
+ what="the reader to pull keystrokes sitting behind a full input queue",
+ )
+ assert session.attached, "a full queue is not a disconnect"
+ bridge.release.set()
+ expected = b"A" + b"".join(extra)
+ await async_wait_until(
+ lambda: bytes(bridge.written) == expected, timeout=5,
+ what="every keystroke to reach the slow program",
+ )
+ tab.close_tab()
+ async with asyncio.timeout(5):
+ await handler
+ assert session.alive and not bridge._closed
+ assert registry._sessions.get(session.key) is session
+ assert not session.attached and session._ws is None
+ finally:
+ bridge.release.set()
+ if handler is not None:
+ handler.cancel()
+ await asyncio.gather(handler, return_exceptions=True)
+ registry._sessions.clear()
+ registry.owners.clear()
+ await registry.close_all()