Skip to content

The agent that grows with you

fix(pty): keep a reattach replay above a floor, grow it back, and say when it trims output

emozillad96595c30e33
diff --git a/hermes_cli/pty_session.py b/hermes_cli/pty_session.py
index 7f88c4f2596..f07e6740fbe 100644
--- a/hermes_cli/pty_session.py
+++ b/hermes_cli/pty_session.py
@@ -32,6 +32,10 @@ _VIEWER_SEND_TIMEOUT_SECONDS = 5.0
# to cross a link this slow (about 256 kbit/s; 37 s for 1 MiB). A replay that misses that deadline is
# halved for the session's next attach, so a viewer on a slower link redials onto a replay it carries.
_REPLAY_MIN_BYTES_PER_SECOND = 32 * 1024
+# The smallest replay a timed-out one shrinks to, so a reattach always shows the newest output. Its
+# deadline needs about 3 KB/s, less than a link needs to keep up with live output (one frame per
+# stall limit).
+_REPLAY_FLOOR_BYTES = 16 * 1024
# A viewer send that fails because the browser is gone: Starlette raises WebSocketDisconnect for a
# dead transport and RuntimeError for a socket already closed; OSError covers the TimeoutError of
# a viewer that stopped reading.
@@ -98,7 +102,9 @@ class PtySession:
self.active_session_file = active_session_file
self.active_session_cleanup: Optional[Callable[[], None]] = None
self.buffer = RingBuffer(buffer_cap)
- # How many of the buffer's newest bytes a reattach replays; a replay that times out halves it.
+ # How many of the buffer's newest bytes a reattach replays: a replay that times out halves it
+ # (down to _REPLAY_FLOOR_BYTES), one that arrives well inside its deadline doubles it back.
+ self._buffer_cap = buffer_cap
self._replay_limit = buffer_cap
self.alive = True
self.attached = False
@@ -201,11 +207,16 @@ class PtySession:
if self._ws is ws and self.alive:
self.bridge.resize(cols=cols, rows=rows)
- async def attach(self, ws, *, force_redraw: bool = False, initial_text: Optional[str] = None) -> bool:
+ async def attach(
+ self, ws, *, force_redraw: bool = False,
+ initial_text: Optional[str | Callable[[bool], str]] = None,
+ ) -> bool:
"""Attach a browser terminal and replay buffered PTY output.
The TUI renders differentially on an alternate screen, so a bounded ANSI tail is not a
self-contained frame; ``force_redraw`` asks the live TUI for one full redraw after replay.
+ ``initial_text`` is sent before the replay; given as a function, it is called with whether
+ the replay leaves out older output (dropped from the buffer or past the replay limit).
"""
self._attach_generation += 1
generation = self._attach_generation
@@ -231,14 +242,16 @@ class PtySession:
self._viewer_lock.release()
try:
async with output_lock:
- snap = self.buffer.snapshot()
- snap = snap[len(snap) - min(len(snap), self._replay_limit):]
+ held = self.buffer.snapshot()
+ snap = held[len(held) - min(len(held), self._replay_limit):]
+ trimmed = self.buffer.truncated or len(snap) < len(held)
if old_ws is not ws:
await _close_ws(old_ws, WS_CLOSE_SUPERSEDED)
if self._ws is not ws:
return False
if initial_text is not None:
- await asyncio.wait_for(ws.send_text(initial_text), timeout=_VIEWER_SEND_TIMEOUT_SECONDS)
+ text = initial_text(trimmed) if callable(initial_text) else initial_text
+ await asyncio.wait_for(ws.send_text(text), timeout=_VIEWER_SEND_TIMEOUT_SECONDS)
if self._ws is not ws:
return False
# Metadata-bearing host clients treat exactly the next binary
@@ -246,11 +259,16 @@ class PtySession:
# frame never loses terminal-query replies on an empty history.
if snap or initial_text is not None:
replay_timeout = _VIEWER_SEND_TIMEOUT_SECONDS + len(snap) / _REPLAY_MIN_BYTES_PER_SECOND
+ started = time.monotonic()
try:
await asyncio.wait_for(ws.send_bytes(snap), timeout=replay_timeout)
except TimeoutError:
- self._replay_limit = len(snap) // 2
+ self._replay_limit = max(_REPLAY_FLOOR_BYTES, len(snap) // 2)
raise
+ # A replay that arrived within half its deadline means twice as much fits in
+ # the larger replay's deadline, so a limit an earlier slow viewer cut grows back.
+ if len(snap) < len(held) and time.monotonic() - started <= replay_timeout / 2:
+ self._replay_limit = min(self._buffer_cap, 2 * len(snap))
except _VIEWER_SEND_ERRORS:
self.detach(ws)
return False
diff --git a/hermes_cli/web_host_terminal_sessions.py b/hermes_cli/web_host_terminal_sessions.py
index 04d951d7dfe..aef3f52c7ca 100644
--- a/hermes_cli/web_host_terminal_sessions.py
+++ b/hermes_cli/web_host_terminal_sessions.py
@@ -219,11 +219,11 @@ async def host_terminal_lifespan(app):
await registry.close_all()
-def _metadata(token, owner, session, registry, *, reconnected):
+def _metadata(token, owner, registry, *, reconnected, truncated):
return META_PREFIX + json.dumps({
"shell": owner.shell, "terminalId": token, "cwd": owner.cwd,
"reconnected": reconnected, "retentionSeconds": registry._ttl,
- "truncated": session.buffer.truncated,
+ "truncated": truncated,
}, separators=(",", ":"))
@@ -358,8 +358,12 @@ async def host_terminal(ws: WebSocket) -> None:
return
session, owner = admitted
token = session.key
- initial_text = _metadata(
- token, owner, session, registry, reconnected=ws.query_params.get("attach") is not None)
+ reconnected = ws.query_params.get("attach") is not None
+
+ def initial_text(truncated: bool) -> str:
+ # Says whether the replay that follows leaves out older output.
+ return _metadata(token, owner, registry, reconnected=reconnected, truncated=truncated)
+
if not await session.attach(ws, initial_text=initial_text):
if session._ws is None:
await _close_ws(ws, WS_CLOSE_VIEWER_STALLED, WS_CLOSE_VIEWER_STALLED_REASON)
diff --git a/tests/hermes_cli/test_pty_session.py b/tests/hermes_cli/test_pty_session.py
index c3442cbd194..fe9e049661b 100644
--- a/tests/hermes_cli/test_pty_session.py
+++ b/tests/hermes_cli/test_pty_session.py
@@ -238,6 +238,7 @@ async def test_redials_over_a_link_too_slow_for_the_whole_replay_reach_a_replay_
monkeypatch.setattr(pty_session, "_VIEWER_SEND_TIMEOUT_SECONDS", 0.01)
monkeypatch.setattr(pty_session, "_REPLAY_MIN_BYTES_PER_SECOND", 1024 ** 3)
+ monkeypatch.setattr(pty_session, "_REPLAY_FLOOR_BYTES", 1024, raising=False)
history = bytes(range(256)) * 256
session = PtySession("k", FakeBridge([]), buffer_cap=len(history), read_timeout=0.01)
session.buffer.append(history)
@@ -259,6 +260,101 @@ async def test_redials_over_a_link_too_slow_for_the_whole_replay_reach_a_replay_
assert session._ws is viewer and viewer.close_code is None
+async def test_a_shrunk_replay_keeps_a_floor_and_grows_back_once_viewers_keep_up(monkeypatch):
+ """Replays that keep timing out shrink the next one only down to a floor, so a reattach still
+ shows the newest output instead of nothing. Replays that arrive well inside their deadline grow
+ it back, so a viewer on a good link gets the whole retained history again after a few
+ reattaches."""
+ from hermes_cli import pty_session
+ from hermes_cli.pty_session import PtySession
+
+ monkeypatch.setattr(pty_session, "_VIEWER_SEND_TIMEOUT_SECONDS", 0.01)
+ monkeypatch.setattr(pty_session, "_REPLAY_MIN_BYTES_PER_SECOND", 1024 ** 3)
+ monkeypatch.setattr(pty_session, "_REPLAY_FLOOR_BYTES", 1024, raising=False)
+ history = bytes(range(256)) * 256
+ session = PtySession("k", FakeBridge([]), buffer_cap=len(history), read_timeout=0.01)
+ session.buffer.append(history)
+
+ for _stall in range(20):
+ await session.attach(NarrowLinkWS(0)) # never finishes a frame of output
+
+ replays = []
+ for _reattach in range(10):
+ viewer = FakeWS()
+ assert await session.attach(viewer)
+ replays.append(viewer.sent[0][1] if viewer.sent else b"")
+ if replays[-1] == history:
+ break
+ assert replays[0] == history[-1024:]
+ assert all(history.endswith(replay) for replay in replays)
+ assert [len(replay) for replay in replays] == sorted(len(replay) for replay in replays)
+ assert replays[-1] == history
+
+
+async def test_a_reattach_that_replays_part_of_the_history_says_output_was_trimmed(tmp_path, monkeypatch):
+ """The host terminal's metadata frame tells the browser ``truncated: true`` whenever the replay
+ after it leaves out older output, including output the session still holds but a shrunk replay
+ omits, so the "output was trimmed" notice shows. A replay of everything says ``false``."""
+ import json
+ import secrets
+ from types import SimpleNamespace
+
+ from hermes_cli import pty_session
+ from hermes_cli.web_host_terminal import META_PREFIX
+ from hermes_cli.web_host_terminal_sessions import HostOwner, HostTerminalRegistry, host_terminal
+
+ home = tmp_path / "home"
+ home.mkdir()
+ monkeypatch.setenv("HERMES_HOME", str(home))
+ monkeypatch.setattr(pty_session, "_VIEWER_SEND_TIMEOUT_SECONDS", 0.01)
+ monkeypatch.setattr(pty_session, "_REPLAY_MIN_BYTES_PER_SECOND", 1024 ** 3)
+ history = bytes(range(256)) * 256
+ registry = HostTerminalRegistry()
+ token = secrets.token_urlsafe(32)
+ session = pty_session.PtySession(token, FakeBridge([]), buffer_cap=1024 * 1024, read_timeout=0.01)
+ session.buffer.append(history)
+ registry._sessions[token] = session
+ registry.owners[token] = HostOwner(("local", "owner"), home.resolve(), "sh", str(tmp_path))
+ app = SimpleNamespace(state=SimpleNamespace(host_terminals=registry))
+
+ class Viewer(NarrowLinkWS):
+ """A browser tab reattaching to the shell, then closing."""
+
+ def __init__(self, capacity):
+ super().__init__(capacity)
+ self.app = app
+ self.query_params = {"attach": token}
+ self._hermes_auth_identity = {"user_id": "owner", "provider": "local"}
+
+ async def receive(self):
+ return {"type": "websocket.disconnect"}
+
+ def metadata(viewer):
+ kind, text = viewer.sent[0]
+ assert kind == "text" and text.startswith(META_PREFIX)
+ return json.loads(text.removeprefix(META_PREFIX))
+
+ try:
+ stalled = Viewer(0)
+ await host_terminal(stalled)
+ assert stalled.close_code == 1001 and metadata(stalled)["truncated"] is False
+
+ partial = Viewer(len(history))
+ await host_terminal(partial)
+ kind, replay = partial.sent[1]
+ assert kind == "bytes" and len(replay) < len(history) and history.endswith(replay)
+ assert metadata(partial)["truncated"] is True
+
+ whole = Viewer(len(history))
+ await host_terminal(whole)
+ assert whole.sent[1] == ("bytes", history)
+ assert metadata(whole)["truncated"] is False
+ finally:
+ await registry.close_all()
+
+
@pytest.mark.asyncio
async def test_reattach_can_force_complete_tui_redraw_after_replay():
"""A fresh terminal cannot reconstruct a differential ANSI tail alone."""