Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

test(ci): cover resident large-chat reopening in UI benchmarks (#2301)

Florian Brand1a891aa1b765
diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.md
index 14681418b4..fc36b6ffef 100644
--- a/scripts/benchmarks/README.md
+++ b/scripts/benchmarks/README.md
@@ -163,6 +163,10 @@ Each trial stops all benchmark-user processes first, then measures:
- **Open another session from agents view:** typing the target's unique session-id prefix,
right-arrow to open, until the target transcript renders and echoes. This is the full
"session → agents view → another session" round trip with many sessions on disk.
+- **Reopen resident large session:** return to agents view and select the large session just opened.
+ Time right-arrow through transcript tail and editor echo, excluding search setup and probe cleanup.
+ This exercises attach-snapshot reuse after model-catalog refresh (#2296), separately from cold
+ worker creation. Readiness retries the editor probe every 20 ms.
- **Open chain parent from agents view:** back to the agents view, search the subagent chain's
root session by id prefix, right-arrow to open. The parent becomes live.
- **Open subagent session at depth 6:** back to the agents view, then drill into the live chain:
diff --git a/scripts/benchmarks/report.py b/scripts/benchmarks/report.py
index 4598dfed51..40fd9eff5e 100644
--- a/scripts/benchmarks/report.py
+++ b/scripts/benchmarks/report.py
@@ -122,6 +122,10 @@ UI_METRICS = (
"agents_open", "Open another session from agents view", 1000, "ms", 0.1, PERFORMANCE_NOISE_FLOOR
),
Definition("agents_open_cpu", "CPU, open from agents view", 1000, "ms", 0.05, PERFORMANCE_NOISE_FLOOR),
+ Definition("agents_reopen", "Reopen resident large session", 1000, "ms", 0.02, PERFORMANCE_NOISE_FLOOR),
+ Definition(
+ "agents_reopen_cpu", "CPU, reopen resident session", 1000, "ms", 0.01, PERFORMANCE_NOISE_FLOOR
+ ),
Definition(
"subagent_open",
"Open subagent session at depth 6",
@@ -393,8 +397,8 @@ def render(report: Report) -> str:
"40 ledger fan-out children, and a 6-deep subagent chain (~46 spawn edges).",
"Large fixtures hold 1,999 complete triples (~5 MB JSONL); medium 119; subagents 399 each.",
"Interactions: cold --resume of a large session, warm /resume switch, left-arrow to agents view,",
- "roster settle with many saved sessions, search-and-open of another large session, and opening",
- "the chain parent, then a clear-expand-step drill into the live chain to depth 6.",
+ "roster settle with many saved sessions, search-and-open of another large session,",
+ "reattaching to that resident session, opening the chain parent, and drilling to depth 6.",
"Readiness is the rendered transcript tail plus a confirmed editor echo.",
"CPU metrics sum utime+stime across the whole benchmark-user process tree per interaction.",
"UI memory sums RSS after the interactions; PTY byte counts are in the raw results.",
diff --git a/scripts/benchmarks/schema.py b/scripts/benchmarks/schema.py
index 5c8e0f20de..a938b38bcb 100644
--- a/scripts/benchmarks/schema.py
+++ b/scripts/benchmarks/schema.py
@@ -37,6 +37,8 @@ Metric = Literal[
"agents_roster_cpu",
"agents_open",
"agents_open_cpu",
+ "agents_reopen",
+ "agents_reopen_cpu",
"subagent_open",
"subagent_open_cpu",
"parent_open",
@@ -61,6 +63,8 @@ UI_METRIC_KEYS = frozenset(
"agents_roster_cpu",
"agents_open",
"agents_open_cpu",
+ "agents_reopen",
+ "agents_reopen_cpu",
"subagent_open",
"subagent_open_cpu",
"parent_open",
@@ -242,6 +246,8 @@ PHASE_METRICS: dict[str, tuple[Metric, ...]] = {
"agents_roster_cpu",
"agents_open",
"agents_open_cpu",
+ "agents_reopen",
+ "agents_reopen_cpu",
"subagent_open",
"subagent_open_cpu",
"parent_open",
diff --git a/scripts/benchmarks/terminal.py b/scripts/benchmarks/terminal.py
index 9529add64d..191cebd5a0 100644
--- a/scripts/benchmarks/terminal.py
+++ b/scripts/benchmarks/terminal.py
@@ -128,9 +128,11 @@ class Terminal:
def until(self, predicate: Callable[[Display], bool], seconds: float) -> None:
deadline = time.perf_counter() + seconds
- while not predicate(self.display):
+ while (2026 << 5) in self.display.screen.mode or not predicate(self.display):
if time.perf_counter() >= deadline:
- raise TimeoutError("Timed out waiting for terminal display")
+ raise TimeoutError(
+ "Timed out waiting for terminal display to reach the expected terminal state"
+ )
self.pump()
def until_output(self, predicate: Callable[[str], bool], seconds: float) -> None:
diff --git a/scripts/benchmarks/tests/test_terminal_readiness.py b/scripts/benchmarks/tests/test_terminal_readiness.py
index eb27f9a7b3..b4d8e135a8 100644
--- a/scripts/benchmarks/tests/test_terminal_readiness.py
+++ b/scripts/benchmarks/tests/test_terminal_readiness.py
@@ -13,6 +13,7 @@ from unittest.mock import Mock, patch
import pexpect
from terminal import MAX_PENDING_INPUT, PROBE_INTERVAL, Display, Terminal
+from ui import input_ready
class Clock:
@@ -146,6 +147,25 @@ class ReadinessTests(unittest.TestCase):
terminal.until_output(lambda output: "fresh agents view" in output, 0.1)
self.assertIn("fresh agents view", terminal.display.text())
+ def test_transcript_readiness_uses_real_terminal_and_excludes_cleanup(self):
+ terminal, editor, clock = self.launch(label="target-tail", accept_after=0.1, cleanup_delay=0.2)
+ ready_at = input_ready(terminal, "target-tail", timeout=1)
+ self.assertLess(ready_at - editor.start, 0.15)
+ self.assertGreater(clock.now - ready_at, 0.19)
+ self.assertEqual(editor.editor, "")
+
+ def test_transcript_tail_is_required_before_editor_probe(self):
+ terminal, editor, _ = self.launch(label="other-tail")
+ with self.assertRaisesRegex(TimeoutError, "expected terminal state"):
+ input_ready(terminal, "target-tail", timeout=0.1)
+ self.assertEqual(editor.sent, [])
+
+ def test_until_does_not_accept_a_partial_synchronized_frame(self):
+ terminal, editor, clock = self.launch(label="\x1b[?2026htarget-tail")
+ editor.schedule(clock.now + 0.1, "\x1b[?2026l")
+ terminal.until(lambda display: "target-tail" in display.text(), 1)
+ self.assertGreaterEqual(clock.now - editor.start, 0.1)
+
def test_changed_absent_and_stale_labels_do_not_gate_input(self):
for label in ("agents/resume", "manage", "sessions / continue", "", "benchready"):
with self.subTest(label=label):
@@ -449,6 +469,13 @@ class RealPTYReadinessTests(unittest.TestCase):
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("close stopped child", result.stdout)
+ def test_real_pty_transcript_and_editor_readiness(self):
+ terminal = self.launch()
+ ready_at = input_ready(terminal, "changed startup header", timeout=3)
+ self.assertGreater(ready_at - terminal.started, 0.35)
+ self.assertIn("begin typing here", terminal.display.text())
+ self.assertNotIn("RuntimeError", "".join(terminal.raw))
+
def test_real_pty_delayed_raw_mode_and_input_handler(self):
terminal = self.launch()
elapsed = terminal.ready(seconds=3)
diff --git a/scripts/benchmarks/tests/test_ui.py b/scripts/benchmarks/tests/test_ui.py
index 3b93e45559..9d92014594 100644
--- a/scripts/benchmarks/tests/test_ui.py
+++ b/scripts/benchmarks/tests/test_ui.py
@@ -4,11 +4,13 @@ import hashlib
import json
import os
import tempfile
+import time
import unittest
from datetime import UTC, datetime
from pathlib import Path
+from types import SimpleNamespace
from typing import get_args
-from unittest.mock import patch
+from unittest.mock import Mock, patch
import ui
from report import UI_METRICS, render
@@ -227,6 +229,66 @@ class ProbeLogicTests(unittest.TestCase):
move_down(terminal, 3)
self.assertEqual(terminal.sent, [ui.DOWN_ARROW] * 3)
+ def test_navigation_survives_persisted_agents_search(self):
+ terminal = FakeTerminal([""])
+ terminal.started = time.perf_counter()
+ terminal.ready = Mock()
+ terminal.close = Mock()
+ query = ""
+ returned_queries = []
+
+ def send(data):
+ nonlocal query
+ terminal.sent.append(data)
+ if data == ui.LEFT_ARROW:
+ returned_queries.append(query)
+ # The state-aware navigation consumes a fresh agents-view render on entry.
+ terminal.frames.append(
+ "agents 0 running, 1 idle, 153 inactive\n"
+ + (query or "Search sessions")
+ + "\nCtrl+N new"
+ )
+ elif data == ui.RIGHT_ARROW or data.startswith("/resume"):
+ terminal.display.set("chat editor")
+ return
+ elif data.startswith(ui.BACKSPACE):
+ query = query[: -len(data)]
+ elif not data.startswith("\x1b"):
+ query += data
+ terminal.display.set(
+ "agents 0 running, 1 idle, 153 inactive\n"
+ + (query or "Search sessions")
+ + "\nui-bench-large-01\nui-bench-root-00"
+ )
+
+ def stats(_uid):
+ # End after navigation so this test cannot launch subsequent benchmark workloads.
+ if "subagent_open_cpu" in side.metrics:
+ raise RuntimeError("navigation test complete")
+ return [ProcessMemory(pid=1, name="daemon", rss=100)]
+
+ terminal.child.send = send
+ with (
+ tempfile.TemporaryDirectory() as directory,
+ patch("ui.Terminal", return_value=terminal),
+ patch("ui.input_ready", side_effect=lambda *args: time.perf_counter()),
+ patch("ui.expand_subagents"),
+ patch("ui.wait_for_roster", return_value=0.1),
+ patch("ui.write_fixtures"),
+ patch("ui.pwd.getpwnam", return_value=SimpleNamespace(pw_uid=123)),
+ patch("ui.process_stats", side_effect=stats),
+ patch("worker.environment", return_value={}),
+ patch("worker.stop_processes"),
+ ):
+ side = Side(sha="a" * 40, metrics={"install": [Observation(trial=0, value=1)]})
+ root = Path(directory)
+ ui_measure(ui_request(), side, 0, results=root, homes=root, user="benchmark1")
+ for metric in ("agents_reopen", "parent_open", "subagent_open"):
+ self.assertIsNone(side.metrics[metric][0].error)
+ self.assertEqual(side.metrics["ui_rss"][0].error, "RuntimeError: navigation test complete")
+ spec = fixture_spec()
+ self.assertEqual(returned_queries, ["", spec.open_id[:8], spec.open_id[:8], spec.root[:8]])
+
def test_expand_subagents_requires_a_newly_expanded_row(self):
# An ancestor already shows "▾", so only a growing count proves the selected row expanded;
# the first expand keystroke is eaten and the retry lands.
diff --git a/scripts/benchmarks/ui.py b/scripts/benchmarks/ui.py
index 4a84fec6f4..48b74d8c33 100644
--- a/scripts/benchmarks/ui.py
+++ b/scripts/benchmarks/ui.py
@@ -36,7 +36,6 @@ TOOL_OUTPUT_LINES = 60
SEARCH_PLACEHOLDER = "Search sessions"
AGENTS_VIEW_FOOTER = "Ctrl+N new"
ROSTER_COUNT = re.compile(r"agents\s+(\d+) running, (\d+) idle, (\d+) inactive")
-READY_MARKER = "benchready"
LEFT_ARROW = "\x1b[D"
RIGHT_ARROW = "\x1b[C"
DOWN_ARROW = "\x1b[B"
@@ -344,36 +343,10 @@ def total_cpu(stats: list[ProcessMemory]) -> float:
return sum(process.cpu or 0.0 for process in stats)
-def quiet(terminal: Terminal, seconds: float = 0.6, cap: float = 8.0) -> None:
- """Pump output until the terminal stays silent, so the editor can accept input again."""
- started = time.perf_counter()
- last_output = time.perf_counter()
- while time.perf_counter() - started < cap:
- if terminal.pump():
- last_output = time.perf_counter()
- elif time.perf_counter() - last_output >= seconds:
- return
-
-
-def input_ready(terminal: Terminal, marker: str, *, timeout: float = 120) -> None:
- """Wait for a rendered marker, then confirm the editor echoes, retrying past input loss.
-
- Keystrokes sent while a freshly opened session is still mounting can be dropped,
- so the echo is retried with a quiet window instead of failing the trial.
- """
+def input_ready(terminal: Terminal, marker: str, *, timeout: float = 120) -> float:
+ """Return the first successful editor-echo timestamp after the transcript tail appears."""
terminal.until(lambda display: marker in display.text(), timeout)
- for _ in range(5):
- quiet(terminal, 0.8)
- terminal.child.send(READY_MARKER)
- try:
- terminal.until(lambda display: READY_MARKER in display.text(), 6)
- except TimeoutError:
- terminal.child.send(BACKSPACE * (len(READY_MARKER) + 2))
- continue
- terminal.child.send(BACKSPACE * len(READY_MARKER))
- terminal.until(lambda display: READY_MARKER not in display.text(), 10)
- return
- raise TimeoutError("Editor did not echo the readiness marker")
+ return terminal.started + terminal.ready(timeout)
def clear_search(terminal: Terminal, *, attempts: int = 5) -> None:
@@ -730,6 +703,23 @@ def ui_measure(request: Request, side: Side, trial: int, *, results: Path, homes
record(side, "agents_open_cpu", trial, cpu) # type: ignore[arg-type]
note("agents_open", seconds=elapsed, cpu=cpu, pty_bytes=terminal.bytes - bytes_start)
+ # Reattach to the worker just opened, excluding search/navigation setup from timing.
+ metric = "agents_reopen"
+ open_agents_view(terminal, clear=True)
+ type_query(terminal, spec.open_id[:8])
+ terminal.until(lambda display: session_name("large", 1) in display.text(), 60)
+ terminal.settle(0.8)
+ started = time.perf_counter()
+ cpu_start = cpu_total()
+ bytes_start = terminal.bytes
+ terminal.child.send(RIGHT_ARROW)
+ ready_at = input_ready(terminal, tail_marker("large", 1))
+ elapsed = ready_at - started
+ cpu = cpu_total() - cpu_start
+ record(side, "agents_reopen", trial, elapsed)
+ record(side, "agents_reopen_cpu", trial, cpu)
+ note("agents_reopen", seconds=elapsed, cpu=cpu, pty_bytes=terminal.bytes - bytes_start)
+
# Chain parent: open the root so the deepest subagent opens against a live parent.
metric = "parent_open"
started = time.perf_counter()