A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
test(ci): cover resident large-chat reopening in UI benchmarks (#2301)
Florian Brand1a891aa1b765
diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.mdindex 14681418b4..fc36b6ffef 100644--- a/scripts/benchmarks/README.md+++ b/scripts/benchmarks/README.md@@ -163,6 +163,10 @@ Each trial stops all benchmark-user processes first, then measures:- **Open another session from agents view:** typing the target's unique session-id prefix,right-arrow to open, until the target transcript renders and echoes. This is the full"session → agents view → another session" round trip with many sessions on disk.+- **Reopen resident large session:** return to agents view and select the large session just opened.+ Time right-arrow through transcript tail and editor echo, excluding search setup and probe cleanup.+ This exercises attach-snapshot reuse after model-catalog refresh (#2296), separately from cold+ worker creation. Readiness retries the editor probe every 20 ms.- **Open chain parent from agents view:** back to the agents view, search the subagent chain'sroot session by id prefix, right-arrow to open. The parent becomes live.- **Open subagent session at depth 6:** back to the agents view, then drill into the live chain:diff --git a/scripts/benchmarks/report.py b/scripts/benchmarks/report.pyindex 4598dfed51..40fd9eff5e 100644--- a/scripts/benchmarks/report.py+++ b/scripts/benchmarks/report.py@@ -122,6 +122,10 @@ UI_METRICS = ("agents_open", "Open another session from agents view", 1000, "ms", 0.1, PERFORMANCE_NOISE_FLOOR),Definition("agents_open_cpu", "CPU, open from agents view", 1000, "ms", 0.05, PERFORMANCE_NOISE_FLOOR),+ Definition("agents_reopen", "Reopen resident large session", 1000, "ms", 0.02, PERFORMANCE_NOISE_FLOOR),+ Definition(+ "agents_reopen_cpu", "CPU, reopen resident session", 1000, "ms", 0.01, PERFORMANCE_NOISE_FLOOR+ ),Definition("subagent_open","Open subagent session at depth 6",@@ -393,8 +397,8 @@ def render(report: Report) -> str:"40 ledger fan-out children, and a 6-deep subagent chain (~46 spawn edges).","Large fixtures hold 1,999 complete triples (~5 MB JSONL); medium 119; subagents 399 each.","Interactions: cold --resume of a large session, warm /resume switch, left-arrow to agents view,",- "roster settle with many saved sessions, search-and-open of another large session, and opening",- "the chain parent, then a clear-expand-step drill into the live chain to depth 6.",+ "roster settle with many saved sessions, search-and-open of another large session,",+ "reattaching to that resident session, opening the chain parent, and drilling to depth 6.","Readiness is the rendered transcript tail plus a confirmed editor echo.","CPU metrics sum utime+stime across the whole benchmark-user process tree per interaction.","UI memory sums RSS after the interactions; PTY byte counts are in the raw results.",diff --git a/scripts/benchmarks/schema.py b/scripts/benchmarks/schema.pyindex 5c8e0f20de..a938b38bcb 100644--- a/scripts/benchmarks/schema.py+++ b/scripts/benchmarks/schema.py@@ -37,6 +37,8 @@ Metric = Literal["agents_roster_cpu","agents_open","agents_open_cpu",+ "agents_reopen",+ "agents_reopen_cpu","subagent_open","subagent_open_cpu","parent_open",@@ -61,6 +63,8 @@ UI_METRIC_KEYS = frozenset("agents_roster_cpu","agents_open","agents_open_cpu",+ "agents_reopen",+ "agents_reopen_cpu","subagent_open","subagent_open_cpu","parent_open",@@ -242,6 +246,8 @@ PHASE_METRICS: dict[str, tuple[Metric, ...]] = {"agents_roster_cpu","agents_open","agents_open_cpu",+ "agents_reopen",+ "agents_reopen_cpu","subagent_open","subagent_open_cpu","parent_open",diff --git a/scripts/benchmarks/terminal.py b/scripts/benchmarks/terminal.pyindex 9529add64d..191cebd5a0 100644--- a/scripts/benchmarks/terminal.py+++ b/scripts/benchmarks/terminal.py@@ -128,9 +128,11 @@ class Terminal:def until(self, predicate: Callable[[Display], bool], seconds: float) -> None:deadline = time.perf_counter() + seconds- while not predicate(self.display):+ while (2026 << 5) in self.display.screen.mode or not predicate(self.display):if time.perf_counter() >= deadline:- raise TimeoutError("Timed out waiting for terminal display")+ raise TimeoutError(+ "Timed out waiting for terminal display to reach the expected terminal state"+ )self.pump()def until_output(self, predicate: Callable[[str], bool], seconds: float) -> None:diff --git a/scripts/benchmarks/tests/test_terminal_readiness.py b/scripts/benchmarks/tests/test_terminal_readiness.pyindex eb27f9a7b3..b4d8e135a8 100644--- a/scripts/benchmarks/tests/test_terminal_readiness.py+++ b/scripts/benchmarks/tests/test_terminal_readiness.py@@ -13,6 +13,7 @@ from unittest.mock import Mock, patchimport pexpectfrom terminal import MAX_PENDING_INPUT, PROBE_INTERVAL, Display, Terminal+from ui import input_readyclass Clock:@@ -146,6 +147,25 @@ class ReadinessTests(unittest.TestCase):terminal.until_output(lambda output: "fresh agents view" in output, 0.1)self.assertIn("fresh agents view", terminal.display.text())+ def test_transcript_readiness_uses_real_terminal_and_excludes_cleanup(self):+ terminal, editor, clock = self.launch(label="target-tail", accept_after=0.1, cleanup_delay=0.2)+ ready_at = input_ready(terminal, "target-tail", timeout=1)+ self.assertLess(ready_at - editor.start, 0.15)+ self.assertGreater(clock.now - ready_at, 0.19)+ self.assertEqual(editor.editor, "")++ def test_transcript_tail_is_required_before_editor_probe(self):+ terminal, editor, _ = self.launch(label="other-tail")+ with self.assertRaisesRegex(TimeoutError, "expected terminal state"):+ input_ready(terminal, "target-tail", timeout=0.1)+ self.assertEqual(editor.sent, [])++ def test_until_does_not_accept_a_partial_synchronized_frame(self):+ terminal, editor, clock = self.launch(label="\x1b[?2026htarget-tail")+ editor.schedule(clock.now + 0.1, "\x1b[?2026l")+ terminal.until(lambda display: "target-tail" in display.text(), 1)+ self.assertGreaterEqual(clock.now - editor.start, 0.1)+def test_changed_absent_and_stale_labels_do_not_gate_input(self):for label in ("agents/resume", "manage", "sessions / continue", "", "benchready"):with self.subTest(label=label):@@ -449,6 +469,13 @@ class RealPTYReadinessTests(unittest.TestCase):self.assertEqual(result.returncode, 0, result.stdout + result.stderr)self.assertIn("close stopped child", result.stdout)+ def test_real_pty_transcript_and_editor_readiness(self):+ terminal = self.launch()+ ready_at = input_ready(terminal, "changed startup header", timeout=3)+ self.assertGreater(ready_at - terminal.started, 0.35)+ self.assertIn("begin typing here", terminal.display.text())+ self.assertNotIn("RuntimeError", "".join(terminal.raw))+def test_real_pty_delayed_raw_mode_and_input_handler(self):terminal = self.launch()elapsed = terminal.ready(seconds=3)diff --git a/scripts/benchmarks/tests/test_ui.py b/scripts/benchmarks/tests/test_ui.pyindex 3b93e45559..9d92014594 100644--- a/scripts/benchmarks/tests/test_ui.py+++ b/scripts/benchmarks/tests/test_ui.py@@ -4,11 +4,13 @@ import hashlibimport jsonimport osimport tempfile+import timeimport unittestfrom datetime import UTC, datetimefrom pathlib import Path+from types import SimpleNamespacefrom typing import get_args-from unittest.mock import patch+from unittest.mock import Mock, patchimport uifrom report import UI_METRICS, render@@ -227,6 +229,66 @@ class ProbeLogicTests(unittest.TestCase):move_down(terminal, 3)self.assertEqual(terminal.sent, [ui.DOWN_ARROW] * 3)+ def test_navigation_survives_persisted_agents_search(self):+ terminal = FakeTerminal([""])+ terminal.started = time.perf_counter()+ terminal.ready = Mock()+ terminal.close = Mock()+ query = ""+ returned_queries = []++ def send(data):+ nonlocal query+ terminal.sent.append(data)+ if data == ui.LEFT_ARROW:+ returned_queries.append(query)+ # The state-aware navigation consumes a fresh agents-view render on entry.+ terminal.frames.append(+ "agents 0 running, 1 idle, 153 inactive\n"+ + (query or "Search sessions")+ + "\nCtrl+N new"+ )+ elif data == ui.RIGHT_ARROW or data.startswith("/resume"):+ terminal.display.set("chat editor")+ return+ elif data.startswith(ui.BACKSPACE):+ query = query[: -len(data)]+ elif not data.startswith("\x1b"):+ query += data+ terminal.display.set(+ "agents 0 running, 1 idle, 153 inactive\n"+ + (query or "Search sessions")+ + "\nui-bench-large-01\nui-bench-root-00"+ )++ def stats(_uid):+ # End after navigation so this test cannot launch subsequent benchmark workloads.+ if "subagent_open_cpu" in side.metrics:+ raise RuntimeError("navigation test complete")+ return [ProcessMemory(pid=1, name="daemon", rss=100)]++ terminal.child.send = send+ with (+ tempfile.TemporaryDirectory() as directory,+ patch("ui.Terminal", return_value=terminal),+ patch("ui.input_ready", side_effect=lambda *args: time.perf_counter()),+ patch("ui.expand_subagents"),+ patch("ui.wait_for_roster", return_value=0.1),+ patch("ui.write_fixtures"),+ patch("ui.pwd.getpwnam", return_value=SimpleNamespace(pw_uid=123)),+ patch("ui.process_stats", side_effect=stats),+ patch("worker.environment", return_value={}),+ patch("worker.stop_processes"),+ ):+ side = Side(sha="a" * 40, metrics={"install": [Observation(trial=0, value=1)]})+ root = Path(directory)+ ui_measure(ui_request(), side, 0, results=root, homes=root, user="benchmark1")+ for metric in ("agents_reopen", "parent_open", "subagent_open"):+ self.assertIsNone(side.metrics[metric][0].error)+ self.assertEqual(side.metrics["ui_rss"][0].error, "RuntimeError: navigation test complete")+ spec = fixture_spec()+ self.assertEqual(returned_queries, ["", spec.open_id[:8], spec.open_id[:8], spec.root[:8]])+def test_expand_subagents_requires_a_newly_expanded_row(self):# An ancestor already shows "▾", so only a growing count proves the selected row expanded;# the first expand keystroke is eaten and the retry lands.diff --git a/scripts/benchmarks/ui.py b/scripts/benchmarks/ui.pyindex 4a84fec6f4..48b74d8c33 100644--- a/scripts/benchmarks/ui.py+++ b/scripts/benchmarks/ui.py@@ -36,7 +36,6 @@ TOOL_OUTPUT_LINES = 60SEARCH_PLACEHOLDER = "Search sessions"AGENTS_VIEW_FOOTER = "Ctrl+N new"ROSTER_COUNT = re.compile(r"agents\s+(\d+) running, (\d+) idle, (\d+) inactive")-READY_MARKER = "benchready"LEFT_ARROW = "\x1b[D"RIGHT_ARROW = "\x1b[C"DOWN_ARROW = "\x1b[B"@@ -344,36 +343,10 @@ def total_cpu(stats: list[ProcessMemory]) -> float:return sum(process.cpu or 0.0 for process in stats)-def quiet(terminal: Terminal, seconds: float = 0.6, cap: float = 8.0) -> None:- """Pump output until the terminal stays silent, so the editor can accept input again."""- started = time.perf_counter()- last_output = time.perf_counter()- while time.perf_counter() - started < cap:- if terminal.pump():- last_output = time.perf_counter()- elif time.perf_counter() - last_output >= seconds:- return---def input_ready(terminal: Terminal, marker: str, *, timeout: float = 120) -> None:- """Wait for a rendered marker, then confirm the editor echoes, retrying past input loss.-- Keystrokes sent while a freshly opened session is still mounting can be dropped,- so the echo is retried with a quiet window instead of failing the trial.- """+def input_ready(terminal: Terminal, marker: str, *, timeout: float = 120) -> float:+ """Return the first successful editor-echo timestamp after the transcript tail appears."""terminal.until(lambda display: marker in display.text(), timeout)- for _ in range(5):- quiet(terminal, 0.8)- terminal.child.send(READY_MARKER)- try:- terminal.until(lambda display: READY_MARKER in display.text(), 6)- except TimeoutError:- terminal.child.send(BACKSPACE * (len(READY_MARKER) + 2))- continue- terminal.child.send(BACKSPACE * len(READY_MARKER))- terminal.until(lambda display: READY_MARKER not in display.text(), 10)- return- raise TimeoutError("Editor did not echo the readiness marker")+ return terminal.started + terminal.ready(timeout)def clear_search(terminal: Terminal, *, attempts: int = 5) -> None:@@ -730,6 +703,23 @@ def ui_measure(request: Request, side: Side, trial: int, *, results: Path, homesrecord(side, "agents_open_cpu", trial, cpu) # type: ignore[arg-type]note("agents_open", seconds=elapsed, cpu=cpu, pty_bytes=terminal.bytes - bytes_start)+ # Reattach to the worker just opened, excluding search/navigation setup from timing.+ metric = "agents_reopen"+ open_agents_view(terminal, clear=True)+ type_query(terminal, spec.open_id[:8])+ terminal.until(lambda display: session_name("large", 1) in display.text(), 60)+ terminal.settle(0.8)+ started = time.perf_counter()+ cpu_start = cpu_total()+ bytes_start = terminal.bytes+ terminal.child.send(RIGHT_ARROW)+ ready_at = input_ready(terminal, tail_marker("large", 1))+ elapsed = ready_at - started+ cpu = cpu_total() - cpu_start+ record(side, "agents_reopen", trial, elapsed)+ record(side, "agents_reopen_cpu", trial, cpu)+ note("agents_reopen", seconds=elapsed, cpu=cpu, pty_bytes=terminal.bytes - bytes_start)+# Chain parent: open the root so the deepest subagent opens against a live parent.metric = "parent_open"started = time.perf_counter()