A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
[RSI, performance] decode private frames in linear time across socket chunks (#2399)
Seth Karten756a8ce83edb
diff --git a/packages/coding-agent/.changes/cb-perf-switch-resume.md b/packages/coding-agent/.changes/cb-perf-switch-resume.mdnew file mode 100644index 0000000000..aaacab895e--- /dev/null+++ b/packages/coding-agent/.changes/cb-perf-switch-resume.md@@ -0,0 +1 @@+- Private worker framing now decodes large frames in linear time across socket chunks: delivered chunks are kept as-is and a completed frame's bytes are joined exactly once, and consumed chunks are skipped via a head cursor with amortized compaction instead of being shifted off the queue one per chunk, so a multi-MB frame arriving in small writes can no longer pin the event loop.diff --git a/packages/coding-agent/src/modes/session-worker/private-framing.ts b/packages/coding-agent/src/modes/session-worker/private-framing.tsindex 4caf143f6a..8190c59690 100644--- a/packages/coding-agent/src/modes/session-worker/private-framing.ts+++ b/packages/coding-agent/src/modes/session-worker/private-framing.ts@@ -2,6 +2,9 @@ import type { Duplex } from "node:stream";const FRAME_PREFIX_BYTES = 8;+/** Minimum spent entries before compaction splices the consumed prefix away. */+const PENDING_COMPACTION_MIN_HEAD = 32;+export interface PrivateFrameLimits {maxHeaderBytes: number;maxPayloadBytes: number;@@ -51,7 +54,25 @@ export function encodePrivateFrame<THeader extends object>(}export class PrivateFrameDecoder<THeader extends object> {- private buffered: Buffer<ArrayBufferLike> = Buffer.alloc(0);+ /**+ * Received-but-unparsed bytes, kept as the chunks the socket delivered.+ *+ * We never concatenate into a single growing buffer: appending to one+ * accumulator on every socket read is O(n^2) when a single frame is split+ * across many reads (e.g. a multi-MB snapshot response arriving in 8KB+ * chunks). Instead, each chunk is stored as-is and a frame's bytes are+ * joined exactly once, when the frame completes. Spent chunks are skipped+ * via a head cursor rather than shifted off one per chunk — shifting every+ * fully consumed chunk is itself quadratic in the chunk count — and the+ * spent prefix is compacted in amortized O(1) per chunk. Mirrors the JSONL+ * line reader's rationale in ../rpc/jsonl.ts.+ */+ private pending: Buffer[] = [];+ private unreadBytes = 0;+ /** Index of the first chunk still holding unread bytes; entries before it are spent. */+ private head = 0;+ /** Bytes already consumed within pending[head]. */+ private offset = 0;constructor(private readonly validateHeader: PrivateFrameHeaderValidator<THeader>,@@ -59,20 +80,20 @@ export class PrivateFrameDecoder<THeader extends object> {) {}get bufferedBytes(): number {- return this.buffered.length;+ return this.unreadBytes;}push(chunk: Uint8Array): PrivateFrame<THeader>[] {if (chunk.length > 0) {- const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);- this.buffered = this.buffered.length === 0 ? buffer : Buffer.concat([this.buffered, buffer]);+ this.pending.push(Buffer.isBuffer(chunk) ? (chunk as Buffer) : Buffer.from(chunk));+ this.unreadBytes += chunk.length;}const frames: PrivateFrame<THeader>[] = [];- let offset = 0;- while (this.buffered.length - offset >= FRAME_PREFIX_BYTES) {- const headerLength = this.buffered.readUInt32BE(offset);- const payloadLength = this.buffered.readUInt32BE(offset + 4);+ while (this.unreadBytes >= FRAME_PREFIX_BYTES) {+ const prefix = this.slice(0, FRAME_PREFIX_BYTES);+ const headerLength = prefix.readUInt32BE(0);+ const payloadLength = prefix.readUInt32BE(4);assertFrameLength("header length", headerLength, this.limits.maxHeaderBytes);assertFrameLength("payload length", payloadLength, this.limits.maxPayloadBytes);if (headerLength === 0) {@@ -80,15 +101,15 @@ export class PrivateFrameDecoder<THeader extends object> {}const frameLength = FRAME_PREFIX_BYTES + headerLength + payloadLength;- if (this.buffered.length - offset < frameLength) {+ if (this.unreadBytes < frameLength) {break;}- const headerStart = offset + FRAME_PREFIX_BYTES;+ const headerStart = FRAME_PREFIX_BYTES;const payloadStart = headerStart + headerLength;let decoded: unknown;try {- decoded = JSON.parse(this.buffered.toString("utf8", headerStart, payloadStart));+ decoded = JSON.parse(this.slice(headerStart, payloadStart).toString("utf8"));} catch (error) {throw new Error(`Invalid private frame header JSON: ${error instanceof Error ? error.message : String(error)}`,@@ -100,20 +121,63 @@ export class PrivateFrameDecoder<THeader extends object> {frames.push({header: decoded,- payload: Buffer.from(this.buffered.subarray(payloadStart, payloadStart + payloadLength)),+ payload: this.slice(payloadStart, frameLength),});- offset += frameLength;+ this.consume(frameLength);}- if (offset > 0) {- this.buffered = Buffer.from(this.buffered.subarray(offset));- }return frames;}finish(): void {- if (this.buffered.length !== 0) {- throw new Error(`Private frame channel ended with ${this.buffered.length} incomplete bytes`);+ if (this.unreadBytes !== 0) {+ throw new Error(`Private frame channel ended with ${this.unreadBytes} incomplete bytes`);+ }+ }++ /** Copy [start, endExclusive) of the unread bytes across pending chunks. */+ private slice(start: number, endExclusive: number): Buffer {+ const parts: Buffer[] = [];+ let position = 0;+ for (let index = this.head; index < this.pending.length && position < endExclusive; index++) {+ const chunk = this.pending[index];+ const usable = index === this.head ? chunk.subarray(this.offset) : chunk;+ const chunkEnd = position + usable.length;+ if (chunkEnd > start) {+ parts.push(+ usable.subarray(Math.max(start, position) - position, Math.min(endExclusive, chunkEnd) - position),+ );+ }+ position = chunkEnd;+ }+ if (parts.length === 1) {+ return Buffer.from(parts[0]);+ }+ return Buffer.concat(parts, endExclusive - start);+ }++ private consume(count: number): void {+ this.unreadBytes -= count;+ this.offset += count;+ while (this.head < this.pending.length && this.offset >= this.pending[this.head].length) {+ this.offset -= this.pending[this.head].length;+ this.head++;+ }++ // Fully drained: clear in one shot instead of one shift per chunk.+ if (this.head === this.pending.length) {+ this.pending.length = 0;+ this.head = 0;+ this.offset = 0;+ return;+ }++ // Splicing moves pending.length - head entries. At this threshold that is+ // at most head, and head only advances by one per consumed chunk, so+ // compaction stays amortized O(1) per chunk.+ if (this.head >= PENDING_COMPACTION_MIN_HEAD && this.head * 2 >= this.pending.length) {+ this.pending.splice(0, this.head);+ this.head = 0;}}}diff --git a/packages/coding-agent/test/session-worker-private-framing.test.ts b/packages/coding-agent/test/session-worker-private-framing.test.tsindex ebb6116f3a..bb406a5b1b 100644--- a/packages/coding-agent/test/session-worker-private-framing.test.ts+++ b/packages/coding-agent/test/session-worker-private-framing.test.ts@@ -1,7 +1,9 @@+import { performance } from "node:perf_hooks";import { PassThrough } from "node:stream";import { describe, expect, it } from "vitest";import {encodePrivateFrame,+ type PrivateFrame,PrivateFrameDecoder,PrivateFramedChannel,type PrivateFrameHeaderValidator,@@ -25,21 +27,27 @@ const isTestHeader: PrivateFrameHeaderValidator<TestHeader> = (value: unknown):describe("private worker framing", () => {it("decodes headers and opaque payloads across arbitrary chunk boundaries", () => {- const first = encodePrivateFrame({ type: "event", requestId: "one" }, Buffer.from([0, 1, 2, 255]));- const second = encodePrivateFrame({ type: "response", requestId: "two" }, Buffer.from("payload"));- const combined = Buffer.concat([first, second]);- const decoder = new PrivateFrameDecoder(isTestHeader);- const frames = [];-- for (let offset = 0; offset < combined.length; offset += 3) {- frames.push(...decoder.push(combined.subarray(offset, offset + 3)));- }- decoder.finish();-- expect(frames).toEqual([+ const frames = [+ encodePrivateFrame({ type: "event", requestId: "one" }, Buffer.from([0, 1, 2, 255])),+ encodePrivateFrame({ type: "response", requestId: "two" }, Buffer.from("payload")),+ encodePrivateFrame({ type: "event" }, Buffer.alloc(0)),+ ];+ const expected = [{ header: { type: "event", requestId: "one" }, payload: Buffer.from([0, 1, 2, 255]) },{ header: { type: "response", requestId: "two" }, payload: Buffer.from("payload") },- ]);+ { header: { type: "event" }, payload: Buffer.alloc(0) },+ ];+ // One byte at a time and three-byte chunks both cover splits at every boundary.+ const combined = Buffer.concat(frames);+ for (const chunkSize of [3, 1]) {+ const decoder = new PrivateFrameDecoder(isTestHeader);+ const decoded: PrivateFrame<TestHeader>[] = [];+ for (let offset = 0; offset < combined.length; offset += chunkSize) {+ decoded.push(...decoder.push(combined.subarray(offset, Math.min(offset + chunkSize, combined.length))));+ }+ decoder.finish();+ expect(decoded).toEqual(expected);+ }});it("rejects invalid lengths, JSON, and routing headers", () => {@@ -66,6 +74,77 @@ describe("private worker framing", () => {expect(() => decoder.finish()).toThrow("incomplete bytes");});+ it("tracks unread bytes while frames span chunk boundaries", () => {+ const frame = encodePrivateFrame({ type: "event" }, Buffer.from("a".repeat(100)));+ const decoder = new PrivateFrameDecoder(isTestHeader);+ expect(decoder.bufferedBytes).toBe(0);+ decoder.push(frame.subarray(0, 20));+ expect(decoder.bufferedBytes).toBe(20);+ decoder.push(frame.subarray(20, 50));+ expect(decoder.bufferedBytes).toBe(50);+ // A complete frame is decoded in one push; nothing stays buffered.+ const decoded = decoder.push(frame.subarray(50));+ expect(decoded).toEqual([{ header: { type: "event" }, payload: Buffer.from("a".repeat(100)) }]);+ expect(decoder.bufferedBytes).toBe(0);+ decoder.finish();+ });++ it("decodes large frames delivered in small chunks within per-size budgets", () => {+ // A decoder that re-copies its whole pending buffer per socket read is quadratic+ // in the chunk count; both vectors finish far under budget on the linear path.+ for (const [payloadBytes, chunkBytes, budgetMs] of [+ [8 * 1024 * 1024, 4096, 2000],+ [64 * 1024 * 1024, 256, 2500],+ ] as const) {+ const frame = encodePrivateFrame({ type: "event", requestId: "large" }, Buffer.alloc(payloadBytes, 7));+ const decoder = new PrivateFrameDecoder(isTestHeader);++ const started = performance.now();+ const decoded: PrivateFrame<TestHeader>[] = [];+ for (let offset = 0; offset < frame.length; offset += chunkBytes) {+ decoded.push(...decoder.push(frame.subarray(offset, Math.min(offset + chunkBytes, frame.length))));+ }+ const elapsed = performance.now() - started;++ expect(decoded).toHaveLength(1);+ expect(decoded[0]?.header).toEqual({ type: "event", requestId: "large" });+ const payload = decoded[0]?.payload;+ expect(payload?.length).toBe(payloadBytes);+ expect(payload?.[0]).toBe(7);+ expect(payload?.[payloadBytes - 1]).toBe(7);+ expect(elapsed).toBeLessThan(budgetMs);+ decoder.finish();+ }+ });++ it("consumes a frame split across many chunks without calling Array.prototype.shift", () => {+ // The head cursor must skip spent chunks instead of shifting each one off+ // the front, which is quadratic in the chunk count.+ const frame = encodePrivateFrame({ type: "event", requestId: "large" }, Buffer.alloc(8 * 1024 * 1024, 3));+ const decoder = new PrivateFrameDecoder(isTestHeader);+ const decoded: PrivateFrame<TestHeader>[] = [];++ const originalShift = Array.prototype.shift;+ let shiftCalls = 0;+ Array.prototype.shift = function (this: unknown[]): unknown {+ shiftCalls += 1;+ return originalShift.apply(this);+ };+ try {+ for (let offset = 0; offset < frame.length; offset += 1024) {+ decoded.push(...decoder.push(frame.subarray(offset, Math.min(offset + 1024, frame.length))));+ }+ } finally {+ Array.prototype.shift = originalShift;+ }++ expect(shiftCalls).toBe(0);+ expect(decoded).toHaveLength(1);+ expect(decoded[0]?.header).toEqual({ type: "event", requestId: "large" });+ expect(decoded[0]?.payload.length).toBe(8 * 1024 * 1024);+ decoder.finish();+ });+it("sends frames through a duplex channel without interpreting payload bytes", async () => {const stream = new PassThrough();const channel = new PrivateFramedChannel(stream, isTestHeader);diff --git a/scripts/benchmarks/README.md b/scripts/benchmarks/README.mdindex fc36b6ffef..a637194637 100644--- a/scripts/benchmarks/README.md+++ b/scripts/benchmarks/README.md@@ -60,12 +60,13 @@ and other missing tools. Their setup time and disk usage are outside the install- **Installed footprint:** apparent bytes added after first use in the first fresh home, including stock Python,runtime, and tool assets; excluding download caches, session history, and logs. Shared systemdependencies supplied by the base image and the fixture repository are excluded.+- **Private frame decode:** one 32 MB frame with a snapshot-chunk routing header pushed through `PrivateFrameDecoder` in 8 KiB socket chunks, seconds per complete decode. This is the wire shape of the multi-MB snapshot and response frames on the daemon-worker and peer transport channels.- **Idle memory:** summed RSS across the benchmark user's entire process tree after input readinessand a one-second settle. Raw results include each process and PSS when Linux permits reading it.The controller, PTY harness, artifact server, and build user are excluded. RSS can double-countshared pages.-Startup and memory use 10 trials per revision. Installation uses three; sizes are measured once. UI interactions use 3 trials per revision (`ui_trials`).+Startup and memory use 10 trials per revision. Installation uses three; sizes are measured once. UI interactions use 3 trials per revision (`ui_trials`). Transport probes also use 10 trials: they run compiled Node code from the prepared source build, not the installed home, so they measure the source revision even when the installation step failed.Compiled revisions provision pinned Bun tooling and build their Linux x64 archive during untimedsetup. The installer selects its normal default from the available artifacts; the harness does notforce Node or compiled mode. Loopback downloads use the installer's explicit test exception, whilediff --git a/scripts/benchmarks/controller.py b/scripts/benchmarks/controller.pyindex 8077175281..b268c4daf0 100644--- a/scripts/benchmarks/controller.py+++ b/scripts/benchmarks/controller.py@@ -26,7 +26,16 @@ from schema import (REMOTE = "/opt/prime-benchmark"OWNER_LABEL = "prime-agent-benchmarks-v1"-FILES = ("schema.py", "terminal.py", "kernel.py", "worker.py", "ui.py", "pyproject.toml", "uv.lock")+FILES = (+ "schema.py",+ "terminal.py",+ "kernel.py",+ "worker.py",+ "ui.py",+ "frame-decode-bench.mjs",+ "pyproject.toml",+ "uv.lock",+)def elapsed_seconds(start: datetime) -> float:@@ -279,6 +288,7 @@ class Controller:("install", self.report.config.install_trials),("measure", self.report.config.trials),("runtime", self.report.config.trials),+ ("transport", self.report.config.trials),("ui", self.report.config.ui_trials),):blocked: set[str] = set()diff --git a/scripts/benchmarks/frame-decode-bench.mjs b/scripts/benchmarks/frame-decode-bench.mjsnew file mode 100644index 0000000000..ccaf7b65d6--- /dev/null+++ b/scripts/benchmarks/frame-decode-bench.mjs@@ -0,0 +1,96 @@+/**+ * Private-frame decode benchmark: one multi-MB frame with a snapshot-chunk+ * routing header, delivered in small socket-size chunks through+ * PrivateFrameDecoder - the wire shape of the multi-MB snapshot and response+ * frames on the daemon-worker and peer transport channels, where chunked+ * delivery made the old per-read concatenation quadratic.+ *+ * Runs against the prepared source build, so the same trusted harness measures+ * main and the PR head. Prints one RESULT line with the decode seconds; the+ * benchmark worker records it as the frame_decode metric.+ *+ * Usage: node frame-decode-bench.mjs --dist <packages/coding-agent dist>+ */+import { performance } from "node:perf_hooks";+import { join } from "node:path";+import { pathToFileURL } from "node:url";++const PAYLOAD_BYTES = 32 * 1024 * 1024;+const WARMUP_BYTES = 256 * 1024;+/** Matches the 8 KiB socket reads observed on the profiled channels. */+const CHUNK_BYTES = 8 * 1024;++function parseArgs(argv) {+ const args = { dist: "" };+ for (let index = 2; index < argv.length; index++) {+ if (argv[index] === "--dist" && argv[index + 1]) {+ args.dist = argv[index + 1];+ index++;+ }+ }+ if (!args.dist) {+ throw new Error("usage: node frame-decode-bench.mjs --dist <coding-agent dist>");+ }+ return args;+}++async function main() {+ const { dist } = parseArgs(process.argv);+ const { encodePrivateFrame, PrivateFrameDecoder } = await import(+ pathToFileURL(join(dist, "modes", "session-worker", "private-framing.js")).href+ );+ const validateHeader = (value) =>+ typeof value === "object" &&+ value !== null &&+ value.kind === "outbound" &&+ typeof value.outboundType === "string" &&+ value.snapshotPurpose === "replacement";+ const header = {+ kind: "outbound",+ outboundType: "session_snapshot_chunk",+ requestId: "bench-frame-decode",+ snapshotId: "bench-frame-decode",+ snapshotPurpose: "replacement",+ };+ const decoder = new PrivateFrameDecoder(validateHeader);++ // Frames are built before the timer: the metric is the chunked decode loop+ // only, not the encoding's allocations and copies.+ const buildFrame = (payloadBytes) => encodePrivateFrame(header, Buffer.alloc(payloadBytes, 7));++ const decode = (frame, payloadBytes) => {+ const decoded = [];+ for (let offset = 0; offset < frame.length; offset += CHUNK_BYTES) {+ decoded.push(...decoder.push(frame.subarray(offset, Math.min(offset + CHUNK_BYTES, frame.length))));+ }+ if (decoded.length !== 1) {+ throw new Error(`expected one decoded frame, got ${decoded.length}`);+ }+ const payload = decoded[0].payload;+ if (+ payload.length !== payloadBytes ||+ payload[payload.length - 1] !== 7 ||+ !validateHeader(decoded[0].header) ||+ decoded[0].header.snapshotId !== header.snapshotId ||+ decoder.bufferedBytes !== 0+ ) {+ throw new Error("decoded frame does not match the encoded frame");+ }+ return decoded[0];+ };++ decode(buildFrame(WARMUP_BYTES), WARMUP_BYTES);+ const frame = buildFrame(PAYLOAD_BYTES);+ const started = performance.now();+ decode(frame, PAYLOAD_BYTES);+ const seconds = (performance.now() - started) / 1000;+ decoder.finish();+ process.stdout.write(+ `RESULT ${JSON.stringify({ value: seconds, payload_bytes: PAYLOAD_BYTES, chunk_bytes: CHUNK_BYTES })}\n`+ );+}++main().catch((error) => {+ process.stderr.write(`frame-decode-bench failed: ${error?.stack ?? error}\n`);+ process.exit(1);+});diff --git a/scripts/benchmarks/report.py b/scripts/benchmarks/report.pyindex 40fd9eff5e..3eb9ce0ad2 100644--- a/scripts/benchmarks/report.py+++ b/scripts/benchmarks/report.py@@ -107,6 +107,16 @@ RUNTIME_METRICS = (PERFORMANCE_NOISE_FLOOR,),)+TRANSPORT_METRICS = (+ Definition(+ "frame_decode",+ "Private frame decode, 32 MiB in 8 KiB chunks",+ 1000,+ "ms",+ 0.02,+ PERFORMANCE_NOISE_FLOOR,+ ),+)UI_METRICS = (Definition("resume_large", "Resume large session (cold)", 1000, "ms", 0.1, PERFORMANCE_NOISE_FLOOR),Definition("resume_large_cpu", "CPU, resume large session", 1000, "ms", 0.05, PERFORMANCE_NOISE_FLOOR),@@ -252,7 +262,7 @@ def comparison(def comparisons(report: Report) -> dict[Metric, Comparison]:results = {}- for definition in (*METRICS, *RUNTIME_METRICS, *UI_METRICS):+ for definition in (*METRICS, *RUNTIME_METRICS, *TRANSPORT_METRICS, *UI_METRICS):expected = report.config.install_trials if definition.key == "install" else report.config.trialsif definition.key in ("bundle", "disk"):expected = 1@@ -336,7 +346,7 @@ def render(report: Report) -> str:"| --- | ---: | ---: | ---: |",])- for definition in (*METRICS, *RUNTIME_METRICS, *UI_METRICS):+ for definition in (*METRICS, *RUNTIME_METRICS, *TRANSPORT_METRICS, *UI_METRICS):if definition == RUNTIME_METRICS[0]:lines.extend([@@ -347,6 +357,16 @@ def render(report: Report) -> str:"| --- | ---: | ---: | ---: |",])+ if definition == TRANSPORT_METRICS[0]:+ lines.extend(+ [+ "",+ "**Session transport**",+ "",+ "| Metric | Main | This PR | Change |",+ "| --- | ---: | ---: | ---: |",+ ]+ )if definition == UI_METRICS[0]:lines.extend([@@ -393,6 +413,9 @@ def render(report: Report) -> str:"State fixture: a 10,000-row × 8-column integer DataFrame and a 10,000-integer list.","Restore runs in a fresh kernel, including pandas imports; kernel startup is excluded.","Kernel RSS covers the isolated Python process; loaded RSS follows the pandas workload.",+ "Transport benches run node against the prepared source build, outside the installed home.",+ "Frame decode times one 32 MiB private frame, snapshot-chunk header, pushed in",+ "8 KiB chunks; the wire shape of multi-MB frames on the daemon-worker channels.","UI trials use a fresh fixture set: 194 top-level sessions including one ~40 MB transcript,","40 ledger fan-out children, and a 6-deep subagent chain (~46 spawn edges).","Large fixtures hold 1,999 complete triples (~5 MB JSONL); medium 119; subagents 399 each.",@@ -416,7 +439,7 @@ def render(report: Report) -> str:"| --- | ---: | ---: | ---: | ---: |",])- for definition in (*METRICS, *RUNTIME_METRICS, *UI_METRICS):+ for definition in (*METRICS, *RUNTIME_METRICS, *TRANSPORT_METRICS, *UI_METRICS):left = report.main.metrics.get(definition.key, [])right = report.pr_head.metrics.get(definition.key, [])lines.append(diff --git a/scripts/benchmarks/schema.py b/scripts/benchmarks/schema.pyindex a938b38bcb..2cd3144e36 100644--- a/scripts/benchmarks/schema.py+++ b/scripts/benchmarks/schema.py@@ -27,6 +27,7 @@ Metric = Literal["restore","kernel_rss","loaded_rss",+ "frame_decode","resume_large","resume_large_cpu","switch_large",@@ -235,6 +236,7 @@ PHASE_METRICS: dict[str, tuple[Metric, ...]] = {"kernel_rss","loaded_rss",),+ "transport": ("frame_decode",),"ui": ("resume_large","resume_large_cpu",diff --git a/scripts/benchmarks/tests/test_benchmarks.py b/scripts/benchmarks/tests/test_benchmarks.pyindex b3ad28f187..0964c905f8 100644--- a/scripts/benchmarks/tests/test_benchmarks.py+++ b/scripts/benchmarks/tests/test_benchmarks.py@@ -18,7 +18,7 @@ from pydantic import ValidationErrorfrom cli import completed_report, main, validate_completion, workflow_sourcefrom controller import Canceled, Controller, cleanup, labelsfrom github import TITLE, GitHub-from report import MARKER, METRICS, RUNTIME_METRICS, UI_METRICS, comparison, render+from report import MARKER, METRICS, RUNTIME_METRICS, TRANSPORT_METRICS, UI_METRICS, comparison, renderfrom schema import (UI_METRIC_KEYS,Config,@@ -177,7 +177,7 @@ class ReportTests(unittest.TestCase):def test_compact_tables_summarize_outcomes_without_treating_missing_samples_as_unchanged(self):report = fixture()report.status = "partial"- for definition in (*METRICS, *RUNTIME_METRICS):+ for definition in (*METRICS, *RUNTIME_METRICS, *TRANSPORT_METRICS):count = 1 if definition.key in ("bundle", "disk") else (3 if definition.key == "install" else 10)report.main.metrics[definition.key] = observations(*([2.0] * count))report.pr_head.metrics[definition.key] = observations(*([2.0] * count))@@ -187,12 +187,12 @@ class ReportTests(unittest.TestCase):report.pr_head.metrics["bundle"] = []text = render(report)self.assertIn(- "**Overall: 1 regressed · 1 improved · 13 no clear change · 1 incomplete · "+ "**Overall: 1 regressed · 1 improved · 14 no clear change · 1 incomplete · "f"{len(UI_METRICS) + 1} unavailable.**",text,)tables = text.split("<details>")[0]- self.assertEqual(tables.count("| Metric | Main | This PR | Change |"), 3)+ self.assertEqual(tables.count("| Metric | Main | This PR | Change |"), 4)for row in tables.splitlines():if row.startswith("|"):self.assertEqual(len(row.strip("|").split("|")), 4)@@ -226,7 +226,7 @@ class ReportTests(unittest.TestCase):self.assertIn("0/0", text)self.assertNotIn("TTFT", text)self.assertNotIn("Pinference", text)- for definition in RUNTIME_METRICS:+ for definition in (*RUNTIME_METRICS, *TRANSPORT_METRICS):self.assertIn(definition.title, text)def test_schema_rejects_nonfinite_negative_duplicate_and_wrong_revision(self):@@ -713,7 +713,7 @@ class MeasurementTests(unittest.TestCase):def test_completion_requires_every_metric(self):side = Side(sha=SHA)- for definition in (*METRICS, *RUNTIME_METRICS, *UI_METRICS):+ for definition in (*METRICS, *RUNTIME_METRICS, *TRANSPORT_METRICS, *UI_METRICS):count = (1if definition.key in ("bundle", "disk")@@ -721,7 +721,7 @@ class MeasurementTests(unittest.TestCase):)side.metrics[definition.key] = observations(*([1] * count))self.assertTrue(side_complete(side, 10, 3, 2))- for definition in (*METRICS, *RUNTIME_METRICS, *UI_METRICS):+ for definition in (*METRICS, *RUNTIME_METRICS, *TRANSPORT_METRICS, *UI_METRICS):incomplete = side.model_copy(deep=True)incomplete.metrics[definition.key][-1] = Observation(trial=0, error="failed")self.assertFalse(side_complete(incomplete, 10, 3, 2))diff --git a/scripts/benchmarks/tests/test_transport.py b/scripts/benchmarks/tests/test_transport.pynew file mode 100644index 0000000000..8b0a9eca28--- /dev/null+++ b/scripts/benchmarks/tests/test_transport.py@@ -0,0 +1,84 @@+from __future__ import annotations++import signal+import subprocess+import tempfile+import unittest+from pathlib import Path+from types import SimpleNamespace+from unittest.mock import Mock, call, patch++from schema import PHASE_METRICS, ROOT, Observation, Side+from worker import TRANSPORT_BENCHES, stop_transport_processes, transport, transport_value+++class TransportTests(unittest.TestCase):+ def test_records_each_bench_result_and_failure_with_explicit_timeouts(self):+ side = Side(sha="a" * 40)+ script, metric, timeout = TRANSPORT_BENCHES[0]+ failure = subprocess.CalledProcessError(1, "node", output="frame-decode-bench failed: exploded\n")+ run_as = Mock(+ side_effect=['RESULT {"value": 1.5}\n', failure, subprocess.TimeoutExpired("node", timeout)]+ )+ with tempfile.TemporaryDirectory() as directory:+ with (+ patch("worker.run_as", run_as),+ patch("worker.RESULTS", Path(directory)),+ patch("worker.stop_transport_processes"),+ ):+ for trial in range(3):+ transport(side, trial)+ saved = sorted(path.name for path in Path(directory).glob("*.output"))+ self.assertEqual(+ run_as.call_args,+ call(+ "builder",+ [+ "node",+ str(ROOT / script),+ "--dist",+ str(Path("/home/builder/source/packages/coding-agent/dist")),+ ],+ Path("/home/builder/source"),+ timeout=timeout,+ merge_output=True,+ ),+ )+ self.assertEqual(side.metrics[metric][0], Observation(trial=0, value=1.5))+ self.assertIn("exploded", side.metrics[metric][1].error)+ self.assertIn("timed out", side.metrics[metric][2].error)+ self.assertEqual(saved, [f"{metric}-0.output", f"{metric}-1.output"])+ self.assertEqual({name for _, name, _ in TRANSPORT_BENCHES}, set(PHASE_METRICS["transport"]))+ self.assertTrue(all(limit > 0 for _, _, limit in TRANSPORT_BENCHES))++ def test_transport_value_reads_the_result_line_and_rejects_malformed_output(self):+ self.assertEqual(transport_value('noise\nRESULT {"value": 12}'), 12.0)+ for output in ("", 'RESULT {"value": "12"}', 'RESULT {"value": true}', "RESULT [1, 2, 3]"):+ with self.subTest(output=output):+ self.assertRaises((RuntimeError, ValueError), transport_value, output)+++class StopTransportProcessesTests(unittest.TestCase):+ def test_sweeps_only_builder_node_processes_and_raises_when_strays_survive(self):+ node = SimpleNamespace(name="node", pid=4242)+ kills = []++ def run_sweep(memory):+ os = Mock()+ os.kill = lambda pid, sig: kills.append((pid, sig))+ with (+ patch("worker.pwd", Mock(getpwnam=Mock(return_value=SimpleNamespace(pw_uid=1500)))),+ patch("worker.memory", memory),+ patch("worker.os", os),+ ):+ stop_transport_processes()++ sightings = [[node, SimpleNamespace(name="python3", pid=4243)], []]+ run_sweep(Mock(side_effect=lambda uid: sightings.pop(0)))+ self.assertEqual(kills, [(4242, signal.SIGTERM)]) # the python3 artifact server stays up+ with self.assertRaisesRegex(RuntimeError, "node processes remain"):+ run_sweep(Mock(return_value=[node]))+++if __name__ == "__main__":+ unittest.main()diff --git a/scripts/benchmarks/worker.py b/scripts/benchmarks/worker.pyindex 93f60b23b1..145ffed215 100644--- a/scripts/benchmarks/worker.py+++ b/scripts/benchmarks/worker.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport argparseimport hashlibimport json+import mathimport osimport pwdimport shutil@@ -41,6 +42,8 @@ RESULTS = ROOT / "results"VERSION = "0.0.0-benchmark"ORIGIN = "http://127.0.0.1:18741"BUN_VERSION = "1.4.0"+# Node transport benches: harness script, recorded metric, and the run timeout.+TRANSPORT_BENCHES: tuple[tuple[str, Metric, int], ...] = (("frame-decode-bench.mjs", "frame_decode", 180),)def clean_error(error: Exception) -> str:@@ -545,9 +548,75 @@ def runtime(side: Side, trial: int) -> None:shutil.rmtree(state)+def transport_value(output: str) -> float:+ """Read the single RESULT line a transport harness prints."""+ for line in output.splitlines():+ if line.startswith("RESULT "):+ result = json.loads(line.removeprefix("RESULT "))+ value = result.get("value") if isinstance(result, dict) else None+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ raise RuntimeError("Transport benchmark result is not a number")+ if not math.isfinite(value) or value < 0:+ raise RuntimeError("Transport benchmark result is not a non-negative number")+ return float(value)+ raise RuntimeError("Transport benchmark printed no RESULT line")+++def stop_transport_processes() -> None:+ """Stop leftover builder-owned node processes; the python3 artifact server stays up."""+ uid = pwd.getpwnam("builder").pw_uid+ for sig, delay in ((signal.SIGTERM, 3), (signal.SIGKILL, 2)):+ agents = [process.pid for process in memory(uid) if process.name == "node"]+ if not agents:+ return+ for pid in agents:+ try:+ os.kill(pid, sig)+ except ProcessLookupError:+ pass+ deadline = time.monotonic() + delay+ while time.monotonic() < deadline:+ if not any(process.name == "node" for process in memory(uid)):+ return+ time.sleep(0.05)+ if any(process.name == "node" for process in memory(uid)):+ raise RuntimeError("Builder node processes remain after the transport phase")+++def transport(side: Side, trial: int) -> None:+ """Run the node transport benches against the prepared source build."""+ dist = SOURCE / "packages/coding-agent/dist"+ try:+ for script, metric, timeout in TRANSPORT_BENCHES:+ try:+ output = run_as(+ "builder",+ ["node", str(ROOT / script), "--dist", str(dist)],+ SOURCE,+ timeout=timeout,+ merge_output=True,+ )+ (RESULTS / f"{metric}-{trial}.output").write_text(output)+ record(side, metric, trial, transport_value(output))+ except subprocess.CalledProcessError as error:+ # Keep the harness's own failure text available in the saved output+ # and in the recorded error, not only in the sandbox log.+ failure_output = error.output if isinstance(error.output, str) else ""+ (RESULTS / f"{metric}-{trial}.output").write_text(failure_output or clean_error(error))+ detail = clean_error(error)+ first_line = failure_output.strip().splitlines()[0] if failure_output.strip() else ""+ if first_line:+ detail = f"{detail}: {first_line}"[:500]+ record(side, metric, trial, error=detail)+ except Exception as error:+ record(side, metric, trial, error=clean_error(error))+ finally:+ stop_transport_processes()++def main() -> None:parser = argparse.ArgumentParser()- parser.add_argument("phase", choices=("prepare", "install", "measure", "runtime", "ui"))+ parser.add_argument("phase", choices=("prepare", "install", "measure", "runtime", "transport", "ui"))parser.add_argument("--trial", type=int, default=0)args = parser.parse_args()RESULTS.mkdir(exist_ok=True)@@ -569,6 +638,8 @@ def main() -> None:install(request, result.side, args.trial)elif args.phase == "measure":measure(request, result.side, args.trial)+ elif args.phase == "transport":+ transport(result.side, args.trial)elif args.phase == "ui":from ui import ui_measure