Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

perf CI: pa-perf corpora + pty bench + the perf workflow (#3607)

Sebastian Müller41946110baad
diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml
new file mode 100644
index 0000000000..522f514617
--- /dev/null
+++ b/.github/workflows/perf.yml
@@ -0,0 +1,185 @@
+# The performance-test lane: a nightly + opt-in-per-PR benchmark of the
+# interactive surfaces (cold/warm start, agents view, open/switch session,
+# @ completion) over deterministic fake stores, driven through the REAL
+# release binary under a pty by `crates/pa-perf`.
+#
+# Triggers:
+# schedule — nightly on main (the label path is how a branch proves
+# itself; workflow_dispatch/schedule only fire once this
+# file is on main).
+# workflow_dispatch — manual.
+# pull_request [labeled, synchronize] — gated on the `perf-test` label:
+# the label IS the opt-in (deliberately including draft
+# PRs — labeling a draft is how this lane proves itself
+# before review), and a synchronize re-runs only while the
+# label stays on. Never runs for unlabeled PRs: a 4-way
+# release-build matrix on every PR would bill the org
+# runners hard.
+#
+# Job graph:
+# build — release prime-agent + pa-perf once, uploaded as one artifact
+# (every bench leg downloads the same bytes).
+# bench — one matrix leg per corpus profile: generate the corpus
+# (cold + warm), bootstrap the kernel venv (the same prep
+# ci.yml's kernel e2e shards use), run every scenario 10x,
+# upload the result JSONs.
+# report — aggregate every result against crates/pa-perf/baseline.json:
+# the $GITHUB_STEP_SUMMARY table (scenario, profile, median,
+# p95, baseline, delta) always prints deltas and fails ONLY on a
+# >2x median regression; scenarios marked report-only in the
+# baseline (spread >20% across 3 CI runs) never fail the gate.
+#
+# Cache policy: Swatinem's cache with a dedicated shared-key, saved ONLY on
+# main-anchored runs (schedule, or a dispatch on main). PR legs restore and
+# never save — PR-ref cache entries are the eviction pressure on the
+# release-build caches, and the repo sits at the 10 GB Actions-cache quota.
+name: perf
+
+on:
+ schedule:
+ - cron: "30 3 * * *"
+ workflow_dispatch:
+ pull_request:
+ types: [labeled, synchronize]
+ branches: [main]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: perf-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ CARGO_TERM_COLOR: always
+ CARGO_INCREMENTAL: 0
+
+jobs:
+ build:
+ name: build release binaries
+ runs-on: ubuntu-24.04
+ timeout-minutes: 45
+ if: >-
+ github.event_name == 'schedule' ||
+ github.event_name == 'workflow_dispatch' ||
+ contains(github.event.pull_request.labels.*.name, 'perf-test')
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - uses: dtolnay/rust-toolchain@ce678459e9fc7500d337468f904b95f1b5c10b5e # 1.98.1 action pin; the SHA selects the toolchain
+ - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
+ with:
+ # One entry per main-anchored run: PR legs restore, never save
+ # (the save-if gate below), so this lane cannot evict the
+ # release-build caches.
+ shared-key: perf-build
+ cache-on-failure: true
+ save-if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') }}
+ - name: Build (release, locked)
+ run: cargo build --release --locked -p pa-cli --bin prime-agent -p pa-perf --bin pa-perf
+ - name: Upload the binaries
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: perf-bins
+ path: |
+ target/release/prime-agent
+ target/release/pa-perf
+ if-no-files-found: error
+
+ bench:
+ name: bench (${{ matrix.profile }})
+ needs: build
+ runs-on: ubuntu-24.04
+ timeout-minutes: 90
+ strategy:
+ fail-fast: false
+ matrix:
+ profile: [small, medium, large, xl]
+ steps:
+ # The checkout provides the packaged kernel runtime (prime-agent-runtime/)
+ # the release binary resolves through PI_PACKAGE_DIR, plus skills/.
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+ with:
+ name: perf-bins
+ path: bins
+ - name: Stage the binaries
+ run: |
+ set -euo pipefail
+ install -Dm755 bins/prime-agent "$HOME/.local/bin/prime-agent"
+ install -Dm755 bins/pa-perf "$HOME/.local/bin/pa-perf"
+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+ - name: Install uv (the kernel venv bootstrap below uses it)
+ run: |
+ curl -LsSf https://astral.sh/uv/install.sh | sh
+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+ # The same prep ci.yml's kernel-e2e shards run: the venv must be ready
+ # before the first session prewarm, or the first bench run pays the
+ # bootstrap inside its (discarded) warm-up.
+ - name: Bootstrap the kernel venv
+ env:
+ PI_PACKAGE_DIR: ${{ github.workspace }}
+ run: prime-agent --prime-agent-bootstrap
+ # One driver call: pa-perf run generates the cold and warm corpora,
+ # routes every scenario to the corpus state it needs, and writes one
+ # result JSON per scenario into <out>/results/.
+ - name: Run the scenarios
+ env:
+ PI_PACKAGE_DIR: ${{ github.workspace }}
+ run: |
+ set -euo pipefail
+ pa-perf run \
+ --binary "$HOME/.local/bin/prime-agent" \
+ --profile ${{ matrix.profile }} \
+ --out "$RUNNER_TEMP/perf" \
+ --runs 10
+ - name: Upload the results
+ if: always()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: perf-results-${{ matrix.profile }}
+ path: ${{ runner.temp }}/perf/results/*.json
+ if-no-files-found: error
+ # The raw pty streams of the launches that did not complete: the CI
+ # counterpart of the empty-tail failures (nothing debuggable in a
+ # timeout message alone).
+ - name: Upload the failure pty dumps
+ if: failure()
+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ with:
+ name: perf-failure-streams-${{ matrix.profile }}
+ path: |
+ ${{ runner.temp }}/perf/corpus-cold/failed-streams/*
+ ${{ runner.temp }}/perf/corpus-warm/failed-streams/*
+ if-no-files-found: ignore
+
+ report:
+ name: report + regression gate
+ needs: bench
+ runs-on: ubuntu-24.04
+ timeout-minutes: 10
+ steps:
+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+ with:
+ persist-credentials: false
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+ with:
+ name: perf-bins
+ path: bins
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+ with:
+ pattern: perf-results-*
+ merge-multiple: true
+ path: results
+ - name: Compare against the baseline
+ run: |
+ set -euo pipefail
+ install -Dm755 bins/pa-perf "$HOME/.local/bin/pa-perf"
+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+ pa-perf report \
+ --baseline crates/pa-perf/baseline.json \
+ --markdown "$GITHUB_STEP_SUMMARY" \
+ --results results/*.json
diff --git a/AGENTS.md b/AGENTS.md
index 90efbf8717..dac9a9543a 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -196,6 +196,7 @@ The table summarizes each crate's current direct workspace dependencies:
| `pa-daemon` | Supervisor, workers, wire protocol | `pa-types`, `pa-core`, `pa-agent`, `pa-ai`, `pa-telemetry`, `pa-models` |
| `pa-tui` | Terminal UI, daemon-wire client | `pa-types` |
| `pa-cli` | `prime-agent` binary, composition root | `pa-types`, `pa-core`, `pa-ai`, `pa-agent`, `pa-daemon`, `pa-tui`, `pa-telemetry` |
+| `pa-perf` | Performance corpora + pty bench harness (dev/CI tooling, never shipped) | `pa-types`, `pa-core`, `pa-daemon` |
Layering rules:
diff --git a/Cargo.lock b/Cargo.lock
index 2d73147bc7..5ab287b87d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -79,6 +79,12 @@ version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
+[[package]]
+name = "arrayvec"
+version = "0.7.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
+
[[package]]
name = "asn1-rs"
version = "0.7.2"
@@ -1596,6 +1602,22 @@ dependencies = [
"tracing",
]
+[[package]]
+name = "pa-perf"
+version = "0.10.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "nix",
+ "pa-core",
+ "pa-daemon",
+ "pa-types",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "vt100",
+]
+
[[package]]
name = "pa-telemetry"
version = "0.10.0"
@@ -2948,6 +2970,39 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
+[[package]]
+name = "vt100"
+version = "0.15.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de"
+dependencies = [
+ "itoa",
+ "log",
+ "unicode-width 0.1.14",
+ "vte",
+]
+
+[[package]]
+name = "vte"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197"
+dependencies = [
+ "arrayvec",
+ "utf8parse",
+ "vte_generate_state_changes",
+]
+
+[[package]]
+name = "vte_generate_state_changes"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+]
+
[[package]]
name = "walkdir"
version = "2.5.0"
diff --git a/Cargo.toml b/Cargo.toml
index 38e1485b47..a340943b0e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,6 +10,7 @@ members = [
"crates/pa-daemon",
"crates/pa-tui",
"crates/pa-cli",
+ "crates/pa-perf",
]
# Crossterm 0.28.1's kitty support check holds the process-wide event reader
diff --git a/Makefile b/Makefile
index 00f85e1c9f..c178551f47 100644
--- a/Makefile
+++ b/Makefile
@@ -165,6 +165,18 @@ package:
catalog-assets-gates:
python3 scripts/release/test_catalog_assets.py
+# Local one-command perf run (crates/pa-perf/README.md documents the
+# scenario set): build the release binaries, generate the small corpus,
+# bench every scenario 10x, print the table. Run it on a quiet machine —
+# the numbers are wall-clock latencies of the interactive surfaces.
+perf:
+ cargo build --release --locked -p pa-cli --bin prime-agent -p pa-perf --bin pa-perf
+ ./target/release/pa-perf run --binary target/release/prime-agent \
+ --profile small --out target/perf --runs 10
+ ./target/release/pa-perf report --baseline crates/pa-perf/baseline.json \
+ --results target/perf/results/*.json
+
+
# The changelog fold's contract battery (RELEASE-FLOW-PROPOSAL.md §8): the
# fold is the release-PR's changelog half, and the pathspec security case is
# first - a fragment NAME is data, never a git pathspec, so no root-level
@@ -187,7 +199,7 @@ shard-gates:
python3 scripts/test_ci_test_shard.py
python3 scripts/test_ci_pr_crates.py
-.PHONY: check deny windows-cross actionlint glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates restamp-gates shard-gates
+.PHONY: check deny windows-cross actionlint glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates restamp-gates shard-gates perf
# Offline packaging contract tests for updaters shipped since Prime Agent 0.6.
legacy-upgrade-gates:
diff --git a/crates/pa-perf/Cargo.toml b/crates/pa-perf/Cargo.toml
new file mode 100644
index 0000000000..7f1ed12b61
--- /dev/null
+++ b/crates/pa-perf/Cargo.toml
@@ -0,0 +1,41 @@
+# pa-perf: the performance corpus generator + bench harness (dev/CI tooling).
+# Top of the dependency graph: nothing depends on it, and it never ships —
+# the release lanes build only `-p pa-cli --bin prime-agent`.
+[package]
+name = "pa-perf"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+publish = false
+
+[dependencies]
+anyhow = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+# CLI parsing: the workspace already resolves clap for pa-tui.
+clap = { version = "4", features = ["derive"] }
+# The screen model for content needles the raw byte stream cannot see (the
+# agents view's `Inactive (N)` heading repaints as cell diffs on big
+# corpora). MIT, no_std-free, never shipped — the only new third-party dep
+# this crate carries (see README + the PR body).
+vt100 = "0.15"
+# pty + poll for the bench harness: the same nix version the workspace
+# already resolves for pa-core/pa-cli.
+[target.'cfg(unix)'.dependencies]
+nix = { version = "0.29", features = ["term", "signal", "process", "fs", "poll"] }
+
+[dependencies.pa-types]
+path = "../pa-types"
+
+[dependencies.pa-core]
+path = "../pa-core"
+
+[dependencies.pa-daemon]
+path = "../pa-daemon"
+
+[dev-dependencies]
+tempfile = "3"
+
+# The library's rows/entries ride the product's own serializers (pa-types
+# payloads inside pa-daemon's SessionEntry), so generated corpora exercise
+# the exact wire format the product reads.
diff --git a/crates/pa-perf/README.md b/crates/pa-perf/README.md
new file mode 100644
index 0000000000..5a46a4036e
--- /dev/null
+++ b/crates/pa-perf/README.md
@@ -0,0 +1,178 @@
+# pa-perf
+
+The prime-agent performance tooling: a deterministic corpus generator over
+the product's real on-disk layout, and a pty bench harness that drives the
+REAL release binary (`target/release/prime-agent`) headlessly. A dev/CI
+crate at the top of the workspace dependency graph — nothing depends on it,
+and it never ships (the release lanes build `-p pa-cli --bin prime-agent`
+only).
+
+## Scope and non-goals
+
+- Scope: faking realistic agent-dir state at scale (sessions, subagent
+ trees, artifacts, the spawn ledger, the cwd project tree), and measuring
+ the interactive surfaces (start, agents view, open/switch session,
+ @ completion) through the real binary.
+- Non-goals: benchmarks of provider/LLM paths (the harness stays offline),
+ and any product-code changes. The new `vt100` screen-model dependency
+ is described below; clap and nix were already in the tree.
+
+## The corpus generator
+
+```
+pa-perf corpus --profile medium --out DIR [--seed N] [knob overrides]
+```
+
+`DIR` must be absent or empty. A populated output is rejected before any writes,
+so stale sessions cannot enter the new corpus or its restore baseline. Choose a
+fresh directory for another standalone generation. The suite command can be run
+again in the same output: it replaces only its own corpus directories after
+verifying their manifests, and refuses unrelated populated directories.
+
+Writes the product's real layout under `DIR/`:
+
+```
+DIR/
+ corpus.json # manifest: counts, needle markers, the file-set baseline
+ project/ # the sessions' cwd + @-completion tree
+ agent/ # point PRIME_AGENT_CODING_AGENT_DIR here
+ settings.json # onboarding pre-answered, traces off
+ sessions/<uuid>.jsonl # current-format session files (flat)
+ sessions-archive/<uuid>.jsonl
+ rlm-ledger/<hash16>.jsonl # written through pa-daemon's RlmSpawnLedger
+ session-artifacts/... # kernel-state stubs, harness/, sub-<id>/ trees
+```
+
+Row payloads are serialized from the product's own types (pa-types message
+shapes inside pa-daemon's `SessionEntry` envelope and `session_header_line`
+header) — no hand-copied JSON. Entry ids are counter-based unique 8-hex: the
+product's own ids are uuid4 prefixes with a 100-try collision check, which
+collide around ~22K rows and silently break branch reconstruction.
+
+### Knobs
+
+`--profile small|medium|large|xl` picks the defaults; every knob overrides:
+
+| flag | meaning | small | medium | large | xl |
+|---|---|---|---|---|---|
+| `--sessions` | top-level sessions | 10 | 100 | 1,000 | 2,500 |
+| `--rows` | rows per session | 100 | 2,000 | 10,000 | 400 |
+| `--huge-rows` | rows of the huge marker session | 2,000 | 4,000 | 20,000 | 2,000 |
+| `--child-rows` | rows per subagent child | 40 | 100 | 150 | 80 |
+| `--fanout` | children per parent per level | 2 | 3 | 3 | 2 |
+| `--depth` | subagent tree depth | 2 | 2 | 3 | 2 |
+| `--tree-parents` | sessions growing a tree | 2 | 5 | 8 | 10 |
+| `--wide-children` | one parent's direct children (the operator's ~110-wide real parent) | 0 | 0 | 110 | 0 |
+| `--archive` | sessions in sessions-archive/ | 3 | 20 | 200 | 500 |
+| `--cwd-files` | files in the project tree | 200 | 2,000 | 4,000 | 2,000 |
+| `--artifacts` | per-session artifact partitions (on/off) | on | on | on | on |
+| `--cache` | cold/warm (see below) | cold | cold | cold | cold |
+| `--seed` | the deterministic seed | 7 | 7 | 7 | 7 |
+
+Total rows: small ~3.5K, medium ~250K, large ~10.1M, xl ~1.2M (medium is
+~210 MB on disk; large ~3.5 GB; xl ~1 GB, generated in ~5 s).
+
+### Determinism
+
+Same seed + same canonical path => a byte-identical tree, file mtimes
+included (roster order is mtime-desc). Two stamps are the product writer's
+own wall clock and are excluded from that contract: the ledger records'
+`at` fields, and the corpus-root path embedded in child headers/ledger
+records (`parentSession`/`child` are canonical paths).
+
+### Caches (cold vs warm)
+
+- `--cache warm` runs the product's own `pa_core::session::window::WindowedSessionStore::open`
+ over every openable session file, writing real `<id>.window-cache.json`
+ sidecars. On a real store warm window-cache sidecars are the common state
+ (the operator's live store: 71 of 111 sessions carry one).
+- The `<id>.info-cache.json` sidecar is written only at worker lease
+ release (pa-daemon, no public writer), so generated corpora carry none —
+ matching real stores (9 of 111). The `agents-view-warm` scenario measures
+ the warm scan via the daemon's in-process cache instead.
+- `pa-perf run` generates BOTH corpora (cold for the cold scenarios, warm
+ for warm-start/warm-ready) — every scenario runs on the state it needs.
+- The corpus settings disable BOTH archive rules (`sessionArchiveMaxAgeDays`
+ / `sessionArchiveMaxSessions` "off"): the product's defaults sweep the
+ live store down to 200 unarchived sessions (the operator's real store:
+ 196 live + 211 archived), so large/xl with archiving off are scaling
+ stress profiles, not typical stores, and the boot sweep itself is not
+ measured.
+
+## The bench harness
+
+```
+# the whole suite (CI + make perf): generates the corpora, routes every
+# scenario to the corpus state it needs, writes one results/<scenario>.json
+pa-perf run --binary target/release/prime-agent --profile medium --out DIR [--runs N]
+
+# one scenario over one corpus (investigation)
+pa-perf bench --binary target/release/prime-agent --corpus DIR --scenario S --runs 10 --json OUT
+```
+
+Scenarios (timed from a triggering action to an observable needle — literal
+paints as byte needles, cell-diffed content through a vt100 screen model;
+the liveness deadline only fails broken runs):
+
+| scenario | corpus | trigger -> needle |
+|---|---|---|
+| `cold-start` | cold | exec -> the chat first paint (the splash paints before any daemon work — the product's own paint contract) |
+| `cold-ready` | cold | the same launch, exec -> the tray's session-usage pill, the first attach-only content (daemon spawn + session create + attach included) |
+| `warm-start` | warm | exec -> first paint, daemon pre-started |
+| `warm-ready` | warm | the same launch, exec -> the attach pill |
+| `agents-view` | cold | left-arrow -> the completed `Inactive (N)` roster heading in the screen model |
+| `agents-view-warm` | cold | the same daemon's second populate (its in-process scan cache warm) |
+| `open-session` | cold | `--resume <huge session>` -> the huge session's tail marker |
+| `session-switch` | cold | enter on the uniquely-filtered switch-b row -> B's transcript marker |
+| `at-completion` | cold | the `@` keystroke -> the first completion menu row |
+
+Mechanics (the repo's pty-e2e discipline): the TUI child runs in its own
+session with NO controlling tty (`setsid` pre-exec; vendored crossterm
+falls back to the stdio pty), 120x36, kitty probe answered identically every
+run, `PRIME_AGENT_CODING_AGENT_DIR` + `--daemon-socket` isolate it, and the
+env is rebuilt from scratch (no ambient credentials leak in). Between runs
+the harness restores the corpus to the manifest's file set (the product's
+new session files, sidecars, logs, and leases are deleted) so every run
+measures identical state. A discarded warm-up run precedes the samples.
+
+Teardown signals only the harness's own child pids; the daemon stops by the
+shutdown protocol and the run FAILS if it keeps accepting — never a
+pid/name-based kill.
+
+## report + baseline
+
+```
+pa-perf report --baseline crates/pa-perf/baseline.json --results results/*.json [--markdown OUT]
+```
+
+Prints the aggregate table (scenario/profile, median, p95, baseline, delta)
+and exits nonzero ONLY when a non-report-only median exceeds twice the
+baseline. Noisy scenarios (>20% spread across 3 CI runs) are marked
+`report_only` in the baseline: their deltas always print, the gate never
+fails. `baseline.json` is committed from CI numbers, not laptop/sandbox
+ones.
+
+## Local run
+
+`make perf` builds the release binaries and runs the whole suite
+(`pa-perf run --profile small`) with the report table printed at the end.
+Run it on a quiet machine; the numbers are wall-clock latencies.
+
+## Dependencies
+
+pa-perf adds ONE new third-party dependency: `vt100` (MIT), the screen
+model behind the content needles (the agents view's heading counts repaint
+as terminal cell diffs, which the raw byte stream cannot see). The crate
+never ships — the release lanes build `-p pa-cli --bin prime-agent` only —
+and `make deny` stays clean.
+
+## CI
+
+The bench matrix runs all four profiles at 10 runs each; the xl
+(2500-session) agents-view populate measures ~2.7 s on an 8-core sandbox.
+
+`.github/workflows/perf.yml`: nightly schedule + workflow_dispatch +
+label-gated pull requests (`perf-test` label, [labeled, synchronize] —
+labeling is the explicit opt-in). One release build, a bench matrix over
+the four profiles, and a report job writing the `$GITHUB_STEP_SUMMARY`
+table with the >2x gate.
diff --git a/crates/pa-perf/baseline.json b/crates/pa-perf/baseline.json
new file mode 100644
index 0000000000..57d7333c7c
--- /dev/null
+++ b/crates/pa-perf/baseline.json
@@ -0,0 +1,185 @@
+{
+ "schema": 1,
+ "entries": {
+ "session-switch/large": {
+ "median_ms": 1034.4,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "agents-view/large": {
+ "median_ms": 13458.7,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "at-completion/large": {
+ "median_ms": 222.3,
+ "report_only": false,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "cold-start/large": {
+ "median_ms": 22.2,
+ "report_only": false,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "warm-ready/large": {
+ "median_ms": 130.5,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "agents-view-warm/large": {
+ "median_ms": 11333.6,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "cold-ready/large": {
+ "median_ms": 147.4,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "warm-start/large": {
+ "median_ms": 12.7,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "open-session/large": {
+ "median_ms": 2485.8,
+ "report_only": false,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "at-completion/medium": {
+ "median_ms": 158.4,
+ "report_only": false,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "open-session/medium": {
+ "median_ms": 314.4,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "cold-ready/medium": {
+ "median_ms": 56.3,
+ "report_only": false,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "agents-view-warm/medium": {
+ "median_ms": 186.6,
+ "report_only": false,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "warm-start/medium": {
+ "median_ms": 10.9,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "cold-start/medium": {
+ "median_ms": 19.2,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "session-switch/medium": {
+ "median_ms": 244.9,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "agents-view/medium": {
+ "median_ms": 551.9,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "warm-ready/medium": {
+ "median_ms": 56.6,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "at-completion/small": {
+ "median_ms": 132.3,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "agents-view-warm/small": {
+ "median_ms": 109.0,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "cold-start/small": {
+ "median_ms": 19.5,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "session-switch/small": {
+ "median_ms": 121.6,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "warm-ready/small": {
+ "median_ms": 40.6,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "cold-ready/small": {
+ "median_ms": 49.4,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "open-session/small": {
+ "median_ms": 165.3,
+ "report_only": false,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "agents-view/small": {
+ "median_ms": 64.7,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "warm-start/small": {
+ "median_ms": 11.2,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "cold-start/xl": {
+ "median_ms": 22.3,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "warm-ready/xl": {
+ "median_ms": 83.8,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "cold-ready/xl": {
+ "median_ms": 95.0,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "agents-view/xl": {
+ "median_ms": 3562.7,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "warm-start/xl": {
+ "median_ms": 16.0,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "open-session/xl": {
+ "median_ms": 279.6,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "at-completion/xl": {
+ "median_ms": 200.2,
+ "report_only": false,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "agents-view-warm/xl": {
+ "median_ms": 3492.8,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ },
+ "session-switch/xl": {
+ "median_ms": 2435.4,
+ "report_only": true,
+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"
+ }
+ }
+}
diff --git a/crates/pa-perf/src/bench/mod.rs b/crates/pa-perf/src/bench/mod.rs
new file mode 100644
index 0000000000..ddf861d295
--- /dev/null
+++ b/crates/pa-perf/src/bench/mod.rs
@@ -0,0 +1,139 @@
+//! The bench harness: the scenario set, the options, and the result types
+//! (the unix-only runner lives in `runner`/`pty`).
+
+use std::path::PathBuf;
+
+use clap::ValueEnum;
+use serde::{Deserialize, Serialize};
+
+/// The measured scenario set. Each is timed from a triggering action to an
+/// observable needle — literal paints as byte needles, cell-diffed content
+/// through the vt100 screen model; the deadline only fails a broken run.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum)]
+#[serde(rename_all = "kebab-case")]
+pub enum Scenario {
+ /// No daemon, cold corpus: exec -> the chat surface's first paint.
+ ColdStart,
+ /// The same launch as cold-start, timed to the attach: exec -> the
+ /// tray's session-usage pill (the first content the surface can only
+ /// paint once the daemon session is attached).
+ ColdReady,
+ /// Daemon pre-started, warm corpus: exec -> the first paint.
+ WarmStart,
+ /// The warm launch timed to the attach pill.
+ WarmReady,
+ /// Fresh daemon+TUI: left-arrow -> the roster scan delivered the oldest
+ /// session (its unique search-result row paints).
+ AgentsView,
+ /// The same daemon's second populate (its in-process scan cache warm):
+ /// a second TUI's left-arrow -> the completed roster.
+ AgentsViewWarm,
+ /// Fresh daemon: `--resume <huge session>` -> the huge marker paints.
+ OpenSession,
+ /// On session A: agents view -> uniquely filter to B -> enter -> B's
+ /// transcript marker.
+ SessionSwitch,
+ /// The `@` keystroke -> the first completion menu row.
+ AtCompletion,
+}
+
+/// Every bench knob for one scenario over one corpus.
+#[derive(Debug, Clone)]
+pub struct BenchOptions {
+ /// The release binary under test (`target/release/prime-agent`).
+ pub binary: PathBuf,
+ /// The corpus root (holds `corpus.json`).
+ pub corpus: PathBuf,
+ pub scenario: Scenario,
+ /// Sampled runs (one discarded warm-up always runs before them).
+ pub runs: usize,
+ /// Where the result JSON is written.
+ pub json: PathBuf,
+ /// The per-run liveness deadline (seconds) — a broken run fails
+ /// instead of hanging.
+ pub liveness_secs: u64,
+}
+
+/// The whole-suite driver: generates the cold and warm corpora, runs every
+/// scenario on the corpus state it needs (cold scenarios on `--cache
+/// cold`, warm-* on `--cache warm`), and writes one result JSON per
+/// scenario.
+#[derive(Debug, Clone)]
+pub struct RunOptions {
+ /// The release binary under test.
+ pub binary: PathBuf,
+ /// The named corpus profile.
+ pub profile: crate::corpus::Profile,
+ /// The output root: the two corpora and `results/` land under it.
+ pub out: PathBuf,
+ /// The deterministic seed.
+ pub seed: u64,
+ /// Sampled runs per scenario (one discarded warm-up always runs first).
+ pub runs: usize,
+ /// The per-run liveness deadline (seconds).
+ pub liveness_secs: u64,
+}
+
+/// The measured outcome of one scenario over one corpus.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BenchResult {
+ pub schema: u32,
+ pub scenario: Scenario,
+ pub profile: String,
+ pub runs: usize,
+ pub samples_ms: Vec<f64>,
+ pub median_ms: f64,
+ pub p95_ms: f64,
+ pub needle: String,
+ pub binary: String,
+ pub corpus: BenchCorpus,
+}
+
+/// The corpus summary the result records (baseline keys use scenario/profile).
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BenchCorpus {
+ pub root: PathBuf,
+ pub sessions: usize,
+ pub total_rows: usize,
+ pub child_sessions: usize,
+ pub archive: usize,
+}
+
+/// Run one scenario over one corpus (`options.runs` sampled runs plus one
+/// discarded warm-up), then write the result JSON.
+///
+/// # Errors
+///
+/// Propagates spawn, needle-timeout, daemon, and IO failures; errors
+/// immediately on non-unix platforms.
+#[cfg(unix)]
+pub fn run(options: &BenchOptions) -> anyhow::Result<BenchResult> {
+ runner::run(options)
+}
+
+/// Run the whole scenario suite on freshly generated corpora (`RunOptions`).
+///
+/// # Errors
+///
+/// Propagates corpus-generation, spawn, needle-timeout, daemon, and IO
+/// failures; errors immediately on non-unix platforms.
+#[cfg(unix)]
+pub fn run_suite(options: &RunOptions) -> anyhow::Result<Vec<BenchResult>> {
+ runner::run_suite(options)
+}
+
+#[cfg(not(unix))]
+pub fn run(_options: &BenchOptions) -> anyhow::Result<BenchResult> {
+ anyhow::bail!("the bench harness is unix-only")
+}
+
+#[cfg(not(unix))]
+pub fn run_suite(_options: &RunOptions) -> anyhow::Result<Vec<BenchResult>> {
+ anyhow::bail!("the bench harness is unix-only")
+}
+
+#[cfg(unix)]
+mod pty;
+/// The unix-only harness (pty, daemon lifecycle, scenario runners).
+#[cfg(unix)]
+mod runner;
diff --git a/crates/pa-perf/src/bench/pty.rs b/crates/pa-perf/src/bench/pty.rs
new file mode 100644
index 0000000000..97de98a7c4
--- /dev/null
+++ b/crates/pa-perf/src/bench/pty.rs
@@ -0,0 +1,409 @@
+//! The pty harness: one product child (the real release binary) on a raw pty
+//! in its own session with NO controlling tty — vendored crossterm opens
+//! `/dev/tty` first in `window_size()` and falls back to the stdio pty
+//! slave, the same discipline the repo's pty e2es use.
+
+use std::io::Read;
+use std::os::fd::{AsFd, AsRawFd, OwnedFd};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+use nix::fcntl::{fcntl, FcntlArg::F_SETFL, OFlag};
+use nix::poll::{poll, PollFd, PollFlags};
+use nix::pty::{openpty, Winsize};
+
+/// The kitty-probe pair: the capability query the product's vendored
+/// crossterm emits, and the flags+DA1 answer a kitty terminal gives. The
+/// bench answers identically every run so the probe path is deterministic.
+pub(crate) const KITTY_QUERY: &[u8] = b"\x1b[?u";
+pub(crate) const KITTY_ANSWER: &[u8] = b"\x1b[?7u\x1b[?62;c";
+
+/// The idle poll slice between empty reads (poll(2) drives the wakeups).
+const IDLE_POLL: Duration = Duration::from_millis(25);
+
+/// One product child on a pty, plus the collected master stream and the
+/// decoded screen (a vt100 model): byte needles cover literal paints; the
+/// screen model covers content that only exists as cell diffs (the agents
+/// view's heading counts).
+pub(crate) struct PtyChild {
+ pub(crate) child: Child,
+ master: std::fs::File,
+ pub(crate) output: Vec<u8>,
+ screen: vt100::Parser,
+ /// The kitty probe is answered exactly once (the answer goes to the
+ /// child's stdin, so it never shows up in the read stream — a
+ /// stream-search for it would re-send forever).
+ kitty_answered: bool,
+ /// Set only after the exited child's final PTY bytes have drained.
+ exited: Option<std::process::ExitStatus>,
+}
+
+impl PtyChild {
+ /// Spawn `binary` with the pty slave as stdio, its own session, and no
+ /// controlling terminal: `setsid` in the pre-exec hook (the child is
+ /// never a process-group leader, so it always succeeds).
+ ///
+ /// # Errors
+ ///
+ /// Propagates pty and spawn failures.
+ pub(crate) fn spawn(
+ binary: &std::path::Path,
+ args: &[std::ffi::OsString],
+ env: &[(String, String)],
+ cwd: &std::path::Path,
+ cols: u16,
+ rows: u16,
+ ) -> anyhow::Result<Self> {
+ let pty = openpty(
+ Some(&Winsize {
+ ws_row: rows,
+ ws_col: cols,
+ ws_xpixel: 0,
+ ws_ypixel: 0,
+ }),
+ None,
+ )?;
+ let mut command = Command::new(binary);
+ command
+ .args(args)
+ .current_dir(cwd)
+ .env_clear()
+ .envs(env.iter().map(|(key, value)| (key, value)))
+ .stdin(slave_as_stdio(&pty.slave))
+ .stdout(slave_as_stdio(&pty.slave))
+ .stderr(slave_as_stdio(&pty.slave));
+ pa_core::platform::process::set_new_session(&mut command);
+ let fd = pty.master.as_raw_fd();
+ fcntl(fd, F_SETFL(OFlag::O_NONBLOCK))?;
+ let child = command.spawn()?;
+ drop(pty.slave);
+ Ok(Self {
+ child,
+ master: pty.master.into(),
+ output: Vec::new(),
+ screen: vt100::Parser::new(rows, cols, 0),
+ kitty_answered: false,
+ exited: None,
+ })
+ }
+
+ /// Write to the child's input (the pty master's write end).
+ ///
+ /// # Errors
+ ///
+ /// Propagates write failures.
+ pub(crate) fn write(&mut self, payload: &[u8]) -> anyhow::Result<()> {
+ use std::io::Write;
+ self.master.write_all(payload)?;
+ Ok(())
+ }
+
+ /// The current stream mark (needles wait from here).
+ pub(crate) fn mark(&self) -> usize {
+ self.output.len()
+ }
+
+ pub(crate) fn output_from(&self, mark: usize) -> &[u8] {
+ &self.output[mark..]
+ }
+
+ /// Drain until `needle` appears in the stream collected since `mark`,
+ /// bounded by `deadline`. Returns the Instant the needle was seen.
+ ///
+ /// # Errors
+ ///
+ /// Errors when the deadline passes (with the stream tail attached) or
+ /// the child exits with a failure code.
+ pub(crate) fn wait_from(
+ &mut self,
+ mark: usize,
+ needle: &[u8],
+ what: &str,
+ deadline: Instant,
+ ) -> anyhow::Result<Instant> {
+ // Search only the bytes that arrived since the last check (plus the
+ // needle overlap, never reaching before the mark): a fresh
+ // full-stream scan per read is quadratic, and the harness must
+ // never out-CPU the product it measures.
+ let mut scanned = mark;
+ loop {
+ let from = scanned
+ .saturating_sub(needle.len().saturating_sub(1))
+ .max(mark);
+ if find(&self.output[from..], needle).is_some() {
+ return Ok(Instant::now());
+ }
+ scanned = self.output.len();
+ if let Some(status) = self.exited {
+ anyhow::bail!(
+ "child exited with {status} before {what}; pty tail:\n{}",
+ String::from_utf8_lossy(&self.output[self.output.len().saturating_sub(4000)..])
+ );
+ }
+ if Instant::now() >= deadline {
+ let tail =
+ &self.output_from(mark)[self.output_from(mark).len().saturating_sub(4000)..];
+ anyhow::bail!(
+ "timeout waiting for {what} (needle {:?}); pty tail since mark:\n{}",
+ String::from_utf8_lossy(needle),
+ String::from_utf8_lossy(tail),
+ );
+ }
+ self.pump(Some(deadline))?;
+ }
+ }
+
+ /// One poll(2) slice on the master: reads whatever is ready, appends to
+ /// the stream, and answers the kitty probe once it appears.
+ fn pump(&mut self, deadline: Option<Instant>) -> anyhow::Result<()> {
+ let timeout = match deadline {
+ Some(end) => {
+ let remaining = end.saturating_duration_since(Instant::now());
+ remaining.min(IDLE_POLL)
+ }
+ None => IDLE_POLL,
+ };
+ let millis = timeout.as_millis().min(u16::MAX as u128) as u16;
+ let mut poller = [PollFd::new(self.master.as_fd(), PollFlags::POLLIN)];
+ let ready = poll(&mut poller, millis)?;
+ // After exit, keep reading any buffered final paints before reporting
+ // the status. A poll timeout alone must not hide an already dead UI.
+ let status = self.child.try_wait()?;
+ if ready == 0 && status.is_none() {
+ return Ok(());
+ }
+ let mut buffer = [0u8; 8192];
+ let read = self.master.read(&mut buffer);
+ match read {
+ Ok(0) => self.exited = status,
+ Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}
+ Err(error)
+ if error.kind() == std::io::ErrorKind::WouldBlock
+ || error.raw_os_error() == Some(nix::errno::Errno::EIO as i32) =>
+ {
+ self.exited = status;
+ }
+ Err(error) => return Err(error.into()),
+ Ok(n) => {
+ let chunk = &buffer[..n];
+ self.receive(chunk)?;
+ }
+ }
+ Ok(())
+ }
+
+ /// Include the previous read's overlap when a terminal query is split.
+ fn receive(&mut self, chunk: &[u8]) -> anyhow::Result<()> {
+ let from = self.output.len().saturating_sub(KITTY_QUERY.len() - 1);
+ self.output.extend_from_slice(chunk);
+ self.screen.process(chunk);
+ if !self.kitty_answered && find(&self.output[from..], KITTY_QUERY).is_some() {
+ self.write(KITTY_ANSWER)?;
+ self.kitty_answered = true;
+ }
+ Ok(())
+ }
+
+ /// Wait until the decoded screen contains `needle`, bounded by
+ /// `deadline`. Content needles the byte stream cannot see (cell-diffed
+ /// counters, repainted rows) live here.
+ ///
+ /// # Errors
+ ///
+ /// Errors when the deadline passes (with the decoded screen attached).
+ pub(crate) fn wait_screen_contains(
+ &mut self,
+ needle: &str,
+ what: &str,
+ deadline: Instant,
+ ) -> anyhow::Result<Instant> {
+ loop {
+ if self.screen_text().contains(needle) {
+ return Ok(Instant::now());
+ }
+ if let Some(status) = self.exited {
+ anyhow::bail!(
+ "child exited with {status} before {what}; pty tail:\n{}",
+ String::from_utf8_lossy(&self.output[self.output.len().saturating_sub(4000)..])
+ );
+ }
+ if Instant::now() >= deadline {
+ anyhow::bail!(
+ "timeout waiting for {what} (screen needle {needle:?}); screen:\n{}",
+ self.screen_text(),
+ );
+ }
+ self.pump(Some(deadline))?;
+ }
+ }
+
+ /// The decoded screen's text (rows joined; trailing blanks trimmed).
+ fn screen_text(&mut self) -> String {
+ self.screen.screen().contents().trim_end().to_string()
+ }
+
+ /// The child's pid (teardown signals this pid only).
+ pub(crate) fn pid(&self) -> i32 {
+ self.child.id() as i32
+ }
+
+ /// Stop the child: SIGTERM, a short grace, then SIGKILL; reap it.
+ pub(crate) fn finish(&mut self) {
+ // Drop also cleans up failures before Launch has been constructed.
+ // Avoid signalling an already-reaped child if finish ran earlier.
+ if self.child.try_wait().is_ok_and(|status| status.is_some()) {
+ return;
+ }
+ let _ = pa_core::platform::process::kill_pid(
+ self.pid(),
+ pa_core::platform::process::Signal::Term,
+ );
+ let deadline = Instant::now() + Duration::from_secs(2);
+ while Instant::now() < deadline {
+ if self.child.try_wait().is_ok_and(|status| status.is_some()) {
+ return;
+ }
+ std::thread::sleep(Duration::from_millis(25));
+ }
+ let _ = pa_core::platform::process::kill_pid(
+ self.pid(),
+ pa_core::platform::process::Signal::Kill,
+ );
+ let _ = self.child.wait();
+ }
+}
+
+impl Drop for PtyChild {
+ fn drop(&mut self) {
+ self.finish();
+ }
+}
+
+fn slave_as_stdio(slave: &OwnedFd) -> Stdio {
+ slave.try_clone().expect("clone pty slave").into()
+}
+
+fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
+ if needle.is_empty() {
+ return None;
+ }
+ haystack.windows(needle.len()).position(|w| w == needle)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn kitty_queries_are_answered_at_every_read_boundary_once() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ for split in 1..KITTY_QUERY.len() {
+ let mut child = PtyChild::spawn(
+ std::path::Path::new("/bin/cat"),
+ &[],
+ &[],
+ dir.path(),
+ 120,
+ 36,
+ )
+ .expect("spawn mock terminal peer");
+ child.receive(&KITTY_QUERY[..split]).expect("first chunk");
+ assert!(!child.kitty_answered);
+ child.receive(&KITTY_QUERY[split..]).expect("second chunk");
+ assert!(child.kitty_answered, "unanswered query split at {split}");
+ // A repeated probe must not write another answer, even if the
+ // master is now closed: answer write failures propagate.
+ let closed = std::fs::File::open("/dev/null").expect("read-only master");
+ child.master = closed;
+ child.receive(KITTY_QUERY).expect("already answered");
+ }
+ }
+
+ #[test]
+ fn exited_children_fail_missing_needles_without_waiting_for_liveness_timeout() {
+ let directory = tempfile::tempdir().expect("mock directory");
+ let mut child = PtyChild::spawn(
+ std::path::Path::new("/bin/sh"),
+ &["-c".into(), "printf 'fatal mock diagnostic'; exit 7".into()],
+ &[],
+ directory.path(),
+ 120,
+ 36,
+ )
+ .expect("spawn exiting mock");
+ let started = Instant::now();
+ let error = child
+ .wait_from(
+ 0,
+ b"missing",
+ "first paint",
+ started + Duration::from_secs(10),
+ )
+ .expect_err("exited child has no paint");
+ let message = error.to_string();
+ assert!(message.contains("exit status: 7"), "{message}");
+ assert!(message.contains("fatal mock diagnostic"), "{message}");
+ assert!(started.elapsed() < Duration::from_secs(5));
+ assert!(child
+ .wait_screen_contains("missing", "ready", started + Duration::from_secs(10))
+ .expect_err("screen wait also sees exit")
+ .to_string()
+ .contains("exit status: 7"));
+ }
+
+ #[test]
+ fn final_output_is_drained_before_reporting_child_exit() {
+ let directory = tempfile::tempdir().expect("mock directory");
+ let mut child = PtyChild::spawn(
+ std::path::Path::new("/bin/sh"),
+ &[
+ "-c".into(),
+ "printf '%20000s' x; printf 'last paint'; exit 0".into(),
+ ],
+ &[],
+ directory.path(),
+ 120,
+ 36,
+ )
+ .expect("spawn final paint mock");
+ child
+ .wait_from(
+ 0,
+ b"last paint",
+ "final paint",
+ Instant::now() + Duration::from_secs(5),
+ )
+ .expect("final bytes survive child exit");
+ assert!(child.output.len() > 8192, "exercise more than one PTY read");
+ child
+ .wait_screen_contains(
+ "last paint",
+ "final screen",
+ Instant::now() + Duration::from_secs(5),
+ )
+ .expect("final screen survives child exit");
+ }
+
+ #[test]
+ fn an_initial_paint_timeout_reaps_the_owned_child() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let mut child = PtyChild::spawn(
+ std::path::Path::new("/bin/cat"),
+ &[],
+ &[],
+ dir.path(),
+ 120,
+ 36,
+ )
+ .expect("spawn the mock, never the product");
+ let pid = nix::unistd::Pid::from_raw(child.pid());
+ assert!(child
+ .wait_from(0, b"missing first paint", "first paint", Instant::now())
+ .is_err());
+ drop(child);
+ assert_eq!(
+ nix::sys::signal::kill(pid, None),
+ Err(nix::errno::Errno::ESRCH)
+ );
+ }
+}
diff --git a/crates/pa-perf/src/bench/runner.rs b/crates/pa-perf/src/bench/runner.rs
new file mode 100644
index 0000000000..aee1a9562c
--- /dev/null
+++ b/crates/pa-perf/src/bench/runner.rs
@@ -0,0 +1,1044 @@
+//! The scenario runner: pty harness, daemon lifecycle, per-run corpus
+//! restore, and the scenario mechanics. Unix-only.
+
+use std::collections::BTreeMap;
+use std::io::{BufRead, BufReader, Write};
+use std::path::{Path, PathBuf};
+use std::time::{Duration, Instant};
+
+use anyhow::Context;
+
+use crate::bench::{BenchCorpus, BenchOptions, BenchResult, RunOptions, Scenario};
+use crate::corpus::{self, CacheMode, CorpusOptions, Manifest};
+
+use super::pty::PtyChild;
+
+/// The first paint's needle: the brand splash is the empty chat's header
+/// (the corpus settings keep the TUI out of the first-run flow). Pinned
+/// empirically against the real binary.
+const FIRST_PAINT_NEEDLE: &[u8] = "▗▄▄█▀".as_bytes();
+
+/// The ready needle: the tray's session-usage pill ("model · tokens
+/// (percent)") — the first content the chat surface can only paint once
+/// the daemon session is attached. A fresh session's usage is always
+/// "0 (0%)".
+const READY_NEEDLE: &str = "0 (0%)";
+
+/// One bench run's anchored paths.
+struct RunCtx {
+ binary: PathBuf,
+ manifest: Manifest,
+ corpus_root: PathBuf,
+ project_dir: PathBuf,
+ agent_dir: PathBuf,
+ liveness: Duration,
+}
+
+impl RunCtx {
+ /// The manifest's agent dir paths, from the corpus at `root`.
+ fn load(binary: &Path, corpus_root: &Path, liveness: Duration) -> anyhow::Result<Self> {
+ let manifest_path = corpus_root.join("corpus.json");
+ let manifest: Manifest = serde_json::from_str(
+ &std::fs::read_to_string(&manifest_path)
+ .with_context(|| format!("read {}", manifest_path.display()))?,
+ )
+ .with_context(|| format!("parse {}", manifest_path.display()))?;
+ Ok(Self {
+ // Spawns run in the generated project directory. Resolve the
+ // caller's path before changing cwd (including `make perf`).
+ binary: binary
+ .canonicalize()
+ .with_context(|| format!("resolve benchmark binary {}", binary.display()))?,
+ project_dir: manifest.paths.project_dir.clone(),
+ agent_dir: manifest.paths.agent_dir.clone(),
+ corpus_root: corpus_root
+ .canonicalize()
+ .with_context(|| format!("resolve corpus root {}", corpus_root.display()))?,
+ manifest,
+ liveness,
+ })
+ }
+}
+
+/// The child environment: hermetic (a clean env plus the isolation vars).
+fn child_env(ctx: &RunCtx) -> Vec<(String, String)> {
+ let mut env = vec![
+ (
+ "PRIME_AGENT_CODING_AGENT_DIR".to_string(),
+ ctx.agent_dir.display().to_string(),
+ ),
+ ("DO_NOT_TRACK".to_string(), "1".to_string()),
+ ("TERM".to_string(), "xterm-ghostty".to_string()),
+ ("TERM_PROGRAM".to_string(), "ghostty".to_string()),
+ ];
+ // PI_PACKAGE_DIR carries the packaged runtime dir to the binary and its
+ // workers (a release binary resolves prime-agent-runtime/ beside the
+ // executable or through it); PATH/HOME/TMPDIR are the ambient basics.
+ for carry in ["PATH", "HOME", "TMPDIR", "PI_PACKAGE_DIR"] {
+ if let Some(value) = std::env::var_os(carry) {
+ env.push((carry.to_string(), value.to_string_lossy().to_string()));
+ }
+ }
+ env
+}
+
+// The single-scenario entry (`pa-perf bench`): one scenario over the given
+// corpus — the debugging/investigation path.
+
+/// Run one scenario `options.runs` times (plus one discarded warm-up), then
+/// write the result JSON.
+///
+/// # Errors
+///
+/// Propagates spawn, needle-timeout, daemon, and IO failures.
+pub(super) fn run(options: &BenchOptions) -> anyhow::Result<BenchResult> {
+ anyhow::ensure!(options.runs > 0, "sampled runs must be positive");
+ let ctx = RunCtx::load(
+ &options.binary,
+ &options.corpus,
+ Duration::from_secs(options.liveness_secs),
+ )?;
+ let samples = sample_runs(&ctx, options.runs, |index| {
+ scenario_once(&ctx, options.scenario, index)
+ })?;
+ let result = build_result(&ctx, options.scenario, samples);
+ write_result(&result, &options.json)?;
+ Ok(result)
+}
+
+/// Restore the manifest baseline before every standalone launch, including
+/// its discarded warm-up. The scenario callback keeps this loop testable
+/// with filesystem mutations instead of running the product.
+fn sample_runs(
+ ctx: &RunCtx,
+ runs: usize,
+ mut scenario: impl FnMut(usize) -> anyhow::Result<f64>,
+) -> anyhow::Result<Vec<f64>> {
+ let mut samples = Vec::with_capacity(runs + 1);
+ let total = runs + 1;
+ for index in 0..total {
+ restore(ctx)?;
+ let started = Instant::now();
+ let outcome = scenario(index);
+ if index == 0 {
+ outcome?;
+ continue;
+ }
+ let sample = outcome.context("a sampled run failed")?;
+ samples.push(sample);
+ println!(
+ "run {index}/{runs}: {sample:.1}ms (wall {:.1}s)",
+ started.elapsed().as_secs_f32()
+ );
+ }
+ Ok(samples)
+}
+
+/// One scenario pass; returns the sample in ms.
+fn scenario_once(ctx: &RunCtx, scenario: Scenario, run: usize) -> anyhow::Result<f64> {
+ let (sample, _) = launch(ctx, scenario, run)?;
+ Ok(sample)
+}
+
+/// The ONE launch path: the pre-started daemon for warm-* scenarios, the
+/// scenario's own resume target + first needle, the post-launch mechanics,
+/// and the daemon teardown on every path. Both `run()` (single scenario)
+/// and `run_suite()` call this.
+fn launch(ctx: &RunCtx, scenario: Scenario, run: usize) -> anyhow::Result<(f64, Launch)> {
+ let socket = run_socket(ctx, run);
+ std::fs::create_dir_all(socket.parent().expect("a socket dir"))?;
+ let mut daemon = if matches!(
+ scenario,
+ Scenario::WarmStart | Scenario::WarmReady | Scenario::AgentsViewWarm
+ ) {
+ // AgentsViewWarm pre-starts so both TUIs share the daemon the first
+ // populate warms; the single-scenario path cannot express that (its
+ // second launch would spawn a second daemon), so it runs the first
+ // populate only and the sample is the suite's warm number.
+ Some(start_daemon(ctx, &socket)?)
+ } else {
+ None
+ };
+ let outcome: anyhow::Result<(f64, Launch)> = (|| {
+ let mut launched = spawn_for_scenario(ctx, &socket, scenario, run)?;
+ let sample = launch_scenario(ctx, &mut launched, scenario)?;
+ // The measured client must disconnect before daemon shutdown.
+ // Keeping it alive while waiting for daemon exit can hold server
+ // connection draining open or start the client's reconnect path.
+ launched.child.finish();
+ Ok((sample, launched))
+ })();
+ let stopped = stop_daemon(&socket);
+ // Socket closure alone does not reap a daemon we directly spawned.
+ // Keep ownership through shutdown, and fail before the guard's forced
+ // cleanup if it does not exit normally.
+ let exited = if let Some(daemon) = &mut daemon {
+ let deadline = Instant::now() + Duration::from_secs(5);
+ loop {
+ if let Some(status) = daemon.child.try_wait()? {
+ break if status.success() {
+ Ok(())
+ } else {
+ Err(anyhow::anyhow!("the owned daemon exited with {status}"))
+ };
+ }
+ if Instant::now() >= deadline {
+ break Err(anyhow::anyhow!(
+ "the owned daemon did not exit after shutdown"
+ ));
+ }
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ } else {
+ Ok(())
+ };
+ let result = outcome?;
+ stopped?;
+ exited.with_context(|| format!("{scenario:?} run {run} daemon cleanup"))?;
+ Ok(result)
+}
+
+// The whole-suite driver (`pa-perf run`): generates the cold and warm
+// corpora and runs every scenario on the corpus state it needs.
+
+/// Run the full scenario schedule on freshly generated corpora, writing one
+/// result JSON per scenario under `<out>/results/`.
+///
+/// # Errors
+///
+/// Propagates corpus-generation, spawn, needle-timeout, daemon, and IO
+/// failures.
+pub(super) fn run_suite(options: &RunOptions) -> anyhow::Result<Vec<BenchResult>> {
+ anyhow::ensure!(options.runs > 0, "sampled runs must be positive");
+ let liveness = Duration::from_secs(options.liveness_secs);
+ let knobs = || CorpusOptions {
+ seed: options.seed,
+ ..options.profile.defaults()
+ };
+ println!("=== generating the cold corpus");
+ corpus::regenerate_for_suite(&CorpusOptions {
+ out: options.out.join("corpus-cold"),
+ ..knobs()
+ })?;
+ println!("=== generating the warm corpus");
+ corpus::regenerate_for_suite(&CorpusOptions {
+ out: options.out.join("corpus-warm"),
+ cache: CacheMode::Warm,
+ ..knobs()
+ })?;
+ let cold_ctx = RunCtx::load(&options.binary, &options.out.join("corpus-cold"), liveness)?;
+ let warm_ctx = RunCtx::load(&options.binary, &options.out.join("corpus-warm"), liveness)?;
+
+ let mut samples: BTreeMap<Scenario, Vec<f64>> = BTreeMap::new();
+ let total = options.runs + 1;
+ for index in 0..total {
+ let started = Instant::now();
+ // The schedule: cold scenarios on the cold corpus, warm-* on the
+ // warm corpus. `launch_cold`/`launch_warm` below run one scenario
+ // per launch; the cold-start/cold-ready (and warm-*) pairs share
+ // one launch each.
+ restore(&cold_ctx)?;
+ let (start_sample, launched) = launch(&cold_ctx, Scenario::ColdStart, index)?;
+ let ready_sample = launched
+ .ready_ms
+ .context("the cold launch never attached")?;
+ push(&mut samples, index, Scenario::ColdStart, start_sample);
+ push(&mut samples, index, Scenario::ColdReady, ready_sample);
+ for scenario in [
+ Scenario::AgentsView,
+ Scenario::AgentsViewWarm,
+ Scenario::OpenSession,
+ Scenario::SessionSwitch,
+ Scenario::AtCompletion,
+ ] {
+ restore(&cold_ctx)?;
+ let (sample, _) = launch(&cold_ctx, scenario, index)?;
+ push(&mut samples, index, scenario, sample);
+ }
+ restore(&warm_ctx)?;
+ let (start_sample, launched) = launch(&warm_ctx, Scenario::WarmStart, index)?;
+ let ready_sample = launched
+ .ready_ms
+ .context("the warm launch never attached")?;
+ push(&mut samples, index, Scenario::WarmStart, start_sample);
+ push(&mut samples, index, Scenario::WarmReady, ready_sample);
+ println!(
+ "run {index}/{} complete ({:.1}s)",
+ options.runs,
+ started.elapsed().as_secs_f32()
+ );
+ }
+
+ let results_dir = options.out.join("results");
+ std::fs::create_dir_all(&results_dir)?;
+ let mut results = Vec::new();
+ for (scenario, mut samples) in samples {
+ samples.sort_by(f64::total_cmp);
+ if samples.len() != options.runs {
+ anyhow::bail!(
+ "{scenario:?} collected {} samples, expected {}",
+ samples.len(),
+ options.runs
+ );
+ }
+ let ctx = corpus_for(scenario, &cold_ctx, &warm_ctx);
+ let result = build_result(ctx, scenario, samples);
+ let name = crate::report::key_of(scenario);
+ // Per-profile unique names: the CI legs merge their artifacts into
+ // one results dir, and same-named files would overwrite each other.
+ let profile = crate::report::profile_key(options.profile);
+ write_result(&result, &results_dir.join(format!("{profile}-{name}.json")))?;
+ println!(
+ "scenario={name} profile={} median={:.1}ms p95={:.1}ms",
+ crate::report::profile_key(options.profile),
+ result.median_ms,
+ result.p95_ms
+ );
+ results.push(result);
+ }
+ Ok(results)
+}
+
+/// Keep result provenance paired with the corpus used by the schedule.
+fn corpus_for<'a>(scenario: Scenario, cold: &'a RunCtx, warm: &'a RunCtx) -> &'a RunCtx {
+ match scenario {
+ Scenario::WarmStart | Scenario::WarmReady => warm,
+ Scenario::ColdStart
+ | Scenario::ColdReady
+ | Scenario::AgentsView
+ | Scenario::AgentsViewWarm
+ | Scenario::OpenSession
+ | Scenario::SessionSwitch
+ | Scenario::AtCompletion => cold,
+ }
+}
+
+/// Push a sample unless this is the discarded warm-up run.
+fn push(samples: &mut BTreeMap<Scenario, Vec<f64>>, index: usize, scenario: Scenario, sample: f64) {
+ if index > 0 {
+ samples.entry(scenario).or_default().push(sample);
+ }
+}
+
+/// The resume target + first needle per scenario, then the spawn.
+fn spawn_for_scenario(
+ ctx: &RunCtx,
+ socket: &Path,
+ scenario: Scenario,
+ run: usize,
+) -> anyhow::Result<Launch> {
+ let (resume, first_needle) = match scenario {
+ Scenario::OpenSession => (
+ Some(ctx.manifest.markers.huge.file.clone()),
+ ctx.manifest.markers.huge.needle.as_bytes().to_vec(),
+ ),
+ Scenario::SessionSwitch => (
+ Some(ctx.manifest.markers.switch_a.file.clone()),
+ ctx.manifest.markers.switch_a.needle.as_bytes().to_vec(),
+ ),
+ _ => (None, FIRST_PAINT_NEEDLE.to_vec()),
+ };
+ let dump = ctx
+ .corpus_root
+ .join("failed-streams")
+ .join(format!("{}-run{run}.bin", crate::report::key_of(scenario)));
+ let mut launch = Launch::spawn(ctx, socket, resume.as_deref(), &first_needle, dump)?;
+ if matches!(
+ scenario,
+ Scenario::ColdStart | Scenario::WarmStart | Scenario::ColdReady | Scenario::WarmReady
+ ) {
+ launch.wait_ready()?;
+ }
+ Ok(launch)
+}
+
+/// The scenario mechanics after the launch needle; a completed launch
+/// never dumps its stream.
+fn launch_scenario(ctx: &RunCtx, launch: &mut Launch, scenario: Scenario) -> anyhow::Result<f64> {
+ let sample = post_launch(ctx, launch, scenario)?;
+ launch.completed = true;
+ Ok(sample)
+}
+
+/// The scenario mechanics after the launch needle.
+fn post_launch(ctx: &RunCtx, launch: &mut Launch, scenario: Scenario) -> anyhow::Result<f64> {
+ match scenario {
+ Scenario::ColdStart | Scenario::WarmStart | Scenario::OpenSession => Ok(launch.first_ms),
+ Scenario::AgentsView => agents_view_populate(ctx, &mut launch.child),
+ Scenario::AgentsViewWarm => {
+ // The first populate warms the daemon's in-process scan cache;
+ // the second TUI measures the warm path on the same daemon.
+ agents_view_populate(ctx, &mut launch.child)?;
+ let socket = launch.socket.clone();
+ let dump = ctx
+ .corpus_root
+ .join("failed-streams")
+ .join("agents-view-warm-second.bin");
+ let mut second = Launch::spawn(ctx, &socket, None, FIRST_PAINT_NEEDLE, dump)?;
+ let sample = agents_view_populate(ctx, &mut second.child)?;
+ second.completed = true;
+ Ok(sample)
+ }
+ Scenario::SessionSwitch => switch_to_target_b(ctx, &mut launch.child),
+ Scenario::AtCompletion => at_completion(ctx, &mut launch.child),
+ Scenario::ColdReady | Scenario::WarmReady => Ok(launch.ready_ms.unwrap_or(launch.first_ms)),
+ }
+}
+
+/// One TUI child plus its teardown: Drop finishes the child, so every path —
+/// success, `?` bubbles, panics — cleans up.
+struct Launch {
+ child: PtyChild,
+ socket: PathBuf,
+ first_ms: f64,
+ ready_ms: Option<f64>,
+ spawned: Instant,
+ liveness: Duration,
+ /// Where the raw pty stream lands when the launch does NOT complete (the
+ /// CI failure artifact for empty-tail debugging).
+ stream_dump: Option<PathBuf>,
+ completed: bool,
+}
+
+impl Launch {
+ /// Spawn the TUI (fresh daemon when none listens on `socket`), wait the
+ /// first needle (the brand-splash paint, or the resumed transcript's
+ /// marker), and record exec -> needle.
+ fn spawn(
+ ctx: &RunCtx,
+ socket: &Path,
+ resume: Option<&Path>,
+ first_needle: &[u8],
+ stream_dump: PathBuf,
+ ) -> anyhow::Result<Self> {
+ let mut args = vec![
+ "--daemon-socket".into(),
+ socket.display().to_string().into(),
+ ];
+ if let Some(file) = resume {
+ args.push("--resume".into());
+ args.push(file.display().to_string().into());
+ }
+ let started = Instant::now();
+ let child = PtyChild::spawn(
+ &ctx.binary,
+ &args,
+ &child_env(ctx),
+ &ctx.project_dir,
+ 120,
+ 36,
+ )?;
+ let what = if resume.is_some() {
+ "the resumed transcript paint"
+ } else {
+ "the first paint"
+ };
+ let mut launch = Self {
+ first_ms: 0.0,
+ child,
+ socket: socket.to_path_buf(),
+ ready_ms: None,
+ spawned: started,
+ liveness: ctx.liveness,
+ stream_dump: Some(stream_dump),
+ completed: false,
+ };
+ launch.child.wait_from(
+ launch.child.mark(),
+ first_needle,
+ what,
+ started + ctx.liveness,
+ )?;
+ launch.first_ms = started.elapsed().as_secs_f64() * 1000.0;
+ Ok(launch)
+ }
+
+ /// The ready metric: exec -> the tray's session-usage pill, the first
+ /// attach-only content on the chat surface.
+ fn wait_ready(&mut self) -> anyhow::Result<()> {
+ self.child.wait_screen_contains(
+ READY_NEEDLE,
+ "the attach pill",
+ self.spawned + self.liveness,
+ )?;
+ self.ready_ms = Some(self.spawned.elapsed().as_secs_f64() * 1000.0);
+ Ok(())
+ }
+}
+
+impl Drop for Launch {
+ fn drop(&mut self) {
+ if !self.completed {
+ if let Some(path) = &self.stream_dump {
+ if let Some(parent) = path.parent() {
+ let _ = std::fs::create_dir_all(parent);
+ }
+ let _ = std::fs::write(path, &self.child.output);
+ }
+ }
+ self.child.finish();
+ }
+}
+
+/// The left-arrow opens the agents view; the populate needle is the
+/// completed `Inactive (N)` roster heading in the decoded screen (the
+/// heading count repaints as cell diffs, so the byte stream cannot see it
+/// — the screen model does). The count includes the TUI's own fresh
+/// session (rendered as one more inactive row).
+fn agents_view_populate(ctx: &RunCtx, child: &mut PtyChild) -> anyhow::Result<f64> {
+ let needle = format!("Inactive ({})", ctx.manifest.markers.roster_inactive + 1);
+ let started = Instant::now();
+ child.write(b"\x1b[D")?;
+ child.wait_screen_contains(
+ &needle,
+ "the agents view roster",
+ Instant::now() + ctx.liveness,
+ )?;
+ Ok(started.elapsed().as_secs_f64() * 1000.0)
+}
+
+/// From the opened session A: open the agents view, uniquely filter to
+/// switch-b, and time enter -> B's transcript marker. The `zq` token in
+/// switch-b's name matches no other row (no generated text carries the
+/// letter z), so Enter always drills THE row.
+fn switch_to_target_b(ctx: &RunCtx, child: &mut PtyChild) -> anyhow::Result<f64> {
+ child.write(b"\x1b[D")?;
+ // Wait the completed roster before typing: keys racing the scan flood
+ // were the CI-runner failure (the query echoed nothing). The resumed
+ // session A renders as one IDLE row, so the completed heading counts
+ // the remaining sessions.
+ child.wait_screen_contains(
+ &format!("Inactive ({})", ctx.manifest.markers.roster_inactive - 1),
+ "the agents view roster",
+ Instant::now() + ctx.liveness,
+ )?;
+ child.write(ctx.manifest.markers.switch_query.as_bytes())?;
+ // The echoed query does not contain the row's "switch" prefix, so this
+ // needle is the picker's result row, not the echo.
+ child.wait_from(
+ child.mark(),
+ ctx.manifest.markers.switch_row.as_bytes(),
+ "the switch-b picker row",
+ Instant::now() + ctx.liveness,
+ )?;
+ let mark = child.mark();
+ let started = Instant::now();
+ child.write(b"\r")?;
+ child.wait_from(
+ mark,
+ ctx.manifest.markers.switch_b.needle.as_bytes(),
+ "the switch-b transcript",
+ Instant::now() + ctx.liveness,
+ )?;
+ Ok(started.elapsed().as_secs_f64() * 1000.0)
+}
+
+/// The `@` completion: the trigger is the `@` keystroke alone; the needle
+/// is the first painted completion menu row (the menu lists the project
+/// tree's entries — `src/` is a top-level dir of the generated project
+/// that the pre-menu screen never contains).
+fn at_completion(ctx: &RunCtx, child: &mut PtyChild) -> anyhow::Result<f64> {
+ let started = Instant::now();
+ child.write(b"@")?;
+ // The menu lists the project tree's entries — `src/` is a top-level
+ // dir of the generated project that the pre-menu screen never contains
+ // (the visible first rows are dirs, not the .rs files).
+ child.wait_screen_contains(
+ "src/",
+ "the completion menu row",
+ Instant::now() + ctx.liveness,
+ )?;
+ Ok(started.elapsed().as_secs_f64() * 1000.0)
+}
+
+/// The per-run isolated socket path.
+fn run_socket(ctx: &RunCtx, run: usize) -> PathBuf {
+ ctx.corpus_root
+ .join("bench-runs")
+ .join(format!("run-{run}"))
+ .join("daemon.sock")
+}
+
+/// Start the daemon (own session, detached) and wait for the socket to
+/// accept — the observable readiness.
+fn start_daemon(ctx: &RunCtx, socket: &Path) -> anyhow::Result<DaemonChild> {
+ std::fs::create_dir_all(socket.parent().expect("a socket dir"))?;
+ let args: [std::ffi::OsString; 4] = [
+ "--mode".into(),
+ "daemon".into(),
+ "--daemon-socket".into(),
+ socket.display().to_string().into(),
+ ];
+ let env = child_env(ctx);
+ let mut command = std::process::Command::new(&ctx.binary);
+ command
+ .args(&args)
+ .env_clear()
+ .envs(env.iter().map(|(k, v)| (k, v)))
+ .current_dir(&ctx.project_dir)
+ .stdin(std::process::Stdio::null())
+ .stdout(std::process::Stdio::null())
+ .stderr(std::process::Stdio::null());
+ pa_core::platform::process::set_new_session(&mut command);
+ DaemonChild {
+ child: command.spawn()?,
+ }
+ .wait_until_ready(socket, Instant::now() + Duration::from_secs(30))
+}
+
+/// Direct child ownership survives startup and normal protocol shutdown.
+/// Drop is an error-path fallback, restricted to this exact child handle.
+struct DaemonChild {
+ child: std::process::Child,
+}
+
+impl DaemonChild {
+ fn wait_until_ready(mut self, socket: &Path, deadline: Instant) -> anyhow::Result<Self> {
+ loop {
+ if let Some(status) = self.child.try_wait()? {
+ anyhow::bail!("the daemon exited before socket readiness: {status}");
+ }
+ if socket_accepts(socket) {
+ return Ok(self);
+ }
+ if Instant::now() >= deadline {
+ anyhow::bail!("the daemon socket never accepted: {}", socket.display());
+ }
+ std::thread::sleep(Duration::from_millis(5));
+ }
+ }
+}
+
+impl Drop for DaemonChild {
+ fn drop(&mut self) {
+ if self.child.try_wait().is_ok_and(|status| status.is_some()) {
+ return;
+ }
+ let _ = pa_core::platform::process::kill_pid(
+ self.child.id() as i32,
+ pa_core::platform::process::Signal::Kill,
+ );
+ let _ = self.child.wait();
+ }
+}
+
+fn socket_accepts(socket: &Path) -> bool {
+ std::os::unix::net::UnixStream::connect(socket).is_ok()
+}
+
+/// Stop the daemon on `socket` by the shutdown protocol: the hello line,
+/// the shutdown command, one response read, then wait until the socket no
+/// longer accepts (the observable exit). A daemon that never stops fails
+/// the run loudly — no pid sweeps.
+fn stop_daemon(socket: &Path) -> anyhow::Result<()> {
+ protocol_shutdown(socket);
+ let deadline = Instant::now() + Duration::from_secs(10);
+ while Instant::now() < deadline {
+ if !socket_accepts(socket) {
+ let _ = std::fs::remove_file(socket);
+ return Ok(());
+ }
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ anyhow::bail!(
+ "the daemon on {} kept accepting after the shutdown protocol; the run is not clean",
+ socket.display()
+ )
+}
+
+/// The daemon-shutdown protocol (the e2es' `graceful_shutdown` shape): the
+/// hello line, the shutdown command, one response read.
+fn protocol_shutdown(socket: &Path) {
+ let Ok(stream) = std::os::unix::net::UnixStream::connect(socket) else {
+ return;
+ };
+ let Ok(write_half) = stream.try_clone() else {
+ return;
+ };
+ let mut reader = BufReader::new(stream);
+ let mut writer = write_half;
+ // The hello can stall too: bound every read, not just the reply.
+ if reader
+ .get_ref()
+ .set_read_timeout(Some(Duration::from_secs(5)))
+ .is_err()
+ {
+ return;
+ }
+ let mut hello = String::new();
+ if !matches!(reader.read_line(&mut hello), Ok(bytes) if bytes > 0) {
+ return;
+ }
+ let command = serde_json::json!({
+ "type": "command",
+ "id": "bench-shutdown",
+ "protocol": { "name": "prime-agent.daemon", "version": 7 },
+ "command": { "type": "shutdown" },
+ });
+ let Ok(mut line) = serde_json::to_string(&command) else {
+ return;
+ };
+ line.push('\n');
+ if writer.write_all(line.as_bytes()).is_err() {
+ return;
+ }
+ let _ = writer.flush();
+ let _ = reader
+ .get_ref()
+ .set_read_timeout(Some(Duration::from_secs(5)));
+ let mut response = String::new();
+ let _ = reader.read_line(&mut response);
+}
+
+/// Restore the corpus's exact generated state: delete every file under the
+/// agent dir that the manifest does not list (the product's new session
+/// files, sidecars, logs, leases), then prune the emptied dirs.
+fn restore(ctx: &RunCtx) -> anyhow::Result<()> {
+ let baseline: std::collections::HashSet<&str> =
+ ctx.manifest.files.iter().map(String::as_str).collect();
+ let mut dirs = Vec::new();
+ let mut stack = vec![ctx.agent_dir.clone()];
+ while let Some(dir) = stack.pop() {
+ for entry in std::fs::read_dir(&dir)? {
+ let entry = entry?;
+ let path = entry.path();
+ // Never follow product-created links outside the owned corpus.
+ if entry.file_type()?.is_dir() {
+ dirs.push(path.clone());
+ stack.push(path);
+ } else if !baseline.contains(path.to_string_lossy().as_ref()) {
+ let _ = std::fs::remove_file(&path);
+ }
+ }
+ }
+ for dir in dirs.iter().rev() {
+ let _ = std::fs::remove_dir(dir);
+ }
+ Ok(())
+}
+
+/// The result object for one scenario's samples.
+fn build_result(ctx: &RunCtx, scenario: Scenario, samples: Vec<f64>) -> BenchResult {
+ let (median_ms, p95_ms) = stats(&samples);
+ BenchResult {
+ schema: 1,
+ scenario,
+ profile: crate::report::profile_key(ctx.manifest.profile),
+ runs: samples.len(),
+ median_ms,
+ p95_ms,
+ samples_ms: samples,
+ needle: needle_of(scenario, ctx),
+ binary: ctx.binary.display().to_string(),
+ corpus: BenchCorpus {
+ root: ctx.corpus_root.clone(),
+ sessions: ctx.manifest.counts.sessions,
+ total_rows: ctx.manifest.counts.total_rows,
+ child_sessions: ctx.manifest.counts.child_sessions,
+ archive: ctx.manifest.counts.archive,
+ },
+ }
+}
+
+/// Median + p95 over the samples.
+fn stats(samples: &[f64]) -> (f64, f64) {
+ let mut sorted = samples.to_vec();
+ sorted.sort_by(f64::total_cmp);
+ let median = if sorted.len() % 2 == 1 {
+ sorted[sorted.len() / 2]
+ } else {
+ (sorted[sorted.len() / 2 - 1] + sorted[sorted.len() / 2]) / 2.0
+ };
+ let p95_index = ((sorted.len() as f64 - 1.0) * 0.95).ceil() as usize;
+ (median, sorted[p95_index.min(sorted.len() - 1)])
+}
+
+fn needle_of(scenario: Scenario, ctx: &RunCtx) -> String {
+ match scenario {
+ Scenario::ColdStart | Scenario::WarmStart => {
+ "first paint (the brand splash over the empty new session)".to_string()
+ }
+ Scenario::ColdReady | Scenario::WarmReady => {
+ "the attach pill (the tray's session-usage row)".to_string()
+ }
+ Scenario::AgentsView | Scenario::AgentsViewWarm => format!(
+ "the completed roster heading (Inactive ({}))",
+ ctx.manifest.markers.roster_inactive + 1
+ ),
+ Scenario::OpenSession => {
+ format!(
+ "the huge session marker {}",
+ ctx.manifest.markers.huge.needle
+ )
+ }
+ Scenario::SessionSwitch => format!(
+ "the enter drill-in to switch-b (its transcript marker {})",
+ ctx.manifest.markers.switch_b.needle
+ ),
+ Scenario::AtCompletion => "the first @ completion menu row".to_string(),
+ }
+}
+
+fn write_result(result: &BenchResult, path: &Path) -> anyhow::Result<()> {
+ let json = serde_json::to_string_pretty(result)?;
+ std::fs::write(path, json).with_context(|| format!("write {}", path.display()))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn tiny_context(root: &Path) -> RunCtx {
+ corpus::generate(&CorpusOptions {
+ out: root.to_path_buf(),
+ sessions: 3,
+ rows: 100,
+ huge_rows: 100,
+ tree_parents: 0,
+ archive: 0,
+ cwd_files: 1,
+ ..crate::corpus::Profile::Small.defaults()
+ })
+ .expect("generate the tiny corpus");
+ // A relative file is sufficient: loading does not execute it.
+ RunCtx::load(Path::new("Cargo.toml"), root, Duration::ZERO).expect("load corpus")
+ }
+
+ #[test]
+ fn the_binary_path_is_anchored_before_entering_the_corpus_project() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let ctx = tiny_context(dir.path());
+ let binary = Path::new("Cargo.toml");
+ assert_eq!(
+ ctx.binary,
+ binary.canonicalize().expect("anchor the binary")
+ );
+ }
+
+ #[test]
+ fn relative_corpus_paths_anchor_the_socket_before_changing_project() {
+ let dir = tempfile::tempdir_in(".").expect("relative corpus directory");
+ // TempDir may return an absolute path even for a relative parent.
+ // The directory lives in cwd, so its name is the caller's relative path.
+ let relative_root = Path::new(dir.path().file_name().expect("temporary directory name"));
+ assert!(
+ relative_root.is_relative(),
+ "exercise a relative caller path"
+ );
+ let ctx = tiny_context(relative_root);
+ assert!(run_socket(&ctx, 0).is_absolute());
+ assert_eq!(
+ ctx.corpus_root,
+ dir.path().canonicalize().expect("absolute root")
+ );
+ assert_ne!(ctx.project_dir, ctx.corpus_root);
+ }
+
+ #[test]
+ fn warm_results_record_the_corpus_that_was_measured() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let cold = tiny_context(&dir.path().join("cold"));
+ let warm = tiny_context(&dir.path().join("warm"));
+ for scenario in [
+ Scenario::WarmStart,
+ Scenario::WarmReady,
+ Scenario::ColdStart,
+ ] {
+ let measured = corpus_for(scenario, &cold, &warm);
+ let result = build_result(measured, scenario, vec![1.0, 2.0]);
+ let expected = if scenario == Scenario::ColdStart {
+ &cold
+ } else {
+ &warm
+ };
+ assert_eq!(result.corpus.root, expected.corpus_root);
+ }
+ }
+
+ #[test]
+ fn zero_samples_fail_before_loading_or_generating_corpora() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let absent = dir.path().join("must-not-be-created");
+ let standalone = run(&BenchOptions {
+ binary: absent.clone(),
+ corpus: absent.clone(),
+ scenario: Scenario::ColdStart,
+ runs: 0,
+ json: absent.clone(),
+ liveness_secs: 1,
+ });
+ assert_eq!(
+ standalone.expect_err("zero runs rejected").to_string(),
+ "sampled runs must be positive"
+ );
+ let suite = run_suite(&RunOptions {
+ binary: absent.clone(),
+ profile: crate::corpus::Profile::Small,
+ out: absent.clone(),
+ seed: 1,
+ runs: 0,
+ liveness_secs: 1,
+ });
+ assert_eq!(
+ suite.expect_err("zero suite runs rejected").to_string(),
+ "sampled runs must be positive"
+ );
+ assert!(!absent.exists(), "validation must precede output mutation");
+ }
+
+ #[test]
+ fn a_first_paint_failure_retains_the_raw_stream() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let dir = tempfile::tempdir().expect("temp dir");
+ let mut ctx = tiny_context(dir.path());
+ let mock = dir.path().join("mock-ui");
+ std::fs::write(
+ &mock,
+ "#!/bin/sh\nprintf 'diagnostic before missing paint\\n'\nexec /bin/cat\n",
+ )
+ .expect("write mock UI");
+ std::fs::set_permissions(&mock, std::fs::Permissions::from_mode(0o700))
+ .expect("executable mock");
+ ctx.binary = mock;
+ ctx.liveness = Duration::from_secs(1);
+ let dump = dir.path().join("failed-streams/first.bin");
+ let launched = Launch::spawn(
+ &ctx,
+ &dir.path().join("s"),
+ None,
+ b"absent paint",
+ dump.clone(),
+ );
+ assert!(
+ launched.is_err(),
+ "the mock never paints the requested needle"
+ );
+ assert!(std::fs::read(&dump)
+ .expect("failure dump")
+ .windows(b"diagnostic before missing paint".len())
+ .any(|bytes| bytes == b"diagnostic before missing paint"));
+ }
+
+ #[test]
+ fn restore_removes_links_without_traversing_external_targets() {
+ use std::os::unix::fs::symlink;
+
+ let directory = tempfile::tempdir().expect("corpus directory");
+ let ctx = tiny_context(directory.path());
+ let external = tempfile::tempdir().expect("external directory");
+ let sentinel = external.path().join("sentinel");
+ std::fs::write(&sentinel, b"external data").expect("external sentinel");
+ let directory_link = ctx.agent_dir.join("outside-dir");
+ let file_link = ctx.agent_dir.join("outside-file");
+ symlink(external.path(), &directory_link).expect("directory link");
+ symlink(&sentinel, &file_link).expect("file link");
+ restore(&ctx).expect("restore without following links");
+ assert_eq!(
+ std::fs::read(&sentinel).expect("external file retained"),
+ b"external data"
+ );
+ assert!(external.path().is_dir());
+ assert!(std::fs::symlink_metadata(directory_link).is_err());
+ assert!(std::fs::symlink_metadata(file_link).is_err());
+ for file in &ctx.manifest.files {
+ assert!(Path::new(file).exists(), "baseline file missing: {file}");
+ }
+ }
+
+ #[test]
+ fn every_standalone_sample_restores_the_manifest_baseline() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let ctx = tiny_context(dir.path());
+ let sidecar = ctx.agent_dir.join("sessions/mock.window-cache.json");
+ // Even the discarded warm-up starts from the original baseline.
+ std::fs::write(&sidecar, "previous invocation").expect("old sidecar");
+ let mut launches = Vec::new();
+ let samples = sample_runs(&ctx, 2, |index| {
+ assert!(
+ !sidecar.exists(),
+ "sample {index} inherited a prior sidecar"
+ );
+ for file in &ctx.manifest.files {
+ assert!(Path::new(file).exists(), "baseline file missing: {file}");
+ }
+ std::fs::write(&sidecar, "written by the mock scenario")?;
+ launches.push(index);
+ Ok(index as f64)
+ })
+ .expect("sample the mock scenario");
+ assert_eq!((launches, samples), (vec![0, 1, 2], vec![1.0, 2.0]));
+ }
+
+ #[test]
+ fn a_daemon_startup_timeout_kills_and_reaps_its_owned_child() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let child = std::process::Command::new("/bin/sleep")
+ .arg("60")
+ .spawn()
+ .expect("spawn the mock daemon, never the product");
+ let pid = nix::unistd::Pid::from_raw(child.id() as i32);
+ let result = DaemonChild { child }.wait_until_ready(&dir.path().join("s"), Instant::now());
+ assert!(result.is_err());
+ assert_eq!(
+ nix::sys::signal::kill(pid, None),
+ Err(nix::errno::Errno::ESRCH)
+ );
+ }
+
+ #[test]
+ fn a_measured_ui_disconnects_before_waiting_for_daemon_exit() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let dir = tempfile::tempdir().expect("temp dir");
+ let mut ctx = tiny_context(dir.path());
+ let binary = dir.path().join("mock-product.py");
+ std::fs::write(
+ &binary,
+ include_str!("../../tests/fixtures/draining_daemon.py"),
+ )
+ .expect("write the mock modes");
+ std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o700))
+ .expect("make the mock executable");
+ ctx.binary = binary;
+ ctx.liveness = Duration::from_secs(10);
+ let (sample, launched) = launch(&ctx, Scenario::WarmStart, 0)
+ .expect("the daemon must drain its measured UI connection before exit");
+ assert!(sample >= 0.0);
+ assert!(
+ launched.ready_ms.is_some(),
+ "the actual mock paint/attach ran"
+ );
+ assert_eq!(
+ std::fs::read_to_string(run_socket(&ctx, 0).with_extension("drained"))
+ .expect("the mock daemon reached normal drain completion"),
+ "all UI/control connections closed\n"
+ );
+ }
+
+ #[test]
+ fn a_daemon_that_never_sends_hello_cannot_hang_shutdown() {
+ use std::os::unix::net::UnixListener;
+
+ let dir = tempfile::tempdir().expect("temp dir");
+ // Keep the Unix socket path short on macOS too.
+ let socket = dir.path().join("s");
+ let listener = UnixListener::bind(&socket).expect("mock listener");
+ let (release, released) = std::sync::mpsc::channel();
+ let peer = std::thread::spawn(move || {
+ let (_stream, _) = listener.accept().expect("accept shutdown client");
+ // The bounded watchdog also lets this test terminate on the
+ // old implementation, where hello has no read timeout.
+ let _ = released.recv_timeout(Duration::from_secs(8));
+ });
+ let started = Instant::now();
+ protocol_shutdown(&socket);
+ let elapsed = started.elapsed();
+ let _ = release.send(());
+ peer.join().expect("mock peer exited");
+ assert!(
+ elapsed < Duration::from_secs(7),
+ "hello blocked for {elapsed:?}"
+ );
+ }
+}
diff --git a/crates/pa-perf/src/corpus/artifacts.rs b/crates/pa-perf/src/corpus/artifacts.rs
new file mode 100644
index 0000000000..75a4674a71
--- /dev/null
+++ b/crates/pa-perf/src/corpus/artifacts.rs
@@ -0,0 +1,261 @@
+//! The session-artifact trees: per-session artifact dirs (kernel-state
+//! stubs, harness state, retired heartbeat jobs), the subagent session
+//! files, and the ledger edges that make them roster-visible — all through
+//! the product's own writers where they exist.
+
+use std::path::{Path, PathBuf};
+
+use pa_core::refinement::{
+ HarnessEntry, HarnessRefinementEvent, HarnessScope, HarnessState, RefinementKind,
+};
+use pa_daemon::rlm_ledger::{RlmSpawnInput, RlmSpawnLedger};
+use serde_json::json;
+
+use crate::corpus::rows::SessionSpec;
+use crate::rng::{iso_from_millis, Rng};
+
+/// The subagent tree shape: fanout per level, total depth, and the row count
+/// of each child session file, plus whether to populate artifact partitions.
+pub(crate) struct SubagentTree {
+ pub fanout: usize,
+ pub max_depth: u32,
+ pub child_rows: usize,
+ pub artifacts: bool,
+}
+
+/// The worker-name bank for subagent ledger edges.
+const WORKER_NAMES: [&str; 12] = [
+ "worker",
+ "helper",
+ "scout",
+ "reviewer",
+ "implementer",
+ "planner",
+ "auditor",
+ "fixer",
+ "researcher",
+ "summarizer",
+ "verifier",
+ "optimizer",
+];
+
+/// A valid empty-dict pickle payload (protocol 4): the kernel-state sidecar
+/// stub unpickles cleanly to zero restored names.
+const EMPTY_PICKLE: &[u8] = b"\x80\x04}\x94.";
+
+/// Write one session's artifact partition: the kernel-state manifest and
+/// payload, plus (per `rng`) a local harness store and a retired heartbeat.
+///
+/// # Errors
+///
+/// Propagates filesystem failures.
+pub(crate) fn write_artifact_dir(
+ dir: &Path,
+ session_id: &str,
+ seed_text: &str,
+ rng: &mut Rng,
+) -> anyhow::Result<()> {
+ std::fs::create_dir_all(dir)?;
+ std::fs::write(
+ dir.join("kernel-state.json"),
+ json!({ "version": 1, "savedNames": [], "skipped": [] }).to_string(),
+ )?;
+ std::fs::write(dir.join("kernel-state.dill"), EMPTY_PICKLE)?;
+
+ if rng.chance(30) {
+ let harness_dir = dir.join("harness");
+ std::fs::create_dir_all(&harness_dir)?;
+ let entries = (1..=1 + rng.below(3))
+ .map(|index| {
+ (
+ format!("{seed_text}-memory-{index}"),
+ format!("corpus memory entry {index}: {}", seed_text),
+ )
+ })
+ .collect::<Vec<_>>();
+ let at = iso_from_millis(1_700_000_000_000 + rng.below(10_000_000));
+ let state = HarnessState {
+ schema: 1,
+ entries: [(
+ RefinementKind::Memory,
+ entries
+ .iter()
+ .map(|(id, content)| {
+ (
+ id.clone(),
+ HarnessEntry {
+ id: id.clone(),
+ kind: RefinementKind::Memory,
+ title: format!("corpus memory {}", id),
+ content: content.clone(),
+ path: "general".to_string(),
+ scope: Some(HarnessScope::Local),
+ reference: serde_json::Map::new(),
+ arguments: serde_json::Map::new(),
+ metadata: serde_json::Map::new(),
+ source: "agent".to_string(),
+ created_at: at.clone(),
+ updated_at: at.clone(),
+ version: 1,
+ },
+ )
+ })
+ .collect(),
+ )]
+ .into_iter()
+ .collect(),
+ refinements: vec![HarnessRefinementEvent {
+ id: format!("{seed_text}-event-1"),
+ trigger: "test".to_string(),
+ changes: vec![format!("create memory:{seed_text}-memory-1")],
+ evidence: String::new(),
+ outcome: String::new(),
+ created_at: at,
+ }],
+ };
+ std::fs::write(
+ harness_dir.join("harness_state.json"),
+ serde_json::to_string(&state)?,
+ )?;
+ std::fs::write(
+ harness_dir.join("refinement_history.jsonl"),
+ serde_json::to_string(&state.refinements[0])? + "\n",
+ )?;
+ }
+
+ if rng.chance(10) {
+ std::fs::write(
+ dir.join("scheduled-jobs.json"),
+ json!({
+ "jobs": [{
+ "id": format!("{:08x}-b1c7-4f80-9d0e-be9ecdc78a65", rng.below(1 << 32)),
+ "status": "cancelled",
+ "source": "rlm_heartbeat",
+ "runtimeKind": "top-level",
+ "deliveryMode": "follow_up",
+ "activeSessionId": rng.next_u64().to_string(),
+ "sessionId": session_id,
+ "sessionFile": String::new(),
+ "cwd": "/tmp",
+ "label": WORKER_NAMES[rng.below(WORKER_NAMES.len() as u64) as usize],
+ "prompt": "retired corpus heartbeat",
+ "schedule": { "kind": "interval", "expression": "every 30m", "intervalMs": 1_800_000 },
+ "createdAt": iso_from_millis(1_700_000_000_000),
+ "updatedAt": iso_from_millis(1_700_010_000_000),
+ "runCount": 0,
+ }]
+ })
+ .to_string(),
+ )?;
+ }
+ Ok(())
+}
+
+/// One subagent tree node's spawn: the child session dir + file, its own
+/// artifact partition, the ledger edge, and the recursion into grandchildren.
+/// The child session file lives at
+/// `<agent>/session-artifacts/<parent-session-id>/sub-<childId>/` (the
+/// product's `rlm_child_session_dir` layout); grandchildren key on the
+/// child's own session id at the artifact root.
+///
+/// # Errors
+///
+/// Propagates ledger and filesystem failures.
+#[allow(clippy::too_many_arguments)] // one node carries the ledger, parent identity and tree-generation context
+pub(crate) fn spawn_child(
+ ledger: &RlmSpawnLedger,
+ artifacts_root: &Path,
+ parent_session_id: &str,
+ parent_file: &Path,
+ depth: u32,
+ tree: &SubagentTree,
+ cwd: &str,
+ rng: &mut Rng,
+) -> anyhow::Result<Vec<PathBuf>> {
+ let mut written = Vec::new();
+ if depth > tree.max_depth {
+ return Ok(written);
+ }
+ for _ in 0..tree.fanout {
+ let child_tag = format!("sub-{:07x}", rng.below(1 << 28));
+ let child_dir = artifacts_root.join(parent_session_id).join(&child_tag);
+ std::fs::create_dir_all(&child_dir)?;
+ let child_start = 1_700_000_000_000 + rng.below(90_000_000_000);
+ let child_session_id = crate::corpus::rows::session_id(rng, child_start);
+ let child_file = child_dir.join(format!("{child_session_id}.jsonl"));
+ crate::corpus::rows::write_session_file(
+ &child_file,
+ &SessionSpec {
+ id: child_session_id.clone(),
+ start_ms: child_start,
+ cwd: cwd.to_string(),
+ rows: tree.child_rows,
+ name: None,
+ marker: None,
+ parent_session: Some(parent_file.display().to_string()),
+ rlm_depth: None,
+ },
+ rng,
+ )?;
+ crate::corpus::set_mtime(&child_file, child_start + 10_000)?;
+ // The child's OWN artifact partition sits under the parent's tree.
+ if tree.artifacts {
+ write_artifact_dir(
+ &artifacts_root
+ .join(parent_session_id)
+ .join("session-artifacts")
+ .join(&child_session_id),
+ &child_session_id,
+ &child_session_id,
+ rng,
+ )?;
+ }
+ ledger.append_spawn(&RlmSpawnInput {
+ child_id: child_tag.clone(),
+ parent: parent_file.display().to_string(),
+ child: child_file.display().to_string(),
+ depth,
+ name: WORKER_NAMES[rng.below(WORKER_NAMES.len() as u64) as usize].to_string(),
+ })?;
+ written.push(child_file.clone());
+ if depth < tree.max_depth {
+ written.extend(spawn_child(
+ ledger,
+ artifacts_root,
+ &child_session_id,
+ &child_file,
+ depth + 1,
+ tree,
+ cwd,
+ rng,
+ )?);
+ }
+ }
+ Ok(written)
+}
+
+#[cfg(test)]
+mod tests {
+ #[test]
+ fn generated_kernel_snapshot_unpickles_to_an_empty_dictionary() {
+ let directory = tempfile::tempdir().expect("artifact directory");
+ super::write_artifact_dir(
+ directory.path(),
+ "session",
+ "seed",
+ &mut crate::rng::Rng::new(7),
+ )
+ .expect("write real artifact partition");
+ let output = std::process::Command::new("python3")
+ .arg("-c")
+ .arg("import pickle,sys; value=pickle.load(open(sys.argv[1],'rb')); assert type(value) is dict and not value")
+ .arg(directory.path().join("kernel-state.dill"))
+ .output()
+ .expect("run hosted Python pickle reader");
+ assert!(
+ output.status.success(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ }
+}
diff --git a/crates/pa-perf/src/corpus/mod.rs b/crates/pa-perf/src/corpus/mod.rs
new file mode 100644
index 0000000000..5e0b208a6f
--- /dev/null
+++ b/crates/pa-perf/src/corpus/mod.rs
@@ -0,0 +1,728 @@
+//! The deterministic corpus generator: one `generate` pass writes the full
+//! agent-state layout under an isolated `--out` dir and returns the manifest
+//! the bench harness needles read.
+
+pub(crate) mod artifacts;
+pub(crate) mod rows;
+
+use std::path::{Path, PathBuf};
+
+use anyhow::Context;
+use clap::ValueEnum;
+use pa_daemon::rlm_ledger::RlmSpawnLedger;
+use serde::{Deserialize, Serialize};
+
+use crate::rng::Rng;
+use artifacts::SubagentTree;
+use rows::SessionSpec;
+
+/// The named corpus profiles (sizes overridable knob by knob).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ValueEnum)]
+#[serde(rename_all = "lowercase")]
+pub enum Profile {
+ Small,
+ Medium,
+ Large,
+ Xl,
+}
+
+impl Profile {
+ /// The profile's knob defaults.
+ #[must_use]
+ pub fn defaults(self) -> CorpusOptions {
+ let (sessions, rows, child_rows, fanout, depth, tree_parents, wide, archive, cwd_files) =
+ match self {
+ Self::Small => (10, 100, 40, 2, 2, 2, 0, 3, 200),
+ Self::Medium => (100, 2_000, 100, 3, 2, 5, 0, 20, 2_000),
+ Self::Large => (1_000, 10_000, 150, 3, 3, 8, 110, 200, 4_000),
+ Self::Xl => (2_500, 400, 80, 2, 2, 10, 0, 500, 2_000),
+ };
+ CorpusOptions {
+ profile: self,
+ out: PathBuf::new(),
+ seed: 7,
+ sessions,
+ rows,
+ child_rows,
+ huge_rows: rows.saturating_mul(2).max(2_000),
+ fanout,
+ depth,
+ tree_parents,
+ wide_children: wide,
+ archive,
+ cwd_files,
+ artifacts: true,
+ cache: CacheMode::Cold,
+ }
+ }
+}
+
+/// Whether the warm-up pass runs the product's window-store open over the
+/// generated session files (writing their window-cache sidecars).
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ValueEnum)]
+#[serde(rename_all = "lowercase")]
+pub enum CacheMode {
+ Cold,
+ Warm,
+}
+
+/// Every corpus knob; all overridable from the CLI (README.md documents each).
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CorpusOptions {
+ pub profile: Profile,
+ pub out: PathBuf,
+ pub seed: u64,
+ /// Top-level sessions in the sessions dir.
+ pub sessions: usize,
+ /// Rows per session (the huge session gets `huge_rows`).
+ pub rows: usize,
+ /// Rows per subagent child session.
+ pub child_rows: usize,
+ /// Rows of the one huge session (the open-session scenario's target).
+ pub huge_rows: usize,
+ /// Subagent children per parent per level.
+ pub fanout: usize,
+ /// Subagent tree depth (levels below the parent session).
+ pub depth: u32,
+ /// How many sessions grow a subagent tree.
+ pub tree_parents: usize,
+ /// One parent grows this many direct children (the operator's ~110-wide
+ /// real parent); 0 disables.
+ pub wide_children: usize,
+ /// Sessions in sessions-archive/.
+ pub archive: usize,
+ /// Files in the generated project dir (the sessions' cwd, the
+ /// @-completion walk's tree).
+ pub cwd_files: usize,
+ /// Write the per-session artifact partitions.
+ pub artifacts: bool,
+ pub cache: CacheMode,
+}
+
+/// The session-name task bank (roster rows carry these as `session_info`).
+const TASKS: [&str; 16] = [
+ "ship the bench harness",
+ "review the corpus generator",
+ "fix the roster scan",
+ "port the window cache",
+ "audit the ledger replay",
+ "write the CI workflow",
+ "profile the daemon boot",
+ "tune the pty harness",
+ "split the scan fold",
+ "measure first paint",
+ "repair the lease release",
+ "seed the archive sweep",
+ "draft the README",
+ "run the regression gates",
+ "bisect the paint delay",
+ "close the perf budget",
+];
+
+/// A fixed generation base (unix ms): sessions spread back from it, so the
+/// tree is seed-deterministic — never wall-clock dependent. (The RLM ledger's
+/// own `at` stamps come from the product writer; see README's determinism
+/// note.)
+const BASE_MS: u64 = 1_790_000_000_000;
+/// The mtime step between sessions (roster order is mtime-desc; the
+/// spread is realism only — the archive-off settings are the sweep guard).
+const MTIME_STEP_MS: u64 = 720_000;
+/// The header-timestamp hour step (a session's rows spread over an hour).
+const HOUR_MS: u64 = 3_600_000;
+
+/// The corpus manifest: what the bench reads (needles are DATA, never
+/// layout-position), and the file-set baseline the per-run restore uses.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Manifest {
+ pub schema: u32,
+ pub profile: Profile,
+ pub seed: u64,
+ pub paths: ManifestPaths,
+ pub counts: ManifestCounts,
+ /// Every generated file under the corpus root (the restore baseline:
+ /// the bench deletes anything else the product wrote between runs).
+ pub files: Vec<String>,
+ pub markers: ManifestMarkers,
+}
+
+/// The corpus's on-disk anchor paths (absolute, canonical).
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ManifestPaths {
+ pub agent_dir: PathBuf,
+ pub sessions_dir: PathBuf,
+ pub archive_dir: PathBuf,
+ pub project_dir: PathBuf,
+ pub manifest_file: PathBuf,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ManifestCounts {
+ pub sessions: usize,
+ pub archive: usize,
+ pub child_sessions: usize,
+ pub total_rows: usize,
+ pub subagent_edges: usize,
+ pub cwd_files: usize,
+}
+
+/// The bench needles: marker strings and expected rendered counts, all
+/// recorded by the generator.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ManifestMarkers {
+ pub huge: MarkerSession,
+ pub switch_a: MarkerSession,
+ pub switch_b: MarkerSession,
+ /// The agents view's expected `Inactive (N)` heading count.
+ pub roster_inactive: usize,
+ /// The session-switch search query (uniquely matches the switch-b row).
+ pub switch_query: String,
+ /// The switch-b row's rendered name (the picker paints it fully on the
+ /// empty-picker -> hit transition; the echoed query does not contain it).
+ pub switch_row: String,
+ /// The @-completion target file name under the project dir.
+ pub at_file: String,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct MarkerSession {
+ pub id: String,
+ pub file: PathBuf,
+ pub needle: String,
+}
+
+/// One marker session captured during generation.
+struct Marker {
+ id: String,
+ file: PathBuf,
+}
+
+impl Marker {
+ fn session(&self, needle: String) -> MarkerSession {
+ MarkerSession {
+ id: self.id.clone(),
+ file: self.file.clone(),
+ needle,
+ }
+ }
+}
+
+/// Generate the corpus: the full agent-state tree under `options.out`,
+/// then the manifest at `corpus.json`.
+///
+/// # Errors
+///
+/// Propagates filesystem, serialization, and ledger failures.
+pub fn generate(options: &CorpusOptions) -> anyhow::Result<Manifest> {
+ anyhow::ensure!(
+ options.sessions >= 3,
+ "the corpus needs at least 3 sessions (the huge, switch-a, and switch-b markers)"
+ );
+ let out = canonical_root(&options.out)?;
+ anyhow::ensure!(
+ std::fs::read_dir(&out)?.next().transpose()?.is_none(),
+ "corpus output directory must be empty: {}",
+ out.display()
+ );
+ let agent_dir = out.join("agent");
+ let sessions_dir = agent_dir.join("sessions");
+ let archive_dir = agent_dir.join("sessions-archive");
+ let project_dir = out.join("project");
+ for dir in [&agent_dir, &sessions_dir, &archive_dir, &project_dir] {
+ std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;
+ }
+ let cwd = project_dir.display().to_string();
+
+ // Onboarding is pre-answered, traces stay off, and BOTH archive rules
+ // are disabled: the daemon's boot sweep would otherwise retire the
+ // corpus's own sessions (the defaults keep 200 files / 30 days) and
+ // move them out from under the bench's manifest baseline.
+ std::fs::write(
+ agent_dir.join("settings.json"),
+ serde_json::json!({
+ "onboardingShown": true,
+ "agentTraces": { "enabled": false },
+ "sessionArchiveMaxAgeDays": "off",
+ "sessionArchiveMaxSessions": "off",
+ })
+ .to_string(),
+ )?;
+ let at_file = write_project(&project_dir, options.cwd_files)?;
+
+ let mut rng = Rng::new(options.seed);
+ let mut total_rows = 0usize;
+ let mut child_sessions = 0usize;
+ let ledger = RlmSpawnLedger::new(&agent_dir, &sessions_dir, |_| {});
+
+ let mut mtime = BASE_MS + MTIME_STEP_MS;
+ let mut huge: Option<Marker> = None;
+ let mut switch_a: Option<Marker> = None;
+ let mut switch_b: Option<Marker> = None;
+ for index in 0..options.sessions {
+ let start = BASE_MS - index as u64 * MTIME_STEP_MS;
+ let (rows, name, marker) = match index {
+ 0 => (
+ options.huge_rows,
+ Some("huge corpus session".to_string()),
+ Some(format!("corpus huge marker {:x}", options.seed)),
+ ),
+ 1 => (
+ options.rows,
+ Some("switch target a".to_string()),
+ Some(format!("corpus switch marker a {:x}", options.seed)),
+ ),
+ // The `zq` token makes switch-b the agents-view search's ONLY
+ // possible hit: no generated text (names or messages) contains
+ // the letter z, so the query can never rank another row first.
+ 2 => (
+ options.rows,
+ Some("switch target zqmark".to_string()),
+ Some(format!("corpus switch marker b {:x}", options.seed)),
+ ),
+ _ => (
+ options.rows,
+ rng.chance(70)
+ .then(|| format!("{} #{index}", TASKS[rng.below(TASKS.len() as u64) as usize])),
+ None,
+ ),
+ };
+ let id = rows::session_id(&mut rng, start);
+ let file = sessions_dir.join(format!("{id}.jsonl"));
+ let written = rows::write_session_file(
+ &file,
+ &SessionSpec {
+ id: id.clone(),
+ start_ms: start,
+ cwd: cwd.clone(),
+ rows,
+ name,
+ marker: marker.clone(),
+ parent_session: None,
+ rlm_depth: Some(0),
+ },
+ &mut rng,
+ )?;
+ total_rows += written.rows;
+ if let Some(marker) = &marker {
+ anyhow::ensure!(
+ written.last_user_text == *marker,
+ "the {} session's marker did not land on the final user row",
+ file.display()
+ );
+ let captured = Marker {
+ id: id.clone(),
+ file: file.clone(),
+ };
+ match marker.as_str() {
+ m if m.starts_with("corpus huge") => huge = Some(captured),
+ m if m.starts_with("corpus switch marker a") => switch_a = Some(captured),
+ m if m.starts_with("corpus switch marker b") => switch_b = Some(captured),
+ _ => {}
+ }
+ }
+ set_mtime(&file, mtime)?;
+ mtime -= MTIME_STEP_MS;
+
+ if options.artifacts {
+ let artifacts_root = agent_dir.join("session-artifacts");
+ artifacts::write_artifact_dir(&artifacts_root.join(&id), &id, &id, &mut rng)?;
+ }
+
+ // Subagent trees: the first `tree_parents` sessions grow one; the
+ // first of those also grows the wide-children set.
+ if index > 2 && index <= 2 + options.tree_parents {
+ let tree = SubagentTree {
+ fanout: options.fanout,
+ max_depth: options.depth,
+ child_rows: options.child_rows,
+ artifacts: options.artifacts,
+ };
+ child_sessions += spawn_and_count(
+ &ledger,
+ &agent_dir.join("session-artifacts"),
+ &id,
+ &file,
+ &tree,
+ &cwd,
+ &mut rng,
+ &mut total_rows,
+ )?;
+ if index == 3 && options.wide_children > 0 {
+ let wide_tree = SubagentTree {
+ fanout: options.wide_children,
+ max_depth: 1,
+ child_rows: options.child_rows,
+ artifacts: options.artifacts,
+ };
+ child_sessions += spawn_and_count(
+ &ledger,
+ &agent_dir.join("session-artifacts"),
+ &id,
+ &file,
+ &wide_tree,
+ &cwd,
+ &mut rng,
+ &mut total_rows,
+ )?;
+ }
+ }
+ }
+
+ // Archived sessions: plain files in the archive dir (older mtimes).
+ for index in 0..options.archive {
+ let start = BASE_MS / 2 - index as u64 * HOUR_MS;
+ let id = rows::session_id(&mut rng, start);
+ let file = archive_dir.join(format!("{id}.jsonl"));
+ let written = rows::write_session_file(
+ &file,
+ &SessionSpec {
+ id: id.clone(),
+ start_ms: start,
+ cwd: cwd.clone(),
+ rows: options.rows,
+ name: None,
+ marker: None,
+ parent_session: None,
+ rlm_depth: Some(0),
+ },
+ &mut rng,
+ )?;
+ total_rows += written.rows;
+ set_mtime(&file, start + HOUR_MS)?;
+ }
+
+ // The warm-up pass: the product's own window-store open writes the
+ // window-cache sidecars (the info-cache sidecar has no product writer
+ // outside a worker lease release; the bench measures that warmth via
+ // the daemon's own process cache — see README).
+ if options.cache == CacheMode::Warm {
+ warm_sidecars(&sessions_dir, &archive_dir)?;
+ }
+
+ let manifest = Manifest {
+ schema: 1,
+ profile: options.profile,
+ seed: options.seed,
+ paths: ManifestPaths {
+ agent_dir,
+ sessions_dir,
+ archive_dir,
+ project_dir,
+ manifest_file: out.join("corpus.json"),
+ },
+ counts: ManifestCounts {
+ sessions: options.sessions,
+ archive: options.archive,
+ child_sessions,
+ total_rows,
+ subagent_edges: child_sessions,
+ cwd_files: options.cwd_files,
+ },
+ files: collect_files(&out)?,
+ markers: ManifestMarkers {
+ huge: marker_session(huge, &format!("corpus huge marker {:x}", options.seed))?,
+ switch_a: marker_session(
+ switch_a,
+ &format!("corpus switch marker a {:x}", options.seed),
+ )?,
+ switch_b: marker_session(
+ switch_b,
+ &format!("corpus switch marker b {:x}", options.seed),
+ )?,
+ roster_inactive: options.sessions,
+ switch_query: "target zqmark".to_string(),
+ switch_row: "switch target zqmark".to_string(),
+ at_file: at_file.clone(),
+ },
+ };
+ std::fs::write(
+ out.join("corpus.json"),
+ serde_json::to_string_pretty(&manifest)?,
+ )?;
+ Ok(manifest)
+}
+
+/// Replace only a prior corpus owned by this suite; arbitrary populated output
+/// remains an error. remove_dir_all does not follow nested directory symlinks.
+#[cfg(any(unix, test))]
+pub(crate) fn regenerate_for_suite(options: &CorpusOptions) -> anyhow::Result<Manifest> {
+ anyhow::ensure!(
+ options.sessions >= 3,
+ "the corpus needs at least 3 sessions"
+ );
+ let root = &options.out;
+ if let Ok(metadata) = std::fs::symlink_metadata(root) {
+ anyhow::ensure!(
+ !metadata.file_type().is_symlink(),
+ "suite corpus root must not be a symlink: {}",
+ root.display()
+ );
+ }
+ if root.exists() && std::fs::read_dir(root)?.next().transpose()?.is_some() {
+ let canonical = root.canonicalize()?;
+ let manifest: Manifest =
+ serde_json::from_slice(&std::fs::read(canonical.join("corpus.json"))?)
+ .context("refuse to replace output without a corpus manifest")?;
+ anyhow::ensure!(
+ manifest.schema == 1
+ && manifest.paths.manifest_file == canonical.join("corpus.json")
+ && manifest.paths.agent_dir == canonical.join("agent")
+ && manifest.paths.project_dir == canonical.join("project"),
+ "refuse to replace a corpus whose manifest belongs to another root"
+ );
+ std::fs::remove_dir_all(root)?;
+ }
+ generate(options)
+}
+
+/// A one-child spawn through the product's ledger, then the recursion's row
+/// accounting.
+#[allow(clippy::too_many_arguments)] // the spawn inputs are the tree site's own facts
+fn spawn_and_count(
+ ledger: &RlmSpawnLedger,
+ artifacts_root: &Path,
+ id: &str,
+ file: &Path,
+ tree: &SubagentTree,
+ cwd: &str,
+ rng: &mut Rng,
+ total_rows: &mut usize,
+) -> anyhow::Result<usize> {
+ let children = artifacts::spawn_child(ledger, artifacts_root, id, file, 1, tree, cwd, rng)?;
+ for child in &children {
+ *total_rows += count_rows(child);
+ }
+ Ok(children.len())
+}
+
+fn marker_session(marker: Option<Marker>, needle: &str) -> anyhow::Result<MarkerSession> {
+ marker.map_or_else(
+ || anyhow::bail!("the marker session was not generated"),
+ |marker| Ok(marker.session(needle.to_string())),
+ )
+}
+
+/// Deterministic project tree: five dirs, plain content files, plus the
+/// @-completion marker file the bench needles on.
+fn write_project(project: &Path, files: usize) -> anyhow::Result<String> {
+ let dirs = ["src", "src/kernel", "tests", "docs", "scripts"];
+ for dir in dirs {
+ std::fs::create_dir_all(project.join(dir))?;
+ }
+ for index in 0..files {
+ let dir = dirs[index % dirs.len()];
+ let name = if index % 50 == 0 {
+ format!("mod{index}.rs")
+ } else {
+ format!("file{index:05}.rs")
+ };
+ std::fs::write(
+ project.join(dir).join(name),
+ format!("// corpus file {index}\nfn stub{index}() {{}}\n"),
+ )?;
+ }
+ let marker = "zzq-corpus-marker.md";
+ std::fs::write(project.join(marker), "# corpus @ completion marker file\n")?;
+ Ok(marker.to_string())
+}
+
+pub(crate) fn set_mtime(path: &Path, millis: u64) -> anyhow::Result<()> {
+ let time = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_millis(millis);
+ let file = std::fs::File::options().write(true).open(path)?;
+ file.set_times(
+ std::fs::FileTimes::new()
+ .set_modified(time)
+ .set_accessed(time),
+ )?;
+ Ok(())
+}
+
+fn count_rows(path: &Path) -> usize {
+ std::fs::read_to_string(path)
+ .map(|text| text.lines().filter(|line| !line.is_empty()).count())
+ .unwrap_or(0)
+}
+
+fn warm_sidecars(sessions_dir: &Path, archive_dir: &Path) -> anyhow::Result<()> {
+ for dir in [sessions_dir, archive_dir] {
+ for entry in std::fs::read_dir(dir)? {
+ let path = entry?.path();
+ if path.extension().is_some_and(|ext| ext == "jsonl") {
+ let _ = pa_core::session::window::WindowedSessionStore::open(&path)?;
+ }
+ }
+ }
+ Ok(())
+}
+
+fn collect_files(root: &Path) -> anyhow::Result<Vec<String>> {
+ fn walk(dir: &Path, out: &mut Vec<String>) -> anyhow::Result<()> {
+ let mut entries = std::fs::read_dir(dir)?.collect::<Result<Vec<_>, _>>()?;
+ entries.sort_by_key(|entry| entry.file_name());
+ for entry in entries {
+ let path = entry.path();
+ if path.is_dir() {
+ walk(&path, out)?;
+ } else {
+ out.push(path.display().to_string());
+ }
+ }
+ Ok(())
+ }
+ let mut files = Vec::new();
+ walk(root, &mut files)?;
+ files.sort();
+ Ok(files)
+}
+
+fn canonical_root(out: &Path) -> anyhow::Result<PathBuf> {
+ std::fs::create_dir_all(out).with_context(|| format!("create {}", out.display()))?;
+ // macOS /tmp -> /private/tmp: the ledger hash, the header parentSession
+ // paths, and every ledger record key on canonical paths.
+ out.canonicalize()
+ .with_context(|| format!("canonicalize {}", out.display()))
+}
+
+#[cfg(test)]
+mod tests {
+ fn tiny_options(out: &std::path::Path) -> super::CorpusOptions {
+ let mut options = super::Profile::Small.defaults();
+ options.out = out.to_path_buf();
+ options.sessions = 4;
+ options.rows = 8;
+ options.huge_rows = 8;
+ options.child_rows = 8;
+ options.archive = 0;
+ options.cwd_files = 0;
+ options.tree_parents = 1;
+ options.wide_children = 0;
+ options.artifacts = false;
+ options
+ }
+
+ #[test]
+ fn generation_rejects_stale_output_without_changing_existing_files() {
+ let directory = tempfile::tempdir().expect("output directory");
+ let options = tiny_options(directory.path());
+ super::generate(&options).expect("generate initial corpus");
+ let sentinel = directory.path().join("unrelated.txt");
+ std::fs::write(&sentinel, b"keep this file").expect("write unrelated file");
+ let manifest = std::fs::read(directory.path().join("corpus.json")).expect("manifest");
+ let error = super::generate(&options).expect_err("reject previously populated output");
+ assert!(error.to_string().contains("must be empty"), "{error:#}");
+ assert_eq!(
+ std::fs::read(&sentinel).expect("sentinel retained"),
+ b"keep this file"
+ );
+ assert_eq!(
+ std::fs::read(directory.path().join("corpus.json")).expect("old manifest"),
+ manifest
+ );
+ }
+
+ #[test]
+ fn depth_zero_generates_no_recursive_children_or_spawn_edges() {
+ let directory = tempfile::tempdir().expect("output directory");
+ let mut options = tiny_options(directory.path());
+ options.depth = 0;
+ options.fanout = 3;
+ let manifest = super::generate(&options).expect("generate depth-zero corpus");
+ assert_eq!(manifest.counts.child_sessions, 0);
+ assert_eq!(manifest.counts.subagent_edges, 0);
+ assert!(!manifest.files.iter().any(|file| file.contains("sub-")));
+ }
+
+ #[test]
+ fn artifact_option_preserves_deep_and_wide_children_without_optional_payloads() {
+ for artifacts in [false, true] {
+ let directory = tempfile::tempdir().expect("output directory");
+ let mut options = tiny_options(directory.path());
+ options.artifacts = artifacts;
+ options.fanout = 2;
+ options.depth = 2;
+ options.wide_children = 3;
+ let manifest = super::generate(&options).expect("generate child trees");
+ // Two direct children, four grandchildren, and three wide children.
+ assert_eq!(manifest.counts.child_sessions, 9);
+ let ledger = pa_daemon::rlm_ledger::RlmSpawnLedger::new(
+ &manifest.paths.agent_dir,
+ &manifest.paths.sessions_dir,
+ |_| {},
+ );
+ let edges = ledger.live_edges().expect("replay actual spawn ledger");
+ assert_eq!(edges.len(), 9);
+ assert!(edges.iter().any(|edge| edge.depth == 2));
+ for edge in edges {
+ assert!(std::path::Path::new(&edge.child).is_file());
+ assert!(manifest.files.contains(&edge.child));
+ }
+ let payloads = manifest
+ .files
+ .iter()
+ .filter(|file| {
+ file.ends_with("kernel-state.json") || file.ends_with("kernel-state.dill")
+ })
+ .count();
+ if artifacts {
+ assert_eq!(payloads, 2 * (options.sessions + 9));
+ } else {
+ assert_eq!(payloads, 0);
+ assert!(!manifest.files.iter().any(|file| {
+ file.contains("/harness/") || file.ends_with("scheduled-jobs.json")
+ }));
+ }
+ }
+ }
+
+ #[test]
+ fn suite_regeneration_replaces_owned_corpora_without_admitting_stale_files() {
+ let directory = tempfile::tempdir().expect("suite output");
+ let options = tiny_options(&directory.path().join("corpus-cold"));
+ super::generate(&options).expect("initial corpus");
+ let stale = options.out.join("agent/sessions/stale.jsonl");
+ std::fs::write(&stale, "stale transcript").expect("stale file");
+ let manifest = super::regenerate_for_suite(&options).expect("repeat suite generation");
+ assert!(!stale.exists());
+ assert!(!manifest
+ .files
+ .iter()
+ .any(|file| file.ends_with("stale.jsonl")));
+ assert_eq!(manifest.counts.sessions, options.sessions);
+ }
+
+ #[test]
+ fn suite_regeneration_refuses_unowned_populated_output() {
+ let directory = tempfile::tempdir().expect("suite output");
+ let sentinel = directory.path().join("unrelated.txt");
+ std::fs::write(&sentinel, "preserve me").expect("unrelated file");
+ assert!(super::regenerate_for_suite(&tiny_options(directory.path())).is_err());
+ assert_eq!(std::fs::read_to_string(sentinel).unwrap(), "preserve me");
+ }
+
+ #[cfg(unix)]
+ #[test]
+ fn suite_regeneration_does_not_follow_external_directory_symlinks() {
+ let directory = tempfile::tempdir().expect("suite output");
+ let outside = tempfile::tempdir().expect("outside tree");
+ let sentinel = outside.path().join("sentinel");
+ std::fs::write(&sentinel, "outside survives").expect("sentinel");
+ let options = tiny_options(&directory.path().join("corpus-cold"));
+ super::generate(&options).expect("initial corpus");
+ std::os::unix::fs::symlink(outside.path(), options.out.join("external-link"))
+ .expect("external symlink");
+ super::regenerate_for_suite(&options).expect("replace owned corpus");
+ assert_eq!(
+ std::fs::read_to_string(sentinel).unwrap(),
+ "outside survives"
+ );
+ }
+ #[cfg(unix)]
+ #[test]
+ fn suite_regeneration_rejects_even_an_empty_external_root_symlink() {
+ let directory = tempfile::tempdir().expect("suite output");
+ let outside = tempfile::tempdir().expect("empty outside directory");
+ let link = directory.path().join("corpus-cold");
+ std::os::unix::fs::symlink(outside.path(), &link).expect("root symlink");
+ assert!(super::regenerate_for_suite(&tiny_options(&link)).is_err());
+ assert!(std::fs::read_dir(outside.path()).unwrap().next().is_none());
+ }
+}
diff --git a/crates/pa-perf/src/corpus/rows.rs b/crates/pa-perf/src/corpus/rows.rs
new file mode 100644
index 0000000000..721e7b7ddb
--- /dev/null
+++ b/crates/pa-perf/src/corpus/rows.rs
@@ -0,0 +1,560 @@
+//! One session file's rows: the entry builders over the product's own
+//! serializers (pa-types payloads inside pa-daemon's `SessionEntry`), the
+//! conversation turn generator, and the marker discipline the bench
+//! needles read from the corpus manifest.
+
+use std::io::Write;
+use std::path::Path;
+
+use pa_daemon::session_store::{session_header_line, SessionEntry};
+use pa_types::ai::{
+ AssistantContentBlock, AssistantMessage, StopReason, TextContent, ThinkingContent, ToolCall,
+ ToolResultMessage, Usage, UsageCost, UserContent, UserContentBlock, UserMessage,
+};
+use pa_types::session::{
+ CompactionEntry, CustomMessageEntry, ModelChangeEntry, ServiceTierChangeEntry, SessionHeader,
+ SessionState, SessionStateEntry, SessionStateStatus, ThinkingLevelChangeEntry,
+};
+
+use crate::rng::{iso_from_millis, Rng};
+
+/// The top-level tools the product exposes to the model.
+const TOOLS: [&str; 5] = ["bash", "edit", "python_repl", "websearch", "attach_image"];
+/// The `custom_message` customType bank (a measured real-session mix).
+const CUSTOM_TYPES: [&str; 8] = [
+ "async_bash_completion",
+ "heartbeat_prompt",
+ "agent_message",
+ "rlm_child_terminal_notice",
+ "session_slash_command",
+ "ipython_state",
+ "provider_retry_outcome",
+ "session_renamed",
+];
+/// Providers and models a real store carries. No id contains the letter
+/// z: the switch-b row's `zqmark` name must be the only z in the corpus so
+/// the switch query can never fuzzy-match another row.
+const MODELS: [(&str, &str); 5] = [
+ ("prime-inference", "glm-5.2"),
+ ("prime-inference", "openai/gpt-5.6-sol"),
+ ("anthropic", "claude-5-sonnet"),
+ ("openai", "gpt-5.4"),
+ ("google", "gemini-4-pro"),
+];
+/// The user-message word bank: plain lowercase words, joined into sentences.
+const WORDS: [&str; 49] = [
+ "the",
+ "quick",
+ "bench",
+ "corpus",
+ "session",
+ "kernel",
+ "worker",
+ "daemon",
+ "roster",
+ "scan",
+ "window",
+ "cache",
+ "entry",
+ "branch",
+ "leaf",
+ "header",
+ "row",
+ "fold",
+ "usage",
+ "token",
+ "model",
+ "provider",
+ "stream",
+ "turn",
+ "tool",
+ "result",
+ "call",
+ "prompt",
+ "context",
+ "compaction",
+ "summary",
+ "refine",
+ "harness",
+ "memory",
+ "skill",
+ "label",
+ "tree",
+ "spawn",
+ "child",
+ "depth",
+ "fanout",
+ "seed",
+ "profile",
+ "median",
+ "p95",
+ "latency",
+ "paint",
+ "frame",
+ "needle",
+];
+/// Bash-output shapes for tool results.
+const TOOL_OUTPUTS: [&str; 6] = [
+ "warning: unused variable: `baseline`\n --> src/lib.rs:42:13\n",
+ "test result: ok. 96 passed; 0 failed; 3 ignored\n",
+ "commit 33da15a2c (HEAD -> main)\nAuthor: dev <[email protected]>\n",
+ "total 24\ndrwxr-xr-x 7 user staff 224 Oct 14 09:02 .\n",
+ "updating crates.io index\n Compiling pa-perf v0.10.0\n",
+ "{\"median_ms\": 812.3, \"p95_ms\": 941.0, \"runs\": 10}\n",
+];
+
+/// One session to generate.
+pub(crate) struct SessionSpec {
+ pub id: String,
+ /// Session-start unix milliseconds; entry timestamps derive from it.
+ pub start_ms: u64,
+ pub cwd: String,
+ pub rows: usize,
+ pub name: Option<String>,
+ /// A unique string placed in the LAST user message (the bench needle:
+ /// the transcript paints the session tail on open).
+ pub marker: Option<String>,
+ pub parent_session: Option<String>,
+ pub rlm_depth: Option<u64>,
+}
+
+/// What one generated session produced.
+pub(crate) struct Written {
+ pub rows: usize,
+ /// The final user-message text (the marker, when one was requested).
+ pub last_user_text: String,
+}
+
+/// The deterministic uuid-v7-shaped id stream: timestamp prefix + rng tail
+/// with the version/variant nibbles v7 carries.
+pub(crate) fn session_id(rng: &mut Rng, start_ms: u64) -> String {
+ let rand_a = (rng.next_u64() & 0xFFFF) as u32;
+ let rand_b = rng.next_u64();
+ let tail = rng.next_u64();
+ let a = (start_ms & 0xFFFF_FFFF) as u32;
+ format!(
+ "{a:08x}-{rand_a:04x}-7{:03x}-8{:03x}-{:05x}{:07x}",
+ (rand_b >> 20) & 0xFFF,
+ (tail >> 52) as u32 & 0xFFF,
+ (rand_b & 0x000F_FFFF) as u32,
+ (tail & 0x0FFF_FFFF) as u32,
+ )
+}
+
+fn sentence(rng: &mut Rng, words: usize) -> String {
+ (0..words)
+ .map(|_| WORDS[rng.below(WORDS.len() as u64) as usize])
+ .collect::<Vec<_>>()
+ .join(" ")
+}
+
+fn user_text(rng: &mut Rng) -> String {
+ (0..(1 + rng.below(3)))
+ .map(|_| {
+ let words = 6 + rng.below(12);
+ let mut text = sentence(rng, words as usize);
+ if rng.chance(12) {
+ text.push_str(&format!(" ({})", sentence(rng, 2)));
+ }
+ text.push('.');
+ text
+ })
+ .collect::<String>()
+}
+
+fn assistant_text(rng: &mut Rng) -> String {
+ let mut text = String::new();
+ for _ in 0..(1 + rng.below(3)) {
+ for _ in 0..(2 + rng.below(4)) {
+ let words = 8 + rng.below(16);
+ text.push_str(&sentence(rng, words as usize));
+ text.push_str(". ");
+ }
+ text.push('\n');
+ }
+ text
+}
+
+fn tool_output(rng: &mut Rng) -> String {
+ let mut text = TOOL_OUTPUTS[rng.below(TOOL_OUTPUTS.len() as u64) as usize].to_string();
+ text.push_str(&sentence(rng, 3));
+ text.push('\n');
+ text
+}
+
+fn usage(rng: &mut Rng) -> Usage {
+ let input = 500 + rng.below(8000);
+ let cache_read = input * (50 + rng.below(46)) / 100;
+ let output = 50 + rng.below(2000);
+ let cache_write = rng.below(500);
+ let total = input + output + cache_read + cache_write;
+ let cost = |n: u64| pa_types::JsNumber::from(n as f64 / 1000.0);
+ Usage {
+ input,
+ output,
+ cache_read,
+ cache_write,
+ total_tokens: total,
+ cost: UsageCost {
+ input: cost(input),
+ output: cost(output),
+ cache_read: cost(cache_read),
+ cache_write: cost(cache_write),
+ total: cost(total),
+ },
+ }
+}
+
+/// The row writer: one `SessionEntry` line, exactly as the product's store
+/// serializes it (`type`, `id`, `parentId`, `timestamp`, then the payload).
+struct Rows {
+ writer: std::io::BufWriter<std::fs::File>,
+ counter: u64,
+ millis: u64,
+ parent: Option<String>,
+}
+
+impl Rows {
+ fn next_id(&mut self) -> String {
+ let id = format!("{:08x}", self.counter);
+ self.counter += 1;
+ id
+ }
+
+ fn push(
+ &mut self,
+ type_: &str,
+ fields: serde_json::Value,
+ rng: &mut Rng,
+ ) -> anyhow::Result<()> {
+ self.millis += 20 + rng.below(400);
+ let entry = SessionEntry {
+ type_: type_.to_string(),
+ id: self.next_id(),
+ parent_id: self.parent.clone(),
+ timestamp: iso_from_millis(self.millis),
+ fields,
+ };
+ let mut line = serde_json::to_string(&entry).expect("the entry serializes");
+ line.push('\n');
+ self.writer.write_all(line.as_bytes())?;
+ self.parent = Some(entry.id);
+ Ok(())
+ }
+
+ fn rows_written(&self) -> u64 {
+ self.counter
+ }
+}
+
+/// One session file: the header line, then the chained entries. Returns the
+/// row count and the last user-message text.
+///
+/// # Errors
+///
+/// Propagates serialization and filesystem failures.
+pub(crate) fn write_session_file(
+ path: &Path,
+ spec: &SessionSpec,
+ rng: &mut Rng,
+) -> anyhow::Result<Written> {
+ let (provider, model) = MODELS[rng.below(MODELS.len() as u64) as usize];
+ let header = SessionHeader {
+ version: Some(3),
+ id: spec.id.clone(),
+ timestamp: iso_from_millis(spec.start_ms),
+ cwd: spec.cwd.clone(),
+ parent_session: spec.parent_session.clone(),
+ rlm_depth: spec.rlm_depth,
+ git: None,
+ rest: serde_json::Map::new(),
+ };
+ let file = std::fs::File::create(path)?;
+ let mut rows = Rows {
+ writer: std::io::BufWriter::new(file),
+ counter: 0,
+ millis: spec.start_ms,
+ parent: None,
+ };
+ let mut header_line =
+ serde_json::to_string(&session_header_line(&header)).expect("the header serializes");
+ header_line.push('\n');
+ rows.writer.write_all(header_line.as_bytes())?;
+
+ rows.push(
+ "model_change",
+ serde_json::to_value(ModelChangeEntry {
+ provider: provider.to_string(),
+ model_id: model.to_string(),
+ })?,
+ rng,
+ )?;
+ rows.push(
+ "thinking_level_change",
+ serde_json::to_value(ThinkingLevelChangeEntry {
+ thinking_level: if rng.chance(70) { "high" } else { "medium" }.to_string(),
+ })?,
+ rng,
+ )?;
+ rows.push(
+ "service_tier_change",
+ serde_json::to_value(ServiceTierChangeEntry {
+ service_tier: Some(pa_types::ai::ServiceTier::Default),
+ })?,
+ rng,
+ )?;
+ rows.push(
+ "session_state",
+ serde_json::to_value(SessionStateEntry {
+ state: SessionState {
+ status: SessionStateStatus::Active,
+ },
+ })?,
+ rng,
+ )?;
+ if let Some(name) = &spec.name {
+ rows.push("session_info", serde_json::json!({ "name": name }), rng)?;
+ }
+
+ // The conversation: fill to `rows`, keeping a compaction boundary near
+ // 60% for long sessions and the marker on the final user message.
+ let target = spec.rows as u64;
+ let mut compaction_at = (target >= 1500).then(|| target * 6 / 10);
+ let mut last_user_text = String::new();
+ while rows.rows_written() < target {
+ if compaction_at.is_some_and(|threshold| rows.rows_written() >= threshold) {
+ compaction_at = None;
+ let summary = format!("Compacted earlier entries: {}.", sentence(rng, 6));
+ rows.push(
+ "compaction",
+ serde_json::to_value(CompactionEntry {
+ summary: summary.clone(),
+ first_kept_entry_id: format!("{:08x}", rows.counter),
+ tokens_before: 40_000 + rng.below(60_000),
+ details: None,
+ from_hook: Some(false),
+ custom_instructions: None,
+ usage: Some(usage(rng)),
+ harness_digest: None,
+ harness_state_fingerprint: None,
+ })?,
+ rng,
+ )?;
+ continue;
+ }
+
+ let text = match &spec.marker {
+ Some(marker) if target - rows.rows_written() <= 4 => marker.clone(),
+ _ => user_text(rng),
+ };
+ let content = if rng.chance(30) {
+ UserContent::Blocks(vec![UserContentBlock::Text(TextContent {
+ text: text.clone(),
+ text_signature: None,
+ rest: serde_json::Map::new(),
+ })])
+ } else {
+ UserContent::Text(text.clone())
+ };
+ rows.push(
+ "message",
+ serde_json::json!({
+ "message": pa_types::session::AgentMessage::User(UserMessage {
+ content,
+ timestamp: rows.millis,
+ rest: serde_json::Map::new(),
+ })
+ }),
+ rng,
+ )?;
+ last_user_text = text;
+
+ // The assistant turn: thinking + text + a tool call, then the result.
+ let tool = rng.chance(45).then(|| {
+ let name = TOOLS[rng.below(TOOLS.len() as u64) as usize];
+ let id = format!("call-{:08x}", rows.counter);
+ let mut arguments = serde_json::Map::new();
+ arguments.insert(
+ "path".to_string(),
+ serde_json::json!(format!("src/mod{}.rs", rng.below(400))),
+ );
+ arguments.insert("query".to_string(), serde_json::json!(sentence(rng, 3)));
+ (id, name)
+ });
+ let mut blocks = Vec::new();
+ if rng.chance(60) {
+ blocks.push(AssistantContentBlock::Thinking(ThinkingContent {
+ thinking: assistant_text(rng),
+ thinking_signature: None,
+ redacted: None,
+ rest: serde_json::Map::new(),
+ }));
+ }
+ blocks.push(AssistantContentBlock::Text(TextContent {
+ text: assistant_text(rng),
+ text_signature: None,
+ rest: serde_json::Map::new(),
+ }));
+ if let Some((id, name)) = &tool {
+ blocks.push(AssistantContentBlock::ToolCall(ToolCall {
+ id: id.clone(),
+ name: (*name).to_string(),
+ arguments: {
+ let mut map = serde_json::Map::new();
+ map.insert(
+ "path".to_string(),
+ serde_json::json!(format!("src/mod{}.rs", rng.below(400))),
+ );
+ map.insert("query".to_string(), serde_json::json!(sentence(rng, 3)));
+ map
+ },
+ thought_signature: None,
+ rest: serde_json::Map::new(),
+ }));
+ }
+ let stop_reason = if tool.is_some() {
+ StopReason::ToolUse
+ } else if rng.chance(2) {
+ StopReason::Error
+ } else {
+ StopReason::Stop
+ };
+ rows.push(
+ "message",
+ serde_json::json!({
+ "message": pa_types::session::AgentMessage::Assistant(AssistantMessage {
+ content: blocks,
+ api: "openai".to_string(),
+ provider: provider.to_string(),
+ model: model.to_string(),
+ response_model: None,
+ response_id: None,
+ diagnostics: None,
+ usage: usage(rng),
+ stop_reason,
+ stop_reason_raw: None,
+ error_message: None,
+ timestamp: rows.millis,
+ rest: serde_json::Map::new(),
+ })
+ }),
+ rng,
+ )?;
+
+ if let Some((call_id, name)) = tool {
+ rows.push(
+ "message",
+ serde_json::json!({
+ "message": pa_types::session::AgentMessage::ToolResult(ToolResultMessage {
+ tool_call_id: call_id,
+ tool_name: name.to_string(),
+ content: vec![UserContentBlock::Text(TextContent {
+ text: tool_output(rng),
+ text_signature: None,
+ rest: serde_json::Map::new(),
+ })],
+ details: None,
+ is_error: rng.chance(10),
+ timestamp: rows.millis,
+ rest: serde_json::Map::new(),
+ })
+ }),
+ rng,
+ )?;
+ }
+
+ if rng.chance(7) {
+ let custom = CUSTOM_TYPES[rng.below(CUSTOM_TYPES.len() as u64) as usize];
+ rows.push(
+ "custom_message",
+ serde_json::to_value(CustomMessageEntry {
+ custom_type: custom.to_string(),
+ content: UserContent::Text(format!("[{custom}] {}", sentence(rng, 8))),
+ details: None,
+ display: false,
+ rest: serde_json::Map::new(),
+ })?,
+ rng,
+ )?;
+ }
+ }
+
+ rows.writer.flush()?;
+ Ok(Written {
+ rows: rows.counter as usize,
+ last_user_text,
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ fn spec(rows: usize) -> super::SessionSpec {
+ super::SessionSpec {
+ id: "corpus-regression".to_string(),
+ start_ms: 1_700_000_000_000,
+ cwd: "/tmp".to_string(),
+ rows,
+ name: Some("regression".to_string()),
+ marker: Some("final user marker".to_string()),
+ parent_session: None,
+ rlm_depth: Some(0),
+ }
+ }
+
+ #[test]
+ fn final_user_marker_survives_all_small_turn_shapes() {
+ let directory = tempfile::tempdir().expect("session directory");
+ for seed in 0..16 {
+ for target in 9..32 {
+ let written = super::write_session_file(
+ &directory.path().join("session.jsonl"),
+ &spec(target),
+ &mut crate::rng::Rng::new(seed),
+ )
+ .expect("write session");
+ assert_eq!(
+ written.last_user_text, "final user marker",
+ "seed={seed}, target={target}"
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn long_sessions_crossing_compaction_threshold_have_exactly_one_boundary() {
+ let directory = tempfile::tempdir().expect("session directory");
+ let path = directory.path().join("session.jsonl");
+ for seed in 0..8 {
+ super::write_session_file(&path, &spec(1500), &mut crate::rng::Rng::new(seed))
+ .expect("write long session");
+ let rows: Vec<serde_json::Value> = std::fs::read_to_string(&path)
+ .expect("read session")
+ .lines()
+ .map(|line| serde_json::from_str(line).expect("valid row"))
+ .collect();
+ assert_eq!(
+ rows.iter()
+ .filter(|row| row["type"] == "compaction")
+ .count(),
+ 1,
+ "seed={seed}"
+ );
+ let entries = pa_core::session::parse_session_entries(
+ &std::fs::read_to_string(&path).expect("read generated session"),
+ );
+ let context = pa_core::session::build_session_context(&entries, None);
+ assert_eq!(
+ context
+ .messages
+ .iter()
+ .filter(|message| matches!(
+ message,
+ pa_types::session::AgentMessage::CompactionSummary(_)
+ ))
+ .count(),
+ 1,
+ "the real context builder must not duplicate the summary; seed={seed}"
+ );
+ }
+ }
+}
diff --git a/crates/pa-perf/src/lib.rs b/crates/pa-perf/src/lib.rs
new file mode 100644
index 0000000000..33a6b6c71f
--- /dev/null
+++ b/crates/pa-perf/src/lib.rs
@@ -0,0 +1,15 @@
+//! pa-perf: the prime-agent performance corpus generator and TUI bench
+//! harness. A dev/CI tooling crate at the top of the workspace dependency
+//! graph — nothing depends on it, and it never ships in a release.
+//!
+//! The corpus generator writes the product's real on-disk layout under an
+//! isolated agent dir, using the product's own serializers and writers
+//! (pa-types payloads, pa-daemon `SessionEntry`/`session_header_line`/
+//! `RlmSpawnLedger`, pa-core's `WindowedSessionStore` for the warm
+//! sidecars). The bench harness drives the real release binary under a pty.
+
+pub mod bench;
+pub mod corpus;
+pub mod report;
+
+mod rng;
diff --git a/crates/pa-perf/src/main.rs b/crates/pa-perf/src/main.rs
new file mode 100644
index 0000000000..a02a05b9df
--- /dev/null
+++ b/crates/pa-perf/src/main.rs
@@ -0,0 +1,252 @@
+//! The `pa-perf` CLI: `corpus`, `bench`, `run`, and `report`.
+
+use std::path::PathBuf;
+
+use clap::{Parser, Subcommand};
+use pa_perf::bench::{BenchOptions, RunOptions, Scenario};
+use pa_perf::corpus::{self, CacheMode, CorpusOptions, Profile};
+use pa_perf::report::{self, Baseline};
+
+#[derive(Parser)]
+#[command(name = "pa-perf", about = "prime-agent performance tooling", version)]
+struct Cli {
+ #[command(subcommand)]
+ command: Command,
+}
+
+#[derive(Subcommand)]
+enum Command {
+ /// Generate a deterministic corpus under an isolated agent dir.
+ Corpus(CorpusArgs),
+ /// Drive the real release binary under a pty and measure one scenario.
+ Bench(BenchArgs),
+ /// Generate the corpora and run every scenario on the corpus state it
+ /// needs (the CI and `make perf` entry point).
+ Run(RunArgs),
+ /// Compare result JSONs against the baseline; fail on >2x regressions.
+ Report(ReportArgs),
+}
+
+#[derive(clap::Args)]
+struct CorpusArgs {
+ /// The named profile providing the default knob values.
+ #[arg(long, value_enum, default_value_t = Profile::Small)]
+ profile: Profile,
+ /// The corpus root: created, then canonicalized (macOS /tmp -> /private/tmp).
+ #[arg(long)]
+ out: PathBuf,
+ /// The deterministic seed.
+ #[arg(long, default_value_t = 7)]
+ seed: u64,
+ /// Top-level sessions in the sessions dir.
+ #[arg(long)]
+ sessions: Option<usize>,
+ /// Rows per session.
+ #[arg(long)]
+ rows: Option<usize>,
+ /// Rows per subagent child session.
+ #[arg(long)]
+ child_rows: Option<usize>,
+ /// Rows of the one huge session.
+ #[arg(long)]
+ huge_rows: Option<usize>,
+ /// Subagent children per parent per level.
+ #[arg(long)]
+ fanout: Option<usize>,
+ /// Subagent tree depth.
+ #[arg(long)]
+ depth: Option<u32>,
+ /// How many sessions grow a subagent tree.
+ #[arg(long)]
+ tree_parents: Option<usize>,
+ /// One parent grows this many direct children (0 disables).
+ #[arg(long)]
+ wide_children: Option<usize>,
+ /// Sessions in sessions-archive/.
+ #[arg(long)]
+ archive: Option<usize>,
+ /// Files in the generated project dir.
+ #[arg(long)]
+ cwd_files: Option<usize>,
+ /// Write the per-session artifact partitions.
+ #[arg(long, default_value_t = true)]
+ artifacts: bool,
+ /// Write the window-cache sidecars via the product's window-store open.
+ #[arg(long, value_enum, default_value_t = CacheMode::Cold)]
+ cache: CacheMode,
+}
+
+#[derive(clap::Args)]
+struct BenchArgs {
+ /// The release binary under test.
+ #[arg(long, default_value = "target/release/prime-agent")]
+ binary: PathBuf,
+ /// The corpus root (holds corpus.json).
+ #[arg(long)]
+ corpus: PathBuf,
+ /// The scenario to measure.
+ #[arg(long, value_enum)]
+ scenario: Scenario,
+ /// Sampled runs (a warm-up run is always discarded first).
+ #[arg(long, default_value_t = 10)]
+ runs: usize,
+ /// Where the result JSON is written.
+ #[arg(long)]
+ json: PathBuf,
+ /// The per-run liveness deadline (seconds).
+ #[arg(long, default_value_t = 180)]
+ liveness_secs: u64,
+}
+
+#[derive(clap::Args)]
+struct RunArgs {
+ /// The release binary under test.
+ #[arg(long, default_value = "target/release/prime-agent")]
+ binary: PathBuf,
+ /// The named corpus profile for both generated corpora.
+ #[arg(long, value_enum, default_value_t = Profile::Small)]
+ profile: Profile,
+ /// The output root: the two corpora and results/ land under it.
+ #[arg(long, default_value = "target/perf")]
+ out: PathBuf,
+ /// The deterministic seed.
+ #[arg(long, default_value_t = 7)]
+ seed: u64,
+ /// Sampled runs per scenario (a warm-up run is always discarded first).
+ #[arg(long, default_value_t = 10)]
+ runs: usize,
+ /// The per-run liveness deadline (seconds).
+ #[arg(long, default_value_t = 180)]
+ liveness_secs: u64,
+}
+
+#[derive(clap::Args)]
+struct ReportArgs {
+ /// One or more bench result JSONs.
+ #[arg(long, num_args = 1..)]
+ results: Vec<PathBuf>,
+ /// The committed baseline file.
+ #[arg(long, default_value = "crates/pa-perf/baseline.json")]
+ baseline: PathBuf,
+ /// Where the markdown table is written (stdout when absent).
+ #[arg(long)]
+ markdown: Option<PathBuf>,
+}
+
+fn main() -> anyhow::Result<()> {
+ match Cli::parse().command {
+ Command::Corpus(args) => corpus_command(args),
+ Command::Bench(args) => bench_command(args),
+ Command::Run(args) => run_command(args),
+ Command::Report(args) => report_command(args),
+ }
+}
+
+fn corpus_command(args: CorpusArgs) -> anyhow::Result<()> {
+ let mut options = CorpusOptions {
+ out: args.out.clone(),
+ ..args.profile.defaults()
+ };
+ options.seed = args.seed;
+ if let Some(value) = args.sessions {
+ options.sessions = value;
+ }
+ if let Some(value) = args.rows {
+ options.rows = value;
+ }
+ if let Some(value) = args.child_rows {
+ options.child_rows = value;
+ }
+ if let Some(value) = args.huge_rows {
+ options.huge_rows = value;
+ }
+ if let Some(value) = args.fanout {
+ options.fanout = value;
+ }
+ if let Some(value) = args.depth {
+ options.depth = value;
+ }
+ if let Some(value) = args.tree_parents {
+ options.tree_parents = value;
+ }
+ if let Some(value) = args.wide_children {
+ options.wide_children = value;
+ }
+ if let Some(value) = args.archive {
+ options.archive = value;
+ }
+ if let Some(value) = args.cwd_files {
+ options.cwd_files = value;
+ }
+ options.artifacts = args.artifacts;
+ options.cache = args.cache;
+ let manifest = corpus::generate(&options)?;
+ println!(
+ "corpus: profile={} sessions={} rows={} total_rows={} child_sessions={} archive={} cache={}",
+ report::profile_key(options.profile),
+ manifest.counts.sessions,
+ options.rows,
+ manifest.counts.total_rows,
+ manifest.counts.child_sessions,
+ manifest.counts.archive,
+ report::profile_key_cache(options.cache),
+ );
+ println!("huge needle: {}", manifest.markers.huge.needle);
+ println!("manifest: {}", manifest.paths.manifest_file.display());
+ Ok(())
+}
+
+fn bench_command(args: BenchArgs) -> anyhow::Result<()> {
+ let options = BenchOptions {
+ binary: args.binary,
+ corpus: args.corpus,
+ scenario: args.scenario,
+ runs: args.runs,
+ json: args.json,
+ liveness_secs: args.liveness_secs,
+ };
+ let result = pa_perf::bench::run(&options)?;
+ println!(
+ "scenario={} profile={} median={:.1}ms p95={:.1}ms runs={}",
+ report::key_of(result.scenario),
+ result.profile,
+ result.median_ms,
+ result.p95_ms,
+ result.runs,
+ );
+ Ok(())
+}
+
+fn run_command(args: RunArgs) -> anyhow::Result<()> {
+ let options = RunOptions {
+ binary: args.binary,
+ profile: args.profile,
+ out: args.out,
+ seed: args.seed,
+ runs: args.runs,
+ liveness_secs: args.liveness_secs,
+ };
+ pa_perf::bench::run_suite(&options)?;
+ Ok(())
+}
+
+fn report_command(args: ReportArgs) -> anyhow::Result<()> {
+ let mut results = Vec::new();
+ for path in &args.results {
+ let result: pa_perf::bench::BenchResult =
+ serde_json::from_str(&std::fs::read_to_string(path)?)?;
+ results.push(result);
+ }
+ let baseline: Baseline = serde_json::from_str(&std::fs::read_to_string(&args.baseline)?)?;
+ let report = report::report(&results, &baseline);
+ match &args.markdown {
+ Some(path) => std::fs::write(path, &report.markdown)?,
+ None => println!("{}", report.markdown),
+ }
+ if report.regressions.is_empty() {
+ println!("no >2x regressions against the baseline");
+ Ok(())
+ } else {
+ anyhow::bail!("regressions over 2x: {}", report.regressions.join(", "));
+ }
+}
diff --git a/crates/pa-perf/src/report.rs b/crates/pa-perf/src/report.rs
new file mode 100644
index 0000000000..4dfd2993dc
--- /dev/null
+++ b/crates/pa-perf/src/report.rs
@@ -0,0 +1,112 @@
+//! The baseline comparison: the aggregate table (median, p95, baseline,
+//! delta) and the >2x regression gate (report-only scenarios always print
+//! deltas and never fail).
+
+use std::collections::BTreeMap;
+
+use serde::{Deserialize, Serialize};
+
+use crate::bench::{BenchResult, Scenario};
+use crate::corpus::Profile;
+
+/// The committed baseline: one entry per scenario/profile key.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Baseline {
+ pub schema: u32,
+ /// "scenario/profile" -> the measured baseline.
+ pub entries: BTreeMap<String, BaselineEntry>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct BaselineEntry {
+ /// The baseline median (ms).
+ pub median_ms: f64,
+ /// Noisy scenarios (>20% spread across 3 CI runs): deltas print, the
+ /// gate never fails.
+ #[serde(default)]
+ pub report_only: bool,
+ /// Where the number came from (a CI run URL).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub source: Option<String>,
+}
+
+/// The regression gate's ceiling: a median must stay under baseline * 2.
+const REGRESSION_FACTOR: f64 = 2.0;
+
+/// The comparison outcome: the markdown table and the failing rows.
+#[derive(Debug)]
+pub struct Report {
+ pub markdown: String,
+ pub regressions: Vec<String>,
+}
+
+/// Compare results against the baseline; the key is scenario/profile (the
+/// serde names — one source for the wire, the CLI, and the keys).
+#[must_use]
+pub fn report(results: &[BenchResult], baseline: &Baseline) -> Report {
+ let mut rows = Vec::new();
+ let mut regressions = Vec::new();
+ for result in results {
+ let key = format!("{}/{}", key_of(result.scenario), result.profile);
+ let median = result.median_ms;
+ let Some(entry) = baseline.entries.get(&key) else {
+ rows.push(format!(
+ "| {} | {median:.1} | {:.1} | — | — | (no baseline) |",
+ key, result.p95_ms
+ ));
+ continue;
+ };
+ let delta = median / entry.median_ms - 1.0;
+ let verdict = if entry.report_only {
+ "report-only".to_string()
+ } else if median > entry.median_ms * REGRESSION_FACTOR {
+ regressions.push(key.clone());
+ "REGRESSION".to_string()
+ } else {
+ format!("{:+.0}%", delta * 100.0)
+ };
+ rows.push(format!(
+ "| {} | {median:.1} | {:.1} | {:.1} | {:+.0}% | {verdict} |",
+ key,
+ result.p95_ms,
+ entry.median_ms,
+ delta * 100.0
+ ));
+ }
+ let markdown = format!(
+ "| scenario/profile | median ms | p95 ms | baseline ms | delta | verdict |\n|---|---|---|---|---|---|\n{}",
+ rows.join("\n")
+ );
+ Report {
+ markdown,
+ regressions,
+ }
+}
+
+/// The serde name of a scenario (the baseline key's scenario half — the
+/// same name the CLI and the wire use).
+#[must_use]
+pub fn key_of(scenario: Scenario) -> String {
+ serde_json::to_value(scenario)
+ .ok()
+ .and_then(|value| value.as_str().map(str::to_string))
+ .unwrap_or_default()
+}
+
+/// The serde name of a profile (the baseline key's profile half).
+#[must_use]
+pub fn profile_key(profile: Profile) -> String {
+ serde_json::to_value(profile)
+ .ok()
+ .and_then(|value| value.as_str().map(str::to_string))
+ .unwrap_or_default()
+}
+
+/// The serde name of a cache mode (display only).
+#[must_use]
+pub fn profile_key_cache(mode: crate::corpus::CacheMode) -> String {
+ serde_json::to_value(mode)
+ .ok()
+ .and_then(|value| value.as_str().map(str::to_string))
+ .unwrap_or_default()
+}
diff --git a/crates/pa-perf/src/rng.rs b/crates/pa-perf/src/rng.rs
new file mode 100644
index 0000000000..f4ca5b5914
--- /dev/null
+++ b/crates/pa-perf/src/rng.rs
@@ -0,0 +1,65 @@
+//! The corpus PRNG: a tiny deterministic splitmix64 (no external dependency).
+//! The same seed produces a byte-identical corpus tree, file mtimes included.
+
+/// A deterministic splitmix64 stream for corpus generation.
+#[derive(Debug, Clone)]
+pub(crate) struct Rng(u64);
+
+impl Rng {
+ /// A stream from `seed`.
+ pub(crate) const fn new(seed: u64) -> Self {
+ Self(seed)
+ }
+
+ /// The next raw value.
+ pub(crate) fn next_u64(&mut self) -> u64 {
+ self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
+ let mut z = self.0;
+ z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
+ z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
+ z ^ (z >> 31)
+ }
+
+ /// A uniform value below `bound` (an empty bound answers 0).
+ pub(crate) fn below(&mut self, bound: u64) -> u64 {
+ if bound == 0 {
+ return 0;
+ }
+ self.next_u64() % bound
+ }
+
+ /// A true chance of `percent` in 100.
+ pub(crate) fn chance(&mut self, percent: u64) -> bool {
+ self.below(100) < percent
+ }
+}
+
+/// Civil-date conversion (days since epoch -> y/m/d), the inverse of the
+/// session parser's `civil_days`: keeps timestamp formatting allocation-free.
+fn civil_from_days(days: i64) -> (i64, u32, u32) {
+ let z = days + 719_468;
+ let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
+ let doe = z - era * 146_097;
+ let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146_096) / 365;
+ let y = yoe + era * 400;
+ let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
+ let mp = (5 * doy + 2) / 153;
+ let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
+ let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
+ (if m <= 2 { y + 1 } else { y }, m, d)
+}
+
+/// ISO-8601 (UTC, millisecond precision) from unix milliseconds — the
+/// product's entry-timestamp format.
+#[must_use]
+pub(crate) fn iso_from_millis(millis: u64) -> String {
+ let secs = (millis / 1000) as i64;
+ let rem = millis % 1000;
+ let days = secs.div_euclid(86_400);
+ let secs_of_day = secs.rem_euclid(86_400);
+ let (year, month, day) = civil_from_days(days);
+ let hour = secs_of_day / 3600;
+ let minute = (secs_of_day % 3600) / 60;
+ let second = secs_of_day % 60;
+ format!("{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}.{rem:03}Z")
+}
diff --git a/crates/pa-perf/tests/corpus_verification.rs b/crates/pa-perf/tests/corpus_verification.rs
new file mode 100644
index 0000000000..dfdcdcea48
--- /dev/null
+++ b/crates/pa-perf/tests/corpus_verification.rs
@@ -0,0 +1,379 @@
+//! The corpus verification battery: every generated file loads through the
+//! PRODUCT's own readers (pa-daemon SessionFile, pa-core parse, the RLM
+//! ledger replay), the tree is seed-deterministic, and the marker discipline
+//! holds. Kept small and fast: tiny corpora in temp dirs only.
+
+use std::collections::HashSet;
+use std::path::Path;
+
+use pa_core::session::parse_session_entries;
+use pa_daemon::rlm_ledger::RlmSpawnLedger;
+use pa_daemon::session_store::SessionFile;
+use pa_perf::corpus::{generate, CacheMode, CorpusOptions, Profile};
+
+fn tiny(profile: Profile, out: &Path) -> CorpusOptions {
+ CorpusOptions {
+ out: out.to_path_buf(),
+ ..profile.defaults()
+ }
+}
+
+/// Every generated session file must parse through the daemon's own store.
+#[test]
+fn sessions_load_through_the_product_store() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let options = tiny(Profile::Small, dir.path());
+ let manifest = generate(&options).expect("generate");
+ let mut checked = 0;
+ for file in &manifest.files {
+ let path = std::path::PathBuf::from(file);
+ if path.parent() != Some(&manifest.paths.sessions_dir) {
+ continue;
+ }
+ let store = SessionFile::open(&path).expect("the product store opens the file");
+ assert_eq!(store.header.id, path.file_stem().unwrap().to_string_lossy());
+ assert!(!store.entries().is_empty(), "the session carried its rows");
+ checked += 1;
+ }
+ assert_eq!(checked, options.sessions, "every sessions-dir file checked");
+}
+
+/// No row may degrade to an Unknown entry: the corpus speaks the wire format.
+#[test]
+fn rows_never_degrade_to_unknown_entries() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let options = tiny(Profile::Small, dir.path());
+ let manifest = generate(&options).expect("generate");
+ let huge = &manifest.markers.huge;
+ let text = std::fs::read_to_string(&huge.file).expect("read the huge session");
+ let entries = parse_session_entries(&text);
+ assert!(
+ entries
+ .iter()
+ .all(|entry| !matches!(entry, pa_types::session::FileEntry::Unknown { .. })),
+ "no unknown rows in the generated huge session"
+ );
+}
+
+/// The marker lands on the final user row of each marker session.
+#[test]
+fn markers_land_on_the_final_user_message() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let options = tiny(Profile::Small, dir.path());
+ let manifest = generate(&options).expect("generate");
+ for marker in [
+ &manifest.markers.huge,
+ &manifest.markers.switch_a,
+ &manifest.markers.switch_b,
+ ] {
+ let text = std::fs::read_to_string(&marker.file).expect("read session");
+ let last_user = text
+ .lines()
+ .rev()
+ .find_map(|line| {
+ let value: serde_json::Value = serde_json::from_str(line).ok()?;
+ (value.get("type") == Some(&serde_json::json!("message"))
+ && value["message"]["role"] == serde_json::json!("user"))
+ .then(|| value["message"]["content"].to_string())
+ })
+ .expect("a final user message");
+ assert!(
+ last_user.contains(&marker.needle),
+ "the needle is the final user message: {last_user}"
+ );
+ }
+}
+
+/// The ledger the corpus writes replays through the product's own reader.
+#[test]
+fn the_ledger_replays_through_the_product_reader() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let options = tiny(Profile::Small, dir.path());
+ let manifest = generate(&options).expect("generate");
+ let ledger = RlmSpawnLedger::new(
+ &manifest.paths.agent_dir,
+ &manifest.paths.sessions_dir,
+ |_| {},
+ );
+ let edges = ledger.live_edges().expect("the ledger replays");
+ assert_eq!(
+ edges.len(),
+ manifest.counts.subagent_edges,
+ "every spawn edge replays as live"
+ );
+ assert!(
+ edges
+ .iter()
+ .all(|edge| edge.depth >= 1 && edge.child.contains("session-artifacts")),
+ "edges carry the artifact-tree child files"
+ );
+}
+
+/// Same seed, same tree: every file the generator owns is byte-identical
+/// (the ledger's `at` stamps come from the product writer's wall clock; the
+/// edge content is compared field-by-field instead).
+#[test]
+fn the_same_seed_rebuilds_the_identical_tree() {
+ let first = tempfile::tempdir().expect("temp dir");
+ let second = tempfile::tempdir().expect("temp dir");
+ let first_root = first.path().canonicalize().expect("canonical first");
+ let second_root = second.path().canonicalize().expect("canonical second");
+ generate(&tiny(Profile::Small, &first_root)).expect("first generate");
+ generate(&tiny(Profile::Small, &second_root)).expect("second generate");
+ // The ledger file NAME is the hash of the canonical sessions dir, so
+ // two corpus roots name it differently; normalize it for comparison.
+ fn normalized(mut files: Vec<String>) -> Vec<String> {
+ for file in &mut files {
+ if let Some(hash) = file
+ .strip_prefix("agent/rlm-ledger/")
+ .and_then(|rest| rest.strip_suffix(".jsonl"))
+ {
+ if !hash.contains('/') {
+ *file = "agent/rlm-ledger/<hash>.jsonl".to_string();
+ }
+ }
+ }
+ files
+ }
+ let first_files = normalized(relative_files(&first_root));
+ let second_files = normalized(relative_files(&second_root));
+ let mut first_sorted = first_files.clone();
+ first_sorted.sort();
+ let mut second_sorted = second_files.clone();
+ second_sorted.sort();
+ assert_eq!(first_sorted, second_sorted, "the file sets match");
+
+ for relative in &first_sorted {
+ let a = first_root.join(relative);
+ let b = second_root.join(relative);
+ if relative.starts_with("agent/rlm-ledger/") {
+ let a = ledger_of(&first_root);
+ let b = ledger_of(&second_root);
+ let edges = |path: &Path, root: &Path| -> Vec<(String, String, u32)> {
+ std::fs::read_to_string(path)
+ .expect("read ledger")
+ .lines()
+ .filter_map(|line| {
+ let value: serde_json::Value = serde_json::from_str(line).ok()?;
+ (value["op"] == serde_json::json!("spawn")).then(|| {
+ let child = Path::new(value["child"].as_str().unwrap_or_default());
+ let child = child
+ .strip_prefix(root)
+ .map_or_else(|_| child.to_path_buf(), Path::to_path_buf);
+ (
+ value["childId"].as_str().unwrap_or_default().to_string(),
+ child.to_string_lossy().to_string(),
+ value["depth"].as_u64().unwrap_or_default() as u32,
+ )
+ })
+ })
+ .collect()
+ };
+ assert_eq!(
+ edges(&a, &first_root),
+ edges(&b, &second_root),
+ "the ledger edges match"
+ );
+ continue;
+ }
+ // corpus.json embeds absolute paths and the ledger's hash name; the
+ // ledger itself and child headers embed the corpus root in
+ // parentSession/child paths. Normalize the root, skip the manifest.
+ if relative == "corpus.json" {
+ continue;
+ }
+ let read = |path: &Path, root: &Path| -> Vec<u8> {
+ let bytes = std::fs::read(path).expect("read");
+ let root = root.to_string_lossy().into_owned().into_bytes();
+ let mut normalized = Vec::with_capacity(bytes.len());
+ let mut index = 0;
+ while index < bytes.len() {
+ if bytes[index..].starts_with(&root) {
+ normalized.extend_from_slice(b"<root>");
+ index += root.len();
+ } else {
+ normalized.push(bytes[index]);
+ index += 1;
+ }
+ }
+ normalized
+ };
+ assert_eq!(
+ read(&a, &first_root),
+ read(&b, &second_root),
+ "{relative} differs between seeds-equal runs"
+ );
+ }
+}
+
+/// Entry ids are unique 8-hex counters even past the ~22k-row collision
+/// cliff the product's uuid-prefix ids hit.
+#[test]
+fn entry_ids_stay_unique_past_the_uuid_prefix_collision_cliff() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let options = CorpusOptions {
+ sessions: 3,
+ rows: 25_000,
+ child_rows: 0,
+ fanout: 0,
+ depth: 0,
+ tree_parents: 0,
+ wide_children: 0,
+ archive: 0,
+ cwd_files: 5,
+ ..tiny(Profile::Small, dir.path())
+ };
+ let manifest = generate(&options).expect("generate");
+ let file = manifest
+ .paths
+ .sessions_dir
+ .join(format!("{}.jsonl", manifest.markers.huge.id));
+ let text = std::fs::read_to_string(&file).expect("read session");
+ let mut ids = HashSet::new();
+ let mut rows = 0;
+ for line in text.lines() {
+ let value: serde_json::Value = serde_json::from_str(line).expect("a parseable row");
+ if value["type"] == serde_json::json!("session") {
+ continue;
+ }
+ let id = value["id"].as_str().expect("an id").to_string();
+ assert_eq!(id.len(), 8, "the id is 8-hex");
+ assert!(id.chars().all(|c| c.is_ascii_hexdigit()), "hex id: {id}");
+ ids.insert(id);
+ rows += 1;
+ }
+ assert_eq!(ids.len(), rows, "no id collides across {rows} rows");
+}
+
+/// The warm pass writes the product's own window-cache sidecars.
+#[test]
+fn the_warm_pass_writes_product_sidecars() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let mut options = tiny(Profile::Small, dir.path());
+ options.cache = CacheMode::Warm;
+ let manifest = generate(&options).expect("generate");
+ let sidecars = manifest
+ .files
+ .iter()
+ .filter(|file| file.ends_with("window-cache.json"))
+ .count();
+ assert_eq!(
+ sidecars,
+ options.sessions + options.archive,
+ "every openable session gained a window-cache sidecar"
+ );
+}
+
+/// The switch-b row is the ONLY session the switch query can match. The
+/// agents-view search is a fuzzy subsequence match over each row's name
+/// AND message text, so a common token could rank another row first; the
+/// guarantee needs the `zq` token to be literally impossible elsewhere —
+/// no generated corpus text (names or message bodies) contains the letter
+/// z at all (the ISO `Z` header suffix is the one allowed z).
+#[test]
+fn the_switch_query_can_only_match_switch_b() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let options = tiny(Profile::Small, dir.path());
+ let manifest = generate(&options).expect("generate");
+ assert_eq!(manifest.markers.switch_row, "switch target zqmark");
+ assert!(manifest.markers.switch_query.contains("zqmark"));
+ assert!(manifest.markers.switch_query != manifest.markers.switch_row);
+ let mut scanned = 0;
+ for file in &manifest.files {
+ let path = std::path::PathBuf::from(file);
+ if path.parent() == Some(&manifest.paths.sessions_dir) {
+ // The search corpus folds MESSAGE text and names (not header
+ // timestamps, whose ISO `Z` suffix is the one allowed z).
+ for line in std::fs::read_to_string(&path)
+ .expect("read session")
+ .lines()
+ {
+ if line.contains('z')
+ && !line.contains(r#""type":"session""#)
+ && !line.contains(&manifest.markers.switch_row)
+ {
+ panic!("a non-header row carries z: {line}");
+ }
+ }
+ scanned += 1;
+ }
+ }
+ assert_eq!(scanned, options.sessions, "every session checked");
+}
+
+/// The corpus settings disable BOTH archive rules: the daemon's boot sweep
+/// would otherwise retire corpus sessions out from under the manifest
+/// baseline (the defaults keep 200 files / 30 days).
+#[test]
+fn the_archive_policy_is_off() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let options = tiny(Profile::Small, dir.path());
+ let manifest = generate(&options).expect("generate");
+ let settings: serde_json::Value = serde_json::from_str(
+ &std::fs::read_to_string(manifest.paths.agent_dir.join("settings.json")).expect("settings"),
+ )
+ .expect("a settings object");
+ assert_eq!(
+ settings["sessionArchiveMaxAgeDays"],
+ serde_json::json!("off")
+ );
+ assert_eq!(
+ settings["sessionArchiveMaxSessions"],
+ serde_json::json!("off")
+ );
+}
+
+/// The corpus manifest records the generated file set (the bench's
+/// per-run restore baseline) and the archive dir holds the archived files.
+#[test]
+fn the_manifest_is_the_file_set_baseline() {
+ let dir = tempfile::tempdir().expect("temp dir");
+ let options = tiny(Profile::Small, dir.path());
+ let manifest = generate(&options).expect("generate");
+ let on_disk = relative_files(dir.path());
+ assert_eq!(
+ manifest.files.len() + 1, // corpus.json is written after the walk
+ on_disk.len(),
+ "the manifest lists every generated file"
+ );
+ let archived = on_disk
+ .iter()
+ .filter(|file| file.starts_with("agent/sessions-archive/"))
+ .count();
+ assert_eq!(archived, options.archive);
+}
+
+/// The one ledger file under a corpus root (its name is the sessions-dir
+/// hash, which differs per root).
+fn ledger_of(root: &Path) -> std::path::PathBuf {
+ let dir = root.join("agent/rlm-ledger");
+ std::fs::read_dir(dir)
+ .expect("read the ledger dir")
+ .map(|entry| entry.expect("entry").path())
+ .find(|path| path.extension().is_some_and(|ext| ext == "jsonl"))
+ .expect("the ledger file")
+}
+
+fn relative_files(root: &Path) -> Vec<String> {
+ fn walk(dir: &Path, root: &Path, out: &mut Vec<String>) {
+ let entries = std::fs::read_dir(dir).expect("read dir");
+ for entry in entries {
+ let entry = entry.expect("entry");
+ let path = entry.path();
+ if path.is_dir() {
+ walk(&path, root, out);
+ } else {
+ out.push(
+ path.strip_prefix(root)
+ .expect("inside root")
+ .to_string_lossy()
+ .to_string(),
+ );
+ }
+ }
+ }
+ let mut files = Vec::new();
+ walk(root, root, &mut files);
+ files.sort();
+ files
+}
diff --git a/crates/pa-perf/tests/fixtures/draining_daemon.py b/crates/pa-perf/tests/fixtures/draining_daemon.py
new file mode 100644
index 0000000000..98b28c1616
--- /dev/null
+++ b/crates/pa-perf/tests/fixtures/draining_daemon.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Mock product modes: daemon shutdown waits for its held UI connection.
+
+Only used by the Rust harness regression; never loads or launches the product.
+"""
+import json
+from pathlib import Path
+import select
+import socket
+import sys
+import time
+
+path = Path(sys.argv[sys.argv.index('--daemon-socket') + 1])
+if '--mode' not in sys.argv:
+ with socket.socket(socket.AF_UNIX) as peer:
+ peer.connect(str(path))
+ peer.recv(4096)
+ peer.sendall(b'held-ui\n')
+ sys.stdout.write('▗▄▄█▀ mock paint 0 (0%)\n')
+ sys.stdout.flush()
+ # The mock UI remains connected until its owned process is stopped.
+ while peer.recv(4096):
+ pass
+ sys.exit(0)
+
+listener = socket.socket(socket.AF_UNIX)
+listener.bind(str(path))
+listener.listen()
+peers = {}
+accepting = True
+shutdown = False
+deadline = time.monotonic() + 20
+try:
+ while accepting or peers:
+ assert time.monotonic() < deadline, 'mock daemon connection drain timed out'
+ ready, _, _ = select.select(([listener] if accepting else []) + list(peers), [], [], 1)
+ for peer in ready:
+ if peer is listener:
+ client, _ = listener.accept()
+ peers[client] = b''
+ try:
+ client.sendall(b'{"type":"daemon_hello"}\n')
+ except (BrokenPipeError, ConnectionResetError):
+ client.close()
+ del peers[client]
+ continue
+ try:
+ data = peer.recv(4096)
+ except ConnectionResetError:
+ data = b''
+ if not data:
+ peer.close()
+ del peers[peer]
+ continue
+ peers[peer] += data
+ if peers[peer] == b'held-ui\n':
+ continue
+ if b'\n' not in peers[peer]:
+ continue
+ request = json.loads(peers[peer])
+ assert request['command']['type'] == 'shutdown'
+ peer.sendall((json.dumps({'type': 'response', 'id': request['id'],
+ 'success': True}) + '\n').encode())
+ listener.close()
+ accepting = False
+ shutdown = True
+ assert shutdown, 'mock daemon never received protocol shutdown'
+ path.with_suffix('.drained').write_text('all UI/control connections closed\n')
+finally:
+ listener.close()
+ for peer in peers:
+ peer.close()
diff --git a/crates/pa-perf/tests/report_regression.rs b/crates/pa-perf/tests/report_regression.rs
new file mode 100644
index 0000000000..e1007c471b
--- /dev/null
+++ b/crates/pa-perf/tests/report_regression.rs
@@ -0,0 +1,74 @@
+//! The report battery: the >2x regression gate, the report-only exempt,
+//! and the missing-baseline row.
+
+use std::collections::BTreeMap;
+
+use pa_perf::bench::{BenchCorpus, BenchResult, Scenario};
+use pa_perf::report::{report, Baseline, BaselineEntry};
+
+fn result(scenario: Scenario, median_ms: f64) -> BenchResult {
+ BenchResult {
+ schema: 1,
+ scenario,
+ profile: "small".to_string(),
+ runs: 10,
+ samples_ms: vec![median_ms],
+ median_ms,
+ p95_ms: median_ms,
+ needle: String::new(),
+ binary: String::new(),
+ corpus: BenchCorpus {
+ root: std::path::PathBuf::new(),
+ sessions: 10,
+ total_rows: 3_500,
+ child_sessions: 12,
+ archive: 3,
+ },
+ }
+}
+
+fn baseline(median_ms: f64, report_only: bool) -> Baseline {
+ Baseline {
+ schema: 1,
+ entries: BTreeMap::from([(
+ "cold-start/small".to_string(),
+ BaselineEntry {
+ median_ms,
+ report_only,
+ source: None,
+ },
+ )]),
+ }
+}
+
+#[test]
+fn a_two_x_regression_fails_and_deltas_always_print() {
+ let report = report(
+ &[result(Scenario::ColdStart, 2_100.0)],
+ &baseline(1_000.0, false),
+ );
+ assert_eq!(report.regressions, ["cold-start/small"]);
+ assert!(report.markdown.contains("cold-start/small"));
+ assert!(report.markdown.contains("REGRESSION"));
+}
+
+#[test]
+fn a_report_only_row_never_fails_the_gate() {
+ let report = report(
+ &[result(Scenario::ColdStart, 5_000.0)],
+ &baseline(1_000.0, true),
+ );
+ assert!(report.regressions.is_empty());
+ assert!(report.markdown.contains("report-only"));
+}
+
+#[test]
+fn a_missing_baseline_prints_no_baseline() {
+ let report = report(
+ &[result(Scenario::OpenSession, 147.0)],
+ &baseline(1_000.0, false),
+ );
+ assert!(report.regressions.is_empty());
+ assert!(report.markdown.contains("open-session/small"));
+ assert!(report.markdown.contains("(no baseline)"));
+}