A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
perf CI: pa-perf corpora + pty bench + the perf workflow (#3607)
Sebastian Müller41946110baad
diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.ymlnew file mode 100644index 0000000000..522f514617--- /dev/null+++ b/.github/workflows/perf.yml@@ -0,0 +1,185 @@+# The performance-test lane: a nightly + opt-in-per-PR benchmark of the+# interactive surfaces (cold/warm start, agents view, open/switch session,+# @ completion) over deterministic fake stores, driven through the REAL+# release binary under a pty by `crates/pa-perf`.+#+# Triggers:+# schedule — nightly on main (the label path is how a branch proves+# itself; workflow_dispatch/schedule only fire once this+# file is on main).+# workflow_dispatch — manual.+# pull_request [labeled, synchronize] — gated on the `perf-test` label:+# the label IS the opt-in (deliberately including draft+# PRs — labeling a draft is how this lane proves itself+# before review), and a synchronize re-runs only while the+# label stays on. Never runs for unlabeled PRs: a 4-way+# release-build matrix on every PR would bill the org+# runners hard.+#+# Job graph:+# build — release prime-agent + pa-perf once, uploaded as one artifact+# (every bench leg downloads the same bytes).+# bench — one matrix leg per corpus profile: generate the corpus+# (cold + warm), bootstrap the kernel venv (the same prep+# ci.yml's kernel e2e shards use), run every scenario 10x,+# upload the result JSONs.+# report — aggregate every result against crates/pa-perf/baseline.json:+# the $GITHUB_STEP_SUMMARY table (scenario, profile, median,+# p95, baseline, delta) always prints deltas and fails ONLY on a+# >2x median regression; scenarios marked report-only in the+# baseline (spread >20% across 3 CI runs) never fail the gate.+#+# Cache policy: Swatinem's cache with a dedicated shared-key, saved ONLY on+# main-anchored runs (schedule, or a dispatch on main). PR legs restore and+# never save — PR-ref cache entries are the eviction pressure on the+# release-build caches, and the repo sits at the 10 GB Actions-cache quota.+name: perf++on:+ schedule:+ - cron: "30 3 * * *"+ workflow_dispatch:+ pull_request:+ types: [labeled, synchronize]+ branches: [main]++permissions:+ contents: read++concurrency:+ group: perf-${{ github.ref }}+ cancel-in-progress: true++env:+ CARGO_TERM_COLOR: always+ CARGO_INCREMENTAL: 0++jobs:+ build:+ name: build release binaries+ runs-on: ubuntu-24.04+ timeout-minutes: 45+ if: >-+ github.event_name == 'schedule' ||+ github.event_name == 'workflow_dispatch' ||+ contains(github.event.pull_request.labels.*.name, 'perf-test')+ steps:+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ persist-credentials: false+ - uses: dtolnay/rust-toolchain@ce678459e9fc7500d337468f904b95f1b5c10b5e # 1.98.1 action pin; the SHA selects the toolchain+ - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2+ with:+ # One entry per main-anchored run: PR legs restore, never save+ # (the save-if gate below), so this lane cannot evict the+ # release-build caches.+ shared-key: perf-build+ cache-on-failure: true+ save-if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') }}+ - name: Build (release, locked)+ run: cargo build --release --locked -p pa-cli --bin prime-agent -p pa-perf --bin pa-perf+ - name: Upload the binaries+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ with:+ name: perf-bins+ path: |+ target/release/prime-agent+ target/release/pa-perf+ if-no-files-found: error++ bench:+ name: bench (${{ matrix.profile }})+ needs: build+ runs-on: ubuntu-24.04+ timeout-minutes: 90+ strategy:+ fail-fast: false+ matrix:+ profile: [small, medium, large, xl]+ steps:+ # The checkout provides the packaged kernel runtime (prime-agent-runtime/)+ # the release binary resolves through PI_PACKAGE_DIR, plus skills/.+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ persist-credentials: false+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0+ with:+ name: perf-bins+ path: bins+ - name: Stage the binaries+ run: |+ set -euo pipefail+ install -Dm755 bins/prime-agent "$HOME/.local/bin/prime-agent"+ install -Dm755 bins/pa-perf "$HOME/.local/bin/pa-perf"+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"+ - name: Install uv (the kernel venv bootstrap below uses it)+ run: |+ curl -LsSf https://astral.sh/uv/install.sh | sh+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"+ # The same prep ci.yml's kernel-e2e shards run: the venv must be ready+ # before the first session prewarm, or the first bench run pays the+ # bootstrap inside its (discarded) warm-up.+ - name: Bootstrap the kernel venv+ env:+ PI_PACKAGE_DIR: ${{ github.workspace }}+ run: prime-agent --prime-agent-bootstrap+ # One driver call: pa-perf run generates the cold and warm corpora,+ # routes every scenario to the corpus state it needs, and writes one+ # result JSON per scenario into <out>/results/.+ - name: Run the scenarios+ env:+ PI_PACKAGE_DIR: ${{ github.workspace }}+ run: |+ set -euo pipefail+ pa-perf run \+ --binary "$HOME/.local/bin/prime-agent" \+ --profile ${{ matrix.profile }} \+ --out "$RUNNER_TEMP/perf" \+ --runs 10+ - name: Upload the results+ if: always()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ with:+ name: perf-results-${{ matrix.profile }}+ path: ${{ runner.temp }}/perf/results/*.json+ if-no-files-found: error+ # The raw pty streams of the launches that did not complete: the CI+ # counterpart of the empty-tail failures (nothing debuggable in a+ # timeout message alone).+ - name: Upload the failure pty dumps+ if: failure()+ uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ with:+ name: perf-failure-streams-${{ matrix.profile }}+ path: |+ ${{ runner.temp }}/perf/corpus-cold/failed-streams/*+ ${{ runner.temp }}/perf/corpus-warm/failed-streams/*+ if-no-files-found: ignore++ report:+ name: report + regression gate+ needs: bench+ runs-on: ubuntu-24.04+ timeout-minutes: 10+ steps:+ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2+ with:+ persist-credentials: false+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0+ with:+ name: perf-bins+ path: bins+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0+ with:+ pattern: perf-results-*+ merge-multiple: true+ path: results+ - name: Compare against the baseline+ run: |+ set -euo pipefail+ install -Dm755 bins/pa-perf "$HOME/.local/bin/pa-perf"+ echo "$HOME/.local/bin" >> "$GITHUB_PATH"+ pa-perf report \+ --baseline crates/pa-perf/baseline.json \+ --markdown "$GITHUB_STEP_SUMMARY" \+ --results results/*.jsondiff --git a/AGENTS.md b/AGENTS.mdindex 90efbf8717..dac9a9543a 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -196,6 +196,7 @@ The table summarizes each crate's current direct workspace dependencies:| `pa-daemon` | Supervisor, workers, wire protocol | `pa-types`, `pa-core`, `pa-agent`, `pa-ai`, `pa-telemetry`, `pa-models` || `pa-tui` | Terminal UI, daemon-wire client | `pa-types` || `pa-cli` | `prime-agent` binary, composition root | `pa-types`, `pa-core`, `pa-ai`, `pa-agent`, `pa-daemon`, `pa-tui`, `pa-telemetry` |+| `pa-perf` | Performance corpora + pty bench harness (dev/CI tooling, never shipped) | `pa-types`, `pa-core`, `pa-daemon` |Layering rules:diff --git a/Cargo.lock b/Cargo.lockindex 2d73147bc7..5ab287b87d 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -79,6 +79,12 @@ version = "1.0.104"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"+[[package]]+name = "arrayvec"+version = "0.7.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"+[[package]]name = "asn1-rs"version = "0.7.2"@@ -1596,6 +1602,22 @@ dependencies = ["tracing",]+[[package]]+name = "pa-perf"+version = "0.10.0"+dependencies = [+ "anyhow",+ "clap",+ "nix",+ "pa-core",+ "pa-daemon",+ "pa-types",+ "serde",+ "serde_json",+ "tempfile",+ "vt100",+]+[[package]]name = "pa-telemetry"version = "0.10.0"@@ -2948,6 +2970,39 @@ version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"+[[package]]+name = "vt100"+version = "0.15.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "84cd863bf0db7e392ba3bd04994be3473491b31e66340672af5d11943c6274de"+dependencies = [+ "itoa",+ "log",+ "unicode-width 0.1.14",+ "vte",+]++[[package]]+name = "vte"+version = "0.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f5022b5fbf9407086c180e9557be968742d839e68346af7792b8592489732197"+dependencies = [+ "arrayvec",+ "utf8parse",+ "vte_generate_state_changes",+]++[[package]]+name = "vte_generate_state_changes"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2e369bee1b05d510a7b4ed645f5faa90619e05437111783ea5848f28d97d3c2e"+dependencies = [+ "proc-macro2",+ "quote",+]+[[package]]name = "walkdir"version = "2.5.0"diff --git a/Cargo.toml b/Cargo.tomlindex 38e1485b47..a340943b0e 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -10,6 +10,7 @@ members = ["crates/pa-daemon","crates/pa-tui","crates/pa-cli",+ "crates/pa-perf",]# Crossterm 0.28.1's kitty support check holds the process-wide event readerdiff --git a/Makefile b/Makefileindex 00f85e1c9f..c178551f47 100644--- a/Makefile+++ b/Makefile@@ -165,6 +165,18 @@ package:catalog-assets-gates:python3 scripts/release/test_catalog_assets.py+# Local one-command perf run (crates/pa-perf/README.md documents the+# scenario set): build the release binaries, generate the small corpus,+# bench every scenario 10x, print the table. Run it on a quiet machine —+# the numbers are wall-clock latencies of the interactive surfaces.+perf:+ cargo build --release --locked -p pa-cli --bin prime-agent -p pa-perf --bin pa-perf+ ./target/release/pa-perf run --binary target/release/prime-agent \+ --profile small --out target/perf --runs 10+ ./target/release/pa-perf report --baseline crates/pa-perf/baseline.json \+ --results target/perf/results/*.json++# The changelog fold's contract battery (RELEASE-FLOW-PROPOSAL.md §8): the# fold is the release-PR's changelog half, and the pathspec security case is# first - a fragment NAME is data, never a git pathspec, so no root-level@@ -187,7 +199,7 @@ shard-gates:python3 scripts/test_ci_test_shard.pypython3 scripts/test_ci_pr_crates.py-.PHONY: check deny windows-cross actionlint glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates restamp-gates shard-gates+.PHONY: check deny windows-cross actionlint glibc-gate release-dry-run continuous-dry-run audit-build package catalog-assets catalog-assets-fixture catalog-assets-gates fold-gates restamp-gates shard-gates perf# Offline packaging contract tests for updaters shipped since Prime Agent 0.6.legacy-upgrade-gates:diff --git a/crates/pa-perf/Cargo.toml b/crates/pa-perf/Cargo.tomlnew file mode 100644index 0000000000..7f1ed12b61--- /dev/null+++ b/crates/pa-perf/Cargo.toml@@ -0,0 +1,41 @@+# pa-perf: the performance corpus generator + bench harness (dev/CI tooling).+# Top of the dependency graph: nothing depends on it, and it never ships —+# the release lanes build only `-p pa-cli --bin prime-agent`.+[package]+name = "pa-perf"+version.workspace = true+edition.workspace = true+license.workspace = true+publish = false++[dependencies]+anyhow = { workspace = true }+serde = { workspace = true }+serde_json = { workspace = true }+# CLI parsing: the workspace already resolves clap for pa-tui.+clap = { version = "4", features = ["derive"] }+# The screen model for content needles the raw byte stream cannot see (the+# agents view's `Inactive (N)` heading repaints as cell diffs on big+# corpora). MIT, no_std-free, never shipped — the only new third-party dep+# this crate carries (see README + the PR body).+vt100 = "0.15"+# pty + poll for the bench harness: the same nix version the workspace+# already resolves for pa-core/pa-cli.+[target.'cfg(unix)'.dependencies]+nix = { version = "0.29", features = ["term", "signal", "process", "fs", "poll"] }++[dependencies.pa-types]+path = "../pa-types"++[dependencies.pa-core]+path = "../pa-core"++[dependencies.pa-daemon]+path = "../pa-daemon"++[dev-dependencies]+tempfile = "3"++# The library's rows/entries ride the product's own serializers (pa-types+# payloads inside pa-daemon's SessionEntry), so generated corpora exercise+# the exact wire format the product reads.diff --git a/crates/pa-perf/README.md b/crates/pa-perf/README.mdnew file mode 100644index 0000000000..5a46a4036e--- /dev/null+++ b/crates/pa-perf/README.md@@ -0,0 +1,178 @@+# pa-perf++The prime-agent performance tooling: a deterministic corpus generator over+the product's real on-disk layout, and a pty bench harness that drives the+REAL release binary (`target/release/prime-agent`) headlessly. A dev/CI+crate at the top of the workspace dependency graph — nothing depends on it,+and it never ships (the release lanes build `-p pa-cli --bin prime-agent`+only).++## Scope and non-goals++- Scope: faking realistic agent-dir state at scale (sessions, subagent+ trees, artifacts, the spawn ledger, the cwd project tree), and measuring+ the interactive surfaces (start, agents view, open/switch session,+ @ completion) through the real binary.+- Non-goals: benchmarks of provider/LLM paths (the harness stays offline),+ and any product-code changes. The new `vt100` screen-model dependency+ is described below; clap and nix were already in the tree.++## The corpus generator++```+pa-perf corpus --profile medium --out DIR [--seed N] [knob overrides]+```++`DIR` must be absent or empty. A populated output is rejected before any writes,+so stale sessions cannot enter the new corpus or its restore baseline. Choose a+fresh directory for another standalone generation. The suite command can be run+again in the same output: it replaces only its own corpus directories after+verifying their manifests, and refuses unrelated populated directories.++Writes the product's real layout under `DIR/`:++```+DIR/+ corpus.json # manifest: counts, needle markers, the file-set baseline+ project/ # the sessions' cwd + @-completion tree+ agent/ # point PRIME_AGENT_CODING_AGENT_DIR here+ settings.json # onboarding pre-answered, traces off+ sessions/<uuid>.jsonl # current-format session files (flat)+ sessions-archive/<uuid>.jsonl+ rlm-ledger/<hash16>.jsonl # written through pa-daemon's RlmSpawnLedger+ session-artifacts/... # kernel-state stubs, harness/, sub-<id>/ trees+```++Row payloads are serialized from the product's own types (pa-types message+shapes inside pa-daemon's `SessionEntry` envelope and `session_header_line`+header) — no hand-copied JSON. Entry ids are counter-based unique 8-hex: the+product's own ids are uuid4 prefixes with a 100-try collision check, which+collide around ~22K rows and silently break branch reconstruction.++### Knobs++`--profile small|medium|large|xl` picks the defaults; every knob overrides:++| flag | meaning | small | medium | large | xl |+|---|---|---|---|---|---|+| `--sessions` | top-level sessions | 10 | 100 | 1,000 | 2,500 |+| `--rows` | rows per session | 100 | 2,000 | 10,000 | 400 |+| `--huge-rows` | rows of the huge marker session | 2,000 | 4,000 | 20,000 | 2,000 |+| `--child-rows` | rows per subagent child | 40 | 100 | 150 | 80 |+| `--fanout` | children per parent per level | 2 | 3 | 3 | 2 |+| `--depth` | subagent tree depth | 2 | 2 | 3 | 2 |+| `--tree-parents` | sessions growing a tree | 2 | 5 | 8 | 10 |+| `--wide-children` | one parent's direct children (the operator's ~110-wide real parent) | 0 | 0 | 110 | 0 |+| `--archive` | sessions in sessions-archive/ | 3 | 20 | 200 | 500 |+| `--cwd-files` | files in the project tree | 200 | 2,000 | 4,000 | 2,000 |+| `--artifacts` | per-session artifact partitions (on/off) | on | on | on | on |+| `--cache` | cold/warm (see below) | cold | cold | cold | cold |+| `--seed` | the deterministic seed | 7 | 7 | 7 | 7 |++Total rows: small ~3.5K, medium ~250K, large ~10.1M, xl ~1.2M (medium is+~210 MB on disk; large ~3.5 GB; xl ~1 GB, generated in ~5 s).++### Determinism++Same seed + same canonical path => a byte-identical tree, file mtimes+included (roster order is mtime-desc). Two stamps are the product writer's+own wall clock and are excluded from that contract: the ledger records'+`at` fields, and the corpus-root path embedded in child headers/ledger+records (`parentSession`/`child` are canonical paths).++### Caches (cold vs warm)++- `--cache warm` runs the product's own `pa_core::session::window::WindowedSessionStore::open`+ over every openable session file, writing real `<id>.window-cache.json`+ sidecars. On a real store warm window-cache sidecars are the common state+ (the operator's live store: 71 of 111 sessions carry one).+- The `<id>.info-cache.json` sidecar is written only at worker lease+ release (pa-daemon, no public writer), so generated corpora carry none —+ matching real stores (9 of 111). The `agents-view-warm` scenario measures+ the warm scan via the daemon's in-process cache instead.+- `pa-perf run` generates BOTH corpora (cold for the cold scenarios, warm+ for warm-start/warm-ready) — every scenario runs on the state it needs.+- The corpus settings disable BOTH archive rules (`sessionArchiveMaxAgeDays`+ / `sessionArchiveMaxSessions` "off"): the product's defaults sweep the+ live store down to 200 unarchived sessions (the operator's real store:+ 196 live + 211 archived), so large/xl with archiving off are scaling+ stress profiles, not typical stores, and the boot sweep itself is not+ measured.++## The bench harness++```+# the whole suite (CI + make perf): generates the corpora, routes every+# scenario to the corpus state it needs, writes one results/<scenario>.json+pa-perf run --binary target/release/prime-agent --profile medium --out DIR [--runs N]++# one scenario over one corpus (investigation)+pa-perf bench --binary target/release/prime-agent --corpus DIR --scenario S --runs 10 --json OUT+```++Scenarios (timed from a triggering action to an observable needle — literal+paints as byte needles, cell-diffed content through a vt100 screen model;+the liveness deadline only fails broken runs):++| scenario | corpus | trigger -> needle |+|---|---|---|+| `cold-start` | cold | exec -> the chat first paint (the splash paints before any daemon work — the product's own paint contract) |+| `cold-ready` | cold | the same launch, exec -> the tray's session-usage pill, the first attach-only content (daemon spawn + session create + attach included) |+| `warm-start` | warm | exec -> first paint, daemon pre-started |+| `warm-ready` | warm | the same launch, exec -> the attach pill |+| `agents-view` | cold | left-arrow -> the completed `Inactive (N)` roster heading in the screen model |+| `agents-view-warm` | cold | the same daemon's second populate (its in-process scan cache warm) |+| `open-session` | cold | `--resume <huge session>` -> the huge session's tail marker |+| `session-switch` | cold | enter on the uniquely-filtered switch-b row -> B's transcript marker |+| `at-completion` | cold | the `@` keystroke -> the first completion menu row |++Mechanics (the repo's pty-e2e discipline): the TUI child runs in its own+session with NO controlling tty (`setsid` pre-exec; vendored crossterm+falls back to the stdio pty), 120x36, kitty probe answered identically every+run, `PRIME_AGENT_CODING_AGENT_DIR` + `--daemon-socket` isolate it, and the+env is rebuilt from scratch (no ambient credentials leak in). Between runs+the harness restores the corpus to the manifest's file set (the product's+new session files, sidecars, logs, and leases are deleted) so every run+measures identical state. A discarded warm-up run precedes the samples.++Teardown signals only the harness's own child pids; the daemon stops by the+shutdown protocol and the run FAILS if it keeps accepting — never a+pid/name-based kill.++## report + baseline++```+pa-perf report --baseline crates/pa-perf/baseline.json --results results/*.json [--markdown OUT]+```++Prints the aggregate table (scenario/profile, median, p95, baseline, delta)+and exits nonzero ONLY when a non-report-only median exceeds twice the+baseline. Noisy scenarios (>20% spread across 3 CI runs) are marked+`report_only` in the baseline: their deltas always print, the gate never+fails. `baseline.json` is committed from CI numbers, not laptop/sandbox+ones.++## Local run++`make perf` builds the release binaries and runs the whole suite+(`pa-perf run --profile small`) with the report table printed at the end.+Run it on a quiet machine; the numbers are wall-clock latencies.++## Dependencies++pa-perf adds ONE new third-party dependency: `vt100` (MIT), the screen+model behind the content needles (the agents view's heading counts repaint+as terminal cell diffs, which the raw byte stream cannot see). The crate+never ships — the release lanes build `-p pa-cli --bin prime-agent` only —+and `make deny` stays clean.++## CI++The bench matrix runs all four profiles at 10 runs each; the xl+(2500-session) agents-view populate measures ~2.7 s on an 8-core sandbox.++`.github/workflows/perf.yml`: nightly schedule + workflow_dispatch ++label-gated pull requests (`perf-test` label, [labeled, synchronize] —+labeling is the explicit opt-in). One release build, a bench matrix over+the four profiles, and a report job writing the `$GITHUB_STEP_SUMMARY`+table with the >2x gate.diff --git a/crates/pa-perf/baseline.json b/crates/pa-perf/baseline.jsonnew file mode 100644index 0000000000..57d7333c7c--- /dev/null+++ b/crates/pa-perf/baseline.json@@ -0,0 +1,185 @@+{+ "schema": 1,+ "entries": {+ "session-switch/large": {+ "median_ms": 1034.4,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "agents-view/large": {+ "median_ms": 13458.7,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "at-completion/large": {+ "median_ms": 222.3,+ "report_only": false,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "cold-start/large": {+ "median_ms": 22.2,+ "report_only": false,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "warm-ready/large": {+ "median_ms": 130.5,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "agents-view-warm/large": {+ "median_ms": 11333.6,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "cold-ready/large": {+ "median_ms": 147.4,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "warm-start/large": {+ "median_ms": 12.7,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "open-session/large": {+ "median_ms": 2485.8,+ "report_only": false,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "at-completion/medium": {+ "median_ms": 158.4,+ "report_only": false,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "open-session/medium": {+ "median_ms": 314.4,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "cold-ready/medium": {+ "median_ms": 56.3,+ "report_only": false,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "agents-view-warm/medium": {+ "median_ms": 186.6,+ "report_only": false,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "warm-start/medium": {+ "median_ms": 10.9,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "cold-start/medium": {+ "median_ms": 19.2,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "session-switch/medium": {+ "median_ms": 244.9,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "agents-view/medium": {+ "median_ms": 551.9,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "warm-ready/medium": {+ "median_ms": 56.6,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "at-completion/small": {+ "median_ms": 132.3,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "agents-view-warm/small": {+ "median_ms": 109.0,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "cold-start/small": {+ "median_ms": 19.5,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "session-switch/small": {+ "median_ms": 121.6,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "warm-ready/small": {+ "median_ms": 40.6,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "cold-ready/small": {+ "median_ms": 49.4,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "open-session/small": {+ "median_ms": 165.3,+ "report_only": false,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "agents-view/small": {+ "median_ms": 64.7,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "warm-start/small": {+ "median_ms": 11.2,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "cold-start/xl": {+ "median_ms": 22.3,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "warm-ready/xl": {+ "median_ms": 83.8,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "cold-ready/xl": {+ "median_ms": 95.0,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "agents-view/xl": {+ "median_ms": 3562.7,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "warm-start/xl": {+ "median_ms": 16.0,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "open-session/xl": {+ "median_ms": 279.6,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "at-completion/xl": {+ "median_ms": 200.2,+ "report_only": false,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "agents-view-warm/xl": {+ "median_ms": 3492.8,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ },+ "session-switch/xl": {+ "median_ms": 2435.4,+ "report_only": true,+ "source": "https://github.com/PrimeIntellect-ai/prime-agent/actions/runs/38630077435 (attempts 1-3)"+ }+ }+}diff --git a/crates/pa-perf/src/bench/mod.rs b/crates/pa-perf/src/bench/mod.rsnew file mode 100644index 0000000000..ddf861d295--- /dev/null+++ b/crates/pa-perf/src/bench/mod.rs@@ -0,0 +1,139 @@+//! The bench harness: the scenario set, the options, and the result types+//! (the unix-only runner lives in `runner`/`pty`).++use std::path::PathBuf;++use clap::ValueEnum;+use serde::{Deserialize, Serialize};++/// The measured scenario set. Each is timed from a triggering action to an+/// observable needle — literal paints as byte needles, cell-diffed content+/// through the vt100 screen model; the deadline only fails a broken run.+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize, ValueEnum)]+#[serde(rename_all = "kebab-case")]+pub enum Scenario {+ /// No daemon, cold corpus: exec -> the chat surface's first paint.+ ColdStart,+ /// The same launch as cold-start, timed to the attach: exec -> the+ /// tray's session-usage pill (the first content the surface can only+ /// paint once the daemon session is attached).+ ColdReady,+ /// Daemon pre-started, warm corpus: exec -> the first paint.+ WarmStart,+ /// The warm launch timed to the attach pill.+ WarmReady,+ /// Fresh daemon+TUI: left-arrow -> the roster scan delivered the oldest+ /// session (its unique search-result row paints).+ AgentsView,+ /// The same daemon's second populate (its in-process scan cache warm):+ /// a second TUI's left-arrow -> the completed roster.+ AgentsViewWarm,+ /// Fresh daemon: `--resume <huge session>` -> the huge marker paints.+ OpenSession,+ /// On session A: agents view -> uniquely filter to B -> enter -> B's+ /// transcript marker.+ SessionSwitch,+ /// The `@` keystroke -> the first completion menu row.+ AtCompletion,+}++/// Every bench knob for one scenario over one corpus.+#[derive(Debug, Clone)]+pub struct BenchOptions {+ /// The release binary under test (`target/release/prime-agent`).+ pub binary: PathBuf,+ /// The corpus root (holds `corpus.json`).+ pub corpus: PathBuf,+ pub scenario: Scenario,+ /// Sampled runs (one discarded warm-up always runs before them).+ pub runs: usize,+ /// Where the result JSON is written.+ pub json: PathBuf,+ /// The per-run liveness deadline (seconds) — a broken run fails+ /// instead of hanging.+ pub liveness_secs: u64,+}++/// The whole-suite driver: generates the cold and warm corpora, runs every+/// scenario on the corpus state it needs (cold scenarios on `--cache+/// cold`, warm-* on `--cache warm`), and writes one result JSON per+/// scenario.+#[derive(Debug, Clone)]+pub struct RunOptions {+ /// The release binary under test.+ pub binary: PathBuf,+ /// The named corpus profile.+ pub profile: crate::corpus::Profile,+ /// The output root: the two corpora and `results/` land under it.+ pub out: PathBuf,+ /// The deterministic seed.+ pub seed: u64,+ /// Sampled runs per scenario (one discarded warm-up always runs first).+ pub runs: usize,+ /// The per-run liveness deadline (seconds).+ pub liveness_secs: u64,+}++/// The measured outcome of one scenario over one corpus.+#[derive(Debug, Serialize, Deserialize)]+pub struct BenchResult {+ pub schema: u32,+ pub scenario: Scenario,+ pub profile: String,+ pub runs: usize,+ pub samples_ms: Vec<f64>,+ pub median_ms: f64,+ pub p95_ms: f64,+ pub needle: String,+ pub binary: String,+ pub corpus: BenchCorpus,+}++/// The corpus summary the result records (baseline keys use scenario/profile).+#[derive(Debug, Serialize, Deserialize)]+pub struct BenchCorpus {+ pub root: PathBuf,+ pub sessions: usize,+ pub total_rows: usize,+ pub child_sessions: usize,+ pub archive: usize,+}++/// Run one scenario over one corpus (`options.runs` sampled runs plus one+/// discarded warm-up), then write the result JSON.+///+/// # Errors+///+/// Propagates spawn, needle-timeout, daemon, and IO failures; errors+/// immediately on non-unix platforms.+#[cfg(unix)]+pub fn run(options: &BenchOptions) -> anyhow::Result<BenchResult> {+ runner::run(options)+}++/// Run the whole scenario suite on freshly generated corpora (`RunOptions`).+///+/// # Errors+///+/// Propagates corpus-generation, spawn, needle-timeout, daemon, and IO+/// failures; errors immediately on non-unix platforms.+#[cfg(unix)]+pub fn run_suite(options: &RunOptions) -> anyhow::Result<Vec<BenchResult>> {+ runner::run_suite(options)+}++#[cfg(not(unix))]+pub fn run(_options: &BenchOptions) -> anyhow::Result<BenchResult> {+ anyhow::bail!("the bench harness is unix-only")+}++#[cfg(not(unix))]+pub fn run_suite(_options: &RunOptions) -> anyhow::Result<Vec<BenchResult>> {+ anyhow::bail!("the bench harness is unix-only")+}++#[cfg(unix)]+mod pty;+/// The unix-only harness (pty, daemon lifecycle, scenario runners).+#[cfg(unix)]+mod runner;diff --git a/crates/pa-perf/src/bench/pty.rs b/crates/pa-perf/src/bench/pty.rsnew file mode 100644index 0000000000..97de98a7c4--- /dev/null+++ b/crates/pa-perf/src/bench/pty.rs@@ -0,0 +1,409 @@+//! The pty harness: one product child (the real release binary) on a raw pty+//! in its own session with NO controlling tty — vendored crossterm opens+//! `/dev/tty` first in `window_size()` and falls back to the stdio pty+//! slave, the same discipline the repo's pty e2es use.++use std::io::Read;+use std::os::fd::{AsFd, AsRawFd, OwnedFd};+use std::process::{Child, Command, Stdio};+use std::time::{Duration, Instant};++use nix::fcntl::{fcntl, FcntlArg::F_SETFL, OFlag};+use nix::poll::{poll, PollFd, PollFlags};+use nix::pty::{openpty, Winsize};++/// The kitty-probe pair: the capability query the product's vendored+/// crossterm emits, and the flags+DA1 answer a kitty terminal gives. The+/// bench answers identically every run so the probe path is deterministic.+pub(crate) const KITTY_QUERY: &[u8] = b"\x1b[?u";+pub(crate) const KITTY_ANSWER: &[u8] = b"\x1b[?7u\x1b[?62;c";++/// The idle poll slice between empty reads (poll(2) drives the wakeups).+const IDLE_POLL: Duration = Duration::from_millis(25);++/// One product child on a pty, plus the collected master stream and the+/// decoded screen (a vt100 model): byte needles cover literal paints; the+/// screen model covers content that only exists as cell diffs (the agents+/// view's heading counts).+pub(crate) struct PtyChild {+ pub(crate) child: Child,+ master: std::fs::File,+ pub(crate) output: Vec<u8>,+ screen: vt100::Parser,+ /// The kitty probe is answered exactly once (the answer goes to the+ /// child's stdin, so it never shows up in the read stream — a+ /// stream-search for it would re-send forever).+ kitty_answered: bool,+ /// Set only after the exited child's final PTY bytes have drained.+ exited: Option<std::process::ExitStatus>,+}++impl PtyChild {+ /// Spawn `binary` with the pty slave as stdio, its own session, and no+ /// controlling terminal: `setsid` in the pre-exec hook (the child is+ /// never a process-group leader, so it always succeeds).+ ///+ /// # Errors+ ///+ /// Propagates pty and spawn failures.+ pub(crate) fn spawn(+ binary: &std::path::Path,+ args: &[std::ffi::OsString],+ env: &[(String, String)],+ cwd: &std::path::Path,+ cols: u16,+ rows: u16,+ ) -> anyhow::Result<Self> {+ let pty = openpty(+ Some(&Winsize {+ ws_row: rows,+ ws_col: cols,+ ws_xpixel: 0,+ ws_ypixel: 0,+ }),+ None,+ )?;+ let mut command = Command::new(binary);+ command+ .args(args)+ .current_dir(cwd)+ .env_clear()+ .envs(env.iter().map(|(key, value)| (key, value)))+ .stdin(slave_as_stdio(&pty.slave))+ .stdout(slave_as_stdio(&pty.slave))+ .stderr(slave_as_stdio(&pty.slave));+ pa_core::platform::process::set_new_session(&mut command);+ let fd = pty.master.as_raw_fd();+ fcntl(fd, F_SETFL(OFlag::O_NONBLOCK))?;+ let child = command.spawn()?;+ drop(pty.slave);+ Ok(Self {+ child,+ master: pty.master.into(),+ output: Vec::new(),+ screen: vt100::Parser::new(rows, cols, 0),+ kitty_answered: false,+ exited: None,+ })+ }++ /// Write to the child's input (the pty master's write end).+ ///+ /// # Errors+ ///+ /// Propagates write failures.+ pub(crate) fn write(&mut self, payload: &[u8]) -> anyhow::Result<()> {+ use std::io::Write;+ self.master.write_all(payload)?;+ Ok(())+ }++ /// The current stream mark (needles wait from here).+ pub(crate) fn mark(&self) -> usize {+ self.output.len()+ }++ pub(crate) fn output_from(&self, mark: usize) -> &[u8] {+ &self.output[mark..]+ }++ /// Drain until `needle` appears in the stream collected since `mark`,+ /// bounded by `deadline`. Returns the Instant the needle was seen.+ ///+ /// # Errors+ ///+ /// Errors when the deadline passes (with the stream tail attached) or+ /// the child exits with a failure code.+ pub(crate) fn wait_from(+ &mut self,+ mark: usize,+ needle: &[u8],+ what: &str,+ deadline: Instant,+ ) -> anyhow::Result<Instant> {+ // Search only the bytes that arrived since the last check (plus the+ // needle overlap, never reaching before the mark): a fresh+ // full-stream scan per read is quadratic, and the harness must+ // never out-CPU the product it measures.+ let mut scanned = mark;+ loop {+ let from = scanned+ .saturating_sub(needle.len().saturating_sub(1))+ .max(mark);+ if find(&self.output[from..], needle).is_some() {+ return Ok(Instant::now());+ }+ scanned = self.output.len();+ if let Some(status) = self.exited {+ anyhow::bail!(+ "child exited with {status} before {what}; pty tail:\n{}",+ String::from_utf8_lossy(&self.output[self.output.len().saturating_sub(4000)..])+ );+ }+ if Instant::now() >= deadline {+ let tail =+ &self.output_from(mark)[self.output_from(mark).len().saturating_sub(4000)..];+ anyhow::bail!(+ "timeout waiting for {what} (needle {:?}); pty tail since mark:\n{}",+ String::from_utf8_lossy(needle),+ String::from_utf8_lossy(tail),+ );+ }+ self.pump(Some(deadline))?;+ }+ }++ /// One poll(2) slice on the master: reads whatever is ready, appends to+ /// the stream, and answers the kitty probe once it appears.+ fn pump(&mut self, deadline: Option<Instant>) -> anyhow::Result<()> {+ let timeout = match deadline {+ Some(end) => {+ let remaining = end.saturating_duration_since(Instant::now());+ remaining.min(IDLE_POLL)+ }+ None => IDLE_POLL,+ };+ let millis = timeout.as_millis().min(u16::MAX as u128) as u16;+ let mut poller = [PollFd::new(self.master.as_fd(), PollFlags::POLLIN)];+ let ready = poll(&mut poller, millis)?;+ // After exit, keep reading any buffered final paints before reporting+ // the status. A poll timeout alone must not hide an already dead UI.+ let status = self.child.try_wait()?;+ if ready == 0 && status.is_none() {+ return Ok(());+ }+ let mut buffer = [0u8; 8192];+ let read = self.master.read(&mut buffer);+ match read {+ Ok(0) => self.exited = status,+ Err(error) if error.kind() == std::io::ErrorKind::Interrupted => {}+ Err(error)+ if error.kind() == std::io::ErrorKind::WouldBlock+ || error.raw_os_error() == Some(nix::errno::Errno::EIO as i32) =>+ {+ self.exited = status;+ }+ Err(error) => return Err(error.into()),+ Ok(n) => {+ let chunk = &buffer[..n];+ self.receive(chunk)?;+ }+ }+ Ok(())+ }++ /// Include the previous read's overlap when a terminal query is split.+ fn receive(&mut self, chunk: &[u8]) -> anyhow::Result<()> {+ let from = self.output.len().saturating_sub(KITTY_QUERY.len() - 1);+ self.output.extend_from_slice(chunk);+ self.screen.process(chunk);+ if !self.kitty_answered && find(&self.output[from..], KITTY_QUERY).is_some() {+ self.write(KITTY_ANSWER)?;+ self.kitty_answered = true;+ }+ Ok(())+ }++ /// Wait until the decoded screen contains `needle`, bounded by+ /// `deadline`. Content needles the byte stream cannot see (cell-diffed+ /// counters, repainted rows) live here.+ ///+ /// # Errors+ ///+ /// Errors when the deadline passes (with the decoded screen attached).+ pub(crate) fn wait_screen_contains(+ &mut self,+ needle: &str,+ what: &str,+ deadline: Instant,+ ) -> anyhow::Result<Instant> {+ loop {+ if self.screen_text().contains(needle) {+ return Ok(Instant::now());+ }+ if let Some(status) = self.exited {+ anyhow::bail!(+ "child exited with {status} before {what}; pty tail:\n{}",+ String::from_utf8_lossy(&self.output[self.output.len().saturating_sub(4000)..])+ );+ }+ if Instant::now() >= deadline {+ anyhow::bail!(+ "timeout waiting for {what} (screen needle {needle:?}); screen:\n{}",+ self.screen_text(),+ );+ }+ self.pump(Some(deadline))?;+ }+ }++ /// The decoded screen's text (rows joined; trailing blanks trimmed).+ fn screen_text(&mut self) -> String {+ self.screen.screen().contents().trim_end().to_string()+ }++ /// The child's pid (teardown signals this pid only).+ pub(crate) fn pid(&self) -> i32 {+ self.child.id() as i32+ }++ /// Stop the child: SIGTERM, a short grace, then SIGKILL; reap it.+ pub(crate) fn finish(&mut self) {+ // Drop also cleans up failures before Launch has been constructed.+ // Avoid signalling an already-reaped child if finish ran earlier.+ if self.child.try_wait().is_ok_and(|status| status.is_some()) {+ return;+ }+ let _ = pa_core::platform::process::kill_pid(+ self.pid(),+ pa_core::platform::process::Signal::Term,+ );+ let deadline = Instant::now() + Duration::from_secs(2);+ while Instant::now() < deadline {+ if self.child.try_wait().is_ok_and(|status| status.is_some()) {+ return;+ }+ std::thread::sleep(Duration::from_millis(25));+ }+ let _ = pa_core::platform::process::kill_pid(+ self.pid(),+ pa_core::platform::process::Signal::Kill,+ );+ let _ = self.child.wait();+ }+}++impl Drop for PtyChild {+ fn drop(&mut self) {+ self.finish();+ }+}++fn slave_as_stdio(slave: &OwnedFd) -> Stdio {+ slave.try_clone().expect("clone pty slave").into()+}++fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {+ if needle.is_empty() {+ return None;+ }+ haystack.windows(needle.len()).position(|w| w == needle)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn kitty_queries_are_answered_at_every_read_boundary_once() {+ let dir = tempfile::tempdir().expect("temp dir");+ for split in 1..KITTY_QUERY.len() {+ let mut child = PtyChild::spawn(+ std::path::Path::new("/bin/cat"),+ &[],+ &[],+ dir.path(),+ 120,+ 36,+ )+ .expect("spawn mock terminal peer");+ child.receive(&KITTY_QUERY[..split]).expect("first chunk");+ assert!(!child.kitty_answered);+ child.receive(&KITTY_QUERY[split..]).expect("second chunk");+ assert!(child.kitty_answered, "unanswered query split at {split}");+ // A repeated probe must not write another answer, even if the+ // master is now closed: answer write failures propagate.+ let closed = std::fs::File::open("/dev/null").expect("read-only master");+ child.master = closed;+ child.receive(KITTY_QUERY).expect("already answered");+ }+ }++ #[test]+ fn exited_children_fail_missing_needles_without_waiting_for_liveness_timeout() {+ let directory = tempfile::tempdir().expect("mock directory");+ let mut child = PtyChild::spawn(+ std::path::Path::new("/bin/sh"),+ &["-c".into(), "printf 'fatal mock diagnostic'; exit 7".into()],+ &[],+ directory.path(),+ 120,+ 36,+ )+ .expect("spawn exiting mock");+ let started = Instant::now();+ let error = child+ .wait_from(+ 0,+ b"missing",+ "first paint",+ started + Duration::from_secs(10),+ )+ .expect_err("exited child has no paint");+ let message = error.to_string();+ assert!(message.contains("exit status: 7"), "{message}");+ assert!(message.contains("fatal mock diagnostic"), "{message}");+ assert!(started.elapsed() < Duration::from_secs(5));+ assert!(child+ .wait_screen_contains("missing", "ready", started + Duration::from_secs(10))+ .expect_err("screen wait also sees exit")+ .to_string()+ .contains("exit status: 7"));+ }++ #[test]+ fn final_output_is_drained_before_reporting_child_exit() {+ let directory = tempfile::tempdir().expect("mock directory");+ let mut child = PtyChild::spawn(+ std::path::Path::new("/bin/sh"),+ &[+ "-c".into(),+ "printf '%20000s' x; printf 'last paint'; exit 0".into(),+ ],+ &[],+ directory.path(),+ 120,+ 36,+ )+ .expect("spawn final paint mock");+ child+ .wait_from(+ 0,+ b"last paint",+ "final paint",+ Instant::now() + Duration::from_secs(5),+ )+ .expect("final bytes survive child exit");+ assert!(child.output.len() > 8192, "exercise more than one PTY read");+ child+ .wait_screen_contains(+ "last paint",+ "final screen",+ Instant::now() + Duration::from_secs(5),+ )+ .expect("final screen survives child exit");+ }++ #[test]+ fn an_initial_paint_timeout_reaps_the_owned_child() {+ let dir = tempfile::tempdir().expect("temp dir");+ let mut child = PtyChild::spawn(+ std::path::Path::new("/bin/cat"),+ &[],+ &[],+ dir.path(),+ 120,+ 36,+ )+ .expect("spawn the mock, never the product");+ let pid = nix::unistd::Pid::from_raw(child.pid());+ assert!(child+ .wait_from(0, b"missing first paint", "first paint", Instant::now())+ .is_err());+ drop(child);+ assert_eq!(+ nix::sys::signal::kill(pid, None),+ Err(nix::errno::Errno::ESRCH)+ );+ }+}diff --git a/crates/pa-perf/src/bench/runner.rs b/crates/pa-perf/src/bench/runner.rsnew file mode 100644index 0000000000..aee1a9562c--- /dev/null+++ b/crates/pa-perf/src/bench/runner.rs@@ -0,0 +1,1044 @@+//! The scenario runner: pty harness, daemon lifecycle, per-run corpus+//! restore, and the scenario mechanics. Unix-only.++use std::collections::BTreeMap;+use std::io::{BufRead, BufReader, Write};+use std::path::{Path, PathBuf};+use std::time::{Duration, Instant};++use anyhow::Context;++use crate::bench::{BenchCorpus, BenchOptions, BenchResult, RunOptions, Scenario};+use crate::corpus::{self, CacheMode, CorpusOptions, Manifest};++use super::pty::PtyChild;++/// The first paint's needle: the brand splash is the empty chat's header+/// (the corpus settings keep the TUI out of the first-run flow). Pinned+/// empirically against the real binary.+const FIRST_PAINT_NEEDLE: &[u8] = "▗▄▄█▀".as_bytes();++/// The ready needle: the tray's session-usage pill ("model · tokens+/// (percent)") — the first content the chat surface can only paint once+/// the daemon session is attached. A fresh session's usage is always+/// "0 (0%)".+const READY_NEEDLE: &str = "0 (0%)";++/// One bench run's anchored paths.+struct RunCtx {+ binary: PathBuf,+ manifest: Manifest,+ corpus_root: PathBuf,+ project_dir: PathBuf,+ agent_dir: PathBuf,+ liveness: Duration,+}++impl RunCtx {+ /// The manifest's agent dir paths, from the corpus at `root`.+ fn load(binary: &Path, corpus_root: &Path, liveness: Duration) -> anyhow::Result<Self> {+ let manifest_path = corpus_root.join("corpus.json");+ let manifest: Manifest = serde_json::from_str(+ &std::fs::read_to_string(&manifest_path)+ .with_context(|| format!("read {}", manifest_path.display()))?,+ )+ .with_context(|| format!("parse {}", manifest_path.display()))?;+ Ok(Self {+ // Spawns run in the generated project directory. Resolve the+ // caller's path before changing cwd (including `make perf`).+ binary: binary+ .canonicalize()+ .with_context(|| format!("resolve benchmark binary {}", binary.display()))?,+ project_dir: manifest.paths.project_dir.clone(),+ agent_dir: manifest.paths.agent_dir.clone(),+ corpus_root: corpus_root+ .canonicalize()+ .with_context(|| format!("resolve corpus root {}", corpus_root.display()))?,+ manifest,+ liveness,+ })+ }+}++/// The child environment: hermetic (a clean env plus the isolation vars).+fn child_env(ctx: &RunCtx) -> Vec<(String, String)> {+ let mut env = vec![+ (+ "PRIME_AGENT_CODING_AGENT_DIR".to_string(),+ ctx.agent_dir.display().to_string(),+ ),+ ("DO_NOT_TRACK".to_string(), "1".to_string()),+ ("TERM".to_string(), "xterm-ghostty".to_string()),+ ("TERM_PROGRAM".to_string(), "ghostty".to_string()),+ ];+ // PI_PACKAGE_DIR carries the packaged runtime dir to the binary and its+ // workers (a release binary resolves prime-agent-runtime/ beside the+ // executable or through it); PATH/HOME/TMPDIR are the ambient basics.+ for carry in ["PATH", "HOME", "TMPDIR", "PI_PACKAGE_DIR"] {+ if let Some(value) = std::env::var_os(carry) {+ env.push((carry.to_string(), value.to_string_lossy().to_string()));+ }+ }+ env+}++// The single-scenario entry (`pa-perf bench`): one scenario over the given+// corpus — the debugging/investigation path.++/// Run one scenario `options.runs` times (plus one discarded warm-up), then+/// write the result JSON.+///+/// # Errors+///+/// Propagates spawn, needle-timeout, daemon, and IO failures.+pub(super) fn run(options: &BenchOptions) -> anyhow::Result<BenchResult> {+ anyhow::ensure!(options.runs > 0, "sampled runs must be positive");+ let ctx = RunCtx::load(+ &options.binary,+ &options.corpus,+ Duration::from_secs(options.liveness_secs),+ )?;+ let samples = sample_runs(&ctx, options.runs, |index| {+ scenario_once(&ctx, options.scenario, index)+ })?;+ let result = build_result(&ctx, options.scenario, samples);+ write_result(&result, &options.json)?;+ Ok(result)+}++/// Restore the manifest baseline before every standalone launch, including+/// its discarded warm-up. The scenario callback keeps this loop testable+/// with filesystem mutations instead of running the product.+fn sample_runs(+ ctx: &RunCtx,+ runs: usize,+ mut scenario: impl FnMut(usize) -> anyhow::Result<f64>,+) -> anyhow::Result<Vec<f64>> {+ let mut samples = Vec::with_capacity(runs + 1);+ let total = runs + 1;+ for index in 0..total {+ restore(ctx)?;+ let started = Instant::now();+ let outcome = scenario(index);+ if index == 0 {+ outcome?;+ continue;+ }+ let sample = outcome.context("a sampled run failed")?;+ samples.push(sample);+ println!(+ "run {index}/{runs}: {sample:.1}ms (wall {:.1}s)",+ started.elapsed().as_secs_f32()+ );+ }+ Ok(samples)+}++/// One scenario pass; returns the sample in ms.+fn scenario_once(ctx: &RunCtx, scenario: Scenario, run: usize) -> anyhow::Result<f64> {+ let (sample, _) = launch(ctx, scenario, run)?;+ Ok(sample)+}++/// The ONE launch path: the pre-started daemon for warm-* scenarios, the+/// scenario's own resume target + first needle, the post-launch mechanics,+/// and the daemon teardown on every path. Both `run()` (single scenario)+/// and `run_suite()` call this.+fn launch(ctx: &RunCtx, scenario: Scenario, run: usize) -> anyhow::Result<(f64, Launch)> {+ let socket = run_socket(ctx, run);+ std::fs::create_dir_all(socket.parent().expect("a socket dir"))?;+ let mut daemon = if matches!(+ scenario,+ Scenario::WarmStart | Scenario::WarmReady | Scenario::AgentsViewWarm+ ) {+ // AgentsViewWarm pre-starts so both TUIs share the daemon the first+ // populate warms; the single-scenario path cannot express that (its+ // second launch would spawn a second daemon), so it runs the first+ // populate only and the sample is the suite's warm number.+ Some(start_daemon(ctx, &socket)?)+ } else {+ None+ };+ let outcome: anyhow::Result<(f64, Launch)> = (|| {+ let mut launched = spawn_for_scenario(ctx, &socket, scenario, run)?;+ let sample = launch_scenario(ctx, &mut launched, scenario)?;+ // The measured client must disconnect before daemon shutdown.+ // Keeping it alive while waiting for daemon exit can hold server+ // connection draining open or start the client's reconnect path.+ launched.child.finish();+ Ok((sample, launched))+ })();+ let stopped = stop_daemon(&socket);+ // Socket closure alone does not reap a daemon we directly spawned.+ // Keep ownership through shutdown, and fail before the guard's forced+ // cleanup if it does not exit normally.+ let exited = if let Some(daemon) = &mut daemon {+ let deadline = Instant::now() + Duration::from_secs(5);+ loop {+ if let Some(status) = daemon.child.try_wait()? {+ break if status.success() {+ Ok(())+ } else {+ Err(anyhow::anyhow!("the owned daemon exited with {status}"))+ };+ }+ if Instant::now() >= deadline {+ break Err(anyhow::anyhow!(+ "the owned daemon did not exit after shutdown"+ ));+ }+ std::thread::sleep(Duration::from_millis(5));+ }+ } else {+ Ok(())+ };+ let result = outcome?;+ stopped?;+ exited.with_context(|| format!("{scenario:?} run {run} daemon cleanup"))?;+ Ok(result)+}++// The whole-suite driver (`pa-perf run`): generates the cold and warm+// corpora and runs every scenario on the corpus state it needs.++/// Run the full scenario schedule on freshly generated corpora, writing one+/// result JSON per scenario under `<out>/results/`.+///+/// # Errors+///+/// Propagates corpus-generation, spawn, needle-timeout, daemon, and IO+/// failures.+pub(super) fn run_suite(options: &RunOptions) -> anyhow::Result<Vec<BenchResult>> {+ anyhow::ensure!(options.runs > 0, "sampled runs must be positive");+ let liveness = Duration::from_secs(options.liveness_secs);+ let knobs = || CorpusOptions {+ seed: options.seed,+ ..options.profile.defaults()+ };+ println!("=== generating the cold corpus");+ corpus::regenerate_for_suite(&CorpusOptions {+ out: options.out.join("corpus-cold"),+ ..knobs()+ })?;+ println!("=== generating the warm corpus");+ corpus::regenerate_for_suite(&CorpusOptions {+ out: options.out.join("corpus-warm"),+ cache: CacheMode::Warm,+ ..knobs()+ })?;+ let cold_ctx = RunCtx::load(&options.binary, &options.out.join("corpus-cold"), liveness)?;+ let warm_ctx = RunCtx::load(&options.binary, &options.out.join("corpus-warm"), liveness)?;++ let mut samples: BTreeMap<Scenario, Vec<f64>> = BTreeMap::new();+ let total = options.runs + 1;+ for index in 0..total {+ let started = Instant::now();+ // The schedule: cold scenarios on the cold corpus, warm-* on the+ // warm corpus. `launch_cold`/`launch_warm` below run one scenario+ // per launch; the cold-start/cold-ready (and warm-*) pairs share+ // one launch each.+ restore(&cold_ctx)?;+ let (start_sample, launched) = launch(&cold_ctx, Scenario::ColdStart, index)?;+ let ready_sample = launched+ .ready_ms+ .context("the cold launch never attached")?;+ push(&mut samples, index, Scenario::ColdStart, start_sample);+ push(&mut samples, index, Scenario::ColdReady, ready_sample);+ for scenario in [+ Scenario::AgentsView,+ Scenario::AgentsViewWarm,+ Scenario::OpenSession,+ Scenario::SessionSwitch,+ Scenario::AtCompletion,+ ] {+ restore(&cold_ctx)?;+ let (sample, _) = launch(&cold_ctx, scenario, index)?;+ push(&mut samples, index, scenario, sample);+ }+ restore(&warm_ctx)?;+ let (start_sample, launched) = launch(&warm_ctx, Scenario::WarmStart, index)?;+ let ready_sample = launched+ .ready_ms+ .context("the warm launch never attached")?;+ push(&mut samples, index, Scenario::WarmStart, start_sample);+ push(&mut samples, index, Scenario::WarmReady, ready_sample);+ println!(+ "run {index}/{} complete ({:.1}s)",+ options.runs,+ started.elapsed().as_secs_f32()+ );+ }++ let results_dir = options.out.join("results");+ std::fs::create_dir_all(&results_dir)?;+ let mut results = Vec::new();+ for (scenario, mut samples) in samples {+ samples.sort_by(f64::total_cmp);+ if samples.len() != options.runs {+ anyhow::bail!(+ "{scenario:?} collected {} samples, expected {}",+ samples.len(),+ options.runs+ );+ }+ let ctx = corpus_for(scenario, &cold_ctx, &warm_ctx);+ let result = build_result(ctx, scenario, samples);+ let name = crate::report::key_of(scenario);+ // Per-profile unique names: the CI legs merge their artifacts into+ // one results dir, and same-named files would overwrite each other.+ let profile = crate::report::profile_key(options.profile);+ write_result(&result, &results_dir.join(format!("{profile}-{name}.json")))?;+ println!(+ "scenario={name} profile={} median={:.1}ms p95={:.1}ms",+ crate::report::profile_key(options.profile),+ result.median_ms,+ result.p95_ms+ );+ results.push(result);+ }+ Ok(results)+}++/// Keep result provenance paired with the corpus used by the schedule.+fn corpus_for<'a>(scenario: Scenario, cold: &'a RunCtx, warm: &'a RunCtx) -> &'a RunCtx {+ match scenario {+ Scenario::WarmStart | Scenario::WarmReady => warm,+ Scenario::ColdStart+ | Scenario::ColdReady+ | Scenario::AgentsView+ | Scenario::AgentsViewWarm+ | Scenario::OpenSession+ | Scenario::SessionSwitch+ | Scenario::AtCompletion => cold,+ }+}++/// Push a sample unless this is the discarded warm-up run.+fn push(samples: &mut BTreeMap<Scenario, Vec<f64>>, index: usize, scenario: Scenario, sample: f64) {+ if index > 0 {+ samples.entry(scenario).or_default().push(sample);+ }+}++/// The resume target + first needle per scenario, then the spawn.+fn spawn_for_scenario(+ ctx: &RunCtx,+ socket: &Path,+ scenario: Scenario,+ run: usize,+) -> anyhow::Result<Launch> {+ let (resume, first_needle) = match scenario {+ Scenario::OpenSession => (+ Some(ctx.manifest.markers.huge.file.clone()),+ ctx.manifest.markers.huge.needle.as_bytes().to_vec(),+ ),+ Scenario::SessionSwitch => (+ Some(ctx.manifest.markers.switch_a.file.clone()),+ ctx.manifest.markers.switch_a.needle.as_bytes().to_vec(),+ ),+ _ => (None, FIRST_PAINT_NEEDLE.to_vec()),+ };+ let dump = ctx+ .corpus_root+ .join("failed-streams")+ .join(format!("{}-run{run}.bin", crate::report::key_of(scenario)));+ let mut launch = Launch::spawn(ctx, socket, resume.as_deref(), &first_needle, dump)?;+ if matches!(+ scenario,+ Scenario::ColdStart | Scenario::WarmStart | Scenario::ColdReady | Scenario::WarmReady+ ) {+ launch.wait_ready()?;+ }+ Ok(launch)+}++/// The scenario mechanics after the launch needle; a completed launch+/// never dumps its stream.+fn launch_scenario(ctx: &RunCtx, launch: &mut Launch, scenario: Scenario) -> anyhow::Result<f64> {+ let sample = post_launch(ctx, launch, scenario)?;+ launch.completed = true;+ Ok(sample)+}++/// The scenario mechanics after the launch needle.+fn post_launch(ctx: &RunCtx, launch: &mut Launch, scenario: Scenario) -> anyhow::Result<f64> {+ match scenario {+ Scenario::ColdStart | Scenario::WarmStart | Scenario::OpenSession => Ok(launch.first_ms),+ Scenario::AgentsView => agents_view_populate(ctx, &mut launch.child),+ Scenario::AgentsViewWarm => {+ // The first populate warms the daemon's in-process scan cache;+ // the second TUI measures the warm path on the same daemon.+ agents_view_populate(ctx, &mut launch.child)?;+ let socket = launch.socket.clone();+ let dump = ctx+ .corpus_root+ .join("failed-streams")+ .join("agents-view-warm-second.bin");+ let mut second = Launch::spawn(ctx, &socket, None, FIRST_PAINT_NEEDLE, dump)?;+ let sample = agents_view_populate(ctx, &mut second.child)?;+ second.completed = true;+ Ok(sample)+ }+ Scenario::SessionSwitch => switch_to_target_b(ctx, &mut launch.child),+ Scenario::AtCompletion => at_completion(ctx, &mut launch.child),+ Scenario::ColdReady | Scenario::WarmReady => Ok(launch.ready_ms.unwrap_or(launch.first_ms)),+ }+}++/// One TUI child plus its teardown: Drop finishes the child, so every path —+/// success, `?` bubbles, panics — cleans up.+struct Launch {+ child: PtyChild,+ socket: PathBuf,+ first_ms: f64,+ ready_ms: Option<f64>,+ spawned: Instant,+ liveness: Duration,+ /// Where the raw pty stream lands when the launch does NOT complete (the+ /// CI failure artifact for empty-tail debugging).+ stream_dump: Option<PathBuf>,+ completed: bool,+}++impl Launch {+ /// Spawn the TUI (fresh daemon when none listens on `socket`), wait the+ /// first needle (the brand-splash paint, or the resumed transcript's+ /// marker), and record exec -> needle.+ fn spawn(+ ctx: &RunCtx,+ socket: &Path,+ resume: Option<&Path>,+ first_needle: &[u8],+ stream_dump: PathBuf,+ ) -> anyhow::Result<Self> {+ let mut args = vec![+ "--daemon-socket".into(),+ socket.display().to_string().into(),+ ];+ if let Some(file) = resume {+ args.push("--resume".into());+ args.push(file.display().to_string().into());+ }+ let started = Instant::now();+ let child = PtyChild::spawn(+ &ctx.binary,+ &args,+ &child_env(ctx),+ &ctx.project_dir,+ 120,+ 36,+ )?;+ let what = if resume.is_some() {+ "the resumed transcript paint"+ } else {+ "the first paint"+ };+ let mut launch = Self {+ first_ms: 0.0,+ child,+ socket: socket.to_path_buf(),+ ready_ms: None,+ spawned: started,+ liveness: ctx.liveness,+ stream_dump: Some(stream_dump),+ completed: false,+ };+ launch.child.wait_from(+ launch.child.mark(),+ first_needle,+ what,+ started + ctx.liveness,+ )?;+ launch.first_ms = started.elapsed().as_secs_f64() * 1000.0;+ Ok(launch)+ }++ /// The ready metric: exec -> the tray's session-usage pill, the first+ /// attach-only content on the chat surface.+ fn wait_ready(&mut self) -> anyhow::Result<()> {+ self.child.wait_screen_contains(+ READY_NEEDLE,+ "the attach pill",+ self.spawned + self.liveness,+ )?;+ self.ready_ms = Some(self.spawned.elapsed().as_secs_f64() * 1000.0);+ Ok(())+ }+}++impl Drop for Launch {+ fn drop(&mut self) {+ if !self.completed {+ if let Some(path) = &self.stream_dump {+ if let Some(parent) = path.parent() {+ let _ = std::fs::create_dir_all(parent);+ }+ let _ = std::fs::write(path, &self.child.output);+ }+ }+ self.child.finish();+ }+}++/// The left-arrow opens the agents view; the populate needle is the+/// completed `Inactive (N)` roster heading in the decoded screen (the+/// heading count repaints as cell diffs, so the byte stream cannot see it+/// — the screen model does). The count includes the TUI's own fresh+/// session (rendered as one more inactive row).+fn agents_view_populate(ctx: &RunCtx, child: &mut PtyChild) -> anyhow::Result<f64> {+ let needle = format!("Inactive ({})", ctx.manifest.markers.roster_inactive + 1);+ let started = Instant::now();+ child.write(b"\x1b[D")?;+ child.wait_screen_contains(+ &needle,+ "the agents view roster",+ Instant::now() + ctx.liveness,+ )?;+ Ok(started.elapsed().as_secs_f64() * 1000.0)+}++/// From the opened session A: open the agents view, uniquely filter to+/// switch-b, and time enter -> B's transcript marker. The `zq` token in+/// switch-b's name matches no other row (no generated text carries the+/// letter z), so Enter always drills THE row.+fn switch_to_target_b(ctx: &RunCtx, child: &mut PtyChild) -> anyhow::Result<f64> {+ child.write(b"\x1b[D")?;+ // Wait the completed roster before typing: keys racing the scan flood+ // were the CI-runner failure (the query echoed nothing). The resumed+ // session A renders as one IDLE row, so the completed heading counts+ // the remaining sessions.+ child.wait_screen_contains(+ &format!("Inactive ({})", ctx.manifest.markers.roster_inactive - 1),+ "the agents view roster",+ Instant::now() + ctx.liveness,+ )?;+ child.write(ctx.manifest.markers.switch_query.as_bytes())?;+ // The echoed query does not contain the row's "switch" prefix, so this+ // needle is the picker's result row, not the echo.+ child.wait_from(+ child.mark(),+ ctx.manifest.markers.switch_row.as_bytes(),+ "the switch-b picker row",+ Instant::now() + ctx.liveness,+ )?;+ let mark = child.mark();+ let started = Instant::now();+ child.write(b"\r")?;+ child.wait_from(+ mark,+ ctx.manifest.markers.switch_b.needle.as_bytes(),+ "the switch-b transcript",+ Instant::now() + ctx.liveness,+ )?;+ Ok(started.elapsed().as_secs_f64() * 1000.0)+}++/// The `@` completion: the trigger is the `@` keystroke alone; the needle+/// is the first painted completion menu row (the menu lists the project+/// tree's entries — `src/` is a top-level dir of the generated project+/// that the pre-menu screen never contains).+fn at_completion(ctx: &RunCtx, child: &mut PtyChild) -> anyhow::Result<f64> {+ let started = Instant::now();+ child.write(b"@")?;+ // The menu lists the project tree's entries — `src/` is a top-level+ // dir of the generated project that the pre-menu screen never contains+ // (the visible first rows are dirs, not the .rs files).+ child.wait_screen_contains(+ "src/",+ "the completion menu row",+ Instant::now() + ctx.liveness,+ )?;+ Ok(started.elapsed().as_secs_f64() * 1000.0)+}++/// The per-run isolated socket path.+fn run_socket(ctx: &RunCtx, run: usize) -> PathBuf {+ ctx.corpus_root+ .join("bench-runs")+ .join(format!("run-{run}"))+ .join("daemon.sock")+}++/// Start the daemon (own session, detached) and wait for the socket to+/// accept — the observable readiness.+fn start_daemon(ctx: &RunCtx, socket: &Path) -> anyhow::Result<DaemonChild> {+ std::fs::create_dir_all(socket.parent().expect("a socket dir"))?;+ let args: [std::ffi::OsString; 4] = [+ "--mode".into(),+ "daemon".into(),+ "--daemon-socket".into(),+ socket.display().to_string().into(),+ ];+ let env = child_env(ctx);+ let mut command = std::process::Command::new(&ctx.binary);+ command+ .args(&args)+ .env_clear()+ .envs(env.iter().map(|(k, v)| (k, v)))+ .current_dir(&ctx.project_dir)+ .stdin(std::process::Stdio::null())+ .stdout(std::process::Stdio::null())+ .stderr(std::process::Stdio::null());+ pa_core::platform::process::set_new_session(&mut command);+ DaemonChild {+ child: command.spawn()?,+ }+ .wait_until_ready(socket, Instant::now() + Duration::from_secs(30))+}++/// Direct child ownership survives startup and normal protocol shutdown.+/// Drop is an error-path fallback, restricted to this exact child handle.+struct DaemonChild {+ child: std::process::Child,+}++impl DaemonChild {+ fn wait_until_ready(mut self, socket: &Path, deadline: Instant) -> anyhow::Result<Self> {+ loop {+ if let Some(status) = self.child.try_wait()? {+ anyhow::bail!("the daemon exited before socket readiness: {status}");+ }+ if socket_accepts(socket) {+ return Ok(self);+ }+ if Instant::now() >= deadline {+ anyhow::bail!("the daemon socket never accepted: {}", socket.display());+ }+ std::thread::sleep(Duration::from_millis(5));+ }+ }+}++impl Drop for DaemonChild {+ fn drop(&mut self) {+ if self.child.try_wait().is_ok_and(|status| status.is_some()) {+ return;+ }+ let _ = pa_core::platform::process::kill_pid(+ self.child.id() as i32,+ pa_core::platform::process::Signal::Kill,+ );+ let _ = self.child.wait();+ }+}++fn socket_accepts(socket: &Path) -> bool {+ std::os::unix::net::UnixStream::connect(socket).is_ok()+}++/// Stop the daemon on `socket` by the shutdown protocol: the hello line,+/// the shutdown command, one response read, then wait until the socket no+/// longer accepts (the observable exit). A daemon that never stops fails+/// the run loudly — no pid sweeps.+fn stop_daemon(socket: &Path) -> anyhow::Result<()> {+ protocol_shutdown(socket);+ let deadline = Instant::now() + Duration::from_secs(10);+ while Instant::now() < deadline {+ if !socket_accepts(socket) {+ let _ = std::fs::remove_file(socket);+ return Ok(());+ }+ std::thread::sleep(Duration::from_millis(10));+ }+ anyhow::bail!(+ "the daemon on {} kept accepting after the shutdown protocol; the run is not clean",+ socket.display()+ )+}++/// The daemon-shutdown protocol (the e2es' `graceful_shutdown` shape): the+/// hello line, the shutdown command, one response read.+fn protocol_shutdown(socket: &Path) {+ let Ok(stream) = std::os::unix::net::UnixStream::connect(socket) else {+ return;+ };+ let Ok(write_half) = stream.try_clone() else {+ return;+ };+ let mut reader = BufReader::new(stream);+ let mut writer = write_half;+ // The hello can stall too: bound every read, not just the reply.+ if reader+ .get_ref()+ .set_read_timeout(Some(Duration::from_secs(5)))+ .is_err()+ {+ return;+ }+ let mut hello = String::new();+ if !matches!(reader.read_line(&mut hello), Ok(bytes) if bytes > 0) {+ return;+ }+ let command = serde_json::json!({+ "type": "command",+ "id": "bench-shutdown",+ "protocol": { "name": "prime-agent.daemon", "version": 7 },+ "command": { "type": "shutdown" },+ });+ let Ok(mut line) = serde_json::to_string(&command) else {+ return;+ };+ line.push('\n');+ if writer.write_all(line.as_bytes()).is_err() {+ return;+ }+ let _ = writer.flush();+ let _ = reader+ .get_ref()+ .set_read_timeout(Some(Duration::from_secs(5)));+ let mut response = String::new();+ let _ = reader.read_line(&mut response);+}++/// Restore the corpus's exact generated state: delete every file under the+/// agent dir that the manifest does not list (the product's new session+/// files, sidecars, logs, leases), then prune the emptied dirs.+fn restore(ctx: &RunCtx) -> anyhow::Result<()> {+ let baseline: std::collections::HashSet<&str> =+ ctx.manifest.files.iter().map(String::as_str).collect();+ let mut dirs = Vec::new();+ let mut stack = vec![ctx.agent_dir.clone()];+ while let Some(dir) = stack.pop() {+ for entry in std::fs::read_dir(&dir)? {+ let entry = entry?;+ let path = entry.path();+ // Never follow product-created links outside the owned corpus.+ if entry.file_type()?.is_dir() {+ dirs.push(path.clone());+ stack.push(path);+ } else if !baseline.contains(path.to_string_lossy().as_ref()) {+ let _ = std::fs::remove_file(&path);+ }+ }+ }+ for dir in dirs.iter().rev() {+ let _ = std::fs::remove_dir(dir);+ }+ Ok(())+}++/// The result object for one scenario's samples.+fn build_result(ctx: &RunCtx, scenario: Scenario, samples: Vec<f64>) -> BenchResult {+ let (median_ms, p95_ms) = stats(&samples);+ BenchResult {+ schema: 1,+ scenario,+ profile: crate::report::profile_key(ctx.manifest.profile),+ runs: samples.len(),+ median_ms,+ p95_ms,+ samples_ms: samples,+ needle: needle_of(scenario, ctx),+ binary: ctx.binary.display().to_string(),+ corpus: BenchCorpus {+ root: ctx.corpus_root.clone(),+ sessions: ctx.manifest.counts.sessions,+ total_rows: ctx.manifest.counts.total_rows,+ child_sessions: ctx.manifest.counts.child_sessions,+ archive: ctx.manifest.counts.archive,+ },+ }+}++/// Median + p95 over the samples.+fn stats(samples: &[f64]) -> (f64, f64) {+ let mut sorted = samples.to_vec();+ sorted.sort_by(f64::total_cmp);+ let median = if sorted.len() % 2 == 1 {+ sorted[sorted.len() / 2]+ } else {+ (sorted[sorted.len() / 2 - 1] + sorted[sorted.len() / 2]) / 2.0+ };+ let p95_index = ((sorted.len() as f64 - 1.0) * 0.95).ceil() as usize;+ (median, sorted[p95_index.min(sorted.len() - 1)])+}++fn needle_of(scenario: Scenario, ctx: &RunCtx) -> String {+ match scenario {+ Scenario::ColdStart | Scenario::WarmStart => {+ "first paint (the brand splash over the empty new session)".to_string()+ }+ Scenario::ColdReady | Scenario::WarmReady => {+ "the attach pill (the tray's session-usage row)".to_string()+ }+ Scenario::AgentsView | Scenario::AgentsViewWarm => format!(+ "the completed roster heading (Inactive ({}))",+ ctx.manifest.markers.roster_inactive + 1+ ),+ Scenario::OpenSession => {+ format!(+ "the huge session marker {}",+ ctx.manifest.markers.huge.needle+ )+ }+ Scenario::SessionSwitch => format!(+ "the enter drill-in to switch-b (its transcript marker {})",+ ctx.manifest.markers.switch_b.needle+ ),+ Scenario::AtCompletion => "the first @ completion menu row".to_string(),+ }+}++fn write_result(result: &BenchResult, path: &Path) -> anyhow::Result<()> {+ let json = serde_json::to_string_pretty(result)?;+ std::fs::write(path, json).with_context(|| format!("write {}", path.display()))+}++#[cfg(test)]+mod tests {+ use super::*;++ fn tiny_context(root: &Path) -> RunCtx {+ corpus::generate(&CorpusOptions {+ out: root.to_path_buf(),+ sessions: 3,+ rows: 100,+ huge_rows: 100,+ tree_parents: 0,+ archive: 0,+ cwd_files: 1,+ ..crate::corpus::Profile::Small.defaults()+ })+ .expect("generate the tiny corpus");+ // A relative file is sufficient: loading does not execute it.+ RunCtx::load(Path::new("Cargo.toml"), root, Duration::ZERO).expect("load corpus")+ }++ #[test]+ fn the_binary_path_is_anchored_before_entering_the_corpus_project() {+ let dir = tempfile::tempdir().expect("temp dir");+ let ctx = tiny_context(dir.path());+ let binary = Path::new("Cargo.toml");+ assert_eq!(+ ctx.binary,+ binary.canonicalize().expect("anchor the binary")+ );+ }++ #[test]+ fn relative_corpus_paths_anchor_the_socket_before_changing_project() {+ let dir = tempfile::tempdir_in(".").expect("relative corpus directory");+ // TempDir may return an absolute path even for a relative parent.+ // The directory lives in cwd, so its name is the caller's relative path.+ let relative_root = Path::new(dir.path().file_name().expect("temporary directory name"));+ assert!(+ relative_root.is_relative(),+ "exercise a relative caller path"+ );+ let ctx = tiny_context(relative_root);+ assert!(run_socket(&ctx, 0).is_absolute());+ assert_eq!(+ ctx.corpus_root,+ dir.path().canonicalize().expect("absolute root")+ );+ assert_ne!(ctx.project_dir, ctx.corpus_root);+ }++ #[test]+ fn warm_results_record_the_corpus_that_was_measured() {+ let dir = tempfile::tempdir().expect("temp dir");+ let cold = tiny_context(&dir.path().join("cold"));+ let warm = tiny_context(&dir.path().join("warm"));+ for scenario in [+ Scenario::WarmStart,+ Scenario::WarmReady,+ Scenario::ColdStart,+ ] {+ let measured = corpus_for(scenario, &cold, &warm);+ let result = build_result(measured, scenario, vec![1.0, 2.0]);+ let expected = if scenario == Scenario::ColdStart {+ &cold+ } else {+ &warm+ };+ assert_eq!(result.corpus.root, expected.corpus_root);+ }+ }++ #[test]+ fn zero_samples_fail_before_loading_or_generating_corpora() {+ let dir = tempfile::tempdir().expect("temp dir");+ let absent = dir.path().join("must-not-be-created");+ let standalone = run(&BenchOptions {+ binary: absent.clone(),+ corpus: absent.clone(),+ scenario: Scenario::ColdStart,+ runs: 0,+ json: absent.clone(),+ liveness_secs: 1,+ });+ assert_eq!(+ standalone.expect_err("zero runs rejected").to_string(),+ "sampled runs must be positive"+ );+ let suite = run_suite(&RunOptions {+ binary: absent.clone(),+ profile: crate::corpus::Profile::Small,+ out: absent.clone(),+ seed: 1,+ runs: 0,+ liveness_secs: 1,+ });+ assert_eq!(+ suite.expect_err("zero suite runs rejected").to_string(),+ "sampled runs must be positive"+ );+ assert!(!absent.exists(), "validation must precede output mutation");+ }++ #[test]+ fn a_first_paint_failure_retains_the_raw_stream() {+ use std::os::unix::fs::PermissionsExt;++ let dir = tempfile::tempdir().expect("temp dir");+ let mut ctx = tiny_context(dir.path());+ let mock = dir.path().join("mock-ui");+ std::fs::write(+ &mock,+ "#!/bin/sh\nprintf 'diagnostic before missing paint\\n'\nexec /bin/cat\n",+ )+ .expect("write mock UI");+ std::fs::set_permissions(&mock, std::fs::Permissions::from_mode(0o700))+ .expect("executable mock");+ ctx.binary = mock;+ ctx.liveness = Duration::from_secs(1);+ let dump = dir.path().join("failed-streams/first.bin");+ let launched = Launch::spawn(+ &ctx,+ &dir.path().join("s"),+ None,+ b"absent paint",+ dump.clone(),+ );+ assert!(+ launched.is_err(),+ "the mock never paints the requested needle"+ );+ assert!(std::fs::read(&dump)+ .expect("failure dump")+ .windows(b"diagnostic before missing paint".len())+ .any(|bytes| bytes == b"diagnostic before missing paint"));+ }++ #[test]+ fn restore_removes_links_without_traversing_external_targets() {+ use std::os::unix::fs::symlink;++ let directory = tempfile::tempdir().expect("corpus directory");+ let ctx = tiny_context(directory.path());+ let external = tempfile::tempdir().expect("external directory");+ let sentinel = external.path().join("sentinel");+ std::fs::write(&sentinel, b"external data").expect("external sentinel");+ let directory_link = ctx.agent_dir.join("outside-dir");+ let file_link = ctx.agent_dir.join("outside-file");+ symlink(external.path(), &directory_link).expect("directory link");+ symlink(&sentinel, &file_link).expect("file link");+ restore(&ctx).expect("restore without following links");+ assert_eq!(+ std::fs::read(&sentinel).expect("external file retained"),+ b"external data"+ );+ assert!(external.path().is_dir());+ assert!(std::fs::symlink_metadata(directory_link).is_err());+ assert!(std::fs::symlink_metadata(file_link).is_err());+ for file in &ctx.manifest.files {+ assert!(Path::new(file).exists(), "baseline file missing: {file}");+ }+ }++ #[test]+ fn every_standalone_sample_restores_the_manifest_baseline() {+ let dir = tempfile::tempdir().expect("temp dir");+ let ctx = tiny_context(dir.path());+ let sidecar = ctx.agent_dir.join("sessions/mock.window-cache.json");+ // Even the discarded warm-up starts from the original baseline.+ std::fs::write(&sidecar, "previous invocation").expect("old sidecar");+ let mut launches = Vec::new();+ let samples = sample_runs(&ctx, 2, |index| {+ assert!(+ !sidecar.exists(),+ "sample {index} inherited a prior sidecar"+ );+ for file in &ctx.manifest.files {+ assert!(Path::new(file).exists(), "baseline file missing: {file}");+ }+ std::fs::write(&sidecar, "written by the mock scenario")?;+ launches.push(index);+ Ok(index as f64)+ })+ .expect("sample the mock scenario");+ assert_eq!((launches, samples), (vec![0, 1, 2], vec![1.0, 2.0]));+ }++ #[test]+ fn a_daemon_startup_timeout_kills_and_reaps_its_owned_child() {+ let dir = tempfile::tempdir().expect("temp dir");+ let child = std::process::Command::new("/bin/sleep")+ .arg("60")+ .spawn()+ .expect("spawn the mock daemon, never the product");+ let pid = nix::unistd::Pid::from_raw(child.id() as i32);+ let result = DaemonChild { child }.wait_until_ready(&dir.path().join("s"), Instant::now());+ assert!(result.is_err());+ assert_eq!(+ nix::sys::signal::kill(pid, None),+ Err(nix::errno::Errno::ESRCH)+ );+ }++ #[test]+ fn a_measured_ui_disconnects_before_waiting_for_daemon_exit() {+ use std::os::unix::fs::PermissionsExt;++ let dir = tempfile::tempdir().expect("temp dir");+ let mut ctx = tiny_context(dir.path());+ let binary = dir.path().join("mock-product.py");+ std::fs::write(+ &binary,+ include_str!("../../tests/fixtures/draining_daemon.py"),+ )+ .expect("write the mock modes");+ std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o700))+ .expect("make the mock executable");+ ctx.binary = binary;+ ctx.liveness = Duration::from_secs(10);+ let (sample, launched) = launch(&ctx, Scenario::WarmStart, 0)+ .expect("the daemon must drain its measured UI connection before exit");+ assert!(sample >= 0.0);+ assert!(+ launched.ready_ms.is_some(),+ "the actual mock paint/attach ran"+ );+ assert_eq!(+ std::fs::read_to_string(run_socket(&ctx, 0).with_extension("drained"))+ .expect("the mock daemon reached normal drain completion"),+ "all UI/control connections closed\n"+ );+ }++ #[test]+ fn a_daemon_that_never_sends_hello_cannot_hang_shutdown() {+ use std::os::unix::net::UnixListener;++ let dir = tempfile::tempdir().expect("temp dir");+ // Keep the Unix socket path short on macOS too.+ let socket = dir.path().join("s");+ let listener = UnixListener::bind(&socket).expect("mock listener");+ let (release, released) = std::sync::mpsc::channel();+ let peer = std::thread::spawn(move || {+ let (_stream, _) = listener.accept().expect("accept shutdown client");+ // The bounded watchdog also lets this test terminate on the+ // old implementation, where hello has no read timeout.+ let _ = released.recv_timeout(Duration::from_secs(8));+ });+ let started = Instant::now();+ protocol_shutdown(&socket);+ let elapsed = started.elapsed();+ let _ = release.send(());+ peer.join().expect("mock peer exited");+ assert!(+ elapsed < Duration::from_secs(7),+ "hello blocked for {elapsed:?}"+ );+ }+}diff --git a/crates/pa-perf/src/corpus/artifacts.rs b/crates/pa-perf/src/corpus/artifacts.rsnew file mode 100644index 0000000000..75a4674a71--- /dev/null+++ b/crates/pa-perf/src/corpus/artifacts.rs@@ -0,0 +1,261 @@+//! The session-artifact trees: per-session artifact dirs (kernel-state+//! stubs, harness state, retired heartbeat jobs), the subagent session+//! files, and the ledger edges that make them roster-visible — all through+//! the product's own writers where they exist.++use std::path::{Path, PathBuf};++use pa_core::refinement::{+ HarnessEntry, HarnessRefinementEvent, HarnessScope, HarnessState, RefinementKind,+};+use pa_daemon::rlm_ledger::{RlmSpawnInput, RlmSpawnLedger};+use serde_json::json;++use crate::corpus::rows::SessionSpec;+use crate::rng::{iso_from_millis, Rng};++/// The subagent tree shape: fanout per level, total depth, and the row count+/// of each child session file, plus whether to populate artifact partitions.+pub(crate) struct SubagentTree {+ pub fanout: usize,+ pub max_depth: u32,+ pub child_rows: usize,+ pub artifacts: bool,+}++/// The worker-name bank for subagent ledger edges.+const WORKER_NAMES: [&str; 12] = [+ "worker",+ "helper",+ "scout",+ "reviewer",+ "implementer",+ "planner",+ "auditor",+ "fixer",+ "researcher",+ "summarizer",+ "verifier",+ "optimizer",+];++/// A valid empty-dict pickle payload (protocol 4): the kernel-state sidecar+/// stub unpickles cleanly to zero restored names.+const EMPTY_PICKLE: &[u8] = b"\x80\x04}\x94.";++/// Write one session's artifact partition: the kernel-state manifest and+/// payload, plus (per `rng`) a local harness store and a retired heartbeat.+///+/// # Errors+///+/// Propagates filesystem failures.+pub(crate) fn write_artifact_dir(+ dir: &Path,+ session_id: &str,+ seed_text: &str,+ rng: &mut Rng,+) -> anyhow::Result<()> {+ std::fs::create_dir_all(dir)?;+ std::fs::write(+ dir.join("kernel-state.json"),+ json!({ "version": 1, "savedNames": [], "skipped": [] }).to_string(),+ )?;+ std::fs::write(dir.join("kernel-state.dill"), EMPTY_PICKLE)?;++ if rng.chance(30) {+ let harness_dir = dir.join("harness");+ std::fs::create_dir_all(&harness_dir)?;+ let entries = (1..=1 + rng.below(3))+ .map(|index| {+ (+ format!("{seed_text}-memory-{index}"),+ format!("corpus memory entry {index}: {}", seed_text),+ )+ })+ .collect::<Vec<_>>();+ let at = iso_from_millis(1_700_000_000_000 + rng.below(10_000_000));+ let state = HarnessState {+ schema: 1,+ entries: [(+ RefinementKind::Memory,+ entries+ .iter()+ .map(|(id, content)| {+ (+ id.clone(),+ HarnessEntry {+ id: id.clone(),+ kind: RefinementKind::Memory,+ title: format!("corpus memory {}", id),+ content: content.clone(),+ path: "general".to_string(),+ scope: Some(HarnessScope::Local),+ reference: serde_json::Map::new(),+ arguments: serde_json::Map::new(),+ metadata: serde_json::Map::new(),+ source: "agent".to_string(),+ created_at: at.clone(),+ updated_at: at.clone(),+ version: 1,+ },+ )+ })+ .collect(),+ )]+ .into_iter()+ .collect(),+ refinements: vec![HarnessRefinementEvent {+ id: format!("{seed_text}-event-1"),+ trigger: "test".to_string(),+ changes: vec![format!("create memory:{seed_text}-memory-1")],+ evidence: String::new(),+ outcome: String::new(),+ created_at: at,+ }],+ };+ std::fs::write(+ harness_dir.join("harness_state.json"),+ serde_json::to_string(&state)?,+ )?;+ std::fs::write(+ harness_dir.join("refinement_history.jsonl"),+ serde_json::to_string(&state.refinements[0])? + "\n",+ )?;+ }++ if rng.chance(10) {+ std::fs::write(+ dir.join("scheduled-jobs.json"),+ json!({+ "jobs": [{+ "id": format!("{:08x}-b1c7-4f80-9d0e-be9ecdc78a65", rng.below(1 << 32)),+ "status": "cancelled",+ "source": "rlm_heartbeat",+ "runtimeKind": "top-level",+ "deliveryMode": "follow_up",+ "activeSessionId": rng.next_u64().to_string(),+ "sessionId": session_id,+ "sessionFile": String::new(),+ "cwd": "/tmp",+ "label": WORKER_NAMES[rng.below(WORKER_NAMES.len() as u64) as usize],+ "prompt": "retired corpus heartbeat",+ "schedule": { "kind": "interval", "expression": "every 30m", "intervalMs": 1_800_000 },+ "createdAt": iso_from_millis(1_700_000_000_000),+ "updatedAt": iso_from_millis(1_700_010_000_000),+ "runCount": 0,+ }]+ })+ .to_string(),+ )?;+ }+ Ok(())+}++/// One subagent tree node's spawn: the child session dir + file, its own+/// artifact partition, the ledger edge, and the recursion into grandchildren.+/// The child session file lives at+/// `<agent>/session-artifacts/<parent-session-id>/sub-<childId>/` (the+/// product's `rlm_child_session_dir` layout); grandchildren key on the+/// child's own session id at the artifact root.+///+/// # Errors+///+/// Propagates ledger and filesystem failures.+#[allow(clippy::too_many_arguments)] // one node carries the ledger, parent identity and tree-generation context+pub(crate) fn spawn_child(+ ledger: &RlmSpawnLedger,+ artifacts_root: &Path,+ parent_session_id: &str,+ parent_file: &Path,+ depth: u32,+ tree: &SubagentTree,+ cwd: &str,+ rng: &mut Rng,+) -> anyhow::Result<Vec<PathBuf>> {+ let mut written = Vec::new();+ if depth > tree.max_depth {+ return Ok(written);+ }+ for _ in 0..tree.fanout {+ let child_tag = format!("sub-{:07x}", rng.below(1 << 28));+ let child_dir = artifacts_root.join(parent_session_id).join(&child_tag);+ std::fs::create_dir_all(&child_dir)?;+ let child_start = 1_700_000_000_000 + rng.below(90_000_000_000);+ let child_session_id = crate::corpus::rows::session_id(rng, child_start);+ let child_file = child_dir.join(format!("{child_session_id}.jsonl"));+ crate::corpus::rows::write_session_file(+ &child_file,+ &SessionSpec {+ id: child_session_id.clone(),+ start_ms: child_start,+ cwd: cwd.to_string(),+ rows: tree.child_rows,+ name: None,+ marker: None,+ parent_session: Some(parent_file.display().to_string()),+ rlm_depth: None,+ },+ rng,+ )?;+ crate::corpus::set_mtime(&child_file, child_start + 10_000)?;+ // The child's OWN artifact partition sits under the parent's tree.+ if tree.artifacts {+ write_artifact_dir(+ &artifacts_root+ .join(parent_session_id)+ .join("session-artifacts")+ .join(&child_session_id),+ &child_session_id,+ &child_session_id,+ rng,+ )?;+ }+ ledger.append_spawn(&RlmSpawnInput {+ child_id: child_tag.clone(),+ parent: parent_file.display().to_string(),+ child: child_file.display().to_string(),+ depth,+ name: WORKER_NAMES[rng.below(WORKER_NAMES.len() as u64) as usize].to_string(),+ })?;+ written.push(child_file.clone());+ if depth < tree.max_depth {+ written.extend(spawn_child(+ ledger,+ artifacts_root,+ &child_session_id,+ &child_file,+ depth + 1,+ tree,+ cwd,+ rng,+ )?);+ }+ }+ Ok(written)+}++#[cfg(test)]+mod tests {+ #[test]+ fn generated_kernel_snapshot_unpickles_to_an_empty_dictionary() {+ let directory = tempfile::tempdir().expect("artifact directory");+ super::write_artifact_dir(+ directory.path(),+ "session",+ "seed",+ &mut crate::rng::Rng::new(7),+ )+ .expect("write real artifact partition");+ let output = std::process::Command::new("python3")+ .arg("-c")+ .arg("import pickle,sys; value=pickle.load(open(sys.argv[1],'rb')); assert type(value) is dict and not value")+ .arg(directory.path().join("kernel-state.dill"))+ .output()+ .expect("run hosted Python pickle reader");+ assert!(+ output.status.success(),+ "{}",+ String::from_utf8_lossy(&output.stderr)+ );+ }+}diff --git a/crates/pa-perf/src/corpus/mod.rs b/crates/pa-perf/src/corpus/mod.rsnew file mode 100644index 0000000000..5e0b208a6f--- /dev/null+++ b/crates/pa-perf/src/corpus/mod.rs@@ -0,0 +1,728 @@+//! The deterministic corpus generator: one `generate` pass writes the full+//! agent-state layout under an isolated `--out` dir and returns the manifest+//! the bench harness needles read.++pub(crate) mod artifacts;+pub(crate) mod rows;++use std::path::{Path, PathBuf};++use anyhow::Context;+use clap::ValueEnum;+use pa_daemon::rlm_ledger::RlmSpawnLedger;+use serde::{Deserialize, Serialize};++use crate::rng::Rng;+use artifacts::SubagentTree;+use rows::SessionSpec;++/// The named corpus profiles (sizes overridable knob by knob).+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ValueEnum)]+#[serde(rename_all = "lowercase")]+pub enum Profile {+ Small,+ Medium,+ Large,+ Xl,+}++impl Profile {+ /// The profile's knob defaults.+ #[must_use]+ pub fn defaults(self) -> CorpusOptions {+ let (sessions, rows, child_rows, fanout, depth, tree_parents, wide, archive, cwd_files) =+ match self {+ Self::Small => (10, 100, 40, 2, 2, 2, 0, 3, 200),+ Self::Medium => (100, 2_000, 100, 3, 2, 5, 0, 20, 2_000),+ Self::Large => (1_000, 10_000, 150, 3, 3, 8, 110, 200, 4_000),+ Self::Xl => (2_500, 400, 80, 2, 2, 10, 0, 500, 2_000),+ };+ CorpusOptions {+ profile: self,+ out: PathBuf::new(),+ seed: 7,+ sessions,+ rows,+ child_rows,+ huge_rows: rows.saturating_mul(2).max(2_000),+ fanout,+ depth,+ tree_parents,+ wide_children: wide,+ archive,+ cwd_files,+ artifacts: true,+ cache: CacheMode::Cold,+ }+ }+}++/// Whether the warm-up pass runs the product's window-store open over the+/// generated session files (writing their window-cache sidecars).+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ValueEnum)]+#[serde(rename_all = "lowercase")]+pub enum CacheMode {+ Cold,+ Warm,+}++/// Every corpus knob; all overridable from the CLI (README.md documents each).+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct CorpusOptions {+ pub profile: Profile,+ pub out: PathBuf,+ pub seed: u64,+ /// Top-level sessions in the sessions dir.+ pub sessions: usize,+ /// Rows per session (the huge session gets `huge_rows`).+ pub rows: usize,+ /// Rows per subagent child session.+ pub child_rows: usize,+ /// Rows of the one huge session (the open-session scenario's target).+ pub huge_rows: usize,+ /// Subagent children per parent per level.+ pub fanout: usize,+ /// Subagent tree depth (levels below the parent session).+ pub depth: u32,+ /// How many sessions grow a subagent tree.+ pub tree_parents: usize,+ /// One parent grows this many direct children (the operator's ~110-wide+ /// real parent); 0 disables.+ pub wide_children: usize,+ /// Sessions in sessions-archive/.+ pub archive: usize,+ /// Files in the generated project dir (the sessions' cwd, the+ /// @-completion walk's tree).+ pub cwd_files: usize,+ /// Write the per-session artifact partitions.+ pub artifacts: bool,+ pub cache: CacheMode,+}++/// The session-name task bank (roster rows carry these as `session_info`).+const TASKS: [&str; 16] = [+ "ship the bench harness",+ "review the corpus generator",+ "fix the roster scan",+ "port the window cache",+ "audit the ledger replay",+ "write the CI workflow",+ "profile the daemon boot",+ "tune the pty harness",+ "split the scan fold",+ "measure first paint",+ "repair the lease release",+ "seed the archive sweep",+ "draft the README",+ "run the regression gates",+ "bisect the paint delay",+ "close the perf budget",+];++/// A fixed generation base (unix ms): sessions spread back from it, so the+/// tree is seed-deterministic — never wall-clock dependent. (The RLM ledger's+/// own `at` stamps come from the product writer; see README's determinism+/// note.)+const BASE_MS: u64 = 1_790_000_000_000;+/// The mtime step between sessions (roster order is mtime-desc; the+/// spread is realism only — the archive-off settings are the sweep guard).+const MTIME_STEP_MS: u64 = 720_000;+/// The header-timestamp hour step (a session's rows spread over an hour).+const HOUR_MS: u64 = 3_600_000;++/// The corpus manifest: what the bench reads (needles are DATA, never+/// layout-position), and the file-set baseline the per-run restore uses.+#[derive(Debug, Serialize, Deserialize)]+pub struct Manifest {+ pub schema: u32,+ pub profile: Profile,+ pub seed: u64,+ pub paths: ManifestPaths,+ pub counts: ManifestCounts,+ /// Every generated file under the corpus root (the restore baseline:+ /// the bench deletes anything else the product wrote between runs).+ pub files: Vec<String>,+ pub markers: ManifestMarkers,+}++/// The corpus's on-disk anchor paths (absolute, canonical).+#[derive(Debug, Serialize, Deserialize)]+pub struct ManifestPaths {+ pub agent_dir: PathBuf,+ pub sessions_dir: PathBuf,+ pub archive_dir: PathBuf,+ pub project_dir: PathBuf,+ pub manifest_file: PathBuf,+}++#[derive(Debug, Serialize, Deserialize)]+pub struct ManifestCounts {+ pub sessions: usize,+ pub archive: usize,+ pub child_sessions: usize,+ pub total_rows: usize,+ pub subagent_edges: usize,+ pub cwd_files: usize,+}++/// The bench needles: marker strings and expected rendered counts, all+/// recorded by the generator.+#[derive(Debug, Serialize, Deserialize)]+pub struct ManifestMarkers {+ pub huge: MarkerSession,+ pub switch_a: MarkerSession,+ pub switch_b: MarkerSession,+ /// The agents view's expected `Inactive (N)` heading count.+ pub roster_inactive: usize,+ /// The session-switch search query (uniquely matches the switch-b row).+ pub switch_query: String,+ /// The switch-b row's rendered name (the picker paints it fully on the+ /// empty-picker -> hit transition; the echoed query does not contain it).+ pub switch_row: String,+ /// The @-completion target file name under the project dir.+ pub at_file: String,+}++#[derive(Debug, Serialize, Deserialize)]+pub struct MarkerSession {+ pub id: String,+ pub file: PathBuf,+ pub needle: String,+}++/// One marker session captured during generation.+struct Marker {+ id: String,+ file: PathBuf,+}++impl Marker {+ fn session(&self, needle: String) -> MarkerSession {+ MarkerSession {+ id: self.id.clone(),+ file: self.file.clone(),+ needle,+ }+ }+}++/// Generate the corpus: the full agent-state tree under `options.out`,+/// then the manifest at `corpus.json`.+///+/// # Errors+///+/// Propagates filesystem, serialization, and ledger failures.+pub fn generate(options: &CorpusOptions) -> anyhow::Result<Manifest> {+ anyhow::ensure!(+ options.sessions >= 3,+ "the corpus needs at least 3 sessions (the huge, switch-a, and switch-b markers)"+ );+ let out = canonical_root(&options.out)?;+ anyhow::ensure!(+ std::fs::read_dir(&out)?.next().transpose()?.is_none(),+ "corpus output directory must be empty: {}",+ out.display()+ );+ let agent_dir = out.join("agent");+ let sessions_dir = agent_dir.join("sessions");+ let archive_dir = agent_dir.join("sessions-archive");+ let project_dir = out.join("project");+ for dir in [&agent_dir, &sessions_dir, &archive_dir, &project_dir] {+ std::fs::create_dir_all(dir).with_context(|| format!("create {}", dir.display()))?;+ }+ let cwd = project_dir.display().to_string();++ // Onboarding is pre-answered, traces stay off, and BOTH archive rules+ // are disabled: the daemon's boot sweep would otherwise retire the+ // corpus's own sessions (the defaults keep 200 files / 30 days) and+ // move them out from under the bench's manifest baseline.+ std::fs::write(+ agent_dir.join("settings.json"),+ serde_json::json!({+ "onboardingShown": true,+ "agentTraces": { "enabled": false },+ "sessionArchiveMaxAgeDays": "off",+ "sessionArchiveMaxSessions": "off",+ })+ .to_string(),+ )?;+ let at_file = write_project(&project_dir, options.cwd_files)?;++ let mut rng = Rng::new(options.seed);+ let mut total_rows = 0usize;+ let mut child_sessions = 0usize;+ let ledger = RlmSpawnLedger::new(&agent_dir, &sessions_dir, |_| {});++ let mut mtime = BASE_MS + MTIME_STEP_MS;+ let mut huge: Option<Marker> = None;+ let mut switch_a: Option<Marker> = None;+ let mut switch_b: Option<Marker> = None;+ for index in 0..options.sessions {+ let start = BASE_MS - index as u64 * MTIME_STEP_MS;+ let (rows, name, marker) = match index {+ 0 => (+ options.huge_rows,+ Some("huge corpus session".to_string()),+ Some(format!("corpus huge marker {:x}", options.seed)),+ ),+ 1 => (+ options.rows,+ Some("switch target a".to_string()),+ Some(format!("corpus switch marker a {:x}", options.seed)),+ ),+ // The `zq` token makes switch-b the agents-view search's ONLY+ // possible hit: no generated text (names or messages) contains+ // the letter z, so the query can never rank another row first.+ 2 => (+ options.rows,+ Some("switch target zqmark".to_string()),+ Some(format!("corpus switch marker b {:x}", options.seed)),+ ),+ _ => (+ options.rows,+ rng.chance(70)+ .then(|| format!("{} #{index}", TASKS[rng.below(TASKS.len() as u64) as usize])),+ None,+ ),+ };+ let id = rows::session_id(&mut rng, start);+ let file = sessions_dir.join(format!("{id}.jsonl"));+ let written = rows::write_session_file(+ &file,+ &SessionSpec {+ id: id.clone(),+ start_ms: start,+ cwd: cwd.clone(),+ rows,+ name,+ marker: marker.clone(),+ parent_session: None,+ rlm_depth: Some(0),+ },+ &mut rng,+ )?;+ total_rows += written.rows;+ if let Some(marker) = &marker {+ anyhow::ensure!(+ written.last_user_text == *marker,+ "the {} session's marker did not land on the final user row",+ file.display()+ );+ let captured = Marker {+ id: id.clone(),+ file: file.clone(),+ };+ match marker.as_str() {+ m if m.starts_with("corpus huge") => huge = Some(captured),+ m if m.starts_with("corpus switch marker a") => switch_a = Some(captured),+ m if m.starts_with("corpus switch marker b") => switch_b = Some(captured),+ _ => {}+ }+ }+ set_mtime(&file, mtime)?;+ mtime -= MTIME_STEP_MS;++ if options.artifacts {+ let artifacts_root = agent_dir.join("session-artifacts");+ artifacts::write_artifact_dir(&artifacts_root.join(&id), &id, &id, &mut rng)?;+ }++ // Subagent trees: the first `tree_parents` sessions grow one; the+ // first of those also grows the wide-children set.+ if index > 2 && index <= 2 + options.tree_parents {+ let tree = SubagentTree {+ fanout: options.fanout,+ max_depth: options.depth,+ child_rows: options.child_rows,+ artifacts: options.artifacts,+ };+ child_sessions += spawn_and_count(+ &ledger,+ &agent_dir.join("session-artifacts"),+ &id,+ &file,+ &tree,+ &cwd,+ &mut rng,+ &mut total_rows,+ )?;+ if index == 3 && options.wide_children > 0 {+ let wide_tree = SubagentTree {+ fanout: options.wide_children,+ max_depth: 1,+ child_rows: options.child_rows,+ artifacts: options.artifacts,+ };+ child_sessions += spawn_and_count(+ &ledger,+ &agent_dir.join("session-artifacts"),+ &id,+ &file,+ &wide_tree,+ &cwd,+ &mut rng,+ &mut total_rows,+ )?;+ }+ }+ }++ // Archived sessions: plain files in the archive dir (older mtimes).+ for index in 0..options.archive {+ let start = BASE_MS / 2 - index as u64 * HOUR_MS;+ let id = rows::session_id(&mut rng, start);+ let file = archive_dir.join(format!("{id}.jsonl"));+ let written = rows::write_session_file(+ &file,+ &SessionSpec {+ id: id.clone(),+ start_ms: start,+ cwd: cwd.clone(),+ rows: options.rows,+ name: None,+ marker: None,+ parent_session: None,+ rlm_depth: Some(0),+ },+ &mut rng,+ )?;+ total_rows += written.rows;+ set_mtime(&file, start + HOUR_MS)?;+ }++ // The warm-up pass: the product's own window-store open writes the+ // window-cache sidecars (the info-cache sidecar has no product writer+ // outside a worker lease release; the bench measures that warmth via+ // the daemon's own process cache — see README).+ if options.cache == CacheMode::Warm {+ warm_sidecars(&sessions_dir, &archive_dir)?;+ }++ let manifest = Manifest {+ schema: 1,+ profile: options.profile,+ seed: options.seed,+ paths: ManifestPaths {+ agent_dir,+ sessions_dir,+ archive_dir,+ project_dir,+ manifest_file: out.join("corpus.json"),+ },+ counts: ManifestCounts {+ sessions: options.sessions,+ archive: options.archive,+ child_sessions,+ total_rows,+ subagent_edges: child_sessions,+ cwd_files: options.cwd_files,+ },+ files: collect_files(&out)?,+ markers: ManifestMarkers {+ huge: marker_session(huge, &format!("corpus huge marker {:x}", options.seed))?,+ switch_a: marker_session(+ switch_a,+ &format!("corpus switch marker a {:x}", options.seed),+ )?,+ switch_b: marker_session(+ switch_b,+ &format!("corpus switch marker b {:x}", options.seed),+ )?,+ roster_inactive: options.sessions,+ switch_query: "target zqmark".to_string(),+ switch_row: "switch target zqmark".to_string(),+ at_file: at_file.clone(),+ },+ };+ std::fs::write(+ out.join("corpus.json"),+ serde_json::to_string_pretty(&manifest)?,+ )?;+ Ok(manifest)+}++/// Replace only a prior corpus owned by this suite; arbitrary populated output+/// remains an error. remove_dir_all does not follow nested directory symlinks.+#[cfg(any(unix, test))]+pub(crate) fn regenerate_for_suite(options: &CorpusOptions) -> anyhow::Result<Manifest> {+ anyhow::ensure!(+ options.sessions >= 3,+ "the corpus needs at least 3 sessions"+ );+ let root = &options.out;+ if let Ok(metadata) = std::fs::symlink_metadata(root) {+ anyhow::ensure!(+ !metadata.file_type().is_symlink(),+ "suite corpus root must not be a symlink: {}",+ root.display()+ );+ }+ if root.exists() && std::fs::read_dir(root)?.next().transpose()?.is_some() {+ let canonical = root.canonicalize()?;+ let manifest: Manifest =+ serde_json::from_slice(&std::fs::read(canonical.join("corpus.json"))?)+ .context("refuse to replace output without a corpus manifest")?;+ anyhow::ensure!(+ manifest.schema == 1+ && manifest.paths.manifest_file == canonical.join("corpus.json")+ && manifest.paths.agent_dir == canonical.join("agent")+ && manifest.paths.project_dir == canonical.join("project"),+ "refuse to replace a corpus whose manifest belongs to another root"+ );+ std::fs::remove_dir_all(root)?;+ }+ generate(options)+}++/// A one-child spawn through the product's ledger, then the recursion's row+/// accounting.+#[allow(clippy::too_many_arguments)] // the spawn inputs are the tree site's own facts+fn spawn_and_count(+ ledger: &RlmSpawnLedger,+ artifacts_root: &Path,+ id: &str,+ file: &Path,+ tree: &SubagentTree,+ cwd: &str,+ rng: &mut Rng,+ total_rows: &mut usize,+) -> anyhow::Result<usize> {+ let children = artifacts::spawn_child(ledger, artifacts_root, id, file, 1, tree, cwd, rng)?;+ for child in &children {+ *total_rows += count_rows(child);+ }+ Ok(children.len())+}++fn marker_session(marker: Option<Marker>, needle: &str) -> anyhow::Result<MarkerSession> {+ marker.map_or_else(+ || anyhow::bail!("the marker session was not generated"),+ |marker| Ok(marker.session(needle.to_string())),+ )+}++/// Deterministic project tree: five dirs, plain content files, plus the+/// @-completion marker file the bench needles on.+fn write_project(project: &Path, files: usize) -> anyhow::Result<String> {+ let dirs = ["src", "src/kernel", "tests", "docs", "scripts"];+ for dir in dirs {+ std::fs::create_dir_all(project.join(dir))?;+ }+ for index in 0..files {+ let dir = dirs[index % dirs.len()];+ let name = if index % 50 == 0 {+ format!("mod{index}.rs")+ } else {+ format!("file{index:05}.rs")+ };+ std::fs::write(+ project.join(dir).join(name),+ format!("// corpus file {index}\nfn stub{index}() {{}}\n"),+ )?;+ }+ let marker = "zzq-corpus-marker.md";+ std::fs::write(project.join(marker), "# corpus @ completion marker file\n")?;+ Ok(marker.to_string())+}++pub(crate) fn set_mtime(path: &Path, millis: u64) -> anyhow::Result<()> {+ let time = std::time::SystemTime::UNIX_EPOCH + std::time::Duration::from_millis(millis);+ let file = std::fs::File::options().write(true).open(path)?;+ file.set_times(+ std::fs::FileTimes::new()+ .set_modified(time)+ .set_accessed(time),+ )?;+ Ok(())+}++fn count_rows(path: &Path) -> usize {+ std::fs::read_to_string(path)+ .map(|text| text.lines().filter(|line| !line.is_empty()).count())+ .unwrap_or(0)+}++fn warm_sidecars(sessions_dir: &Path, archive_dir: &Path) -> anyhow::Result<()> {+ for dir in [sessions_dir, archive_dir] {+ for entry in std::fs::read_dir(dir)? {+ let path = entry?.path();+ if path.extension().is_some_and(|ext| ext == "jsonl") {+ let _ = pa_core::session::window::WindowedSessionStore::open(&path)?;+ }+ }+ }+ Ok(())+}++fn collect_files(root: &Path) -> anyhow::Result<Vec<String>> {+ fn walk(dir: &Path, out: &mut Vec<String>) -> anyhow::Result<()> {+ let mut entries = std::fs::read_dir(dir)?.collect::<Result<Vec<_>, _>>()?;+ entries.sort_by_key(|entry| entry.file_name());+ for entry in entries {+ let path = entry.path();+ if path.is_dir() {+ walk(&path, out)?;+ } else {+ out.push(path.display().to_string());+ }+ }+ Ok(())+ }+ let mut files = Vec::new();+ walk(root, &mut files)?;+ files.sort();+ Ok(files)+}++fn canonical_root(out: &Path) -> anyhow::Result<PathBuf> {+ std::fs::create_dir_all(out).with_context(|| format!("create {}", out.display()))?;+ // macOS /tmp -> /private/tmp: the ledger hash, the header parentSession+ // paths, and every ledger record key on canonical paths.+ out.canonicalize()+ .with_context(|| format!("canonicalize {}", out.display()))+}++#[cfg(test)]+mod tests {+ fn tiny_options(out: &std::path::Path) -> super::CorpusOptions {+ let mut options = super::Profile::Small.defaults();+ options.out = out.to_path_buf();+ options.sessions = 4;+ options.rows = 8;+ options.huge_rows = 8;+ options.child_rows = 8;+ options.archive = 0;+ options.cwd_files = 0;+ options.tree_parents = 1;+ options.wide_children = 0;+ options.artifacts = false;+ options+ }++ #[test]+ fn generation_rejects_stale_output_without_changing_existing_files() {+ let directory = tempfile::tempdir().expect("output directory");+ let options = tiny_options(directory.path());+ super::generate(&options).expect("generate initial corpus");+ let sentinel = directory.path().join("unrelated.txt");+ std::fs::write(&sentinel, b"keep this file").expect("write unrelated file");+ let manifest = std::fs::read(directory.path().join("corpus.json")).expect("manifest");+ let error = super::generate(&options).expect_err("reject previously populated output");+ assert!(error.to_string().contains("must be empty"), "{error:#}");+ assert_eq!(+ std::fs::read(&sentinel).expect("sentinel retained"),+ b"keep this file"+ );+ assert_eq!(+ std::fs::read(directory.path().join("corpus.json")).expect("old manifest"),+ manifest+ );+ }++ #[test]+ fn depth_zero_generates_no_recursive_children_or_spawn_edges() {+ let directory = tempfile::tempdir().expect("output directory");+ let mut options = tiny_options(directory.path());+ options.depth = 0;+ options.fanout = 3;+ let manifest = super::generate(&options).expect("generate depth-zero corpus");+ assert_eq!(manifest.counts.child_sessions, 0);+ assert_eq!(manifest.counts.subagent_edges, 0);+ assert!(!manifest.files.iter().any(|file| file.contains("sub-")));+ }++ #[test]+ fn artifact_option_preserves_deep_and_wide_children_without_optional_payloads() {+ for artifacts in [false, true] {+ let directory = tempfile::tempdir().expect("output directory");+ let mut options = tiny_options(directory.path());+ options.artifacts = artifacts;+ options.fanout = 2;+ options.depth = 2;+ options.wide_children = 3;+ let manifest = super::generate(&options).expect("generate child trees");+ // Two direct children, four grandchildren, and three wide children.+ assert_eq!(manifest.counts.child_sessions, 9);+ let ledger = pa_daemon::rlm_ledger::RlmSpawnLedger::new(+ &manifest.paths.agent_dir,+ &manifest.paths.sessions_dir,+ |_| {},+ );+ let edges = ledger.live_edges().expect("replay actual spawn ledger");+ assert_eq!(edges.len(), 9);+ assert!(edges.iter().any(|edge| edge.depth == 2));+ for edge in edges {+ assert!(std::path::Path::new(&edge.child).is_file());+ assert!(manifest.files.contains(&edge.child));+ }+ let payloads = manifest+ .files+ .iter()+ .filter(|file| {+ file.ends_with("kernel-state.json") || file.ends_with("kernel-state.dill")+ })+ .count();+ if artifacts {+ assert_eq!(payloads, 2 * (options.sessions + 9));+ } else {+ assert_eq!(payloads, 0);+ assert!(!manifest.files.iter().any(|file| {+ file.contains("/harness/") || file.ends_with("scheduled-jobs.json")+ }));+ }+ }+ }++ #[test]+ fn suite_regeneration_replaces_owned_corpora_without_admitting_stale_files() {+ let directory = tempfile::tempdir().expect("suite output");+ let options = tiny_options(&directory.path().join("corpus-cold"));+ super::generate(&options).expect("initial corpus");+ let stale = options.out.join("agent/sessions/stale.jsonl");+ std::fs::write(&stale, "stale transcript").expect("stale file");+ let manifest = super::regenerate_for_suite(&options).expect("repeat suite generation");+ assert!(!stale.exists());+ assert!(!manifest+ .files+ .iter()+ .any(|file| file.ends_with("stale.jsonl")));+ assert_eq!(manifest.counts.sessions, options.sessions);+ }++ #[test]+ fn suite_regeneration_refuses_unowned_populated_output() {+ let directory = tempfile::tempdir().expect("suite output");+ let sentinel = directory.path().join("unrelated.txt");+ std::fs::write(&sentinel, "preserve me").expect("unrelated file");+ assert!(super::regenerate_for_suite(&tiny_options(directory.path())).is_err());+ assert_eq!(std::fs::read_to_string(sentinel).unwrap(), "preserve me");+ }++ #[cfg(unix)]+ #[test]+ fn suite_regeneration_does_not_follow_external_directory_symlinks() {+ let directory = tempfile::tempdir().expect("suite output");+ let outside = tempfile::tempdir().expect("outside tree");+ let sentinel = outside.path().join("sentinel");+ std::fs::write(&sentinel, "outside survives").expect("sentinel");+ let options = tiny_options(&directory.path().join("corpus-cold"));+ super::generate(&options).expect("initial corpus");+ std::os::unix::fs::symlink(outside.path(), options.out.join("external-link"))+ .expect("external symlink");+ super::regenerate_for_suite(&options).expect("replace owned corpus");+ assert_eq!(+ std::fs::read_to_string(sentinel).unwrap(),+ "outside survives"+ );+ }+ #[cfg(unix)]+ #[test]+ fn suite_regeneration_rejects_even_an_empty_external_root_symlink() {+ let directory = tempfile::tempdir().expect("suite output");+ let outside = tempfile::tempdir().expect("empty outside directory");+ let link = directory.path().join("corpus-cold");+ std::os::unix::fs::symlink(outside.path(), &link).expect("root symlink");+ assert!(super::regenerate_for_suite(&tiny_options(&link)).is_err());+ assert!(std::fs::read_dir(outside.path()).unwrap().next().is_none());+ }+}diff --git a/crates/pa-perf/src/corpus/rows.rs b/crates/pa-perf/src/corpus/rows.rsnew file mode 100644index 0000000000..721e7b7ddb--- /dev/null+++ b/crates/pa-perf/src/corpus/rows.rs@@ -0,0 +1,560 @@+//! One session file's rows: the entry builders over the product's own+//! serializers (pa-types payloads inside pa-daemon's `SessionEntry`), the+//! conversation turn generator, and the marker discipline the bench+//! needles read from the corpus manifest.++use std::io::Write;+use std::path::Path;++use pa_daemon::session_store::{session_header_line, SessionEntry};+use pa_types::ai::{+ AssistantContentBlock, AssistantMessage, StopReason, TextContent, ThinkingContent, ToolCall,+ ToolResultMessage, Usage, UsageCost, UserContent, UserContentBlock, UserMessage,+};+use pa_types::session::{+ CompactionEntry, CustomMessageEntry, ModelChangeEntry, ServiceTierChangeEntry, SessionHeader,+ SessionState, SessionStateEntry, SessionStateStatus, ThinkingLevelChangeEntry,+};++use crate::rng::{iso_from_millis, Rng};++/// The top-level tools the product exposes to the model.+const TOOLS: [&str; 5] = ["bash", "edit", "python_repl", "websearch", "attach_image"];+/// The `custom_message` customType bank (a measured real-session mix).+const CUSTOM_TYPES: [&str; 8] = [+ "async_bash_completion",+ "heartbeat_prompt",+ "agent_message",+ "rlm_child_terminal_notice",+ "session_slash_command",+ "ipython_state",+ "provider_retry_outcome",+ "session_renamed",+];+/// Providers and models a real store carries. No id contains the letter+/// z: the switch-b row's `zqmark` name must be the only z in the corpus so+/// the switch query can never fuzzy-match another row.+const MODELS: [(&str, &str); 5] = [+ ("prime-inference", "glm-5.2"),+ ("prime-inference", "openai/gpt-5.6-sol"),+ ("anthropic", "claude-5-sonnet"),+ ("openai", "gpt-5.4"),+ ("google", "gemini-4-pro"),+];+/// The user-message word bank: plain lowercase words, joined into sentences.+const WORDS: [&str; 49] = [+ "the",+ "quick",+ "bench",+ "corpus",+ "session",+ "kernel",+ "worker",+ "daemon",+ "roster",+ "scan",+ "window",+ "cache",+ "entry",+ "branch",+ "leaf",+ "header",+ "row",+ "fold",+ "usage",+ "token",+ "model",+ "provider",+ "stream",+ "turn",+ "tool",+ "result",+ "call",+ "prompt",+ "context",+ "compaction",+ "summary",+ "refine",+ "harness",+ "memory",+ "skill",+ "label",+ "tree",+ "spawn",+ "child",+ "depth",+ "fanout",+ "seed",+ "profile",+ "median",+ "p95",+ "latency",+ "paint",+ "frame",+ "needle",+];+/// Bash-output shapes for tool results.+const TOOL_OUTPUTS: [&str; 6] = [+ "warning: unused variable: `baseline`\n --> src/lib.rs:42:13\n",+ "test result: ok. 96 passed; 0 failed; 3 ignored\n",+ "commit 33da15a2c (HEAD -> main)\nAuthor: dev <[email protected]>\n",+ "total 24\ndrwxr-xr-x 7 user staff 224 Oct 14 09:02 .\n",+ "updating crates.io index\n Compiling pa-perf v0.10.0\n",+ "{\"median_ms\": 812.3, \"p95_ms\": 941.0, \"runs\": 10}\n",+];++/// One session to generate.+pub(crate) struct SessionSpec {+ pub id: String,+ /// Session-start unix milliseconds; entry timestamps derive from it.+ pub start_ms: u64,+ pub cwd: String,+ pub rows: usize,+ pub name: Option<String>,+ /// A unique string placed in the LAST user message (the bench needle:+ /// the transcript paints the session tail on open).+ pub marker: Option<String>,+ pub parent_session: Option<String>,+ pub rlm_depth: Option<u64>,+}++/// What one generated session produced.+pub(crate) struct Written {+ pub rows: usize,+ /// The final user-message text (the marker, when one was requested).+ pub last_user_text: String,+}++/// The deterministic uuid-v7-shaped id stream: timestamp prefix + rng tail+/// with the version/variant nibbles v7 carries.+pub(crate) fn session_id(rng: &mut Rng, start_ms: u64) -> String {+ let rand_a = (rng.next_u64() & 0xFFFF) as u32;+ let rand_b = rng.next_u64();+ let tail = rng.next_u64();+ let a = (start_ms & 0xFFFF_FFFF) as u32;+ format!(+ "{a:08x}-{rand_a:04x}-7{:03x}-8{:03x}-{:05x}{:07x}",+ (rand_b >> 20) & 0xFFF,+ (tail >> 52) as u32 & 0xFFF,+ (rand_b & 0x000F_FFFF) as u32,+ (tail & 0x0FFF_FFFF) as u32,+ )+}++fn sentence(rng: &mut Rng, words: usize) -> String {+ (0..words)+ .map(|_| WORDS[rng.below(WORDS.len() as u64) as usize])+ .collect::<Vec<_>>()+ .join(" ")+}++fn user_text(rng: &mut Rng) -> String {+ (0..(1 + rng.below(3)))+ .map(|_| {+ let words = 6 + rng.below(12);+ let mut text = sentence(rng, words as usize);+ if rng.chance(12) {+ text.push_str(&format!(" ({})", sentence(rng, 2)));+ }+ text.push('.');+ text+ })+ .collect::<String>()+}++fn assistant_text(rng: &mut Rng) -> String {+ let mut text = String::new();+ for _ in 0..(1 + rng.below(3)) {+ for _ in 0..(2 + rng.below(4)) {+ let words = 8 + rng.below(16);+ text.push_str(&sentence(rng, words as usize));+ text.push_str(". ");+ }+ text.push('\n');+ }+ text+}++fn tool_output(rng: &mut Rng) -> String {+ let mut text = TOOL_OUTPUTS[rng.below(TOOL_OUTPUTS.len() as u64) as usize].to_string();+ text.push_str(&sentence(rng, 3));+ text.push('\n');+ text+}++fn usage(rng: &mut Rng) -> Usage {+ let input = 500 + rng.below(8000);+ let cache_read = input * (50 + rng.below(46)) / 100;+ let output = 50 + rng.below(2000);+ let cache_write = rng.below(500);+ let total = input + output + cache_read + cache_write;+ let cost = |n: u64| pa_types::JsNumber::from(n as f64 / 1000.0);+ Usage {+ input,+ output,+ cache_read,+ cache_write,+ total_tokens: total,+ cost: UsageCost {+ input: cost(input),+ output: cost(output),+ cache_read: cost(cache_read),+ cache_write: cost(cache_write),+ total: cost(total),+ },+ }+}++/// The row writer: one `SessionEntry` line, exactly as the product's store+/// serializes it (`type`, `id`, `parentId`, `timestamp`, then the payload).+struct Rows {+ writer: std::io::BufWriter<std::fs::File>,+ counter: u64,+ millis: u64,+ parent: Option<String>,+}++impl Rows {+ fn next_id(&mut self) -> String {+ let id = format!("{:08x}", self.counter);+ self.counter += 1;+ id+ }++ fn push(+ &mut self,+ type_: &str,+ fields: serde_json::Value,+ rng: &mut Rng,+ ) -> anyhow::Result<()> {+ self.millis += 20 + rng.below(400);+ let entry = SessionEntry {+ type_: type_.to_string(),+ id: self.next_id(),+ parent_id: self.parent.clone(),+ timestamp: iso_from_millis(self.millis),+ fields,+ };+ let mut line = serde_json::to_string(&entry).expect("the entry serializes");+ line.push('\n');+ self.writer.write_all(line.as_bytes())?;+ self.parent = Some(entry.id);+ Ok(())+ }++ fn rows_written(&self) -> u64 {+ self.counter+ }+}++/// One session file: the header line, then the chained entries. Returns the+/// row count and the last user-message text.+///+/// # Errors+///+/// Propagates serialization and filesystem failures.+pub(crate) fn write_session_file(+ path: &Path,+ spec: &SessionSpec,+ rng: &mut Rng,+) -> anyhow::Result<Written> {+ let (provider, model) = MODELS[rng.below(MODELS.len() as u64) as usize];+ let header = SessionHeader {+ version: Some(3),+ id: spec.id.clone(),+ timestamp: iso_from_millis(spec.start_ms),+ cwd: spec.cwd.clone(),+ parent_session: spec.parent_session.clone(),+ rlm_depth: spec.rlm_depth,+ git: None,+ rest: serde_json::Map::new(),+ };+ let file = std::fs::File::create(path)?;+ let mut rows = Rows {+ writer: std::io::BufWriter::new(file),+ counter: 0,+ millis: spec.start_ms,+ parent: None,+ };+ let mut header_line =+ serde_json::to_string(&session_header_line(&header)).expect("the header serializes");+ header_line.push('\n');+ rows.writer.write_all(header_line.as_bytes())?;++ rows.push(+ "model_change",+ serde_json::to_value(ModelChangeEntry {+ provider: provider.to_string(),+ model_id: model.to_string(),+ })?,+ rng,+ )?;+ rows.push(+ "thinking_level_change",+ serde_json::to_value(ThinkingLevelChangeEntry {+ thinking_level: if rng.chance(70) { "high" } else { "medium" }.to_string(),+ })?,+ rng,+ )?;+ rows.push(+ "service_tier_change",+ serde_json::to_value(ServiceTierChangeEntry {+ service_tier: Some(pa_types::ai::ServiceTier::Default),+ })?,+ rng,+ )?;+ rows.push(+ "session_state",+ serde_json::to_value(SessionStateEntry {+ state: SessionState {+ status: SessionStateStatus::Active,+ },+ })?,+ rng,+ )?;+ if let Some(name) = &spec.name {+ rows.push("session_info", serde_json::json!({ "name": name }), rng)?;+ }++ // The conversation: fill to `rows`, keeping a compaction boundary near+ // 60% for long sessions and the marker on the final user message.+ let target = spec.rows as u64;+ let mut compaction_at = (target >= 1500).then(|| target * 6 / 10);+ let mut last_user_text = String::new();+ while rows.rows_written() < target {+ if compaction_at.is_some_and(|threshold| rows.rows_written() >= threshold) {+ compaction_at = None;+ let summary = format!("Compacted earlier entries: {}.", sentence(rng, 6));+ rows.push(+ "compaction",+ serde_json::to_value(CompactionEntry {+ summary: summary.clone(),+ first_kept_entry_id: format!("{:08x}", rows.counter),+ tokens_before: 40_000 + rng.below(60_000),+ details: None,+ from_hook: Some(false),+ custom_instructions: None,+ usage: Some(usage(rng)),+ harness_digest: None,+ harness_state_fingerprint: None,+ })?,+ rng,+ )?;+ continue;+ }++ let text = match &spec.marker {+ Some(marker) if target - rows.rows_written() <= 4 => marker.clone(),+ _ => user_text(rng),+ };+ let content = if rng.chance(30) {+ UserContent::Blocks(vec![UserContentBlock::Text(TextContent {+ text: text.clone(),+ text_signature: None,+ rest: serde_json::Map::new(),+ })])+ } else {+ UserContent::Text(text.clone())+ };+ rows.push(+ "message",+ serde_json::json!({+ "message": pa_types::session::AgentMessage::User(UserMessage {+ content,+ timestamp: rows.millis,+ rest: serde_json::Map::new(),+ })+ }),+ rng,+ )?;+ last_user_text = text;++ // The assistant turn: thinking + text + a tool call, then the result.+ let tool = rng.chance(45).then(|| {+ let name = TOOLS[rng.below(TOOLS.len() as u64) as usize];+ let id = format!("call-{:08x}", rows.counter);+ let mut arguments = serde_json::Map::new();+ arguments.insert(+ "path".to_string(),+ serde_json::json!(format!("src/mod{}.rs", rng.below(400))),+ );+ arguments.insert("query".to_string(), serde_json::json!(sentence(rng, 3)));+ (id, name)+ });+ let mut blocks = Vec::new();+ if rng.chance(60) {+ blocks.push(AssistantContentBlock::Thinking(ThinkingContent {+ thinking: assistant_text(rng),+ thinking_signature: None,+ redacted: None,+ rest: serde_json::Map::new(),+ }));+ }+ blocks.push(AssistantContentBlock::Text(TextContent {+ text: assistant_text(rng),+ text_signature: None,+ rest: serde_json::Map::new(),+ }));+ if let Some((id, name)) = &tool {+ blocks.push(AssistantContentBlock::ToolCall(ToolCall {+ id: id.clone(),+ name: (*name).to_string(),+ arguments: {+ let mut map = serde_json::Map::new();+ map.insert(+ "path".to_string(),+ serde_json::json!(format!("src/mod{}.rs", rng.below(400))),+ );+ map.insert("query".to_string(), serde_json::json!(sentence(rng, 3)));+ map+ },+ thought_signature: None,+ rest: serde_json::Map::new(),+ }));+ }+ let stop_reason = if tool.is_some() {+ StopReason::ToolUse+ } else if rng.chance(2) {+ StopReason::Error+ } else {+ StopReason::Stop+ };+ rows.push(+ "message",+ serde_json::json!({+ "message": pa_types::session::AgentMessage::Assistant(AssistantMessage {+ content: blocks,+ api: "openai".to_string(),+ provider: provider.to_string(),+ model: model.to_string(),+ response_model: None,+ response_id: None,+ diagnostics: None,+ usage: usage(rng),+ stop_reason,+ stop_reason_raw: None,+ error_message: None,+ timestamp: rows.millis,+ rest: serde_json::Map::new(),+ })+ }),+ rng,+ )?;++ if let Some((call_id, name)) = tool {+ rows.push(+ "message",+ serde_json::json!({+ "message": pa_types::session::AgentMessage::ToolResult(ToolResultMessage {+ tool_call_id: call_id,+ tool_name: name.to_string(),+ content: vec![UserContentBlock::Text(TextContent {+ text: tool_output(rng),+ text_signature: None,+ rest: serde_json::Map::new(),+ })],+ details: None,+ is_error: rng.chance(10),+ timestamp: rows.millis,+ rest: serde_json::Map::new(),+ })+ }),+ rng,+ )?;+ }++ if rng.chance(7) {+ let custom = CUSTOM_TYPES[rng.below(CUSTOM_TYPES.len() as u64) as usize];+ rows.push(+ "custom_message",+ serde_json::to_value(CustomMessageEntry {+ custom_type: custom.to_string(),+ content: UserContent::Text(format!("[{custom}] {}", sentence(rng, 8))),+ details: None,+ display: false,+ rest: serde_json::Map::new(),+ })?,+ rng,+ )?;+ }+ }++ rows.writer.flush()?;+ Ok(Written {+ rows: rows.counter as usize,+ last_user_text,+ })+}++#[cfg(test)]+mod tests {+ fn spec(rows: usize) -> super::SessionSpec {+ super::SessionSpec {+ id: "corpus-regression".to_string(),+ start_ms: 1_700_000_000_000,+ cwd: "/tmp".to_string(),+ rows,+ name: Some("regression".to_string()),+ marker: Some("final user marker".to_string()),+ parent_session: None,+ rlm_depth: Some(0),+ }+ }++ #[test]+ fn final_user_marker_survives_all_small_turn_shapes() {+ let directory = tempfile::tempdir().expect("session directory");+ for seed in 0..16 {+ for target in 9..32 {+ let written = super::write_session_file(+ &directory.path().join("session.jsonl"),+ &spec(target),+ &mut crate::rng::Rng::new(seed),+ )+ .expect("write session");+ assert_eq!(+ written.last_user_text, "final user marker",+ "seed={seed}, target={target}"+ );+ }+ }+ }++ #[test]+ fn long_sessions_crossing_compaction_threshold_have_exactly_one_boundary() {+ let directory = tempfile::tempdir().expect("session directory");+ let path = directory.path().join("session.jsonl");+ for seed in 0..8 {+ super::write_session_file(&path, &spec(1500), &mut crate::rng::Rng::new(seed))+ .expect("write long session");+ let rows: Vec<serde_json::Value> = std::fs::read_to_string(&path)+ .expect("read session")+ .lines()+ .map(|line| serde_json::from_str(line).expect("valid row"))+ .collect();+ assert_eq!(+ rows.iter()+ .filter(|row| row["type"] == "compaction")+ .count(),+ 1,+ "seed={seed}"+ );+ let entries = pa_core::session::parse_session_entries(+ &std::fs::read_to_string(&path).expect("read generated session"),+ );+ let context = pa_core::session::build_session_context(&entries, None);+ assert_eq!(+ context+ .messages+ .iter()+ .filter(|message| matches!(+ message,+ pa_types::session::AgentMessage::CompactionSummary(_)+ ))+ .count(),+ 1,+ "the real context builder must not duplicate the summary; seed={seed}"+ );+ }+ }+}diff --git a/crates/pa-perf/src/lib.rs b/crates/pa-perf/src/lib.rsnew file mode 100644index 0000000000..33a6b6c71f--- /dev/null+++ b/crates/pa-perf/src/lib.rs@@ -0,0 +1,15 @@+//! pa-perf: the prime-agent performance corpus generator and TUI bench+//! harness. A dev/CI tooling crate at the top of the workspace dependency+//! graph — nothing depends on it, and it never ships in a release.+//!+//! The corpus generator writes the product's real on-disk layout under an+//! isolated agent dir, using the product's own serializers and writers+//! (pa-types payloads, pa-daemon `SessionEntry`/`session_header_line`/+//! `RlmSpawnLedger`, pa-core's `WindowedSessionStore` for the warm+//! sidecars). The bench harness drives the real release binary under a pty.++pub mod bench;+pub mod corpus;+pub mod report;++mod rng;diff --git a/crates/pa-perf/src/main.rs b/crates/pa-perf/src/main.rsnew file mode 100644index 0000000000..a02a05b9df--- /dev/null+++ b/crates/pa-perf/src/main.rs@@ -0,0 +1,252 @@+//! The `pa-perf` CLI: `corpus`, `bench`, `run`, and `report`.++use std::path::PathBuf;++use clap::{Parser, Subcommand};+use pa_perf::bench::{BenchOptions, RunOptions, Scenario};+use pa_perf::corpus::{self, CacheMode, CorpusOptions, Profile};+use pa_perf::report::{self, Baseline};++#[derive(Parser)]+#[command(name = "pa-perf", about = "prime-agent performance tooling", version)]+struct Cli {+ #[command(subcommand)]+ command: Command,+}++#[derive(Subcommand)]+enum Command {+ /// Generate a deterministic corpus under an isolated agent dir.+ Corpus(CorpusArgs),+ /// Drive the real release binary under a pty and measure one scenario.+ Bench(BenchArgs),+ /// Generate the corpora and run every scenario on the corpus state it+ /// needs (the CI and `make perf` entry point).+ Run(RunArgs),+ /// Compare result JSONs against the baseline; fail on >2x regressions.+ Report(ReportArgs),+}++#[derive(clap::Args)]+struct CorpusArgs {+ /// The named profile providing the default knob values.+ #[arg(long, value_enum, default_value_t = Profile::Small)]+ profile: Profile,+ /// The corpus root: created, then canonicalized (macOS /tmp -> /private/tmp).+ #[arg(long)]+ out: PathBuf,+ /// The deterministic seed.+ #[arg(long, default_value_t = 7)]+ seed: u64,+ /// Top-level sessions in the sessions dir.+ #[arg(long)]+ sessions: Option<usize>,+ /// Rows per session.+ #[arg(long)]+ rows: Option<usize>,+ /// Rows per subagent child session.+ #[arg(long)]+ child_rows: Option<usize>,+ /// Rows of the one huge session.+ #[arg(long)]+ huge_rows: Option<usize>,+ /// Subagent children per parent per level.+ #[arg(long)]+ fanout: Option<usize>,+ /// Subagent tree depth.+ #[arg(long)]+ depth: Option<u32>,+ /// How many sessions grow a subagent tree.+ #[arg(long)]+ tree_parents: Option<usize>,+ /// One parent grows this many direct children (0 disables).+ #[arg(long)]+ wide_children: Option<usize>,+ /// Sessions in sessions-archive/.+ #[arg(long)]+ archive: Option<usize>,+ /// Files in the generated project dir.+ #[arg(long)]+ cwd_files: Option<usize>,+ /// Write the per-session artifact partitions.+ #[arg(long, default_value_t = true)]+ artifacts: bool,+ /// Write the window-cache sidecars via the product's window-store open.+ #[arg(long, value_enum, default_value_t = CacheMode::Cold)]+ cache: CacheMode,+}++#[derive(clap::Args)]+struct BenchArgs {+ /// The release binary under test.+ #[arg(long, default_value = "target/release/prime-agent")]+ binary: PathBuf,+ /// The corpus root (holds corpus.json).+ #[arg(long)]+ corpus: PathBuf,+ /// The scenario to measure.+ #[arg(long, value_enum)]+ scenario: Scenario,+ /// Sampled runs (a warm-up run is always discarded first).+ #[arg(long, default_value_t = 10)]+ runs: usize,+ /// Where the result JSON is written.+ #[arg(long)]+ json: PathBuf,+ /// The per-run liveness deadline (seconds).+ #[arg(long, default_value_t = 180)]+ liveness_secs: u64,+}++#[derive(clap::Args)]+struct RunArgs {+ /// The release binary under test.+ #[arg(long, default_value = "target/release/prime-agent")]+ binary: PathBuf,+ /// The named corpus profile for both generated corpora.+ #[arg(long, value_enum, default_value_t = Profile::Small)]+ profile: Profile,+ /// The output root: the two corpora and results/ land under it.+ #[arg(long, default_value = "target/perf")]+ out: PathBuf,+ /// The deterministic seed.+ #[arg(long, default_value_t = 7)]+ seed: u64,+ /// Sampled runs per scenario (a warm-up run is always discarded first).+ #[arg(long, default_value_t = 10)]+ runs: usize,+ /// The per-run liveness deadline (seconds).+ #[arg(long, default_value_t = 180)]+ liveness_secs: u64,+}++#[derive(clap::Args)]+struct ReportArgs {+ /// One or more bench result JSONs.+ #[arg(long, num_args = 1..)]+ results: Vec<PathBuf>,+ /// The committed baseline file.+ #[arg(long, default_value = "crates/pa-perf/baseline.json")]+ baseline: PathBuf,+ /// Where the markdown table is written (stdout when absent).+ #[arg(long)]+ markdown: Option<PathBuf>,+}++fn main() -> anyhow::Result<()> {+ match Cli::parse().command {+ Command::Corpus(args) => corpus_command(args),+ Command::Bench(args) => bench_command(args),+ Command::Run(args) => run_command(args),+ Command::Report(args) => report_command(args),+ }+}++fn corpus_command(args: CorpusArgs) -> anyhow::Result<()> {+ let mut options = CorpusOptions {+ out: args.out.clone(),+ ..args.profile.defaults()+ };+ options.seed = args.seed;+ if let Some(value) = args.sessions {+ options.sessions = value;+ }+ if let Some(value) = args.rows {+ options.rows = value;+ }+ if let Some(value) = args.child_rows {+ options.child_rows = value;+ }+ if let Some(value) = args.huge_rows {+ options.huge_rows = value;+ }+ if let Some(value) = args.fanout {+ options.fanout = value;+ }+ if let Some(value) = args.depth {+ options.depth = value;+ }+ if let Some(value) = args.tree_parents {+ options.tree_parents = value;+ }+ if let Some(value) = args.wide_children {+ options.wide_children = value;+ }+ if let Some(value) = args.archive {+ options.archive = value;+ }+ if let Some(value) = args.cwd_files {+ options.cwd_files = value;+ }+ options.artifacts = args.artifacts;+ options.cache = args.cache;+ let manifest = corpus::generate(&options)?;+ println!(+ "corpus: profile={} sessions={} rows={} total_rows={} child_sessions={} archive={} cache={}",+ report::profile_key(options.profile),+ manifest.counts.sessions,+ options.rows,+ manifest.counts.total_rows,+ manifest.counts.child_sessions,+ manifest.counts.archive,+ report::profile_key_cache(options.cache),+ );+ println!("huge needle: {}", manifest.markers.huge.needle);+ println!("manifest: {}", manifest.paths.manifest_file.display());+ Ok(())+}++fn bench_command(args: BenchArgs) -> anyhow::Result<()> {+ let options = BenchOptions {+ binary: args.binary,+ corpus: args.corpus,+ scenario: args.scenario,+ runs: args.runs,+ json: args.json,+ liveness_secs: args.liveness_secs,+ };+ let result = pa_perf::bench::run(&options)?;+ println!(+ "scenario={} profile={} median={:.1}ms p95={:.1}ms runs={}",+ report::key_of(result.scenario),+ result.profile,+ result.median_ms,+ result.p95_ms,+ result.runs,+ );+ Ok(())+}++fn run_command(args: RunArgs) -> anyhow::Result<()> {+ let options = RunOptions {+ binary: args.binary,+ profile: args.profile,+ out: args.out,+ seed: args.seed,+ runs: args.runs,+ liveness_secs: args.liveness_secs,+ };+ pa_perf::bench::run_suite(&options)?;+ Ok(())+}++fn report_command(args: ReportArgs) -> anyhow::Result<()> {+ let mut results = Vec::new();+ for path in &args.results {+ let result: pa_perf::bench::BenchResult =+ serde_json::from_str(&std::fs::read_to_string(path)?)?;+ results.push(result);+ }+ let baseline: Baseline = serde_json::from_str(&std::fs::read_to_string(&args.baseline)?)?;+ let report = report::report(&results, &baseline);+ match &args.markdown {+ Some(path) => std::fs::write(path, &report.markdown)?,+ None => println!("{}", report.markdown),+ }+ if report.regressions.is_empty() {+ println!("no >2x regressions against the baseline");+ Ok(())+ } else {+ anyhow::bail!("regressions over 2x: {}", report.regressions.join(", "));+ }+}diff --git a/crates/pa-perf/src/report.rs b/crates/pa-perf/src/report.rsnew file mode 100644index 0000000000..4dfd2993dc--- /dev/null+++ b/crates/pa-perf/src/report.rs@@ -0,0 +1,112 @@+//! The baseline comparison: the aggregate table (median, p95, baseline,+//! delta) and the >2x regression gate (report-only scenarios always print+//! deltas and never fail).++use std::collections::BTreeMap;++use serde::{Deserialize, Serialize};++use crate::bench::{BenchResult, Scenario};+use crate::corpus::Profile;++/// The committed baseline: one entry per scenario/profile key.+#[derive(Debug, Serialize, Deserialize)]+pub struct Baseline {+ pub schema: u32,+ /// "scenario/profile" -> the measured baseline.+ pub entries: BTreeMap<String, BaselineEntry>,+}++#[derive(Debug, Serialize, Deserialize)]+pub struct BaselineEntry {+ /// The baseline median (ms).+ pub median_ms: f64,+ /// Noisy scenarios (>20% spread across 3 CI runs): deltas print, the+ /// gate never fails.+ #[serde(default)]+ pub report_only: bool,+ /// Where the number came from (a CI run URL).+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub source: Option<String>,+}++/// The regression gate's ceiling: a median must stay under baseline * 2.+const REGRESSION_FACTOR: f64 = 2.0;++/// The comparison outcome: the markdown table and the failing rows.+#[derive(Debug)]+pub struct Report {+ pub markdown: String,+ pub regressions: Vec<String>,+}++/// Compare results against the baseline; the key is scenario/profile (the+/// serde names — one source for the wire, the CLI, and the keys).+#[must_use]+pub fn report(results: &[BenchResult], baseline: &Baseline) -> Report {+ let mut rows = Vec::new();+ let mut regressions = Vec::new();+ for result in results {+ let key = format!("{}/{}", key_of(result.scenario), result.profile);+ let median = result.median_ms;+ let Some(entry) = baseline.entries.get(&key) else {+ rows.push(format!(+ "| {} | {median:.1} | {:.1} | — | — | (no baseline) |",+ key, result.p95_ms+ ));+ continue;+ };+ let delta = median / entry.median_ms - 1.0;+ let verdict = if entry.report_only {+ "report-only".to_string()+ } else if median > entry.median_ms * REGRESSION_FACTOR {+ regressions.push(key.clone());+ "REGRESSION".to_string()+ } else {+ format!("{:+.0}%", delta * 100.0)+ };+ rows.push(format!(+ "| {} | {median:.1} | {:.1} | {:.1} | {:+.0}% | {verdict} |",+ key,+ result.p95_ms,+ entry.median_ms,+ delta * 100.0+ ));+ }+ let markdown = format!(+ "| scenario/profile | median ms | p95 ms | baseline ms | delta | verdict |\n|---|---|---|---|---|---|\n{}",+ rows.join("\n")+ );+ Report {+ markdown,+ regressions,+ }+}++/// The serde name of a scenario (the baseline key's scenario half — the+/// same name the CLI and the wire use).+#[must_use]+pub fn key_of(scenario: Scenario) -> String {+ serde_json::to_value(scenario)+ .ok()+ .and_then(|value| value.as_str().map(str::to_string))+ .unwrap_or_default()+}++/// The serde name of a profile (the baseline key's profile half).+#[must_use]+pub fn profile_key(profile: Profile) -> String {+ serde_json::to_value(profile)+ .ok()+ .and_then(|value| value.as_str().map(str::to_string))+ .unwrap_or_default()+}++/// The serde name of a cache mode (display only).+#[must_use]+pub fn profile_key_cache(mode: crate::corpus::CacheMode) -> String {+ serde_json::to_value(mode)+ .ok()+ .and_then(|value| value.as_str().map(str::to_string))+ .unwrap_or_default()+}diff --git a/crates/pa-perf/src/rng.rs b/crates/pa-perf/src/rng.rsnew file mode 100644index 0000000000..f4ca5b5914--- /dev/null+++ b/crates/pa-perf/src/rng.rs@@ -0,0 +1,65 @@+//! The corpus PRNG: a tiny deterministic splitmix64 (no external dependency).+//! The same seed produces a byte-identical corpus tree, file mtimes included.++/// A deterministic splitmix64 stream for corpus generation.+#[derive(Debug, Clone)]+pub(crate) struct Rng(u64);++impl Rng {+ /// A stream from `seed`.+ pub(crate) const fn new(seed: u64) -> Self {+ Self(seed)+ }++ /// The next raw value.+ pub(crate) fn next_u64(&mut self) -> u64 {+ self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);+ let mut z = self.0;+ z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);+ z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);+ z ^ (z >> 31)+ }++ /// A uniform value below `bound` (an empty bound answers 0).+ pub(crate) fn below(&mut self, bound: u64) -> u64 {+ if bound == 0 {+ return 0;+ }+ self.next_u64() % bound+ }++ /// A true chance of `percent` in 100.+ pub(crate) fn chance(&mut self, percent: u64) -> bool {+ self.below(100) < percent+ }+}++/// Civil-date conversion (days since epoch -> y/m/d), the inverse of the+/// session parser's `civil_days`: keeps timestamp formatting allocation-free.+fn civil_from_days(days: i64) -> (i64, u32, u32) {+ let z = days + 719_468;+ let era = if z >= 0 { z } else { z - 146_096 } / 146_097;+ let doe = z - era * 146_097;+ let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146_096) / 365;+ let y = yoe + era * 400;+ let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);+ let mp = (5 * doy + 2) / 153;+ let d = (doy - (153 * mp + 2) / 5 + 1) as u32;+ let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;+ (if m <= 2 { y + 1 } else { y }, m, d)+}++/// ISO-8601 (UTC, millisecond precision) from unix milliseconds — the+/// product's entry-timestamp format.+#[must_use]+pub(crate) fn iso_from_millis(millis: u64) -> String {+ let secs = (millis / 1000) as i64;+ let rem = millis % 1000;+ let days = secs.div_euclid(86_400);+ let secs_of_day = secs.rem_euclid(86_400);+ let (year, month, day) = civil_from_days(days);+ let hour = secs_of_day / 3600;+ let minute = (secs_of_day % 3600) / 60;+ let second = secs_of_day % 60;+ format!("{year:04}-{month:02}-{day:02}T{hour:02}:{minute:02}:{second:02}.{rem:03}Z")+}diff --git a/crates/pa-perf/tests/corpus_verification.rs b/crates/pa-perf/tests/corpus_verification.rsnew file mode 100644index 0000000000..dfdcdcea48--- /dev/null+++ b/crates/pa-perf/tests/corpus_verification.rs@@ -0,0 +1,379 @@+//! The corpus verification battery: every generated file loads through the+//! PRODUCT's own readers (pa-daemon SessionFile, pa-core parse, the RLM+//! ledger replay), the tree is seed-deterministic, and the marker discipline+//! holds. Kept small and fast: tiny corpora in temp dirs only.++use std::collections::HashSet;+use std::path::Path;++use pa_core::session::parse_session_entries;+use pa_daemon::rlm_ledger::RlmSpawnLedger;+use pa_daemon::session_store::SessionFile;+use pa_perf::corpus::{generate, CacheMode, CorpusOptions, Profile};++fn tiny(profile: Profile, out: &Path) -> CorpusOptions {+ CorpusOptions {+ out: out.to_path_buf(),+ ..profile.defaults()+ }+}++/// Every generated session file must parse through the daemon's own store.+#[test]+fn sessions_load_through_the_product_store() {+ let dir = tempfile::tempdir().expect("temp dir");+ let options = tiny(Profile::Small, dir.path());+ let manifest = generate(&options).expect("generate");+ let mut checked = 0;+ for file in &manifest.files {+ let path = std::path::PathBuf::from(file);+ if path.parent() != Some(&manifest.paths.sessions_dir) {+ continue;+ }+ let store = SessionFile::open(&path).expect("the product store opens the file");+ assert_eq!(store.header.id, path.file_stem().unwrap().to_string_lossy());+ assert!(!store.entries().is_empty(), "the session carried its rows");+ checked += 1;+ }+ assert_eq!(checked, options.sessions, "every sessions-dir file checked");+}++/// No row may degrade to an Unknown entry: the corpus speaks the wire format.+#[test]+fn rows_never_degrade_to_unknown_entries() {+ let dir = tempfile::tempdir().expect("temp dir");+ let options = tiny(Profile::Small, dir.path());+ let manifest = generate(&options).expect("generate");+ let huge = &manifest.markers.huge;+ let text = std::fs::read_to_string(&huge.file).expect("read the huge session");+ let entries = parse_session_entries(&text);+ assert!(+ entries+ .iter()+ .all(|entry| !matches!(entry, pa_types::session::FileEntry::Unknown { .. })),+ "no unknown rows in the generated huge session"+ );+}++/// The marker lands on the final user row of each marker session.+#[test]+fn markers_land_on_the_final_user_message() {+ let dir = tempfile::tempdir().expect("temp dir");+ let options = tiny(Profile::Small, dir.path());+ let manifest = generate(&options).expect("generate");+ for marker in [+ &manifest.markers.huge,+ &manifest.markers.switch_a,+ &manifest.markers.switch_b,+ ] {+ let text = std::fs::read_to_string(&marker.file).expect("read session");+ let last_user = text+ .lines()+ .rev()+ .find_map(|line| {+ let value: serde_json::Value = serde_json::from_str(line).ok()?;+ (value.get("type") == Some(&serde_json::json!("message"))+ && value["message"]["role"] == serde_json::json!("user"))+ .then(|| value["message"]["content"].to_string())+ })+ .expect("a final user message");+ assert!(+ last_user.contains(&marker.needle),+ "the needle is the final user message: {last_user}"+ );+ }+}++/// The ledger the corpus writes replays through the product's own reader.+#[test]+fn the_ledger_replays_through_the_product_reader() {+ let dir = tempfile::tempdir().expect("temp dir");+ let options = tiny(Profile::Small, dir.path());+ let manifest = generate(&options).expect("generate");+ let ledger = RlmSpawnLedger::new(+ &manifest.paths.agent_dir,+ &manifest.paths.sessions_dir,+ |_| {},+ );+ let edges = ledger.live_edges().expect("the ledger replays");+ assert_eq!(+ edges.len(),+ manifest.counts.subagent_edges,+ "every spawn edge replays as live"+ );+ assert!(+ edges+ .iter()+ .all(|edge| edge.depth >= 1 && edge.child.contains("session-artifacts")),+ "edges carry the artifact-tree child files"+ );+}++/// Same seed, same tree: every file the generator owns is byte-identical+/// (the ledger's `at` stamps come from the product writer's wall clock; the+/// edge content is compared field-by-field instead).+#[test]+fn the_same_seed_rebuilds_the_identical_tree() {+ let first = tempfile::tempdir().expect("temp dir");+ let second = tempfile::tempdir().expect("temp dir");+ let first_root = first.path().canonicalize().expect("canonical first");+ let second_root = second.path().canonicalize().expect("canonical second");+ generate(&tiny(Profile::Small, &first_root)).expect("first generate");+ generate(&tiny(Profile::Small, &second_root)).expect("second generate");+ // The ledger file NAME is the hash of the canonical sessions dir, so+ // two corpus roots name it differently; normalize it for comparison.+ fn normalized(mut files: Vec<String>) -> Vec<String> {+ for file in &mut files {+ if let Some(hash) = file+ .strip_prefix("agent/rlm-ledger/")+ .and_then(|rest| rest.strip_suffix(".jsonl"))+ {+ if !hash.contains('/') {+ *file = "agent/rlm-ledger/<hash>.jsonl".to_string();+ }+ }+ }+ files+ }+ let first_files = normalized(relative_files(&first_root));+ let second_files = normalized(relative_files(&second_root));+ let mut first_sorted = first_files.clone();+ first_sorted.sort();+ let mut second_sorted = second_files.clone();+ second_sorted.sort();+ assert_eq!(first_sorted, second_sorted, "the file sets match");++ for relative in &first_sorted {+ let a = first_root.join(relative);+ let b = second_root.join(relative);+ if relative.starts_with("agent/rlm-ledger/") {+ let a = ledger_of(&first_root);+ let b = ledger_of(&second_root);+ let edges = |path: &Path, root: &Path| -> Vec<(String, String, u32)> {+ std::fs::read_to_string(path)+ .expect("read ledger")+ .lines()+ .filter_map(|line| {+ let value: serde_json::Value = serde_json::from_str(line).ok()?;+ (value["op"] == serde_json::json!("spawn")).then(|| {+ let child = Path::new(value["child"].as_str().unwrap_or_default());+ let child = child+ .strip_prefix(root)+ .map_or_else(|_| child.to_path_buf(), Path::to_path_buf);+ (+ value["childId"].as_str().unwrap_or_default().to_string(),+ child.to_string_lossy().to_string(),+ value["depth"].as_u64().unwrap_or_default() as u32,+ )+ })+ })+ .collect()+ };+ assert_eq!(+ edges(&a, &first_root),+ edges(&b, &second_root),+ "the ledger edges match"+ );+ continue;+ }+ // corpus.json embeds absolute paths and the ledger's hash name; the+ // ledger itself and child headers embed the corpus root in+ // parentSession/child paths. Normalize the root, skip the manifest.+ if relative == "corpus.json" {+ continue;+ }+ let read = |path: &Path, root: &Path| -> Vec<u8> {+ let bytes = std::fs::read(path).expect("read");+ let root = root.to_string_lossy().into_owned().into_bytes();+ let mut normalized = Vec::with_capacity(bytes.len());+ let mut index = 0;+ while index < bytes.len() {+ if bytes[index..].starts_with(&root) {+ normalized.extend_from_slice(b"<root>");+ index += root.len();+ } else {+ normalized.push(bytes[index]);+ index += 1;+ }+ }+ normalized+ };+ assert_eq!(+ read(&a, &first_root),+ read(&b, &second_root),+ "{relative} differs between seeds-equal runs"+ );+ }+}++/// Entry ids are unique 8-hex counters even past the ~22k-row collision+/// cliff the product's uuid-prefix ids hit.+#[test]+fn entry_ids_stay_unique_past_the_uuid_prefix_collision_cliff() {+ let dir = tempfile::tempdir().expect("temp dir");+ let options = CorpusOptions {+ sessions: 3,+ rows: 25_000,+ child_rows: 0,+ fanout: 0,+ depth: 0,+ tree_parents: 0,+ wide_children: 0,+ archive: 0,+ cwd_files: 5,+ ..tiny(Profile::Small, dir.path())+ };+ let manifest = generate(&options).expect("generate");+ let file = manifest+ .paths+ .sessions_dir+ .join(format!("{}.jsonl", manifest.markers.huge.id));+ let text = std::fs::read_to_string(&file).expect("read session");+ let mut ids = HashSet::new();+ let mut rows = 0;+ for line in text.lines() {+ let value: serde_json::Value = serde_json::from_str(line).expect("a parseable row");+ if value["type"] == serde_json::json!("session") {+ continue;+ }+ let id = value["id"].as_str().expect("an id").to_string();+ assert_eq!(id.len(), 8, "the id is 8-hex");+ assert!(id.chars().all(|c| c.is_ascii_hexdigit()), "hex id: {id}");+ ids.insert(id);+ rows += 1;+ }+ assert_eq!(ids.len(), rows, "no id collides across {rows} rows");+}++/// The warm pass writes the product's own window-cache sidecars.+#[test]+fn the_warm_pass_writes_product_sidecars() {+ let dir = tempfile::tempdir().expect("temp dir");+ let mut options = tiny(Profile::Small, dir.path());+ options.cache = CacheMode::Warm;+ let manifest = generate(&options).expect("generate");+ let sidecars = manifest+ .files+ .iter()+ .filter(|file| file.ends_with("window-cache.json"))+ .count();+ assert_eq!(+ sidecars,+ options.sessions + options.archive,+ "every openable session gained a window-cache sidecar"+ );+}++/// The switch-b row is the ONLY session the switch query can match. The+/// agents-view search is a fuzzy subsequence match over each row's name+/// AND message text, so a common token could rank another row first; the+/// guarantee needs the `zq` token to be literally impossible elsewhere —+/// no generated corpus text (names or message bodies) contains the letter+/// z at all (the ISO `Z` header suffix is the one allowed z).+#[test]+fn the_switch_query_can_only_match_switch_b() {+ let dir = tempfile::tempdir().expect("temp dir");+ let options = tiny(Profile::Small, dir.path());+ let manifest = generate(&options).expect("generate");+ assert_eq!(manifest.markers.switch_row, "switch target zqmark");+ assert!(manifest.markers.switch_query.contains("zqmark"));+ assert!(manifest.markers.switch_query != manifest.markers.switch_row);+ let mut scanned = 0;+ for file in &manifest.files {+ let path = std::path::PathBuf::from(file);+ if path.parent() == Some(&manifest.paths.sessions_dir) {+ // The search corpus folds MESSAGE text and names (not header+ // timestamps, whose ISO `Z` suffix is the one allowed z).+ for line in std::fs::read_to_string(&path)+ .expect("read session")+ .lines()+ {+ if line.contains('z')+ && !line.contains(r#""type":"session""#)+ && !line.contains(&manifest.markers.switch_row)+ {+ panic!("a non-header row carries z: {line}");+ }+ }+ scanned += 1;+ }+ }+ assert_eq!(scanned, options.sessions, "every session checked");+}++/// The corpus settings disable BOTH archive rules: the daemon's boot sweep+/// would otherwise retire corpus sessions out from under the manifest+/// baseline (the defaults keep 200 files / 30 days).+#[test]+fn the_archive_policy_is_off() {+ let dir = tempfile::tempdir().expect("temp dir");+ let options = tiny(Profile::Small, dir.path());+ let manifest = generate(&options).expect("generate");+ let settings: serde_json::Value = serde_json::from_str(+ &std::fs::read_to_string(manifest.paths.agent_dir.join("settings.json")).expect("settings"),+ )+ .expect("a settings object");+ assert_eq!(+ settings["sessionArchiveMaxAgeDays"],+ serde_json::json!("off")+ );+ assert_eq!(+ settings["sessionArchiveMaxSessions"],+ serde_json::json!("off")+ );+}++/// The corpus manifest records the generated file set (the bench's+/// per-run restore baseline) and the archive dir holds the archived files.+#[test]+fn the_manifest_is_the_file_set_baseline() {+ let dir = tempfile::tempdir().expect("temp dir");+ let options = tiny(Profile::Small, dir.path());+ let manifest = generate(&options).expect("generate");+ let on_disk = relative_files(dir.path());+ assert_eq!(+ manifest.files.len() + 1, // corpus.json is written after the walk+ on_disk.len(),+ "the manifest lists every generated file"+ );+ let archived = on_disk+ .iter()+ .filter(|file| file.starts_with("agent/sessions-archive/"))+ .count();+ assert_eq!(archived, options.archive);+}++/// The one ledger file under a corpus root (its name is the sessions-dir+/// hash, which differs per root).+fn ledger_of(root: &Path) -> std::path::PathBuf {+ let dir = root.join("agent/rlm-ledger");+ std::fs::read_dir(dir)+ .expect("read the ledger dir")+ .map(|entry| entry.expect("entry").path())+ .find(|path| path.extension().is_some_and(|ext| ext == "jsonl"))+ .expect("the ledger file")+}++fn relative_files(root: &Path) -> Vec<String> {+ fn walk(dir: &Path, root: &Path, out: &mut Vec<String>) {+ let entries = std::fs::read_dir(dir).expect("read dir");+ for entry in entries {+ let entry = entry.expect("entry");+ let path = entry.path();+ if path.is_dir() {+ walk(&path, root, out);+ } else {+ out.push(+ path.strip_prefix(root)+ .expect("inside root")+ .to_string_lossy()+ .to_string(),+ );+ }+ }+ }+ let mut files = Vec::new();+ walk(root, root, &mut files);+ files.sort();+ files+}diff --git a/crates/pa-perf/tests/fixtures/draining_daemon.py b/crates/pa-perf/tests/fixtures/draining_daemon.pynew file mode 100644index 0000000000..98b28c1616--- /dev/null+++ b/crates/pa-perf/tests/fixtures/draining_daemon.py@@ -0,0 +1,72 @@+#!/usr/bin/env python3+"""Mock product modes: daemon shutdown waits for its held UI connection.++Only used by the Rust harness regression; never loads or launches the product.+"""+import json+from pathlib import Path+import select+import socket+import sys+import time++path = Path(sys.argv[sys.argv.index('--daemon-socket') + 1])+if '--mode' not in sys.argv:+ with socket.socket(socket.AF_UNIX) as peer:+ peer.connect(str(path))+ peer.recv(4096)+ peer.sendall(b'held-ui\n')+ sys.stdout.write('▗▄▄█▀ mock paint 0 (0%)\n')+ sys.stdout.flush()+ # The mock UI remains connected until its owned process is stopped.+ while peer.recv(4096):+ pass+ sys.exit(0)++listener = socket.socket(socket.AF_UNIX)+listener.bind(str(path))+listener.listen()+peers = {}+accepting = True+shutdown = False+deadline = time.monotonic() + 20+try:+ while accepting or peers:+ assert time.monotonic() < deadline, 'mock daemon connection drain timed out'+ ready, _, _ = select.select(([listener] if accepting else []) + list(peers), [], [], 1)+ for peer in ready:+ if peer is listener:+ client, _ = listener.accept()+ peers[client] = b''+ try:+ client.sendall(b'{"type":"daemon_hello"}\n')+ except (BrokenPipeError, ConnectionResetError):+ client.close()+ del peers[client]+ continue+ try:+ data = peer.recv(4096)+ except ConnectionResetError:+ data = b''+ if not data:+ peer.close()+ del peers[peer]+ continue+ peers[peer] += data+ if peers[peer] == b'held-ui\n':+ continue+ if b'\n' not in peers[peer]:+ continue+ request = json.loads(peers[peer])+ assert request['command']['type'] == 'shutdown'+ peer.sendall((json.dumps({'type': 'response', 'id': request['id'],+ 'success': True}) + '\n').encode())+ listener.close()+ accepting = False+ shutdown = True+ assert shutdown, 'mock daemon never received protocol shutdown'+ path.with_suffix('.drained').write_text('all UI/control connections closed\n')+finally:+ listener.close()+ for peer in peers:+ peer.close()diff --git a/crates/pa-perf/tests/report_regression.rs b/crates/pa-perf/tests/report_regression.rsnew file mode 100644index 0000000000..e1007c471b--- /dev/null+++ b/crates/pa-perf/tests/report_regression.rs@@ -0,0 +1,74 @@+//! The report battery: the >2x regression gate, the report-only exempt,+//! and the missing-baseline row.++use std::collections::BTreeMap;++use pa_perf::bench::{BenchCorpus, BenchResult, Scenario};+use pa_perf::report::{report, Baseline, BaselineEntry};++fn result(scenario: Scenario, median_ms: f64) -> BenchResult {+ BenchResult {+ schema: 1,+ scenario,+ profile: "small".to_string(),+ runs: 10,+ samples_ms: vec![median_ms],+ median_ms,+ p95_ms: median_ms,+ needle: String::new(),+ binary: String::new(),+ corpus: BenchCorpus {+ root: std::path::PathBuf::new(),+ sessions: 10,+ total_rows: 3_500,+ child_sessions: 12,+ archive: 3,+ },+ }+}++fn baseline(median_ms: f64, report_only: bool) -> Baseline {+ Baseline {+ schema: 1,+ entries: BTreeMap::from([(+ "cold-start/small".to_string(),+ BaselineEntry {+ median_ms,+ report_only,+ source: None,+ },+ )]),+ }+}++#[test]+fn a_two_x_regression_fails_and_deltas_always_print() {+ let report = report(+ &[result(Scenario::ColdStart, 2_100.0)],+ &baseline(1_000.0, false),+ );+ assert_eq!(report.regressions, ["cold-start/small"]);+ assert!(report.markdown.contains("cold-start/small"));+ assert!(report.markdown.contains("REGRESSION"));+}++#[test]+fn a_report_only_row_never_fails_the_gate() {+ let report = report(+ &[result(Scenario::ColdStart, 5_000.0)],+ &baseline(1_000.0, true),+ );+ assert!(report.regressions.is_empty());+ assert!(report.markdown.contains("report-only"));+}++#[test]+fn a_missing_baseline_prints_no_baseline() {+ let report = report(+ &[result(Scenario::OpenSession, 147.0)],+ &baseline(1_000.0, false),+ );+ assert!(report.regressions.is_empty());+ assert!(report.markdown.contains("open-session/small"));+ assert!(report.markdown.contains("(no baseline)"));+}