Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): fail pull requests that make export, seek or playback slower

Richie McIlroyeaed5bc44e12
diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml
new file mode 100644
index 000000000..63abf699d
--- /dev/null
+++ b/.github/workflows/perf-budget.yml
@@ -0,0 +1,139 @@
+name: Perf budget
+
+# Builds the engine suite's export, seek and playback binaries for a pull
+# request and for its merge base on the same Blacksmith macOS machine, runs
+# each workload in interleaved pairs and fails when the pull request is
+# measurably slower (scripts/verify/perf_ab.py). Runs for ready (non-draft)
+# pull requests that change an engine crate or a high-risk region, like the
+# Engine verify FULL tier; every other pull request gets a green verdict
+# without running. The "Perf budget" job is the one to require.
+
+on:
+ pull_request:
+ types: [opened, synchronize, reopened, ready_for_review]
+ workflow_dispatch:
+ inputs:
+ base:
+ description: "Ref to compare against (default: the merge base with main)"
+ default: ""
+
+concurrency:
+ group: perf-budget-${{ github.event.pull_request.number || github.ref }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ scope:
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ outputs:
+ run: ${{ steps.scope.outputs.run }}
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ - name: Scope
+ id: scope
+ env:
+ GH_TOKEN: ${{ github.token }}
+ EVENT: ${{ github.event_name }}
+ PR: ${{ github.event.pull_request.number }}
+ DRAFT: ${{ github.event.pull_request.draft }}
+ RUSTUP_TOOLCHAIN: stable
+ run: |
+ set -euo pipefail
+ if [[ "$EVENT" == workflow_dispatch ]]; then
+ echo "run=true" >> "$GITHUB_OUTPUT"; exit 0
+ fi
+ if [[ "$DRAFT" == true ]]; then
+ echo "Draft: the perf budget runs once the pull request is ready." | tee -a "$GITHUB_STEP_SUMMARY"
+ echo "run=false" >> "$GITHUB_OUTPUT"; exit 0
+ fi
+ gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"
+ python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"
+ if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" \
+ || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf_(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
+ echo "run=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"
+ echo "run=false" >> "$GITHUB_OUTPUT"
+ fi
+
+ measure:
+ needs: scope
+ if: needs.scope.outputs.run == 'true'
+ runs-on: blacksmith-6vcpu-macos-26
+ timeout-minutes: 120
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ fetch-depth: 0
+
+ - name: Setup
+ uses: ./.github/actions/engine-verify-setup
+ with:
+ target: aarch64-apple-darwin
+ tier: fast
+
+ - name: Perf build cache
+ uses: actions/cache@v4
+ with:
+ path: target/perf-verify/
+ key: perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-${{ github.run_id }}
+ restore-keys: |
+ perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-
+ perf-verify-v1-
+
+ - name: Base worktree
+ env:
+ BASE_INPUT: ${{ inputs.base }}
+ PR_BASE: ${{ github.event.pull_request.base.sha }}
+ run: |
+ set -euo pipefail
+ base="${BASE_INPUT:-}"
+ if [[ -z "$base" ]]; then
+ base="$(git merge-base HEAD "${PR_BASE:-origin/main}")"
+ fi
+ echo "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY"
+ git worktree add --detach "$RUNNER_TEMP/base" "$base"
+ # The base builds with the head's native dependencies.
+ ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target"
+
+ - name: Build head
+ run: python3 scripts/verify/perf_build.py . "$RUNNER_TEMP/perf/head"
+
+ - name: Build base
+ run: python3 scripts/verify/perf_build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"
+
+ - name: Measure
+ run: |
+ python3 scripts/verify/perf_ab.py \
+ --base "$RUNNER_TEMP/perf/base/binaries.json" \
+ --head "$RUNNER_TEMP/perf/head/binaries.json" \
+ --fixtures "$RUNNER_TEMP/perf/fixtures" \
+ --out "$RUNNER_TEMP/perf/runs" \
+ --summary "$GITHUB_STEP_SUMMARY"
+
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ if: always()
+ with:
+ name: perf-budget
+ path: ${{ runner.temp }}/perf/runs/**/perf-budget.md
+ if-no-files-found: ignore
+
+ verdict:
+ name: Perf budget
+ needs: [scope, measure]
+ if: always()
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ steps:
+ - env:
+ RUN: ${{ needs.scope.outputs.run }}
+ SCOPE: ${{ needs.scope.result }}
+ MEASURE: ${{ needs.measure.result }}
+ run: |
+ if [[ "$SCOPE" != success ]]; then echo "scope failed"; exit 1; fi
+ if [[ "$RUN" != true ]]; then echo "not measured"; exit 0; fi
+ [[ "$MEASURE" == success ]]
diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf_ab.py
new file mode 100644
index 000000000..0aa1aeb5b
--- /dev/null
+++ b/scripts/verify/perf_ab.py
@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Paired performance comparison of two builds of the engine suite binaries.
+
+Runs each workload alternately with the base and head binaries on the same
+machine and compares the medians of the test process's CPU time (user +
+system, from the kernel's rusage) and wall time. CPU time ignores scheduling
+noise better than wall time; pairing cancels most of the machine's state.
+
+ perf_ab.py --base base.json --head head.json --fixtures DIR --out DIR
+
+base.json / head.json map a test target ("export", "seek") to its binary and
+the crate directory to run it from:
+ {"worktree": "...", "binaries": {"export": "...", "seek": "..."}}
+"""
+import argparse
+import json
+import os
+import statistics
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+WORKLOADS = [
+ # name, test target, case glob, tier
+ ("export-basic", "export", "export/basic/*", "fast"),
+ ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast"),
+ ("seek-storm", "seek", "seek/*/storm30/*", "fast"),
+ ("playback-basic", "playback", "playback/basic/*", "fast"),
+]
+
+# A head build is slower only when its median CPU time exceeds the base's by
+# more than this, and in most pairs.
+CPU_LIMIT = 0.05
+WALL_WARN = 0.08
+
+
+def run_once(side, target, case, tier, fixtures, report):
+ binary = side["binaries"][target]
+ env = dict(
+ os.environ,
+ CAP_VERIFY_FIXTURE_DIR=str(fixtures),
+ CAP_VERIFY_REPORT_DIR=str(report),
+ CAP_VERIFY_CASE=case,
+ CAP_VERIFY_TIER=tier,
+ )
+ report.mkdir(parents=True, exist_ok=True)
+ started = time.monotonic()
+ with (report / "test.log").open("w") as log:
+ process = subprocess.Popen(
+ [binary, "--test-threads", "1", "--nocapture"],
+ cwd=Path(side["worktree"]) / "crates/verify",
+ env=env,
+ stdout=log,
+ stderr=subprocess.STDOUT,
+ )
+ _, status, usage = os.wait4(process.pid, 0)
+ wall = time.monotonic() - started
+ outcomes = {}
+ for path in report.glob("cap-verify-*.jsonl"):
+ for line in path.read_text().splitlines():
+ try:
+ row = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if row.get("case"):
+ outcomes[row["case"]] = row.get("outcome")
+ return {
+ "cpu": usage.ru_utime + usage.ru_stime,
+ "wall": wall,
+ "exit": os.waitstatus_to_exitcode(status),
+ "outcomes": outcomes,
+ }
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--base", required=True)
+ parser.add_argument("--head", required=True)
+ parser.add_argument("--fixtures", required=True)
+ parser.add_argument("--out", required=True)
+ parser.add_argument("--pairs", type=int, default=3)
+ parser.add_argument("--summary")
+ args = parser.parse_args()
+
+ sides = {
+ "base": json.loads(Path(args.base).read_text()),
+ "head": json.loads(Path(args.head).read_text()),
+ }
+ fixtures = Path(args.fixtures)
+ out = Path(args.out)
+ out.mkdir(parents=True, exist_ok=True)
+
+ rows = []
+ failed = []
+ for name, target, case, tier in WORKLOADS:
+ # Untimed warm-up per side: generates fixtures and warms caches.
+ for label, side in sides.items():
+ run_once(side, target, case, tier, fixtures, out / name / f"warm-{label}")
+ samples = {"base": [], "head": []}
+ for pair in range(args.pairs):
+ order = ("base", "head") if pair % 2 == 0 else ("head", "base")
+ for label in order:
+ result = run_once(
+ sides[label], target, case, tier, fixtures, out / name / f"{pair}-{label}"
+ )
+ samples[label].append(result)
+ print(
+ f"{name} pair {pair} {label}: cpu {result['cpu']:.2f}s wall {result['wall']:.2f}s exit {result['exit']}",
+ flush=True,
+ )
+ statuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}
+ if statuses["base"][0] != statuses["head"][0]:
+ # Correctness is the suite's job; a case that now passes or fails
+ # can change how much work runs, so its time is not comparable.
+ rows.append((name, None, None, None, None, "case outcomes differ: timing not compared"))
+ continue
+ cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}
+ wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}
+ cpu_delta = cpu["head"] / cpu["base"] - 1
+ wall_delta = wall["head"] / wall["base"] - 1
+ slower_pairs = sum(
+ h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])
+ )
+ verdict = "ok"
+ if cpu_delta > CPU_LIMIT and slower_pairs * 4 >= args.pairs * 3:
+ verdict = f"SLOWER: CPU +{cpu_delta:.1%}, slower in {slower_pairs} of {args.pairs} pairs"
+ failed.append(name)
+ elif wall_delta > WALL_WARN:
+ verdict = f"wall +{wall_delta:.1%} (warning only)"
+ rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict))
+
+ lines = [
+ "## Perf budget",
+ "",
+ f"Median of {args.pairs} interleaved pairs, test process CPU time (user + system) and wall time.",
+ "",
+ "| Workload | CPU base | CPU head | Change | Wall base | Wall head | Change | Verdict |",
+ "| --- | --- | --- | --- | --- | --- | --- | --- |",
+ ]
+ for name, cb, ch, wb, wh, verdict in rows:
+ if cb is None:
+ lines.append(f"| {name} | | | | | | | {verdict} |")
+ else:
+ lines.append(
+ f"| {name} | {cb:.2f} s | {ch:.2f} s | {ch / cb - 1:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"
+ )
+ text = "\n".join(lines) + "\n"
+ print(text)
+ (out / "perf-budget.md").write_text(text)
+ if args.summary:
+ with open(args.summary, "a") as summary:
+ summary.write(text)
+ if failed:
+ print(f"::error::slower than the base build: {', '.join(failed)}")
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf_build.py
new file mode 100644
index 000000000..13b261738
--- /dev/null
+++ b/scripts/verify/perf_build.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Builds the engine suite's export, seek and playback test binaries of one
+worktree with the perf-verify profile and copies them out, so a second build
+in the same target directory cannot replace them.
+
+ perf_build.py WORKTREE OUT_DIR -> OUT_DIR/binaries.json
+"""
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+TARGETS = ["export", "seek", "playback"]
+# Release optimisation without link-time optimisation or single-unit codegen,
+# so two builds per pull request stay affordable. Passed as --config so a base
+# commit without this profile builds it too.
+PROFILE = [
+ "--config", 'profile.perf-verify.inherits="release"',
+ "--config", "profile.perf-verify.lto=false",
+ "--config", "profile.perf-verify.codegen-units=16",
+ "--config", "profile.perf-verify.incremental=false",
+ "--config", "profile.perf-verify.debug=0",
+]
+
+
+def main():
+ worktree, out = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()
+ out.mkdir(parents=True, exist_ok=True)
+ command = ["cargo", *PROFILE, "test", "--locked", "-p", "cap-verify", "--profile", "perf-verify", "--no-run", "--message-format", "json"]
+ for target in TARGETS:
+ command += ["--test", target]
+ result = subprocess.run(command, cwd=worktree, stdout=subprocess.PIPE, text=True, check=True)
+ binaries = {}
+ for line in result.stdout.splitlines():
+ try:
+ row = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ if row.get("reason") == "compiler-artifact" and row.get("executable") and row["target"]["name"] in TARGETS:
+ source = Path(row["executable"])
+ deps = out / "deps"
+ deps.mkdir(exist_ok=True)
+ dest = deps / source.name
+ shutil.copy2(source, dest)
+ binaries[row["target"]["name"]] = str(dest)
+ # macOS binaries load FFmpeg from @executable_path/../Frameworks,
+ # which scripts/setup.js fills for the debug and release profiles.
+ target_dir = source.parent.parent.parent
+ link = out / "Frameworks"
+ for profile in (source.parent.parent.name, "release", "debug"):
+ frameworks = target_dir / profile / "Frameworks"
+ if frameworks.exists() and not link.exists():
+ link.symlink_to(frameworks, target_is_directory=True)
+ break
+ missing = [t for t in TARGETS if t not in binaries]
+ if missing:
+ sys.exit(f"missing test binaries: {missing}")
+ (out / "binaries.json").write_text(json.dumps({"worktree": str(worktree), "binaries": binaries}, indent=2))
+ print(json.dumps(binaries, indent=2))
+
+
+if __name__ == "__main__":
+ main()