Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): fail pull requests that make export, seek or playback slower
Richie McIlroyeaed5bc44e12
diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.ymlnew file mode 100644index 000000000..63abf699d--- /dev/null+++ b/.github/workflows/perf-budget.yml@@ -0,0 +1,139 @@+name: Perf budget++# Builds the engine suite's export, seek and playback binaries for a pull+# request and for its merge base on the same Blacksmith macOS machine, runs+# each workload in interleaved pairs and fails when the pull request is+# measurably slower (scripts/verify/perf_ab.py). Runs for ready (non-draft)+# pull requests that change an engine crate or a high-risk region, like the+# Engine verify FULL tier; every other pull request gets a green verdict+# without running. The "Perf budget" job is the one to require.++on:+ pull_request:+ types: [opened, synchronize, reopened, ready_for_review]+ workflow_dispatch:+ inputs:+ base:+ description: "Ref to compare against (default: the merge base with main)"+ default: ""++concurrency:+ group: perf-budget-${{ github.event.pull_request.number || github.ref }}+ cancel-in-progress: true++permissions:+ contents: read++jobs:+ scope:+ runs-on: blacksmith-2vcpu-ubuntu-2404+ outputs:+ run: ${{ steps.scope.outputs.run }}+ steps:+ - uses: actions/checkout@v4+ with:+ persist-credentials: false+ - name: Scope+ id: scope+ env:+ GH_TOKEN: ${{ github.token }}+ EVENT: ${{ github.event_name }}+ PR: ${{ github.event.pull_request.number }}+ DRAFT: ${{ github.event.pull_request.draft }}+ RUSTUP_TOOLCHAIN: stable+ run: |+ set -euo pipefail+ if [[ "$EVENT" == workflow_dispatch ]]; then+ echo "run=true" >> "$GITHUB_OUTPUT"; exit 0+ fi+ if [[ "$DRAFT" == true ]]; then+ echo "Draft: the perf budget runs once the pull request is ready." | tee -a "$GITHUB_STEP_SUMMARY"+ echo "run=false" >> "$GITHUB_OUTPUT"; exit 0+ fi+ gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"+ python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"+ if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" \+ || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf_(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then+ echo "run=true" >> "$GITHUB_OUTPUT"+ else+ echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"+ echo "run=false" >> "$GITHUB_OUTPUT"+ fi++ measure:+ needs: scope+ if: needs.scope.outputs.run == 'true'+ runs-on: blacksmith-6vcpu-macos-26+ timeout-minutes: 120+ steps:+ - uses: actions/checkout@v4+ with:+ persist-credentials: false+ fetch-depth: 0++ - name: Setup+ uses: ./.github/actions/engine-verify-setup+ with:+ target: aarch64-apple-darwin+ tier: fast++ - name: Perf build cache+ uses: actions/cache@v4+ with:+ path: target/perf-verify/+ key: perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-${{ github.run_id }}+ restore-keys: |+ perf-verify-v1-${{ hashFiles('Cargo.lock', 'rust-toolchain.toml') }}-+ perf-verify-v1-++ - name: Base worktree+ env:+ BASE_INPUT: ${{ inputs.base }}+ PR_BASE: ${{ github.event.pull_request.base.sha }}+ run: |+ set -euo pipefail+ base="${BASE_INPUT:-}"+ if [[ -z "$base" ]]; then+ base="$(git merge-base HEAD "${PR_BASE:-origin/main}")"+ fi+ echo "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY"+ git worktree add --detach "$RUNNER_TEMP/base" "$base"+ # The base builds with the head's native dependencies.+ ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target"++ - name: Build head+ run: python3 scripts/verify/perf_build.py . "$RUNNER_TEMP/perf/head"++ - name: Build base+ run: python3 scripts/verify/perf_build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"++ - name: Measure+ run: |+ python3 scripts/verify/perf_ab.py \+ --base "$RUNNER_TEMP/perf/base/binaries.json" \+ --head "$RUNNER_TEMP/perf/head/binaries.json" \+ --fixtures "$RUNNER_TEMP/perf/fixtures" \+ --out "$RUNNER_TEMP/perf/runs" \+ --summary "$GITHUB_STEP_SUMMARY"++ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ if: always()+ with:+ name: perf-budget+ path: ${{ runner.temp }}/perf/runs/**/perf-budget.md+ if-no-files-found: ignore++ verdict:+ name: Perf budget+ needs: [scope, measure]+ if: always()+ runs-on: blacksmith-2vcpu-ubuntu-2404+ steps:+ - env:+ RUN: ${{ needs.scope.outputs.run }}+ SCOPE: ${{ needs.scope.result }}+ MEASURE: ${{ needs.measure.result }}+ run: |+ if [[ "$SCOPE" != success ]]; then echo "scope failed"; exit 1; fi+ if [[ "$RUN" != true ]]; then echo "not measured"; exit 0; fi+ [[ "$MEASURE" == success ]]diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf_ab.pynew file mode 100644index 000000000..0aa1aeb5b--- /dev/null+++ b/scripts/verify/perf_ab.py@@ -0,0 +1,161 @@+#!/usr/bin/env python3+"""Paired performance comparison of two builds of the engine suite binaries.++Runs each workload alternately with the base and head binaries on the same+machine and compares the medians of the test process's CPU time (user ++system, from the kernel's rusage) and wall time. CPU time ignores scheduling+noise better than wall time; pairing cancels most of the machine's state.++ perf_ab.py --base base.json --head head.json --fixtures DIR --out DIR++base.json / head.json map a test target ("export", "seek") to its binary and+the crate directory to run it from:+ {"worktree": "...", "binaries": {"export": "...", "seek": "..."}}+"""+import argparse+import json+import os+import statistics+import subprocess+import sys+import time+from pathlib import Path++WORKLOADS = [+ # name, test target, case glob, tier+ ("export-basic", "export", "export/basic/*", "fast"),+ ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast"),+ ("seek-storm", "seek", "seek/*/storm30/*", "fast"),+ ("playback-basic", "playback", "playback/basic/*", "fast"),+]++# A head build is slower only when its median CPU time exceeds the base's by+# more than this, and in most pairs.+CPU_LIMIT = 0.05+WALL_WARN = 0.08+++def run_once(side, target, case, tier, fixtures, report):+ binary = side["binaries"][target]+ env = dict(+ os.environ,+ CAP_VERIFY_FIXTURE_DIR=str(fixtures),+ CAP_VERIFY_REPORT_DIR=str(report),+ CAP_VERIFY_CASE=case,+ CAP_VERIFY_TIER=tier,+ )+ report.mkdir(parents=True, exist_ok=True)+ started = time.monotonic()+ with (report / "test.log").open("w") as log:+ process = subprocess.Popen(+ [binary, "--test-threads", "1", "--nocapture"],+ cwd=Path(side["worktree"]) / "crates/verify",+ env=env,+ stdout=log,+ stderr=subprocess.STDOUT,+ )+ _, status, usage = os.wait4(process.pid, 0)+ wall = time.monotonic() - started+ outcomes = {}+ for path in report.glob("cap-verify-*.jsonl"):+ for line in path.read_text().splitlines():+ try:+ row = json.loads(line)+ except json.JSONDecodeError:+ continue+ if row.get("case"):+ outcomes[row["case"]] = row.get("outcome")+ return {+ "cpu": usage.ru_utime + usage.ru_stime,+ "wall": wall,+ "exit": os.waitstatus_to_exitcode(status),+ "outcomes": outcomes,+ }+++def main():+ parser = argparse.ArgumentParser()+ parser.add_argument("--base", required=True)+ parser.add_argument("--head", required=True)+ parser.add_argument("--fixtures", required=True)+ parser.add_argument("--out", required=True)+ parser.add_argument("--pairs", type=int, default=3)+ parser.add_argument("--summary")+ args = parser.parse_args()++ sides = {+ "base": json.loads(Path(args.base).read_text()),+ "head": json.loads(Path(args.head).read_text()),+ }+ fixtures = Path(args.fixtures)+ out = Path(args.out)+ out.mkdir(parents=True, exist_ok=True)++ rows = []+ failed = []+ for name, target, case, tier in WORKLOADS:+ # Untimed warm-up per side: generates fixtures and warms caches.+ for label, side in sides.items():+ run_once(side, target, case, tier, fixtures, out / name / f"warm-{label}")+ samples = {"base": [], "head": []}+ for pair in range(args.pairs):+ order = ("base", "head") if pair % 2 == 0 else ("head", "base")+ for label in order:+ result = run_once(+ sides[label], target, case, tier, fixtures, out / name / f"{pair}-{label}"+ )+ samples[label].append(result)+ print(+ f"{name} pair {pair} {label}: cpu {result['cpu']:.2f}s wall {result['wall']:.2f}s exit {result['exit']}",+ flush=True,+ )+ statuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}+ if statuses["base"][0] != statuses["head"][0]:+ # Correctness is the suite's job; a case that now passes or fails+ # can change how much work runs, so its time is not comparable.+ rows.append((name, None, None, None, None, "case outcomes differ: timing not compared"))+ continue+ cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}+ wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}+ cpu_delta = cpu["head"] / cpu["base"] - 1+ wall_delta = wall["head"] / wall["base"] - 1+ slower_pairs = sum(+ h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])+ )+ verdict = "ok"+ if cpu_delta > CPU_LIMIT and slower_pairs * 4 >= args.pairs * 3:+ verdict = f"SLOWER: CPU +{cpu_delta:.1%}, slower in {slower_pairs} of {args.pairs} pairs"+ failed.append(name)+ elif wall_delta > WALL_WARN:+ verdict = f"wall +{wall_delta:.1%} (warning only)"+ rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict))++ lines = [+ "## Perf budget",+ "",+ f"Median of {args.pairs} interleaved pairs, test process CPU time (user + system) and wall time.",+ "",+ "| Workload | CPU base | CPU head | Change | Wall base | Wall head | Change | Verdict |",+ "| --- | --- | --- | --- | --- | --- | --- | --- |",+ ]+ for name, cb, ch, wb, wh, verdict in rows:+ if cb is None:+ lines.append(f"| {name} | | | | | | | {verdict} |")+ else:+ lines.append(+ f"| {name} | {cb:.2f} s | {ch:.2f} s | {ch / cb - 1:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"+ )+ text = "\n".join(lines) + "\n"+ print(text)+ (out / "perf-budget.md").write_text(text)+ if args.summary:+ with open(args.summary, "a") as summary:+ summary.write(text)+ if failed:+ print(f"::error::slower than the base build: {', '.join(failed)}")+ return 1+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf_build.pynew file mode 100644index 000000000..13b261738--- /dev/null+++ b/scripts/verify/perf_build.py@@ -0,0 +1,65 @@+#!/usr/bin/env python3+"""Builds the engine suite's export, seek and playback test binaries of one+worktree with the perf-verify profile and copies them out, so a second build+in the same target directory cannot replace them.++ perf_build.py WORKTREE OUT_DIR -> OUT_DIR/binaries.json+"""+import json+import os+import shutil+import subprocess+import sys+from pathlib import Path++TARGETS = ["export", "seek", "playback"]+# Release optimisation without link-time optimisation or single-unit codegen,+# so two builds per pull request stay affordable. Passed as --config so a base+# commit without this profile builds it too.+PROFILE = [+ "--config", 'profile.perf-verify.inherits="release"',+ "--config", "profile.perf-verify.lto=false",+ "--config", "profile.perf-verify.codegen-units=16",+ "--config", "profile.perf-verify.incremental=false",+ "--config", "profile.perf-verify.debug=0",+]+++def main():+ worktree, out = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()+ out.mkdir(parents=True, exist_ok=True)+ command = ["cargo", *PROFILE, "test", "--locked", "-p", "cap-verify", "--profile", "perf-verify", "--no-run", "--message-format", "json"]+ for target in TARGETS:+ command += ["--test", target]+ result = subprocess.run(command, cwd=worktree, stdout=subprocess.PIPE, text=True, check=True)+ binaries = {}+ for line in result.stdout.splitlines():+ try:+ row = json.loads(line)+ except json.JSONDecodeError:+ continue+ if row.get("reason") == "compiler-artifact" and row.get("executable") and row["target"]["name"] in TARGETS:+ source = Path(row["executable"])+ deps = out / "deps"+ deps.mkdir(exist_ok=True)+ dest = deps / source.name+ shutil.copy2(source, dest)+ binaries[row["target"]["name"]] = str(dest)+ # macOS binaries load FFmpeg from @executable_path/../Frameworks,+ # which scripts/setup.js fills for the debug and release profiles.+ target_dir = source.parent.parent.parent+ link = out / "Frameworks"+ for profile in (source.parent.parent.name, "release", "debug"):+ frameworks = target_dir / profile / "Frameworks"+ if frameworks.exists() and not link.exists():+ link.symlink_to(frameworks, target_is_directory=True)+ break+ missing = [t for t in TARGETS if t not in binaries]+ if missing:+ sys.exit(f"missing test binaries: {missing}")+ (out / "binaries.json").write_text(json.dumps({"worktree": str(worktree), "binaries": binaries}, indent=2))+ print(json.dumps(binaries, indent=2))+++if __name__ == "__main__":+ main()