Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): base builds with the head's cargo config, compare every run's outcomes, kebab-case scripts
Richie McIlroy1a1845067bb6
diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.ymlindex 63abf699d..57c4595fb 100644--- a/.github/workflows/perf-budget.yml+++ b/.github/workflows/perf-budget.yml@@ -1,12 +1,8 @@name: Perf budget-# Builds the engine suite's export, seek and playback binaries for a pull-# request and for its merge base on the same Blacksmith macOS machine, runs-# each workload in interleaved pairs and fails when the pull request is-# measurably slower (scripts/verify/perf_ab.py). Runs for ready (non-draft)-# pull requests that change an engine crate or a high-risk region, like the-# Engine verify FULL tier; every other pull request gets a green verdict-# without running. The "Perf budget" job is the one to require.+# Head and base are built and timed on one machine, interleaved, because+# Blacksmith VMs have no hardware instruction counters and machine state+# drifts between jobs. "Perf budget" is the job to require.on:pull_request:@@ -53,7 +49,7 @@ jobs:gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" \- || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf_(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then+ || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf-(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; thenecho "run=true" >> "$GITHUB_OUTPUT"elseecho "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"@@ -98,18 +94,22 @@ jobs:fiecho "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY"git worktree add --detach "$RUNNER_TEMP/base" "$base"- # The base builds with the head's native dependencies.+ # The base builds against the head's native dependencies: the same+ # target dir and the FFMPEG_DIR that scripts/setup.js writes to the+ # (git-ignored) .cargo/config.toml.ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target"+ mkdir -p "$RUNNER_TEMP/base/.cargo"+ cp .cargo/config.toml "$RUNNER_TEMP/base/.cargo/config.toml"- name: Build head- run: python3 scripts/verify/perf_build.py . "$RUNNER_TEMP/perf/head"+ run: python3 scripts/verify/perf-build.py . "$RUNNER_TEMP/perf/head"- name: Build base- run: python3 scripts/verify/perf_build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"+ run: python3 scripts/verify/perf-build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"- name: Measurerun: |- python3 scripts/verify/perf_ab.py \+ python3 scripts/verify/perf-ab.py \--base "$RUNNER_TEMP/perf/base/binaries.json" \--head "$RUNNER_TEMP/perf/head/binaries.json" \--fixtures "$RUNNER_TEMP/perf/fixtures" \diff --git a/scripts/verify/__pycache__/perf-ab.cpython-314.pyc b/scripts/verify/__pycache__/perf-ab.cpython-314.pycnew file mode 100644index 000000000..6d5549359Binary files /dev/null and b/scripts/verify/__pycache__/perf-ab.cpython-314.pyc differdiff --git a/scripts/verify/__pycache__/perf-build.cpython-314.pyc b/scripts/verify/__pycache__/perf-build.cpython-314.pycnew file mode 100644index 000000000..8f5e41315Binary files /dev/null and b/scripts/verify/__pycache__/perf-build.cpython-314.pyc differdiff --git a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc b/scripts/verify/__pycache__/perf_ab.cpython-314.pycindex 4615bd1af..672b6f1fd 100644Binary files a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc and b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc differdiff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf-ab.pysimilarity index 94%rename from scripts/verify/perf_ab.pyrename to scripts/verify/perf-ab.pyindex 452b54177..2d492060e 100644--- a/scripts/verify/perf_ab.py+++ b/scripts/verify/perf-ab.py@@ -6,7 +6,7 @@ machine and compares the medians of the test process's CPU time (user +system, from the kernel's rusage) and wall time. CPU time ignores schedulingnoise better than wall time; pairing cancels most of the machine's state.- perf_ab.py --base base.json --head head.json --fixtures DIR --out DIR+ perf-ab.py --base base.json --head head.json --fixtures DIR --out DIRbase.json / head.json map a test target ("export", "seek") to its binary andthe crate directory to run it from:@@ -27,7 +27,6 @@ from pathlib import Path# +0.8%, seek-storm +0.9%, export-jitter-camera-40s -4.4%. Real-time playback# CPU varied by 15% there, so playback is reported without a limit.WORKLOADS = [- # name, test target, case glob, tier, CPU limit (None: report only)("export-basic", "export", "export/basic/*", "fast", 0.05),("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.08),("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.05),@@ -108,7 +107,7 @@ def main():print("\n".join(tail))return Falsefor name, target, case, tier, cpu_limit in WORKLOADS:- # Untimed warm-up per side: generates fixtures and warms caches.+ # The first run of a side generates fixtures and fills caches.usable = Truefor label, side in sides.items():report = out / name / f"warm-{label}"@@ -133,11 +132,11 @@ def main():if name in broken:rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))continue- statuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}- if statuses["base"][0] != statuses["head"][0]:+ outcomes = [s["outcomes"] for runs in samples.values() for s in runs]+ if any(o != outcomes[0] for o in outcomes):# Correctness is the suite's job; a case that now passes or fails# can change how much work runs, so its time is not comparable.- rows.append((name, None, None, None, None, "case outcomes differ: timing not compared"))+ rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared"))continuecpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf-build.pysimilarity index 97%rename from scripts/verify/perf_build.pyrename to scripts/verify/perf-build.pyindex 72702cb5a..e970415eb 100644--- a/scripts/verify/perf_build.py+++ b/scripts/verify/perf-build.py@@ -3,7 +3,7 @@worktree with the perf-verify profile and copies them out, so a second buildin the same target directory cannot replace them.- perf_build.py WORKTREE OUT_DIR -> OUT_DIR/binaries.json+ perf-build.py WORKTREE OUT_DIR -> OUT_DIR/binaries.json"""import jsonimport os