Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): base builds with the head's cargo config, compare every run's outcomes, kebab-case scripts

Richie McIlroy1a1845067bb6
diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml
index 63abf699d..57c4595fb 100644
--- a/.github/workflows/perf-budget.yml
+++ b/.github/workflows/perf-budget.yml
@@ -1,12 +1,8 @@
name: Perf budget
-# Builds the engine suite's export, seek and playback binaries for a pull
-# request and for its merge base on the same Blacksmith macOS machine, runs
-# each workload in interleaved pairs and fails when the pull request is
-# measurably slower (scripts/verify/perf_ab.py). Runs for ready (non-draft)
-# pull requests that change an engine crate or a high-risk region, like the
-# Engine verify FULL tier; every other pull request gets a green verdict
-# without running. The "Perf budget" job is the one to require.
+# Head and base are built and timed on one machine, interleaved, because
+# Blacksmith VMs have no hardware instruction counters and machine state
+# drifts between jobs. "Perf budget" is the job to require.
on:
pull_request:
@@ -53,7 +49,7 @@ jobs:
gh api "repos/$GITHUB_REPOSITORY/pulls/$PR/files" --paginate --jq '.[].filename' > "$RUNNER_TEMP/changed.txt"
python3 scripts/verify/plan.py risk --files "$RUNNER_TEMP/changed.txt" --output "$RUNNER_TEMP/risk.txt"
if grep -qx 'escalate=true' "$RUNNER_TEMP/risk.txt" || grep -qx 'engine=true' "$RUNNER_TEMP/risk.txt" \
- || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf_(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
+ || grep -qE '^(\.github/workflows/perf-budget\.yml|scripts/verify/perf-(ab|build)\.py)$' "$RUNNER_TEMP/changed.txt"; then
echo "run=true" >> "$GITHUB_OUTPUT"
else
echo "No engine or high-risk code changed: nothing to time." | tee -a "$GITHUB_STEP_SUMMARY"
@@ -98,18 +94,22 @@ jobs:
fi
echo "Comparing $(git rev-parse --short HEAD) with base $(git rev-parse --short "$base")" | tee -a "$GITHUB_STEP_SUMMARY"
git worktree add --detach "$RUNNER_TEMP/base" "$base"
- # The base builds with the head's native dependencies.
+ # The base builds against the head's native dependencies: the same
+ # target dir and the FFMPEG_DIR that scripts/setup.js writes to the
+ # (git-ignored) .cargo/config.toml.
ln -s "$GITHUB_WORKSPACE/target" "$RUNNER_TEMP/base/target"
+ mkdir -p "$RUNNER_TEMP/base/.cargo"
+ cp .cargo/config.toml "$RUNNER_TEMP/base/.cargo/config.toml"
- name: Build head
- run: python3 scripts/verify/perf_build.py . "$RUNNER_TEMP/perf/head"
+ run: python3 scripts/verify/perf-build.py . "$RUNNER_TEMP/perf/head"
- name: Build base
- run: python3 scripts/verify/perf_build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"
+ run: python3 scripts/verify/perf-build.py "$RUNNER_TEMP/base" "$RUNNER_TEMP/perf/base"
- name: Measure
run: |
- python3 scripts/verify/perf_ab.py \
+ python3 scripts/verify/perf-ab.py \
--base "$RUNNER_TEMP/perf/base/binaries.json" \
--head "$RUNNER_TEMP/perf/head/binaries.json" \
--fixtures "$RUNNER_TEMP/perf/fixtures" \
diff --git a/scripts/verify/__pycache__/perf-ab.cpython-314.pyc b/scripts/verify/__pycache__/perf-ab.cpython-314.pyc
new file mode 100644
index 000000000..6d5549359
Binary files /dev/null and b/scripts/verify/__pycache__/perf-ab.cpython-314.pyc differ
diff --git a/scripts/verify/__pycache__/perf-build.cpython-314.pyc b/scripts/verify/__pycache__/perf-build.cpython-314.pyc
new file mode 100644
index 000000000..8f5e41315
Binary files /dev/null and b/scripts/verify/__pycache__/perf-build.cpython-314.pyc differ
diff --git a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc
index 4615bd1af..672b6f1fd 100644
Binary files a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc and b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc differ
diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf-ab.py
similarity index 94%
rename from scripts/verify/perf_ab.py
rename to scripts/verify/perf-ab.py
index 452b54177..2d492060e 100644
--- a/scripts/verify/perf_ab.py
+++ b/scripts/verify/perf-ab.py
@@ -6,7 +6,7 @@ machine and compares the medians of the test process's CPU time (user +
system, from the kernel's rusage) and wall time. CPU time ignores scheduling
noise better than wall time; pairing cancels most of the machine's state.
- perf_ab.py --base base.json --head head.json --fixtures DIR --out DIR
+ perf-ab.py --base base.json --head head.json --fixtures DIR --out DIR
base.json / head.json map a test target ("export", "seek") to its binary and
the crate directory to run it from:
@@ -27,7 +27,6 @@ from pathlib import Path
# +0.8%, seek-storm +0.9%, export-jitter-camera-40s -4.4%. Real-time playback
# CPU varied by 15% there, so playback is reported without a limit.
WORKLOADS = [
- # name, test target, case glob, tier, CPU limit (None: report only)
("export-basic", "export", "export/basic/*", "fast", 0.05),
("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.08),
("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.05),
@@ -108,7 +107,7 @@ def main():
print("\n".join(tail))
return False
for name, target, case, tier, cpu_limit in WORKLOADS:
- # Untimed warm-up per side: generates fixtures and warms caches.
+ # The first run of a side generates fixtures and fills caches.
usable = True
for label, side in sides.items():
report = out / name / f"warm-{label}"
@@ -133,11 +132,11 @@ def main():
if name in broken:
rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
continue
- statuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}
- if statuses["base"][0] != statuses["head"][0]:
+ outcomes = [s["outcomes"] for runs in samples.values() for s in runs]
+ if any(o != outcomes[0] for o in outcomes):
# Correctness is the suite's job; a case that now passes or fails
# can change how much work runs, so its time is not comparable.
- rows.append((name, None, None, None, None, "case outcomes differ: timing not compared"))
+ rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared"))
continue
cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}
wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}
diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf-build.py
similarity index 97%
rename from scripts/verify/perf_build.py
rename to scripts/verify/perf-build.py
index 72702cb5a..e970415eb 100644
--- a/scripts/verify/perf_build.py
+++ b/scripts/verify/perf-build.py
@@ -3,7 +3,7 @@
worktree with the perf-verify profile and copies them out, so a second build
in the same target directory cannot replace them.
- perf_build.py WORKTREE OUT_DIR -> OUT_DIR/binaries.json
+ perf-build.py WORKTREE OUT_DIR -> OUT_DIR/binaries.json
"""
import json
import os