Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): judge the perf budget on per-pair CPU ratios, 10% limits where Blacksmith noise reaches 5%
Richie McIlroydfad1c164c1b
diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.pyindex 2d492060e..6b5b1bfb9 100644--- a/scripts/verify/perf-ab.py+++ b/scripts/verify/perf-ab.py@@ -21,15 +21,18 @@ import sysimport timefrom pathlib import Path-# A head build is slower when its median CPU time exceeds the base's by more-# than the workload's limit, and in most pairs. Limits sit above the noise of-# identical code on blacksmith-6vcpu-macos-26 (run 38995088761): export-basic-# +0.8%, seek-storm +0.9%, export-jitter-camera-40s -4.4%. Real-time playback-# CPU varied by 15% there, so playback is reported without a limit.+# A head build is slower when the median of its per-pair CPU ratios to the+# base exceeds the workload's limit and it is slower in most pairs. Adjacent+# runs of a pair share the machine's state; across a job a Blacksmith VM's+# speed drifts (one seek-storm job ran 10.1 to 13.2 s per run for the same+# code). Limits sit above the median-ratio noise of identical code on+# blacksmith-6vcpu-macos-26 (runs 38995088761, 38998485953): export-basic+# 1.1%, export-jitter-camera-40s 4.4%, seek-storm 5.0%. Real-time playback's+# CPU varied by 15%, so it is reported without a limit.WORKLOADS = [("export-basic", "export", "export/basic/*", "fast", 0.05),- ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.08),- ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.05),+ ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.10),+ ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.10),("playback-basic", "playback", "playback/basic/*", "fast", None),]@@ -113,7 +116,7 @@ def main():report = out / name / f"warm-{label}"usable &= check(name, label, run_once(side, target, case, tier, fixtures, report), report)if not usable:- rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))+ rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None))broken.append(name)continuesamples = {"base": [], "head": []}@@ -130,17 +133,19 @@ def main():flush=True,)if name in broken:- rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))+ rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None))continueoutcomes = [s["outcomes"] for runs in samples.values() for s in runs]if any(o != outcomes[0] for o in outcomes):# Correctness is the suite's job; a case that now passes or fails# can change how much work runs, so its time is not comparable.- rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared"))+ rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared", None))continuecpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}- cpu_delta = cpu["head"] / cpu["base"] - 1+ cpu_delta = statistics.median(+ h["cpu"] / b["cpu"] for b, h in zip(samples["base"], samples["head"])+ ) - 1wall_delta = wall["head"] / wall["base"] - 1slower_pairs = sum(h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])@@ -151,22 +156,22 @@ def main():failed.append(name)elif wall_delta > WALL_WARN:verdict = f"wall +{wall_delta:.1%} (warning only)"- rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict))+ rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict, cpu_delta))lines = ["## Perf budget","",- f"Median of {args.pairs} interleaved pairs, test process CPU time (user + system) and wall time.",+ f"{args.pairs} interleaved pairs: median test process CPU time (user + system) and wall time per side, and the median of the per-pair CPU ratios.","",- "| Workload | CPU base | CPU head | Change | Wall base | Wall head | Change | Verdict |",+ "| Workload | CPU base | CPU head | CPU change (median of pairs) | Wall base | Wall head | Wall change | Verdict |","| --- | --- | --- | --- | --- | --- | --- | --- |",]- for name, cb, ch, wb, wh, verdict in rows:+ for name, cb, ch, wb, wh, verdict, cpu_delta in rows:if cb is None:lines.append(f"| {name} | | | | | | | {verdict} |")else:lines.append(- f"| {name} | {cb:.2f} s | {ch:.2f} s | {ch / cb - 1:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"+ f"| {name} | {cb:.2f} s | {ch:.2f} s | {cpu_delta:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |")text = "\n".join(lines) + "\n"print(text)