Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): per-workload perf limits from the Blacksmith noise floor, playback report-only
Richie McIlroyb5ba81549ae9
diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf_ab.pyindex 2458d2518..452b54177 100644--- a/scripts/verify/perf_ab.py+++ b/scripts/verify/perf_ab.py@@ -21,17 +21,19 @@ import sysimport timefrom pathlib import Path+# A head build is slower when its median CPU time exceeds the base's by more+# than the workload's limit, and in most pairs. Limits sit above the noise of+# identical code on blacksmith-6vcpu-macos-26 (run 38995088761): export-basic+# +0.8%, seek-storm +0.9%, export-jitter-camera-40s -4.4%. Real-time playback+# CPU varied by 15% there, so playback is reported without a limit.WORKLOADS = [- # name, test target, case glob, tier- ("export-basic", "export", "export/basic/*", "fast"),- ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast"),- ("seek-storm", "seek", "seek/*/storm30/*", "fast"),- ("playback-basic", "playback", "playback/basic/*", "fast"),+ # name, test target, case glob, tier, CPU limit (None: report only)+ ("export-basic", "export", "export/basic/*", "fast", 0.05),+ ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.08),+ ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.05),+ ("playback-basic", "playback", "playback/basic/*", "fast", None),]-# A head build is slower only when its median CPU time exceeds the base's by-# more than this, and in most pairs.-CPU_LIMIT = 0.05WALL_WARN = 0.08@@ -105,7 +107,7 @@ def main():print(f"::error::{name} {label}: exit {result['exit']}, {len(result['outcomes'])} cases reported")print("\n".join(tail))return False- for name, target, case, tier in WORKLOADS:+ for name, target, case, tier, cpu_limit in WORKLOADS:# Untimed warm-up per side: generates fixtures and warms caches.usable = Truefor label, side in sides.items():@@ -144,8 +146,8 @@ def main():slower_pairs = sum(h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"]))- verdict = "ok"- if cpu_delta > CPU_LIMIT and slower_pairs * 4 >= args.pairs * 3:+ verdict = "ok" if cpu_limit is not None else "reported only"+ if cpu_limit is not None and cpu_delta > cpu_limit and slower_pairs * 4 >= args.pairs * 3:verdict = f"SLOWER: CPU +{cpu_delta:.1%}, slower in {slower_pairs} of {args.pairs} pairs"failed.append(name)elif wall_delta > WALL_WARN: