Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): judge the perf budget on per-pair CPU ratios, 10% limits where Blacksmith noise reaches 5%

Richie McIlroydfad1c164c1b
diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.py
index 2d492060e..6b5b1bfb9 100644
--- a/scripts/verify/perf-ab.py
+++ b/scripts/verify/perf-ab.py
@@ -21,15 +21,18 @@ import sys
import time
from pathlib import Path
-# A head build is slower when its median CPU time exceeds the base's by more
-# than the workload's limit, and in most pairs. Limits sit above the noise of
-# identical code on blacksmith-6vcpu-macos-26 (run 38995088761): export-basic
-# +0.8%, seek-storm +0.9%, export-jitter-camera-40s -4.4%. Real-time playback
-# CPU varied by 15% there, so playback is reported without a limit.
+# A head build is slower when the median of its per-pair CPU ratios to the
+# base exceeds the workload's limit and it is slower in most pairs. Adjacent
+# runs of a pair share the machine's state; across a job a Blacksmith VM's
+# speed drifts (one seek-storm job ran 10.1 to 13.2 s per run for the same
+# code). Limits sit above the median-ratio noise of identical code on
+# blacksmith-6vcpu-macos-26 (runs 38995088761, 38998485953): export-basic
+# 1.1%, export-jitter-camera-40s 4.4%, seek-storm 5.0%. Real-time playback's
+# CPU varied by 15%, so it is reported without a limit.
WORKLOADS = [
("export-basic", "export", "export/basic/*", "fast", 0.05),
- ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.08),
- ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.05),
+ ("export-jitter-camera-40s", "export", "export/jitter-camera-40s/*", "fast", 0.10),
+ ("seek-storm", "seek", "seek/*/storm30/*", "fast", 0.10),
("playback-basic", "playback", "playback/basic/*", "fast", None),
]
@@ -113,7 +116,7 @@ def main():
report = out / name / f"warm-{label}"
usable &= check(name, label, run_once(side, target, case, tier, fixtures, report), report)
if not usable:
- rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
+ rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None))
broken.append(name)
continue
samples = {"base": [], "head": []}
@@ -130,17 +133,19 @@ def main():
flush=True,
)
if name in broken:
- rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
+ rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases", None))
continue
outcomes = [s["outcomes"] for runs in samples.values() for s in runs]
if any(o != outcomes[0] for o in outcomes):
# Correctness is the suite's job; a case that now passes or fails
# can change how much work runs, so its time is not comparable.
- rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared"))
+ rows.append((name, None, None, None, None, "case outcomes differ between runs: timing not compared", None))
continue
cpu = {label: statistics.median(s["cpu"] for s in runs) for label, runs in samples.items()}
wall = {label: statistics.median(s["wall"] for s in runs) for label, runs in samples.items()}
- cpu_delta = cpu["head"] / cpu["base"] - 1
+ cpu_delta = statistics.median(
+ h["cpu"] / b["cpu"] for b, h in zip(samples["base"], samples["head"])
+ ) - 1
wall_delta = wall["head"] / wall["base"] - 1
slower_pairs = sum(
h["cpu"] > b["cpu"] for b, h in zip(samples["base"], samples["head"])
@@ -151,22 +156,22 @@ def main():
failed.append(name)
elif wall_delta > WALL_WARN:
verdict = f"wall +{wall_delta:.1%} (warning only)"
- rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict))
+ rows.append((name, cpu["base"], cpu["head"], wall["base"], wall["head"], verdict, cpu_delta))
lines = [
"## Perf budget",
"",
- f"Median of {args.pairs} interleaved pairs, test process CPU time (user + system) and wall time.",
+ f"{args.pairs} interleaved pairs: median test process CPU time (user + system) and wall time per side, and the median of the per-pair CPU ratios.",
"",
- "| Workload | CPU base | CPU head | Change | Wall base | Wall head | Change | Verdict |",
+ "| Workload | CPU base | CPU head | CPU change (median of pairs) | Wall base | Wall head | Wall change | Verdict |",
"| --- | --- | --- | --- | --- | --- | --- | --- |",
]
- for name, cb, ch, wb, wh, verdict in rows:
+ for name, cb, ch, wb, wh, verdict, cpu_delta in rows:
if cb is None:
lines.append(f"| {name} | | | | | | | {verdict} |")
else:
lines.append(
- f"| {name} | {cb:.2f} s | {ch:.2f} s | {ch / cb - 1:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"
+ f"| {name} | {cb:.2f} s | {ch:.2f} s | {cpu_delta:+.1%} | {wb:.2f} s | {wh:.2f} s | {wh / wb - 1:+.1%} | {verdict} |"
)
text = "\n".join(lines) + "\n"
print(text)