Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

ci(verify): fail the perf budget when a binary does not run, and find target/Frameworks

Richie McIlroyb1928a797d54
diff --git a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc
new file mode 100644
index 000000000..4615bd1af
Binary files /dev/null and b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc differ
diff --git a/scripts/verify/__pycache__/perf_build.cpython-314.pyc b/scripts/verify/__pycache__/perf_build.cpython-314.pyc
new file mode 100644
index 000000000..1d6c453fb
Binary files /dev/null and b/scripts/verify/__pycache__/perf_build.cpython-314.pyc differ
diff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf_ab.py
index 0aa1aeb5b..2458d2518 100644
--- a/scripts/verify/perf_ab.py
+++ b/scripts/verify/perf_ab.py
@@ -93,22 +93,44 @@ def main():
rows = []
failed = []
+ broken = []
+
+ def check(name, label, result, report):
+ # A test binary exits 0, or 101 when a case fails. Anything else (a
+ # signal, a missing library) means nothing was measured.
+ if result["exit"] in (0, 101) and result["outcomes"]:
+ return True
+ log = report / "test.log"
+ tail = log.read_text(errors="replace").splitlines()[-15:] if log.exists() else []
+ print(f"::error::{name} {label}: exit {result['exit']}, {len(result['outcomes'])} cases reported")
+ print("\n".join(tail))
+ return False
for name, target, case, tier in WORKLOADS:
# Untimed warm-up per side: generates fixtures and warms caches.
+ usable = True
for label, side in sides.items():
- run_once(side, target, case, tier, fixtures, out / name / f"warm-{label}")
+ report = out / name / f"warm-{label}"
+ usable &= check(name, label, run_once(side, target, case, tier, fixtures, report), report)
+ if not usable:
+ rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
+ broken.append(name)
+ continue
samples = {"base": [], "head": []}
for pair in range(args.pairs):
order = ("base", "head") if pair % 2 == 0 else ("head", "base")
for label in order:
- result = run_once(
- sides[label], target, case, tier, fixtures, out / name / f"{pair}-{label}"
- )
+ report = out / name / f"{pair}-{label}"
+ result = run_once(sides[label], target, case, tier, fixtures, report)
+ if not check(name, label, result, report):
+ broken.append(name)
samples[label].append(result)
print(
f"{name} pair {pair} {label}: cpu {result['cpu']:.2f}s wall {result['wall']:.2f}s exit {result['exit']}",
flush=True,
)
+ if name in broken:
+ rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))
+ continue
statuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}
if statuses["base"][0] != statuses["head"][0]:
# Correctness is the suite's job; a case that now passes or fails
@@ -151,10 +173,11 @@ def main():
if args.summary:
with open(args.summary, "a") as summary:
summary.write(text)
+ if broken:
+ print(f"::error::not measured: {', '.join(sorted(set(broken)))}")
if failed:
print(f"::error::slower than the base build: {', '.join(failed)}")
- return 1
- return 0
+ return 1 if broken or failed else 0
if __name__ == "__main__":
diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf_build.py
index 13b261738..72702cb5a 100644
--- a/scripts/verify/perf_build.py
+++ b/scripts/verify/perf_build.py
@@ -45,18 +45,22 @@ def main():
dest = deps / source.name
shutil.copy2(source, dest)
binaries[row["target"]["name"]] = str(dest)
- # macOS binaries load FFmpeg from @executable_path/../Frameworks,
- # which scripts/setup.js fills for the debug and release profiles.
+ # macOS binaries load FFmpeg from @executable_path/../Frameworks.
+ # scripts/setup.js puts the framework in target/Frameworks; local
+ # builds may also have one per profile.
target_dir = source.parent.parent.parent
link = out / "Frameworks"
- for profile in (source.parent.parent.name, "release", "debug"):
- frameworks = target_dir / profile / "Frameworks"
+ candidates = [target_dir / source.parent.parent.name / "Frameworks", target_dir / "Frameworks"]
+ candidates += [target_dir / profile / "Frameworks" for profile in ("release", "debug")]
+ for frameworks in candidates:
if frameworks.exists() and not link.exists():
link.symlink_to(frameworks, target_is_directory=True)
break
missing = [t for t in TARGETS if t not in binaries]
if missing:
sys.exit(f"missing test binaries: {missing}")
+ if sys.platform == "darwin" and not (out / "Frameworks").exists():
+ sys.exit("no FFmpeg Frameworks directory next to the target dir: run scripts/setup.js")
(out / "binaries.json").write_text(json.dumps({"worktree": str(worktree), "binaries": binaries}, indent=2))
print(json.dumps(binaries, indent=2))