Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
ci(verify): fail the perf budget when a binary does not run, and find target/Frameworks
Richie McIlroyb1928a797d54
diff --git a/scripts/verify/__pycache__/perf_ab.cpython-314.pyc b/scripts/verify/__pycache__/perf_ab.cpython-314.pycnew file mode 100644index 000000000..4615bd1afBinary files /dev/null and b/scripts/verify/__pycache__/perf_ab.cpython-314.pyc differdiff --git a/scripts/verify/__pycache__/perf_build.cpython-314.pyc b/scripts/verify/__pycache__/perf_build.cpython-314.pycnew file mode 100644index 000000000..1d6c453fbBinary files /dev/null and b/scripts/verify/__pycache__/perf_build.cpython-314.pyc differdiff --git a/scripts/verify/perf_ab.py b/scripts/verify/perf_ab.pyindex 0aa1aeb5b..2458d2518 100644--- a/scripts/verify/perf_ab.py+++ b/scripts/verify/perf_ab.py@@ -93,22 +93,44 @@ def main():rows = []failed = []+ broken = []++ def check(name, label, result, report):+ # A test binary exits 0, or 101 when a case fails. Anything else (a+ # signal, a missing library) means nothing was measured.+ if result["exit"] in (0, 101) and result["outcomes"]:+ return True+ log = report / "test.log"+ tail = log.read_text(errors="replace").splitlines()[-15:] if log.exists() else []+ print(f"::error::{name} {label}: exit {result['exit']}, {len(result['outcomes'])} cases reported")+ print("\n".join(tail))+ return Falsefor name, target, case, tier in WORKLOADS:# Untimed warm-up per side: generates fixtures and warms caches.+ usable = Truefor label, side in sides.items():- run_once(side, target, case, tier, fixtures, out / name / f"warm-{label}")+ report = out / name / f"warm-{label}"+ usable &= check(name, label, run_once(side, target, case, tier, fixtures, report), report)+ if not usable:+ rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))+ broken.append(name)+ continuesamples = {"base": [], "head": []}for pair in range(args.pairs):order = ("base", "head") if pair % 2 == 0 else ("head", "base")for label in order:- result = run_once(- sides[label], target, case, tier, fixtures, out / name / f"{pair}-{label}"- )+ report = out / name / f"{pair}-{label}"+ result = run_once(sides[label], target, case, tier, fixtures, report)+ if not check(name, label, result, report):+ broken.append(name)samples[label].append(result)print(f"{name} pair {pair} {label}: cpu {result['cpu']:.2f}s wall {result['wall']:.2f}s exit {result['exit']}",flush=True,)+ if name in broken:+ rows.append((name, None, None, None, None, "BROKEN: a test binary did not run its cases"))+ continuestatuses = {label: [s["outcomes"] for s in runs] for label, runs in samples.items()}if statuses["base"][0] != statuses["head"][0]:# Correctness is the suite's job; a case that now passes or fails@@ -151,10 +173,11 @@ def main():if args.summary:with open(args.summary, "a") as summary:summary.write(text)+ if broken:+ print(f"::error::not measured: {', '.join(sorted(set(broken)))}")if failed:print(f"::error::slower than the base build: {', '.join(failed)}")- return 1- return 0+ return 1 if broken or failed else 0if __name__ == "__main__":diff --git a/scripts/verify/perf_build.py b/scripts/verify/perf_build.pyindex 13b261738..72702cb5a 100644--- a/scripts/verify/perf_build.py+++ b/scripts/verify/perf_build.py@@ -45,18 +45,22 @@ def main():dest = deps / source.nameshutil.copy2(source, dest)binaries[row["target"]["name"]] = str(dest)- # macOS binaries load FFmpeg from @executable_path/../Frameworks,- # which scripts/setup.js fills for the debug and release profiles.+ # macOS binaries load FFmpeg from @executable_path/../Frameworks.+ # scripts/setup.js puts the framework in target/Frameworks; local+ # builds may also have one per profile.target_dir = source.parent.parent.parentlink = out / "Frameworks"- for profile in (source.parent.parent.name, "release", "debug"):- frameworks = target_dir / profile / "Frameworks"+ candidates = [target_dir / source.parent.parent.name / "Frameworks", target_dir / "Frameworks"]+ candidates += [target_dir / profile / "Frameworks" for profile in ("release", "debug")]+ for frameworks in candidates:if frameworks.exists() and not link.exists():link.symlink_to(frameworks, target_is_directory=True)breakmissing = [t for t in TARGETS if t not in binaries]if missing:sys.exit(f"missing test binaries: {missing}")+ if sys.platform == "darwin" and not (out / "Frameworks").exists():+ sys.exit("no FFmpeg Frameworks directory next to the target dir: run scripts/setup.js")(out / "binaries.json").write_text(json.dumps({"worktree": str(worktree), "binaries": binaries}, indent=2))print(json.dumps(binaries, indent=2))