Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(ci): rebuild local crates for trustworthy performance comparisons
Richie McIlroy33d9f4a4b457
diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.ymlindex 07a176dc7..806f8b617 100644--- a/.github/workflows/engine-verify.yml+++ b/.github/workflows/engine-verify.yml@@ -88,6 +88,7 @@ jobs:run: |python3 scripts/verify/plan.py test > /dev/nullpython3 scripts/verify/test_diff.py+ python3 scripts/verify/test_perf_build.pyif grep -rn --include='expectations*.toml' '^reason = "TODO' crates; thenecho "::error::blessed expectations still have TODO reasons: describe each failure (and link its fix) before merging"exit 1diff --git a/scripts/verify/perf-build.py b/scripts/verify/perf-build.pyindex e970415eb..7ed48548d 100644--- a/scripts/verify/perf-build.py+++ b/scripts/verify/perf-build.py@@ -28,10 +28,25 @@ PROFILE = [def main():worktree, out = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()out.mkdir(parents=True, exist_ok=True)+ metadata = json.loads(subprocess.check_output(+ ["cargo", "metadata", "--format-version", "1", "--locked"],+ cwd=worktree,+ text=True,+ ))+ # Another CI cache may have normalized source mtimes, hiding changes from+ # Cargo. Rebuild local crates while retaining registry and git dependencies.+ clean = ["cargo", *PROFILE, "clean", "--profile", "perf-verify"]+ for package in metadata["packages"]:+ if package["source"] is None:+ clean += ["-p", package["id"]]+ subprocess.run(clean, cwd=worktree, check=True)command = ["cargo", *PROFILE, "test", "--locked", "-p", "cap-verify", "--profile", "perf-verify", "--no-run", "--message-format", "json"]for target in TARGETS:command += ["--test", target]- result = subprocess.run(command, cwd=worktree, stdout=subprocess.PIPE, text=True, check=True)+ result = subprocess.run(command, cwd=worktree, stdout=subprocess.PIPE, text=True)+ if result.returncode:+ print(result.stdout, end="", flush=True)+ result.check_returncode()binaries = {}for line in result.stdout.splitlines():try:diff --git a/scripts/verify/test_perf_build.py b/scripts/verify/test_perf_build.pynew file mode 100644index 000000000..048a7312d--- /dev/null+++ b/scripts/verify/test_perf_build.py@@ -0,0 +1,64 @@+import json+import os+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++SCRIPT = Path(__file__).with_name("perf-build.py").resolve()+OLD = 946684800+++class PerformanceBuildCacheTests(unittest.TestCase):+ def test_changed_library_is_rebuilt_after_another_profile_normalizes_its_mtime(self):+ with tempfile.TemporaryDirectory() as temporary:+ root = Path(temporary)+ (root / "src").mkdir()+ (root / "tests").mkdir()+ (root / "target" / "Frameworks").mkdir(parents=True)+ (root / "Cargo.toml").write_text(+ '[package]\nname = "cap-verify"\nversion = "0.1.0"\nedition = "2021"\n'+ )+ source = root / "src" / "lib.rs"+ source.write_text("pub fn value() -> u32 { 1 }\n")+ for target in ["export", "seek", "playback"]:+ (root / "tests" / f"{target}.rs").write_text(+ '#[test]\nfn current_library() {\n'+ 'let expected: u32 = std::env::var("EXPECTED").unwrap().parse().unwrap();\n'+ 'assert_eq!(cap_verify::value(), expected);\n}\n'+ )+ env = dict(os.environ, CARGO_TARGET_DIR=str(root / "target"))+ self.run_command(root, env, "git", "init", "--quiet")+ self.run_command(root, env, "cargo", "generate-lockfile", "--offline")+ for version in [1, 2, 1]:+ source.write_text(f"pub fn value() -> u32 {{ {version} }}\n")+ self.run_command(root, env, "git", "add", "Cargo.toml", "Cargo.lock", "src", "tests")+ os.utime(source, (OLD, OLD))+ output = root / f"build-{version}"+ self.run_command(root, env, sys.executable, str(SCRIPT), str(root), str(output))+ manifest = json.loads((output / "binaries.json").read_text())+ self.run_command(+ root,+ dict(env, EXPECTED=str(version)),+ manifest["binaries"]["playback"],+ )+ source.write_text("pub fn broken(\n")+ self.run_command(root, env, "git", "add", "src/lib.rs")+ result = subprocess.run(+ [sys.executable, str(SCRIPT), str(root), str(root / "broken-build")],+ cwd=root,+ env=env,+ capture_output=True,+ text=True,+ )+ self.assertNotEqual(result.returncode, 0)+ self.assertIn('"reason":"compiler-message"', result.stdout)++ def run_command(self, root, env, *command):+ result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True)+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)+++if __name__ == "__main__":+ unittest.main()