Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(ci): rebuild local crates for trustworthy performance comparisons

Richie McIlroy33d9f4a4b457
diff --git a/.github/workflows/engine-verify.yml b/.github/workflows/engine-verify.yml
index 07a176dc7..806f8b617 100644
--- a/.github/workflows/engine-verify.yml
+++ b/.github/workflows/engine-verify.yml
@@ -88,6 +88,7 @@ jobs:
run: |
python3 scripts/verify/plan.py test > /dev/null
python3 scripts/verify/test_diff.py
+ python3 scripts/verify/test_perf_build.py
if grep -rn --include='expectations*.toml' '^reason = "TODO' crates; then
echo "::error::blessed expectations still have TODO reasons: describe each failure (and link its fix) before merging"
exit 1
diff --git a/scripts/verify/perf-build.py b/scripts/verify/perf-build.py
index e970415eb..7ed48548d 100644
--- a/scripts/verify/perf-build.py
+++ b/scripts/verify/perf-build.py
@@ -28,10 +28,25 @@ PROFILE = [
def main():
worktree, out = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()
out.mkdir(parents=True, exist_ok=True)
+ metadata = json.loads(subprocess.check_output(
+ ["cargo", "metadata", "--format-version", "1", "--locked"],
+ cwd=worktree,
+ text=True,
+ ))
+ # Another CI cache may have normalized source mtimes, hiding changes from
+ # Cargo. Rebuild local crates while retaining registry and git dependencies.
+ clean = ["cargo", *PROFILE, "clean", "--profile", "perf-verify"]
+ for package in metadata["packages"]:
+ if package["source"] is None:
+ clean += ["-p", package["id"]]
+ subprocess.run(clean, cwd=worktree, check=True)
command = ["cargo", *PROFILE, "test", "--locked", "-p", "cap-verify", "--profile", "perf-verify", "--no-run", "--message-format", "json"]
for target in TARGETS:
command += ["--test", target]
- result = subprocess.run(command, cwd=worktree, stdout=subprocess.PIPE, text=True, check=True)
+ result = subprocess.run(command, cwd=worktree, stdout=subprocess.PIPE, text=True)
+ if result.returncode:
+ print(result.stdout, end="", flush=True)
+ result.check_returncode()
binaries = {}
for line in result.stdout.splitlines():
try:
diff --git a/scripts/verify/test_perf_build.py b/scripts/verify/test_perf_build.py
new file mode 100644
index 000000000..048a7312d
--- /dev/null
+++ b/scripts/verify/test_perf_build.py
@@ -0,0 +1,64 @@
+import json
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT = Path(__file__).with_name("perf-build.py").resolve()
+OLD = 946684800
+
+
+class PerformanceBuildCacheTests(unittest.TestCase):
+ def test_changed_library_is_rebuilt_after_another_profile_normalizes_its_mtime(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ (root / "src").mkdir()
+ (root / "tests").mkdir()
+ (root / "target" / "Frameworks").mkdir(parents=True)
+ (root / "Cargo.toml").write_text(
+ '[package]\nname = "cap-verify"\nversion = "0.1.0"\nedition = "2021"\n'
+ )
+ source = root / "src" / "lib.rs"
+ source.write_text("pub fn value() -> u32 { 1 }\n")
+ for target in ["export", "seek", "playback"]:
+ (root / "tests" / f"{target}.rs").write_text(
+ '#[test]\nfn current_library() {\n'
+ 'let expected: u32 = std::env::var("EXPECTED").unwrap().parse().unwrap();\n'
+ 'assert_eq!(cap_verify::value(), expected);\n}\n'
+ )
+ env = dict(os.environ, CARGO_TARGET_DIR=str(root / "target"))
+ self.run_command(root, env, "git", "init", "--quiet")
+ self.run_command(root, env, "cargo", "generate-lockfile", "--offline")
+ for version in [1, 2, 1]:
+ source.write_text(f"pub fn value() -> u32 {{ {version} }}\n")
+ self.run_command(root, env, "git", "add", "Cargo.toml", "Cargo.lock", "src", "tests")
+ os.utime(source, (OLD, OLD))
+ output = root / f"build-{version}"
+ self.run_command(root, env, sys.executable, str(SCRIPT), str(root), str(output))
+ manifest = json.loads((output / "binaries.json").read_text())
+ self.run_command(
+ root,
+ dict(env, EXPECTED=str(version)),
+ manifest["binaries"]["playback"],
+ )
+ source.write_text("pub fn broken(\n")
+ self.run_command(root, env, "git", "add", "src/lib.rs")
+ result = subprocess.run(
+ [sys.executable, str(SCRIPT), str(root), str(root / "broken-build")],
+ cwd=root,
+ env=env,
+ capture_output=True,
+ text=True,
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertIn('"reason":"compiler-message"', result.stdout)
+
+ def run_command(self, root, env, *command):
+ result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True)
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+
+
+if __name__ == "__main__":
+ unittest.main()