Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(verify): reject stale performance reports on reruns

Akshar Patel5a63119b48d7
diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.yml
index 999f0b463..f1629edb3 100644
--- a/.github/workflows/perf-budget.yml
+++ b/.github/workflows/perf-budget.yml
@@ -29,6 +29,8 @@ jobs:
- uses: actions/checkout@v4
with:
persist-credentials: false
+ - name: Test performance report isolation
+ run: python3 -B scripts/verify/test_perf_ab.py
- name: Scope
id: scope
env:
diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.py
index 2d48a702c..0e4c8de5b 100644
--- a/scripts/verify/perf-ab.py
+++ b/scripts/verify/perf-ab.py
@@ -50,7 +50,7 @@ def run_once(side, target, case, tier, fixtures, report):
CAP_VERIFY_CASE=case,
CAP_VERIFY_TIER=tier,
)
- report.mkdir(parents=True, exist_ok=True)
+ report.mkdir(parents=True)
started = time.monotonic()
with (report / "test.log").open("w") as log:
process = subprocess.Popen(
@@ -87,7 +87,7 @@ def main():
parser.add_argument("--base", required=True)
parser.add_argument("--head", required=True)
parser.add_argument("--fixtures", required=True)
- parser.add_argument("--out", required=True)
+ parser.add_argument("--out", required=True, help="use a fresh path for each comparison; existing reports are preserved")
parser.add_argument("--pairs", type=int, default=3)
parser.add_argument("--summary")
args = parser.parse_args()
diff --git a/scripts/verify/test_perf_ab.py b/scripts/verify/test_perf_ab.py
new file mode 100644
index 000000000..8b7f91256
--- /dev/null
+++ b/scripts/verify/test_perf_ab.py
@@ -0,0 +1,91 @@
+import importlib.util
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+SCRIPT = Path(__file__).with_name("perf-ab.py").resolve()
+SPEC = importlib.util.spec_from_file_location("perf_ab", SCRIPT)
+PERF = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(PERF)
+
+
[email protected](hasattr(os, "wait4"), "The performance runner requires POSIX wait4")
+class PerformanceReportTests(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temporary.cleanup)
+ self.root = Path(self.temporary.name).resolve()
+ (self.root / "crates" / "verify").mkdir(parents=True)
+
+ def side(self, records, exit_code=0):
+ binary = self.root / "fixture.py"
+ binary.with_suffix(".json").write_text(json.dumps({"records": records, "exit": exit_code}))
+ binary.write_text(f"#!{sys.executable}\n" + r"""import json
+import os
+from pathlib import Path
+import sys
+
+Path(__file__).with_suffix(".started").touch()
+fixture = json.loads(Path(__file__).with_suffix(".json").read_text())
+if fixture["records"]:
+ report = Path(os.environ["CAP_VERIFY_REPORT_DIR"])
+ path = report / f"cap-verify-{os.getpid()}.jsonl"
+ path.write_text("".join(json.dumps(row) + "\n" for row in fixture["records"]))
+print("fixture completed")
+sys.exit(fixture["exit"])
+""")
+ binary.chmod(0o700)
+ return {
+ "worktree": str(self.root),
+ "binaries": {target: str(binary) for _, target, _, _, _ in PERF.WORKLOADS},
+ }
+
+ def test_existing_reports_are_preserved_and_never_used(self):
+ side = self.side([])
+ report = self.root / "reports"
+ report.mkdir()
+ stale = report / "cap-verify-123.jsonl"
+ previous = json.dumps({"case": "export/basic/output/mp4/native", "outcome": "pass"}) + "\n"
+ stale.write_text(previous)
+ log = report / "test.log"
+ log.write_text("previous run\n")
+ with self.assertRaises(FileExistsError):
+ PERF.run_once(side, "export", "export/basic/*", "fast", self.root / "fixtures", report)
+ self.assertEqual(stale.read_text(), previous)
+ self.assertEqual(log.read_text(), "previous run\n")
+ self.assertFalse((self.root / "fixture.started").exists())
+
+ def test_fresh_reports_preserve_exit_status_and_outcomes(self):
+ outcomes = ["pass", "xfail", "xpass", "fail", "unsupported", "inconclusive"]
+ rows = [{"case": f"export/basic/{outcome}", "outcome": outcome} for outcome in outcomes]
+ result = PERF.run_once(
+ self.side(rows, 101), "export", "export/basic/*", "fast",
+ self.root / "fixtures", self.root / "reports",
+ )
+ self.assertEqual(result["exit"], 101)
+ self.assertEqual(result["outcomes"], {
+ f"export/basic/{outcome}": "ran" if outcome in {"pass", "xfail", "xpass"} else outcome
+ for outcome in outcomes
+ })
+
+ def test_cli_rejects_fresh_runs_that_report_no_cases(self):
+ manifest = self.root / "binaries.json"
+ manifest.write_text(json.dumps(self.side([])))
+ output = self.root / "output"
+ output.mkdir()
+ result = subprocess.run([
+ sys.executable, "-B", str(SCRIPT), "--base", str(manifest), "--head", str(manifest),
+ "--fixtures", str(self.root / "fixtures"), "--out", str(output), "--pairs", "1",
+ ], capture_output=True, text=True, timeout=30)
+ self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
+ self.assertIn("not measured:", result.stdout)
+ self.assertEqual((output / "perf-budget.md").read_text().count("BROKEN:"), len(PERF.WORKLOADS))
+
+
+if __name__ == "__main__":
+ unittest.main()