Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(verify): reject stale performance reports on reruns
Akshar Patel5a63119b48d7
diff --git a/.github/workflows/perf-budget.yml b/.github/workflows/perf-budget.ymlindex 999f0b463..f1629edb3 100644--- a/.github/workflows/perf-budget.yml+++ b/.github/workflows/perf-budget.yml@@ -29,6 +29,8 @@ jobs:- uses: actions/checkout@v4with:persist-credentials: false+ - name: Test performance report isolation+ run: python3 -B scripts/verify/test_perf_ab.py- name: Scopeid: scopeenv:diff --git a/scripts/verify/perf-ab.py b/scripts/verify/perf-ab.pyindex 2d48a702c..0e4c8de5b 100644--- a/scripts/verify/perf-ab.py+++ b/scripts/verify/perf-ab.py@@ -50,7 +50,7 @@ def run_once(side, target, case, tier, fixtures, report):CAP_VERIFY_CASE=case,CAP_VERIFY_TIER=tier,)- report.mkdir(parents=True, exist_ok=True)+ report.mkdir(parents=True)started = time.monotonic()with (report / "test.log").open("w") as log:process = subprocess.Popen(@@ -87,7 +87,7 @@ def main():parser.add_argument("--base", required=True)parser.add_argument("--head", required=True)parser.add_argument("--fixtures", required=True)- parser.add_argument("--out", required=True)+ parser.add_argument("--out", required=True, help="use a fresh path for each comparison; existing reports are preserved")parser.add_argument("--pairs", type=int, default=3)parser.add_argument("--summary")args = parser.parse_args()diff --git a/scripts/verify/test_perf_ab.py b/scripts/verify/test_perf_ab.pynew file mode 100644index 000000000..8b7f91256--- /dev/null+++ b/scripts/verify/test_perf_ab.py@@ -0,0 +1,91 @@+import importlib.util+import json+import os+from pathlib import Path+import subprocess+import sys+import tempfile+import unittest+++SCRIPT = Path(__file__).with_name("perf-ab.py").resolve()+SPEC = importlib.util.spec_from_file_location("perf_ab", SCRIPT)+PERF = importlib.util.module_from_spec(SPEC)+SPEC.loader.exec_module(PERF)++[email protected](hasattr(os, "wait4"), "The performance runner requires POSIX wait4")+class PerformanceReportTests(unittest.TestCase):+ def setUp(self):+ self.temporary = tempfile.TemporaryDirectory()+ self.addCleanup(self.temporary.cleanup)+ self.root = Path(self.temporary.name).resolve()+ (self.root / "crates" / "verify").mkdir(parents=True)++ def side(self, records, exit_code=0):+ binary = self.root / "fixture.py"+ binary.with_suffix(".json").write_text(json.dumps({"records": records, "exit": exit_code}))+ binary.write_text(f"#!{sys.executable}\n" + r"""import json+import os+from pathlib import Path+import sys++Path(__file__).with_suffix(".started").touch()+fixture = json.loads(Path(__file__).with_suffix(".json").read_text())+if fixture["records"]:+ report = Path(os.environ["CAP_VERIFY_REPORT_DIR"])+ path = report / f"cap-verify-{os.getpid()}.jsonl"+ path.write_text("".join(json.dumps(row) + "\n" for row in fixture["records"]))+print("fixture completed")+sys.exit(fixture["exit"])+""")+ binary.chmod(0o700)+ return {+ "worktree": str(self.root),+ "binaries": {target: str(binary) for _, target, _, _, _ in PERF.WORKLOADS},+ }++ def test_existing_reports_are_preserved_and_never_used(self):+ side = self.side([])+ report = self.root / "reports"+ report.mkdir()+ stale = report / "cap-verify-123.jsonl"+ previous = json.dumps({"case": "export/basic/output/mp4/native", "outcome": "pass"}) + "\n"+ stale.write_text(previous)+ log = report / "test.log"+ log.write_text("previous run\n")+ with self.assertRaises(FileExistsError):+ PERF.run_once(side, "export", "export/basic/*", "fast", self.root / "fixtures", report)+ self.assertEqual(stale.read_text(), previous)+ self.assertEqual(log.read_text(), "previous run\n")+ self.assertFalse((self.root / "fixture.started").exists())++ def test_fresh_reports_preserve_exit_status_and_outcomes(self):+ outcomes = ["pass", "xfail", "xpass", "fail", "unsupported", "inconclusive"]+ rows = [{"case": f"export/basic/{outcome}", "outcome": outcome} for outcome in outcomes]+ result = PERF.run_once(+ self.side(rows, 101), "export", "export/basic/*", "fast",+ self.root / "fixtures", self.root / "reports",+ )+ self.assertEqual(result["exit"], 101)+ self.assertEqual(result["outcomes"], {+ f"export/basic/{outcome}": "ran" if outcome in {"pass", "xfail", "xpass"} else outcome+ for outcome in outcomes+ })++ def test_cli_rejects_fresh_runs_that_report_no_cases(self):+ manifest = self.root / "binaries.json"+ manifest.write_text(json.dumps(self.side([])))+ output = self.root / "output"+ output.mkdir()+ result = subprocess.run([+ sys.executable, "-B", str(SCRIPT), "--base", str(manifest), "--head", str(manifest),+ "--fixtures", str(self.root / "fixtures"), "--out", str(output), "--pairs", "1",+ ], capture_output=True, text=True, timeout=30)+ self.assertEqual(result.returncode, 1, result.stdout + result.stderr)+ self.assertIn("not measured:", result.stdout)+ self.assertEqual((output / "perf-budget.md").read_text().count("BROKEN:"), len(PERF.WORKLOADS))+++if __name__ == "__main__":+ unittest.main()