Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix(scripts): check benchmark helpers up front and reject decode runs with failed reads
Marcos Daniel Rivera Gonzalez0505abc52ad7
diff --git a/scripts/capbench.py b/scripts/capbench.pyindex 94e6769dc..12f072b1a 100644--- a/scripts/capbench.py+++ b/scripts/capbench.py@@ -209,6 +209,13 @@ def section_floats(lines, start_pattern, pattern, group=1):return None+def on_path(name):+ path = shutil.which(name)+ if path is None:+ raise FileNotFoundError(f"missing {name}; put it on PATH")+ return path++def ffprobe_video(path):result = subprocess.run([@@ -273,6 +280,7 @@ def export_bench(fps, resolution, quality, extra=()):"argv": argv,"parse": parse_export,"post": post,+ "requires": lambda ctx: on_path("ffprobe"),"timeout": 900,"env": {"CAP_DISABLE_ENCODER_SELF_TEST": None},"better": {@@ -329,6 +337,14 @@ def parse_decode(lines):"random_avg_ms": float_after(lines, r"Avg access time:\s*([\d.]+)ms"),"random_p95_ms": section_floats(lines, r"RANDOM ACCESS PERFORMANCE", r"P95:\s*([\d.]+)ms"),}+ # Timings cover only the reads that succeeded, so a run with failed reads looks faster.+ failed = sum(+ int(match.group(2))+ for _, line in lines+ if (match := re.match(r"(Frames failed|Seek failures|Failures):\s*(\d+)", line.strip()))+ )+ if failed:+ raise InvalidRun(f"decode-benchmark reported {failed} failed reads")return {key: value for key, value in metrics.items() if value is not None}@@ -399,6 +415,7 @@ def record_bench(mode, fps, seconds):"argv": argv,"parse": lambda lines: {},"post": post,+ "requires": lambda ctx: ctx.bin("cap-test"),"timeout": seconds + 120,"env": {},"needs_screen": True,@@ -520,15 +537,15 @@ def run_once(name, spec, ctx, out_dir, keep_log):animation.kill()animation.wait()metrics = {key: value for key, value in measured.items() if key != "exit"}- metrics.update(spec["parse"](lines))artifact = Nonecode = measured["exit"]- if code == 0 and spec.get("post"):- try:+ try:+ metrics.update(spec["parse"](lines))+ if code == 0 and spec.get("post"):artifact = spec["post"](ctx, out_dir, metrics)- except InvalidRun as error:- code = "invalid"- lines.append((measured["wall_s"], f"capbench: {error}"))+ except InvalidRun as error:+ code = "invalid" if code == 0 else code+ lines.append((measured["wall_s"], f"capbench: {error}"))missing = missing_metrics(spec["better"], metrics)if code == 0 and missing:code = "invalid"@@ -590,6 +607,7 @@ def cmd_run(args):try:for name in names:BENCHES[name]["argv"](ctx, work)+ BENCHES[name].get("requires", lambda ctx: None)(ctx)except FileNotFoundError as error:sys.exit(str(error))if work.exists():diff --git a/scripts/test-capbench.py b/scripts/test-capbench.pyindex 035193826..d38d5fa8b 100644--- a/scripts/test-capbench.py+++ b/scripts/test-capbench.py@@ -3,6 +3,7 @@ import importlib.utilimport jsonfrom pathlib import Pathimport sys+from types import SimpleNamespaceimport unittest@@ -90,6 +91,18 @@ class CapbenchTests(unittest.TestCase):"random_avg_ms": 107.38, "random_p95_ms": 150.12,})+ def test_decode_runs_with_failed_reads_are_invalid(self):+ failed = DECODE_OUTPUT + "\nRANDOM ACCESS PERFORMANCE\n Failures: 50\n"+ with self.assertRaisesRegex(capbench.InvalidRun, "50 failed reads"):+ capbench.parse_decode(timed(failed))+ clean = DECODE_OUTPUT + "\n Frames failed: 0\n"+ self.assertIn("sequential_fps", capbench.parse_decode(timed(clean)))++ def test_startup_check_finds_missing_helpers(self):+ args = SimpleNamespace(fixture=".", bin_dir=str(Path(__file__).parent), screen=None, animate=False, affinity=None)+ with self.assertRaisesRegex(FileNotFoundError, "cap-test"):+ capbench.BENCHES["record-studio-60"]["requires"](capbench.Context(args))+def test_startup_metrics(self):lines = timed("Startup: avg=1159.3ms p50=671.2ms p95=2229.8ms\nFirst preview: avg=37.0ms p50=44.4ms p95=45.9ms")self.assertEqual(capbench.parse_startup(lines), {"startup_p50_ms": 671.2, "first_preview_p50_ms": 44.4})