Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(scripts): check benchmark helpers up front and reject decode runs with failed reads

Marcos Daniel Rivera Gonzalez0505abc52ad7
diff --git a/scripts/capbench.py b/scripts/capbench.py
index 94e6769dc..12f072b1a 100644
--- a/scripts/capbench.py
+++ b/scripts/capbench.py
@@ -209,6 +209,13 @@ def section_floats(lines, start_pattern, pattern, group=1):
return None
+def on_path(name):
+ path = shutil.which(name)
+ if path is None:
+ raise FileNotFoundError(f"missing {name}; put it on PATH")
+ return path
+
+
def ffprobe_video(path):
result = subprocess.run(
[
@@ -273,6 +280,7 @@ def export_bench(fps, resolution, quality, extra=()):
"argv": argv,
"parse": parse_export,
"post": post,
+ "requires": lambda ctx: on_path("ffprobe"),
"timeout": 900,
"env": {"CAP_DISABLE_ENCODER_SELF_TEST": None},
"better": {
@@ -329,6 +337,14 @@ def parse_decode(lines):
"random_avg_ms": float_after(lines, r"Avg access time:\s*([\d.]+)ms"),
"random_p95_ms": section_floats(lines, r"RANDOM ACCESS PERFORMANCE", r"P95:\s*([\d.]+)ms"),
}
+ # Timings cover only the reads that succeeded, so a run with failed reads looks faster.
+ failed = sum(
+ int(match.group(2))
+ for _, line in lines
+ if (match := re.match(r"(Frames failed|Seek failures|Failures):\s*(\d+)", line.strip()))
+ )
+ if failed:
+ raise InvalidRun(f"decode-benchmark reported {failed} failed reads")
return {key: value for key, value in metrics.items() if value is not None}
@@ -399,6 +415,7 @@ def record_bench(mode, fps, seconds):
"argv": argv,
"parse": lambda lines: {},
"post": post,
+ "requires": lambda ctx: ctx.bin("cap-test"),
"timeout": seconds + 120,
"env": {},
"needs_screen": True,
@@ -520,15 +537,15 @@ def run_once(name, spec, ctx, out_dir, keep_log):
animation.kill()
animation.wait()
metrics = {key: value for key, value in measured.items() if key != "exit"}
- metrics.update(spec["parse"](lines))
artifact = None
code = measured["exit"]
- if code == 0 and spec.get("post"):
- try:
+ try:
+ metrics.update(spec["parse"](lines))
+ if code == 0 and spec.get("post"):
artifact = spec["post"](ctx, out_dir, metrics)
- except InvalidRun as error:
- code = "invalid"
- lines.append((measured["wall_s"], f"capbench: {error}"))
+ except InvalidRun as error:
+ code = "invalid" if code == 0 else code
+ lines.append((measured["wall_s"], f"capbench: {error}"))
missing = missing_metrics(spec["better"], metrics)
if code == 0 and missing:
code = "invalid"
@@ -590,6 +607,7 @@ def cmd_run(args):
try:
for name in names:
BENCHES[name]["argv"](ctx, work)
+ BENCHES[name].get("requires", lambda ctx: None)(ctx)
except FileNotFoundError as error:
sys.exit(str(error))
if work.exists():
diff --git a/scripts/test-capbench.py b/scripts/test-capbench.py
index 035193826..d38d5fa8b 100644
--- a/scripts/test-capbench.py
+++ b/scripts/test-capbench.py
@@ -3,6 +3,7 @@ import importlib.util
import json
from pathlib import Path
import sys
+from types import SimpleNamespace
import unittest
@@ -90,6 +91,18 @@ class CapbenchTests(unittest.TestCase):
"random_avg_ms": 107.38, "random_p95_ms": 150.12,
})
+ def test_decode_runs_with_failed_reads_are_invalid(self):
+ failed = DECODE_OUTPUT + "\nRANDOM ACCESS PERFORMANCE\n Failures: 50\n"
+ with self.assertRaisesRegex(capbench.InvalidRun, "50 failed reads"):
+ capbench.parse_decode(timed(failed))
+ clean = DECODE_OUTPUT + "\n Frames failed: 0\n"
+ self.assertIn("sequential_fps", capbench.parse_decode(timed(clean)))
+
+ def test_startup_check_finds_missing_helpers(self):
+ args = SimpleNamespace(fixture=".", bin_dir=str(Path(__file__).parent), screen=None, animate=False, affinity=None)
+ with self.assertRaisesRegex(FileNotFoundError, "cap-test"):
+ capbench.BENCHES["record-studio-60"]["requires"](capbench.Context(args))
+
def test_startup_metrics(self):
lines = timed("Startup: avg=1159.3ms p50=671.2ms p95=2229.8ms\nFirst preview: avg=37.0ms p50=44.4ms p95=45.9ms")
self.assertEqual(capbench.parse_startup(lines), {"startup_p50_ms": 671.2, "first_preview_p50_ms": 44.4})