Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix(scripts): reject benchmark runs that report no measurements

Marcos Daniel Rivera Gonzalez9611675195a2
diff --git a/scripts/capbench.py b/scripts/capbench.py
index 600fa3ae1..94e6769dc 100644
--- a/scripts/capbench.py
+++ b/scripts/capbench.py
@@ -368,6 +368,14 @@ def record_bench(mode, fps, seconds):
"--fps", str(fps), "--duration", str(seconds), "--path", str(out_dir / "rec.cap"),
]
+ better = {
+ "cpu_per_rec_s": "lower", "peak_commit_mb": "lower", "delivered_fps": "higher",
+ "captured_ratio": "higher", "overhead_s": "lower",
+ }
+ if mode == "instant":
+ # cap-test validate reports no frame count or duration for instant recordings.
+ del better["delivered_fps"], better["captured_ratio"]
+
def post(ctx, out_dir, metrics):
project = out_dir / "rec.cap"
validation = subprocess.run(
@@ -394,10 +402,7 @@ def record_bench(mode, fps, seconds):
"timeout": seconds + 120,
"env": {},
"needs_screen": True,
- "better": {
- "cpu_per_rec_s": "lower", "peak_commit_mb": "lower", "delivered_fps": "higher",
- "captured_ratio": "higher", "overhead_s": "lower",
- },
+ "better": better,
}
@@ -495,6 +500,15 @@ def start_animation(bounds):
return process
+# Job accounting, and the metrics derived from it, exist only where the job object does (Windows).
+JOB_METRICS = {"cpu_s", "peak_commit_mb", "cpu_per_rec_s"}
+
+
+def missing_metrics(better, metrics):
+ optional = set() if "cpu_s" in metrics else JOB_METRICS
+ return sorted(key for key in better if key not in metrics and key not in optional)
+
+
def run_once(name, spec, ctx, out_dir, keep_log):
out_dir.mkdir(parents=True, exist_ok=True)
argv = spec["argv"](ctx, out_dir)
@@ -515,6 +529,10 @@ def run_once(name, spec, ctx, out_dir, keep_log):
except InvalidRun as error:
code = "invalid"
lines.append((measured["wall_s"], f"capbench: {error}"))
+ missing = missing_metrics(spec["better"], metrics)
+ if code == 0 and missing:
+ code = "invalid"
+ lines.append((measured["wall_s"], f"capbench: no {', '.join(missing)} in this run's output"))
if keep_log or code != 0:
(out_dir / "output.log").write_text("\n".join(line for _, line in lines), encoding="utf-8")
return code, metrics, artifact
@@ -610,9 +628,6 @@ def cmd_run(args):
status = "ok" if code == 0 else f"exit={code}"
summary = ", ".join(f"{key}={value:.3f}" for key, value in sorted(metrics.items()) if key in spec["better"])
print(f"[{phase} {rep}] {label} {name}: {status} {summary}", flush=True)
- missing = sorted(key for key in spec["better"] if key not in metrics)
- if code == 0 and missing:
- print(f" warning: no {', '.join(missing)} in this run's output", flush=True)
if phase == "warmup":
continue
if code != 0:
diff --git a/scripts/test-capbench.py b/scripts/test-capbench.py
index 9eefd98ab..035193826 100644
--- a/scripts/test-capbench.py
+++ b/scripts/test-capbench.py
@@ -126,6 +126,17 @@ class CapbenchTests(unittest.TestCase):
with self.assertRaises(capbench.InvalidRun):
capbench.validated_video("thread 'main' panicked", 101)
+ def test_runs_missing_a_metric_are_invalid(self):
+ better = capbench.BENCHES["decode-display"]["better"]
+ complete = {key: 1.0 for key in better}
+ self.assertEqual(capbench.missing_metrics(better, complete), [])
+ crashed = {"wall_s": 0.4, "cpu_s": 0.1, "peak_commit_mb": 40.0}
+ self.assertIn("sequential_fps", capbench.missing_metrics(better, crashed))
+ no_job = {key: 1.0 for key in better if key not in capbench.JOB_METRICS}
+ self.assertEqual(capbench.missing_metrics(better, no_job), [])
+ self.assertNotIn("delivered_fps", capbench.BENCHES["record-instant-30"]["better"])
+ self.assertIn("delivered_fps", capbench.BENCHES["record-studio-60"]["better"])
+
def test_variant_env_is_scoped_per_label(self):
env = capbench.parse_variant_env(["base", "t4"], ["t4:CAP_EXP_DECODE_THREADS=4", "t4:X=a=b"])
self.assertEqual(env, {"base": {}, "t4": {"CAP_EXP_DECODE_THREADS": "4", "X": "a=b"}})