Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: compare failure severity independently of witness position
Richie McIlroyab45fd46ab4a
diff --git a/scripts/verify/diff.py b/scripts/verify/diff.pyindex 095fb6527..d9332ad19 100644--- a/scripts/verify/diff.py+++ b/scripts/verify/diff.py@@ -279,6 +279,8 @@ def compare_case(b, m, rel, abs_):better.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}")bm, mm = b["measures"].get(cls, {}), m["measures"].get(cls, {})for metric in sorted(set(bm) | set(mm)):+ if metric == "frame":+ continueif fresh or timing_resolution(metric) is not None:bv, mv = distribution(b, cls, metric), distribution(m, cls, metric)resolution = metric_resolution(metric)diff --git a/scripts/verify/test_diff.py b/scripts/verify/test_diff.pyindex 7bca84137..83eb11006 100644--- a/scripts/verify/test_diff.py+++ b/scripts/verify/test_diff.py@@ -34,6 +34,29 @@ def measured(value, case="export/instant-gated-mic-only/output/mp4/native", cls=class ComparisonTests(unittest.TestCase):+ def test_witness_frame_movement_does_not_change_severity(self):+ for cls in ["stale", "future", "unreadable", "timestamps", "nondeterministic"]:+ with self.subTest(cls=cls):+ base = measured(4, case="editing/export/basic/seed-10", cls=cls, metric="screen-frames")+ base["measures"][cls]["frame"] = [63, 63]+ main = json.loads(json.dumps(base))+ main["measures"][cls]["frame"] = [109, 109]+ self.assertEqual(compare([base], [main]), ("EQUAL", []))+ main["measures"][cls]["screen-frames"] = [3, 3]+ self.assertEqual(compare([base], [main])[0], "BETTER")+ main["measures"][cls]["screen-frames"] = [5, 5]+ self.assertEqual(compare([base], [main])[0], "WORSE")++ def test_witness_frame_does_not_hide_new_classes_or_larger_deltas(self):+ for cls in ["stale", "future", "unreadable", "grid-rounding"]:+ with self.subTest(cls=cls):+ main = measured(109, case="export/basic", cls=cls, metric="frame")+ self.assertEqual(compare([dict(record(), case="export/basic")], [main])[0], "WORSE")+ base = json.loads(json.dumps(main))+ base["measures"][cls]["delta"] = [1, 1]+ main["measures"][cls]["delta"] = [2, 2]+ self.assertEqual(compare([base], [main])[0], "WORSE")+def test_continuous_metrics_require_three_independent_runs(self):self.assertEqual(compare([measured(15.2)], [measured(15.3)])[0], "INSUFFICIENT")self.assertEqual(compare([measured(15.2, sample="same") for _ in range(3)], [measured(16) for _ in range(3)])[0], "INSUFFICIENT")