Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: compare failure severity independently of witness position

Richie McIlroyab45fd46ab4a
diff --git a/scripts/verify/diff.py b/scripts/verify/diff.py
index 095fb6527..d9332ad19 100644
--- a/scripts/verify/diff.py
+++ b/scripts/verify/diff.py
@@ -279,6 +279,8 @@ def compare_case(b, m, rel, abs_):
better.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}")
bm, mm = b["measures"].get(cls, {}), m["measures"].get(cls, {})
for metric in sorted(set(bm) | set(mm)):
+ if metric == "frame":
+ continue
if fresh or timing_resolution(metric) is not None:
bv, mv = distribution(b, cls, metric), distribution(m, cls, metric)
resolution = metric_resolution(metric)
diff --git a/scripts/verify/test_diff.py b/scripts/verify/test_diff.py
index 7bca84137..83eb11006 100644
--- a/scripts/verify/test_diff.py
+++ b/scripts/verify/test_diff.py
@@ -34,6 +34,29 @@ def measured(value, case="export/instant-gated-mic-only/output/mp4/native", cls=
class ComparisonTests(unittest.TestCase):
+ def test_witness_frame_movement_does_not_change_severity(self):
+ for cls in ["stale", "future", "unreadable", "timestamps", "nondeterministic"]:
+ with self.subTest(cls=cls):
+ base = measured(4, case="editing/export/basic/seed-10", cls=cls, metric="screen-frames")
+ base["measures"][cls]["frame"] = [63, 63]
+ main = json.loads(json.dumps(base))
+ main["measures"][cls]["frame"] = [109, 109]
+ self.assertEqual(compare([base], [main]), ("EQUAL", []))
+ main["measures"][cls]["screen-frames"] = [3, 3]
+ self.assertEqual(compare([base], [main])[0], "BETTER")
+ main["measures"][cls]["screen-frames"] = [5, 5]
+ self.assertEqual(compare([base], [main])[0], "WORSE")
+
+ def test_witness_frame_does_not_hide_new_classes_or_larger_deltas(self):
+ for cls in ["stale", "future", "unreadable", "grid-rounding"]:
+ with self.subTest(cls=cls):
+ main = measured(109, case="export/basic", cls=cls, metric="frame")
+ self.assertEqual(compare([dict(record(), case="export/basic")], [main])[0], "WORSE")
+ base = json.loads(json.dumps(main))
+ base["measures"][cls]["delta"] = [1, 1]
+ main["measures"][cls]["delta"] = [2, 2]
+ self.assertEqual(compare([base], [main])[0], "WORSE")
+
def test_continuous_metrics_require_three_independent_runs(self):
self.assertEqual(compare([measured(15.2)], [measured(15.3)])[0], "INSUFFICIENT")
self.assertEqual(compare([measured(15.2, sample="same") for _ in range(3)], [measured(16) for _ in range(3)])[0], "INSUFFICIENT")