Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: compare fresh timing measurements across independent runs
Richie McIlroy5fa33b019e52
diff --git a/scripts/verify/diff.py b/scripts/verify/diff.pyindex 712f36a8b..2e88142c7 100644--- a/scripts/verify/diff.py+++ b/scripts/verify/diff.py@@ -1,6 +1,7 @@import argparseimport fnmatchimport json+import mathimport osimport reimport sys@@ -107,6 +108,43 @@ def magnitude(rng):return None+def normalized_measures(record):+ measures = {cls: dict(metrics or {}) for cls, metrics in (record.get("measures") or {}).items()}+ if record.get("case") != "export/instant/output/mp4/native":+ return measures+ unobserved = record.get("unobserved") or {}+ if "sync-mic" in unobserved or "sync-system" in unobserved:+ return measures+ mic = measures.get("sync-mic", {}).get("offset-ms")+ system = measures.get("sync-system", {}).get("offset-ms")+ if mic is not None and system is not None:+ # Historical source offsets are already rounded to 0.1 ms. Subtract their+ # signed intervals without inventing precision or replacing newer telemetry.+ metrics = measures["sync-system"]+ metrics.setdefault("mic-offset-ms", mic)+ metrics.setdefault("system-minus-mic-ms", [round(system[0] - mic[1], 1), round(system[1] - mic[0], 1)])+ return measures+++def fresh_fixture(case):+ return case.startswith(("recording/", "crash/", "export/instant", "export/rec-"))+++def timing_resolution(metric):+ if metric.endswith("-ms"):+ return 0.1+ if metric == "secs":+ return 0.01+ if metric.endswith("-s"):+ return 1e-9+ return None+++def metric_resolution(metric):+ resolution = timing_resolution(metric)+ return resolution if resolution is not None else 1.0++def summarize(recs):ran = [r for r in recs if str(r.get("outcome", "")).lower() != "unsupported"]decisive = [r for r in ran if str(r.get("outcome", "")).lower() != "inconclusive"] or ran@@ -124,7 +162,7 @@ def summarize(recs):for cls in classes:class_runs[cls] += 1for r in group:- for cls, metrics in (r.get("measures") or {}).items():+ for cls, metrics in normalized_measures(r).items():for metric, rng in (metrics or {}).items():value = magnitude(rng)if value is None:@@ -171,6 +209,11 @@ def larger(a, b, rel, abs_):return a > b + max(abs_, rel * abs(b))+def beyond_resolution(a, b, rel, abs_):+ limit = b + max(abs_, rel * abs(b))+ return a > limit and not math.isclose(a, limit, rel_tol=1e-12, abs_tol=1e-12)++def compare_case(b, m, rel, abs_):worse, better, pending, overlap = [], [], [], []@@ -184,6 +227,14 @@ def compare_case(b, m, rel, abs_):lost = coverage_lost(b, m)worse.extend(f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost)bc, mc = b["classes"], m["classes"]+ fresh = fresh_fixture(b["case"]) or fresh_fixture(m["case"])+ if fresh and (b["runs"] < 3 or m["runs"] < 3):+ pending.append("fresh fixture comparison requires at least 3 independent runs per side on this OS image")+ sampled_classes = {+ cls for cls in set(bc) | set(mc)+ if fresh or any(timing_resolution(metric) is not None for metric in+ set(b["measures"].get(cls, {})) | set(m["measures"].get(cls, {})))+ }stochastic = "missing-after-hold"distribution_classes = (set(bc) | set(mc)) & {stochastic}if b["case"].startswith("recording/") or m["case"].startswith("recording/"):@@ -202,13 +253,23 @@ def compare_case(b, m, rel, abs_):elif bv != mv:overlap.append(description + " (ranges overlap)")for cls in sorted(set(mc) - set(bc) - set(gained) - {stochastic}):- worse.append(f"+{cls}")+ if cls in sampled_classes and (b["runs"] < 3 or m["runs"] < 3):+ pending.append(f"+{cls} (requires at least 3 independent runs per side on this OS image)")+ elif cls in sampled_classes and mc[cls] != m["runs"]:+ overlap.append(f"+{cls} in {mc[cls]}/{m['runs']} main runs (ranges overlap)")+ else:+ worse.append(f"+{cls}")for cls in sorted(set(bc) - set(mc) - {stochastic}):if cls in m["unobserved"]:continue- better.append(f"-{cls}")+ if cls in sampled_classes and (b["runs"] < 3 or m["runs"] < 3):+ pending.append(f"-{cls} (requires at least 3 independent runs per side on this OS image)")+ elif cls in sampled_classes and bc[cls] != b["runs"]:+ overlap.append(f"-{cls} in {bc[cls]}/{b['runs']} baseline runs (ranges overlap)")+ else:+ better.append(f"-{cls}")for cls in sorted((set(bc) & set(mc)) - {stochastic}):- if b["runs"] and m["runs"] and (b["runs"] > 1 or m["runs"] > 1):+ if cls not in sampled_classes and b["runs"] and m["runs"] and (b["runs"] > 1 or m["runs"] > 1):bf, mf = bc[cls] / b["runs"], mc[cls] / m["runs"]if mf > bf:worse.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}")@@ -216,7 +277,19 @@ def compare_case(b, m, rel, abs_):better.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}")bm, mm = b["measures"].get(cls, {}), m["measures"].get(cls, {})for metric in sorted(set(bm) | set(mm)):- if metric in bm and metric in mm:+ if fresh or timing_resolution(metric) is not None:+ bv, mv = distribution(b, cls, metric), distribution(m, cls, metric)+ resolution = metric_resolution(metric)+ description = f"{cls}.{metric} baseline={bv}, main={mv}, resolution={resolution:g}"+ if len(bv) < 3 or len(mv) < 3:+ pending.append(description + " (requires at least 3 independent runs per side on this OS image)")+ elif beyond_resolution(min(mv), max(bv), rel, max(abs_, resolution)):+ worse.append(description + " (every main value above baseline range by more than resolution)")+ elif beyond_resolution(min(bv), max(mv), rel, max(abs_, resolution)):+ better.append(description + " (every main value below baseline range by more than resolution)")+ elif bv != mv:+ overlap.append(description + " (ranges overlap within resolution)")+ elif metric in bm and metric in mm:if larger(mm[metric], bm[metric], rel, abs_):worse.append(f"{cls}.{metric} {fmt(bm[metric])} -> {fmt(mm[metric])}")elif larger(bm[metric], mm[metric], rel, abs_):diff --git a/scripts/verify/test_diff.py b/scripts/verify/test_diff.pyindex 005e00873..c5298f0a9 100644--- a/scripts/verify/test_diff.py+++ b/scripts/verify/test_diff.py@@ -27,7 +27,75 @@ def compare(base, main):return diff.compare_case(diff.summarize(base), diff.summarize(main), 0, 0)+def measured(value, case="export/instant-gated-mic-only/output/mp4/native", cls="sync-mic", metric="offset-ms", sample=None):+ result = record(sample=sample)+ result.update(case=case, failures={cls: []}, measures={cls: {metric: [value, value]}})+ return result++class ComparisonTests(unittest.TestCase):+ def test_continuous_metrics_require_three_independent_runs(self):+ self.assertEqual(compare([measured(15.2)], [measured(15.3)])[0], "INSUFFICIENT")+ self.assertEqual(compare([measured(15.2, sample="same") for _ in range(3)], [measured(16) for _ in range(3)])[0], "INSUFFICIENT")++ def test_continuous_overlap_and_one_resolution_step_are_not_worse(self):+ base = [measured(v) for v in [15.1, 15.2, 15.3]]+ for values in [[15.2, 15.3, 15.4], [15.4, 15.4, 15.4]]:+ with self.subTest(values=values):+ self.assertEqual(compare(base, [measured(v) for v in values])[0], "OVERLAP")++ def test_continuous_separated_ranges_are_worse(self):+ self.assertEqual(compare([measured(v) for v in [15.1, 15.2, 15.3]], [measured(v) for v in [15.5, 15.6, 15.5]])[0], "WORSE")++ def test_signed_offsets_compare_magnitude(self):+ self.assertEqual(compare([measured(-15) for _ in range(3)], [measured(-17) for _ in range(3)])[0], "WORSE")++ def test_timing_metric_outside_recording_cases_uses_distribution(self):+ self.assertEqual(compare([measured(1, case="playback/basic", metric="warm-up-s")], [measured(2, case="playback/basic", metric="warm-up-s")])[0], "INSUFFICIENT")++ def test_fresh_crash_counts_use_frame_resolution(self):+ base = [measured(176, case="crash/recorder-kill/oop/random0", cls="loss", metric="frames") for _ in range(3)]+ for value, expected in [(177, "OVERLAP"), (178, "WORSE")]:+ main = [measured(value, case="crash/recorder-kill/oop/random0", cls="loss", metric="frames") for _ in range(3)]+ self.assertEqual(compare(base, main)[0], expected)++ def test_missing_metric_is_insufficient_not_zero(self):+ base = [measured(15) for _ in range(3)]+ base[0]["measures"] = {}+ self.assertEqual(compare(base, [measured(17) for _ in range(3)])[0], "INSUFFICIENT")++ def test_new_fresh_class_needs_consistent_occurrence(self):+ base = [dict(record(), case="crash/example") for _ in range(3)]+ main = [measured(1, case="crash/example", cls="unexpected", metric="count") for _ in range(3)]+ self.assertEqual(compare(base[:1], main[:1])[0], "INSUFFICIENT")+ self.assertEqual(compare(base, main)[0], "WORSE")+ self.assertEqual(compare(base, main[:2] + base[:1])[0], "OVERLAP")++ def test_legacy_instant_telemetry_preserves_signed_interval(self):+ rec = measured(-45.9, case="export/instant/output/mp4/native", cls="sync-system")+ rec["measures"]["sync-mic"] = {"offset-ms": [14.0, 14.2]}+ rec["measures"]["sync-system"]["offset-ms"] = [-46.0, -45.8]+ actual = diff.normalized_measures(rec)["sync-system"]+ self.assertEqual(actual["mic-offset-ms"], [14.0, 14.2])+ self.assertEqual(actual["system-minus-mic-ms"], [-60.2, -59.8])+ self.assertNotIn("mic-offset-ms", rec["measures"]["sync-system"])++ def test_legacy_normalization_does_not_override_measurements_or_invent_sources(self):+ rec = measured(8.5, case="export/instant/output/mp4/native", cls="sync-system")+ self.assertNotIn("mic-offset-ms", diff.normalized_measures(rec)["sync-system"])+ rec["measures"]["sync-mic"] = {"offset-ms": [15.1, 15.1]}+ rec["measures"]["sync-system"]["system-minus-mic-ms"] = [-6.7, -6.7]+ self.assertEqual(diff.normalized_measures(rec)["sync-system"]["system-minus-mic-ms"], [-6.7, -6.7])+ rec["unobserved"] = {"sync-mic": "track absent"}+ self.assertNotIn("mic-offset-ms", diff.normalized_measures(rec)["sync-system"])++ def test_legacy_and_current_instant_records_compare_equally(self):+ legacy = measured(8.5, case="export/instant/output/mp4/native", cls="sync-system")+ legacy["measures"]["sync-mic"] = {"offset-ms": [15.1, 15.1]}+ current = json.loads(json.dumps(legacy))+ current["measures"]["sync-system"].update({"mic-offset-ms": [15.1, 15.1], "system-minus-mic-ms": [-6.6, -6.6]})+ self.assertEqual(compare([legacy] * 3, [current] * 3), ("EQUAL", []))+def test_lost_coverage_is_worse_for_passing_and_failing_baselines(self):main = record(unobserved={"sync-screen-mic": "track absent"})for failed in [False, True]: