Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: compare fresh timing measurements across independent runs

Richie McIlroy5fa33b019e52
diff --git a/scripts/verify/diff.py b/scripts/verify/diff.py
index 712f36a8b..2e88142c7 100644
--- a/scripts/verify/diff.py
+++ b/scripts/verify/diff.py
@@ -1,6 +1,7 @@
import argparse
import fnmatch
import json
+import math
import os
import re
import sys
@@ -107,6 +108,43 @@ def magnitude(rng):
return None
+def normalized_measures(record):
+ measures = {cls: dict(metrics or {}) for cls, metrics in (record.get("measures") or {}).items()}
+ if record.get("case") != "export/instant/output/mp4/native":
+ return measures
+ unobserved = record.get("unobserved") or {}
+ if "sync-mic" in unobserved or "sync-system" in unobserved:
+ return measures
+ mic = measures.get("sync-mic", {}).get("offset-ms")
+ system = measures.get("sync-system", {}).get("offset-ms")
+ if mic is not None and system is not None:
+ # Historical source offsets are already rounded to 0.1 ms. Subtract their
+ # signed intervals without inventing precision or replacing newer telemetry.
+ metrics = measures["sync-system"]
+ metrics.setdefault("mic-offset-ms", mic)
+ metrics.setdefault("system-minus-mic-ms", [round(system[0] - mic[1], 1), round(system[1] - mic[0], 1)])
+ return measures
+
+
+def fresh_fixture(case):
+ return case.startswith(("recording/", "crash/", "export/instant", "export/rec-"))
+
+
+def timing_resolution(metric):
+ if metric.endswith("-ms"):
+ return 0.1
+ if metric == "secs":
+ return 0.01
+ if metric.endswith("-s"):
+ return 1e-9
+ return None
+
+
+def metric_resolution(metric):
+ resolution = timing_resolution(metric)
+ return resolution if resolution is not None else 1.0
+
+
def summarize(recs):
ran = [r for r in recs if str(r.get("outcome", "")).lower() != "unsupported"]
decisive = [r for r in ran if str(r.get("outcome", "")).lower() != "inconclusive"] or ran
@@ -124,7 +162,7 @@ def summarize(recs):
for cls in classes:
class_runs[cls] += 1
for r in group:
- for cls, metrics in (r.get("measures") or {}).items():
+ for cls, metrics in normalized_measures(r).items():
for metric, rng in (metrics or {}).items():
value = magnitude(rng)
if value is None:
@@ -171,6 +209,11 @@ def larger(a, b, rel, abs_):
return a > b + max(abs_, rel * abs(b))
+def beyond_resolution(a, b, rel, abs_):
+ limit = b + max(abs_, rel * abs(b))
+ return a > limit and not math.isclose(a, limit, rel_tol=1e-12, abs_tol=1e-12)
+
+
def compare_case(b, m, rel, abs_):
worse, better, pending, overlap = [], [], [], []
@@ -184,6 +227,14 @@ def compare_case(b, m, rel, abs_):
lost = coverage_lost(b, m)
worse.extend(f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost)
bc, mc = b["classes"], m["classes"]
+ fresh = fresh_fixture(b["case"]) or fresh_fixture(m["case"])
+ if fresh and (b["runs"] < 3 or m["runs"] < 3):
+ pending.append("fresh fixture comparison requires at least 3 independent runs per side on this OS image")
+ sampled_classes = {
+ cls for cls in set(bc) | set(mc)
+ if fresh or any(timing_resolution(metric) is not None for metric in
+ set(b["measures"].get(cls, {})) | set(m["measures"].get(cls, {})))
+ }
stochastic = "missing-after-hold"
distribution_classes = (set(bc) | set(mc)) & {stochastic}
if b["case"].startswith("recording/") or m["case"].startswith("recording/"):
@@ -202,13 +253,23 @@ def compare_case(b, m, rel, abs_):
elif bv != mv:
overlap.append(description + " (ranges overlap)")
for cls in sorted(set(mc) - set(bc) - set(gained) - {stochastic}):
- worse.append(f"+{cls}")
+ if cls in sampled_classes and (b["runs"] < 3 or m["runs"] < 3):
+ pending.append(f"+{cls} (requires at least 3 independent runs per side on this OS image)")
+ elif cls in sampled_classes and mc[cls] != m["runs"]:
+ overlap.append(f"+{cls} in {mc[cls]}/{m['runs']} main runs (ranges overlap)")
+ else:
+ worse.append(f"+{cls}")
for cls in sorted(set(bc) - set(mc) - {stochastic}):
if cls in m["unobserved"]:
continue
- better.append(f"-{cls}")
+ if cls in sampled_classes and (b["runs"] < 3 or m["runs"] < 3):
+ pending.append(f"-{cls} (requires at least 3 independent runs per side on this OS image)")
+ elif cls in sampled_classes and bc[cls] != b["runs"]:
+ overlap.append(f"-{cls} in {bc[cls]}/{b['runs']} baseline runs (ranges overlap)")
+ else:
+ better.append(f"-{cls}")
for cls in sorted((set(bc) & set(mc)) - {stochastic}):
- if b["runs"] and m["runs"] and (b["runs"] > 1 or m["runs"] > 1):
+ if cls not in sampled_classes and b["runs"] and m["runs"] and (b["runs"] > 1 or m["runs"] > 1):
bf, mf = bc[cls] / b["runs"], mc[cls] / m["runs"]
if mf > bf:
worse.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}")
@@ -216,7 +277,19 @@ def compare_case(b, m, rel, abs_):
better.append(f"{cls} fails {mc[cls]}/{m['runs']} vs {bc[cls]}/{b['runs']}")
bm, mm = b["measures"].get(cls, {}), m["measures"].get(cls, {})
for metric in sorted(set(bm) | set(mm)):
- if metric in bm and metric in mm:
+ if fresh or timing_resolution(metric) is not None:
+ bv, mv = distribution(b, cls, metric), distribution(m, cls, metric)
+ resolution = metric_resolution(metric)
+ description = f"{cls}.{metric} baseline={bv}, main={mv}, resolution={resolution:g}"
+ if len(bv) < 3 or len(mv) < 3:
+ pending.append(description + " (requires at least 3 independent runs per side on this OS image)")
+ elif beyond_resolution(min(mv), max(bv), rel, max(abs_, resolution)):
+ worse.append(description + " (every main value above baseline range by more than resolution)")
+ elif beyond_resolution(min(bv), max(mv), rel, max(abs_, resolution)):
+ better.append(description + " (every main value below baseline range by more than resolution)")
+ elif bv != mv:
+ overlap.append(description + " (ranges overlap within resolution)")
+ elif metric in bm and metric in mm:
if larger(mm[metric], bm[metric], rel, abs_):
worse.append(f"{cls}.{metric} {fmt(bm[metric])} -> {fmt(mm[metric])}")
elif larger(bm[metric], mm[metric], rel, abs_):
diff --git a/scripts/verify/test_diff.py b/scripts/verify/test_diff.py
index 005e00873..c5298f0a9 100644
--- a/scripts/verify/test_diff.py
+++ b/scripts/verify/test_diff.py
@@ -27,7 +27,75 @@ def compare(base, main):
return diff.compare_case(diff.summarize(base), diff.summarize(main), 0, 0)
+def measured(value, case="export/instant-gated-mic-only/output/mp4/native", cls="sync-mic", metric="offset-ms", sample=None):
+ result = record(sample=sample)
+ result.update(case=case, failures={cls: []}, measures={cls: {metric: [value, value]}})
+ return result
+
+
class ComparisonTests(unittest.TestCase):
+ def test_continuous_metrics_require_three_independent_runs(self):
+ self.assertEqual(compare([measured(15.2)], [measured(15.3)])[0], "INSUFFICIENT")
+ self.assertEqual(compare([measured(15.2, sample="same") for _ in range(3)], [measured(16) for _ in range(3)])[0], "INSUFFICIENT")
+
+ def test_continuous_overlap_and_one_resolution_step_are_not_worse(self):
+ base = [measured(v) for v in [15.1, 15.2, 15.3]]
+ for values in [[15.2, 15.3, 15.4], [15.4, 15.4, 15.4]]:
+ with self.subTest(values=values):
+ self.assertEqual(compare(base, [measured(v) for v in values])[0], "OVERLAP")
+
+ def test_continuous_separated_ranges_are_worse(self):
+ self.assertEqual(compare([measured(v) for v in [15.1, 15.2, 15.3]], [measured(v) for v in [15.5, 15.6, 15.5]])[0], "WORSE")
+
+ def test_signed_offsets_compare_magnitude(self):
+ self.assertEqual(compare([measured(-15) for _ in range(3)], [measured(-17) for _ in range(3)])[0], "WORSE")
+
+ def test_timing_metric_outside_recording_cases_uses_distribution(self):
+ self.assertEqual(compare([measured(1, case="playback/basic", metric="warm-up-s")], [measured(2, case="playback/basic", metric="warm-up-s")])[0], "INSUFFICIENT")
+
+ def test_fresh_crash_counts_use_frame_resolution(self):
+ base = [measured(176, case="crash/recorder-kill/oop/random0", cls="loss", metric="frames") for _ in range(3)]
+ for value, expected in [(177, "OVERLAP"), (178, "WORSE")]:
+ main = [measured(value, case="crash/recorder-kill/oop/random0", cls="loss", metric="frames") for _ in range(3)]
+ self.assertEqual(compare(base, main)[0], expected)
+
+ def test_missing_metric_is_insufficient_not_zero(self):
+ base = [measured(15) for _ in range(3)]
+ base[0]["measures"] = {}
+ self.assertEqual(compare(base, [measured(17) for _ in range(3)])[0], "INSUFFICIENT")
+
+ def test_new_fresh_class_needs_consistent_occurrence(self):
+ base = [dict(record(), case="crash/example") for _ in range(3)]
+ main = [measured(1, case="crash/example", cls="unexpected", metric="count") for _ in range(3)]
+ self.assertEqual(compare(base[:1], main[:1])[0], "INSUFFICIENT")
+ self.assertEqual(compare(base, main)[0], "WORSE")
+ self.assertEqual(compare(base, main[:2] + base[:1])[0], "OVERLAP")
+
+ def test_legacy_instant_telemetry_preserves_signed_interval(self):
+ rec = measured(-45.9, case="export/instant/output/mp4/native", cls="sync-system")
+ rec["measures"]["sync-mic"] = {"offset-ms": [14.0, 14.2]}
+ rec["measures"]["sync-system"]["offset-ms"] = [-46.0, -45.8]
+ actual = diff.normalized_measures(rec)["sync-system"]
+ self.assertEqual(actual["mic-offset-ms"], [14.0, 14.2])
+ self.assertEqual(actual["system-minus-mic-ms"], [-60.2, -59.8])
+ self.assertNotIn("mic-offset-ms", rec["measures"]["sync-system"])
+
+ def test_legacy_normalization_does_not_override_measurements_or_invent_sources(self):
+ rec = measured(8.5, case="export/instant/output/mp4/native", cls="sync-system")
+ self.assertNotIn("mic-offset-ms", diff.normalized_measures(rec)["sync-system"])
+ rec["measures"]["sync-mic"] = {"offset-ms": [15.1, 15.1]}
+ rec["measures"]["sync-system"]["system-minus-mic-ms"] = [-6.7, -6.7]
+ self.assertEqual(diff.normalized_measures(rec)["sync-system"]["system-minus-mic-ms"], [-6.7, -6.7])
+ rec["unobserved"] = {"sync-mic": "track absent"}
+ self.assertNotIn("mic-offset-ms", diff.normalized_measures(rec)["sync-system"])
+
+ def test_legacy_and_current_instant_records_compare_equally(self):
+ legacy = measured(8.5, case="export/instant/output/mp4/native", cls="sync-system")
+ legacy["measures"]["sync-mic"] = {"offset-ms": [15.1, 15.1]}
+ current = json.loads(json.dumps(legacy))
+ current["measures"]["sync-system"].update({"mic-offset-ms": [15.1, 15.1], "system-minus-mic-ms": [-6.6, -6.6]})
+ self.assertEqual(compare([legacy] * 3, [current] * 3), ("EQUAL", []))
+
def test_lost_coverage_is_worse_for_passing_and_failing_baselines(self):
main = record(unobserved={"sync-screen-mic": "track absent"})
for failed in [False, True]: