Open source Loom alternative. Beautiful, shareable screen recordings.
Rust appcapcossloommacnextjsnextjs15open-sourceossreactrecordrustscreen-capturescreen-recorderscreenshotsolidjstauritauri-apptypescriptvite
fix: report lost baseline coverage as worse [skip ci]
Richie McIlroyd6c00b5eb85f
diff --git a/scripts/verify/diff.py b/scripts/verify/diff.pyindex f7f63d1d3..712f36a8b 100644--- a/scripts/verify/diff.py+++ b/scripts/verify/diff.py@@ -151,6 +151,10 @@ def coverage_gained(b, m):return sorted(b["unobserved"] - m["unobserved"]) if m["ran"] else []+def coverage_lost(b, m):+ return sorted(m["unobserved"] - b["unobserved"]) if b["ran"] and m["ran"] else []++def distribution(summary, cls, metric):values = []for sample in summary["samples"]:@@ -177,12 +181,14 @@ def compare_case(b, m, rel, abs_):return "N/A", ["unsupported on baseline: " + "; ".join(b.get("why", [])[:1])]if not b["ran"] and not m["ran"]:return "EQUAL", []+ lost = coverage_lost(b, m)+ worse.extend(f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost)bc, mc = b["classes"], m["classes"]stochastic = "missing-after-hold"distribution_classes = (set(bc) | set(mc)) & {stochastic}if b["case"].startswith("recording/") or m["case"].startswith("recording/"):distribution_classes.add(stochastic)- for cls in sorted(distribution_classes):+ for cls in sorted(distribution_classes - set(lost)):metrics = {"count"} | set(b["measures"].get(cls, {})) | set(m["measures"].get(cls, {}))for metric in sorted(metrics):bv, mv = distribution(b, cls, metric), distribution(m, cls, metric)@@ -252,8 +258,8 @@ def main():out = {}print(f"# {args.title}\n")print(f"tolerance: rel {args.rel_tol}, abs {args.abs_tol}\n")- print("| lane | cases base/main | WORSE | BETTER | EQUAL | coverage gained | insufficient | overlapping distributions | n/a on base | only base | only main | junit WORSE | junit BETTER |")- print("|---|---|---|---|---|---|---|---|---|---|---|---|---|")+ print("| lane | cases base/main | WORSE | BETTER | EQUAL | coverage gained | coverage lost | insufficient | overlapping distributions | n/a on base | only base | only main | junit WORSE | junit BETTER |")+ print("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|")detail = []for lane in lanes:b, m = brec.get(lane, {}), mrec.get(lane, {})@@ -278,6 +284,9 @@ def main():gained = coverage_gained(bs, ms)if gained and verdict != "COVERAGE-GAINED":res["COVERAGE-GAINED"].append((key, [f"{cls}: baseline unobserved -> main observed" for cls in gained]))+ lost = coverage_lost(bs, ms)+ if lost:+ res["COVERAGE-LOST"].append((key, [f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost]))bj, mj = bjunit.get(lane, {}), mjunit.get(lane, {})jworse, jbetter = [], []bad = {"fail", "error"}@@ -298,12 +307,12 @@ def main():jworse.append((test, ["fail -> error (crash/timeout)"]))print(f"| {lane} | {len(b)}/{len(m)} | {len(res['WORSE'])} | {len(res['BETTER'])} | {len(res['EQUAL'])}"- f" | {len(res['COVERAGE-GAINED'])} | {len(res['INSUFFICIENT'])} | {len(res['OVERLAP'])} | {len(res['N/A'])} | {len(res['ONLY-BASE'])} | {len(res['ONLY-MAIN'])} | {len(jworse)} | {len(jbetter)} |"+ f" | {len(res['COVERAGE-GAINED'])} | {len(res['COVERAGE-LOST'])} | {len(res['INSUFFICIENT'])} | {len(res['OVERLAP'])} | {len(res['N/A'])} | {len(res['ONLY-BASE'])} | {len(res['ONLY-MAIN'])} | {len(jworse)} | {len(jbetter)} |")out[lane] = {k: v for k, v in res.items()}out[lane]["JUNIT-WORSE"] = jworseout[lane]["JUNIT-BETTER"] = jbetter- for label in ("WORSE", "BETTER", "COVERAGE-GAINED", "INSUFFICIENT", "OVERLAP", "JUNIT-WORSE", "JUNIT-BETTER", "N/A", "ONLY-BASE", "ONLY-MAIN"):+ for label in ("WORSE", "BETTER", "COVERAGE-GAINED", "COVERAGE-LOST", "INSUFFICIENT", "OVERLAP", "JUNIT-WORSE", "JUNIT-BETTER", "N/A", "ONLY-BASE", "ONLY-MAIN"):items = out[lane].get(label) or []if not items:continuediff --git a/scripts/verify/test_diff.py b/scripts/verify/test_diff.pyindex 8b6fe3963..005e00873 100644--- a/scripts/verify/test_diff.py+++ b/scripts/verify/test_diff.py@@ -1,4 +1,8 @@import importlib.util+import json+import subprocess+import sys+import tempfileimport unittestfrom pathlib import Path@@ -24,6 +28,60 @@ def compare(base, main):class ComparisonTests(unittest.TestCase):+ def test_lost_coverage_is_worse_for_passing_and_failing_baselines(self):+ main = record(unobserved={"sync-screen-mic": "track absent"})+ for failed in [False, True]:+ with self.subTest(failed=failed):+ base = record()+ if failed:+ base["failures"] = {"sync-screen-mic": []}+ verdict, reasons = compare([base], [main])+ self.assertEqual(verdict, "WORSE")+ self.assertEqual(reasons, ["sync-screen-mic: baseline observed -> main unobserved (lost coverage)"])++ def test_coverage_gain_does_not_hide_simultaneous_loss(self):+ base = record(unobserved={"sync-screen-mic": "track absent"})+ main = record(unobserved={"sync-screen-system": "track absent"})+ bs, ms = diff.summarize([base]), diff.summarize([main])+ self.assertEqual(diff.compare_case(bs, ms, 0, 0)[0], "WORSE")+ self.assertEqual(diff.coverage_gained(bs, ms), ["sync-screen-mic"])+ self.assertEqual(diff.coverage_lost(bs, ms), ["sync-screen-system"])++ def test_partial_baseline_observation_still_detects_total_loss(self):+ unseen = record(unobserved={"sync-screen-mic": "track absent"})+ self.assertEqual(compare([unseen, record()], [unseen, unseen])[0], "WORSE")++ def test_partial_main_observation_is_not_total_loss(self):+ unseen = record(unobserved={"sync-screen-mic": "track absent"})+ self.assertEqual(diff.coverage_lost(diff.summarize([record()]), diff.summarize([unseen, record()])), [])++ def test_unobserved_on_both_sides_is_not_lost_coverage(self):+ unseen = record(unobserved={"sync-screen-mic": "track absent"})+ self.assertEqual(compare([unseen], [unseen]), ("EQUAL", []))++ def test_lost_distribution_coverage_is_worse_instead_of_insufficient(self):+ unseen = record(unobserved={"missing-after-hold": "track absent"})+ self.assertEqual(compare([record(1)], [unseen])[0], "WORSE")++ def test_cli_reports_coverage_lost_and_gained_separately(self):+ with tempfile.TemporaryDirectory() as root:+ root = Path(root)+ base = record(unobserved={"sync-screen-mic": "track absent"})+ main = record(unobserved={"sync-screen-system": "track absent"})+ for side, rec in [("base", base), ("main", main)]:+ path = root / side / "verify-report-macos-fast"+ path.mkdir(parents=True)+ rec.update({"suite": "export", "case": "a"})+ (path / "records.jsonl").write_text(json.dumps(rec) + "\n")+ result = subprocess.run(+ [sys.executable, "-B", str(Path(__file__).with_name("diff.py")), str(root / "base"), str(root / "main"), "--json", str(root / "diff.json")],+ check=True, capture_output=True, text=True,+ )+ self.assertIn("| coverage gained | coverage lost |", result.stdout)+ output = json.loads((root / "diff.json").read_text())["macos"]+ for category in ["WORSE", "COVERAGE-LOST", "COVERAGE-GAINED"]:+ self.assertEqual(output[category][0][0], "export::a")+def test_absent_track_is_coverage_gained_even_when_main_fails(self):base = record(unobserved={"sync-screen-mic": "track absent"})main = record()