Skip to content

armgit / Cap

Public Imported from GitHub
0

Open source Loom alternative. Beautiful, shareable screen recordings.

fix: report lost baseline coverage as worse [skip ci]

Richie McIlroyd6c00b5eb85f
diff --git a/scripts/verify/diff.py b/scripts/verify/diff.py
index f7f63d1d3..712f36a8b 100644
--- a/scripts/verify/diff.py
+++ b/scripts/verify/diff.py
@@ -151,6 +151,10 @@ def coverage_gained(b, m):
return sorted(b["unobserved"] - m["unobserved"]) if m["ran"] else []
+def coverage_lost(b, m):
+ return sorted(m["unobserved"] - b["unobserved"]) if b["ran"] and m["ran"] else []
+
+
def distribution(summary, cls, metric):
values = []
for sample in summary["samples"]:
@@ -177,12 +181,14 @@ def compare_case(b, m, rel, abs_):
return "N/A", ["unsupported on baseline: " + "; ".join(b.get("why", [])[:1])]
if not b["ran"] and not m["ran"]:
return "EQUAL", []
+ lost = coverage_lost(b, m)
+ worse.extend(f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost)
bc, mc = b["classes"], m["classes"]
stochastic = "missing-after-hold"
distribution_classes = (set(bc) | set(mc)) & {stochastic}
if b["case"].startswith("recording/") or m["case"].startswith("recording/"):
distribution_classes.add(stochastic)
- for cls in sorted(distribution_classes):
+ for cls in sorted(distribution_classes - set(lost)):
metrics = {"count"} | set(b["measures"].get(cls, {})) | set(m["measures"].get(cls, {}))
for metric in sorted(metrics):
bv, mv = distribution(b, cls, metric), distribution(m, cls, metric)
@@ -252,8 +258,8 @@ def main():
out = {}
print(f"# {args.title}\n")
print(f"tolerance: rel {args.rel_tol}, abs {args.abs_tol}\n")
- print("| lane | cases base/main | WORSE | BETTER | EQUAL | coverage gained | insufficient | overlapping distributions | n/a on base | only base | only main | junit WORSE | junit BETTER |")
- print("|---|---|---|---|---|---|---|---|---|---|---|---|---|")
+ print("| lane | cases base/main | WORSE | BETTER | EQUAL | coverage gained | coverage lost | insufficient | overlapping distributions | n/a on base | only base | only main | junit WORSE | junit BETTER |")
+ print("|---|---|---|---|---|---|---|---|---|---|---|---|---|---|")
detail = []
for lane in lanes:
b, m = brec.get(lane, {}), mrec.get(lane, {})
@@ -278,6 +284,9 @@ def main():
gained = coverage_gained(bs, ms)
if gained and verdict != "COVERAGE-GAINED":
res["COVERAGE-GAINED"].append((key, [f"{cls}: baseline unobserved -> main observed" for cls in gained]))
+ lost = coverage_lost(bs, ms)
+ if lost:
+ res["COVERAGE-LOST"].append((key, [f"{cls}: baseline observed -> main unobserved (lost coverage)" for cls in lost]))
bj, mj = bjunit.get(lane, {}), mjunit.get(lane, {})
jworse, jbetter = [], []
bad = {"fail", "error"}
@@ -298,12 +307,12 @@ def main():
jworse.append((test, ["fail -> error (crash/timeout)"]))
print(
f"| {lane} | {len(b)}/{len(m)} | {len(res['WORSE'])} | {len(res['BETTER'])} | {len(res['EQUAL'])}"
- f" | {len(res['COVERAGE-GAINED'])} | {len(res['INSUFFICIENT'])} | {len(res['OVERLAP'])} | {len(res['N/A'])} | {len(res['ONLY-BASE'])} | {len(res['ONLY-MAIN'])} | {len(jworse)} | {len(jbetter)} |"
+ f" | {len(res['COVERAGE-GAINED'])} | {len(res['COVERAGE-LOST'])} | {len(res['INSUFFICIENT'])} | {len(res['OVERLAP'])} | {len(res['N/A'])} | {len(res['ONLY-BASE'])} | {len(res['ONLY-MAIN'])} | {len(jworse)} | {len(jbetter)} |"
)
out[lane] = {k: v for k, v in res.items()}
out[lane]["JUNIT-WORSE"] = jworse
out[lane]["JUNIT-BETTER"] = jbetter
- for label in ("WORSE", "BETTER", "COVERAGE-GAINED", "INSUFFICIENT", "OVERLAP", "JUNIT-WORSE", "JUNIT-BETTER", "N/A", "ONLY-BASE", "ONLY-MAIN"):
+ for label in ("WORSE", "BETTER", "COVERAGE-GAINED", "COVERAGE-LOST", "INSUFFICIENT", "OVERLAP", "JUNIT-WORSE", "JUNIT-BETTER", "N/A", "ONLY-BASE", "ONLY-MAIN"):
items = out[lane].get(label) or []
if not items:
continue
diff --git a/scripts/verify/test_diff.py b/scripts/verify/test_diff.py
index 8b6fe3963..005e00873 100644
--- a/scripts/verify/test_diff.py
+++ b/scripts/verify/test_diff.py
@@ -1,4 +1,8 @@
import importlib.util
+import json
+import subprocess
+import sys
+import tempfile
import unittest
from pathlib import Path
@@ -24,6 +28,60 @@ def compare(base, main):
class ComparisonTests(unittest.TestCase):
+ def test_lost_coverage_is_worse_for_passing_and_failing_baselines(self):
+ main = record(unobserved={"sync-screen-mic": "track absent"})
+ for failed in [False, True]:
+ with self.subTest(failed=failed):
+ base = record()
+ if failed:
+ base["failures"] = {"sync-screen-mic": []}
+ verdict, reasons = compare([base], [main])
+ self.assertEqual(verdict, "WORSE")
+ self.assertEqual(reasons, ["sync-screen-mic: baseline observed -> main unobserved (lost coverage)"])
+
+ def test_coverage_gain_does_not_hide_simultaneous_loss(self):
+ base = record(unobserved={"sync-screen-mic": "track absent"})
+ main = record(unobserved={"sync-screen-system": "track absent"})
+ bs, ms = diff.summarize([base]), diff.summarize([main])
+ self.assertEqual(diff.compare_case(bs, ms, 0, 0)[0], "WORSE")
+ self.assertEqual(diff.coverage_gained(bs, ms), ["sync-screen-mic"])
+ self.assertEqual(diff.coverage_lost(bs, ms), ["sync-screen-system"])
+
+ def test_partial_baseline_observation_still_detects_total_loss(self):
+ unseen = record(unobserved={"sync-screen-mic": "track absent"})
+ self.assertEqual(compare([unseen, record()], [unseen, unseen])[0], "WORSE")
+
+ def test_partial_main_observation_is_not_total_loss(self):
+ unseen = record(unobserved={"sync-screen-mic": "track absent"})
+ self.assertEqual(diff.coverage_lost(diff.summarize([record()]), diff.summarize([unseen, record()])), [])
+
+ def test_unobserved_on_both_sides_is_not_lost_coverage(self):
+ unseen = record(unobserved={"sync-screen-mic": "track absent"})
+ self.assertEqual(compare([unseen], [unseen]), ("EQUAL", []))
+
+ def test_lost_distribution_coverage_is_worse_instead_of_insufficient(self):
+ unseen = record(unobserved={"missing-after-hold": "track absent"})
+ self.assertEqual(compare([record(1)], [unseen])[0], "WORSE")
+
+ def test_cli_reports_coverage_lost_and_gained_separately(self):
+ with tempfile.TemporaryDirectory() as root:
+ root = Path(root)
+ base = record(unobserved={"sync-screen-mic": "track absent"})
+ main = record(unobserved={"sync-screen-system": "track absent"})
+ for side, rec in [("base", base), ("main", main)]:
+ path = root / side / "verify-report-macos-fast"
+ path.mkdir(parents=True)
+ rec.update({"suite": "export", "case": "a"})
+ (path / "records.jsonl").write_text(json.dumps(rec) + "\n")
+ result = subprocess.run(
+ [sys.executable, "-B", str(Path(__file__).with_name("diff.py")), str(root / "base"), str(root / "main"), "--json", str(root / "diff.json")],
+ check=True, capture_output=True, text=True,
+ )
+ self.assertIn("| coverage gained | coverage lost |", result.stdout)
+ output = json.loads((root / "diff.json").read_text())["macos"]
+ for category in ["WORSE", "COVERAGE-LOST", "COVERAGE-GAINED"]:
+ self.assertEqual(output[category][0][0], "export::a")
+
def test_absent_track_is_coverage_gained_even_when_main_fails(self):
base = record(unobserved={"sync-screen-mic": "track absent"})
main = record()