The open-source Arc replacement for macOS. Spaces, pinned tabs, split view, Little Windows and Arc's shortcuts on Chromium, with your Arc sidebar imported in one go.
Mutation testing fits its hour again: a shuffled sample, a new one each week
Victor5e79e3ab4fdf
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.ymlindex a610c5d..5038304 100644--- a/.github/workflows/nightly.yml+++ b/.github/workflows/nightly.yml@@ -112,5 +112,9 @@ jobs:timeout-minutes: 90steps:- uses: actions/checkout@v7+ # About a thousand mutants are more than the job's hour: a shuffled sample (a new order each+ # ISO week) runs until the time is up, and the score is over the ones it reached.- name: Mutants of the pure code (70 % must be caught)+ env:+ CRA_MUTATION_MINUTES: "70"run: scripts/mutate.pydiff --git a/docs/development.md b/docs/development.mdindex f076f47..4d0214e 100644--- a/docs/development.md+++ b/docs/development.md@@ -73,7 +73,7 @@ the save panel.scripts/coverage.py # line coverage, and the changed-lines gate against origin/mainscripts/coverage.py --base <ref> # the same against another base (CI passes the PR's)scripts/fuzz.sh # mutated importer fixtures under AddressSanitizer- scripts/mutate.py [word] # mutation testing of the pure code, minutes; a word limits it+ scripts/mutate.py [word] # mutation testing of the pure code, about an hour; a word limits it[testing.md](testing.md) has the rules, the layers, the gates and how to write a scenario.diff --git a/docs/testing.md b/docs/testing.mdindex 2c05453..b8fe405 100644--- a/docs/testing.md+++ b/docs/testing.md@@ -57,7 +57,8 @@ no page content), so it checks geometry, ink and fills.- **CraCore ≥ 85 % of lines.** There is no target for the whole repo: one would be met bytesting the easy files.- **Mutation score ≥ 70 %** (`scripts/mutate.py`, weekly): one operator flipped at a time in- the pure code must fail a test.+ the pure code must fail a test. About a thousand mutants are more than the CI job's hour, so+ it tests a shuffled sample for 70 minutes (`CRA_MUTATION_MINUTES`), a new order each ISO week.- **Scenario invariants**, the same for every scenario: the app is still running at the end,no new crash report for its processes, no `fault:` line in `cra.log`, the sidebar sound(`SidebarIntegrity`), main-thread stalls inside the scenario's budget, Cra's memory insidediff --git a/scripts/mutate.py b/scripts/mutate.pyindex 910bff8..58d5554 100755--- a/scripts/mutate.py+++ b/scripts/mutate.py@@ -6,14 +6,22 @@ drops a !) in one of the files below and runs the tests that cover that file. Atests catch is killed; one they don't survives, and points at a decision nothing checks.Below 70 % killed fails. Weekly in CI (scripts take minutes: every mutant is a compile).+There are about a thousand mutants, more than a CI job's hour holds, so they are tested in a+shuffled order until CRA_MUTATION_MINUTES (default: no limit) runs out, and the score is over the+ones tested. CRA_MUTATION_SEED picks the order (default: the ISO week), so each week's run samples+a different part of the code and a rerun in the same week repeats it.+scripts/mutate.py every targetscripts/mutate.py DevToolsToggle targets whose path contains the word"""+import datetimeimport globimport os+import randomimport reimport subprocessimport sys+import timeROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))MIN_SCORE = 70.0@@ -53,48 +61,58 @@ def mutants(text):def main():words = [a for a in sys.argv[1:] if not a.startswith("-")]- killed, survived, invalid = 0, [], 0+ minutes = float(os.environ.get("CRA_MUTATION_MINUTES") or 0)+ deadline = time.monotonic() + minutes * 60 if minutes else None+ year, week, _ = datetime.date.today().isocalendar()+ seed = os.environ.get("CRA_MUTATION_SEED") or f"{year}-W{week:02d}"+ work = []for rel, cmd in TARGETS:if words and not any(w in rel for w in words):continuepath = os.path.join(ROOT, rel)- if not os.path.exists(path):- continue- original = open(path, encoding="utf-8").read()- baseline = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)- if baseline.returncode != 0:- sys.exit(f"{' '.join(cmd)} fails before any mutation:\n{baseline.stdout[-1500:]}{baseline.stderr[-1500:]}")- k = s = 0+ if os.path.exists(path):+ original = open(path, encoding="utf-8").read()+ work += [(rel, cmd, original, line, what, text) for line, what, text in mutants(original)]+ random.Random(seed).shuffle(work)+ print(f"{len(work)} mutants, order {seed}" + (f", {minutes:g} minutes" if minutes else ""), flush=True)+ killed, survived, invalid, checked = 0, [], 0, set()+ for rel, cmd, original, line, what, text in work:+ if deadline and time.monotonic() > deadline:+ break+ if rel not in checked:+ baseline = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)+ if baseline.returncode != 0:+ sys.exit(f"{' '.join(cmd)} fails before any mutation:\n{baseline.stdout[-1500:]}{baseline.stderr[-1500:]}")+ checked.add(rel)+ path = os.path.join(ROOT, rel)try:- for line, what, text in mutants(original):- with open(path, "w", encoding="utf-8") as f:- f.write(text)- try:- r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=300)- out = r.stdout + r.stderr- except subprocess.TimeoutExpired:- k += 1 # a mutant that hangs the tests is caught- continue- if r.returncode == 0:- s += 1- survived.append(f"{rel}:{line}: {what}")- elif "error:" in out and "FAIL:" not in out and "failed" not in out.lower().split("error:")[0][-200:]:- invalid += 1 # did not compile: not a mutant- else:- k += 1+ with open(path, "w", encoding="utf-8") as f:+ f.write(text)+ try:+ r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=300)+ out = r.stdout + r.stderr+ except subprocess.TimeoutExpired:+ killed += 1 # a mutant that hangs the tests is caught+ continue+ if r.returncode == 0:+ survived.append(f"{rel}:{line}: {what}")+ print(f"survived {rel}:{line}: {what}", flush=True)+ elif "error:" in out and "FAIL:" not in out and "failed" not in out.lower().split("error:")[0][-200:]:+ invalid += 1 # did not compile: not a mutant+ else:+ killed += 1finally:with open(path, "w", encoding="utf-8") as f:f.write(original)- killed += k- print(f"{rel}: {k} killed, {s} survived")total = killed + len(survived)score = 100.0 * killed / total if total else 100.0- print(f"\nMutation score: {score:.1f} % ({killed} of {total} killed, {invalid} did not compile)")- for s in survived:- print(f" survived {s}")+ left = len(work) - total - invalid+ print(f"\nMutation score: {score:.1f} % ({killed} of {total} killed, {invalid} did not compile"+ + (f", {left} not reached in {minutes:g} minutes" if left else "") + ")", flush=True)if os.environ.get("GITHUB_STEP_SUMMARY"):with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:- f.write(f"## Mutation testing\n\n{score:.1f} % killed ({killed} of {total})\n\n" + "".join(f"- survived {s}\n" for s in survived))+ f.write(f"## Mutation testing\n\n{score:.1f} % killed ({killed} of {total}, order {seed}"+ + (f", {left} not reached" if left else "") + ")\n\n" + "".join(f"- survived {s}\n" for s in survived))return 0 if score >= MIN_SCORE else 1