Skip to content

armgit / cra

Public Imported from GitHub
0

The open-source Arc replacement for macOS. Spaces, pinned tabs, split view, Little Windows and Arc's shortcuts on Chromium, with your Arc sidebar imported in one go.

Mutation testing fits its hour again: a shuffled sample, a new one each week

Victor5e79e3ab4fdf
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index a610c5d..5038304 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -112,5 +112,9 @@ jobs:
timeout-minutes: 90
steps:
- uses: actions/checkout@v7
+ # About a thousand mutants are more than the job's hour: a shuffled sample (a new order each
+ # ISO week) runs until the time is up, and the score is over the ones it reached.
- name: Mutants of the pure code (70 % must be caught)
+ env:
+ CRA_MUTATION_MINUTES: "70"
run: scripts/mutate.py
diff --git a/docs/development.md b/docs/development.md
index f076f47..4d0214e 100644
--- a/docs/development.md
+++ b/docs/development.md
@@ -73,7 +73,7 @@ the save panel.
scripts/coverage.py # line coverage, and the changed-lines gate against origin/main
scripts/coverage.py --base <ref> # the same against another base (CI passes the PR's)
scripts/fuzz.sh # mutated importer fixtures under AddressSanitizer
- scripts/mutate.py [word] # mutation testing of the pure code, minutes; a word limits it
+ scripts/mutate.py [word] # mutation testing of the pure code, about an hour; a word limits it
[testing.md](testing.md) has the rules, the layers, the gates and how to write a scenario.
diff --git a/docs/testing.md b/docs/testing.md
index 2c05453..b8fe405 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -57,7 +57,8 @@ no page content), so it checks geometry, ink and fills.
- **CraCore ≥ 85 % of lines.** There is no target for the whole repo: one would be met by
testing the easy files.
- **Mutation score ≥ 70 %** (`scripts/mutate.py`, weekly): one operator flipped at a time in
- the pure code must fail a test.
+ the pure code must fail a test. About a thousand mutants are more than the CI job's hour, so
+ it tests a shuffled sample for 70 minutes (`CRA_MUTATION_MINUTES`), a new order each ISO week.
- **Scenario invariants**, the same for every scenario: the app is still running at the end,
no new crash report for its processes, no `fault:` line in `cra.log`, the sidebar sound
(`SidebarIntegrity`), main-thread stalls inside the scenario's budget, Cra's memory inside
diff --git a/scripts/mutate.py b/scripts/mutate.py
index 910bff8..58d5554 100755
--- a/scripts/mutate.py
+++ b/scripts/mutate.py
@@ -6,14 +6,22 @@ drops a !) in one of the files below and runs the tests that cover that file. A
tests catch is killed; one they don't survives, and points at a decision nothing checks.
Below 70 % killed fails. Weekly in CI (scripts take minutes: every mutant is a compile).
+There are about a thousand mutants, more than a CI job's hour holds, so they are tested in a
+shuffled order until CRA_MUTATION_MINUTES (default: no limit) runs out, and the score is over the
+ones tested. CRA_MUTATION_SEED picks the order (default: the ISO week), so each week's run samples
+a different part of the code and a rerun in the same week repeats it.
+
scripts/mutate.py every target
scripts/mutate.py DevToolsToggle targets whose path contains the word
"""
+import datetime
import glob
import os
+import random
import re
import subprocess
import sys
+import time
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
MIN_SCORE = 70.0
@@ -53,48 +61,58 @@ def mutants(text):
def main():
words = [a for a in sys.argv[1:] if not a.startswith("-")]
- killed, survived, invalid = 0, [], 0
+ minutes = float(os.environ.get("CRA_MUTATION_MINUTES") or 0)
+ deadline = time.monotonic() + minutes * 60 if minutes else None
+ year, week, _ = datetime.date.today().isocalendar()
+ seed = os.environ.get("CRA_MUTATION_SEED") or f"{year}-W{week:02d}"
+ work = []
for rel, cmd in TARGETS:
if words and not any(w in rel for w in words):
continue
path = os.path.join(ROOT, rel)
- if not os.path.exists(path):
- continue
- original = open(path, encoding="utf-8").read()
- baseline = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
- if baseline.returncode != 0:
- sys.exit(f"{' '.join(cmd)} fails before any mutation:\n{baseline.stdout[-1500:]}{baseline.stderr[-1500:]}")
- k = s = 0
+ if os.path.exists(path):
+ original = open(path, encoding="utf-8").read()
+ work += [(rel, cmd, original, line, what, text) for line, what, text in mutants(original)]
+ random.Random(seed).shuffle(work)
+ print(f"{len(work)} mutants, order {seed}" + (f", {minutes:g} minutes" if minutes else ""), flush=True)
+ killed, survived, invalid, checked = 0, [], 0, set()
+ for rel, cmd, original, line, what, text in work:
+ if deadline and time.monotonic() > deadline:
+ break
+ if rel not in checked:
+ baseline = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
+ if baseline.returncode != 0:
+ sys.exit(f"{' '.join(cmd)} fails before any mutation:\n{baseline.stdout[-1500:]}{baseline.stderr[-1500:]}")
+ checked.add(rel)
+ path = os.path.join(ROOT, rel)
try:
- for line, what, text in mutants(original):
- with open(path, "w", encoding="utf-8") as f:
- f.write(text)
- try:
- r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=300)
- out = r.stdout + r.stderr
- except subprocess.TimeoutExpired:
- k += 1 # a mutant that hangs the tests is caught
- continue
- if r.returncode == 0:
- s += 1
- survived.append(f"{rel}:{line}: {what}")
- elif "error:" in out and "FAIL:" not in out and "failed" not in out.lower().split("error:")[0][-200:]:
- invalid += 1 # did not compile: not a mutant
- else:
- k += 1
+ with open(path, "w", encoding="utf-8") as f:
+ f.write(text)
+ try:
+ r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=300)
+ out = r.stdout + r.stderr
+ except subprocess.TimeoutExpired:
+ killed += 1 # a mutant that hangs the tests is caught
+ continue
+ if r.returncode == 0:
+ survived.append(f"{rel}:{line}: {what}")
+ print(f"survived {rel}:{line}: {what}", flush=True)
+ elif "error:" in out and "FAIL:" not in out and "failed" not in out.lower().split("error:")[0][-200:]:
+ invalid += 1 # did not compile: not a mutant
+ else:
+ killed += 1
finally:
with open(path, "w", encoding="utf-8") as f:
f.write(original)
- killed += k
- print(f"{rel}: {k} killed, {s} survived")
total = killed + len(survived)
score = 100.0 * killed / total if total else 100.0
- print(f"\nMutation score: {score:.1f} % ({killed} of {total} killed, {invalid} did not compile)")
- for s in survived:
- print(f" survived {s}")
+ left = len(work) - total - invalid
+ print(f"\nMutation score: {score:.1f} % ({killed} of {total} killed, {invalid} did not compile"
+ + (f", {left} not reached in {minutes:g} minutes" if left else "") + ")", flush=True)
if os.environ.get("GITHUB_STEP_SUMMARY"):
with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f:
- f.write(f"## Mutation testing\n\n{score:.1f} % killed ({killed} of {total})\n\n" + "".join(f"- survived {s}\n" for s in survived))
+ f.write(f"## Mutation testing\n\n{score:.1f} % killed ({killed} of {total}, order {seed}"
+ + (f", {left} not reached" if left else "") + ")\n\n" + "".join(f"- survived {s}\n" for s in survived))
return 0 if score >= MIN_SCORE else 1