SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.
Mutation testing: judge kernel mutants by the black-box suites (#2660)
Jackson Ourslandbc68e14c3e8e
diff --git a/.cargo/mutants.toml b/.cargo/mutants.tomlnew file mode 100644index 0000000000..2cc2da3ea4--- /dev/null+++ b/.cargo/mutants.toml@@ -0,0 +1,14 @@+# cargo-mutants (docs/mutation-testing.md). Mutants are judged by the black-box suites that will+# outlive the kernel backend: the reference tests (tests/oracle, through the oracle_cases test)+# and the pull-request sample of the robustness goldens (every 8th seed, as ci.yml). The+# kernel's own unit tests are left out on purpose: a mutant only they catch is a behaviour the+# backend-neutral suites would miss when the kernel is swapped.+profile = "mutants"+additional_cargo_test_args = [+ "--test", "oracle_cases", "--lib", "--", "--include-ignored",+ "oracle_suite",+ "robustness_goldens::seed_00", "robustness_goldens::seed_08", "robustness_goldens::seed_16",+ "robustness_goldens::seed_24", "robustness_goldens::seed_32", "robustness_goldens::seed_40",+ "robustness_goldens::seed_48", "robustness_goldens::seed_56", "robustness_goldens::seed_64",+ "robustness_goldens::seed_72", "robustness_goldens::seed_80", "robustness_goldens::seed_88",+]diff --git a/.gitignore b/.gitignoreindex 0a9913f05c..8e03052511 100644--- a/.gitignore+++ b/.gitignore@@ -7,3 +7,6 @@/.claude/worktrees//target-snapshot__pycache__/++# cargo-mutants output (docs/mutation-testing.md)+mutants.out*/diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 971144e4e1..b4567e198c 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -148,6 +148,8 @@ the most complete description of the house style.A geometry change that changes the robustness goldens re-records them in the same pull request:`ROBUSTNESS_GOLDENS_BLESS="why it changed" cargo test --profile ci -p solidcraft-engine --lib robustness_goldens::`(see [docs/robustness.md](docs/robustness.md)); CI runs every seed on pull requests that touch geometry.+[docs/mutation-testing.md](docs/mutation-testing.md) shows how to find kernel behaviour these+black-box suites miss.**UI scenarios** drive the real app headless: start a tool, click a plane or a sketch point,press keys, then assert on the result.diff --git a/Cargo.toml b/Cargo.tomlindex ba9d538400..6369851fe4 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -142,3 +142,9 @@ matext4cgmath = { path = "vendor/matext4cgmath" }# Keep OpenGL pipelines, resolve textures and CAD callbacks at the same safe sample count.egui-wgpu = { path = "vendor/egui-wgpu" }++# Mutation testing (cargo-mutants, docs/mutation-testing.md): the ci profile with incremental+# builds, since every mutant rebuilds the crates above the one it changes.+[profile.mutants]+inherits = "ci"+incremental = truediff --git a/apps/solidcraft-cli/tests/oracle_cases.rs b/apps/solidcraft-cli/tests/oracle_cases.rsnew file mode 100644index 0000000000..76b0627633--- /dev/null+++ b/apps/solidcraft-cli/tests/oracle_cases.rs@@ -0,0 +1,66 @@+//! The reference tests (tests/oracle, docs/oracle.md) as a `cargo test`, so tools that drive+//! `cargo test` (cargo-mutants) can use them: every case with expected values and no+//! `known_failure` mark must pass, as `cargo xtask oracle` requires.++use std::path::Path;+use std::process::Command;++use serde_json::Value;++#[test]+#[ignore = "run by cargo-mutants (.cargo/mutants.toml); CI replays the oracle through cargo xtask oracle"]+fn oracle_suite() {+ let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");+ let dir = root.join("tests/oracle");+ let mut cases: Vec<String> = std::fs::read_dir(&dir)+ .unwrap()+ .flatten()+ .map(|e| e.path())+ .filter(|p| p.join("recipe.json").is_file())+ .map(|p| format!("tests/oracle/{}", p.file_name().unwrap().to_string_lossy()))+ .collect();+ cases.sort();+ assert!(cases.len() >= 50, "oracle cases in {}", dir.display());+ // One process per core, cases dealt round-robin so the slow ones spread out.+ let n = std::thread::available_parallelism().map_or(4, |n| n.get()).min(cases.len());+ let children: Vec<_> = (0..n)+ .map(|i| {+ let chunk: Vec<&String> = cases.iter().skip(i).step_by(n).collect();+ Command::new(env!("CARGO_BIN_EXE_solidcraft-cli"))+ .current_dir(&root)+ .arg("oracle")+ .arg("--json")+ .args(chunk)+ .stdout(std::process::Stdio::piped())+ .stderr(std::process::Stdio::piped())+ .spawn()+ .unwrap()+ })+ .collect();+ let mut results: Vec<Value> = Vec::new();+ for child in children {+ let out = child.wait_with_output().unwrap();+ let part: Vec<Value> =+ serde_json::from_slice(&out.stdout).unwrap_or_else(|e| panic!("oracle JSON: {e}\n{}", String::from_utf8_lossy(&out.stderr)));+ results.extend(part);+ }+ assert_eq!(results.len(), cases.len());+ let mut failed = Vec::new();+ for r in &results {+ let case = r["case"].as_str().unwrap_or("");+ let want: Value =+ std::fs::read_to_string(dir.join(case).join("expected.json")).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default();+ if want.is_null() || want["known_failure"].is_string() || r["pass"].as_bool().unwrap_or(false) {+ continue;+ }+ let checks: Vec<String> = r["checks"]+ .as_array()+ .into_iter()+ .flatten()+ .filter(|c| !c["ok"].as_bool().unwrap_or(false))+ .map(|c| format!("{} got {} want {}", c["check"].as_str().unwrap_or(""), c["got"], c["want"]))+ .collect();+ failed.push(format!("{case}: {} {}", r["error"].as_str().unwrap_or(""), checks.join("; ")));+ }+ assert!(failed.is_empty(), "oracle cases failed:\n{}", failed.join("\n"));+}diff --git a/docs/mutation-testing.md b/docs/mutation-testing.mdnew file mode 100644index 0000000000..74c54f49f5--- /dev/null+++ b/docs/mutation-testing.md@@ -0,0 +1,69 @@+# Mutation testing++SolidCraft will move from truck to solidcore (#1597). The tests that have to catch regressions+across that switch are the ones that don't depend on the kernel: the reference tests+([docs/oracle.md](oracle.md)), the robustness goldens ([docs/robustness.md](robustness.md)) and+the engine's command tests. Most of the kernel's own unit tests call truck-specific code and go+away with it.++Mutation testing checks those suites. [cargo-mutants](https://mutants.rs) makes small changes to+the source, one at a time (`*` becomes `/`, a condition is negated, a function returns a default+value), rebuilds, and runs the tests. A mutant the tests don't catch is a change in behaviour+nobody would notice, and solidcore could get the same thing wrong without anyone noticing.++## What judges a mutant++`.cargo/mutants.toml` judges mutants by the black-box suites only:++- `oracle_suite` (`apps/solidcraft-cli/tests/oracle_cases.rs`): every case in `tests/oracle` with+ expected values and no `known_failure` must pass, as `cargo xtask oracle` requires. It is+ `#[ignore]`d so CI doesn't replay the oracle twice.+- The pull-request sample of the robustness goldens: every 8th seed, as `ci.yml` runs it.++The kernel's unit tests are left out on purpose. A mutant that only they catch marks a behaviour+the black-box suites would miss once the kernel is swapped.++## Running it++```+cargo install cargo-mutants --locked+CARGO_TARGET_DIR=$PWD/target cargo mutants --in-place \+ -f crates/kernel/src/measure.rs \+ --test-package solidcraft-cli --test-package solidcraft-engine \+ --baseline skip --timeout 900 --build-timeout 900+```++- Use a worktree you don't otherwise touch: `--in-place` writes each mutant into the source tree+ and restores it afterwards.+- `--shard 0/200 --sharding round-robin` with `-d crates/kernel` takes an even sample (every 200th+ mutant) of the whole crate.+- The `mutants` profile in `Cargo.toml` is the `ci` profile with incremental builds. It cuts the+ rebuild after a kernel change from about 3 minutes to about 15 seconds.+- A mutant costs 1.5–5 minutes on a 12-core Mac, mostly the tests. The kernel has about 40,000+ mutants, so run a sample or one file at a time, never the whole crate. This is a tool for+ finding gaps, not a CI gate.++Two problems with cargo-mutants 27.1 are behind the flags above:++- The baseline (the unmutated run) ignores `--test-package` and tests the mutated crate, which+ fails. Skip it with `--baseline skip`, set `--timeout` yourself, and check the unmutated tree+ passes first.+- `--re` and `--exclude-re` don't filter "delete field" mutants. Saved diffs in+ `mutants.out/diff/` name the mutation, not the file, in their `+++` line: apply one with+ `patch <file> < <diff>`.++## Pilot results (October 2026, `cb857f1f`)++| Sample | Mutants that build | Caught by the black-box suites |+|---|---:|---:|+| `crates/kernel/src/measure.rs`, every mutant | 85 | 85% |+| Whole kernel, every 200th mutant | 173 | 15% (95% interval 10–21%) |++Out of 49 of the kernel sample's survivors, the kernel's own unit tests caught 21 (43%) and+nothing caught 28 (57%). About 11% of survivors swap `>` and `>=` on floating-point values that+are never exactly equal, so they don't change behaviour.++The black-box suites are weakest on tessellation and the display mesh (0 of 19 mutants caught),+offset, shell, emboss and lip (0 of 14), blends (15%) and STEP/IGES exchange (17%). For a+survivor that a kernel unit test catches, that test's design is the starting point for a new+oracle case or engine test.