Skip to content

SolidCraft: an open-source parametric 3D CAD application built in pure Rust. Desktop, web, and automation through MCP.

Mutation testing: judge kernel mutants by the black-box suites (#2660)

Jackson Ourslandbc68e14c3e8e
diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml
new file mode 100644
index 0000000000..2cc2da3ea4
--- /dev/null
+++ b/.cargo/mutants.toml
@@ -0,0 +1,14 @@
+# cargo-mutants (docs/mutation-testing.md). Mutants are judged by the black-box suites that will
+# outlive the kernel backend: the reference tests (tests/oracle, through the oracle_cases test)
+# and the pull-request sample of the robustness goldens (every 8th seed, as ci.yml). The
+# kernel's own unit tests are left out on purpose: a mutant only they catch is a behaviour the
+# backend-neutral suites would miss when the kernel is swapped.
+profile = "mutants"
+additional_cargo_test_args = [
+ "--test", "oracle_cases", "--lib", "--", "--include-ignored",
+ "oracle_suite",
+ "robustness_goldens::seed_00", "robustness_goldens::seed_08", "robustness_goldens::seed_16",
+ "robustness_goldens::seed_24", "robustness_goldens::seed_32", "robustness_goldens::seed_40",
+ "robustness_goldens::seed_48", "robustness_goldens::seed_56", "robustness_goldens::seed_64",
+ "robustness_goldens::seed_72", "robustness_goldens::seed_80", "robustness_goldens::seed_88",
+]
diff --git a/.gitignore b/.gitignore
index 0a9913f05c..8e03052511 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,3 +7,6 @@
/.claude/worktrees/
/target-snapshot
__pycache__/
+
+# cargo-mutants output (docs/mutation-testing.md)
+mutants.out*/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 971144e4e1..b4567e198c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -148,6 +148,8 @@ the most complete description of the house style.
A geometry change that changes the robustness goldens re-records them in the same pull request:
`ROBUSTNESS_GOLDENS_BLESS="why it changed" cargo test --profile ci -p solidcraft-engine --lib robustness_goldens::`
(see [docs/robustness.md](docs/robustness.md)); CI runs every seed on pull requests that touch geometry.
+[docs/mutation-testing.md](docs/mutation-testing.md) shows how to find kernel behaviour these
+black-box suites miss.
**UI scenarios** drive the real app headless: start a tool, click a plane or a sketch point,
press keys, then assert on the result.
diff --git a/Cargo.toml b/Cargo.toml
index ba9d538400..6369851fe4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -142,3 +142,9 @@ matext4cgmath = { path = "vendor/matext4cgmath" }
# Keep OpenGL pipelines, resolve textures and CAD callbacks at the same safe sample count.
egui-wgpu = { path = "vendor/egui-wgpu" }
+
+# Mutation testing (cargo-mutants, docs/mutation-testing.md): the ci profile with incremental
+# builds, since every mutant rebuilds the crates above the one it changes.
+[profile.mutants]
+inherits = "ci"
+incremental = true
diff --git a/apps/solidcraft-cli/tests/oracle_cases.rs b/apps/solidcraft-cli/tests/oracle_cases.rs
new file mode 100644
index 0000000000..76b0627633
--- /dev/null
+++ b/apps/solidcraft-cli/tests/oracle_cases.rs
@@ -0,0 +1,66 @@
+//! The reference tests (tests/oracle, docs/oracle.md) as a `cargo test`, so tools that drive
+//! `cargo test` (cargo-mutants) can use them: every case with expected values and no
+//! `known_failure` mark must pass, as `cargo xtask oracle` requires.
+
+use std::path::Path;
+use std::process::Command;
+
+use serde_json::Value;
+
+#[test]
+#[ignore = "run by cargo-mutants (.cargo/mutants.toml); CI replays the oracle through cargo xtask oracle"]
+fn oracle_suite() {
+ let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
+ let dir = root.join("tests/oracle");
+ let mut cases: Vec<String> = std::fs::read_dir(&dir)
+ .unwrap()
+ .flatten()
+ .map(|e| e.path())
+ .filter(|p| p.join("recipe.json").is_file())
+ .map(|p| format!("tests/oracle/{}", p.file_name().unwrap().to_string_lossy()))
+ .collect();
+ cases.sort();
+ assert!(cases.len() >= 50, "oracle cases in {}", dir.display());
+ // One process per core, cases dealt round-robin so the slow ones spread out.
+ let n = std::thread::available_parallelism().map_or(4, |n| n.get()).min(cases.len());
+ let children: Vec<_> = (0..n)
+ .map(|i| {
+ let chunk: Vec<&String> = cases.iter().skip(i).step_by(n).collect();
+ Command::new(env!("CARGO_BIN_EXE_solidcraft-cli"))
+ .current_dir(&root)
+ .arg("oracle")
+ .arg("--json")
+ .args(chunk)
+ .stdout(std::process::Stdio::piped())
+ .stderr(std::process::Stdio::piped())
+ .spawn()
+ .unwrap()
+ })
+ .collect();
+ let mut results: Vec<Value> = Vec::new();
+ for child in children {
+ let out = child.wait_with_output().unwrap();
+ let part: Vec<Value> =
+ serde_json::from_slice(&out.stdout).unwrap_or_else(|e| panic!("oracle JSON: {e}\n{}", String::from_utf8_lossy(&out.stderr)));
+ results.extend(part);
+ }
+ assert_eq!(results.len(), cases.len());
+ let mut failed = Vec::new();
+ for r in &results {
+ let case = r["case"].as_str().unwrap_or("");
+ let want: Value =
+ std::fs::read_to_string(dir.join(case).join("expected.json")).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default();
+ if want.is_null() || want["known_failure"].is_string() || r["pass"].as_bool().unwrap_or(false) {
+ continue;
+ }
+ let checks: Vec<String> = r["checks"]
+ .as_array()
+ .into_iter()
+ .flatten()
+ .filter(|c| !c["ok"].as_bool().unwrap_or(false))
+ .map(|c| format!("{} got {} want {}", c["check"].as_str().unwrap_or(""), c["got"], c["want"]))
+ .collect();
+ failed.push(format!("{case}: {} {}", r["error"].as_str().unwrap_or(""), checks.join("; ")));
+ }
+ assert!(failed.is_empty(), "oracle cases failed:\n{}", failed.join("\n"));
+}
diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md
new file mode 100644
index 0000000000..74c54f49f5
--- /dev/null
+++ b/docs/mutation-testing.md
@@ -0,0 +1,69 @@
+# Mutation testing
+
+SolidCraft will move from truck to solidcore (#1597). The tests that have to catch regressions
+across that switch are the ones that don't depend on the kernel: the reference tests
+([docs/oracle.md](oracle.md)), the robustness goldens ([docs/robustness.md](robustness.md)) and
+the engine's command tests. Most of the kernel's own unit tests call truck-specific code and go
+away with it.
+
+Mutation testing checks those suites. [cargo-mutants](https://mutants.rs) makes small changes to
+the source, one at a time (`*` becomes `/`, a condition is negated, a function returns a default
+value), rebuilds, and runs the tests. A mutant the tests don't catch is a change in behaviour
+nobody would notice, and solidcore could get the same thing wrong without anyone noticing.
+
+## What judges a mutant
+
+`.cargo/mutants.toml` judges mutants by the black-box suites only:
+
+- `oracle_suite` (`apps/solidcraft-cli/tests/oracle_cases.rs`): every case in `tests/oracle` with
+ expected values and no `known_failure` must pass, as `cargo xtask oracle` requires. It is
+ `#[ignore]`d so CI doesn't replay the oracle twice.
+- The pull-request sample of the robustness goldens: every 8th seed, as `ci.yml` runs it.
+
+The kernel's unit tests are left out on purpose. A mutant that only they catch marks a behaviour
+the black-box suites would miss once the kernel is swapped.
+
+## Running it
+
+```
+cargo install cargo-mutants --locked
+CARGO_TARGET_DIR=$PWD/target cargo mutants --in-place \
+ -f crates/kernel/src/measure.rs \
+ --test-package solidcraft-cli --test-package solidcraft-engine \
+ --baseline skip --timeout 900 --build-timeout 900
+```
+
+- Use a worktree you don't otherwise touch: `--in-place` writes each mutant into the source tree
+ and restores it afterwards.
+- `--shard 0/200 --sharding round-robin` with `-d crates/kernel` takes an even sample (every 200th
+ mutant) of the whole crate.
+- The `mutants` profile in `Cargo.toml` is the `ci` profile with incremental builds. It cuts the
+ rebuild after a kernel change from about 3 minutes to about 15 seconds.
+- A mutant costs 1.5–5 minutes on a 12-core Mac, mostly the tests. The kernel has about 40,000
+ mutants, so run a sample or one file at a time, never the whole crate. This is a tool for
+ finding gaps, not a CI gate.
+
+Two problems with cargo-mutants 27.1 are behind the flags above:
+
+- The baseline (the unmutated run) ignores `--test-package` and tests the mutated crate, which
+ fails. Skip it with `--baseline skip`, set `--timeout` yourself, and check the unmutated tree
+ passes first.
+- `--re` and `--exclude-re` don't filter "delete field" mutants. Saved diffs in
+ `mutants.out/diff/` name the mutation, not the file, in their `+++` line: apply one with
+ `patch <file> < <diff>`.
+
+## Pilot results (October 2026, `cb857f1f`)
+
+| Sample | Mutants that build | Caught by the black-box suites |
+|---|---:|---:|
+| `crates/kernel/src/measure.rs`, every mutant | 85 | 85% |
+| Whole kernel, every 200th mutant | 173 | 15% (95% interval 10–21%) |
+
+Out of 49 of the kernel sample's survivors, the kernel's own unit tests caught 21 (43%) and
+nothing caught 28 (57%). About 11% of survivors swap `>` and `>=` on floating-point values that
+are never exactly equal, so they don't change behaviour.
+
+The black-box suites are weakest on tessellation and the display mesh (0 of 19 mutants caught),
+offset, shell, emboss and lip (0 of 14), blends (15%) and STEP/IGES exchange (17%). For a
+survivor that a kernel unit test catches, that test's design is the starting point for a new
+oracle case or engine test.