Skip to content

Plan, apply and roll back changes to a production ontology, with a blast radius report and a proof an auditor can re-check.

Closes #203: the command line reaches the verified SHACL evaluator (#246)

Fabio Rovaic2201cb1ac18
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 5bdfd1d4..f1b90360 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -494,6 +494,14 @@ jobs:
# three-megabyte inputs. The published vectors are checked at compile
# time by #guard; this is the other half.
run: OO_REQUIRE_FIXTURES=1 cargo test --test checker_self_identification_test -- --nocapture
+ - name: Reach the verified SHACL evaluator from the COMMAND LINE
+ # The other half of #203. The tool could reach `oo-shacl` and the CLI
+ # could not, so a user without an MCP client could not obtain the
+ # repository's strongest SHACL answer at all. Strict for the reason the
+ # leg above is strict, and for one more: two of these tests run the
+ # batch executor and the single-command path, which are different code
+ # that have already differed once.
+ run: OO_REQUIRE_FIXTURES=1 cargo test --test cli_verified_shacl_test -- --nocapture
- name: Check the description-logic refutation certificates
# The other half of the line above. That one certifies that an ontology
# HAS a model by exhibiting one; this certifies that it has NONE by
diff --git a/docs/ci-gates.md b/docs/ci-gates.md
index 1949dee5..5de0c2fe 100644
--- a/docs/ci-gates.md
+++ b/docs/ci-gates.md
@@ -61,6 +61,7 @@ job invokes the file at all.
| `shacl_verified_tool_test.rs` | lake + `oo-shacl` | `lean` job | **strict** |
| `shacl_ignored_is_named_test.rs` | lake + `oo-shacl` | `lean` job | **strict** |
| `checker_self_identification_test.rs` | lake + every checker | `lean` job | **strict** |
+| `cli_verified_shacl_test.rs` | lake + `oo-shacl` | `lean` job | **strict** |
| `dl_refutation_certificate_test.rs` | lake + `oo-dlrefute` | `lean` job | **strict** |
| `lean_refutation_test.rs` | lake + `oo-refute` | `lean` job | **strict** |
| `lean_refutation_producer_test.rs` | lake + `oo-refute` | `lean` job | **strict** |
diff --git a/src/batch.rs b/src/batch.rs
index 5eaf14d5..e06374d6 100644
--- a/src/batch.rs
+++ b/src/batch.rs
@@ -576,12 +576,47 @@ impl BatchRunner {
}
}
+ /// SHACL, on either of the two evaluators.
+ ///
+ /// `--verified` runs `oo-shacl`, whose agreement with the Recommendation is
+ /// the machine-checked theorem `Shacl.validate_spec`. Until this flag
+ /// existed that evaluator was reachable from the MCP tool `onto_shacl` and
+ /// from nothing else, so a user of the command line could not obtain the
+ /// repository's strongest SHACL answer at all (#203).
+ ///
+ /// The two are NOT the same question and the reports say so: the verified
+ /// one carries a `verified` key that the unverified one never emits, and
+ /// `undetermined` is a first-class answer there rather than a silent
+ /// omission from a `skipped_constraints` list.
fn exec_shacl(&self, args: &[String]) -> Value {
use crate::shacl::ShaclValidator;
let shapes_path = match args.first() {
Some(p) => p,
None => return json!({"error": "shacl requires a shapes file path"}),
};
+ if args.iter().any(|a| a == "--verified") {
+ // Refused rather than ignored, the same answer `onto_shacl` gives.
+ // The verified evaluator reads one N-Triples dump of the store and
+ // has no notion of a temporal scope, so honouring the argument is
+ // impossible and dropping it would answer a different question
+ // from the one that was asked.
+ if Self::flag_value(args, "--valid-at").is_some()
+ || Self::flag_value(args, "--as-of").is_some()
+ || args.iter().any(|a| a == "--all-versions")
+ {
+ return json!({"error":
+ "--verified cannot be combined with --valid-at, --as-of or \
+ --all-versions. The verified evaluator reads the whole store and has \
+ no temporal scope, so the scope would be silently dropped. Run the \
+ scoped question without --verified, or the verified question without \
+ a scope."});
+ }
+ return match std::fs::read_to_string(shapes_path) {
+ Ok(shapes) => crate::shacl_verified::validate_verified(&self.graph, &shapes)
+ .unwrap_or_else(|e| json!({"error": e.to_string()})),
+ Err(e) => json!({"error": e.to_string()}),
+ };
+ }
let request = match crate::temporal::ScopeRequest::from_args(
Self::flag_value(args, "--valid-at").as_deref(),
Self::flag_value(args, "--as-of").as_deref(),
diff --git a/src/main.rs b/src/main.rs
index 78a36de2..a64cd322 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -445,6 +445,22 @@ enum Commands {
/// refused together with --valid-at or --as-of.
#[arg(long)]
all_versions: bool,
+ /// Run the VERIFIED evaluator, `oo-shacl`, whose agreement with the
+ /// SHACL Recommendation is the machine-checked theorem
+ /// `Shacl.validate_spec`, instead of the SPARQL-compiling one.
+ ///
+ /// A different question, not a better setting on the same one. It
+ /// covers SHACL Core and refuses `sh:sparql` and user-defined
+ /// components outright; the default path runs those and skips others,
+ /// so neither is a superset. Its report carries a `verified` key the
+ /// default path never emits, and `undetermined` is a first-class
+ /// answer there rather than a silent omission.
+ ///
+ /// Refused together with --valid-at, --as-of and --all-versions: the
+ /// verified evaluator has no temporal scope, so honouring one is
+ /// impossible and dropping it would answer a different question.
+ #[arg(long)]
+ verified: bool,
},
/// Closed-world vocab check: flag data terms not declared in the loaded ontology
VocabCheck { data: String },
@@ -1144,6 +1160,7 @@ impl Commands {
valid_at,
as_of,
all_versions,
+ verified,
} => {
let mut a = vec![absolutize(shapes)];
if let Some(v) = valid_at {
@@ -1157,6 +1174,9 @@ impl Commands {
if *all_versions {
a.push("--all-versions".into());
}
+ if *verified {
+ a.push("--verified".into());
+ }
cmd("shacl", a)
}
Commands::Status => cmd("status", vec![]),
@@ -2855,10 +2875,30 @@ async fn async_main() -> anyhow::Result<()> {
valid_at,
as_of,
all_versions,
+ verified,
} => {
use open_ontologies::shacl::ShaclValidator;
let (_db, graph) = setup(&cli.data_dir)?;
let shapes_content = std::fs::read_to_string(&shapes)?;
+ if verified {
+ // Refused rather than ignored: the verified evaluator reads one
+ // N-Triples dump of the store and has no temporal scope, so a
+ // scope could only be dropped, and dropping it would answer a
+ // different question from the one that was asked.
+ let result = if valid_at.is_some() || as_of.is_some() || all_versions {
+ serde_json::json!({"error":
+ "--verified cannot be combined with --valid-at, --as-of or \
+ --all-versions. The verified evaluator reads the whole store and \
+ has no temporal scope, so the scope would be silently dropped. Run \
+ the scoped question without --verified, or the verified question \
+ without a scope."})
+ } else {
+ open_ontologies::shacl_verified::validate_verified(&graph, &shapes_content)
+ .unwrap_or_else(|e| serde_json::json!({"error": e.to_string()}))
+ };
+ output_result(&result.to_string(), cli.pretty);
+ return Ok(());
+ }
let result = match open_ontologies::temporal::ScopeRequest::from_args(
valid_at.as_deref(),
as_of.as_deref(),
@@ -3651,8 +3691,13 @@ mod proxy_serialization_tests {
Commands::Fol { out: "/tmp/fol".into(), format: "tptp".into(), smt_domain: None, clif_dialect: "iso".into(), clif_comments: "standalone".into(), goals: None, goals_skip_columns: 0 },
Commands::FolModel { out: "/tmp/folmodel".into(), solver: "z3".into(), max_domain: 16, timeout_secs: 30, unbounded_probe: true, goals: None, goals_skip_columns: 0, checker: None },
Commands::FolProve { out: Some("/tmp/folprove".into()), prover: "vampire".into(), timeout_secs: 30, goals: None, goals_skip_columns: 0, problem: None, proof: None },
- Commands::Shacl { shapes: "s.ttl".into(), valid_at: None, as_of: None, all_versions: false },
- Commands::Shacl { shapes: "s.ttl".into(), valid_at: None, as_of: None, all_versions: true },
+ Commands::Shacl { shapes: "s.ttl".into(), valid_at: None, as_of: None, all_versions: false, verified: false },
+ Commands::Shacl { shapes: "s.ttl".into(), valid_at: None, as_of: None, all_versions: true, verified: false },
+ // The verified form proxies too, for the reason the scoped `reason`
+ // form above is here: a daemon-backed `shacl --verified` that
+ // silently dropped the flag would run the UNVERIFIED evaluator and
+ // answer with a conformance verdict that no theorem stands behind.
+ Commands::Shacl { shapes: "s.ttl".into(), valid_at: None, as_of: None, all_versions: false, verified: true },
Commands::Status,
Commands::Pull { url: "http://example.org".into(), sparql: false, query: None },
Commands::Push { endpoint: "http://example.org".into(), graph: None },
diff --git a/tests/checker_self_identification_test.rs b/tests/checker_self_identification_test.rs
index c7e284e8..f5b00bee 100644
--- a/tests/checker_self_identification_test.rs
+++ b/tests/checker_self_identification_test.rs
@@ -87,10 +87,11 @@ fn skip() -> bool {
}
fn version_block(exe: &str) -> serde_json::Value {
- let out = Command::new(bin_dir().join(exe))
- .arg("--version")
- .output()
- .unwrap_or_else(|e| panic!("run {exe} --version: {e}"));
+ let out = {
+ let _gate = common::exec_gate();
+ Command::new(bin_dir().join(exe)).arg("--version").output()
+ }
+ .unwrap_or_else(|e| panic!("run {exe} --version: {e}"));
assert_eq!(out.status.code(), Some(0), "{exe} --version must exit 0");
let text = String::from_utf8_lossy(&out.stdout).to_string();
let v: serde_json::Value = serde_json::from_str(&text)
@@ -165,11 +166,11 @@ fn the_verdict_carries_the_same_block_as_version() {
std::fs::write(&a, "<a>\t<b>\t<c>\n").unwrap();
std::fs::write(&der, "").unwrap();
- let out = Command::new(bin_dir().join("oo-cert"))
- .arg(&a)
- .arg(&der)
- .output()
- .expect("run oo-cert");
+ let out = {
+ let _gate = common::exec_gate();
+ Command::new(bin_dir().join("oo-cert")).arg(&a).arg(&der).output()
+ }
+ .expect("run oo-cert");
let text = String::from_utf8_lossy(&out.stdout).to_string();
let v: serde_json::Value = serde_json::from_str(&text).expect("oo-cert prints JSON");
assert_eq!(v["theorem"], "OOCert.certificate_sound", "{text}");
@@ -203,7 +204,11 @@ fn the_rust_side_reads_the_block_and_invents_nothing() {
let bin = bin_dir().join("oo-cert");
let mut cmd = Command::new(&bin);
cmd.arg(&a).arg(&der);
- let run = CheckerRun::spawn(&CheckerBinary::found_at(bin.clone()), cmd).expect("runs");
+ let run = {
+ let _gate = common::exec_gate();
+ CheckerRun::spawn(&CheckerBinary::found_at(bin.clone()), cmd)
+ }
+ .expect("runs");
let block = run.checker_block().expect("an oo-cert since #204 prints one");
assert_eq!(block["self_sha256"].as_str().unwrap(), sha256_of(&bin));
}
@@ -222,14 +227,24 @@ fn something_that_prints_no_block_yields_none() {
"#!/bin/sh\necho '{\"ok\":true,\"theorem\":\"OOCert.certificate_sound\"}'\nexit 0\n"
.to_string()
};
- std::fs::write(&script, body).unwrap();
- #[cfg(unix)]
{
- use std::os::unix::fs::PermissionsExt as _;
- std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
+ // Written with no fork in flight. This test wrote a script and executed
+ // it while four siblings were forking `--version`, and CI answered
+ // `ExecutableFileBusy`. See `common::exec_gate`.
+ let _gate = common::exec_gate();
+ std::fs::write(&script, body).unwrap();
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt as _;
+ std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)).unwrap();
+ }
}
let cmd = Command::new(&script);
- let run = CheckerRun::spawn(&CheckerBinary::found_at(script), cmd).expect("runs");
+ let run = {
+ let _gate = common::exec_gate();
+ CheckerRun::spawn(&CheckerBinary::found_at(script), cmd)
+ }
+ .expect("runs");
assert_eq!(run.named_theorem().as_deref(), Some("OOCert.certificate_sound"));
assert!(
run.checker_block().is_none(),
diff --git a/tests/cli_verified_shacl_test.rs b/tests/cli_verified_shacl_test.rs
new file mode 100644
index 00000000..a3d97c0a
--- /dev/null
+++ b/tests/cli_verified_shacl_test.rs
@@ -0,0 +1,249 @@
+//! The verified SHACL evaluator, from the command line.
+//!
+//! Issue #203 is titled "the verified evaluator is unreachable from
+//! `onto_shacl` AND THE CLI". PR #210 closed the first half: `onto_shacl`
+//! grew `verified: true` and the MCP tool can now reach `oo-shacl`, whose
+//! agreement with the Recommendation is the machine-checked theorem
+//! `Shacl.validate_spec`. The command line could not. A user without an MCP
+//! client could not obtain this repository's strongest SHACL answer at all,
+//! which is the same defect the issue opened about, surviving in the half of
+//! the interface that most people reach for first.
+//!
+//! `--verified` closes it, on both CLI paths: the single-command one in
+//! `main.rs` and the batch executor in `batch.rs`, which are different code
+//! and were separately capable of not having it.
+//!
+//! # Two questions, not one setting
+//!
+//! The flag does not make the same answer better. The verified evaluator
+//! covers SHACL Core and refuses `sh:sparql` and user-defined components
+//! outright; the default path runs those and skips others. Neither is a
+//! superset, so the reports must be tellable apart, and they are: the verified
+//! report carries a `verified` key the default path never emits.
+//! `the_default_path_never_says_verified` is that test, and it needs no
+//! checker, so it runs everywhere.
+//!
+//! # The refusal is the interesting half
+//!
+//! A temporal scope cannot be honoured by an evaluator that reads one
+//! N-Triples dump of the whole store. Silently dropping it would answer a
+//! different question from the one that was asked, which is precisely the
+//! failure mode the unverified path's `skipped_constraints` list exists to
+//! prevent. So it is refused, with the reason, and
+//! `a_scope_with_verified_is_refused_and_not_dropped` runs without a checker
+//! too, because argument handling should not be testable only on machines with
+//! a Lean toolchain.
+
+mod common;
+
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+fn run(dir: &Path, args: &[&str]) -> (i32, String) {
+ let out = Command::new(env!("CARGO_BIN_EXE_open-ontologies"))
+ .arg("--data-dir")
+ .arg(dir.join("data"))
+ .args(args)
+ .output()
+ .expect("the binary runs");
+ (
+ out.status.code().unwrap_or(-1),
+ format!(
+ "{}{}",
+ String::from_utf8_lossy(&out.stdout),
+ String::from_utf8_lossy(&out.stderr)
+ ),
+ )
+}
+
+fn scratch(tag: &str) -> PathBuf {
+ let d = std::env::temp_dir().join(format!("oo-cli-vshacl-{tag}-{}", std::process::id()));
+ let _ = std::fs::remove_dir_all(&d);
+ std::fs::create_dir_all(&d).unwrap();
+ d
+}
+
+fn write(dir: &Path, name: &str, body: &str) -> String {
+ let p = dir.join(name);
+ std::fs::write(&p, body).unwrap();
+ p.to_str().unwrap().to_string()
+}
+
+const SHAPES: &str = r#"
+@prefix sh: <http://www.w3.org/ns/shacl#> .
+@prefix ex: <http://ex.org/> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+ex:PersonShape a sh:NodeShape ;
+ sh:targetClass ex:Person ;
+ sh:property [ sh:path ex:name ; sh:datatype xsd:string ; sh:minCount 1 ] .
+"#;
+
+/// `alice` satisfies the shape and `bob` does not: he is a Person with no
+/// name, and the shape says `sh:minCount 1`. Both are here so a verdict can be
+/// wrong in either direction — a run that blames nobody and a run that blames
+/// everybody both fail, where a fixture with only a violation would let the
+/// second through.
+const DATA: &str = r#"
+@prefix ex: <http://ex.org/> .
+ex:alice a ex:Person ; ex:name "Alice" .
+ex:bob a ex:Person .
+"#;
+
+/// Where the verified evaluator would be found, by the rule `shacl_verified`
+/// uses: an explicit `$OO_SHACL` is an instruction, then the build directory.
+fn checker_present() -> bool {
+ if let Ok(p) = std::env::var("OO_SHACL") {
+ return Path::new(&p).exists();
+ }
+ PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+ .join("lean/.lake/build/bin/oo-shacl")
+ .exists()
+}
+
+fn skip() -> bool {
+ common::skip_unless(
+ checker_present(),
+ "the verified SHACL evaluator (oo-shacl)",
+ "run `lake build oo-shacl` in lean/, or point $OO_SHACL at a built one",
+ )
+}
+
+#[test]
+fn the_default_path_never_says_verified() {
+ let d = scratch("default");
+ let shapes = write(&d, "shapes.ttl", SHAPES);
+ let data = write(&d, "data.ttl", DATA);
+ let _ = run(&d, &["load", &data]);
+ let (_code, out) = run(&d, &["shacl", &shapes]);
+ assert!(
+ !out.contains("\"verified\""),
+ "the unverified path must not emit a key that says a theorem was involved: {out}"
+ );
+}
+
+#[test]
+fn a_scope_with_verified_is_refused_and_not_dropped() {
+ // No checker needed: this is argument handling, and it must be testable on
+ // a machine with no Lean toolchain.
+ let d = scratch("scope");
+ let shapes = write(&d, "shapes.ttl", SHAPES);
+ for scope in [
+ vec!["--valid-at", "2026-01-01T00:00:00Z"],
+ vec!["--as-of", "2026-01-01T00:00:00Z"],
+ vec!["--all-versions"],
+ ] {
+ let mut args = vec!["shacl", shapes.as_str(), "--verified"];
+ args.extend_from_slice(&scope);
+ let (_code, out) = run(&d, &args);
+ assert!(
+ out.contains("cannot be combined"),
+ "a scope the verified evaluator cannot honour must be REFUSED, not dropped. \
+ Args {scope:?} produced: {out}"
+ );
+ assert!(
+ !out.contains("\"conforms\":true"),
+ "and the refusal must not also carry a conformance answer: {out}"
+ );
+ }
+}
+
+/// Reachability, and only reachability.
+///
+/// The single-command path cannot carry data between invocations in the
+/// default storage mode, so the store this validates is EMPTY and the verdict
+/// is `conforms: true` over nothing. That is worth saying out loud rather than
+/// leaving as a test that looks substantive and is not: what it proves is that
+/// the flag reaches `oo-shacl` and that the theorem in the report is the
+/// checker's own. The verdict with data in it is the batch test below.
+#[test]
+fn the_verified_flag_reaches_the_lean_evaluator() {
+ if skip() {
+ return;
+ }
+ let d = scratch("reach");
+ let shapes = write(&d, "shapes.ttl", SHAPES);
+ let (_code, out) = run(&d, &["shacl", &shapes, "--verified"]);
+ assert!(
+ out.contains("\"verified\""),
+ "the verified path must say so in its report: {out}"
+ );
+ // The theorem is the checker's own word, echoed. If it were ever a Rust
+ // literal the whole layer would be decoration, which is why this is
+ // asserted and not merely documented.
+ assert!(
+ out.contains("Shacl.validate_spec"),
+ "an accepted run names the theorem it discharged: {out}"
+ );
+}
+
+/// The batch executor is different code from the single-command path and was
+/// separately capable of not having the flag. It is also the only CLI route
+/// that can load and then validate in one process, so it is where the verdict
+/// has data under it.
+#[test]
+fn the_batch_path_finds_the_violation_the_shapes_describe() {
+ if skip() {
+ return;
+ }
+ let d = scratch("batch");
+ let shapes = write(&d, "shapes.ttl", SHAPES);
+ let data = write(&d, "data.ttl", DATA);
+ let plan = write(
+ &d,
+ "plan.json",
+ &format!(
+ r#"[{{"command":"load","args":[{data:?}]}},
+ {{"command":"shacl","args":[{shapes:?}, "--verified"]}}]"#
+ ),
+ );
+ let (_code, out) = run(&d, &["batch", &plan]);
+ assert!(out.contains("\"verified\""), "batch must reach the same evaluator: {out}");
+ // `ex:bob` is a Person with no name, and the shape says minCount 1. A run
+ // that reported `conforms: true` here would be reachable and wrong, which
+ // is the failure the previous test cannot see.
+ assert!(
+ out.contains("\"conforms\": false") || out.contains("\"conforms\":false"),
+ "the verified evaluator must find the missing name: {out}"
+ );
+ assert!(
+ out.contains("http://ex.org/bob"),
+ "and it must name the node it blames, not merely say something failed: {out}"
+ );
+ assert!(
+ !out.contains("http://ex.org/alice"),
+ "alice has a name and must not be blamed: {out}"
+ );
+}
+
+/// The differential the issue said would fall out of this for free.
+///
+/// The two evaluators are not comparable in general: neither covers what the
+/// other does. On a graph inside SHACL Core they must not DISAGREE, and this
+/// is the first gate on `onto_shacl` that does not rest on SPARQL being right.
+#[test]
+fn the_two_evaluators_agree_on_a_graph_inside_shacl_core() {
+ if skip() {
+ return;
+ }
+ let d = scratch("differential");
+ let shapes = write(&d, "shapes.ttl", SHAPES);
+ let data = write(&d, "data.ttl", DATA);
+ let plan = write(
+ &d,
+ "plan.json",
+ &format!(
+ r#"[{{"command":"load","args":[{data:?}]}},
+ {{"command":"shacl","args":[{shapes:?}]}}]"#
+ ),
+ );
+ let (_code, unverified) = run(&d, &["batch", &plan]);
+ assert!(
+ unverified.contains("http://ex.org/bob"),
+ "the SPARQL-compiling path must blame bob too, or the two evaluators disagree \
+ on a graph both of them fully cover: {unverified}"
+ );
+ assert!(
+ !unverified.contains("\"verified\""),
+ "and it must still not claim a theorem: {unverified}"
+ );
+}
diff --git a/tests/common/mod.rs b/tests/common/mod.rs
index 202768c7..0d296ce3 100644
--- a/tests/common/mod.rs
+++ b/tests/common/mod.rs
@@ -27,3 +27,45 @@ pub fn skip_unless(available: bool, what: &str, how_to_get_it: &str) -> bool {
eprintln!("SKIPPED_FIXTURE: {msg}");
true
}
+
+/// Writing a file and forking a process must not overlap in one test binary.
+///
+/// Confirmed from a CI log rather than guessed at. On 21 September 2026 a lean
+/// job printed
+///
+/// ```text
+/// runs: Os { code: 26, kind: ExecutableFileBusy, message: "Text file busy" }
+/// ```
+///
+/// A thread writing a script holds a write file descriptor to it. If another
+/// thread forks in that instant, the child inherits that descriptor. The writer
+/// closes its own copy and execs the script, but the child still holds one
+/// until it reaches its own exec, and Linux refuses to exec a file any process
+/// has open for writing. The spawn returns `ExecutableFileBusy`.
+///
+/// Giving each script its own path does NOT fix it, and that was tried: the
+/// race is between ANY write and ANY fork, not between two writers of one
+/// file. It never reproduces on macOS, which does not raise this error here,
+/// so it looks like a CI defect and is not one.
+///
+/// Only threads of one process matter, because a descriptor is inherited by a
+/// fork and is not shared between unrelated processes, so a mutex per test
+/// binary is the whole fix. It lives here because the same defect has now
+/// appeared in two test files and the next one should reach for this instead
+/// of rediscovering it.
+///
+/// Hold it across the write AND across the spawn. Both are short.
+///
+/// ```ignore
+/// let script = { let _gate = common::exec_gate(); write_script() };
+/// let out = { let _gate = common::exec_gate(); Command::new(&script).output() };
+/// ```
+///
+/// `unwrap_or_else(|e| e.into_inner())`: a test that panics holding this lock
+/// poisons it, and the next test would then fail for a reason unrelated to
+/// what it checks.
+#[allow(dead_code)]
+pub fn exec_gate() -> std::sync::MutexGuard<'static, ()> {
+ static GATE: std::sync::Mutex<()> = std::sync::Mutex::new(());
+ GATE.lock().unwrap_or_else(|e| e.into_inner())
+}
diff --git a/tests/verdict_vocabulary_test.rs b/tests/verdict_vocabulary_test.rs
index 4de2cf92..1d6ed729 100644
--- a/tests/verdict_vocabulary_test.rs
+++ b/tests/verdict_vocabulary_test.rs
@@ -25,6 +25,8 @@
//! the engine does: it runs a process that exits zero through the crate's own
//! `run_checker`. There is no other way to get one, which is the point.
+mod common;
+
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
@@ -88,11 +90,15 @@ fn script_saying(code: i32, theorem: &str) -> PathBuf {
} else {
format!("#!/bin/sh\necho '{say}'\nexit {code}\n")
};
- std::fs::write(&p, body).unwrap();
- #[cfg(unix)]
{
- use std::os::unix::fs::PermissionsExt as _;
- std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
+ // The write and the chmod happen with no fork in flight; see EXEC_GATE.
+ let _gate = common::exec_gate();
+ std::fs::write(&p, body).unwrap();
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt as _;
+ std::fs::set_permissions(&p, std::fs::Permissions::from_mode(0o755)).unwrap();
+ }
}
p
}
@@ -110,7 +116,11 @@ fn earned(theorem: &'static str) -> Certified {
// reads. A shell that exits zero in silence is a checker that named
// nothing, and it correctly mints nothing.
let (bin, cmd) = shell_naming(theorem);
- let run = CheckerRun::spawn(&bin, cmd).expect("the system shell must be runnable");
+ let run = {
+ let _gate = common::exec_gate();
+ CheckerRun::spawn(&bin, cmd)
+ }
+ .expect("the system shell must be runnable");
assert_eq!(run.exit(), 0);
run.accepted_naming(&[theorem]).expect("exit 0 naming it mints the token")
}
@@ -328,7 +338,11 @@ fn a_theorem_is_named_only_where_the_evidence_is() {
/// and neither is an acceptance. A run that could not start is `Absent`.
#[test]
fn only_a_zero_exit_produces_an_acceptance() {
- let dir = std::env::temp_dir().join("oo-verdict-vocabulary");
+ // Per PROCESS, not per machine. A fixed name under the system temp
+ // directory is shared by every concurrent run on the box, and these two
+ // files are truncated on entry, so a second run could empty them under the
+ // first one's feet.
+ let dir = std::env::temp_dir().join(format!("oo-verdict-vocabulary-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let a = dir.join("asserted.tsv");
let d = dir.join("derivations.tsv");
@@ -337,21 +351,42 @@ fn only_a_zero_exit_produces_an_acceptance() {
let run = |code: i32| {
let script = script_exiting(code);
+ let _gate = common::exec_gate();
pe::run_checker(CertKind::OoCert, Some(script.as_path()), &a, &d, None)
};
- assert!(matches!(run(0), CheckerStatus::Accepted(_)));
- assert!(matches!(run(1), CheckerStatus::Rejected { .. }));
- assert!(matches!(run(2), CheckerStatus::Unreadable { .. }));
- assert!(matches!(run(3), CheckerStatus::Unreadable { .. }));
+ // `assert!(matches!(...))` prints nothing but the line number, and this
+ // test went red twice on CI and green everywhere else. A failure has to
+ // say WHICH status it got, or the next person reads a line number and
+ // guesses, which is what happened.
+ let got = |st: &CheckerStatus| -> String {
+ match st {
+ CheckerStatus::Accepted(_) => "accepted".to_string(),
+ CheckerStatus::Rejected { stdout } => format!("rejected, stdout {stdout:?}"),
+ CheckerStatus::Unreadable { stdout } => format!("unreadable, stdout {stdout:?}"),
+ CheckerStatus::Absent { what, .. } => format!("ABSENT: {what}"),
+ CheckerStatus::NotNeeded { what } => format!("not needed: {what}"),
+ }
+ };
+ let r0 = run(0);
+ assert!(matches!(r0, CheckerStatus::Accepted(_)), "exit 0 must accept, got {}", got(&r0));
+ let r1 = run(1);
+ assert!(matches!(r1, CheckerStatus::Rejected { .. }), "exit 1 must reject, got {}", got(&r1));
+ let r2 = run(2);
+ assert!(matches!(r2, CheckerStatus::Unreadable { .. }), "exit 2 unreadable, got {}", got(&r2));
+ let r3 = run(3);
+ assert!(matches!(r3, CheckerStatus::Unreadable { .. }), "exit 3 unreadable, got {}", got(&r3));
- let absent = pe::run_checker(
- CertKind::OoCert,
- Some(Path::new("/nonexistent/oo-cert")),
- &a,
- &d,
- None,
- );
+ let absent = {
+ let _gate = common::exec_gate();
+ pe::run_checker(
+ CertKind::OoCert,
+ Some(Path::new("/nonexistent/oo-cert")),
+ &a,
+ &d,
+ None,
+ )
+ };
assert!(absent.is_absent(), "a checker that cannot be found accepted nothing");
// And the acceptance carries the theorem the KIND names, not one the
@@ -364,7 +399,11 @@ fn only_a_zero_exit_produces_an_acceptance() {
// tell an entailment from an entailment-under-supplied-rules told neither.
for named in ["OOCert.horn_certificate_sound", "OOCert.entails_of_builtin_horn"] {
let script = script_saying(0, named);
- match pe::run_checker(CertKind::OoHorn, Some(script.as_path()), &a, &d, Some(a.as_path())) {
+ let status = {
+ let _gate = common::exec_gate();
+ pe::run_checker(CertKind::OoHorn, Some(script.as_path()), &a, &d, Some(a.as_path()))
+ };
+ match status {
CheckerStatus::Accepted(acc) => {
assert_eq!(acc.theorem(), named, "the token must carry what oo-horn printed");
assert_eq!(acc.certified().theorem(), named);
@@ -380,7 +419,10 @@ fn only_a_zero_exit_produces_an_acceptance() {
let wrong = script_saying(0, "OOCert.something_else_entirely");
assert!(
!matches!(
- pe::run_checker(CertKind::OoHorn, Some(wrong.as_path()), &a, &d, Some(a.as_path())),
+ {
+ let _gate = common::exec_gate();
+ pe::run_checker(CertKind::OoHorn, Some(wrong.as_path()), &a, &d, Some(a.as_path()))
+ },
CheckerStatus::Accepted(_)
),
"a checker naming an unexpected theorem must not mint an acceptance"