A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
ISLE: Veri: include counterexample in CI log for failures (#14269)
Alexa VanHattum6d8c2dd210d7
diff --git a/.github/workflows/main.yml b/.github/workflows/main.ymlindex 55eb675595..fc20b1fa5e 100644--- a/.github/workflows/main.yml+++ b/.github/workflows/main.yml@@ -1432,6 +1432,8 @@ jobs:# Per-SMT-query timeout, in seconds (see the `--timeout` option in# cranelift/isle/veri/veri/src/bin/veri.rs).ISLE_VERI_TIMEOUT: "90"+ # Print the counterexample for each verification failure.+ ISLE_VERI_PRINT_COUNTEREXAMPLE: "1"run: |mkdir -p cranelift/isle/veri/cache./cranelift/isle/veri/verify.sh rebuild-cache aarch64 opt x64-iadd-base-casediff --git a/cranelift/isle/veri/veri/src/bin/veri.rs b/cranelift/isle/veri/veri/src/bin/veri.rsindex 5ad30f1e40..6f0e40093f 100644--- a/cranelift/isle/veri/veri/src/bin/veri.rs+++ b/cranelift/isle/veri/veri/src/bin/veri.rs@@ -88,6 +88,15 @@ struct Opts {#[arg(long, default_value_t = true, action = clap::ArgAction::Set)]results_to_log_dir: bool,+ /// Print the counterexample for each verification failure.+ /// At most 25 counterexamples are printed.+ #[arg(+ long,+ env = "ISLE_VERI_PRINT_COUNTEREXAMPLE",+ value_parser = clap::builder::FalseyValueParser::new(),+ )]+ print_counterexample: bool,+/// Skip solver.#[arg(long, env = "ISLE_VERI_SKIP_SOLVER")]skip_solver: bool,@@ -285,6 +294,7 @@ fn main() -> Result<()> {runner.set_log_dir(log_dir);}runner.set_results_to_log_dir(opts.results_to_log_dir);+ runner.set_print_counterexample(opts.print_counterexample);runner.skip_solver(opts.skip_solver);runner.debug(opts.debug);diff --git a/cranelift/isle/veri/veri/src/runner.rs b/cranelift/isle/veri/veri/src/runner.rsindex 2819a3167c..4f31bb11a2 100644--- a/cranelift/isle/veri/veri/src/runner.rs+++ b/cranelift/isle/veri/veri/src/runner.rs@@ -30,6 +30,10 @@ use crate::{const LOG_DIR: &str = ".veriisle";+/// Cap on how many counterexamples `--print-counterexample` writes to the+/// summary.+const MAX_PRINTED_COUNTEREXAMPLES: usize = 25;+#[derive(Debug, Clone, Copy)]pub enum SolverBackend {Z3,@@ -426,6 +430,7 @@ pub struct FailureRecord {pub description: String,pub instantiation_index: usize,pub failure_path: PathBuf,+ pub counterexample: Option<String>,}/// An expansion that could not be processed at all (for example, because a term@@ -529,6 +534,7 @@ pub struct Runner {log_dir: PathBuf,skip_solver: bool,results_to_log_dir: bool,+ print_counterexample: bool,debug: bool,/// Shared cache of SMT query results (None = no caching).@@ -553,6 +559,7 @@ impl Runner {log_dir: PathBuf::from(LOG_DIR),results_to_log_dir: false,skip_solver: false,+ print_counterexample: false,debug: false,cache: None,})@@ -637,6 +644,10 @@ impl Runner {self.results_to_log_dir = enabled;}+ pub fn set_print_counterexample(&mut self, enabled: bool) {+ self.print_counterexample = enabled;+ }+pub fn skip_solver(&mut self, skip: bool) {self.skip_solver = skip;}@@ -820,12 +831,36 @@ impl Runner {"=== VERIFICATION FAILURES ({n}) ===",n = verification_failures.len());+ let mut printed = 0;for failure in &verification_failures {let line = format_line(failure);eprintln!("FAILURE {line}");if let Some(f) = summary.as_mut() {let _ = writeln!(f, "{line}");}++ // With `--print-counterexample`, follow each failure with the+ // model the solver found+ let Some(counterexample) = &failure.counterexample else {+ continue;+ };+ if printed < MAX_PRINTED_COUNTEREXAMPLES {+ eprintln!(+ "#{id}\t{description}\tinstantiation={inst}",+ id = failure.expansion_id,+ description = failure.description,+ inst = failure.instantiation_index,+ );+ eprintln!("model:");+ eprint!("{counterexample}");+ printed += 1;+ }+ }+ if printed > 0 && verification_failures.len() > printed {+ eprintln!(+ "... and {n} more counterexamples not printed; see the failure.out files.",+ n = verification_failures.len() - printed+ );}log::warn!("verification failures: {n}",@@ -1382,6 +1417,7 @@ impl Runner {description: description.to_string(),instantiation_index,failure_path: unknown_path,+ counterexample: None,});return Ok(VerifyReport {@@ -1401,6 +1437,10 @@ impl Runner {writeln!(output, "\t\tverification = {verification}")?;Ok(match verification {Verification::Failure(model) => {+ let mut rendered = Vec::new();+ conditions.write_model(&mut rendered, &model, &self.prog)?;+ let rendered = String::from_utf8(rendered)?;+let failure_path = log_dir.join("failure.out");let mut failure_file = Self::open_log_file(log_dir.clone(), "failure.out")?;writeln!(@@ -1410,7 +1450,7 @@ impl Runner {writeln!(failure_file, "expansion:")?;write_expansion(&mut failure_file, &self.prog, expansion)?;writeln!(failure_file, "model:")?;- conditions.write_model(&mut failure_file, &model, &self.prog)?;+ write!(failure_file, "{rendered}")?;writeln!(output, "\t\tfailure written to {}", failure_path.display())?;log::warn!(@@ -1424,6 +1464,7 @@ impl Runner {description: description.to_string(),instantiation_index,failure_path,+ counterexample: self.print_counterexample.then_some(rendered),});VerifyReport {