Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

ISLE: Veri: include counterexample in CI log for failures (#14269)

Alexa VanHattum6d8c2dd210d7
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 55eb675595..fc20b1fa5e 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -1432,6 +1432,8 @@ jobs:
# Per-SMT-query timeout, in seconds (see the `--timeout` option in
# cranelift/isle/veri/veri/src/bin/veri.rs).
ISLE_VERI_TIMEOUT: "90"
+ # Print the counterexample for each verification failure.
+ ISLE_VERI_PRINT_COUNTEREXAMPLE: "1"
run: |
mkdir -p cranelift/isle/veri/cache
./cranelift/isle/veri/verify.sh rebuild-cache aarch64 opt x64-iadd-base-case
diff --git a/cranelift/isle/veri/veri/src/bin/veri.rs b/cranelift/isle/veri/veri/src/bin/veri.rs
index 5ad30f1e40..6f0e40093f 100644
--- a/cranelift/isle/veri/veri/src/bin/veri.rs
+++ b/cranelift/isle/veri/veri/src/bin/veri.rs
@@ -88,6 +88,15 @@ struct Opts {
#[arg(long, default_value_t = true, action = clap::ArgAction::Set)]
results_to_log_dir: bool,
+ /// Print the counterexample for each verification failure.
+ /// At most 25 counterexamples are printed.
+ #[arg(
+ long,
+ env = "ISLE_VERI_PRINT_COUNTEREXAMPLE",
+ value_parser = clap::builder::FalseyValueParser::new(),
+ )]
+ print_counterexample: bool,
+
/// Skip solver.
#[arg(long, env = "ISLE_VERI_SKIP_SOLVER")]
skip_solver: bool,
@@ -285,6 +294,7 @@ fn main() -> Result<()> {
runner.set_log_dir(log_dir);
}
runner.set_results_to_log_dir(opts.results_to_log_dir);
+ runner.set_print_counterexample(opts.print_counterexample);
runner.skip_solver(opts.skip_solver);
runner.debug(opts.debug);
diff --git a/cranelift/isle/veri/veri/src/runner.rs b/cranelift/isle/veri/veri/src/runner.rs
index 2819a3167c..4f31bb11a2 100644
--- a/cranelift/isle/veri/veri/src/runner.rs
+++ b/cranelift/isle/veri/veri/src/runner.rs
@@ -30,6 +30,10 @@ use crate::{
const LOG_DIR: &str = ".veriisle";
+/// Cap on how many counterexamples `--print-counterexample` writes to the
+/// summary.
+const MAX_PRINTED_COUNTEREXAMPLES: usize = 25;
+
#[derive(Debug, Clone, Copy)]
pub enum SolverBackend {
Z3,
@@ -426,6 +430,7 @@ pub struct FailureRecord {
pub description: String,
pub instantiation_index: usize,
pub failure_path: PathBuf,
+ pub counterexample: Option<String>,
}
/// An expansion that could not be processed at all (for example, because a term
@@ -529,6 +534,7 @@ pub struct Runner {
log_dir: PathBuf,
skip_solver: bool,
results_to_log_dir: bool,
+ print_counterexample: bool,
debug: bool,
/// Shared cache of SMT query results (None = no caching).
@@ -553,6 +559,7 @@ impl Runner {
log_dir: PathBuf::from(LOG_DIR),
results_to_log_dir: false,
skip_solver: false,
+ print_counterexample: false,
debug: false,
cache: None,
})
@@ -637,6 +644,10 @@ impl Runner {
self.results_to_log_dir = enabled;
}
+ pub fn set_print_counterexample(&mut self, enabled: bool) {
+ self.print_counterexample = enabled;
+ }
+
pub fn skip_solver(&mut self, skip: bool) {
self.skip_solver = skip;
}
@@ -820,12 +831,36 @@ impl Runner {
"=== VERIFICATION FAILURES ({n}) ===",
n = verification_failures.len()
);
+ let mut printed = 0;
for failure in &verification_failures {
let line = format_line(failure);
eprintln!("FAILURE {line}");
if let Some(f) = summary.as_mut() {
let _ = writeln!(f, "{line}");
}
+
+ // With `--print-counterexample`, follow each failure with the
+ // model the solver found
+ let Some(counterexample) = &failure.counterexample else {
+ continue;
+ };
+ if printed < MAX_PRINTED_COUNTEREXAMPLES {
+ eprintln!(
+ "#{id}\t{description}\tinstantiation={inst}",
+ id = failure.expansion_id,
+ description = failure.description,
+ inst = failure.instantiation_index,
+ );
+ eprintln!("model:");
+ eprint!("{counterexample}");
+ printed += 1;
+ }
+ }
+ if printed > 0 && verification_failures.len() > printed {
+ eprintln!(
+ "... and {n} more counterexamples not printed; see the failure.out files.",
+ n = verification_failures.len() - printed
+ );
}
log::warn!(
"verification failures: {n}",
@@ -1382,6 +1417,7 @@ impl Runner {
description: description.to_string(),
instantiation_index,
failure_path: unknown_path,
+ counterexample: None,
});
return Ok(VerifyReport {
@@ -1401,6 +1437,10 @@ impl Runner {
writeln!(output, "\t\tverification = {verification}")?;
Ok(match verification {
Verification::Failure(model) => {
+ let mut rendered = Vec::new();
+ conditions.write_model(&mut rendered, &model, &self.prog)?;
+ let rendered = String::from_utf8(rendered)?;
+
let failure_path = log_dir.join("failure.out");
let mut failure_file = Self::open_log_file(log_dir.clone(), "failure.out")?;
writeln!(
@@ -1410,7 +1450,7 @@ impl Runner {
writeln!(failure_file, "expansion:")?;
write_expansion(&mut failure_file, &self.prog, expansion)?;
writeln!(failure_file, "model:")?;
- conditions.write_model(&mut failure_file, &model, &self.prog)?;
+ write!(failure_file, "{rendered}")?;
writeln!(output, "\t\tfailure written to {}", failure_path.display())?;
log::warn!(
@@ -1424,6 +1464,7 @@ impl Runner {
description: description.to_string(),
instantiation_index,
failure_path,
+ counterexample: self.print_counterexample.then_some(rendered),
});
VerifyReport {