A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cleanup/add some testing in `wasmtime-fuzzing` (#12038)
Alex Crichton882f22a7bd49
diff --git a/crates/fuzzing/src/generators/component_types.rs b/crates/fuzzing/src/generators/component_types.rsindex 5bcd76a2a5..5b1def366e 100644--- a/crates/fuzzing/src/generators/component_types.rs+++ b/crates/fuzzing/src/generators/component_types.rs@@ -187,3 +187,25 @@ whereOk(())}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::test::test_n_times;+ use wasmtime_test_util::component_fuzz::{TestCase, Type};++ #[test]+ fn static_api_smoke_test() {+ test_n_times(10, |(), u| {+ let case = TestCase {+ params: vec![&Type::S32, &Type::Bool, &Type::String],+ result: Some(&Type::String),+ encoding1: u.arbitrary()?,+ encoding2: u.arbitrary()?,+ };++ let declarations = case.declarations();+ static_api_test::<(i32, bool, String), (String,)>(u, &declarations)+ });+ }+}diff --git a/crates/fuzzing/src/generators/stacks.rs b/crates/fuzzing/src/generators/stacks.rsindex e7a3fcb31f..0e9e87e3f5 100644--- a/crates/fuzzing/src/generators/stacks.rs+++ b/crates/fuzzing/src/generators/stacks.rs@@ -404,21 +404,15 @@ impl Stacks {#[cfg(test)]mod tests {use super::*;- use rand::prelude::*;use wasmparser::Validator;#[test]fn stacks_generates_valid_wasm_modules() {- let mut rng = SmallRng::seed_from_u64(0);- let mut buf = vec![0; 2048];- for _ in 0..1024 {- rng.fill_bytes(&mut buf);- let u = Unstructured::new(&buf);- if let Ok(stacks) = Stacks::arbitrary_take_rest(u) {- let wasm = stacks.wasm();- validate(&wasm);- }- }+ crate::test::test_n_times(10, |stacks: Stacks, _u| {+ let wasm = stacks.wasm();+ validate(&wasm);+ Ok(())+ })}fn validate(wasm: &[u8]) {diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rsindex b5a405eb8b..0a2134e6f4 100644--- a/crates/fuzzing/src/lib.rs+++ b/crates/fuzzing/src/lib.rs@@ -29,3 +29,44 @@ pub fn init_fuzzing() {let _ = env_logger::try_init();});}++#[cfg(test)]+mod test {+ use arbitrary::{Arbitrary, Unstructured};+ use rand::prelude::*;++ pub fn gen_until_pass<T: for<'a> Arbitrary<'a>>(+ mut f: impl FnMut(T, &mut Unstructured<'_>) -> anyhow::Result<bool>,+ ) -> bool {+ let mut rng = SmallRng::seed_from_u64(0);+ let mut buf = vec![0; 2048];+ let n = 3000;+ for _ in 0..n {+ rng.fill_bytes(&mut buf);+ let mut u = Unstructured::new(&buf);++ if let Ok(config) = u.arbitrary() {+ if f(config, &mut u).unwrap() {+ return true;+ }+ }+ }+ false+ }++ /// Runs `f` with random data until it returns `Ok(())` `iters` times.+ pub fn test_n_times<T: for<'a> Arbitrary<'a>>(+ iters: u32,+ mut f: impl FnMut(T, &mut Unstructured<'_>) -> arbitrary::Result<()>,+ ) {+ let mut to_test = 0..iters;+ let ok = gen_until_pass(|a, b| {+ if f(a, b).is_ok() {+ Ok(to_test.next().is_none())+ } else {+ Ok(false)+ }+ });+ assert!(ok);+ }+}diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rsindex 9f6ef62ab9..090db067b3 100644--- a/crates/fuzzing/src/oracles.rs+++ b/crates/fuzzing/src/oracles.rs@@ -1369,45 +1369,9 @@ pub fn call_async(wasm: &[u8], config: &generators::Config, mut poll_amts: &[u32#[cfg(test)]mod tests {use super::*;- use arbitrary::Unstructured;- use rand::prelude::*;+ use crate::test::{gen_until_pass, test_n_times};use wasmparser::{Validator, WasmFeatures};- fn gen_until_pass<T: for<'a> Arbitrary<'a>>(- mut f: impl FnMut(T, &mut Unstructured<'_>) -> Result<bool>,- ) -> bool {- let mut rng = SmallRng::seed_from_u64(0);- let mut buf = vec![0; 2048];- let n = 3000;- for _ in 0..n {- rng.fill_bytes(&mut buf);- let mut u = Unstructured::new(&buf);-- if let Ok(config) = u.arbitrary() {- if f(config, &mut u).unwrap() {- return true;- }- }- }- false- }-- /// Runs `f` with random data until it returns `Ok(())` `iters` times.- fn test_n_times<T: for<'a> Arbitrary<'a>>(- iters: u32,- mut f: impl FnMut(T, &mut Unstructured<'_>) -> arbitrary::Result<()>,- ) {- let mut to_test = 0..iters;- let ok = gen_until_pass(|a, b| {- if f(a, b).is_ok() {- Ok(to_test.next().is_none())- } else {- Ok(false)- }- });- assert!(ok);- }-// Test that the `gc_ops` fuzzer eventually runs the gc function in the host.// We've historically had issues where this fuzzer accidentally wasn't fuzzing// anything for a long time so this is an attempt to prevent that from happening@@ -1501,4 +1465,9 @@ mod tests {fn wast_smoke_test() {test_n_times(50, |(), u| super::wast_test(u));}++ #[test]+ fn dynamic_component_api_smoke_test() {+ test_n_times(50, |(), u| super::dynamic_component_api_target(u));+ }}diff --git a/crates/fuzzing/src/oracles/engine.rs b/crates/fuzzing/src/oracles/engine.rsindex e64eca7aff..0da6c0beb2 100644--- a/crates/fuzzing/src/oracles/engine.rs+++ b/crates/fuzzing/src/oracles/engine.rs@@ -181,27 +181,16 @@ pub fn smoke_test_engine<T>() whereT: DiffEngine,{- use rand::prelude::*;-- let mut rng = SmallRng::seed_from_u64(0);- let mut buf = vec![0; 2048];- let n = 100;- for _ in 0..n {- rng.fill_bytes(&mut buf);- let mut u = Unstructured::new(&buf);- let mut config = match u.arbitrary::<Config>() {- Ok(config) => config,- Err(_) => continue,- };+ crate::test::test_n_times(5, |mut config: Config, u| {// This will ensure that wasmtime, which uses this configuration// settings, can guaranteed instantiate a module.config.set_differential_config();- let mut engine = match mk_engine(&mut u, &mut config) {+ let mut engine = match mk_engine(u, &mut config) {Ok(engine) => engine,Err(e) => {println!("skip {e:?}");- continue;+ return Ok(());}};@@ -240,8 +229,6 @@ pub fn smoke_test_engine<T>(}}- return;- }-- panic!("after {n} runs nothing ever ran, something is probably wrong");+ Ok(())+ })}diff --git a/crates/fuzzing/src/oracles/memory.rs b/crates/fuzzing/src/oracles/memory.rsindex 667139ecd4..d6ed4806d6 100644--- a/crates/fuzzing/src/oracles/memory.rs+++ b/crates/fuzzing/src/oracles/memory.rs@@ -392,20 +392,13 @@ fn build_wasm(image: &HeapImage, offset: u32) -> Vec<u8> {#[cfg(test)]mod tests {use super::*;- use arbitrary::{Arbitrary, Unstructured};- use rand::prelude::*;+ use crate::test::test_n_times;#[test]fn smoke_test_memory_access() {- let mut rng = SmallRng::seed_from_u64(0);- let mut buf = vec![0; 1024];-- for _ in 0..1024 {- rng.fill_bytes(&mut buf);- let u = Unstructured::new(&buf);- if let Ok(input) = MemoryAccesses::arbitrary_take_rest(u) {- check_memory_accesses(input);- }- }+ test_n_times(50, |input: MemoryAccesses, _u| {+ check_memory_accesses(input);+ Ok(())+ })}}diff --git a/crates/fuzzing/src/oracles/stacks.rs b/crates/fuzzing/src/oracles/stacks.rsindex 0d49fcc5ca..7a9ec51e1a 100644--- a/crates/fuzzing/src/oracles/stacks.rs+++ b/crates/fuzzing/src/oracles/stacks.rs@@ -138,30 +138,15 @@ fn assert_stack_matches(#[cfg(test)]mod tests {use super::*;- use arbitrary::{Arbitrary, Unstructured};- use rand::prelude::*;+ use crate::test::gen_until_pass;const TARGET_STACK_DEPTH: usize = 10;#[test]fn smoke_test() {- let mut rng = SmallRng::seed_from_u64(0);- let mut buf = vec![0; 2048];-- for _ in 0..1024 {- rng.fill_bytes(&mut buf);- let u = Unstructured::new(&buf);- if let Ok(stacks) = Stacks::arbitrary_take_rest(u) {- let max_stack_depth = check_stacks(stacks);- if max_stack_depth >= TARGET_STACK_DEPTH {- return;- }- }- }-- panic!(- "never generated a `Stacks` test case that reached {TARGET_STACK_DEPTH} \- deep stack frames",- );+ gen_until_pass(|stacks: Stacks, _u| {+ let max_stack_depth = check_stacks(stacks);+ Ok(max_stack_depth >= TARGET_STACK_DEPTH)+ });}}