Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cleanup/add some testing in `wasmtime-fuzzing` (#12038)

Alex Crichton882f22a7bd49
diff --git a/crates/fuzzing/src/generators/component_types.rs b/crates/fuzzing/src/generators/component_types.rs
index 5bcd76a2a5..5b1def366e 100644
--- a/crates/fuzzing/src/generators/component_types.rs
+++ b/crates/fuzzing/src/generators/component_types.rs
@@ -187,3 +187,25 @@ where
Ok(())
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::test::test_n_times;
+ use wasmtime_test_util::component_fuzz::{TestCase, Type};
+
+ #[test]
+ fn static_api_smoke_test() {
+ test_n_times(10, |(), u| {
+ let case = TestCase {
+ params: vec![&Type::S32, &Type::Bool, &Type::String],
+ result: Some(&Type::String),
+ encoding1: u.arbitrary()?,
+ encoding2: u.arbitrary()?,
+ };
+
+ let declarations = case.declarations();
+ static_api_test::<(i32, bool, String), (String,)>(u, &declarations)
+ });
+ }
+}
diff --git a/crates/fuzzing/src/generators/stacks.rs b/crates/fuzzing/src/generators/stacks.rs
index e7a3fcb31f..0e9e87e3f5 100644
--- a/crates/fuzzing/src/generators/stacks.rs
+++ b/crates/fuzzing/src/generators/stacks.rs
@@ -404,21 +404,15 @@ impl Stacks {
#[cfg(test)]
mod tests {
use super::*;
- use rand::prelude::*;
use wasmparser::Validator;
#[test]
fn stacks_generates_valid_wasm_modules() {
- let mut rng = SmallRng::seed_from_u64(0);
- let mut buf = vec![0; 2048];
- for _ in 0..1024 {
- rng.fill_bytes(&mut buf);
- let u = Unstructured::new(&buf);
- if let Ok(stacks) = Stacks::arbitrary_take_rest(u) {
- let wasm = stacks.wasm();
- validate(&wasm);
- }
- }
+ crate::test::test_n_times(10, |stacks: Stacks, _u| {
+ let wasm = stacks.wasm();
+ validate(&wasm);
+ Ok(())
+ })
}
fn validate(wasm: &[u8]) {
diff --git a/crates/fuzzing/src/lib.rs b/crates/fuzzing/src/lib.rs
index b5a405eb8b..0a2134e6f4 100644
--- a/crates/fuzzing/src/lib.rs
+++ b/crates/fuzzing/src/lib.rs
@@ -29,3 +29,44 @@ pub fn init_fuzzing() {
let _ = env_logger::try_init();
});
}
+
+#[cfg(test)]
+mod test {
+ use arbitrary::{Arbitrary, Unstructured};
+ use rand::prelude::*;
+
+ pub fn gen_until_pass<T: for<'a> Arbitrary<'a>>(
+ mut f: impl FnMut(T, &mut Unstructured<'_>) -> anyhow::Result<bool>,
+ ) -> bool {
+ let mut rng = SmallRng::seed_from_u64(0);
+ let mut buf = vec![0; 2048];
+ let n = 3000;
+ for _ in 0..n {
+ rng.fill_bytes(&mut buf);
+ let mut u = Unstructured::new(&buf);
+
+ if let Ok(config) = u.arbitrary() {
+ if f(config, &mut u).unwrap() {
+ return true;
+ }
+ }
+ }
+ false
+ }
+
+ /// Runs `f` with random data until it returns `Ok(())` `iters` times.
+ pub fn test_n_times<T: for<'a> Arbitrary<'a>>(
+ iters: u32,
+ mut f: impl FnMut(T, &mut Unstructured<'_>) -> arbitrary::Result<()>,
+ ) {
+ let mut to_test = 0..iters;
+ let ok = gen_until_pass(|a, b| {
+ if f(a, b).is_ok() {
+ Ok(to_test.next().is_none())
+ } else {
+ Ok(false)
+ }
+ });
+ assert!(ok);
+ }
+}
diff --git a/crates/fuzzing/src/oracles.rs b/crates/fuzzing/src/oracles.rs
index 9f6ef62ab9..090db067b3 100644
--- a/crates/fuzzing/src/oracles.rs
+++ b/crates/fuzzing/src/oracles.rs
@@ -1369,45 +1369,9 @@ pub fn call_async(wasm: &[u8], config: &generators::Config, mut poll_amts: &[u32
#[cfg(test)]
mod tests {
use super::*;
- use arbitrary::Unstructured;
- use rand::prelude::*;
+ use crate::test::{gen_until_pass, test_n_times};
use wasmparser::{Validator, WasmFeatures};
- fn gen_until_pass<T: for<'a> Arbitrary<'a>>(
- mut f: impl FnMut(T, &mut Unstructured<'_>) -> Result<bool>,
- ) -> bool {
- let mut rng = SmallRng::seed_from_u64(0);
- let mut buf = vec![0; 2048];
- let n = 3000;
- for _ in 0..n {
- rng.fill_bytes(&mut buf);
- let mut u = Unstructured::new(&buf);
-
- if let Ok(config) = u.arbitrary() {
- if f(config, &mut u).unwrap() {
- return true;
- }
- }
- }
- false
- }
-
- /// Runs `f` with random data until it returns `Ok(())` `iters` times.
- fn test_n_times<T: for<'a> Arbitrary<'a>>(
- iters: u32,
- mut f: impl FnMut(T, &mut Unstructured<'_>) -> arbitrary::Result<()>,
- ) {
- let mut to_test = 0..iters;
- let ok = gen_until_pass(|a, b| {
- if f(a, b).is_ok() {
- Ok(to_test.next().is_none())
- } else {
- Ok(false)
- }
- });
- assert!(ok);
- }
-
// Test that the `gc_ops` fuzzer eventually runs the gc function in the host.
// We've historically had issues where this fuzzer accidentally wasn't fuzzing
// anything for a long time so this is an attempt to prevent that from happening
@@ -1501,4 +1465,9 @@ mod tests {
fn wast_smoke_test() {
test_n_times(50, |(), u| super::wast_test(u));
}
+
+ #[test]
+ fn dynamic_component_api_smoke_test() {
+ test_n_times(50, |(), u| super::dynamic_component_api_target(u));
+ }
}
diff --git a/crates/fuzzing/src/oracles/engine.rs b/crates/fuzzing/src/oracles/engine.rs
index e64eca7aff..0da6c0beb2 100644
--- a/crates/fuzzing/src/oracles/engine.rs
+++ b/crates/fuzzing/src/oracles/engine.rs
@@ -181,27 +181,16 @@ pub fn smoke_test_engine<T>(
) where
T: DiffEngine,
{
- use rand::prelude::*;
-
- let mut rng = SmallRng::seed_from_u64(0);
- let mut buf = vec![0; 2048];
- let n = 100;
- for _ in 0..n {
- rng.fill_bytes(&mut buf);
- let mut u = Unstructured::new(&buf);
- let mut config = match u.arbitrary::<Config>() {
- Ok(config) => config,
- Err(_) => continue,
- };
+ crate::test::test_n_times(5, |mut config: Config, u| {
// This will ensure that wasmtime, which uses this configuration
// settings, can guaranteed instantiate a module.
config.set_differential_config();
- let mut engine = match mk_engine(&mut u, &mut config) {
+ let mut engine = match mk_engine(u, &mut config) {
Ok(engine) => engine,
Err(e) => {
println!("skip {e:?}");
- continue;
+ return Ok(());
}
};
@@ -240,8 +229,6 @@ pub fn smoke_test_engine<T>(
}
}
- return;
- }
-
- panic!("after {n} runs nothing ever ran, something is probably wrong");
+ Ok(())
+ })
}
diff --git a/crates/fuzzing/src/oracles/memory.rs b/crates/fuzzing/src/oracles/memory.rs
index 667139ecd4..d6ed4806d6 100644
--- a/crates/fuzzing/src/oracles/memory.rs
+++ b/crates/fuzzing/src/oracles/memory.rs
@@ -392,20 +392,13 @@ fn build_wasm(image: &HeapImage, offset: u32) -> Vec<u8> {
#[cfg(test)]
mod tests {
use super::*;
- use arbitrary::{Arbitrary, Unstructured};
- use rand::prelude::*;
+ use crate::test::test_n_times;
#[test]
fn smoke_test_memory_access() {
- let mut rng = SmallRng::seed_from_u64(0);
- let mut buf = vec![0; 1024];
-
- for _ in 0..1024 {
- rng.fill_bytes(&mut buf);
- let u = Unstructured::new(&buf);
- if let Ok(input) = MemoryAccesses::arbitrary_take_rest(u) {
- check_memory_accesses(input);
- }
- }
+ test_n_times(50, |input: MemoryAccesses, _u| {
+ check_memory_accesses(input);
+ Ok(())
+ })
}
}
diff --git a/crates/fuzzing/src/oracles/stacks.rs b/crates/fuzzing/src/oracles/stacks.rs
index 0d49fcc5ca..7a9ec51e1a 100644
--- a/crates/fuzzing/src/oracles/stacks.rs
+++ b/crates/fuzzing/src/oracles/stacks.rs
@@ -138,30 +138,15 @@ fn assert_stack_matches(
#[cfg(test)]
mod tests {
use super::*;
- use arbitrary::{Arbitrary, Unstructured};
- use rand::prelude::*;
+ use crate::test::gen_until_pass;
const TARGET_STACK_DEPTH: usize = 10;
#[test]
fn smoke_test() {
- let mut rng = SmallRng::seed_from_u64(0);
- let mut buf = vec![0; 2048];
-
- for _ in 0..1024 {
- rng.fill_bytes(&mut buf);
- let u = Unstructured::new(&buf);
- if let Ok(stacks) = Stacks::arbitrary_take_rest(u) {
- let max_stack_depth = check_stacks(stacks);
- if max_stack_depth >= TARGET_STACK_DEPTH {
- return;
- }
- }
- }
-
- panic!(
- "never generated a `Stacks` test case that reached {TARGET_STACK_DEPTH} \
- deep stack frames",
- );
+ gen_until_pass(|stacks: Stacks, _u| {
+ let max_stack_depth = check_stacks(stacks);
+ Ok(max_stack_depth >= TARGET_STACK_DEPTH)
+ });
}
}