A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
//! Utilities for testing and fuzzing out-of-memory handling.
//!
//! Inspired by SpiderMonkey's `oomTest()` helper:
//! <https://firefox-source-docs.mozilla.org/js/hacking_tips.html#how-to-debug-oomtest-failures>
use backtrace::Backtrace;
use rand::{RngExt, SeedableRng, rngs::SmallRng};
use std::{
alloc::GlobalAlloc,
cell::Cell,
mem,
pin::Pin,
ptr,
task::{Context, Poll},
time,
};
use wasmtime_core::error::{Context as _, OutOfMemory, Result, bail, ensure};
/// An allocator for use with `OomTest`.
#[non_exhaustive]
pub struct OomTestAllocator;
impl OomTestAllocator {
/// Create a new OOM test allocator.
pub const fn new() -> Self {
OomTestAllocator
}
}
#[derive(Clone, Debug, Default, PartialEq, Eq)]
enum OomState {
/// We are in code that is not part of an OOM test.
#[default]
OutsideOomTest,
/// We are inside an OOM test's dry run, counting total allocations.
Counting { count: u32 },
/// We are inside an OOM test and should inject an OOM when the counter
/// reaches zero.
OomOnAlloc {
counter: u32,
allow_alloc_after: bool,
alloc_succeeds_after: bool,
},
/// We are inside an OOM test and we already injected an OOM.
DidOom {
allow_alloc: bool,
alloc_succeeds_after: bool,
},
}
thread_local! {
static OOM_STATE: Cell<OomState> = const { Cell::new(OomState::OutsideOomTest) };
}
/// Set the new OOM state, returning the old state.
fn set_oom_state(state: OomState) -> OomState {
OOM_STATE.with(|s| s.replace(state))
}
/// RAII helper to set the OOM state within a block of code and reset it upon
/// exiting that block (even if exiting via panic unwinding).
struct ScopedOomState<'a> {
state: &'a mut OomState,
}
impl<'a> ScopedOomState<'a> {
/// The OOM state will be initialized to `*state` on construction, and then
/// `state` will be updated to be the old OOM state that existed when the
/// original state is replaced on this type's drop.
fn new(state: &'a mut OomState) -> Self {
*state = set_oom_state(mem::take(state));
ScopedOomState { state }
}
}
impl Drop for ScopedOomState<'_> {
fn drop(&mut self) {
*self.state = set_oom_state(mem::take(self.state));
}
}
unsafe impl GlobalAlloc for OomTestAllocator {
unsafe fn alloc(&self, layout: std::alloc::Layout) -> *mut u8 {
let old_state = set_oom_state(OomState::OutsideOomTest);
let new_state;
let ptr;
'outside_oom_test: {
// NB: It's okay to log/backtrace/etc... in this block because the
// current state is `OutsideOomTest`, so any re-entrant allocations
// will be passed through to the system allocator.
// Don't panic on allocation-after-OOM attempts if we
// are already in the middle of panicking. That will
// cause an abort and we won't get as good of an error
// message for the original panic, which is most likely
// some kind of test failure.
if old_state == OomState::OutsideOomTest || std::thread::panicking() {
new_state = old_state;
ptr = unsafe { std::alloc::System.alloc(layout) };
break 'outside_oom_test;
}
let bt = Backtrace::new();
let bt = format!("{bt:?}");
// XXX: `env_logger` internally buffers writes in a `Vec` which
// means our OOM tests might sporadically fail when you enable
// logging to debug stuff, so simply let the allocation through if
// `env_logger` is on the stack.
if bt.contains("env_logger") {
new_state = old_state;
ptr = unsafe { std::alloc::System.alloc(layout) };
break 'outside_oom_test;
}
match old_state {
OomState::OutsideOomTest => unreachable!("handled above"),
OomState::Counting { count } => {
new_state = OomState::Counting { count: count + 1 };
ptr = unsafe { std::alloc::System.alloc(layout) };
}
OomState::OomOnAlloc {
counter: 0,
allow_alloc_after,
alloc_succeeds_after,
} => {
log::debug!(
"injecting OOM for allocation: {layout:?}\nAllocation backtrace:\n{bt}"
);
new_state = OomState::DidOom {
allow_alloc: allow_alloc_after,
alloc_succeeds_after,
};
ptr = ptr::null_mut();
}
OomState::OomOnAlloc {
counter: c,
allow_alloc_after,
alloc_succeeds_after,
} => {
new_state = OomState::OomOnAlloc {
counter: c - 1,
allow_alloc_after,
alloc_succeeds_after,
};
ptr = unsafe { std::alloc::System.alloc(layout) };
}
OomState::DidOom {
allow_alloc,
alloc_succeeds_after,
} => {
log::trace!("Attempt to allocate {layout:?} after OOM:\n{bt}");
if allow_alloc {
new_state = OomState::DidOom {
allow_alloc: true,
alloc_succeeds_after,
};
ptr = if alloc_succeeds_after {
unsafe { std::alloc::System.alloc(layout) }
} else {
ptr::null_mut()
};
} else {
panic!(
"OOM test attempted to allocate after OOM: {layout:?}\n\
\n\
Hint: if this is acceptable, configure the OOM test to allow allocation \
after OOM with `OomTest::allow_alloc_after_oom`"
)
}
}
}
}
set_oom_state(new_state);
ptr
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: std::alloc::Layout) {
unsafe {
std::alloc::System.dealloc(ptr, layout);
}
}
}
/// A test helper that checks that some code handles OOM correctly.
///
/// `OomTest` will only work correctly when `OomTestAllocator` is configured as
/// the global allocator.
///
/// `OomTest` does not support reentrancy, so you cannot run an `OomTest` within
/// an `OomTest`.
///
/// # Example
///
/// ```no_run
/// use std::time::Duration;
/// use wasmtime::Result;
/// use wasmtime_fuzzing::oom::{OomTest, OomTestAllocator};
///
/// #[global_allocator]
/// static GLOBAL_ALLOCATOR: OomTestAllocator = OomTestAllocator::new();
///
/// #[test]
/// fn my_oom_test() -> Result<()> {
/// OomTest::new()
/// .max_iters(1_000_000)
/// .max_duration(Duration::from_secs(5))
/// .allow_alloc_after_oom(true)
/// .test(|| {
/// todo!("insert code here that should handle OOM here...")
/// })
/// }
/// ```
///
/// Use `test_with_reuse` when you also want to check that some context is still
/// usable after the OOM, e.g. that nothing was left half-registered in state
/// that is shared with an `Engine`:
///
/// ```no_run
/// use wasmtime::Result;
/// use wasmtime_fuzzing::oom::OomTest;
///
/// #[test]
/// fn my_oom_reuse_test() -> Result<()> {
/// OomTest::new().test_with_reuse(
/// || todo!("create the context that is reused after the OOM..."),
/// |context| todo!("insert code here that should handle OOM here..."),
/// |context| todo!("check that `context` is still usable..."),
/// )
/// }
/// ```
pub struct OomTest {
max_iters: Option<u32>,
max_duration: Option<time::Duration>,
allow_alloc_after_oom: bool,
alloc_succeeds_after_oom: bool,
allow_missed_oom_errors: bool,
seed: u64,
}
impl OomTest {
/// Create a new OOM test.
///
/// By default there is no iteration or time limit, tests will be executed
/// until the pass (or fail).
pub fn new() -> Self {
let _ = env_logger::try_init();
// NB: `std::backtrace::Backtrace` doesn't have ways to handle
// OOM. Ideally we would just disable the `"backtrace"` cargo feature,
// but workspace feature resolution doesn't play nice with that.
wasmtime_core::error::disable_backtrace();
OomTest {
max_iters: None,
max_duration: None,
allow_alloc_after_oom: false,
alloc_succeeds_after_oom: false,
allow_missed_oom_errors: false,
seed: 0,
}
}
/// Configure the maximum number of times to run an OOM test.
pub fn max_iters(&mut self, max_iters: u32) -> &mut Self {
self.max_iters = Some(max_iters);
self
}
/// Configure the maximum duration of time to run an OOM text.
pub fn max_duration(&mut self, max_duration: time::Duration) -> &mut Self {
self.max_duration = Some(max_duration);
self
}
/// Configure whether to allow allocation attempts after an OOM has already
/// been injected.
///
/// The default is `false`.
pub fn allow_alloc_after_oom(&mut self, allow: bool) -> &mut Self {
self.allow_alloc_after_oom = allow;
self
}
/// Configure whether allocations after an OOM (assuming
/// `allow_alloc_after_oom(true)`) will succeed or will always return null.
///
/// The default is `false`.
pub fn alloc_succeeds_after_oom(&mut self, succeeds: bool) -> &mut Self {
self.alloc_succeeds_after_oom = succeeds;
self
}
/// Configure whether to allow a test to pass if it does not return
/// `Err(OutOfMemory)` when a synthetic OOM was injected (perhaps because it
/// is exercising logic that is robust to OOM).
///
/// The default is `false`.
pub fn allow_missed_oom_errors(&mut self, allow: bool) -> &mut Self {
self.allow_missed_oom_errors = allow;
self
}
/// Configure the seed for the RNG used in `fuzz` mode.
///
/// The default is `0`.
pub fn seed(&mut self, seed: u64) -> &mut Self {
self.seed = seed;
self
}
/// Repeatedly run the given test function, injecting OOMs at different
/// times and checking that it correctly handles them.
///
/// The test function should not use threads, or else allocations may not be
/// tracked correctly and OOM injection may be incorrect.
///
/// The test function should return an `Err(_)` if and only if it encounters
/// an OOM.
///
/// Returns early once the test function returns `Ok(())` before an OOM has
/// been injected.
pub fn test(&self, test_func: impl Fn() -> Result<()>) -> Result<()> {
let future = self.test_async(|| async { test_func() });
crate::block_on(future)
}
/// The same as `test` but `async`.
pub async fn test_async(&self, test_func: impl AsyncFn() -> Result<()>) -> Result<()> {
self.test_with_reuse_async(
async || Ok(()),
async |_: &mut ()| test_func().await,
async |_: ()| Ok(()),
)
.await
}
/// The same as `test`, but additionally exercising context that is reused
/// after the OOM is triggered.
///
/// Each iteration runs three phases in sequence:
///
/// 1. `make_context` creates the context that is shared between the two
/// phases that follow. It is called afresh every iteration, so that each
/// iteration begins from identical state.
///
/// 2. `test_func` is the main OOM test, and is the only phase into which
/// OOMs are injected. It is otherwise the same as the function given to
/// `test`.
///
/// 3. `post_oom_func` checks that we recovered from the OOM correctly and
/// that we can still interact with the context afterwards, e.g. that
/// nothing was left half-registered in shared state, rather than
/// properly cleaned up, when the OOM happened.
///
/// OOMs are never injected into `make_context` or `post_oom_func` and
/// neither's allocations are counted. An `Err(_)` returned from either is a
/// test failure.
pub fn test_with_reuse<T>(
&self,
make_context: impl Fn() -> Result<T>,
test_func: impl Fn(&mut T) -> Result<()>,
post_oom_func: impl Fn(T) -> Result<()>,
) -> Result<()> {
let future = self.test_with_reuse_async(
async || make_context(),
async |context: &mut T| test_func(context),
async |context: T| post_oom_func(context),
);
crate::block_on(future)
}
/// The same as `test_with_reuse` but `async`.
pub async fn test_with_reuse_async<T>(
&self,
make_context: impl AsyncFn() -> Result<T>,
test_func: impl AsyncFn(&mut T) -> Result<()>,
post_oom_func: impl AsyncFn(T) -> Result<()>,
) -> Result<()> {
let total_allocs =
Self::count_allocations(&make_context, &test_func, &post_oom_func).await?;
let start = time::Instant::now();
for i in 0..total_allocs {
if self.max_iters.is_some_and(|n| i >= n)
|| self.max_duration.is_some_and(|d| start.elapsed() >= d)
{
break;
}
log::trace!("=== Injecting OOM after {i} allocations ===");
self.run_one_oom_injection(&make_context, &test_func, &post_oom_func, i)
.await?;
}
Ok(())
}
/// Similar to `test` but instead of exhaustively injecting OOMs on every
/// allocation, randomly chooses which allocation to OOM each iteration.
///
/// Requires `max_iters` to be set.
pub fn fuzz(&self, test_func: impl Fn() -> Result<()>) -> Result<()> {
let future = self.fuzz_async(|| async { test_func() });
crate::block_on(future)
}
/// The same as `fuzz` but `async`.
pub async fn fuzz_async(&self, test_func: impl AsyncFn() -> Result<()>) -> Result<()> {
let max_iters = match self.max_iters {
Some(n) => n,
None => bail!("OomTest::fuzz requires max_iters to be set"),
};
let make_context = async || Ok(());
let test_func = async |_: &mut ()| test_func().await;
let post_oom_func = async |_: ()| Ok(());
let total_allocs =
Self::count_allocations(&make_context, &test_func, &post_oom_func).await?;
if total_allocs == 0 {
return Ok(());
}
let mut rng = SmallRng::seed_from_u64(self.seed);
for _ in 0..max_iters {
let i = rng.random_range(0..total_allocs);
log::trace!("=== Injecting OOM after {i} allocations (fuzz) ===");
self.run_one_oom_injection(&make_context, &test_func, &post_oom_func, i)
.await?;
}
Ok(())
}
/// Run the test function once, injecting OOM on the `i`th allocation, and
/// check that the result correctly reflects whether an OOM was hit.
async fn run_one_oom_injection<T>(
&self,
make_context: &impl AsyncFn() -> Result<T>,
test_func: &impl AsyncFn(&mut T) -> Result<()>,
post_oom_func: &impl AsyncFn(T) -> Result<()>,
i: u32,
) -> Result<()> {
let mut context = make_context()
.await
.context("OOM test failed to create its reused context")?;
let (result, oom_state) = {
let future = std::pin::pin!(test_func(&mut context));
OomTestFuture::new(
future,
OomState::OomOnAlloc {
counter: i,
allow_alloc_after: self.allow_alloc_after_oom,
alloc_succeeds_after: self.alloc_succeeds_after_oom,
},
)
.await
};
match (result, oom_state) {
(_, OomState::OutsideOomTest | OomState::Counting { .. }) => unreachable!(),
// The test function completed successfully before we reached
// this allocation point (allocation count may differ slightly
// between the counting run and OOM injection runs).
(Ok(()), OomState::OomOnAlloc { .. }) => {}
// We injected an OOM and the test function handled it correctly.
(Err(e), OomState::DidOom { .. }) if e.is::<OutOfMemory>() => {}
// Missed OOMs.
(Ok(()), OomState::DidOom { .. }) => {
ensure!(
self.allow_missed_oom_errors,
"OOM test function missed an OOM: returned Ok(())"
);
}
(Err(e), OomState::DidOom { .. }) => {
if !self.allow_missed_oom_errors {
return Err(
e.context("OOM test function missed an OOM: returned non-OOM error")
);
}
}
// Unexpected error.
(Err(e), OomState::OomOnAlloc { .. }) => {
if !self.allow_missed_oom_errors {
return Err(
e.context("OOM test function returned an error when there was no OOM")
);
}
}
}
post_oom_func(context)
.await
.context("OOM test's post-OOM function failed")
}
/// Run the test function without injecting OOMs, counting allocations.
///
/// Repeats until the count stabilizes across two consecutive runs,
/// since the first run may change internal state (e.g. grow hash map
/// capacities) that subsequent runs reuse.
async fn count_allocations<T>(
make_context: &impl AsyncFn() -> Result<T>,
test_func: &impl AsyncFn(&mut T) -> Result<()>,
post_oom_func: &impl AsyncFn(T) -> Result<()>,
) -> Result<u32> {
const MAX_DRY_RUN_ITERS: usize = 10;
let mut total_allocs = u32::MAX;
for i in 0..MAX_DRY_RUN_ITERS {
log::trace!("=== Counting allocations (dry run {i}) ===");
let mut context = make_context()
.await
.context("OOM test failed to create its reused context")?;
let (result, oom_state) = {
let future = std::pin::pin!(test_func(&mut context));
OomTestFuture::new(future, OomState::Counting { count: 0 }).await
};
result?;
post_oom_func(context)
.await
.context("OOM test's post-OOM function failed")?;
let count = match oom_state {
OomState::Counting { count } => count,
_ => unreachable!(),
};
log::trace!("=== Counted {count} allocations ===");
if count == total_allocs {
return Ok(total_allocs);
}
total_allocs = count;
}
bail!(
"allocation count did not stabilize after {MAX_DRY_RUN_ITERS} \
dry-run iterations (last count: {total_allocs})"
)
}
}
struct OomTestFuture<'a, F> {
inner: Pin<&'a mut F>,
state: OomState,
}
impl<'a, F> OomTestFuture<'a, F> {
fn new(inner: Pin<&'a mut F>, state: OomState) -> Self {
OomTestFuture { inner, state }
}
}
impl<F> Future for OomTestFuture<'_, F>
where
F: Future<Output = Result<()>>,
{
type Output = (Result<()>, OomState);
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
log::trace!("OomTestFuture::poll: Entered ---");
let result = {
let OomTestFuture { inner, state } = &mut *self;
let guard = ScopedOomState::new(state);
assert_eq!(*guard.state, OomState::OutsideOomTest);
inner.as_mut().poll(cx)
};
log::trace!("OomTestFuture::poll: inner result: {result:?}");
log::trace!("OomTestFuture::poll: old state: {:?}", self.state);
match result {
Poll::Pending => {
set_oom_state(OomState::OutsideOomTest);
Poll::Pending
}
Poll::Ready(result) => Poll::Ready((result, mem::take(&mut self.state))),
}
}
}