Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Add GC zeal assertions (#12933)

Nick Fitzgeralddad8432da2ad
diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs
index d95109dfe6..45f52d6769 100644
--- a/crates/cranelift/src/func_environ/gc/enabled.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled.rs
@@ -94,8 +94,55 @@ fn unbarriered_store_gc_ref(
Ok(())
}
-/// Emit code to read a struct field or array element from its raw address in
-/// the GC heap.
+/// Emit inline CLIF code that asserts an object's `VMGcKind` matches the
+/// expected kind. Only emits code when `cfg(gc_zeal)` is enabled.
+///
+/// `gc_ref` must be a non-null, non-i31 GC reference (i32 heap index).
+fn emit_gc_kind_assert(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ gc_ref: ir::Value,
+ expected_kind: VMGcKind,
+) {
+ if !cfg!(gc_zeal) {
+ return;
+ }
+
+ func_env.trapz(builder, gc_ref, crate::TRAP_NULL_REFERENCE);
+
+ let kind_addr = func_env.prepare_gc_ref_access(
+ builder,
+ gc_ref,
+ BoundsCheck::StaticObjectField {
+ offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,
+ access_size: wasmtime_environ::VM_GC_KIND_SIZE,
+ object_size: wasmtime_environ::VM_GC_HEADER_SIZE,
+ },
+ );
+ let kind_and_reserved_bits = builder.ins().load(
+ ir::types::I32,
+ ir::MemFlags::trusted().with_readonly(),
+ kind_addr,
+ 0,
+ );
+ let kind_mask = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(VMGcKind::MASK));
+ let actual_kind = builder.ins().band(kind_and_reserved_bits, kind_mask);
+
+ let expected_kind = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));
+
+ // NB: Do a subtype check rather than a strict equality check. See
+ // `VMGcKind::matches` for details.
+ let and = builder.ins().band(actual_kind, expected_kind);
+ let matches = builder.ins().icmp(IntCC::Equal, and, expected_kind);
+
+ builder.ins().trapz(matches, TRAP_INTERNAL_ASSERT);
+}
+
+/// Read a struct field or array element from its raw address in the GC heap.
///
/// The given address MUST have already been bounds-checked via
/// `prepare_gc_ref_access`.
@@ -331,6 +378,8 @@ pub fn translate_struct_get(
// type info from `wasmparser` and through to here is a bit funky.
func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);
+ emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);
+
let field_index = usize::try_from(field_index).unwrap();
let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();
@@ -378,6 +427,8 @@ pub fn translate_struct_set(
// TODO: See comment in `translate_struct_get` about the `trapz`.
func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);
+ emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);
+
let field_index = usize::try_from(field_index).unwrap();
let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();
@@ -979,6 +1030,8 @@ pub fn translate_array_get(
) -> WasmResult<ir::Value> {
log::trace!("translate_array_get({array_type_index:?}, {array_ref:?}, {index:?})");
+ emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);
+
let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();
let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index);
@@ -1000,6 +1053,8 @@ pub fn translate_array_set(
) -> WasmResult<()> {
log::trace!("translate_array_set({array_type_index:?}, {array_ref:?}, {index:?}, {value:?})");
+ emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);
+
let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();
let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index);
diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs
index 492f31f953..5de8ba4a24 100644
--- a/crates/environ/src/gc.rs
+++ b/crates/environ/src/gc.rs
@@ -499,15 +499,8 @@ impl VMGcKind {
#[inline]
pub fn from_high_bits_of_u32(val: u32) -> VMGcKind {
let masked = val & Self::MASK;
- let result = match masked {
- x if x == Self::ExternRef.as_u32() => Self::ExternRef,
- x if x == Self::AnyRef.as_u32() => Self::AnyRef,
- x if x == Self::EqRef.as_u32() => Self::EqRef,
- x if x == Self::ArrayRef.as_u32() => Self::ArrayRef,
- x if x == Self::StructRef.as_u32() => Self::StructRef,
- x if x == Self::ExnRef.as_u32() => Self::ExnRef,
- _ => panic!("invalid `VMGcKind`: {masked:#032b}"),
- };
+ let result = Self::try_from_u32(masked)
+ .unwrap_or_else(|| panic!("invalid `VMGcKind`: {masked:#032b}"));
let poison_kind = u32::from_le_bytes([POISON, POISON, POISON, POISON]) & VMGcKind::MASK;
debug_assert_ne!(
@@ -531,6 +524,22 @@ impl VMGcKind {
pub fn as_u32(self) -> u32 {
self as u32
}
+
+ /// Try to convert a `u32` into a `VMGcKind`.
+ ///
+ /// Returns `None` if the value doesn't match any known kind.
+ #[inline]
+ pub fn try_from_u32(x: u32) -> Option<VMGcKind> {
+ match x {
+ _ if x == Self::ExternRef.as_u32() => Some(Self::ExternRef),
+ _ if x == Self::AnyRef.as_u32() => Some(Self::AnyRef),
+ _ if x == Self::EqRef.as_u32() => Some(Self::EqRef),
+ _ if x == Self::ArrayRef.as_u32() => Some(Self::ArrayRef),
+ _ if x == Self::StructRef.as_u32() => Some(Self::StructRef),
+ _ if x == Self::ExnRef.as_u32() => Some(Self::ExnRef),
+ _ => None,
+ }
+ }
}
#[cfg(test)]
diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs
index 10e1805d08..4656866578 100644
--- a/crates/wasmtime/src/runtime/store/gc.rs
+++ b/crates/wasmtime/src/runtime/store/gc.rs
@@ -56,11 +56,16 @@ impl StoreOpaque {
bytes_needed: Option<u64>,
asyncness: Asyncness,
) {
- if let Some(n) = bytes_needed {
+ if let Some(n) = bytes_needed
+ // The gc_zeal's allocation counter will pass `bytes_needed == 0` to
+ // signify that we shouldn't grow the GC heap, just do a collection.
+ && n > 0
+ {
if self.grow_gc_heap(limiter, n).await.is_ok() {
return;
}
}
+
self.do_gc(asyncness).await;
}
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index 6067b565d5..b33365a43c 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -48,6 +48,18 @@ pub struct GcStore {
/// The function-references table for this GC heap.
pub func_ref_table: FuncRefTable,
+
+ /// An allocation counter that triggers GC when it reaches zero.
+ ///
+ /// Initialized from the `WASMTIME_GC_ZEAL_ALLOC_COUNTER` environment
+ /// variable. Decremented on every allocation and when it hits zero, a GC is
+ /// forced and the counter is reset.
+ #[cfg(all(gc_zeal, feature = "std"))]
+ gc_zeal_alloc_counter: Option<NonZeroU32>,
+
+ /// The initial value to reset the counter to after it triggers.
+ #[cfg(all(gc_zeal, feature = "std"))]
+ gc_zeal_alloc_counter_init: Option<NonZeroU32>,
}
impl GcStore {
@@ -55,11 +67,29 @@ impl GcStore {
pub fn new(allocation_index: GcHeapAllocationIndex, gc_heap: Box<dyn GcHeap>) -> Self {
let host_data_table = ExternRefHostDataTable::default();
let func_ref_table = FuncRefTable::default();
+
+ #[cfg(all(gc_zeal, feature = "std"))]
+ let gc_zeal_alloc_counter_init =
+ std::env::var("WASMTIME_GC_ZEAL_ALLOC_COUNTER")
+ .ok()
+ .map(|v| {
+ v.parse::<NonZeroU32>().unwrap_or_else(|_| {
+ panic!(
+ "`WASMTIME_GC_ZEAL_ALLOC_COUNTER` must be a non-zero \
+ `u32` value, got: {v}"
+ )
+ })
+ });
+
Self {
allocation_index,
gc_heap,
host_data_table,
func_ref_table,
+ #[cfg(all(gc_zeal, feature = "std"))]
+ gc_zeal_alloc_counter: gc_zeal_alloc_counter_init,
+ #[cfg(all(gc_zeal, feature = "std"))]
+ gc_zeal_alloc_counter_init,
}
}
@@ -231,6 +261,20 @@ impl GcStore {
header: VMGcHeader,
layout: Layout,
) -> Result<Result<VMGcRef, u64>> {
+ // When gc_zeal is enabled with an allocation counter, decrement it and
+ // force a GC cycle when it reaches zero by returning a fake OOM.
+ #[cfg(all(gc_zeal, feature = "std"))]
+ if let Some(counter) = self.gc_zeal_alloc_counter.take() {
+ match NonZeroU32::new(counter.get() - 1) {
+ Some(c) => self.gc_zeal_alloc_counter = Some(c),
+ None => {
+ log::trace!("gc_zeal: allocation counter reached zero, forcing GC");
+ self.gc_zeal_alloc_counter = self.gc_zeal_alloc_counter_init;
+ return Ok(Err(0));
+ }
+ }
+ }
+
self.gc_heap.alloc_raw(header, layout)
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index 94bc99abb9..774f4c4c3f 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -64,7 +64,8 @@ use core::{
};
use wasmtime_environ::drc::{ARRAY_LENGTH_OFFSET, DrcTypeLayouts};
use wasmtime_environ::{
- GcArrayLayout, GcLayout, GcStructLayout, GcTypeLayouts, VMGcKind, VMSharedTypeIndex,
+ GcArrayLayout, GcLayout, GcStructLayout, GcTypeLayouts, POISON, VMGcKind, VMSharedTypeIndex,
+ gc_assert,
};
#[expect(clippy::cast_possible_truncation, reason = "known to not overflow")]
@@ -177,10 +178,15 @@ impl DrcHeap {
let drc_ref = drc_ref(&gc_ref);
let size = self.index(drc_ref).object_size();
let layout = FreeList::layout(size);
- self.free_list
- .as_mut()
- .unwrap()
- .dealloc(gc_ref.as_heap_index().unwrap(), layout);
+ let index = gc_ref.as_heap_index().unwrap();
+
+ // Poison the freed memory so that any stale access is detectable.
+ if cfg!(gc_zeal) {
+ let index = usize::try_from(index.get()).unwrap();
+ self.heap_slice_mut()[index..][..layout.size()].fill(POISON);
+ }
+
+ self.free_list.as_mut().unwrap().dealloc(index, layout);
}
/// Increment the ref count for the associated object.
@@ -311,6 +317,7 @@ impl DrcHeap {
debug_assert!(header.kind().matches(VMGcKind::ExternRef));
return;
};
+
match self
.trace_infos
.get(&ty)
@@ -321,11 +328,24 @@ impl DrcHeap {
let data = self.gc_object_data(gc_ref);
for offset in gc_ref_offsets {
let raw = data.read_u32(*offset);
- if let Some(gc_ref) = VMGcRef::from_raw_u32(raw) {
+ if let Some(gc_ref) = VMGcRef::from_raw_u32(raw)
+ && !gc_ref.is_i31()
+ {
+ debug_assert!(
+ {
+ let header = self.header(&gc_ref);
+ let kind = header.kind().as_u32();
+ VMGcKind::try_from_u32(kind).is_some()
+ },
+ "trace_gc_ref: struct field at offset {offset} references object \
+ with invalid `VMGcKind`",
+ );
+
stack.push(gc_ref);
}
}
}
+
TraceInfo::Array { gc_ref_elems } => {
if !*gc_ref_elems {
return;
@@ -338,7 +358,19 @@ impl DrcHeap {
let elem_offset = GC_REF_ARRAY_ELEMS_OFFSET
+ i * u32::try_from(mem::size_of::<u32>()).unwrap();
let raw = data.read_u32(elem_offset);
- if let Some(gc_ref) = VMGcRef::from_raw_u32(raw) {
+ if let Some(gc_ref) = VMGcRef::from_raw_u32(raw)
+ && !gc_ref.is_i31()
+ {
+ debug_assert!(
+ {
+ let header = self.header(&gc_ref);
+ let kind = header.kind().as_u32();
+ VMGcKind::try_from_u32(kind).is_some()
+ },
+ "trace_gc_ref: array element at index {i} references object \
+ with invalid `VMGcKind`",
+ );
+
stack.push(gc_ref);
}
}
@@ -359,6 +391,64 @@ impl DrcHeap {
})
}
+ /// Assert the integrity of the over-approximated stack roots list.
+ fn assert_over_approximated_stack_roots_integrity(&self) {
+ if !cfg!(gc_zeal) {
+ return;
+ }
+
+ let mut visited = HashSet::new();
+ for gc_ref in self.iter_over_approximated_stack_roots() {
+ let idx = gc_ref.as_heap_index().unwrap().get();
+
+ // Each entry must have a valid `VMGcKind`.
+ let header = self.header(&gc_ref);
+ let kind = header.kind().as_u32();
+ assert!(
+ VMGcKind::try_from_u32(kind).is_some(),
+ "over-approx list: entry at heap index {idx} has invalid VMGcKind {kind:#034b}",
+ );
+
+ // Each entry must have its in-list bit set.
+ let drc_header = self.index(drc_ref(&gc_ref));
+ assert!(
+ drc_header.is_in_over_approximated_stack_roots(),
+ "over-approx list: entry at heap index {idx} does not have in-list bit set",
+ );
+
+ // Each entry must have a nonzero ref count.
+ assert_ne!(
+ drc_header.ref_count, 0,
+ "over-approx list: entry at heap index {idx} has zero ref count",
+ );
+
+ // No cycles or duplicates.
+ assert!(
+ visited.insert(idx),
+ "over-approx list: cycle or duplicate detected at heap index {idx}",
+ );
+ }
+ }
+
+ /// Assert that every free block in the free list is filled with the poison
+ /// pattern.
+ fn assert_free_blocks_are_poisoned(&self) {
+ if !cfg!(gc_zeal) {
+ return;
+ }
+
+ let free_list = self.free_list.as_ref().unwrap();
+ for (index, len) in free_list.iter_free_blocks() {
+ let start = usize::try_from(index).unwrap();
+ let size = usize::try_from(len).unwrap();
+ let slice = &self.heap_slice()[start..][..size];
+ assert!(
+ slice.iter().all(|&b| b == POISON),
+ "free block at heap index {start} (size {size}) is not fully poisoned",
+ );
+ }
+ }
+
fn trace(&mut self, roots: &mut GcRootsIter<'_>) {
// The `over_approx_set` is used for `debug_assert!`s checking that
// every reference we read out from the stack via stack maps is actually
@@ -685,6 +775,12 @@ unsafe impl GcHeap for DrcHeap {
self.free_list = Some(FreeList::new(len));
self.vmmemory = Some(memory.vmmemory());
self.memory = Some(memory);
+
+ // Poison the entire heap so any access to uninitialized memory is
+ // detectable.
+ if cfg!(gc_zeal) {
+ self.heap_slice_mut().fill(POISON);
+ }
}
fn detach(&mut self) -> crate::vm::Memory {
@@ -795,11 +891,27 @@ unsafe impl GcHeap for DrcHeap {
}
fn header(&self, gc_ref: &VMGcRef) -> &VMGcHeader {
- self.index(gc_ref.as_typed_unchecked())
+ let header: &VMGcHeader = self.index(gc_ref.as_typed_unchecked());
+
+ debug_assert!(
+ VMGcKind::try_from_u32(header.kind().as_u32()).is_some(),
+ "header: invalid VMGcKind {:#010x} at gc_ref {gc_ref:#p}",
+ header.kind().as_u32(),
+ );
+
+ header
}
fn header_mut(&mut self, gc_ref: &VMGcRef) -> &mut VMGcHeader {
- self.index_mut(gc_ref.as_typed_unchecked())
+ let header: &mut VMGcHeader = self.index_mut(gc_ref.as_typed_unchecked());
+
+ debug_assert!(
+ VMGcKind::try_from_u32(header.kind().as_u32()).is_some(),
+ "header_mut: invalid VMGcKind {:#010x} at gc_ref {gc_ref:#p}",
+ header.kind().as_u32(),
+ );
+
+ header
}
fn object_size(&self, gc_ref: &VMGcRef) -> usize {
@@ -828,6 +940,19 @@ unsafe impl GcHeap for DrcHeap {
Some(index) => VMGcRef::from_heap_index(index).unwrap(),
};
+ // Assert that the newly-allocated memory is still filled with the
+ // poison pattern, and hasn't been corrupted since deallocation (or
+ // initial heap creation).
+ if cfg!(gc_zeal) {
+ let start = usize::try_from(gc_ref.as_heap_index().unwrap().get()).unwrap();
+ let slice = &self.heap_slice()[start..][..layout.size()];
+ gc_assert!(
+ slice.iter().all(|&b| b == POISON),
+ "newly allocated GC object at index {start} is not fully poisoned; \
+ freed memory was corrupted",
+ );
+ }
+
*self.index_mut(drc_ref(&gc_ref)) = VMDrcHeader {
header,
ref_count: 1,
@@ -922,6 +1047,16 @@ unsafe impl GcHeap for DrcHeap {
self.vmmemory = Some(memory.vmmemory());
self.memory = Some(memory);
+ // Poison the newly-grown region so stale accesses are detectable.
+ if cfg!(gc_zeal) {
+ let old_cap = self.free_list.as_ref().unwrap().current_capacity();
+ let new_bytes = usize::try_from(delta_bytes_grown).unwrap();
+ let slice = self.heap_slice_mut();
+ if old_cap + new_bytes <= slice.len() {
+ slice[old_cap..old_cap + new_bytes].fill(POISON);
+ }
+ }
+
self.free_list
.as_mut()
.unwrap()
@@ -958,14 +1093,30 @@ impl<'a> GarbageCollection<'a> for DrcCollection<'a> {
match self.phase {
DrcCollectionPhase::Trace => {
log::trace!("Begin DRC trace");
+
+ self.heap.assert_over_approximated_stack_roots_integrity();
+ self.heap.assert_free_blocks_are_poisoned();
+
self.heap.trace(&mut self.roots);
+
+ self.heap.assert_over_approximated_stack_roots_integrity();
+ self.heap.assert_free_blocks_are_poisoned();
+
log::trace!("End DRC trace");
self.phase = DrcCollectionPhase::Sweep;
GcProgress::Continue
}
DrcCollectionPhase::Sweep => {
log::trace!("Begin DRC sweep");
+
+ self.heap.assert_over_approximated_stack_roots_integrity();
+ self.heap.assert_free_blocks_are_poisoned();
+
self.heap.sweep(self.host_data_table);
+
+ self.heap.assert_over_approximated_stack_roots_integrity();
+ self.heap.assert_free_blocks_are_poisoned();
+
log::trace!("End DRC sweep");
self.phase = DrcCollectionPhase::Done;
GcProgress::Complete
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs
index 7d0fb7c877..e638e5ea90 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs
@@ -40,6 +40,11 @@ impl FreeList {
Layout::from_size_align(size, ALIGN_USIZE).unwrap()
}
+ /// Get the current total capacity this free list manages.
+ pub fn current_capacity(&self) -> usize {
+ self.capacity
+ }
+
/// Create a new `FreeList` for a contiguous region of memory of the given
/// size.
pub fn new(capacity: usize) -> Self {
@@ -340,6 +345,11 @@ impl FreeList {
self.check_integrity();
}
+ /// Iterate over all free blocks as `(index, len)` pairs.
+ pub fn iter_free_blocks(&self) -> impl Iterator<Item = (u32, u32)> + '_ {
+ self.free_block_index_to_len.iter().map(|(&i, &l)| (i, l))
+ }
+
/// Assert that the free list is valid:
///
/// 1. All blocks are within `ALIGN..self.capacity`