A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Add GC zeal assertions (#12933)
Nick Fitzgeralddad8432da2ad
diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rsindex d95109dfe6..45f52d6769 100644--- a/crates/cranelift/src/func_environ/gc/enabled.rs+++ b/crates/cranelift/src/func_environ/gc/enabled.rs@@ -94,8 +94,55 @@ fn unbarriered_store_gc_ref(Ok(())}-/// Emit code to read a struct field or array element from its raw address in-/// the GC heap.+/// Emit inline CLIF code that asserts an object's `VMGcKind` matches the+/// expected kind. Only emits code when `cfg(gc_zeal)` is enabled.+///+/// `gc_ref` must be a non-null, non-i31 GC reference (i32 heap index).+fn emit_gc_kind_assert(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ gc_ref: ir::Value,+ expected_kind: VMGcKind,+) {+ if !cfg!(gc_zeal) {+ return;+ }++ func_env.trapz(builder, gc_ref, crate::TRAP_NULL_REFERENCE);++ let kind_addr = func_env.prepare_gc_ref_access(+ builder,+ gc_ref,+ BoundsCheck::StaticObjectField {+ offset: wasmtime_environ::VM_GC_HEADER_KIND_OFFSET,+ access_size: wasmtime_environ::VM_GC_KIND_SIZE,+ object_size: wasmtime_environ::VM_GC_HEADER_SIZE,+ },+ );+ let kind_and_reserved_bits = builder.ins().load(+ ir::types::I32,+ ir::MemFlags::trusted().with_readonly(),+ kind_addr,+ 0,+ );+ let kind_mask = builder+ .ins()+ .iconst(ir::types::I32, i64::from(VMGcKind::MASK));+ let actual_kind = builder.ins().band(kind_and_reserved_bits, kind_mask);++ let expected_kind = builder+ .ins()+ .iconst(ir::types::I32, i64::from(expected_kind.as_u32()));++ // NB: Do a subtype check rather than a strict equality check. See+ // `VMGcKind::matches` for details.+ let and = builder.ins().band(actual_kind, expected_kind);+ let matches = builder.ins().icmp(IntCC::Equal, and, expected_kind);++ builder.ins().trapz(matches, TRAP_INTERNAL_ASSERT);+}++/// Read a struct field or array element from its raw address in the GC heap.////// The given address MUST have already been bounds-checked via/// `prepare_gc_ref_access`.@@ -331,6 +378,8 @@ pub fn translate_struct_get(// type info from `wasmparser` and through to here is a bit funky.func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);+ emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);+let field_index = usize::try_from(field_index).unwrap();let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();@@ -378,6 +427,8 @@ pub fn translate_struct_set(// TODO: See comment in `translate_struct_get` about the `trapz`.func_env.trapz(builder, struct_ref, crate::TRAP_NULL_REFERENCE);+ emit_gc_kind_assert(func_env, builder, struct_ref, VMGcKind::StructRef);+let field_index = usize::try_from(field_index).unwrap();let interned_type_index = func_env.module.types[struct_type_index].unwrap_module_type_index();@@ -979,6 +1030,8 @@ pub fn translate_array_get() -> WasmResult<ir::Value> {log::trace!("translate_array_get({array_type_index:?}, {array_ref:?}, {index:?})");+ emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);+let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index);@@ -1000,6 +1053,8 @@ pub fn translate_array_set() -> WasmResult<()> {log::trace!("translate_array_set({array_type_index:?}, {array_ref:?}, {index:?}, {value:?})");+ emit_gc_kind_assert(func_env, builder, array_ref, VMGcKind::ArrayRef);+let array_type_index = func_env.module.types[array_type_index].unwrap_module_type_index();let elem_addr = array_elem_addr(func_env, builder, array_type_index, array_ref, index);diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rsindex 492f31f953..5de8ba4a24 100644--- a/crates/environ/src/gc.rs+++ b/crates/environ/src/gc.rs@@ -499,15 +499,8 @@ impl VMGcKind {#[inline]pub fn from_high_bits_of_u32(val: u32) -> VMGcKind {let masked = val & Self::MASK;- let result = match masked {- x if x == Self::ExternRef.as_u32() => Self::ExternRef,- x if x == Self::AnyRef.as_u32() => Self::AnyRef,- x if x == Self::EqRef.as_u32() => Self::EqRef,- x if x == Self::ArrayRef.as_u32() => Self::ArrayRef,- x if x == Self::StructRef.as_u32() => Self::StructRef,- x if x == Self::ExnRef.as_u32() => Self::ExnRef,- _ => panic!("invalid `VMGcKind`: {masked:#032b}"),- };+ let result = Self::try_from_u32(masked)+ .unwrap_or_else(|| panic!("invalid `VMGcKind`: {masked:#032b}"));let poison_kind = u32::from_le_bytes([POISON, POISON, POISON, POISON]) & VMGcKind::MASK;debug_assert_ne!(@@ -531,6 +524,22 @@ impl VMGcKind {pub fn as_u32(self) -> u32 {self as u32}++ /// Try to convert a `u32` into a `VMGcKind`.+ ///+ /// Returns `None` if the value doesn't match any known kind.+ #[inline]+ pub fn try_from_u32(x: u32) -> Option<VMGcKind> {+ match x {+ _ if x == Self::ExternRef.as_u32() => Some(Self::ExternRef),+ _ if x == Self::AnyRef.as_u32() => Some(Self::AnyRef),+ _ if x == Self::EqRef.as_u32() => Some(Self::EqRef),+ _ if x == Self::ArrayRef.as_u32() => Some(Self::ArrayRef),+ _ if x == Self::StructRef.as_u32() => Some(Self::StructRef),+ _ if x == Self::ExnRef.as_u32() => Some(Self::ExnRef),+ _ => None,+ }+ }}#[cfg(test)]diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rsindex 10e1805d08..4656866578 100644--- a/crates/wasmtime/src/runtime/store/gc.rs+++ b/crates/wasmtime/src/runtime/store/gc.rs@@ -56,11 +56,16 @@ impl StoreOpaque {bytes_needed: Option<u64>,asyncness: Asyncness,) {- if let Some(n) = bytes_needed {+ if let Some(n) = bytes_needed+ // The gc_zeal's allocation counter will pass `bytes_needed == 0` to+ // signify that we shouldn't grow the GC heap, just do a collection.+ && n > 0+ {if self.grow_gc_heap(limiter, n).await.is_ok() {return;}}+self.do_gc(asyncness).await;}diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rsindex 6067b565d5..b33365a43c 100644--- a/crates/wasmtime/src/runtime/vm/gc.rs+++ b/crates/wasmtime/src/runtime/vm/gc.rs@@ -48,6 +48,18 @@ pub struct GcStore {/// The function-references table for this GC heap.pub func_ref_table: FuncRefTable,++ /// An allocation counter that triggers GC when it reaches zero.+ ///+ /// Initialized from the `WASMTIME_GC_ZEAL_ALLOC_COUNTER` environment+ /// variable. Decremented on every allocation and when it hits zero, a GC is+ /// forced and the counter is reset.+ #[cfg(all(gc_zeal, feature = "std"))]+ gc_zeal_alloc_counter: Option<NonZeroU32>,++ /// The initial value to reset the counter to after it triggers.+ #[cfg(all(gc_zeal, feature = "std"))]+ gc_zeal_alloc_counter_init: Option<NonZeroU32>,}impl GcStore {@@ -55,11 +67,29 @@ impl GcStore {pub fn new(allocation_index: GcHeapAllocationIndex, gc_heap: Box<dyn GcHeap>) -> Self {let host_data_table = ExternRefHostDataTable::default();let func_ref_table = FuncRefTable::default();++ #[cfg(all(gc_zeal, feature = "std"))]+ let gc_zeal_alloc_counter_init =+ std::env::var("WASMTIME_GC_ZEAL_ALLOC_COUNTER")+ .ok()+ .map(|v| {+ v.parse::<NonZeroU32>().unwrap_or_else(|_| {+ panic!(+ "`WASMTIME_GC_ZEAL_ALLOC_COUNTER` must be a non-zero \+ `u32` value, got: {v}"+ )+ })+ });+Self {allocation_index,gc_heap,host_data_table,func_ref_table,+ #[cfg(all(gc_zeal, feature = "std"))]+ gc_zeal_alloc_counter: gc_zeal_alloc_counter_init,+ #[cfg(all(gc_zeal, feature = "std"))]+ gc_zeal_alloc_counter_init,}}@@ -231,6 +261,20 @@ impl GcStore {header: VMGcHeader,layout: Layout,) -> Result<Result<VMGcRef, u64>> {+ // When gc_zeal is enabled with an allocation counter, decrement it and+ // force a GC cycle when it reaches zero by returning a fake OOM.+ #[cfg(all(gc_zeal, feature = "std"))]+ if let Some(counter) = self.gc_zeal_alloc_counter.take() {+ match NonZeroU32::new(counter.get() - 1) {+ Some(c) => self.gc_zeal_alloc_counter = Some(c),+ None => {+ log::trace!("gc_zeal: allocation counter reached zero, forcing GC");+ self.gc_zeal_alloc_counter = self.gc_zeal_alloc_counter_init;+ return Ok(Err(0));+ }+ }+ }+self.gc_heap.alloc_raw(header, layout)}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rsindex 94bc99abb9..774f4c4c3f 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs@@ -64,7 +64,8 @@ use core::{};use wasmtime_environ::drc::{ARRAY_LENGTH_OFFSET, DrcTypeLayouts};use wasmtime_environ::{- GcArrayLayout, GcLayout, GcStructLayout, GcTypeLayouts, VMGcKind, VMSharedTypeIndex,+ GcArrayLayout, GcLayout, GcStructLayout, GcTypeLayouts, POISON, VMGcKind, VMSharedTypeIndex,+ gc_assert,};#[expect(clippy::cast_possible_truncation, reason = "known to not overflow")]@@ -177,10 +178,15 @@ impl DrcHeap {let drc_ref = drc_ref(&gc_ref);let size = self.index(drc_ref).object_size();let layout = FreeList::layout(size);- self.free_list- .as_mut()- .unwrap()- .dealloc(gc_ref.as_heap_index().unwrap(), layout);+ let index = gc_ref.as_heap_index().unwrap();++ // Poison the freed memory so that any stale access is detectable.+ if cfg!(gc_zeal) {+ let index = usize::try_from(index.get()).unwrap();+ self.heap_slice_mut()[index..][..layout.size()].fill(POISON);+ }++ self.free_list.as_mut().unwrap().dealloc(index, layout);}/// Increment the ref count for the associated object.@@ -311,6 +317,7 @@ impl DrcHeap {debug_assert!(header.kind().matches(VMGcKind::ExternRef));return;};+match self.trace_infos.get(&ty)@@ -321,11 +328,24 @@ impl DrcHeap {let data = self.gc_object_data(gc_ref);for offset in gc_ref_offsets {let raw = data.read_u32(*offset);- if let Some(gc_ref) = VMGcRef::from_raw_u32(raw) {+ if let Some(gc_ref) = VMGcRef::from_raw_u32(raw)+ && !gc_ref.is_i31()+ {+ debug_assert!(+ {+ let header = self.header(&gc_ref);+ let kind = header.kind().as_u32();+ VMGcKind::try_from_u32(kind).is_some()+ },+ "trace_gc_ref: struct field at offset {offset} references object \+ with invalid `VMGcKind`",+ );+stack.push(gc_ref);}}}+TraceInfo::Array { gc_ref_elems } => {if !*gc_ref_elems {return;@@ -338,7 +358,19 @@ impl DrcHeap {let elem_offset = GC_REF_ARRAY_ELEMS_OFFSET+ i * u32::try_from(mem::size_of::<u32>()).unwrap();let raw = data.read_u32(elem_offset);- if let Some(gc_ref) = VMGcRef::from_raw_u32(raw) {+ if let Some(gc_ref) = VMGcRef::from_raw_u32(raw)+ && !gc_ref.is_i31()+ {+ debug_assert!(+ {+ let header = self.header(&gc_ref);+ let kind = header.kind().as_u32();+ VMGcKind::try_from_u32(kind).is_some()+ },+ "trace_gc_ref: array element at index {i} references object \+ with invalid `VMGcKind`",+ );+stack.push(gc_ref);}}@@ -359,6 +391,64 @@ impl DrcHeap {})}+ /// Assert the integrity of the over-approximated stack roots list.+ fn assert_over_approximated_stack_roots_integrity(&self) {+ if !cfg!(gc_zeal) {+ return;+ }++ let mut visited = HashSet::new();+ for gc_ref in self.iter_over_approximated_stack_roots() {+ let idx = gc_ref.as_heap_index().unwrap().get();++ // Each entry must have a valid `VMGcKind`.+ let header = self.header(&gc_ref);+ let kind = header.kind().as_u32();+ assert!(+ VMGcKind::try_from_u32(kind).is_some(),+ "over-approx list: entry at heap index {idx} has invalid VMGcKind {kind:#034b}",+ );++ // Each entry must have its in-list bit set.+ let drc_header = self.index(drc_ref(&gc_ref));+ assert!(+ drc_header.is_in_over_approximated_stack_roots(),+ "over-approx list: entry at heap index {idx} does not have in-list bit set",+ );++ // Each entry must have a nonzero ref count.+ assert_ne!(+ drc_header.ref_count, 0,+ "over-approx list: entry at heap index {idx} has zero ref count",+ );++ // No cycles or duplicates.+ assert!(+ visited.insert(idx),+ "over-approx list: cycle or duplicate detected at heap index {idx}",+ );+ }+ }++ /// Assert that every free block in the free list is filled with the poison+ /// pattern.+ fn assert_free_blocks_are_poisoned(&self) {+ if !cfg!(gc_zeal) {+ return;+ }++ let free_list = self.free_list.as_ref().unwrap();+ for (index, len) in free_list.iter_free_blocks() {+ let start = usize::try_from(index).unwrap();+ let size = usize::try_from(len).unwrap();+ let slice = &self.heap_slice()[start..][..size];+ assert!(+ slice.iter().all(|&b| b == POISON),+ "free block at heap index {start} (size {size}) is not fully poisoned",+ );+ }+ }+fn trace(&mut self, roots: &mut GcRootsIter<'_>) {// The `over_approx_set` is used for `debug_assert!`s checking that// every reference we read out from the stack via stack maps is actually@@ -685,6 +775,12 @@ unsafe impl GcHeap for DrcHeap {self.free_list = Some(FreeList::new(len));self.vmmemory = Some(memory.vmmemory());self.memory = Some(memory);++ // Poison the entire heap so any access to uninitialized memory is+ // detectable.+ if cfg!(gc_zeal) {+ self.heap_slice_mut().fill(POISON);+ }}fn detach(&mut self) -> crate::vm::Memory {@@ -795,11 +891,27 @@ unsafe impl GcHeap for DrcHeap {}fn header(&self, gc_ref: &VMGcRef) -> &VMGcHeader {- self.index(gc_ref.as_typed_unchecked())+ let header: &VMGcHeader = self.index(gc_ref.as_typed_unchecked());++ debug_assert!(+ VMGcKind::try_from_u32(header.kind().as_u32()).is_some(),+ "header: invalid VMGcKind {:#010x} at gc_ref {gc_ref:#p}",+ header.kind().as_u32(),+ );++ header}fn header_mut(&mut self, gc_ref: &VMGcRef) -> &mut VMGcHeader {- self.index_mut(gc_ref.as_typed_unchecked())+ let header: &mut VMGcHeader = self.index_mut(gc_ref.as_typed_unchecked());++ debug_assert!(+ VMGcKind::try_from_u32(header.kind().as_u32()).is_some(),+ "header_mut: invalid VMGcKind {:#010x} at gc_ref {gc_ref:#p}",+ header.kind().as_u32(),+ );++ header}fn object_size(&self, gc_ref: &VMGcRef) -> usize {@@ -828,6 +940,19 @@ unsafe impl GcHeap for DrcHeap {Some(index) => VMGcRef::from_heap_index(index).unwrap(),};+ // Assert that the newly-allocated memory is still filled with the+ // poison pattern, and hasn't been corrupted since deallocation (or+ // initial heap creation).+ if cfg!(gc_zeal) {+ let start = usize::try_from(gc_ref.as_heap_index().unwrap().get()).unwrap();+ let slice = &self.heap_slice()[start..][..layout.size()];+ gc_assert!(+ slice.iter().all(|&b| b == POISON),+ "newly allocated GC object at index {start} is not fully poisoned; \+ freed memory was corrupted",+ );+ }+*self.index_mut(drc_ref(&gc_ref)) = VMDrcHeader {header,ref_count: 1,@@ -922,6 +1047,16 @@ unsafe impl GcHeap for DrcHeap {self.vmmemory = Some(memory.vmmemory());self.memory = Some(memory);+ // Poison the newly-grown region so stale accesses are detectable.+ if cfg!(gc_zeal) {+ let old_cap = self.free_list.as_ref().unwrap().current_capacity();+ let new_bytes = usize::try_from(delta_bytes_grown).unwrap();+ let slice = self.heap_slice_mut();+ if old_cap + new_bytes <= slice.len() {+ slice[old_cap..old_cap + new_bytes].fill(POISON);+ }+ }+self.free_list.as_mut().unwrap()@@ -958,14 +1093,30 @@ impl<'a> GarbageCollection<'a> for DrcCollection<'a> {match self.phase {DrcCollectionPhase::Trace => {log::trace!("Begin DRC trace");++ self.heap.assert_over_approximated_stack_roots_integrity();+ self.heap.assert_free_blocks_are_poisoned();+self.heap.trace(&mut self.roots);++ self.heap.assert_over_approximated_stack_roots_integrity();+ self.heap.assert_free_blocks_are_poisoned();+log::trace!("End DRC trace");self.phase = DrcCollectionPhase::Sweep;GcProgress::Continue}DrcCollectionPhase::Sweep => {log::trace!("Begin DRC sweep");++ self.heap.assert_over_approximated_stack_roots_integrity();+ self.heap.assert_free_blocks_are_poisoned();+self.heap.sweep(self.host_data_table);++ self.heap.assert_over_approximated_stack_roots_integrity();+ self.heap.assert_free_blocks_are_poisoned();+log::trace!("End DRC sweep");self.phase = DrcCollectionPhase::Done;GcProgress::Completediff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rsindex 7d0fb7c877..e638e5ea90 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs@@ -40,6 +40,11 @@ impl FreeList {Layout::from_size_align(size, ALIGN_USIZE).unwrap()}+ /// Get the current total capacity this free list manages.+ pub fn current_capacity(&self) -> usize {+ self.capacity+ }+/// Create a new `FreeList` for a contiguous region of memory of the given/// size.pub fn new(capacity: usize) -> Self {@@ -340,6 +345,11 @@ impl FreeList {self.check_integrity();}+ /// Iterate over all free blocks as `(index, len)` pairs.+ pub fn iter_free_blocks(&self) -> impl Iterator<Item = (u32, u32)> + '_ {+ self.free_block_index_to_len.iter().map(|(&i, &l)| (i, l))+ }+/// Assert that the free list is valid:////// 1. All blocks are within `ALIGN..self.capacity`