A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Remove `ensure_trace_info` methods (#13843)
Alex Crichton2fb59986ce75
diff --git a/.github/actions/apt-get-install/action.yml b/.github/actions/apt-get-install/action.ymlindex 1659482de6..1d5d3e6a68 100644--- a/.github/actions/apt-get-install/action.yml+++ b/.github/actions/apt-get-install/action.yml@@ -33,8 +33,18 @@ runs:- name: Update apt-getshell: bash- run: sudo apt-get update+ run: |+ for attempt in 1 2 3 4 5; do+ [ "$attempt" = "5" ] && exit 1+ sudo apt-get update && break+ sleep 5+ done- name: Install packagesshell: bash- run: sudo apt-get install -y ${{ inputs.packages }}+ run: |+ for attempt in 1 2 3 4 5; do+ [ "$attempt" = "5" ] && exit 1+ sudo apt-get install -y ${{ inputs.packages }} && break+ sleep 5+ donediff --git a/crates/wasmtime/src/runtime/coredump.rs b/crates/wasmtime/src/runtime/coredump.rsindex c8c8fe366e..031fb7c0e2 100644--- a/crates/wasmtime/src/runtime/coredump.rs+++ b/crates/wasmtime/src/runtime/coredump.rs@@ -39,7 +39,11 @@ pub struct WasmCoreDump {impl WasmCoreDump {pub(crate) fn new(store: &mut StoreOpaque, backtrace: WasmBacktrace) -> WasmCoreDump {- let modules: Vec<_> = store.modules().all_modules().cloned().collect();+ let modules = store+ .modules()+ .all_modules()+ .map(|(_, m)| m.clone())+ .collect::<Vec<_>>();let instances: Vec<Instance> = store.all_instances().collect();let store_memories: Vec<Memory> =store.all_memories().filter_map(|m| m.unshared()).collect();diff --git a/crates/wasmtime/src/runtime/debug.rs b/crates/wasmtime/src/runtime/debug.rsindex 36a886664b..c9d27abe76 100644--- a/crates/wasmtime/src/runtime/debug.rs+++ b/crates/wasmtime/src/runtime/debug.rs@@ -170,7 +170,10 @@ impl StoreOpaque {return vec![];}- self.modules().all_modules().cloned().collect()+ self.modules()+ .all_modules()+ .map(|(_, m)| m.clone())+ .collect()}}@@ -1182,7 +1185,11 @@ impl<'a> BreakpointEdit<'a> {// re-publishing code.return Ok(());}- let modules = self.registry.all_modules().cloned().collect::<Vec<_>>();+ let modules = self+ .registry+ .all_modules()+ .map(|(_, m)| m.clone())+ .collect::<Vec<_>>();for module in modules {let mem =Self::get_code_memory(self.state, self.registry, &mut self.dirty_modules, &module)?;diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rsindex 22914fbdda..0f0a60a67e 100644--- a/crates/wasmtime/src/runtime/func.rs+++ b/crates/wasmtime/src/runtime/func.rs@@ -2394,7 +2394,7 @@ impl HostFunc {};let (gc_lifo_scope, ret) = {- let gc_lifo_scope = store.0.gc_roots().enter_lifo_scope();+ let gc_lifo_scope = store.0.enter_gc_lifo_scope();let mut args = NonNull::slice_from_raw_parts(args.cast(), args_len);// SAFETY: it's a contract of this function itself that the valuesdiff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rsindex eb2f392270..706de77994 100644--- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs+++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs@@ -1,4 +1,4 @@-use crate::runtime::vm::{GcStore, VMGcRef};+use crate::runtime::vm::VMGcRef;use crate::{AsContext, AsContextMut, GcRef, Result, RootedGcRef,store::{AutoAssertNoGc, StoreOpaque},@@ -34,17 +34,6 @@ mod sealed {}pub(crate) use sealed::*;-#[derive(Debug, Default)]-pub(crate) struct RootSet {}--impl RootSet {- pub(crate) fn enter_lifo_scope(&self) -> usize {- usize::MAX- }-- pub(crate) fn exit_lifo_scope(&mut self, _gc_store: Option<&mut GcStore>, _scope: usize) {}-}-/// This type is disabled because the `gc` cargo feature was not enabled at/// compile time.pub struct Rooted<T: GcRef> {diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rsindex 1b4a539710..47856b5cc7 100644--- a/crates/wasmtime/src/runtime/instance.rs+++ b/crates/wasmtime/src/runtime/instance.rs@@ -311,13 +311,6 @@ impl Instance {// Allocate the GC heap, if necessary.if module.env_module().needs_gc_heap {store.ensure_gc_store(limiter.as_deref_mut()).await?;-- // Eagerly register trace info for all types in this module.- if let Some(gc_store) = store.optional_gc_store_mut() {- for (_, ty) in module.signatures().as_module_map().iter() {- gc_store.ensure_trace_info(*ty);- }- }}// Register the module just before instantiation to ensure we keep the modulediff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rsindex 3952255804..19b0f1f21c 100644--- a/crates/wasmtime/src/runtime/module.rs+++ b/crates/wasmtime/src/runtime/module.rs@@ -657,6 +657,11 @@ impl Module {self.inner.code.module_types()}+ #[cfg(any(+ feature = "gc-drc",+ feature = "gc-copying",+ feature = "component-model"+ ))]pub(crate) fn signatures(&self) -> &crate::type_registry::TypeCollection {self.inner.code.signatures()}diff --git a/crates/wasmtime/src/runtime/module/registry.rs b/crates/wasmtime/src/runtime/module/registry.rsindex cd05f1b44b..97c2e43e23 100644--- a/crates/wasmtime/src/runtime/module/registry.rs+++ b/crates/wasmtime/src/runtime/module/registry.rs@@ -210,9 +210,9 @@ impl ModuleRegistry {}/// Gets an iterator over all modules in the registry.- #[cfg(any(feature = "coredump", feature = "debug"))]- pub fn all_modules(&self) -> impl Iterator<Item = &'_ Module> + '_ {- self.modules.values()+ #[cfg(any(feature = "coredump", feature = "debug", feature = "gc"))]+ pub fn all_modules(&self) -> impl Iterator<Item = (RegisteredModuleId, &'_ Module)> + '_ {+ self.modules.iter()}/// Registers a new module with the registry.diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 78c46017b6..fae724cbba 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -76,14 +76,11 @@//! contents of `StoreOpaque`. This is an invariant that we, as the authors of//! `wasmtime`, must uphold for the public interface to be safe.-use crate::RootSet;use crate::error::OutOfMemory;#[cfg(feature = "async")]use crate::fiber;use crate::module::{RegisterBreakpointState, RegisteredModuleId};use crate::prelude::*;-#[cfg(feature = "gc")]-use crate::runtime::vm::GcRootsList;#[cfg(feature = "stack-switching")]use crate::runtime::vm::VMContRef;use crate::runtime::vm::mpk::ProtectionKey;@@ -129,6 +126,8 @@ pub use self::async_::CallHookHandler;#[cfg(feature = "gc")]mod gc;+#[cfg(not(feature = "gc"))]+mod gc_disabled;/// A [`Store`] is a collection of WebAssembly instances and host-defined state.///@@ -477,27 +476,8 @@ pub struct StoreOpaque {host_globals: TryPrimaryMap<DefinedGlobalIndex, StoreBox<VMHostGlobalContext>>,// GC-related fields.gc_store: Option<GcStore>,- gc_roots: RootSet,- #[cfg(feature = "gc")]- gc_roots_list: GcRootsList,- // Types for which the embedder has created an allocator for.#[cfg(feature = "gc")]- gc_host_alloc_types: crate::hash_set::HashSet<crate::type_registry::RegisteredType>,- /// Pending exception, if any. This is also a GC root, because it- /// needs to be rooted somewhere between the time that a pending- /// exception is set and the time that the handling code takes the- /// exception object. We use this rooting strategy rather than a- /// root in an `Err` branch of a `Result` on the host side because- /// it is less error-prone with respect to rooting behavior. See- /// `throw()`, `take_pending_exception()`,- /// `peek_pending_exception()`, `has_pending_exception()`, and- /// `catch()`.- ///- /// Also note that the underlying reference here is a `VMExnRef`, a- /// refinement of `VMGcRef`, but rooting APIs right now make it difficult to- /// work with that directly so this is stored as `VMGcRef` instead.- #[cfg(feature = "gc")]- pending_exception: Option<VMGcRef>,+ gc_data: gc::StoreGcData,// Numbers of resources instantiated in this store, and their limitsinstance_count: usize,@@ -753,13 +733,8 @@ impl<T> Store<T> {instances: TryPrimaryMap::new(),signal_handler: None,gc_store: None,- gc_roots: RootSet::default(),- #[cfg(feature = "gc")]- gc_roots_list: GcRootsList::default(),#[cfg(feature = "gc")]- gc_host_alloc_types: Default::default(),- #[cfg(feature = "gc")]- pending_exception: None,+ gc_data: Default::default(),modules: ModuleRegistry::default(),func_refs: FuncRefs::default(),host_globals: TryPrimaryMap::new(),@@ -1797,117 +1772,6 @@ impl StoreOpaque {&mut self.vm_store_context}- /// Performs a lazy allocation of the `GcStore` within this store, returning- /// the previous allocation if it's already present.- ///- /// This method will, if necessary, allocate a new `GcStore` -- linear- /// memory and all. This is a blocking operation due to- /// `ResourceLimiterAsync` which means that this should only be executed- /// in a fiber context at this time.- #[inline]- pub(crate) async fn ensure_gc_store(- &mut self,- limiter: Option<&mut StoreResourceLimiter<'_>>,- ) -> Result<&mut GcStore> {- if self.gc_store.is_some() {- return Ok(self.gc_store.as_mut().unwrap());- }- self.allocate_gc_store(limiter).await- }-- #[inline(never)]- async fn allocate_gc_store(- &mut self,- limiter: Option<&mut StoreResourceLimiter<'_>>,- ) -> Result<&mut GcStore> {- log::trace!("allocating GC heap for store {:?}", self.id());-- assert!(self.gc_store.is_none());- assert_eq!(- self.vm_store_context.gc_heap.get_mut().base.as_non_null(),- NonNull::dangling(),- );- assert_eq!(self.vm_store_context.gc_heap.get_mut().current_length(), 0);-- let gc_store = allocate_gc_store(self, limiter).await?;- *self.vm_store_context.gc_heap.get_mut() = gc_store.vmmemory_definition();- return Ok(self.gc_store.insert(gc_store));-- #[cfg(feature = "gc")]- async fn allocate_gc_store(- store: &mut StoreOpaque,- limiter: Option<&mut StoreResourceLimiter<'_>>,- ) -> Result<GcStore> {- use wasmtime_environ::packed_option::ReservedValue;-- let engine = store.engine();- let mem_ty = engine.tunables().gc_heap_memory_type();-- ensure!(- engine.features().gc_types(),- "cannot allocate a GC store when GC is disabled at configuration time"- );- let gc_runtime = engine- .gc_runtime()- .context("no GC runtime: GC disabled at compile time or configuration time")?;-- // First, allocate the memory that will be our GC heap's storage.- let mut request = InstanceAllocationRequest {- id: InstanceId::reserved_value(),- runtime_info: engine.empty_module_runtime_info(),- imports: vm::Imports::default(),- store,- limiter,- };-- let (mem_alloc_index, mem) = engine- .allocator()- .allocate_memory(- &mut request,- &mem_ty,- None,- wasmtime_environ::MemoryKind::GcHeap,- )- .await?;-- // Then, allocate the actual GC heap, passing in that memory- // storage.- let (index, mut heap) =- match engine- .allocator()- .allocate_gc_heap(engine, &**gc_runtime, mem_alloc_index)- {- Ok(pair) => pair,- Err(e) => unsafe {- engine- .allocator()- .deallocate_memory(None, mem_alloc_index, mem);- return Err(e);- },- };- heap.attach(mem);-- let mut gc_store = GcStore::new(index, heap, engine.tunables().gc_zeal_alloc_counter);-- // Eagerly register trace info for any host-created types (via- // StructRefPre/ArrayRefPre) that were created before this GC- // store was allocated.- for ty in &store.gc_host_alloc_types {- gc_store.ensure_trace_info(ty.index());- }-- Ok(gc_store)- }-- #[cfg(not(feature = "gc"))]- async fn allocate_gc_store(- _: &mut StoreOpaque,- _: Option<&mut StoreResourceLimiter<'_>>,- ) -> Result<GcStore> {- bail!("cannot allocate a GC store: the `gc` feature was disabled at compile time")- }- }-/// Attempts to access the GC store that has been previously allocated.////// This method will return `Some` if the GC store was previously allocated.@@ -1957,22 +1821,6 @@ impl StoreOpaque {self.gc_store.as_mut()}- #[inline]- pub(crate) fn gc_roots(&self) -> &RootSet {- &self.gc_roots- }-- #[inline]- #[cfg(feature = "gc")]- pub(crate) fn gc_roots_mut(&mut self) -> &mut RootSet {- &mut self.gc_roots- }-- #[inline]- pub(crate) fn exit_gc_lifo_scope(&mut self, scope: usize) {- self.gc_roots.exit_lifo_scope(self.gc_store.as_mut(), scope);- }-/// Helper function execute a `init_gc_ref` when placing `gc_ref` in `dest`.////// This avoids allocating `GcStore` where possible.@@ -2361,18 +2209,6 @@ at https://bytecodealliance.org/security.Ok(id)}- /// Tests whether there is a pending exception.- pub fn has_pending_exception(&self) -> bool {- #[cfg(feature = "gc")]- {- self.pending_exception.is_some()- }- #[cfg(not(feature = "gc"))]- {- false- }- }-#[cfg(target_has_atomic = "64")]pub(crate) fn set_epoch_deadline(&mut self, delta: u64) {// Set a new deadline based on the "epoch deadline delta".@@ -2423,11 +2259,6 @@ at https://bytecodealliance.org/security.// otherwise fall back to the runtime-agnostic code.yield_now().await}-- #[cfg(not(feature = "gc"))]- pub(crate) fn require_gc_store_mut(&mut self) -> Result<&mut GcStore> {- bail!("GC is disabled")- }}#[cfg(any(feature = "async", feature = "gc"))]diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rsindex 419face962..67b4964987 100644--- a/crates/wasmtime/src/runtime/store/gc.rs+++ b/crates/wasmtime/src/runtime/store/gc.rs@@ -1,12 +1,16 @@//! GC-related methods for stores.-use crate::error::Context;+use crate::RootSet;+use crate::error::{Context, ensure};use crate::module::ModuleRegistry;use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter, yield_now,};use crate::type_registry::RegisteredType;-use crate::vm::{self, Backtrace, Frame, GcRootsList, GcStore, SendSyncPtr, VMGcRef};+use crate::vm::{+ self, Backtrace, Frame, GcRootsList, GcStore, InstanceAllocationRequest, SendSyncPtr,+ StoreGcHostAllocTypes, TraceInfo, VMGcRef,+};use crate::{ExnRef, GcHeapOutOfMemory, Result, Rooted, Store, StoreContextMut, ThrownException, bail,};@@ -16,6 +20,29 @@ use core::num::NonZeroU32;use core::ops::{Deref, DerefMut};use core::ptr::NonNull;use wasmtime_environ::DefinedTagIndex;+use wasmtime_environ::packed_option::ReservedValue;++#[derive(Default)]+pub(crate) struct StoreGcData {+ gc_roots: RootSet,+ gc_roots_list: GcRootsList,+ // Types for which the embedder has created an allocator for.+ gc_host_alloc_types: StoreGcHostAllocTypes,+ /// Pending exception, if any. This is also a GC root, because it+ /// needs to be rooted somewhere between the time that a pending+ /// exception is set and the time that the handling code takes the+ /// exception object. We use this rooting strategy rather than a+ /// root in an `Err` branch of a `Result` on the host side because+ /// it is less error-prone with respect to rooting behavior. See+ /// `throw()`, `take_pending_exception()`,+ /// `peek_pending_exception()`, `has_pending_exception()`, and+ /// `catch()`.+ ///+ /// Also note that the underlying reference here is a `VMExnRef`, a+ /// refinement of `VMGcRef`, but rooting APIs right now make it difficult to+ /// work with that directly so this is stored as `VMGcRef` instead.+ pending_exception: Option<VMGcRef>,+}impl<T> Store<T> {/// Perform garbage collection.@@ -188,7 +215,7 @@ impl StoreOpaque {// borrows.pub(crate) fn trim_gc_liveness_flags(&mut self, eager: bool) {if let Some(gc_store) = self.gc_store.as_mut() {- self.gc_roots.trim_liveness_flags(gc_store, eager);+ self.gc_data.gc_roots.trim_liveness_flags(gc_store, eager);}}@@ -455,7 +482,7 @@ impl StoreOpaque {pub(crate) fn set_pending_exception(&mut self, exnref: &VMGcRef) -> crate::Error {debug_assert!(exnref.is_exnref(&*self.unwrap_gc_store_mut().gc_heap));let gc_store = self.gc_store.as_mut().unwrap();- match gc_store.write_gc_ref(&mut self.pending_exception, Some(exnref)) {+ match gc_store.write_gc_ref(&mut self.gc_data.pending_exception, Some(exnref)) {Ok(()) => ThrownException.into(),Err(e) => e,}@@ -464,7 +491,7 @@ impl StoreOpaque {/// Takes the pending exception from this store, if any, and exposes it to/// WebAssembly, returning the raw representation.pub(crate) fn expose_pending_exception_to_wasm(&mut self) -> Option<NonZeroU32> {- let exnref = self.pending_exception.take()?;+ let exnref = self.gc_data.pending_exception.take()?;let gc_store = self.unwrap_gc_store_mut();debug_assert!(exnref.is_exnref(&*gc_store.gc_heap));Some(gc_store.expose_gc_ref_to_wasm(exnref).unwrap())@@ -473,7 +500,7 @@ impl StoreOpaque {/// Takes the pending exception of the store, yielding ownership of its/// reference to the `Rooted` that's returned.fn take_pending_exception_rooted(&mut self) -> Option<Rooted<ExnRef>> {- let vmexnref = self.pending_exception.take()?;+ let vmexnref = self.gc_data.pending_exception.take()?;debug_assert!(vmexnref.is_exnref(&*self.unwrap_gc_store().gc_heap));let mut nogc = AutoAssertNoGc::new(self);Some(Rooted::new(&mut nogc, vmexnref))@@ -484,7 +511,7 @@ impl StoreOpaque {pub(crate) fn pending_exception_tag_and_instance(&mut self,) -> Option<(InstanceId, DefinedTagIndex)> {- let pending_exnref = self.pending_exception.as_ref()?.unchecked_copy();+ let pending_exnref = self.gc_data.pending_exception.as_ref()?.unchecked_copy();debug_assert!(pending_exnref.is_exnref(&*self.unwrap_gc_store_mut().gc_heap));let mut store = AutoAssertNoGc::new(self);@@ -503,7 +530,7 @@ impl StoreOpaque {pub(crate) fn pending_exception_owned_rooted(&mut self,) -> Result<Option<crate::OwnedRooted<ExnRef>>, crate::OutOfMemory> {- let pending = match &self.pending_exception {+ let pending = match &self.gc_data.pending_exception {Some(r) => r,None => return Ok(None),};@@ -581,13 +608,17 @@ impl StoreOpaque {// Take the GC roots out of `self` so we can borrow it mutably but still// call mutable methods on `self`.- let mut roots = core::mem::take(&mut self.gc_roots_list);+ let mut roots = core::mem::take(&mut self.gc_data.gc_roots_list);self.trace_roots(&mut roots, asyncness).await;- self.unwrap_gc_store_mut()+ self.gc_store+ .as_mut()+ .unwrap().gc(asyncness,unsafe { roots.iter() },+ &self.modules,+ &self.gc_data.gc_host_alloc_types,// TODO: Once `Config` has an optional `AsyncFn` field for// yielding to the current async runtime// (e.g. `tokio::task::yield_now`), use that if set; otherwise@@ -598,7 +629,7 @@ impl StoreOpaque {// Restore the GC roots for the next GC.roots.clear();- self.gc_roots_list = roots;+ self.gc_data.gc_roots_list = roots;if log::log_enabled!(log::Level::Trace) {let gc_store = self.gc_store.as_ref().unwrap();@@ -803,13 +834,13 @@ impl StoreOpaque {fn trace_user_roots(&mut self, gc_roots_list: &mut GcRootsList) {log::trace!("Begin trace GC roots :: user");- self.gc_roots.trace_roots(gc_roots_list);+ self.gc_data.gc_roots.trace_roots(gc_roots_list);log::trace!("End trace GC roots :: user");}fn trace_pending_exception_roots(&mut self, gc_roots_list: &mut GcRootsList) {log::trace!("Begin trace GC roots :: pending exception");- if let Some(pending_exception) = self.pending_exception.as_mut() {+ if let Some(pending_exception) = self.gc_data.pending_exception.as_mut() {unsafe {gc_roots_list.add_vmgcref_root(pending_exception.into(), "Pending exception");}@@ -824,13 +855,120 @@ impl StoreOpaque {/// reclaimed (since it is possible that none of the Wasm modules in this/// store are holding it alive).pub(crate) fn insert_gc_host_alloc_type(&mut self, ty: RegisteredType) {- // If a GC heap is already allocated, eagerly register trace info- // now. Otherwise, trace info will be registered when the GC heap- // is allocated in `StoreOpaque::allocate_gc_store`.- if let Some(gc_store) = self.optional_gc_store_mut() {- gc_store.ensure_trace_info(ty.index());+ let trace_info = ty.layout().map(TraceInfo::new);+ self.gc_data+ .gc_host_alloc_types+ .insert(ty.index(), (ty, trace_info));+ }++ /// Performs a lazy allocation of the `GcStore` within this store, returning+ /// the previous allocation if it's already present.+ ///+ /// This method will, if necessary, allocate a new `GcStore` -- linear+ /// memory and all. This is a blocking operation due to+ /// `ResourceLimiterAsync` which means that this should only be executed+ /// in a fiber context at this time.+ #[inline]+ pub(crate) async fn ensure_gc_store(+ &mut self,+ limiter: Option<&mut StoreResourceLimiter<'_>>,+ ) -> Result<&mut GcStore> {+ if self.gc_store.is_some() {+ return Ok(self.gc_store.as_mut().unwrap());}- self.gc_host_alloc_types.insert(ty);+ self.allocate_gc_store(limiter).await+ }++ #[inline(never)]+ async fn allocate_gc_store(+ &mut self,+ limiter: Option<&mut StoreResourceLimiter<'_>>,+ ) -> Result<&mut GcStore> {+ log::trace!("allocating GC heap for store {:?}", self.id());++ assert!(self.gc_store.is_none());+ assert_eq!(+ self.vm_store_context.gc_heap.get_mut().base.as_non_null(),+ NonNull::dangling(),+ );+ assert_eq!(self.vm_store_context.gc_heap.get_mut().current_length(), 0);++ let engine = self.engine();+ let mem_ty = engine.tunables().gc_heap_memory_type();+ ensure!(+ engine.features().gc_types(),+ "cannot allocate a GC store when GC is disabled at configuration time"+ );+ let gc_runtime = engine+ .gc_runtime()+ .context("no GC runtime: GC disabled at compile time or configuration time")?;++ // First, allocate the memory that will be our GC heap's storage.+ let mut request = InstanceAllocationRequest {+ id: InstanceId::reserved_value(),+ runtime_info: engine.empty_module_runtime_info(),+ imports: vm::Imports::default(),+ store: self,+ limiter,+ };++ let (mem_alloc_index, mem) = engine+ .allocator()+ .allocate_memory(+ &mut request,+ &mem_ty,+ None,+ wasmtime_environ::MemoryKind::GcHeap,+ )+ .await?;++ // Then, allocate the actual GC heap, passing in that memory+ // storage.+ let (index, mut heap) =+ match engine+ .allocator()+ .allocate_gc_heap(engine, &**gc_runtime, mem_alloc_index)+ {+ Ok(pair) => pair,+ Err(e) => unsafe {+ engine+ .allocator()+ .deallocate_memory(None, mem_alloc_index, mem);+ return Err(e);+ },+ };+ heap.attach(mem);++ let gc_store = GcStore::new(index, heap, engine.tunables().gc_zeal_alloc_counter);+ *self.vm_store_context.gc_heap.get_mut() = gc_store.vmmemory_definition();+ Ok(self.gc_store.insert(gc_store))+ }++ /// Tests whether there is a pending exception.+ pub fn has_pending_exception(&self) -> bool {+ self.gc_data.pending_exception.is_some()+ }++ #[inline]+ pub(crate) fn gc_roots(&self) -> &RootSet {+ &self.gc_data.gc_roots+ }++ #[inline]+ pub(crate) fn gc_roots_mut(&mut self) -> &mut RootSet {+ &mut self.gc_data.gc_roots+ }++ #[inline]+ pub(crate) fn enter_gc_lifo_scope(&self) -> usize {+ self.gc_data.gc_roots.enter_lifo_scope()+ }++ #[inline]+ pub(crate) fn exit_gc_lifo_scope(&mut self, scope: usize) {+ self.gc_data+ .gc_roots+ .exit_lifo_scope(self.gc_store.as_mut(), scope);}}diff --git a/crates/wasmtime/src/runtime/store/gc_disabled.rs b/crates/wasmtime/src/runtime/store/gc_disabled.rsnew file mode 100644index 0000000000..a57713d754--- /dev/null+++ b/crates/wasmtime/src/runtime/store/gc_disabled.rs@@ -0,0 +1,29 @@+use crate::store::{StoreOpaque, StoreResourceLimiter};+use crate::vm::GcStore;+use crate::{Result, bail};++impl StoreOpaque {+ #[inline]+ pub(crate) async fn ensure_gc_store(+ &mut self,+ _limiter: Option<&mut StoreResourceLimiter<'_>>,+ ) -> Result<&mut GcStore> {+ bail!("cannot allocate a GC store: the `gc` feature was disabled at compile time")+ }++ pub(crate) fn has_pending_exception(&self) -> bool {+ false+ }++ pub(crate) fn require_gc_store_mut(&mut self) -> Result<&mut GcStore> {+ bail!("GC is disabled")+ }++ #[inline]+ pub(crate) fn enter_gc_lifo_scope(&self) -> usize {+ 0+ }++ #[inline]+ pub(crate) fn exit_gc_lifo_scope(&mut self, _scope: usize) {}+}diff --git a/crates/wasmtime/src/runtime/type_registry.rs b/crates/wasmtime/src/runtime/type_registry.rsindex da4e76cdb2..0d98863617 100644--- a/crates/wasmtime/src/runtime/type_registry.rs+++ b/crates/wasmtime/src/runtime/type_registry.rs@@ -5,9 +5,10 @@use crate::Engine;use crate::error::OutOfMemory;+use crate::hash_map::HashMap;use crate::prelude::*;use crate::sync::RwLock;-use crate::vm::GcRuntime;+use crate::vm::{GcRuntime, TraceInfo};use alloc::sync::Arc;use core::cell::Cell;use core::iter;@@ -91,6 +92,8 @@ pub struct TypeCollection {rec_groups: TryVec<RecGroupEntry>,types: TryPrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,trampolines: TrySecondaryMap<VMSharedTypeIndex, PackedOption<ModuleInternedTypeIndex>>,+ trace_info: TryPrimaryMap<ModuleInternedTypeIndex, Option<TraceInfo>>,+ types_with_trace_info: HashMap<VMSharedTypeIndex, ModuleInternedTypeIndex>,}impl Debug for TypeCollection {@@ -100,11 +103,14 @@ impl Debug for TypeCollection {rec_groups,types,trampolines,+ trace_info,+ types_with_trace_info: _,} = self;f.debug_struct("TypeCollection").field("rec_groups", rec_groups).field("types", types).field("trampolines", trampolines)+ .field("trace_info", trace_info).finish_non_exhaustive()}}@@ -138,7 +144,7 @@ impl Engine {let gc_runtime = engine.gc_runtime().map(|rt| &**rt);// First, register these types in this engine's registry.- let (rec_groups, types) = registry+ let (rec_groups, types, trace_info) = registry.0.write().register_module_types(gc_runtime, module_types)?;@@ -155,6 +161,15 @@ impl Engine {}});+ // Build up the map of vm indices to module indices for when types have+ // trace info associated with them.+ let mut types_with_trace_info = TryHashMap::new();+ for (module_ty, info) in trace_info.iter() {+ if info.is_some() {+ types_with_trace_info.insert(types[module_ty], module_ty)?;+ }+ }+// Then build our map from each function type's engine index to the// module-index of its trampoline. Trampoline functions are queried by// module-index in a compiled module, and doing this engine-to-module@@ -187,6 +202,8 @@ impl Engine {rec_groups,types,trampolines,+ trace_info,+ types_with_trace_info: types_with_trace_info.into(),})}}@@ -223,6 +240,23 @@ impl TypeCollection {log::trace!("TypeCollection::trampoline_type({ty:?}) -> {trampoline_ty:?}");trampoline_ty}++ /// Returns the `TraceInfo`, if any, for the `index` provided.+ #[inline]+ #[cfg(feature = "gc")]+ pub fn trace_info(&self, index: ModuleInternedTypeIndex) -> Option<&TraceInfo> {+ self.trace_info.get(index).and_then(|s| s.as_ref())+ }++ /// If `index` has `TraceInfo` associated with it then the corresponding+ /// `ModuleInternedTypeIndex` for `index` is returned.+ #[inline]+ pub fn shared_type_with_trace_info(+ &self,+ index: VMSharedTypeIndex,+ ) -> Option<ModuleInternedTypeIndex> {+ self.types_with_trace_info.get(&index).copied()+ }}impl Drop for TypeCollection {@@ -696,6 +730,7 @@ impl TypeRegistryInner {(TryVec<RecGroupEntry>,TryPrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,+ TryPrimaryMap<ModuleInternedTypeIndex, Option<TraceInfo>>,),OutOfMemory,> {@@ -707,8 +742,15 @@ impl TypeRegistryInner {// The map from a module type index to an engine type index for these// module types.let mut map = TryPrimaryMap::<ModuleInternedTypeIndex, VMSharedTypeIndex>::new();+ let mut trace_info = TryPrimaryMap::new();- if let Err(e) = self.register_module_types_impl(gc_runtime, types, &mut entries, &mut map) {+ if let Err(e) = self.register_module_types_impl(+ gc_runtime,+ types,+ &mut entries,+ &mut map,+ &mut trace_info,+ ) {for entry in entries {if entry.decref("register_module_types cleanup") {self.unregister_entry(entry);@@ -719,7 +761,7 @@ impl TypeRegistryInner {log::trace!("End registering module types");- Ok((entries, map))+ Ok((entries, map, trace_info))}fn register_module_types_impl(@@ -728,9 +770,11 @@ impl TypeRegistryInner {types: &ModuleTypes,entries: &mut TryVec<RecGroupEntry>,map: &mut TryPrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,+ trace_info: &mut TryPrimaryMap<ModuleInternedTypeIndex, Option<TraceInfo>>,) -> Result<(), OutOfMemory> {entries.reserve(types.rec_groups().len())?;map.reserve(types.wasm_types().len())?;+ trace_info.reserve(types.wasm_types().len())?;for (_rec_group_index, module_group) in types.rec_groups() {let entry = self.register_rec_group(@@ -747,6 +791,14 @@ impl TypeRegistryInner {{let module_ty2 = map.push(*engine_ty).expect("reserved capacity");assert_eq!(module_ty, module_ty2);++ let info = self+ .type_to_gc_layout+ .get(*engine_ty)+ .and_then(|l| l.as_ref())+ .map(TraceInfo::new);+ let module_ty3 = trace_info.push(info).expect("reserved capacity");+ assert_eq!(module_ty, module_ty3);}entries.push(entry).expect("reserved capacity");diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rsindex 47aeeafa41..5b0aae7248 100644--- a/crates/wasmtime/src/runtime/vm/gc.rs+++ b/crates/wasmtime/src/runtime/vm/gc.rs@@ -22,13 +22,16 @@ pub use gc_runtime::*;pub use host_data::*;pub use i31::*;+use crate::hash_map::HashMap;+use crate::module::ModuleRegistry;use crate::prelude::*;use crate::runtime::vm::{GcHeapAllocationIndex, VMMemoryDefinition};use crate::store::Asyncness;+use crate::type_registry::RegisteredType;use core::any::Any;use core::mem::MaybeUninit;use core::{alloc::Layout, num::NonZeroU32};-use wasmtime_environ::{GcArrayLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};+use wasmtime_environ::{GcArrayLayout, GcLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};/// GC-related data that is one-to-one with a `wasmtime::Store`.///@@ -68,6 +71,22 @@ pub struct GcStore {gc_zeal_alloc_counter_init: Option<NonZeroU32>,}+/// Convenience type definition for the storage, within a `Store`, of+/// host-allocated types that are one-off used for host allocation.+pub type StoreGcHostAllocTypes = HashMap<VMSharedTypeIndex, (RegisteredType, Option<TraceInfo>)>;++/// Contextual information used when performing a GC to trace GC references as+/// necesary.+pub struct GcStoreTraceState<'a> {+ /// The backing data of `externref`s, deleted from when a GC reference is+ /// reclaimed, for example.+ pub host_data_table: &'a mut ExternRefHostDataTable,+ /// All known modules in this store.+ pub modules: &'a ModuleRegistry,+ /// All known host-registered types in this store.+ pub gc_host_alloc_types: &'a StoreGcHostAllocTypes,+}+impl GcStore {/// Create a new `GcStore`.pub fn new(@@ -108,9 +127,16 @@ impl GcStore {&mut self,asyncness: Asyncness,roots: GcRootsIter<'_>,+ modules: &ModuleRegistry,+ gc_host_alloc_types: &StoreGcHostAllocTypes,yield_fn: impl AsyncFn(),) -> Result<()> {- let collection = self.gc_heap.gc(roots, &mut self.host_data_table);+ let mut trace_state = GcStoreTraceState {+ host_data_table: &mut self.host_data_table,+ modules,+ gc_host_alloc_types,+ };+ let collection = self.gc_heap.gc(roots, &mut trace_state);collect_async(collection, asyncness, yield_fn).await?;self.last_post_gc_allocated_bytes = Some({let size = self.gc_heap.allocated_bytes();@@ -299,11 +325,6 @@ impl GcStore {self.gc_heap.alloc_raw(header, layout)}- /// Eagerly ensure tracing info is registered for the given type.- pub fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {- self.gc_heap.ensure_trace_info(ty)- }-/// Allocate an uninitialized struct with the given type index and layout.////// This does NOT check that the index is currently allocated in the types@@ -394,3 +415,42 @@ impl GcStore {}}}++/// How to trace a GC object.+#[derive(Debug)]+pub enum TraceInfo {+ /// How to trace an array.+ Array {+ /// Whether this array type's elements are GC references, and need+ /// tracing.+ #[cfg_attr(+ not(feature = "gc-drc"),+ allow(dead_code, reason = "easier not to cfg on/off")+ )]+ gc_ref_elems: bool,+ },++ /// How to trace a struct.+ Struct {+ /// The offsets of each GC reference field that needs tracing in+ /// instances of this struct type.+ gc_ref_offsets: Box<[u32]>,+ },+}++impl TraceInfo {+ pub(crate) fn new(gc_layout: &GcLayout) -> Self {+ match gc_layout {+ GcLayout::Array(l) => TraceInfo::Array {+ gc_ref_elems: l.elems_are_gc_refs,+ },+ GcLayout::Struct(l) => TraceInfo::Struct {+ gc_ref_offsets: l+ .fields+ .iter()+ .filter_map(|f| if f.is_gc_ref { Some(f.offset) } else { None })+ .collect(),+ },+ }+ }+}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled.rs b/crates/wasmtime/src/runtime/vm/gc/enabled.rsindex 478cd85fec..fc8bebb58a 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled.rs@@ -7,7 +7,7 @@ mod externref;mod free_list;mod structref;#[cfg(any(feature = "gc-drc", feature = "gc-copying"))]-mod trace_info;+mod trace_infos;pub use arrayref::*;pub use exnref::*;diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rsindex f90d03adcb..83b8e0bb2f 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs@@ -11,11 +11,11 @@//! This collector does not require any read or write barriers.use super::VMArrayRef;-use super::trace_info::{TraceInfo, TraceInfos};+use super::trace_infos::TraceInfos;use crate::runtime::vm::{- ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,- GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMExternRef, VMGcHeader,- VMGcRef, VMMemoryDefinition,+ ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,+ GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMExternRef,+ VMGcHeader, VMGcRef, VMMemoryDefinition,};use crate::{Engine, bail_bug, prelude::*};use core::{@@ -42,8 +42,8 @@ unsafe impl GcRuntime for CopyingCollector {&self.layouts}- fn new_gc_heap(&self, engine: &Engine) -> Result<Box<dyn GcHeap>> {- let heap = CopyingHeap::new(engine)?;+ fn new_gc_heap(&self, _engine: &Engine) -> Result<Box<dyn GcHeap>> {+ let heap = CopyingHeap::new()?;Ok(Box::new(heap) as _)}}@@ -300,10 +300,10 @@ struct CopyingHeap {}impl CopyingHeap {- fn new(engine: &Engine) -> Result<Self> {+ fn new() -> Result<Self> {log::trace!("allocating new copying heap");Ok(Self {- trace_infos: TraceInfos::new(engine, GC_REF_ARRAY_ELEMS_OFFSET),+ trace_infos: TraceInfos::new(),no_gc_count: 0,memory: None,vmmemory: None,@@ -594,7 +594,12 @@ survived collection, since the active space is the same size as the idle space",/// Trace a grey object's outgoing edges, copying their referents into the/// new semi-space if necessary, and updating the object's references with/// their forwarded locations in the new semi-space.- fn scan(&mut self, gc_ref: &VMGcRef, trace_infos: &TraceInfos) -> Result<()> {+ fn scan(+ &mut self,+ gc_ref: &VMGcRef,+ trace_infos: &mut TraceInfos,+ trace_state: &mut GcStoreTraceState<'_>,+ ) -> Result<()> {debug_assert!(!gc_ref.is_i31());let index = gc_ref.heap_index()?.get();debug_assert!(self.is_in_active_space(index));@@ -633,7 +638,7 @@ survived collection, since the active space is the same size as the idle space",let Some(ty) = ty else {bail_bug!("out-of-line trace info but no type index");};- match trace_infos.trace_info(&ty) {+ match trace_infos.trace_info(&ty, trace_state) {TraceInfo::Struct { gc_ref_offsets } => {for &offset in gc_ref_offsets {self.scan_field(object_start, offset)?;@@ -732,10 +737,6 @@ unsafe impl GcHeap for CopyingHeap {memory.take().unwrap()}- fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {- self.trace_infos.ensure(ty);- }-fn as_any(&self) -> &dyn Any {self as _}@@ -849,18 +850,6 @@ unsafe impl GcHeap for CopyingHeap {);debug_assert_eq!(header.reserved_u26() & HEADER_COPIED_BIT, 0);- // We must have trace info for every GC type that we allocate in this- // heap. Trace info is eagerly registered during module instantiation- // and `StructRefPre`/`ArrayRefPre` construction.- if let Some(ty) = header.ty() {- debug_assert!(- self.trace_infos.contains(&ty),- "trace info for {ty:?} should have been eagerly registered",- );- } else {- debug_assert_eq!(header.kind(), VMGcKind::ExternRef);- }-let size = u32::try_from(layout.size()).unwrap();// Round up the allocation size to ALIGN so that the next bump-pointer// allocation is also aligned.@@ -966,15 +955,18 @@ unsafe impl GcHeap for CopyingHeap {usize::try_from(self.bump_ptr() - self.active_space_start).unwrap()}- fn gc<'a>(+ fn gc<'a, 'b>(&'a mut self,roots: GcRootsIter<'a>,- host_data_table: &'a mut ExternRefHostDataTable,- ) -> Box<dyn GarbageCollection<'a> + 'a> {+ trace_state: &'a mut GcStoreTraceState<'b>,+ ) -> Box<dyn GarbageCollection + 'a>+ where+ 'b: 'a,+ {assert_eq!(self.no_gc_count, 0, "Cannot GC inside a no-GC scope!");Box::new(CopyingCollection {roots: Some(roots),- host_data_table,+ trace_state,heap: self,phase: CopyingCollectionPhase::ProcessRoots,})@@ -1036,9 +1028,9 @@ unsafe impl GcHeap for CopyingHeap {const OBJECTS_PER_INCREMENT: usize = 16_384;-struct CopyingCollection<'a> {+struct CopyingCollection<'a, 'b> {roots: Option<GcRootsIter<'a>>,- host_data_table: &'a mut ExternRefHostDataTable,+ trace_state: &'a mut GcStoreTraceState<'b>,heap: &'a mut CopyingHeap,phase: CopyingCollectionPhase,}@@ -1050,7 +1042,7 @@ enum CopyingCollectionPhase {Done,}-impl CopyingCollection<'_> {+impl CopyingCollection<'_, '_> {/// Forward all GC roots from the idle space to the active space.fn process_roots(&mut self) -> Result<()> {log::trace!("Begin processing GC roots");@@ -1098,7 +1090,7 @@ impl CopyingCollection<'_> {/// increment regardless of whether the worklist still has items.fn process_worklist_increment(&mut self) -> Result<GcProgress> {log::trace!("Begin processing worklist increment");- let trace_infos = mem::take(&mut self.heap.trace_infos);+ let mut trace_infos = mem::take(&mut self.heap.trace_infos);let mut count = 0;let has_more = loop {if count >= OBJECTS_PER_INCREMENT {@@ -1107,7 +1099,8 @@ impl CopyingCollection<'_> {match self.heap.worklist_pop()? {Some(gc_ref) => {debug_assert!(self.heap.is_in_active_space(gc_ref.heap_index()?.get()));- self.heap.scan(&gc_ref, &trace_infos)?;+ self.heap+ .scan(&gc_ref, &mut trace_infos, self.trace_state)?;count += 1;}None => break false,@@ -1133,7 +1126,7 @@ impl CopyingCollection<'_> {if !header.copied() {let typed: &TypedGcRef<VMCopyingExternRef> = gc_ref.as_typed_unchecked();let host_data_id = self.heap.index(typed)?.host_data;- self.host_data_table.dealloc(host_data_id)?;+ self.trace_state.host_data_table.dealloc(host_data_id)?;}link = self.heap@@ -1180,7 +1173,7 @@ impl CopyingCollection<'_> {}}-impl GarbageCollection<'_> for CopyingCollection<'_> {+impl GarbageCollection for CopyingCollection<'_, '_> {fn collect_increment(&mut self) -> Result<GcProgress> {match self.phase {CopyingCollectionPhase::ProcessRoots => self.begin_collection(),@@ -1191,7 +1184,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {}}-impl Drop for CopyingCollection<'_> {+impl Drop for CopyingCollection<'_, '_> {fn drop(&mut self) {// A collection is logically atomic with respect to the mutator: the// mutator must never run while the heap is in a half-collecteddiff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rsindex 6bed2e510d..9532e43f05 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs@@ -46,12 +46,12 @@use super::VMArrayRef;use super::free_list::FreeList;-use super::trace_info::{TraceInfo, TraceInfos};+use super::trace_infos::TraceInfos;use crate::hash_set::HashSet;use crate::runtime::vm::{- ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,- GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMExternRef, VMGcHeader,- VMGcObjectData, VMGcRef,+ ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,+ GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMExternRef,+ VMGcHeader, VMGcObjectData, VMGcRef,};use crate::vm::VMMemoryDefinition;use crate::{Engine, Trap, bail_bug, prelude::*};@@ -91,8 +91,8 @@ unsafe impl GcRuntime for DrcCollector {&self.layouts}- fn new_gc_heap(&self, engine: &Engine) -> Result<Box<dyn GcHeap>> {- let heap = DrcHeap::new(engine)?;+ fn new_gc_heap(&self, _engine: &Engine) -> Result<Box<dyn GcHeap>> {+ let heap = DrcHeap::new()?;Ok(Box::new(heap) as _)}}@@ -220,10 +220,10 @@ struct TracingAllocs {impl DrcHeap {/// Construct a new, default DRC heap.- fn new(engine: &Engine) -> Result<Self> {+ fn new() -> Result<Self> {log::trace!("allocating new DRC heap");Ok(Self {- trace_infos: TraceInfos::new(engine, GC_REF_ARRAY_ELEMS_OFFSET),+ trace_infos: TraceInfos::new(),no_gc_count: 0,vmctx_data: Box::default(),memory: None,@@ -311,10 +311,7 @@ impl DrcHeap {/// any that have reached a reference count of 0. Note that this is modeled/// with an explicit stack rather than a recursive function to ensure that/// this cannot overflow the host stack.- fn process_dec_ref_stack(- &mut self,- host_data_table: &mut ExternRefHostDataTable,- ) -> Result<()> {+ fn process_dec_ref_stack(&mut self, trace_state: &mut GcStoreTraceState<'_>) -> Result<()> {let allocs = match self.tracing_allocs.take() {Some(allocs) => allocs,None => bail_bug!("allocs missing during tracing"),@@ -324,6 +321,7 @@ impl DrcHeap {this.tracing_allocs = Some(allocs);});let (this, allocs) = &mut *undo;+ let this: &mut Self = this;let stack = &mut allocs.dec_ref_stack;let large_array_stack = &mut allocs.large_array_dec_ref_stack;let to_dealloc = &mut allocs.to_dealloc;@@ -355,7 +353,7 @@ impl DrcHeap {// Trace: enqueue child GC refs for dec-ref'ing.if let Some(ty) = ty {- match this.trace_infos.trace_info(&ty) {+ match this.trace_infos.trace_info(&ty, trace_state) {TraceInfo::Struct { gc_ref_offsets } => {stack.reserve(gc_ref_offsets.len());let data = this.gc_object_data(&gc_ref)?;@@ -401,12 +399,12 @@ impl DrcHeap {// data, and `ty` is `None` only for `externref`s, so we skip// this for `struct` and `array` objects entirely.debug_assert!(drc_header.header.kind().matches(VMGcKind::ExternRef));- let externref = match gc_ref.as_typed::<VMDrcExternRef>(*this) {+ let externref = match gc_ref.as_typed::<VMDrcExternRef>(this) {Some(r) => r,None => bail_bug!("expected externref"),};let host_data_id = this.index(externref)?.host_data;- host_data_table.dealloc(host_data_id)?;+ trace_state.host_data_table.dealloc(host_data_id)?;}to_dealloc.push(gc_ref);@@ -613,7 +611,7 @@ impl DrcHeap {/// Sweep the bump allocation table after we've discovered our precise stack/// roots.- fn sweep(&mut self, host_data_table: &mut ExternRefHostDataTable) -> Result<()> {+ fn sweep(&mut self, trace_state: &mut GcStoreTraceState<'_>) -> Result<()> {if log::log_enabled!(log::Level::Trace) {Self::log_gc_ref_set("over-approximated-stack-roots set before sweeping",@@ -704,7 +702,7 @@ impl DrcHeap {if let Some(allocs) = &self.tracing_allocs&& !allocs.dec_ref_stack.is_empty(){- self.process_dec_ref_stack(host_data_table)?;+ self.process_dec_ref_stack(trace_state)?;}self.vmctx_data@@ -946,10 +944,6 @@ unsafe impl GcHeap for DrcHeap {memory.take().unwrap()}- fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {- self.trace_infos.ensure(ty);- }-fn as_any(&self) -> &dyn Any {self as _}@@ -1086,21 +1080,6 @@ unsafe impl GcHeap for DrcHeap {debug_assert!(FreeList::can_align_to(layout.align()));debug_assert_eq!(header.reserved_u26(), 0);- // We must have trace info for every GC type that we allocate in this- // heap. Trace info is eagerly registered during module instantiation- // and `StructRefPre`/`ArrayRefPre` construction. The only kinds of GC- // objects we allocate that do not have an associated- // `VMSharedTypeIndex` are `externref`s, and they don't have any GC- // edges.- if let Some(ty) = header.ty() {- debug_assert!(- self.trace_infos.contains(&ty),- "trace info for {ty:?} should have been eagerly registered",- );- } else {- debug_assert_eq!(header.kind(), VMGcKind::ExternRef);- }-let object_size = u32::try_from(layout.size()).unwrap();let alloc_size = FreeList::aligned_size(object_size).ok_or(Trap::AllocationTooLarge)?;@@ -1197,15 +1176,18 @@ unsafe impl GcHeap for DrcHeap {self.allocated_bytes}- fn gc<'a>(+ fn gc<'a, 'b>(&'a mut self,roots: GcRootsIter<'a>,- host_data_table: &'a mut ExternRefHostDataTable,- ) -> Box<dyn GarbageCollection<'a> + 'a> {+ trace_state: &'a mut GcStoreTraceState<'b>,+ ) -> Box<dyn GarbageCollection + 'a>+ where+ 'b: 'a,+ {assert_eq!(self.no_gc_count, 0, "Cannot GC inside a no-GC scope!");Box::new(DrcCollection {roots,- host_data_table,+ trace_state,heap: self,phase: DrcCollectionPhase::Trace,})@@ -1256,9 +1238,9 @@ unsafe impl GcHeap for DrcHeap {}}-struct DrcCollection<'a> {+struct DrcCollection<'a, 'b> {roots: GcRootsIter<'a>,- host_data_table: &'a mut ExternRefHostDataTable,+ trace_state: &'a mut GcStoreTraceState<'b>,heap: &'a mut DrcHeap,phase: DrcCollectionPhase,}@@ -1269,7 +1251,7 @@ enum DrcCollectionPhase {Done,}-impl<'a> GarbageCollection<'a> for DrcCollection<'a> {+impl GarbageCollection for DrcCollection<'_, '_> {fn collect_increment(&mut self) -> Result<GcProgress> {match self.phase {DrcCollectionPhase::Trace => {@@ -1300,7 +1282,7 @@ impl<'a> GarbageCollection<'a> for DrcCollection<'a> {self.heap.assert_over_approximated_stack_roots_integrity()?;self.heap.assert_free_blocks_are_poisoned();- self.heap.sweep(self.host_data_table)?;+ self.heap.sweep(self.trace_state)?;self.heap.assert_over_approximated_stack_roots_integrity()?;self.heap.assert_free_blocks_are_poisoned();diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rsindex 7b93e6eb98..c4f06f3dd4 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs@@ -9,8 +9,8 @@ use crate::{Engine, Trap,prelude::*,vm::{- ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,- GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMGcHeader, VMGcRef,+ ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,+ GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TypedGcRef, VMGcHeader, VMGcRef,VMMemoryDefinition,},};@@ -341,11 +341,14 @@ unsafe impl GcHeap for NullHeap {usize::try_from(next.get()).unwrap() - 1}- fn gc<'a>(+ fn gc<'a, 'b>(&'a mut self,_roots: GcRootsIter<'a>,- _host_data_table: &'a mut ExternRefHostDataTable,- ) -> Box<dyn GarbageCollection<'a> + 'a> {+ _trace_state: &'a mut GcStoreTraceState<'b>,+ ) -> Box<dyn GarbageCollection + 'a>+ where+ 'b: 'a,+ {assert_eq!(self.no_gc_count, 0, "Cannot GC inside a no-GC scope!");Box::new(NullCollection {})}@@ -358,7 +361,7 @@ unsafe impl GcHeap for NullHeap {struct NullCollection {}-impl<'a> GarbageCollection<'a> for NullCollection {+impl GarbageCollection for NullCollection {fn collect_increment(&mut self) -> Result<GcProgress> {Ok(GcProgress::Complete)}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rsdeleted file mode 100644index b53d25c6ea..0000000000--- a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rs+++ /dev/null@@ -1,166 +0,0 @@-//! Shared tracing information for GC collectors.-//!-//! Both the DRC and copying collectors need to know how to trace GC objects to-//! find outgoing GC-reference edges. This module provides a shared `TraceInfos`-//! type that both collectors use.--use crate::hash_map::HashMap;-use crate::{Engine, EngineWeak};-use alloc::boxed::Box;-use core::hash::BuildHasher;-use wasmtime_environ::{GcLayout, VMSharedTypeIndex};--/// How to trace a GC object.-pub(super) enum TraceInfo {- /// How to trace an array.- Array {- /// Whether this array type's elements are GC references, and need- /// tracing.- #[cfg_attr(- not(feature = "gc-drc"),- allow(dead_code, reason = "easier not to cfg on/off")- )]- gc_ref_elems: bool,- },-- /// How to trace a struct.- Struct {- /// The offsets of each GC reference field that needs tracing in- /// instances of this struct type.- gc_ref_offsets: Box<[u32]>,- },-}--/// A hasher that doesn't hash, for use in the trace-info hash map, where we are-/// just using scalar keys and aren't overly concerned with collision-based DoS.-#[derive(Default)]-struct NopHasher(u64);--impl BuildHasher for NopHasher {- type Hasher = Self;-- #[inline]- fn build_hasher(&self) -> Self::Hasher {- NopHasher::default()- }-}--impl core::hash::Hasher for NopHasher {- #[inline]- fn finish(&self) -> u64 {- self.0- }-- #[inline]- fn write(&mut self, bytes: &[u8]) {- let mut hash = self.0.to_ne_bytes();- let n = hash.len().min(bytes.len());- hash[..n].copy_from_slice(bytes);- self.0 = u64::from_ne_bytes(hash);- }-- #[inline]- fn write_u8(&mut self, i: u8) {- self.write_u64(i.into());- }-- #[inline]- fn write_u16(&mut self, i: u16) {- self.write_u64(i.into())- }-- #[inline]- fn write_u32(&mut self, i: u32) {- self.write_u64(i.into())- }-- #[inline]- fn write_u64(&mut self, i: u64) {- self.0 = i;- }-- #[inline]- fn write_usize(&mut self, i: usize) {- self.write_u64(i.try_into().unwrap());- }-}--/// A map from GC type indices to their tracing information.-#[derive(Default)]-pub(super) struct TraceInfos {- engine: EngineWeak,- map: HashMap<VMSharedTypeIndex, TraceInfo, NopHasher>,- gc_ref_array_elems_offset: u32,-}--impl TraceInfos {- /// Create a new `TraceInfos` with the given engine and expected array- /// element offset for GC-ref arrays.- pub fn new(engine: &Engine, gc_ref_array_elems_offset: u32) -> Self {- let mut map = HashMap::default();- map.reserve(1);- Self {- engine: engine.weak(),- map,- gc_ref_array_elems_offset,- }- }-- fn engine(&self) -> Engine {- self.engine.upgrade().unwrap()- }-- /// Remove all trace info from this collection.- pub fn clear(&mut self) {- self.map.clear();- }-- /// Index into the trace infos, panicking if the type is not present.- pub fn trace_info(&self, ty: &VMSharedTypeIndex) -> &TraceInfo {- &self.map[ty]- }-- /// Returns whether we already have tracing information for the given type.- pub fn contains(&self, ty: &VMSharedTypeIndex) -> bool {- self.map.contains_key(ty)- }-- /// Ensure that we have tracing information for the given type.- pub fn ensure(&mut self, ty: VMSharedTypeIndex) {- if self.map.contains_key(&ty) {- return;- }- self.insert_new(ty);- }-- fn insert_new(&mut self, ty: VMSharedTypeIndex) {- debug_assert!(!self.map.contains_key(&ty));-- let engine = self.engine();- let Some(gc_layout) = engine.signatures().layout(ty) else {- // Not a GC type (e.g. a function type); no trace info needed.- return;- };-- let info = match gc_layout {- GcLayout::Array(l) => {- if l.elems_are_gc_refs {- debug_assert_eq!(l.elem_offset(0), Some(self.gc_ref_array_elems_offset));- }- TraceInfo::Array {- gc_ref_elems: l.elems_are_gc_refs,- }- }- GcLayout::Struct(l) => TraceInfo::Struct {- gc_ref_offsets: l- .fields- .iter()- .filter_map(|f| if f.is_gc_ref { Some(f.offset) } else { None })- .collect(),- },- };-- let old_entry = self.map.insert(ty, info);- debug_assert!(old_entry.is_none());- }-}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rsnew file mode 100644index 0000000000..0fd20c6bd3--- /dev/null+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs@@ -0,0 +1,149 @@+//! Shared tracing information for GC collectors.+//!+//! Both the DRC and copying collectors need to know how to trace GC objects to+//! find outgoing GC-reference edges. This module provides a shared `TraceInfos`+//! type that both collectors use.++use crate::hash_map::{Entry, HashMap};+use crate::module::RegisteredModuleId;+use crate::vm::{GcStoreTraceState, TraceInfo};+use core::hash::BuildHasher;+use wasmtime_environ::{ModuleInternedTypeIndex, VMSharedTypeIndex};++/// A hasher that doesn't hash, for use in the trace-info hash map, where we are+/// just using scalar keys and aren't overly concerned with collision-based DoS.+#[derive(Default)]+struct NopHasher(u64);++impl BuildHasher for NopHasher {+ type Hasher = Self;++ #[inline]+ fn build_hasher(&self) -> Self::Hasher {+ NopHasher::default()+ }+}++impl core::hash::Hasher for NopHasher {+ #[inline]+ fn finish(&self) -> u64 {+ self.0+ }++ #[inline]+ fn write(&mut self, bytes: &[u8]) {+ let mut hash = self.0.to_ne_bytes();+ let n = hash.len().min(bytes.len());+ hash[..n].copy_from_slice(bytes);+ self.0 = u64::from_ne_bytes(hash);+ }++ #[inline]+ fn write_u8(&mut self, i: u8) {+ self.write_u64(i.into());+ }++ #[inline]+ fn write_u16(&mut self, i: u16) {+ self.write_u64(i.into())+ }++ #[inline]+ fn write_u32(&mut self, i: u32) {+ self.write_u64(i.into())+ }++ #[inline]+ fn write_u64(&mut self, i: u64) {+ self.0 = i;+ }++ #[inline]+ fn write_usize(&mut self, i: usize) {+ self.write_u64(i.try_into().unwrap());+ }+}++#[derive(Clone, Copy)]+enum TraceInfoLoc {+ HostType(VMSharedTypeIndex),+ Module(RegisteredModuleId, ModuleInternedTypeIndex),+}++/// A map from GC type indices to where their tracing information can be found.+#[derive(Default)]+pub(super) struct TraceInfos {+ map: HashMap<VMSharedTypeIndex, TraceInfoLoc, NopHasher>,+}++impl TraceInfos {+ /// Create a new `TraceInfos` with the given engine and expected array+ /// element offset for GC-ref arrays.+ pub fn new() -> Self {+ Self {+ map: HashMap::default(),+ }+ }++ /// Remove all trace info from this collection.+ pub fn clear(&mut self) {+ self.map.clear();+ }++ /// Lookup trace information for `ty`, panicking if it can't be found.+ pub fn trace_info<'a>(+ &mut self,+ ty: &VMSharedTypeIndex,+ state: &'a GcStoreTraceState<'_>,+ ) -> &'a TraceInfo {+ self.trace_info_(ty, state)+ .unwrap_or_else(|| panic!("failed to find trace information for {ty:?}"))+ }++ fn trace_info_<'a>(+ &mut self,+ ty: &VMSharedTypeIndex,+ state: &'a GcStoreTraceState<'_>,+ ) -> Option<&'a TraceInfo> {+ // Determine where the trace information for `ty` is stored. This lookup+ // is cached within `self.map` to avoid hitting `find_trace_info` too+ // often.+ let loc = match self.map.entry(*ty) {+ Entry::Occupied(e) => e.into_mut(),+ Entry::Vacant(e) => e.insert(find_trace_info(ty, state)?),+ };++ // Use `loc` to, relatively quickly, go to the trace information as+ // stored within `state`.+ Some(match loc {+ TraceInfoLoc::HostType(ty) => state.gc_host_alloc_types[ty].1.as_ref()?,+ TraceInfoLoc::Module(module_id, ty) => state+ .modules+ .module_by_id(*module_id)?+ .signatures()+ .trace_info(*ty)?,+ })+ }+}++/// Locates the trace information for `ty` within `state`.+///+/// This is a one-time operation per-store which is used to determine where+/// exactly trace information can be found. This is structured to notably be+/// relatively cheap to compute but additionally require zero up-front compute+/// in terms of instantiation or when a store is created.+fn find_trace_info(ty: &VMSharedTypeIndex, state: &GcStoreTraceState<'_>) -> Option<TraceInfoLoc> {+ if state.gc_host_alloc_types.contains_key(ty) {+ return Some(TraceInfoLoc::HostType(*ty));+ }++ // It's expected that most stores have a small number of modules, hence the+ // linear iteration here.+ for (id, module) in state.modules.all_modules() {+ if let Some(module_ty) = module.signatures().shared_type_with_trace_info(*ty) {+ return Some(TraceInfoLoc::Module(id, module_ty));+ }+ }++ None+}diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rsindex e69ca8d9b1..20fb2cf287 100644--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs@@ -3,7 +3,7 @@use crate::bail_bug;use crate::prelude::*;use crate::runtime::vm::{- ExternRefHostDataId, ExternRefHostDataTable, GcHeapObject, SendSyncPtr, TypedGcRef, VMArrayRef,+ ExternRefHostDataId, GcHeapObject, GcStoreTraceState, SendSyncPtr, TypedGcRef, VMArrayRef,VMExternRef, VMGcHeader, VMGcObjectData, VMGcRef, ValRaw,};use crate::store::Asyncness;@@ -110,18 +110,6 @@ pub unsafe trait GcHeap: 'static + Send + Sync {/// any virtual memory mappings.fn detach(&mut self) -> crate::vm::Memory;- /// Eagerly ensure that tracing information is registered for the given GC- /// type.- ///- /// This is called during module instantiation for every GC type in the- /// module's type collection, and during `StructRefPre` and `ArrayRefPre`- /// construction for host-allocated types.- ///- /// The default implementation is a no-op, which is appropriate for- /// collectors that do not need per-type tracing info (e.g. the null- /// collector).- fn ensure_trace_info(&mut self, _ty: VMSharedTypeIndex) {}-////////////////////////////////////////////////////////////////////////////// `Any` methods@@ -396,11 +384,13 @@ pub unsafe trait GcHeap: 'static + Send + Sync {/// incorrect results.////// This method should panic if we are in a no-GC scope.- fn gc<'a>(+ fn gc<'a, 'b>(&'a mut self,roots: GcRootsIter<'a>,- host_data_table: &'a mut ExternRefHostDataTable,- ) -> Box<dyn GarbageCollection<'a> + 'a>;+ trace_state: &'a mut GcStoreTraceState<'b>,+ ) -> Box<dyn GarbageCollection + 'a>+ where+ 'b: 'a;////////////////////////////////////////////////////////////////////////////// JIT-Code Interaction Methods@@ -783,7 +773,7 @@ impl GcRoot<'_> {/// When using fuel and/or epochs, consumers can also use `collect_increment`/// directly and choose to abandon further execution in this GC's heap's whole/// store if the GC is taking too long to complete.-pub trait GarbageCollection<'a>: Send + Sync {+pub trait GarbageCollection: Send + Sync {/// Perform an incremental slice of this garbage collection process.////// Upon completion of the slice, a `GcProgress` is returned which informs@@ -819,8 +809,8 @@ pub enum GcProgress {/// Asynchronously run the given garbage collection process to completion,/// cooperatively yielding back to the event loop after each increment of work.-pub async fn collect_async<'a>(- mut collection: Box<dyn GarbageCollection<'a> + 'a>,+pub async fn collect_async(+ mut collection: Box<dyn GarbageCollection + '_>,asyncness: Asyncness,yield_fn: impl AsyncFn(),) -> Result<()> {@@ -850,7 +840,7 @@ mod collect_async_tests {fn is_send_and_sync() {fn _assert_send_sync<T: Send + Sync>(_: T) {}- fn _foo<'a>(collection: Box<dyn GarbageCollection<'a>>) {+ fn _foo(collection: Box<dyn GarbageCollection>) {_assert_send_sync(collect_async(collection, Asyncness::Yes, async || ()));}}