Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Remove `ensure_trace_info` methods (#13843)

Alex Crichton2fb59986ce75
diff --git a/.github/actions/apt-get-install/action.yml b/.github/actions/apt-get-install/action.yml
index 1659482de6..1d5d3e6a68 100644
--- a/.github/actions/apt-get-install/action.yml
+++ b/.github/actions/apt-get-install/action.yml
@@ -33,8 +33,18 @@ runs:
- name: Update apt-get
shell: bash
- run: sudo apt-get update
+ run: |
+ for attempt in 1 2 3 4 5; do
+ [ "$attempt" = "5" ] && exit 1
+ sudo apt-get update && break
+ sleep 5
+ done
- name: Install packages
shell: bash
- run: sudo apt-get install -y ${{ inputs.packages }}
+ run: |
+ for attempt in 1 2 3 4 5; do
+ [ "$attempt" = "5" ] && exit 1
+ sudo apt-get install -y ${{ inputs.packages }} && break
+ sleep 5
+ done
diff --git a/crates/wasmtime/src/runtime/coredump.rs b/crates/wasmtime/src/runtime/coredump.rs
index c8c8fe366e..031fb7c0e2 100644
--- a/crates/wasmtime/src/runtime/coredump.rs
+++ b/crates/wasmtime/src/runtime/coredump.rs
@@ -39,7 +39,11 @@ pub struct WasmCoreDump {
impl WasmCoreDump {
pub(crate) fn new(store: &mut StoreOpaque, backtrace: WasmBacktrace) -> WasmCoreDump {
- let modules: Vec<_> = store.modules().all_modules().cloned().collect();
+ let modules = store
+ .modules()
+ .all_modules()
+ .map(|(_, m)| m.clone())
+ .collect::<Vec<_>>();
let instances: Vec<Instance> = store.all_instances().collect();
let store_memories: Vec<Memory> =
store.all_memories().filter_map(|m| m.unshared()).collect();
diff --git a/crates/wasmtime/src/runtime/debug.rs b/crates/wasmtime/src/runtime/debug.rs
index 36a886664b..c9d27abe76 100644
--- a/crates/wasmtime/src/runtime/debug.rs
+++ b/crates/wasmtime/src/runtime/debug.rs
@@ -170,7 +170,10 @@ impl StoreOpaque {
return vec![];
}
- self.modules().all_modules().cloned().collect()
+ self.modules()
+ .all_modules()
+ .map(|(_, m)| m.clone())
+ .collect()
}
}
@@ -1182,7 +1185,11 @@ impl<'a> BreakpointEdit<'a> {
// re-publishing code.
return Ok(());
}
- let modules = self.registry.all_modules().cloned().collect::<Vec<_>>();
+ let modules = self
+ .registry
+ .all_modules()
+ .map(|(_, m)| m.clone())
+ .collect::<Vec<_>>();
for module in modules {
let mem =
Self::get_code_memory(self.state, self.registry, &mut self.dirty_modules, &module)?;
diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs
index 22914fbdda..0f0a60a67e 100644
--- a/crates/wasmtime/src/runtime/func.rs
+++ b/crates/wasmtime/src/runtime/func.rs
@@ -2394,7 +2394,7 @@ impl HostFunc {
};
let (gc_lifo_scope, ret) = {
- let gc_lifo_scope = store.0.gc_roots().enter_lifo_scope();
+ let gc_lifo_scope = store.0.enter_gc_lifo_scope();
let mut args = NonNull::slice_from_raw_parts(args.cast(), args_len);
// SAFETY: it's a contract of this function itself that the values
diff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
index eb2f392270..706de77994 100644
--- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
@@ -1,4 +1,4 @@
-use crate::runtime::vm::{GcStore, VMGcRef};
+use crate::runtime::vm::VMGcRef;
use crate::{
AsContext, AsContextMut, GcRef, Result, RootedGcRef,
store::{AutoAssertNoGc, StoreOpaque},
@@ -34,17 +34,6 @@ mod sealed {
}
pub(crate) use sealed::*;
-#[derive(Debug, Default)]
-pub(crate) struct RootSet {}
-
-impl RootSet {
- pub(crate) fn enter_lifo_scope(&self) -> usize {
- usize::MAX
- }
-
- pub(crate) fn exit_lifo_scope(&mut self, _gc_store: Option<&mut GcStore>, _scope: usize) {}
-}
-
/// This type is disabled because the `gc` cargo feature was not enabled at
/// compile time.
pub struct Rooted<T: GcRef> {
diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs
index 1b4a539710..47856b5cc7 100644
--- a/crates/wasmtime/src/runtime/instance.rs
+++ b/crates/wasmtime/src/runtime/instance.rs
@@ -311,13 +311,6 @@ impl Instance {
// Allocate the GC heap, if necessary.
if module.env_module().needs_gc_heap {
store.ensure_gc_store(limiter.as_deref_mut()).await?;
-
- // Eagerly register trace info for all types in this module.
- if let Some(gc_store) = store.optional_gc_store_mut() {
- for (_, ty) in module.signatures().as_module_map().iter() {
- gc_store.ensure_trace_info(*ty);
- }
- }
}
// Register the module just before instantiation to ensure we keep the module
diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rs
index 3952255804..19b0f1f21c 100644
--- a/crates/wasmtime/src/runtime/module.rs
+++ b/crates/wasmtime/src/runtime/module.rs
@@ -657,6 +657,11 @@ impl Module {
self.inner.code.module_types()
}
+ #[cfg(any(
+ feature = "gc-drc",
+ feature = "gc-copying",
+ feature = "component-model"
+ ))]
pub(crate) fn signatures(&self) -> &crate::type_registry::TypeCollection {
self.inner.code.signatures()
}
diff --git a/crates/wasmtime/src/runtime/module/registry.rs b/crates/wasmtime/src/runtime/module/registry.rs
index cd05f1b44b..97c2e43e23 100644
--- a/crates/wasmtime/src/runtime/module/registry.rs
+++ b/crates/wasmtime/src/runtime/module/registry.rs
@@ -210,9 +210,9 @@ impl ModuleRegistry {
}
/// Gets an iterator over all modules in the registry.
- #[cfg(any(feature = "coredump", feature = "debug"))]
- pub fn all_modules(&self) -> impl Iterator<Item = &'_ Module> + '_ {
- self.modules.values()
+ #[cfg(any(feature = "coredump", feature = "debug", feature = "gc"))]
+ pub fn all_modules(&self) -> impl Iterator<Item = (RegisteredModuleId, &'_ Module)> + '_ {
+ self.modules.iter()
}
/// Registers a new module with the registry.
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index 78c46017b6..fae724cbba 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -76,14 +76,11 @@
//! contents of `StoreOpaque`. This is an invariant that we, as the authors of
//! `wasmtime`, must uphold for the public interface to be safe.
-use crate::RootSet;
use crate::error::OutOfMemory;
#[cfg(feature = "async")]
use crate::fiber;
use crate::module::{RegisterBreakpointState, RegisteredModuleId};
use crate::prelude::*;
-#[cfg(feature = "gc")]
-use crate::runtime::vm::GcRootsList;
#[cfg(feature = "stack-switching")]
use crate::runtime::vm::VMContRef;
use crate::runtime::vm::mpk::ProtectionKey;
@@ -129,6 +126,8 @@ pub use self::async_::CallHookHandler;
#[cfg(feature = "gc")]
mod gc;
+#[cfg(not(feature = "gc"))]
+mod gc_disabled;
/// A [`Store`] is a collection of WebAssembly instances and host-defined state.
///
@@ -477,27 +476,8 @@ pub struct StoreOpaque {
host_globals: TryPrimaryMap<DefinedGlobalIndex, StoreBox<VMHostGlobalContext>>,
// GC-related fields.
gc_store: Option<GcStore>,
- gc_roots: RootSet,
- #[cfg(feature = "gc")]
- gc_roots_list: GcRootsList,
- // Types for which the embedder has created an allocator for.
#[cfg(feature = "gc")]
- gc_host_alloc_types: crate::hash_set::HashSet<crate::type_registry::RegisteredType>,
- /// Pending exception, if any. This is also a GC root, because it
- /// needs to be rooted somewhere between the time that a pending
- /// exception is set and the time that the handling code takes the
- /// exception object. We use this rooting strategy rather than a
- /// root in an `Err` branch of a `Result` on the host side because
- /// it is less error-prone with respect to rooting behavior. See
- /// `throw()`, `take_pending_exception()`,
- /// `peek_pending_exception()`, `has_pending_exception()`, and
- /// `catch()`.
- ///
- /// Also note that the underlying reference here is a `VMExnRef`, a
- /// refinement of `VMGcRef`, but rooting APIs right now make it difficult to
- /// work with that directly so this is stored as `VMGcRef` instead.
- #[cfg(feature = "gc")]
- pending_exception: Option<VMGcRef>,
+ gc_data: gc::StoreGcData,
// Numbers of resources instantiated in this store, and their limits
instance_count: usize,
@@ -753,13 +733,8 @@ impl<T> Store<T> {
instances: TryPrimaryMap::new(),
signal_handler: None,
gc_store: None,
- gc_roots: RootSet::default(),
- #[cfg(feature = "gc")]
- gc_roots_list: GcRootsList::default(),
#[cfg(feature = "gc")]
- gc_host_alloc_types: Default::default(),
- #[cfg(feature = "gc")]
- pending_exception: None,
+ gc_data: Default::default(),
modules: ModuleRegistry::default(),
func_refs: FuncRefs::default(),
host_globals: TryPrimaryMap::new(),
@@ -1797,117 +1772,6 @@ impl StoreOpaque {
&mut self.vm_store_context
}
- /// Performs a lazy allocation of the `GcStore` within this store, returning
- /// the previous allocation if it's already present.
- ///
- /// This method will, if necessary, allocate a new `GcStore` -- linear
- /// memory and all. This is a blocking operation due to
- /// `ResourceLimiterAsync` which means that this should only be executed
- /// in a fiber context at this time.
- #[inline]
- pub(crate) async fn ensure_gc_store(
- &mut self,
- limiter: Option<&mut StoreResourceLimiter<'_>>,
- ) -> Result<&mut GcStore> {
- if self.gc_store.is_some() {
- return Ok(self.gc_store.as_mut().unwrap());
- }
- self.allocate_gc_store(limiter).await
- }
-
- #[inline(never)]
- async fn allocate_gc_store(
- &mut self,
- limiter: Option<&mut StoreResourceLimiter<'_>>,
- ) -> Result<&mut GcStore> {
- log::trace!("allocating GC heap for store {:?}", self.id());
-
- assert!(self.gc_store.is_none());
- assert_eq!(
- self.vm_store_context.gc_heap.get_mut().base.as_non_null(),
- NonNull::dangling(),
- );
- assert_eq!(self.vm_store_context.gc_heap.get_mut().current_length(), 0);
-
- let gc_store = allocate_gc_store(self, limiter).await?;
- *self.vm_store_context.gc_heap.get_mut() = gc_store.vmmemory_definition();
- return Ok(self.gc_store.insert(gc_store));
-
- #[cfg(feature = "gc")]
- async fn allocate_gc_store(
- store: &mut StoreOpaque,
- limiter: Option<&mut StoreResourceLimiter<'_>>,
- ) -> Result<GcStore> {
- use wasmtime_environ::packed_option::ReservedValue;
-
- let engine = store.engine();
- let mem_ty = engine.tunables().gc_heap_memory_type();
-
- ensure!(
- engine.features().gc_types(),
- "cannot allocate a GC store when GC is disabled at configuration time"
- );
- let gc_runtime = engine
- .gc_runtime()
- .context("no GC runtime: GC disabled at compile time or configuration time")?;
-
- // First, allocate the memory that will be our GC heap's storage.
- let mut request = InstanceAllocationRequest {
- id: InstanceId::reserved_value(),
- runtime_info: engine.empty_module_runtime_info(),
- imports: vm::Imports::default(),
- store,
- limiter,
- };
-
- let (mem_alloc_index, mem) = engine
- .allocator()
- .allocate_memory(
- &mut request,
- &mem_ty,
- None,
- wasmtime_environ::MemoryKind::GcHeap,
- )
- .await?;
-
- // Then, allocate the actual GC heap, passing in that memory
- // storage.
- let (index, mut heap) =
- match engine
- .allocator()
- .allocate_gc_heap(engine, &**gc_runtime, mem_alloc_index)
- {
- Ok(pair) => pair,
- Err(e) => unsafe {
- engine
- .allocator()
- .deallocate_memory(None, mem_alloc_index, mem);
- return Err(e);
- },
- };
- heap.attach(mem);
-
- let mut gc_store = GcStore::new(index, heap, engine.tunables().gc_zeal_alloc_counter);
-
- // Eagerly register trace info for any host-created types (via
- // StructRefPre/ArrayRefPre) that were created before this GC
- // store was allocated.
- for ty in &store.gc_host_alloc_types {
- gc_store.ensure_trace_info(ty.index());
- }
-
- Ok(gc_store)
- }
-
- #[cfg(not(feature = "gc"))]
- async fn allocate_gc_store(
- _: &mut StoreOpaque,
- _: Option<&mut StoreResourceLimiter<'_>>,
- ) -> Result<GcStore> {
- bail!("cannot allocate a GC store: the `gc` feature was disabled at compile time")
- }
- }
-
/// Attempts to access the GC store that has been previously allocated.
///
/// This method will return `Some` if the GC store was previously allocated.
@@ -1957,22 +1821,6 @@ impl StoreOpaque {
self.gc_store.as_mut()
}
- #[inline]
- pub(crate) fn gc_roots(&self) -> &RootSet {
- &self.gc_roots
- }
-
- #[inline]
- #[cfg(feature = "gc")]
- pub(crate) fn gc_roots_mut(&mut self) -> &mut RootSet {
- &mut self.gc_roots
- }
-
- #[inline]
- pub(crate) fn exit_gc_lifo_scope(&mut self, scope: usize) {
- self.gc_roots.exit_lifo_scope(self.gc_store.as_mut(), scope);
- }
-
/// Helper function execute a `init_gc_ref` when placing `gc_ref` in `dest`.
///
/// This avoids allocating `GcStore` where possible.
@@ -2361,18 +2209,6 @@ at https://bytecodealliance.org/security.
Ok(id)
}
- /// Tests whether there is a pending exception.
- pub fn has_pending_exception(&self) -> bool {
- #[cfg(feature = "gc")]
- {
- self.pending_exception.is_some()
- }
- #[cfg(not(feature = "gc"))]
- {
- false
- }
- }
-
#[cfg(target_has_atomic = "64")]
pub(crate) fn set_epoch_deadline(&mut self, delta: u64) {
// Set a new deadline based on the "epoch deadline delta".
@@ -2423,11 +2259,6 @@ at https://bytecodealliance.org/security.
// otherwise fall back to the runtime-agnostic code.
yield_now().await
}
-
- #[cfg(not(feature = "gc"))]
- pub(crate) fn require_gc_store_mut(&mut self) -> Result<&mut GcStore> {
- bail!("GC is disabled")
- }
}
#[cfg(any(feature = "async", feature = "gc"))]
diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs
index 419face962..67b4964987 100644
--- a/crates/wasmtime/src/runtime/store/gc.rs
+++ b/crates/wasmtime/src/runtime/store/gc.rs
@@ -1,12 +1,16 @@
//! GC-related methods for stores.
-use crate::error::Context;
+use crate::RootSet;
+use crate::error::{Context, ensure};
use crate::module::ModuleRegistry;
use crate::store::{
Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter, yield_now,
};
use crate::type_registry::RegisteredType;
-use crate::vm::{self, Backtrace, Frame, GcRootsList, GcStore, SendSyncPtr, VMGcRef};
+use crate::vm::{
+ self, Backtrace, Frame, GcRootsList, GcStore, InstanceAllocationRequest, SendSyncPtr,
+ StoreGcHostAllocTypes, TraceInfo, VMGcRef,
+};
use crate::{
ExnRef, GcHeapOutOfMemory, Result, Rooted, Store, StoreContextMut, ThrownException, bail,
};
@@ -16,6 +20,29 @@ use core::num::NonZeroU32;
use core::ops::{Deref, DerefMut};
use core::ptr::NonNull;
use wasmtime_environ::DefinedTagIndex;
+use wasmtime_environ::packed_option::ReservedValue;
+
+#[derive(Default)]
+pub(crate) struct StoreGcData {
+ gc_roots: RootSet,
+ gc_roots_list: GcRootsList,
+ // Types for which the embedder has created an allocator for.
+ gc_host_alloc_types: StoreGcHostAllocTypes,
+ /// Pending exception, if any. This is also a GC root, because it
+ /// needs to be rooted somewhere between the time that a pending
+ /// exception is set and the time that the handling code takes the
+ /// exception object. We use this rooting strategy rather than a
+ /// root in an `Err` branch of a `Result` on the host side because
+ /// it is less error-prone with respect to rooting behavior. See
+ /// `throw()`, `take_pending_exception()`,
+ /// `peek_pending_exception()`, `has_pending_exception()`, and
+ /// `catch()`.
+ ///
+ /// Also note that the underlying reference here is a `VMExnRef`, a
+ /// refinement of `VMGcRef`, but rooting APIs right now make it difficult to
+ /// work with that directly so this is stored as `VMGcRef` instead.
+ pending_exception: Option<VMGcRef>,
+}
impl<T> Store<T> {
/// Perform garbage collection.
@@ -188,7 +215,7 @@ impl StoreOpaque {
// borrows.
pub(crate) fn trim_gc_liveness_flags(&mut self, eager: bool) {
if let Some(gc_store) = self.gc_store.as_mut() {
- self.gc_roots.trim_liveness_flags(gc_store, eager);
+ self.gc_data.gc_roots.trim_liveness_flags(gc_store, eager);
}
}
@@ -455,7 +482,7 @@ impl StoreOpaque {
pub(crate) fn set_pending_exception(&mut self, exnref: &VMGcRef) -> crate::Error {
debug_assert!(exnref.is_exnref(&*self.unwrap_gc_store_mut().gc_heap));
let gc_store = self.gc_store.as_mut().unwrap();
- match gc_store.write_gc_ref(&mut self.pending_exception, Some(exnref)) {
+ match gc_store.write_gc_ref(&mut self.gc_data.pending_exception, Some(exnref)) {
Ok(()) => ThrownException.into(),
Err(e) => e,
}
@@ -464,7 +491,7 @@ impl StoreOpaque {
/// Takes the pending exception from this store, if any, and exposes it to
/// WebAssembly, returning the raw representation.
pub(crate) fn expose_pending_exception_to_wasm(&mut self) -> Option<NonZeroU32> {
- let exnref = self.pending_exception.take()?;
+ let exnref = self.gc_data.pending_exception.take()?;
let gc_store = self.unwrap_gc_store_mut();
debug_assert!(exnref.is_exnref(&*gc_store.gc_heap));
Some(gc_store.expose_gc_ref_to_wasm(exnref).unwrap())
@@ -473,7 +500,7 @@ impl StoreOpaque {
/// Takes the pending exception of the store, yielding ownership of its
/// reference to the `Rooted` that's returned.
fn take_pending_exception_rooted(&mut self) -> Option<Rooted<ExnRef>> {
- let vmexnref = self.pending_exception.take()?;
+ let vmexnref = self.gc_data.pending_exception.take()?;
debug_assert!(vmexnref.is_exnref(&*self.unwrap_gc_store().gc_heap));
let mut nogc = AutoAssertNoGc::new(self);
Some(Rooted::new(&mut nogc, vmexnref))
@@ -484,7 +511,7 @@ impl StoreOpaque {
pub(crate) fn pending_exception_tag_and_instance(
&mut self,
) -> Option<(InstanceId, DefinedTagIndex)> {
- let pending_exnref = self.pending_exception.as_ref()?.unchecked_copy();
+ let pending_exnref = self.gc_data.pending_exception.as_ref()?.unchecked_copy();
debug_assert!(pending_exnref.is_exnref(&*self.unwrap_gc_store_mut().gc_heap));
let mut store = AutoAssertNoGc::new(self);
@@ -503,7 +530,7 @@ impl StoreOpaque {
pub(crate) fn pending_exception_owned_rooted(
&mut self,
) -> Result<Option<crate::OwnedRooted<ExnRef>>, crate::OutOfMemory> {
- let pending = match &self.pending_exception {
+ let pending = match &self.gc_data.pending_exception {
Some(r) => r,
None => return Ok(None),
};
@@ -581,13 +608,17 @@ impl StoreOpaque {
// Take the GC roots out of `self` so we can borrow it mutably but still
// call mutable methods on `self`.
- let mut roots = core::mem::take(&mut self.gc_roots_list);
+ let mut roots = core::mem::take(&mut self.gc_data.gc_roots_list);
self.trace_roots(&mut roots, asyncness).await;
- self.unwrap_gc_store_mut()
+ self.gc_store
+ .as_mut()
+ .unwrap()
.gc(
asyncness,
unsafe { roots.iter() },
+ &self.modules,
+ &self.gc_data.gc_host_alloc_types,
// TODO: Once `Config` has an optional `AsyncFn` field for
// yielding to the current async runtime
// (e.g. `tokio::task::yield_now`), use that if set; otherwise
@@ -598,7 +629,7 @@ impl StoreOpaque {
// Restore the GC roots for the next GC.
roots.clear();
- self.gc_roots_list = roots;
+ self.gc_data.gc_roots_list = roots;
if log::log_enabled!(log::Level::Trace) {
let gc_store = self.gc_store.as_ref().unwrap();
@@ -803,13 +834,13 @@ impl StoreOpaque {
fn trace_user_roots(&mut self, gc_roots_list: &mut GcRootsList) {
log::trace!("Begin trace GC roots :: user");
- self.gc_roots.trace_roots(gc_roots_list);
+ self.gc_data.gc_roots.trace_roots(gc_roots_list);
log::trace!("End trace GC roots :: user");
}
fn trace_pending_exception_roots(&mut self, gc_roots_list: &mut GcRootsList) {
log::trace!("Begin trace GC roots :: pending exception");
- if let Some(pending_exception) = self.pending_exception.as_mut() {
+ if let Some(pending_exception) = self.gc_data.pending_exception.as_mut() {
unsafe {
gc_roots_list.add_vmgcref_root(pending_exception.into(), "Pending exception");
}
@@ -824,13 +855,120 @@ impl StoreOpaque {
/// reclaimed (since it is possible that none of the Wasm modules in this
/// store are holding it alive).
pub(crate) fn insert_gc_host_alloc_type(&mut self, ty: RegisteredType) {
- // If a GC heap is already allocated, eagerly register trace info
- // now. Otherwise, trace info will be registered when the GC heap
- // is allocated in `StoreOpaque::allocate_gc_store`.
- if let Some(gc_store) = self.optional_gc_store_mut() {
- gc_store.ensure_trace_info(ty.index());
+ let trace_info = ty.layout().map(TraceInfo::new);
+ self.gc_data
+ .gc_host_alloc_types
+ .insert(ty.index(), (ty, trace_info));
+ }
+
+ /// Performs a lazy allocation of the `GcStore` within this store, returning
+ /// the previous allocation if it's already present.
+ ///
+ /// This method will, if necessary, allocate a new `GcStore` -- linear
+ /// memory and all. This is a blocking operation due to
+ /// `ResourceLimiterAsync` which means that this should only be executed
+ /// in a fiber context at this time.
+ #[inline]
+ pub(crate) async fn ensure_gc_store(
+ &mut self,
+ limiter: Option<&mut StoreResourceLimiter<'_>>,
+ ) -> Result<&mut GcStore> {
+ if self.gc_store.is_some() {
+ return Ok(self.gc_store.as_mut().unwrap());
}
- self.gc_host_alloc_types.insert(ty);
+ self.allocate_gc_store(limiter).await
+ }
+
+ #[inline(never)]
+ async fn allocate_gc_store(
+ &mut self,
+ limiter: Option<&mut StoreResourceLimiter<'_>>,
+ ) -> Result<&mut GcStore> {
+ log::trace!("allocating GC heap for store {:?}", self.id());
+
+ assert!(self.gc_store.is_none());
+ assert_eq!(
+ self.vm_store_context.gc_heap.get_mut().base.as_non_null(),
+ NonNull::dangling(),
+ );
+ assert_eq!(self.vm_store_context.gc_heap.get_mut().current_length(), 0);
+
+ let engine = self.engine();
+ let mem_ty = engine.tunables().gc_heap_memory_type();
+ ensure!(
+ engine.features().gc_types(),
+ "cannot allocate a GC store when GC is disabled at configuration time"
+ );
+ let gc_runtime = engine
+ .gc_runtime()
+ .context("no GC runtime: GC disabled at compile time or configuration time")?;
+
+ // First, allocate the memory that will be our GC heap's storage.
+ let mut request = InstanceAllocationRequest {
+ id: InstanceId::reserved_value(),
+ runtime_info: engine.empty_module_runtime_info(),
+ imports: vm::Imports::default(),
+ store: self,
+ limiter,
+ };
+
+ let (mem_alloc_index, mem) = engine
+ .allocator()
+ .allocate_memory(
+ &mut request,
+ &mem_ty,
+ None,
+ wasmtime_environ::MemoryKind::GcHeap,
+ )
+ .await?;
+
+ // Then, allocate the actual GC heap, passing in that memory
+ // storage.
+ let (index, mut heap) =
+ match engine
+ .allocator()
+ .allocate_gc_heap(engine, &**gc_runtime, mem_alloc_index)
+ {
+ Ok(pair) => pair,
+ Err(e) => unsafe {
+ engine
+ .allocator()
+ .deallocate_memory(None, mem_alloc_index, mem);
+ return Err(e);
+ },
+ };
+ heap.attach(mem);
+
+ let gc_store = GcStore::new(index, heap, engine.tunables().gc_zeal_alloc_counter);
+ *self.vm_store_context.gc_heap.get_mut() = gc_store.vmmemory_definition();
+ Ok(self.gc_store.insert(gc_store))
+ }
+
+ /// Tests whether there is a pending exception.
+ pub fn has_pending_exception(&self) -> bool {
+ self.gc_data.pending_exception.is_some()
+ }
+
+ #[inline]
+ pub(crate) fn gc_roots(&self) -> &RootSet {
+ &self.gc_data.gc_roots
+ }
+
+ #[inline]
+ pub(crate) fn gc_roots_mut(&mut self) -> &mut RootSet {
+ &mut self.gc_data.gc_roots
+ }
+
+ #[inline]
+ pub(crate) fn enter_gc_lifo_scope(&self) -> usize {
+ self.gc_data.gc_roots.enter_lifo_scope()
+ }
+
+ #[inline]
+ pub(crate) fn exit_gc_lifo_scope(&mut self, scope: usize) {
+ self.gc_data
+ .gc_roots
+ .exit_lifo_scope(self.gc_store.as_mut(), scope);
}
}
diff --git a/crates/wasmtime/src/runtime/store/gc_disabled.rs b/crates/wasmtime/src/runtime/store/gc_disabled.rs
new file mode 100644
index 0000000000..a57713d754
--- /dev/null
+++ b/crates/wasmtime/src/runtime/store/gc_disabled.rs
@@ -0,0 +1,29 @@
+use crate::store::{StoreOpaque, StoreResourceLimiter};
+use crate::vm::GcStore;
+use crate::{Result, bail};
+
+impl StoreOpaque {
+ #[inline]
+ pub(crate) async fn ensure_gc_store(
+ &mut self,
+ _limiter: Option<&mut StoreResourceLimiter<'_>>,
+ ) -> Result<&mut GcStore> {
+ bail!("cannot allocate a GC store: the `gc` feature was disabled at compile time")
+ }
+
+ pub(crate) fn has_pending_exception(&self) -> bool {
+ false
+ }
+
+ pub(crate) fn require_gc_store_mut(&mut self) -> Result<&mut GcStore> {
+ bail!("GC is disabled")
+ }
+
+ #[inline]
+ pub(crate) fn enter_gc_lifo_scope(&self) -> usize {
+ 0
+ }
+
+ #[inline]
+ pub(crate) fn exit_gc_lifo_scope(&mut self, _scope: usize) {}
+}
diff --git a/crates/wasmtime/src/runtime/type_registry.rs b/crates/wasmtime/src/runtime/type_registry.rs
index da4e76cdb2..0d98863617 100644
--- a/crates/wasmtime/src/runtime/type_registry.rs
+++ b/crates/wasmtime/src/runtime/type_registry.rs
@@ -5,9 +5,10 @@
use crate::Engine;
use crate::error::OutOfMemory;
+use crate::hash_map::HashMap;
use crate::prelude::*;
use crate::sync::RwLock;
-use crate::vm::GcRuntime;
+use crate::vm::{GcRuntime, TraceInfo};
use alloc::sync::Arc;
use core::cell::Cell;
use core::iter;
@@ -91,6 +92,8 @@ pub struct TypeCollection {
rec_groups: TryVec<RecGroupEntry>,
types: TryPrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,
trampolines: TrySecondaryMap<VMSharedTypeIndex, PackedOption<ModuleInternedTypeIndex>>,
+ trace_info: TryPrimaryMap<ModuleInternedTypeIndex, Option<TraceInfo>>,
+ types_with_trace_info: HashMap<VMSharedTypeIndex, ModuleInternedTypeIndex>,
}
impl Debug for TypeCollection {
@@ -100,11 +103,14 @@ impl Debug for TypeCollection {
rec_groups,
types,
trampolines,
+ trace_info,
+ types_with_trace_info: _,
} = self;
f.debug_struct("TypeCollection")
.field("rec_groups", rec_groups)
.field("types", types)
.field("trampolines", trampolines)
+ .field("trace_info", trace_info)
.finish_non_exhaustive()
}
}
@@ -138,7 +144,7 @@ impl Engine {
let gc_runtime = engine.gc_runtime().map(|rt| &**rt);
// First, register these types in this engine's registry.
- let (rec_groups, types) = registry
+ let (rec_groups, types, trace_info) = registry
.0
.write()
.register_module_types(gc_runtime, module_types)?;
@@ -155,6 +161,15 @@ impl Engine {
}
});
+ // Build up the map of vm indices to module indices for when types have
+ // trace info associated with them.
+ let mut types_with_trace_info = TryHashMap::new();
+ for (module_ty, info) in trace_info.iter() {
+ if info.is_some() {
+ types_with_trace_info.insert(types[module_ty], module_ty)?;
+ }
+ }
+
// Then build our map from each function type's engine index to the
// module-index of its trampoline. Trampoline functions are queried by
// module-index in a compiled module, and doing this engine-to-module
@@ -187,6 +202,8 @@ impl Engine {
rec_groups,
types,
trampolines,
+ trace_info,
+ types_with_trace_info: types_with_trace_info.into(),
})
}
}
@@ -223,6 +240,23 @@ impl TypeCollection {
log::trace!("TypeCollection::trampoline_type({ty:?}) -> {trampoline_ty:?}");
trampoline_ty
}
+
+ /// Returns the `TraceInfo`, if any, for the `index` provided.
+ #[inline]
+ #[cfg(feature = "gc")]
+ pub fn trace_info(&self, index: ModuleInternedTypeIndex) -> Option<&TraceInfo> {
+ self.trace_info.get(index).and_then(|s| s.as_ref())
+ }
+
+ /// If `index` has `TraceInfo` associated with it then the corresponding
+ /// `ModuleInternedTypeIndex` for `index` is returned.
+ #[inline]
+ pub fn shared_type_with_trace_info(
+ &self,
+ index: VMSharedTypeIndex,
+ ) -> Option<ModuleInternedTypeIndex> {
+ self.types_with_trace_info.get(&index).copied()
+ }
}
impl Drop for TypeCollection {
@@ -696,6 +730,7 @@ impl TypeRegistryInner {
(
TryVec<RecGroupEntry>,
TryPrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,
+ TryPrimaryMap<ModuleInternedTypeIndex, Option<TraceInfo>>,
),
OutOfMemory,
> {
@@ -707,8 +742,15 @@ impl TypeRegistryInner {
// The map from a module type index to an engine type index for these
// module types.
let mut map = TryPrimaryMap::<ModuleInternedTypeIndex, VMSharedTypeIndex>::new();
+ let mut trace_info = TryPrimaryMap::new();
- if let Err(e) = self.register_module_types_impl(gc_runtime, types, &mut entries, &mut map) {
+ if let Err(e) = self.register_module_types_impl(
+ gc_runtime,
+ types,
+ &mut entries,
+ &mut map,
+ &mut trace_info,
+ ) {
for entry in entries {
if entry.decref("register_module_types cleanup") {
self.unregister_entry(entry);
@@ -719,7 +761,7 @@ impl TypeRegistryInner {
log::trace!("End registering module types");
- Ok((entries, map))
+ Ok((entries, map, trace_info))
}
fn register_module_types_impl(
@@ -728,9 +770,11 @@ impl TypeRegistryInner {
types: &ModuleTypes,
entries: &mut TryVec<RecGroupEntry>,
map: &mut TryPrimaryMap<ModuleInternedTypeIndex, VMSharedTypeIndex>,
+ trace_info: &mut TryPrimaryMap<ModuleInternedTypeIndex, Option<TraceInfo>>,
) -> Result<(), OutOfMemory> {
entries.reserve(types.rec_groups().len())?;
map.reserve(types.wasm_types().len())?;
+ trace_info.reserve(types.wasm_types().len())?;
for (_rec_group_index, module_group) in types.rec_groups() {
let entry = self.register_rec_group(
@@ -747,6 +791,14 @@ impl TypeRegistryInner {
{
let module_ty2 = map.push(*engine_ty).expect("reserved capacity");
assert_eq!(module_ty, module_ty2);
+
+ let info = self
+ .type_to_gc_layout
+ .get(*engine_ty)
+ .and_then(|l| l.as_ref())
+ .map(TraceInfo::new);
+ let module_ty3 = trace_info.push(info).expect("reserved capacity");
+ assert_eq!(module_ty, module_ty3);
}
entries.push(entry).expect("reserved capacity");
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index 47aeeafa41..5b0aae7248 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -22,13 +22,16 @@ pub use gc_runtime::*;
pub use host_data::*;
pub use i31::*;
+use crate::hash_map::HashMap;
+use crate::module::ModuleRegistry;
use crate::prelude::*;
use crate::runtime::vm::{GcHeapAllocationIndex, VMMemoryDefinition};
use crate::store::Asyncness;
+use crate::type_registry::RegisteredType;
use core::any::Any;
use core::mem::MaybeUninit;
use core::{alloc::Layout, num::NonZeroU32};
-use wasmtime_environ::{GcArrayLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};
+use wasmtime_environ::{GcArrayLayout, GcLayout, GcStructLayout, VMGcKind, VMSharedTypeIndex};
/// GC-related data that is one-to-one with a `wasmtime::Store`.
///
@@ -68,6 +71,22 @@ pub struct GcStore {
gc_zeal_alloc_counter_init: Option<NonZeroU32>,
}
+/// Convenience type definition for the storage, within a `Store`, of
+/// host-allocated types that are one-off used for host allocation.
+pub type StoreGcHostAllocTypes = HashMap<VMSharedTypeIndex, (RegisteredType, Option<TraceInfo>)>;
+
+/// Contextual information used when performing a GC to trace GC references as
+/// necesary.
+pub struct GcStoreTraceState<'a> {
+ /// The backing data of `externref`s, deleted from when a GC reference is
+ /// reclaimed, for example.
+ pub host_data_table: &'a mut ExternRefHostDataTable,
+ /// All known modules in this store.
+ pub modules: &'a ModuleRegistry,
+ /// All known host-registered types in this store.
+ pub gc_host_alloc_types: &'a StoreGcHostAllocTypes,
+}
+
impl GcStore {
/// Create a new `GcStore`.
pub fn new(
@@ -108,9 +127,16 @@ impl GcStore {
&mut self,
asyncness: Asyncness,
roots: GcRootsIter<'_>,
+ modules: &ModuleRegistry,
+ gc_host_alloc_types: &StoreGcHostAllocTypes,
yield_fn: impl AsyncFn(),
) -> Result<()> {
- let collection = self.gc_heap.gc(roots, &mut self.host_data_table);
+ let mut trace_state = GcStoreTraceState {
+ host_data_table: &mut self.host_data_table,
+ modules,
+ gc_host_alloc_types,
+ };
+ let collection = self.gc_heap.gc(roots, &mut trace_state);
collect_async(collection, asyncness, yield_fn).await?;
self.last_post_gc_allocated_bytes = Some({
let size = self.gc_heap.allocated_bytes();
@@ -299,11 +325,6 @@ impl GcStore {
self.gc_heap.alloc_raw(header, layout)
}
- /// Eagerly ensure tracing info is registered for the given type.
- pub fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {
- self.gc_heap.ensure_trace_info(ty)
- }
-
/// Allocate an uninitialized struct with the given type index and layout.
///
/// This does NOT check that the index is currently allocated in the types
@@ -394,3 +415,42 @@ impl GcStore {
}
}
}
+
+/// How to trace a GC object.
+#[derive(Debug)]
+pub enum TraceInfo {
+ /// How to trace an array.
+ Array {
+ /// Whether this array type's elements are GC references, and need
+ /// tracing.
+ #[cfg_attr(
+ not(feature = "gc-drc"),
+ allow(dead_code, reason = "easier not to cfg on/off")
+ )]
+ gc_ref_elems: bool,
+ },
+
+ /// How to trace a struct.
+ Struct {
+ /// The offsets of each GC reference field that needs tracing in
+ /// instances of this struct type.
+ gc_ref_offsets: Box<[u32]>,
+ },
+}
+
+impl TraceInfo {
+ pub(crate) fn new(gc_layout: &GcLayout) -> Self {
+ match gc_layout {
+ GcLayout::Array(l) => TraceInfo::Array {
+ gc_ref_elems: l.elems_are_gc_refs,
+ },
+ GcLayout::Struct(l) => TraceInfo::Struct {
+ gc_ref_offsets: l
+ .fields
+ .iter()
+ .filter_map(|f| if f.is_gc_ref { Some(f.offset) } else { None })
+ .collect(),
+ },
+ }
+ }
+}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled.rs b/crates/wasmtime/src/runtime/vm/gc/enabled.rs
index 478cd85fec..fc8bebb58a 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled.rs
@@ -7,7 +7,7 @@ mod externref;
mod free_list;
mod structref;
#[cfg(any(feature = "gc-drc", feature = "gc-copying"))]
-mod trace_info;
+mod trace_infos;
pub use arrayref::*;
pub use exnref::*;
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
index f90d03adcb..83b8e0bb2f 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
@@ -11,11 +11,11 @@
//! This collector does not require any read or write barriers.
use super::VMArrayRef;
-use super::trace_info::{TraceInfo, TraceInfos};
+use super::trace_infos::TraceInfos;
use crate::runtime::vm::{
- ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,
- GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMExternRef, VMGcHeader,
- VMGcRef, VMMemoryDefinition,
+ ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,
+ GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMExternRef,
+ VMGcHeader, VMGcRef, VMMemoryDefinition,
};
use crate::{Engine, bail_bug, prelude::*};
use core::{
@@ -42,8 +42,8 @@ unsafe impl GcRuntime for CopyingCollector {
&self.layouts
}
- fn new_gc_heap(&self, engine: &Engine) -> Result<Box<dyn GcHeap>> {
- let heap = CopyingHeap::new(engine)?;
+ fn new_gc_heap(&self, _engine: &Engine) -> Result<Box<dyn GcHeap>> {
+ let heap = CopyingHeap::new()?;
Ok(Box::new(heap) as _)
}
}
@@ -300,10 +300,10 @@ struct CopyingHeap {
}
impl CopyingHeap {
- fn new(engine: &Engine) -> Result<Self> {
+ fn new() -> Result<Self> {
log::trace!("allocating new copying heap");
Ok(Self {
- trace_infos: TraceInfos::new(engine, GC_REF_ARRAY_ELEMS_OFFSET),
+ trace_infos: TraceInfos::new(),
no_gc_count: 0,
memory: None,
vmmemory: None,
@@ -594,7 +594,12 @@ survived collection, since the active space is the same size as the idle space",
/// Trace a grey object's outgoing edges, copying their referents into the
/// new semi-space if necessary, and updating the object's references with
/// their forwarded locations in the new semi-space.
- fn scan(&mut self, gc_ref: &VMGcRef, trace_infos: &TraceInfos) -> Result<()> {
+ fn scan(
+ &mut self,
+ gc_ref: &VMGcRef,
+ trace_infos: &mut TraceInfos,
+ trace_state: &mut GcStoreTraceState<'_>,
+ ) -> Result<()> {
debug_assert!(!gc_ref.is_i31());
let index = gc_ref.heap_index()?.get();
debug_assert!(self.is_in_active_space(index));
@@ -633,7 +638,7 @@ survived collection, since the active space is the same size as the idle space",
let Some(ty) = ty else {
bail_bug!("out-of-line trace info but no type index");
};
- match trace_infos.trace_info(&ty) {
+ match trace_infos.trace_info(&ty, trace_state) {
TraceInfo::Struct { gc_ref_offsets } => {
for &offset in gc_ref_offsets {
self.scan_field(object_start, offset)?;
@@ -732,10 +737,6 @@ unsafe impl GcHeap for CopyingHeap {
memory.take().unwrap()
}
- fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {
- self.trace_infos.ensure(ty);
- }
-
fn as_any(&self) -> &dyn Any {
self as _
}
@@ -849,18 +850,6 @@ unsafe impl GcHeap for CopyingHeap {
);
debug_assert_eq!(header.reserved_u26() & HEADER_COPIED_BIT, 0);
- // We must have trace info for every GC type that we allocate in this
- // heap. Trace info is eagerly registered during module instantiation
- // and `StructRefPre`/`ArrayRefPre` construction.
- if let Some(ty) = header.ty() {
- debug_assert!(
- self.trace_infos.contains(&ty),
- "trace info for {ty:?} should have been eagerly registered",
- );
- } else {
- debug_assert_eq!(header.kind(), VMGcKind::ExternRef);
- }
-
let size = u32::try_from(layout.size()).unwrap();
// Round up the allocation size to ALIGN so that the next bump-pointer
// allocation is also aligned.
@@ -966,15 +955,18 @@ unsafe impl GcHeap for CopyingHeap {
usize::try_from(self.bump_ptr() - self.active_space_start).unwrap()
}
- fn gc<'a>(
+ fn gc<'a, 'b>(
&'a mut self,
roots: GcRootsIter<'a>,
- host_data_table: &'a mut ExternRefHostDataTable,
- ) -> Box<dyn GarbageCollection<'a> + 'a> {
+ trace_state: &'a mut GcStoreTraceState<'b>,
+ ) -> Box<dyn GarbageCollection + 'a>
+ where
+ 'b: 'a,
+ {
assert_eq!(self.no_gc_count, 0, "Cannot GC inside a no-GC scope!");
Box::new(CopyingCollection {
roots: Some(roots),
- host_data_table,
+ trace_state,
heap: self,
phase: CopyingCollectionPhase::ProcessRoots,
})
@@ -1036,9 +1028,9 @@ unsafe impl GcHeap for CopyingHeap {
const OBJECTS_PER_INCREMENT: usize = 16_384;
-struct CopyingCollection<'a> {
+struct CopyingCollection<'a, 'b> {
roots: Option<GcRootsIter<'a>>,
- host_data_table: &'a mut ExternRefHostDataTable,
+ trace_state: &'a mut GcStoreTraceState<'b>,
heap: &'a mut CopyingHeap,
phase: CopyingCollectionPhase,
}
@@ -1050,7 +1042,7 @@ enum CopyingCollectionPhase {
Done,
}
-impl CopyingCollection<'_> {
+impl CopyingCollection<'_, '_> {
/// Forward all GC roots from the idle space to the active space.
fn process_roots(&mut self) -> Result<()> {
log::trace!("Begin processing GC roots");
@@ -1098,7 +1090,7 @@ impl CopyingCollection<'_> {
/// increment regardless of whether the worklist still has items.
fn process_worklist_increment(&mut self) -> Result<GcProgress> {
log::trace!("Begin processing worklist increment");
- let trace_infos = mem::take(&mut self.heap.trace_infos);
+ let mut trace_infos = mem::take(&mut self.heap.trace_infos);
let mut count = 0;
let has_more = loop {
if count >= OBJECTS_PER_INCREMENT {
@@ -1107,7 +1099,8 @@ impl CopyingCollection<'_> {
match self.heap.worklist_pop()? {
Some(gc_ref) => {
debug_assert!(self.heap.is_in_active_space(gc_ref.heap_index()?.get()));
- self.heap.scan(&gc_ref, &trace_infos)?;
+ self.heap
+ .scan(&gc_ref, &mut trace_infos, self.trace_state)?;
count += 1;
}
None => break false,
@@ -1133,7 +1126,7 @@ impl CopyingCollection<'_> {
if !header.copied() {
let typed: &TypedGcRef<VMCopyingExternRef> = gc_ref.as_typed_unchecked();
let host_data_id = self.heap.index(typed)?.host_data;
- self.host_data_table.dealloc(host_data_id)?;
+ self.trace_state.host_data_table.dealloc(host_data_id)?;
}
link = self
.heap
@@ -1180,7 +1173,7 @@ impl CopyingCollection<'_> {
}
}
-impl GarbageCollection<'_> for CopyingCollection<'_> {
+impl GarbageCollection for CopyingCollection<'_, '_> {
fn collect_increment(&mut self) -> Result<GcProgress> {
match self.phase {
CopyingCollectionPhase::ProcessRoots => self.begin_collection(),
@@ -1191,7 +1184,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {
}
}
-impl Drop for CopyingCollection<'_> {
+impl Drop for CopyingCollection<'_, '_> {
fn drop(&mut self) {
// A collection is logically atomic with respect to the mutator: the
// mutator must never run while the heap is in a half-collected
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index 6bed2e510d..9532e43f05 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -46,12 +46,12 @@
use super::VMArrayRef;
use super::free_list::FreeList;
-use super::trace_info::{TraceInfo, TraceInfos};
+use super::trace_infos::TraceInfos;
use crate::hash_set::HashSet;
use crate::runtime::vm::{
- ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,
- GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMExternRef, VMGcHeader,
- VMGcObjectData, VMGcRef,
+ ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,
+ GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMExternRef,
+ VMGcHeader, VMGcObjectData, VMGcRef,
};
use crate::vm::VMMemoryDefinition;
use crate::{Engine, Trap, bail_bug, prelude::*};
@@ -91,8 +91,8 @@ unsafe impl GcRuntime for DrcCollector {
&self.layouts
}
- fn new_gc_heap(&self, engine: &Engine) -> Result<Box<dyn GcHeap>> {
- let heap = DrcHeap::new(engine)?;
+ fn new_gc_heap(&self, _engine: &Engine) -> Result<Box<dyn GcHeap>> {
+ let heap = DrcHeap::new()?;
Ok(Box::new(heap) as _)
}
}
@@ -220,10 +220,10 @@ struct TracingAllocs {
impl DrcHeap {
/// Construct a new, default DRC heap.
- fn new(engine: &Engine) -> Result<Self> {
+ fn new() -> Result<Self> {
log::trace!("allocating new DRC heap");
Ok(Self {
- trace_infos: TraceInfos::new(engine, GC_REF_ARRAY_ELEMS_OFFSET),
+ trace_infos: TraceInfos::new(),
no_gc_count: 0,
vmctx_data: Box::default(),
memory: None,
@@ -311,10 +311,7 @@ impl DrcHeap {
/// any that have reached a reference count of 0. Note that this is modeled
/// with an explicit stack rather than a recursive function to ensure that
/// this cannot overflow the host stack.
- fn process_dec_ref_stack(
- &mut self,
- host_data_table: &mut ExternRefHostDataTable,
- ) -> Result<()> {
+ fn process_dec_ref_stack(&mut self, trace_state: &mut GcStoreTraceState<'_>) -> Result<()> {
let allocs = match self.tracing_allocs.take() {
Some(allocs) => allocs,
None => bail_bug!("allocs missing during tracing"),
@@ -324,6 +321,7 @@ impl DrcHeap {
this.tracing_allocs = Some(allocs);
});
let (this, allocs) = &mut *undo;
+ let this: &mut Self = this;
let stack = &mut allocs.dec_ref_stack;
let large_array_stack = &mut allocs.large_array_dec_ref_stack;
let to_dealloc = &mut allocs.to_dealloc;
@@ -355,7 +353,7 @@ impl DrcHeap {
// Trace: enqueue child GC refs for dec-ref'ing.
if let Some(ty) = ty {
- match this.trace_infos.trace_info(&ty) {
+ match this.trace_infos.trace_info(&ty, trace_state) {
TraceInfo::Struct { gc_ref_offsets } => {
stack.reserve(gc_ref_offsets.len());
let data = this.gc_object_data(&gc_ref)?;
@@ -401,12 +399,12 @@ impl DrcHeap {
// data, and `ty` is `None` only for `externref`s, so we skip
// this for `struct` and `array` objects entirely.
debug_assert!(drc_header.header.kind().matches(VMGcKind::ExternRef));
- let externref = match gc_ref.as_typed::<VMDrcExternRef>(*this) {
+ let externref = match gc_ref.as_typed::<VMDrcExternRef>(this) {
Some(r) => r,
None => bail_bug!("expected externref"),
};
let host_data_id = this.index(externref)?.host_data;
- host_data_table.dealloc(host_data_id)?;
+ trace_state.host_data_table.dealloc(host_data_id)?;
}
to_dealloc.push(gc_ref);
@@ -613,7 +611,7 @@ impl DrcHeap {
/// Sweep the bump allocation table after we've discovered our precise stack
/// roots.
- fn sweep(&mut self, host_data_table: &mut ExternRefHostDataTable) -> Result<()> {
+ fn sweep(&mut self, trace_state: &mut GcStoreTraceState<'_>) -> Result<()> {
if log::log_enabled!(log::Level::Trace) {
Self::log_gc_ref_set(
"over-approximated-stack-roots set before sweeping",
@@ -704,7 +702,7 @@ impl DrcHeap {
if let Some(allocs) = &self.tracing_allocs
&& !allocs.dec_ref_stack.is_empty()
{
- self.process_dec_ref_stack(host_data_table)?;
+ self.process_dec_ref_stack(trace_state)?;
}
self.vmctx_data
@@ -946,10 +944,6 @@ unsafe impl GcHeap for DrcHeap {
memory.take().unwrap()
}
- fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {
- self.trace_infos.ensure(ty);
- }
-
fn as_any(&self) -> &dyn Any {
self as _
}
@@ -1086,21 +1080,6 @@ unsafe impl GcHeap for DrcHeap {
debug_assert!(FreeList::can_align_to(layout.align()));
debug_assert_eq!(header.reserved_u26(), 0);
- // We must have trace info for every GC type that we allocate in this
- // heap. Trace info is eagerly registered during module instantiation
- // and `StructRefPre`/`ArrayRefPre` construction. The only kinds of GC
- // objects we allocate that do not have an associated
- // `VMSharedTypeIndex` are `externref`s, and they don't have any GC
- // edges.
- if let Some(ty) = header.ty() {
- debug_assert!(
- self.trace_infos.contains(&ty),
- "trace info for {ty:?} should have been eagerly registered",
- );
- } else {
- debug_assert_eq!(header.kind(), VMGcKind::ExternRef);
- }
-
let object_size = u32::try_from(layout.size()).unwrap();
let alloc_size = FreeList::aligned_size(object_size).ok_or(Trap::AllocationTooLarge)?;
@@ -1197,15 +1176,18 @@ unsafe impl GcHeap for DrcHeap {
self.allocated_bytes
}
- fn gc<'a>(
+ fn gc<'a, 'b>(
&'a mut self,
roots: GcRootsIter<'a>,
- host_data_table: &'a mut ExternRefHostDataTable,
- ) -> Box<dyn GarbageCollection<'a> + 'a> {
+ trace_state: &'a mut GcStoreTraceState<'b>,
+ ) -> Box<dyn GarbageCollection + 'a>
+ where
+ 'b: 'a,
+ {
assert_eq!(self.no_gc_count, 0, "Cannot GC inside a no-GC scope!");
Box::new(DrcCollection {
roots,
- host_data_table,
+ trace_state,
heap: self,
phase: DrcCollectionPhase::Trace,
})
@@ -1256,9 +1238,9 @@ unsafe impl GcHeap for DrcHeap {
}
}
-struct DrcCollection<'a> {
+struct DrcCollection<'a, 'b> {
roots: GcRootsIter<'a>,
- host_data_table: &'a mut ExternRefHostDataTable,
+ trace_state: &'a mut GcStoreTraceState<'b>,
heap: &'a mut DrcHeap,
phase: DrcCollectionPhase,
}
@@ -1269,7 +1251,7 @@ enum DrcCollectionPhase {
Done,
}
-impl<'a> GarbageCollection<'a> for DrcCollection<'a> {
+impl GarbageCollection for DrcCollection<'_, '_> {
fn collect_increment(&mut self) -> Result<GcProgress> {
match self.phase {
DrcCollectionPhase::Trace => {
@@ -1300,7 +1282,7 @@ impl<'a> GarbageCollection<'a> for DrcCollection<'a> {
self.heap.assert_over_approximated_stack_roots_integrity()?;
self.heap.assert_free_blocks_are_poisoned();
- self.heap.sweep(self.host_data_table)?;
+ self.heap.sweep(self.trace_state)?;
self.heap.assert_over_approximated_stack_roots_integrity()?;
self.heap.assert_free_blocks_are_poisoned();
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
index 7b93e6eb98..c4f06f3dd4 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
@@ -9,8 +9,8 @@ use crate::{
Engine, Trap,
prelude::*,
vm::{
- ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,
- GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMGcHeader, VMGcRef,
+ ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,
+ GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TypedGcRef, VMGcHeader, VMGcRef,
VMMemoryDefinition,
},
};
@@ -341,11 +341,14 @@ unsafe impl GcHeap for NullHeap {
usize::try_from(next.get()).unwrap() - 1
}
- fn gc<'a>(
+ fn gc<'a, 'b>(
&'a mut self,
_roots: GcRootsIter<'a>,
- _host_data_table: &'a mut ExternRefHostDataTable,
- ) -> Box<dyn GarbageCollection<'a> + 'a> {
+ _trace_state: &'a mut GcStoreTraceState<'b>,
+ ) -> Box<dyn GarbageCollection + 'a>
+ where
+ 'b: 'a,
+ {
assert_eq!(self.no_gc_count, 0, "Cannot GC inside a no-GC scope!");
Box::new(NullCollection {})
}
@@ -358,7 +361,7 @@ unsafe impl GcHeap for NullHeap {
struct NullCollection {}
-impl<'a> GarbageCollection<'a> for NullCollection {
+impl GarbageCollection for NullCollection {
fn collect_increment(&mut self) -> Result<GcProgress> {
Ok(GcProgress::Complete)
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rs
deleted file mode 100644
index b53d25c6ea..0000000000
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rs
+++ /dev/null
@@ -1,166 +0,0 @@
-//! Shared tracing information for GC collectors.
-//!
-//! Both the DRC and copying collectors need to know how to trace GC objects to
-//! find outgoing GC-reference edges. This module provides a shared `TraceInfos`
-//! type that both collectors use.
-
-use crate::hash_map::HashMap;
-use crate::{Engine, EngineWeak};
-use alloc::boxed::Box;
-use core::hash::BuildHasher;
-use wasmtime_environ::{GcLayout, VMSharedTypeIndex};
-
-/// How to trace a GC object.
-pub(super) enum TraceInfo {
- /// How to trace an array.
- Array {
- /// Whether this array type's elements are GC references, and need
- /// tracing.
- #[cfg_attr(
- not(feature = "gc-drc"),
- allow(dead_code, reason = "easier not to cfg on/off")
- )]
- gc_ref_elems: bool,
- },
-
- /// How to trace a struct.
- Struct {
- /// The offsets of each GC reference field that needs tracing in
- /// instances of this struct type.
- gc_ref_offsets: Box<[u32]>,
- },
-}
-
-/// A hasher that doesn't hash, for use in the trace-info hash map, where we are
-/// just using scalar keys and aren't overly concerned with collision-based DoS.
-#[derive(Default)]
-struct NopHasher(u64);
-
-impl BuildHasher for NopHasher {
- type Hasher = Self;
-
- #[inline]
- fn build_hasher(&self) -> Self::Hasher {
- NopHasher::default()
- }
-}
-
-impl core::hash::Hasher for NopHasher {
- #[inline]
- fn finish(&self) -> u64 {
- self.0
- }
-
- #[inline]
- fn write(&mut self, bytes: &[u8]) {
- let mut hash = self.0.to_ne_bytes();
- let n = hash.len().min(bytes.len());
- hash[..n].copy_from_slice(bytes);
- self.0 = u64::from_ne_bytes(hash);
- }
-
- #[inline]
- fn write_u8(&mut self, i: u8) {
- self.write_u64(i.into());
- }
-
- #[inline]
- fn write_u16(&mut self, i: u16) {
- self.write_u64(i.into())
- }
-
- #[inline]
- fn write_u32(&mut self, i: u32) {
- self.write_u64(i.into())
- }
-
- #[inline]
- fn write_u64(&mut self, i: u64) {
- self.0 = i;
- }
-
- #[inline]
- fn write_usize(&mut self, i: usize) {
- self.write_u64(i.try_into().unwrap());
- }
-}
-
-/// A map from GC type indices to their tracing information.
-#[derive(Default)]
-pub(super) struct TraceInfos {
- engine: EngineWeak,
- map: HashMap<VMSharedTypeIndex, TraceInfo, NopHasher>,
- gc_ref_array_elems_offset: u32,
-}
-
-impl TraceInfos {
- /// Create a new `TraceInfos` with the given engine and expected array
- /// element offset for GC-ref arrays.
- pub fn new(engine: &Engine, gc_ref_array_elems_offset: u32) -> Self {
- let mut map = HashMap::default();
- map.reserve(1);
- Self {
- engine: engine.weak(),
- map,
- gc_ref_array_elems_offset,
- }
- }
-
- fn engine(&self) -> Engine {
- self.engine.upgrade().unwrap()
- }
-
- /// Remove all trace info from this collection.
- pub fn clear(&mut self) {
- self.map.clear();
- }
-
- /// Index into the trace infos, panicking if the type is not present.
- pub fn trace_info(&self, ty: &VMSharedTypeIndex) -> &TraceInfo {
- &self.map[ty]
- }
-
- /// Returns whether we already have tracing information for the given type.
- pub fn contains(&self, ty: &VMSharedTypeIndex) -> bool {
- self.map.contains_key(ty)
- }
-
- /// Ensure that we have tracing information for the given type.
- pub fn ensure(&mut self, ty: VMSharedTypeIndex) {
- if self.map.contains_key(&ty) {
- return;
- }
- self.insert_new(ty);
- }
-
- fn insert_new(&mut self, ty: VMSharedTypeIndex) {
- debug_assert!(!self.map.contains_key(&ty));
-
- let engine = self.engine();
- let Some(gc_layout) = engine.signatures().layout(ty) else {
- // Not a GC type (e.g. a function type); no trace info needed.
- return;
- };
-
- let info = match gc_layout {
- GcLayout::Array(l) => {
- if l.elems_are_gc_refs {
- debug_assert_eq!(l.elem_offset(0), Some(self.gc_ref_array_elems_offset));
- }
- TraceInfo::Array {
- gc_ref_elems: l.elems_are_gc_refs,
- }
- }
- GcLayout::Struct(l) => TraceInfo::Struct {
- gc_ref_offsets: l
- .fields
- .iter()
- .filter_map(|f| if f.is_gc_ref { Some(f.offset) } else { None })
- .collect(),
- },
- };
-
- let old_entry = self.map.insert(ty, info);
- debug_assert!(old_entry.is_none());
- }
-}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs
new file mode 100644
index 0000000000..0fd20c6bd3
--- /dev/null
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs
@@ -0,0 +1,149 @@
+//! Shared tracing information for GC collectors.
+//!
+//! Both the DRC and copying collectors need to know how to trace GC objects to
+//! find outgoing GC-reference edges. This module provides a shared `TraceInfos`
+//! type that both collectors use.
+
+use crate::hash_map::{Entry, HashMap};
+use crate::module::RegisteredModuleId;
+use crate::vm::{GcStoreTraceState, TraceInfo};
+use core::hash::BuildHasher;
+use wasmtime_environ::{ModuleInternedTypeIndex, VMSharedTypeIndex};
+
+/// A hasher that doesn't hash, for use in the trace-info hash map, where we are
+/// just using scalar keys and aren't overly concerned with collision-based DoS.
+#[derive(Default)]
+struct NopHasher(u64);
+
+impl BuildHasher for NopHasher {
+ type Hasher = Self;
+
+ #[inline]
+ fn build_hasher(&self) -> Self::Hasher {
+ NopHasher::default()
+ }
+}
+
+impl core::hash::Hasher for NopHasher {
+ #[inline]
+ fn finish(&self) -> u64 {
+ self.0
+ }
+
+ #[inline]
+ fn write(&mut self, bytes: &[u8]) {
+ let mut hash = self.0.to_ne_bytes();
+ let n = hash.len().min(bytes.len());
+ hash[..n].copy_from_slice(bytes);
+ self.0 = u64::from_ne_bytes(hash);
+ }
+
+ #[inline]
+ fn write_u8(&mut self, i: u8) {
+ self.write_u64(i.into());
+ }
+
+ #[inline]
+ fn write_u16(&mut self, i: u16) {
+ self.write_u64(i.into())
+ }
+
+ #[inline]
+ fn write_u32(&mut self, i: u32) {
+ self.write_u64(i.into())
+ }
+
+ #[inline]
+ fn write_u64(&mut self, i: u64) {
+ self.0 = i;
+ }
+
+ #[inline]
+ fn write_usize(&mut self, i: usize) {
+ self.write_u64(i.try_into().unwrap());
+ }
+}
+
+#[derive(Clone, Copy)]
+enum TraceInfoLoc {
+ HostType(VMSharedTypeIndex),
+ Module(RegisteredModuleId, ModuleInternedTypeIndex),
+}
+
+/// A map from GC type indices to where their tracing information can be found.
+#[derive(Default)]
+pub(super) struct TraceInfos {
+ map: HashMap<VMSharedTypeIndex, TraceInfoLoc, NopHasher>,
+}
+
+impl TraceInfos {
+ /// Create a new `TraceInfos` with the given engine and expected array
+ /// element offset for GC-ref arrays.
+ pub fn new() -> Self {
+ Self {
+ map: HashMap::default(),
+ }
+ }
+
+ /// Remove all trace info from this collection.
+ pub fn clear(&mut self) {
+ self.map.clear();
+ }
+
+ /// Lookup trace information for `ty`, panicking if it can't be found.
+ pub fn trace_info<'a>(
+ &mut self,
+ ty: &VMSharedTypeIndex,
+ state: &'a GcStoreTraceState<'_>,
+ ) -> &'a TraceInfo {
+ self.trace_info_(ty, state)
+ .unwrap_or_else(|| panic!("failed to find trace information for {ty:?}"))
+ }
+
+ fn trace_info_<'a>(
+ &mut self,
+ ty: &VMSharedTypeIndex,
+ state: &'a GcStoreTraceState<'_>,
+ ) -> Option<&'a TraceInfo> {
+ // Determine where the trace information for `ty` is stored. This lookup
+ // is cached within `self.map` to avoid hitting `find_trace_info` too
+ // often.
+ let loc = match self.map.entry(*ty) {
+ Entry::Occupied(e) => e.into_mut(),
+ Entry::Vacant(e) => e.insert(find_trace_info(ty, state)?),
+ };
+
+ // Use `loc` to, relatively quickly, go to the trace information as
+ // stored within `state`.
+ Some(match loc {
+ TraceInfoLoc::HostType(ty) => state.gc_host_alloc_types[ty].1.as_ref()?,
+ TraceInfoLoc::Module(module_id, ty) => state
+ .modules
+ .module_by_id(*module_id)?
+ .signatures()
+ .trace_info(*ty)?,
+ })
+ }
+}
+
+/// Locates the trace information for `ty` within `state`.
+///
+/// This is a one-time operation per-store which is used to determine where
+/// exactly trace information can be found. This is structured to notably be
+/// relatively cheap to compute but additionally require zero up-front compute
+/// in terms of instantiation or when a store is created.
+fn find_trace_info(ty: &VMSharedTypeIndex, state: &GcStoreTraceState<'_>) -> Option<TraceInfoLoc> {
+ if state.gc_host_alloc_types.contains_key(ty) {
+ return Some(TraceInfoLoc::HostType(*ty));
+ }
+
+ // It's expected that most stores have a small number of modules, hence the
+ // linear iteration here.
+ for (id, module) in state.modules.all_modules() {
+ if let Some(module_ty) = module.signatures().shared_type_with_trace_info(*ty) {
+ return Some(TraceInfoLoc::Module(id, module_ty));
+ }
+ }
+
+ None
+}
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
index e69ca8d9b1..20fb2cf287 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
@@ -3,7 +3,7 @@
use crate::bail_bug;
use crate::prelude::*;
use crate::runtime::vm::{
- ExternRefHostDataId, ExternRefHostDataTable, GcHeapObject, SendSyncPtr, TypedGcRef, VMArrayRef,
+ ExternRefHostDataId, GcHeapObject, GcStoreTraceState, SendSyncPtr, TypedGcRef, VMArrayRef,
VMExternRef, VMGcHeader, VMGcObjectData, VMGcRef, ValRaw,
};
use crate::store::Asyncness;
@@ -110,18 +110,6 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// any virtual memory mappings.
fn detach(&mut self) -> crate::vm::Memory;
- /// Eagerly ensure that tracing information is registered for the given GC
- /// type.
- ///
- /// This is called during module instantiation for every GC type in the
- /// module's type collection, and during `StructRefPre` and `ArrayRefPre`
- /// construction for host-allocated types.
- ///
- /// The default implementation is a no-op, which is appropriate for
- /// collectors that do not need per-type tracing info (e.g. the null
- /// collector).
- fn ensure_trace_info(&mut self, _ty: VMSharedTypeIndex) {}
-
////////////////////////////////////////////////////////////////////////////
// `Any` methods
@@ -396,11 +384,13 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// incorrect results.
///
/// This method should panic if we are in a no-GC scope.
- fn gc<'a>(
+ fn gc<'a, 'b>(
&'a mut self,
roots: GcRootsIter<'a>,
- host_data_table: &'a mut ExternRefHostDataTable,
- ) -> Box<dyn GarbageCollection<'a> + 'a>;
+ trace_state: &'a mut GcStoreTraceState<'b>,
+ ) -> Box<dyn GarbageCollection + 'a>
+ where
+ 'b: 'a;
////////////////////////////////////////////////////////////////////////////
// JIT-Code Interaction Methods
@@ -783,7 +773,7 @@ impl GcRoot<'_> {
/// When using fuel and/or epochs, consumers can also use `collect_increment`
/// directly and choose to abandon further execution in this GC's heap's whole
/// store if the GC is taking too long to complete.
-pub trait GarbageCollection<'a>: Send + Sync {
+pub trait GarbageCollection: Send + Sync {
/// Perform an incremental slice of this garbage collection process.
///
/// Upon completion of the slice, a `GcProgress` is returned which informs
@@ -819,8 +809,8 @@ pub enum GcProgress {
/// Asynchronously run the given garbage collection process to completion,
/// cooperatively yielding back to the event loop after each increment of work.
-pub async fn collect_async<'a>(
- mut collection: Box<dyn GarbageCollection<'a> + 'a>,
+pub async fn collect_async(
+ mut collection: Box<dyn GarbageCollection + '_>,
asyncness: Asyncness,
yield_fn: impl AsyncFn(),
) -> Result<()> {
@@ -850,7 +840,7 @@ mod collect_async_tests {
fn is_send_and_sync() {
fn _assert_send_sync<T: Send + Sync>(_: T) {}
- fn _foo<'a>(collection: Box<dyn GarbageCollection<'a>>) {
+ fn _foo(collection: Box<dyn GarbageCollection>) {
_assert_send_sync(collect_async(collection, Asyncness::Yes, async || ()));
}
}