A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Eagerly register `TraceInfo` for types with the `GcHeap` (#13317)
Nick Fitzgeraldb2788afe8248
diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rsindex fb8b2d8126..266d066de9 100644--- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs@@ -76,6 +76,14 @@ impl ArrayRefPre {pub(crate) fn _new(store: &mut StoreOpaque, ty: ArrayType) -> Self {store.insert_gc_host_alloc_type(ty.registered_type().clone());++ // If a GC heap is already allocated, eagerly register trace info+ // now. Otherwise, trace info will be registered when the GC heap+ // is allocated in `StoreOpaque::allocate_gc_store`.+ if let Some(gc_store) = store.optional_gc_store_mut() {+ gc_store.ensure_trace_info(ty.registered_type().index());+ }+let store_id = store.id();ArrayRefPre { store_id, ty }}diff --git a/crates/wasmtime/src/runtime/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/gc/enabled/structref.rsindex e3b755159d..d9d5dc5c8b 100644--- a/crates/wasmtime/src/runtime/gc/enabled/structref.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/structref.rs@@ -76,8 +76,15 @@ impl StructRefPre {pub(crate) fn _new(store: &mut StoreOpaque, ty: StructType) -> Self {store.insert_gc_host_alloc_type(ty.registered_type().clone());- let store_id = store.id();+ // If a GC heap is already allocated, eagerly register trace info+ // now. Otherwise, trace info will be registered when the GC heap+ // is allocated in `StoreOpaque::allocate_gc_store`.+ if let Some(gc_store) = store.optional_gc_store_mut() {+ gc_store.ensure_trace_info(ty.registered_type().index());+ }++ let store_id = store.id();StructRefPre { store_id, ty }}diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rsindex 6d67feed45..f2133cc275 100644--- a/crates/wasmtime/src/runtime/instance.rs+++ b/crates/wasmtime/src/runtime/instance.rs@@ -308,6 +308,13 @@ impl Instance {// Allocate the GC heap, if necessary.if module.env_module().needs_gc_heap {store.ensure_gc_store(limiter.as_deref_mut()).await?;++ // Eagerly register trace info for all types in this module.+ if let Some(gc_store) = store.optional_gc_store_mut() {+ for (_, ty) in module.signatures().as_module_map().iter() {+ gc_store.ensure_trace_info(*ty);+ }+ }}let compiled_module = module.compiled_module();diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rsindex a34e68c7cd..bc6bcfcb4e 100644--- a/crates/wasmtime/src/runtime/module.rs+++ b/crates/wasmtime/src/runtime/module.rs@@ -661,7 +661,6 @@ impl Module {self.inner.code.module_types()}- #[cfg(any(feature = "component-model", feature = "gc-drc"))]pub(crate) fn signatures(&self) -> &crate::type_registry::TypeCollection {self.inner.code.signatures()}diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 18f7b040ae..6af09b19a1 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -1995,11 +1995,16 @@ impl StoreOpaque {.allocator().allocate_gc_heap(engine, &**gc_runtime, mem_alloc_index, mem)?;- Ok(GcStore::new(- index,- heap,- engine.tunables().gc_zeal_alloc_counter,- ))+ let mut gc_store = GcStore::new(index, heap, engine.tunables().gc_zeal_alloc_counter);++ // Eagerly register trace info for any host-created types (via+ // StructRefPre/ArrayRefPre) that were created before this GC+ // store was allocated.+ for ty in &store.gc_host_alloc_types {+ gc_store.ensure_trace_info(ty.index());+ }++ Ok(gc_store)}#[cfg(not(feature = "gc"))]diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rsindex 85827282f5..ea4d197eca 100644--- a/crates/wasmtime/src/runtime/vm/gc.rs+++ b/crates/wasmtime/src/runtime/vm/gc.rs@@ -291,6 +291,11 @@ impl GcStore {self.gc_heap.alloc_raw(header, layout)}+ /// Eagerly ensure tracing info is registered for the given type.+ pub fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {+ self.gc_heap.ensure_trace_info(ty)+ }+/// Allocate an uninitialized struct with the given type index and layout.////// This does NOT check that the index is currently allocated in the typesdiff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rsindex 7170c1608d..a5d70100bc 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs@@ -612,6 +612,10 @@ unsafe impl GcHeap for CopyingHeap {memory.take().unwrap()}+ fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {+ self.trace_infos.ensure(ty);+ }+fn as_any(&self) -> &dyn Any {self as _}@@ -720,9 +724,13 @@ unsafe impl GcHeap for CopyingHeap {debug_assert_eq!(header.reserved_u26(), 0);// We must have trace info for every GC type that we allocate in this- // heap.+ // heap. Trace info is eagerly registered during module instantiation+ // and `StructRefPre`/`ArrayRefPre` construction.if let Some(ty) = header.ty() {- self.trace_infos.ensure(ty);+ debug_assert!(+ self.trace_infos.contains(&ty),+ "trace info for {ty:?} should have been eagerly registered",+ );} else {debug_assert_eq!(header.kind(), VMGcKind::ExternRef);}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rsindex f03bf3ec17..816c432428 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs@@ -808,6 +808,10 @@ unsafe impl GcHeap for DrcHeap {memory.take().unwrap()}+ fn ensure_trace_info(&mut self, ty: VMSharedTypeIndex) {+ self.trace_infos.ensure(ty);+ }+fn as_any(&self) -> &dyn Any {self as _}@@ -931,11 +935,16 @@ unsafe impl GcHeap for DrcHeap {debug_assert_eq!(header.reserved_u26(), 0);// We must have trace info for every GC type that we allocate in this- // heap. The only kinds of GC objects we allocate that do not have an- // associated `VMSharedTypeIndex` are `externref`s, and they don't have- // any GC edges.+ // heap. Trace info is eagerly registered during module instantiation+ // and `StructRefPre`/`ArrayRefPre` construction. The only kinds of GC+ // objects we allocate that do not have an associated+ // `VMSharedTypeIndex` are `externref`s, and they don't have any GC+ // edges.if let Some(ty) = header.ty() {- self.trace_infos.ensure(ty);+ debug_assert!(+ self.trace_infos.contains(&ty),+ "trace info for {ty:?} should have been eagerly registered",+ );} else {debug_assert_eq!(header.kind(), VMGcKind::ExternRef);}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rsindex 6a909857ee..4f9de0d8b5 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_info.rs@@ -111,6 +111,11 @@ impl TraceInfos {&self.map[ty]}+ /// Returns whether we already have tracing information for the given type.+ pub fn contains(&self, ty: &VMSharedTypeIndex) -> bool {+ self.map.contains_key(ty)+ }+/// Ensure that we have tracing information for the given type.pub fn ensure(&mut self, ty: VMSharedTypeIndex) {if self.map.contains_key(&ty) {@@ -123,10 +128,10 @@ impl TraceInfos {debug_assert!(!self.map.contains_key(&ty));let engine = self.engine();- let gc_layout = engine- .signatures()- .layout(ty)- .unwrap_or_else(|| panic!("should have a GC layout for {ty:?}"));+ let Some(gc_layout) = engine.signatures().layout(ty) else {+ // Not a GC type (e.g. a function type); no trace info needed.+ return;+ };let info = match gc_layout {GcLayout::Array(l) => {diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rsindex 8fe05f1e3b..8eab8d3fbf 100644--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs@@ -109,6 +109,18 @@ pub unsafe trait GcHeap: 'static + Send + Sync {/// any virtual memory mappings.fn detach(&mut self) -> crate::vm::Memory;+ /// Eagerly ensure that tracing information is registered for the given GC+ /// type.+ ///+ /// This is called during module instantiation for every GC type in the+ /// module's type collection, and during `StructRefPre` and `ArrayRefPre`+ /// construction for host-allocated types.+ ///+ /// The default implementation is a no-op, which is appropriate for+ /// collectors that do not need per-type tracing info (e.g. the null+ /// collector).+ fn ensure_trace_info(&mut self, _ty: VMSharedTypeIndex) {}+////////////////////////////////////////////////////////////////////////////// `Any` methods