A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Replace almost every manually-written `AliasRegions` helper method with macro-generated versions (#14267)
Nick Fitzgerald7a882e9dca76
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex d1700dbe85..857a8c1376 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -6,7 +6,8 @@ use cranelift_codegen::{};use wasmtime_environ::{BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, GetPtrSize,- ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex, StaticModuleIndex, VMOffsets,+ ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, PtrSize as _, RuntimeDataIndex,+ StaticModuleIndex, VMOffsets, VmctxArrayIndex as _,component::{ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,@@ -571,6 +572,11 @@ impl<'a, Offsets> Field<'a, Offsets> {/// fn current_length(self) -> Field<'a, Offsets> { ... }/// }/// ```+///+/// A field marked `#[aggregate]` gets no accessor, because it has no single+/// Cranelift type. A field marked `#[indexed]` is a fixed-size array of a+/// scalar type; its accessor takes the element index, and each element gets its+/// own alias region.#[allow(unused_macro_rules,reason = "the `#[readonly]`/`#[can_move]` marker arms are generated \@@ -597,6 +603,8 @@ macro_rules! define_vm_type_alias_region_helpers {(@field_ty $pt:expr, i64) => { ir::types::I64 };(@field_ty $pt:expr, u64) => { ir::types::I64 };(@field_ty $pt:expr, u32) => { ir::types::I32 };+ (@field_ty $pt:expr, NonZeroU32) => { ir::types::I32 };+ (@field_ty $pt:expr, Option < VMGcRef >) => { ir::types::I32 };// `VMLazyThread` is a pointer-sized bitpacked integer; see its definition in// `for_each_vm_type!`.(@field_ty $pt:expr, VMLazyThread) => { $pt };@@ -607,6 +615,13 @@ macro_rules! define_vm_type_alias_region_helpers {(@field_ty $pt:expr, DefinedTagIndex) => { ir::types::I32 };(@field_ty $pt:expr, VMGlobalKind) => { ir::types::I64 };+ // Classify an `#[indexed]` field's array type to+ // `(element ir::Type, element size in bytes, element count)`.+ (@array_info $pt:expr, UnsafeCell < $inner:tt >) => {+ define_vm_type_alias_region_helpers!(@array_info $pt, $inner)+ };+ (@array_info $pt:expr, [u32; $n:expr]) => { (ir::types::I32, 4u32, $n) };+// Apply a field attribute to its access flags. The `#[readonly]` and// `#[can_move]` markers map to the corresponding `MemFlagsData` builder// methods; doc comments have no effect on flags; any other attribute is a@@ -615,13 +630,68 @@ macro_rules! define_vm_type_alias_region_helpers {(@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() };(@apply_attr $flags:expr, [doc = $d:literal]) => { $flags };+ // Emit the accessor method for a single ordinary (scalar) field.+ (@method $Name:ident $snake:ident plain $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ]) => {+ #[doc = concat!(+ "Get the [`Field`] for the `", stringify!($fname),+ "` field of `", stringify!($Name), "`."+ )]+ pub fn $fname(self) -> Field<'a, Offsets> {+ let offset = u32::from(+ self.regions.offsets.get_ptr_size().$snake().$fname()+ );+ let flags = ir::MemFlagsData::trusted();+ $(+ let flags = define_vm_type_alias_region_helpers!(+ @apply_attr flags, $fattr+ );+ )*+ let ty = define_vm_type_alias_region_helpers!(+ @field_ty self.regions.pointer_type, $($fty)*+ );+ Field::new(self.regions, VmType::$Name, offset, flags, ty)+ }+ };++ // Emit the accessor method for an `#[indexed]` field: a fixed-size array of+ // a scalar type, indexed by a compile-time constant.+ (@method $Name:ident $snake:ident indexed $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ]) => {+ #[doc = concat!(+ "Get the [`Field`] for the `index`th element of the `",+ stringify!($fname), "` field of `", stringify!($Name), "`."+ )]+ pub fn $fname(self, index: u8) -> Field<'a, Offsets> {+ let (ty, elem_size, len) = define_vm_type_alias_region_helpers!(+ @array_info self.regions.pointer_type, $($fty)*+ );+ assert!(+ usize::from(index) < len,+ concat!(+ "index out of bounds for `", stringify!($Name), "::",+ stringify!($fname), "`"+ ),+ );+ let base = u32::from(+ self.regions.offsets.get_ptr_size().$snake().$fname()+ );+ let offset = base + u32::from(index) * elem_size;+ let flags = ir::MemFlagsData::trusted();+ $(+ let flags = define_vm_type_alias_region_helpers!(+ @apply_attr flags, $fattr+ );+ )*+ Field::new(self.regions, VmType::$Name, offset, flags, ty)+ }+ };+// Emit the accessor `struct` and per-field methods for one `VM*` type.//- // Fields arrive pre-split into `{ $fname [ $attrs... ] [ $fty... ] }` groups- // (see `@munch` below); the terminal arm consumes those groups once the- // struct body has been fully peeled apart.+ // Fields arrive pre-split into `{ $kind $fname [ $attrs... ] [ $fty... ] }`+ // groups (see `@munch` below); the terminal arm consumes those groups once+ // the struct body has been fully peeled apart.(@emit $Name:ident $snake:ident {- $( { $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] } )*+ $( { $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] } )*}) => {#[doc = concat!("An [`AliasRegions`] accessor for the fields of a `",@@ -657,32 +727,17 @@ macro_rules! define_vm_type_alias_region_helpers {Offsets: GetPtrSize,{$(- #[doc = concat!(- "Get the [`Field`] for the `", stringify!($fname),- "` field of `", stringify!($Name), "`."- )]- pub fn $fname(self) -> Field<'a, Offsets> {- let offset = u32::from(- self.regions.offsets.get_ptr_size().$snake().$fname()- );- let flags = ir::MemFlagsData::trusted();- $(- let flags = define_vm_type_alias_region_helpers!(- @apply_attr flags, $fattr- );- )*- let ty = define_vm_type_alias_region_helpers!(- @field_ty self.regions.pointer_type, $($fty)*- );- Field::new(self.regions, VmType::$Name, offset, flags, ty)- }+ define_vm_type_alias_region_helpers!(+ @method $Name $snake $fkind $fname [ $($fattr)* ] [ $($fty)* ]+ );)*}};// Peel fields off the raw struct body one at a time, accumulating completed- // `{ $fname [ $attrs... ] [ $fty... ] }` groups plus the pending attributes- // for the field currently being parsed.+ // `{ $kind $fname [ $attrs... ] [ $fty... ] }` groups plus the pending+ // attributes for the field currently being parsed. `$kind` is `plain` or+ // `indexed`, and selects which `@method` arm emits the field's accessor.//// Splitting the body by hand (rather than matching `$fty:tt $(< $fgen:ty// >)?` within a repetition) is what lets each field's type reach the@@ -692,35 +747,43 @@ macro_rules! define_vm_type_alias_region_helpers {define_vm_type_alias_region_helpers!(@emit $Name $snake { $($groups)* });};// A field marked `#[aggregate]` is a composite (a nested struct or array)- // that is not accessed as a whole by Wasm code, but piecewise- // instead. Therefore, there is no single Cranelift type for such fields, so- // no accessors are generated for them; interior accesses are computed from- // the field's offset instead, which the generated `offsets::*` methods- // still provide.+ // so there is no single Cranelift type for such fields, so no accessors are+ // generated for them; interior accesses are computed from the field's+ // offset instead, which the generated `offsets::*` methods still provide.(@munch $Name:ident $snake:ident { $($groups:tt)* }$(#[doc = $fdoc:literal])* #[aggregate] $fvis:vis $fname:ident : $fty:ty , $($rest:tt)*) => {define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* } $($rest)*);};+ // A field marked `#[indexed]` is a fixed-size array of a scalar type whose+ // elements are accessed individually by a compile-time constant index. It+ // has no single Cranelift type either, but its elements do, so it gets an+ // index-taking accessor.+ (@munch $Name:ident $snake:ident { $($groups:tt)* }+ $(#[doc = $fdoc:literal])* #[indexed] $fvis:vis $fname:ident : $($rest:tt)*+ ) => {+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } indexed $fname [] [] $($rest)*);+ };// Consume one field's attributes, visibility, and name, then collect its// type tokens.(@munch $Name:ident $snake:ident { $($groups:tt)* }$(# $fattr:tt)* $fvis:vis $fname:ident : $($rest:tt)*) => {- define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($fattr)* ] [] $($rest)*);+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } plain $fname [ $($fattr)* ] [] $($rest)*);};// Accumulate one field's type tokens up to its terminating comma, then// append the completed group.- (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {- define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fname [ $($fattr)* ] [ $($fty)* ] } } $($rest)*);+ (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {+ define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fkind $fname [ $($fattr)* ] [ $($fty)* ] } } $($rest)*);};- (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {- define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($fattr)* ] [ $($fty)* $tok ] $($rest)*);+ (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fkind $fname [ $($fattr)* ] [ $($fty)* $tok ] $($rest)*);};// Top-level entry: the list of `VM*` type definitions.( $($(#[doc = $sdoc:literal])*+ $(#[cfg($($scfg:tt)*)])?$(#[derive($($d:ident),*)])?#[repr($($repr:tt)*)]#[snake_name = $snake:ident]@@ -760,6 +823,11 @@ wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers);/// accessing that `VM*` type through a pointer would. For the few aggregates that/// have no `VM*` type of their own, and hence no alias region of their own, there/// are hand-written helpers below.+///+/// A field marked `#[pointee(..)]` additionally gets an accessor for what it+/// points *at*, which lives outside the vmctx. For example,+/// `alias_regions.vmctx().builtin_functions_array(i)` is the `i`th builtin in+/// the array that the `VMContext::builtin_functions` field points at.#[allow(unused_macro_rules,reason = "entry shapes and marker attributes are handled uniformly for both \@@ -770,6 +838,9 @@ macro_rules! define_vmctx_alias_region_helpers {// pointer type as an `ir::Type`).(@field_ty $pt:expr, u32) => { ir::types::I32 };(@field_ty $pt:expr, VmPtr < $g:ident >) => { $pt };+ (@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };+ (@field_ty $pt:expr, AtomicU64) => { ir::types::I64 };+ (@field_ty $pt:expr, unsafe extern "C" fn) => { $pt };// Determine the Cranelift type a field is *accessed* as: the classification// of its `#[access_as = T]` type if it has one, and of its declared type@@ -789,6 +860,7 @@ macro_rules! define_vmctx_alias_region_helpers {(@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() };(@apply_attr $flags:expr, [access_as = $($t:tt)*]) => { $flags };(@apply_attr $flags:expr, [ptr_size_offset]) => { $flags };+ (@apply_attr $flags:expr, [pointee( $($t:tt)* )]) => { $flags };// Compute a field's access flags and Cranelift type from its declared type// and marker attributes, and build the `Field` for it at `$offset`.@@ -803,6 +875,55 @@ macro_rules! define_vmctx_alias_region_helpers {Field::new(this.regions, VmType::$Name, offset, flags, ty)}};+ // Scan a field's attributes for a `#[pointee(..)]` and emit the accessor it+ // describes, if any.+ (@pointee_methods $Offsets:tt [ ]) => {};+ (@pointee_methods $Offsets:tt [ #[pointee( $($p:tt)* )] $($rest:tt)* ]) => {+ define_vmctx_alias_region_helpers!(@pointee $Offsets $($p)*);+ };+ (@pointee_methods $Offsets:tt [ # $skip:tt $($rest:tt)* ]) => {+ define_vmctx_alias_region_helpers!(@pointee_methods $Offsets [ $($rest)* ]);+ };++ // An array of pointees.+ (@pointee [ $($Offsets:tt)* ]+ $(# $pattr:tt)* $Region:ident as $pname:ident [ $Index:ident ] : $($pty:tt)*+ ) => {+ #[doc = concat!(+ "Get the [`Field`] for the `index`th element of the out-of-line `",+ stringify!($Region), "` array.\n\nThe returned [`Field`] is relative \+ to the array's base pointer, not to the vmctx."+ )]+ pub fn $pname(self, index: $Index) -> Field<'a, $($Offsets)*> {+ let ty = define_vmctx_alias_region_helpers!(+ @field_ty self.regions.pointer_type, $($pty)*+ );+ let offset = index.vmctx_array_index().checked_mul(ty.bytes()).unwrap();+ let flags = ir::MemFlagsData::trusted();+ $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $pattr); )*+ Field::new(self.regions, VmType::$Region, offset, flags, ty)+ }+ };++ // A single pointee.+ (@pointee [ $($Offsets:tt)* ]+ $(# $pattr:tt)* $Region:ident as $pname:ident : $($pty:tt)*+ ) => {+ #[doc = concat!(+ "Get the [`Field`] for the out-of-line `", stringify!($Region),+ "`.\n\nThe returned [`Field`] is relative to the pointer to it, not \+ to the vmctx."+ )]+ pub fn $pname(self) -> Field<'a, $($Offsets)*> {+ let ty = define_vmctx_alias_region_helpers!(+ @field_ty self.regions.pointer_type, $($pty)*+ );+ let flags = ir::MemFlagsData::trusted();+ $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $pattr); )*+ Field::new(self.regions, VmType::$Region, 0, flags, ty)+ }+ };+// ### `static` Section Entries(@static_entry $Name:ident $snake:ident align { $al:tt }) => {};@@ -823,6 +944,8 @@ macro_rules! define_vmctx_alias_region_helpers {@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ])}++ define_vmctx_alias_region_helpers!(@pointee_methods [ Offsets ] [ $(# $fattr)* ]);};// ### `dynamic` Section Entries Marked `#[ptr_size_offset]`@@ -891,6 +1014,10 @@ macro_rules! define_vmctx_alias_region_helpers {@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ])}++ define_vmctx_alias_region_helpers!(+ @pointee_methods [ $($Offsets)* ] [ $(# $fattr)* ]+ );};// A single field, whether unconditionally present (`field`) or only@@ -910,6 +1037,10 @@ macro_rules! define_vmctx_alias_region_helpers {@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ])}++ define_vmctx_alias_region_helpers!(+ @pointee_methods [ $($Offsets)* ] [ $(# $fattr)* ]+ );};// Emit the `impl` block holding the accessors for the dynamically-positioned@@ -1099,40 +1230,6 @@ where)}- fn vmstore_context_load(- &mut self,- cursor: &mut FuncCursor<'_>,- ty: ir::Type,- base_flags: ir::MemFlagsData,- vmstore_ctx: ir::Value,- offset: u32,- ) -> ir::Value {- let region = self.vmstore_context_region(cursor.func, offset);- cursor.ins().load(- ty,- base_flags.with_alias_region(Some(region)),- vmstore_ctx,- i32::try_from(offset).unwrap(),- )- }-- fn vmstore_context_store(- &mut self,- cursor: &mut FuncCursor<'_>,- base_flags: ir::MemFlagsData,- vmstore_ctx: ir::Value,- offset: u32,- val: ir::Value,- ) {- let region = self.vmstore_context_region(cursor.func, offset);- cursor.ins().store(- base_flags.with_alias_region(Some(region)),- val,- vmstore_ctx,- i32::try_from(offset).unwrap(),- );- }-/// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).////// The caller supplies the base flags because whether the base pointer is@@ -1208,297 +1305,13 @@ wherelet offset = self.offsets.get_ptr_size().vm_store_context().stack_chain();self.vmstore_context_region(func, offset.into())}-- /// Load a `VMStoreContext` component-context slot.- ///- /// The slot is indexed by a compile-time constant, so the alias region is- /// keyed on the precise per-slot offset.- pub fn vmstore_context_component_context_slot(- &mut self,- cursor: &mut FuncCursor<'_>,- ty: ir::Type,- vmstore_ctx: ir::Value,- slot: u8,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- ty,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .component_context_slot(slot)- .into(),- )- }-- /// Store a `VMStoreContext` component-context slot.- ///- /// The slot is indexed by a compile-time constant, so the alias region is- /// keyed on the precise per-slot offset.- pub fn store_vmstore_context_component_context_slot(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- slot: u8,- val: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .component_context_slot(slot)- .into(),- val,- )- }-}--/// `VMDeferredThread`-related methods.-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- fn vmdeferred_thread_region(- &mut self,- func: &mut ir::Function,- offset: u32,- ) -> ir::AliasRegion {- self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMDeferredThread,- offset,- },- )- }-- fn vmdeferred_thread_load(- &mut self,- cursor: &mut FuncCursor<'_>,- ty: ir::Type,- base_flags: ir::MemFlagsData,- vmdeferred_thread_ptr: ir::Value,- offset: u32,- ) -> ir::Value {- let region = self.vmdeferred_thread_region(cursor.func, offset);- cursor.ins().load(- ty,- base_flags.with_alias_region(Some(region)),- vmdeferred_thread_ptr,- i32::try_from(offset).unwrap(),- )- }-- fn vmdeferred_thread_store(- &mut self,- cursor: &mut FuncCursor<'_>,- base_flags: ir::MemFlagsData,- vmdeferred_thread_ptr: ir::Value,- offset: u32,- val: ir::Value,- ) {- let region = self.vmdeferred_thread_region(cursor.func, offset);- cursor.ins().store(- base_flags.with_alias_region(Some(region)),- val,- vmdeferred_thread_ptr,- i32::try_from(offset).unwrap(),- );- }-- /// Load `VMDeferredThread::saved_context[i]` (a saved `context.{get,set}`- /// slot).- pub fn vmdeferred_thread_saved_context(- &mut self,- cursor: &mut FuncCursor<'_>,- vmdeferred_thread_ptr: ir::Value,- i: u8,- ) -> ir::Value {- self.vmdeferred_thread_load(- cursor,- ir::types::I32,- ir::MemFlagsData::trusted(),- vmdeferred_thread_ptr,- self.offsets- .get_ptr_size()- .vm_deferred_thread()- .saved_context_slot(i)- .into(),- )- }-- /// Store `VMDeferredThread::saved_context[i]`.- pub fn store_vmdeferred_thread_saved_context(- &mut self,- cursor: &mut FuncCursor<'_>,- vmdeferred_thread_ptr: ir::Value,- i: u8,- val: ir::Value,- ) {- self.vmdeferred_thread_store(- cursor,- ir::MemFlagsData::trusted(),- vmdeferred_thread_ptr,- self.offsets- .get_ptr_size()- .vm_deferred_thread()- .saved_context_slot(i)- .into(),- val,- )- }-}--/// `[VMSharedTypeIndex]`-related methods.-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- /// Load a `VMSharedTypeIndex` element out of the `[VMSharedTypeIndex]`- /// array pointed at by the `VMContext::type_ids_array` field.- pub fn type_ids_array_element(- &mut self,- cursor: &mut FuncCursor<'_>,- array: ir::Value,- ty: ModuleInternedTypeIndex,- ) -> ir::Value {- let load_ty = ir::Type::int_with_byte_size(- self.offsets- .get_ptr_size()- .size_of_vmshared_type_index()- .into(),- )- .unwrap();-- let offset = ty.as_u32().checked_mul(load_ty.bytes()).unwrap();- let region = self.region(- cursor.func,- AliasRegionKey::Vm {- ty: VmType::TypeIdsArray,- offset,- },- );-- cursor.ins().load(- load_ty,- ir::MemFlagsData::trusted()- .with_readonly()- .with_can_move()- .with_alias_region(Some(region)),- array,- i32::try_from(offset).unwrap(),- )- }-}--/// Epoch counter-related methods.-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- /// Dereference the epoch pointer (an `*const AtomicU64` previously loaded- /// out of the vmctx's `epoch_ptr` field) to read the current epoch counter.- pub fn epoch_counter(- &mut self,- cursor: &mut FuncCursor<'_>,- epoch_ptr: ir::Value,- ) -> ir::Value {- let region = self.region(- cursor.func,- AliasRegionKey::Vm {- ty: VmType::EpochCounter,- offset: 0,- },- );- cursor.ins().load(- ir::types::I64,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- epoch_ptr,- 0,- )- }}-/// Builtin-functions array methods.+/// Component-specific methods.impl<Offsets> AliasRegions<Offsets>whereOffsets: GetPtrSize,{- /// Load a host function pointer element out of the builtin-functions array- /// (`VMContext::builtin_functions`)- pub fn builtin_functions_array_element(- &mut self,- cursor: &mut FuncCursor<'_>,- array: ir::Value,- builtin: BuiltinFunctionIndex,- ) -> ir::Value {- let offset = builtin- .index()- .checked_mul(self.pointer_type.bytes())- .unwrap();-- let region = self.region(- cursor.func,- AliasRegionKey::Vm {- ty: VmType::BuiltinFunctionsArray,- offset,- },- );-- cursor.ins().load(- self.pointer_type,- ir::MemFlagsData::trusted()- .with_readonly()- .with_can_move()- .with_alias_region(Some(region)),- array,- i32::try_from(offset).unwrap(),- )- }-}--/// Component builtin-functions array methods.-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- /// Load a host function pointer element out of the component- /// builtin-functions array (`VMComponentContext::builtins`).- pub fn component_builtin_functions_array_element(- &mut self,- cursor: &mut FuncCursor<'_>,- array: ir::Value,- builtin: ComponentBuiltinFunctionIndex,- ) -> ir::Value {- let offset = builtin- .index()- .checked_mul(self.pointer_type.bytes())- .unwrap();-- let region = self.region(- cursor.func,- AliasRegionKey::Vm {- ty: VmType::ComponentBuiltinFunctionsArray,- offset,- },- );-- cursor.ins().load(- self.pointer_type,- ir::MemFlagsData::trusted()- .with_readonly()- .with_can_move()- .with_alias_region(Some(region)),- array,- i32::try_from(offset).unwrap(),- )- }-/// Get the alias region for the `ValRaw` array used to marshal arguments/// and results across the array calling convention used by various/// trampolines.@@ -1609,236 +1422,6 @@ impl AliasRegions<VMComponentOffsets<u8>> {}}-/// Methods for the collectors' private heap-data structs.-///-/// Each struct is a separate allocation reached through a `*mut _` stored in the-/// `VMContext`. Their fields are *not* GC heap locations, so they are tagged with-/// the owning struct's own region (keyed on the field offset within the struct)-/// rather than the `GcHeap` region. These helpers emit the field load/store.-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- fn vmdrc_heap_data_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {- self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMDrcHeapData,- offset,- },- )- }-- /// Emit a load of the DRC over-approximated-stack-roots list head, given the- /// a `*mut VMDrcHeapData`.- pub fn vmdrc_heap_data_over_approximated_stack_roots(- &mut self,- cursor: &mut FuncCursor<'_>,- drc_heap_data: ir::Value,- ) -> ir::Value {- let offset = self- .offsets- .get_ptr_size()- .vmdrc_heap_data_over_approximated_stack_roots();- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());- cursor.ins().load(- ir::types::I32,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- drc_heap_data,- i32::from(offset),- )- }-- /// Emit a store to the DRC over-approximated-stack-roots list head.- pub fn store_vmdrc_heap_data_over_approximated_stack_roots(- &mut self,- cursor: &mut FuncCursor<'_>,- drc_heap_data: ir::Value,- val: ir::Value,- ) {- let offset = self- .offsets- .get_ptr_size()- .vmdrc_heap_data_over_approximated_stack_roots();- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());- cursor.ins().store(- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- val,- drc_heap_data,- i32::from(offset),- );- }-- /// Emit a load of the current over-approximated-stack-roots list length.- pub fn vmdrc_heap_data_current_over_approximated_stack_roots_len(- &mut self,- cursor: &mut FuncCursor<'_>,- drc_heap_data: ir::Value,- ) -> ir::Value {- let offset = self- .offsets- .get_ptr_size()- .vmdrc_heap_data_current_over_approximated_stack_roots_len();- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());- cursor.ins().load(- ir::types::I32,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- drc_heap_data,- i32::from(offset),- )- }-- /// Emit a store to the current over-approximated-stack-roots list length.- pub fn store_vmdrc_heap_data_current_over_approximated_stack_roots_len(- &mut self,- cursor: &mut FuncCursor<'_>,- drc_heap_data: ir::Value,- len: ir::Value,- ) {- let offset = self- .offsets- .get_ptr_size()- .vmdrc_heap_data_current_over_approximated_stack_roots_len();- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());- cursor.ins().store(- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- len,- drc_heap_data,- i32::from(offset),- );- }-- /// Emit a load of the over-approximated-stack-roots list length after the- /// last GC.- pub fn vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(- &mut self,- cursor: &mut FuncCursor<'_>,- drc_heap_data: ir::Value,- ) -> ir::Value {- let offset = self- .offsets- .get_ptr_size()- .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc();- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());- cursor.ins().load(- ir::types::I32,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- drc_heap_data,- i32::from(offset),- )- }-- fn vmcopying_heap_data_region(- &mut self,- func: &mut ir::Function,- offset: u32,- ) -> ir::AliasRegion {- self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMCopyingHeapData,- offset,- },- )- }-- /// Emit a load of the copying collector's bump pointer.- pub fn vmcopying_heap_data_bump_ptr(- &mut self,- cursor: &mut FuncCursor<'_>,- copying_heap_data: ir::Value,- ) -> ir::Value {- let offset = self.offsets.get_ptr_size().vmcopying_heap_data_bump_ptr();- let region = self.vmcopying_heap_data_region(cursor.func, offset.into());- cursor.ins().load(- ir::types::I32,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- copying_heap_data,- i32::from(offset),- )- }-- /// Emit a store to the copying collector's bump pointer.- pub fn store_vmcopying_heap_data_bump_ptr(- &mut self,- cursor: &mut FuncCursor<'_>,- copying_heap_data: ir::Value,- val: ir::Value,- ) {- let offset = self.offsets.get_ptr_size().vmcopying_heap_data_bump_ptr();- let region = self.vmcopying_heap_data_region(cursor.func, offset.into());- cursor.ins().store(- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- val,- copying_heap_data,- i32::from(offset),- );- }-- /// Emit a load of the copying collector's active-space-end pointer.- pub fn vmcopying_heap_data_active_space_end(- &mut self,- cursor: &mut FuncCursor<'_>,- copying_heap_data: ir::Value,- ) -> ir::Value {- let offset = self- .offsets- .get_ptr_size()- .vmcopying_heap_data_active_space_end();- let region = self.vmcopying_heap_data_region(cursor.func, offset.into());- cursor.ins().load(- ir::types::I32,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- copying_heap_data,- i32::from(offset),- )- }-- /// Emit a load of the null collector's bump finger (the first and only field- /// of its heap data, at offset 0).- pub fn vmnull_heap_data_bump_finger(- &mut self,- cursor: &mut FuncCursor<'_>,- null_collector_heap_data: ir::Value,- ) -> ir::Value {- let region = self.region(- cursor.func,- AliasRegionKey::Vm {- ty: VmType::VMNullHeapData,- offset: 0,- },- );- cursor.ins().load(- ir::types::I32,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- null_collector_heap_data,- 0,- )- }-- /// Emit a store to the null collector's bump finger.- pub fn store_vmnull_heap_data_bump_finger(- &mut self,- cursor: &mut FuncCursor<'_>,- null_collector_heap_data: ir::Value,- val: ir::Value,- ) {- let region = self.region(- cursor.func,- AliasRegionKey::Vm {- ty: VmType::VMNullHeapData,- offset: 0,- },- );- cursor.ins().store(- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- val,- null_collector_heap_data,- 0,- );- }-}-/// Stack-switching and continuation-object methods.impl<Offsets> AliasRegions<Offsets>wherediff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex 4665fcd1eb..ed95a74f24 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -1388,11 +1388,10 @@ impl Compiler {.load(&mut builder.cursor(), vmctx);// ... and then load the entry in the array that corresponds to this// builtin.- let func_addr = alias_regions.builtin_functions_array_element(- &mut builder.cursor(),- array_addr,- builtin,- );+ let func_addr = alias_regions+ .vmctx()+ .builtin_functions_array(builtin)+ .load(&mut builder.cursor(), array_addr);let sig = builder.func.import_signature(sig);self.call_indirect_host(builder, builtin, sig, func_addr, args)diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex b32ae9a203..fa48ca3fba 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -1398,11 +1398,9 @@ impl<'a> TrampolineCompiler<'a> {.load(&mut self.builder.cursor(), vmctx);// Next load the function pointer at `offset` and return that.self.alias_regions- .component_builtin_functions_array_element(- &mut self.builder.cursor(),- builtins_array,- index,- )+ .vmcomponent()+ .builtins_array(index)+ .load(&mut self.builder.cursor(), builtins_array)}/// Get a function's parameters regardless of the ABI in use.@@ -2356,16 +2354,6 @@ whereintrinsic: UnsafeIntrinsic,params: &[ir::Value],) -> Option<ir::Value> {- // This is the width of the type being loaded from Wasmtime's- // `VMStoreContext` slot and it depends on the intrinsic.- let ty = match intrinsic {- UnsafeIntrinsic::ContextGetI32_0- | UnsafeIntrinsic::ContextSetI32_0- | UnsafeIntrinsic::ContextGetI32_1- | UnsafeIntrinsic::ContextSetI32_1 => ir::types::I32,- _ => unreachable!(),- };-let slot = match intrinsic {UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextSetI32_0 => 0,UnsafeIntrinsic::ContextGetI32_1 | UnsafeIntrinsic::ContextSetI32_1 => 1,@@ -2377,24 +2365,18 @@ wherelet context = self.traps.alias_regions()- .vmstore_context_component_context_slot(- &mut self.builder.cursor(),- ty,- vmstore_context,- slot,- );+ .vm_store_context()+ .component_context(slot)+ .load(&mut self.builder.cursor(), vmstore_context);Some(context)}UnsafeIntrinsic::ContextSetI32_0 | UnsafeIntrinsic::ContextSetI32_1 => {let new_context = params[2];self.traps.alias_regions()- .store_vmstore_context_component_context_slot(- &mut self.builder.cursor(),- vmstore_context,- slot,- new_context,- );+ .vm_store_context()+ .component_context(slot)+ .store(&mut self.builder.cursor(), vmstore_context, new_context);None}_ => unreachable!(),diff --git a/crates/cranelift/src/component_sync_call.rs b/crates/cranelift/src/component_sync_call.rsindex 831603afcf..876bb4e170 100644--- a/crates/cranelift/src/component_sync_call.rs+++ b/crates/cranelift/src/component_sync_call.rs@@ -98,23 +98,19 @@ where// Save the caller's context slots into the frame and reset the live values// to 0 for the freshly-entered (deferred) thread.for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {- let saved = alias_regions.vmstore_context_component_context_slot(- &mut builder.cursor(),- ir::types::I32,- vmstore,- i,- );- alias_regions.store_vmdeferred_thread_saved_context(+ let saved = alias_regions+ .vm_store_context()+ .component_context(i)+ .load(&mut builder.cursor(), vmstore);+ alias_regions.vm_deferred_thread().saved_context(i).store(&mut builder.cursor(),slot_addr,- i,saved,);let zero = builder.ins().iconst(ir::types::I32, 0);- alias_regions.store_vmstore_context_component_context_slot(+ alias_regions.vm_store_context().component_context(i).store(&mut builder.cursor(),vmstore,- i,zero,);}@@ -195,12 +191,13 @@ where.current_thread().store(&mut builder.cursor(), vmstore, parent);for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {- let saved =- alias_regions.vmdeferred_thread_saved_context(&mut builder.cursor(), slot_addr, i);- alias_regions.store_vmstore_context_component_context_slot(+ let saved = alias_regions+ .vm_deferred_thread()+ .saved_context(i)+ .load(&mut builder.cursor(), slot_addr);+ alias_regions.vm_store_context().component_context(i).store(&mut builder.cursor(),vmstore,- i,saved,);}diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex a617dc06c1..3445f11c9c 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -804,7 +804,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {fn epoch_load_current(&mut self, builder: &mut FunctionBuilder<'_>) -> ir::Value {let addr = builder.use_var(self.epoch_ptr_var);self.alias_regions- .epoch_counter(&mut builder.cursor(), addr)+ .vmctx()+ .epoch_counter()+ .load(&mut builder.cursor(), addr)}fn epoch_check(&mut self, builder: &mut FunctionBuilder<'_>) {@@ -1162,12 +1164,12 @@ impl<'module_environment> FuncEnvironment<'module_environment> {// Load the base pointer of the array of `VMSharedTypeIndex`es.let shared_indices = self.alias_regions.vmctx().type_ids().load(pos, vmctx);- // Calculate the offset in that array for this type's entry.-// Load the`VMSharedTypeIndex` that this `ModuleInternedTypeIndex` is// associated with at runtime from the array.self.alias_regions- .type_ids_array_element(pos, shared_indices, interned_ty)+ .vmctx()+ .type_ids_array(interned_ty)+ .load(pos, shared_indices)}/// Does this function need a GC heap?diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rsindex 7f3a56541e..7cf5628d0b 100644--- a/crates/cranelift/src/func_environ/gc/copying.rs+++ b/crates/cranelift/src/func_environ/gc/copying.rs@@ -50,10 +50,14 @@ impl CopyingCompiler {) -> (ir::Value, ir::Value) {let bump_ptr = func_env.alias_regions- .vmcopying_heap_data_bump_ptr(&mut builder.cursor(), ptr_to_heap_data);+ .vm_copying_heap_data()+ .bump_ptr()+ .load(&mut builder.cursor(), ptr_to_heap_data);let active_space_end = func_env.alias_regions- .vmcopying_heap_data_active_space_end(&mut builder.cursor(), ptr_to_heap_data);+ .vm_copying_heap_data()+ .active_space_end()+ .load(&mut builder.cursor(), ptr_to_heap_data);(bump_ptr, active_space_end)}@@ -64,11 +68,11 @@ impl CopyingCompiler {heap_data_ptr: ir::Value,end_of_object: ir::Value,) {- func_env.alias_regions.store_vmcopying_heap_data_bump_ptr(- &mut builder.cursor(),- heap_data_ptr,- end_of_object,- );+ func_env+ .alias_regions+ .vm_copying_heap_data()+ .bump_ptr()+ .store(&mut builder.cursor(), heap_data_ptr, end_of_object);}/// Round `size` (an `i32`) up to `ALIGN`, returning the result as an `i64`.diff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rsindex da3d205965..50a453fd1f 100644--- a/crates/cranelift/src/func_environ/gc/drc.rs+++ b/crates/cranelift/src/func_environ/gc/drc.rs@@ -113,7 +113,9 @@ impl DrcCompiler {// element.let next = func_env.alias_regions- .vmdrc_heap_data_over_approximated_stack_roots(&mut builder.cursor(), heap_data);+ .vm_drc_heap_data()+ .over_approximated_stack_roots()+ .load(&mut builder.cursor(), heap_data);// Update our object's header to point to `next` and consider itself part of the list.self.set_next_over_approximated_stack_root(func_env, builder, gc_ref, next);@@ -125,11 +127,9 @@ impl DrcCompiler {// Commit this object as the new head of the list.func_env.alias_regions- .store_vmdrc_heap_data_over_approximated_stack_roots(- &mut builder.cursor(),- heap_data,- gc_ref,- );+ .vm_drc_heap_data()+ .over_approximated_stack_roots()+ .store(&mut builder.cursor(), heap_data, gc_ref);// Increment the list's length.//@@ -140,19 +140,16 @@ impl DrcCompiler {// `u32::MAX`.let current_len = func_env.alias_regions- .vmdrc_heap_data_current_over_approximated_stack_roots_len(- &mut builder.cursor(),- heap_data,- );+ .vm_drc_heap_data()+ .current_over_approximated_stack_roots_len()+ .load(&mut builder.cursor(), heap_data);let one = builder.ins().iconst(ir::types::I32, 1);let new_current_len = builder.ins().iadd(current_len, one);func_env.alias_regions- .store_vmdrc_heap_data_current_over_approximated_stack_roots_len(- &mut builder.cursor(),- heap_data,- new_current_len,- );+ .vm_drc_heap_data()+ .current_over_approximated_stack_roots_len()+ .store(&mut builder.cursor(), heap_data, new_current_len);}/// Trigger a GC when the over-approximated-stack-roots list has doubled@@ -178,16 +175,14 @@ impl DrcCompiler {.load(&mut builder.cursor(), vmctx);let current_len = func_env.alias_regions- .vmdrc_heap_data_current_over_approximated_stack_roots_len(- &mut builder.cursor(),- heap_data,- );+ .vm_drc_heap_data()+ .current_over_approximated_stack_roots_len()+ .load(&mut builder.cursor(), heap_data);let last_len = func_env.alias_regions- .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(- &mut builder.cursor(),- heap_data,- );+ .vm_drc_heap_data()+ .over_approximated_stack_roots_len_after_last_gc()+ .load(&mut builder.cursor(), heap_data);let doubled_last_len = builder.ins().iadd(last_len, last_len);let min_threshold = builder.ins().iconst(diff --git a/crates/cranelift/src/func_environ/gc/null.rs b/crates/cranelift/src/func_environ/gc/null.rsindex 454166c278..479a9b6fd0 100644--- a/crates/cranelift/src/func_environ/gc/null.rs+++ b/crates/cranelift/src/func_environ/gc/null.rs@@ -77,7 +77,9 @@ impl NullCompiler {.load(&mut builder.cursor(), vmctx);let next = func_env.alias_regions- .vmnull_heap_data_bump_finger(&mut builder.cursor(), ptr_to_next);+ .vm_null_heap_data()+ .next()+ .load(&mut builder.cursor(), ptr_to_next);// Increment the bump "pointer" to the requested alignment://@@ -168,7 +170,7 @@ impl NullCompiler {ptr_to_object,i32::try_from(wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET).unwrap(),);- func_env.alias_regions.store_vmnull_heap_data_bump_finger(+ func_env.alias_regions.vm_null_heap_data().next().store(&mut builder.cursor(),ptr_to_next,end_of_object,diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rsindex 6da4e3b2cd..d59f3c16af 100644--- a/crates/environ/src/builtin.rs+++ b/crates/environ/src/builtin.rs@@ -212,6 +212,15 @@ macro_rules! declare_builtin_index {$for_each_builtin!(declare_builtin_index_constructors);}+ // Builtins are reached by indexing the array of function pointers that+ // a vmctx points at.+ impl $crate::VmctxArrayIndex for $index_name {+ #[inline]+ fn vmctx_array_index(self) -> u32 {+ self.index()+ }+ }+#[cfg(test)]impl arbitrary::Arbitrary<'_> for $index_name {fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<Self> {diff --git a/crates/environ/src/vmctxtypes.rs b/crates/environ/src/vmctxtypes.rsindex 25a65119c7..36fac62515 100644--- a/crates/environ/src/vmctxtypes.rs+++ b/crates/environ/src/vmctxtypes.rs@@ -91,6 +91,17 @@/// on the vmctx's shape. These fields get generated accessors that do not/// need to be parameterized over `VMOffsets`, only `GetPtrSize`.///+/// * `#[pointee(<attrs> <Region> as <name> $([<IndexType>])? : <ty>)]`: this+/// field is a pointer to memory that lives *outside* the vmctx and so is not+/// part of its layout at all, but that compiled code reaches only by loading+/// this field first. `<Region>` names the alias region that memory belongs+/// to, `<name>` is the accessor generated for it, and `<ty>` is the type it is+/// accessed as. With an `[<IndexType>]`, the pointee is an array of `<ty>`+/// indexed by `<IndexType>` and each element gets its own alias region;+/// without one, it is a single `<ty>`. The pointee's `<attrs>` are its own:+/// how compiled code may treat a load of the pointee is independent of how it+/// may treat a load of the pointer.+////// Doc comments are deliberately *not* accepted on fields; use `//` comments for/// prose about the layout. Accessor documentation is synthesized from the field/// names instead, so that there is only one place a field can be described.@@ -121,15 +132,41 @@ macro_rules! for_each_vmctx_type {field { #[readonly] #[can_move] store_context: VmPtr<VMStoreContext> }- field { #[readonly] #[can_move] builtin_functions: VmPtr<VMBuiltinFunctionsArray> }+ // NB: `VMBuiltinFunctionsArray` is a `repr(C)` struct of+ // `unsafe extern "C" fn` fields rather than a true array,+ // so its elements are pointer-width and pointer-aligned.+ field {+ #[readonly]+ #[can_move]+ #[pointee(+ #[readonly]+ #[can_move]+ BuiltinFunctionsArray as builtin_functions_array[BuiltinFunctionIndex]:+ unsafe extern "C" fn+ )]+ builtin_functions: VmPtr<VMBuiltinFunctionsArray>+ }- field { epoch_ptr: VmPtr<AtomicU64> }+ field {+ #[pointee(EpochCounter as epoch_counter: AtomicU64)]+ epoch_ptr: VmPtr<AtomicU64>+ }// A pointer that different collectors use however they see// fit.field { #[readonly] #[can_move] gc_heap_data: VmPtr<u8> }- field { #[readonly] #[can_move] type_ids: VmPtr<VMSharedTypeIndex> }+ field {+ #[readonly]+ #[can_move]+ #[pointee(+ #[readonly]+ #[can_move]+ TypeIdsArray as type_ids_array[ModuleInternedTypeIndex]:+ VMSharedTypeIndex+ )]+ type_ids: VmPtr<VMSharedTypeIndex>+ }}// Variable-width fields come after the fixed-width fields@@ -222,7 +259,16 @@ macro_rules! for_each_vmctx_type {align { ptr }- field { #[readonly] builtins: VmPtr<VMComponentBuiltins> }+ field {+ #[readonly]+ #[pointee(+ #[readonly]+ #[can_move]+ ComponentBuiltinFunctionsArray as builtins_array[ComponentBuiltinFunctionIndex]:+ unsafe extern "C" fn+ )]+ builtins: VmPtr<VMComponentBuiltins>+ }field { #[readonly] #[can_move] store_context: VmPtr<VMStoreContext> }}diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex 3293e9745d..9c877d9526 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -8,8 +8,8 @@use crate::{DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex,- FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex,- TagIndex,+ FuncRefIndex, GlobalIndex, MemoryIndex, Module, ModuleInternedTypeIndex, OwnedMemoryIndex,+ RuntimeDataIndex, TableIndex, TagIndex,};/// Number of slots in for `component_context` in the `VMStoreContext`. This is@@ -51,6 +51,10 @@ macro_rules! define_vm_type_offsets {(@size ($p:expr) i64) => { 8u32 };(@size ($p:expr) u64) => { 8u32 };(@size ($p:expr) u32) => { 4u32 };+ // `VMGcRef` is a `NonZeroU32` newtype, so `Option<VMGcRef>` is niche-packed+ // back down into four bytes.+ (@size ($p:expr) NonZeroU32) => { 4u32 };+ (@size ($p:expr) Option < VMGcRef >) => { 4u32 };(@size ($p:expr) [u8; 16]) => { 16u32 };(@size ($p:expr) [u32; $n:expr]) => { 4u32 * u32::try_from($n).unwrap() };(@size ($p:expr) VMSharedTypeIndex) => { u32::from(($p).size_of_vmshared_type_index()) };@@ -85,6 +89,8 @@ macro_rules! define_vm_type_offsets {(@align ($p:expr) i64) => { 8u32 };(@align ($p:expr) u64) => { 8u32 };(@align ($p:expr) u32) => { 4u32 };+ (@align ($p:expr) NonZeroU32) => { 4u32 };+ (@align ($p:expr) Option < VMGcRef >) => { 4u32 };(@align ($p:expr) [u8; 16]) => { 16u32 };(@align ($p:expr) [u32; $n:expr]) => { 4u32 };(@align ($p:expr) VMSharedTypeIndex) => { u32::from(($p).align_of_vmshared_type_index()) };@@ -205,6 +211,7 @@ macro_rules! define_vm_type_offsets {// Top-level entry: the list of `VM*` type definitions.( $($(#[doc = $sdoc:literal])*+ $(#[cfg($($scfg:tt)*)])?$(#[derive($($d:ident),*)])?#[repr($($repr:tt)*)]#[snake_name = $snake:ident]@@ -358,11 +365,6 @@ pub mod offsets {for_each_vmctx_type!(define_vmctx_static_offsets);}-/// The size, in bytes, of one `context.{get,set}` slot. These slots are `u32`s,-/// both in `VMStoreContext::component_context` and in-/// `VMDeferredThread::saved_context`.-const COMPONENT_CONTEXT_SLOT_SIZE: u8 = 4;-/// Offsets within a `VMStoreContext` that are not simply the offset of one of/// its fields, and so are not generated by `for_each_vm_type!`.impl<P: PtrSize> offsets::VMStoreContext<P> {@@ -380,22 +382,6 @@ impl<P: PtrSize> offsets::VMStoreContext<P> {debug_assert!(offset < self.last_wasm_exit_trampoline_fp());offset}-- /// The offset of the `component_context[i]` slot within a `VMStoreContext`.- pub fn component_context_slot(&self, i: u8) -> u8 {- assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);- self.component_context() + i * COMPONENT_CONTEXT_SLOT_SIZE- }-}--/// Offsets within a `VMDeferredThread` that are not simply the offset of one of-/// its fields, and so are not generated by `for_each_vm_type!`.-impl<P: PtrSize> offsets::VMDeferredThread<P> {- /// The offset of the `saved_context[i]` slot within a `VMDeferredThread`.- pub fn saved_context_slot(&self, i: u8) -> u8 {- assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);- self.saved_context() + i * COMPONENT_CONTEXT_SLOT_SIZE- }}/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to `PtrSize` for@@ -403,6 +389,7 @@ impl<P: PtrSize> offsets::VMDeferredThread<P> {macro_rules! define_ptr_size_vm_type_accessors {( $($(#[doc = $sdoc:literal])*+ $(#[cfg($($scfg:tt)*)])?$(#[derive($($d:ident),*)])?#[repr($($repr:tt)*)]#[snake_name = $snake:ident]@@ -688,65 +675,6 @@ pub trait PtrSize {fn vmcontref_values(&self) -> u8 {self.vmcontref_args() + self.size_of_vmhostarray()}-- /// Return the offset of the `over_approximated_stack_roots` field within- /// `VMDrcHeapData`.- #[inline]- fn vmdrc_heap_data_over_approximated_stack_roots(&self) -> u8 {- 0- }-- /// Return the offset of the `current_over_approximated_stack_roots_len`- /// field within `VMDrcHeapData`.- #[inline]- fn vmdrc_heap_data_current_over_approximated_stack_roots_len(&self) -> u8 {- 4- }-- /// Return the offset of the- /// `over_approximated_stack_roots_len_after_last_gc` field within- /// `VMDrcHeapData`.- #[inline]- fn vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(&self) -> u8 {- 8- }-- /// Return the size of `VMDrcHeapData`.- #[inline]- fn size_of_vmdrc_heap_data(&self) -> u8 {- 12- }-- /// Return the alignment of `VMDrcHeapData`.- #[inline]- fn align_of_vmdrc_heap_data(&self) -> u8 {- 4- }-- /// Return the offset of the `bump_ptr` field within `VMCopyingHeapData`.- #[inline]- fn vmcopying_heap_data_bump_ptr(&self) -> u8 {- 0- }-- /// Return the offset of the `active_space_end` field within- /// `VMCopyingHeapData`.- #[inline]- fn vmcopying_heap_data_active_space_end(&self) -> u8 {- 4- }-- /// Return the size of `VMCopyingHeapData`.- #[inline]- fn size_of_vmcopying_heap_data(&self) -> u8 {- 8- }-- /// Return the alignment of `VMCopyingHeapData`.- #[inline]- fn align_of_vmcopying_heap_data(&self) -> u8 {- 4- }}/// A trait to abstract over various types that contain a `P: PtrSize`.@@ -1023,6 +951,7 @@ impl_vmctx_array_index! {DefinedTagIndex,FuncRefIndex,RuntimeDataIndex,+ ModuleInternedTypeIndex,}/// Generate the accessors for the offsets of `VMContext`'sdiff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rsindex e6eadf1d4a..71ba0f3691 100644--- a/crates/environ/src/vmtypes.rs+++ b/crates/environ/src/vmtypes.rs@@ -20,6 +20,14 @@////// * Doc-comment attributes (`#[doc = "..."]`).///+/// * An optional `#[cfg(...)]` attribute, gating the type on the Cargo features+/// whose code actually uses it.+///+/// Only the runtime's `struct` definition (and its layout test) is gated by+/// it. A type's offsets and alias regions are always generated because the+/// `wasmtime-environ` and `wasmtime-internal-cranelift` crates do not have+/// Cargo features for all the various Wasm features.+////// * An optional `#[derive(...)]` attribute.////// * A `#[repr(...)]` attribute.@@ -37,6 +45,13 @@/// is still generated, since that is what interior accesses are computed/// relative to.///+/// * `#[indexed]`: this field is a fixed-size array of a scalar type, whose+/// elements compiled Wasm code accesses individually by a compile-time+/// constant index. Like `#[aggregate]`, the field has no single Cranelift+/// type; unlike `#[aggregate]`, its elements all do, so an alias-region+/// accessor taking that index is generated for it and each element gets its+/// own alias region.+////// * `#[readonly]` and/or `#[can_move]`: describe how Cranelift may treat loads/// and stores of that field.#[macro_export]@@ -456,7 +471,7 @@ macro_rules! for_each_vm_type {/// This is saved/restored when threads are swapped in the component model.////// NB: `UnsafeCell` because JIT code writes to the slots.- #[aggregate]+ #[indexed]pub component_context: UnsafeCell<[u32; NUM_COMPONENT_CONTEXT_SLOTS]>,/// JIT-visible current thread for the component model's sync-to-sync@@ -542,9 +557,68 @@ macro_rules! for_each_vm_type {pub callee_instance: u32,/// The caller thread's `context.{get,set}` slots, saved on entry and/// restored on the fast-path exit (or recovered while forcing).- #[aggregate]+ #[indexed]pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],}++ /// The deferred-reference-counting collector's JIT-accessible heap+ /// data.+ ///+ /// This is a separate allocation, reached through the+ /// `VMContext::gc_heap_data` pointer. Its fields are not GC heap+ /// locations, so they get this type's own alias regions rather than+ /// the GC heap's.+ ///+ /// `wasmtime::runtime::vm::gc::enabled::drc` owns one of these,+ /// wrapped in a cell because compiled Wasm writes to it, and+ /// accesses it only through that wrapper's methods.+ #[cfg(feature = "gc-drc")]+ #[derive(Default)]+ #[repr(C)]+ #[snake_name = vm_drc_heap_data]+ pub struct VMDrcHeapData {+ /// The head of the over-approximated-stack-roots list.+ pub over_approximated_stack_roots: Option<VMGcRef>,++ /// The current size of the over-approximated-stack-roots list.+ pub current_over_approximated_stack_roots_len: u32,++ /// The size of the over-approximated-stack-roots list+ /// immediately after the last GC.+ pub over_approximated_stack_roots_len_after_last_gc: u32,+ }++ /// The copying collector's JIT-accessible bump-allocation state.+ ///+ /// Like [`VMDrcHeapData`], this is a separate allocation reached+ /// through `VMContext::gc_heap_data`, and is owned, wrapped in a+ /// cell, by `wasmtime::runtime::vm::gc::enabled::copying`.+ #[cfg(feature = "gc-copying")]+ #[derive(Default)]+ #[repr(C)]+ #[snake_name = vm_copying_heap_data]+ pub struct VMCopyingHeapData {+ /// Current bump pointer (an index into the GC heap).+ pub bump_ptr: u32,++ /// End of the active semi-space.+ pub active_space_end: u32,+ }++ /// The null collector's JIT-accessible bump-allocation state.+ ///+ /// Unlike the two above, this is not a separate allocation: it is+ /// the first field of `NullHeap` (in+ /// `wasmtime::runtime::vm::gc::enabled::null`), again wrapped in a+ /// cell, and compiled Wasm reaches it through a pointer to that+ /// field.+ #[cfg(feature = "gc-null")]+ #[repr(C)]+ #[snake_name = vm_null_heap_data]+ pub struct VMNullHeapData {+ /// The bump-allocation finger, an index into the GC heap.+ pub next: NonZeroU32,+ }}};}diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex 2f503d1606..5e77c7abd5 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -126,8 +126,14 @@ pub use crate::runtime::vm::throw::*;pub use crate::runtime::vm::traphandlers::*;#[cfg(feature = "component-model")]pub use crate::runtime::vm::vmcontext::VMArrayCallFunction;+#[cfg(feature = "gc-copying")]+pub use crate::runtime::vm::vmcontext::VMCopyingHeapData;+#[cfg(feature = "gc-drc")]+pub use crate::runtime::vm::vmcontext::VMDrcHeapData;#[cfg(feature = "component-model-async")]pub use crate::runtime::vm::vmcontext::VMLazyThread;+#[cfg(feature = "gc-null")]+pub use crate::runtime::vm::vmcontext::VMNullHeapData;pub use crate::runtime::vm::vmcontext::{VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition,VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext,diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rsindex cb0ba1dd04..cafb4d0bc8 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs@@ -14,8 +14,8 @@ use super::VMArrayRef;use super::trace_infos::TraceInfos;use crate::runtime::vm::{ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,- GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMExternRef,- VMGcHeader, VMGcRef, VMMemoryDefinition,+ GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMCopyingHeapData,+ VMExternRef, VMGcHeader, VMGcRef, VMMemoryDefinition,};use crate::{Engine, bail_bug, prelude::*};use core::{@@ -176,26 +176,20 @@ unsafe impl GcHeapObject for VMCopyingExternRef {}}-/// JIT-accessible bump-allocation state for the copying collector.+/// A [`VMCopyingHeapData`] as this heap owns it.///-/// NB: Layout is defined by constants in `wasmtime_environ::copying`. Keep in-/// sync!-#[derive(Default)]-#[repr(C)]-struct VMCopyingHeapDataInner {- /// Current bump pointer (index into the GC heap).- bump_ptr: u32,- /// End of the active semi-space.- active_space_end: u32,-}-+/// Compiled Wasm holds a pointer to the inner `VMCopyingHeapData` and bumps its+/// allocation pointer, so all of the runtime's own accesses go through an+/// `UnsafeCell`. Both this wrapper and `SendSyncUnsafeCell` are+/// `repr(transparent)`, so the layout that compiled code sees is exactly the+/// generated one.#[derive(Default)]#[repr(transparent)]-struct VMCopyingHeapData {- inner: SendSyncUnsafeCell<VMCopyingHeapDataInner>,+struct VMCopyingHeapDataCell {+ inner: SendSyncUnsafeCell<VMCopyingHeapData>,}-impl VMCopyingHeapData {+impl VMCopyingHeapDataCell {fn bump_ptr(&self) -> u32 {// Safety: `inner` is valid to read from.unsafe { (*self.inner.get()).bump_ptr }@@ -266,7 +260,7 @@ struct CopyingHeap {////// NB: The bump pointer is written to by compiled Wasm code via the pointer/// returned by `vmctx_gc_heap_data`.- vmctx_data: VMCopyingHeapData,+ vmctx_data: VMCopyingHeapDataCell,/// The start of the active semi-space.active_space_start: u32,@@ -307,7 +301,7 @@ impl CopyingHeap {no_gc_count: 0,memory: None,vmmemory: None,- vmctx_data: VMCopyingHeapData::default(),+ vmctx_data: VMCopyingHeapDataCell::default(),active_space_start: 0,idle_space_start: 0,idle_space_end: 0,@@ -1220,7 +1214,6 @@ impl Drop for CopyingCollection<'_, '_> {#[cfg(test)]mod tests {use super::*;- use wasmtime_environ::{HostPtr, PtrSize};#[test]fn vm_copying_header_size_align() {@@ -1272,46 +1265,6 @@ mod tests {);}- #[test]- fn vm_copying_heap_data_bump_ptr_offset() {- assert_eq!(- HostPtr.vmcopying_heap_data_bump_ptr() as usize,- core::mem::offset_of!(VMCopyingHeapDataInner, bump_ptr),- );- }-- #[test]- fn vm_copying_heap_data_active_space_end_offset() {- assert_eq!(- HostPtr.vmcopying_heap_data_active_space_end() as usize,- core::mem::offset_of!(VMCopyingHeapDataInner, active_space_end),- );- }-- #[test]- fn vm_copying_heap_data_size() {- assert_eq!(- HostPtr.size_of_vmcopying_heap_data() as usize,- core::mem::size_of::<VMCopyingHeapData>(),- );- assert_eq!(- HostPtr.size_of_vmcopying_heap_data() as usize,- core::mem::size_of::<VMCopyingHeapDataInner>(),- );- }-- #[test]- fn vm_copying_heap_data_align() {- assert_eq!(- HostPtr.align_of_vmcopying_heap_data() as usize,- core::mem::align_of::<VMCopyingHeapData>(),- );- assert_eq!(- HostPtr.align_of_vmcopying_heap_data() as usize,- core::mem::align_of::<VMCopyingHeapDataInner>(),- );- }-#[test]fn semi_space_split_stays_aligned() {fn check(len: usize) {diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rsindex 9532e43f05..0252fd36c6 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs@@ -53,7 +53,7 @@ use crate::runtime::vm::{GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMExternRef,VMGcHeader, VMGcObjectData, VMGcRef,};-use crate::vm::VMMemoryDefinition;+use crate::vm::{VMDrcHeapData, VMMemoryDefinition};use crate::{Engine, Trap, bail_bug, prelude::*};use core::sync::atomic::AtomicUsize;use core::{@@ -97,28 +97,17 @@ unsafe impl GcRuntime for DrcCollector {}}-/// JIT-accessible DRC heap data.-#[derive(Default)]-#[repr(C)]-struct VMDrcHeapDataInner {- /// The head of the over-approximated-stack-roots list.- over_approximated_stack_roots: Option<VMGcRef>,-- /// The current size of the over-approximated-stack-roots list.- current_over_approximated_stack_roots_len: u32,-- /// The size of the over-approximated-stack-roots list immediately after the- /// last GC.- over_approximated_stack_roots_len_after_last_gc: u32,-}-+/// A `VMDrcHeapData` inside a `SendSyncUnsafeCell`.+///+/// Compiled Wasm holds a pointer to the inner `VMDrcHeapData` and writes to it,+/// so all of the runtime's own accesses go through an `UnsafeCell`.#[derive(Default)]#[repr(transparent)]-struct VMDrcHeapData {- inner: SendSyncUnsafeCell<VMDrcHeapDataInner>,+struct VMDrcHeapDataCell {+ inner: SendSyncUnsafeCell<VMDrcHeapData>,}-impl VMDrcHeapData {+impl VMDrcHeapDataCell {fn over_approximated_stack_roots(&self) -> Option<VMGcRef> {// Safety: `inner` is valid to read from.unsafe {@@ -177,7 +166,7 @@ struct DrcHeap {////// Note that this is exposed directly to compiled Wasm code through the/// vmctx, so must not move.- vmctx_data: Box<VMDrcHeapData>,+ vmctx_data: Box<VMDrcHeapDataCell>,/// The storage for the GC heap itself.memory: Option<crate::vm::Memory>,@@ -928,7 +917,7 @@ unsafe impl GcHeap for DrcHeap {} = self;*no_gc_count = 0;- **vmctx_data = VMDrcHeapData::default();+ **vmctx_data = VMDrcHeapDataCell::default();*free_list = None;*vmmemory = None;*allocated_bytes = 0;@@ -1194,7 +1183,7 @@ unsafe impl GcHeap for DrcHeap {}unsafe fn vmctx_gc_heap_data(&self) -> NonNull<u8> {- let ptr: NonNull<VMDrcHeapData> = NonNull::from(&*self.vmctx_data);+ let ptr: NonNull<VMDrcHeapDataCell> = NonNull::from(&*self.vmctx_data);ptr.cast()}@@ -1335,7 +1324,7 @@ impl<T> DerefMut for DebugOnly<T> {#[cfg(test)]mod tests {use super::*;- use wasmtime_environ::{HostPtr, PtrSize};+ use wasmtime_environ::HostPtr;#[test]fn vm_drc_header_size_align() {@@ -1393,58 +1382,4 @@ mod tests {u32::try_from(actual_offset).unwrap(),);}-- #[test]- fn vm_drc_heap_data_over_approximated_stack_roots_offset() {- assert_eq!(- HostPtr.vmdrc_heap_data_over_approximated_stack_roots() as usize,- core::mem::offset_of!(VMDrcHeapDataInner, over_approximated_stack_roots),- );- }-- #[test]- fn vm_drc_heap_data_current_over_approximated_stack_roots_len_offset() {- assert_eq!(- HostPtr.vmdrc_heap_data_current_over_approximated_stack_roots_len() as usize,- core::mem::offset_of!(- VMDrcHeapDataInner,- current_over_approximated_stack_roots_len- ),- );- }-- #[test]- fn vm_drc_heap_data_over_approximated_stack_roots_len_after_last_gc_offset() {- assert_eq!(- HostPtr.vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc() as usize,- core::mem::offset_of!(- VMDrcHeapDataInner,- over_approximated_stack_roots_len_after_last_gc- ),- );- }-- #[test]- fn vm_drc_heap_data_size() {- assert_eq!(- HostPtr.size_of_vmdrc_heap_data() as usize,- core::mem::size_of::<VMDrcHeapData>(),- );- assert_eq!(- HostPtr.size_of_vmdrc_heap_data() as usize,- core::mem::size_of::<VMDrcHeapDataInner>(),- );- }-- #[test]- fn vm_drc_heap_data_align() {- assert_eq!(- HostPtr.align_of_vmdrc_heap_data() as usize,- core::mem::align_of::<VMDrcHeapData>(),- );- assert_eq!(- HostPtr.align_of_vmdrc_heap_data() as usize,- core::mem::align_of::<VMDrcHeapDataInner>(),- );- }}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rsindex c4f06f3dd4..6ad8a79e70 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs@@ -11,7 +11,7 @@ use crate::{vm::{ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TypedGcRef, VMGcHeader, VMGcRef,- VMMemoryDefinition,+ VMMemoryDefinition, VMNullHeapData,},};use core::ptr::NonNull;@@ -38,14 +38,26 @@ unsafe impl GcRuntime for NullCollector {}}+impl VMNullHeapData {+ /// The heap data for a detached heap.+ fn detached() -> Self {+ VMNullHeapData {+ next: NonZeroU32::new(u32::MAX).unwrap(),+ }+ }+}+/// A GC heap for the null collector.+///+/// Compiled Wasm reaches the JIT-accessible bump-allocation state through a+/// pointer to `vmctx_data`, and accesses nothing else in here, so that field+/// must stay first.#[repr(C)]struct NullHeap {- /// Bump-allocation finger indexing within `1..self.heap.len()`.+ /// The bump-allocation finger, indexing within `1..self.heap.len()`.///- /// NB: this is an `UnsafeCell` because it is written to by compiled Wasm- /// code.- next: SendSyncUnsafeCell<NonZeroU32>,+ /// NB: this is in a cell because it is written to by compiled Wasm code.+ vmctx_data: SendSyncUnsafeCell<VMNullHeapData>,/// The number of active no-gc scopes at the current moment.no_gc_count: usize,@@ -111,7 +123,7 @@ impl NullHeap {fn new() -> Result<Self> {Ok(Self {no_gc_count: 0,- next: SendSyncUnsafeCell::new(NonZeroU32::new(u32::MAX).unwrap()),+ vmctx_data: SendSyncUnsafeCell::new(VMNullHeapData::detached()),memory: None,})}@@ -140,7 +152,7 @@ impl NullHeap {None => return Err(crate::Trap::AllocationTooLarge.into()),};- let next = *self.next.get_mut();+ let next = self.vmctx_data.get_mut().next;// Increment the bump pointer to the layout's requested alignment.let aligned = match u32::try_from(layout.align())@@ -163,7 +175,7 @@ impl NullHeap {}// Update the bump pointer, write the header, and return the GC ref.- *self.next.get_mut() = NonZeroU32::new(end_of_object).unwrap();+ self.vmctx_data.get_mut().next = NonZeroU32::new(end_of_object).unwrap();let aligned = NonZeroU32::new(aligned).unwrap();let gc_ref = VMGcRef::from_heap_index(aligned).unwrap();@@ -184,22 +196,21 @@ unsafe impl GcHeap for NullHeap {fn attach(&mut self, memory: crate::vm::Memory) {assert!(!self.is_attached());self.memory = Some(memory);- self.next = SendSyncUnsafeCell::new(NonZeroU32::new(1).unwrap());+ self.vmctx_data.get_mut().next = NonZeroU32::new(1).unwrap();}fn detach(&mut self) -> crate::vm::Memory {assert!(self.is_attached());let NullHeap {- next,+ vmctx_data,no_gc_count,memory,} = self;- *next.get_mut() = NonZeroU32::new(1).unwrap();+ *vmctx_data.get_mut() = VMNullHeapData::detached();*no_gc_count = 0;- self.next = SendSyncUnsafeCell::new(NonZeroU32::new(u32::MAX).unwrap());memory.take().unwrap()}@@ -337,7 +348,7 @@ unsafe impl GcHeap for NullHeap {// the heap up to the bump pointer is allocated. Subtract 1 because// the bump pointer starts at index 1 (index 0 is unused since// `VMGcRef` uses `NonZeroU32`), not because any byte was allocated.- let next = unsafe { *self.next.get() };+ let next = unsafe { (*self.vmctx_data.get()).next };usize::try_from(next.get()).unwrap() - 1}@@ -354,8 +365,8 @@ unsafe impl GcHeap for NullHeap {}unsafe fn vmctx_gc_heap_data(&self) -> NonNull<u8> {- let ptr_to_next: *mut NonZeroU32 = unsafe { self.next.get() };- NonNull::new(ptr_to_next).unwrap().cast()+ let ptr: *mut VMNullHeapData = unsafe { self.vmctx_data.get() };+ NonNull::new(ptr).unwrap().cast()}}diff --git a/crates/wasmtime/src/runtime/vm/send_sync_unsafe_cell.rs b/crates/wasmtime/src/runtime/vm/send_sync_unsafe_cell.rsindex f1b16d1a94..b20bc5baf1 100644--- a/crates/wasmtime/src/runtime/vm/send_sync_unsafe_cell.rs+++ b/crates/wasmtime/src/runtime/vm/send_sync_unsafe_cell.rs@@ -2,7 +2,10 @@ use core::cell::UnsafeCell;/// A wrapper around `UnsafeCell` that implements `Send` and `Sync` for types/// that are themselves `Send` and `Sync`.+///+/// This type is `repr(transparent)` and has exactly `T`'s layout.#[derive(Default)]+#[repr(transparent)]pub struct SendSyncUnsafeCell<T>(UnsafeCell<T>);// Safety: `T` is `Send` and users guarantee that any pointers derived from thediff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex 7d02bfb98b..17afb693c1 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -13,6 +13,8 @@ use core::ffi::c_void;use core::fmt;use core::marker;use core::mem::{self, MaybeUninit};+#[cfg(feature = "gc-null")]+use core::num::NonZeroU32;use core::ops::Range;use core::ptr::{self, NonNull};use core::sync::atomic::{AtomicUsize, Ordering};@@ -196,6 +198,7 @@ unsafe impl VmSafe for VMTagImport {}macro_rules! define_vm_types {( $($(#[doc = $sdoc:literal])*+ $(#[cfg($($scfg:tt)*)])?$(#[derive($($d:ident),*)])?#[repr($($repr:tt)*)]#[snake_name = $snake:ident]@@ -203,6 +206,7 @@ macro_rules! define_vm_types {$($(#[doc = $fdoc:literal])*$(#[aggregate])?+ $(#[indexed])?$(#[readonly])?$(#[can_move])?$fvis:vis $fname:ident : $fty:tt $(< $fgen:ty >)? ,@@ -211,6 +215,7 @@ macro_rules! define_vm_types {)* ) => {$($(#[doc = $sdoc])*+ $(#[cfg($($scfg)*)])?$(#[derive($($d),*)])?#[repr($($repr)*)]$svis struct $Name {@@ -223,13 +228,15 @@ macro_rules! define_vm_types {#[cfg(test)]mod test_vm_type_layouts {- use super::{ $( $Name, )* };use core::mem::{align_of, offset_of, size_of};use wasmtime_environ::{HostPtr, PtrSize};$(+ $(#[cfg($($scfg)*)])?#[test]fn $snake() {+ use super::$Name;+let host = HostPtr;let offsets = host.$snake();@@ -823,11 +830,10 @@ mod test_vmstore_context {use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};/// Check the `VMStoreContext` offsets that `for_each_vm_type!` does *not*- /// generate: the offsets reaching into the inlined `gc_heap`, and the- /// indexed `component_context` slot accessor.+ /// generate: the offsets reaching into the inlined `gc_heap`.///- /// Every plain field offset, plus the size and alignment of the type, is- /// already checked by the generated `test_vm_type_layouts::vm_store_context`.+ /// Every field offset, plus the size and alignment of the type, is already+ /// checked by the generated `test_vm_type_layouts::vm_store_context`.#[test]fn derived_field_offsets() {let module = Module::new(StaticModuleIndex::from_u32(0));@@ -840,20 +846,6 @@ mod test_vmstore_context {offset_of!(VMStoreContext, gc_heap) + offset_of!(VMMemoryDefinition, current_length),usize::from(offsets.ptr.vm_store_context().gc_heap_current_length()));- assert_eq!(- offset_of!(VMStoreContext, component_context),- usize::from(offsets.ptr.vm_store_context().component_context_slot(0))- );-- // Make sure that the calculation for the size of a slot is also- // accurate.- let slot_width = offsets.ptr.vm_store_context().component_context_slot(1)- - offsets.ptr.vm_store_context().component_context_slot(0);- let mut default = VMStoreContext::default();- assert_eq!(- size_of_val(&default.component_context.get_mut()[0]),- usize::from(slot_width)- );}}@@ -924,30 +916,6 @@ mod test_vmlazy_thread {}}-#[cfg(test)]-mod test_vmdeferred_thread {- use super::*;- use core::mem::offset_of;- use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};-- /// Check the indexed `saved_context` slot accessor, which- /// `for_each_vm_type!` does not generate.- ///- /// Every plain field offset, plus the size and alignment of the type, is- /// already checked by the generated- /// `test_vm_type_layouts::vm_deferred_thread`.- #[test]- fn deferred_thread_derived_field_offsets() {- let module = Module::new(StaticModuleIndex::from_u32(0));- let offsets = VMOffsets::new(HostPtr, &module);- let ptr = offsets.ptr;- assert_eq!(- offset_of!(VMDeferredThread, saved_context),- usize::from(ptr.vm_deferred_thread().saved_context_slot(0))- );- }-}-/// The VM "context", which is pointed to by the `vmctx` arg in Cranelift./// This has information about globals, memories, tables, and other runtime/// state associated with the current instance.diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rsindex abfc88bea6..8f3a1b45a3 100644--- a/winch/codegen/src/codegen/drc.rs+++ b/winch/codegen/src/codegen/drc.rs@@ -243,13 +243,15 @@ whereself.env.vmoffsets.ptr- .vmdrc_heap_data_over_approximated_stack_roots(),+ .vm_drc_heap_data()+ .over_approximated_stack_roots(),);let roots_len_offset = u32::from(self.env.vmoffsets.ptr- .vmdrc_heap_data_current_over_approximated_stack_roots_len(),+ .vm_drc_heap_data()+ .current_over_approximated_stack_roots_len(),);let next_offset = self.env@@ -341,7 +343,8 @@ whereself.env.vmoffsets.ptr- .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(),+ .vm_drc_heap_data()+ .over_approximated_stack_roots_len_after_last_gc(),);let skip_gc = self.masm.get_label()?;self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {