Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Replace almost every manually-written `AliasRegions` helper method with macro-generated versions (#14267)

Nick Fitzgerald7a882e9dca76
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index d1700dbe85..857a8c1376 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -6,7 +6,8 @@ use cranelift_codegen::{
};
use wasmtime_environ::{
BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, GetPtrSize,
- ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex, StaticModuleIndex, VMOffsets,
+ ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, PtrSize as _, RuntimeDataIndex,
+ StaticModuleIndex, VMOffsets, VmctxArrayIndex as _,
component::{
ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,
RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,
@@ -571,6 +572,11 @@ impl<'a, Offsets> Field<'a, Offsets> {
/// fn current_length(self) -> Field<'a, Offsets> { ... }
/// }
/// ```
+///
+/// A field marked `#[aggregate]` gets no accessor, because it has no single
+/// Cranelift type. A field marked `#[indexed]` is a fixed-size array of a
+/// scalar type; its accessor takes the element index, and each element gets its
+/// own alias region.
#[allow(
unused_macro_rules,
reason = "the `#[readonly]`/`#[can_move]` marker arms are generated \
@@ -597,6 +603,8 @@ macro_rules! define_vm_type_alias_region_helpers {
(@field_ty $pt:expr, i64) => { ir::types::I64 };
(@field_ty $pt:expr, u64) => { ir::types::I64 };
(@field_ty $pt:expr, u32) => { ir::types::I32 };
+ (@field_ty $pt:expr, NonZeroU32) => { ir::types::I32 };
+ (@field_ty $pt:expr, Option < VMGcRef >) => { ir::types::I32 };
// `VMLazyThread` is a pointer-sized bitpacked integer; see its definition in
// `for_each_vm_type!`.
(@field_ty $pt:expr, VMLazyThread) => { $pt };
@@ -607,6 +615,13 @@ macro_rules! define_vm_type_alias_region_helpers {
(@field_ty $pt:expr, DefinedTagIndex) => { ir::types::I32 };
(@field_ty $pt:expr, VMGlobalKind) => { ir::types::I64 };
+ // Classify an `#[indexed]` field's array type to
+ // `(element ir::Type, element size in bytes, element count)`.
+ (@array_info $pt:expr, UnsafeCell < $inner:tt >) => {
+ define_vm_type_alias_region_helpers!(@array_info $pt, $inner)
+ };
+ (@array_info $pt:expr, [u32; $n:expr]) => { (ir::types::I32, 4u32, $n) };
+
// Apply a field attribute to its access flags. The `#[readonly]` and
// `#[can_move]` markers map to the corresponding `MemFlagsData` builder
// methods; doc comments have no effect on flags; any other attribute is a
@@ -615,13 +630,68 @@ macro_rules! define_vm_type_alias_region_helpers {
(@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() };
(@apply_attr $flags:expr, [doc = $d:literal]) => { $flags };
+ // Emit the accessor method for a single ordinary (scalar) field.
+ (@method $Name:ident $snake:ident plain $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ]) => {
+ #[doc = concat!(
+ "Get the [`Field`] for the `", stringify!($fname),
+ "` field of `", stringify!($Name), "`."
+ )]
+ pub fn $fname(self) -> Field<'a, Offsets> {
+ let offset = u32::from(
+ self.regions.offsets.get_ptr_size().$snake().$fname()
+ );
+ let flags = ir::MemFlagsData::trusted();
+ $(
+ let flags = define_vm_type_alias_region_helpers!(
+ @apply_attr flags, $fattr
+ );
+ )*
+ let ty = define_vm_type_alias_region_helpers!(
+ @field_ty self.regions.pointer_type, $($fty)*
+ );
+ Field::new(self.regions, VmType::$Name, offset, flags, ty)
+ }
+ };
+
+ // Emit the accessor method for an `#[indexed]` field: a fixed-size array of
+ // a scalar type, indexed by a compile-time constant.
+ (@method $Name:ident $snake:ident indexed $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ]) => {
+ #[doc = concat!(
+ "Get the [`Field`] for the `index`th element of the `",
+ stringify!($fname), "` field of `", stringify!($Name), "`."
+ )]
+ pub fn $fname(self, index: u8) -> Field<'a, Offsets> {
+ let (ty, elem_size, len) = define_vm_type_alias_region_helpers!(
+ @array_info self.regions.pointer_type, $($fty)*
+ );
+ assert!(
+ usize::from(index) < len,
+ concat!(
+ "index out of bounds for `", stringify!($Name), "::",
+ stringify!($fname), "`"
+ ),
+ );
+ let base = u32::from(
+ self.regions.offsets.get_ptr_size().$snake().$fname()
+ );
+ let offset = base + u32::from(index) * elem_size;
+ let flags = ir::MemFlagsData::trusted();
+ $(
+ let flags = define_vm_type_alias_region_helpers!(
+ @apply_attr flags, $fattr
+ );
+ )*
+ Field::new(self.regions, VmType::$Name, offset, flags, ty)
+ }
+ };
+
// Emit the accessor `struct` and per-field methods for one `VM*` type.
//
- // Fields arrive pre-split into `{ $fname [ $attrs... ] [ $fty... ] }` groups
- // (see `@munch` below); the terminal arm consumes those groups once the
- // struct body has been fully peeled apart.
+ // Fields arrive pre-split into `{ $kind $fname [ $attrs... ] [ $fty... ] }`
+ // groups (see `@munch` below); the terminal arm consumes those groups once
+ // the struct body has been fully peeled apart.
(@emit $Name:ident $snake:ident {
- $( { $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] } )*
+ $( { $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] } )*
}) => {
#[doc = concat!(
"An [`AliasRegions`] accessor for the fields of a `",
@@ -657,32 +727,17 @@ macro_rules! define_vm_type_alias_region_helpers {
Offsets: GetPtrSize,
{
$(
- #[doc = concat!(
- "Get the [`Field`] for the `", stringify!($fname),
- "` field of `", stringify!($Name), "`."
- )]
- pub fn $fname(self) -> Field<'a, Offsets> {
- let offset = u32::from(
- self.regions.offsets.get_ptr_size().$snake().$fname()
- );
- let flags = ir::MemFlagsData::trusted();
- $(
- let flags = define_vm_type_alias_region_helpers!(
- @apply_attr flags, $fattr
- );
- )*
- let ty = define_vm_type_alias_region_helpers!(
- @field_ty self.regions.pointer_type, $($fty)*
- );
- Field::new(self.regions, VmType::$Name, offset, flags, ty)
- }
+ define_vm_type_alias_region_helpers!(
+ @method $Name $snake $fkind $fname [ $($fattr)* ] [ $($fty)* ]
+ );
)*
}
};
// Peel fields off the raw struct body one at a time, accumulating completed
- // `{ $fname [ $attrs... ] [ $fty... ] }` groups plus the pending attributes
- // for the field currently being parsed.
+ // `{ $kind $fname [ $attrs... ] [ $fty... ] }` groups plus the pending
+ // attributes for the field currently being parsed. `$kind` is `plain` or
+ // `indexed`, and selects which `@method` arm emits the field's accessor.
//
// Splitting the body by hand (rather than matching `$fty:tt $(< $fgen:ty
// >)?` within a repetition) is what lets each field's type reach the
@@ -692,35 +747,43 @@ macro_rules! define_vm_type_alias_region_helpers {
define_vm_type_alias_region_helpers!(@emit $Name $snake { $($groups)* });
};
// A field marked `#[aggregate]` is a composite (a nested struct or array)
- // that is not accessed as a whole by Wasm code, but piecewise
- // instead. Therefore, there is no single Cranelift type for such fields, so
- // no accessors are generated for them; interior accesses are computed from
- // the field's offset instead, which the generated `offsets::*` methods
- // still provide.
+ // so there is no single Cranelift type for such fields, so no accessors are
+ // generated for them; interior accesses are computed from the field's
+ // offset instead, which the generated `offsets::*` methods still provide.
(@munch $Name:ident $snake:ident { $($groups:tt)* }
$(#[doc = $fdoc:literal])* #[aggregate] $fvis:vis $fname:ident : $fty:ty , $($rest:tt)*
) => {
define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* } $($rest)*);
};
+ // A field marked `#[indexed]` is a fixed-size array of a scalar type whose
+ // elements are accessed individually by a compile-time constant index. It
+ // has no single Cranelift type either, but its elements do, so it gets an
+ // index-taking accessor.
+ (@munch $Name:ident $snake:ident { $($groups:tt)* }
+ $(#[doc = $fdoc:literal])* #[indexed] $fvis:vis $fname:ident : $($rest:tt)*
+ ) => {
+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } indexed $fname [] [] $($rest)*);
+ };
// Consume one field's attributes, visibility, and name, then collect its
// type tokens.
(@munch $Name:ident $snake:ident { $($groups:tt)* }
$(# $fattr:tt)* $fvis:vis $fname:ident : $($rest:tt)*
) => {
- define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($fattr)* ] [] $($rest)*);
+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } plain $fname [ $($fattr)* ] [] $($rest)*);
};
// Accumulate one field's type tokens up to its terminating comma, then
// append the completed group.
- (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {
- define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fname [ $($fattr)* ] [ $($fty)* ] } } $($rest)*);
+ (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {
+ define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fkind $fname [ $($fattr)* ] [ $($fty)* ] } } $($rest)*);
};
- (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {
- define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($fattr)* ] [ $($fty)* $tok ] $($rest)*);
+ (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {
+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fkind $fname [ $($fattr)* ] [ $($fty)* $tok ] $($rest)*);
};
// Top-level entry: the list of `VM*` type definitions.
( $(
$(#[doc = $sdoc:literal])*
+ $(#[cfg($($scfg:tt)*)])?
$(#[derive($($d:ident),*)])?
#[repr($($repr:tt)*)]
#[snake_name = $snake:ident]
@@ -760,6 +823,11 @@ wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers);
/// accessing that `VM*` type through a pointer would. For the few aggregates that
/// have no `VM*` type of their own, and hence no alias region of their own, there
/// are hand-written helpers below.
+///
+/// A field marked `#[pointee(..)]` additionally gets an accessor for what it
+/// points *at*, which lives outside the vmctx. For example,
+/// `alias_regions.vmctx().builtin_functions_array(i)` is the `i`th builtin in
+/// the array that the `VMContext::builtin_functions` field points at.
#[allow(
unused_macro_rules,
reason = "entry shapes and marker attributes are handled uniformly for both \
@@ -770,6 +838,9 @@ macro_rules! define_vmctx_alias_region_helpers {
// pointer type as an `ir::Type`).
(@field_ty $pt:expr, u32) => { ir::types::I32 };
(@field_ty $pt:expr, VmPtr < $g:ident >) => { $pt };
+ (@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };
+ (@field_ty $pt:expr, AtomicU64) => { ir::types::I64 };
+ (@field_ty $pt:expr, unsafe extern "C" fn) => { $pt };
// Determine the Cranelift type a field is *accessed* as: the classification
// of its `#[access_as = T]` type if it has one, and of its declared type
@@ -789,6 +860,7 @@ macro_rules! define_vmctx_alias_region_helpers {
(@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() };
(@apply_attr $flags:expr, [access_as = $($t:tt)*]) => { $flags };
(@apply_attr $flags:expr, [ptr_size_offset]) => { $flags };
+ (@apply_attr $flags:expr, [pointee( $($t:tt)* )]) => { $flags };
// Compute a field's access flags and Cranelift type from its declared type
// and marker attributes, and build the `Field` for it at `$offset`.
@@ -803,6 +875,55 @@ macro_rules! define_vmctx_alias_region_helpers {
Field::new(this.regions, VmType::$Name, offset, flags, ty)
}};
+ // Scan a field's attributes for a `#[pointee(..)]` and emit the accessor it
+ // describes, if any.
+ (@pointee_methods $Offsets:tt [ ]) => {};
+ (@pointee_methods $Offsets:tt [ #[pointee( $($p:tt)* )] $($rest:tt)* ]) => {
+ define_vmctx_alias_region_helpers!(@pointee $Offsets $($p)*);
+ };
+ (@pointee_methods $Offsets:tt [ # $skip:tt $($rest:tt)* ]) => {
+ define_vmctx_alias_region_helpers!(@pointee_methods $Offsets [ $($rest)* ]);
+ };
+
+ // An array of pointees.
+ (@pointee [ $($Offsets:tt)* ]
+ $(# $pattr:tt)* $Region:ident as $pname:ident [ $Index:ident ] : $($pty:tt)*
+ ) => {
+ #[doc = concat!(
+ "Get the [`Field`] for the `index`th element of the out-of-line `",
+ stringify!($Region), "` array.\n\nThe returned [`Field`] is relative \
+ to the array's base pointer, not to the vmctx."
+ )]
+ pub fn $pname(self, index: $Index) -> Field<'a, $($Offsets)*> {
+ let ty = define_vmctx_alias_region_helpers!(
+ @field_ty self.regions.pointer_type, $($pty)*
+ );
+ let offset = index.vmctx_array_index().checked_mul(ty.bytes()).unwrap();
+ let flags = ir::MemFlagsData::trusted();
+ $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $pattr); )*
+ Field::new(self.regions, VmType::$Region, offset, flags, ty)
+ }
+ };
+
+ // A single pointee.
+ (@pointee [ $($Offsets:tt)* ]
+ $(# $pattr:tt)* $Region:ident as $pname:ident : $($pty:tt)*
+ ) => {
+ #[doc = concat!(
+ "Get the [`Field`] for the out-of-line `", stringify!($Region),
+ "`.\n\nThe returned [`Field`] is relative to the pointer to it, not \
+ to the vmctx."
+ )]
+ pub fn $pname(self) -> Field<'a, $($Offsets)*> {
+ let ty = define_vmctx_alias_region_helpers!(
+ @field_ty self.regions.pointer_type, $($pty)*
+ );
+ let flags = ir::MemFlagsData::trusted();
+ $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $pattr); )*
+ Field::new(self.regions, VmType::$Region, 0, flags, ty)
+ }
+ };
+
// ### `static` Section Entries
(@static_entry $Name:ident $snake:ident align { $al:tt }) => {};
@@ -823,6 +944,8 @@ macro_rules! define_vmctx_alias_region_helpers {
@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
)
}
+
+ define_vmctx_alias_region_helpers!(@pointee_methods [ Offsets ] [ $(# $fattr)* ]);
};
// ### `dynamic` Section Entries Marked `#[ptr_size_offset]`
@@ -891,6 +1014,10 @@ macro_rules! define_vmctx_alias_region_helpers {
@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
)
}
+
+ define_vmctx_alias_region_helpers!(
+ @pointee_methods [ $($Offsets)* ] [ $(# $fattr)* ]
+ );
};
// A single field, whether unconditionally present (`field`) or only
@@ -910,6 +1037,10 @@ macro_rules! define_vmctx_alias_region_helpers {
@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
)
}
+
+ define_vmctx_alias_region_helpers!(
+ @pointee_methods [ $($Offsets)* ] [ $(# $fattr)* ]
+ );
};
// Emit the `impl` block holding the accessors for the dynamically-positioned
@@ -1099,40 +1230,6 @@ where
)
}
- fn vmstore_context_load(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- ty: ir::Type,
- base_flags: ir::MemFlagsData,
- vmstore_ctx: ir::Value,
- offset: u32,
- ) -> ir::Value {
- let region = self.vmstore_context_region(cursor.func, offset);
- cursor.ins().load(
- ty,
- base_flags.with_alias_region(Some(region)),
- vmstore_ctx,
- i32::try_from(offset).unwrap(),
- )
- }
-
- fn vmstore_context_store(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- base_flags: ir::MemFlagsData,
- vmstore_ctx: ir::Value,
- offset: u32,
- val: ir::Value,
- ) {
- let region = self.vmstore_context_region(cursor.func, offset);
- cursor.ins().store(
- base_flags.with_alias_region(Some(region)),
- val,
- vmstore_ctx,
- i32::try_from(offset).unwrap(),
- );
- }
-
/// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).
///
/// The caller supplies the base flags because whether the base pointer is
@@ -1208,297 +1305,13 @@ where
let offset = self.offsets.get_ptr_size().vm_store_context().stack_chain();
self.vmstore_context_region(func, offset.into())
}
-
- /// Load a `VMStoreContext` component-context slot.
- ///
- /// The slot is indexed by a compile-time constant, so the alias region is
- /// keyed on the precise per-slot offset.
- pub fn vmstore_context_component_context_slot(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- ty: ir::Type,
- vmstore_ctx: ir::Value,
- slot: u8,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- ty,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .component_context_slot(slot)
- .into(),
- )
- }
-
- /// Store a `VMStoreContext` component-context slot.
- ///
- /// The slot is indexed by a compile-time constant, so the alias region is
- /// keyed on the precise per-slot offset.
- pub fn store_vmstore_context_component_context_slot(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- slot: u8,
- val: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .component_context_slot(slot)
- .into(),
- val,
- )
- }
-}
-
-/// `VMDeferredThread`-related methods.
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- fn vmdeferred_thread_region(
- &mut self,
- func: &mut ir::Function,
- offset: u32,
- ) -> ir::AliasRegion {
- self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMDeferredThread,
- offset,
- },
- )
- }
-
- fn vmdeferred_thread_load(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- ty: ir::Type,
- base_flags: ir::MemFlagsData,
- vmdeferred_thread_ptr: ir::Value,
- offset: u32,
- ) -> ir::Value {
- let region = self.vmdeferred_thread_region(cursor.func, offset);
- cursor.ins().load(
- ty,
- base_flags.with_alias_region(Some(region)),
- vmdeferred_thread_ptr,
- i32::try_from(offset).unwrap(),
- )
- }
-
- fn vmdeferred_thread_store(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- base_flags: ir::MemFlagsData,
- vmdeferred_thread_ptr: ir::Value,
- offset: u32,
- val: ir::Value,
- ) {
- let region = self.vmdeferred_thread_region(cursor.func, offset);
- cursor.ins().store(
- base_flags.with_alias_region(Some(region)),
- val,
- vmdeferred_thread_ptr,
- i32::try_from(offset).unwrap(),
- );
- }
-
- /// Load `VMDeferredThread::saved_context[i]` (a saved `context.{get,set}`
- /// slot).
- pub fn vmdeferred_thread_saved_context(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmdeferred_thread_ptr: ir::Value,
- i: u8,
- ) -> ir::Value {
- self.vmdeferred_thread_load(
- cursor,
- ir::types::I32,
- ir::MemFlagsData::trusted(),
- vmdeferred_thread_ptr,
- self.offsets
- .get_ptr_size()
- .vm_deferred_thread()
- .saved_context_slot(i)
- .into(),
- )
- }
-
- /// Store `VMDeferredThread::saved_context[i]`.
- pub fn store_vmdeferred_thread_saved_context(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmdeferred_thread_ptr: ir::Value,
- i: u8,
- val: ir::Value,
- ) {
- self.vmdeferred_thread_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmdeferred_thread_ptr,
- self.offsets
- .get_ptr_size()
- .vm_deferred_thread()
- .saved_context_slot(i)
- .into(),
- val,
- )
- }
-}
-
-/// `[VMSharedTypeIndex]`-related methods.
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- /// Load a `VMSharedTypeIndex` element out of the `[VMSharedTypeIndex]`
- /// array pointed at by the `VMContext::type_ids_array` field.
- pub fn type_ids_array_element(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- array: ir::Value,
- ty: ModuleInternedTypeIndex,
- ) -> ir::Value {
- let load_ty = ir::Type::int_with_byte_size(
- self.offsets
- .get_ptr_size()
- .size_of_vmshared_type_index()
- .into(),
- )
- .unwrap();
-
- let offset = ty.as_u32().checked_mul(load_ty.bytes()).unwrap();
- let region = self.region(
- cursor.func,
- AliasRegionKey::Vm {
- ty: VmType::TypeIdsArray,
- offset,
- },
- );
-
- cursor.ins().load(
- load_ty,
- ir::MemFlagsData::trusted()
- .with_readonly()
- .with_can_move()
- .with_alias_region(Some(region)),
- array,
- i32::try_from(offset).unwrap(),
- )
- }
-}
-
-/// Epoch counter-related methods.
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- /// Dereference the epoch pointer (an `*const AtomicU64` previously loaded
- /// out of the vmctx's `epoch_ptr` field) to read the current epoch counter.
- pub fn epoch_counter(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- epoch_ptr: ir::Value,
- ) -> ir::Value {
- let region = self.region(
- cursor.func,
- AliasRegionKey::Vm {
- ty: VmType::EpochCounter,
- offset: 0,
- },
- );
- cursor.ins().load(
- ir::types::I64,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- epoch_ptr,
- 0,
- )
- }
}
-/// Builtin-functions array methods.
+/// Component-specific methods.
impl<Offsets> AliasRegions<Offsets>
where
Offsets: GetPtrSize,
{
- /// Load a host function pointer element out of the builtin-functions array
- /// (`VMContext::builtin_functions`)
- pub fn builtin_functions_array_element(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- array: ir::Value,
- builtin: BuiltinFunctionIndex,
- ) -> ir::Value {
- let offset = builtin
- .index()
- .checked_mul(self.pointer_type.bytes())
- .unwrap();
-
- let region = self.region(
- cursor.func,
- AliasRegionKey::Vm {
- ty: VmType::BuiltinFunctionsArray,
- offset,
- },
- );
-
- cursor.ins().load(
- self.pointer_type,
- ir::MemFlagsData::trusted()
- .with_readonly()
- .with_can_move()
- .with_alias_region(Some(region)),
- array,
- i32::try_from(offset).unwrap(),
- )
- }
-}
-
-/// Component builtin-functions array methods.
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- /// Load a host function pointer element out of the component
- /// builtin-functions array (`VMComponentContext::builtins`).
- pub fn component_builtin_functions_array_element(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- array: ir::Value,
- builtin: ComponentBuiltinFunctionIndex,
- ) -> ir::Value {
- let offset = builtin
- .index()
- .checked_mul(self.pointer_type.bytes())
- .unwrap();
-
- let region = self.region(
- cursor.func,
- AliasRegionKey::Vm {
- ty: VmType::ComponentBuiltinFunctionsArray,
- offset,
- },
- );
-
- cursor.ins().load(
- self.pointer_type,
- ir::MemFlagsData::trusted()
- .with_readonly()
- .with_can_move()
- .with_alias_region(Some(region)),
- array,
- i32::try_from(offset).unwrap(),
- )
- }
-
/// Get the alias region for the `ValRaw` array used to marshal arguments
/// and results across the array calling convention used by various
/// trampolines.
@@ -1609,236 +1422,6 @@ impl AliasRegions<VMComponentOffsets<u8>> {
}
}
-/// Methods for the collectors' private heap-data structs.
-///
-/// Each struct is a separate allocation reached through a `*mut _` stored in the
-/// `VMContext`. Their fields are *not* GC heap locations, so they are tagged with
-/// the owning struct's own region (keyed on the field offset within the struct)
-/// rather than the `GcHeap` region. These helpers emit the field load/store.
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- fn vmdrc_heap_data_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {
- self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMDrcHeapData,
- offset,
- },
- )
- }
-
- /// Emit a load of the DRC over-approximated-stack-roots list head, given the
- /// a `*mut VMDrcHeapData`.
- pub fn vmdrc_heap_data_over_approximated_stack_roots(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- drc_heap_data: ir::Value,
- ) -> ir::Value {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmdrc_heap_data_over_approximated_stack_roots();
- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());
- cursor.ins().load(
- ir::types::I32,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- drc_heap_data,
- i32::from(offset),
- )
- }
-
- /// Emit a store to the DRC over-approximated-stack-roots list head.
- pub fn store_vmdrc_heap_data_over_approximated_stack_roots(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- drc_heap_data: ir::Value,
- val: ir::Value,
- ) {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmdrc_heap_data_over_approximated_stack_roots();
- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());
- cursor.ins().store(
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- val,
- drc_heap_data,
- i32::from(offset),
- );
- }
-
- /// Emit a load of the current over-approximated-stack-roots list length.
- pub fn vmdrc_heap_data_current_over_approximated_stack_roots_len(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- drc_heap_data: ir::Value,
- ) -> ir::Value {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmdrc_heap_data_current_over_approximated_stack_roots_len();
- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());
- cursor.ins().load(
- ir::types::I32,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- drc_heap_data,
- i32::from(offset),
- )
- }
-
- /// Emit a store to the current over-approximated-stack-roots list length.
- pub fn store_vmdrc_heap_data_current_over_approximated_stack_roots_len(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- drc_heap_data: ir::Value,
- len: ir::Value,
- ) {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmdrc_heap_data_current_over_approximated_stack_roots_len();
- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());
- cursor.ins().store(
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- len,
- drc_heap_data,
- i32::from(offset),
- );
- }
-
- /// Emit a load of the over-approximated-stack-roots list length after the
- /// last GC.
- pub fn vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- drc_heap_data: ir::Value,
- ) -> ir::Value {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc();
- let region = self.vmdrc_heap_data_region(cursor.func, offset.into());
- cursor.ins().load(
- ir::types::I32,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- drc_heap_data,
- i32::from(offset),
- )
- }
-
- fn vmcopying_heap_data_region(
- &mut self,
- func: &mut ir::Function,
- offset: u32,
- ) -> ir::AliasRegion {
- self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMCopyingHeapData,
- offset,
- },
- )
- }
-
- /// Emit a load of the copying collector's bump pointer.
- pub fn vmcopying_heap_data_bump_ptr(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- copying_heap_data: ir::Value,
- ) -> ir::Value {
- let offset = self.offsets.get_ptr_size().vmcopying_heap_data_bump_ptr();
- let region = self.vmcopying_heap_data_region(cursor.func, offset.into());
- cursor.ins().load(
- ir::types::I32,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- copying_heap_data,
- i32::from(offset),
- )
- }
-
- /// Emit a store to the copying collector's bump pointer.
- pub fn store_vmcopying_heap_data_bump_ptr(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- copying_heap_data: ir::Value,
- val: ir::Value,
- ) {
- let offset = self.offsets.get_ptr_size().vmcopying_heap_data_bump_ptr();
- let region = self.vmcopying_heap_data_region(cursor.func, offset.into());
- cursor.ins().store(
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- val,
- copying_heap_data,
- i32::from(offset),
- );
- }
-
- /// Emit a load of the copying collector's active-space-end pointer.
- pub fn vmcopying_heap_data_active_space_end(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- copying_heap_data: ir::Value,
- ) -> ir::Value {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmcopying_heap_data_active_space_end();
- let region = self.vmcopying_heap_data_region(cursor.func, offset.into());
- cursor.ins().load(
- ir::types::I32,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- copying_heap_data,
- i32::from(offset),
- )
- }
-
- /// Emit a load of the null collector's bump finger (the first and only field
- /// of its heap data, at offset 0).
- pub fn vmnull_heap_data_bump_finger(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- null_collector_heap_data: ir::Value,
- ) -> ir::Value {
- let region = self.region(
- cursor.func,
- AliasRegionKey::Vm {
- ty: VmType::VMNullHeapData,
- offset: 0,
- },
- );
- cursor.ins().load(
- ir::types::I32,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- null_collector_heap_data,
- 0,
- )
- }
-
- /// Emit a store to the null collector's bump finger.
- pub fn store_vmnull_heap_data_bump_finger(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- null_collector_heap_data: ir::Value,
- val: ir::Value,
- ) {
- let region = self.region(
- cursor.func,
- AliasRegionKey::Vm {
- ty: VmType::VMNullHeapData,
- offset: 0,
- },
- );
- cursor.ins().store(
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- val,
- null_collector_heap_data,
- 0,
- );
- }
-}
-
/// Stack-switching and continuation-object methods.
impl<Offsets> AliasRegions<Offsets>
where
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 4665fcd1eb..ed95a74f24 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -1388,11 +1388,10 @@ impl Compiler {
.load(&mut builder.cursor(), vmctx);
// ... and then load the entry in the array that corresponds to this
// builtin.
- let func_addr = alias_regions.builtin_functions_array_element(
- &mut builder.cursor(),
- array_addr,
- builtin,
- );
+ let func_addr = alias_regions
+ .vmctx()
+ .builtin_functions_array(builtin)
+ .load(&mut builder.cursor(), array_addr);
let sig = builder.func.import_signature(sig);
self.call_indirect_host(builder, builtin, sig, func_addr, args)
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index b32ae9a203..fa48ca3fba 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -1398,11 +1398,9 @@ impl<'a> TrampolineCompiler<'a> {
.load(&mut self.builder.cursor(), vmctx);
// Next load the function pointer at `offset` and return that.
self.alias_regions
- .component_builtin_functions_array_element(
- &mut self.builder.cursor(),
- builtins_array,
- index,
- )
+ .vmcomponent()
+ .builtins_array(index)
+ .load(&mut self.builder.cursor(), builtins_array)
}
/// Get a function's parameters regardless of the ABI in use.
@@ -2356,16 +2354,6 @@ where
intrinsic: UnsafeIntrinsic,
params: &[ir::Value],
) -> Option<ir::Value> {
- // This is the width of the type being loaded from Wasmtime's
- // `VMStoreContext` slot and it depends on the intrinsic.
- let ty = match intrinsic {
- UnsafeIntrinsic::ContextGetI32_0
- | UnsafeIntrinsic::ContextSetI32_0
- | UnsafeIntrinsic::ContextGetI32_1
- | UnsafeIntrinsic::ContextSetI32_1 => ir::types::I32,
- _ => unreachable!(),
- };
-
let slot = match intrinsic {
UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextSetI32_0 => 0,
UnsafeIntrinsic::ContextGetI32_1 | UnsafeIntrinsic::ContextSetI32_1 => 1,
@@ -2377,24 +2365,18 @@ where
let context = self
.traps
.alias_regions()
- .vmstore_context_component_context_slot(
- &mut self.builder.cursor(),
- ty,
- vmstore_context,
- slot,
- );
+ .vm_store_context()
+ .component_context(slot)
+ .load(&mut self.builder.cursor(), vmstore_context);
Some(context)
}
UnsafeIntrinsic::ContextSetI32_0 | UnsafeIntrinsic::ContextSetI32_1 => {
let new_context = params[2];
self.traps
.alias_regions()
- .store_vmstore_context_component_context_slot(
- &mut self.builder.cursor(),
- vmstore_context,
- slot,
- new_context,
- );
+ .vm_store_context()
+ .component_context(slot)
+ .store(&mut self.builder.cursor(), vmstore_context, new_context);
None
}
_ => unreachable!(),
diff --git a/crates/cranelift/src/component_sync_call.rs b/crates/cranelift/src/component_sync_call.rs
index 831603afcf..876bb4e170 100644
--- a/crates/cranelift/src/component_sync_call.rs
+++ b/crates/cranelift/src/component_sync_call.rs
@@ -98,23 +98,19 @@ where
// Save the caller's context slots into the frame and reset the live values
// to 0 for the freshly-entered (deferred) thread.
for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
- let saved = alias_regions.vmstore_context_component_context_slot(
- &mut builder.cursor(),
- ir::types::I32,
- vmstore,
- i,
- );
- alias_regions.store_vmdeferred_thread_saved_context(
+ let saved = alias_regions
+ .vm_store_context()
+ .component_context(i)
+ .load(&mut builder.cursor(), vmstore);
+ alias_regions.vm_deferred_thread().saved_context(i).store(
&mut builder.cursor(),
slot_addr,
- i,
saved,
);
let zero = builder.ins().iconst(ir::types::I32, 0);
- alias_regions.store_vmstore_context_component_context_slot(
+ alias_regions.vm_store_context().component_context(i).store(
&mut builder.cursor(),
vmstore,
- i,
zero,
);
}
@@ -195,12 +191,13 @@ where
.current_thread()
.store(&mut builder.cursor(), vmstore, parent);
for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
- let saved =
- alias_regions.vmdeferred_thread_saved_context(&mut builder.cursor(), slot_addr, i);
- alias_regions.store_vmstore_context_component_context_slot(
+ let saved = alias_regions
+ .vm_deferred_thread()
+ .saved_context(i)
+ .load(&mut builder.cursor(), slot_addr);
+ alias_regions.vm_store_context().component_context(i).store(
&mut builder.cursor(),
vmstore,
- i,
saved,
);
}
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index a617dc06c1..3445f11c9c 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -804,7 +804,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
fn epoch_load_current(&mut self, builder: &mut FunctionBuilder<'_>) -> ir::Value {
let addr = builder.use_var(self.epoch_ptr_var);
self.alias_regions
- .epoch_counter(&mut builder.cursor(), addr)
+ .vmctx()
+ .epoch_counter()
+ .load(&mut builder.cursor(), addr)
}
fn epoch_check(&mut self, builder: &mut FunctionBuilder<'_>) {
@@ -1162,12 +1164,12 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
// Load the base pointer of the array of `VMSharedTypeIndex`es.
let shared_indices = self.alias_regions.vmctx().type_ids().load(pos, vmctx);
- // Calculate the offset in that array for this type's entry.
-
// Load the`VMSharedTypeIndex` that this `ModuleInternedTypeIndex` is
// associated with at runtime from the array.
self.alias_regions
- .type_ids_array_element(pos, shared_indices, interned_ty)
+ .vmctx()
+ .type_ids_array(interned_ty)
+ .load(pos, shared_indices)
}
/// Does this function need a GC heap?
diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rs
index 7f3a56541e..7cf5628d0b 100644
--- a/crates/cranelift/src/func_environ/gc/copying.rs
+++ b/crates/cranelift/src/func_environ/gc/copying.rs
@@ -50,10 +50,14 @@ impl CopyingCompiler {
) -> (ir::Value, ir::Value) {
let bump_ptr = func_env
.alias_regions
- .vmcopying_heap_data_bump_ptr(&mut builder.cursor(), ptr_to_heap_data);
+ .vm_copying_heap_data()
+ .bump_ptr()
+ .load(&mut builder.cursor(), ptr_to_heap_data);
let active_space_end = func_env
.alias_regions
- .vmcopying_heap_data_active_space_end(&mut builder.cursor(), ptr_to_heap_data);
+ .vm_copying_heap_data()
+ .active_space_end()
+ .load(&mut builder.cursor(), ptr_to_heap_data);
(bump_ptr, active_space_end)
}
@@ -64,11 +68,11 @@ impl CopyingCompiler {
heap_data_ptr: ir::Value,
end_of_object: ir::Value,
) {
- func_env.alias_regions.store_vmcopying_heap_data_bump_ptr(
- &mut builder.cursor(),
- heap_data_ptr,
- end_of_object,
- );
+ func_env
+ .alias_regions
+ .vm_copying_heap_data()
+ .bump_ptr()
+ .store(&mut builder.cursor(), heap_data_ptr, end_of_object);
}
/// Round `size` (an `i32`) up to `ALIGN`, returning the result as an `i64`.
diff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rs
index da3d205965..50a453fd1f 100644
--- a/crates/cranelift/src/func_environ/gc/drc.rs
+++ b/crates/cranelift/src/func_environ/gc/drc.rs
@@ -113,7 +113,9 @@ impl DrcCompiler {
// element.
let next = func_env
.alias_regions
- .vmdrc_heap_data_over_approximated_stack_roots(&mut builder.cursor(), heap_data);
+ .vm_drc_heap_data()
+ .over_approximated_stack_roots()
+ .load(&mut builder.cursor(), heap_data);
// Update our object's header to point to `next` and consider itself part of the list.
self.set_next_over_approximated_stack_root(func_env, builder, gc_ref, next);
@@ -125,11 +127,9 @@ impl DrcCompiler {
// Commit this object as the new head of the list.
func_env
.alias_regions
- .store_vmdrc_heap_data_over_approximated_stack_roots(
- &mut builder.cursor(),
- heap_data,
- gc_ref,
- );
+ .vm_drc_heap_data()
+ .over_approximated_stack_roots()
+ .store(&mut builder.cursor(), heap_data, gc_ref);
// Increment the list's length.
//
@@ -140,19 +140,16 @@ impl DrcCompiler {
// `u32::MAX`.
let current_len = func_env
.alias_regions
- .vmdrc_heap_data_current_over_approximated_stack_roots_len(
- &mut builder.cursor(),
- heap_data,
- );
+ .vm_drc_heap_data()
+ .current_over_approximated_stack_roots_len()
+ .load(&mut builder.cursor(), heap_data);
let one = builder.ins().iconst(ir::types::I32, 1);
let new_current_len = builder.ins().iadd(current_len, one);
func_env
.alias_regions
- .store_vmdrc_heap_data_current_over_approximated_stack_roots_len(
- &mut builder.cursor(),
- heap_data,
- new_current_len,
- );
+ .vm_drc_heap_data()
+ .current_over_approximated_stack_roots_len()
+ .store(&mut builder.cursor(), heap_data, new_current_len);
}
/// Trigger a GC when the over-approximated-stack-roots list has doubled
@@ -178,16 +175,14 @@ impl DrcCompiler {
.load(&mut builder.cursor(), vmctx);
let current_len = func_env
.alias_regions
- .vmdrc_heap_data_current_over_approximated_stack_roots_len(
- &mut builder.cursor(),
- heap_data,
- );
+ .vm_drc_heap_data()
+ .current_over_approximated_stack_roots_len()
+ .load(&mut builder.cursor(), heap_data);
let last_len = func_env
.alias_regions
- .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(
- &mut builder.cursor(),
- heap_data,
- );
+ .vm_drc_heap_data()
+ .over_approximated_stack_roots_len_after_last_gc()
+ .load(&mut builder.cursor(), heap_data);
let doubled_last_len = builder.ins().iadd(last_len, last_len);
let min_threshold = builder.ins().iconst(
diff --git a/crates/cranelift/src/func_environ/gc/null.rs b/crates/cranelift/src/func_environ/gc/null.rs
index 454166c278..479a9b6fd0 100644
--- a/crates/cranelift/src/func_environ/gc/null.rs
+++ b/crates/cranelift/src/func_environ/gc/null.rs
@@ -77,7 +77,9 @@ impl NullCompiler {
.load(&mut builder.cursor(), vmctx);
let next = func_env
.alias_regions
- .vmnull_heap_data_bump_finger(&mut builder.cursor(), ptr_to_next);
+ .vm_null_heap_data()
+ .next()
+ .load(&mut builder.cursor(), ptr_to_next);
// Increment the bump "pointer" to the requested alignment:
//
@@ -168,7 +170,7 @@ impl NullCompiler {
ptr_to_object,
i32::try_from(wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET).unwrap(),
);
- func_env.alias_regions.store_vmnull_heap_data_bump_finger(
+ func_env.alias_regions.vm_null_heap_data().next().store(
&mut builder.cursor(),
ptr_to_next,
end_of_object,
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index 6da4e3b2cd..d59f3c16af 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -212,6 +212,15 @@ macro_rules! declare_builtin_index {
$for_each_builtin!(declare_builtin_index_constructors);
}
+ // Builtins are reached by indexing the array of function pointers that
+ // a vmctx points at.
+ impl $crate::VmctxArrayIndex for $index_name {
+ #[inline]
+ fn vmctx_array_index(self) -> u32 {
+ self.index()
+ }
+ }
+
#[cfg(test)]
impl arbitrary::Arbitrary<'_> for $index_name {
fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<Self> {
diff --git a/crates/environ/src/vmctxtypes.rs b/crates/environ/src/vmctxtypes.rs
index 25a65119c7..36fac62515 100644
--- a/crates/environ/src/vmctxtypes.rs
+++ b/crates/environ/src/vmctxtypes.rs
@@ -91,6 +91,17 @@
/// on the vmctx's shape. These fields get generated accessors that do not
/// need to be parameterized over `VMOffsets`, only `GetPtrSize`.
///
+/// * `#[pointee(<attrs> <Region> as <name> $([<IndexType>])? : <ty>)]`: this
+/// field is a pointer to memory that lives *outside* the vmctx and so is not
+/// part of its layout at all, but that compiled code reaches only by loading
+/// this field first. `<Region>` names the alias region that memory belongs
+/// to, `<name>` is the accessor generated for it, and `<ty>` is the type it is
+/// accessed as. With an `[<IndexType>]`, the pointee is an array of `<ty>`
+/// indexed by `<IndexType>` and each element gets its own alias region;
+/// without one, it is a single `<ty>`. The pointee's `<attrs>` are its own:
+/// how compiled code may treat a load of the pointee is independent of how it
+/// may treat a load of the pointer.
+///
/// Doc comments are deliberately *not* accepted on fields; use `//` comments for
/// prose about the layout. Accessor documentation is synthesized from the field
/// names instead, so that there is only one place a field can be described.
@@ -121,15 +132,41 @@ macro_rules! for_each_vmctx_type {
field { #[readonly] #[can_move] store_context: VmPtr<VMStoreContext> }
- field { #[readonly] #[can_move] builtin_functions: VmPtr<VMBuiltinFunctionsArray> }
+ // NB: `VMBuiltinFunctionsArray` is a `repr(C)` struct of
+ // `unsafe extern "C" fn` fields rather than a true array,
+ // so its elements are pointer-width and pointer-aligned.
+ field {
+ #[readonly]
+ #[can_move]
+ #[pointee(
+ #[readonly]
+ #[can_move]
+ BuiltinFunctionsArray as builtin_functions_array[BuiltinFunctionIndex]:
+ unsafe extern "C" fn
+ )]
+ builtin_functions: VmPtr<VMBuiltinFunctionsArray>
+ }
- field { epoch_ptr: VmPtr<AtomicU64> }
+ field {
+ #[pointee(EpochCounter as epoch_counter: AtomicU64)]
+ epoch_ptr: VmPtr<AtomicU64>
+ }
// A pointer that different collectors use however they see
// fit.
field { #[readonly] #[can_move] gc_heap_data: VmPtr<u8> }
- field { #[readonly] #[can_move] type_ids: VmPtr<VMSharedTypeIndex> }
+ field {
+ #[readonly]
+ #[can_move]
+ #[pointee(
+ #[readonly]
+ #[can_move]
+ TypeIdsArray as type_ids_array[ModuleInternedTypeIndex]:
+ VMSharedTypeIndex
+ )]
+ type_ids: VmPtr<VMSharedTypeIndex>
+ }
}
// Variable-width fields come after the fixed-width fields
@@ -222,7 +259,16 @@ macro_rules! for_each_vmctx_type {
align { ptr }
- field { #[readonly] builtins: VmPtr<VMComponentBuiltins> }
+ field {
+ #[readonly]
+ #[pointee(
+ #[readonly]
+ #[can_move]
+ ComponentBuiltinFunctionsArray as builtins_array[ComponentBuiltinFunctionIndex]:
+ unsafe extern "C" fn
+ )]
+ builtins: VmPtr<VMComponentBuiltins>
+ }
field { #[readonly] #[can_move] store_context: VmPtr<VMStoreContext> }
}
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 3293e9745d..9c877d9526 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -8,8 +8,8 @@
use crate::{
DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex,
- FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex,
- TagIndex,
+ FuncRefIndex, GlobalIndex, MemoryIndex, Module, ModuleInternedTypeIndex, OwnedMemoryIndex,
+ RuntimeDataIndex, TableIndex, TagIndex,
};
/// Number of slots in for `component_context` in the `VMStoreContext`. This is
@@ -51,6 +51,10 @@ macro_rules! define_vm_type_offsets {
(@size ($p:expr) i64) => { 8u32 };
(@size ($p:expr) u64) => { 8u32 };
(@size ($p:expr) u32) => { 4u32 };
+ // `VMGcRef` is a `NonZeroU32` newtype, so `Option<VMGcRef>` is niche-packed
+ // back down into four bytes.
+ (@size ($p:expr) NonZeroU32) => { 4u32 };
+ (@size ($p:expr) Option < VMGcRef >) => { 4u32 };
(@size ($p:expr) [u8; 16]) => { 16u32 };
(@size ($p:expr) [u32; $n:expr]) => { 4u32 * u32::try_from($n).unwrap() };
(@size ($p:expr) VMSharedTypeIndex) => { u32::from(($p).size_of_vmshared_type_index()) };
@@ -85,6 +89,8 @@ macro_rules! define_vm_type_offsets {
(@align ($p:expr) i64) => { 8u32 };
(@align ($p:expr) u64) => { 8u32 };
(@align ($p:expr) u32) => { 4u32 };
+ (@align ($p:expr) NonZeroU32) => { 4u32 };
+ (@align ($p:expr) Option < VMGcRef >) => { 4u32 };
(@align ($p:expr) [u8; 16]) => { 16u32 };
(@align ($p:expr) [u32; $n:expr]) => { 4u32 };
(@align ($p:expr) VMSharedTypeIndex) => { u32::from(($p).align_of_vmshared_type_index()) };
@@ -205,6 +211,7 @@ macro_rules! define_vm_type_offsets {
// Top-level entry: the list of `VM*` type definitions.
( $(
$(#[doc = $sdoc:literal])*
+ $(#[cfg($($scfg:tt)*)])?
$(#[derive($($d:ident),*)])?
#[repr($($repr:tt)*)]
#[snake_name = $snake:ident]
@@ -358,11 +365,6 @@ pub mod offsets {
for_each_vmctx_type!(define_vmctx_static_offsets);
}
-/// The size, in bytes, of one `context.{get,set}` slot. These slots are `u32`s,
-/// both in `VMStoreContext::component_context` and in
-/// `VMDeferredThread::saved_context`.
-const COMPONENT_CONTEXT_SLOT_SIZE: u8 = 4;
-
/// Offsets within a `VMStoreContext` that are not simply the offset of one of
/// its fields, and so are not generated by `for_each_vm_type!`.
impl<P: PtrSize> offsets::VMStoreContext<P> {
@@ -380,22 +382,6 @@ impl<P: PtrSize> offsets::VMStoreContext<P> {
debug_assert!(offset < self.last_wasm_exit_trampoline_fp());
offset
}
-
- /// The offset of the `component_context[i]` slot within a `VMStoreContext`.
- pub fn component_context_slot(&self, i: u8) -> u8 {
- assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);
- self.component_context() + i * COMPONENT_CONTEXT_SLOT_SIZE
- }
-}
-
-/// Offsets within a `VMDeferredThread` that are not simply the offset of one of
-/// its fields, and so are not generated by `for_each_vm_type!`.
-impl<P: PtrSize> offsets::VMDeferredThread<P> {
- /// The offset of the `saved_context[i]` slot within a `VMDeferredThread`.
- pub fn saved_context_slot(&self, i: u8) -> u8 {
- assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);
- self.saved_context() + i * COMPONENT_CONTEXT_SLOT_SIZE
- }
}
/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to `PtrSize` for
@@ -403,6 +389,7 @@ impl<P: PtrSize> offsets::VMDeferredThread<P> {
macro_rules! define_ptr_size_vm_type_accessors {
( $(
$(#[doc = $sdoc:literal])*
+ $(#[cfg($($scfg:tt)*)])?
$(#[derive($($d:ident),*)])?
#[repr($($repr:tt)*)]
#[snake_name = $snake:ident]
@@ -688,65 +675,6 @@ pub trait PtrSize {
fn vmcontref_values(&self) -> u8 {
self.vmcontref_args() + self.size_of_vmhostarray()
}
-
- /// Return the offset of the `over_approximated_stack_roots` field within
- /// `VMDrcHeapData`.
- #[inline]
- fn vmdrc_heap_data_over_approximated_stack_roots(&self) -> u8 {
- 0
- }
-
- /// Return the offset of the `current_over_approximated_stack_roots_len`
- /// field within `VMDrcHeapData`.
- #[inline]
- fn vmdrc_heap_data_current_over_approximated_stack_roots_len(&self) -> u8 {
- 4
- }
-
- /// Return the offset of the
- /// `over_approximated_stack_roots_len_after_last_gc` field within
- /// `VMDrcHeapData`.
- #[inline]
- fn vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(&self) -> u8 {
- 8
- }
-
- /// Return the size of `VMDrcHeapData`.
- #[inline]
- fn size_of_vmdrc_heap_data(&self) -> u8 {
- 12
- }
-
- /// Return the alignment of `VMDrcHeapData`.
- #[inline]
- fn align_of_vmdrc_heap_data(&self) -> u8 {
- 4
- }
-
- /// Return the offset of the `bump_ptr` field within `VMCopyingHeapData`.
- #[inline]
- fn vmcopying_heap_data_bump_ptr(&self) -> u8 {
- 0
- }
-
- /// Return the offset of the `active_space_end` field within
- /// `VMCopyingHeapData`.
- #[inline]
- fn vmcopying_heap_data_active_space_end(&self) -> u8 {
- 4
- }
-
- /// Return the size of `VMCopyingHeapData`.
- #[inline]
- fn size_of_vmcopying_heap_data(&self) -> u8 {
- 8
- }
-
- /// Return the alignment of `VMCopyingHeapData`.
- #[inline]
- fn align_of_vmcopying_heap_data(&self) -> u8 {
- 4
- }
}
/// A trait to abstract over various types that contain a `P: PtrSize`.
@@ -1023,6 +951,7 @@ impl_vmctx_array_index! {
DefinedTagIndex,
FuncRefIndex,
RuntimeDataIndex,
+ ModuleInternedTypeIndex,
}
/// Generate the accessors for the offsets of `VMContext`'s
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
index e6eadf1d4a..71ba0f3691 100644
--- a/crates/environ/src/vmtypes.rs
+++ b/crates/environ/src/vmtypes.rs
@@ -20,6 +20,14 @@
///
/// * Doc-comment attributes (`#[doc = "..."]`).
///
+/// * An optional `#[cfg(...)]` attribute, gating the type on the Cargo features
+/// whose code actually uses it.
+///
+/// Only the runtime's `struct` definition (and its layout test) is gated by
+/// it. A type's offsets and alias regions are always generated because the
+/// `wasmtime-environ` and `wasmtime-internal-cranelift` crates do not have
+/// Cargo features for all the various Wasm features.
+///
/// * An optional `#[derive(...)]` attribute.
///
/// * A `#[repr(...)]` attribute.
@@ -37,6 +45,13 @@
/// is still generated, since that is what interior accesses are computed
/// relative to.
///
+/// * `#[indexed]`: this field is a fixed-size array of a scalar type, whose
+/// elements compiled Wasm code accesses individually by a compile-time
+/// constant index. Like `#[aggregate]`, the field has no single Cranelift
+/// type; unlike `#[aggregate]`, its elements all do, so an alias-region
+/// accessor taking that index is generated for it and each element gets its
+/// own alias region.
+///
/// * `#[readonly]` and/or `#[can_move]`: describe how Cranelift may treat loads
/// and stores of that field.
#[macro_export]
@@ -456,7 +471,7 @@ macro_rules! for_each_vm_type {
/// This is saved/restored when threads are swapped in the component model.
///
/// NB: `UnsafeCell` because JIT code writes to the slots.
- #[aggregate]
+ #[indexed]
pub component_context: UnsafeCell<[u32; NUM_COMPONENT_CONTEXT_SLOTS]>,
/// JIT-visible current thread for the component model's sync-to-sync
@@ -542,9 +557,68 @@ macro_rules! for_each_vm_type {
pub callee_instance: u32,
/// The caller thread's `context.{get,set}` slots, saved on entry and
/// restored on the fast-path exit (or recovered while forcing).
- #[aggregate]
+ #[indexed]
pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
}
+
+ /// The deferred-reference-counting collector's JIT-accessible heap
+ /// data.
+ ///
+ /// This is a separate allocation, reached through the
+ /// `VMContext::gc_heap_data` pointer. Its fields are not GC heap
+ /// locations, so they get this type's own alias regions rather than
+ /// the GC heap's.
+ ///
+ /// `wasmtime::runtime::vm::gc::enabled::drc` owns one of these,
+ /// wrapped in a cell because compiled Wasm writes to it, and
+ /// accesses it only through that wrapper's methods.
+ #[cfg(feature = "gc-drc")]
+ #[derive(Default)]
+ #[repr(C)]
+ #[snake_name = vm_drc_heap_data]
+ pub struct VMDrcHeapData {
+ /// The head of the over-approximated-stack-roots list.
+ pub over_approximated_stack_roots: Option<VMGcRef>,
+
+ /// The current size of the over-approximated-stack-roots list.
+ pub current_over_approximated_stack_roots_len: u32,
+
+ /// The size of the over-approximated-stack-roots list
+ /// immediately after the last GC.
+ pub over_approximated_stack_roots_len_after_last_gc: u32,
+ }
+
+ /// The copying collector's JIT-accessible bump-allocation state.
+ ///
+ /// Like [`VMDrcHeapData`], this is a separate allocation reached
+ /// through `VMContext::gc_heap_data`, and is owned, wrapped in a
+ /// cell, by `wasmtime::runtime::vm::gc::enabled::copying`.
+ #[cfg(feature = "gc-copying")]
+ #[derive(Default)]
+ #[repr(C)]
+ #[snake_name = vm_copying_heap_data]
+ pub struct VMCopyingHeapData {
+ /// Current bump pointer (an index into the GC heap).
+ pub bump_ptr: u32,
+
+ /// End of the active semi-space.
+ pub active_space_end: u32,
+ }
+
+ /// The null collector's JIT-accessible bump-allocation state.
+ ///
+ /// Unlike the two above, this is not a separate allocation: it is
+ /// the first field of `NullHeap` (in
+ /// `wasmtime::runtime::vm::gc::enabled::null`), again wrapped in a
+ /// cell, and compiled Wasm reaches it through a pointer to that
+ /// field.
+ #[cfg(feature = "gc-null")]
+ #[repr(C)]
+ #[snake_name = vm_null_heap_data]
+ pub struct VMNullHeapData {
+ /// The bump-allocation finger, an index into the GC heap.
+ pub next: NonZeroU32,
+ }
}
};
}
diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs
index 2f503d1606..5e77c7abd5 100644
--- a/crates/wasmtime/src/runtime/vm.rs
+++ b/crates/wasmtime/src/runtime/vm.rs
@@ -126,8 +126,14 @@ pub use crate::runtime::vm::throw::*;
pub use crate::runtime::vm::traphandlers::*;
#[cfg(feature = "component-model")]
pub use crate::runtime::vm::vmcontext::VMArrayCallFunction;
+#[cfg(feature = "gc-copying")]
+pub use crate::runtime::vm::vmcontext::VMCopyingHeapData;
+#[cfg(feature = "gc-drc")]
+pub use crate::runtime::vm::vmcontext::VMDrcHeapData;
#[cfg(feature = "component-model-async")]
pub use crate::runtime::vm::vmcontext::VMLazyThread;
+#[cfg(feature = "gc-null")]
+pub use crate::runtime::vm::vmcontext::VMNullHeapData;
pub use crate::runtime::vm::vmcontext::{
VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition,
VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext,
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
index cb0ba1dd04..cafb4d0bc8 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
@@ -14,8 +14,8 @@ use super::VMArrayRef;
use super::trace_infos::TraceInfos;
use crate::runtime::vm::{
ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,
- GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMExternRef,
- VMGcHeader, VMGcRef, VMMemoryDefinition,
+ GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMCopyingHeapData,
+ VMExternRef, VMGcHeader, VMGcRef, VMMemoryDefinition,
};
use crate::{Engine, bail_bug, prelude::*};
use core::{
@@ -176,26 +176,20 @@ unsafe impl GcHeapObject for VMCopyingExternRef {
}
}
-/// JIT-accessible bump-allocation state for the copying collector.
+/// A [`VMCopyingHeapData`] as this heap owns it.
///
-/// NB: Layout is defined by constants in `wasmtime_environ::copying`. Keep in
-/// sync!
-#[derive(Default)]
-#[repr(C)]
-struct VMCopyingHeapDataInner {
- /// Current bump pointer (index into the GC heap).
- bump_ptr: u32,
- /// End of the active semi-space.
- active_space_end: u32,
-}
-
+/// Compiled Wasm holds a pointer to the inner `VMCopyingHeapData` and bumps its
+/// allocation pointer, so all of the runtime's own accesses go through an
+/// `UnsafeCell`. Both this wrapper and `SendSyncUnsafeCell` are
+/// `repr(transparent)`, so the layout that compiled code sees is exactly the
+/// generated one.
#[derive(Default)]
#[repr(transparent)]
-struct VMCopyingHeapData {
- inner: SendSyncUnsafeCell<VMCopyingHeapDataInner>,
+struct VMCopyingHeapDataCell {
+ inner: SendSyncUnsafeCell<VMCopyingHeapData>,
}
-impl VMCopyingHeapData {
+impl VMCopyingHeapDataCell {
fn bump_ptr(&self) -> u32 {
// Safety: `inner` is valid to read from.
unsafe { (*self.inner.get()).bump_ptr }
@@ -266,7 +260,7 @@ struct CopyingHeap {
///
/// NB: The bump pointer is written to by compiled Wasm code via the pointer
/// returned by `vmctx_gc_heap_data`.
- vmctx_data: VMCopyingHeapData,
+ vmctx_data: VMCopyingHeapDataCell,
/// The start of the active semi-space.
active_space_start: u32,
@@ -307,7 +301,7 @@ impl CopyingHeap {
no_gc_count: 0,
memory: None,
vmmemory: None,
- vmctx_data: VMCopyingHeapData::default(),
+ vmctx_data: VMCopyingHeapDataCell::default(),
active_space_start: 0,
idle_space_start: 0,
idle_space_end: 0,
@@ -1220,7 +1214,6 @@ impl Drop for CopyingCollection<'_, '_> {
#[cfg(test)]
mod tests {
use super::*;
- use wasmtime_environ::{HostPtr, PtrSize};
#[test]
fn vm_copying_header_size_align() {
@@ -1272,46 +1265,6 @@ mod tests {
);
}
- #[test]
- fn vm_copying_heap_data_bump_ptr_offset() {
- assert_eq!(
- HostPtr.vmcopying_heap_data_bump_ptr() as usize,
- core::mem::offset_of!(VMCopyingHeapDataInner, bump_ptr),
- );
- }
-
- #[test]
- fn vm_copying_heap_data_active_space_end_offset() {
- assert_eq!(
- HostPtr.vmcopying_heap_data_active_space_end() as usize,
- core::mem::offset_of!(VMCopyingHeapDataInner, active_space_end),
- );
- }
-
- #[test]
- fn vm_copying_heap_data_size() {
- assert_eq!(
- HostPtr.size_of_vmcopying_heap_data() as usize,
- core::mem::size_of::<VMCopyingHeapData>(),
- );
- assert_eq!(
- HostPtr.size_of_vmcopying_heap_data() as usize,
- core::mem::size_of::<VMCopyingHeapDataInner>(),
- );
- }
-
- #[test]
- fn vm_copying_heap_data_align() {
- assert_eq!(
- HostPtr.align_of_vmcopying_heap_data() as usize,
- core::mem::align_of::<VMCopyingHeapData>(),
- );
- assert_eq!(
- HostPtr.align_of_vmcopying_heap_data() as usize,
- core::mem::align_of::<VMCopyingHeapDataInner>(),
- );
- }
-
#[test]
fn semi_space_split_stays_aligned() {
fn check(len: usize) {
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index 9532e43f05..0252fd36c6 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -53,7 +53,7 @@ use crate::runtime::vm::{
GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TraceInfo, TypedGcRef, VMExternRef,
VMGcHeader, VMGcObjectData, VMGcRef,
};
-use crate::vm::VMMemoryDefinition;
+use crate::vm::{VMDrcHeapData, VMMemoryDefinition};
use crate::{Engine, Trap, bail_bug, prelude::*};
use core::sync::atomic::AtomicUsize;
use core::{
@@ -97,28 +97,17 @@ unsafe impl GcRuntime for DrcCollector {
}
}
-/// JIT-accessible DRC heap data.
-#[derive(Default)]
-#[repr(C)]
-struct VMDrcHeapDataInner {
- /// The head of the over-approximated-stack-roots list.
- over_approximated_stack_roots: Option<VMGcRef>,
-
- /// The current size of the over-approximated-stack-roots list.
- current_over_approximated_stack_roots_len: u32,
-
- /// The size of the over-approximated-stack-roots list immediately after the
- /// last GC.
- over_approximated_stack_roots_len_after_last_gc: u32,
-}
-
+/// A `VMDrcHeapData` inside a `SendSyncUnsafeCell`.
+///
+/// Compiled Wasm holds a pointer to the inner `VMDrcHeapData` and writes to it,
+/// so all of the runtime's own accesses go through an `UnsafeCell`.
#[derive(Default)]
#[repr(transparent)]
-struct VMDrcHeapData {
- inner: SendSyncUnsafeCell<VMDrcHeapDataInner>,
+struct VMDrcHeapDataCell {
+ inner: SendSyncUnsafeCell<VMDrcHeapData>,
}
-impl VMDrcHeapData {
+impl VMDrcHeapDataCell {
fn over_approximated_stack_roots(&self) -> Option<VMGcRef> {
// Safety: `inner` is valid to read from.
unsafe {
@@ -177,7 +166,7 @@ struct DrcHeap {
///
/// Note that this is exposed directly to compiled Wasm code through the
/// vmctx, so must not move.
- vmctx_data: Box<VMDrcHeapData>,
+ vmctx_data: Box<VMDrcHeapDataCell>,
/// The storage for the GC heap itself.
memory: Option<crate::vm::Memory>,
@@ -928,7 +917,7 @@ unsafe impl GcHeap for DrcHeap {
} = self;
*no_gc_count = 0;
- **vmctx_data = VMDrcHeapData::default();
+ **vmctx_data = VMDrcHeapDataCell::default();
*free_list = None;
*vmmemory = None;
*allocated_bytes = 0;
@@ -1194,7 +1183,7 @@ unsafe impl GcHeap for DrcHeap {
}
unsafe fn vmctx_gc_heap_data(&self) -> NonNull<u8> {
- let ptr: NonNull<VMDrcHeapData> = NonNull::from(&*self.vmctx_data);
+ let ptr: NonNull<VMDrcHeapDataCell> = NonNull::from(&*self.vmctx_data);
ptr.cast()
}
@@ -1335,7 +1324,7 @@ impl<T> DerefMut for DebugOnly<T> {
#[cfg(test)]
mod tests {
use super::*;
- use wasmtime_environ::{HostPtr, PtrSize};
+ use wasmtime_environ::HostPtr;
#[test]
fn vm_drc_header_size_align() {
@@ -1393,58 +1382,4 @@ mod tests {
u32::try_from(actual_offset).unwrap(),
);
}
-
- #[test]
- fn vm_drc_heap_data_over_approximated_stack_roots_offset() {
- assert_eq!(
- HostPtr.vmdrc_heap_data_over_approximated_stack_roots() as usize,
- core::mem::offset_of!(VMDrcHeapDataInner, over_approximated_stack_roots),
- );
- }
-
- #[test]
- fn vm_drc_heap_data_current_over_approximated_stack_roots_len_offset() {
- assert_eq!(
- HostPtr.vmdrc_heap_data_current_over_approximated_stack_roots_len() as usize,
- core::mem::offset_of!(
- VMDrcHeapDataInner,
- current_over_approximated_stack_roots_len
- ),
- );
- }
-
- #[test]
- fn vm_drc_heap_data_over_approximated_stack_roots_len_after_last_gc_offset() {
- assert_eq!(
- HostPtr.vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc() as usize,
- core::mem::offset_of!(
- VMDrcHeapDataInner,
- over_approximated_stack_roots_len_after_last_gc
- ),
- );
- }
-
- #[test]
- fn vm_drc_heap_data_size() {
- assert_eq!(
- HostPtr.size_of_vmdrc_heap_data() as usize,
- core::mem::size_of::<VMDrcHeapData>(),
- );
- assert_eq!(
- HostPtr.size_of_vmdrc_heap_data() as usize,
- core::mem::size_of::<VMDrcHeapDataInner>(),
- );
- }
-
- #[test]
- fn vm_drc_heap_data_align() {
- assert_eq!(
- HostPtr.align_of_vmdrc_heap_data() as usize,
- core::mem::align_of::<VMDrcHeapData>(),
- );
- assert_eq!(
- HostPtr.align_of_vmdrc_heap_data() as usize,
- core::mem::align_of::<VMDrcHeapDataInner>(),
- );
- }
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
index c4f06f3dd4..6ad8a79e70 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
@@ -11,7 +11,7 @@ use crate::{
vm::{
ExternRefHostDataId, GarbageCollection, GcHeap, GcHeapObject, GcProgress, GcRootsIter,
GcRuntime, GcStoreTraceState, SendSyncUnsafeCell, TypedGcRef, VMGcHeader, VMGcRef,
- VMMemoryDefinition,
+ VMMemoryDefinition, VMNullHeapData,
},
};
use core::ptr::NonNull;
@@ -38,14 +38,26 @@ unsafe impl GcRuntime for NullCollector {
}
}
+impl VMNullHeapData {
+ /// The heap data for a detached heap.
+ fn detached() -> Self {
+ VMNullHeapData {
+ next: NonZeroU32::new(u32::MAX).unwrap(),
+ }
+ }
+}
+
/// A GC heap for the null collector.
+///
+/// Compiled Wasm reaches the JIT-accessible bump-allocation state through a
+/// pointer to `vmctx_data`, and accesses nothing else in here, so that field
+/// must stay first.
#[repr(C)]
struct NullHeap {
- /// Bump-allocation finger indexing within `1..self.heap.len()`.
+ /// The bump-allocation finger, indexing within `1..self.heap.len()`.
///
- /// NB: this is an `UnsafeCell` because it is written to by compiled Wasm
- /// code.
- next: SendSyncUnsafeCell<NonZeroU32>,
+ /// NB: this is in a cell because it is written to by compiled Wasm code.
+ vmctx_data: SendSyncUnsafeCell<VMNullHeapData>,
/// The number of active no-gc scopes at the current moment.
no_gc_count: usize,
@@ -111,7 +123,7 @@ impl NullHeap {
fn new() -> Result<Self> {
Ok(Self {
no_gc_count: 0,
- next: SendSyncUnsafeCell::new(NonZeroU32::new(u32::MAX).unwrap()),
+ vmctx_data: SendSyncUnsafeCell::new(VMNullHeapData::detached()),
memory: None,
})
}
@@ -140,7 +152,7 @@ impl NullHeap {
None => return Err(crate::Trap::AllocationTooLarge.into()),
};
- let next = *self.next.get_mut();
+ let next = self.vmctx_data.get_mut().next;
// Increment the bump pointer to the layout's requested alignment.
let aligned = match u32::try_from(layout.align())
@@ -163,7 +175,7 @@ impl NullHeap {
}
// Update the bump pointer, write the header, and return the GC ref.
- *self.next.get_mut() = NonZeroU32::new(end_of_object).unwrap();
+ self.vmctx_data.get_mut().next = NonZeroU32::new(end_of_object).unwrap();
let aligned = NonZeroU32::new(aligned).unwrap();
let gc_ref = VMGcRef::from_heap_index(aligned).unwrap();
@@ -184,22 +196,21 @@ unsafe impl GcHeap for NullHeap {
fn attach(&mut self, memory: crate::vm::Memory) {
assert!(!self.is_attached());
self.memory = Some(memory);
- self.next = SendSyncUnsafeCell::new(NonZeroU32::new(1).unwrap());
+ self.vmctx_data.get_mut().next = NonZeroU32::new(1).unwrap();
}
fn detach(&mut self) -> crate::vm::Memory {
assert!(self.is_attached());
let NullHeap {
- next,
+ vmctx_data,
no_gc_count,
memory,
} = self;
- *next.get_mut() = NonZeroU32::new(1).unwrap();
+ *vmctx_data.get_mut() = VMNullHeapData::detached();
*no_gc_count = 0;
- self.next = SendSyncUnsafeCell::new(NonZeroU32::new(u32::MAX).unwrap());
memory.take().unwrap()
}
@@ -337,7 +348,7 @@ unsafe impl GcHeap for NullHeap {
// the heap up to the bump pointer is allocated. Subtract 1 because
// the bump pointer starts at index 1 (index 0 is unused since
// `VMGcRef` uses `NonZeroU32`), not because any byte was allocated.
- let next = unsafe { *self.next.get() };
+ let next = unsafe { (*self.vmctx_data.get()).next };
usize::try_from(next.get()).unwrap() - 1
}
@@ -354,8 +365,8 @@ unsafe impl GcHeap for NullHeap {
}
unsafe fn vmctx_gc_heap_data(&self) -> NonNull<u8> {
- let ptr_to_next: *mut NonZeroU32 = unsafe { self.next.get() };
- NonNull::new(ptr_to_next).unwrap().cast()
+ let ptr: *mut VMNullHeapData = unsafe { self.vmctx_data.get() };
+ NonNull::new(ptr).unwrap().cast()
}
}
diff --git a/crates/wasmtime/src/runtime/vm/send_sync_unsafe_cell.rs b/crates/wasmtime/src/runtime/vm/send_sync_unsafe_cell.rs
index f1b16d1a94..b20bc5baf1 100644
--- a/crates/wasmtime/src/runtime/vm/send_sync_unsafe_cell.rs
+++ b/crates/wasmtime/src/runtime/vm/send_sync_unsafe_cell.rs
@@ -2,7 +2,10 @@ use core::cell::UnsafeCell;
/// A wrapper around `UnsafeCell` that implements `Send` and `Sync` for types
/// that are themselves `Send` and `Sync`.
+///
+/// This type is `repr(transparent)` and has exactly `T`'s layout.
#[derive(Default)]
+#[repr(transparent)]
pub struct SendSyncUnsafeCell<T>(UnsafeCell<T>);
// Safety: `T` is `Send` and users guarantee that any pointers derived from the
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 7d02bfb98b..17afb693c1 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -13,6 +13,8 @@ use core::ffi::c_void;
use core::fmt;
use core::marker;
use core::mem::{self, MaybeUninit};
+#[cfg(feature = "gc-null")]
+use core::num::NonZeroU32;
use core::ops::Range;
use core::ptr::{self, NonNull};
use core::sync::atomic::{AtomicUsize, Ordering};
@@ -196,6 +198,7 @@ unsafe impl VmSafe for VMTagImport {}
macro_rules! define_vm_types {
( $(
$(#[doc = $sdoc:literal])*
+ $(#[cfg($($scfg:tt)*)])?
$(#[derive($($d:ident),*)])?
#[repr($($repr:tt)*)]
#[snake_name = $snake:ident]
@@ -203,6 +206,7 @@ macro_rules! define_vm_types {
$(
$(#[doc = $fdoc:literal])*
$(#[aggregate])?
+ $(#[indexed])?
$(#[readonly])?
$(#[can_move])?
$fvis:vis $fname:ident : $fty:tt $(< $fgen:ty >)? ,
@@ -211,6 +215,7 @@ macro_rules! define_vm_types {
)* ) => {
$(
$(#[doc = $sdoc])*
+ $(#[cfg($($scfg)*)])?
$(#[derive($($d),*)])?
#[repr($($repr)*)]
$svis struct $Name {
@@ -223,13 +228,15 @@ macro_rules! define_vm_types {
#[cfg(test)]
mod test_vm_type_layouts {
- use super::{ $( $Name, )* };
use core::mem::{align_of, offset_of, size_of};
use wasmtime_environ::{HostPtr, PtrSize};
$(
+ $(#[cfg($($scfg)*)])?
#[test]
fn $snake() {
+ use super::$Name;
+
let host = HostPtr;
let offsets = host.$snake();
@@ -823,11 +830,10 @@ mod test_vmstore_context {
use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
/// Check the `VMStoreContext` offsets that `for_each_vm_type!` does *not*
- /// generate: the offsets reaching into the inlined `gc_heap`, and the
- /// indexed `component_context` slot accessor.
+ /// generate: the offsets reaching into the inlined `gc_heap`.
///
- /// Every plain field offset, plus the size and alignment of the type, is
- /// already checked by the generated `test_vm_type_layouts::vm_store_context`.
+ /// Every field offset, plus the size and alignment of the type, is already
+ /// checked by the generated `test_vm_type_layouts::vm_store_context`.
#[test]
fn derived_field_offsets() {
let module = Module::new(StaticModuleIndex::from_u32(0));
@@ -840,20 +846,6 @@ mod test_vmstore_context {
offset_of!(VMStoreContext, gc_heap) + offset_of!(VMMemoryDefinition, current_length),
usize::from(offsets.ptr.vm_store_context().gc_heap_current_length())
);
- assert_eq!(
- offset_of!(VMStoreContext, component_context),
- usize::from(offsets.ptr.vm_store_context().component_context_slot(0))
- );
-
- // Make sure that the calculation for the size of a slot is also
- // accurate.
- let slot_width = offsets.ptr.vm_store_context().component_context_slot(1)
- - offsets.ptr.vm_store_context().component_context_slot(0);
- let mut default = VMStoreContext::default();
- assert_eq!(
- size_of_val(&default.component_context.get_mut()[0]),
- usize::from(slot_width)
- );
}
}
@@ -924,30 +916,6 @@ mod test_vmlazy_thread {
}
}
-#[cfg(test)]
-mod test_vmdeferred_thread {
- use super::*;
- use core::mem::offset_of;
- use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
-
- /// Check the indexed `saved_context` slot accessor, which
- /// `for_each_vm_type!` does not generate.
- ///
- /// Every plain field offset, plus the size and alignment of the type, is
- /// already checked by the generated
- /// `test_vm_type_layouts::vm_deferred_thread`.
- #[test]
- fn deferred_thread_derived_field_offsets() {
- let module = Module::new(StaticModuleIndex::from_u32(0));
- let offsets = VMOffsets::new(HostPtr, &module);
- let ptr = offsets.ptr;
- assert_eq!(
- offset_of!(VMDeferredThread, saved_context),
- usize::from(ptr.vm_deferred_thread().saved_context_slot(0))
- );
- }
-}
-
/// The VM "context", which is pointed to by the `vmctx` arg in Cranelift.
/// This has information about globals, memories, tables, and other runtime
/// state associated with the current instance.
diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rs
index abfc88bea6..8f3a1b45a3 100644
--- a/winch/codegen/src/codegen/drc.rs
+++ b/winch/codegen/src/codegen/drc.rs
@@ -243,13 +243,15 @@ where
self.env
.vmoffsets
.ptr
- .vmdrc_heap_data_over_approximated_stack_roots(),
+ .vm_drc_heap_data()
+ .over_approximated_stack_roots(),
);
let roots_len_offset = u32::from(
self.env
.vmoffsets
.ptr
- .vmdrc_heap_data_current_over_approximated_stack_roots_len(),
+ .vm_drc_heap_data()
+ .current_over_approximated_stack_roots_len(),
);
let next_offset = self
.env
@@ -341,7 +343,8 @@ where
self.env
.vmoffsets
.ptr
- .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(),
+ .vm_drc_heap_data()
+ .over_approximated_stack_roots_len_after_last_gc(),
);
let skip_gc = self.masm.get_label()?;
self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {