Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Define `VMContext` and `VMComponentContext` in a macro (#14083)

Nick Fitzgerald2ecd55d74146
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index 873f34b7a2..9963f5ef22 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -5,13 +5,12 @@ use cranelift_codegen::{
ir::{self, InstBuilder as _},
};
use wasmtime_environ::{
- BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex,
- GetPtrSize, GlobalIndex, MemoryIndex, ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex,
- StaticModuleIndex, TableIndex, TagIndex, VMOffsets,
+ BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, GetPtrSize,
+ ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex, StaticModuleIndex, VMOffsets,
component::{
ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,
RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,
- VMComponentOffsets,
+ RuntimeReallocIndex, VMComponentOffsets,
},
};
@@ -501,10 +500,6 @@ impl<'a, Offsets> Field<'a, Offsets> {
/// Emit a store of `value` to this field relative to `ptr`, a pointer to
/// the containing `VM*` structure.
- #[allow(
- dead_code,
- reason = "part of the general `Field` API; not all fields are stored to yet"
- )]
pub fn store(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value, value: ir::Value) {
let flags = self.flags_with_region(cursor.func);
cursor
@@ -703,594 +698,294 @@ macro_rules! define_vm_type_alias_region_helpers {
}
wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers);
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- /// Create a new `AliasRegions`.
- pub fn new(offsets: Offsets) -> Self {
- Self {
- pointer_type: ir::Type::int_with_byte_size(offsets.get_ptr_size().size().into())
- .unwrap(),
- offsets,
- cache: std::collections::HashMap::default(),
- }
- }
-
- /// Get the alias region for accesses into the GC heap.
- pub fn gc_heap_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
- self.region(func, AliasRegionKey::GcHeap)
- }
-
- /// Get the alias region for an imported or exported memory access (shared
- /// across all imported/exported memories).
- pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
- self.region(func, AliasRegionKey::PublicMemory)
- }
-
- /// Get the alias region for accessing a defined memory that is not
- /// exported.
- pub fn defined_memory_region(
- &mut self,
- func: &mut ir::Function,
- module: StaticModuleIndex,
- index: DefinedMemoryIndex,
- ) -> ir::AliasRegion {
- self.region(func, AliasRegionKey::DefinedMemory { module, index })
- }
-
- /// Get the alias region for an imported or exported table access (shared
- /// across all imported/exported memories).
- pub fn public_table_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
- self.region(func, AliasRegionKey::PublicTable)
- }
-
- /// Get the alias region for accessing a defined table that is not
- /// exported.
- pub fn defined_table_region(
- &mut self,
- func: &mut ir::Function,
- module: StaticModuleIndex,
- index: DefinedTableIndex,
- ) -> ir::AliasRegion {
- self.region(func, AliasRegionKey::DefinedTable { module, index })
- }
-
- /// Get the alias region for an imported or exported global access (shared
- /// across all imported/exported memories).
- pub fn public_global_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
- self.region(func, AliasRegionKey::PublicGlobal)
- }
-
- /// Get the alias region for accessing a defined global that is not
- /// exported.
- pub fn defined_global_region(
- &mut self,
- func: &mut ir::Function,
- module: StaticModuleIndex,
- index: DefinedGlobalIndex,
- ) -> ir::AliasRegion {
- self.region(func, AliasRegionKey::DefinedGlobal { module, index })
- }
-}
-
-/// `VMContext`-related methods that are valid for any `VMContext`, regardless
-/// of its particular `VMOffsets`.
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- /// Get the alias region for the given offset into the `VMContext`.
- fn vmctx_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {
- self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMContext,
- offset,
- },
- )
- }
+/// Define, for each of Wasmtime's vmctx types, an [`AliasRegions`] accessor
+/// that returns a wrapper exposing a [`Field`]-returning method per field of
+/// that vmctx.
+///
+/// For example, `alias_regions.vmctx().epoch_ptr()` is the `VMContext::epoch_ptr`
+/// field, and `alias_regions.vmcomponent().callbacks(i)` is the `i`th element of
+/// the `VMComponentContext`'s runtime-callbacks array.
+///
+/// A vmctx's `static` fields sit at offsets that depend only on the target
+/// pointer size, so their accessors are available for any `Offsets: GetPtrSize`.
+/// Its `dynamic` fields sit at offsets that additionally depend on the module or
+/// component being compiled, so their accessors are only available when the
+/// `AliasRegions` carries that vmctx's own fully-computed offsets.
+///
+/// A field marked `#[aggregate]` gets no accessor, for the same reason it gets
+/// none in [`define_vm_type_alias_region_helpers!`]: it has no single Cranelift
+/// type. Instead, its interior is reached by rebasing a [`Field`] of the nested
+/// `VM*` type onto the aggregate's own offset within the vmctx (see
+/// [`Field::relative_to`]), which produces exactly the same alias region as
+/// accessing that `VM*` type through a pointer would. For the few aggregates that
+/// have no `VM*` type of their own, and hence no alias region of their own, there
+/// are hand-written helpers below.
+#[allow(
+ unused_macro_rules,
+ reason = "entry shapes and marker attributes are handled uniformly for both \
+ sections of both vmctx types, but not every combination occurs"
+)]
+macro_rules! define_vmctx_alias_region_helpers {
+ // Classify a field type to its Cranelift `ir::Type`, given `$pt` (the target
+ // pointer type as an `ir::Type`).
+ (@field_ty $pt:expr, u32) => { ir::types::I32 };
+ (@field_ty $pt:expr, VmPtr < $g:ident >) => { $pt };
- fn vmctx_load(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- ty: ir::Type,
- base_flags: ir::MemFlagsData,
- vmctx: ir::Value,
- offset: u32,
- ) -> ir::Value {
- let region = self.vmctx_region(cursor.func, offset);
- cursor.ins().load(
- ty,
- base_flags.with_alias_region(Some(region)),
- vmctx,
- i32::try_from(offset).unwrap(),
- )
- }
+ // Determine the Cranelift type a field is *accessed* as: the classification
+ // of its `#[access_as = T]` type if it has one, and of its declared type
+ // otherwise.
+ (@access_ty $pt:expr, [ $($fty:tt)* ] []) => {
+ define_vmctx_alias_region_helpers!(@field_ty $pt, $($fty)*)
+ };
+ (@access_ty $pt:expr, $fty:tt [ #[access_as = $($t:tt)*] $($rest:tt)* ]) => {
+ define_vmctx_alias_region_helpers!(@field_ty $pt, $($t)*)
+ };
+ (@access_ty $pt:expr, $fty:tt [ # $skip:tt $($rest:tt)* ]) => {
+ define_vmctx_alias_region_helpers!(@access_ty $pt, $fty [ $($rest)* ])
+ };
- fn vmctx_store(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- base_flags: ir::MemFlagsData,
- vmctx: ir::Value,
- offset: u32,
- val: ir::Value,
- ) {
- let region = self.vmctx_region(cursor.func, offset);
- cursor.ins().store(
- base_flags.with_alias_region(Some(region)),
- val,
- vmctx,
- i32::try_from(offset).unwrap(),
+ // Apply a field attribute to its access flags.
+ (@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly() };
+ (@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() };
+ (@apply_attr $flags:expr, [access_as = $($t:tt)*]) => { $flags };
+
+ // Compute a field's access flags and Cranelift type from its declared type
+ // and marker attributes, and build the `Field` for it at `$offset`.
+ (@field $Name:ident ($self:expr, $offset:expr) [ $($fty:tt)* ] [ $(# $fattr:tt)* ]) => {{
+ let this = $self;
+ let flags = ir::MemFlagsData::trusted();
+ $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $fattr); )*
+ let ty = define_vmctx_alias_region_helpers!(
+ @access_ty this.regions.pointer_type, [ $($fty)* ] [ $(# $fattr)* ]
);
- }
+ let offset = $offset;
+ Field::new(this.regions, VmType::$Name, offset, flags, ty)
+ }};
- /// Load the `VMContext::magic` field.
- pub fn vmctx_magic(&mut self, cursor: &mut FuncCursor<'_>, vmctx: ir::Value) -> ir::Value {
- self.vmctx_load(
- cursor,
- ir::types::I32,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets.get_ptr_size().vmctx_magic().into(),
- )
- }
+ // ### `static` Section Entries
- /// Load the `*mut VMStoreContext` value out of the given `*mut VMContext`.
- pub fn vmctx_store_context(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- ) -> ir::Value {
- self.vmctx_store_context_load(cursor.func)
- .emit(cursor, vmctx)
- }
+ (@static_entry $Name:ident $snake:ident align { $al:tt }) => {};
- /// Get a `Load` for the `*mut VMStoreContext` value out of a `*mut VMContext`.
- pub fn vmctx_store_context_load(&mut self, func: &mut ir::Function) -> Load {
- let offset = u32::from(self.offsets.get_ptr_size().vmctx_store_context());
- let region = self.vmctx_region(func, offset);
- Load {
- offset,
- flags: ir::MemFlagsData::trusted()
- .with_readonly()
- .with_can_move()
- .with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
- }
-
- /// Load the `*mut i64` epoch pointer out of the given `*mut VMContext`.
- pub fn vmctx_epoch_ptr(&mut self, cursor: &mut FuncCursor<'_>, vmctx: ir::Value) -> ir::Value {
- self.vmctx_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.get_ptr_size().vmctx_epoch_ptr().into(),
- )
- }
+ // Aggregates get no accessor.
+ (@static_entry $Name:ident $snake:ident $kind:ident { #[aggregate] $($rest:tt)* }) => {};
- /// Load the base pointer of the `[VMSharedTypeIndex]` array out of the
- /// given `*mut VMContext`.
- pub fn vmctx_shared_type_ids_array(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- ) -> ir::Value {
- self.vmctx_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets.get_ptr_size().vmctx_type_ids_array().into(),
- )
- }
-
- /// Load the collector's heap data pointer out of the `*mut VMContext`.
- pub fn vmctx_gc_heap_data(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- ) -> ir::Value {
- self.vmctx_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets.get_ptr_size().vmctx_gc_heap_data().into(),
- )
- }
-
- /// Load the base pointer to the builtin-functions array from a `*mut
- /// VMContext`.
- pub fn vmctx_builtin_functions(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- ) -> ir::Value {
- self.vmctx_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets
- .get_ptr_size()
- .vmcontext_builtin_functions()
- .into(),
- )
- }
-}
-
-/// `VMContext`-related methods that are specific to a particular Wasm module's
-/// `VMOffsets`.
-impl AliasRegions<VMOffsets<u8>> {
- /// Like `vmctx_load`, but tags the load with a per-import alias region
- /// rather than the coarse `VMContext` region. Used for fields of the
- /// `VM*Import` structs inlined into the `VMContext`.
- fn vmimport_load(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- ty: ir::Type,
- base_flags: ir::MemFlagsData,
- vmctx: ir::Value,
- vmctx_offset: u32,
- field_offset: u32,
- vm_type: VmType,
- ) -> ir::Value {
- let region = self.region(
- cursor.func,
- AliasRegionKey::Vm {
- ty: vm_type,
- offset: field_offset,
- },
- );
- cursor.ins().load(
- ty,
- base_flags.with_alias_region(Some(region)),
- vmctx,
- i32::try_from(vmctx_offset).unwrap(),
- )
- }
-
- /// Load the imported tag's `VMTagImport::vmctx` field from the `*mut
- /// VMContext`.
- pub fn vmctx_vmtag_import_vmctx(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- tag: TagIndex,
- ) -> ir::Value {
- self.vmimport_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets.vmctx_vmtag_import_vmctx(tag),
- self.offsets.ptr.vm_tag_import().vmctx().into(),
- VmType::VMTagImport,
- )
- }
-
- /// Load the imported tag's `VMTagImport::index` field from the `*mut
- /// VMContext`.
- pub fn vmctx_vmtag_import_index(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- tag: TagIndex,
- ) -> ir::Value {
- self.vmimport_load(
- cursor,
- ir::types::I32,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets.vmctx_vmtag_import_index(tag),
- self.offsets.ptr.vm_tag_import().index().into(),
- VmType::VMTagImport,
- )
- }
+ (@static_entry $Name:ident $snake:ident field {
+ $(# $fattr:tt)* $fname:ident : $($fty:tt)*
+ }) => {
+ #[doc = concat!(
+ "Get the [`Field`] for the `", stringify!($fname), "` field of `",
+ stringify!($Name), "`."
+ )]
+ pub fn $fname(self) -> Field<'a, Offsets> {
+ let offset = u32::from(self.regions.offsets.get_ptr_size().$snake().$fname());
+ define_vmctx_alias_region_helpers!(
+ @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
+ )
+ }
+ };
- /// Load the imported tag's `VMTagImport::from` field from the `*mut
- /// VMContext`.
- pub fn vmctx_vmtag_import_from(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- tag: TagIndex,
- ) -> ir::Value {
- self.vmimport_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets.vmctx_vmtag_import_from(tag),
- self.offsets.ptr.vm_tag_import().from().into(),
- VmType::VMTagImport,
- )
- }
+ // ### `dynamic` Section Entries
- /// Load the import function's `VMFunctionImport::vmctx` field from the
- /// `*mut VMContext`.
- pub fn vmctx_vmfunction_import_vmctx(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- func: FuncIndex,
- ) -> ir::Value {
- self.vmimport_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets.vmctx_vmfunction_import_vmctx(func),
- self.offsets.ptr.vm_function_import().vmctx().into(),
- VmType::VMFunctionImport,
- )
- }
+ (@dynamic_entry $Name:ident $Offsets:tt align { $al:tt }) => {};
- /// Load the import function's `VMFunctionImport::wasm_call` field from the
- /// `*mut VMContext`.
- pub fn vmctx_vmfunction_import_wasm_call(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- func: FuncIndex,
- ) -> ir::Value {
- self.vmimport_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- self.offsets.vmctx_vmfunction_import_wasm_call(func),
- self.offsets.ptr.vm_function_import().wasm_call().into(),
- VmType::VMFunctionImport,
- )
- }
+ // Aggregates get no accessor.
+ (@dynamic_entry $Name:ident $Offsets:tt $kind:ident { #[aggregate] $($rest:tt)* }) => {};
- /// Load the imported memory's `VMMemoryImport::vmctx` field from the `*mut
- /// VMContext`.
- pub fn vmctx_vmmemory_import_vmctx(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- memory: MemoryIndex,
- ) -> ir::Value {
- let mem_offset = self.offsets.vmctx_vmmemory_import(memory);
- let mem_vmctx_offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().vmctx());
- self.vmimport_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- mem_vmctx_offset,
- self.offsets.ptr.vm_memory_import().vmctx().into(),
- VmType::VMMemoryImport,
- )
- }
+ (@dynamic_entry $Name:ident [ $($Offsets:tt)* ] array {
+ $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*
+ }) => {
+ #[doc = concat!(
+ "Get the [`Field`] for the `index`th element of `", stringify!($Name),
+ "`'s `", stringify!($fname), "` array.\n\nPanics if `index` is out of \
+ bounds for this vmctx."
+ )]
+ pub fn $fname(self, index: $Index) -> Field<'a, $($Offsets)*> {
+ let offset = self.regions.offsets.$fname().at(index);
+ define_vmctx_alias_region_helpers!(
+ @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
+ )
+ }
+ };
- /// Load the imported memory's `VMMemoryImport::index` field from the `*mut
- /// VMContext`.
- pub fn vmctx_vmmemory_import_index(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- memory: MemoryIndex,
- ) -> ir::Value {
- let mem_offset = self.offsets.vmctx_vmmemory_import(memory);
- let mem_index_offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().index());
- self.vmimport_load(
- cursor,
- ir::types::I32,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- mem_index_offset,
- self.offsets.ptr.vm_memory_import().index().into(),
- VmType::VMMemoryImport,
- )
- }
+ // A single field, whether unconditionally present (`field`) or only
+ // conditionally (`optional`).
+ (@dynamic_entry $Name:ident [ $($Offsets:tt)* ] $kind:ident {
+ $(# $fattr:tt)* $fname:ident $([ if $flag:ident ])? : $($fty:tt)*
+ }) => {
+ #[doc = concat!(
+ "Get the [`Field`] for the `", stringify!($fname), "` field of `",
+ stringify!($Name), "`."
+ $(, "\n\nPanics if `", stringify!($flag), "` is false, in which case \
+ this field is not present at all.")?
+ )]
+ pub fn $fname(self) -> Field<'a, $($Offsets)*> {
+ let offset = self.regions.offsets.$fname();
+ define_vmctx_alias_region_helpers!(
+ @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
+ )
+ }
+ };
- /// Load the imported memory's `VMMemoryImport::from` field from the `*mut
- /// VMContext`.
- pub fn vmctx_vmmemory_import_from(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- memory: MemoryIndex,
- ) -> ir::Value {
- self.vmctx_vmmemory_import_from_load(cursor.func, memory)
- .emit(cursor, vmctx)
- }
+ // Emit the `impl` block holding the accessors for the dynamically-positioned
+ // fields.
+ (@dynamic_impl $Name:ident [ $($Offsets:tt)* ] $OffsetsTt:tt {
+ $($kind:ident $entry:tt)*
+ }) => {
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for every field; not all fields are \
+ accessed by compiled code"
+ )]
+ impl<'a> $Name<'a, $($Offsets)*> {
+ $(
+ define_vmctx_alias_region_helpers!(
+ @dynamic_entry $Name $OffsetsTt $kind $entry
+ );
+ )*
+ }
+ };
- /// Get a `Load` for the imported memory's `VMMemoryImport::from` field from
- /// a `*mut VMContext`.
- pub fn vmctx_vmmemory_import_from_load(
- &mut self,
- func: &mut ir::Function,
- memory: MemoryIndex,
- ) -> Load {
- let mem_offset = self.offsets.vmctx_vmmemory_import(memory);
- let offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().from());
- let region = self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMMemoryImport,
- offset: self.offsets.ptr.vm_memory_import().from().into(),
- },
- );
- Load {
- offset,
- flags: ir::MemFlagsData::trusted()
- .with_readonly()
- .with_can_move()
- .with_alias_region(Some(region)),
- ty: self.pointer_type,
+ // Emit the accessor `struct` and both `impl` blocks for one vmctx type.
+ (@emit $Name:ident $snake:ident $Offsets:tt
+ static { $($skind:ident $sentry:tt)* }
+ dynamic { $($dyn:tt)* }
+ ) => {
+ #[doc = concat!(
+ "An [`AliasRegions`] accessor for the fields of a `", stringify!($Name),
+ "`."
+ )]
+ pub struct $Name<'a, Offsets> {
+ regions: &'a mut AliasRegions<Offsets>,
}
- }
- /// Load the imported table's `VMTableImport::vmctx` field from the `*mut
- /// VMContext`.
- pub fn vmctx_vmtable_import_vmctx(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- table: TableIndex,
- ) -> ir::Value {
- let table_offset = self.offsets.vmctx_vmtable_import(table);
- let table_vmctx_offset =
- table_offset + u32::from(self.offsets.ptr.vm_table_import().vmctx());
- self.vmimport_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- table_vmctx_offset,
- self.offsets.ptr.vm_table_import().vmctx().into(),
- VmType::VMTableImport,
- )
- }
+ impl<Offsets> AliasRegions<Offsets> {
+ #[doc = concat!(
+ "Get an accessor for the fields of a `", stringify!($Name), "`."
+ )]
+ pub fn $snake(&mut self) -> $Name<'_, Offsets> {
+ $Name { regions: self }
+ }
+ }
- /// Load the imported table's `VMTableImport::index` field from the `*mut
- /// VMContext`.
- pub fn vmctx_vmtable_import_index(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- table: TableIndex,
- ) -> ir::Value {
- let table_offset = self.offsets.vmctx_vmtable_import(table);
- let table_index_offset =
- table_offset + u32::from(self.offsets.ptr.vm_table_import().index());
- self.vmimport_load(
- cursor,
- ir::types::I32,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- table_index_offset,
- self.offsets.ptr.vm_table_import().index().into(),
- VmType::VMTableImport,
- )
- }
+ // A statically-positioned field's offset depends only on the target
+ // pointer size, so these accessors work with any `Offsets`.
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for every field; not all fields are \
+ accessed by compiled code"
+ )]
+ impl<'a, Offsets> $Name<'a, Offsets>
+ where
+ Offsets: GetPtrSize,
+ {
+ $( define_vmctx_alias_region_helpers!(@static_entry $Name $snake $skind $sentry); )*
+ }
- /// Get a `Load` for the imported table's `VMTableImport::from` field (a
- /// `*mut VMTableDefinition`) out of a `*mut VMContext`.
- pub fn vmctx_vmtable_from_load(&mut self, func: &mut ir::Function, table: TableIndex) -> Load {
- let offset = self.offsets.vmctx_vmtable_from(table);
- let region = self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMTableImport,
- offset: self.offsets.ptr.vm_table_import().from().into(),
- },
+ // A dynamically-positioned field's offset depends on the module or
+ // component being compiled, so these accessors require this vmctx's own
+ // fully-computed offsets.
+ define_vmctx_alias_region_helpers!(
+ @dynamic_impl $Name $Offsets $Offsets { $($dyn)* }
);
- Load {
- offset,
- flags: ir::MemFlagsData::trusted()
- .with_readonly()
- .with_can_move()
- .with_alias_region(Some(region)),
- ty: self.pointer_type,
+ };
+
+ // Map each vmctx type to the offsets type that computes its
+ // dynamically-positioned fields' offsets.
+ (@one VMContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => {
+ define_vmctx_alias_region_helpers!(@emit VMContext $snake [ VMOffsets<u8> ]
+ static { $($stat)* } dynamic { $($dyn)* });
+ };
+ (@one VMComponentContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => {
+ define_vmctx_alias_region_helpers!(
+ @emit VMComponentContext $snake [ VMComponentOffsets<u8> ]
+ static { $($stat)* } dynamic { $($dyn)* }
+ );
+ };
+
+ // Top-level entry.
+ ( $(
+ {
+ $Name:ident $snake:ident
+ static { $($stat:tt)* }
+ dynamic { $($dyn:tt)* }
+ }
+ )* ) => {
+ $(
+ define_vmctx_alias_region_helpers!(@one $Name $snake
+ static { $($stat)* } dynamic { $($dyn)* });
+ )*
+ };
+}
+wasmtime_environ::for_each_vmctx_type!(define_vmctx_alias_region_helpers);
+
+impl<Offsets> AliasRegions<Offsets>
+where
+ Offsets: GetPtrSize,
+{
+ /// Create a new `AliasRegions`.
+ pub fn new(offsets: Offsets) -> Self {
+ Self {
+ pointer_type: ir::Type::int_with_byte_size(offsets.get_ptr_size().size().into())
+ .unwrap(),
+ offsets,
+ cache: std::collections::HashMap::default(),
}
}
- /// Load the imported global's address (`VMGlobalImport::from`) out of the
- /// `*mut VMContext`.
- pub fn vmctx_vmglobal_import_from(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- global: GlobalIndex,
- ) -> ir::Value {
- let from_offset = self.offsets.vmctx_vmglobal_import_from(global);
- self.vmimport_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- from_offset,
- self.offsets.ptr.vm_global_import().from().into(),
- VmType::VMGlobalImport,
- )
+ /// Get the alias region for accesses into the GC heap.
+ pub fn gc_heap_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
+ self.region(func, AliasRegionKey::GcHeap)
}
- /// Load the defined memory's `*mut VMMemoryDefinition` out of the `*mut
- /// VMContext`.
- pub fn vmctx_vmmemory_pointer(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- memory: DefinedMemoryIndex,
- ) -> ir::Value {
- self.vmctx_vmmemory_pointer_load(cursor.func, memory)
- .emit(cursor, vmctx)
+ /// Get the alias region for an imported or exported memory access (shared
+ /// across all imported/exported memories).
+ pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
+ self.region(func, AliasRegionKey::PublicMemory)
}
- /// Get a `Load` for the defined memory's `*mut VMMemoryDefinition` out of a
- /// `*mut VMContext`.
- pub fn vmctx_vmmemory_pointer_load(
+ /// Get the alias region for accessing a defined memory that is not
+ /// exported.
+ pub fn defined_memory_region(
&mut self,
func: &mut ir::Function,
- memory: DefinedMemoryIndex,
- ) -> Load {
- let offset = self.offsets.vmctx_vmmemory_pointer(memory);
- let region = self.vmctx_region(func, offset);
- Load {
- offset,
- flags: ir::MemFlagsData::trusted()
- .with_readonly()
- .with_can_move()
- .with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
+ module: StaticModuleIndex,
+ index: DefinedMemoryIndex,
+ ) -> ir::AliasRegion {
+ self.region(func, AliasRegionKey::DefinedMemory { module, index })
}
- /// Load the base of the given runtime data out of the `*mut VMContext`.
- pub fn vmctx_runtime_data_base(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- runtime_data: RuntimeDataIndex,
- ) -> ir::Value {
- self.vmctx_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.vmctx_runtime_data_base(runtime_data),
- )
+ /// Get the alias region for an imported or exported table access (shared
+ /// across all imported/exported memories).
+ pub fn public_table_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
+ self.region(func, AliasRegionKey::PublicTable)
}
- /// Load the length of the given runtime data out of the `*mut VMContext`.
- pub fn vmctx_runtime_data_length(
+ /// Get the alias region for accessing a defined table that is not
+ /// exported.
+ pub fn defined_table_region(
&mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- runtime_data: RuntimeDataIndex,
- ) -> ir::Value {
- self.vmctx_load(
- cursor,
- ir::types::I32,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.vmctx_runtime_data_length(runtime_data),
- )
+ func: &mut ir::Function,
+ module: StaticModuleIndex,
+ index: DefinedTableIndex,
+ ) -> ir::AliasRegion {
+ self.region(func, AliasRegionKey::DefinedTable { module, index })
+ }
+
+ /// Get the alias region for an imported or exported global access (shared
+ /// across all imported/exported memories).
+ pub fn public_global_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
+ self.region(func, AliasRegionKey::PublicGlobal)
}
- /// Load the length of the given runtime data out of the `*mut VMContext`.
- pub fn store_vmctx_runtime_data_length(
+ /// Get the alias region for accessing a defined global that is not
+ /// exported.
+ pub fn defined_global_region(
&mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- runtime_data: RuntimeDataIndex,
- new_length: ir::Value,
- ) {
- self.vmctx_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.vmctx_runtime_data_length(runtime_data),
- new_length,
- )
+ func: &mut ir::Function,
+ module: StaticModuleIndex,
+ index: DefinedGlobalIndex,
+ ) -> ir::AliasRegion {
+ self.region(func, AliasRegionKey::DefinedGlobal { module, index })
}
}
@@ -2332,243 +2027,45 @@ where
}
}
-/// `VMComponentContext`-related methods, used when compiling component
-/// trampolines.
+/// `VMComponentContext` fields that are not simply one of the layout's own
+/// fields, and so are not generated by [`define_vmctx_alias_region_helpers!`].
impl AliasRegions<VMComponentOffsets<u8>> {
- fn vmcomponent_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {
- self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMComponentContext,
- offset,
- },
- )
- }
-
- fn vmcomponent_load(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- ty: ir::Type,
- base_flags: ir::MemFlagsData,
- vmctx: ir::Value,
- offset: u32,
- ) -> ir::Value {
- let region = self.vmcomponent_region(cursor.func, offset);
- cursor.ins().load(
- ty,
- base_flags.with_alias_region(Some(region)),
- vmctx,
- i32::try_from(offset).unwrap(),
- )
- }
-
- fn vmcomponent_store(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- base_flags: ir::MemFlagsData,
- vmctx: ir::Value,
- offset: u32,
- val: ir::Value,
- ) {
- let region = self.vmcomponent_region(cursor.func, offset);
- cursor.ins().store(
- base_flags.with_alias_region(Some(region)),
- val,
- vmctx,
- i32::try_from(offset).unwrap(),
- );
- }
-
- /// Load a lowering's host-data pointer from the `VMComponentContext`.
- pub fn vmcomponent_lowering_data(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- index: LoweredIndex,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.lowering_data(index),
- )
- }
-
- /// Load a lowering's host callee pointer from the `VMComponentContext`.
+ /// Get the [`Field`] for the `callee` of the `index`th lowering in the
+ /// `VMComponentContext`'s `lowerings` array.
pub fn vmcomponent_lowering_callee(
&mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
index: LoweredIndex,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.lowering_callee(index),
- )
- }
-
- /// Load the current task's `may_block` flag from the `VMComponentContext`.
- pub fn vmcomponent_task_may_block(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- ir::types::I32,
- ir::MemFlagsData::trusted().with_readonly(),
- vmctx,
- self.offsets.task_may_block(),
- )
- }
-
- /// Store the current task's `may_block` flag into the `VMComponentContext`.
- pub fn store_vmcomponent_task_may_block(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- val: ir::Value,
- ) {
- self.vmcomponent_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.task_may_block(),
- val,
- )
- }
-
- /// Load a resource's destructor function pointer from the
- /// `VMComponentContext`.
- pub fn vmcomponent_resource_destructor(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- index: ResourceIndex,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly(),
- vmctx,
- self.offsets.resource_destructor(index),
- )
- }
-
- /// Load a runtime memory's `*mut VMMemoryDefinition` from the
- /// `VMComponentContext`.
- pub fn vmcomponent_runtime_memory(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- index: RuntimeMemoryIndex,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.runtime_memory(index),
- )
- }
-
- /// Load a runtime callback function pointer from the `VMComponentContext`.
- pub fn vmcomponent_runtime_callback(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- index: RuntimeCallbackIndex,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.runtime_callback(index),
- )
+ ) -> Field<'_, VMComponentOffsets<u8>> {
+ let offset = self.offsets.lowering_callee(index);
+ self.vmlowering_field(offset)
}
- /// Load a runtime post-return function pointer from the
- /// `VMComponentContext`.
- pub fn vmcomponent_runtime_post_return(
+ /// Get the [`Field`] for the host data of the `index`th lowering in the
+ /// `VMComponentContext`'s `lowerings` array.
+ pub fn vmcomponent_lowering_data(
&mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- index: RuntimePostReturnIndex,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.runtime_post_return(index),
- )
+ index: LoweredIndex,
+ ) -> Field<'_, VMComponentOffsets<u8>> {
+ let offset = self.offsets.lowering_data(index);
+ self.vmlowering_field(offset)
}
- /// Load the base pointer of the component builtins array from the
+ /// Get the [`Field`] at `offset` within a `VMLowering` inlined into the
/// `VMComponentContext`.
- pub fn vmcomponent_builtins(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly(),
- vmctx,
- self.offsets.builtins(),
- )
- }
-
- /// Load a component instance's `may_leave` flag from the `VMComponentContext`.
- pub fn vmcomponent_instance_may_leave(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmctx: ir::Value,
- instance: RuntimeComponentInstanceIndex,
- ) -> ir::Value {
- self.vmcomponent_load(
- cursor,
- ir::types::I32,
+ ///
+ /// Unlike the other aggregates inlined into a vmctx, `VMLowering` is not one
+ /// of the types defined by `for_each_vm_type!`, and so has no alias region of
+ /// its own to rebase onto the aggregate's offset. Its fields are therefore
+ /// part of the containing `VMComponentContext`'s region, keyed by their
+ /// offset within it.
+ fn vmlowering_field(&mut self, offset: u32) -> Field<'_, VMComponentOffsets<u8>> {
+ let ty = self.pointer_type;
+ Field::new(
+ self,
+ VmType::VMComponentContext,
+ offset,
ir::MemFlagsData::trusted(),
- vmctx,
- self.offsets.may_leave(instance),
- )
- }
-}
-
-/// `VMComponentContext`-related methods that need to be generic over `Offsets`
-/// due to the call context.
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- /// Load a field at `offset` within a `VMComponentContext`.
- pub fn vmcomponent_context_generic_load(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- ty: ir::Type,
- base_flags: ir::MemFlagsData,
- vmctx: ir::Value,
- offset: u32,
- ) -> ir::Value {
- let region = self.region(
- cursor.func,
- AliasRegionKey::Vm {
- ty: VmType::VMComponentContext,
- offset,
- },
- );
- cursor.ins().load(
ty,
- base_flags.with_alias_region(Some(region)),
- vmctx,
- i32::try_from(offset).unwrap(),
)
}
}
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 5f3fdee1c6..7aa781c40f 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -230,8 +230,10 @@ impl Compiler {
caller_vmctx,
wasmtime_environ::VMCONTEXT_MAGIC,
);
- let vm_store_context =
- alias_regions.vmctx_store_context(&mut builder.cursor(), caller_vmctx);
+ let vm_store_context = alias_regions
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), caller_vmctx);
save_last_wasm_exit_fp_and_pc(
&mut builder,
pointer_type,
@@ -266,8 +268,10 @@ impl Compiler {
// Increment the "execution version" on the VMStoreContext if
// guest debugging is enabled.
if self.tunables.debug_guest {
- let vmstore_ctx_ptr =
- alias_regions.vmctx_store_context(&mut builder.cursor(), caller_vmctx);
+ let vmstore_ctx_ptr = alias_regions
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), caller_vmctx);
let old_version = alias_regions
.vmstore_context_execution_version(&mut builder.cursor(), vmstore_ctx_ptr);
let new_version = builder.ins().iadd_imm_s(old_version, 1);
@@ -343,7 +347,10 @@ impl Compiler {
vmctx,
wasmtime_environ::VMCONTEXT_MAGIC,
);
- let vm_store_context = alias_regions.vmctx_store_context(&mut builder.cursor(), vmctx);
+ let vm_store_context = alias_regions
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx);
save_last_wasm_exit_fp_and_pc(
&mut builder,
pointer_type,
@@ -550,7 +557,9 @@ impl wasmtime_environ::Compiler for Compiler {
if !isa.triple().is_pulley() {
let store_ctx = func_env
.alias_regions
- .vmctx_store_context_load(&mut context.func);
+ .vmctx()
+ .store_context()
+ .to_deferred_load(&mut context.func);
let stack_limit = func_env
.alias_regions
.vmstore_context_stack_limit_load(&mut context.func);
@@ -608,7 +617,7 @@ impl wasmtime_environ::Compiler for Compiler {
symbol,
wasmtime_environ::VMCONTEXT_MAGIC,
|alias_regions, _pointer_type, cursor, vmctx| {
- alias_regions.vmctx_store_context(cursor, vmctx)
+ alias_regions.vmctx().store_context().load(cursor, vmctx)
},
)
}
@@ -656,7 +665,7 @@ impl wasmtime_environ::Compiler for Compiler {
symbol,
wasmtime_environ::VMCONTEXT_MAGIC,
|alias_regions, _pointer_type, cursor, vmctx| {
- alias_regions.vmctx_store_context(cursor, vmctx)
+ alias_regions.vmctx().store_context().load(cursor, vmctx)
},
),
// Delegate to a helper to finish compiling this.
@@ -1358,7 +1367,10 @@ impl Compiler {
{
// Builtins are stored in an array in all `VMContext`s. First load the
// base pointer of the array...
- let array_addr = alias_regions.vmctx_builtin_functions(&mut builder.cursor(), vmctx);
+ let array_addr = alias_regions
+ .vmctx()
+ .builtin_functions()
+ .load(&mut builder.cursor(), vmctx);
// ... and then load the entry in the array that corresponds to this
// builtin.
let func_addr = alias_regions.builtin_functions_array_element(
@@ -1408,7 +1420,14 @@ impl Compiler {
if !self.emit_debug_checks {
return;
}
- let magic = alias_regions.vmctx_magic(&mut builder.cursor(), vmctx);
+ // NB: a `VMComponentContext`'s `magic` field is deliberately read
+ // through the `VMContext` accessor: both types keep `magic` at offset
+ // zero, and this check is the one place that reads a vmctx's magic
+ // without knowing which of the two it has.
+ let magic = alias_regions
+ .vmctx()
+ .magic()
+ .load(&mut builder.cursor(), vmctx);
let is_expected_vmctx = builder.ins().icmp_imm_s(
ir::condcodes::IntCC::Equal,
magic,
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index a4c98bebab..4580e60113 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -155,11 +155,10 @@ impl<'a> TrampolineCompiler<'a> {
WasmArgs::ValRawList,
|me, params| {
let vmctx = params[0];
- let lowering_data = me.alias_regions.vmcomponent_lowering_data(
- &mut me.builder.cursor(),
- vmctx,
- *index,
- );
+ let lowering_data = me
+ .alias_regions
+ .vmcomponent_lowering_data(*index)
+ .load(&mut me.builder.cursor(), vmctx);
params.extend([
lowering_data,
me.index_value(*lower_ty),
@@ -990,11 +989,10 @@ impl<'a> TrampolineCompiler<'a> {
HostCallee::Lowering(index) => {
// Load host function pointer from the vmcontext and then call that
// indirect function pointer with the list of arguments.
- let host_fn = self.alias_regions.vmcomponent_lowering_callee(
- &mut self.builder.cursor(),
- vmctx,
- index,
- );
+ let host_fn = self
+ .alias_regions
+ .vmcomponent_lowering_callee(index)
+ .load(&mut self.builder.cursor(), vmctx);
let host_sig = {
let mut sig = ir::Signature::new(CallConv::triple_default(self.isa.triple()));
for param in host_args.iter() {
@@ -1206,9 +1204,12 @@ impl<'a> TrampolineCompiler<'a> {
// Stash the old value of `may_block` and then set it to false.
let old_may_block = self
.alias_regions
- .vmcomponent_task_may_block(&mut self.builder.cursor(), vmctx);
+ .vmcomponent()
+ .task_may_block()
+ .readonly()
+ .load(&mut self.builder.cursor(), vmctx);
let zero = self.builder.ins().iconst(ir::types::I32, i64::from(0));
- self.alias_regions.store_vmcomponent_task_may_block(
+ self.alias_regions.vmcomponent().task_may_block().store(
&mut self.builder.cursor(),
vmctx,
zero,
@@ -1251,11 +1252,11 @@ impl<'a> TrampolineCompiler<'a> {
// NB: despite the vmcontext storing nullable funcrefs for function
// pointers we know this is statically never null due to the
// `has_destructor` check above.
- let dtor_func_ref = self.alias_regions.vmcomponent_resource_destructor(
- &mut self.builder.cursor(),
- vmctx,
- index,
- );
+ let dtor_func_ref = self
+ .alias_regions
+ .vmcomponent()
+ .resource_destructors(index)
+ .load(&mut self.builder.cursor(), vmctx);
if self.compiler.emit_debug_checks {
self.builder
.ins()
@@ -1323,7 +1324,7 @@ impl<'a> TrampolineCompiler<'a> {
self.raise_if_host_trapped(result.unwrap());
// Restore the old value of `may_block`
- self.alias_regions.store_vmcomponent_task_may_block(
+ self.alias_regions.vmcomponent().task_may_block().store(
&mut self.builder.cursor(),
vmctx,
old_may_block,
@@ -1351,7 +1352,9 @@ impl<'a> TrampolineCompiler<'a> {
fn load_memory(&mut self, vmctx: ir::Value, memory: RuntimeMemoryIndex) -> ir::Value {
self.alias_regions
- .vmcomponent_runtime_memory(&mut self.builder.cursor(), vmctx, memory)
+ .vmcomponent()
+ .memories(memory)
+ .load(&mut self.builder.cursor(), vmctx)
}
fn load_callback(
@@ -1361,11 +1364,11 @@ impl<'a> TrampolineCompiler<'a> {
) -> ir::Value {
let pointer_type = self.isa.pointer_type();
match callback {
- Some(idx) => self.alias_regions.vmcomponent_runtime_callback(
- &mut self.builder.cursor(),
- vmctx,
- idx,
- ),
+ Some(idx) => self
+ .alias_regions
+ .vmcomponent()
+ .callbacks(idx)
+ .load(&mut self.builder.cursor(), vmctx),
None => self.builder.ins().iconst(pointer_type, 0),
}
}
@@ -1377,11 +1380,11 @@ impl<'a> TrampolineCompiler<'a> {
) -> ir::Value {
let pointer_type = self.isa.pointer_type();
match post_return {
- Some(idx) => self.alias_regions.vmcomponent_runtime_post_return(
- &mut self.builder.cursor(),
- vmctx,
- idx,
- ),
+ Some(idx) => self
+ .alias_regions
+ .vmcomponent()
+ .post_returns(idx)
+ .load(&mut self.builder.cursor(), vmctx),
None => self.builder.ins().iconst(pointer_type, 0),
}
}
@@ -1399,7 +1402,9 @@ impl<'a> TrampolineCompiler<'a> {
// per-process.
let builtins_array = self
.alias_regions
- .vmcomponent_builtins(&mut self.builder.cursor(), vmctx);
+ .vmcomponent()
+ .builtins()
+ .load(&mut self.builder.cursor(), vmctx);
// Next load the function pointer at `offset` and return that.
self.alias_regions
.component_builtin_functions_array_element(
@@ -1540,11 +1545,11 @@ impl<'a> TrampolineCompiler<'a> {
fn check_may_leave_instance(&mut self, instance: RuntimeComponentInstanceIndex) {
let vmctx = self.builder.func.dfg.block_params(self.block0)[0];
- let may_leave = self.alias_regions.vmcomponent_instance_may_leave(
- &mut self.builder.cursor(),
- vmctx,
- instance,
- );
+ let may_leave = self
+ .alias_regions
+ .vmcomponent()
+ .may_leave(instance)
+ .load(&mut self.builder.cursor(), vmctx);
let (mut traps, builder) = self.traps();
traps.trapz(builder, may_leave, TRAP_CANNOT_LEAVE_COMPONENT);
}
@@ -1578,7 +1583,9 @@ impl<'a> TrampolineCompiler<'a> {
fn load_vm_store_context(&mut self) -> ir::Value {
let caller_vmctx = self.abi_load_params()[1];
self.alias_regions
- .vmctx_store_context(&mut self.builder.cursor(), caller_vmctx)
+ .vmctx()
+ .store_context()
+ .load(&mut self.builder.cursor(), caller_vmctx)
}
}
@@ -1687,22 +1694,17 @@ impl ComponentCompiler for Compiler {
// Implement the array-abi trampoline in terms of calling the
// wasm-abi trampoline.
Abi::Array => {
- let offsets =
- VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);
return Ok(self.array_to_wasm_trampoline(
key,
FuncKey::ComponentTrampoline(Abi::Wasm, trampoline_index),
sig,
symbol,
wasmtime_environ::component::VMCOMPONENT_MAGIC,
- |alias_regions, pointer_type, cursor, vmctx| {
- alias_regions.vmcomponent_context_generic_load(
- cursor,
- pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- offsets.vm_store_context(),
- )
+ |alias_regions, _pointer_type, cursor, vmctx| {
+ alias_regions
+ .vmcomponent()
+ .store_context()
+ .load(cursor, vmctx)
},
)?);
}
@@ -1771,22 +1773,17 @@ impl ComponentCompiler for Compiler {
// Implement the array-abi trampoline in terms of calling the
// wasm-abi trampoline.
Abi::Array => {
- let offsets =
- VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);
return Ok(self.array_to_wasm_trampoline(
FuncKey::UnsafeIntrinsic(abi, intrinsic),
FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic),
&wasm_func_ty,
symbol,
wasmtime_environ::component::VMCOMPONENT_MAGIC,
- |alias_regions, pointer_type, cursor, vmctx| {
- alias_regions.vmcomponent_context_generic_load(
- cursor,
- pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmctx,
- offsets.vm_store_context(),
- )
+ |alias_regions, _pointer_type, cursor, vmctx| {
+ alias_regions
+ .vmcomponent()
+ .store_context()
+ .load(cursor, vmctx)
},
)?);
}
@@ -2424,7 +2421,9 @@ where
let caller_vmctx = params[1];
self.traps
.alias_regions()
- .vmctx_store_context(&mut self.builder.cursor(), caller_vmctx)
+ .vmctx()
+ .store_context()
+ .load(&mut self.builder.cursor(), caller_vmctx)
}
}
diff --git a/crates/cranelift/src/debug.rs b/crates/cranelift/src/debug.rs
index 55a0a7a2fb..dbfedf8631 100644
--- a/crates/cranelift/src/debug.rs
+++ b/crates/cranelift/src/debug.rs
@@ -102,17 +102,20 @@ impl<'a> Compilation<'a> {
let memory_offset = if ofs.num_imported_memories > 0 {
let index = MemoryIndex::new(0);
ModuleMemoryOffset::Imported {
- offset_to_vm_memory_definition: ofs.vmctx_vmmemory_import(index)
+ offset_to_vm_memory_definition: ofs.imported_memories().at(index)
+ u32::from(ofs.ptr.vm_memory_import().from()),
offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(),
}
} else if ofs.num_owned_memories > 0 {
let index = OwnedMemoryIndex::new(0);
- ModuleMemoryOffset::Defined(ofs.vmctx_vmmemory_definition_base(index))
+ ModuleMemoryOffset::Defined(
+ ofs.owned_memories().at(index)
+ + u32::from(ofs.ptr.vm_memory_definition().base()),
+ )
} else if ofs.num_defined_memories > 0 {
let index = DefinedMemoryIndex::new(0);
ModuleMemoryOffset::Imported {
- offset_to_vm_memory_definition: ofs.vmctx_vmmemory_pointer(index),
+ offset_to_vm_memory_definition: ofs.memories().at(index),
offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(),
}
} else {
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 6a58b20fe9..71b07ef666 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -450,12 +450,16 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
) -> (ir::Value, i32) {
let vmctx = self.vmctx_val(pos);
if let Some(def_index) = self.module.defined_global_index(index) {
- let offset = i32::try_from(self.offsets.vmctx_vmglobal_definition(def_index)).unwrap();
+ let offset = i32::try_from(self.offsets.globals().at(def_index)).unwrap();
(vmctx, offset)
} else {
+ let import_off = self.offsets.imported_globals().at(index);
let addr = self
.alias_regions
- .vmctx_vmglobal_import_from(pos, vmctx, index);
+ .vm_global_import()
+ .from()
+ .relative_to(import_off)
+ .load(pos, vmctx);
(addr, 0)
}
}
@@ -464,7 +468,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
fn get_vmstore_context_ptr(&mut self, builder: &mut FunctionBuilder) -> ir::Value {
let vmctx = self.vmctx_val(&mut builder.cursor());
self.alias_regions
- .vmctx_store_context(&mut builder.cursor(), vmctx)
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx)
}
fn fuel_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) {
@@ -737,7 +743,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
fn epoch_ptr(&mut self, builder: &mut FunctionBuilder<'_>) -> ir::Value {
let vmctx = self.vmctx_val(&mut builder.cursor());
self.alias_regions
- .vmctx_epoch_ptr(&mut builder.cursor(), vmctx)
+ .vmctx()
+ .epoch_ptr()
+ .load(&mut builder.cursor(), vmctx)
}
fn epoch_load_current(&mut self, builder: &mut FunctionBuilder<'_>) -> ir::Value {
@@ -1097,7 +1105,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
) -> ir::Value {
let vmctx = self.vmctx_val(pos);
// Load the base pointer of the array of `VMSharedTypeIndex`es.
- let shared_indices = self.alias_regions.vmctx_shared_type_ids_array(pos, vmctx);
+ let shared_indices = self.alias_regions.vmctx().type_ids().load(pos, vmctx);
// Calculate the offset in that array for this type's entry.
@@ -1550,7 +1558,9 @@ impl FuncEnvironment<'_> {
// atomically growing it.
let mem = self
.alias_regions
- .vmctx_vmmemory_pointer_load(func, def_index);
+ .vmctx()
+ .memories(def_index)
+ .to_deferred_load(func);
let mut base = self.alias_regions.vm_memory_definition().base();
base.can_move();
if base_readonly {
@@ -1572,7 +1582,7 @@ impl FuncEnvironment<'_> {
// relative to the `vmctx` itself: fold that offset into each
// field's offset.
let owned_index = self.module.owned_memory_index(def_index);
- let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index);
+ let vmctx_off = self.offsets.owned_memories().at(owned_index);
let mut base = self.alias_regions.vm_memory_definition().base();
base.can_move();
if base_readonly {
@@ -1591,9 +1601,13 @@ impl FuncEnvironment<'_> {
)
}
} else {
+ let import_off = self.offsets.imported_memories().at(index);
let mem = self
.alias_regions
- .vmctx_vmmemory_import_from_load(func, index);
+ .vm_memory_import()
+ .from()
+ .relative_to(import_off)
+ .to_deferred_load(func);
let mut base = self.alias_regions.vm_memory_definition().base();
base.can_move();
if base_readonly {
@@ -1646,7 +1660,7 @@ impl FuncEnvironment<'_> {
if let Some(def_index) = self.module.defined_table_index(index) {
// A defined table's `VMTableDefinition` is inlined into the vmctx,
// reached at an absolute `vmctx` offset.
- let vmctx_off = self.offsets.vmctx_vmtable_definition(def_index);
+ let vmctx_off = self.offsets.tables().at(def_index);
let mut base = self.alias_regions.vm_table_definition().base();
if is_static {
base.readonly().can_move();
@@ -1674,7 +1688,13 @@ impl FuncEnvironment<'_> {
} else {
// An imported table is reached through a `*mut VMTableDefinition`
// loaded from the `vmctx`.
- let from = self.alias_regions.vmctx_vmtable_from_load(func, index);
+ let import_off = self.offsets.imported_tables().at(index);
+ let from = self
+ .alias_regions
+ .vm_table_import()
+ .from()
+ .relative_to(import_off)
+ .to_deferred_load(func);
let mut base = self.alias_regions.vm_table_definition().base();
if is_static {
base.readonly().can_move();
@@ -1735,16 +1755,19 @@ impl FuncEnvironment<'_> {
} else {
// An imported tag -- we need to load the VMTagImport struct.
let vmctx = self.vmctx_val(&mut builder.cursor());
- let from_vmctx = self.alias_regions.vmctx_vmtag_import_vmctx(
- &mut builder.cursor(),
- vmctx,
- tag_index,
- );
- let index = self.alias_regions.vmctx_vmtag_import_index(
- &mut builder.cursor(),
- vmctx,
- tag_index,
- );
+ let import_off = self.offsets.imported_tags().at(tag_index);
+ let from_vmctx = self
+ .alias_regions
+ .vm_tag_import()
+ .vmctx()
+ .relative_to(import_off)
+ .load(&mut builder.cursor(), vmctx);
+ let index = self
+ .alias_regions
+ .vm_tag_import()
+ .index()
+ .relative_to(import_off)
+ .load(&mut builder.cursor(), vmctx);
let builtin = self.builtin_functions.get_instance_id(builder.func);
let call = builder.ins().call(builtin, &[from_vmctx]);
let from_instance_id = builder.func.dfg.inst_results(call)[0];
@@ -1841,11 +1864,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// First append the callee vmctx address.
let vmctx = self.env.vmctx_val(&mut self.builder.cursor());
- let callee_vmctx = self.env.alias_regions.vmctx_vmfunction_import_vmctx(
- &mut self.builder.cursor(),
- vmctx,
- callee_index,
- );
+ let import_off = self.env.offsets.imported_functions().at(callee_index);
+ let callee_vmctx = self
+ .env
+ .alias_regions
+ .vm_function_import()
+ .vmctx()
+ .relative_to(import_off)
+ .load(&mut self.builder.cursor(), vmctx);
real_call_args.push(callee_vmctx);
real_call_args.push(caller_vmctx);
@@ -1929,11 +1955,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
}
FactInlineIntrinsic::EnterSyncCall | FactInlineIntrinsic::ExitSyncCall => {}
}
- let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(
- &mut self.builder.cursor(),
- vmctx,
- callee_index,
- );
+ let import_off = self.env.offsets.imported_functions().at(callee_index);
+ let func_addr = self
+ .env
+ .alias_regions
+ .vm_function_import()
+ .wasm_call()
+ .relative_to(import_off)
+ .load(&mut self.builder.cursor(), vmctx);
Ok(Reachability::Reachable(self.indirect_call_inst(
sig_ref,
func_addr,
@@ -1947,11 +1976,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// and with different functions. Either way, we have to do the
// indirect call.
None => {
- let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(
- &mut self.builder.cursor(),
- vmctx,
- callee_index,
- );
+ let import_off = self.env.offsets.imported_functions().at(callee_index);
+ let func_addr = self
+ .env
+ .alias_regions
+ .vm_function_import()
+ .wasm_call()
+ .relative_to(import_off)
+ .load(&mut self.builder.cursor(), vmctx);
Ok(Reachability::Reachable(self.indirect_call_inst(
sig_ref,
func_addr,
@@ -2143,11 +2175,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// equivalent out-of-line teardown via the `exit_sync_call` libcall.
self.builder.switch_to_block(slow_block);
let vmctx = self.env.vmctx_val(&mut self.builder.cursor());
- let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(
- &mut self.builder.cursor(),
- vmctx,
- callee_index,
- );
+ let import_off = self.env.offsets.imported_functions().at(callee_index);
+ let func_addr = self
+ .env
+ .alias_regions
+ .vm_function_import()
+ .wasm_call()
+ .relative_to(import_off)
+ .load(&mut self.builder.cursor(), vmctx);
self.indirect_call_inst(sig_ref, func_addr, real_call_args);
self.builder.ins().jump(cont_block, &[]);
@@ -3518,12 +3553,19 @@ impl FuncEnvironment<'_> {
// This is an imported memory, so load the vmctx/defined index from
// the import definition itself.
None => {
+ let import_off = self.offsets.imported_memories().at(index);
let vmctx = self
.alias_regions
- .vmctx_vmmemory_import_vmctx(pos, cur_vmctx, index);
+ .vm_memory_import()
+ .vmctx()
+ .relative_to(import_off)
+ .load(pos, cur_vmctx);
let index = self
.alias_regions
- .vmctx_vmmemory_import_index(pos, cur_vmctx, index);
+ .vm_memory_import()
+ .index()
+ .relative_to(import_off)
+ .load(pos, cur_vmctx);
(vmctx, index)
}
}
@@ -3544,12 +3586,19 @@ impl FuncEnvironment<'_> {
match self.module.defined_table_index(index) {
Some(index) => (cur_vmctx, pos.ins().iconst(I32, i64::from(index.as_u32()))),
None => {
+ let import_off = self.offsets.imported_tables().at(index);
let vmctx = self
.alias_regions
- .vmctx_vmtable_import_vmctx(pos, cur_vmctx, index);
+ .vm_table_import()
+ .vmctx()
+ .relative_to(import_off)
+ .load(pos, cur_vmctx);
let index = self
.alias_regions
- .vmctx_vmtable_import_index(pos, cur_vmctx, index);
+ .vm_table_import()
+ .index()
+ .relative_to(import_off)
+ .load(pos, cur_vmctx);
(vmctx, index)
}
}
@@ -3600,7 +3649,9 @@ impl FuncEnvironment<'_> {
if is_shared {
let mem_ptr = self
.alias_regions
- .vmctx_vmmemory_pointer(pos, vmctx, def_index);
+ .vmctx()
+ .memories(def_index)
+ .load(pos, vmctx);
self.alias_regions
.vm_memory_definition()
.current_length()
@@ -3609,7 +3660,7 @@ impl FuncEnvironment<'_> {
// A defined, owned memory's `VMMemoryDefinition` is inlined into
// the `vmctx` at an absolute offset.
let owned_index = self.module.owned_memory_index(def_index);
- let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index);
+ let vmctx_off = self.offsets.owned_memories().at(owned_index);
self.alias_regions
.vm_memory_definition()
.current_length()
@@ -3617,9 +3668,13 @@ impl FuncEnvironment<'_> {
.load(pos, vmctx)
}
} else {
+ let import_off = self.offsets.imported_memories().at(index);
let mem_ptr = self
.alias_regions
- .vmctx_vmmemory_import_from(pos, vmctx, index);
+ .vm_memory_import()
+ .from()
+ .relative_to(import_off)
+ .load(pos, vmctx);
if is_shared {
self.alias_regions
.vm_memory_definition()
@@ -4040,12 +4095,10 @@ impl FuncEnvironment<'_> {
// the value 0 to the `VMContext`'s slot for this passive data segment.
let vmctx = self.vmctx_val(&mut pos);
let new_length = pos.ins().iconst(I32, 0);
- self.alias_regions.store_vmctx_runtime_data_length(
- &mut pos,
- vmctx,
- runtime_index,
- new_length,
- );
+ self.alias_regions
+ .vmctx()
+ .runtime_data_lengths(runtime_index)
+ .store(&mut pos, vmctx, new_length);
Ok(())
}
@@ -4331,7 +4384,9 @@ impl FuncEnvironment<'_> {
) -> ir::Value {
let vmctx = self.vmctx_val(&mut builder.cursor());
self.alias_regions
- .vmctx_runtime_data_length(&mut builder.cursor(), vmctx, runtime_index)
+ .vmctx()
+ .runtime_data_lengths(runtime_index)
+ .load(&mut builder.cursor(), vmctx)
}
fn load_runtime_data_length_as_pointer(
@@ -4350,7 +4405,9 @@ impl FuncEnvironment<'_> {
) -> ir::Value {
let vmctx = self.vmctx_val(&mut builder.cursor());
self.alias_regions
- .vmctx_runtime_data_base(&mut builder.cursor(), vmctx, runtime_index)
+ .vmctx()
+ .runtime_data_bases(runtime_index)
+ .load(&mut builder.cursor(), vmctx)
}
pub fn translate_table_copy(
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index ac8275f027..b88d21c050 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -1524,7 +1524,11 @@ impl FuncEnvironment<'_> {
return heap;
}
- let store_ctx = self.alias_regions.vmctx_store_context_load(func);
+ let store_ctx = self
+ .alias_regions
+ .vmctx()
+ .store_context()
+ .to_deferred_load(func);
// The base pointer's load is `can_move` and `readonly` when the GC
// heap's base can never move.
diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rs
index e4eac77885..7f3a56541e 100644
--- a/crates/cranelift/src/func_environ/gc/copying.rs
+++ b/crates/cranelift/src/func_environ/gc/copying.rs
@@ -34,7 +34,9 @@ impl CopyingCompiler {
let vmctx = func_env.vmctx_val(&mut builder.cursor());
func_env
.alias_regions
- .vmctx_gc_heap_data(&mut builder.cursor(), vmctx)
+ .vmctx()
+ .gc_heap_data()
+ .load(&mut builder.cursor(), vmctx)
}
/// Load the current bump pointer and active-space end from a `*mut
diff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rs
index 8a9de06f63..da3d205965 100644
--- a/crates/cranelift/src/func_environ/gc/drc.rs
+++ b/crates/cranelift/src/func_environ/gc/drc.rs
@@ -105,7 +105,9 @@ impl DrcCompiler {
let vmctx = func_env.vmctx_val(&mut builder.cursor());
let heap_data = func_env
.alias_regions
- .vmctx_gc_heap_data(&mut builder.cursor(), vmctx);
+ .vmctx()
+ .gc_heap_data()
+ .load(&mut builder.cursor(), vmctx);
// Load the current first list element, which will be our new next list
// element.
@@ -171,7 +173,9 @@ impl DrcCompiler {
let vmctx = func_env.vmctx_val(&mut builder.cursor());
let heap_data = func_env
.alias_regions
- .vmctx_gc_heap_data(&mut builder.cursor(), vmctx);
+ .vmctx()
+ .gc_heap_data()
+ .load(&mut builder.cursor(), vmctx);
let current_len = func_env
.alias_regions
.vmdrc_heap_data_current_over_approximated_stack_roots_len(
diff --git a/crates/cranelift/src/func_environ/gc/null.rs b/crates/cranelift/src/func_environ/gc/null.rs
index b951d9b30b..454166c278 100644
--- a/crates/cranelift/src/func_environ/gc/null.rs
+++ b/crates/cranelift/src/func_environ/gc/null.rs
@@ -72,7 +72,9 @@ impl NullCompiler {
let vmctx = func_env.vmctx_val(&mut builder.cursor());
let ptr_to_next = func_env
.alias_regions
- .vmctx_gc_heap_data(&mut builder.cursor(), vmctx);
+ .vmctx()
+ .gc_heap_data()
+ .load(&mut builder.cursor(), vmctx);
let next = func_env
.alias_regions
.vmnull_heap_data_bump_finger(&mut builder.cursor(), ptr_to_next);
diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
index e6a84e9a27..56386c797c 100644
--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs
+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
@@ -1012,11 +1012,15 @@ pub(crate) fn tag_address<'a>(
let vmctx = env.vmctx_val(&mut builder.cursor());
let tag_index = wasmtime_environ::TagIndex::from_u32(index);
if let Some(def_index) = env.module.defined_tag_index(tag_index) {
- let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap();
+ let offset = i32::try_from(env.offsets.tags().at(def_index)).unwrap();
builder.ins().iadd_imm_s(vmctx, i64::from(offset))
} else {
+ let import_off = env.offsets.imported_tags().at(tag_index);
env.alias_regions
- .vmctx_vmtag_import_from(&mut builder.cursor(), vmctx, tag_index)
+ .vm_tag_import()
+ .from()
+ .relative_to(import_off)
+ .load(&mut builder.cursor(), vmctx)
}
}
@@ -1033,7 +1037,9 @@ pub fn vmctx_load_stack_chain<'a>(
// First we need to get the `VMStoreContext`.
let vm_store_context = env
.alias_regions
- .vmctx_store_context(&mut builder.cursor(), vmctx);
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx);
let stack_chain_region = env
.alias_regions
@@ -1061,7 +1067,9 @@ pub fn vmctx_store_stack_chain<'a>(
// First we need to get the `VMStoreContext`.
let vm_store_context = env
.alias_regions
- .vmctx_store_context(&mut builder.cursor(), vmctx);
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx);
let stack_chain_region = env
.alias_regions
@@ -1565,7 +1573,9 @@ fn translate_resume_impl<'a>(
// of the invariants that we maintain for the various stack limits.
let vm_runtime_limits_ptr = env
.alias_regions
- .vmctx_store_context(&mut builder.cursor(), vmctx);
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx);
parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true);
resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);
@@ -2032,7 +2042,9 @@ pub(crate) fn translate_switch<'a>(
// switcher continuation.
let vm_runtime_limits_ptr = env
.alias_regions
- .vmctx_store_context(&mut builder.cursor(), vmctx);
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx);
switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false);
let revision = switcher_contref.get_revision(env, builder);
diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rs
index 1b1c148346..937b744527 100644
--- a/crates/environ/src/component/vmcomponent_offsets.rs
+++ b/crates/environ/src/component/vmcomponent_offsets.rs
@@ -1,20 +1,9 @@
-// Currently the `VMComponentContext` allocation by field looks like this:
-//
-// struct VMComponentContext {
-// magic: u32,
-// builtins: &'static VMComponentBuiltins,
-// limits: *const VMStoreContext,
-// may_leave: [VMGlobalDefinition; component.num_runtime_component_instances],
-// task_may_block: u32,
-// trampoline_func_refs: [VMFuncRef; component.num_trampolines],
-// unsafe_intrinsics: [VMFuncRef; component.num_unsafe_intrinsics],
-// lowerings: [VMLowering; component.num_lowerings],
-// memories: [*mut VMMemoryDefinition; component.num_runtime_memories],
-// tables: [VMTable; component.num_runtime_tables],
-// reallocs: [*mut VMFuncRef; component.num_runtime_reallocs],
-// post_returns: [*mut VMFuncRef; component.num_runtime_post_returns],
-// resource_destructors: [*mut VMFuncRef; component.num_resources],
-// }
+//! Offsets of the fields within a `VMComponentContext`.
+//!
+//! The layout itself is not defined here: it is defined once, alongside
+//! `VMContext`'s, in `for_each_vmctx_type!`. Everything in this module is either
+//! generated from that definition or is an offset that is not simply the offset
+//! of one of the layout's fields.
use crate::GetPtrSize;
use crate::PtrSize;
@@ -55,10 +44,11 @@ pub struct VMComponentOffsets<P> {
/// Number of resources within a component which need destructors stored.
pub num_resources: u32,
- // precalculated offsets of various member fields
- magic: u32,
- builtins: u32,
- vm_store_context: u32,
+ // Precalculated offsets of the dynamically-positioned fields, one per entry
+ // in `VMComponentContext`'s `dynamic` section in `for_each_vmctx_type!`,
+ // plus this `VMComponentContext`'s total size. These are all computed by the
+ // generated `compute_field_offsets` and read by the generated accessors of
+ // the same names.
may_leave: u32,
task_may_block: u32,
trampoline_func_refs: u32,
@@ -82,16 +72,66 @@ impl<P: PtrSize> GetPtrSize for VMComponentOffsets<P> {
}
}
-#[inline]
-fn align(offset: u32, align: u32) -> u32 {
- assert!(align.is_power_of_two());
- (offset + (align - 1)) & !(align - 1)
+impl_vmctx_array_index! {
+ RuntimeComponentInstanceIndex,
+ TrampolineIndex,
+ LoweredIndex,
+ RuntimeMemoryIndex,
+ RuntimeTableIndex,
+ RuntimeReallocIndex,
+ RuntimeCallbackIndex,
+ RuntimePostReturnIndex,
+ ResourceIndex,
}
+impl crate::VmctxArrayIndex for UnsafeIntrinsic {
+ #[inline]
+ fn vmctx_array_index(self) -> u32 {
+ self.index()
+ }
+}
+
+/// Generate the offsets of `VMComponentContext`'s dynamically-positioned fields,
+/// ignoring every other vmctx type.
+macro_rules! define_vmcomponent_offsets_dynamic_offsets {
+ (@one VMComponentContext $snake:ident { $($dyn:tt)* }) => {
+ /// Offsets of the dynamically-positioned fields of `VMComponentContext`.
+ impl<P: PtrSize> VMComponentOffsets<P> {
+ define_vmctx_dynamic_offsets!(@accessors (self) [ $($dyn)* ]);
+
+ define_vmctx_dynamic_offsets!(@compute_fn (self, next) $snake [ $($dyn)* ]);
+
+ /// Return the size of the `VMComponentContext` allocation.
+ #[inline]
+ pub fn size_of_vmctx(&self) -> u32 {
+ self.size
+ }
+ }
+ };
+ (@one $other:ident $snake:ident { $($dyn:tt)* }) => {};
+
+ ( $(
+ {
+ $Name:ident $snake:ident
+ static { $($stat:tt)* }
+ dynamic { $($dyn:tt)* }
+ }
+ )* ) => {
+ $( define_vmcomponent_offsets_dynamic_offsets!(@one $Name $snake { $($dyn)* }); )*
+ };
+}
+
+for_each_vmctx_type!(define_vmcomponent_offsets_dynamic_offsets);
+
impl<P: PtrSize> VMComponentOffsets<P> {
/// Creates a new set of offsets for the `component` specified configured
/// additionally for the `ptr` size specified.
pub fn new(ptr: P, component: &Component) -> Self {
+ // This is required by the implementation of
+ // `VMComponentContext::from_opaque`. If this value changes then this
+ // location needs to be updated.
+ assert_eq!(ptr.vmcomponent().magic(), 0);
+
let mut ret = Self {
ptr,
num_lowerings: component.num_lowerings,
@@ -125,9 +165,6 @@ impl<P: PtrSize> VMComponentOffsets<P> {
0
},
num_resources: component.num_resources,
- magic: 0,
- builtins: 0,
- vm_store_context: 0,
may_leave: 0,
task_may_block: 0,
trampoline_func_refs: 0,
@@ -142,58 +179,9 @@ impl<P: PtrSize> VMComponentOffsets<P> {
size: 0,
};
- // Convenience functions for checked addition and multiplication.
- // As side effect this reduces binary size by using only a single
- // `#[track_caller]` location for each function instead of one for
- // each individual invocation.
- #[inline]
- fn cmul(count: u32, size: u8) -> u32 {
- count.checked_mul(u32::from(size)).unwrap()
- }
-
- let mut next_field_offset = 0;
-
- macro_rules! fields {
- (size($field:ident) = $size:expr, $($rest:tt)*) => {
- ret.$field = next_field_offset;
- next_field_offset = next_field_offset.checked_add(u32::from($size)).unwrap();
- fields!($($rest)*);
- };
- (align($align:expr), $($rest:tt)*) => {
- next_field_offset = align(next_field_offset, $align);
- fields!($($rest)*);
- };
- () => {};
- }
-
- fields! {
- size(magic) = 4u32,
- align(u32::from(ret.ptr.size())),
- size(builtins) = ret.ptr.size(),
- size(vm_store_context) = ret.ptr.size(),
- align(16),
- size(may_leave) = cmul(ret.num_runtime_component_instances, ret.ptr.vm_global_definition().size()),
- size(task_may_block) = ret.ptr.vm_global_definition().size(),
- align(u32::from(ret.ptr.size())),
- size(trampoline_func_refs) = cmul(ret.num_trampolines, ret.ptr.vm_func_ref().size()),
- size(intrinsic_func_refs) = cmul(ret.num_unsafe_intrinsics, ret.ptr.vm_func_ref().size()),
- size(lowerings) = cmul(ret.num_lowerings, ret.ptr.size() * 2),
- size(memories) = cmul(ret.num_runtime_memories, ret.ptr.size()),
- size(tables) = cmul(ret.num_runtime_tables, ret.ptr.vm_table_import().size()),
- size(reallocs) = cmul(ret.num_runtime_reallocs, ret.ptr.size()),
- size(callbacks) = cmul(ret.num_runtime_callbacks, ret.ptr.size()),
- size(post_returns) = cmul(ret.num_runtime_post_returns, ret.ptr.size()),
- size(resource_destructors) = cmul(ret.num_resources, ret.ptr.size()),
- }
-
- ret.size = next_field_offset;
-
- // This is required by the implementation of
- // `VMComponentContext::from_opaque`. If this value changes then this
- // location needs to be updated.
- assert_eq!(ret.magic, 0);
+ ret.compute_field_offsets();
- return ret;
+ ret
}
/// The size, in bytes, of the host pointer.
@@ -202,86 +190,16 @@ impl<P: PtrSize> VMComponentOffsets<P> {
self.ptr.size()
}
- /// The offset of the `magic` field.
- #[inline]
- pub fn magic(&self) -> u32 {
- self.magic
- }
-
- /// The offset of the `builtins` field.
- #[inline]
- pub fn builtins(&self) -> u32 {
- self.builtins
- }
-
- /// The offset of the `may_leave` flag for the given component instance.
- #[inline]
- pub fn may_leave(&self, index: RuntimeComponentInstanceIndex) -> u32 {
- assert!(index.as_u32() < self.num_runtime_component_instances);
- self.may_leave + index.as_u32() * u32::from(self.ptr.vm_global_definition().size())
- }
-
- /// The offset of the `task_may_block` field.
- pub fn task_may_block(&self) -> u32 {
- self.task_may_block
- }
-
- /// The offset of the `vm_store_context` field.
- #[inline]
- pub fn vm_store_context(&self) -> u32 {
- self.vm_store_context
- }
-
- /// The offset of the `trampoline_func_refs` field.
- #[inline]
- pub fn trampoline_func_refs(&self) -> u32 {
- self.trampoline_func_refs
- }
-
- /// The offset of `VMFuncRef` for the `index` specified.
- #[inline]
- pub fn trampoline_func_ref(&self, index: TrampolineIndex) -> u32 {
- assert!(index.as_u32() < self.num_trampolines);
- self.trampoline_func_refs() + index.as_u32() * u32::from(self.ptr.vm_func_ref().size())
- }
-
- /// The offset of the `unsafe_intrinsic_func_refs` field.
- #[inline]
- pub fn unsafe_intrinsic_func_refs(&self) -> u32 {
- self.intrinsic_func_refs
- }
-
- /// The offset of the `VMFuncRef` for the `intrinsic` specified.
- #[inline]
- pub fn unsafe_intrinsic_func_ref(&self, intrinsic: UnsafeIntrinsic) -> u32 {
- assert!(intrinsic.index() < self.num_unsafe_intrinsics);
- self.unsafe_intrinsic_func_refs()
- + intrinsic.index() * u32::from(self.ptr.vm_func_ref().size())
- }
-
- /// The offset of the `lowerings` field.
- #[inline]
- pub fn lowerings(&self) -> u32 {
- self.lowerings
- }
-
- /// The offset of the `VMLowering` for the `index` specified.
- #[inline]
- pub fn lowering(&self, index: LoweredIndex) -> u32 {
- assert!(index.as_u32() < self.num_lowerings);
- self.lowerings() + index.as_u32() * u32::from(2 * self.ptr.size())
- }
-
/// The offset of the `callee` for the `index` specified.
#[inline]
pub fn lowering_callee(&self, index: LoweredIndex) -> u32 {
- self.lowering(index) + self.lowering_callee_offset()
+ self.lowerings().at(index) + self.lowering_callee_offset()
}
/// The offset of the `data` for the `index` specified.
#[inline]
pub fn lowering_data(&self, index: LoweredIndex) -> u32 {
- self.lowering(index) + self.lowering_data_offset()
+ self.lowerings().at(index) + self.lowering_data_offset()
}
/// The size of the `VMLowering` type
@@ -301,93 +219,4 @@ impl<P: PtrSize> VMComponentOffsets<P> {
pub fn lowering_data_offset(&self) -> u32 {
u32::from(self.ptr.size())
}
-
- /// The offset of the base of the `runtime_memories` field
- #[inline]
- pub fn runtime_memories(&self) -> u32 {
- self.memories
- }
-
- /// The offset of the `*mut VMMemoryDefinition` for the runtime index
- /// provided.
- #[inline]
- pub fn runtime_memory(&self, index: RuntimeMemoryIndex) -> u32 {
- assert!(index.as_u32() < self.num_runtime_memories);
- self.runtime_memories() + index.as_u32() * u32::from(self.ptr.size())
- }
-
- /// The offset of the base of the `runtime_tables` field
- #[inline]
- pub fn runtime_tables(&self) -> u32 {
- self.tables
- }
-
- /// The offset of the table for the runtime index provided.
- #[inline]
- pub fn runtime_table(&self, index: RuntimeTableIndex) -> u32 {
- assert!(index.as_u32() < self.num_runtime_tables);
- self.runtime_tables() + index.as_u32() * u32::from(self.ptr.vm_table_import().size())
- }
-
- /// The offset of the base of the `runtime_reallocs` field
- #[inline]
- pub fn runtime_reallocs(&self) -> u32 {
- self.reallocs
- }
-
- /// The offset of the `*mut VMFuncRef` for the runtime index
- /// provided.
- #[inline]
- pub fn runtime_realloc(&self, index: RuntimeReallocIndex) -> u32 {
- assert!(index.as_u32() < self.num_runtime_reallocs);
- self.runtime_reallocs() + index.as_u32() * u32::from(self.ptr.size())
- }
-
- /// The offset of the base of the `runtime_callbacks` field
- #[inline]
- pub fn runtime_callbacks(&self) -> u32 {
- self.callbacks
- }
-
- /// The offset of the `*mut VMFuncRef` for the runtime index
- /// provided.
- #[inline]
- pub fn runtime_callback(&self, index: RuntimeCallbackIndex) -> u32 {
- assert!(index.as_u32() < self.num_runtime_callbacks);
- self.runtime_callbacks() + index.as_u32() * u32::from(self.ptr.size())
- }
-
- /// The offset of the base of the `runtime_post_returns` field
- #[inline]
- pub fn runtime_post_returns(&self) -> u32 {
- self.post_returns
- }
-
- /// The offset of the `*mut VMFuncRef` for the runtime index
- /// provided.
- #[inline]
- pub fn runtime_post_return(&self, index: RuntimePostReturnIndex) -> u32 {
- assert!(index.as_u32() < self.num_runtime_post_returns);
- self.runtime_post_returns() + index.as_u32() * u32::from(self.ptr.size())
- }
-
- /// The offset of the base of the `resource_destructors` field
- #[inline]
- pub fn resource_destructors(&self) -> u32 {
- self.resource_destructors
- }
-
- /// The offset of the `*mut VMFuncRef` for the runtime index
- /// provided.
- #[inline]
- pub fn resource_destructor(&self, index: ResourceIndex) -> u32 {
- assert!(index.as_u32() < self.num_resources);
- self.resource_destructors() + index.as_u32() * u32::from(self.ptr.size())
- }
-
- /// Return the size of the `VMComponentContext` allocation.
- #[inline]
- pub fn size_of_vmctx(&self) -> u32 {
- self.size
- }
}
diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs
index 2be10ed159..774f140423 100644
--- a/crates/environ/src/lib.rs
+++ b/crates/environ/src/lib.rs
@@ -43,6 +43,8 @@ mod trap_encoding;
mod tunables;
mod types;
#[macro_use]
+mod vmctxtypes;
+#[macro_use]
mod vmtypes;
mod vmoffsets;
mod wasm_error;
@@ -66,6 +68,7 @@ pub use crate::string_pool::{Atom, StringPool};
pub use crate::trap_encoding::*;
pub use crate::tunables::*;
pub use crate::types::*;
+pub use crate::vmctxtypes::{ArrayOffsets, VmctxArrayIndex};
pub use crate::vmoffsets::*;
pub use crate::wasm_error::*;
pub use object;
diff --git a/crates/environ/src/vmctxtypes.rs b/crates/environ/src/vmctxtypes.rs
new file mode 100644
index 0000000000..ddac95c64e
--- /dev/null
+++ b/crates/environ/src/vmctxtypes.rs
@@ -0,0 +1,514 @@
+//! Centralized definitions of the layouts of Wasmtime's two "vmctx" types:
+//! `VMContext`, the runtime context for a core Wasm instance, and
+//! `VMComponentContext`, the runtime context for a component.
+//!
+//! Unlike the `VM*` types defined by `for_each_vm_type!`, neither of these has
+//! a corresponding `#[repr(C)]` Rust `struct`: their sizes depend on the module
+//! or component being instantiated, so they are dynamically laid out and
+//! accessed exclusively through the computed offsets in `VMOffsets` and
+//! `VMComponentOffsets`. That layout is defined exactly once here, via the
+//! higher-order `for_each_vmctx_type!` macro, and each consumer generates its
+//! view of it from that single source of truth.
+
+/// Invoke the given macro `$mac` once, passing it the layout of each of
+/// Wasmtime's "vmctx" types.
+///
+/// This is a higher-order macro: callers define a `macro_rules!` macro that
+/// matches the grammar described below and pass its name as an argument to this
+/// macro's invocation, e.g. `for_each_vmctx_type!(define_vmctx_offsets)`.
+///
+/// # Grammar
+///
+/// The layout below is written in exactly the same grammar that it is handed to
+/// `$mac` in: this macro has a single rule, and that rule does nothing but
+/// forward those tokens along. Writing the layout in the grammar that consumers
+/// match makes it more verbose than a bespoke input syntax would be, but in
+/// exchange there is no normalization pass in between, so what a consumer matches
+/// is exactly what a reader of the layout sees.
+///
+/// `$mac` receives one brace-delimited group per vmctx type:
+///
+/// ```ignore
+/// {
+/// VMContext vmctx
+/// static { ...entries... }
+/// dynamic { ...entries... }
+/// }
+/// ```
+///
+/// where `vmctx` is the name used for accessor methods of that type, `static`
+/// holds the fixed-width prefix whose offsets depend only on the target pointer
+/// size, and `dynamic` holds the rest, whose offsets additionally depend on the
+/// module or component being compiled.
+///
+/// Each entry within a section is a keyword naming the entry's shape followed by
+/// a single delimited group, so that a consumer can iterate over a section with
+/// `$( ...$kind:ident $entry:tt... )*` and dispatch on one whole entry at a time
+/// instead of munching the section token by token. An entry is one of:
+///
+/// * `align { ptr }` or `align { N }`: round the running offset up to the target
+/// pointer size, or to `N` bytes. Alignment is *always* explicit: it is never
+/// derived from a field's type, because the layout being described here does
+/// not necessarily align every field to its natural alignment, and inserting
+/// padding that the layout does not actually have would silently corrupt every
+/// subsequent offset.
+///
+/// * `field { <attrs> <name>: <ty> }`: a single field.
+///
+/// * `array { <attrs> <name>[<count>; <IndexType>]: <ty> }` (`dynamic` only): an
+/// array of `self.<count>` elements, indexed by `<IndexType>`.
+///
+/// * `optional { <attrs> <name>[if <flag>]: <ty> }` (`dynamic` only): a field that
+/// is present when `self.<flag>` is true and absent (zero-sized) otherwise.
+///
+/// A field's type is always the last thing in its entry. Consumers capture it
+/// as a trailing `$($fty:tt)*` and re-dispatch on its tokens only where they
+/// actually need to classify it into a size or a Cranelift type; a `:ty`
+/// capture would be opaque forever, and so could never be classified at
+/// all.
+///
+/// `<attrs>` is a possibly-empty sequence of marker attributes, which consumers
+/// match with `$(# $fattr:tt)*`. They do not affect the layout, but they do affect
+/// the accesses generated for the field:
+///
+/// * `#[aggregate]`: this field is a composite (a nested struct, or an array of
+/// them) rather than a single scalar. Compiled code accesses such a field's
+/// interior piecewise, so there is no one Cranelift type for the field as a
+/// whole and no alias-region accessor is generated for it. Its offset is still
+/// generated, since that is what interior accesses are computed relative to.
+///
+/// * `#[readonly]` and/or `#[can_move]`: describe how Cranelift may treat loads
+/// and stores of this field.
+///
+/// * `#[access_as = Type]`: this field is *declared* with one type (because that
+/// is what determines its size and stride) but *accessed* as another. For
+/// example, the component context's `may_leave` flags are each stored in a
+/// whole `VMGlobalDefinition` but only ever accessed as a `u32`.
+///
+/// Doc comments are deliberately *not* accepted on fields; use `//` comments for
+/// prose about the layout. Accessor documentation is synthesized from the field
+/// names instead, so that there is only one place a field can be described.
+///
+/// A consumer that only cares about one of the two types can filter with a
+/// literal-name arm followed by a catch-all, e.g.
+///
+/// ```ignore
+/// (@one VMContext $snake:ident dynamic { $($dyn:tt)* }) => { ...generate... };
+/// (@one $other:ident $snake:ident dynamic { $($dyn:tt)* }) => {};
+/// ```
+#[macro_export]
+macro_rules! for_each_vmctx_type {
+ ($mac:ident) => {
+ $mac! {
+ {
+ VMContext vmctx
+
+ // Fixed-width data comes first so that the calculation of these
+ // fields' offsets is a compile-time constant when using
+ // `HostPtr`.
+ static {
+ field { #[readonly] #[can_move] magic: u32 }
+
+ // NB: this is where the four bytes of padding after `magic`
+ // live on targets with eight-byte pointers.
+ align { ptr }
+
+ field { #[readonly] #[can_move] store_context: VmPtr<VMStoreContext> }
+
+ field { #[readonly] #[can_move] builtin_functions: VmPtr<VMBuiltinFunctionsArray> }
+
+ field { epoch_ptr: VmPtr<AtomicU64> }
+
+ // A pointer that different collectors use however they see
+ // fit.
+ field { #[readonly] #[can_move] gc_heap_data: VmPtr<u8> }
+
+ field { #[readonly] #[can_move] type_ids: VmPtr<VMSharedTypeIndex> }
+ }
+
+ // Variable-width fields come after the fixed-width fields
+ // above. Memory-related items are placed first as they are some
+ // of the most frequently accessed items, and minimizing their
+ // offset can shrink the size of load/store instruction offset
+ // immediates on platforms like x64 and Pulley (e.g. fit in an
+ // 8-bit offset instead of needing a 32-bit offset).
+ dynamic {
+ array {
+ #[aggregate]
+ imported_memories[num_imported_memories; MemoryIndex]: VMMemoryImport
+ }
+
+ array {
+ #[readonly]
+ #[can_move]
+ memories[num_defined_memories; DefinedMemoryIndex]: VmPtr<VMMemoryDefinition>
+ }
+
+ array {
+ #[aggregate]
+ owned_memories[num_owned_memories; OwnedMemoryIndex]: VMMemoryDefinition
+ }
+
+ array {
+ #[aggregate]
+ imported_functions[num_imported_functions; FuncIndex]: VMFunctionImport
+ }
+
+ array {
+ #[aggregate]
+ imported_tables[num_imported_tables; TableIndex]: VMTableImport
+ }
+
+ array {
+ #[aggregate]
+ imported_globals[num_imported_globals; GlobalIndex]: VMGlobalImport
+ }
+
+ array {
+ #[aggregate]
+ imported_tags[num_imported_tags; TagIndex]: VMTagImport
+ }
+
+ array {
+ #[aggregate]
+ tables[num_defined_tables; DefinedTableIndex]: VMTableDefinition
+ }
+
+ align { 16 }
+
+ array {
+ #[aggregate]
+ globals[num_defined_globals; DefinedGlobalIndex]: VMGlobalDefinition
+ }
+
+ array {
+ #[aggregate]
+ tags[num_defined_tags; DefinedTagIndex]: VMTagDefinition
+ }
+
+ array {
+ #[aggregate]
+ func_refs[num_escaped_funcs; FuncRefIndex]: VMFuncRef
+ }
+
+ optional {
+ #[aggregate]
+ startup_func_ref[if has_startup_func]: VMFuncRef
+ }
+
+ array {
+ runtime_data_bases[num_runtime_data; RuntimeDataIndex]: VmPtr<u8>
+ }
+
+ array {
+ runtime_data_lengths[num_runtime_data; RuntimeDataIndex]: u32
+ }
+ }
+ }
+
+ {
+ VMComponentContext vmcomponent
+
+ static {
+ // NB: `magic` must be at offset zero; this is relied upon by
+ // `VMComponentContext::from_opaque`.
+ field { #[readonly] #[can_move] magic: u32 }
+
+ align { ptr }
+
+ field { #[readonly] builtins: VmPtr<VMComponentBuiltins> }
+
+ field { #[readonly] #[can_move] store_context: VmPtr<VMStoreContext> }
+ }
+
+ dynamic {
+ align { 16 }
+
+ // Each of these flags gets a whole `VMGlobalDefinition`'s
+ // worth of space, but only its first four bytes are ever
+ // accessed.
+ array {
+ #[access_as = u32]
+ may_leave[num_runtime_component_instances; RuntimeComponentInstanceIndex]: VMGlobalDefinition
+ }
+
+ field { #[access_as = u32] task_may_block: VMGlobalDefinition }
+
+ align { ptr }
+
+ array {
+ #[aggregate]
+ trampoline_func_refs[num_trampolines; TrampolineIndex]: VMFuncRef
+ }
+
+ array {
+ #[aggregate]
+ intrinsic_func_refs[num_unsafe_intrinsics; UnsafeIntrinsic]: VMFuncRef
+ }
+
+ array {
+ #[aggregate]
+ lowerings[num_lowerings; LoweredIndex]: VMLowering
+ }
+
+ array {
+ memories
+ [num_runtime_memories; RuntimeMemoryIndex]: VmPtr<VMMemoryDefinition>
+ }
+
+ array {
+ #[aggregate]
+ tables[num_runtime_tables; RuntimeTableIndex]: VMTableImport
+ }
+
+ array {
+ reallocs[num_runtime_reallocs; RuntimeReallocIndex]: VmPtr<VMFuncRef>
+ }
+
+ array {
+ callbacks[num_runtime_callbacks; RuntimeCallbackIndex]: VmPtr<VMFuncRef>
+ }
+
+ array {
+ post_returns[num_runtime_post_returns; RuntimePostReturnIndex]: VmPtr<VMFuncRef>
+ }
+
+ array {
+ #[readonly]
+ resource_destructors[num_resources; ResourceIndex]: VmPtr<VMFuncRef>
+ }
+ }
+ }
+ }
+ };
+}
+
+/// Round `offset` up to a multiple of `align`.
+#[inline]
+pub(crate) fn align_up(offset: u32, align: u32) -> u32 {
+ debug_assert!(align.is_power_of_two());
+ (offset + (align - 1)) & !(align - 1)
+}
+
+/// Add two offsets, panicking on overflow.
+#[inline]
+pub(crate) fn cadd(a: u32, b: u32) -> u32 {
+ a.checked_add(b).unwrap()
+}
+
+/// Multiply an element count by an element size, panicking on overflow.
+#[inline]
+pub(crate) fn cmul(count: u32, size: u32) -> u32 {
+ count.checked_mul(size).unwrap()
+}
+
+/// Classify one of the field types accepted by `for_each_vmctx_type!` to its
+/// size in bytes as a `u32`, given a `PtrSize`.
+#[allow(
+ unused_macro_rules,
+ reason = "some element types only appear in `VMComponentContext`, and so are \
+ unused when the `component-model` feature is disabled"
+)]
+macro_rules! vmctx_field_size {
+ (($p:expr) u32) => {
+ 4u32
+ };
+ (($p:expr) VmPtr < $g:ident >) => {
+ u32::from($p)
+ };
+ // `VMLowering` is a pair of pointers, and is not itself defined by
+ // `for_each_vm_type!`.
+ (($p:expr) VMLowering) => {
+ 2u32 * u32::from($p)
+ };
+ // Anything else names one of the `VM*` types, and so has an
+ // `offsets::VMFoo` generated for it by `for_each_vm_type!`. Deferring to
+ // that keeps this macro from having to mirror the list of `VM*` types, and
+ // a type that has no such entry fails to resolve here.
+ (($p:expr) $Name:ident) => {
+ u32::from(crate::vmoffsets::offsets::$Name($p).size())
+ };
+}
+
+/// Classify a `for_each_vmctx_type!` alignment step to its alignment in bytes
+/// as a `u32`, given a `PtrSize`.
+macro_rules! vmctx_align_value {
+ (($p:expr) ptr) => {
+ u32::from($p)
+ };
+ (($p:expr) $n:literal) => {
+ $n
+ };
+}
+
+/// Generate the accessors for, and the layout computation of, the
+/// dynamically-positioned fields of one of the vmctx types.
+#[allow(
+ unused_macro_rules,
+ reason = "single dynamically-positioned fields only appear in \
+ `VMComponentContext`, and so are unused when the `component-model` \
+ feature is disabled"
+)]
+macro_rules! define_vmctx_dynamic_offsets {
+ (@accessors ($s:ident) [ $($kind:ident $entry:tt)* ]) => {
+ $( define_vmctx_dynamic_offsets!(@accessor ($s) $kind $entry); )*
+ };
+
+ (@accessor ($s:ident) align { $al:tt }) => {};
+
+ (@accessor ($s:ident) field { $(# $fattr:tt)* $fname:ident : $($fty:tt)* }) => {
+ #[doc = concat!("The offset of the `", stringify!($fname), "` field.")]
+ #[inline]
+ pub fn $fname(&$s) -> u32 {
+ $s.$fname
+ }
+ };
+
+ (@accessor ($s:ident) array {
+ $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*
+ }) => {
+ #[doc = concat!("The offsets of the `", stringify!($fname), "` array.")]
+ #[inline]
+ pub fn $fname(&$s) -> $crate::ArrayOffsets<$Index> {
+ $crate::ArrayOffsets::new(
+ $s.$fname,
+ vmctx_field_size!(($s.ptr.size()) $($fty)*),
+ $s.$count,
+ )
+ }
+ };
+
+ (@accessor ($s:ident) optional {
+ $(# $fattr:tt)* $fname:ident [ if $flag:ident ] : $($fty:tt)*
+ }) => {
+ #[doc = concat!(
+ "The offset of the `", stringify!($fname), "` field.\n\n",
+ "Panics if `", stringify!($flag), "` is false, in which case this \
+ field is not present at all."
+ )]
+ #[inline]
+ pub fn $fname(&$s) -> u32 {
+ assert!($s.$flag);
+ $s.$fname
+ }
+ };
+
+ (@compute_fn ($s:ident, $next:ident) $snake:ident [ $($kind:ident $entry:tt)* ]) => {
+ /// Compute the offset of each dynamically-positioned field, and this
+ /// vmctx's total size.
+ fn compute_field_offsets(&mut $s) {
+ let mut $next = u32::from($s.ptr.$snake().end_of_static_fields());
+ $( define_vmctx_dynamic_offsets!(@compute ($s, $next) $kind $entry); )*
+ $s.size = $next;
+ }
+ };
+
+ (@compute ($s:ident, $next:ident) align { $al:tt }) => {
+ $next = crate::vmctxtypes::align_up($next, vmctx_align_value!(($s.ptr.size()) $al));
+ };
+
+ (@compute ($s:ident, $next:ident) field {
+ $(# $fattr:tt)* $fname:ident : $($fty:tt)*
+ }) => {
+ $s.$fname = $next;
+ $next = crate::vmctxtypes::cadd($next, vmctx_field_size!(($s.ptr.size()) $($fty)*));
+ };
+
+ (@compute ($s:ident, $next:ident) array {
+ $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*
+ }) => {
+ $s.$fname = $next;
+ $next = crate::vmctxtypes::cadd(
+ $next,
+ crate::vmctxtypes::cmul($s.$count, vmctx_field_size!(($s.ptr.size()) $($fty)*)),
+ );
+ };
+
+ (@compute ($s:ident, $next:ident) optional {
+ $(# $fattr:tt)* $fname:ident [ if $flag:ident ] : $($fty:tt)*
+ }) => {
+ $s.$fname = $next;
+ $next = crate::vmctxtypes::cadd(
+ $next,
+ if $s.$flag {
+ vmctx_field_size!(($s.ptr.size()) $($fty)*)
+ } else {
+ 0
+ },
+ );
+ };
+}
+
+/// The offsets of one array field within a vmctx.
+#[derive(Debug, Clone, Copy)]
+pub struct ArrayOffsets<I> {
+ begin: u32,
+ stride: u32,
+ count: u32,
+ _index: core::marker::PhantomData<I>,
+}
+
+impl<I> ArrayOffsets<I> {
+ /// Create the offsets for an array of `count` elements which begins at
+ /// `begin` within its vmctx and whose elements are `stride` bytes apart.
+ #[inline]
+ pub fn new(begin: u32, stride: u32, count: u32) -> Self {
+ ArrayOffsets {
+ begin,
+ stride,
+ count,
+ _index: core::marker::PhantomData,
+ }
+ }
+
+ /// The offset of the start of this array within its vmctx.
+ #[inline]
+ pub fn begin(&self) -> u32 {
+ self.begin
+ }
+
+ /// The number of bytes between the start of consecutive elements of this
+ /// array.
+ #[inline]
+ pub fn stride(&self) -> u32 {
+ self.stride
+ }
+
+ /// The number of elements in this array.
+ #[inline]
+ pub fn count(&self) -> u32 {
+ self.count
+ }
+}
+
+impl<I: VmctxArrayIndex> ArrayOffsets<I> {
+ /// The offset of the given element within this array's vmctx.
+ ///
+ /// Panics if `index` is out of bounds for this array.
+ #[inline]
+ pub fn at(&self, index: I) -> u32 {
+ let index = index.vmctx_array_index();
+ assert!(index < self.count);
+ self.begin + index * self.stride
+ }
+}
+
+/// An index type that can be used to index one of a vmctx's arrays.
+pub trait VmctxArrayIndex: Copy {
+ /// This index's position within its array.
+ fn vmctx_array_index(self) -> u32;
+}
+
+/// Implement `VmctxArrayIndex` for entity references.
+macro_rules! impl_vmctx_array_index {
+ ($($ty:ty),* $(,)?) => {
+ $(
+ impl $crate::VmctxArrayIndex for $ty {
+ #[inline]
+ fn vmctx_array_index(self) -> u32 {
+ self.as_u32()
+ }
+ }
+ )*
+ };
+}
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 79f7bda603..f4f6b33b14 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -1,47 +1,16 @@
//! Offsets and sizes of various structs in `wasmtime::runtime::vm::*` that are
//! accessed directly by compiled Wasm code.
-// Currently the `VMContext` allocation by field looks like this:
-//
-// struct VMContext {
-// // Fixed-width data comes first so the calculation of the offset of
-// // these fields is a compile-time constant when using `HostPtr`.
-// magic: u32,
-// _padding: u32, // (On 64-bit systems)
-// vm_store_context: *const VMStoreContext,
-// builtin_functions: *mut VMBuiltinFunctionsArray,
-// epoch_ptr: *mut AtomicU64,
-// gc_heap_data: *mut T, // Collector-specific pointer
-// type_ids: *const VMSharedTypeIndex,
-//
-// // Variable-width fields come after the fixed-width fields above. Place
-// // memory-related items first as they're some of the most frequently
-// // accessed items and minimizing their offset in this structure can
-// // shrink the size of load/store instruction offset immediates on
-// // platforms like x64 and Pulley (e.g. fit in an 8-bit offset instead
-// // of needing a 32-bit offset)
-// imported_memories: [VMMemoryImport; module.num_imported_memories],
-// memories: [*mut VMMemoryDefinition; module.num_defined_memories],
-// owned_memories: [VMMemoryDefinition; module.num_owned_memories],
-// imported_functions: [VMFunctionImport; module.num_imported_functions],
-// imported_tables: [VMTableImport; module.num_imported_tables],
-// imported_globals: [VMGlobalImport; module.num_imported_globals],
-// imported_tags: [VMTagImport; module.num_imported_tags],
-// tables: [VMTableDefinition; module.num_defined_tables],
-// globals: [VMGlobalDefinition; module.num_defined_globals],
-// tags: [VMTagDefinition; module.num_defined_tags],
-// func_refs: [VMFuncRef; module.num_escaped_funcs],
-// startup_func_ref: [VMFuncRef; module.has_startup_func ? 1 : 0],
-// runtime_data_bases: [*const u8; module.num_runtime_data],
-// runtime_data_lengths: [u32; module.num_runtime_data],
-// }
+// The `VMContext` layout is not defined here: it is defined once, alongside
+// `VMComponentContext`'s, in `for_each_vmctx_type!`. Everything in this module
+// is either generated from that definition or is an offset that is not simply
+// the offset of one of the layout's fields.
use crate::{
DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex,
FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex,
TagIndex,
};
-use cranelift_entity::packed_option::ReservedValue;
/// Number of slots in for `component_context` in the `VMStoreContext`. This is
/// defined by the component model's `context.{get,set}` intrinsics.
@@ -243,25 +212,87 @@ macro_rules! define_vm_type_offsets {
$($body:tt)*
}
)* ) => {
- /// Offsets of fields within the various `VM*` types, parameterized over
- /// a target [`PtrSize`] so that they can be computed during cross
- /// compilation.
- ///
- /// These types are namespaced within their own module so that they never
- /// collide with the real definitions of the `VM*` types themselves.
- pub mod offsets {
- use super::{align, PtrSize, NUM_COMPONENT_CONTEXT_SLOTS};
+ $(
+ #[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")]
+ pub struct $Name<P: PtrSize>(pub P);
- $(
- #[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")]
- pub struct $Name<P: PtrSize>(pub P);
+ define_vm_type_offsets!(@impl $Name [$($repr)*] {} $($body)*);
+ )*
+ };
+}
+
+/// Generate a `struct VMContext<P: PtrSize>(P)`-style wrapper for each vmctx
+/// type, with a method per statically-positioned field returning that field's
+/// offset.
+macro_rules! define_vmctx_static_offsets {
+ // Munch the `static` section, threading a closed-form expression for the
+ // running offset.
+ (@chain $p:ident [ $($prev:tt)* ] []) => {
+ /// The offset just past this type's last statically-positioned field.
+ ///
+ /// Everything after this point is dynamically sized.
+ #[inline]
+ pub fn end_of_static_fields(&self) -> u8 {
+ let $p = self.0.size();
+ let _ = $p;
+ u8::try_from($($prev)*).unwrap()
+ }
+ };
+ (@chain $p:ident [ $($prev:tt)* ] [ align { $al:tt } $($rest:tt)* ]) => {
+ define_vmctx_static_offsets!(
+ @chain $p
+ [ crate::vmctxtypes::align_up($($prev)*, vmctx_align_value!(($p) $al)) ]
+ [ $($rest)* ]
+ );
+ };
+ (@chain $p:ident [ $($prev:tt)* ] [
+ field { $(# $fattr:tt)* $fname:ident : $($fty:tt)* } $($rest:tt)*
+ ]) => {
+ #[doc = concat!("The offset of the `", stringify!($fname), "` field.")]
+ #[inline]
+ pub fn $fname(&self) -> u8 {
+ let $p = self.0.size();
+ let _ = $p;
+ u8::try_from($($prev)*).unwrap()
+ }
+ define_vmctx_static_offsets!(
+ @chain $p
+ [ $($prev)* + vmctx_field_size!(($p) $($fty)*) ]
+ [ $($rest)* ]
+ );
+ };
- define_vm_type_offsets!(@impl $Name [$($repr)*] {} $($body)*);
- )*
+ ( $(
+ {
+ $Name:ident $snake:ident
+ static { $($stat:tt)* }
+ dynamic { $($dyn:tt)* }
}
+ )* ) => {
+ $(
+ #[doc = concat!("Offsets of the statically-positioned fields within the `",
+ stringify!($Name), "` type.")]
+ pub struct $Name<P: PtrSize>(pub P);
+
+ impl<P: PtrSize> $Name<P> {
+ define_vmctx_static_offsets!(@chain ptr [ 0u32 ] [ $($stat)* ]);
+ }
+ )*
};
}
-for_each_vm_type!(define_vm_type_offsets);
+
+/// Offsets of fields within the various `VM*` types and within the two vmctx
+/// types, parameterized over a target `PtrSize` so that they can be computed
+/// during cross compilation.
+///
+/// These types are namespaced within their own module so that they never collide
+/// with the real definitions of the `VM*` types themselves.
+pub mod offsets {
+ use super::{NUM_COMPONENT_CONTEXT_SLOTS, PtrSize, align};
+
+ for_each_vm_type!(define_vm_type_offsets);
+ for_each_vmctx_type!(define_vmctx_static_offsets);
+}
/// The size, in bytes, of one `context.{get,set}` slot. These slots are `u32`s,
/// both in `VMStoreContext::component_context` and in
@@ -303,9 +334,8 @@ impl<P: PtrSize> offsets::VMDeferredThread<P> {
}
}
-/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to [`PtrSize`] for
-/// each `VM*` type, using the `#[snake_name = ...]` attribute for the method
-/// name.
+/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to `PtrSize` for
+/// each `VM*` type.
macro_rules! define_ptr_size_vm_type_accessors {
( $(
$(#[doc = $sdoc:literal])*
@@ -313,8 +343,6 @@ macro_rules! define_ptr_size_vm_type_accessors {
#[repr($($repr:tt)*)]
#[snake_name = $snake:ident]
$svis:vis struct $Name:ident {
- // This macro only needs each type's name and snake name, so the
- // body is captured raw rather than parsed into fields.
$($body:tt)*
}
)* ) => {
@@ -328,6 +356,27 @@ macro_rules! define_ptr_size_vm_type_accessors {
};
}
+/// Add a `fn vmctx(&self) -> offsets::VMContext<&Self>` accessor to `PtrSize`
+/// for each vmctx type.
+macro_rules! define_ptr_size_vmctx_type_accessors {
+ ( $(
+ {
+ $Name:ident $snake:ident
+ static { $($stat:tt)* }
+ dynamic { $($dyn:tt)* }
+ }
+ )* ) => {
+ $(
+ #[doc = concat!("Get the [`offsets::", stringify!($Name),
+ "`] offsets for this pointer size.")]
+ #[inline]
+ fn $snake(&self) -> offsets::$Name<&Self> {
+ offsets::$Name(self)
+ }
+ )*
+ };
+}
+
/// This class computes offsets to fields within `VMContext` and other
/// related structs that JIT code accesses directly.
#[derive(Debug, Clone, Copy)]
@@ -362,18 +411,18 @@ pub struct VMOffsets<P> {
/// Whether or not the module has a start function.
pub has_startup_func: bool,
- // precalculated offsets of various member fields
+ // Precalculated offsets of the dynamically-positioned fields.
+ imported_memories: u32,
+ memories: u32,
+ owned_memories: u32,
imported_functions: u32,
imported_tables: u32,
- imported_memories: u32,
imported_globals: u32,
imported_tags: u32,
- defined_tables: u32,
- defined_memories: u32,
- owned_memories: u32,
- defined_globals: u32,
- defined_tags: u32,
- defined_func_refs: u32,
+ tables: u32,
+ globals: u32,
+ tags: u32,
+ func_refs: u32,
startup_func_ref: u32,
runtime_data_bases: u32,
runtime_data_lengths: u32,
@@ -386,23 +435,12 @@ pub trait PtrSize {
fn size(&self) -> u8;
// Generate a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor for each
- // `VM*` type, giving access to that type's field offsets, size, and
- // alignment for this pointer size.
+ // `VM*` type.
for_each_vm_type!(define_ptr_size_vm_type_accessors);
- /// The offset of the `VMContext::store_context` field
- fn vmcontext_store_context(&self) -> u8 {
- u8::try_from(align(
- u32::try_from(core::mem::size_of::<u32>()).unwrap(),
- u32::from(self.size()),
- ))
- .unwrap()
- }
-
- /// The offset of the `VMContext::builtin_functions` field
- fn vmcontext_builtin_functions(&self) -> u8 {
- self.vmcontext_store_context() + self.size()
- }
+ // Generate a `fn vmctx(&self) -> offsets::VMContext<&Self>` accessor for
+ // each vmctx type.
+ for_each_vmctx_type!(define_ptr_size_vmctx_type_accessors);
/// Return the size of `VMSharedTypeIndex`.
#[inline]
@@ -587,43 +625,6 @@ pub trait PtrSize {
self.vmcontref_args() + self.size_of_vmhostarray()
}
- /// Return the offset to the `magic` value in this `VMContext`.
- #[inline]
- fn vmctx_magic(&self) -> u8 {
- // This is required by the implementation of `VMContext::instance` and
- // `VMContext::instance_mut`. If this value changes then those locations
- // need to be updated.
- 0
- }
-
- /// Return the offset to the `VMStoreContext` structure
- #[inline]
- fn vmctx_store_context(&self) -> u8 {
- self.vmctx_magic() + self.size()
- }
-
- /// Return the offset to the `VMBuiltinFunctionsArray` structure
- #[inline]
- fn vmctx_builtin_functions(&self) -> u8 {
- self.vmctx_store_context() + self.size()
- }
-
- /// Return the offset to the `*const AtomicU64` epoch-counter
- /// pointer.
- #[inline]
- fn vmctx_epoch_ptr(&self) -> u8 {
- self.vmctx_builtin_functions() + self.size()
- }
-
- /// Return the offset to the `*mut T` collector-specific data.
- ///
- /// This is a pointer that different collectors can use however they see
- /// fit.
- #[inline]
- fn vmctx_gc_heap_data(&self) -> u8 {
- self.vmctx_epoch_ptr() + self.size()
- }
-
/// Return the offset of the `over_approximated_stack_roots` field within
/// `VMDrcHeapData`.
#[inline]
@@ -682,20 +683,6 @@ pub trait PtrSize {
fn align_of_vmcopying_heap_data(&self) -> u8 {
4
}
-
- /// The offset of the `type_ids` array pointer.
- #[inline]
- fn vmctx_type_ids_array(&self) -> u8 {
- self.vmctx_gc_heap_data() + self.size()
- }
-
- /// The end of statically known offsets in `VMContext`.
- ///
- /// Data after this is dynamically sized.
- #[inline]
- fn vmctx_dynamic_data_start(&self) -> u8 {
- self.vmctx_type_ids_array() + self.size()
- }
}
/// A trait to abstract over various types that contain a `P: PtrSize`.
@@ -871,16 +858,16 @@ impl<P: PtrSize> VMOffsets<P> {
runtime_data_lengths: "runtime data lengths",
runtime_data_bases: "runtime data base pointers",
startup_func_ref: "startup funcref",
- defined_func_refs: "module functions",
- defined_tags: "defined tags",
- defined_globals: "defined globals",
- defined_tables: "defined tables",
+ func_refs: "module functions",
+ tags: "defined tags",
+ globals: "defined globals",
+ tables: "defined tables",
imported_tags: "imported tags",
imported_globals: "imported globals",
imported_tables: "imported tables",
imported_functions: "imported functions",
owned_memories: "owned memories",
- defined_memories: "defined memories",
+ memories: "defined memories",
imported_memories: "imported memories",
}
}
@@ -912,90 +899,24 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
num_escaped_funcs: fields.num_escaped_funcs,
num_runtime_data: fields.num_runtime_data,
has_startup_func: fields.has_startup_func,
+ imported_memories: 0,
+ memories: 0,
+ owned_memories: 0,
imported_functions: 0,
imported_tables: 0,
- imported_memories: 0,
imported_globals: 0,
imported_tags: 0,
- defined_tables: 0,
- defined_memories: 0,
- owned_memories: 0,
- defined_globals: 0,
- defined_tags: 0,
- defined_func_refs: 0,
+ tables: 0,
+ globals: 0,
+ tags: 0,
+ func_refs: 0,
startup_func_ref: 0,
runtime_data_bases: 0,
runtime_data_lengths: 0,
size: 0,
};
-
- // Convenience functions for checked addition and multiplication.
- // As side effect this reduces binary size by using only a single
- // `#[track_caller]` location for each function instead of one for
- // each individual invocation.
- #[inline]
- fn cadd(count: u32, size: u32) -> u32 {
- count.checked_add(size).unwrap()
- }
-
- #[inline]
- fn cmul(count: u32, size: u8) -> u32 {
- count.checked_mul(u32::from(size)).unwrap()
- }
-
- let mut next_field_offset = u32::from(ret.ptr.vmctx_dynamic_data_start());
-
- macro_rules! fields {
- (size($field:ident) = $size:expr, $($rest:tt)*) => {
- ret.$field = next_field_offset;
- next_field_offset = cadd(next_field_offset, u32::from($size));
- fields!($($rest)*);
- };
- (align($align:expr), $($rest:tt)*) => {
- next_field_offset = align(next_field_offset, $align);
- fields!($($rest)*);
- };
- () => {};
- }
-
- fields! {
- size(imported_memories)
- = cmul(ret.num_imported_memories, ret.ptr.vm_memory_import().size()),
- size(defined_memories)
- = cmul(ret.num_defined_memories, ret.ptr.size_of_vmmemory_pointer()),
- size(owned_memories)
- = cmul(ret.num_owned_memories, ret.ptr.vm_memory_definition().size()),
- size(imported_functions)
- = cmul(ret.num_imported_functions, ret.ptr.vm_function_import().size()),
- size(imported_tables)
- = cmul(ret.num_imported_tables, ret.ptr.vm_table_import().size()),
- size(imported_globals)
- = cmul(ret.num_imported_globals, ret.ptr.vm_global_import().size()),
- size(imported_tags)
- = cmul(ret.num_imported_tags, ret.ptr.vm_tag_import().size()),
- size(defined_tables)
- = cmul(ret.num_defined_tables, ret.ptr.vm_table_definition().size()),
- align(16),
- size(defined_globals)
- = cmul(ret.num_defined_globals, ret.ptr.vm_global_definition().size()),
- size(defined_tags)
- = cmul(ret.num_defined_tags, ret.ptr.vm_tag_definition().size()),
- size(defined_func_refs) = cmul(
- ret.num_escaped_funcs,
- ret.ptr.vm_func_ref().size(),
- ),
- size(startup_func_ref) = if ret.has_startup_func {
- ret.ptr.vm_func_ref().size()
- } else {
- 0
- },
- size(runtime_data_bases) = cmul(ret.num_runtime_data, ret.ptr.size()),
- size(runtime_data_lengths) = cmul(ret.num_runtime_data, 4),
- }
-
- ret.size = next_field_offset;
-
- return ret;
+ ret.compute_field_offsets();
+ ret
}
}
@@ -1025,286 +946,50 @@ impl<P: PtrSize> VMOffsets<P> {
}
}
-/// Offsets for `VMContext`.
-impl<P: PtrSize> VMOffsets<P> {
- /// The offset of the `tables` array.
- #[inline]
- pub fn vmctx_imported_functions_begin(&self) -> u32 {
- self.imported_functions
- }
-
- /// The offset of the `tables` array.
- #[inline]
- pub fn vmctx_imported_tables_begin(&self) -> u32 {
- self.imported_tables
- }
-
- /// The offset of the `memories` array.
- #[inline]
- pub fn vmctx_imported_memories_begin(&self) -> u32 {
- self.imported_memories
- }
-
- /// The offset of the `globals` array.
- #[inline]
- pub fn vmctx_imported_globals_begin(&self) -> u32 {
- self.imported_globals
- }
-
- /// The offset of the `tags` array.
- #[inline]
- pub fn vmctx_imported_tags_begin(&self) -> u32 {
- self.imported_tags
- }
-
- /// The offset of the `tables` array.
- #[inline]
- pub fn vmctx_tables_begin(&self) -> u32 {
- self.defined_tables
- }
-
- /// The offset of the `memories` array.
- #[inline]
- pub fn vmctx_memories_begin(&self) -> u32 {
- self.defined_memories
- }
-
- /// The offset of the `owned_memories` array.
- #[inline]
- pub fn vmctx_owned_memories_begin(&self) -> u32 {
- self.owned_memories
- }
-
- /// The offset of the `globals` array.
- #[inline]
- pub fn vmctx_globals_begin(&self) -> u32 {
- self.defined_globals
- }
-
- /// The offset of the `tags` array.
- #[inline]
- pub fn vmctx_tags_begin(&self) -> u32 {
- self.defined_tags
- }
-
- /// The offset of the `func_refs` array.
- #[inline]
- pub fn vmctx_func_refs_begin(&self) -> u32 {
- self.defined_func_refs
- }
-
- /// The offset of the `runtime_data_bases` array.
- #[inline]
- pub fn vmctx_runtime_data_bases_begin(&self) -> u32 {
- self.runtime_data_bases
- }
-
- /// The offset of the `runtime_data_lengths` array.
- #[inline]
- pub fn vmctx_runtime_data_lengths_begin(&self) -> u32 {
- self.runtime_data_lengths
- }
-
- /// Return the size of the `VMContext` allocation.
- #[inline]
- pub fn size_of_vmctx(&self) -> u32 {
- self.size
- }
-
- /// Return the offset to `VMFunctionImport` index `index`.
- #[inline]
- pub fn vmctx_vmfunction_import(&self, index: FuncIndex) -> u32 {
- assert!(index.as_u32() < self.num_imported_functions);
- self.vmctx_imported_functions_begin()
- + index.as_u32() * u32::from(self.ptr.vm_function_import().size())
- }
-
- /// Return the offset to `VMTable` index `index`.
- #[inline]
- pub fn vmctx_vmtable_import(&self, index: TableIndex) -> u32 {
- assert!(index.as_u32() < self.num_imported_tables);
- self.vmctx_imported_tables_begin()
- + index.as_u32() * u32::from(self.ptr.vm_table_import().size())
- }
-
- /// Return the offset to `VMMemoryImport` index `index`.
- #[inline]
- pub fn vmctx_vmmemory_import(&self, index: MemoryIndex) -> u32 {
- assert!(index.as_u32() < self.num_imported_memories);
- self.vmctx_imported_memories_begin()
- + index.as_u32() * u32::from(self.ptr.vm_memory_import().size())
- }
-
- /// Return the offset to `VMGlobalImport` index `index`.
- #[inline]
- pub fn vmctx_vmglobal_import(&self, index: GlobalIndex) -> u32 {
- assert!(index.as_u32() < self.num_imported_globals);
- self.vmctx_imported_globals_begin()
- + index.as_u32() * u32::from(self.ptr.vm_global_import().size())
- }
-
- /// Return the offset to `VMTagImport` index `index`.
- #[inline]
- pub fn vmctx_vmtag_import(&self, index: TagIndex) -> u32 {
- assert!(index.as_u32() < self.num_imported_tags);
- self.vmctx_imported_tags_begin()
- + index.as_u32() * u32::from(self.ptr.vm_tag_import().size())
- }
-
- /// Return the offset to `VMTableDefinition` index `index`.
- #[inline]
- pub fn vmctx_vmtable_definition(&self, index: DefinedTableIndex) -> u32 {
- assert!(index.as_u32() < self.num_defined_tables);
- self.vmctx_tables_begin()
- + index.as_u32() * u32::from(self.ptr.vm_table_definition().size())
- }
-
- /// Return the offset to the `*mut VMMemoryDefinition` at index `index`.
- #[inline]
- pub fn vmctx_vmmemory_pointer(&self, index: DefinedMemoryIndex) -> u32 {
- assert!(index.as_u32() < self.num_defined_memories);
- self.vmctx_memories_begin()
- + index.as_u32() * u32::from(self.ptr.size_of_vmmemory_pointer())
- }
-
- /// Return the offset to the owned `VMMemoryDefinition` at index `index`.
- #[inline]
- pub fn vmctx_vmmemory_definition(&self, index: OwnedMemoryIndex) -> u32 {
- assert!(index.as_u32() < self.num_owned_memories);
- self.vmctx_owned_memories_begin()
- + index.as_u32() * u32::from(self.ptr.vm_memory_definition().size())
- }
-
- /// Return the offset to the `VMGlobalDefinition` index `index`.
- #[inline]
- pub fn vmctx_vmglobal_definition(&self, index: DefinedGlobalIndex) -> u32 {
- assert!(index.as_u32() < self.num_defined_globals);
- self.vmctx_globals_begin()
- + index.as_u32() * u32::from(self.ptr.vm_global_definition().size())
- }
-
- /// Return the offset to the `VMTagDefinition` index `index`.
- #[inline]
- pub fn vmctx_vmtag_definition(&self, index: DefinedTagIndex) -> u32 {
- assert!(index.as_u32() < self.num_defined_tags);
- self.vmctx_tags_begin() + index.as_u32() * u32::from(self.ptr.vm_tag_definition().size())
- }
-
- /// Return the offset to the `VMFuncRef` for the given function
- /// index (either imported or defined).
- #[inline]
- pub fn vmctx_func_ref(&self, index: FuncRefIndex) -> u32 {
- assert!(!index.is_reserved_value());
- assert!(index.as_u32() < self.num_escaped_funcs);
- self.vmctx_func_refs_begin() + index.as_u32() * u32::from(self.ptr.vm_func_ref().size())
- }
-
- /// Returns the offset to the `VMFuncRef` for the module startup function.
- ///
- /// Panics if this module does not have a startup function.
- #[inline]
- pub fn vmctx_startup_func_ref(&self) -> u32 {
- assert!(self.has_startup_func);
- self.startup_func_ref
- }
-
- /// Return the offset to the base of the runtime data segment at `index`.
- #[inline]
- pub fn vmctx_runtime_data_base(&self, index: RuntimeDataIndex) -> u32 {
- assert!(!index.is_reserved_value());
- assert!(index.as_u32() < self.num_runtime_data);
- self.vmctx_runtime_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size())
- }
-
- /// Return the offset to the length of the runtime data segment at `index`.
- #[inline]
- pub fn vmctx_runtime_data_length(&self, index: RuntimeDataIndex) -> u32 {
- assert!(!index.is_reserved_value());
- assert!(index.as_u32() < self.num_runtime_data);
- self.vmctx_runtime_data_lengths_begin() + index.as_u32() * 4
- }
-
- /// Return the offset to the `wasm_call` field in `*const VMFunctionBody` index `index`.
- #[inline]
- pub fn vmctx_vmfunction_import_wasm_call(&self, index: FuncIndex) -> u32 {
- self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().wasm_call())
- }
-
- /// Return the offset to the `array_call` field in `*const VMFunctionBody` index `index`.
- #[inline]
- pub fn vmctx_vmfunction_import_array_call(&self, index: FuncIndex) -> u32 {
- self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().array_call())
- }
-
- /// Return the offset to the `vmctx` field in `*const VMFunctionBody` index `index`.
- #[inline]
- pub fn vmctx_vmfunction_import_vmctx(&self, index: FuncIndex) -> u32 {
- self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().vmctx())
- }
-
- /// Return the offset to the `from` field in the imported `VMTable` at index
- /// `index`.
- #[inline]
- pub fn vmctx_vmtable_from(&self, index: TableIndex) -> u32 {
- self.vmctx_vmtable_import(index) + u32::from(self.ptr.vm_table_import().from())
- }
-
- /// Return the offset to the `base` field in `VMTableDefinition` index `index`.
- #[inline]
- pub fn vmctx_vmtable_definition_base(&self, index: DefinedTableIndex) -> u32 {
- self.vmctx_vmtable_definition(index) + u32::from(self.ptr.vm_table_definition().base())
- }
-
- /// Return the offset to the `current_elements` field in `VMTableDefinition` index `index`.
- #[inline]
- pub fn vmctx_vmtable_definition_current_elements(&self, index: DefinedTableIndex) -> u32 {
- self.vmctx_vmtable_definition(index)
- + u32::from(self.ptr.vm_table_definition().current_elements())
- }
-
- /// Return the offset to the `from` field in `VMMemoryImport` index `index`.
- #[inline]
- pub fn vmctx_vmmemory_import_from(&self, index: MemoryIndex) -> u32 {
- self.vmctx_vmmemory_import(index) + u32::from(self.ptr.vm_memory_import().from())
- }
-
- /// Return the offset to the `base` field in `VMMemoryDefinition` index `index`.
- #[inline]
- pub fn vmctx_vmmemory_definition_base(&self, index: OwnedMemoryIndex) -> u32 {
- self.vmctx_vmmemory_definition(index) + u32::from(self.ptr.vm_memory_definition().base())
- }
-
- /// Return the offset to the `current_length` field in `VMMemoryDefinition` index `index`.
- #[inline]
- pub fn vmctx_vmmemory_definition_current_length(&self, index: OwnedMemoryIndex) -> u32 {
- self.vmctx_vmmemory_definition(index)
- + u32::from(self.ptr.vm_memory_definition().current_length())
- }
+impl_vmctx_array_index! {
+ MemoryIndex,
+ DefinedMemoryIndex,
+ OwnedMemoryIndex,
+ FuncIndex,
+ TableIndex,
+ DefinedTableIndex,
+ GlobalIndex,
+ DefinedGlobalIndex,
+ TagIndex,
+ DefinedTagIndex,
+ FuncRefIndex,
+ RuntimeDataIndex,
+}
- /// Return the offset to the `from` field in `VMGlobalImport` index `index`.
- #[inline]
- pub fn vmctx_vmglobal_import_from(&self, index: GlobalIndex) -> u32 {
- self.vmctx_vmglobal_import(index) + u32::from(self.ptr.vm_global_import().from())
- }
+/// Generate the accessors for the offsets of `VMContext`'s
+/// dynamically-positioned fields.
+macro_rules! define_vmoffsets_dynamic_offsets {
+ (@one VMContext $snake:ident { $($dyn:tt)* }) => {
+ /// Offsets of the dynamically-positioned fields of `VMContext`.
+ impl<P: PtrSize> VMOffsets<P> {
+ define_vmctx_dynamic_offsets!(@accessors (self) [ $($dyn)* ]);
+ define_vmctx_dynamic_offsets!(@compute_fn (self, next) $snake [ $($dyn)* ]);
- /// Return the offset to the `from` field in `VMTagImport` index `index`.
- #[inline]
- pub fn vmctx_vmtag_import_from(&self, index: TagIndex) -> u32 {
- self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().from())
- }
-
- /// Return the offset to the `vmctx` field in `VMTagImport` index `index`.
- #[inline]
- pub fn vmctx_vmtag_import_vmctx(&self, index: TagIndex) -> u32 {
- self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().vmctx())
- }
+ /// Return the size of the `VMContext` allocation.
+ #[inline]
+ pub fn size_of_vmctx(&self) -> u32 {
+ self.size
+ }
+ }
+ };
+ (@one $other:ident $snake:ident { $($dyn:tt)* }) => {};
- /// Return the offset to the `index` field in `VMTagImport` index `index`.
- #[inline]
- pub fn vmctx_vmtag_import_index(&self, index: TagIndex) -> u32 {
- self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().index())
- }
+ ( $(
+ {
+ $Name:ident $snake:ident
+ static { $($stat:tt)* }
+ dynamic { $($dyn:tt)* }
+ }
+ )* ) => {
+ $( define_vmoffsets_dynamic_offsets!(@one $Name $snake { $($dyn)* }); )*
+ };
}
+for_each_vmctx_type!(define_vmoffsets_dynamic_offsets);
/// Offsets for `VMGcHeader`.
impl<P: PtrSize> VMOffsets<P> {
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
index 867c1b8060..8011ddd8c0 100644
--- a/crates/environ/src/vmtypes.rs
+++ b/crates/environ/src/vmtypes.rs
@@ -147,10 +147,14 @@ macro_rules! for_each_vm_type {
#[snake_name = vm_function_import]
pub struct VMFunctionImport {
/// Same as `VMFuncRef::array_call`.
+ #[readonly]
+ #[can_move]
pub array_call: VmPtr<VMArrayCallFunction>,
/// Same as `VMFuncRef::wasm_call`, except always non-null. Must be filled
/// in by the time Wasm is importing this function!
+ #[readonly]
+ #[can_move]
pub wasm_call: VmPtr<VMWasmCallFunction>,
/// Function signature's _actual_ type id.
@@ -158,9 +162,13 @@ macro_rules! for_each_vm_type {
/// This is the type that the function was defined with, not the type that
/// it was imported as. These two can be different in the face of subtyping
/// and we need the former for to correctly implement dynamic downcasts.
+ #[readonly]
+ #[can_move]
pub type_index: VMSharedTypeIndex,
/// Same as `VMFuncRef::vmctx`.
+ #[readonly]
+ #[can_move]
pub vmctx: VmPtr<VMOpaqueContext>,
}
@@ -171,12 +179,18 @@ macro_rules! for_each_vm_type {
#[snake_name = vm_table_import]
pub struct VMTableImport {
/// A pointer to the imported table description.
+ #[readonly]
+ #[can_move]
pub from: VmPtr<VMTableDefinition>,
/// A pointer to the `VMContext` that owns the table description.
+ #[readonly]
+ #[can_move]
pub vmctx: VmPtr<VMContext>,
/// The table index, within `vmctx`, this definition resides at.
+ #[readonly]
+ #[can_move]
pub index: DefinedTableIndex,
}
@@ -187,12 +201,18 @@ macro_rules! for_each_vm_type {
#[snake_name = vm_memory_import]
pub struct VMMemoryImport {
/// A pointer to the imported memory description.
+ #[readonly]
+ #[can_move]
pub from: VmPtr<VMMemoryDefinition>,
/// A pointer to the `VMContext` that owns the memory description.
+ #[readonly]
+ #[can_move]
pub vmctx: VmPtr<VMContext>,
/// The index of the memory in the containing `vmctx`.
+ #[readonly]
+ #[can_move]
pub index: DefinedMemoryIndex,
}
@@ -207,6 +227,8 @@ macro_rules! for_each_vm_type {
#[snake_name = vm_global_import]
pub struct VMGlobalImport {
/// A pointer to the imported global variable description.
+ #[readonly]
+ #[can_move]
pub from: VmPtr<VMGlobalDefinition>,
/// A pointer to the context that owns the global.
@@ -215,10 +237,14 @@ macro_rules! for_each_vm_type {
/// for `VMGlobalKind::Host`, it's a `VMContext` for
/// `VMGlobalKind::Instance`, and it's `VMComponentContext` for
/// `VMGlobalKind::ComponentFlags`.
+ #[readonly]
+ #[can_move]
pub vmctx: Option<VmPtr<VMOpaqueContext>>,
/// The kind of global, and extra location information in addition to
/// `vmctx` above.
+ #[readonly]
+ #[can_move]
pub kind: VMGlobalKind,
}
@@ -229,12 +255,18 @@ macro_rules! for_each_vm_type {
#[snake_name = vm_tag_import]
pub struct VMTagImport {
/// A pointer to the imported tag description.
+ #[readonly]
+ #[can_move]
pub from: VmPtr<VMTagDefinition>,
/// The instance that owns this tag.
+ #[readonly]
+ #[can_move]
pub vmctx: VmPtr<VMContext>,
/// The index of the tag in the containing `vmctx`.
+ #[readonly]
+ #[can_move]
pub index: DefinedTagIndex,
}
diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs
index de21c6738b..bafaada3d7 100644
--- a/crates/wasmtime/src/runtime/vm/component.rs
+++ b/crates/wasmtime/src/runtime/vm/component.rs
@@ -30,7 +30,7 @@ use core::ptr::NonNull;
use wasmtime_environ::component::*;
use wasmtime_environ::error::OutOfMemory;
use wasmtime_environ::prelude::TryPrimaryMap;
-use wasmtime_environ::{HostPtr, PrimaryMap, VMSharedTypeIndex};
+use wasmtime_environ::{HostPtr, PrimaryMap, PtrSize as _, VMSharedTypeIndex};
#[allow(
clippy::cast_possible_truncation,
@@ -365,8 +365,8 @@ impl ComponentInstance {
#[inline]
pub fn instance_flags(&self, instance: RuntimeComponentInstanceIndex) -> InstanceFlags {
unsafe {
- let ptr =
- self.vmctx_plus_offset_raw::<VMGlobalDefinition>(self.offsets.may_leave(instance));
+ let ptr = self
+ .vmctx_plus_offset_raw::<VMGlobalDefinition>(self.offsets.may_leave().at(instance));
InstanceFlags(SendSyncPtr::new(ptr))
}
}
@@ -378,7 +378,7 @@ impl ComponentInstance {
/// during the instantiation process of a component.
pub fn runtime_memory(&self, idx: RuntimeMemoryIndex) -> NonNull<VMMemoryDefinition> {
unsafe {
- let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.runtime_memory(idx));
+ let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.memories().at(idx));
debug_assert!(ret.as_ptr() as usize != INVALID_PTR);
ret.as_non_null()
}
@@ -391,7 +391,7 @@ impl ComponentInstance {
/// during the instantiation process of a component.
pub fn runtime_table(&self, idx: RuntimeTableIndex) -> VMTableImport {
unsafe {
- let ret = *self.vmctx_plus_offset::<VMTableImport>(self.offsets.runtime_table(idx));
+ let ret = *self.vmctx_plus_offset::<VMTableImport>(self.offsets.tables().at(idx));
debug_assert!(ret.from.as_ptr() as usize != INVALID_PTR);
debug_assert!(ret.vmctx.as_ptr() as usize != INVALID_PTR);
ret
@@ -430,7 +430,7 @@ impl ComponentInstance {
/// during the instantiation process of a component.
pub fn runtime_realloc(&self, idx: RuntimeReallocIndex) -> NonNull<VMFuncRef> {
unsafe {
- let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.runtime_realloc(idx));
+ let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.reallocs().at(idx));
debug_assert!(ret.as_ptr() as usize != INVALID_PTR);
ret.as_non_null()
}
@@ -442,7 +442,7 @@ impl ComponentInstance {
/// during the instantiation process of a component.
pub fn runtime_callback(&self, idx: RuntimeCallbackIndex) -> NonNull<VMFuncRef> {
unsafe {
- let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.runtime_callback(idx));
+ let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.callbacks().at(idx));
debug_assert!(ret.as_ptr() as usize != INVALID_PTR);
ret.as_non_null()
}
@@ -454,7 +454,7 @@ impl ComponentInstance {
/// during the instantiation process of a component.
pub fn runtime_post_return(&self, idx: RuntimePostReturnIndex) -> NonNull<VMFuncRef> {
unsafe {
- let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.runtime_post_return(idx));
+ let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.post_returns().at(idx));
debug_assert!(ret.as_ptr() as usize != INVALID_PTR);
ret.as_non_null()
}
@@ -467,7 +467,7 @@ impl ComponentInstance {
/// during the instantiation process of a component.
pub fn lowering(&self, idx: LoweredIndex) -> VMLowering {
unsafe {
- let ret = *self.vmctx_plus_offset::<VMLowering>(self.offsets.lowering(idx));
+ let ret = *self.vmctx_plus_offset::<VMLowering>(self.offsets.lowerings().at(idx));
debug_assert!(ret.callee.as_ptr() as usize != INVALID_PTR);
debug_assert!(ret.data.as_ptr() as usize != INVALID_PTR);
ret
@@ -484,7 +484,7 @@ impl ComponentInstance {
/// during the instantiation process of a component.
pub fn trampoline_func_ref(&self, idx: TrampolineIndex) -> NonNull<VMFuncRef> {
unsafe {
- let offset = self.offsets.trampoline_func_ref(idx);
+ let offset = self.offsets.trampoline_func_refs().at(idx);
let ret = self.vmctx_plus_offset_raw::<VMFuncRef>(offset);
debug_assert!(
mem::transmute::<Option<VmPtr<VMWasmCallFunction>>, usize>(ret.as_ref().wasm_call)
@@ -498,7 +498,7 @@ impl ComponentInstance {
/// Get the core Wasm function reference for the given unsafe intrinsic.
pub fn unsafe_intrinsic_func_ref(&self, idx: UnsafeIntrinsic) -> NonNull<VMFuncRef> {
unsafe {
- let offset = self.offsets.unsafe_intrinsic_func_ref(idx);
+ let offset = self.offsets.intrinsic_func_refs().at(idx);
let ret = self.vmctx_plus_offset_raw::<VMFuncRef>(offset);
debug_assert!(
mem::transmute::<Option<VmPtr<VMWasmCallFunction>>, usize>(ret.as_ref().wasm_call)
@@ -523,7 +523,7 @@ impl ComponentInstance {
ptr: NonNull<VMMemoryDefinition>,
) {
unsafe {
- let offset = self.offsets.runtime_memory(idx);
+ let offset = self.offsets.memories().at(idx);
let storage = self.vmctx_plus_offset_mut::<VmPtr<VMMemoryDefinition>>(offset);
debug_assert!((*storage).as_ptr() as usize == INVALID_PTR);
*storage = ptr.into();
@@ -537,7 +537,7 @@ impl ComponentInstance {
ptr: NonNull<VMFuncRef>,
) {
unsafe {
- let offset = self.offsets.runtime_realloc(idx);
+ let offset = self.offsets.reallocs().at(idx);
let storage = self.vmctx_plus_offset_mut::<VmPtr<VMFuncRef>>(offset);
debug_assert!((*storage).as_ptr() as usize == INVALID_PTR);
*storage = ptr.into();
@@ -551,7 +551,7 @@ impl ComponentInstance {
ptr: NonNull<VMFuncRef>,
) {
unsafe {
- let offset = self.offsets.runtime_callback(idx);
+ let offset = self.offsets.callbacks().at(idx);
let storage = self.vmctx_plus_offset_mut::<VmPtr<VMFuncRef>>(offset);
debug_assert!((*storage).as_ptr() as usize == INVALID_PTR);
*storage = ptr.into();
@@ -565,7 +565,7 @@ impl ComponentInstance {
ptr: NonNull<VMFuncRef>,
) {
unsafe {
- let offset = self.offsets.runtime_post_return(idx);
+ let offset = self.offsets.post_returns().at(idx);
let storage = self.vmctx_plus_offset_mut::<VmPtr<VMFuncRef>>(offset);
debug_assert!((*storage).as_ptr() as usize == INVALID_PTR);
*storage = ptr.into();
@@ -582,7 +582,7 @@ impl ComponentInstance {
/// here is never needed prior to it being configured here in the instance.
pub fn set_runtime_table(self: Pin<&mut Self>, idx: RuntimeTableIndex, import: VMTableImport) {
unsafe {
- let offset = self.offsets.runtime_table(idx);
+ let offset = self.offsets.tables().at(idx);
let storage = self.vmctx_plus_offset_mut::<VMTableImport>(offset);
debug_assert!((*storage).vmctx.as_ptr() as usize == INVALID_PTR);
debug_assert!((*storage).from.as_ptr() as usize == INVALID_PTR);
@@ -598,7 +598,7 @@ impl ComponentInstance {
debug_assert!(*self.vmctx_plus_offset::<usize>(callee) == INVALID_PTR);
let data = self.offsets.lowering_data(idx);
debug_assert!(*self.vmctx_plus_offset::<usize>(data) == INVALID_PTR);
- let offset = self.offsets.lowering(idx);
+ let offset = self.offsets.lowerings().at(idx);
*self.vmctx_plus_offset_mut(offset) = lowering;
}
}
@@ -612,7 +612,7 @@ impl ComponentInstance {
type_index: VMSharedTypeIndex,
) {
unsafe {
- let offset = self.offsets.trampoline_func_ref(idx);
+ let offset = self.offsets.trampoline_func_refs().at(idx);
debug_assert!(*self.vmctx_plus_offset::<usize>(offset) == INVALID_PTR);
let vmctx = VMOpaqueContext::from_vmcomponent(self.vmctx());
*self.vmctx_plus_offset_mut(offset) = VMFuncRef {
@@ -633,7 +633,7 @@ impl ComponentInstance {
type_index: VMSharedTypeIndex,
) {
unsafe {
- let offset = self.offsets.unsafe_intrinsic_func_ref(intrinsic);
+ let offset = self.offsets.intrinsic_func_refs().at(intrinsic);
debug_assert!(*self.vmctx_plus_offset::<usize>(offset) == INVALID_PTR);
let vmctx = VMOpaqueContext::from_vmcomponent(self.vmctx());
*self.vmctx_plus_offset_mut(offset) = VMFuncRef {
@@ -655,7 +655,7 @@ impl ComponentInstance {
dtor: Option<NonNull<VMFuncRef>>,
) {
unsafe {
- let offset = self.offsets.resource_destructor(idx);
+ let offset = self.offsets.resource_destructors().at(idx);
debug_assert!(*self.vmctx_plus_offset::<usize>(offset) == INVALID_PTR);
*self.vmctx_plus_offset_mut(offset) = dtor.map(VmPtr::from);
}
@@ -667,14 +667,14 @@ impl ComponentInstance {
/// after instantiation.
pub fn resource_destructor(&self, idx: ResourceIndex) -> Option<NonNull<VMFuncRef>> {
unsafe {
- let offset = self.offsets.resource_destructor(idx);
+ let offset = self.offsets.resource_destructors().at(idx);
debug_assert!(*self.vmctx_plus_offset::<usize>(offset) != INVALID_PTR);
(*self.vmctx_plus_offset::<Option<VmPtr<VMFuncRef>>>(offset)).map(|p| p.as_non_null())
}
}
unsafe fn initialize_vmctx(mut self: Pin<&mut Self>) {
- let offset = self.offsets.magic();
+ let offset = u32::from(self.offsets.ptr.vmcomponent().magic());
// SAFETY: it's safe to write the magic value during initialization and
// this is also the right type of value to write.
unsafe {
@@ -687,14 +687,14 @@ impl ComponentInstance {
// is also the right type of value to store in the vmctx.
static BUILTINS: libcalls::VMComponentBuiltins = libcalls::VMComponentBuiltins::INIT;
let ptr = BUILTINS.expose_provenance();
- let offset = self.offsets.builtins();
+ let offset = u32::from(self.offsets.ptr.vmcomponent().builtins());
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) = VmPtr::from(ptr);
}
// SAFETY: it's safe to initialize the vmctx in this function and this
// is also the right type of value to store in the vmctx.
- let offset = self.offsets.vm_store_context();
+ let offset = u32::from(self.offsets.ptr.vmcomponent().store_context());
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) =
VmPtr::from(self.store.0.as_ref().vm_store_context_ptr());
@@ -734,7 +734,7 @@ impl ComponentInstance {
}
for i in 0..self.offsets.num_trampolines {
let i = TrampolineIndex::from_u32(i);
- let offset = self.offsets.trampoline_func_ref(i);
+ let offset = self.offsets.trampoline_func_refs().at(i);
// SAFETY: see above
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
@@ -742,7 +742,7 @@ impl ComponentInstance {
}
for i in 0..self.offsets.num_unsafe_intrinsics {
let i = UnsafeIntrinsic::from_u32(i);
- let offset = self.offsets.unsafe_intrinsic_func_ref(i);
+ let offset = self.offsets.intrinsic_func_refs().at(i);
// SAFETY: see above
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
@@ -750,7 +750,7 @@ impl ComponentInstance {
}
for i in 0..self.offsets.num_runtime_memories {
let i = RuntimeMemoryIndex::from_u32(i);
- let offset = self.offsets.runtime_memory(i);
+ let offset = self.offsets.memories().at(i);
// SAFETY: see above
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
@@ -758,7 +758,7 @@ impl ComponentInstance {
}
for i in 0..self.offsets.num_runtime_reallocs {
let i = RuntimeReallocIndex::from_u32(i);
- let offset = self.offsets.runtime_realloc(i);
+ let offset = self.offsets.reallocs().at(i);
// SAFETY: see above
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
@@ -766,7 +766,7 @@ impl ComponentInstance {
}
for i in 0..self.offsets.num_runtime_callbacks {
let i = RuntimeCallbackIndex::from_u32(i);
- let offset = self.offsets.runtime_callback(i);
+ let offset = self.offsets.callbacks().at(i);
// SAFETY: see above
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
@@ -774,7 +774,7 @@ impl ComponentInstance {
}
for i in 0..self.offsets.num_runtime_post_returns {
let i = RuntimePostReturnIndex::from_u32(i);
- let offset = self.offsets.runtime_post_return(i);
+ let offset = self.offsets.post_returns().at(i);
// SAFETY: see above
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
@@ -782,7 +782,7 @@ impl ComponentInstance {
}
for i in 0..self.offsets.num_resources {
let i = ResourceIndex::from_u32(i);
- let offset = self.offsets.resource_destructor(i);
+ let offset = self.offsets.resource_destructors().at(i);
// SAFETY: see above
unsafe {
*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
@@ -790,7 +790,7 @@ impl ComponentInstance {
}
for i in 0..self.offsets.num_runtime_tables {
let i = RuntimeTableIndex::from_u32(i);
- let offset = self.offsets.runtime_table(i);
+ let offset = self.offsets.tables().at(i);
// SAFETY: see above
#[allow(clippy::cast_possible_truncation, reason = "known to not overflow")]
unsafe {
diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs
index 06ba7e4455..321f140def 100644
--- a/crates/wasmtime/src/runtime/vm/instance.rs
+++ b/crates/wasmtime/src/runtime/vm/instance.rs
@@ -418,32 +418,32 @@ impl Instance {
/// Return the indexed `VMFunctionImport`.
fn imported_function(&self, index: FuncIndex) -> &VMFunctionImport {
- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmfunction_import(index)) }
+ unsafe { self.vmctx_plus_offset(self.offsets().imported_functions().at(index)) }
}
/// Return the index `VMTableImport`.
fn imported_table(&self, index: TableIndex) -> &VMTableImport {
- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtable_import(index)) }
+ unsafe { self.vmctx_plus_offset(self.offsets().imported_tables().at(index)) }
}
/// Return the indexed `VMMemoryImport`.
fn imported_memory(&self, index: MemoryIndex) -> &VMMemoryImport {
- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmmemory_import(index)) }
+ unsafe { self.vmctx_plus_offset(self.offsets().imported_memories().at(index)) }
}
/// Return the indexed `VMGlobalImport`.
fn imported_global(&self, index: GlobalIndex) -> &VMGlobalImport {
- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmglobal_import(index)) }
+ unsafe { self.vmctx_plus_offset(self.offsets().imported_globals().at(index)) }
}
/// Return the indexed `VMTagImport`.
fn imported_tag(&self, index: TagIndex) -> &VMTagImport {
- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtag_import(index)) }
+ unsafe { self.vmctx_plus_offset(self.offsets().imported_tags().at(index)) }
}
/// Return the indexed `VMTagDefinition`.
pub fn tag_ptr(&self, index: DefinedTagIndex) -> NonNull<VMTagDefinition> {
- unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmtag_definition(index)) }
+ unsafe { self.vmctx_plus_offset_raw(self.offsets().tags().at(index)) }
}
/// Return the indexed `VMTableDefinition`.
@@ -461,7 +461,7 @@ impl Instance {
/// Return a pointer to the `index`'th table within this instance, stored
/// in vmctx memory.
pub fn table_ptr(&self, index: DefinedTableIndex) -> NonNull<VMTableDefinition> {
- unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmtable_definition(index)) }
+ unsafe { self.vmctx_plus_offset_raw(self.offsets().tables().at(index)) }
}
/// Get a locally defined or imported memory.
@@ -496,14 +496,14 @@ impl Instance {
#[inline]
pub fn memory_ptr(&self, index: DefinedMemoryIndex) -> NonNull<VMMemoryDefinition> {
unsafe {
- self.vmctx_plus_offset::<VmPtr<_>>(self.offsets().vmctx_vmmemory_pointer(index))
+ self.vmctx_plus_offset::<VmPtr<_>>(self.offsets().memories().at(index))
.as_non_null()
}
}
/// Return the indexed `VMGlobalDefinition`.
pub fn global_ptr(&self, index: DefinedGlobalIndex) -> NonNull<VMGlobalDefinition> {
- unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmglobal_definition(index)) }
+ unsafe { self.vmctx_plus_offset_raw(self.offsets().globals().at(index)) }
}
/// Get all globals within this instance.
@@ -538,19 +538,19 @@ impl Instance {
/// Return a pointer to the interrupts structure
#[inline]
pub fn vm_store_context(&self) -> NonNull<Option<VmPtr<VMStoreContext>>> {
- unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx_store_context()) }
+ unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx().store_context()) }
}
/// Return a pointer to the global epoch counter used by this instance.
#[cfg(target_has_atomic = "64")]
pub fn epoch_ptr(self: Pin<&mut Self>) -> &mut Option<VmPtr<AtomicU64>> {
- let offset = self.offsets().ptr.vmctx_epoch_ptr();
+ let offset = self.offsets().ptr.vmctx().epoch_ptr();
unsafe { self.vmctx_plus_offset_mut(offset) }
}
/// Return a pointer to the collector-specific heap data.
pub fn gc_heap_data(self: Pin<&mut Self>) -> &mut Option<VmPtr<u8>> {
- let offset = self.offsets().ptr.vmctx_gc_heap_data();
+ let offset = self.offsets().ptr.vmctx().gc_heap_data();
unsafe { self.vmctx_plus_offset_mut(offset) }
}
@@ -841,7 +841,7 @@ impl Instance {
}
fn type_ids_array(&self) -> NonNull<VmPtr<VMSharedTypeIndex>> {
- unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx_type_ids_array()) }
+ unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx().type_ids()) }
}
/// Get a `&VMFuncRef` for the given `FuncIndex`.
@@ -879,7 +879,7 @@ impl Instance {
let index = module.func_index(def_index);
let func = &module.functions[index];
let type_index = func.signature.unwrap_engine_type_index();
- let vmctx_offset = self.offsets().vmctx_func_ref(func.func_ref);
+ let vmctx_offset = self.offsets().func_refs().at(func.func_ref);
let array_to_wasm_key = FuncKey::ArrayToWasmTrampoline(module.module_index, def_index);
let wasm_key = FuncKey::DefinedWasmFunction(module.module_index, def_index);
// SAFETY: the type/offset/keys here are all valid for the defined
@@ -906,7 +906,7 @@ impl Instance {
t.unwrap_engine_type_index()
}
};
- let vmctx_offset = self.offsets().vmctx_startup_func_ref();
+ let vmctx_offset = self.offsets().startup_func_ref();
let array_to_wasm_key = FuncKey::ModuleStartup(Abi::Array, module.module_index);
let wasm_key = FuncKey::ModuleStartup(Abi::Wasm, module.module_index);
// SAFETY: the type/offset/keys here are all valid for the module
@@ -1185,7 +1185,7 @@ impl Instance {
unsafe {
let offsets = instance.runtime_info.offsets();
instance
- .vmctx_plus_offset_raw::<u32>(offsets.ptr.vmctx_magic())
+ .vmctx_plus_offset_raw::<u32>(offsets.ptr.vmctx().magic())
.write(VMCONTEXT_MAGIC);
}
@@ -1213,7 +1213,7 @@ impl Instance {
let ptr = BUILTINS.expose_provenance();
let offsets = instance.runtime_info.offsets();
instance
- .vmctx_plus_offset_raw(offsets.ptr.vmctx_builtin_functions())
+ .vmctx_plus_offset_raw(offsets.ptr.vmctx().builtin_functions())
.write(VmPtr::from(ptr));
}
@@ -1227,7 +1227,7 @@ impl Instance {
ptr::copy_nonoverlapping(
imports.functions.as_ptr(),
instance
- .vmctx_plus_offset_raw(offsets.vmctx_imported_functions_begin())
+ .vmctx_plus_offset_raw(offsets.imported_functions().begin())
.as_ptr(),
imports.functions.len(),
);
@@ -1235,7 +1235,7 @@ impl Instance {
ptr::copy_nonoverlapping(
imports.tables.as_ptr(),
instance
- .vmctx_plus_offset_raw(offsets.vmctx_imported_tables_begin())
+ .vmctx_plus_offset_raw(offsets.imported_tables().begin())
.as_ptr(),
imports.tables.len(),
);
@@ -1243,7 +1243,7 @@ impl Instance {
ptr::copy_nonoverlapping(
imports.memories.as_ptr(),
instance
- .vmctx_plus_offset_raw(offsets.vmctx_imported_memories_begin())
+ .vmctx_plus_offset_raw(offsets.imported_memories().begin())
.as_ptr(),
imports.memories.len(),
);
@@ -1251,7 +1251,7 @@ impl Instance {
ptr::copy_nonoverlapping(
imports.globals.as_ptr(),
instance
- .vmctx_plus_offset_raw(offsets.vmctx_imported_globals_begin())
+ .vmctx_plus_offset_raw(offsets.imported_globals().begin())
.as_ptr(),
imports.globals.len(),
);
@@ -1259,7 +1259,7 @@ impl Instance {
ptr::copy_nonoverlapping(
imports.tags.as_ptr(),
instance
- .vmctx_plus_offset_raw(offsets.vmctx_imported_tags_begin())
+ .vmctx_plus_offset_raw(offsets.imported_tags().begin())
.as_ptr(),
imports.tags.len(),
);
@@ -1277,7 +1277,7 @@ impl Instance {
// valid.
unsafe {
let offsets = instance.runtime_info.offsets();
- let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_tables_begin());
+ let mut ptr = instance.vmctx_plus_offset_raw(offsets.tables().begin());
let tables = instance.as_mut().tables_mut();
for i in 0..module.num_defined_tables() {
ptr.write(tables[DefinedTableIndex::new(i)].1.vmtable());
@@ -1296,9 +1296,8 @@ impl Instance {
// valid.
unsafe {
let offsets = instance.runtime_info.offsets();
- let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_memories_begin());
- let mut owned_ptr =
- instance.vmctx_plus_offset_raw(offsets.vmctx_owned_memories_begin());
+ let mut ptr = instance.vmctx_plus_offset_raw(offsets.memories().begin());
+ let mut owned_ptr = instance.vmctx_plus_offset_raw(offsets.owned_memories().begin());
let memories = instance.as_mut().memories_mut();
for i in 0..module.num_defined_memories() {
let defined_memory_index = DefinedMemoryIndex::new(i);
@@ -1355,7 +1354,7 @@ impl Instance {
// valid.
unsafe {
let offsets = instance.runtime_info.offsets();
- let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_tags_begin());
+ let mut ptr = instance.vmctx_plus_offset_raw(offsets.tags().begin());
for i in 0..module.num_defined_tags() {
let defined_index = DefinedTagIndex::new(i);
let tag_index = module.tag_index(defined_index);
@@ -1375,9 +1374,8 @@ impl Instance {
unsafe {
let offsets = instance.runtime_info.offsets();
let mut lengths =
- instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_lengths_begin());
- let mut bases =
- instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_bases_begin());
+ instance.vmctx_plus_offset_raw(offsets.runtime_data_lengths().begin());
+ let mut bases = instance.vmctx_plus_offset_raw(offsets.runtime_data_bases().begin());
for i in module.runtime_data.keys() {
let data = instance.runtime_data(i);
lengths.write(u32::try_from(data.len()).unwrap());
@@ -1408,10 +1406,10 @@ impl Instance {
let offsets = instance.runtime_info.offsets();
unsafe {
instance
- .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_length(*data))
+ .vmctx_plus_offset_raw(offsets.runtime_data_lengths().at(*data))
.write(0u32);
instance
- .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_base(*data))
+ .vmctx_plus_offset_raw(offsets.runtime_data_bases().at(*data))
.write(0usize);
}
}
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 0a9c934d1c..de5b079425 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -331,7 +331,7 @@ mod test_vmglobal_definition {
fn check_vmglobal_begins_aligned() {
let module = Module::new(StaticModuleIndex::from_u32(0));
let offsets = VMOffsets::new(HostPtr, &module);
- assert_eq!(offsets.vmctx_globals_begin() % 16, 0);
+ assert_eq!(offsets.globals().begin() % 16, 0);
}
#[test]
@@ -544,7 +544,7 @@ mod test_vmtag_definition {
fn check_vmtag_begins_aligned() {
let module = Module::new(StaticModuleIndex::from_u32(0));
let offsets = VMOffsets::new(HostPtr, &module);
- assert_eq!(offsets.vmctx_tags_begin() % 16, 0);
+ assert_eq!(offsets.tags().begin() % 16, 0);
}
}
@@ -969,7 +969,7 @@ impl VMContext {
#[inline]
pub unsafe fn from_opaque(opaque: NonNull<VMOpaqueContext>) -> NonNull<VMContext> {
// Note that in general the offset of the "magic" field is stored in
- // `VMOffsets::vmctx_magic`. Given though that this is a sanity check
+ // `VMContext::magic`. Given though that this is a sanity check
// about converting this pointer to another type we ideally don't want
// to read the offset from potentially corrupt memory. Instead it would
// be better to catch errors here as soon as possible.
diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rs
index e0de2888bb..a45b8d839d 100644
--- a/winch/codegen/src/codegen/call.rs
+++ b/winch/codegen/src/codegen/call.rs
@@ -213,11 +213,13 @@ impl FnCall {
context.without::<Result<(Reg, Reg)>, M, _>(&sig.regs, masm, |context, masm| {
Ok((context.any_gpr(masm)?, context.any_gpr(masm)?))
})??;
- let callee_vmctx_offset = vmoffsets.vmctx_vmfunction_import_vmctx(index);
+ let vmimport = vmoffsets.imported_functions().at(index);
+ let callee_vmctx_offset = vmimport + u32::from(vmoffsets.ptr.vm_function_import().vmctx());
let callee_vmctx_addr = masm.address_at_vmctx(callee_vmctx_offset)?;
masm.load_ptr(callee_vmctx_addr, writable!(callee_vmctx))?;
- let callee_body_offset = vmoffsets.vmctx_vmfunction_import_wasm_call(index);
+ let callee_body_offset =
+ vmimport + u32::from(vmoffsets.ptr.vm_function_import().wasm_call());
let callee_addr = masm.address_at_vmctx(callee_body_offset)?;
masm.load_ptr(callee_addr, writable!(callee))?;
diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rs
index d3eb52d321..3604ed76d6 100644
--- a/winch/codegen/src/codegen/drc.rs
+++ b/winch/codegen/src/codegen/drc.rs
@@ -225,7 +225,7 @@ where
gc_ref: Reg,
object_addr: Reg,
) -> Result<(Reg, Reg)> {
- let heap_data_offset = self.env.vmoffsets.ptr.vmctx_gc_heap_data();
+ let heap_data_offset = self.env.vmoffsets.ptr.vmctx().gc_heap_data();
let roots_head_offset = u32::from(
self.env
.vmoffsets
diff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rs
index f53fdba350..07f88c120b 100644
--- a/winch/codegen/src/codegen/env.rs
+++ b/winch/codegen/src/codegen/env.rs
@@ -215,12 +215,13 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
let ty = self.translation.module.globals[index].wasm_ty;
let val = || match self.translation.module.defined_global_index(index) {
Some(defined_index) => GlobalData {
- offset: self.vmoffsets.vmctx_vmglobal_definition(defined_index),
+ offset: self.vmoffsets.globals().at(defined_index),
imported: false,
ty,
},
None => GlobalData {
- offset: self.vmoffsets.vmctx_vmglobal_import_from(index),
+ offset: self.vmoffsets.imported_globals().at(index)
+ + u32::from(self.vmoffsets.ptr.vm_global_import().from()),
imported: true,
ty,
},
@@ -238,12 +239,18 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
match self.translation.module.defined_table_index(index) {
Some(defined) => (
None,
- self.vmoffsets.vmctx_vmtable_definition_base(defined),
- self.vmoffsets
- .vmctx_vmtable_definition_current_elements(defined),
+ self.vmoffsets.tables().at(defined)
+ + u32::from(self.vmoffsets.ptr.vm_table_definition().base()),
+ self.vmoffsets.tables().at(defined)
+ + u32::from(
+ self.vmoffsets.ptr.vm_table_definition().current_elements(),
+ ),
),
None => (
- Some(self.vmoffsets.vmctx_vmtable_from(index)),
+ Some(
+ self.vmoffsets.imported_tables().at(index)
+ + u32::from(self.vmoffsets.ptr.vm_table_import().from()),
+ ),
self.vmoffsets.ptr.vm_table_definition().base().into(),
self.vmoffsets
.ptr
@@ -274,39 +281,48 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
match self.resolved_heaps.entry(index) {
Occupied(entry) => *entry.get(),
Vacant(entry) => {
- let (import_from, base_offset, current_length_offset) =
- match self.translation.module.defined_memory_index(index) {
- Some(defined) => {
- if is_shared {
- (
- Some(self.vmoffsets.vmctx_vmmemory_pointer(defined)),
- self.vmoffsets.ptr.vm_memory_definition().base().into(),
- self.vmoffsets
- .ptr
- .vm_memory_definition()
- .current_length()
- .into(),
- )
- } else {
- let owned = self.translation.module.owned_memory_index(defined);
- (
- None,
- self.vmoffsets.vmctx_vmmemory_definition_base(owned),
- self.vmoffsets
- .vmctx_vmmemory_definition_current_length(owned),
- )
- }
+ let (import_from, base_offset, current_length_offset) = match self
+ .translation
+ .module
+ .defined_memory_index(index)
+ {
+ Some(defined) => {
+ if is_shared {
+ (
+ Some(self.vmoffsets.memories().at(defined)),
+ self.vmoffsets.ptr.vm_memory_definition().base().into(),
+ self.vmoffsets
+ .ptr
+ .vm_memory_definition()
+ .current_length()
+ .into(),
+ )
+ } else {
+ let owned = self.translation.module.owned_memory_index(defined);
+ (
+ None,
+ self.vmoffsets.owned_memories().at(owned)
+ + u32::from(self.vmoffsets.ptr.vm_memory_definition().base()),
+ self.vmoffsets.owned_memories().at(owned)
+ + u32::from(
+ self.vmoffsets.ptr.vm_memory_definition().current_length(),
+ ),
+ )
}
- None => (
- Some(self.vmoffsets.vmctx_vmmemory_import_from(index)),
- self.vmoffsets.ptr.vm_memory_definition().base().into(),
- self.vmoffsets
- .ptr
- .vm_memory_definition()
- .current_length()
- .into(),
+ }
+ None => (
+ Some(
+ self.vmoffsets.imported_memories().at(index)
+ + u32::from(self.vmoffsets.ptr.vm_memory_import().from()),
),
- };
+ self.vmoffsets.ptr.vm_memory_definition().base().into(),
+ self.vmoffsets
+ .ptr
+ .vm_memory_definition()
+ .current_length()
+ .into(),
+ ),
+ };
let memory = &self.translation.module.memories[index];
diff --git a/winch/codegen/src/codegen/gc.rs b/winch/codegen/src/codegen/gc.rs
index a7f7ad86d5..33b93371b2 100644
--- a/winch/codegen/src/codegen/gc.rs
+++ b/winch/codegen/src/codegen/gc.rs
@@ -58,7 +58,7 @@ where
/// Both returned registers are allocated from the code-generation context
/// and are owned by the caller. The caller must eventually free them.
pub(super) fn emit_load_gc_heap_base_and_bound(&mut self) -> Result<(Reg, Reg)> {
- let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
+ let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context();
let gc_heap_base_offset = self.env.vmoffsets.ptr.vm_store_context().gc_heap_base();
let gc_heap_len_offset = self
.env
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 1577c091ae..e8c20b4e99 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -514,7 +514,7 @@ where
.as_u32()
.checked_mul(sig_index_bytes.into())
.unwrap();
- let signatures_base_offset = self.env.vmoffsets.ptr.vmctx_type_ids_array();
+ let signatures_base_offset = self.env.vmoffsets.ptr.vmctx().type_ids();
let funcref_sig_offset = self.env.vmoffsets.ptr.vm_func_ref().type_index();
// Get the caller id.
let caller_id = self.context.any_gpr(self.masm)?;
@@ -1795,7 +1795,8 @@ where
let data_segment_length_offset = self
.env
.vmoffsets
- .vmctx_runtime_data_length(runtime_data_index);
+ .runtime_data_lengths()
+ .at(runtime_data_index);
let tmp1 = self.context.any_gpr(self.masm)?;
let tmp2 = self.context.any_gpr(self.masm)?;
self.masm.load(
@@ -1822,7 +1823,8 @@ where
let data_segment_base_offset = self
.env
.vmoffsets
- .vmctx_runtime_data_base(runtime_data_index);
+ .runtime_data_bases()
+ .at(runtime_data_index);
self.masm.load(
self.masm.address_at_vmctx(data_segment_base_offset)?,
writable!(tmp1),
@@ -1860,7 +1862,8 @@ where
let data_segment_offset = self
.env
.vmoffsets
- .vmctx_runtime_data_length(runtime_data_index);
+ .runtime_data_lengths()
+ .at(runtime_data_index);
let len_addr = self.masm.address_at_vmctx(data_segment_offset)?;
self.masm.store(RegImm::i32(0), len_addr, OperandSize::S32)
}
@@ -2146,7 +2149,7 @@ where
/// Emits a series of instructions that load the `fuel_consumed` field from
/// `VMStoreContext`.
fn emit_load_fuel_consumed(&mut self, fuel_reg: Reg) -> Result<()> {
- let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
+ let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context();
let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed();
self.masm.load_ptr(
self.masm
@@ -2224,8 +2227,8 @@ where
epoch_deadline_reg: Reg,
epoch_counter_reg: Reg,
) -> Result<()> {
- let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx_epoch_ptr();
- let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
+ let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx().epoch_ptr();
+ let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context();
let epoch_deadline_offset = self.env.vmoffsets.ptr.vm_store_context().epoch_deadline();
// Load the current epoch value into `epoch_counter_var`.
@@ -2266,7 +2269,7 @@ where
return Ok(());
}
- let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
+ let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context();
let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed();
let limits_reg = self.context.any_gpr(self.masm)?;
@@ -2575,7 +2578,7 @@ where
// is loaded from the `VMMemoryImport` and the vmctx is loaded from
// the vmctx itself.
None => {
- let vmimport = self.env.vmoffsets.vmctx_vmmemory_import(mem);
+ let vmimport = self.env.vmoffsets.imported_memories().at(mem);
let vmctx_offset =
vmimport + u32::from(self.env.vmoffsets.ptr.vm_memory_import().vmctx());
let index_offset =
@@ -2607,7 +2610,7 @@ where
Ok(Callee::Builtin(builtin))
}
None => {
- let vmimport = self.env.vmoffsets.vmctx_vmtable_import(table);
+ let vmimport = self.env.vmoffsets.imported_tables().at(table);
let vmctx_offset =
vmimport + u32::from(self.env.vmoffsets.ptr.vm_table_import().vmctx());
let index_offset =
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 276bbfd4dc..0c1328736b 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -191,7 +191,7 @@ impl Masm for MacroAssembler {
masm.with_scratch::<IntScratch, _>(|masm, scratch_stk_limit| {
masm.with_scratch::<IntScratch, _>(|masm, scratch_tmp| {
masm.load_ptr(
- masm.address_at_reg(vmctx, ptr_size_u8.vmcontext_store_context().into())?,
+ masm.address_at_reg(vmctx, ptr_size_u8.vmctx().store_context().into())?,
scratch_stk_limit.writable(),
)?;
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index a455f828dd..e7fe439b24 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -129,7 +129,7 @@ impl Masm for MacroAssembler {
self.with_scratch::<IntScratch, _>(|masm, scratch| {
masm.load_ptr(
- masm.address_at_reg(vmctx, ptr_size.vmcontext_store_context().into())?,
+ masm.address_at_reg(vmctx, ptr_size.vmctx().store_context().into())?,
scratch.writable(),
)?;