A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Define `VMContext` and `VMComponentContext` in a macro (#14083)
Nick Fitzgerald2ecd55d74146
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex 873f34b7a2..9963f5ef22 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -5,13 +5,12 @@ use cranelift_codegen::{ir::{self, InstBuilder as _},};use wasmtime_environ::{- BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex,- GetPtrSize, GlobalIndex, MemoryIndex, ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex,- StaticModuleIndex, TableIndex, TagIndex, VMOffsets,+ BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, GetPtrSize,+ ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex, StaticModuleIndex, VMOffsets,component::{ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,- VMComponentOffsets,+ RuntimeReallocIndex, VMComponentOffsets,},};@@ -501,10 +500,6 @@ impl<'a, Offsets> Field<'a, Offsets> {/// Emit a store of `value` to this field relative to `ptr`, a pointer to/// the containing `VM*` structure.- #[allow(- dead_code,- reason = "part of the general `Field` API; not all fields are stored to yet"- )]pub fn store(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value, value: ir::Value) {let flags = self.flags_with_region(cursor.func);cursor@@ -703,594 +698,294 @@ macro_rules! define_vm_type_alias_region_helpers {}wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers);-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- /// Create a new `AliasRegions`.- pub fn new(offsets: Offsets) -> Self {- Self {- pointer_type: ir::Type::int_with_byte_size(offsets.get_ptr_size().size().into())- .unwrap(),- offsets,- cache: std::collections::HashMap::default(),- }- }-- /// Get the alias region for accesses into the GC heap.- pub fn gc_heap_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {- self.region(func, AliasRegionKey::GcHeap)- }-- /// Get the alias region for an imported or exported memory access (shared- /// across all imported/exported memories).- pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {- self.region(func, AliasRegionKey::PublicMemory)- }-- /// Get the alias region for accessing a defined memory that is not- /// exported.- pub fn defined_memory_region(- &mut self,- func: &mut ir::Function,- module: StaticModuleIndex,- index: DefinedMemoryIndex,- ) -> ir::AliasRegion {- self.region(func, AliasRegionKey::DefinedMemory { module, index })- }-- /// Get the alias region for an imported or exported table access (shared- /// across all imported/exported memories).- pub fn public_table_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {- self.region(func, AliasRegionKey::PublicTable)- }-- /// Get the alias region for accessing a defined table that is not- /// exported.- pub fn defined_table_region(- &mut self,- func: &mut ir::Function,- module: StaticModuleIndex,- index: DefinedTableIndex,- ) -> ir::AliasRegion {- self.region(func, AliasRegionKey::DefinedTable { module, index })- }-- /// Get the alias region for an imported or exported global access (shared- /// across all imported/exported memories).- pub fn public_global_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {- self.region(func, AliasRegionKey::PublicGlobal)- }-- /// Get the alias region for accessing a defined global that is not- /// exported.- pub fn defined_global_region(- &mut self,- func: &mut ir::Function,- module: StaticModuleIndex,- index: DefinedGlobalIndex,- ) -> ir::AliasRegion {- self.region(func, AliasRegionKey::DefinedGlobal { module, index })- }-}--/// `VMContext`-related methods that are valid for any `VMContext`, regardless-/// of its particular `VMOffsets`.-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- /// Get the alias region for the given offset into the `VMContext`.- fn vmctx_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {- self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMContext,- offset,- },- )- }+/// Define, for each of Wasmtime's vmctx types, an [`AliasRegions`] accessor+/// that returns a wrapper exposing a [`Field`]-returning method per field of+/// that vmctx.+///+/// For example, `alias_regions.vmctx().epoch_ptr()` is the `VMContext::epoch_ptr`+/// field, and `alias_regions.vmcomponent().callbacks(i)` is the `i`th element of+/// the `VMComponentContext`'s runtime-callbacks array.+///+/// A vmctx's `static` fields sit at offsets that depend only on the target+/// pointer size, so their accessors are available for any `Offsets: GetPtrSize`.+/// Its `dynamic` fields sit at offsets that additionally depend on the module or+/// component being compiled, so their accessors are only available when the+/// `AliasRegions` carries that vmctx's own fully-computed offsets.+///+/// A field marked `#[aggregate]` gets no accessor, for the same reason it gets+/// none in [`define_vm_type_alias_region_helpers!`]: it has no single Cranelift+/// type. Instead, its interior is reached by rebasing a [`Field`] of the nested+/// `VM*` type onto the aggregate's own offset within the vmctx (see+/// [`Field::relative_to`]), which produces exactly the same alias region as+/// accessing that `VM*` type through a pointer would. For the few aggregates that+/// have no `VM*` type of their own, and hence no alias region of their own, there+/// are hand-written helpers below.+#[allow(+ unused_macro_rules,+ reason = "entry shapes and marker attributes are handled uniformly for both \+ sections of both vmctx types, but not every combination occurs"+)]+macro_rules! define_vmctx_alias_region_helpers {+ // Classify a field type to its Cranelift `ir::Type`, given `$pt` (the target+ // pointer type as an `ir::Type`).+ (@field_ty $pt:expr, u32) => { ir::types::I32 };+ (@field_ty $pt:expr, VmPtr < $g:ident >) => { $pt };- fn vmctx_load(- &mut self,- cursor: &mut FuncCursor<'_>,- ty: ir::Type,- base_flags: ir::MemFlagsData,- vmctx: ir::Value,- offset: u32,- ) -> ir::Value {- let region = self.vmctx_region(cursor.func, offset);- cursor.ins().load(- ty,- base_flags.with_alias_region(Some(region)),- vmctx,- i32::try_from(offset).unwrap(),- )- }+ // Determine the Cranelift type a field is *accessed* as: the classification+ // of its `#[access_as = T]` type if it has one, and of its declared type+ // otherwise.+ (@access_ty $pt:expr, [ $($fty:tt)* ] []) => {+ define_vmctx_alias_region_helpers!(@field_ty $pt, $($fty)*)+ };+ (@access_ty $pt:expr, $fty:tt [ #[access_as = $($t:tt)*] $($rest:tt)* ]) => {+ define_vmctx_alias_region_helpers!(@field_ty $pt, $($t)*)+ };+ (@access_ty $pt:expr, $fty:tt [ # $skip:tt $($rest:tt)* ]) => {+ define_vmctx_alias_region_helpers!(@access_ty $pt, $fty [ $($rest)* ])+ };- fn vmctx_store(- &mut self,- cursor: &mut FuncCursor<'_>,- base_flags: ir::MemFlagsData,- vmctx: ir::Value,- offset: u32,- val: ir::Value,- ) {- let region = self.vmctx_region(cursor.func, offset);- cursor.ins().store(- base_flags.with_alias_region(Some(region)),- val,- vmctx,- i32::try_from(offset).unwrap(),+ // Apply a field attribute to its access flags.+ (@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly() };+ (@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() };+ (@apply_attr $flags:expr, [access_as = $($t:tt)*]) => { $flags };++ // Compute a field's access flags and Cranelift type from its declared type+ // and marker attributes, and build the `Field` for it at `$offset`.+ (@field $Name:ident ($self:expr, $offset:expr) [ $($fty:tt)* ] [ $(# $fattr:tt)* ]) => {{+ let this = $self;+ let flags = ir::MemFlagsData::trusted();+ $( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $fattr); )*+ let ty = define_vmctx_alias_region_helpers!(+ @access_ty this.regions.pointer_type, [ $($fty)* ] [ $(# $fattr)* ]);- }+ let offset = $offset;+ Field::new(this.regions, VmType::$Name, offset, flags, ty)+ }};- /// Load the `VMContext::magic` field.- pub fn vmctx_magic(&mut self, cursor: &mut FuncCursor<'_>, vmctx: ir::Value) -> ir::Value {- self.vmctx_load(- cursor,- ir::types::I32,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets.get_ptr_size().vmctx_magic().into(),- )- }+ // ### `static` Section Entries- /// Load the `*mut VMStoreContext` value out of the given `*mut VMContext`.- pub fn vmctx_store_context(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- ) -> ir::Value {- self.vmctx_store_context_load(cursor.func)- .emit(cursor, vmctx)- }+ (@static_entry $Name:ident $snake:ident align { $al:tt }) => {};- /// Get a `Load` for the `*mut VMStoreContext` value out of a `*mut VMContext`.- pub fn vmctx_store_context_load(&mut self, func: &mut ir::Function) -> Load {- let offset = u32::from(self.offsets.get_ptr_size().vmctx_store_context());- let region = self.vmctx_region(func, offset);- Load {- offset,- flags: ir::MemFlagsData::trusted()- .with_readonly()- .with_can_move()- .with_alias_region(Some(region)),- ty: self.pointer_type,- }- }-- /// Load the `*mut i64` epoch pointer out of the given `*mut VMContext`.- pub fn vmctx_epoch_ptr(&mut self, cursor: &mut FuncCursor<'_>, vmctx: ir::Value) -> ir::Value {- self.vmctx_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.get_ptr_size().vmctx_epoch_ptr().into(),- )- }+ // Aggregates get no accessor.+ (@static_entry $Name:ident $snake:ident $kind:ident { #[aggregate] $($rest:tt)* }) => {};- /// Load the base pointer of the `[VMSharedTypeIndex]` array out of the- /// given `*mut VMContext`.- pub fn vmctx_shared_type_ids_array(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- ) -> ir::Value {- self.vmctx_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets.get_ptr_size().vmctx_type_ids_array().into(),- )- }-- /// Load the collector's heap data pointer out of the `*mut VMContext`.- pub fn vmctx_gc_heap_data(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- ) -> ir::Value {- self.vmctx_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets.get_ptr_size().vmctx_gc_heap_data().into(),- )- }-- /// Load the base pointer to the builtin-functions array from a `*mut- /// VMContext`.- pub fn vmctx_builtin_functions(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- ) -> ir::Value {- self.vmctx_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets- .get_ptr_size()- .vmcontext_builtin_functions()- .into(),- )- }-}--/// `VMContext`-related methods that are specific to a particular Wasm module's-/// `VMOffsets`.-impl AliasRegions<VMOffsets<u8>> {- /// Like `vmctx_load`, but tags the load with a per-import alias region- /// rather than the coarse `VMContext` region. Used for fields of the- /// `VM*Import` structs inlined into the `VMContext`.- fn vmimport_load(- &mut self,- cursor: &mut FuncCursor<'_>,- ty: ir::Type,- base_flags: ir::MemFlagsData,- vmctx: ir::Value,- vmctx_offset: u32,- field_offset: u32,- vm_type: VmType,- ) -> ir::Value {- let region = self.region(- cursor.func,- AliasRegionKey::Vm {- ty: vm_type,- offset: field_offset,- },- );- cursor.ins().load(- ty,- base_flags.with_alias_region(Some(region)),- vmctx,- i32::try_from(vmctx_offset).unwrap(),- )- }-- /// Load the imported tag's `VMTagImport::vmctx` field from the `*mut- /// VMContext`.- pub fn vmctx_vmtag_import_vmctx(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- tag: TagIndex,- ) -> ir::Value {- self.vmimport_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets.vmctx_vmtag_import_vmctx(tag),- self.offsets.ptr.vm_tag_import().vmctx().into(),- VmType::VMTagImport,- )- }-- /// Load the imported tag's `VMTagImport::index` field from the `*mut- /// VMContext`.- pub fn vmctx_vmtag_import_index(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- tag: TagIndex,- ) -> ir::Value {- self.vmimport_load(- cursor,- ir::types::I32,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets.vmctx_vmtag_import_index(tag),- self.offsets.ptr.vm_tag_import().index().into(),- VmType::VMTagImport,- )- }+ (@static_entry $Name:ident $snake:ident field {+ $(# $fattr:tt)* $fname:ident : $($fty:tt)*+ }) => {+ #[doc = concat!(+ "Get the [`Field`] for the `", stringify!($fname), "` field of `",+ stringify!($Name), "`."+ )]+ pub fn $fname(self) -> Field<'a, Offsets> {+ let offset = u32::from(self.regions.offsets.get_ptr_size().$snake().$fname());+ define_vmctx_alias_region_helpers!(+ @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]+ )+ }+ };- /// Load the imported tag's `VMTagImport::from` field from the `*mut- /// VMContext`.- pub fn vmctx_vmtag_import_from(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- tag: TagIndex,- ) -> ir::Value {- self.vmimport_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets.vmctx_vmtag_import_from(tag),- self.offsets.ptr.vm_tag_import().from().into(),- VmType::VMTagImport,- )- }+ // ### `dynamic` Section Entries- /// Load the import function's `VMFunctionImport::vmctx` field from the- /// `*mut VMContext`.- pub fn vmctx_vmfunction_import_vmctx(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- func: FuncIndex,- ) -> ir::Value {- self.vmimport_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets.vmctx_vmfunction_import_vmctx(func),- self.offsets.ptr.vm_function_import().vmctx().into(),- VmType::VMFunctionImport,- )- }+ (@dynamic_entry $Name:ident $Offsets:tt align { $al:tt }) => {};- /// Load the import function's `VMFunctionImport::wasm_call` field from the- /// `*mut VMContext`.- pub fn vmctx_vmfunction_import_wasm_call(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- func: FuncIndex,- ) -> ir::Value {- self.vmimport_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- self.offsets.vmctx_vmfunction_import_wasm_call(func),- self.offsets.ptr.vm_function_import().wasm_call().into(),- VmType::VMFunctionImport,- )- }+ // Aggregates get no accessor.+ (@dynamic_entry $Name:ident $Offsets:tt $kind:ident { #[aggregate] $($rest:tt)* }) => {};- /// Load the imported memory's `VMMemoryImport::vmctx` field from the `*mut- /// VMContext`.- pub fn vmctx_vmmemory_import_vmctx(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- memory: MemoryIndex,- ) -> ir::Value {- let mem_offset = self.offsets.vmctx_vmmemory_import(memory);- let mem_vmctx_offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().vmctx());- self.vmimport_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- mem_vmctx_offset,- self.offsets.ptr.vm_memory_import().vmctx().into(),- VmType::VMMemoryImport,- )- }+ (@dynamic_entry $Name:ident [ $($Offsets:tt)* ] array {+ $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*+ }) => {+ #[doc = concat!(+ "Get the [`Field`] for the `index`th element of `", stringify!($Name),+ "`'s `", stringify!($fname), "` array.\n\nPanics if `index` is out of \+ bounds for this vmctx."+ )]+ pub fn $fname(self, index: $Index) -> Field<'a, $($Offsets)*> {+ let offset = self.regions.offsets.$fname().at(index);+ define_vmctx_alias_region_helpers!(+ @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]+ )+ }+ };- /// Load the imported memory's `VMMemoryImport::index` field from the `*mut- /// VMContext`.- pub fn vmctx_vmmemory_import_index(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- memory: MemoryIndex,- ) -> ir::Value {- let mem_offset = self.offsets.vmctx_vmmemory_import(memory);- let mem_index_offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().index());- self.vmimport_load(- cursor,- ir::types::I32,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- mem_index_offset,- self.offsets.ptr.vm_memory_import().index().into(),- VmType::VMMemoryImport,- )- }+ // A single field, whether unconditionally present (`field`) or only+ // conditionally (`optional`).+ (@dynamic_entry $Name:ident [ $($Offsets:tt)* ] $kind:ident {+ $(# $fattr:tt)* $fname:ident $([ if $flag:ident ])? : $($fty:tt)*+ }) => {+ #[doc = concat!(+ "Get the [`Field`] for the `", stringify!($fname), "` field of `",+ stringify!($Name), "`."+ $(, "\n\nPanics if `", stringify!($flag), "` is false, in which case \+ this field is not present at all.")?+ )]+ pub fn $fname(self) -> Field<'a, $($Offsets)*> {+ let offset = self.regions.offsets.$fname();+ define_vmctx_alias_region_helpers!(+ @field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]+ )+ }+ };- /// Load the imported memory's `VMMemoryImport::from` field from the `*mut- /// VMContext`.- pub fn vmctx_vmmemory_import_from(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- memory: MemoryIndex,- ) -> ir::Value {- self.vmctx_vmmemory_import_from_load(cursor.func, memory)- .emit(cursor, vmctx)- }+ // Emit the `impl` block holding the accessors for the dynamically-positioned+ // fields.+ (@dynamic_impl $Name:ident [ $($Offsets:tt)* ] $OffsetsTt:tt {+ $($kind:ident $entry:tt)*+ }) => {+ #[allow(+ dead_code,+ reason = "generated uniformly for every field; not all fields are \+ accessed by compiled code"+ )]+ impl<'a> $Name<'a, $($Offsets)*> {+ $(+ define_vmctx_alias_region_helpers!(+ @dynamic_entry $Name $OffsetsTt $kind $entry+ );+ )*+ }+ };- /// Get a `Load` for the imported memory's `VMMemoryImport::from` field from- /// a `*mut VMContext`.- pub fn vmctx_vmmemory_import_from_load(- &mut self,- func: &mut ir::Function,- memory: MemoryIndex,- ) -> Load {- let mem_offset = self.offsets.vmctx_vmmemory_import(memory);- let offset = mem_offset + u32::from(self.offsets.ptr.vm_memory_import().from());- let region = self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMMemoryImport,- offset: self.offsets.ptr.vm_memory_import().from().into(),- },- );- Load {- offset,- flags: ir::MemFlagsData::trusted()- .with_readonly()- .with_can_move()- .with_alias_region(Some(region)),- ty: self.pointer_type,+ // Emit the accessor `struct` and both `impl` blocks for one vmctx type.+ (@emit $Name:ident $snake:ident $Offsets:tt+ static { $($skind:ident $sentry:tt)* }+ dynamic { $($dyn:tt)* }+ ) => {+ #[doc = concat!(+ "An [`AliasRegions`] accessor for the fields of a `", stringify!($Name),+ "`."+ )]+ pub struct $Name<'a, Offsets> {+ regions: &'a mut AliasRegions<Offsets>,}- }- /// Load the imported table's `VMTableImport::vmctx` field from the `*mut- /// VMContext`.- pub fn vmctx_vmtable_import_vmctx(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- table: TableIndex,- ) -> ir::Value {- let table_offset = self.offsets.vmctx_vmtable_import(table);- let table_vmctx_offset =- table_offset + u32::from(self.offsets.ptr.vm_table_import().vmctx());- self.vmimport_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- table_vmctx_offset,- self.offsets.ptr.vm_table_import().vmctx().into(),- VmType::VMTableImport,- )- }+ impl<Offsets> AliasRegions<Offsets> {+ #[doc = concat!(+ "Get an accessor for the fields of a `", stringify!($Name), "`."+ )]+ pub fn $snake(&mut self) -> $Name<'_, Offsets> {+ $Name { regions: self }+ }+ }- /// Load the imported table's `VMTableImport::index` field from the `*mut- /// VMContext`.- pub fn vmctx_vmtable_import_index(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- table: TableIndex,- ) -> ir::Value {- let table_offset = self.offsets.vmctx_vmtable_import(table);- let table_index_offset =- table_offset + u32::from(self.offsets.ptr.vm_table_import().index());- self.vmimport_load(- cursor,- ir::types::I32,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- table_index_offset,- self.offsets.ptr.vm_table_import().index().into(),- VmType::VMTableImport,- )- }+ // A statically-positioned field's offset depends only on the target+ // pointer size, so these accessors work with any `Offsets`.+ #[allow(+ dead_code,+ reason = "generated uniformly for every field; not all fields are \+ accessed by compiled code"+ )]+ impl<'a, Offsets> $Name<'a, Offsets>+ where+ Offsets: GetPtrSize,+ {+ $( define_vmctx_alias_region_helpers!(@static_entry $Name $snake $skind $sentry); )*+ }- /// Get a `Load` for the imported table's `VMTableImport::from` field (a- /// `*mut VMTableDefinition`) out of a `*mut VMContext`.- pub fn vmctx_vmtable_from_load(&mut self, func: &mut ir::Function, table: TableIndex) -> Load {- let offset = self.offsets.vmctx_vmtable_from(table);- let region = self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMTableImport,- offset: self.offsets.ptr.vm_table_import().from().into(),- },+ // A dynamically-positioned field's offset depends on the module or+ // component being compiled, so these accessors require this vmctx's own+ // fully-computed offsets.+ define_vmctx_alias_region_helpers!(+ @dynamic_impl $Name $Offsets $Offsets { $($dyn)* });- Load {- offset,- flags: ir::MemFlagsData::trusted()- .with_readonly()- .with_can_move()- .with_alias_region(Some(region)),- ty: self.pointer_type,+ };++ // Map each vmctx type to the offsets type that computes its+ // dynamically-positioned fields' offsets.+ (@one VMContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => {+ define_vmctx_alias_region_helpers!(@emit VMContext $snake [ VMOffsets<u8> ]+ static { $($stat)* } dynamic { $($dyn)* });+ };+ (@one VMComponentContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => {+ define_vmctx_alias_region_helpers!(+ @emit VMComponentContext $snake [ VMComponentOffsets<u8> ]+ static { $($stat)* } dynamic { $($dyn)* }+ );+ };++ // Top-level entry.+ ( $(+ {+ $Name:ident $snake:ident+ static { $($stat:tt)* }+ dynamic { $($dyn:tt)* }+ }+ )* ) => {+ $(+ define_vmctx_alias_region_helpers!(@one $Name $snake+ static { $($stat)* } dynamic { $($dyn)* });+ )*+ };+}+wasmtime_environ::for_each_vmctx_type!(define_vmctx_alias_region_helpers);++impl<Offsets> AliasRegions<Offsets>+where+ Offsets: GetPtrSize,+{+ /// Create a new `AliasRegions`.+ pub fn new(offsets: Offsets) -> Self {+ Self {+ pointer_type: ir::Type::int_with_byte_size(offsets.get_ptr_size().size().into())+ .unwrap(),+ offsets,+ cache: std::collections::HashMap::default(),}}- /// Load the imported global's address (`VMGlobalImport::from`) out of the- /// `*mut VMContext`.- pub fn vmctx_vmglobal_import_from(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- global: GlobalIndex,- ) -> ir::Value {- let from_offset = self.offsets.vmctx_vmglobal_import_from(global);- self.vmimport_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- from_offset,- self.offsets.ptr.vm_global_import().from().into(),- VmType::VMGlobalImport,- )+ /// Get the alias region for accesses into the GC heap.+ pub fn gc_heap_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {+ self.region(func, AliasRegionKey::GcHeap)}- /// Load the defined memory's `*mut VMMemoryDefinition` out of the `*mut- /// VMContext`.- pub fn vmctx_vmmemory_pointer(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- memory: DefinedMemoryIndex,- ) -> ir::Value {- self.vmctx_vmmemory_pointer_load(cursor.func, memory)- .emit(cursor, vmctx)+ /// Get the alias region for an imported or exported memory access (shared+ /// across all imported/exported memories).+ pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {+ self.region(func, AliasRegionKey::PublicMemory)}- /// Get a `Load` for the defined memory's `*mut VMMemoryDefinition` out of a- /// `*mut VMContext`.- pub fn vmctx_vmmemory_pointer_load(+ /// Get the alias region for accessing a defined memory that is not+ /// exported.+ pub fn defined_memory_region(&mut self,func: &mut ir::Function,- memory: DefinedMemoryIndex,- ) -> Load {- let offset = self.offsets.vmctx_vmmemory_pointer(memory);- let region = self.vmctx_region(func, offset);- Load {- offset,- flags: ir::MemFlagsData::trusted()- .with_readonly()- .with_can_move()- .with_alias_region(Some(region)),- ty: self.pointer_type,- }+ module: StaticModuleIndex,+ index: DefinedMemoryIndex,+ ) -> ir::AliasRegion {+ self.region(func, AliasRegionKey::DefinedMemory { module, index })}- /// Load the base of the given runtime data out of the `*mut VMContext`.- pub fn vmctx_runtime_data_base(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- runtime_data: RuntimeDataIndex,- ) -> ir::Value {- self.vmctx_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.vmctx_runtime_data_base(runtime_data),- )+ /// Get the alias region for an imported or exported table access (shared+ /// across all imported/exported memories).+ pub fn public_table_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {+ self.region(func, AliasRegionKey::PublicTable)}- /// Load the length of the given runtime data out of the `*mut VMContext`.- pub fn vmctx_runtime_data_length(+ /// Get the alias region for accessing a defined table that is not+ /// exported.+ pub fn defined_table_region(&mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- runtime_data: RuntimeDataIndex,- ) -> ir::Value {- self.vmctx_load(- cursor,- ir::types::I32,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.vmctx_runtime_data_length(runtime_data),- )+ func: &mut ir::Function,+ module: StaticModuleIndex,+ index: DefinedTableIndex,+ ) -> ir::AliasRegion {+ self.region(func, AliasRegionKey::DefinedTable { module, index })+ }++ /// Get the alias region for an imported or exported global access (shared+ /// across all imported/exported memories).+ pub fn public_global_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {+ self.region(func, AliasRegionKey::PublicGlobal)}- /// Load the length of the given runtime data out of the `*mut VMContext`.- pub fn store_vmctx_runtime_data_length(+ /// Get the alias region for accessing a defined global that is not+ /// exported.+ pub fn defined_global_region(&mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- runtime_data: RuntimeDataIndex,- new_length: ir::Value,- ) {- self.vmctx_store(- cursor,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.vmctx_runtime_data_length(runtime_data),- new_length,- )+ func: &mut ir::Function,+ module: StaticModuleIndex,+ index: DefinedGlobalIndex,+ ) -> ir::AliasRegion {+ self.region(func, AliasRegionKey::DefinedGlobal { module, index })}}@@ -2332,243 +2027,45 @@ where}}-/// `VMComponentContext`-related methods, used when compiling component-/// trampolines.+/// `VMComponentContext` fields that are not simply one of the layout's own+/// fields, and so are not generated by [`define_vmctx_alias_region_helpers!`].impl AliasRegions<VMComponentOffsets<u8>> {- fn vmcomponent_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {- self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMComponentContext,- offset,- },- )- }-- fn vmcomponent_load(- &mut self,- cursor: &mut FuncCursor<'_>,- ty: ir::Type,- base_flags: ir::MemFlagsData,- vmctx: ir::Value,- offset: u32,- ) -> ir::Value {- let region = self.vmcomponent_region(cursor.func, offset);- cursor.ins().load(- ty,- base_flags.with_alias_region(Some(region)),- vmctx,- i32::try_from(offset).unwrap(),- )- }-- fn vmcomponent_store(- &mut self,- cursor: &mut FuncCursor<'_>,- base_flags: ir::MemFlagsData,- vmctx: ir::Value,- offset: u32,- val: ir::Value,- ) {- let region = self.vmcomponent_region(cursor.func, offset);- cursor.ins().store(- base_flags.with_alias_region(Some(region)),- val,- vmctx,- i32::try_from(offset).unwrap(),- );- }-- /// Load a lowering's host-data pointer from the `VMComponentContext`.- pub fn vmcomponent_lowering_data(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- index: LoweredIndex,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.lowering_data(index),- )- }-- /// Load a lowering's host callee pointer from the `VMComponentContext`.+ /// Get the [`Field`] for the `callee` of the `index`th lowering in the+ /// `VMComponentContext`'s `lowerings` array.pub fn vmcomponent_lowering_callee(&mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,index: LoweredIndex,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.lowering_callee(index),- )- }-- /// Load the current task's `may_block` flag from the `VMComponentContext`.- pub fn vmcomponent_task_may_block(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- ir::types::I32,- ir::MemFlagsData::trusted().with_readonly(),- vmctx,- self.offsets.task_may_block(),- )- }-- /// Store the current task's `may_block` flag into the `VMComponentContext`.- pub fn store_vmcomponent_task_may_block(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- val: ir::Value,- ) {- self.vmcomponent_store(- cursor,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.task_may_block(),- val,- )- }-- /// Load a resource's destructor function pointer from the- /// `VMComponentContext`.- pub fn vmcomponent_resource_destructor(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- index: ResourceIndex,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly(),- vmctx,- self.offsets.resource_destructor(index),- )- }-- /// Load a runtime memory's `*mut VMMemoryDefinition` from the- /// `VMComponentContext`.- pub fn vmcomponent_runtime_memory(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- index: RuntimeMemoryIndex,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.runtime_memory(index),- )- }-- /// Load a runtime callback function pointer from the `VMComponentContext`.- pub fn vmcomponent_runtime_callback(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- index: RuntimeCallbackIndex,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.runtime_callback(index),- )+ ) -> Field<'_, VMComponentOffsets<u8>> {+ let offset = self.offsets.lowering_callee(index);+ self.vmlowering_field(offset)}- /// Load a runtime post-return function pointer from the- /// `VMComponentContext`.- pub fn vmcomponent_runtime_post_return(+ /// Get the [`Field`] for the host data of the `index`th lowering in the+ /// `VMComponentContext`'s `lowerings` array.+ pub fn vmcomponent_lowering_data(&mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- index: RuntimePostReturnIndex,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmctx,- self.offsets.runtime_post_return(index),- )+ index: LoweredIndex,+ ) -> Field<'_, VMComponentOffsets<u8>> {+ let offset = self.offsets.lowering_data(index);+ self.vmlowering_field(offset)}- /// Load the base pointer of the component builtins array from the+ /// Get the [`Field`] at `offset` within a `VMLowering` inlined into the/// `VMComponentContext`.- pub fn vmcomponent_builtins(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly(),- vmctx,- self.offsets.builtins(),- )- }-- /// Load a component instance's `may_leave` flag from the `VMComponentContext`.- pub fn vmcomponent_instance_may_leave(- &mut self,- cursor: &mut FuncCursor<'_>,- vmctx: ir::Value,- instance: RuntimeComponentInstanceIndex,- ) -> ir::Value {- self.vmcomponent_load(- cursor,- ir::types::I32,+ ///+ /// Unlike the other aggregates inlined into a vmctx, `VMLowering` is not one+ /// of the types defined by `for_each_vm_type!`, and so has no alias region of+ /// its own to rebase onto the aggregate's offset. Its fields are therefore+ /// part of the containing `VMComponentContext`'s region, keyed by their+ /// offset within it.+ fn vmlowering_field(&mut self, offset: u32) -> Field<'_, VMComponentOffsets<u8>> {+ let ty = self.pointer_type;+ Field::new(+ self,+ VmType::VMComponentContext,+ offset,ir::MemFlagsData::trusted(),- vmctx,- self.offsets.may_leave(instance),- )- }-}--/// `VMComponentContext`-related methods that need to be generic over `Offsets`-/// due to the call context.-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- /// Load a field at `offset` within a `VMComponentContext`.- pub fn vmcomponent_context_generic_load(- &mut self,- cursor: &mut FuncCursor<'_>,- ty: ir::Type,- base_flags: ir::MemFlagsData,- vmctx: ir::Value,- offset: u32,- ) -> ir::Value {- let region = self.region(- cursor.func,- AliasRegionKey::Vm {- ty: VmType::VMComponentContext,- offset,- },- );- cursor.ins().load(ty,- base_flags.with_alias_region(Some(region)),- vmctx,- i32::try_from(offset).unwrap(),)}}diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex 5f3fdee1c6..7aa781c40f 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -230,8 +230,10 @@ impl Compiler {caller_vmctx,wasmtime_environ::VMCONTEXT_MAGIC,);- let vm_store_context =- alias_regions.vmctx_store_context(&mut builder.cursor(), caller_vmctx);+ let vm_store_context = alias_regions+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), caller_vmctx);save_last_wasm_exit_fp_and_pc(&mut builder,pointer_type,@@ -266,8 +268,10 @@ impl Compiler {// Increment the "execution version" on the VMStoreContext if// guest debugging is enabled.if self.tunables.debug_guest {- let vmstore_ctx_ptr =- alias_regions.vmctx_store_context(&mut builder.cursor(), caller_vmctx);+ let vmstore_ctx_ptr = alias_regions+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), caller_vmctx);let old_version = alias_regions.vmstore_context_execution_version(&mut builder.cursor(), vmstore_ctx_ptr);let new_version = builder.ins().iadd_imm_s(old_version, 1);@@ -343,7 +347,10 @@ impl Compiler {vmctx,wasmtime_environ::VMCONTEXT_MAGIC,);- let vm_store_context = alias_regions.vmctx_store_context(&mut builder.cursor(), vmctx);+ let vm_store_context = alias_regions+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx);save_last_wasm_exit_fp_and_pc(&mut builder,pointer_type,@@ -550,7 +557,9 @@ impl wasmtime_environ::Compiler for Compiler {if !isa.triple().is_pulley() {let store_ctx = func_env.alias_regions- .vmctx_store_context_load(&mut context.func);+ .vmctx()+ .store_context()+ .to_deferred_load(&mut context.func);let stack_limit = func_env.alias_regions.vmstore_context_stack_limit_load(&mut context.func);@@ -608,7 +617,7 @@ impl wasmtime_environ::Compiler for Compiler {symbol,wasmtime_environ::VMCONTEXT_MAGIC,|alias_regions, _pointer_type, cursor, vmctx| {- alias_regions.vmctx_store_context(cursor, vmctx)+ alias_regions.vmctx().store_context().load(cursor, vmctx)},)}@@ -656,7 +665,7 @@ impl wasmtime_environ::Compiler for Compiler {symbol,wasmtime_environ::VMCONTEXT_MAGIC,|alias_regions, _pointer_type, cursor, vmctx| {- alias_regions.vmctx_store_context(cursor, vmctx)+ alias_regions.vmctx().store_context().load(cursor, vmctx)},),// Delegate to a helper to finish compiling this.@@ -1358,7 +1367,10 @@ impl Compiler {{// Builtins are stored in an array in all `VMContext`s. First load the// base pointer of the array...- let array_addr = alias_regions.vmctx_builtin_functions(&mut builder.cursor(), vmctx);+ let array_addr = alias_regions+ .vmctx()+ .builtin_functions()+ .load(&mut builder.cursor(), vmctx);// ... and then load the entry in the array that corresponds to this// builtin.let func_addr = alias_regions.builtin_functions_array_element(@@ -1408,7 +1420,14 @@ impl Compiler {if !self.emit_debug_checks {return;}- let magic = alias_regions.vmctx_magic(&mut builder.cursor(), vmctx);+ // NB: a `VMComponentContext`'s `magic` field is deliberately read+ // through the `VMContext` accessor: both types keep `magic` at offset+ // zero, and this check is the one place that reads a vmctx's magic+ // without knowing which of the two it has.+ let magic = alias_regions+ .vmctx()+ .magic()+ .load(&mut builder.cursor(), vmctx);let is_expected_vmctx = builder.ins().icmp_imm_s(ir::condcodes::IntCC::Equal,magic,diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex a4c98bebab..4580e60113 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -155,11 +155,10 @@ impl<'a> TrampolineCompiler<'a> {WasmArgs::ValRawList,|me, params| {let vmctx = params[0];- let lowering_data = me.alias_regions.vmcomponent_lowering_data(- &mut me.builder.cursor(),- vmctx,- *index,- );+ let lowering_data = me+ .alias_regions+ .vmcomponent_lowering_data(*index)+ .load(&mut me.builder.cursor(), vmctx);params.extend([lowering_data,me.index_value(*lower_ty),@@ -990,11 +989,10 @@ impl<'a> TrampolineCompiler<'a> {HostCallee::Lowering(index) => {// Load host function pointer from the vmcontext and then call that// indirect function pointer with the list of arguments.- let host_fn = self.alias_regions.vmcomponent_lowering_callee(- &mut self.builder.cursor(),- vmctx,- index,- );+ let host_fn = self+ .alias_regions+ .vmcomponent_lowering_callee(index)+ .load(&mut self.builder.cursor(), vmctx);let host_sig = {let mut sig = ir::Signature::new(CallConv::triple_default(self.isa.triple()));for param in host_args.iter() {@@ -1206,9 +1204,12 @@ impl<'a> TrampolineCompiler<'a> {// Stash the old value of `may_block` and then set it to false.let old_may_block = self.alias_regions- .vmcomponent_task_may_block(&mut self.builder.cursor(), vmctx);+ .vmcomponent()+ .task_may_block()+ .readonly()+ .load(&mut self.builder.cursor(), vmctx);let zero = self.builder.ins().iconst(ir::types::I32, i64::from(0));- self.alias_regions.store_vmcomponent_task_may_block(+ self.alias_regions.vmcomponent().task_may_block().store(&mut self.builder.cursor(),vmctx,zero,@@ -1251,11 +1252,11 @@ impl<'a> TrampolineCompiler<'a> {// NB: despite the vmcontext storing nullable funcrefs for function// pointers we know this is statically never null due to the// `has_destructor` check above.- let dtor_func_ref = self.alias_regions.vmcomponent_resource_destructor(- &mut self.builder.cursor(),- vmctx,- index,- );+ let dtor_func_ref = self+ .alias_regions+ .vmcomponent()+ .resource_destructors(index)+ .load(&mut self.builder.cursor(), vmctx);if self.compiler.emit_debug_checks {self.builder.ins()@@ -1323,7 +1324,7 @@ impl<'a> TrampolineCompiler<'a> {self.raise_if_host_trapped(result.unwrap());// Restore the old value of `may_block`- self.alias_regions.store_vmcomponent_task_may_block(+ self.alias_regions.vmcomponent().task_may_block().store(&mut self.builder.cursor(),vmctx,old_may_block,@@ -1351,7 +1352,9 @@ impl<'a> TrampolineCompiler<'a> {fn load_memory(&mut self, vmctx: ir::Value, memory: RuntimeMemoryIndex) -> ir::Value {self.alias_regions- .vmcomponent_runtime_memory(&mut self.builder.cursor(), vmctx, memory)+ .vmcomponent()+ .memories(memory)+ .load(&mut self.builder.cursor(), vmctx)}fn load_callback(@@ -1361,11 +1364,11 @@ impl<'a> TrampolineCompiler<'a> {) -> ir::Value {let pointer_type = self.isa.pointer_type();match callback {- Some(idx) => self.alias_regions.vmcomponent_runtime_callback(- &mut self.builder.cursor(),- vmctx,- idx,- ),+ Some(idx) => self+ .alias_regions+ .vmcomponent()+ .callbacks(idx)+ .load(&mut self.builder.cursor(), vmctx),None => self.builder.ins().iconst(pointer_type, 0),}}@@ -1377,11 +1380,11 @@ impl<'a> TrampolineCompiler<'a> {) -> ir::Value {let pointer_type = self.isa.pointer_type();match post_return {- Some(idx) => self.alias_regions.vmcomponent_runtime_post_return(- &mut self.builder.cursor(),- vmctx,- idx,- ),+ Some(idx) => self+ .alias_regions+ .vmcomponent()+ .post_returns(idx)+ .load(&mut self.builder.cursor(), vmctx),None => self.builder.ins().iconst(pointer_type, 0),}}@@ -1399,7 +1402,9 @@ impl<'a> TrampolineCompiler<'a> {// per-process.let builtins_array = self.alias_regions- .vmcomponent_builtins(&mut self.builder.cursor(), vmctx);+ .vmcomponent()+ .builtins()+ .load(&mut self.builder.cursor(), vmctx);// Next load the function pointer at `offset` and return that.self.alias_regions.component_builtin_functions_array_element(@@ -1540,11 +1545,11 @@ impl<'a> TrampolineCompiler<'a> {fn check_may_leave_instance(&mut self, instance: RuntimeComponentInstanceIndex) {let vmctx = self.builder.func.dfg.block_params(self.block0)[0];- let may_leave = self.alias_regions.vmcomponent_instance_may_leave(- &mut self.builder.cursor(),- vmctx,- instance,- );+ let may_leave = self+ .alias_regions+ .vmcomponent()+ .may_leave(instance)+ .load(&mut self.builder.cursor(), vmctx);let (mut traps, builder) = self.traps();traps.trapz(builder, may_leave, TRAP_CANNOT_LEAVE_COMPONENT);}@@ -1578,7 +1583,9 @@ impl<'a> TrampolineCompiler<'a> {fn load_vm_store_context(&mut self) -> ir::Value {let caller_vmctx = self.abi_load_params()[1];self.alias_regions- .vmctx_store_context(&mut self.builder.cursor(), caller_vmctx)+ .vmctx()+ .store_context()+ .load(&mut self.builder.cursor(), caller_vmctx)}}@@ -1687,22 +1694,17 @@ impl ComponentCompiler for Compiler {// Implement the array-abi trampoline in terms of calling the// wasm-abi trampoline.Abi::Array => {- let offsets =- VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);return Ok(self.array_to_wasm_trampoline(key,FuncKey::ComponentTrampoline(Abi::Wasm, trampoline_index),sig,symbol,wasmtime_environ::component::VMCOMPONENT_MAGIC,- |alias_regions, pointer_type, cursor, vmctx| {- alias_regions.vmcomponent_context_generic_load(- cursor,- pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- offsets.vm_store_context(),- )+ |alias_regions, _pointer_type, cursor, vmctx| {+ alias_regions+ .vmcomponent()+ .store_context()+ .load(cursor, vmctx)},)?);}@@ -1771,22 +1773,17 @@ impl ComponentCompiler for Compiler {// Implement the array-abi trampoline in terms of calling the// wasm-abi trampoline.Abi::Array => {- let offsets =- VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);return Ok(self.array_to_wasm_trampoline(FuncKey::UnsafeIntrinsic(abi, intrinsic),FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic),&wasm_func_ty,symbol,wasmtime_environ::component::VMCOMPONENT_MAGIC,- |alias_regions, pointer_type, cursor, vmctx| {- alias_regions.vmcomponent_context_generic_load(- cursor,- pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmctx,- offsets.vm_store_context(),- )+ |alias_regions, _pointer_type, cursor, vmctx| {+ alias_regions+ .vmcomponent()+ .store_context()+ .load(cursor, vmctx)},)?);}@@ -2424,7 +2421,9 @@ wherelet caller_vmctx = params[1];self.traps.alias_regions()- .vmctx_store_context(&mut self.builder.cursor(), caller_vmctx)+ .vmctx()+ .store_context()+ .load(&mut self.builder.cursor(), caller_vmctx)}}diff --git a/crates/cranelift/src/debug.rs b/crates/cranelift/src/debug.rsindex 55a0a7a2fb..dbfedf8631 100644--- a/crates/cranelift/src/debug.rs+++ b/crates/cranelift/src/debug.rs@@ -102,17 +102,20 @@ impl<'a> Compilation<'a> {let memory_offset = if ofs.num_imported_memories > 0 {let index = MemoryIndex::new(0);ModuleMemoryOffset::Imported {- offset_to_vm_memory_definition: ofs.vmctx_vmmemory_import(index)+ offset_to_vm_memory_definition: ofs.imported_memories().at(index)+ u32::from(ofs.ptr.vm_memory_import().from()),offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(),}} else if ofs.num_owned_memories > 0 {let index = OwnedMemoryIndex::new(0);- ModuleMemoryOffset::Defined(ofs.vmctx_vmmemory_definition_base(index))+ ModuleMemoryOffset::Defined(+ ofs.owned_memories().at(index)+ + u32::from(ofs.ptr.vm_memory_definition().base()),+ )} else if ofs.num_defined_memories > 0 {let index = DefinedMemoryIndex::new(0);ModuleMemoryOffset::Imported {- offset_to_vm_memory_definition: ofs.vmctx_vmmemory_pointer(index),+ offset_to_vm_memory_definition: ofs.memories().at(index),offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(),}} else {diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 6a58b20fe9..71b07ef666 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -450,12 +450,16 @@ impl<'module_environment> FuncEnvironment<'module_environment> {) -> (ir::Value, i32) {let vmctx = self.vmctx_val(pos);if let Some(def_index) = self.module.defined_global_index(index) {- let offset = i32::try_from(self.offsets.vmctx_vmglobal_definition(def_index)).unwrap();+ let offset = i32::try_from(self.offsets.globals().at(def_index)).unwrap();(vmctx, offset)} else {+ let import_off = self.offsets.imported_globals().at(index);let addr = self.alias_regions- .vmctx_vmglobal_import_from(pos, vmctx, index);+ .vm_global_import()+ .from()+ .relative_to(import_off)+ .load(pos, vmctx);(addr, 0)}}@@ -464,7 +468,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {fn get_vmstore_context_ptr(&mut self, builder: &mut FunctionBuilder) -> ir::Value {let vmctx = self.vmctx_val(&mut builder.cursor());self.alias_regions- .vmctx_store_context(&mut builder.cursor(), vmctx)+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx)}fn fuel_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) {@@ -737,7 +743,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {fn epoch_ptr(&mut self, builder: &mut FunctionBuilder<'_>) -> ir::Value {let vmctx = self.vmctx_val(&mut builder.cursor());self.alias_regions- .vmctx_epoch_ptr(&mut builder.cursor(), vmctx)+ .vmctx()+ .epoch_ptr()+ .load(&mut builder.cursor(), vmctx)}fn epoch_load_current(&mut self, builder: &mut FunctionBuilder<'_>) -> ir::Value {@@ -1097,7 +1105,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {) -> ir::Value {let vmctx = self.vmctx_val(pos);// Load the base pointer of the array of `VMSharedTypeIndex`es.- let shared_indices = self.alias_regions.vmctx_shared_type_ids_array(pos, vmctx);+ let shared_indices = self.alias_regions.vmctx().type_ids().load(pos, vmctx);// Calculate the offset in that array for this type's entry.@@ -1550,7 +1558,9 @@ impl FuncEnvironment<'_> {// atomically growing it.let mem = self.alias_regions- .vmctx_vmmemory_pointer_load(func, def_index);+ .vmctx()+ .memories(def_index)+ .to_deferred_load(func);let mut base = self.alias_regions.vm_memory_definition().base();base.can_move();if base_readonly {@@ -1572,7 +1582,7 @@ impl FuncEnvironment<'_> {// relative to the `vmctx` itself: fold that offset into each// field's offset.let owned_index = self.module.owned_memory_index(def_index);- let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index);+ let vmctx_off = self.offsets.owned_memories().at(owned_index);let mut base = self.alias_regions.vm_memory_definition().base();base.can_move();if base_readonly {@@ -1591,9 +1601,13 @@ impl FuncEnvironment<'_> {)}} else {+ let import_off = self.offsets.imported_memories().at(index);let mem = self.alias_regions- .vmctx_vmmemory_import_from_load(func, index);+ .vm_memory_import()+ .from()+ .relative_to(import_off)+ .to_deferred_load(func);let mut base = self.alias_regions.vm_memory_definition().base();base.can_move();if base_readonly {@@ -1646,7 +1660,7 @@ impl FuncEnvironment<'_> {if let Some(def_index) = self.module.defined_table_index(index) {// A defined table's `VMTableDefinition` is inlined into the vmctx,// reached at an absolute `vmctx` offset.- let vmctx_off = self.offsets.vmctx_vmtable_definition(def_index);+ let vmctx_off = self.offsets.tables().at(def_index);let mut base = self.alias_regions.vm_table_definition().base();if is_static {base.readonly().can_move();@@ -1674,7 +1688,13 @@ impl FuncEnvironment<'_> {} else {// An imported table is reached through a `*mut VMTableDefinition`// loaded from the `vmctx`.- let from = self.alias_regions.vmctx_vmtable_from_load(func, index);+ let import_off = self.offsets.imported_tables().at(index);+ let from = self+ .alias_regions+ .vm_table_import()+ .from()+ .relative_to(import_off)+ .to_deferred_load(func);let mut base = self.alias_regions.vm_table_definition().base();if is_static {base.readonly().can_move();@@ -1735,16 +1755,19 @@ impl FuncEnvironment<'_> {} else {// An imported tag -- we need to load the VMTagImport struct.let vmctx = self.vmctx_val(&mut builder.cursor());- let from_vmctx = self.alias_regions.vmctx_vmtag_import_vmctx(- &mut builder.cursor(),- vmctx,- tag_index,- );- let index = self.alias_regions.vmctx_vmtag_import_index(- &mut builder.cursor(),- vmctx,- tag_index,- );+ let import_off = self.offsets.imported_tags().at(tag_index);+ let from_vmctx = self+ .alias_regions+ .vm_tag_import()+ .vmctx()+ .relative_to(import_off)+ .load(&mut builder.cursor(), vmctx);+ let index = self+ .alias_regions+ .vm_tag_import()+ .index()+ .relative_to(import_off)+ .load(&mut builder.cursor(), vmctx);let builtin = self.builtin_functions.get_instance_id(builder.func);let call = builder.ins().call(builtin, &[from_vmctx]);let from_instance_id = builder.func.dfg.inst_results(call)[0];@@ -1841,11 +1864,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// First append the callee vmctx address.let vmctx = self.env.vmctx_val(&mut self.builder.cursor());- let callee_vmctx = self.env.alias_regions.vmctx_vmfunction_import_vmctx(- &mut self.builder.cursor(),- vmctx,- callee_index,- );+ let import_off = self.env.offsets.imported_functions().at(callee_index);+ let callee_vmctx = self+ .env+ .alias_regions+ .vm_function_import()+ .vmctx()+ .relative_to(import_off)+ .load(&mut self.builder.cursor(), vmctx);real_call_args.push(callee_vmctx);real_call_args.push(caller_vmctx);@@ -1929,11 +1955,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {}FactInlineIntrinsic::EnterSyncCall | FactInlineIntrinsic::ExitSyncCall => {}}- let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(- &mut self.builder.cursor(),- vmctx,- callee_index,- );+ let import_off = self.env.offsets.imported_functions().at(callee_index);+ let func_addr = self+ .env+ .alias_regions+ .vm_function_import()+ .wasm_call()+ .relative_to(import_off)+ .load(&mut self.builder.cursor(), vmctx);Ok(Reachability::Reachable(self.indirect_call_inst(sig_ref,func_addr,@@ -1947,11 +1976,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// and with different functions. Either way, we have to do the// indirect call.None => {- let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(- &mut self.builder.cursor(),- vmctx,- callee_index,- );+ let import_off = self.env.offsets.imported_functions().at(callee_index);+ let func_addr = self+ .env+ .alias_regions+ .vm_function_import()+ .wasm_call()+ .relative_to(import_off)+ .load(&mut self.builder.cursor(), vmctx);Ok(Reachability::Reachable(self.indirect_call_inst(sig_ref,func_addr,@@ -2143,11 +2175,14 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// equivalent out-of-line teardown via the `exit_sync_call` libcall.self.builder.switch_to_block(slow_block);let vmctx = self.env.vmctx_val(&mut self.builder.cursor());- let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(- &mut self.builder.cursor(),- vmctx,- callee_index,- );+ let import_off = self.env.offsets.imported_functions().at(callee_index);+ let func_addr = self+ .env+ .alias_regions+ .vm_function_import()+ .wasm_call()+ .relative_to(import_off)+ .load(&mut self.builder.cursor(), vmctx);self.indirect_call_inst(sig_ref, func_addr, real_call_args);self.builder.ins().jump(cont_block, &[]);@@ -3518,12 +3553,19 @@ impl FuncEnvironment<'_> {// This is an imported memory, so load the vmctx/defined index from// the import definition itself.None => {+ let import_off = self.offsets.imported_memories().at(index);let vmctx = self.alias_regions- .vmctx_vmmemory_import_vmctx(pos, cur_vmctx, index);+ .vm_memory_import()+ .vmctx()+ .relative_to(import_off)+ .load(pos, cur_vmctx);let index = self.alias_regions- .vmctx_vmmemory_import_index(pos, cur_vmctx, index);+ .vm_memory_import()+ .index()+ .relative_to(import_off)+ .load(pos, cur_vmctx);(vmctx, index)}}@@ -3544,12 +3586,19 @@ impl FuncEnvironment<'_> {match self.module.defined_table_index(index) {Some(index) => (cur_vmctx, pos.ins().iconst(I32, i64::from(index.as_u32()))),None => {+ let import_off = self.offsets.imported_tables().at(index);let vmctx = self.alias_regions- .vmctx_vmtable_import_vmctx(pos, cur_vmctx, index);+ .vm_table_import()+ .vmctx()+ .relative_to(import_off)+ .load(pos, cur_vmctx);let index = self.alias_regions- .vmctx_vmtable_import_index(pos, cur_vmctx, index);+ .vm_table_import()+ .index()+ .relative_to(import_off)+ .load(pos, cur_vmctx);(vmctx, index)}}@@ -3600,7 +3649,9 @@ impl FuncEnvironment<'_> {if is_shared {let mem_ptr = self.alias_regions- .vmctx_vmmemory_pointer(pos, vmctx, def_index);+ .vmctx()+ .memories(def_index)+ .load(pos, vmctx);self.alias_regions.vm_memory_definition().current_length()@@ -3609,7 +3660,7 @@ impl FuncEnvironment<'_> {// A defined, owned memory's `VMMemoryDefinition` is inlined into// the `vmctx` at an absolute offset.let owned_index = self.module.owned_memory_index(def_index);- let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index);+ let vmctx_off = self.offsets.owned_memories().at(owned_index);self.alias_regions.vm_memory_definition().current_length()@@ -3617,9 +3668,13 @@ impl FuncEnvironment<'_> {.load(pos, vmctx)}} else {+ let import_off = self.offsets.imported_memories().at(index);let mem_ptr = self.alias_regions- .vmctx_vmmemory_import_from(pos, vmctx, index);+ .vm_memory_import()+ .from()+ .relative_to(import_off)+ .load(pos, vmctx);if is_shared {self.alias_regions.vm_memory_definition()@@ -4040,12 +4095,10 @@ impl FuncEnvironment<'_> {// the value 0 to the `VMContext`'s slot for this passive data segment.let vmctx = self.vmctx_val(&mut pos);let new_length = pos.ins().iconst(I32, 0);- self.alias_regions.store_vmctx_runtime_data_length(- &mut pos,- vmctx,- runtime_index,- new_length,- );+ self.alias_regions+ .vmctx()+ .runtime_data_lengths(runtime_index)+ .store(&mut pos, vmctx, new_length);Ok(())}@@ -4331,7 +4384,9 @@ impl FuncEnvironment<'_> {) -> ir::Value {let vmctx = self.vmctx_val(&mut builder.cursor());self.alias_regions- .vmctx_runtime_data_length(&mut builder.cursor(), vmctx, runtime_index)+ .vmctx()+ .runtime_data_lengths(runtime_index)+ .load(&mut builder.cursor(), vmctx)}fn load_runtime_data_length_as_pointer(@@ -4350,7 +4405,9 @@ impl FuncEnvironment<'_> {) -> ir::Value {let vmctx = self.vmctx_val(&mut builder.cursor());self.alias_regions- .vmctx_runtime_data_base(&mut builder.cursor(), vmctx, runtime_index)+ .vmctx()+ .runtime_data_bases(runtime_index)+ .load(&mut builder.cursor(), vmctx)}pub fn translate_table_copy(diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rsindex ac8275f027..b88d21c050 100644--- a/crates/cranelift/src/func_environ/gc.rs+++ b/crates/cranelift/src/func_environ/gc.rs@@ -1524,7 +1524,11 @@ impl FuncEnvironment<'_> {return heap;}- let store_ctx = self.alias_regions.vmctx_store_context_load(func);+ let store_ctx = self+ .alias_regions+ .vmctx()+ .store_context()+ .to_deferred_load(func);// The base pointer's load is `can_move` and `readonly` when the GC// heap's base can never move.diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rsindex e4eac77885..7f3a56541e 100644--- a/crates/cranelift/src/func_environ/gc/copying.rs+++ b/crates/cranelift/src/func_environ/gc/copying.rs@@ -34,7 +34,9 @@ impl CopyingCompiler {let vmctx = func_env.vmctx_val(&mut builder.cursor());func_env.alias_regions- .vmctx_gc_heap_data(&mut builder.cursor(), vmctx)+ .vmctx()+ .gc_heap_data()+ .load(&mut builder.cursor(), vmctx)}/// Load the current bump pointer and active-space end from a `*mutdiff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rsindex 8a9de06f63..da3d205965 100644--- a/crates/cranelift/src/func_environ/gc/drc.rs+++ b/crates/cranelift/src/func_environ/gc/drc.rs@@ -105,7 +105,9 @@ impl DrcCompiler {let vmctx = func_env.vmctx_val(&mut builder.cursor());let heap_data = func_env.alias_regions- .vmctx_gc_heap_data(&mut builder.cursor(), vmctx);+ .vmctx()+ .gc_heap_data()+ .load(&mut builder.cursor(), vmctx);// Load the current first list element, which will be our new next list// element.@@ -171,7 +173,9 @@ impl DrcCompiler {let vmctx = func_env.vmctx_val(&mut builder.cursor());let heap_data = func_env.alias_regions- .vmctx_gc_heap_data(&mut builder.cursor(), vmctx);+ .vmctx()+ .gc_heap_data()+ .load(&mut builder.cursor(), vmctx);let current_len = func_env.alias_regions.vmdrc_heap_data_current_over_approximated_stack_roots_len(diff --git a/crates/cranelift/src/func_environ/gc/null.rs b/crates/cranelift/src/func_environ/gc/null.rsindex b951d9b30b..454166c278 100644--- a/crates/cranelift/src/func_environ/gc/null.rs+++ b/crates/cranelift/src/func_environ/gc/null.rs@@ -72,7 +72,9 @@ impl NullCompiler {let vmctx = func_env.vmctx_val(&mut builder.cursor());let ptr_to_next = func_env.alias_regions- .vmctx_gc_heap_data(&mut builder.cursor(), vmctx);+ .vmctx()+ .gc_heap_data()+ .load(&mut builder.cursor(), vmctx);let next = func_env.alias_regions.vmnull_heap_data_bump_finger(&mut builder.cursor(), ptr_to_next);diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rsindex e6a84e9a27..56386c797c 100644--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs@@ -1012,11 +1012,15 @@ pub(crate) fn tag_address<'a>(let vmctx = env.vmctx_val(&mut builder.cursor());let tag_index = wasmtime_environ::TagIndex::from_u32(index);if let Some(def_index) = env.module.defined_tag_index(tag_index) {- let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap();+ let offset = i32::try_from(env.offsets.tags().at(def_index)).unwrap();builder.ins().iadd_imm_s(vmctx, i64::from(offset))} else {+ let import_off = env.offsets.imported_tags().at(tag_index);env.alias_regions- .vmctx_vmtag_import_from(&mut builder.cursor(), vmctx, tag_index)+ .vm_tag_import()+ .from()+ .relative_to(import_off)+ .load(&mut builder.cursor(), vmctx)}}@@ -1033,7 +1037,9 @@ pub fn vmctx_load_stack_chain<'a>(// First we need to get the `VMStoreContext`.let vm_store_context = env.alias_regions- .vmctx_store_context(&mut builder.cursor(), vmctx);+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx);let stack_chain_region = env.alias_regions@@ -1061,7 +1067,9 @@ pub fn vmctx_store_stack_chain<'a>(// First we need to get the `VMStoreContext`.let vm_store_context = env.alias_regions- .vmctx_store_context(&mut builder.cursor(), vmctx);+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx);let stack_chain_region = env.alias_regions@@ -1565,7 +1573,9 @@ fn translate_resume_impl<'a>(// of the invariants that we maintain for the various stack limits.let vm_runtime_limits_ptr = env.alias_regions- .vmctx_store_context(&mut builder.cursor(), vmctx);+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx);parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true);resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);@@ -2032,7 +2042,9 @@ pub(crate) fn translate_switch<'a>(// switcher continuation.let vm_runtime_limits_ptr = env.alias_regions- .vmctx_store_context(&mut builder.cursor(), vmctx);+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx);switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false);let revision = switcher_contref.get_revision(env, builder);diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rsindex 1b1c148346..937b744527 100644--- a/crates/environ/src/component/vmcomponent_offsets.rs+++ b/crates/environ/src/component/vmcomponent_offsets.rs@@ -1,20 +1,9 @@-// Currently the `VMComponentContext` allocation by field looks like this:-//-// struct VMComponentContext {-// magic: u32,-// builtins: &'static VMComponentBuiltins,-// limits: *const VMStoreContext,-// may_leave: [VMGlobalDefinition; component.num_runtime_component_instances],-// task_may_block: u32,-// trampoline_func_refs: [VMFuncRef; component.num_trampolines],-// unsafe_intrinsics: [VMFuncRef; component.num_unsafe_intrinsics],-// lowerings: [VMLowering; component.num_lowerings],-// memories: [*mut VMMemoryDefinition; component.num_runtime_memories],-// tables: [VMTable; component.num_runtime_tables],-// reallocs: [*mut VMFuncRef; component.num_runtime_reallocs],-// post_returns: [*mut VMFuncRef; component.num_runtime_post_returns],-// resource_destructors: [*mut VMFuncRef; component.num_resources],-// }+//! Offsets of the fields within a `VMComponentContext`.+//!+//! The layout itself is not defined here: it is defined once, alongside+//! `VMContext`'s, in `for_each_vmctx_type!`. Everything in this module is either+//! generated from that definition or is an offset that is not simply the offset+//! of one of the layout's fields.use crate::GetPtrSize;use crate::PtrSize;@@ -55,10 +44,11 @@ pub struct VMComponentOffsets<P> {/// Number of resources within a component which need destructors stored.pub num_resources: u32,- // precalculated offsets of various member fields- magic: u32,- builtins: u32,- vm_store_context: u32,+ // Precalculated offsets of the dynamically-positioned fields, one per entry+ // in `VMComponentContext`'s `dynamic` section in `for_each_vmctx_type!`,+ // plus this `VMComponentContext`'s total size. These are all computed by the+ // generated `compute_field_offsets` and read by the generated accessors of+ // the same names.may_leave: u32,task_may_block: u32,trampoline_func_refs: u32,@@ -82,16 +72,66 @@ impl<P: PtrSize> GetPtrSize for VMComponentOffsets<P> {}}-#[inline]-fn align(offset: u32, align: u32) -> u32 {- assert!(align.is_power_of_two());- (offset + (align - 1)) & !(align - 1)+impl_vmctx_array_index! {+ RuntimeComponentInstanceIndex,+ TrampolineIndex,+ LoweredIndex,+ RuntimeMemoryIndex,+ RuntimeTableIndex,+ RuntimeReallocIndex,+ RuntimeCallbackIndex,+ RuntimePostReturnIndex,+ ResourceIndex,}+impl crate::VmctxArrayIndex for UnsafeIntrinsic {+ #[inline]+ fn vmctx_array_index(self) -> u32 {+ self.index()+ }+}++/// Generate the offsets of `VMComponentContext`'s dynamically-positioned fields,+/// ignoring every other vmctx type.+macro_rules! define_vmcomponent_offsets_dynamic_offsets {+ (@one VMComponentContext $snake:ident { $($dyn:tt)* }) => {+ /// Offsets of the dynamically-positioned fields of `VMComponentContext`.+ impl<P: PtrSize> VMComponentOffsets<P> {+ define_vmctx_dynamic_offsets!(@accessors (self) [ $($dyn)* ]);++ define_vmctx_dynamic_offsets!(@compute_fn (self, next) $snake [ $($dyn)* ]);++ /// Return the size of the `VMComponentContext` allocation.+ #[inline]+ pub fn size_of_vmctx(&self) -> u32 {+ self.size+ }+ }+ };+ (@one $other:ident $snake:ident { $($dyn:tt)* }) => {};++ ( $(+ {+ $Name:ident $snake:ident+ static { $($stat:tt)* }+ dynamic { $($dyn:tt)* }+ }+ )* ) => {+ $( define_vmcomponent_offsets_dynamic_offsets!(@one $Name $snake { $($dyn)* }); )*+ };+}++for_each_vmctx_type!(define_vmcomponent_offsets_dynamic_offsets);+impl<P: PtrSize> VMComponentOffsets<P> {/// Creates a new set of offsets for the `component` specified configured/// additionally for the `ptr` size specified.pub fn new(ptr: P, component: &Component) -> Self {+ // This is required by the implementation of+ // `VMComponentContext::from_opaque`. If this value changes then this+ // location needs to be updated.+ assert_eq!(ptr.vmcomponent().magic(), 0);+let mut ret = Self {ptr,num_lowerings: component.num_lowerings,@@ -125,9 +165,6 @@ impl<P: PtrSize> VMComponentOffsets<P> {0},num_resources: component.num_resources,- magic: 0,- builtins: 0,- vm_store_context: 0,may_leave: 0,task_may_block: 0,trampoline_func_refs: 0,@@ -142,58 +179,9 @@ impl<P: PtrSize> VMComponentOffsets<P> {size: 0,};- // Convenience functions for checked addition and multiplication.- // As side effect this reduces binary size by using only a single- // `#[track_caller]` location for each function instead of one for- // each individual invocation.- #[inline]- fn cmul(count: u32, size: u8) -> u32 {- count.checked_mul(u32::from(size)).unwrap()- }-- let mut next_field_offset = 0;-- macro_rules! fields {- (size($field:ident) = $size:expr, $($rest:tt)*) => {- ret.$field = next_field_offset;- next_field_offset = next_field_offset.checked_add(u32::from($size)).unwrap();- fields!($($rest)*);- };- (align($align:expr), $($rest:tt)*) => {- next_field_offset = align(next_field_offset, $align);- fields!($($rest)*);- };- () => {};- }-- fields! {- size(magic) = 4u32,- align(u32::from(ret.ptr.size())),- size(builtins) = ret.ptr.size(),- size(vm_store_context) = ret.ptr.size(),- align(16),- size(may_leave) = cmul(ret.num_runtime_component_instances, ret.ptr.vm_global_definition().size()),- size(task_may_block) = ret.ptr.vm_global_definition().size(),- align(u32::from(ret.ptr.size())),- size(trampoline_func_refs) = cmul(ret.num_trampolines, ret.ptr.vm_func_ref().size()),- size(intrinsic_func_refs) = cmul(ret.num_unsafe_intrinsics, ret.ptr.vm_func_ref().size()),- size(lowerings) = cmul(ret.num_lowerings, ret.ptr.size() * 2),- size(memories) = cmul(ret.num_runtime_memories, ret.ptr.size()),- size(tables) = cmul(ret.num_runtime_tables, ret.ptr.vm_table_import().size()),- size(reallocs) = cmul(ret.num_runtime_reallocs, ret.ptr.size()),- size(callbacks) = cmul(ret.num_runtime_callbacks, ret.ptr.size()),- size(post_returns) = cmul(ret.num_runtime_post_returns, ret.ptr.size()),- size(resource_destructors) = cmul(ret.num_resources, ret.ptr.size()),- }-- ret.size = next_field_offset;-- // This is required by the implementation of- // `VMComponentContext::from_opaque`. If this value changes then this- // location needs to be updated.- assert_eq!(ret.magic, 0);+ ret.compute_field_offsets();- return ret;+ ret}/// The size, in bytes, of the host pointer.@@ -202,86 +190,16 @@ impl<P: PtrSize> VMComponentOffsets<P> {self.ptr.size()}- /// The offset of the `magic` field.- #[inline]- pub fn magic(&self) -> u32 {- self.magic- }-- /// The offset of the `builtins` field.- #[inline]- pub fn builtins(&self) -> u32 {- self.builtins- }-- /// The offset of the `may_leave` flag for the given component instance.- #[inline]- pub fn may_leave(&self, index: RuntimeComponentInstanceIndex) -> u32 {- assert!(index.as_u32() < self.num_runtime_component_instances);- self.may_leave + index.as_u32() * u32::from(self.ptr.vm_global_definition().size())- }-- /// The offset of the `task_may_block` field.- pub fn task_may_block(&self) -> u32 {- self.task_may_block- }-- /// The offset of the `vm_store_context` field.- #[inline]- pub fn vm_store_context(&self) -> u32 {- self.vm_store_context- }-- /// The offset of the `trampoline_func_refs` field.- #[inline]- pub fn trampoline_func_refs(&self) -> u32 {- self.trampoline_func_refs- }-- /// The offset of `VMFuncRef` for the `index` specified.- #[inline]- pub fn trampoline_func_ref(&self, index: TrampolineIndex) -> u32 {- assert!(index.as_u32() < self.num_trampolines);- self.trampoline_func_refs() + index.as_u32() * u32::from(self.ptr.vm_func_ref().size())- }-- /// The offset of the `unsafe_intrinsic_func_refs` field.- #[inline]- pub fn unsafe_intrinsic_func_refs(&self) -> u32 {- self.intrinsic_func_refs- }-- /// The offset of the `VMFuncRef` for the `intrinsic` specified.- #[inline]- pub fn unsafe_intrinsic_func_ref(&self, intrinsic: UnsafeIntrinsic) -> u32 {- assert!(intrinsic.index() < self.num_unsafe_intrinsics);- self.unsafe_intrinsic_func_refs()- + intrinsic.index() * u32::from(self.ptr.vm_func_ref().size())- }-- /// The offset of the `lowerings` field.- #[inline]- pub fn lowerings(&self) -> u32 {- self.lowerings- }-- /// The offset of the `VMLowering` for the `index` specified.- #[inline]- pub fn lowering(&self, index: LoweredIndex) -> u32 {- assert!(index.as_u32() < self.num_lowerings);- self.lowerings() + index.as_u32() * u32::from(2 * self.ptr.size())- }-/// The offset of the `callee` for the `index` specified.#[inline]pub fn lowering_callee(&self, index: LoweredIndex) -> u32 {- self.lowering(index) + self.lowering_callee_offset()+ self.lowerings().at(index) + self.lowering_callee_offset()}/// The offset of the `data` for the `index` specified.#[inline]pub fn lowering_data(&self, index: LoweredIndex) -> u32 {- self.lowering(index) + self.lowering_data_offset()+ self.lowerings().at(index) + self.lowering_data_offset()}/// The size of the `VMLowering` type@@ -301,93 +219,4 @@ impl<P: PtrSize> VMComponentOffsets<P> {pub fn lowering_data_offset(&self) -> u32 {u32::from(self.ptr.size())}-- /// The offset of the base of the `runtime_memories` field- #[inline]- pub fn runtime_memories(&self) -> u32 {- self.memories- }-- /// The offset of the `*mut VMMemoryDefinition` for the runtime index- /// provided.- #[inline]- pub fn runtime_memory(&self, index: RuntimeMemoryIndex) -> u32 {- assert!(index.as_u32() < self.num_runtime_memories);- self.runtime_memories() + index.as_u32() * u32::from(self.ptr.size())- }-- /// The offset of the base of the `runtime_tables` field- #[inline]- pub fn runtime_tables(&self) -> u32 {- self.tables- }-- /// The offset of the table for the runtime index provided.- #[inline]- pub fn runtime_table(&self, index: RuntimeTableIndex) -> u32 {- assert!(index.as_u32() < self.num_runtime_tables);- self.runtime_tables() + index.as_u32() * u32::from(self.ptr.vm_table_import().size())- }-- /// The offset of the base of the `runtime_reallocs` field- #[inline]- pub fn runtime_reallocs(&self) -> u32 {- self.reallocs- }-- /// The offset of the `*mut VMFuncRef` for the runtime index- /// provided.- #[inline]- pub fn runtime_realloc(&self, index: RuntimeReallocIndex) -> u32 {- assert!(index.as_u32() < self.num_runtime_reallocs);- self.runtime_reallocs() + index.as_u32() * u32::from(self.ptr.size())- }-- /// The offset of the base of the `runtime_callbacks` field- #[inline]- pub fn runtime_callbacks(&self) -> u32 {- self.callbacks- }-- /// The offset of the `*mut VMFuncRef` for the runtime index- /// provided.- #[inline]- pub fn runtime_callback(&self, index: RuntimeCallbackIndex) -> u32 {- assert!(index.as_u32() < self.num_runtime_callbacks);- self.runtime_callbacks() + index.as_u32() * u32::from(self.ptr.size())- }-- /// The offset of the base of the `runtime_post_returns` field- #[inline]- pub fn runtime_post_returns(&self) -> u32 {- self.post_returns- }-- /// The offset of the `*mut VMFuncRef` for the runtime index- /// provided.- #[inline]- pub fn runtime_post_return(&self, index: RuntimePostReturnIndex) -> u32 {- assert!(index.as_u32() < self.num_runtime_post_returns);- self.runtime_post_returns() + index.as_u32() * u32::from(self.ptr.size())- }-- /// The offset of the base of the `resource_destructors` field- #[inline]- pub fn resource_destructors(&self) -> u32 {- self.resource_destructors- }-- /// The offset of the `*mut VMFuncRef` for the runtime index- /// provided.- #[inline]- pub fn resource_destructor(&self, index: ResourceIndex) -> u32 {- assert!(index.as_u32() < self.num_resources);- self.resource_destructors() + index.as_u32() * u32::from(self.ptr.size())- }-- /// Return the size of the `VMComponentContext` allocation.- #[inline]- pub fn size_of_vmctx(&self) -> u32 {- self.size- }}diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rsindex 2be10ed159..774f140423 100644--- a/crates/environ/src/lib.rs+++ b/crates/environ/src/lib.rs@@ -43,6 +43,8 @@ mod trap_encoding;mod tunables;mod types;#[macro_use]+mod vmctxtypes;+#[macro_use]mod vmtypes;mod vmoffsets;mod wasm_error;@@ -66,6 +68,7 @@ pub use crate::string_pool::{Atom, StringPool};pub use crate::trap_encoding::*;pub use crate::tunables::*;pub use crate::types::*;+pub use crate::vmctxtypes::{ArrayOffsets, VmctxArrayIndex};pub use crate::vmoffsets::*;pub use crate::wasm_error::*;pub use object;diff --git a/crates/environ/src/vmctxtypes.rs b/crates/environ/src/vmctxtypes.rsnew file mode 100644index 0000000000..ddac95c64e--- /dev/null+++ b/crates/environ/src/vmctxtypes.rs@@ -0,0 +1,514 @@+//! Centralized definitions of the layouts of Wasmtime's two "vmctx" types:+//! `VMContext`, the runtime context for a core Wasm instance, and+//! `VMComponentContext`, the runtime context for a component.+//!+//! Unlike the `VM*` types defined by `for_each_vm_type!`, neither of these has+//! a corresponding `#[repr(C)]` Rust `struct`: their sizes depend on the module+//! or component being instantiated, so they are dynamically laid out and+//! accessed exclusively through the computed offsets in `VMOffsets` and+//! `VMComponentOffsets`. That layout is defined exactly once here, via the+//! higher-order `for_each_vmctx_type!` macro, and each consumer generates its+//! view of it from that single source of truth.++/// Invoke the given macro `$mac` once, passing it the layout of each of+/// Wasmtime's "vmctx" types.+///+/// This is a higher-order macro: callers define a `macro_rules!` macro that+/// matches the grammar described below and pass its name as an argument to this+/// macro's invocation, e.g. `for_each_vmctx_type!(define_vmctx_offsets)`.+///+/// # Grammar+///+/// The layout below is written in exactly the same grammar that it is handed to+/// `$mac` in: this macro has a single rule, and that rule does nothing but+/// forward those tokens along. Writing the layout in the grammar that consumers+/// match makes it more verbose than a bespoke input syntax would be, but in+/// exchange there is no normalization pass in between, so what a consumer matches+/// is exactly what a reader of the layout sees.+///+/// `$mac` receives one brace-delimited group per vmctx type:+///+/// ```ignore+/// {+/// VMContext vmctx+/// static { ...entries... }+/// dynamic { ...entries... }+/// }+/// ```+///+/// where `vmctx` is the name used for accessor methods of that type, `static`+/// holds the fixed-width prefix whose offsets depend only on the target pointer+/// size, and `dynamic` holds the rest, whose offsets additionally depend on the+/// module or component being compiled.+///+/// Each entry within a section is a keyword naming the entry's shape followed by+/// a single delimited group, so that a consumer can iterate over a section with+/// `$( ...$kind:ident $entry:tt... )*` and dispatch on one whole entry at a time+/// instead of munching the section token by token. An entry is one of:+///+/// * `align { ptr }` or `align { N }`: round the running offset up to the target+/// pointer size, or to `N` bytes. Alignment is *always* explicit: it is never+/// derived from a field's type, because the layout being described here does+/// not necessarily align every field to its natural alignment, and inserting+/// padding that the layout does not actually have would silently corrupt every+/// subsequent offset.+///+/// * `field { <attrs> <name>: <ty> }`: a single field.+///+/// * `array { <attrs> <name>[<count>; <IndexType>]: <ty> }` (`dynamic` only): an+/// array of `self.<count>` elements, indexed by `<IndexType>`.+///+/// * `optional { <attrs> <name>[if <flag>]: <ty> }` (`dynamic` only): a field that+/// is present when `self.<flag>` is true and absent (zero-sized) otherwise.+///+/// A field's type is always the last thing in its entry. Consumers capture it+/// as a trailing `$($fty:tt)*` and re-dispatch on its tokens only where they+/// actually need to classify it into a size or a Cranelift type; a `:ty`+/// capture would be opaque forever, and so could never be classified at+/// all.+///+/// `<attrs>` is a possibly-empty sequence of marker attributes, which consumers+/// match with `$(# $fattr:tt)*`. They do not affect the layout, but they do affect+/// the accesses generated for the field:+///+/// * `#[aggregate]`: this field is a composite (a nested struct, or an array of+/// them) rather than a single scalar. Compiled code accesses such a field's+/// interior piecewise, so there is no one Cranelift type for the field as a+/// whole and no alias-region accessor is generated for it. Its offset is still+/// generated, since that is what interior accesses are computed relative to.+///+/// * `#[readonly]` and/or `#[can_move]`: describe how Cranelift may treat loads+/// and stores of this field.+///+/// * `#[access_as = Type]`: this field is *declared* with one type (because that+/// is what determines its size and stride) but *accessed* as another. For+/// example, the component context's `may_leave` flags are each stored in a+/// whole `VMGlobalDefinition` but only ever accessed as a `u32`.+///+/// Doc comments are deliberately *not* accepted on fields; use `//` comments for+/// prose about the layout. Accessor documentation is synthesized from the field+/// names instead, so that there is only one place a field can be described.+///+/// A consumer that only cares about one of the two types can filter with a+/// literal-name arm followed by a catch-all, e.g.+///+/// ```ignore+/// (@one VMContext $snake:ident dynamic { $($dyn:tt)* }) => { ...generate... };+/// (@one $other:ident $snake:ident dynamic { $($dyn:tt)* }) => {};+/// ```+#[macro_export]+macro_rules! for_each_vmctx_type {+ ($mac:ident) => {+ $mac! {+ {+ VMContext vmctx++ // Fixed-width data comes first so that the calculation of these+ // fields' offsets is a compile-time constant when using+ // `HostPtr`.+ static {+ field { #[readonly] #[can_move] magic: u32 }++ // NB: this is where the four bytes of padding after `magic`+ // live on targets with eight-byte pointers.+ align { ptr }++ field { #[readonly] #[can_move] store_context: VmPtr<VMStoreContext> }++ field { #[readonly] #[can_move] builtin_functions: VmPtr<VMBuiltinFunctionsArray> }++ field { epoch_ptr: VmPtr<AtomicU64> }++ // A pointer that different collectors use however they see+ // fit.+ field { #[readonly] #[can_move] gc_heap_data: VmPtr<u8> }++ field { #[readonly] #[can_move] type_ids: VmPtr<VMSharedTypeIndex> }+ }++ // Variable-width fields come after the fixed-width fields+ // above. Memory-related items are placed first as they are some+ // of the most frequently accessed items, and minimizing their+ // offset can shrink the size of load/store instruction offset+ // immediates on platforms like x64 and Pulley (e.g. fit in an+ // 8-bit offset instead of needing a 32-bit offset).+ dynamic {+ array {+ #[aggregate]+ imported_memories[num_imported_memories; MemoryIndex]: VMMemoryImport+ }++ array {+ #[readonly]+ #[can_move]+ memories[num_defined_memories; DefinedMemoryIndex]: VmPtr<VMMemoryDefinition>+ }++ array {+ #[aggregate]+ owned_memories[num_owned_memories; OwnedMemoryIndex]: VMMemoryDefinition+ }++ array {+ #[aggregate]+ imported_functions[num_imported_functions; FuncIndex]: VMFunctionImport+ }++ array {+ #[aggregate]+ imported_tables[num_imported_tables; TableIndex]: VMTableImport+ }++ array {+ #[aggregate]+ imported_globals[num_imported_globals; GlobalIndex]: VMGlobalImport+ }++ array {+ #[aggregate]+ imported_tags[num_imported_tags; TagIndex]: VMTagImport+ }++ array {+ #[aggregate]+ tables[num_defined_tables; DefinedTableIndex]: VMTableDefinition+ }++ align { 16 }++ array {+ #[aggregate]+ globals[num_defined_globals; DefinedGlobalIndex]: VMGlobalDefinition+ }++ array {+ #[aggregate]+ tags[num_defined_tags; DefinedTagIndex]: VMTagDefinition+ }++ array {+ #[aggregate]+ func_refs[num_escaped_funcs; FuncRefIndex]: VMFuncRef+ }++ optional {+ #[aggregate]+ startup_func_ref[if has_startup_func]: VMFuncRef+ }++ array {+ runtime_data_bases[num_runtime_data; RuntimeDataIndex]: VmPtr<u8>+ }++ array {+ runtime_data_lengths[num_runtime_data; RuntimeDataIndex]: u32+ }+ }+ }++ {+ VMComponentContext vmcomponent++ static {+ // NB: `magic` must be at offset zero; this is relied upon by+ // `VMComponentContext::from_opaque`.+ field { #[readonly] #[can_move] magic: u32 }++ align { ptr }++ field { #[readonly] builtins: VmPtr<VMComponentBuiltins> }++ field { #[readonly] #[can_move] store_context: VmPtr<VMStoreContext> }+ }++ dynamic {+ align { 16 }++ // Each of these flags gets a whole `VMGlobalDefinition`'s+ // worth of space, but only its first four bytes are ever+ // accessed.+ array {+ #[access_as = u32]+ may_leave[num_runtime_component_instances; RuntimeComponentInstanceIndex]: VMGlobalDefinition+ }++ field { #[access_as = u32] task_may_block: VMGlobalDefinition }++ align { ptr }++ array {+ #[aggregate]+ trampoline_func_refs[num_trampolines; TrampolineIndex]: VMFuncRef+ }++ array {+ #[aggregate]+ intrinsic_func_refs[num_unsafe_intrinsics; UnsafeIntrinsic]: VMFuncRef+ }++ array {+ #[aggregate]+ lowerings[num_lowerings; LoweredIndex]: VMLowering+ }++ array {+ memories+ [num_runtime_memories; RuntimeMemoryIndex]: VmPtr<VMMemoryDefinition>+ }++ array {+ #[aggregate]+ tables[num_runtime_tables; RuntimeTableIndex]: VMTableImport+ }++ array {+ reallocs[num_runtime_reallocs; RuntimeReallocIndex]: VmPtr<VMFuncRef>+ }++ array {+ callbacks[num_runtime_callbacks; RuntimeCallbackIndex]: VmPtr<VMFuncRef>+ }++ array {+ post_returns[num_runtime_post_returns; RuntimePostReturnIndex]: VmPtr<VMFuncRef>+ }++ array {+ #[readonly]+ resource_destructors[num_resources; ResourceIndex]: VmPtr<VMFuncRef>+ }+ }+ }+ }+ };+}++/// Round `offset` up to a multiple of `align`.+#[inline]+pub(crate) fn align_up(offset: u32, align: u32) -> u32 {+ debug_assert!(align.is_power_of_two());+ (offset + (align - 1)) & !(align - 1)+}++/// Add two offsets, panicking on overflow.+#[inline]+pub(crate) fn cadd(a: u32, b: u32) -> u32 {+ a.checked_add(b).unwrap()+}++/// Multiply an element count by an element size, panicking on overflow.+#[inline]+pub(crate) fn cmul(count: u32, size: u32) -> u32 {+ count.checked_mul(size).unwrap()+}++/// Classify one of the field types accepted by `for_each_vmctx_type!` to its+/// size in bytes as a `u32`, given a `PtrSize`.+#[allow(+ unused_macro_rules,+ reason = "some element types only appear in `VMComponentContext`, and so are \+ unused when the `component-model` feature is disabled"+)]+macro_rules! vmctx_field_size {+ (($p:expr) u32) => {+ 4u32+ };+ (($p:expr) VmPtr < $g:ident >) => {+ u32::from($p)+ };+ // `VMLowering` is a pair of pointers, and is not itself defined by+ // `for_each_vm_type!`.+ (($p:expr) VMLowering) => {+ 2u32 * u32::from($p)+ };+ // Anything else names one of the `VM*` types, and so has an+ // `offsets::VMFoo` generated for it by `for_each_vm_type!`. Deferring to+ // that keeps this macro from having to mirror the list of `VM*` types, and+ // a type that has no such entry fails to resolve here.+ (($p:expr) $Name:ident) => {+ u32::from(crate::vmoffsets::offsets::$Name($p).size())+ };+}++/// Classify a `for_each_vmctx_type!` alignment step to its alignment in bytes+/// as a `u32`, given a `PtrSize`.+macro_rules! vmctx_align_value {+ (($p:expr) ptr) => {+ u32::from($p)+ };+ (($p:expr) $n:literal) => {+ $n+ };+}++/// Generate the accessors for, and the layout computation of, the+/// dynamically-positioned fields of one of the vmctx types.+#[allow(+ unused_macro_rules,+ reason = "single dynamically-positioned fields only appear in \+ `VMComponentContext`, and so are unused when the `component-model` \+ feature is disabled"+)]+macro_rules! define_vmctx_dynamic_offsets {+ (@accessors ($s:ident) [ $($kind:ident $entry:tt)* ]) => {+ $( define_vmctx_dynamic_offsets!(@accessor ($s) $kind $entry); )*+ };++ (@accessor ($s:ident) align { $al:tt }) => {};++ (@accessor ($s:ident) field { $(# $fattr:tt)* $fname:ident : $($fty:tt)* }) => {+ #[doc = concat!("The offset of the `", stringify!($fname), "` field.")]+ #[inline]+ pub fn $fname(&$s) -> u32 {+ $s.$fname+ }+ };++ (@accessor ($s:ident) array {+ $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*+ }) => {+ #[doc = concat!("The offsets of the `", stringify!($fname), "` array.")]+ #[inline]+ pub fn $fname(&$s) -> $crate::ArrayOffsets<$Index> {+ $crate::ArrayOffsets::new(+ $s.$fname,+ vmctx_field_size!(($s.ptr.size()) $($fty)*),+ $s.$count,+ )+ }+ };++ (@accessor ($s:ident) optional {+ $(# $fattr:tt)* $fname:ident [ if $flag:ident ] : $($fty:tt)*+ }) => {+ #[doc = concat!(+ "The offset of the `", stringify!($fname), "` field.\n\n",+ "Panics if `", stringify!($flag), "` is false, in which case this \+ field is not present at all."+ )]+ #[inline]+ pub fn $fname(&$s) -> u32 {+ assert!($s.$flag);+ $s.$fname+ }+ };++ (@compute_fn ($s:ident, $next:ident) $snake:ident [ $($kind:ident $entry:tt)* ]) => {+ /// Compute the offset of each dynamically-positioned field, and this+ /// vmctx's total size.+ fn compute_field_offsets(&mut $s) {+ let mut $next = u32::from($s.ptr.$snake().end_of_static_fields());+ $( define_vmctx_dynamic_offsets!(@compute ($s, $next) $kind $entry); )*+ $s.size = $next;+ }+ };++ (@compute ($s:ident, $next:ident) align { $al:tt }) => {+ $next = crate::vmctxtypes::align_up($next, vmctx_align_value!(($s.ptr.size()) $al));+ };++ (@compute ($s:ident, $next:ident) field {+ $(# $fattr:tt)* $fname:ident : $($fty:tt)*+ }) => {+ $s.$fname = $next;+ $next = crate::vmctxtypes::cadd($next, vmctx_field_size!(($s.ptr.size()) $($fty)*));+ };++ (@compute ($s:ident, $next:ident) array {+ $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*+ }) => {+ $s.$fname = $next;+ $next = crate::vmctxtypes::cadd(+ $next,+ crate::vmctxtypes::cmul($s.$count, vmctx_field_size!(($s.ptr.size()) $($fty)*)),+ );+ };++ (@compute ($s:ident, $next:ident) optional {+ $(# $fattr:tt)* $fname:ident [ if $flag:ident ] : $($fty:tt)*+ }) => {+ $s.$fname = $next;+ $next = crate::vmctxtypes::cadd(+ $next,+ if $s.$flag {+ vmctx_field_size!(($s.ptr.size()) $($fty)*)+ } else {+ 0+ },+ );+ };+}++/// The offsets of one array field within a vmctx.+#[derive(Debug, Clone, Copy)]+pub struct ArrayOffsets<I> {+ begin: u32,+ stride: u32,+ count: u32,+ _index: core::marker::PhantomData<I>,+}++impl<I> ArrayOffsets<I> {+ /// Create the offsets for an array of `count` elements which begins at+ /// `begin` within its vmctx and whose elements are `stride` bytes apart.+ #[inline]+ pub fn new(begin: u32, stride: u32, count: u32) -> Self {+ ArrayOffsets {+ begin,+ stride,+ count,+ _index: core::marker::PhantomData,+ }+ }++ /// The offset of the start of this array within its vmctx.+ #[inline]+ pub fn begin(&self) -> u32 {+ self.begin+ }++ /// The number of bytes between the start of consecutive elements of this+ /// array.+ #[inline]+ pub fn stride(&self) -> u32 {+ self.stride+ }++ /// The number of elements in this array.+ #[inline]+ pub fn count(&self) -> u32 {+ self.count+ }+}++impl<I: VmctxArrayIndex> ArrayOffsets<I> {+ /// The offset of the given element within this array's vmctx.+ ///+ /// Panics if `index` is out of bounds for this array.+ #[inline]+ pub fn at(&self, index: I) -> u32 {+ let index = index.vmctx_array_index();+ assert!(index < self.count);+ self.begin + index * self.stride+ }+}++/// An index type that can be used to index one of a vmctx's arrays.+pub trait VmctxArrayIndex: Copy {+ /// This index's position within its array.+ fn vmctx_array_index(self) -> u32;+}++/// Implement `VmctxArrayIndex` for entity references.+macro_rules! impl_vmctx_array_index {+ ($($ty:ty),* $(,)?) => {+ $(+ impl $crate::VmctxArrayIndex for $ty {+ #[inline]+ fn vmctx_array_index(self) -> u32 {+ self.as_u32()+ }+ }+ )*+ };+}diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex 79f7bda603..f4f6b33b14 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -1,47 +1,16 @@//! Offsets and sizes of various structs in `wasmtime::runtime::vm::*` that are//! accessed directly by compiled Wasm code.-// Currently the `VMContext` allocation by field looks like this:-//-// struct VMContext {-// // Fixed-width data comes first so the calculation of the offset of-// // these fields is a compile-time constant when using `HostPtr`.-// magic: u32,-// _padding: u32, // (On 64-bit systems)-// vm_store_context: *const VMStoreContext,-// builtin_functions: *mut VMBuiltinFunctionsArray,-// epoch_ptr: *mut AtomicU64,-// gc_heap_data: *mut T, // Collector-specific pointer-// type_ids: *const VMSharedTypeIndex,-//-// // Variable-width fields come after the fixed-width fields above. Place-// // memory-related items first as they're some of the most frequently-// // accessed items and minimizing their offset in this structure can-// // shrink the size of load/store instruction offset immediates on-// // platforms like x64 and Pulley (e.g. fit in an 8-bit offset instead-// // of needing a 32-bit offset)-// imported_memories: [VMMemoryImport; module.num_imported_memories],-// memories: [*mut VMMemoryDefinition; module.num_defined_memories],-// owned_memories: [VMMemoryDefinition; module.num_owned_memories],-// imported_functions: [VMFunctionImport; module.num_imported_functions],-// imported_tables: [VMTableImport; module.num_imported_tables],-// imported_globals: [VMGlobalImport; module.num_imported_globals],-// imported_tags: [VMTagImport; module.num_imported_tags],-// tables: [VMTableDefinition; module.num_defined_tables],-// globals: [VMGlobalDefinition; module.num_defined_globals],-// tags: [VMTagDefinition; module.num_defined_tags],-// func_refs: [VMFuncRef; module.num_escaped_funcs],-// startup_func_ref: [VMFuncRef; module.has_startup_func ? 1 : 0],-// runtime_data_bases: [*const u8; module.num_runtime_data],-// runtime_data_lengths: [u32; module.num_runtime_data],-// }+// The `VMContext` layout is not defined here: it is defined once, alongside+// `VMComponentContext`'s, in `for_each_vmctx_type!`. Everything in this module+// is either generated from that definition or is an offset that is not simply+// the offset of one of the layout's fields.use crate::{DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, DefinedTagIndex, FuncIndex,FuncRefIndex, GlobalIndex, MemoryIndex, Module, OwnedMemoryIndex, RuntimeDataIndex, TableIndex,TagIndex,};-use cranelift_entity::packed_option::ReservedValue;/// Number of slots in for `component_context` in the `VMStoreContext`. This is/// defined by the component model's `context.{get,set}` intrinsics.@@ -243,25 +212,87 @@ macro_rules! define_vm_type_offsets {$($body:tt)*})* ) => {- /// Offsets of fields within the various `VM*` types, parameterized over- /// a target [`PtrSize`] so that they can be computed during cross- /// compilation.- ///- /// These types are namespaced within their own module so that they never- /// collide with the real definitions of the `VM*` types themselves.- pub mod offsets {- use super::{align, PtrSize, NUM_COMPONENT_CONTEXT_SLOTS};+ $(+ #[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")]+ pub struct $Name<P: PtrSize>(pub P);- $(- #[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")]- pub struct $Name<P: PtrSize>(pub P);+ define_vm_type_offsets!(@impl $Name [$($repr)*] {} $($body)*);+ )*+ };+}++/// Generate a `struct VMContext<P: PtrSize>(P)`-style wrapper for each vmctx+/// type, with a method per statically-positioned field returning that field's+/// offset.+macro_rules! define_vmctx_static_offsets {+ // Munch the `static` section, threading a closed-form expression for the+ // running offset.+ (@chain $p:ident [ $($prev:tt)* ] []) => {+ /// The offset just past this type's last statically-positioned field.+ ///+ /// Everything after this point is dynamically sized.+ #[inline]+ pub fn end_of_static_fields(&self) -> u8 {+ let $p = self.0.size();+ let _ = $p;+ u8::try_from($($prev)*).unwrap()+ }+ };+ (@chain $p:ident [ $($prev:tt)* ] [ align { $al:tt } $($rest:tt)* ]) => {+ define_vmctx_static_offsets!(+ @chain $p+ [ crate::vmctxtypes::align_up($($prev)*, vmctx_align_value!(($p) $al)) ]+ [ $($rest)* ]+ );+ };+ (@chain $p:ident [ $($prev:tt)* ] [+ field { $(# $fattr:tt)* $fname:ident : $($fty:tt)* } $($rest:tt)*+ ]) => {+ #[doc = concat!("The offset of the `", stringify!($fname), "` field.")]+ #[inline]+ pub fn $fname(&self) -> u8 {+ let $p = self.0.size();+ let _ = $p;+ u8::try_from($($prev)*).unwrap()+ }+ define_vmctx_static_offsets!(+ @chain $p+ [ $($prev)* + vmctx_field_size!(($p) $($fty)*) ]+ [ $($rest)* ]+ );+ };- define_vm_type_offsets!(@impl $Name [$($repr)*] {} $($body)*);- )*+ ( $(+ {+ $Name:ident $snake:ident+ static { $($stat:tt)* }+ dynamic { $($dyn:tt)* }}+ )* ) => {+ $(+ #[doc = concat!("Offsets of the statically-positioned fields within the `",+ stringify!($Name), "` type.")]+ pub struct $Name<P: PtrSize>(pub P);++ impl<P: PtrSize> $Name<P> {+ define_vmctx_static_offsets!(@chain ptr [ 0u32 ] [ $($stat)* ]);+ }+ )*};}-for_each_vm_type!(define_vm_type_offsets);++/// Offsets of fields within the various `VM*` types and within the two vmctx+/// types, parameterized over a target `PtrSize` so that they can be computed+/// during cross compilation.+///+/// These types are namespaced within their own module so that they never collide+/// with the real definitions of the `VM*` types themselves.+pub mod offsets {+ use super::{NUM_COMPONENT_CONTEXT_SLOTS, PtrSize, align};++ for_each_vm_type!(define_vm_type_offsets);+ for_each_vmctx_type!(define_vmctx_static_offsets);+}/// The size, in bytes, of one `context.{get,set}` slot. These slots are `u32`s,/// both in `VMStoreContext::component_context` and in@@ -303,9 +334,8 @@ impl<P: PtrSize> offsets::VMDeferredThread<P> {}}-/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to [`PtrSize`] for-/// each `VM*` type, using the `#[snake_name = ...]` attribute for the method-/// name.+/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to `PtrSize` for+/// each `VM*` type.macro_rules! define_ptr_size_vm_type_accessors {( $($(#[doc = $sdoc:literal])*@@ -313,8 +343,6 @@ macro_rules! define_ptr_size_vm_type_accessors {#[repr($($repr:tt)*)]#[snake_name = $snake:ident]$svis:vis struct $Name:ident {- // This macro only needs each type's name and snake name, so the- // body is captured raw rather than parsed into fields.$($body:tt)*})* ) => {@@ -328,6 +356,27 @@ macro_rules! define_ptr_size_vm_type_accessors {};}+/// Add a `fn vmctx(&self) -> offsets::VMContext<&Self>` accessor to `PtrSize`+/// for each vmctx type.+macro_rules! define_ptr_size_vmctx_type_accessors {+ ( $(+ {+ $Name:ident $snake:ident+ static { $($stat:tt)* }+ dynamic { $($dyn:tt)* }+ }+ )* ) => {+ $(+ #[doc = concat!("Get the [`offsets::", stringify!($Name),+ "`] offsets for this pointer size.")]+ #[inline]+ fn $snake(&self) -> offsets::$Name<&Self> {+ offsets::$Name(self)+ }+ )*+ };+}+/// This class computes offsets to fields within `VMContext` and other/// related structs that JIT code accesses directly.#[derive(Debug, Clone, Copy)]@@ -362,18 +411,18 @@ pub struct VMOffsets<P> {/// Whether or not the module has a start function.pub has_startup_func: bool,- // precalculated offsets of various member fields+ // Precalculated offsets of the dynamically-positioned fields.+ imported_memories: u32,+ memories: u32,+ owned_memories: u32,imported_functions: u32,imported_tables: u32,- imported_memories: u32,imported_globals: u32,imported_tags: u32,- defined_tables: u32,- defined_memories: u32,- owned_memories: u32,- defined_globals: u32,- defined_tags: u32,- defined_func_refs: u32,+ tables: u32,+ globals: u32,+ tags: u32,+ func_refs: u32,startup_func_ref: u32,runtime_data_bases: u32,runtime_data_lengths: u32,@@ -386,23 +435,12 @@ pub trait PtrSize {fn size(&self) -> u8;// Generate a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor for each- // `VM*` type, giving access to that type's field offsets, size, and- // alignment for this pointer size.+ // `VM*` type.for_each_vm_type!(define_ptr_size_vm_type_accessors);- /// The offset of the `VMContext::store_context` field- fn vmcontext_store_context(&self) -> u8 {- u8::try_from(align(- u32::try_from(core::mem::size_of::<u32>()).unwrap(),- u32::from(self.size()),- ))- .unwrap()- }-- /// The offset of the `VMContext::builtin_functions` field- fn vmcontext_builtin_functions(&self) -> u8 {- self.vmcontext_store_context() + self.size()- }+ // Generate a `fn vmctx(&self) -> offsets::VMContext<&Self>` accessor for+ // each vmctx type.+ for_each_vmctx_type!(define_ptr_size_vmctx_type_accessors);/// Return the size of `VMSharedTypeIndex`.#[inline]@@ -587,43 +625,6 @@ pub trait PtrSize {self.vmcontref_args() + self.size_of_vmhostarray()}- /// Return the offset to the `magic` value in this `VMContext`.- #[inline]- fn vmctx_magic(&self) -> u8 {- // This is required by the implementation of `VMContext::instance` and- // `VMContext::instance_mut`. If this value changes then those locations- // need to be updated.- 0- }-- /// Return the offset to the `VMStoreContext` structure- #[inline]- fn vmctx_store_context(&self) -> u8 {- self.vmctx_magic() + self.size()- }-- /// Return the offset to the `VMBuiltinFunctionsArray` structure- #[inline]- fn vmctx_builtin_functions(&self) -> u8 {- self.vmctx_store_context() + self.size()- }-- /// Return the offset to the `*const AtomicU64` epoch-counter- /// pointer.- #[inline]- fn vmctx_epoch_ptr(&self) -> u8 {- self.vmctx_builtin_functions() + self.size()- }-- /// Return the offset to the `*mut T` collector-specific data.- ///- /// This is a pointer that different collectors can use however they see- /// fit.- #[inline]- fn vmctx_gc_heap_data(&self) -> u8 {- self.vmctx_epoch_ptr() + self.size()- }-/// Return the offset of the `over_approximated_stack_roots` field within/// `VMDrcHeapData`.#[inline]@@ -682,20 +683,6 @@ pub trait PtrSize {fn align_of_vmcopying_heap_data(&self) -> u8 {4}-- /// The offset of the `type_ids` array pointer.- #[inline]- fn vmctx_type_ids_array(&self) -> u8 {- self.vmctx_gc_heap_data() + self.size()- }-- /// The end of statically known offsets in `VMContext`.- ///- /// Data after this is dynamically sized.- #[inline]- fn vmctx_dynamic_data_start(&self) -> u8 {- self.vmctx_type_ids_array() + self.size()- }}/// A trait to abstract over various types that contain a `P: PtrSize`.@@ -871,16 +858,16 @@ impl<P: PtrSize> VMOffsets<P> {runtime_data_lengths: "runtime data lengths",runtime_data_bases: "runtime data base pointers",startup_func_ref: "startup funcref",- defined_func_refs: "module functions",- defined_tags: "defined tags",- defined_globals: "defined globals",- defined_tables: "defined tables",+ func_refs: "module functions",+ tags: "defined tags",+ globals: "defined globals",+ tables: "defined tables",imported_tags: "imported tags",imported_globals: "imported globals",imported_tables: "imported tables",imported_functions: "imported functions",owned_memories: "owned memories",- defined_memories: "defined memories",+ memories: "defined memories",imported_memories: "imported memories",}}@@ -912,90 +899,24 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {num_escaped_funcs: fields.num_escaped_funcs,num_runtime_data: fields.num_runtime_data,has_startup_func: fields.has_startup_func,+ imported_memories: 0,+ memories: 0,+ owned_memories: 0,imported_functions: 0,imported_tables: 0,- imported_memories: 0,imported_globals: 0,imported_tags: 0,- defined_tables: 0,- defined_memories: 0,- owned_memories: 0,- defined_globals: 0,- defined_tags: 0,- defined_func_refs: 0,+ tables: 0,+ globals: 0,+ tags: 0,+ func_refs: 0,startup_func_ref: 0,runtime_data_bases: 0,runtime_data_lengths: 0,size: 0,};-- // Convenience functions for checked addition and multiplication.- // As side effect this reduces binary size by using only a single- // `#[track_caller]` location for each function instead of one for- // each individual invocation.- #[inline]- fn cadd(count: u32, size: u32) -> u32 {- count.checked_add(size).unwrap()- }-- #[inline]- fn cmul(count: u32, size: u8) -> u32 {- count.checked_mul(u32::from(size)).unwrap()- }-- let mut next_field_offset = u32::from(ret.ptr.vmctx_dynamic_data_start());-- macro_rules! fields {- (size($field:ident) = $size:expr, $($rest:tt)*) => {- ret.$field = next_field_offset;- next_field_offset = cadd(next_field_offset, u32::from($size));- fields!($($rest)*);- };- (align($align:expr), $($rest:tt)*) => {- next_field_offset = align(next_field_offset, $align);- fields!($($rest)*);- };- () => {};- }-- fields! {- size(imported_memories)- = cmul(ret.num_imported_memories, ret.ptr.vm_memory_import().size()),- size(defined_memories)- = cmul(ret.num_defined_memories, ret.ptr.size_of_vmmemory_pointer()),- size(owned_memories)- = cmul(ret.num_owned_memories, ret.ptr.vm_memory_definition().size()),- size(imported_functions)- = cmul(ret.num_imported_functions, ret.ptr.vm_function_import().size()),- size(imported_tables)- = cmul(ret.num_imported_tables, ret.ptr.vm_table_import().size()),- size(imported_globals)- = cmul(ret.num_imported_globals, ret.ptr.vm_global_import().size()),- size(imported_tags)- = cmul(ret.num_imported_tags, ret.ptr.vm_tag_import().size()),- size(defined_tables)- = cmul(ret.num_defined_tables, ret.ptr.vm_table_definition().size()),- align(16),- size(defined_globals)- = cmul(ret.num_defined_globals, ret.ptr.vm_global_definition().size()),- size(defined_tags)- = cmul(ret.num_defined_tags, ret.ptr.vm_tag_definition().size()),- size(defined_func_refs) = cmul(- ret.num_escaped_funcs,- ret.ptr.vm_func_ref().size(),- ),- size(startup_func_ref) = if ret.has_startup_func {- ret.ptr.vm_func_ref().size()- } else {- 0- },- size(runtime_data_bases) = cmul(ret.num_runtime_data, ret.ptr.size()),- size(runtime_data_lengths) = cmul(ret.num_runtime_data, 4),- }-- ret.size = next_field_offset;-- return ret;+ ret.compute_field_offsets();+ ret}}@@ -1025,286 +946,50 @@ impl<P: PtrSize> VMOffsets<P> {}}-/// Offsets for `VMContext`.-impl<P: PtrSize> VMOffsets<P> {- /// The offset of the `tables` array.- #[inline]- pub fn vmctx_imported_functions_begin(&self) -> u32 {- self.imported_functions- }-- /// The offset of the `tables` array.- #[inline]- pub fn vmctx_imported_tables_begin(&self) -> u32 {- self.imported_tables- }-- /// The offset of the `memories` array.- #[inline]- pub fn vmctx_imported_memories_begin(&self) -> u32 {- self.imported_memories- }-- /// The offset of the `globals` array.- #[inline]- pub fn vmctx_imported_globals_begin(&self) -> u32 {- self.imported_globals- }-- /// The offset of the `tags` array.- #[inline]- pub fn vmctx_imported_tags_begin(&self) -> u32 {- self.imported_tags- }-- /// The offset of the `tables` array.- #[inline]- pub fn vmctx_tables_begin(&self) -> u32 {- self.defined_tables- }-- /// The offset of the `memories` array.- #[inline]- pub fn vmctx_memories_begin(&self) -> u32 {- self.defined_memories- }-- /// The offset of the `owned_memories` array.- #[inline]- pub fn vmctx_owned_memories_begin(&self) -> u32 {- self.owned_memories- }-- /// The offset of the `globals` array.- #[inline]- pub fn vmctx_globals_begin(&self) -> u32 {- self.defined_globals- }-- /// The offset of the `tags` array.- #[inline]- pub fn vmctx_tags_begin(&self) -> u32 {- self.defined_tags- }-- /// The offset of the `func_refs` array.- #[inline]- pub fn vmctx_func_refs_begin(&self) -> u32 {- self.defined_func_refs- }-- /// The offset of the `runtime_data_bases` array.- #[inline]- pub fn vmctx_runtime_data_bases_begin(&self) -> u32 {- self.runtime_data_bases- }-- /// The offset of the `runtime_data_lengths` array.- #[inline]- pub fn vmctx_runtime_data_lengths_begin(&self) -> u32 {- self.runtime_data_lengths- }-- /// Return the size of the `VMContext` allocation.- #[inline]- pub fn size_of_vmctx(&self) -> u32 {- self.size- }-- /// Return the offset to `VMFunctionImport` index `index`.- #[inline]- pub fn vmctx_vmfunction_import(&self, index: FuncIndex) -> u32 {- assert!(index.as_u32() < self.num_imported_functions);- self.vmctx_imported_functions_begin()- + index.as_u32() * u32::from(self.ptr.vm_function_import().size())- }-- /// Return the offset to `VMTable` index `index`.- #[inline]- pub fn vmctx_vmtable_import(&self, index: TableIndex) -> u32 {- assert!(index.as_u32() < self.num_imported_tables);- self.vmctx_imported_tables_begin()- + index.as_u32() * u32::from(self.ptr.vm_table_import().size())- }-- /// Return the offset to `VMMemoryImport` index `index`.- #[inline]- pub fn vmctx_vmmemory_import(&self, index: MemoryIndex) -> u32 {- assert!(index.as_u32() < self.num_imported_memories);- self.vmctx_imported_memories_begin()- + index.as_u32() * u32::from(self.ptr.vm_memory_import().size())- }-- /// Return the offset to `VMGlobalImport` index `index`.- #[inline]- pub fn vmctx_vmglobal_import(&self, index: GlobalIndex) -> u32 {- assert!(index.as_u32() < self.num_imported_globals);- self.vmctx_imported_globals_begin()- + index.as_u32() * u32::from(self.ptr.vm_global_import().size())- }-- /// Return the offset to `VMTagImport` index `index`.- #[inline]- pub fn vmctx_vmtag_import(&self, index: TagIndex) -> u32 {- assert!(index.as_u32() < self.num_imported_tags);- self.vmctx_imported_tags_begin()- + index.as_u32() * u32::from(self.ptr.vm_tag_import().size())- }-- /// Return the offset to `VMTableDefinition` index `index`.- #[inline]- pub fn vmctx_vmtable_definition(&self, index: DefinedTableIndex) -> u32 {- assert!(index.as_u32() < self.num_defined_tables);- self.vmctx_tables_begin()- + index.as_u32() * u32::from(self.ptr.vm_table_definition().size())- }-- /// Return the offset to the `*mut VMMemoryDefinition` at index `index`.- #[inline]- pub fn vmctx_vmmemory_pointer(&self, index: DefinedMemoryIndex) -> u32 {- assert!(index.as_u32() < self.num_defined_memories);- self.vmctx_memories_begin()- + index.as_u32() * u32::from(self.ptr.size_of_vmmemory_pointer())- }-- /// Return the offset to the owned `VMMemoryDefinition` at index `index`.- #[inline]- pub fn vmctx_vmmemory_definition(&self, index: OwnedMemoryIndex) -> u32 {- assert!(index.as_u32() < self.num_owned_memories);- self.vmctx_owned_memories_begin()- + index.as_u32() * u32::from(self.ptr.vm_memory_definition().size())- }-- /// Return the offset to the `VMGlobalDefinition` index `index`.- #[inline]- pub fn vmctx_vmglobal_definition(&self, index: DefinedGlobalIndex) -> u32 {- assert!(index.as_u32() < self.num_defined_globals);- self.vmctx_globals_begin()- + index.as_u32() * u32::from(self.ptr.vm_global_definition().size())- }-- /// Return the offset to the `VMTagDefinition` index `index`.- #[inline]- pub fn vmctx_vmtag_definition(&self, index: DefinedTagIndex) -> u32 {- assert!(index.as_u32() < self.num_defined_tags);- self.vmctx_tags_begin() + index.as_u32() * u32::from(self.ptr.vm_tag_definition().size())- }-- /// Return the offset to the `VMFuncRef` for the given function- /// index (either imported or defined).- #[inline]- pub fn vmctx_func_ref(&self, index: FuncRefIndex) -> u32 {- assert!(!index.is_reserved_value());- assert!(index.as_u32() < self.num_escaped_funcs);- self.vmctx_func_refs_begin() + index.as_u32() * u32::from(self.ptr.vm_func_ref().size())- }-- /// Returns the offset to the `VMFuncRef` for the module startup function.- ///- /// Panics if this module does not have a startup function.- #[inline]- pub fn vmctx_startup_func_ref(&self) -> u32 {- assert!(self.has_startup_func);- self.startup_func_ref- }-- /// Return the offset to the base of the runtime data segment at `index`.- #[inline]- pub fn vmctx_runtime_data_base(&self, index: RuntimeDataIndex) -> u32 {- assert!(!index.is_reserved_value());- assert!(index.as_u32() < self.num_runtime_data);- self.vmctx_runtime_data_bases_begin() + index.as_u32() * u32::from(self.ptr.size())- }-- /// Return the offset to the length of the runtime data segment at `index`.- #[inline]- pub fn vmctx_runtime_data_length(&self, index: RuntimeDataIndex) -> u32 {- assert!(!index.is_reserved_value());- assert!(index.as_u32() < self.num_runtime_data);- self.vmctx_runtime_data_lengths_begin() + index.as_u32() * 4- }-- /// Return the offset to the `wasm_call` field in `*const VMFunctionBody` index `index`.- #[inline]- pub fn vmctx_vmfunction_import_wasm_call(&self, index: FuncIndex) -> u32 {- self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().wasm_call())- }-- /// Return the offset to the `array_call` field in `*const VMFunctionBody` index `index`.- #[inline]- pub fn vmctx_vmfunction_import_array_call(&self, index: FuncIndex) -> u32 {- self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().array_call())- }-- /// Return the offset to the `vmctx` field in `*const VMFunctionBody` index `index`.- #[inline]- pub fn vmctx_vmfunction_import_vmctx(&self, index: FuncIndex) -> u32 {- self.vmctx_vmfunction_import(index) + u32::from(self.ptr.vm_function_import().vmctx())- }-- /// Return the offset to the `from` field in the imported `VMTable` at index- /// `index`.- #[inline]- pub fn vmctx_vmtable_from(&self, index: TableIndex) -> u32 {- self.vmctx_vmtable_import(index) + u32::from(self.ptr.vm_table_import().from())- }-- /// Return the offset to the `base` field in `VMTableDefinition` index `index`.- #[inline]- pub fn vmctx_vmtable_definition_base(&self, index: DefinedTableIndex) -> u32 {- self.vmctx_vmtable_definition(index) + u32::from(self.ptr.vm_table_definition().base())- }-- /// Return the offset to the `current_elements` field in `VMTableDefinition` index `index`.- #[inline]- pub fn vmctx_vmtable_definition_current_elements(&self, index: DefinedTableIndex) -> u32 {- self.vmctx_vmtable_definition(index)- + u32::from(self.ptr.vm_table_definition().current_elements())- }-- /// Return the offset to the `from` field in `VMMemoryImport` index `index`.- #[inline]- pub fn vmctx_vmmemory_import_from(&self, index: MemoryIndex) -> u32 {- self.vmctx_vmmemory_import(index) + u32::from(self.ptr.vm_memory_import().from())- }-- /// Return the offset to the `base` field in `VMMemoryDefinition` index `index`.- #[inline]- pub fn vmctx_vmmemory_definition_base(&self, index: OwnedMemoryIndex) -> u32 {- self.vmctx_vmmemory_definition(index) + u32::from(self.ptr.vm_memory_definition().base())- }-- /// Return the offset to the `current_length` field in `VMMemoryDefinition` index `index`.- #[inline]- pub fn vmctx_vmmemory_definition_current_length(&self, index: OwnedMemoryIndex) -> u32 {- self.vmctx_vmmemory_definition(index)- + u32::from(self.ptr.vm_memory_definition().current_length())- }+impl_vmctx_array_index! {+ MemoryIndex,+ DefinedMemoryIndex,+ OwnedMemoryIndex,+ FuncIndex,+ TableIndex,+ DefinedTableIndex,+ GlobalIndex,+ DefinedGlobalIndex,+ TagIndex,+ DefinedTagIndex,+ FuncRefIndex,+ RuntimeDataIndex,+}- /// Return the offset to the `from` field in `VMGlobalImport` index `index`.- #[inline]- pub fn vmctx_vmglobal_import_from(&self, index: GlobalIndex) -> u32 {- self.vmctx_vmglobal_import(index) + u32::from(self.ptr.vm_global_import().from())- }+/// Generate the accessors for the offsets of `VMContext`'s+/// dynamically-positioned fields.+macro_rules! define_vmoffsets_dynamic_offsets {+ (@one VMContext $snake:ident { $($dyn:tt)* }) => {+ /// Offsets of the dynamically-positioned fields of `VMContext`.+ impl<P: PtrSize> VMOffsets<P> {+ define_vmctx_dynamic_offsets!(@accessors (self) [ $($dyn)* ]);+ define_vmctx_dynamic_offsets!(@compute_fn (self, next) $snake [ $($dyn)* ]);- /// Return the offset to the `from` field in `VMTagImport` index `index`.- #[inline]- pub fn vmctx_vmtag_import_from(&self, index: TagIndex) -> u32 {- self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().from())- }-- /// Return the offset to the `vmctx` field in `VMTagImport` index `index`.- #[inline]- pub fn vmctx_vmtag_import_vmctx(&self, index: TagIndex) -> u32 {- self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().vmctx())- }+ /// Return the size of the `VMContext` allocation.+ #[inline]+ pub fn size_of_vmctx(&self) -> u32 {+ self.size+ }+ }+ };+ (@one $other:ident $snake:ident { $($dyn:tt)* }) => {};- /// Return the offset to the `index` field in `VMTagImport` index `index`.- #[inline]- pub fn vmctx_vmtag_import_index(&self, index: TagIndex) -> u32 {- self.vmctx_vmtag_import(index) + u32::from(self.ptr.vm_tag_import().index())- }+ ( $(+ {+ $Name:ident $snake:ident+ static { $($stat:tt)* }+ dynamic { $($dyn:tt)* }+ }+ )* ) => {+ $( define_vmoffsets_dynamic_offsets!(@one $Name $snake { $($dyn)* }); )*+ };}+for_each_vmctx_type!(define_vmoffsets_dynamic_offsets);/// Offsets for `VMGcHeader`.impl<P: PtrSize> VMOffsets<P> {diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rsindex 867c1b8060..8011ddd8c0 100644--- a/crates/environ/src/vmtypes.rs+++ b/crates/environ/src/vmtypes.rs@@ -147,10 +147,14 @@ macro_rules! for_each_vm_type {#[snake_name = vm_function_import]pub struct VMFunctionImport {/// Same as `VMFuncRef::array_call`.+ #[readonly]+ #[can_move]pub array_call: VmPtr<VMArrayCallFunction>,/// Same as `VMFuncRef::wasm_call`, except always non-null. Must be filled/// in by the time Wasm is importing this function!+ #[readonly]+ #[can_move]pub wasm_call: VmPtr<VMWasmCallFunction>,/// Function signature's _actual_ type id.@@ -158,9 +162,13 @@ macro_rules! for_each_vm_type {/// This is the type that the function was defined with, not the type that/// it was imported as. These two can be different in the face of subtyping/// and we need the former for to correctly implement dynamic downcasts.+ #[readonly]+ #[can_move]pub type_index: VMSharedTypeIndex,/// Same as `VMFuncRef::vmctx`.+ #[readonly]+ #[can_move]pub vmctx: VmPtr<VMOpaqueContext>,}@@ -171,12 +179,18 @@ macro_rules! for_each_vm_type {#[snake_name = vm_table_import]pub struct VMTableImport {/// A pointer to the imported table description.+ #[readonly]+ #[can_move]pub from: VmPtr<VMTableDefinition>,/// A pointer to the `VMContext` that owns the table description.+ #[readonly]+ #[can_move]pub vmctx: VmPtr<VMContext>,/// The table index, within `vmctx`, this definition resides at.+ #[readonly]+ #[can_move]pub index: DefinedTableIndex,}@@ -187,12 +201,18 @@ macro_rules! for_each_vm_type {#[snake_name = vm_memory_import]pub struct VMMemoryImport {/// A pointer to the imported memory description.+ #[readonly]+ #[can_move]pub from: VmPtr<VMMemoryDefinition>,/// A pointer to the `VMContext` that owns the memory description.+ #[readonly]+ #[can_move]pub vmctx: VmPtr<VMContext>,/// The index of the memory in the containing `vmctx`.+ #[readonly]+ #[can_move]pub index: DefinedMemoryIndex,}@@ -207,6 +227,8 @@ macro_rules! for_each_vm_type {#[snake_name = vm_global_import]pub struct VMGlobalImport {/// A pointer to the imported global variable description.+ #[readonly]+ #[can_move]pub from: VmPtr<VMGlobalDefinition>,/// A pointer to the context that owns the global.@@ -215,10 +237,14 @@ macro_rules! for_each_vm_type {/// for `VMGlobalKind::Host`, it's a `VMContext` for/// `VMGlobalKind::Instance`, and it's `VMComponentContext` for/// `VMGlobalKind::ComponentFlags`.+ #[readonly]+ #[can_move]pub vmctx: Option<VmPtr<VMOpaqueContext>>,/// The kind of global, and extra location information in addition to/// `vmctx` above.+ #[readonly]+ #[can_move]pub kind: VMGlobalKind,}@@ -229,12 +255,18 @@ macro_rules! for_each_vm_type {#[snake_name = vm_tag_import]pub struct VMTagImport {/// A pointer to the imported tag description.+ #[readonly]+ #[can_move]pub from: VmPtr<VMTagDefinition>,/// The instance that owns this tag.+ #[readonly]+ #[can_move]pub vmctx: VmPtr<VMContext>,/// The index of the tag in the containing `vmctx`.+ #[readonly]+ #[can_move]pub index: DefinedTagIndex,}diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rsindex de21c6738b..bafaada3d7 100644--- a/crates/wasmtime/src/runtime/vm/component.rs+++ b/crates/wasmtime/src/runtime/vm/component.rs@@ -30,7 +30,7 @@ use core::ptr::NonNull;use wasmtime_environ::component::*;use wasmtime_environ::error::OutOfMemory;use wasmtime_environ::prelude::TryPrimaryMap;-use wasmtime_environ::{HostPtr, PrimaryMap, VMSharedTypeIndex};+use wasmtime_environ::{HostPtr, PrimaryMap, PtrSize as _, VMSharedTypeIndex};#[allow(clippy::cast_possible_truncation,@@ -365,8 +365,8 @@ impl ComponentInstance {#[inline]pub fn instance_flags(&self, instance: RuntimeComponentInstanceIndex) -> InstanceFlags {unsafe {- let ptr =- self.vmctx_plus_offset_raw::<VMGlobalDefinition>(self.offsets.may_leave(instance));+ let ptr = self+ .vmctx_plus_offset_raw::<VMGlobalDefinition>(self.offsets.may_leave().at(instance));InstanceFlags(SendSyncPtr::new(ptr))}}@@ -378,7 +378,7 @@ impl ComponentInstance {/// during the instantiation process of a component.pub fn runtime_memory(&self, idx: RuntimeMemoryIndex) -> NonNull<VMMemoryDefinition> {unsafe {- let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.runtime_memory(idx));+ let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.memories().at(idx));debug_assert!(ret.as_ptr() as usize != INVALID_PTR);ret.as_non_null()}@@ -391,7 +391,7 @@ impl ComponentInstance {/// during the instantiation process of a component.pub fn runtime_table(&self, idx: RuntimeTableIndex) -> VMTableImport {unsafe {- let ret = *self.vmctx_plus_offset::<VMTableImport>(self.offsets.runtime_table(idx));+ let ret = *self.vmctx_plus_offset::<VMTableImport>(self.offsets.tables().at(idx));debug_assert!(ret.from.as_ptr() as usize != INVALID_PTR);debug_assert!(ret.vmctx.as_ptr() as usize != INVALID_PTR);ret@@ -430,7 +430,7 @@ impl ComponentInstance {/// during the instantiation process of a component.pub fn runtime_realloc(&self, idx: RuntimeReallocIndex) -> NonNull<VMFuncRef> {unsafe {- let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.runtime_realloc(idx));+ let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.reallocs().at(idx));debug_assert!(ret.as_ptr() as usize != INVALID_PTR);ret.as_non_null()}@@ -442,7 +442,7 @@ impl ComponentInstance {/// during the instantiation process of a component.pub fn runtime_callback(&self, idx: RuntimeCallbackIndex) -> NonNull<VMFuncRef> {unsafe {- let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.runtime_callback(idx));+ let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.callbacks().at(idx));debug_assert!(ret.as_ptr() as usize != INVALID_PTR);ret.as_non_null()}@@ -454,7 +454,7 @@ impl ComponentInstance {/// during the instantiation process of a component.pub fn runtime_post_return(&self, idx: RuntimePostReturnIndex) -> NonNull<VMFuncRef> {unsafe {- let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.runtime_post_return(idx));+ let ret = *self.vmctx_plus_offset::<VmPtr<_>>(self.offsets.post_returns().at(idx));debug_assert!(ret.as_ptr() as usize != INVALID_PTR);ret.as_non_null()}@@ -467,7 +467,7 @@ impl ComponentInstance {/// during the instantiation process of a component.pub fn lowering(&self, idx: LoweredIndex) -> VMLowering {unsafe {- let ret = *self.vmctx_plus_offset::<VMLowering>(self.offsets.lowering(idx));+ let ret = *self.vmctx_plus_offset::<VMLowering>(self.offsets.lowerings().at(idx));debug_assert!(ret.callee.as_ptr() as usize != INVALID_PTR);debug_assert!(ret.data.as_ptr() as usize != INVALID_PTR);ret@@ -484,7 +484,7 @@ impl ComponentInstance {/// during the instantiation process of a component.pub fn trampoline_func_ref(&self, idx: TrampolineIndex) -> NonNull<VMFuncRef> {unsafe {- let offset = self.offsets.trampoline_func_ref(idx);+ let offset = self.offsets.trampoline_func_refs().at(idx);let ret = self.vmctx_plus_offset_raw::<VMFuncRef>(offset);debug_assert!(mem::transmute::<Option<VmPtr<VMWasmCallFunction>>, usize>(ret.as_ref().wasm_call)@@ -498,7 +498,7 @@ impl ComponentInstance {/// Get the core Wasm function reference for the given unsafe intrinsic.pub fn unsafe_intrinsic_func_ref(&self, idx: UnsafeIntrinsic) -> NonNull<VMFuncRef> {unsafe {- let offset = self.offsets.unsafe_intrinsic_func_ref(idx);+ let offset = self.offsets.intrinsic_func_refs().at(idx);let ret = self.vmctx_plus_offset_raw::<VMFuncRef>(offset);debug_assert!(mem::transmute::<Option<VmPtr<VMWasmCallFunction>>, usize>(ret.as_ref().wasm_call)@@ -523,7 +523,7 @@ impl ComponentInstance {ptr: NonNull<VMMemoryDefinition>,) {unsafe {- let offset = self.offsets.runtime_memory(idx);+ let offset = self.offsets.memories().at(idx);let storage = self.vmctx_plus_offset_mut::<VmPtr<VMMemoryDefinition>>(offset);debug_assert!((*storage).as_ptr() as usize == INVALID_PTR);*storage = ptr.into();@@ -537,7 +537,7 @@ impl ComponentInstance {ptr: NonNull<VMFuncRef>,) {unsafe {- let offset = self.offsets.runtime_realloc(idx);+ let offset = self.offsets.reallocs().at(idx);let storage = self.vmctx_plus_offset_mut::<VmPtr<VMFuncRef>>(offset);debug_assert!((*storage).as_ptr() as usize == INVALID_PTR);*storage = ptr.into();@@ -551,7 +551,7 @@ impl ComponentInstance {ptr: NonNull<VMFuncRef>,) {unsafe {- let offset = self.offsets.runtime_callback(idx);+ let offset = self.offsets.callbacks().at(idx);let storage = self.vmctx_plus_offset_mut::<VmPtr<VMFuncRef>>(offset);debug_assert!((*storage).as_ptr() as usize == INVALID_PTR);*storage = ptr.into();@@ -565,7 +565,7 @@ impl ComponentInstance {ptr: NonNull<VMFuncRef>,) {unsafe {- let offset = self.offsets.runtime_post_return(idx);+ let offset = self.offsets.post_returns().at(idx);let storage = self.vmctx_plus_offset_mut::<VmPtr<VMFuncRef>>(offset);debug_assert!((*storage).as_ptr() as usize == INVALID_PTR);*storage = ptr.into();@@ -582,7 +582,7 @@ impl ComponentInstance {/// here is never needed prior to it being configured here in the instance.pub fn set_runtime_table(self: Pin<&mut Self>, idx: RuntimeTableIndex, import: VMTableImport) {unsafe {- let offset = self.offsets.runtime_table(idx);+ let offset = self.offsets.tables().at(idx);let storage = self.vmctx_plus_offset_mut::<VMTableImport>(offset);debug_assert!((*storage).vmctx.as_ptr() as usize == INVALID_PTR);debug_assert!((*storage).from.as_ptr() as usize == INVALID_PTR);@@ -598,7 +598,7 @@ impl ComponentInstance {debug_assert!(*self.vmctx_plus_offset::<usize>(callee) == INVALID_PTR);let data = self.offsets.lowering_data(idx);debug_assert!(*self.vmctx_plus_offset::<usize>(data) == INVALID_PTR);- let offset = self.offsets.lowering(idx);+ let offset = self.offsets.lowerings().at(idx);*self.vmctx_plus_offset_mut(offset) = lowering;}}@@ -612,7 +612,7 @@ impl ComponentInstance {type_index: VMSharedTypeIndex,) {unsafe {- let offset = self.offsets.trampoline_func_ref(idx);+ let offset = self.offsets.trampoline_func_refs().at(idx);debug_assert!(*self.vmctx_plus_offset::<usize>(offset) == INVALID_PTR);let vmctx = VMOpaqueContext::from_vmcomponent(self.vmctx());*self.vmctx_plus_offset_mut(offset) = VMFuncRef {@@ -633,7 +633,7 @@ impl ComponentInstance {type_index: VMSharedTypeIndex,) {unsafe {- let offset = self.offsets.unsafe_intrinsic_func_ref(intrinsic);+ let offset = self.offsets.intrinsic_func_refs().at(intrinsic);debug_assert!(*self.vmctx_plus_offset::<usize>(offset) == INVALID_PTR);let vmctx = VMOpaqueContext::from_vmcomponent(self.vmctx());*self.vmctx_plus_offset_mut(offset) = VMFuncRef {@@ -655,7 +655,7 @@ impl ComponentInstance {dtor: Option<NonNull<VMFuncRef>>,) {unsafe {- let offset = self.offsets.resource_destructor(idx);+ let offset = self.offsets.resource_destructors().at(idx);debug_assert!(*self.vmctx_plus_offset::<usize>(offset) == INVALID_PTR);*self.vmctx_plus_offset_mut(offset) = dtor.map(VmPtr::from);}@@ -667,14 +667,14 @@ impl ComponentInstance {/// after instantiation.pub fn resource_destructor(&self, idx: ResourceIndex) -> Option<NonNull<VMFuncRef>> {unsafe {- let offset = self.offsets.resource_destructor(idx);+ let offset = self.offsets.resource_destructors().at(idx);debug_assert!(*self.vmctx_plus_offset::<usize>(offset) != INVALID_PTR);(*self.vmctx_plus_offset::<Option<VmPtr<VMFuncRef>>>(offset)).map(|p| p.as_non_null())}}unsafe fn initialize_vmctx(mut self: Pin<&mut Self>) {- let offset = self.offsets.magic();+ let offset = u32::from(self.offsets.ptr.vmcomponent().magic());// SAFETY: it's safe to write the magic value during initialization and// this is also the right type of value to write.unsafe {@@ -687,14 +687,14 @@ impl ComponentInstance {// is also the right type of value to store in the vmctx.static BUILTINS: libcalls::VMComponentBuiltins = libcalls::VMComponentBuiltins::INIT;let ptr = BUILTINS.expose_provenance();- let offset = self.offsets.builtins();+ let offset = u32::from(self.offsets.ptr.vmcomponent().builtins());unsafe {*self.as_mut().vmctx_plus_offset_mut(offset) = VmPtr::from(ptr);}// SAFETY: it's safe to initialize the vmctx in this function and this// is also the right type of value to store in the vmctx.- let offset = self.offsets.vm_store_context();+ let offset = u32::from(self.offsets.ptr.vmcomponent().store_context());unsafe {*self.as_mut().vmctx_plus_offset_mut(offset) =VmPtr::from(self.store.0.as_ref().vm_store_context_ptr());@@ -734,7 +734,7 @@ impl ComponentInstance {}for i in 0..self.offsets.num_trampolines {let i = TrampolineIndex::from_u32(i);- let offset = self.offsets.trampoline_func_ref(i);+ let offset = self.offsets.trampoline_func_refs().at(i);// SAFETY: see aboveunsafe {*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;@@ -742,7 +742,7 @@ impl ComponentInstance {}for i in 0..self.offsets.num_unsafe_intrinsics {let i = UnsafeIntrinsic::from_u32(i);- let offset = self.offsets.unsafe_intrinsic_func_ref(i);+ let offset = self.offsets.intrinsic_func_refs().at(i);// SAFETY: see aboveunsafe {*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;@@ -750,7 +750,7 @@ impl ComponentInstance {}for i in 0..self.offsets.num_runtime_memories {let i = RuntimeMemoryIndex::from_u32(i);- let offset = self.offsets.runtime_memory(i);+ let offset = self.offsets.memories().at(i);// SAFETY: see aboveunsafe {*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;@@ -758,7 +758,7 @@ impl ComponentInstance {}for i in 0..self.offsets.num_runtime_reallocs {let i = RuntimeReallocIndex::from_u32(i);- let offset = self.offsets.runtime_realloc(i);+ let offset = self.offsets.reallocs().at(i);// SAFETY: see aboveunsafe {*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;@@ -766,7 +766,7 @@ impl ComponentInstance {}for i in 0..self.offsets.num_runtime_callbacks {let i = RuntimeCallbackIndex::from_u32(i);- let offset = self.offsets.runtime_callback(i);+ let offset = self.offsets.callbacks().at(i);// SAFETY: see aboveunsafe {*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;@@ -774,7 +774,7 @@ impl ComponentInstance {}for i in 0..self.offsets.num_runtime_post_returns {let i = RuntimePostReturnIndex::from_u32(i);- let offset = self.offsets.runtime_post_return(i);+ let offset = self.offsets.post_returns().at(i);// SAFETY: see aboveunsafe {*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;@@ -782,7 +782,7 @@ impl ComponentInstance {}for i in 0..self.offsets.num_resources {let i = ResourceIndex::from_u32(i);- let offset = self.offsets.resource_destructor(i);+ let offset = self.offsets.resource_destructors().at(i);// SAFETY: see aboveunsafe {*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;@@ -790,7 +790,7 @@ impl ComponentInstance {}for i in 0..self.offsets.num_runtime_tables {let i = RuntimeTableIndex::from_u32(i);- let offset = self.offsets.runtime_table(i);+ let offset = self.offsets.tables().at(i);// SAFETY: see above#[allow(clippy::cast_possible_truncation, reason = "known to not overflow")]unsafe {diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rsindex 06ba7e4455..321f140def 100644--- a/crates/wasmtime/src/runtime/vm/instance.rs+++ b/crates/wasmtime/src/runtime/vm/instance.rs@@ -418,32 +418,32 @@ impl Instance {/// Return the indexed `VMFunctionImport`.fn imported_function(&self, index: FuncIndex) -> &VMFunctionImport {- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmfunction_import(index)) }+ unsafe { self.vmctx_plus_offset(self.offsets().imported_functions().at(index)) }}/// Return the index `VMTableImport`.fn imported_table(&self, index: TableIndex) -> &VMTableImport {- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtable_import(index)) }+ unsafe { self.vmctx_plus_offset(self.offsets().imported_tables().at(index)) }}/// Return the indexed `VMMemoryImport`.fn imported_memory(&self, index: MemoryIndex) -> &VMMemoryImport {- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmmemory_import(index)) }+ unsafe { self.vmctx_plus_offset(self.offsets().imported_memories().at(index)) }}/// Return the indexed `VMGlobalImport`.fn imported_global(&self, index: GlobalIndex) -> &VMGlobalImport {- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmglobal_import(index)) }+ unsafe { self.vmctx_plus_offset(self.offsets().imported_globals().at(index)) }}/// Return the indexed `VMTagImport`.fn imported_tag(&self, index: TagIndex) -> &VMTagImport {- unsafe { self.vmctx_plus_offset(self.offsets().vmctx_vmtag_import(index)) }+ unsafe { self.vmctx_plus_offset(self.offsets().imported_tags().at(index)) }}/// Return the indexed `VMTagDefinition`.pub fn tag_ptr(&self, index: DefinedTagIndex) -> NonNull<VMTagDefinition> {- unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmtag_definition(index)) }+ unsafe { self.vmctx_plus_offset_raw(self.offsets().tags().at(index)) }}/// Return the indexed `VMTableDefinition`.@@ -461,7 +461,7 @@ impl Instance {/// Return a pointer to the `index`'th table within this instance, stored/// in vmctx memory.pub fn table_ptr(&self, index: DefinedTableIndex) -> NonNull<VMTableDefinition> {- unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmtable_definition(index)) }+ unsafe { self.vmctx_plus_offset_raw(self.offsets().tables().at(index)) }}/// Get a locally defined or imported memory.@@ -496,14 +496,14 @@ impl Instance {#[inline]pub fn memory_ptr(&self, index: DefinedMemoryIndex) -> NonNull<VMMemoryDefinition> {unsafe {- self.vmctx_plus_offset::<VmPtr<_>>(self.offsets().vmctx_vmmemory_pointer(index))+ self.vmctx_plus_offset::<VmPtr<_>>(self.offsets().memories().at(index)).as_non_null()}}/// Return the indexed `VMGlobalDefinition`.pub fn global_ptr(&self, index: DefinedGlobalIndex) -> NonNull<VMGlobalDefinition> {- unsafe { self.vmctx_plus_offset_raw(self.offsets().vmctx_vmglobal_definition(index)) }+ unsafe { self.vmctx_plus_offset_raw(self.offsets().globals().at(index)) }}/// Get all globals within this instance.@@ -538,19 +538,19 @@ impl Instance {/// Return a pointer to the interrupts structure#[inline]pub fn vm_store_context(&self) -> NonNull<Option<VmPtr<VMStoreContext>>> {- unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx_store_context()) }+ unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx().store_context()) }}/// Return a pointer to the global epoch counter used by this instance.#[cfg(target_has_atomic = "64")]pub fn epoch_ptr(self: Pin<&mut Self>) -> &mut Option<VmPtr<AtomicU64>> {- let offset = self.offsets().ptr.vmctx_epoch_ptr();+ let offset = self.offsets().ptr.vmctx().epoch_ptr();unsafe { self.vmctx_plus_offset_mut(offset) }}/// Return a pointer to the collector-specific heap data.pub fn gc_heap_data(self: Pin<&mut Self>) -> &mut Option<VmPtr<u8>> {- let offset = self.offsets().ptr.vmctx_gc_heap_data();+ let offset = self.offsets().ptr.vmctx().gc_heap_data();unsafe { self.vmctx_plus_offset_mut(offset) }}@@ -841,7 +841,7 @@ impl Instance {}fn type_ids_array(&self) -> NonNull<VmPtr<VMSharedTypeIndex>> {- unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx_type_ids_array()) }+ unsafe { self.vmctx_plus_offset_raw(self.offsets().ptr.vmctx().type_ids()) }}/// Get a `&VMFuncRef` for the given `FuncIndex`.@@ -879,7 +879,7 @@ impl Instance {let index = module.func_index(def_index);let func = &module.functions[index];let type_index = func.signature.unwrap_engine_type_index();- let vmctx_offset = self.offsets().vmctx_func_ref(func.func_ref);+ let vmctx_offset = self.offsets().func_refs().at(func.func_ref);let array_to_wasm_key = FuncKey::ArrayToWasmTrampoline(module.module_index, def_index);let wasm_key = FuncKey::DefinedWasmFunction(module.module_index, def_index);// SAFETY: the type/offset/keys here are all valid for the defined@@ -906,7 +906,7 @@ impl Instance {t.unwrap_engine_type_index()}};- let vmctx_offset = self.offsets().vmctx_startup_func_ref();+ let vmctx_offset = self.offsets().startup_func_ref();let array_to_wasm_key = FuncKey::ModuleStartup(Abi::Array, module.module_index);let wasm_key = FuncKey::ModuleStartup(Abi::Wasm, module.module_index);// SAFETY: the type/offset/keys here are all valid for the module@@ -1185,7 +1185,7 @@ impl Instance {unsafe {let offsets = instance.runtime_info.offsets();instance- .vmctx_plus_offset_raw::<u32>(offsets.ptr.vmctx_magic())+ .vmctx_plus_offset_raw::<u32>(offsets.ptr.vmctx().magic()).write(VMCONTEXT_MAGIC);}@@ -1213,7 +1213,7 @@ impl Instance {let ptr = BUILTINS.expose_provenance();let offsets = instance.runtime_info.offsets();instance- .vmctx_plus_offset_raw(offsets.ptr.vmctx_builtin_functions())+ .vmctx_plus_offset_raw(offsets.ptr.vmctx().builtin_functions()).write(VmPtr::from(ptr));}@@ -1227,7 +1227,7 @@ impl Instance {ptr::copy_nonoverlapping(imports.functions.as_ptr(),instance- .vmctx_plus_offset_raw(offsets.vmctx_imported_functions_begin())+ .vmctx_plus_offset_raw(offsets.imported_functions().begin()).as_ptr(),imports.functions.len(),);@@ -1235,7 +1235,7 @@ impl Instance {ptr::copy_nonoverlapping(imports.tables.as_ptr(),instance- .vmctx_plus_offset_raw(offsets.vmctx_imported_tables_begin())+ .vmctx_plus_offset_raw(offsets.imported_tables().begin()).as_ptr(),imports.tables.len(),);@@ -1243,7 +1243,7 @@ impl Instance {ptr::copy_nonoverlapping(imports.memories.as_ptr(),instance- .vmctx_plus_offset_raw(offsets.vmctx_imported_memories_begin())+ .vmctx_plus_offset_raw(offsets.imported_memories().begin()).as_ptr(),imports.memories.len(),);@@ -1251,7 +1251,7 @@ impl Instance {ptr::copy_nonoverlapping(imports.globals.as_ptr(),instance- .vmctx_plus_offset_raw(offsets.vmctx_imported_globals_begin())+ .vmctx_plus_offset_raw(offsets.imported_globals().begin()).as_ptr(),imports.globals.len(),);@@ -1259,7 +1259,7 @@ impl Instance {ptr::copy_nonoverlapping(imports.tags.as_ptr(),instance- .vmctx_plus_offset_raw(offsets.vmctx_imported_tags_begin())+ .vmctx_plus_offset_raw(offsets.imported_tags().begin()).as_ptr(),imports.tags.len(),);@@ -1277,7 +1277,7 @@ impl Instance {// valid.unsafe {let offsets = instance.runtime_info.offsets();- let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_tables_begin());+ let mut ptr = instance.vmctx_plus_offset_raw(offsets.tables().begin());let tables = instance.as_mut().tables_mut();for i in 0..module.num_defined_tables() {ptr.write(tables[DefinedTableIndex::new(i)].1.vmtable());@@ -1296,9 +1296,8 @@ impl Instance {// valid.unsafe {let offsets = instance.runtime_info.offsets();- let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_memories_begin());- let mut owned_ptr =- instance.vmctx_plus_offset_raw(offsets.vmctx_owned_memories_begin());+ let mut ptr = instance.vmctx_plus_offset_raw(offsets.memories().begin());+ let mut owned_ptr = instance.vmctx_plus_offset_raw(offsets.owned_memories().begin());let memories = instance.as_mut().memories_mut();for i in 0..module.num_defined_memories() {let defined_memory_index = DefinedMemoryIndex::new(i);@@ -1355,7 +1354,7 @@ impl Instance {// valid.unsafe {let offsets = instance.runtime_info.offsets();- let mut ptr = instance.vmctx_plus_offset_raw(offsets.vmctx_tags_begin());+ let mut ptr = instance.vmctx_plus_offset_raw(offsets.tags().begin());for i in 0..module.num_defined_tags() {let defined_index = DefinedTagIndex::new(i);let tag_index = module.tag_index(defined_index);@@ -1375,9 +1374,8 @@ impl Instance {unsafe {let offsets = instance.runtime_info.offsets();let mut lengths =- instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_lengths_begin());- let mut bases =- instance.vmctx_plus_offset_raw(offsets.vmctx_runtime_data_bases_begin());+ instance.vmctx_plus_offset_raw(offsets.runtime_data_lengths().begin());+ let mut bases = instance.vmctx_plus_offset_raw(offsets.runtime_data_bases().begin());for i in module.runtime_data.keys() {let data = instance.runtime_data(i);lengths.write(u32::try_from(data.len()).unwrap());@@ -1408,10 +1406,10 @@ impl Instance {let offsets = instance.runtime_info.offsets();unsafe {instance- .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_length(*data))+ .vmctx_plus_offset_raw(offsets.runtime_data_lengths().at(*data)).write(0u32);instance- .vmctx_plus_offset_raw(offsets.vmctx_runtime_data_base(*data))+ .vmctx_plus_offset_raw(offsets.runtime_data_bases().at(*data)).write(0usize);}}diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex 0a9c934d1c..de5b079425 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -331,7 +331,7 @@ mod test_vmglobal_definition {fn check_vmglobal_begins_aligned() {let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);- assert_eq!(offsets.vmctx_globals_begin() % 16, 0);+ assert_eq!(offsets.globals().begin() % 16, 0);}#[test]@@ -544,7 +544,7 @@ mod test_vmtag_definition {fn check_vmtag_begins_aligned() {let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);- assert_eq!(offsets.vmctx_tags_begin() % 16, 0);+ assert_eq!(offsets.tags().begin() % 16, 0);}}@@ -969,7 +969,7 @@ impl VMContext {#[inline]pub unsafe fn from_opaque(opaque: NonNull<VMOpaqueContext>) -> NonNull<VMContext> {// Note that in general the offset of the "magic" field is stored in- // `VMOffsets::vmctx_magic`. Given though that this is a sanity check+ // `VMContext::magic`. Given though that this is a sanity check// about converting this pointer to another type we ideally don't want// to read the offset from potentially corrupt memory. Instead it would// be better to catch errors here as soon as possible.diff --git a/winch/codegen/src/codegen/call.rs b/winch/codegen/src/codegen/call.rsindex e0de2888bb..a45b8d839d 100644--- a/winch/codegen/src/codegen/call.rs+++ b/winch/codegen/src/codegen/call.rs@@ -213,11 +213,13 @@ impl FnCall {context.without::<Result<(Reg, Reg)>, M, _>(&sig.regs, masm, |context, masm| {Ok((context.any_gpr(masm)?, context.any_gpr(masm)?))})??;- let callee_vmctx_offset = vmoffsets.vmctx_vmfunction_import_vmctx(index);+ let vmimport = vmoffsets.imported_functions().at(index);+ let callee_vmctx_offset = vmimport + u32::from(vmoffsets.ptr.vm_function_import().vmctx());let callee_vmctx_addr = masm.address_at_vmctx(callee_vmctx_offset)?;masm.load_ptr(callee_vmctx_addr, writable!(callee_vmctx))?;- let callee_body_offset = vmoffsets.vmctx_vmfunction_import_wasm_call(index);+ let callee_body_offset =+ vmimport + u32::from(vmoffsets.ptr.vm_function_import().wasm_call());let callee_addr = masm.address_at_vmctx(callee_body_offset)?;masm.load_ptr(callee_addr, writable!(callee))?;diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rsindex d3eb52d321..3604ed76d6 100644--- a/winch/codegen/src/codegen/drc.rs+++ b/winch/codegen/src/codegen/drc.rs@@ -225,7 +225,7 @@ wheregc_ref: Reg,object_addr: Reg,) -> Result<(Reg, Reg)> {- let heap_data_offset = self.env.vmoffsets.ptr.vmctx_gc_heap_data();+ let heap_data_offset = self.env.vmoffsets.ptr.vmctx().gc_heap_data();let roots_head_offset = u32::from(self.env.vmoffsetsdiff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rsindex f53fdba350..07f88c120b 100644--- a/winch/codegen/src/codegen/env.rs+++ b/winch/codegen/src/codegen/env.rs@@ -215,12 +215,13 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {let ty = self.translation.module.globals[index].wasm_ty;let val = || match self.translation.module.defined_global_index(index) {Some(defined_index) => GlobalData {- offset: self.vmoffsets.vmctx_vmglobal_definition(defined_index),+ offset: self.vmoffsets.globals().at(defined_index),imported: false,ty,},None => GlobalData {- offset: self.vmoffsets.vmctx_vmglobal_import_from(index),+ offset: self.vmoffsets.imported_globals().at(index)+ + u32::from(self.vmoffsets.ptr.vm_global_import().from()),imported: true,ty,},@@ -238,12 +239,18 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {match self.translation.module.defined_table_index(index) {Some(defined) => (None,- self.vmoffsets.vmctx_vmtable_definition_base(defined),- self.vmoffsets- .vmctx_vmtable_definition_current_elements(defined),+ self.vmoffsets.tables().at(defined)+ + u32::from(self.vmoffsets.ptr.vm_table_definition().base()),+ self.vmoffsets.tables().at(defined)+ + u32::from(+ self.vmoffsets.ptr.vm_table_definition().current_elements(),+ ),),None => (- Some(self.vmoffsets.vmctx_vmtable_from(index)),+ Some(+ self.vmoffsets.imported_tables().at(index)+ + u32::from(self.vmoffsets.ptr.vm_table_import().from()),+ ),self.vmoffsets.ptr.vm_table_definition().base().into(),self.vmoffsets.ptr@@ -274,39 +281,48 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {match self.resolved_heaps.entry(index) {Occupied(entry) => *entry.get(),Vacant(entry) => {- let (import_from, base_offset, current_length_offset) =- match self.translation.module.defined_memory_index(index) {- Some(defined) => {- if is_shared {- (- Some(self.vmoffsets.vmctx_vmmemory_pointer(defined)),- self.vmoffsets.ptr.vm_memory_definition().base().into(),- self.vmoffsets- .ptr- .vm_memory_definition()- .current_length()- .into(),- )- } else {- let owned = self.translation.module.owned_memory_index(defined);- (- None,- self.vmoffsets.vmctx_vmmemory_definition_base(owned),- self.vmoffsets- .vmctx_vmmemory_definition_current_length(owned),- )- }+ let (import_from, base_offset, current_length_offset) = match self+ .translation+ .module+ .defined_memory_index(index)+ {+ Some(defined) => {+ if is_shared {+ (+ Some(self.vmoffsets.memories().at(defined)),+ self.vmoffsets.ptr.vm_memory_definition().base().into(),+ self.vmoffsets+ .ptr+ .vm_memory_definition()+ .current_length()+ .into(),+ )+ } else {+ let owned = self.translation.module.owned_memory_index(defined);+ (+ None,+ self.vmoffsets.owned_memories().at(owned)+ + u32::from(self.vmoffsets.ptr.vm_memory_definition().base()),+ self.vmoffsets.owned_memories().at(owned)+ + u32::from(+ self.vmoffsets.ptr.vm_memory_definition().current_length(),+ ),+ )}- None => (- Some(self.vmoffsets.vmctx_vmmemory_import_from(index)),- self.vmoffsets.ptr.vm_memory_definition().base().into(),- self.vmoffsets- .ptr- .vm_memory_definition()- .current_length()- .into(),+ }+ None => (+ Some(+ self.vmoffsets.imported_memories().at(index)+ + u32::from(self.vmoffsets.ptr.vm_memory_import().from()),),- };+ self.vmoffsets.ptr.vm_memory_definition().base().into(),+ self.vmoffsets+ .ptr+ .vm_memory_definition()+ .current_length()+ .into(),+ ),+ };let memory = &self.translation.module.memories[index];diff --git a/winch/codegen/src/codegen/gc.rs b/winch/codegen/src/codegen/gc.rsindex a7f7ad86d5..33b93371b2 100644--- a/winch/codegen/src/codegen/gc.rs+++ b/winch/codegen/src/codegen/gc.rs@@ -58,7 +58,7 @@ where/// Both returned registers are allocated from the code-generation context/// and are owned by the caller. The caller must eventually free them.pub(super) fn emit_load_gc_heap_base_and_bound(&mut self) -> Result<(Reg, Reg)> {- let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();+ let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context();let gc_heap_base_offset = self.env.vmoffsets.ptr.vm_store_context().gc_heap_base();let gc_heap_len_offset = self.envdiff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex 1577c091ae..e8c20b4e99 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -514,7 +514,7 @@ where.as_u32().checked_mul(sig_index_bytes.into()).unwrap();- let signatures_base_offset = self.env.vmoffsets.ptr.vmctx_type_ids_array();+ let signatures_base_offset = self.env.vmoffsets.ptr.vmctx().type_ids();let funcref_sig_offset = self.env.vmoffsets.ptr.vm_func_ref().type_index();// Get the caller id.let caller_id = self.context.any_gpr(self.masm)?;@@ -1795,7 +1795,8 @@ wherelet data_segment_length_offset = self.env.vmoffsets- .vmctx_runtime_data_length(runtime_data_index);+ .runtime_data_lengths()+ .at(runtime_data_index);let tmp1 = self.context.any_gpr(self.masm)?;let tmp2 = self.context.any_gpr(self.masm)?;self.masm.load(@@ -1822,7 +1823,8 @@ wherelet data_segment_base_offset = self.env.vmoffsets- .vmctx_runtime_data_base(runtime_data_index);+ .runtime_data_bases()+ .at(runtime_data_index);self.masm.load(self.masm.address_at_vmctx(data_segment_base_offset)?,writable!(tmp1),@@ -1860,7 +1862,8 @@ wherelet data_segment_offset = self.env.vmoffsets- .vmctx_runtime_data_length(runtime_data_index);+ .runtime_data_lengths()+ .at(runtime_data_index);let len_addr = self.masm.address_at_vmctx(data_segment_offset)?;self.masm.store(RegImm::i32(0), len_addr, OperandSize::S32)}@@ -2146,7 +2149,7 @@ where/// Emits a series of instructions that load the `fuel_consumed` field from/// `VMStoreContext`.fn emit_load_fuel_consumed(&mut self, fuel_reg: Reg) -> Result<()> {- let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();+ let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context();let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed();self.masm.load_ptr(self.masm@@ -2224,8 +2227,8 @@ whereepoch_deadline_reg: Reg,epoch_counter_reg: Reg,) -> Result<()> {- let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx_epoch_ptr();- let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();+ let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx().epoch_ptr();+ let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context();let epoch_deadline_offset = self.env.vmoffsets.ptr.vm_store_context().epoch_deadline();// Load the current epoch value into `epoch_counter_var`.@@ -2266,7 +2269,7 @@ wherereturn Ok(());}- let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();+ let store_context_offset = self.env.vmoffsets.ptr.vmctx().store_context();let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed();let limits_reg = self.context.any_gpr(self.masm)?;@@ -2575,7 +2578,7 @@ where// is loaded from the `VMMemoryImport` and the vmctx is loaded from// the vmctx itself.None => {- let vmimport = self.env.vmoffsets.vmctx_vmmemory_import(mem);+ let vmimport = self.env.vmoffsets.imported_memories().at(mem);let vmctx_offset =vmimport + u32::from(self.env.vmoffsets.ptr.vm_memory_import().vmctx());let index_offset =@@ -2607,7 +2610,7 @@ whereOk(Callee::Builtin(builtin))}None => {- let vmimport = self.env.vmoffsets.vmctx_vmtable_import(table);+ let vmimport = self.env.vmoffsets.imported_tables().at(table);let vmctx_offset =vmimport + u32::from(self.env.vmoffsets.ptr.vm_table_import().vmctx());let index_offset =diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex 276bbfd4dc..0c1328736b 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -191,7 +191,7 @@ impl Masm for MacroAssembler {masm.with_scratch::<IntScratch, _>(|masm, scratch_stk_limit| {masm.with_scratch::<IntScratch, _>(|masm, scratch_tmp| {masm.load_ptr(- masm.address_at_reg(vmctx, ptr_size_u8.vmcontext_store_context().into())?,+ masm.address_at_reg(vmctx, ptr_size_u8.vmctx().store_context().into())?,scratch_stk_limit.writable(),)?;diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rsindex a455f828dd..e7fe439b24 100644--- a/winch/codegen/src/isa/x64/masm.rs+++ b/winch/codegen/src/isa/x64/masm.rs@@ -129,7 +129,7 @@ impl Masm for MacroAssembler {self.with_scratch::<IntScratch, _>(|masm, scratch| {masm.load_ptr(- masm.address_at_reg(vmctx, ptr_size.vmcontext_store_context().into())?,+ masm.address_at_reg(vmctx, ptr_size.vmctx().store_context().into())?,scratch.writable(),)?;