A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Introduce a `for_each_vm_type!` macro (#13967)
Nick Fitzgerald4c4ed70f3cec
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex 8e92a96360..8d2a427cc6 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -6,8 +6,8 @@ use cranelift_codegen::{};use wasmtime_environ::{BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex,- GetPtrSize, GlobalIndex, MemoryIndex, ModuleInternedTypeIndex, OwnedMemoryIndex, PtrSize as _,- RuntimeDataIndex, StaticModuleIndex, TableIndex, TagIndex, VMOffsets,+ GetPtrSize, GlobalIndex, MemoryIndex, ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex,+ StaticModuleIndex, TableIndex, TagIndex, VMOffsets,component::{ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,@@ -21,6 +21,18 @@ enum VmType {VMStoreContext,VMMemoryDefinition,VMTableDefinition,+ // NB: these two are currently only referenced by macro-generated+ // `AliasRegions` helpers that are not all wired up to call sites yet.+ #[allow(+ dead_code,+ reason = "generated uniformly for all VM types via `for_each_vm_type!`"+ )]+ VMGlobalDefinition,+ #[allow(+ dead_code,+ reason = "generated uniformly for all VM types via `for_each_vm_type!`"+ )]+ VMTagDefinition,VMComponentContext,VMDrcHeapData,VMCopyingHeapData,@@ -171,6 +183,8 @@ impl AliasRegionKey {const HOST_VAL_RAW_KIND: u32 = Self::new_kind(0b011110);const ELEMENT_SEGMENT_KIND: u32 = Self::new_kind(0b011111);const DATA_SEGMENT_KIND: u32 = Self::new_kind(0b100000);+ const VM_GLOBAL_DEFINITION_KIND: u32 = Self::new_kind(0b100001);+ const VM_TAG_DEFINITION_KIND: u32 = Self::new_kind(0b100010);/// Encode this key into a raw `u32` suitable for use as an/// `AliasRegionData::user_id`.@@ -183,6 +197,8 @@ impl AliasRegionKey {VmType::VMStoreContext => Self::VM_STORE_CONTEXT_KIND,VmType::VMMemoryDefinition => Self::VM_MEMORY_DEFINITION_KIND,VmType::VMTableDefinition => Self::VM_TABLE_DEFINITION_KIND,+ VmType::VMGlobalDefinition => Self::VM_GLOBAL_DEFINITION_KIND,+ VmType::VMTagDefinition => Self::VM_TAG_DEFINITION_KIND,VmType::VMComponentContext => Self::VM_COMPONENT_CONTEXT_KIND,VmType::VMDrcHeapData => Self::VM_DRC_HEAP_DATA_KIND,VmType::VMCopyingHeapData => Self::VM_COPYING_HEAP_DATA_KIND,@@ -330,6 +346,288 @@ impl<Offsets> AliasRegions<Offsets> {}}+/// A single field within one of Wasmtime's `VM*` types, along with everything+/// needed to emit a correctly alias-analyzed load or store of that field.+///+/// A `Field` is obtained from the `Field`-returning accessors generated by+/// [`define_vm_type_alias_region_helpers!`], for example+/// `alias_regions.vm_memory_definition().base()`.+///+/// This is the evolved form of the old `Load` type: in addition to producing a+/// deferred [`Load`] descriptor (via [`Field::to_deferred_load`]) for use in a+/// `VmctxLoadChain`, a `Field` can directly emit loads and stores relative to a+/// pointer to the containing `VM*` structure.+pub struct Field<'a, Offsets> {+ /// The alias-region cache used to get-or-create this field's alias region.+ regions: &'a mut AliasRegions<Offsets>,+ /// The key identifying this field's alias region.+ ///+ /// This is fixed when the `Field` is created, derived from the field's+ /// offset *within* its containing `VM*` type, and is deliberately+ /// independent of [`Field::relative_to`]: rebasing the load/store+ /// (e.g. to make the access relative to the `vmctx` rather than to a pointer+ /// to the containing structure) must not change which alias region the+ /// access belongs to.+ key: AliasRegionKey,+ /// The offset added to the base value when emitting a load or store of this+ /// field.+ ///+ /// Initially the field's offset *within* its containing `VM*` type; callers+ /// may rebase it via [`Field::relative_to`].+ offset: u32,+ /// The base memory flags for accesses of this field, before this field's+ /// alias region is mixed in.+ flags: ir::MemFlagsData,+ /// The Cranelift type of this field.+ ty: ir::Type,+}++impl<'a, Offsets> Field<'a, Offsets> {+ /// Create a new `Field` for the given `offset` within `vm_type`, loaded or+ /// stored with the given base `flags` and Cranelift type `ty`.+ fn new(+ regions: &'a mut AliasRegions<Offsets>,+ vm_type: VmType,+ offset: u32,+ flags: ir::MemFlagsData,+ ty: ir::Type,+ ) -> Self {+ Field {+ regions,+ key: AliasRegionKey::Vm {+ ty: vm_type,+ offset,+ },+ offset,+ flags,+ ty,+ }+ }++ /// Mark accesses of this field as `readonly`.+ ///+ /// Whether a field is `readonly` often depends on dynamic properties of the+ /// module being compiled (e.g. whether a memory can be relocated) rather+ /// than being a static property of the field; this method allows callers+ /// to mark the load as `readonly` in these cases.+ pub fn readonly(&mut self) -> &mut Self {+ self.flags.set_readonly();+ self+ }++ /// Mark accesses of this field as `can_move`.+ ///+ /// See the note on [`Field::readonly`].+ pub fn can_move(&mut self) -> &mut Self {+ self.flags = self.flags.with_can_move();+ self+ }++ /// Cast this field to the given type.+ ///+ /// This can be used, for example, to cast a `Field` that points to a+ /// `VMGlobalDefinition`'s storage (a `[u8; 16]` represented as+ /// `ir::types::I8X16`) to the global's actual Wasm type's representation+ /// (`ir::types::I32` for a Wasm `i32`).+ pub fn cast(&mut self, ty: ir::Type) -> &mut Self {+ self.ty = ty;+ self+ }++ /// Rebase this field's load or store to be relative to a new base.+ ///+ /// The `struct_offset` parameter is the offset of this field's containing+ /// `VM*` structure within the new base.+ ///+ /// A `Field` starts out relative to a pointer to its containing `VM*`+ /// structure. When that structure is inlined directly into a larger one (as+ /// an owned memory's `VMMemoryDefinition` is inlined into the `vmctx`), use+ /// this method to fold the structure's own offset within the larger+ /// structure into this `Field`, so that the resulting access is relative to+ /// the larger structure directly.+ pub fn relative_to(mut self, struct_offset: u32) -> Self {+ self.offset += struct_offset;+ self+ }++ /// Get-or-create this field's alias region and mix it into this field's+ /// base flags.+ fn flags_with_region(&mut self, func: &mut ir::Function) -> ir::MemFlagsData {+ let region = self.regions.region(func, self.key);+ self.flags.with_alias_region(Some(region))+ }++ /// Get a deferred [`Load`] descriptor for this field, for use in a+ /// `VmctxLoadChain`.+ ///+ /// The load is emitted at the field's current offset relative to its base+ /// value. By default that is the field's offset within its containing `VM*`+ /// structure, so the load is relative to a pointer to that structure; to+ /// make it relative to something else (e.g. the `vmctx`, when the structure+ /// is inlined into the `vmctx`), call [`Field::relative_to`] first.+ pub fn to_deferred_load(&mut self, func: &mut ir::Function) -> Load {+ let flags = self.flags_with_region(func);+ Load {+ offset: self.offset,+ flags,+ ty: self.ty,+ }+ }++ /// Emit a load of this field relative to `ptr`, a pointer to the containing+ /// `VM*` structure.+ pub fn load(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value) -> ir::Value {+ let load = self.to_deferred_load(cursor.func);+ load.emit(cursor, ptr)+ }++ /// Emit an atomic load of this field relative to `ptr`, a pointer to the+ /// containing `VM*` structure.+ pub fn load_atomic(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value) -> ir::Value {+ let flags = self.flags_with_region(cursor.func);+ let ty = self.ty;+ let pointer_type = self.regions.pointer_type;+ let offset = cursor.ins().iconst(pointer_type, i64::from(self.offset));+ let addr = cursor.ins().iadd(ptr, offset);+ cursor.ins().atomic_load(ty, flags, addr)+ }++ /// Emit a store of `value` to this field relative to `ptr`, a pointer to+ /// the containing `VM*` structure.+ #[allow(+ dead_code,+ reason = "part of the general `Field` API; not all fields are stored to yet"+ )]+ pub fn store(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value, value: ir::Value) {+ let flags = self.flags_with_region(cursor.func);+ cursor+ .ins()+ .store(flags, value, ptr, i32::try_from(self.offset).unwrap());+ }+}++/// Define, for each `VM*` type, an `AliasRegions` accessor that returns a+/// wrapper exposing a [`Field`]-returning method per field of that type.+///+/// For example, given+///+/// ```ignore+/// struct VMMemoryDefinition {+/// base: VmPtr<u8>,+/// current_length: AtomicUsize,+/// }+/// ```+///+/// this macro generates:+///+/// ```ignore+/// impl<Offsets> AliasRegions<Offsets> {+/// fn vm_memory_definition(&mut self) -> VMMemoryDefinition<'_, Offsets> { ... }+/// }+///+/// struct VMMemoryDefinition<'a, Offsets> { ... }+///+/// impl<'a, Offsets: GetPtrSize> VMMemoryDefinition<'a, Offsets> {+/// fn base(self) -> Field<'a, Offsets> { ... }+/// fn current_length(self) -> Field<'a, Offsets> { ... }+/// }+/// ```+#[allow(+ unused_macro_rules,+ reason = "the `#[readonly]`/`#[can_move]` marker arms are generated \+ uniformly but not exercised until a VM type uses those markers"+)]+macro_rules! define_vm_type_alias_region_helpers {+ // Classify a field type to its Cranelift `ir::Type`, given `$pt` (the+ // target pointer type as an `ir::Type`).+ (@field_ty $pt:expr, VmPtr < u8 >) => { $pt };+ (@field_ty $pt:expr, AtomicUsize) => { $pt };+ (@field_ty $pt:expr, usize) => { $pt };+ (@field_ty $pt:expr, [u8; 16]) => { ir::types::I8X16 };+ (@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };++ // Apply a field attribute to its access flags. The `#[readonly]` and+ // `#[can_move]` markers map to the corresponding `MemFlagsData` builder+ // methods; doc comments have no effect on flags; any other attribute is a+ // compile error.+ (@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly() };+ (@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() };+ (@apply_attr $flags:expr, [doc = $d:literal]) => { $flags };++ // Top-level entry: the list of `VM*` type definitions.+ ( $(+ $(#[doc = $sdoc:literal])*+ $(#[derive($($d:ident),*)])?+ #[repr($($repr:tt)*)]+ #[snake_name = $snake:ident]+ $svis:vis struct $Name:ident {+ $(+ $( # $fattr:tt )*+ $fvis:vis $fname:ident : $fty:tt $(< $fgen:tt >)? ,+ )*+ }+ )* ) => {+ $(+ #[doc = concat!(+ "An [`AliasRegions`] accessor for the fields of a `",+ stringify!($Name), "`."+ )]+ #[allow(+ dead_code,+ reason = "generated uniformly for all VM types; not all accessors are used yet"+ )]+ pub struct $Name<'a, Offsets> {+ regions: &'a mut AliasRegions<Offsets>,+ }++ #[allow(+ dead_code,+ reason = "generated uniformly for all VM types; not all accessors are used yet"+ )]+ impl<Offsets> AliasRegions<Offsets> {+ #[doc = concat!(+ "Get an accessor for the fields of a `", stringify!($Name), "`."+ )]+ pub fn $snake(&mut self) -> $Name<'_, Offsets> {+ $Name { regions: self }+ }+ }++ #[allow(+ dead_code,+ reason = "generated uniformly for all VM types; not all accessors are used yet"+ )]+ impl<'a, Offsets> $Name<'a, Offsets>+ where+ Offsets: GetPtrSize,+ {+ $(+ #[doc = concat!(+ "Get the [`Field`] for the `", stringify!($fname),+ "` field of `", stringify!($Name), "`."+ )]+ pub fn $fname(self) -> Field<'a, Offsets> {+ let offset = u32::from(+ self.regions.offsets.get_ptr_size().$snake().$fname()+ );+ let flags = ir::MemFlagsData::trusted();+ $(+ let flags = define_vm_type_alias_region_helpers!(+ @apply_attr flags, $fattr+ );+ )*+ let ty = define_vm_type_alias_region_helpers!(+ @field_ty self.regions.pointer_type, $fty $(< $fgen >)?+ );+ Field::new(self.regions, VmType::$Name, offset, flags, ty)+ }+ )*+ }+ )*+ };+}+wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers);+impl<Offsets> AliasRegions<Offsets>whereOffsets: GetPtrSize,@@ -917,160 +1215,6 @@ impl AliasRegions<VMOffsets<u8>> {new_length,)}-- /// Get a `Load` for an inlined-in-the-`vmctx` `VMMemoryDefinition`'s `base`- /// field.- pub fn vmctx_vmmemory_definition_base_load(- &mut self,- func: &mut ir::Function,- memory: OwnedMemoryIndex,- base_flags: ir::MemFlagsData,- ) -> Load {- let field = self.offsets.ptr.vmmemory_definition_base();- let region = self.vmmemory_definition_region(func, field.into());- Load {- offset: self.offsets.vmctx_vmmemory_definition_base(memory),- flags: base_flags.with_alias_region(Some(region)),- ty: self.pointer_type,- }- }-- /// Load an inlined-in-the-`vmctx` `VMMemoryDefinition`'s `base` field.- pub fn vmctx_vmmemory_definition_base(- &mut self,- cursor: &mut FuncCursor<'_>,- memory: OwnedMemoryIndex,- base_flags: ir::MemFlagsData,- vmctx: ir::Value,- ) -> ir::Value {- self.vmctx_vmmemory_definition_base_load(cursor.func, memory, base_flags)- .emit(cursor, vmctx)- }-- /// Get a `Load` for an inlined-in-the-`vmctx` `VMMemoryDefinition`'s `current_length`- /// field.- pub fn vmctx_vmmemory_definition_current_length_load(- &mut self,- func: &mut ir::Function,- memory: OwnedMemoryIndex,- ) -> Load {- let field = self.offsets.ptr.vmmemory_definition_current_length();- let region = self.vmmemory_definition_region(func, field.into());- Load {- offset: self- .offsets- .vmctx_vmmemory_definition_current_length(memory),- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),- ty: self.pointer_type,- }- }-- /// Load an inlined-in-the-`vmctx` `VMMemoryDefinition`'s `current_length` field.- pub fn vmctx_vmmemory_definition_current_length(- &mut self,- cursor: &mut FuncCursor<'_>,- memory: OwnedMemoryIndex,- vmctx: ir::Value,- ) -> ir::Value {- self.vmctx_vmmemory_definition_current_length_load(cursor.func, memory)- .emit(cursor, vmctx)- }-- fn vmtable_definition_region(- &mut self,- func: &mut ir::Function,- offset: u32,- ) -> ir::AliasRegion {- self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMTableDefinition,- offset,- },- )- }-- /// Get a `Load` for an inlined-in-the-`vmctx` `VMTableDefinition`'s `base`- /// field.- pub fn vmctx_vmtable_definition_base_load(- &mut self,- func: &mut ir::Function,- table: DefinedTableIndex,- base_flags: ir::MemFlagsData,- ) -> Load {- // NB: The region is keyed on the field's offset within the- // `VMTableDefinition`, not the `vmctx`, so that defined- // (`vmctx`-inlined) and imported (via-pointer) tables share one region- // per field.- let field = self.offsets.vmtable_definition_base();- let region = self.vmtable_definition_region(func, field.into());-- Load {- offset: self.offsets.vmctx_vmtable_definition_base(table),- flags: base_flags.with_alias_region(Some(region)),- ty: self.pointer_type,- }- }-- /// Get a `Load` for an inlined-in-the-`vmctx` `VMTableDefinition`'s- /// `current_elements` field.- ///- /// The caller supplies `ty` because the field's width depends on the table- /// elements' type.- pub fn vmctx_vmtable_definition_current_elements_load(- &mut self,- func: &mut ir::Function,- table: DefinedTableIndex,- ty: ir::Type,- ) -> Load {- // See note in `vmctx_vmtable_definition_base_load`.- let field = self.offsets.vmtable_definition_current_elements();- let region = self.vmtable_definition_region(func, field.into());-- Load {- offset: self- .offsets- .vmctx_vmtable_definition_current_elements(table),- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),- ty,- }- }-- /// Get a `Load` for the `VMTableDefinition::base` field reached through a- /// `*mut VMTableDefinition` (an imported table), for use in a- /// `VmctxLoadChain`.- pub fn vmtable_definition_base_load(- &mut self,- func: &mut ir::Function,- base_flags: ir::MemFlagsData,- ) -> Load {- let offset = self.offsets.vmtable_definition_base().into();- let region = self.vmtable_definition_region(func, offset);- Load {- offset,- flags: base_flags.with_alias_region(Some(region)),- ty: self.pointer_type,- }- }-- /// Get a `Load` for the `VMTableDefinition::current_elements` field reached- /// through a `*mut VMTableDefinition` (an imported table).- ///- /// The caller supplies `ty` because the field's width depends on the table- /// elements' type.- pub fn vmtable_definition_current_elements_load(- &mut self,- func: &mut ir::Function,- ty: ir::Type,- ) -> Load {- let offset = self.offsets.vmtable_definition_current_elements().into();- let region = self.vmtable_definition_region(func, offset);- Load {- offset,- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),- ty,- }- }}/// `VMStoreContext`-related methods.@@ -1736,111 +1880,6 @@ where}}-/// `VMMemoryDefinition`-related methods.-///-/// The `base` and `current_length` fields are reached either directly through-/// the `vmctx` (for an owned, inline memory) or through a `*mut-/// VMMemoryDefinition` (for a shared/imported memory). Both cases must share-/// one region per field, so the region is keyed on the field's offset *within*-/// the `VMMemoryDefinition` regardless of how the field is addressed; this is-/// required for soundness under inlining (cf. `memory_alias_region`).-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- fn vmmemory_definition_region(- &mut self,- func: &mut ir::Function,- offset: u32,- ) -> ir::AliasRegion {- self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMMemoryDefinition,- offset,- },- )- }-- /// Create a `Load` for the `VMMemoryDefinition::base` field, for use in a- /// `VmctxLoadChain`.- pub fn vmmemory_definition_base_load(- &mut self,- func: &mut ir::Function,- base_flags: ir::MemFlagsData,- ) -> Load {- let offset = self- .offsets- .get_ptr_size()- .vmmemory_definition_base()- .into();- let region = self.vmmemory_definition_region(func, offset);- Load {- offset,- flags: base_flags.with_alias_region(Some(region)),- ty: self.pointer_type,- }- }-- /// Create a `Load` for the `VMMemoryDefinition::current_length` field, for- /// use in a `VmctxLoadChain`.- pub fn vmmemory_definition_current_length_load(&mut self, func: &mut ir::Function) -> Load {- let offset = self- .offsets- .get_ptr_size()- .vmmemory_definition_current_length()- .into();- let region = self.vmmemory_definition_region(func, offset);- Load {- offset,- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),- ty: self.pointer_type,- }- }-- /// Emit a load of the `VMMemoryDefinition::base` field from the given `vmmemory_definition`.- pub fn vmmemory_definition_base(- &mut self,- cursor: &mut FuncCursor<'_>,- vmmemory_definition: ir::Value,- ) -> ir::Value {- self.vmmemory_definition_base_load(cursor.func, ir::MemFlagsData::trusted())- .emit(cursor, vmmemory_definition)- }-- /// Emit a (non-atomic) load of the `VMMemoryDefinition::current_length`- /// field from the given `vmmemory_definition`.- pub fn vmmemory_definition_current_length(- &mut self,- cursor: &mut FuncCursor<'_>,- vmmemory_definition: ir::Value,- ) -> ir::Value {- self.vmmemory_definition_current_length_load(cursor.func)- .emit(cursor, vmmemory_definition)- }-- /// Emit an atomic load of the `VMMemoryDefinition::current_length` field out- /// of a `*mut VMMemoryDefinition`.- pub fn vmmemory_definition_current_length_atomic(- &mut self,- cursor: &mut FuncCursor<'_>,- vmmemory_definition: ir::Value,- ) -> ir::Value {- let offset = self- .offsets- .get_ptr_size()- .vmmemory_definition_current_length();- let region = self.vmmemory_definition_region(cursor.func, offset.into());- let offset = cursor.ins().iconst(self.pointer_type, i64::from(offset));- let ptr = cursor.ins().iadd(vmmemory_definition, offset);- cursor.ins().atomic_load(- self.pointer_type,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- ptr,- )- }-}-/// `VMFuncRef`-related methods.impl<Offsets> AliasRegions<Offsets>wherediff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex 34a7def1a4..3d49e8daac 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -2057,7 +2057,9 @@ impl TrampolineCompiler<'_> {fn load_runtime_memory_base(&mut self, vmctx: ir::Value, mem: RuntimeMemoryIndex) -> ir::Value {let from_vmmemory_definition = self.load_memory(vmctx, mem);self.alias_regions- .vmmemory_definition_base(&mut self.builder.cursor(), from_vmmemory_definition)+ .vm_memory_definition()+ .base()+ .load(&mut self.builder.cursor(), from_vmmemory_definition)}}diff --git a/crates/cranelift/src/debug.rs b/crates/cranelift/src/debug.rsindex e2938ada26..39fd74a151 100644--- a/crates/cranelift/src/debug.rs+++ b/crates/cranelift/src/debug.rs@@ -104,7 +104,7 @@ impl<'a> Compilation<'a> {ModuleMemoryOffset::Imported {offset_to_vm_memory_definition: ofs.vmctx_vmmemory_import(index)+ u32::from(ofs.vmmemory_import_from()),- offset_to_memory_base: ofs.ptr.vmmemory_definition_base().into(),+ offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(),}} else if ofs.num_owned_memories > 0 {let index = OwnedMemoryIndex::new(0);@@ -113,7 +113,7 @@ impl<'a> Compilation<'a> {let index = DefinedMemoryIndex::new(0);ModuleMemoryOffset::Imported {offset_to_vm_memory_definition: ofs.vmctx_vmmemory_pointer(index),- offset_to_memory_base: ofs.ptr.vmmemory_definition_base().into(),+ offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(),}} else {ModuleMemoryOffset::Nonediff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex e791c092c2..d1a6b753ad 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -1563,10 +1563,7 @@ impl FuncEnvironment<'_> {// The base pointer load is `can_move`, and additionally `readonly` when// this memory's base can never move.let memory_tunables = MemoryTunables::new(self.tunables, MemoryKind::LinearMemory);- let mut base_flags = ir::MemFlagsData::trusted().with_can_move();- if !memory.memory_may_move(&memory_tunables) {- base_flags.set_readonly();- }+ let base_readonly = !memory.memory_may_move(&memory_tunables);if let Some(def_index) = self.module.defined_memory_index(index) {if is_shared {@@ -1577,49 +1574,63 @@ impl FuncEnvironment<'_> {let mem = self.alias_regions.vmctx_vmmemory_pointer_load(func, def_index);+ let mut base = self.alias_regions.vm_memory_definition().base();+ base.can_move();+ if base_readonly {+ base.readonly();+ }+ let base = base.to_deferred_load(func);+ let len = self+ .alias_regions+ .vm_memory_definition()+ .current_length()+ .to_deferred_load(func);(- VmctxLoadChain::new(smallvec![- mem,- self.alias_regions- .vmmemory_definition_base_load(func, base_flags),- ]),- VmctxLoadChain::new(smallvec![- mem,- self.alias_regions- .vmmemory_definition_current_length_load(func),- ]),+ VmctxLoadChain::new(smallvec![mem, base]),+ VmctxLoadChain::new(smallvec![mem, len]),)} else {+ // A defined, owned memory's `VMMemoryDefinition` is inlined into+ // the `vmctx` at an absolute offset, so its fields are loaded+ // relative to the `vmctx` itself: fold that offset into each+ // field's offset.let owned_index = self.module.owned_memory_index(def_index);+ let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index);+ let mut base = self.alias_regions.vm_memory_definition().base();+ base.can_move();+ if base_readonly {+ base.readonly();+ }+ let base = base.relative_to(vmctx_off).to_deferred_load(func);+ let len = self+ .alias_regions+ .vm_memory_definition()+ .current_length()+ .relative_to(vmctx_off)+ .to_deferred_load(func);(- VmctxLoadChain::new(smallvec![- self.alias_regions.vmctx_vmmemory_definition_base_load(- func,- owned_index,- base_flags- )- ]),- VmctxLoadChain::new(smallvec![- self.alias_regions- .vmctx_vmmemory_definition_current_length_load(func, owned_index)- ]),+ VmctxLoadChain::new(smallvec![base]),+ VmctxLoadChain::new(smallvec![len]),)}} else {let mem = self.alias_regions.vmctx_vmmemory_import_from_load(func, index);+ let mut base = self.alias_regions.vm_memory_definition().base();+ base.can_move();+ if base_readonly {+ base.readonly();+ }+ let base = base.to_deferred_load(func);+ let len = self+ .alias_regions+ .vm_memory_definition()+ .current_length()+ .to_deferred_load(func);(- VmctxLoadChain::new(smallvec![- mem,- self.alias_regions- .vmmemory_definition_base_load(func, base_flags),- ]),- VmctxLoadChain::new(smallvec![- mem,- self.alias_regions- .vmmemory_definition_current_length_load(func),- ]),+ VmctxLoadChain::new(smallvec![mem, base]),+ VmctxLoadChain::new(smallvec![mem, len]),)}}@@ -1654,29 +1665,31 @@ impl FuncEnvironment<'_> {// A fixed-size table can't be resized, so its base address won't change// and its base load is `readonly` and `can_move`.let is_static = Some(table.limits.min) == table.limits.max;- let mut base_flags = ir::MemFlagsData::trusted();- if is_static {- base_flags = base_flags.with_readonly().with_can_move();- }if let Some(def_index) = self.module.defined_table_index(index) {// A defined table's `VMTableDefinition` is inlined into the vmctx,// reached at an absolute `vmctx` offset.+ let vmctx_off = self.offsets.vmctx_vmtable_definition(def_index);+ let mut base = self.alias_regions.vm_table_definition().base();+ if is_static {+ base.readonly().can_move();+ }let base = VmctxLoadChain::new(smallvec![- self.alias_regions- .vmctx_vmtable_definition_base_load(func, def_index, base_flags)+ base.relative_to(vmctx_off).to_deferred_load(func)]);let bound = if is_static {TableSize::Static {bound: table.limits.min,}} else {+ let mut current_elements =+ self.alias_regions.vm_table_definition().current_elements();+ current_elements.cast(bound_ty);TableSize::Dynamic {bound: VmctxLoadChain::new(smallvec![- self.alias_regions- .vmctx_vmtable_definition_current_elements_load(- func, def_index, bound_ty- )+ current_elements+ .relative_to(vmctx_off)+ .to_deferred_load(func)]),}};@@ -1685,11 +1698,11 @@ impl FuncEnvironment<'_> {// An imported table is reached through a `*mut VMTableDefinition`// loaded from the `vmctx`.let from = self.alias_regions.vmctx_vmtable_from_load(func, index);- let base = VmctxLoadChain::new(smallvec![- from,- self.alias_regions- .vmtable_definition_base_load(func, base_flags),- ]);+ let mut base = self.alias_regions.vm_table_definition().base();+ if is_static {+ base.readonly().can_move();+ }+ let base = VmctxLoadChain::new(smallvec![from, base.to_deferred_load(func)]);let bound = if is_static {TableSize::Static {bound: table.limits.min,@@ -1699,7 +1712,10 @@ impl FuncEnvironment<'_> {bound: VmctxLoadChain::new(smallvec![from,self.alias_regions- .vmtable_definition_current_elements_load(func, bound_ty),+ .vm_table_definition()+ .current_elements()+ .cast(bound_ty)+ .to_deferred_load(func),]),}};@@ -3505,11 +3521,19 @@ impl FuncEnvironment<'_> {.alias_regions.vmctx_vmmemory_pointer(pos, vmctx, def_index);self.alias_regions- .vmmemory_definition_current_length_atomic(pos, mem_ptr)+ .vm_memory_definition()+ .current_length()+ .load_atomic(pos, mem_ptr)} else {+ // A defined, owned memory's `VMMemoryDefinition` is inlined into+ // the `vmctx` at an absolute offset.let owned_index = self.module.owned_memory_index(def_index);+ let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index);self.alias_regions- .vmctx_vmmemory_definition_current_length(pos, owned_index, vmctx)+ .vm_memory_definition()+ .current_length()+ .relative_to(vmctx_off)+ .load(pos, vmctx)}} else {let mem_ptr = self@@ -3517,10 +3541,14 @@ impl FuncEnvironment<'_> {.vmctx_vmmemory_import_from(pos, vmctx, index);if is_shared {self.alias_regions- .vmmemory_definition_current_length_atomic(pos, mem_ptr)+ .vm_memory_definition()+ .current_length()+ .load_atomic(pos, mem_ptr)} else {self.alias_regions- .vmmemory_definition_current_length(pos, mem_ptr)+ .vm_memory_definition()+ .current_length()+ .load(pos, mem_ptr)}}}diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rsindex 9c5f3b3308..129348dfd5 100644--- a/crates/environ/src/component/vmcomponent_offsets.rs+++ b/crates/environ/src/component/vmcomponent_offsets.rs@@ -172,8 +172,8 @@ impl<P: PtrSize> VMComponentOffsets<P> {size(builtins) = ret.ptr.size(),size(vm_store_context) = ret.ptr.size(),align(16),- size(may_leave) = cmul(ret.num_runtime_component_instances, ret.ptr.size_of_vmglobal_definition()),- size(task_may_block) = ret.ptr.size_of_vmglobal_definition(),+ size(may_leave) = cmul(ret.num_runtime_component_instances, ret.ptr.vm_global_definition().size()),+ size(task_may_block) = ret.ptr.vm_global_definition().size(),align(u32::from(ret.ptr.size())),size(trampoline_func_refs) = cmul(ret.num_trampolines, ret.ptr.size_of_vm_func_ref()),size(intrinsic_func_refs) = cmul(ret.num_unsafe_intrinsics, ret.ptr.size_of_vm_func_ref()),@@ -218,7 +218,7 @@ impl<P: PtrSize> VMComponentOffsets<P> {#[inline]pub fn may_leave(&self, index: RuntimeComponentInstanceIndex) -> u32 {assert!(index.as_u32() < self.num_runtime_component_instances);- self.may_leave + index.as_u32() * u32::from(self.ptr.size_of_vmglobal_definition())+ self.may_leave + index.as_u32() * u32::from(self.ptr.vm_global_definition().size())}/// The offset of the `task_may_block` field.diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rsindex cb7b11bd25..2be10ed159 100644--- a/crates/environ/src/lib.rs+++ b/crates/environ/src/lib.rs@@ -42,6 +42,8 @@ mod string_pool;mod trap_encoding;mod tunables;mod types;+#[macro_use]+mod vmtypes;mod vmoffsets;mod wasm_error;diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex 91ffadeaf9..970009e7c7 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -62,6 +62,159 @@ fn align(offset: u32, width: u32) -> u32 {(offset + (width - 1)) / width * width}+/// Generate the [`offsets`] module: a `struct VMFoo<P: PtrSize>(P)` for each+/// `VM*` type, with a method per field returning that field's offset, plus+/// `size` and `align` methods.+macro_rules! define_vm_type_offsets {+ // Classify a field type to its size in bytes as a `u32`, given `$p` (the+ // target pointer size as a `u8`).+ (@size ($p:expr) VmPtr < u8 >) => { u32::from($p) };+ (@size ($p:expr) AtomicUsize) => { u32::from($p) };+ (@size ($p:expr) usize) => { u32::from($p) };+ (@size ($p:expr) [u8; 16]) => { 16u32 };+ (@size ($p:expr) VMSharedTypeIndex) => { u32::from(($p).size_of_vmshared_type_index()) };++ // Classify a field type to its alignment in bytes as a `u32`, given `$p`+ // (the target pointer size as a `u8`).+ (@align ($p:expr) VmPtr < u8 >) => { u32::from($p) };+ (@align ($p:expr) AtomicUsize) => { u32::from($p) };+ (@align ($p:expr) usize) => { u32::from($p) };+ (@align ($p:expr) [u8; 16]) => { 16u32 };+ (@align ($p:expr) VMSharedTypeIndex) => { u32::from(($p).align_of_vmshared_type_index()) };++ // Classify a `#[repr(...)]` to the minimum alignment it forces, as a `u32`.+ (@repr_align C) => { 1u32 };+ (@repr_align C, align($n:literal)) => {{ let align: u32 = $n; align }};++ // Emit a `pub fn` per field returning that field's offset, computed by+ // accumulating the aligned size of each preceding field. `$p`/`$o` are+ // caller-minted identifiers (for the pointer size and running offset) that+ // are threaded through the recursion so their hygiene stays consistent+ // across the emitted `let` bindings.+ (@fields $Name:ident ($p:ident, $o:ident) prefix( $($prefix:tt)* )) => {};+ (@fields $Name:ident ($p:ident, $o:ident) prefix( $($prefix:tt)* )+ [ $fname:ident : $fty:tt $(< $fgen:tt >)? ]+ $($rest:tt)*+ ) => {+ #[doc = concat!(+ "The offset of the `", stringify!($fname),+ "` field of `", stringify!($Name), "`."+ )]+ #[inline]+ pub fn $fname(&self) -> u8 {+ let $p = self.0.size();+ let $o: u32 = 0;+ $($prefix)*+ let $o = align($o, define_vm_type_offsets!(@align ($p) $fty $(< $fgen >)?));+ let _ = $p;+ u8::try_from($o).unwrap()+ }+ define_vm_type_offsets!(@fields $Name ($p, $o)+ prefix(+ $($prefix)*+ let $o = align($o, define_vm_type_offsets!(@align ($p) $fty $(< $fgen >)?));+ let $o = $o + define_vm_type_offsets!(@size ($p) $fty $(< $fgen >)?);+ )+ $($rest)*+ );+ };++ // Top-level entry: the list of `VM*` type definitions.+ ( $(+ $(#[doc = $sdoc:literal])*+ $(#[derive($($d:ident),*)])?+ #[repr($($repr:tt)*)]+ #[snake_name = $snake:ident]+ $svis:vis struct $Name:ident {+ $(+ $(#[doc = $fdoc:literal])*+ $(#[readonly])?+ $(#[can_move])?+ $fvis:vis $fname:ident : $fty:tt $(< $fgen:tt >)? ,+ )*+ }+ )* ) => {+ /// Offsets of fields within the various `VM*` types, parameterized over+ /// a target [`PtrSize`] so that they can be computed during cross+ /// compilation.+ ///+ /// These types are namespaced within their own module so that they never+ /// collide with the real definitions of the `VM*` types themselves.+ pub mod offsets {+ use super::{align, PtrSize};++ $(+ #[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")]+ pub struct $Name<P: PtrSize>(pub P);++ impl<P: PtrSize> $Name<P> {+ define_vm_type_offsets!(@fields $Name (p, o) prefix()+ $( [ $fname : $fty $(< $fgen >)? ] )*+ );++ #[doc = concat!("The alignment of the `", stringify!($Name), "` type.")]+ #[inline]+ pub fn align(&self) -> u8 {+ let p = self.0.size();+ let a: u32 = define_vm_type_offsets!(@repr_align $($repr)*);+ $(+ let a = core::cmp::max(+ a,+ define_vm_type_offsets!(@align (p) $fty $(< $fgen >)?),+ );+ )*+ let _ = p;+ u8::try_from(a).unwrap()+ }++ #[doc = concat!("The size of the `", stringify!($Name), "` type.")]+ #[inline]+ pub fn size(&self) -> u8 {+ let p = self.0.size();+ let o: u32 = 0;+ $(+ let o = align(o, define_vm_type_offsets!(@align (p) $fty $(< $fgen >)?));+ let o = o + define_vm_type_offsets!(@size (p) $fty $(< $fgen >)?);+ )*+ let o = align(o, u32::from(self.align()));+ let _ = p;+ u8::try_from(o).unwrap()+ }+ }+ )*+ }+ };+}+for_each_vm_type!(define_vm_type_offsets);++/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to [`PtrSize`] for+/// each `VM*` type, using the `#[snake_name = ...]` attribute for the method+/// name.+macro_rules! define_ptr_size_vm_type_accessors {+ ( $(+ $(#[doc = $sdoc:literal])*+ $(#[derive($($d:ident),*)])?+ #[repr($($repr:tt)*)]+ #[snake_name = $snake:ident]+ $svis:vis struct $Name:ident {+ $(+ $(#[doc = $fdoc:literal])*+ $(#[readonly])?+ $(#[can_move])?+ $fvis:vis $fname:ident : $fty:tt $(< $fgen:tt >)? ,+ )*+ }+ )* ) => {+ $(+ #[doc = concat!("Get the [`offsets::", stringify!($Name), "`] offsets for this pointer size.")]+ #[inline]+ fn $snake(&self) -> offsets::$Name<&Self> {+ offsets::$Name(self)+ }+ )*+ };+}+/// This class computes offsets to fields within `VMContext` and other/// related structs that JIT code accesses directly.#[derive(Debug, Clone, Copy)]@@ -119,6 +272,11 @@ pub trait PtrSize {/// Returns the pointer size, in bytes, for the target.fn size(&self) -> u8;+ // Generate a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor for each+ // `VM*` type, giving access to that type's field offsets, size, and+ // alignment for this pointer size.+ for_each_vm_type!(define_ptr_size_vm_type_accessors);+/// The offset of the `VMContext::store_context` fieldfn vmcontext_store_context(&self) -> u8 {u8::try_from(align(@@ -169,23 +327,16 @@ pub trait PtrSize {4}- /// Return the size of `VMGlobalDefinition`; this is the size of the largest value type (i.e. a- /// V128).- #[inline]- fn size_of_vmglobal_definition(&self) -> u8 {- 16- }-- /// Return the size of `VMTagDefinition`.+ /// Return the alignment of `VMSharedTypeIndex`.#[inline]- fn size_of_vmtag_definition(&self) -> u8 {+ fn align_of_vmshared_type_index(&self) -> u8 {4}/// This is the size of the largest value type (i.e. a V128).#[inline]fn maximum_value_size(&self) -> u8 {- self.size_of_vmglobal_definition()+ self.vm_global_definition().size()}// Offsets within `VMStoreContext`@@ -223,14 +374,14 @@ pub trait PtrSize {/// Return the offset of the `gc_heap.base` field within a `VMStoreContext`.fn vmstore_context_gc_heap_base(&self) -> u8 {- let offset = self.vmstore_context_gc_heap() + self.vmmemory_definition_base();+ let offset = self.vmstore_context_gc_heap() + self.vm_memory_definition().base();debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());offset}/// Return the offset of the `gc_heap.current_length` field within a `VMStoreContext`.fn vmstore_context_gc_heap_current_length(&self) -> u8 {- let offset = self.vmstore_context_gc_heap() + self.vmmemory_definition_current_length();+ let offset = self.vmstore_context_gc_heap() + self.vm_memory_definition().current_length();debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());offset}@@ -238,7 +389,7 @@ pub trait PtrSize {/// Return the offset of the `last_wasm_exit_trampoline_fp` field/// of `VMStoreContext`.fn vmstore_context_last_wasm_exit_trampoline_fp(&self) -> u8 {- self.vmstore_context_gc_heap() + self.size_of_vmmemory_definition()+ self.vmstore_context_gc_heap() + self.vm_memory_definition().size()}/// Return the offset of the `last_wasm_exit_pc` field of `VMStoreContext`.@@ -325,26 +476,6 @@ pub trait PtrSize {align(u32::from(unaligned), u32::from(self.size())) as u8}- // Offsets within `VMMemoryDefinition`-- /// The offset of the `base` field.- #[inline]- fn vmmemory_definition_base(&self) -> u8 {- 0 * self.size()- }-- /// The offset of the `current_length` field.- #[inline]- fn vmmemory_definition_current_length(&self) -> u8 {- 1 * self.size()- }-- /// Return the size of `VMMemoryDefinition`.- #[inline]- fn size_of_vmmemory_definition(&self) -> u8 {- 2 * self.size()- }-/// Return the size of `*mut VMMemoryDefinition`.#[inline]fn size_of_vmmemory_pointer(&self) -> u8 {@@ -650,6 +781,16 @@ impl PtrSize for u8 {}}+impl<P> PtrSize for &'_ P+where+ P: PtrSize + ?Sized,+{+ #[inline]+ fn size(&self) -> u8 {+ (**self).size()+ }+}+/// Used to construct a `VMOffsets`#[derive(Debug, Clone, Copy)]pub struct VMOffsetsFields<P> {@@ -867,7 +1008,7 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {size(defined_memories)= cmul(ret.num_defined_memories, ret.ptr.size_of_vmmemory_pointer()),size(owned_memories)- = cmul(ret.num_owned_memories, ret.ptr.size_of_vmmemory_definition()),+ = cmul(ret.num_owned_memories, ret.ptr.vm_memory_definition().size()),size(imported_functions)= cmul(ret.num_imported_functions, ret.size_of_vmfunction_import()),size(imported_tables)@@ -877,12 +1018,12 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {size(imported_tags)= cmul(ret.num_imported_tags, ret.size_of_vmtag_import()),size(defined_tables)- = cmul(ret.num_defined_tables, ret.size_of_vmtable_definition()),+ = cmul(ret.num_defined_tables, ret.ptr.vm_table_definition().size()),align(16),size(defined_globals)- = cmul(ret.num_defined_globals, ret.ptr.size_of_vmglobal_definition()),+ = cmul(ret.num_defined_globals, ret.ptr.vm_global_definition().size()),size(defined_tags)- = cmul(ret.num_defined_tags, ret.ptr.size_of_vmtag_definition()),+ = cmul(ret.num_defined_tags, ret.ptr.vm_tag_definition().size()),size(defined_func_refs) = cmul(ret.num_escaped_funcs,ret.ptr.size_of_vm_func_ref(),@@ -971,28 +1112,11 @@ impl<P: PtrSize> VMOffsets<P> {/// Offsets for `VMTableDefinition`.impl<P: PtrSize> VMOffsets<P> {- /// The offset of the `base` field.- #[inline]- pub fn vmtable_definition_base(&self) -> u8 {- 0 * self.pointer_size()- }-- /// The offset of the `current_elements` field.- pub fn vmtable_definition_current_elements(&self) -> u8 {- 1 * self.pointer_size()- }-/// The size of the `current_elements` field.#[inline]pub fn size_of_vmtable_definition_current_elements(&self) -> u8 {self.pointer_size()}-- /// Return the size of `VMTableDefinition`.- #[inline]- pub fn size_of_vmtable_definition(&self) -> u8 {- 2 * self.pointer_size()- }}/// Offsets for `VMMemoryImport`.@@ -1203,7 +1327,8 @@ impl<P: PtrSize> VMOffsets<P> {#[inline]pub fn vmctx_vmtable_definition(&self, index: DefinedTableIndex) -> u32 {assert!(index.as_u32() < self.num_defined_tables);- self.vmctx_tables_begin() + index.as_u32() * u32::from(self.size_of_vmtable_definition())+ self.vmctx_tables_begin()+ + index.as_u32() * u32::from(self.ptr.vm_table_definition().size())}/// Return the offset to the `*mut VMMemoryDefinition` at index `index`.@@ -1219,7 +1344,7 @@ impl<P: PtrSize> VMOffsets<P> {pub fn vmctx_vmmemory_definition(&self, index: OwnedMemoryIndex) -> u32 {assert!(index.as_u32() < self.num_owned_memories);self.vmctx_owned_memories_begin()- + index.as_u32() * u32::from(self.ptr.size_of_vmmemory_definition())+ + index.as_u32() * u32::from(self.ptr.vm_memory_definition().size())}/// Return the offset to the `VMGlobalDefinition` index `index`.@@ -1227,14 +1352,14 @@ impl<P: PtrSize> VMOffsets<P> {pub fn vmctx_vmglobal_definition(&self, index: DefinedGlobalIndex) -> u32 {assert!(index.as_u32() < self.num_defined_globals);self.vmctx_globals_begin()- + index.as_u32() * u32::from(self.ptr.size_of_vmglobal_definition())+ + index.as_u32() * u32::from(self.ptr.vm_global_definition().size())}/// Return the offset to the `VMTagDefinition` index `index`.#[inline]pub fn vmctx_vmtag_definition(&self, index: DefinedTagIndex) -> u32 {assert!(index.as_u32() < self.num_defined_tags);- self.vmctx_tags_begin() + index.as_u32() * u32::from(self.ptr.size_of_vmtag_definition())+ self.vmctx_tags_begin() + index.as_u32() * u32::from(self.ptr.vm_tag_definition().size())}/// Return the offset to the `VMFuncRef` for the given function@@ -1299,13 +1424,14 @@ impl<P: PtrSize> VMOffsets<P> {/// Return the offset to the `base` field in `VMTableDefinition` index `index`.#[inline]pub fn vmctx_vmtable_definition_base(&self, index: DefinedTableIndex) -> u32 {- self.vmctx_vmtable_definition(index) + u32::from(self.vmtable_definition_base())+ self.vmctx_vmtable_definition(index) + u32::from(self.ptr.vm_table_definition().base())}/// Return the offset to the `current_elements` field in `VMTableDefinition` index `index`.#[inline]pub fn vmctx_vmtable_definition_current_elements(&self, index: DefinedTableIndex) -> u32 {- self.vmctx_vmtable_definition(index) + u32::from(self.vmtable_definition_current_elements())+ self.vmctx_vmtable_definition(index)+ + u32::from(self.ptr.vm_table_definition().current_elements())}/// Return the offset to the `from` field in `VMMemoryImport` index `index`.@@ -1317,14 +1443,14 @@ impl<P: PtrSize> VMOffsets<P> {/// Return the offset to the `base` field in `VMMemoryDefinition` index `index`.#[inline]pub fn vmctx_vmmemory_definition_base(&self, index: OwnedMemoryIndex) -> u32 {- self.vmctx_vmmemory_definition(index) + u32::from(self.ptr.vmmemory_definition_base())+ self.vmctx_vmmemory_definition(index) + u32::from(self.ptr.vm_memory_definition().base())}/// Return the offset to the `current_length` field in `VMMemoryDefinition` index `index`.#[inline]pub fn vmctx_vmmemory_definition_current_length(&self, index: OwnedMemoryIndex) -> u32 {self.vmctx_vmmemory_definition(index)- + u32::from(self.ptr.vmmemory_definition_current_length())+ + u32::from(self.ptr.vm_memory_definition().current_length())}/// Return the offset to the `from` field in `VMGlobalImport` index `index`.diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rsnew file mode 100644index 0000000000..5f061d6f6f--- /dev/null+++ b/crates/environ/src/vmtypes.rs@@ -0,0 +1,90 @@+//! Centralized definitions of the various `VM*` types whose layout is shared+//! between the runtime (which uses the actual structures) and the compiler+//! (which uses the types' offsets and has per-type alias regions).+//!+//! To keep these in sync, the shape of each type is defined exactly once here,+//! via the higher-order [`for_each_vm_type!`] macro, and each consumer+//! generates its view of the type from that single source of truth.++/// Invoke the given macro `$mac` once, passing it the definitions of each of the+/// `VM*` types whose layout is shared between the runtime, compilation offsets,+/// and Cranelift alias regions.+///+/// This is a higher-order macro: callers define a `macro_rules!` macro that+/// matches the grammar defined below and pass its name as an argument to this+/// macro's invocation, e.g. `for_each_vm_type!(define_vm_types)`.+///+/// # Grammar+///+/// Each type is emitted as a struct definition preceded by:+///+/// * Doc-comment attributes (`#[doc = "..."]`).+///+/// * An optional `#[derive(...)]` attribute.+///+/// * A `#[repr(...)]` attribute.+///+/// * A `#[snake_name = <ident>]` attribute giving the type's name in+/// `snake_case`, used to generate accessor method names.+///+/// Each field may be preceded by doc-comment attributes and, optionally, the+/// marker attributes `#[readonly]` and/or `#[can_move]` (in that order) which+/// describe how Cranelift may treat loads and stores of that field.+#[macro_export]+macro_rules! for_each_vm_type {+ ($mac:ident) => {+ $mac! {+ /// The fields compiled code needs to access to utilize a WebAssembly linear+ /// memory defined within the instance, namely the start address and the+ /// size in bytes.+ #[derive(Debug)]+ #[repr(C)]+ #[snake_name = vm_memory_definition]+ pub struct VMMemoryDefinition {+ /// The start address.+ pub base: VmPtr<u8>,++ /// The current logical size of this linear memory in bytes.+ ///+ /// This is atomic because shared memories must be able to grow their length+ /// atomically. For relaxed access, see+ /// [`VMMemoryDefinition::current_length()`].+ pub current_length: AtomicUsize,+ }++ /// The fields compiled code needs to access to utilize a WebAssembly table+ /// defined within the instance.+ #[derive(Debug, Copy, Clone)]+ #[repr(C)]+ #[snake_name = vm_table_definition]+ pub struct VMTableDefinition {+ /// Pointer to the table data.+ pub base: VmPtr<u8>,++ /// The current number of elements in the table.+ pub current_elements: usize,+ }++ /// The storage for a WebAssembly global defined within the instance.+ ///+ /// TODO: Pack the globals more densely, rather than using the same size+ /// for every type.+ #[derive(Debug)]+ #[repr(C, align(16))]+ #[snake_name = vm_global_definition]+ pub struct VMGlobalDefinition {+ /// The raw storage for a global's value.+ storage: [u8; 16],+ }++ /// A WebAssembly tag defined within the instance.+ #[derive(Debug)]+ #[repr(C)]+ #[snake_name = vm_tag_definition]+ pub struct VMTagDefinition {+ /// Function signature's type id.+ pub type_index: VMSharedTypeIndex,+ }+ }+ };+}diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex ceaa88cbb3..e207502c4a 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -417,25 +417,85 @@ mod test_vmtag_import {}}-/// The fields compiled code needs to access to utilize a WebAssembly linear-/// memory defined within the instance, namely the start address and the-/// size in bytes.-#[derive(Debug)]-#[repr(C)]-pub struct VMMemoryDefinition {- /// The start address.- pub base: VmPtr<u8>,+/// Define the runtime definitions of the shared `VM*` types.+macro_rules! define_vm_types {+ ( $(+ $(#[doc = $sdoc:literal])*+ $(#[derive($($d:ident),*)])?+ #[repr($($repr:tt)*)]+ #[snake_name = $snake:ident]+ $svis:vis struct $Name:ident {+ $(+ $(#[doc = $fdoc:literal])*+ $(#[readonly])?+ $(#[can_move])?+ $fvis:vis $fname:ident : $fty:tt $(< $fgen:tt >)? ,+ )*+ }+ )* ) => {+ $(+ $(#[doc = $sdoc])*+ $(#[derive($($d),*)])?+ #[repr($($repr)*)]+ $svis struct $Name {+ $(+ $(#[doc = $fdoc])*+ $fvis $fname: $fty $(< $fgen >)?,+ )*+ }+ )*- /// The current logical size of this linear memory in bytes.- ///- /// This is atomic because shared memories must be able to grow their length- /// atomically. For relaxed access, see- /// [`VMMemoryDefinition::current_length()`].- pub current_length: AtomicUsize,+ #[cfg(test)]+ mod test_vm_type_layouts {+ use super::{ $( $Name, )* };+ use core::mem::{align_of, offset_of, size_of};+ use wasmtime_environ::{HostPtr, PtrSize};++ $(+ #[test]+ fn $snake() {+ let host = HostPtr;+ let offsets = host.$snake();++ let expected = usize::from(offsets.size());+ let actual = size_of::<$Name>();+ assert_eq!(+ expected,+ actual,+ "size of {} failed: {expected} (expected) != {actual} (actual)",+ stringify!($Name),+ );++ let expected = usize::from(offsets.align());+ let actual = align_of::<$Name>();+ assert_eq!(+ expected,+ actual,+ "alignment of {} failed: {expected} (expected) != {actual} (actual)",+ stringify!($Name),+ );++ $(+ let expected = usize::from(offsets.$fname());+ let actual = offset_of!($Name, $fname);+ assert_eq!(+ expected,+ actual,+ "offset of {}::{} failed: {expected} (expected) != {actual} (actual)",+ stringify!($Name),+ stringify!($fname),+ );+ )*+ }+ )*+ }+ };}+wasmtime_environ::for_each_vm_type!(define_vm_types);-// SAFETY: the above definition has `repr(C)` and each field individually-// implements `VmSafe`, which satisfies the requirements of this trait.+// SAFETY: `VMMemoryDefinition` is generated with `#[repr(C)]` and each field+// individually implements `VmSafe`, which satisfies the requirements of this+// trait.unsafe impl VmSafe for VMMemoryDefinition {}impl VMMemoryDefinition {@@ -464,98 +524,19 @@ impl VMMemoryDefinition {}}-#[cfg(test)]-mod test_vmmemory_definition {- use super::VMMemoryDefinition;- use core::mem::offset_of;- use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};-- #[test]- fn check_vmmemory_definition_offsets() {- let module = Module::new(StaticModuleIndex::from_u32(0));- let offsets = VMOffsets::new(HostPtr, &module);- assert_eq!(- size_of::<VMMemoryDefinition>(),- usize::from(offsets.ptr.size_of_vmmemory_definition())- );- assert_eq!(- offset_of!(VMMemoryDefinition, base),- usize::from(offsets.ptr.vmmemory_definition_base())- );- assert_eq!(- offset_of!(VMMemoryDefinition, current_length),- usize::from(offsets.ptr.vmmemory_definition_current_length())- );- /* TODO: Assert that the size of `current_length` matches.- assert_eq!(- size_of::<VMMemoryDefinition::current_length>(),- usize::from(offsets.size_of_vmmemory_definition_current_length())- );- */- }-}--/// The fields compiled code needs to access to utilize a WebAssembly table-/// defined within the instance.-#[derive(Debug, Copy, Clone)]-#[repr(C)]-pub struct VMTableDefinition {- /// Pointer to the table data.- pub base: VmPtr<u8>,-- /// The current number of elements in the table.- pub current_elements: usize,-}--// SAFETY: the above structure is repr(C) and only contains `VmSafe` fields.+// SAFETY: `VMTableDefinition` is generated with `#[repr(C)]` and only contains+// `VmSafe` fields.unsafe impl VmSafe for VMTableDefinition {}-#[cfg(test)]-mod test_vmtable_definition {- use super::VMTableDefinition;- use core::mem::offset_of;- use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};-- #[test]- fn check_vmtable_definition_offsets() {- let module = Module::new(StaticModuleIndex::from_u32(0));- let offsets = VMOffsets::new(HostPtr, &module);- assert_eq!(- size_of::<VMTableDefinition>(),- usize::from(offsets.size_of_vmtable_definition())- );- assert_eq!(- offset_of!(VMTableDefinition, base),- usize::from(offsets.vmtable_definition_base())- );- assert_eq!(- offset_of!(VMTableDefinition, current_elements),- usize::from(offsets.vmtable_definition_current_elements())- );- }-}--/// The storage for a WebAssembly global defined within the instance.-///-/// TODO: Pack the globals more densely, rather than using the same size-/// for every type.-#[derive(Debug)]-#[repr(C, align(16))]-pub struct VMGlobalDefinition {- storage: [u8; 16],- // If more elements are added here, remember to add offset_of tests below!-}--// SAFETY: the above structure is repr(C) and only contains `VmSafe` fields.+// SAFETY: `VMGlobalDefinition` is generated with `#[repr(C)]` and only contains+// `VmSafe` fields.unsafe impl VmSafe for VMGlobalDefinition {}#[cfg(test)]mod test_vmglobal_definition {use super::VMGlobalDefinition;use std::mem::{align_of, size_of};- use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmglobal_definition_alignment() {@@ -568,16 +549,6 @@ mod test_vmglobal_definition {assert!(align_of::<VMGlobalDefinition>() >= align_of::<[f64; 2]>());}- #[test]- fn check_vmglobal_definition_offsets() {- let module = Module::new(StaticModuleIndex::from_u32(0));- let offsets = VMOffsets::new(HostPtr, &module);- assert_eq!(- size_of::<VMGlobalDefinition>(),- usize::from(offsets.ptr.size_of_vmglobal_definition())- );- }-#[test]fn check_vmglobal_begins_aligned() {let module = Module::new(StaticModuleIndex::from_u32(0));@@ -777,40 +748,19 @@ mod test_vmshared_type_index {}}-/// A WebAssembly tag defined within the instance.-///-#[derive(Debug)]-#[repr(C)]-pub struct VMTagDefinition {- /// Function signature's type id.- pub type_index: VMSharedTypeIndex,-}-impl VMTagDefinition {pub fn new(type_index: VMSharedTypeIndex) -> Self {Self { type_index }}}-// SAFETY: the above structure is repr(C) and only contains VmSafe-// fields.+// SAFETY: `VMTagDefinition` is generated with `#[repr(C)]` and only contains+// `VmSafe` fields.unsafe impl VmSafe for VMTagDefinition {}#[cfg(test)]mod test_vmtag_definition {- use super::VMTagDefinition;- use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};-- #[test]- fn check_vmtag_definition_offsets() {- let module = Module::new(StaticModuleIndex::from_u32(0));- let offsets = VMOffsets::new(HostPtr, &module);- assert_eq!(- size_of::<VMTagDefinition>(),- usize::from(offsets.ptr.size_of_vmtag_definition())- );- }+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmtag_begins_aligned() {diff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rsindex 2d88150670..f53fdba350 100644--- a/winch/codegen/src/codegen/env.rs+++ b/winch/codegen/src/codegen/env.rs@@ -244,8 +244,12 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {),None => (Some(self.vmoffsets.vmctx_vmtable_from(index)),- self.vmoffsets.vmtable_definition_base().into(),- self.vmoffsets.vmtable_definition_current_elements().into(),+ self.vmoffsets.ptr.vm_table_definition().base().into(),+ self.vmoffsets+ .ptr+ .vm_table_definition()+ .current_elements()+ .into(),),};@@ -276,10 +280,11 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {if is_shared {(Some(self.vmoffsets.vmctx_vmmemory_pointer(defined)),- self.vmoffsets.ptr.vmmemory_definition_base().into(),+ self.vmoffsets.ptr.vm_memory_definition().base().into(),self.vmoffsets.ptr- .vmmemory_definition_current_length()+ .vm_memory_definition()+ .current_length().into(),)} else {@@ -294,10 +299,11 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {}None => (Some(self.vmoffsets.vmctx_vmmemory_import_from(index)),- self.vmoffsets.ptr.vmmemory_definition_base().into(),+ self.vmoffsets.ptr.vm_memory_definition().base().into(),self.vmoffsets.ptr- .vmmemory_definition_current_length()+ .vm_memory_definition()+ .current_length().into(),),};