Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Introduce a `for_each_vm_type!` macro (#13967)

Nick Fitzgerald4c4ed70f3cec
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index 8e92a96360..8d2a427cc6 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -6,8 +6,8 @@ use cranelift_codegen::{
};
use wasmtime_environ::{
BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex,
- GetPtrSize, GlobalIndex, MemoryIndex, ModuleInternedTypeIndex, OwnedMemoryIndex, PtrSize as _,
- RuntimeDataIndex, StaticModuleIndex, TableIndex, TagIndex, VMOffsets,
+ GetPtrSize, GlobalIndex, MemoryIndex, ModuleInternedTypeIndex, PtrSize as _, RuntimeDataIndex,
+ StaticModuleIndex, TableIndex, TagIndex, VMOffsets,
component::{
ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,
RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,
@@ -21,6 +21,18 @@ enum VmType {
VMStoreContext,
VMMemoryDefinition,
VMTableDefinition,
+ // NB: these two are currently only referenced by macro-generated
+ // `AliasRegions` helpers that are not all wired up to call sites yet.
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for all VM types via `for_each_vm_type!`"
+ )]
+ VMGlobalDefinition,
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for all VM types via `for_each_vm_type!`"
+ )]
+ VMTagDefinition,
VMComponentContext,
VMDrcHeapData,
VMCopyingHeapData,
@@ -171,6 +183,8 @@ impl AliasRegionKey {
const HOST_VAL_RAW_KIND: u32 = Self::new_kind(0b011110);
const ELEMENT_SEGMENT_KIND: u32 = Self::new_kind(0b011111);
const DATA_SEGMENT_KIND: u32 = Self::new_kind(0b100000);
+ const VM_GLOBAL_DEFINITION_KIND: u32 = Self::new_kind(0b100001);
+ const VM_TAG_DEFINITION_KIND: u32 = Self::new_kind(0b100010);
/// Encode this key into a raw `u32` suitable for use as an
/// `AliasRegionData::user_id`.
@@ -183,6 +197,8 @@ impl AliasRegionKey {
VmType::VMStoreContext => Self::VM_STORE_CONTEXT_KIND,
VmType::VMMemoryDefinition => Self::VM_MEMORY_DEFINITION_KIND,
VmType::VMTableDefinition => Self::VM_TABLE_DEFINITION_KIND,
+ VmType::VMGlobalDefinition => Self::VM_GLOBAL_DEFINITION_KIND,
+ VmType::VMTagDefinition => Self::VM_TAG_DEFINITION_KIND,
VmType::VMComponentContext => Self::VM_COMPONENT_CONTEXT_KIND,
VmType::VMDrcHeapData => Self::VM_DRC_HEAP_DATA_KIND,
VmType::VMCopyingHeapData => Self::VM_COPYING_HEAP_DATA_KIND,
@@ -330,6 +346,288 @@ impl<Offsets> AliasRegions<Offsets> {
}
}
+/// A single field within one of Wasmtime's `VM*` types, along with everything
+/// needed to emit a correctly alias-analyzed load or store of that field.
+///
+/// A `Field` is obtained from the `Field`-returning accessors generated by
+/// [`define_vm_type_alias_region_helpers!`], for example
+/// `alias_regions.vm_memory_definition().base()`.
+///
+/// This is the evolved form of the old `Load` type: in addition to producing a
+/// deferred [`Load`] descriptor (via [`Field::to_deferred_load`]) for use in a
+/// `VmctxLoadChain`, a `Field` can directly emit loads and stores relative to a
+/// pointer to the containing `VM*` structure.
+pub struct Field<'a, Offsets> {
+ /// The alias-region cache used to get-or-create this field's alias region.
+ regions: &'a mut AliasRegions<Offsets>,
+ /// The key identifying this field's alias region.
+ ///
+ /// This is fixed when the `Field` is created, derived from the field's
+ /// offset *within* its containing `VM*` type, and is deliberately
+ /// independent of [`Field::relative_to`]: rebasing the load/store
+ /// (e.g. to make the access relative to the `vmctx` rather than to a pointer
+ /// to the containing structure) must not change which alias region the
+ /// access belongs to.
+ key: AliasRegionKey,
+ /// The offset added to the base value when emitting a load or store of this
+ /// field.
+ ///
+ /// Initially the field's offset *within* its containing `VM*` type; callers
+ /// may rebase it via [`Field::relative_to`].
+ offset: u32,
+ /// The base memory flags for accesses of this field, before this field's
+ /// alias region is mixed in.
+ flags: ir::MemFlagsData,
+ /// The Cranelift type of this field.
+ ty: ir::Type,
+}
+
+impl<'a, Offsets> Field<'a, Offsets> {
+ /// Create a new `Field` for the given `offset` within `vm_type`, loaded or
+ /// stored with the given base `flags` and Cranelift type `ty`.
+ fn new(
+ regions: &'a mut AliasRegions<Offsets>,
+ vm_type: VmType,
+ offset: u32,
+ flags: ir::MemFlagsData,
+ ty: ir::Type,
+ ) -> Self {
+ Field {
+ regions,
+ key: AliasRegionKey::Vm {
+ ty: vm_type,
+ offset,
+ },
+ offset,
+ flags,
+ ty,
+ }
+ }
+
+ /// Mark accesses of this field as `readonly`.
+ ///
+ /// Whether a field is `readonly` often depends on dynamic properties of the
+ /// module being compiled (e.g. whether a memory can be relocated) rather
+ /// than being a static property of the field; this method allows callers
+ /// to mark the load as `readonly` in these cases.
+ pub fn readonly(&mut self) -> &mut Self {
+ self.flags.set_readonly();
+ self
+ }
+
+ /// Mark accesses of this field as `can_move`.
+ ///
+ /// See the note on [`Field::readonly`].
+ pub fn can_move(&mut self) -> &mut Self {
+ self.flags = self.flags.with_can_move();
+ self
+ }
+
+ /// Cast this field to the given type.
+ ///
+ /// This can be used, for example, to cast a `Field` that points to a
+ /// `VMGlobalDefinition`'s storage (a `[u8; 16]` represented as
+ /// `ir::types::I8X16`) to the global's actual Wasm type's representation
+ /// (`ir::types::I32` for a Wasm `i32`).
+ pub fn cast(&mut self, ty: ir::Type) -> &mut Self {
+ self.ty = ty;
+ self
+ }
+
+ /// Rebase this field's load or store to be relative to a new base.
+ ///
+ /// The `struct_offset` parameter is the offset of this field's containing
+ /// `VM*` structure within the new base.
+ ///
+ /// A `Field` starts out relative to a pointer to its containing `VM*`
+ /// structure. When that structure is inlined directly into a larger one (as
+ /// an owned memory's `VMMemoryDefinition` is inlined into the `vmctx`), use
+ /// this method to fold the structure's own offset within the larger
+ /// structure into this `Field`, so that the resulting access is relative to
+ /// the larger structure directly.
+ pub fn relative_to(mut self, struct_offset: u32) -> Self {
+ self.offset += struct_offset;
+ self
+ }
+
+ /// Get-or-create this field's alias region and mix it into this field's
+ /// base flags.
+ fn flags_with_region(&mut self, func: &mut ir::Function) -> ir::MemFlagsData {
+ let region = self.regions.region(func, self.key);
+ self.flags.with_alias_region(Some(region))
+ }
+
+ /// Get a deferred [`Load`] descriptor for this field, for use in a
+ /// `VmctxLoadChain`.
+ ///
+ /// The load is emitted at the field's current offset relative to its base
+ /// value. By default that is the field's offset within its containing `VM*`
+ /// structure, so the load is relative to a pointer to that structure; to
+ /// make it relative to something else (e.g. the `vmctx`, when the structure
+ /// is inlined into the `vmctx`), call [`Field::relative_to`] first.
+ pub fn to_deferred_load(&mut self, func: &mut ir::Function) -> Load {
+ let flags = self.flags_with_region(func);
+ Load {
+ offset: self.offset,
+ flags,
+ ty: self.ty,
+ }
+ }
+
+ /// Emit a load of this field relative to `ptr`, a pointer to the containing
+ /// `VM*` structure.
+ pub fn load(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value) -> ir::Value {
+ let load = self.to_deferred_load(cursor.func);
+ load.emit(cursor, ptr)
+ }
+
+ /// Emit an atomic load of this field relative to `ptr`, a pointer to the
+ /// containing `VM*` structure.
+ pub fn load_atomic(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value) -> ir::Value {
+ let flags = self.flags_with_region(cursor.func);
+ let ty = self.ty;
+ let pointer_type = self.regions.pointer_type;
+ let offset = cursor.ins().iconst(pointer_type, i64::from(self.offset));
+ let addr = cursor.ins().iadd(ptr, offset);
+ cursor.ins().atomic_load(ty, flags, addr)
+ }
+
+ /// Emit a store of `value` to this field relative to `ptr`, a pointer to
+ /// the containing `VM*` structure.
+ #[allow(
+ dead_code,
+ reason = "part of the general `Field` API; not all fields are stored to yet"
+ )]
+ pub fn store(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value, value: ir::Value) {
+ let flags = self.flags_with_region(cursor.func);
+ cursor
+ .ins()
+ .store(flags, value, ptr, i32::try_from(self.offset).unwrap());
+ }
+}
+
+/// Define, for each `VM*` type, an `AliasRegions` accessor that returns a
+/// wrapper exposing a [`Field`]-returning method per field of that type.
+///
+/// For example, given
+///
+/// ```ignore
+/// struct VMMemoryDefinition {
+/// base: VmPtr<u8>,
+/// current_length: AtomicUsize,
+/// }
+/// ```
+///
+/// this macro generates:
+///
+/// ```ignore
+/// impl<Offsets> AliasRegions<Offsets> {
+/// fn vm_memory_definition(&mut self) -> VMMemoryDefinition<'_, Offsets> { ... }
+/// }
+///
+/// struct VMMemoryDefinition<'a, Offsets> { ... }
+///
+/// impl<'a, Offsets: GetPtrSize> VMMemoryDefinition<'a, Offsets> {
+/// fn base(self) -> Field<'a, Offsets> { ... }
+/// fn current_length(self) -> Field<'a, Offsets> { ... }
+/// }
+/// ```
+#[allow(
+ unused_macro_rules,
+ reason = "the `#[readonly]`/`#[can_move]` marker arms are generated \
+ uniformly but not exercised until a VM type uses those markers"
+)]
+macro_rules! define_vm_type_alias_region_helpers {
+ // Classify a field type to its Cranelift `ir::Type`, given `$pt` (the
+ // target pointer type as an `ir::Type`).
+ (@field_ty $pt:expr, VmPtr < u8 >) => { $pt };
+ (@field_ty $pt:expr, AtomicUsize) => { $pt };
+ (@field_ty $pt:expr, usize) => { $pt };
+ (@field_ty $pt:expr, [u8; 16]) => { ir::types::I8X16 };
+ (@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };
+
+ // Apply a field attribute to its access flags. The `#[readonly]` and
+ // `#[can_move]` markers map to the corresponding `MemFlagsData` builder
+ // methods; doc comments have no effect on flags; any other attribute is a
+ // compile error.
+ (@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly() };
+ (@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move() };
+ (@apply_attr $flags:expr, [doc = $d:literal]) => { $flags };
+
+ // Top-level entry: the list of `VM*` type definitions.
+ ( $(
+ $(#[doc = $sdoc:literal])*
+ $(#[derive($($d:ident),*)])?
+ #[repr($($repr:tt)*)]
+ #[snake_name = $snake:ident]
+ $svis:vis struct $Name:ident {
+ $(
+ $( # $fattr:tt )*
+ $fvis:vis $fname:ident : $fty:tt $(< $fgen:tt >)? ,
+ )*
+ }
+ )* ) => {
+ $(
+ #[doc = concat!(
+ "An [`AliasRegions`] accessor for the fields of a `",
+ stringify!($Name), "`."
+ )]
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for all VM types; not all accessors are used yet"
+ )]
+ pub struct $Name<'a, Offsets> {
+ regions: &'a mut AliasRegions<Offsets>,
+ }
+
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for all VM types; not all accessors are used yet"
+ )]
+ impl<Offsets> AliasRegions<Offsets> {
+ #[doc = concat!(
+ "Get an accessor for the fields of a `", stringify!($Name), "`."
+ )]
+ pub fn $snake(&mut self) -> $Name<'_, Offsets> {
+ $Name { regions: self }
+ }
+ }
+
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for all VM types; not all accessors are used yet"
+ )]
+ impl<'a, Offsets> $Name<'a, Offsets>
+ where
+ Offsets: GetPtrSize,
+ {
+ $(
+ #[doc = concat!(
+ "Get the [`Field`] for the `", stringify!($fname),
+ "` field of `", stringify!($Name), "`."
+ )]
+ pub fn $fname(self) -> Field<'a, Offsets> {
+ let offset = u32::from(
+ self.regions.offsets.get_ptr_size().$snake().$fname()
+ );
+ let flags = ir::MemFlagsData::trusted();
+ $(
+ let flags = define_vm_type_alias_region_helpers!(
+ @apply_attr flags, $fattr
+ );
+ )*
+ let ty = define_vm_type_alias_region_helpers!(
+ @field_ty self.regions.pointer_type, $fty $(< $fgen >)?
+ );
+ Field::new(self.regions, VmType::$Name, offset, flags, ty)
+ }
+ )*
+ }
+ )*
+ };
+}
+wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers);
+
impl<Offsets> AliasRegions<Offsets>
where
Offsets: GetPtrSize,
@@ -917,160 +1215,6 @@ impl AliasRegions<VMOffsets<u8>> {
new_length,
)
}
-
- /// Get a `Load` for an inlined-in-the-`vmctx` `VMMemoryDefinition`'s `base`
- /// field.
- pub fn vmctx_vmmemory_definition_base_load(
- &mut self,
- func: &mut ir::Function,
- memory: OwnedMemoryIndex,
- base_flags: ir::MemFlagsData,
- ) -> Load {
- let field = self.offsets.ptr.vmmemory_definition_base();
- let region = self.vmmemory_definition_region(func, field.into());
- Load {
- offset: self.offsets.vmctx_vmmemory_definition_base(memory),
- flags: base_flags.with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
- }
-
- /// Load an inlined-in-the-`vmctx` `VMMemoryDefinition`'s `base` field.
- pub fn vmctx_vmmemory_definition_base(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- memory: OwnedMemoryIndex,
- base_flags: ir::MemFlagsData,
- vmctx: ir::Value,
- ) -> ir::Value {
- self.vmctx_vmmemory_definition_base_load(cursor.func, memory, base_flags)
- .emit(cursor, vmctx)
- }
-
- /// Get a `Load` for an inlined-in-the-`vmctx` `VMMemoryDefinition`'s `current_length`
- /// field.
- pub fn vmctx_vmmemory_definition_current_length_load(
- &mut self,
- func: &mut ir::Function,
- memory: OwnedMemoryIndex,
- ) -> Load {
- let field = self.offsets.ptr.vmmemory_definition_current_length();
- let region = self.vmmemory_definition_region(func, field.into());
- Load {
- offset: self
- .offsets
- .vmctx_vmmemory_definition_current_length(memory),
- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
- }
-
- /// Load an inlined-in-the-`vmctx` `VMMemoryDefinition`'s `current_length` field.
- pub fn vmctx_vmmemory_definition_current_length(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- memory: OwnedMemoryIndex,
- vmctx: ir::Value,
- ) -> ir::Value {
- self.vmctx_vmmemory_definition_current_length_load(cursor.func, memory)
- .emit(cursor, vmctx)
- }
-
- fn vmtable_definition_region(
- &mut self,
- func: &mut ir::Function,
- offset: u32,
- ) -> ir::AliasRegion {
- self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMTableDefinition,
- offset,
- },
- )
- }
-
- /// Get a `Load` for an inlined-in-the-`vmctx` `VMTableDefinition`'s `base`
- /// field.
- pub fn vmctx_vmtable_definition_base_load(
- &mut self,
- func: &mut ir::Function,
- table: DefinedTableIndex,
- base_flags: ir::MemFlagsData,
- ) -> Load {
- // NB: The region is keyed on the field's offset within the
- // `VMTableDefinition`, not the `vmctx`, so that defined
- // (`vmctx`-inlined) and imported (via-pointer) tables share one region
- // per field.
- let field = self.offsets.vmtable_definition_base();
- let region = self.vmtable_definition_region(func, field.into());
-
- Load {
- offset: self.offsets.vmctx_vmtable_definition_base(table),
- flags: base_flags.with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
- }
-
- /// Get a `Load` for an inlined-in-the-`vmctx` `VMTableDefinition`'s
- /// `current_elements` field.
- ///
- /// The caller supplies `ty` because the field's width depends on the table
- /// elements' type.
- pub fn vmctx_vmtable_definition_current_elements_load(
- &mut self,
- func: &mut ir::Function,
- table: DefinedTableIndex,
- ty: ir::Type,
- ) -> Load {
- // See note in `vmctx_vmtable_definition_base_load`.
- let field = self.offsets.vmtable_definition_current_elements();
- let region = self.vmtable_definition_region(func, field.into());
-
- Load {
- offset: self
- .offsets
- .vmctx_vmtable_definition_current_elements(table),
- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- ty,
- }
- }
-
- /// Get a `Load` for the `VMTableDefinition::base` field reached through a
- /// `*mut VMTableDefinition` (an imported table), for use in a
- /// `VmctxLoadChain`.
- pub fn vmtable_definition_base_load(
- &mut self,
- func: &mut ir::Function,
- base_flags: ir::MemFlagsData,
- ) -> Load {
- let offset = self.offsets.vmtable_definition_base().into();
- let region = self.vmtable_definition_region(func, offset);
- Load {
- offset,
- flags: base_flags.with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
- }
-
- /// Get a `Load` for the `VMTableDefinition::current_elements` field reached
- /// through a `*mut VMTableDefinition` (an imported table).
- ///
- /// The caller supplies `ty` because the field's width depends on the table
- /// elements' type.
- pub fn vmtable_definition_current_elements_load(
- &mut self,
- func: &mut ir::Function,
- ty: ir::Type,
- ) -> Load {
- let offset = self.offsets.vmtable_definition_current_elements().into();
- let region = self.vmtable_definition_region(func, offset);
- Load {
- offset,
- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- ty,
- }
- }
}
/// `VMStoreContext`-related methods.
@@ -1736,111 +1880,6 @@ where
}
}
-/// `VMMemoryDefinition`-related methods.
-///
-/// The `base` and `current_length` fields are reached either directly through
-/// the `vmctx` (for an owned, inline memory) or through a `*mut
-/// VMMemoryDefinition` (for a shared/imported memory). Both cases must share
-/// one region per field, so the region is keyed on the field's offset *within*
-/// the `VMMemoryDefinition` regardless of how the field is addressed; this is
-/// required for soundness under inlining (cf. `memory_alias_region`).
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- fn vmmemory_definition_region(
- &mut self,
- func: &mut ir::Function,
- offset: u32,
- ) -> ir::AliasRegion {
- self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMMemoryDefinition,
- offset,
- },
- )
- }
-
- /// Create a `Load` for the `VMMemoryDefinition::base` field, for use in a
- /// `VmctxLoadChain`.
- pub fn vmmemory_definition_base_load(
- &mut self,
- func: &mut ir::Function,
- base_flags: ir::MemFlagsData,
- ) -> Load {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmmemory_definition_base()
- .into();
- let region = self.vmmemory_definition_region(func, offset);
- Load {
- offset,
- flags: base_flags.with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
- }
-
- /// Create a `Load` for the `VMMemoryDefinition::current_length` field, for
- /// use in a `VmctxLoadChain`.
- pub fn vmmemory_definition_current_length_load(&mut self, func: &mut ir::Function) -> Load {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmmemory_definition_current_length()
- .into();
- let region = self.vmmemory_definition_region(func, offset);
- Load {
- offset,
- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
- }
-
- /// Emit a load of the `VMMemoryDefinition::base` field from the given `vmmemory_definition`.
- pub fn vmmemory_definition_base(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmmemory_definition: ir::Value,
- ) -> ir::Value {
- self.vmmemory_definition_base_load(cursor.func, ir::MemFlagsData::trusted())
- .emit(cursor, vmmemory_definition)
- }
-
- /// Emit a (non-atomic) load of the `VMMemoryDefinition::current_length`
- /// field from the given `vmmemory_definition`.
- pub fn vmmemory_definition_current_length(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmmemory_definition: ir::Value,
- ) -> ir::Value {
- self.vmmemory_definition_current_length_load(cursor.func)
- .emit(cursor, vmmemory_definition)
- }
-
- /// Emit an atomic load of the `VMMemoryDefinition::current_length` field out
- /// of a `*mut VMMemoryDefinition`.
- pub fn vmmemory_definition_current_length_atomic(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmmemory_definition: ir::Value,
- ) -> ir::Value {
- let offset = self
- .offsets
- .get_ptr_size()
- .vmmemory_definition_current_length();
- let region = self.vmmemory_definition_region(cursor.func, offset.into());
- let offset = cursor.ins().iconst(self.pointer_type, i64::from(offset));
- let ptr = cursor.ins().iadd(vmmemory_definition, offset);
- cursor.ins().atomic_load(
- self.pointer_type,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- ptr,
- )
- }
-}
-
/// `VMFuncRef`-related methods.
impl<Offsets> AliasRegions<Offsets>
where
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index 34a7def1a4..3d49e8daac 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -2057,7 +2057,9 @@ impl TrampolineCompiler<'_> {
fn load_runtime_memory_base(&mut self, vmctx: ir::Value, mem: RuntimeMemoryIndex) -> ir::Value {
let from_vmmemory_definition = self.load_memory(vmctx, mem);
self.alias_regions
- .vmmemory_definition_base(&mut self.builder.cursor(), from_vmmemory_definition)
+ .vm_memory_definition()
+ .base()
+ .load(&mut self.builder.cursor(), from_vmmemory_definition)
}
}
diff --git a/crates/cranelift/src/debug.rs b/crates/cranelift/src/debug.rs
index e2938ada26..39fd74a151 100644
--- a/crates/cranelift/src/debug.rs
+++ b/crates/cranelift/src/debug.rs
@@ -104,7 +104,7 @@ impl<'a> Compilation<'a> {
ModuleMemoryOffset::Imported {
offset_to_vm_memory_definition: ofs.vmctx_vmmemory_import(index)
+ u32::from(ofs.vmmemory_import_from()),
- offset_to_memory_base: ofs.ptr.vmmemory_definition_base().into(),
+ offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(),
}
} else if ofs.num_owned_memories > 0 {
let index = OwnedMemoryIndex::new(0);
@@ -113,7 +113,7 @@ impl<'a> Compilation<'a> {
let index = DefinedMemoryIndex::new(0);
ModuleMemoryOffset::Imported {
offset_to_vm_memory_definition: ofs.vmctx_vmmemory_pointer(index),
- offset_to_memory_base: ofs.ptr.vmmemory_definition_base().into(),
+ offset_to_memory_base: ofs.ptr.vm_memory_definition().base().into(),
}
} else {
ModuleMemoryOffset::None
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index e791c092c2..d1a6b753ad 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -1563,10 +1563,7 @@ impl FuncEnvironment<'_> {
// The base pointer load is `can_move`, and additionally `readonly` when
// this memory's base can never move.
let memory_tunables = MemoryTunables::new(self.tunables, MemoryKind::LinearMemory);
- let mut base_flags = ir::MemFlagsData::trusted().with_can_move();
- if !memory.memory_may_move(&memory_tunables) {
- base_flags.set_readonly();
- }
+ let base_readonly = !memory.memory_may_move(&memory_tunables);
if let Some(def_index) = self.module.defined_memory_index(index) {
if is_shared {
@@ -1577,49 +1574,63 @@ impl FuncEnvironment<'_> {
let mem = self
.alias_regions
.vmctx_vmmemory_pointer_load(func, def_index);
+ let mut base = self.alias_regions.vm_memory_definition().base();
+ base.can_move();
+ if base_readonly {
+ base.readonly();
+ }
+ let base = base.to_deferred_load(func);
+ let len = self
+ .alias_regions
+ .vm_memory_definition()
+ .current_length()
+ .to_deferred_load(func);
(
- VmctxLoadChain::new(smallvec![
- mem,
- self.alias_regions
- .vmmemory_definition_base_load(func, base_flags),
- ]),
- VmctxLoadChain::new(smallvec![
- mem,
- self.alias_regions
- .vmmemory_definition_current_length_load(func),
- ]),
+ VmctxLoadChain::new(smallvec![mem, base]),
+ VmctxLoadChain::new(smallvec![mem, len]),
)
} else {
+ // A defined, owned memory's `VMMemoryDefinition` is inlined into
+ // the `vmctx` at an absolute offset, so its fields are loaded
+ // relative to the `vmctx` itself: fold that offset into each
+ // field's offset.
let owned_index = self.module.owned_memory_index(def_index);
+ let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index);
+ let mut base = self.alias_regions.vm_memory_definition().base();
+ base.can_move();
+ if base_readonly {
+ base.readonly();
+ }
+ let base = base.relative_to(vmctx_off).to_deferred_load(func);
+ let len = self
+ .alias_regions
+ .vm_memory_definition()
+ .current_length()
+ .relative_to(vmctx_off)
+ .to_deferred_load(func);
(
- VmctxLoadChain::new(smallvec![
- self.alias_regions.vmctx_vmmemory_definition_base_load(
- func,
- owned_index,
- base_flags
- )
- ]),
- VmctxLoadChain::new(smallvec![
- self.alias_regions
- .vmctx_vmmemory_definition_current_length_load(func, owned_index)
- ]),
+ VmctxLoadChain::new(smallvec![base]),
+ VmctxLoadChain::new(smallvec![len]),
)
}
} else {
let mem = self
.alias_regions
.vmctx_vmmemory_import_from_load(func, index);
+ let mut base = self.alias_regions.vm_memory_definition().base();
+ base.can_move();
+ if base_readonly {
+ base.readonly();
+ }
+ let base = base.to_deferred_load(func);
+ let len = self
+ .alias_regions
+ .vm_memory_definition()
+ .current_length()
+ .to_deferred_load(func);
(
- VmctxLoadChain::new(smallvec![
- mem,
- self.alias_regions
- .vmmemory_definition_base_load(func, base_flags),
- ]),
- VmctxLoadChain::new(smallvec![
- mem,
- self.alias_regions
- .vmmemory_definition_current_length_load(func),
- ]),
+ VmctxLoadChain::new(smallvec![mem, base]),
+ VmctxLoadChain::new(smallvec![mem, len]),
)
}
}
@@ -1654,29 +1665,31 @@ impl FuncEnvironment<'_> {
// A fixed-size table can't be resized, so its base address won't change
// and its base load is `readonly` and `can_move`.
let is_static = Some(table.limits.min) == table.limits.max;
- let mut base_flags = ir::MemFlagsData::trusted();
- if is_static {
- base_flags = base_flags.with_readonly().with_can_move();
- }
if let Some(def_index) = self.module.defined_table_index(index) {
// A defined table's `VMTableDefinition` is inlined into the vmctx,
// reached at an absolute `vmctx` offset.
+ let vmctx_off = self.offsets.vmctx_vmtable_definition(def_index);
+ let mut base = self.alias_regions.vm_table_definition().base();
+ if is_static {
+ base.readonly().can_move();
+ }
let base = VmctxLoadChain::new(smallvec![
- self.alias_regions
- .vmctx_vmtable_definition_base_load(func, def_index, base_flags)
+ base.relative_to(vmctx_off).to_deferred_load(func)
]);
let bound = if is_static {
TableSize::Static {
bound: table.limits.min,
}
} else {
+ let mut current_elements =
+ self.alias_regions.vm_table_definition().current_elements();
+ current_elements.cast(bound_ty);
TableSize::Dynamic {
bound: VmctxLoadChain::new(smallvec![
- self.alias_regions
- .vmctx_vmtable_definition_current_elements_load(
- func, def_index, bound_ty
- )
+ current_elements
+ .relative_to(vmctx_off)
+ .to_deferred_load(func)
]),
}
};
@@ -1685,11 +1698,11 @@ impl FuncEnvironment<'_> {
// An imported table is reached through a `*mut VMTableDefinition`
// loaded from the `vmctx`.
let from = self.alias_regions.vmctx_vmtable_from_load(func, index);
- let base = VmctxLoadChain::new(smallvec![
- from,
- self.alias_regions
- .vmtable_definition_base_load(func, base_flags),
- ]);
+ let mut base = self.alias_regions.vm_table_definition().base();
+ if is_static {
+ base.readonly().can_move();
+ }
+ let base = VmctxLoadChain::new(smallvec![from, base.to_deferred_load(func)]);
let bound = if is_static {
TableSize::Static {
bound: table.limits.min,
@@ -1699,7 +1712,10 @@ impl FuncEnvironment<'_> {
bound: VmctxLoadChain::new(smallvec![
from,
self.alias_regions
- .vmtable_definition_current_elements_load(func, bound_ty),
+ .vm_table_definition()
+ .current_elements()
+ .cast(bound_ty)
+ .to_deferred_load(func),
]),
}
};
@@ -3505,11 +3521,19 @@ impl FuncEnvironment<'_> {
.alias_regions
.vmctx_vmmemory_pointer(pos, vmctx, def_index);
self.alias_regions
- .vmmemory_definition_current_length_atomic(pos, mem_ptr)
+ .vm_memory_definition()
+ .current_length()
+ .load_atomic(pos, mem_ptr)
} else {
+ // A defined, owned memory's `VMMemoryDefinition` is inlined into
+ // the `vmctx` at an absolute offset.
let owned_index = self.module.owned_memory_index(def_index);
+ let vmctx_off = self.offsets.vmctx_vmmemory_definition(owned_index);
self.alias_regions
- .vmctx_vmmemory_definition_current_length(pos, owned_index, vmctx)
+ .vm_memory_definition()
+ .current_length()
+ .relative_to(vmctx_off)
+ .load(pos, vmctx)
}
} else {
let mem_ptr = self
@@ -3517,10 +3541,14 @@ impl FuncEnvironment<'_> {
.vmctx_vmmemory_import_from(pos, vmctx, index);
if is_shared {
self.alias_regions
- .vmmemory_definition_current_length_atomic(pos, mem_ptr)
+ .vm_memory_definition()
+ .current_length()
+ .load_atomic(pos, mem_ptr)
} else {
self.alias_regions
- .vmmemory_definition_current_length(pos, mem_ptr)
+ .vm_memory_definition()
+ .current_length()
+ .load(pos, mem_ptr)
}
}
}
diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rs
index 9c5f3b3308..129348dfd5 100644
--- a/crates/environ/src/component/vmcomponent_offsets.rs
+++ b/crates/environ/src/component/vmcomponent_offsets.rs
@@ -172,8 +172,8 @@ impl<P: PtrSize> VMComponentOffsets<P> {
size(builtins) = ret.ptr.size(),
size(vm_store_context) = ret.ptr.size(),
align(16),
- size(may_leave) = cmul(ret.num_runtime_component_instances, ret.ptr.size_of_vmglobal_definition()),
- size(task_may_block) = ret.ptr.size_of_vmglobal_definition(),
+ size(may_leave) = cmul(ret.num_runtime_component_instances, ret.ptr.vm_global_definition().size()),
+ size(task_may_block) = ret.ptr.vm_global_definition().size(),
align(u32::from(ret.ptr.size())),
size(trampoline_func_refs) = cmul(ret.num_trampolines, ret.ptr.size_of_vm_func_ref()),
size(intrinsic_func_refs) = cmul(ret.num_unsafe_intrinsics, ret.ptr.size_of_vm_func_ref()),
@@ -218,7 +218,7 @@ impl<P: PtrSize> VMComponentOffsets<P> {
#[inline]
pub fn may_leave(&self, index: RuntimeComponentInstanceIndex) -> u32 {
assert!(index.as_u32() < self.num_runtime_component_instances);
- self.may_leave + index.as_u32() * u32::from(self.ptr.size_of_vmglobal_definition())
+ self.may_leave + index.as_u32() * u32::from(self.ptr.vm_global_definition().size())
}
/// The offset of the `task_may_block` field.
diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs
index cb7b11bd25..2be10ed159 100644
--- a/crates/environ/src/lib.rs
+++ b/crates/environ/src/lib.rs
@@ -42,6 +42,8 @@ mod string_pool;
mod trap_encoding;
mod tunables;
mod types;
+#[macro_use]
+mod vmtypes;
mod vmoffsets;
mod wasm_error;
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 91ffadeaf9..970009e7c7 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -62,6 +62,159 @@ fn align(offset: u32, width: u32) -> u32 {
(offset + (width - 1)) / width * width
}
+/// Generate the [`offsets`] module: a `struct VMFoo<P: PtrSize>(P)` for each
+/// `VM*` type, with a method per field returning that field's offset, plus
+/// `size` and `align` methods.
+macro_rules! define_vm_type_offsets {
+ // Classify a field type to its size in bytes as a `u32`, given `$p` (the
+ // target pointer size as a `u8`).
+ (@size ($p:expr) VmPtr < u8 >) => { u32::from($p) };
+ (@size ($p:expr) AtomicUsize) => { u32::from($p) };
+ (@size ($p:expr) usize) => { u32::from($p) };
+ (@size ($p:expr) [u8; 16]) => { 16u32 };
+ (@size ($p:expr) VMSharedTypeIndex) => { u32::from(($p).size_of_vmshared_type_index()) };
+
+ // Classify a field type to its alignment in bytes as a `u32`, given `$p`
+ // (the target pointer size as a `u8`).
+ (@align ($p:expr) VmPtr < u8 >) => { u32::from($p) };
+ (@align ($p:expr) AtomicUsize) => { u32::from($p) };
+ (@align ($p:expr) usize) => { u32::from($p) };
+ (@align ($p:expr) [u8; 16]) => { 16u32 };
+ (@align ($p:expr) VMSharedTypeIndex) => { u32::from(($p).align_of_vmshared_type_index()) };
+
+ // Classify a `#[repr(...)]` to the minimum alignment it forces, as a `u32`.
+ (@repr_align C) => { 1u32 };
+ (@repr_align C, align($n:literal)) => {{ let align: u32 = $n; align }};
+
+ // Emit a `pub fn` per field returning that field's offset, computed by
+ // accumulating the aligned size of each preceding field. `$p`/`$o` are
+ // caller-minted identifiers (for the pointer size and running offset) that
+ // are threaded through the recursion so their hygiene stays consistent
+ // across the emitted `let` bindings.
+ (@fields $Name:ident ($p:ident, $o:ident) prefix( $($prefix:tt)* )) => {};
+ (@fields $Name:ident ($p:ident, $o:ident) prefix( $($prefix:tt)* )
+ [ $fname:ident : $fty:tt $(< $fgen:tt >)? ]
+ $($rest:tt)*
+ ) => {
+ #[doc = concat!(
+ "The offset of the `", stringify!($fname),
+ "` field of `", stringify!($Name), "`."
+ )]
+ #[inline]
+ pub fn $fname(&self) -> u8 {
+ let $p = self.0.size();
+ let $o: u32 = 0;
+ $($prefix)*
+ let $o = align($o, define_vm_type_offsets!(@align ($p) $fty $(< $fgen >)?));
+ let _ = $p;
+ u8::try_from($o).unwrap()
+ }
+ define_vm_type_offsets!(@fields $Name ($p, $o)
+ prefix(
+ $($prefix)*
+ let $o = align($o, define_vm_type_offsets!(@align ($p) $fty $(< $fgen >)?));
+ let $o = $o + define_vm_type_offsets!(@size ($p) $fty $(< $fgen >)?);
+ )
+ $($rest)*
+ );
+ };
+
+ // Top-level entry: the list of `VM*` type definitions.
+ ( $(
+ $(#[doc = $sdoc:literal])*
+ $(#[derive($($d:ident),*)])?
+ #[repr($($repr:tt)*)]
+ #[snake_name = $snake:ident]
+ $svis:vis struct $Name:ident {
+ $(
+ $(#[doc = $fdoc:literal])*
+ $(#[readonly])?
+ $(#[can_move])?
+ $fvis:vis $fname:ident : $fty:tt $(< $fgen:tt >)? ,
+ )*
+ }
+ )* ) => {
+ /// Offsets of fields within the various `VM*` types, parameterized over
+ /// a target [`PtrSize`] so that they can be computed during cross
+ /// compilation.
+ ///
+ /// These types are namespaced within their own module so that they never
+ /// collide with the real definitions of the `VM*` types themselves.
+ pub mod offsets {
+ use super::{align, PtrSize};
+
+ $(
+ #[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")]
+ pub struct $Name<P: PtrSize>(pub P);
+
+ impl<P: PtrSize> $Name<P> {
+ define_vm_type_offsets!(@fields $Name (p, o) prefix()
+ $( [ $fname : $fty $(< $fgen >)? ] )*
+ );
+
+ #[doc = concat!("The alignment of the `", stringify!($Name), "` type.")]
+ #[inline]
+ pub fn align(&self) -> u8 {
+ let p = self.0.size();
+ let a: u32 = define_vm_type_offsets!(@repr_align $($repr)*);
+ $(
+ let a = core::cmp::max(
+ a,
+ define_vm_type_offsets!(@align (p) $fty $(< $fgen >)?),
+ );
+ )*
+ let _ = p;
+ u8::try_from(a).unwrap()
+ }
+
+ #[doc = concat!("The size of the `", stringify!($Name), "` type.")]
+ #[inline]
+ pub fn size(&self) -> u8 {
+ let p = self.0.size();
+ let o: u32 = 0;
+ $(
+ let o = align(o, define_vm_type_offsets!(@align (p) $fty $(< $fgen >)?));
+ let o = o + define_vm_type_offsets!(@size (p) $fty $(< $fgen >)?);
+ )*
+ let o = align(o, u32::from(self.align()));
+ let _ = p;
+ u8::try_from(o).unwrap()
+ }
+ }
+ )*
+ }
+ };
+}
+for_each_vm_type!(define_vm_type_offsets);
+
+/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to [`PtrSize`] for
+/// each `VM*` type, using the `#[snake_name = ...]` attribute for the method
+/// name.
+macro_rules! define_ptr_size_vm_type_accessors {
+ ( $(
+ $(#[doc = $sdoc:literal])*
+ $(#[derive($($d:ident),*)])?
+ #[repr($($repr:tt)*)]
+ #[snake_name = $snake:ident]
+ $svis:vis struct $Name:ident {
+ $(
+ $(#[doc = $fdoc:literal])*
+ $(#[readonly])?
+ $(#[can_move])?
+ $fvis:vis $fname:ident : $fty:tt $(< $fgen:tt >)? ,
+ )*
+ }
+ )* ) => {
+ $(
+ #[doc = concat!("Get the [`offsets::", stringify!($Name), "`] offsets for this pointer size.")]
+ #[inline]
+ fn $snake(&self) -> offsets::$Name<&Self> {
+ offsets::$Name(self)
+ }
+ )*
+ };
+}
+
/// This class computes offsets to fields within `VMContext` and other
/// related structs that JIT code accesses directly.
#[derive(Debug, Clone, Copy)]
@@ -119,6 +272,11 @@ pub trait PtrSize {
/// Returns the pointer size, in bytes, for the target.
fn size(&self) -> u8;
+ // Generate a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor for each
+ // `VM*` type, giving access to that type's field offsets, size, and
+ // alignment for this pointer size.
+ for_each_vm_type!(define_ptr_size_vm_type_accessors);
+
/// The offset of the `VMContext::store_context` field
fn vmcontext_store_context(&self) -> u8 {
u8::try_from(align(
@@ -169,23 +327,16 @@ pub trait PtrSize {
4
}
- /// Return the size of `VMGlobalDefinition`; this is the size of the largest value type (i.e. a
- /// V128).
- #[inline]
- fn size_of_vmglobal_definition(&self) -> u8 {
- 16
- }
-
- /// Return the size of `VMTagDefinition`.
+ /// Return the alignment of `VMSharedTypeIndex`.
#[inline]
- fn size_of_vmtag_definition(&self) -> u8 {
+ fn align_of_vmshared_type_index(&self) -> u8 {
4
}
/// This is the size of the largest value type (i.e. a V128).
#[inline]
fn maximum_value_size(&self) -> u8 {
- self.size_of_vmglobal_definition()
+ self.vm_global_definition().size()
}
// Offsets within `VMStoreContext`
@@ -223,14 +374,14 @@ pub trait PtrSize {
/// Return the offset of the `gc_heap.base` field within a `VMStoreContext`.
fn vmstore_context_gc_heap_base(&self) -> u8 {
- let offset = self.vmstore_context_gc_heap() + self.vmmemory_definition_base();
+ let offset = self.vmstore_context_gc_heap() + self.vm_memory_definition().base();
debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());
offset
}
/// Return the offset of the `gc_heap.current_length` field within a `VMStoreContext`.
fn vmstore_context_gc_heap_current_length(&self) -> u8 {
- let offset = self.vmstore_context_gc_heap() + self.vmmemory_definition_current_length();
+ let offset = self.vmstore_context_gc_heap() + self.vm_memory_definition().current_length();
debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());
offset
}
@@ -238,7 +389,7 @@ pub trait PtrSize {
/// Return the offset of the `last_wasm_exit_trampoline_fp` field
/// of `VMStoreContext`.
fn vmstore_context_last_wasm_exit_trampoline_fp(&self) -> u8 {
- self.vmstore_context_gc_heap() + self.size_of_vmmemory_definition()
+ self.vmstore_context_gc_heap() + self.vm_memory_definition().size()
}
/// Return the offset of the `last_wasm_exit_pc` field of `VMStoreContext`.
@@ -325,26 +476,6 @@ pub trait PtrSize {
align(u32::from(unaligned), u32::from(self.size())) as u8
}
- // Offsets within `VMMemoryDefinition`
-
- /// The offset of the `base` field.
- #[inline]
- fn vmmemory_definition_base(&self) -> u8 {
- 0 * self.size()
- }
-
- /// The offset of the `current_length` field.
- #[inline]
- fn vmmemory_definition_current_length(&self) -> u8 {
- 1 * self.size()
- }
-
- /// Return the size of `VMMemoryDefinition`.
- #[inline]
- fn size_of_vmmemory_definition(&self) -> u8 {
- 2 * self.size()
- }
-
/// Return the size of `*mut VMMemoryDefinition`.
#[inline]
fn size_of_vmmemory_pointer(&self) -> u8 {
@@ -650,6 +781,16 @@ impl PtrSize for u8 {
}
}
+impl<P> PtrSize for &'_ P
+where
+ P: PtrSize + ?Sized,
+{
+ #[inline]
+ fn size(&self) -> u8 {
+ (**self).size()
+ }
+}
+
/// Used to construct a `VMOffsets`
#[derive(Debug, Clone, Copy)]
pub struct VMOffsetsFields<P> {
@@ -867,7 +1008,7 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
size(defined_memories)
= cmul(ret.num_defined_memories, ret.ptr.size_of_vmmemory_pointer()),
size(owned_memories)
- = cmul(ret.num_owned_memories, ret.ptr.size_of_vmmemory_definition()),
+ = cmul(ret.num_owned_memories, ret.ptr.vm_memory_definition().size()),
size(imported_functions)
= cmul(ret.num_imported_functions, ret.size_of_vmfunction_import()),
size(imported_tables)
@@ -877,12 +1018,12 @@ impl<P: PtrSize> From<VMOffsetsFields<P>> for VMOffsets<P> {
size(imported_tags)
= cmul(ret.num_imported_tags, ret.size_of_vmtag_import()),
size(defined_tables)
- = cmul(ret.num_defined_tables, ret.size_of_vmtable_definition()),
+ = cmul(ret.num_defined_tables, ret.ptr.vm_table_definition().size()),
align(16),
size(defined_globals)
- = cmul(ret.num_defined_globals, ret.ptr.size_of_vmglobal_definition()),
+ = cmul(ret.num_defined_globals, ret.ptr.vm_global_definition().size()),
size(defined_tags)
- = cmul(ret.num_defined_tags, ret.ptr.size_of_vmtag_definition()),
+ = cmul(ret.num_defined_tags, ret.ptr.vm_tag_definition().size()),
size(defined_func_refs) = cmul(
ret.num_escaped_funcs,
ret.ptr.size_of_vm_func_ref(),
@@ -971,28 +1112,11 @@ impl<P: PtrSize> VMOffsets<P> {
/// Offsets for `VMTableDefinition`.
impl<P: PtrSize> VMOffsets<P> {
- /// The offset of the `base` field.
- #[inline]
- pub fn vmtable_definition_base(&self) -> u8 {
- 0 * self.pointer_size()
- }
-
- /// The offset of the `current_elements` field.
- pub fn vmtable_definition_current_elements(&self) -> u8 {
- 1 * self.pointer_size()
- }
-
/// The size of the `current_elements` field.
#[inline]
pub fn size_of_vmtable_definition_current_elements(&self) -> u8 {
self.pointer_size()
}
-
- /// Return the size of `VMTableDefinition`.
- #[inline]
- pub fn size_of_vmtable_definition(&self) -> u8 {
- 2 * self.pointer_size()
- }
}
/// Offsets for `VMMemoryImport`.
@@ -1203,7 +1327,8 @@ impl<P: PtrSize> VMOffsets<P> {
#[inline]
pub fn vmctx_vmtable_definition(&self, index: DefinedTableIndex) -> u32 {
assert!(index.as_u32() < self.num_defined_tables);
- self.vmctx_tables_begin() + index.as_u32() * u32::from(self.size_of_vmtable_definition())
+ self.vmctx_tables_begin()
+ + index.as_u32() * u32::from(self.ptr.vm_table_definition().size())
}
/// Return the offset to the `*mut VMMemoryDefinition` at index `index`.
@@ -1219,7 +1344,7 @@ impl<P: PtrSize> VMOffsets<P> {
pub fn vmctx_vmmemory_definition(&self, index: OwnedMemoryIndex) -> u32 {
assert!(index.as_u32() < self.num_owned_memories);
self.vmctx_owned_memories_begin()
- + index.as_u32() * u32::from(self.ptr.size_of_vmmemory_definition())
+ + index.as_u32() * u32::from(self.ptr.vm_memory_definition().size())
}
/// Return the offset to the `VMGlobalDefinition` index `index`.
@@ -1227,14 +1352,14 @@ impl<P: PtrSize> VMOffsets<P> {
pub fn vmctx_vmglobal_definition(&self, index: DefinedGlobalIndex) -> u32 {
assert!(index.as_u32() < self.num_defined_globals);
self.vmctx_globals_begin()
- + index.as_u32() * u32::from(self.ptr.size_of_vmglobal_definition())
+ + index.as_u32() * u32::from(self.ptr.vm_global_definition().size())
}
/// Return the offset to the `VMTagDefinition` index `index`.
#[inline]
pub fn vmctx_vmtag_definition(&self, index: DefinedTagIndex) -> u32 {
assert!(index.as_u32() < self.num_defined_tags);
- self.vmctx_tags_begin() + index.as_u32() * u32::from(self.ptr.size_of_vmtag_definition())
+ self.vmctx_tags_begin() + index.as_u32() * u32::from(self.ptr.vm_tag_definition().size())
}
/// Return the offset to the `VMFuncRef` for the given function
@@ -1299,13 +1424,14 @@ impl<P: PtrSize> VMOffsets<P> {
/// Return the offset to the `base` field in `VMTableDefinition` index `index`.
#[inline]
pub fn vmctx_vmtable_definition_base(&self, index: DefinedTableIndex) -> u32 {
- self.vmctx_vmtable_definition(index) + u32::from(self.vmtable_definition_base())
+ self.vmctx_vmtable_definition(index) + u32::from(self.ptr.vm_table_definition().base())
}
/// Return the offset to the `current_elements` field in `VMTableDefinition` index `index`.
#[inline]
pub fn vmctx_vmtable_definition_current_elements(&self, index: DefinedTableIndex) -> u32 {
- self.vmctx_vmtable_definition(index) + u32::from(self.vmtable_definition_current_elements())
+ self.vmctx_vmtable_definition(index)
+ + u32::from(self.ptr.vm_table_definition().current_elements())
}
/// Return the offset to the `from` field in `VMMemoryImport` index `index`.
@@ -1317,14 +1443,14 @@ impl<P: PtrSize> VMOffsets<P> {
/// Return the offset to the `base` field in `VMMemoryDefinition` index `index`.
#[inline]
pub fn vmctx_vmmemory_definition_base(&self, index: OwnedMemoryIndex) -> u32 {
- self.vmctx_vmmemory_definition(index) + u32::from(self.ptr.vmmemory_definition_base())
+ self.vmctx_vmmemory_definition(index) + u32::from(self.ptr.vm_memory_definition().base())
}
/// Return the offset to the `current_length` field in `VMMemoryDefinition` index `index`.
#[inline]
pub fn vmctx_vmmemory_definition_current_length(&self, index: OwnedMemoryIndex) -> u32 {
self.vmctx_vmmemory_definition(index)
- + u32::from(self.ptr.vmmemory_definition_current_length())
+ + u32::from(self.ptr.vm_memory_definition().current_length())
}
/// Return the offset to the `from` field in `VMGlobalImport` index `index`.
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
new file mode 100644
index 0000000000..5f061d6f6f
--- /dev/null
+++ b/crates/environ/src/vmtypes.rs
@@ -0,0 +1,90 @@
+//! Centralized definitions of the various `VM*` types whose layout is shared
+//! between the runtime (which uses the actual structures) and the compiler
+//! (which uses the types' offsets and has per-type alias regions).
+//!
+//! To keep these in sync, the shape of each type is defined exactly once here,
+//! via the higher-order [`for_each_vm_type!`] macro, and each consumer
+//! generates its view of the type from that single source of truth.
+
+/// Invoke the given macro `$mac` once, passing it the definitions of each of the
+/// `VM*` types whose layout is shared between the runtime, compilation offsets,
+/// and Cranelift alias regions.
+///
+/// This is a higher-order macro: callers define a `macro_rules!` macro that
+/// matches the grammar defined below and pass its name as an argument to this
+/// macro's invocation, e.g. `for_each_vm_type!(define_vm_types)`.
+///
+/// # Grammar
+///
+/// Each type is emitted as a struct definition preceded by:
+///
+/// * Doc-comment attributes (`#[doc = "..."]`).
+///
+/// * An optional `#[derive(...)]` attribute.
+///
+/// * A `#[repr(...)]` attribute.
+///
+/// * A `#[snake_name = <ident>]` attribute giving the type's name in
+/// `snake_case`, used to generate accessor method names.
+///
+/// Each field may be preceded by doc-comment attributes and, optionally, the
+/// marker attributes `#[readonly]` and/or `#[can_move]` (in that order) which
+/// describe how Cranelift may treat loads and stores of that field.
+#[macro_export]
+macro_rules! for_each_vm_type {
+ ($mac:ident) => {
+ $mac! {
+ /// The fields compiled code needs to access to utilize a WebAssembly linear
+ /// memory defined within the instance, namely the start address and the
+ /// size in bytes.
+ #[derive(Debug)]
+ #[repr(C)]
+ #[snake_name = vm_memory_definition]
+ pub struct VMMemoryDefinition {
+ /// The start address.
+ pub base: VmPtr<u8>,
+
+ /// The current logical size of this linear memory in bytes.
+ ///
+ /// This is atomic because shared memories must be able to grow their length
+ /// atomically. For relaxed access, see
+ /// [`VMMemoryDefinition::current_length()`].
+ pub current_length: AtomicUsize,
+ }
+
+ /// The fields compiled code needs to access to utilize a WebAssembly table
+ /// defined within the instance.
+ #[derive(Debug, Copy, Clone)]
+ #[repr(C)]
+ #[snake_name = vm_table_definition]
+ pub struct VMTableDefinition {
+ /// Pointer to the table data.
+ pub base: VmPtr<u8>,
+
+ /// The current number of elements in the table.
+ pub current_elements: usize,
+ }
+
+ /// The storage for a WebAssembly global defined within the instance.
+ ///
+ /// TODO: Pack the globals more densely, rather than using the same size
+ /// for every type.
+ #[derive(Debug)]
+ #[repr(C, align(16))]
+ #[snake_name = vm_global_definition]
+ pub struct VMGlobalDefinition {
+ /// The raw storage for a global's value.
+ storage: [u8; 16],
+ }
+
+ /// A WebAssembly tag defined within the instance.
+ #[derive(Debug)]
+ #[repr(C)]
+ #[snake_name = vm_tag_definition]
+ pub struct VMTagDefinition {
+ /// Function signature's type id.
+ pub type_index: VMSharedTypeIndex,
+ }
+ }
+ };
+}
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index ceaa88cbb3..e207502c4a 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -417,25 +417,85 @@ mod test_vmtag_import {
}
}
-/// The fields compiled code needs to access to utilize a WebAssembly linear
-/// memory defined within the instance, namely the start address and the
-/// size in bytes.
-#[derive(Debug)]
-#[repr(C)]
-pub struct VMMemoryDefinition {
- /// The start address.
- pub base: VmPtr<u8>,
+/// Define the runtime definitions of the shared `VM*` types.
+macro_rules! define_vm_types {
+ ( $(
+ $(#[doc = $sdoc:literal])*
+ $(#[derive($($d:ident),*)])?
+ #[repr($($repr:tt)*)]
+ #[snake_name = $snake:ident]
+ $svis:vis struct $Name:ident {
+ $(
+ $(#[doc = $fdoc:literal])*
+ $(#[readonly])?
+ $(#[can_move])?
+ $fvis:vis $fname:ident : $fty:tt $(< $fgen:tt >)? ,
+ )*
+ }
+ )* ) => {
+ $(
+ $(#[doc = $sdoc])*
+ $(#[derive($($d),*)])?
+ #[repr($($repr)*)]
+ $svis struct $Name {
+ $(
+ $(#[doc = $fdoc])*
+ $fvis $fname: $fty $(< $fgen >)?,
+ )*
+ }
+ )*
- /// The current logical size of this linear memory in bytes.
- ///
- /// This is atomic because shared memories must be able to grow their length
- /// atomically. For relaxed access, see
- /// [`VMMemoryDefinition::current_length()`].
- pub current_length: AtomicUsize,
+ #[cfg(test)]
+ mod test_vm_type_layouts {
+ use super::{ $( $Name, )* };
+ use core::mem::{align_of, offset_of, size_of};
+ use wasmtime_environ::{HostPtr, PtrSize};
+
+ $(
+ #[test]
+ fn $snake() {
+ let host = HostPtr;
+ let offsets = host.$snake();
+
+ let expected = usize::from(offsets.size());
+ let actual = size_of::<$Name>();
+ assert_eq!(
+ expected,
+ actual,
+ "size of {} failed: {expected} (expected) != {actual} (actual)",
+ stringify!($Name),
+ );
+
+ let expected = usize::from(offsets.align());
+ let actual = align_of::<$Name>();
+ assert_eq!(
+ expected,
+ actual,
+ "alignment of {} failed: {expected} (expected) != {actual} (actual)",
+ stringify!($Name),
+ );
+
+ $(
+ let expected = usize::from(offsets.$fname());
+ let actual = offset_of!($Name, $fname);
+ assert_eq!(
+ expected,
+ actual,
+ "offset of {}::{} failed: {expected} (expected) != {actual} (actual)",
+ stringify!($Name),
+ stringify!($fname),
+ );
+ )*
+ }
+ )*
+ }
+ };
}
+wasmtime_environ::for_each_vm_type!(define_vm_types);
-// SAFETY: the above definition has `repr(C)` and each field individually
-// implements `VmSafe`, which satisfies the requirements of this trait.
+// SAFETY: `VMMemoryDefinition` is generated with `#[repr(C)]` and each field
+// individually implements `VmSafe`, which satisfies the requirements of this
+// trait.
unsafe impl VmSafe for VMMemoryDefinition {}
impl VMMemoryDefinition {
@@ -464,98 +524,19 @@ impl VMMemoryDefinition {
}
}
-#[cfg(test)]
-mod test_vmmemory_definition {
- use super::VMMemoryDefinition;
- use core::mem::offset_of;
- use std::mem::size_of;
- use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
-
- #[test]
- fn check_vmmemory_definition_offsets() {
- let module = Module::new(StaticModuleIndex::from_u32(0));
- let offsets = VMOffsets::new(HostPtr, &module);
- assert_eq!(
- size_of::<VMMemoryDefinition>(),
- usize::from(offsets.ptr.size_of_vmmemory_definition())
- );
- assert_eq!(
- offset_of!(VMMemoryDefinition, base),
- usize::from(offsets.ptr.vmmemory_definition_base())
- );
- assert_eq!(
- offset_of!(VMMemoryDefinition, current_length),
- usize::from(offsets.ptr.vmmemory_definition_current_length())
- );
- /* TODO: Assert that the size of `current_length` matches.
- assert_eq!(
- size_of::<VMMemoryDefinition::current_length>(),
- usize::from(offsets.size_of_vmmemory_definition_current_length())
- );
- */
- }
-}
-
-/// The fields compiled code needs to access to utilize a WebAssembly table
-/// defined within the instance.
-#[derive(Debug, Copy, Clone)]
-#[repr(C)]
-pub struct VMTableDefinition {
- /// Pointer to the table data.
- pub base: VmPtr<u8>,
-
- /// The current number of elements in the table.
- pub current_elements: usize,
-}
-
-// SAFETY: the above structure is repr(C) and only contains `VmSafe` fields.
+// SAFETY: `VMTableDefinition` is generated with `#[repr(C)]` and only contains
+// `VmSafe` fields.
unsafe impl VmSafe for VMTableDefinition {}
-#[cfg(test)]
-mod test_vmtable_definition {
- use super::VMTableDefinition;
- use core::mem::offset_of;
- use std::mem::size_of;
- use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};
-
- #[test]
- fn check_vmtable_definition_offsets() {
- let module = Module::new(StaticModuleIndex::from_u32(0));
- let offsets = VMOffsets::new(HostPtr, &module);
- assert_eq!(
- size_of::<VMTableDefinition>(),
- usize::from(offsets.size_of_vmtable_definition())
- );
- assert_eq!(
- offset_of!(VMTableDefinition, base),
- usize::from(offsets.vmtable_definition_base())
- );
- assert_eq!(
- offset_of!(VMTableDefinition, current_elements),
- usize::from(offsets.vmtable_definition_current_elements())
- );
- }
-}
-
-/// The storage for a WebAssembly global defined within the instance.
-///
-/// TODO: Pack the globals more densely, rather than using the same size
-/// for every type.
-#[derive(Debug)]
-#[repr(C, align(16))]
-pub struct VMGlobalDefinition {
- storage: [u8; 16],
- // If more elements are added here, remember to add offset_of tests below!
-}
-
-// SAFETY: the above structure is repr(C) and only contains `VmSafe` fields.
+// SAFETY: `VMGlobalDefinition` is generated with `#[repr(C)]` and only contains
+// `VmSafe` fields.
unsafe impl VmSafe for VMGlobalDefinition {}
#[cfg(test)]
mod test_vmglobal_definition {
use super::VMGlobalDefinition;
use std::mem::{align_of, size_of};
- use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};
#[test]
fn check_vmglobal_definition_alignment() {
@@ -568,16 +549,6 @@ mod test_vmglobal_definition {
assert!(align_of::<VMGlobalDefinition>() >= align_of::<[f64; 2]>());
}
- #[test]
- fn check_vmglobal_definition_offsets() {
- let module = Module::new(StaticModuleIndex::from_u32(0));
- let offsets = VMOffsets::new(HostPtr, &module);
- assert_eq!(
- size_of::<VMGlobalDefinition>(),
- usize::from(offsets.ptr.size_of_vmglobal_definition())
- );
- }
-
#[test]
fn check_vmglobal_begins_aligned() {
let module = Module::new(StaticModuleIndex::from_u32(0));
@@ -777,40 +748,19 @@ mod test_vmshared_type_index {
}
}
-/// A WebAssembly tag defined within the instance.
-///
-#[derive(Debug)]
-#[repr(C)]
-pub struct VMTagDefinition {
- /// Function signature's type id.
- pub type_index: VMSharedTypeIndex,
-}
-
impl VMTagDefinition {
pub fn new(type_index: VMSharedTypeIndex) -> Self {
Self { type_index }
}
}
-// SAFETY: the above structure is repr(C) and only contains VmSafe
-// fields.
+// SAFETY: `VMTagDefinition` is generated with `#[repr(C)]` and only contains
+// `VmSafe` fields.
unsafe impl VmSafe for VMTagDefinition {}
#[cfg(test)]
mod test_vmtag_definition {
- use super::VMTagDefinition;
- use std::mem::size_of;
- use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
-
- #[test]
- fn check_vmtag_definition_offsets() {
- let module = Module::new(StaticModuleIndex::from_u32(0));
- let offsets = VMOffsets::new(HostPtr, &module);
- assert_eq!(
- size_of::<VMTagDefinition>(),
- usize::from(offsets.ptr.size_of_vmtag_definition())
- );
- }
+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};
#[test]
fn check_vmtag_begins_aligned() {
diff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rs
index 2d88150670..f53fdba350 100644
--- a/winch/codegen/src/codegen/env.rs
+++ b/winch/codegen/src/codegen/env.rs
@@ -244,8 +244,12 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
),
None => (
Some(self.vmoffsets.vmctx_vmtable_from(index)),
- self.vmoffsets.vmtable_definition_base().into(),
- self.vmoffsets.vmtable_definition_current_elements().into(),
+ self.vmoffsets.ptr.vm_table_definition().base().into(),
+ self.vmoffsets
+ .ptr
+ .vm_table_definition()
+ .current_elements()
+ .into(),
),
};
@@ -276,10 +280,11 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
if is_shared {
(
Some(self.vmoffsets.vmctx_vmmemory_pointer(defined)),
- self.vmoffsets.ptr.vmmemory_definition_base().into(),
+ self.vmoffsets.ptr.vm_memory_definition().base().into(),
self.vmoffsets
.ptr
- .vmmemory_definition_current_length()
+ .vm_memory_definition()
+ .current_length()
.into(),
)
} else {
@@ -294,10 +299,11 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {
}
None => (
Some(self.vmoffsets.vmctx_vmmemory_import_from(index)),
- self.vmoffsets.ptr.vmmemory_definition_base().into(),
+ self.vmoffsets.ptr.vm_memory_definition().base().into(),
self.vmoffsets
.ptr
- .vmmemory_definition_current_length()
+ .vm_memory_definition()
+ .current_length()
.into(),
),
};