A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
//! Alias regions for the memory that compiled Wasm code accesses.
//!
//! Every load and store Wasmtime emits carries a
//! `cranelift_codegen::ir::AliasRegion` naming the memory it touches, so that
//! Cranelift's alias analysis can better eliminate redundant loads, dead
//! stores, etc. This module, and primarily its `AliasRegions` type, parcels
//! those regions out.
//!
//! `AliasRegions` accessors come from two places:
//!
//! 1. Fields of the `VM*` types and of the vmctx types get one region per
//! field, and an accessor per field is generated from our
//! `for_each_vm[ctx]_type!` macros. An `enum` gets only two: one for its
//! tag, one shared by all variants' fields, which overlap in memory.
//!
//! 2. There are a handful of hand-written methods for memory regions that are
//! not part of the vmctx or a `VM*` type at all (linear memories, the GC
//! heap, stack slots, etc...).
//!
//! We strive to avoid adding new hand-written methods as much as possible.
use crate::translate::Load;
use cranelift_codegen::{
cursor::FuncCursor,
ir::{self, InstBuilder as _},
};
use std::collections::HashMap;
use std::hash::{Hash as _, Hasher};
use wasmtime_environ::{
BuiltinFunctionIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, GetPtrSize,
MemoryIndex, ModuleInternedTypeIndex, ModuleTypesBuilder, NUM_COMPONENT_CONTEXT_SLOTS,
PtrSize as _, RuntimeDataIndex, StaticModuleIndex, VMOffsets, VmctxArrayIndex as _,
WasmCompositeInnerType,
component::{
ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,
RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,
RuntimeReallocIndex, VMComponentOffsets,
},
};
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
enum VmType {
VMContext,
VMStoreContext,
VMMemoryDefinition,
VMTableDefinition,
#[allow(
dead_code,
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
)]
VMTagDefinition,
VMComponentContext,
VMDrcHeapData,
VMCopyingHeapData,
VMNullHeapData,
VMDeferredThread,
VMStackLimits,
#[allow(
dead_code,
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
)]
VMLazyThread,
VMContRef,
VMCommonStackInformation,
VMHostArray,
ContinuationStackMemory,
VMFunctionImport,
VMMemoryImport,
VMTableImport,
VMTagImport,
VMGlobalImport,
VMFuncRef,
TypeIdsArray,
EpochCounter,
BuiltinFunctionsArray,
ComponentBuiltinFunctionsArray,
HostValRaw,
VMPayloads,
VMRawContObj,
VMGcHeader,
VMDrcHeader,
VMCopyingHeader,
#[allow(
dead_code,
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
)]
VMAsanStackSupport,
VMResourceIndex,
VMHandleTable,
VMHandleTableSlot,
VMTypedResource,
#[allow(
dead_code,
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
)]
VMScope,
#[allow(
dead_code,
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
)]
VMTransmitLocalState,
}
/// A key that uniquely identifies an alias region across an entire compilation.
///
/// This is used to assign stable `user_id`s to `AliasRegionData` entries so
/// that alias regions can be deduplicated during inlining.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
enum AliasRegionKey {
/// An access of a field within a VM data structure of type `ty`.
Vm {
/// The type of VM data structure being accessed.
ty: VmType,
/// The offset of the accessed field *within* the `ty` structure (or
/// the base offset of the array, for array fields, or the payload
/// offset, for an enum's variants' fields).
offset: u32,
},
/// An access of a memory that crosses a module boundary and whose
/// definition we do not statically know (shared across all such memories).
PublicMemory,
/// A defined memory access.
DefinedMemory {
/// The static module index.
module: StaticModuleIndex,
/// The defined memory index within the module.
index: DefinedMemoryIndex,
},
/// An access of a table that crosses a module boundary and whose definition
/// we do not statically know (shared across all such tables).
PublicTable,
/// A defined table access.
DefinedTable {
/// The static module index.
module: StaticModuleIndex,
/// The defined table index within the module.
index: DefinedTableIndex,
},
/// An access of a global that crosses a module boundary and whose definition
/// we do not statically know (shared across all such globals).
PublicGlobal,
/// A defined global access.
DefinedGlobal {
/// The static module index.
module: StaticModuleIndex,
/// The defined global index within the module.
index: DefinedGlobalIndex,
},
/// An access of a GC array's `length` word.
GcArrayLength,
/// An access of a GC struct's field.
GcStructField {
/// The first supertype that introduced this field.
ty: ModuleInternedTypeIndex,
/// The index of the field being accessed.
field: u32,
},
/// An access of a GC array's elements, which all share one region.
GcArrayElements {
/// The first supertype that introduced these elements.
ty: ModuleInternedTypeIndex,
},
/// An access of an exception object's payload field.
GcExnPayload {
/// The exception type.
exn_ty: ModuleInternedTypeIndex,
/// The payload field being accessed.
field: u32,
},
/// A stack slot access.
Stack {
/// The stack slot being accessed.
slot: ir::StackSlot,
},
/// All unsafe intrinsics share a single alias region.
///
/// By contract, unsafe intrinsics cannot access our internal `VM*` types or
/// linear memories or anything else that has its own dedicated alias
/// region. However, we cannot guarantee anything about the (lack of)
/// aliasing of the embedder's data structures that the unsafe intrinsics do
/// access, so all their accesses are lumped together into the same region.
UnsafeIntrinsicMemory,
/// An access of a `ValRaw` inside a passive element segment.
ElementSegment,
/// An access of the bytes inside a data segment.
DataSegment,
/// An access of an exception object's tag-instance word.
GcExnTagInstance,
/// An access of an exception object's tag-defined word.
GcExnTagDefined,
/// An access of the shadow bytes of a linear memory, for watchpoints.
ShadowMemory,
}
impl AliasRegionKey {
/// Encode this key into a raw `u32` suitable for use as an
/// `AliasRegionData::user_id`.
///
/// We lossily encode the key into a `user_id` via a single-byte hash, so
/// there are at most 256 alias regions. This avoids compile-time blowups
/// associated with many alias regions.
///
/// Note that mapping multiple alias regions onto the same `user_id` is
/// always okay: by collapsing two logical regions into one actual region,
/// we are letting Cranelift believe they *might* alias even when they
/// actually cannot. At worst this prevents some optimization. The opposite,
/// however -- if we were to place accesses that *could* alias in two
/// different regions -- is not sound. That would let Cranelift optimize
/// based on a false premise, leading to misoptimizations and symptoms like
/// memory writes "disappearing".
///
/// Simultaneously, this mapping is independent of the particular set of
/// `AliasRegionKey`s used in a particular Wasm-to-CLIF translation, which
/// is necessary for inlining. If we assigned `user_id = 1` to the first
/// `AliasRegionKey`, `user_id = 2` to the second, etc... then inlining one
/// function into another would mean that the `user_id`s for the same
/// `AliasRegionKey` could be inconsistent with each other, which also leads
/// to the miscompilation scenario described above.
pub(crate) fn into_raw(self) -> u32 {
/// A version of `rustc_hash::FxHasher` that is guaranteed to provide
/// deterministic hashing across Wasmtime builds and processes.
struct AliasRegionHasher {
hash: u64,
}
impl AliasRegionHasher {
fn new(seed: u64) -> Self {
AliasRegionHasher { hash: seed }
}
fn add(&mut self, i: u64) {
const K: u64 = 0x517c_c1b7_2722_0a95;
self.hash = (self.hash.rotate_left(5) ^ i).wrapping_mul(K);
}
}
impl Hasher for AliasRegionHasher {
fn finish(&self) -> u64 {
self.hash
}
fn write(&mut self, bytes: &[u8]) {
for b in bytes {
self.add((*b).into());
}
}
fn write_u8(&mut self, i: u8) {
self.add(i.into());
}
fn write_u16(&mut self, i: u16) {
self.add(i.into());
}
fn write_u32(&mut self, i: u32) {
self.add(i.into());
}
fn write_u64(&mut self, i: u64) {
self.add(i);
}
fn write_u128(&mut self, i: u128) {
self.add(u64::try_from(i & u128::from(u64::MAX)).unwrap());
self.add(u64::try_from(i >> 64).unwrap());
}
fn write_usize(&mut self, i: usize) {
self.add(u64::try_from(i).unwrap());
}
}
/// The `AliasRegionHasher` seed, chosen to shift which keys collide
/// with which.
///
/// Collisions are inevitable because we are lossily mapping our
/// ~"infinite" keys onto only 256 actual alias regions. However, not
/// all collisions are equal. The cost of a pair of globals colliding is
/// marginal. A collision between `VMContext::store_context` and the
/// first defined memory's base pointer, two of the most-frequently used
/// regions in essentially every function, would be expensive.
///
/// This value was picked by brute-force search as one for which none
/// of the following collide with each other: the fixed, non-indexed
/// keys appearing anywhere in `tests/disas`, and the first five defined
/// memories, tables, and globals of the first module. If a new `VM*`
/// field or `AliasRegionKey` variant introduces such a collision later,
/// the `tests/disas` expectations will show it as two regions merging
/// into one, and a new seed can be searched for in the same way.
const SEED: u64 = 15912981;
let mut hasher = AliasRegionHasher::new(SEED);
self.hash(&mut hasher);
let hash = hasher.finish();
// `xor`-fold the hash down to a single byte.
hash.to_le_bytes().into_iter().fold(0, |a, b| a ^ b).into()
}
}
/// Which logical part of a GC object an access targets.
#[derive(Clone, Copy, Debug)]
pub enum GcAccess {
/// An access of a struct's field, by the type at the access site.
StructField {
ty: ModuleInternedTypeIndex,
field: u32,
},
/// An access of one of an array's elements, by the type at the access site.
ArrayElements { ty: ModuleInternedTypeIndex },
/// An access of an exception object's payload field.
ExnPayload {
exn_ty: ModuleInternedTypeIndex,
field: u32,
},
/// An access of `VMGcHeader::kind` or the reserved bits packed into it.
HeaderKind,
/// An access of `VMGcHeader::ty`.
HeaderTypeIndex,
/// An access of `VMDrcHeader::ref_count`.
DrcRefCount,
/// An access of `VMDrcHeader::next_over_approximated_stack_root`.
DrcNextOverApproximatedStackRoot,
/// An access of `VMCopyingHeader::object_size`.
CopyingObjectSize,
/// An access of an array's `length` word.
ArrayLength,
/// An access of an exception object's tag-instance word.
ExnTagInstance,
/// An access of an exception object's tag-defined word.
ExnTagDefined,
}
/// Alias region bookkeeping and load/store helper type.
///
/// This is scoped to a single function: it hands out `ir::AliasRegion`s, which
/// are indices into one function's `ir::AliasRegionSet`, so a given
/// `AliasRegions` must not be reused across functions.
pub struct AliasRegions<Offsets> {
pointer_type: ir::Type,
offsets: Offsets,
/// Cache for `Self::gc_introducer`, keyed on the access site's type and the
/// field index, if any.
gc_introducer_cache: HashMap<(ModuleInternedTypeIndex, u32), ModuleInternedTypeIndex>,
}
impl<Offsets> AliasRegions<Offsets> {
/// Get the offsets this `AliasRegions` computes its field offsets from.
pub fn offsets(&self) -> &Offsets {
&self.offsets
}
/// Make the alias region for a stack map.
pub fn stack_map_region(
regions: &mut ir::AliasRegionSet,
_ty: ir::Type,
slot: ir::StackSlot,
_offset: u32,
) -> Option<ir::AliasRegion> {
Some(Self::insert_region(regions, AliasRegionKey::Stack { slot }))
}
/// Get the alias region for a stack slot.
pub fn stack_slot_region(
&mut self,
func: &mut ir::Function,
slot: ir::StackSlot,
) -> ir::AliasRegion {
self.region(func, AliasRegionKey::Stack { slot })
}
/// Get the alias region for the given key.
fn region(&mut self, func: &mut ir::Function, key: AliasRegionKey) -> ir::AliasRegion {
Self::insert_region(&mut func.dfg.alias_regions, key)
}
/// Get or create the alias region for the given key.
fn insert_region(regions: &mut ir::AliasRegionSet, key: AliasRegionKey) -> ir::AliasRegion {
let user_id = key.into_raw();
let region = regions.insert(ir::AliasRegionData {
user_id,
description: "".into(),
});
log::trace!("alias region: {key:?} => {region} (user_id = {user_id})");
region
}
}
/// A single field within one of Wasmtime's `VM*` types, along with everything
/// needed to emit a correctly alias-analyzed load or store of that field.
///
/// A `Field` is obtained from the `Field`-returning accessors generated by
/// [`define_vm_type_alias_region_helpers!`], for example
/// `alias_regions.vm_memory_definition().base()`.
///
/// This is the evolved form of the old `Load` type: in addition to producing a
/// deferred [`Load`] descriptor (via [`Field::to_deferred_load`]) for use in a
/// `VmctxLoadChain`, a `Field` can directly emit loads and stores relative to a
/// pointer to the containing `VM*` structure.
pub struct Field<'a, Offsets> {
/// The alias-region cache used to get-or-create this field's alias region.
regions: &'a mut AliasRegions<Offsets>,
/// The key identifying this field's alias region.
///
/// This is fixed when the `Field` is created, derived from the field's
/// offset *within* its containing `VM*` type, and is deliberately
/// independent of [`Field::relative_to`]: rebasing the load/store
/// (e.g. to make the access relative to the `vmctx` rather than to a pointer
/// to the containing structure) must not change which alias region the
/// access belongs to.
key: AliasRegionKey,
/// The offset added to the base value when emitting a load or store of this
/// field.
///
/// Initially the field's offset *within* its containing `VM*` type; callers
/// may rebase it via [`Field::relative_to`].
offset: u32,
/// The base memory flags for accesses of this field, before this field's
/// alias region is mixed in.
flags: ir::MemFlagsData,
/// The Cranelift type of this field.
ty: ir::Type,
}
impl<'a, Offsets> Field<'a, Offsets> {
/// Create a new `Field` for the given `offset` within `vm_type`, loaded or
/// stored with the given base `flags` and Cranelift type `ty`.
fn new(
regions: &'a mut AliasRegions<Offsets>,
vm_type: VmType,
offset: u32,
flags: ir::MemFlagsData,
ty: ir::Type,
) -> Self {
Self::new_in_region(regions, vm_type, offset, offset, flags, ty)
}
/// As `new`, but keyed on `region_offset` rather than the field's own
/// `offset`, so that fields overlapping in memory (an enum's variants'
/// fields) share one region instead of claiming they cannot alias.
fn new_in_region(
regions: &'a mut AliasRegions<Offsets>,
vm_type: VmType,
region_offset: u32,
offset: u32,
flags: ir::MemFlagsData,
ty: ir::Type,
) -> Self {
Field {
regions,
key: AliasRegionKey::Vm {
ty: vm_type,
offset: region_offset,
},
offset,
flags,
ty,
}
}
/// Mark accesses of this field as `readonly`.
///
/// Whether a field is `readonly` often depends on dynamic properties of the
/// module being compiled (e.g. whether a memory can be relocated) rather
/// than being a static property of the field; this method allows callers
/// to mark the load as `readonly` in these cases.
pub fn readonly(mut self) -> Self {
self.flags.set_readonly(true);
self
}
/// Mark accesses of this field as `readonly` if and only if `readonly` is
/// `true`.
///
/// See the note on [`Field::readonly`].
pub fn readonly_if(self, readonly: bool) -> Self {
if readonly { self.readonly() } else { self }
}
/// Mark accesses of this field as `can_move`.
///
/// See the note on [`Field::readonly`].
pub fn can_move(mut self) -> Self {
self.flags = self.flags.with_can_move(true);
self
}
/// Mark accesses of this field as `can_move` if and only if `can_move` is
/// `true`.
///
/// See the note on [`Field::readonly`].
pub fn can_move_if(self, can_move: bool) -> Self {
if can_move { self.can_move() } else { self }
}
/// Set the trap code for accesses of this field.
///
/// A `Field`'s accesses do not trap by default.
///
/// Note that when signals-based traps are disabled, callers must use the
/// explicit call-to-host trapping codegen instead.
pub fn trap_code(mut self, code: Option<ir::TrapCode>) -> Self {
self.flags = self.flags.with_trap_code(code);
self
}
/// Cast this field to the given type.
///
/// This can be used, for example, to load a field as a narrower or
/// differently-typed value than its declared Cranelift type.
pub fn cast(mut self, ty: ir::Type) -> Self {
self.ty = ty;
self
}
/// Rebase this field's load or store to be relative to a new base.
///
/// The `struct_offset` parameter is the offset of this field's containing
/// `VM*` structure within the new base.
///
/// A `Field` starts out relative to a pointer to its containing `VM*`
/// structure. When that structure is inlined directly into a larger one (as
/// an owned memory's `VMMemoryDefinition` is inlined into the `vmctx`), use
/// this method to fold the structure's own offset within the larger
/// structure into this `Field`, so that the resulting access is relative to
/// the larger structure directly.
pub fn relative_to(mut self, struct_offset: u32) -> Self {
self.offset += struct_offset;
self
}
/// Get-or-create this field's alias region.
pub fn region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.regions.region(func, self.key)
}
/// Get-or-create this field's alias region and mix it into this field's
/// base flags.
fn flags_with_region(&mut self, func: &mut ir::Function) -> ir::MemFlagsData {
let region = self.region(func);
self.flags.with_alias_region(Some(region))
}
/// Get a deferred [`Load`] descriptor for this field, for use in a
/// `VmctxLoadChain`.
///
/// The load is emitted at the field's current offset relative to its base
/// value. By default that is the field's offset within its containing `VM*`
/// structure, so the load is relative to a pointer to that structure; to
/// make it relative to something else (e.g. the `vmctx`, when the structure
/// is inlined into the `vmctx`), call [`Field::relative_to`] first.
pub fn to_deferred_load(&mut self, func: &mut ir::Function) -> Load {
let flags = self.flags_with_region(func);
Load {
offset: self.offset,
flags,
ty: self.ty,
}
}
/// Emit a load of this field relative to `ptr`, a pointer to the containing
/// `VM*` structure.
pub fn load(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value) -> ir::Value {
let load = self.to_deferred_load(cursor.func);
load.emit(cursor, ptr)
}
/// Emit an atomic load of this field relative to `ptr`, a pointer to the
/// containing `VM*` structure.
pub fn load_atomic(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value) -> ir::Value {
let flags = self.flags_with_region(cursor.func);
let ty = self.ty;
let pointer_type = self.regions.pointer_type;
let offset = cursor.ins().iconst(pointer_type, i64::from(self.offset));
let addr = cursor.ins().iadd(ptr, offset);
cursor.ins().atomic_load(ty, flags, addr)
}
/// Emit a store of `value` to this field relative to `ptr`, a pointer to
/// the containing `VM*` structure.
pub fn store(&mut self, cursor: &mut FuncCursor<'_>, ptr: ir::Value, value: ir::Value) {
let flags = self.flags_with_region(cursor.func);
cursor
.ins()
.store(flags, value, ptr, i32::try_from(self.offset).unwrap());
}
/// Like [`Self::store`], except removes the `readonly` flag for this region
/// if one would otherwise be specified.
///
/// This is useful for `#[readonly]` fields that are initialized precisely
/// once before use, such as `VMFuncRef` fields.
pub fn store_without_readonly(
&mut self,
cursor: &mut FuncCursor<'_>,
ptr: ir::Value,
value: ir::Value,
) {
let flags = self.flags_with_region(cursor.func).with_readonly(false);
cursor
.ins()
.store(flags, value, ptr, i32::try_from(self.offset).unwrap());
}
}
/// Define, for each `VM*` type, an `AliasRegions` accessor that returns a
/// wrapper exposing a [`Field`]-returning method per field of that type.
///
/// For example, given
///
/// ```ignore
/// struct VMMemoryDefinition {
/// base: VmPtr<u8>,
/// current_length: AtomicUsize,
/// }
/// ```
///
/// this macro generates:
///
/// ```ignore
/// impl<Offsets> AliasRegions<Offsets> {
/// fn vm_memory_definition(&mut self) -> VMMemoryDefinition<'_, Offsets> { ... }
/// }
///
/// struct VMMemoryDefinition<'a, Offsets> { ... }
///
/// impl<'a, Offsets: GetPtrSize> VMMemoryDefinition<'a, Offsets> {
/// fn base(self) -> Field<'a, Offsets> { ... }
/// fn current_length(self) -> Field<'a, Offsets> { ... }
/// }
/// ```
///
/// A field marked `#[aggregate]` gets no accessor, because it has no single
/// Cranelift type. A field marked `#[indexed]` is a fixed-size array of a
/// scalar type; its accessor takes the element index, and each element gets its
/// own alias region.
#[allow(
unused_macro_rules,
reason = "the `#[readonly]`/`#[can_move]` marker arms are generated \
uniformly but not exercised until a VM type uses those markers"
)]
macro_rules! define_vm_type_alias_region_helpers {
// `UnsafeCell<T>` is `repr(transparent)` and is accessed exactly as its `T`
// would be; delegate to the inner type.
(@field_ty $pt:expr, UnsafeCell < $inner:tt >) => {
define_vm_type_alias_region_helpers!(@field_ty $pt, $inner)
};
// Classify a field type to its Cranelift `ir::Type`, given `$pt` (the
// target pointer type as an `ir::Type`).
//
// Note that there is deliberately no arm for a composite field (a nested
// struct, an array, or a range): such a field has no single Cranelift type,
// and is marked `#[aggregate]` in `for_each_vm_type!` so that no accessor is
// generated for it at all.
(@field_ty $pt:expr, VmPtr < $g:ty >) => { $pt };
(@field_ty $pt:expr, VmPtr < VmPtr < $g:ty >>) => { $pt };
(@field_ty $pt:expr, Option < VmPtr < $g:ty >>) => { $pt };
(@field_ty $pt:expr, AtomicUsize) => { $pt };
(@field_ty $pt:expr, usize) => { $pt };
(@field_ty $pt:expr, * mut $g:ty) => { $pt };
(@field_ty $pt:expr, i64) => { ir::types::I64 };
(@field_ty $pt:expr, u64) => { ir::types::I64 };
(@field_ty $pt:expr, u32) => { ir::types::I32 };
(@field_ty $pt:expr, u16) => { ir::types::I16 };
(@field_ty $pt:expr, u8) => { ir::types::I8 };
(@field_ty $pt:expr, NonZeroU32) => { ir::types::I32 };
(@field_ty $pt:expr, Option < VMGcRef >) => { ir::types::I32 };
// `VMLazyThread` is a pointer-sized bitpacked integer; see its definition in
// `for_each_vm_type!`.
(@field_ty $pt:expr, VMLazyThread) => { $pt };
(@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };
(@field_ty $pt:expr, DefinedTableIndex) => { ir::types::I32 };
(@field_ty $pt:expr, DefinedMemoryIndex) => { ir::types::I32 };
(@field_ty $pt:expr, DefinedTagIndex) => { ir::types::I32 };
(@field_ty $pt:expr, VMGlobalKind) => { ir::types::I64 };
(@field_ty $pt:expr, VMStackState) => { ir::types::I32 };
// Classify an `#[indexed]` field's array type to
// `(element ir::Type, element size in bytes, element count)`.
(@array_info $pt:expr, UnsafeCell < $inner:tt >) => {
define_vm_type_alias_region_helpers!(@array_info $pt, $inner)
};
(@array_info $pt:expr, [u32; $n:expr]) => { (ir::types::I32, 4u32, $n) };
// Apply a field attribute to its access flags. The `#[readonly]` and
// `#[can_move]` markers map to the corresponding `MemFlagsData` builder
// methods; doc comments have no effect on flags; any other attribute is a
// compile error.
(@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly(true) };
(@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move(true) };
(@apply_attr $flags:expr, [doc = $d:literal]) => { $flags };
// Emit the accessor method for a single ordinary (scalar) field.
(@method $Name:ident $snake:ident plain $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ]) => {
#[doc = concat!(
"Get the [`Field`] for the `", stringify!($fname),
"` field of `", stringify!($Name), "`."
)]
pub fn $fname(self) -> Field<'a, Offsets> {
let offset = u32::from(
self.regions.offsets.get_ptr_size().$snake().$fname()
);
let flags = ir::MemFlagsData::trusted();
$(
let flags = define_vm_type_alias_region_helpers!(
@apply_attr flags, $fattr
);
)*
let ty = define_vm_type_alias_region_helpers!(
@field_ty self.regions.pointer_type, $($fty)*
);
Field::new(self.regions, VmType::$Name, offset, flags, ty)
}
};
// Emit the accessor method for an `#[indexed]` field: a fixed-size array of
// a scalar type, indexed by a compile-time constant.
(@method $Name:ident $snake:ident indexed $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ]) => {
#[doc = concat!(
"Get the [`Field`] for the `index`th element of the `",
stringify!($fname), "` field of `", stringify!($Name), "`."
)]
pub fn $fname(self, index: u8) -> Field<'a, Offsets> {
let (ty, elem_size, len) = define_vm_type_alias_region_helpers!(
@array_info self.regions.pointer_type, $($fty)*
);
assert!(
usize::from(index) < len,
concat!(
"index out of bounds for `", stringify!($Name), "::",
stringify!($fname), "`"
),
);
let base = u32::from(
self.regions.offsets.get_ptr_size().$snake().$fname()
);
let offset = base + u32::from(index) * elem_size;
let flags = ir::MemFlagsData::trusted();
$(
let flags = define_vm_type_alias_region_helpers!(
@apply_attr flags, $fattr
);
)*
Field::new(self.regions, VmType::$Name, offset, flags, ty)
}
};
// Emit the accessor `struct` and per-field methods for one `VM*` type.
//
// Fields arrive pre-split into `{ $kind $fname [ $attrs... ] [ $fty... ] }`
// groups (see `@munch` below); the terminal arm consumes those groups once
// the struct body has been fully peeled apart.
(@emit $Name:ident $snake:ident {
$( { $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] } )*
}) => {
#[doc = concat!(
"An [`AliasRegions`] accessor for the fields of a `",
stringify!($Name), "`."
)]
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
pub struct $Name<'a, Offsets> {
regions: &'a mut AliasRegions<Offsets>,
}
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
impl<Offsets> AliasRegions<Offsets> {
#[doc = concat!(
"Get an accessor for the fields of a `", stringify!($Name), "`."
)]
pub fn $snake(&mut self) -> $Name<'_, Offsets> {
$Name { regions: self }
}
}
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
impl<'a, Offsets> $Name<'a, Offsets>
where
Offsets: GetPtrSize,
{
$(
define_vm_type_alias_region_helpers!(
@method $Name $snake $fkind $fname [ $($fattr)* ] [ $($fty)* ]
);
)*
}
};
// Peel fields off the raw struct body one at a time, accumulating completed
// `{ $kind $fname [ $attrs... ] [ $fty... ] }` groups plus the pending
// attributes for the field currently being parsed. `$kind` is `plain` or
// `indexed`, and selects which `@method` arm emits the field's accessor.
//
// Splitting the body by hand (rather than matching `$fty:tt $(< $fgen:ty
// >)?` within a repetition) is what lets each field's type reach the
// `@field_ty` classifier as raw tokens, so it can require `Option`s to
// specifically be `Option<VmPtr<_>>`.
(@munch $Name:ident $snake:ident { $($groups:tt)* }) => {
define_vm_type_alias_region_helpers!(@emit $Name $snake { $($groups)* });
};
// A field marked `#[aggregate]` is a composite (a nested struct or array)
// so there is no single Cranelift type for such fields, so no accessors are
// generated for them; interior accesses are computed from the field's
// offset instead, which the generated `offsets::*` methods still provide.
(@munch $Name:ident $snake:ident { $($groups:tt)* }
$(#[doc = $fdoc:literal])* #[aggregate] $fvis:vis $fname:ident : $fty:ty , $($rest:tt)*
) => {
define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* } $($rest)*);
};
// A field marked `#[indexed]` is a fixed-size array of a scalar type whose
// elements are accessed individually by a compile-time constant index. It
// has no single Cranelift type either, but its elements do, so it gets an
// index-taking accessor.
(@munch $Name:ident $snake:ident { $($groups:tt)* }
$(#[doc = $fdoc:literal])* #[indexed] $fvis:vis $fname:ident : $($rest:tt)*
) => {
define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } indexed $fname [] [] $($rest)*);
};
// Consume one field's attributes, visibility, and name, then collect its
// type tokens.
(@munch $Name:ident $snake:ident { $($groups:tt)* }
$(# $fattr:tt)* $fvis:vis $fname:ident : $($rest:tt)*
) => {
define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } plain $fname [ $($fattr)* ] [] $($rest)*);
};
// Accumulate one field's type tokens up to its terminating comma, then
// append the completed group.
(@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {
define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fkind $fname [ $($fattr)* ] [ $($fty)* ] } } $($rest)*);
};
(@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {
define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fkind $fname [ $($fattr)* ] [ $($fty)* $tok ] $($rest)*);
};
// Emit the accessor `struct` and per-field methods for one enum variant,
// so that `alias_regions.vm_foo().variant_a().field()` mirrors the
// generated offset accessors.
(@emit_variant $Name:ident $snake:ident $VariantAccess:ident $vsnake:ident {
$( { $fkind:ident $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] } )*
}) => {
#[doc = concat!(
"An [`AliasRegions`] accessor for the fields of the `",
stringify!($Name), "::", stringify!($VariantAccess), "` variant."
)]
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
pub struct $VariantAccess<'a, Offsets> {
regions: &'a mut AliasRegions<Offsets>,
}
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
impl<'a, Offsets> $Name<'a, Offsets> {
#[doc = concat!(
"Get an accessor for the fields of the `", stringify!($Name),
"::", stringify!($VariantAccess), "` variant."
)]
pub fn $vsnake(self) -> $VariantAccess<'a, Offsets> {
$VariantAccess { regions: self.regions }
}
}
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
impl<'a, Offsets> $VariantAccess<'a, Offsets>
where
Offsets: GetPtrSize,
{
$(
define_vm_type_alias_region_helpers!(
@variant_method $Name $snake $vsnake $fkind $fname
[ $($fattr)* ] [ $($fty)* ]
);
)*
}
};
// As `@method` above, but the offset comes from the variant's own offsets
// and the region is the enum's shared payload one.
(@variant_method $Name:ident $snake:ident $vsnake:ident plain $fname:ident
[ $($fattr:tt)* ] [ $($fty:tt)* ]
) => {
#[doc = concat!(
"Get the [`Field`] for the `", stringify!($fname),
"` field of this variant of `", stringify!($Name), "`.\n\n\
All variants' fields share one alias region."
)]
pub fn $fname(self) -> Field<'a, Offsets> {
let ptr_size = self.regions.offsets.get_ptr_size();
let payload = u32::from(ptr_size.$snake().payload());
let offset = u32::from(ptr_size.$snake().$vsnake().$fname());
let flags = ir::MemFlagsData::trusted();
$(
let flags = define_vm_type_alias_region_helpers!(
@apply_attr flags, $fattr
);
)*
let ty = define_vm_type_alias_region_helpers!(
@field_ty self.regions.pointer_type, $($fty)*
);
Field::new_in_region(self.regions, VmType::$Name, payload, offset, flags, ty)
}
};
// Emit the accessor `struct` for a `#[repr(C, u32)]` enum: its `u32` tag,
// plus one nested accessor type per variant.
//
// An enum gets two regions, not one per field: variants' fields occupy the
// same payload bytes, so they can alias both across and within a variant.
(@emit_enum $Name:ident $snake:ident $tag:ident {
$( { $VariantAccess:ident $vsnake:ident { $($vgroups:tt)* } } )*
}) => {
#[doc = concat!(
"An [`AliasRegions`] accessor for the tag and variants of the `",
stringify!($Name), "` enum."
)]
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
pub struct $Name<'a, Offsets> {
regions: &'a mut AliasRegions<Offsets>,
}
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
impl<Offsets> AliasRegions<Offsets> {
#[doc = concat!(
"Get an accessor for the tag and variants of a `",
stringify!($Name), "`."
)]
pub fn $snake(&mut self) -> $Name<'_, Offsets> {
$Name { regions: self }
}
}
#[allow(
dead_code,
reason = "generated uniformly for all VM types; not all accessors are used yet"
)]
impl<'a, Offsets> $Name<'a, Offsets>
where
Offsets: GetPtrSize,
{
#[doc = concat!(
"Get the [`Field`] for the `", stringify!($tag), "` tag of `",
stringify!($Name), "`.\n\n\
The tag has its own alias region, distinct from the variants'."
)]
pub fn tag(self) -> Field<'a, Offsets> {
let offset = u32::from(
self.regions.offsets.get_ptr_size().$snake().tag()
);
let flags = ir::MemFlagsData::trusted();
let ty = define_vm_type_alias_region_helpers!(
@field_ty self.regions.pointer_type, $tag
);
Field::new(self.regions, VmType::$Name, offset, flags, ty)
}
}
$(
define_vm_type_alias_region_helpers!(
@emit_variant $Name $snake $VariantAccess $vsnake { $($vgroups)* }
);
)*
};
// Peel variants off an enum's body one at a time; `@munch_variant` handles
// each variant's own body and hands the remainder back.
(@munch_enum $Name:ident $snake:ident $tag:ident { $($vs:tt)* }) => {
define_vm_type_alias_region_helpers!(@emit_enum $Name $snake $tag { $($vs)* });
};
(@munch_enum $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
$(#[doc = $vdoc:literal])*
#[snake_name = $vsnake:ident]
$(#[$vattr:meta])*
$Variant:ident { $($vbody:tt)* } = $disc:literal ,
$($rest:tt)*
) => {
define_vm_type_alias_region_helpers!(
@munch_variant $Name $snake $tag { $($vs)* } $Variant $vsnake {}
[ $($vbody)* ] [ $($rest)* ]
);
};
// As `@munch` for a struct body, resuming `@munch_enum` on the remainder.
(@munch_variant $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
$Variant:ident $vsnake:ident { $($g:tt)* } [] [ $($rest:tt)* ]
) => {
define_vm_type_alias_region_helpers!(
@munch_enum $Name $snake $tag
{ $($vs)* { $Variant $vsnake { $($g)* } } }
$($rest)*
);
};
// As in `@munch`, an `#[aggregate]` field gets no accessor.
(@munch_variant $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
$Variant:ident $vsnake:ident { $($g:tt)* }
[ $(#[doc = $fdoc:literal])* #[aggregate] $fname:ident : $fty:ty , $($vrest:tt)* ] $rest:tt
) => {
define_vm_type_alias_region_helpers!(
@munch_variant $Name $snake $tag { $($vs)* } $Variant $vsnake { $($g)* }
[ $($vrest)* ] $rest
);
};
(@munch_variant $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
$Variant:ident $vsnake:ident { $($g:tt)* }
[ $(# $fattr:tt)* $fname:ident : $($vrest:tt)* ] $rest:tt
) => {
define_vm_type_alias_region_helpers!(
@munch_variant_ty $Name $snake $tag { $($vs)* } $Variant $vsnake { $($g)* }
$fname [ $($fattr)* ] [] [ $($vrest)* ] $rest
);
};
(@munch_variant_ty $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
$Variant:ident $vsnake:ident { $($g:tt)* }
$fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] [ , $($vrest:tt)* ] $rest:tt
) => {
define_vm_type_alias_region_helpers!(
@munch_variant $Name $snake $tag { $($vs)* } $Variant $vsnake
{ $($g)* { plain $fname [ $($fattr)* ] [ $($fty)* ] } }
[ $($vrest)* ] $rest
);
};
(@munch_variant_ty $Name:ident $snake:ident $tag:ident { $($vs:tt)* }
$Variant:ident $vsnake:ident { $($g:tt)* }
$fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] [ $tok:tt $($vrest:tt)* ] $rest:tt
) => {
define_vm_type_alias_region_helpers!(
@munch_variant_ty $Name $snake $tag { $($vs)* } $Variant $vsnake { $($g)* }
$fname [ $($fattr)* ] [ $($fty)* $tok ] [ $($vrest)* ] $rest
);
};
// Dispatch on `struct` vs `enum`.
(@def struct $Name:ident $snake:ident $repr:tt { $($body:tt)* }) => {
define_vm_type_alias_region_helpers!(@munch $Name $snake {} $($body)*);
};
// See the matching arm in `define_vm_type_offsets!`.
(@def enum $Name:ident $snake:ident [ C, $tag:ident ] { $($body:tt)* }) => {
define_vm_type_alias_region_helpers!(@munch_enum $Name $snake $tag {} $($body)*);
};
// Top-level entry: the list of `VM*` type definitions. Each is either a
// `struct` or a `#[repr(C, u32)]` `enum`; `@def` dispatches on which.
( $(
$(#[doc = $sdoc:literal])*
$(#[cfg($($scfg:tt)*)])?
$(#[derive($($d:ident),*)])?
#[repr($($repr:tt)*)]
#[snake_name = $snake:ident]
$svis:vis $kind:ident $Name:ident {
$($body:tt)*
}
)* ) => {
$(
define_vm_type_alias_region_helpers!(@def $kind $Name $snake [$($repr)*] { $($body)* });
)*
};
}
wasmtime_environ::for_each_vm_type!(define_vm_type_alias_region_helpers);
/// Define, for each of Wasmtime's vmctx types, an [`AliasRegions`] accessor
/// that returns a wrapper exposing a [`Field`]-returning method per field of
/// that vmctx.
///
/// For example, `alias_regions.vmctx().epoch_ptr()` is the `VMContext::epoch_ptr`
/// field, and `alias_regions.vmcomponent().callbacks(i)` is the `i`th element of
/// the `VMComponentContext`'s runtime-callbacks array.
///
/// A vmctx's `static` fields sit at offsets that depend only on the target
/// pointer size, so their accessors are available for any `Offsets: GetPtrSize`.
/// Its `dynamic` fields sit at offsets that additionally depend on the module or
/// component being compiled, so their accessors are only available when the
/// `AliasRegions` carries that vmctx's own fully-computed offsets. The exception
/// is a `dynamic` field marked `#[ptr_size_offset]`, whose offsets are derived
/// only from the pointer size; these live in the `Offsets: GetPtrSize` block as
/// well.
///
/// A field marked `#[aggregate]` gets no accessor, for the same reason it gets
/// none in [`define_vm_type_alias_region_helpers!`]: it has no single Cranelift
/// type. Instead, its interior is reached by rebasing a [`Field`] of the nested
/// `VM*` type onto the aggregate's own offset within the vmctx (see
/// [`Field::relative_to`]), which produces exactly the same alias region as
/// accessing that `VM*` type through a pointer would. For the few aggregates that
/// have no `VM*` type of their own, and hence no alias region of their own, there
/// are hand-written helpers below.
///
/// A field marked `#[pointee(..)]` additionally gets an accessor for what it
/// points *at*, which lives outside the vmctx. For example,
/// `alias_regions.vmctx().builtin_functions_array(i)` is the `i`th builtin in
/// the array that the `VMContext::builtin_functions` field points at.
#[allow(
unused_macro_rules,
reason = "entry shapes and marker attributes are handled uniformly for both \
sections of both vmctx types, but not every combination occurs"
)]
macro_rules! define_vmctx_alias_region_helpers {
// Classify a field type to its Cranelift `ir::Type`, given `$pt` (the target
// pointer type as an `ir::Type`).
(@field_ty $pt:expr, u32) => { ir::types::I32 };
(@field_ty $pt:expr, VmPtr < $g:ident >) => { $pt };
(@field_ty $pt:expr, VmPtr < VmPtr < $g:ident >>) => { $pt };
(@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };
(@field_ty $pt:expr, AtomicU64) => { ir::types::I64 };
(@field_ty $pt:expr, unsafe extern "C" fn) => { $pt };
// Determine the Cranelift type a field is *accessed* as: the classification
// of its `#[access_as = T]` type if it has one, and of its declared type
// otherwise.
(@access_ty $pt:expr, [ $($fty:tt)* ] []) => {
define_vmctx_alias_region_helpers!(@field_ty $pt, $($fty)*)
};
(@access_ty $pt:expr, $fty:tt [ #[access_as = $($t:tt)*] $($rest:tt)* ]) => {
define_vmctx_alias_region_helpers!(@field_ty $pt, $($t)*)
};
(@access_ty $pt:expr, $fty:tt [ # $skip:tt $($rest:tt)* ]) => {
define_vmctx_alias_region_helpers!(@access_ty $pt, $fty [ $($rest)* ])
};
// Apply a field attribute to its access flags.
(@apply_attr $flags:expr, [readonly]) => { $flags.with_readonly(true) };
(@apply_attr $flags:expr, [can_move]) => { $flags.with_can_move(true) };
(@apply_attr $flags:expr, [access_as = $($t:tt)*]) => { $flags };
(@apply_attr $flags:expr, [ptr_size_offset]) => { $flags };
(@apply_attr $flags:expr, [pointee( $($t:tt)* )]) => { $flags };
// Compute a field's access flags and Cranelift type from its declared type
// and marker attributes, and build the `Field` for it at `$offset`.
(@field $Name:ident ($self:expr, $offset:expr) [ $($fty:tt)* ] [ $(# $fattr:tt)* ]) => {{
let this = $self;
let flags = ir::MemFlagsData::trusted();
$( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $fattr); )*
let ty = define_vmctx_alias_region_helpers!(
@access_ty this.regions.pointer_type, [ $($fty)* ] [ $(# $fattr)* ]
);
let offset = $offset;
Field::new(this.regions, VmType::$Name, offset, flags, ty)
}};
// Scan a field's attributes for a `#[pointee(..)]` and emit the accessor it
// describes, if any.
(@pointee_methods $Offsets:tt [ ]) => {};
(@pointee_methods $Offsets:tt [ #[pointee( $($p:tt)* )] $($rest:tt)* ]) => {
define_vmctx_alias_region_helpers!(@pointee $Offsets $($p)*);
};
(@pointee_methods $Offsets:tt [ # $skip:tt $($rest:tt)* ]) => {
define_vmctx_alias_region_helpers!(@pointee_methods $Offsets [ $($rest)* ]);
};
// An array of pointees.
(@pointee [ $($Offsets:tt)* ]
$(# $pattr:tt)* $Region:ident as $pname:ident [ $Index:ident ] : $($pty:tt)*
) => {
#[doc = concat!(
"Get the [`Field`] for the `index`th element of the out-of-line `",
stringify!($Region), "` array.\n\nThe returned [`Field`] is relative \
to the array's base pointer, not to the vmctx."
)]
pub fn $pname(self, index: $Index) -> Field<'a, $($Offsets)*> {
let ty = define_vmctx_alias_region_helpers!(
@field_ty self.regions.pointer_type, $($pty)*
);
let offset = index.vmctx_array_index().checked_mul(ty.bytes()).unwrap();
let flags = ir::MemFlagsData::trusted();
$( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $pattr); )*
Field::new(self.regions, VmType::$Region, offset, flags, ty)
}
};
// A single pointee.
(@pointee [ $($Offsets:tt)* ]
$(# $pattr:tt)* $Region:ident as $pname:ident : $($pty:tt)*
) => {
#[doc = concat!(
"Get the [`Field`] for the out-of-line `", stringify!($Region),
"`.\n\nThe returned [`Field`] is relative to the pointer to it, not \
to the vmctx."
)]
pub fn $pname(self) -> Field<'a, $($Offsets)*> {
let ty = define_vmctx_alias_region_helpers!(
@field_ty self.regions.pointer_type, $($pty)*
);
let flags = ir::MemFlagsData::trusted();
$( let flags = define_vmctx_alias_region_helpers!(@apply_attr flags, $pattr); )*
Field::new(self.regions, VmType::$Region, 0, flags, ty)
}
};
// ### `static` Section Entries
(@static_entry $Name:ident $snake:ident align { $al:tt }) => {};
// Aggregates get no accessor.
(@static_entry $Name:ident $snake:ident $kind:ident { #[aggregate] $($rest:tt)* }) => {};
(@static_entry $Name:ident $snake:ident field {
$(# $fattr:tt)* $fname:ident : $($fty:tt)*
}) => {
#[doc = concat!(
"Get the [`Field`] for the `", stringify!($fname), "` field of `",
stringify!($Name), "`."
)]
pub fn $fname(self) -> Field<'a, Offsets> {
let offset = u32::from(self.regions.offsets.get_ptr_size().$snake().$fname());
define_vmctx_alias_region_helpers!(
@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
)
}
define_vmctx_alias_region_helpers!(@pointee_methods [ Offsets ] [ $(# $fattr)* ]);
};
// ### `dynamic` Section Entries Marked `#[ptr_size_offset]`
//
// These get their offsets from the pointer-size-only `offsets::VMFoo<P>`
// wrapper, and don't require a full `VMOffsets` parameterization.
(@ptr_size_entry $Name:ident $snake:ident field {
#[ptr_size_offset] $(# $fattr:tt)* $fname:ident : $($fty:tt)*
}) => {
#[doc = concat!(
"Get the [`Field`] for the `", stringify!($fname), "` field of `",
stringify!($Name), "`."
)]
pub fn $fname(self) -> Field<'a, Offsets> {
let offset = self.regions.offsets.get_ptr_size().$snake().$fname();
define_vmctx_alias_region_helpers!(
@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
)
}
};
(@ptr_size_entry $Name:ident $snake:ident array {
#[ptr_size_offset] $(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*
}) => {
#[doc = concat!(
"Get the [`Field`] for the `index`th element of `", stringify!($Name),
"`'s `", stringify!($fname), "` array.\n\nThis is not bounds checked: \
the array's length depends on the module or component being compiled, \
which is precisely what this accessor does not require knowing."
)]
pub fn $fname(self, index: $Index) -> Field<'a, Offsets> {
let offset = self.regions.offsets.get_ptr_size().$snake().$fname(index);
define_vmctx_alias_region_helpers!(
@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
)
}
};
(@ptr_size_entry $Name:ident $snake:ident $kind:ident $entry:tt) => {};
// ### `dynamic` Section Entries
// Entries marked `#[ptr_size_offset]` were already handled above; emitting
// them here too would be a duplicate definition.
(@dynamic_entry $Name:ident $Offsets:tt $kind:ident {
#[ptr_size_offset] $($rest:tt)*
}) => {};
(@dynamic_entry $Name:ident $Offsets:tt align { $al:tt }) => {};
// Aggregates get no accessor.
(@dynamic_entry $Name:ident $Offsets:tt $kind:ident { #[aggregate] $($rest:tt)* }) => {};
(@dynamic_entry $Name:ident [ $($Offsets:tt)* ] array {
$(# $fattr:tt)* $fname:ident [ $count:ident ; $Index:ident ] : $($fty:tt)*
}) => {
#[doc = concat!(
"Get the [`Field`] for the `index`th element of `", stringify!($Name),
"`'s `", stringify!($fname), "` array.\n\nPanics if `index` is out of \
bounds for this vmctx."
)]
pub fn $fname(self, index: $Index) -> Field<'a, $($Offsets)*> {
let offset = self.regions.offsets.$fname().at(index);
define_vmctx_alias_region_helpers!(
@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
)
}
define_vmctx_alias_region_helpers!(
@pointee_methods [ $($Offsets)* ] [ $(# $fattr)* ]
);
};
// A single field, whether unconditionally present (`field`) or only
// conditionally (`optional`).
(@dynamic_entry $Name:ident [ $($Offsets:tt)* ] $kind:ident {
$(# $fattr:tt)* $fname:ident $([ if $flag:ident ])? : $($fty:tt)*
}) => {
#[doc = concat!(
"Get the [`Field`] for the `", stringify!($fname), "` field of `",
stringify!($Name), "`."
$(, "\n\nPanics if `", stringify!($flag), "` is false, in which case \
this field is not present at all.")?
)]
pub fn $fname(self) -> Field<'a, $($Offsets)*> {
let offset = self.regions.offsets.$fname();
define_vmctx_alias_region_helpers!(
@field $Name (self, offset) [ $($fty)* ] [ $(# $fattr)* ]
)
}
define_vmctx_alias_region_helpers!(
@pointee_methods [ $($Offsets)* ] [ $(# $fattr)* ]
);
};
// Emit the `impl` block holding the accessors for the dynamically-positioned
// fields.
(@dynamic_impl $Name:ident [ $($Offsets:tt)* ] $OffsetsTt:tt {
$($kind:ident $entry:tt)*
}) => {
#[allow(
dead_code,
reason = "generated uniformly for every field; not all fields are \
accessed by compiled code"
)]
impl<'a> $Name<'a, $($Offsets)*> {
$(
define_vmctx_alias_region_helpers!(
@dynamic_entry $Name $OffsetsTt $kind $entry
);
)*
}
};
// Emit the accessor `struct` and both `impl` blocks for one vmctx type.
(@emit $Name:ident $snake:ident $Offsets:tt
static { $($skind:ident $sentry:tt)* }
dynamic { $($dkind:ident $dentry:tt)* }
) => {
#[doc = concat!(
"An [`AliasRegions`] accessor for the fields of a `", stringify!($Name),
"`."
)]
pub struct $Name<'a, Offsets> {
regions: &'a mut AliasRegions<Offsets>,
}
impl<Offsets> AliasRegions<Offsets> {
#[doc = concat!(
"Get an accessor for the fields of a `", stringify!($Name), "`."
)]
pub fn $snake(&mut self) -> $Name<'_, Offsets> {
$Name { regions: self }
}
}
// A statically-positioned field's offset depends only on the target
// pointer size, so these accessors work with any `Offsets`.
#[allow(
dead_code,
reason = "generated uniformly for every field; not all fields are \
accessed by compiled code"
)]
impl<'a, Offsets> $Name<'a, Offsets>
where
Offsets: GetPtrSize,
{
$( define_vmctx_alias_region_helpers!(@static_entry $Name $snake $skind $sentry); )*
$( define_vmctx_alias_region_helpers!(@ptr_size_entry $Name $snake $dkind $dentry); )*
}
// A dynamically-positioned field's offset depends on the module or
// component being compiled, so these accessors require this vmctx's own
// fully-computed offsets.
define_vmctx_alias_region_helpers!(
@dynamic_impl $Name $Offsets $Offsets { $($dkind $dentry)* }
);
};
// Map each vmctx type to the offsets type that computes its
// dynamically-positioned fields' offsets.
(@one VMContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => {
define_vmctx_alias_region_helpers!(@emit VMContext $snake [ VMOffsets<u8> ]
static { $($stat)* } dynamic { $($dyn)* });
};
(@one VMComponentContext $snake:ident static { $($stat:tt)* } dynamic { $($dyn:tt)* }) => {
define_vmctx_alias_region_helpers!(
@emit VMComponentContext $snake [ VMComponentOffsets<u8> ]
static { $($stat)* } dynamic { $($dyn)* }
);
};
// Top-level entry.
( $(
{
$Name:ident $snake:ident
static { $($stat:tt)* }
dynamic { $($dyn:tt)* }
}
)* ) => {
$(
define_vmctx_alias_region_helpers!(@one $Name $snake
static { $($stat)* } dynamic { $($dyn)* });
)*
};
}
wasmtime_environ::for_each_vmctx_type!(define_vmctx_alias_region_helpers);
impl<Offsets> AliasRegions<Offsets>
where
Offsets: GetPtrSize,
{
/// Create a new `AliasRegions`.
pub fn new(offsets: Offsets) -> Self {
Self {
pointer_type: ir::Type::int_with_byte_size(offsets.get_ptr_size().size().into())
.unwrap(),
offsets,
gc_introducer_cache: HashMap::new(),
}
}
/// The Cranelift type of a pointer on the target being compiled for.
pub fn pointer_type(&self) -> ir::Type {
self.pointer_type
}
/// The pointer size of the target being compiled for, used to compute the
/// layout of Wasmtime's vmctx types.
pub fn ptr_size(&self) -> &Offsets::Ptr {
self.offsets.get_ptr_size()
}
/// Get the alias region for the given GC object access.
pub fn gc_access_region(
&mut self,
func: &mut ir::Function,
types: &ModuleTypesBuilder,
access: GcAccess,
) -> ir::AliasRegion {
let key = match access {
GcAccess::HeaderKind => return self.vm_gc_header().kind().region(func),
GcAccess::HeaderTypeIndex => return self.vm_gc_header().ty().region(func),
GcAccess::DrcRefCount => return self.vm_drc_header().ref_count().region(func),
GcAccess::DrcNextOverApproximatedStackRoot => {
return self
.vm_drc_header()
.next_over_approximated_stack_root()
.region(func);
}
GcAccess::CopyingObjectSize => {
return self.vm_copying_header().object_size().region(func);
}
GcAccess::ArrayLength => AliasRegionKey::GcArrayLength,
GcAccess::ExnTagInstance => AliasRegionKey::GcExnTagInstance,
GcAccess::ExnTagDefined => AliasRegionKey::GcExnTagDefined,
GcAccess::StructField { ty, field } => AliasRegionKey::GcStructField {
ty: self.gc_introducer(types, ty, field),
field,
},
GcAccess::ArrayElements { ty } => AliasRegionKey::GcArrayElements {
ty: self.gc_introducer(types, ty, 0),
},
GcAccess::ExnPayload { exn_ty, field } => {
AliasRegionKey::GcExnPayload { exn_ty, field }
}
};
self.region(func, key)
}
/// Find the type that introduced the given field (or, for arrays, the
/// elements) into `ty`'s subtyping hierarchy, memoizing the answer.
fn gc_introducer(
&mut self,
types: &ModuleTypesBuilder,
ty: ModuleInternedTypeIndex,
field: u32,
) -> ModuleInternedTypeIndex {
if let Some(introducer) = self.gc_introducer_cache.get(&(ty, field)) {
return *introducer;
}
let mut introducer = ty;
while let Some(supertype) = types[introducer].supertype {
let supertype = supertype.unwrap_module_type_index();
let supertype_declares_field = match &types[supertype].composite_type.inner {
// Subtyping a struct can only append new fields, so we need
// only look at the field length.
WasmCompositeInnerType::Struct(s) => field < u32::try_from(s.fields.len()).unwrap(),
// Array elements are always introduced by the first type
// without a supertype.
WasmCompositeInnerType::Array(_) => true,
inner @ WasmCompositeInnerType::Func(_)
| inner @ WasmCompositeInnerType::Cont(_)
| inner @ WasmCompositeInnerType::Exn(_) => {
unreachable!("not a GC type with subtyping: {inner:?}")
}
};
if supertype_declares_field {
introducer = supertype;
} else {
break;
}
}
self.gc_introducer_cache.insert((ty, field), introducer);
introducer
}
/// Get the alias region shared by all memories that cross a module boundary
/// and whose definition we do not statically know.
pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.region(func, AliasRegionKey::PublicMemory)
}
/// Get the alias region for accessing a particular defined memory.
pub fn defined_memory_region(
&mut self,
func: &mut ir::Function,
module: StaticModuleIndex,
index: DefinedMemoryIndex,
) -> ir::AliasRegion {
self.region(func, AliasRegionKey::DefinedMemory { module, index })
}
/// Get the alias region shared by all tables that cross a module boundary
/// and whose definition we do not statically know.
pub fn public_table_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.region(func, AliasRegionKey::PublicTable)
}
/// Get the alias region for accessing a particular defined table.
pub fn defined_table_region(
&mut self,
func: &mut ir::Function,
module: StaticModuleIndex,
index: DefinedTableIndex,
) -> ir::AliasRegion {
self.region(func, AliasRegionKey::DefinedTable { module, index })
}
/// Get the alias region shared by all globals that cross a module boundary
/// and whose definition we do not statically know.
pub fn public_global_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.region(func, AliasRegionKey::PublicGlobal)
}
/// Get the alias region for accessing a particular defined global.
pub fn defined_global_region(
&mut self,
func: &mut ir::Function,
module: StaticModuleIndex,
index: DefinedGlobalIndex,
) -> ir::AliasRegion {
self.region(func, AliasRegionKey::DefinedGlobal { module, index })
}
}
/// `VMStoreContext`-related methods.
impl<Offsets> AliasRegions<Offsets>
where
Offsets: GetPtrSize,
{
fn vmstore_context_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {
self.region(
func,
AliasRegionKey::Vm {
ty: VmType::VMStoreContext,
offset,
},
)
}
/// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).
///
/// The caller supplies the base flags because whether the base pointer is
/// `readonly`/`can_move` depends on the GC heap's tunables.
pub fn vmstore_context_gc_heap_base_load(
&mut self,
func: &mut ir::Function,
base_flags: ir::MemFlagsData,
) -> Load {
let offset = self
.offsets
.get_ptr_size()
.vm_store_context()
.gc_heap_base()
.into();
let region = self.vmstore_context_region(func, offset);
Load {
offset,
flags: base_flags.with_alias_region(Some(region)),
ty: self.pointer_type,
}
}
/// Load the GC heap base pointer (`VMStoreContext::gc_heap.base`).
///
/// The caller supplies the base flags because whether the base pointer is
/// `readonly`/`can_move` depends on the GC heap's tunables.
pub fn vmstore_context_gc_heap_base(
&mut self,
cursor: &mut FuncCursor<'_>,
base_flags: ir::MemFlagsData,
vmstore_ctx: ir::Value,
) -> ir::Value {
self.vmstore_context_gc_heap_base_load(cursor.func, base_flags)
.emit(cursor, vmstore_ctx)
}
/// Get a `Load` of the GC heap bound (`VMStoreContext::gc_heap.current_length`).
pub fn vmstore_context_gc_heap_current_length_load(&mut self, func: &mut ir::Function) -> Load {
let offset = self
.offsets
.get_ptr_size()
.vm_store_context()
.gc_heap_current_length()
.into();
let region = self.vmstore_context_region(func, offset);
Load {
offset,
flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),
ty: self.pointer_type,
}
}
/// Load the GC heap bound (`VMStoreContext::gc_heap.current_length`).
pub fn vmstore_context_gc_heap_current_length(
&mut self,
cursor: &mut FuncCursor<'_>,
vmstore_ctx: ir::Value,
) -> ir::Value {
self.vmstore_context_gc_heap_current_length_load(cursor.func)
.emit(cursor, vmstore_ctx)
}
/// Get the alias region for the `VMStoreContext::stack_chain` field.
///
/// The `VMStackChain` is two pointers wide and is emitted by the stack
/// switching `VMStackChain` load/store helpers, which take a region
/// argument; this provides that region.
pub fn vmstore_context_stack_chain_region(
&mut self,
func: &mut ir::Function,
) -> ir::AliasRegion {
let offset = self.offsets.get_ptr_size().vm_store_context().stack_chain();
self.vmstore_context_region(func, offset.into())
}
}
/// Component-specific methods.
impl<Offsets> AliasRegions<Offsets>
where
Offsets: GetPtrSize,
{
/// Get the alias region for the `ValRaw` array used to marshal arguments
/// and results across the array calling convention used by various
/// trampolines.
pub fn host_val_raw_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.region(
func,
AliasRegionKey::Vm {
ty: VmType::HostValRaw,
offset: 0,
},
)
}
}
/// Passive data and element segment-related methods.
impl<Offsets> AliasRegions<Offsets>
where
Offsets: GetPtrSize,
{
/// Get the alias region for the runtime `ValRaw` storage of passive element
/// segments (written by passive-element initialization and read by
/// `table.init` and `array.{new,init}_elem`).
pub fn element_segment_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.region(func, AliasRegionKey::ElementSegment)
}
/// Get the alias region for the bytes of data segments (read by
/// `memory.init` and `array.{new,init}_data`).
pub fn data_segment_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.region(func, AliasRegionKey::DataSegment)
}
/// Get the alias region for the shadow bytes of linear memories, which
/// watchpoint checks read before stores.
pub fn shadow_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.region(func, AliasRegionKey::ShadowMemory)
}
}
/// Unsafe intrinsic-related methods.
impl<Offsets> AliasRegions<Offsets>
where
Offsets: GetPtrSize,
{
/// Perform an unsafe intrinsic's load.
pub fn unsafe_intrinsic_load(
&mut self,
cursor: &mut FuncCursor<'_>,
ty: ir::Type,
base_flags: ir::MemFlagsData,
addr: ir::Value,
) -> ir::Value {
let region = self.region(cursor.func, AliasRegionKey::UnsafeIntrinsicMemory);
cursor
.ins()
.load(ty, base_flags.with_alias_region(Some(region)), addr, 0)
}
/// Perform an unsafe intrinsic's store.
pub fn unsafe_intrinsic_store(
&mut self,
cursor: &mut FuncCursor<'_>,
base_flags: ir::MemFlagsData,
val: ir::Value,
addr: ir::Value,
) {
let region = self.region(cursor.func, AliasRegionKey::UnsafeIntrinsicMemory);
cursor
.ins()
.store(base_flags.with_alias_region(Some(region)), val, addr, 0);
}
}
/// `VMComponentContext` fields that are not simply one of the layout's own
/// fields, and so are not generated by [`define_vmctx_alias_region_helpers!`].
impl AliasRegions<VMComponentOffsets<u8>> {
/// Get the [`Field`] for the `callee` of the `index`th lowering in the
/// `VMComponentContext`'s `lowerings` array.
pub fn vmcomponent_lowering_callee(
&mut self,
index: LoweredIndex,
) -> Field<'_, VMComponentOffsets<u8>> {
let offset = self.offsets.lowering_callee(index);
self.vmlowering_field(offset)
}
/// Get the [`Field`] for the host data of the `index`th lowering in the
/// `VMComponentContext`'s `lowerings` array.
pub fn vmcomponent_lowering_data(
&mut self,
index: LoweredIndex,
) -> Field<'_, VMComponentOffsets<u8>> {
let offset = self.offsets.lowering_data(index);
self.vmlowering_field(offset)
}
/// Get the [`Field`] at `offset` within a `VMLowering` inlined into the
/// `VMComponentContext`.
///
/// Unlike the other aggregates inlined into a vmctx, `VMLowering` is not one
/// of the types defined by `for_each_vm_type!`, and so has no alias region of
/// its own to rebase onto the aggregate's offset. Its fields are therefore
/// part of the containing `VMComponentContext`'s region, keyed by their
/// offset within it.
fn vmlowering_field(&mut self, offset: u32) -> Field<'_, VMComponentOffsets<u8>> {
let ty = self.pointer_type;
Field::new(
self,
VmType::VMComponentContext,
offset,
ir::MemFlagsData::trusted(),
ty,
)
}
}
/// Stack-switching and continuation-object methods.
impl<Offsets> AliasRegions<Offsets>
where
Offsets: GetPtrSize,
{
/// Get the alias region for the `VMContRef::parent_chain` field.
///
/// Like `VMStoreContext::stack_chain`, this field is a two-pointer-wide
/// `VMStackChain`, which is not yet defined by `for_each_vm_type!` and so
/// has no alias region of its own. Its bytes are therefore mapped to the
/// containing `VMContRef`'s regions, keyed by the field's offset within it.
pub fn vm_cont_ref_parent_chain_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
let offset = self.offsets.get_ptr_size().vm_cont_ref().parent_chain();
self.region(
func,
AliasRegionKey::Vm {
ty: VmType::VMContRef,
offset: offset.into(),
},
)
}
/// Get the alias region for the top-of-stack pointer at the start of the
/// `VMContRef::stack` field.
///
/// `VMContinuationStack` is a platform-specific struct private to the
/// runtime, is not one of the types defined by `for_each_vm_type!`, and so
/// has no alias region of its own; as with `parent_chain` above, its bytes
/// are mapped to the containing `VMContRef`'s regions.
pub fn vm_cont_ref_top_of_stack_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
let offset = self.offsets.get_ptr_size().vm_cont_ref().stack();
self.region(
func,
AliasRegionKey::Vm {
ty: VmType::VMContRef,
offset: offset.into(),
},
)
}
/// Region for a continuation's stack memory: its payload/handler data
/// buffers and the control-context records stored on the continuation
/// stack.
///
/// These are distinct from the `VMContRef` object itself (which only holds
/// pointers to them).
pub fn continuation_stack_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
self.region(
func,
AliasRegionKey::Vm {
ty: VmType::ContinuationStackMemory,
offset: 0,
},
)
}
}
/// Debug-assert that every CLIF memory instruction in `func` carries an alias
/// region.
///
/// The Wasm-to-CLIF translation in `crates/cranelift/*` tags 100% of the memory
/// instructions it emits (loads, stores, and atomics) with an alias region.
/// This helper upholds that invariant; it is called at each
/// `FunctionBuilder::finalize` site, after finalization so that the stack-map
/// spills inserted by the safepoint pass (tagged via the
/// `FunctionBuilder::make_stack_map_alias_region` hook) are checked too.
pub(crate) fn debug_assert_all_mem_insts_have_alias_regions(func: &ir::Function) {
if cfg!(debug_assertions) {
for block in func.layout.blocks() {
for inst in func.layout.block_insts(block) {
// Only loads, stores, and atomics access memory. Some non-memory
// instructions (e.g. `bitcast`) also carry `MemFlags` purely for
// lane/byte-order and legitimately have no alias region, so they
// are excluded here.
let opcode = func.dfg.insts[inst].opcode();
if !opcode.can_load() && !opcode.can_store() {
continue;
}
if let Some(flags) = func.dfg.insts[inst].memflags() {
debug_assert!(
func.dfg.mem_flags[flags].alias_region().is_some(),
"CLIF memory instruction emitted without an alias region: {}",
func.dfg.display_inst(inst),
);
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Collisions are sound and only cost an optimization, so this is not a
/// correctness test: it exists so that anything reshuffling the key hashes
/// says so here rather than as `tests/disas` diffs where two regions have
/// quietly merged. If it fires, find a new `SEED` as described on
/// `into_raw`, additionally covering the keys below.
#[test]
fn gc_keys_do_not_collide() {
let ty = ModuleInternedTypeIndex::from_u32;
let mut keys = vec![
AliasRegionKey::GcArrayLength,
AliasRegionKey::GcExnTagInstance,
AliasRegionKey::GcExnTagDefined,
AliasRegionKey::Vm {
ty: VmType::VMGcHeader,
offset: 0,
},
AliasRegionKey::Vm {
ty: VmType::VMGcHeader,
offset: 4,
},
AliasRegionKey::Vm {
ty: VmType::VMDrcHeader,
offset: 8,
},
AliasRegionKey::Vm {
ty: VmType::VMDrcHeader,
offset: 16,
},
AliasRegionKey::Vm {
ty: VmType::VMCopyingHeader,
offset: 8,
},
];
for t in 0..3 {
for field in 0..2 {
keys.push(AliasRegionKey::GcStructField { ty: ty(t), field });
}
keys.push(AliasRegionKey::GcArrayElements { ty: ty(t) });
}
for (i, a) in keys.iter().enumerate() {
for b in &keys[i + 1..] {
assert_ne!(
a.into_raw(),
b.into_raw(),
"alias region keys {a:?} and {b:?} collide on \
user_id {}; a new `SEED` is needed",
a.into_raw(),
);
}
}
}
}