Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Migrate some `VM*` type fields to macro-generated helpers (#14195)

Nick Fitzgeralddece27d3abcf
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index a55f7b5533..d1700dbe85 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -318,6 +318,11 @@ pub struct AliasRegions<Offsets> {
}
impl<Offsets> AliasRegions<Offsets> {
+ /// Get the offsets this `AliasRegions` computes its field offsets from.
+ pub fn offsets(&self) -> &Offsets {
+ &self.offsets
+ }
+
/// Make the alias region for a stack map.
pub fn stack_map_region(
regions: &mut ir::AliasRegionSet,
@@ -416,26 +421,53 @@ impl<'a, Offsets> Field<'a, Offsets> {
/// module being compiled (e.g. whether a memory can be relocated) rather
/// than being a static property of the field; this method allows callers
/// to mark the load as `readonly` in these cases.
- pub fn readonly(&mut self) -> &mut Self {
+ pub fn readonly(mut self) -> Self {
self.flags.set_readonly();
self
}
+ /// Mark accesses of this field as `readonly` if and only if `readonly` is
+ /// `true`.
+ ///
+ /// See the note on [`Field::readonly`].
+ pub fn readonly_if(self, readonly: bool) -> Self {
+ if readonly { self.readonly() } else { self }
+ }
+
/// Mark accesses of this field as `can_move`.
///
/// See the note on [`Field::readonly`].
- pub fn can_move(&mut self) -> &mut Self {
+ pub fn can_move(mut self) -> Self {
self.flags = self.flags.with_can_move();
self
}
+ /// Mark accesses of this field as `can_move` if and only if `can_move` is
+ /// `true`.
+ ///
+ /// See the note on [`Field::readonly`].
+ pub fn can_move_if(self, can_move: bool) -> Self {
+ if can_move { self.can_move() } else { self }
+ }
+
+ /// Set the trap code for accesses of this field.
+ ///
+ /// A `Field`'s accesses do not trap by default.
+ ///
+ /// Note that when signals-based traps are disabled, callers must use the
+ /// explicit call-to-host trapping codegen instead.
+ pub fn trap_code(mut self, code: Option<ir::TrapCode>) -> Self {
+ self.flags = self.flags.with_trap_code(code);
+ self
+ }
+
/// Cast this field to the given type.
///
/// This can be used, for example, to cast a `Field` that points to a
/// `VMGlobalDefinition`'s storage (a `[u8; 16]` represented as
/// `ir::types::I8X16`) to the global's actual Wasm type's representation
/// (`ir::types::I32` for a Wasm `i32`).
- pub fn cast(&mut self, ty: ir::Type) -> &mut Self {
+ pub fn cast(mut self, ty: ir::Type) -> Self {
self.ty = ty;
self
}
@@ -1101,208 +1133,6 @@ where
);
}
- /// Load a pointer to the `*mut T` store data from a `*mut VMStoreContext`.
- pub fn vmstore_context_store_data(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted().with_readonly().with_can_move(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .store_data()
- .into(),
- )
- }
-
- /// Load the `VMStoreContext::execution_version` field.
- pub fn vmstore_context_execution_version(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- ir::types::I64,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .execution_version()
- .into(),
- )
- }
-
- /// Store the `VMStoreContext::execution_version` field.
- pub fn store_vmstore_context_execution_version(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- new_version: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .execution_version()
- .into(),
- new_version,
- )
- }
-
- /// Load the `VMStoreContext::fuel_consumed` field.
- pub fn vmstore_context_fuel_consumed(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- ir::types::I64,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .fuel_consumed()
- .into(),
- )
- }
-
- /// Store the `VMStoreContext::fuel_consumed` field.
- pub fn store_vmstore_context_fuel_consumed(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- fuel_consumed: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .fuel_consumed()
- .into(),
- fuel_consumed,
- )
- }
-
- /// Load the `VMStoreContext::epoch_deadline` field.
- pub fn vmstore_context_epoch_deadline(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- ir::types::I64,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .epoch_deadline()
- .into(),
- )
- }
-
- /// Get a `Load` for the `VmStoreContext::stack_limits` field.
- pub fn vmstore_context_stack_limit_load(&mut self, func: &mut ir::Function) -> Load {
- let offset = self
- .offsets
- .get_ptr_size()
- .vm_store_context()
- .stack_limit()
- .into();
- let region = self.vmstore_context_region(func, offset);
- Load {
- offset,
- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- ty: self.pointer_type,
- }
- }
-
- /// Load the `VMStoreContext::stack_limit` field.
- pub fn vmstore_context_stack_limit(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_stack_limit_load(cursor.func)
- .emit(cursor, vmstore_ctx)
- }
-
- /// Store the `VMStoreContext::stack_limit` field.
- pub fn store_vmstore_context_stack_limit(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- stack_limit: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .stack_limit()
- .into(),
- stack_limit,
- )
- }
-
- /// Load the `VMStoreContext::current_thread` field (the JIT-visible
- /// deferred-thread pointer; see `VMLazyThread`).
- pub fn vmstore_context_current_thread(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .current_thread()
- .into(),
- )
- }
-
- /// Store the `VMStoreContext::current_thread` field.
- pub fn store_vmstore_context_current_thread(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- new_thread: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .current_thread()
- .into(),
- new_thread,
- )
- }
-
/// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).
///
/// The caller supplies the base flags because whether the base pointer is
@@ -1366,163 +1196,6 @@ where
.emit(cursor, vmstore_ctx)
}
- /// Load the `VMStoreContext::last_wasm_entry_fp` field.
- pub fn vmstore_context_last_wasm_entry_fp(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .last_wasm_entry_fp()
- .into(),
- )
- }
-
- /// Load the `VMStoreContext::last_wasm_entry_sp` field.
- pub fn vmstore_context_last_wasm_entry_sp(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .last_wasm_entry_sp()
- .into(),
- )
- }
-
- /// Load the `VMStoreContext::last_wasm_entry_trap_handler` field.
- pub fn vmstore_context_last_wasm_entry_trap_handler(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- ) -> ir::Value {
- self.vmstore_context_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .last_wasm_entry_trap_handler()
- .into(),
- )
- }
-
- /// Store the `VMStoreContext::last_wasm_entry_fp` field.
- pub fn store_vmstore_context_last_wasm_entry_fp(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- fp: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .last_wasm_entry_fp()
- .into(),
- fp,
- )
- }
-
- /// Store the `VMStoreContext::last_wasm_entry_sp` field.
- pub fn store_vmstore_context_last_wasm_entry_sp(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- sp: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .last_wasm_entry_sp()
- .into(),
- sp,
- )
- }
-
- /// Store the `VMStoreContext::last_wasm_entry_trap_handler` field.
- pub fn store_vmstore_context_last_wasm_entry_trap_handler(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- trap_handler: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .last_wasm_entry_trap_handler()
- .into(),
- trap_handler,
- )
- }
-
- /// Store the `VMStoreContext::last_wasm_exit_trampoline_fp` field.
- pub fn store_vmstore_context_last_wasm_exit_trampoline_fp(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- fp: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .last_wasm_exit_trampoline_fp()
- .into(),
- fp,
- )
- }
-
- /// Store the `VMStoreContext::last_wasm_exit_pc` field.
- pub fn store_vmstore_context_last_wasm_exit_pc(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmstore_ctx: ir::Value,
- pc: ir::Value,
- ) {
- self.vmstore_context_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmstore_ctx,
- self.offsets
- .get_ptr_size()
- .vm_store_context()
- .last_wasm_exit_pc()
- .into(),
- pc,
- )
- }
-
/// Get the alias region for the `VMStoreContext::stack_chain` field.
///
/// The `VMStackChain` is two pointers wide and is emitted by the stack
@@ -1638,105 +1311,6 @@ where
);
}
- /// Load `VMDeferredThread::parent` (the current thread this frame replaced).
- pub fn vmdeferred_thread_parent(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmdeferred_thread_ptr: ir::Value,
- ) -> ir::Value {
- self.vmdeferred_thread_load(
- cursor,
- self.pointer_type,
- ir::MemFlagsData::trusted(),
- vmdeferred_thread_ptr,
- self.offsets
- .get_ptr_size()
- .vm_deferred_thread()
- .parent()
- .into(),
- )
- }
-
- /// Store `VMDeferredThread::parent`.
- pub fn store_vmdeferred_thread_parent(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmdeferred_thread_ptr: ir::Value,
- parent: ir::Value,
- ) {
- self.vmdeferred_thread_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmdeferred_thread_ptr,
- self.offsets
- .get_ptr_size()
- .vm_deferred_thread()
- .parent()
- .into(),
- parent,
- )
- }
-
- /// Store `VMDeferredThread::caller_instance`.
- pub fn store_vmdeferred_thread_caller_instance(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmdeferred_thread_ptr: ir::Value,
- caller_instance: ir::Value,
- ) {
- self.vmdeferred_thread_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmdeferred_thread_ptr,
- self.offsets
- .get_ptr_size()
- .vm_deferred_thread()
- .caller_instance()
- .into(),
- caller_instance,
- )
- }
-
- /// Store `VMDeferredThread::callee_async`.
- pub fn store_vmdeferred_thread_callee_async(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmdeferred_thread_ptr: ir::Value,
- callee_async: ir::Value,
- ) {
- self.vmdeferred_thread_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmdeferred_thread_ptr,
- self.offsets
- .get_ptr_size()
- .vm_deferred_thread()
- .callee_async()
- .into(),
- callee_async,
- )
- }
-
- /// Store `VMDeferredThread::callee_instance`.
- pub fn store_vmdeferred_thread_callee_instance(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- vmdeferred_thread_ptr: ir::Value,
- callee_instance: ir::Value,
- ) {
- self.vmdeferred_thread_store(
- cursor,
- ir::MemFlagsData::trusted(),
- vmdeferred_thread_ptr,
- self.offsets
- .get_ptr_size()
- .vm_deferred_thread()
- .callee_instance()
- .into(),
- callee_instance,
- )
- }
-
/// Load `VMDeferredThread::saved_context[i]` (a saved `context.{get,set}`
/// slot).
pub fn vmdeferred_thread_saved_context(
@@ -1780,104 +1354,6 @@ where
}
}
-/// `VMFuncRef`-related methods.
-impl<Offsets> AliasRegions<Offsets>
-where
- Offsets: GetPtrSize,
-{
- fn vmfuncref_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {
- self.region(
- func,
- AliasRegionKey::Vm {
- ty: VmType::VMFuncRef,
- offset,
- },
- )
- }
-
- /// Load the `VMFuncRef::type_index` field out of a `*const VMFuncRef`.
- pub fn vmfuncref_type_index(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- base_flags: ir::MemFlagsData,
- funcref: ir::Value,
- ) -> ir::Value {
- let offset = self.offsets.get_ptr_size().vm_func_ref().type_index();
- let region = self.vmfuncref_region(cursor.func, offset.into());
- let ty = ir::Type::int_with_byte_size(
- self.offsets
- .get_ptr_size()
- .size_of_vmshared_type_index()
- .into(),
- )
- .unwrap();
- cursor.ins().load(
- ty,
- base_flags.with_alias_region(Some(region)),
- funcref,
- i32::from(offset),
- )
- }
-
- /// Load the `VMFuncRef::wasm_call` field out of a `*const VMFuncRef`.
- ///
- /// The caller supplies the base flags because this load may carry an
- /// optional trap code for the null-funcref case.
- pub fn vmfuncref_wasm_call(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- base_flags: ir::MemFlagsData,
- funcref: ir::Value,
- ) -> ir::Value {
- let offset = self.offsets.get_ptr_size().vm_func_ref().wasm_call();
- let region = self.vmfuncref_region(cursor.func, offset.into());
- cursor.ins().load(
- self.pointer_type,
- base_flags.with_alias_region(Some(region)),
- funcref,
- i32::from(offset),
- )
- }
-
- /// Load the `VMFuncRef::vmctx` field out of a `*const VMFuncRef`.
- pub fn vmfuncref_vmctx(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- base_flags: ir::MemFlagsData,
- funcref: ir::Value,
- ) -> ir::Value {
- let offset = self.offsets.get_ptr_size().vm_func_ref().vmctx();
- let region = self.vmfuncref_region(cursor.func, offset.into());
- cursor.ins().load(
- self.pointer_type,
- base_flags.with_alias_region(Some(region)),
- funcref,
- i32::from(offset),
- )
- }
-
- /// Load the `array_call` field of the `VMFuncRef` inlined in a
- /// `VMArrayCallHostFuncContext`.
- pub fn vmarray_call_host_func_context_array_call(
- &mut self,
- cursor: &mut FuncCursor<'_>,
- host_func_ctx: ir::Value,
- ) -> ir::Value {
- let func_ref = self
- .offsets
- .get_ptr_size()
- .vmarray_call_host_func_context_func_ref();
- let field = self.offsets.get_ptr_size().vm_func_ref().array_call();
- let region = self.vmfuncref_region(cursor.func, field.into());
- cursor.ins().load(
- self.pointer_type,
- ir::MemFlagsData::trusted().with_alias_region(Some(region)),
- host_func_ctx,
- i32::from(func_ref) + i32::from(field),
- )
- }
-}
-
/// `[VMSharedTypeIndex]`-related methods.
impl<Offsets> AliasRegions<Offsets>
where
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 7aa781c40f..3e4c7b25b0 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -41,7 +41,7 @@ use wasmtime_environ::{
Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,
DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder,
FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, GetPtrSize, HostCall,
- InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder,
+ InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize,
StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables,
WasmFuncType, WasmValType, prelude::*,
};
@@ -252,8 +252,15 @@ impl Compiler {
// Load the actual callee out of the
// `VMArrayCallHostFuncContext::host_func`.
+ let func_ref_offset = alias_regions
+ .offsets()
+ .get_ptr_size()
+ .vmarray_call_host_func_context_func_ref();
let callee = alias_regions
- .vmarray_call_host_func_context_array_call(&mut builder.cursor(), callee_vmctx);
+ .vm_func_ref()
+ .array_call()
+ .relative_to(func_ref_offset.into())
+ .load(&mut builder.cursor(), callee_vmctx);
// Do an indirect call to the callee.
let callee_signature = builder.func.import_signature(array_call_sig);
@@ -273,9 +280,11 @@ impl Compiler {
.store_context()
.load(&mut builder.cursor(), caller_vmctx);
let old_version = alias_regions
- .vmstore_context_execution_version(&mut builder.cursor(), vmstore_ctx_ptr);
+ .vm_store_context()
+ .execution_version()
+ .load(&mut builder.cursor(), vmstore_ctx_ptr);
let new_version = builder.ins().iadd_imm_s(old_version, 1);
- alias_regions.store_vmstore_context_execution_version(
+ alias_regions.vm_store_context().execution_version().store(
&mut builder.cursor(),
vmstore_ctx_ptr,
new_version,
@@ -562,7 +571,9 @@ impl wasmtime_environ::Compiler for Compiler {
.to_deferred_load(&mut context.func);
let stack_limit = func_env
.alias_regions
- .vmstore_context_stack_limit_load(&mut context.func);
+ .vm_store_context()
+ .stack_limit()
+ .to_deferred_load(&mut context.func);
let stack_limit = VmctxLoadChain::new([store_ctx, stack_limit].into());
if self.tunables.signals_based_traps {
let stack_limit = stack_limit.emit_global(&mut context.func);
@@ -1588,7 +1599,7 @@ impl Compiler {
pointer_type,
i64::try_from(wasmtime_environ::VM_LAZY_THREAD_FORCED).unwrap(),
);
- alias_regions.store_vmstore_context_current_thread(
+ alias_regions.vm_store_context().current_thread().store(
&mut builder.cursor(),
vm_store_ctx,
forced,
@@ -1858,13 +1869,13 @@ fn save_last_wasm_entry_context<O>(
{
// Save the current fp/sp of the entry trampoline into the `VMStoreContext`.
let fp = builder.ins().get_frame_pointer(pointer_type);
- alias_regions.store_vmstore_context_last_wasm_entry_fp(
+ alias_regions.vm_store_context().last_wasm_entry_fp().store(
&mut builder.cursor(),
vm_store_context,
fp,
);
let sp = builder.ins().get_stack_pointer(pointer_type);
- alias_regions.store_vmstore_context_last_wasm_entry_sp(
+ alias_regions.vm_store_context().last_wasm_entry_sp().store(
&mut builder.cursor(),
vm_store_context,
sp,
@@ -1875,11 +1886,10 @@ fn save_last_wasm_entry_context<O>(
let trap_handler = builder
.ins()
.get_exception_handler_address(pointer_type, block, 0);
- alias_regions.store_vmstore_context_last_wasm_entry_trap_handler(
- &mut builder.cursor(),
- vm_store_context,
- trap_handler,
- );
+ alias_regions
+ .vm_store_context()
+ .last_wasm_entry_trap_handler()
+ .store(&mut builder.cursor(), vm_store_context, trap_handler);
}
fn save_last_wasm_exit_fp_and_pc<O>(
@@ -1894,15 +1904,18 @@ fn save_last_wasm_exit_fp_and_pc<O>(
// this so that it can know the SP (bottom of frame) for the very
// last Wasm frame.
let trampoline_fp = builder.ins().get_frame_pointer(pointer_type);
- alias_regions.store_vmstore_context_last_wasm_exit_trampoline_fp(
- &mut builder.cursor(),
- limits,
- trampoline_fp,
- );
+ alias_regions
+ .vm_store_context()
+ .last_wasm_exit_trampoline_fp()
+ .store(&mut builder.cursor(), limits, trampoline_fp);
// Finally save the Wasm return address to the limits.
let wasm_pc = builder.ins().get_return_address(pointer_type);
- alias_regions.store_vmstore_context_last_wasm_exit_pc(&mut builder.cursor(), limits, wasm_pc);
+ alias_regions.vm_store_context().last_wasm_exit_pc().store(
+ &mut builder.cursor(),
+ limits,
+ wasm_pc,
+ );
}
fn key_to_name(key: FuncKey) -> ir::UserFuncName {
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index 2fd2f207c4..1a77491e78 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -1188,8 +1188,6 @@ impl<'a> TrampolineCompiler<'a> {
&[],
);
- let trusted = ir::MemFlagsData::trusted().with_readonly();
-
self.builder.switch_to_block(run_destructor_block);
// If this is a component-defined resource, the `may_leave` flag must be
@@ -1245,16 +1243,16 @@ impl<'a> TrampolineCompiler<'a> {
.ins()
.trapz(dtor_func_ref, TRAP_INTERNAL_ASSERT);
}
- let func_addr = self.alias_regions.vmfuncref_wasm_call(
- &mut self.builder.cursor(),
- trusted,
- dtor_func_ref,
- );
- let callee_vmctx = self.alias_regions.vmfuncref_vmctx(
- &mut self.builder.cursor(),
- trusted,
- dtor_func_ref,
- );
+ let func_addr = self
+ .alias_regions
+ .vm_func_ref()
+ .wasm_call()
+ .load(&mut self.builder.cursor(), dtor_func_ref);
+ let callee_vmctx = self
+ .alias_regions
+ .vm_func_ref()
+ .vmctx()
+ .load(&mut self.builder.cursor(), dtor_func_ref);
let sig = crate::wasm_call_signature(self.isa, self.signature, &self.compiler.tunables);
let sig_ref = self.builder.import_signature(sig);
@@ -2164,7 +2162,9 @@ where
let data_address = self
.traps
.alias_regions()
- .vmstore_context_store_data(&mut self.builder.cursor(), store_ctx);
+ .vm_store_context()
+ .store_data()
+ .load(&mut self.builder.cursor(), store_ctx);
// Zero-extend the address if we are on a 32-bit architecture.
let data_address = match pointer_type.bits() {
diff --git a/crates/cranelift/src/component_sync_call.rs b/crates/cranelift/src/component_sync_call.rs
index 6f2d7581f6..831603afcf 100644
--- a/crates/cranelift/src/component_sync_call.rs
+++ b/crates/cranelift/src/component_sync_call.rs
@@ -69,21 +69,27 @@ where
.load(&mut builder.cursor(), vmctx);
// Link the previous current thread in as this frame's parent.
- let parent = alias_regions.vmstore_context_current_thread(&mut builder.cursor(), vmstore);
- alias_regions.store_vmdeferred_thread_parent(&mut builder.cursor(), slot_addr, parent);
+ let parent = alias_regions
+ .vm_store_context()
+ .current_thread()
+ .load(&mut builder.cursor(), vmstore);
+ alias_regions
+ .vm_deferred_thread()
+ .parent()
+ .store(&mut builder.cursor(), slot_addr, parent);
// Record the deferred `enter_sync_call` arguments.
- alias_regions.store_vmdeferred_thread_caller_instance(
+ alias_regions.vm_deferred_thread().caller_instance().store(
&mut builder.cursor(),
slot_addr,
args.caller_instance,
);
- alias_regions.store_vmdeferred_thread_callee_async(
+ alias_regions.vm_deferred_thread().callee_async().store(
&mut builder.cursor(),
slot_addr,
args.callee_async,
);
- alias_regions.store_vmdeferred_thread_callee_instance(
+ alias_regions.vm_deferred_thread().callee_instance().store(
&mut builder.cursor(),
slot_addr,
args.callee_instance,
@@ -114,7 +120,11 @@ where
}
// Publish the deferred thread as the store's current thread.
- alias_regions.store_vmstore_context_current_thread(&mut builder.cursor(), vmstore, slot_addr);
+ alias_regions.vm_store_context().current_thread().store(
+ &mut builder.cursor(),
+ vmstore,
+ slot_addr,
+ );
slot
}
@@ -159,7 +169,10 @@ where
.vmctx()
.store_context()
.load(&mut builder.cursor(), vmctx);
- let cur = alias_regions.vmstore_context_current_thread(&mut builder.cursor(), vmstore);
+ let cur = alias_regions
+ .vm_store_context()
+ .current_thread()
+ .load(&mut builder.cursor(), vmstore);
let is_fast = builder.ins().icmp(IntCC::Equal, cur, slot_addr);
let fast_block = builder.create_block();
@@ -173,8 +186,14 @@ where
// Fast path: pop the deferred thread and restore the caller's context.
builder.switch_to_block(fast_block);
- let parent = alias_regions.vmdeferred_thread_parent(&mut builder.cursor(), slot_addr);
- alias_regions.store_vmstore_context_current_thread(&mut builder.cursor(), vmstore, parent);
+ let parent = alias_regions
+ .vm_deferred_thread()
+ .parent()
+ .load(&mut builder.cursor(), slot_addr);
+ alias_regions
+ .vm_store_context()
+ .current_thread()
+ .store(&mut builder.cursor(), vmstore, parent);
for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
let saved =
alias_regions.vmdeferred_thread_saved_context(&mut builder.cursor(), slot_addr, i);
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index fe762b7074..c93057f432 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -667,7 +667,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
let vmstore_ctx = self.get_vmstore_context_ptr(builder);
let fuel = self
.alias_regions
- .vmstore_context_fuel_consumed(&mut builder.cursor(), vmstore_ctx);
+ .vm_store_context()
+ .fuel_consumed()
+ .load(&mut builder.cursor(), vmstore_ctx);
builder.def_var(self.fuel_var, fuel);
}
@@ -676,7 +678,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
fn fuel_save_from_var(&mut self, builder: &mut FunctionBuilder<'_>) {
let vmstore_ctx = self.get_vmstore_context_ptr(builder);
let fuel_consumed = builder.use_var(self.fuel_var);
- self.alias_regions.store_vmstore_context_fuel_consumed(
+ self.alias_regions.vm_store_context().fuel_consumed().store(
&mut builder.cursor(),
vmstore_ctx,
fuel_consumed,
@@ -859,7 +861,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
let vmstore_ctx = self.get_vmstore_context_ptr(builder);
let deadline = self
.alias_regions
- .vmstore_context_epoch_deadline(&mut builder.cursor(), vmstore_ctx);
+ .vm_store_context()
+ .epoch_deadline()
+ .load(&mut builder.cursor(), vmstore_ctx);
builder.def_var(self.epoch_deadline_var, deadline);
self.epoch_check_cached(builder, cur_epoch_value, continuation_block);
@@ -1617,12 +1621,13 @@ impl FuncEnvironment<'_> {
.vmctx()
.memories(def_index)
.to_deferred_load(func);
- let mut base = self.alias_regions.vm_memory_definition().base();
- base.can_move();
- if base_readonly {
- base.readonly();
- }
- let base = base.to_deferred_load(func);
+ let base = self
+ .alias_regions
+ .vm_memory_definition()
+ .base()
+ .can_move()
+ .readonly_if(base_readonly)
+ .to_deferred_load(func);
let len = self
.alias_regions
.vm_memory_definition()
@@ -1639,12 +1644,14 @@ impl FuncEnvironment<'_> {
// field's offset.
let owned_index = self.module.owned_memory_index(def_index);
let vmctx_off = self.offsets.owned_memories().at(owned_index);
- let mut base = self.alias_regions.vm_memory_definition().base();
- base.can_move();
- if base_readonly {
- base.readonly();
- }
- let base = base.relative_to(vmctx_off).to_deferred_load(func);
+ let base = self
+ .alias_regions
+ .vm_memory_definition()
+ .base()
+ .can_move()
+ .readonly_if(base_readonly)
+ .relative_to(vmctx_off)
+ .to_deferred_load(func);
let len = self
.alias_regions
.vm_memory_definition()
@@ -1664,12 +1671,13 @@ impl FuncEnvironment<'_> {
.from()
.relative_to(import_off)
.to_deferred_load(func);
- let mut base = self.alias_regions.vm_memory_definition().base();
- base.can_move();
- if base_readonly {
- base.readonly();
- }
- let base = base.to_deferred_load(func);
+ let base = self
+ .alias_regions
+ .vm_memory_definition()
+ .base()
+ .can_move()
+ .readonly_if(base_readonly)
+ .to_deferred_load(func);
let len = self
.alias_regions
.vm_memory_definition()
@@ -1717,24 +1725,26 @@ impl FuncEnvironment<'_> {
// A defined table's `VMTableDefinition` is inlined into the vmctx,
// reached at an absolute `vmctx` offset.
let vmctx_off = self.offsets.tables().at(def_index);
- let mut base = self.alias_regions.vm_table_definition().base();
- if is_static {
- base.readonly().can_move();
- }
let base = VmctxLoadChain::new(smallvec![
- base.relative_to(vmctx_off).to_deferred_load(func)
+ self.alias_regions
+ .vm_table_definition()
+ .base()
+ .readonly_if(is_static)
+ .can_move_if(is_static)
+ .relative_to(vmctx_off)
+ .to_deferred_load(func)
]);
let bound = if is_static {
TableSize::Static {
bound: table.limits.min,
}
} else {
- let mut current_elements =
- self.alias_regions.vm_table_definition().current_elements();
- current_elements.cast(bound_ty);
TableSize::Dynamic {
bound: VmctxLoadChain::new(smallvec![
- current_elements
+ self.alias_regions
+ .vm_table_definition()
+ .current_elements()
+ .cast(bound_ty)
.relative_to(vmctx_off)
.to_deferred_load(func)
]),
@@ -1751,11 +1761,15 @@ impl FuncEnvironment<'_> {
.from()
.relative_to(import_off)
.to_deferred_load(func);
- let mut base = self.alias_regions.vm_table_definition().base();
- if is_static {
- base.readonly().can_move();
- }
- let base = VmctxLoadChain::new(smallvec![from, base.to_deferred_load(func)]);
+ let base = VmctxLoadChain::new(smallvec![
+ from,
+ self.alias_regions
+ .vm_table_definition()
+ .base()
+ .readonly_if(is_static)
+ .can_move_if(is_static)
+ .to_deferred_load(func),
+ ]);
let bound = if is_static {
TableSize::Static {
bound: table.limits.min,
@@ -2266,18 +2280,20 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
//
// Note that the callee may be null in which case this load may
// trap. If so use the `TRAP_INDIRECT_CALL_TO_NULL` trap code.
- let mut mem_flags = ir::MemFlagsData::trusted().with_readonly();
- if self.env.clif_memory_traps_enabled() {
- mem_flags = mem_flags.with_trap_code(Some(crate::TRAP_INDIRECT_CALL_TO_NULL));
+ let trap_code = if self.env.clif_memory_traps_enabled() {
+ Some(crate::TRAP_INDIRECT_CALL_TO_NULL)
} else {
self.env
.trapz(self.builder, funcref_ptr, crate::TRAP_INDIRECT_CALL_TO_NULL);
- }
- let callee_sig_id = self.env.alias_regions.vmfuncref_type_index(
- &mut self.builder.cursor(),
- mem_flags,
- funcref_ptr,
- );
+ None
+ };
+ let callee_sig_id = self
+ .env
+ .alias_regions
+ .vm_func_ref()
+ .type_index()
+ .trap_code(trap_code)
+ .load(&mut self.builder.cursor(), funcref_ptr);
// Check that they match: in the case of Wasm GC, this means doing a
// full subtype check. Otherwise, we do a simple equality check.
@@ -2335,24 +2351,27 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// optional trap code provided by the caller of `unchecked_call` which
// will handle the case where this is either already known to be
// non-null or may trap.
- let mem_flags = ir::MemFlagsData::trusted().with_readonly();
- let mut callee_flags = mem_flags;
- if self.env.clif_memory_traps_enabled() {
- callee_flags = callee_flags.with_trap_code(callee_load_trap_code);
+ let callee_load_trap_code = if self.env.clif_memory_traps_enabled() {
+ callee_load_trap_code
} else {
if let Some(trap) = callee_load_trap_code {
self.env.trapz(self.builder, callee, trap);
}
- }
- let func_addr = self.env.alias_regions.vmfuncref_wasm_call(
- &mut self.builder.cursor(),
- callee_flags,
- callee,
- );
- let callee_vmctx =
- self.env
- .alias_regions
- .vmfuncref_vmctx(&mut self.builder.cursor(), mem_flags, callee);
+ None
+ };
+ let func_addr = self
+ .env
+ .alias_regions
+ .vm_func_ref()
+ .wasm_call()
+ .trap_code(callee_load_trap_code)
+ .load(&mut self.builder.cursor(), callee);
+ let callee_vmctx = self
+ .env
+ .alias_regions
+ .vm_func_ref()
+ .vmctx()
+ .load(&mut self.builder.cursor(), callee);
(func_addr, callee_vmctx)
}
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index b88d21c050..8987cd9a25 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -1306,11 +1306,11 @@ pub fn translate_ref_test(
let expected_shared_ty =
func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);
- let actual_shared_ty = func_env.alias_regions.vmfuncref_type_index(
- &mut builder.cursor(),
- ir::MemFlagsData::trusted().with_readonly(),
- val,
- );
+ let actual_shared_ty = func_env
+ .alias_regions
+ .vm_func_ref()
+ .type_index()
+ .load(&mut builder.cursor(), val);
func_env.is_subtype(
builder,
diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
index 56386c797c..d263620b3c 100644
--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs
+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
@@ -790,7 +790,7 @@ pub(crate) mod stack_switching_helpers {
let stack_limit = env
.alias_regions
.stack_limit(&mut builder.cursor(), stack_limits_ptr);
- env.alias_regions.store_vmstore_context_stack_limit(
+ env.alias_regions.vm_store_context().stack_limit().store(
&mut builder.cursor(),
vmruntime_limits_ptr,
stack_limit,
@@ -799,26 +799,34 @@ pub(crate) mod stack_switching_helpers {
let last_wasm_entry_fp = env
.alias_regions
.last_wasm_entry_fp(&mut builder.cursor(), stack_limits_ptr);
- env.alias_regions.store_vmstore_context_last_wasm_entry_fp(
- &mut builder.cursor(),
- vmruntime_limits_ptr,
- last_wasm_entry_fp,
- );
+ env.alias_regions
+ .vm_store_context()
+ .last_wasm_entry_fp()
+ .store(
+ &mut builder.cursor(),
+ vmruntime_limits_ptr,
+ last_wasm_entry_fp,
+ );
let last_wasm_entry_sp = env
.alias_regions
.last_wasm_entry_sp(&mut builder.cursor(), stack_limits_ptr);
- env.alias_regions.store_vmstore_context_last_wasm_entry_sp(
- &mut builder.cursor(),
- vmruntime_limits_ptr,
- last_wasm_entry_sp,
- );
+ env.alias_regions
+ .vm_store_context()
+ .last_wasm_entry_sp()
+ .store(
+ &mut builder.cursor(),
+ vmruntime_limits_ptr,
+ last_wasm_entry_sp,
+ );
let last_wasm_entry_trap_handler = env
.alias_regions
.last_wasm_entry_trap_handler(&mut builder.cursor(), stack_limits_ptr);
env.alias_regions
- .store_vmstore_context_last_wasm_entry_trap_handler(
+ .vm_store_context()
+ .last_wasm_entry_trap_handler()
+ .store(
&mut builder.cursor(),
vmruntime_limits_ptr,
last_wasm_entry_trap_handler,
@@ -843,16 +851,19 @@ pub(crate) mod stack_switching_helpers {
// continuation's inline `VMStackLimits`.
let last_wasm_entry_fp = env
.alias_regions
- .vmstore_context_last_wasm_entry_fp(&mut builder.cursor(), vmruntime_limits_ptr);
+ .vm_store_context()
+ .last_wasm_entry_fp()
+ .load(&mut builder.cursor(), vmruntime_limits_ptr);
let last_wasm_entry_sp = env
.alias_regions
- .vmstore_context_last_wasm_entry_sp(&mut builder.cursor(), vmruntime_limits_ptr);
+ .vm_store_context()
+ .last_wasm_entry_sp()
+ .load(&mut builder.cursor(), vmruntime_limits_ptr);
let last_wasm_entry_trap_handler = env
.alias_regions
- .vmstore_context_last_wasm_entry_trap_handler(
- &mut builder.cursor(),
- vmruntime_limits_ptr,
- );
+ .vm_store_context()
+ .last_wasm_entry_trap_handler()
+ .load(&mut builder.cursor(), vmruntime_limits_ptr);
let vmcontref_region = env.alias_regions.vmcontref_region(builder.func);
let our_memflags =
@@ -884,7 +895,9 @@ pub(crate) mod stack_switching_helpers {
// Load from the `VMStoreContext`...
let stack_limit = env
.alias_regions
- .vmstore_context_stack_limit(&mut builder.cursor(), vmruntime_limits_ptr);
+ .vm_store_context()
+ .stack_limit()
+ .load(&mut builder.cursor(), vmruntime_limits_ptr);
// ...and store to this continuation's inline `VMStackLimits`.
let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit();
builder.ins().store(
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
index 8011ddd8c0..e6eadf1d4a 100644
--- a/crates/environ/src/vmtypes.rs
+++ b/crates/environ/src/vmtypes.rs
@@ -125,9 +125,19 @@ macro_rules! for_each_vm_type {
/// it means that the Wasm cannot actually call this function. But it does
/// mean that this field needs to be an `Option` even though it is non-null
/// the vast vast vast majority of the time.
+ ///
+ /// Once a `VMFuncRef` is exposed to compiled code this field
+ /// never changes again, so accesses of it are `readonly`. It is
+ /// not `can_move`, however: the load may be the one that traps
+ /// on a null funcref, and moving it would move that trap.
+ #[readonly]
pub wasm_call: Option<VmPtr<VMWasmCallFunction>>,
/// Function signature's type id.
+ ///
+ /// See the note about `readonly` and not `can_move` on
+ /// `wasm_call`.
+ #[readonly]
pub type_index: VMSharedTypeIndex,
/// The VM state associated with this function.
@@ -136,6 +146,10 @@ macro_rules! for_each_vm_type {
/// function being referenced: for core Wasm functions, this is a `*mut
/// VMContext`, for host functions it is a `*mut VMHostFuncContext`, and for
/// component functions it is a `*mut VMComponentContext`.
+ ///
+ /// See the note about `readonly` and not `can_move` on
+ /// `wasm_call`.
+ #[readonly]
pub vmctx: VmPtr<VMOpaqueContext>,
}
@@ -413,6 +427,11 @@ macro_rules! for_each_vm_type {
/// A pointer to the embedder's `T` inside a `Store<T>`, for use with the
/// `store-data-address` unsafe intrinsic.
+ ///
+ /// This pointer is fixed for the lifetime of the store, so loads
+ /// of it are `readonly` and `can_move`.
+ #[readonly]
+ #[can_move]
pub store_data: VmPtr<()>,
/// The range, in addresses, of the guard page that is currently in use.