A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Migrate some `VM*` type fields to macro-generated helpers (#14195)
Nick Fitzgeralddece27d3abcf
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex a55f7b5533..d1700dbe85 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -318,6 +318,11 @@ pub struct AliasRegions<Offsets> {}impl<Offsets> AliasRegions<Offsets> {+ /// Get the offsets this `AliasRegions` computes its field offsets from.+ pub fn offsets(&self) -> &Offsets {+ &self.offsets+ }+/// Make the alias region for a stack map.pub fn stack_map_region(regions: &mut ir::AliasRegionSet,@@ -416,26 +421,53 @@ impl<'a, Offsets> Field<'a, Offsets> {/// module being compiled (e.g. whether a memory can be relocated) rather/// than being a static property of the field; this method allows callers/// to mark the load as `readonly` in these cases.- pub fn readonly(&mut self) -> &mut Self {+ pub fn readonly(mut self) -> Self {self.flags.set_readonly();self}+ /// Mark accesses of this field as `readonly` if and only if `readonly` is+ /// `true`.+ ///+ /// See the note on [`Field::readonly`].+ pub fn readonly_if(self, readonly: bool) -> Self {+ if readonly { self.readonly() } else { self }+ }+/// Mark accesses of this field as `can_move`.////// See the note on [`Field::readonly`].- pub fn can_move(&mut self) -> &mut Self {+ pub fn can_move(mut self) -> Self {self.flags = self.flags.with_can_move();self}+ /// Mark accesses of this field as `can_move` if and only if `can_move` is+ /// `true`.+ ///+ /// See the note on [`Field::readonly`].+ pub fn can_move_if(self, can_move: bool) -> Self {+ if can_move { self.can_move() } else { self }+ }++ /// Set the trap code for accesses of this field.+ ///+ /// A `Field`'s accesses do not trap by default.+ ///+ /// Note that when signals-based traps are disabled, callers must use the+ /// explicit call-to-host trapping codegen instead.+ pub fn trap_code(mut self, code: Option<ir::TrapCode>) -> Self {+ self.flags = self.flags.with_trap_code(code);+ self+ }+/// Cast this field to the given type.////// This can be used, for example, to cast a `Field` that points to a/// `VMGlobalDefinition`'s storage (a `[u8; 16]` represented as/// `ir::types::I8X16`) to the global's actual Wasm type's representation/// (`ir::types::I32` for a Wasm `i32`).- pub fn cast(&mut self, ty: ir::Type) -> &mut Self {+ pub fn cast(mut self, ty: ir::Type) -> Self {self.ty = ty;self}@@ -1101,208 +1133,6 @@ where);}- /// Load a pointer to the `*mut T` store data from a `*mut VMStoreContext`.- pub fn vmstore_context_store_data(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted().with_readonly().with_can_move(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .store_data()- .into(),- )- }-- /// Load the `VMStoreContext::execution_version` field.- pub fn vmstore_context_execution_version(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- ir::types::I64,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .execution_version()- .into(),- )- }-- /// Store the `VMStoreContext::execution_version` field.- pub fn store_vmstore_context_execution_version(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- new_version: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .execution_version()- .into(),- new_version,- )- }-- /// Load the `VMStoreContext::fuel_consumed` field.- pub fn vmstore_context_fuel_consumed(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- ir::types::I64,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .fuel_consumed()- .into(),- )- }-- /// Store the `VMStoreContext::fuel_consumed` field.- pub fn store_vmstore_context_fuel_consumed(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- fuel_consumed: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .fuel_consumed()- .into(),- fuel_consumed,- )- }-- /// Load the `VMStoreContext::epoch_deadline` field.- pub fn vmstore_context_epoch_deadline(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- ir::types::I64,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .epoch_deadline()- .into(),- )- }-- /// Get a `Load` for the `VmStoreContext::stack_limits` field.- pub fn vmstore_context_stack_limit_load(&mut self, func: &mut ir::Function) -> Load {- let offset = self- .offsets- .get_ptr_size()- .vm_store_context()- .stack_limit()- .into();- let region = self.vmstore_context_region(func, offset);- Load {- offset,- flags: ir::MemFlagsData::trusted().with_alias_region(Some(region)),- ty: self.pointer_type,- }- }-- /// Load the `VMStoreContext::stack_limit` field.- pub fn vmstore_context_stack_limit(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_stack_limit_load(cursor.func)- .emit(cursor, vmstore_ctx)- }-- /// Store the `VMStoreContext::stack_limit` field.- pub fn store_vmstore_context_stack_limit(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- stack_limit: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .stack_limit()- .into(),- stack_limit,- )- }-- /// Load the `VMStoreContext::current_thread` field (the JIT-visible- /// deferred-thread pointer; see `VMLazyThread`).- pub fn vmstore_context_current_thread(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .current_thread()- .into(),- )- }-- /// Store the `VMStoreContext::current_thread` field.- pub fn store_vmstore_context_current_thread(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- new_thread: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .current_thread()- .into(),- new_thread,- )- }-/// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).////// The caller supplies the base flags because whether the base pointer is@@ -1366,163 +1196,6 @@ where.emit(cursor, vmstore_ctx)}- /// Load the `VMStoreContext::last_wasm_entry_fp` field.- pub fn vmstore_context_last_wasm_entry_fp(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .last_wasm_entry_fp()- .into(),- )- }-- /// Load the `VMStoreContext::last_wasm_entry_sp` field.- pub fn vmstore_context_last_wasm_entry_sp(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .last_wasm_entry_sp()- .into(),- )- }-- /// Load the `VMStoreContext::last_wasm_entry_trap_handler` field.- pub fn vmstore_context_last_wasm_entry_trap_handler(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- ) -> ir::Value {- self.vmstore_context_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .last_wasm_entry_trap_handler()- .into(),- )- }-- /// Store the `VMStoreContext::last_wasm_entry_fp` field.- pub fn store_vmstore_context_last_wasm_entry_fp(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- fp: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .last_wasm_entry_fp()- .into(),- fp,- )- }-- /// Store the `VMStoreContext::last_wasm_entry_sp` field.- pub fn store_vmstore_context_last_wasm_entry_sp(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- sp: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .last_wasm_entry_sp()- .into(),- sp,- )- }-- /// Store the `VMStoreContext::last_wasm_entry_trap_handler` field.- pub fn store_vmstore_context_last_wasm_entry_trap_handler(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- trap_handler: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .last_wasm_entry_trap_handler()- .into(),- trap_handler,- )- }-- /// Store the `VMStoreContext::last_wasm_exit_trampoline_fp` field.- pub fn store_vmstore_context_last_wasm_exit_trampoline_fp(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- fp: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .last_wasm_exit_trampoline_fp()- .into(),- fp,- )- }-- /// Store the `VMStoreContext::last_wasm_exit_pc` field.- pub fn store_vmstore_context_last_wasm_exit_pc(- &mut self,- cursor: &mut FuncCursor<'_>,- vmstore_ctx: ir::Value,- pc: ir::Value,- ) {- self.vmstore_context_store(- cursor,- ir::MemFlagsData::trusted(),- vmstore_ctx,- self.offsets- .get_ptr_size()- .vm_store_context()- .last_wasm_exit_pc()- .into(),- pc,- )- }-/// Get the alias region for the `VMStoreContext::stack_chain` field.////// The `VMStackChain` is two pointers wide and is emitted by the stack@@ -1638,105 +1311,6 @@ where);}- /// Load `VMDeferredThread::parent` (the current thread this frame replaced).- pub fn vmdeferred_thread_parent(- &mut self,- cursor: &mut FuncCursor<'_>,- vmdeferred_thread_ptr: ir::Value,- ) -> ir::Value {- self.vmdeferred_thread_load(- cursor,- self.pointer_type,- ir::MemFlagsData::trusted(),- vmdeferred_thread_ptr,- self.offsets- .get_ptr_size()- .vm_deferred_thread()- .parent()- .into(),- )- }-- /// Store `VMDeferredThread::parent`.- pub fn store_vmdeferred_thread_parent(- &mut self,- cursor: &mut FuncCursor<'_>,- vmdeferred_thread_ptr: ir::Value,- parent: ir::Value,- ) {- self.vmdeferred_thread_store(- cursor,- ir::MemFlagsData::trusted(),- vmdeferred_thread_ptr,- self.offsets- .get_ptr_size()- .vm_deferred_thread()- .parent()- .into(),- parent,- )- }-- /// Store `VMDeferredThread::caller_instance`.- pub fn store_vmdeferred_thread_caller_instance(- &mut self,- cursor: &mut FuncCursor<'_>,- vmdeferred_thread_ptr: ir::Value,- caller_instance: ir::Value,- ) {- self.vmdeferred_thread_store(- cursor,- ir::MemFlagsData::trusted(),- vmdeferred_thread_ptr,- self.offsets- .get_ptr_size()- .vm_deferred_thread()- .caller_instance()- .into(),- caller_instance,- )- }-- /// Store `VMDeferredThread::callee_async`.- pub fn store_vmdeferred_thread_callee_async(- &mut self,- cursor: &mut FuncCursor<'_>,- vmdeferred_thread_ptr: ir::Value,- callee_async: ir::Value,- ) {- self.vmdeferred_thread_store(- cursor,- ir::MemFlagsData::trusted(),- vmdeferred_thread_ptr,- self.offsets- .get_ptr_size()- .vm_deferred_thread()- .callee_async()- .into(),- callee_async,- )- }-- /// Store `VMDeferredThread::callee_instance`.- pub fn store_vmdeferred_thread_callee_instance(- &mut self,- cursor: &mut FuncCursor<'_>,- vmdeferred_thread_ptr: ir::Value,- callee_instance: ir::Value,- ) {- self.vmdeferred_thread_store(- cursor,- ir::MemFlagsData::trusted(),- vmdeferred_thread_ptr,- self.offsets- .get_ptr_size()- .vm_deferred_thread()- .callee_instance()- .into(),- callee_instance,- )- }-/// Load `VMDeferredThread::saved_context[i]` (a saved `context.{get,set}`/// slot).pub fn vmdeferred_thread_saved_context(@@ -1780,104 +1354,6 @@ where}}-/// `VMFuncRef`-related methods.-impl<Offsets> AliasRegions<Offsets>-where- Offsets: GetPtrSize,-{- fn vmfuncref_region(&mut self, func: &mut ir::Function, offset: u32) -> ir::AliasRegion {- self.region(- func,- AliasRegionKey::Vm {- ty: VmType::VMFuncRef,- offset,- },- )- }-- /// Load the `VMFuncRef::type_index` field out of a `*const VMFuncRef`.- pub fn vmfuncref_type_index(- &mut self,- cursor: &mut FuncCursor<'_>,- base_flags: ir::MemFlagsData,- funcref: ir::Value,- ) -> ir::Value {- let offset = self.offsets.get_ptr_size().vm_func_ref().type_index();- let region = self.vmfuncref_region(cursor.func, offset.into());- let ty = ir::Type::int_with_byte_size(- self.offsets- .get_ptr_size()- .size_of_vmshared_type_index()- .into(),- )- .unwrap();- cursor.ins().load(- ty,- base_flags.with_alias_region(Some(region)),- funcref,- i32::from(offset),- )- }-- /// Load the `VMFuncRef::wasm_call` field out of a `*const VMFuncRef`.- ///- /// The caller supplies the base flags because this load may carry an- /// optional trap code for the null-funcref case.- pub fn vmfuncref_wasm_call(- &mut self,- cursor: &mut FuncCursor<'_>,- base_flags: ir::MemFlagsData,- funcref: ir::Value,- ) -> ir::Value {- let offset = self.offsets.get_ptr_size().vm_func_ref().wasm_call();- let region = self.vmfuncref_region(cursor.func, offset.into());- cursor.ins().load(- self.pointer_type,- base_flags.with_alias_region(Some(region)),- funcref,- i32::from(offset),- )- }-- /// Load the `VMFuncRef::vmctx` field out of a `*const VMFuncRef`.- pub fn vmfuncref_vmctx(- &mut self,- cursor: &mut FuncCursor<'_>,- base_flags: ir::MemFlagsData,- funcref: ir::Value,- ) -> ir::Value {- let offset = self.offsets.get_ptr_size().vm_func_ref().vmctx();- let region = self.vmfuncref_region(cursor.func, offset.into());- cursor.ins().load(- self.pointer_type,- base_flags.with_alias_region(Some(region)),- funcref,- i32::from(offset),- )- }-- /// Load the `array_call` field of the `VMFuncRef` inlined in a- /// `VMArrayCallHostFuncContext`.- pub fn vmarray_call_host_func_context_array_call(- &mut self,- cursor: &mut FuncCursor<'_>,- host_func_ctx: ir::Value,- ) -> ir::Value {- let func_ref = self- .offsets- .get_ptr_size()- .vmarray_call_host_func_context_func_ref();- let field = self.offsets.get_ptr_size().vm_func_ref().array_call();- let region = self.vmfuncref_region(cursor.func, field.into());- cursor.ins().load(- self.pointer_type,- ir::MemFlagsData::trusted().with_alias_region(Some(region)),- host_func_ctx,- i32::from(func_ref) + i32::from(field),- )- }-}-/// `[VMSharedTypeIndex]`-related methods.impl<Offsets> AliasRegions<Offsets>wherediff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex 7aa781c40f..3e4c7b25b0 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -41,7 +41,7 @@ use wasmtime_environ::{Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder,FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, GetPtrSize, HostCall,- InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder,+ InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize,StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables,WasmFuncType, WasmValType, prelude::*,};@@ -252,8 +252,15 @@ impl Compiler {// Load the actual callee out of the// `VMArrayCallHostFuncContext::host_func`.+ let func_ref_offset = alias_regions+ .offsets()+ .get_ptr_size()+ .vmarray_call_host_func_context_func_ref();let callee = alias_regions- .vmarray_call_host_func_context_array_call(&mut builder.cursor(), callee_vmctx);+ .vm_func_ref()+ .array_call()+ .relative_to(func_ref_offset.into())+ .load(&mut builder.cursor(), callee_vmctx);// Do an indirect call to the callee.let callee_signature = builder.func.import_signature(array_call_sig);@@ -273,9 +280,11 @@ impl Compiler {.store_context().load(&mut builder.cursor(), caller_vmctx);let old_version = alias_regions- .vmstore_context_execution_version(&mut builder.cursor(), vmstore_ctx_ptr);+ .vm_store_context()+ .execution_version()+ .load(&mut builder.cursor(), vmstore_ctx_ptr);let new_version = builder.ins().iadd_imm_s(old_version, 1);- alias_regions.store_vmstore_context_execution_version(+ alias_regions.vm_store_context().execution_version().store(&mut builder.cursor(),vmstore_ctx_ptr,new_version,@@ -562,7 +571,9 @@ impl wasmtime_environ::Compiler for Compiler {.to_deferred_load(&mut context.func);let stack_limit = func_env.alias_regions- .vmstore_context_stack_limit_load(&mut context.func);+ .vm_store_context()+ .stack_limit()+ .to_deferred_load(&mut context.func);let stack_limit = VmctxLoadChain::new([store_ctx, stack_limit].into());if self.tunables.signals_based_traps {let stack_limit = stack_limit.emit_global(&mut context.func);@@ -1588,7 +1599,7 @@ impl Compiler {pointer_type,i64::try_from(wasmtime_environ::VM_LAZY_THREAD_FORCED).unwrap(),);- alias_regions.store_vmstore_context_current_thread(+ alias_regions.vm_store_context().current_thread().store(&mut builder.cursor(),vm_store_ctx,forced,@@ -1858,13 +1869,13 @@ fn save_last_wasm_entry_context<O>({// Save the current fp/sp of the entry trampoline into the `VMStoreContext`.let fp = builder.ins().get_frame_pointer(pointer_type);- alias_regions.store_vmstore_context_last_wasm_entry_fp(+ alias_regions.vm_store_context().last_wasm_entry_fp().store(&mut builder.cursor(),vm_store_context,fp,);let sp = builder.ins().get_stack_pointer(pointer_type);- alias_regions.store_vmstore_context_last_wasm_entry_sp(+ alias_regions.vm_store_context().last_wasm_entry_sp().store(&mut builder.cursor(),vm_store_context,sp,@@ -1875,11 +1886,10 @@ fn save_last_wasm_entry_context<O>(let trap_handler = builder.ins().get_exception_handler_address(pointer_type, block, 0);- alias_regions.store_vmstore_context_last_wasm_entry_trap_handler(- &mut builder.cursor(),- vm_store_context,- trap_handler,- );+ alias_regions+ .vm_store_context()+ .last_wasm_entry_trap_handler()+ .store(&mut builder.cursor(), vm_store_context, trap_handler);}fn save_last_wasm_exit_fp_and_pc<O>(@@ -1894,15 +1904,18 @@ fn save_last_wasm_exit_fp_and_pc<O>(// this so that it can know the SP (bottom of frame) for the very// last Wasm frame.let trampoline_fp = builder.ins().get_frame_pointer(pointer_type);- alias_regions.store_vmstore_context_last_wasm_exit_trampoline_fp(- &mut builder.cursor(),- limits,- trampoline_fp,- );+ alias_regions+ .vm_store_context()+ .last_wasm_exit_trampoline_fp()+ .store(&mut builder.cursor(), limits, trampoline_fp);// Finally save the Wasm return address to the limits.let wasm_pc = builder.ins().get_return_address(pointer_type);- alias_regions.store_vmstore_context_last_wasm_exit_pc(&mut builder.cursor(), limits, wasm_pc);+ alias_regions.vm_store_context().last_wasm_exit_pc().store(+ &mut builder.cursor(),+ limits,+ wasm_pc,+ );}fn key_to_name(key: FuncKey) -> ir::UserFuncName {diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex 2fd2f207c4..1a77491e78 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -1188,8 +1188,6 @@ impl<'a> TrampolineCompiler<'a> {&[],);- let trusted = ir::MemFlagsData::trusted().with_readonly();-self.builder.switch_to_block(run_destructor_block);// If this is a component-defined resource, the `may_leave` flag must be@@ -1245,16 +1243,16 @@ impl<'a> TrampolineCompiler<'a> {.ins().trapz(dtor_func_ref, TRAP_INTERNAL_ASSERT);}- let func_addr = self.alias_regions.vmfuncref_wasm_call(- &mut self.builder.cursor(),- trusted,- dtor_func_ref,- );- let callee_vmctx = self.alias_regions.vmfuncref_vmctx(- &mut self.builder.cursor(),- trusted,- dtor_func_ref,- );+ let func_addr = self+ .alias_regions+ .vm_func_ref()+ .wasm_call()+ .load(&mut self.builder.cursor(), dtor_func_ref);+ let callee_vmctx = self+ .alias_regions+ .vm_func_ref()+ .vmctx()+ .load(&mut self.builder.cursor(), dtor_func_ref);let sig = crate::wasm_call_signature(self.isa, self.signature, &self.compiler.tunables);let sig_ref = self.builder.import_signature(sig);@@ -2164,7 +2162,9 @@ wherelet data_address = self.traps.alias_regions()- .vmstore_context_store_data(&mut self.builder.cursor(), store_ctx);+ .vm_store_context()+ .store_data()+ .load(&mut self.builder.cursor(), store_ctx);// Zero-extend the address if we are on a 32-bit architecture.let data_address = match pointer_type.bits() {diff --git a/crates/cranelift/src/component_sync_call.rs b/crates/cranelift/src/component_sync_call.rsindex 6f2d7581f6..831603afcf 100644--- a/crates/cranelift/src/component_sync_call.rs+++ b/crates/cranelift/src/component_sync_call.rs@@ -69,21 +69,27 @@ where.load(&mut builder.cursor(), vmctx);// Link the previous current thread in as this frame's parent.- let parent = alias_regions.vmstore_context_current_thread(&mut builder.cursor(), vmstore);- alias_regions.store_vmdeferred_thread_parent(&mut builder.cursor(), slot_addr, parent);+ let parent = alias_regions+ .vm_store_context()+ .current_thread()+ .load(&mut builder.cursor(), vmstore);+ alias_regions+ .vm_deferred_thread()+ .parent()+ .store(&mut builder.cursor(), slot_addr, parent);// Record the deferred `enter_sync_call` arguments.- alias_regions.store_vmdeferred_thread_caller_instance(+ alias_regions.vm_deferred_thread().caller_instance().store(&mut builder.cursor(),slot_addr,args.caller_instance,);- alias_regions.store_vmdeferred_thread_callee_async(+ alias_regions.vm_deferred_thread().callee_async().store(&mut builder.cursor(),slot_addr,args.callee_async,);- alias_regions.store_vmdeferred_thread_callee_instance(+ alias_regions.vm_deferred_thread().callee_instance().store(&mut builder.cursor(),slot_addr,args.callee_instance,@@ -114,7 +120,11 @@ where}// Publish the deferred thread as the store's current thread.- alias_regions.store_vmstore_context_current_thread(&mut builder.cursor(), vmstore, slot_addr);+ alias_regions.vm_store_context().current_thread().store(+ &mut builder.cursor(),+ vmstore,+ slot_addr,+ );slot}@@ -159,7 +169,10 @@ where.vmctx().store_context().load(&mut builder.cursor(), vmctx);- let cur = alias_regions.vmstore_context_current_thread(&mut builder.cursor(), vmstore);+ let cur = alias_regions+ .vm_store_context()+ .current_thread()+ .load(&mut builder.cursor(), vmstore);let is_fast = builder.ins().icmp(IntCC::Equal, cur, slot_addr);let fast_block = builder.create_block();@@ -173,8 +186,14 @@ where// Fast path: pop the deferred thread and restore the caller's context.builder.switch_to_block(fast_block);- let parent = alias_regions.vmdeferred_thread_parent(&mut builder.cursor(), slot_addr);- alias_regions.store_vmstore_context_current_thread(&mut builder.cursor(), vmstore, parent);+ let parent = alias_regions+ .vm_deferred_thread()+ .parent()+ .load(&mut builder.cursor(), slot_addr);+ alias_regions+ .vm_store_context()+ .current_thread()+ .store(&mut builder.cursor(), vmstore, parent);for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {let saved =alias_regions.vmdeferred_thread_saved_context(&mut builder.cursor(), slot_addr, i);diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex fe762b7074..c93057f432 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -667,7 +667,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {let vmstore_ctx = self.get_vmstore_context_ptr(builder);let fuel = self.alias_regions- .vmstore_context_fuel_consumed(&mut builder.cursor(), vmstore_ctx);+ .vm_store_context()+ .fuel_consumed()+ .load(&mut builder.cursor(), vmstore_ctx);builder.def_var(self.fuel_var, fuel);}@@ -676,7 +678,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {fn fuel_save_from_var(&mut self, builder: &mut FunctionBuilder<'_>) {let vmstore_ctx = self.get_vmstore_context_ptr(builder);let fuel_consumed = builder.use_var(self.fuel_var);- self.alias_regions.store_vmstore_context_fuel_consumed(+ self.alias_regions.vm_store_context().fuel_consumed().store(&mut builder.cursor(),vmstore_ctx,fuel_consumed,@@ -859,7 +861,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {let vmstore_ctx = self.get_vmstore_context_ptr(builder);let deadline = self.alias_regions- .vmstore_context_epoch_deadline(&mut builder.cursor(), vmstore_ctx);+ .vm_store_context()+ .epoch_deadline()+ .load(&mut builder.cursor(), vmstore_ctx);builder.def_var(self.epoch_deadline_var, deadline);self.epoch_check_cached(builder, cur_epoch_value, continuation_block);@@ -1617,12 +1621,13 @@ impl FuncEnvironment<'_> {.vmctx().memories(def_index).to_deferred_load(func);- let mut base = self.alias_regions.vm_memory_definition().base();- base.can_move();- if base_readonly {- base.readonly();- }- let base = base.to_deferred_load(func);+ let base = self+ .alias_regions+ .vm_memory_definition()+ .base()+ .can_move()+ .readonly_if(base_readonly)+ .to_deferred_load(func);let len = self.alias_regions.vm_memory_definition()@@ -1639,12 +1644,14 @@ impl FuncEnvironment<'_> {// field's offset.let owned_index = self.module.owned_memory_index(def_index);let vmctx_off = self.offsets.owned_memories().at(owned_index);- let mut base = self.alias_regions.vm_memory_definition().base();- base.can_move();- if base_readonly {- base.readonly();- }- let base = base.relative_to(vmctx_off).to_deferred_load(func);+ let base = self+ .alias_regions+ .vm_memory_definition()+ .base()+ .can_move()+ .readonly_if(base_readonly)+ .relative_to(vmctx_off)+ .to_deferred_load(func);let len = self.alias_regions.vm_memory_definition()@@ -1664,12 +1671,13 @@ impl FuncEnvironment<'_> {.from().relative_to(import_off).to_deferred_load(func);- let mut base = self.alias_regions.vm_memory_definition().base();- base.can_move();- if base_readonly {- base.readonly();- }- let base = base.to_deferred_load(func);+ let base = self+ .alias_regions+ .vm_memory_definition()+ .base()+ .can_move()+ .readonly_if(base_readonly)+ .to_deferred_load(func);let len = self.alias_regions.vm_memory_definition()@@ -1717,24 +1725,26 @@ impl FuncEnvironment<'_> {// A defined table's `VMTableDefinition` is inlined into the vmctx,// reached at an absolute `vmctx` offset.let vmctx_off = self.offsets.tables().at(def_index);- let mut base = self.alias_regions.vm_table_definition().base();- if is_static {- base.readonly().can_move();- }let base = VmctxLoadChain::new(smallvec![- base.relative_to(vmctx_off).to_deferred_load(func)+ self.alias_regions+ .vm_table_definition()+ .base()+ .readonly_if(is_static)+ .can_move_if(is_static)+ .relative_to(vmctx_off)+ .to_deferred_load(func)]);let bound = if is_static {TableSize::Static {bound: table.limits.min,}} else {- let mut current_elements =- self.alias_regions.vm_table_definition().current_elements();- current_elements.cast(bound_ty);TableSize::Dynamic {bound: VmctxLoadChain::new(smallvec![- current_elements+ self.alias_regions+ .vm_table_definition()+ .current_elements()+ .cast(bound_ty).relative_to(vmctx_off).to_deferred_load(func)]),@@ -1751,11 +1761,15 @@ impl FuncEnvironment<'_> {.from().relative_to(import_off).to_deferred_load(func);- let mut base = self.alias_regions.vm_table_definition().base();- if is_static {- base.readonly().can_move();- }- let base = VmctxLoadChain::new(smallvec![from, base.to_deferred_load(func)]);+ let base = VmctxLoadChain::new(smallvec![+ from,+ self.alias_regions+ .vm_table_definition()+ .base()+ .readonly_if(is_static)+ .can_move_if(is_static)+ .to_deferred_load(func),+ ]);let bound = if is_static {TableSize::Static {bound: table.limits.min,@@ -2266,18 +2280,20 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {//// Note that the callee may be null in which case this load may// trap. If so use the `TRAP_INDIRECT_CALL_TO_NULL` trap code.- let mut mem_flags = ir::MemFlagsData::trusted().with_readonly();- if self.env.clif_memory_traps_enabled() {- mem_flags = mem_flags.with_trap_code(Some(crate::TRAP_INDIRECT_CALL_TO_NULL));+ let trap_code = if self.env.clif_memory_traps_enabled() {+ Some(crate::TRAP_INDIRECT_CALL_TO_NULL)} else {self.env.trapz(self.builder, funcref_ptr, crate::TRAP_INDIRECT_CALL_TO_NULL);- }- let callee_sig_id = self.env.alias_regions.vmfuncref_type_index(- &mut self.builder.cursor(),- mem_flags,- funcref_ptr,- );+ None+ };+ let callee_sig_id = self+ .env+ .alias_regions+ .vm_func_ref()+ .type_index()+ .trap_code(trap_code)+ .load(&mut self.builder.cursor(), funcref_ptr);// Check that they match: in the case of Wasm GC, this means doing a// full subtype check. Otherwise, we do a simple equality check.@@ -2335,24 +2351,27 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// optional trap code provided by the caller of `unchecked_call` which// will handle the case where this is either already known to be// non-null or may trap.- let mem_flags = ir::MemFlagsData::trusted().with_readonly();- let mut callee_flags = mem_flags;- if self.env.clif_memory_traps_enabled() {- callee_flags = callee_flags.with_trap_code(callee_load_trap_code);+ let callee_load_trap_code = if self.env.clif_memory_traps_enabled() {+ callee_load_trap_code} else {if let Some(trap) = callee_load_trap_code {self.env.trapz(self.builder, callee, trap);}- }- let func_addr = self.env.alias_regions.vmfuncref_wasm_call(- &mut self.builder.cursor(),- callee_flags,- callee,- );- let callee_vmctx =- self.env- .alias_regions- .vmfuncref_vmctx(&mut self.builder.cursor(), mem_flags, callee);+ None+ };+ let func_addr = self+ .env+ .alias_regions+ .vm_func_ref()+ .wasm_call()+ .trap_code(callee_load_trap_code)+ .load(&mut self.builder.cursor(), callee);+ let callee_vmctx = self+ .env+ .alias_regions+ .vm_func_ref()+ .vmctx()+ .load(&mut self.builder.cursor(), callee);(func_addr, callee_vmctx)}diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rsindex b88d21c050..8987cd9a25 100644--- a/crates/cranelift/src/func_environ/gc.rs+++ b/crates/cranelift/src/func_environ/gc.rs@@ -1306,11 +1306,11 @@ pub fn translate_ref_test(let expected_shared_ty =func_env.module_interned_to_shared_ty(&mut builder.cursor(), expected_interned_ty);- let actual_shared_ty = func_env.alias_regions.vmfuncref_type_index(- &mut builder.cursor(),- ir::MemFlagsData::trusted().with_readonly(),- val,- );+ let actual_shared_ty = func_env+ .alias_regions+ .vm_func_ref()+ .type_index()+ .load(&mut builder.cursor(), val);func_env.is_subtype(builder,diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rsindex 56386c797c..d263620b3c 100644--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs@@ -790,7 +790,7 @@ pub(crate) mod stack_switching_helpers {let stack_limit = env.alias_regions.stack_limit(&mut builder.cursor(), stack_limits_ptr);- env.alias_regions.store_vmstore_context_stack_limit(+ env.alias_regions.vm_store_context().stack_limit().store(&mut builder.cursor(),vmruntime_limits_ptr,stack_limit,@@ -799,26 +799,34 @@ pub(crate) mod stack_switching_helpers {let last_wasm_entry_fp = env.alias_regions.last_wasm_entry_fp(&mut builder.cursor(), stack_limits_ptr);- env.alias_regions.store_vmstore_context_last_wasm_entry_fp(- &mut builder.cursor(),- vmruntime_limits_ptr,- last_wasm_entry_fp,- );+ env.alias_regions+ .vm_store_context()+ .last_wasm_entry_fp()+ .store(+ &mut builder.cursor(),+ vmruntime_limits_ptr,+ last_wasm_entry_fp,+ );let last_wasm_entry_sp = env.alias_regions.last_wasm_entry_sp(&mut builder.cursor(), stack_limits_ptr);- env.alias_regions.store_vmstore_context_last_wasm_entry_sp(- &mut builder.cursor(),- vmruntime_limits_ptr,- last_wasm_entry_sp,- );+ env.alias_regions+ .vm_store_context()+ .last_wasm_entry_sp()+ .store(+ &mut builder.cursor(),+ vmruntime_limits_ptr,+ last_wasm_entry_sp,+ );let last_wasm_entry_trap_handler = env.alias_regions.last_wasm_entry_trap_handler(&mut builder.cursor(), stack_limits_ptr);env.alias_regions- .store_vmstore_context_last_wasm_entry_trap_handler(+ .vm_store_context()+ .last_wasm_entry_trap_handler()+ .store(&mut builder.cursor(),vmruntime_limits_ptr,last_wasm_entry_trap_handler,@@ -843,16 +851,19 @@ pub(crate) mod stack_switching_helpers {// continuation's inline `VMStackLimits`.let last_wasm_entry_fp = env.alias_regions- .vmstore_context_last_wasm_entry_fp(&mut builder.cursor(), vmruntime_limits_ptr);+ .vm_store_context()+ .last_wasm_entry_fp()+ .load(&mut builder.cursor(), vmruntime_limits_ptr);let last_wasm_entry_sp = env.alias_regions- .vmstore_context_last_wasm_entry_sp(&mut builder.cursor(), vmruntime_limits_ptr);+ .vm_store_context()+ .last_wasm_entry_sp()+ .load(&mut builder.cursor(), vmruntime_limits_ptr);let last_wasm_entry_trap_handler = env.alias_regions- .vmstore_context_last_wasm_entry_trap_handler(- &mut builder.cursor(),- vmruntime_limits_ptr,- );+ .vm_store_context()+ .last_wasm_entry_trap_handler()+ .load(&mut builder.cursor(), vmruntime_limits_ptr);let vmcontref_region = env.alias_regions.vmcontref_region(builder.func);let our_memflags =@@ -884,7 +895,9 @@ pub(crate) mod stack_switching_helpers {// Load from the `VMStoreContext`...let stack_limit = env.alias_regions- .vmstore_context_stack_limit(&mut builder.cursor(), vmruntime_limits_ptr);+ .vm_store_context()+ .stack_limit()+ .load(&mut builder.cursor(), vmruntime_limits_ptr);// ...and store to this continuation's inline `VMStackLimits`.let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit();builder.ins().store(diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rsindex 8011ddd8c0..e6eadf1d4a 100644--- a/crates/environ/src/vmtypes.rs+++ b/crates/environ/src/vmtypes.rs@@ -125,9 +125,19 @@ macro_rules! for_each_vm_type {/// it means that the Wasm cannot actually call this function. But it does/// mean that this field needs to be an `Option` even though it is non-null/// the vast vast vast majority of the time.+ ///+ /// Once a `VMFuncRef` is exposed to compiled code this field+ /// never changes again, so accesses of it are `readonly`. It is+ /// not `can_move`, however: the load may be the one that traps+ /// on a null funcref, and moving it would move that trap.+ #[readonly]pub wasm_call: Option<VmPtr<VMWasmCallFunction>>,/// Function signature's type id.+ ///+ /// See the note about `readonly` and not `can_move` on+ /// `wasm_call`.+ #[readonly]pub type_index: VMSharedTypeIndex,/// The VM state associated with this function.@@ -136,6 +146,10 @@ macro_rules! for_each_vm_type {/// function being referenced: for core Wasm functions, this is a `*mut/// VMContext`, for host functions it is a `*mut VMHostFuncContext`, and for/// component functions it is a `*mut VMComponentContext`.+ ///+ /// See the note about `readonly` and not `can_move` on+ /// `wasm_call`.+ #[readonly]pub vmctx: VmPtr<VMOpaqueContext>,}@@ -413,6 +427,11 @@ macro_rules! for_each_vm_type {/// A pointer to the embedder's `T` inside a `Store<T>`, for use with the/// `store-data-address` unsafe intrinsic.+ ///+ /// This pointer is fixed for the lifetime of the store, so loads+ /// of it are `readonly` and `can_move`.+ #[readonly]+ #[can_move]pub store_data: VmPtr<()>,/// The range, in addresses, of the guard page that is currently in use.