A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Tag some component-model loads and stores with alias regions (#13792)
Nick Fitzgeraldd37b5466bcf0
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex ff70cdf87d..d11da57d11 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -9,8 +9,9 @@ use wasmtime_environ::{GetPtrSize, GlobalIndex, MemoryIndex, ModuleInternedTypeIndex, OwnedMemoryIndex, PtrSize as _,RuntimeDataIndex, StaticModuleIndex, TableIndex, TagIndex, VMOffsets,component::{- LoweredIndex, ResourceIndex, RuntimeCallbackIndex, RuntimeComponentInstanceIndex,- RuntimeMemoryIndex, RuntimePostReturnIndex, VMComponentOffsets,+ ComponentBuiltinFunctionIndex, LoweredIndex, ResourceIndex, RuntimeCallbackIndex,+ RuntimeComponentInstanceIndex, RuntimeMemoryIndex, RuntimePostReturnIndex,+ VMComponentOffsets,},};@@ -36,6 +37,7 @@ enum VmType {TypeIdsArray,EpochCounter,BuiltinFunctionsArray,+ ComponentBuiltinFunctionsArray,}/// A key that uniquely identifies an alias region across an entire compilation.@@ -100,6 +102,15 @@ enum AliasRegionKey {/// The stack slot being accessed.slot: ir::StackSlot,},++ /// All unsafe intrinsics share a single alias region.+ ///+ /// By contract, unsafe intrinsics cannot access our internal `VM*` types or+ /// linear memories or anything else that has its own dedicated alias+ /// region. However, we cannot guarantee anything about the (lack of)+ /// aliasing of the embedder's data structures that the unsafe intrinsics do+ /// access, so all their accesses are lumped together into the same region.+ UnsafeIntrinsicMemory,}impl AliasRegionKey {@@ -148,6 +159,8 @@ impl AliasRegionKey {const TYPE_IDS_ARRAY_KIND: u32 = Self::new_kind(0b11001);const EPOCH_COUNTER_KIND: u32 = Self::new_kind(0b11010);const BUILTIN_FUNCTIONS_KIND: u32 = Self::new_kind(0b11011);+ const COMPONENT_BUILTIN_FUNCTIONS_KIND: u32 = Self::new_kind(0b11100);+ const UNSAFE_INTRINSIC_MEMORY_KIND: u32 = Self::new_kind(0b11101);/// Encode this key into a raw `u32` suitable for use as an/// `AliasRegionData::user_id`.@@ -176,6 +189,9 @@ impl AliasRegionKey {VmType::TypeIdsArray => Self::TYPE_IDS_ARRAY_KIND,VmType::EpochCounter => Self::EPOCH_COUNTER_KIND,VmType::BuiltinFunctionsArray => Self::BUILTIN_FUNCTIONS_KIND,+ VmType::ComponentBuiltinFunctionsArray => {+ Self::COMPONENT_BUILTIN_FUNCTIONS_KIND+ }};kind | (offset & Self::OFFSET_MASK)}@@ -215,6 +231,7 @@ impl AliasRegionKey {debug_assert_eq!(slot.as_u32() & Self::KIND_MASK, 0);Self::STACK_KIND | (slot.as_u32() & Self::OFFSET_MASK)}+ AliasRegionKey::UnsafeIntrinsicMemory => Self::UNSAFE_INTRINSIC_MEMORY_KIND,}}}@@ -237,6 +254,7 @@ impl fmt::Debug for AliasRegionKey {}AliasRegionKey::GcHeap => write!(f, "GcHeap"),AliasRegionKey::Stack { slot } => write!(f, "Stack({slot:?})"),+ AliasRegionKey::UnsafeIntrinsicMemory => write!(f, "UnsafeIntrinsicMemory"),}}}@@ -278,6 +296,23 @@ impl<Offsets> AliasRegions<Offsets> {Some(regions.insert(key.into()))}}++ /// Get the alias region for a stack slot.+ pub fn stack_slot_region(+ &mut self,+ func: &mut ir::Function,+ slot: ir::StackSlot,+ ) -> ir::AliasRegion {+ self.region(func, AliasRegionKey::Stack { slot })+ }++ /// Get the alias region for the given key.+ fn region(&mut self, func: &mut ir::Function, key: AliasRegionKey) -> ir::AliasRegion {+ *self+ .cache+ .entry(key)+ .or_insert_with(|| func.dfg.alias_regions.insert(key.into()))+ }}impl<Offsets> AliasRegions<Offsets>@@ -294,14 +329,6 @@ where}}- /// Get the alias region for the given key.- fn region(&mut self, func: &mut ir::Function, key: AliasRegionKey) -> ir::AliasRegion {- *self- .cache- .entry(key)- .or_insert_with(|| func.dfg.alias_regions.insert(key.into()))- }-/// Get the alias region for accesses into the GC heap.pub fn gc_heap_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {self.region(func, AliasRegionKey::GcHeap)@@ -1951,9 +1978,8 @@ impl<Offsets> AliasRegions<Offsets>whereOffsets: GetPtrSize,{- /// Load a host function pointer element out of a builtin-functions array- /// (either `VMContext::builtin_functions` or- /// `VMComponentContext::builtin_functions`).+ /// Load a host function pointer element out of the builtin-functions array+ /// (`VMContext::builtin_functions`)pub fn builtin_functions_array_element(&mut self,cursor: &mut FuncCursor<'_>,@@ -1985,6 +2011,78 @@ where}}+/// Component builtin-functions array methods.+impl<Offsets> AliasRegions<Offsets>+where+ Offsets: GetPtrSize,+{+ /// Load a host function pointer element out of the component+ /// builtin-functions array (`VMComponentContext::builtins`).+ pub fn component_builtin_functions_array_element(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ array: ir::Value,+ builtin: ComponentBuiltinFunctionIndex,+ ) -> ir::Value {+ let offset = builtin+ .index()+ .checked_mul(self.pointer_type.bytes())+ .unwrap();++ let region = self.region(+ cursor.func,+ AliasRegionKey::Vm {+ ty: VmType::ComponentBuiltinFunctionsArray,+ offset,+ },+ );++ cursor.ins().load(+ self.pointer_type,+ ir::MemFlagsData::trusted()+ .with_readonly()+ .with_can_move()+ .with_alias_region(Some(region)),+ array,+ i32::try_from(offset).unwrap(),+ )+ }+}++/// Unsafe intrinsic-related methods.+impl<Offsets> AliasRegions<Offsets>+where+ Offsets: GetPtrSize,+{+ /// Perform an unsafe intrinsic's load.+ pub fn unsafe_intrinsic_load(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ ty: ir::Type,+ base_flags: ir::MemFlagsData,+ addr: ir::Value,+ ) -> ir::Value {+ let region = self.region(cursor.func, AliasRegionKey::UnsafeIntrinsicMemory);+ cursor+ .ins()+ .load(ty, base_flags.with_alias_region(Some(region)), addr, 0)+ }++ /// Perform an unsafe intrinsic's store.+ pub fn unsafe_intrinsic_store(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ base_flags: ir::MemFlagsData,+ val: ir::Value,+ addr: ir::Value,+ ) {+ let region = self.region(cursor.func, AliasRegionKey::UnsafeIntrinsicMemory);+ cursor+ .ins()+ .store(base_flags.with_alias_region(Some(region)), val, addr, 0);+ }+}+/// `VMComponentContext`-related methods, used when compiling component/// trampolines.impl AliasRegions<VMComponentOffsets<u8>> {@@ -2195,6 +2293,37 @@ impl AliasRegions<VMComponentOffsets<u8>> {}}+/// `VMComponentContext`-related methods that need to be generic over `Offsets`+/// due to the call context.+impl<Offsets> AliasRegions<Offsets>+where+ Offsets: GetPtrSize,+{+ /// Load a field at `offset` within a `VMComponentContext`.+ pub fn vmcomponent_context_generic_load(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ ty: ir::Type,+ base_flags: ir::MemFlagsData,+ vmctx: ir::Value,+ offset: u32,+ ) -> ir::Value {+ let region = self.region(+ cursor.func,+ AliasRegionKey::Vm {+ ty: VmType::VMComponentContext,+ offset,+ },+ );+ cursor.ins().load(+ ty,+ base_flags.with_alias_region(Some(region)),+ vmctx,+ i32::try_from(offset).unwrap(),+ )+ }+}+/// Methods for the collectors' private heap-data structs.////// Each struct is a separate allocation reached through a `*mut _` stored in thediff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex a26d1bd120..8a7cc2be67 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -10,14 +10,14 @@ use crate::{};use cranelift_codegen::cursor::FuncCursor;use cranelift_codegen::ir::condcodes::IntCC;-use cranelift_codegen::ir::{self, InstBuilder, MemFlagsData, Value};+use cranelift_codegen::ir::{self, InstBuilder, Value};use cranelift_codegen::isa::{CallConv, TargetIsa};use cranelift_frontend::FunctionBuilder;use wasmtime_environ::GetPtrSize;use wasmtime_environ::error::{Result, bail};use wasmtime_environ::{Abi, BuiltinFunctionIndex, CompiledFunctionBody, EntityRef, FuncKey, HostCall, PanicOnOom as _,- PtrSize, TrapSentinel, Tunables, WasmFuncType, WasmValType, component::*,+ TrapSentinel, Tunables, WasmFuncType, WasmValType, component::*,fact::PREPARE_CALL_FIXED_PARAMS,};@@ -27,7 +27,6 @@ struct TrampolineCompiler<'a> {builder: FunctionBuilder<'a>,component: &'a Component,types: &'a ComponentTypesBuilder,- offsets: VMComponentOffsets<u8>,block0: ir::Block,signature: &'a WasmFuncType,builtins: BuiltinFunctions,@@ -122,7 +121,6 @@ impl<'a> TrampolineCompiler<'a> {builder,component,types,- offsets,block0,signature,builtins: BuiltinFunctions::new(compiler),@@ -1378,19 +1376,18 @@ impl<'a> TrampolineCompiler<'a> {vmctx: ir::Value,index: ComponentBuiltinFunctionIndex,) -> ir::Value {- let pointer_type = self.isa.pointer_type();// First load the pointer to the builtins structure which is static// per-process.let builtins_array = self.alias_regions.vmcomponent_builtins(&mut self.builder.cursor(), vmctx);// Next load the function pointer at `offset` and return that.- self.builder.ins().load(- pointer_type,- MemFlagsData::trusted().with_readonly(),- builtins_array,- i32::try_from(index.index() * u32::from(self.offsets.ptr.size())).unwrap(),- )+ self.alias_regions+ .component_builtin_functions_array_element(+ &mut self.builder.cursor(),+ builtins_array,+ index,+ )}/// Get a function's parameters regardless of the ABI in use.@@ -1678,14 +1675,13 @@ impl ComponentCompiler for Compiler {sig,symbol,wasmtime_environ::component::VMCOMPONENT_MAGIC,- |_alias_regions, pointer_type, cursor, vmctx| {- // TODO: `VMComponentContext` doesn't have its own alias- // region or helpers yet.- cursor.ins().load(+ |alias_regions, pointer_type, cursor, vmctx| {+ alias_regions.vmcomponent_context_generic_load(+ cursor,pointer_type,ir::MemFlagsData::trusted().with_readonly().with_can_move(),vmctx,- i32::try_from(offsets.vm_store_context()).unwrap(),+ offsets.vm_store_context(),)},)?);@@ -1760,14 +1756,13 @@ impl ComponentCompiler for Compiler {&wasm_func_ty,symbol,wasmtime_environ::component::VMCOMPONENT_MAGIC,- |_alias_regions, pointer_type, cursor, vmctx| {- // TODO: `VMComponentContext` doesn't have its own alias- // region or helpers yet.- cursor.ins().load(+ |alias_regions, pointer_type, cursor, vmctx| {+ alias_regions.vmcomponent_context_generic_load(+ cursor,pointer_type,ir::MemFlagsData::trusted().with_readonly().with_can_move(),vmctx,- i32::try_from(offsets.vm_store_context()).unwrap(),+ offsets.vm_store_context(),)},)?);@@ -1927,6 +1922,7 @@ impl TrampolineCompiler<'_> {args.push(self.len_param(4, to64));}};+ let mut retptr_slot = None;if uses_retptr {let slot = self.builder@@ -1936,14 +1932,18 @@ impl TrampolineCompiler<'_> {pointer_type.bytes(),0,));+ retptr_slot = Some(slot);args.push(self.builder.ins().stack_addr(pointer_type, slot, 0));}let call = self.call_libcall(vmctx, get_libcall, &args);let mut results = self.builder.func.dfg.inst_results(call).to_vec();- if uses_retptr {+ if let Some(slot) = retptr_slot {+ let region = self+ .alias_regions+ .stack_slot_region(self.builder.func, slot);results.push(self.builder.ins().load(pointer_type,- ir::MemFlagsData::trusted(),+ ir::MemFlagsData::trusted().with_alias_region(Some(region)),*args.last().unwrap(),0,));@@ -2158,10 +2158,12 @@ where};// Do the load!- let mut value = self- .builder- .ins()- .load(clif_ty, ir::MemFlagsData::trusted(), pointer, 0);+ let mut value = self.traps.alias_regions().unsafe_intrinsic_load(+ &mut self.builder.cursor(),+ clif_ty,+ ir::MemFlagsData::trusted(),+ pointer,+ );// Extend the value, if necessary. When implementing the// `u8-native-load` intrinsic, for example, we will load a Cranelift@@ -2212,9 +2214,12 @@ where}// Do the store!- self.builder- .ins()- .store(ir::MemFlagsData::trusted(), value, pointer, 0);+ self.traps.alias_regions().unsafe_intrinsic_store(+ &mut self.builder.cursor(),+ ir::MemFlagsData::trusted(),+ value,+ pointer,+ );Ok(())}@@ -2250,10 +2255,12 @@ where)?;// Do the load!- let mut value = self- .builder- .ins()- .load(clif_ty, ir::MemFlagsData::trusted(), addr, 0);+ let mut value = self.traps.alias_regions().unsafe_intrinsic_load(+ &mut self.builder.cursor(),+ clif_ty,+ ir::MemFlagsData::trusted(),+ addr,+ );// Zero-extend the loaded value to the Wasm result type, if necessary.let wasm_clif_ty = crate::value_type(self.isa, wasm_ty);@@ -2316,9 +2323,12 @@ where}// Do the store!- self.builder- .ins()- .store(ir::MemFlagsData::trusted(), value, addr, 0);+ self.traps.alias_regions().unsafe_intrinsic_store(+ &mut self.builder.cursor(),+ ir::MemFlagsData::trusted(),+ value,+ addr,+ );Ok(())}diff --git a/tests/disas/component-model/checked-intrinsics-inlined-no-spectre.wat b/tests/disas/component-model/checked-intrinsics-inlined-no-spectre.watindex 201909e711..a70d3413ef 100644--- a/tests/disas/component-model/checked-intrinsics-inlined-no-spectre.wat+++ b/tests/disas/component-model/checked-intrinsics-inlined-no-spectre.wat@@ -52,6 +52,7 @@;; region1 = 134217752 "VMStoreContext+0x18";; region2 = 2415919128 "VMFunctionImport+0x18";; region3 = 134217832 "VMStoreContext+0x68"+;; region4 = 3892314112 "UnsafeIntrinsicMemory";; gv0 = vmctx;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8;; gv2 = load.i64 notrap aligned region1 gv1+24@@ -72,14 +73,14 @@;; @01aa v6 = load.i64 notrap aligned readonly can_move region0 v0+8;; @01aa v7 = load.i64 notrap aligned readonly can_move region3 v6+104;; @01b0 v14 = iadd v7, v2-;; @01b0 v15 = load.i32 notrap aligned v14+;; @01b0 v15 = load.i32 notrap aligned region4 v14;; @01bf v23, v24 = uadd_overflow v2, v9 ; v9 = 4;; @01bf v25 = icmp ugt v23, v3;; @01bf v26 = bor v24, v25;; @01bf trapnz v26, heap_oob;; @01bc v19 = iconst.i32 1;; @01be v20 = iadd v15, v19 ; v19 = 1-;; @01bf store notrap aligned v20, v14+;; @01bf store notrap aligned region4 v20, v14;; @01c1 jump block1;;;; block1:diff --git a/tests/disas/component-model/checked-intrinsics-inlined.wat b/tests/disas/component-model/checked-intrinsics-inlined.watindex 1aa2975090..0bba573ae9 100644--- a/tests/disas/component-model/checked-intrinsics-inlined.wat+++ b/tests/disas/component-model/checked-intrinsics-inlined.wat@@ -53,6 +53,7 @@;; region1 = 134217752 "VMStoreContext+0x18";; region2 = 2415919128 "VMFunctionImport+0x18";; region3 = 134217832 "VMStoreContext+0x68"+;; region4 = 3892314112 "UnsafeIntrinsicMemory";; gv0 = vmctx;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8;; gv2 = load.i64 notrap aligned region1 gv1+24@@ -75,7 +76,7 @@;; @01b0 v14 = iadd v7, v2;; @01b0 v16 = select_spectre_guard v13, v15, v14 ; v15 = 0;; @01b0 trapz v16, heap_oob-;; @01b0 v17 = load.i32 notrap aligned v16+;; @01b0 v17 = load.i32 notrap aligned region4 v16;; @01bf v25, v26 = uadd_overflow v2, v9 ; v9 = 4;; @01bf v27 = icmp ugt v25, v3;; @01bf v28 = bor v26, v27@@ -83,7 +84,7 @@;; @01bf trapz v31, heap_oob;; @01bc v21 = iconst.i32 1;; @01be v22 = iadd v17, v21 ; v21 = 1-;; @01bf store notrap aligned v22, v31+;; @01bf store notrap aligned region4 v22, v31;; @01c1 jump block1;;;; block1:diff --git a/tests/disas/component-model/checked-intrinsics-trampoline.wat b/tests/disas/component-model/checked-intrinsics-trampoline.watindex 4c83fd15b6..1daa742551 100644--- a/tests/disas/component-model/checked-intrinsics-trampoline.wat+++ b/tests/disas/component-model/checked-intrinsics-trampoline.wat@@ -32,6 +32,8 @@));; function u0:0(i64 vmctx, i64, i64, i64, i64) -> i32 tail {+;; region0 = 3892314112 "UnsafeIntrinsicMemory"+;;;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64):;; v5 = iconst.i64 4;; v6, v7 = uadd_overflow v3, v5 ; v5 = 4@@ -41,11 +43,13 @@;; v10 = iadd v2, v3;; v12 = select_spectre_guard v9, v11, v10 ; v11 = 0;; trapz v12, heap_oob-;; v13 = load.i32 notrap aligned v12+;; v13 = load.i32 notrap aligned region0 v12;; return v13;; };;;; function u0:0(i64 vmctx, i64, i64, i64, i64, i32) tail {+;; region0 = 3892314112 "UnsafeIntrinsicMemory"+;;;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64, v5: i32):;; v6 = iconst.i64 4;; v7, v8 = uadd_overflow v3, v6 ; v6 = 4@@ -55,6 +59,6 @@;; v11 = iadd v2, v3;; v13 = select_spectre_guard v10, v12, v11 ; v12 = 0;; trapz v13, heap_oob-;; store notrap aligned v5, v13+;; store notrap aligned region0 v5, v13;; return;; }diff --git a/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat b/tests/disas/component-model/inlining-and-unsafe-intrinsics.watindex bc4bf8ac53..a0bbb61c4b 100644--- a/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat+++ b/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat@@ -47,6 +47,7 @@;; region1 = 134217752 "VMStoreContext+0x18";; region2 = 2415919128 "VMFunctionImport+0x18";; region3 = 134217832 "VMStoreContext+0x68"+;; region4 = 3892314112 "UnsafeIntrinsicMemory";; gv0 = vmctx;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8;; gv2 = load.i64 notrap aligned region1 gv1+24@@ -61,10 +62,10 @@;; block0(v0: i64, v1: i64):;; @0153 v4 = load.i64 notrap aligned readonly can_move region0 v0+8;; @0153 v5 = load.i64 notrap aligned readonly can_move region3 v4+104-;; @0155 v7 = load.i8 notrap aligned v5+;; @0155 v7 = load.i8 notrap aligned region4 v5;; v18 = iconst.i8 1;; v19 = iadd v7, v18 ; v18 = 1-;; @0160 store notrap aligned v19, v5+;; @0160 store notrap aligned region4 v19, v5;; @0162 jump block1;;;; block1:diff --git a/tests/disas/component-model/unsafe-intrinsics-used.wat b/tests/disas/component-model/unsafe-intrinsics-used.watindex 5d17a54bbb..ffa7030174 100644--- a/tests/disas/component-model/unsafe-intrinsics-used.wat+++ b/tests/disas/component-model/unsafe-intrinsics-used.wat@@ -45,8 +45,10 @@;; };;;; function u0:0(i64 vmctx, i64, i64) -> i32 tail {+;; region0 = 3892314112 "UnsafeIntrinsicMemory"+;;;; block0(v0: i64, v1: i64, v2: i64):-;; v3 = load.i8 notrap aligned v2+;; v3 = load.i8 notrap aligned region0 v2;; v4 = uextend.i32 v3;; return v4;; }diff --git a/tests/disas/riscv64-component-builtins.wat b/tests/disas/riscv64-component-builtins.watindex 65ad2d6fc9..99befc345c 100644--- a/tests/disas/riscv64-component-builtins.wat+++ b/tests/disas/riscv64-component-builtins.wat@@ -16,8 +16,9 @@;; region2 = 134217784 "VMStoreContext+0x38";; region3 = 1476395040 "VMComponentContext+0x20";; region4 = 1476395016 "VMComponentContext+0x8"-;; region5 = 16 "VMContext+0x10"-;; region6 = 3623878984 "BuiltinFunctionsArray+0x148"+;; region5 = 3758096400 "ComponentBuiltinFunctionsArray+0x10"+;; region6 = 16 "VMContext+0x10"+;; region7 = 3623878984 "BuiltinFunctionsArray+0x148";; sig0 = (i64 sext, i32 sext, i32 sext, i32 sext) -> i64 sext system_v;; sig1 = (i64 sext vmctx) system_v;;@@ -30,7 +31,7 @@;; v6 = load.i32 notrap aligned region3 v0+32;; trapz v6, user26;; v9 = load.i64 notrap aligned readonly region4 v0+8-;; v10 = load.i64 notrap aligned readonly v9+16+;; v10 = load.i64 notrap aligned readonly can_move region5 v9+16;; v7 = iconst.i32 0;; v11 = call_indirect sig0, v10(v0, v7, v7, v2) ; v7 = 0, v7 = 0;; v12 = iconst.i64 -1@@ -38,8 +39,8 @@;; brif v13, block2, block1;;;; block1 cold:-;; v14 = load.i64 notrap aligned readonly can_move region5 v1+16-;; v15 = load.i64 notrap aligned readonly can_move region6 v14+328+;; v14 = load.i64 notrap aligned readonly can_move region6 v1+16+;; v15 = load.i64 notrap aligned readonly can_move region7 v14+328;; call_indirect sig1, v15(v1);; trap user1;;