A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Tag trampolines' `ValRaw` marshalling with alias regions (#13796)
Nick Fitzgerald8006e967dbed
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex d11da57d11..4d727a25a6 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -38,6 +38,7 @@ enum VmType {EpochCounter,BuiltinFunctionsArray,ComponentBuiltinFunctionsArray,+ HostValRaw,}/// A key that uniquely identifies an alias region across an entire compilation.@@ -161,6 +162,7 @@ impl AliasRegionKey {const BUILTIN_FUNCTIONS_KIND: u32 = Self::new_kind(0b11011);const COMPONENT_BUILTIN_FUNCTIONS_KIND: u32 = Self::new_kind(0b11100);const UNSAFE_INTRINSIC_MEMORY_KIND: u32 = Self::new_kind(0b11101);+ const HOST_VAL_RAW_KIND: u32 = Self::new_kind(0b11110);/// Encode this key into a raw `u32` suitable for use as an/// `AliasRegionData::user_id`.@@ -192,6 +194,7 @@ impl AliasRegionKey {VmType::ComponentBuiltinFunctionsArray => {Self::COMPONENT_BUILTIN_FUNCTIONS_KIND}+ VmType::HostValRaw => Self::HOST_VAL_RAW_KIND,};kind | (offset & Self::OFFSET_MASK)}@@ -1901,6 +1904,27 @@ wherei32::from(offset),)}++ /// Load the `array_call` field of the `VMFuncRef` inlined in a+ /// `VMArrayCallHostFuncContext`.+ pub fn vmarray_call_host_func_context_array_call(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ host_func_ctx: ir::Value,+ ) -> ir::Value {+ let func_ref = self+ .offsets+ .get_ptr_size()+ .vmarray_call_host_func_context_func_ref();+ let field = self.offsets.get_ptr_size().vm_func_ref_array_call();+ let region = self.vmfuncref_region(cursor.func, field.into());+ cursor.ins().load(+ self.pointer_type,+ ir::MemFlagsData::trusted().with_alias_region(Some(region)),+ host_func_ctx,+ i32::from(func_ref) + i32::from(field),+ )+ }}/// `[VMSharedTypeIndex]`-related methods.@@ -2047,6 +2071,19 @@ wherei32::try_from(offset).unwrap(),)}++ /// Get the alias region for the `ValRaw` array used to marshal arguments+ /// and results across the array calling convention used by various+ /// trampolines.+ pub fn host_val_raw_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {+ self.region(+ func,+ AliasRegionKey::Vm {+ ty: VmType::HostValRaw,+ offset: 0,+ },+ )+ }}/// Unsafe intrinsic-related methods.diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex d857b4fced..bfebbd2255 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -41,7 +41,7 @@ use wasmtime_environ::{Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder,FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, GetPtrSize, HostCall,- InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize,+ InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder,StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables,WasmFuncType, WasmValType, prelude::*,};@@ -240,19 +240,18 @@ impl Compiler {);// Spill all wasm arguments to the stack in `ValRaw` slots.- let (args_base, args_len) =- self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]);+ let (args_base, args_len) = self.allocate_stack_array_and_spill_args(+ &mut alias_regions,+ wasm_func_ty,+ &mut builder,+ &args[2..],+ );let args_len = builder.ins().iconst(pointer_type, i64::from(args_len));// Load the actual callee out of the// `VMArrayCallHostFuncContext::host_func`.- let ptr_size = isa.pointer_bytes();- let callee = builder.ins().load(- pointer_type,- MemFlagsData::trusted(),- callee_vmctx,- ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(),- );+ let callee = alias_regions+ .vmarray_call_host_func_context_array_call(&mut builder.cursor(), callee_vmctx);// Do an indirect call to the callee.let callee_signature = builder.func.import_signature(array_call_sig);@@ -284,8 +283,13 @@ impl Compiler {self.raise_if_host_trapped(&mut builder, &mut alias_regions, caller_vmctx, succeeded);// Return results from the array as native return values.- let results =- self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len);+ let results = self.load_values_from_array(+ &mut alias_regions,+ wasm_func_ty.results(),+ &mut builder,+ args_base,+ args_len,+ );builder.ins().return_(&results);builder.finalize(self.isa().frontend_config());@@ -1125,12 +1129,16 @@ impl Compiler {////// The stack slot pointer is returned in addition to the size, in units of/// `ValRaw`, of the stack slot.- fn allocate_stack_array_and_spill_args(+ fn allocate_stack_array_and_spill_args<O>(&self,+ alias_regions: &mut AliasRegions<O>,ty: &WasmFuncType,builder: &mut FunctionBuilder,args: &[ir::Value],- ) -> (Value, u32) {+ ) -> (Value, u32)+ where+ O: GetPtrSize,+ {let isa = &*self.isa;let pointer_type = isa.pointer_type();@@ -1151,7 +1159,14 @@ impl Compiler {let values_vec_len = builder.ins().iconst(ir::types::I32, i64::from(values_vec_len));- self.store_values_to_array(builder, ty.params(), args, values_vec_ptr, values_vec_len);+ self.store_values_to_array(+ alias_regions,+ builder,+ ty.params(),+ args,+ values_vec_ptr,+ values_vec_len,+ );}(values_vec_ptr, values_vec_len)@@ -1163,25 +1178,33 @@ impl Compiler {/// using the array calling convention, or used to store results to the/// array when implementing a function that exposes the array calling/// convention.- fn store_values_to_array(+ fn store_values_to_array<O>(&self,+ alias_regions: &mut AliasRegions<O>,builder: &mut FunctionBuilder,types: &[WasmValType],values: &[Value],values_vec_ptr: Value,values_vec_capacity: Value,- ) {+ ) where+ O: GetPtrSize,+ {debug_assert_eq!(types.len(), values.len());self.debug_assert_enough_capacity_for_length(builder, types.len(), values_vec_capacity);+ if values.is_empty() {+ return;+ }// Note that loads and stores are unconditionally done in the// little-endian format rather than the host's native-endianness,// despite this load/store being unrelated to execution in Wasm itself.// For more details on this see the `ValRaw` type in// `wasmtime::runtime::vm`.+ let region = alias_regions.host_val_raw_region(builder.func);let flags = ir::MemFlagsData::new().with_notrap()- .with_endianness(ir::Endianness::Little);+ .with_endianness(ir::Endianness::Little)+ .with_alias_region(Some(region));let value_size = mem::size_of::<u128>();for (i, val) in values.iter().copied().enumerate() {@@ -1202,23 +1225,32 @@ impl Compiler {/// are building exposes the array calling convention, or it can be used to/// load results out of the array if the trampoline we are building calls a/// function that uses the array calling convention.- fn load_values_from_array(+ fn load_values_from_array<O>(&self,+ alias_regions: &mut AliasRegions<O>,types: &[WasmValType],builder: &mut FunctionBuilder,values_vec_ptr: Value,values_vec_capacity: Value,- ) -> Vec<ir::Value> {+ ) -> Vec<ir::Value>+ where+ O: GetPtrSize,+ {let isa = &*self.isa;let value_size = mem::size_of::<u128>();self.debug_assert_enough_capacity_for_length(builder, types.len(), values_vec_capacity);+ if types.is_empty() {+ return Vec::new();+ }// Note that this is little-endian like `store_values_to_array` above,// see notes there for more information.+ let region = alias_regions.host_val_raw_region(builder.func);let flags = MemFlagsData::new().with_notrap()- .with_endianness(ir::Endianness::Little);+ .with_endianness(ir::Endianness::Little)+ .with_alias_region(Some(region));let mut results = Vec::new();for (i, ty) in types.iter().enumerate() {@@ -1416,6 +1448,7 @@ impl Compiler {// First load the actual arguments out of the array.let mut args = self.load_values_from_array(+ &mut alias_regions,callee_sig.params(),&mut builder,values_vec_ptr,@@ -1505,6 +1538,7 @@ impl Compiler {// provided and return "true" for "returned successfully".builder.switch_to_block(normal_return);self.store_values_to_array(+ &mut alias_regions,&mut builder,callee_sig.results(),&normal_return_values,diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex 8a7cc2be67..8dba0992be 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -645,6 +645,7 @@ impl<'a> TrampolineCompiler<'a> {Trampoline::SyncStartCall { callback } => {let pointer_type = self.isa.pointer_type();let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(+ &mut self.alias_regions,&WasmFuncType::new([], self.signature.results().iter().copied()).panic_on_oom(),&mut self.builder,&[],@@ -877,6 +878,7 @@ impl<'a> TrampolineCompiler<'a> {let pointer_type = self.isa.pointer_type();let (ptr, len) = self.compiler.allocate_stack_array_and_spill_args(+ &mut self.alias_regions,self.signature,&mut self.builder,args,@@ -953,6 +955,7 @@ impl<'a> TrampolineCompiler<'a> {// A mixture of the above two.WasmArgs::InRegistersUpTo(n) => {let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(+ &mut self.alias_regions,&WasmFuncType::new(self.signature.params().iter().skip(n).copied(), []).panic_on_oom(),&mut self.builder,@@ -1032,6 +1035,7 @@ impl<'a> TrampolineCompiler<'a> {let len = len.or(val_raw_len).unwrap();self.raise_if_host_trapped(result.unwrap());let results = self.compiler.load_values_from_array(+ &mut self.alias_regions,self.signature.results(),&mut self.builder,ptr,diff --git a/tests/disas/array-call-trampoline.wat b/tests/disas/array-call-trampoline.watnew file mode 100644index 0000000000..673642e702--- /dev/null+++ b/tests/disas/array-call-trampoline.wat@@ -0,0 +1,47 @@+;;! target = "x86_64"+;;! test = "optimize"+;;! filter = "array_to_wasm_trampoline"++(module+ (func (export "f") (param i32 i64) (result i32 i64)+ (local.get 0)+ (local.get 1)+ )+)+;; function u268435456:0(i64 vmctx, i64, i64, i64) -> i8 system_v {+;; region0 = 4026531840 "HostValRaw+0x0"+;; region1 = 8 "VMContext+0x8"+;; region2 = 134217800 "VMStoreContext+0x48"+;; region3 = 134217792 "VMStoreContext+0x40"+;; region4 = 134217808 "VMStoreContext+0x50"+;; region5 = 134217864 "VMStoreContext+0x88"+;; sig0 = (i64 vmctx, i64, i32, i64) -> i32, i64 tail+;; fn0 = colocated u0:0 sig0+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; jump block1+;;+;; block1:+;; v4 = load.i32 notrap little region0 v2+;; v5 = load.i64 notrap little region0 v2+16+;; v7 = get_frame_pointer.i64+;; v6 = load.i64 notrap aligned readonly can_move region1 v0+8+;; store notrap aligned region2 v7, v6+72+;; v8 = get_stack_pointer.i64+;; store notrap aligned region3 v8, v6+64+;; v9 = get_exception_handler_address.i64 block1, 0+;; store notrap aligned region4 v9, v6+80+;; try_call fn0(v0, v1, v4, v5), sig0, block2(ret0, ret1), [ default: block3 ]+;;+;; block2(v10: i32, v11: i64):+;; store notrap little region0 v10, v2+;; store notrap little region0 v11, v2+16+;; v12 = iconst.i8 1+;; return v12 ; v12 = 1+;;+;; block3:+;; v13 = iconst.i64 1+;; store notrap aligned region5 v13, v6+136 ; v13 = 1+;; v14 = iconst.i8 0+;; return v14 ; v14 = 0+;; }