A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Inline `UnsafeIntrinsic` calls during translation (#13300)
Alex Crichton77dfa747d956
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex 9b7e6bed36..dc2df65574 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -44,7 +44,7 @@ use wasmtime_environ::{use wasmtime_unwinder::ExceptionTableBuilder;#[cfg(feature = "component-model")]-mod component;+pub(crate) mod component;struct IncrementalCacheContext {#[cfg(feature = "incremental-cache")]@@ -1261,7 +1261,7 @@ impl Compiler {self.call_indirect_host(builder, builtin, sig, func_addr, args)}- pub fn isa(&self) -> &dyn TargetIsa {+ pub fn isa(&self) -> &(dyn TargetIsa + 'static) {&*self.isa}diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex 326fe5f6ca..70aabb93f9 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -1550,52 +1550,6 @@ impl<'a> TrampolineCompiler<'a> {i32::from(self.offsets.ptr.vmctx_store_context()),)}-- fn translate_context_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) {- // This is the width of the type being loaded from Wasmtime's- // `VMStoreContext` slot and it depends on the intrinsic.- let ty = match intrinsic {- UnsafeIntrinsic::ContextGetI32_0- | UnsafeIntrinsic::ContextSetI32_0- | UnsafeIntrinsic::ContextGetI32_1- | UnsafeIntrinsic::ContextSetI32_1 => ir::types::I32,- _ => unreachable!(),- };-- let slot = match intrinsic {- UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextSetI32_0 => 0,- UnsafeIntrinsic::ContextGetI32_1 | UnsafeIntrinsic::ContextSetI32_1 => 1,- _ => unreachable!(),- };- let offset = self- .offsets- .ptr- .vmstore_context_component_context_slot(slot);- let params = self.abi_load_params();- let vmstore_context = self.load_vm_store_context();- match intrinsic {- UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextGetI32_1 => {- let context = self.builder.ins().load(- ty,- MemFlags::trusted(),- vmstore_context,- i32::from(offset),- );- self.abi_store_results(&[context]);- }- UnsafeIntrinsic::ContextSetI32_0 | UnsafeIntrinsic::ContextSetI32_1 => {- let new_context = params[2];- self.builder.ins().store(- MemFlags::trusted(),- new_context,- vmstore_context,- i32::from(offset),- );- self.abi_store_results(&[]);- }- _ => unreachable!(),- }- }}// Helper structure to implement `TranslateTrap`. This isn't possible to do@@ -1744,50 +1698,14 @@ impl ComponentCompiler for Compiler {&types,&wasm_func_ty,);-- match intrinsic {- UnsafeIntrinsic::U8NativeLoad- | UnsafeIntrinsic::U16NativeLoad- | UnsafeIntrinsic::U32NativeLoad- | UnsafeIntrinsic::U64NativeLoad => c.translate_load_intrinsic(intrinsic)?,- UnsafeIntrinsic::U8NativeStore- | UnsafeIntrinsic::U16NativeStore- | UnsafeIntrinsic::U32NativeStore- | UnsafeIntrinsic::U64NativeStore => c.translate_store_intrinsic(intrinsic)?,- UnsafeIntrinsic::StoreDataAddress => {- let pointer_type = self.isa.pointer_type();-- // Load the `*mut VMStoreContext` out of our vmctx.- let store_ctx = c.load_vm_store_context();-- // Load the `*mut T` out of the `VMStoreContext`.- let data_address = c.builder.ins().load(- pointer_type,- ir::MemFlags::trusted()- .with_readonly()- .with_alias_region(Some(ir::AliasRegion::Vmctx))- .with_can_move(),- store_ctx,- i32::from(c.offsets.ptr.vmstore_context_store_data()),- );-- // Zero-extend the address if we are on a 32-bit architecture.- let data_address = match pointer_type.bits() {- 32 => c.builder.ins().uextend(ir::types::I64, data_address),- 64 => data_address,- p => bail!("unsupported architecture: no support for {p}-bit pointers"),- };-- c.abi_store_results(&[data_address]);- }-- UnsafeIntrinsic::ContextGetI32_0- | UnsafeIntrinsic::ContextGetI32_1- | UnsafeIntrinsic::ContextSetI32_0- | UnsafeIntrinsic::ContextSetI32_1 => {- c.translate_context_intrinsic(intrinsic);- }+ let params = c.abi_load_params();+ let result = UnsafeIntrinsicCompiler {+ isa: c.isa,+ ptr: &c.offsets.ptr,+ cursor: c.builder.cursor(),}+ .translate(intrinsic, ¶ms)?;+ c.abi_store_results(result.as_slice());c.builder.finalize();compiler.cx.abi = Some(abi);@@ -2015,8 +1933,94 @@ impl TrampolineCompiler<'_> {i32::from(self.offsets.ptr.vmmemory_definition_base()),)}+}++/// A helper structure to translate an `UnsafeIntrinsic`.+///+/// This is used as part of `ComponentCompiler::compile_intrinsic` as well as+/// during normal translation of wasm to known direct calls to `UnsafeIntrinsic`+/// targets.+///+/// This type itself is more of a context type of sorts where it maintains+/// little-to-no state and instead just weaves together all that's necessary for+/// translating an intrinsic.+pub struct UnsafeIntrinsicCompiler<'a> {+ pub isa: &'a (dyn TargetIsa + 'static),+ pub cursor: FuncCursor<'a>,+ pub ptr: &'a (dyn PtrSize + 'static),+}- fn translate_load_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()> {+impl<'a> UnsafeIntrinsicCompiler<'a> {+ /// Translates the `intrinsic` provided which is provided `params` as+ /// arguments.+ ///+ /// Note that `params` is expected to be with `Abi::Wasm` meaning that the+ /// first two parameters are callee/caller vmctx arguments.+ ///+ /// Returns an `Option<ir::Value>` representing an optional return value of+ /// this intrinsic. Not all intrinsics have a return value, and those that+ /// do have at most one return value.+ pub fn translate(+ &mut self,+ intrinsic: UnsafeIntrinsic,+ params: &[ir::Value],+ ) -> Result<Option<ir::Value>> {+ Ok(match intrinsic {+ UnsafeIntrinsic::U8NativeLoad+ | UnsafeIntrinsic::U16NativeLoad+ | UnsafeIntrinsic::U32NativeLoad+ | UnsafeIntrinsic::U64NativeLoad => {+ Some(self.translate_load_intrinsic(intrinsic, params)?)+ }++ UnsafeIntrinsic::U8NativeStore+ | UnsafeIntrinsic::U16NativeStore+ | UnsafeIntrinsic::U32NativeStore+ | UnsafeIntrinsic::U64NativeStore => {+ self.translate_store_intrinsic(intrinsic, params)?;+ None+ }+ UnsafeIntrinsic::StoreDataAddress => {+ let pointer_type = self.isa.pointer_type();++ // Load the `*mut VMStoreContext` out of our vmctx.+ let store_ctx = self.load_vm_store_context(params);++ // Load the `*mut T` out of the `VMStoreContext`.+ let data_address = self.cursor.ins().load(+ pointer_type,+ ir::MemFlags::trusted()+ .with_readonly()+ .with_alias_region(Some(ir::AliasRegion::Vmctx))+ .with_can_move(),+ store_ctx,+ i32::from(self.ptr.vmstore_context_store_data()),+ );++ // Zero-extend the address if we are on a 32-bit architecture.+ let data_address = match pointer_type.bits() {+ 32 => self.cursor.ins().uextend(ir::types::I64, data_address),+ 64 => data_address,+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),+ };++ Some(data_address)+ }++ UnsafeIntrinsic::ContextGetI32_0+ | UnsafeIntrinsic::ContextGetI32_1+ | UnsafeIntrinsic::ContextSetI32_0+ | UnsafeIntrinsic::ContextSetI32_1 => {+ self.translate_context_intrinsic(intrinsic, params)+ }+ })+ }++ fn translate_load_intrinsic(+ &mut self,+ intrinsic: UnsafeIntrinsic,+ params: &[ir::Value],+ ) -> Result<ir::Value> {// Emit code for a native-load intrinsic.debug_assert_eq!(intrinsic.core_params(), &[WasmValType::I64]);debug_assert_eq!(intrinsic.core_results().len(), 1);@@ -2024,21 +2028,21 @@ impl TrampolineCompiler<'_> {let wasm_ty = intrinsic.core_results()[0];let clif_ty = unsafe_intrinsic_clif_results(intrinsic)[0];- let [_callee_vmctx, _caller_vmctx, pointer] = *self.abi_load_params() else {+ let [_callee_vmctx, _caller_vmctx, pointer] = *params else {unreachable!()};// Truncate the pointer, if necessary.- debug_assert_eq!(self.builder.func.dfg.value_type(pointer), ir::types::I64);+ debug_assert_eq!(self.cursor.func.dfg.value_type(pointer), ir::types::I64);let pointer = match self.isa.pointer_bits() {- 32 => self.builder.ins().ireduce(ir::types::I32, pointer),+ 32 => self.cursor.ins().ireduce(ir::types::I32, pointer),64 => pointer,p => bail!("unsupported architecture: no support for {p}-bit pointers"),};// Do the load!let mut value = self- .builder+ .cursor.ins().load(clif_ty, ir::MemFlags::trusted(), pointer, 0);@@ -2051,28 +2055,31 @@ impl TrampolineCompiler<'_> {assert!(clif_ty.bytes() < wasm_clif_ty.bytes());// NB: all of our unsafe intrinsics for native loads are// unsigned, so we always zero-extend.- value = self.builder.ins().uextend(wasm_clif_ty, value);+ value = self.cursor.ins().uextend(wasm_clif_ty, value);}- self.abi_store_results(&[value]);- Ok(())+ Ok(value)}- fn translate_store_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()> {+ fn translate_store_intrinsic(+ &mut self,+ intrinsic: UnsafeIntrinsic,+ params: &[ir::Value],+ ) -> Result<()> {debug_assert!(intrinsic.core_results().is_empty());debug_assert!(matches!(intrinsic.core_params(), [WasmValType::I64, _]));let wasm_ty = intrinsic.core_params()[1];let clif_ty = unsafe_intrinsic_clif_params(intrinsic)[1];- let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *self.abi_load_params() else {+ let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *params else {unreachable!()};// Truncate the pointer, if necessary.- debug_assert_eq!(self.builder.func.dfg.value_type(pointer), ir::types::I64);+ debug_assert_eq!(self.cursor.func.dfg.value_type(pointer), ir::types::I64);let pointer = match self.isa.pointer_bits() {- 32 => self.builder.ins().ireduce(ir::types::I32, pointer),+ 32 => self.cursor.ins().ireduce(ir::types::I32, pointer),64 => pointer,p => bail!("unsupported architecture: no support for {p}-bit pointers"),};@@ -2084,17 +2091,84 @@ impl TrampolineCompiler<'_> {let wasm_ty = crate::value_type(self.isa, wasm_ty);if clif_ty != wasm_ty {assert!(clif_ty.bytes() < wasm_ty.bytes());- value = self.builder.ins().ireduce(clif_ty, value);+ value = self.cursor.ins().ireduce(clif_ty, value);}// Do the store!- self.builder+ self.cursor.ins().store(ir::MemFlags::trusted(), value, pointer, 0);- self.abi_store_results(&[]);Ok(())}++ fn translate_context_intrinsic(+ &mut self,+ intrinsic: UnsafeIntrinsic,+ params: &[ir::Value],+ ) -> Option<ir::Value> {+ // This is the width of the type being loaded from Wasmtime's+ // `VMStoreContext` slot and it depends on the intrinsic.+ let ty = match intrinsic {+ UnsafeIntrinsic::ContextGetI32_0+ | UnsafeIntrinsic::ContextSetI32_0+ | UnsafeIntrinsic::ContextGetI32_1+ | UnsafeIntrinsic::ContextSetI32_1 => ir::types::I32,+ _ => unreachable!(),+ };++ let slot = match intrinsic {+ UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextSetI32_0 => 0,+ UnsafeIntrinsic::ContextGetI32_1 | UnsafeIntrinsic::ContextSetI32_1 => 1,+ _ => unreachable!(),+ };+ let offset = self.ptr.vmstore_context_component_context_slot(slot);+ let vmstore_context = self.load_vm_store_context(params);+ match intrinsic {+ UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextGetI32_1 => {+ let context = self.cursor.ins().load(+ ty,+ MemFlags::trusted(),+ vmstore_context,+ i32::from(offset),+ );+ Some(context)+ }+ UnsafeIntrinsic::ContextSetI32_0 | UnsafeIntrinsic::ContextSetI32_1 => {+ let new_context = params[2];+ self.cursor.ins().store(+ MemFlags::trusted(),+ new_context,+ vmstore_context,+ i32::from(offset),+ );+ None+ }+ _ => unreachable!(),+ }+ }++ /// Loads `*mut VMStoreContext` and returns it.+ ///+ /// Note that the `*mut VMStoreContext` value is the same for all+ /// `VMContext`-like structures in a store. In this case it's loaded from+ /// the *caller* vmctx rather than the *callee* vmctx. The caller is using a+ /// `VMContext` for core wasm which is passed in a register, where the+ /// callee is a `VMComponentContext` loaded from the `VMContext`. By using+ /// the caller vmctx we're able to possibly eliminate the dead load of the+ /// `VMComponentContext` if it's otherwise unused.+ fn load_vm_store_context(&mut self, params: &[ir::Value]) -> ir::Value {+ let caller_vmctx = params[1];+ self.cursor.ins().load(+ self.isa.pointer_type(),+ ir::MemFlags::trusted()+ .with_readonly()+ .with_alias_region(Some(ir::AliasRegion::Vmctx))+ .with_can_move(),+ caller_vmctx,+ i32::from(self.ptr.vmctx_store_context()),+ )+ }}/// Module with macro-generated contents that will return the signature anddiff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 83b8538415..d9acc09612 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -127,7 +127,7 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures);/// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`.pub struct FuncEnvironment<'module_environment> {compiler: &'module_environment Compiler,- isa: &'module_environment (dyn TargetIsa + 'module_environment),+ isa: &'module_environment (dyn TargetIsa + 'static),key: FuncKey,pub(crate) module: &'module_environment Module,types: &'module_environment ModuleTypesBuilder,@@ -1848,11 +1848,22 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// direct call to that function (presumably it will eventually be// inlined).#[cfg(feature = "component-model")]- Some(FuncKey::UnsafeIntrinsic(..)) => {+ Some(FuncKey::UnsafeIntrinsic(abi, intrinsic)) => {let callee = self.env.get_or_create_imported_func_ref(self.builder.func, callee_index);- Ok(self.direct_call_inst(callee, &real_call_args))+ if self.can_directly_inline_unsafe_intrinsic(*abi) {+ let result = super::compiler::component::UnsafeIntrinsicCompiler {+ cursor: self.builder.cursor(),+ isa: self.env.isa,+ ptr: &self.env.offsets.ptr,+ }+ .translate(*intrinsic, &real_call_args)+ .unwrap();+ Ok(result.into_iter().collect())+ } else {+ Ok(self.direct_call_inst(callee, &real_call_args))+ }}// The import is always satisfied with the given defined Wasm@@ -1881,6 +1892,21 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {}}+ /// Determines if a direct inline-during-translation is possible for a call+ /// made to an `UnsafeIntrinsic`.+ ///+ /// This only happens in "normal" circumstances where it's considered safe+ /// to bypass the otherwise off-by-default Cranelift inliner that Wasmtime+ /// has. This is a performance optimization to avoid needing to turn on all+ /// of inlining to get the performance benefit of inlining unsafe+ /// intrinsics. The fallback of issuing a `call` to the intrinsic is always+ /// suitable to do and is used in situations where the call instruction may+ /// have extra context.+ #[cfg(feature = "component-model")]+ fn can_directly_inline_unsafe_intrinsic(&self, abi: wasmtime_environ::Abi) -> bool {+ abi == wasmtime_environ::Abi::Wasm && !self.tail && !self.env.tunables.debug_guest+ }+/// Do a Wasm-level indirect call through the given funcref table.pub fn indirect_call(mut self,diff --git a/tests/disas/component-model/context-intrinsics.wat b/tests/disas/component-model/context-intrinsics.watindex 8a87be4d2c..9a226ad353 100644--- a/tests/disas/component-model/context-intrinsics.wat+++ b/tests/disas/component-model/context-intrinsics.wat@@ -1,6 +1,6 @@;;! target = "x86_64";;! test = 'compile'-;;! flags = '-Wcomponent-model-async -Wcomponent-model-threading -Cinlining=intrinsics'+;;! flags = '-Wcomponent-model-async -Wcomponent-model-threading'(component(core func $context.get0 (canon context.get i32 0))diff --git a/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat b/tests/disas/component-model/inlining-and-unsafe-intrinsics.watindex 452d14b47c..c7cdedc67f 100644--- a/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat+++ b/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat@@ -56,36 +56,12 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64):-;; @0153 jump block2-;;-;; block2:-;; jump block3-;;-;; block3:-;; @0155 jump block4-;;-;; block4:-;; v17 = load.i64 notrap aligned readonly can_move vmctx v0+8-;; v18 = load.i64 notrap aligned readonly can_move vmctx v17+104-;; v20 = load.i8 notrap aligned v18-;; jump block5-;;-;; block5:-;; @0159 jump block6-;;-;; block6:-;; jump block7-;;-;; block7:-;; @0160 jump block8-;;-;; block8:-;; v28 = iconst.i8 1-;; v29 = iadd.i8 v20, v28 ; v28 = 1-;; store notrap aligned v29, v18-;; jump block9-;;-;; block9:+;; @0153 v5 = load.i64 notrap aligned readonly can_move vmctx v0+8+;; @0153 v6 = load.i64 notrap aligned readonly can_move vmctx v5+104+;; @0155 v9 = load.i8 notrap aligned v6+;; v22 = iconst.i8 1+;; v23 = iadd v9, v22 ; v22 = 1+;; @0160 store notrap aligned v23, v6;; @0162 jump block1;;;; block1: