Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Inline `UnsafeIntrinsic` calls during translation (#13300)

Alex Crichton77dfa747d956
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 9b7e6bed36..dc2df65574 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -44,7 +44,7 @@ use wasmtime_environ::{
use wasmtime_unwinder::ExceptionTableBuilder;
#[cfg(feature = "component-model")]
-mod component;
+pub(crate) mod component;
struct IncrementalCacheContext {
#[cfg(feature = "incremental-cache")]
@@ -1261,7 +1261,7 @@ impl Compiler {
self.call_indirect_host(builder, builtin, sig, func_addr, args)
}
- pub fn isa(&self) -> &dyn TargetIsa {
+ pub fn isa(&self) -> &(dyn TargetIsa + 'static) {
&*self.isa
}
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index 326fe5f6ca..70aabb93f9 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -1550,52 +1550,6 @@ impl<'a> TrampolineCompiler<'a> {
i32::from(self.offsets.ptr.vmctx_store_context()),
)
}
-
- fn translate_context_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) {
- // This is the width of the type being loaded from Wasmtime's
- // `VMStoreContext` slot and it depends on the intrinsic.
- let ty = match intrinsic {
- UnsafeIntrinsic::ContextGetI32_0
- | UnsafeIntrinsic::ContextSetI32_0
- | UnsafeIntrinsic::ContextGetI32_1
- | UnsafeIntrinsic::ContextSetI32_1 => ir::types::I32,
- _ => unreachable!(),
- };
-
- let slot = match intrinsic {
- UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextSetI32_0 => 0,
- UnsafeIntrinsic::ContextGetI32_1 | UnsafeIntrinsic::ContextSetI32_1 => 1,
- _ => unreachable!(),
- };
- let offset = self
- .offsets
- .ptr
- .vmstore_context_component_context_slot(slot);
- let params = self.abi_load_params();
- let vmstore_context = self.load_vm_store_context();
- match intrinsic {
- UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextGetI32_1 => {
- let context = self.builder.ins().load(
- ty,
- MemFlags::trusted(),
- vmstore_context,
- i32::from(offset),
- );
- self.abi_store_results(&[context]);
- }
- UnsafeIntrinsic::ContextSetI32_0 | UnsafeIntrinsic::ContextSetI32_1 => {
- let new_context = params[2];
- self.builder.ins().store(
- MemFlags::trusted(),
- new_context,
- vmstore_context,
- i32::from(offset),
- );
- self.abi_store_results(&[]);
- }
- _ => unreachable!(),
- }
- }
}
// Helper structure to implement `TranslateTrap`. This isn't possible to do
@@ -1744,50 +1698,14 @@ impl ComponentCompiler for Compiler {
&types,
&wasm_func_ty,
);
-
- match intrinsic {
- UnsafeIntrinsic::U8NativeLoad
- | UnsafeIntrinsic::U16NativeLoad
- | UnsafeIntrinsic::U32NativeLoad
- | UnsafeIntrinsic::U64NativeLoad => c.translate_load_intrinsic(intrinsic)?,
- UnsafeIntrinsic::U8NativeStore
- | UnsafeIntrinsic::U16NativeStore
- | UnsafeIntrinsic::U32NativeStore
- | UnsafeIntrinsic::U64NativeStore => c.translate_store_intrinsic(intrinsic)?,
- UnsafeIntrinsic::StoreDataAddress => {
- let pointer_type = self.isa.pointer_type();
-
- // Load the `*mut VMStoreContext` out of our vmctx.
- let store_ctx = c.load_vm_store_context();
-
- // Load the `*mut T` out of the `VMStoreContext`.
- let data_address = c.builder.ins().load(
- pointer_type,
- ir::MemFlags::trusted()
- .with_readonly()
- .with_alias_region(Some(ir::AliasRegion::Vmctx))
- .with_can_move(),
- store_ctx,
- i32::from(c.offsets.ptr.vmstore_context_store_data()),
- );
-
- // Zero-extend the address if we are on a 32-bit architecture.
- let data_address = match pointer_type.bits() {
- 32 => c.builder.ins().uextend(ir::types::I64, data_address),
- 64 => data_address,
- p => bail!("unsupported architecture: no support for {p}-bit pointers"),
- };
-
- c.abi_store_results(&[data_address]);
- }
-
- UnsafeIntrinsic::ContextGetI32_0
- | UnsafeIntrinsic::ContextGetI32_1
- | UnsafeIntrinsic::ContextSetI32_0
- | UnsafeIntrinsic::ContextSetI32_1 => {
- c.translate_context_intrinsic(intrinsic);
- }
+ let params = c.abi_load_params();
+ let result = UnsafeIntrinsicCompiler {
+ isa: c.isa,
+ ptr: &c.offsets.ptr,
+ cursor: c.builder.cursor(),
}
+ .translate(intrinsic, &params)?;
+ c.abi_store_results(result.as_slice());
c.builder.finalize();
compiler.cx.abi = Some(abi);
@@ -2015,8 +1933,94 @@ impl TrampolineCompiler<'_> {
i32::from(self.offsets.ptr.vmmemory_definition_base()),
)
}
+}
+
+/// A helper structure to translate an `UnsafeIntrinsic`.
+///
+/// This is used as part of `ComponentCompiler::compile_intrinsic` as well as
+/// during normal translation of wasm to known direct calls to `UnsafeIntrinsic`
+/// targets.
+///
+/// This type itself is more of a context type of sorts where it maintains
+/// little-to-no state and instead just weaves together all that's necessary for
+/// translating an intrinsic.
+pub struct UnsafeIntrinsicCompiler<'a> {
+ pub isa: &'a (dyn TargetIsa + 'static),
+ pub cursor: FuncCursor<'a>,
+ pub ptr: &'a (dyn PtrSize + 'static),
+}
- fn translate_load_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()> {
+impl<'a> UnsafeIntrinsicCompiler<'a> {
+ /// Translates the `intrinsic` provided which is provided `params` as
+ /// arguments.
+ ///
+ /// Note that `params` is expected to be with `Abi::Wasm` meaning that the
+ /// first two parameters are callee/caller vmctx arguments.
+ ///
+ /// Returns an `Option<ir::Value>` representing an optional return value of
+ /// this intrinsic. Not all intrinsics have a return value, and those that
+ /// do have at most one return value.
+ pub fn translate(
+ &mut self,
+ intrinsic: UnsafeIntrinsic,
+ params: &[ir::Value],
+ ) -> Result<Option<ir::Value>> {
+ Ok(match intrinsic {
+ UnsafeIntrinsic::U8NativeLoad
+ | UnsafeIntrinsic::U16NativeLoad
+ | UnsafeIntrinsic::U32NativeLoad
+ | UnsafeIntrinsic::U64NativeLoad => {
+ Some(self.translate_load_intrinsic(intrinsic, params)?)
+ }
+
+ UnsafeIntrinsic::U8NativeStore
+ | UnsafeIntrinsic::U16NativeStore
+ | UnsafeIntrinsic::U32NativeStore
+ | UnsafeIntrinsic::U64NativeStore => {
+ self.translate_store_intrinsic(intrinsic, params)?;
+ None
+ }
+ UnsafeIntrinsic::StoreDataAddress => {
+ let pointer_type = self.isa.pointer_type();
+
+ // Load the `*mut VMStoreContext` out of our vmctx.
+ let store_ctx = self.load_vm_store_context(params);
+
+ // Load the `*mut T` out of the `VMStoreContext`.
+ let data_address = self.cursor.ins().load(
+ pointer_type,
+ ir::MemFlags::trusted()
+ .with_readonly()
+ .with_alias_region(Some(ir::AliasRegion::Vmctx))
+ .with_can_move(),
+ store_ctx,
+ i32::from(self.ptr.vmstore_context_store_data()),
+ );
+
+ // Zero-extend the address if we are on a 32-bit architecture.
+ let data_address = match pointer_type.bits() {
+ 32 => self.cursor.ins().uextend(ir::types::I64, data_address),
+ 64 => data_address,
+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),
+ };
+
+ Some(data_address)
+ }
+
+ UnsafeIntrinsic::ContextGetI32_0
+ | UnsafeIntrinsic::ContextGetI32_1
+ | UnsafeIntrinsic::ContextSetI32_0
+ | UnsafeIntrinsic::ContextSetI32_1 => {
+ self.translate_context_intrinsic(intrinsic, params)
+ }
+ })
+ }
+
+ fn translate_load_intrinsic(
+ &mut self,
+ intrinsic: UnsafeIntrinsic,
+ params: &[ir::Value],
+ ) -> Result<ir::Value> {
// Emit code for a native-load intrinsic.
debug_assert_eq!(intrinsic.core_params(), &[WasmValType::I64]);
debug_assert_eq!(intrinsic.core_results().len(), 1);
@@ -2024,21 +2028,21 @@ impl TrampolineCompiler<'_> {
let wasm_ty = intrinsic.core_results()[0];
let clif_ty = unsafe_intrinsic_clif_results(intrinsic)[0];
- let [_callee_vmctx, _caller_vmctx, pointer] = *self.abi_load_params() else {
+ let [_callee_vmctx, _caller_vmctx, pointer] = *params else {
unreachable!()
};
// Truncate the pointer, if necessary.
- debug_assert_eq!(self.builder.func.dfg.value_type(pointer), ir::types::I64);
+ debug_assert_eq!(self.cursor.func.dfg.value_type(pointer), ir::types::I64);
let pointer = match self.isa.pointer_bits() {
- 32 => self.builder.ins().ireduce(ir::types::I32, pointer),
+ 32 => self.cursor.ins().ireduce(ir::types::I32, pointer),
64 => pointer,
p => bail!("unsupported architecture: no support for {p}-bit pointers"),
};
// Do the load!
let mut value = self
- .builder
+ .cursor
.ins()
.load(clif_ty, ir::MemFlags::trusted(), pointer, 0);
@@ -2051,28 +2055,31 @@ impl TrampolineCompiler<'_> {
assert!(clif_ty.bytes() < wasm_clif_ty.bytes());
// NB: all of our unsafe intrinsics for native loads are
// unsigned, so we always zero-extend.
- value = self.builder.ins().uextend(wasm_clif_ty, value);
+ value = self.cursor.ins().uextend(wasm_clif_ty, value);
}
- self.abi_store_results(&[value]);
- Ok(())
+ Ok(value)
}
- fn translate_store_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()> {
+ fn translate_store_intrinsic(
+ &mut self,
+ intrinsic: UnsafeIntrinsic,
+ params: &[ir::Value],
+ ) -> Result<()> {
debug_assert!(intrinsic.core_results().is_empty());
debug_assert!(matches!(intrinsic.core_params(), [WasmValType::I64, _]));
let wasm_ty = intrinsic.core_params()[1];
let clif_ty = unsafe_intrinsic_clif_params(intrinsic)[1];
- let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *self.abi_load_params() else {
+ let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *params else {
unreachable!()
};
// Truncate the pointer, if necessary.
- debug_assert_eq!(self.builder.func.dfg.value_type(pointer), ir::types::I64);
+ debug_assert_eq!(self.cursor.func.dfg.value_type(pointer), ir::types::I64);
let pointer = match self.isa.pointer_bits() {
- 32 => self.builder.ins().ireduce(ir::types::I32, pointer),
+ 32 => self.cursor.ins().ireduce(ir::types::I32, pointer),
64 => pointer,
p => bail!("unsupported architecture: no support for {p}-bit pointers"),
};
@@ -2084,17 +2091,84 @@ impl TrampolineCompiler<'_> {
let wasm_ty = crate::value_type(self.isa, wasm_ty);
if clif_ty != wasm_ty {
assert!(clif_ty.bytes() < wasm_ty.bytes());
- value = self.builder.ins().ireduce(clif_ty, value);
+ value = self.cursor.ins().ireduce(clif_ty, value);
}
// Do the store!
- self.builder
+ self.cursor
.ins()
.store(ir::MemFlags::trusted(), value, pointer, 0);
- self.abi_store_results(&[]);
Ok(())
}
+
+ fn translate_context_intrinsic(
+ &mut self,
+ intrinsic: UnsafeIntrinsic,
+ params: &[ir::Value],
+ ) -> Option<ir::Value> {
+ // This is the width of the type being loaded from Wasmtime's
+ // `VMStoreContext` slot and it depends on the intrinsic.
+ let ty = match intrinsic {
+ UnsafeIntrinsic::ContextGetI32_0
+ | UnsafeIntrinsic::ContextSetI32_0
+ | UnsafeIntrinsic::ContextGetI32_1
+ | UnsafeIntrinsic::ContextSetI32_1 => ir::types::I32,
+ _ => unreachable!(),
+ };
+
+ let slot = match intrinsic {
+ UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextSetI32_0 => 0,
+ UnsafeIntrinsic::ContextGetI32_1 | UnsafeIntrinsic::ContextSetI32_1 => 1,
+ _ => unreachable!(),
+ };
+ let offset = self.ptr.vmstore_context_component_context_slot(slot);
+ let vmstore_context = self.load_vm_store_context(params);
+ match intrinsic {
+ UnsafeIntrinsic::ContextGetI32_0 | UnsafeIntrinsic::ContextGetI32_1 => {
+ let context = self.cursor.ins().load(
+ ty,
+ MemFlags::trusted(),
+ vmstore_context,
+ i32::from(offset),
+ );
+ Some(context)
+ }
+ UnsafeIntrinsic::ContextSetI32_0 | UnsafeIntrinsic::ContextSetI32_1 => {
+ let new_context = params[2];
+ self.cursor.ins().store(
+ MemFlags::trusted(),
+ new_context,
+ vmstore_context,
+ i32::from(offset),
+ );
+ None
+ }
+ _ => unreachable!(),
+ }
+ }
+
+ /// Loads `*mut VMStoreContext` and returns it.
+ ///
+ /// Note that the `*mut VMStoreContext` value is the same for all
+ /// `VMContext`-like structures in a store. In this case it's loaded from
+ /// the *caller* vmctx rather than the *callee* vmctx. The caller is using a
+ /// `VMContext` for core wasm which is passed in a register, where the
+ /// callee is a `VMComponentContext` loaded from the `VMContext`. By using
+ /// the caller vmctx we're able to possibly eliminate the dead load of the
+ /// `VMComponentContext` if it's otherwise unused.
+ fn load_vm_store_context(&mut self, params: &[ir::Value]) -> ir::Value {
+ let caller_vmctx = params[1];
+ self.cursor.ins().load(
+ self.isa.pointer_type(),
+ ir::MemFlags::trusted()
+ .with_readonly()
+ .with_alias_region(Some(ir::AliasRegion::Vmctx))
+ .with_can_move(),
+ caller_vmctx,
+ i32::from(self.ptr.vmctx_store_context()),
+ )
+ }
}
/// Module with macro-generated contents that will return the signature and
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 83b8538415..d9acc09612 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -127,7 +127,7 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures);
/// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`.
pub struct FuncEnvironment<'module_environment> {
compiler: &'module_environment Compiler,
- isa: &'module_environment (dyn TargetIsa + 'module_environment),
+ isa: &'module_environment (dyn TargetIsa + 'static),
key: FuncKey,
pub(crate) module: &'module_environment Module,
types: &'module_environment ModuleTypesBuilder,
@@ -1848,11 +1848,22 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// direct call to that function (presumably it will eventually be
// inlined).
#[cfg(feature = "component-model")]
- Some(FuncKey::UnsafeIntrinsic(..)) => {
+ Some(FuncKey::UnsafeIntrinsic(abi, intrinsic)) => {
let callee = self
.env
.get_or_create_imported_func_ref(self.builder.func, callee_index);
- Ok(self.direct_call_inst(callee, &real_call_args))
+ if self.can_directly_inline_unsafe_intrinsic(*abi) {
+ let result = super::compiler::component::UnsafeIntrinsicCompiler {
+ cursor: self.builder.cursor(),
+ isa: self.env.isa,
+ ptr: &self.env.offsets.ptr,
+ }
+ .translate(*intrinsic, &real_call_args)
+ .unwrap();
+ Ok(result.into_iter().collect())
+ } else {
+ Ok(self.direct_call_inst(callee, &real_call_args))
+ }
}
// The import is always satisfied with the given defined Wasm
@@ -1881,6 +1892,21 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
}
}
+ /// Determines if a direct inline-during-translation is possible for a call
+ /// made to an `UnsafeIntrinsic`.
+ ///
+ /// This only happens in "normal" circumstances where it's considered safe
+ /// to bypass the otherwise off-by-default Cranelift inliner that Wasmtime
+ /// has. This is a performance optimization to avoid needing to turn on all
+ /// of inlining to get the performance benefit of inlining unsafe
+ /// intrinsics. The fallback of issuing a `call` to the intrinsic is always
+ /// suitable to do and is used in situations where the call instruction may
+ /// have extra context.
+ #[cfg(feature = "component-model")]
+ fn can_directly_inline_unsafe_intrinsic(&self, abi: wasmtime_environ::Abi) -> bool {
+ abi == wasmtime_environ::Abi::Wasm && !self.tail && !self.env.tunables.debug_guest
+ }
+
/// Do a Wasm-level indirect call through the given funcref table.
pub fn indirect_call(
mut self,
diff --git a/tests/disas/component-model/context-intrinsics.wat b/tests/disas/component-model/context-intrinsics.wat
index 8a87be4d2c..9a226ad353 100644
--- a/tests/disas/component-model/context-intrinsics.wat
+++ b/tests/disas/component-model/context-intrinsics.wat
@@ -1,6 +1,6 @@
;;! target = "x86_64"
;;! test = 'compile'
-;;! flags = '-Wcomponent-model-async -Wcomponent-model-threading -Cinlining=intrinsics'
+;;! flags = '-Wcomponent-model-async -Wcomponent-model-threading'
(component
(core func $context.get0 (canon context.get i32 0))
diff --git a/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat b/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat
index 452d14b47c..c7cdedc67f 100644
--- a/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat
+++ b/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat
@@ -56,36 +56,12 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64):
-;; @0153 jump block2
-;;
-;; block2:
-;; jump block3
-;;
-;; block3:
-;; @0155 jump block4
-;;
-;; block4:
-;; v17 = load.i64 notrap aligned readonly can_move vmctx v0+8
-;; v18 = load.i64 notrap aligned readonly can_move vmctx v17+104
-;; v20 = load.i8 notrap aligned v18
-;; jump block5
-;;
-;; block5:
-;; @0159 jump block6
-;;
-;; block6:
-;; jump block7
-;;
-;; block7:
-;; @0160 jump block8
-;;
-;; block8:
-;; v28 = iconst.i8 1
-;; v29 = iadd.i8 v20, v28 ; v28 = 1
-;; store notrap aligned v29, v18
-;; jump block9
-;;
-;; block9:
+;; @0153 v5 = load.i64 notrap aligned readonly can_move vmctx v0+8
+;; @0153 v6 = load.i64 notrap aligned readonly can_move vmctx v5+104
+;; @0155 v9 = load.i8 notrap aligned v6
+;; v22 = iconst.i8 1
+;; v23 = iadd v9, v22 ; v22 = 1
+;; @0160 store notrap aligned v23, v6
;; @0162 jump block1
;;
;; block1: