Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Implement unsafe intrinsics for compile-time builtins (#11825)

Nick Fitzgeraldad56ff980ae5
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index abfc714a9e..0c001f6b18 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -897,7 +897,12 @@ impl InliningCompiler for Compiler {
.user_named_funcs()
.values()
.map(|name| FuncKey::from_raw_parts(name.namespace, name.index))
- .filter(|key| matches!(key, FuncKey::DefinedWasmFunction(_, _))),
+ .filter(|key| match key {
+ FuncKey::DefinedWasmFunction(..) => true,
+ #[cfg(feature = "component-model")]
+ FuncKey::UnsafeIntrinsic(..) => true,
+ _ => false,
+ }),
);
Ok(())
}
@@ -953,25 +958,28 @@ impl InliningCompiler for Compiler {
let callee = &caller.params.user_named_funcs()[callee];
let callee = FuncKey::from_raw_parts(callee.namespace, callee.index);
match callee {
- FuncKey::DefinedWasmFunction(_, _) => match (self.0)(callee) {
- None => InlineCommand::KeepCall,
- Some(func_body) => {
- debug_assert!(!func_body.code.is::<CompiledFunction>());
- debug_assert!(func_body.code.is::<Option<CompilerContext>>());
- let cx = func_body
- .code
- .downcast_ref::<Option<CompilerContext>>()
- .unwrap();
- InlineCommand::Inline {
- callee: Cow::Borrowed(&cx.as_ref().unwrap().codegen_context.func),
- // We've already visited the callee for inlining
- // due to our bottom-up approach, no need to
- // visit it again.
- visit_callee: false,
- }
+ FuncKey::DefinedWasmFunction(..) => {}
+ #[cfg(feature = "component-model")]
+ FuncKey::UnsafeIntrinsic(..) => {}
+ _ => return InlineCommand::KeepCall,
+ }
+ match (self.0)(callee) {
+ None => InlineCommand::KeepCall,
+ Some(func_body) => {
+ debug_assert!(!func_body.code.is::<CompiledFunction>());
+ debug_assert!(func_body.code.is::<Option<CompilerContext>>());
+ let cx = func_body
+ .code
+ .downcast_ref::<Option<CompilerContext>>()
+ .unwrap();
+ InlineCommand::Inline {
+ callee: Cow::Borrowed(&cx.as_ref().unwrap().codegen_context.func),
+ // We've already visited the callee for inlining
+ // due to our bottom-up approach, no need to
+ // visit it again.
+ visit_callee: false,
}
- },
- _ => InlineCommand::KeepCall,
+ }
}
}
}
@@ -1138,7 +1146,7 @@ impl Compiler {
// Note that loads and stores are unconditionally done in the
// little-endian format rather than the host's native-endianness,
- // despite this load/store being unrelated to execution in wasm itself.
+ // despite this load/store being unrelated to execution in Wasm itself.
// For more details on this see the `ValRaw` type in
// `wasmtime::runtime::vm`.
let flags = ir::MemFlags::new()
@@ -1355,6 +1363,7 @@ impl FunctionCompiler<'_> {
let block0 = builder.create_block();
builder.append_block_params_for_function_params(block0);
builder.switch_to_block(block0);
+ builder.ensure_inserted_block();
builder.seal_block(block0);
(builder, block0)
}
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index ad4981cc81..533c150b11 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -1,7 +1,10 @@
//! Compilation support for the component model.
-use crate::{TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT, compiler::Compiler};
-use anyhow::Result;
+use crate::array_call_signature;
+use crate::{
+ TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT, compiler::Compiler, wasm_call_signature,
+};
+use anyhow::{Result, bail};
use cranelift_codegen::ir::condcodes::IntCC;
use cranelift_codegen::ir::{self, InstBuilder, MemFlags, Value};
use cranelift_codegen::isa::{CallConv, TargetIsa};
@@ -95,6 +98,85 @@ enum WasmArgs {
InRegistersUpTo(usize),
}
+/// Get a function's parameters regardless of the ABI in use.
+///
+/// This emits code to load the parameters from the array-call's ABI's values
+/// vector, if necessary.
+fn abi_params(
+ compiler: &Compiler,
+ builder: &mut FunctionBuilder<'_>,
+ abi: Abi,
+ types: &[WasmValType],
+) -> Vec<ir::Value> {
+ let entry_block = builder
+ .func
+ .layout
+ .entry_block()
+ .expect("should have an entry block");
+ let entry_block_params = builder.func.dfg.block_params(entry_block);
+ match abi {
+ // Wasm and native ABIs pass parameters as normal function parameters.
+ Abi::Wasm => entry_block_params.to_vec(),
+
+ // The array ABI passes a pointer/length as the 3rd/4th arguments and
+ // those are used to load the actual Wasm parameters.
+ Abi::Array => {
+ let &[callee_vmctx, caller_vmctx, values_vec_ptr, values_vec_len] = entry_block_params
+ else {
+ unreachable!()
+ };
+ let mut results =
+ compiler.load_values_from_array(types, builder, values_vec_ptr, values_vec_len);
+ results.splice(0..0, [callee_vmctx, caller_vmctx]);
+ results
+ }
+ }
+}
+
+/// Emit code to return the given result values, regardless of the ABI in use.
+fn abi_return(
+ compiler: &Compiler,
+ builder: &mut FunctionBuilder<'_>,
+ abi: Abi,
+ result_types: &[WasmValType],
+ results: &[ir::Value],
+) {
+ match abi {
+ // The Wasm ABI returns values as usual.
+ Abi::Wasm => {
+ builder.ins().return_(results);
+ }
+
+ // The array ABI stores all results in the pointer/length passed
+ // as arguments to this function, which contractually are required
+ // to have enough space for the results.
+ Abi::Array => {
+ let entry_block = builder
+ .func
+ .layout
+ .entry_block()
+ .expect("should have an entry block");
+ let &[_callee_vmctx, _caller_vmctx, values_vec_ptr, values_vec_len] =
+ builder.func.dfg.block_params(entry_block)
+ else {
+ unreachable!()
+ };
+
+ compiler.store_values_to_array(
+ builder,
+ result_types,
+ results,
+ values_vec_ptr,
+ values_vec_len,
+ );
+
+ // A "true" return value indicates a successful, non-trapping call.
+ let true_value = builder.ins().iconst(ir::types::I8, 1);
+ builder.ins().return_(&[true_value]);
+ }
+ }
+}
+
impl<'a> TrampolineCompiler<'a> {
fn new(
compiler: &'a Compiler,
@@ -970,7 +1052,7 @@ impl<'a> TrampolineCompiler<'a> {
// Acquire the result of this function (if any) and interpret it
// according to `host_result`.
//
- // NOte that all match arms here end with `abi_store_results` which
+ // Note that all match arms here end with `abi_store_results` which
// accounts for the ABI of this function when storing results.
let result = self.builder.func.dfg.inst_results(call).get(0).copied();
let result_ty = result.map(|v| self.builder.func.dfg.value_type(v));
@@ -1292,52 +1374,22 @@ impl<'a> TrampolineCompiler<'a> {
}
fn abi_load_params(&mut self) -> Vec<ir::Value> {
- let mut block0_params = self.builder.func.dfg.block_params(self.block0).to_vec();
- match self.abi {
- // Wasm and native ABIs pass parameters as normal function
- // parameters.
- Abi::Wasm => block0_params,
-
- // The array ABI passes a pointer/length as the 3rd/4th arguments
- // and those are used to load the actual wasm parameters.
- Abi::Array => {
- let results = self.compiler.load_values_from_array(
- self.types[self.signature].unwrap_func().params(),
- &mut self.builder,
- block0_params[2],
- block0_params[3],
- );
- block0_params.truncate(2);
- block0_params.extend(results);
- block0_params
- }
- }
+ abi_params(
+ self.compiler,
+ &mut self.builder,
+ self.abi,
+ self.types[self.signature].unwrap_func().params(),
+ )
}
fn abi_store_results(&mut self, results: &[ir::Value]) {
- match self.abi {
- // Wasm/native ABIs return values as usual.
- Abi::Wasm => {
- self.builder.ins().return_(results);
- }
-
- // The array ABI stores all results in the pointer/length passed
- // as arguments to this function, which contractually are required
- // to have enough space for the results.
- Abi::Array => {
- let block0_params = self.builder.func.dfg.block_params(self.block0);
- let (ptr, len) = (block0_params[2], block0_params[3]);
- self.compiler.store_values_to_array(
- &mut self.builder,
- self.types[self.signature].unwrap_func().returns(),
- results,
- ptr,
- len,
- );
- let true_value = self.builder.ins().iconst(ir::types::I8, 1);
- self.builder.ins().return_(&[true_value]);
- }
- }
+ abi_return(
+ self.compiler,
+ &mut self.builder,
+ self.abi,
+ self.types[self.signature].unwrap_func().returns(),
+ results,
+ );
}
fn raise_if_host_trapped(&mut self, succeeded: ir::Value) {
@@ -1442,8 +1494,215 @@ impl ComponentCompiler for Compiler {
needs_gc_heap: false,
})
}
+
+ fn compile_intrinsic(
+ &self,
+ tunables: &Tunables,
+ component: &ComponentTranslation,
+ intrinsic: UnsafeIntrinsic,
+ abi: Abi,
+ _symbol: &str,
+ ) -> Result<CompiledFunctionBody> {
+ let offsets = VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);
+ let mut compiler = self.function_compiler();
+ let cx = &mut compiler.cx;
+ let ctx = &mut cx.codegen_context;
+ let mut builder = FunctionBuilder::new(&mut ctx.func, cx.func_translator.context());
+
+ // Initialize the function, its signature, and entry block.
+ let init = |builder: &mut FunctionBuilder| -> Vec<ir::Value> {
+ let wasm_func_ty = WasmFuncType::new(
+ intrinsic.core_params().into(),
+ intrinsic.core_results().into(),
+ );
+ builder.func.signature = match abi {
+ Abi::Wasm => wasm_call_signature(&*self.isa, &wasm_func_ty, tunables),
+ Abi::Array => array_call_signature(&*self.isa),
+ };
+
+ assert!(builder.func.layout.entry_block().is_none());
+ let entry_block = builder.create_block();
+ builder.append_block_params_for_function_params(entry_block);
+ builder.seal_block(entry_block);
+ builder.switch_to_block(entry_block);
+ builder.ensure_inserted_block();
+
+ abi_params(self, builder, abi, wasm_func_ty.params())
+ };
+
+ // Emit code for a native-load intrinsic.
+ let load = |builder: &mut FunctionBuilder| -> Result<()> {
+ debug_assert_eq!(intrinsic.core_params(), &[WasmValType::I64]);
+ debug_assert_eq!(intrinsic.core_results().len(), 1);
+
+ let wasm_ty = intrinsic.core_results()[0];
+ let clif_ty = unsafe_intrinsic_clif_results(intrinsic)[0];
+
+ let [_callee_vmctx, _caller_vmctx, pointer] = *init(builder) else {
+ unreachable!()
+ };
+
+ // Truncate the pointer, if necessary.
+ debug_assert_eq!(builder.func.dfg.value_type(pointer), ir::types::I64);
+ let pointer = match self.isa.pointer_bits() {
+ 32 => builder.ins().ireduce(ir::types::I32, pointer),
+ 64 => pointer,
+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),
+ };
+
+ // Do the load!
+ let mut value = builder
+ .ins()
+ .load(clif_ty, ir::MemFlags::trusted(), pointer, 0);
+
+ // Extend the value, if necessary. When implementing the
+ // `u8-native-load` intrinsic, for example, we will load a Cranelift
+ // value of type `i8` but we need to extend it to an `i32` because
+ // Wasm doesn't have an `i8` core value type.
+ let wasm_clif_ty = crate::value_type(&*self.isa, wasm_ty);
+ if clif_ty != wasm_clif_ty {
+ assert!(clif_ty.bytes() < wasm_clif_ty.bytes());
+ // NB: all of our unsafe intrinsics for native loads are
+ // unsigned, so we always zero-extend.
+ value = builder.ins().uextend(wasm_clif_ty, value);
+ }
+
+ abi_return(self, builder, abi, &[wasm_ty], &[value]);
+ Ok(())
+ };
+
+ // Emit code for a native-store intrinsic.
+ let store = |builder: &mut FunctionBuilder| -> Result<()> {
+ debug_assert!(intrinsic.core_results().is_empty());
+ debug_assert!(matches!(intrinsic.core_params(), [WasmValType::I64, _]));
+
+ let wasm_ty = intrinsic.core_params()[1];
+ let clif_ty = unsafe_intrinsic_clif_params(intrinsic)[1];
+
+ let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *init(builder) else {
+ unreachable!()
+ };
+
+ // Truncate the pointer, if necessary.
+ debug_assert_eq!(builder.func.dfg.value_type(pointer), ir::types::I64);
+ let pointer = match self.isa.pointer_bits() {
+ 32 => builder.ins().ireduce(ir::types::I32, pointer),
+ 64 => pointer,
+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),
+ };
+
+ // Truncate the value, if necessary. For example, with
+ // `u8-native-store` we will be given an `i32` from Wasm (because
+ // core Wasm does not have an 8-bit integer value type) and we need
+ // to reduce that into an `i8`.
+ let wasm_ty = crate::value_type(&*self.isa, wasm_ty);
+ if clif_ty != wasm_ty {
+ assert!(clif_ty.bytes() < wasm_ty.bytes());
+ value = builder.ins().ireduce(clif_ty, value);
+ }
+
+ // Do the store!
+ builder
+ .ins()
+ .store(ir::MemFlags::trusted(), value, pointer, 0);
+
+ abi_return(self, builder, abi, &[], &[]);
+ Ok(())
+ };
+
+ match intrinsic {
+ UnsafeIntrinsic::U8NativeLoad => load(&mut builder)?,
+ UnsafeIntrinsic::U16NativeLoad => load(&mut builder)?,
+ UnsafeIntrinsic::U32NativeLoad => load(&mut builder)?,
+ UnsafeIntrinsic::U64NativeLoad => load(&mut builder)?,
+ UnsafeIntrinsic::U8NativeStore => store(&mut builder)?,
+ UnsafeIntrinsic::U16NativeStore => store(&mut builder)?,
+ UnsafeIntrinsic::U32NativeStore => store(&mut builder)?,
+ UnsafeIntrinsic::U64NativeStore => store(&mut builder)?,
+ UnsafeIntrinsic::StoreDataAddress => {
+ let [callee_vmctx, _caller_vmctx] = *init(&mut builder) else {
+ unreachable!()
+ };
+ let pointer_type = self.isa.pointer_type();
+
+ // Load the `*mut VMStoreContext` out of our vmctx.
+ let store_ctx = builder.ins().load(
+ pointer_type,
+ ir::MemFlags::trusted()
+ .with_readonly()
+ .with_alias_region(Some(ir::AliasRegion::Vmctx))
+ .with_can_move(),
+ callee_vmctx,
+ i32::try_from(offsets.vm_store_context()).unwrap(),
+ );
+
+ // Load the `*mut T` out of the `VMStoreContext`.
+ let data_address = builder.ins().load(
+ pointer_type,
+ ir::MemFlags::trusted()
+ .with_readonly()
+ .with_alias_region(Some(ir::AliasRegion::Vmctx))
+ .with_can_move(),
+ store_ctx,
+ i32::from(offsets.ptr.vmstore_context_store_data()),
+ );
+
+ // Zero-extend the address if we are on a 32-bit architecture.
+ let data_address = match pointer_type.bits() {
+ 32 => builder.ins().uextend(ir::types::I64, data_address),
+ 64 => data_address,
+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),
+ };
+
+ abi_return(
+ self,
+ &mut builder,
+ abi,
+ intrinsic.core_results(),
+ &[data_address],
+ );
+ }
+ }
+
+ builder.finalize();
+ Ok(CompiledFunctionBody {
+ code: super::box_dyn_any_compiler_context(Some(compiler.cx)),
+ needs_gc_heap: false,
+ })
+ }
}
+macro_rules! unsafe_intrinsic_clif_params_results {
+ (
+ $(
+ $symbol:expr => $variant:ident : $ctor:ident ( $( $param:ident : $param_ty:ident ),* ) $( -> $result_ty:ident )? ;
+ )*
+ ) => {
+ fn unsafe_intrinsic_clif_params(intrinsic: UnsafeIntrinsic) -> &'static [ir::types::Type] {
+ match intrinsic {
+ $(
+ UnsafeIntrinsic::$variant => &[ $( unsafe_intrinsic_clif_params_results!(@clif_type $param_ty) ),* ],
+ )*
+ }
+ }
+
+ fn unsafe_intrinsic_clif_results(intrinsic: UnsafeIntrinsic) -> &'static [ir::types::Type] {
+ match intrinsic {
+ $(
+ UnsafeIntrinsic::$variant => &[ $( unsafe_intrinsic_clif_params_results!(@clif_type $result_ty) )? ],
+ )*
+ }
+ }
+ };
+
+ (@clif_type u8) => { ir::types::I8 };
+ (@clif_type u16) => { ir::types::I16 };
+ (@clif_type u32) => { ir::types::I32 };
+ (@clif_type u64) => { ir::types::I64 };
+}
+
+wasmtime_environ::for_each_unsafe_intrinsic!(unsafe_intrinsic_clif_params_results);
+
impl TrampolineCompiler<'_> {
fn translate_transcode(
&mut self,
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index c26f84a723..52208e7d90 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -1468,9 +1468,21 @@ impl FuncEnvironment<'_> {
.unwrap_module_type_index();
let signature = self.get_or_create_interned_sig_ref(func, ty);
- let (module, def_func_index) =
- self.translation.known_imported_functions[func_index].unwrap();
- let key = FuncKey::DefinedWasmFunction(module, def_func_index);
+ let key = match self.translation.known_imported_functions[func_index] {
+ Some(key @ FuncKey::DefinedWasmFunction(..)) => key,
+
+ #[cfg(feature = "component-model")]
+ Some(key @ FuncKey::UnsafeIntrinsic(..)) => key,
+
+ Some(key) => {
+ panic!("unexpected kind of known-import function: {key:?}")
+ }
+
+ None => panic!(
+ "cannot make an `ir::FuncRef` for a function import that is not statically known"
+ ),
+ };
+
let (namespace, index) = key.into_raw_parts();
let name = ir::ExternalName::User(
func.declare_imported_user_function(ir::UserExternalName { namespace, index }),
@@ -1903,9 +1915,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
mut self,
callee_index: FuncIndex,
sig_ref: ir::SigRef,
- call_args: &[ir::Value],
+ wasm_call_args: &[ir::Value],
) -> WasmResult<CallRets> {
- let mut real_call_args = Vec::with_capacity(call_args.len() + 2);
+ let mut real_call_args = Vec::with_capacity(wasm_call_args.len() + 2);
let caller_vmctx = self
.builder
.func
@@ -1922,7 +1934,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
real_call_args.push(caller_vmctx);
// Then append the regular call arguments.
- real_call_args.extend_from_slice(call_args);
+ real_call_args.extend_from_slice(wasm_call_args);
// Finally, make the direct call!
let callee = self
@@ -1957,25 +1969,47 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
real_call_args.push(callee_vmctx);
real_call_args.push(caller_vmctx);
- // Then append the regular call arguments.
- real_call_args.extend_from_slice(call_args);
+ // Then append the Wasm call arguments.
+ real_call_args.extend_from_slice(wasm_call_args);
// If we statically know the imported function (e.g. this is a
// component-to-component call where we statically know both components)
- // then we can actually still make a direct call (although we do have to
- // pass the callee's vmctx that we just loaded, not our own). Otherwise,
- // we really do an indirect call.
- if self.env.translation.known_imported_functions[callee_index].is_some() {
- let callee = self
- .env
- .get_or_create_imported_func_ref(self.builder.func, callee_index);
- Ok(self.direct_call_inst(callee, &real_call_args))
- } else {
- let func_addr = self
- .builder
- .ins()
- .load(pointer_type, mem_flags, base, body_offset);
- Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))
+ // then we can avoid doing an indirect call.
+ match self.env.translation.known_imported_functions[callee_index].as_ref() {
+ // The import is always a compile-time builtin intrinsic. Make a
+ // direct call to that function (presumably it will eventually be
+ // inlined).
+ #[cfg(feature = "component-model")]
+ Some(FuncKey::UnsafeIntrinsic(..)) => {
+ let callee = self
+ .env
+ .get_or_create_imported_func_ref(self.builder.func, callee_index);
+ Ok(self.direct_call_inst(callee, &real_call_args))
+ }
+
+ // The import is always satisfied with the given defined Wasm
+ // function, so do a direct call to that function! (Although we take
+ // care to still pass its `funcref`'s `vmctx` as the callee `vmctx`
+ // in `real_call_args` and not the caller's.)
+ Some(FuncKey::DefinedWasmFunction(..)) => {
+ let callee = self
+ .env
+ .get_or_create_imported_func_ref(self.builder.func, callee_index);
+ Ok(self.direct_call_inst(callee, &real_call_args))
+ }
+
+ Some(key) => panic!("unexpected kind of known-import function: {key:?}"),
+
+ // Unknown import function or this module is instantiated many times
+ // and with different functions. Either way, we have to do the
+ // indirect call.
+ None => {
+ let func_addr = self
+ .builder
+ .ins()
+ .load(pointer_type, mem_flags, base, body_offset);
+ Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))
+ }
}
}
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index 63d3a4ef62..a2b6e029a4 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -260,9 +260,18 @@ macro_rules! declare_builtin_index {
pub struct $index_name:ident : $for_each_builtin:ident ;
) => {
$(#[$attr])*
- #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+ #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct $index_name(u32);
+ impl core::fmt::Debug for $index_name {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ f.debug_struct(stringify!($index_name))
+ .field("index", &self.0)
+ .field("ctor", &self.ctor_name())
+ .finish()
+ }
+ }
+
impl $index_name {
/// Create a new builtin from its raw index
pub const fn from_u32(i: u32) -> Self {
@@ -275,6 +284,8 @@ macro_rules! declare_builtin_index {
self.0
}
+ $for_each_builtin!(define_ctor_name);
+
$for_each_builtin!(declare_builtin_index_constructors);
}
@@ -287,6 +298,28 @@ macro_rules! declare_builtin_index {
};
}
+/// Helper macro used by the above macro.
+macro_rules! define_ctor_name {
+ (
+ $(
+ $( #[$attr:meta] )*
+ $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
+ )*
+ ) => {
+ /// Returns the name of the constructor that creates this index.
+ pub fn ctor_name(&self) -> &'static str {
+ let mut _i = self.0;
+ $(
+ if _i == 0 {
+ return stringify!($name);
+ }
+ _i -= 1;
+ )*
+ unreachable!()
+ }
+ }
+}
+
/// Helper macro used by the above macro.
macro_rules! declare_builtin_index_constructors {
(
diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs
index 65d438ee88..6894ba713c 100644
--- a/crates/environ/src/compile/module_environ.rs
+++ b/crates/environ/src/compile/module_environ.rs
@@ -2,14 +2,15 @@ use crate::module::{
FuncRefIndex, Initializer, MemoryInitialization, MemoryInitializer, Module, TableSegment,
TableSegmentElements,
};
+use crate::prelude::*;
use crate::{
ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, ElemIndex, EngineOrModuleTypeIndex,
- EntityIndex, EntityType, FuncIndex, GlobalIndex, IndexType, InitMemory, MemoryIndex,
+ EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType, InitMemory, MemoryIndex,
ModuleInternedTypeIndex, ModuleTypesBuilder, PrimaryMap, SizeOverflow, StaticMemoryInitializer,
- TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, WasmError,
- WasmHeapTopType, WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter,
+ StaticModuleIndex, TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert,
+ TypeIndex, WasmError, WasmHeapTopType, WasmHeapType, WasmResult, WasmValType,
+ WasmparserTypeConverter,
};
-use crate::{StaticModuleIndex, prelude::*};
use anyhow::{Result, bail};
use cranelift_entity::SecondaryMap;
use cranelift_entity::packed_option::ReservedValue;
@@ -55,12 +56,15 @@ pub struct ModuleTranslation<'data> {
/// References to the function bodies.
pub function_body_inputs: PrimaryMap<DefinedFuncIndex, FunctionBodyData<'data>>,
- /// For each imported function, the single statically-known defined function
- /// that satisfies that import, if any. This is used to turn what would
- /// otherwise be indirect calls through the imports table into direct calls,
- /// when possible.
- pub known_imported_functions:
- SecondaryMap<FuncIndex, Option<(StaticModuleIndex, DefinedFuncIndex)>>,
+ /// For each imported function, the single statically-known function that
+ /// always satisfies that import, if any.
+ ///
+ /// This is used to turn what would otherwise be indirect calls through the
+ /// imports table into direct calls, when possible.
+ ///
+ /// When filled in, this only ever contains
+ /// `FuncKey::DefinedWasmFunction(..)`s and `FuncKey::Intrinsic(..)`s.
+ pub known_imported_functions: SecondaryMap<FuncIndex, Option<FuncKey>>,
/// A list of type signatures which are considered exported from this
/// module, or those that can possibly be called. This list is sorted, and
diff --git a/crates/environ/src/component.rs b/crates/environ/src/component.rs
index 3c7087728a..414c1a3ec5 100644
--- a/crates/environ/src/component.rs
+++ b/crates/environ/src/component.rs
@@ -53,11 +53,13 @@ pub const START_FLAG_ASYNC_CALLEE: i32 = 1 << 0;
mod artifacts;
mod info;
+mod intrinsic;
mod names;
mod types;
mod vmcomponent_offsets;
pub use self::artifacts::*;
pub use self::info::*;
+pub use self::intrinsic::*;
pub use self::names::*;
pub use self::types::*;
pub use self::vmcomponent_offsets::*;
diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs
index ccfdab4566..08c089c66b 100644
--- a/crates/environ/src/component/compiler.rs
+++ b/crates/environ/src/component/compiler.rs
@@ -1,4 +1,4 @@
-use crate::component::{ComponentTranslation, ComponentTypesBuilder};
+use crate::component::{ComponentTranslation, ComponentTypesBuilder, UnsafeIntrinsic};
use crate::{Abi, CompiledFunctionBody, FuncKey, Tunables};
use anyhow::Result;
@@ -19,4 +19,14 @@ pub trait ComponentCompiler: Send + Sync {
tunables: &Tunables,
symbol: &str,
) -> Result<CompiledFunctionBody>;
+
+ /// Compile the given Wasmtime intrinsic.
+ fn compile_intrinsic(
+ &self,
+ tunables: &Tunables,
+ component: &ComponentTranslation,
+ intrinsic: UnsafeIntrinsic,
+ abi: Abi,
+ symbol: &str,
+ ) -> Result<CompiledFunctionBody>;
}
diff --git a/crates/environ/src/component/dfg.rs b/crates/environ/src/component/dfg.rs
index a85a80cea0..68e234e338 100644
--- a/crates/environ/src/component/dfg.rs
+++ b/crates/environ/src/component/dfg.rs
@@ -31,6 +31,7 @@ use crate::component::*;
use crate::prelude::*;
use crate::{EntityIndex, EntityRef, ModuleInternedTypeIndex, PrimaryMap, WasmValType};
use anyhow::Result;
+use cranelift_entity::packed_option::PackedOption;
use indexmap::IndexMap;
use info::LinearMemoryOptions;
use std::collections::HashMap;
@@ -54,6 +55,10 @@ pub struct ComponentDfg {
/// compiled by Cranelift.
pub trampolines: Intern<TrampolineIndex, (ModuleInternedTypeIndex, Trampoline)>,
+ /// A map from `UnsafeIntrinsic::index()` to that intrinsic's
+ /// module-interned type.
+ pub unsafe_intrinsics: [PackedOption<ModuleInternedTypeIndex>; UnsafeIntrinsic::len() as usize],
+
/// Know reallocation functions which are used by `lowerings` (e.g. will be
/// used by the host)
pub reallocs: Intern<ReallocId, CoreDef>,
@@ -258,6 +263,8 @@ pub enum CoreDef {
Export(CoreExport<EntityIndex>),
InstanceFlags(RuntimeComponentInstanceIndex),
Trampoline(TrampolineIndex),
+ UnsafeIntrinsic(ModuleInternedTypeIndex, UnsafeIntrinsic),
+
/// This is a special variant not present in `info::CoreDef` which
/// represents that this definition refers to a fused adapter function. This
/// adapter is fully processed after the initial translation and
@@ -595,6 +602,7 @@ impl ComponentDfg {
runtime_callbacks: Default::default(),
runtime_instances: Default::default(),
num_lowerings: 0,
+ unsafe_intrinsics: Default::default(),
trampolines: Default::default(),
trampoline_defs: Default::default(),
trampoline_map: Default::default(),
@@ -629,6 +637,7 @@ impl ComponentDfg {
exports,
export_items,
initializers: linearize.initializers,
+ unsafe_intrinsics: linearize.unsafe_intrinsics,
trampolines: linearize.trampolines,
num_lowerings: linearize.num_lowerings,
options: linearize.options,
@@ -666,6 +675,7 @@ impl ComponentDfg {
struct LinearizeDfg<'a> {
dfg: &'a ComponentDfg,
initializers: Vec<GlobalInitializer>,
+ unsafe_intrinsics: [PackedOption<ModuleInternedTypeIndex>; UnsafeIntrinsic::len() as usize],
trampolines: PrimaryMap<TrampolineIndex, ModuleInternedTypeIndex>,
trampoline_defs: PrimaryMap<TrampolineIndex, info::Trampoline>,
options: PrimaryMap<OptionsIndex, info::CanonicalOptions>,
@@ -856,6 +866,13 @@ impl LinearizeDfg<'_> {
CoreDef::InstanceFlags(i) => info::CoreDef::InstanceFlags(*i),
CoreDef::Adapter(id) => info::CoreDef::Export(self.adapter(*id)),
CoreDef::Trampoline(index) => info::CoreDef::Trampoline(self.trampoline(*index)),
+ CoreDef::UnsafeIntrinsic(ty, i) => {
+ let index = usize::try_from(i.index()).unwrap();
+ if self.unsafe_intrinsics[index].is_none() {
+ self.unsafe_intrinsics[index] = Some(*ty).into();
+ }
+ info::CoreDef::UnsafeIntrinsic(*i)
+ }
}
}
diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rs
index d825c9a51e..251d6ef181 100644
--- a/crates/environ/src/component/info.rs
+++ b/crates/environ/src/component/info.rs
@@ -49,6 +49,7 @@
use crate::component::*;
use crate::prelude::*;
use crate::{EntityIndex, ModuleInternedTypeIndex, PrimaryMap, WasmValType};
+use cranelift_entity::packed_option::PackedOption;
use serde_derive::{Deserialize, Serialize};
/// Metadata as a result of compiling a component.
@@ -162,6 +163,10 @@ pub struct Component {
/// WebAssembly type signature of all trampolines.
pub trampolines: PrimaryMap<TrampolineIndex, ModuleInternedTypeIndex>,
+ /// A map from a `UnsafeIntrinsic::index()` to that intrinsic's
+ /// module-interned type.
+ pub unsafe_intrinsics: [PackedOption<ModuleInternedTypeIndex>; UnsafeIntrinsic::len() as usize],
+
/// The number of lowered host functions (maximum `LoweredIndex`) needed to
/// instantiate this component.
pub num_lowerings: u32,
@@ -382,6 +387,8 @@ pub enum CoreDef {
/// This is a reference to a Cranelift-generated trampoline which is
/// described in the `trampolines` array.
Trampoline(TrampolineIndex),
+ /// An intrinsic for compile-time builtins.
+ UnsafeIntrinsic(UnsafeIntrinsic),
}
impl<T> From<CoreExport<T>> for CoreDef
diff --git a/crates/environ/src/component/intrinsic.rs b/crates/environ/src/component/intrinsic.rs
new file mode 100644
index 0000000000..391c9038f8
--- /dev/null
+++ b/crates/environ/src/component/intrinsic.rs
@@ -0,0 +1,147 @@
+//! Wasmtime compile-time intrinsic definitions.
+
+use core::str::FromStr;
+
+use anyhow::{Result, bail};
+use serde_derive::{Deserialize, Serialize};
+
+/// Invoke a macro for each of our unsafe intrinsics.
+#[macro_export]
+macro_rules! for_each_unsafe_intrinsic {
+ ($mac:ident) => {
+ $mac! {
+ "store-data-address" => StoreDataAddress : store_data_address() -> u64;
+
+ "u8-native-load" => U8NativeLoad : u8_native_load(address: u64) -> u8;
+ "u8-native-store" => U8NativeStore : u8_native_store(address: u64, value: u8);
+
+ "u16-native-load" => U16NativeLoad : u16_native_load(address: u64) -> u16;
+ "u16-native-store" => U16NativeStore : u16_native_store(address: u64, value: u16);
+
+ "u32-native-load" => U32NativeLoad : u32_native_load(address: u64) -> u32;
+ "u32-native-store" => U32NativeStore : u32_native_store(address: u64, value: u32);
+
+ "u64-native-load" => U64NativeLoad : u64_native_load(address: u64) -> u64;
+ "u64-native-store" => U64NativeStore : u64_native_store(address: u64, value: u64);
+ }
+ };
+}
+
+macro_rules! define_unsafe_intrinsics {
+ (
+ $(
+ $symbol:expr => $variant:ident : $ctor:ident ( $( $param:ident : $param_ty:ident ),* ) $( -> $result_ty:ident )? ;
+ )*
+ ) => {
+ /// An index type for Wasmtime's intrinsics available to compile-time
+ /// builtins.
+ #[repr(u32)]
+ #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+ pub enum UnsafeIntrinsic {
+ $(
+ #[doc = concat!("The `", $symbol, "` intrinsic.")]
+ $variant,
+ )*
+ }
+
+ impl UnsafeIntrinsic {
+ /// Returns the total number of unsafe intrinsics.
+ pub const fn len() -> u32 {
+ let mut len = 0;
+ $(
+ let _ = Self::$variant;
+ len += 1;
+ )*
+ len
+ }
+
+ /// Construct an `UnsafeIntrinsic` from its `u32` index.
+ ///
+ /// Panics on invalid indices.
+ pub const fn from_u32(i: u32) -> Self {
+ assert!(i < Self::len());
+ $(
+ if i == Self::$variant.index() {
+ return Self::$variant;
+ }
+ )*
+ unreachable!()
+ }
+
+ /// Get this intrinsic's index.
+ pub const fn index(&self) -> u32 {
+ *self as u32
+ }
+
+ /// Get this intrinsic's name.
+ pub const fn name(&self) -> &'static str {
+ match self {
+ $(
+ Self::$variant => $symbol,
+ )*
+ }
+ }
+
+ /// Get this intrinsic's parameters, as core Wasm value types.
+ pub const fn core_params(&self) -> &'static [$crate::WasmValType] {
+ match self {
+ $(
+ Self::$variant => &[ $( define_unsafe_intrinsics!(@core_type $param_ty) ),* ],
+ )*
+ }
+ }
+
+ /// Get this intrinsic's results, as core Wasm value types.
+ pub const fn core_results(&self) -> &'static [$crate::WasmValType] {
+ match self {
+ $(
+ Self::$variant => &[ $( define_unsafe_intrinsics!(@core_type $result_ty) )? ],
+ )*
+ }
+ }
+
+ /// Get this intrinsic's parameters, as component model interface types.
+ pub const fn component_params(&self) -> &'static [$crate::component::InterfaceType] {
+ match self {
+ $(
+ Self::$variant => &[ $( define_unsafe_intrinsics!(@component_type $param_ty) ),* ],
+ )*
+ }
+ }
+
+ /// Get this intrinsic's results, as component model interface types.
+ pub const fn component_results(&self) -> &'static [$crate::component::InterfaceType] {
+ match self {
+ $(
+ Self::$variant => &[ $( define_unsafe_intrinsics!(@component_type $result_ty) ),* ],
+ )*
+ }
+ }
+ }
+
+ impl FromStr for UnsafeIntrinsic {
+ type Err = anyhow::Error;
+
+ fn from_str(s: &str) -> Result<Self> {
+ match s {
+ $(
+ $symbol => Ok(Self::$variant),
+ )*
+ _ => bail!("invalid unsafe intrinsic: {s:?}"),
+ }
+ }
+ }
+ };
+
+ (@core_type u8) => { $crate::WasmValType::I32 };
+ (@core_type u16) => { $crate::WasmValType::I32 };
+ (@core_type u32) => { $crate::WasmValType::I32 };
+ (@core_type u64) => { $crate::WasmValType::I64 };
+
+ (@component_type u8) => { $crate::component::InterfaceType::U8 };
+ (@component_type u16) => { $crate::component::InterfaceType::U16 };
+ (@component_type u32) => { $crate::component::InterfaceType::U32 };
+ (@component_type u64) => { $crate::component::InterfaceType::U64 };
+}
+
+for_each_unsafe_intrinsic!(define_unsafe_intrinsics);
diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rs
index 9cb276f43a..580dc5c826 100644
--- a/crates/environ/src/component/translate.rs
+++ b/crates/environ/src/component/translate.rs
@@ -1,14 +1,17 @@
-use crate::ScopeVec;
+use crate::Abi;
use crate::component::dfg::AbstractInstantiations;
use crate::component::*;
use crate::prelude::*;
use crate::{
- EngineOrModuleTypeIndex, EntityIndex, ModuleEnvironment, ModuleInternedTypeIndex,
- ModuleTranslation, ModuleTypesBuilder, PrimaryMap, TagIndex, Tunables, TypeConvert,
+ EngineOrModuleTypeIndex, EntityIndex, FuncKey, ModuleEnvironment, ModuleInternedTypeIndex,
+ ModuleTranslation, ModuleTypesBuilder, PrimaryMap, ScopeVec, TagIndex, Tunables, TypeConvert,
WasmHeapType, WasmResult, WasmValType,
};
+use anyhow::Context;
use anyhow::anyhow;
+use anyhow::ensure;
use anyhow::{Result, bail};
+use core::str::FromStr;
use cranelift_entity::SecondaryMap;
use cranelift_entity::packed_option::PackedOption;
use indexmap::IndexMap;
@@ -73,6 +76,9 @@ pub struct Translator<'a, 'data> {
/// As frames are popped from `lexical_scopes` their completed component
/// will be pushed onto this list.
static_components: PrimaryMap<StaticComponentIndex, Translation<'data>>,
+
+ /// The top-level import name for Wasmtime's unsafe intrinsics, if any.
+ unsafe_intrinsics_import: Option<&'a str>,
}
/// Representation of the syntactic scope of a component meaning where it is
@@ -176,6 +182,9 @@ enum LocalInitializer<'data> {
// imports
Import(ComponentImportName<'data>, ComponentEntityType),
+ // An import of an intrinsic for compile-time builtins.
+ IntrinsicsImport,
+
// canonical function sections
Lower {
func: ComponentFuncIndex,
@@ -425,9 +434,18 @@ impl<'a, 'data> Translator<'a, 'data> {
static_components: Default::default(),
static_modules: Default::default(),
scope_vec,
+ unsafe_intrinsics_import: None,
}
}
+ /// Expose Wasmtime's unsafe intrinsics under the given top-level import
+ /// name.
+ pub fn expose_unsafe_intrinsics(&mut self, name: &'a str) -> &mut Self {
+ assert!(self.unsafe_intrinsics_import.is_none());
+ self.unsafe_intrinsics_import = Some(name);
+ self
+ }
+
/// Translates the binary `component`.
///
/// This is the workhorse of compilation which will parse all of
@@ -566,7 +584,7 @@ impl<'a, 'data> Translator<'a, 'data> {
.is_none()
);
- match arg {
+ let known_func = match arg {
CoreDef::InstanceFlags(_) => unreachable!("instance flags are not a function"),
// We could in theory inline these trampolines, so it could
@@ -583,6 +601,11 @@ impl<'a, 'data> Translator<'a, 'data> {
// the effort.
CoreDef::Trampoline(_) => continue,
+ // This import is a compile-time builtin intrinsic, we
+ // should inline its implementation during function
+ // translation.
+ CoreDef::UnsafeIntrinsic(i) => FuncKey::UnsafeIntrinsic(Abi::Wasm, *i),
+
// This imported function is an export from another
// instance, a perfect candidate for becoming an inlinable
// direct call!
@@ -613,14 +636,15 @@ impl<'a, 'data> Translator<'a, 'data> {
continue;
};
- assert!(
- self.static_modules[module].known_imported_functions[imported_func]
- .is_none()
- );
- self.static_modules[module].known_imported_functions[imported_func] =
- Some((*arg_module, arg_module_def_func));
+ FuncKey::DefinedWasmFunction(*arg_module, arg_module_def_func)
}
- }
+ };
+
+ assert!(
+ self.static_modules[module].known_imported_functions[imported_func].is_none()
+ );
+ self.static_modules[module].known_imported_functions[imported_func] =
+ Some(known_func);
}
}
}
@@ -725,9 +749,17 @@ impl<'a, 'data> Translator<'a, 'data> {
let ty = types
.component_entity_type_of_import(import.name.0)
.unwrap();
- self.result
- .initializers
- .push(LocalInitializer::Import(import.name, ty));
+
+ if self.is_unsafe_intrinsics_import(import.name.0) {
+ self.check_unsafe_intrinsics_import(import.name.0, ty)?;
+ self.result
+ .initializers
+ .push(LocalInitializer::IntrinsicsImport);
+ } else {
+ self.result
+ .initializers
+ .push(LocalInitializer::Import(import.name, ty));
+ }
}
}
@@ -1555,6 +1587,105 @@ impl<'a, 'data> Translator<'a, 'data> {
let id = types.core_function_at(index);
self.types.module_types_builder().intern_type(types, id)
}
+
+ fn is_unsafe_intrinsics_import(&self, import: &str) -> bool {
+ self.lexical_scopes.is_empty()
+ && self
+ .unsafe_intrinsics_import
+ .is_some_and(|name| import == name)
+ }
+
+ fn check_unsafe_intrinsics_import(&self, import: &str, ty: ComponentEntityType) -> Result<()> {
+ let types = &self.validator.types(0).unwrap();
+
+ let ComponentEntityType::Instance(instance_ty) = ty else {
+ bail!("bad unsafe intrinsics import: import `{import}` must be an instance import")
+ };
+ let instance_ty = &types[instance_ty];
+
+ ensure!(
+ instance_ty.defined_resources.is_empty(),
+ "bad unsafe intrinsics import: import `{import}` cannot define any resources"
+ );
+ ensure!(
+ instance_ty.explicit_resources.is_empty(),
+ "bad unsafe intrinsics import: import `{import}` cannot export any resources"
+ );
+
+ for (name, ty) in &instance_ty.exports {
+ let ComponentEntityType::Func(func_ty) = ty else {
+ bail!(
+ "bad unsafe intrinsics import: imported instance `{import}` must \
+ only export functions"
+ )
+ };
+ let func_ty = &types[*func_ty];
+
+ fn ty_eq(a: &InterfaceType, b: &wasmparser::component_types::ComponentValType) -> bool {
+ use wasmparser::{PrimitiveValType as P, component_types::ComponentValType as C};
+ match (a, b) {
+ (InterfaceType::U8, C::Primitive(P::U8)) => true,
+ (InterfaceType::U8, _) => false,
+
+ (InterfaceType::U16, C::Primitive(P::U16)) => true,
+ (InterfaceType::U16, _) => false,
+
+ (InterfaceType::U32, C::Primitive(P::U32)) => true,
+ (InterfaceType::U32, _) => false,
+
+ (InterfaceType::U64, C::Primitive(P::U64)) => true,
+ (InterfaceType::U64, _) => false,
+
+ (ty, _) => unreachable!("no unsafe intrinsics use {ty:?}"),
+ }
+ }
+
+ fn check_types<'a>(
+ expected: impl ExactSizeIterator<Item = &'a InterfaceType>,
+ actual: impl ExactSizeIterator<Item = &'a wasmparser::component_types::ComponentValType>,
+ kind: &str,
+ import: &str,
+ name: &str,
+ ) -> core::result::Result<(), anyhow::Error> {
+ let expected_len = expected.len();
+ let actual_len = actual.len();
+ ensure!(
+ expected_len == actual_len,
+ "bad unsafe intrinsics import at `{import}`: function `{name}` must have \
+ {expected_len} {kind}, found {actual_len}"
+ );
+
+ for (i, (actual_ty, expected_ty)) in actual.zip(expected).enumerate() {
+ ensure!(
+ ty_eq(expected_ty, actual_ty),
+ "bad unsafe intrinsics import at `{import}`: {kind}[{i}] for function \
+ `{name}` must be `{expected_ty:?}`, found `{actual_ty:?}`"
+ );
+ }
+ Ok(())
+ }
+
+ let intrinsic = UnsafeIntrinsic::from_str(name)
+ .with_context(|| format!("bad unsafe intrinsics import at `{import}`"))?;
+
+ check_types(
+ intrinsic.component_params().iter(),
+ func_ty.params.iter().map(|(_name, ty)| ty),
+ "parameters",
+ &import,
+ &name,
+ )?;
+ check_types(
+ intrinsic.component_results().iter(),
+ func_ty.result.iter(),
+ "results",
+ &import,
+ &name,
+ )?;
+ }
+
+ Ok(())
+ }
}
impl Translation<'_> {
diff --git a/crates/environ/src/component/translate/adapt.rs b/crates/environ/src/component/translate/adapt.rs
index 64c9e1a9c6..ad3a00f525 100644
--- a/crates/environ/src/component/translate/adapt.rs
+++ b/crates/environ/src/component/translate/adapt.rs
@@ -448,7 +448,9 @@ impl PartitionAdapterModules {
}
// These items can't transitively depend on an adapter
- dfg::CoreDef::Trampoline(_) | dfg::CoreDef::InstanceFlags(_) => {}
+ dfg::CoreDef::Trampoline(_)
+ | dfg::CoreDef::InstanceFlags(_)
+ | dfg::CoreDef::UnsafeIntrinsic(..) => {}
}
}
diff --git a/crates/environ/src/component/translate/inline.rs b/crates/environ/src/component/translate/inline.rs
index ec9382c8e9..360f7313f8 100644
--- a/crates/environ/src/component/translate/inline.rs
+++ b/crates/environ/src/component/translate/inline.rs
@@ -47,6 +47,7 @@
use crate::component::translate::*;
use crate::{EntityType, IndexType};
+use core::str::FromStr;
use std::borrow::Cow;
use wasmparser::component_types::{ComponentAnyTypeId, ComponentCoreModuleTypeId};
@@ -302,6 +303,9 @@ enum ModuleInstanceDef<'a> {
#[derive(Clone)]
enum ComponentFuncDef<'a> {
+ /// A compile-time builtin intrinsic.
+ UnsafeIntrinsic(UnsafeIntrinsic),
+
/// A host-imported component function.
Import(ImportPath<'a>),
@@ -318,6 +322,10 @@ enum ComponentFuncDef<'a> {
#[derive(Clone)]
enum ComponentInstanceDef<'a> {
+ /// The `__wasmtime_intrinsics` instance that exports all of our
+ /// compile-time builtin intrinsics.
+ Intrinsics,
+
/// A host-imported instance.
///
/// This typically means that it's "just" a map of named values. It's not
@@ -485,6 +493,12 @@ impl<'a> Inliner<'a> {
frame.push_item(arg.clone());
}
+ IntrinsicsImport => {
+ frame
+ .component_instances
+ .push(ComponentInstanceDef::Intrinsics);
+ }
+
// Lowering a component function to a core wasm function is
// generally what "triggers compilation". Here various metadata is
// recorded and then the final component gets an initializer
@@ -597,6 +611,10 @@ impl<'a> Inliner<'a> {
});
dfg::CoreDef::Adapter(adapter_idx)
}
+
+ ComponentFuncDef::UnsafeIntrinsic(intrinsic) => {
+ dfg::CoreDef::UnsafeIntrinsic(options.core_type, *intrinsic)
+ }
};
frame.funcs.push((lower_core_type, func));
}
@@ -1319,6 +1337,12 @@ impl<'a> Inliner<'a> {
AliasComponentExport(instance, name) => {
match &frame.component_instances[*instance] {
+ ComponentInstanceDef::Intrinsics => {
+ frame.push_item(ComponentItemDef::Func(ComponentFuncDef::UnsafeIntrinsic(
+ UnsafeIntrinsic::from_str(name)?,
+ )));
+ }
+
// Aliasing an export from an imported instance means that
// we're extending the `ImportPath` by one name, represented
// with the clone + push here. Afterwards an appropriate
@@ -1581,11 +1605,23 @@ impl<'a> Inliner<'a> {
"component export `{name}` is a reexport of an imported function which is not implemented"
)
}
+
+ ComponentFuncDef::UnsafeIntrinsic(_) => {
+ bail!(
+ "component export `{name}` is a reexport of an intrinsic function which is not supported"
+ )
+ }
},
ComponentItemDef::Instance(instance) => {
let mut exports = IndexMap::new();
match instance {
+ ComponentInstanceDef::Intrinsics => {
+ bail!(
+ "component export `{name}` is a reexport of the intrinsics instance which is not supported"
+ )
+ }
+
// If this instance is one that was originally imported by
// the component itself then the imports are translated here
// by converting to a `ComponentItemDef` and then
@@ -1848,6 +1884,9 @@ impl<'a> ComponentItemDef<'a> {
ComponentItemDef::from_import(path.push(element), types[ty].exports[element])
.unwrap()
}
+ ComponentInstanceDef::Intrinsics => {
+ unreachable!("intrinsics do not define resources")
+ }
};
}
diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rs
index 435edb2aaf..bbbb0e79a1 100644
--- a/crates/environ/src/component/vmcomponent_offsets.rs
+++ b/crates/environ/src/component/vmcomponent_offsets.rs
@@ -6,6 +6,7 @@
// limits: *const VMStoreContext,
// flags: [VMGlobalDefinition; component.num_runtime_component_instances],
// trampoline_func_refs: [VMFuncRef; component.num_trampolines],
+// unsafe_intrinsics: [VMFuncRef; component.num_unsafe_intrinsics],
// lowerings: [VMLowering; component.num_lowerings],
// memories: [*mut VMMemoryDefinition; component.num_runtime_memories],
// tables: [VMTable; component.num_runtime_tables],
@@ -58,6 +59,9 @@ pub struct VMComponentOffsets<P> {
pub num_runtime_component_instances: u32,
/// Number of cranelift-compiled trampolines required for this component.
pub num_trampolines: u32,
+ /// Number of `VMFuncRef`s for unsafe intrinsics within this component's
+ /// context.
+ pub num_unsafe_intrinsics: u32,
/// Number of resources within a component which need destructors stored.
pub num_resources: u32,
@@ -67,6 +71,7 @@ pub struct VMComponentOffsets<P> {
vm_store_context: u32,
flags: u32,
trampoline_func_refs: u32,
+ intrinsic_func_refs: u32,
lowerings: u32,
memories: u32,
tables: u32,
@@ -97,12 +102,35 @@ impl<P: PtrSize> VMComponentOffsets<P> {
num_runtime_post_returns: component.num_runtime_post_returns,
num_runtime_component_instances: component.num_runtime_component_instances,
num_trampolines: component.trampolines.len().try_into().unwrap(),
+ num_unsafe_intrinsics: if let Some(i) = component
+ .unsafe_intrinsics
+ .iter()
+ .rposition(|x| x.is_some())
+ {
+ // Note: We do not currently have an indirection between "the
+ // `i`th unsafe intrinsic in the vmctx" and
+ // `UnsafeIntrinsic::from_u32(i)`, so therefore if we are
+ // compiling in *any* intrinsics, we need to include space for
+ // all of them up to the max `i` that is used.
+ //
+ // We _could_ introduce such an indirection via a map in
+ // `Component` like `PrimaryMap<UnsafeIntrinsicIndex,
+ // UnsafeIntrinsic>`, and that would allow us to densely pack
+ // intrinsics in the vmctx. However we do not do that today
+ // because there are very few unsafe intrinsics, and we do not
+ // see that changing anytime soon, so we aren't wasting much
+ // space.
+ u32::try_from(i + 1).unwrap()
+ } else {
+ 0
+ },
num_resources: component.num_resources,
magic: 0,
builtins: 0,
vm_store_context: 0,
flags: 0,
trampoline_func_refs: 0,
+ intrinsic_func_refs: 0,
lowerings: 0,
memories: 0,
tables: 0,
@@ -146,6 +174,7 @@ impl<P: PtrSize> VMComponentOffsets<P> {
size(flags) = cmul(ret.num_runtime_component_instances, ret.ptr.size_of_vmglobal_definition()),
align(u32::from(ret.ptr.size())),
size(trampoline_func_refs) = cmul(ret.num_trampolines, ret.ptr.size_of_vm_func_ref()),
+ size(intrinsic_func_refs) = cmul(ret.num_unsafe_intrinsics, ret.ptr.size_of_vm_func_ref()),
size(lowerings) = cmul(ret.num_lowerings, ret.ptr.size() * 2),
size(memories) = cmul(ret.num_runtime_memories, ret.ptr.size()),
size(tables) = cmul(ret.num_runtime_tables, ret.size_of_vmtable_import()),
@@ -209,6 +238,20 @@ impl<P: PtrSize> VMComponentOffsets<P> {
self.trampoline_func_refs() + index.as_u32() * u32::from(self.ptr.size_of_vm_func_ref())
}
+ /// The offset of the `unsafe_intrinsic_func_refs` field.
+ #[inline]
+ pub fn unsafe_intrinsic_func_refs(&self) -> u32 {
+ self.intrinsic_func_refs
+ }
+
+ /// The offset of the `VMFuncRef` for the `intrinsic` specified.
+ #[inline]
+ pub fn unsafe_intrinsic_func_ref(&self, intrinsic: UnsafeIntrinsic) -> u32 {
+ assert!(intrinsic.index() < self.num_unsafe_intrinsics);
+ self.unsafe_intrinsic_func_refs()
+ + intrinsic.index() * u32::from(self.ptr.size_of_vm_func_ref())
+ }
+
/// The offset of the `lowerings` field.
#[inline]
pub fn lowerings(&self) -> u32 {
diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rs
index f09d108d6e..f8b88cf7c7 100644
--- a/crates/environ/src/key.rs
+++ b/crates/environ/src/key.rs
@@ -39,6 +39,10 @@ pub enum FuncKeyKind {
/// A Wasm-caller to array-callee `resource.drop` trampoline.
#[cfg(feature = "component-model")]
ResourceDropTrampoline = FuncKey::new_kind(0b110),
+
+ /// A Wasmtime unsafe intrinsic function.
+ #[cfg(feature = "component-model")]
+ UnsafeIntrinsic = FuncKey::new_kind(0b111),
}
impl From<FuncKeyKind> for u32 {
@@ -68,6 +72,8 @@ impl FuncKeyKind {
x if x == Self::ComponentTrampoline.into() => Self::ComponentTrampoline,
#[cfg(feature = "component-model")]
x if x == Self::ResourceDropTrampoline.into() => Self::ResourceDropTrampoline,
+ #[cfg(feature = "component-model")]
+ x if x == Self::UnsafeIntrinsic.into() => Self::UnsafeIntrinsic,
_ => panic!("invalid raw value passed to `FuncKind::from_raw`: {raw}"),
}
@@ -132,6 +138,12 @@ impl FuncKeyNamespace {
assert_eq!(raw & FuncKey::MODULE_MASK, 0);
Self(raw)
}
+
+ #[cfg(feature = "component-model")]
+ FuncKeyKind::UnsafeIntrinsic => {
+ let _ = Abi::from_raw(raw & FuncKey::MODULE_MASK);
+ Self(raw)
+ }
}
}
@@ -223,6 +235,10 @@ pub enum FuncKey {
/// A Wasm-caller to array-callee `resource.drop` trampoline.
#[cfg(feature = "component-model")]
ResourceDropTrampoline,
+
+ /// A Wasmtime intrinsic function.
+ #[cfg(feature = "component-model")]
+ UnsafeIntrinsic(Abi, component::UnsafeIntrinsic),
}
impl Ord for FuncKey {
@@ -298,6 +314,14 @@ impl FuncKey {
let index = 0;
(namespace, index)
}
+ #[cfg(feature = "component-model")]
+ FuncKey::UnsafeIntrinsic(abi, intrinsic) => {
+ let abi = abi.into_raw();
+ assert_eq!(abi & Self::KIND_MASK, 0);
+ let namespace = FuncKeyKind::UnsafeIntrinsic.into_raw() | abi;
+ let index = intrinsic.index();
+ (namespace, index)
+ }
};
(FuncKeyNamespace(namespace), FuncKeyIndex(index))
}
@@ -389,6 +413,12 @@ impl FuncKey {
assert_eq!(b, 0);
Self::ResourceDropTrampoline
}
+ #[cfg(feature = "component-model")]
+ FuncKeyKind::UnsafeIntrinsic => {
+ let abi = Abi::from_raw(a & Self::MODULE_MASK);
+ let intrinsic = component::UnsafeIntrinsic::from_u32(b);
+ Self::UnsafeIntrinsic(abi, intrinsic)
+ }
}
}
diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs
index fb76f0d1d7..c5c0643c43 100644
--- a/crates/environ/src/module_artifacts.rs
+++ b/crates/environ/src/module_artifacts.rs
@@ -509,7 +509,9 @@ impl CompiledFunctionsTable {
FuncKeyKind::ArrayToWasmTrampoline | FuncKeyKind::WasmToBuiltinTrampoline => false,
#[cfg(feature = "component-model")]
- FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline => true,
+ FuncKeyKind::ComponentTrampoline
+ | FuncKeyKind::ResourceDropTrampoline
+ | FuncKeyKind::UnsafeIntrinsic => true,
}
}
@@ -522,7 +524,9 @@ impl CompiledFunctionsTable {
| FuncKeyKind::WasmToBuiltinTrampoline
| FuncKeyKind::PulleyHostCall => false,
#[cfg(feature = "component-model")]
- FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline => false,
+ FuncKeyKind::ComponentTrampoline
+ | FuncKeyKind::ResourceDropTrampoline
+ | FuncKeyKind::UnsafeIntrinsic => false,
}
}
}
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 08e09beee7..60c88a2c52 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -238,6 +238,11 @@ pub trait PtrSize {
self.vmstore_context_last_wasm_entry_trap_handler() + self.size()
}
+ /// Return the offset of the `stack_chain` field of `VMStoreContext`.
+ fn vmstore_context_store_data(&self) -> u8 {
+ self.vmstore_context_stack_chain() + self.size_of_vmstack_chain()
+ }
+
// Offsets within `VMMemoryDefinition`
/// The offset of the `base` field.
diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs
index 91fd68244f..bdf885263a 100644
--- a/crates/wasmtime/src/compile.rs
+++ b/crates/wasmtime/src/compile.rs
@@ -61,7 +61,7 @@ mod runtime;
/// Additionally compilation returns an `Option` here which is always
/// `Some`, notably compiled metadata about the module in addition to the
/// type information found within.
-pub(crate) fn build_artifacts<T: FinishedObject>(
+pub(crate) fn build_module_artifacts<T: FinishedObject>(
engine: &Engine,
wasm: &[u8],
dwarf_package: Option<&[u8]>,
@@ -142,6 +142,7 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
engine: &Engine,
binary: &[u8],
_dwarf_package: Option<&[u8]>,
+ unsafe_intrinsics_import: Option<&str>,
obj_state: &T::State,
) -> Result<(T, Option<wasmtime_environ::component::ComponentArtifacts>)> {
use wasmtime_environ::ScopeVec;
@@ -155,10 +156,13 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
let scope = ScopeVec::new();
let mut validator = wasmparser::Validator::new_with_features(engine.features());
let mut types = ComponentTypesBuilder::new(&validator);
- let (component, mut module_translations) =
- Translator::new(tunables, &mut validator, &mut types, &scope)
- .translate(binary)
- .context("failed to parse WebAssembly module")?;
+ let mut translator = Translator::new(tunables, &mut validator, &mut types, &scope);
+ if let Some(name) = unsafe_intrinsics_import {
+ translator.expose_unsafe_intrinsics(name);
+ }
+ let (component, mut module_translations) = translator
+ .translate(binary)
+ .context("failed to parse WebAssembly module")?;
let compile_inputs = CompileInputs::for_component(
engine,
@@ -228,12 +232,10 @@ struct CompileOutput<'a> {
struct InlineHeuristicParams<'a> {
tunables: &'a Tunables,
caller_size: u32,
- caller_module: StaticModuleIndex,
- caller_def_func: DefinedFuncIndex,
+ caller_key: FuncKey,
caller_needs_gc_heap: bool,
callee_size: u32,
- callee_module: StaticModuleIndex,
- callee_def_func: DefinedFuncIndex,
+ callee_key: FuncKey,
callee_needs_gc_heap: bool,
}
@@ -280,21 +282,60 @@ impl<'a> CompileInputs<'a> {
>,
) -> Self {
use wasmtime_environ::Abi;
+ use wasmtime_environ::component::UnsafeIntrinsic;
let mut ret = CompileInputs { inputs: vec![] };
ret.collect_inputs_in_translations(types.module_types_builder(), module_translations);
let tunables = engine.tunables();
+ for i in component
+ .component
+ .unsafe_intrinsics
+ .iter()
+ .enumerate()
+ .filter_map(|(i, ty)| if ty.is_some() { Some(i) } else { None })
+ {
+ let i = u32::try_from(i).unwrap();
+ let intrinsic = UnsafeIntrinsic::from_u32(i);
+ for abi in [Abi::Wasm, Abi::Array] {
+ ret.push_input(move |compiler| {
+ let symbol = format!(
+ "unsafe-intrinsics-{}-{}",
+ match abi {
+ Abi::Wasm => "wasm-call",
+ Abi::Array => "array-call",
+ },
+ intrinsic.name(),
+ );
+ Ok(CompileOutput {
+ key: FuncKey::UnsafeIntrinsic(abi, intrinsic),
+ function: compiler
+ .component_compiler()
+ .compile_intrinsic(tunables, component, intrinsic, abi, &symbol)
+ .with_context(|| format!("failed to compile `{symbol}`"))?,
+ symbol,
+ start_srcloc: FilePos::default(),
+ translation: None,
+ func_body: None,
+ })
+ });
+ }
+ }
+
for (idx, trampoline) in component.trampolines.iter() {
for abi in [Abi::Wasm, Abi::Array] {
ret.push_input(move |compiler| {
let key = FuncKey::ComponentTrampoline(abi, idx);
- let mut symbol = trampoline.symbol_name();
- symbol.push_str(match abi {
- Abi::Wasm => "_wasm_call",
- Abi::Array => "_array_call",
- });
+ let symbol = format!(
+ "component-trampolines[{}]-{}-{}",
+ idx.as_u32(),
+ match abi {
+ Abi::Wasm => "wasm-call",
+ Abi::Array => "array-call",
+ },
+ trampoline.symbol_name(),
+ );
let function = compiler
.component_compiler()
.compile_trampoline(component, types, key, abi, tunables, &symbol)
@@ -521,6 +562,18 @@ the use case.
engine.run_maybe_parallel(self.inputs, |f| f(compiler))?
};
+ if cfg!(debug_assertions) {
+ let mut symbols: Vec<_> = raw_outputs.iter().map(|i| &i.symbol).collect();
+ symbols.sort();
+ for w in symbols.windows(2) {
+ assert_ne!(
+ w[0], w[1],
+ "should never have duplicate symbols, but found two functions with the symbol `{}`",
+ w[0]
+ );
+ }
+ }
+
// Now that all functions have been compiled see if any
// wasmtime-builtin functions are necessary. If so those need to be
// collected and then those trampolines additionally need to be
@@ -553,14 +606,46 @@ the use case.
engine.run_maybe_parallel(self.inputs, |f| f(compiler).map(Some))?,
);
- /// Get just the output indices of the Wasm functions from our unlinked
- /// outputs.
- fn wasm_functions<'a>(
+ /// Whether a function (as described by the given `FuncKey`) can
+ /// participate in inlining or not (either as a candidate for being
+ /// inlined into a caller or having a callee inlined into a callsite
+ /// within itself).
+ fn is_inlining_function(key: FuncKey) -> bool {
+ match key {
+ // Wasm functions can both be inlined into other functions and
+ // have other functions inlined into them.
+ FuncKey::DefinedWasmFunction(..) => true,
+
+ // Intrinsics can be inlined into other functions.
+ #[cfg(feature = "component-model")]
+ FuncKey::UnsafeIntrinsic(..) => true,
+
+ // Trampolines cannot participate in inlining since our
+ // unwinding and exceptions infrastructure relies on them being
+ // in their own call frames.
+ FuncKey::ArrayToWasmTrampoline(..)
+ | FuncKey::WasmToArrayTrampoline(..)
+ | FuncKey::WasmToBuiltinTrampoline(..) => false,
+ #[cfg(feature = "component-model")]
+ FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,
+
+ FuncKey::PulleyHostCall(_) => {
+ unreachable!("we don't compile artifacts for Pulley host calls")
+ }
+ }
+ }
+
+ /// Get just the output indices of the functions that can participate in
+ /// inlining from our unlinked outputs.
+ fn inlining_functions<'a>(
outputs: &'a PrimaryMap<OutputIndex, Option<CompileOutput<'_>>>,
) -> impl Iterator<Item = OutputIndex> + 'a {
- outputs.iter().filter_map(|(i, o)| match o.as_ref()?.key {
- FuncKey::DefinedWasmFunction(..) => Some(i),
- _ => None,
+ outputs.iter().filter_map(|(index, output)| {
+ if is_inlining_function(output.as_ref().unwrap().key) {
+ Some(index)
+ } else {
+ None
+ }
})
}
@@ -571,16 +656,9 @@ the use case.
// example when we know that one module's function import is always
// satisfied with a particular `FuncKey::DefinedWasmFunction`. This map
// enables that translation.
- let key_to_output: HashMap<FuncKey, OutputIndex> = outputs
- .iter()
- .filter(|(_, output)| {
- matches!(
- output.as_ref().unwrap().key,
- FuncKey::DefinedWasmFunction(..)
- )
- })
- .map(|(output_index, output)| {
- let output = output.as_ref().unwrap();
+ let key_to_output: HashMap<FuncKey, OutputIndex> = inlining_functions(&outputs)
+ .map(|output_index| {
+ let output = outputs[output_index].as_ref().unwrap();
(output.key, output_index)
})
.collect();
@@ -590,7 +668,7 @@ the use case.
// We only inline Wasm functions, not trampolines, because we rely on
// trampolines being in their own stack frame when we save the entry and
// exit SP, FP, and PC for backtraces in trampolines.
- let call_graph = CallGraph::<OutputIndex>::new(wasm_functions(&outputs), {
+ let call_graph = CallGraph::<OutputIndex>::new(inlining_functions(&outputs), {
let mut func_keys = IndexSet::default();
let outputs = &outputs;
let key_to_output = &key_to_output;
@@ -598,7 +676,7 @@ the use case.
debug_assert!(calls.is_empty());
let output = outputs[output_index].as_ref().unwrap();
- debug_assert!(matches!(output.key, FuncKey::DefinedWasmFunction(..)));
+ debug_assert!(is_inlining_function(output.key));
// Get this function's call graph edges as `FuncKey`s.
func_keys.clear();
@@ -612,6 +690,7 @@ the use case.
.copied()
.filter_map(|key| key_to_output.get(&key)),
);
+
log::trace!(
"call graph edges for {output_index:?} = {:?}: {calls:?}",
output.key
@@ -625,7 +704,8 @@ the use case.
// (because they either do not call each other or are part of a
// mutual-recursion cycle; either way we won't inline members of the
// same layer into each other).
- let strata = stratify::Strata::<OutputIndex>::new(wasm_functions(&outputs), &call_graph);
+ let strata =
+ stratify::Strata::<OutputIndex>::new(inlining_functions(&outputs), &call_graph);
let mut layer_outputs = vec![];
for layer in strata.layers() {
// Temporarily take this layer's outputs out of our unlinked outputs
@@ -641,17 +721,17 @@ the use case.
&mut layer_outputs,
|output: &mut CompileOutput<'_>| {
log::trace!("processing inlining for {:?}", output.key);
- debug_assert!(matches!(output.key, FuncKey::DefinedWasmFunction(..)));
+ debug_assert!(is_inlining_function(output.key));
- let caller_translation = output.translation.unwrap();
- let (caller_module, caller_def_func) =
- output.key.unwrap_defined_wasm_function();
- let caller_needs_gc_heap = caller_translation.module.needs_gc_heap;
+ let caller_key = output.key;
+ let caller_needs_gc_heap =
+ output.translation.is_some_and(|t| t.module.needs_gc_heap);
let caller = &mut output.function;
let mut caller_size = inlining_compiler.size(caller);
inlining_compiler.inline(caller, &mut |callee_key: FuncKey| {
+ log::trace!(" --> considering call to {callee_key:?}");
let callee_output_index: OutputIndex = key_to_output[&callee_key];
// NB: If the callee is not inside `outputs`, then it is
@@ -667,20 +747,17 @@ the use case.
let callee = &callee_output.function;
let callee_size = inlining_compiler.size(callee);
- let callee_needs_gc_heap =
- callee_output.translation.unwrap().module.needs_gc_heap;
+ let callee_needs_gc_heap = callee_output
+ .translation
+ .is_some_and(|t| t.module.needs_gc_heap);
- let (callee_module, callee_def_func) =
- callee_key.unwrap_defined_wasm_function();
if Self::should_inline(InlineHeuristicParams {
tunables: engine.tunables(),
caller_size,
- caller_module,
- caller_def_func,
+ caller_key,
caller_needs_gc_heap,
callee_size,
- callee_module,
- callee_def_func,
+ callee_key,
callee_needs_gc_heap,
}) {
caller_size = caller_size.saturating_add(callee_size);
@@ -736,21 +813,19 @@ the use case.
InlineHeuristicParams {
tunables,
caller_size,
- caller_module,
- caller_def_func,
+ caller_key,
caller_needs_gc_heap,
callee_size,
- callee_module,
- callee_def_func,
+ callee_key,
callee_needs_gc_heap,
}: InlineHeuristicParams,
) -> bool {
log::trace!(
"considering inlining:\n\
- \tcaller = ({caller_module:?}, {caller_def_func:?})\n\
+ \tcaller = {caller_key:?}\n\
\t\tsize = {caller_size}\n\
\t\tneeds_gc_heap = {caller_needs_gc_heap}\n\
- \tcallee = ({callee_module:?}, {callee_def_func:?})\n\
+ \tcallee = {callee_key:?}\n\
\t\tsize = {callee_size}\n\
\t\tneeds_gc_heap = {callee_needs_gc_heap}"
);
@@ -759,10 +834,7 @@ the use case.
tunables.inlining,
"shouldn't even call this method if we aren't configured for inlining"
);
- debug_assert!(
- caller_module != callee_module || caller_def_func != callee_def_func,
- "we never inline recursion"
- );
+ debug_assert_ne!(caller_key, callee_key, "we never inline recursion");
// Put a limit on how large we can make a function via inlining to cap
// code bloat.
@@ -783,23 +855,33 @@ the use case.
// function calls to small callees were presumably annotated with the
// equivalent of `#[inline(never)]` or `#[cold]` but we don't have that
// information anymore.
- if caller_module == callee_module {
- match tunables.inlining_intra_module {
- IntraModuleInlining::Yes => {}
-
- IntraModuleInlining::WhenUsingGc
- if caller_needs_gc_heap || callee_needs_gc_heap => {}
-
- IntraModuleInlining::WhenUsingGc => {
- log::trace!(" --> not inlining: intra-module call that does not use GC");
- return false;
- }
+ match (caller_key, callee_key) {
+ (
+ FuncKey::DefinedWasmFunction(caller_module, _),
+ FuncKey::DefinedWasmFunction(callee_module, _),
+ ) => {
+ if caller_module == callee_module {
+ match tunables.inlining_intra_module {
+ IntraModuleInlining::Yes => {}
+
+ IntraModuleInlining::WhenUsingGc
+ if caller_needs_gc_heap || callee_needs_gc_heap => {}
+
+ IntraModuleInlining::WhenUsingGc => {
+ log::trace!(
+ " --> not inlining: intra-module call that does not use GC"
+ );
+ return false;
+ }
- IntraModuleInlining::No => {
- log::trace!(" --> not inlining: intra-module call");
- return false;
+ IntraModuleInlining::No => {
+ log::trace!(" --> not inlining: intra-module call");
+ return false;
+ }
+ }
}
}
+ _ => {}
}
// Small callees are often worth inlining regardless of the size of the
diff --git a/crates/wasmtime/src/compile/code_builder.rs b/crates/wasmtime/src/compile/code_builder.rs
index fb7737d82b..c7973bff5a 100644
--- a/crates/wasmtime/src/compile/code_builder.rs
+++ b/crates/wasmtime/src/compile/code_builder.rs
@@ -40,6 +40,7 @@ pub struct CodeBuilder<'a> {
wasm_path: Option<Cow<'a, Path>>,
dwarf_package: Option<Cow<'a, [u8]>>,
dwarf_package_path: Option<Cow<'a, Path>>,
+ unsafe_intrinsics_import: Option<String>,
}
/// Return value of [`CodeBuilder::hint`]
@@ -60,6 +61,7 @@ impl<'a> CodeBuilder<'a> {
wasm_path: None,
dwarf_package: None,
dwarf_package_path: None,
+ unsafe_intrinsics_import: None,
}
}
@@ -181,6 +183,472 @@ impl<'a> CodeBuilder<'a> {
.ok_or_else(|| anyhow!("no wasm bytes have been configured"))
}
+ /// Expose Wasmtime's unsafe intrinsics under the given import name.
+ ///
+ /// These intrinsics provide native memory loads and stores to Wasm; they
+ /// are *extremely* unsafe! If you are not absolutely sure that you need
+ /// these unsafe intrinsics, *do not use them!* See the safety section below
+ /// for details.
+ ///
+ /// This functionality is intended to be used when implementing
+ /// "compile-time builtins"; that is, satisfying a Wasm import via
+ /// special-cased, embedder-specific code at compile time. You should never
+ /// use these intrinsics to intentionally subvert the Wasm sandbox. You
+ /// should strive to implement safe functions that encapsulate your uses of
+ /// these intrinsics such that, regardless of any value given as arguments,
+ /// your functions *cannot* result in loading from or storing to invalid
+ /// pointers, or any other kind of unsafety. See below for an example of the
+ /// intended use cases.
+ ///
+ /// Wasmtime's unsafe intrinsics can only be exposed to Wasm components, not
+ /// core modules, currently.
+ ///
+ /// # Safety
+ ///
+ /// Extreme care must be taken when using these intrinsics.
+ ///
+ /// All loads of or stores to pointers derived from `store-data-address` are
+ /// inherently tied to a particular `T` type in a `Store<T>`. It is wildly
+ /// unsafe to run a Wasm program that uses unsafe intrinsics to access the
+ /// store's `T` inside a `Store<U>`. You must only run Wasm that uses unsafe
+ /// intrinsics in a `Store<T>` where the `T` is the type expected by the
+ /// Wasm's unsafe-intrinsics usage.
+ ///
+ /// Furthermore, usage of these intrinsics is not only tied to a particular
+ /// `T` type, but also to `T`'s layout on the host platform. The size and
+ /// alignment of `T`, the offsets of its fields, and those fields' size and
+ /// alignment can all vary across not only architecture but also operating
+ /// system. With care, you can define your `T` type such that its layout is
+ /// identical across the platforms that you run Wasm on, allowing you to
+ /// reuse the same Wasm binary and its unsafe-intrinsics usage on all your
+ /// platforms. Failing that, you must only run a Wasm program that uses
+ /// unsafe intrinsics on the host platform that its unsafe-intrinsic usage
+ /// is specialized to. See the portability section and example below for
+ /// more details.
+ ///
+ /// You are *strongly* encouraged to add assertions for the layout
+ /// properties that your unsafe-intrinsic usage's safety relies upon:
+ ///
+ /// ```rust
+ /// /// This type is used as `wasmtime::Store<MyData>` and accessed by Wasm via
+ /// /// unsafe intrinsics.
+ /// #[repr(C, align(8))]
+ /// struct MyData {
+ /// id: u64,
+ /// counter: u32,
+ /// buf: [u8; 4],
+ /// }
+ ///
+ /// // Assert that the layout is what our Wasm's unsafe-intrinsics usage expects.
+ /// static _MY_DATA_LAYOUT_ASSERTIONS: () = {
+ /// assert!(core::mem::size_of::<MyData>() == 16);
+ /// assert!(core::mem::align_of::<MyData>() == 8);
+ /// assert!(core::mem::offset_of!(MyData, id) == 0);
+ /// assert!(core::mem::offset_of!(MyData, counter) == 8);
+ /// assert!(core::mem::offset_of!(MyData, buf) == 12);
+ /// };
+ /// ```
+ ///
+ /// Finally, every pointer loaded from or stored to must:
+ ///
+ /// * Be non-null
+ ///
+ /// * Be aligned to the access type's natural alignment (e.g. 8-byte alignment
+ /// for `u64`, 4-byte alignment for `u32`, etc...)
+ ///
+ /// * Point to a memory block that is valid to read from (for loads) or
+ /// valid to write to (for stores) under Rust's pointer provenance rules
+ ///
+ /// * Point to a memory block that is at least as large as the access type's
+ /// natural size (e.g. 1 byte for `u8`, 2 bytes for `u16`, etc...)
+ ///
+ /// * Point to a memory block that is not accessed concurrently by any other
+ /// threads
+ ///
+ /// Failure to uphold any of these invariants will lead to unsafety,
+ /// undefined behavior, and/or data races.
+ ///
+ /// # Intrinsics
+ ///
+ /// | Name | Parameters | Results |
+ /// |----------------------|--------------|---------|
+ /// | `u8-native-load` | `u64` | `u8` |
+ /// | `u16-native-load` | `u64` | `u16` |
+ /// | `u32-native-load` | `u64` | `u32` |
+ /// | `u64-native-load` | `u64` | `u64` |
+ /// | `u8-native-store` | `u64`, `u8` | - |
+ /// | `u16-native-load` | `u64`, `u16` | - |
+ /// | `u32-native-load` | `u64`, `u32` | - |
+ /// | `u64-native-load` | `u64`, `u64` | - |
+ /// | `store-data-address` | - | `u64` |
+ ///
+ /// ## `*-native-load`
+ ///
+ /// These intrinsics perform an unsandboxed, unsynchronized load from native
+ /// memory, using the native endianness.
+ ///
+ /// ## `*-native-store`
+ ///
+ /// These intrinsics perform an unsandboxed, unsynchronized store to native
+ /// memory, using the native endianness.
+ ///
+ /// ## `store-data-address`
+ ///
+ /// This intrinsic function returns the pointer to the embedder's `T` data
+ /// inside a `Store<T>`.
+ ///
+ /// In general, all native load and store intinsics should operate on memory
+ /// addresses that are derived from a call to this intrinsic. If you want to
+ /// expose data for raw memory access by Wasm, put it inside the `T` in your
+ /// `Store<T>` and Wasm's access to that data should derive from this
+ /// intrinsic.
+ ///
+ /// # Portability
+ ///
+ /// Loads and stores are always performed using the architecture's native
+ /// endianness.
+ ///
+ /// Addresses passed to and returned from these intrinsics are always
+ /// 64-bits large. The upper half of the value is simply ignored on 32-bit
+ /// architectures.
+ ///
+ /// With care, you can design your store's `T` type such that accessing it
+ /// via these intrinsics is portable, and you can reuse a single Wasm binary
+ /// (and its set of intrinsic calls) across all of the platforms, with the
+ /// following rules of thumb:
+ ///
+ /// * Only access `u8`, `u16`, `u32`, and `u64` data via these intrinsics.
+ ///
+ /// * If you need to access other types of data, encode it into those types
+ /// and then access the encoded data from the intrinsics.
+ ///
+ /// * Use `union`s to encode pointers and pointer-sized data as a `u64` and
+ /// then access it via the `u64-native-{load,store}` intrinsics. See
+ /// `ExposedPointer` in the example below.
+ ///
+ /// # Example
+ ///
+ /// The following example shows how you can use unsafe intrinsics to give
+ /// Wasm direct zero-copy access to a host buffer.
+ ///
+ /// ```rust
+ /// use std::mem;
+ /// use wasmtime::*;
+ ///
+ /// // A `*mut u8` pointer that is exposed directly to Wasm via unsafe intrinsics.
+ /// #[repr(align(8))]
+ /// union ExposedPointer {
+ /// pointer: *mut u8,
+ /// padding: u64,
+ /// }
+ ///
+ /// static _EXPOSED_POINTER_LAYOUT_ASSERTIONS: () = {
+ /// assert!(mem::size_of::<ExposedPointer>() == 8);
+ /// assert!(mem::align_of::<ExposedPointer>() == 8);
+ /// };
+ ///
+ /// impl ExposedPointer {
+ /// /// Wrap the given pointer into an `ExposedPointer`.
+ /// fn new(pointer: *mut u8) -> Self {
+ /// // NB: Zero-initialize to avoid potential footguns with accessing
+ /// // undefined bytes.
+ /// let mut p = Self { padding: 0 };
+ /// p.pointer = pointer;
+ /// p
+ /// }
+ ///
+ /// /// Get the wrapped pointer.
+ /// fn get(&self) -> *mut u8 {
+ /// unsafe { self.pointer }
+ /// }
+ /// }
+ ///
+ /// /// This is the `T` type we will put inside our
+ /// /// `wasmtime::Store<T>`s. It contains a pointer to a heap-allocated buffer
+ /// /// in host memory, which we will give Wasm zero-copy access to via unsafe
+ /// /// intrinsics.
+ /// #[repr(C)]
+ /// struct StoreData {
+ /// buf_ptr: ExposedPointer,
+ /// buf_len: u64,
+ /// }
+ ///
+ /// static _STORE_DATA_LAYOUT_ASSERTIONS: () = {
+ /// assert!(mem::size_of::<StoreData>() == 16);
+ /// assert!(mem::align_of::<StoreData>() == 8);
+ /// assert!(mem::offset_of!(StoreData, buf_ptr) == 0);
+ /// assert!(mem::offset_of!(StoreData, buf_len) == 8);
+ /// };
+ ///
+ /// impl Drop for StoreData {
+ /// fn drop(&mut self) {
+ /// let len = usize::try_from(self.buf_len).unwrap();
+ /// let ptr = std::ptr::slice_from_raw_parts_mut(self.buf_ptr.get(), len);
+ /// unsafe {
+ /// let _ = Box::from_raw(ptr);
+ /// }
+ /// }
+ /// }
+ ///
+ /// impl StoreData {
+ /// /// Create a new `StoreData`, allocating an inner buffer of `capacity`
+ /// /// bytes.
+ /// fn new(bytes: impl IntoIterator<Item = u8>) -> Self {
+ /// let buf: Box<[u8]> = bytes.into_iter().collect();
+ /// let ptr = Box::into_raw(buf);
+ /// Self {
+ /// buf_ptr: ExposedPointer::new(ptr.cast::<u8>()),
+ /// buf_len: u64::try_from(ptr.len()).unwrap(),
+ /// }
+ /// }
+ ///
+ /// /// Get the inner buffer as a shared slice.
+ /// fn buf(&self) -> &[u8] {
+ /// let ptr = self.buf_ptr.get().cast_const();
+ /// let len = usize::try_from(self.buf_len).unwrap();
+ /// unsafe {
+ /// std::slice::from_raw_parts(ptr, len)
+ /// }
+ /// }
+ ///
+ /// /// Get the inner buffer as a mutable slice.
+ /// fn buf_mut(&mut self) -> &mut [u8] {
+ /// let ptr = self.buf_ptr.get();
+ /// let len = usize::try_from(self.buf_len).unwrap();
+ /// unsafe {
+ /// std::slice::from_raw_parts_mut(ptr, len)
+ /// }
+ /// }
+ /// }
+ ///
+ /// # fn main() -> Result<()> {
+ /// // Enable function inlining during compilation. If you are using unsafe intrinsics, you
+ /// // almost assuredly want them inlined to avoid function call overheads.
+ /// let mut config = Config::new();
+ /// config.compiler_inlining(true);
+ ///
+ /// let engine = Engine::new(&config)?;
+ /// let linker = wasmtime::component::Linker::new(&engine);
+ ///
+ /// // Create a new builder for configuring a Wasm compilation.
+ /// let mut builder = CodeBuilder::new(&engine);
+ ///
+ /// // Allow the code we are building to use Wasmtime's unsafe intrinsics.
+ /// //
+ /// // SAFETY: we wrap all usage of the intrinsics in safe APIs and only instantiate the code
+ /// // within a `Store<T>` where `T = StoreData`, as the code expects.
+ /// unsafe {
+ /// builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ /// }
+ ///
+ /// // Provide the Wasm that we are compiling.
+ /// builder.wasm_binary_or_text(r#"
+ /// (component
+ /// ;; Import the unsafe intrinsics that we will use.
+ /// (import "unsafe-intrinsics"
+ /// (instance $intrinsics
+ /// (export "store-data-address" (func (result u64)))
+ /// (export "u64-native-load" (func (param "pointer" u64) (result u64)))
+ /// (export "u8-native-load" (func (param "pointer" u64) (result u8)))
+ /// (export "u8-native-store" (func (param "pointer" u64) (param "value" u8)))
+ /// )
+ /// )
+ ///
+ /// ;; A component that encapsulates the intrinsics' unsafety, exposing a safe API
+ /// ;; built on top of them.
+ /// (component $safe-api
+ /// (import "unsafe-intrinsics"
+ /// (instance $intrinsics
+ /// (export "store-data-address" (func (result u64)))
+ /// (export "u64-native-load" (func (param "pointer" u64) (result u64)))
+ /// (export "u8-native-load" (func (param "pointer" u64) (result u8)))
+ /// (export "u8-native-store" (func (param "pointer" u64) (param "value" u8)))
+ /// )
+ /// )
+ ///
+ /// ;; The core Wasm module that implements the safe API.
+ /// (core module $safe-api-impl
+ /// (import "" "store-data-address" (func $store-data-address (result i64)))
+ /// (import "" "u64-native-load" (func $u64-native-load (param i64) (result i64)))
+ /// (import "" "u8-native-load" (func $u8-native-load (param i64) (result i32)))
+ /// (import "" "u8-native-store" (func $u8-native-store (param i64 i32)))
+ ///
+ /// ;; Load the `StoreData::buf_ptr` field
+ /// (func $get-buf-ptr (result i64)
+ /// (call $u64-native-load (i64.add (call $store-data-address) (i64.const 0)))
+ /// )
+ ///
+ /// ;; Load the `StoreData::buf_len` field
+ /// (func $get-buf-len (result i64)
+ /// (call $u64-native-load (i64.add (call $store-data-address) (i64.const 8)))
+ /// )
+ ///
+ /// ;; Check that `$i` is within `StoreData` buffer's bounds, raising a trap
+ /// ;; otherwise.
+ /// (func $bounds-check (param $i i64)
+ /// (if (i64.lt_u (local.get $i) (call $get-buf-len))
+ /// (then (return))
+ /// (else (unreachable))
+ /// )
+ /// )
+ ///
+ /// ;; A safe function to get the `i`th byte from `StoreData`'s buffer, raising
+ /// ;; a trap on out-of-bounds accesses.
+ /// (func (export "get") (param $i i64) (result i32)
+ /// (call $bounds-check (local.get $i))
+ /// (call $u8-native-load (i64.add (call $get-buf-ptr) (local.get $i)))
+ /// )
+ ///
+ /// ;; A safe function to set the `i`th byte in `StoreData`'s buffer, raising
+ /// ;; a trap on out-of-bounds accesses.
+ /// (func (export "set") (param $i i64) (param $value i32)
+ /// (call $bounds-check (local.get $i))
+ /// (call $u8-native-store (i64.add (call $get-buf-ptr) (local.get $i))
+ /// (local.get $value))
+ /// )
+ ///
+ /// ;; A safe function to get the length of the `StoreData` buffer.
+ /// (func (export "len") (result i64)
+ /// (call $get-buf-len)
+ /// )
+ /// )
+ ///
+ /// ;; Lower the imported intrinsics from component functions to core functions.
+ /// (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))
+ /// (core func $u64-native-load' (canon lower (func $intrinsics "u64-native-load")))
+ /// (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))
+ /// (core func $u8-native-store' (canon lower (func $intrinsics "u8-native-store")))
+ ///
+ /// ;; Instantiate our safe API implementation, passing in the lowered unsafe
+ /// ;; intrinsics as its imports.
+ /// (core instance $instance
+ /// (instantiate $safe-api-impl
+ /// (with "" (instance
+ /// (export "store-data-address" (func $store-data-address'))
+ /// (export "u64-native-load" (func $u64-native-load'))
+ /// (export "u8-native-load" (func $u8-native-load'))
+ /// (export "u8-native-store" (func $u8-native-store'))
+ /// ))
+ /// )
+ /// )
+ ///
+ /// ;; Lift the safe API's exports from core functions to component functions and
+ /// ;; export them.
+ /// (func (export "get") (param "i" u64) (result u8)
+ /// (canon lift (core func $instance "get"))
+ /// )
+ /// (func (export "set") (param "i" u64) (param "value" u8)
+ /// (canon lift (core func $instance "set"))
+ /// )
+ /// (func (export "len") (result u64)
+ /// (canon lift (core func $instance "len"))
+ /// )
+ /// )
+ ///
+ /// ;; A component that uses that safe API to increment each byte in the
+ /// ;; `StoreData` buffer.
+ /// (component $main
+ /// ;; Import the safe API.
+ /// (import "safe-api"
+ /// (instance $safe-api
+ /// (export "get" (func (param "i" u64) (result u8)))
+ /// (export "set" (func (param "i" u64) (param "value" u8)))
+ /// (export "len" (func (result u64)))
+ /// )
+ /// )
+ ///
+ /// ;; Define this component's core module implementation.
+ /// (core module $main-impl
+ /// (import "" "get" (func $get (param i64) (result i32)))
+ /// (import "" "set" (func $set (param i64 i32)))
+ /// (import "" "len" (func $len (result i64)))
+ ///
+ /// (func (export "main")
+ /// (local $i i64)
+ /// (local $n i64)
+ ///
+ /// (local.set $i (i64.const 0))
+ /// (local.set $n (call $len))
+ ///
+ /// (loop $loop
+ /// ;; When we have iterated over every byte in the
+ /// ;; buffer, exit.
+ /// (if (i64.ge_u (local.get $i) (local.get $n))
+ /// (then (return)))
+ ///
+ /// ;; Increment the `i`th byte in the buffer.
+ /// (call $set (local.get $i)
+ /// (i32.add (call $get (local.get $i))
+ /// (i32.const 1)))
+ ///
+ /// ;; Increment `i` and continue to the next iteration
+ /// ;; of the loop.
+ /// (local.set $i (i64.add (local.get $i) (i64.const 1)))
+ /// (br $loop)
+ /// )
+ /// )
+ /// )
+ ///
+ /// ;; Lower the imported safe APIs from component functions to core functions.
+ /// (core func $get' (canon lower (func $safe-api "get")))
+ /// (core func $set' (canon lower (func $safe-api "set")))
+ /// (core func $len' (canon lower (func $safe-api "len")))
+ ///
+ /// ;; Instantiate our module, providing the lowered safe APIs as imports.
+ /// (core instance $instance
+ /// (instantiate $main-impl
+ /// (with "" (instance
+ /// (export "get" (func $get'))
+ /// (export "set" (func $set'))
+ /// (export "len" (func $len'))
+ /// ))
+ /// )
+ /// )
+ ///
+ /// ;; Lift implementation's `main` from a core function to a component function
+ /// ;; and export it.
+ /// (func (export "main")
+ /// (canon lift (core func $instance "main"))
+ /// )
+ /// )
+ ///
+ /// ;; Instantiate our safe API component and our main component that consumes
+ /// ;; it, passing the unsafe intrinsics into the safe API, and the safe API
+ /// ;; into the main component.
+ /// (instance $safe-api-instance
+ /// (instantiate $safe-api (with "unsafe-intrinsics" (instance $intrinsics))))
+ /// (instance $main-instance
+ /// (instantiate $main (with "safe-api" (instance $safe-api-instance))))
+ ///
+ /// ;; Finally, re-export the `main` function!
+ /// (export "main" (func $main-instance "main"))
+ /// )
+ /// "#.as_bytes(), None)?;
+ ///
+ /// // Finish the builder and compile the component.
+ /// let component = builder.compile_component()?;
+ ///
+ /// // Create a new `Store<StoreData>`, wrapping a buffer of the given elements.
+ /// let mut store = Store::new(&engine, StoreData::new([0, 10, 20, 30, 40, 50]));
+ ///
+ /// // Instantiate our component into the store.
+ /// let instance = linker.instantiate(&mut store, &component)?;
+ ///
+ /// // Get the instance's exported `main` function and call it.
+ /// instance
+ /// .get_typed_func::<(), ()>(&mut store, "main")?
+ /// .call(&mut store, ())?;
+ ///
+ /// // Our `StoreData`'s buffer had each element incremented directly from Wasm!
+ /// assert_eq!(store.data().buf(), &[1, 11, 21, 31, 41, 51]);
+ /// # Ok(())
+ /// # }
+ /// ```
+ pub unsafe fn expose_unsafe_intrinsics(&mut self, import_name: impl Into<String>) -> &mut Self {
+ self.unsafe_intrinsics_import = Some(import_name.into());
+ self
+ }
+
/// Explicitly specify DWARF `.dwp` path.
///
/// # Errors
@@ -274,7 +742,12 @@ impl<'a> CodeBuilder<'a> {
pub fn compile_module_serialized(&self) -> Result<Vec<u8>> {
let wasm = self.get_wasm()?;
let dwarf_package = self.get_dwarf_package();
- let (v, _) = super::build_artifacts(self.engine, &wasm, dwarf_package.as_deref(), &())?;
+ ensure!(
+ self.unsafe_intrinsics_import.is_none(),
+ "`CodeBuilder::expose_unsafe_intrinsics` can only be used with components"
+ );
+ let (v, _) =
+ super::build_module_artifacts(self.engine, &wasm, dwarf_package.as_deref(), &())?;
Ok(v)
}
@@ -284,9 +757,19 @@ impl<'a> CodeBuilder<'a> {
#[cfg(feature = "component-model")]
pub fn compile_component_serialized(&self) -> Result<Vec<u8>> {
let bytes = self.get_wasm()?;
- let (v, _) = super::build_component_artifacts(self.engine, &bytes, None, &())?;
+ let (v, _) = super::build_component_artifacts(
+ self.engine,
+ &bytes,
+ None,
+ self.get_unsafe_intrinsics_import(),
+ &(),
+ )?;
Ok(v)
}
+
+ pub(super) fn get_unsafe_intrinsics_import(&self) -> Option<&str> {
+ self.unsafe_intrinsics_import.as_deref()
+ }
}
/// This is a helper struct used when caching to hash the state of an `Engine`
diff --git a/crates/wasmtime/src/compile/runtime.rs b/crates/wasmtime/src/compile/runtime.rs
index 243f85a9c1..794a5029a4 100644
--- a/crates/wasmtime/src/compile/runtime.rs
+++ b/crates/wasmtime/src/compile/runtime.rs
@@ -14,12 +14,14 @@ impl<'a> CodeBuilder<'a> {
&Engine,
&[u8],
Option<&[u8]>,
+ Option<&str>,
&S,
) -> Result<(MmapVecWrapper, Option<T>)>,
state: &S,
) -> Result<(Arc<CodeMemory>, Option<T>)> {
let wasm = self.get_wasm()?;
let dwarf_package = self.get_dwarf_package();
+ let unsafe_intrinsics_import = self.get_unsafe_intrinsics_import();
self.engine
.check_compatible_with_native_host()
@@ -31,6 +33,7 @@ impl<'a> CodeBuilder<'a> {
crate::compile::HashedEngineCompileEnv(self.engine),
&wasm,
&dwarf_package,
+ &unsafe_intrinsics_import,
// Don't hash this as it's just its own "pure" function pointer.
NotHashed(build_artifacts),
// Don't hash the FinishedObject state: this contains
@@ -43,22 +46,31 @@ impl<'a> CodeBuilder<'a> {
.get_data_raw(
&state,
// Cache miss, compute the actual artifacts
- |(engine, wasm, dwarf_package, build_artifacts, state)| -> Result<_> {
+ |(
+ engine,
+ wasm,
+ dwarf_package,
+ unsafe_intrinsics_import,
+ build_artifacts,
+ state,
+ )|
+ -> Result<_> {
let (mmap, info) = (build_artifacts.0)(
engine.0,
wasm,
dwarf_package.as_deref(),
+ **unsafe_intrinsics_import,
state.0,
)?;
let code = publish_mmap(engine.0, mmap.0)?;
Ok((code, info))
},
// Implementation of how to serialize artifacts
- |(_engine, _wasm, _, _, _), (code, _info_and_types)| {
+ |(_engine, _wasm, _, _, _, _), (code, _info_and_types)| {
Some(code.mmap().to_vec())
},
// Cache hit, deserialize the provided artifacts
- |(engine, wasm, _, _, _), serialized_bytes| {
+ |(engine, wasm, _, _, _, _), serialized_bytes| {
let kind = if wasmparser::Parser::is_component(&wasm) {
wasmtime_environ::ObjectKind::Component
} else {
@@ -79,8 +91,13 @@ impl<'a> CodeBuilder<'a> {
#[cfg(not(feature = "cache"))]
{
- let (mmap, info_and_types) =
- build_artifacts(self.engine, &wasm, dwarf_package.as_deref(), state)?;
+ let (mmap, info_and_types) = build_artifacts(
+ self.engine,
+ &wasm,
+ dwarf_package.as_deref(),
+ unsafe_intrinsics_import,
+ state,
+ )?;
let code = publish_mmap(self.engine, mmap.0)?;
return Ok((code, info_and_types));
}
@@ -92,9 +109,18 @@ impl<'a> CodeBuilder<'a> {
/// Note that this method will cache compilations if the `cache` feature is
/// enabled and turned on in [`Config`](crate::Config).
pub fn compile_module(&self) -> Result<Module> {
+ ensure!(
+ self.get_unsafe_intrinsics_import().is_none(),
+ "`CodeBuilder::expose_unsafe_intrinsics` can only be used with components"
+ );
let custom_alignment = self.custom_alignment();
- let (code, info_and_types) =
- self.compile_cached(super::build_artifacts, &custom_alignment)?;
+ let (code, info_and_types) = self.compile_cached(
+ |engine, wasm, dwarf, unsafe_intrinsics_import, state| {
+ assert!(unsafe_intrinsics_import.is_none());
+ super::build_module_artifacts(engine, wasm, dwarf, state)
+ },
+ &custom_alignment,
+ )?;
Module::from_parts(self.engine, code, info_and_types)
}
@@ -103,8 +129,18 @@ impl<'a> CodeBuilder<'a> {
#[cfg(feature = "component-model")]
pub fn compile_component(&self) -> Result<Component> {
let custom_alignment = self.custom_alignment();
- let (code, artifacts) =
- self.compile_cached(super::build_component_artifacts, &custom_alignment)?;
+ let (code, artifacts) = self.compile_cached(
+ |engine, wasm, dwarf, unsafe_intrinsics_import, state| {
+ super::build_component_artifacts(
+ engine,
+ wasm,
+ dwarf,
+ unsafe_intrinsics_import,
+ state,
+ )
+ },
+ &custom_alignment,
+ )?;
Component::from_parts(self.engine, code, artifacts)
}
diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rs
index eb83c44852..2990248d9e 100644
--- a/crates/wasmtime/src/runtime/component/component.rs
+++ b/crates/wasmtime/src/runtime/component/component.rs
@@ -18,7 +18,7 @@ use std::path::Path;
use wasmtime_environ::component::{
CompiledComponentInfo, ComponentArtifacts, ComponentTypes, CoreDef, Export, ExportIndex,
GlobalInitializer, InstantiateModule, NameMapNoIntern, OptionsIndex, StaticModuleIndex,
- TrampolineIndex, TypeComponentIndex, TypeFuncIndex, VMComponentOffsets,
+ TrampolineIndex, TypeComponentIndex, TypeFuncIndex, UnsafeIntrinsic, VMComponentOffsets,
};
use wasmtime_environ::{Abi, CompiledFunctionsTable, FuncKey, TypeTrace};
use wasmtime_environ::{FunctionLoc, HostPtr, ObjectKind, PrimaryMap};
@@ -502,22 +502,46 @@ impl Component {
pub(crate) fn trampoline_ptrs(&self, index: TrampolineIndex) -> AllCallFuncPointers {
let wasm_call = self
- .inner
- .index
- .func_loc(FuncKey::ComponentTrampoline(Abi::Wasm, index))
- .unwrap();
+ .func(FuncKey::ComponentTrampoline(Abi::Wasm, index))
+ .unwrap()
+ .cast();
let array_call = self
- .inner
- .index
- .func_loc(FuncKey::ComponentTrampoline(Abi::Array, index))
- .unwrap();
+ .func(FuncKey::ComponentTrampoline(Abi::Array, index))
+ .unwrap()
+ .cast();
AllCallFuncPointers {
- wasm_call: self.func(wasm_call).cast(),
- array_call: self.func(array_call).cast(),
+ wasm_call,
+ array_call,
}
}
- fn func(&self, loc: &FunctionLoc) -> NonNull<u8> {
+ pub(crate) fn unsafe_intrinsic_ptrs(
+ &self,
+ intrinsic: UnsafeIntrinsic,
+ ) -> Option<AllCallFuncPointers> {
+ let wasm_call = self
+ .func(FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic))?
+ .cast();
+ let array_call = self
+ .func(FuncKey::UnsafeIntrinsic(Abi::Array, intrinsic))?
+ .cast();
+ Some(AllCallFuncPointers {
+ wasm_call,
+ array_call,
+ })
+ }
+
+ /// Look up a function in this component's text section by `FuncKey`.
+ fn func(&self, key: FuncKey) -> Option<NonNull<u8>> {
+ let loc = self.inner.index.func_loc(key)?;
+ Some(self.func_loc_to_pointer(loc))
+ }
+
+ /// Given a function location within this component's text section, get a
+ /// pointer to the function.
+ ///
+ /// Panics on out-of-bounds function locations.
+ fn func_loc_to_pointer(&self, loc: &FunctionLoc) -> NonNull<u8> {
let text = self.text();
let trampoline = &text[loc.start as usize..][..loc.length as usize];
NonNull::from(trampoline).cast()
@@ -553,10 +577,8 @@ impl Component {
// then this can't be called by the component, so it's ok to leave it
// blank.
let wasm_call = self
- .inner
- .index
- .func_loc(FuncKey::ResourceDropTrampoline)
- .map(|loc| self.func(loc).cast().into());
+ .func(FuncKey::ResourceDropTrampoline)
+ .map(|f| f.cast().into());
VMFuncRef {
wasm_call,
diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rs
index a7aa6f1624..4f67d39f65 100644
--- a/crates/wasmtime/src/runtime/component/instance.rs
+++ b/crates/wasmtime/src/runtime/component/instance.rs
@@ -489,6 +489,15 @@ pub(crate) fn lookup_vmdef(
let id = StoreComponentInstanceId::new(store.id(), id);
vm::Export::Global(crate::Global::from_component_flags(id, *idx))
}
+ CoreDef::UnsafeIntrinsic(intrinsic) => {
+ let funcref = store
+ .store_data_mut()
+ .component_instance_mut(id)
+ .unsafe_intrinsic_func_ref(*intrinsic);
+ // SAFETY: as above, the `funcref` is owned by `store` and is valid
+ // within that store, so it's safe to create a `Func`.
+ vm::Export::Function(unsafe { crate::Func::from_vm_func_ref(store.id(), funcref) })
+ }
}
}
@@ -678,6 +687,31 @@ impl<'a> Instantiator<'a> {
);
}
+ // Initialize the unsafe intrinsics used by this component, if any.
+ for (i, module_ty) in env_component
+ .unsafe_intrinsics
+ .iter()
+ .enumerate()
+ .filter_map(|(i, ty)| ty.expand().map(|ty| (i, ty)))
+ {
+ let i = u32::try_from(i).unwrap();
+ let intrinsic = UnsafeIntrinsic::from_u32(i);
+ let ptrs = self.component.unsafe_intrinsic_ptrs(intrinsic).expect(
+ "should have intrinsic pointers given that we assigned the intrinsic a type",
+ );
+ let shared_ty = self
+ .component
+ .signatures()
+ .shared_type(module_ty)
+ .expect("should have a shared type");
+ self.instance_mut(store.0).set_intrinsic(
+ intrinsic,
+ ptrs.wasm_call,
+ ptrs.array_call,
+ shared_ty,
+ );
+ }
+
for initializer in env_component.initializers.iter() {
match initializer {
GlobalInitializer::InstantiateModule(m) => {
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index a87a2c070c..4bde59d5b2 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -245,8 +245,20 @@ pub struct StoreInner<T: 'static> {
#[cfg(target_has_atomic = "64")]
epoch_deadline_behavior:
Option<Box<dyn FnMut(StoreContextMut<T>) -> Result<UpdateDeadline> + Send + Sync>>,
- // for comments about `ManuallyDrop`, see `Store::into_data`
- data: ManuallyDrop<T>,
+
+ /// The user's `T` data.
+ ///
+ /// Don't actually access it via this field, however! Use the
+ /// `Store{,Inner,Context,ContextMut}::data[_mut]` methods instead, to
+ /// preserve stacked borrows and provenance in the face of potential
+ /// direct-access of `T` from Wasm code (via unsafe intrinsics).
+ ///
+ /// The only exception to the above is when taking ownership of the value,
+ /// e.g. in `Store::into_data`, after which nothing can access this field
+ /// via raw pointers anymore so there is no more provenance to preserve.
+ ///
+ /// For comments about `ManuallyDrop`, see `Store::into_data`.
+ data_no_provenance: ManuallyDrop<T>,
}
enum ResourceLimiterInner<T> {
@@ -703,9 +715,13 @@ impl<T> Store<T> {
call_hook: None,
#[cfg(target_has_atomic = "64")]
epoch_deadline_behavior: None,
- data: ManuallyDrop::new(data),
+ data_no_provenance: ManuallyDrop::new(data),
});
+ let store_data =
+ <NonNull<ManuallyDrop<T>>>::from(&mut inner.data_no_provenance).cast::<()>();
+ inner.inner.vm_store_context.store_data = store_data.into();
+
inner.traitobj = StorePtr(Some(NonNull::from(&mut *inner)));
// Wasmtime uses the callee argument to host functions to learn about
@@ -748,13 +764,13 @@ impl<T> Store<T> {
}
}
- /// Access the underlying data owned by this `Store`.
+ /// Access the underlying `T` data owned by this `Store`.
#[inline]
pub fn data(&self) -> &T {
self.inner.data()
}
- /// Access the underlying data owned by this `Store`.
+ /// Access the underlying `T` data owned by this `Store`.
#[inline]
pub fn data_mut(&mut self) -> &mut T {
self.inner.data_mut()
@@ -808,7 +824,7 @@ impl<T> Store<T> {
unsafe {
let mut inner = ManuallyDrop::take(&mut self.inner);
core::mem::forget(self);
- ManuallyDrop::take(&mut inner.data)
+ ManuallyDrop::take(&mut inner.data_no_provenance)
}
}
@@ -870,7 +886,7 @@ impl<T> Store<T> {
// Apply the limits on instances, tables, and memory given by the limiter:
let inner = &mut self.inner;
let (instance_limit, table_limit, memory_limit) = {
- let l = limiter(&mut inner.data);
+ let l = limiter(inner.data_mut());
(l.instances(), l.tables(), l.memories())
};
let innermost = &mut inner.inner;
@@ -1315,12 +1331,49 @@ impl<'a, T> StoreContextMut<'a, T> {
impl<T> StoreInner<T> {
#[inline]
fn data(&self) -> &T {
- &self.data
+ // We are actually just accessing `&self.data_no_provenance` but we must
+ // do so with the `VMStoreContext::store_data` pointer's provenance. If
+ // we did otherwise, i.e. directly accessed the field, we would
+ // invalidate that pointer, which would in turn invalidate any direct
+ // `T` accesses that Wasm code makes via unsafe intrinsics.
+ let data: *const ManuallyDrop<T> = &raw const self.data_no_provenance;
+ let provenance = self.inner.vm_store_context.store_data.as_ptr().cast::<T>();
+ let ptr = provenance.with_addr(data.addr());
+
+ // SAFETY: The pointer is non-null, points to our `T` data, and is valid
+ // to access because of our `&self` borrow.
+ debug_assert_ne!(ptr, core::ptr::null_mut());
+ debug_assert_eq!(ptr.addr(), (&raw const self.data_no_provenance).addr());
+ unsafe { &*ptr }
+ }
+
+ #[inline]
+ fn data_limiter_and_opaque(
+ &mut self,
+ ) -> (
+ &mut T,
+ Option<&mut ResourceLimiterInner<T>>,
+ &mut StoreOpaque,
+ ) {
+ // See the comments about provenance in `StoreInner::data` above.
+ let data: *mut ManuallyDrop<T> = &raw mut self.data_no_provenance;
+ let provenance = self.inner.vm_store_context.store_data.as_ptr().cast::<T>();
+ let ptr = provenance.with_addr(data.addr());
+
+ // SAFETY: The pointer is non-null, points to our `T` data, and is valid
+ // to access because of our `&mut self` borrow.
+ debug_assert_ne!(ptr, core::ptr::null_mut());
+ debug_assert_eq!(ptr.addr(), (&raw const self.data_no_provenance).addr());
+ let data = unsafe { &mut *ptr };
+
+ let limiter = self.limiter.as_mut();
+
+ (data, limiter, &mut self.inner)
}
#[inline]
fn data_mut(&mut self) -> &mut T {
- &mut self.data
+ self.data_limiter_and_opaque().0
}
#[inline]
@@ -2562,14 +2615,15 @@ unsafe impl<T> VMStore for StoreInner<T> {
fn resource_limiter_and_store_opaque(
&mut self,
) -> (Option<StoreResourceLimiter<'_>>, &mut StoreOpaque) {
- (
- self.limiter.as_mut().map(|l| match l {
- ResourceLimiterInner::Sync(s) => StoreResourceLimiter::Sync(s(&mut self.data)),
- #[cfg(feature = "async")]
- ResourceLimiterInner::Async(s) => StoreResourceLimiter::Async(s(&mut self.data)),
- }),
- &mut self.inner,
- )
+ let (data, limiter, opaque) = self.data_limiter_and_opaque();
+
+ let limiter = limiter.map(|l| match l {
+ ResourceLimiterInner::Sync(s) => StoreResourceLimiter::Sync(s(data)),
+ #[cfg(feature = "async")]
+ ResourceLimiterInner::Async(s) => StoreResourceLimiter::Async(s(data)),
+ });
+
+ (limiter, opaque)
}
#[cfg(target_has_atomic = "64")]
@@ -2619,7 +2673,7 @@ impl<T: fmt::Debug> fmt::Debug for Store<T> {
let inner = &**self.inner as *const StoreInner<T>;
f.debug_struct("Store")
.field("inner", &inner)
- .field("data", &self.inner.data)
+ .field("data", self.inner.data())
.finish()
}
}
@@ -2628,9 +2682,9 @@ impl<T> Drop for Store<T> {
fn drop(&mut self) {
self.run_manual_drop_routines();
- // for documentation on this `unsafe`, see `into_data`.
+ // For documentation on this `unsafe`, see `into_data`.
unsafe {
- ManuallyDrop::drop(&mut self.inner.data);
+ ManuallyDrop::drop(&mut self.inner.data_no_provenance);
ManuallyDrop::drop(&mut self.inner);
}
}
@@ -2717,8 +2771,7 @@ impl<T: AsStoreOpaque + ?Sized> AsStoreOpaque for &mut T {
#[cfg(test)]
mod tests {
- use super::{get_fuel, refuel, set_fuel};
- use std::num::NonZeroU64;
+ use super::*;
struct FuelTank {
pub consumed_fuel: i64,
@@ -2835,4 +2888,35 @@ mod tests {
assert_eq!(tank.consumed_fuel, 4);
assert_eq!(tank.get_fuel(), 0);
}
+
+ #[test]
+ fn store_data_provenance() {
+ // Test that we juggle pointer provenance and all that correctly, and
+ // miri is happy with everything, while allowing both Rust code and
+ // "Wasm" to access and modify the store's `T` data. Note that this is
+ // not actually Wasm mutating the store data here because compiling Wasm
+ // under miri is way too slow.
+
+ unsafe fn run_wasm(store: &mut Store<u32>) {
+ let ptr = store
+ .inner
+ .inner
+ .vm_store_context
+ .store_data
+ .as_ptr()
+ .cast::<u32>();
+ unsafe { *ptr += 1 }
+ }
+
+ let engine = Engine::default();
+ let mut store = Store::new(&engine, 0_u32);
+
+ assert_eq!(*store.data(), 0);
+ *store.data_mut() += 1;
+ assert_eq!(*store.data(), 1);
+ unsafe { run_wasm(&mut store) }
+ assert_eq!(*store.data(), 2);
+ *store.data_mut() += 1;
+ assert_eq!(*store.data(), 3);
+ }
}
diff --git a/crates/wasmtime/src/runtime/store/async_.rs b/crates/wasmtime/src/runtime/store/async_.rs
index cefc0e192a..93e3617501 100644
--- a/crates/wasmtime/src/runtime/store/async_.rs
+++ b/crates/wasmtime/src/runtime/store/async_.rs
@@ -51,7 +51,7 @@ impl<T> Store<T> {
// Apply the limits on instances, tables, and memory given by the limiter:
let inner = &mut self.inner;
let (instance_limit, table_limit, memory_limit) = {
- let l = limiter(&mut inner.data);
+ let l = limiter(inner.data_mut());
(l.instances(), l.tables(), l.memories())
};
let innermost = &mut inner.inner;
diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs
index 1e680cd745..079db57bd9 100644
--- a/crates/wasmtime/src/runtime/vm/component.rs
+++ b/crates/wasmtime/src/runtime/vm/component.rs
@@ -428,6 +428,20 @@ impl ComponentInstance {
}
}
+ /// Get the core Wasm function reference for the given unsafe intrinsic.
+ pub fn unsafe_intrinsic_func_ref(&self, idx: UnsafeIntrinsic) -> NonNull<VMFuncRef> {
+ unsafe {
+ let offset = self.offsets.unsafe_intrinsic_func_ref(idx);
+ let ret = self.vmctx_plus_offset_raw::<VMFuncRef>(offset);
+ debug_assert!(
+ mem::transmute::<Option<VmPtr<VMWasmCallFunction>>, usize>(ret.as_ref().wasm_call)
+ != INVALID_PTR
+ );
+ debug_assert!(ret.as_ref().vmctx.as_ptr() as usize != INVALID_PTR);
+ ret
+ }
+ }
+
/// Stores the runtime memory pointer at the index specified.
///
/// This is intended to be called during the instantiation process of a
@@ -543,6 +557,27 @@ impl ComponentInstance {
}
}
+ /// Same as `set_trampoline` but for intrinsic functions.
+ pub fn set_intrinsic(
+ self: Pin<&mut Self>,
+ intrinsic: UnsafeIntrinsic,
+ wasm_call: NonNull<VMWasmCallFunction>,
+ array_call: NonNull<VMArrayCallFunction>,
+ type_index: VMSharedTypeIndex,
+ ) {
+ unsafe {
+ let offset = self.offsets.unsafe_intrinsic_func_ref(intrinsic);
+ debug_assert!(*self.vmctx_plus_offset::<usize>(offset) == INVALID_PTR);
+ let vmctx = VMOpaqueContext::from_vmcomponent(self.vmctx());
+ *self.vmctx_plus_offset_mut(offset) = VMFuncRef {
+ wasm_call: Some(wasm_call.into()),
+ array_call: array_call.into(),
+ type_index,
+ vmctx: vmctx.into(),
+ };
+ }
+ }
+
/// Configures the destructor for a resource at the `idx` specified.
///
/// This is required to be called for each resource as it's defined within a
@@ -638,6 +673,14 @@ impl ComponentInstance {
*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
}
}
+ for i in 0..self.offsets.num_unsafe_intrinsics {
+ let i = UnsafeIntrinsic::from_u32(i);
+ let offset = self.offsets.unsafe_intrinsic_func_ref(i);
+ // SAFETY: see above
+ unsafe {
+ *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;
+ }
+ }
for i in 0..self.offsets.num_runtime_memories {
let i = RuntimeMemoryIndex::from_u32(i);
let offset = self.offsets.runtime_memory(i);
diff --git a/crates/wasmtime/src/runtime/vm/provenance.rs b/crates/wasmtime/src/runtime/vm/provenance.rs
index c10136c757..ea0ca41533 100644
--- a/crates/wasmtime/src/runtime/vm/provenance.rs
+++ b/crates/wasmtime/src/runtime/vm/provenance.rs
@@ -115,6 +115,11 @@ impl<T> VmPtr<T> {
pub fn as_ptr(&self) -> *mut T {
self.as_non_null().as_ptr()
}
+
+ /// Similar to `{NonNull,core::ptr}::dangling()` but for `VmPtr`.
+ pub fn dangling() -> Self {
+ NonNull::dangling().into()
+ }
}
// `VmPtr<T>`, like raw pointers, is trivially `Clone`/`Copy`.
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 24dc0bfd57..ef714422d6 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -1173,6 +1173,10 @@ pub struct VMStoreContext {
/// on `VMStackChain` for details.
pub stack_chain: UnsafeCell<VMStackChain>,
+ /// A pointer to the embedder's `T` inside a `Store<T>`, for use with the
+ /// `store-data-address` unsafe intrinsic.
+ pub store_data: VmPtr<()>,
+
/// The range, in addresses, of the guard page that is currently in use.
///
/// This field is used when signal handlers are run to determine whether a
@@ -1266,6 +1270,7 @@ impl Default for VMStoreContext {
last_wasm_entry_trap_handler: UnsafeCell::new(0),
stack_chain: UnsafeCell::new(VMStackChain::Absent),
async_guard_range: ptr::null_mut()..ptr::null_mut(),
+ store_data: VmPtr::dangling(),
}
}
}
@@ -1327,7 +1332,11 @@ mod test_vmstore_context {
assert_eq!(
offset_of!(VMStoreContext, stack_chain),
usize::from(offsets.ptr.vmstore_context_stack_chain())
- )
+ );
+ assert_eq!(
+ offset_of!(VMStoreContext, store_data),
+ usize::from(offsets.ptr.vmstore_context_store_data())
+ );
}
}
diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rs
index 13a5e80d98..77f76dd742 100644
--- a/crates/winch/src/compiler.rs
+++ b/crates/winch/src/compiler.rs
@@ -6,6 +6,8 @@ use std::mem;
use std::sync::Mutex;
use wasmparser::FuncValidatorAllocations;
use wasmtime_cranelift::CompiledFunction;
+#[cfg(feature = "component-model")]
+use wasmtime_environ::component::ComponentTranslation;
use wasmtime_environ::{
CompileError, CompiledFunctionBody, DefinedFuncIndex, FuncKey, FunctionBodyData, FunctionLoc,
ModuleTranslation, ModuleTypesBuilder, PrimaryMap, StaticModuleIndex, Tunables, VMOffsets,
@@ -398,4 +400,23 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {
}
Ok(body)
}
+
+ fn compile_intrinsic(
+ &self,
+ tunables: &Tunables,
+ component: &ComponentTranslation,
+ intrinsic: wasmtime_environ::component::UnsafeIntrinsic,
+ abi: wasmtime_environ::Abi,
+ symbol: &str,
+ ) -> Result<CompiledFunctionBody> {
+ let mut body = self
+ .0
+ .component_compiler()
+ .compile_intrinsic(tunables, component, intrinsic, abi, symbol)?;
+ if let Some(c) = self.0.inlining_compiler() {
+ c.finish_compiling(&mut body, None, symbol)
+ .map_err(|e| CompileError::Codegen(e.to_string()))?;
+ }
+ Ok(body)
+ }
}
diff --git a/tests/all/intrinsics.rs b/tests/all/intrinsics.rs
new file mode 100644
index 0000000000..0c67749edb
--- /dev/null
+++ b/tests/all/intrinsics.rs
@@ -0,0 +1,643 @@
+//! Tests related to the unsafe Wasmtime intrinsics we can give to components
+//! via `CodeBuilder::expose_unsafe_intrinsics`.
+
+use super::*;
+use std::{cell::UnsafeCell, path::Path, sync::Arc};
+use wasmtime::*;
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn native_loads_and_stores() -> Result<()> {
+ let engine = Engine::default();
+
+ for (comp_ty, core_ty) in [
+ ("u8", "i32"),
+ ("u16", "i32"),
+ ("u32", "i32"),
+ ("u64", "i64"),
+ ] {
+ let mut code_builder = CodeBuilder::new(&engine);
+ let path = format!("intrinsics::native_loads_and_stores::{comp_ty}");
+ let path = Path::new(&path);
+ let wat = format!(
+ r#"
+ (component
+ (import "host" (instance $host (export "get-pointer" (func (result u64)))))
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "{comp_ty}-native-load" (func (param "pointer" u64) (result {comp_ty})))
+ (export "{comp_ty}-native-store" (func (param "pointer" u64) (param "value" {comp_ty})))
+ )
+ )
+
+ (core func $get-pointer' (canon lower (func $host "get-pointer")))
+ (core func $load' (canon lower (func $intrinsics "{comp_ty}-native-load")))
+ (core func $store' (canon lower (func $intrinsics "{comp_ty}-native-store")))
+
+ (core module $m
+ (import "" "get-pointer" (func $get-pointer (result i64)))
+ (import "" "load" (func $load (param i64) (result {core_ty})))
+ (import "" "store" (func $store (param i64 {core_ty})))
+ (func (export "run")
+ (local $pointer i64)
+
+ ;; Get a native pointer from the host.
+ (local.set $pointer (call $get-pointer))
+
+ ;; Assert that loading from the pointer results in `42`.
+ (if ({core_ty}.ne (call $load (local.get $pointer))
+ ({core_ty}.const 42))
+ (then (unreachable)))
+
+ ;; Store `-1` through the pointer.
+ (call $store (local.get $pointer) ({core_ty}.const -1))
+ )
+ )
+
+ (core instance $i
+ (instantiate $m
+ (with "" (instance (export "get-pointer" (func $get-pointer'))
+ (export "load" (func $load'))
+ (export "store" (func $store'))))
+ )
+ )
+
+ (func (export "run") (canon lift (core func $i "run")))
+ )
+ "#
+ );
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let component = code_builder.compile_component()?;
+
+ // Data that will be manipulated directly by Wasm via intrinsics.
+ let mut data = Arc::new(UnsafeCell::new(0x6666666666666666_u64));
+
+ // Write `42` into `data` at the appropriate place for this type.
+ let ptr = data.get();
+ unsafe {
+ match comp_ty {
+ "u8" => ptr.cast::<u8>().write(42),
+ "u16" => ptr.cast::<u16>().write(42),
+ "u32" => ptr.cast::<u32>().write(42),
+ "u64" => ptr.cast::<u64>().write(42),
+ _ => unreachable!(),
+ }
+ }
+
+ // Create a linker and define `host::get-pointer` to return a pointer to
+ // `data`.
+ let mut linker = component::Linker::new(&engine);
+ linker.instance("host")?.func_new("get-pointer", {
+ let ptr = ptr as usize;
+ let ptr = u64::try_from(ptr).unwrap();
+ move |_cx, _args, results| {
+ results[0] = component::Val::U64(ptr);
+ Ok(())
+ }
+ })?;
+
+ // Instantiate the component and call its `run` function.
+ let mut store = Store::new(&engine, ());
+ let instance = linker.instantiate(&mut store, &component)?;
+ let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;
+ run.call(&mut store, ())?;
+
+ // The `run` function should have written `-1` into its view of `data`
+ // and should not have modified any other part of it.
+ drop(linker);
+ let actual = Arc::get_mut(&mut data).unwrap().get_mut().to_ne_bytes();
+ let expected = match comp_ty {
+ "u8" => [0xFF, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66],
+ "u16" => [0xFF, 0xFF, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66],
+ "u32" => [0xFF, 0xFF, 0xFF, 0xFF, 0x66, 0x66, 0x66, 0x66],
+ "u64" => [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF],
+ _ => unreachable!(),
+ };
+ assert_eq!(
+ expected,
+ actual,
+ "expected != actual\n\
+ \texpected: {}\n\
+ \t actual: {}",
+ expected
+ .iter()
+ .map(|b| format!("{b:02x}"))
+ .collect::<Vec<_>>()
+ .join(" "),
+ actual
+ .iter()
+ .map(|b| format!("{b:02x}"))
+ .collect::<Vec<_>>()
+ .join(" "),
+ );
+ }
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn cannot_enable_unsafe_intrinsics_for_core_module() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+ code_builder.wasm_binary_or_text("(module)".as_bytes(), None)?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let err = code_builder.compile_module().unwrap_err();
+ let err = format!("{err:?}");
+ assert!(
+ err.contains("`CodeBuilder::expose_unsafe_intrinsics` can only be used with components"),
+ "unexpected error: {err}"
+ );
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn ref_func_of_intrinsic() -> Result<()> {
+ let mut config = Config::new();
+ config.wasm_function_references(true);
+ config.wasm_gc(true);
+ let engine = Engine::new(&config)?;
+
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u32-native-load" (func (param "pointer" u64) (result u32)))
+ )
+ )
+
+ (core func $load' (canon lower (func $intrinsics "u32-native-load")))
+
+ (core module $m
+ (type $ty (func (param i64) (result i32)))
+ (import "" "load" (func $load (type $ty)))
+ (elem declare func $load)
+ (func (export "run") (param $pointer i64) (result i32)
+ (local $f (ref null func))
+ (local.set $f (ref.func $load))
+ (call_ref $ty (local.get $pointer) (ref.cast (ref $ty) (local.get $f)))
+ )
+ )
+
+ (core instance $i
+ (instantiate $m
+ (with "" (instance (export "load" (func $load'))))
+ )
+ )
+
+ (func (export "run") (param "pointer" u64) (result u32)
+ (canon lift (core func $i "run"))
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let component = code_builder.compile_component()?;
+
+ // Instantiate the component and call its `run` function.
+ let linker = component::Linker::new(&engine);
+ let mut store = Store::new(&engine, ());
+ let instance = linker.instantiate(&mut store, &component)?;
+ let run = instance.get_typed_func::<(u64,), (u32,)>(&mut store, "run")?;
+
+ let data = 0x12345678_u32;
+ let (result,) = run.call(&mut store, (&data as *const _ as usize as u64,))?;
+ assert_eq!(result, data);
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn table_element_segment_with_intrinsic() -> Result<()> {
+ let engine = Engine::default();
+
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u32-native-load" (func (param "pointer" u64) (result u32)))
+ )
+ )
+
+ (core func $load' (canon lower (func $intrinsics "u32-native-load")))
+
+ (core module $m
+ (type $ty (func (param i64) (result i32)))
+ (import "" "load" (func $load (type $ty)))
+ (table $t 1 1 funcref)
+ (elem (table $t) (i32.const 0) func $load)
+ (func (export "run") (param $pointer i64) (result i32)
+ (call_indirect (type $ty) (local.get $pointer) (i32.const 0))
+ )
+ )
+
+ (core instance $i
+ (instantiate $m
+ (with "" (instance (export "load" (func $load'))))
+ )
+ )
+
+ (func (export "run") (param "pointer" u64) (result u32)
+ (canon lift (core func $i "run"))
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let component = code_builder.compile_component()?;
+
+ // Instantiate the component and call its `run` function.
+ let linker = component::Linker::new(&engine);
+ let mut store = Store::new(&engine, ());
+ let instance = linker.instantiate(&mut store, &component)?;
+ let run = instance.get_typed_func::<(u64,), (u32,)>(&mut store, "run")?;
+
+ let data = 0x12345678_u32;
+ let (result,) = run.call(&mut store, (&data as *const _ as usize as u64,))?;
+ assert_eq!(result, data);
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn intrinsics_not_listed_in_imports() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u32-native-load" (func (param "pointer" u64) (result u32)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let component = code_builder.compile_component()?;
+
+ let component_type = component.component_type();
+ let imports = component_type.imports(&engine);
+ assert_eq!(imports.count(), 0);
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn unknown_intrinsic_function() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "unknown" (func (param "pointer" u64) (result u32)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();
+ let err = format!("{err:?}");
+ assert!(
+ err.contains("invalid unsafe intrinsic: \"unknown\""),
+ "unexpected error: {err}"
+ );
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn bad_extra_param() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u8-native-load" (func (param "pointer" u64) (param "extra" u64) (result u8)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();
+ let err = format!("{err:?}");
+ assert!(
+ err.contains(
+ "bad unsafe intrinsics import at `unsafe-intrinsics`: function `u8-native-load` \
+ must have 1 parameters, found 2"
+ ),
+ "unexpected error: {err}"
+ );
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn bad_missing_param() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u8-native-load" (func (result u8)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();
+ let err = format!("{err:?}");
+ assert!(
+ err.contains(
+ "bad unsafe intrinsics import at `unsafe-intrinsics`: function `u8-native-load` \
+ must have 1 parameters, found 0"
+ ),
+ "unexpected error: {err}"
+ );
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn bad_missing_return() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u8-native-load" (func (param "pointer" u64)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();
+ let err = format!("{err:?}");
+ assert!(
+ err.contains(
+ "bad unsafe intrinsics import at `unsafe-intrinsics`: function `u8-native-load` must \
+ have 1 results, found 0"
+ ),
+ "unexpected error: {err}"
+ );
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn bad_extra_return() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u8-native-store" (func (param "pointer" u64) (param "value" u8) (result u32)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();
+ let err = format!("{err:?}");
+ assert!(
+ err.contains(
+ "bad unsafe intrinsics import at `unsafe-intrinsics`: function `u8-native-store` \
+ must have 0 results, found 1"
+ ),
+ "unexpected error: {err}"
+ );
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn bad_param_type() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u8-native-store" (func (param "pointer" u64) (param "value" u16)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();
+ let err = format!("{err:?}");
+ assert!(
+ err.contains(
+ "bad unsafe intrinsics import at `unsafe-intrinsics`: parameters[1] for function \
+ `u8-native-store` must be `U8`, found `Primitive(U16)`"
+ ),
+ "unexpected error: {err}"
+ );
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn bad_result_type() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let path = format!("intrinsics::ref_func_of_intrinsic");
+ let path = Path::new(&path);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "u8-native-load" (func (param "pointer" u64) (result u16)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();
+ let err = format!("{err:?}");
+ assert!(
+ err.contains(
+ "bad unsafe intrinsics import at `unsafe-intrinsics`: results[0] for function \
+ `u8-native-load` must be `U8`, found `Primitive(U16)`"
+ ),
+ "unexpected error: {err}"
+ );
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn store_data_address() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let wat = r#"
+ (component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "store-data-address" (func (result u64)))
+ (export "u8-native-load" (func (param "pointer" u64) (result u8)))
+ (export "u8-native-store" (func (param "pointer" u64) (param "value" u8)))
+ )
+ )
+
+ (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))
+ (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))
+ (core func $u8-native-store' (canon lower (func $intrinsics "u8-native-store")))
+
+ (core module $m
+ (import "" "store-data-address" (func $store-data-address (result i64)))
+ (import "" "u8-native-load" (func $load (param i64) (result i32)))
+ (import "" "u8-native-store" (func $store (param i64 i32)))
+ (func (export "run")
+ ;; Load the store data, it should be 42.
+ (if (i32.ne (call $load (call $store-data-address))
+ (i32.const 42))
+ (then (unreachable)))
+
+ ;; Store 36 into the store data.
+ (call $store (call $store-data-address) (i32.const 36))
+ )
+ )
+
+ (core instance $i
+ (instantiate $m
+ (with "" (instance (export "store-data-address" (func $store-data-address'))
+ (export "u8-native-load" (func $u8-native-load'))
+ (export "u8-native-store" (func $u8-native-store'))))
+ )
+ )
+
+ (func (export "run")
+ (canon lift (core func $i "run"))
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), None)?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");
+ }
+ let component = code_builder.compile_component()?;
+
+ // Instantiate the component and call its `run` function.
+ let linker = component::Linker::new(&engine);
+ let mut store = Store::new(&engine, 42_u8);
+ let instance = linker.instantiate(&mut store, &component)?;
+ let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;
+
+ run.call(&mut store, ())?;
+ assert_eq!(*store.data(), 36);
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn other_import_name() -> Result<()> {
+ let engine = Engine::default();
+ let mut code_builder = CodeBuilder::new(&engine);
+
+ let wat = r#"
+ (component
+ (import "other-name"
+ (instance
+ (export "u32-native-load" (func (param "pointer" u64) (result u32)))
+ )
+ )
+ )
+ "#;
+ code_builder.wasm_binary_or_text(wat.as_bytes(), None)?;
+ unsafe {
+ code_builder.expose_unsafe_intrinsics("other-name");
+ }
+ code_builder.compile_component()?;
+
+ Ok(())
+}
diff --git a/tests/all/main.rs b/tests/all/main.rs
index 5e3568fda8..0c6375b971 100644
--- a/tests/all/main.rs
+++ b/tests/all/main.rs
@@ -27,6 +27,7 @@ mod iloop;
mod import_calling_export;
mod import_indexes;
mod instance;
+mod intrinsics;
mod invoke_func_via_table;
mod limits;
mod linker;
diff --git a/tests/disas.rs b/tests/disas.rs
index 8c75058a1d..7b0a0956db 100644
--- a/tests/disas.rs
+++ b/tests/disas.rs
@@ -50,7 +50,7 @@ use std::io::Write as _;
use std::path::{Path, PathBuf};
use std::process::Stdio;
use tempfile::TempDir;
-use wasmtime::{Engine, OptLevel, Strategy};
+use wasmtime::{CodeBuilder, CodeHint, Engine, OptLevel, Strategy};
use wasmtime_cli_flags::CommonOptions;
fn main() -> Result<()> {
@@ -124,6 +124,7 @@ struct TestConfig {
flags: Option<TestConfigFlags>,
objdump: Option<TestConfigFlags>,
filter: Option<String>,
+ unsafe_intrinsics: Option<String>,
}
#[derive(Debug, Deserialize)]
@@ -208,15 +209,26 @@ impl Test {
}
}
let engine = Engine::new(&config).context("failed to create engine")?;
- let wasm = wat::parse_file(&self.path)?;
- let elf = if wasmparser::Parser::is_component(&wasm) {
- engine
- .precompile_component(&wasm)
- .context("failed to compile component")?
- } else {
- engine
- .precompile_module(&wasm)
- .context("failed to compile module")?
+
+ let mut builder = CodeBuilder::new(&engine);
+ builder.wasm_binary_or_text_file(&self.path)?;
+ if let Some(name) = self.config.unsafe_intrinsics.as_deref() {
+ unsafe {
+ builder.expose_unsafe_intrinsics(name);
+ }
+ }
+
+ let elf = match builder.hint() {
+ Some(CodeHint::Component) => builder
+ .compile_component_serialized()
+ .context("failed to compile component")?,
+ Some(CodeHint::Module) => builder
+ .compile_module_serialized()
+ .context("failed to compile module")?,
+ None => bail!(
+ "contents of `{}` do not look like a Wasm component or module",
+ self.path.display()
+ ),
};
match self.config.test {
@@ -253,7 +265,10 @@ impl Test {
let functions = clifs
.iter()
.map(|clif| {
- let mut funcs = cranelift_reader::parse_functions(clif)?;
+ let mut funcs =
+ cranelift_reader::parse_functions(clif).with_context(|| {
+ format!("failed to parse CLIF:\n\"\"\"\n{clif}\n\"\"\"")
+ })?;
if funcs.len() != 1 {
bail!("expected one function per clif");
}
diff --git a/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat b/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat
new file mode 100644
index 0000000000..128e913579
--- /dev/null
+++ b/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat
@@ -0,0 +1,94 @@
+;;! target = "x86_64"
+;;! test = "optimize"
+;;! filter = "wasm[0]--function"
+;;! flags = "-C inlining=y"
+;;! unsafe_intrinsics = "unsafe-intrinsics"
+
+(component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "store-data-address" (func (result u64)))
+ (export "u8-native-load" (func (param "pointer" u64) (result u8)))
+ (export "u8-native-store" (func (param "pointer" u64) (param "value" u8)))
+ )
+ )
+
+ (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))
+ (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))
+ (core func $u8-native-store' (canon lower (func $intrinsics "u8-native-store")))
+
+ (core module $m
+ (import "" "store-data-address" (func $store-data-address (result i64)))
+ (import "" "u8-native-load" (func $load (param i64) (result i32)))
+ (import "" "u8-native-store" (func $store (param i64 i32)))
+ (func (export "f")
+ (local $x i32)
+ (local.set $x (call $load (call $store-data-address)))
+ (call $store (call $store-data-address)
+ (i32.add (local.get $x) (i32.const 1)))
+ )
+ )
+
+ (core instance $i
+ (instantiate $m
+ (with "" (instance (export "store-data-address" (func $store-data-address'))
+ (export "u8-native-load" (func $u8-native-load'))
+ (export "u8-native-store" (func $u8-native-store'))))
+ )
+ )
+
+ (func (export "f")
+ (canon lift (core func $i "f"))
+ )
+)
+
+;; function u0:0(i64 vmctx, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+16
+;; gv3 = vmctx
+;; sig0 = (i64 vmctx, i64) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i64) -> i32 tail
+;; sig2 = (i64 vmctx, i64, i64, i32) tail
+;; fn0 = colocated u3758096384:0 sig0
+;; fn1 = colocated u3758096384:1 sig1
+;; fn2 = colocated u3758096384:2 sig2
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64):
+;; @0153 jump block2
+;;
+;; block2:
+;; jump block3
+;;
+;; block3:
+;; @0155 jump block4
+;;
+;; block4:
+;; @0153 v4 = load.i64 notrap aligned readonly can_move v0+64
+;; v17 = load.i64 notrap aligned readonly can_move vmctx v4+16
+;; v18 = load.i64 notrap aligned readonly can_move vmctx v17+96
+;; v20 = load.i8 notrap aligned v18
+;; jump block5
+;;
+;; block5:
+;; @0159 jump block6
+;;
+;; block6:
+;; jump block7
+;;
+;; block7:
+;; @0160 jump block8
+;;
+;; block8:
+;; v28 = iconst.i8 1
+;; v29 = iadd.i8 v20, v28 ; v28 = 1
+;; store notrap aligned v29, v18
+;; jump block9
+;;
+;; block9:
+;; @0162 jump block1
+;;
+;; block1:
+;; @0162 return
+;; }
diff --git a/tests/disas/component-model/unsafe-intrinsics-used.wat b/tests/disas/component-model/unsafe-intrinsics-used.wat
new file mode 100644
index 0000000000..07a961cd09
--- /dev/null
+++ b/tests/disas/component-model/unsafe-intrinsics-used.wat
@@ -0,0 +1,49 @@
+;;! target = "x86_64"
+;;! test = "optimize"
+;;! filter = "unsafe-intrinsics-wasm-call"
+;;! unsafe_intrinsics = "unsafe-intrinsics"
+
+;; Test that only the intrinsics that we actually use get compiled.
+
+(component
+ (import "unsafe-intrinsics"
+ (instance $intrinsics
+ (export "store-data-address" (func (result u64)))
+ (export "u8-native-load" (func (param "pointer" u64) (result u8)))
+ )
+ )
+
+ (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))
+ (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))
+
+ (core module $m
+ (import "" "store-data-address" (func $store-data-address (result i64)))
+ (import "" "u8-native-load" (func $load (param i64) (result i32)))
+
+ ;; XXX: if we ever implement gc-sections/DCE during our linking, we will
+ ;; need to update this to `ref.func` the imported functions or whatever
+ ;; to ensure that they are rooted in that analysis and we do end up with
+ ;; non-inlined function bodies for the intrinsics.
+ )
+
+ (core instance $i
+ (instantiate $m
+ (with "" (instance (export "store-data-address" (func $store-data-address'))
+ (export "u8-native-load" (func $u8-native-load'))))
+ )
+ )
+)
+
+;; function u0:0(i64 vmctx, i64) -> i64 tail {
+;; block0(v0: i64, v1: i64):
+;; v2 = load.i64 notrap aligned readonly can_move vmctx v0+16
+;; v3 = load.i64 notrap aligned readonly can_move vmctx v2+96
+;; return v3
+;; }
+;;
+;; function u0:0(i64 vmctx, i64, i64) -> i32 tail {
+;; block0(v0: i64, v1: i64, v2: i64):
+;; v3 = load.i8 notrap aligned v2
+;; v4 = uextend.i32 v3
+;; return v4
+;; }
diff --git a/tests/disas/riscv64-component-builtins-asm.wat b/tests/disas/riscv64-component-builtins-asm.wat
index cbbfd7de46..dd6a465899 100644
--- a/tests/disas/riscv64-component-builtins-asm.wat
+++ b/tests/disas/riscv64-component-builtins-asm.wat
@@ -1,6 +1,6 @@
;;! target = "riscv64"
;;! test = 'compile'
-;;! filter = '_wasm_call'
+;;! filter = 'wasm-call'
;;! objdump = '--funcs all'
(component
@@ -11,7 +11,7 @@
(core instance (instantiate $m (with "" (instance (export "" (func $f))))))
)
-;; component-resource-drop[0]_wasm_call:
+;; component-trampolines[0]-wasm-call-component-resource-drop[0]:
;; addi sp, sp, -0x10
;; sd ra, 8(sp)
;; sd s0, 0(sp)
diff --git a/tests/disas/riscv64-component-builtins.wat b/tests/disas/riscv64-component-builtins.wat
index 1451e59b56..4a1102fed0 100644
--- a/tests/disas/riscv64-component-builtins.wat
+++ b/tests/disas/riscv64-component-builtins.wat
@@ -1,6 +1,6 @@
;;! target = "riscv64"
;;! test = 'optimize'
-;;! filter = 'component-resource-drop[0]_wasm_call'
+;;! filter = 'wasm-call-component-resource-drop[0]'
(component
(type $a (resource (rep i32)))