A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Implement unsafe intrinsics for compile-time builtins (#11825)
Nick Fitzgeraldad56ff980ae5
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex abfc714a9e..0c001f6b18 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -897,7 +897,12 @@ impl InliningCompiler for Compiler {.user_named_funcs().values().map(|name| FuncKey::from_raw_parts(name.namespace, name.index))- .filter(|key| matches!(key, FuncKey::DefinedWasmFunction(_, _))),+ .filter(|key| match key {+ FuncKey::DefinedWasmFunction(..) => true,+ #[cfg(feature = "component-model")]+ FuncKey::UnsafeIntrinsic(..) => true,+ _ => false,+ }),);Ok(())}@@ -953,25 +958,28 @@ impl InliningCompiler for Compiler {let callee = &caller.params.user_named_funcs()[callee];let callee = FuncKey::from_raw_parts(callee.namespace, callee.index);match callee {- FuncKey::DefinedWasmFunction(_, _) => match (self.0)(callee) {- None => InlineCommand::KeepCall,- Some(func_body) => {- debug_assert!(!func_body.code.is::<CompiledFunction>());- debug_assert!(func_body.code.is::<Option<CompilerContext>>());- let cx = func_body- .code- .downcast_ref::<Option<CompilerContext>>()- .unwrap();- InlineCommand::Inline {- callee: Cow::Borrowed(&cx.as_ref().unwrap().codegen_context.func),- // We've already visited the callee for inlining- // due to our bottom-up approach, no need to- // visit it again.- visit_callee: false,- }+ FuncKey::DefinedWasmFunction(..) => {}+ #[cfg(feature = "component-model")]+ FuncKey::UnsafeIntrinsic(..) => {}+ _ => return InlineCommand::KeepCall,+ }+ match (self.0)(callee) {+ None => InlineCommand::KeepCall,+ Some(func_body) => {+ debug_assert!(!func_body.code.is::<CompiledFunction>());+ debug_assert!(func_body.code.is::<Option<CompilerContext>>());+ let cx = func_body+ .code+ .downcast_ref::<Option<CompilerContext>>()+ .unwrap();+ InlineCommand::Inline {+ callee: Cow::Borrowed(&cx.as_ref().unwrap().codegen_context.func),+ // We've already visited the callee for inlining+ // due to our bottom-up approach, no need to+ // visit it again.+ visit_callee: false,}- },- _ => InlineCommand::KeepCall,+ }}}}@@ -1138,7 +1146,7 @@ impl Compiler {// Note that loads and stores are unconditionally done in the// little-endian format rather than the host's native-endianness,- // despite this load/store being unrelated to execution in wasm itself.+ // despite this load/store being unrelated to execution in Wasm itself.// For more details on this see the `ValRaw` type in// `wasmtime::runtime::vm`.let flags = ir::MemFlags::new()@@ -1355,6 +1363,7 @@ impl FunctionCompiler<'_> {let block0 = builder.create_block();builder.append_block_params_for_function_params(block0);builder.switch_to_block(block0);+ builder.ensure_inserted_block();builder.seal_block(block0);(builder, block0)}diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex ad4981cc81..533c150b11 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -1,7 +1,10 @@//! Compilation support for the component model.-use crate::{TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT, compiler::Compiler};-use anyhow::Result;+use crate::array_call_signature;+use crate::{+ TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT, compiler::Compiler, wasm_call_signature,+};+use anyhow::{Result, bail};use cranelift_codegen::ir::condcodes::IntCC;use cranelift_codegen::ir::{self, InstBuilder, MemFlags, Value};use cranelift_codegen::isa::{CallConv, TargetIsa};@@ -95,6 +98,85 @@ enum WasmArgs {InRegistersUpTo(usize),}+/// Get a function's parameters regardless of the ABI in use.+///+/// This emits code to load the parameters from the array-call's ABI's values+/// vector, if necessary.+fn abi_params(+ compiler: &Compiler,+ builder: &mut FunctionBuilder<'_>,+ abi: Abi,+ types: &[WasmValType],+) -> Vec<ir::Value> {+ let entry_block = builder+ .func+ .layout+ .entry_block()+ .expect("should have an entry block");+ let entry_block_params = builder.func.dfg.block_params(entry_block);+ match abi {+ // Wasm and native ABIs pass parameters as normal function parameters.+ Abi::Wasm => entry_block_params.to_vec(),++ // The array ABI passes a pointer/length as the 3rd/4th arguments and+ // those are used to load the actual Wasm parameters.+ Abi::Array => {+ let &[callee_vmctx, caller_vmctx, values_vec_ptr, values_vec_len] = entry_block_params+ else {+ unreachable!()+ };+ let mut results =+ compiler.load_values_from_array(types, builder, values_vec_ptr, values_vec_len);+ results.splice(0..0, [callee_vmctx, caller_vmctx]);+ results+ }+ }+}++/// Emit code to return the given result values, regardless of the ABI in use.+fn abi_return(+ compiler: &Compiler,+ builder: &mut FunctionBuilder<'_>,+ abi: Abi,+ result_types: &[WasmValType],+ results: &[ir::Value],+) {+ match abi {+ // The Wasm ABI returns values as usual.+ Abi::Wasm => {+ builder.ins().return_(results);+ }++ // The array ABI stores all results in the pointer/length passed+ // as arguments to this function, which contractually are required+ // to have enough space for the results.+ Abi::Array => {+ let entry_block = builder+ .func+ .layout+ .entry_block()+ .expect("should have an entry block");+ let &[_callee_vmctx, _caller_vmctx, values_vec_ptr, values_vec_len] =+ builder.func.dfg.block_params(entry_block)+ else {+ unreachable!()+ };++ compiler.store_values_to_array(+ builder,+ result_types,+ results,+ values_vec_ptr,+ values_vec_len,+ );++ // A "true" return value indicates a successful, non-trapping call.+ let true_value = builder.ins().iconst(ir::types::I8, 1);+ builder.ins().return_(&[true_value]);+ }+ }+}+impl<'a> TrampolineCompiler<'a> {fn new(compiler: &'a Compiler,@@ -970,7 +1052,7 @@ impl<'a> TrampolineCompiler<'a> {// Acquire the result of this function (if any) and interpret it// according to `host_result`.//- // NOte that all match arms here end with `abi_store_results` which+ // Note that all match arms here end with `abi_store_results` which// accounts for the ABI of this function when storing results.let result = self.builder.func.dfg.inst_results(call).get(0).copied();let result_ty = result.map(|v| self.builder.func.dfg.value_type(v));@@ -1292,52 +1374,22 @@ impl<'a> TrampolineCompiler<'a> {}fn abi_load_params(&mut self) -> Vec<ir::Value> {- let mut block0_params = self.builder.func.dfg.block_params(self.block0).to_vec();- match self.abi {- // Wasm and native ABIs pass parameters as normal function- // parameters.- Abi::Wasm => block0_params,-- // The array ABI passes a pointer/length as the 3rd/4th arguments- // and those are used to load the actual wasm parameters.- Abi::Array => {- let results = self.compiler.load_values_from_array(- self.types[self.signature].unwrap_func().params(),- &mut self.builder,- block0_params[2],- block0_params[3],- );- block0_params.truncate(2);- block0_params.extend(results);- block0_params- }- }+ abi_params(+ self.compiler,+ &mut self.builder,+ self.abi,+ self.types[self.signature].unwrap_func().params(),+ )}fn abi_store_results(&mut self, results: &[ir::Value]) {- match self.abi {- // Wasm/native ABIs return values as usual.- Abi::Wasm => {- self.builder.ins().return_(results);- }-- // The array ABI stores all results in the pointer/length passed- // as arguments to this function, which contractually are required- // to have enough space for the results.- Abi::Array => {- let block0_params = self.builder.func.dfg.block_params(self.block0);- let (ptr, len) = (block0_params[2], block0_params[3]);- self.compiler.store_values_to_array(- &mut self.builder,- self.types[self.signature].unwrap_func().returns(),- results,- ptr,- len,- );- let true_value = self.builder.ins().iconst(ir::types::I8, 1);- self.builder.ins().return_(&[true_value]);- }- }+ abi_return(+ self.compiler,+ &mut self.builder,+ self.abi,+ self.types[self.signature].unwrap_func().returns(),+ results,+ );}fn raise_if_host_trapped(&mut self, succeeded: ir::Value) {@@ -1442,8 +1494,215 @@ impl ComponentCompiler for Compiler {needs_gc_heap: false,})}++ fn compile_intrinsic(+ &self,+ tunables: &Tunables,+ component: &ComponentTranslation,+ intrinsic: UnsafeIntrinsic,+ abi: Abi,+ _symbol: &str,+ ) -> Result<CompiledFunctionBody> {+ let offsets = VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);+ let mut compiler = self.function_compiler();+ let cx = &mut compiler.cx;+ let ctx = &mut cx.codegen_context;+ let mut builder = FunctionBuilder::new(&mut ctx.func, cx.func_translator.context());++ // Initialize the function, its signature, and entry block.+ let init = |builder: &mut FunctionBuilder| -> Vec<ir::Value> {+ let wasm_func_ty = WasmFuncType::new(+ intrinsic.core_params().into(),+ intrinsic.core_results().into(),+ );+ builder.func.signature = match abi {+ Abi::Wasm => wasm_call_signature(&*self.isa, &wasm_func_ty, tunables),+ Abi::Array => array_call_signature(&*self.isa),+ };++ assert!(builder.func.layout.entry_block().is_none());+ let entry_block = builder.create_block();+ builder.append_block_params_for_function_params(entry_block);+ builder.seal_block(entry_block);+ builder.switch_to_block(entry_block);+ builder.ensure_inserted_block();++ abi_params(self, builder, abi, wasm_func_ty.params())+ };++ // Emit code for a native-load intrinsic.+ let load = |builder: &mut FunctionBuilder| -> Result<()> {+ debug_assert_eq!(intrinsic.core_params(), &[WasmValType::I64]);+ debug_assert_eq!(intrinsic.core_results().len(), 1);++ let wasm_ty = intrinsic.core_results()[0];+ let clif_ty = unsafe_intrinsic_clif_results(intrinsic)[0];++ let [_callee_vmctx, _caller_vmctx, pointer] = *init(builder) else {+ unreachable!()+ };++ // Truncate the pointer, if necessary.+ debug_assert_eq!(builder.func.dfg.value_type(pointer), ir::types::I64);+ let pointer = match self.isa.pointer_bits() {+ 32 => builder.ins().ireduce(ir::types::I32, pointer),+ 64 => pointer,+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),+ };++ // Do the load!+ let mut value = builder+ .ins()+ .load(clif_ty, ir::MemFlags::trusted(), pointer, 0);++ // Extend the value, if necessary. When implementing the+ // `u8-native-load` intrinsic, for example, we will load a Cranelift+ // value of type `i8` but we need to extend it to an `i32` because+ // Wasm doesn't have an `i8` core value type.+ let wasm_clif_ty = crate::value_type(&*self.isa, wasm_ty);+ if clif_ty != wasm_clif_ty {+ assert!(clif_ty.bytes() < wasm_clif_ty.bytes());+ // NB: all of our unsafe intrinsics for native loads are+ // unsigned, so we always zero-extend.+ value = builder.ins().uextend(wasm_clif_ty, value);+ }++ abi_return(self, builder, abi, &[wasm_ty], &[value]);+ Ok(())+ };++ // Emit code for a native-store intrinsic.+ let store = |builder: &mut FunctionBuilder| -> Result<()> {+ debug_assert!(intrinsic.core_results().is_empty());+ debug_assert!(matches!(intrinsic.core_params(), [WasmValType::I64, _]));++ let wasm_ty = intrinsic.core_params()[1];+ let clif_ty = unsafe_intrinsic_clif_params(intrinsic)[1];++ let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *init(builder) else {+ unreachable!()+ };++ // Truncate the pointer, if necessary.+ debug_assert_eq!(builder.func.dfg.value_type(pointer), ir::types::I64);+ let pointer = match self.isa.pointer_bits() {+ 32 => builder.ins().ireduce(ir::types::I32, pointer),+ 64 => pointer,+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),+ };++ // Truncate the value, if necessary. For example, with+ // `u8-native-store` we will be given an `i32` from Wasm (because+ // core Wasm does not have an 8-bit integer value type) and we need+ // to reduce that into an `i8`.+ let wasm_ty = crate::value_type(&*self.isa, wasm_ty);+ if clif_ty != wasm_ty {+ assert!(clif_ty.bytes() < wasm_ty.bytes());+ value = builder.ins().ireduce(clif_ty, value);+ }++ // Do the store!+ builder+ .ins()+ .store(ir::MemFlags::trusted(), value, pointer, 0);++ abi_return(self, builder, abi, &[], &[]);+ Ok(())+ };++ match intrinsic {+ UnsafeIntrinsic::U8NativeLoad => load(&mut builder)?,+ UnsafeIntrinsic::U16NativeLoad => load(&mut builder)?,+ UnsafeIntrinsic::U32NativeLoad => load(&mut builder)?,+ UnsafeIntrinsic::U64NativeLoad => load(&mut builder)?,+ UnsafeIntrinsic::U8NativeStore => store(&mut builder)?,+ UnsafeIntrinsic::U16NativeStore => store(&mut builder)?,+ UnsafeIntrinsic::U32NativeStore => store(&mut builder)?,+ UnsafeIntrinsic::U64NativeStore => store(&mut builder)?,+ UnsafeIntrinsic::StoreDataAddress => {+ let [callee_vmctx, _caller_vmctx] = *init(&mut builder) else {+ unreachable!()+ };+ let pointer_type = self.isa.pointer_type();++ // Load the `*mut VMStoreContext` out of our vmctx.+ let store_ctx = builder.ins().load(+ pointer_type,+ ir::MemFlags::trusted()+ .with_readonly()+ .with_alias_region(Some(ir::AliasRegion::Vmctx))+ .with_can_move(),+ callee_vmctx,+ i32::try_from(offsets.vm_store_context()).unwrap(),+ );++ // Load the `*mut T` out of the `VMStoreContext`.+ let data_address = builder.ins().load(+ pointer_type,+ ir::MemFlags::trusted()+ .with_readonly()+ .with_alias_region(Some(ir::AliasRegion::Vmctx))+ .with_can_move(),+ store_ctx,+ i32::from(offsets.ptr.vmstore_context_store_data()),+ );++ // Zero-extend the address if we are on a 32-bit architecture.+ let data_address = match pointer_type.bits() {+ 32 => builder.ins().uextend(ir::types::I64, data_address),+ 64 => data_address,+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),+ };++ abi_return(+ self,+ &mut builder,+ abi,+ intrinsic.core_results(),+ &[data_address],+ );+ }+ }++ builder.finalize();+ Ok(CompiledFunctionBody {+ code: super::box_dyn_any_compiler_context(Some(compiler.cx)),+ needs_gc_heap: false,+ })+ }}+macro_rules! unsafe_intrinsic_clif_params_results {+ (+ $(+ $symbol:expr => $variant:ident : $ctor:ident ( $( $param:ident : $param_ty:ident ),* ) $( -> $result_ty:ident )? ;+ )*+ ) => {+ fn unsafe_intrinsic_clif_params(intrinsic: UnsafeIntrinsic) -> &'static [ir::types::Type] {+ match intrinsic {+ $(+ UnsafeIntrinsic::$variant => &[ $( unsafe_intrinsic_clif_params_results!(@clif_type $param_ty) ),* ],+ )*+ }+ }++ fn unsafe_intrinsic_clif_results(intrinsic: UnsafeIntrinsic) -> &'static [ir::types::Type] {+ match intrinsic {+ $(+ UnsafeIntrinsic::$variant => &[ $( unsafe_intrinsic_clif_params_results!(@clif_type $result_ty) )? ],+ )*+ }+ }+ };++ (@clif_type u8) => { ir::types::I8 };+ (@clif_type u16) => { ir::types::I16 };+ (@clif_type u32) => { ir::types::I32 };+ (@clif_type u64) => { ir::types::I64 };+}++wasmtime_environ::for_each_unsafe_intrinsic!(unsafe_intrinsic_clif_params_results);+impl TrampolineCompiler<'_> {fn translate_transcode(&mut self,diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex c26f84a723..52208e7d90 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -1468,9 +1468,21 @@ impl FuncEnvironment<'_> {.unwrap_module_type_index();let signature = self.get_or_create_interned_sig_ref(func, ty);- let (module, def_func_index) =- self.translation.known_imported_functions[func_index].unwrap();- let key = FuncKey::DefinedWasmFunction(module, def_func_index);+ let key = match self.translation.known_imported_functions[func_index] {+ Some(key @ FuncKey::DefinedWasmFunction(..)) => key,++ #[cfg(feature = "component-model")]+ Some(key @ FuncKey::UnsafeIntrinsic(..)) => key,++ Some(key) => {+ panic!("unexpected kind of known-import function: {key:?}")+ }++ None => panic!(+ "cannot make an `ir::FuncRef` for a function import that is not statically known"+ ),+ };+let (namespace, index) = key.into_raw_parts();let name = ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName { namespace, index }),@@ -1903,9 +1915,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {mut self,callee_index: FuncIndex,sig_ref: ir::SigRef,- call_args: &[ir::Value],+ wasm_call_args: &[ir::Value],) -> WasmResult<CallRets> {- let mut real_call_args = Vec::with_capacity(call_args.len() + 2);+ let mut real_call_args = Vec::with_capacity(wasm_call_args.len() + 2);let caller_vmctx = self.builder.func@@ -1922,7 +1934,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {real_call_args.push(caller_vmctx);// Then append the regular call arguments.- real_call_args.extend_from_slice(call_args);+ real_call_args.extend_from_slice(wasm_call_args);// Finally, make the direct call!let callee = self@@ -1957,25 +1969,47 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {real_call_args.push(callee_vmctx);real_call_args.push(caller_vmctx);- // Then append the regular call arguments.- real_call_args.extend_from_slice(call_args);+ // Then append the Wasm call arguments.+ real_call_args.extend_from_slice(wasm_call_args);// If we statically know the imported function (e.g. this is a// component-to-component call where we statically know both components)- // then we can actually still make a direct call (although we do have to- // pass the callee's vmctx that we just loaded, not our own). Otherwise,- // we really do an indirect call.- if self.env.translation.known_imported_functions[callee_index].is_some() {- let callee = self- .env- .get_or_create_imported_func_ref(self.builder.func, callee_index);- Ok(self.direct_call_inst(callee, &real_call_args))- } else {- let func_addr = self- .builder- .ins()- .load(pointer_type, mem_flags, base, body_offset);- Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))+ // then we can avoid doing an indirect call.+ match self.env.translation.known_imported_functions[callee_index].as_ref() {+ // The import is always a compile-time builtin intrinsic. Make a+ // direct call to that function (presumably it will eventually be+ // inlined).+ #[cfg(feature = "component-model")]+ Some(FuncKey::UnsafeIntrinsic(..)) => {+ let callee = self+ .env+ .get_or_create_imported_func_ref(self.builder.func, callee_index);+ Ok(self.direct_call_inst(callee, &real_call_args))+ }++ // The import is always satisfied with the given defined Wasm+ // function, so do a direct call to that function! (Although we take+ // care to still pass its `funcref`'s `vmctx` as the callee `vmctx`+ // in `real_call_args` and not the caller's.)+ Some(FuncKey::DefinedWasmFunction(..)) => {+ let callee = self+ .env+ .get_or_create_imported_func_ref(self.builder.func, callee_index);+ Ok(self.direct_call_inst(callee, &real_call_args))+ }++ Some(key) => panic!("unexpected kind of known-import function: {key:?}"),++ // Unknown import function or this module is instantiated many times+ // and with different functions. Either way, we have to do the+ // indirect call.+ None => {+ let func_addr = self+ .builder+ .ins()+ .load(pointer_type, mem_flags, base, body_offset);+ Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))+ }}}diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rsindex 63d3a4ef62..a2b6e029a4 100644--- a/crates/environ/src/builtin.rs+++ b/crates/environ/src/builtin.rs@@ -260,9 +260,18 @@ macro_rules! declare_builtin_index {pub struct $index_name:ident : $for_each_builtin:ident ;) => {$(#[$attr])*- #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+ #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct $index_name(u32);+ impl core::fmt::Debug for $index_name {+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {+ f.debug_struct(stringify!($index_name))+ .field("index", &self.0)+ .field("ctor", &self.ctor_name())+ .finish()+ }+ }+impl $index_name {/// Create a new builtin from its raw indexpub const fn from_u32(i: u32) -> Self {@@ -275,6 +284,8 @@ macro_rules! declare_builtin_index {self.0}+ $for_each_builtin!(define_ctor_name);+$for_each_builtin!(declare_builtin_index_constructors);}@@ -287,6 +298,28 @@ macro_rules! declare_builtin_index {};}+/// Helper macro used by the above macro.+macro_rules! define_ctor_name {+ (+ $(+ $( #[$attr:meta] )*+ $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;+ )*+ ) => {+ /// Returns the name of the constructor that creates this index.+ pub fn ctor_name(&self) -> &'static str {+ let mut _i = self.0;+ $(+ if _i == 0 {+ return stringify!($name);+ }+ _i -= 1;+ )*+ unreachable!()+ }+ }+}+/// Helper macro used by the above macro.macro_rules! declare_builtin_index_constructors {(diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rsindex 65d438ee88..6894ba713c 100644--- a/crates/environ/src/compile/module_environ.rs+++ b/crates/environ/src/compile/module_environ.rs@@ -2,14 +2,15 @@ use crate::module::{FuncRefIndex, Initializer, MemoryInitialization, MemoryInitializer, Module, TableSegment,TableSegmentElements,};+use crate::prelude::*;use crate::{ConstExpr, ConstOp, DataIndex, DefinedFuncIndex, ElemIndex, EngineOrModuleTypeIndex,- EntityIndex, EntityType, FuncIndex, GlobalIndex, IndexType, InitMemory, MemoryIndex,+ EntityIndex, EntityType, FuncIndex, FuncKey, GlobalIndex, IndexType, InitMemory, MemoryIndex,ModuleInternedTypeIndex, ModuleTypesBuilder, PrimaryMap, SizeOverflow, StaticMemoryInitializer,- TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert, TypeIndex, WasmError,- WasmHeapTopType, WasmHeapType, WasmResult, WasmValType, WasmparserTypeConverter,+ StaticModuleIndex, TableIndex, TableInitialValue, Tag, TagIndex, Tunables, TypeConvert,+ TypeIndex, WasmError, WasmHeapTopType, WasmHeapType, WasmResult, WasmValType,+ WasmparserTypeConverter,};-use crate::{StaticModuleIndex, prelude::*};use anyhow::{Result, bail};use cranelift_entity::SecondaryMap;use cranelift_entity::packed_option::ReservedValue;@@ -55,12 +56,15 @@ pub struct ModuleTranslation<'data> {/// References to the function bodies.pub function_body_inputs: PrimaryMap<DefinedFuncIndex, FunctionBodyData<'data>>,- /// For each imported function, the single statically-known defined function- /// that satisfies that import, if any. This is used to turn what would- /// otherwise be indirect calls through the imports table into direct calls,- /// when possible.- pub known_imported_functions:- SecondaryMap<FuncIndex, Option<(StaticModuleIndex, DefinedFuncIndex)>>,+ /// For each imported function, the single statically-known function that+ /// always satisfies that import, if any.+ ///+ /// This is used to turn what would otherwise be indirect calls through the+ /// imports table into direct calls, when possible.+ ///+ /// When filled in, this only ever contains+ /// `FuncKey::DefinedWasmFunction(..)`s and `FuncKey::Intrinsic(..)`s.+ pub known_imported_functions: SecondaryMap<FuncIndex, Option<FuncKey>>,/// A list of type signatures which are considered exported from this/// module, or those that can possibly be called. This list is sorted, anddiff --git a/crates/environ/src/component.rs b/crates/environ/src/component.rsindex 3c7087728a..414c1a3ec5 100644--- a/crates/environ/src/component.rs+++ b/crates/environ/src/component.rs@@ -53,11 +53,13 @@ pub const START_FLAG_ASYNC_CALLEE: i32 = 1 << 0;mod artifacts;mod info;+mod intrinsic;mod names;mod types;mod vmcomponent_offsets;pub use self::artifacts::*;pub use self::info::*;+pub use self::intrinsic::*;pub use self::names::*;pub use self::types::*;pub use self::vmcomponent_offsets::*;diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rsindex ccfdab4566..08c089c66b 100644--- a/crates/environ/src/component/compiler.rs+++ b/crates/environ/src/component/compiler.rs@@ -1,4 +1,4 @@-use crate::component::{ComponentTranslation, ComponentTypesBuilder};+use crate::component::{ComponentTranslation, ComponentTypesBuilder, UnsafeIntrinsic};use crate::{Abi, CompiledFunctionBody, FuncKey, Tunables};use anyhow::Result;@@ -19,4 +19,14 @@ pub trait ComponentCompiler: Send + Sync {tunables: &Tunables,symbol: &str,) -> Result<CompiledFunctionBody>;++ /// Compile the given Wasmtime intrinsic.+ fn compile_intrinsic(+ &self,+ tunables: &Tunables,+ component: &ComponentTranslation,+ intrinsic: UnsafeIntrinsic,+ abi: Abi,+ symbol: &str,+ ) -> Result<CompiledFunctionBody>;}diff --git a/crates/environ/src/component/dfg.rs b/crates/environ/src/component/dfg.rsindex a85a80cea0..68e234e338 100644--- a/crates/environ/src/component/dfg.rs+++ b/crates/environ/src/component/dfg.rs@@ -31,6 +31,7 @@ use crate::component::*;use crate::prelude::*;use crate::{EntityIndex, EntityRef, ModuleInternedTypeIndex, PrimaryMap, WasmValType};use anyhow::Result;+use cranelift_entity::packed_option::PackedOption;use indexmap::IndexMap;use info::LinearMemoryOptions;use std::collections::HashMap;@@ -54,6 +55,10 @@ pub struct ComponentDfg {/// compiled by Cranelift.pub trampolines: Intern<TrampolineIndex, (ModuleInternedTypeIndex, Trampoline)>,+ /// A map from `UnsafeIntrinsic::index()` to that intrinsic's+ /// module-interned type.+ pub unsafe_intrinsics: [PackedOption<ModuleInternedTypeIndex>; UnsafeIntrinsic::len() as usize],+/// Know reallocation functions which are used by `lowerings` (e.g. will be/// used by the host)pub reallocs: Intern<ReallocId, CoreDef>,@@ -258,6 +263,8 @@ pub enum CoreDef {Export(CoreExport<EntityIndex>),InstanceFlags(RuntimeComponentInstanceIndex),Trampoline(TrampolineIndex),+ UnsafeIntrinsic(ModuleInternedTypeIndex, UnsafeIntrinsic),+/// This is a special variant not present in `info::CoreDef` which/// represents that this definition refers to a fused adapter function. This/// adapter is fully processed after the initial translation and@@ -595,6 +602,7 @@ impl ComponentDfg {runtime_callbacks: Default::default(),runtime_instances: Default::default(),num_lowerings: 0,+ unsafe_intrinsics: Default::default(),trampolines: Default::default(),trampoline_defs: Default::default(),trampoline_map: Default::default(),@@ -629,6 +637,7 @@ impl ComponentDfg {exports,export_items,initializers: linearize.initializers,+ unsafe_intrinsics: linearize.unsafe_intrinsics,trampolines: linearize.trampolines,num_lowerings: linearize.num_lowerings,options: linearize.options,@@ -666,6 +675,7 @@ impl ComponentDfg {struct LinearizeDfg<'a> {dfg: &'a ComponentDfg,initializers: Vec<GlobalInitializer>,+ unsafe_intrinsics: [PackedOption<ModuleInternedTypeIndex>; UnsafeIntrinsic::len() as usize],trampolines: PrimaryMap<TrampolineIndex, ModuleInternedTypeIndex>,trampoline_defs: PrimaryMap<TrampolineIndex, info::Trampoline>,options: PrimaryMap<OptionsIndex, info::CanonicalOptions>,@@ -856,6 +866,13 @@ impl LinearizeDfg<'_> {CoreDef::InstanceFlags(i) => info::CoreDef::InstanceFlags(*i),CoreDef::Adapter(id) => info::CoreDef::Export(self.adapter(*id)),CoreDef::Trampoline(index) => info::CoreDef::Trampoline(self.trampoline(*index)),+ CoreDef::UnsafeIntrinsic(ty, i) => {+ let index = usize::try_from(i.index()).unwrap();+ if self.unsafe_intrinsics[index].is_none() {+ self.unsafe_intrinsics[index] = Some(*ty).into();+ }+ info::CoreDef::UnsafeIntrinsic(*i)+ }}}diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rsindex d825c9a51e..251d6ef181 100644--- a/crates/environ/src/component/info.rs+++ b/crates/environ/src/component/info.rs@@ -49,6 +49,7 @@use crate::component::*;use crate::prelude::*;use crate::{EntityIndex, ModuleInternedTypeIndex, PrimaryMap, WasmValType};+use cranelift_entity::packed_option::PackedOption;use serde_derive::{Deserialize, Serialize};/// Metadata as a result of compiling a component.@@ -162,6 +163,10 @@ pub struct Component {/// WebAssembly type signature of all trampolines.pub trampolines: PrimaryMap<TrampolineIndex, ModuleInternedTypeIndex>,+ /// A map from a `UnsafeIntrinsic::index()` to that intrinsic's+ /// module-interned type.+ pub unsafe_intrinsics: [PackedOption<ModuleInternedTypeIndex>; UnsafeIntrinsic::len() as usize],+/// The number of lowered host functions (maximum `LoweredIndex`) needed to/// instantiate this component.pub num_lowerings: u32,@@ -382,6 +387,8 @@ pub enum CoreDef {/// This is a reference to a Cranelift-generated trampoline which is/// described in the `trampolines` array.Trampoline(TrampolineIndex),+ /// An intrinsic for compile-time builtins.+ UnsafeIntrinsic(UnsafeIntrinsic),}impl<T> From<CoreExport<T>> for CoreDefdiff --git a/crates/environ/src/component/intrinsic.rs b/crates/environ/src/component/intrinsic.rsnew file mode 100644index 0000000000..391c9038f8--- /dev/null+++ b/crates/environ/src/component/intrinsic.rs@@ -0,0 +1,147 @@+//! Wasmtime compile-time intrinsic definitions.++use core::str::FromStr;++use anyhow::{Result, bail};+use serde_derive::{Deserialize, Serialize};++/// Invoke a macro for each of our unsafe intrinsics.+#[macro_export]+macro_rules! for_each_unsafe_intrinsic {+ ($mac:ident) => {+ $mac! {+ "store-data-address" => StoreDataAddress : store_data_address() -> u64;++ "u8-native-load" => U8NativeLoad : u8_native_load(address: u64) -> u8;+ "u8-native-store" => U8NativeStore : u8_native_store(address: u64, value: u8);++ "u16-native-load" => U16NativeLoad : u16_native_load(address: u64) -> u16;+ "u16-native-store" => U16NativeStore : u16_native_store(address: u64, value: u16);++ "u32-native-load" => U32NativeLoad : u32_native_load(address: u64) -> u32;+ "u32-native-store" => U32NativeStore : u32_native_store(address: u64, value: u32);++ "u64-native-load" => U64NativeLoad : u64_native_load(address: u64) -> u64;+ "u64-native-store" => U64NativeStore : u64_native_store(address: u64, value: u64);+ }+ };+}++macro_rules! define_unsafe_intrinsics {+ (+ $(+ $symbol:expr => $variant:ident : $ctor:ident ( $( $param:ident : $param_ty:ident ),* ) $( -> $result_ty:ident )? ;+ )*+ ) => {+ /// An index type for Wasmtime's intrinsics available to compile-time+ /// builtins.+ #[repr(u32)]+ #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+ pub enum UnsafeIntrinsic {+ $(+ #[doc = concat!("The `", $symbol, "` intrinsic.")]+ $variant,+ )*+ }++ impl UnsafeIntrinsic {+ /// Returns the total number of unsafe intrinsics.+ pub const fn len() -> u32 {+ let mut len = 0;+ $(+ let _ = Self::$variant;+ len += 1;+ )*+ len+ }++ /// Construct an `UnsafeIntrinsic` from its `u32` index.+ ///+ /// Panics on invalid indices.+ pub const fn from_u32(i: u32) -> Self {+ assert!(i < Self::len());+ $(+ if i == Self::$variant.index() {+ return Self::$variant;+ }+ )*+ unreachable!()+ }++ /// Get this intrinsic's index.+ pub const fn index(&self) -> u32 {+ *self as u32+ }++ /// Get this intrinsic's name.+ pub const fn name(&self) -> &'static str {+ match self {+ $(+ Self::$variant => $symbol,+ )*+ }+ }++ /// Get this intrinsic's parameters, as core Wasm value types.+ pub const fn core_params(&self) -> &'static [$crate::WasmValType] {+ match self {+ $(+ Self::$variant => &[ $( define_unsafe_intrinsics!(@core_type $param_ty) ),* ],+ )*+ }+ }++ /// Get this intrinsic's results, as core Wasm value types.+ pub const fn core_results(&self) -> &'static [$crate::WasmValType] {+ match self {+ $(+ Self::$variant => &[ $( define_unsafe_intrinsics!(@core_type $result_ty) )? ],+ )*+ }+ }++ /// Get this intrinsic's parameters, as component model interface types.+ pub const fn component_params(&self) -> &'static [$crate::component::InterfaceType] {+ match self {+ $(+ Self::$variant => &[ $( define_unsafe_intrinsics!(@component_type $param_ty) ),* ],+ )*+ }+ }++ /// Get this intrinsic's results, as component model interface types.+ pub const fn component_results(&self) -> &'static [$crate::component::InterfaceType] {+ match self {+ $(+ Self::$variant => &[ $( define_unsafe_intrinsics!(@component_type $result_ty) ),* ],+ )*+ }+ }+ }++ impl FromStr for UnsafeIntrinsic {+ type Err = anyhow::Error;++ fn from_str(s: &str) -> Result<Self> {+ match s {+ $(+ $symbol => Ok(Self::$variant),+ )*+ _ => bail!("invalid unsafe intrinsic: {s:?}"),+ }+ }+ }+ };++ (@core_type u8) => { $crate::WasmValType::I32 };+ (@core_type u16) => { $crate::WasmValType::I32 };+ (@core_type u32) => { $crate::WasmValType::I32 };+ (@core_type u64) => { $crate::WasmValType::I64 };++ (@component_type u8) => { $crate::component::InterfaceType::U8 };+ (@component_type u16) => { $crate::component::InterfaceType::U16 };+ (@component_type u32) => { $crate::component::InterfaceType::U32 };+ (@component_type u64) => { $crate::component::InterfaceType::U64 };+}++for_each_unsafe_intrinsic!(define_unsafe_intrinsics);diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rsindex 9cb276f43a..580dc5c826 100644--- a/crates/environ/src/component/translate.rs+++ b/crates/environ/src/component/translate.rs@@ -1,14 +1,17 @@-use crate::ScopeVec;+use crate::Abi;use crate::component::dfg::AbstractInstantiations;use crate::component::*;use crate::prelude::*;use crate::{- EngineOrModuleTypeIndex, EntityIndex, ModuleEnvironment, ModuleInternedTypeIndex,- ModuleTranslation, ModuleTypesBuilder, PrimaryMap, TagIndex, Tunables, TypeConvert,+ EngineOrModuleTypeIndex, EntityIndex, FuncKey, ModuleEnvironment, ModuleInternedTypeIndex,+ ModuleTranslation, ModuleTypesBuilder, PrimaryMap, ScopeVec, TagIndex, Tunables, TypeConvert,WasmHeapType, WasmResult, WasmValType,};+use anyhow::Context;use anyhow::anyhow;+use anyhow::ensure;use anyhow::{Result, bail};+use core::str::FromStr;use cranelift_entity::SecondaryMap;use cranelift_entity::packed_option::PackedOption;use indexmap::IndexMap;@@ -73,6 +76,9 @@ pub struct Translator<'a, 'data> {/// As frames are popped from `lexical_scopes` their completed component/// will be pushed onto this list.static_components: PrimaryMap<StaticComponentIndex, Translation<'data>>,++ /// The top-level import name for Wasmtime's unsafe intrinsics, if any.+ unsafe_intrinsics_import: Option<&'a str>,}/// Representation of the syntactic scope of a component meaning where it is@@ -176,6 +182,9 @@ enum LocalInitializer<'data> {// importsImport(ComponentImportName<'data>, ComponentEntityType),+ // An import of an intrinsic for compile-time builtins.+ IntrinsicsImport,+// canonical function sectionsLower {func: ComponentFuncIndex,@@ -425,9 +434,18 @@ impl<'a, 'data> Translator<'a, 'data> {static_components: Default::default(),static_modules: Default::default(),scope_vec,+ unsafe_intrinsics_import: None,}}+ /// Expose Wasmtime's unsafe intrinsics under the given top-level import+ /// name.+ pub fn expose_unsafe_intrinsics(&mut self, name: &'a str) -> &mut Self {+ assert!(self.unsafe_intrinsics_import.is_none());+ self.unsafe_intrinsics_import = Some(name);+ self+ }+/// Translates the binary `component`.////// This is the workhorse of compilation which will parse all of@@ -566,7 +584,7 @@ impl<'a, 'data> Translator<'a, 'data> {.is_none());- match arg {+ let known_func = match arg {CoreDef::InstanceFlags(_) => unreachable!("instance flags are not a function"),// We could in theory inline these trampolines, so it could@@ -583,6 +601,11 @@ impl<'a, 'data> Translator<'a, 'data> {// the effort.CoreDef::Trampoline(_) => continue,+ // This import is a compile-time builtin intrinsic, we+ // should inline its implementation during function+ // translation.+ CoreDef::UnsafeIntrinsic(i) => FuncKey::UnsafeIntrinsic(Abi::Wasm, *i),+// This imported function is an export from another// instance, a perfect candidate for becoming an inlinable// direct call!@@ -613,14 +636,15 @@ impl<'a, 'data> Translator<'a, 'data> {continue;};- assert!(- self.static_modules[module].known_imported_functions[imported_func]- .is_none()- );- self.static_modules[module].known_imported_functions[imported_func] =- Some((*arg_module, arg_module_def_func));+ FuncKey::DefinedWasmFunction(*arg_module, arg_module_def_func)}- }+ };++ assert!(+ self.static_modules[module].known_imported_functions[imported_func].is_none()+ );+ self.static_modules[module].known_imported_functions[imported_func] =+ Some(known_func);}}}@@ -725,9 +749,17 @@ impl<'a, 'data> Translator<'a, 'data> {let ty = types.component_entity_type_of_import(import.name.0).unwrap();- self.result- .initializers- .push(LocalInitializer::Import(import.name, ty));++ if self.is_unsafe_intrinsics_import(import.name.0) {+ self.check_unsafe_intrinsics_import(import.name.0, ty)?;+ self.result+ .initializers+ .push(LocalInitializer::IntrinsicsImport);+ } else {+ self.result+ .initializers+ .push(LocalInitializer::Import(import.name, ty));+ }}}@@ -1555,6 +1587,105 @@ impl<'a, 'data> Translator<'a, 'data> {let id = types.core_function_at(index);self.types.module_types_builder().intern_type(types, id)}++ fn is_unsafe_intrinsics_import(&self, import: &str) -> bool {+ self.lexical_scopes.is_empty()+ && self+ .unsafe_intrinsics_import+ .is_some_and(|name| import == name)+ }++ fn check_unsafe_intrinsics_import(&self, import: &str, ty: ComponentEntityType) -> Result<()> {+ let types = &self.validator.types(0).unwrap();++ let ComponentEntityType::Instance(instance_ty) = ty else {+ bail!("bad unsafe intrinsics import: import `{import}` must be an instance import")+ };+ let instance_ty = &types[instance_ty];++ ensure!(+ instance_ty.defined_resources.is_empty(),+ "bad unsafe intrinsics import: import `{import}` cannot define any resources"+ );+ ensure!(+ instance_ty.explicit_resources.is_empty(),+ "bad unsafe intrinsics import: import `{import}` cannot export any resources"+ );++ for (name, ty) in &instance_ty.exports {+ let ComponentEntityType::Func(func_ty) = ty else {+ bail!(+ "bad unsafe intrinsics import: imported instance `{import}` must \+ only export functions"+ )+ };+ let func_ty = &types[*func_ty];++ fn ty_eq(a: &InterfaceType, b: &wasmparser::component_types::ComponentValType) -> bool {+ use wasmparser::{PrimitiveValType as P, component_types::ComponentValType as C};+ match (a, b) {+ (InterfaceType::U8, C::Primitive(P::U8)) => true,+ (InterfaceType::U8, _) => false,++ (InterfaceType::U16, C::Primitive(P::U16)) => true,+ (InterfaceType::U16, _) => false,++ (InterfaceType::U32, C::Primitive(P::U32)) => true,+ (InterfaceType::U32, _) => false,++ (InterfaceType::U64, C::Primitive(P::U64)) => true,+ (InterfaceType::U64, _) => false,++ (ty, _) => unreachable!("no unsafe intrinsics use {ty:?}"),+ }+ }++ fn check_types<'a>(+ expected: impl ExactSizeIterator<Item = &'a InterfaceType>,+ actual: impl ExactSizeIterator<Item = &'a wasmparser::component_types::ComponentValType>,+ kind: &str,+ import: &str,+ name: &str,+ ) -> core::result::Result<(), anyhow::Error> {+ let expected_len = expected.len();+ let actual_len = actual.len();+ ensure!(+ expected_len == actual_len,+ "bad unsafe intrinsics import at `{import}`: function `{name}` must have \+ {expected_len} {kind}, found {actual_len}"+ );++ for (i, (actual_ty, expected_ty)) in actual.zip(expected).enumerate() {+ ensure!(+ ty_eq(expected_ty, actual_ty),+ "bad unsafe intrinsics import at `{import}`: {kind}[{i}] for function \+ `{name}` must be `{expected_ty:?}`, found `{actual_ty:?}`"+ );+ }+ Ok(())+ }++ let intrinsic = UnsafeIntrinsic::from_str(name)+ .with_context(|| format!("bad unsafe intrinsics import at `{import}`"))?;++ check_types(+ intrinsic.component_params().iter(),+ func_ty.params.iter().map(|(_name, ty)| ty),+ "parameters",+ &import,+ &name,+ )?;+ check_types(+ intrinsic.component_results().iter(),+ func_ty.result.iter(),+ "results",+ &import,+ &name,+ )?;+ }++ Ok(())+ }}impl Translation<'_> {diff --git a/crates/environ/src/component/translate/adapt.rs b/crates/environ/src/component/translate/adapt.rsindex 64c9e1a9c6..ad3a00f525 100644--- a/crates/environ/src/component/translate/adapt.rs+++ b/crates/environ/src/component/translate/adapt.rs@@ -448,7 +448,9 @@ impl PartitionAdapterModules {}// These items can't transitively depend on an adapter- dfg::CoreDef::Trampoline(_) | dfg::CoreDef::InstanceFlags(_) => {}+ dfg::CoreDef::Trampoline(_)+ | dfg::CoreDef::InstanceFlags(_)+ | dfg::CoreDef::UnsafeIntrinsic(..) => {}}}diff --git a/crates/environ/src/component/translate/inline.rs b/crates/environ/src/component/translate/inline.rsindex ec9382c8e9..360f7313f8 100644--- a/crates/environ/src/component/translate/inline.rs+++ b/crates/environ/src/component/translate/inline.rs@@ -47,6 +47,7 @@use crate::component::translate::*;use crate::{EntityType, IndexType};+use core::str::FromStr;use std::borrow::Cow;use wasmparser::component_types::{ComponentAnyTypeId, ComponentCoreModuleTypeId};@@ -302,6 +303,9 @@ enum ModuleInstanceDef<'a> {#[derive(Clone)]enum ComponentFuncDef<'a> {+ /// A compile-time builtin intrinsic.+ UnsafeIntrinsic(UnsafeIntrinsic),+/// A host-imported component function.Import(ImportPath<'a>),@@ -318,6 +322,10 @@ enum ComponentFuncDef<'a> {#[derive(Clone)]enum ComponentInstanceDef<'a> {+ /// The `__wasmtime_intrinsics` instance that exports all of our+ /// compile-time builtin intrinsics.+ Intrinsics,+/// A host-imported instance.////// This typically means that it's "just" a map of named values. It's not@@ -485,6 +493,12 @@ impl<'a> Inliner<'a> {frame.push_item(arg.clone());}+ IntrinsicsImport => {+ frame+ .component_instances+ .push(ComponentInstanceDef::Intrinsics);+ }+// Lowering a component function to a core wasm function is// generally what "triggers compilation". Here various metadata is// recorded and then the final component gets an initializer@@ -597,6 +611,10 @@ impl<'a> Inliner<'a> {});dfg::CoreDef::Adapter(adapter_idx)}++ ComponentFuncDef::UnsafeIntrinsic(intrinsic) => {+ dfg::CoreDef::UnsafeIntrinsic(options.core_type, *intrinsic)+ }};frame.funcs.push((lower_core_type, func));}@@ -1319,6 +1337,12 @@ impl<'a> Inliner<'a> {AliasComponentExport(instance, name) => {match &frame.component_instances[*instance] {+ ComponentInstanceDef::Intrinsics => {+ frame.push_item(ComponentItemDef::Func(ComponentFuncDef::UnsafeIntrinsic(+ UnsafeIntrinsic::from_str(name)?,+ )));+ }+// Aliasing an export from an imported instance means that// we're extending the `ImportPath` by one name, represented// with the clone + push here. Afterwards an appropriate@@ -1581,11 +1605,23 @@ impl<'a> Inliner<'a> {"component export `{name}` is a reexport of an imported function which is not implemented")}++ ComponentFuncDef::UnsafeIntrinsic(_) => {+ bail!(+ "component export `{name}` is a reexport of an intrinsic function which is not supported"+ )+ }},ComponentItemDef::Instance(instance) => {let mut exports = IndexMap::new();match instance {+ ComponentInstanceDef::Intrinsics => {+ bail!(+ "component export `{name}` is a reexport of the intrinsics instance which is not supported"+ )+ }+// If this instance is one that was originally imported by// the component itself then the imports are translated here// by converting to a `ComponentItemDef` and then@@ -1848,6 +1884,9 @@ impl<'a> ComponentItemDef<'a> {ComponentItemDef::from_import(path.push(element), types[ty].exports[element]).unwrap()}+ ComponentInstanceDef::Intrinsics => {+ unreachable!("intrinsics do not define resources")+ }};}diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rsindex 435edb2aaf..bbbb0e79a1 100644--- a/crates/environ/src/component/vmcomponent_offsets.rs+++ b/crates/environ/src/component/vmcomponent_offsets.rs@@ -6,6 +6,7 @@// limits: *const VMStoreContext,// flags: [VMGlobalDefinition; component.num_runtime_component_instances],// trampoline_func_refs: [VMFuncRef; component.num_trampolines],+// unsafe_intrinsics: [VMFuncRef; component.num_unsafe_intrinsics],// lowerings: [VMLowering; component.num_lowerings],// memories: [*mut VMMemoryDefinition; component.num_runtime_memories],// tables: [VMTable; component.num_runtime_tables],@@ -58,6 +59,9 @@ pub struct VMComponentOffsets<P> {pub num_runtime_component_instances: u32,/// Number of cranelift-compiled trampolines required for this component.pub num_trampolines: u32,+ /// Number of `VMFuncRef`s for unsafe intrinsics within this component's+ /// context.+ pub num_unsafe_intrinsics: u32,/// Number of resources within a component which need destructors stored.pub num_resources: u32,@@ -67,6 +71,7 @@ pub struct VMComponentOffsets<P> {vm_store_context: u32,flags: u32,trampoline_func_refs: u32,+ intrinsic_func_refs: u32,lowerings: u32,memories: u32,tables: u32,@@ -97,12 +102,35 @@ impl<P: PtrSize> VMComponentOffsets<P> {num_runtime_post_returns: component.num_runtime_post_returns,num_runtime_component_instances: component.num_runtime_component_instances,num_trampolines: component.trampolines.len().try_into().unwrap(),+ num_unsafe_intrinsics: if let Some(i) = component+ .unsafe_intrinsics+ .iter()+ .rposition(|x| x.is_some())+ {+ // Note: We do not currently have an indirection between "the+ // `i`th unsafe intrinsic in the vmctx" and+ // `UnsafeIntrinsic::from_u32(i)`, so therefore if we are+ // compiling in *any* intrinsics, we need to include space for+ // all of them up to the max `i` that is used.+ //+ // We _could_ introduce such an indirection via a map in+ // `Component` like `PrimaryMap<UnsafeIntrinsicIndex,+ // UnsafeIntrinsic>`, and that would allow us to densely pack+ // intrinsics in the vmctx. However we do not do that today+ // because there are very few unsafe intrinsics, and we do not+ // see that changing anytime soon, so we aren't wasting much+ // space.+ u32::try_from(i + 1).unwrap()+ } else {+ 0+ },num_resources: component.num_resources,magic: 0,builtins: 0,vm_store_context: 0,flags: 0,trampoline_func_refs: 0,+ intrinsic_func_refs: 0,lowerings: 0,memories: 0,tables: 0,@@ -146,6 +174,7 @@ impl<P: PtrSize> VMComponentOffsets<P> {size(flags) = cmul(ret.num_runtime_component_instances, ret.ptr.size_of_vmglobal_definition()),align(u32::from(ret.ptr.size())),size(trampoline_func_refs) = cmul(ret.num_trampolines, ret.ptr.size_of_vm_func_ref()),+ size(intrinsic_func_refs) = cmul(ret.num_unsafe_intrinsics, ret.ptr.size_of_vm_func_ref()),size(lowerings) = cmul(ret.num_lowerings, ret.ptr.size() * 2),size(memories) = cmul(ret.num_runtime_memories, ret.ptr.size()),size(tables) = cmul(ret.num_runtime_tables, ret.size_of_vmtable_import()),@@ -209,6 +238,20 @@ impl<P: PtrSize> VMComponentOffsets<P> {self.trampoline_func_refs() + index.as_u32() * u32::from(self.ptr.size_of_vm_func_ref())}+ /// The offset of the `unsafe_intrinsic_func_refs` field.+ #[inline]+ pub fn unsafe_intrinsic_func_refs(&self) -> u32 {+ self.intrinsic_func_refs+ }++ /// The offset of the `VMFuncRef` for the `intrinsic` specified.+ #[inline]+ pub fn unsafe_intrinsic_func_ref(&self, intrinsic: UnsafeIntrinsic) -> u32 {+ assert!(intrinsic.index() < self.num_unsafe_intrinsics);+ self.unsafe_intrinsic_func_refs()+ + intrinsic.index() * u32::from(self.ptr.size_of_vm_func_ref())+ }+/// The offset of the `lowerings` field.#[inline]pub fn lowerings(&self) -> u32 {diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rsindex f09d108d6e..f8b88cf7c7 100644--- a/crates/environ/src/key.rs+++ b/crates/environ/src/key.rs@@ -39,6 +39,10 @@ pub enum FuncKeyKind {/// A Wasm-caller to array-callee `resource.drop` trampoline.#[cfg(feature = "component-model")]ResourceDropTrampoline = FuncKey::new_kind(0b110),++ /// A Wasmtime unsafe intrinsic function.+ #[cfg(feature = "component-model")]+ UnsafeIntrinsic = FuncKey::new_kind(0b111),}impl From<FuncKeyKind> for u32 {@@ -68,6 +72,8 @@ impl FuncKeyKind {x if x == Self::ComponentTrampoline.into() => Self::ComponentTrampoline,#[cfg(feature = "component-model")]x if x == Self::ResourceDropTrampoline.into() => Self::ResourceDropTrampoline,+ #[cfg(feature = "component-model")]+ x if x == Self::UnsafeIntrinsic.into() => Self::UnsafeIntrinsic,_ => panic!("invalid raw value passed to `FuncKind::from_raw`: {raw}"),}@@ -132,6 +138,12 @@ impl FuncKeyNamespace {assert_eq!(raw & FuncKey::MODULE_MASK, 0);Self(raw)}++ #[cfg(feature = "component-model")]+ FuncKeyKind::UnsafeIntrinsic => {+ let _ = Abi::from_raw(raw & FuncKey::MODULE_MASK);+ Self(raw)+ }}}@@ -223,6 +235,10 @@ pub enum FuncKey {/// A Wasm-caller to array-callee `resource.drop` trampoline.#[cfg(feature = "component-model")]ResourceDropTrampoline,++ /// A Wasmtime intrinsic function.+ #[cfg(feature = "component-model")]+ UnsafeIntrinsic(Abi, component::UnsafeIntrinsic),}impl Ord for FuncKey {@@ -298,6 +314,14 @@ impl FuncKey {let index = 0;(namespace, index)}+ #[cfg(feature = "component-model")]+ FuncKey::UnsafeIntrinsic(abi, intrinsic) => {+ let abi = abi.into_raw();+ assert_eq!(abi & Self::KIND_MASK, 0);+ let namespace = FuncKeyKind::UnsafeIntrinsic.into_raw() | abi;+ let index = intrinsic.index();+ (namespace, index)+ }};(FuncKeyNamespace(namespace), FuncKeyIndex(index))}@@ -389,6 +413,12 @@ impl FuncKey {assert_eq!(b, 0);Self::ResourceDropTrampoline}+ #[cfg(feature = "component-model")]+ FuncKeyKind::UnsafeIntrinsic => {+ let abi = Abi::from_raw(a & Self::MODULE_MASK);+ let intrinsic = component::UnsafeIntrinsic::from_u32(b);+ Self::UnsafeIntrinsic(abi, intrinsic)+ }}}diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rsindex fb76f0d1d7..c5c0643c43 100644--- a/crates/environ/src/module_artifacts.rs+++ b/crates/environ/src/module_artifacts.rs@@ -509,7 +509,9 @@ impl CompiledFunctionsTable {FuncKeyKind::ArrayToWasmTrampoline | FuncKeyKind::WasmToBuiltinTrampoline => false,#[cfg(feature = "component-model")]- FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline => true,+ FuncKeyKind::ComponentTrampoline+ | FuncKeyKind::ResourceDropTrampoline+ | FuncKeyKind::UnsafeIntrinsic => true,}}@@ -522,7 +524,9 @@ impl CompiledFunctionsTable {| FuncKeyKind::WasmToBuiltinTrampoline| FuncKeyKind::PulleyHostCall => false,#[cfg(feature = "component-model")]- FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline => false,+ FuncKeyKind::ComponentTrampoline+ | FuncKeyKind::ResourceDropTrampoline+ | FuncKeyKind::UnsafeIntrinsic => false,}}}diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex 08e09beee7..60c88a2c52 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -238,6 +238,11 @@ pub trait PtrSize {self.vmstore_context_last_wasm_entry_trap_handler() + self.size()}+ /// Return the offset of the `stack_chain` field of `VMStoreContext`.+ fn vmstore_context_store_data(&self) -> u8 {+ self.vmstore_context_stack_chain() + self.size_of_vmstack_chain()+ }+// Offsets within `VMMemoryDefinition`/// The offset of the `base` field.diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rsindex 91fd68244f..bdf885263a 100644--- a/crates/wasmtime/src/compile.rs+++ b/crates/wasmtime/src/compile.rs@@ -61,7 +61,7 @@ mod runtime;/// Additionally compilation returns an `Option` here which is always/// `Some`, notably compiled metadata about the module in addition to the/// type information found within.-pub(crate) fn build_artifacts<T: FinishedObject>(+pub(crate) fn build_module_artifacts<T: FinishedObject>(engine: &Engine,wasm: &[u8],dwarf_package: Option<&[u8]>,@@ -142,6 +142,7 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(engine: &Engine,binary: &[u8],_dwarf_package: Option<&[u8]>,+ unsafe_intrinsics_import: Option<&str>,obj_state: &T::State,) -> Result<(T, Option<wasmtime_environ::component::ComponentArtifacts>)> {use wasmtime_environ::ScopeVec;@@ -155,10 +156,13 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(let scope = ScopeVec::new();let mut validator = wasmparser::Validator::new_with_features(engine.features());let mut types = ComponentTypesBuilder::new(&validator);- let (component, mut module_translations) =- Translator::new(tunables, &mut validator, &mut types, &scope)- .translate(binary)- .context("failed to parse WebAssembly module")?;+ let mut translator = Translator::new(tunables, &mut validator, &mut types, &scope);+ if let Some(name) = unsafe_intrinsics_import {+ translator.expose_unsafe_intrinsics(name);+ }+ let (component, mut module_translations) = translator+ .translate(binary)+ .context("failed to parse WebAssembly module")?;let compile_inputs = CompileInputs::for_component(engine,@@ -228,12 +232,10 @@ struct CompileOutput<'a> {struct InlineHeuristicParams<'a> {tunables: &'a Tunables,caller_size: u32,- caller_module: StaticModuleIndex,- caller_def_func: DefinedFuncIndex,+ caller_key: FuncKey,caller_needs_gc_heap: bool,callee_size: u32,- callee_module: StaticModuleIndex,- callee_def_func: DefinedFuncIndex,+ callee_key: FuncKey,callee_needs_gc_heap: bool,}@@ -280,21 +282,60 @@ impl<'a> CompileInputs<'a> {>,) -> Self {use wasmtime_environ::Abi;+ use wasmtime_environ::component::UnsafeIntrinsic;let mut ret = CompileInputs { inputs: vec![] };ret.collect_inputs_in_translations(types.module_types_builder(), module_translations);let tunables = engine.tunables();+ for i in component+ .component+ .unsafe_intrinsics+ .iter()+ .enumerate()+ .filter_map(|(i, ty)| if ty.is_some() { Some(i) } else { None })+ {+ let i = u32::try_from(i).unwrap();+ let intrinsic = UnsafeIntrinsic::from_u32(i);+ for abi in [Abi::Wasm, Abi::Array] {+ ret.push_input(move |compiler| {+ let symbol = format!(+ "unsafe-intrinsics-{}-{}",+ match abi {+ Abi::Wasm => "wasm-call",+ Abi::Array => "array-call",+ },+ intrinsic.name(),+ );+ Ok(CompileOutput {+ key: FuncKey::UnsafeIntrinsic(abi, intrinsic),+ function: compiler+ .component_compiler()+ .compile_intrinsic(tunables, component, intrinsic, abi, &symbol)+ .with_context(|| format!("failed to compile `{symbol}`"))?,+ symbol,+ start_srcloc: FilePos::default(),+ translation: None,+ func_body: None,+ })+ });+ }+ }+for (idx, trampoline) in component.trampolines.iter() {for abi in [Abi::Wasm, Abi::Array] {ret.push_input(move |compiler| {let key = FuncKey::ComponentTrampoline(abi, idx);- let mut symbol = trampoline.symbol_name();- symbol.push_str(match abi {- Abi::Wasm => "_wasm_call",- Abi::Array => "_array_call",- });+ let symbol = format!(+ "component-trampolines[{}]-{}-{}",+ idx.as_u32(),+ match abi {+ Abi::Wasm => "wasm-call",+ Abi::Array => "array-call",+ },+ trampoline.symbol_name(),+ );let function = compiler.component_compiler().compile_trampoline(component, types, key, abi, tunables, &symbol)@@ -521,6 +562,18 @@ the use case.engine.run_maybe_parallel(self.inputs, |f| f(compiler))?};+ if cfg!(debug_assertions) {+ let mut symbols: Vec<_> = raw_outputs.iter().map(|i| &i.symbol).collect();+ symbols.sort();+ for w in symbols.windows(2) {+ assert_ne!(+ w[0], w[1],+ "should never have duplicate symbols, but found two functions with the symbol `{}`",+ w[0]+ );+ }+ }+// Now that all functions have been compiled see if any// wasmtime-builtin functions are necessary. If so those need to be// collected and then those trampolines additionally need to be@@ -553,14 +606,46 @@ the use case.engine.run_maybe_parallel(self.inputs, |f| f(compiler).map(Some))?,);- /// Get just the output indices of the Wasm functions from our unlinked- /// outputs.- fn wasm_functions<'a>(+ /// Whether a function (as described by the given `FuncKey`) can+ /// participate in inlining or not (either as a candidate for being+ /// inlined into a caller or having a callee inlined into a callsite+ /// within itself).+ fn is_inlining_function(key: FuncKey) -> bool {+ match key {+ // Wasm functions can both be inlined into other functions and+ // have other functions inlined into them.+ FuncKey::DefinedWasmFunction(..) => true,++ // Intrinsics can be inlined into other functions.+ #[cfg(feature = "component-model")]+ FuncKey::UnsafeIntrinsic(..) => true,++ // Trampolines cannot participate in inlining since our+ // unwinding and exceptions infrastructure relies on them being+ // in their own call frames.+ FuncKey::ArrayToWasmTrampoline(..)+ | FuncKey::WasmToArrayTrampoline(..)+ | FuncKey::WasmToBuiltinTrampoline(..) => false,+ #[cfg(feature = "component-model")]+ FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,++ FuncKey::PulleyHostCall(_) => {+ unreachable!("we don't compile artifacts for Pulley host calls")+ }+ }+ }++ /// Get just the output indices of the functions that can participate in+ /// inlining from our unlinked outputs.+ fn inlining_functions<'a>(outputs: &'a PrimaryMap<OutputIndex, Option<CompileOutput<'_>>>,) -> impl Iterator<Item = OutputIndex> + 'a {- outputs.iter().filter_map(|(i, o)| match o.as_ref()?.key {- FuncKey::DefinedWasmFunction(..) => Some(i),- _ => None,+ outputs.iter().filter_map(|(index, output)| {+ if is_inlining_function(output.as_ref().unwrap().key) {+ Some(index)+ } else {+ None+ }})}@@ -571,16 +656,9 @@ the use case.// example when we know that one module's function import is always// satisfied with a particular `FuncKey::DefinedWasmFunction`. This map// enables that translation.- let key_to_output: HashMap<FuncKey, OutputIndex> = outputs- .iter()- .filter(|(_, output)| {- matches!(- output.as_ref().unwrap().key,- FuncKey::DefinedWasmFunction(..)- )- })- .map(|(output_index, output)| {- let output = output.as_ref().unwrap();+ let key_to_output: HashMap<FuncKey, OutputIndex> = inlining_functions(&outputs)+ .map(|output_index| {+ let output = outputs[output_index].as_ref().unwrap();(output.key, output_index)}).collect();@@ -590,7 +668,7 @@ the use case.// We only inline Wasm functions, not trampolines, because we rely on// trampolines being in their own stack frame when we save the entry and// exit SP, FP, and PC for backtraces in trampolines.- let call_graph = CallGraph::<OutputIndex>::new(wasm_functions(&outputs), {+ let call_graph = CallGraph::<OutputIndex>::new(inlining_functions(&outputs), {let mut func_keys = IndexSet::default();let outputs = &outputs;let key_to_output = &key_to_output;@@ -598,7 +676,7 @@ the use case.debug_assert!(calls.is_empty());let output = outputs[output_index].as_ref().unwrap();- debug_assert!(matches!(output.key, FuncKey::DefinedWasmFunction(..)));+ debug_assert!(is_inlining_function(output.key));// Get this function's call graph edges as `FuncKey`s.func_keys.clear();@@ -612,6 +690,7 @@ the use case..copied().filter_map(|key| key_to_output.get(&key)),);+log::trace!("call graph edges for {output_index:?} = {:?}: {calls:?}",output.key@@ -625,7 +704,8 @@ the use case.// (because they either do not call each other or are part of a// mutual-recursion cycle; either way we won't inline members of the// same layer into each other).- let strata = stratify::Strata::<OutputIndex>::new(wasm_functions(&outputs), &call_graph);+ let strata =+ stratify::Strata::<OutputIndex>::new(inlining_functions(&outputs), &call_graph);let mut layer_outputs = vec![];for layer in strata.layers() {// Temporarily take this layer's outputs out of our unlinked outputs@@ -641,17 +721,17 @@ the use case.&mut layer_outputs,|output: &mut CompileOutput<'_>| {log::trace!("processing inlining for {:?}", output.key);- debug_assert!(matches!(output.key, FuncKey::DefinedWasmFunction(..)));+ debug_assert!(is_inlining_function(output.key));- let caller_translation = output.translation.unwrap();- let (caller_module, caller_def_func) =- output.key.unwrap_defined_wasm_function();- let caller_needs_gc_heap = caller_translation.module.needs_gc_heap;+ let caller_key = output.key;+ let caller_needs_gc_heap =+ output.translation.is_some_and(|t| t.module.needs_gc_heap);let caller = &mut output.function;let mut caller_size = inlining_compiler.size(caller);inlining_compiler.inline(caller, &mut |callee_key: FuncKey| {+ log::trace!(" --> considering call to {callee_key:?}");let callee_output_index: OutputIndex = key_to_output[&callee_key];// NB: If the callee is not inside `outputs`, then it is@@ -667,20 +747,17 @@ the use case.let callee = &callee_output.function;let callee_size = inlining_compiler.size(callee);- let callee_needs_gc_heap =- callee_output.translation.unwrap().module.needs_gc_heap;+ let callee_needs_gc_heap = callee_output+ .translation+ .is_some_and(|t| t.module.needs_gc_heap);- let (callee_module, callee_def_func) =- callee_key.unwrap_defined_wasm_function();if Self::should_inline(InlineHeuristicParams {tunables: engine.tunables(),caller_size,- caller_module,- caller_def_func,+ caller_key,caller_needs_gc_heap,callee_size,- callee_module,- callee_def_func,+ callee_key,callee_needs_gc_heap,}) {caller_size = caller_size.saturating_add(callee_size);@@ -736,21 +813,19 @@ the use case.InlineHeuristicParams {tunables,caller_size,- caller_module,- caller_def_func,+ caller_key,caller_needs_gc_heap,callee_size,- callee_module,- callee_def_func,+ callee_key,callee_needs_gc_heap,}: InlineHeuristicParams,) -> bool {log::trace!("considering inlining:\n\- \tcaller = ({caller_module:?}, {caller_def_func:?})\n\+ \tcaller = {caller_key:?}\n\\t\tsize = {caller_size}\n\\t\tneeds_gc_heap = {caller_needs_gc_heap}\n\- \tcallee = ({callee_module:?}, {callee_def_func:?})\n\+ \tcallee = {callee_key:?}\n\\t\tsize = {callee_size}\n\\t\tneeds_gc_heap = {callee_needs_gc_heap}");@@ -759,10 +834,7 @@ the use case.tunables.inlining,"shouldn't even call this method if we aren't configured for inlining");- debug_assert!(- caller_module != callee_module || caller_def_func != callee_def_func,- "we never inline recursion"- );+ debug_assert_ne!(caller_key, callee_key, "we never inline recursion");// Put a limit on how large we can make a function via inlining to cap// code bloat.@@ -783,23 +855,33 @@ the use case.// function calls to small callees were presumably annotated with the// equivalent of `#[inline(never)]` or `#[cold]` but we don't have that// information anymore.- if caller_module == callee_module {- match tunables.inlining_intra_module {- IntraModuleInlining::Yes => {}-- IntraModuleInlining::WhenUsingGc- if caller_needs_gc_heap || callee_needs_gc_heap => {}-- IntraModuleInlining::WhenUsingGc => {- log::trace!(" --> not inlining: intra-module call that does not use GC");- return false;- }+ match (caller_key, callee_key) {+ (+ FuncKey::DefinedWasmFunction(caller_module, _),+ FuncKey::DefinedWasmFunction(callee_module, _),+ ) => {+ if caller_module == callee_module {+ match tunables.inlining_intra_module {+ IntraModuleInlining::Yes => {}++ IntraModuleInlining::WhenUsingGc+ if caller_needs_gc_heap || callee_needs_gc_heap => {}++ IntraModuleInlining::WhenUsingGc => {+ log::trace!(+ " --> not inlining: intra-module call that does not use GC"+ );+ return false;+ }- IntraModuleInlining::No => {- log::trace!(" --> not inlining: intra-module call");- return false;+ IntraModuleInlining::No => {+ log::trace!(" --> not inlining: intra-module call");+ return false;+ }+ }}}+ _ => {}}// Small callees are often worth inlining regardless of the size of thediff --git a/crates/wasmtime/src/compile/code_builder.rs b/crates/wasmtime/src/compile/code_builder.rsindex fb7737d82b..c7973bff5a 100644--- a/crates/wasmtime/src/compile/code_builder.rs+++ b/crates/wasmtime/src/compile/code_builder.rs@@ -40,6 +40,7 @@ pub struct CodeBuilder<'a> {wasm_path: Option<Cow<'a, Path>>,dwarf_package: Option<Cow<'a, [u8]>>,dwarf_package_path: Option<Cow<'a, Path>>,+ unsafe_intrinsics_import: Option<String>,}/// Return value of [`CodeBuilder::hint`]@@ -60,6 +61,7 @@ impl<'a> CodeBuilder<'a> {wasm_path: None,dwarf_package: None,dwarf_package_path: None,+ unsafe_intrinsics_import: None,}}@@ -181,6 +183,472 @@ impl<'a> CodeBuilder<'a> {.ok_or_else(|| anyhow!("no wasm bytes have been configured"))}+ /// Expose Wasmtime's unsafe intrinsics under the given import name.+ ///+ /// These intrinsics provide native memory loads and stores to Wasm; they+ /// are *extremely* unsafe! If you are not absolutely sure that you need+ /// these unsafe intrinsics, *do not use them!* See the safety section below+ /// for details.+ ///+ /// This functionality is intended to be used when implementing+ /// "compile-time builtins"; that is, satisfying a Wasm import via+ /// special-cased, embedder-specific code at compile time. You should never+ /// use these intrinsics to intentionally subvert the Wasm sandbox. You+ /// should strive to implement safe functions that encapsulate your uses of+ /// these intrinsics such that, regardless of any value given as arguments,+ /// your functions *cannot* result in loading from or storing to invalid+ /// pointers, or any other kind of unsafety. See below for an example of the+ /// intended use cases.+ ///+ /// Wasmtime's unsafe intrinsics can only be exposed to Wasm components, not+ /// core modules, currently.+ ///+ /// # Safety+ ///+ /// Extreme care must be taken when using these intrinsics.+ ///+ /// All loads of or stores to pointers derived from `store-data-address` are+ /// inherently tied to a particular `T` type in a `Store<T>`. It is wildly+ /// unsafe to run a Wasm program that uses unsafe intrinsics to access the+ /// store's `T` inside a `Store<U>`. You must only run Wasm that uses unsafe+ /// intrinsics in a `Store<T>` where the `T` is the type expected by the+ /// Wasm's unsafe-intrinsics usage.+ ///+ /// Furthermore, usage of these intrinsics is not only tied to a particular+ /// `T` type, but also to `T`'s layout on the host platform. The size and+ /// alignment of `T`, the offsets of its fields, and those fields' size and+ /// alignment can all vary across not only architecture but also operating+ /// system. With care, you can define your `T` type such that its layout is+ /// identical across the platforms that you run Wasm on, allowing you to+ /// reuse the same Wasm binary and its unsafe-intrinsics usage on all your+ /// platforms. Failing that, you must only run a Wasm program that uses+ /// unsafe intrinsics on the host platform that its unsafe-intrinsic usage+ /// is specialized to. See the portability section and example below for+ /// more details.+ ///+ /// You are *strongly* encouraged to add assertions for the layout+ /// properties that your unsafe-intrinsic usage's safety relies upon:+ ///+ /// ```rust+ /// /// This type is used as `wasmtime::Store<MyData>` and accessed by Wasm via+ /// /// unsafe intrinsics.+ /// #[repr(C, align(8))]+ /// struct MyData {+ /// id: u64,+ /// counter: u32,+ /// buf: [u8; 4],+ /// }+ ///+ /// // Assert that the layout is what our Wasm's unsafe-intrinsics usage expects.+ /// static _MY_DATA_LAYOUT_ASSERTIONS: () = {+ /// assert!(core::mem::size_of::<MyData>() == 16);+ /// assert!(core::mem::align_of::<MyData>() == 8);+ /// assert!(core::mem::offset_of!(MyData, id) == 0);+ /// assert!(core::mem::offset_of!(MyData, counter) == 8);+ /// assert!(core::mem::offset_of!(MyData, buf) == 12);+ /// };+ /// ```+ ///+ /// Finally, every pointer loaded from or stored to must:+ ///+ /// * Be non-null+ ///+ /// * Be aligned to the access type's natural alignment (e.g. 8-byte alignment+ /// for `u64`, 4-byte alignment for `u32`, etc...)+ ///+ /// * Point to a memory block that is valid to read from (for loads) or+ /// valid to write to (for stores) under Rust's pointer provenance rules+ ///+ /// * Point to a memory block that is at least as large as the access type's+ /// natural size (e.g. 1 byte for `u8`, 2 bytes for `u16`, etc...)+ ///+ /// * Point to a memory block that is not accessed concurrently by any other+ /// threads+ ///+ /// Failure to uphold any of these invariants will lead to unsafety,+ /// undefined behavior, and/or data races.+ ///+ /// # Intrinsics+ ///+ /// | Name | Parameters | Results |+ /// |----------------------|--------------|---------|+ /// | `u8-native-load` | `u64` | `u8` |+ /// | `u16-native-load` | `u64` | `u16` |+ /// | `u32-native-load` | `u64` | `u32` |+ /// | `u64-native-load` | `u64` | `u64` |+ /// | `u8-native-store` | `u64`, `u8` | - |+ /// | `u16-native-load` | `u64`, `u16` | - |+ /// | `u32-native-load` | `u64`, `u32` | - |+ /// | `u64-native-load` | `u64`, `u64` | - |+ /// | `store-data-address` | - | `u64` |+ ///+ /// ## `*-native-load`+ ///+ /// These intrinsics perform an unsandboxed, unsynchronized load from native+ /// memory, using the native endianness.+ ///+ /// ## `*-native-store`+ ///+ /// These intrinsics perform an unsandboxed, unsynchronized store to native+ /// memory, using the native endianness.+ ///+ /// ## `store-data-address`+ ///+ /// This intrinsic function returns the pointer to the embedder's `T` data+ /// inside a `Store<T>`.+ ///+ /// In general, all native load and store intinsics should operate on memory+ /// addresses that are derived from a call to this intrinsic. If you want to+ /// expose data for raw memory access by Wasm, put it inside the `T` in your+ /// `Store<T>` and Wasm's access to that data should derive from this+ /// intrinsic.+ ///+ /// # Portability+ ///+ /// Loads and stores are always performed using the architecture's native+ /// endianness.+ ///+ /// Addresses passed to and returned from these intrinsics are always+ /// 64-bits large. The upper half of the value is simply ignored on 32-bit+ /// architectures.+ ///+ /// With care, you can design your store's `T` type such that accessing it+ /// via these intrinsics is portable, and you can reuse a single Wasm binary+ /// (and its set of intrinsic calls) across all of the platforms, with the+ /// following rules of thumb:+ ///+ /// * Only access `u8`, `u16`, `u32`, and `u64` data via these intrinsics.+ ///+ /// * If you need to access other types of data, encode it into those types+ /// and then access the encoded data from the intrinsics.+ ///+ /// * Use `union`s to encode pointers and pointer-sized data as a `u64` and+ /// then access it via the `u64-native-{load,store}` intrinsics. See+ /// `ExposedPointer` in the example below.+ ///+ /// # Example+ ///+ /// The following example shows how you can use unsafe intrinsics to give+ /// Wasm direct zero-copy access to a host buffer.+ ///+ /// ```rust+ /// use std::mem;+ /// use wasmtime::*;+ ///+ /// // A `*mut u8` pointer that is exposed directly to Wasm via unsafe intrinsics.+ /// #[repr(align(8))]+ /// union ExposedPointer {+ /// pointer: *mut u8,+ /// padding: u64,+ /// }+ ///+ /// static _EXPOSED_POINTER_LAYOUT_ASSERTIONS: () = {+ /// assert!(mem::size_of::<ExposedPointer>() == 8);+ /// assert!(mem::align_of::<ExposedPointer>() == 8);+ /// };+ ///+ /// impl ExposedPointer {+ /// /// Wrap the given pointer into an `ExposedPointer`.+ /// fn new(pointer: *mut u8) -> Self {+ /// // NB: Zero-initialize to avoid potential footguns with accessing+ /// // undefined bytes.+ /// let mut p = Self { padding: 0 };+ /// p.pointer = pointer;+ /// p+ /// }+ ///+ /// /// Get the wrapped pointer.+ /// fn get(&self) -> *mut u8 {+ /// unsafe { self.pointer }+ /// }+ /// }+ ///+ /// /// This is the `T` type we will put inside our+ /// /// `wasmtime::Store<T>`s. It contains a pointer to a heap-allocated buffer+ /// /// in host memory, which we will give Wasm zero-copy access to via unsafe+ /// /// intrinsics.+ /// #[repr(C)]+ /// struct StoreData {+ /// buf_ptr: ExposedPointer,+ /// buf_len: u64,+ /// }+ ///+ /// static _STORE_DATA_LAYOUT_ASSERTIONS: () = {+ /// assert!(mem::size_of::<StoreData>() == 16);+ /// assert!(mem::align_of::<StoreData>() == 8);+ /// assert!(mem::offset_of!(StoreData, buf_ptr) == 0);+ /// assert!(mem::offset_of!(StoreData, buf_len) == 8);+ /// };+ ///+ /// impl Drop for StoreData {+ /// fn drop(&mut self) {+ /// let len = usize::try_from(self.buf_len).unwrap();+ /// let ptr = std::ptr::slice_from_raw_parts_mut(self.buf_ptr.get(), len);+ /// unsafe {+ /// let _ = Box::from_raw(ptr);+ /// }+ /// }+ /// }+ ///+ /// impl StoreData {+ /// /// Create a new `StoreData`, allocating an inner buffer of `capacity`+ /// /// bytes.+ /// fn new(bytes: impl IntoIterator<Item = u8>) -> Self {+ /// let buf: Box<[u8]> = bytes.into_iter().collect();+ /// let ptr = Box::into_raw(buf);+ /// Self {+ /// buf_ptr: ExposedPointer::new(ptr.cast::<u8>()),+ /// buf_len: u64::try_from(ptr.len()).unwrap(),+ /// }+ /// }+ ///+ /// /// Get the inner buffer as a shared slice.+ /// fn buf(&self) -> &[u8] {+ /// let ptr = self.buf_ptr.get().cast_const();+ /// let len = usize::try_from(self.buf_len).unwrap();+ /// unsafe {+ /// std::slice::from_raw_parts(ptr, len)+ /// }+ /// }+ ///+ /// /// Get the inner buffer as a mutable slice.+ /// fn buf_mut(&mut self) -> &mut [u8] {+ /// let ptr = self.buf_ptr.get();+ /// let len = usize::try_from(self.buf_len).unwrap();+ /// unsafe {+ /// std::slice::from_raw_parts_mut(ptr, len)+ /// }+ /// }+ /// }+ ///+ /// # fn main() -> Result<()> {+ /// // Enable function inlining during compilation. If you are using unsafe intrinsics, you+ /// // almost assuredly want them inlined to avoid function call overheads.+ /// let mut config = Config::new();+ /// config.compiler_inlining(true);+ ///+ /// let engine = Engine::new(&config)?;+ /// let linker = wasmtime::component::Linker::new(&engine);+ ///+ /// // Create a new builder for configuring a Wasm compilation.+ /// let mut builder = CodeBuilder::new(&engine);+ ///+ /// // Allow the code we are building to use Wasmtime's unsafe intrinsics.+ /// //+ /// // SAFETY: we wrap all usage of the intrinsics in safe APIs and only instantiate the code+ /// // within a `Store<T>` where `T = StoreData`, as the code expects.+ /// unsafe {+ /// builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ /// }+ ///+ /// // Provide the Wasm that we are compiling.+ /// builder.wasm_binary_or_text(r#"+ /// (component+ /// ;; Import the unsafe intrinsics that we will use.+ /// (import "unsafe-intrinsics"+ /// (instance $intrinsics+ /// (export "store-data-address" (func (result u64)))+ /// (export "u64-native-load" (func (param "pointer" u64) (result u64)))+ /// (export "u8-native-load" (func (param "pointer" u64) (result u8)))+ /// (export "u8-native-store" (func (param "pointer" u64) (param "value" u8)))+ /// )+ /// )+ ///+ /// ;; A component that encapsulates the intrinsics' unsafety, exposing a safe API+ /// ;; built on top of them.+ /// (component $safe-api+ /// (import "unsafe-intrinsics"+ /// (instance $intrinsics+ /// (export "store-data-address" (func (result u64)))+ /// (export "u64-native-load" (func (param "pointer" u64) (result u64)))+ /// (export "u8-native-load" (func (param "pointer" u64) (result u8)))+ /// (export "u8-native-store" (func (param "pointer" u64) (param "value" u8)))+ /// )+ /// )+ ///+ /// ;; The core Wasm module that implements the safe API.+ /// (core module $safe-api-impl+ /// (import "" "store-data-address" (func $store-data-address (result i64)))+ /// (import "" "u64-native-load" (func $u64-native-load (param i64) (result i64)))+ /// (import "" "u8-native-load" (func $u8-native-load (param i64) (result i32)))+ /// (import "" "u8-native-store" (func $u8-native-store (param i64 i32)))+ ///+ /// ;; Load the `StoreData::buf_ptr` field+ /// (func $get-buf-ptr (result i64)+ /// (call $u64-native-load (i64.add (call $store-data-address) (i64.const 0)))+ /// )+ ///+ /// ;; Load the `StoreData::buf_len` field+ /// (func $get-buf-len (result i64)+ /// (call $u64-native-load (i64.add (call $store-data-address) (i64.const 8)))+ /// )+ ///+ /// ;; Check that `$i` is within `StoreData` buffer's bounds, raising a trap+ /// ;; otherwise.+ /// (func $bounds-check (param $i i64)+ /// (if (i64.lt_u (local.get $i) (call $get-buf-len))+ /// (then (return))+ /// (else (unreachable))+ /// )+ /// )+ ///+ /// ;; A safe function to get the `i`th byte from `StoreData`'s buffer, raising+ /// ;; a trap on out-of-bounds accesses.+ /// (func (export "get") (param $i i64) (result i32)+ /// (call $bounds-check (local.get $i))+ /// (call $u8-native-load (i64.add (call $get-buf-ptr) (local.get $i)))+ /// )+ ///+ /// ;; A safe function to set the `i`th byte in `StoreData`'s buffer, raising+ /// ;; a trap on out-of-bounds accesses.+ /// (func (export "set") (param $i i64) (param $value i32)+ /// (call $bounds-check (local.get $i))+ /// (call $u8-native-store (i64.add (call $get-buf-ptr) (local.get $i))+ /// (local.get $value))+ /// )+ ///+ /// ;; A safe function to get the length of the `StoreData` buffer.+ /// (func (export "len") (result i64)+ /// (call $get-buf-len)+ /// )+ /// )+ ///+ /// ;; Lower the imported intrinsics from component functions to core functions.+ /// (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))+ /// (core func $u64-native-load' (canon lower (func $intrinsics "u64-native-load")))+ /// (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))+ /// (core func $u8-native-store' (canon lower (func $intrinsics "u8-native-store")))+ ///+ /// ;; Instantiate our safe API implementation, passing in the lowered unsafe+ /// ;; intrinsics as its imports.+ /// (core instance $instance+ /// (instantiate $safe-api-impl+ /// (with "" (instance+ /// (export "store-data-address" (func $store-data-address'))+ /// (export "u64-native-load" (func $u64-native-load'))+ /// (export "u8-native-load" (func $u8-native-load'))+ /// (export "u8-native-store" (func $u8-native-store'))+ /// ))+ /// )+ /// )+ ///+ /// ;; Lift the safe API's exports from core functions to component functions and+ /// ;; export them.+ /// (func (export "get") (param "i" u64) (result u8)+ /// (canon lift (core func $instance "get"))+ /// )+ /// (func (export "set") (param "i" u64) (param "value" u8)+ /// (canon lift (core func $instance "set"))+ /// )+ /// (func (export "len") (result u64)+ /// (canon lift (core func $instance "len"))+ /// )+ /// )+ ///+ /// ;; A component that uses that safe API to increment each byte in the+ /// ;; `StoreData` buffer.+ /// (component $main+ /// ;; Import the safe API.+ /// (import "safe-api"+ /// (instance $safe-api+ /// (export "get" (func (param "i" u64) (result u8)))+ /// (export "set" (func (param "i" u64) (param "value" u8)))+ /// (export "len" (func (result u64)))+ /// )+ /// )+ ///+ /// ;; Define this component's core module implementation.+ /// (core module $main-impl+ /// (import "" "get" (func $get (param i64) (result i32)))+ /// (import "" "set" (func $set (param i64 i32)))+ /// (import "" "len" (func $len (result i64)))+ ///+ /// (func (export "main")+ /// (local $i i64)+ /// (local $n i64)+ ///+ /// (local.set $i (i64.const 0))+ /// (local.set $n (call $len))+ ///+ /// (loop $loop+ /// ;; When we have iterated over every byte in the+ /// ;; buffer, exit.+ /// (if (i64.ge_u (local.get $i) (local.get $n))+ /// (then (return)))+ ///+ /// ;; Increment the `i`th byte in the buffer.+ /// (call $set (local.get $i)+ /// (i32.add (call $get (local.get $i))+ /// (i32.const 1)))+ ///+ /// ;; Increment `i` and continue to the next iteration+ /// ;; of the loop.+ /// (local.set $i (i64.add (local.get $i) (i64.const 1)))+ /// (br $loop)+ /// )+ /// )+ /// )+ ///+ /// ;; Lower the imported safe APIs from component functions to core functions.+ /// (core func $get' (canon lower (func $safe-api "get")))+ /// (core func $set' (canon lower (func $safe-api "set")))+ /// (core func $len' (canon lower (func $safe-api "len")))+ ///+ /// ;; Instantiate our module, providing the lowered safe APIs as imports.+ /// (core instance $instance+ /// (instantiate $main-impl+ /// (with "" (instance+ /// (export "get" (func $get'))+ /// (export "set" (func $set'))+ /// (export "len" (func $len'))+ /// ))+ /// )+ /// )+ ///+ /// ;; Lift implementation's `main` from a core function to a component function+ /// ;; and export it.+ /// (func (export "main")+ /// (canon lift (core func $instance "main"))+ /// )+ /// )+ ///+ /// ;; Instantiate our safe API component and our main component that consumes+ /// ;; it, passing the unsafe intrinsics into the safe API, and the safe API+ /// ;; into the main component.+ /// (instance $safe-api-instance+ /// (instantiate $safe-api (with "unsafe-intrinsics" (instance $intrinsics))))+ /// (instance $main-instance+ /// (instantiate $main (with "safe-api" (instance $safe-api-instance))))+ ///+ /// ;; Finally, re-export the `main` function!+ /// (export "main" (func $main-instance "main"))+ /// )+ /// "#.as_bytes(), None)?;+ ///+ /// // Finish the builder and compile the component.+ /// let component = builder.compile_component()?;+ ///+ /// // Create a new `Store<StoreData>`, wrapping a buffer of the given elements.+ /// let mut store = Store::new(&engine, StoreData::new([0, 10, 20, 30, 40, 50]));+ ///+ /// // Instantiate our component into the store.+ /// let instance = linker.instantiate(&mut store, &component)?;+ ///+ /// // Get the instance's exported `main` function and call it.+ /// instance+ /// .get_typed_func::<(), ()>(&mut store, "main")?+ /// .call(&mut store, ())?;+ ///+ /// // Our `StoreData`'s buffer had each element incremented directly from Wasm!+ /// assert_eq!(store.data().buf(), &[1, 11, 21, 31, 41, 51]);+ /// # Ok(())+ /// # }+ /// ```+ pub unsafe fn expose_unsafe_intrinsics(&mut self, import_name: impl Into<String>) -> &mut Self {+ self.unsafe_intrinsics_import = Some(import_name.into());+ self+ }+/// Explicitly specify DWARF `.dwp` path.////// # Errors@@ -274,7 +742,12 @@ impl<'a> CodeBuilder<'a> {pub fn compile_module_serialized(&self) -> Result<Vec<u8>> {let wasm = self.get_wasm()?;let dwarf_package = self.get_dwarf_package();- let (v, _) = super::build_artifacts(self.engine, &wasm, dwarf_package.as_deref(), &())?;+ ensure!(+ self.unsafe_intrinsics_import.is_none(),+ "`CodeBuilder::expose_unsafe_intrinsics` can only be used with components"+ );+ let (v, _) =+ super::build_module_artifacts(self.engine, &wasm, dwarf_package.as_deref(), &())?;Ok(v)}@@ -284,9 +757,19 @@ impl<'a> CodeBuilder<'a> {#[cfg(feature = "component-model")]pub fn compile_component_serialized(&self) -> Result<Vec<u8>> {let bytes = self.get_wasm()?;- let (v, _) = super::build_component_artifacts(self.engine, &bytes, None, &())?;+ let (v, _) = super::build_component_artifacts(+ self.engine,+ &bytes,+ None,+ self.get_unsafe_intrinsics_import(),+ &(),+ )?;Ok(v)}++ pub(super) fn get_unsafe_intrinsics_import(&self) -> Option<&str> {+ self.unsafe_intrinsics_import.as_deref()+ }}/// This is a helper struct used when caching to hash the state of an `Engine`diff --git a/crates/wasmtime/src/compile/runtime.rs b/crates/wasmtime/src/compile/runtime.rsindex 243f85a9c1..794a5029a4 100644--- a/crates/wasmtime/src/compile/runtime.rs+++ b/crates/wasmtime/src/compile/runtime.rs@@ -14,12 +14,14 @@ impl<'a> CodeBuilder<'a> {&Engine,&[u8],Option<&[u8]>,+ Option<&str>,&S,) -> Result<(MmapVecWrapper, Option<T>)>,state: &S,) -> Result<(Arc<CodeMemory>, Option<T>)> {let wasm = self.get_wasm()?;let dwarf_package = self.get_dwarf_package();+ let unsafe_intrinsics_import = self.get_unsafe_intrinsics_import();self.engine.check_compatible_with_native_host()@@ -31,6 +33,7 @@ impl<'a> CodeBuilder<'a> {crate::compile::HashedEngineCompileEnv(self.engine),&wasm,&dwarf_package,+ &unsafe_intrinsics_import,// Don't hash this as it's just its own "pure" function pointer.NotHashed(build_artifacts),// Don't hash the FinishedObject state: this contains@@ -43,22 +46,31 @@ impl<'a> CodeBuilder<'a> {.get_data_raw(&state,// Cache miss, compute the actual artifacts- |(engine, wasm, dwarf_package, build_artifacts, state)| -> Result<_> {+ |(+ engine,+ wasm,+ dwarf_package,+ unsafe_intrinsics_import,+ build_artifacts,+ state,+ )|+ -> Result<_> {let (mmap, info) = (build_artifacts.0)(engine.0,wasm,dwarf_package.as_deref(),+ **unsafe_intrinsics_import,state.0,)?;let code = publish_mmap(engine.0, mmap.0)?;Ok((code, info))},// Implementation of how to serialize artifacts- |(_engine, _wasm, _, _, _), (code, _info_and_types)| {+ |(_engine, _wasm, _, _, _, _), (code, _info_and_types)| {Some(code.mmap().to_vec())},// Cache hit, deserialize the provided artifacts- |(engine, wasm, _, _, _), serialized_bytes| {+ |(engine, wasm, _, _, _, _), serialized_bytes| {let kind = if wasmparser::Parser::is_component(&wasm) {wasmtime_environ::ObjectKind::Component} else {@@ -79,8 +91,13 @@ impl<'a> CodeBuilder<'a> {#[cfg(not(feature = "cache"))]{- let (mmap, info_and_types) =- build_artifacts(self.engine, &wasm, dwarf_package.as_deref(), state)?;+ let (mmap, info_and_types) = build_artifacts(+ self.engine,+ &wasm,+ dwarf_package.as_deref(),+ unsafe_intrinsics_import,+ state,+ )?;let code = publish_mmap(self.engine, mmap.0)?;return Ok((code, info_and_types));}@@ -92,9 +109,18 @@ impl<'a> CodeBuilder<'a> {/// Note that this method will cache compilations if the `cache` feature is/// enabled and turned on in [`Config`](crate::Config).pub fn compile_module(&self) -> Result<Module> {+ ensure!(+ self.get_unsafe_intrinsics_import().is_none(),+ "`CodeBuilder::expose_unsafe_intrinsics` can only be used with components"+ );let custom_alignment = self.custom_alignment();- let (code, info_and_types) =- self.compile_cached(super::build_artifacts, &custom_alignment)?;+ let (code, info_and_types) = self.compile_cached(+ |engine, wasm, dwarf, unsafe_intrinsics_import, state| {+ assert!(unsafe_intrinsics_import.is_none());+ super::build_module_artifacts(engine, wasm, dwarf, state)+ },+ &custom_alignment,+ )?;Module::from_parts(self.engine, code, info_and_types)}@@ -103,8 +129,18 @@ impl<'a> CodeBuilder<'a> {#[cfg(feature = "component-model")]pub fn compile_component(&self) -> Result<Component> {let custom_alignment = self.custom_alignment();- let (code, artifacts) =- self.compile_cached(super::build_component_artifacts, &custom_alignment)?;+ let (code, artifacts) = self.compile_cached(+ |engine, wasm, dwarf, unsafe_intrinsics_import, state| {+ super::build_component_artifacts(+ engine,+ wasm,+ dwarf,+ unsafe_intrinsics_import,+ state,+ )+ },+ &custom_alignment,+ )?;Component::from_parts(self.engine, code, artifacts)}diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rsindex eb83c44852..2990248d9e 100644--- a/crates/wasmtime/src/runtime/component/component.rs+++ b/crates/wasmtime/src/runtime/component/component.rs@@ -18,7 +18,7 @@ use std::path::Path;use wasmtime_environ::component::{CompiledComponentInfo, ComponentArtifacts, ComponentTypes, CoreDef, Export, ExportIndex,GlobalInitializer, InstantiateModule, NameMapNoIntern, OptionsIndex, StaticModuleIndex,- TrampolineIndex, TypeComponentIndex, TypeFuncIndex, VMComponentOffsets,+ TrampolineIndex, TypeComponentIndex, TypeFuncIndex, UnsafeIntrinsic, VMComponentOffsets,};use wasmtime_environ::{Abi, CompiledFunctionsTable, FuncKey, TypeTrace};use wasmtime_environ::{FunctionLoc, HostPtr, ObjectKind, PrimaryMap};@@ -502,22 +502,46 @@ impl Component {pub(crate) fn trampoline_ptrs(&self, index: TrampolineIndex) -> AllCallFuncPointers {let wasm_call = self- .inner- .index- .func_loc(FuncKey::ComponentTrampoline(Abi::Wasm, index))- .unwrap();+ .func(FuncKey::ComponentTrampoline(Abi::Wasm, index))+ .unwrap()+ .cast();let array_call = self- .inner- .index- .func_loc(FuncKey::ComponentTrampoline(Abi::Array, index))- .unwrap();+ .func(FuncKey::ComponentTrampoline(Abi::Array, index))+ .unwrap()+ .cast();AllCallFuncPointers {- wasm_call: self.func(wasm_call).cast(),- array_call: self.func(array_call).cast(),+ wasm_call,+ array_call,}}- fn func(&self, loc: &FunctionLoc) -> NonNull<u8> {+ pub(crate) fn unsafe_intrinsic_ptrs(+ &self,+ intrinsic: UnsafeIntrinsic,+ ) -> Option<AllCallFuncPointers> {+ let wasm_call = self+ .func(FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic))?+ .cast();+ let array_call = self+ .func(FuncKey::UnsafeIntrinsic(Abi::Array, intrinsic))?+ .cast();+ Some(AllCallFuncPointers {+ wasm_call,+ array_call,+ })+ }++ /// Look up a function in this component's text section by `FuncKey`.+ fn func(&self, key: FuncKey) -> Option<NonNull<u8>> {+ let loc = self.inner.index.func_loc(key)?;+ Some(self.func_loc_to_pointer(loc))+ }++ /// Given a function location within this component's text section, get a+ /// pointer to the function.+ ///+ /// Panics on out-of-bounds function locations.+ fn func_loc_to_pointer(&self, loc: &FunctionLoc) -> NonNull<u8> {let text = self.text();let trampoline = &text[loc.start as usize..][..loc.length as usize];NonNull::from(trampoline).cast()@@ -553,10 +577,8 @@ impl Component {// then this can't be called by the component, so it's ok to leave it// blank.let wasm_call = self- .inner- .index- .func_loc(FuncKey::ResourceDropTrampoline)- .map(|loc| self.func(loc).cast().into());+ .func(FuncKey::ResourceDropTrampoline)+ .map(|f| f.cast().into());VMFuncRef {wasm_call,diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rsindex a7aa6f1624..4f67d39f65 100644--- a/crates/wasmtime/src/runtime/component/instance.rs+++ b/crates/wasmtime/src/runtime/component/instance.rs@@ -489,6 +489,15 @@ pub(crate) fn lookup_vmdef(let id = StoreComponentInstanceId::new(store.id(), id);vm::Export::Global(crate::Global::from_component_flags(id, *idx))}+ CoreDef::UnsafeIntrinsic(intrinsic) => {+ let funcref = store+ .store_data_mut()+ .component_instance_mut(id)+ .unsafe_intrinsic_func_ref(*intrinsic);+ // SAFETY: as above, the `funcref` is owned by `store` and is valid+ // within that store, so it's safe to create a `Func`.+ vm::Export::Function(unsafe { crate::Func::from_vm_func_ref(store.id(), funcref) })+ }}}@@ -678,6 +687,31 @@ impl<'a> Instantiator<'a> {);}+ // Initialize the unsafe intrinsics used by this component, if any.+ for (i, module_ty) in env_component+ .unsafe_intrinsics+ .iter()+ .enumerate()+ .filter_map(|(i, ty)| ty.expand().map(|ty| (i, ty)))+ {+ let i = u32::try_from(i).unwrap();+ let intrinsic = UnsafeIntrinsic::from_u32(i);+ let ptrs = self.component.unsafe_intrinsic_ptrs(intrinsic).expect(+ "should have intrinsic pointers given that we assigned the intrinsic a type",+ );+ let shared_ty = self+ .component+ .signatures()+ .shared_type(module_ty)+ .expect("should have a shared type");+ self.instance_mut(store.0).set_intrinsic(+ intrinsic,+ ptrs.wasm_call,+ ptrs.array_call,+ shared_ty,+ );+ }+for initializer in env_component.initializers.iter() {match initializer {GlobalInitializer::InstantiateModule(m) => {diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex a87a2c070c..4bde59d5b2 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -245,8 +245,20 @@ pub struct StoreInner<T: 'static> {#[cfg(target_has_atomic = "64")]epoch_deadline_behavior:Option<Box<dyn FnMut(StoreContextMut<T>) -> Result<UpdateDeadline> + Send + Sync>>,- // for comments about `ManuallyDrop`, see `Store::into_data`- data: ManuallyDrop<T>,++ /// The user's `T` data.+ ///+ /// Don't actually access it via this field, however! Use the+ /// `Store{,Inner,Context,ContextMut}::data[_mut]` methods instead, to+ /// preserve stacked borrows and provenance in the face of potential+ /// direct-access of `T` from Wasm code (via unsafe intrinsics).+ ///+ /// The only exception to the above is when taking ownership of the value,+ /// e.g. in `Store::into_data`, after which nothing can access this field+ /// via raw pointers anymore so there is no more provenance to preserve.+ ///+ /// For comments about `ManuallyDrop`, see `Store::into_data`.+ data_no_provenance: ManuallyDrop<T>,}enum ResourceLimiterInner<T> {@@ -703,9 +715,13 @@ impl<T> Store<T> {call_hook: None,#[cfg(target_has_atomic = "64")]epoch_deadline_behavior: None,- data: ManuallyDrop::new(data),+ data_no_provenance: ManuallyDrop::new(data),});+ let store_data =+ <NonNull<ManuallyDrop<T>>>::from(&mut inner.data_no_provenance).cast::<()>();+ inner.inner.vm_store_context.store_data = store_data.into();+inner.traitobj = StorePtr(Some(NonNull::from(&mut *inner)));// Wasmtime uses the callee argument to host functions to learn about@@ -748,13 +764,13 @@ impl<T> Store<T> {}}- /// Access the underlying data owned by this `Store`.+ /// Access the underlying `T` data owned by this `Store`.#[inline]pub fn data(&self) -> &T {self.inner.data()}- /// Access the underlying data owned by this `Store`.+ /// Access the underlying `T` data owned by this `Store`.#[inline]pub fn data_mut(&mut self) -> &mut T {self.inner.data_mut()@@ -808,7 +824,7 @@ impl<T> Store<T> {unsafe {let mut inner = ManuallyDrop::take(&mut self.inner);core::mem::forget(self);- ManuallyDrop::take(&mut inner.data)+ ManuallyDrop::take(&mut inner.data_no_provenance)}}@@ -870,7 +886,7 @@ impl<T> Store<T> {// Apply the limits on instances, tables, and memory given by the limiter:let inner = &mut self.inner;let (instance_limit, table_limit, memory_limit) = {- let l = limiter(&mut inner.data);+ let l = limiter(inner.data_mut());(l.instances(), l.tables(), l.memories())};let innermost = &mut inner.inner;@@ -1315,12 +1331,49 @@ impl<'a, T> StoreContextMut<'a, T> {impl<T> StoreInner<T> {#[inline]fn data(&self) -> &T {- &self.data+ // We are actually just accessing `&self.data_no_provenance` but we must+ // do so with the `VMStoreContext::store_data` pointer's provenance. If+ // we did otherwise, i.e. directly accessed the field, we would+ // invalidate that pointer, which would in turn invalidate any direct+ // `T` accesses that Wasm code makes via unsafe intrinsics.+ let data: *const ManuallyDrop<T> = &raw const self.data_no_provenance;+ let provenance = self.inner.vm_store_context.store_data.as_ptr().cast::<T>();+ let ptr = provenance.with_addr(data.addr());++ // SAFETY: The pointer is non-null, points to our `T` data, and is valid+ // to access because of our `&self` borrow.+ debug_assert_ne!(ptr, core::ptr::null_mut());+ debug_assert_eq!(ptr.addr(), (&raw const self.data_no_provenance).addr());+ unsafe { &*ptr }+ }++ #[inline]+ fn data_limiter_and_opaque(+ &mut self,+ ) -> (+ &mut T,+ Option<&mut ResourceLimiterInner<T>>,+ &mut StoreOpaque,+ ) {+ // See the comments about provenance in `StoreInner::data` above.+ let data: *mut ManuallyDrop<T> = &raw mut self.data_no_provenance;+ let provenance = self.inner.vm_store_context.store_data.as_ptr().cast::<T>();+ let ptr = provenance.with_addr(data.addr());++ // SAFETY: The pointer is non-null, points to our `T` data, and is valid+ // to access because of our `&mut self` borrow.+ debug_assert_ne!(ptr, core::ptr::null_mut());+ debug_assert_eq!(ptr.addr(), (&raw const self.data_no_provenance).addr());+ let data = unsafe { &mut *ptr };++ let limiter = self.limiter.as_mut();++ (data, limiter, &mut self.inner)}#[inline]fn data_mut(&mut self) -> &mut T {- &mut self.data+ self.data_limiter_and_opaque().0}#[inline]@@ -2562,14 +2615,15 @@ unsafe impl<T> VMStore for StoreInner<T> {fn resource_limiter_and_store_opaque(&mut self,) -> (Option<StoreResourceLimiter<'_>>, &mut StoreOpaque) {- (- self.limiter.as_mut().map(|l| match l {- ResourceLimiterInner::Sync(s) => StoreResourceLimiter::Sync(s(&mut self.data)),- #[cfg(feature = "async")]- ResourceLimiterInner::Async(s) => StoreResourceLimiter::Async(s(&mut self.data)),- }),- &mut self.inner,- )+ let (data, limiter, opaque) = self.data_limiter_and_opaque();++ let limiter = limiter.map(|l| match l {+ ResourceLimiterInner::Sync(s) => StoreResourceLimiter::Sync(s(data)),+ #[cfg(feature = "async")]+ ResourceLimiterInner::Async(s) => StoreResourceLimiter::Async(s(data)),+ });++ (limiter, opaque)}#[cfg(target_has_atomic = "64")]@@ -2619,7 +2673,7 @@ impl<T: fmt::Debug> fmt::Debug for Store<T> {let inner = &**self.inner as *const StoreInner<T>;f.debug_struct("Store").field("inner", &inner)- .field("data", &self.inner.data)+ .field("data", self.inner.data()).finish()}}@@ -2628,9 +2682,9 @@ impl<T> Drop for Store<T> {fn drop(&mut self) {self.run_manual_drop_routines();- // for documentation on this `unsafe`, see `into_data`.+ // For documentation on this `unsafe`, see `into_data`.unsafe {- ManuallyDrop::drop(&mut self.inner.data);+ ManuallyDrop::drop(&mut self.inner.data_no_provenance);ManuallyDrop::drop(&mut self.inner);}}@@ -2717,8 +2771,7 @@ impl<T: AsStoreOpaque + ?Sized> AsStoreOpaque for &mut T {#[cfg(test)]mod tests {- use super::{get_fuel, refuel, set_fuel};- use std::num::NonZeroU64;+ use super::*;struct FuelTank {pub consumed_fuel: i64,@@ -2835,4 +2888,35 @@ mod tests {assert_eq!(tank.consumed_fuel, 4);assert_eq!(tank.get_fuel(), 0);}++ #[test]+ fn store_data_provenance() {+ // Test that we juggle pointer provenance and all that correctly, and+ // miri is happy with everything, while allowing both Rust code and+ // "Wasm" to access and modify the store's `T` data. Note that this is+ // not actually Wasm mutating the store data here because compiling Wasm+ // under miri is way too slow.++ unsafe fn run_wasm(store: &mut Store<u32>) {+ let ptr = store+ .inner+ .inner+ .vm_store_context+ .store_data+ .as_ptr()+ .cast::<u32>();+ unsafe { *ptr += 1 }+ }++ let engine = Engine::default();+ let mut store = Store::new(&engine, 0_u32);++ assert_eq!(*store.data(), 0);+ *store.data_mut() += 1;+ assert_eq!(*store.data(), 1);+ unsafe { run_wasm(&mut store) }+ assert_eq!(*store.data(), 2);+ *store.data_mut() += 1;+ assert_eq!(*store.data(), 3);+ }}diff --git a/crates/wasmtime/src/runtime/store/async_.rs b/crates/wasmtime/src/runtime/store/async_.rsindex cefc0e192a..93e3617501 100644--- a/crates/wasmtime/src/runtime/store/async_.rs+++ b/crates/wasmtime/src/runtime/store/async_.rs@@ -51,7 +51,7 @@ impl<T> Store<T> {// Apply the limits on instances, tables, and memory given by the limiter:let inner = &mut self.inner;let (instance_limit, table_limit, memory_limit) = {- let l = limiter(&mut inner.data);+ let l = limiter(inner.data_mut());(l.instances(), l.tables(), l.memories())};let innermost = &mut inner.inner;diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rsindex 1e680cd745..079db57bd9 100644--- a/crates/wasmtime/src/runtime/vm/component.rs+++ b/crates/wasmtime/src/runtime/vm/component.rs@@ -428,6 +428,20 @@ impl ComponentInstance {}}+ /// Get the core Wasm function reference for the given unsafe intrinsic.+ pub fn unsafe_intrinsic_func_ref(&self, idx: UnsafeIntrinsic) -> NonNull<VMFuncRef> {+ unsafe {+ let offset = self.offsets.unsafe_intrinsic_func_ref(idx);+ let ret = self.vmctx_plus_offset_raw::<VMFuncRef>(offset);+ debug_assert!(+ mem::transmute::<Option<VmPtr<VMWasmCallFunction>>, usize>(ret.as_ref().wasm_call)+ != INVALID_PTR+ );+ debug_assert!(ret.as_ref().vmctx.as_ptr() as usize != INVALID_PTR);+ ret+ }+ }+/// Stores the runtime memory pointer at the index specified.////// This is intended to be called during the instantiation process of a@@ -543,6 +557,27 @@ impl ComponentInstance {}}+ /// Same as `set_trampoline` but for intrinsic functions.+ pub fn set_intrinsic(+ self: Pin<&mut Self>,+ intrinsic: UnsafeIntrinsic,+ wasm_call: NonNull<VMWasmCallFunction>,+ array_call: NonNull<VMArrayCallFunction>,+ type_index: VMSharedTypeIndex,+ ) {+ unsafe {+ let offset = self.offsets.unsafe_intrinsic_func_ref(intrinsic);+ debug_assert!(*self.vmctx_plus_offset::<usize>(offset) == INVALID_PTR);+ let vmctx = VMOpaqueContext::from_vmcomponent(self.vmctx());+ *self.vmctx_plus_offset_mut(offset) = VMFuncRef {+ wasm_call: Some(wasm_call.into()),+ array_call: array_call.into(),+ type_index,+ vmctx: vmctx.into(),+ };+ }+ }+/// Configures the destructor for a resource at the `idx` specified.////// This is required to be called for each resource as it's defined within a@@ -638,6 +673,14 @@ impl ComponentInstance {*self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;}}+ for i in 0..self.offsets.num_unsafe_intrinsics {+ let i = UnsafeIntrinsic::from_u32(i);+ let offset = self.offsets.unsafe_intrinsic_func_ref(i);+ // SAFETY: see above+ unsafe {+ *self.as_mut().vmctx_plus_offset_mut(offset) = INVALID_PTR;+ }+ }for i in 0..self.offsets.num_runtime_memories {let i = RuntimeMemoryIndex::from_u32(i);let offset = self.offsets.runtime_memory(i);diff --git a/crates/wasmtime/src/runtime/vm/provenance.rs b/crates/wasmtime/src/runtime/vm/provenance.rsindex c10136c757..ea0ca41533 100644--- a/crates/wasmtime/src/runtime/vm/provenance.rs+++ b/crates/wasmtime/src/runtime/vm/provenance.rs@@ -115,6 +115,11 @@ impl<T> VmPtr<T> {pub fn as_ptr(&self) -> *mut T {self.as_non_null().as_ptr()}++ /// Similar to `{NonNull,core::ptr}::dangling()` but for `VmPtr`.+ pub fn dangling() -> Self {+ NonNull::dangling().into()+ }}// `VmPtr<T>`, like raw pointers, is trivially `Clone`/`Copy`.diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex 24dc0bfd57..ef714422d6 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -1173,6 +1173,10 @@ pub struct VMStoreContext {/// on `VMStackChain` for details.pub stack_chain: UnsafeCell<VMStackChain>,+ /// A pointer to the embedder's `T` inside a `Store<T>`, for use with the+ /// `store-data-address` unsafe intrinsic.+ pub store_data: VmPtr<()>,+/// The range, in addresses, of the guard page that is currently in use.////// This field is used when signal handlers are run to determine whether a@@ -1266,6 +1270,7 @@ impl Default for VMStoreContext {last_wasm_entry_trap_handler: UnsafeCell::new(0),stack_chain: UnsafeCell::new(VMStackChain::Absent),async_guard_range: ptr::null_mut()..ptr::null_mut(),+ store_data: VmPtr::dangling(),}}}@@ -1327,7 +1332,11 @@ mod test_vmstore_context {assert_eq!(offset_of!(VMStoreContext, stack_chain),usize::from(offsets.ptr.vmstore_context_stack_chain())- )+ );+ assert_eq!(+ offset_of!(VMStoreContext, store_data),+ usize::from(offsets.ptr.vmstore_context_store_data())+ );}}diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rsindex 13a5e80d98..77f76dd742 100644--- a/crates/winch/src/compiler.rs+++ b/crates/winch/src/compiler.rs@@ -6,6 +6,8 @@ use std::mem;use std::sync::Mutex;use wasmparser::FuncValidatorAllocations;use wasmtime_cranelift::CompiledFunction;+#[cfg(feature = "component-model")]+use wasmtime_environ::component::ComponentTranslation;use wasmtime_environ::{CompileError, CompiledFunctionBody, DefinedFuncIndex, FuncKey, FunctionBodyData, FunctionLoc,ModuleTranslation, ModuleTypesBuilder, PrimaryMap, StaticModuleIndex, Tunables, VMOffsets,@@ -398,4 +400,23 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {}Ok(body)}++ fn compile_intrinsic(+ &self,+ tunables: &Tunables,+ component: &ComponentTranslation,+ intrinsic: wasmtime_environ::component::UnsafeIntrinsic,+ abi: wasmtime_environ::Abi,+ symbol: &str,+ ) -> Result<CompiledFunctionBody> {+ let mut body = self+ .0+ .component_compiler()+ .compile_intrinsic(tunables, component, intrinsic, abi, symbol)?;+ if let Some(c) = self.0.inlining_compiler() {+ c.finish_compiling(&mut body, None, symbol)+ .map_err(|e| CompileError::Codegen(e.to_string()))?;+ }+ Ok(body)+ }}diff --git a/tests/all/intrinsics.rs b/tests/all/intrinsics.rsnew file mode 100644index 0000000000..0c67749edb--- /dev/null+++ b/tests/all/intrinsics.rs@@ -0,0 +1,643 @@+//! Tests related to the unsafe Wasmtime intrinsics we can give to components+//! via `CodeBuilder::expose_unsafe_intrinsics`.++use super::*;+use std::{cell::UnsafeCell, path::Path, sync::Arc};+use wasmtime::*;++#[test]+#[cfg_attr(miri, ignore)]+fn native_loads_and_stores() -> Result<()> {+ let engine = Engine::default();++ for (comp_ty, core_ty) in [+ ("u8", "i32"),+ ("u16", "i32"),+ ("u32", "i32"),+ ("u64", "i64"),+ ] {+ let mut code_builder = CodeBuilder::new(&engine);+ let path = format!("intrinsics::native_loads_and_stores::{comp_ty}");+ let path = Path::new(&path);+ let wat = format!(+ r#"+ (component+ (import "host" (instance $host (export "get-pointer" (func (result u64)))))+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "{comp_ty}-native-load" (func (param "pointer" u64) (result {comp_ty})))+ (export "{comp_ty}-native-store" (func (param "pointer" u64) (param "value" {comp_ty})))+ )+ )++ (core func $get-pointer' (canon lower (func $host "get-pointer")))+ (core func $load' (canon lower (func $intrinsics "{comp_ty}-native-load")))+ (core func $store' (canon lower (func $intrinsics "{comp_ty}-native-store")))++ (core module $m+ (import "" "get-pointer" (func $get-pointer (result i64)))+ (import "" "load" (func $load (param i64) (result {core_ty})))+ (import "" "store" (func $store (param i64 {core_ty})))+ (func (export "run")+ (local $pointer i64)++ ;; Get a native pointer from the host.+ (local.set $pointer (call $get-pointer))++ ;; Assert that loading from the pointer results in `42`.+ (if ({core_ty}.ne (call $load (local.get $pointer))+ ({core_ty}.const 42))+ (then (unreachable)))++ ;; Store `-1` through the pointer.+ (call $store (local.get $pointer) ({core_ty}.const -1))+ )+ )++ (core instance $i+ (instantiate $m+ (with "" (instance (export "get-pointer" (func $get-pointer'))+ (export "load" (func $load'))+ (export "store" (func $store'))))+ )+ )++ (func (export "run") (canon lift (core func $i "run")))+ )+ "#+ );+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let component = code_builder.compile_component()?;++ // Data that will be manipulated directly by Wasm via intrinsics.+ let mut data = Arc::new(UnsafeCell::new(0x6666666666666666_u64));++ // Write `42` into `data` at the appropriate place for this type.+ let ptr = data.get();+ unsafe {+ match comp_ty {+ "u8" => ptr.cast::<u8>().write(42),+ "u16" => ptr.cast::<u16>().write(42),+ "u32" => ptr.cast::<u32>().write(42),+ "u64" => ptr.cast::<u64>().write(42),+ _ => unreachable!(),+ }+ }++ // Create a linker and define `host::get-pointer` to return a pointer to+ // `data`.+ let mut linker = component::Linker::new(&engine);+ linker.instance("host")?.func_new("get-pointer", {+ let ptr = ptr as usize;+ let ptr = u64::try_from(ptr).unwrap();+ move |_cx, _args, results| {+ results[0] = component::Val::U64(ptr);+ Ok(())+ }+ })?;++ // Instantiate the component and call its `run` function.+ let mut store = Store::new(&engine, ());+ let instance = linker.instantiate(&mut store, &component)?;+ let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;+ run.call(&mut store, ())?;++ // The `run` function should have written `-1` into its view of `data`+ // and should not have modified any other part of it.+ drop(linker);+ let actual = Arc::get_mut(&mut data).unwrap().get_mut().to_ne_bytes();+ let expected = match comp_ty {+ "u8" => [0xFF, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66],+ "u16" => [0xFF, 0xFF, 0x66, 0x66, 0x66, 0x66, 0x66, 0x66],+ "u32" => [0xFF, 0xFF, 0xFF, 0xFF, 0x66, 0x66, 0x66, 0x66],+ "u64" => [0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF],+ _ => unreachable!(),+ };+ assert_eq!(+ expected,+ actual,+ "expected != actual\n\+ \texpected: {}\n\+ \t actual: {}",+ expected+ .iter()+ .map(|b| format!("{b:02x}"))+ .collect::<Vec<_>>()+ .join(" "),+ actual+ .iter()+ .map(|b| format!("{b:02x}"))+ .collect::<Vec<_>>()+ .join(" "),+ );+ }+ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn cannot_enable_unsafe_intrinsics_for_core_module() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);+ code_builder.wasm_binary_or_text("(module)".as_bytes(), None)?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let err = code_builder.compile_module().unwrap_err();+ let err = format!("{err:?}");+ assert!(+ err.contains("`CodeBuilder::expose_unsafe_intrinsics` can only be used with components"),+ "unexpected error: {err}"+ );+ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn ref_func_of_intrinsic() -> Result<()> {+ let mut config = Config::new();+ config.wasm_function_references(true);+ config.wasm_gc(true);+ let engine = Engine::new(&config)?;++ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u32-native-load" (func (param "pointer" u64) (result u32)))+ )+ )++ (core func $load' (canon lower (func $intrinsics "u32-native-load")))++ (core module $m+ (type $ty (func (param i64) (result i32)))+ (import "" "load" (func $load (type $ty)))+ (elem declare func $load)+ (func (export "run") (param $pointer i64) (result i32)+ (local $f (ref null func))+ (local.set $f (ref.func $load))+ (call_ref $ty (local.get $pointer) (ref.cast (ref $ty) (local.get $f)))+ )+ )++ (core instance $i+ (instantiate $m+ (with "" (instance (export "load" (func $load'))))+ )+ )++ (func (export "run") (param "pointer" u64) (result u32)+ (canon lift (core func $i "run"))+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let component = code_builder.compile_component()?;++ // Instantiate the component and call its `run` function.+ let linker = component::Linker::new(&engine);+ let mut store = Store::new(&engine, ());+ let instance = linker.instantiate(&mut store, &component)?;+ let run = instance.get_typed_func::<(u64,), (u32,)>(&mut store, "run")?;++ let data = 0x12345678_u32;+ let (result,) = run.call(&mut store, (&data as *const _ as usize as u64,))?;+ assert_eq!(result, data);++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn table_element_segment_with_intrinsic() -> Result<()> {+ let engine = Engine::default();++ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u32-native-load" (func (param "pointer" u64) (result u32)))+ )+ )++ (core func $load' (canon lower (func $intrinsics "u32-native-load")))++ (core module $m+ (type $ty (func (param i64) (result i32)))+ (import "" "load" (func $load (type $ty)))+ (table $t 1 1 funcref)+ (elem (table $t) (i32.const 0) func $load)+ (func (export "run") (param $pointer i64) (result i32)+ (call_indirect (type $ty) (local.get $pointer) (i32.const 0))+ )+ )++ (core instance $i+ (instantiate $m+ (with "" (instance (export "load" (func $load'))))+ )+ )++ (func (export "run") (param "pointer" u64) (result u32)+ (canon lift (core func $i "run"))+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let component = code_builder.compile_component()?;++ // Instantiate the component and call its `run` function.+ let linker = component::Linker::new(&engine);+ let mut store = Store::new(&engine, ());+ let instance = linker.instantiate(&mut store, &component)?;+ let run = instance.get_typed_func::<(u64,), (u32,)>(&mut store, "run")?;++ let data = 0x12345678_u32;+ let (result,) = run.call(&mut store, (&data as *const _ as usize as u64,))?;+ assert_eq!(result, data);++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn intrinsics_not_listed_in_imports() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u32-native-load" (func (param "pointer" u64) (result u32)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let component = code_builder.compile_component()?;++ let component_type = component.component_type();+ let imports = component_type.imports(&engine);+ assert_eq!(imports.count(), 0);++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn unknown_intrinsic_function() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "unknown" (func (param "pointer" u64) (result u32)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();+ let err = format!("{err:?}");+ assert!(+ err.contains("invalid unsafe intrinsic: \"unknown\""),+ "unexpected error: {err}"+ );++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn bad_extra_param() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u8-native-load" (func (param "pointer" u64) (param "extra" u64) (result u8)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();+ let err = format!("{err:?}");+ assert!(+ err.contains(+ "bad unsafe intrinsics import at `unsafe-intrinsics`: function `u8-native-load` \+ must have 1 parameters, found 2"+ ),+ "unexpected error: {err}"+ );++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn bad_missing_param() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u8-native-load" (func (result u8)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();+ let err = format!("{err:?}");+ assert!(+ err.contains(+ "bad unsafe intrinsics import at `unsafe-intrinsics`: function `u8-native-load` \+ must have 1 parameters, found 0"+ ),+ "unexpected error: {err}"+ );++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn bad_missing_return() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u8-native-load" (func (param "pointer" u64)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();+ let err = format!("{err:?}");+ assert!(+ err.contains(+ "bad unsafe intrinsics import at `unsafe-intrinsics`: function `u8-native-load` must \+ have 1 results, found 0"+ ),+ "unexpected error: {err}"+ );++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn bad_extra_return() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u8-native-store" (func (param "pointer" u64) (param "value" u8) (result u32)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();+ let err = format!("{err:?}");+ assert!(+ err.contains(+ "bad unsafe intrinsics import at `unsafe-intrinsics`: function `u8-native-store` \+ must have 0 results, found 1"+ ),+ "unexpected error: {err}"+ );++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn bad_param_type() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u8-native-store" (func (param "pointer" u64) (param "value" u16)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();+ let err = format!("{err:?}");+ assert!(+ err.contains(+ "bad unsafe intrinsics import at `unsafe-intrinsics`: parameters[1] for function \+ `u8-native-store` must be `U8`, found `Primitive(U16)`"+ ),+ "unexpected error: {err}"+ );++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn bad_result_type() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let path = format!("intrinsics::ref_func_of_intrinsic");+ let path = Path::new(&path);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "u8-native-load" (func (param "pointer" u64) (result u16)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), Some(path))?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let err = code_builder.compile_component().map(|_| ()).unwrap_err();+ let err = format!("{err:?}");+ assert!(+ err.contains(+ "bad unsafe intrinsics import at `unsafe-intrinsics`: results[0] for function \+ `u8-native-load` must be `U8`, found `Primitive(U16)`"+ ),+ "unexpected error: {err}"+ );++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn store_data_address() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let wat = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "store-data-address" (func (result u64)))+ (export "u8-native-load" (func (param "pointer" u64) (result u8)))+ (export "u8-native-store" (func (param "pointer" u64) (param "value" u8)))+ )+ )++ (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))+ (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))+ (core func $u8-native-store' (canon lower (func $intrinsics "u8-native-store")))++ (core module $m+ (import "" "store-data-address" (func $store-data-address (result i64)))+ (import "" "u8-native-load" (func $load (param i64) (result i32)))+ (import "" "u8-native-store" (func $store (param i64 i32)))+ (func (export "run")+ ;; Load the store data, it should be 42.+ (if (i32.ne (call $load (call $store-data-address))+ (i32.const 42))+ (then (unreachable)))++ ;; Store 36 into the store data.+ (call $store (call $store-data-address) (i32.const 36))+ )+ )++ (core instance $i+ (instantiate $m+ (with "" (instance (export "store-data-address" (func $store-data-address'))+ (export "u8-native-load" (func $u8-native-load'))+ (export "u8-native-store" (func $u8-native-store'))))+ )+ )++ (func (export "run")+ (canon lift (core func $i "run"))+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), None)?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("unsafe-intrinsics");+ }+ let component = code_builder.compile_component()?;++ // Instantiate the component and call its `run` function.+ let linker = component::Linker::new(&engine);+ let mut store = Store::new(&engine, 42_u8);+ let instance = linker.instantiate(&mut store, &component)?;+ let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;++ run.call(&mut store, ())?;+ assert_eq!(*store.data(), 36);++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn other_import_name() -> Result<()> {+ let engine = Engine::default();+ let mut code_builder = CodeBuilder::new(&engine);++ let wat = r#"+ (component+ (import "other-name"+ (instance+ (export "u32-native-load" (func (param "pointer" u64) (result u32)))+ )+ )+ )+ "#;+ code_builder.wasm_binary_or_text(wat.as_bytes(), None)?;+ unsafe {+ code_builder.expose_unsafe_intrinsics("other-name");+ }+ code_builder.compile_component()?;++ Ok(())+}diff --git a/tests/all/main.rs b/tests/all/main.rsindex 5e3568fda8..0c6375b971 100644--- a/tests/all/main.rs+++ b/tests/all/main.rs@@ -27,6 +27,7 @@ mod iloop;mod import_calling_export;mod import_indexes;mod instance;+mod intrinsics;mod invoke_func_via_table;mod limits;mod linker;diff --git a/tests/disas.rs b/tests/disas.rsindex 8c75058a1d..7b0a0956db 100644--- a/tests/disas.rs+++ b/tests/disas.rs@@ -50,7 +50,7 @@ use std::io::Write as _;use std::path::{Path, PathBuf};use std::process::Stdio;use tempfile::TempDir;-use wasmtime::{Engine, OptLevel, Strategy};+use wasmtime::{CodeBuilder, CodeHint, Engine, OptLevel, Strategy};use wasmtime_cli_flags::CommonOptions;fn main() -> Result<()> {@@ -124,6 +124,7 @@ struct TestConfig {flags: Option<TestConfigFlags>,objdump: Option<TestConfigFlags>,filter: Option<String>,+ unsafe_intrinsics: Option<String>,}#[derive(Debug, Deserialize)]@@ -208,15 +209,26 @@ impl Test {}}let engine = Engine::new(&config).context("failed to create engine")?;- let wasm = wat::parse_file(&self.path)?;- let elf = if wasmparser::Parser::is_component(&wasm) {- engine- .precompile_component(&wasm)- .context("failed to compile component")?- } else {- engine- .precompile_module(&wasm)- .context("failed to compile module")?++ let mut builder = CodeBuilder::new(&engine);+ builder.wasm_binary_or_text_file(&self.path)?;+ if let Some(name) = self.config.unsafe_intrinsics.as_deref() {+ unsafe {+ builder.expose_unsafe_intrinsics(name);+ }+ }++ let elf = match builder.hint() {+ Some(CodeHint::Component) => builder+ .compile_component_serialized()+ .context("failed to compile component")?,+ Some(CodeHint::Module) => builder+ .compile_module_serialized()+ .context("failed to compile module")?,+ None => bail!(+ "contents of `{}` do not look like a Wasm component or module",+ self.path.display()+ ),};match self.config.test {@@ -253,7 +265,10 @@ impl Test {let functions = clifs.iter().map(|clif| {- let mut funcs = cranelift_reader::parse_functions(clif)?;+ let mut funcs =+ cranelift_reader::parse_functions(clif).with_context(|| {+ format!("failed to parse CLIF:\n\"\"\"\n{clif}\n\"\"\"")+ })?;if funcs.len() != 1 {bail!("expected one function per clif");}diff --git a/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat b/tests/disas/component-model/inlining-and-unsafe-intrinsics.watnew file mode 100644index 0000000000..128e913579--- /dev/null+++ b/tests/disas/component-model/inlining-and-unsafe-intrinsics.wat@@ -0,0 +1,94 @@+;;! target = "x86_64"+;;! test = "optimize"+;;! filter = "wasm[0]--function"+;;! flags = "-C inlining=y"+;;! unsafe_intrinsics = "unsafe-intrinsics"++(component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "store-data-address" (func (result u64)))+ (export "u8-native-load" (func (param "pointer" u64) (result u8)))+ (export "u8-native-store" (func (param "pointer" u64) (param "value" u8)))+ )+ )++ (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))+ (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))+ (core func $u8-native-store' (canon lower (func $intrinsics "u8-native-store")))++ (core module $m+ (import "" "store-data-address" (func $store-data-address (result i64)))+ (import "" "u8-native-load" (func $load (param i64) (result i32)))+ (import "" "u8-native-store" (func $store (param i64 i32)))+ (func (export "f")+ (local $x i32)+ (local.set $x (call $load (call $store-data-address)))+ (call $store (call $store-data-address)+ (i32.add (local.get $x) (i32.const 1)))+ )+ )++ (core instance $i+ (instantiate $m+ (with "" (instance (export "store-data-address" (func $store-data-address'))+ (export "u8-native-load" (func $u8-native-load'))+ (export "u8-native-store" (func $u8-native-store'))))+ )+ )++ (func (export "f")+ (canon lift (core func $i "f"))+ )+)++;; function u0:0(i64 vmctx, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; gv3 = vmctx+;; sig0 = (i64 vmctx, i64) -> i64 tail+;; sig1 = (i64 vmctx, i64, i64) -> i32 tail+;; sig2 = (i64 vmctx, i64, i64, i32) tail+;; fn0 = colocated u3758096384:0 sig0+;; fn1 = colocated u3758096384:1 sig1+;; fn2 = colocated u3758096384:2 sig2+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64):+;; @0153 jump block2+;;+;; block2:+;; jump block3+;;+;; block3:+;; @0155 jump block4+;;+;; block4:+;; @0153 v4 = load.i64 notrap aligned readonly can_move v0+64+;; v17 = load.i64 notrap aligned readonly can_move vmctx v4+16+;; v18 = load.i64 notrap aligned readonly can_move vmctx v17+96+;; v20 = load.i8 notrap aligned v18+;; jump block5+;;+;; block5:+;; @0159 jump block6+;;+;; block6:+;; jump block7+;;+;; block7:+;; @0160 jump block8+;;+;; block8:+;; v28 = iconst.i8 1+;; v29 = iadd.i8 v20, v28 ; v28 = 1+;; store notrap aligned v29, v18+;; jump block9+;;+;; block9:+;; @0162 jump block1+;;+;; block1:+;; @0162 return+;; }diff --git a/tests/disas/component-model/unsafe-intrinsics-used.wat b/tests/disas/component-model/unsafe-intrinsics-used.watnew file mode 100644index 0000000000..07a961cd09--- /dev/null+++ b/tests/disas/component-model/unsafe-intrinsics-used.wat@@ -0,0 +1,49 @@+;;! target = "x86_64"+;;! test = "optimize"+;;! filter = "unsafe-intrinsics-wasm-call"+;;! unsafe_intrinsics = "unsafe-intrinsics"++;; Test that only the intrinsics that we actually use get compiled.++(component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "store-data-address" (func (result u64)))+ (export "u8-native-load" (func (param "pointer" u64) (result u8)))+ )+ )++ (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))+ (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))++ (core module $m+ (import "" "store-data-address" (func $store-data-address (result i64)))+ (import "" "u8-native-load" (func $load (param i64) (result i32)))++ ;; XXX: if we ever implement gc-sections/DCE during our linking, we will+ ;; need to update this to `ref.func` the imported functions or whatever+ ;; to ensure that they are rooted in that analysis and we do end up with+ ;; non-inlined function bodies for the intrinsics.+ )++ (core instance $i+ (instantiate $m+ (with "" (instance (export "store-data-address" (func $store-data-address'))+ (export "u8-native-load" (func $u8-native-load'))))+ )+ )+)++;; function u0:0(i64 vmctx, i64) -> i64 tail {+;; block0(v0: i64, v1: i64):+;; v2 = load.i64 notrap aligned readonly can_move vmctx v0+16+;; v3 = load.i64 notrap aligned readonly can_move vmctx v2+96+;; return v3+;; }+;;+;; function u0:0(i64 vmctx, i64, i64) -> i32 tail {+;; block0(v0: i64, v1: i64, v2: i64):+;; v3 = load.i8 notrap aligned v2+;; v4 = uextend.i32 v3+;; return v4+;; }diff --git a/tests/disas/riscv64-component-builtins-asm.wat b/tests/disas/riscv64-component-builtins-asm.watindex cbbfd7de46..dd6a465899 100644--- a/tests/disas/riscv64-component-builtins-asm.wat+++ b/tests/disas/riscv64-component-builtins-asm.wat@@ -1,6 +1,6 @@;;! target = "riscv64";;! test = 'compile'-;;! filter = '_wasm_call'+;;! filter = 'wasm-call';;! objdump = '--funcs all'(component@@ -11,7 +11,7 @@(core instance (instantiate $m (with "" (instance (export "" (func $f)))))))-;; component-resource-drop[0]_wasm_call:+;; component-trampolines[0]-wasm-call-component-resource-drop[0]:;; addi sp, sp, -0x10;; sd ra, 8(sp);; sd s0, 0(sp)diff --git a/tests/disas/riscv64-component-builtins.wat b/tests/disas/riscv64-component-builtins.watindex 1451e59b56..4a1102fed0 100644--- a/tests/disas/riscv64-component-builtins.wat+++ b/tests/disas/riscv64-component-builtins.wat@@ -1,6 +1,6 @@;;! target = "riscv64";;! test = 'optimize'-;;! filter = 'component-resource-drop[0]_wasm_call'+;;! filter = 'wasm-call-component-resource-drop[0]'(component(type $a (resource (rep i32)))