A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
wasmtime: Cache `VMFuncRef` in `component::Func` (#14189)
Adam Bratschi-Kayeffb04089ea89
diff --git a/benches/call.rs b/benches/call.rsindex 7e35b7fe42..b48448273c 100644--- a/benches/call.rs+++ b/benches/call.rs@@ -45,12 +45,15 @@ impl IsAsync {}}-fn engines() -> Vec<(Engine, IsAsync)> {+fn engines(concurrency_support: bool) -> Vec<(Engine, IsAsync)> {let mut config = Config::new();#[cfg(feature = "component-model")]config.wasm_component_model(true);+ #[cfg(feature = "component-model-async")]+ config.concurrency_support(concurrency_support);+let mut pool = PoolingAllocationConfig::default();if std::env::var("WASMTIME_TEST_FORCE_MPK").is_ok() {pool.memory_protection_keys(Enabled::Yes);@@ -79,7 +82,7 @@ fn engines() -> Vec<(Engine, IsAsync)> {/// Benchmarks the overhead of calling WebAssembly from the host in various/// configurations.fn host_to_wasm(c: &mut Criterion) {- for (engine, is_async) in engines() {+ for (engine, is_async) in engines(false) {let mut store = Store::new(&engine, ());let module = Module::new(&engine,@@ -249,7 +252,7 @@ fn wasm_to_host(c: &mut Criterion) {)"#;- for (engine, is_async) in engines() {+ for (engine, is_async) in engines(false) {let mut store = Store::new(&engine, ());let module = Module::new(&engine, module).unwrap();@@ -548,8 +551,22 @@ mod component {tuples!(A B);tuples!(A B C);+ fn engines() -> Vec<(String, Engine, IsAsync)> {+ let mut result: Vec<_> = super::engines(false)+ .into_iter()+ .map(|(e, a)| ("no-concurrent".to_string(), e, a))+ .collect();+ #[cfg(feature = "component-model-async")]+ result.extend(+ super::engines(true)+ .into_iter()+ .map(|(e, a)| ("concurrent".to_string(), e, a)),+ );+ result+ }+fn host_to_wasm(c: &mut Criterion) {- for (engine, is_async) in engines() {+ for (concurrent, engine, is_async) in engines() {let mut store = Store::new(&engine, ());let component = Component::new(@@ -599,12 +616,12 @@ mod component {};// Bench once without any call hooks configured- let name = format!("{}/no-hook", is_async.desc());+ let name = format!("{}/{}/no-hook", concurrent, is_async.desc());bench_calls(&mut c.benchmark_group(&name), &mut store);// Bench again with a "call hook" enabledstore.call_hook(|_, _| Ok(()));- let name = format!("{}/hook-sync", is_async.desc());+ let name = format!("{}/{}/hook-sync", concurrent, is_async.desc());bench_calls(&mut c.benchmark_group(&name), &mut store);}}@@ -738,19 +755,19 @@ mod component {)"#;- for (engine, is_async) in engines() {+ for (concurrent, engine, is_async) in engines() {let mut store = Store::new(&engine, ());let component = component::Component::new(&engine, module).unwrap();bench_calls(- &mut c.benchmark_group(&format!("{}/no-hook", is_async.desc())),+ &mut c.benchmark_group(&format!("{}/{}/no-hook", concurrent, is_async.desc())),&mut store,&component,is_async,);store.call_hook(|_, _| Ok(()));bench_calls(- &mut c.benchmark_group(&format!("{}/hook-sync", is_async.desc())),+ &mut c.benchmark_group(&format!("{}/{}/hook-sync", concurrent, is_async.desc())),&mut store,&component,is_async,diff --git a/crates/c-api/include/wasmtime/component/func.h b/crates/c-api/include/wasmtime/component/func.hindex 2811610e44..a2b48a2987 100644--- a/crates/c-api/include/wasmtime/component/func.h+++ b/crates/c-api/include/wasmtime/component/func.h@@ -33,6 +33,9 @@ typedef struct wasmtime_component_func {/// Private internal wasmtime information.uint32_t __private2;++ /// Private internal wasmtime information.+ void *__private3;} wasmtime_component_func_t;/// \brief Returns the type of this function.diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex 079ce49e6e..2bae1c0ccb 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -5828,11 +5828,11 @@ pub(crate) fn prepare_call<T, R>(handle: Func,param_count: usize,host_future_present: bool,- lower_params: impl FnOnce(Func, StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>+ lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>+ Send+ Sync+ 'static,- lift_result: impl FnOnce(Func, &mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>+ lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>+ Send+ Sync+ 'static,@@ -5861,11 +5861,11 @@ pub(crate) fn prepare_call<T, R>(let thread = GuestTask::new(state,Box::new(for_any_lower(move |store, params| {- lower_params(handle, token.as_context_mut(store), params)+ lower_params(token.as_context_mut(store), params)})),LiftResult {lift: Box::new(for_any_lift(move |store, result| {- lift_result(handle, store, result)+ lift_result(store, result)})),ty: task_return_type,memory,@@ -5976,7 +5976,9 @@ fn queue_call0<T: 'static>(let callback = raw_options.callback;let instance = handle.instance();let callee = handle.lifted_core_func(store.0);- let post_return = handle.post_return_core_func(store.0);+ let post_return = raw_options+ .post_return+ .map(|i| instance.id().get(store.0).runtime_post_return(i));let callback = callback.map(|i| {let instance = instance.id().get(store.0);SendSyncPtr::new(instance.runtime_callback(i))diff --git a/crates/wasmtime/src/runtime/component/concurrent/func.rs b/crates/wasmtime/src/runtime/component/concurrent/func.rsindex 371bb8e621..bedebac33a 100644--- a/crates/wasmtime/src/runtime/component/concurrent/func.rs+++ b/crates/wasmtime/src/runtime/component/concurrent/func.rs@@ -171,24 +171,27 @@ impl Func {params: Vec<Val>,) -> Result<PreparedCall<Vec<Val>>> {let store = store.as_context_mut();+ let (options, flags, ty, raw_options) = self.abi_info(store.0);+ let async_ = raw_options.async_;+ let instance = self.instance();concurrent::prepare_call(store,self,MAX_FLAT_PARAMS,false,- move |func, store, params_out| {- func.with_lower_context(store, |cx, ty| {+ move |store, params_out| {+ Func::with_lower_context(instance, store, options, flags, ty, |cx, ty| {Self::lower_args(cx, ¶ms, ty, params_out)})},- move |func, store, results| {- let max_flat = if func.abi_async(store) {+ move |store, results| {+ let max_flat = if async_ {MAX_FLAT_PARAMS} else {MAX_FLAT_RESULTS};- let results = func.with_lift_context(store, |cx, ty| {+ let results = Func::with_lift_context(instance, store, options, ty, |cx, ty| {Self::lift_results(cx, ty, results, max_flat)?.collect::<Result<Vec<_>>>()})?;Ok(Box::new(results))@@ -431,22 +434,27 @@ where} else {1};- let max_results = if self.func().abi_async(store.0) {+ let (options, flags, ty, raw_options) = self.func().abi_info(store.0);+ let instance = self.func().instance();+ let max_results = if raw_options.async_ {MAX_FLAT_PARAMS} else {MAX_FLAT_RESULTS};+concurrent::prepare_call(store,*self.func(),param_count,host_future_present,- move |func, store, params_out| {- func.with_lower_context(store, |cx, ty| lower(cx, ty, params_out))+ move |store, params_out| {+ Func::with_lower_context(instance, store, options, flags, ty, |cx, ty| {+ lower(cx, ty, params_out)+ })},- move |func, store, results| {+ move |store, results| {let result = if Return::flatten_count() <= max_results {- func.with_lift_context(store, |cx, ty| {+ Func::with_lift_context(instance, store, options, ty, |cx, ty| {// SAFETY: Per the safety requirements documented for the// `ComponentType` trait, `Return::Lower` must be// compatible at the binary level with a `[ValRaw; N]`,@@ -468,7 +476,7 @@ whereSelf::lift_stack_result(cx, ty, results)})?} else {- func.with_lift_context(store, |cx, ty| {+ Func::with_lift_context(instance, store, options, ty, |cx, ty| {Self::lift_heap_result(cx, ty, &results[0])})?};diff --git a/crates/wasmtime/src/runtime/component/func.rs b/crates/wasmtime/src/runtime/component/func.rsindex 5c7f07ba2d..2527a17919 100644--- a/crates/wasmtime/src/runtime/component/func.rs+++ b/crates/wasmtime/src/runtime/component/func.rs@@ -5,7 +5,7 @@ use crate::component::types::ComponentFunc;use crate::component::values::Val;use crate::prelude::*;use crate::runtime::vm::component::{ComponentInstance, InstanceFlags};-use crate::runtime::vm::{Export, VMFuncRef};+use crate::runtime::vm::{Export, SendSyncPtr, VMFuncRef};use crate::store::StoreOpaque;use crate::{AsContext, AsContextMut, StoreContextMut, ValRaw};use core::mem::{self, MaybeUninit};@@ -33,24 +33,55 @@ pub use self::typed::*;#[repr(C)] // here for the C API.pub struct Func {instance: Instance,++ /// The export index of this lifted function within its component.index: ExportIndex,++ /// The resolved core `VMFuncRef` for this lifted function, whose lifetime+ /// is bound to the `Store` this `Func` belongs to.+ ///+ /// Note that this field has an `unsafe_*` prefix to discourage use of it.+ /// This is only safe to read/use if the store that owns `instance`+ /// (identified by `instance.id().store_id()`) is in scope. Use the+ /// `self.lifted_core_func()` method instead of this field to perform this+ /// check.+ unsafe_func_ref: SendSyncPtr<VMFuncRef>,}-// Double-check that the C representation in `component/instance.h` matches our+// Double-check that the C representation in `component/func.h` matches our// in-Rust representation here in terms of size/alignment/etc.const _: () = {#[repr(C)]struct T(u64, u32);#[repr(C)]- struct C(T, u32);+ struct C(T, u32, *mut u8);assert!(core::mem::size_of::<C>() == core::mem::size_of::<Func>());assert!(core::mem::align_of::<C>() == core::mem::align_of::<Func>());assert!(core::mem::offset_of!(Func, instance) == 0);};impl Func {- pub(crate) fn from_lifted_func(instance: Instance, index: ExportIndex) -> Func {- Func { instance, index }+ pub(crate) fn from_lifted_func(+ store: &mut StoreOpaque,+ instance: Instance,+ index: ExportIndex,+ ) -> Func {+ let def = {+ let vminstance = instance.id().get(store);+ let (_ty, def, _options) = vminstance.component().export_lifted_function(index);+ def.clone()+ };+ let unsafe_func_ref = match instance.lookup_vmdef(store, &def) {+ Export::Function(f) => f.vm_func_ref(store),+ _ => unreachable!(),+ }+ .into();++ Func {+ instance,+ index,+ unsafe_func_ref,+ }}/// Attempt to cast this [`Func`] to a statically typed [`TypedFunc`] with@@ -179,10 +210,7 @@ impl Func {/// Get the type of this function.pub fn ty(&self, store: impl AsContext) -> ComponentFunc {- self.ty_(store.as_context().0)- }-- fn ty_(&self, store: &StoreOpaque) -> ComponentFunc {+ let store = store.as_context().0;let cx = InstanceType::new(self.instance.id().get(store));let ty = self.ty_index(store);ComponentFunc::from(ty, &cx)@@ -336,7 +364,7 @@ impl Func {// safe in Rust, however, due to `ValRaw` being a `union`. The// contents should dynamically not be read due to the type of the// function used here matching the actual lift.- let (_, post_return_arg) = unsafe {+ unsafe {self.call_raw(store.as_context_mut(),|cx, ty, dst: &mut MaybeUninit<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>| {@@ -355,30 +383,16 @@ impl Func {}Ok(())},- )?- };-- self.post_return_impl(store, post_return_arg)- }-- pub(crate) fn lifted_core_func(&self, store: &mut StoreOpaque) -> NonNull<VMFuncRef> {- let def = {- let instance = self.instance.id().get(store);- let (_ty, def, _options) = instance.component().export_lifted_function(self.index);- def.clone()- };- match self.instance.lookup_vmdef(store, &def) {- Export::Function(f) => f.vm_func_ref(store),- _ => unreachable!(),+ )?;}++ Ok(())}- pub(crate) fn post_return_core_func(&self, store: &StoreOpaque) -> Option<NonNull<VMFuncRef>> {- let instance = self.instance.id().get(store);- let component = instance.component();- let (_ty, _def, options) = component.export_lifted_function(self.index);- let post_return = component.env_component().options[options].post_return;- post_return.map(|i| instance.runtime_post_return(i))+ #[inline]+ pub(crate) fn lifted_core_func(&self, store: &StoreOpaque) -> NonNull<VMFuncRef> {+ self.instance.id().assert_belongs_to(store.id());+ self.unsafe_func_ref.as_non_null()}pub(crate) fn abi_async(&self, store: &StoreOpaque) -> bool {@@ -433,21 +447,25 @@ impl Func {&mut MaybeUninit<LowerParams>,) -> Result<()>,lift: impl FnOnce(&mut LiftContext<'_>, InterfaceType, &LowerReturn) -> Result<Return>,- ) -> Result<(Return, ValRaw)>+ ) -> Result<Return>whereLowerParams: Copy,LowerReturn: Copy,{let export = self.lifted_core_func(store.0);- let (_options, _flags, _ty, raw_options) = self.abi_info(store.0);++ let (options_idx, flags, ty, raw_options) = self.abi_info(store.0);+ let post_return = raw_options+ .post_return+ .map(|i| self.instance.id().get(store.0).runtime_post_return(i));let instance = self.instance.runtime_instance(raw_options.instance);+ let async_ = raw_options.async_;if !store.0.may_enter(instance)? {bail!(crate::Trap::CannotEnterComponent);}- let async_type = self.abi_async(store.0);- store.0.enter_guest_sync_call(None, async_type, instance)?;+ store.0.enter_guest_sync_call(None, async_, instance)?;#[repr(C)]union Union<Params: Copy, Return: Copy> {@@ -472,9 +490,14 @@ impl Func {assert!(mem::align_of_val(map_maybe_uninit!(space.params)) == val_align);assert!(mem::align_of_val(map_maybe_uninit!(space.ret)) == val_align);- self.with_lower_context(store.as_context_mut(), |cx, ty| {- lower(cx, ty, map_maybe_uninit!(space.params))- })?;+ Func::with_lower_context(+ self.instance,+ store.as_context_mut(),+ options_idx,+ flags,+ ty,+ |cx, ty| lower(cx, ty, map_maybe_uninit!(space.params)),+ )?;// SAFETY: We are providing the guarantee that all the inputs are valid.// The various pointers passed in for the function are all valid since@@ -511,27 +534,28 @@ impl Func {// return values).let ret: &LowerReturn = unsafe { map_maybe_uninit!(space.ret).assume_init_ref() };- // Lift the result into the host while managing post-return state- // here as well.- //- // After a successful lift the return value of the function, which- // is currently required to be 0 or 1 values according to the- // canonical ABI, is saved within the `Store`'s `FuncData`. This'll- // later get used in post-return.- let val = self.with_lift_context(store.0, |cx, ty| lift(cx, ty, ret))?;+ let val = Func::with_lift_context(self.instance, store.0, options_idx, ty, |cx, ty| {+ lift(cx, ty, ret)+ })?;// SAFETY: it's a contract of this function that `LowerReturn` is an// appropriate representation of the result of this function.let ret_slice = unsafe { storage_as_slice(ret) };+ let post_return_arg = match ret_slice.len() {+ 0 => ValRaw::i32(0),+ 1 => ret_slice[0],+ _ => unreachable!(),+ };- Ok((- val,- match ret_slice.len() {- 0 => ValRaw::i32(0),- 1 => ret_slice[0],- _ => unreachable!(),- },- ))+ // SAFETY: `post_return` and `flags` were resolved from this function's+ // own canonical options above, and `store` is the store this call is+ // running in.+ unsafe {+ call_post_return(&mut store, post_return, post_return_arg, flags)?;+ }+ store.0.exit_guest_sync_call()?;++ Ok(val)}#[doc(hidden)]@@ -547,23 +571,6 @@ impl Func {Ok(())}- pub(crate) fn post_return_impl(&self, mut store: impl AsContextMut, arg: ValRaw) -> Result<()> {- let mut store = store.as_context_mut();-- let index = self.index;- let vminstance = self.instance.id().get(store.0);- let component = vminstance.component();- let (_ty, _def, options) = component.export_lifted_function(index);- let post_return = self.post_return_core_func(store.0);- let flags = vminstance.instance_flags(component.env_component().options[options].instance);-- unsafe {- call_post_return(&mut store, post_return, arg, flags)?;- store.0.exit_guest_sync_call()?;- }- Ok(())- }-pub(crate) fn lower_args<T>(cx: &mut LowerContext<'_, T>,params: &[Val],@@ -659,44 +666,46 @@ impl Func {self.instance}- /// Creates a `LowerContext` using the configuration values of this lifted- /// function.+ /// Creates a `LowerContext` using the provided configuration values and runs+ /// the given `lower` closure within it.////// The `lower` closure provided should perform the actual lowering and/// return the result of the lowering operation which is then returned from/// this function as well.pub(crate) fn with_lower_context<T>(- self,+ instance: Instance,mut store: StoreContextMut<T>,+ options: OptionsIndex,+ mut flags: InstanceFlags,+ ty: TypeFuncIndex,lower: impl FnOnce(&mut LowerContext<T>, InterfaceType) -> Result<()>,) -> Result<()> {- let (options_idx, mut flags, ty, _) = self.abi_info(store.0);-// Perform the actual lowering, where while this is running the// component is forbidden from calling imports.unsafe {debug_assert!(flags.may_leave());flags.set_may_leave(false);}- let mut cx = LowerContext::new(store.as_context_mut(), options_idx, self.instance);+ let mut cx = LowerContext::new(store.as_context_mut(), options, instance);let param_ty = InterfaceType::Tuple(cx.types[ty].params);let result = lower(&mut cx, param_ty);unsafe { flags.set_may_leave(true) };result}- /// Creates a `LiftContext` using the configuration values with this lifted- /// function.+ /// Creates a `LiftContext` using the provided configuration values and runs+ /// the given `lift` closure within it.////// The closure `lift` provided should actually perform the lift itself and/// the result of that closure is returned from this function call as well.pub(crate) fn with_lift_context<R>(- self,+ instance: Instance,store: &mut StoreOpaque,+ options: OptionsIndex,+ ty: TypeFuncIndex,lift: impl FnOnce(&mut LiftContext, InterfaceType) -> Result<R>,) -> Result<R> {- let (options, _flags, ty, _) = self.abi_info(store);- let mut cx = LiftContext::new(store, options, self.instance)?;+ let mut cx = LiftContext::new(store, options, instance)?;let ty = InterfaceType::Tuple(cx.types[ty].results);lift(&mut cx, ty)}diff --git a/crates/wasmtime/src/runtime/component/func/typed.rs b/crates/wasmtime/src/runtime/component/func/typed.rsindex 9cac53d770..8216080393 100644--- a/crates/wasmtime/src/runtime/component/func/typed.rs+++ b/crates/wasmtime/src/runtime/component/func/typed.rs@@ -228,7 +228,7 @@ where// safety requirements of `Lift` and `Lower` on `Params` and `Return` in// combination with checking the various possible branches here and// dispatching to appropriately typed functions.- let (result, post_return_arg) = unsafe {+ let result = unsafe {// This type is used as `LowerParams` for `call_raw` which is either// `Params::Lower` or `ValRaw` representing it's either on the stack// or it's on the heap. This allocates 1 extra `ValRaw` on the stack@@ -262,8 +262,6 @@ where}}?;- self.func.post_return_impl(store, post_return_arg)?;-Ok(result)}diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rsindex 453b6dcf57..95468d3c9d 100644--- a/crates/wasmtime/src/runtime/component/instance.rs+++ b/crates/wasmtime/src/runtime/component/instance.rs@@ -167,7 +167,7 @@ impl Instance {}// And package up the indices!- Some(Func::from_lifted_func(*self, index))+ Some(Func::from_lifted_func(store, *self, index))}/// Looks up an exported [`Func`] value by name and with its type.diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rsindex b9520fdfb5..c8ab027e51 100644--- a/crates/wasmtime/src/runtime/func.rs+++ b/crates/wasmtime/src/runtime/func.rs@@ -277,7 +277,7 @@ pub struct Func {/// Note that this field has an `unsafe_*` prefix to discourage use of it./// This is only safe to read/use if `self.store` is validated to belong to/// an ambiently provided `StoreOpaque` or similar. Use the- /// `self.func_ref()` method instead of this field to perform this check.+ /// `self.vm_func_ref()` method instead of this field to perform this check.unsafe_func_ref: SendSyncPtr<VMFuncRef>,}