A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Consolidate component-related store data (#12549)
Alex Crichton7e11f18221ba
diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex b17aaee78f..4e68182263 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -1805,7 +1805,7 @@ impl StoreOpaque {if !task.returned_or_cancelled() {log::trace!("push call context for {guest_task:?}");let call_context = task.call_context.take().unwrap();- self.component_resource_state().0.push(call_context);+ self.component_call_contexts_mut().push(call_context);}Ok(())}@@ -1820,7 +1820,7 @@ impl StoreOpaque {.returned_or_cancelled(){log::trace!("pop call context for {guest_task:?}");- let call_context = Some(self.component_resource_state().0.pop().unwrap());+ let call_context = Some(self.component_call_contexts_mut().pop().unwrap());self.concurrent_state_mut().get_mut(guest_task)?.call_context = call_context;@@ -2745,8 +2745,7 @@ impl Instance {token.as_context_mut(store).0- .component_resource_state()- .0+ .component_call_contexts_mut().push(call_context);}});@@ -2761,8 +2760,7 @@ impl Instance {token.as_context_mut(store).0- .component_resource_state()- .0+ .component_call_contexts_mut().pop().unwrap(),);diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rsindex dd512ddcb7..9cd336a7fc 100644--- a/crates/wasmtime/src/runtime/component/instance.rs+++ b/crates/wasmtime/src/runtime/component/instance.rs@@ -728,7 +728,8 @@ impl<'a> Instantiator<'a> {imports: &'a Arc<PrimaryMap<RuntimeImportIndex, RuntimeImport>>,) -> Result<Instantiator<'a>> {let env_component = component.env_component();- store.register_component(component)?;+ let (modules, engine) = store.modules_and_engine_mut();+ modules.register_component(component, engine)?;let imported_resources: ImportedResources =PrimaryMap::with_capacity(env_component.imported_resources.len());diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rsindex 0b2d571aa5..81b2dbddbe 100644--- a/crates/wasmtime/src/runtime/component/store.rs+++ b/crates/wasmtime/src/runtime/component/store.rs@@ -1,19 +1,42 @@+use crate::runtime::component::concurrent::ConcurrentState;+use crate::runtime::component::{HostResourceData, Instance};+use crate::runtime::vm;#[cfg(feature = "component-model-async")]use crate::runtime::vm::VMStore;+use crate::runtime::vm::component::{CallContexts, HandleTable};use crate::runtime::vm::component::{ComponentInstance, OwnedComponentInstance};use crate::store::{StoreData, StoreId, StoreOpaque};+use crate::{Engine, StoreContextMut};#[cfg(feature = "component-model-async")]use alloc::vec::Vec;use core::pin::Pin;use wasmtime_environ::PrimaryMap;use wasmtime_environ::component::RuntimeComponentInstanceIndex;-#[derive(Default)]+/// Extensions to `Store` which are only relevant for component-related+/// information.pub struct ComponentStoreData {+ /// All component instances, in a similar manner to how core wasm instances+ /// are managed.instances: PrimaryMap<ComponentInstanceId, Option<OwnedComponentInstance>>,/// Whether an instance belonging to this store has trapped.trapped: bool,++ /// Total number of component instances in this store, used to track+ /// resources in the instance allocator.+ num_component_instances: usize,++ /// Runtime state for components used in the handling of resources, borrow,+ /// and calls. These also interact with the `ResourceAny` type and its+ /// internal representation.+ component_host_table: HandleTable,+ component_calls: CallContexts,+ host_resource_data: HostResourceData,++ /// Metadata/tasks/etc related to component-model-async and concurrency+ /// support.+ concurrent_state: Option<ConcurrentState>,}#[derive(Copy, Clone, Debug, PartialEq, Eq)]@@ -26,29 +49,51 @@ pub struct RuntimeInstance {pub index: RuntimeComponentInstanceIndex,}-impl StoreOpaque {- pub(crate) fn trapped(&self) -> bool {- self.store_data().components.trapped- }-- pub(crate) fn set_trapped(&mut self) {- self.store_data_mut().components.trapped = true;+impl ComponentStoreData {+ pub fn new(engine: &Engine) -> ComponentStoreData {+ ComponentStoreData {+ instances: Default::default(),+ trapped: false,+ num_component_instances: 0,+ component_host_table: Default::default(),+ component_calls: Default::default(),+ host_resource_data: Default::default(),+ concurrent_state: if engine.tunables().concurrency_support {+ #[cfg(feature = "component-model-async")]+ {+ Some(Default::default())+ }+ #[cfg(not(feature = "component-model-async"))]+ {+ unreachable!()+ }+ } else {+ None+ },+ }}-}-impl StoreData {- pub(crate) fn push_component_instance(- &mut self,- data: OwnedComponentInstance,- ) -> ComponentInstanceId {- let expected = data.get().id();- let ret = self.components.instances.push(Some(data));- assert_eq!(expected, ret);- ret+ /// Hook used just before a `Store` is dropped to dispose of anything+ /// necessary.+ ///+ /// Used at this time to deallocate fibers related to concurrency support.+ pub fn run_manual_drop_routines<T>(store: StoreContextMut<T>) {+ // We need to drop the fibers of each component instance before+ // attempting to drop the instances themselves since the fibers may need+ // to be resumed and allowed to exit cleanly before we yank the state+ // out from under them.+ //+ // This will also drop any futures which might use a `&Accessor` fields+ // in their `Drop::drop` implementations, in which case they'll need to+ // be called from with in the context of a `tls::set` closure.+ #[cfg(feature = "component-model-async")]+ if store.0.component_data().concurrent_state.is_some() {+ ComponentStoreData::drop_fibers_and_futures(store.0);+ }+ #[cfg(not(feature = "component-model-async"))]+ let _ = store;}-}-impl ComponentStoreData {pub fn next_component_instance_id(&self) -> ComponentInstanceId {self.instances.next_key()}@@ -86,32 +131,11 @@ impl ComponentStoreData {);}}-}--impl StoreData {- pub(crate) fn component_instance(&self, id: ComponentInstanceId) -> &ComponentInstance {- self.components.instances[id].as_ref().unwrap().get()- }- pub(crate) fn component_instance_mut(- &mut self,- id: ComponentInstanceId,- ) -> Pin<&mut ComponentInstance> {- self.components.instances[id].as_mut().unwrap().get_mut()- }-}--impl StoreOpaque {- pub(crate) fn component_instance(&self, id: ComponentInstanceId) -> &ComponentInstance {- self.store_data().component_instance(id)- }-- #[cfg(feature = "component-model-async")]- pub(crate) fn component_instance_mut(- &mut self,- id: ComponentInstanceId,- ) -> Pin<&mut ComponentInstance> {- self.store_data_mut().component_instance_mut(id)+ pub fn decrement_allocator_resources(&mut self, allocator: &dyn vm::InstanceAllocator) {+ for _ in 0..self.num_component_instances {+ allocator.decrement_component_instance_count();+ }}}@@ -194,11 +218,149 @@ impl StoreComponentInstanceId {}/// Same as `get_mut`, but borrows less of a store.- pub(crate) fn from_data_get_mut<'a>(- &self,- store: &'a mut StoreData,- ) -> Pin<&'a mut ComponentInstance> {+ fn from_data_get_mut<'a>(&self, store: &'a mut StoreData) -> Pin<&'a mut ComponentInstance> {self.assert_belongs_to(store.id());store.component_instance_mut(self.instance)}}++impl StoreData {+ pub(crate) fn push_component_instance(+ &mut self,+ data: OwnedComponentInstance,+ ) -> ComponentInstanceId {+ let expected = data.get().id();+ let ret = self.components.instances.push(Some(data));+ assert_eq!(expected, ret);+ ret+ }++ pub(crate) fn component_instance(&self, id: ComponentInstanceId) -> &ComponentInstance {+ self.components.instances[id].as_ref().unwrap().get()+ }++ pub(crate) fn component_instance_mut(+ &mut self,+ id: ComponentInstanceId,+ ) -> Pin<&mut ComponentInstance> {+ self.components.instances[id].as_mut().unwrap().get_mut()+ }+}++impl StoreOpaque {+ pub(crate) fn trapped(&self) -> bool {+ self.store_data().components.trapped+ }++ pub(crate) fn set_trapped(&mut self) {+ self.store_data_mut().components.trapped = true;+ }++ pub(crate) fn component_data(&self) -> &ComponentStoreData {+ &self.store_data().components+ }++ pub(crate) fn component_data_mut(&mut self) -> &mut ComponentStoreData {+ &mut self.store_data_mut().components+ }++ pub(crate) fn component_call_contexts_mut(&mut self) -> &mut CallContexts {+ &mut self.component_data_mut().component_calls+ }++ pub(crate) fn push_component_instance(&mut self, instance: Instance) {+ // We don't actually need the instance itself right now, but it seems+ // like something we will almost certainly eventually want to keep+ // around, so force callers to provide it.+ let _ = instance;++ self.component_data_mut().num_component_instances += 1;+ }++ pub(crate) fn component_instance(&self, id: ComponentInstanceId) -> &ComponentInstance {+ self.store_data().component_instance(id)+ }++ #[cfg(feature = "component-model-async")]+ pub(crate) fn component_instance_mut(+ &mut self,+ id: ComponentInstanceId,+ ) -> Pin<&mut ComponentInstance> {+ self.store_data_mut().component_instance_mut(id)+ }++ #[cfg(feature = "component-model-async")]+ pub(crate) fn concurrent_state_mut(&mut self) -> &mut ConcurrentState {+ debug_assert!(self.concurrency_support());+ self.component_data_mut().concurrent_state.as_mut().unwrap()+ }++ #[inline]+ pub(crate) fn concurrency_support(&self) -> bool {+ let support = self.component_data().concurrent_state.is_some();+ debug_assert_eq!(support, self.engine().tunables().concurrency_support);+ support+ }++ pub(crate) fn component_resource_state_with_instance_and_concurrent_state(+ &mut self,+ instance: Instance,+ ) -> (+ &mut CallContexts,+ &mut HandleTable,+ &mut HostResourceData,+ Pin<&mut ComponentInstance>,+ Option<&mut ConcurrentState>,+ ) {+ let instance = instance.id();+ instance.assert_belongs_to(self.id());+ let data = self.component_data_mut();+ (+ &mut data.component_calls,+ &mut data.component_host_table,+ &mut data.host_resource_data,+ data.instances[instance.instance]+ .as_mut()+ .unwrap()+ .get_mut(),+ data.concurrent_state.as_mut(),+ )+ }++ pub(crate) fn component_resource_tables(+ &mut self,+ instance: Option<Instance>,+ ) -> vm::component::ResourceTables<'_> {+ self.component_resource_tables_and_host_resource_data(instance)+ .0+ }++ pub(crate) fn component_resource_tables_and_host_resource_data(+ &mut self,+ instance: Option<Instance>,+ ) -> (+ vm::component::ResourceTables<'_>,+ &mut crate::component::HostResourceData,+ ) {+ let store_id = self.id();+ let data = self.component_data_mut();+ let guest = instance.map(|i| {+ let i = i.id();+ i.assert_belongs_to(store_id);+ data.instances[i.instance]+ .as_mut()+ .unwrap()+ .get_mut()+ .instance_states()+ });++ (+ vm::component::ResourceTables {+ host_table: Some(&mut data.component_host_table),+ calls: &mut data.component_calls,+ guest,+ },+ &mut data.host_resource_data,+ )+ }+}diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rsindex 2a55d96234..5c871b4757 100644--- a/crates/wasmtime/src/runtime/instance.rs+++ b/crates/wasmtime/src/runtime/instance.rs@@ -225,7 +225,8 @@ impl Instance {// Note that under normal operation this shouldn't do much as the list// of funcs-with-holes should generally be empty. As a result the// process of filling this out is not super optimized at this point.- store.register_module(module)?;+ let (modules, engine) = store.modules_and_engine_mut();+ modules.register_module(module, engine)?;let (funcrefs, modules) = store.func_refs_and_modules();funcrefs.fill(modules);@@ -304,7 +305,8 @@ impl Instance {// Register the module just before instantiation to ensure we keep the module// properly referenced while in use by the store.- let module_id = store.register_module(module)?;+ let (modules, engine) = store.modules_and_engine_mut();+ let module_id = modules.register_module(module, engine)?;// The first thing we do is issue an instance allocation request// to the instance allocator. This, on success, will give us an@@ -950,7 +952,8 @@ fn pre_instantiate_raw() -> Result<OwnedImports> {// Register this module and use it to fill out any funcref wasm_call holes// we can. For more comments on this see `typecheck_externs`.- store.register_module(module)?;+ let (modules, engine) = store.modules_and_engine_mut();+ modules.register_module(module, engine)?;let (funcrefs, modules) = store.func_refs_and_modules();funcrefs.fill(modules);diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex d680479573..8e7edfc48a 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -81,10 +81,6 @@ use crate::OwnedRooted;use crate::RootSet;#[cfg(feature = "gc")]use crate::ThrownException;-#[cfg(feature = "component-model-async")]-use crate::component::ComponentStoreData;-#[cfg(feature = "component-model")]-use crate::component::concurrent;use crate::error::OutOfMemory;#[cfg(feature = "async")]use crate::fiber;@@ -479,8 +475,6 @@ pub struct StoreOpaque {instances: wasmtime_environ::collections::PrimaryMap<InstanceId, StoreInstance>,- #[cfg(feature = "component-model")]- num_component_instances: usize,signal_handler: Option<SignalHandler>,modules: ModuleRegistry,func_refs: FuncRefs,@@ -540,18 +534,6 @@ pub struct StoreOpaque {/// guest code.pkey: Option<ProtectionKey>,- /// Runtime state for components used in the handling of resources, borrow,- /// and calls. These also interact with the `ResourceAny` type and its- /// internal representation.- #[cfg(feature = "component-model")]- component_host_table: vm::component::HandleTable,- #[cfg(feature = "component-model")]- component_calls: vm::component::CallContexts,- #[cfg(feature = "component-model")]- host_resource_data: crate::component::HostResourceData,- #[cfg(feature = "component-model")]- concurrent_state: Option<concurrent::ConcurrentState>,-/// State related to the executor of wasm code.////// For example if Pulley is enabled and configured then this will store a@@ -735,7 +717,7 @@ impl<T> Store<T> {/// Like `Store::new` but returns an error on allocation failure.pub fn try_new(engine: &Engine, data: T) -> Result<Self> {- let store_data = StoreData::new();+ let store_data = StoreData::new(engine);log::trace!("creating new store {:?}", store_data.id());let pkey = engine.allocator().next_available_pkey();@@ -747,8 +729,6 @@ impl<T> Store<T> {#[cfg(feature = "stack-switching")]continuations: Vec::new(),instances: wasmtime_environ::collections::PrimaryMap::new(),- #[cfg(feature = "component-model")]- num_component_instances: 0,signal_handler: None,gc_store: None,gc_roots: RootSet::default(),@@ -777,26 +757,7 @@ impl<T> Store<T> {hostcall_val_storage: Vec::new(),wasm_val_raw_storage: Vec::new(),pkey,- #[cfg(feature = "component-model")]- component_host_table: Default::default(),- #[cfg(feature = "component-model")]- component_calls: Default::default(),- #[cfg(feature = "component-model")]- host_resource_data: Default::default(),executor: Executor::new(engine)?,- #[cfg(feature = "component-model")]- concurrent_state: if engine.tunables().concurrency_support {- #[cfg(feature = "component-model-async")]- {- Some(Default::default())- }- #[cfg(not(feature = "component-model-async"))]- {- unreachable!()- }- } else {- None- },#[cfg(feature = "debug")]breakpoints: Default::default(),};@@ -874,18 +835,7 @@ impl<T> Store<T> {}fn run_manual_drop_routines(&mut self) {- // We need to drop the fibers of each component instance before- // attempting to drop the instances themselves since the fibers may need- // to be resumed and allowed to exit cleanly before we yank the state- // out from under them.- //- // This will also drop any futures which might use a `&Accessor` fields- // in their `Drop::drop` implementations, in which case they'll need to- // be called from with in the context of a `tls::set` closure.- #[cfg(feature = "component-model-async")]- if self.inner.concurrent_state.is_some() {- ComponentStoreData::drop_fibers_and_futures(&mut **self.inner);- }+ StoreData::run_manual_drop_routines(StoreContextMut(&mut self.inner));// Ensure all fiber stacks, even cached ones, are all flushed out to the// instance allocator.@@ -1711,16 +1661,9 @@ impl StoreOpaque {&self.modules}- pub(crate) fn register_module(&mut self, module: &Module) -> Result<RegisteredModuleId> {- self.modules.register_module(module, &self.engine)- }-- #[cfg(feature = "component-model")]- pub(crate) fn register_component(- &mut self,- component: &crate::component::Component,- ) -> Result<()> {- self.modules.register_component(component, &self.engine)+ #[inline]+ pub(crate) fn modules_and_engine_mut(&mut self) -> (&mut ModuleRegistry, &Engine) {+ (&mut self.modules, &self.engine)}pub(crate) fn func_refs_and_modules(&mut self) -> (&mut FuncRefs, &ModuleRegistry) {@@ -2548,104 +2491,11 @@ at https://bytecodealliance.org/security.self.pkey}- #[inline]- #[cfg(feature = "component-model-async")]- pub(crate) fn component_resource_state(- &mut self,- ) -> (- &mut vm::component::CallContexts,- &mut vm::component::HandleTable,- &mut crate::component::HostResourceData,- ) {- (- &mut self.component_calls,- &mut self.component_host_table,- &mut self.host_resource_data,- )- }-- #[cfg(feature = "component-model")]- pub(crate) fn push_component_instance(&mut self, instance: crate::component::Instance) {- // We don't actually need the instance itself right now, but it seems- // like something we will almost certainly eventually want to keep- // around, so force callers to provide it.- let _ = instance;-- self.num_component_instances += 1;- }-- #[cfg(feature = "component-model")]- pub(crate) fn component_resource_state_with_instance_and_concurrent_state(- &mut self,- instance: crate::component::Instance,- ) -> (- &mut vm::component::CallContexts,- &mut vm::component::HandleTable,- &mut crate::component::HostResourceData,- Pin<&mut vm::component::ComponentInstance>,- Option<&mut concurrent::ConcurrentState>,- ) {- (- &mut self.component_calls,- &mut self.component_host_table,- &mut self.host_resource_data,- instance.id().from_data_get_mut(&mut self.store_data),- self.concurrent_state.as_mut(),- )- }-- #[cfg(feature = "component-model")]- pub(crate) fn component_resource_tables(- &mut self,- instance: Option<crate::component::Instance>,- ) -> vm::component::ResourceTables<'_> {- self.component_resource_tables_and_host_resource_data(instance)- .0- }-- #[cfg(feature = "component-model")]- pub(crate) fn component_resource_tables_and_host_resource_data(- &mut self,- instance: Option<crate::component::Instance>,- ) -> (- vm::component::ResourceTables<'_>,- &mut crate::component::HostResourceData,- ) {- let guest = instance.map(|i| {- i.id()- .from_data_get_mut(&mut self.store_data)- .instance_states()- });-- (- vm::component::ResourceTables {- host_table: Some(&mut self.component_host_table),- calls: &mut self.component_calls,- guest,- },- &mut self.host_resource_data,- )- }-#[cfg(feature = "async")]pub(crate) fn fiber_async_state_mut(&mut self) -> &mut fiber::AsyncState {&mut self.async_state}- #[cfg(feature = "component-model-async")]- pub(crate) fn concurrent_state_mut(&mut self) -> &mut concurrent::ConcurrentState {- debug_assert!(self.concurrency_support());- self.concurrent_state.as_mut().unwrap()- }-- #[inline]- #[cfg(feature = "component-model")]- pub(crate) fn concurrency_support(&self) -> bool {- let support = self.concurrent_state.is_some();- debug_assert_eq!(support, self.engine().tunables().concurrency_support);- support- }-#[cfg(feature = "async")]pub(crate) fn has_pkey(&self) -> bool {self.pkey.is_some()@@ -2889,6 +2739,11 @@ unsafe impl<T> VMStore for StoreInner<T> {self}+ #[cfg(feature = "component-model")]+ fn component_calls(&mut self) -> &mut vm::component::CallContexts {+ self.component_call_contexts_mut()+ }+fn store_opaque(&self) -> &StoreOpaque {&self.inner}@@ -2926,11 +2781,6 @@ unsafe impl<T> VMStore for StoreInner<T> {update}- #[cfg(feature = "component-model")]- fn component_calls(&mut self) -> &mut vm::component::CallContexts {- &mut self.component_calls- }-#[cfg(feature = "debug")]fn block_on_debug_handler(&mut self, event: crate::DebugEvent<'_>) -> crate::Result<()> {if let Some(handler) = self.debug_handler.take() {@@ -3019,12 +2869,7 @@ impl Drop for StoreOpaque {allocator.deallocate_module(&mut instance.handle);}- #[cfg(feature = "component-model")]- {- for _ in 0..self.num_component_instances {- allocator.decrement_component_instance_count();- }- }+ self.store_data.decrement_allocator_resources(allocator);}}}diff --git a/crates/wasmtime/src/runtime/store/data.rs b/crates/wasmtime/src/runtime/store/data.rsindex 265e7a1ce8..dd0110b1f9 100644--- a/crates/wasmtime/src/runtime/store/data.rs+++ b/crates/wasmtime/src/runtime/store/data.rs@@ -1,7 +1,7 @@use crate::module::ModuleRegistry;use crate::runtime::vm::{self, GcStore, VMStore};use crate::store::StoreOpaque;-use crate::{StoreContext, StoreContextMut};+use crate::{Engine, StoreContext, StoreContextMut};use core::num::NonZeroU64;use core::ops::{Index, IndexMut};use core::pin::Pin;@@ -21,17 +21,33 @@ pub struct StoreData {}impl StoreData {- pub fn new() -> StoreData {+ pub fn new(engine: &Engine) -> StoreData {+ #[cfg(not(feature = "component-model"))]+ let _ = engine;StoreData {id: StoreId::allocate(),#[cfg(feature = "component-model")]- components: Default::default(),+ components: crate::component::ComponentStoreData::new(engine),}}pub fn id(&self) -> StoreId {self.id}++ pub fn run_manual_drop_routines<T>(store: StoreContextMut<T>) {+ #[cfg(feature = "component-model")]+ crate::component::ComponentStoreData::run_manual_drop_routines(store);+ #[cfg(not(feature = "component-model"))]+ let _ = store;+ }++ pub fn decrement_allocator_resources(&mut self, allocator: &dyn vm::InstanceAllocator) {+ #[cfg(feature = "component-model")]+ self.components.decrement_allocator_resources(allocator);+ #[cfg(not(feature = "component-model"))]+ let _ = allocator;+ }}// forward StoreOpaque => StoreData