A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
merge `concurrent::table::Table` back into `ResourceTable` (#11547)
Joel Dice624c82356b79
diff --git a/crates/misc/component-async-tests/tests/scenario/round_trip.rs b/crates/misc/component-async-tests/tests/scenario/round_trip.rsindex a40dd26f9b..b420e2fe4f 100644--- a/crates/misc/component-async-tests/tests/scenario/round_trip.rs+++ b/crates/misc/component-async-tests/tests/scenario/round_trip.rs@@ -232,7 +232,6 @@ pub async fn test_round_trip(let mut store = make_store();let instance = linker.instantiate_async(&mut store, &component).await?;- instance.enable_concurrent_state_debug(&mut store, true);let round_trip =component_async_tests::round_trip::bindings::RoundTrip::new(&mut store, &instance)?;diff --git a/crates/misc/component-async-tests/tests/scenario/round_trip_many.rs b/crates/misc/component-async-tests/tests/scenario/round_trip_many.rsindex c9d59a271f..0a6ff8930b 100644--- a/crates/misc/component-async-tests/tests/scenario/round_trip_many.rs+++ b/crates/misc/component-async-tests/tests/scenario/round_trip_many.rs@@ -245,7 +245,6 @@ async fn test_round_trip_many(let mut store = make_store();let instance = linker.instantiate_async(&mut store, &component).await?;- instance.enable_concurrent_state_debug(&mut store, true);let round_trip_many = component_async_tests::round_trip_many::bindings::RoundTripMany::new(&mut store, &instance,)?;diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex 498e379469..8cffc1c5be 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -48,13 +48,16 @@//! convenient to use in host functions.use crate::component::func::{self, Func, Options};-use crate::component::{Component, ComponentInstanceId, HasData, HasSelf, Instance};+use crate::component::{+ Component, ComponentInstanceId, HasData, HasSelf, Instance, Resource, ResourceTable,+ ResourceTableError,+};use crate::fiber::{self, StoreFiber, StoreFiberYield};use crate::store::{StoreInner, StoreOpaque, StoreToken};use crate::vm::component::{CallContext, ComponentInstance, InstanceFlags, ResourceTables, TransmitLocalState,};-use crate::vm::{SendSyncPtr, VMFuncRef, VMMemoryDefinition, VMStore};+use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMMemoryDefinition, VMStore};use crate::{AsContext, AsContextMut, StoreContext, StoreContextMut, ValRaw};use anyhow::{Context as _, Result, anyhow, bail};use error_contexts::GlobalErrorContextRefCount;@@ -75,10 +78,9 @@ use std::ops::DerefMut;use std::pin::{Pin, pin};use std::ptr::{self, NonNull};use std::slice;-use std::sync::Mutex;use std::task::{Context, Poll, Waker};use std::vec::Vec;-use table::{Table, TableDebug, TableError, TableId};+use table::{TableDebug, TableId};use wasmtime_environ::component::{CanonicalOptions, CanonicalOptionsDataModel, ExportIndex, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS,OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,@@ -307,7 +309,7 @@ where////// * It's intentional that `Accessor` cannot be cloned, it needs to stay within/// the lifetime of a single future.-/// * A futures is expected to, however, close over an `Accessor` and keep it+/// * A future is expected to, however, close over an `Accessor` and keep it/// alive probably for the duration of the entire future./// * Different host futures will be given different `Accessor`s, and that's/// intentional.@@ -632,7 +634,7 @@ impl GuestCall {/// - the call is for a not-yet started task and the (sub-)component/// instance to be called has backpressure enabledfn is_ready(&self, state: &mut ConcurrentState) -> Result<bool> {- let task_instance = state.get(self.task)?.instance;+ let task_instance = state.get_mut(self.task)?.instance;let state = state.instance_state(task_instance);let ready = match &self.kind {GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,@@ -650,7 +652,7 @@ impl GuestCall {/// Job to be run on a worker fiber.enum WorkerItem {GuestCall(GuestCall),- Function(Mutex<Box<dyn FnOnce(&mut dyn VMStore, Instance) -> Result<()> + Send>>),+ Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore, Instance) -> Result<()> + Send>>),}/// Represents state related to an in-progress poll operation (e.g. `task.poll`@@ -667,7 +669,7 @@ struct PollParams {/// component instance.enum WorkItem {/// A host task to be pushed to `ConcurrentState::futures`.- PushFuture(Mutex<HostTaskFuture>),+ PushFuture(AlwaysMut<HostTaskFuture>),/// A fiber to resume.ResumeFiber(StoreFiber<'static>),/// A pending call into guest code for a given guest task.@@ -675,7 +677,7 @@ enum WorkItem {/// A pending `task.poll` or `CallbackCode.POLL` operation.Poll(PollParams),/// A job to run on a worker fiber.- WorkerFunction(Mutex<Box<dyn FnOnce(&mut dyn VMStore, Instance) -> Result<()> + Send>>),+ WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore, Instance) -> Result<()> + Send>>),}impl fmt::Debug for WorkItem {@@ -928,14 +930,14 @@ impl ComponentInstance {let concurrent_state = self.concurrent_state_mut();let (waitable, expected_caller_instance, delete) = if is_host {let id = TableId::<HostTask>::new(rep);- let task = concurrent_state.get(id)?;+ let task = concurrent_state.get_mut(id)?;if task.join_handle.is_some() {bail!("cannot drop a subtask which has not yet resolved");}(Waitable::Host(id), task.caller_instance, true)} else {let id = TableId::<GuestTask>::new(rep);- let task = concurrent_state.get(id)?;+ let task = concurrent_state.get_mut(id)?;if task.lift_result.is_some() {bail!("cannot drop a subtask which has not yet resolved");}@@ -966,23 +968,6 @@ impl ComponentInstance {}impl Instance {- /// Enable or disable concurrent state debugging mode for e.g. integration- /// tests.- ///- /// This will avoid re-using deleted handles, making it easier to catch- /// e.g. "use-after-delete" and "double-delete" errors. It can also make- /// reading trace output easier since it ensures handles are never- /// repurposed.- #[doc(hidden)]- pub fn enable_concurrent_state_debug(&self, mut store: impl AsContextMut, enable: bool) {- self.id()- .get_mut(store.as_context_mut().0)- .concurrent_state_mut()- .table- .enable_debug(enable);- // TODO: do the same for the tables holding guest-facing handles- }-/// Assert that all the relevant tables and queues in the concurrent state/// for this instance are empty.///@@ -1003,19 +988,15 @@ impl Instance {.all(|(_, table)| table.is_empty()));let state = instance.concurrent_state_mut();- assert!(state.table.is_empty(), "non-empty table: {:?}", state.table);+ assert!(+ state.table.get_mut().is_empty(),+ "non-empty table: {:?}",+ state.table+ );assert!(state.high_priority.is_empty());assert!(state.low_priority.is_empty());assert!(state.guest_task.is_none());- assert!(- state- .futures- .get_mut()- .unwrap()- .as_ref()- .unwrap()- .is_empty()- );+ assert!(state.futures.get_mut().as_ref().unwrap().is_empty());assert!(state.instance_states@@ -1177,7 +1158,6 @@ impl Instance {.concurrent_state_mut(store.0).futures.get_mut()- .unwrap().take().unwrap();let mut next = pin!(futures.next());@@ -1207,8 +1187,9 @@ impl Instance {// by adding another variant to `HostOutput` to// distinguish which ones need it and which// don't.- self.concurrent_state_mut(store.0)- .push_high_priority(WorkItem::WorkerFunction(Mutex::new(fun)))+ self.concurrent_state_mut(store.0).push_high_priority(+ WorkItem::WorkerFunction(AlwaysMut::new(fun)),+ )}}Poll::Ready(true)@@ -1301,11 +1282,7 @@ impl Instance {// Put the `ConcurrentState::futures` back into the instance before// we return or handle any work items since one or more of those// items might append more futures.- *self- .concurrent_state_mut(store.0)- .futures- .get_mut()- .unwrap() = Some(futures);+ *self.concurrent_state_mut(store.0).futures.get_mut() = Some(futures);match result? {// The future we were passed as an argument completed, so we@@ -1334,10 +1311,9 @@ impl Instance {self.concurrent_state_mut(store.0).futures.get_mut()- .unwrap().as_mut().unwrap()- .push(future.into_inner().unwrap());+ .push(future.into_inner());}WorkItem::ResumeFiber(fiber) => {self.resume_fiber(store.0, fiber).await?;@@ -1361,7 +1337,7 @@ impl Instance {}}- let runtime_instance = state.get(call.task)?.instance;+ let runtime_instance = state.get_mut(call.task)?.instance;state.instance_state(runtime_instance).pending@@ -1453,7 +1429,7 @@ impl Instance {loop {match self.concurrent_state_mut(store).worker_item.take().unwrap() {WorkerItem::GuestCall(call) => self.handle_guest_call(store, call)?,- WorkerItem::Function(fun) => fun.into_inner().unwrap()(store, self)?,+ WorkerItem::Function(fun) => fun.into_inner()(store, self)?,}self.suspend(store, SuspendReason::NeedWork)?;@@ -1589,7 +1565,7 @@ impl Instance {) -> Result<()> {if self.concurrent_state_mut(store)- .get(guest_task)?+ .get_mut(guest_task)?.lift_result.is_some(){@@ -1698,7 +1674,10 @@ impl Instance {)};- let callee_instance = self.concurrent_state_mut(store.0).get(guest_task)?.instance;+ let callee_instance = self+ .concurrent_state_mut(store.0)+ .get_mut(guest_task)?+ .instance;let fun = if callback.is_some() {assert!(async_);@@ -1788,7 +1767,7 @@ impl Instance {// been called.if instance.concurrent_state_mut(store)- .get(guest_task)?+ .get_mut(guest_task)?.lift_result.is_some(){@@ -1804,7 +1783,7 @@ impl Instance {let state = instance.concurrent_state_mut(store);state.exit_instance(callee_instance)?;- assert!(state.get(guest_task)?.result.is_none());+ assert!(state.get_mut(guest_task)?.result.is_none());state.get_mut(guest_task)?.lift_result.take().unwrap()};@@ -1827,7 +1806,7 @@ impl Instance {if instance.concurrent_state_mut(store)- .get(guest_task)?+ .get_mut(guest_task)?.call_post_return_automatically(){unsafe { flags.set_needs_post_return(false) }@@ -2138,7 +2117,7 @@ impl Instance {let async_caller = storage.is_none();let state = self.concurrent_state_mut(store.0);let guest_task = state.guest_task.unwrap();- let may_enter_after_call = state.get(guest_task)?.call_post_return_automatically();+ let may_enter_after_call = state.get_mut(guest_task)?.call_post_return_automatically();let callee = SendSyncPtr::new(NonNull::new(callee).unwrap());let param_count = usize::try_from(param_count).unwrap();assert!(param_count <= MAX_FLAT_PARAMS);@@ -2574,7 +2553,7 @@ impl Instance {.ok_or_else(|| {anyhow!("`task.return` or `task.cancel` called more than once for current task")})?;- assert!(state.get(guest_task)?.result.is_none());+ assert!(state.get_mut(guest_task)?.result.is_none());let invalid = ty != lift.ty|| string_encoding != lift.string_encoding@@ -2648,7 +2627,7 @@ impl Instance {) -> Result<()> {if self.concurrent_state_mut(store)- .get(guest_task)?+ .get_mut(guest_task)?.call_post_return_automatically(){let (calls, host_table, _, instance) = store@@ -2667,7 +2646,7 @@ impl Instance {// should be a non-`None` value.let function_index = self.concurrent_state_mut(store)- .get(guest_task)?+ .get_mut(guest_task)?.function_index.unwrap();@@ -2784,7 +2763,7 @@ impl Instance {log::trace!("waitable check for {guest_task:?}; set {set:?}");let state = self.concurrent_state_mut(store);- let task = state.get(guest_task)?;+ let task = state.get_mut(guest_task)?;if wait && task.callback.is_some() {bail!("cannot call `task.wait` from async-lifted export with callback");@@ -2795,7 +2774,7 @@ impl Instance {if wait {let set = set.unwrap();- if task.event.is_none() && state.get(set)?.ready.is_empty() {+ if task.event.is_none() && state.get_mut(set)?.ready.is_empty() {let old = state.get_mut(guest_task)?.wake_on_cancel.replace(set);assert!(old.is_none());@@ -2868,12 +2847,14 @@ impl Instance {let id = TableId::<HostTask>::new(rep);(Waitable::Host(id),- self.concurrent_state_mut(store).get(id)?.caller_instance,+ self.concurrent_state_mut(store)+ .get_mut(id)?+ .caller_instance,)} else {let id = TableId::<GuestTask>::new(rep);if let Caller::Guest { instance, .. } =- &self.concurrent_state_mut(store).get(id)?.caller+ &self.concurrent_state_mut(store).get_mut(id)?.caller{(Waitable::Guest(id), *instance)} else {@@ -3962,15 +3943,11 @@ pub struct ConcurrentState {guest_task: Option<TableId<GuestTask>>,/// The set of pending host and background tasks, if any.///- /// We must wrap this in a `Mutex` to ensure that `ComponentInstance` and- /// `Store` satisfy a `Sync` bound, but it can't actually be accessed from- /// more than one thread at a time.- ////// See `ComponentInstance::poll_until` for where we temporarily take this/// out, poll it, then put it back to avoid any mutable aliasing hazards.- futures: Mutex<Option<FuturesUnordered<HostTaskFuture>>>,+ futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,/// The table of waitables, waitable sets, etc.- table: Table,+ table: AlwaysMut<ResourceTable>,/// Per (sub-)component instance states.////// See `InstanceState` for details and note that this map is lazily@@ -4016,8 +3993,8 @@ impl ConcurrentState {pub(crate) fn new(component: &Component) -> Self {Self {guest_task: None,- table: Table::new(),- futures: Mutex::new(Some(FuturesUnordered::new())),+ table: AlwaysMut::new(ResourceTable::new()),+ futures: AlwaysMut::new(Some(FuturesUnordered::new())),instance_states: HashMap::new(),high_priority: Vec::new(),low_priority: Vec::new(),@@ -4050,7 +4027,7 @@ impl ConcurrentState {fibers: &mut Vec<StoreFiber<'static>>,futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,) {- for entry in self.table.iter_mut() {+ for entry in self.table.get_mut().iter_mut() {if let Some(set) = entry.downcast_mut::<WaitableSet>() {for mode in mem::take(&mut set.waiting).into_values() {if let WaitMode::Fiber(fiber) = mode {@@ -4073,10 +4050,9 @@ impl ConcurrentState {WorkItem::PushFuture(future) => {self.futures.get_mut()- .unwrap().as_mut().unwrap()- .push(future.into_inner().unwrap());+ .push(future.into_inner());}_ => {}}@@ -4086,7 +4062,7 @@ impl ConcurrentState {take_items(&mut self.high_priority);take_items(&mut self.low_priority);- if let Some(them) = self.futures.get_mut().unwrap().take() {+ if let Some(them) = self.futures.get_mut().take() {futures.push(them);}}@@ -4095,36 +4071,39 @@ impl ConcurrentState {self.instance_states.entry(instance).or_default()}- fn push<V: Send + Sync + 'static>(&mut self, value: V) -> Result<TableId<V>, TableError> {- self.table.push(value)- }-- fn get<V: 'static>(&self, id: TableId<V>) -> Result<&V, TableError> {- self.table.get(id)+ fn push<V: Send + Sync + 'static>(+ &mut self,+ value: V,+ ) -> Result<TableId<V>, ResourceTableError> {+ self.table.get_mut().push(value).map(TableId::from)}- fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, TableError> {- self.table.get_mut(id)+ fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {+ self.table.get_mut().get_mut(&Resource::from(id))}- pub fn add_child<T, U>(+ pub fn add_child<T: 'static, U: 'static>(&mut self,child: TableId<T>,parent: TableId<U>,- ) -> Result<(), TableError> {- self.table.add_child(child, parent)+ ) -> Result<(), ResourceTableError> {+ self.table+ .get_mut()+ .add_child(Resource::from(child), Resource::from(parent))}- pub fn remove_child<T, U>(+ pub fn remove_child<T: 'static, U: 'static>(&mut self,child: TableId<T>,parent: TableId<U>,- ) -> Result<(), TableError> {- self.table.remove_child(child, parent)+ ) -> Result<(), ResourceTableError> {+ self.table+ .get_mut()+ .remove_child(Resource::from(child), Resource::from(parent))}- fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, TableError> {- self.table.delete(id)+ fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {+ self.table.get_mut().delete(Resource::from(id))}fn push_future(&mut self, future: HostTaskFuture) {@@ -4134,7 +4113,7 @@ impl ConcurrentState {// so it has exclusive access while polling it. Therefore, we push a// work item to the "high priority" queue, which will actually push to// `ConcurrentState::futures` later.- self.push_high_priority(WorkItem::PushFuture(Mutex::new(future)));+ self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));}fn push_high_priority(&mut self, item: WorkItem) {@@ -4157,7 +4136,7 @@ impl ConcurrentState {/// if there are no activation records on the stack (i.e. the "may_enter"/// field is `true`) for that instance.fn may_enter(&mut self, mut guest_task: TableId<GuestTask>) -> bool {- let guest_instance = self.get(guest_task).unwrap().instance;+ let guest_instance = self.get_mut(guest_task).unwrap().instance;// Walk the task tree back to the root, looking for potential// reentrance.@@ -4237,7 +4216,7 @@ impl ConcurrentState {/// Implements the `context.get` intrinsic.pub(crate) fn context_get(&mut self, slot: u32) -> Result<u32> {let task = self.guest_task.unwrap();- let val = self.get(task)?.context[usize::try_from(slot).unwrap()];+ let val = self.get_mut(task)?.context[usize::try_from(slot).unwrap()];log::trace!("context_get {task:?} slot {slot} val {val:#x}");Ok(val)}diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rsindex 327431a24f..3da9eedf4c 100644--- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs+++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs@@ -6,8 +6,8 @@ use crate::component::matching::InstanceType;use crate::component::values::{ErrorContextAny, FutureAny, StreamAny};use crate::component::{AsAccessor, Instance, Lower, Val, WasmList, WasmStr};use crate::store::{StoreOpaque, StoreToken};-use crate::vm::VMStore;use crate::vm::component::{ComponentInstance, HandleTable, TransmitLocalState};+use crate::vm::{AlwaysMut, VMStore};use crate::{AsContextMut, StoreContextMut, ValRaw};use anyhow::{Context, Result, anyhow, bail};use buffers::Extender;@@ -21,7 +21,7 @@ use std::marker::PhantomData;use std::mem::{self, MaybeUninit};use std::pin::Pin;use std::string::{String, ToString};-use std::sync::{Arc, Mutex};+use std::sync::Arc;use std::task::{Poll, Waker};use std::vec::Vec;use wasmtime_environ::component::{@@ -291,7 +291,7 @@ async fn watch_reader(accessor: impl AsAccessor, instance: Instance, id: TableId.as_accessor().with(|mut access| {let concurrent_state = instance.concurrent_state_mut(access.as_context_mut().0);- let state_id = concurrent_state.get(id)?.state;+ let state_id = concurrent_state.get_mut(id)?.state;let state = concurrent_state.get_mut(state_id)?;anyhow::Ok(if matches!(&state.read, ReadState::Dropped) {Poll::Ready(())@@ -313,7 +313,7 @@ async fn watch_writer(accessor: impl AsAccessor, instance: Instance, id: TableId.as_accessor().with(|mut access| {let concurrent_state = instance.concurrent_state_mut(access.as_context_mut().0);- let state_id = concurrent_state.get(id)?.state;+ let state_id = concurrent_state.get_mut(id)?.state;let state = concurrent_state.get_mut(state_id)?;anyhow::Ok(if matches!(@@ -436,7 +436,7 @@ impl<T> FutureWriter<T> {whereT: func::Lower + Send + Sync + 'static,{- let id = mem::replace(&mut self.id, TableId::new(0));+ let id = mem::replace(&mut self.id, TableId::new(u32::MAX));let default = self.default;self.instance.host_drop_writer(store.as_context_mut(), id, Some(&move || Ok(default())))@@ -638,7 +638,7 @@ impl<T> FutureReader<T> {};let store = store.as_context_mut();let id = TableId::<TransmitHandle>::new(*rep);- instance.concurrent_state_mut(store.0).get(id)?; // Just make sure it's present+ instance.concurrent_state_mut(store.0).get_mut(id)?; // Just make sure it's presentOk(Self::new(id, instance))}@@ -659,7 +659,7 @@ impl<T> FutureReader<T> {future.common.handle = None;let state = future.state;- if concurrent_state.get(state)?.done {+ if concurrent_state.get_mut(state)?.done {bail!("cannot lift future after previous read succeeded");}@@ -678,7 +678,7 @@ impl<T> FutureReader<T> {/// a panic.pub fn close(&mut self, mut store: impl AsContextMut) {// `self` should never be used again, but leave an invalid handle there just in case.- let id = mem::replace(&mut self.id, TableId::new(0));+ let id = mem::replace(&mut self.id, TableId::new(u32::MAX));self.instance.host_drop_reader(store.as_context_mut().0, id, TransmitKind::Future).unwrap();@@ -721,8 +721,8 @@ pub(crate) fn lower_future_to_index<U>(InterfaceType::Future(dst) => {let concurrent_state = cx.instance_mut().concurrent_state_mut();let id = TableId::<TransmitHandle>::new(rep);- let state = concurrent_state.get(id)?.state;- let rep = concurrent_state.get(state)?.read_handle.rep();+ let state = concurrent_state.get_mut(id)?.state;+ let rep = concurrent_state.get_mut(state)?.read_handle.rep();let handle = cx.instance_mut()@@ -994,7 +994,7 @@ impl<T> StreamWriter<T> {/// a panic.pub fn close(&mut self, mut store: impl AsContextMut) {// `self` should never be used again, but leave an invalid handle there just in case.- let id = mem::replace(&mut self.id, TableId::new(0));+ let id = mem::replace(&mut self.id, TableId::new(u32::MAX));self.instance.host_drop_writer(store.as_context_mut(), id, None::<&dyn Fn() -> Result<()>>).unwrap()@@ -1210,7 +1210,7 @@ impl<T> StreamReader<T> {};let store = store.as_context_mut();let id = TableId::<TransmitHandle>::new(*rep);- instance.concurrent_state_mut(store.0).get(id)?; // Just make sure it's present+ instance.concurrent_state_mut(store.0).get_mut(id)?; // Just make sure it's presentOk(Self::new(id, instance))}@@ -1246,7 +1246,7 @@ impl<T> StreamReader<T> {/// a panic.pub fn close(&mut self, mut store: impl AsContextMut) {// `self` should never be used again, but leave an invalid handle there just in case.- let id = mem::replace(&mut self.id, TableId::new(0));+ let id = mem::replace(&mut self.id, TableId::new(u32::MAX));self.instance.host_drop_reader(store.as_context_mut().0, id, TransmitKind::Stream).unwrap()@@ -1289,8 +1289,8 @@ pub(crate) fn lower_stream_to_index<U>(InterfaceType::Stream(dst) => {let concurrent_state = cx.instance_mut().concurrent_state_mut();let id = TableId::<TransmitHandle>::new(rep);- let state = concurrent_state.get(id)?.state;- let rep = concurrent_state.get(state)?.read_handle.rep();+ let state = concurrent_state.get_mut(id)?.state;+ let rep = concurrent_state.get_mut(state)?.read_handle.rep();let handle = cx.instance_mut()@@ -1620,8 +1620,8 @@ struct TransmitState {impl Default for TransmitState {fn default() -> Self {Self {- write_handle: TableId::new(0),- read_handle: TableId::new(0),+ write_handle: TableId::new(u32::MAX),+ read_handle: TableId::new(u32::MAX),read: ReadState::Open,write: WriteState::Open,reader_watcher: None,@@ -1790,7 +1790,7 @@ impl Instance {kind: TransmitKind,post_write: PostWrite,) -> Result<Result<HostResult<B>, oneshot::Receiver<HostResult<B>>>> {- let transmit_id = self.concurrent_state_mut(store.0).get(id)?.state;+ let transmit_id = self.concurrent_state_mut(store.0).get_mut(id)?.state;let transmit = self.concurrent_state_mut(store.0).get_mut(transmit_id)@@ -1887,7 +1887,7 @@ impl Instance {let (tx, rx) = oneshot::channel();let token = StoreToken::new(store.as_context_mut());self.concurrent_state_mut(store.0).push_high_priority(- WorkItem::WorkerFunction(Mutex::new(Box::new(move |store, _| {+ WorkItem::WorkerFunction(AlwaysMut::new(Box::new(move |store, _| {_ = tx.send(accept(token.as_context_mut(store))?);Ok(())}))),@@ -1956,7 +1956,7 @@ impl Instance {mut buffer: B,kind: TransmitKind,) -> Result<Result<HostResult<B>, oneshot::Receiver<HostResult<B>>>> {- let transmit_id = self.concurrent_state_mut(store.0).get(id)?.state;+ let transmit_id = self.concurrent_state_mut(store.0).get_mut(id)?.state;let transmit = self.concurrent_state_mut(store.0).get_mut(transmit_id)@@ -2093,7 +2093,7 @@ impl Instance {id: TableId<TransmitHandle>,kind: TransmitKind,) -> Result<()> {- let transmit_id = self.concurrent_state_mut(store).get(id)?.state;+ let transmit_id = self.concurrent_state_mut(store).get_mut(id)?.state;let state = self.concurrent_state_mut(store);let transmit = state.get_mut(transmit_id)@@ -2182,7 +2182,7 @@ impl Instance {id: TableId<TransmitHandle>,default: Option<&dyn Fn() -> Result<T>>,) -> Result<()> {- let transmit_id = self.concurrent_state_mut(store.0).get(id)?.state;+ let transmit_id = self.concurrent_state_mut(store.0).get_mut(id)?.state;let transmit = self.concurrent_state_mut(store.0).get_mut(transmit_id)@@ -2499,7 +2499,7 @@ impl Instance {*state = TransmitLocalState::Busy;let transmit_handle = TableId::<TransmitHandle>::new(rep);let concurrent_state = self.concurrent_state_mut(store.0);- let transmit_id = concurrent_state.get(transmit_handle)?.state;+ let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;let transmit = concurrent_state.get_mut(transmit_id)?;log::trace!("guest_write {transmit_handle:?} (handle {handle}; state {transmit_id:?}); {:?}",@@ -2712,7 +2712,7 @@ impl Instance {*state = TransmitLocalState::Busy;let transmit_handle = TableId::<TransmitHandle>::new(rep);let concurrent_state = self.concurrent_state_mut(store.0);- let transmit_id = concurrent_state.get(transmit_handle)?.state;+ let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;let transmit = concurrent_state.get_mut(transmit_id)?;log::trace!("guest_read {transmit_handle:?} (handle {handle}; state {transmit_id:?}); {:?}",@@ -3106,7 +3106,7 @@ impl ComponentInstance {}}let state = self.concurrent_state_mut();- let rep = state.get(id)?.state.rep();+ let rep = state.get_mut(id)?.state.rep();state.host_cancel_write(rep)}@@ -3132,7 +3132,7 @@ impl ComponentInstance {}}let state = self.concurrent_state_mut();- let rep = state.get(id)?.state.rep();+ let rep = state.get_mut(id)?.state.rep();state.host_cancel_read(rep)}diff --git a/crates/wasmtime/src/runtime/component/concurrent/table.rs b/crates/wasmtime/src/runtime/component/concurrent/table.rsindex 7829c05786..6f14c405a3 100644--- a/crates/wasmtime/src/runtime/component/concurrent/table.rs+++ b/crates/wasmtime/src/runtime/component/concurrent/table.rs@@ -1,20 +1,14 @@-// TODO: This duplicates a lot of resource_table.rs; consider reducing that-// duplication: https://github.com/bytecodealliance/wasmtime/issues/11190.-//-// The main difference between this and resource_table.rs is that the key type,-// `TableId<T>` implements `Copy`, making them much easier to work with than-// `Resource<T>`. I've also added a `Table::delete_any` function, useful for-// implementing `subtask.drop`.--use std::any::Any;-use std::boxed::Box;+use crate::component::Resource;use std::cmp::Ordering;-use std::collections::BTreeSet;use std::fmt;use std::hash::{Hash, Hasher};use std::marker::PhantomData;-use std::vec::Vec;+/// Represents a `ResourceTable` entry for a `waitable` or `waitable-set`.+///+/// This is equivalent to a `Resource<T>`, except without any tracking of borrow+/// status (since neither `waitable`s nor `waitable-set`s can be borrowed) or+/// other resource-specific bookkeeping.pub struct TableId<T> {rep: u32,_marker: PhantomData<fn() -> T>,@@ -79,313 +73,17 @@ impl<T> TableId<T> {}}-#[derive(Debug)]-/// Errors returned by operations on `Table`-pub enum TableError {- /// Table has no free keys- Full,- /// Entry not present in table- NotPresent,- /// Resource present in table, but with a different type- WrongType,- /// Entry cannot be deleted because child entrys exist in the table.- HasChildren,-}--impl std::fmt::Display for TableError {- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {- match self {- Self::Full => write!(f, "table has no free keys"),- Self::NotPresent => write!(f, "entry not present"),- Self::WrongType => write!(f, "entry is of another type"),- Self::HasChildren => write!(f, "entry has children"),- }- }-}-impl std::error::Error for TableError {}--/// The `Table` type maps a `TableId` to its entry.-#[derive(Default)]-pub struct Table {- entries: Vec<Entry>,- free_head: Option<usize>,- debug: bool,-}--enum Entry {- Free { next: Option<usize> },- Occupied { entry: TableEntry },-}--impl Entry {- pub fn occupied(&self) -> Option<&TableEntry> {- match self {- Self::Occupied { entry } => Some(entry),- Self::Free { .. } => None,- }- }-- pub fn occupied_mut(&mut self) -> Option<&mut TableEntry> {- match self {- Self::Occupied { entry } => Some(entry),- Self::Free { .. } => None,- }- }-}--struct Tombstone;--/// This structure tracks parent and child relationships for a given table entry.-///-/// Parents and children are referred to by table index. We maintain the-/// following invariants to prevent orphans and cycles:-/// * parent can only be assigned on creating the entry.-/// * parent, if some, must exist when creating the entry.-/// * whenever a child is created, its index is added to children.-/// * whenever a child is deleted, its index is removed from children.-/// * an entry with children may not be deleted.-struct TableEntry {- /// The entry in the table- entry: Box<dyn Any + Send + Sync>,- /// The index of the parent of this entry, if it has one.- parent: Option<u32>,- /// The indicies of any children of this entry.- children: BTreeSet<u32>,-}--impl TableEntry {- fn new(entry: Box<dyn Any + Send + Sync>, parent: Option<u32>) -> Self {+impl<T: 'static> From<Resource<T>> for TableId<T> {+ fn from(value: Resource<T>) -> Self {Self {- entry,- parent,- children: BTreeSet::new(),- }- }- fn add_child(&mut self, child: u32) {- assert!(self.children.insert(child));- }- fn remove_child(&mut self, child: u32) {- assert!(self.children.remove(&child));- }-}--impl Table {- /// Create an empty table- pub fn new() -> Self {- let mut me = Self {- entries: Vec::new(),- free_head: None,- debug: false,- };-- // Reserve 0 as an invalid entry.- me.push(Tombstone).unwrap();-- me- }-- /// Returns whether or not this table is empty.- pub fn is_empty(&self) -> bool {- self.entries.iter().all(|entry| match entry {- Entry::Free { .. } => true,- Entry::Occupied { entry } => entry.entry.downcast_ref::<Tombstone>().is_some(),- })- }-- /// Enable or disable "debug mode".- ///- /// When this is enabled, the `delete` method will leave a tombstone in- /// place of the deleted item rather than add the entry to the free list.- /// This can help uncover "use-after-delete" or "double-delete" bugs which- /// might otherwise go unnoticed if an entry is repopulated.- pub fn enable_debug(&mut self, enable: bool) {- self.debug = enable;- }-- /// Inserts a new entry into this table, returning a corresponding- /// `TableId<T>` which can be used to refer to it after it was inserted.- pub fn push<T: Send + Sync + 'static>(&mut self, entry: T) -> Result<TableId<T>, TableError> {- let idx = self.push_(TableEntry::new(Box::new(entry), None))?;- Ok(TableId::new(idx))- }-- /// Pop an index off of the free list, if it's not empty.- fn pop_free_list(&mut self) -> Option<usize> {- if let Some(ix) = self.free_head {- // Advance free_head to the next entry if one is available.- match &self.entries[ix] {- Entry::Free { next } => self.free_head = *next,- Entry::Occupied { .. } => unreachable!(),- }- Some(ix)- } else {- None- }- }-- /// Free an entry in the table, returning its [`TableEntry`]. Add the index to the free list.- fn free_entry(&mut self, ix: usize) -> TableEntry {- if self.debug {- match std::mem::replace(- &mut self.entries[ix],- Entry::Occupied {- entry: TableEntry {- entry: Box::new(Tombstone),- parent: None,- children: BTreeSet::new(),- },- },- ) {- Entry::Occupied { entry } => entry,- Entry::Free { .. } => unreachable!(),- }- } else {- let entry = match std::mem::replace(- &mut self.entries[ix],- Entry::Free {- next: self.free_head,- },- ) {- Entry::Occupied { entry } => entry,- Entry::Free { .. } => unreachable!(),- };-- self.free_head = Some(ix);-- entry- }- }-- /// Push a new entry into the table, returning its handle. This will prefer to use free entries- /// if they exist, falling back on pushing new entries onto the end of the table.- fn push_(&mut self, e: TableEntry) -> Result<u32, TableError> {- if let Some(free) = self.pop_free_list() {- self.entries[free] = Entry::Occupied { entry: e };- Ok(u32::try_from(free).unwrap())- } else {- let ix = self- .entries- .len()- .try_into()- .map_err(|_| TableError::Full)?;- self.entries.push(Entry::Occupied { entry: e });- Ok(ix)- }- }-- fn occupied(&self, key: u32) -> Result<&TableEntry, TableError> {- self.entries- .get(key as usize)- .and_then(Entry::occupied)- .ok_or(TableError::NotPresent)- }-- fn occupied_mut(&mut self, key: u32) -> Result<&mut TableEntry, TableError> {- self.entries- .get_mut(key as usize)- .and_then(Entry::occupied_mut)- .ok_or(TableError::NotPresent)- }-- pub fn add_child<T, U>(- &mut self,- child: TableId<T>,- parent: TableId<U>,- ) -> Result<(), TableError> {- let entry = self.occupied_mut(child.rep())?;- assert!(entry.parent.is_none());- entry.parent = Some(parent.rep());- self.occupied_mut(parent.rep())?.add_child(child.rep());- Ok(())- }-- pub fn remove_child<T, U>(- &mut self,- child: TableId<T>,- parent: TableId<U>,- ) -> Result<(), TableError> {- let entry = self.occupied_mut(child.rep())?;- assert_eq!(entry.parent, Some(parent.rep()));- entry.parent = None;- self.occupied_mut(parent.rep())?.remove_child(child.rep());- Ok(())- }-- /// Get an immutable reference to a task of a given type at a given index.- ///- /// Multiple shared references can be borrowed at any given time.- pub fn get<T: 'static>(&self, key: TableId<T>) -> Result<&T, TableError> {- self.get_(key.rep())?- .downcast_ref()- .ok_or(TableError::WrongType)- }-- fn get_(&self, key: u32) -> Result<&dyn Any, TableError> {- let r = self.occupied(key)?;- Ok(&*r.entry)- }-- /// Get an mutable reference to a task of a given type at a given index.- pub fn get_mut<T: 'static>(&mut self, key: TableId<T>) -> Result<&mut T, TableError> {- self.get_mut_(key.rep())?- .downcast_mut()- .ok_or(TableError::WrongType)- }-- pub fn get_mut_(&mut self, key: u32) -> Result<&mut dyn Any, TableError> {- let r = self.occupied_mut(key)?;- Ok(&mut *r.entry)- }-- /// Delete the specified task- pub fn delete<T: 'static>(&mut self, key: TableId<T>) -> Result<T, TableError> {- self.delete_entry(key.rep())?- .entry- .downcast()- .map(|v| *v)- .map_err(|_| TableError::WrongType)- }-- fn delete_entry(&mut self, key: u32) -> Result<TableEntry, TableError> {- if !self.occupied(key)?.children.is_empty() {- return Err(TableError::HasChildren);- }- let e = self.free_entry(key as usize);- if let Some(parent) = e.parent {- // Remove deleted task from parent's child list. Parent must still- // be present because it cant be deleted while still having- // children:- self.occupied_mut(parent)- .expect("missing parent")- .remove_child(key);+ rep: value.rep(),+ _marker: PhantomData,}- Ok(e)- }-- pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut (dyn Any + Send + Sync)> {- self.entries.iter_mut().filter_map(|entry| match entry {- Entry::Occupied { entry } => Some(&mut *entry.entry),- Entry::Free { .. } => None,- })}}-impl fmt::Debug for Table {- fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {- write!(f, "[")?;- let mut wrote = false;- for (index, entry) in self.entries.iter().enumerate() {- if let Entry::Occupied { entry } = entry {- if entry.entry.downcast_ref::<Tombstone>().is_none() {- if wrote {- write!(f, ", ")?;- } else {- wrote = true;- }- write!(f, "{index}")?;- }- }- }- write!(f, "]")+impl<T: 'static> From<TableId<T>> for Resource<T> {+ fn from(value: TableId<T>) -> Self {+ Resource::new_own(value.rep)}}diff --git a/crates/wasmtime/src/runtime/component/resource_table.rs b/crates/wasmtime/src/runtime/component/resource_table.rsindex ab8e0e3c60..e427518bb2 100644--- a/crates/wasmtime/src/runtime/component/resource_table.rs+++ b/crates/wasmtime/src/runtime/component/resource_table.rs@@ -3,6 +3,7 @@ use crate::prelude::*;use alloc::collections::{BTreeMap, BTreeSet};use core::any::Any;use core::fmt;+use core::mem;#[derive(Debug)]/// Errors returned by operations on `ResourceTable`@@ -32,7 +33,6 @@ impl fmt::Display for ResourceTableError {impl core::error::Error for ResourceTableError {}/// The `ResourceTable` type maps a `Resource<T>` to its `T`.-#[derive(Debug)]pub struct ResourceTable {entries: Vec<Entry>,free_head: Option<usize>,@@ -60,12 +60,13 @@ impl Entry {}}+struct Tombstone;+/// This structure tracks parent and child relationships for a given table entry.////// Parents and children are referred to by table index. We maintain the-/// following invariants to prevent orphans and cycles:-/// * parent can only be assigned on creating the entry.-/// * parent, if some, must exist when creating the entry.+/// following invariants:+/// * the parent must exist when adding a child./// * whenever a child is created, its index is added to children./// * whenever a child is deleted, its index is removed from children./// * an entry with children may not be deleted.@@ -106,6 +107,17 @@ impl ResourceTable {}}+ /// Returns whether or not this table is empty.+ ///+ /// Note that this is an `O(n)` operation, where `n` is the number of+ /// entries in the backing `Vec`.+ pub fn is_empty(&self) -> bool {+ self.entries.iter().all(|entry| match entry {+ Entry::Free { .. } => true,+ Entry::Occupied { entry } => entry.entry.downcast_ref::<Tombstone>().is_some(),+ })+ }+/// Create an empty table with at least the specified capacity.pub fn with_capacity(capacity: usize) -> Self {ResourceTable {@@ -139,20 +151,39 @@ impl ResourceTable {}/// Free an entry in the table, returning its [`TableEntry`]. Add the index to the free list.- fn free_entry(&mut self, ix: usize) -> TableEntry {- let entry = match core::mem::replace(- &mut self.entries[ix],- Entry::Free {- next: self.free_head,- },- ) {- Entry::Occupied { entry } => entry,- Entry::Free { .. } => unreachable!(),- };-- self.free_head = Some(ix);-- entry+ fn free_entry(&mut self, ix: usize, debug: bool) -> TableEntry {+ if debug {+ // Instead of making this entry available for reuse, we leave a+ // tombstone in debug mode. This helps detect use-after-delete and+ // double-delete bugs.+ match mem::replace(+ &mut self.entries[ix],+ Entry::Occupied {+ entry: TableEntry {+ entry: Box::new(Tombstone),+ parent: None,+ children: BTreeSet::new(),+ },+ },+ ) {+ Entry::Occupied { entry } => entry,+ Entry::Free { .. } => unreachable!(),+ }+ } else {+ let entry = match core::mem::replace(+ &mut self.entries[ix],+ Entry::Free {+ next: self.free_head,+ },+ ) {+ Entry::Occupied { entry } => entry,+ Entry::Free { .. } => unreachable!(),+ };++ self.free_head = Some(ix);++ entry+ }}/// Push a new entry into the table, returning its handle. This will prefer to use free entries@@ -201,11 +232,6 @@ impl ResourceTable {/// locking overhead and design issues, such as child existence extending/// lifetime of parent referent even after parent resource is destroyed,/// possibility for deadlocks.- ///- /// Parent-child relationships may not be modified once created. There- /// is no way to observe these relationships through the [`ResourceTable`]- /// methods except for erroring on deletion, or the [`std::fmt::Debug`]- /// impl.pub fn push_child<T, U>(&mut self,entry: T,@@ -222,6 +248,32 @@ impl ResourceTable {Ok(Resource::new_own(child))}+ /// Add an already-resident child to a resource.+ pub fn add_child<T: 'static, U: 'static>(+ &mut self,+ child: Resource<T>,+ parent: Resource<U>,+ ) -> Result<(), ResourceTableError> {+ let entry = self.occupied_mut(child.rep())?;+ assert!(entry.parent.is_none());+ entry.parent = Some(parent.rep());+ self.occupied_mut(parent.rep())?.add_child(child.rep());+ Ok(())+ }++ /// Remove a child to from a resource (but leave it in the table).+ pub fn remove_child<T: 'static, U: 'static>(+ &mut self,+ child: Resource<T>,+ parent: Resource<U>,+ ) -> Result<(), ResourceTableError> {+ let entry = self.occupied_mut(child.rep())?;+ assert_eq!(entry.parent, Some(parent.rep()));+ entry.parent = None;+ self.occupied_mut(parent.rep())?.remove_child(child.rep());+ Ok(())+ }+/// Get an immutable reference to a resource of a given type at a given/// index.///@@ -254,24 +306,35 @@ impl ResourceTable {Ok(&mut *r.entry)}- /// Same as `delete`, but typed+ /// Remove the specified entry from the table.pub fn delete<T>(&mut self, resource: Resource<T>) -> Result<T, ResourceTableError>+ where+ T: Any,+ {+ self.delete_maybe_debug(resource, cfg!(debug_assertions))+ }++ fn delete_maybe_debug<T>(+ &mut self,+ resource: Resource<T>,+ debug: bool,+ ) -> Result<T, ResourceTableError>whereT: Any,{debug_assert!(resource.owned());- let entry = self.delete_entry(resource.rep())?;+ let entry = self.delete_entry(resource.rep(), debug)?;match entry.entry.downcast() {Ok(t) => Ok(*t),Err(_e) => Err(ResourceTableError::WrongType),}}- fn delete_entry(&mut self, key: u32) -> Result<TableEntry, ResourceTableError> {+ fn delete_entry(&mut self, key: u32, debug: bool) -> Result<TableEntry, ResourceTableError> {if !self.occupied(key)?.children.is_empty() {return Err(ResourceTableError::HasChildren);}- let e = self.free_entry(key as usize);+ let e = self.free_entry(key as usize, debug);if let Some(parent) = e.parent {// Remove deleted resource from parent's child list.// Parent must still be present because it can't be deleted while still having@@ -314,6 +377,14 @@ impl ResourceTable {child.entry.as_ref()}))}++ /// Iterate over all the entries in this table.+ pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut (dyn Any + Send)> {+ self.entries.iter_mut().filter_map(|entry| match entry {+ Entry::Occupied { entry } => Some(&mut *entry.entry),+ Entry::Free { .. } => None,+ })+ }}impl Default for ResourceTable {@@ -322,6 +393,26 @@ impl Default for ResourceTable {}}+impl fmt::Debug for ResourceTable {+ fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {+ write!(f, "[")?;+ let mut wrote = false;+ for (index, entry) in self.entries.iter().enumerate() {+ if let Entry::Occupied { entry } = entry {+ if entry.entry.downcast_ref::<Tombstone>().is_none() {+ if wrote {+ write!(f, ", ")?;+ } else {+ wrote = true;+ }+ write!(f, "{index}")?;+ }+ }+ }+ write!(f, "]")+ }+}+#[test]pub fn test_free_list() {let mut table = ResourceTable::new();@@ -333,13 +424,13 @@ pub fn test_free_list() {assert_eq!(y.rep(), 1);// Deleting x should put it on the free list, so the next entry should have the same rep.- table.delete(x).unwrap();+ table.delete_maybe_debug(x, false).unwrap();let x = table.push(()).unwrap();assert_eq!(x.rep(), 0);// Deleting x and then y should yield indices 1 and then 0 for new entries.- table.delete(x).unwrap();- table.delete(y).unwrap();+ table.delete_maybe_debug(x, false).unwrap();+ table.delete_maybe_debug(y, false).unwrap();let y = table.push(()).unwrap();assert_eq!(y.rep(), 1);