A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Refactor borrow state tracking for async tasks (#12550)
Alex Crichton3764e7573e36
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex 7a960de15f..9b0d7a3798 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -47,9 +47,6 @@ impl From<GetLibcallFn> for HostCallee {/// How to interpret the results of a host function.enum HostResult {- /// The host function has no results.- None,-/// The host function returns the sentinel specified which is interpreted/// and translated to the real return value.Sentinel(TrapSentinel),@@ -622,22 +619,6 @@ impl<'a> TrampolineCompiler<'a> {|_, _| {},);}- Trampoline::ResourceEnterCall => {- self.translate_libcall(- host::resource_enter_call,- HostResult::None,- WasmArgs::InRegisters,- |_, _| {},- );- }- Trampoline::ResourceExitCall => {- self.translate_libcall(- host::resource_exit_call,- TrapSentinel::Falsy,- WasmArgs::InRegisters,- |_, _| {},- );- }Trampoline::PrepareCall { memory } => {self.translate_libcall(host::prepare_call,@@ -1043,10 +1024,6 @@ impl<'a> TrampolineCompiler<'a> {self.abi_store_results(&[]);}HostResult::Sentinel(_) => todo!("support additional return types if/when necessary"),- HostResult::None => {- assert!(result.is_none());- self.abi_store_results(&[]);- }HostResult::MultiValue { ptr, len } => {let ptr = ptr.or(val_raw_ptr).unwrap();@@ -1472,8 +1449,6 @@ impl<'a> TrampolineCompiler<'a> {// directly from guest wasm, so no check is needed.Trampoline::ResourceTransferOwn| Trampoline::ResourceTransferBorrow- | Trampoline::ResourceEnterCall- | Trampoline::ResourceExitCall| Trampoline::PrepareCall { .. }| Trampoline::SyncStartCall { .. }| Trampoline::AsyncStartCall { .. }diff --git a/crates/environ/src/component.rs b/crates/environ/src/component.rsindex edc02acac5..2e3a826585 100644--- a/crates/environ/src/component.rs+++ b/crates/environ/src/component.rs@@ -96,12 +96,8 @@ macro_rules! foreach_builtin_component_function {resource_transfer_own(vmctx: vmctx, src_idx: u32, src_table: u32, dst_table: u32) -> u64;resource_transfer_borrow(vmctx: vmctx, src_idx: u32, src_table: u32, dst_table: u32) -> u64;- resource_enter_call(vmctx: vmctx);- resource_exit_call(vmctx: vmctx) -> bool;- #[cfg(feature = "component-model-async")]enter_sync_call(vmctx: vmctx, caller_instance: u32, callee_async: u32, callee_instance: u32) -> bool;- #[cfg(feature = "component-model-async")]exit_sync_call(vmctx: vmctx) -> bool;#[cfg(feature = "component-model-async")]diff --git a/crates/environ/src/component/dfg.rs b/crates/environ/src/component/dfg.rsindex f1c4cef3f4..4a76153a85 100644--- a/crates/environ/src/component/dfg.rs+++ b/crates/environ/src/component/dfg.rs@@ -460,8 +460,6 @@ pub enum Trampoline {},ResourceTransferOwn,ResourceTransferBorrow,- ResourceEnterCall,- ResourceExitCall,PrepareCall {memory: Option<MemoryId>,},@@ -1146,8 +1144,6 @@ impl LinearizeDfg<'_> {},Trampoline::ResourceTransferOwn => info::Trampoline::ResourceTransferOwn,Trampoline::ResourceTransferBorrow => info::Trampoline::ResourceTransferBorrow,- Trampoline::ResourceEnterCall => info::Trampoline::ResourceEnterCall,- Trampoline::ResourceExitCall => info::Trampoline::ResourceExitCall,Trampoline::PrepareCall { memory } => info::Trampoline::PrepareCall {memory: memory.map(|v| self.runtime_memory(v)),},diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rsindex 2aff54ae68..9f49acd1f5 100644--- a/crates/environ/src/component/info.rs+++ b/crates/environ/src/component/info.rs@@ -1044,16 +1044,6 @@ pub enum Trampoline {/// Same as `ResourceTransferOwn` but for borrows.ResourceTransferBorrow,- /// An intrinsic used by FACT-generated modules which indicates that a call- /// is being entered and resource-related metadata needs to be configured.- ///- /// Note that this is currently only invoked when borrowed resources are- /// detected, otherwise this is "optimized out".- ResourceEnterCall,-- /// Same as `ResourceEnterCall` except for when exiting a call.- ResourceExitCall,-/// An intrinsic used by FACT-generated modules to prepare a call involving/// an async-lowered import and/or an async-lifted export.PrepareCall {@@ -1239,8 +1229,6 @@ impl Trampoline {ErrorContextDrop { .. } => format!("error-context-drop"),ResourceTransferOwn => format!("component-resource-transfer-own"),ResourceTransferBorrow => format!("component-resource-transfer-borrow"),- ResourceEnterCall => format!("component-resource-enter-call"),- ResourceExitCall => format!("component-resource-exit-call"),PrepareCall { .. } => format!("component-prepare-call"),SyncStartCall { .. } => format!("component-sync-start-call"),AsyncStartCall { .. } => format!("component-async-start-call"),diff --git a/crates/environ/src/component/translate/adapt.rs b/crates/environ/src/component/translate/adapt.rsindex 618934ec70..d21476d974 100644--- a/crates/environ/src/component/translate/adapt.rs+++ b/crates/environ/src/component/translate/adapt.rs@@ -325,8 +325,6 @@ fn fact_import_to_core_def(fact::Import::ResourceTransferBorrow => {simple_intrinsic(dfg::Trampoline::ResourceTransferBorrow)}- fact::Import::ResourceEnterCall => simple_intrinsic(dfg::Trampoline::ResourceEnterCall),- fact::Import::ResourceExitCall => simple_intrinsic(dfg::Trampoline::ResourceExitCall),fact::Import::PrepareCall { memory } => simple_intrinsic(dfg::Trampoline::PrepareCall {memory: memory.as_ref().map(|v| dfg.memories.push(unwrap_memory(v))),}),diff --git a/crates/environ/src/fact.rs b/crates/environ/src/fact.rsindex f3a131ba15..e3be059bbf 100644--- a/crates/environ/src/fact.rs+++ b/crates/environ/src/fact.rs@@ -76,8 +76,6 @@ pub struct Module<'a> {/// Cached versions of imported trampolines for working with resources.imported_resource_transfer_own: Option<FuncIndex>,imported_resource_transfer_borrow: Option<FuncIndex>,- imported_resource_enter_call: Option<FuncIndex>,- imported_resource_exit_call: Option<FuncIndex>,// Cached versions of imported trampolines for working with the async ABI.imported_async_start_calls: HashMap<(Option<FuncIndex>, Option<FuncIndex>), FuncIndex>,@@ -261,8 +259,6 @@ impl<'a> Module<'a> {helper_worklist: Vec::new(),imported_resource_transfer_own: None,imported_resource_transfer_borrow: None,- imported_resource_enter_call: None,- imported_resource_exit_call: None,imported_async_start_calls: HashMap::new(),imported_future_transfer: None,imported_stream_transfer: None,@@ -720,28 +716,6 @@ impl<'a> Module<'a> {)}- fn import_resource_enter_call(&mut self) -> FuncIndex {- self.import_simple(- "resource",- "enter-call",- &[],- &[],- Import::ResourceEnterCall,- |me| &mut me.imported_resource_enter_call,- )- }-- fn import_resource_exit_call(&mut self) -> FuncIndex {- self.import_simple(- "resource",- "exit-call",- &[],- &[],- Import::ResourceExitCall,- |me| &mut me.imported_resource_exit_call,- )- }-fn import_enter_sync_call(&mut self) -> FuncIndex {self.import_simple("async",@@ -881,11 +855,6 @@ pub enum Import {ResourceTransferOwn,/// Transfers a borrowed resource from one table to another.ResourceTransferBorrow,- /// Sets up entry metadata for a borrow resources when a call starts.- ResourceEnterCall,- /// Tears down a previous entry and handles checking borrow-related- /// metadata.- ResourceExitCall,/// An intrinsic used by FACT-generated modules to begin a call involving/// an async-lowered import and/or an async-lifted export.PrepareCall {diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rsindex 79cf0a49eb..e729b1a032 100644--- a/crates/environ/src/fact/trampoline.rs+++ b/crates/environ/src/fact/trampoline.rs@@ -797,15 +797,14 @@ impl<'a, 'b> Compiler<'a, 'b> {self.instruction(Call(enter_sync_call.as_u32()));old_task_may_block+ } else if self.emit_resource_call {+ let enter_sync_call = self.module.import_enter_sync_call();+ self.instruction(Call(enter_sync_call.as_u32()));+ None} else {None};- if self.emit_resource_call {- let enter = self.module.import_resource_enter_call();- self.instruction(Call(enter.as_u32()));- }-// Perform the translation of arguments. Note that `FLAG_MAY_LEAVE` is// cleared around this invocation for the callee as per the// `canon_lift` definition in the spec. Additionally note that the@@ -866,18 +865,15 @@ impl<'a, 'b> Compiler<'a, 'b> {self.free_temp_local(tmp);}- if self.emit_resource_call {- let exit = self.module.import_resource_exit_call();- self.instruction(Call(exit.as_u32()));- }-- if self.module.tunables.concurrency_support {+ if self.emit_resource_call || self.module.tunables.concurrency_support {// Pop the task we pushed earlier off of the current task stack.//// FIXME: Apply the optimizations described in #12311.let exit_sync_call = self.module.import_exit_sync_call();self.instruction(Call(exit_sync_call.as_u32()));+ }+ if self.module.tunables.concurrency_support {// Restore old `may_block_field`if let Some(old_task_may_block) = old_task_may_block {let task_may_block = self.module.import_task_may_block();diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex f6b1b7d955..cd72725aa3 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -57,8 +57,7 @@ pub fn find_tests(root: &Path) -> Result<Vec<WastTest>> {// panics, loops, etc).{let skip_list = &[- // FIXME(#12510)- "drop-cross-task-borrow.wast",+ // .. empty currently ..];tests.retain(|test| {test.pathdiff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex 60adfd06da..4cb543014d 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -61,8 +61,7 @@ use crate::vm::component::{CallContext, ComponentInstance, InstanceState};use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMMemoryDefinition, VMStore};use crate::{AsContext, AsContextMut, FuncType, Result, StoreContext, StoreContextMut, ValRaw, ValType,- bail,- error::{Context as _, format_err},+ bail, error::format_err,};use error_contexts::GlobalErrorContextRefCount;use futures::channel::oneshot;@@ -718,26 +717,9 @@ pub(crate) enum WaitResult {pub(crate) fn poll_and_block<R: Send + Sync + 'static>(store: &mut dyn VMStore,future: impl Future<Output = Result<R>> + Send + 'static,- caller_instance: RuntimeInstance,) -> Result<R> {let state = store.concurrent_state_mut();-- let caller = state.guest_thread.unwrap();-- // Save any existing result stashed in `GuestTask::result` so we can replace- // it with the new result.- let old_result = state- .get_mut(caller.task)- .with_context(|| format!("bad handle: {caller:?}"))?- .result- .take();-- // Add a temporary host task into the table so we can track its progress.- // Note that we'll never allocate a waitable handle for the guest since- // we're being called synchronously.- let task = state.push(HostTask::new(caller_instance, None))?;-- log::trace!("new host task child of {caller:?}: {task:?}");+ let task = state.unwrap_current_host_thread();// Wrap the future in a closure which will take care of stashing the result// in `GuestTask::result` and resuming this fiber when the host task@@ -746,7 +728,7 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(let result = future.await?;tls::get(move |store| {let state = store.concurrent_state_mut();- state.get_mut(caller.task)?.result = Some(Box::new(result) as _);+ state.get_mut(task)?.result = Some(Box::new(result) as _);Waitable::Host(task).set_event(state,@@ -769,20 +751,18 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(});match poll {- Poll::Ready(result) => {- // It completed immediately; check the result and delete the task.- result?;- log::trace!("delete host task {task:?} (already ready)");- store.concurrent_state_mut().delete(task)?;- }+ // It completed immediately; check the result and delete the task.+ Poll::Ready(result) => result?,++ // It did not complete immediately; add it to+ // `ConcurrentState::futures` so it will be polled via the event loop;+ // then use `GuestTask::sync_call_set` to wait for the task to+ // complete, suspending the current fiber until it does so.Poll::Pending => {- // It did not complete immediately; add it to- // `ConcurrentState::futures` so it will be polled via the event- // loop; then use `GuestTask::sync_call_set` to wait for the task to- // complete, suspending the current fiber until it does so.let state = store.concurrent_state_mut();state.push_future(future);+ let caller = state.get_mut(task)?.caller;let set = state.get_mut(caller.task)?.sync_call_set;Waitable::Host(task).join(state, Some(set))?;@@ -791,17 +771,23 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(thread: caller,skip_may_block_check: false,})?;++ // Remove the `task` from the `sync_call_set` to ensure that when+ // this function returns and the task is deleted that there are no+ // more lingering references to this host task.+ Waitable::Host(task).join(store.concurrent_state_mut(), None)?;}}// Retrieve and return the result.- Ok(*mem::replace(- &mut store.concurrent_state_mut().get_mut(caller.task)?.result,- old_result,- )- .unwrap()- .downcast()- .unwrap())+ Ok(*store+ .concurrent_state_mut()+ .get_mut(task)?+ .result+ .take()+ .unwrap()+ .downcast()+ .unwrap())}/// Execute the specified guest call.@@ -823,14 +809,12 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {call.thread,);- let old_thread = store.set_thread(Some(call.thread));+ let old_thread = store.set_thread(call.thread);log::trace!("GuestCallKind::DeliverEvent: replaced {old_thread:?} with {:?} as current thread",call.thread);- store.maybe_push_call_context(call.thread.task)?;-store.enter_instance(runtime_instance);let callback = store@@ -849,8 +833,6 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {store.exit_instance(runtime_instance)?;- store.maybe_pop_call_context(call.thread.task)?;-store.set_thread(old_thread);next = instance.handle_callback_code(@@ -894,6 +876,11 @@ impl<T> Store<T> {self.as_context_mut().assert_concurrent_state_empty();}+ #[doc(hidden)]+ pub fn concurrent_state_table_size(&mut self) -> usize {+ self.as_context_mut().concurrent_state_table_size()+ }+/// Convenience wrapper for [`StoreContextMut::spawn`].pub fn spawn(&mut self, task: impl AccessorTask<T, HasSelf<T>>) -> JoinHandlewhere@@ -912,6 +899,8 @@ impl<T> StoreContextMut<'_, T> {/// after each test concludes. This should help us catch leaks, e.g. guest/// tasks which haven't been deleted despite having completed and having/// been dropped by their supertasks.+ ///+ /// Only intended for use in Wasmtime's own testing.#[doc(hidden)]pub fn assert_concurrent_state_empty(self) {let store = self.0;@@ -927,11 +916,25 @@ impl<T> StoreContextMut<'_, T> {);assert!(state.high_priority.is_empty());assert!(state.low_priority.is_empty());- assert!(state.guest_thread.is_none());+ assert!(state.current_thread.is_none());assert!(state.futures.get_mut().as_ref().unwrap().is_empty());assert!(state.global_error_context_ref_counts.is_empty());}+ /// Helper function to perform tests over the size of the concurrent state+ /// table which can be useful for detecting leaks.+ ///+ /// Only intended for use in Wasmtime's own testing.+ #[doc(hidden)]+ pub fn concurrent_state_table_size(&mut self) -> usize {+ self.0+ .concurrent_state_mut()+ .table+ .get_mut()+ .iter_mut()+ .count()+ }+/// Spawn a background task to run as part of this instance's event loop.////// The task will receive an `&Accessor<U>` and run concurrently with@@ -1393,17 +1396,20 @@ impl StoreOpaque {/// This task will only be used for the purpose of handling calls to/// intrinsic functions; both parameter lowering and result lifting are/// assumed to be taken care of elsewhere.- pub(crate) fn enter_sync_call(+ pub(crate) fn enter_guest_sync_call(&mut self,guest_caller: Option<RuntimeInstance>,callee_async: bool,callee: RuntimeInstance,) -> Result<()> {log::trace!("enter sync call {callee:?}");+ if !self.concurrency_support() {+ return Ok(self.enter_call_not_concurrent());+ }let state = self.concurrent_state_mut();- let thread = state.guest_thread;- let instance = if let Some(thread) = thread {+ let thread = state.current_thread;+ let instance = if let Some(thread) = thread.guest() {Some(state.get_mut(thread.task)?.instance)} else {None@@ -1420,14 +1426,14 @@ impl StoreOpaque {if let Some(caller) = guest_caller {assert_eq!(caller, instance.unwrap());Caller::Guest {- thread: thread.unwrap(),+ thread: *thread.guest().unwrap(),}} else {Caller::Host {tx: None,exit_tx: Arc::new(oneshot::channel().0),host_future_present: false,- caller: state.guest_thread,+ caller: thread,}},None,@@ -1447,21 +1453,24 @@ impl StoreOpaque {let state = self.concurrent_state_mut();state.get_mut(guest_task)?.threads.insert(guest_thread);if guest_caller.is_some() {- let thread = state.guest_thread.unwrap();+ let thread = thread.guest().unwrap();state.get_mut(thread.task)?.subtasks.insert(guest_task);}- self.set_thread(Some(QualifiedThreadId {+ self.set_thread(QualifiedThreadId {task: guest_task,thread: guest_thread,- }));+ });Ok(())}/// Pop a `GuestTask` previously pushed using `enter_sync_call`.- pub(crate) fn exit_sync_call(&mut self, guest_caller: bool) -> Result<()> {- let thread = self.set_thread(None).unwrap();+ pub(crate) fn exit_guest_sync_call(&mut self, guest_caller: bool) -> Result<()> {+ if !self.concurrency_support() {+ return Ok(self.exit_call_not_concurrent());+ }+ let thread = *self.set_thread(CurrentThread::None).guest().unwrap();let instance = self.concurrent_state_mut().get_mut(thread.task)?.instance;log::trace!("exit sync call {instance:?}");Instance::from_wasmtime(self, instance.instance).cleanup_thread(@@ -1475,7 +1484,7 @@ impl StoreOpaque {let caller = match &task.caller {&Caller::Guest { thread } => {assert!(guest_caller);- Some(thread)+ thread.into()}&Caller::Host { caller, .. } => {assert!(!guest_caller);@@ -1493,6 +1502,36 @@ impl StoreOpaque {Ok(())}+ /// Similar to `enter_guest_sync_call` except for when the guest makes a+ /// transition to the host.+ ///+ /// FIXME: this is called for all guest->host transitions and performs some+ /// relatively expensive table manipulations. This would ideally be+ /// optimized to avoid the full allocation of a `HostTask` in at least some+ /// situations.+ pub fn enter_host_call(&mut self) -> Result<()> {+ let state = self.concurrent_state_mut();+ let caller = state.unwrap_current_guest_thread();+ let task = state.push(HostTask::new(caller))?;+ log::trace!("new host task {task:?}");+ self.set_thread(task);+ Ok(())+ }++ /// Dual of `enter_host_call` and signifies that the host has finished and+ /// will be cleaned up.+ ///+ /// Note that this isn't invoked when the host is invoked asynchronously and+ /// the host isn't complete yet. In that situation the host task persists+ /// and will be cleaned up separately.+ pub fn exit_host_call(&mut self) -> Result<()> {+ let task = self.concurrent_state_mut().unwrap_current_host_thread();+ log::trace!("delete host task {task:?}");+ let task = self.concurrent_state_mut().delete(task)?;+ self.set_thread(task.caller);+ Ok(())+ }+/// Determine whether the specified instance may be entered from the host.////// We return `true` here only if all of the following hold:@@ -1508,11 +1547,11 @@ impl StoreOpaque {return true;}let state = self.concurrent_state_mut();- let mut cur = state.guest_thread;+ let mut cur = state.current_thread;loop {match cur {- None => break true,- Some(thread) => {+ CurrentThread::None => break true,+ CurrentThread::Guest(thread) => {let task = state.get_mut(thread.task).unwrap();// Note that we only compare top-level instance IDs here.@@ -1529,6 +1568,9 @@ impl StoreOpaque {Caller::Guest { thread } => thread.into(),};}+ CurrentThread::Host(id) => {+ cur = state.get_mut(id).unwrap().caller.into();+ }}}}@@ -1540,24 +1582,22 @@ impl StoreOpaque {.instance_state(instance.index)}- fn set_thread(&mut self, thread: Option<QualifiedThreadId>) -> Option<QualifiedThreadId> {+ fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> CurrentThread {// Each time we switch threads, we conservatively set `task_may_block`// to `false` for the component instance we're switching away from (if// any), meaning it will be `false` for any new thread created for that// instance unless explicitly set otherwise.let state = self.concurrent_state_mut();- let old_thread = state.guest_thread.take();- if let Some(old_thread) = old_thread {+ let old_thread = mem::replace(&mut state.current_thread, thread.into());+ if let Some(old_thread) = old_thread.guest() {let instance = state.get_mut(old_thread.task).unwrap().instance.instance;self.component_instance_mut(instance).set_task_may_block(false)}- self.concurrent_state_mut().guest_thread = thread;-// If we're switching to a new thread, set its component instance's// `task_may_block` according to where it left off.- if thread.is_some() {+ if self.concurrent_state_mut().current_thread.guest().is_some() {self.set_task_may_block();}@@ -1568,7 +1608,7 @@ impl StoreOpaque {/// prior to entering Wasm code.fn set_task_may_block(&mut self) {let state = self.concurrent_state_mut();- let guest_thread = state.guest_thread.unwrap();+ let guest_thread = state.unwrap_current_guest_thread();let instance = state.get_mut(guest_thread.task).unwrap().instance.instance;let may_block = self.concurrent_state_mut().may_block(guest_thread.task);self.component_instance_mut(instance)@@ -1580,7 +1620,7 @@ impl StoreOpaque {return Ok(());}let state = self.concurrent_state_mut();- let task = state.guest_thread.unwrap().task;+ let task = state.unwrap_current_guest_thread().task;let instance = state.get_mut(task).unwrap().instance.instance;let task_may_block = self.component_instance(instance).get_task_may_block();@@ -1658,7 +1698,7 @@ impl StoreOpaque {/// Resume the specified fiber, giving it exclusive access to the specified/// store.async fn resume_fiber(&mut self, fiber: StoreFiber<'static>) -> Result<()> {- let old_thread = self.concurrent_state_mut().guest_thread;+ let old_thread = self.concurrent_state_mut().current_thread;log::trace!("resume_fiber: save current thread {old_thread:?}");let fiber = fiber::resolve_or_release(self, fiber).await?;@@ -1667,7 +1707,7 @@ impl StoreOpaque {let state = self.concurrent_state_mut();- if let Some(ref ot) = old_thread {+ if let Some(ot) = old_thread.guest() {state.get_mut(ot.thread)?.state = GuestThreadState::Running;}log::trace!("resume_fiber: restore current thread {old_thread:?}");@@ -1723,11 +1763,10 @@ impl StoreOpaque {SuspendReason::NeedWork => None,};- let old_guest_thread = if let Some(task) = task {- self.maybe_pop_call_context(task)?;- self.concurrent_state_mut().guest_thread+ let old_guest_thread = if task.is_some() {+ self.concurrent_state_mut().current_thread} else {- None+ CurrentThread::None};// We should not have reached here unless either there's no current@@ -1749,6 +1788,7 @@ impl StoreOpaque {..}) || old_guest_thread+ .guest().map(|thread| self.concurrent_state_mut().may_block(thread.task)).unwrap_or(true));@@ -1759,49 +1799,16 @@ impl StoreOpaque {self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;- if let Some(task) = task {+ if task.is_some() {self.set_thread(old_guest_thread);- self.maybe_push_call_context(task)?;}Ok(())}- /// Push the call context for managing resource borrows for the specified- /// guest task if it has not yet either returned a result or cancelled- /// itself.- fn maybe_push_call_context(&mut self, guest_task: TableId<GuestTask>) -> Result<()> {- let task = self.concurrent_state_mut().get_mut(guest_task)?;-- if !task.returned_or_cancelled() {- log::trace!("push call context for {guest_task:?}");- let call_context = task.call_context.take().unwrap();- self.component_call_contexts_mut().push(call_context);- }- Ok(())- }-- /// Pop the call context for managing resource borrows for the specified- /// guest task if it has not yet either returned a result or cancelled- /// itself.- fn maybe_pop_call_context(&mut self, guest_task: TableId<GuestTask>) -> Result<()> {- if !self- .concurrent_state_mut()- .get_mut(guest_task)?- .returned_or_cancelled()- {- log::trace!("pop call context for {guest_task:?}");- let call_context = Some(self.component_call_contexts_mut().pop().unwrap());- self.concurrent_state_mut()- .get_mut(guest_task)?- .call_context = call_context;- }- Ok(())- }-fn wait_for_event(&mut self, waitable: Waitable) -> Result<()> {let state = self.concurrent_state_mut();- let caller = state.guest_thread.unwrap();+ let caller = state.unwrap_current_guest_thread();let old_set = waitable.common(state)?.set;let set = state.get_mut(caller.task)?.sync_call_set;waitable.join(state, Some(set))?;@@ -2121,13 +2128,11 @@ impl Instance {store,callee_instance.index,)?;- let old_thread = store.set_thread(Some(guest_thread));+ let old_thread = store.set_thread(guest_thread);log::trace!("stackless call: replaced {old_thread:?} with {guest_thread:?} as current thread");- store.maybe_push_call_context(guest_thread.task)?;-store.enter_instance(callee_instance);// SAFETY: See the documentation for `make_call` to review the@@ -2140,11 +2145,10 @@ impl Instance {store.exit_instance(callee_instance)?;- store.maybe_pop_call_context(guest_thread.task)?;-store.set_thread(old_thread);let state = store.concurrent_state_mut();old_thread+ .guest().map(|t| state.get_mut(t.thread).unwrap().state = GuestThreadState::Running);log::trace!("stackless call: restored {old_thread:?} as current thread");@@ -2163,14 +2167,12 @@ impl Instance {store,callee_instance.index,)?;- let old_thread = store.set_thread(Some(guest_thread));+ let old_thread = store.set_thread(guest_thread);log::trace!("sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",);let flags = self.id().get(store).instance_flags(callee_instance.index);- store.maybe_push_call_context(guest_thread.task)?;-// Unless this is a callback-less (i.e. stackful)// async-lifted export, we need to record that the instance// cannot be entered until the call returns.@@ -2243,8 +2245,6 @@ impl Instance {store.set_thread(old_thread);- store.maybe_pop_call_context(guest_thread.task)?;-let state = store.concurrent_state_mut();let task = state.get_mut(guest_thread.task)?;@@ -2341,7 +2341,7 @@ impl Instance {let return_ = SendSyncPtr::new(NonNull::new(return_).unwrap());let token = StoreToken::new(store.as_context_mut());let state = store.0.concurrent_state_mut();- let old_thread = state.guest_thread.unwrap();+ let old_thread = state.unwrap_current_guest_thread();assert_eq!(state.get_mut(old_thread.task)?.instance,@@ -2396,7 +2396,7 @@ impl Instance {}dst.copy_from_slice(&src[..dst.len()]);let state = store.0.concurrent_state_mut();- Waitable::Guest(state.guest_thread.unwrap().task).set_event(+ Waitable::Guest(state.unwrap_current_guest_thread().task).set_event(state,Some(Event::Subtask {status: Status::Started,@@ -2427,7 +2427,7 @@ impl Instance {)?;}let state = store.0.concurrent_state_mut();- let thread = state.guest_thread.unwrap();+ let thread = state.unwrap_current_guest_thread();if sync_caller {state.get_mut(thread.task)?.sync_result = SyncResult::Produced(if let ResultInfo::Stack { result_count } = &result_info {@@ -2470,10 +2470,10 @@ impl Instance {// Make the new thread the current one so that `Self::start_call` knows// which one to start.- store.0.set_thread(Some(QualifiedThreadId {+ store.0.set_thread(QualifiedThreadId {task: guest_task,thread: guest_thread,- }));+ });log::trace!("pushed {guest_task:?}:{guest_thread:?} as current thread; old thread was {old_thread:?}");@@ -2538,7 +2538,7 @@ impl Instance {let token = StoreToken::new(store.as_context_mut());let async_caller = storage.is_none();let state = store.0.concurrent_state_mut();- let guest_thread = state.guest_thread.unwrap();+ let guest_thread = state.unwrap_current_guest_thread();let callee_async = state.get_mut(guest_thread.task)?.async_function;let callee = SendSyncPtr::new(NonNull::new(callee).unwrap());let param_count = usize::try_from(param_count).unwrap();@@ -2657,7 +2657,7 @@ impl Instance {guest_waitable.join(store.0.concurrent_state_mut(), old_set)?;// Reset the current thread to point to the caller as it resumes control.- store.0.set_thread(Some(caller));+ store.0.set_thread(caller);store.0.concurrent_state_mut().get_mut(caller.thread)?.state = GuestThreadState::Running;log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");@@ -2696,64 +2696,20 @@ impl Instance {self,mut store: StoreContextMut<'_, T>,future: impl Future<Output = Result<R>> + Send + 'static,- caller_instance: RuntimeComponentInstanceIndex,lower: impl FnOnce(StoreContextMut<T>, R) -> Result<()> + Send + 'static,) -> Result<Option<u32>> {let token = StoreToken::new(store.as_context_mut());let state = store.0.concurrent_state_mut();- let caller = state.guest_thread.unwrap();+ let task = state.unwrap_current_host_thread();// Create an abortable future which hooks calls to poll and manages call// context state for the future.- let (join_handle, future) = JoinHandle::run(async move {- let mut future = pin!(future);- let mut call_context = None;- future::poll_fn(move |cx| {- // Push the call context for managing any resource borrows- // for the task.- tls::get(|store| {- if let Some(call_context) = call_context.take() {- token- .as_context_mut(store)- .0- .component_call_contexts_mut()- .push(call_context);- }- });-- let result = future.as_mut().poll(cx);-- if result.is_pending() {- // Pop the call context for managing any resource- // borrows for the task.- tls::get(|store| {- call_context = Some(- token- .as_context_mut(store)- .0- .component_call_contexts_mut()- .pop()- .unwrap(),- );- });- }- result- })- .await- });-- // We create a new host task even though it might complete immediately- // (in which case we won't need to pass a waitable back to the guest).- // If it does complete immediately, we'll remove it before we return.- let task = state.push(HostTask::new(- RuntimeInstance {- instance: self.id().instance(),- index: caller_instance,- },- Some(join_handle),- ))?;-- log::trace!("new host task child of {caller:?}: {task:?}");+ let (join_handle, future) = JoinHandle::run(future);+ {+ let task = state.get_mut(task)?;+ assert!(task.join_handle.is_none());+ task.join_handle = Some(join_handle);+ }let mut future = Box::pin(future);@@ -2768,17 +2724,20 @@ impl Instance {});match poll {+ // It finished immediately; lower the result and delete the task.Poll::Ready(Some(result)) => {- // It finished immediately; lower the result and delete the- // task.lower(store.as_context_mut(), result?)?;- log::trace!("delete host task {task:?} (already ready)");- store.0.concurrent_state_mut().delete(task)?;return Ok(None);}++ // Shouldn't be possible since the future isn't cancelled via the+ // `join_handle`.Poll::Ready(None) => unreachable!(),++ // Future isn't ready yet, so fall through.Poll::Pending => {}}+// It hasn't finished yet; add the future to// `ConcurrentState::futures` so it will be polled by the event// loop and allocate a waitable handle to return to the guest.@@ -2793,22 +2752,34 @@ impl Instance {None => return Ok(()),};let on_complete = move |store: &mut dyn VMStore| {- lower(token.as_context_mut(store), result)?;- let state = store.concurrent_state_mut();+ // Restore the `current_thread` to be the host so `lower` knows+ // how to manipulate borrows and knows which scope of borrows+ // to check.+ let mut store = token.as_context_mut(store);+ let state = store.0.concurrent_state_mut();+ assert!(state.current_thread.is_none());+ store.0.set_thread(task);++ lower(store.as_context_mut(), result)?;+ let state = store.0.concurrent_state_mut();state.get_mut(task)?.join_handle.take();Waitable::Host(task).set_event(state,Some(Event::Subtask {status: Status::Returned,}),- )+ )?;++ // Go back to "no current thread" at the end.+ store.0.set_thread(CurrentThread::None);+ Ok(())};++ // Here we schedule a task to run on a worker fiber to do the+ // lowering since it may involve a call to the guest's realloc+ // function. This is necessary because calling the guest while+ // there are host embedder frames on the stack is unsound.tls::get(move |store| {- // Here we schedule a task to run on a worker fiber to do- // the lowering since it may involve a call to the guest's- // realloc function. This is necessary because calling the- // guest while there are host embedder frames on the stack- // is unsound.store.concurrent_state_mut().push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(@@ -2818,17 +2789,24 @@ impl Instance {})});- store.0.concurrent_state_mut().push_future(future);+ // Make this task visible to the guest and then record what it+ // was made visible as.+ let state = store.0.concurrent_state_mut();+ state.push_future(future);+ let caller = state.get_mut(task)?.caller;+ let instance = state.get_mut(caller.task)?.instance;let handle = store.0- .instance_state(RuntimeInstance {- instance: self.id().instance(),- index: caller_instance,- })+ .instance_state(instance).handle_table().subtask_insert_host(task.rep())?;store.0.concurrent_state_mut().get_mut(task)?.common.handle = Some(handle);- log::trace!("assign {task:?} handle {handle} for {caller:?} instance {caller_instance:?}");+ log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");++ // Restore the currently running thread to this host task's+ // caller. Note that the host task isn't deallocated as it's+ // within the store and will get deallocated later.+ store.0.set_thread(caller);Ok(Some(handle))}@@ -2842,7 +2820,7 @@ impl Instance {storage: &[ValRaw],) -> Result<()> {let state = store.concurrent_state_mut();- let guest_thread = state.guest_thread.unwrap();+ let guest_thread = state.unwrap_current_guest_thread();let lift = state.get_mut(guest_thread.task)?.lift_result@@ -2888,7 +2866,7 @@ impl Instance {/// Implements the `task.cancel` intrinsic.pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {let state = store.concurrent_state_mut();- let guest_thread = state.guest_thread.unwrap();+ let guest_thread = state.unwrap_current_guest_thread();let task = state.get_mut(guest_thread.task)?;if !task.cancel_sent {bail!("`task.cancel` called by task which has not been cancelled")@@ -2921,7 +2899,9 @@ impl Instance {result: Box<dyn Any + Send + Sync>,status: Status,) -> Result<()> {- store.component_resource_tables(Some(self)).exit_call()?;+ store+ .component_resource_tables(Some(self))+ .validate_scope_exit()?;let state = store.concurrent_state_mut();let task = state.get_mut(guest_task)?;@@ -3031,23 +3011,23 @@ impl Instance {.subtask_remove(task_id)?;let concurrent_state = store.concurrent_state_mut();- let (waitable, expected_caller_instance, delete) = if is_host {+ let (waitable, expected_caller, delete) = if is_host {let id = TableId::<HostTask>::new(rep);let task = concurrent_state.get_mut(id)?;if task.join_handle.is_some() {bail!("cannot drop a subtask which has not yet resolved");}- (Waitable::Host(id), task.caller_instance, true)+ (Waitable::Host(id), task.caller, true)} else {let id = TableId::<GuestTask>::new(rep);let task = concurrent_state.get_mut(id)?;if task.lift_result.is_some() {bail!("cannot drop a subtask which has not yet resolved");}- if let &Caller::Guest { thread } = &task.caller {+ if let Caller::Guest { thread } = task.caller {(Waitable::Guest(id),- concurrent_state.get_mut(thread.task)?.instance,+ thread,concurrent_state.get_mut(id)?.exited,)} else {@@ -3069,11 +3049,8 @@ impl Instance {// this should never fail unless there's a bug in Wasmtime, but we check// here to be sure:assert_eq!(- expected_caller_instance,- RuntimeInstance {- instance: self.id().instance(),- index: caller_instance- }+ expected_caller,+ concurrent_state.unwrap_current_guest_thread(),);log::trace!("subtask_drop {waitable:?} (handle {task_id})");Ok(())@@ -3154,7 +3131,10 @@ impl Instance {/// Implements the `thread.index` intrinsic.pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {- let thread_id = store.concurrent_state_mut().guest_thread.unwrap().thread;+ let thread_id = store+ .concurrent_state_mut()+ .unwrap_current_guest_thread()+ .thread;// The unwrap is safe because `instance_rep` must be `Some` by this pointOk(store.concurrent_state_mut()@@ -3191,21 +3171,17 @@ impl Instance {let token = StoreToken::new(store.as_context_mut());let start_func = Box::new(move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {- let old_thread = store.set_thread(Some(guest_thread));+ let old_thread = store.set_thread(guest_thread);log::trace!("thread start: replaced {old_thread:?} with {guest_thread:?} as current thread");- store.maybe_push_call_context(guest_thread.task)?;-let mut store = token.as_context_mut(store);let mut params = [ValRaw::i32(context)];// Use call_unchecked rather than call or call_async, as we don't want to run the function// on a separate fiber if we're running in an async store.unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };- store.0.maybe_pop_call_context(guest_thread.task)?;-self.cleanup_thread(store.0, guest_thread, runtime_instance)?;log::trace!("explicit thread {guest_thread:?} completed");let state = store.0.concurrent_state_mut();@@ -3216,6 +3192,7 @@ impl Instance {store.0.set_thread(old_thread);let state = store.0.concurrent_state_mut();old_thread+ .guest().map(|t| state.get_mut(t.thread).unwrap().state = GuestThreadState::Running);if state.get_mut(guest_thread.task)?.ready_to_delete() {Waitable::Guest(guest_thread.task).delete_from(state)?;@@ -3227,7 +3204,7 @@ impl Instance {);let state = store.0.concurrent_state_mut();- let current_thread = state.guest_thread.unwrap();+ let current_thread = state.unwrap_current_guest_thread();let parent_task = current_thread.task;let new_thread = GuestThread::new_explicit(parent_task, start_func);@@ -3308,11 +3285,12 @@ impl Instance {yielding: bool,to_thread: Option<u32>,) -> Result<WaitResult> {+ let guest_thread = store.concurrent_state_mut().unwrap_current_guest_thread();if to_thread.is_none() {let state = store.concurrent_state_mut();if yielding {// This is a `thread.yield` call- if !state.may_block(state.guest_thread.unwrap().task) {+ if !state.may_block(guest_thread.task) {// The spec defines `thread.yield` to be a no-op in a// non-blocking context, so we return immediately without giving// any other thread a chance to run.@@ -3335,8 +3313,6 @@ impl Instance {self.resume_suspended_thread(store, caller, thread, true)?;}- let state = store.concurrent_state_mut();- let guest_thread = state.guest_thread.unwrap();let reason = if yielding {SuspendReason::Yielding {thread: guest_thread,@@ -3372,7 +3348,7 @@ impl Instance {check: WaitableCheck,params: WaitableCheckParams,) -> Result<u32> {- let guest_thread = store.concurrent_state_mut().guest_thread.unwrap();+ let guest_thread = store.concurrent_state_mut().unwrap_current_guest_thread();log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);@@ -3471,19 +3447,16 @@ impl Instance {}).handle_table().subtask_rep(task_id)?;- let (waitable, expected_caller_instance) = if is_host {+ let (waitable, expected_caller) = if is_host {let id = TableId::<HostTask>::new(rep);(Waitable::Host(id),- store.concurrent_state_mut().get_mut(id)?.caller_instance,+ store.concurrent_state_mut().get_mut(id)?.caller,)} else {let id = TableId::<GuestTask>::new(rep);- if let &Caller::Guest { thread } = &store.concurrent_state_mut().get_mut(id)?.caller {- (- Waitable::Guest(id),- store.concurrent_state_mut().get_mut(thread.task)?.instance,- )+ if let Caller::Guest { thread } = store.concurrent_state_mut().get_mut(id)?.caller {+ (Waitable::Guest(id), thread)} else {unreachable!()}@@ -3491,24 +3464,21 @@ impl Instance {// Since waitables can neither be passed between instances nor forged,// this should never fail unless there's a bug in Wasmtime, but we check// here to be sure:+ let concurrent_state = store.concurrent_state_mut();assert_eq!(- expected_caller_instance,- RuntimeInstance {- instance: self.id().instance(),- index: caller_instance- }+ expected_caller,+ concurrent_state.unwrap_current_guest_thread(),);log::trace!("subtask_cancel {waitable:?} (handle {task_id})");- let concurrent_state = store.concurrent_state_mut();if let Waitable::Host(host_task) = waitable {if let Some(handle) = concurrent_state.get_mut(host_task)?.join_handle.take() {handle.abort();return Ok(Status::ReturnCancelled as u32);}} else {- let caller = concurrent_state.guest_thread.unwrap();+ let caller = concurrent_state.unwrap_current_guest_thread();let guest_task = TableId::<GuestTask>::new(rep);let task = concurrent_state.get_mut(guest_task)?;if !task.already_lowered_parameters() {@@ -4096,18 +4066,36 @@ impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;/// Represents the state of a pending host task.+///+/// This is used to represent tasks when the guest calls into the host.struct HostTask {common: WaitableCommon,- caller_instance: RuntimeInstance,++ /// Guest thread which called the host.+ caller: QualifiedThreadId,++ /// State of borrows/etc the host needs to track. Used when the guest passes+ /// borrows to the host, for example.+ call_context: CallContext,++ /// For host tasks which end up doing some asynchronous work (e.g.+ /// async-lowered and didn't complete on the first poll) this handle is used+ /// as a signal to cancel the future as it resides in the store's+ /// `FuturesUnordered`.join_handle: Option<JoinHandle>,++ /// Box<Any> of the result of this host task.+ result: Option<LiftedResult>,}impl HostTask {- fn new(caller_instance: RuntimeInstance, join_handle: Option<JoinHandle>) -> Self {+ fn new(caller: QualifiedThreadId) -> Self {Self {common: WaitableCommon::default(),- caller_instance,- join_handle,+ call_context: CallContext::default(),+ caller,+ join_handle: None,+ result: None,}}}@@ -4136,11 +4124,10 @@ enum Caller {/// If true, there's a host future that must be dropped before the task/// can be deleted.host_future_present: bool,- /// If `Some`, represents the `QualifiedThreadId` caller of the host- /// function which called back into a guest. Note that this thread- /// could belong to an entirely unrelated top-level component instance- /// than the one the host called into.- caller: Option<QualifiedThreadId>,+ /// Represents the caller of the host function which called back into a+ /// guest. Note that this thread could belong to an entirely unrelated+ /// top-level component instance than the one the host called into.+ caller: CurrentThread,},/// Another guest thread called the guest taskGuest {@@ -4302,9 +4289,11 @@ pub(crate) struct GuestTask {callback: Option<CallbackFn>,/// See `Caller`caller: Caller,- /// A place to stash the call context for managing resource borrows while- /// switching between guest tasks.- call_context: Option<CallContext>,+ /// Borrow state for this task.+ ///+ /// Keeps track of `borrow<T>` received to this task to ensure that+ /// everything is dropped by the time it exits.+ call_context: CallContext,/// A place to stash the lowered result for a sync-to-async call until it/// can be returned to the caller.sync_result: SyncResult,@@ -4407,7 +4396,7 @@ impl GuestTask {result: None,callback,caller,- call_context: Some(CallContext::default()),+ call_context: CallContext::default(),sync_result: SyncResult::NotProduced,cancel_sent: false,starting_sent: false,@@ -4710,10 +4699,49 @@ impl ConcurrentInstanceState {}}+#[derive(Debug, Copy, Clone)]+enum CurrentThread {+ Guest(QualifiedThreadId),+ Host(TableId<HostTask>),+ None,+}++impl CurrentThread {+ fn guest(&self) -> Option<&QualifiedThreadId> {+ match self {+ Self::Guest(id) => Some(id),+ _ => None,+ }+ }++ fn host(&self) -> Option<TableId<HostTask>> {+ match self {+ Self::Host(id) => Some(*id),+ _ => None,+ }+ }++ fn is_none(&self) -> bool {+ matches!(self, Self::None)+ }+}++impl From<QualifiedThreadId> for CurrentThread {+ fn from(id: QualifiedThreadId) -> Self {+ Self::Guest(id)+ }+}++impl From<TableId<HostTask>> for CurrentThread {+ fn from(id: TableId<HostTask>) -> Self {+ Self::Host(id)+ }+}+/// Represents the Component Model Async state of a store.pub struct ConcurrentState {- /// The currently running guest thread, if any.- guest_thread: Option<QualifiedThreadId>,+ /// The currently running thread, if any.+ current_thread: CurrentThread,/// The set of pending host and background tasks, if any.///@@ -4756,7 +4784,7 @@ pub struct ConcurrentState {impl Default for ConcurrentState {fn default() -> Self {Self {- guest_thread: None,+ current_thread: CurrentThread::None,table: AlwaysMut::new(ResourceTable::new()),futures: AlwaysMut::new(Some(FuturesUnordered::new())),high_priority: Vec::new(),@@ -4915,7 +4943,7 @@ impl ConcurrentState {/// Implements the `context.get` intrinsic.pub(crate) fn context_get(&mut self, slot: u32) -> Result<u32> {- let thread = self.guest_thread.unwrap();+ let thread = self.unwrap_current_guest_thread();let val = self.get_mut(thread.thread)?.context[usize::try_from(slot).unwrap()];log::trace!("context_get {thread:?} slot {slot} val {val:#x}");Ok(val)@@ -4923,7 +4951,7 @@ impl ConcurrentState {/// Implements the `context.set` intrinsic.pub(crate) fn context_set(&mut self, slot: u32, val: u32) -> Result<()> {- let thread = self.guest_thread.unwrap();+ let thread = self.unwrap_current_guest_thread();log::trace!("context_set {thread:?} slot {slot} val {val:#x}");self.get_mut(thread.thread)?.context[usize::try_from(slot).unwrap()] = val;Ok(())@@ -4932,7 +4960,7 @@ impl ConcurrentState {/// Returns whether there's a pending cancellation on the current guest thread,/// consuming the event if so.fn take_pending_cancellation(&mut self) -> bool {- let thread = self.guest_thread.unwrap();+ let thread = self.unwrap_current_guest_thread();if let Some(event) = self.get_mut(thread.task).unwrap().event.take() {assert!(matches!(event, Event::Cancelled));true@@ -4953,6 +4981,42 @@ impl ConcurrentState {let task = self.get_mut(task).unwrap();task.async_function || task.returned_or_cancelled()}++ /// Used by `ResourceTables` to acquire the current `CallContext` for the+ /// specified task.+ ///+ /// The `task` is bit-packed as returned by `current_call_context_scope_id`+ /// below.+ pub fn call_context(&mut self, task: u32) -> &mut CallContext {+ let (task, is_host) = (task >> 1, task & 1 == 1);+ if is_host {+ let task: TableId<HostTask> = TableId::new(task);+ &mut self.get_mut(task).unwrap().call_context+ } else {+ let task: TableId<GuestTask> = TableId::new(task);+ &mut self.get_mut(task).unwrap().call_context+ }+ }++ /// Used by `ResourceTables` to record the scope of a borrow to get undone+ /// in the future.+ pub fn current_call_context_scope_id(&self) -> u32 {+ let (bits, is_host) = match self.current_thread {+ CurrentThread::Guest(id) => (id.task.rep(), false),+ CurrentThread::Host(id) => (id.rep(), true),+ CurrentThread::None => unreachable!(),+ };+ assert_eq!((bits << 1) >> 1, bits);+ (bits << 1) | u32::from(is_host)+ }++ fn unwrap_current_guest_thread(&self) -> QualifiedThreadId {+ *self.current_thread.guest().unwrap()+ }++ fn unwrap_current_host_thread(&self) -> TableId<HostTask> {+ self.current_thread.host().unwrap()+ }}/// Provide a type hint to compiler about the shape of a parameter lower@@ -5131,7 +5195,7 @@ pub(crate) fn prepare_call<T, R>(instance: handle.instance().id().instance(),index: component_instance,};- let caller = state.guest_thread;+ let caller = state.current_thread;let task = GuestTask::new(state,Box::new(for_any_lower(move |store, params| {diff --git a/crates/wasmtime/src/runtime/component/concurrent_disabled.rs b/crates/wasmtime/src/runtime/component/concurrent_disabled.rsindex 629c2c1fc9..16d73cadfa 100644--- a/crates/wasmtime/src/runtime/component/concurrent_disabled.rs+++ b/crates/wasmtime/src/runtime/component/concurrent_disabled.rs@@ -2,6 +2,7 @@ use crate::component::func::{LiftContext, LowerContext};use crate::component::matching::InstanceType;use crate::component::{ComponentType, Lift, Lower, RuntimeInstance, Val};use crate::store::StoreOpaque;+use crate::vm::component::CallContext;use crate::{Result, bail, error::format_err};use core::convert::Infallible;use core::mem::MaybeUninit;@@ -9,6 +10,16 @@ use wasmtime_environ::component::{CanonicalAbiInfo, InterfaceType};pub enum ConcurrentState {}+impl ConcurrentState {+ pub fn call_context(&mut self, _: u32) -> &mut CallContext {+ match *self {}+ }++ pub fn current_call_context_scope_id(&self) -> u32 {+ match *self {}+ }+}+fn should_have_failed_validation<T>(what: &str) -> Result<T> {// This should be unreachable; if we trap here, it indicates a// bug in Wasmtime rather than in the guest.@@ -151,21 +162,25 @@ unsafe impl Lower for StreamAny {}impl StoreOpaque {- pub(crate) fn enter_sync_call(+ pub(crate) fn enter_guest_sync_call(&mut self,_guest_caller: Option<RuntimeInstance>,_callee_async: bool,_callee: RuntimeInstance,) -> Result<()> {- // If we've reached here, the user somehow managed to enable the- // `component-model-async` runtime config feature without enabling the- // corresponding compile-time feature.- unreachable!()+ Ok(self.enter_call_not_concurrent())+ }++ pub(crate) fn exit_guest_sync_call(&mut self, _guest_caller: bool) -> Result<()> {+ Ok(self.exit_call_not_concurrent())+ }++ pub(crate) fn enter_host_call(&mut self) -> Result<()> {+ Ok(self.enter_call_not_concurrent())}- pub(crate) fn exit_sync_call(&mut self, _guest_caller: bool) -> Result<()> {- // See comment in `enter_sync_call`- unreachable!()+ pub(crate) fn exit_host_call(&mut self) -> Result<()> {+ Ok(self.exit_call_not_concurrent())}pub(crate) fn check_blocking(&mut self) -> crate::Result<()> {diff --git a/crates/wasmtime/src/runtime/component/func.rs b/crates/wasmtime/src/runtime/component/func.rsindex 4c81947d67..3cf236d2fa 100644--- a/crates/wasmtime/src/runtime/component/func.rs+++ b/crates/wasmtime/src/runtime/component/func.rs@@ -603,10 +603,8 @@ impl Func {bail!(crate::Trap::CannotEnterComponent);}- if store.0.concurrency_support() {- let async_type = self.abi_async(store.0);- store.0.enter_sync_call(None, async_type, instance)?;- }+ let async_type = self.abi_async(store.0);+ store.0.enter_guest_sync_call(None, async_type, instance)?;#[repr(C)]union Union<Params: Copy, Return: Copy> {@@ -632,7 +630,6 @@ impl Func {assert!(mem::align_of_val(map_maybe_uninit!(space.ret)) == val_align);self.with_lower_context(store.as_context_mut(), |cx, ty| {- cx.enter_call();lower(cx, ty, map_maybe_uninit!(space.params))})?;@@ -716,11 +713,8 @@ impl Func {store.0.component_resource_tables(Some(self.instance))- .exit_call()?;-- if store.0.concurrency_support() {- store.0.exit_sync_call(false)?;- }+ .validate_scope_exit()?;+ store.0.exit_guest_sync_call(false)?;}Ok(())}diff --git a/crates/wasmtime/src/runtime/component/func/host.rs b/crates/wasmtime/src/runtime/component/func/host.rsindex b746f75c63..c4b8d8bb88 100644--- a/crates/wasmtime/src/runtime/component/func/host.rs+++ b/crates/wasmtime/src/runtime/component/func/host.rs@@ -1,7 +1,5 @@//! Implementation of calling Rust-defined functions from components.-#[cfg(feature = "component-model-async")]-use crate::component::RuntimeInstance;#[cfg(feature = "component-model-async")]use crate::component::concurrent;#[cfg(feature = "component-model-async")]@@ -363,26 +361,50 @@ where/// arguments are translated to Rust types.fn entrypoint(&self,- store: StoreContextMut<'_, T>,+ mut store: StoreContextMut<'_, T>,instance: Instance,ty: TypeFuncIndex,options: OptionsIndex,storage: &mut [MaybeUninit<ValRaw>],) -> Result<()> {let vminstance = instance.id().get(store.0);- let opts = &vminstance.component().env_component().options[options];+ let async_ = vminstance.component().env_component().options[options].async_;- if opts.async_ {+ // If this is a synchronous-lower of a host-async function, then the+ // guest is blocking. Test, in the context of the guest task, if that's+ // allowed.+ if !async_ && Self::ASYNC {+ store.0.check_blocking()?;+ }++ // Enter the host by pushing a `HostTask` into the concurrent state.+ store.0.enter_host_call()?;++ let task_exited = if async_ {#[cfg(feature = "component-model-async")]- return self.call_async_lower(store, instance, ty, options, storage);+ {+ self.call_async_lower(store.as_context_mut(), instance, ty, options, storage)?+ }#[cfg(not(feature = "component-model-async"))]unreachable!("async-lowered imports should have failed validation \when `component-model-async` feature disabled");} else {- self.call_sync_lower(store, instance, ty, options, storage)+ self.call_sync_lower(store.as_context_mut(), instance, ty, options, storage)?;+ true+ };++ // If the host task exited, then it's popped and deallocated.+ //+ // Note that if the host task did not exit then the `call_async_lower`+ // function transitively would have updated the current guest thread to+ // the caller of this host function.+ if task_exited {+ store.0.exit_host_call()?;}++ Ok(())}/// Implementation of the "sync" ABI.@@ -400,29 +422,13 @@ whereoptions: OptionsIndex,storage: &mut [MaybeUninit<ValRaw>],) -> Result<()> {- if Self::ASYNC {- // The caller has synchronously lowered an async function, meaning- // the caller can only call it from an async task (i.e. a task- // created via a call to an async export). Otherwise, we'll trap.- store.0.check_blocking()?;- }-let mut lift = LiftContext::new(store.0.store_opaque_mut(), options, instance);let (params, rest) = self.load_params(&mut lift, ty, MAX_FLAT_PARAMS, storage)?;- #[cfg(feature = "component-model-async")]- let caller_instance = lift.options().instance;let ret = match self.run(store.as_context_mut(), params) {HostResult::Done(result) => result?,#[cfg(feature = "component-model-async")]- HostResult::Future(future) => concurrent::poll_and_block(- store.0,- future,- RuntimeInstance {- instance: instance.id().instance(),- index: caller_instance,- },- )?,+ HostResult::Future(future) => concurrent::poll_and_block(store.0, future)?,};let mut lower = LowerContext::new(store, options, instance);@@ -457,7 +463,7 @@ wherety: TypeFuncIndex,options: OptionsIndex,storage: &mut [MaybeUninit<ValRaw>],- ) -> Result<()> {+ ) -> Result<bool> {use wasmtime_environ::component::MAX_FLAT_ASYNC_PARAMS;let (component, store) = instance.component_and_store_mut(store.0);@@ -468,7 +474,6 @@ where// Lift the parameters, either from flat storage or from linear// memory.let mut lift = LiftContext::new(store.0.store_opaque_mut(), options, instance);- let caller_instance = lift.options().instance;let (params, rest) = self.load_params(&mut lift, ty, MAX_FLAT_ASYNC_PARAMS, storage)?;// Load/validate the return pointer, if present.@@ -501,7 +506,7 @@ where}#[cfg(feature = "component-model-async")]HostResult::Future(future) => {- instance.first_poll(store, future, caller_instance, move |store, ret| {+ instance.first_poll(store, future, move |store, ret| {Self::lower_result_and_exit_call(&mut LowerContext::new(store, options, instance),ty,@@ -518,7 +523,7 @@ whereStatus::Returned.pack(None)}));- Ok(())+ Ok(task.is_none())}/// Loads parameters the wasm arguments `storage`.@@ -535,7 +540,6 @@ wherelet fty = &lift.types[ty];let param_tys = &lift.types[fty.params];let param_flat_count = param_tys.abi.flat_count(max_flat_params);- lift.enter_call();let src = match param_flat_count {Some(cnt) => {let params = &storage[..cnt];@@ -576,7 +580,7 @@ whereunsafe {flags.set_may_leave(true);}- lower.exit_call()?;+ lower.validate_scope_exit()?;Ok(())}}diff --git a/crates/wasmtime/src/runtime/component/func/options.rs b/crates/wasmtime/src/runtime/component/func/options.rsindex 61788584f4..4cce6bd8e5 100644--- a/crates/wasmtime/src/runtime/component/func/options.rs+++ b/crates/wasmtime/src/runtime/component/func/options.rs@@ -1,11 +1,13 @@use crate::StoreContextMut;+#[cfg(feature = "component-model-async")]use crate::component::concurrent::ConcurrentState;use crate::component::matching::InstanceType;use crate::component::resources::{HostResourceData, HostResourceIndex, HostResourceTables};+use crate::component::store::ComponentTaskState;use crate::component::{Instance, ResourceType, RuntimeInstance};use crate::prelude::*;use crate::runtime::vm::VMFuncRef;-use crate::runtime::vm::component::{CallContexts, ComponentInstance, HandleTable, ResourceTables};+use crate::runtime::vm::component::{ComponentInstance, HandleTable, ResourceTables};use crate::store::{StoreId, StoreOpaque};use alloc::sync::Arc;use core::pin::Pin;@@ -283,16 +285,10 @@ impl<'a, T: 'static> LowerContext<'a, T> {HostResourceTables::from_parts(tables, data)}- /// See [`HostResourceTables::enter_call`].+ /// See [`HostResourceTables::validate_scope_exit`].#[inline]- pub fn enter_call(&mut self) {- self.resource_tables().enter_call()- }-- /// See [`HostResourceTables::exit_call`].- #[inline]- pub fn exit_call(&mut self) -> Result<()> {- self.resource_tables().exit_call()+ pub fn validate_scope_exit(&mut self) -> Result<()> {+ self.resource_tables().validate_scope_exit()}}@@ -318,13 +314,7 @@ pub struct LiftContext<'a> {host_table: &'a mut HandleTable,host_resource_data: &'a mut HostResourceData,- calls: &'a mut CallContexts,-- #[cfg_attr(- not(feature = "component-model-async"),- allow(unused, reason = "easier to not #[cfg] away")- )]- concurrent_state: Option<&'a mut ConcurrentState>,+ task_state: &'a mut ComponentTaskState,}#[doc(hidden)]@@ -345,8 +335,8 @@ impl<'a> LiftContext<'a> {// at this time.let memory =instance_handle.options_memory(unsafe { &*(store as *const StoreOpaque) }, options);- let (calls, host_table, host_resource_data, instance, concurrent_state) =- store.component_resource_state_with_instance_and_concurrent_state(instance_handle);+ let (task_state, host_table, host_resource_data, instance) =+ store.lift_context_parts(instance_handle);let (component, instance) = instance.component_and_self();LiftContext {@@ -356,10 +346,9 @@ impl<'a> LiftContext<'a> {types: component.types(),instance,instance_handle,- calls,+ task_state,host_table,host_resource_data,- concurrent_state,}}@@ -401,7 +390,7 @@ impl<'a> LiftContext<'a> {#[cfg(feature = "component-model-async")]pub(crate) fn concurrent_state_mut(&mut self) -> &mut ConcurrentState {- self.concurrent_state.as_deref_mut().unwrap()+ self.task_state.concurrent_state_mut()}/// Lifts an `own` resource from the guest at the `idx` specified into its@@ -460,23 +449,17 @@ impl<'a> LiftContext<'a> {fn resource_tables(&mut self) -> HostResourceTables<'_> {HostResourceTables::from_parts(ResourceTables {- host_table: Some(self.host_table),- calls: self.calls,+ host_table: self.host_table,+ task_state: self.task_state,guest: Some(self.instance.as_mut().instance_states()),},self.host_resource_data,)}- /// See [`HostResourceTables::enter_call`].- #[inline]- pub fn enter_call(&mut self) {- self.resource_tables().enter_call()- }-- /// See [`HostResourceTables::exit_call`].+ /// See [`HostResourceTables::validate_scope_exit`].#[inline]- pub fn exit_call(&mut self) -> Result<()> {- self.resource_tables().exit_call()+ pub fn validate_scope_exit(&mut self) -> Result<()> {+ self.resource_tables().validate_scope_exit()}}diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rsindex 9cd336a7fc..d92b5d8c8b 100644--- a/crates/wasmtime/src/runtime/component/instance.rs+++ b/crates/wasmtime/src/runtime/component/instance.rs@@ -446,14 +446,6 @@ impl Instance {tables.resource_lower_borrow(TypedResource::Component { ty: dst, rep })}- pub(crate) fn resource_enter_call(self, store: &mut StoreOpaque) {- store.component_resource_tables(Some(self)).enter_call()- }-- pub(crate) fn resource_exit_call(self, store: &mut StoreOpaque) -> Result<()> {- store.component_resource_tables(Some(self)).exit_call()- }-pub(crate) fn lookup_vmdef(&self, store: &mut StoreOpaque, def: &CoreDef) -> vm::Export {lookup_vmdef(store, self.id.instance(), def)}@@ -850,10 +842,8 @@ impl<'a> Instantiator<'a> {}};- let exit = if let Some(component_instance) = *component_instance- && store.0.concurrency_support()- {- store.0.enter_sync_call(+ let exit = if let Some(component_instance) = *component_instance {+ store.0.enter_guest_sync_call(None,false,RuntimeInstance {@@ -881,7 +871,7 @@ impl<'a> Instantiator<'a> {};if exit {- store.0.exit_sync_call(false)?;+ store.0.exit_guest_sync_call(false)?;}self.instance_mut(store.0).push_instance_id(i.id());diff --git a/crates/wasmtime/src/runtime/component/resources/any.rs b/crates/wasmtime/src/runtime/component/resources/any.rsindex 27cb760c9e..6c3db02a8e 100644--- a/crates/wasmtime/src/runtime/component/resources/any.rs+++ b/crates/wasmtime/src/runtime/component/resources/any.rs@@ -197,14 +197,15 @@ impl ResourceAny {};let mut args = [ValRaw::u32(rep)];- let exit = if let Some(instance) = slot.instance- && store.0.concurrency_support()- {- store.0.enter_sync_call(None, false, instance)?;- true- } else {- false- };+ // Setup async-level task infrastructure for this call. This, for+ // example, prevents the destructor from blocking.+ //+ // Note that if `slot.instance` is `None` then this is skipped. That+ // means that this is a host resource being destroyed by the host. In+ // that case restrictions around blocking and such are exempt.+ if let Some(instance) = slot.instance {+ store.0.enter_guest_sync_call(None, false, instance)?;+ }// This should be safe because `dtor` has been checked to belong to the// `store` provided which means it's valid and still alive. Additionally@@ -215,8 +216,8 @@ impl ResourceAny {crate::Func::call_unchecked_raw(store, dtor, NonNull::from(&mut args))?;}- if exit {- store.0.exit_sync_call(false)?;+ if slot.instance.is_some() {+ store.0.exit_guest_sync_call(false)?;}Ok(())diff --git a/crates/wasmtime/src/runtime/component/resources/host_tables.rs b/crates/wasmtime/src/runtime/component/resources/host_tables.rsindex a6f70e46b9..625035296b 100644--- a/crates/wasmtime/src/runtime/component/resources/host_tables.rs+++ b/crates/wasmtime/src/runtime/component/resources/host_tables.rs@@ -309,17 +309,10 @@ impl<'a> HostResourceTables<'a> {.resource_lift_borrow(TypedResourceIndex::Component { ty, index })}- /// Begins a call into the component instance, starting recording of- /// metadata related to resource borrowing.- #[inline]- pub fn enter_call(&mut self) {- self.tables.enter_call()- }-/// Completes a call into the component instance, validating that it's ok to/// complete by ensuring the are no remaining active borrows.#[inline]- pub fn exit_call(&mut self) -> Result<()> {- self.tables.exit_call()+ pub fn validate_scope_exit(&mut self) -> Result<()> {+ self.tables.validate_scope_exit()}}diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rsindex 81b2dbddbe..32ad58d765 100644--- a/crates/wasmtime/src/runtime/component/store.rs+++ b/crates/wasmtime/src/runtime/component/store.rs@@ -1,14 +1,14 @@+use crate::prelude::*;use crate::runtime::component::concurrent::ConcurrentState;use crate::runtime::component::{HostResourceData, Instance};use crate::runtime::vm;#[cfg(feature = "component-model-async")]use crate::runtime::vm::VMStore;-use crate::runtime::vm::component::{CallContexts, HandleTable};-use crate::runtime::vm::component::{ComponentInstance, OwnedComponentInstance};+use crate::runtime::vm::component::{+ CallContext, ComponentInstance, HandleTable, OwnedComponentInstance,+};use crate::store::{StoreData, StoreId, StoreOpaque};use crate::{Engine, StoreContextMut};-#[cfg(feature = "component-model-async")]-use alloc::vec::Vec;use core::pin::Pin;use wasmtime_environ::PrimaryMap;use wasmtime_environ::component::RuntimeComponentInstanceIndex;@@ -31,12 +31,22 @@ pub struct ComponentStoreData {/// and calls. These also interact with the `ResourceAny` type and its/// internal representation.component_host_table: HandleTable,- component_calls: CallContexts,host_resource_data: HostResourceData,/// Metadata/tasks/etc related to component-model-async and concurrency/// support.- concurrent_state: Option<ConcurrentState>,+ task_state: ComponentTaskState,+}++/// State tracking for tasks within components.+pub enum ComponentTaskState {+ /// Used when `Config::concurrency_support` is disabled. Here there are no+ /// async tasks but there's still state for borrows that needs managing.+ NotConcurrent(ComponentTasksNotConcurrent),++ /// Used when `Config::concurrency_support` is enabled and has+ /// full state for all async tasks.+ Concurrent(ConcurrentState),}#[derive(Copy, Clone, Debug, PartialEq, Eq)]@@ -56,19 +66,21 @@ impl ComponentStoreData {trapped: false,num_component_instances: 0,component_host_table: Default::default(),- component_calls: Default::default(),host_resource_data: Default::default(),- concurrent_state: if engine.tunables().concurrency_support {+ task_state: if engine.tunables().concurrency_support {#[cfg(feature = "component-model-async")]{- Some(Default::default())+ ComponentTaskState::Concurrent(Default::default())}#[cfg(not(feature = "component-model-async"))]{+ // This should be validated in `Config` where if+ // `concurrency_support` is enabled but compile time support+ // isn't available then an `Engine` isn't creatable.unreachable!()}} else {- None+ ComponentTaskState::NotConcurrent(Default::default())},}}@@ -87,7 +99,7 @@ impl ComponentStoreData {// in their `Drop::drop` implementations, in which case they'll need to// be called from with in the context of a `tls::set` closure.#[cfg(feature = "component-model-async")]- if store.0.component_data().concurrent_state.is_some() {+ if store.0.component_data().task_state.is_concurrent() {ComponentStoreData::drop_fibers_and_futures(store.0);}#[cfg(not(feature = "component-model-async"))]@@ -256,6 +268,7 @@ impl StoreOpaque {self.store_data_mut().components.trapped = true;}+ #[cfg(feature = "component-model-async")]pub(crate) fn component_data(&self) -> &ComponentStoreData {&self.store_data().components}@@ -264,8 +277,8 @@ impl StoreOpaque {&mut self.store_data_mut().components}- pub(crate) fn component_call_contexts_mut(&mut self) -> &mut CallContexts {- &mut self.component_data_mut().component_calls+ pub(crate) fn component_task_state_mut(&mut self) -> &mut ComponentTaskState {+ &mut self.component_data_mut().task_state}pub(crate) fn push_component_instance(&mut self, instance: Instance) {@@ -292,38 +305,37 @@ impl StoreOpaque {#[cfg(feature = "component-model-async")]pub(crate) fn concurrent_state_mut(&mut self) -> &mut ConcurrentState {debug_assert!(self.concurrency_support());- self.component_data_mut().concurrent_state.as_mut().unwrap()+ self.component_data_mut().task_state.concurrent_state_mut()}#[inline]+ #[cfg(feature = "component-model-async")]pub(crate) fn concurrency_support(&self) -> bool {- let support = self.component_data().concurrent_state.is_some();+ let support = self.component_data().task_state.is_concurrent();debug_assert_eq!(support, self.engine().tunables().concurrency_support);support}- pub(crate) fn component_resource_state_with_instance_and_concurrent_state(+ pub(crate) fn lift_context_parts(&mut self,instance: Instance,) -> (- &mut CallContexts,+ &mut ComponentTaskState,&mut HandleTable,&mut HostResourceData,Pin<&mut ComponentInstance>,- Option<&mut ConcurrentState>,) {let instance = instance.id();instance.assert_belongs_to(self.id());let data = self.component_data_mut();(- &mut data.component_calls,+ &mut data.task_state,&mut data.component_host_table,&mut data.host_resource_data,data.instances[instance.instance].as_mut().unwrap().get_mut(),- data.concurrent_state.as_mut(),)}@@ -356,11 +368,67 @@ impl StoreOpaque {(vm::component::ResourceTables {- host_table: Some(&mut data.component_host_table),- calls: &mut data.component_calls,+ host_table: &mut data.component_host_table,+ task_state: &mut data.task_state,guest,},&mut data.host_resource_data,)}++ pub(crate) fn enter_call_not_concurrent(&mut self) {+ let state = match &mut self.component_data_mut().task_state {+ ComponentTaskState::NotConcurrent(state) => state,+ ComponentTaskState::Concurrent(_) => unreachable!(),+ };+ state.scopes.push(CallContext::default());+ }++ pub(crate) fn exit_call_not_concurrent(&mut self) {+ let state = match &mut self.component_data_mut().task_state {+ ComponentTaskState::NotConcurrent(state) => state,+ ComponentTaskState::Concurrent(_) => unreachable!(),+ };+ state.scopes.pop();+ }+}++#[derive(Default)]+pub struct ComponentTasksNotConcurrent {+ scopes: Vec<CallContext>,+}++impl ComponentTaskState {+ pub fn call_context(&mut self, id: u32) -> &mut CallContext {+ match self {+ ComponentTaskState::NotConcurrent(state) => &mut state.scopes[id as usize],+ ComponentTaskState::Concurrent(state) => state.call_context(id),+ }+ }++ pub fn current_call_context_scope_id(&self) -> u32 {+ match self {+ ComponentTaskState::NotConcurrent(state) => {+ u32::try_from(state.scopes.len() - 1).unwrap()+ }+ ComponentTaskState::Concurrent(state) => state.current_call_context_scope_id(),+ }+ }++ pub fn concurrent_state_mut(&mut self) -> &mut ConcurrentState {+ match self {+ ComponentTaskState::Concurrent(state) => state,+ ComponentTaskState::NotConcurrent(_) => {+ panic!("expected concurrent state to be present")+ }+ }+ }++ #[cfg(feature = "component-model-async")]+ fn is_concurrent(&self) -> bool {+ match self {+ ComponentTaskState::Concurrent(_) => true,+ ComponentTaskState::NotConcurrent(_) => false,+ }+ }}diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 8e7edfc48a..dc2d34121d 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -2740,8 +2740,8 @@ unsafe impl<T> VMStore for StoreInner<T> {}#[cfg(feature = "component-model")]- fn component_calls(&mut self) -> &mut vm::component::CallContexts {- self.component_call_contexts_mut()+ fn component_task_state_mut(&mut self) -> &mut crate::component::store::ComponentTaskState {+ StoreOpaque::component_task_state_mut(self)}fn store_opaque(&self) -> &StoreOpaque {diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex cef56814f4..35934f49af 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -223,7 +223,7 @@ pub unsafe trait VMStore: 'static {/// Metadata required for resources for the component model.#[cfg(feature = "component-model")]- fn component_calls(&mut self) -> &mut component::CallContexts;+ fn component_task_state_mut(&mut self) -> &mut crate::component::store::ComponentTaskState;#[cfg(feature = "component-model-async")]fn component_async_store(diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rsindex 9850b28366..52c2a2c530 100644--- a/crates/wasmtime/src/runtime/vm/component.rs+++ b/crates/wasmtime/src/runtime/vm/component.rs@@ -44,9 +44,7 @@ mod resources;pub use self::handle_table::{HandleTable, RemovedResource};#[cfg(feature = "component-model-async")]pub use self::handle_table::{ThreadHandleTable, TransmitLocalState, Waitable};-#[cfg(feature = "component-model-async")]-pub use self::resources::CallContext;-pub use self::resources::{CallContexts, ResourceTables, TypedResource, TypedResourceIndex};+pub use self::resources::{CallContext, ResourceTables, TypedResource, TypedResourceIndex};/// Represents the state of a (sub-)component instance.#[derive(Default)]diff --git a/crates/wasmtime/src/runtime/vm/component/handle_table.rs b/crates/wasmtime/src/runtime/vm/component/handle_table.rsindex 0fc19800b0..6976497df9 100644--- a/crates/wasmtime/src/runtime/vm/component/handle_table.rs+++ b/crates/wasmtime/src/runtime/vm/component/handle_table.rs@@ -36,7 +36,7 @@ pub enum RemovedResource {/// An `own` resource was removed with the specified `rep`Own { rep: u32 },/// A `borrow` resource was removed originally created within `scope`.- Borrow { scope: usize },+ Borrow { scope: u32 },}/// Different kinds of waitables returned by [`HandleTable::waitable_rep`].@@ -68,7 +68,7 @@ enum Slot {/// count of the `scope`.ResourceBorrow {resource: TypedResource,- scope: usize,+ scope: u32,},/// Represents a host task handle.@@ -195,7 +195,7 @@ impl HandleTable {/// Inserts a new `borrow` resource into this table whose type/rep are/// specified by `resource`. The `scope` specified is used by/// `CallContexts` to manage lending information.- pub fn resource_borrow_insert(&mut self, resource: TypedResource, scope: usize) -> Result<u32> {+ pub fn resource_borrow_insert(&mut self, resource: TypedResource, scope: u32) -> Result<u32> {self.insert(Slot::ResourceBorrow { resource, scope })}diff --git a/crates/wasmtime/src/runtime/vm/component/libcalls.rs b/crates/wasmtime/src/runtime/vm/component/libcalls.rsindex fa2808602d..655524af81 100644--- a/crates/wasmtime/src/runtime/vm/component/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/component/libcalls.rs@@ -4,7 +4,6 @@ use crate::component::Instance;#[cfg(feature = "component-model-async")]use crate::component::concurrent::WaitResult;use crate::prelude::*;-#[cfg(feature = "component-model-async")]use crate::runtime::component::RuntimeInstance;#[cfg(feature = "component-model-async")]use crate::runtime::component::concurrent::ResourcePair;@@ -653,21 +652,12 @@ fn resource_transfer_borrow(instance.resource_transfer_borrow(store, src_idx, src_table, dst_table)}-fn resource_enter_call(store: &mut dyn VMStore, instance: Instance) {- instance.resource_enter_call(store)-}--fn resource_exit_call(store: &mut dyn VMStore, instance: Instance) -> Result<()> {- instance.resource_exit_call(store)-}-fn trap(_store: &mut dyn VMStore, _instance: Instance, code: u32) -> Result<()> {Err(wasmtime_environ::Trap::from_u8(u8::try_from(code).unwrap()).unwrap().into())}-#[cfg(feature = "component-model-async")]fn enter_sync_call(store: &mut dyn VMStore,instance: Instance,@@ -675,7 +665,7 @@ fn enter_sync_call(callee_async: u32,callee_instance: u32,) -> Result<()> {- store.enter_sync_call(+ store.enter_guest_sync_call(Some(RuntimeInstance {instance: instance.id().instance(),index: RuntimeComponentInstanceIndex::from_u32(caller_instance),@@ -688,9 +678,11 @@ fn enter_sync_call()}-#[cfg(feature = "component-model-async")]-fn exit_sync_call(store: &mut dyn VMStore, _instance: Instance) -> Result<()> {- store.exit_sync_call(true)+fn exit_sync_call(store: &mut dyn VMStore, instance: Instance) -> Result<()> {+ store+ .component_resource_tables(Some(instance))+ .validate_scope_exit()?;+ store.exit_guest_sync_call(true)}#[cfg(feature = "component-model-async")]diff --git a/crates/wasmtime/src/runtime/vm/component/resources.rs b/crates/wasmtime/src/runtime/vm/component/resources.rsindex 11473f45bf..56deccdd43 100644--- a/crates/wasmtime/src/runtime/vm/component/resources.rs+++ b/crates/wasmtime/src/runtime/vm/component/resources.rs@@ -11,7 +11,7 @@//! * `ResourceTables` - the "here's everything" context which is required to//! perform canonical ABI operations.//!-//! * `CallContexts` - store-local information about active calls and borrows+//! * `CallContext` - per-task information about active calls and borrows//! and runtime state tracking that to ensure that everything is handled//! correctly.//!@@ -21,9 +21,11 @@//! namely in the `Resource<T>` and `ResourceAny` types.use super::{HandleTable, InstanceState, RemovedResource};+use crate::component::store::ComponentTaskState;use crate::prelude::*;use core::error::Error;use core::fmt;+use core::mem;use wasmtime_environ::PrimaryMap;use wasmtime_environ::component::{ComponentTypes, RuntimeComponentInstanceIndex, TypeResourceTableIndex,@@ -64,11 +66,11 @@ pub struct ResourceTables<'a> {/// as-if they're in-component resources. The major distinction though is/// that this is a heterogeneous table instead of only containing a single/// type.- pub host_table: Option<&'a mut HandleTable>,+ pub host_table: &'a mut HandleTable,- /// Scope information about calls actively in use to track information such+ /// Task information about calls actively in use to track information such/// as borrow counts.- pub calls: &'a mut CallContexts,+ pub task_state: &'a mut ComponentTaskState,}/// Typed representation of a "rep" for a resource.@@ -164,26 +166,6 @@ impl TypedResourceIndex {}}-/// State related to borrows and calls within a component.-///-/// This is created once per `Store` and updated and modified throughout the-/// lifetime of the store. This primarily tracks borrow counts and what slots-/// should be updated when calls go out of scope.-#[derive(Default)]-pub struct CallContexts {- scopes: Vec<CallContext>,-}--impl CallContexts {- pub fn push(&mut self, cx: CallContext) {- self.scopes.push(cx);- }-- pub fn pop(&mut self) -> Option<CallContext> {- self.scopes.pop()- }-}-/// State related to borrows for a specific call.#[derive(Default)]pub struct CallContext {@@ -194,7 +176,7 @@ pub struct CallContext {impl ResourceTables<'_> {fn table_for_resource(&mut self, resource: &TypedResource) -> &mut HandleTable {match resource {- TypedResource::Host(_) => self.host_table.as_mut().unwrap(),+ TypedResource::Host(_) => self.host_table,TypedResource::Component { ty, .. } => {let (states, types) = self.guest.as_mut().unwrap();states[types[*ty].unwrap_concrete_instance()].handle_table()@@ -204,7 +186,7 @@ impl ResourceTables<'_> {fn table_for_index(&mut self, index: &TypedResourceIndex) -> &mut HandleTable {match index {- TypedResourceIndex::Host(_) => self.host_table.as_mut().unwrap(),+ TypedResourceIndex::Host(_) => self.host_table,TypedResourceIndex::Component { ty, .. } => {let (states, types) = self.guest.as_mut().unwrap();states[types[*ty].unwrap_concrete_instance()].handle_table()@@ -244,7 +226,7 @@ impl ResourceTables<'_> {match self.table_for_index(&index).remove_resource(index)? {RemovedResource::Own { rep } => Ok(Some(rep)),RemovedResource::Borrow { scope } => {- self.calls.scopes[scope].borrow_count -= 1;+ self.task_state.call_context(scope).borrow_count -= 1;Ok(None)}}@@ -291,8 +273,8 @@ impl ResourceTables<'_> {pub fn resource_lift_borrow(&mut self, index: TypedResourceIndex) -> Result<u32> {let (rep, is_own) = self.table_for_index(&index).resource_lend(index)?;if is_own {- let scope = self.calls.scopes.last_mut().unwrap();- scope.lenders.push(index);+ let scope = self.task_state.current_call_context_scope_id();+ self.task_state.call_context(scope).lenders.push(index);}Ok(rep)}@@ -310,37 +292,31 @@ impl ResourceTables<'_> {/// `VMComponentContext` which handles the special case of avoiding borrow/// tracking entirely.pub fn resource_lower_borrow(&mut self, resource: TypedResource) -> Result<u32> {- let scope = self.calls.scopes.len() - 1;- let borrow_count = &mut self.calls.scopes.last_mut().unwrap().borrow_count;- *borrow_count = borrow_count.checked_add(1).unwrap();+ let scope = self.task_state.current_call_context_scope_id();+ let cx = self.task_state.call_context(scope);+ cx.borrow_count = cx.borrow_count.checked_add(1).unwrap();self.table_for_resource(&resource).resource_borrow_insert(resource, scope)}- /// Enters a new calling context, starting a fresh count of borrows and- /// such.- #[inline]- pub fn enter_call(&mut self) {- self.calls.scopes.push(CallContext::default());- }-- /// Exits the previously pushed calling context.+ /// Validates that the current scope can be exited.///- /// This requires all information to be available within this- /// `ResourceTables` and is only called during lowering/lifting operations- /// at this time.+ /// This will ensure that this context's active borrows have all been+ /// dropped. This will then commit the lend decrements back to the owned+ /// resources that were originally passed in.#[inline]- pub fn exit_call(&mut self) -> Result<()> {- let cx = self.calls.scopes.pop().unwrap();+ pub fn validate_scope_exit(&mut self) -> Result<()> {+ let current = self.task_state.current_call_context_scope_id();+ let cx = self.task_state.call_context(current);if cx.borrow_count > 0 {bail!("borrow handles still remain at the end of the call")}- for lender in cx.lenders.iter() {+ for lender in mem::take(&mut cx.lenders) {// Note the panics here which should never get triggered in theory// due to the dynamic tracking of borrows and such employed for// resources.- self.table_for_index(lender)- .resource_undo_lend(*lender)+ self.table_for_index(&lender)+ .resource_undo_lend(lender).unwrap();}Ok(())diff --git a/tests/all/component_model/async.rs b/tests/all/component_model/async.rsindex 949639e1dd..396079cab3 100644--- a/tests/all/component_model/async.rs+++ b/tests/all/component_model/async.rs@@ -2,8 +2,7 @@ use crate::async_functions::{PollOnce, execute_across_threads};use std::pin::Pin;use std::task::{Context, Poll};use wasmtime::Result;-use wasmtime::{Config, component::*};-use wasmtime::{Engine, Store, StoreContextMut, Trap};+use wasmtime::{AsContextMut, Config, Engine, Store, StoreContextMut, Trap, component::*};use wasmtime_component_util::REALLOC_AND_FREE;/// This is super::func::thunks, except with an async store.@@ -873,3 +872,177 @@ async fn require_concurrency_support() -> Result<()> {Ok(())}++#[tokio::test]+#[cfg_attr(miri, ignore)]+async fn cancel_host_task_does_not_leak() -> Result<()> {+ let mut config = Config::new();+ config.wasm_component_model_async(true);+ let engine = Engine::new(&config)?;++ let mut store = Store::new(&engine, ());+ let component = Component::new(+ &engine,+ r#"(component+ (import "f" (func $f async))++ (core module $m+ (import "" "f" (func $f (result i32)))+ (import "" "cancel" (func $cancel (param i32) (result i32)))+ (import "" "drop" (func $drop (param i32)))+ (func (export "run")+ (local i32)++ ;; start the subtask, asserting it's `STARTED`+ call $f+ local.tee 0+ i32.const 0xf+ i32.and+ i32.const 1 ;; STARTED+ i32.ne+ if unreachable end++ ;; extract the task id+ local.get 0+ i32.const 4+ i32.shr_u+ local.set 0++ ;; cancel the subtask asserting it's `RETURN_CANCELLED`+ local.get 0+ call $cancel+ i32.const 4 ;; RETURN_CANCELLED+ i32.ne+ if unreachable end++ ;; drop the subtask+ local.get 0+ call $drop+ )+ )+ (core func $f (canon lower (func $f) async))+ (core func $cancel (canon subtask.cancel))+ (core func $drop (canon subtask.drop))+ (core instance $i (instantiate $m+ (with "" (instance+ (export "f" (func $f))+ (export "cancel" (func $cancel))+ (export "drop" (func $drop))+ ))+ ))++ (func (export "f") async+ (canon lift (core func $i "run")))+++ )"#,+ )?;++ let mut linker = Linker::new(&engine);+ linker.root().func_wrap_concurrent("f", |_, ()| {+ Box::pin(async move {+ std::future::pending::<()>().await;+ Ok(())+ })+ })?;+ let instance = linker.instantiate_async(&mut store, &component).await?;+ let func = instance.get_typed_func::<(), ()>(&mut store, "f")?;+ store+ .run_concurrent(async |store| -> wasmtime::Result<()> {+ func.call_concurrent(store, ()).await?;++ for _ in 0..5 {+ tokio::task::yield_now().await;+ }+ Ok(())+ })+ .await??;++ // The host task was cancelled, nothing should remain.+ store.assert_concurrent_state_empty();++ Ok(())+}++#[tokio::test]+#[cfg_attr(miri, ignore)]+async fn sync_lower_async_host_does_not_leak() -> Result<()> {+ let mut config = Config::new();+ config.wasm_component_model_async(true);+ let engine = Engine::new(&config)?;++ let mut store = Store::new(&engine, 0);+ let component = Component::new(+ &engine,+ r#"(component+ (import "f" (func $f async))++ (core module $m+ (import "" "f" (func $f))+ (func (export "run")+ (local $c i32)++ ;; call the host 100 times+ loop+ call $f+ (local.tee $c (i32.add (local.get $c) (i32.const 1)))+ i32.const 100+ i32.ne+ if br 1 end+ end+ )+ )+ (core func $f (canon lower (func $f) ))+ (core instance $i (instantiate $m+ (with "" (instance+ (export "f" (func $f))+ ))+ ))++ (func (export "f") async+ (canon lift (core func $i "run")))+++ )"#,+ )?;++ let mut linker = Linker::<usize>::new(&engine);+ linker+ .root()+ .func_wrap_concurrent("f", |accessor, (): ()| {+ Box::pin(async move {+ // Ensure that this doesn't hit the fast path of "ready on+ // first poll"+ for _ in 0..5 {+ tokio::task::yield_now().await;+ }++ // Keep track of the maximum size of the table in+ // concurrent_state.+ accessor.with(|mut s| {+ let cur = s.as_context_mut().concurrent_state_table_size();+ let max = s.data_mut();+ *max = (*max).max(cur);+ });+ Ok(())+ })+ })?;+ let instance = linker.instantiate_async(&mut store, &component).await?;+ let func = instance.get_typed_func::<(), ()>(&mut store, "f")?;+ func.call_async(&mut store, ()).await?;++ // First-level assertion: nothing should remain after the guest has exited.+ store.assert_concurrent_state_empty();++ // Second-level assertion: state should be incrementally cleaned up along+ // the way as the guest calls the host. Things shouldn't leak until the+ // guest exits at the end, for example.+ assert!(+ *store.data() < 100,+ "the store peaked at over 100 items in the concurrent table which \+ indicates that something isn't getting cleaned up between executions \+ of the host"+ );++ Ok(())+}diff --git a/tests/all/component_model/resources.rs b/tests/all/component_model/resources.rsindex 80fce005d6..dd57c35618 100644--- a/tests/all/component_model/resources.rs+++ b/tests/all/component_model/resources.rs@@ -2,7 +2,7 @@use wasmtime::Result;use wasmtime::component::*;-use wasmtime::{Store, Trap};+use wasmtime::{Config, Engine, Store, Trap};#[test]fn host_resource_types() -> Result<()> {@@ -1626,3 +1626,101 @@ fn intrinsic_trampolines() -> Result<()> {assert!(rep.call(&mut store, (r,)).is_err());Ok(())}++#[tokio::test]+async fn drop_after_sync_lowered_async_host_function() -> Result<()> {+ let mut config = Config::new();+ config.wasm_component_model_async(true);+ let engine = Engine::new(&config)?;+ let mut store = Store::new(&engine, ());+ let mut linker = Linker::new(&engine);+ let c = Component::new(+ &engine,+ r#"+(component+ (import "r" (type $r (sub resource)))+ ;;(import "f" (func $f async (param "x" (borrow $r)) (param "y" (borrow $r))))+ ;;(import "n" (func $n (result (own $r))))+ (import "f" (func $f async))+++ (component $A+ (import "f" (func $f async))+ (core module $m+ (import "" "f" (func $f))+ (func (export "call") (result i32) call $f i32.const 0)+ (func (export "cb") (param i32 i32 i32) (result i32) unreachable)+ )+ (core func $f (canon lower (func $f)))+ (core instance $i (instantiate $m+ (with "" (instance (export "f" (func $f))))+ ))+ (func (export "call") async+ (canon lift (core func $i "call")+ async+ (callback (func $i "cb"))))+ )++ (component $B+ (import "r" (type $r (sub resource)))+ (import "f" (func $f async))+ (core module $m+ (import "" "f" (func $f (result i32)))+ (import "" "r" (func $r))+ (import "" "drop" (func $drop (param i32)))+ (func (export "call") (param i32) (result i32)+ ;; Call into component `$A` which is an async-lifted function. This+ ;; function will call a sync-lowered async function from the host. In+ ;; effect this means that the task started here by calling `$f` is+ ;; blocked.+ call $f+ drop ;; ignore the `SUBTASK_STARTED` status code++ ;; Now drop our borrow that this function was provided. This is+ ;; entirely disconnected from `$A` and it should be using the borrow+ ;; tracking of this task, no other task.+ (call $drop (local.get 0))++ ;; Now finish up this task without actually waiting on `$A`.+ call $r+ i32.const 0+ )+ (func (export "cb") (param i32 i32 i32) (result i32) unreachable)+ )+ (core func $f (canon lower (func $f) async))+ (core func $r (canon task.return))+ (core func $drop (canon resource.drop $r))+ (core instance $i (instantiate $m+ (with "" (instance+ (export "f" (func $f))+ (export "r" (func $r))+ (export "drop" (func $drop))+ ))+ ))+ (func (export "call") async (param "x" (borrow $r))+ (canon lift (core func $i "call")+ async+ (callback (func $i "cb"))))+ )+ (instance $a (instantiate $A (with "f" (func $f))))+ (instance $b (instantiate $B+ (with "f" (func $a "call"))+ (with "r" (type $r))+))+ (export "call" (func $b "call"))+)+"#,+ )?;++ let mut root = linker.root();+ root.resource("r", ResourceType::host::<u32>(), |_, _| Ok(()))?;+ root.func_wrap_concurrent("f", |_accessor, ()| {+ Box::pin(async move { std::future::pending::<Result<()>>().await })+ })?;++ let i = linker.instantiate_async(&mut store, &c).await?;+ let call = i.get_typed_func::<(Resource<u32>,), ()>(&mut store, "call")?;+ let r = Resource::new_borrow(3);+ call.call_async(&mut store, (r,)).await?;+ Ok(())+}