A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
component: protect fiber handoffs without adding thread states (#14620)
B. Naut504e231a77d2
diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex f4cb1f6572..ca903081b6 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -70,6 +70,7 @@ use crate::vm::{use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};use crate::{Instance as ModuleInstance, bail_bug};use alloc::borrow::ToOwned;+use alloc::collections::btree_map::Entry;use alloc::collections::{BTreeMap, BTreeSet, VecDeque};use core::any::Any;use core::cell::UnsafeCell;@@ -843,6 +844,24 @@ enum WorkItem {WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),}+/// A fiber temporarily outside the store's scheduler state.+///+/// Construct this before returning a future so cancellation before its first+/// poll also disposes of the fiber. Successful handoffs take the fiber only+/// after all fallible destination checks have completed.+struct ResumingFiber<'a> {+ store: &'a mut StoreOpaque,+ fiber: Option<StoreFiber<'static>>,+}++impl Drop for ResumingFiber<'_> {+ fn drop(&mut self) {+ if let Some(mut fiber) = self.fiber.take() {+ fiber.dispose(self.store);+ }+ }+}+impl fmt::Debug for WorkItem {fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {match self {@@ -1611,88 +1630,111 @@ impl<T> StoreContextMut<'_, T> {}/// Handle the specified work item, possibly resuming a fiber if applicable.- async fn handle_work_item(self, item: WorkItem) -> Result<()> {- log::trace!("handle work item {item:?}");- match item {- WorkItem::PushFuture(future) => {- self.0- .concurrent_state_mut()?- .futures_mut()?- .push(future.into_inner());- }- WorkItem::ResumeFiber { fiber, .. } => {- self.0.resume_fiber(fiber).await?;- }- WorkItem::ResumeThread { thread, .. } => {- if let GuestThreadState::Ready { fiber, .. } = mem::replace(- &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state,- GuestThreadState::Running,- ) {- self.0.resume_fiber(fiber).await?;- } else {- bail_bug!("cannot resume non-pending thread {thread:?}");- }+ fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {+ // Construct the guarded future before returning, even when the caller+ // never polls this work item. Other variants keep their fibers in the+ // store until their first poll.+ let item = match item {+ WorkItem::ResumeFiber {+ instance,+ thread,+ fiber,+ } => {+ let future = self.0.resume_fiber(fiber);+ log::trace!(+ "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"+ );+ return futures::future::Either::Left(future);}- WorkItem::GuestCall { call, .. } => {- if call.is_ready(self.0)? {+ item => item,+ };+ futures::future::Either::Right(async move {+ log::trace!("handle work item {item:?}");+ match item {+ WorkItem::PushFuture(future) => {self.0.concurrent_state_mut()?- .get_mut(call.thread.thread)?- .wake_on_cancel = WakeOnCancel::None;- self.run_on_worker(WorkerItem::GuestCall(call)).await?;- } else {- let state = self.0.concurrent_state_mut()?;- let task = state.get_mut(call.thread.task)?;- if !task.starting_sent {- task.starting_sent = true;- if let GuestCallKind::StartImplicit(_) = &call.kind {- Waitable::Guest(call.thread.task).set_event(- state,- Some(Event::Subtask {- status: Status::Starting,- }),- )?;- }+ .futures_mut()?+ .push(future.into_inner());+ }+ WorkItem::ResumeFiber { .. } => unreachable!(),+ WorkItem::ResumeThread { thread, .. } => {+ let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;+ if !matches!(slot, GuestThreadState::Ready { .. }) {+ bail_bug!("cannot resume non-pending thread {thread:?}");}-- let instance = state.get_mut(call.thread.task)?.instance;- // If this call is for a callback-lifted task waiting on a- // waitable set, it can't run until `instance` is enterable- // again. Meanwhile, the set may have an event which another- // thread waiting on it could take, and that thread may- // even be the reason `instance` isn't enterable, so wake- // the next waiter up as well if there's still an event- // for it. Whichever thread runs first takes the event, and- // the other goes back to waiting if none is left.- if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {+ let GuestThreadState::Ready { fiber } =+ mem::replace(slot, GuestThreadState::Running)+ else {+ unreachable!()+ };+ self.0.resume_fiber(fiber).await?;+ }+ WorkItem::GuestCall { call, .. } => {+ if call.is_ready(self.0)? {+ self.0+ .concurrent_state_mut()?+ .get_mut(call.thread.thread)?+ .wake_on_cancel = WakeOnCancel::None;+ self.run_on_worker(WorkerItem::GuestCall(call)).await?;+ } else {let state = self.0.concurrent_state_mut()?;- if !state.get_mut(*set)?.ready.is_empty() {- state.wake_waiter(*set)?;+ let task = state.get_mut(call.thread.task)?;+ if !task.starting_sent {+ task.starting_sent = true;+ if let GuestCallKind::StartImplicit(_) = &call.kind {+ Waitable::Guest(call.thread.task).set_event(+ state,+ Some(Event::Subtask {+ status: Status::Starting,+ }),+ )?;+ }+ }++ let instance = state.get_mut(call.thread.task)?.instance;+ // If this call is for a callback-lifted task waiting on a+ // waitable set, it can't run until `instance` is enterable+ // again. Meanwhile, the set may have an event which another+ // thread waiting on it could take, and that thread may+ // even be the reason `instance` isn't enterable, so wake+ // the next waiter up as well if there's still an event+ // for it. Whichever thread runs first takes the event, and+ // the other goes back to waiting if none is left.+ if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {+ let state = self.0.concurrent_state_mut()?;+ if !state.get_mut(*set)?.ready.is_empty() {+ state.wake_waiter(*set)?;+ }}+ self.0+ .instance_state(instance)+ .concurrent_state()+ .pending+ .insert(call.thread, call.kind);++ // Switch back to the caller (or canceller) immediately if+ // applicable since we aren't yet able to run the subtask it+ // yielded to.+ self.0.concurrent_state_mut()?.take_next_switch_item()?;}- self.0- .instance_state(instance)- .concurrent_state()- .pending- .insert(call.thread, call.kind);-- // Switch back to the caller (or canceller) immediately if- // applicable since we aren't yet able to run the subtask it- // yielded to.- self.0.concurrent_state_mut()?.take_next_switch_item()?;+ }+ WorkItem::WorkerFunction(fun) => {+ self.run_on_worker(WorkerItem::Function(fun)).await?;}}- WorkItem::WorkerFunction(fun) => {- self.run_on_worker(WorkerItem::Function(fun)).await?;- }- }- Ok(())+ Ok(())+ })}/// Execute the specified guest call on a worker fiber.async fn run_on_worker(self, item: WorkerItem) -> Result<()> {- let worker = if let Some(fiber) = self.0.concurrent_state_mut()?.worker.take() {+ let state = self.0.concurrent_state_mut()?;+ if state.worker_item.is_some() {+ bail_bug!("worker item already set when taking worker fiber");+ }+ let worker = if let Some(fiber) = state.worker.take() {fiber} else {// SAFETY: the `make_fiber_unchecked` function is unsafe because the@@ -1730,9 +1772,11 @@ impl<T> StoreContextMut<'_, T> {}};- let worker_item = &mut self.0.concurrent_state_mut()?.worker_item;- assert!(worker_item.is_none());- *worker_item = Some(item);+ // No fallible operation between taking the fiber and guarding it in+ // resume_fiber; the current thread was forced before the take.+ self.0+ .concurrent_state_mut_already_forced_current_thread()+ .worker_item = Some(item);self.0.resume_fiber(worker).await}@@ -2289,82 +2333,99 @@ impl StoreOpaque {/// Resume the specified fiber, giving it exclusive access to the specified/// store.- async fn resume_fiber(&mut self, fiber: StoreFiber<'static>) -> Result<()> {- let old_thread = self.current_thread()?;- log::trace!("resume_fiber: save current thread {old_thread:?}");-- let fiber = fiber::resolve_or_release(self, fiber).await?;+ fn resume_fiber(+ &mut self,+ fiber: StoreFiber<'static>,+ ) -> impl Future<Output = Result<()>> + '_ {+ let mut resuming = ResumingFiber {+ store: self,+ fiber: Some(fiber),+ };+ async move {+ let old_thread = resuming.store.current_thread()?;+ log::trace!("resume_fiber: save current thread {old_thread:?}");- self.set_thread(old_thread)?;+ let fiber = resuming.fiber.take().unwrap();+ resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;- let state = self.concurrent_state_mut()?;+ resuming.store.set_thread(old_thread)?;- if let Some(ot) = old_thread.guest() {- state.get_mut(ot.thread)?.state = GuestThreadState::Running;- }- log::trace!("resume_fiber: restore current thread {old_thread:?}");+ let state = resuming.store.concurrent_state_mut()?;- if let Some(mut fiber) = fiber {- log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);- // See the `SuspendReason` documentation for what each case means.- let reason = match state.suspend_reason.take() {- Some(r) => r,- None => bail_bug!("suspend reason missing when resuming fiber"),- };- match reason {- SuspendReason::NeedWork => {- if state.worker.is_none() {- state.worker = Some(fiber);- } else {- fiber.dispose(self);+ if let Some(ot) = old_thread.guest() {+ state.set_thread_running(ot.thread)?;+ }+ log::trace!("resume_fiber: restore current thread {old_thread:?}");++ if resuming.fiber.is_some() {+ log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);+ // See the `SuspendReason` documentation for what each case means.+ let reason = match state.suspend_reason.take() {+ Some(r) => r,+ None => bail_bug!("suspend reason missing when resuming fiber"),+ };+ match reason {+ SuspendReason::NeedWork => {+ if state.worker.is_none() {+ state.worker = resuming.fiber.take();+ }}- }- SuspendReason::Yielding { thread } => {- state.get_mut(thread.thread)?.state = GuestThreadState::Ready { fiber };- let instance = state.get_mut(thread.task)?.instance;- state.push_low_priority(WorkItem::ResumeThread { instance, thread });- }- SuspendReason::ExplicitlySuspending { thread } => {- state.get_mut(thread.thread)?.state = GuestThreadState::Suspended(fiber);- }- SuspendReason::Waiting { set, thread } => {- let old = state- .get_mut(set)?- .waiting- .insert(thread, WaitMode::Fiber(fiber));- assert!(old.is_none());- }- SuspendReason::YieldingToSubtask { thread } => {- // In this case, the thread has either invoked or sent a- // cancel request to a subtask, and is now yielding to that- // subtask. According to the CM spec, that subtask may only- // yield back to the original thread the first time it- // suspends or exits (or a thread that it has resumed- // suspends or exits, etc.), which we ensure by setting- // `ConcurrentState::next_switch_item` here.-- let item = WorkItem::ResumeFiber {- instance: state.get_mut(thread.task)?.instance,- thread,- fiber,- };-- log::trace!("set next switch item to {item:?}");- if state.next_switch_item.replace(item).is_some() {- // This should be unreachable per the save/restore code- // in `Self::suspend`.- bail_bug!(- "`ConcurrentState::next_switch_item` was already `Some(_)` when \+ SuspendReason::Yielding { thread } => {+ let slot = &mut state.get_mut(thread.thread)?.state;+ slot.check_no_fiber()?;+ *slot = GuestThreadState::Ready {+ fiber: resuming.fiber.take().unwrap(),+ };+ let instance = state.get_mut(thread.task)?.instance;+ state.push_low_priority(WorkItem::ResumeThread { instance, thread });+ }+ SuspendReason::ExplicitlySuspending { thread } => {+ let slot = &mut state.get_mut(thread.thread)?.state;+ slot.check_no_fiber()?;+ *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());+ }+ SuspendReason::Waiting { set, thread } => {+ match state.get_mut(set)?.waiting.entry(thread) {+ Entry::Vacant(entry) => {+ entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));+ }+ Entry::Occupied(_) => {+ bail_bug!("entry unexpectedly already exists for {thread:?}");+ }+ }+ }+ SuspendReason::YieldingToSubtask { thread } => {+ // In this case, the thread has either invoked or sent a+ // cancel request to a subtask, and is now yielding to that+ // subtask. According to the CM spec, that subtask may only+ // yield back to the original thread the first time it+ // suspends or exits (or a thread that it has resumed+ // suspends or exits, etc.), which we ensure by setting+ // `ConcurrentState::next_switch_item` here.++ let instance = state.get_mut(thread.task)?.instance;+ if state.next_switch_item.is_some() {+ // This should be unreachable per the save/restore code+ // in `Self::suspend`.+ bail_bug!(+ "`ConcurrentState::next_switch_item` was already `Some(_)` when \a thread wanted to wait on a subtask"- );+ );+ }+ let item = state.next_switch_item.insert(WorkItem::ResumeFiber {+ instance,+ thread,+ fiber: resuming.fiber.take().unwrap(),+ });+ log::trace!("set next switch item to {item:?}");}- }- };- } else {- log::trace!("resume_fiber: fiber has exited");- }+ };+ } else {+ log::trace!("resume_fiber: fiber has exited");+ }- Ok(())+ Ok(())+ }}/// Suspend the current fiber, storing the reason in@@ -2400,11 +2461,7 @@ impl StoreOpaque {};let old_next_switch_item = if save_and_restore_next_switch_item {- let item = state.next_switch_item.take();- // Note that we store it in the table here rather than directly in a- // local variable to ensure the fiber is disposed of properly if we- // end up trapping or panicking.- Some(state.push(item)?)+ Some(state.save_next_switch_item()?)} else {None};@@ -2448,7 +2505,7 @@ impl StoreOpaque {if let Some(item) = old_next_switch_item {let state = self.concurrent_state_mut()?;- state.next_switch_item = state.delete(item)?;+ state.restore_next_switch_item(item)?;}Ok(())@@ -2944,8 +3001,7 @@ impl Instance {store.concurrent_state_mut()?- .get_mut(guest_thread.thread)?- .state = GuestThreadState::Running;+ .set_thread_running(guest_thread.thread)?;let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;let lower = match task.lower_params.take() {Some(l) => l,@@ -3023,7 +3079,7 @@ impl Instance {store.set_thread(old_thread)?;let state = store.concurrent_state_mut()?;if let Some(t) = old_thread.guest() {- state.get_mut(t.thread)?.state = GuestThreadState::Running;+ state.set_thread_running(t.thread)?;}log::trace!("stackless call: restored {old_thread:?} as current thread");@@ -3580,8 +3636,7 @@ impl Instance {store.0.concurrent_state_mut()?- .get_mut(caller.thread)?- .state = GuestThreadState::Running;+ .set_thread_running(caller.thread)?;log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");if async_caller {@@ -4153,7 +4208,7 @@ impl Instance {log::trace!("explicit thread {guest_thread:?} completed");let state = store.0.concurrent_state_mut()?;if let Some(t) = old_thread.guest() {- state.get_mut(t.thread)?.state = GuestThreadState::Running;+ state.set_thread_running(t.thread)?;}log::trace!("thread start: restored {old_thread:?} as current thread");@@ -4241,10 +4296,18 @@ impl Instance {// When resuming a thread it must be in a suspended state otherwise// this operation is a trap.match &thread.state {- GuestThreadState::NotStartedExplicit(_) | GuestThreadState::Suspended(_) => {}+ GuestThreadState::NotStartedExplicit(_) => {}+ GuestThreadState::Suspended(_) => {+ log::trace!("resuming thread {thread_id:?} that was suspended");+ state.schedule_suspended_thread(+ guest_thread,+ self.runtime_instance(runtime_instance),+ priority,+ )?;+ return Ok(true);+ }_ => bail!(Trap::CannotResumeThread),}-match mem::replace(&mut thread.state, GuestThreadState::Running) {GuestThreadState::NotStartedExplicit(start_func) => {log::trace!("starting thread {guest_thread:?}");@@ -4257,20 +4320,7 @@ impl Instance {})),},};- store- .concurrent_state_mut()?- .push_work_item(guest_call, priority)?;- }- GuestThreadState::Suspended(fiber) => {- log::trace!("resuming thread {thread_id:?} that was suspended");- store.concurrent_state_mut()?.push_work_item(- WorkItem::ResumeFiber {- instance: self.runtime_instance(runtime_instance),- thread: guest_thread,- fiber,- },- priority,- )?;+ state.push_work_item(guest_call, priority)?;}other => {thread.state = other;@@ -4554,21 +4604,16 @@ impl Instance {match thread_mut.wake_on_cancel.take() {WakeOnCancel::Waiting(set) => {// The thread is in a cancellable wait, so wake it up:- let item = match concurrent_state.get_mut(set)?.waiting.remove(&thread)- {- Some(WaitMode::Callback(instance)) => WorkItem::GuestCall {- instance: runtime_instance,- call: GuestCall {- thread,- kind: GuestCallKind::DeliverEvent {- instance,- set: Some(set),- },+ let instance = concurrent_state.take_callback_waiter(set, thread)?;+ let item = WorkItem::GuestCall {+ instance: runtime_instance,+ call: GuestCall {+ thread,+ kind: GuestCallKind::DeliverEvent {+ instance,+ set: Some(set),},},- other => bail_bug!(- "expected `Some(WaitMode::Callback(_))`; got `{other:?}`"- ),};concurrent_state.set_switch_item(item)?;@@ -4628,21 +4673,14 @@ impl Instance {// Save and later restore `next_switch_item` during a sync cancel so// we don't try to switch to it while blocking.- let old_next_switch_item = {- let state = store.concurrent_state_mut()?;- let item = state.next_switch_item.take();- // Note that we store it in the table here rather than directly- // in a local variable to ensure the fiber is disposed of- // properly if we end up trapping or panicking.- state.push(item)?- };+ let old_next_switch_item = store.concurrent_state_mut()?.save_next_switch_item()?;// Wait for this waitable to get signaled with its terminal// status. Once that's done fall through to the shared code.store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;let state = store.concurrent_state_mut()?;- state.next_switch_item = state.delete(old_next_switch_item)?;+ state.restore_next_switch_item(old_next_switch_item)?;// .. fall through to determine what event's in store for us.}@@ -5270,6 +5308,18 @@ enum GuestThreadState {Completed,}+impl GuestThreadState {+ fn check_no_fiber(&self) -> Result<()> {+ match self {+ Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),+ Self::NotStartedImplicit+ | Self::NotStartedExplicit(_)+ | Self::Running+ | Self::Completed => Ok(()),+ }+ }+}+impl fmt::Debug for GuestThreadState {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {match self {@@ -6031,8 +6081,30 @@ impl ConcurrentState {fibers: &mut Vec<StoreFiber<'static>>,futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,) {+ let ConcurrentState {+ table,+ worker,+ switch_item,+ next_switch_item,+ high_priority,+ low_priority,+ saved_next_switch_items,+ futures: pending_futures,+ worker_item: _,+ unforced_current_thread: _,+ deferred_host_call_context: _,+ suspend_reason: _,+ global_error_context_ref_counts: _,+ interesting_tasks: _,+ interesting_tasks_empty_waker: _,+ ready_for_concurrent_call_waker: _,+ event_loop_running: _,+ #[cfg(feature = "task-group-hook")]+ task_group_hook: _,+ } = self;+let mut items = Vec::new();- for (_, entry) in self.table.get_mut().iter_mut() {+ for (_, entry) in table.get_mut().iter_mut() {if let Some(set) = entry.downcast_mut::<WaitableSet>() {for mode in mem::take(&mut set.waiting).into_values() {match mode {@@ -6043,10 +6115,14 @@ impl ConcurrentState {}}} else if let Some(thread) = entry.downcast_mut::<GuestThread>() {- if let GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber, .. } =- mem::replace(&mut thread.state, GuestThreadState::Completed)- {- fibers.push(fiber);+ match mem::replace(&mut thread.state, GuestThreadState::Completed) {+ GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {+ fibers.push(fiber);+ }+ GuestThreadState::NotStartedImplicit+ | GuestThreadState::NotStartedExplicit(_)+ | GuestThreadState::Running+ | GuestThreadState::Completed => {}}} else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {if let Some(item) = item.take() {@@ -6055,7 +6131,7 @@ impl ConcurrentState {}}- if let Some(fiber) = self.worker.take() {+ if let Some(fiber) = worker.take() {fibers.push(fiber);}@@ -6064,7 +6140,7 @@ impl ConcurrentState {fibers.push(fiber);}WorkItem::PushFuture(future) => {- self.futures+ pending_futures.get_mut().as_mut().unwrap()@@ -6078,26 +6154,26 @@ impl ConcurrentState {for item in items {handle_item(item);}- if let Some(item) = self.switch_item.take() {+ if let Some(item) = switch_item.take() {handle_item(item);}- if let Some(item) = self.next_switch_item.take() {+ if let Some(item) = next_switch_item.take() {handle_item(item);}- for item in mem::take(&mut self.high_priority) {+ for item in mem::take(high_priority) {handle_item(item);}- for item in mem::take(&mut self.low_priority) {+ for item in mem::take(low_priority) {handle_item(item);}- for item in mem::take(&mut self.saved_next_switch_items)+ for item in mem::take(saved_next_switch_items).into_iter().filter_map(|v| v){handle_item(item);}- if let Some(them) = self.futures.get_mut().take() {+ if let Some(them) = pending_futures.get_mut().take() {futures.push(them);}}@@ -6148,10 +6224,14 @@ impl ConcurrentState {}}} else if let Some(thread) = entry.downcast_mut::<GuestThread>() {- if let GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber, .. } =- &mut thread.state- {- fiber.trace_gc_roots(modules, unwind, gc_roots_list);+ match &mut thread.state {+ GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {+ fiber.trace_gc_roots(modules, unwind, gc_roots_list);+ }+ GuestThreadState::NotStartedImplicit+ | GuestThreadState::NotStartedExplicit(_)+ | GuestThreadState::Running+ | GuestThreadState::Completed => {}}} else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =entry.downcast_mut::<Option<WorkItem>>()@@ -6197,6 +6277,75 @@ impl ConcurrentState {}}+ fn schedule_suspended_thread(+ &mut self,+ thread: QualifiedThreadId,+ instance: RuntimeInstance,+ priority: Priority,+ ) -> Result<()> {+ // Validate the destination while the source still owns the fiber.+ if matches!(priority, Priority::Switch) && self.switch_item.is_some() {+ bail_bug!("switch item already set");+ }+ let slot = &mut self.get_mut(thread.thread)?.state;+ if !matches!(slot, GuestThreadState::Suspended(_)) {+ bail!(Trap::CannotResumeThread);+ }+ let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)+ else {+ unreachable!()+ };+ let item = WorkItem::ResumeFiber {+ instance,+ thread,+ fiber,+ };+ match priority {+ Priority::Switch => self.switch_item = Some(item),+ Priority::High => self.push_high_priority(item),+ Priority::Low => self.push_low_priority(item),+ }+ Ok(())+ }++ fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {+ let slot = &mut self.get_mut(thread)?.state;+ slot.check_no_fiber()?;+ *slot = GuestThreadState::Running;+ Ok(())+ }++ fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {+ // push can consume and drop its argument when the table is full.+ // Reserve with an empty item before taking the fiber-owning source.+ let saved = self.push(None::<WorkItem>)?;+ let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;+ *slot = self.next_switch_item.take();+ Ok(saved)+ }++ fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {+ if self.next_switch_item.is_some() {+ bail_bug!("next switch item already set when restoring");+ }+ self.next_switch_item = self.delete(saved)?;+ Ok(())+ }++ fn take_callback_waiter(+ &mut self,+ set: TableId<WaitableSet>,+ thread: QualifiedThreadId,+ ) -> Result<Instance> {+ let waiting = &mut self.get_mut(set)?.waiting;+ let instance = match waiting.get(&thread) {+ Some(WaitMode::Callback(instance)) => *instance,+ other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),+ };+ waiting.remove(&thread);+ Ok(instance)+ }+fn push<V: Send + Sync + 'static>(&mut self,value: V,@@ -6243,13 +6392,15 @@ impl ConcurrentState {}fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {- log::trace!("set switch item: {item:?}");-if self.switch_item.is_some() {+ // Preserve ownership on this internal-error path. The caller+ // must stop scheduling after the error; teardown disposes the item.+ self.push_high_priority(item);bail_bug!("switch item already set");}self.switch_item = Some(item);+ log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());Ok(())}@@ -6264,13 +6415,19 @@ impl ConcurrentState {/// Wake the first thread waiting on `set`, if any, so that it may receive/// one of the set's pending events.fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {- let Some((thread, mode)) = self.get_mut(set)?.waiting.pop_first() else {+ // Finish every fallible lookup and check before taking the waiter out+ // of the set, so that an error leaves its fiber owned by the set.+ let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {return Ok(());};- let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel.take();- assert!(wake_on_cancel.is_none() || wake_on_cancel == WakeOnCancel::Waiting(set));-+ let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;+ if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {+ bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");+ }let instance = self.get_mut(thread.task)?.instance;+ self.get_mut(thread.thread)?.wake_on_cancel.take();+ let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();+let item = match mode {WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {instance,@@ -6293,13 +6450,19 @@ impl ConcurrentState {}fn push_high_priority(&mut self, item: WorkItem) {- log::trace!("push high priority: {item:?}");self.high_priority.push_front(item);+ log::trace!(+ "push high priority: {:?}",+ self.high_priority.front().unwrap()+ );}fn push_low_priority(&mut self, item: WorkItem) {- log::trace!("push low priority: {item:?}");self.low_priority.push_front(item);+ log::trace!(+ "push low priority: {:?}",+ self.low_priority.front().unwrap()+ );}fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {@@ -6353,28 +6516,22 @@ impl ConcurrentState {whereF: FnMut(&WorkItem) -> bool,{- // Note the use of `.rev()` below to preserve ordering given that items+ // Scan from the back to preserve the ordering of items// are popped from the back of the `VecDeque`s by `poll_until` and// pushed to the front by `push_{high,low}_priority`.- for item in mem::take(&mut self.high_priority).into_iter().rev() {- if self.switch_item.is_none() && predicate(&item) {- self.set_switch_item(item)?;- } else {- self.push_high_priority(item);+ // Inspect items while they are still owned by the store. A predicate+ // (including tracing) may panic; it must not drop an extracted queue.+ if self.switch_item.is_none() {+ if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {+ self.switch_item = self.high_priority.remove(index);}}-if self.switch_item.is_none() {- for item in mem::take(&mut self.low_priority).into_iter().rev() {- if self.switch_item.is_none() && predicate(&item) {- self.set_switch_item(item)?;- } else {- self.push_low_priority(item);- }+ if let Some(index) = self.low_priority.iter().rposition(predicate) {+ self.switch_item = self.low_priority.remove(index);}}-Ok(self.switch_item.is_some())}@@ -6894,3 +7051,6 @@ mod tests {Ok(())}}++#[cfg(test)]+mod fiber_ownership_tests;diff --git a/crates/wasmtime/src/runtime/component/concurrent/fiber_ownership_tests.rs b/crates/wasmtime/src/runtime/component/concurrent/fiber_ownership_tests.rsnew file mode 100644index 0000000000..a11305f4dd--- /dev/null+++ b/crates/wasmtime/src/runtime/component/concurrent/fiber_ownership_tests.rs@@ -0,0 +1,913 @@+use super::*;+use std::sync::Arc;+use std::sync::atomic::{AtomicBool, Ordering};++struct Exited(Arc<AtomicBool>);++impl Drop for Exited {+ fn drop(&mut self) {+ self.0.store(true, Ordering::SeqCst);+ }+}++#[test]+#[cfg_attr(miri, ignore)]+fn failed_waiter_wakeup_restores_store_ownership() {+ let mut config = crate::Config::new();+ config.wasm_component_model_async(true);+ let engine = crate::Engine::new(&config).unwrap();+ let mut store = Store::new(&engine, ());+ let store = store.as_context_mut().0;++ let fiber = unsafe { fiber::make_fiber_unchecked(store, |_| Ok(())) }.unwrap();+ let state = store.concurrent_state_mut().unwrap();+ let set = state.push(WaitableSet::default()).unwrap();+ let group = state.make_task_group().unwrap();+ let host = state+ .push(HostTask {+ common: WaitableCommon::default(),+ call_context: CallContext::default(),+ state: HostTaskState::CalleeStarted,+ group,+ })+ .unwrap();+ let thread = QualifiedThreadId {+ task: TableId::new(u32::MAX),+ thread: TableId::new(u32::MAX),+ };+ state.get_mut(host).unwrap().common.set = Some(set);+ state+ .get_mut(set)+ .unwrap()+ .waiting+ .insert(thread, WaitMode::Fiber(fiber));++ let error = Waitable::Host(host).mark_ready(state).unwrap_err();+ assert!(matches!(+ error.downcast_ref::<ResourceTableError>(),+ Some(ResourceTableError::NotPresent)+ ));+ assert!(state.get_mut(set).unwrap().waiting.contains_key(&thread));+ // Dropping the store disposes the restored live fiber.+}++#[test]+#[cfg_attr(miri, ignore)]+fn invalid_waiting_set_returns_error_without_dropping_live_fiber() {+ let mut config = crate::Config::new();+ config.wasm_component_model_async(true);+ let engine = crate::Engine::new(&config).unwrap();+ let mut store = Store::new(&engine, ());+ let store = store.as_context_mut().0;+ let exited = Arc::new(AtomicBool::new(false));+ let fiber_exited = exited.clone();++ // A suspended fiber names a waitable set that does not exist. The+ // destination lookup must return its original table error while the+ // live fiber is disposed using the store.+ let fiber = unsafe {+ fiber::make_fiber_unchecked(store, move |store| {+ let _exited = Exited(fiber_exited);+ store.concurrent_state_mut()?.suspend_reason = Some(SuspendReason::Waiting {+ set: TableId::new(u32::MAX),+ thread: QualifiedThreadId {+ task: TableId::new(u32::MAX),+ thread: TableId::new(u32::MAX),+ },+ });+ store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ Ok(())+ })+ }+ .unwrap();++ let result = {+ let mut future = Box::pin(store.resume_fiber(fiber));+ future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ };+ let Poll::Ready(Err(error)) = result else {+ panic!("expected resource table error from invalid waitable set");+ };+ assert!(matches!(+ error.downcast_ref::<ResourceTableError>(),+ Some(ResourceTableError::NotPresent)+ ));+ assert!(exited.load(Ordering::SeqCst));+}++// Exercise each suspend destination with an invalid thread handle. The+// returned table error and the exit marker show that a live fiber did not+// escape both store ownership and disposal.+fn invalid_thread_suspend_returns_original_error(reason: fn(QualifiedThreadId) -> SuspendReason) {+ let mut config = crate::Config::new();+ config.wasm_component_model_async(true);+ let engine = crate::Engine::new(&config).unwrap();+ let mut store = Store::new(&engine, ());+ let store = store.as_context_mut().0;+ let exited = Arc::new(AtomicBool::new(false));+ let fiber_exited = exited.clone();++ let fiber = unsafe {+ fiber::make_fiber_unchecked(store, move |store| {+ let _exited = Exited(fiber_exited);+ store.concurrent_state_mut()?.suspend_reason = Some(reason(QualifiedThreadId {+ task: TableId::new(u32::MAX),+ thread: TableId::new(u32::MAX),+ }));+ store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ Ok(())+ })+ }+ .unwrap();++ let result = {+ let mut future = Box::pin(store.resume_fiber(fiber));+ future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ };+ let Poll::Ready(Err(error)) = result else {+ panic!("expected resource table error from invalid thread");+ };+ assert!(matches!(+ error.downcast_ref::<ResourceTableError>(),+ Some(ResourceTableError::NotPresent)+ ));+ assert!(exited.load(Ordering::SeqCst));+}++#[test]+#[cfg_attr(miri, ignore)]+fn occupied_switch_keeps_incoming_fiber_in_store() {+ use std::panic::{AssertUnwindSafe, catch_unwind};++ let mut config = crate::Config::new();+ config.wasm_component_model_async(true);+ let engine = crate::Engine::new(&config).unwrap();+ let mut store = Store::new(&engine, ());+ let store = store.as_context_mut().0;+ let fiber = unsafe { fiber::make_fiber_unchecked(store, |_| Ok(())) }.unwrap();+ let instance = RuntimeInstance {+ instance: crate::component::ComponentInstanceId::from_u32(0),+ index: RuntimeComponentInstanceIndex::from_u32(0),+ };+ let thread = QualifiedThreadId {+ task: TableId::new(u32::MAX),+ thread: TableId::new(u32::MAX),+ };+ let state = store.concurrent_state_mut().unwrap();+ state.switch_item = Some(WorkItem::ResumeThread { instance, thread });++ let result = catch_unwind(AssertUnwindSafe(|| {+ state.push_work_item(+ WorkItem::ResumeFiber {+ instance,+ thread,+ fiber,+ },+ Priority::Switch,+ )+ }));+ if cfg!(debug_assertions) {+ let panic = result.expect_err("debug builds must keep the original bail_bug panic");+ let message = panic+ .downcast_ref::<String>()+ .map(String::as_str)+ .or_else(|| panic.downcast_ref::<&str>().copied())+ .unwrap();+ assert!(message.contains("BUG: switch item already set"));+ } else {+ let error = result.unwrap().unwrap_err();+ assert!(error.is::<crate::WasmtimeBug>());+ assert!(error.to_string().contains("switch item already set"));+ }+ assert_eq!(state.high_priority.len(), 1);+ // The store disposes the fiber retained in its queue.+}++#[test]+#[cfg_attr(miri, ignore)]+fn yielding_invalid_thread_returns_table_error() {+ invalid_thread_suspend_returns_original_error(|thread| SuspendReason::Yielding { thread });+}++#[test]+#[cfg_attr(miri, ignore)]+fn explicitly_suspending_invalid_thread_returns_table_error() {+ invalid_thread_suspend_returns_original_error(|thread| SuspendReason::ExplicitlySuspending {+ thread,+ });+}++#[test]+#[cfg_attr(miri, ignore)]+fn yielding_to_subtask_invalid_task_returns_table_error() {+ invalid_thread_suspend_returns_original_error(|thread| SuspendReason::YieldingToSubtask {+ thread,+ });+}++fn store() -> Store<()> {+ let mut config = crate::Config::new();+ config.wasm_component_model_async(true);+ Store::new(&crate::Engine::new(&config).unwrap(), ())+}++fn instance() -> RuntimeInstance {+ RuntimeInstance {+ instance: crate::component::ComponentInstanceId::from_u32(0),+ index: RuntimeComponentInstanceIndex::from_u32(0),+ }+}++fn thread() -> QualifiedThreadId {+ QualifiedThreadId {+ task: TableId::new(u32::MAX),+ thread: TableId::new(u32::MAX),+ }+}++fn live(store: &mut StoreOpaque) -> StoreFiber<'static> {+ unsafe { fiber::make_fiber_unchecked(store, |_| Ok(())) }.unwrap()+}++fn bug(f: impl FnOnce() -> Result<()>, message: &str) {+ use std::panic::{AssertUnwindSafe, catch_unwind};+ let result = catch_unwind(AssertUnwindSafe(f));+ if cfg!(debug_assertions) {+ let panic = result.expect_err("debug bail_bug must panic");+ let text = panic+ .downcast_ref::<String>()+ .map(String::as_str)+ .or_else(|| panic.downcast_ref::<&str>().copied())+ .unwrap();+ assert!(text.contains(message), "{text}");+ } else {+ let error = result.unwrap().unwrap_err();+ assert!(error.is::<crate::WasmtimeBug>());+ assert!(error.to_string().contains(message), "{error}");+ }+}++fn add_thread(state: &mut ConcurrentState, fiber: StoreFiber<'static>) -> QualifiedThreadId {+ let sync_call_set = state.push(WaitableSet::default()).unwrap();+ let q = thread();+ let t = state+ .push(GuestThread {+ context: [0; NUM_COMPONENT_CONTEXT_SLOTS],+ parent_task: q.task,+ wake_on_cancel: WakeOnCancel::None,+ state: GuestThreadState::Suspended(fiber),+ instance_rep: None,+ sync_call_set,+ old_do_not_suspend: None,+ })+ .unwrap();+ QualifiedThreadId { thread: t, ..q }+}++#[test]+#[cfg_attr(miri, ignore)]+fn unpolled_resume_disposes_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = live(store);+ drop(store.resume_fiber(fiber));+}++#[test]+#[cfg_attr(miri, ignore)]+fn missing_suspend_reason_disposes_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = unsafe {+ fiber::make_fiber_unchecked(store, |store| {+ store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ Ok(())+ })+ }+ .unwrap();+ bug(+ || {+ let mut future = Box::pin(store.resume_fiber(fiber));+ match future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ {+ Poll::Ready(r) => r,+ _ => panic!("expected completion"),+ }+ },+ "suspend reason missing when resuming fiber",+ );+}++#[test]+#[cfg_attr(miri, ignore)]+fn full_table_during_switch_save_keeps_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = live(store);+ let state = store.concurrent_state_mut().unwrap();+ state.next_switch_item = Some(WorkItem::ResumeFiber {+ instance: instance(),+ thread: thread(),+ fiber,+ });+ state.table.get_mut().set_max_capacity(0);+ let error = store+ .suspend(SuspendReason::YieldingToSubtask { thread: thread() })+ .unwrap_err();+ assert!(matches!(+ error.downcast_ref::<ResourceTableError>(),+ Some(ResourceTableError::Full)+ ));+ assert!(+ store+ .concurrent_state_mut()+ .unwrap()+ .next_switch_item+ .is_some()+ );+}++#[test]+#[cfg_attr(miri, ignore)]+fn occupied_restore_keeps_both_fibers() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let a = live(store);+ let b = live(store);+ let state = store.concurrent_state_mut().unwrap();+ state.next_switch_item = Some(WorkItem::ResumeFiber {+ instance: instance(),+ thread: thread(),+ fiber: a,+ });+ let saved = state.save_next_switch_item().unwrap();+ state.next_switch_item = Some(WorkItem::ResumeFiber {+ instance: instance(),+ thread: thread(),+ fiber: b,+ });+ bug(+ || state.restore_next_switch_item(saved),+ "next switch item already set when restoring",+ );+ assert!(state.get_mut(saved).unwrap().is_some());+ assert!(state.next_switch_item.is_some());+}++#[test]+#[cfg_attr(miri, ignore)]+fn callback_cancel_rejects_fiber_waiter_without_removing_it() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = live(store);+ let state = store.concurrent_state_mut().unwrap();+ let set = state.push(WaitableSet::default()).unwrap();+ state+ .get_mut(set)+ .unwrap()+ .waiting+ .insert(thread(), WaitMode::Fiber(fiber));+ bug(+ || state.take_callback_waiter(set, thread()).map(|_| ()),+ "expected",+ );+ assert!(state.get_mut(set).unwrap().waiting.contains_key(&thread()));+}++#[test]+#[cfg_attr(miri, ignore)]+fn ready_dispatch_rejects_suspended_thread_without_taking_fiber() {+ let mut s = store();+ let fiber = live(s.as_context_mut().0);+ let q = add_thread(s.as_context_mut().0.concurrent_state_mut().unwrap(), fiber);+ bug(+ || {+ let mut future =+ Box::pin(s.as_context_mut().handle_work_item(WorkItem::ResumeThread {+ instance: instance(),+ thread: q,+ }));+ match future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ {+ Poll::Ready(r) => r,+ _ => panic!("expected completion"),+ }+ },+ "cannot resume non-pending thread",+ );+ assert!(matches!(+ s.as_context_mut()+ .0+ .concurrent_state_mut()+ .unwrap()+ .get_mut(q.thread)+ .unwrap()+ .state,+ GuestThreadState::Suspended(_)+ ));+}++#[test]+#[cfg_attr(miri, ignore)]+fn running_update_keeps_suspended_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = live(store);+ let state = store.concurrent_state_mut().unwrap();+ let q = add_thread(state, fiber);+ bug(+ || state.set_thread_running(q.thread),+ "thread already owns a fiber",+ );+ assert!(matches!(+ state.get_mut(q.thread).unwrap().state,+ GuestThreadState::Suspended(_)+ ));+}++#[test]+#[cfg_attr(miri, ignore)]+fn worker_item_conflict_keeps_worker_in_store() {+ let mut s = store();+ let fiber = live(s.as_context_mut().0);+ let state = s.as_context_mut().0.concurrent_state_mut().unwrap();+ state.worker = Some(fiber);+ state.worker_item = Some(WorkerItem::Function(AlwaysMut::new(Box::new(|_| Ok(())))));+ bug(+ || {+ let cx = &mut Context::from_waker(Waker::noop());+ let mut future = Box::pin(+ s.as_context_mut()+ .run_on_worker(WorkerItem::Function(AlwaysMut::new(Box::new(|_| Ok(()))))),+ );+ loop {+ match future.as_mut().poll(cx) {+ Poll::Pending => {}+ Poll::Ready(r) => return r,+ }+ }+ },+ "worker item already set",+ );+ assert!(+ s.as_context_mut()+ .0+ .concurrent_state_mut()+ .unwrap()+ .worker+ .is_some()+ );+}++#[test]+#[cfg_attr(miri, ignore)]+fn promotion_preserves_remaining_queue_fibers() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let a = live(store);+ let b = live(store);+ let state = store.concurrent_state_mut().unwrap();+ let q = thread();+ state.push_low_priority(WorkItem::ResumeFiber {+ instance: instance(),+ thread: q,+ fiber: a,+ });+ state.push_low_priority(WorkItem::ResumeFiber {+ instance: instance(),+ thread: q,+ fiber: b,+ });+ assert!(state.promote_work_item_matching(|_| true).unwrap());+ assert!(state.switch_item.is_some());+ assert_eq!(state.low_priority.len(), 1);+}++#[test]+#[cfg_attr(miri, ignore)]+fn initial_thread_lookup_error_disposes_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = live(store);+ let state = store.concurrent_state_mut().unwrap();+ state.unforced_current_thread = CurrentThread::Guest(thread());+ // The missing base task is checked before walking deferred frames.+ // This marker is never dereferenced; clear it again after the error.+ *store.vm_store_context_mut().current_thread_mut() =+ VMLazyThread::deferred(NonNull::dangling());+ let mut future = Box::pin(store.resume_fiber(fiber));+ let Poll::Ready(Err(error)) = future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ else {+ panic!("expected lookup error");+ };+ assert!(matches!(+ error.downcast_ref::<ResourceTableError>(),+ Some(ResourceTableError::NotPresent)+ ));+ drop(future);+ *store.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();+}++#[test]+#[cfg_attr(miri, ignore)]+fn post_release_thread_lookup_error_disposes_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ store+ .concurrent_state_mut()+ .unwrap()+ .unforced_current_thread = CurrentThread::Guest(thread());+ let fiber = unsafe {+ fiber::make_fiber_unchecked(store, |store| {+ store.concurrent_state_mut()?.suspend_reason = Some(SuspendReason::NeedWork);+ store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ Ok(())+ })+ }+ .unwrap();+ let mut future = Box::pin(store.resume_fiber(fiber));+ let Poll::Ready(Err(error)) = future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ else {+ panic!("expected post-release lookup error");+ };+ assert!(matches!(+ error.downcast_ref::<ResourceTableError>(),+ Some(ResourceTableError::NotPresent)+ ));+}++#[test]+#[cfg_attr(miri, ignore)]+fn suspended_destination_keeps_old_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let old = live(store);+ let q = add_thread(store.concurrent_state_mut().unwrap(), old);+ let fiber = unsafe {+ fiber::make_fiber_unchecked(store, move |store| {+ store.concurrent_state_mut()?.suspend_reason =+ Some(SuspendReason::ExplicitlySuspending { thread: q });+ store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ Ok(())+ })+ }+ .unwrap();+ bug(+ || {+ let mut future = Box::pin(store.resume_fiber(fiber));+ match future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ {+ Poll::Ready(r) => r,+ _ => panic!("expected completion"),+ }+ },+ "thread already owns a fiber",+ );+ assert!(matches!(+ store+ .concurrent_state_mut()+ .unwrap()+ .get_mut(q.thread)+ .unwrap()+ .state,+ GuestThreadState::Suspended(_)+ ));+}++#[test]+#[cfg_attr(miri, ignore)]+fn duplicate_waiter_keeps_old_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let old = live(store);+ let state = store.concurrent_state_mut().unwrap();+ let set = state.push(WaitableSet::default()).unwrap();+ state+ .get_mut(set)+ .unwrap()+ .waiting+ .insert(thread(), WaitMode::Fiber(old));+ let fiber = unsafe {+ fiber::make_fiber_unchecked(store, move |store| {+ store.concurrent_state_mut()?.suspend_reason = Some(SuspendReason::Waiting {+ set,+ thread: thread(),+ });+ store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ Ok(())+ })+ }+ .unwrap();+ bug(+ || {+ let mut future = Box::pin(store.resume_fiber(fiber));+ let cx = &mut Context::from_waker(Waker::noop());+ loop {+ match future.as_mut().poll(cx) {+ Poll::Pending => {}+ Poll::Ready(r) => return r,+ }+ }+ },+ "entry unexpectedly already exists",+ );+ assert!(+ store+ .concurrent_state_mut()+ .unwrap()+ .get_mut(set)+ .unwrap()+ .waiting+ .contains_key(&thread())+ );+}++#[test]+#[cfg_attr(miri, ignore)]+fn invalid_waiter_task_lookup_keeps_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = live(store);+ let old = live(store);+ let state = store.concurrent_state_mut().unwrap();+ let q = add_thread(state, old);+ let set = state.push(WaitableSet::default()).unwrap();+ let group = state.make_task_group().unwrap();+ let host = state+ .push(HostTask {+ common: WaitableCommon {+ set: Some(set),+ ..WaitableCommon::default()+ },+ call_context: CallContext::default(),+ state: HostTaskState::CalleeStarted,+ group,+ })+ .unwrap();+ state+ .get_mut(set)+ .unwrap()+ .waiting+ .insert(q, WaitMode::Fiber(fiber));+ let error = Waitable::Host(host).mark_ready(state).unwrap_err();+ assert!(matches!(+ error.downcast_ref::<ResourceTableError>(),+ Some(ResourceTableError::NotPresent)+ ));+ assert!(state.get_mut(set).unwrap().waiting.contains_key(&q));+}++#[test]+#[cfg_attr(miri, ignore)]+fn subtask_lookup_error_keeps_parked_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let old = live(store);+ store.concurrent_state_mut().unwrap().next_switch_item = Some(WorkItem::ResumeFiber {+ instance: instance(),+ thread: thread(),+ fiber: old,+ });+ // Invalid task lookup still precedes the occupied-slot check.+ // Both incoming and parked fibers must survive that early error.+ let fiber = unsafe {+ fiber::make_fiber_unchecked(store, move |store| {+ store.concurrent_state_mut()?.suspend_reason =+ Some(SuspendReason::YieldingToSubtask { thread: thread() });+ store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ Ok(())+ })+ }+ .unwrap();+ let mut future = Box::pin(store.resume_fiber(fiber));+ let Poll::Ready(Err(error)) = future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ else {+ panic!("expected original task lookup error");+ };+ assert!(matches!(+ error.downcast_ref::<ResourceTableError>(),+ Some(ResourceTableError::NotPresent)+ ));+ drop(future);+ assert!(+ store+ .concurrent_state_mut()+ .unwrap()+ .next_switch_item+ .is_some()+ );+}++#[test]+#[cfg_attr(miri, ignore)]+fn occupied_resume_schedule_keeps_suspended_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = live(store);+ let state = store.concurrent_state_mut().unwrap();+ let q = add_thread(state, fiber);+ state.switch_item = Some(WorkItem::ResumeThread {+ instance: instance(),+ thread: thread(),+ });+ bug(+ || state.schedule_suspended_thread(q, instance(), Priority::Switch),+ "switch item already set",+ );+ assert!(matches!(+ state.get_mut(q.thread).unwrap().state,+ GuestThreadState::Suspended(_)+ ));+ assert!(state.high_priority.is_empty());+}+#[test]+#[cfg_attr(miri, ignore)]+fn schedule_rejects_ready_thread_without_dropping_fiber() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let fiber = live(store);+ let state = store.concurrent_state_mut().unwrap();+ let q = add_thread(state, fiber);+ let slot = &mut state.get_mut(q.thread).unwrap().state;+ let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running) else {+ unreachable!()+ };+ *slot = GuestThreadState::Ready { fiber };+ let error = state+ .schedule_suspended_thread(q, instance(), Priority::Low)+ .unwrap_err();+ assert!(matches!(+ error.downcast_ref::<Trap>(),+ Some(Trap::CannotResumeThread)+ ));+ assert!(matches!(+ state.get_mut(q.thread).unwrap().state,+ GuestThreadState::Ready { .. }+ ));+}++#[test]+#[cfg_attr(miri, ignore)]+fn occupied_subtask_switch_keeps_both_fibers() {+ let mut s = store();+ let engine = s.engine().clone();+ let component = crate::component::Component::new(+ &engine,+ r#"+ (component+ (core module $m (func (export "f")))+ (core instance $i (instantiate $m))+ (func (export "f") (canon lift (core func $i "f"))))+ "#,+ )+ .unwrap();+ let inst = crate::component::Linker::new(&engine)+ .instantiate(&mut s, &component)+ .unwrap();+ let func = inst.get_func(&mut s, "f").unwrap();+ let mut results = [];+ let call = func+ .start_call_concurrent(&mut s, &[], &mut results)+ .unwrap();+ let task = s+ .as_context_mut()+ .0+ .concurrent_state_mut()+ .unwrap()+ .table+ .get_mut()+ .iter_mut()+ .find_map(|(rep, entry)| entry.downcast_mut::<GuestTask>().map(|_| TableId::new(rep)))+ .unwrap();+ let store = s.as_context_mut().0;+ let q = QualifiedThreadId {+ task,+ thread: thread().thread,+ };+ let parked = live(store);+ store.concurrent_state_mut().unwrap().next_switch_item = Some(WorkItem::ResumeFiber {+ instance: instance(),+ thread: thread(),+ fiber: parked,+ });+ let fiber = unsafe {+ fiber::make_fiber_unchecked(store, move |store| {+ store.concurrent_state_mut()?.suspend_reason =+ Some(SuspendReason::YieldingToSubtask { thread: q });+ store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ Ok(())+ })+ }+ .unwrap();+ bug(+ || {+ let mut future = Box::pin(store.resume_fiber(fiber));+ match future+ .as_mut()+ .poll(&mut Context::from_waker(Waker::noop()))+ {+ Poll::Ready(r) => r,+ _ => panic!("expected completion"),+ }+ },+ "was already",+ );+ assert!(+ store+ .concurrent_state_mut()+ .unwrap()+ .next_switch_item+ .is_some()+ );+ drop(call);+}++#[test]+#[cfg_attr(miri, ignore)]+fn waiter_assertion_keeps_fiber_in_set() {+ let mut s = store();+ let store = s.as_context_mut().0;+ let old = live(store);+ let fiber = live(store);+ let state = store.concurrent_state_mut().unwrap();+ let q = add_thread(state, old);+ let set = state.push(WaitableSet::default()).unwrap();+ state.get_mut(q.thread).unwrap().wake_on_cancel = WakeOnCancel::Yielding;+ let group = state.make_task_group().unwrap();+ let host = state+ .push(HostTask {+ common: WaitableCommon {+ set: Some(set),+ ..WaitableCommon::default()+ },+ call_context: CallContext::default(),+ state: HostTaskState::CalleeStarted,+ group,+ })+ .unwrap();+ state+ .get_mut(set)+ .unwrap()+ .waiting+ .insert(q, WaitMode::Fiber(fiber));+ bug(+ || Waitable::Host(host).mark_ready(state),+ "unexpected wake_on_cancel",+ );+ assert!(state.get_mut(set).unwrap().waiting.contains_key(&q));+}++#[test]+#[cfg_attr(miri, ignore)]+fn unpolled_work_item_disposes_fiber() {+ let mut s = store();+ let fiber = live(s.as_context_mut().0);+ drop(s.as_context_mut().handle_work_item(WorkItem::ResumeFiber {+ instance: instance(),+ thread: thread(),+ fiber,+ }));+}++#[test]+#[cfg_attr(miri, ignore)]+fn promotion_panic_keeps_queued_fibers() {+ use std::panic::{AssertUnwindSafe, catch_unwind};+ let mut s = store();+ let store = s.as_context_mut().0;+ let a = live(store);+ let b = live(store);+ let state = store.concurrent_state_mut().unwrap();+ state.push_high_priority(WorkItem::ResumeFiber {+ instance: instance(),+ thread: thread(),+ fiber: a,+ });+ state.push_low_priority(WorkItem::ResumeFiber {+ instance: instance(),+ thread: thread(),+ fiber: b,+ });+ let result = catch_unwind(AssertUnwindSafe(|| {+ state.promote_work_item_matching(|_| panic!("predicate"))+ }));+ assert!(result.is_err());+ assert_eq!(state.high_priority.len(), 1);+ assert_eq!(state.low_priority.len(), 1);+}diff --git a/tests/misc_testsuite/component-model/async/thread-wait-resume.wast b/tests/misc_testsuite/component-model/async/thread-wait-resume.wastnew file mode 100644index 0000000000..04f59c7847--- /dev/null+++ b/tests/misc_testsuite/component-model/async/thread-wait-resume.wast@@ -0,0 +1,78 @@+;;! component_model_async = true+;;! component_model_threading = true++;; A thread blocked in waitable-set.wait is still running for the purposes of+;; thread.resume-later and thread.suspend-then-resume. In particular, neither+;; operation may wake it without a waitable event.+(component definition $C+ (core module $libc+ (memory (export "mem") 1)+ (table (export "table") 1 funcref))+ (core instance $libc (instantiate $libc))+ (core type $start (func (param i32)))+ (core func $thread.new-indirect+ (canon thread.new-indirect $start (core table $libc "table")))+ (core func $thread.yield-then-resume (canon thread.yield-then-resume))+ (core func $thread.resume-later (canon thread.resume-later))+ (core func $thread.suspend-then-resume (canon thread.suspend-then-resume))+ (type $stream (stream u8))+ (core func $stream.new (canon stream.new $stream))+ (core func $waitable-set.new (canon waitable-set.new))+ (core func $waitable.join (canon waitable.join))+ (core func $waitable-set.wait+ (canon waitable-set.wait (memory (core memory $libc "mem"))))++ (core module $m+ (import "libc" "mem" (memory 1))+ (import "libc" "table" (table $table 1 funcref))+ (import "" "thread.new-indirect" (func $new (param i32 i32) (result i32)))+ (import "" "thread.yield-then-resume" (func $yield-to (param i32) (result i32)))+ (import "" "thread.resume-later" (func $resume-later (param i32)))+ (import "" "thread.suspend-then-resume" (func $suspend-to (param i32) (result i32)))+ (import "" "stream.new" (func $stream-new (result i64)))+ (import "" "waitable-set.new" (func $set-new (result i32)))+ (import "" "waitable.join" (func $join (param i32 i32)))+ (import "" "waitable-set.wait" (func $wait (param i32 i32) (result i32)))++ (func $waiter (param i32)+ (local $ends i64) (local $set i32)+ (local.set $ends (call $stream-new))+ (local.set $set (call $set-new))+ (call $join (i32.wrap_i64 (local.get $ends)) (local.get $set))+ ;; The writable end stays open, so there is no event to wake the waiter.+ (drop (call $wait (local.get $set) (i32.const 0))))+ (elem (table $table) (i32.const 0) func $waiter)++ (func $start-waiter (result i32)+ (local $id i32)+ (local.set $id (call $new (i32.const 0) (i32.const 0)))+ (drop (call $yield-to (local.get $id)))+ (local.get $id))++ (func (export "resume-later-waiter")+ (call $resume-later (call $start-waiter)))+ (func (export "suspend-then-resume-waiter")+ (drop (call $suspend-to (call $start-waiter)))))++ (core instance $i (instantiate $m+ (with "libc" (instance $libc))+ (with "" (instance+ (export "thread.new-indirect" (func $thread.new-indirect))+ (export "thread.yield-then-resume" (func $thread.yield-then-resume))+ (export "thread.resume-later" (func $thread.resume-later))+ (export "thread.suspend-then-resume" (func $thread.suspend-then-resume))+ (export "stream.new" (func $stream.new))+ (export "waitable-set.new" (func $waitable-set.new))+ (export "waitable.join" (func $waitable.join))+ (export "waitable-set.wait" (func $waitable-set.wait))))))+ (func (export "resume-later-waiter") async+ (canon lift (core func $i "resume-later-waiter")))+ (func (export "suspend-then-resume-waiter") async+ (canon lift (core func $i "suspend-then-resume-waiter"))))++(component instance $resume-later $C)+(assert_trap (invoke "resume-later-waiter")+ "cannot resume thread which is not suspended")+(component instance $suspend-then-resume $C)+(assert_trap (invoke "suspend-then-resume-waiter")+ "cannot resume thread which is not suspended")