A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
yield to executor when polling with zero-duration wait (#13085)
Joel Dice80be8d62d837
diff --git a/crates/test-programs/src/bin/p2_sleep.rs b/crates/test-programs/src/bin/p2_sleep.rsindex a6c8dae398..f8b835ccd1 100644--- a/crates/test-programs/src/bin/p2_sleep.rs+++ b/crates/test-programs/src/bin/p2_sleep.rs@@ -18,16 +18,10 @@ fn sleep_0ms() {let p = monotonic_clock::subscribe_instant(monotonic_clock::now());p.block();let p = monotonic_clock::subscribe_duration(0);- assert!(- p.ready(),- "timer subscription with duration 0 is ready immediately"- );+ p.block();}fn sleep_backwards_in_time() {let p = monotonic_clock::subscribe_instant(monotonic_clock::now() - 1);- assert!(- p.ready(),- "timer subscription for instant which has passed is ready immediately"- );+ p.block();}diff --git a/crates/test-programs/src/bin/p2_tcp_busy_poll.rs b/crates/test-programs/src/bin/p2_tcp_busy_poll.rsnew file mode 100644index 0000000000..9f59886586--- /dev/null+++ b/crates/test-programs/src/bin/p2_tcp_busy_poll.rs@@ -0,0 +1,72 @@+use test_programs::sockets::supports_ipv6;+use test_programs::wasi::clocks::monotonic_clock;+use test_programs::wasi::io::poll;+use test_programs::wasi::sockets::network::{+ IpAddressFamily, IpSocketAddress, Ipv4SocketAddress, Ipv6SocketAddress, Network,+};+use test_programs::wasi::sockets::tcp::TcpSocket;++// Historically, `wasmtime-wasi` had a bug such that polling with a+// zero-duration (i.e. always-ready) wait would starve the host executor and+// prevent e.g. socket readiness from being delivered. Here we verify that such+// starvation does not happen.+fn test_tcp_busy_poll(family: IpAddressFamily, address: IpSocketAddress) {+ let zero_wait = monotonic_clock::subscribe_duration(0);++ let net = Network::default();++ let listener = TcpSocket::new(family).unwrap();+ listener.blocking_bind(&net, address).unwrap();+ listener.set_listen_backlog_size(32).unwrap();+ listener.blocking_listen().unwrap();++ let address = listener.local_address().unwrap();++ let message = b"Hello, world!";++ for _ in 0..100 {+ let client = TcpSocket::new(family).unwrap();+ let (_rx, tx) = client.blocking_connect(&net, address).unwrap();+ tx.blocking_write_util(message).unwrap();++ let (_accepted, rx, _tx) = listener.blocking_accept().unwrap();+ let rx_ready = rx.subscribe();+ let mut counter = 0;+ loop {+ if counter > 1_000_000 {+ panic!("socket still not ready!");+ }++ let ready = poll::poll(&[&zero_wait, &rx_ready]);+ if ready.contains(&1) {+ break;+ }+ counter += 1;+ }++ let data = rx.read(message.len().try_into().unwrap()).unwrap();+ assert_eq!(data, message); // Not guaranteed to work but should work in practice.+ }+}++fn main() {+ test_tcp_busy_poll(+ IpAddressFamily::Ipv4,+ IpSocketAddress::Ipv4(Ipv4SocketAddress {+ port: 0,+ address: (127, 0, 0, 1),+ }),+ );++ if supports_ipv6() {+ test_tcp_busy_poll(+ IpAddressFamily::Ipv6,+ IpSocketAddress::Ipv6(Ipv6SocketAddress {+ port: 0,+ address: (0, 0, 0, 0, 0, 0, 0, 1),+ flow_info: 0,+ scope_id: 0,+ }),+ );+ }+}diff --git a/crates/test-programs/src/bin/p3_sockets_tcp_busy_poll.rs b/crates/test-programs/src/bin/p3_sockets_tcp_busy_poll.rsnew file mode 100644index 0000000000..6583eed0df--- /dev/null+++ b/crates/test-programs/src/bin/p3_sockets_tcp_busy_poll.rs@@ -0,0 +1,123 @@+use futures::join;+use std::ptr;+use test_programs::async_::{+ BLOCKED, EVENT_NONE, EVENT_STREAM_READ, thread_yield, waitable_join, waitable_set_drop,+ waitable_set_new, waitable_set_poll,+};+use test_programs::p3::wasi::sockets::types::{+ IpAddressFamily, IpSocketAddress, Ipv4SocketAddress, Ipv6SocketAddress, TcpSocket,+};+use test_programs::p3::wit_stream;+use test_programs::sockets::supports_ipv6;+use wit_bindgen::StreamResult;++#[link(wasm_import_module = "wasi:http/[email protected]")]+unsafe extern "C" {+ #[link_name = "[async-lower][stream-read-0][static]request.new"]+ fn stream_read(_: u32, _: *mut u8, _: usize) -> u32;+}++// Historically, `wasmtime-wasi` had a bug such that polling would starve the+// host executor and prevent e.g. socket readiness from being delivered. Here+// we verify that such starvation does not happen.+async fn test_tcp_busy_poll(family: IpAddressFamily, address: IpSocketAddress) {+ let listener = TcpSocket::create(family).unwrap();+ listener.bind(address).unwrap();+ listener.set_listen_backlog_size(32).unwrap();+ let mut accept = listener.listen().unwrap();++ let address = listener.get_local_address().unwrap();++ let message = b"Hello, world!";++ for _ in 0..100 {+ let client = TcpSocket::create(family).unwrap();+ client.connect(address).await.unwrap();+ let (mut data_tx, data_rx) = wit_stream::new();+ join!(+ async {+ client.send(data_rx).await.unwrap();+ },+ async {+ let remaining = data_tx.write_all(message.into()).await;+ assert!(remaining.is_empty());+ drop(data_tx);+ }+ );++ let sock = accept.next().await.unwrap();+ let (mut data_rx, fut) = sock.receive();++ unsafe {+ if (stream_read(data_rx.handle(), ptr::null_mut(), 0)) == BLOCKED {+ let set = waitable_set_new();+ waitable_join(data_rx.handle(), set);+ let mut counter = 0;+ loop {+ if counter > 1_000_000 {+ panic!("socket still not ready!");+ }++ let (mut event, mut waitable, _) = waitable_set_poll(set);+ if event == EVENT_NONE {+ // `waitable-set.poll` does not yield, so we+ // call `thread.yield` to give the executor a+ // chance to run, then poll one more time.+ assert!(thread_yield() == 0);+ (event, waitable, _) = waitable_set_poll(set);+ }+ if event == EVENT_STREAM_READ && waitable == data_rx.handle() {+ waitable_join(data_rx.handle(), 0);+ waitable_set_drop(set);+ break;+ }+ counter += 1;+ }+ }+ }++ let (result, data) = data_rx.read(Vec::with_capacity(100)).await;+ assert_eq!(result, StreamResult::Complete(message.len()));+ assert_eq!(data, message); // Not guaranteed to work but should work in practice.++ let (result, data) = data_rx.read(Vec::with_capacity(1)).await;+ assert_eq!(result, StreamResult::Dropped);+ assert_eq!(data, []);++ fut.await.unwrap();+ }+}++struct Component;++test_programs::p3::export!(Component);++impl test_programs::p3::exports::wasi::cli::run::Guest for Component {+ async fn run() -> Result<(), ()> {+ test_tcp_busy_poll(+ IpAddressFamily::Ipv4,+ IpSocketAddress::Ipv4(Ipv4SocketAddress {+ port: 0,+ address: (127, 0, 0, 1),+ }),+ )+ .await;++ if supports_ipv6() {+ test_tcp_busy_poll(+ IpAddressFamily::Ipv6,+ IpSocketAddress::Ipv6(Ipv6SocketAddress {+ port: 0,+ address: (0, 0, 0, 0, 0, 0, 0, 1),+ flow_info: 0,+ scope_id: 0,+ }),+ )+ .await;+ }++ Ok(())+ }+}++fn main() {}diff --git a/crates/wasi/src/p2/host/clocks.rs b/crates/wasi/src/p2/host/clocks.rsindex 6097835778..3427c9cfd3 100644--- a/crates/wasi/src/p2/host/clocks.rs+++ b/crates/wasi/src/p2/host/clocks.rs@@ -124,7 +124,26 @@ enum Deadline {impl Pollable for Deadline {async fn ready(&mut self) {match self {- Deadline::Past => {}+ Deadline::Past => {+ // It is important we yield to Tokio here; otherwise we risk+ // starving `mio` such that it is unable to signal readiness for+ // other pollables (e.g. TCP sockets) when the guest is polling+ // in a busy loop.+ //+ // This is somewhat of a hack to ensure that+ // `wasmtime-wasi-io`'s implementation of `wasi:io/poll` does+ // not starve `mio` when the guest calls `wasi:io/poll#poll` in+ // a busy loop with a zero timeout. It relies on the guest+ // using the most natural approach to making a non-blocking call+ // to `wasi:io/poll#poll`, which is to include a zero-duration+ // `monotonic_clock::subscribe_{instant,duration}` in the list+ // of pollables. That's what `wasi-libc`'s `poll(2)`+ // implementation does as of this writing, for example. There+ // are hypothetically other ways to generate a pollable that's+ // always immediately ready, which this hack doesn't cover, but+ // we consider this sufficient for now.+ tokio::task::yield_now().await+ }Deadline::Instant(instant) => tokio::time::sleep_until(*instant).await,Deadline::Never => std::future::pending().await,}diff --git a/crates/wasi/tests/all/p2/async_.rs b/crates/wasi/tests/all/p2/async_.rsindex 41fdd2842d..bd8eb9400e 100644--- a/crates/wasi/tests/all/p2/async_.rs+++ b/crates/wasi/tests/all/p2/async_.rs@@ -307,6 +307,10 @@ async fn p2_tcp_listen() {run(P2_TCP_LISTEN_COMPONENT, false).await.unwrap()}#[test_log::test(tokio::test(flavor = "multi_thread"))]+async fn p2_tcp_busy_poll() {+ run(P2_TCP_BUSY_POLL_COMPONENT, false).await.unwrap()+}+#[test_log::test(tokio::test(flavor = "multi_thread"))]async fn p2_udp_sockopts() {run(P2_UDP_SOCKOPTS_COMPONENT, false).await.unwrap()}diff --git a/crates/wasi/tests/all/p2/sync.rs b/crates/wasi/tests/all/p2/sync.rsindex b7cbdc8e12..4bce4681fa 100644--- a/crates/wasi/tests/all/p2/sync.rs+++ b/crates/wasi/tests/all/p2/sync.rs@@ -293,6 +293,10 @@ fn p2_tcp_listen() {run(P2_TCP_LISTEN_COMPONENT, false).unwrap()}#[test_log::test]+fn p2_tcp_busy_poll() {+ run(P2_TCP_BUSY_POLL_COMPONENT, false).unwrap()+}+#[test_log::test]fn p2_udp_sockopts() {run(P2_UDP_SOCKOPTS_COMPONENT, false).unwrap()}diff --git a/crates/wasi/tests/all/p3/mod.rs b/crates/wasi/tests/all/p3/mod.rsindex 10998cb1c6..b3f5eb4329 100644--- a/crates/wasi/tests/all/p3/mod.rs+++ b/crates/wasi/tests/all/p3/mod.rs@@ -115,6 +115,11 @@ async fn p3_sockets_tcp_streams() -> wasmtime::Result<()> {run(P3_SOCKETS_TCP_STREAMS_COMPONENT).await}+#[test_log::test(tokio::test(flavor = "multi_thread"))]+async fn p3_sockets_tcp_busy_poll() -> wasmtime::Result<()> {+ run(P3_SOCKETS_TCP_BUSY_POLL_COMPONENT).await+}+#[test_log::test(tokio::test(flavor = "multi_thread"))]async fn p3_sockets_udp_bind() -> wasmtime::Result<()> {run(P3_SOCKETS_UDP_BIND_COMPONENT).awaitdiff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex ace17b3e99..1570050359 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -1196,8 +1196,12 @@ impl<T> StoreContextMut<'_, T> {enum PollResult<R> {Complete(R),- ProcessWork(Vec<WorkItem>),+ ProcessWork {+ ready: Vec<WorkItem>,+ low_priority: bool,+ },}+let result = future::poll_fn(|cx| {// First, poll the future we were passed as an argument and// return immediately if it's ready.@@ -1220,13 +1224,23 @@ impl<T> StoreContextMut<'_, T> {Poll::Pending => Poll::Pending,};- // Next, collect the next batch of work items to process, if any.- // This will be either all of the high-priority work items, or if- // there are none, a single low-priority work item.+ // Next, collect the next batch of work items to process, if+ // any. This will be either all of the high-priority work+ // items, or if there are none, a single low-priority work item.let state = reset.store.0.concurrent_state_mut();- let ready = state.collect_work_items_to_run();+ let mut ready = mem::take(&mut state.high_priority);+ let mut low_priority = false;+ if ready.is_empty() {+ if let Some(item) = state.low_priority.pop_back() {+ ready.push(item);+ low_priority = true;+ }+ }if !ready.is_empty() {- return Poll::Ready(Ok(PollResult::ProcessWork(ready)));+ return Poll::Ready(Ok(PollResult::ProcessWork {+ ready,+ low_priority,+ }));}// Finally, if we have nothing else to do right now, determine what to do@@ -1239,7 +1253,10 @@ impl<T> StoreContextMut<'_, T> {// successfully, so we return now and continue// the outer loop in case there is another one// ready to complete.- Poll::Ready(Ok(PollResult::ProcessWork(Vec::new())))+ Poll::Ready(Ok(PollResult::ProcessWork {+ ready: Vec::new(),+ low_priority: false,+ }))}Poll::Ready(false) => {// Poll the future we were passed one last time@@ -1287,7 +1304,10 @@ impl<T> StoreContextMut<'_, T> {PollResult::Complete(value) => break Ok(value),// The future we were passed has not yet completed, so handle// any work items and then loop again.- PollResult::ProcessWork(ready) => {+ PollResult::ProcessWork {+ ready,+ low_priority,+ } => {struct Dispose<'a, T: 'static, I: Iterator<Item = WorkItem>> {store: StoreContextMut<'a, T>,ready: I,@@ -1312,6 +1332,31 @@ impl<T> StoreContextMut<'_, T> {ready: ready.into_iter(),};+ // If we're about to run a low-priority task, first yield to+ // the executor. This ensures that it won't be starved of+ // the ability to e.g. update the readiness of sockets,+ // etc. which the guest may be using `thread.yield` along+ // with `waitable-set.poll` to monitor in a CPU-heavy loop.+ //+ // This works for e.g. `thread.yield` and callbacks which+ // return `CALLBACK_CODE_YIELD` because we queue a low+ // priority item to resume the task (i.e. resume the thread+ // or call the callback, respectively) just prior to+ // suspending it. Indeed, as of this writing those are the+ // _only_ situations we queue low-priority tasks.+ // Therefore, we interpret the guest's request to yield as+ // meaning "yield to other guest tasks _and_/_or_ host+ // operations such as updating socket readiness", the latter+ // being the async runtime's responsibility.+ //+ // In the future, if this ends up causing measurable+ // performance issues, this could be optimized such that we+ // only yield periodically (e.g. for batches of low priority+ // items) and not for each and every idividual item.+ if low_priority {+ dispose.store.0.yield_now().await+ }+while let Some(item) = dispose.ready.next() {dispose.store@@ -4972,19 +5017,6 @@ impl ConcurrentState {}}- /// Collect the next set of work items to run. This will be either all- /// high-priority items, or a single low-priority item if there are no- /// high-priority items.- fn collect_work_items_to_run(&mut self) -> Vec<WorkItem> {- let mut ready = mem::take(&mut self.high_priority);- if ready.is_empty() {- if let Some(item) = self.low_priority.pop_back() {- ready.push(item);- }- }- ready- }-fn push<V: Send + Sync + 'static>(&mut self,value: V,diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 564571c9b1..3bbe7ccab4 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -106,12 +106,16 @@ use crate::{ExnRef, Rooted};use crate::{Global, Instance, Table};use core::convert::Infallible;use core::fmt;+#[cfg(any(feature = "async", feature = "gc"))]+use core::future;use core::marker;use core::mem::{self, ManuallyDrop, MaybeUninit};use core::num::NonZeroU64;use core::ops::{Deref, DerefMut};use core::pin::Pin;use core::ptr::NonNull;+#[cfg(any(feature = "async", feature = "gc"))]+use core::task::Poll;use wasmtime_environ::{DefinedGlobalIndex, DefinedTableIndex, EntityRef, TripleExt};mod context;@@ -2116,7 +2120,15 @@ impl StoreOpaque {self.trace_roots(&mut roots, asyncness).await;self.unwrap_gc_store_mut()- .gc(asyncness, unsafe { roots.iter() })+ .gc(+ asyncness,+ unsafe { roots.iter() },+ // TODO: Once `Config` has an optional `AsyncFn` field for+ // yielding to the current async runtime+ // (e.g. `tokio::task::yield_now`), use that if set; otherwise+ // fall back to the runtime-agnostic code.+ yield_now,+ ).await;// Restore the GC roots for the next GC.@@ -2135,30 +2147,30 @@ impl StoreOpaque {self.trace_wasm_stack_roots(gc_roots_list);if asyncness != Asyncness::No {- vm::Yield::new().await;+ self.yield_now().await;}#[cfg(feature = "stack-switching")]{self.trace_wasm_continuation_roots(gc_roots_list);if asyncness != Asyncness::No {- vm::Yield::new().await;+ self.yield_now().await;}}self.trace_vmctx_roots(gc_roots_list);if asyncness != Asyncness::No {- vm::Yield::new().await;+ self.yield_now().await;}self.trace_instance_roots(gc_roots_list);if asyncness != Asyncness::No {- vm::Yield::new().await;+ self.yield_now().await;}self.trace_user_roots(gc_roots_list);if asyncness != Asyncness::No {- vm::Yield::new().await;+ self.yield_now().await;}self.trace_pending_exception_roots(gc_roots_list);@@ -2810,6 +2822,29 @@ at https://bytecodealliance.org/security.Asyncness::No => {}}}++ #[cfg(any(feature = "async", feature = "gc"))]+ pub(crate) async fn yield_now(&self) {+ // TODO: Once `Config` has an optional `AsyncFn` field for yielding to the+ // current async runtime (e.g. `tokio::task::yield_now`), use that if set;+ // otherwise fall back to the runtime-agnostic code.+ yield_now().await+ }+}++#[cfg(any(feature = "async", feature = "gc"))]+async fn yield_now() {+ let mut yielded = false;+ future::poll_fn(move |cx| {+ if yielded {+ Poll::Ready(())+ } else {+ yielded = true;+ cx.waker().wake_by_ref();+ Poll::Pending+ }+ })+ .await;}/// Helper parameter to [`StoreOpaque::allocate_instance`].diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex 35934f49af..d3fe27fc5b 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -151,11 +151,6 @@ mod cow_disabled;#[cfg(has_virtual_memory)]mod mmap;-#[cfg(any(feature = "async", feature = "gc"))]-mod async_yield;-#[cfg(any(feature = "async", feature = "gc"))]-pub use crate::runtime::vm::async_yield::*;-#[cfg(feature = "gc-null")]mod send_sync_unsafe_cell;#[cfg(feature = "gc-null")]diff --git a/crates/wasmtime/src/runtime/vm/async_yield.rs b/crates/wasmtime/src/runtime/vm/async_yield.rsdeleted file mode 100644index 4b53f99049..0000000000--- a/crates/wasmtime/src/runtime/vm/async_yield.rs+++ /dev/null@@ -1,35 +0,0 @@-use core::{- future::Future,- pin::Pin,- task::{Context, Poll},-};--/// A small future that yields once and then returns.-#[derive(Default)]-pub struct Yield {- yielded: bool,-}--impl Yield {- /// Create a new `Yield`.- pub fn new() -> Self {- Self::default()- }-}--impl Future for Yield {- type Output = ();-- fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {- if self.yielded {- Poll::Ready(())- } else {- // Flag ourselves as yielded to return next time, and also- // flag the waker that we're already ready to get- // re-enqueued for another poll.- self.yielded = true;- cx.waker().wake_by_ref();- Poll::Pending- }- }-}diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rsindex 84e6c76acb..74717de0c1 100644--- a/crates/wasmtime/src/runtime/vm/gc.rs+++ b/crates/wasmtime/src/runtime/vm/gc.rs@@ -102,9 +102,14 @@ impl GcStore {}/// Asynchronously perform garbage collection within this heap.- pub async fn gc(&mut self, asyncness: Asyncness, roots: GcRootsIter<'_>) {+ pub async fn gc(+ &mut self,+ asyncness: Asyncness,+ roots: GcRootsIter<'_>,+ yield_fn: impl AsyncFn(),+ ) {let collection = self.gc_heap.gc(roots, &mut self.host_data_table);- collect_async(collection, asyncness).await;+ collect_async(collection, asyncness, yield_fn).await;self.last_post_gc_allocated_bytes = Some({let size = self.gc_heap.allocated_bytes();log::trace!("After collection, GC heap size = {size} bytes");diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rsindex d386ba4378..836a70417b 100644--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs@@ -755,13 +755,19 @@ pub enum GcProgress {pub async fn collect_async<'a>(mut collection: Box<dyn GarbageCollection<'a> + 'a>,asyncness: Asyncness,+ yield_fn: impl AsyncFn(),) {+ #[cfg(not(feature = "async"))]+ {+ _ = yield_fn;+ }+loop {match collection.collect_increment() {GcProgress::Continue => {if asyncness != Asyncness::No {#[cfg(feature = "async")]- crate::runtime::vm::Yield::new().await+ yield_fn().await}}GcProgress::Complete => return,@@ -778,7 +784,7 @@ mod collect_async_tests {fn _assert_send_sync<T: Send + Sync>(_: T) {}fn _foo<'a>(collection: Box<dyn GarbageCollection<'a>>) {- _assert_send_sync(collect_async(collection, Asyncness::Yes));+ _assert_send_sync(collect_async(collection, Asyncness::Yes, async || ()));}}}diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex 5dbd0489d1..013b651cba 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -1243,7 +1243,7 @@ fn out_of_gas(store: &mut dyn VMStore, _instance: InstanceId) -> Result<()> {}#[cfg(feature = "async")]if store.fuel_yield_interval.is_some() {- crate::runtime::vm::Yield::new().await;+ store.yield_now().await;}Ok(())})?@@ -1279,7 +1279,7 @@ fn new_epoch(store: &mut dyn VMStore, _instance: InstanceId) -> Result<NextEpochan async wasm entrypoint",);}- crate::runtime::vm::Yield::new().await;+ store.yield_now().await;delta}#[cfg(feature = "async")]