Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

make `Destination::as_direct` work for both host and guest readers (#11612)

Joel Dice6f0253930ea6
diff --git a/crates/misc/component-async-tests/src/util.rs b/crates/misc/component-async-tests/src/util.rs
index 40dc572d5e..70be467885 100644
--- a/crates/misc/component-async-tests/src/util.rs
+++ b/crates/misc/component-async-tests/src/util.rs
@@ -53,7 +53,7 @@ impl<D, T: Send + Sync + Lower + 'static, S: Stream<Item = T> + Send + 'static>
self: Pin<&mut Self>,
cx: &mut Context<'_>,
_: StoreContextMut<D>,
- destination: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ mut destination: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<Result<StreamResult>> {
// SAFETY: This is a standard pin-projection, and we never move
@@ -94,7 +94,7 @@ impl<D, T: Lift + 'static, S: Sink<T, Error: std::error::Error + Send + Sync> +
self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<D>,
- source: &mut Source<Self::Item>,
+ mut source: Source<Self::Item>,
finish: bool,
) -> Poll<Result<StreamResult>> {
// SAFETY: This is a standard pin-projection, and we never move
diff --git a/crates/misc/component-async-tests/tests/scenario/streams.rs b/crates/misc/component-async-tests/tests/scenario/streams.rs
index e63f2d3031..a0a934de61 100644
--- a/crates/misc/component-async-tests/tests/scenario/streams.rs
+++ b/crates/misc/component-async-tests/tests/scenario/streams.rs
@@ -6,18 +6,107 @@ use {
util::{OneshotConsumer, OneshotProducer, PipeConsumer, PipeProducer},
},
futures::{
- SinkExt, StreamExt,
+ Sink, SinkExt, Stream, StreamExt,
channel::{mpsc, oneshot},
future,
},
- std::sync::{Arc, Mutex},
+ std::{
+ pin::Pin,
+ sync::{Arc, Mutex},
+ task::{Context, Poll},
+ },
wasmtime::{
- Engine, Store,
- component::{FutureReader, Linker, ResourceTable, StreamReader},
+ Engine, Store, StoreContextMut,
+ component::{
+ Destination, FutureReader, Linker, ResourceTable, Source, StreamConsumer,
+ StreamProducer, StreamReader, StreamResult,
+ },
},
wasmtime_wasi::WasiCtxBuilder,
};
+pub struct DirectPipeProducer<S>(S);
+
+impl<D, S: Stream<Item = u8> + Send + 'static> StreamProducer<D> for DirectPipeProducer<S> {
+ type Item = u8;
+ type Buffer = Option<u8>;
+
+ fn poll_produce<'a>(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ store: StoreContextMut<D>,
+ destination: Destination<'a, Self::Item, Self::Buffer>,
+ finish: bool,
+ ) -> Poll<Result<StreamResult>> {
+ // SAFETY: This is a standard pin-projection, and we never move
+ // out of `self`.
+ let stream = unsafe { self.map_unchecked_mut(|v| &mut v.0) };
+
+ match stream.poll_next(cx) {
+ Poll::Pending => {
+ if finish {
+ Poll::Ready(Ok(StreamResult::Cancelled))
+ } else {
+ Poll::Pending
+ }
+ }
+ Poll::Ready(Some(item)) => {
+ let mut destination = destination.as_direct(store, 1);
+ destination.remaining()[0] = item;
+ destination.mark_written(1);
+ Poll::Ready(Ok(StreamResult::Completed))
+ }
+ Poll::Ready(None) => Poll::Ready(Ok(StreamResult::Dropped)),
+ }
+ }
+}
+
+pub struct DirectPipeConsumer<S>(S);
+
+impl<D, S: Sink<u8, Error: std::error::Error + Send + Sync> + Send + 'static> StreamConsumer<D>
+ for DirectPipeConsumer<S>
+{
+ type Item = u8;
+
+ fn poll_consume(
+ self: Pin<&mut Self>,
+ cx: &mut Context<'_>,
+ store: StoreContextMut<D>,
+ source: Source<Self::Item>,
+ finish: bool,
+ ) -> Poll<Result<StreamResult>> {
+ // SAFETY: This is a standard pin-projection, and we never move
+ // out of `self`.
+ let mut sink = unsafe { self.map_unchecked_mut(|v| &mut v.0) };
+
+ let on_pending = || {
+ if finish {
+ Poll::Ready(Ok(StreamResult::Cancelled))
+ } else {
+ Poll::Pending
+ }
+ };
+
+ match sink.as_mut().poll_flush(cx) {
+ Poll::Pending => on_pending(),
+ Poll::Ready(result) => {
+ result?;
+ match sink.as_mut().poll_ready(cx) {
+ Poll::Pending => on_pending(),
+ Poll::Ready(result) => {
+ result?;
+ let mut source = source.as_direct(store);
+ let item = source.remaining()[0];
+ source.mark_read(1);
+ sink.start_send(item)?;
+ Poll::Ready(Ok(StreamResult::Completed))
+ }
+ }
+ }
+ }
+ }
+}
+
#[tokio::test]
pub async fn async_closed_streams() -> Result<()> {
let engine = Engine::new(&config())?;
@@ -49,35 +138,45 @@ pub async fn async_closed_streams() -> Result<()> {
let value = 42_u8;
// First, test stream host->host
- {
- let (mut input_tx, input_rx) = mpsc::channel(1);
- let (output_tx, mut output_rx) = mpsc::channel(1);
- StreamReader::new(instance, &mut store, PipeProducer::new(input_rx))
- .pipe(&mut store, PipeConsumer::new(output_tx));
-
- instance
- .run_concurrent(&mut store, async |_| {
- let (a, b) = future::join(
- async {
- for &value in &values {
- input_tx.send(value).await?;
- }
- drop(input_tx);
- anyhow::Ok(())
- },
- async {
- for &value in &values {
- assert_eq!(Some(value), output_rx.next().await);
- }
- assert!(output_rx.next().await.is_none());
- Ok(())
- },
- )
- .await;
-
- a.and(b)
- })
- .await??;
+ for direct_producer in [true, false] {
+ for direct_consumer in [true, false] {
+ let (mut input_tx, input_rx) = mpsc::channel(1);
+ let (output_tx, mut output_rx) = mpsc::channel(1);
+ let reader = if direct_producer {
+ StreamReader::new(instance, &mut store, DirectPipeProducer(input_rx))
+ } else {
+ StreamReader::new(instance, &mut store, PipeProducer::new(input_rx))
+ };
+ if direct_consumer {
+ reader.pipe(&mut store, DirectPipeConsumer(output_tx));
+ } else {
+ reader.pipe(&mut store, PipeConsumer::new(output_tx));
+ }
+
+ instance
+ .run_concurrent(&mut store, async |_| {
+ let (a, b) = future::join(
+ async {
+ for &value in &values {
+ input_tx.send(value).await?;
+ }
+ drop(input_tx);
+ anyhow::Ok(())
+ },
+ async {
+ for &value in &values {
+ assert_eq!(Some(value), output_rx.next().await);
+ }
+ assert!(output_rx.next().await.is_none());
+ Ok(())
+ },
+ )
+ .await;
+
+ a.and(b)
+ })
+ .await??;
+ }
}
// Next, test futures host->host
diff --git a/crates/misc/component-async-tests/tests/scenario/transmit.rs b/crates/misc/component-async-tests/tests/scenario/transmit.rs
index 80de705e73..77a0cfc98f 100644
--- a/crates/misc/component-async-tests/tests/scenario/transmit.rs
+++ b/crates/misc/component-async-tests/tests/scenario/transmit.rs
@@ -42,7 +42,7 @@ impl<D> StreamProducer<D> for ReadinessProducer {
self: Pin<&mut Self>,
cx: &mut Context<'_>,
mut store: StoreContextMut<'a, D>,
- destination: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ destination: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<Result<StreamResult>> {
let me = self.get_mut();
@@ -62,7 +62,7 @@ impl<D> StreamProducer<D> for ReadinessProducer {
Poll::Ready(Ok(StreamResult::Completed))
} else {
assert_eq!(capacity, Some(me.buffer.len()));
- let mut destination = destination.as_direct_destination(store).unwrap();
+ let mut destination = destination.as_direct(store, me.buffer.len());
destination.remaining().copy_from_slice(&me.buffer);
destination.mark_written(me.buffer.len());
@@ -85,7 +85,7 @@ impl<D> StreamConsumer<D> for ReadinessConsumer {
self: Pin<&mut Self>,
cx: &mut Context<'_>,
mut store: StoreContextMut<D>,
- source: &mut Source<Self::Item>,
+ source: Source<Self::Item>,
finish: bool,
) -> Poll<Result<StreamResult>> {
let me = self.get_mut();
@@ -105,7 +105,7 @@ impl<D> StreamConsumer<D> for ReadinessConsumer {
Poll::Ready(Ok(StreamResult::Completed))
} else {
assert_eq!(available, me.expected.len());
- let mut source = source.as_direct_source(store);
+ let mut source = source.as_direct(store);
assert_eq!(&me.expected, source.remaining());
source.mark_read(me.expected.len());
diff --git a/crates/wasi/src/p3/cli/host.rs b/crates/wasi/src/p3/cli/host.rs
index 3915f5754c..42756263e9 100644
--- a/crates/wasi/src/p3/cli/host.rs
+++ b/crates/wasi/src/p3/cli/host.rs
@@ -12,11 +12,11 @@ use core::pin::Pin;
use core::task::{Context, Poll};
use std::io::Cursor;
use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
-use wasmtime::StoreContextMut;
use wasmtime::component::{
Accessor, Destination, Resource, Source, StreamConsumer, StreamProducer, StreamReader,
StreamResult,
};
+use wasmtime::{AsContextMut as _, StoreContextMut};
struct InputStreamProducer {
rx: Pin<Box<dyn AsyncRead + Send + Sync>>,
@@ -29,54 +29,32 @@ impl<D> StreamProducer<D> for InputStreamProducer {
fn poll_produce<'a>(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
- store: StoreContextMut<'a, D>,
- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ mut store: StoreContextMut<'a, D>,
+ dst: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
- if let Some(mut dst) = dst.as_direct_destination(store) {
- if !dst.remaining().is_empty() {
- let mut buf = ReadBuf::new(dst.remaining());
- match self.rx.as_mut().poll_read(cx, &mut buf) {
- Poll::Ready(Ok(())) if buf.filled().is_empty() => {
- return Poll::Ready(Ok(StreamResult::Dropped));
- }
- Poll::Ready(Ok(())) => {
- let n = buf.filled().len();
- dst.mark_written(n);
- return Poll::Ready(Ok(StreamResult::Completed));
- }
- Poll::Ready(Err(..)) => {
- // TODO: Report the error to the guest
- return Poll::Ready(Ok(StreamResult::Dropped));
- }
- Poll::Pending if finish => return Poll::Ready(Ok(StreamResult::Cancelled)),
- Poll::Pending => return Poll::Pending,
+ if let Some(0) = dst.remaining(store.as_context_mut()) {
+ Poll::Ready(Ok(StreamResult::Completed))
+ } else {
+ let mut dst = dst.as_direct(store, DEFAULT_BUFFER_CAPACITY);
+ let mut buf = ReadBuf::new(dst.remaining());
+ match self.rx.as_mut().poll_read(cx, &mut buf) {
+ Poll::Ready(Ok(())) if buf.filled().is_empty() => {
+ Poll::Ready(Ok(StreamResult::Dropped))
}
+ Poll::Ready(Ok(())) => {
+ let n = buf.filled().len();
+ dst.mark_written(n);
+ Poll::Ready(Ok(StreamResult::Completed))
+ }
+ Poll::Ready(Err(..)) => {
+ // TODO: Report the error to the guest
+ Poll::Ready(Ok(StreamResult::Dropped))
+ }
+ Poll::Pending if finish => Poll::Ready(Ok(StreamResult::Cancelled)),
+ Poll::Pending => Poll::Pending,
}
}
- let mut buf = dst.take_buffer().into_inner();
- buf.clear();
- buf.reserve(DEFAULT_BUFFER_CAPACITY);
- let mut rbuf = ReadBuf::uninit(buf.spare_capacity_mut());
- match self.rx.as_mut().poll_read(cx, &mut rbuf) {
- Poll::Ready(Ok(())) if rbuf.filled().is_empty() => {
- Poll::Ready(Ok(StreamResult::Dropped))
- }
- Poll::Ready(Ok(())) => {
- let n = rbuf.filled().len();
- // SAFETY: `ReadyBuf::filled` promised us `count` bytes have
- // been initialized.
- unsafe { buf.set_len(n) };
- dst.set_buffer(Cursor::new(buf));
- Poll::Ready(Ok(StreamResult::Completed))
- }
- Poll::Ready(Err(..)) => {
- // TODO: Report the error to the guest
- Poll::Ready(Ok(StreamResult::Dropped))
- }
- Poll::Pending if finish => Poll::Ready(Ok(StreamResult::Cancelled)),
- Poll::Pending => Poll::Pending,
- }
}
}
@@ -91,10 +69,10 @@ impl<D> StreamConsumer<D> for OutputStreamConsumer {
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<D>,
- src: &mut Source<Self::Item>,
+ src: Source<Self::Item>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
- let mut src = src.as_direct_source(store);
+ let mut src = src.as_direct(store);
let buf = src.remaining();
match self.tx.as_mut().poll_write(cx, buf) {
Poll::Ready(Ok(n)) if buf.is_empty() => {
diff --git a/crates/wasi/src/p3/filesystem/host.rs b/crates/wasi/src/p3/filesystem/host.rs
index a0ffe530e6..763040934b 100644
--- a/crates/wasi/src/p3/filesystem/host.rs
+++ b/crates/wasi/src/p3/filesystem/host.rs
@@ -145,7 +145,7 @@ impl<D> StreamProducer<D> for ReadStreamProducer {
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<'a, D>,
- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ mut dst: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
if let Some(task) = self.task.as_mut() {
@@ -183,58 +183,33 @@ impl<D> StreamProducer<D> for ReadStreamProducer {
return Poll::Ready(Ok(StreamResult::Cancelled));
}
if let Some(file) = self.file.as_blocking_file() {
- if let Some(mut dst) = dst.as_direct_destination(store) {
- let buf = dst.remaining();
- if !buf.is_empty() {
- match file.read_at(buf, self.offset) {
- Ok(0) => {
- self.close(Ok(()));
+ let mut dst = dst.as_direct(store, DEFAULT_BUFFER_CAPACITY);
+ let buf = dst.remaining();
+ if buf.is_empty() {
+ return Poll::Ready(Ok(StreamResult::Completed));
+ } else {
+ match file.read_at(buf, self.offset) {
+ Ok(0) => {
+ self.close(Ok(()));
+ return Poll::Ready(Ok(StreamResult::Dropped));
+ }
+ Ok(n) => {
+ dst.mark_written(n);
+ let Ok(n) = n.try_into() else {
+ self.close(Err(ErrorCode::Overflow));
return Poll::Ready(Ok(StreamResult::Dropped));
- }
- Ok(n) => {
- dst.mark_written(n);
- let Ok(n) = n.try_into() else {
- self.close(Err(ErrorCode::Overflow));
- return Poll::Ready(Ok(StreamResult::Dropped));
- };
- let Some(n) = self.offset.checked_add(n) else {
- self.close(Err(ErrorCode::Overflow));
- return Poll::Ready(Ok(StreamResult::Dropped));
- };
- self.offset = n;
- return Poll::Ready(Ok(StreamResult::Completed));
- }
- Err(err) => {
- self.close(Err(err.into()));
+ };
+ let Some(n) = self.offset.checked_add(n) else {
+ self.close(Err(ErrorCode::Overflow));
return Poll::Ready(Ok(StreamResult::Dropped));
- }
+ };
+ self.offset = n;
+ return Poll::Ready(Ok(StreamResult::Completed));
}
- }
- }
- let mut buf = dst.take_buffer().into_inner();
- buf.resize(DEFAULT_BUFFER_CAPACITY, 0);
- match file.read_at(&mut buf, self.offset) {
- Ok(0) => {
- self.close(Ok(()));
- return Poll::Ready(Ok(StreamResult::Dropped));
- }
- Ok(n) => {
- buf.truncate(n);
- dst.set_buffer(Cursor::new(buf));
- let Ok(n) = n.try_into() else {
- self.close(Err(ErrorCode::Overflow));
+ Err(err) => {
+ self.close(Err(err.into()));
return Poll::Ready(Ok(StreamResult::Dropped));
- };
- let Some(n) = self.offset.checked_add(n) else {
- self.close(Err(ErrorCode::Overflow));
- return Poll::Ready(Ok(StreamResult::Dropped));
- };
- self.offset = n;
- return Poll::Ready(Ok(StreamResult::Completed));
- }
- Err(err) => {
- self.close(Err(err.into()));
- return Poll::Ready(Ok(StreamResult::Dropped));
+ }
}
}
}
@@ -343,7 +318,7 @@ impl<D> StreamProducer<D> for BlockingDirectoryStreamProducer {
mut self: Pin<&mut Self>,
_: &mut Context<'_>,
_: StoreContextMut<'a, D>,
- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ mut dst: Destination<'a, Self::Item, Self::Buffer>,
_finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
let entries = match self.dir.entries() {
@@ -407,7 +382,7 @@ impl<D> StreamProducer<D> for NonblockingDirectoryStreamProducer {
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<'a, D>,
- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ mut dst: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
match mem::replace(&mut self.0, DirStreamState::Closed) {
@@ -493,10 +468,10 @@ impl<D> StreamConsumer<D> for WriteStreamConsumer {
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<D>,
- src: &mut Source<Self::Item>,
+ src: Source<Self::Item>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
- let mut src = src.as_direct_source(store);
+ let mut src = src.as_direct(store);
if let Some(task) = self.task.as_mut() {
let res = ready!(Pin::new(task).poll(cx));
self.task = None;
@@ -615,10 +590,10 @@ impl<D> StreamConsumer<D> for AppendStreamConsumer {
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<D>,
- src: &mut Source<Self::Item>,
+ src: Source<Self::Item>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
- let mut src = src.as_direct_source(store);
+ let mut src = src.as_direct(store);
if let Some(task) = self.task.as_mut() {
let res = ready!(Pin::new(task).poll(cx));
self.task = None;
diff --git a/crates/wasi/src/p3/mod.rs b/crates/wasi/src/p3/mod.rs
index c3cdf63117..67c2b6699c 100644
--- a/crates/wasi/src/p3/mod.rs
+++ b/crates/wasi/src/p3/mod.rs
@@ -46,7 +46,7 @@ impl<T: Send + Sync + 'static, D> StreamProducer<D> for StreamEmptyProducer<T> {
self: Pin<&mut Self>,
_: &mut Context<'_>,
_: StoreContextMut<'a, D>,
- _: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ _: Destination<'a, Self::Item, Self::Buffer>,
_: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
Poll::Ready(Ok(StreamResult::Dropped))
diff --git a/crates/wasi/src/p3/sockets/host/types/tcp.rs b/crates/wasi/src/p3/sockets/host/types/tcp.rs
index 0b755cde13..6dc5caee8f 100644
--- a/crates/wasi/src/p3/sockets/host/types/tcp.rs
+++ b/crates/wasi/src/p3/sockets/host/types/tcp.rs
@@ -18,11 +18,11 @@ use std::net::{Shutdown, SocketAddr};
use std::sync::Arc;
use tokio::net::{TcpListener, TcpStream};
use tokio::sync::oneshot;
-use wasmtime::StoreContextMut;
use wasmtime::component::{
Accessor, Destination, FutureReader, Resource, ResourceTable, Source, StreamConsumer,
StreamProducer, StreamReader, StreamResult,
};
+use wasmtime::{AsContextMut as _, StoreContextMut};
fn get_socket<'a>(
table: &'a ResourceTable,
@@ -62,7 +62,7 @@ where
self: Pin<&mut Self>,
cx: &mut Context<'_>,
mut store: StoreContextMut<'a, D>,
- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ mut dst: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
let res = match self.listener.poll_accept(cx) {
@@ -111,58 +111,42 @@ impl<D> StreamProducer<D> for ReceiveStreamProducer {
fn poll_produce<'a>(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
- store: StoreContextMut<'a, D>,
- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ mut store: StoreContextMut<'a, D>,
+ dst: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
let res = 'result: {
- if let Some(mut dst) = dst.as_direct_destination(store) {
- let buf = dst.remaining();
- if !buf.is_empty() {
- loop {
- match self.stream.try_read(buf) {
- Ok(0) => break 'result Ok(()),
- Ok(n) => {
- dst.mark_written(n);
- return Poll::Ready(Ok(StreamResult::Completed));
- }
- Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
- match self.stream.poll_read_ready(cx) {
- Poll::Ready(Ok(())) => continue,
- Poll::Ready(Err(err)) => break 'result Err(err.into()),
- Poll::Pending if finish => {
- return Poll::Ready(Ok(StreamResult::Cancelled));
- }
- Poll::Pending => return Poll::Pending,
- }
- }
- Err(err) => break 'result Err(err.into()),
- }
+ if let Some(0) = dst.remaining(store.as_context_mut()) {
+ match self.stream.poll_read_ready(cx) {
+ Poll::Ready(Ok(())) => return Poll::Ready(Ok(StreamResult::Completed)),
+ Poll::Ready(Err(err)) => break 'result Err(err.into()),
+ Poll::Pending if finish => {
+ return Poll::Ready(Ok(StreamResult::Cancelled));
}
+ Poll::Pending => return Poll::Pending,
}
- }
-
- let mut buf = dst.take_buffer().into_inner();
- buf.clear();
- buf.reserve(DEFAULT_BUFFER_CAPACITY);
- loop {
- match self.stream.try_read_buf(&mut buf) {
- Ok(0) => break 'result Ok(()),
- Ok(..) => {
- dst.set_buffer(Cursor::new(buf));
- return Poll::Ready(Ok(StreamResult::Completed));
- }
- Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
- match self.stream.poll_read_ready(cx) {
- Poll::Ready(Ok(())) => continue,
- Poll::Ready(Err(err)) => break 'result Err(err.into()),
- Poll::Pending if finish => {
- return Poll::Ready(Ok(StreamResult::Cancelled));
+ } else {
+ let mut dst = dst.as_direct(store, DEFAULT_BUFFER_CAPACITY);
+ let buf = dst.remaining();
+ loop {
+ match self.stream.try_read(buf) {
+ Ok(0) => break 'result Ok(()),
+ Ok(n) => {
+ dst.mark_written(n);
+ return Poll::Ready(Ok(StreamResult::Completed));
+ }
+ Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {
+ match self.stream.poll_read_ready(cx) {
+ Poll::Ready(Ok(())) => continue,
+ Poll::Ready(Err(err)) => break 'result Err(err.into()),
+ Poll::Pending if finish => {
+ return Poll::Ready(Ok(StreamResult::Cancelled));
+ }
+ Poll::Pending => return Poll::Pending,
}
- Poll::Pending => return Poll::Pending,
}
+ Err(err) => break 'result Err(err.into()),
}
- Err(err) => break 'result Err(err.into()),
}
}
};
@@ -201,10 +185,10 @@ impl<D> StreamConsumer<D> for SendStreamConsumer {
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<D>,
- src: &mut Source<Self::Item>,
+ src: Source<Self::Item>,
finish: bool,
) -> Poll<wasmtime::Result<StreamResult>> {
- let mut src = src.as_direct_source(store);
+ let mut src = src.as_direct(store);
let res = 'result: {
if src.remaining().is_empty() {
match self.stream.poll_write_ready(cx) {
diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs
index 9ae9dfc5e5..9bc43d25fe 100644
--- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs
+++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs
@@ -10,13 +10,13 @@ use crate::vm::component::{ComponentInstance, HandleTable, TransmitLocalState};
use crate::vm::{AlwaysMut, VMStore};
use crate::{AsContextMut, StoreContextMut, ValRaw};
use anyhow::{Context as _, Result, anyhow, bail};
-use buffers::Extender;
-use buffers::UntypedWriteBuffer;
+use buffers::{Extender, SliceBuffer, UntypedWriteBuffer};
use futures::FutureExt;
use futures::channel::oneshot;
use std::boxed::Box;
use std::fmt;
use std::future;
+use std::io::Cursor;
use std::iter;
use std::marker::PhantomData;
use std::mem::{self, MaybeUninit};
@@ -218,6 +218,7 @@ pub struct Destination<'a, T, B> {
instance: Instance,
id: TableId<TransmitState>,
buffer: &'a mut B,
+ host_buffer: Option<&'a mut Cursor<Vec<u8>>>,
_phantom: PhantomData<fn() -> T>,
}
@@ -240,9 +241,9 @@ impl<'a, T, B> Destination<'a, T, B> {
/// the `StreamProducer::poll_produce` call to which this `Destination` was
/// passed returns (unless overwritten by another call to `set_buffer`).
///
- /// If items are stored via a buffer _and_ written via a `DirectDestination`
- /// view of `self`, then the items in the buffer will be delivered after the
- /// ones written using `DirectDestination`.
+ /// If items are stored via this buffer _and_ written via a
+ /// `DirectDestination` view of `self`, then the items in the buffer will be
+ /// delivered after the ones written using `DirectDestination`.
pub fn set_buffer(&mut self, buffer: B) {
*self.buffer = buffer;
}
@@ -279,25 +280,33 @@ impl<'a, T, B> Destination<'a, T, B> {
}
impl<'a, B> Destination<'a, u8, B> {
- /// Return a `DirectDestination` view of `self` if the guest is reading.
- pub fn as_direct_destination<D>(
- &mut self,
+ /// Return a `DirectDestination` view of `self`.
+ ///
+ /// If the reader is a guest, this will provide direct access to the guest's
+ /// read buffer. If the reader is a host, this will provide access to a
+ /// buffer which will be delivered to the host before any items stored using
+ /// `Destination::set_buffer`.
+ ///
+ /// `capacity` will only be used if the reader is a host, in which case it
+ /// will update the length of the buffer, possibly zero-initializing the new
+ /// elements if the new length is larger than the old length.
+ pub fn as_direct<D>(
+ mut self,
store: StoreContextMut<'a, D>,
- ) -> Option<DirectDestination<'a, D>> {
- if let ReadState::GuestReady { .. } = self
- .instance
- .concurrent_state_mut(store.0)
- .get_mut(self.id)
- .unwrap()
- .read
- {
- Some(DirectDestination {
- instance: self.instance,
- id: self.id,
- store,
- })
- } else {
- None
+ capacity: usize,
+ ) -> DirectDestination<'a, D> {
+ if let Some(buffer) = self.host_buffer.as_deref_mut() {
+ buffer.set_position(0);
+ if buffer.get_mut().is_empty() {
+ buffer.get_mut().resize(capacity, 0);
+ }
+ }
+
+ DirectDestination {
+ instance: self.instance,
+ id: self.id,
+ host_buffer: self.host_buffer,
+ store,
}
}
}
@@ -307,37 +316,42 @@ impl<'a, B> Destination<'a, u8, B> {
pub struct DirectDestination<'a, D: 'static> {
instance: Instance,
id: TableId<TransmitState>,
+ host_buffer: Option<&'a mut Cursor<Vec<u8>>>,
store: StoreContextMut<'a, D>,
}
impl<D: 'static> DirectDestination<'_, D> {
/// Provide direct access to the writer's buffer.
pub fn remaining(&mut self) -> &mut [u8] {
- let transmit = self
- .instance
- .concurrent_state_mut(self.store.as_context_mut().0)
- .get_mut(self.id)
- .unwrap();
+ if let Some(buffer) = self.host_buffer.as_deref_mut() {
+ buffer.get_mut()
+ } else {
+ let transmit = self
+ .instance
+ .concurrent_state_mut(self.store.as_context_mut().0)
+ .get_mut(self.id)
+ .unwrap();
- let &ReadState::GuestReady {
- address,
- count,
- options,
- ..
- } = &transmit.read
- else {
- unreachable!()
- };
+ let &ReadState::GuestReady {
+ address,
+ count,
+ options,
+ ..
+ } = &transmit.read
+ else {
+ unreachable!();
+ };
- let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {
- unreachable!()
- };
+ let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {
+ unreachable!()
+ };
- options
- .memory_mut(self.store.0)
- .get_mut((address + guest_offset)..)
- .and_then(|b| b.get_mut(..(count - guest_offset)))
- .unwrap()
+ options
+ .memory_mut(self.store.0)
+ .get_mut((address + guest_offset)..)
+ .and_then(|b| b.get_mut(..(count - guest_offset)))
+ .unwrap()
+ }
}
/// Mark the specified number of bytes as written to the writer's buffer.
@@ -345,27 +359,38 @@ impl<D: 'static> DirectDestination<'_, D> {
/// This will panic if the count is larger than the size of the
/// buffer returned by `Self::remaining`.
pub fn mark_written(&mut self, count: usize) {
- let transmit = self
- .instance
- .concurrent_state_mut(self.store.as_context_mut().0)
- .get_mut(self.id)
- .unwrap();
+ if let Some(buffer) = self.host_buffer.as_deref_mut() {
+ buffer.set_position(
+ buffer
+ .position()
+ .checked_add(u64::try_from(count).unwrap())
+ .unwrap(),
+ );
+ } else {
+ let transmit = self
+ .instance
+ .concurrent_state_mut(self.store.as_context_mut().0)
+ .get_mut(self.id)
+ .unwrap();
- let ReadState::GuestReady {
- count: read_count, ..
- } = &transmit.read
- else {
- unreachable!()
- };
+ let ReadState::GuestReady {
+ count: read_count, ..
+ } = &transmit.read
+ else {
+ unreachable!();
+ };
- let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {
- unreachable!()
- };
+ let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {
+ unreachable!()
+ };
- if *guest_offset + count > *read_count {
- panic!("write count ({count}) must be less than or equal to read count ({read_count})")
- } else {
- *guest_offset += count;
+ if *guest_offset + count > *read_count {
+ panic!(
+ "write count ({count}) must be less than or equal to read count ({read_count})"
+ )
+ } else {
+ *guest_offset += count;
+ }
}
}
}
@@ -443,21 +468,32 @@ pub trait StreamProducer<D>: Send + 'static {
/// If more items are written to `destination` than the reader has immediate
/// capacity to accept, they will be retained in memory by the caller and
/// used to satisify future reads, in which case `poll_produce` will only be
- /// called again once all those items have been delivered. This is
- /// particularly important for zero-length reads, in which case the
- /// implementation is expected to either:
+ /// called again once all those items have been delivered.
+ ///
+ /// If this function is called with zero capacity
+ /// (i.e. `Destination::remaining` returns `Some(0)`), the implementation
+ /// should either:
///
- /// 1. Produce at least one item (if possible, and if `finish` is false) so
- /// that it is ready to be delivered immediately upon the next
- /// non-zero-length read.
+ /// - Return `Poll::Ready(Ok(StreamResult::Completed))` without writing
+ /// anything if it expects to be able to produce items immediately
+ /// (i.e. without first returning `Poll::Pending`) the next time
+ /// `poll_produce` is called with non-zero capacity _or_ if that cannot be
+ /// reliably determined.
///
- /// 2. Produce at least one item the next time `poll_produce` is called with
- /// non-zero capacity and `finish` set to false.
+ /// - Return `Poll::Pending` if the next call to `poll_produce` with
+ /// non-zero capacity is likely to also return `Poll::Pending`.
+ ///
+ /// - Return `Poll::Ready(Ok(StreamResult::Completed))` after calling
+ /// `Destination::set_buffer` with one more more items. Note, however, that
+ /// this creates the hazard that the items will never be received by the
+ /// guest if it decides not to do another non-zero-length read before
+ /// closing the stream. Moreover, if `Self::Item` is e.g. a `Resource<_>`,
+ /// they may end up leaking in that scenario.
fn poll_produce<'a>(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<'a, D>,
- destination: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ destination: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<Result<StreamResult>>;
}
@@ -553,15 +589,13 @@ impl<T> Source<'_, T> {
}
}
-impl Source<'_, u8> {
+impl<'a> Source<'a, u8> {
/// Return a `DirectSource` view of `self`.
- pub fn as_direct_source<'a, D>(
- &mut self,
- store: StoreContextMut<'a, D>,
- ) -> DirectSource<'a, D> {
+ pub fn as_direct<D>(self, store: StoreContextMut<'a, D>) -> DirectSource<'a, D> {
DirectSource {
instance: self.instance,
id: self.id,
+ host_buffer: self.host_buffer,
store,
}
}
@@ -572,37 +606,42 @@ impl Source<'_, u8> {
pub struct DirectSource<'a, D: 'static> {
instance: Instance,
id: TableId<TransmitState>,
+ host_buffer: Option<&'a mut dyn WriteBuffer<u8>>,
store: StoreContextMut<'a, D>,
}
impl<D: 'static> DirectSource<'_, D> {
/// Provide direct access to the writer's buffer.
pub fn remaining(&mut self) -> &[u8] {
- let transmit = self
- .instance
- .concurrent_state_mut(self.store.as_context_mut().0)
- .get_mut(self.id)
- .unwrap();
+ if let Some(buffer) = self.host_buffer.as_deref_mut() {
+ buffer.remaining()
+ } else {
+ let transmit = self
+ .instance
+ .concurrent_state_mut(self.store.as_context_mut().0)
+ .get_mut(self.id)
+ .unwrap();
- let &WriteState::GuestReady {
- address,
- count,
- options,
- ..
- } = &transmit.write
- else {
- unreachable!()
- };
+ let &WriteState::GuestReady {
+ address,
+ count,
+ options,
+ ..
+ } = &transmit.write
+ else {
+ unreachable!()
+ };
- let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
- unreachable!()
- };
+ let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
+ unreachable!()
+ };
- options
- .memory(self.store.0)
- .get((address + guest_offset)..)
- .and_then(|b| b.get(..(count - guest_offset)))
- .unwrap()
+ options
+ .memory(self.store.0)
+ .get((address + guest_offset)..)
+ .and_then(|b| b.get(..(count - guest_offset)))
+ .unwrap()
+ }
}
/// Mark the specified number of bytes as read from the writer's buffer.
@@ -610,27 +649,33 @@ impl<D: 'static> DirectSource<'_, D> {
/// This will panic if the count is larger than the size of the buffer
/// returned by `Self::remaining`.
pub fn mark_read(&mut self, count: usize) {
- let transmit = self
- .instance
- .concurrent_state_mut(self.store.as_context_mut().0)
- .get_mut(self.id)
- .unwrap();
+ if let Some(buffer) = self.host_buffer.as_deref_mut() {
+ buffer.skip(count);
+ } else {
+ let transmit = self
+ .instance
+ .concurrent_state_mut(self.store.as_context_mut().0)
+ .get_mut(self.id)
+ .unwrap();
- let WriteState::GuestReady {
- count: write_count, ..
- } = &transmit.write
- else {
- unreachable!()
- };
+ let WriteState::GuestReady {
+ count: write_count, ..
+ } = &transmit.write
+ else {
+ unreachable!()
+ };
- let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {
- unreachable!()
- };
+ let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {
+ unreachable!()
+ };
- if *guest_offset + count > *write_count {
- panic!("read count ({count}) must be less than or equal to write count ({write_count})")
- } else {
- *guest_offset += count;
+ if *guest_offset + count > *write_count {
+ panic!(
+ "read count ({count}) must be less than or equal to write count ({write_count})"
+ )
+ } else {
+ *guest_offset += count;
+ }
}
}
}
@@ -726,7 +771,7 @@ pub trait StreamConsumer<D>: Send + 'static {
self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<D>,
- source: &mut Source<Self::Item>,
+ source: Source<'_, Self::Item>,
finish: bool,
) -> Poll<Result<StreamResult>>;
}
@@ -787,7 +832,7 @@ impl<T> FutureReader<T> {
self: Pin<&mut Self>,
cx: &mut Context<'_>,
store: StoreContextMut<D>,
- destination: &'a mut Destination<'a, Self::Item, Self::Buffer>,
+ mut destination: Destination<'a, Self::Item, Self::Buffer>,
finish: bool,
) -> Poll<Result<StreamResult>> {
// SAFETY: This is a standard pin-projection, and we never move
@@ -864,7 +909,7 @@ impl<T> FutureReader<T> {
self: Pin<&mut Self>,
cx: &mut Context<'_>,
mut store: StoreContextMut<D>,
- source: &mut Source<Self::Item>,
+ mut source: Source<Self::Item>,
finish: bool,
) -> Poll<Result<StreamResult>> {
let me = self.get_mut();
@@ -1681,6 +1726,8 @@ enum ReadState {
+ Send
+ Sync,
>,
+ buffer: Vec<u8>,
+ limit: usize,
},
/// The read end has been dropped.
Dropped,
@@ -1731,41 +1778,62 @@ impl Instance {
let (result, cancelled) = if buffer.remaining().is_empty() {
future::poll_fn(|cx| {
tls::get(|store| {
- let &WriteState::HostReady { cancel, .. } =
- &self.concurrent_state_mut(store).get_mut(id).unwrap().write
- else {
+ let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();
+
+ let &WriteState::HostReady { cancel, .. } = &transmit.write else {
unreachable!();
};
+ let mut host_buffer =
+ if let ReadState::HostToHost { buffer, .. } = &mut transmit.read {
+ Some(Cursor::new(mem::take(buffer)))
+ } else {
+ None
+ };
+
let poll = mine.as_mut().poll_produce(
cx,
token.as_context_mut(store),
- &mut Destination {
+ Destination {
instance: self,
id,
buffer: &mut buffer,
+ host_buffer: host_buffer.as_mut(),
_phantom: PhantomData,
},
cancel,
);
+ let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();
+
+ let host_offset = if let (
+ Some(host_buffer),
+ ReadState::HostToHost { buffer, limit, .. },
+ ) = (host_buffer, &mut transmit.read)
+ {
+ *limit = usize::try_from(host_buffer.position()).unwrap();
+ *buffer = host_buffer.into_inner();
+ *limit
+ } else {
+ 0
+ };
+
{
let WriteState::HostReady {
guest_offset,
cancel,
cancel_waker,
..
- } = &mut self
- .concurrent_state_mut(store)
- .get_mut(id)
- .unwrap()
- .write
+ } = &mut transmit.write
else {
unreachable!();
};
if let Poll::Pending = &poll {
- if !buffer.remaining().is_empty() || *guest_offset > 0 {
+ if !buffer.remaining().is_empty()
+ || *guest_offset > 0
+ || host_offset > 0
+ {
return Poll::Ready(Err(anyhow!(
"StreamProducer::poll_produce returned Poll::Pending \
after producing at least one item"
@@ -1787,24 +1855,27 @@ impl Instance {
(StreamResult::Completed, false)
};
- let (guest_offset, count) = tls::get(|store| {
+ let (guest_offset, host_offset, count) = tls::get(|store| {
let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();
- (
- match &transmit.write {
- &WriteState::HostReady { guest_offset, .. } => guest_offset,
- _ => unreachable!(),
- },
- match &transmit.read {
- &ReadState::GuestReady { count, .. } => count,
- ReadState::HostToHost { .. } => 1,
- _ => unreachable!(),
- },
- )
+ let (count, host_offset) = match &transmit.read {
+ &ReadState::GuestReady { count, .. } => (count, 0),
+ &ReadState::HostToHost { limit, .. } => (1, limit),
+ _ => unreachable!(),
+ };
+ let guest_offset = match &transmit.write {
+ &WriteState::HostReady { guest_offset, .. } => guest_offset,
+ _ => unreachable!(),
+ };
+ (guest_offset, host_offset, count)
});
match result {
StreamResult::Completed => {
- if count > 1 && buffer.remaining().is_empty() && guest_offset == 0 {
+ if count > 1
+ && buffer.remaining().is_empty()
+ && guest_offset == 0
+ && host_offset == 0
+ {
bail!(
"StreamProducer::poll_produce returned StreamResult::Completed \
without producing any items"
@@ -1822,11 +1893,11 @@ impl Instance {
StreamResult::Dropped => {}
}
- let write = !buffer.remaining().is_empty();
+ let write_buffer = !buffer.remaining().is_empty() || host_offset > 0;
*producer.lock().unwrap() = Some((mine, buffer));
- if write {
+ if write_buffer {
self.write(token, id, producer, kind).await?;
}
@@ -1876,7 +1947,7 @@ impl Instance {
let poll = mine.as_mut().poll_consume(
cx,
token.as_context_mut(store),
- &mut Source {
+ Source {
instance: self,
id,
host_buffer: host_buffer.as_deref_mut(),
@@ -1995,6 +2066,8 @@ impl Instance {
let consume = consume_with_buffer.clone();
async move { consume(Some(input.get_mut::<C::Item>())).await }.boxed()
}),
+ buffer: Vec::new(),
+ limit: 0,
};
let future = async move {
@@ -2141,18 +2214,40 @@ impl Instance {
Ok(())
}
- ReadState::HostToHost { accept } => {
- let (mine, mut buffer) = pair.lock().unwrap().take().unwrap();
+ ReadState::HostToHost {
+ accept,
+ mut buffer,
+ limit,
+ } => {
+ let mut state = StreamResult::Completed;
+ let mut position = 0;
- let state = accept(&mut UntypedWriteBuffer::new(&mut buffer)).await?;
+ while !matches!(state, StreamResult::Dropped) && position < limit {
+ let mut slice_buffer = SliceBuffer::new(buffer, position, limit);
+ state = accept(&mut UntypedWriteBuffer::new(&mut slice_buffer)).await?;
+ (buffer, position, _) = slice_buffer.into_parts();
+ }
- *pair.lock().unwrap() = Some((mine, buffer));
+ {
+ let (mine, mut buffer) = pair.lock().unwrap().take().unwrap();
+
+ while !(matches!(state, StreamResult::Dropped) || buffer.remaining().is_empty())
+ {
+ state = accept(&mut UntypedWriteBuffer::new(&mut buffer)).await?;
+ }
+
+ *pair.lock().unwrap() = Some((mine, buffer));
+ }
tls::get(|store| {
self.concurrent_state_mut(store).get_mut(id)?.read = match state {
StreamResult::Dropped => ReadState::Dropped,
StreamResult::Completed | StreamResult::Cancelled => {
- ReadState::HostToHost { accept }
+ ReadState::HostToHost {
+ accept,
+ buffer,
+ limit: 0,
+ }
}
};
diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams/buffers.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams/buffers.rs
index 4f9dea603f..7fc10d3fe0 100644
--- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams/buffers.rs
+++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams/buffers.rs
@@ -214,6 +214,54 @@ impl<T: Send + Sync + 'static> ReadBuffer<T> for Option<T> {
}
}
+/// A `WriteBuffer` implementation, backed by a `Vec<u8>`, a position, and a limit.
+pub struct SliceBuffer {
+ buffer: Vec<u8>,
+ offset: usize,
+ limit: usize,
+}
+
+impl SliceBuffer {
+ pub fn new(buffer: Vec<u8>, offset: usize, limit: usize) -> Self {
+ assert!(limit <= buffer.len());
+ Self {
+ buffer,
+ offset,
+ limit,
+ }
+ }
+
+ pub fn into_parts(self) -> (Vec<u8>, usize, usize) {
+ (self.buffer, self.offset, self.limit)
+ }
+}
+
+// SAFETY: the `take` implementation below guarantees that the `fun` closure is
+// provided with fully initialized items due to all elements in the slice being
+// initialized.
+unsafe impl WriteBuffer<u8> for SliceBuffer {
+ fn remaining(&self) -> &[u8] {
+ &self.buffer[self.offset..self.limit]
+ }
+
+ fn skip(&mut self, count: usize) {
+ assert!(self.offset + count <= self.limit);
+ self.offset += count;
+ }
+
+ fn take(&mut self, count: usize, fun: &mut dyn FnMut(&[MaybeUninit<u8>])) {
+ assert!(count <= self.remaining().len());
+ self.offset += count;
+ // SAFETY: Transmuting from `&[u8]` to `&[MaybeUninit<u8>]` should
+ // always be sound.
+ fun(unsafe {
+ mem::transmute::<&[u8], &[MaybeUninit<u8>]>(
+ &self.buffer[self.offset - count..self.limit],
+ )
+ });
+ }
+}
+
/// A `WriteBuffer` implementation, backed by a `Vec`.
pub struct VecBuffer<T> {
buffer: Vec<MaybeUninit<T>>,
@@ -282,7 +330,7 @@ unsafe impl<T: Send + Sync + 'static> WriteBuffer<T> for VecBuffer<T> {
// ensure that if `fun` panics that the items are still considered
// transferred.
self.offset += count;
- fun(&mut self.buffer[self.offset - count..]);
+ fun(&self.buffer[self.offset - count..]);
}
}