A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
make `Destination::as_direct` work for both host and guest readers (#11612)
Joel Dice6f0253930ea6
diff --git a/crates/misc/component-async-tests/src/util.rs b/crates/misc/component-async-tests/src/util.rsindex 40dc572d5e..70be467885 100644--- a/crates/misc/component-async-tests/src/util.rs+++ b/crates/misc/component-async-tests/src/util.rs@@ -53,7 +53,7 @@ impl<D, T: Send + Sync + Lower + 'static, S: Stream<Item = T> + Send + 'static>self: Pin<&mut Self>,cx: &mut Context<'_>,_: StoreContextMut<D>,- destination: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ mut destination: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<Result<StreamResult>> {// SAFETY: This is a standard pin-projection, and we never move@@ -94,7 +94,7 @@ impl<D, T: Lift + 'static, S: Sink<T, Error: std::error::Error + Send + Sync> +self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<D>,- source: &mut Source<Self::Item>,+ mut source: Source<Self::Item>,finish: bool,) -> Poll<Result<StreamResult>> {// SAFETY: This is a standard pin-projection, and we never movediff --git a/crates/misc/component-async-tests/tests/scenario/streams.rs b/crates/misc/component-async-tests/tests/scenario/streams.rsindex e63f2d3031..a0a934de61 100644--- a/crates/misc/component-async-tests/tests/scenario/streams.rs+++ b/crates/misc/component-async-tests/tests/scenario/streams.rs@@ -6,18 +6,107 @@ use {util::{OneshotConsumer, OneshotProducer, PipeConsumer, PipeProducer},},futures::{- SinkExt, StreamExt,+ Sink, SinkExt, Stream, StreamExt,channel::{mpsc, oneshot},future,},- std::sync::{Arc, Mutex},+ std::{+ pin::Pin,+ sync::{Arc, Mutex},+ task::{Context, Poll},+ },wasmtime::{- Engine, Store,- component::{FutureReader, Linker, ResourceTable, StreamReader},+ Engine, Store, StoreContextMut,+ component::{+ Destination, FutureReader, Linker, ResourceTable, Source, StreamConsumer,+ StreamProducer, StreamReader, StreamResult,+ },},wasmtime_wasi::WasiCtxBuilder,};+pub struct DirectPipeProducer<S>(S);++impl<D, S: Stream<Item = u8> + Send + 'static> StreamProducer<D> for DirectPipeProducer<S> {+ type Item = u8;+ type Buffer = Option<u8>;++ fn poll_produce<'a>(+ self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ store: StoreContextMut<D>,+ destination: Destination<'a, Self::Item, Self::Buffer>,+ finish: bool,+ ) -> Poll<Result<StreamResult>> {+ // SAFETY: This is a standard pin-projection, and we never move+ // out of `self`.+ let stream = unsafe { self.map_unchecked_mut(|v| &mut v.0) };++ match stream.poll_next(cx) {+ Poll::Pending => {+ if finish {+ Poll::Ready(Ok(StreamResult::Cancelled))+ } else {+ Poll::Pending+ }+ }+ Poll::Ready(Some(item)) => {+ let mut destination = destination.as_direct(store, 1);+ destination.remaining()[0] = item;+ destination.mark_written(1);+ Poll::Ready(Ok(StreamResult::Completed))+ }+ Poll::Ready(None) => Poll::Ready(Ok(StreamResult::Dropped)),+ }+ }+}++pub struct DirectPipeConsumer<S>(S);++impl<D, S: Sink<u8, Error: std::error::Error + Send + Sync> + Send + 'static> StreamConsumer<D>+ for DirectPipeConsumer<S>+{+ type Item = u8;++ fn poll_consume(+ self: Pin<&mut Self>,+ cx: &mut Context<'_>,+ store: StoreContextMut<D>,+ source: Source<Self::Item>,+ finish: bool,+ ) -> Poll<Result<StreamResult>> {+ // SAFETY: This is a standard pin-projection, and we never move+ // out of `self`.+ let mut sink = unsafe { self.map_unchecked_mut(|v| &mut v.0) };++ let on_pending = || {+ if finish {+ Poll::Ready(Ok(StreamResult::Cancelled))+ } else {+ Poll::Pending+ }+ };++ match sink.as_mut().poll_flush(cx) {+ Poll::Pending => on_pending(),+ Poll::Ready(result) => {+ result?;+ match sink.as_mut().poll_ready(cx) {+ Poll::Pending => on_pending(),+ Poll::Ready(result) => {+ result?;+ let mut source = source.as_direct(store);+ let item = source.remaining()[0];+ source.mark_read(1);+ sink.start_send(item)?;+ Poll::Ready(Ok(StreamResult::Completed))+ }+ }+ }+ }+ }+}+#[tokio::test]pub async fn async_closed_streams() -> Result<()> {let engine = Engine::new(&config())?;@@ -49,35 +138,45 @@ pub async fn async_closed_streams() -> Result<()> {let value = 42_u8;// First, test stream host->host- {- let (mut input_tx, input_rx) = mpsc::channel(1);- let (output_tx, mut output_rx) = mpsc::channel(1);- StreamReader::new(instance, &mut store, PipeProducer::new(input_rx))- .pipe(&mut store, PipeConsumer::new(output_tx));-- instance- .run_concurrent(&mut store, async |_| {- let (a, b) = future::join(- async {- for &value in &values {- input_tx.send(value).await?;- }- drop(input_tx);- anyhow::Ok(())- },- async {- for &value in &values {- assert_eq!(Some(value), output_rx.next().await);- }- assert!(output_rx.next().await.is_none());- Ok(())- },- )- .await;-- a.and(b)- })- .await??;+ for direct_producer in [true, false] {+ for direct_consumer in [true, false] {+ let (mut input_tx, input_rx) = mpsc::channel(1);+ let (output_tx, mut output_rx) = mpsc::channel(1);+ let reader = if direct_producer {+ StreamReader::new(instance, &mut store, DirectPipeProducer(input_rx))+ } else {+ StreamReader::new(instance, &mut store, PipeProducer::new(input_rx))+ };+ if direct_consumer {+ reader.pipe(&mut store, DirectPipeConsumer(output_tx));+ } else {+ reader.pipe(&mut store, PipeConsumer::new(output_tx));+ }++ instance+ .run_concurrent(&mut store, async |_| {+ let (a, b) = future::join(+ async {+ for &value in &values {+ input_tx.send(value).await?;+ }+ drop(input_tx);+ anyhow::Ok(())+ },+ async {+ for &value in &values {+ assert_eq!(Some(value), output_rx.next().await);+ }+ assert!(output_rx.next().await.is_none());+ Ok(())+ },+ )+ .await;++ a.and(b)+ })+ .await??;+ }}// Next, test futures host->hostdiff --git a/crates/misc/component-async-tests/tests/scenario/transmit.rs b/crates/misc/component-async-tests/tests/scenario/transmit.rsindex 80de705e73..77a0cfc98f 100644--- a/crates/misc/component-async-tests/tests/scenario/transmit.rs+++ b/crates/misc/component-async-tests/tests/scenario/transmit.rs@@ -42,7 +42,7 @@ impl<D> StreamProducer<D> for ReadinessProducer {self: Pin<&mut Self>,cx: &mut Context<'_>,mut store: StoreContextMut<'a, D>,- destination: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ destination: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<Result<StreamResult>> {let me = self.get_mut();@@ -62,7 +62,7 @@ impl<D> StreamProducer<D> for ReadinessProducer {Poll::Ready(Ok(StreamResult::Completed))} else {assert_eq!(capacity, Some(me.buffer.len()));- let mut destination = destination.as_direct_destination(store).unwrap();+ let mut destination = destination.as_direct(store, me.buffer.len());destination.remaining().copy_from_slice(&me.buffer);destination.mark_written(me.buffer.len());@@ -85,7 +85,7 @@ impl<D> StreamConsumer<D> for ReadinessConsumer {self: Pin<&mut Self>,cx: &mut Context<'_>,mut store: StoreContextMut<D>,- source: &mut Source<Self::Item>,+ source: Source<Self::Item>,finish: bool,) -> Poll<Result<StreamResult>> {let me = self.get_mut();@@ -105,7 +105,7 @@ impl<D> StreamConsumer<D> for ReadinessConsumer {Poll::Ready(Ok(StreamResult::Completed))} else {assert_eq!(available, me.expected.len());- let mut source = source.as_direct_source(store);+ let mut source = source.as_direct(store);assert_eq!(&me.expected, source.remaining());source.mark_read(me.expected.len());diff --git a/crates/wasi/src/p3/cli/host.rs b/crates/wasi/src/p3/cli/host.rsindex 3915f5754c..42756263e9 100644--- a/crates/wasi/src/p3/cli/host.rs+++ b/crates/wasi/src/p3/cli/host.rs@@ -12,11 +12,11 @@ use core::pin::Pin;use core::task::{Context, Poll};use std::io::Cursor;use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};-use wasmtime::StoreContextMut;use wasmtime::component::{Accessor, Destination, Resource, Source, StreamConsumer, StreamProducer, StreamReader,StreamResult,};+use wasmtime::{AsContextMut as _, StoreContextMut};struct InputStreamProducer {rx: Pin<Box<dyn AsyncRead + Send + Sync>>,@@ -29,54 +29,32 @@ impl<D> StreamProducer<D> for InputStreamProducer {fn poll_produce<'a>(mut self: Pin<&mut Self>,cx: &mut Context<'_>,- store: StoreContextMut<'a, D>,- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ mut store: StoreContextMut<'a, D>,+ dst: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {- if let Some(mut dst) = dst.as_direct_destination(store) {- if !dst.remaining().is_empty() {- let mut buf = ReadBuf::new(dst.remaining());- match self.rx.as_mut().poll_read(cx, &mut buf) {- Poll::Ready(Ok(())) if buf.filled().is_empty() => {- return Poll::Ready(Ok(StreamResult::Dropped));- }- Poll::Ready(Ok(())) => {- let n = buf.filled().len();- dst.mark_written(n);- return Poll::Ready(Ok(StreamResult::Completed));- }- Poll::Ready(Err(..)) => {- // TODO: Report the error to the guest- return Poll::Ready(Ok(StreamResult::Dropped));- }- Poll::Pending if finish => return Poll::Ready(Ok(StreamResult::Cancelled)),- Poll::Pending => return Poll::Pending,+ if let Some(0) = dst.remaining(store.as_context_mut()) {+ Poll::Ready(Ok(StreamResult::Completed))+ } else {+ let mut dst = dst.as_direct(store, DEFAULT_BUFFER_CAPACITY);+ let mut buf = ReadBuf::new(dst.remaining());+ match self.rx.as_mut().poll_read(cx, &mut buf) {+ Poll::Ready(Ok(())) if buf.filled().is_empty() => {+ Poll::Ready(Ok(StreamResult::Dropped))}+ Poll::Ready(Ok(())) => {+ let n = buf.filled().len();+ dst.mark_written(n);+ Poll::Ready(Ok(StreamResult::Completed))+ }+ Poll::Ready(Err(..)) => {+ // TODO: Report the error to the guest+ Poll::Ready(Ok(StreamResult::Dropped))+ }+ Poll::Pending if finish => Poll::Ready(Ok(StreamResult::Cancelled)),+ Poll::Pending => Poll::Pending,}}- let mut buf = dst.take_buffer().into_inner();- buf.clear();- buf.reserve(DEFAULT_BUFFER_CAPACITY);- let mut rbuf = ReadBuf::uninit(buf.spare_capacity_mut());- match self.rx.as_mut().poll_read(cx, &mut rbuf) {- Poll::Ready(Ok(())) if rbuf.filled().is_empty() => {- Poll::Ready(Ok(StreamResult::Dropped))- }- Poll::Ready(Ok(())) => {- let n = rbuf.filled().len();- // SAFETY: `ReadyBuf::filled` promised us `count` bytes have- // been initialized.- unsafe { buf.set_len(n) };- dst.set_buffer(Cursor::new(buf));- Poll::Ready(Ok(StreamResult::Completed))- }- Poll::Ready(Err(..)) => {- // TODO: Report the error to the guest- Poll::Ready(Ok(StreamResult::Dropped))- }- Poll::Pending if finish => Poll::Ready(Ok(StreamResult::Cancelled)),- Poll::Pending => Poll::Pending,- }}}@@ -91,10 +69,10 @@ impl<D> StreamConsumer<D> for OutputStreamConsumer {mut self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<D>,- src: &mut Source<Self::Item>,+ src: Source<Self::Item>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {- let mut src = src.as_direct_source(store);+ let mut src = src.as_direct(store);let buf = src.remaining();match self.tx.as_mut().poll_write(cx, buf) {Poll::Ready(Ok(n)) if buf.is_empty() => {diff --git a/crates/wasi/src/p3/filesystem/host.rs b/crates/wasi/src/p3/filesystem/host.rsindex a0ffe530e6..763040934b 100644--- a/crates/wasi/src/p3/filesystem/host.rs+++ b/crates/wasi/src/p3/filesystem/host.rs@@ -145,7 +145,7 @@ impl<D> StreamProducer<D> for ReadStreamProducer {mut self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<'a, D>,- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ mut dst: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {if let Some(task) = self.task.as_mut() {@@ -183,58 +183,33 @@ impl<D> StreamProducer<D> for ReadStreamProducer {return Poll::Ready(Ok(StreamResult::Cancelled));}if let Some(file) = self.file.as_blocking_file() {- if let Some(mut dst) = dst.as_direct_destination(store) {- let buf = dst.remaining();- if !buf.is_empty() {- match file.read_at(buf, self.offset) {- Ok(0) => {- self.close(Ok(()));+ let mut dst = dst.as_direct(store, DEFAULT_BUFFER_CAPACITY);+ let buf = dst.remaining();+ if buf.is_empty() {+ return Poll::Ready(Ok(StreamResult::Completed));+ } else {+ match file.read_at(buf, self.offset) {+ Ok(0) => {+ self.close(Ok(()));+ return Poll::Ready(Ok(StreamResult::Dropped));+ }+ Ok(n) => {+ dst.mark_written(n);+ let Ok(n) = n.try_into() else {+ self.close(Err(ErrorCode::Overflow));return Poll::Ready(Ok(StreamResult::Dropped));- }- Ok(n) => {- dst.mark_written(n);- let Ok(n) = n.try_into() else {- self.close(Err(ErrorCode::Overflow));- return Poll::Ready(Ok(StreamResult::Dropped));- };- let Some(n) = self.offset.checked_add(n) else {- self.close(Err(ErrorCode::Overflow));- return Poll::Ready(Ok(StreamResult::Dropped));- };- self.offset = n;- return Poll::Ready(Ok(StreamResult::Completed));- }- Err(err) => {- self.close(Err(err.into()));+ };+ let Some(n) = self.offset.checked_add(n) else {+ self.close(Err(ErrorCode::Overflow));return Poll::Ready(Ok(StreamResult::Dropped));- }+ };+ self.offset = n;+ return Poll::Ready(Ok(StreamResult::Completed));}- }- }- let mut buf = dst.take_buffer().into_inner();- buf.resize(DEFAULT_BUFFER_CAPACITY, 0);- match file.read_at(&mut buf, self.offset) {- Ok(0) => {- self.close(Ok(()));- return Poll::Ready(Ok(StreamResult::Dropped));- }- Ok(n) => {- buf.truncate(n);- dst.set_buffer(Cursor::new(buf));- let Ok(n) = n.try_into() else {- self.close(Err(ErrorCode::Overflow));+ Err(err) => {+ self.close(Err(err.into()));return Poll::Ready(Ok(StreamResult::Dropped));- };- let Some(n) = self.offset.checked_add(n) else {- self.close(Err(ErrorCode::Overflow));- return Poll::Ready(Ok(StreamResult::Dropped));- };- self.offset = n;- return Poll::Ready(Ok(StreamResult::Completed));- }- Err(err) => {- self.close(Err(err.into()));- return Poll::Ready(Ok(StreamResult::Dropped));+ }}}}@@ -343,7 +318,7 @@ impl<D> StreamProducer<D> for BlockingDirectoryStreamProducer {mut self: Pin<&mut Self>,_: &mut Context<'_>,_: StoreContextMut<'a, D>,- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ mut dst: Destination<'a, Self::Item, Self::Buffer>,_finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {let entries = match self.dir.entries() {@@ -407,7 +382,7 @@ impl<D> StreamProducer<D> for NonblockingDirectoryStreamProducer {mut self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<'a, D>,- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ mut dst: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {match mem::replace(&mut self.0, DirStreamState::Closed) {@@ -493,10 +468,10 @@ impl<D> StreamConsumer<D> for WriteStreamConsumer {mut self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<D>,- src: &mut Source<Self::Item>,+ src: Source<Self::Item>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {- let mut src = src.as_direct_source(store);+ let mut src = src.as_direct(store);if let Some(task) = self.task.as_mut() {let res = ready!(Pin::new(task).poll(cx));self.task = None;@@ -615,10 +590,10 @@ impl<D> StreamConsumer<D> for AppendStreamConsumer {mut self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<D>,- src: &mut Source<Self::Item>,+ src: Source<Self::Item>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {- let mut src = src.as_direct_source(store);+ let mut src = src.as_direct(store);if let Some(task) = self.task.as_mut() {let res = ready!(Pin::new(task).poll(cx));self.task = None;diff --git a/crates/wasi/src/p3/mod.rs b/crates/wasi/src/p3/mod.rsindex c3cdf63117..67c2b6699c 100644--- a/crates/wasi/src/p3/mod.rs+++ b/crates/wasi/src/p3/mod.rs@@ -46,7 +46,7 @@ impl<T: Send + Sync + 'static, D> StreamProducer<D> for StreamEmptyProducer<T> {self: Pin<&mut Self>,_: &mut Context<'_>,_: StoreContextMut<'a, D>,- _: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ _: Destination<'a, Self::Item, Self::Buffer>,_: bool,) -> Poll<wasmtime::Result<StreamResult>> {Poll::Ready(Ok(StreamResult::Dropped))diff --git a/crates/wasi/src/p3/sockets/host/types/tcp.rs b/crates/wasi/src/p3/sockets/host/types/tcp.rsindex 0b755cde13..6dc5caee8f 100644--- a/crates/wasi/src/p3/sockets/host/types/tcp.rs+++ b/crates/wasi/src/p3/sockets/host/types/tcp.rs@@ -18,11 +18,11 @@ use std::net::{Shutdown, SocketAddr};use std::sync::Arc;use tokio::net::{TcpListener, TcpStream};use tokio::sync::oneshot;-use wasmtime::StoreContextMut;use wasmtime::component::{Accessor, Destination, FutureReader, Resource, ResourceTable, Source, StreamConsumer,StreamProducer, StreamReader, StreamResult,};+use wasmtime::{AsContextMut as _, StoreContextMut};fn get_socket<'a>(table: &'a ResourceTable,@@ -62,7 +62,7 @@ whereself: Pin<&mut Self>,cx: &mut Context<'_>,mut store: StoreContextMut<'a, D>,- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ mut dst: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {let res = match self.listener.poll_accept(cx) {@@ -111,58 +111,42 @@ impl<D> StreamProducer<D> for ReceiveStreamProducer {fn poll_produce<'a>(mut self: Pin<&mut Self>,cx: &mut Context<'_>,- store: StoreContextMut<'a, D>,- dst: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ mut store: StoreContextMut<'a, D>,+ dst: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {let res = 'result: {- if let Some(mut dst) = dst.as_direct_destination(store) {- let buf = dst.remaining();- if !buf.is_empty() {- loop {- match self.stream.try_read(buf) {- Ok(0) => break 'result Ok(()),- Ok(n) => {- dst.mark_written(n);- return Poll::Ready(Ok(StreamResult::Completed));- }- Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {- match self.stream.poll_read_ready(cx) {- Poll::Ready(Ok(())) => continue,- Poll::Ready(Err(err)) => break 'result Err(err.into()),- Poll::Pending if finish => {- return Poll::Ready(Ok(StreamResult::Cancelled));- }- Poll::Pending => return Poll::Pending,- }- }- Err(err) => break 'result Err(err.into()),- }+ if let Some(0) = dst.remaining(store.as_context_mut()) {+ match self.stream.poll_read_ready(cx) {+ Poll::Ready(Ok(())) => return Poll::Ready(Ok(StreamResult::Completed)),+ Poll::Ready(Err(err)) => break 'result Err(err.into()),+ Poll::Pending if finish => {+ return Poll::Ready(Ok(StreamResult::Cancelled));}+ Poll::Pending => return Poll::Pending,}- }-- let mut buf = dst.take_buffer().into_inner();- buf.clear();- buf.reserve(DEFAULT_BUFFER_CAPACITY);- loop {- match self.stream.try_read_buf(&mut buf) {- Ok(0) => break 'result Ok(()),- Ok(..) => {- dst.set_buffer(Cursor::new(buf));- return Poll::Ready(Ok(StreamResult::Completed));- }- Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {- match self.stream.poll_read_ready(cx) {- Poll::Ready(Ok(())) => continue,- Poll::Ready(Err(err)) => break 'result Err(err.into()),- Poll::Pending if finish => {- return Poll::Ready(Ok(StreamResult::Cancelled));+ } else {+ let mut dst = dst.as_direct(store, DEFAULT_BUFFER_CAPACITY);+ let buf = dst.remaining();+ loop {+ match self.stream.try_read(buf) {+ Ok(0) => break 'result Ok(()),+ Ok(n) => {+ dst.mark_written(n);+ return Poll::Ready(Ok(StreamResult::Completed));+ }+ Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => {+ match self.stream.poll_read_ready(cx) {+ Poll::Ready(Ok(())) => continue,+ Poll::Ready(Err(err)) => break 'result Err(err.into()),+ Poll::Pending if finish => {+ return Poll::Ready(Ok(StreamResult::Cancelled));+ }+ Poll::Pending => return Poll::Pending,}- Poll::Pending => return Poll::Pending,}+ Err(err) => break 'result Err(err.into()),}- Err(err) => break 'result Err(err.into()),}}};@@ -201,10 +185,10 @@ impl<D> StreamConsumer<D> for SendStreamConsumer {mut self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<D>,- src: &mut Source<Self::Item>,+ src: Source<Self::Item>,finish: bool,) -> Poll<wasmtime::Result<StreamResult>> {- let mut src = src.as_direct_source(store);+ let mut src = src.as_direct(store);let res = 'result: {if src.remaining().is_empty() {match self.stream.poll_write_ready(cx) {diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rsindex 9ae9dfc5e5..9bc43d25fe 100644--- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs+++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs@@ -10,13 +10,13 @@ use crate::vm::component::{ComponentInstance, HandleTable, TransmitLocalState};use crate::vm::{AlwaysMut, VMStore};use crate::{AsContextMut, StoreContextMut, ValRaw};use anyhow::{Context as _, Result, anyhow, bail};-use buffers::Extender;-use buffers::UntypedWriteBuffer;+use buffers::{Extender, SliceBuffer, UntypedWriteBuffer};use futures::FutureExt;use futures::channel::oneshot;use std::boxed::Box;use std::fmt;use std::future;+use std::io::Cursor;use std::iter;use std::marker::PhantomData;use std::mem::{self, MaybeUninit};@@ -218,6 +218,7 @@ pub struct Destination<'a, T, B> {instance: Instance,id: TableId<TransmitState>,buffer: &'a mut B,+ host_buffer: Option<&'a mut Cursor<Vec<u8>>>,_phantom: PhantomData<fn() -> T>,}@@ -240,9 +241,9 @@ impl<'a, T, B> Destination<'a, T, B> {/// the `StreamProducer::poll_produce` call to which this `Destination` was/// passed returns (unless overwritten by another call to `set_buffer`).///- /// If items are stored via a buffer _and_ written via a `DirectDestination`- /// view of `self`, then the items in the buffer will be delivered after the- /// ones written using `DirectDestination`.+ /// If items are stored via this buffer _and_ written via a+ /// `DirectDestination` view of `self`, then the items in the buffer will be+ /// delivered after the ones written using `DirectDestination`.pub fn set_buffer(&mut self, buffer: B) {*self.buffer = buffer;}@@ -279,25 +280,33 @@ impl<'a, T, B> Destination<'a, T, B> {}impl<'a, B> Destination<'a, u8, B> {- /// Return a `DirectDestination` view of `self` if the guest is reading.- pub fn as_direct_destination<D>(- &mut self,+ /// Return a `DirectDestination` view of `self`.+ ///+ /// If the reader is a guest, this will provide direct access to the guest's+ /// read buffer. If the reader is a host, this will provide access to a+ /// buffer which will be delivered to the host before any items stored using+ /// `Destination::set_buffer`.+ ///+ /// `capacity` will only be used if the reader is a host, in which case it+ /// will update the length of the buffer, possibly zero-initializing the new+ /// elements if the new length is larger than the old length.+ pub fn as_direct<D>(+ mut self,store: StoreContextMut<'a, D>,- ) -> Option<DirectDestination<'a, D>> {- if let ReadState::GuestReady { .. } = self- .instance- .concurrent_state_mut(store.0)- .get_mut(self.id)- .unwrap()- .read- {- Some(DirectDestination {- instance: self.instance,- id: self.id,- store,- })- } else {- None+ capacity: usize,+ ) -> DirectDestination<'a, D> {+ if let Some(buffer) = self.host_buffer.as_deref_mut() {+ buffer.set_position(0);+ if buffer.get_mut().is_empty() {+ buffer.get_mut().resize(capacity, 0);+ }+ }++ DirectDestination {+ instance: self.instance,+ id: self.id,+ host_buffer: self.host_buffer,+ store,}}}@@ -307,37 +316,42 @@ impl<'a, B> Destination<'a, u8, B> {pub struct DirectDestination<'a, D: 'static> {instance: Instance,id: TableId<TransmitState>,+ host_buffer: Option<&'a mut Cursor<Vec<u8>>>,store: StoreContextMut<'a, D>,}impl<D: 'static> DirectDestination<'_, D> {/// Provide direct access to the writer's buffer.pub fn remaining(&mut self) -> &mut [u8] {- let transmit = self- .instance- .concurrent_state_mut(self.store.as_context_mut().0)- .get_mut(self.id)- .unwrap();+ if let Some(buffer) = self.host_buffer.as_deref_mut() {+ buffer.get_mut()+ } else {+ let transmit = self+ .instance+ .concurrent_state_mut(self.store.as_context_mut().0)+ .get_mut(self.id)+ .unwrap();- let &ReadState::GuestReady {- address,- count,- options,- ..- } = &transmit.read- else {- unreachable!()- };+ let &ReadState::GuestReady {+ address,+ count,+ options,+ ..+ } = &transmit.read+ else {+ unreachable!();+ };- let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {- unreachable!()- };+ let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {+ unreachable!()+ };- options- .memory_mut(self.store.0)- .get_mut((address + guest_offset)..)- .and_then(|b| b.get_mut(..(count - guest_offset)))- .unwrap()+ options+ .memory_mut(self.store.0)+ .get_mut((address + guest_offset)..)+ .and_then(|b| b.get_mut(..(count - guest_offset)))+ .unwrap()+ }}/// Mark the specified number of bytes as written to the writer's buffer.@@ -345,27 +359,38 @@ impl<D: 'static> DirectDestination<'_, D> {/// This will panic if the count is larger than the size of the/// buffer returned by `Self::remaining`.pub fn mark_written(&mut self, count: usize) {- let transmit = self- .instance- .concurrent_state_mut(self.store.as_context_mut().0)- .get_mut(self.id)- .unwrap();+ if let Some(buffer) = self.host_buffer.as_deref_mut() {+ buffer.set_position(+ buffer+ .position()+ .checked_add(u64::try_from(count).unwrap())+ .unwrap(),+ );+ } else {+ let transmit = self+ .instance+ .concurrent_state_mut(self.store.as_context_mut().0)+ .get_mut(self.id)+ .unwrap();- let ReadState::GuestReady {- count: read_count, ..- } = &transmit.read- else {- unreachable!()- };+ let ReadState::GuestReady {+ count: read_count, ..+ } = &transmit.read+ else {+ unreachable!();+ };- let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {- unreachable!()- };+ let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {+ unreachable!()+ };- if *guest_offset + count > *read_count {- panic!("write count ({count}) must be less than or equal to read count ({read_count})")- } else {- *guest_offset += count;+ if *guest_offset + count > *read_count {+ panic!(+ "write count ({count}) must be less than or equal to read count ({read_count})"+ )+ } else {+ *guest_offset += count;+ }}}}@@ -443,21 +468,32 @@ pub trait StreamProducer<D>: Send + 'static {/// If more items are written to `destination` than the reader has immediate/// capacity to accept, they will be retained in memory by the caller and/// used to satisify future reads, in which case `poll_produce` will only be- /// called again once all those items have been delivered. This is- /// particularly important for zero-length reads, in which case the- /// implementation is expected to either:+ /// called again once all those items have been delivered.+ ///+ /// If this function is called with zero capacity+ /// (i.e. `Destination::remaining` returns `Some(0)`), the implementation+ /// should either:///- /// 1. Produce at least one item (if possible, and if `finish` is false) so- /// that it is ready to be delivered immediately upon the next- /// non-zero-length read.+ /// - Return `Poll::Ready(Ok(StreamResult::Completed))` without writing+ /// anything if it expects to be able to produce items immediately+ /// (i.e. without first returning `Poll::Pending`) the next time+ /// `poll_produce` is called with non-zero capacity _or_ if that cannot be+ /// reliably determined.///- /// 2. Produce at least one item the next time `poll_produce` is called with- /// non-zero capacity and `finish` set to false.+ /// - Return `Poll::Pending` if the next call to `poll_produce` with+ /// non-zero capacity is likely to also return `Poll::Pending`.+ ///+ /// - Return `Poll::Ready(Ok(StreamResult::Completed))` after calling+ /// `Destination::set_buffer` with one more more items. Note, however, that+ /// this creates the hazard that the items will never be received by the+ /// guest if it decides not to do another non-zero-length read before+ /// closing the stream. Moreover, if `Self::Item` is e.g. a `Resource<_>`,+ /// they may end up leaking in that scenario.fn poll_produce<'a>(self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<'a, D>,- destination: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ destination: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<Result<StreamResult>>;}@@ -553,15 +589,13 @@ impl<T> Source<'_, T> {}}-impl Source<'_, u8> {+impl<'a> Source<'a, u8> {/// Return a `DirectSource` view of `self`.- pub fn as_direct_source<'a, D>(- &mut self,- store: StoreContextMut<'a, D>,- ) -> DirectSource<'a, D> {+ pub fn as_direct<D>(self, store: StoreContextMut<'a, D>) -> DirectSource<'a, D> {DirectSource {instance: self.instance,id: self.id,+ host_buffer: self.host_buffer,store,}}@@ -572,37 +606,42 @@ impl Source<'_, u8> {pub struct DirectSource<'a, D: 'static> {instance: Instance,id: TableId<TransmitState>,+ host_buffer: Option<&'a mut dyn WriteBuffer<u8>>,store: StoreContextMut<'a, D>,}impl<D: 'static> DirectSource<'_, D> {/// Provide direct access to the writer's buffer.pub fn remaining(&mut self) -> &[u8] {- let transmit = self- .instance- .concurrent_state_mut(self.store.as_context_mut().0)- .get_mut(self.id)- .unwrap();+ if let Some(buffer) = self.host_buffer.as_deref_mut() {+ buffer.remaining()+ } else {+ let transmit = self+ .instance+ .concurrent_state_mut(self.store.as_context_mut().0)+ .get_mut(self.id)+ .unwrap();- let &WriteState::GuestReady {- address,- count,- options,- ..- } = &transmit.write- else {- unreachable!()- };+ let &WriteState::GuestReady {+ address,+ count,+ options,+ ..+ } = &transmit.write+ else {+ unreachable!()+ };- let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {- unreachable!()- };+ let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {+ unreachable!()+ };- options- .memory(self.store.0)- .get((address + guest_offset)..)- .and_then(|b| b.get(..(count - guest_offset)))- .unwrap()+ options+ .memory(self.store.0)+ .get((address + guest_offset)..)+ .and_then(|b| b.get(..(count - guest_offset)))+ .unwrap()+ }}/// Mark the specified number of bytes as read from the writer's buffer.@@ -610,27 +649,33 @@ impl<D: 'static> DirectSource<'_, D> {/// This will panic if the count is larger than the size of the buffer/// returned by `Self::remaining`.pub fn mark_read(&mut self, count: usize) {- let transmit = self- .instance- .concurrent_state_mut(self.store.as_context_mut().0)- .get_mut(self.id)- .unwrap();+ if let Some(buffer) = self.host_buffer.as_deref_mut() {+ buffer.skip(count);+ } else {+ let transmit = self+ .instance+ .concurrent_state_mut(self.store.as_context_mut().0)+ .get_mut(self.id)+ .unwrap();- let WriteState::GuestReady {- count: write_count, ..- } = &transmit.write- else {- unreachable!()- };+ let WriteState::GuestReady {+ count: write_count, ..+ } = &transmit.write+ else {+ unreachable!()+ };- let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {- unreachable!()- };+ let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {+ unreachable!()+ };- if *guest_offset + count > *write_count {- panic!("read count ({count}) must be less than or equal to write count ({write_count})")- } else {- *guest_offset += count;+ if *guest_offset + count > *write_count {+ panic!(+ "read count ({count}) must be less than or equal to write count ({write_count})"+ )+ } else {+ *guest_offset += count;+ }}}}@@ -726,7 +771,7 @@ pub trait StreamConsumer<D>: Send + 'static {self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<D>,- source: &mut Source<Self::Item>,+ source: Source<'_, Self::Item>,finish: bool,) -> Poll<Result<StreamResult>>;}@@ -787,7 +832,7 @@ impl<T> FutureReader<T> {self: Pin<&mut Self>,cx: &mut Context<'_>,store: StoreContextMut<D>,- destination: &'a mut Destination<'a, Self::Item, Self::Buffer>,+ mut destination: Destination<'a, Self::Item, Self::Buffer>,finish: bool,) -> Poll<Result<StreamResult>> {// SAFETY: This is a standard pin-projection, and we never move@@ -864,7 +909,7 @@ impl<T> FutureReader<T> {self: Pin<&mut Self>,cx: &mut Context<'_>,mut store: StoreContextMut<D>,- source: &mut Source<Self::Item>,+ mut source: Source<Self::Item>,finish: bool,) -> Poll<Result<StreamResult>> {let me = self.get_mut();@@ -1681,6 +1726,8 @@ enum ReadState {+ Send+ Sync,>,+ buffer: Vec<u8>,+ limit: usize,},/// The read end has been dropped.Dropped,@@ -1731,41 +1778,62 @@ impl Instance {let (result, cancelled) = if buffer.remaining().is_empty() {future::poll_fn(|cx| {tls::get(|store| {- let &WriteState::HostReady { cancel, .. } =- &self.concurrent_state_mut(store).get_mut(id).unwrap().write- else {+ let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();++ let &WriteState::HostReady { cancel, .. } = &transmit.write else {unreachable!();};+ let mut host_buffer =+ if let ReadState::HostToHost { buffer, .. } = &mut transmit.read {+ Some(Cursor::new(mem::take(buffer)))+ } else {+ None+ };+let poll = mine.as_mut().poll_produce(cx,token.as_context_mut(store),- &mut Destination {+ Destination {instance: self,id,buffer: &mut buffer,+ host_buffer: host_buffer.as_mut(),_phantom: PhantomData,},cancel,);+ let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();++ let host_offset = if let (+ Some(host_buffer),+ ReadState::HostToHost { buffer, limit, .. },+ ) = (host_buffer, &mut transmit.read)+ {+ *limit = usize::try_from(host_buffer.position()).unwrap();+ *buffer = host_buffer.into_inner();+ *limit+ } else {+ 0+ };+{let WriteState::HostReady {guest_offset,cancel,cancel_waker,..- } = &mut self- .concurrent_state_mut(store)- .get_mut(id)- .unwrap()- .write+ } = &mut transmit.writeelse {unreachable!();};if let Poll::Pending = &poll {- if !buffer.remaining().is_empty() || *guest_offset > 0 {+ if !buffer.remaining().is_empty()+ || *guest_offset > 0+ || host_offset > 0+ {return Poll::Ready(Err(anyhow!("StreamProducer::poll_produce returned Poll::Pending \after producing at least one item"@@ -1787,24 +1855,27 @@ impl Instance {(StreamResult::Completed, false)};- let (guest_offset, count) = tls::get(|store| {+ let (guest_offset, host_offset, count) = tls::get(|store| {let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();- (- match &transmit.write {- &WriteState::HostReady { guest_offset, .. } => guest_offset,- _ => unreachable!(),- },- match &transmit.read {- &ReadState::GuestReady { count, .. } => count,- ReadState::HostToHost { .. } => 1,- _ => unreachable!(),- },- )+ let (count, host_offset) = match &transmit.read {+ &ReadState::GuestReady { count, .. } => (count, 0),+ &ReadState::HostToHost { limit, .. } => (1, limit),+ _ => unreachable!(),+ };+ let guest_offset = match &transmit.write {+ &WriteState::HostReady { guest_offset, .. } => guest_offset,+ _ => unreachable!(),+ };+ (guest_offset, host_offset, count)});match result {StreamResult::Completed => {- if count > 1 && buffer.remaining().is_empty() && guest_offset == 0 {+ if count > 1+ && buffer.remaining().is_empty()+ && guest_offset == 0+ && host_offset == 0+ {bail!("StreamProducer::poll_produce returned StreamResult::Completed \without producing any items"@@ -1822,11 +1893,11 @@ impl Instance {StreamResult::Dropped => {}}- let write = !buffer.remaining().is_empty();+ let write_buffer = !buffer.remaining().is_empty() || host_offset > 0;*producer.lock().unwrap() = Some((mine, buffer));- if write {+ if write_buffer {self.write(token, id, producer, kind).await?;}@@ -1876,7 +1947,7 @@ impl Instance {let poll = mine.as_mut().poll_consume(cx,token.as_context_mut(store),- &mut Source {+ Source {instance: self,id,host_buffer: host_buffer.as_deref_mut(),@@ -1995,6 +2066,8 @@ impl Instance {let consume = consume_with_buffer.clone();async move { consume(Some(input.get_mut::<C::Item>())).await }.boxed()}),+ buffer: Vec::new(),+ limit: 0,};let future = async move {@@ -2141,18 +2214,40 @@ impl Instance {Ok(())}- ReadState::HostToHost { accept } => {- let (mine, mut buffer) = pair.lock().unwrap().take().unwrap();+ ReadState::HostToHost {+ accept,+ mut buffer,+ limit,+ } => {+ let mut state = StreamResult::Completed;+ let mut position = 0;- let state = accept(&mut UntypedWriteBuffer::new(&mut buffer)).await?;+ while !matches!(state, StreamResult::Dropped) && position < limit {+ let mut slice_buffer = SliceBuffer::new(buffer, position, limit);+ state = accept(&mut UntypedWriteBuffer::new(&mut slice_buffer)).await?;+ (buffer, position, _) = slice_buffer.into_parts();+ }- *pair.lock().unwrap() = Some((mine, buffer));+ {+ let (mine, mut buffer) = pair.lock().unwrap().take().unwrap();++ while !(matches!(state, StreamResult::Dropped) || buffer.remaining().is_empty())+ {+ state = accept(&mut UntypedWriteBuffer::new(&mut buffer)).await?;+ }++ *pair.lock().unwrap() = Some((mine, buffer));+ }tls::get(|store| {self.concurrent_state_mut(store).get_mut(id)?.read = match state {StreamResult::Dropped => ReadState::Dropped,StreamResult::Completed | StreamResult::Cancelled => {- ReadState::HostToHost { accept }+ ReadState::HostToHost {+ accept,+ buffer,+ limit: 0,+ }}};diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams/buffers.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams/buffers.rsindex 4f9dea603f..7fc10d3fe0 100644--- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams/buffers.rs+++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams/buffers.rs@@ -214,6 +214,54 @@ impl<T: Send + Sync + 'static> ReadBuffer<T> for Option<T> {}}+/// A `WriteBuffer` implementation, backed by a `Vec<u8>`, a position, and a limit.+pub struct SliceBuffer {+ buffer: Vec<u8>,+ offset: usize,+ limit: usize,+}++impl SliceBuffer {+ pub fn new(buffer: Vec<u8>, offset: usize, limit: usize) -> Self {+ assert!(limit <= buffer.len());+ Self {+ buffer,+ offset,+ limit,+ }+ }++ pub fn into_parts(self) -> (Vec<u8>, usize, usize) {+ (self.buffer, self.offset, self.limit)+ }+}++// SAFETY: the `take` implementation below guarantees that the `fun` closure is+// provided with fully initialized items due to all elements in the slice being+// initialized.+unsafe impl WriteBuffer<u8> for SliceBuffer {+ fn remaining(&self) -> &[u8] {+ &self.buffer[self.offset..self.limit]+ }++ fn skip(&mut self, count: usize) {+ assert!(self.offset + count <= self.limit);+ self.offset += count;+ }++ fn take(&mut self, count: usize, fun: &mut dyn FnMut(&[MaybeUninit<u8>])) {+ assert!(count <= self.remaining().len());+ self.offset += count;+ // SAFETY: Transmuting from `&[u8]` to `&[MaybeUninit<u8>]` should+ // always be sound.+ fun(unsafe {+ mem::transmute::<&[u8], &[MaybeUninit<u8>]>(+ &self.buffer[self.offset - count..self.limit],+ )+ });+ }+}+/// A `WriteBuffer` implementation, backed by a `Vec`.pub struct VecBuffer<T> {buffer: Vec<MaybeUninit<T>>,@@ -282,7 +330,7 @@ unsafe impl<T: Send + Sync + 'static> WriteBuffer<T> for VecBuffer<T> {// ensure that if `fun` panics that the items are still considered// transferred.self.offset += count;- fun(&mut self.buffer[self.offset - count..]);+ fun(&self.buffer[self.offset - count..]);}}