A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
move `ConcurrentState` from `ComponentInstance` to `Store` (#11796)
Joel Dice7e39c25e2741
diff --git a/crates/misc/component-async-tests/src/resource_stream.rs b/crates/misc/component-async-tests/src/resource_stream.rsindex baf3bdcecf..a47e721ed6 100644--- a/crates/misc/component-async-tests/src/resource_stream.rs+++ b/crates/misc/component-async-tests/src/resource_stream.rs@@ -44,8 +44,7 @@ impl bindings::local::local::resource_stream::HostWithStore for Ctx {tx.try_send(access.get().table.push(ResourceStreamX)?).unwrap()}- let instance = access.instance();- Ok(StreamReader::new(instance, access, PipeProducer::new(rx)))+ Ok(StreamReader::new(access, PipeProducer::new(rx)))})}}diff --git a/crates/misc/component-async-tests/tests/scenario/borrowing.rs b/crates/misc/component-async-tests/tests/scenario/borrowing.rsindex 7e1f7cfd1c..698091a3d5 100644--- a/crates/misc/component-async-tests/tests/scenario/borrowing.rs+++ b/crates/misc/component-async-tests/tests/scenario/borrowing.rs@@ -102,12 +102,14 @@ pub async fn test_run_bool(components: &[&str], v: bool) -> Result<()> {});}- let instance = linker.instantiate_async(&mut store, &component).await?;let borrowing_host =- component_async_tests::borrowing_host::bindings::BorrowingHost::new(&mut store, &instance)?;+ component_async_tests::borrowing_host::bindings::BorrowingHost::instantiate_async(+ &mut store, &component, &linker,+ )+ .await?;- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {// Start three concurrent calls and then join them all:let mut futures = FuturesUnordered::new();for _ in 0..3 {diff --git a/crates/misc/component-async-tests/tests/scenario/post_return.rs b/crates/misc/component-async-tests/tests/scenario/post_return.rsindex 61d014feac..6a3bf6ef2c 100644--- a/crates/misc/component-async-tests/tests/scenario/post_return.rs+++ b/crates/misc/component-async-tests/tests/scenario/post_return.rs@@ -8,7 +8,7 @@ use std::pin::pin;use std::sync::{Arc, Mutex};use std::task::Poll;use std::time::Duration;-use wasmtime::component::{Accessor, Instance, Linker, ResourceTable};+use wasmtime::component::{Accessor, Linker, ResourceTable};use wasmtime::{AsContextMut, Engine, Store, StoreContextMut};use wasmtime_wasi::WasiCtxBuilder;@@ -71,8 +71,10 @@ async fn test_sleep_post_return(components: &[&str]) -> Result<()> {},);- let instance = linker.instantiate_async(&mut store, &component).await?;- let guest = sleep_post_return::SleepPostReturnCallee::new(&mut store, &instance)?;+ let guest = sleep_post_return::SleepPostReturnCallee::instantiate_async(+ &mut store, &component, &linker,+ )+ .await?;async fn run_with(accessor: &Accessor<Ctx>,@@ -93,18 +95,17 @@ async fn test_sleep_post_return(components: &[&str]) -> Result<()> {async fn run(store: StoreContextMut<'_, Ctx>,- instance: Instance,guest: &sleep_post_return::SleepPostReturnCallee,) -> Result<()> {- instance- .run_concurrent(store, async |accessor| {+ store+ .run_concurrent(async |accessor| {run_with(accessor, guest).await?;// Go idle for a bit before doing it again. This tests that- // `Instance::run_concurrent` is okay with having no outstanding- // guest or host tasks to poll for a while, trusting that we'll- // resolve the future independently, with or without giving it- // more work to do.+ // `StoreContextMut::run_concurrent` is okay with having no+ // outstanding guest or host tasks to poll for a while, trusting+ // that we'll resolve the future independently, with or without+ // giving it more work to do.util::sleep(Duration::from_millis(100)).await;run_with(accessor, guest).await?;@@ -114,23 +115,21 @@ async fn test_sleep_post_return(components: &[&str]) -> Result<()> {.await?}- run(store.as_context_mut(), instance, &guest).await?;+ run(store.as_context_mut(), &guest).await?;// At this point, all subtasks should have exited, meaning no waitables,- // tasks, or other concurrent state should remain present in the instance.- instance.assert_concurrent_state_empty(&mut store);+ // tasks, or other concurrent state should remain present in the store.+ store.assert_concurrent_state_empty();// Do it again, but this time cancel the event loop before it exits:assert!(- future::poll_fn(|cx| Poll::Ready(- pin!(run(store.as_context_mut(), instance, &guest)).poll(cx)- ))- .await- .is_pending()+ future::poll_fn(|cx| Poll::Ready(pin!(run(store.as_context_mut(), &guest)).poll(cx)))+ .await+ .is_pending());// Assuming the event loop is cancel-safe, this should complete without// errors or panics:- run(store.as_context_mut(), instance, &guest).await?;+ run(store.as_context_mut(), &guest).await?;Ok(())}diff --git a/crates/misc/component-async-tests/tests/scenario/round_trip.rs b/crates/misc/component-async-tests/tests/scenario/round_trip.rsindex c641ee5166..2d49c02cce 100644--- a/crates/misc/component-async-tests/tests/scenario/round_trip.rs+++ b/crates/misc/component-async-tests/tests/scenario/round_trip.rs@@ -236,9 +236,9 @@ pub async fn test_round_trip(component_async_tests::round_trip::bindings::RoundTrip::new(&mut store, &instance)?;if call_style == 0 || !cfg!(miri) {- // Run the test using `Instance::run_concurrent`:- instance- .run_concurrent(&mut store, {+ // Run the test using `StoreContextMut::run_concurrent`:+ store+ .run_concurrent({let inputs_and_outputs = inputs_and_outputs.iter().map(|(a, b)| (String::from(*a), String::from(*b)))@@ -266,7 +266,7 @@ pub async fn test_round_trip(}).await??;- instance.assert_concurrent_state_empty(&mut store);+ store.assert_concurrent_state_empty();}if call_style == 1 || !cfg!(miri) {@@ -309,23 +309,18 @@ pub async fn test_round_trip(}let (tx, rx) = oneshot::channel();- instance.spawn(- &mut store,- Task {- instance,- inputs_and_outputs: inputs_and_outputs- .iter()- .map(|(a, b)| (String::from(*a), String::from(*b)))- .collect::<Vec<_>>(),- tx,- },- );-- instance- .run_concurrent(&mut store, async |_| rx.await)- .await??;-- instance.assert_concurrent_state_empty(&mut store);+ store.spawn(Task {+ instance,+ inputs_and_outputs: inputs_and_outputs+ .iter()+ .map(|(a, b)| (String::from(*a), String::from(*b)))+ .collect::<Vec<_>>(),+ tx,+ });++ store.run_concurrent(async |_| rx.await).await??;++ store.assert_concurrent_state_empty();}if call_style == 2 || !cfg!(miri) {@@ -345,7 +340,7 @@ pub async fn test_round_trip();}- instance.assert_concurrent_state_empty(&mut store);+ store.assert_concurrent_state_empty();}}@@ -382,8 +377,8 @@ pub async fn test_round_trip(.ok_or_else(|| anyhow!("can't find `[async]foo` in instance"))?;if call_style == 3 || !cfg!(miri) {- instance- .run_concurrent(&mut store, async |store| {+ store+ .run_concurrent(async |store| {// Start three concurrent calls and then join them all:let mut futures = FuturesUnordered::new();for (input, output) in inputs_and_outputs {@@ -411,7 +406,7 @@ pub async fn test_round_trip(}).await??;- instance.assert_concurrent_state_empty(&mut store);+ store.assert_concurrent_state_empty();}if call_style == 4 || !cfg!(miri) {@@ -432,7 +427,7 @@ pub async fn test_round_trip(foo_function.post_return_async(&mut store).await?;}- instance.assert_concurrent_state_empty(&mut store);+ store.assert_concurrent_state_empty();}}diff --git a/crates/misc/component-async-tests/tests/scenario/round_trip_direct.rs b/crates/misc/component-async-tests/tests/scenario/round_trip_direct.rsindex 8f61aae27b..fc0921888d 100644--- a/crates/misc/component-async-tests/tests/scenario/round_trip_direct.rs+++ b/crates/misc/component-async-tests/tests/scenario/round_trip_direct.rs@@ -60,13 +60,14 @@ async fn test_round_trip_direct(let mut store = make_store();- let instance = linker.instantiate_async(&mut store, &component).await?;- let round_trip = component_async_tests::round_trip_direct::bindings::RoundTripDirect::new(- &mut store, &instance,- )?;-- instance- .run_concurrent(&mut store, {+ let round_trip =+ component_async_tests::round_trip_direct::bindings::RoundTripDirect::instantiate_async(+ &mut store, &component, &linker,+ )+ .await?;++ store+ .run_concurrent({let input = input.to_owned();let expected_output = expected_output.to_owned();async move |accessor| {@@ -115,8 +116,8 @@ async fn test_round_trip_direct(.ok_or_else(|| anyhow!("can't find `foo` in instance"))?;// Start three concurrent calls and then join them all:- instance- .run_concurrent(&mut store, async |store| -> wasmtime::Result<_> {+ store+ .run_concurrent(async |store| -> wasmtime::Result<_> {let mut futures = FuturesUnordered::new();for _ in 0..3 {futures.push(async move {diff --git a/crates/misc/component-async-tests/tests/scenario/round_trip_many.rs b/crates/misc/component-async-tests/tests/scenario/round_trip_many.rsindex 0a6ff8930b..85a1ffb924 100644--- a/crates/misc/component-async-tests/tests/scenario/round_trip_many.rs+++ b/crates/misc/component-async-tests/tests/scenario/round_trip_many.rs@@ -250,8 +250,8 @@ async fn test_round_trip_many()?;if call_style == 0 {- instance- .run_concurrent(&mut store, {+ store+ .run_concurrent({let c = c.clone();let e = e.clone();let f = f.clone();@@ -293,7 +293,7 @@ async fn test_round_trip_many(}).await??;- instance.assert_concurrent_state_empty(&mut store);+ store.assert_concurrent_state_empty();}if call_style == 1 {@@ -329,7 +329,7 @@ async fn test_round_trip_many();}- instance.assert_concurrent_state_empty(&mut store);+ store.assert_concurrent_state_empty();}}@@ -391,8 +391,8 @@ async fn test_round_trip_many(};if call_style == 2 {- instance- .run_concurrent(&mut store, async |store| -> wasmtime::Result<_> {+ store+ .run_concurrent(async |store| -> wasmtime::Result<_> {// Start three concurrent calls and then join them all:let mut futures = FuturesUnordered::new();for (input, output) in inputs_and_outputs {@@ -416,7 +416,7 @@ async fn test_round_trip_many(}).await??;- instance.assert_concurrent_state_empty(&mut store);+ store.assert_concurrent_state_empty();}if call_style == 3 {@@ -433,7 +433,7 @@ async fn test_round_trip_many(foo_function.post_return_async(&mut store).await?;}- instance.assert_concurrent_state_empty(&mut store);+ store.assert_concurrent_state_empty();}}diff --git a/crates/misc/component-async-tests/tests/scenario/streams.rs b/crates/misc/component-async-tests/tests/scenario/streams.rsindex 3434dff629..eb251e76a4 100644--- a/crates/misc/component-async-tests/tests/scenario/streams.rs+++ b/crates/misc/component-async-tests/tests/scenario/streams.rs@@ -143,9 +143,9 @@ pub async fn async_closed_streams() -> Result<()> {let (mut input_tx, input_rx) = mpsc::channel(1);let (output_tx, mut output_rx) = mpsc::channel(1);let reader = if direct_producer {- StreamReader::new(instance, &mut store, DirectPipeProducer(input_rx))+ StreamReader::new(&mut store, DirectPipeProducer(input_rx))} else {- StreamReader::new(instance, &mut store, PipeProducer::new(input_rx))+ StreamReader::new(&mut store, PipeProducer::new(input_rx))};if direct_consumer {reader.pipe(&mut store, DirectPipeConsumer(output_tx));@@ -153,8 +153,8 @@ pub async fn async_closed_streams() -> Result<()> {reader.pipe(&mut store, PipeConsumer::new(output_tx));}- instance- .run_concurrent(&mut store, async |_| {+ store+ .run_concurrent(async |_| {let (a, b) = future::join(async {for &value in &values {@@ -183,11 +183,11 @@ pub async fn async_closed_streams() -> Result<()> {{let (input_tx, input_rx) = oneshot::channel();let (output_tx, output_rx) = oneshot::channel();- FutureReader::new(instance, &mut store, OneshotProducer::new(input_rx))+ FutureReader::new(&mut store, OneshotProducer::new(input_rx)).pipe(&mut store, OneshotConsumer::new(output_tx));- instance- .run_concurrent(&mut store, async |_| {+ store+ .run_concurrent(async |_| {_ = input_tx.send(value);assert_eq!(value, output_rx.await?);anyhow::Ok(())@@ -198,14 +198,14 @@ pub async fn async_closed_streams() -> Result<()> {// Next, test stream host->guest{let (mut tx, rx) = mpsc::channel(1);- let rx = StreamReader::new(instance, &mut store, PipeProducer::new(rx));+ let rx = StreamReader::new(&mut store, PipeProducer::new(rx));let closed_streams = closed_streams::bindings::ClosedStreams::new(&mut store, &instance)?;let values = values.clone();- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {let (a, b) = future::join(async {for &value in &values {@@ -230,14 +230,14 @@ pub async fn async_closed_streams() -> Result<()> {// Next, test futures host->guest{let (tx, rx) = oneshot::channel();- let rx = FutureReader::new(instance, &mut store, OneshotProducer::new(rx));+ let rx = FutureReader::new(&mut store, OneshotProducer::new(rx));let (_, rx_ignored) = oneshot::channel();- let rx_ignored = FutureReader::new(instance, &mut store, OneshotProducer::new(rx_ignored));+ let rx_ignored = FutureReader::new(&mut store, OneshotProducer::new(rx_ignored));let closed_streams = closed_streams::bindings::ClosedStreams::new(&mut store, &instance)?;- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {_ = tx.send(value);closed_streams.local_local_closed()@@ -284,8 +284,8 @@ pub async fn async_closed_stream() -> Result<()> {let instance = linker.instantiate_async(&mut store, &component).await?;let guest = closed_stream::ClosedStreamGuest::new(&mut store, &instance)?;- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {let stream = guest.local_local_closed_stream().call_get(accessor).await?;let (tx, mut rx) = mpsc::channel(1);diff --git a/crates/misc/component-async-tests/tests/scenario/transmit.rs b/crates/misc/component-async-tests/tests/scenario/transmit.rsindex ff900480d9..d31ef9960c 100644--- a/crates/misc/component-async-tests/tests/scenario/transmit.rs+++ b/crates/misc/component-async-tests/tests/scenario/transmit.rs@@ -354,11 +354,10 @@ pub async fn async_readiness() -> Result<()> {},);- let instance = linker.instantiate_async(&mut store, &component).await?;- let readiness_guest = readiness::ReadinessGuest::new(&mut store, &instance)?;+ let readiness_guest =+ readiness::ReadinessGuest::instantiate_async(&mut store, &component, &linker).await?;let expected = vec![2u8, 4, 6, 8, 9];let rx = StreamReader::new(- instance,&mut store,DelayedStreamProducer {inner: BufferStreamProducer {@@ -367,8 +366,8 @@ pub async fn async_readiness() -> Result<()> {sleep: sleep(),},);- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {let ((rx, expected), task_exit) = readiness_guest.local_local_readiness().call_start(accessor, rx, expected)@@ -498,10 +497,10 @@ async fn test_cancel(mode: Mode) -> Result<()> {},);- let instance = linker.instantiate_async(&mut store, &component).await?;- let cancel_host = cancel::CancelHost::new(&mut store, &instance)?;- instance- .run_concurrent(&mut store, async move |accessor| {+ let cancel_host =+ cancel::CancelHost::instantiate_async(&mut store, &component, &linker).await?;+ store+ .run_concurrent(async move |accessor| {cancel_host.local_local_cancel().call_run(accessor, mode, delay_millis())@@ -544,7 +543,7 @@ pub trait TransmitTest {store: impl AsContextMut<Data = Ctx>,component: &Component,linker: &Linker<Ctx>,- ) -> impl Future<Output = Result<(Self::Instance, Instance)>>;+ ) -> impl Future<Output = Result<Self::Instance>>;fn call<'a>(accessor: &'a Accessor<Ctx, HasSelf<Ctx>>,@@ -561,7 +560,6 @@ pub trait TransmitTest {fn from_result(store: impl AsContextMut<Data = Ctx>,- instance: Instance,result: Self::Result,) -> Result<(StreamReader<String>,@@ -587,13 +585,11 @@ impl TransmitTest for StaticTransmitTest {);async fn instantiate(- mut store: impl AsContextMut<Data = Ctx>,+ store: impl AsContextMut<Data = Ctx>,component: &Component,linker: &Linker<Ctx>,- ) -> Result<(Self::Instance, Instance)> {- let instance = linker.instantiate_async(&mut store, component).await?;- let callee = transmit::bindings::TransmitCallee::new(store, &instance)?;- Ok((callee, instance))+ ) -> Result<Self::Instance> {+ transmit::bindings::TransmitCallee::instantiate_async(store, &component, &linker).await}fn call<'a>(@@ -617,7 +613,6 @@ impl TransmitTest for StaticTransmitTest {fn from_result(_: impl AsContextMut<Data = Ctx>,- _: Instance,result: Self::Result,) -> Result<(StreamReader<String>,@@ -639,9 +634,8 @@ impl TransmitTest for DynamicTransmitTest {store: impl AsContextMut<Data = Ctx>,component: &Component,linker: &Linker<Ctx>,- ) -> Result<(Self::Instance, Instance)> {- let instance = linker.instantiate_async(store, component).await?;- Ok((instance, instance))+ ) -> Result<Self::Instance> {+ linker.instantiate_async(store, component).await}async fn call<'a>(@@ -688,7 +682,6 @@ impl TransmitTest for DynamicTransmitTest {fn from_result(mut store: impl AsContextMut<Data = Ctx>,- instance: Instance,result: Self::Result,) -> Result<(StreamReader<String>,@@ -698,9 +691,9 @@ impl TransmitTest for DynamicTransmitTest {let Val::Tuple(fields) = result else {unreachable!()};- let stream = StreamReader::from_val(store.as_context_mut(), instance, &fields[0])?;- let future1 = FutureReader::from_val(store.as_context_mut(), instance, &fields[1])?;- let future2 = FutureReader::from_val(store.as_context_mut(), instance, &fields[2])?;+ let stream = StreamReader::from_val(store.as_context_mut(), &fields[0])?;+ let future1 = FutureReader::from_val(store.as_context_mut(), &fields[1])?;+ let future2 = FutureReader::from_val(store.as_context_mut(), &fields[2])?;Ok((stream, future1, future2))}}@@ -729,7 +722,7 @@ async fn test_transmit_with<Test: TransmitTest + 'static>(component: &str) -> Re},);- let (test, instance) = Test::instantiate(&mut store, &component, &linker).await?;+ let test = Test::instantiate(&mut store, &component, &linker).await?;enum Event<Test: TransmitTest> {Result(Test::Result),@@ -744,26 +737,17 @@ async fn test_transmit_with<Test: TransmitTest + 'static>(component: &str) -> Re}let (mut control_tx, control_rx) = mpsc::channel(1);- let control_rx = StreamReader::new(instance, &mut store, PipeProducer::new(control_rx));+ let control_rx = StreamReader::new(&mut store, PipeProducer::new(control_rx));let (mut caller_stream_tx, caller_stream_rx) = mpsc::channel(1);- let caller_stream_rx =- StreamReader::new(instance, &mut store, PipeProducer::new(caller_stream_rx));+ let caller_stream_rx = StreamReader::new(&mut store, PipeProducer::new(caller_stream_rx));let (caller_future1_tx, caller_future1_rx) = oneshot::channel();- let caller_future1_rx = FutureReader::new(- instance,- &mut store,- OneshotProducer::new(caller_future1_rx),- );+ let caller_future1_rx = FutureReader::new(&mut store, OneshotProducer::new(caller_future1_rx));let (_, caller_future2_rx) = oneshot::channel();- let caller_future2_rx = FutureReader::new(- instance,- &mut store,- OneshotProducer::new(caller_future2_rx),- );+ let caller_future2_rx = FutureReader::new(&mut store, OneshotProducer::new(caller_future2_rx));let (callee_future1_tx, callee_future1_rx) = oneshot::channel();let (callee_stream_tx, callee_stream_rx) = mpsc::channel(1);- instance- .run_concurrent(&mut store, async |accessor| {+ store+ .run_concurrent(async |accessor| {let mut caller_future1_tx = Some(caller_future1_tx);let mut callee_future1_tx = Some(callee_future1_tx);let mut callee_future1_rx = Some(callee_future1_rx);@@ -810,7 +794,7 @@ async fn test_transmit_with<Test: TransmitTest + 'static>(component: &str) -> ReEvent::Result(result) => {accessor.with(|mut store| {let (callee_stream_rx, callee_future1_rx, _) =- Test::from_result(&mut store, instance, result)?;+ Test::from_result(&mut store, result)?;callee_stream_rx.pipe(&mut store,PipeConsumer::new(callee_stream_tx.take().unwrap()),@@ -950,13 +934,9 @@ async fn test_synchronous_transmit(component: &str, procrastinate: bool) -> Resusleep: sleep(),};let stream = if procrastinate {- StreamReader::new(- instance,- &mut store,- ProcrastinatingStreamProducer(producer),- )+ StreamReader::new(&mut store, ProcrastinatingStreamProducer(producer))} else {- StreamReader::new(instance, &mut store, producer)+ StreamReader::new(&mut store, producer)};let future_expected = 10;let producer = DelayedFutureProducer {@@ -966,16 +946,12 @@ async fn test_synchronous_transmit(component: &str, procrastinate: bool) -> Resusleep: sleep(),};let future = if procrastinate {- FutureReader::new(- instance,- &mut store,- ProcrastinatingFutureProducer(producer),- )+ FutureReader::new(&mut store, ProcrastinatingFutureProducer(producer))} else {- FutureReader::new(instance, &mut store, producer)+ FutureReader::new(&mut store, producer)};- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {let ((stream, stream_expected, future, future_expected), task_exit) = guest.local_local_synchronous_transmit().call_start(accessor, stream, stream_expected, future, future_expected)diff --git a/crates/misc/component-async-tests/tests/scenario/util.rs b/crates/misc/component-async-tests/tests/scenario/util.rsindex 2ee4577cd2..10aeae2f4a 100644--- a/crates/misc/component-async-tests/tests/scenario/util.rs+++ b/crates/misc/component-async-tests/tests/scenario/util.rs@@ -210,13 +210,14 @@ pub async fn test_run_with_count(components: &[&str], count: usize) -> Result<()});}- let instance = linker.instantiate_async(&mut store, &component).await?;- let yield_host =- component_async_tests::yield_host::bindings::YieldHost::new(&mut store, &instance)?;+ let yield_host = component_async_tests::yield_host::bindings::YieldHost::instantiate_async(+ &mut store, &component, &linker,+ )+ .await?;// Start `count` concurrent calls and then join them all:- instance- .run_concurrent(&mut store, async |store| {+ store+ .run_concurrent(async |store| {let mut futures = FuturesUnordered::new();for _ in 0..count {futures.push(yield_host.local_local_run().call_run(store));diff --git a/crates/test-programs/src/bin/p3_http_proxy.rs b/crates/test-programs/src/bin/p3_http_proxy.rsindex 05fc2120e2..532d9d37e0 100644--- a/crates/test-programs/src/bin/p3_http_proxy.rs+++ b/crates/test-programs/src/bin/p3_http_proxy.rs@@ -15,7 +15,7 @@ struct Component;test_programs::p3::proxy::export!(Component);impl Handler for Component {- /// Return a response which echoes the request headers, body, and trailers.+ // Forward the request body and trailers to a URL specified in a header.async fn handle(request: Request) -> Result<Response, ErrorCode> {let headers = request.get_headers().copy_all();Ok(diff --git a/crates/wasi-http/src/p3/body.rs b/crates/wasi-http/src/p3/body.rsindex 0779f4c7db..1d74ffb3f9 100644--- a/crates/wasi-http/src/p3/body.rs+++ b/crates/wasi-http/src/p3/body.rs@@ -93,19 +93,13 @@ impl Body {result_tx,} => {fut.pipe(&mut store, BodyResultConsumer(Some(result_tx)));- let instance = store.instance();- (- StreamReader::new(instance, &mut store, iter::empty()),- trailers_rx,- )+ (StreamReader::new(&mut store, iter::empty()), trailers_rx)}Body::Host { body, result_tx } => {fut.pipe(&mut store, BodyResultConsumer(Some(result_tx)));- let instance = store.instance();let (trailers_tx, trailers_rx) = oneshot::channel();(StreamReader::new(- instance,&mut store,HostBodyStreamProducer {body,@@ -113,7 +107,7 @@ impl Body {getter,},),- FutureReader::new(instance, &mut store, trailers_rx),+ FutureReader::new(&mut store, trailers_rx),)}}diff --git a/crates/wasi-http/src/p3/host/types.rs b/crates/wasi-http/src/p3/host/types.rsindex 08db168b75..d0ea83ba06 100644--- a/crates/wasi-http/src/p3/host/types.rs+++ b/crates/wasi-http/src/p3/host/types.rs@@ -319,7 +319,6 @@ impl HostRequestWithStore for WasiHttp {trailers: FutureReader<Result<Option<Resource<Trailers>>, ErrorCode>>,options: Option<Resource<RequestOptions>>,) -> wasmtime::Result<(Resource<Request>, FutureReader<Result<(), ErrorCode>>)> {- let instance = store.instance();store.with(|mut store| {let (result_tx, result_rx) = oneshot::channel();let body = match contents@@ -357,11 +356,7 @@ impl HostRequestWithStore for WasiHttp {let req = table.push(req).context("failed to push request to table")?;Ok((req,- FutureReader::new(- instance,- &mut store,- GuestBodyResultProducer::Receiver(result_rx),- ),+ FutureReader::new(&mut store, GuestBodyResultProducer::Receiver(result_rx)),))})}@@ -613,7 +608,6 @@ impl HostResponseWithStore for WasiHttp {contents: Option<StreamReader<u8>>,trailers: FutureReader<Result<Option<Resource<Trailers>>, ErrorCode>>,) -> wasmtime::Result<(Resource<Response>, FutureReader<Result<(), ErrorCode>>)> {- let instance = store.instance();store.with(|mut store| {let (result_tx, result_rx) = oneshot::channel();let body = match contents@@ -646,11 +640,7 @@ impl HostResponseWithStore for WasiHttp {.context("failed to push response to table")?;Ok((res,- FutureReader::new(- instance,- &mut store,- GuestBodyResultProducer::Receiver(result_rx),- ),+ FutureReader::new(&mut store, GuestBodyResultProducer::Receiver(result_rx)),))})}diff --git a/crates/wasi-http/tests/all/p3/mod.rs b/crates/wasi-http/tests/all/p3/mod.rsindex eadd64988d..279d928c46 100644--- a/crates/wasi-http/tests/all/p3/mod.rs+++ b/crates/wasi-http/tests/all/p3/mod.rs@@ -138,12 +138,9 @@ async fn run_cli(path: &str, server: &Server) -> anyhow::Result<()> {wasmtime_wasi::p3::add_to_linker(&mut linker).context("failed to link `wasi:[email protected]`")?;wasmtime_wasi_http::p3::add_to_linker(&mut linker).context("failed to link `wasi:[email protected]`")?;- let instance = linker.instantiate_async(&mut store, &component).await?;- let command = Command::new(&mut store, &instance)?;- instance- .run_concurrent(store, async |store| {- command.wasi_cli_run().call_run(store).await- })+ let command = Command::instantiate_async(&mut store, &component, &linker).await?;+ store+ .run_concurrent(async |store| command.wasi_cli_run().call_run(store).await).await.context("failed to call `wasi:cli/run#run`")?.context("guest trapped")?@@ -170,14 +167,13 @@ async fn run_http<E: Into<ErrorCode> + 'static>(wasmtime_wasi::p3::add_to_linker(&mut linker).context("failed to link `wasi:[email protected]`")?;wasmtime_wasi_http::p3::add_to_linker(&mut linker).context("failed to link `wasi:[email protected]`")?;- let instance = linker.instantiate_async(&mut store, &component).await?;- let proxy = Proxy::new(&mut store, &instance)?;+ let proxy = Proxy::instantiate_async(&mut store, &component, &linker).await?;let (req, io) = Request::from_http(req);let (tx, rx) = tokio::sync::oneshot::channel();let ((handle_result, ()), res) = try_join!(async move {- instance- .run_concurrent(&mut store, async |store| {+ store+ .run_concurrent(async |store| {try_join!(async {let (res, task) = match proxy.handle(store, req).await? {diff --git a/crates/wasi/src/p3/bindings.rs b/crates/wasi/src/p3/bindings.rsindex f0237ad08f..d5dfac16b7 100644--- a/crates/wasi/src/p3/bindings.rs+++ b/crates/wasi/src/p3/bindings.rs@@ -156,9 +156,8 @@ pub use self::generated::wasi::*;/// );////// // Instantiate the component and we're off to the races.-/// let instance = linker.instantiate_async(&mut store, &component).await?;-/// let command = Command::new(&mut store, &instance)?;-/// let program_result = instance.run_concurrent(&mut store, async move |store| {+/// let command = Command::instantiate_async(&mut store, &component, &linker).await?;+/// let program_result = store.run_concurrent(async move |store| {/// command.wasi_cli_run().call_run(store).await/// }).await??;/// match program_result {diff --git a/crates/wasi/src/p3/cli/host.rs b/crates/wasi/src/p3/cli/host.rsindex f18e3c1444..7ab3a1047e 100644--- a/crates/wasi/src/p3/cli/host.rs+++ b/crates/wasi/src/p3/cli/host.rs@@ -196,19 +196,17 @@ impl stdin::HostWithStore for WasiCli {async fn read_via_stream<U>(store: &Accessor<U, Self>,) -> wasmtime::Result<(StreamReader<u8>, FutureReader<Result<(), ErrorCode>>)> {- let instance = store.instance();store.with(|mut store| {let rx = store.get().ctx.stdin.async_stream();let (result_tx, result_rx) = oneshot::channel();let stream = StreamReader::new(- instance,&mut store,InputStreamProducer {rx: Box::into_pin(rx),result_tx: Some(result_tx),},);- let future = FutureReader::new(instance, &mut store, async {+ let future = FutureReader::new(&mut store, async {anyhow::Ok(match result_rx.await {Ok(err) => Err(err),Err(_) => Ok(()),diff --git a/crates/wasi/src/p3/filesystem/host.rs b/crates/wasi/src/p3/filesystem/host.rsindex df94cdefb7..edeabf6a25 100644--- a/crates/wasi/src/p3/filesystem/host.rs+++ b/crates/wasi/src/p3/filesystem/host.rs@@ -490,13 +490,12 @@ impl types::HostDescriptorWithStore for WasiFilesystem {fd: Resource<Descriptor>,offset: Filesize,) -> wasmtime::Result<(StreamReader<u8>, FutureReader<Result<(), ErrorCode>>)> {- let instance = store.instance();store.with(|mut store| {let file = get_file(store.get().table, &fd)?;if !file.perms.contains(FilePerms::READ) {return Ok((- StreamReader::new(instance, &mut store, iter::empty()),- FutureReader::new(instance, &mut store, async {+ StreamReader::new(&mut store, iter::empty()),+ FutureReader::new(&mut store, async {anyhow::Ok(Err(ErrorCode::NotPermitted))}),));@@ -506,7 +505,6 @@ impl types::HostDescriptorWithStore for WasiFilesystem {let (result_tx, result_rx) = oneshot::channel();Ok((StreamReader::new(- instance,&mut store,ReadStreamProducer {file,@@ -515,7 +513,7 @@ impl types::HostDescriptorWithStore for WasiFilesystem {task: None,},),- FutureReader::new(instance, &mut store, result_rx),+ FutureReader::new(&mut store, result_rx),))})}@@ -634,13 +632,12 @@ impl types::HostDescriptorWithStore for WasiFilesystem {StreamReader<DirectoryEntry>,FutureReader<Result<(), ErrorCode>>,)> {- let instance = store.instance();store.with(|mut store| {let dir = get_dir(store.get().table, &fd)?;if !dir.perms.contains(DirPerms::READ) {return Ok((- StreamReader::new(instance, &mut store, iter::empty()),- FutureReader::new(instance, &mut store, async {+ StreamReader::new(&mut store, iter::empty()),+ FutureReader::new(&mut store, async {anyhow::Ok(Err(ErrorCode::NotPermitted))}),));@@ -651,7 +648,6 @@ impl types::HostDescriptorWithStore for WasiFilesystem {let stream = if allow_blocking_current_thread {match dir.entries() {Ok(readdir) => StreamReader::new(- instance,&mut store,FallibleIteratorProducer::new(readdir.filter_map(|e| map_dir_entry(e).transpose()),@@ -660,13 +656,13 @@ impl types::HostDescriptorWithStore for WasiFilesystem {),Err(e) => {result_tx.send(Err(e.into())).unwrap();- StreamReader::new(instance, &mut store, iter::empty())+ StreamReader::new(&mut store, iter::empty())}}} else {- StreamReader::new(instance, &mut store, ReadDirStream::new(dir, result_tx))+ StreamReader::new(&mut store, ReadDirStream::new(dir, result_tx))};- Ok((stream, FutureReader::new(instance, &mut store, result_rx)))+ Ok((stream, FutureReader::new(&mut store, result_rx)))})}diff --git a/crates/wasi/src/p3/sockets/host/types/tcp.rs b/crates/wasi/src/p3/sockets/host/types/tcp.rsindex 441ce14db7..b29f3b7d56 100644--- a/crates/wasi/src/p3/sockets/host/types/tcp.rs+++ b/crates/wasi/src/p3/sockets/host/types/tcp.rs@@ -282,7 +282,6 @@ impl HostTcpSocketWithStore for WasiSockets {store: &Accessor<T, Self>,socket: Resource<TcpSocket>,) -> SocketResult<StreamReader<Resource<TcpSocket>>> {- let instance = store.instance();let getter = store.getter();store.with(|mut store| {let socket = get_socket_mut(store.get().table, &socket)?;@@ -292,7 +291,6 @@ impl HostTcpSocketWithStore for WasiSockets {let family = socket.address_family();let options = socket.non_inherited_options().clone();Ok(StreamReader::new(- instance,&mut store,ListenStreamProducer {listener,@@ -334,7 +332,6 @@ impl HostTcpSocketWithStore for WasiSockets {store: &Accessor<T, Self>,socket: Resource<TcpSocket>,) -> wasmtime::Result<(StreamReader<u8>, FutureReader<Result<(), ErrorCode>>)> {- let instance = store.instance();store.with(|mut store| {let socket = get_socket_mut(store.get().table, &socket)?;match socket.start_receive() {@@ -343,19 +340,18 @@ impl HostTcpSocketWithStore for WasiSockets {let (result_tx, result_rx) = oneshot::channel();Ok((StreamReader::new(- instance,&mut store,ReceiveStreamProducer {stream,result: Some(result_tx),},),- FutureReader::new(instance, &mut store, result_rx),+ FutureReader::new(&mut store, result_rx),))}None => Ok((- StreamReader::new(instance, &mut store, iter::empty()),- FutureReader::new(instance, &mut store, async {+ StreamReader::new(&mut store, iter::empty()),+ FutureReader::new(&mut store, async {anyhow::Ok(Err(ErrorCode::InvalidState))}),)),diff --git a/crates/wasi/tests/all/p3/mod.rs b/crates/wasi/tests/all/p3/mod.rsindex 40d06b5e1e..95c16be477 100644--- a/crates/wasi/tests/all/p3/mod.rs+++ b/crates/wasi/tests/all/p3/mod.rs@@ -33,13 +33,11 @@ async fn run_allow_blocking_current_thread(table: Default::default(),})?;let component = Component::from_file(&engine, path)?;- let instance = linker.instantiate_async(&mut store, &component).await?;- let command =- Command::new(&mut store, &instance).context("failed to instantiate `wasi:cli/command`")?;- instance- .run_concurrent(&mut store, async move |store| {- command.wasi_cli_run().call_run(store).await- })+ let command = Command::instantiate_async(&mut store, &component, &linker)+ .await+ .context("failed to instantiate `wasi:cli/command`")?;+ store+ .run_concurrent(async move |store| command.wasi_cli_run().call_run(store).await).await.context("failed to call `wasi:cli/run#run`")?.context("guest trapped")?diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex 65eea83f3a..abfb131480 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -8,11 +8,11 @@//!//! At the core of this support is an event loop which schedules and switches//! between guest tasks and any host tasks they create. Each-//! `ComponentInstance` will have at most one event loop running at any given+//! `Store` will have at most one event loop running at any given//! time, and that loop may be suspended and resumed by the host embedder using-//! e.g. `Instance::run_concurrent`. The `ComponentInstance::poll_until`+//! e.g. `StoreContextMut::run_concurrent`. The `StoreContextMut::poll_until`//! function contains the loop itself, while the-//! `ComponentInstance::concurrent_state` field holds its state.+//! `StoreOpaque::concurrent_state` field holds its state.//!//! # Public API Overview//!@@ -21,39 +21,39 @@//! - `[Typed]Func::call_concurrent`: Start a host->guest call to an//! async-lifted or sync-lifted import, creating a guest task.//!-//! - `Instance::run_concurrent`: Run the event loop for the specified instance,-//! allowing any and all tasks belonging to that instance to make progress.+//! - `StoreContextMut::run_concurrent`: Run the event loop for the specified+//! instance, allowing any and all tasks belonging to that instance to make+//! progress.//!-//! - `Instance::spawn`: Run a background task as part of the event loop for the-//! specified instance.+//! - `StoreContextMut::spawn`: Run a background task as part of the event loop+//! for the specified instance.//!-//! - `Instance::{future,stream}`: Create a new Component Model `future` or-//! `stream`; the read end may be passed to the guest.+//! - `{Future,Stream}Reader::new`: Create a new Component Model `future` or+//! `stream` which may be passed to the guest. This takes a+//! `{Future,Stream}Producer` implementation which will be polled for items when+//! the consumer requests them.//!-//! - `{Future,Stream}Reader::read` and `{Future,Stream}Writer::write`: read-//! from or write to a future or stream, respectively.+//! - `{Future,Stream}Reader::pipe`: Consume a `future` or `stream` by+//! connecting it to a `{Future,Stream}Consumer` which will consume any items+//! produced by the write end.//!//! ## Host Task API (e.g. implementing concurrent host functions and background tasks)//!//! - `LinkerInstance::func_wrap_concurrent`: Register a concurrent host//! function with the linker. That function will take an `Accessor` as its-//! first parameter, which provides access to the store and instance between-//! (but not across) await points.+//! first parameter, which provides access to the store between (but not across)+//! await points.//!-//! - `Accessor::with`: Access the store, its associated data, and the current-//! instance.+//! - `Accessor::with`: Access the store and its associated data.//!//! - `Accessor::spawn`: Run a background task as part of the event loop for the-//! specified instance. This is equivalent to `Instance::spawn` but more-//! convenient to use in host functions.+//! store. This is equivalent to `StoreContextMut::spawn` but more convenient to use+//! in host functions.use crate::component::func::{self, Func, Options};-use crate::component::{- Component, ComponentInstanceId, HasData, HasSelf, Instance, Resource, ResourceTable,- ResourceTableError,-};+use crate::component::{HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError};use crate::fiber::{self, StoreFiber, StoreFiberYield};-use crate::store::{StoreInner, StoreOpaque, StoreToken};+use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};use crate::vm::component::{CallContext, ComponentInstance, InstanceFlags, ResourceTables, TransmitLocalState,};@@ -207,7 +207,6 @@ const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYpub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {store: StoreContextMut<'a, T>,get_data: fn(&mut T) -> D::Data<'_>,- instance: Option<Instance>,}impl<'a, T, D> Access<'a, T, D>@@ -217,11 +216,7 @@ where{/// Creates a new [`Access`] from its component parts.pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {- Self {- store,- get_data,- instance: None,- }+ Self { store, get_data }}/// Get mutable access to the store data.@@ -243,17 +238,11 @@ where{let accessor = Accessor {get_data: self.get_data,- instance: self.instance,token: StoreToken::new(self.store.as_context_mut()),};- self.instance- .unwrap()- .spawn_with_accessor(self.store.as_context_mut(), accessor, task)- }-- /// Retrieve the component instance of the caller.- pub fn instance(&self) -> Instance {- self.instance.unwrap()+ self.store+ .as_context_mut()+ .spawn_with_accessor(accessor, task)}}@@ -344,7 +333,6 @@ where{token: StoreToken<T>,get_data: fn(&mut T) -> D::Data<'_>,- instance: Option<Instance>,}/// A helper trait to take any type of accessor-with-data in functions.@@ -359,8 +347,9 @@ where/// `Accessor<T, D>` it can still be passed to a function which just needs a/// store accessor.///-/// Acquiring an [`Accessor`] can be done through [`Instance::run_concurrent`]-/// for example or in a host function through+/// Acquiring an [`Accessor`] can be done through+/// [`StoreContextMut::run_concurrent`] for example or in a host function+/// through/// [`Linker::func_wrap_concurrent`](crate::component::Linker::func_wrap_concurrent).pub trait AsAccessor {/// The `T` in `Store<T>` that this accessor refers to.@@ -428,14 +417,10 @@ impl<T> Accessor<T> {/// another type (e.g. a field of that data or a wrapper around it).////// - `spawn`: used to queue spawned background tasks to be run later- ///- /// - `instance`: used to access the `Instance` to which this `Accessor`- /// (and the future which closes over it) belongs- pub(crate) fn new(token: StoreToken<T>, instance: Option<Instance>) -> Self {+ pub(crate) fn new(token: StoreToken<T>) -> Self {Self {token,get_data: |x| x,- instance,}}}@@ -466,7 +451,6 @@ wherefun(Access {store: self.token.as_context_mut(vmstore),get_data: self.get_data,- instance: self.instance,})})}@@ -500,13 +484,12 @@ whereAccessor {token: self.token,get_data,- instance: self.instance,}}/// Spawn a background task which will receive an `&Accessor<T, D>` and/// run concurrently with any other tasks in progress for the current- /// instance.+ /// store.////// This is particularly useful for host functions which return a `stream`/// or `future` such that the code to write to the write end of that@@ -523,39 +506,29 @@ wherewhereT: 'static,{- let instance = self.instance.unwrap();let accessor = self.clone_for_spawn();- self.with(|mut access| {- instance.spawn_with_accessor(access.as_context_mut(), accessor, task)- })- }-- /// Retrieve the component instance of the caller.- pub fn instance(&self) -> Instance {- self.instance.unwrap()+ self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))}fn clone_for_spawn(&self) -> Self {Self {token: self.token,get_data: self.get_data,- instance: self.instance,}}}/// Represents a task which may be provided to `Accessor::spawn`,-/// `Accessor::forward`, or `Instance::spawn`.+/// `Accessor::forward`, or `StorecContextMut::spawn`.// TODO: Replace this with `std::ops::AsyncFnOnce` when that becomes a viable// option.//-// `AsyncFnOnce` is still nightly-only in latest stable Rust version as of this-// writing (1.84.1), and even with 1.85.0-beta it's not possible to specify-// e.g. `Send` and `Sync` bounds on the `Future` type returned by an-// `AsyncFnOnce`. Also, using `F: Future<Output = Result<()>> + Send + Sync,-// FN: FnOnce(&Accessor<T>) -> F + Send + Sync + 'static` fails with a type-// mismatch error when we try to pass it an async closure (e.g. `async move |_|-// { ... }`). So this seems to be the best we can do for the time being.+// As of this writing, it's not possible to specify e.g. `Send` and `Sync`+// bounds on the `Future` type returned by an `AsyncFnOnce`. Also, using `F:+// Future<Output = Result<()>> + Send + Sync, FN: FnOnce(&Accessor<T>) -> F ++// Send + Sync + 'static` fails with a type mismatch error when we try to pass+// it an async closure (e.g. `async move |_| { ... }`). So this seems to be the+// best we can do for the time being.pub trait AccessorTask<T, D, R>: Send + 'staticwhereD: HasData + ?Sized,@@ -585,7 +558,7 @@ enum WaitMode {Fiber(StoreFiber<'static>),/// The guest task is waiting via a callback declared as part of an/// async-lifted export.- Callback,+ Callback(Instance),}/// Represents the reason a fiber is suspending itself.@@ -610,6 +583,8 @@ enum GuestCallKind {/// Indicates there's an event to deliver to the task, possibly related to a/// waitable set the task has been waiting on or polling.DeliverEvent {+ /// The instance to which the task belongs.+ instance: Instance,/// The waitable set the event belongs to, if any.////// If this is `None` the event will be waiting in the@@ -618,13 +593,17 @@ enum GuestCallKind {},/// Indicates that a new guest task call is pending and may be executed/// using the specified closure.- Start(Box<dyn FnOnce(&mut dyn VMStore, Instance) -> Result<()> + Send + Sync>),+ Start(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),}impl fmt::Debug for GuestCallKind {fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {match self {- Self::DeliverEvent { set } => f.debug_struct("DeliverEvent").field("set", set).finish(),+ Self::DeliverEvent { instance, set } => f+ .debug_struct("DeliverEvent")+ .field("instance", instance)+ .field("set", set)+ .finish(),Self::Start(_) => f.debug_tuple("Start").finish(),}}@@ -666,14 +645,16 @@ impl GuestCall {/// Job to be run on a worker fiber.enum WorkerItem {GuestCall(GuestCall),- Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore, Instance) -> Result<()> + Send>>),+ Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),}/// Represents state related to an in-progress poll operation (e.g. `task.poll`/// or `CallbackCode.POLL`).#[derive(Debug)]struct PollParams {- /// Identifies the polling task.+ /// The instance to which the polling task belongs.+ instance: Instance,+ /// The polling task.task: TableId<GuestTask>,/// The waitable set being polled.set: TableId<WaitableSet>,@@ -691,7 +672,7 @@ enum WorkItem {/// A pending `task.poll` or `CallbackCode.POLL` operation.Poll(PollParams),/// A job to run on a worker fiber.- WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore, Instance) -> Result<()> + Send>>),+ WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),}impl fmt::Debug for WorkItem {@@ -706,337 +687,261 @@ impl fmt::Debug for WorkItem {}}-impl ComponentInstance {- /// Handle the `CallbackCode` returned from an async-lifted export or its- /// callback.- ///- /// If `initial_call` is `true`, then the code was received from the- /// async-lifted export; otherwise, it was received from its callback.- fn handle_callback_code(- mut self: Pin<&mut Self>,- guest_task: TableId<GuestTask>,- runtime_instance: RuntimeComponentInstanceIndex,- code: u32,- initial_call: bool,- ) -> Result<()> {- let (code, set) = unpack_callback_code(code);-- log::trace!("received callback code from {guest_task:?}: {code} (set: {set})");-- let state = self.as_mut().concurrent_state_mut();- let task = state.get_mut(guest_task)?;-- if task.lift_result.is_some() {- if code == callback_code::EXIT {- return Err(anyhow!(crate::Trap::NoAsyncResult));- }- if initial_call {- // Notify any current or future waiters that this subtask has- // started.- Waitable::Guest(guest_task).set_event(- state,- Some(Event::Subtask {- status: Status::Started,- }),- )?;- }- }-- let get_set = |instance: Pin<&mut Self>, handle| {- if handle == 0 {- bail!("invalid waitable-set handle");+/// Poll the specified future until it completes on behalf of a guest->host call+/// using a sync-lowered import.+///+/// This is similar to `Instance::first_poll` except it's for sync-lowered+/// imports, meaning we don't need to handle cancellation and we can block the+/// caller until the task completes, at which point the caller can handle+/// lowering the result to the guest's stack and linear memory.+pub(crate) fn poll_and_block<R: Send + Sync + 'static>(+ store: &mut dyn VMStore,+ future: impl Future<Output = Result<R>> + Send + 'static,+ caller_instance: RuntimeComponentInstanceIndex,+) -> Result<R> {+ let state = store.concurrent_state_mut();++ // If there is no current guest task set, that means the host function was+ // registered using e.g. `LinkerInstance::func_wrap`, in which case it+ // should complete immediately.+ let Some(caller) = state.guest_task else {+ return match pin!(future).poll(&mut Context::from_waker(&Waker::noop())) {+ Poll::Ready(result) => result,+ Poll::Pending => {+ unreachable!()}+ };+ };- let set = instance.guest_tables().0[runtime_instance].waitable_set_rep(handle)?;+ // Save any existing result stashed in `GuestTask::result` so we can replace+ // it with the new result.+ let old_result = state+ .get_mut(caller)+ .with_context(|| format!("bad handle: {caller:?}"))?+ .result+ .take();++ // Add a temporary host task into the table so we can track its progress.+ // Note that we'll never allocate a waitable handle for the guest since+ // we're being called synchronously.+ let task = state.push(HostTask::new(caller_instance, None))?;++ log::trace!("new host task child of {caller:?}: {task:?}");++ // Wrap the future in a closure which will take care of stashing the result+ // in `GuestTask::result` and resuming this fiber when the host task+ // completes.+ let mut future = Box::pin(async move {+ let result = future.await?;+ tls::get(move |store| {+ let state = store.concurrent_state_mut();+ state.get_mut(caller)?.result = Some(Box::new(result) as _);++ Waitable::Host(task).set_event(+ state,+ Some(Event::Subtask {+ status: Status::Returned,+ }),+ )?;- Ok(TableId::<WaitableSet>::new(set))- };+ Ok(())+ })+ }) as HostTaskFuture;- match code {- callback_code::EXIT => {- let task = state.get_mut(guest_task)?;- match &task.caller {- Caller::Host {- remove_task_automatically,- ..- } => {- if *remove_task_automatically {- log::trace!("handle_callback_code will delete task {guest_task:?}");- Waitable::Guest(guest_task).delete_from(state)?;- }- }- Caller::Guest { .. } => {- task.exited = true;- task.callback = None;- }- }- }- callback_code::YIELD => {- // Push this task onto the "low priority" queue so it runs after- // any other tasks have had a chance to run.- let task = state.get_mut(guest_task)?;- assert!(task.event.is_none());- task.event = Some(Event::None);- state.push_low_priority(WorkItem::GuestCall(GuestCall {- task: guest_task,- kind: GuestCallKind::DeliverEvent { set: None },- }));- }- callback_code::WAIT | callback_code::POLL => {- let set = get_set(self.as_mut(), set)?;- let state = self.concurrent_state_mut();+ // Finally, poll the future. We can use a dummy `Waker` here because we'll+ // add the future to `ConcurrentState::futures` and poll it automatically+ // from the event loop if it doesn't complete immediately here.+ let poll = tls::set(store, || {+ future+ .as_mut()+ .poll(&mut Context::from_waker(&Waker::noop()))+ });- if state.get_mut(guest_task)?.event.is_some()- || !state.get_mut(set)?.ready.is_empty()- {- // An event is immediately available; deliver it ASAP.- state.push_high_priority(WorkItem::GuestCall(GuestCall {- task: guest_task,- kind: GuestCallKind::DeliverEvent { set: Some(set) },- }));- } else {- // No event is immediately available.- match code {- callback_code::POLL => {- // We're polling, so just yield and check whether an- // event has arrived after that.- state.push_low_priority(WorkItem::Poll(PollParams {- task: guest_task,- set,- }));- }- callback_code::WAIT => {- // We're waiting, so register to be woken up when an- // event is published for this waitable set.- //- // Here we also set `GuestTask::wake_on_cancel`- // which allows `subtask.cancel` to interrupt the- // wait.- let old = state.get_mut(guest_task)?.wake_on_cancel.replace(set);- assert!(old.is_none());- let old = state- .get_mut(set)?- .waiting- .insert(guest_task, WaitMode::Callback);- assert!(old.is_none());- }- _ => unreachable!(),- }- }- }- _ => bail!("unsupported callback code: {code}"),+ match poll {+ Poll::Ready(result) => {+ // It completed immediately; check the result and delete the task.+ result?;+ log::trace!("delete host task {task:?} (already ready)");+ store.concurrent_state_mut().delete(task)?;+ }+ Poll::Pending => {+ // It did not complete immediately; add it to+ // `ConcurrentState::futures` so it will be polled via the event+ // loop; then use `GuestTask::sync_call_set` to wait for the task to+ // complete, suspending the current fiber until it does so.+ let state = store.concurrent_state_mut();+ state.push_future(future);++ let set = state.get_mut(caller)?.sync_call_set;+ Waitable::Host(task).join(state, Some(set))?;++ store.suspend(SuspendReason::Waiting { set, task: caller })?;}-- Ok(())}- /// Get the next pending event for the specified task and (optional)- /// waitable set, along with the waitable handle if applicable.- fn get_event(- mut self: Pin<&mut Self>,- guest_task: TableId<GuestTask>,- set: Option<TableId<WaitableSet>>,- cancellable: bool,- ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {- let state = self.as_mut().concurrent_state_mut();+ // Retrieve and return the result.+ Ok(*mem::replace(+ &mut store.concurrent_state_mut().get_mut(caller)?.result,+ old_result,+ )+ .unwrap()+ .downcast()+ .unwrap())+}- if let Some(event) = state.get_mut(guest_task)?.event.take() {- log::trace!("deliver event {event:?} to {guest_task:?}");+/// Execute the specified guest call.+fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {+ match call.kind {+ GuestCallKind::DeliverEvent { instance, set } => {+ let (event, waitable) = instance.get_event(store, call.task, set, true)?.unwrap();+ let state = store.concurrent_state_mut();+ let task = state.get_mut(call.task)?;+ let runtime_instance = task.instance;+ let handle = waitable.map(|(_, v)| v).unwrap_or(0);- if cancellable || !matches!(event, Event::Cancelled) {- return Ok(Some((event, None)));- } else {- state.get_mut(guest_task)?.event = Some(event);- }- }+ log::trace!(+ "use callback to deliver event {event:?} to {:?} for {waitable:?}",+ call.task,+ );- Ok(- if let Some((set, waitable)) = set- .and_then(|set| {- state- .get_mut(set)- .map(|v| v.ready.pop_first().map(|v| (set, v)))- .transpose()- })- .transpose()?- {- let common = waitable.common(state)?;- let handle = common.handle.unwrap();- let event = common.event.take().unwrap();+ let old_task = state.guest_task.replace(call.task);+ log::trace!(+ "GuestCallKind::DeliverEvent: replaced {old_task:?} with {:?} as current task",+ call.task+ );- log::trace!(- "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"- );+ store.maybe_push_call_context(call.task)?;- waitable.on_delivery(self, event);+ let state = store.concurrent_state_mut();+ state.enter_instance(runtime_instance);- Some((event, Some((waitable, handle))))- } else {- None- },- )- }+ let callback = state.get_mut(call.task)?.callback.take().unwrap();- /// Implements the `waitable-set.new` intrinsic.- pub(crate) fn waitable_set_new(- mut self: Pin<&mut Self>,- caller_instance: RuntimeComponentInstanceIndex,- ) -> Result<u32> {- self.check_may_leave(caller_instance)?;- let set = self- .as_mut()- .concurrent_state_mut()- .push(WaitableSet::default())?;- let handle = self.guest_tables().0[caller_instance].waitable_set_insert(set.rep())?;- log::trace!("new waitable set {set:?} (handle {handle})");- Ok(handle)- }+ let code = callback(store, runtime_instance, event, handle)?;- /// Implements the `waitable-set.drop` intrinsic.- pub(crate) fn waitable_set_drop(- mut self: Pin<&mut Self>,- caller_instance: RuntimeComponentInstanceIndex,- set: u32,- ) -> Result<()> {- self.check_may_leave(caller_instance)?;- let rep = self.as_mut().guest_tables().0[caller_instance].waitable_set_remove(set)?;+ let state = store.concurrent_state_mut();- log::trace!("drop waitable set {rep} (handle {set})");+ state.get_mut(call.task)?.callback = Some(callback);- let set = self- .concurrent_state_mut()- .delete(TableId::<WaitableSet>::new(rep))?;+ state.exit_instance(runtime_instance)?;- if !set.waiting.is_empty() {- bail!("cannot drop waitable set with waiters");- }+ store.maybe_pop_call_context(call.task)?;- Ok(())+ instance.handle_callback_code(store, call.task, runtime_instance, code, false)?;++ store.concurrent_state_mut().guest_task = old_task;+ log::trace!("GuestCallKind::DeliverEvent: restored {old_task:?} as current task");+ }+ GuestCallKind::Start(fun) => {+ fun(store)?;+ }}- /// Implements the `waitable.join` intrinsic.- pub(crate) fn waitable_join(- mut self: Pin<&mut Self>,- caller_instance: RuntimeComponentInstanceIndex,- waitable_handle: u32,- set_handle: u32,- ) -> Result<()> {- self.check_may_leave(caller_instance)?;- let waitable = Waitable::from_instance(self.as_mut(), caller_instance, waitable_handle)?;+ Ok(())+}- let set = if set_handle == 0 {- None- } else {- let set =- self.as_mut().guest_tables().0[caller_instance].waitable_set_rep(set_handle)?;+impl<T> Store<T> {+ /// Convenience wrapper for [`StoreContextMut::run_concurrent`].+ pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>+ where+ T: Send + 'static,+ {+ self.as_context_mut().run_concurrent(fun).await+ }- Some(TableId::<WaitableSet>::new(set))- };+ #[doc(hidden)]+ pub fn assert_concurrent_state_empty(&mut self) {+ self.as_context_mut().assert_concurrent_state_empty();+ }- log::trace!(- "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",+ /// Convenience wrapper for [`StoreContextMut::spawn`].+ pub fn spawn(&mut self, task: impl AccessorTask<T, HasSelf<T>, Result<()>>) -> JoinHandle+ where+ T: 'static,+ {+ self.as_context_mut().spawn(task)+ }+}++impl<T> StoreContextMut<'_, T> {+ /// Assert that all the relevant tables and queues in the concurrent state+ /// for this store are empty.+ ///+ /// This is for sanity checking in integration tests+ /// (e.g. `component-async-tests`) that the relevant state has been cleared+ /// after each test concludes. This should help us catch leaks, e.g. guest+ /// tasks which haven't been deleted despite having completed and having+ /// been dropped by their supertasks.+ #[doc(hidden)]+ pub fn assert_concurrent_state_empty(self) {+ let store = self.0;+ store+ .store_data_mut()+ .components+ .assert_guest_tables_empty();+ let state = store.concurrent_state_mut();+ assert!(+ state.table.get_mut().is_empty(),+ "non-empty table: {:?}",+ state.table.get_mut()+ );+ assert!(state.high_priority.is_empty());+ assert!(state.low_priority.is_empty());+ assert!(state.guest_task.is_none());+ assert!(state.futures.get_mut().as_ref().unwrap().is_empty());+ assert!(+ state+ .instance_states+ .iter()+ .all(|(_, state)| state.pending.is_empty()));+ assert!(state.global_error_context_ref_counts.is_empty());+ }- waitable.join(self.concurrent_state_mut(), set)+ /// Spawn a background task to run as part of this instance's event loop.+ ///+ /// The task will receive an `&Accessor<U>` and run concurrently with+ /// any other tasks in progress for the instance.+ ///+ /// Note that the task will only make progress if and when the event loop+ /// for this instance is run.+ ///+ /// The returned [`SpawnHandle`] may be used to cancel the task.+ pub fn spawn(mut self, task: impl AccessorTask<T, HasSelf<T>, Result<()>>) -> JoinHandle+ where+ T: 'static,+ {+ let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));+ self.spawn_with_accessor(accessor, task)}- /// Implements the `subtask.drop` intrinsic.- pub(crate) fn subtask_drop(- mut self: Pin<&mut Self>,- caller_instance: RuntimeComponentInstanceIndex,- task_id: u32,- ) -> Result<()> {- self.check_may_leave(caller_instance)?;- self.as_mut().waitable_join(caller_instance, task_id, 0)?;-- let (rep, is_host) =- self.as_mut().guest_tables().0[caller_instance].subtask_remove(task_id)?;-- let concurrent_state = self.concurrent_state_mut();- let (waitable, expected_caller_instance, delete) = if is_host {- let id = TableId::<HostTask>::new(rep);- let task = concurrent_state.get_mut(id)?;- if task.join_handle.is_some() {- bail!("cannot drop a subtask which has not yet resolved");- }- (Waitable::Host(id), task.caller_instance, true)- } else {- let id = TableId::<GuestTask>::new(rep);- let task = concurrent_state.get_mut(id)?;- if task.lift_result.is_some() {- bail!("cannot drop a subtask which has not yet resolved");- }- if let Caller::Guest { instance, .. } = &task.caller {- (Waitable::Guest(id), *instance, task.exited)- } else {- unreachable!()- }- };-- waitable.common(concurrent_state)?.handle = None;-- if waitable.take_event(concurrent_state)?.is_some() {- bail!("cannot drop a subtask with an undelivered event");- }-- if delete {- waitable.delete_from(concurrent_state)?;- }-- // Since waitables can neither be passed between instances nor forged,- // this should never fail unless there's a bug in Wasmtime, but we check- // here to be sure:- assert_eq!(expected_caller_instance, caller_instance);- log::trace!("subtask_drop {waitable:?} (handle {task_id})");- Ok(())- }-}--impl Instance {- /// Assert that all the relevant tables and queues in the concurrent state- /// for this instance are empty.- ///- /// This is for sanity checking in integration tests- /// (e.g. `component-async-tests`) that the relevant state has been cleared- /// after each test concludes. This should help us catch leaks, e.g. guest- /// tasks which haven't been deleted despite having completed and having- /// been dropped by their supertasks.- #[doc(hidden)]- pub fn assert_concurrent_state_empty(&self, mut store: impl AsContextMut) {- let mut instance = self.id().get_mut(store.as_context_mut().0);- assert!(- instance- .as_mut()- .guest_tables()- .0- .iter()- .all(|(_, table)| table.is_empty())- );- let state = instance.concurrent_state_mut();- assert!(- state.table.get_mut().is_empty(),- "non-empty table: {:?}",- state.table.get_mut()- );- assert!(state.high_priority.is_empty());- assert!(state.low_priority.is_empty());- assert!(state.guest_task.is_none());- assert!(state.futures.get_mut().as_ref().unwrap().is_empty());- assert!(- state- .instance_states- .iter()- .all(|(_, state)| state.pending.is_empty())- );- assert!(state.global_error_context_ref_counts.is_empty());+ /// Internal implementation of `spawn` functions where a `store` is+ /// available along with an `Accessor`.+ fn spawn_with_accessor<D>(+ self,+ accessor: Accessor<T, D>,+ task: impl AccessorTask<T, D, Result<()>>,+ ) -> JoinHandle+ where+ T: 'static,+ D: HasData + ?Sized,+ {+ // Create an "abortable future" here where internally the future will+ // hook calls to poll and possibly spawn more background tasks on each+ // iteration.+ let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });+ self.0+ .concurrent_state_mut()+ .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));+ handle}- /// Run the specified closure `fun` to completion as part of this instance's+ /// Run the specified closure `fun` to completion as part of this store's/// event loop.///- /// This will run `fun` as part of this instance's event loop until it+ /// This will run `fun` as part of this store's event loop until it/// yields a result. `fun` is provided an [`Accessor`], which provides- /// controlled access to the `Store` and its data.+ /// controlled access to the store and its data.////// This function can be used to invoke [`Func::call_concurrent`] for/// example within the async closure provided here.@@ -1064,7 +969,7 @@ impl Instance {/// # let instance = linker.instantiate_async(&mut store, &component).await?;/// # let foo = instance.get_typed_func::<(Resource<MyResource>,), (Resource<MyResource>,)>(&mut store, "foo")?;/// # let bar = instance.get_typed_func::<(u32,), ()>(&mut store, "bar")?;- /// instance.run_concurrent(&mut store, async |accessor| -> wasmtime::Result<_> {+ /// store.run_concurrent(async |accessor| -> wasmtime::Result<_> {/// let resource = accessor.with(|mut access| access.get().table.push(MyResource(42)))?;/// let (another_resource,) = foo.call_concurrent(accessor, (resource,)).await?.0;/// let value = accessor.with(|mut access| access.get().table.delete(another_resource))?;@@ -1074,31 +979,25 @@ impl Instance {/// # Ok(())/// # }/// ```- pub async fn run_concurrent<T, R>(- self,- store: impl AsContextMut<Data = T>,- fun: impl AsyncFnOnce(&Accessor<T>) -> R,- ) -> Result<R>+ pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>whereT: Send + 'static,{- self.do_run_concurrent(store, fun, false).await+ self.do_run_concurrent(fun, false).await}- pub(super) async fn run_concurrent_trap_on_idle<T, R>(+ pub(super) async fn run_concurrent_trap_on_idle<R>(self,- store: impl AsContextMut<Data = T>,fun: impl AsyncFnOnce(&Accessor<T>) -> R,) -> Result<R>whereT: Send + 'static,{- self.do_run_concurrent(store, fun, true).await+ self.do_run_concurrent(fun, true).await}- async fn do_run_concurrent<T, R>(- self,- mut store: impl AsContextMut<Data = T>,+ async fn do_run_concurrent<R>(+ mut self,fun: impl AsyncFnOnce(&Accessor<T>) -> R,trap_on_idle: bool,) -> Result<R>@@ -1106,8 +1005,7 @@ impl Instance {T: Send + 'static,{check_recursive_run();- let mut store = store.as_context_mut();- let token = StoreToken::new(store.as_context_mut());+ let token = StoreToken::new(self.as_context_mut());struct Dropper<'a, T: 'static, V> {store: StoreContextMut<'a, T>,@@ -1126,67 +1024,28 @@ impl Instance {}}- let accessor = &Accessor::new(token, Some(self));+ let accessor = &Accessor::new(token);let dropper = &mut Dropper {- store,+ store: self,value: ManuallyDrop::new(fun(accessor)),};// SAFETY: We never move `dropper` nor its `value` field.let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };- self.poll_until(dropper.store.as_context_mut(), future, trap_on_idle)+ dropper+ .store+ .as_context_mut()+ .poll_until(future, trap_on_idle).await}- /// Spawn a background task to run as part of this instance's event loop.- ///- /// The task will receive an `&Accessor<U>` and run concurrently with- /// any other tasks in progress for the instance.- ///- /// Note that the task will only make progress if and when the event loop- /// for this instance is run.- ///- /// The returned [`SpawnHandle`] may be used to cancel the task.- pub fn spawn<U: 'static>(- self,- mut store: impl AsContextMut<Data = U>,- task: impl AccessorTask<U, HasSelf<U>, Result<()>>,- ) -> JoinHandle {- let mut store = store.as_context_mut();- let accessor = Accessor::new(StoreToken::new(store.as_context_mut()), Some(self));- self.spawn_with_accessor(store, accessor, task)- }-- /// Internal implementation of `spawn` functions where a `store` is- /// available along with an `Accessor`.- fn spawn_with_accessor<T, D>(- self,- mut store: StoreContextMut<T>,- accessor: Accessor<T, D>,- task: impl AccessorTask<T, D, Result<()>>,- ) -> JoinHandle- where- T: 'static,- D: HasData + ?Sized,- {- let store = store.as_context_mut();-- // Create an "abortable future" here where internally the future will- // hook calls to poll and possibly spawn more background tasks on each- // iteration.- let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });- self.concurrent_state_mut(store.0)- .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));-- handle- }-- /// Run this instance's event loop.+ /// Run this store's event loop.///- /// The returned future will resolve when the specified future completes.- async fn poll_until<T, R>(- self,- mut store: StoreContextMut<'_, T>,+ /// The returned future will resolve when the specified future completes or,+ /// if `trap_on_idle` is true, when the event loop can't make further+ /// progress.+ async fn poll_until<R>(+ mut self,mut future: Pin<&mut impl Future<Output = R>>,trap_on_idle: bool,) -> Result<R>@@ -1195,18 +1054,13 @@ impl Instance {{struct Reset<'a, T: 'static> {store: StoreContextMut<'a, T>,- instance: Instance,futures: Option<FuturesUnordered<HostTaskFuture>>,}impl<'a, T> Drop for Reset<'a, T> {fn drop(&mut self) {if let Some(futures) = self.futures.take() {- *self- .instance- .concurrent_state_mut(self.store.0)- .futures- .get_mut() = Some(futures);+ *self.store.0.concurrent_state_mut().futures.get_mut() = Some(futures);}}}@@ -1215,10 +1069,9 @@ impl Instance {// Take `ConcurrentState::futures` out of the instance so we can// poll it while also safely giving any of the futures inside access// to `self`.- let futures = self.concurrent_state_mut(store.0).futures.get_mut().take();+ let futures = self.0.concurrent_state_mut().futures.get_mut().take();let mut reset = Reset {- store: store.as_context_mut(),- instance: self,+ store: self.as_context_mut(),futures,};let mut next = pin!(reset.futures.as_mut().unwrap().next());@@ -1226,15 +1079,14 @@ impl Instance {let result = future::poll_fn(|cx| {// First, poll the future we were passed as an argument and// return immediately if it's ready.- if let Poll::Ready(value) = self.set_tls(reset.store.0, || future.as_mut().poll(cx))- {+ if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {return Poll::Ready(Ok(Either::Left(value)));}// Next, poll `ConcurrentState::futures` (which includes any// pending host tasks and/or background tasks), returning// immediately if one of them fails.- let next = match self.set_tls(reset.store.0, || next.as_mut().poll(cx)) {+ let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {Poll::Ready(Some(output)) => {match output {Err(e) => return Poll::Ready(Err(e)),@@ -1246,11 +1098,9 @@ impl Instance {Poll::Pending => Poll::Pending,};- let mut instance = self.id().get_mut(reset.store.0);-// Next, check the "high priority" work queue and return// immediately if it has at least one item.- let state = instance.as_mut().concurrent_state_mut();+ let state = reset.store.0.concurrent_state_mut();let ready = mem::take(&mut state.high_priority);let ready = if ready.is_empty() {// Next, check the "low priority" work queue and return@@ -1271,7 +1121,7 @@ impl Instance {// in case one of `ConcurrentState::futures` had// the side effect of unblocking it.if let Poll::Ready(value) =- self.set_tls(reset.store.0, || future.as_mut().poll(cx))+ tls::set(reset.store.0, || future.as_mut().poll(cx)){Poll::Ready(Ok(Either::Left(value)))} else {@@ -1341,12 +1191,15 @@ impl Instance {}let mut dispose = Dispose {- store: store.as_context_mut(),+ store: self.as_context_mut(),ready: ready.into_iter(),};while let Some(item) = dispose.ready.next() {- self.handle_work_item(dispose.store.as_context_mut(), item)+ dispose+ .store+ .as_context_mut()+ .handle_work_item(item).await?;}}@@ -1355,15 +1208,15 @@ impl Instance {}/// Handle the specified work item, possibly resuming a fiber if applicable.- async fn handle_work_item<T: Send>(- self,- store: StoreContextMut<'_, T>,- item: WorkItem,- ) -> Result<()> {+ async fn handle_work_item(self, item: WorkItem) -> Result<()>+ where+ T: Send,+ {log::trace!("handle work item {item:?}");match item {WorkItem::PushFuture(future) => {- self.concurrent_state_mut(store.0)+ self.0+ .concurrent_state_mut().futures.get_mut().as_mut()@@ -1371,13 +1224,12 @@ impl Instance {.push(future.into_inner());}WorkItem::ResumeFiber(fiber) => {- self.resume_fiber(store.0, fiber).await?;+ self.0.resume_fiber(fiber).await?;}WorkItem::GuestCall(call) => {- let state = self.concurrent_state_mut(store.0);+ let state = self.0.concurrent_state_mut();if call.is_ready(state)? {- self.run_on_worker(store, WorkerItem::GuestCall(call))- .await?;+ self.run_on_worker(WorkerItem::GuestCall(call)).await?;} else {let task = state.get_mut(call.task)?;if !task.starting_sent {@@ -1400,7 +1252,7 @@ impl Instance {}}WorkItem::Poll(params) => {- let state = self.concurrent_state_mut(store.0);+ let state = self.0.concurrent_state_mut();if state.get_mut(params.task)?.event.is_some()|| !state.get_mut(params.set)?.ready.is_empty(){@@ -1409,6 +1261,7 @@ impl Instance {state.push_high_priority(WorkItem::GuestCall(GuestCall {task: params.task,kind: GuestCallKind::DeliverEvent {+ instance: params.instance,set: Some(params.set),},}));@@ -1419,28 +1272,78 @@ impl Instance {state.push_high_priority(WorkItem::GuestCall(GuestCall {task: params.task,kind: GuestCallKind::DeliverEvent {+ instance: params.instance,set: Some(params.set),},}));}}WorkItem::WorkerFunction(fun) => {- self.run_on_worker(store, WorkerItem::Function(fun)).await?;+ self.run_on_worker(WorkerItem::Function(fun)).await?;}}Ok(())}+ /// Execute the specified guest call on a worker fiber.+ async fn run_on_worker(self, item: WorkerItem) -> Result<()>+ where+ T: Send,+ {+ let worker = if let Some(fiber) = self.0.concurrent_state_mut().worker.take() {+ fiber+ } else {+ fiber::make_fiber(self.0, move |store| {+ loop {+ match store.concurrent_state_mut().worker_item.take().unwrap() {+ WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,+ WorkerItem::Function(fun) => fun.into_inner()(store)?,+ }++ store.suspend(SuspendReason::NeedWork)?;+ }+ })?+ };++ let worker_item = &mut self.0.concurrent_state_mut().worker_item;+ assert!(worker_item.is_none());+ *worker_item = Some(item);++ self.0.resume_fiber(worker).await+ }++ /// Wrap the specified host function in a future which will call it, passing+ /// it an `&Accessor<T>`.+ ///+ /// See the `Accessor` documentation for details.+ pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static+ where+ T: 'static,+ F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>+ + Send+ + Sync+ + 'static,+ R: Send + Sync + 'static,+ {+ let token = StoreToken::new(self);+ async move {+ let mut accessor = Accessor::new(token);+ closure(&mut accessor).await+ }+ }+}++impl StoreOpaque {/// Resume the specified fiber, giving it exclusive access to the specified/// store.- async fn resume_fiber(self, store: &mut StoreOpaque, fiber: StoreFiber<'static>) -> Result<()> {- let old_task = self.concurrent_state_mut(store).guest_task;+ async fn resume_fiber(&mut self, fiber: StoreFiber<'static>) -> Result<()> {+ let old_task = self.concurrent_state_mut().guest_task;log::trace!("resume_fiber: save current task {old_task:?}");- let fiber = fiber::resolve_or_release(store, fiber).await?;+ let fiber = fiber::resolve_or_release(self, fiber).await?;- let state = self.concurrent_state_mut(store);+ let state = self.concurrent_state_mut();state.guest_task = old_task;log::trace!("resume_fiber: restore current task {old_task:?}");@@ -1452,7 +1355,7 @@ impl Instance {if state.worker.is_none() {state.worker = Some(fiber);} else {- fiber.dispose(store);+ fiber.dispose(self);}}SuspendReason::Yielding { .. } => {@@ -1471,100 +1374,12 @@ impl Instance {Ok(())}- /// Execute the specified guest call on a worker fiber.- async fn run_on_worker<T: Send>(- self,- store: StoreContextMut<'_, T>,- item: WorkerItem,- ) -> Result<()> {- let worker = if let Some(fiber) = self.concurrent_state_mut(store.0).worker.take() {- fiber- } else {- fiber::make_fiber(store.0, move |store| {- loop {- match self.concurrent_state_mut(store).worker_item.take().unwrap() {- WorkerItem::GuestCall(call) => self.handle_guest_call(store, call)?,- WorkerItem::Function(fun) => fun.into_inner()(store, self)?,- }-- self.suspend(store, SuspendReason::NeedWork)?;- }- })?- };-- let worker_item = &mut self.concurrent_state_mut(store.0).worker_item;- assert!(worker_item.is_none());- *worker_item = Some(item);-- self.resume_fiber(store.0, worker).await- }-- /// Execute the specified guest call.- fn handle_guest_call(self, store: &mut dyn VMStore, call: GuestCall) -> Result<()> {- match call.kind {- GuestCallKind::DeliverEvent { set } => {- let (event, waitable) = self- .id()- .get_mut(store)- .get_event(call.task, set, true)?- .unwrap();- let state = self.concurrent_state_mut(store);- let task = state.get_mut(call.task)?;- let runtime_instance = task.instance;- let handle = waitable.map(|(_, v)| v).unwrap_or(0);-- log::trace!(- "use callback to deliver event {event:?} to {:?} for {waitable:?}",- call.task,- );-- let old_task = state.guest_task.replace(call.task);- log::trace!(- "GuestCallKind::DeliverEvent: replaced {old_task:?} with {:?} as current task",- call.task- );-- self.maybe_push_call_context(store.store_opaque_mut(), call.task)?;-- let state = self.concurrent_state_mut(store);- state.enter_instance(runtime_instance);-- let callback = state.get_mut(call.task)?.callback.take().unwrap();-- let code = callback(store, self, runtime_instance, event, handle)?;-- let state = self.concurrent_state_mut(store);-- state.get_mut(call.task)?.callback = Some(callback);-- state.exit_instance(runtime_instance)?;-- self.maybe_pop_call_context(store.store_opaque_mut(), call.task)?;-- self.id().get_mut(store).handle_callback_code(- call.task,- runtime_instance,- code,- false,- )?;-- self.concurrent_state_mut(store).guest_task = old_task;- log::trace!("GuestCallKind::DeliverEvent: restored {old_task:?} as current task");- }- GuestCallKind::Start(fun) => {- fun(store, self)?;- }- }-- Ok(())- }-/// Suspend the current fiber, storing the reason in/// `ConcurrentState::suspend_reason` to indicate the conditions under which/// it should be resumed.////// See the `SuspendReason` documentation for details.- fn suspend(self, store: &mut dyn VMStore, reason: SuspendReason) -> Result<()> {+ fn suspend(&mut self, reason: SuspendReason) -> Result<()> {log::trace!("suspend fiber: {reason:?}");// If we're yielding or waiting on behalf of a guest task, we'll need to@@ -1576,21 +1391,21 @@ impl Instance {};let old_guest_task = if let Some(task) = task {- self.maybe_pop_call_context(store, task)?;- self.concurrent_state_mut(store).guest_task+ self.maybe_pop_call_context(task)?;+ self.concurrent_state_mut().guest_task} else {None};- let suspend_reason = &mut self.concurrent_state_mut(store).suspend_reason;+ let suspend_reason = &mut self.concurrent_state_mut().suspend_reason;assert!(suspend_reason.is_none());*suspend_reason = Some(reason);- store.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;+ self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;if let Some(task) = task {- self.concurrent_state_mut(store).guest_task = old_guest_task;- self.maybe_push_call_context(store, task)?;+ self.concurrent_state_mut().guest_task = old_guest_task;+ self.maybe_push_call_context(task)?;}Ok(())@@ -1599,16 +1414,12 @@ impl Instance {/// Push the call context for managing resource borrows for the specified/// guest task if it has not yet either returned a result or cancelled/// itself.- fn maybe_push_call_context(- self,- store: &mut StoreOpaque,- guest_task: TableId<GuestTask>,- ) -> Result<()> {- let task = self.concurrent_state_mut(store).get_mut(guest_task)?;+ fn maybe_push_call_context(&mut self, guest_task: TableId<GuestTask>) -> Result<()> {+ let task = self.concurrent_state_mut().get_mut(guest_task)?;if task.lift_result.is_some() {log::trace!("push call context for {guest_task:?}");let call_context = task.call_context.take().unwrap();- store.component_resource_state().0.push(call_context);+ self.component_resource_state().0.push(call_context);}Ok(())}@@ -1616,26 +1427,215 @@ impl Instance {/// Pop the call context for managing resource borrows for the specified/// guest task if it has not yet either returned a result or cancelled/// itself.- fn maybe_pop_call_context(- self,- store: &mut StoreOpaque,- guest_task: TableId<GuestTask>,- ) -> Result<()> {+ fn maybe_pop_call_context(&mut self, guest_task: TableId<GuestTask>) -> Result<()> {if self- .concurrent_state_mut(store)+ .concurrent_state_mut().get_mut(guest_task)?.lift_result.is_some(){log::trace!("pop call context for {guest_task:?}");- let call_context = Some(store.component_resource_state().0.pop().unwrap());- self.concurrent_state_mut(store)+ let call_context = Some(self.component_resource_state().0.pop().unwrap());+ self.concurrent_state_mut().get_mut(guest_task)?.call_context = call_context;}Ok(())}+ fn wait_for_event(&mut self, waitable: Waitable) -> Result<()> {+ let state = self.concurrent_state_mut();+ let caller = state.guest_task.unwrap();+ let old_set = waitable.common(state)?.set;+ let set = state.get_mut(caller)?.sync_call_set;+ waitable.join(state, Some(set))?;+ self.suspend(SuspendReason::Waiting { set, task: caller })?;+ let state = self.concurrent_state_mut();+ waitable.join(state, old_set)+ }+}++impl Instance {+ /// Get the next pending event for the specified task and (optional)+ /// waitable set, along with the waitable handle if applicable.+ fn get_event(+ self,+ store: &mut StoreOpaque,+ guest_task: TableId<GuestTask>,+ set: Option<TableId<WaitableSet>>,+ cancellable: bool,+ ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {+ let state = store.concurrent_state_mut();++ if let Some(event) = state.get_mut(guest_task)?.event.take() {+ log::trace!("deliver event {event:?} to {guest_task:?}");++ if cancellable || !matches!(event, Event::Cancelled) {+ return Ok(Some((event, None)));+ } else {+ state.get_mut(guest_task)?.event = Some(event);+ }+ }++ Ok(+ if let Some((set, waitable)) = set+ .and_then(|set| {+ state+ .get_mut(set)+ .map(|v| v.ready.pop_first().map(|v| (set, v)))+ .transpose()+ })+ .transpose()?+ {+ let common = waitable.common(state)?;+ let handle = common.handle.unwrap();+ let event = common.event.take().unwrap();++ log::trace!(+ "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"+ );++ waitable.on_delivery(self.id().get_mut(store), event);++ Some((event, Some((waitable, handle))))+ } else {+ None+ },+ )+ }++ /// Handle the `CallbackCode` returned from an async-lifted export or its+ /// callback.+ ///+ /// If `initial_call` is `true`, then the code was received from the+ /// async-lifted export; otherwise, it was received from its callback.+ fn handle_callback_code(+ self,+ store: &mut StoreOpaque,+ guest_task: TableId<GuestTask>,+ runtime_instance: RuntimeComponentInstanceIndex,+ code: u32,+ initial_call: bool,+ ) -> Result<()> {+ let (code, set) = unpack_callback_code(code);++ log::trace!("received callback code from {guest_task:?}: {code} (set: {set})");++ let state = store.concurrent_state_mut();+ let task = state.get_mut(guest_task)?;++ if task.lift_result.is_some() {+ if code == callback_code::EXIT {+ return Err(anyhow!(crate::Trap::NoAsyncResult));+ }+ if initial_call {+ // Notify any current or future waiters that this subtask has+ // started.+ Waitable::Guest(guest_task).set_event(+ state,+ Some(Event::Subtask {+ status: Status::Started,+ }),+ )?;+ }+ }++ let get_set = |store, handle| {+ if handle == 0 {+ bail!("invalid waitable-set handle");+ }++ let set = self.id().get_mut(store).guest_tables().0[runtime_instance]+ .waitable_set_rep(handle)?;++ Ok(TableId::<WaitableSet>::new(set))+ };++ match code {+ callback_code::EXIT => {+ let task = state.get_mut(guest_task)?;+ match &task.caller {+ Caller::Host {+ remove_task_automatically,+ ..+ } => {+ if *remove_task_automatically {+ log::trace!("handle_callback_code will delete task {guest_task:?}");+ Waitable::Guest(guest_task).delete_from(state)?;+ }+ }+ Caller::Guest { .. } => {+ task.exited = true;+ task.callback = None;+ }+ }+ }+ callback_code::YIELD => {+ // Push this task onto the "low priority" queue so it runs after+ // any other tasks have had a chance to run.+ let task = state.get_mut(guest_task)?;+ assert!(task.event.is_none());+ task.event = Some(Event::None);+ state.push_low_priority(WorkItem::GuestCall(GuestCall {+ task: guest_task,+ kind: GuestCallKind::DeliverEvent {+ instance: self,+ set: None,+ },+ }));+ }+ callback_code::WAIT | callback_code::POLL => {+ let set = get_set(store, set)?;+ let state = store.concurrent_state_mut();++ if state.get_mut(guest_task)?.event.is_some()+ || !state.get_mut(set)?.ready.is_empty()+ {+ // An event is immediately available; deliver it ASAP.+ state.push_high_priority(WorkItem::GuestCall(GuestCall {+ task: guest_task,+ kind: GuestCallKind::DeliverEvent {+ instance: self,+ set: Some(set),+ },+ }));+ } else {+ // No event is immediately available.+ match code {+ callback_code::POLL => {+ // We're polling, so just yield and check whether an+ // event has arrived after that.+ state.push_low_priority(WorkItem::Poll(PollParams {+ instance: self,+ task: guest_task,+ set,+ }));+ }+ callback_code::WAIT => {+ // We're waiting, so register to be woken up when an+ // event is published for this waitable set.+ //+ // Here we also set `GuestTask::wake_on_cancel`+ // which allows `subtask.cancel` to interrupt the+ // wait.+ let old = state.get_mut(guest_task)?.wake_on_cancel.replace(set);+ assert!(old.is_none());+ let old = state+ .get_mut(set)?+ .waiting+ .insert(guest_task, WaitMode::Callback(self));+ assert!(old.is_none());+ }+ _ => unreachable!(),+ }+ }+ }+ _ => bail!("unsupported callback code: {code}"),+ }++ Ok(())+ }+/// Add the specified guest call to the "high priority" work item queue, to/// be started as soon as backpressure and/or reentrance rules allow.///@@ -1675,22 +1675,19 @@ impl Instance {param_count: usize,result_count: usize,flags: Option<InstanceFlags>,- ) -> impl FnOnce(- &mut dyn VMStore,- Instance,- ) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>+ ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>+ Send+ Sync+ 'static+ use<T> {let token = StoreToken::new(store);- move |store: &mut dyn VMStore, instance: Instance| {+ move |store: &mut dyn VMStore| {let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];- let task = instance.concurrent_state_mut(store).get_mut(guest_task)?;+ let task = store.concurrent_state_mut().get_mut(guest_task)?;let may_enter_after_call = task.call_post_return_automatically();let lower = task.lower_params.take().unwrap();- lower(store, instance, &mut storage[..param_count])?;+ lower(store, &mut storage[..param_count])?;let mut store = token.as_context_mut(store);@@ -1732,27 +1729,19 @@ impl Instance {)};- let callee_instance = self- .concurrent_state_mut(store.0)- .get_mut(guest_task)?- .instance;+ let callee_instance = store.0.concurrent_state_mut().get_mut(guest_task)?.instance;let fun = if callback.is_some() {assert!(async_);- Box::new(move |store: &mut dyn VMStore, instance: Instance| {- let old_task = instance- .concurrent_state_mut(store)- .guest_task- .replace(guest_task);+ Box::new(move |store: &mut dyn VMStore| {+ let old_task = store.concurrent_state_mut().guest_task.replace(guest_task);log::trace!("stackless call: replaced {old_task:?} with {guest_task:?} as current task");- instance.maybe_push_call_context(store.store_opaque_mut(), guest_task)?;+ store.maybe_push_call_context(guest_task)?;- instance- .concurrent_state_mut(store)- .enter_instance(callee_instance);+ store.concurrent_state_mut().enter_instance(callee_instance);// SAFETY: See the documentation for `make_call` to review the// contract we must uphold for `call` here.@@ -1760,15 +1749,15 @@ impl Instance {// Per the contract described in the `queue_call`// documentation, the `callee` pointer which `call` closes// over must be valid.- let storage = call(store, instance)?;+ let storage = call(store)?;- instance- .concurrent_state_mut(store)+ store+ .concurrent_state_mut().exit_instance(callee_instance)?;- instance.maybe_pop_call_context(store.store_opaque_mut(), guest_task)?;+ store.maybe_pop_call_context(guest_task)?;- let state = instance.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();state.guest_task = old_task;log::trace!("stackless call: restored {old_task:?} as current task");@@ -1776,38 +1765,27 @@ impl Instance {// function returns a `i32` result.let code = unsafe { storage[0].assume_init() }.get_i32() as u32;- instance.id().get_mut(store).handle_callback_code(- guest_task,- callee_instance,- code,- true,- )?;+ self.handle_callback_code(store, guest_task, callee_instance, code, true)?;Ok(())- })- as Box<dyn FnOnce(&mut dyn VMStore, Instance) -> Result<()> + Send + Sync>+ }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>} else {let token = StoreToken::new(store.as_context_mut());- Box::new(move |store: &mut dyn VMStore, instance: Instance| {- let old_task = instance- .concurrent_state_mut(store)- .guest_task- .replace(guest_task);+ Box::new(move |store: &mut dyn VMStore| {+ let old_task = store.concurrent_state_mut().guest_task.replace(guest_task);log::trace!("stackful call: replaced {old_task:?} with {guest_task:?} as current task",);- let mut flags = instance.id().get(store).instance_flags(callee_instance);+ let mut flags = self.id().get(store).instance_flags(callee_instance);- instance.maybe_push_call_context(store.store_opaque_mut(), guest_task)?;+ store.maybe_push_call_context(guest_task)?;// Unless this is a callback-less (i.e. stackful)// async-lifted export, we need to record that the instance// cannot be entered until the call returns.if !async_ {- instance- .concurrent_state_mut(store)- .enter_instance(callee_instance);+ store.concurrent_state_mut().enter_instance(callee_instance);}// SAFETY: See the documentation for `make_call` to review the@@ -1816,15 +1794,15 @@ impl Instance {// Per the contract described in the `queue_call`// documentation, the `callee` pointer which `call` closes// over must be valid.- let storage = call(store, instance)?;+ let storage = call(store)?;if async_ {// This is a callback-less (i.e. stackful) async-lifted// export, so there is no post-return function, and// either `task.return` or `task.cancel` should have// been called.- if instance- .concurrent_state_mut(store)+ if store+ .concurrent_state_mut().get_mut(guest_task)?.lift_result.is_some()@@ -1838,7 +1816,7 @@ impl Instance {// subtask has returned.let lift = {- let state = instance.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();state.exit_instance(callee_instance)?;assert!(state.get_mut(guest_task)?.result.is_none());@@ -1848,7 +1826,7 @@ impl Instance {// SAFETY: `result_count` represents the number of core Wasm// results returned, per `wasmparser`.- let result = (lift.lift)(store, instance, unsafe {+ let result = (lift.lift)(store, unsafe {mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(&storage[..result_count],)@@ -1862,8 +1840,8 @@ impl Instance {_ => unreachable!(),};- if instance- .concurrent_state_mut(store)+ if store+ .concurrent_state_mut().get_mut(guest_task)?.call_post_return_automatically(){@@ -1895,7 +1873,7 @@ impl Instance {}}- instance.task_complete(+ self.task_complete(store,guest_task,result,@@ -1904,9 +1882,9 @@ impl Instance {)?;}- instance.maybe_pop_call_context(store.store_opaque_mut(), guest_task)?;+ store.maybe_pop_call_context(guest_task)?;- let task = instance.concurrent_state_mut(store).get_mut(guest_task)?;+ let task = store.concurrent_state_mut().get_mut(guest_task)?;match &task.caller {Caller::Host {@@ -1915,7 +1893,7 @@ impl Instance {} => {if *remove_task_automatically {Waitable::Guest(guest_task)- .delete_from(instance.concurrent_state_mut(store))?;+ .delete_from(store.concurrent_state_mut())?;}}Caller::Guest { .. } => {@@ -1927,7 +1905,9 @@ impl Instance {})};- self.concurrent_state_mut(store.0)+ store+ .0+ .concurrent_state_mut().push_high_priority(WorkItem::GuestCall(GuestCall {task: guest_task,kind: GuestCallKind::Start(fun),@@ -1996,11 +1976,11 @@ impl Instance {let start = SendSyncPtr::new(NonNull::new(start).unwrap());let return_ = SendSyncPtr::new(NonNull::new(return_).unwrap());let token = StoreToken::new(store.as_context_mut());- let state = self.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();let old_task = state.guest_task.take();let new_task = GuestTask::new(state,- Box::new(move |store, instance, dst| {+ Box::new(move |store, dst| {let mut store = token.as_context_mut(store);assert!(dst.len() <= MAX_FLAT_PARAMS);let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];@@ -2041,7 +2021,7 @@ impl Instance {)?;}dst.copy_from_slice(&src[..dst.len()]);- let state = instance.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();let task = state.guest_task.unwrap();Waitable::Guest(task).set_event(state,@@ -2052,7 +2032,7 @@ impl Instance {Ok(())}),LiftResult {- lift: Box::new(move |store, instance, src| {+ lift: Box::new(move |store, src| {// SAFETY: See comment in closure passed as `lower_params`// parameter above.let mut store = token.as_context_mut(store);@@ -2073,7 +2053,7 @@ impl Instance {my_src.as_mut_slice().into(),)?;}- let state = instance.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();let task = state.guest_task.unwrap();if sync_caller {state.get_mut(task)?.sync_result =@@ -2181,7 +2161,7 @@ impl Instance {) -> Result<u32> {let token = StoreToken::new(store.as_context_mut());let async_caller = storage.is_none();- let state = self.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();let guest_task = state.guest_task.unwrap();let may_enter_after_call = state.get_mut(guest_task)?.call_post_return_automatically();let callee = SendSyncPtr::new(NonNull::new(callee).unwrap());@@ -2192,25 +2172,23 @@ impl Instance {let task = state.get_mut(guest_task)?;if !callback.is_null() {- // We're calling an async-lifted export with a callback, so store- // the callback and related context as part of the task so we can- // call it later when needed.- let callback = SendSyncPtr::new(NonNull::new(callback).unwrap());- task.callback = Some(Box::new(- move |store, instance, runtime_instance, event, handle| {- let store = token.as_context_mut(store);- unsafe {- instance.call_callback::<T>(- store,- runtime_instance,- callback,- event,- handle,- may_enter_after_call,- )- }- },- ));+ // We're calling an async-lifted export with a callback, so store+ // the callback and related context as part of the task so we can+ // call it later when needed.+ let callback = SendSyncPtr::new(NonNull::new(callback).unwrap());+ task.callback = Some(Box::new(move |store, runtime_instance, event, handle| {+ let store = token.as_context_mut(store);+ unsafe {+ self.call_callback::<T>(+ store,+ runtime_instance,+ callback,+ event,+ handle,+ may_enter_after_call,+ )+ }+ }));}let Caller::Guest {@@ -2248,7 +2226,7 @@ impl Instance {)?;}- let state = self.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();// Use the caller's `GuestTask::sync_call_set` to register interest in// the subtask...@@ -2273,9 +2251,11 @@ impl Instance {// before committing to such an optimization. And again, we'd need to// update the spec to allow that.let (status, waitable) = loop {- self.suspend(store.0, SuspendReason::Waiting { set, task: caller })?;+ store+ .0+ .suspend(SuspendReason::Waiting { set, task: caller })?;- let state = self.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();let event = guest_waitable.take_event(state)?;let Some(Event::Subtask { status }) = event else {@@ -2293,7 +2273,9 @@ impl Instance {// waitable and return the status.let handle = self.id().get_mut(store.0).guest_tables().0[caller_instance].subtask_insert_guest(guest_task.rep())?;- self.concurrent_state_mut(store.0)+ store+ .0+ .concurrent_state_mut().get_mut(guest_task)?.common.handle = Some(handle);@@ -2305,7 +2287,7 @@ impl Instance {}};- let state = self.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();guest_waitable.join(state, old_set)?;@@ -2336,30 +2318,6 @@ impl Instance {Ok(status.pack(waitable))}- /// Wrap the specified host function in a future which will call it, passing- /// it an `&Accessor<T>`.- ///- /// See the `Accessor` documentation for details.- pub(crate) fn wrap_call<T, F, R>(- self,- store: StoreContextMut<T>,- closure: F,- ) -> impl Future<Output = Result<R>> + 'static- where- T: 'static,- F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>- + Send- + Sync- + 'static,- R: Send + Sync + 'static,- {- let token = StoreToken::new(store);- async move {- let mut accessor = Accessor::new(token, Some(self));- closure(&mut accessor).await- }- }-/// Poll the specified future once on behalf of a guest->host call using an/// async-lowered import.///@@ -2376,10 +2334,10 @@ impl Instance {mut store: StoreContextMut<T>,future: impl Future<Output = Result<R>> + Send + 'static,caller_instance: RuntimeComponentInstanceIndex,- lower: impl FnOnce(StoreContextMut<T>, Instance, R) -> Result<()> + Send + 'static,+ lower: impl FnOnce(StoreContextMut<T>, R) -> Result<()> + Send + 'static,) -> Result<Option<u32>> {let token = StoreToken::new(store.as_context_mut());- let state = self.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();let caller = state.guest_task.unwrap();// Create an abortable future which hooks calls to poll and manages call@@ -2436,7 +2394,7 @@ impl Instance {// we'll add the future to `ConcurrentState::futures` and poll it// automatically from the event loop if it doesn't complete immediately// here.- let poll = self.set_tls(store.0, || {+ let poll = tls::set(store.0, || {future.as_mut().poll(&mut Context::from_waker(&Waker::noop()))@@ -2447,9 +2405,9 @@ impl Instance {Poll::Ready(Some(result)) => {// It finished immediately; lower the result and delete the// task.- lower(store.as_context_mut(), self, result?)?;+ lower(store.as_context_mut(), result?)?;log::trace!("delete host task {task:?} (already ready)");- self.concurrent_state_mut(store.0).delete(task)?;+ store.0.concurrent_state_mut().delete(task)?;None}Poll::Pending => {@@ -2460,42 +2418,40 @@ impl Instance {// Wrap the future in a closure responsible for lowering the result into// the guest's stack and memory, as well as notifying any waiters that// the task returned.- let future = Box::pin(async move {- let result = match future.await {- Some(result) => result?,- // Task was cancelled; nothing left to do.- None => return Ok(()),- };- tls::get(move |store| {- // Here we schedule a task to run on a worker fiber to do- // the lowering since it may involve a call to the guest's- // realloc function. This is necessary because calling the- // guest while there are host embedder frames on the stack- // is unsound.- self.concurrent_state_mut(store).push_high_priority(- WorkItem::WorkerFunction(AlwaysMut::new(Box::new(move |store, _| {- lower(token.as_context_mut(store), self, result)?;- let state = self.concurrent_state_mut(store);- state.get_mut(task)?.join_handle.take();- Waitable::Host(task).set_event(- state,- Some(Event::Subtask {- status: Status::Returned,- }),- )- }))),- );- Ok(())- })- });+ let future =+ Box::pin(async move {+ let result = match future.await {+ Some(result) => result?,+ // Task was cancelled; nothing left to do.+ None => return Ok(()),+ };+ tls::get(move |store| {+ // Here we schedule a task to run on a worker fiber to do+ // the lowering since it may involve a call to the guest's+ // realloc function. This is necessary because calling the+ // guest while there are host embedder frames on the stack+ // is unsound.+ store.concurrent_state_mut().push_high_priority(+ WorkItem::WorkerFunction(AlwaysMut::new(Box::new(move |store| {+ lower(token.as_context_mut(store), result)?;+ let state = store.concurrent_state_mut();+ state.get_mut(task)?.join_handle.take();+ Waitable::Host(task).set_event(+ state,+ Some(Event::Subtask {+ status: Status::Returned,+ }),+ )+ }))),+ );+ Ok(())+ })+ });- self.concurrent_state_mut(store.0).push_future(future);+ store.0.concurrent_state_mut().push_future(future);let handle = self.id().get_mut(store.0).guest_tables().0[caller_instance].subtask_insert_host(task.rep())?;- self.concurrent_state_mut(store.0)- .get_mut(task)?- .common- .handle = Some(handle);+ store.0.concurrent_state_mut().get_mut(task)?.common.handle = Some(handle);log::trace!("assign {task:?} handle {handle} for {caller:?} instance {caller_instance:?}");@@ -2504,111 +2460,6 @@ impl Instance {})}- /// Poll the specified future until it completes on behalf of a guest->host- /// call using a sync-lowered import.- ///- /// This is similar to `Self::first_poll` except it's for sync-lowered- /// imports, meaning we don't need to handle cancellation and we can block- /// the caller until the task completes, at which point the caller can- /// handle lowering the result to the guest's stack and linear memory.- pub(crate) fn poll_and_block<R: Send + Sync + 'static>(- self,- store: &mut dyn VMStore,- future: impl Future<Output = Result<R>> + Send + 'static,- caller_instance: RuntimeComponentInstanceIndex,- ) -> Result<R> {- let state = self.concurrent_state_mut(store);-- // If there is no current guest task set, that means the host function- // was registered using e.g. `LinkerInstance::func_wrap`, in which case- // it should complete immediately.- let Some(caller) = state.guest_task else {- return match pin!(future).poll(&mut Context::from_waker(&Waker::noop())) {- Poll::Ready(result) => result,- Poll::Pending => {- unreachable!()- }- };- };-- // Save any existing result stashed in `GuestTask::result` so we can- // replace it with the new result.- let old_result = state- .get_mut(caller)- .with_context(|| format!("bad handle: {caller:?}"))?- .result- .take();-- // Add a temporary host task into the table so we can track its- // progress. Note that we'll never allocate a waitable handle for the- // guest since we're being called synchronously.- let task = state.push(HostTask::new(caller_instance, None))?;-- log::trace!("new host task child of {caller:?}: {task:?}");-- // Wrap the future in a closure which will take care of stashing the- // result in `GuestTask::result` and resuming this fiber when the host- // task completes.- let mut future = Box::pin(async move {- let result = future.await?;- tls::get(move |store| {- let state = self.concurrent_state_mut(store);- state.get_mut(caller)?.result = Some(Box::new(result) as _);-- Waitable::Host(task).set_event(- state,- Some(Event::Subtask {- status: Status::Returned,- }),- )?;-- Ok(())- })- }) as HostTaskFuture;-- // Finally, poll the future. We can use a dummy `Waker` here because- // we'll add the future to `ConcurrentState::futures` and poll it- // automatically from the event loop if it doesn't complete immediately- // here.- let poll = self.set_tls(store, || {- future- .as_mut()- .poll(&mut Context::from_waker(&Waker::noop()))- });-- match poll {- Poll::Ready(result) => {- // It completed immediately; check the result and delete the task.- result?;- log::trace!("delete host task {task:?} (already ready)");- self.concurrent_state_mut(store).delete(task)?;- }- Poll::Pending => {- // It did not complete immediately; add it to- // `ConcurrentState::futures` so it will be polled via the event- // loop; then use `GuestTask::sync_call_set` to wait for the- // task to complete, suspending the current fiber until it does- // so.- let state = self.concurrent_state_mut(store);- state.push_future(future);-- let set = state.get_mut(caller)?.sync_call_set;- Waitable::Host(task).join(state, Some(set))?;-- self.suspend(store, SuspendReason::Waiting { set, task: caller })?;- }- }-- // Retrieve and return the result.- Ok(*mem::replace(- &mut self.concurrent_state_mut(store).get_mut(caller)?.result,- old_result,- )- .unwrap()- .downcast()- .unwrap())- }-/// Implements the `task.return` intrinsic, lifting the result for the/// current guest task.pub(crate) fn task_return(@@ -2620,12 +2471,7 @@ impl Instance {storage: &[ValRaw],) -> Result<()> {self.id().get(store).check_may_leave(caller)?;- let state = self.concurrent_state_mut(store);- let CanonicalOptions {- string_encoding,- data_model,- ..- } = *state.options(options);+ let state = store.concurrent_state_mut();let guest_task = state.guest_task.unwrap();let lift = state.get_mut(guest_task)?@@ -2636,8 +2482,14 @@ impl Instance {})?;assert!(state.get_mut(guest_task)?.result.is_none());+ let CanonicalOptions {+ string_encoding,+ data_model,+ ..+ } = &self.id().get(store).component().env_component().options[options];+let invalid = ty != lift.ty- || string_encoding != lift.string_encoding+ || string_encoding != &lift.string_encoding|| match data_model {CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {Some(memory) => {@@ -2659,7 +2511,7 @@ impl Instance {log::trace!("task.return for {guest_task:?}");- let result = (lift.lift)(store, self, storage)?;+ let result = (lift.lift)(store, storage)?;self.task_complete(store, guest_task, result, Status::Returned, ValRaw::i32(0))}@@ -2667,11 +2519,11 @@ impl Instance {/// Implements the `task.cancel` intrinsic.pub(crate) fn task_cancel(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,) -> Result<()> {self.id().get(store).check_may_leave(caller)?;- let state = self.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();let guest_task = state.guest_task.unwrap();let task = state.get_mut(guest_task)?;if !task.cancel_sent {@@ -2701,20 +2553,19 @@ impl Instance {/// waiters that the task has completed.fn task_complete(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,guest_task: TableId<GuestTask>,result: Box<dyn Any + Send + Sync>,status: Status,post_return_arg: ValRaw,) -> Result<()> {- if self- .concurrent_state_mut(store)+ if store+ .concurrent_state_mut().get_mut(guest_task)?.call_post_return_automatically(){- let (calls, host_table, _, instance) = store- .store_opaque_mut()- .component_resource_state_with_instance(self);+ let (calls, host_table, _, instance) =+ store.component_resource_state_with_instance(self);ResourceTables {calls,host_table: Some(host_table),@@ -2726,8 +2577,8 @@ impl Instance {// would be false for a `GuestTask` is for host-to-guest calls using// `[Typed]Func::call_async`, in which case the `function_index`// should be a non-`None` value.- let function_index = self- .concurrent_state_mut(store)+ let function_index = store+ .concurrent_state_mut().get_mut(guest_task)?.function_index.unwrap();@@ -2737,7 +2588,7 @@ impl Instance {.post_return_arg_set(function_index, post_return_arg);}- let state = self.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();let task = state.get_mut(guest_task)?;if let Caller::Host { tx, .. } = &mut task.caller {@@ -2752,19 +2603,139 @@ impl Instance {Ok(())}+ /// Implements the `waitable-set.new` intrinsic.+ pub(crate) fn waitable_set_new(+ self,+ store: &mut StoreOpaque,+ caller_instance: RuntimeComponentInstanceIndex,+ ) -> Result<u32> {+ self.id().get_mut(store).check_may_leave(caller_instance)?;+ let set = store.concurrent_state_mut().push(WaitableSet::default())?;+ let handle = self.id().get_mut(store).guest_tables().0[caller_instance]+ .waitable_set_insert(set.rep())?;+ log::trace!("new waitable set {set:?} (handle {handle})");+ Ok(handle)+ }++ /// Implements the `waitable-set.drop` intrinsic.+ pub(crate) fn waitable_set_drop(+ self,+ store: &mut StoreOpaque,+ caller_instance: RuntimeComponentInstanceIndex,+ set: u32,+ ) -> Result<()> {+ self.id().get_mut(store).check_may_leave(caller_instance)?;+ let rep =+ self.id().get_mut(store).guest_tables().0[caller_instance].waitable_set_remove(set)?;++ log::trace!("drop waitable set {rep} (handle {set})");++ let set = store+ .concurrent_state_mut()+ .delete(TableId::<WaitableSet>::new(rep))?;++ if !set.waiting.is_empty() {+ bail!("cannot drop waitable set with waiters");+ }++ Ok(())+ }++ /// Implements the `waitable.join` intrinsic.+ pub(crate) fn waitable_join(+ self,+ store: &mut StoreOpaque,+ caller_instance: RuntimeComponentInstanceIndex,+ waitable_handle: u32,+ set_handle: u32,+ ) -> Result<()> {+ let mut instance = self.id().get_mut(store);+ instance.check_may_leave(caller_instance)?;+ let waitable =+ Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;++ let set = if set_handle == 0 {+ None+ } else {+ let set = instance.guest_tables().0[caller_instance].waitable_set_rep(set_handle)?;++ Some(TableId::<WaitableSet>::new(set))+ };++ log::trace!(+ "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",+ );++ waitable.join(store.concurrent_state_mut(), set)+ }++ /// Implements the `subtask.drop` intrinsic.+ pub(crate) fn subtask_drop(+ self,+ store: &mut StoreOpaque,+ caller_instance: RuntimeComponentInstanceIndex,+ task_id: u32,+ ) -> Result<()> {+ self.id().get_mut(store).check_may_leave(caller_instance)?;+ self.waitable_join(store, caller_instance, task_id, 0)?;++ let (rep, is_host) =+ self.id().get_mut(store).guest_tables().0[caller_instance].subtask_remove(task_id)?;++ let concurrent_state = store.concurrent_state_mut();+ let (waitable, expected_caller_instance, delete) = if is_host {+ let id = TableId::<HostTask>::new(rep);+ let task = concurrent_state.get_mut(id)?;+ if task.join_handle.is_some() {+ bail!("cannot drop a subtask which has not yet resolved");+ }+ (Waitable::Host(id), task.caller_instance, true)+ } else {+ let id = TableId::<GuestTask>::new(rep);+ let task = concurrent_state.get_mut(id)?;+ if task.lift_result.is_some() {+ bail!("cannot drop a subtask which has not yet resolved");+ }+ if let Caller::Guest { instance, .. } = &task.caller {+ (Waitable::Guest(id), *instance, task.exited)+ } else {+ unreachable!()+ }+ };++ waitable.common(concurrent_state)?.handle = None;++ if waitable.take_event(concurrent_state)?.is_some() {+ bail!("cannot drop a subtask with an undelivered event");+ }++ if delete {+ waitable.delete_from(concurrent_state)?;+ }++ // Since waitables can neither be passed between instances nor forged,+ // this should never fail unless there's a bug in Wasmtime, but we check+ // here to be sure:+ assert_eq!(expected_caller_instance, caller_instance);+ log::trace!("subtask_drop {waitable:?} (handle {task_id})");+ Ok(())+ }+/// Implements the `waitable-set.wait` intrinsic.pub(crate) fn waitable_set_wait(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,options: OptionsIndex,set: u32,payload: u32,) -> Result<u32> {self.id().get(store).check_may_leave(caller)?;- let opts = self.concurrent_state_mut(store).options(options);- let cancellable = opts.cancellable;- let caller_instance = opts.instance;+ let &CanonicalOptions {+ cancellable,+ instance: caller_instance,+ ..+ } = &self.id().get(store).component().env_component().options[options];let rep =self.id().get_mut(store).guest_tables().0[caller_instance].waitable_set_rep(set)?;@@ -2782,16 +2753,18 @@ impl Instance {/// Implements the `waitable-set.poll` intrinsic.pub(crate) fn waitable_set_poll(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,options: OptionsIndex,set: u32,payload: u32,) -> Result<u32> {self.id().get(store).check_may_leave(caller)?;- let opts = self.concurrent_state_mut(store).options(options);- let cancellable = opts.cancellable;- let caller_instance = opts.instance;+ let &CanonicalOptions {+ cancellable,+ instance: caller_instance,+ ..+ } = &self.id().get(store).component().env_component().options[options];let rep =self.id().get_mut(store).guest_tables().0[caller_instance].waitable_set_rep(set)?;@@ -2809,7 +2782,7 @@ impl Instance {/// Implements the `thread.yield` intrinsic.pub(crate) fn thread_yield(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,cancellable: bool,) -> Result<bool> {@@ -2817,7 +2790,7 @@ impl Instance {self.waitable_check(store, cancellable, WaitableCheck::Yield).map(|_| {if cancellable {- let state = self.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();let task = state.guest_task.unwrap();if let Some(event) = state.get_mut(task).unwrap().event.take() {assert!(matches!(event, Event::Cancelled));@@ -2835,11 +2808,11 @@ impl Instance {/// `yield` intrinsics.fn waitable_check(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,cancellable: bool,check: WaitableCheck,) -> Result<u32> {- let guest_task = self.concurrent_state_mut(store).guest_task.unwrap();+ let guest_task = store.concurrent_state_mut().guest_task.unwrap();let (wait, set) = match &check {WaitableCheck::Wait(params) => (true, Some(params.set)),@@ -2848,11 +2821,11 @@ impl Instance {};// First, suspend this fiber, allowing any other tasks to run.- self.suspend(store, SuspendReason::Yielding { task: guest_task })?;+ store.suspend(SuspendReason::Yielding { task: guest_task })?;log::trace!("waitable check for {guest_task:?}; set {set:?}");- let state = self.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();let task = state.get_mut(guest_task)?;// If we're waiting, and there are no events immediately available,@@ -2869,13 +2842,10 @@ impl Instance {assert!(old.is_none());}- self.suspend(- store,- SuspendReason::Waiting {- set,- task: guest_task,- },- )?;+ store.suspend(SuspendReason::Waiting {+ set,+ task: guest_task,+ })?;}}@@ -2884,11 +2854,7 @@ impl Instance {let result = match check {// Deliver any pending events to the guest and return.WaitableCheck::Wait(params) | WaitableCheck::Poll(params) => {- let event = self.id().get_mut(store).get_event(- guest_task,- Some(params.set),- cancellable,- )?;+ let event = self.get_event(store, guest_task, Some(params.set), cancellable)?;let (ordinal, handle, result) = if wait {let (event, waitable) = event.unwrap();@@ -2909,7 +2875,6 @@ impl Instance {(ordinal, 0, result)}};- let store = store.store_opaque_mut();let options = Options::new_index(store, self, params.options);let ptr = func::validate_inbounds::<(u32, u32)>(options.memory_mut(store),@@ -2928,7 +2893,7 @@ impl Instance {/// Implements the `subtask.cancel` intrinsic.pub(crate) fn subtask_cancel(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller_instance: RuntimeComponentInstanceIndex,async_: bool,task_id: u32,@@ -2940,14 +2905,12 @@ impl Instance {let id = TableId::<HostTask>::new(rep);(Waitable::Host(id),- self.concurrent_state_mut(store)- .get_mut(id)?- .caller_instance,+ store.concurrent_state_mut().get_mut(id)?.caller_instance,)} else {let id = TableId::<GuestTask>::new(rep);if let Caller::Guest { instance, .. } =- &self.concurrent_state_mut(store).get_mut(id)?.caller+ &store.concurrent_state_mut().get_mut(id)?.caller{(Waitable::Guest(id), *instance)} else {@@ -2961,7 +2924,7 @@ impl Instance {log::trace!("subtask_cancel {waitable:?} (handle {task_id})");- let concurrent_state = self.concurrent_state_mut(store);+ let concurrent_state = store.concurrent_state_mut();if let Waitable::Host(host_task) = waitable {if let Some(handle) = concurrent_state.get_mut(host_task)?.join_handle.take() {handle.abort();@@ -3005,17 +2968,20 @@ impl Instance {.unwrap(){WaitMode::Fiber(fiber) => WorkItem::ResumeFiber(fiber),- WaitMode::Callback => WorkItem::GuestCall(GuestCall {+ WaitMode::Callback(instance) => WorkItem::GuestCall(GuestCall {task: guest_task,- kind: GuestCallKind::DeliverEvent { set: None },+ kind: GuestCallKind::DeliverEvent {+ instance,+ set: None,+ },}),};concurrent_state.push_high_priority(item);- self.suspend(store, SuspendReason::Yielding { task: caller })?;+ store.suspend(SuspendReason::Yielding { task: caller })?;}- let concurrent_state = self.concurrent_state_mut(store);+ let concurrent_state = store.concurrent_state_mut();let task = concurrent_state.get_mut(guest_task)?;if task.lift_result.is_some() {// Still not yet returned or cancelled; if `async_`, return@@ -3023,13 +2989,13 @@ impl Instance {if async_ {return Ok(BLOCKED);} else {- self.wait_for_event(store, Waitable::Guest(guest_task))?;+ store.wait_for_event(Waitable::Guest(guest_task))?;}}}}- let event = waitable.take_event(self.concurrent_state_mut(store))?;+ let event = waitable.take_event(store.concurrent_state_mut())?;if let Some(Event::Subtask {status: status @ (Status::Returned | Status::ReturnCancelled),}) = event@@ -3040,67 +3006,25 @@ impl Instance {}}- fn wait_for_event(self, store: &mut dyn VMStore, waitable: Waitable) -> Result<()> {- let state = self.concurrent_state_mut(store);- let caller = state.guest_task.unwrap();- let old_set = waitable.common(state)?.set;- let set = state.get_mut(caller)?.sync_call_set;- waitable.join(state, Some(set))?;- self.suspend(store, SuspendReason::Waiting { set, task: caller })?;- let state = self.concurrent_state_mut(store);- waitable.join(state, old_set)- }-- /// Configures TLS state so `store` will be available via `tls::get` within- /// the closure `f` provided.- ///- /// This is used to ensure that `Future::poll`, which doesn't take a `store`- /// parameter, is able to get access to the `store` during future poll- /// methods.- fn set_tls<R>(self, store: &mut dyn VMStore, f: impl FnOnce() -> R) -> R {- struct Reset<'a>(&'a mut dyn VMStore, Option<ComponentInstanceId>);-- impl Drop for Reset<'_> {- fn drop(&mut self) {- self.0.concurrent_async_state_mut().current_instance = self.1;- }- }- let prev = mem::replace(- &mut store.concurrent_async_state_mut().current_instance,- Some(self.id().instance()),- );- let reset = Reset(store, prev);-- tls::set(reset.0, f)- }-- /// Convenience function to reduce boilerplate.- pub(crate) fn concurrent_state_mut<'a>(- &self,- store: &'a mut StoreOpaque,- ) -> &'a mut ConcurrentState {- self.id().get_mut(store).concurrent_state_mut()- }-pub(crate) fn context_get(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,slot: u32,) -> Result<u32> {self.id().get(store).check_may_leave(caller)?;- self.concurrent_state_mut(store).context_get(slot)+ store.concurrent_state_mut().context_get(slot)}pub(crate) fn context_set(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,slot: u32,value: u32,) -> Result<()> {self.id().get(store).check_may_leave(caller)?;- self.concurrent_state_mut(store).context_set(slot, value)+ store.concurrent_state_mut().context_set(slot, value)}}@@ -3466,7 +3390,7 @@ impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {writer: u32,) -> Result<()> {instance.id().get(self).check_may_leave(caller)?;- instance.guest_drop_writable(StoreContextMut(self), TransmitIndex::Future(ty), writer)+ instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)}fn flat_stream_write(@@ -3535,7 +3459,7 @@ impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {writer: u32,) -> Result<()> {instance.id().get(self).check_may_leave(caller)?;- instance.guest_drop_writable(StoreContextMut(self), TransmitIndex::Stream(ty), writer)+ instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)}fn error_context_debug_message(@@ -3587,7 +3511,7 @@ impl TableDebug for HostTask {}type CallbackFn = Box<- dyn Fn(&mut dyn VMStore, Instance, RuntimeComponentInstanceIndex, Event, u32) -> Result<u32>+ dyn Fn(&mut dyn VMStore, RuntimeComponentInstanceIndex, Event, u32) -> Result<u32>+ Send+ Sync+ 'static,@@ -3925,9 +3849,12 @@ impl Waitable {let item = match mode {WaitMode::Fiber(fiber) => WorkItem::ResumeFiber(fiber),- WaitMode::Callback => WorkItem::GuestCall(GuestCall {+ WaitMode::Callback(instance) => WorkItem::GuestCall(GuestCall {task,- kind: GuestCallKind::DeliverEvent { set: Some(set) },+ kind: GuestCallKind::DeliverEvent {+ instance,+ set: Some(set),+ },}),};state.push_high_priority(item);@@ -4031,15 +3958,12 @@ impl TableDebug for WaitableSet {}/// Type-erased closure to lower the parameters for a guest task.-type RawLower = Box<- dyn FnOnce(&mut dyn VMStore, Instance, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,->;+type RawLower =+ Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;/// Type-erased closure to lift the result for a guest task.type RawLift = Box<- dyn FnOnce(&mut dyn VMStore, Instance, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>- + Send- + Sync,+ dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,>;/// Type erased result of a guest task which may be downcast to the expected@@ -4051,23 +3975,6 @@ type LiftedResult = Box<dyn Any + Send + Sync>;/// been lowered to a guest task's stack and linear memory.struct DummyResult;-/// Represents the state of a currently executing fiber which has been resumed-/// via `self::poll_fn`.-pub(crate) struct AsyncState {- /// The current instance being polled, if any, which is used to perform- /// checks to ensure that futures are always polled within the correct- /// instance.- current_instance: Option<ComponentInstanceId>,-}--impl Default for AsyncState {- fn default() -> Self {- Self {- current_instance: None,- }- }-}-/// Represents the Component Model Async state of a (sub-)component instance.#[derive(Default)]struct InstanceState {@@ -4080,8 +3987,7 @@ struct InstanceState {pending: BTreeMap<TableId<GuestTask>, GuestCallKind>,}-/// Represents the Component Model Async state of a top-level component instance-/// (i.e. a `super::ComponentInstance`).+/// Represents the Component Model Async state of a store.pub struct ConcurrentState {/// The currently running guest task, if any.guest_task: Option<TableId<GuestTask>>,@@ -4127,14 +4033,10 @@ pub struct ConcurrentState {/// as a `TableId<ErrorContextState>`.global_error_context_ref_counts:BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,-- /// Mirror of type information in `ComponentInstance`, placed here for- /// convenience at the cost of an extra `Arc` clone.- component: Component,}-impl ConcurrentState {- pub(crate) fn new(component: &Component) -> Self {+impl Default for ConcurrentState {+ fn default() -> Self {Self {guest_task: None,table: AlwaysMut::new(ResourceTable::new()),@@ -4146,10 +4048,11 @@ impl ConcurrentState {worker: None,worker_item: None,global_error_context_ref_counts: BTreeMap::new(),- component: component.clone(),}}+}+impl ConcurrentState {/// Take ownership of any fibers and futures owned by this object.////// This should be used when disposing of the `Store` containing this object@@ -4372,16 +4275,12 @@ impl ConcurrentState {self.get_mut(task)?.context[usize::try_from(slot).unwrap()] = val;Ok(())}-- fn options(&self, options: OptionsIndex) -> &CanonicalOptions {- &self.component.env_component().options[options]- }}/// Provide a type hint to compiler about the shape of a parameter lower/// closure.fn for_any_lower<- F: FnOnce(&mut dyn VMStore, Instance, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,+ F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,>(fun: F,) -> F {@@ -4390,9 +4289,7 @@ fn for_any_lower</// Provide a type hint to compiler about the shape of a result lift closure.fn for_any_lift<- F: FnOnce(&mut dyn VMStore, Instance, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>- + Send- + Sync,+ F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,>(fun: F,) -> F {@@ -4400,11 +4297,11 @@ fn for_any_lift<}/// Wrap the specified future in a `poll_fn` which asserts that the future is-/// only polled from the event loop of the specified `Instance`.+/// only polled from the event loop of the specified `Store`.///-/// See `Instance::run_concurrent` for details.+/// See `StoreContextMut::run_concurrent` for details.fn checked<F: Future + Send + 'static>(- instance: Instance,+ id: StoreId,fut: F,) -> impl Future<Output = F::Output> + Send + 'static {async move {@@ -4413,15 +4310,12 @@ fn checked<F: Future + Send + 'static>(let message = "\`Future`s which depend on asynchronous component tasks, streams, or \futures to complete may only be polled from the event loop of the \- instance from which they originated. Please use \- `Instance::{run_concurrent,spawn}` to poll or await them.\+ store to which they belong. Please use \+ `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\";tls::try_get(|store| {let matched = match store {- tls::TryGet::Some(store) => {- let a = store.concurrent_async_state_mut().current_instance;- a == Some(instance.id().instance())- }+ tls::TryGet::Some(store) => store.id() == id,tls::TryGet::Taken | tls::TryGet::None => false,};@@ -4435,12 +4329,12 @@ fn checked<F: Future + Send + 'static>(}}-/// Assert that `Instance::run_concurrent` has not been called from within an-/// instance's event loop.+/// Assert that `StoreContextMut::run_concurrent` has not been called from+/// within an store's event loop.fn check_recursive_run() {tls::try_get(|store| {if !matches!(store, tls::TryGet::None) {- panic!("Recursive `Instance::run_concurrent` calls not supported")+ panic!("Recursive `StoreContextMut::run_concurrent` calls not supported")}});}@@ -4485,16 +4379,12 @@ pub(crate) struct PreparedCall<R> {impl<R> PreparedCall<R> {/// Get a copy of the `TaskId` for this `PreparedCall`.pub(crate) fn task_id(&self) -> TaskId {- TaskId {- handle: self.handle,- task: self.task,- }+ TaskId { task: self.task }}}/// Represents a task created by `prepare_call`.pub(crate) struct TaskId {- handle: Func,task: TableId<GuestTask>,}@@ -4509,7 +4399,7 @@ impl TaskId {/// it should only be called once for a given task, and only after either/// the task has completed or the instance has trapped.pub(crate) fn remove<T>(&self, store: StoreContextMut<T>) -> Result<()> {- Waitable::Guest(self.task).delete_from(self.handle.instance().concurrent_state_mut(store.0))+ Waitable::Guest(self.task).delete_from(store.0.concurrent_state_mut())}}@@ -4542,7 +4432,7 @@ pub(crate) fn prepare_call<T, R>(let memory = options.memory_raw().map(SendSyncPtr::new);let string_encoding = options.string_encoding();let token = StoreToken::new(store.as_context_mut());- let state = handle.instance().concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();assert!(state.guest_task.is_none());@@ -4551,13 +4441,11 @@ pub(crate) fn prepare_call<T, R>(let mut task = GuestTask::new(state,- Box::new(for_any_lower(move |store, instance, params| {- debug_assert!(instance.id() == handle.instance().id());+ Box::new(for_any_lower(move |store, params| {lower_params(handle, token.as_context_mut(store), params)})),LiftResult {- lift: Box::new(for_any_lift(move |store, instance, result| {- debug_assert!(instance.id() == handle.instance().id());+ lift: Box::new(for_any_lift(move |store, result| {lift_result(handle, store, result)})),ty: task_return_type,@@ -4572,12 +4460,9 @@ pub(crate) fn prepare_call<T, R>(},callback.map(|callback| {let callback = SendSyncPtr::new(callback);+ let instance = handle.instance();Box::new(- move |store: &mut dyn VMStore,- instance: Instance,- runtime_instance,- event,- handle| {+ move |store: &mut dyn VMStore, runtime_instance, event, handle| {let store = token.as_context_mut(store);// SAFETY: Per the contract of `prepare_call`, the callback// will remain valid at least as long is this task exists.@@ -4615,7 +4500,7 @@ pub(crate) fn prepare_call<T, R>(////// The returned future will resolve to the result once it is available, but/// must only be polled via the instance's event loop. See-/// `Instance::run_concurrent` for details.+/// `StoreContextMut::run_concurrent` for details.pub(crate) fn queue_call<T: 'static, R: Send + 'static>(mut store: StoreContextMut<T>,prepared: PreparedCall<R>,@@ -4632,7 +4517,7 @@ pub(crate) fn queue_call<T: 'static, R: Send + 'static>(queue_call0(store.as_context_mut(), handle, task, param_count)?;Ok(checked(- handle.instance(),+ store.0.id(),rx.map(move |result| {result.map(|v| (*v.downcast().unwrap(), exit_rx))diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rsindex 6f90d4f790..a64ad8012c 100644--- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs+++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs@@ -216,7 +216,6 @@ pub(super) struct FlatAbi {/// Represents the buffer for a host- or guest-initiated stream read.pub struct Destination<'a, T, B> {- instance: Instance,id: TableId<TransmitState>,buffer: &'a mut B,host_buffer: Option<&'a mut Cursor<Vec<u8>>>,@@ -227,7 +226,6 @@ impl<'a, T, B> Destination<'a, T, B> {/// Reborrow `self` so it can be used again later.pub fn reborrow(&mut self) -> Destination<'_, T, B> {Destination {- instance: self.instance,id: self.id,buffer: &mut *self.buffer,host_buffer: self.host_buffer.as_deref_mut(),@@ -273,9 +271,10 @@ impl<'a, T, B> Destination<'a, T, B> {/// without blocking (or reach a terminal state such as end-of-stream),/// meaning the next non-zero read must complete without blocking.pub fn remaining(&self, mut store: impl AsContextMut) -> Option<usize> {- let transmit = self- .instance- .concurrent_state_mut(store.as_context_mut().0)+ let transmit = store+ .as_context_mut()+ .0+ .concurrent_state_mut().get_mut(self.id).unwrap();@@ -315,7 +314,6 @@ impl<'a, B> Destination<'a, u8, B> {}DirectDestination {- instance: self.instance,id: self.id,host_buffer: self.host_buffer,store,@@ -326,7 +324,6 @@ impl<'a, B> Destination<'a, u8, B> {/// Represents a read from a `stream<u8>`, providing direct access to the/// writer's buffer.pub struct DirectDestination<'a, D: 'static> {- instance: Instance,id: TableId<TransmitState>,host_buffer: Option<&'a mut Cursor<Vec<u8>>>,store: StoreContextMut<'a, D>,@@ -339,8 +336,10 @@ impl<D: 'static> DirectDestination<'_, D> {buffer.get_mut()} else {let transmit = self- .instance- .concurrent_state_mut(self.store.as_context_mut().0)+ .store+ .as_context_mut()+ .0+ .concurrent_state_mut().get_mut(self.id).unwrap();@@ -380,8 +379,10 @@ impl<D: 'static> DirectDestination<'_, D> {);} else {let transmit = self- .instance- .concurrent_state_mut(self.store.as_context_mut().0)+ .store+ .as_context_mut()+ .0+ .concurrent_state_mut().get_mut(self.id).unwrap();@@ -651,7 +652,7 @@ impl<D> StreamProducer<D> for bytes::BytesMut {/// Represents the buffer for a host- or guest-initiated stream write.pub struct Source<'a, T> {- instance: Instance,+ instance: Option<Instance>,id: TableId<TransmitState>,host_buffer: Option<&'a mut dyn WriteBuffer<T>>,}@@ -677,10 +678,7 @@ impl<'a, T> Source<'a, T> {buffer.move_from(*input, count);} else {let store = store.as_context_mut();- let transmit = self- .instance- .concurrent_state_mut(store.0)- .get_mut(self.id)?;+ let transmit = store.0.concurrent_state_mut().get_mut(self.id)?;let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {unreachable!();@@ -697,7 +695,8 @@ impl<'a, T> Source<'a, T> {unreachable!()};- let cx = &mut LiftContext::new(store.0.store_opaque_mut(), &options, self.instance);+ let cx =+ &mut LiftContext::new(store.0.store_opaque_mut(), &options, self.instance.unwrap());let ty = payload(ty, cx.types);let old_remaining = buffer.remaining_capacity();lift::<T, B, S::Data>(@@ -708,10 +707,7 @@ impl<'a, T> Source<'a, T> {count - guest_offset,)?;- let transmit = self- .instance- .concurrent_state_mut(store.0)- .get_mut(self.id)?;+ let transmit = store.0.concurrent_state_mut().get_mut(self.id)?;let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {unreachable!();@@ -729,9 +725,10 @@ impl<'a, T> Source<'a, T> {whereT: 'static,{- let transmit = self- .instance- .concurrent_state_mut(store.as_context_mut().0)+ let transmit = store+ .as_context_mut()+ .0+ .concurrent_state_mut().get_mut(self.id).unwrap();@@ -753,7 +750,6 @@ impl<'a> Source<'a, u8> {/// Return a `DirectSource` view of `self`.pub fn as_direct<D>(self, store: StoreContextMut<'a, D>) -> DirectSource<'a, D> {DirectSource {- instance: self.instance,id: self.id,host_buffer: self.host_buffer,store,@@ -764,7 +760,6 @@ impl<'a> Source<'a, u8> {/// Represents a write to a `stream<u8>`, providing direct access to the/// writer's buffer.pub struct DirectSource<'a, D: 'static> {- instance: Instance,id: TableId<TransmitState>,host_buffer: Option<&'a mut dyn WriteBuffer<u8>>,store: StoreContextMut<'a, D>,@@ -777,8 +772,10 @@ impl<D: 'static> DirectSource<'_, D> {buffer.remaining()} else {let transmit = self- .instance- .concurrent_state_mut(self.store.as_context_mut().0)+ .store+ .as_context_mut()+ .0+ .concurrent_state_mut().get_mut(self.id).unwrap();@@ -813,8 +810,10 @@ impl<D: 'static> DirectSource<'_, D> {buffer.skip(count);} else {let transmit = self- .instance- .concurrent_state_mut(self.store.as_context_mut().0)+ .store+ .as_context_mut()+ .0+ .concurrent_state_mut().get_mut(self.id).unwrap();@@ -1012,7 +1011,6 @@ pub trait FutureConsumer<D>: Send + 'static {/// end will hang indefinitely. Consider using [`GuardedFutureReader`] to/// ensure that disposal happens automatically.pub struct FutureReader<T> {- instance: Instance,id: TableId<TransmitHandle>,_phantom: PhantomData<T>,}@@ -1020,7 +1018,6 @@ pub struct FutureReader<T> {impl<T> FutureReader<T> {/// Create a new future with the specified producer.pub fn new<S: AsContextMut>(- instance: Instance,mut store: S,producer: impl FutureProducer<S::Data, Item = T>,) -> Self@@ -1065,18 +1062,14 @@ impl<T> FutureReader<T> {}Self::new_(- instance.new_transmit(- store.as_context_mut(),- TransmitKind::Future,- Producer(producer),- ),- instance,+ store+ .as_context_mut()+ .new_transmit(TransmitKind::Future, Producer(producer)),)}- fn new_(id: TableId<TransmitHandle>, instance: Instance) -> Self {+ fn new_(id: TableId<TransmitHandle>) -> Self {Self {- instance,id,_phantom: PhantomData,}@@ -1085,7 +1078,7 @@ impl<T> FutureReader<T> {/// Set the consumer that accepts the result of this future.pub fn pipe<S: AsContextMut>(self,- store: S,+ mut store: S,consumer: impl FutureConsumer<S::Data, Item = T> + Unpin,) whereT: func::Lift + 'static,@@ -1128,8 +1121,9 @@ impl<T> FutureReader<T> {}}- self.instance- .set_consumer(store, self.id, TransmitKind::Future, Consumer(consumer));+ store+ .as_context_mut()+ .set_consumer(self.id, TransmitKind::Future, Consumer(consumer));}/// Convert this `FutureReader` into a [`Val`].@@ -1139,18 +1133,14 @@ impl<T> FutureReader<T> {}/// Attempt to convert the specified [`Val`] to a `FutureReader`.- pub fn from_val(- mut store: impl AsContextMut<Data: Send>,- instance: Instance,- value: &Val,- ) -> Result<Self> {+ pub fn from_val(mut store: impl AsContextMut<Data: Send>, value: &Val) -> Result<Self> {let Val::Future(FutureAny(rep)) = value else {bail!("expected `future`; got `{}`", value.desc());};let store = store.as_context_mut();let id = TableId::<TransmitHandle>::new(*rep);- instance.concurrent_state_mut(store.0).get_mut(id)?; // Just make sure it's present- Ok(Self::new_(id, instance))+ store.0.concurrent_state_mut().get_mut(id)?; // Just make sure it's present+ Ok(Self::new_(id))}/// Transfer ownership of the read end of a future from a guest to the host.@@ -1165,7 +1155,7 @@ impl<T> FutureReader<T> {bail!("cannot lift future after being notified that the writable end dropped");}let id = TableId::<TransmitHandle>::new(rep);- let concurrent_state = cx.instance_mut().concurrent_state_mut();+ let concurrent_state = cx.concurrent_state_mut();let future = concurrent_state.get_mut(id)?;future.common.handle = None;let state = future.state;@@ -1174,7 +1164,7 @@ impl<T> FutureReader<T> {bail!("cannot lift future after previous read succeeded");}- Ok(Self::new_(id, cx.instance_handle()))+ Ok(Self::new_(id))}_ => func::bad_type_info(),}@@ -1190,8 +1180,10 @@ impl<T> FutureReader<T> {pub fn close(&mut self, mut store: impl AsContextMut) {// `self` should never be used again, but leave an invalid handle there just in case.let id = mem::replace(&mut self.id, TableId::new(u32::MAX));- self.instance- .host_drop_reader(store.as_context_mut().0, id, TransmitKind::Future)+ store+ .as_context_mut()+ .0+ .host_drop_reader(id, TransmitKind::Future).unwrap();}@@ -1217,7 +1209,6 @@ impl<T> fmt::Debug for FutureReader<T> {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {f.debug_struct("FutureReader").field("id", &self.id)- .field("instance", &self.instance).finish()}}@@ -1230,7 +1221,7 @@ pub(crate) fn lower_future_to_index<U>() -> Result<u32> {match ty {InterfaceType::Future(dst) => {- let concurrent_state = cx.instance_mut().concurrent_state_mut();+ let concurrent_state = cx.store.0.concurrent_state_mut();let id = TableId::<TransmitHandle>::new(rep);let state = concurrent_state.get_mut(id)?.state;let rep = concurrent_state.get_mut(state)?.read_handle.rep();@@ -1240,11 +1231,7 @@ pub(crate) fn lower_future_to_index<U>(.table_for_transmit(TransmitIndex::Future(dst)).future_insert_read(dst, rep)?;- cx.instance_mut()- .concurrent_state_mut()- .get_mut(id)?- .common- .handle = Some(handle);+ cx.store.0.concurrent_state_mut().get_mut(id)?.common.handle = Some(handle);Ok(handle)}@@ -1379,7 +1366,6 @@ where/// indefinitely. Consider using [`GuardedStreamReader`] to ensure that/// disposal happens automatically.pub struct StreamReader<T> {- instance: Instance,id: TableId<TransmitHandle>,_phantom: PhantomData<T>,}@@ -1387,22 +1373,21 @@ pub struct StreamReader<T> {impl<T> StreamReader<T> {/// Create a new stream with the specified producer.pub fn new<S: AsContextMut>(- instance: Instance,- store: S,+ mut store: S,producer: impl StreamProducer<S::Data, Item = T>,) -> SelfwhereT: func::Lower + func::Lift + Send + Sync + 'static,{Self::new_(- instance.new_transmit(store, TransmitKind::Stream, producer),- instance,+ store+ .as_context_mut()+ .new_transmit(TransmitKind::Stream, producer),)}- fn new_(id: TableId<TransmitHandle>, instance: Instance) -> Self {+ fn new_(id: TableId<TransmitHandle>) -> Self {Self {- instance,id,_phantom: PhantomData,}@@ -1426,7 +1411,7 @@ impl<T> StreamReader<T> {/// along with `TypeId::of::<V>()`.pub fn try_into<V: 'static>(mut self, mut store: impl AsContextMut) -> Result<V, Self> {let store = store.as_context_mut();- let state = self.instance.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();let id = state.get_mut(self.id).unwrap().state;if let WriteState::HostReady { try_into, .. } = &state.get_mut(id).unwrap().write {match try_into(TypeId::of::<V>()) {@@ -1442,12 +1427,16 @@ impl<T> StreamReader<T> {}/// Set the consumer that accepts the items delivered to this stream.- pub fn pipe<S: AsContextMut>(self, store: S, consumer: impl StreamConsumer<S::Data, Item = T>)- where+ pub fn pipe<S: AsContextMut>(+ self,+ mut store: S,+ consumer: impl StreamConsumer<S::Data, Item = T>,+ ) whereT: 'static,{- self.instance- .set_consumer(store, self.id, TransmitKind::Stream, consumer);+ store+ .as_context_mut()+ .set_consumer(self.id, TransmitKind::Stream, consumer);}/// Convert this `StreamReader` into a [`Val`].@@ -1457,18 +1446,14 @@ impl<T> StreamReader<T> {}/// Attempt to convert the specified [`Val`] to a `StreamReader`.- pub fn from_val(- mut store: impl AsContextMut<Data: Send>,- instance: Instance,- value: &Val,- ) -> Result<Self> {+ pub fn from_val(mut store: impl AsContextMut<Data: Send>, value: &Val) -> Result<Self> {let Val::Stream(StreamAny(rep)) = value else {bail!("expected `stream`; got `{}`", value.desc());};let store = store.as_context_mut();let id = TableId::<TransmitHandle>::new(*rep);- instance.concurrent_state_mut(store.0).get_mut(id)?; // Just make sure it's present- Ok(Self::new_(id, instance))+ store.0.concurrent_state_mut().get_mut(id)?; // Just make sure it's present+ Ok(Self::new_(id))}/// Transfer ownership of the read end of a stream from a guest to the host.@@ -1483,12 +1468,8 @@ impl<T> StreamReader<T> {bail!("cannot lift stream after being notified that the writable end dropped");}let id = TableId::<TransmitHandle>::new(rep);- cx.instance_mut()- .concurrent_state_mut()- .get_mut(id)?- .common- .handle = None;- Ok(Self::new_(id, cx.instance_handle()))+ cx.concurrent_state_mut().get_mut(id)?.common.handle = None;+ Ok(Self::new_(id))}_ => func::bad_type_info(),}@@ -1504,8 +1485,10 @@ impl<T> StreamReader<T> {pub fn close(&mut self, mut store: impl AsContextMut) {// `self` should never be used again, but leave an invalid handle there just in case.let id = mem::replace(&mut self.id, TableId::new(u32::MAX));- self.instance- .host_drop_reader(store.as_context_mut().0, id, TransmitKind::Stream)+ store+ .as_context_mut()+ .0+ .host_drop_reader(id, TransmitKind::Stream).unwrap()}@@ -1531,7 +1514,6 @@ impl<T> fmt::Debug for StreamReader<T> {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {f.debug_struct("StreamReader").field("id", &self.id)- .field("instance", &self.instance).finish()}}@@ -1544,7 +1526,7 @@ pub(crate) fn lower_stream_to_index<U>() -> Result<u32> {match ty {InterfaceType::Stream(dst) => {- let concurrent_state = cx.instance_mut().concurrent_state_mut();+ let concurrent_state = cx.store.0.concurrent_state_mut();let id = TableId::<TransmitHandle>::new(rep);let state = concurrent_state.get_mut(id)?.state;let rep = concurrent_state.get_mut(state)?.read_handle.rep();@@ -1554,11 +1536,7 @@ pub(crate) fn lower_stream_to_index<U>(.table_for_transmit(TransmitIndex::Stream(dst)).stream_insert_read(dst, rep)?;- cx.instance_mut()- .concurrent_state_mut()- .get_mut(id)?- .common- .handle = Some(handle);+ cx.store.0.concurrent_state_mut().get_mut(id)?.common.handle = Some(handle);Ok(handle)}@@ -1858,7 +1836,9 @@ impl TableDebug for TransmitState {}type PollStream = Box<- dyn Fn() -> Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>> + Send + Sync,+ dyn Fn(Option<Instance>) -> Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>+ + Send+ + Sync,>;type TryInto = Box<dyn Fn(TypeId) -> Option<Box<dyn Any>> + Send + Sync>;@@ -1869,6 +1849,7 @@ enum WriteState {Open,/// The write end is owned by a guest task and a write is pending.GuestReady {+ instance: Instance,ty: TransmitIndex,flat_abi: Option<FlatAbi>,options: Options,@@ -1957,132 +1938,389 @@ fn return_code(kind: TransmitKind, state: StreamResult, guest_offset: usize) ->}}-impl Instance {- fn new_transmit<S: AsContextMut, P: StreamProducer<S::Data>>(- self,- mut store: S,+impl StoreOpaque {+ fn pipe_from_guest(+ &mut self,kind: TransmitKind,- producer: P,- ) -> TableId<TransmitHandle>- where- P::Item: func::Lower,- {- let mut store = store.as_context_mut();- let token = StoreToken::new(store.as_context_mut());- let state = self.concurrent_state_mut(store.0);- let (_, read) = state.new_transmit().unwrap();- let producer = Arc::new(Mutex::new(Some((Box::pin(producer), P::Buffer::default()))));- let id = state.get_mut(read).unwrap().state;- let produce = Box::new({- let producer = producer.clone();- move || {- let producer = producer.clone();- async move {- let (mut mine, mut buffer) = producer.lock().unwrap().take().unwrap();-- let (result, cancelled) = if buffer.remaining().is_empty() {- future::poll_fn(|cx| {- tls::get(|store| {- let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();+ id: TableId<TransmitState>,+ future: Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,+ ) {+ let future = async move {+ let stream_state = future.await?;+ tls::get(|store| {+ let state = store.concurrent_state_mut();+ let transmit = state.get_mut(id)?;+ let ReadState::HostReady {+ consume,+ guest_offset,+ ..+ } = mem::replace(&mut transmit.read, ReadState::Open)+ else {+ unreachable!();+ };+ let code = return_code(kind, stream_state, guest_offset);+ transmit.read = match stream_state {+ StreamResult::Dropped => ReadState::Dropped,+ StreamResult::Completed | StreamResult::Cancelled => ReadState::HostReady {+ consume,+ guest_offset: 0,+ cancel: false,+ cancel_waker: None,+ },+ };+ let WriteState::GuestReady { ty, handle, .. } =+ mem::replace(&mut transmit.write, WriteState::Open)+ else {+ unreachable!();+ };+ state.send_write_result(ty, id, handle, code)?;+ Ok(())+ })+ };- let &WriteState::HostReady { cancel, .. } = &transmit.write else {- unreachable!();- };+ self.concurrent_state_mut().push_future(future.boxed());+ }- let mut host_buffer =- if let ReadState::HostToHost { buffer, .. } = &mut transmit.read {- Some(Cursor::new(mem::take(buffer)))- } else {- None- };+ fn pipe_to_guest(+ &mut self,+ kind: TransmitKind,+ id: TableId<TransmitState>,+ future: Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,+ ) {+ let future = async move {+ let stream_state = future.await?;+ tls::get(|store| {+ let state = store.concurrent_state_mut();+ let transmit = state.get_mut(id)?;+ let WriteState::HostReady {+ produce,+ try_into,+ guest_offset,+ ..+ } = mem::replace(&mut transmit.write, WriteState::Open)+ else {+ unreachable!();+ };+ let code = return_code(kind, stream_state, guest_offset);+ transmit.write = match stream_state {+ StreamResult::Dropped => WriteState::Dropped,+ StreamResult::Completed | StreamResult::Cancelled => WriteState::HostReady {+ produce,+ try_into,+ guest_offset: 0,+ cancel: false,+ cancel_waker: None,+ },+ };+ let ReadState::GuestReady { ty, handle, .. } =+ mem::replace(&mut transmit.read, ReadState::Open)+ else {+ unreachable!();+ };+ state.send_read_result(ty, id, handle, code)?;+ Ok(())+ })+ };- let poll = mine.as_mut().poll_produce(- cx,- token.as_context_mut(store),- Destination {- instance: self,- id,- buffer: &mut buffer,- host_buffer: host_buffer.as_mut(),- _phantom: PhantomData,- },- cancel,- );+ self.concurrent_state_mut().push_future(future.boxed());+ }- let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();+ /// Drop the read end of a stream or future read from the host.+ fn host_drop_reader(&mut self, id: TableId<TransmitHandle>, kind: TransmitKind) -> Result<()> {+ let state = self.concurrent_state_mut();+ let transmit_id = state.get_mut(id)?.state;+ let transmit = state+ .get_mut(transmit_id)+ .with_context(|| format!("error closing reader {transmit_id:?}"))?;+ log::trace!(+ "host_drop_reader state {transmit_id:?}; read state {:?} write state {:?}",+ transmit.read,+ transmit.write+ );- let host_offset = if let (- Some(host_buffer),- ReadState::HostToHost { buffer, limit, .. },- ) = (host_buffer, &mut transmit.read)- {- *limit = usize::try_from(host_buffer.position()).unwrap();- *buffer = host_buffer.into_inner();- *limit- } else {- 0- };+ transmit.read = ReadState::Dropped;- {- let WriteState::HostReady {- guest_offset,- cancel,- cancel_waker,- ..- } = &mut transmit.write- else {- unreachable!();- };+ // If the write end is already dropped, it should stay dropped,+ // otherwise, it should be opened.+ let new_state = if let WriteState::Dropped = &transmit.write {+ WriteState::Dropped+ } else {+ WriteState::Open+ };- if poll.is_pending() {- if !buffer.remaining().is_empty()- || *guest_offset > 0- || host_offset > 0- {- return Poll::Ready(Err(anyhow!(- "StreamProducer::poll_produce returned Poll::Pending \- after producing at least one item"- )));- }- *cancel_waker = Some(cx.waker().clone());- } else {- *cancel_waker = None;- *cancel = false;- }- }+ let write_handle = transmit.write_handle;- poll.map(|v| v.map(|result| (result, cancel)))- })- })- .await?- } else {- (StreamResult::Completed, false)- };+ match mem::replace(&mut transmit.write, new_state) {+ // If a guest is waiting to write, notify it that the read end has+ // been dropped.+ WriteState::GuestReady { ty, handle, .. } => {+ state.update_event(+ write_handle.rep(),+ match ty {+ TransmitIndex::Future(ty) => Event::FutureWrite {+ code: ReturnCode::Dropped(0),+ pending: Some((ty, handle)),+ },+ TransmitIndex::Stream(ty) => Event::StreamWrite {+ code: ReturnCode::Dropped(0),+ pending: Some((ty, handle)),+ },+ },+ )?;+ }- let (guest_offset, host_offset, count) = tls::get(|store| {- let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();- let (count, host_offset) = match &transmit.read {- &ReadState::GuestReady { count, .. } => (count, 0),- &ReadState::HostToHost { limit, .. } => (1, limit),- _ => unreachable!(),- };- let guest_offset = match &transmit.write {- &WriteState::HostReady { guest_offset, .. } => guest_offset,- _ => unreachable!(),- };- (guest_offset, host_offset, count)- });+ WriteState::HostReady { .. } => {}- match result {- StreamResult::Completed => {- if count > 1- && buffer.remaining().is_empty()- && guest_offset == 0- && host_offset == 0- {- bail!(- "StreamProducer::poll_produce returned StreamResult::Completed \- without producing any items"+ WriteState::Open => {+ state.update_event(+ write_handle.rep(),+ match kind {+ TransmitKind::Future => Event::FutureWrite {+ code: ReturnCode::Dropped(0),+ pending: None,+ },+ TransmitKind::Stream => Event::StreamWrite {+ code: ReturnCode::Dropped(0),+ pending: None,+ },+ },+ )?;+ }++ WriteState::Dropped => {+ log::trace!("host_drop_reader delete {transmit_id:?}");+ state.delete_transmit(transmit_id)?;+ }+ }+ Ok(())+ }++ /// Drop the write end of a stream or future read from the host.+ fn host_drop_writer(+ &mut self,+ id: TableId<TransmitHandle>,+ on_drop_open: Option<fn() -> Result<()>>,+ ) -> Result<()> {+ let state = self.concurrent_state_mut();+ let transmit_id = state.get_mut(id)?.state;+ let transmit = state+ .get_mut(transmit_id)+ .with_context(|| format!("error closing writer {transmit_id:?}"))?;+ log::trace!(+ "host_drop_writer state {transmit_id:?}; write state {:?} read state {:?}",+ transmit.read,+ transmit.write+ );++ // Existing queued transmits must be updated with information for the impending writer closure+ match &mut transmit.write {+ WriteState::GuestReady { .. } => {+ unreachable!("can't call `host_drop_writer` on a guest-owned writer");+ }+ WriteState::HostReady { .. } => {}+ v @ WriteState::Open => {+ if let (Some(on_drop_open), false) = (+ on_drop_open,+ transmit.done || matches!(transmit.read, ReadState::Dropped),+ ) {+ on_drop_open()?;+ } else {+ *v = WriteState::Dropped;+ }+ }+ WriteState::Dropped => unreachable!("write state is already dropped"),+ }++ let transmit = self.concurrent_state_mut().get_mut(transmit_id)?;++ // If the existing read state is dropped, then there's nothing to read+ // and we can keep it that way.+ //+ // If the read state was any other state, then we must set the new state to open+ // to indicate that there *is* data to be read+ let new_state = if let ReadState::Dropped = &transmit.read {+ ReadState::Dropped+ } else {+ ReadState::Open+ };++ let read_handle = transmit.read_handle;++ // Swap in the new read state+ match mem::replace(&mut transmit.read, new_state) {+ // If the guest was ready to read, then we cannot drop the reader (or writer);+ // we must deliver the event, and update the state associated with the handle to+ // represent that a read must be performed+ ReadState::GuestReady { ty, handle, .. } => {+ // Ensure the final read of the guest is queued, with appropriate closure indicator+ self.concurrent_state_mut().update_event(+ read_handle.rep(),+ match ty {+ TransmitIndex::Future(ty) => Event::FutureRead {+ code: ReturnCode::Dropped(0),+ pending: Some((ty, handle)),+ },+ TransmitIndex::Stream(ty) => Event::StreamRead {+ code: ReturnCode::Dropped(0),+ pending: Some((ty, handle)),+ },+ },+ )?;+ }++ ReadState::HostReady { .. } | ReadState::HostToHost { .. } => {}++ // If the read state is open, then there are no registered readers of the stream/future+ ReadState::Open => {+ self.concurrent_state_mut().update_event(+ read_handle.rep(),+ match on_drop_open {+ Some(_) => Event::FutureRead {+ code: ReturnCode::Dropped(0),+ pending: None,+ },+ None => Event::StreamRead {+ code: ReturnCode::Dropped(0),+ pending: None,+ },+ },+ )?;+ }++ // If the read state was already dropped, then we can remove the transmit state completely+ // (both writer and reader have been dropped)+ ReadState::Dropped => {+ log::trace!("host_drop_writer delete {transmit_id:?}");+ self.concurrent_state_mut().delete_transmit(transmit_id)?;+ }+ }+ Ok(())+ }+}++impl<T> StoreContextMut<'_, T> {+ fn new_transmit<P: StreamProducer<T>>(+ mut self,+ kind: TransmitKind,+ producer: P,+ ) -> TableId<TransmitHandle>+ where+ P::Item: func::Lower,+ {+ let token = StoreToken::new(self.as_context_mut());+ let state = self.0.concurrent_state_mut();+ let (_, read) = state.new_transmit().unwrap();+ let producer = Arc::new(Mutex::new(Some((Box::pin(producer), P::Buffer::default()))));+ let id = state.get_mut(read).unwrap().state;+ let produce = Box::new({+ let producer = producer.clone();+ move |instance| {+ let producer = producer.clone();+ async move {+ let (mut mine, mut buffer) = producer.lock().unwrap().take().unwrap();++ let (result, cancelled) = if buffer.remaining().is_empty() {+ future::poll_fn(|cx| {+ tls::get(|store| {+ let transmit = store.concurrent_state_mut().get_mut(id).unwrap();++ let &WriteState::HostReady { cancel, .. } = &transmit.write else {+ unreachable!();+ };++ let mut host_buffer =+ if let ReadState::HostToHost { buffer, .. } = &mut transmit.read {+ Some(Cursor::new(mem::take(buffer)))+ } else {+ None+ };++ let poll = mine.as_mut().poll_produce(+ cx,+ token.as_context_mut(store),+ Destination {+ id,+ buffer: &mut buffer,+ host_buffer: host_buffer.as_mut(),+ _phantom: PhantomData,+ },+ cancel,+ );++ let transmit = store.concurrent_state_mut().get_mut(id).unwrap();++ let host_offset = if let (+ Some(host_buffer),+ ReadState::HostToHost { buffer, limit, .. },+ ) = (host_buffer, &mut transmit.read)+ {+ *limit = usize::try_from(host_buffer.position()).unwrap();+ *buffer = host_buffer.into_inner();+ *limit+ } else {+ 0+ };++ {+ let WriteState::HostReady {+ guest_offset,+ cancel,+ cancel_waker,+ ..+ } = &mut transmit.write+ else {+ unreachable!();+ };++ if poll.is_pending() {+ if !buffer.remaining().is_empty()+ || *guest_offset > 0+ || host_offset > 0+ {+ return Poll::Ready(Err(anyhow!(+ "StreamProducer::poll_produce returned Poll::Pending \+ after producing at least one item"+ )));+ }+ *cancel_waker = Some(cx.waker().clone());+ } else {+ *cancel_waker = None;+ *cancel = false;+ }+ }++ poll.map(|v| v.map(|result| (result, cancel)))+ })+ })+ .await?+ } else {+ (StreamResult::Completed, false)+ };++ let (guest_offset, host_offset, count) = tls::get(|store| {+ let transmit = store.concurrent_state_mut().get_mut(id).unwrap();+ let (count, host_offset) = match &transmit.read {+ &ReadState::GuestReady { count, .. } => (count, 0),+ &ReadState::HostToHost { limit, .. } => (1, limit),+ _ => unreachable!(),+ };+ let guest_offset = match &transmit.write {+ &WriteState::HostReady { guest_offset, .. } => guest_offset,+ _ => unreachable!(),+ };+ (guest_offset, host_offset, count)+ });++ match result {+ StreamResult::Completed => {+ if count > 1+ && buffer.remaining().is_empty()+ && guest_offset == 0+ && host_offset == 0+ {+ bail!(+ "StreamProducer::poll_produce returned StreamResult::Completed \+ without producing any items");}}@@ -2102,7 +2340,7 @@ impl Instance {*producer.lock().unwrap() = Some((mine, buffer));if write_buffer {- self.write(token, id, producer, kind).await?;+ write(instance, token, id, producer, kind).await?;}Ok(result)@@ -2130,22 +2368,20 @@ impl Instance {read}- fn set_consumer<S: AsContextMut, C: StreamConsumer<S::Data>>(- self,- mut store: S,+ fn set_consumer<C: StreamConsumer<T>>(+ mut self,id: TableId<TransmitHandle>,kind: TransmitKind,consumer: C,) {- let mut store = store.as_context_mut();- let token = StoreToken::new(store.as_context_mut());- let state = self.concurrent_state_mut(store.0);+ let token = StoreToken::new(self.as_context_mut());+ let state = self.0.concurrent_state_mut();let id = state.get_mut(id).unwrap().state;let transmit = state.get_mut(id).unwrap();let consumer = Arc::new(Mutex::new(Some(Box::pin(consumer))));let consume_with_buffer = {let consumer = consumer.clone();- async move |mut host_buffer: Option<&mut dyn WriteBuffer<C::Item>>| {+ async move |instance, mut host_buffer: Option<&mut dyn WriteBuffer<C::Item>>| {let mut mine = consumer.lock().unwrap().take().unwrap();let host_buffer_remaining_before =@@ -2153,18 +2389,17 @@ impl Instance {let (result, cancelled) = future::poll_fn(|cx| {tls::get(|store| {- let cancel =- match &self.concurrent_state_mut(store).get_mut(id).unwrap().read {- &ReadState::HostReady { cancel, .. } => cancel,- ReadState::Open => false,- _ => unreachable!(),- };+ let cancel = match &store.concurrent_state_mut().get_mut(id).unwrap().read {+ &ReadState::HostReady { cancel, .. } => cancel,+ ReadState::Open => false,+ _ => unreachable!(),+ };let poll = mine.as_mut().poll_consume(cx,token.as_context_mut(store),Source {- instance: self,+ instance,id,host_buffer: host_buffer.as_deref_mut(),},@@ -2175,7 +2410,7 @@ impl Instance {cancel_waker,cancel,..- } = &mut self.concurrent_state_mut(store).get_mut(id).unwrap().read+ } = &mut store.concurrent_state_mut().get_mut(id).unwrap().read{if poll.is_pending() {*cancel_waker = Some(cx.waker().clone());@@ -2191,7 +2426,7 @@ impl Instance {.await?;let (guest_offset, count) = tls::get(|store| {- let transmit = self.concurrent_state_mut(store).get_mut(id).unwrap();+ let transmit = store.concurrent_state_mut().get_mut(id).unwrap();(match &transmit.read {&ReadState::HostReady { guest_offset, .. } => guest_offset,@@ -2223,7 +2458,7 @@ impl Instance {if let TransmitKind::Future = kind {tls::get(|store| {- self.concurrent_state_mut(store).get_mut(id).unwrap().done = true;+ store.concurrent_state_mut().get_mut(id).unwrap().done = true;});}}@@ -2245,9 +2480,9 @@ impl Instance {};let consume = {let consume = consume_with_buffer.clone();- Box::new(move || {+ Box::new(move |instance| {let consume = consume.clone();- async move { consume(None).await }.boxed()+ async move { consume(instance, None).await }.boxed()})};@@ -2260,21 +2495,21 @@ impl Instance {cancel_waker: None,};}- WriteState::GuestReady { .. } => {- let future = consume();+ &WriteState::GuestReady { instance, .. } => {+ let future = consume(Some(instance));transmit.read = ReadState::HostReady {consume,guest_offset: 0,cancel: false,cancel_waker: None,};- self.pipe_from_guest(store.0, kind, id, future);+ self.0.pipe_from_guest(kind, id, future);}WriteState::HostReady { .. } => {let WriteState::HostReady { produce, .. } = mem::replace(&mut transmit.write,WriteState::HostReady {- produce: Box::new(|| unreachable!()),+ produce: Box::new(|_| unreachable!()),try_into: Box::new(|_| unreachable!()),guest_offset: 0,cancel: false,@@ -2287,7 +2522,7 @@ impl Instance {transmit.read = ReadState::HostToHost {accept: Box::new(move |input| {let consume = consume_with_buffer.clone();- async move { consume(Some(input.get_mut::<C::Item>())).await }.boxed()+ async move { consume(None, Some(input.get_mut::<C::Item>())).await }.boxed()}),buffer: Vec::new(),limit: 0,@@ -2297,14 +2532,14 @@ impl Instance {loop {if tls::get(|store| {anyhow::Ok(matches!(- self.concurrent_state_mut(store).get_mut(id)?.read,+ store.concurrent_state_mut().get_mut(id)?.read,ReadState::Dropped))})? {break Ok(());}- match produce().await? {+ match produce(None).await? {StreamResult::Completed | StreamResult::Cancelled => {}StreamResult::Dropped => break Ok(()),}@@ -2315,7 +2550,7 @@ impl Instance {}}.map(move |result| {- tls::get(|store| self.concurrent_state_mut(store).delete_transmit(id))?;+ tls::get(|store| store.concurrent_state_mut().delete_transmit(id))?;result});@@ -2323,444 +2558,259 @@ impl Instance {}WriteState::Dropped => {let reader = transmit.read_handle;- self.host_drop_reader(store.0, reader, kind).unwrap();+ self.0.host_drop_reader(reader, kind).unwrap();}}}+}- async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: WriteBuffer<T>>(- self,- token: StoreToken<D>,- id: TableId<TransmitState>,- pair: Arc<Mutex<Option<(P, B)>>>,- kind: TransmitKind,- ) -> Result<()> {- let (read, guest_offset) = tls::get(|store| {- let transmit = self.concurrent_state_mut(store).get_mut(id)?;+async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: WriteBuffer<T>>(+ instance: Option<Instance>,+ token: StoreToken<D>,+ id: TableId<TransmitState>,+ pair: Arc<Mutex<Option<(P, B)>>>,+ kind: TransmitKind,+) -> Result<()> {+ let (read, guest_offset) = tls::get(|store| {+ let transmit = store.concurrent_state_mut().get_mut(id)?;- let guest_offset = if let &WriteState::HostReady { guest_offset, .. } = &transmit.write- {- Some(guest_offset)- } else {- None- };+ let guest_offset = if let &WriteState::HostReady { guest_offset, .. } = &transmit.write {+ Some(guest_offset)+ } else {+ None+ };- anyhow::Ok((- mem::replace(&mut transmit.read, ReadState::Open),- guest_offset,- ))- })?;-- match read {- ReadState::GuestReady {- ty,- flat_abi,- options,- address,- count,- handle,- } => {- let guest_offset = guest_offset.unwrap();+ anyhow::Ok((+ mem::replace(&mut transmit.read, ReadState::Open),+ guest_offset,+ ))+ })?;++ match read {+ ReadState::GuestReady {+ ty,+ flat_abi,+ options,+ address,+ count,+ handle,+ } => {+ let guest_offset = guest_offset.unwrap();++ if let TransmitKind::Future = kind {+ tls::get(|store| {+ store.concurrent_state_mut().get_mut(id)?.done = true;+ anyhow::Ok(())+ })?;+ }- if let TransmitKind::Future = kind {- tls::get(|store| {- self.concurrent_state_mut(store).get_mut(id)?.done = true;- anyhow::Ok(())- })?;+ let old_remaining = pair.lock().unwrap().as_mut().unwrap().1.remaining().len();+ let accept = {+ let pair = pair.clone();+ move |mut store: StoreContextMut<D>| {+ lower::<T, B, D>(+ store.as_context_mut(),+ instance.unwrap(),+ &options,+ ty,+ address + (T::SIZE32 * guest_offset),+ count - guest_offset,+ &mut pair.lock().unwrap().as_mut().unwrap().1,+ )?;+ anyhow::Ok(())}+ };- let old_remaining = pair.lock().unwrap().as_mut().unwrap().1.remaining().len();- let accept = {- let pair = pair.clone();- move |mut store: StoreContextMut<D>| {- lower::<T, B, D>(- store.as_context_mut(),- self,- &options,- ty,- address + (T::SIZE32 * guest_offset),- count - guest_offset,- &mut pair.lock().unwrap().as_mut().unwrap().1,- )?;- anyhow::Ok(())- }+ if guest_offset < count {+ if T::MAY_REQUIRE_REALLOC {+ // For payloads which may require a realloc call, use a+ // oneshot::channel and background task. This is+ // necessary because calling the guest while there are+ // host embedder frames on the stack is unsound.+ let (tx, rx) = oneshot::channel();+ tls::get(move |store| {+ store+ .concurrent_state_mut()+ .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(+ move |store| {+ _ = tx.send(accept(token.as_context_mut(store))?);+ Ok(())+ },+ ))))+ });+ rx.await?+ } else {+ // Optimize flat payloads (i.e. those which do not+ // require calling the guest's realloc function) by+ // lowering directly instead of using a oneshot::channel+ // and background task.+ tls::get(|store| accept(token.as_context_mut(store)))?};+ }- if guest_offset < count {- if T::MAY_REQUIRE_REALLOC {- // For payloads which may require a realloc call, use a- // oneshot::channel and background task. This is- // necessary because calling the guest while there are- // host embedder frames on the stack is unsound.- let (tx, rx) = oneshot::channel();- tls::get(move |store| {- self.concurrent_state_mut(store).push_high_priority(- WorkItem::WorkerFunction(AlwaysMut::new(Box::new(- move |store, _| {- _ = tx.send(accept(token.as_context_mut(store))?);- Ok(())- },- ))),- )- });- rx.await?- } else {- // Optimize flat payloads (i.e. those which do not- // require calling the guest's realloc function) by- // lowering directly instead of using a oneshot::channel- // and background task.- tls::get(|store| accept(token.as_context_mut(store)))?- };- }+ tls::get(|store| {+ let count =+ old_remaining - pair.lock().unwrap().as_mut().unwrap().1.remaining().len();- tls::get(|store| {- let count =- old_remaining - pair.lock().unwrap().as_mut().unwrap().1.remaining().len();+ let transmit = store.concurrent_state_mut().get_mut(id)?;- let transmit = self.concurrent_state_mut(store).get_mut(id)?;+ let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {+ unreachable!();+ };- let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {- unreachable!();- };+ *guest_offset += count;- *guest_offset += count;+ transmit.read = ReadState::GuestReady {+ ty,+ flat_abi,+ options,+ address,+ count,+ handle,+ };- transmit.read = ReadState::GuestReady {- ty,- flat_abi,- options,- address,- count,- handle,- };+ anyhow::Ok(())+ })?;- anyhow::Ok(())- })?;+ Ok(())+ }- Ok(())+ ReadState::HostToHost {+ accept,+ mut buffer,+ limit,+ } => {+ let mut state = StreamResult::Completed;+ let mut position = 0;++ while !matches!(state, StreamResult::Dropped) && position < limit {+ let mut slice_buffer = SliceBuffer::new(buffer, position, limit);+ state = accept(&mut UntypedWriteBuffer::new(&mut slice_buffer)).await?;+ (buffer, position, _) = slice_buffer.into_parts();}- ReadState::HostToHost {- accept,- mut buffer,- limit,- } => {- let mut state = StreamResult::Completed;- let mut position = 0;-- while !matches!(state, StreamResult::Dropped) && position < limit {- let mut slice_buffer = SliceBuffer::new(buffer, position, limit);- state = accept(&mut UntypedWriteBuffer::new(&mut slice_buffer)).await?;- (buffer, position, _) = slice_buffer.into_parts();- }-- {- let (mine, mut buffer) = pair.lock().unwrap().take().unwrap();-- while !(matches!(state, StreamResult::Dropped) || buffer.remaining().is_empty())- {- state = accept(&mut UntypedWriteBuffer::new(&mut buffer)).await?;- }+ {+ let (mine, mut buffer) = pair.lock().unwrap().take().unwrap();- *pair.lock().unwrap() = Some((mine, buffer));+ while !(matches!(state, StreamResult::Dropped) || buffer.remaining().is_empty()) {+ state = accept(&mut UntypedWriteBuffer::new(&mut buffer)).await?;}- tls::get(|store| {- self.concurrent_state_mut(store).get_mut(id)?.read = match state {- StreamResult::Dropped => ReadState::Dropped,- StreamResult::Completed | StreamResult::Cancelled => {- ReadState::HostToHost {- accept,- buffer,- limit: 0,- }- }- };-- anyhow::Ok(())- })?;- Ok(())+ *pair.lock().unwrap() = Some((mine, buffer));}- _ => unreachable!(),- }- }-- fn pipe_from_guest(- self,- store: &mut dyn VMStore,- kind: TransmitKind,- id: TableId<TransmitState>,- future: Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,- ) {- let future = async move {- let stream_state = future.await?;tls::get(|store| {- let state = self.concurrent_state_mut(store);- let transmit = state.get_mut(id)?;- let ReadState::HostReady {- consume,- guest_offset,- ..- } = mem::replace(&mut transmit.read, ReadState::Open)- else {- unreachable!();- };- let code = return_code(kind, stream_state, guest_offset);- transmit.read = match stream_state {+ store.concurrent_state_mut().get_mut(id)?.read = match state {StreamResult::Dropped => ReadState::Dropped,- StreamResult::Completed | StreamResult::Cancelled => ReadState::HostReady {- consume,- guest_offset: 0,- cancel: false,- cancel_waker: None,+ StreamResult::Completed | StreamResult::Cancelled => ReadState::HostToHost {+ accept,+ buffer,+ limit: 0,},};- let WriteState::GuestReady { ty, handle, .. } =- mem::replace(&mut transmit.write, WriteState::Open)- else {- unreachable!();- };- state.send_write_result(ty, id, handle, code)?;- Ok(())- })- };-- self.concurrent_state_mut(store).push_future(future.boxed());- }- fn pipe_to_guest(- self,- store: &mut dyn VMStore,- kind: TransmitKind,- id: TableId<TransmitState>,- future: Pin<Box<dyn Future<Output = Result<StreamResult>> + Send + 'static>>,- ) {- let future = async move {- let stream_state = future.await?;- tls::get(|store| {- let state = self.concurrent_state_mut(store);- let transmit = state.get_mut(id)?;- let WriteState::HostReady {- produce,- try_into,- guest_offset,- ..- } = mem::replace(&mut transmit.write, WriteState::Open)- else {- unreachable!();- };- let code = return_code(kind, stream_state, guest_offset);- transmit.write = match stream_state {- StreamResult::Dropped => WriteState::Dropped,- StreamResult::Completed | StreamResult::Cancelled => WriteState::HostReady {- produce,- try_into,- guest_offset: 0,- cancel: false,- cancel_waker: None,- },- };- let ReadState::GuestReady { ty, handle, .. } =- mem::replace(&mut transmit.read, ReadState::Open)- else {- unreachable!();- };- state.send_read_result(ty, id, handle, code)?;- Ok(())- })- };+ anyhow::Ok(())+ })?;+ Ok(())+ }- self.concurrent_state_mut(store).push_future(future.boxed());+ _ => unreachable!(),}+}- /// Drop the read end of a stream or future read from the host.- fn host_drop_reader(+impl Instance {+ /// Handle a host- or guest-initiated write by delivering the item(s) to the+ /// `StreamConsumer` for the specified stream or future.+ fn consume(self,store: &mut dyn VMStore,- id: TableId<TransmitHandle>,kind: TransmitKind,- ) -> Result<()> {- let transmit_id = self.concurrent_state_mut(store).get_mut(id)?.state;- let state = self.concurrent_state_mut(store);- let transmit = state- .get_mut(transmit_id)- .with_context(|| format!("error closing reader {transmit_id:?}"))?;- log::trace!(- "host_drop_reader state {transmit_id:?}; read state {:?} write state {:?}",- transmit.read,- transmit.write- );-- transmit.read = ReadState::Dropped;-- // If the write end is already dropped, it should stay dropped,- // otherwise, it should be opened.- let new_state = if let WriteState::Dropped = &transmit.write {- WriteState::Dropped- } else {- WriteState::Open- };-- let write_handle = transmit.write_handle;-- match mem::replace(&mut transmit.write, new_state) {- // If a guest is waiting to write, notify it that the read end has- // been dropped.- WriteState::GuestReady { ty, handle, .. } => {- state.update_event(- write_handle.rep(),- match ty {- TransmitIndex::Future(ty) => Event::FutureWrite {- code: ReturnCode::Dropped(0),- pending: Some((ty, handle)),- },- TransmitIndex::Stream(ty) => Event::StreamWrite {- code: ReturnCode::Dropped(0),- pending: Some((ty, handle)),- },- },- )?;- }-- WriteState::HostReady { .. } => {}-- WriteState::Open => {- state.update_event(- write_handle.rep(),- match kind {- TransmitKind::Future => Event::FutureWrite {- code: ReturnCode::Dropped(0),- pending: None,- },- TransmitKind::Stream => Event::StreamWrite {- code: ReturnCode::Dropped(0),- pending: None,- },- },- )?;- }-- WriteState::Dropped => {- log::trace!("host_drop_reader delete {transmit_id:?}");- state.delete_transmit(transmit_id)?;- }- }- Ok(())- }-- /// Drop the write end of a stream or future read from the host.- fn host_drop_writer<U>(- self,- store: StoreContextMut<U>,- id: TableId<TransmitHandle>,- on_drop_open: Option<fn() -> Result<()>>,- ) -> Result<()> {- let transmit_id = self.concurrent_state_mut(store.0).get_mut(id)?.state;- let transmit = self- .concurrent_state_mut(store.0)- .get_mut(transmit_id)- .with_context(|| format!("error closing writer {transmit_id:?}"))?;- log::trace!(- "host_drop_writer state {transmit_id:?}; write state {:?} read state {:?}",- transmit.read,- transmit.write- );-- // Existing queued transmits must be updated with information for the impending writer closure- match &mut transmit.write {- WriteState::GuestReady { .. } => {- unreachable!("can't call `host_drop_writer` on a guest-owned writer");- }- WriteState::HostReady { .. } => {}- v @ WriteState::Open => {- if let (Some(on_drop_open), false) = (- on_drop_open,- transmit.done || matches!(transmit.read, ReadState::Dropped),- ) {- on_drop_open()?;- } else {- *v = WriteState::Dropped;- }- }- WriteState::Dropped => unreachable!("write state is already dropped"),- }-- let transmit = self.concurrent_state_mut(store.0).get_mut(transmit_id)?;-- // If the existing read state is dropped, then there's nothing to read- // and we can keep it that way.- //- // If the read state was any other state, then we must set the new state to open- // to indicate that there *is* data to be read- let new_state = if let ReadState::Dropped = &transmit.read {- ReadState::Dropped- } else {- ReadState::Open+ transmit_id: TableId<TransmitState>,+ consume: PollStream,+ guest_offset: usize,+ cancel: bool,+ ) -> Result<ReturnCode> {+ let mut future = consume(Some(self));+ store.concurrent_state_mut().get_mut(transmit_id)?.read = ReadState::HostReady {+ consume,+ guest_offset,+ cancel,+ cancel_waker: None,};+ let poll = tls::set(store, || {+ future+ .as_mut()+ .poll(&mut Context::from_waker(&Waker::noop()))+ });- let read_handle = transmit.read_handle;-- // Swap in the new read state- match mem::replace(&mut transmit.read, new_state) {- // If the guest was ready to read, then we cannot drop the reader (or writer);- // we must deliver the event, and update the state associated with the handle to- // represent that a read must be performed- ReadState::GuestReady { ty, handle, .. } => {- // Ensure the final read of the guest is queued, with appropriate closure indicator- self.concurrent_state_mut(store.0).update_event(- read_handle.rep(),- match ty {- TransmitIndex::Future(ty) => Event::FutureRead {- code: ReturnCode::Dropped(0),- pending: Some((ty, handle)),- },- TransmitIndex::Stream(ty) => Event::StreamRead {- code: ReturnCode::Dropped(0),- pending: Some((ty, handle)),- },- },- )?;+ Ok(match poll {+ Poll::Ready(state) => {+ let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;+ let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {+ unreachable!();+ };+ let code = return_code(kind, state?, mem::replace(guest_offset, 0));+ transmit.write = WriteState::Open;+ code+ }+ Poll::Pending => {+ store.pipe_from_guest(kind, transmit_id, future);+ ReturnCode::Blocked}+ })+ }- ReadState::HostReady { .. } | ReadState::HostToHost { .. } => {}+ /// Handle a host- or guest-initiated read by polling the `StreamProducer`+ /// for the specified stream or future for items.+ fn produce(+ self,+ store: &mut dyn VMStore,+ kind: TransmitKind,+ transmit_id: TableId<TransmitState>,+ produce: PollStream,+ try_into: TryInto,+ guest_offset: usize,+ cancel: bool,+ ) -> Result<ReturnCode> {+ let mut future = produce(Some(self));+ store.concurrent_state_mut().get_mut(transmit_id)?.write = WriteState::HostReady {+ produce,+ try_into,+ guest_offset,+ cancel,+ cancel_waker: None,+ };+ let poll = tls::set(store, || {+ future+ .as_mut()+ .poll(&mut Context::from_waker(&Waker::noop()))+ });- // If the read state is open, then there are no registered readers of the stream/future- ReadState::Open => {- self.concurrent_state_mut(store.0).update_event(- read_handle.rep(),- match on_drop_open {- Some(_) => Event::FutureRead {- code: ReturnCode::Dropped(0),- pending: None,- },- None => Event::StreamRead {- code: ReturnCode::Dropped(0),- pending: None,- },- },- )?;+ Ok(match poll {+ Poll::Ready(state) => {+ let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;+ let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {+ unreachable!();+ };+ let code = return_code(kind, state?, mem::replace(guest_offset, 0));+ transmit.read = ReadState::Open;+ code}-- // If the read state was already dropped, then we can remove the transmit state completely- // (both writer and reader have been dropped)- ReadState::Dropped => {- log::trace!("host_drop_writer delete {transmit_id:?}");- self.concurrent_state_mut(store.0)- .delete_transmit(transmit_id)?;+ Poll::Pending => {+ store.pipe_to_guest(kind, transmit_id, future);+ ReturnCode::Blocked}- }- Ok(())+ })}/// Drop the writable end of the specified stream or future from the guest.- pub(super) fn guest_drop_writable<T>(+ pub(super) fn guest_drop_writable(self,- store: StoreContextMut<T>,+ store: &mut StoreOpaque,ty: TransmitIndex,writer: u32,) -> Result<()> {- let table = self.id().get_mut(store.0).table_for_transmit(ty);+ let table = self.id().get_mut(store).table_for_transmit(ty);let transmit_rep = match ty {TransmitIndex::Future(ty) => table.future_remove_writable(ty, writer)?,TransmitIndex::Stream(ty) => table.stream_remove_writable(ty, writer)?,@@ -2769,9 +2819,8 @@ impl Instance {let id = TableId::<TransmitHandle>::new(transmit_rep);log::trace!("guest_drop_writable: drop writer {id:?}");match ty {- TransmitIndex::Stream(_) => self.host_drop_writer(store, id, None),- TransmitIndex::Future(_) => self.host_drop_writer(- store,+ TransmitIndex::Stream(_) => store.host_drop_writer(id, None),+ TransmitIndex::Future(_) => store.host_drop_writer(id,Some(|| {Err(anyhow!(@@ -2984,7 +3033,7 @@ impl Instance {*state = TransmitLocalState::Busy;let transmit_handle = TableId::<TransmitHandle>::new(rep);- let concurrent_state = self.concurrent_state_mut(store.0);+ let concurrent_state = store.0.concurrent_state_mut();let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;let transmit = concurrent_state.get_mut(transmit_id)?;log::trace!(@@ -3006,6 +3055,7 @@ impl Instance {let transmit = me.get_mut(transmit_id)?;assert!(matches!(&transmit.write, WriteState::Open));transmit.write = WriteState::GuestReady {+ instance: self,ty,flat_abi,options,@@ -3078,7 +3128,7 @@ impl Instance {let item_size = payload(ty, types).map(|ty| usize::try_from(types.canonical_abi(&ty).size32).unwrap()).unwrap_or(0);- let concurrent_state = instance.concurrent_state_mut();+ let concurrent_state = store.0.concurrent_state_mut();if read_complete {let count = u32::try_from(count).unwrap();let total = if let Some(Event::StreamRead {@@ -3171,47 +3221,6 @@ impl Instance {Ok(result)}- /// Handle a host- or guest-initiated write by delivering the item(s) to the- /// `StreamConsumer` for the specified stream or future.- fn consume(- self,- store: &mut dyn VMStore,- kind: TransmitKind,- transmit_id: TableId<TransmitState>,- consume: PollStream,- guest_offset: usize,- cancel: bool,- ) -> Result<ReturnCode> {- let mut future = consume();- self.concurrent_state_mut(store).get_mut(transmit_id)?.read = ReadState::HostReady {- consume,- guest_offset,- cancel,- cancel_waker: None,- };- let poll = self.set_tls(store, || {- future- .as_mut()- .poll(&mut Context::from_waker(&Waker::noop()))- });-- Ok(match poll {- Poll::Ready(state) => {- let transmit = self.concurrent_state_mut(store).get_mut(transmit_id)?;- let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {- unreachable!();- };- let code = return_code(kind, state?, mem::replace(guest_offset, 0));- transmit.write = WriteState::Open;- code- }- Poll::Pending => {- self.pipe_from_guest(store, kind, transmit_id, future);- ReturnCode::Blocked- }- })- }-/// Read from the specified stream or future from the guest.pub(super) fn guest_read<T: 'static>(self,@@ -3238,7 +3247,7 @@ impl Instance {*state = TransmitLocalState::Busy;let transmit_handle = TableId::<TransmitHandle>::new(rep);- let concurrent_state = self.concurrent_state_mut(store.0);+ let concurrent_state = store.0.concurrent_state_mut();let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;let transmit = concurrent_state.get_mut(transmit_id)?;log::trace!(@@ -3272,6 +3281,7 @@ impl Instance {let mut result = match mem::replace(&mut transmit.write, new_state) {WriteState::GuestReady {+ instance: _,ty: write_ty,flat_abi: write_flat_abi,options: write_options,@@ -3315,7 +3325,7 @@ impl Instance {let item_size = payload(ty, types).map(|ty| usize::try_from(types.canonical_abi(&ty).size32).unwrap()).unwrap_or(0);- let concurrent_state = instance.concurrent_state_mut();+ let concurrent_state = store.0.concurrent_state_mut();if write_complete {let count = u32::try_from(count).unwrap();@@ -3342,6 +3352,7 @@ impl Instance {if write_buffer_remaining {let transmit = concurrent_state.get_mut(transmit_id)?;transmit.write = WriteState::GuestReady {+ instance: self,ty: write_ty,flat_abi: write_flat_abi,options: write_options,@@ -3408,57 +3419,14 @@ impl Instance {Ok(result)}- /// Handle a host- or guest-initiated read by polling the `StreamProducer`- /// for the specified stream or future for items.- fn produce(- self,- store: &mut dyn VMStore,- kind: TransmitKind,- transmit_id: TableId<TransmitState>,- produce: PollStream,- try_into: TryInto,- guest_offset: usize,- cancel: bool,- ) -> Result<ReturnCode> {- let mut future = produce();- self.concurrent_state_mut(store).get_mut(transmit_id)?.write = WriteState::HostReady {- produce,- try_into,- guest_offset,- cancel,- cancel_waker: None,- };- let poll = self.set_tls(store, || {- future- .as_mut()- .poll(&mut Context::from_waker(&Waker::noop()))- });-- Ok(match poll {- Poll::Ready(state) => {- let transmit = self.concurrent_state_mut(store).get_mut(transmit_id)?;- let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {- unreachable!();- };- let code = return_code(kind, state?, mem::replace(guest_offset, 0));- transmit.read = ReadState::Open;- code- }- Poll::Pending => {- self.pipe_to_guest(store, kind, transmit_id, future);- ReturnCode::Blocked- }- })- }-fn wait_for_write(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,handle: TableId<TransmitHandle>,) -> Result<ReturnCode> {let waitable = Waitable::Transmit(handle);- self.wait_for_event(store, waitable)?;- let event = waitable.take_event(self.concurrent_state_mut(store))?;+ store.wait_for_event(waitable)?;+ let event = waitable.take_event(store.concurrent_state_mut())?;if let Some(event @ (Event::StreamWrite { code, .. } | Event::FutureWrite { code, .. })) =event{@@ -3472,11 +3440,11 @@ impl Instance {/// Cancel a pending stream or future write.fn cancel_write(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,transmit_id: TableId<TransmitState>,async_: bool,) -> Result<ReturnCode> {- let state = self.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();let transmit = state.get_mut(transmit_id)?;log::trace!("host_cancel_write state {transmit_id:?}; write state {:?} read state {:?}",@@ -3511,8 +3479,8 @@ impl Instance {if async_ {ReturnCode::Blocked} else {- let handle = self- .concurrent_state_mut(store)+ let handle = store+ .concurrent_state_mut().get_mut(transmit_id)?.write_handle;self.wait_for_write(store, handle)?@@ -3521,7 +3489,7 @@ impl Instance {ReturnCode::Cancelled(0)};- let transmit = self.concurrent_state_mut(store).get_mut(transmit_id)?;+ let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;match &transmit.write {WriteState::GuestReady { .. } => {@@ -3538,12 +3506,12 @@ impl Instance {fn wait_for_read(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,handle: TableId<TransmitHandle>,) -> Result<ReturnCode> {let waitable = Waitable::Transmit(handle);- self.wait_for_event(store, waitable)?;- let event = waitable.take_event(self.concurrent_state_mut(store))?;+ store.wait_for_event(waitable)?;+ let event = waitable.take_event(store.concurrent_state_mut())?;if let Some(event @ (Event::StreamRead { code, .. } | Event::FutureRead { code, .. })) =event{@@ -3557,11 +3525,11 @@ impl Instance {/// Cancel a pending stream or future read.fn cancel_read(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,transmit_id: TableId<TransmitState>,async_: bool,) -> Result<ReturnCode> {- let state = self.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();let transmit = state.get_mut(transmit_id)?;log::trace!("host_cancel_read state {transmit_id:?}; read state {:?} write state {:?}",@@ -3596,8 +3564,8 @@ impl Instance {if async_ {ReturnCode::Blocked} else {- let handle = self- .concurrent_state_mut(store)+ let handle = store+ .concurrent_state_mut().get_mut(transmit_id)?.read_handle;self.wait_for_read(store, handle)?@@ -3606,7 +3574,7 @@ impl Instance {ReturnCode::Cancelled(0)};- let transmit = self.concurrent_state_mut(store).get_mut(transmit_id)?;+ let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;match &transmit.read {ReadState::GuestReady { .. } => {@@ -3626,7 +3594,7 @@ impl Instance {/// Cancel a pending write for the specified stream or future from the guest.fn guest_cancel_write(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,ty: TransmitIndex,async_: bool,writer: u32,@@ -3644,7 +3612,7 @@ impl Instance {}TransmitLocalState::Busy => {}}- let transmit_id = self.concurrent_state_mut(store).get_mut(id)?.state;+ let transmit_id = store.concurrent_state_mut().get_mut(id)?.state;let code = self.cancel_write(store, transmit_id, async_)?;if !matches!(code, ReturnCode::Blocked) {let state =@@ -3660,7 +3628,7 @@ impl Instance {/// Cancel a pending read for the specified stream or future from the guest.fn guest_cancel_read(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,ty: TransmitIndex,async_: bool,reader: u32,@@ -3678,7 +3646,7 @@ impl Instance {}TransmitLocalState::Busy => {}}- let transmit_id = self.concurrent_state_mut(store).get_mut(id)?.state;+ let transmit_id = store.concurrent_state_mut().get_mut(id)?.state;let code = self.cancel_read(store, transmit_id, async_)?;if !matches!(code, ReturnCode::Blocked) {let state =@@ -3694,7 +3662,7 @@ impl Instance {/// Drop the readable end of the specified stream or future from the guest.fn guest_drop_readable(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,ty: TransmitIndex,reader: u32,) -> Result<()> {@@ -3709,7 +3677,7 @@ impl Instance {};let id = TableId::<TransmitHandle>::new(rep);log::trace!("guest_drop_readable: drop reader {id:?}");- self.host_drop_reader(store, id, kind)+ store.host_drop_reader(id, kind)}/// Create a new error context for the given component.@@ -3741,7 +3709,7 @@ impl Instance {.to_str_from_memory(options.memory(store))?.to_string(),};- let state = self.concurrent_state_mut(store);+ let state = store.concurrent_state_mut();let table_id = state.push(err_ctx)?;let global_ref_count_idx =TypeComponentGlobalErrorContextTableIndex::from_u32(table_id.rep());@@ -3782,7 +3750,7 @@ impl Instance {.table_for_error_context(ty).error_context_rep(err_ctx_handle)?;- let state = self.concurrent_state_mut(store.0);+ let state = store.0.concurrent_state_mut();// Get the state associated with the error contextlet ErrorContextState { debug_msg } =state.get_mut(TableId::<ErrorContextState>::new(handle_table_id_rep))?;@@ -3809,7 +3777,7 @@ impl Instance {/// Implements the `future.cancel-read` intrinsic.pub(crate) fn future_cancel_read(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeFutureTableIndex,async_: bool,@@ -3823,7 +3791,7 @@ impl Instance {/// Implements the `future.cancel-write` intrinsic.pub(crate) fn future_cancel_write(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeFutureTableIndex,async_: bool,@@ -3837,7 +3805,7 @@ impl Instance {/// Implements the `stream.cancel-read` intrinsic.pub(crate) fn stream_cancel_read(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeStreamTableIndex,async_: bool,@@ -3851,7 +3819,7 @@ impl Instance {/// Implements the `stream.cancel-write` intrinsic.pub(crate) fn stream_cancel_write(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeStreamTableIndex,async_: bool,@@ -3865,7 +3833,7 @@ impl Instance {/// Implements the `future.drop-readable` intrinsic.pub(crate) fn future_drop_readable(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeFutureTableIndex,reader: u32,@@ -3877,7 +3845,7 @@ impl Instance {/// Implements the `stream.drop-readable` intrinsic.pub(crate) fn stream_drop_readable(self,- store: &mut dyn VMStore,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeStreamTableIndex,reader: u32,@@ -3885,42 +3853,14 @@ impl Instance {self.id().get(store).check_may_leave(caller)?;self.guest_drop_readable(store, TransmitIndex::Stream(ty), reader)}-}--impl ComponentInstance {- fn table_for_transmit(self: Pin<&mut Self>, ty: TransmitIndex) -> &mut HandleTable {- let (tables, types) = self.guest_tables();- let runtime_instance = match ty {- TransmitIndex::Stream(ty) => types[ty].instance,- TransmitIndex::Future(ty) => types[ty].instance,- };- &mut tables[runtime_instance]- }-- fn table_for_error_context(- self: Pin<&mut Self>,- ty: TypeComponentLocalErrorContextTableIndex,- ) -> &mut HandleTable {- let (tables, types) = self.guest_tables();- let runtime_instance = types[ty].instance;- &mut tables[runtime_instance]- }-- fn get_mut_by_index(- self: Pin<&mut Self>,- ty: TransmitIndex,- index: u32,- ) -> Result<(u32, &mut TransmitLocalState)> {- get_mut_by_index_from(self.table_for_transmit(ty), ty, index)- }/// Allocate a new future or stream and grant ownership of both the read and/// write ends to the (sub-)component instance to which the specified/// `TransmitIndex` belongs.- fn guest_new(mut self: Pin<&mut Self>, ty: TransmitIndex) -> Result<ResourcePair> {- let (write, read) = self.as_mut().concurrent_state_mut().new_transmit()?;+ fn guest_new(self, store: &mut StoreOpaque, ty: TransmitIndex) -> Result<ResourcePair> {+ let (write, read) = store.concurrent_state_mut().new_transmit()?;- let table = self.as_mut().table_for_transmit(ty);+ let table = self.id().get_mut(store).table_for_transmit(ty);let (read_handle, write_handle) = match ty {TransmitIndex::Future(ty) => (table.future_insert_read(ty, read.rep())?,@@ -3932,7 +3872,7 @@ impl ComponentInstance {),};- let state = self.as_mut().concurrent_state_mut();+ let state = store.concurrent_state_mut();state.get_mut(read)?.common.handle = Some(read_handle);state.get_mut(write)?.common.handle = Some(write_handle);@@ -3944,20 +3884,22 @@ impl ComponentInstance {/// Drop the specified error context.pub(crate) fn error_context_drop(- mut self: Pin<&mut Self>,+ self,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeComponentLocalErrorContextTableIndex,error_context: u32,) -> Result<()> {- self.check_may_leave(caller)?;+ let instance = self.id().get_mut(store);+ instance.check_may_leave(caller)?;- let local_handle_table = self.as_mut().table_for_error_context(ty);+ let local_handle_table = instance.table_for_error_context(ty);let rep = local_handle_table.error_context_drop(error_context)?;let global_ref_count_idx = TypeComponentGlobalErrorContextTableIndex::from_u32(rep);- let state = self.concurrent_state_mut();+ let state = store.concurrent_state_mut();let GlobalErrorContextRefCount(global_ref_count) = state.global_error_context_ref_counts.get_mut(&global_ref_count_idx)@@ -3982,12 +3924,14 @@ impl ComponentInstance {/// Transfer ownership of the specified stream or future read end from one/// guest to another.fn guest_transfer(- mut self: Pin<&mut Self>,+ self,+ store: &mut StoreOpaque,src_idx: u32,src: TransmitIndex,dst: TransmitIndex,) -> Result<u32> {- let src_table = self.as_mut().table_for_transmit(src);+ let mut instance = self.id().get_mut(store);+ let src_table = instance.as_mut().table_for_transmit(src);let (rep, is_done) = match src {TransmitIndex::Future(idx) => src_table.future_remove_readable(idx, src_idx)?,TransmitIndex::Stream(idx) => src_table.stream_remove_readable(idx, src_idx)?,@@ -3995,12 +3939,13 @@ impl ComponentInstance {if is_done {bail!("cannot lift after being notified that the writable end dropped");}- let dst_table = self.as_mut().table_for_transmit(dst);+ let dst_table = instance.table_for_transmit(dst);let handle = match dst {TransmitIndex::Future(idx) => dst_table.future_insert_read(idx, rep),TransmitIndex::Stream(idx) => dst_table.stream_insert_read(idx, rep),}?;- self.concurrent_state_mut()+ store+ .concurrent_state_mut().get_mut(TableId::<TransmitHandle>::new(rep))?.common.handle = Some(handle);@@ -4009,33 +3954,37 @@ impl ComponentInstance {/// Implements the `future.new` intrinsic.pub(crate) fn future_new(- self: Pin<&mut Self>,+ self,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeFutureTableIndex,) -> Result<ResourcePair> {- self.check_may_leave(caller)?;- self.guest_new(TransmitIndex::Future(ty))+ self.id().get(store).check_may_leave(caller)?;+ self.guest_new(store, TransmitIndex::Future(ty))}/// Implements the `stream.new` intrinsic.pub(crate) fn stream_new(- self: Pin<&mut Self>,+ self,+ store: &mut StoreOpaque,caller: RuntimeComponentInstanceIndex,ty: TypeStreamTableIndex,) -> Result<ResourcePair> {- self.check_may_leave(caller)?;- self.guest_new(TransmitIndex::Stream(ty))+ self.id().get(store).check_may_leave(caller)?;+ self.guest_new(store, TransmitIndex::Stream(ty))}/// Transfer ownership of the specified future read end from one guest to/// another.pub(crate) fn future_transfer(- self: Pin<&mut Self>,+ self,+ store: &mut StoreOpaque,src_idx: u32,src: TypeFutureTableIndex,dst: TypeFutureTableIndex,) -> Result<u32> {self.guest_transfer(+ store,src_idx,TransmitIndex::Future(src),TransmitIndex::Future(dst),@@ -4045,12 +3994,14 @@ impl ComponentInstance {/// Transfer ownership of the specified stream read end from one guest to/// another.pub(crate) fn stream_transfer(- self: Pin<&mut Self>,+ self,+ store: &mut StoreOpaque,src_idx: u32,src: TypeStreamTableIndex,dst: TypeStreamTableIndex,) -> Result<u32> {self.guest_transfer(+ store,src_idx,TransmitIndex::Stream(src),TransmitIndex::Stream(dst),@@ -4059,24 +4010,25 @@ impl ComponentInstance {/// Copy the specified error context from one component to another.pub(crate) fn error_context_transfer(- mut self: Pin<&mut Self>,+ self,+ store: &mut StoreOpaque,src_idx: u32,src: TypeComponentLocalErrorContextTableIndex,dst: TypeComponentLocalErrorContextTableIndex,) -> Result<u32> {- let rep = self+ let mut instance = self.id().get_mut(store);+ let rep = instance.as_mut().table_for_error_context(src).error_context_rep(src_idx)?;- let dst_idx = self- .as_mut()+ let dst_idx = instance.table_for_error_context(dst).error_context_insert(rep)?;// Update the global (cross-subcomponent) count for error contexts// as the new component has essentially created a new reference that will// be dropped/handled independently- let global_ref_count = self+ let global_ref_count = store.concurrent_state_mut().global_error_context_ref_counts.get_mut(&TypeComponentGlobalErrorContextTableIndex::from_u32(rep))@@ -4087,6 +4039,34 @@ impl ComponentInstance {}}+impl ComponentInstance {+ fn table_for_transmit(self: Pin<&mut Self>, ty: TransmitIndex) -> &mut HandleTable {+ let (tables, types) = self.guest_tables();+ let runtime_instance = match ty {+ TransmitIndex::Stream(ty) => types[ty].instance,+ TransmitIndex::Future(ty) => types[ty].instance,+ };+ &mut tables[runtime_instance]+ }++ fn table_for_error_context(+ self: Pin<&mut Self>,+ ty: TypeComponentLocalErrorContextTableIndex,+ ) -> &mut HandleTable {+ let (tables, types) = self.guest_tables();+ let runtime_instance = types[ty].instance;+ &mut tables[runtime_instance]+ }++ fn get_mut_by_index(+ self: Pin<&mut Self>,+ ty: TransmitIndex,+ index: u32,+ ) -> Result<(u32, &mut TransmitLocalState)> {+ get_mut_by_index_from(self.table_for_transmit(ty), ty, index)+ }+}+impl ConcurrentState {fn send_write_result(&mut self,diff --git a/crates/wasmtime/src/runtime/component/concurrent_disabled.rs b/crates/wasmtime/src/runtime/component/concurrent_disabled.rsindex 1866761807..5fb313801f 100644--- a/crates/wasmtime/src/runtime/component/concurrent_disabled.rs+++ b/crates/wasmtime/src/runtime/component/concurrent_disabled.rs@@ -1,6 +1,6 @@use crate::Uninhabited;+use crate::component::Val;use crate::component::func::{ComponentType, LiftContext, LowerContext};-use crate::component::{Instance, Val};use crate::runtime::vm::VMStore;use anyhow::{Result, anyhow};use core::future::Future;@@ -9,6 +9,9 @@ use core::pin::pin;use core::task::{Context, Poll, Waker};use wasmtime_environ::component::{InterfaceType, RuntimeComponentInstanceIndex};+#[derive(Default)]+pub struct ConcurrentState;+fn should_have_failed_validation<T>(what: &str) -> Result<T> {// This should be unreachable; if we trap here, it indicates a// bug in Wasmtime rather than in the guest.@@ -18,17 +21,14 @@ fn should_have_failed_validation<T>(what: &str) -> Result<T> {))}-impl Instance {- pub(crate) fn poll_and_block<R: Send + Sync + 'static>(- self,- _store: &mut dyn VMStore,- future: impl Future<Output = Result<R>> + Send + 'static,- _caller_instance: RuntimeComponentInstanceIndex,- ) -> Result<R> {- match pin!(future).poll(&mut Context::from_waker(Waker::noop())) {- Poll::Ready(result) => result,- Poll::Pending => should_have_failed_validation("async lowered import"),- }+pub(crate) fn poll_and_block<R: Send + Sync + 'static>(+ _store: &mut dyn VMStore,+ future: impl Future<Output = Result<R>> + Send + 'static,+ _caller_instance: RuntimeComponentInstanceIndex,+) -> Result<R> {+ match pin!(future).poll(&mut Context::from_waker(Waker::noop())) {+ Poll::Ready(result) => result,+ Poll::Pending => should_have_failed_validation("async lowered import"),}}diff --git a/crates/wasmtime/src/runtime/component/func.rs b/crates/wasmtime/src/runtime/component/func.rsindex d41ba92abd..497134c45a 100644--- a/crates/wasmtime/src/runtime/component/func.rs+++ b/crates/wasmtime/src/runtime/component/func.rs@@ -276,12 +276,12 @@ impl Func {params: &[Val],results: &mut [Val],) -> Result<()> {- let mut store = store.as_context_mut();+ let store = store.as_context_mut();#[cfg(feature = "component-model-async")]{- self.instance- .run_concurrent_trap_on_idle(&mut store, async |store| {+ store+ .run_concurrent_trap_on_idle(async |store| {self.call_concurrent_dynamic(store, params, results, false).await.map(drop)@@ -294,6 +294,7 @@ impl Func {store.0.async_support(),"cannot use `call_async` without enabling async support in the config");+ let mut store = store;store.on_fiber(|store| self.call_impl(store, params, results)).await?diff --git a/crates/wasmtime/src/runtime/component/func/host.rs b/crates/wasmtime/src/runtime/component/func/host.rsindex fc6eedffdf..dbfe00277d 100644--- a/crates/wasmtime/src/runtime/component/func/host.rs+++ b/crates/wasmtime/src/runtime/component/func/host.rs@@ -1,3 +1,4 @@+use crate::component::concurrent;#[cfg(feature = "component-model-async")]use crate::component::concurrent::{Accessor, Status};use crate::component::func::{LiftContext, LowerContext, Options};@@ -42,7 +43,7 @@ enum HostResult<T> {impl HostFunc {fn from_canonical<T, F, P, R>(func: F) -> Arc<HostFunc>where- F: Fn(StoreContextMut<'_, T>, Instance, P) -> HostResult<R> + Send + Sync + 'static,+ F: Fn(StoreContextMut<'_, T>, P) -> HostResult<R> + Send + Sync + 'static,P: ComponentNamedList + Lift + 'static,R: ComponentNamedList + Lower + 'static,T: 'static,@@ -62,7 +63,7 @@ impl HostFunc {P: ComponentNamedList + Lift + 'static,R: ComponentNamedList + Lower + 'static,{- Self::from_canonical::<T, _, _, _>(move |store, _, params| {+ Self::from_canonical::<T, _, _, _>(move |store, params| {HostResult::Done(func(store, params))})}@@ -79,10 +80,10 @@ impl HostFunc {R: ComponentNamedList + Lower + 'static,{let func = Arc::new(func);- Self::from_canonical::<T, _, _, _>(move |store, instance, params| {+ Self::from_canonical::<T, _, _, _>(move |store, params| {let func = func.clone();HostResult::Future(Box::pin(- instance.wrap_call(store, move |accessor| func(accessor, params)),+ store.wrap_call(move |accessor| func(accessor, params)),))})}@@ -96,7 +97,7 @@ impl HostFunc {storage_len: usize,) -> boolwhere- F: Fn(StoreContextMut<'_, T>, Instance, P) -> HostResult<R> + Send + Sync + 'static,+ F: Fn(StoreContextMut<'_, T>, P) -> HostResult<R> + Send + Sync + 'static,P: ComponentNamedList + Lift,R: ComponentNamedList + Lower + 'static,T: 'static,@@ -110,7 +111,7 @@ impl HostFunc {TypeFuncIndex::from_u32(ty),OptionsIndex::from_u32(options),NonNull::slice_from_raw_parts(storage, storage_len).as_mut(),- move |store, instance, args| (*data.as_ptr())(store, instance, args),+ move |store, args| (*data.as_ptr())(store, args),)})}@@ -120,7 +121,6 @@ impl HostFunc {whereF: Fn(StoreContextMut<'_, T>,- Instance,Vec<Val>,usize,) -> Pin<Box<dyn Future<Output = Result<Vec<Val>>> + Send + 'static>>@@ -143,13 +143,11 @@ impl HostFunc {whereF: Fn(StoreContextMut<'_, T>, &[Val], &mut [Val]) -> Result<()> + Send + Sync + 'static,{- Self::new_dynamic_canonical::<T, _>(- move |store, _, mut params_and_results, result_start| {- let (params, results) = params_and_results.split_at_mut(result_start);- let result = func(store, params, results).map(move |()| params_and_results);- Box::pin(async move { result })- },- )+ Self::new_dynamic_canonical::<T, _>(move |store, mut params_and_results, result_start| {+ let (params, results) = params_and_results.split_at_mut(result_start);+ let result = func(store, params, results).map(move |()| params_and_results);+ Box::pin(async move { result })+ })}#[cfg(feature = "component-model-async")]@@ -166,18 +164,16 @@ impl HostFunc {+ 'static,{let func = Arc::new(func);- Self::new_dynamic_canonical::<T, _>(- move |store, instance, mut params_and_results, result_start| {- let func = func.clone();- Box::pin(instance.wrap_call(store, move |accessor| {- Box::pin(async move {- let (params, results) = params_and_results.split_at_mut(result_start);- func(accessor, params, results).await?;- Ok(params_and_results)- })- }))- },- )+ Self::new_dynamic_canonical::<T, _>(move |store, mut params_and_results, result_start| {+ let func = func.clone();+ Box::pin(store.wrap_call(move |accessor| {+ Box::pin(async move {+ let (params, results) = params_and_results.split_at_mut(result_start);+ func(accessor, params, results).await?;+ Ok(params_and_results)+ })+ }))+ })}pub fn typecheck(&self, ty: TypeFuncIndex, types: &InstanceType<'_>) -> Result<()> {@@ -235,7 +231,7 @@ unsafe fn call_host<T, Params, Return, F>(closure: F,) -> Result<()>where- F: Fn(StoreContextMut<'_, T>, Instance, Params) -> HostResult<Return> + Send + Sync + 'static,+ F: Fn(StoreContextMut<'_, T>, Params) -> HostResult<Return> + Send + Sync + 'static,Params: Lift,Return: Lower + 'static,{@@ -286,11 +282,11 @@ where}};- let host_result = closure(store.as_context_mut(), instance, params);+ let host_result = closure(store.as_context_mut(), params);let mut lower_result = {let types = types.clone();- move |store: StoreContextMut<T>, instance: Instance, ret: Return| {+ move |store: StoreContextMut<T>, ret: Return| {unsafe {flags.set_may_leave(false);}@@ -305,7 +301,7 @@ where};let task = match host_result {HostResult::Done(result) => {- lower_result(store.as_context_mut(), instance, result?)?;+ lower_result(store.as_context_mut(), result?)?;None}#[cfg(feature = "component-model-async")]@@ -340,11 +336,11 @@ wherelift.enter_call();let params = storage.lift_params(&mut lift, param_tys)?;- let ret = match closure(store.as_context_mut(), instance, params) {+ let ret = match closure(store.as_context_mut(), params) {HostResult::Done(result) => result?,#[cfg(feature = "component-model-async")]HostResult::Future(future) => {- instance.poll_and_block(store.0, future, caller_instance)?+ concurrent::poll_and_block(store.0, future, caller_instance)?}};@@ -704,7 +700,6 @@ unsafe fn call_host_dynamic<T, F>(whereF: Fn(StoreContextMut<'_, T>,- Instance,Vec<Val>,usize,) -> Pin<Box<dyn Future<Output = Result<Vec<Val>>> + Send + 'static>>@@ -768,17 +763,12 @@ wherevalidate_inbounds_dynamic(&result_tys.abi, lower.as_slice_mut(), &retptr)?};- let future = closure(- store.as_context_mut(),- instance,- params_and_results,- result_start,- );+ let future = closure(store.as_context_mut(), params_and_results, result_start);let task = instance.first_poll(store, future, caller_instance, {let types = types.clone();let result_tys = func_ty.results;- move |store: StoreContextMut<T>, instance: Instance, result_vals: Vec<Val>| {+ move |store: StoreContextMut<T>, result_vals: Vec<Val>| {let result_tys = &types[result_tys];let result_vals = &result_vals[result_start..];assert_eq!(result_vals.len(), result_tys.types.len());@@ -820,13 +810,8 @@ where);}} else {- let future = closure(- store.as_context_mut(),- instance,- params_and_results,- result_start,- );- let result_vals = instance.poll_and_block(store.0, future, caller_instance)?;+ let future = closure(store.as_context_mut(), params_and_results, result_start);+ let result_vals = concurrent::poll_and_block(store.0, future, caller_instance)?;let result_vals = &result_vals[result_start..];unsafe {@@ -930,7 +915,6 @@ extern "C" fn dynamic_entrypoint<T, F>(whereF: Fn(StoreContextMut<'_, T>,- Instance,Vec<Val>,usize,) -> Pin<Box<dyn Future<Output = Result<Vec<Val>>> + Send + 'static>>@@ -948,9 +932,7 @@ whereTypeFuncIndex::from_u32(ty),OptionsIndex::from_u32(options),NonNull::slice_from_raw_parts(storage, storage_len).as_mut(),- move |store, instance, params, results| {- (*data.as_ptr())(store, instance, params, results)- },+ move |store, params, results| (*data.as_ptr())(store, params, results),)})}diff --git a/crates/wasmtime/src/runtime/component/func/options.rs b/crates/wasmtime/src/runtime/component/func/options.rsindex b17a71d1b9..8b32a53de7 100644--- a/crates/wasmtime/src/runtime/component/func/options.rs+++ b/crates/wasmtime/src/runtime/component/func/options.rs@@ -1,3 +1,4 @@+use crate::component::concurrent::ConcurrentState;use crate::component::matching::InstanceType;use crate::component::resources::{HostResourceData, HostResourceIndex, HostResourceTables};use crate::component::{Instance, ResourceType};@@ -477,6 +478,9 @@ pub struct LiftContext<'a> {host_resource_data: &'a mut HostResourceData,calls: &'a mut CallContexts,++ #[cfg_attr(not(feature = "component-model-async"), allow(unused))]+ concurrent_state: &'a mut ConcurrentState,}#[doc(hidden)]@@ -497,8 +501,8 @@ impl<'a> LiftContext<'a> {let memory = options.memory.map(|_| options.memory(unsafe { &*(store as *const StoreOpaque) }));- let (calls, host_table, host_resource_data, instance) =- store.component_resource_state_with_instance(instance_handle);+ let (calls, host_table, host_resource_data, instance, concurrent_state) =+ store.component_resource_state_with_instance_and_concurrent_state(instance_handle);let (component, instance) = instance.component_and_self();LiftContext {@@ -510,6 +514,7 @@ impl<'a> LiftContext<'a> {calls,host_table,host_resource_data,+ concurrent_state,}}@@ -539,6 +544,11 @@ impl<'a> LiftContext<'a> {self.instance_handle}+ #[cfg(feature = "component-model-async")]+ pub(crate) fn concurrent_state_mut(&mut self) -> &mut ConcurrentState {+ self.concurrent_state+ }+/// Lifts an `own` resource from the guest at the `idx` specified into its/// representation.///diff --git a/crates/wasmtime/src/runtime/component/func/typed.rs b/crates/wasmtime/src/runtime/component/func/typed.rsindex 1ebf013d79..788ee52247 100644--- a/crates/wasmtime/src/runtime/component/func/typed.rs+++ b/crates/wasmtime/src/runtime/component/func/typed.rs@@ -232,11 +232,10 @@ where};let result = concurrent::queue_call(wrapper.store.as_context_mut(), prepared)?;- self.func- .instance- .run_concurrent_trap_on_idle(wrapper.store.as_context_mut(), async |_| {- Ok(result.await?.0)- })+ wrapper+ .store+ .as_context_mut()+ .run_concurrent_trap_on_idle(async |_| Ok(result.await?.0)).await?}#[cfg(not(feature = "component-model-async"))]diff --git a/crates/wasmtime/src/runtime/component/linker.rs b/crates/wasmtime/src/runtime/component/linker.rsindex 0f364cd5b4..903de4d924 100644--- a/crates/wasmtime/src/runtime/component/linker.rs+++ b/crates/wasmtime/src/runtime/component/linker.rs@@ -823,18 +823,8 @@ impl<T: 'static> LinkerInstance<'_, T> {let dtor = dtor.clone();cx.as_context_mut().block_on(move |mut store| {Box::pin(async move {- // NOTE: We currently pass `None` as the `instance`- // parameter to `Accessor::new` because we don't have ready- // access to it, meaning `dtor` will panic if it tries to- // use `Accessor::instance`. We could plumb that through- // from the `wasmtime-cranelift`-generated code, but we plan- // to remove `Accessor::instance` once all instances in a- // store share the same concurrent state, at which point we- // won't need it anyway.- let accessor = &Accessor::new(- crate::store::StoreToken::new(store.as_context_mut()),- None,- );+ let accessor =+ &Accessor::new(crate::store::StoreToken::new(store.as_context_mut()));let mut future = std::pin::pin!(dtor(accessor, param));std::future::poll_fn(|cx| {crate::component::concurrent::tls::set(store.0, || {diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rsindex 46ef308069..762c031706 100644--- a/crates/wasmtime/src/runtime/component/store.rs+++ b/crates/wasmtime/src/runtime/component/store.rs@@ -37,16 +37,9 @@ impl ComponentStoreData {pub(crate) fn drop_fibers_and_futures(store: &mut dyn VMStore) {let mut fibers = Vec::new();let mut futures = Vec::new();- for (_, instance) in store.store_data_mut().components.instances.iter_mut() {- let Some(instance) = instance.as_mut() else {- continue;- };-- instance- .get_mut()- .concurrent_state_mut()- .take_fibers_and_futures(&mut fibers, &mut futures);- }+ store+ .concurrent_state_mut()+ .take_fibers_and_futures(&mut fibers, &mut futures);for mut fiber in fibers {fiber.dispose(store);@@ -54,6 +47,24 @@ impl ComponentStoreData {crate::component::concurrent::tls::set(store, move || drop(futures));}++ #[cfg(feature = "component-model-async")]+ pub(crate) fn assert_guest_tables_empty(&mut self) {+ for (_, instance) in self.instances.iter_mut() {+ let Some(instance) = instance.as_mut() else {+ continue;+ };++ assert!(+ instance+ .get_mut()+ .guest_tables()+ .0+ .iter()+ .all(|(_, table)| table.is_empty())+ );+ }+ }}impl StoreData {diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 47fccb0684..ca54b0f084 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -81,7 +81,7 @@ use crate::RootSet;use crate::ThrownException;#[cfg(feature = "component-model-async")]use crate::component::ComponentStoreData;-#[cfg(feature = "component-model-async")]+#[cfg(feature = "component-model")]use crate::component::concurrent;#[cfg(feature = "async")]use crate::fiber;@@ -480,9 +480,8 @@ pub struct StoreOpaque {component_calls: vm::component::CallContexts,#[cfg(feature = "component-model")]host_resource_data: crate::component::HostResourceData,-- #[cfg(feature = "component-model-async")]- concurrent_async_state: concurrent::AsyncState,+ #[cfg(feature = "component-model")]+ concurrent_state: concurrent::ConcurrentState,/// State related to the executor of wasm code.///@@ -695,8 +694,8 @@ impl<T> Store<T> {#[cfg(feature = "component-model")]host_resource_data: Default::default(),executor: Executor::new(engine),- #[cfg(feature = "component-model-async")]- concurrent_async_state: Default::default(),+ #[cfg(feature = "component-model")]+ concurrent_state: Default::default(),};let mut inner = Box::new(StoreInner {inner,@@ -2301,14 +2300,34 @@ at https://bytecodealliance.org/security.)}+ #[cfg(feature = "component-model")]+ pub(crate) fn component_resource_state_with_instance_and_concurrent_state(+ &mut self,+ instance: crate::component::Instance,+ ) -> (+ &mut vm::component::CallContexts,+ &mut vm::component::HandleTable,+ &mut crate::component::HostResourceData,+ Pin<&mut vm::component::ComponentInstance>,+ &mut concurrent::ConcurrentState,+ ) {+ (+ &mut self.component_calls,+ &mut self.component_host_table,+ &mut self.host_resource_data,+ instance.id().from_data_get_mut(&mut self.store_data),+ &mut self.concurrent_state,+ )+ }+#[cfg(feature = "async")]pub(crate) fn fiber_async_state_mut(&mut self) -> &mut fiber::AsyncState {&mut self.async_state}#[cfg(feature = "component-model-async")]- pub(crate) fn concurrent_async_state_mut(&mut self) -> &mut concurrent::AsyncState {- &mut self.concurrent_async_state+ pub(crate) fn concurrent_state_mut(&mut self) -> &mut concurrent::ConcurrentState {+ &mut self.concurrent_state}#[cfg(feature = "async")]diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rsindex a8fc564d0e..ec9c658cc2 100644--- a/crates/wasmtime/src/runtime/vm/component.rs+++ b/crates/wasmtime/src/runtime/vm/component.rs@@ -42,9 +42,6 @@ pub use self::handle_table::{TransmitLocalState, Waitable};pub use self::resources::CallContext;pub use self::resources::{CallContexts, ResourceTables, TypedResource, TypedResourceIndex};-#[cfg(feature = "component-model-async")]-use crate::component::concurrent;-/// Runtime representation of a component instance and all state necessary for/// the instance itself.///@@ -88,11 +85,6 @@ pub struct ComponentInstance {/// vice-versa.instance_handle_tables: PrimaryMap<RuntimeComponentInstanceIndex, HandleTable>,- /// State related to async for this component, e.g. futures, streams, tasks,- /// etc.- #[cfg(feature = "component-model-async")]- concurrent_state: concurrent::ConcurrentState,-/// What all compile-time-identified core instances are mapped to within the/// `Store` that this component belongs to.instances: PrimaryMap<RuntimeInstanceIndex, InstanceId>,@@ -314,8 +306,6 @@ impl ComponentInstance {imports: imports.clone(),store: VMStoreRawPtr(store),post_return_arg: None,- #[cfg(feature = "component-model-async")]- concurrent_state: concurrent::ConcurrentState::new(component),vmctx: OwnedVMContext::new(),});unsafe {@@ -884,13 +874,6 @@ impl ComponentInstance {unsafe { &mut self.get_unchecked_mut().post_return_arg }}- #[cfg(feature = "component-model-async")]- pub(crate) fn concurrent_state_mut(self: Pin<&mut Self>) -> &mut concurrent::ConcurrentState {- // SAFETY: we've chosen the `Pin` guarantee of `Self` to not apply to- // the map returned.- unsafe { &mut self.get_unchecked_mut().concurrent_state }- }-pub(crate) fn check_may_leave(&self,instance: RuntimeComponentInstanceIndex,diff --git a/crates/wasmtime/src/runtime/vm/component/libcalls.rs b/crates/wasmtime/src/runtime/vm/component/libcalls.rsindex 4346c56a12..5996ba3d3b 100644--- a/crates/wasmtime/src/runtime/vm/component/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/component/libcalls.rs@@ -669,11 +669,11 @@ fn trap(_store: &mut dyn VMStore, _instance: Instance, code: u8) -> Result<()> {#[cfg(feature = "component-model-async")]fn backpressure_set(store: &mut dyn VMStore,- instance: Instance,+ _instance: Instance,caller_instance: u32,enabled: u32,) -> Result<()> {- instance.concurrent_state_mut(store).backpressure_modify(+ store.concurrent_state_mut().backpressure_modify(RuntimeComponentInstanceIndex::from_u32(caller_instance),|_| Some(if enabled != 0 { 1 } else { 0 }),)@@ -682,11 +682,11 @@ fn backpressure_set(#[cfg(feature = "component-model-async")]fn backpressure_modify(store: &mut dyn VMStore,- instance: Instance,+ _instance: Instance,caller_instance: u32,increment: u8,) -> Result<()> {- instance.concurrent_state_mut(store).backpressure_modify(+ store.concurrent_state_mut().backpressure_modify(RuntimeComponentInstanceIndex::from_u32(caller_instance),|old| {if increment != 0 {@@ -731,10 +731,10 @@ fn waitable_set_new(instance: Instance,caller_instance: u32,) -> Result<u32> {- instance- .id()- .get_mut(store)- .waitable_set_new(RuntimeComponentInstanceIndex::from_u32(caller_instance))+ instance.waitable_set_new(+ store,+ RuntimeComponentInstanceIndex::from_u32(caller_instance),+ )}#[cfg(feature = "component-model-async")]@@ -780,7 +780,8 @@ fn waitable_set_drop(caller_instance: u32,set: u32,) -> Result<()> {- instance.id().get_mut(store).waitable_set_drop(+ instance.waitable_set_drop(+ store,RuntimeComponentInstanceIndex::from_u32(caller_instance),set,)@@ -794,7 +795,8 @@ fn waitable_join(waitable: u32,set: u32,) -> Result<()> {- instance.id().get_mut(store).waitable_join(+ instance.waitable_join(+ store,RuntimeComponentInstanceIndex::from_u32(caller_instance),waitable,set,@@ -822,7 +824,8 @@ fn subtask_drop(caller_instance: u32,task_id: u32,) -> Result<()> {- instance.id().get_mut(store).subtask_drop(+ instance.subtask_drop(+ store,RuntimeComponentInstanceIndex::from_u32(caller_instance),task_id,)@@ -930,7 +933,8 @@ fn future_transfer(src_table: u32,dst_table: u32,) -> Result<u32> {- instance.id().get_mut(store).future_transfer(+ instance.future_transfer(+ store,src_idx,TypeFutureTableIndex::from_u32(src_table),TypeFutureTableIndex::from_u32(dst_table),@@ -945,7 +949,8 @@ fn stream_transfer(src_table: u32,dst_table: u32,) -> Result<u32> {- instance.id().get_mut(store).stream_transfer(+ instance.stream_transfer(+ store,src_idx,TypeStreamTableIndex::from_u32(src_table),TypeStreamTableIndex::from_u32(dst_table),@@ -962,10 +967,7 @@ fn error_context_transfer() -> Result<u32> {let src_table = TypeComponentLocalErrorContextTableIndex::from_u32(src_table);let dst_table = TypeComponentLocalErrorContextTableIndex::from_u32(dst_table);- instance- .id()- .get_mut(store)- .error_context_transfer(src_idx, src_table, dst_table)+ instance.error_context_transfer(store, src_idx, src_table, dst_table)}#[cfg(feature = "component-model-async")]@@ -986,7 +988,8 @@ fn future_new(caller_instance: u32,ty: u32,) -> Result<ResourcePair> {- instance.id().get_mut(store).future_new(+ instance.future_new(+ store,RuntimeComponentInstanceIndex::from_u32(caller_instance),TypeFutureTableIndex::from_u32(ty),)@@ -1107,7 +1110,8 @@ fn stream_new(caller_instance: u32,ty: u32,) -> Result<ResourcePair> {- instance.id().get_mut(store).stream_new(+ instance.stream_new(+ store,RuntimeComponentInstanceIndex::from_u32(caller_instance),TypeStreamTableIndex::from_u32(ty),)@@ -1325,7 +1329,8 @@ fn error_context_drop(ty: u32,err_ctx_handle: u32,) -> Result<()> {- instance.id().get_mut(store).error_context_drop(+ instance.error_context_drop(+ store,RuntimeComponentInstanceIndex::from_u32(caller_instance),TypeComponentLocalErrorContextTableIndex::from_u32(ty),err_ctx_handle,diff --git a/src/commands/run.rs b/src/commands/run.rsindex 2e658b725f..4ecf26d7f9 100644--- a/src/commands/run.rs+++ b/src/commands/run.rs@@ -588,10 +588,8 @@ impl RunCommand {if self.run.common.wasi.p3.unwrap_or(crate::common::P3_DEFAULT) {if let Ok(command) = wasmtime_wasi::p3::bindings::Command::new(&mut *store, &instance) {result = Some(- instance- .run_concurrent(&mut *store, async |store| {- command.wasi_cli_run().call_run(store).await- })+ store+ .run_concurrent(async |store| command.wasi_cli_run().call_run(store).await).await?,);}diff --git a/src/commands/serve.rs b/src/commands/serve.rsindex 907a83eb60..7467dc4ebf 100644--- a/src/commands/serve.rs+++ b/src/commands/serve.rs@@ -389,8 +389,8 @@ impl ServeCommand {let instance = linker.instantiate_pre(&component)?;#[cfg(feature = "component-model-async")]- let instance = match wasmtime_wasi_http::p3::bindings::ProxyIndices::new(&instance) {- Ok(indices) => ProxyPre::P3(indices, instance),+ let instance = match wasmtime_wasi_http::p3::bindings::ProxyPre::new(instance.clone()) {+ Ok(pre) => ProxyPre::P3(pre),Err(_) => ProxyPre::P2(p2::ProxyPre::new(instance)?),};#[cfg(not(feature = "component-model-async"))]@@ -734,10 +734,7 @@ struct ProxyHandlerInner {enum ProxyPre {P2(p2::ProxyPre<Host>),#[cfg(feature = "component-model-async")]- P3(- wasmtime_wasi_http::p3::bindings::ProxyIndices,- wasmtime::component::InstancePre<Host>,- ),+ P3(wasmtime_wasi_http::p3::bindings::ProxyPre<Host>),}impl ProxyPre {@@ -745,11 +742,7 @@ impl ProxyPre {Ok(match self {ProxyPre::P2(pre) => Proxy::P2(pre.instantiate_async(store).await?),#[cfg(feature = "component-model-async")]- ProxyPre::P3(indices, pre) => {- let instance = pre.instantiate_async(&mut *store).await?;- let proxy = indices.load(&mut *store, &instance)?;- Proxy::P3(proxy, instance)- }+ ProxyPre::P3(pre) => Proxy::P3(pre.instantiate_async(store).await?),})}}@@ -757,10 +750,7 @@ impl ProxyPre {enum Proxy {P2(p2::Proxy),#[cfg(feature = "component-model-async")]- P3(- wasmtime_wasi_http::p3::bindings::Proxy,- wasmtime::component::Instance,- ),+ P3(wasmtime_wasi_http::p3::bindings::Proxy),}impl ProxyHandlerInner {@@ -852,14 +842,14 @@ async fn handle_request(Ok(result.map(|body| body.map_err(|e| e.into()).boxed()))}#[cfg(feature = "component-model-async")]- Proxy::P3(proxy, instance) => {+ Proxy::P3(proxy) => {use wasmtime_wasi_http::p3::bindings::http::types::{ErrorCode, Request};let (tx, rx) = tokio::sync::oneshot::channel();tokio::task::spawn(async move {- let guest_result = instance- .run_concurrent(&mut store, async move |store| {+ let guest_result = store+ .run_concurrent(async move |store| {let (req, body) = req.into_parts();let body = body.map_err(ErrorCode::from_hyper_request_error);let req = http::Request::from_parts(req, body);diff --git a/tests/all/component_model/bindgen.rs b/tests/all/component_model/bindgen.rsindex e537f6d040..387f1c2f94 100644--- a/tests/all/component_model/bindgen.rs+++ b/tests/all/component_model/bindgen.rs@@ -121,10 +121,9 @@ mod no_imports_concurrent {let linker = Linker::new(&engine);let mut store = Store::new(&engine, ());- let instance = linker.instantiate_async(&mut store, &component).await?;- let no_imports = NoImports::new(&mut store, &instance)?;- instance- .run_concurrent(&mut store, async move |accessor| {+ let no_imports = NoImports::instantiate_async(&mut store, &component, &linker).await?;+ store+ .run_concurrent(async move |accessor| {let mut futures = FuturesUnordered::new();futures.push(no_imports.call_bar(accessor).boxed());futures.push(no_imports.foo().call_foo(accessor).boxed());@@ -286,12 +285,9 @@ mod one_import_concurrent {let mut linker = Linker::new(&engine);foo::add_to_linker::<_, MyImports>(&mut linker, |x| x)?;let mut store = Store::new(&engine, MyImports::default());- let instance = linker.instantiate_async(&mut store, &component).await?;- let no_imports = NoImports::new(&mut store, &instance)?;- instance- .run_concurrent(&mut store, async move |accessor| {- no_imports.call_bar(accessor).await- })+ let no_imports = NoImports::instantiate_async(&mut store, &component, &linker).await?;+ store+ .run_concurrent(async move |accessor| no_imports.call_bar(accessor).await).await??;assert!(store.data().hit);Ok(())diff --git a/tests/all/component_model/func.rs b/tests/all/component_model/func.rsindex 158cc9adfa..22e280a617 100644--- a/tests/all/component_model/func.rs+++ b/tests/all/component_model/func.rs@@ -920,8 +920,8 @@ async fn async_reentrance() -> Result<()> {.await?;let func = instance.get_typed_func::<(u32,), (u32,)>(&mut store, "export")?;let message = "cannot enter component instance";- match instance- .run_concurrent(&mut store, async move |accessor| {+ match store+ .run_concurrent(async move |accessor| {anyhow::Ok(func.call_concurrent(accessor, (42,)).await?.0)}).await@@ -1050,8 +1050,8 @@ async fn task_return_trap(component: &str, substring: &str) -> Result<()> {.await?;let func = instance.get_typed_func::<(), ()>(&mut store, "foo")?;- match instance- .run_concurrent(&mut store, async move |accessor| {+ match store+ .run_concurrent(async move |accessor| {anyhow::Ok(func.call_concurrent(accessor, ()).await?.0)}).await@@ -1249,8 +1249,8 @@ async fn test_many_parameters(dynamic: bool, concurrent: bool) -> Result<()> {let mut results = vec![Val::Bool(false)];if concurrent {- instance- .run_concurrent(&mut store, async |store| {+ store+ .run_concurrent(async |store| {func.call_concurrent(store, &input, &mut results).await?;anyhow::Ok(())})@@ -1294,8 +1294,8 @@ async fn test_many_parameters(dynamic: bool, concurrent: bool) -> Result<()> {), ((Vec<u8>, u32),)>(&mut store, "many-param")?;if concurrent {- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {anyhow::Ok(func.call_concurrent(accessor, input).await?.0)}).await??@@ -1683,8 +1683,8 @@ async fn test_many_results(dynamic: bool, concurrent: bool) -> Result<()> {let mut results = vec![Val::Bool(false)];if concurrent {- instance- .run_concurrent(&mut store, async |store| {+ store+ .run_concurrent(async |store| {func.call_concurrent(store, &[], &mut results).await?;anyhow::Ok(())})@@ -1773,8 +1773,8 @@ async fn test_many_results(dynamic: bool, concurrent: bool) -> Result<()> {),)>(&mut store, "many-results")?;if concurrent {- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {anyhow::Ok(func.call_concurrent(accessor, ()).await?.0)}).await??diff --git a/tests/all/component_model/import.rs b/tests/all/component_model/import.rsindex 09e5881f53..9917ad7b20 100644--- a/tests/all/component_model/import.rs+++ b/tests/all/component_model/import.rs@@ -833,8 +833,8 @@ async fn test_stack_and_heap_args_and_rets(concurrent: bool) -> Result<()> {let run = instance.get_typed_func::<(), ()>(&mut store, "run")?;if concurrent {- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {anyhow::Ok(run.call_concurrent(accessor, ()).await?.0)}).await??;@@ -957,8 +957,8 @@ async fn test_stack_and_heap_args_and_rets(concurrent: bool) -> Result<()> {let run = instance.get_func(&mut store, "run").unwrap();if concurrent {- instance- .run_concurrent(&mut store, async |store| {+ store+ .run_concurrent(async |store| {run.call_concurrent(store, &[], &mut []).await?;anyhow::Ok(())})diff --git a/tests/all/pulley.rs b/tests/all/pulley.rsindex eea59a97f1..870d621c34 100644--- a/tests/all/pulley.rs+++ b/tests/all/pulley.rs@@ -452,29 +452,29 @@ async fn pulley_provenance_test_async_components() -> Result<()> {let instance = linker.instantiate_async(&mut store, &component).await?;let run = instance.get_typed_func::<(), ()>(&mut store, "run-stackless")?;- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {anyhow::Ok(run.call_concurrent(accessor, ()).await?.0)}).await??;let run = instance.get_typed_func::<(), ()>(&mut store, "run-stackful")?;- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {anyhow::Ok(run.call_concurrent(accessor, ()).await?.0)}).await??;let run = instance.get_typed_func::<(), ()>(&mut store, "run-stackless-stackless")?;- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {anyhow::Ok(run.call_concurrent(accessor, ()).await?.0)}).await??;let run = instance.get_typed_func::<(), ()>(&mut store, "run-stackful-stackful")?;- instance- .run_concurrent(&mut store, async move |accessor| {+ store+ .run_concurrent(async move |accessor| {anyhow::Ok(run.call_concurrent(accessor, ()).await?.0)}).await??;