Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

support WASIp3 instance reuse in `wasmtime serve` (#11807)

Joel Dicedcd654469e78
diff --git a/Cargo.toml b/Cargo.toml
index e4e2c29333..aa7cbc1d91 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -87,6 +87,7 @@ tokio = { workspace = true, optional = true, features = [ "signal", "macros" ] }
hyper = { workspace = true, optional = true }
http = { workspace = true, optional = true }
http-body-util = { workspace = true, optional = true }
+futures = { workspace = true, optional = true }
[target.'cfg(unix)'.dependencies]
rustix = { workspace = true, features = ["mm", "process"] }
@@ -504,6 +505,7 @@ component-model-async = [
"component-model",
"wasmtime-wasi?/p3",
"wasmtime-wasi-http?/p3",
+ "dep:futures",
]
# This feature, when enabled, will statically compile out all logging statements
@@ -549,6 +551,7 @@ debug = ["wasmtime-cli-flags/debug", "wasmtime/debug"]
# for more information on each subcommand.
serve = [
"wasi-http",
+ "wasmtime-wasi-http/component-model-async",
"component-model",
"dep:http-body-util",
"dep:http",
diff --git a/crates/test-programs/src/bin/p2_cli_serve_hello_world.rs b/crates/test-programs/src/bin/p2_cli_serve_hello_world.rs
new file mode 100644
index 0000000000..cd7c789a68
--- /dev/null
+++ b/crates/test-programs/src/bin/p2_cli_serve_hello_world.rs
@@ -0,0 +1,27 @@
+use test_programs::proxy;
+use test_programs::wasi::http::types::{
+ Fields, IncomingRequest, OutgoingBody, OutgoingResponse, ResponseOutparam,
+};
+
+struct T;
+
+proxy::export!(T);
+
+impl proxy::exports::wasi::http::incoming_handler::Guest for T {
+ fn handle(_: IncomingRequest, outparam: ResponseOutparam) {
+ let fields = Fields::new();
+ let resp = OutgoingResponse::new(fields);
+ let body = resp.body().expect("outgoing response");
+
+ ResponseOutparam::set(outparam, Ok(resp));
+
+ let out = body.write().expect("outgoing stream");
+ out.blocking_write_and_flush(b"Hello, WASI!")
+ .expect("writing response");
+
+ drop(out);
+ OutgoingBody::finish(body, None).expect("outgoing-body.finish");
+ }
+}
+
+fn main() {}
diff --git a/crates/test-programs/src/bin/p3_cli_serve_sleep.rs b/crates/test-programs/src/bin/p3_cli_serve_sleep.rs
new file mode 100644
index 0000000000..eed10b49bb
--- /dev/null
+++ b/crates/test-programs/src/bin/p3_cli_serve_sleep.rs
@@ -0,0 +1,18 @@
+use test_programs::p3::proxy;
+use test_programs::p3::wasi::clocks::monotonic_clock;
+use test_programs::p3::wasi::http::types::{ErrorCode, Request, Response};
+
+struct T;
+
+proxy::export!(T);
+
+impl proxy::exports::wasi::http::handler::Guest for T {
+ async fn handle(_request: Request) -> Result<Response, ErrorCode> {
+ monotonic_clock::wait_for(u64::MAX).await;
+ unreachable!()
+ }
+}
+
+fn main() {
+ unreachable!()
+}
diff --git a/crates/wasi-http/Cargo.toml b/crates/wasi-http/Cargo.toml
index 0b360ac16c..8e46aa4e51 100644
--- a/crates/wasi-http/Cargo.toml
+++ b/crates/wasi-http/Cargo.toml
@@ -18,6 +18,7 @@ all-features = true
default = ["default-send-request"]
default-send-request = ["dep:tokio-rustls", "dep:rustls", "dep:webpki-roots"]
p3 = ["wasmtime-wasi/p3", "dep:tokio-util"]
+component-model-async = ["futures/alloc", "wasmtime/component-model-async"]
[dependencies]
anyhow = { workspace = true }
diff --git a/crates/wasi-http/src/handler.rs b/crates/wasi-http/src/handler.rs
new file mode 100644
index 0000000000..0f6335eae9
--- /dev/null
+++ b/crates/wasi-http/src/handler.rs
@@ -0,0 +1,634 @@
+//! Provides utilities useful for dispatching incoming HTTP requests
+//! `wasi:http/handler` guest instances.
+
+#[cfg(feature = "p3")]
+use crate::p3;
+use anyhow::{Result, anyhow};
+use futures::stream::{FuturesUnordered, StreamExt};
+use std::collections::VecDeque;
+use std::collections::btree_map::{BTreeMap, Entry};
+use std::future;
+use std::pin::{Pin, pin};
+use std::sync::{
+ Arc, Mutex,
+ atomic::{
+ AtomicBool, AtomicU64, AtomicUsize,
+ Ordering::{Relaxed, SeqCst},
+ },
+};
+use std::task::Poll;
+use std::time::{Duration, Instant};
+use tokio::sync::Notify;
+use wasmtime::AsContextMut;
+use wasmtime::component::Accessor;
+use wasmtime::{Store, StoreContextMut};
+
+/// Alternative p2 bindings generated with `exports: { default: async | store }`
+/// so we can use `TypedFunc::call_concurrent` with both p2 and p3 instances.
+pub mod p2 {
+ #[expect(missing_docs, reason = "bindgen-generated code")]
+ pub mod bindings {
+ wasmtime::component::bindgen!({
+ path: "wit",
+ world: "wasi:http/proxy",
+ imports: { default: tracing },
+ exports: { default: async | store },
+ require_store_data_send: true,
+ with: {
+ // http is in this crate
+ "wasi:http": crate::bindings::http,
+ // Upstream package dependencies
+ "wasi:io": wasmtime_wasi::p2::bindings::io,
+ }
+ });
+
+ pub use wasi::*;
+ }
+}
+
+/// Represents either a `wasi:http/[email protected]` or
+/// `wasi:http/[email protected]` pre-instance.
+pub enum ProxyPre<T: 'static> {
+ /// A `wasi:http/[email protected]` pre-instance.
+ P2(p2::bindings::ProxyPre<T>),
+ /// A `wasi:http/[email protected]` pre-instance.
+ #[cfg(feature = "p3")]
+ P3(p3::bindings::ProxyPre<T>),
+}
+
+impl<T: 'static> ProxyPre<T> {
+ async fn instantiate_async(&self, store: impl AsContextMut<Data = T>) -> Result<Proxy>
+ where
+ T: Send,
+ {
+ Ok(match self {
+ Self::P2(pre) => Proxy::P2(pre.instantiate_async(store).await?),
+ #[cfg(feature = "p3")]
+ Self::P3(pre) => Proxy::P3(pre.instantiate_async(store).await?),
+ })
+ }
+}
+
+/// Represents either a `wasi:http/[email protected]` or
+/// `wasi:http/[email protected]` instance.
+pub enum Proxy {
+ /// A `wasi:http/[email protected]` instance.
+ P2(p2::bindings::Proxy),
+ /// A `wasi:http/[email protected]` instance.
+ #[cfg(feature = "p3")]
+ P3(p3::bindings::Proxy),
+}
+
+/// Represents a task to run using a `wasi:http/[email protected]` or
+/// `wasi:http/[email protected]` instance.
+pub type TaskFn<T> = Box<
+ dyn for<'a> FnOnce(&'a Accessor<T>, &'a Proxy) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>
+ + Send,
+>;
+
+/// Async MPMC channel where each item is delivered to at most one consumer.
+struct Queue<T> {
+ queue: Mutex<VecDeque<T>>,
+ notify: Notify,
+}
+
+impl<T> Default for Queue<T> {
+ fn default() -> Self {
+ Self {
+ queue: Default::default(),
+ notify: Default::default(),
+ }
+ }
+}
+
+impl<T> Queue<T> {
+ fn is_empty(&self) -> bool {
+ self.queue.lock().unwrap().is_empty()
+ }
+
+ fn push(&self, item: T) {
+ self.queue.lock().unwrap().push_back(item);
+ self.notify.notify_one();
+ }
+
+ fn try_pop(&self) -> Option<T> {
+ self.queue.lock().unwrap().pop_front()
+ }
+
+ async fn pop(&self) -> T {
+ // This code comes from the Unbound MPMC Channel example in [the
+ // `tokio::sync::Notify`
+ // docs](https://docs.rs/tokio/latest/tokio/sync/struct.Notify.html).
+
+ let mut notified = pin!(self.notify.notified());
+
+ loop {
+ notified.as_mut().enable();
+ if let Some(item) = self.try_pop() {
+ return item;
+ }
+ notified.as_mut().await;
+ notified.set(self.notify.notified());
+ }
+ }
+}
+
+/// Bundles a [`Store`] with a callback to write a profile (if configured).
+pub struct StoreBundle<T: 'static> {
+ /// The [`Store`] to use to handle requests.
+ pub store: Store<T>,
+ /// Callback to write a profile (if enabled) once all requests have been
+ /// handled.
+ pub write_profile: Box<dyn FnOnce(StoreContextMut<T>) + Send>,
+}
+
+/// Represents the application-specific state of a web server.
+pub trait HandlerState: 'static + Sync + Send {
+ /// The type of the associated data for [`Store`]s created using
+ /// [`new_store`].
+ type StoreData: Send;
+
+ /// Create a new [`Store`] for handling one or more requests.
+ ///
+ /// The `req_id` parameter is the value passed in the call to
+ /// [`ProxyHandler::spawn`] that created the worker to which the new `Store`
+ /// will belong. See that function's documentation for details.
+ fn new_store(&self, req_id: Option<u64>) -> Result<StoreBundle<Self::StoreData>>;
+
+ /// Maximum time allowed to handle a request.
+ ///
+ /// In practice, a guest may be allowed to run up to 2x this time in the
+ /// case of instance reuse to avoid penalizing concurrent requests being
+ /// handled by the same instance.
+ fn request_timeout(&self) -> Duration;
+
+ /// Maximum time to keep an idle instance around before dropping it.
+ fn idle_instance_timeout(&self) -> Duration;
+
+ /// Maximum number of requests to handle using a single instance before
+ /// dropping it.
+ fn max_instance_reuse_count(&self) -> usize;
+
+ /// Maximum number of requests to handle concurrently using a single
+ /// instance.
+ fn max_instance_concurrent_reuse_count(&self) -> usize;
+
+ /// Called when a worker exits with an error.
+ fn handle_worker_error(&self, error: anyhow::Error);
+}
+
+struct ProxyHandlerInner<S: HandlerState> {
+ state: S,
+ instance_pre: ProxyPre<S::StoreData>,
+ next_id: AtomicU64,
+ task_queue: Queue<TaskFn<S::StoreData>>,
+ worker_count: AtomicUsize,
+}
+
+/// Helper utility to track the start times of tasks accepted by a worker.
+///
+/// This is used to ensure that timeouts are enforced even when the
+/// `StoreContextMut::run_concurrent` event loop is unable to make progress due
+/// to the guest either busy looping or being blocked on a synchronous call to a
+/// host function which has exclusive access to the `Store`.
+#[derive(Default)]
+struct StartTimes(BTreeMap<Instant, usize>);
+
+impl StartTimes {
+ fn add(&mut self, time: Instant) {
+ *self.0.entry(time).or_insert(0) += 1;
+ }
+
+ fn remove(&mut self, time: Instant) {
+ let Entry::Occupied(mut entry) = self.0.entry(time) else {
+ unreachable!()
+ };
+ match *entry.get() {
+ 0 => unreachable!(),
+ 1 => {
+ entry.remove();
+ }
+ _ => {
+ *entry.get_mut() -= 1;
+ }
+ }
+ }
+
+ fn earliest(&self) -> Option<Instant> {
+ self.0.first_key_value().map(|(&k, _)| k)
+ }
+}
+
+struct Worker<S>
+where
+ S: HandlerState,
+{
+ handler: ProxyHandler<S>,
+ available: bool,
+}
+
+impl<S> Worker<S>
+where
+ S: HandlerState,
+{
+ fn set_available(&mut self, available: bool) {
+ if available != self.available {
+ self.available = available;
+ if available {
+ self.handler.0.worker_count.fetch_add(1, Relaxed);
+ } else {
+ // Here we use `SeqCst` to ensure the load/store is ordered
+ // correctly with respect to the `Queue::is_empty` check we do
+ // below.
+ let count = self.handler.0.worker_count.fetch_sub(1, SeqCst);
+ // This addresses what would otherwise be a race condition in
+ // `ProxyHandler::spawn` where it only starts a worker if the
+ // available worker count is zero. If we decrement the count to
+ // zero right after `ProxyHandler::spawn` checks it, then no
+ // worker will be started; thus it becomes our responsibility to
+ // start a worker here instead.
+ if count == 1 && !self.handler.0.task_queue.is_empty() {
+ self.handler.start_worker(None, None);
+ }
+ }
+ }
+ }
+
+ async fn run(mut self, task: Option<TaskFn<S::StoreData>>, req_id: Option<u64>) {
+ if let Err(error) = self.run_(task, req_id).await {
+ self.handler.0.state.handle_worker_error(error);
+ }
+ }
+
+ async fn run_(
+ &mut self,
+ task: Option<TaskFn<S::StoreData>>,
+ req_id: Option<u64>,
+ ) -> Result<()> {
+ // NB: The code the follows is rather subtle in that it is structured
+ // carefully to provide a few key invariants related to how instance
+ // reuse and request timeouts interact:
+ //
+ // - A task must never be allowed to run for more than 2x the request
+ // timeout, if any.
+ //
+ // - Every task we accept here must be allowed to run for at least 1x
+ // the request timeout, if any.
+ //
+ // - When more than one task is run concurrently in the same instance,
+ // we must stop accepting new tasks as soon as any existing task reaches
+ // the request timeout. This serves to cap the amount of time we need
+ // to keep the instance alive before _all_ tasks have either completed
+ // or timed out.
+ //
+ // As of this writing, there's an additional wrinkle that makes
+ // guaranteeing those invariants particularly tricky: per #11869 and
+ // #11870, busy guest loops, epoch interruption, and host functions
+ // registered using `Linker::func_{wrap,new}_async` all require
+ // blocking, exclusive access to the `Store`, which effectively prevents
+ // the `StoreContextMut::run_concurrent` event loop from making
+ // progress. That, in turn, prevents any concurrent tasks from
+ // executing, and also prevents the `AsyncFnOnce` passed to
+ // `run_concurrent` from being polled. Consequently, we must rely on a
+ // "second line of defense" to ensure tasks are timed out promptly,
+ // which is to check for timeouts _outside_ the `run_concurrent` future.
+ // Once the aforementioned issues have been addressed, we'll be able to
+ // remove that check and its associated baggage.
+
+ let handler = &self.handler.0;
+
+ let StoreBundle {
+ mut store,
+ write_profile,
+ } = handler.state.new_store(req_id)?;
+
+ let request_timeout = handler.state.request_timeout();
+ let idle_instance_timeout = handler.state.idle_instance_timeout();
+ let max_instance_reuse_count = handler.state.max_instance_reuse_count();
+ let max_instance_concurrent_reuse_count =
+ handler.state.max_instance_concurrent_reuse_count();
+
+ let proxy = &handler.instance_pre.instantiate_async(&mut store).await?;
+ let accept_concurrent = AtomicBool::new(true);
+ let task_start_times = Mutex::new(StartTimes::default());
+
+ let mut future = pin!(store.run_concurrent(async |accessor| {
+ let mut reuse_count = 0;
+ let mut timed_out = false;
+ let mut futures = FuturesUnordered::new();
+
+ let accept_task = |task: TaskFn<S::StoreData>,
+ futures: &mut FuturesUnordered<_>,
+ reuse_count: &mut usize| {
+ // Set `accept_concurrent` to false, conservatively assuming
+ // that the new task will be CPU-bound, at least to begin with.
+ // Only once the `StoreContextMut::run_concurrent` event loop
+ // returns `Pending` will we set `accept_concurrent` back to
+ // true and consider accepting more tasks.
+ //
+ // This approach avoids taking on more than one CPU-bound task
+ // at a time, which would hurt throughput vs. leaving the
+ // additional tasks for other workers to handle.
+ accept_concurrent.store(false, Relaxed);
+ *reuse_count += 1;
+
+ let start_time = Instant::now().checked_add(request_timeout);
+ if let Some(start_time) = start_time {
+ task_start_times.lock().unwrap().add(start_time);
+ }
+
+ futures.push(tokio::time::timeout(request_timeout, async move {
+ (task)(accessor, proxy).await;
+ start_time
+ }));
+ };
+
+ if let Some(task) = task {
+ accept_task(task, &mut futures, &mut reuse_count);
+ }
+
+ let handler = self.handler.clone();
+ while !(futures.is_empty() && reuse_count >= max_instance_reuse_count) {
+ let new_task = {
+ let future_count = futures.len();
+ let mut next_future = pin!(async {
+ if futures.is_empty() {
+ future::pending().await
+ } else {
+ futures.next().await.unwrap()
+ }
+ });
+ let mut next_task = pin!(tokio::time::timeout(
+ if future_count == 0 {
+ idle_instance_timeout
+ } else {
+ Duration::MAX
+ },
+ handler.0.task_queue.pop()
+ ));
+ // Poll any existing tasks, and if they're all `Pending`
+ // _and_ we haven't reached any reuse limits yet, poll for a
+ // new task from the queue.
+ //
+ // Note the the order of operations here is important. By
+ // polling `next_future` first, we'll disover any tasks that
+ // may have timed out, at which point we'll stop accepting
+ // new tasks altogether (see below for details). This is
+ // especially imporant in the case where the task was
+ // blocked on a synchronous call to a host function which
+ // has exclusive access to the `Store`; once that call
+ // finishes, the first think we need to do is time out the
+ // task. If we were to poll for a new task first, then we'd
+ // have to wait for _that_ task to finish or time out before
+ // we could kill the instance.
+ future::poll_fn(|cx| match next_future.as_mut().poll(cx) {
+ Poll::Pending => {
+ // Note that `Pending` here doesn't necessarily mean
+ // all tasks are blocked on I/O. They might simply
+ // be waiting for some deferred work to be done by
+ // the next turn of the
+ // `StoreContextMut::run_concurrent` event loop.
+ // Therefore, we check `accept_concurrent` here and
+ // only advertise we have capacity for another task
+ // if either we have no tasks at all or all our
+ // tasks really are blocked on I/O.
+ self.set_available(
+ reuse_count < max_instance_reuse_count
+ && future_count < max_instance_concurrent_reuse_count
+ && (future_count == 0 || accept_concurrent.load(Relaxed)),
+ );
+
+ if self.available {
+ next_task.as_mut().poll(cx).map(Some)
+ } else {
+ Poll::Pending
+ }
+ }
+ Poll::Ready(Ok(start_time)) => {
+ // Task completed; carry on!
+ if let Some(start_time) = start_time {
+ task_start_times.lock().unwrap().remove(start_time);
+ }
+ Poll::Ready(None)
+ }
+ Poll::Ready(Err(_)) => {
+ // Task timed out; stop accepting new tasks, but
+ // continue polling until any other, in-progress
+ // tasks until they have either finished or timed
+ // out. This effectively kicks off a "graceful
+ // shutdown" of the worker, allowing any other
+ // concurrent tasks time to finish before we drop
+ // the instance.
+ //
+ // TODO: We should also send a cancel request to the
+ // timed-out task to give it a chance to shut down
+ // gracefully (and delay dropping the instance for a
+ // reasonable amount of time), but as of this
+ // writing Wasmtime does not yet provide an API for
+ // doing that. See issue #11833.
+ timed_out = true;
+ reuse_count = max_instance_reuse_count;
+ Poll::Ready(None)
+ }
+ })
+ .await
+ };
+
+ match new_task {
+ Some(Ok(task)) => {
+ accept_task(task, &mut futures, &mut reuse_count);
+ }
+ Some(Err(_)) => break,
+ None => {}
+ }
+ }
+
+ accessor.with(|mut access| write_profile(access.as_context_mut()));
+
+ if timed_out {
+ Err(anyhow!("guest timed out"))
+ } else {
+ anyhow::Ok(())
+ }
+ }));
+
+ let mut sleep = pin!(tokio::time::sleep(Duration::MAX));
+
+ future::poll_fn(|cx| {
+ let poll = future.as_mut().poll(cx);
+ if poll.is_pending() {
+ // If the future returns `Pending`, that's either because it's
+ // idle (in which case it can definitely accept a new task) or
+ // because all its tasks are awaiting I/O, in which case it may
+ // have capacity for additional tasks to run concurrently.
+ //
+ // However, if one of the tasks is blocked on a sync call to a
+ // host function which has exclusive access to the `Store`, the
+ // `StoreContextMut::run_concurrent` event loop will be unable
+ // to make progress until that call finishes. Similarly, if the
+ // task loops indefinitely, subject only to epoch interruption,
+ // the event loop will also be stuck. Either way, any task
+ // timeouts created inside the `AsyncFnOnce` we passed to
+ // `run_concurrent` won't have a chance to trigger.
+ // Consequently, we need to _also_ enforce timeouts here,
+ // outside the event loop.
+ //
+ // Therefore, we check if the oldest outstanding task has been
+ // running for at least `request_timeout*2`, which is the
+ // maximum time needed for any other concurrent tasks to
+ // complete or time out, at which point we can safely discard
+ // the instance. If that deadline has not yet arrived, we
+ // schedule a wakeup to occur when it does.
+ //
+ // We uphold the "never kill an instance with a task which has
+ // been running for less than the request timeout" invariant
+ // here by noting that this timeout will only trigger if the
+ // `AsyncFnOnce` we passed to `run_concurrent` has been unable
+ // to run for at least the past `request_timeout` amount of
+ // time, meaning it can't possibly have accepted a task newer
+ // than that.
+ if let Some(deadline) = task_start_times
+ .lock()
+ .unwrap()
+ .earliest()
+ .and_then(|v| v.checked_add(request_timeout.saturating_mul(2)))
+ {
+ sleep.as_mut().reset(deadline.into());
+ // Note that this will schedule a wakeup for later if the
+ // deadline has not yet arrived:
+ if sleep.as_mut().poll(cx).is_ready() {
+ // Deadline has been reached; kill the instance with an
+ // error.
+ return Poll::Ready(Err(anyhow!("guest timed out")));
+ }
+ }
+
+ // Otherwise, if no timeouts have elapsed, we set
+ // `accept_concurrent` to true and, if it wasn't already true
+ // before, poll the future one more time so it can ask for
+ // another task if appropriate.
+ if !accept_concurrent.swap(true, Relaxed) {
+ return future.as_mut().poll(cx);
+ }
+ }
+
+ poll
+ })
+ .await?
+ }
+}
+
+impl<S> Drop for Worker<S>
+where
+ S: HandlerState,
+{
+ fn drop(&mut self) {
+ self.set_available(false);
+ }
+}
+
+/// Represents the state of a web server.
+///
+/// Note that this supports optional instance reuse, enabled when
+/// `S::max_instance_reuse_count()` returns a number greater than one. See
+/// [`Self::push`] for details.
+pub struct ProxyHandler<S: HandlerState>(Arc<ProxyHandlerInner<S>>);
+
+impl<S: HandlerState> Clone for ProxyHandler<S> {
+ fn clone(&self) -> Self {
+ Self(self.0.clone())
+ }
+}
+
+impl<S> ProxyHandler<S>
+where
+ S: HandlerState,
+{
+ /// Create a new `ProxyHandler` with the specified application state and
+ /// pre-instance.
+ pub fn new(state: S, instance_pre: ProxyPre<S::StoreData>) -> Self {
+ Self(Arc::new(ProxyHandlerInner {
+ state,
+ instance_pre,
+ next_id: AtomicU64::from(0),
+ task_queue: Default::default(),
+ worker_count: AtomicUsize::from(0),
+ }))
+ }
+
+ /// Push a task to the task queue for this handler.
+ ///
+ /// This will either spawn a new background worker to run the task or
+ /// deliver it to an already-running worker.
+ ///
+ /// The `req_id` will be passed to `<S as HandlerState>::new_store` _if_ a
+ /// new worker is started for this task. It is intended to be used as a
+ /// "request identifier" corresponding to that task and can be used e.g. to
+ /// prefix all logging from the `Store` with that identifier. Note that a
+ /// non-`None` value only makes sense when `<S as
+ /// HandlerState>::max_instance_reuse_count == 1`; otherwise the identifier
+ /// will not match subsequent tasks handled by the worker.
+ pub fn spawn(&self, req_id: Option<u64>, task: TaskFn<S::StoreData>) {
+ match self.0.state.max_instance_reuse_count() {
+ 0 => panic!("`max_instance_reuse_count` must be at least 1"),
+ _ => {
+ if self.0.worker_count.load(Relaxed) == 0 {
+ // There are no available workers; skip the queue and pass
+ // the task directly to the worker, which improves
+ // performance as measured by `wasmtime-server-rps.sh` by
+ // about 15%.
+ self.start_worker(Some(task), req_id);
+ } else {
+ self.0.task_queue.push(task);
+ // Start a new worker to handle the task if the last worker
+ // just went unavailable. See also `Worker::set_available`
+ // for what happens if the available worker count goes to
+ // zero right after we check it here, and note that we only
+ // check the count _after_ we've pushed the task to the
+ // queue. We use `SeqCst` here to ensure that we get an
+ // updated view of `worker_count` as it exists after the
+ // `Queue::push` above.
+ //
+ // The upshot is that at least one (or more) of the
+ // following will happen:
+ //
+ // - An existing worker will accept the task
+ // - We'll start a new worker here to accept the task
+ // - `Worker::set_available` will start a new worker to accept the task
+ //
+ // I.e. it should not be possible for the task to be
+ // orphaned indefinitely in the queue without being
+ // accepted.
+ if self.0.worker_count.load(SeqCst) == 0 {
+ self.start_worker(None, None);
+ }
+ }
+ }
+ }
+ }
+
+ /// Generate a unique request ID.
+ pub fn next_req_id(&self) -> u64 {
+ self.0.next_id.fetch_add(1, Relaxed)
+ }
+
+ /// Return a reference to the application state.
+ pub fn state(&self) -> &S {
+ &self.0.state
+ }
+
+ /// Return a reference to the pre-instance.
+ pub fn instance_pre(&self) -> &ProxyPre<S::StoreData> {
+ &self.0.instance_pre
+ }
+
+ fn start_worker(&self, task: Option<TaskFn<S::StoreData>>, req_id: Option<u64>) {
+ tokio::spawn(
+ Worker {
+ handler: self.clone(),
+ available: false,
+ }
+ .run(task, req_id),
+ );
+ }
+}
diff --git a/crates/wasi-http/src/lib.rs b/crates/wasi-http/src/lib.rs
index b7efd26542..fe50030bb3 100644
--- a/crates/wasi-http/src/lib.rs
+++ b/crates/wasi-http/src/lib.rs
@@ -225,6 +225,8 @@ mod http_impl;
mod types_impl;
pub mod body;
+#[cfg(feature = "component-model-async")]
+pub mod handler;
pub mod io;
pub mod types;
diff --git a/crates/wasi/tests/all/p3/mod.rs b/crates/wasi/tests/all/p3/mod.rs
index df299075cd..3faf1b7b36 100644
--- a/crates/wasi/tests/all/p3/mod.rs
+++ b/crates/wasi/tests/all/p3/mod.rs
@@ -168,3 +168,9 @@ fn p3_cli_much_stdout() {}
reason = "tested in the wasi-cli crate, satisfying foreach_api! macro"
)]
fn p3_cli_serve_hello_world() {}
+
+#[expect(
+ dead_code,
+ reason = "tested in the wasi-cli crate, satisfying foreach_api! macro"
+)]
+fn p3_cli_serve_sleep() {}
diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rs
index 30aea75e7b..9aca9de4bc 100644
--- a/crates/wasmtime/src/runtime/component/concurrent.rs
+++ b/crates/wasmtime/src/runtime/component/concurrent.rs
@@ -1103,9 +1103,9 @@ impl<T> StoreContextMut<'_, T> {
}
loop {
- // Take `ConcurrentState::futures` out of the instance so we can
- // poll it while also safely giving any of the futures inside access
- // to `self`.
+ // Take `ConcurrentState::futures` out of the store so we can poll
+ // it while also safely giving any of the futures inside access to
+ // `self`.
let futures = self.0.concurrent_state_mut().futures.get_mut().take();
let mut reset = Reset {
store: self.as_context_mut(),
@@ -1196,9 +1196,9 @@ impl<T> StoreContextMut<'_, T> {
})
.await;
- // Put the `ConcurrentState::futures` back into the instance before
- // we return or handle any work items since one or more of those
- // items might append more futures.
+ // Put the `ConcurrentState::futures` back into the store before we
+ // return or handle any work items since one or more of those items
+ // might append more futures.
drop(reset);
match result? {
diff --git a/src/commands/serve.rs b/src/commands/serve.rs
index edf479d76f..8d776a1f95 100644
--- a/src/commands/serve.rs
+++ b/src/commands/serve.rs
@@ -1,7 +1,8 @@
use crate::common::{Profile, RunCommon, RunTarget};
-use anyhow::{Result, bail};
+use anyhow::{Context as _, Result, bail};
use bytes::Bytes;
use clap::Parser;
+use futures::future::FutureExt;
use http::{Response, StatusCode};
use http_body_util::BodyExt as _;
use http_body_util::combinators::BoxBody;
@@ -9,22 +10,24 @@ use std::convert::Infallible;
use std::net::SocketAddr;
use std::pin::Pin;
use std::task::{Context, Poll};
-use std::time::Instant;
use std::{
path::PathBuf,
sync::{
Arc, Mutex,
- atomic::{AtomicBool, AtomicU64, Ordering},
+ atomic::{AtomicBool, Ordering},
},
time::Duration,
};
use tokio::io::{self, AsyncWrite};
use tokio::sync::Notify;
use wasmtime::component::{Component, Linker, ResourceTable};
-use wasmtime::{Engine, Store, StoreLimits, UpdateDeadline};
+use wasmtime::{Engine, Store, StoreContextMut, StoreLimits, UpdateDeadline};
+use wasmtime_cli_flags::opt::WasmtimeOptionValue;
use wasmtime_wasi::p2::{StreamError, StreamResult};
use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiCtxView, WasiView};
-use wasmtime_wasi_http::bindings as p2;
+#[cfg(feature = "component-model-async")]
+use wasmtime_wasi_http::handler::p2::bindings as p2;
+use wasmtime_wasi_http::handler::{HandlerState, Proxy, ProxyHandler, ProxyPre, StoreBundle};
use wasmtime_wasi_http::io::TokioIo;
use wasmtime_wasi_http::{
DEFAULT_OUTGOING_BODY_BUFFER_CHUNKS, DEFAULT_OUTGOING_BODY_CHUNK_SIZE, WasiHttpCtx,
@@ -38,6 +41,10 @@ use wasmtime_wasi_keyvalue::{WasiKeyValue, WasiKeyValueCtx, WasiKeyValueCtxBuild
#[cfg(feature = "wasi-nn")]
use wasmtime_wasi_nn::wit::WasiNnCtx;
+const DEFAULT_WASIP3_MAX_INSTANCE_REUSE_COUNT: usize = 128;
+const DEFAULT_WASIP2_MAX_INSTANCE_REUSE_COUNT: usize = 1;
+const DEFAULT_WASIP3_MAX_INSTANCE_CONCURRENT_REUSE_COUNT: usize = 16;
+
struct Host {
table: wasmtime::component::ResourceTable,
ctx: WasiCtx,
@@ -106,6 +113,10 @@ const DEFAULT_ADDR: std::net::SocketAddr = std::net::SocketAddr::new(
8080,
);
+fn parse_duration(s: &str) -> Result<Duration, String> {
+ Duration::parse(Some(s)).map_err(|e| e.to_string())
+}
+
/// Runs a WebAssembly module
#[derive(Parser)]
pub struct ServeCommand {
@@ -131,6 +142,31 @@ pub struct ServeCommand {
/// The WebAssembly component to run.
#[arg(value_name = "WASM", required = true)]
component: PathBuf,
+
+ /// Maximum number of requests to send to a single component instance before
+ /// dropping it.
+ ///
+ /// This defaults to 1 for WASIp2 components and 128 for WASIp3 components.
+ #[arg(long)]
+ max_instance_reuse_count: Option<usize>,
+
+ /// Maximum number of concurrent requests to send to a single component
+ /// instance.
+ ///
+ /// This defaults to 1 for WASIp2 components and 16 for WASIp3 components.
+ /// Note that setting it to more than 1 will have no effect for WASIp2
+ /// components since they cannot be called concurrently.
+ #[arg(long)]
+ max_instance_concurrent_reuse_count: Option<usize>,
+
+ /// Time to hold an idle component instance for possible reuse before
+ /// dropping it.
+ ///
+ /// A number with no suffix or with an `s` suffix is interpreted as seconds;
+ /// other accepted suffixes include `ms` (milliseconds), `us` or `μs`
+ /// (microseconds), and `ns` (nanoseconds).
+ #[arg(long, default_value = "1s", value_parser = parse_duration)]
+ idle_instance_timeout: Duration,
}
impl ServeCommand {
@@ -171,20 +207,25 @@ impl ServeCommand {
Ok(())
}
- fn new_store(&self, engine: &Engine, req_id: u64) -> Result<Store<Host>> {
+ fn new_store(&self, engine: &Engine, req_id: Option<u64>) -> Result<Store<Host>> {
let mut builder = WasiCtxBuilder::new();
self.run.configure_wasip2(&mut builder)?;
- builder.env("REQUEST_ID", req_id.to_string());
+ if let Some(req_id) = req_id {
+ builder.env("REQUEST_ID", req_id.to_string());
+ }
let stdout_prefix: String;
let stderr_prefix: String;
- if self.no_logging_prefix {
- stdout_prefix = "".to_string();
- stderr_prefix = "".to_string();
- } else {
- stdout_prefix = format!("stdout [{req_id}] :: ");
- stderr_prefix = format!("stderr [{req_id}] :: ");
+ match req_id {
+ Some(req_id) if !self.no_logging_prefix => {
+ stdout_prefix = format!("stdout [{req_id}] :: ");
+ stderr_prefix = format!("stderr [{req_id}] :: ");
+ }
+ _ => {
+ stdout_prefix = "".to_string();
+ stderr_prefix = "".to_string();
+ }
}
builder.stdout(LogStream::new(stdout_prefix, Output::Stdout));
builder.stderr(LogStream::new(stderr_prefix, Output::Stderr));
@@ -449,7 +490,31 @@ impl ServeCommand {
};
let _epoch_thread = epoch_interval.map(|t| EpochThread::spawn(t, engine.clone()));
- let handler = ProxyHandler::new(self, engine, instance);
+ let max_instance_reuse_count = self.max_instance_reuse_count.unwrap_or_else(|| {
+ if let ProxyPre::P3(_) = &instance {
+ DEFAULT_WASIP3_MAX_INSTANCE_REUSE_COUNT
+ } else {
+ DEFAULT_WASIP2_MAX_INSTANCE_REUSE_COUNT
+ }
+ });
+
+ let max_instance_concurrent_reuse_count = if let ProxyPre::P3(_) = &instance {
+ self.max_instance_concurrent_reuse_count
+ .unwrap_or(DEFAULT_WASIP3_MAX_INSTANCE_CONCURRENT_REUSE_COUNT)
+ } else {
+ 1
+ };
+
+ let handler = ProxyHandler::new(
+ HostHandlerState {
+ cmd: self,
+ engine,
+ component,
+ max_instance_reuse_count,
+ max_instance_concurrent_reuse_count,
+ },
+ instance,
+ );
loop {
// Wait for a socket, but also "race" against shutdown to break out
@@ -459,7 +524,14 @@ impl ServeCommand {
_ = shutdown.requested.notified() => break,
v = listener.accept() => v?,
};
- let comp = component.clone();
+
+ // The Nagle algorithm can impose a significant latency penalty
+ // (e.g. 40ms on Linux) on guests which write small, intermittent
+ // response body chunks (e.g. SSE streams). Here we disable that
+ // algorithm and rely on the guest to buffer if appropriate to avoid
+ // TCP fragmentation.
+ stream.set_nodelay(true)?;
+
let stream = TokioIo::new(stream);
let h = handler.clone();
let shutdown_guard = shutdown.clone().increment();
@@ -469,11 +541,10 @@ impl ServeCommand {
.serve_connection(
stream,
hyper::service::service_fn(move |req| {
- let comp = comp.clone();
let h = h.clone();
async move {
use http_body_util::{BodyExt, Full};
- match handle_request(h, req, comp).await {
+ match handle_request(h, req).await {
Ok(r) => Ok::<_, Infallible>(r),
Err(e) => {
eprintln!("error: {e:?}");
@@ -531,6 +602,48 @@ impl ServeCommand {
}
}
+struct HostHandlerState {
+ cmd: ServeCommand,
+ engine: Engine,
+ component: Component,
+ max_instance_reuse_count: usize,
+ max_instance_concurrent_reuse_count: usize,
+}
+
+impl HandlerState for HostHandlerState {
+ type StoreData = Host;
+
+ fn new_store(&self, req_id: Option<u64>) -> Result<StoreBundle<Host>> {
+ let mut store = self.cmd.new_store(&self.engine, req_id)?;
+ let write_profile = setup_epoch_handler(&self.cmd, &mut store, self.component.clone())?;
+
+ Ok(StoreBundle {
+ store,
+ write_profile,
+ })
+ }
+
+ fn request_timeout(&self) -> Duration {
+ self.cmd.run.common.wasm.timeout.unwrap_or(Duration::MAX)
+ }
+
+ fn idle_instance_timeout(&self) -> Duration {
+ self.cmd.idle_instance_timeout
+ }
+
+ fn max_instance_reuse_count(&self) -> usize {
+ self.max_instance_reuse_count
+ }
+
+ fn max_instance_concurrent_reuse_count(&self) -> usize {
+ self.max_instance_concurrent_reuse_count
+ }
+
+ fn handle_worker_error(&self, error: anyhow::Error) {
+ eprintln!("worker error: {error}");
+ }
+}
+
/// Helper structure to manage graceful shutdown int he accept loop above.
#[derive(Default)]
struct GracefulShutdown {
@@ -618,7 +731,7 @@ impl Drop for EpochThread {
}
}
-type WriteProfile = Box<dyn FnOnce(&mut Store<Host>) + Send>;
+type WriteProfile = Box<dyn FnOnce(StoreContextMut<Host>) + Send>;
fn setup_epoch_handler(
cmd: &ServeCommand,
@@ -628,7 +741,7 @@ fn setup_epoch_handler(
// Profiling Enabled
if let Some(Profile::Guest { interval, path }) = &cmd.run.profile {
#[cfg(feature = "profiling")]
- return setup_guest_profiler(cmd, store, path.clone(), *interval, component.clone());
+ return setup_guest_profiler(store, path.clone(), *interval, component.clone());
#[cfg(not(feature = "profiling"))]
{
let _ = (path, interval);
@@ -646,7 +759,6 @@ fn setup_epoch_handler(
#[cfg(feature = "profiling")]
fn setup_guest_profiler(
- cmd: &ServeCommand,
store: &mut Store<Host>,
path: String,
interval: Duration,
@@ -681,28 +793,17 @@ fn setup_guest_profiler(
Ok(())
});
- let start = Instant::now();
- let timeout = cmd.run.common.wasm.timeout;
store.epoch_deadline_callback(move |store| {
sample(store, |profiler, store| {
profiler.sample(store, std::time::Duration::ZERO)
});
- // Originally epoch counting was used here; this is problematic in
- // a lot of cases due to there being a lot of time (e.g. in hostcalls)
- // when we are not expected to get sample hits.
- if let Some(timeout) = timeout {
- if start.elapsed() > timeout {
- bail!("Timeout expired");
- }
- }
-
Ok(UpdateDeadline::Continue(1))
});
store.set_epoch_deadline(1);
- let write_profile = Box::new(move |store: &mut Store<Host>| {
+ let write_profile = Box::new(move |mut store: StoreContextMut<Host>| {
let profiler = Arc::try_unwrap(store.data_mut().guest_profiler.take().unwrap())
.expect("profiling doesn't support threads yet");
if let Err(e) = std::fs::File::create(&path)
@@ -720,65 +821,15 @@ fn setup_guest_profiler(
Ok(write_profile)
}
-struct ProxyHandlerInner {
- cmd: ServeCommand,
- engine: Engine,
- instance_pre: ProxyPre,
- next_id: AtomicU64,
-}
-
-enum ProxyPre {
- P2(p2::ProxyPre<Host>),
- #[cfg(feature = "component-model-async")]
- P3(wasmtime_wasi_http::p3::bindings::ProxyPre<Host>),
-}
-
-impl ProxyPre {
- async fn instantiate(&self, store: &mut Store<Host>) -> Result<Proxy> {
- Ok(match self {
- ProxyPre::P2(pre) => Proxy::P2(pre.instantiate_async(store).await?),
- #[cfg(feature = "component-model-async")]
- ProxyPre::P3(pre) => Proxy::P3(pre.instantiate_async(store).await?),
- })
- }
-}
-
-enum Proxy {
- P2(p2::Proxy),
- #[cfg(feature = "component-model-async")]
- P3(wasmtime_wasi_http::p3::bindings::Proxy),
-}
-
-impl ProxyHandlerInner {
- fn next_req_id(&self) -> u64 {
- self.next_id.fetch_add(1, Ordering::Relaxed)
- }
-}
-
-#[derive(Clone)]
-struct ProxyHandler(Arc<ProxyHandlerInner>);
-
-impl ProxyHandler {
- fn new(cmd: ServeCommand, engine: Engine, instance_pre: ProxyPre) -> Self {
- Self(Arc::new(ProxyHandlerInner {
- cmd,
- engine,
- instance_pre,
- next_id: AtomicU64::from(0),
- }))
- }
-}
-
type Request = hyper::Request<hyper::body::Incoming>;
async fn handle_request(
- ProxyHandler(inner): ProxyHandler,
+ handler: ProxyHandler<HostHandlerState>,
req: Request,
- component: Component,
) -> Result<hyper::Response<BoxBody<Bytes, anyhow::Error>>> {
- let (sender, receiver) = tokio::sync::oneshot::channel();
+ use tokio::sync::oneshot;
- let req_id = inner.next_req_id();
+ let req_id = handler.next_req_id();
log::info!(
"Request {req_id} handling {} to {}",
@@ -786,97 +837,102 @@ async fn handle_request(
req.uri()
);
- let mut store = inner.cmd.new_store(&inner.engine, req_id)?;
-
- let write_profile = setup_epoch_handler(&inner.cmd, &mut store, component.clone())?;
- let timeout = inner.cmd.run.common.wasm.timeout.unwrap_or(Duration::MAX);
-
- match inner.instance_pre.instantiate(&mut store).await? {
- Proxy::P2(proxy) => {
- let req = store
- .data_mut()
- .new_incoming_request(p2::http::types::Scheme::Http, req)?;
- let out = store.data_mut().new_response_outparam(sender)?;
- let task = tokio::task::spawn(async move {
- let result = tokio::time::timeout(
- timeout,
- proxy
- .wasi_http_incoming_handler()
- .call_handle(&mut store, req, out),
- )
- .await
- .unwrap_or_else(|_| bail!("guest timed out"));
- if let Err(e) = result {
- log::error!("[{req_id}] :: {e:?}");
- return Err(e);
- }
-
- write_profile(&mut store);
-
- Ok(())
- });
+ // Here we must declare different channel types for p2 and p3 since p2's
+ // `WasiHttpView::new_response_outparam` expects a specific kind of sender
+ // that uses `p2::http::types::ErrorCode`, and we don't want to have to
+ // convert from the p3 `ErrorCode` to the p2 one, only to convert again to
+ // `anyhow::Error`.
+
+ type P2Response = Result<
+ hyper::Response<wasmtime_wasi_http::body::HyperOutgoingBody>,
+ p2::http::types::ErrorCode,
+ >;
+ type P3Response = hyper::Response<BoxBody<Bytes, anyhow::Error>>;
+
+ enum Sender {
+ P2(oneshot::Sender<P2Response>),
+ P3(oneshot::Sender<P3Response>),
+ }
- let result = match receiver.await {
- Ok(Ok(resp)) => resp,
- Ok(Err(e)) => bail!(e),
- Err(_) => {
- // An error in the receiver (`RecvError`) only indicates that the
- // task exited before a response was sent (i.e., the sender was
- // dropped); it does not describe the underlying cause of failure.
- // Instead we retrieve and propagate the error from inside the task
- // which should more clearly tell the user what went wrong. Note
- // that we assume the task has already exited at this point so the
- // `await` should resolve immediately.
- let e = match task.await {
- Ok(Ok(())) => {
- bail!("guest never invoked `response-outparam::set` method")
- }
- Ok(Err(e)) => e,
- Err(e) => e.into(),
- };
- bail!(e.context("guest never invoked `response-outparam::set` method"))
- }
- };
+ enum Receiver {
+ P2(oneshot::Receiver<P2Response>),
+ P3(oneshot::Receiver<P3Response>),
+ }
- Ok(result.map(|body| body.map_err(|e| e.into()).boxed()))
+ let (tx, rx) = match handler.instance_pre() {
+ ProxyPre::P2(_) => {
+ let (tx, rx) = oneshot::channel();
+ (Sender::P2(tx), Receiver::P2(rx))
}
- #[cfg(feature = "component-model-async")]
- Proxy::P3(proxy) => {
- use wasmtime_wasi_http::p3::bindings::http::types::{ErrorCode, Request};
-
- let (tx, rx) = tokio::sync::oneshot::channel();
+ ProxyPre::P3(_) => {
+ let (tx, rx) = oneshot::channel();
+ (Sender::P3(tx), Receiver::P3(rx))
+ }
+ };
- tokio::task::spawn(async move {
- let guest_result = store.run_concurrent(async move |store| {
- let (req, body) = req.into_parts();
- let body = body.map_err(ErrorCode::from_hyper_request_error);
- let req = http::Request::from_parts(req, body);
- let (request, request_io_result) = Request::from_http(req);
- let (res, task) = proxy.handle(store, request).await??;
- let res =
- store.with(|mut store| res.into_http(&mut store, request_io_result))?;
-
- _ = tx.send(res);
-
- task.block(store).await;
- anyhow::Ok(())
- });
-
- let guest_result = tokio::time::timeout(timeout, guest_result)
- .await
- .unwrap_or_else(|_| bail!("guest timed out"))?;
- if let Err(e) = guest_result {
- log::error!("[{req_id}] :: {e:?}");
- return Err(e);
+ handler.spawn(
+ if handler.state().max_instance_reuse_count() == 1 {
+ Some(req_id)
+ } else {
+ None
+ },
+ Box::new(move |store, proxy| {
+ Box::pin(
+ async move {
+ match proxy {
+ Proxy::P2(proxy) => {
+ let Sender::P2(tx) = tx else { unreachable!() };
+ let (req, out) = store.with(move |mut store| {
+ let req = store
+ .data_mut()
+ .new_incoming_request(p2::http::types::Scheme::Http, req)?;
+ let out = store.data_mut().new_response_outparam(tx)?;
+ anyhow::Ok((req, out))
+ })?;
+
+ proxy
+ .wasi_http_incoming_handler()
+ .call_handle(store, req, out)
+ .await
+ }
+ Proxy::P3(proxy) => {
+ use wasmtime_wasi_http::p3::bindings::http::types::{
+ ErrorCode, Request,
+ };
+
+ let Sender::P3(tx) = tx else { unreachable!() };
+ let (req, body) = req.into_parts();
+ let body = body.map_err(ErrorCode::from_hyper_request_error);
+ let req = http::Request::from_parts(req, body);
+ let (request, request_io_result) = Request::from_http(req);
+ let (res, task) = proxy.handle(store, request).await??;
+ let res = store
+ .with(|mut store| res.into_http(&mut store, request_io_result))?;
+ _ = tx.send(res.map(|body| body.map_err(|e| e.into()).boxed()));
+
+ // Wait for the task to finish.
+ task.block(store).await;
+ Ok(())
+ }
+ }
}
+ .map(move |result| {
+ if let Err(error) = result {
+ eprintln!("[{req_id}] :: {error:?}");
+ }
+ }),
+ )
+ }),
+ );
- write_profile(&mut store);
-
- anyhow::Ok(())
- });
- Ok(rx.await?.map(|body| body.map_err(|err| err.into()).boxed()))
- }
- }
+ Ok(match rx {
+ Receiver::P2(rx) => rx
+ .await
+ .context("guest never invoked `response-outparam::set` method")?
+ .map_err(|e| anyhow::Error::from(e))?
+ .map(|body| body.map_err(|e| e.into()).boxed()),
+ Receiver::P3(rx) => rx.await?,
+ })
}
#[derive(Clone)]
diff --git a/tests/all/cli_tests.rs b/tests/all/cli_tests.rs
index e08d0074c2..a84bf616ec 100644
--- a/tests/all/cli_tests.rs
+++ b/tests/all/cli_tests.rs
@@ -1631,19 +1631,29 @@ mod test_programs {
async fn send_request(&self, req: http::Request<String>) -> Result<http::Response<String>> {
let (mut send, conn_task) = self.start_requests().await?;
+ let response = Self::send_request_with(&mut send, req).await?;
+ drop(send);
+
+ conn_task.await??;
+
+ Ok(response)
+ }
+
+ async fn send_request_with(
+ send: &mut hyper::client::conn::http1::SendRequest<String>,
+ req: http::Request<String>,
+ ) -> Result<http::Response<String>> {
let response = send
.send_request(req)
.await
.context("error sending request")?;
- drop(send);
+
let (parts, body) = response.into_parts();
let body = body.collect().await.context("failed to read body")?;
assert!(body.trailers().is_none());
let body = std::str::from_utf8(&body.to_bytes())?.to_string();
- conn_task.await??;
-
Ok(http::Response::from_parts(parts, body))
}
@@ -2238,44 +2248,239 @@ start a print 1234
#[tokio::test]
#[cfg_attr(not(feature = "component-model-async"), ignore)]
async fn p3_cli_serve_hello_world() -> Result<()> {
- let server = WasmtimeServe::new(P3_CLI_SERVE_HELLO_WORLD_COMPONENT, |cmd| {
+ cli_serve_hello_world(P3_CLI_SERVE_HELLO_WORLD_COMPONENT, 1, 1, |cmd| {
cmd.arg("-Wcomponent-model-async");
cmd.arg("-Sp3,cli");
- })?;
+ })
+ .await
+ }
- let result = server
- .send_request(
- hyper::Request::builder()
- .uri("http://localhost/")
- .body(String::new())
- .context("failed to make request")?,
- )
- .await?;
+ #[tokio::test]
+ async fn p2_cli_serve_hello_world() -> Result<()> {
+ cli_serve_hello_world(P2_CLI_SERVE_HELLO_WORLD_COMPONENT, 1, 1, |cmd| {
+ cmd.arg("-Scli");
+ })
+ .await
+ }
- assert!(result.status().is_success());
- assert_eq!(result.body(), "Hello, WASI!");
+ const CONNECTION_COUNT_MANY: usize = 20;
+ const REQUESTS_PER_CONNECTION_MANY: usize = 5;
- server.finish()?;
+ #[tokio::test]
+ #[cfg_attr(not(feature = "component-model-async"), ignore)]
+ async fn p3_cli_serve_hello_world_many() -> Result<()> {
+ cli_serve_hello_world(
+ P3_CLI_SERVE_HELLO_WORLD_COMPONENT,
+ CONNECTION_COUNT_MANY,
+ REQUESTS_PER_CONNECTION_MANY,
+ |cmd| {
+ cmd.arg("-Wcomponent-model-async");
+ cmd.arg("-Sp3,cli");
+ },
+ )
+ .await
+ }
+
+ #[tokio::test]
+ #[cfg_attr(not(feature = "component-model-async"), ignore)]
+ async fn p3_cli_serve_hello_world_many_no_reuse() -> Result<()> {
+ cli_serve_hello_world(
+ P3_CLI_SERVE_HELLO_WORLD_COMPONENT,
+ CONNECTION_COUNT_MANY,
+ REQUESTS_PER_CONNECTION_MANY,
+ |cmd| {
+ cmd.arg("-Wcomponent-model-async");
+ cmd.arg("-Sp3,cli");
+ cmd.arg("--max-instance-reuse-count=1");
+ },
+ )
+ .await
+ }
+
+ #[tokio::test]
+ #[cfg_attr(not(feature = "component-model-async"), ignore)]
+ async fn p3_cli_serve_hello_world_many_no_concurrent_reuse() -> Result<()> {
+ cli_serve_hello_world(
+ P3_CLI_SERVE_HELLO_WORLD_COMPONENT,
+ CONNECTION_COUNT_MANY,
+ REQUESTS_PER_CONNECTION_MANY,
+ |cmd| {
+ cmd.arg("-Wcomponent-model-async");
+ cmd.arg("-Sp3,cli");
+ cmd.arg("--max-instance-concurrent-reuse-count=1");
+ },
+ )
+ .await
+ }
+
+ #[tokio::test]
+ async fn p2_cli_serve_hello_world_many() -> Result<()> {
+ cli_serve_hello_world(
+ P2_CLI_SERVE_HELLO_WORLD_COMPONENT,
+ CONNECTION_COUNT_MANY,
+ REQUESTS_PER_CONNECTION_MANY,
+ |cmd| {
+ cmd.arg("-Scli");
+ },
+ )
+ .await
+ }
+
+ #[tokio::test]
+ async fn p2_cli_serve_hello_world_many_with_reuse() -> Result<()> {
+ cli_serve_hello_world(
+ P2_CLI_SERVE_HELLO_WORLD_COMPONENT,
+ CONNECTION_COUNT_MANY,
+ REQUESTS_PER_CONNECTION_MANY,
+ |cmd| {
+ cmd.arg("-Scli");
+ cmd.arg("--max-instance-reuse-count=128");
+ },
+ )
+ .await
+ }
+
+ async fn cli_serve_hello_world(
+ component: &str,
+ connection_count: usize,
+ requests_per_connection: usize,
+ configure: impl FnOnce(&mut Command),
+ ) -> Result<()> {
+ let server = std::sync::Arc::new(WasmtimeServe::new(component, configure)?);
+
+ let tasks = (0..connection_count).map({
+ let server = server.clone();
+ move |_| {
+ tokio::task::spawn({
+ let server = server.clone();
+ async move {
+ let (mut send, conn_task) = server.start_requests().await?;
+
+ for _ in 0..requests_per_connection {
+ let result = WasmtimeServe::send_request_with(
+ &mut send,
+ hyper::Request::builder()
+ .uri("http://localhost/")
+ .body(String::new())
+ .context("failed to make request")?,
+ )
+ .await?;
+
+ assert!(result.status().is_success());
+ assert_eq!(result.body(), "Hello, WASI!");
+ }
+
+ drop(send);
+
+ conn_task.await??;
+
+ anyhow::Ok(())
+ }
+ })
+ }
+ });
+
+ for task in tasks {
+ task.await??;
+ }
+
+ std::sync::Arc::into_inner(server).unwrap().finish()?;
Ok(())
}
#[tokio::test]
async fn p2_cli_serve_sleep() -> Result<()> {
- let server = WasmtimeServe::new(P2_CLI_SERVE_SLEEP_COMPONENT, |cmd| {
- cmd.arg("-Scli").arg("-Wtimeout=100us");
- })?;
+ cli_serve_sleep(P2_CLI_SERVE_SLEEP_COMPONENT, 1, 1, |cmd| {
+ cmd.arg("-Scli");
+ })
+ .await
+ }
- let resp = server
- .send_request(
- hyper::Request::builder()
- .uri("http://localhost/")
- .body(String::new())
- .context("failed to make request")?,
- )
- .await?;
+ #[tokio::test]
+ #[cfg_attr(not(feature = "component-model-async"), ignore)]
+ async fn p3_cli_serve_sleep() -> Result<()> {
+ cli_serve_sleep(P3_CLI_SERVE_SLEEP_COMPONENT, 1, 1, |cmd| {
+ cmd.arg("-Wcomponent-model-async");
+ cmd.arg("-Sp3,cli");
+ })
+ .await
+ }
- assert!(resp.status().is_server_error());
- let (stdout, stderr) = server.finish()?;
+ #[tokio::test]
+ async fn p2_cli_serve_sleep_many() -> Result<()> {
+ cli_serve_sleep(
+ P2_CLI_SERVE_SLEEP_COMPONENT,
+ CONNECTION_COUNT_MANY,
+ REQUESTS_PER_CONNECTION_MANY,
+ |cmd| {
+ cmd.arg("-Scli");
+ },
+ )
+ .await
+ }
+
+ #[tokio::test]
+ #[cfg_attr(not(feature = "component-model-async"), ignore)]
+ async fn p3_cli_serve_sleep_many() -> Result<()> {
+ cli_serve_sleep(
+ P3_CLI_SERVE_SLEEP_COMPONENT,
+ CONNECTION_COUNT_MANY,
+ REQUESTS_PER_CONNECTION_MANY,
+ |cmd| {
+ cmd.arg("-Wcomponent-model-async");
+ cmd.arg("-Sp3,cli");
+ },
+ )
+ .await
+ }
+
+ async fn cli_serve_sleep(
+ component: &str,
+ connection_count: usize,
+ requests_per_connection: usize,
+ configure: impl FnOnce(&mut Command),
+ ) -> Result<()> {
+ let server = std::sync::Arc::new(WasmtimeServe::new(component, move |cmd| {
+ configure(cmd);
+ cmd.arg("-Wtimeout=100us");
+ })?);
+
+ let tasks = (0..connection_count).map({
+ let server = server.clone();
+ move |_| {
+ tokio::task::spawn({
+ let server = server.clone();
+ async move {
+ let (mut send, conn_task) = server.start_requests().await?;
+
+ for _ in 0..requests_per_connection {
+ let result = WasmtimeServe::send_request_with(
+ &mut send,
+ hyper::Request::builder()
+ .uri("http://localhost/")
+ .body(String::new())
+ .context("failed to make request")?,
+ )
+ .await?;
+
+ assert!(result.status().is_server_error());
+ }
+
+ drop(send);
+
+ conn_task.await??;
+
+ anyhow::Ok(())
+ }
+ })
+ }
+ });
+
+ for task in tasks {
+ task.await??;
+ }
+
+ let (stdout, stderr) = std::sync::Arc::into_inner(server).unwrap().finish()?;
assert_eq!(stdout, "");
assert!(stderr.contains("guest timed out"), "bad stderr: {stderr}");
Ok(())