Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Update wasi-testsuite and get it running (#13592)

Alex Crichtonec47fbcd4c8c
diff --git a/tests/wasi-testsuite b/tests/wasi-testsuite
index 26ea2ff9dd..a667df7a6f 160000
--- a/tests/wasi-testsuite
+++ b/tests/wasi-testsuite
@@ -1 +1 @@
-Subproject commit 26ea2ff9ddc2aad1b61b40c73786869af156995a
+Subproject commit a667df7a6f2b0c23cae7c57c7fe1df0868951d7c
diff --git a/tests/wasi.rs b/tests/wasi.rs
index 4a4fdc8bf4..7baa5b9171 100644
--- a/tests/wasi.rs
+++ b/tests/wasi.rs
@@ -3,22 +3,30 @@
//!
//! [wasi-testsuite]: https://github.com/WebAssembly/wasi-testsuite
+use http_body_util::BodyExt;
use libtest_mimic::{Arguments, Trial};
use serde_derive::Deserialize;
use std::collections::HashMap;
-use std::fmt::Write;
+use std::fmt::Write as _;
use std::fs;
+use std::io::{Read, Write};
+use std::net::{SocketAddr, TcpStream};
use std::path::Path;
-use std::process::Output;
+use std::process::{Child, Stdio};
use tempfile::TempDir;
+use wasmtime::error::Context;
use wasmtime::{Result, ToWasmtimeResult as _, format_err};
use wit_component::ComponentEncoder;
const KNOWN_FAILURES: &[&str] = &[
- "filesystem-hard-links",
- "filesystem-read-directory",
// FIXME(#11524)
"remove_directory_trailing_slashes",
+ // FIXME(#12475)
+ "sockets-udp-send",
+ // FIXME(#13396)
+ #[cfg(target_os = "linux")]
+ "sockets-tcp-connect",
+ // FIXME(#13396)
#[cfg(target_vendor = "apple")]
"filesystem-advise",
// FIXME(WebAssembly/wasi-testsuite#128)
@@ -88,15 +96,26 @@ const KNOWN_FAILURES: &[&str] = &[
"symlink_loop",
#[cfg(windows)]
"unlink_file_trailing_slashes",
- // Once cm-async changes have percolated this can be removed.
- "filesystem-flags-and-type",
- "multi-clock-wait",
- "monotonic-clock",
- "filesystem-advise",
- // Wasmtime's snapshot of WASIp3 APIs is different than what these tests are
- // expecting.
- "wall-clock",
- "http-response",
+ #[cfg(windows)]
+ "filesystem-unlink-errors",
+ #[cfg(windows)]
+ "filesystem-stat",
+ #[cfg(windows)]
+ "filesystem-set-size",
+ #[cfg(windows)]
+ "filesystem-rename",
+ #[cfg(windows)]
+ "filesystem-read-directory",
+ #[cfg(windows)]
+ "filesystem-open-errors",
+ #[cfg(windows)]
+ "filesystem-mkdir-rmdir",
+ #[cfg(windows)]
+ "filesystem-metadata-hash",
+ #[cfg(windows)]
+ "filesystem-hard-links",
+ #[cfg(windows)]
+ "filesystem-io",
];
fn main() -> Result<()> {
@@ -168,75 +187,217 @@ fn run_test(path: &Path, componentize: bool) -> Result<()> {
path.to_path_buf()
};
- let Spec {
- args,
- dirs,
- env,
- exit_code: _,
- stderr: _,
- stdout: _,
- } = &spec;
- let mut cmd = wasmtime_test_util::command(wasmtime);
- cmd.arg("run");
- for dir in dirs {
- cmd.arg("--dir");
- let src = parent_dir.join(dir);
- let dst = td.path().join(dir);
- cp_r(&src, &dst)?;
- cmd.arg(format!("{}::{dir}", dst.display()));
- }
- for (k, v) in env {
- cmd.arg("--env");
- cmd.arg(format!("{k}={v}"));
- }
- let mut should_fail = KNOWN_FAILURES.contains(&test_name);
- if path.iter().any(|p| p == "wasm32-wasip3") {
- cmd.arg("-Sp3,http").arg("-Wcomponent-model-async");
- if !cfg!(feature = "component-model-async") {
- should_fail = true;
- }
- }
- cmd.arg(path);
- cmd.args(args);
+ let spec_debug = format!("{spec:#?}");
+ let spec_world = spec.world.clone();
+ let mut child = KillOnDrop(None);
+ let mut cmd_debug = String::new();
+ let proposals = spec.proposals.clone();
+ let mut streams = HashMap::new();
+ let mut http_addr = None;
+ let mut killed = false;
- let result = cmd.output()?;
- td.disable_cleanup(true);
- let ok = spec == result;
- match (ok, should_fail) {
- // If this test passed and is not a known failure, or if it failed and
- // it's a known failure, then flag this test as "ok".
- (true, false) | (false, true) => Ok(()),
-
- // If this test failed and it's not known to fail, explain why.
- (false, false) => {
- td.disable_cleanup(false);
- let mut msg = String::new();
- writeln!(msg, " command: {cmd:?}")?;
- writeln!(msg, " spec: {spec:#?}")?;
- writeln!(msg, " result.status: {}", result.status)?;
- if !result.stdout.is_empty() {
- write!(
- msg,
- " result.stdout:\n {}",
- String::from_utf8_lossy(&result.stdout).replace("\n", "\n ")
- )?;
+ for operation in spec.operations() {
+ log::info!("execute: {operation:?}");
+ match operation {
+ Operation::Run { args, dirs, env } => {
+ assert!(child.0.is_none());
+ let mut cmd = wasmtime_test_util::command(wasmtime);
+ match spec_world.as_deref() {
+ Some("wasi:http/service") => {
+ cmd.arg("serve");
+ cmd.arg("--addr=127.0.0.1:0");
+ }
+ Some(world) => panic!("unknown world {world}"),
+ None => {
+ cmd.arg("run");
+ }
+ }
+ for dir in dirs {
+ cmd.arg("--dir");
+ let src = parent_dir.join(&dir);
+ let dst = td.path().join(&dir);
+ cp_r(&src, &dst)?;
+ cmd.arg(format!("{}::{dir}", dst.display()));
+ }
+ for (k, v) in env {
+ cmd.arg("--env");
+ cmd.arg(format!("{k}={v}"));
+ }
+ if path.iter().any(|p| p == "wasm32-wasip3") {
+ cmd.arg("-Sp3").arg("-Wcomponent-model-async");
+ }
+ for proposal in proposals.as_deref().unwrap_or(&[]) {
+ match proposal {
+ WasiProposal::Sockets => {
+ cmd.arg("-Sinherit-network");
+ }
+ WasiProposal::Http => {
+ cmd.arg("-Shttp,cli");
+ }
+ };
+ }
+ cmd.arg(&path);
+ cmd.args(args);
+ cmd.stdout(Stdio::piped());
+ cmd.stderr(Stdio::piped());
+ cmd.stdin(Stdio::piped());
+
+ cmd_debug = format!("{cmd:?}");
+ child.0 = Some(cmd.spawn()?);
}
- if !result.stderr.is_empty() {
- writeln!(
- msg,
- " result.stderr:\n {}",
- String::from_utf8_lossy(&result.stderr).replace("\n", "\n ")
- )?;
+ Operation::Write { id, payload } => {
+ let child = child.0.as_mut().unwrap();
+ let stream = match id {
+ StreamId::Stdin => child.stdin.as_mut().unwrap(),
+ StreamId::Stdout | StreamId::Stderr => {
+ panic!("cannot write to stdout or stderr")
+ }
+ };
+ stream.write_all(payload.as_bytes())?;
}
- wasmtime::bail!("{msg}\nFAILED! The result does not match the specification");
- }
+ Operation::Read { id, payload } => {
+ let child = child.0.as_mut().unwrap();
+ let stream = match id {
+ StreamId::Stdout => child.stdout.as_mut().unwrap() as &mut dyn Read,
+ StreamId::Stderr => child.stderr.as_mut().unwrap() as &mut dyn Read,
+ StreamId::Stdin => panic!("cannot read from stdin"),
+ };
+ let mut buf = vec![0; payload.len()];
+ stream.read_exact(&mut buf)?;
+ if payload != String::from_utf8_lossy(&buf) {
+ wasmtime::bail!(
+ "unexpected output from {id:?}: expected {payload:?}, got {:?}",
+ String::from_utf8_lossy(&buf)
+ );
+ }
+ }
+ Operation::Connect {
+ id,
+ protocol_type: ProtocolType::Tcp,
+ } => {
+ let child = child.0.as_mut().unwrap();
+ let mut buf = [0; 200];
+ let n = child.stdout.as_mut().unwrap().read(&mut buf)?;
+ let addr = String::from_utf8_lossy(&buf[..n]);
+ let stream = TcpStream::connect(addr.trim())?;
+ let prev = streams.insert(id, stream);
+ assert!(prev.is_none());
+ }
+ Operation::Send { id, payload } => {
+ let stream = streams.get_mut(&id).unwrap();
+ stream.write_all(payload.as_bytes())?;
+ }
+ Operation::Recv { id, payload } => {
+ let stream = streams.get_mut(&id).unwrap();
+ let mut buf = vec![0; payload.len()];
+ stream.read_exact(&mut buf)?;
+ if payload != String::from_utf8_lossy(&buf) {
+ wasmtime::bail!(
+ "unexpected output from stream {id:?}: expected {payload:?}, got {:?}",
+ String::from_utf8_lossy(&buf)
+ );
+ }
+ }
+ Operation::Request { req } => {
+ let http_addr = match &http_addr {
+ None => {
+ let child = child.0.as_mut().unwrap();
+ let mut buf = [0; 200];
+ let mut i = 0;
+ loop {
+ let n = child.stderr.as_mut().unwrap().read(&mut buf[i..])?;
+ assert!(n > 0);
+ i += n;
+ if buf[i - 1] == b'\n' {
+ break;
+ }
+ }
+ let addr = String::from_utf8_lossy(&buf[..i]);
+ let addr = addr
+ .trim()
+ .strip_prefix("Serving HTTP on http://")
+ .unwrap()
+ .strip_suffix("/")
+ .unwrap()
+ .parse::<SocketAddr>()
+ .unwrap();
+ http_addr = Some(addr);
+ addr
+ }
+ Some(addr) => *addr,
+ };
+
+ tokio::runtime::Builder::new_current_thread()
+ .enable_io()
+ .build()?
+ .block_on(send_request(http_addr, req))?;
+ }
+ Operation::Kill {
+ signal: Signal::Sigint,
+ } => {
+ let child = child.0.as_mut().unwrap();
+ child.kill()?;
+ killed = true;
+ }
+ Operation::Wait {
+ exit_code,
+ stderr,
+ stdout,
+ } => {
+ let result = child.0.take().unwrap().wait_with_output()?;
+ td.disable_cleanup(true);
+ let ok = (Some(exit_code.unwrap_or(0)) == result.status.code() || killed)
+ && matches_or_missing(&stdout, &result.stdout)
+ && matches_or_missing(&stderr, &result.stderr);
+ let mut should_fail = KNOWN_FAILURES.contains(&test_name);
+ if path.iter().any(|p| p == "wasm32-wasip3")
+ && !cfg!(feature = "component-model-async")
+ {
+ should_fail = true;
+ }
- // If this test passed but it's flagged as should be failed, then fail
- // this test for someone to update `KNOWN_FAILURES`.
- (true, true) => {
- wasmtime::bail!("test passed but it's listed in `KNOWN_FAILURES`")
+ match (ok, should_fail) {
+ // If this test passed and is not a known failure, or if it failed and
+ // it's a known failure, then flag this test as "ok".
+ (true, false) | (false, true) => {}
+
+ // If this test failed and it's not known to fail, explain why.
+ (false, false) => {
+ td.disable_cleanup(false);
+ let mut msg = String::new();
+ writeln!(msg, " command: {cmd_debug}")?;
+ writeln!(msg, " spec: {spec_debug}")?;
+ writeln!(msg, " result.status: {}", result.status)?;
+ if !result.stdout.is_empty() {
+ write!(
+ msg,
+ " result.stdout:\n {}",
+ String::from_utf8_lossy(&result.stdout).replace("\n", "\n ")
+ )?;
+ }
+ if !result.stderr.is_empty() {
+ writeln!(
+ msg,
+ " result.stderr:\n {}",
+ String::from_utf8_lossy(&result.stderr).replace("\n", "\n ")
+ )?;
+ }
+ wasmtime::bail!(
+ "{msg}\nFAILED! The result does not match the specification"
+ );
+ }
+
+ // If this test passed but it's flagged as should be failed, then fail
+ // this test for someone to update `KNOWN_FAILURES`.
+ (true, true) => {
+ wasmtime::bail!("test passed but it's listed in `KNOWN_FAILURES`")
+ }
+ }
+ }
}
}
+ assert!(child.0.is_none());
+ Ok(())
}
fn cp_r(path: &Path, dst: &Path) -> Result<()> {
@@ -254,27 +415,212 @@ fn cp_r(path: &Path, dst: &Path) -> Result<()> {
Ok(())
}
+struct KillOnDrop(Option<Child>);
+
+impl Drop for KillOnDrop {
+ fn drop(&mut self) {
+ if let Some(child) = &mut self.0 {
+ let _ = child.kill();
+ }
+ }
+}
+
+/// Sends `HttpRequest` to `addr`, asserting the response matches the expected
+/// response specified within `req`.
+async fn send_request(addr: SocketAddr, req: HttpRequest) -> Result<()> {
+ let tcp = TcpStream::connect(addr).with_context(|| format!("failed to connect to {addr:?}"))?;
+ tcp.set_nonblocking(true)?;
+ let tcp = tokio::net::TcpStream::from_std(tcp)?;
+ let tcp = wasmtime_wasi_http::io::TokioIo::new(tcp);
+ let (mut send, conn) = hyper::client::conn::http1::handshake(tcp)
+ .await
+ .context("failed http handshake")?;
+ tokio::task::spawn(conn);
+ let response = send
+ .send_request(
+ http::Request::builder()
+ .method(http::Method::from(req.method))
+ .uri(req.path)
+ .body(String::new())
+ .unwrap(),
+ )
+ .await
+ .context("error sending request")?;
+ let (parts, body) = response.into_parts();
+
+ let body = body.collect().await.context("failed to read body")?;
+ assert!(body.trailers().is_none());
+ let body = std::str::from_utf8(&body.to_bytes())?.to_string();
+
+ assert_eq!(parts.status.as_u16(), req.response.status);
+ for header in &req.response.headers {
+ let value = parts
+ .headers
+ .get(header.0.as_str())
+ .ok_or_else(|| format_err!("missing header {} in response", header.0))?;
+ if value != header.1.as_str() {
+ wasmtime::bail!(
+ "unexpected value for header {}: expected {:?}, got {:?}",
+ header.0,
+ header.1,
+ value.to_str()?
+ );
+ }
+ }
+ if body != req.response.body {
+ wasmtime::bail!(
+ "unexpected body: expected {:?}, got {:?}",
+ req.response.body,
+ body
+ );
+ }
+
+ Ok(())
+}
+
#[derive(Debug, Default, Deserialize)]
+#[serde(deny_unknown_fields)]
struct Spec {
- #[serde(default)]
- args: Vec<String>,
- #[serde(default)]
- dirs: Vec<String>,
- #[serde(default)]
- env: HashMap<String, String>,
+ proposals: Option<Vec<WasiProposal>>,
+ world: Option<String>,
+ operations: Option<Vec<Operation>>,
+
+ args: Option<Vec<String>>,
+ dirs: Option<Vec<String>>,
+ env: Option<HashMap<String, String>>,
exit_code: Option<i32>,
- stderr: Option<String>,
stdout: Option<String>,
}
-impl PartialEq<Output> for Spec {
- fn eq(&self, other: &Output) -> bool {
- self.exit_code.unwrap_or(0) == other.status.code().unwrap()
- && matches_or_missing(&self.stdout, &other.stdout)
- && matches_or_missing(&self.stderr, &other.stderr)
+impl Spec {
+ fn operations(mut self) -> Vec<Operation> {
+ if let Some(ops) = self.operations.take() {
+ assert!(self.args.is_none());
+ assert!(self.dirs.is_none());
+ assert!(self.env.is_none());
+ assert!(self.exit_code.is_none());
+ assert!(self.stdout.is_none());
+ return ops;
+ }
+
+ vec![
+ Operation::Run {
+ args: self.args.take().unwrap_or_default(),
+ dirs: self.dirs.take().unwrap_or_default(),
+ env: self.env.take().unwrap_or_default(),
+ },
+ Operation::Wait {
+ exit_code: self.exit_code,
+ stderr: None,
+ stdout: self.stdout.take(),
+ },
+ ]
+ }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
+enum Operation {
+ Run {
+ #[serde(default)]
+ args: Vec<String>,
+ #[serde(default)]
+ dirs: Vec<String>,
+ #[serde(default)]
+ env: HashMap<String, String>,
+ },
+ Write {
+ id: StreamId,
+ payload: String,
+ },
+ Read {
+ id: StreamId,
+ payload: String,
+ },
+ Connect {
+ id: String,
+ protocol_type: ProtocolType,
+ },
+ Send {
+ id: String,
+ payload: String,
+ },
+ Recv {
+ id: String,
+ payload: String,
+ },
+ Request {
+ #[serde(flatten)]
+ req: HttpRequest,
+ },
+ Kill {
+ signal: Signal,
+ },
+ Wait {
+ exit_code: Option<i32>,
+ stderr: Option<String>,
+ stdout: Option<String>,
+ },
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+enum StreamId {
+ Stdin,
+ Stdout,
+ Stderr,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+enum ProtocolType {
+ Tcp,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "UPPERCASE")]
+enum HttpMethod {
+ Get,
+ Post,
+}
+
+impl From<HttpMethod> for http::Method {
+ fn from(method: HttpMethod) -> Self {
+ match method {
+ HttpMethod::Get => http::Method::GET,
+ HttpMethod::Post => http::Method::POST,
+ }
}
}
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "UPPERCASE")]
+enum Signal {
+ Sigint,
+}
+#[derive(Debug, Deserialize)]
+struct HttpRequest {
+ method: HttpMethod,
+ path: String,
+ response: HttpResponse,
+}
+
+#[derive(Debug, Deserialize)]
+struct HttpResponse {
+ status: u16,
+ #[serde(default)]
+ headers: HashMap<String, String>,
+ #[serde(default)]
+ body: String,
+}
+
+#[derive(Debug, Deserialize, Clone, Copy)]
+#[serde(rename_all = "snake_case")]
+enum WasiProposal {
+ Sockets,
+ Http,
+}
+
fn matches_or_missing(a: &Option<String>, b: &[u8]) -> bool {
a.as_ref()
.map(|s| s == &String::from_utf8_lossy(b))