Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Optimize runtime of `test-artifacts` build script (#13089)

Alex Crichton09cb178cc585
diff --git a/Cargo.lock b/Cargo.lock
index c6b26aaa8f..c25835df72 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3803,6 +3803,7 @@ version = "0.0.0"
dependencies = [
"cargo_metadata",
"heck 0.5.0",
+ "rayon",
"serde",
"serde_derive",
"wasmtime",
diff --git a/crates/test-programs/artifacts/Cargo.toml b/crates/test-programs/artifacts/Cargo.toml
index 6481b4d442..d838499ec7 100644
--- a/crates/test-programs/artifacts/Cargo.toml
+++ b/crates/test-programs/artifacts/Cargo.toml
@@ -21,3 +21,4 @@ cargo_metadata = "0.19.2"
wasmtime-test-util = { workspace = true, features = ['wast'] }
serde_derive = { workspace = true }
serde = { workspace = true }
+rayon = { workspace = true }
diff --git a/crates/test-programs/artifacts/build.rs b/crates/test-programs/artifacts/build.rs
index cb9467aa88..37b632c941 100644
--- a/crates/test-programs/artifacts/build.rs
+++ b/crates/test-programs/artifacts/build.rs
@@ -1,9 +1,11 @@
use heck::*;
+use rayon::prelude::*;
use std::collections::{BTreeMap, HashSet};
use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
+use std::time::SystemTime;
use wit_component::ComponentEncoder;
fn main() {
@@ -62,6 +64,7 @@ impl Artifacts {
let mut kinds = BTreeMap::new();
let missing_sdk_path =
PathBuf::from("Asset not compiled, WASI_SDK_PATH missing at compile time");
+ let mut components = Vec::new();
for test in tests.iter() {
let shouty_snake = test.name.to_shouty_snake_case();
let snake = test.name.to_snake_case();
@@ -122,14 +125,20 @@ impl Artifacts {
{
continue;
}
- let adapter = match test.name.as_str() {
+ let (adapter, mtime) = match test.name.as_str() {
"reactor" => &reactor_adapter,
s if s.starts_with("p3_") => &reactor_adapter,
s if s.starts_with("p2_api_proxy") => &proxy_adapter,
_ => &command_adapter,
};
let path = match &test.core_wasm {
- Some(path) => self.compile_component(path, adapter),
+ Some(path) => {
+ let out_dir = path.parent().unwrap();
+ let stem = path.file_stem().unwrap().to_str().unwrap();
+ let component_path = out_dir.join(format!("{stem}.component.wasm"));
+ components.push((path, adapter, mtime.as_ref(), component_path.clone()));
+ component_path
+ }
None => missing_sdk_path.clone(),
};
generated_code +=
@@ -141,6 +150,12 @@ impl Artifacts {
);
}
+ components
+ .par_iter()
+ .for_each(|(wasm, adapter, mtime, component_path)| {
+ self.compile_component(wasm, adapter, *mtime, component_path);
+ });
+
for (kind, targets) in kinds {
generated_code += &format!("#[macro_export]");
generated_code += &format!("macro_rules! foreach_{kind} {{\n");
@@ -206,13 +221,15 @@ impl Artifacts {
generated_code: &mut String,
name: &str,
features: &[&str],
- ) -> Vec<u8> {
+ ) -> (Vec<u8>, Option<SystemTime>) {
let mut cmd = cargo();
cmd.arg("build")
.arg("--release")
+ .arg("-vv")
.arg("--package=wasi-preview1-component-adapter")
.arg("--target=wasm32-unknown-unknown")
- .env("CARGO_TARGET_DIR", &self.out_dir)
+ .env("CARGO_BUILD_BUILD_DIR", &self.out_dir)
+ .env("CARGO_TARGET_DIR", &self.out_dir.join(name))
.env("RUSTFLAGS", rustflags())
.env_remove("CARGO_ENCODED_RUSTFLAGS");
for f in features {
@@ -224,24 +241,55 @@ impl Artifacts {
let artifact = self
.out_dir
+ .join(name)
.join("wasm32-unknown-unknown")
.join("release")
.join("wasi_snapshot_preview1.wasm");
let adapter = self
.out_dir
.join(format!("wasi_snapshot_preview1.{name}.wasm"));
- std::fs::copy(&artifact, &adapter).unwrap();
+
+ if let Ok(prev) = std::fs::read(&adapter)
+ && let Ok(cur) = std::fs::read(&artifact)
+ && prev == cur
+ {
+ // nothing to do ...
+ } else {
+ if adapter.exists() {
+ fs::remove_file(&adapter).unwrap();
+ }
+ std::fs::hard_link(&artifact, &adapter)
+ .or_else(|_| std::fs::copy(&artifact, &adapter).map(|_| ()))
+ .unwrap();
+ }
self.read_deps_of(&artifact);
println!("wasi {name} adapter: {:?}", &adapter);
generated_code.push_str(&format!(
"pub const ADAPTER_{}: &'static str = {adapter:?};\n",
name.to_shouty_snake_case(),
));
- fs::read(&adapter).unwrap()
+ (fs::read(&adapter).unwrap(), mtime(&adapter))
}
// Compile a component, return the path of the binary:
- fn compile_component(&self, wasm: &Path, adapter: &[u8]) -> PathBuf {
+ fn compile_component(
+ &self,
+ wasm: &Path,
+ adapter: &[u8],
+ adapter_mtime: Option<&SystemTime>,
+ component_path: &Path,
+ ) {
+ // If the component exists and was last updated after the inputs that
+ // make it up then there's no need to recreate it.
+ if let Some(wasm_mtime) = mtime(wasm)
+ && let Some(adapter_mtime) = adapter_mtime
+ && let Some(component_mtime) = mtime(&component_path)
+ && wasm_mtime < component_mtime
+ && *adapter_mtime < component_mtime
+ {
+ println!("reusing cached component for {wasm:?}");
+ return;
+ }
println!("creating a component from {wasm:?}");
let module = fs::read(wasm).expect("read wasm module");
let component = ComponentEncoder::default()
@@ -252,51 +300,55 @@ impl Artifacts {
.unwrap()
.encode()
.expect("module can be translated to a component");
- let out_dir = wasm.parent().unwrap();
- let stem = wasm.file_stem().unwrap().to_str().unwrap();
- let component_path = out_dir.join(format!("{stem}.component.wasm"));
fs::write(&component_path, component).expect("write component to disk");
- component_path
}
fn build_non_rust_tests(&mut self, tests: &mut Vec<Test>) {
const ASSETS_REL_SRC_DIR: &'static str = "../src/bin";
println!("cargo:rerun-if-changed={ASSETS_REL_SRC_DIR}");
- for entry in fs::read_dir(ASSETS_REL_SRC_DIR).unwrap() {
- let entry = entry.unwrap();
- let path = entry.path();
- let name = path.file_stem().unwrap().to_str().unwrap().to_owned();
- match path.extension().and_then(|s| s.to_str()) {
- // Compile C/C++ tests with clang
- Some("c") | Some("cc") => self.build_c_or_cpp_test(path, name, tests),
-
- // just a header, part of another test.
- Some("h") => {}
-
- // Convert the text format to binary and use it as a test.
- Some("wat") => {
- let wasm = wat::parse_file(&path).unwrap();
- let core_wasm = self.out_dir.join(&name).with_extension("wasm");
- fs::write(&core_wasm, &wasm).unwrap();
- tests.push(Test {
- name,
- core_wasm: Some(core_wasm),
- });
+ let entries = fs::read_dir(ASSETS_REL_SRC_DIR)
+ .unwrap()
+ .map(|e| e.unwrap())
+ .collect::<Vec<_>>();
+ let mut c_tests = entries
+ .par_iter()
+ .flat_map(|entry| {
+ let path = entry.path();
+ let name = path.file_stem().unwrap().to_str().unwrap().to_owned();
+ match path.extension().and_then(|s| s.to_str()) {
+ // Compile C/C++ tests with clang
+ Some("c") | Some("cc") => self.build_c_or_cpp_test(path, name),
+
+ // just a header, part of another test.
+ Some("h") => None,
+
+ // Convert the text format to binary and use it as a test.
+ Some("wat") => {
+ let wasm = wat::parse_file(&path).unwrap();
+ let core_wasm = self.out_dir.join(&name).with_extension("wasm");
+ fs::write(&core_wasm, &wasm).unwrap();
+ Some(Test {
+ name,
+ core_wasm: Some(core_wasm),
+ })
+ }
+
+ // these are built above in `build_rust_tests`
+ Some("rs") => None,
+
+ // Prevent stray files for now that we don't understand.
+ Some(_) => panic!("unknown file extension on {path:?}"),
+
+ None => unreachable!("no extension in path {path:?}"),
}
-
- // these are built above in `build_rust_tests`
- Some("rs") => {}
-
- // Prevent stray files for now that we don't understand.
- Some(_) => panic!("unknown file extension on {path:?}"),
-
- None => unreachable!("no extension in path {path:?}"),
- }
- }
+ })
+ .collect::<Vec<_>>();
+ c_tests.sort_by_key(|t| t.name.clone());
+ tests.extend(c_tests);
}
- fn build_c_or_cpp_test(&mut self, path: PathBuf, name: String, tests: &mut Vec<Test>) {
+ fn build_c_or_cpp_test(&self, path: PathBuf, name: String) -> Option<Test> {
println!("compiling {path:?}");
println!("cargo:rerun-if-changed={}", path.display());
let contents = std::fs::read_to_string(&path).unwrap();
@@ -304,7 +356,7 @@ impl Artifacts {
wasmtime_test_util::wast::parse_test_config::<CTestConfig>(&contents, "//!").unwrap();
if config.skip {
- return;
+ return None;
}
// The debug tests relying on these assets are ignored by default,
@@ -317,11 +369,10 @@ impl Artifacts {
let wasi_sdk_path = match env::var_os("WASI_SDK_PATH") {
Some(path) => PathBuf::from(path),
None => {
- tests.push(Test {
+ return Some(Test {
name,
core_wasm: None,
});
- return;
}
};
@@ -352,7 +403,7 @@ impl Artifacts {
assert!(dwp.status().expect("failed to spawn llvm-dwp").success());
}
- tests.push(Test {
+ return Some(Test {
name,
core_wasm: Some(wasm_path),
});
@@ -425,3 +476,7 @@ fn rustflags() -> &'static str {
_ => "",
}
}
+
+fn mtime(path: &Path) -> Option<SystemTime> {
+ path.metadata().ok()?.modified().ok()
+}