A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Compose compile-time builtins in memory (#15025)
dyslexicsteakb34fad65f3b7
diff --git a/crates/misc/component-async-tests/Cargo.toml b/crates/misc/component-async-tests/Cargo.tomlindex 1a6f1c406d..01283fe67b 100644--- a/crates/misc/component-async-tests/Cargo.toml+++ b/crates/misc/component-async-tests/Cargo.toml@@ -12,7 +12,6 @@ workspace = true[dependencies]futures = { workspace = true, features = ['async-await'] }-tempfile = { workspace = true }tokio = { workspace = true, features = ["fs","process",@@ -36,3 +35,4 @@ wasm-compose = { workspace = true }test-programs-artifacts = { workspace = true }bytes = { workspace = true }once_cell = { version = "1.12.0" }+tempfile = { workspace = true }diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.tomlindex d93b8809a4..3d5fd5756a 100644--- a/crates/wasmtime/Cargo.toml+++ b/crates/wasmtime/Cargo.toml@@ -36,7 +36,6 @@ wasm-encoder = { workspace = true, optional = true }wasm-wave = { workspace = true, optional = true }wasm-compose = { workspace = true, optional = true }wit-parser = { workspace = true, optional = true }-tempfile = { workspace = true, optional = true }libc = { workspace = true }log = { workspace = true }wat = { workspace = true, optional = true }@@ -426,7 +425,7 @@ debug = []# Enables support for defining compile-time builtins.-compile-time-builtins = ['anyhow', 'dep:wasm-compose', 'dep:tempfile']+compile-time-builtins = ['anyhow', 'dep:wasm-compose']# Enable support for the common base infrastructure of record/replayrr = [diff --git a/crates/wasmtime/src/compile/code_builder/compile_time_builtins.rs b/crates/wasmtime/src/compile/code_builder/compile_time_builtins.rsindex aaf111ccb1..e30526ddf6 100644--- a/crates/wasmtime/src/compile/code_builder/compile_time_builtins.rs+++ b/crates/wasmtime/src/compile/code_builder/compile_time_builtins.rs@@ -1,5 +1,84 @@use super::*;use crate::ToWasmtimeResult as _;+use wasm_compose::graph::{CompositionGraph, EncodeOptions, ExportIndex, ImportIndex, InstanceId};++/// Returns the instance imports of `component`, in declaration order.+fn instance_imports<'c>(+ component: &'c wasm_compose::graph::Component<'_>,+) -> impl Iterator<Item = (ImportIndex, &'c str)> {+ component.imports().filter_map(|(index, name, ty)| {+ matches!(ty, wasmparser::ComponentTypeRef::Instance(_)).then_some((index, name))+ })+}++/// Composes `main_wasm` with the compile-time builtins it imports.+///+/// Each imported builtin is instantiated once and satisfies every instance+/// import named after it, including other builtins' imports. Other imports,+/// such as the unsafe intrinsics, remain imports, merged by name.+fn compose(+ main_wasm: &[u8],+ builtins: &HashMap<Cow<'_, str>, Cow<'_, [u8]>>,+ features: wasmparser::WasmFeatures,+) -> Result<Vec<u8>> {+ let mut validator = wasmparser::Validator::new_with_features(features);+ let mut graph = CompositionGraph::new();++ let main = wasm_compose::graph::Component::from_bytes(&mut validator, "main", main_wasm)+ .to_wasmtime_result()?;+ let main_imports: Vec<String> = instance_imports(&main)+ .map(|(_, name)| name.to_owned())+ .collect();+ let main = graph.add_component(main).to_wasmtime_result()?;+ let main = graph.instantiate(main).to_wasmtime_result()?;++ // Instantiate in import order, so the output doesn't depend on hash map+ // order.+ let mut instances: Vec<(&str, InstanceId)> = Vec::new();+ for name in &main_imports {+ let Some((name, bytes)) = builtins.get_key_value(name.as_str()) else {+ continue;+ };++ // Graph labels must be unique and valid but aren't encoded, so number+ // them rather than reuse import names, which could clash with `main`.+ let label = format!("builtin{}", instances.len());++ // wasm-compose's errors use the label, so name the builtin here.+ let component = wasm_compose::graph::Component::from_bytes(&mut validator, label, &**bytes)+ .to_wasmtime_result()+ .with_context(|| format!("invalid compile-time builtin `{name}`"))?;+ let component = graph.add_component(component).to_wasmtime_result()?;++ instances.push((name, graph.instantiate(component).to_wasmtime_result()?));+ }++ // Collect the connections first, since connecting needs the graph+ // mutably.+ let mut connections: Vec<(InstanceId, InstanceId, ImportIndex)> = Vec::new();+ for target in core::iter::once(main).chain(instances.iter().map(|&(_, instance)| instance)) {+ let (_, component) = graph.get_component_of_instance(target).unwrap();+ for (import, name) in instance_imports(component) {+ if let Some(&(_, source)) = instances.iter().find(|&&(builtin, _)| builtin == name) {+ connections.push((source, target, import));+ }+ }+ }++ for (source, target, import) in connections {+ graph+ .connect(source, None::<ExportIndex>, target, import)+ .to_wasmtime_result()?;+ }++ graph+ .encode(EncodeOptions {+ define_components: true,+ export: Some(main),+ validate: false,+ })+ .to_wasmtime_result()+}impl<'a> CodeBuilder<'a> {pub(crate) fn get_compile_time_builtins(&self) -> &HashMap<Cow<'a, str>, Cow<'a, [u8]>> {@@ -15,48 +94,22 @@ impl<'a> CodeBuilder<'a> {}let imports = self.check_imports_for_compile_time_builtins(&main_wasm)?;- if imports.is_empty() {+ let builtins = self.get_compile_time_builtins();++ // Nothing to compose if the main Wasm imports none of the builtins.+ if !imports.iter().any(|import| builtins.contains_key(*import)) {drop(imports);return Ok(main_wasm.into());}- let tempdir = tempfile::TempDir::new().context("failed to create a temporary directory")?;- let deps = tempdir.path().join("_deps");- std::fs::create_dir(&deps)- .with_context(|| format!("failed to create directory: {}", deps.display()))?;-- let main_wasm_path = tempdir.path().join("_main.wasm");- std::fs::write(&main_wasm_path, &main_wasm)- .with_context(|| format!("failed to write to file: {}", main_wasm_path.display()))?;-- let mut config = wasm_compose::config::Config::default();- for (name, bytes) in self.get_compile_time_builtins() {- let name: &str = &*name;- if !imports.contains(&name) {- continue;- }-- let mut path = deps.join(Path::new(name));- path.set_extension("wasm");-- std::fs::write(&path, &bytes)- .with_context(|| format!("failed to write to file: {}", path.display()))?;-- config- .dependencies- .insert(name.to_string(), wasm_compose::config::Dependency { path });- }-- let composer = wasm_compose::composer::ComponentComposer::new(&main_wasm_path, &config);- let composed = composer.compose().to_wasmtime_result()?;- Ok(composed.into())+ Ok(compose(main_wasm, builtins, self.engine.features())?.into())}/// Check that the main Wasm doesn't import unsafe intrinsics, keeping the/// TCB to just the compile-time builtins' implementation.///- /// Returns the Wasm's top-level instance imports for `wasm-compose`- /// configuration.+ /// Returns the Wasm's top-level instance imports, so that composition can+ /// be skipped when none of them is a compile-time builtin.fn check_imports_for_compile_time_builtins<'b>(&self,main_wasm: &'b [u8],diff --git a/tests/all/compile_time_builtins.rs b/tests/all/compile_time_builtins.rsindex 3218538d44..c8e4d61f2a 100644--- a/tests/all/compile_time_builtins.rs+++ b/tests/all/compile_time_builtins.rs@@ -263,3 +263,305 @@ fn import_erased() -> Result<()> {Ok(())}++/// A compile-time builtin that exports `get`, returning `value`.+fn constant_builtin(value: u32) -> String {+ format!(+ r#"+ (component+ (core module $m+ (func (export "get") (result i32)+ (i32.const {value})+ )+ )++ (core instance $i (instantiate $m))++ (func (export "get") (result u32)+ (canon lift (core func $i "get"))+ )+ )+ "#+ )+}++/// A main Wasm that imports each of `names` as an instance exporting `get`,+/// and exports `sum`, the sum of every `get`.+fn summing_main(names: &[&str]) -> String {+ let mut imports = String::new();+ let mut lowers = String::new();+ let mut params = String::new();+ let mut calls = String::from("(i32.const 0)");+ let mut args = String::new();++ for (i, name) in names.iter().enumerate() {+ imports += &format!(+ r#"(import "{name}" (instance $import{i} (export "get" (func (result u32)))))"#+ );+ lowers += &format!(r#"(core func $get{i} (canon lower (func $import{i} "get")))"#);+ params += &format!(r#"(import "" "get{i}" (func $get{i} (result i32)))"#);+ calls = format!("(i32.add {calls} (call $get{i}))");+ args += &format!(r#"(export "get{i}" (func $get{i}))"#);+ }++ format!(+ r#"+ (component+ {imports}+ {lowers}++ (core module $m+ {params}+ (func (export "sum") (result i32)+ {calls}+ )+ )++ (core instance $i (instantiate $m (with "" (instance {args}))))++ (func (export "sum") (result u32)+ (canon lift (core func $i "sum"))+ )+ )+ "#+ )+}++/// Instantiates `component` and returns the result of calling its `sum`.+fn call_sum(engine: &Engine, component: &component::Component) -> Result<u32> {+ let linker = component::Linker::new(engine);+ let mut store = Store::new(engine, ());+ let instance = linker.instantiate(&mut store, component)?;+ let (result,) = instance+ .get_typed_func::<(), (u32,)>(&mut store, "sum")?+ .call(&mut store, ())?;++ Ok(result)+}++#[test]+#[cfg_attr(miri, ignore)]+fn interface_name() -> Result<()> {+ let engine = Engine::default();++ let component = unsafe {+ CodeBuilder::new(&engine)+ .expose_unsafe_intrinsics("unsafe-intrinsics")+ .compile_time_builtins_binary_or_text(+ "my:pkg/[email protected]",+ constant_builtin(42).as_bytes(),+ Some(Path::new("if.wat")),+ )?+ .wasm_binary_or_text(+ summing_main(&["my:pkg/[email protected]"]).as_bytes(),+ Some(Path::new("main.wat")),+ )?+ .compile_component()?+ };++ assert_eq!(call_sum(&engine, &component)?, 42);+ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn versions_of_one_interface() -> Result<()> {+ let engine = Engine::default();++ let component = unsafe {+ CodeBuilder::new(&engine)+ .expose_unsafe_intrinsics("unsafe-intrinsics")+ .compile_time_builtins_binary_or_text(+ "a:b/[email protected]",+ constant_builtin(1).as_bytes(),+ Some(Path::new("c-1.0.0.wat")),+ )?+ .compile_time_builtins_binary_or_text(+ "a:b/[email protected]",+ constant_builtin(10).as_bytes(),+ Some(Path::new("c-1.0.1.wat")),+ )?+ .wasm_binary_or_text(+ summing_main(&["a:b/[email protected]", "a:b/[email protected]"]).as_bytes(),+ Some(Path::new("main.wat")),+ )?+ .compile_component()?+ };++ assert_eq!(call_sum(&engine, &component)?, 11);+ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn builtins_share_intrinsics() -> Result<()> {+ // Both builtins' intrinsics imports merge into one.+ const BUILTIN: &str = r#"+ (component+ (import "unsafe-intrinsics"+ (instance $intrinsics+ (export "store-data-address" (func (result u64)))+ (export "u8-native-load" (func (param "pointer" u64) (result u8)))+ )+ )++ (core func $store-data-address' (canon lower (func $intrinsics "store-data-address")))+ (core func $u8-native-load' (canon lower (func $intrinsics "u8-native-load")))++ (core module $m+ (import "" "store-data-address" (func $store-data-address (result i64)))+ (import "" "u8-native-load" (func $u8-native-load (param i64) (result i32)))+ (func (export "get") (result i32)+ (call $u8-native-load (call $store-data-address))+ )+ )++ (core instance $i+ (instantiate $m+ (with "" (instance (export "store-data-address" (func $store-data-address'))+ (export "u8-native-load" (func $u8-native-load'))))+ )+ )++ (func (export "get") (result u32)+ (canon lift (core func $i "get"))+ )+ )+ "#;++ let engine = Engine::default();+ let linker = component::Linker::new(&engine);++ let component = unsafe {+ CodeBuilder::new(&engine)+ .expose_unsafe_intrinsics("unsafe-intrinsics")+ .compile_time_builtins_binary_or_text(+ "a:b/first",+ BUILTIN.as_bytes(),+ Some(Path::new("first.wat")),+ )?+ .compile_time_builtins_binary_or_text(+ "a:b/second",+ BUILTIN.as_bytes(),+ Some(Path::new("second.wat")),+ )?+ .wasm_binary_or_text(+ summing_main(&["a:b/first", "a:b/second"]).as_bytes(),+ Some(Path::new("main.wat")),+ )?+ .compile_component()?+ };++ let mut store = Store::new(&engine, 42_u8);+ let instance = linker.instantiate(&mut store, &component)?;++ let (result,) = instance+ .get_typed_func::<(), (u32,)>(&mut store, "sum")?+ .call(&mut store, ())?;++ assert_eq!(result, 84);++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn unused_compile_time_builtins_are_not_instantiated() -> Result<()> {+ // `a:b/unused` traps on instantiation, so this only works if it's left out.+ const TRAPS: &str = r#"+ (component+ (core module $m+ (func $trap unreachable)+ (start $trap)+ )+ (core instance (instantiate $m))+ )+ "#;++ let engine = Engine::default();++ let component = unsafe {+ CodeBuilder::new(&engine)+ .expose_unsafe_intrinsics("unsafe-intrinsics")+ .compile_time_builtins_binary_or_text(+ "a:b/used",+ constant_builtin(42).as_bytes(),+ Some(Path::new("used.wat")),+ )?+ .compile_time_builtins_binary_or_text(+ "a:b/unused",+ TRAPS.as_bytes(),+ Some(Path::new("unused.wat")),+ )?+ .wasm_binary_or_text(+ summing_main(&["a:b/used"]).as_bytes(),+ Some(Path::new("main.wat")),+ )?+ .compile_component()?+ };++ assert_eq!(call_sum(&engine, &component)?, 42);+ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn invalid_compile_time_builtins_are_named_in_errors() -> Result<()> {+ // Exports a function that doesn't exist, so it fails validation.+ let engine = Engine::default();++ let err = unsafe {+ CodeBuilder::new(&engine)+ .expose_unsafe_intrinsics("unsafe-intrinsics")+ .compile_time_builtins_binary_or_text(+ "a:b/broken",+ r#"(component (export "get" (func 0)))"#.as_bytes(),+ Some(Path::new("broken.wat")),+ )?+ .wasm_binary_or_text(+ summing_main(&["a:b/broken"]).as_bytes(),+ Some(Path::new("main.wat")),+ )?+ .compile_component()+ .map(|_| ())+ .unwrap_err()+ };++ err.assert_contains("`a:b/broken`");+ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn unused_compile_time_builtins_with_other_imports() -> Result<()> {+ let engine = Engine::default();++ let component = unsafe {+ CodeBuilder::new(&engine)+ .expose_unsafe_intrinsics("unsafe-intrinsics")+ .compile_time_builtins_binary_or_text(+ "compile-time-api",+ "(component)".as_bytes(),+ Some(Path::new("compile-time-api.wat")),+ )?+ .wasm_binary_or_text(+ r#"+ (component+ (import "link-time-api" (instance))+ )+ "#+ .as_bytes(),+ Some(Path::new("main.wat")),+ )?+ .compile_component()?+ };++ let component_type = component.component_type();+ let imports = component_type+ .imports(&engine)+ .map(|(name, _ty)| name)+ .collect::<Vec<_>>();+ assert_eq!(imports, ["link-time-api"]);++ Ok(())+}