A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
wizer: Don't copy all memory out of a store (#11891)
Alex Crichtonc7dc0eb15764
diff --git a/crates/wizer/src/component.rs b/crates/wizer/src/component.rsindex f87a4245bf..ec0385538d 100644--- a/crates/wizer/src/component.rs+++ b/crates/wizer/src/component.rs@@ -58,7 +58,8 @@ pub trait ComponentInstanceState: Send {&mut self,instance: &str,func: &str,- ) -> impl Future<Output = Vec<u8>> + Send;+ contents: impl FnOnce(&[u8]) + Send,+ ) -> impl Future<Output = ()> + Send;/// Same as [`Self::call_func_ret_list_u8`], but for the `s32` WIT type.fn call_func_ret_s32(&mut self, instance: &str, func: &str)diff --git a/crates/wizer/src/component/snapshot.rs b/crates/wizer/src/component/snapshot.rsindex 00f401eaf5..a78e86dc7a 100644--- a/crates/wizer/src/component/snapshot.rs+++ b/crates/wizer/src/component/snapshot.rs@@ -115,7 +115,7 @@ where}}- async fn memory_contents(&mut self, name: &str) -> Vec<u8> {+ async fn memory_contents(&mut self, name: &str, contents: impl FnOnce(&[u8]) + Send) {let Accessor::Memory {accessor_export_name,..@@ -124,7 +124,7 @@ wherepanic!("expected memory accessor for {name}");};self.ctx- .call_func_ret_list_u8(WIZER_INSTANCE, accessor_export_name)+ .call_func_ret_list_u8(WIZER_INSTANCE, accessor_export_name, contents).await}}diff --git a/crates/wizer/src/component/wasmtime.rs b/crates/wizer/src/component/wasmtime.rsindex 83d8370d52..bf9f7d87e6 100644--- a/crates/wizer/src/component/wasmtime.rs+++ b/crates/wizer/src/component/wasmtime.rs@@ -1,7 +1,9 @@use crate::Wizer;use crate::component::ComponentInstanceState;use anyhow::{Context, anyhow};-use wasmtime::component::{Component, ComponentExportIndex, Instance, Lift, types::ComponentItem};+use wasmtime::component::{+ Component, ComponentExportIndex, Instance, Lift, WasmList, types::ComponentItem,+};use wasmtime::{Result, Store};impl Wizer {@@ -84,7 +86,12 @@ pub struct WasmtimeWizerComponent<'a, T: 'static> {}impl<T: Send> WasmtimeWizerComponent<'_, T> {- async fn call_func<R>(&mut self, instance: &str, func: &str) -> R+ async fn call_func<R, R2>(+ &mut self,+ instance: &str,+ func: &str,+ use_ret: impl FnOnce(&mut Store<T>, R) -> R2,+ ) -> R2whereR: Lift + 'static,{@@ -102,29 +109,40 @@ impl<T: Send> WasmtimeWizerComponent<'_, T> {.get_typed_func::<(), (R,)>(&mut *self.store, func_export).unwrap();let ret = func.call_async(&mut *self.store, ()).await.unwrap().0;+ let ret = use_ret(&mut *self.store, ret);func.post_return_async(&mut *self.store).await.unwrap();ret}}impl<T: Send> ComponentInstanceState for WasmtimeWizerComponent<'_, T> {- async fn call_func_ret_list_u8(&mut self, instance: &str, func: &str) -> Vec<u8> {- self.call_func(instance, func).await+ async fn call_func_ret_list_u8(+ &mut self,+ instance: &str,+ func: &str,+ contents: impl FnOnce(&[u8]) + Send,+ ) {+ self.call_func(instance, func, |store, list: WasmList<u8>| {+ contents(list.as_le_slice(&store));+ })+ .await}async fn call_func_ret_s32(&mut self, instance: &str, func: &str) -> i32 {- self.call_func(instance, func).await+ self.call_func(instance, func, |_, r| r).await}async fn call_func_ret_s64(&mut self, instance: &str, func: &str) -> i64 {- self.call_func(instance, func).await+ self.call_func(instance, func, |_, r| r).await}async fn call_func_ret_f32(&mut self, instance: &str, func: &str) -> u32 {- self.call_func::<f32>(instance, func).await.to_bits()+ self.call_func(instance, func, |_, r: f32| r.to_bits())+ .await}async fn call_func_ret_f64(&mut self, instance: &str, func: &str) -> u64 {- self.call_func::<f64>(instance, func).await.to_bits()+ self.call_func(instance, func, |_, r: f64| r.to_bits())+ .await}}diff --git a/crates/wizer/src/lib.rs b/crates/wizer/src/lib.rsindex 3975f8dcb3..f5f04189c6 100644--- a/crates/wizer/src/lib.rs+++ b/crates/wizer/src/lib.rs@@ -386,5 +386,9 @@ pub trait InstanceState {/// # Panics////// This function panics if `name` isn't an exported memory.- fn memory_contents(&mut self, name: &str) -> impl Future<Output = Vec<u8>> + Send;+ fn memory_contents(+ &mut self,+ name: &str,+ contents: impl FnOnce(&[u8]) + Send,+ ) -> impl Future<Output = ()> + Send;}diff --git a/crates/wizer/src/rewrite.rs b/crates/wizer/src/rewrite.rsindex df37e6b645..5f854af485 100644--- a/crates/wizer/src/rewrite.rs+++ b/crates/wizer/src/rewrite.rs@@ -25,11 +25,12 @@ impl Wizer {// than the original, uninitialized data segments.let add_data_segments = |data_section: &mut wasm_encoder::DataSection| {for seg in &snapshot.data_segments {- data_section.active(- seg.memory_index,- &ConstExpr::i32_const(seg.offset as i32),- seg.data().iter().copied(),- );+ let offset = if seg.is64 {+ ConstExpr::i64_const(seg.offset.cast_signed())+ } else {+ ConstExpr::i32_const(u32::try_from(seg.offset).unwrap().cast_signed())+ };+ data_section.active(seg.memory_index, &offset, seg.data.iter().copied());}};diff --git a/crates/wizer/src/snapshot.rs b/crates/wizer/src/snapshot.rsindex 387e856ab3..cbd246770d 100644--- a/crates/wizer/src/snapshot.rs+++ b/crates/wizer/src/snapshot.rs@@ -2,7 +2,7 @@ use crate::InstanceState;use crate::info::ModuleContext;use rayon::iter::{IntoParallelIterator, ParallelExtend, ParallelIterator};use std::convert::TryFrom;-use std::sync::Arc;+use std::ops::Range;/// The maximum number of data segments that we will emit. Most/// engines support more than this, but we want to leave some@@ -41,47 +41,13 @@ pub struct DataSegment {pub memory_index: u32,/// This data segment's initialized memory that it originated from.- pub memory: Arc<Vec<u8>>,+ pub data: Vec<u8>,/// The offset within the memory that `data` should be copied to.- pub offset: u32,+ pub offset: u64,- /// This segment's length.- pub len: u32,-}--impl DataSegment {- pub fn data(&self) -> &[u8] {- let start = usize::try_from(self.offset).unwrap();- let end = start + usize::try_from(self.len).unwrap();- &self.memory[start..end]- }-}--impl DataSegment {- /// What is the gap between two consecutive data segments?- ///- /// `self` must be in front of `other` and they must not overlap with each- /// other.- fn gap(&self, other: &Self) -> u32 {- debug_assert_eq!(self.memory_index, other.memory_index);- debug_assert!(self.offset + self.len <= other.offset);- other.offset - (self.offset + self.len)- }-- /// Merge two consecutive data segments.- ///- /// `self` must be in front of `other` and they must not overlap with each- /// other.- fn merge(&self, other: &Self) -> DataSegment {- let gap = self.gap(other);-- DataSegment {- offset: self.offset,- len: self.len + gap + other.len,- ..self.clone()- }- }+ /// Whether or not `memory_index` is a 64-bit memory.+ pub is64: bool,}/// Snapshot the given instance's globals, memories, and instances from the Wasm@@ -116,6 +82,34 @@ async fn snapshot_globals(ret}+#[derive(Clone)]+struct DataSegmentRange {+ memory_index: u32,+ range: Range<usize>,+}++impl DataSegmentRange {+ /// What is the gap between two consecutive data segments?+ ///+ /// `self` must be in front of `other` and they must not overlap with each+ /// other.+ fn gap(&self, other: &Self) -> usize {+ debug_assert_eq!(self.memory_index, other.memory_index);+ debug_assert!(self.range.end <= other.range.start);+ other.range.start - self.range.end+ }++ /// Merge two consecutive data segments.+ ///+ /// `self` must be in front of `other` and they must not overlap with each+ /// other.+ fn merge(&mut self, other: &Self) {+ debug_assert_eq!(self.memory_index, other.memory_index);+ debug_assert!(self.range.end <= other.range.start);+ self.range.end = other.range.end;+ }+}+/// Find the initialized minimum page size of each memory, as well as all/// regions of non-zero memory.async fn snapshot_memories(@@ -131,51 +125,52 @@ async fn snapshot_memories(.defined_memories().zip(module.defined_memory_exports.as_ref().unwrap());for ((memory_index, ty), name) in iter {- let memory = Arc::new(instance.memory_contents(&name).await);- let page_size = 1 << ty.page_size_log2.unwrap_or(16);- let num_wasm_pages = memory.len() / page_size;- memory_mins.push(num_wasm_pages as u64);-- let memory_data = &memory[..];-- // Consider each Wasm page in parallel. Create data segments for each- // region of non-zero memory.- data_segments.par_extend((0..num_wasm_pages).into_par_iter().flat_map(|i| {- let page_end = (i + 1) * page_size;- let mut start = i * page_size;- let mut segments = vec![];- while start < page_end {- let nonzero = match memory_data[start..page_end]- .iter()- .position(|byte| *byte != 0)- {- None => break,- Some(i) => i,- };- start += nonzero;- let end = memory_data[start..page_end]- .iter()- .position(|byte| *byte == 0)- .map_or(page_end, |zero| start + zero);- segments.push(DataSegment {- memory_index,- memory: memory.clone(),- offset: u32::try_from(start).unwrap(),- len: u32::try_from(end - start).unwrap(),- });- start = end;- }- segments- }));+ instance+ .memory_contents(&name, |memory| {+ let page_size = 1 << ty.page_size_log2.unwrap_or(16);+ let num_wasm_pages = memory.len() / page_size;+ memory_mins.push(num_wasm_pages as u64);++ let memory_data = &memory[..];++ // Consider each Wasm page in parallel. Create data segments for each+ // region of non-zero memory.+ data_segments.par_extend((0..num_wasm_pages).into_par_iter().flat_map(|i| {+ let page_end = (i + 1) * page_size;+ let mut start = i * page_size;+ let mut segments = vec![];+ while start < page_end {+ let nonzero = match memory_data[start..page_end]+ .iter()+ .position(|byte| *byte != 0)+ {+ None => break,+ Some(i) => i,+ };+ start += nonzero;+ let end = memory_data[start..page_end]+ .iter()+ .position(|byte| *byte == 0)+ .map_or(page_end, |zero| start + zero);+ segments.push(DataSegmentRange {+ memory_index,+ range: start..end,+ });+ start = end;+ }+ segments+ }));+ })+ .await;}if data_segments.is_empty() {- return (memory_mins, data_segments);+ return (memory_mins, Vec::new());}// Sort data segments to enforce determinism in the face of the// parallelism above.- data_segments.sort_by_key(|s| (s.memory_index, s.offset));+ data_segments.sort_by_key(|s| (s.memory_index, s.range.start));// Merge any contiguous segments (caused by spanning a Wasm page boundary,// and therefore created in separate logical threads above) or pages that@@ -184,7 +179,7 @@ async fn snapshot_memories(// LEB, two for the memory offset init expression (one for the `i32.const`// opcode and another for the constant immediate LEB), and finally one for// the data length LEB).- const MIN_ACTIVE_SEGMENT_OVERHEAD: u32 = 4;+ const MIN_ACTIVE_SEGMENT_OVERHEAD: usize = 4;let mut merged_data_segments = Vec::with_capacity(data_segments.len());merged_data_segments.push(data_segments[0].clone());for b in &data_segments[1..] {@@ -206,19 +201,47 @@ async fn snapshot_memories(// Okay, merge them together into `a` (so that the next iteration can// merge it with its predecessor) and then omit `b`!- let merged = a.merge(b);- *a = merged;+ a.merge(b);}remove_excess_segments(&mut merged_data_segments);- (memory_mins, merged_data_segments)+ // With the final set of data segments now extract the actual data of each+ // memory, copying it into a `DataSegment`, to return the final list of+ // segments.+ //+ // Here the memories are iterated over again and, in tandem, the+ // `merged_data_segments` list is traversed to extract a `DataSegment` for+ // each range that `merged_data_segments` indicates. This relies on+ // `merged_data_segments` being a sorted list by `memory_index` at least.+ let mut final_data_segments = Vec::with_capacity(merged_data_segments.len());+ let mut merged = merged_data_segments.iter().peekable();+ let iter = module+ .defined_memories()+ .zip(module.defined_memory_exports.as_ref().unwrap());+ for ((memory_index, ty), name) in iter {+ instance+ .memory_contents(&name, |memory| {+ while let Some(segment) = merged.next_if(|s| s.memory_index == memory_index) {+ final_data_segments.push(DataSegment {+ memory_index,+ data: memory[segment.range.clone()].to_vec(),+ offset: segment.range.start.try_into().unwrap(),+ is64: ty.memory64,+ });+ }+ })+ .await;+ }+ assert!(merged.next().is_none());++ (memory_mins, final_data_segments)}/// Engines apply a limit on how many segments a module may contain, and Wizer/// can run afoul of it. When that happens, we need to merge data segments/// together until our number of data segments fits within the limit.-fn remove_excess_segments(merged_data_segments: &mut Vec<DataSegment>) {+fn remove_excess_segments(merged_data_segments: &mut Vec<DataSegmentRange>) {if merged_data_segments.len() < MAX_DATA_SEGMENTS {return;}@@ -243,7 +266,14 @@ fn remove_excess_segments(merged_data_segments: &mut Vec<DataSegment>) {if w[0].memory_index != w[1].memory_index {continue;}- let gap = w[0].gap(&w[1]);+ let gap = match u32::try_from(w[0].gap(&w[1])) {+ Ok(gap) => gap,+ // If the gap is larger than 4G then don't consider these two data+ // segments for merging and assume there's enough other data+ // segments close enough together to still consider for merging to+ // get under the limit.+ Err(_) => continue,+ };let index = u32::try_from(index).unwrap();smallest_gaps.push(GapIndex { gap, index });}@@ -256,8 +286,10 @@ fn remove_excess_segments(merged_data_segments: &mut Vec<DataSegment>) {smallest_gaps.sort_unstable_by(|a, b| a.index.cmp(&b.index).reverse());for GapIndex { index, .. } in smallest_gaps {let index = usize::try_from(index).unwrap();- let merged = merged_data_segments[index].merge(&merged_data_segments[index + 1]);- merged_data_segments[index] = merged;+ let [a, b] = merged_data_segments+ .get_disjoint_mut([index, index + 1])+ .unwrap();+ a.merge(b);// Okay to use `swap_remove` here because, even though it makes// `merged_data_segments` unsorted, the segments are still sorted within@@ -269,5 +301,5 @@ fn remove_excess_segments(merged_data_segments: &mut Vec<DataSegment>) {// Finally, sort the data segments again so that our output is// deterministic.- merged_data_segments.sort_by_key(|s| (s.memory_index, s.offset));+ merged_data_segments.sort_by_key(|s| (s.memory_index, s.range.start));}diff --git a/crates/wizer/src/wasmtime.rs b/crates/wizer/src/wasmtime.rsindex 9d1f29fac2..289df62304 100644--- a/crates/wizer/src/wasmtime.rs+++ b/crates/wizer/src/wasmtime.rs@@ -105,8 +105,8 @@ impl<T: Send> InstanceState for WasmtimeWizer<'_, T> {}}- async fn memory_contents(&mut self, name: &str) -> Vec<u8> {+ async fn memory_contents(&mut self, name: &str, contents: impl FnOnce(&[u8]) + Send) {let memory = self.instance.get_memory(&mut *self.store, name).unwrap();- memory.data(&self.store).to_vec()+ contents(memory.data(&self.store))}}diff --git a/crates/wizer/tests/all/tests.rs b/crates/wizer/tests/all/tests.rsindex 895b9e6945..46fed8a0f3 100644--- a/crates/wizer/tests/all/tests.rs+++ b/crates/wizer/tests/all/tests.rs@@ -992,3 +992,28 @@ async fn memory_init_and_data_segments() -> Result<()> {let wizer = get_wizer();wizen_and_run_wasm(&[], 0x02010403 + 0x06050201, &wasm, wizer).await}++#[tokio::test]+async fn memory64() -> Result<()> {+ let _ = env_logger::try_init();+ let wasm = wat_to_wasm(+ r#"+(module+ (memory i64 1)++ (func (export "wizer-initialize")+ i64.const 0+ i32.const 10+ i32.store+ )++ (func (export "run") (result i32)+ i64.const 0+ i32.load+ )+)+ "#,+ )?;+ let wizer = get_wizer();+ wizen_and_run_wasm(&[], 10, &wasm, wizer).await+}