A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Moving to `LinearMemory` trait and fixing custom memories (#14885)
Mat Gienieczko65fb6644964e
diff --git a/crates/wasmtime/src/runtime/memory.rs b/crates/wasmtime/src/runtime/memory.rsindex 1eac1e3503..e4f01edc37 100644--- a/crates/wasmtime/src/runtime/memory.rs+++ b/crates/wasmtime/src/runtime/memory.rs@@ -6,6 +6,8 @@ use crate::trampoline::generate_memory_export;#[cfg(feature = "async")]use crate::vm::VMStore;use crate::{AsContext, AsContextMut, Engine, MemoryType, StoreContext, StoreContextMut};+use alloc::sync::Arc;+use core::any::Any;use core::cell::UnsafeCell;use core::fmt;use core::slice;@@ -731,24 +733,74 @@ impl Memory {/// MemoryCreator, one can supply wasmtime with custom allocated host managed/// memory.///+/// Note that this is a relatively advanced feature and it is recommended to be+/// familiar with wasmtime runtime code to use it.+///+/// In the following, `minimum`, `reserved_size_in_bytes`, and `guard_size_in_bytes` arguments are given to a+/// [`MemoryCreator::new_memory`] implementation.+////// # Safety///-/// The memory should be page aligned and a multiple of page size.-/// To prevent possible silent overflows, the memory should be protected by a-/// guard page. Additionally the safety concerns explained in ['Memory'], for-/// accessing the memory apply here as well.+/// Wasmtime's compiled code and runtime rely on the following invariants for memory safety.+/// Note that while we strive to be thorough in this documentation, we do not guarantee that the+/// following points are exhaustive.+/// When using a custom memory backend in a security-sensitive context auditing just the external+/// implementation of this trait is not sufficient.///-/// The value returned by [`LinearMemory::byte_size`] must be at least the-/// `minimum` size the memory was created with, must be at least `new_size`-/// after a successful [`LinearMemory::grow_to`], and must never decrease.-/// At least that many bytes starting at [`LinearMemory::as_ptr`] must be-/// valid to read and write. Compiled WebAssembly code relies on these-/// properties when performing bounds checks, so violating them can lead to-/// out-of-bounds memory accesses.+/// - **Valid region.** From creation until the memory is dropped, the first+/// [`byte_size`] bytes starting at the [`as_ptr`]+/// memory location must be allocated, initialized, readable, and writeable.+/// The pointer must be non-null.+/// Nothing other than Wasmtime may read, write, unmap, or otherwise modify this region while the+/// memory is in use by Wasmtime.+/// If the memory moves (although see conditions on stability below), the previous region is no+/// longer in use by Wasmtime.+/// Moreover, be aware that Wasmtime currently has a soundness hole for shared memory,+/// as multiple threads may access the same slice of memory mutably at the same time (aliasing) (see+/// [issue #4203](https://github.com/bytecodealliance/wasmtime/issues/4203)).///-/// Note that this is a relatively advanced feature and it is recommended to be-/// familiar with wasmtime runtime code to use it.-pub unsafe trait LinearMemory: Send + Sync + 'static {+/// - **Size.** [`byte_size`] must be at least `minimum` after creation+/// and at least `new_size` after a successful call to [`grow_to`],+/// and it must never decrease.+///+/// - **Base pointer stability.** A call to [`as_ptr`] must return the same+/// address until the next successful call to [`grow_to`].+/// Moreover, a successful [`grow_to(new_size)`][`grow_to`] call where `new_size` is at most the [`byte_capacity`](LinearMemory::byte_capacity)+/// reported before the call must not change the result of [`as_ptr`].+/// If static memory is forced (e.g. when using shared memory or when [`Config::memory_may_move`](crate::Config::memory_may_move) is disabled), Wasmtime will never ask the memory to grow beyond its [`byte_capacity`].+///+/// - **Base pointer alignment.** The base pointer returned by [`as_ptr`] must always be 8-byte aligned.+/// Moreover, if `guard_size_in_bytes` is non-zero, or `reserved_size_in_bytes` is `Some` and non-zero, then [`as_ptr`] must be+/// aligned to the host page size at all times.+///+/// - **Inaccessible region.** If: `guard_size_in_bytes` is non-zero, or `reserved_size_in_bytes` is `Some` and non-zero;+/// then every byte from `as_ptr() + byte_size()`, rounded up to the next host page boundary, up to+/// `as_ptr() + max(reserved_size_in_bytes, byte_size()) + guard_size_in_bytes` must be+/// inaccessible, so that any access to it faults. This must hold at all times, including after growth.+///+/// Additionally, observe the safety requirements when the memory is accessed, as explained in [`Memory`].+///+/// # Correctness+///+/// The following constraints do not cause memory safety issues, but break WebAssembly semantics,+/// or expose potentially unwanted data to WebAssembly programs:+///+/// - [`byte_size`] should be exactly `minimum` after creation and exactly `new_size` after a+/// successful call to [`grow_to(new_size)`][`grow_to`]. Wasmtime only requests sizes that are+/// multiples of the Wasm page size.+///+/// - The memory's contents must be zeroed at creation, and bytes made accessible by [`grow_to`] must+/// be zeroed.+///+/// - If [`init_from_image`] returns `Ok(true)`, then the bytes at [`MemoryImage::linear_memory_offset`]+/// must equal the image and all other bytes must be zero.+///+/// [`grow_to`]: LinearMemory::grow_to+/// [`byte_size`]: LinearMemory::byte_size+/// [`as_ptr`]: LinearMemory::as_ptr+/// [`byte_capacity`]: LinearMemory::byte_capacity+/// [`init_from_image`]: LinearMemory::init_from_image+pub unsafe trait LinearMemory: Any + Send + Sync {/// Returns the number of allocated bytes which are accessible at this time.fn byte_size(&self) -> usize;@@ -767,6 +819,151 @@ pub unsafe trait LinearMemory: Send + Sync + 'static {/// Return the allocated memory as a mutable pointer to u8.fn as_ptr(&self) -> *mut u8;++ /// Optionally initializes this memory's contents from a copy-on-write+ /// [`MemoryImage`].+ ///+ /// When [copy-on-write memory initialization][cow_config] is enabled and a+ /// module's data segments for this memory can be represented as an image,+ /// Wasmtime calls this method once, right after the memory is created+ /// and before anything else reads or writes it.+ ///+ /// If the function succeeds with `false`, the runtime assumes the memory was not initialized.+ /// It then fully copies the module's data segments into the memory region.+ /// This is always correct and is essentially equivalent to CoW being disabled.+ ///+ /// If the function succeeds with `true`, the runtime assumes the memory's contents match `image` exactly.+ /// The bytes of the image must start at [`MemoryImage::linear_memory_offset`] and all other bytes must be zero.+ ///+ /// ## Shortcut for memory maps+ ///+ /// [`MemoryImage`] exposes a shortcut, [`MemoryImage::map_at`], which allows an implementation+ /// backed by a memory map to replace a portion of its mapping with the pages of the image.+ /// Consult that function's documentation for safety notes.+ ///+ /// Returning an error causes instantiation to fail with that error.+ ///+ /// [cow_config]: crate::Config::memory_init_cow+ fn init_from_image(&mut self, image: &MemoryImage) -> Result<bool> {+ let _ = image;+ Ok(false)+ }+}++/// A copy-on-write image of the initial contents of a linear memory.+///+/// When [`Config::memory_init_cow`](crate::Config::memory_init_cow) is enabled+/// Wasmtime prepares, where possible, an image of a module's data segments for+/// each of its linear memories that can be mapped into memory copy-on-write+/// instead of being copied byte-by-byte. Such images are handed to+/// [`LinearMemory::init_from_image`] for the specific memory implementation to map into its+/// underlying representation.+///+/// An image covers [`len()`](MemoryImage::len) bytes of the linear memory starting at+/// [`linear_memory_offset()`](MemoryImage::linear_memory_offset).+/// Both values are multiples of the host page size, and the image always lies within+/// the minimum size of the memory it was created for.+/// All other bytes are expected to be zero.+///+/// Cloning a `MemoryImage` is cheap and keeps the underlying image alive.+#[derive(Clone)]+pub struct MemoryImage {+ image: Arc<vm::MemoryImage>,+}++impl MemoryImage {+ pub(crate) fn new(image: Arc<vm::MemoryImage>) -> MemoryImage {+ MemoryImage { image }+ }++ /// This is a convenience method for the internal `MmapMemory` to access the underlying image+ /// directly and use its [`map_at`](vm::MemoryImage::map_at) function.+ #[cfg(has_virtual_memory)]+ pub(crate) fn vm(&self) -> &vm::MemoryImage {+ &self.image+ }++ /// Returns the offset, in bytes, within linear memory where this image+ /// starts.+ ///+ /// This is guaranteed to be a multiple of the host page size.+ pub fn linear_memory_offset(&self) -> usize {+ self.image.linear_memory_offset()+ }++ /// Returns the length, in bytes, of this image.+ ///+ /// This is guaranteed to be a multiple of the host page size.+ pub fn len(&self) -> usize {+ self.image.len()+ }++ /// Returns whether this image is empty (length equal to zero).+ pub fn is_empty(&self) -> bool {+ self.len() == 0+ }++ /// Returns the bytes of this image.+ ///+ /// The returned slice is [`len()`](MemoryImage::len) bytes long and holds+ /// the initial contents of linear memory starting at+ /// [`linear_memory_offset()`](MemoryImage::linear_memory_offset). All+ /// bytes of the memory outside of that range are initially zero.+ pub fn data(&self) -> &[u8] {+ self.image.data()+ }++ /// Maps this image copy-on-write into the linear memory starting at+ /// `base`.+ ///+ /// The bytes `base + linear_memory_offset()` up to+ /// `base + linear_memory_offset() + len()` are replaced with a private,+ /// readable and writable copy-on-write mapping of the image. Nothing+ /// outside of that range is modified.+ ///+ /// Returns an error if `base` is not aligned to the host page size or if+ /// the underlying mapping operation fails.+ ///+ /// # Safety+ ///+ /// The caller must exclusively own the virtual memory range being mapped+ /// over, nothing may hold references into it, and it must be valid for+ /// its pages to be replaced by a new mapping (for example by `mmap` with+ /// `MAP_FIXED` on Unix). Any previous contents and memory protections of+ /// that range are lost. The mapping lives until the range is unmapped or+ /// mapped over again by its owner. In particular, the runtime does not unmap the image by itself.+ /// The owner of the virtual memory range must clean it up after usage.+ ///+ /// The typical usage is owning a memory-mapped region and requesting the image to be mapped+ /// into a subrange of that mapping. Unmapping the entire region at the end of the memory's+ /// lifetime takes care of the cleanup.+ pub unsafe fn map_at(&self, base: *mut u8) -> Result<()> {+ // SAFETY: forwarded to the caller.+ unsafe { self.image.map_at_ptr(base) }+ }+}++impl fmt::Debug for MemoryImage {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("MemoryImage")+ .field("linear_memory_offset", &self.linear_memory_offset())+ .field("len", &self.len())+ .finish_non_exhaustive()+ }+}++impl PartialEq for MemoryImage {+ fn eq(&self, other: &Self) -> bool {+ Arc::ptr_eq(&self.image, &other.image)+ }+}++impl Eq for MemoryImage {}++impl core::hash::Hash for MemoryImage {+ fn hash<H: core::hash::Hasher>(&self, state: &mut H) {+ core::ptr::hash(Arc::as_ptr(&self.image), state)+ }}/// A memory creator. Can be used to provide a memory creator@@ -1171,4 +1368,127 @@ mod tests {Ok(())}++ /// Offset of the data segment used by the `MemoryImage` tests: inside the+ /// second Wasm page, but not at the start of a host page.+ #[cfg(all(target_os = "linux", not(miri)))]+ const IMAGE_DATA_OFFSET: usize = 0x10000 + 8;++ /// Compiles a module whose single memory is initialized by one data segment+ /// at `IMAGE_DATA_OFFSET` and returns the CoW image Wasmtime builds for it.+ /// The module is never instantiated, we just take the image of the memory.+ #[cfg(all(target_os = "linux", not(miri)))]+ fn memory_image_for_data(data: &str) -> Result<MemoryImage> {+ let mut config = Config::new();+ config.memory_init_cow(true);+ let engine = Engine::new(&config)?;+ let module = Module::new(+ &engine,+ // Module with (at least) 2 pages of memory with data written at IMAGE_DATA_OFFSET.+ format!("(module (memory 2) (data (i32.const {IMAGE_DATA_OFFSET}) \"{data}\"))"),+ )?;+ let images = module.memory_images()?.expect("module has memory images");+ let image = images+ .get_memory_image(wasmtime_environ::DefinedMemoryIndex::from_u32(0))+ .expect("memory 0 has an image");+ Ok(MemoryImage::new(image.clone()))+ }++ /// The public accessors report the internal image's placement, and:+ /// - both values are multiples of the host page size; and+ /// - the image covers the module's data.+ #[test]+ #[cfg(all(target_os = "linux", not(miri)))]+ fn memory_image_accessors() -> Result<()> {+ let page_size = crate::runtime::vm::host_page_size();+ let image = memory_image_for_data("hello")?;+ let offset = image.linear_memory_offset();+ let len = image.len();++ assert_eq!(offset, image.vm().linear_memory_offset());+ assert_eq!(len, image.vm().len());+ assert_eq!(offset % page_size, 0);+ assert_eq!(len % page_size, 0);+ assert!(!image.is_empty());+ // Image starts at or before the data...+ assert!(offset <= IMAGE_DATA_OFFSET);+ // ... and reaches past the end of the data.+ assert!(offset + len >= IMAGE_DATA_OFFSET + "hello".len());++ let data = image.data();+ assert_eq!(data.len(), len);+ assert_eq!(data, image.vm().data());+ let data_start = IMAGE_DATA_OFFSET - offset;+ assert_eq!(&data[data_start..][..5], b"hello");+ assert!(data[..data_start].iter().all(|b| *b == 0));+ assert!(data[data_start + 5..].iter().all(|b| *b == 0));++ // Clones share the same underlying image.+ let clone = image.clone();+ assert!(core::ptr::eq(clone.vm(), image.vm()));+ Ok(())+ }++ #[test]+ #[cfg(all(target_os = "linux", not(miri)))]+ fn memory_image_identity() -> Result<()> {+ use std::hash::{BuildHasher, RandomState};++ let mut config = Config::new();+ config.memory_init_cow(true);+ let engine = Engine::new(&config)?;+ let wat = r#"(module (memory 1) (data (i32.const 0) "hello"))"#;+ let image_of = |module: &Module| -> Result<MemoryImage> {+ let images = module.memory_images()?.expect("module has memory images");+ let image = images+ .get_memory_image(wasmtime_environ::DefinedMemoryIndex::from_u32(0))+ .expect("memory 0 has an image");+ Ok(MemoryImage::new(image.clone()))+ };++ let module = Module::new(&engine, wat)?;+ let image = image_of(&module)?;+ let same = image_of(&module)?; // Same module instance gives the same image.+ let clone = image.clone(); // Clone just clones an Arc.++ assert_eq!(image, same);+ assert_eq!(image, clone);++ let hasher = RandomState::new();+ assert_eq!(hasher.hash_one(&image), hasher.hash_one(&same));+ assert_eq!(hasher.hash_one(&image), hasher.hash_one(&clone));++ Ok(())+ }++ /// `MemoryImage::map_at` places the image's bytes at+ /// `base + linear_memory_offset()` and zeroes the rest.+ #[test]+ #[cfg(all(target_os = "linux", not(miri)))]+ fn memory_image_map_at() -> Result<()> {+ use crate::runtime::vm::{HostAlignedByteCount, Mmap};++ let image = memory_image_for_data("hello")?;+ let size =+ HostAlignedByteCount::new_rounded_up(image.linear_memory_offset() + image.len())?;+ let mmap = Mmap::accessible_reserved(HostAlignedByteCount::ZERO, size)?;+ let base = mmap.as_non_null().as_ptr();++ // SAFETY: `mmap` was just created, is owned by this test and covers+ // the whole range the image is mapped into.+ unsafe { image.map_at(base)? };++ // SAFETY: `map_at` made the image's range readable.+ let mapped = unsafe {+ core::slice::from_raw_parts(base.add(image.linear_memory_offset()), image.len())+ };+ let data_start = IMAGE_DATA_OFFSET - image.linear_memory_offset();++ assert_eq!(mapped, image.data());+ assert_eq!(&mapped[data_start..][..5], b"hello");+ assert!(mapped[..data_start].iter().all(|b| *b == 0));+ assert!(mapped[data_start + 5..].iter().all(|b| *b == 0));++ Ok(())+ }}diff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rsindex 71c8883344..b906b616dd 100644--- a/crates/wasmtime/src/runtime/trampoline/memory.rs+++ b/crates/wasmtime/src/runtime/trampoline/memory.rs@@ -4,8 +4,8 @@ use crate::prelude::*;use crate::runtime::vm::mpk::ProtectionKey;use crate::runtime::vm::{CompiledModuleId, InstanceAllocationRequest, InstanceAllocator, Memory, MemoryAllocationIndex,- MemoryBase, ModuleRuntimeInfo, OnDemandInstanceAllocator, RuntimeLinearMemory,- RuntimeMemoryCreator, SharedMemory, Table, TableAllocationIndex,+ ModuleRuntimeInfo, OnDemandInstanceAllocator, RuntimeMemoryCreator, SharedMemory, Table,+ TableAllocationIndex,};use crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque, StoreResourceLimiter};use alloc::sync::Arc;@@ -49,10 +49,16 @@ pub async fn create_memory(// We create an instance in the on-demand allocator when creating handles// associated with external objects. The configured instance allocator// should only be used when creating module instances as we don't want host- // objects to count towards instance limits.+ // objects to count towards instance limits. Host-created memories must+ // still use the configured memory creator, though, see+ // `Config::with_host_memory`.let allocator = SingleMemoryInstance {preallocation,- ondemand: OnDemandInstanceAllocator::default(),+ ondemand: OnDemandInstanceAllocator::new(+ store.engine().config().mem_creator.clone(),+ 0,+ false,+ ),};unsafe {store@@ -68,36 +74,6 @@ pub async fn create_memory(}}-struct LinearMemoryProxy {- mem: Box<dyn LinearMemory>,-}--impl RuntimeLinearMemory for LinearMemoryProxy {- fn byte_size(&self) -> usize {- self.mem.byte_size()- }-- fn byte_capacity(&self) -> usize {- self.mem.byte_capacity()- }-- fn grow_to(&mut self, new_size: usize) -> Result<()> {- self.mem.grow_to(new_size)- }-- fn base(&self) -> MemoryBase {- MemoryBase::new_raw(self.mem.as_ptr())- }-- fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {- let base = core::ptr::NonNull::new(self.mem.as_ptr()).unwrap();- crate::vm::VMMemoryDefinition {- base: base.into(),- current_length: self.mem.byte_size().into(),- }- }-}-#[derive(Clone)]pub(crate) struct MemoryCreatorProxy(pub Arc<dyn MemoryCreator>);@@ -108,7 +84,7 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy {memory_tunables: &MemoryTunables<'_>,minimum: usize,maximum: Option<usize>,- ) -> Result<Box<dyn RuntimeLinearMemory>> {+ ) -> Result<Box<dyn LinearMemory>> {let reserved_size_in_bytes = Some(memory_tunables.reservation().try_into().unwrap());self.0.new_memory(@@ -118,7 +94,6 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy {reserved_size_in_bytes,usize::try_from(memory_tunables.guard_size()).unwrap(),)- .map(|mem| Box::new(LinearMemoryProxy { mem }) as Box<dyn RuntimeLinearMemory>).map_err(|e| format_err!(e))}}diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex e3a3606d12..269486ef3f 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -109,9 +109,7 @@ pub use crate::runtime::vm::instance::{PoolConcurrencyLimitError, PoolingAllocatorMetrics, PoolingInstanceAllocator,};pub use crate::runtime::vm::interpreter::*;-pub use crate::runtime::vm::memory::{- Memory, MemoryBase, RuntimeLinearMemory, RuntimeMemoryCreator, SharedMemory,-};+pub use crate::runtime::vm::memory::{Memory, RuntimeMemoryCreator, SharedMemory};pub use crate::runtime::vm::mmap_vec::MmapVec;pub use crate::runtime::vm::provenance::*;pub use crate::runtime::vm::stack_switching::*;@@ -174,10 +172,12 @@ cfg_select! {has_virtual_memory => {pub use crate::runtime::vm::byte_count::*;pub use crate::runtime::vm::mmap::{Mmap, MmapOffset};- pub use self::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};+ pub use self::cow::{MemoryImage, ModuleMemoryImages};+ #[cfg(feature = "pooling-allocator")]+ pub use self::cow::MemoryImageSlot;}_ => {- pub use self::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};+ pub use self::cow_disabled::{MemoryImage, ModuleMemoryImages};}}diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rsindex 398d86f935..ac7d4dd9fc 100644--- a/crates/wasmtime/src/runtime/vm/cow.rs+++ b/crates/wasmtime/src/runtime/vm/cow.rs@@ -146,7 +146,51 @@ impl MemoryImage {Ok(None)}- unsafe fn map_at(&self, mmap_base: &MmapOffset) -> Result<()> {+ /// Offset, in bytes, within linear memory at which this image starts.+ pub(crate) fn linear_memory_offset(&self) -> usize {+ self.linear_memory_offset.byte_count()+ }++ /// Length, in bytes, of this image.+ pub(crate) fn len(&self) -> usize {+ self.len.byte_count()+ }++ /// The bytes of this image, as stored in the module's data.+ /// The length is exactly [`self.len`](Self::len).+ pub(crate) fn data(&self) -> &[u8] {+ &self.module_source.wasm_data()[self.module_source_offset..][..self.len.byte_count()]+ }++ /// Maps this image at `base + self.linear_memory_offset`.+ ///+ /// # Safety+ ///+ /// See [`crate::MemoryImage::map_at`].+ pub(crate) unsafe fn map_at_ptr(&self, base: *mut u8) -> Result<()> {+ if base.addr() % host_page_size() != 0 {+ bail!("cannot map a memory image at {base:p}: not host-page-aligned");+ }+ if self.len.is_zero() {+ return Ok(());+ }+ unsafe {+ self.source.map_at(+ base.add(self.linear_memory_offset.byte_count()),+ self.source_offset,+ self.len.byte_count(),+ )?;+ }+ Ok(())+ }++ /// Maps this image into `mmap_base + self.linear_memory_offset`.+ ///+ /// # Safety+ ///+ /// The caller must ensure that no one else has a reference to the range+ /// of `mmap_base` being mapped over.+ pub(crate) unsafe fn map_at(&self, mmap_base: &MmapOffset) -> Result<()> {unsafe {mmap_base.map_image_at(&self.source,@@ -302,6 +346,10 @@ impl ModuleMemoryImages {/// original mappings, effectively resetting everything back to its initial/// state. Non-linux platforms will replace all memory below `self.accessible`/// with a fresh zero'd mmap, meaning that reuse is effectively not supported.+#[cfg_attr(+ not(feature = "pooling-allocator"),+ expect(dead_code, reason = "only used by the pooling allocator")+)]pub struct MemoryImageSlot {/// The mmap and offset within it that contains the linear memory for this/// slot.@@ -359,6 +407,10 @@ impl fmt::Debug for MemoryImageSlot {}}+#[cfg_attr(+ not(feature = "pooling-allocator"),+ expect(dead_code, reason = "only used by the pooling allocator")+)]impl MemoryImageSlot {/// Create a new MemoryImageSlot. Assumes that there is an anonymous/// mmap backing in the given range to start.@@ -540,7 +592,11 @@ impl MemoryImageSlot {////// Returns the number of bytes still resident in memory after this function/// has returned.- #[allow(dead_code, reason = "only used in some cfgs")]++ #[cfg_attr(+ not(feature = "pooling-allocator"),+ expect(dead_code, reason = "only used by the pooling allocator")+ )]pub(crate) fn clear_and_remain_ready(&mut self,pagemap: Option<&PageMap>,@@ -556,7 +612,10 @@ impl MemoryImageSlot {Ok(bytes_resident)}- #[allow(dead_code, reason = "only used in some cfgs")]+ #[cfg_attr(+ not(feature = "pooling-allocator"),+ expect(dead_code, reason = "only used by the pooling allocator")+ )]unsafe fn reset_all_memory_contents(&mut self,pagemap: Option<&PageMap>,@@ -582,7 +641,10 @@ impl MemoryImageSlot {}}- #[allow(dead_code, reason = "only used in some cfgs")]+ #[cfg_attr(+ not(feature = "pooling-allocator"),+ expect(dead_code, reason = "only used by the pooling allocator")+ )]unsafe fn reset_with_original_mapping(&mut self,pagemap: Option<&PageMap>,@@ -742,7 +804,10 @@ impl MemoryImageSlot {self.image.is_some()}- #[allow(dead_code, reason = "only used in some cfgs")]+ #[cfg_attr(+ not(feature = "pooling-allocator"),+ expect(dead_code, reason = "only used by the pooling allocator")+ )]pub(crate) fn is_dirty(&self) -> bool {self.dirty}@@ -772,7 +837,7 @@ impl MemoryImageSlot {}}-#[cfg(all(test, target_os = "linux", not(miri)))]+#[cfg(all(test, target_os = "linux", not(miri), feature = "pooling-allocator"))]mod test {use super::*;use crate::runtime::vm::mmap::{AlignedLength, Mmap};diff --git a/crates/wasmtime/src/runtime/vm/cow_disabled.rs b/crates/wasmtime/src/runtime/vm/cow_disabled.rsindex 90a92e63f8..9595c0d455 100644--- a/crates/wasmtime/src/runtime/vm/cow_disabled.rs+++ b/crates/wasmtime/src/runtime/vm/cow_disabled.rs@@ -23,25 +23,30 @@ impl ModuleMemoryImages {#[derive(Debug, PartialEq)]pub enum MemoryImage {}-impl ModuleMemoryImages {- pub fn new(- _engine: &Engine,- _module: &Module,- _source: &Arc<impl ModuleMemoryImageSource>,- ) -> Result<Option<ModuleMemoryImages>> {- Ok(None)+impl MemoryImage {+ pub(crate) fn linear_memory_offset(&self) -> usize {+ match *self {}}-}-#[derive(Debug)]-pub enum MemoryImageSlot {}+ pub(crate) fn len(&self) -> usize {+ match *self {}+ }-impl MemoryImageSlot {- pub(crate) fn set_heap_limit(&mut self, _size_bytes: usize) -> Result<()> {+ pub(crate) fn data(&self) -> &[u8] {match *self {}}- pub(crate) fn has_image(&self) -> bool {+ pub(crate) unsafe fn map_at_ptr(&self, _base: *mut u8) -> Result<()> {match *self {}}}++impl ModuleMemoryImages {+ pub fn new(+ _engine: &Engine,+ _module: &Module,+ _source: &Arc<impl ModuleMemoryImageSource>,+ ) -> Result<Option<ModuleMemoryImages>> {+ Ok(None)+ }+}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rsindex 9d69157901..41e67b6beb 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs@@ -244,7 +244,7 @@ struct CopyingHeap {memory: Option<crate::vm::Memory>,/// The cached `VMMemoryDefinition` for `self.memory` so that we don't have- /// to make indirect calls through a `dyn RuntimeLinearMemory` object.+ /// to make indirect calls through a `dyn LinearMemory` object.////// Must be updated and kept in sync with `self.memory`, cleared when the/// memory is taken and updated when the memory is replaced.diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rsindex 0d2f7428aa..e841fe93bf 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs@@ -172,7 +172,7 @@ struct DrcHeap {memory: Option<crate::vm::Memory>,/// The cached `VMMemoryDefinition` for `self.memory` so that we don't have- /// to make indirect calls through a `dyn RuntimeLinearMemory` object.+ /// to make indirect calls through a `dyn LinearMemory` object.////// Must be updated and kept in sync with `self.memory`, cleared when the/// memory is taken and updated when the memory is replaced.diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rsindex ca1425c63d..f18ae7f13b 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs@@ -58,8 +58,8 @@ use crate::config::InstanceLimits;use crate::config::PoolingAllocationConfig;use crate::prelude::*;use crate::runtime::vm::{- CompiledModuleId, InstanceAllocationRequest, Memory, MemoryBase, MemoryImageSlot, Mmap,- MmapOffset, mmap::AlignedLength,+ CompiledModuleId, InstanceAllocationRequest, Memory, MemoryImageSlot, Mmap, MmapOffset,+ mmap::AlignedLength,};use crate::vm::sys::vm::{decommit_pages, iovec};use crate::{@@ -432,7 +432,7 @@ impl MemoryPool {let memory = Memory::new_static(ty,&memory_tunables,- MemoryBase::Mmap(base),+ base,base_capacity.byte_count(),slot,request.limiter.as_deref_mut(),diff --git a/crates/wasmtime/src/runtime/vm/memory.rs b/crates/wasmtime/src/runtime/vm/memory.rsindex ef68214ed7..aa02d508c8 100644--- a/crates/wasmtime/src/runtime/vm/memory.rs+++ b/crates/wasmtime/src/runtime/vm/memory.rs@@ -24,7 +24,7 @@//! ▼//! ┌─────────────────────┐//! │ │-//! │ RuntimeLinearMemory ├─────────────┬───────────────┐+//! │ LinearMemory ├─────────────┬───────────────┐//! │ │ │ │//! └──────────┬──────────┘ │ │//! │ │ │@@ -52,35 +52,34 @@//! instance-local state of `Memory`. One example is that `LocalMemory::grow`//! has most of the logic around memory growth.//!-//! * `RuntimeLinearMemory` - this is a trait which `LocalMemory` delegates to.-//! This trait is intentionally relatively simple to be exposed in Wasmtime's-//! embedder API. This is exposed all the way through `wasmtime::Config` so-//! embedders can provide arbitrary implementations.+//! * `LinearMemory` - this is the public trait which+//! `LocalMemory` delegates to. This is exposed all the+//! way through `wasmtime::Config` so embedders can provide arbitrary+//! implementations, and Wasmtime's own memories implement it as well.//!-//! * `MmapMemory` - this is an implementation of `RuntimeLinearMemory` in terms-//! of the platform's mmap primitive.+//! * `MmapMemory` - this is an implementation of `LinearMemory` in terms of+//! the platform's mmap primitive.//!-//! * `StaticMemory` - this is an implementation of `RuntimeLinearMemory`-//! for the pooling allocator where the base pointer is already allocated-//! and contents are managed through `MemoryImageSlot`.+//! * `StaticMemory` - this is an implementation of `LinearMemory` for the+//! pooling allocator where the base pointer is already allocated and+//! contents are managed through `MemoryImageSlot`.//!//! Other important types for memories are `MemoryImage` and `MemoryImageSlot`-//! which manage CoW state for memories. This is implemented at the-//! `LocalMemory` layer.-//!-//! FIXME: don't have both RuntimeLinearMemory and wasmtime::LinearMemory, they-//! should be merged together.+//! which manage CoW state for memories. Initializing a memory from a+//! `MemoryImage` is delegated to the `LinearMemory` itself through+//! `LinearMemory::init_from_image`, and the pooling allocator's+//! `StaticMemory` owns a `MemoryImageSlot` which is reused across instances.//!//! FIXME: don't have both RuntimeMemoryCreator and wasmtime::MemoryCreator,//! they should be merged together.-use crate::Engine;use crate::prelude::*;use crate::runtime::store::StoreResourceLimiter;+use crate::runtime::vm::MemoryImage;use crate::runtime::vm::vmcontext::VMMemoryDefinition;-#[cfg(has_virtual_memory)]-use crate::runtime::vm::{HostAlignedByteCount, MmapOffset};-use crate::runtime::vm::{MemoryImage, MemoryImageSlot, SendSyncPtr};+#[cfg(feature = "pooling-allocator")]+use crate::runtime::vm::{MemoryImageSlot, MmapOffset};+use crate::{Engine, LinearMemory};use alloc::sync::Arc;use core::{ops::Range, ptr::NonNull};use wasmtime_environ::{MemoryKind, MemoryTunables};@@ -113,28 +112,29 @@ pub use shared_memory_disabled::SharedMemory;/// A memory allocatorpub trait RuntimeMemoryCreator: Send + Sync {- /// Create new RuntimeLinearMemory+ /// Create a new `LinearMemory`.fn new_memory(&self,ty: &wasmtime_environ::Memory,memory_tunables: &MemoryTunables<'_>,minimum: usize,maximum: Option<usize>,- ) -> Result<Box<dyn RuntimeLinearMemory>>;+ ) -> Result<Box<dyn LinearMemory>>;}/// A default memory allocator used by Wasmtimepub struct DefaultMemoryCreator;impl RuntimeMemoryCreator for DefaultMemoryCreator {- /// Create new MmapMemory+ /// Create new LinearMemory.+ /// Chooses [`MmapMemory`] if it is available and tunables require it, falling back to [`MallocMemory`] otherwise.fn new_memory(&self,ty: &wasmtime_environ::Memory,memory_tunables: &MemoryTunables<'_>,minimum: usize,maximum: Option<usize>,- ) -> Result<Box<dyn RuntimeLinearMemory>> {+ ) -> Result<Box<dyn LinearMemory>> {#[cfg(has_virtual_memory)]if memory_tunables.tunables().signals_based_traps|| memory_tunables.guard_size() > 0@@ -143,93 +143,18 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator {{return Ok(try_new::<Box<_>>(MmapMemory::new(ty, memory_tunables, minimum, maximum)?)?- as Box<dyn RuntimeLinearMemory>,+ as Box<dyn LinearMemory>,);}let _ = maximum;Ok(try_new::<Box<_>>(MallocMemory::new(ty, memory_tunables, minimum)?)?- as Box<dyn RuntimeLinearMemory>,+ as Box<dyn LinearMemory>,)}}-/// A linear memory and its backing storage.-pub trait RuntimeLinearMemory: Send + Sync {- /// Returns the number bytes that this linear memory can access.- fn byte_size(&self) -> usize;-- /// Returns the maximal number of bytes the current allocation can access.- ///- /// Growth up to this value should not relocate the base pointer.- fn byte_capacity(&self) -> usize;-- /// Grow memory to the specified amount of bytes.- ///- /// Returns an error if memory can't be grown by the specified amount- /// of bytes.- fn grow_to(&mut self, size: usize) -> Result<()>;-- /// Returns a pointer to the base of this linear memory allocation.- ///- /// This is either a raw pointer, or a reference to an mmap along with an- /// offset within it.- fn base(&self) -> MemoryBase;-- /// Get a `VMMemoryDefinition` for this linear memory.- fn vmmemory(&self) -> VMMemoryDefinition;-- /// Internal method for Wasmtime when used in conjunction with CoW images.- /// This is used to inform the underlying memory that the size of memory has- /// changed.- ///- /// Note that this is hidden and panics by default as embedders using custom- /// memory without CoW images shouldn't have to worry about this.- #[doc(hidden)]- fn set_byte_size(&mut self, len: usize) {- let _ = len;- panic!("CoW images used with this memory and it doesn't support it");- }-}--/// The base pointer of a memory allocation.-#[derive(Clone, Debug)]-pub enum MemoryBase {- /// A raw pointer into memory.- ///- /// This may or may not be host-page-aligned.- Raw(SendSyncPtr<u8>),-- /// An mmap along with an offset into it.- #[cfg(has_virtual_memory)]- Mmap(MmapOffset),-}--impl MemoryBase {- /// Creates a new `MemoryBase` from a raw pointer.- ///- /// The pointer must be non-null, and it must be logically `Send + Sync`.- pub fn new_raw(ptr: *mut u8) -> Self {- Self::Raw(NonNull::new(ptr).expect("pointer is non-null").into())- }-- /// Returns the actual memory address in memory that is represented by this- /// base.- pub fn as_non_null(&self) -> NonNull<u8> {- match self {- Self::Raw(ptr) => ptr.as_non_null(),- #[cfg(has_virtual_memory)]- Self::Mmap(mmap_offset) => mmap_offset.as_non_null(),- }- }-- /// Same as `as_non_null`, but different return type.- pub fn as_mut_ptr(&self) -> *mut u8 {- self.as_non_null().as_ptr()- }-}-/// Representation of a runtime wasm linear memory.pub enum Memory {Local(LocalMemory),@@ -260,26 +185,34 @@ impl Memory {}/// Create a new static (immovable) memory instance for the specified plan.+ ///+ /// The `memory_image_slot` must have been already instantiated by the pooling allocator.#[cfg(feature = "pooling-allocator")]pub async fn new_static(ty: &wasmtime_environ::Memory,memory_tunables: &MemoryTunables<'_>,- base: MemoryBase,+ base: MmapOffset,base_capacity: usize,- memory_image: MemoryImageSlot,+ memory_image_slot: MemoryImageSlot,limiter: Option<&mut StoreResourceLimiter<'_>>,) -> Result<Self> {let (minimum, maximum) = Self::limit_new(ty, limiter).await?;- let pooled_memory = StaticMemory::new(base, base_capacity, minimum, maximum)?;++ // Assert the memory image was instantiated.+ debug_assert!(memory_image_slot.is_dirty());+ // The pooling allocator has already instantiated the image, so it only needs+ // initialization if the slot holds no image.+ let needs_init = !memory_image_slot.has_image();+ let pooled_memory =+ StaticMemory::new(base, base_capacity, minimum, maximum, memory_image_slot)?;let allocation = try_new::<Box<_>>(pooled_memory)?;// Configure some defaults a bit differently for this memory within the- // `LocalMemory` structure created, notably we already have- // `memory_image` and regardless of configuration settings this memory- // can't move its base pointer since it's a fixed allocation.+ // `LocalMemory` structure created:+ // - record whether the data segments still need to be copied in;+ // - regardless of configuration settings this memory can't move its base pointer since it's a fixed allocation.let mut memory = LocalMemory::new(ty, memory_tunables, allocation, None)?;- assert!(memory.memory_image.is_none());- memory.memory_image = Some(memory_image);+ memory.needs_init = needs_init;memory.memory_may_move = false;Ok(if ty.shared {@@ -534,61 +467,38 @@ impl Memory {/// the implementation basis for a `SharedMemory` behind an `RwLock` for/// example.pub struct LocalMemory {- alloc: Box<dyn RuntimeLinearMemory>,+ alloc: Box<dyn LinearMemory>,ty: wasmtime_environ::Memory,+ /// True only if this memory may move its base pointer.memory_may_move: bool,memory_guard_size: usize,memory_reservation: usize,- /// An optional CoW mapping that provides the initial content of this- /// memory.- memory_image: Option<MemoryImageSlot>,+ /// Whether the data segments of this memory still need to be copied into it.+ /// The memory might be already initialized from a pooled allocator or by the particular+ /// allocation's implementation, in which case this is false.+ needs_init: bool,}impl LocalMemory {pub fn new(ty: &wasmtime_environ::Memory,memory_tunables: &MemoryTunables<'_>,- alloc: Box<dyn RuntimeLinearMemory>,+ mut alloc: Box<dyn LinearMemory>,memory_image: Option<&Arc<MemoryImage>>,) -> Result<LocalMemory> {- // If a memory image was specified, try to create the MemoryImageSlot on- // top of our mmap.- let memory_image = match memory_image {- #[cfg(has_virtual_memory)]- Some(image) => {- // We currently don't support memory_image if- // `RuntimeLinearMemory::byte_size` is not a multiple of the host page- // size. See https://github.com/bytecodealliance/wasmtime/issues/9660.- if let Ok(byte_size) = HostAlignedByteCount::new(alloc.byte_size()) {- // memory_image is CoW-based so it is expected to be backed- // by an mmap.- let mmap_base = match alloc.base() {- MemoryBase::Mmap(offset) => offset,- MemoryBase::Raw { .. } => {- unreachable!("memory_image is Some only for mmap-based memories")- }- };-- // Memories allocated on demand are never striped with MPK- // protection keys, so this slot has no key to preserve.- let mut slot =- MemoryImageSlot::create(mmap_base, byte_size, alloc.byte_capacity(), None);- slot.instantiate(alloc.byte_size(), Some(image), ty, memory_tunables)?;- Some(slot)- } else {- None- }- }- #[cfg(not(has_virtual_memory))]- Some(_) => unreachable!(),- None => None,+ // If a memory image was specified, give the allocation a chance to+ // initialize itself from it. If it declines then the data segments+ // are copied in as usual.+ let initialized = match memory_image {+ Some(image) => alloc.init_from_image(&crate::MemoryImage::new(image.clone()))?,+ None => false,};Ok(LocalMemory {ty: *ty,alloc,memory_may_move: ty.memory_may_move(memory_tunables),- memory_image,+ needs_init: !initialized,memory_guard_size: memory_tunables.guard_size().try_into().unwrap(),memory_reservation: memory_tunables.reservation().try_into().unwrap(),})@@ -660,7 +570,7 @@ impl LocalMemory {// Save the original base pointer to assert the invariant that growth up// to the byte capacity never relocates the base pointer.- let base_ptr_before = self.alloc.base().as_mut_ptr();+ let base_ptr_before = self.alloc.as_ptr();let required_to_not_move_memory = new_byte_size <= self.alloc.byte_capacity();let result = (|| -> Result<()> {@@ -677,30 +587,6 @@ impl LocalMemory {bail!("Memory maximum size exceeded");}- // If we have a CoW image overlay then let it manage accessible- // bytes. Once the heap limit is modified inform the underlying- // allocation that the size has changed.- //- // If the growth is going beyond the size of the heap image then- // discard it. This should only happen for `MmapMemory` where- // `no_clear_on_drop` is set so the destructor doesn't do anything.- // For now be maximally sure about this by asserting that memory can- // indeed move and that we're on unix. If this wants to run- // somewhere else like Windows or with other allocations this may- // need adjusting.- if let Some(image) = &mut self.memory_image {- if new_byte_size <= self.alloc.byte_capacity() {- image.set_heap_limit(new_byte_size)?;- self.alloc.set_byte_size(new_byte_size);- return Ok(());- }- assert!(cfg!(unix));- assert!(self.memory_may_move);- self.memory_image = None;- }-- // And failing all that fall back to the underlying allocation to- // grow it.self.alloc.grow_to(new_byte_size)})();@@ -709,7 +595,11 @@ impl LocalMemory {// On successful growth double-check that the base pointer// didn't move if it shouldn't have.if required_to_not_move_memory {- assert_eq!(base_ptr_before, self.alloc.base().as_mut_ptr());+ assert_eq!(+ base_ptr_before,+ self.alloc.as_ptr(),+ "linear memory moved on grow when it must not have"+ );}Ok(Some((old_byte_size, new_byte_size)))@@ -727,8 +617,14 @@ impl LocalMemory {}}+ /// Return the bounds of the memory (base, byte_size) for consumption by WebAssembly code in a+ /// repr(C) struct.pub fn vmmemory(&self) -> VMMemoryDefinition {- self.alloc.vmmemory()+ let base = NonNull::new(self.alloc.as_ptr()).expect("linear memory base is non-null");+ VMMemoryDefinition {+ base: base.into(),+ current_length: self.alloc.byte_size().into(),+ }}pub fn byte_size(&self) -> usize {@@ -736,14 +632,11 @@ impl LocalMemory {}pub fn needs_init(&self) -> bool {- match &self.memory_image {- Some(image) => !image.has_image(),- None => true,- }+ self.needs_init}pub fn wasm_accessible(&self) -> Range<usize> {- let base = self.alloc.base().as_mut_ptr() as usize;+ let base = self.alloc.as_ptr() as usize;// From the base add://// * max(capacity, reservation) -- all memory is guaranteed to have at@@ -758,9 +651,16 @@ impl LocalMemory {base..end}+ /// Assume the inner allocation is a [`StaticMemory`] created for a pooling allocator and unwrap+ /// the inner [`MemoryImageSlot`].+ /// This function panics if this memory was not created from a [`StaticMemory`] allocation.#[cfg(feature = "pooling-allocator")]pub fn unwrap_static_image(self) -> MemoryImageSlot {- self.memory_image.unwrap()+ let alloc: Box<dyn core::any::Any> = self.alloc;+ alloc+ .downcast::<StaticMemory>()+ .unwrap_or_else(|_| panic!("expected a pooling allocator memory"))+ .into_image_slot()}}diff --git a/crates/wasmtime/src/runtime/vm/memory/malloc.rs b/crates/wasmtime/src/runtime/vm/memory/malloc.rsindex f07a5995d5..5046b585c2 100644--- a/crates/wasmtime/src/runtime/vm/memory/malloc.rs+++ b/crates/wasmtime/src/runtime/vm/memory/malloc.rs@@ -1,12 +1,12 @@-//! Support for implementing the [`RuntimeLinearMemory`] trait in terms of a+//! Support for implementing the [`LinearMemory`] trait in terms of a//! platform memory allocation primitive (e.g. `malloc`)//!//! Note that memory is allocated here using `Vec::try_reserve` to explicitly//! handle memory allocation failures.+use crate::LinearMemory;use crate::prelude::*;use crate::runtime::vm::SendSyncPtr;-use crate::runtime::vm::memory::{MemoryBase, RuntimeLinearMemory};use core::mem;use core::ptr::NonNull;use wasmtime_environ::MemoryTunables;@@ -66,7 +66,16 @@ impl MallocMemory {}}-impl RuntimeLinearMemory for MallocMemory {+// SAFETY:+// - **Valid region.** We use a Vec with at least `minimum` bytes, possibly more with 16-alignment.+// - **Size.** Is exactly `minimum` at creation and is set to exactly `new_size` on growth.+// - **Base pointer stability.** `grow_to` calls `Vec::try_reserve`, which is guaranteed to not+// reallocate if the new length does not exceed capacity. The Vec's capacity is what is reported in+// `byte_capacity`.+// - **Base pointer alignment.** The pointer is 16-byte aligned. The other conditions are checked+// to be false within the constructor, so we do not require host page alignment.+// - **Inaccessible region** moot because the conditions are checked to be false.+unsafe impl LinearMemory for MallocMemory {fn byte_size(&self) -> usize {self.byte_len}@@ -89,16 +98,8 @@ impl RuntimeLinearMemory for MallocMemory {Ok(())}- fn base(&self) -> MemoryBase {- MemoryBase::Raw(self.base_ptr)- }-- fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {- let base = self.base_ptr.as_non_null();- crate::vm::VMMemoryDefinition {- base: base.into(),- current_length: self.byte_len.into(),- }+ fn as_ptr(&self) -> *mut u8 {+ self.base_ptr.as_ptr()}}diff --git a/crates/wasmtime/src/runtime/vm/memory/mmap.rs b/crates/wasmtime/src/runtime/vm/memory/mmap.rsindex bdd9c3f1ef..2cf6f46009 100644--- a/crates/wasmtime/src/runtime/vm/memory/mmap.rs+++ b/crates/wasmtime/src/runtime/vm/memory/mmap.rs@@ -1,15 +1,23 @@-//! Support for implementing the [`RuntimeLinearMemory`] trait in terms of a+//! Support for implementing the [`LinearMemory`] trait in terms of a//! platform mmap primitive.use crate::prelude::*;-use crate::runtime::vm::memory::RuntimeLinearMemory;use crate::runtime::vm::{HostAlignedByteCount, Mmap, mmap::AlignedLength};+use crate::{LinearMemory, MemoryImage};use alloc::sync::Arc;use wasmtime_environ::MemoryTunables;-use super::MemoryBase;-/// A linear memory instance.+///+/// The layout of a single Mmap is:+///+/// ```text+/// [ pre_guard ][ accessible = len rounded up | PROT_NONE up to capacity ][ offset_guard ]+/// ^ as_ptr()+/// |<------------------ current_capacity() --------------->|+/// ```+///+/// The amount of reserved memory depends on tunables (see the [`new`](MmapMemory::new) function).#[derive(Debug)]pub struct MmapMemory {// The underlying allocation.@@ -144,7 +152,17 @@ impl MmapMemory {}}-impl RuntimeLinearMemory for MmapMemory {+/// SAFETY:+///+/// - **Valid region.** The memory map is always kept valid for self.len bytes.+/// - **Size.** [`MmapMemory::new`] maps at least `minimum` bytes.+/// - **Base pointer stability.** `grow_to` does not modify the mapping while the request is within+/// capacity.+/// - **Base pointer alignment.** The mmap call always returns a page-aligned pointer, and guard+/// bytes are ensured to be a multiple of page-size.+/// - **Inaccessible region.** Inaccessible memory is protected (implementation delegated to the+/// `sys` submodule).+unsafe impl LinearMemory for MmapMemory {fn byte_size(&self) -> usize {self.len}@@ -221,31 +239,32 @@ impl RuntimeLinearMemory for MmapMemory {Ok(())}- fn set_byte_size(&mut self, len: usize) {- self.len = len;- }-- fn base(&self) -> MemoryBase {- MemoryBase::Mmap(+ fn as_ptr(&self) -> *mut u8 {+ // SAFETY: `pre_guard_size` is within the mmap allocation.+ unsafe {self.mmap- .offset(self.pre_guard_size)- .expect("pre_guard_size is in bounds"),- )+ .as_non_null()+ .byte_add(self.pre_guard_size.byte_count())+ .as_ptr()+ }}- fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {- let pre_guard_size = self.pre_guard_size.byte_count();- assert!(pre_guard_size <= self.mmap.len());- let pre_guard_size = isize::try_from(pre_guard_size).unwrap();- let mmap = self.mmap.as_non_null();- let base = unsafe {- // Safety: `pre_guard_size` is within the mmap allocation.- mmap.offset(pre_guard_size)- };-- crate::vm::VMMemoryDefinition {- base: base.into(),- current_length: self.len.into(),+ fn init_from_image(&mut self, image: &MemoryImage) -> Result<bool> {+ // Images always fit within the minimum size of a memory, which is+ // accessible from creation. If this ever changes our assumptions are incorrect for CoW, so+ // explode loudly.+ let image_end = image.linear_memory_offset().checked_add(image.len());+ assert!(+ image_end.is_some_and(|end| end <= self.len),+ "MemoryImage does not fit into the MmapMemory"+ );+ let base = self.mmap.offset(self.pre_guard_size)?;+ // SAFETY: this memory was just created and no one else has a reference+ // to it yet. The image's range lies within this memory's accessible+ // region of the mmap, which this memory exclusively owns.+ unsafe {+ image.vm().map_at(&base)?;}+ Ok(true)}}diff --git a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rsindex 9f0721be11..4a93bb197c 100644--- a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs+++ b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs@@ -41,8 +41,7 @@ impl SharedMemory {// `assert_ready` should never panic.let (minimum_bytes, maximum_bytes) = vm::assert_ready(Memory::limit_new(ty, None))?;let mmap_memory = MmapMemory::new(ty, &memory_tunables, minimum_bytes, maximum_bytes)?;- let boxed: Box<dyn crate::runtime::vm::RuntimeLinearMemory> =- try_new::<Box<_>>(mmap_memory)?;+ let boxed: Box<dyn crate::LinearMemory> = try_new::<Box<_>>(mmap_memory)?;Self::wrap(engine,ty,@@ -87,7 +86,7 @@ impl SharedMemory {NonNull::from(&self.0.def.0)}- /// Same as `RuntimeLinearMemory::grow`, except with `&self`.+ /// Same as `LocalMemory::grow`, except with `&self`.pub fn grow(&self, delta_pages: u64) -> Result<Option<(usize, usize)>, Error> {let mut memory = self.0.memory.write().unwrap();// Without a limiter being passed in this shouldn't have an await point,diff --git a/crates/wasmtime/src/runtime/vm/memory/static_.rs b/crates/wasmtime/src/runtime/vm/memory/static_.rsindex 1646eb4a9d..602b3e3352 100644--- a/crates/wasmtime/src/runtime/vm/memory/static_.rs+++ b/crates/wasmtime/src/runtime/vm/memory/static_.rs@@ -1,30 +1,45 @@-//! Support for implementing the [`RuntimeLinearMemory`] trait in terms of a+//! Support for implementing the [`LinearMemory`] trait in terms of a//! fixed allocation that cannot move.+use crate::LinearMemory;use crate::prelude::*;-use crate::runtime::vm::MemoryBase;-use crate::runtime::vm::memory::RuntimeLinearMemory;+use crate::runtime::vm::{MemoryImageSlot, MmapOffset, SendSyncPtr};/// A "static" memory where the lifetime of the backing memory is managed/// elsewhere. Currently used with the pooling allocator.pub struct StaticMemory {/// The base pointer of this static memory, wrapped up in a send/sync/// wrapper.- base: MemoryBase,+ base: SendSyncPtr<u8>,/// The byte capacity of the `base` pointer.capacity: usize,/// The current size, in bytes, of this memory.size: usize,++ /// The slot of the pooling allocator in which this memory lives.+ ///+ /// The slot has already been instantiated for this memory, including+ /// mapping its CoW image, if any. It manages which bytes of the slot are+ /// accessible as this memory grows, and is handed back to the pooling+ /// allocator with [`StaticMemory::into_image_slot`] when this memory is+ /// deallocated.+ image_slot: MemoryImageSlot,}impl StaticMemory {+ /// Create a new static memory at the given memory map location.+ /// The `image_slot` must be already instantiated *for this memory*.+ /// So the `slot.base` is the same location as `base`, `slot.static_size` >= `base_capacity`,+ /// `initial_size` is accessible in the `slot`, and when relying on guards no more than+ /// `initial_size` rounded to full host pages may be accessible.pub fn new(- base: MemoryBase,+ base: MmapOffset,base_capacity: usize,initial_size: usize,maximum_size: Option<usize>,+ image_slot: MemoryImageSlot,) -> Result<Self> {if base_capacity < initial_size {bail!(@@ -40,14 +55,40 @@ impl StaticMemory {};Ok(Self {- base,+ base: base.as_non_null().into(),capacity: base_capacity,size: initial_size,+ image_slot,})}++ /// Returns the slot state of this memory back to the pooling allocator.+ pub fn into_image_slot(self) -> MemoryImageSlot {+ self.image_slot+ }}-impl RuntimeLinearMemory for StaticMemory {+// SAFETY:+// Note that these points, in particular alignment, rely on the behavior of the pooling allocator.+//+// - **Valid region.** `base` points into the pooling allocator's mapping at the slot owned+// exclusively by this memory (the pool's index allocator hands each slot to one memory at a time).+// `image_slot` manages that same region and keeps the first `size` bytes, rounded up to the host+// page size, accessible. It also holds a reference to the pool's mapping, which keeps+// `base` valid for this memory's lifetime.+//+// - **Size.** Is exactly `initial_size` at creation and is set to exactly `new_byte_size` on growth.+//+// - **Base pointer stability.** `base` never changes. Growth beyond `capacity` is rejected by the+// assertion in `grow_to`, and `LocalMemory` never requests it since `Memory::new_static` forces+// `memory_may_move = false`.+//+// - **Base pointer alignment.** The slot's base is the pool's mapping plus the pre-slab guard plus+// a multiple of the slot size, all of which are multiples of the host page size, so `base` is+// host-page-aligned.+//+// - **Inaccessible region.** This relies on the guard mechanism outlined in the `MemoryPool` documentation.+unsafe impl LinearMemory for StaticMemory {fn byte_size(&self) -> usize {self.size}@@ -61,23 +102,14 @@ impl RuntimeLinearMemory for StaticMemory {// prior to arriving here.assert!(new_byte_size <= self.capacity);- // Update our accounting of the available size.+ // Let the slot make the new bytes accessible, then update our+ // accounting of the available size.+ self.image_slot.set_heap_limit(new_byte_size)?;self.size = new_byte_size;Ok(())}- fn set_byte_size(&mut self, len: usize) {- self.size = len;- }-- fn base(&self) -> MemoryBase {- self.base.clone()- }-- fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {- crate::vm::VMMemoryDefinition {- base: self.base.as_non_null().into(),- current_length: self.size.into(),- }+ fn as_ptr(&self) -> *mut u8 {+ self.base.as_ptr()}}diff --git a/crates/wasmtime/src/runtime/vm/mmap.rs b/crates/wasmtime/src/runtime/vm/mmap.rsindex e63e037498..144c50d8a0 100644--- a/crates/wasmtime/src/runtime/vm/mmap.rs+++ b/crates/wasmtime/src/runtime/vm/mmap.rs@@ -454,11 +454,21 @@ impl MmapOffset {.offset.checked_add(memory_offset).expect("self.offset + memory_offset is in bounds");+ if !total_offset+ .checked_add(memory_len)+ .is_ok_and(|end| end <= self.mmap.len_aligned())+ {+ bail!("image mapping must be in bounds of the mmap")+ }unsafe {- self.mmap- .sys- .map_image_at(image_source, source_offset, total_offset, memory_len)+ let base = self+ .mmap+ .as_non_null()+ .byte_add(total_offset.byte_count())+ .as_ptr();+ image_source.map_at(base, source_offset, memory_len.byte_count())?;}+ Ok(())}}@@ -490,4 +500,68 @@ mod tests {.expect("make_readonly succeeded");};}++ /// Creates a two-page inaccessible mmap and a one-page image source whose+ /// bytes are all `0x67`.+ #[cfg(all(target_os = "linux", not(miri)))]+ fn mmap_and_image_source() -> (Arc<Mmap<AlignedLength>>, MemoryImageSource) {+ let page_size = HostAlignedByteCount::host_page_size();+ let pagex2 = page_size.checked_mul(2).unwrap();+ let mmap = Mmap::accessible_reserved(HostAlignedByteCount::ZERO, pagex2)+ .expect("allocated memory");+ let data = vec![0x67; page_size.byte_count()];+ let source = MemoryImageSource::from_data(&data)+ .expect("created image source")+ .expect("memfd is supported");+ (Arc::new(mmap), source)+ }++ #[test]+ #[cfg(all(target_os = "linux", not(miri)))]+ fn map_image_at_up_to_end_of_mmap() {+ let page_size = HostAlignedByteCount::host_page_size();+ let (mmap, source) = mmap_and_image_source();+ let base = mmap.offset(page_size).unwrap();+ // SAFETY: the mmap was just created and nothing references it.+ unsafe {+ base.map_image_at(&source, 0, HostAlignedByteCount::ZERO, page_size)+ .expect("mapped image");+ let mapped = core::slice::from_raw_parts(base.as_mut_ptr(), page_size.byte_count());+ assert!(mapped.iter().all(|b| *b == 0x67));+ }+ }++ #[test]+ #[cfg(all(target_os = "linux", not(miri)))]+ fn map_image_at_past_end_of_mmap() {+ let page_size = HostAlignedByteCount::host_page_size();+ let (mmap, source) = mmap_and_image_source();+ let base = mmap.offset(page_size).unwrap();+ // End the mapping at the highest page-aligned address: `offset + len`+ // doesn't overflow, but `len` exceeds any address space, so should the+ // assertion ever regress the kernel rejects the `mmap` instead of+ // mapping over memory that this test doesn't own.+ let max_aligned = usize::MAX - usize::MAX % page_size.byte_count();+ let max = HostAlignedByteCount::new(max_aligned).unwrap();+ let huge = max.checked_sub(page_size).unwrap();+ // SAFETY: the mmap was just created and nothing references it. The+ // call is expected to return Err before mapping anything.+ let result = unsafe { base.map_image_at(&source, 0, HostAlignedByteCount::ZERO, huge) };+ assert!(result.is_err());+ }++ /// An offset and length whose sum overflows must be rejected as well.+ #[test]+ #[cfg(all(target_os = "linux", not(miri)))]+ fn map_image_at_overflowing_length() {+ let page_size = HostAlignedByteCount::host_page_size();+ let (mmap, source) = mmap_and_image_source();+ let base = mmap.offset(page_size).unwrap();+ let max_aligned = usize::MAX - usize::MAX % page_size.byte_count();+ let max = HostAlignedByteCount::new(max_aligned).unwrap();+ // SAFETY: the mmap was just created and nothing references it. The+ // call is expected to panic before mapping anything.+ let result = unsafe { base.map_image_at(&source, 0, HostAlignedByteCount::ZERO, max) };+ assert!(result.is_err());+ }}diff --git a/crates/wasmtime/src/runtime/vm/sys/custom/mmap.rs b/crates/wasmtime/src/runtime/vm/sys/custom/mmap.rsindex bc2ff78d8b..30a9242b5e 100644--- a/crates/wasmtime/src/runtime/vm/sys/custom/mmap.rs+++ b/crates/wasmtime/src/runtime/vm/sys/custom/mmap.rs@@ -1,6 +1,6 @@use super::cvt;use crate::prelude::*;-use crate::runtime::vm::sys::{capi, vm::MemoryImageSource};+use crate::runtime::vm::sys::capi;use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};use core::ops::Range;use core::ptr::{self, NonNull};@@ -122,28 +122,6 @@ impl Mmap {}Ok(())}-- pub unsafe fn map_image_at(- &self,- image_source: &MemoryImageSource,- source_offset: u64,- memory_offset: HostAlignedByteCount,- memory_len: HostAlignedByteCount,- ) -> Result<()> {- assert_eq!(source_offset, 0);- unsafe {- let base = self- .memory- .as_ptr()- .byte_add(memory_offset.byte_count())- .cast();- cvt(capi::wasmtime_memory_image_map_at(- image_source.image_ptr().as_ptr(),- base,- memory_len.byte_count(),- ))- }- }}impl Drop for Mmap {diff --git a/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs b/crates/wasmtime/src/runtime/vm/sys/custom/vm.rsindex 60321f5674..7febc5153e 100644--- a/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs+++ b/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs@@ -95,9 +95,16 @@ impl MemoryImageSource {}}- #[inline]- pub(super) fn image_ptr(&self) -> SendSyncPtr<capi::wasmtime_memory_image> {- self.data+ /// See the documentation of the unix implementation of this method.+ pub unsafe fn map_at(&self, base: *mut u8, source_offset: u64, len: usize) -> Result<()> {+ assert_eq!(source_offset, 0);+ unsafe {+ cvt(capi::wasmtime_memory_image_map_at(+ self.data.as_ptr(),+ base.cast(),+ len,+ ))+ }}pub unsafe fn remap_as_zeros_at(&self, base: *mut u8, len: usize) -> Result<()> {diff --git a/crates/wasmtime/src/runtime/vm/sys/miri/mmap.rs b/crates/wasmtime/src/runtime/vm/sys/miri/mmap.rsindex 1afbc44f33..e33f236760 100644--- a/crates/wasmtime/src/runtime/vm/sys/miri/mmap.rs+++ b/crates/wasmtime/src/runtime/vm/sys/miri/mmap.rs@@ -6,7 +6,6 @@//! but it's enough to get various tests running relying on memories and such.use crate::prelude::*;-use crate::runtime::vm::sys::vm::MemoryImageSource;use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};use std::alloc::{self, Layout};use std::fs::File;@@ -118,16 +117,6 @@ impl Mmap {pub unsafe fn make_readwrite(&self, _range: Range<usize>) -> Result<()> {Ok(())}-- pub unsafe fn map_image_at(- &self,- image_source: &MemoryImageSource,- _source_offset: u64,- _memory_offset: HostAlignedByteCount,- _memory_len: HostAlignedByteCount,- ) -> Result<()> {- match *image_source {}- }}impl Drop for Mmap {diff --git a/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs b/crates/wasmtime/src/runtime/vm/sys/miri/vm.rsindex 00446aedf4..487a4cc3d7 100644--- a/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs+++ b/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs@@ -69,6 +69,15 @@ impl MemoryImageSource {Ok(None)}+ pub unsafe fn map_at(+ &self,+ _base: *mut u8,+ _source_offset: u64,+ _len: usize,+ ) -> io::Result<()> {+ match *self {}+ }+pub unsafe fn remap_as_zeros_at(&self, _base: *mut u8, _len: usize) -> io::Result<()> {match *self {}}diff --git a/crates/wasmtime/src/runtime/vm/sys/unix/mmap.rs b/crates/wasmtime/src/runtime/vm/sys/unix/mmap.rsindex 8d89bbb125..5b2301c760 100644--- a/crates/wasmtime/src/runtime/vm/sys/unix/mmap.rs+++ b/crates/wasmtime/src/runtime/vm/sys/unix/mmap.rs@@ -1,5 +1,4 @@use crate::prelude::*;-use crate::runtime::vm::sys::vm::MemoryImageSource;use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};use rustix::mm::{MprotectFlags, mprotect};use std::ops::Range;@@ -212,28 +211,6 @@ impl Mmap {Ok(())}-- pub unsafe fn map_image_at(- &self,- image_source: &MemoryImageSource,- source_offset: u64,- memory_offset: HostAlignedByteCount,- memory_len: HostAlignedByteCount,- ) -> Result<()> {- unsafe {- let map_base = self.memory.as_ptr().byte_add(memory_offset.byte_count());- let ptr = rustix::mm::mmap(- map_base.cast(),- memory_len.byte_count(),- rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE,- rustix::mm::MapFlags::PRIVATE | rustix::mm::MapFlags::FIXED,- image_source.as_file(),- source_offset,- )?;- assert_eq!(map_base.cast(), ptr);- };- Ok(())- }}impl Drop for Mmap {diff --git a/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs b/crates/wasmtime/src/runtime/vm/sys/unix/vm.rsindex 79ab69293c..b7db22041f 100644--- a/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs+++ b/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs@@ -1,6 +1,6 @@use crate::runtime::vm::sys::DecommitBehavior;use rustix::fd::AsRawFd;-use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect};+use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap, mmap_anonymous, mprotect};use std::fs::File;use std::io;#[cfg(feature = "std")]@@ -172,7 +172,7 @@ impl MemoryImageSource {Ok(Some(MemoryImageSource::Memfd(memfd)))}- pub(super) fn as_file(&self) -> &File {+ fn as_file(&self) -> &File {match *self {#[cfg(feature = "std")]MemoryImageSource::Mmap(ref file) => file,@@ -181,6 +181,30 @@ impl MemoryImageSource {}}+ /// Maps `len` bytes of this image, starting at `source_offset` within the+ /// source, copy-on-write and read/write at `base`, replacing whatever was+ /// mapped there before.+ ///+ /// # Safety+ ///+ /// `base` must be host-page-aligned and `base..base + len` must be a+ /// region of address space that the caller exclusively owns and that may+ /// have its pages replaced.+ pub unsafe fn map_at(&self, base: *mut u8, source_offset: u64, len: usize) -> io::Result<()> {+ let ptr = unsafe {+ mmap(+ base.cast(),+ len,+ ProtFlags::READ | ProtFlags::WRITE,+ MapFlags::PRIVATE | MapFlags::FIXED,+ self.as_file(),+ source_offset,+ )?+ };+ assert_eq!(base, ptr.cast());+ Ok(())+ }+pub unsafe fn remap_as_zeros_at(&self, base: *mut u8, len: usize) -> io::Result<()> {let ptr = unsafe {mmap_anonymous(diff --git a/crates/wasmtime/src/runtime/vm/sys/windows/mmap.rs b/crates/wasmtime/src/runtime/vm/sys/windows/mmap.rsindex 137887600a..792946a292 100644--- a/crates/wasmtime/src/runtime/vm/sys/windows/mmap.rs+++ b/crates/wasmtime/src/runtime/vm/sys/windows/mmap.rs@@ -1,5 +1,4 @@use crate::prelude::*;-use crate::runtime::vm::sys::vm::MemoryImageSource;use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};use std::fs::{File, OpenOptions};use std::io;@@ -259,16 +258,6 @@ impl Mmap {}Ok(())}-- pub unsafe fn map_image_at(- &self,- image_source: &MemoryImageSource,- _source_offset: u64,- _memory_offset: HostAlignedByteCount,- _memory_len: HostAlignedByteCount,- ) -> Result<()> {- match *image_source {}- }}impl Drop for Mmap {diff --git a/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs b/crates/wasmtime/src/runtime/vm/sys/windows/vm.rsindex cf767135fa..d3bdce281a 100644--- a/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs+++ b/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs@@ -80,6 +80,15 @@ impl MemoryImageSource {Ok(None)}+ pub unsafe fn map_at(+ &self,+ _base: *mut u8,+ _source_offset: u64,+ _len: usize,+ ) -> io::Result<()> {+ match *self {}+ }+pub unsafe fn remap_as_zeros_at(&self, _base: *mut u8, _len: usize) -> io::Result<()> {match *self {}}diff --git a/tests/all/memory_creator.rs b/tests/all/memory_creator.rsindex 8d7a1fc8fe..340894cf85 100644--- a/tests/all/memory_creator.rs+++ b/tests/all/memory_creator.rs@@ -1,193 +1,545 @@-#[cfg(all(not(target_os = "windows"), not(miri)))]-mod not_for_windows {- use wasmtime::*;+//! Tests of custom linear memories provided with `Config::with_host_memory`.+// None of this works on miri since we use actual system mmap utilities.+#![cfg(not(miri))]- use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect, munmap};+use std::ptr;+use std::sync::Arc;+use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};+use wasmtime::*;- use std::ptr::null_mut;- use std::sync::{Arc, Mutex};+const WASM_PAGE: usize = 0x10000;+const GUARD_SIZE: usize = WASM_PAGE;+const MAP_CAPACITY: usize = 10 << 20; // 10MiB as max cap for mmap.- struct CustomMemory {- mem: usize,- size: usize,- guard_size: usize,- used_wasm_bytes: usize,- glob_bytes_counter: Arc<Mutex<usize>>,+/// What `CustomMemory` does when asked to initialize itself from a CoW memory+/// image.+#[derive(Clone, Copy, PartialEq)]+#[cfg_attr(+ not(target_os = "linux"),+ allow(dead_code, reason = "CoW images are only built on Linux")+)]+enum ImageMode {+ /// Map the image with `MemoryImage::map_at`, return true.+ Map,+ /// Do not map, return false, let Wasmtime copy the data.+ ReturnFalse,+ /// Return true *without* initializing the memory.+ /// This is for tests to check that wasmtime does not manually copy anything.+ Lie,+ /// Fail initialization.+ Fail,+}++/// Observability of calls for tests.+#[derive(Default)]+struct Counters {+ num_memories_created: AtomicUsize,+ num_bytes_accessible: AtomicUsize,+ grow_to_calls: AtomicUsize,+ init_from_image_calls: AtomicUsize,+ images_mapped: AtomicUsize,+}++struct SendSyncPtr(*mut u8);+unsafe impl Send for SendSyncPtr {}+unsafe impl Sync for SendSyncPtr {}++/// A linear memory backed by an mmap.+/// `size` bytes are accessible, followed by inaccessible space up to `capacity` and a+/// guard region of `GUARD_SIZE` bytes.+struct CustomMemory {+ base: SendSyncPtr,+ capacity: usize,+ size: usize,+ /// Whether every `grow_to` moves the memory to a new reservation.+ /// When false, growing beyond capacity fails (static memory).+ move_on_grow: bool,+ image_mode: ImageMode,+ counters: Arc<Counters>,+}++impl CustomMemory {+ fn new(+ minimum: usize,+ capacity: usize,+ move_on_grow: bool,+ image_mode: ImageMode,+ counters: Arc<Counters>,+ ) -> Self {+ let capacity = capacity.max(minimum);+ // SAFETY: the reservation is new and `minimum` bytes fit in it.+ let base = unsafe {+ let base = sys::reserve(capacity + GUARD_SIZE);+ sys::make_available(base, minimum);+ base+ };+ counters.num_bytes_accessible.fetch_add(minimum, SeqCst);+ Self {+ base: SendSyncPtr(base),+ capacity,+ size: minimum,+ move_on_grow,+ image_mode,+ counters,+ }}- impl CustomMemory {- unsafe fn new(minimum: usize, maximum: usize, glob_counter: Arc<Mutex<usize>>) -> Self {- let page_size = rustix::param::page_size();- let guard_size = page_size;- let size = maximum + guard_size;- // We rely on the Wasm page size being multiple of host page size.- assert_eq!(size % page_size, 0);+ fn base(&self) -> *mut u8 {+ self.base.0+ }+}- let mem = unsafe {- mmap_anonymous(null_mut(), size, ProtFlags::empty(), MapFlags::PRIVATE)- .expect("mmap failed")- };+impl Drop for CustomMemory {+ fn drop(&mut self) {+ self.counters+ .num_bytes_accessible+ .fetch_sub(self.size, SeqCst);+ // SAFETY: this memory owns its reservation, which nothing uses anymore.+ unsafe { sys::release(self.base(), self.capacity + GUARD_SIZE) };+ }+}++// SAFETY: the first `size` bytes at `base` are committed, readable, writable+// and zero-initialized, and the rest of the reservation up to `capacity` plus+// the guard region is inaccessible. Reservations are aligned to at least the+// host page size. Without `move_on_grow` the memory never moves and refuses to+// grow beyond its capacity. With `move_on_grow` every growth moves it to a new+// reservation, which is only valid for growth beyond the capacity; growing+// within the capacity then deliberately violates the base pointer stability+// requirement, which one test uses to check that Wasmtime detects it.+unsafe impl LinearMemory for CustomMemory {+ fn byte_size(&self) -> usize {+ self.size+ }- // NOTE: mmap_anonymous returns zero initialized memory, which is relied upon by this- // API.+ fn byte_capacity(&self) -> usize {+ self.capacity+ }+ fn grow_to(&mut self, new_size: usize) -> Result<()> {+ self.counters.grow_to_calls.fetch_add(1, SeqCst);+ if self.move_on_grow {+ let capacity = new_size.max(self.capacity);+ // SAFETY: the new reservation is fresh, and the old one is owned+ // by this memory and released after copying its contents.unsafe {- mprotect(mem, minimum, MprotectFlags::READ | MprotectFlags::WRITE)- .expect("mprotect failed");+ let base = sys::reserve(capacity + GUARD_SIZE);+ sys::make_available(base, new_size);+ ptr::copy_nonoverlapping(self.base(), base, self.size);+ sys::release(self.base(), self.capacity + GUARD_SIZE);+ self.base = SendSyncPtr(base);}- *glob_counter.lock().unwrap() += minimum;-- Self {- mem: mem as usize,- size,- guard_size,- used_wasm_bytes: minimum,- glob_bytes_counter: glob_counter,+ self.capacity = capacity;+ } else {+ if new_size > self.capacity {+ bail!("custom memory cannot grow beyond its capacity");}+ // SAFETY: the new bytes lie within this memory's reservation. The+ // size is a multiple of 64K, so the start is page aligned.+ unsafe { sys::make_available(self.base().add(self.size), new_size - self.size) };}+ self.counters+ .num_bytes_accessible+ .fetch_add(new_size - self.size, SeqCst);+ self.size = new_size;+ Ok(())}- impl Drop for CustomMemory {- fn drop(&mut self) {- *self.glob_bytes_counter.lock().unwrap() -= self.used_wasm_bytes;- unsafe { munmap(self.mem as *mut _, self.size).expect("munmap failed") };- }+ fn as_ptr(&self) -> *mut u8 {+ self.base()}- unsafe impl LinearMemory for CustomMemory {- fn byte_size(&self) -> usize {- self.used_wasm_bytes- }-- fn byte_capacity(&self) -> usize {- self.size - self.guard_size+ fn init_from_image(&mut self, image: &MemoryImage) -> Result<bool> {+ self.counters.init_from_image_calls.fetch_add(1, SeqCst);+ match self.image_mode {+ ImageMode::Map => {+ assert!(image.linear_memory_offset() + image.len() <= self.size);+ // SAFETY: this memory owns its whole reservation and nothing+ // has accessed it yet.+ unsafe { image.map_at(self.base())? };+ self.counters.images_mapped.fetch_add(1, SeqCst);+ Ok(true)+ }+ ImageMode::ReturnFalse => Ok(false),+ ImageMode::Lie => Ok(true),+ ImageMode::Fail => bail!("custom memory refused its image"),}+ }+}- fn grow_to(&mut self, new_size: usize) -> wasmtime::Result<()> {- println!("grow to {new_size:x}");- let delta = new_size - self.used_wasm_bytes;- unsafe {- let start = (self.mem as *mut u8).add(self.used_wasm_bytes) as _;- mprotect(start, delta, MprotectFlags::READ | MprotectFlags::WRITE)- .expect("mprotect failed");- }+struct CustomMemoryCreator {+ image_mode: ImageMode,+ capacity_of_each: usize,+ move_on_grow: bool,+ counters: Arc<Counters>,+}- *self.glob_bytes_counter.lock().unwrap() += delta;- self.used_wasm_bytes = new_size;- Ok(())+impl CustomMemoryCreator {+ fn new(image_mode: ImageMode) -> Self {+ Self {+ image_mode,+ capacity_of_each: MAP_CAPACITY,+ move_on_grow: false,+ counters: Arc::default(),}+ }- fn as_ptr(&self) -> *mut u8 {- self.mem as *mut u8- }+ fn with_capacity(mut self, capacity: usize) -> Self {+ self.capacity_of_each = capacity;+ self}- struct CustomMemoryCreator {- pub num_created_memories: Mutex<usize>,- pub num_total_bytes: Arc<Mutex<usize>>,+ fn moving(mut self) -> Self {+ self.move_on_grow = true;+ self}+}- impl CustomMemoryCreator {- pub fn new() -> Self {- Self {- num_created_memories: Mutex::new(0),- num_total_bytes: Arc::new(Mutex::new(0)),- }- }+unsafe impl MemoryCreator for CustomMemoryCreator {+ fn new_memory(+ &self,+ ty: MemoryType,+ minimum: usize,+ _maximum: Option<usize>,+ reserved_size: Option<usize>,+ guard_size: usize,+ ) -> Result<Box<dyn LinearMemory>, String> {+ assert_eq!(guard_size, 0);+ assert_eq!(reserved_size, Some(0));+ assert!(!ty.is_64());+ let capacity = self.capacity_of_each;+ self.counters.num_memories_created.fetch_add(1, SeqCst);+ Ok(Box::new(CustomMemory::new(+ minimum,+ capacity,+ self.move_on_grow,+ self.image_mode,+ self.counters.clone(),+ )))}+}- unsafe impl MemoryCreator for CustomMemoryCreator {- fn new_memory(- &self,- ty: MemoryType,- minimum: usize,- maximum: Option<usize>,- reserved_size: Option<usize>,- guard_size: usize,- ) -> Result<Box<dyn LinearMemory>, String> {- assert_eq!(guard_size, 0);- assert_eq!(reserved_size, Some(0));- assert!(!ty.is_64());- unsafe {- // Cap the maximum at 10MiB to reduce the virtual memory- // allocated by this test to execute on 32-bit platforms.- let mem = Box::new(CustomMemory::new(- minimum,- maximum.unwrap_or(10 << 20),- self.num_total_bytes.clone(),- ));- *self.num_created_memories.lock().unwrap() += 1;- Ok(mem)- }- }+/// Returns a store whose memories are created by `creator`, and the creator's+/// counters. `configure` can further adjust the engine's configuration.+fn store(+ creator: CustomMemoryCreator,+ configure: impl FnOnce(&mut Config),+) -> (Store<()>, Arc<Counters>) {+ let counters = creator.counters.clone();+ let mut config = Config::new();+ config+ .with_host_memory(Arc::new(creator))+ .memory_reservation(0)+ .memory_guard_size(0);+ configure(&mut config);+ (Store::new(&Engine::new(&config).unwrap(), ()), counters)+}++/// A module whose memory is initialized by a data segment.+const DATA_MODULE: &str = r#"+ (module+ (memory (export "memory") 1 4)+ (data (i32.const 0) "hello")+ (func (export "load") (param i32) (result i32)+ (i32.load8_u (local.get 0)))+ (func (export "store") (param i32 i32)+ (i32.store8 (local.get 0) (local.get 1)))+ (func (export "grow") (param i32) (result i32)+ (memory.grow (local.get 0)))+ )+"#;++/// This function tests that:+/// - "hello" is accessible in the Wasm memory where it's expected to (in static data);+/// - this stays true after growing the memory by a page.+fn check_data_module(store: &mut Store<()>, instance: Instance) -> Result<()> {+ let load = instance.get_typed_func::<i32, i32>(&mut *store, "load")?;+ let grow = instance.get_typed_func::<i32, i32>(&mut *store, "grow")?;+ for (i, byte) in b"hello\0".iter().enumerate() {+ assert_eq!(load.call(&mut *store, i as i32)?, i32::from(*byte));+ }+ assert_eq!(grow.call(&mut *store, 1)?, 1);+ for (i, byte) in b"hello\0".iter().enumerate() {+ assert_eq!(load.call(&mut *store, i as i32)?, i32::from(*byte));}+ assert_eq!(load.call(&mut *store, WASM_PAGE as i32)?, 0);+ Ok(())+}- fn config() -> (Store<()>, Arc<CustomMemoryCreator>) {- let mem_creator = Arc::new(CustomMemoryCreator::new());- let mut config = Config::new();- config- .with_host_memory(mem_creator.clone())- .memory_reservation(0)- .memory_guard_size(0);- (Store::new(&Engine::new(&config).unwrap(), ()), mem_creator)+// CoW of in-memory data requires memfd, which is linux-only.+#[cfg(target_os = "linux")]+mod linux_only {+ use super::*;+ // Regression test for [#10740](https://github.com/bytecodealliance/wasmtime/issues/10740):+ // a custom memory that does not support CoW forces a copy from Wasmtime.+ #[test]+ #[cfg(target_os = "linux")]+ fn host_memory_with_cow_image_when_impl_refuses_mapping() -> Result<()> {+ let (mut store, counters) =+ store(CustomMemoryCreator::new(ImageMode::ReturnFalse), |cfg| {+ cfg.memory_init_cow(true);+ });+ let module = Module::new(store.engine(), DATA_MODULE)?;+ let instance = Instance::new(&mut store, &module, &[])?;+ check_data_module(&mut store, instance)?;+ assert_eq!(counters.num_memories_created.load(SeqCst), 1);+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 1);+ assert_eq!(counters.images_mapped.load(SeqCst), 0);+ Ok(())+ }++ // When `init_from_image` returns `true`, Wasmtime trusts it and doesn't copy+ // the module's data segments into the memory.+ // This is relying on our CustomMemory breaking the contract (ImageMode::Lie) and not actually performing the mapping.+ #[test]+ #[cfg(target_os = "linux")]+ fn host_memory_init_from_image_true_skips_data_segments() -> Result<()> {+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Lie), |cfg| {+ cfg.memory_init_cow(true);+ });+ let module = Module::new(store.engine(), DATA_MODULE)?;+ let instance = Instance::new(&mut store, &module, &[])?;+ let load = instance.get_typed_func::<i32, i32>(&mut store, "load")?;+ // The memory claimed to hold the image but left itself zeroed.+ assert_eq!(load.call(&mut store, 0)?, 0); // Would be 'h' if data was copied.+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 1);+ Ok(())}+ // This checks the CoW semantics: writing to one instantiated memory does not leak into memories+ // later instantiated from the same image.#[test]- fn host_memory() -> wasmtime::Result<()> {- let (mut store, mem_creator) = config();- let module = Module::new(- store.engine(),- r#"- (module- (memory (export "memory") 1)- )- "#,- )?;- Instance::new(&mut store, &module, &[])?;-- assert_eq!(*mem_creator.num_created_memories.lock().unwrap(), 1);+ #[cfg(target_os = "linux")]+ fn host_memory_with_cow_image_when_impl_maps() -> Result<()> {+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Map), |cfg| {+ cfg.memory_init_cow(true);+ });+ let module = Module::new(store.engine(), DATA_MODULE)?;++ let first_instance = Instance::new(&mut store, &module, &[])?;+ check_data_module(&mut store, first_instance)?;++ let first_store = first_instance.get_typed_func::<(i32, i32), ()>(&mut store, "store")?;+ first_store.call(&mut store, (0, i32::from(b'x')))?;+ let second_instance = Instance::new(&mut store, &module, &[])?;++ let first_load = first_instance.get_typed_func::<i32, i32>(&mut store, "load")?;+ assert_eq!(first_load.call(&mut store, 0)?, i32::from(b'x'));+ let second_load = second_instance.get_typed_func::<i32, i32>(&mut store, "load")?;+ assert_eq!(second_load.call(&mut store, 0)?, i32::from(b'h'));++ assert_eq!(counters.num_memories_created.load(SeqCst), 2);+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 2);+ assert_eq!(counters.images_mapped.load(SeqCst), 2);Ok(())}+ // Check that failures from implementations are propagated as-is (part of the public contract).#[test]- fn host_memory_grow() -> wasmtime::Result<()> {- let (mut store, mem_creator) = config();- let module = Module::new(- store.engine(),- r#"- (module- (func $f (drop (memory.grow (i32.const 1))))- (memory (export "memory") 1 2)- (start $f)- )- "#,- )?;-- Instance::new(&mut store, &module, &[])?;- let instance2 = Instance::new(&mut store, &module, &[])?;-- assert_eq!(*mem_creator.num_created_memories.lock().unwrap(), 2);-- assert_eq!(- instance2- .get_memory(&mut store, "memory")- .unwrap()- .size(&store),- 2+ #[cfg(target_os = "linux")]+ fn host_memory_init_from_image_error_fails_instantiation() -> Result<()> {+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Fail), |cfg| {+ cfg.memory_init_cow(true);+ });+ let module = Module::new(store.engine(), DATA_MODULE)?;+ let err = Instance::new(&mut store, &module, &[]).unwrap_err();+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 1);+ assert!(+ format!("{err:?}").contains("custom memory refused its image"),+ "unexpected error: {err:?}");+ Ok(())+ }+}- // we take the lock outside the assert, so it won't get poisoned on assert failure- let tot_pages = *mem_creator.num_total_bytes.lock().unwrap();- assert_eq!(- tot_pages,- (4 * wasmtime_environ::Memory::DEFAULT_PAGE_SIZE) as usize- );+// With CoW disabled `init_from_image` should not be called,+// and the data segments should be copied into the custom memory.+#[test]+fn host_memory_without_cow_does_not_call_init_from_image() -> Result<()> {+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Fail), |cfg| {+ cfg.memory_init_cow(false);+ });+ let module = Module::new(store.engine(), DATA_MODULE)?;+ let instance = Instance::new(&mut store, &module, &[])?;+ check_data_module(&mut store, instance)?;+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 0);+ Ok(())+}- drop(store);- let tot_pages = *mem_creator.num_total_bytes.lock().unwrap();- assert_eq!(tot_pages, 0);+// Regression test: memories created by the host with `Memory::new` used to+// ignore the configured `MemoryCreator`.+#[test]+fn host_created_memory_uses_creator() -> Result<()> {+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Fail), |_| {});- Ok(())+ let memory = Memory::new(&mut store, MemoryType::new(1, Some(3))?)?;+ assert_eq!(counters.num_memories_created.load(SeqCst), 1);+ assert_eq!(counters.num_bytes_accessible.load(SeqCst), WASM_PAGE);++ // Growing from the host goes through `CustomMemory::grow_to`...+ memory.grow(&mut store, 1)?;+ assert_eq!(counters.num_bytes_accessible.load(SeqCst), 2 * WASM_PAGE);++ // ... and so does growing from Wasm.+ memory.data_mut(&mut store)[WASM_PAGE] = 42;+ let module = Module::new(+ store.engine(),+ r#"+ (module+ (memory (import "" "memory") 1)+ (func (export "grow") (result i32) (memory.grow (i32.const 1)))+ (func (export "load") (param i32) (result i32)+ (i32.load8_u (local.get 0)))+ )+ "#,+ )?;+ let instance = Instance::new(&mut store, &module, &[memory.into()])?;+ let load = instance.get_typed_func::<i32, i32>(&mut store, "load")?;+ let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?;++ assert_eq!(load.call(&mut store, WASM_PAGE as i32)?, 42);+ assert_eq!(grow.call(&mut store, ())?, 2);+ assert_eq!(counters.num_bytes_accessible.load(SeqCst), 3 * WASM_PAGE);++ // The importing instance didn't create a memory of its own.+ assert_eq!(counters.num_memories_created.load(SeqCst), 1);++ drop(store);+ assert_eq!(counters.num_bytes_accessible.load(SeqCst), 0);++ Ok(())+}++// A memory that moves when growing beyond its capacity keeps its contents,+// and wasm code uses the new location afterwards.+#[test]+fn growth_that_moves_memory() -> Result<()> {+ let creator = CustomMemoryCreator::new(ImageMode::ReturnFalse)+ .with_capacity(WASM_PAGE)+ .moving();+ let (mut store, counters) = store(creator, |_| {});+ let module = Module::new(store.engine(), DATA_MODULE)?;+ let instance = Instance::new(&mut store, &module, &[])?;+ let memory = instance.get_memory(&mut store, "memory").unwrap();+ let load = instance.get_typed_func::<i32, i32>(&mut store, "load")?;+ let store_byte = instance.get_typed_func::<(i32, i32), ()>(&mut store, "store")?;+ let grow = instance.get_typed_func::<i32, i32>(&mut store, "grow")?;++ store_byte.call(&mut store, (7, 42))?;+ let base_before = memory.data_ptr(&store);++ assert_eq!(grow.call(&mut store, 1)?, 1);+ assert_eq!(counters.grow_to_calls.load(SeqCst), 1);+ // Sanity check: memory moved somewhere else.+ assert_ne!(memory.data_ptr(&store), base_before);+ assert_eq!(memory.data_size(&store), 2 * WASM_PAGE);++ assert_eq!(load.call(&mut store, 7)?, 42);+ store_byte.call(&mut store, (WASM_PAGE as i32 + 7, 43))?;+ assert_eq!(load.call(&mut store, WASM_PAGE as i32 + 7)?, 43);+ assert_eq!(memory.data(&store)[WASM_PAGE + 7], 43);+ Ok(())+}++// With `Config::memory_may_move(false)` a memory still grows within its+// capacity, but growth beyond it is refused without asking the memory.+#[test]+fn growth_limited_to_capacity_when_memory_may_not_move() -> Result<()> {+ let creator = CustomMemoryCreator::new(ImageMode::ReturnFalse).with_capacity(2 * WASM_PAGE);+ let (mut store, counters) = store(creator, |config| {+ config.memory_may_move(false);+ });+ let module = Module::new(store.engine(), DATA_MODULE)?;+ let instance = Instance::new(&mut store, &module, &[])?;+ let memory = instance.get_memory(&mut store, "memory").unwrap();+ let grow = instance.get_typed_func::<i32, i32>(&mut store, "grow")?;+ let base = memory.data_ptr(&store);+ assert_eq!(memory.data_size(&store), WASM_PAGE);++ assert_eq!(grow.call(&mut store, 1)?, 1);+ assert_eq!(counters.grow_to_calls.load(SeqCst), 1);+ assert_eq!(memory.data_size(&store), 2 * WASM_PAGE);+ assert_eq!(memory.data_ptr(&store), base);++ assert_eq!(grow.call(&mut store, 1)?, -1);+ assert_eq!(counters.grow_to_calls.load(SeqCst), 1);+ assert_eq!(memory.data_size(&store), 2 * WASM_PAGE);+ assert_eq!(memory.data_ptr(&store), base);+ Ok(())+}++// A memory that moves while growing within its capacity violates the `LinearMemory` contract.+// Wasmtime does not guarantee that this is detected, but it is an easy check that prevents+// potentially catastrophic errors, so we make sure it leads to a panic.+#[test]+#[should_panic(expected = "linear memory moved on grow when it must not have")]+fn growth_within_capacity_that_moves_memory_panics() {+ let creator = CustomMemoryCreator::new(ImageMode::ReturnFalse)+ .with_capacity(2 * WASM_PAGE)+ .moving();+ let (mut store, _counters) = store(creator, |_| {});+ let memory = Memory::new(&mut store, MemoryType::new(1, None).unwrap()).unwrap();+ let _ = memory.grow(&mut store, 1);+}++// Mmap providers for unix and windows.+mod sys {+ #[cfg(unix)]+ pub use unix::{make_available, release, reserve};+ #[cfg(windows)]+ pub use windows::{make_available, release, reserve};++ #[cfg(unix)]+ mod unix {+ use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect, munmap};++ pub unsafe fn reserve(size: usize) -> *mut u8 {+ let ptr = unsafe {+ mmap_anonymous(+ std::ptr::null_mut(),+ size,+ ProtFlags::empty(),+ MapFlags::PRIVATE,+ )+ };+ ptr.expect("mmap failed").cast()+ }++ pub unsafe fn make_available(ptr: *mut u8, len: usize) {+ unsafe { mprotect(ptr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE) }+ .expect("mprotect failed");+ }++ pub unsafe fn release(ptr: *mut u8, size: usize) {+ unsafe { munmap(ptr.cast(), size) }.expect("munmap failed");+ }+ }++ #[cfg(windows)]+ mod windows {+ use windows_sys::Win32::System::Memory::{+ MEM_COMMIT, MEM_RELEASE, MEM_RESERVE, PAGE_NOACCESS, PAGE_READWRITE, VirtualAlloc,+ VirtualFree,+ };++ pub unsafe fn reserve(size: usize) -> *mut u8 {+ let ptr = unsafe { VirtualAlloc(std::ptr::null(), size, MEM_RESERVE, PAGE_NOACCESS) };+ assert!(!ptr.is_null(), "VirtualAlloc(MEM_RESERVE) failed");+ ptr.cast()+ }++ pub unsafe fn make_available(ptr: *mut u8, len: usize) {+ if len == 0 {+ return;+ }+ let ret = unsafe { VirtualAlloc(ptr.cast(), len, MEM_COMMIT, PAGE_READWRITE) };+ assert!(!ret.is_null(), "VirtualAlloc(MEM_COMMIT) failed");+ }++ pub unsafe fn release(ptr: *mut u8, _size: usize) {+ let ret = unsafe { VirtualFree(ptr.cast(), 0, MEM_RELEASE) };+ assert_ne!(ret, 0, "VirtualFree failed");+ }}}