Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Moving to `LinearMemory` trait and fixing custom memories (#14885)

Mat Gienieczko65fb6644964e
diff --git a/crates/wasmtime/src/runtime/memory.rs b/crates/wasmtime/src/runtime/memory.rs
index 1eac1e3503..e4f01edc37 100644
--- a/crates/wasmtime/src/runtime/memory.rs
+++ b/crates/wasmtime/src/runtime/memory.rs
@@ -6,6 +6,8 @@ use crate::trampoline::generate_memory_export;
#[cfg(feature = "async")]
use crate::vm::VMStore;
use crate::{AsContext, AsContextMut, Engine, MemoryType, StoreContext, StoreContextMut};
+use alloc::sync::Arc;
+use core::any::Any;
use core::cell::UnsafeCell;
use core::fmt;
use core::slice;
@@ -731,24 +733,74 @@ impl Memory {
/// MemoryCreator, one can supply wasmtime with custom allocated host managed
/// memory.
///
+/// Note that this is a relatively advanced feature and it is recommended to be
+/// familiar with wasmtime runtime code to use it.
+///
+/// In the following, `minimum`, `reserved_size_in_bytes`, and `guard_size_in_bytes` arguments are given to a
+/// [`MemoryCreator::new_memory`] implementation.
+///
/// # Safety
///
-/// The memory should be page aligned and a multiple of page size.
-/// To prevent possible silent overflows, the memory should be protected by a
-/// guard page. Additionally the safety concerns explained in ['Memory'], for
-/// accessing the memory apply here as well.
+/// Wasmtime's compiled code and runtime rely on the following invariants for memory safety.
+/// Note that while we strive to be thorough in this documentation, we do not guarantee that the
+/// following points are exhaustive.
+/// When using a custom memory backend in a security-sensitive context auditing just the external
+/// implementation of this trait is not sufficient.
///
-/// The value returned by [`LinearMemory::byte_size`] must be at least the
-/// `minimum` size the memory was created with, must be at least `new_size`
-/// after a successful [`LinearMemory::grow_to`], and must never decrease.
-/// At least that many bytes starting at [`LinearMemory::as_ptr`] must be
-/// valid to read and write. Compiled WebAssembly code relies on these
-/// properties when performing bounds checks, so violating them can lead to
-/// out-of-bounds memory accesses.
+/// - **Valid region.** From creation until the memory is dropped, the first
+/// [`byte_size`] bytes starting at the [`as_ptr`]
+/// memory location must be allocated, initialized, readable, and writeable.
+/// The pointer must be non-null.
+/// Nothing other than Wasmtime may read, write, unmap, or otherwise modify this region while the
+/// memory is in use by Wasmtime.
+/// If the memory moves (although see conditions on stability below), the previous region is no
+/// longer in use by Wasmtime.
+/// Moreover, be aware that Wasmtime currently has a soundness hole for shared memory,
+/// as multiple threads may access the same slice of memory mutably at the same time (aliasing) (see
+/// [issue #4203](https://github.com/bytecodealliance/wasmtime/issues/4203)).
///
-/// Note that this is a relatively advanced feature and it is recommended to be
-/// familiar with wasmtime runtime code to use it.
-pub unsafe trait LinearMemory: Send + Sync + 'static {
+/// - **Size.** [`byte_size`] must be at least `minimum` after creation
+/// and at least `new_size` after a successful call to [`grow_to`],
+/// and it must never decrease.
+///
+/// - **Base pointer stability.** A call to [`as_ptr`] must return the same
+/// address until the next successful call to [`grow_to`].
+/// Moreover, a successful [`grow_to(new_size)`][`grow_to`] call where `new_size` is at most the [`byte_capacity`](LinearMemory::byte_capacity)
+/// reported before the call must not change the result of [`as_ptr`].
+/// If static memory is forced (e.g. when using shared memory or when [`Config::memory_may_move`](crate::Config::memory_may_move) is disabled), Wasmtime will never ask the memory to grow beyond its [`byte_capacity`].
+///
+/// - **Base pointer alignment.** The base pointer returned by [`as_ptr`] must always be 8-byte aligned.
+/// Moreover, if `guard_size_in_bytes` is non-zero, or `reserved_size_in_bytes` is `Some` and non-zero, then [`as_ptr`] must be
+/// aligned to the host page size at all times.
+///
+/// - **Inaccessible region.** If: `guard_size_in_bytes` is non-zero, or `reserved_size_in_bytes` is `Some` and non-zero;
+/// then every byte from `as_ptr() + byte_size()`, rounded up to the next host page boundary, up to
+/// `as_ptr() + max(reserved_size_in_bytes, byte_size()) + guard_size_in_bytes` must be
+/// inaccessible, so that any access to it faults. This must hold at all times, including after growth.
+///
+/// Additionally, observe the safety requirements when the memory is accessed, as explained in [`Memory`].
+///
+/// # Correctness
+///
+/// The following constraints do not cause memory safety issues, but break WebAssembly semantics,
+/// or expose potentially unwanted data to WebAssembly programs:
+///
+/// - [`byte_size`] should be exactly `minimum` after creation and exactly `new_size` after a
+/// successful call to [`grow_to(new_size)`][`grow_to`]. Wasmtime only requests sizes that are
+/// multiples of the Wasm page size.
+///
+/// - The memory's contents must be zeroed at creation, and bytes made accessible by [`grow_to`] must
+/// be zeroed.
+///
+/// - If [`init_from_image`] returns `Ok(true)`, then the bytes at [`MemoryImage::linear_memory_offset`]
+/// must equal the image and all other bytes must be zero.
+///
+/// [`grow_to`]: LinearMemory::grow_to
+/// [`byte_size`]: LinearMemory::byte_size
+/// [`as_ptr`]: LinearMemory::as_ptr
+/// [`byte_capacity`]: LinearMemory::byte_capacity
+/// [`init_from_image`]: LinearMemory::init_from_image
+pub unsafe trait LinearMemory: Any + Send + Sync {
/// Returns the number of allocated bytes which are accessible at this time.
fn byte_size(&self) -> usize;
@@ -767,6 +819,151 @@ pub unsafe trait LinearMemory: Send + Sync + 'static {
/// Return the allocated memory as a mutable pointer to u8.
fn as_ptr(&self) -> *mut u8;
+
+ /// Optionally initializes this memory's contents from a copy-on-write
+ /// [`MemoryImage`].
+ ///
+ /// When [copy-on-write memory initialization][cow_config] is enabled and a
+ /// module's data segments for this memory can be represented as an image,
+ /// Wasmtime calls this method once, right after the memory is created
+ /// and before anything else reads or writes it.
+ ///
+ /// If the function succeeds with `false`, the runtime assumes the memory was not initialized.
+ /// It then fully copies the module's data segments into the memory region.
+ /// This is always correct and is essentially equivalent to CoW being disabled.
+ ///
+ /// If the function succeeds with `true`, the runtime assumes the memory's contents match `image` exactly.
+ /// The bytes of the image must start at [`MemoryImage::linear_memory_offset`] and all other bytes must be zero.
+ ///
+ /// ## Shortcut for memory maps
+ ///
+ /// [`MemoryImage`] exposes a shortcut, [`MemoryImage::map_at`], which allows an implementation
+ /// backed by a memory map to replace a portion of its mapping with the pages of the image.
+ /// Consult that function's documentation for safety notes.
+ ///
+ /// Returning an error causes instantiation to fail with that error.
+ ///
+ /// [cow_config]: crate::Config::memory_init_cow
+ fn init_from_image(&mut self, image: &MemoryImage) -> Result<bool> {
+ let _ = image;
+ Ok(false)
+ }
+}
+
+/// A copy-on-write image of the initial contents of a linear memory.
+///
+/// When [`Config::memory_init_cow`](crate::Config::memory_init_cow) is enabled
+/// Wasmtime prepares, where possible, an image of a module's data segments for
+/// each of its linear memories that can be mapped into memory copy-on-write
+/// instead of being copied byte-by-byte. Such images are handed to
+/// [`LinearMemory::init_from_image`] for the specific memory implementation to map into its
+/// underlying representation.
+///
+/// An image covers [`len()`](MemoryImage::len) bytes of the linear memory starting at
+/// [`linear_memory_offset()`](MemoryImage::linear_memory_offset).
+/// Both values are multiples of the host page size, and the image always lies within
+/// the minimum size of the memory it was created for.
+/// All other bytes are expected to be zero.
+///
+/// Cloning a `MemoryImage` is cheap and keeps the underlying image alive.
+#[derive(Clone)]
+pub struct MemoryImage {
+ image: Arc<vm::MemoryImage>,
+}
+
+impl MemoryImage {
+ pub(crate) fn new(image: Arc<vm::MemoryImage>) -> MemoryImage {
+ MemoryImage { image }
+ }
+
+ /// This is a convenience method for the internal `MmapMemory` to access the underlying image
+ /// directly and use its [`map_at`](vm::MemoryImage::map_at) function.
+ #[cfg(has_virtual_memory)]
+ pub(crate) fn vm(&self) -> &vm::MemoryImage {
+ &self.image
+ }
+
+ /// Returns the offset, in bytes, within linear memory where this image
+ /// starts.
+ ///
+ /// This is guaranteed to be a multiple of the host page size.
+ pub fn linear_memory_offset(&self) -> usize {
+ self.image.linear_memory_offset()
+ }
+
+ /// Returns the length, in bytes, of this image.
+ ///
+ /// This is guaranteed to be a multiple of the host page size.
+ pub fn len(&self) -> usize {
+ self.image.len()
+ }
+
+ /// Returns whether this image is empty (length equal to zero).
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ /// Returns the bytes of this image.
+ ///
+ /// The returned slice is [`len()`](MemoryImage::len) bytes long and holds
+ /// the initial contents of linear memory starting at
+ /// [`linear_memory_offset()`](MemoryImage::linear_memory_offset). All
+ /// bytes of the memory outside of that range are initially zero.
+ pub fn data(&self) -> &[u8] {
+ self.image.data()
+ }
+
+ /// Maps this image copy-on-write into the linear memory starting at
+ /// `base`.
+ ///
+ /// The bytes `base + linear_memory_offset()` up to
+ /// `base + linear_memory_offset() + len()` are replaced with a private,
+ /// readable and writable copy-on-write mapping of the image. Nothing
+ /// outside of that range is modified.
+ ///
+ /// Returns an error if `base` is not aligned to the host page size or if
+ /// the underlying mapping operation fails.
+ ///
+ /// # Safety
+ ///
+ /// The caller must exclusively own the virtual memory range being mapped
+ /// over, nothing may hold references into it, and it must be valid for
+ /// its pages to be replaced by a new mapping (for example by `mmap` with
+ /// `MAP_FIXED` on Unix). Any previous contents and memory protections of
+ /// that range are lost. The mapping lives until the range is unmapped or
+ /// mapped over again by its owner. In particular, the runtime does not unmap the image by itself.
+ /// The owner of the virtual memory range must clean it up after usage.
+ ///
+ /// The typical usage is owning a memory-mapped region and requesting the image to be mapped
+ /// into a subrange of that mapping. Unmapping the entire region at the end of the memory's
+ /// lifetime takes care of the cleanup.
+ pub unsafe fn map_at(&self, base: *mut u8) -> Result<()> {
+ // SAFETY: forwarded to the caller.
+ unsafe { self.image.map_at_ptr(base) }
+ }
+}
+
+impl fmt::Debug for MemoryImage {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("MemoryImage")
+ .field("linear_memory_offset", &self.linear_memory_offset())
+ .field("len", &self.len())
+ .finish_non_exhaustive()
+ }
+}
+
+impl PartialEq for MemoryImage {
+ fn eq(&self, other: &Self) -> bool {
+ Arc::ptr_eq(&self.image, &other.image)
+ }
+}
+
+impl Eq for MemoryImage {}
+
+impl core::hash::Hash for MemoryImage {
+ fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
+ core::ptr::hash(Arc::as_ptr(&self.image), state)
+ }
}
/// A memory creator. Can be used to provide a memory creator
@@ -1171,4 +1368,127 @@ mod tests {
Ok(())
}
+
+ /// Offset of the data segment used by the `MemoryImage` tests: inside the
+ /// second Wasm page, but not at the start of a host page.
+ #[cfg(all(target_os = "linux", not(miri)))]
+ const IMAGE_DATA_OFFSET: usize = 0x10000 + 8;
+
+ /// Compiles a module whose single memory is initialized by one data segment
+ /// at `IMAGE_DATA_OFFSET` and returns the CoW image Wasmtime builds for it.
+ /// The module is never instantiated, we just take the image of the memory.
+ #[cfg(all(target_os = "linux", not(miri)))]
+ fn memory_image_for_data(data: &str) -> Result<MemoryImage> {
+ let mut config = Config::new();
+ config.memory_init_cow(true);
+ let engine = Engine::new(&config)?;
+ let module = Module::new(
+ &engine,
+ // Module with (at least) 2 pages of memory with data written at IMAGE_DATA_OFFSET.
+ format!("(module (memory 2) (data (i32.const {IMAGE_DATA_OFFSET}) \"{data}\"))"),
+ )?;
+ let images = module.memory_images()?.expect("module has memory images");
+ let image = images
+ .get_memory_image(wasmtime_environ::DefinedMemoryIndex::from_u32(0))
+ .expect("memory 0 has an image");
+ Ok(MemoryImage::new(image.clone()))
+ }
+
+ /// The public accessors report the internal image's placement, and:
+ /// - both values are multiples of the host page size; and
+ /// - the image covers the module's data.
+ #[test]
+ #[cfg(all(target_os = "linux", not(miri)))]
+ fn memory_image_accessors() -> Result<()> {
+ let page_size = crate::runtime::vm::host_page_size();
+ let image = memory_image_for_data("hello")?;
+ let offset = image.linear_memory_offset();
+ let len = image.len();
+
+ assert_eq!(offset, image.vm().linear_memory_offset());
+ assert_eq!(len, image.vm().len());
+ assert_eq!(offset % page_size, 0);
+ assert_eq!(len % page_size, 0);
+ assert!(!image.is_empty());
+ // Image starts at or before the data...
+ assert!(offset <= IMAGE_DATA_OFFSET);
+ // ... and reaches past the end of the data.
+ assert!(offset + len >= IMAGE_DATA_OFFSET + "hello".len());
+
+ let data = image.data();
+ assert_eq!(data.len(), len);
+ assert_eq!(data, image.vm().data());
+ let data_start = IMAGE_DATA_OFFSET - offset;
+ assert_eq!(&data[data_start..][..5], b"hello");
+ assert!(data[..data_start].iter().all(|b| *b == 0));
+ assert!(data[data_start + 5..].iter().all(|b| *b == 0));
+
+ // Clones share the same underlying image.
+ let clone = image.clone();
+ assert!(core::ptr::eq(clone.vm(), image.vm()));
+ Ok(())
+ }
+
+ #[test]
+ #[cfg(all(target_os = "linux", not(miri)))]
+ fn memory_image_identity() -> Result<()> {
+ use std::hash::{BuildHasher, RandomState};
+
+ let mut config = Config::new();
+ config.memory_init_cow(true);
+ let engine = Engine::new(&config)?;
+ let wat = r#"(module (memory 1) (data (i32.const 0) "hello"))"#;
+ let image_of = |module: &Module| -> Result<MemoryImage> {
+ let images = module.memory_images()?.expect("module has memory images");
+ let image = images
+ .get_memory_image(wasmtime_environ::DefinedMemoryIndex::from_u32(0))
+ .expect("memory 0 has an image");
+ Ok(MemoryImage::new(image.clone()))
+ };
+
+ let module = Module::new(&engine, wat)?;
+ let image = image_of(&module)?;
+ let same = image_of(&module)?; // Same module instance gives the same image.
+ let clone = image.clone(); // Clone just clones an Arc.
+
+ assert_eq!(image, same);
+ assert_eq!(image, clone);
+
+ let hasher = RandomState::new();
+ assert_eq!(hasher.hash_one(&image), hasher.hash_one(&same));
+ assert_eq!(hasher.hash_one(&image), hasher.hash_one(&clone));
+
+ Ok(())
+ }
+
+ /// `MemoryImage::map_at` places the image's bytes at
+ /// `base + linear_memory_offset()` and zeroes the rest.
+ #[test]
+ #[cfg(all(target_os = "linux", not(miri)))]
+ fn memory_image_map_at() -> Result<()> {
+ use crate::runtime::vm::{HostAlignedByteCount, Mmap};
+
+ let image = memory_image_for_data("hello")?;
+ let size =
+ HostAlignedByteCount::new_rounded_up(image.linear_memory_offset() + image.len())?;
+ let mmap = Mmap::accessible_reserved(HostAlignedByteCount::ZERO, size)?;
+ let base = mmap.as_non_null().as_ptr();
+
+ // SAFETY: `mmap` was just created, is owned by this test and covers
+ // the whole range the image is mapped into.
+ unsafe { image.map_at(base)? };
+
+ // SAFETY: `map_at` made the image's range readable.
+ let mapped = unsafe {
+ core::slice::from_raw_parts(base.add(image.linear_memory_offset()), image.len())
+ };
+ let data_start = IMAGE_DATA_OFFSET - image.linear_memory_offset();
+
+ assert_eq!(mapped, image.data());
+ assert_eq!(&mapped[data_start..][..5], b"hello");
+ assert!(mapped[..data_start].iter().all(|b| *b == 0));
+ assert!(mapped[data_start + 5..].iter().all(|b| *b == 0));
+
+ Ok(())
+ }
}
diff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rs
index 71c8883344..b906b616dd 100644
--- a/crates/wasmtime/src/runtime/trampoline/memory.rs
+++ b/crates/wasmtime/src/runtime/trampoline/memory.rs
@@ -4,8 +4,8 @@ use crate::prelude::*;
use crate::runtime::vm::mpk::ProtectionKey;
use crate::runtime::vm::{
CompiledModuleId, InstanceAllocationRequest, InstanceAllocator, Memory, MemoryAllocationIndex,
- MemoryBase, ModuleRuntimeInfo, OnDemandInstanceAllocator, RuntimeLinearMemory,
- RuntimeMemoryCreator, SharedMemory, Table, TableAllocationIndex,
+ ModuleRuntimeInfo, OnDemandInstanceAllocator, RuntimeMemoryCreator, SharedMemory, Table,
+ TableAllocationIndex,
};
use crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque, StoreResourceLimiter};
use alloc::sync::Arc;
@@ -49,10 +49,16 @@ pub async fn create_memory(
// We create an instance in the on-demand allocator when creating handles
// associated with external objects. The configured instance allocator
// should only be used when creating module instances as we don't want host
- // objects to count towards instance limits.
+ // objects to count towards instance limits. Host-created memories must
+ // still use the configured memory creator, though, see
+ // `Config::with_host_memory`.
let allocator = SingleMemoryInstance {
preallocation,
- ondemand: OnDemandInstanceAllocator::default(),
+ ondemand: OnDemandInstanceAllocator::new(
+ store.engine().config().mem_creator.clone(),
+ 0,
+ false,
+ ),
};
unsafe {
store
@@ -68,36 +74,6 @@ pub async fn create_memory(
}
}
-struct LinearMemoryProxy {
- mem: Box<dyn LinearMemory>,
-}
-
-impl RuntimeLinearMemory for LinearMemoryProxy {
- fn byte_size(&self) -> usize {
- self.mem.byte_size()
- }
-
- fn byte_capacity(&self) -> usize {
- self.mem.byte_capacity()
- }
-
- fn grow_to(&mut self, new_size: usize) -> Result<()> {
- self.mem.grow_to(new_size)
- }
-
- fn base(&self) -> MemoryBase {
- MemoryBase::new_raw(self.mem.as_ptr())
- }
-
- fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {
- let base = core::ptr::NonNull::new(self.mem.as_ptr()).unwrap();
- crate::vm::VMMemoryDefinition {
- base: base.into(),
- current_length: self.mem.byte_size().into(),
- }
- }
-}
-
#[derive(Clone)]
pub(crate) struct MemoryCreatorProxy(pub Arc<dyn MemoryCreator>);
@@ -108,7 +84,7 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy {
memory_tunables: &MemoryTunables<'_>,
minimum: usize,
maximum: Option<usize>,
- ) -> Result<Box<dyn RuntimeLinearMemory>> {
+ ) -> Result<Box<dyn LinearMemory>> {
let reserved_size_in_bytes = Some(memory_tunables.reservation().try_into().unwrap());
self.0
.new_memory(
@@ -118,7 +94,6 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy {
reserved_size_in_bytes,
usize::try_from(memory_tunables.guard_size()).unwrap(),
)
- .map(|mem| Box::new(LinearMemoryProxy { mem }) as Box<dyn RuntimeLinearMemory>)
.map_err(|e| format_err!(e))
}
}
diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs
index e3a3606d12..269486ef3f 100644
--- a/crates/wasmtime/src/runtime/vm.rs
+++ b/crates/wasmtime/src/runtime/vm.rs
@@ -109,9 +109,7 @@ pub use crate::runtime::vm::instance::{
PoolConcurrencyLimitError, PoolingAllocatorMetrics, PoolingInstanceAllocator,
};
pub use crate::runtime::vm::interpreter::*;
-pub use crate::runtime::vm::memory::{
- Memory, MemoryBase, RuntimeLinearMemory, RuntimeMemoryCreator, SharedMemory,
-};
+pub use crate::runtime::vm::memory::{Memory, RuntimeMemoryCreator, SharedMemory};
pub use crate::runtime::vm::mmap_vec::MmapVec;
pub use crate::runtime::vm::provenance::*;
pub use crate::runtime::vm::stack_switching::*;
@@ -174,10 +172,12 @@ cfg_select! {
has_virtual_memory => {
pub use crate::runtime::vm::byte_count::*;
pub use crate::runtime::vm::mmap::{Mmap, MmapOffset};
- pub use self::cow::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
+ pub use self::cow::{MemoryImage, ModuleMemoryImages};
+ #[cfg(feature = "pooling-allocator")]
+ pub use self::cow::MemoryImageSlot;
}
_ => {
- pub use self::cow_disabled::{MemoryImage, MemoryImageSlot, ModuleMemoryImages};
+ pub use self::cow_disabled::{MemoryImage, ModuleMemoryImages};
}
}
diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rs
index 398d86f935..ac7d4dd9fc 100644
--- a/crates/wasmtime/src/runtime/vm/cow.rs
+++ b/crates/wasmtime/src/runtime/vm/cow.rs
@@ -146,7 +146,51 @@ impl MemoryImage {
Ok(None)
}
- unsafe fn map_at(&self, mmap_base: &MmapOffset) -> Result<()> {
+ /// Offset, in bytes, within linear memory at which this image starts.
+ pub(crate) fn linear_memory_offset(&self) -> usize {
+ self.linear_memory_offset.byte_count()
+ }
+
+ /// Length, in bytes, of this image.
+ pub(crate) fn len(&self) -> usize {
+ self.len.byte_count()
+ }
+
+ /// The bytes of this image, as stored in the module's data.
+ /// The length is exactly [`self.len`](Self::len).
+ pub(crate) fn data(&self) -> &[u8] {
+ &self.module_source.wasm_data()[self.module_source_offset..][..self.len.byte_count()]
+ }
+
+ /// Maps this image at `base + self.linear_memory_offset`.
+ ///
+ /// # Safety
+ ///
+ /// See [`crate::MemoryImage::map_at`].
+ pub(crate) unsafe fn map_at_ptr(&self, base: *mut u8) -> Result<()> {
+ if base.addr() % host_page_size() != 0 {
+ bail!("cannot map a memory image at {base:p}: not host-page-aligned");
+ }
+ if self.len.is_zero() {
+ return Ok(());
+ }
+ unsafe {
+ self.source.map_at(
+ base.add(self.linear_memory_offset.byte_count()),
+ self.source_offset,
+ self.len.byte_count(),
+ )?;
+ }
+ Ok(())
+ }
+
+ /// Maps this image into `mmap_base + self.linear_memory_offset`.
+ ///
+ /// # Safety
+ ///
+ /// The caller must ensure that no one else has a reference to the range
+ /// of `mmap_base` being mapped over.
+ pub(crate) unsafe fn map_at(&self, mmap_base: &MmapOffset) -> Result<()> {
unsafe {
mmap_base.map_image_at(
&self.source,
@@ -302,6 +346,10 @@ impl ModuleMemoryImages {
/// original mappings, effectively resetting everything back to its initial
/// state. Non-linux platforms will replace all memory below `self.accessible`
/// with a fresh zero'd mmap, meaning that reuse is effectively not supported.
+#[cfg_attr(
+ not(feature = "pooling-allocator"),
+ expect(dead_code, reason = "only used by the pooling allocator")
+)]
pub struct MemoryImageSlot {
/// The mmap and offset within it that contains the linear memory for this
/// slot.
@@ -359,6 +407,10 @@ impl fmt::Debug for MemoryImageSlot {
}
}
+#[cfg_attr(
+ not(feature = "pooling-allocator"),
+ expect(dead_code, reason = "only used by the pooling allocator")
+)]
impl MemoryImageSlot {
/// Create a new MemoryImageSlot. Assumes that there is an anonymous
/// mmap backing in the given range to start.
@@ -540,7 +592,11 @@ impl MemoryImageSlot {
///
/// Returns the number of bytes still resident in memory after this function
/// has returned.
- #[allow(dead_code, reason = "only used in some cfgs")]
+
+ #[cfg_attr(
+ not(feature = "pooling-allocator"),
+ expect(dead_code, reason = "only used by the pooling allocator")
+ )]
pub(crate) fn clear_and_remain_ready(
&mut self,
pagemap: Option<&PageMap>,
@@ -556,7 +612,10 @@ impl MemoryImageSlot {
Ok(bytes_resident)
}
- #[allow(dead_code, reason = "only used in some cfgs")]
+ #[cfg_attr(
+ not(feature = "pooling-allocator"),
+ expect(dead_code, reason = "only used by the pooling allocator")
+ )]
unsafe fn reset_all_memory_contents(
&mut self,
pagemap: Option<&PageMap>,
@@ -582,7 +641,10 @@ impl MemoryImageSlot {
}
}
- #[allow(dead_code, reason = "only used in some cfgs")]
+ #[cfg_attr(
+ not(feature = "pooling-allocator"),
+ expect(dead_code, reason = "only used by the pooling allocator")
+ )]
unsafe fn reset_with_original_mapping(
&mut self,
pagemap: Option<&PageMap>,
@@ -742,7 +804,10 @@ impl MemoryImageSlot {
self.image.is_some()
}
- #[allow(dead_code, reason = "only used in some cfgs")]
+ #[cfg_attr(
+ not(feature = "pooling-allocator"),
+ expect(dead_code, reason = "only used by the pooling allocator")
+ )]
pub(crate) fn is_dirty(&self) -> bool {
self.dirty
}
@@ -772,7 +837,7 @@ impl MemoryImageSlot {
}
}
-#[cfg(all(test, target_os = "linux", not(miri)))]
+#[cfg(all(test, target_os = "linux", not(miri), feature = "pooling-allocator"))]
mod test {
use super::*;
use crate::runtime::vm::mmap::{AlignedLength, Mmap};
diff --git a/crates/wasmtime/src/runtime/vm/cow_disabled.rs b/crates/wasmtime/src/runtime/vm/cow_disabled.rs
index 90a92e63f8..9595c0d455 100644
--- a/crates/wasmtime/src/runtime/vm/cow_disabled.rs
+++ b/crates/wasmtime/src/runtime/vm/cow_disabled.rs
@@ -23,25 +23,30 @@ impl ModuleMemoryImages {
#[derive(Debug, PartialEq)]
pub enum MemoryImage {}
-impl ModuleMemoryImages {
- pub fn new(
- _engine: &Engine,
- _module: &Module,
- _source: &Arc<impl ModuleMemoryImageSource>,
- ) -> Result<Option<ModuleMemoryImages>> {
- Ok(None)
+impl MemoryImage {
+ pub(crate) fn linear_memory_offset(&self) -> usize {
+ match *self {}
}
-}
-#[derive(Debug)]
-pub enum MemoryImageSlot {}
+ pub(crate) fn len(&self) -> usize {
+ match *self {}
+ }
-impl MemoryImageSlot {
- pub(crate) fn set_heap_limit(&mut self, _size_bytes: usize) -> Result<()> {
+ pub(crate) fn data(&self) -> &[u8] {
match *self {}
}
- pub(crate) fn has_image(&self) -> bool {
+ pub(crate) unsafe fn map_at_ptr(&self, _base: *mut u8) -> Result<()> {
match *self {}
}
}
+
+impl ModuleMemoryImages {
+ pub fn new(
+ _engine: &Engine,
+ _module: &Module,
+ _source: &Arc<impl ModuleMemoryImageSource>,
+ ) -> Result<Option<ModuleMemoryImages>> {
+ Ok(None)
+ }
+}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
index 9d69157901..41e67b6beb 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
@@ -244,7 +244,7 @@ struct CopyingHeap {
memory: Option<crate::vm::Memory>,
/// The cached `VMMemoryDefinition` for `self.memory` so that we don't have
- /// to make indirect calls through a `dyn RuntimeLinearMemory` object.
+ /// to make indirect calls through a `dyn LinearMemory` object.
///
/// Must be updated and kept in sync with `self.memory`, cleared when the
/// memory is taken and updated when the memory is replaced.
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index 0d2f7428aa..e841fe93bf 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -172,7 +172,7 @@ struct DrcHeap {
memory: Option<crate::vm::Memory>,
/// The cached `VMMemoryDefinition` for `self.memory` so that we don't have
- /// to make indirect calls through a `dyn RuntimeLinearMemory` object.
+ /// to make indirect calls through a `dyn LinearMemory` object.
///
/// Must be updated and kept in sync with `self.memory`, cleared when the
/// memory is taken and updated when the memory is replaced.
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
index ca1425c63d..f18ae7f13b 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
@@ -58,8 +58,8 @@ use crate::config::InstanceLimits;
use crate::config::PoolingAllocationConfig;
use crate::prelude::*;
use crate::runtime::vm::{
- CompiledModuleId, InstanceAllocationRequest, Memory, MemoryBase, MemoryImageSlot, Mmap,
- MmapOffset, mmap::AlignedLength,
+ CompiledModuleId, InstanceAllocationRequest, Memory, MemoryImageSlot, Mmap, MmapOffset,
+ mmap::AlignedLength,
};
use crate::vm::sys::vm::{decommit_pages, iovec};
use crate::{
@@ -432,7 +432,7 @@ impl MemoryPool {
let memory = Memory::new_static(
ty,
&memory_tunables,
- MemoryBase::Mmap(base),
+ base,
base_capacity.byte_count(),
slot,
request.limiter.as_deref_mut(),
diff --git a/crates/wasmtime/src/runtime/vm/memory.rs b/crates/wasmtime/src/runtime/vm/memory.rs
index ef68214ed7..aa02d508c8 100644
--- a/crates/wasmtime/src/runtime/vm/memory.rs
+++ b/crates/wasmtime/src/runtime/vm/memory.rs
@@ -24,7 +24,7 @@
//! ▼
//! ┌─────────────────────┐
//! │ │
-//! │ RuntimeLinearMemory ├─────────────┬───────────────┐
+//! │ LinearMemory ├─────────────┬───────────────┐
//! │ │ │ │
//! └──────────┬──────────┘ │ │
//! │ │ │
@@ -52,35 +52,34 @@
//! instance-local state of `Memory`. One example is that `LocalMemory::grow`
//! has most of the logic around memory growth.
//!
-//! * `RuntimeLinearMemory` - this is a trait which `LocalMemory` delegates to.
-//! This trait is intentionally relatively simple to be exposed in Wasmtime's
-//! embedder API. This is exposed all the way through `wasmtime::Config` so
-//! embedders can provide arbitrary implementations.
+//! * `LinearMemory` - this is the public trait which
+//! `LocalMemory` delegates to. This is exposed all the
+//! way through `wasmtime::Config` so embedders can provide arbitrary
+//! implementations, and Wasmtime's own memories implement it as well.
//!
-//! * `MmapMemory` - this is an implementation of `RuntimeLinearMemory` in terms
-//! of the platform's mmap primitive.
+//! * `MmapMemory` - this is an implementation of `LinearMemory` in terms of
+//! the platform's mmap primitive.
//!
-//! * `StaticMemory` - this is an implementation of `RuntimeLinearMemory`
-//! for the pooling allocator where the base pointer is already allocated
-//! and contents are managed through `MemoryImageSlot`.
+//! * `StaticMemory` - this is an implementation of `LinearMemory` for the
+//! pooling allocator where the base pointer is already allocated and
+//! contents are managed through `MemoryImageSlot`.
//!
//! Other important types for memories are `MemoryImage` and `MemoryImageSlot`
-//! which manage CoW state for memories. This is implemented at the
-//! `LocalMemory` layer.
-//!
-//! FIXME: don't have both RuntimeLinearMemory and wasmtime::LinearMemory, they
-//! should be merged together.
+//! which manage CoW state for memories. Initializing a memory from a
+//! `MemoryImage` is delegated to the `LinearMemory` itself through
+//! `LinearMemory::init_from_image`, and the pooling allocator's
+//! `StaticMemory` owns a `MemoryImageSlot` which is reused across instances.
//!
//! FIXME: don't have both RuntimeMemoryCreator and wasmtime::MemoryCreator,
//! they should be merged together.
-use crate::Engine;
use crate::prelude::*;
use crate::runtime::store::StoreResourceLimiter;
+use crate::runtime::vm::MemoryImage;
use crate::runtime::vm::vmcontext::VMMemoryDefinition;
-#[cfg(has_virtual_memory)]
-use crate::runtime::vm::{HostAlignedByteCount, MmapOffset};
-use crate::runtime::vm::{MemoryImage, MemoryImageSlot, SendSyncPtr};
+#[cfg(feature = "pooling-allocator")]
+use crate::runtime::vm::{MemoryImageSlot, MmapOffset};
+use crate::{Engine, LinearMemory};
use alloc::sync::Arc;
use core::{ops::Range, ptr::NonNull};
use wasmtime_environ::{MemoryKind, MemoryTunables};
@@ -113,28 +112,29 @@ pub use shared_memory_disabled::SharedMemory;
/// A memory allocator
pub trait RuntimeMemoryCreator: Send + Sync {
- /// Create new RuntimeLinearMemory
+ /// Create a new `LinearMemory`.
fn new_memory(
&self,
ty: &wasmtime_environ::Memory,
memory_tunables: &MemoryTunables<'_>,
minimum: usize,
maximum: Option<usize>,
- ) -> Result<Box<dyn RuntimeLinearMemory>>;
+ ) -> Result<Box<dyn LinearMemory>>;
}
/// A default memory allocator used by Wasmtime
pub struct DefaultMemoryCreator;
impl RuntimeMemoryCreator for DefaultMemoryCreator {
- /// Create new MmapMemory
+ /// Create new LinearMemory.
+ /// Chooses [`MmapMemory`] if it is available and tunables require it, falling back to [`MallocMemory`] otherwise.
fn new_memory(
&self,
ty: &wasmtime_environ::Memory,
memory_tunables: &MemoryTunables<'_>,
minimum: usize,
maximum: Option<usize>,
- ) -> Result<Box<dyn RuntimeLinearMemory>> {
+ ) -> Result<Box<dyn LinearMemory>> {
#[cfg(has_virtual_memory)]
if memory_tunables.tunables().signals_based_traps
|| memory_tunables.guard_size() > 0
@@ -143,93 +143,18 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator {
{
return Ok(
try_new::<Box<_>>(MmapMemory::new(ty, memory_tunables, minimum, maximum)?)?
- as Box<dyn RuntimeLinearMemory>,
+ as Box<dyn LinearMemory>,
);
}
let _ = maximum;
Ok(
try_new::<Box<_>>(MallocMemory::new(ty, memory_tunables, minimum)?)?
- as Box<dyn RuntimeLinearMemory>,
+ as Box<dyn LinearMemory>,
)
}
}
-/// A linear memory and its backing storage.
-pub trait RuntimeLinearMemory: Send + Sync {
- /// Returns the number bytes that this linear memory can access.
- fn byte_size(&self) -> usize;
-
- /// Returns the maximal number of bytes the current allocation can access.
- ///
- /// Growth up to this value should not relocate the base pointer.
- fn byte_capacity(&self) -> usize;
-
- /// Grow memory to the specified amount of bytes.
- ///
- /// Returns an error if memory can't be grown by the specified amount
- /// of bytes.
- fn grow_to(&mut self, size: usize) -> Result<()>;
-
- /// Returns a pointer to the base of this linear memory allocation.
- ///
- /// This is either a raw pointer, or a reference to an mmap along with an
- /// offset within it.
- fn base(&self) -> MemoryBase;
-
- /// Get a `VMMemoryDefinition` for this linear memory.
- fn vmmemory(&self) -> VMMemoryDefinition;
-
- /// Internal method for Wasmtime when used in conjunction with CoW images.
- /// This is used to inform the underlying memory that the size of memory has
- /// changed.
- ///
- /// Note that this is hidden and panics by default as embedders using custom
- /// memory without CoW images shouldn't have to worry about this.
- #[doc(hidden)]
- fn set_byte_size(&mut self, len: usize) {
- let _ = len;
- panic!("CoW images used with this memory and it doesn't support it");
- }
-}
-
-/// The base pointer of a memory allocation.
-#[derive(Clone, Debug)]
-pub enum MemoryBase {
- /// A raw pointer into memory.
- ///
- /// This may or may not be host-page-aligned.
- Raw(SendSyncPtr<u8>),
-
- /// An mmap along with an offset into it.
- #[cfg(has_virtual_memory)]
- Mmap(MmapOffset),
-}
-
-impl MemoryBase {
- /// Creates a new `MemoryBase` from a raw pointer.
- ///
- /// The pointer must be non-null, and it must be logically `Send + Sync`.
- pub fn new_raw(ptr: *mut u8) -> Self {
- Self::Raw(NonNull::new(ptr).expect("pointer is non-null").into())
- }
-
- /// Returns the actual memory address in memory that is represented by this
- /// base.
- pub fn as_non_null(&self) -> NonNull<u8> {
- match self {
- Self::Raw(ptr) => ptr.as_non_null(),
- #[cfg(has_virtual_memory)]
- Self::Mmap(mmap_offset) => mmap_offset.as_non_null(),
- }
- }
-
- /// Same as `as_non_null`, but different return type.
- pub fn as_mut_ptr(&self) -> *mut u8 {
- self.as_non_null().as_ptr()
- }
-}
-
/// Representation of a runtime wasm linear memory.
pub enum Memory {
Local(LocalMemory),
@@ -260,26 +185,34 @@ impl Memory {
}
/// Create a new static (immovable) memory instance for the specified plan.
+ ///
+ /// The `memory_image_slot` must have been already instantiated by the pooling allocator.
#[cfg(feature = "pooling-allocator")]
pub async fn new_static(
ty: &wasmtime_environ::Memory,
memory_tunables: &MemoryTunables<'_>,
- base: MemoryBase,
+ base: MmapOffset,
base_capacity: usize,
- memory_image: MemoryImageSlot,
+ memory_image_slot: MemoryImageSlot,
limiter: Option<&mut StoreResourceLimiter<'_>>,
) -> Result<Self> {
let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
- let pooled_memory = StaticMemory::new(base, base_capacity, minimum, maximum)?;
+
+ // Assert the memory image was instantiated.
+ debug_assert!(memory_image_slot.is_dirty());
+ // The pooling allocator has already instantiated the image, so it only needs
+ // initialization if the slot holds no image.
+ let needs_init = !memory_image_slot.has_image();
+ let pooled_memory =
+ StaticMemory::new(base, base_capacity, minimum, maximum, memory_image_slot)?;
let allocation = try_new::<Box<_>>(pooled_memory)?;
// Configure some defaults a bit differently for this memory within the
- // `LocalMemory` structure created, notably we already have
- // `memory_image` and regardless of configuration settings this memory
- // can't move its base pointer since it's a fixed allocation.
+ // `LocalMemory` structure created:
+ // - record whether the data segments still need to be copied in;
+ // - regardless of configuration settings this memory can't move its base pointer since it's a fixed allocation.
let mut memory = LocalMemory::new(ty, memory_tunables, allocation, None)?;
- assert!(memory.memory_image.is_none());
- memory.memory_image = Some(memory_image);
+ memory.needs_init = needs_init;
memory.memory_may_move = false;
Ok(if ty.shared {
@@ -534,61 +467,38 @@ impl Memory {
/// the implementation basis for a `SharedMemory` behind an `RwLock` for
/// example.
pub struct LocalMemory {
- alloc: Box<dyn RuntimeLinearMemory>,
+ alloc: Box<dyn LinearMemory>,
ty: wasmtime_environ::Memory,
+ /// True only if this memory may move its base pointer.
memory_may_move: bool,
memory_guard_size: usize,
memory_reservation: usize,
- /// An optional CoW mapping that provides the initial content of this
- /// memory.
- memory_image: Option<MemoryImageSlot>,
+ /// Whether the data segments of this memory still need to be copied into it.
+ /// The memory might be already initialized from a pooled allocator or by the particular
+ /// allocation's implementation, in which case this is false.
+ needs_init: bool,
}
impl LocalMemory {
pub fn new(
ty: &wasmtime_environ::Memory,
memory_tunables: &MemoryTunables<'_>,
- alloc: Box<dyn RuntimeLinearMemory>,
+ mut alloc: Box<dyn LinearMemory>,
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<LocalMemory> {
- // If a memory image was specified, try to create the MemoryImageSlot on
- // top of our mmap.
- let memory_image = match memory_image {
- #[cfg(has_virtual_memory)]
- Some(image) => {
- // We currently don't support memory_image if
- // `RuntimeLinearMemory::byte_size` is not a multiple of the host page
- // size. See https://github.com/bytecodealliance/wasmtime/issues/9660.
- if let Ok(byte_size) = HostAlignedByteCount::new(alloc.byte_size()) {
- // memory_image is CoW-based so it is expected to be backed
- // by an mmap.
- let mmap_base = match alloc.base() {
- MemoryBase::Mmap(offset) => offset,
- MemoryBase::Raw { .. } => {
- unreachable!("memory_image is Some only for mmap-based memories")
- }
- };
-
- // Memories allocated on demand are never striped with MPK
- // protection keys, so this slot has no key to preserve.
- let mut slot =
- MemoryImageSlot::create(mmap_base, byte_size, alloc.byte_capacity(), None);
- slot.instantiate(alloc.byte_size(), Some(image), ty, memory_tunables)?;
- Some(slot)
- } else {
- None
- }
- }
- #[cfg(not(has_virtual_memory))]
- Some(_) => unreachable!(),
- None => None,
+ // If a memory image was specified, give the allocation a chance to
+ // initialize itself from it. If it declines then the data segments
+ // are copied in as usual.
+ let initialized = match memory_image {
+ Some(image) => alloc.init_from_image(&crate::MemoryImage::new(image.clone()))?,
+ None => false,
};
Ok(LocalMemory {
ty: *ty,
alloc,
memory_may_move: ty.memory_may_move(memory_tunables),
- memory_image,
+ needs_init: !initialized,
memory_guard_size: memory_tunables.guard_size().try_into().unwrap(),
memory_reservation: memory_tunables.reservation().try_into().unwrap(),
})
@@ -660,7 +570,7 @@ impl LocalMemory {
// Save the original base pointer to assert the invariant that growth up
// to the byte capacity never relocates the base pointer.
- let base_ptr_before = self.alloc.base().as_mut_ptr();
+ let base_ptr_before = self.alloc.as_ptr();
let required_to_not_move_memory = new_byte_size <= self.alloc.byte_capacity();
let result = (|| -> Result<()> {
@@ -677,30 +587,6 @@ impl LocalMemory {
bail!("Memory maximum size exceeded");
}
- // If we have a CoW image overlay then let it manage accessible
- // bytes. Once the heap limit is modified inform the underlying
- // allocation that the size has changed.
- //
- // If the growth is going beyond the size of the heap image then
- // discard it. This should only happen for `MmapMemory` where
- // `no_clear_on_drop` is set so the destructor doesn't do anything.
- // For now be maximally sure about this by asserting that memory can
- // indeed move and that we're on unix. If this wants to run
- // somewhere else like Windows or with other allocations this may
- // need adjusting.
- if let Some(image) = &mut self.memory_image {
- if new_byte_size <= self.alloc.byte_capacity() {
- image.set_heap_limit(new_byte_size)?;
- self.alloc.set_byte_size(new_byte_size);
- return Ok(());
- }
- assert!(cfg!(unix));
- assert!(self.memory_may_move);
- self.memory_image = None;
- }
-
- // And failing all that fall back to the underlying allocation to
- // grow it.
self.alloc.grow_to(new_byte_size)
})();
@@ -709,7 +595,11 @@ impl LocalMemory {
// On successful growth double-check that the base pointer
// didn't move if it shouldn't have.
if required_to_not_move_memory {
- assert_eq!(base_ptr_before, self.alloc.base().as_mut_ptr());
+ assert_eq!(
+ base_ptr_before,
+ self.alloc.as_ptr(),
+ "linear memory moved on grow when it must not have"
+ );
}
Ok(Some((old_byte_size, new_byte_size)))
@@ -727,8 +617,14 @@ impl LocalMemory {
}
}
+ /// Return the bounds of the memory (base, byte_size) for consumption by WebAssembly code in a
+ /// repr(C) struct.
pub fn vmmemory(&self) -> VMMemoryDefinition {
- self.alloc.vmmemory()
+ let base = NonNull::new(self.alloc.as_ptr()).expect("linear memory base is non-null");
+ VMMemoryDefinition {
+ base: base.into(),
+ current_length: self.alloc.byte_size().into(),
+ }
}
pub fn byte_size(&self) -> usize {
@@ -736,14 +632,11 @@ impl LocalMemory {
}
pub fn needs_init(&self) -> bool {
- match &self.memory_image {
- Some(image) => !image.has_image(),
- None => true,
- }
+ self.needs_init
}
pub fn wasm_accessible(&self) -> Range<usize> {
- let base = self.alloc.base().as_mut_ptr() as usize;
+ let base = self.alloc.as_ptr() as usize;
// From the base add:
//
// * max(capacity, reservation) -- all memory is guaranteed to have at
@@ -758,9 +651,16 @@ impl LocalMemory {
base..end
}
+ /// Assume the inner allocation is a [`StaticMemory`] created for a pooling allocator and unwrap
+ /// the inner [`MemoryImageSlot`].
+ /// This function panics if this memory was not created from a [`StaticMemory`] allocation.
#[cfg(feature = "pooling-allocator")]
pub fn unwrap_static_image(self) -> MemoryImageSlot {
- self.memory_image.unwrap()
+ let alloc: Box<dyn core::any::Any> = self.alloc;
+ alloc
+ .downcast::<StaticMemory>()
+ .unwrap_or_else(|_| panic!("expected a pooling allocator memory"))
+ .into_image_slot()
}
}
diff --git a/crates/wasmtime/src/runtime/vm/memory/malloc.rs b/crates/wasmtime/src/runtime/vm/memory/malloc.rs
index f07a5995d5..5046b585c2 100644
--- a/crates/wasmtime/src/runtime/vm/memory/malloc.rs
+++ b/crates/wasmtime/src/runtime/vm/memory/malloc.rs
@@ -1,12 +1,12 @@
-//! Support for implementing the [`RuntimeLinearMemory`] trait in terms of a
+//! Support for implementing the [`LinearMemory`] trait in terms of a
//! platform memory allocation primitive (e.g. `malloc`)
//!
//! Note that memory is allocated here using `Vec::try_reserve` to explicitly
//! handle memory allocation failures.
+use crate::LinearMemory;
use crate::prelude::*;
use crate::runtime::vm::SendSyncPtr;
-use crate::runtime::vm::memory::{MemoryBase, RuntimeLinearMemory};
use core::mem;
use core::ptr::NonNull;
use wasmtime_environ::MemoryTunables;
@@ -66,7 +66,16 @@ impl MallocMemory {
}
}
-impl RuntimeLinearMemory for MallocMemory {
+// SAFETY:
+// - **Valid region.** We use a Vec with at least `minimum` bytes, possibly more with 16-alignment.
+// - **Size.** Is exactly `minimum` at creation and is set to exactly `new_size` on growth.
+// - **Base pointer stability.** `grow_to` calls `Vec::try_reserve`, which is guaranteed to not
+// reallocate if the new length does not exceed capacity. The Vec's capacity is what is reported in
+// `byte_capacity`.
+// - **Base pointer alignment.** The pointer is 16-byte aligned. The other conditions are checked
+// to be false within the constructor, so we do not require host page alignment.
+// - **Inaccessible region** moot because the conditions are checked to be false.
+unsafe impl LinearMemory for MallocMemory {
fn byte_size(&self) -> usize {
self.byte_len
}
@@ -89,16 +98,8 @@ impl RuntimeLinearMemory for MallocMemory {
Ok(())
}
- fn base(&self) -> MemoryBase {
- MemoryBase::Raw(self.base_ptr)
- }
-
- fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {
- let base = self.base_ptr.as_non_null();
- crate::vm::VMMemoryDefinition {
- base: base.into(),
- current_length: self.byte_len.into(),
- }
+ fn as_ptr(&self) -> *mut u8 {
+ self.base_ptr.as_ptr()
}
}
diff --git a/crates/wasmtime/src/runtime/vm/memory/mmap.rs b/crates/wasmtime/src/runtime/vm/memory/mmap.rs
index bdd9c3f1ef..2cf6f46009 100644
--- a/crates/wasmtime/src/runtime/vm/memory/mmap.rs
+++ b/crates/wasmtime/src/runtime/vm/memory/mmap.rs
@@ -1,15 +1,23 @@
-//! Support for implementing the [`RuntimeLinearMemory`] trait in terms of a
+//! Support for implementing the [`LinearMemory`] trait in terms of a
//! platform mmap primitive.
use crate::prelude::*;
-use crate::runtime::vm::memory::RuntimeLinearMemory;
use crate::runtime::vm::{HostAlignedByteCount, Mmap, mmap::AlignedLength};
+use crate::{LinearMemory, MemoryImage};
use alloc::sync::Arc;
use wasmtime_environ::MemoryTunables;
-use super::MemoryBase;
-
/// A linear memory instance.
+///
+/// The layout of a single Mmap is:
+///
+/// ```text
+/// [ pre_guard ][ accessible = len rounded up | PROT_NONE up to capacity ][ offset_guard ]
+/// ^ as_ptr()
+/// |<------------------ current_capacity() --------------->|
+/// ```
+///
+/// The amount of reserved memory depends on tunables (see the [`new`](MmapMemory::new) function).
#[derive(Debug)]
pub struct MmapMemory {
// The underlying allocation.
@@ -144,7 +152,17 @@ impl MmapMemory {
}
}
-impl RuntimeLinearMemory for MmapMemory {
+/// SAFETY:
+///
+/// - **Valid region.** The memory map is always kept valid for self.len bytes.
+/// - **Size.** [`MmapMemory::new`] maps at least `minimum` bytes.
+/// - **Base pointer stability.** `grow_to` does not modify the mapping while the request is within
+/// capacity.
+/// - **Base pointer alignment.** The mmap call always returns a page-aligned pointer, and guard
+/// bytes are ensured to be a multiple of page-size.
+/// - **Inaccessible region.** Inaccessible memory is protected (implementation delegated to the
+/// `sys` submodule).
+unsafe impl LinearMemory for MmapMemory {
fn byte_size(&self) -> usize {
self.len
}
@@ -221,31 +239,32 @@ impl RuntimeLinearMemory for MmapMemory {
Ok(())
}
- fn set_byte_size(&mut self, len: usize) {
- self.len = len;
- }
-
- fn base(&self) -> MemoryBase {
- MemoryBase::Mmap(
+ fn as_ptr(&self) -> *mut u8 {
+ // SAFETY: `pre_guard_size` is within the mmap allocation.
+ unsafe {
self.mmap
- .offset(self.pre_guard_size)
- .expect("pre_guard_size is in bounds"),
- )
+ .as_non_null()
+ .byte_add(self.pre_guard_size.byte_count())
+ .as_ptr()
+ }
}
- fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {
- let pre_guard_size = self.pre_guard_size.byte_count();
- assert!(pre_guard_size <= self.mmap.len());
- let pre_guard_size = isize::try_from(pre_guard_size).unwrap();
- let mmap = self.mmap.as_non_null();
- let base = unsafe {
- // Safety: `pre_guard_size` is within the mmap allocation.
- mmap.offset(pre_guard_size)
- };
-
- crate::vm::VMMemoryDefinition {
- base: base.into(),
- current_length: self.len.into(),
+ fn init_from_image(&mut self, image: &MemoryImage) -> Result<bool> {
+ // Images always fit within the minimum size of a memory, which is
+ // accessible from creation. If this ever changes our assumptions are incorrect for CoW, so
+ // explode loudly.
+ let image_end = image.linear_memory_offset().checked_add(image.len());
+ assert!(
+ image_end.is_some_and(|end| end <= self.len),
+ "MemoryImage does not fit into the MmapMemory"
+ );
+ let base = self.mmap.offset(self.pre_guard_size)?;
+ // SAFETY: this memory was just created and no one else has a reference
+ // to it yet. The image's range lies within this memory's accessible
+ // region of the mmap, which this memory exclusively owns.
+ unsafe {
+ image.vm().map_at(&base)?;
}
+ Ok(true)
}
}
diff --git a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs
index 9f0721be11..4a93bb197c 100644
--- a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs
+++ b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs
@@ -41,8 +41,7 @@ impl SharedMemory {
// `assert_ready` should never panic.
let (minimum_bytes, maximum_bytes) = vm::assert_ready(Memory::limit_new(ty, None))?;
let mmap_memory = MmapMemory::new(ty, &memory_tunables, minimum_bytes, maximum_bytes)?;
- let boxed: Box<dyn crate::runtime::vm::RuntimeLinearMemory> =
- try_new::<Box<_>>(mmap_memory)?;
+ let boxed: Box<dyn crate::LinearMemory> = try_new::<Box<_>>(mmap_memory)?;
Self::wrap(
engine,
ty,
@@ -87,7 +86,7 @@ impl SharedMemory {
NonNull::from(&self.0.def.0)
}
- /// Same as `RuntimeLinearMemory::grow`, except with `&self`.
+ /// Same as `LocalMemory::grow`, except with `&self`.
pub fn grow(&self, delta_pages: u64) -> Result<Option<(usize, usize)>, Error> {
let mut memory = self.0.memory.write().unwrap();
// Without a limiter being passed in this shouldn't have an await point,
diff --git a/crates/wasmtime/src/runtime/vm/memory/static_.rs b/crates/wasmtime/src/runtime/vm/memory/static_.rs
index 1646eb4a9d..602b3e3352 100644
--- a/crates/wasmtime/src/runtime/vm/memory/static_.rs
+++ b/crates/wasmtime/src/runtime/vm/memory/static_.rs
@@ -1,30 +1,45 @@
-//! Support for implementing the [`RuntimeLinearMemory`] trait in terms of a
+//! Support for implementing the [`LinearMemory`] trait in terms of a
//! fixed allocation that cannot move.
+use crate::LinearMemory;
use crate::prelude::*;
-use crate::runtime::vm::MemoryBase;
-use crate::runtime::vm::memory::RuntimeLinearMemory;
+use crate::runtime::vm::{MemoryImageSlot, MmapOffset, SendSyncPtr};
/// A "static" memory where the lifetime of the backing memory is managed
/// elsewhere. Currently used with the pooling allocator.
pub struct StaticMemory {
/// The base pointer of this static memory, wrapped up in a send/sync
/// wrapper.
- base: MemoryBase,
+ base: SendSyncPtr<u8>,
/// The byte capacity of the `base` pointer.
capacity: usize,
/// The current size, in bytes, of this memory.
size: usize,
+
+ /// The slot of the pooling allocator in which this memory lives.
+ ///
+ /// The slot has already been instantiated for this memory, including
+ /// mapping its CoW image, if any. It manages which bytes of the slot are
+ /// accessible as this memory grows, and is handed back to the pooling
+ /// allocator with [`StaticMemory::into_image_slot`] when this memory is
+ /// deallocated.
+ image_slot: MemoryImageSlot,
}
impl StaticMemory {
+ /// Create a new static memory at the given memory map location.
+ /// The `image_slot` must be already instantiated *for this memory*.
+ /// So the `slot.base` is the same location as `base`, `slot.static_size` >= `base_capacity`,
+ /// `initial_size` is accessible in the `slot`, and when relying on guards no more than
+ /// `initial_size` rounded to full host pages may be accessible.
pub fn new(
- base: MemoryBase,
+ base: MmapOffset,
base_capacity: usize,
initial_size: usize,
maximum_size: Option<usize>,
+ image_slot: MemoryImageSlot,
) -> Result<Self> {
if base_capacity < initial_size {
bail!(
@@ -40,14 +55,40 @@ impl StaticMemory {
};
Ok(Self {
- base,
+ base: base.as_non_null().into(),
capacity: base_capacity,
size: initial_size,
+ image_slot,
})
}
+
+ /// Returns the slot state of this memory back to the pooling allocator.
+ pub fn into_image_slot(self) -> MemoryImageSlot {
+ self.image_slot
+ }
}
-impl RuntimeLinearMemory for StaticMemory {
+// SAFETY:
+// Note that these points, in particular alignment, rely on the behavior of the pooling allocator.
+//
+// - **Valid region.** `base` points into the pooling allocator's mapping at the slot owned
+// exclusively by this memory (the pool's index allocator hands each slot to one memory at a time).
+// `image_slot` manages that same region and keeps the first `size` bytes, rounded up to the host
+// page size, accessible. It also holds a reference to the pool's mapping, which keeps
+// `base` valid for this memory's lifetime.
+//
+// - **Size.** Is exactly `initial_size` at creation and is set to exactly `new_byte_size` on growth.
+//
+// - **Base pointer stability.** `base` never changes. Growth beyond `capacity` is rejected by the
+// assertion in `grow_to`, and `LocalMemory` never requests it since `Memory::new_static` forces
+// `memory_may_move = false`.
+//
+// - **Base pointer alignment.** The slot's base is the pool's mapping plus the pre-slab guard plus
+// a multiple of the slot size, all of which are multiples of the host page size, so `base` is
+// host-page-aligned.
+//
+// - **Inaccessible region.** This relies on the guard mechanism outlined in the `MemoryPool` documentation.
+unsafe impl LinearMemory for StaticMemory {
fn byte_size(&self) -> usize {
self.size
}
@@ -61,23 +102,14 @@ impl RuntimeLinearMemory for StaticMemory {
// prior to arriving here.
assert!(new_byte_size <= self.capacity);
- // Update our accounting of the available size.
+ // Let the slot make the new bytes accessible, then update our
+ // accounting of the available size.
+ self.image_slot.set_heap_limit(new_byte_size)?;
self.size = new_byte_size;
Ok(())
}
- fn set_byte_size(&mut self, len: usize) {
- self.size = len;
- }
-
- fn base(&self) -> MemoryBase {
- self.base.clone()
- }
-
- fn vmmemory(&self) -> crate::vm::VMMemoryDefinition {
- crate::vm::VMMemoryDefinition {
- base: self.base.as_non_null().into(),
- current_length: self.size.into(),
- }
+ fn as_ptr(&self) -> *mut u8 {
+ self.base.as_ptr()
}
}
diff --git a/crates/wasmtime/src/runtime/vm/mmap.rs b/crates/wasmtime/src/runtime/vm/mmap.rs
index e63e037498..144c50d8a0 100644
--- a/crates/wasmtime/src/runtime/vm/mmap.rs
+++ b/crates/wasmtime/src/runtime/vm/mmap.rs
@@ -454,11 +454,21 @@ impl MmapOffset {
.offset
.checked_add(memory_offset)
.expect("self.offset + memory_offset is in bounds");
+ if !total_offset
+ .checked_add(memory_len)
+ .is_ok_and(|end| end <= self.mmap.len_aligned())
+ {
+ bail!("image mapping must be in bounds of the mmap")
+ }
unsafe {
- self.mmap
- .sys
- .map_image_at(image_source, source_offset, total_offset, memory_len)
+ let base = self
+ .mmap
+ .as_non_null()
+ .byte_add(total_offset.byte_count())
+ .as_ptr();
+ image_source.map_at(base, source_offset, memory_len.byte_count())?;
}
+ Ok(())
}
}
@@ -490,4 +500,68 @@ mod tests {
.expect("make_readonly succeeded");
};
}
+
+ /// Creates a two-page inaccessible mmap and a one-page image source whose
+ /// bytes are all `0x67`.
+ #[cfg(all(target_os = "linux", not(miri)))]
+ fn mmap_and_image_source() -> (Arc<Mmap<AlignedLength>>, MemoryImageSource) {
+ let page_size = HostAlignedByteCount::host_page_size();
+ let pagex2 = page_size.checked_mul(2).unwrap();
+ let mmap = Mmap::accessible_reserved(HostAlignedByteCount::ZERO, pagex2)
+ .expect("allocated memory");
+ let data = vec![0x67; page_size.byte_count()];
+ let source = MemoryImageSource::from_data(&data)
+ .expect("created image source")
+ .expect("memfd is supported");
+ (Arc::new(mmap), source)
+ }
+
+ #[test]
+ #[cfg(all(target_os = "linux", not(miri)))]
+ fn map_image_at_up_to_end_of_mmap() {
+ let page_size = HostAlignedByteCount::host_page_size();
+ let (mmap, source) = mmap_and_image_source();
+ let base = mmap.offset(page_size).unwrap();
+ // SAFETY: the mmap was just created and nothing references it.
+ unsafe {
+ base.map_image_at(&source, 0, HostAlignedByteCount::ZERO, page_size)
+ .expect("mapped image");
+ let mapped = core::slice::from_raw_parts(base.as_mut_ptr(), page_size.byte_count());
+ assert!(mapped.iter().all(|b| *b == 0x67));
+ }
+ }
+
+ #[test]
+ #[cfg(all(target_os = "linux", not(miri)))]
+ fn map_image_at_past_end_of_mmap() {
+ let page_size = HostAlignedByteCount::host_page_size();
+ let (mmap, source) = mmap_and_image_source();
+ let base = mmap.offset(page_size).unwrap();
+ // End the mapping at the highest page-aligned address: `offset + len`
+ // doesn't overflow, but `len` exceeds any address space, so should the
+ // assertion ever regress the kernel rejects the `mmap` instead of
+ // mapping over memory that this test doesn't own.
+ let max_aligned = usize::MAX - usize::MAX % page_size.byte_count();
+ let max = HostAlignedByteCount::new(max_aligned).unwrap();
+ let huge = max.checked_sub(page_size).unwrap();
+ // SAFETY: the mmap was just created and nothing references it. The
+ // call is expected to return Err before mapping anything.
+ let result = unsafe { base.map_image_at(&source, 0, HostAlignedByteCount::ZERO, huge) };
+ assert!(result.is_err());
+ }
+
+ /// An offset and length whose sum overflows must be rejected as well.
+ #[test]
+ #[cfg(all(target_os = "linux", not(miri)))]
+ fn map_image_at_overflowing_length() {
+ let page_size = HostAlignedByteCount::host_page_size();
+ let (mmap, source) = mmap_and_image_source();
+ let base = mmap.offset(page_size).unwrap();
+ let max_aligned = usize::MAX - usize::MAX % page_size.byte_count();
+ let max = HostAlignedByteCount::new(max_aligned).unwrap();
+ // SAFETY: the mmap was just created and nothing references it. The
+ // call is expected to panic before mapping anything.
+ let result = unsafe { base.map_image_at(&source, 0, HostAlignedByteCount::ZERO, max) };
+ assert!(result.is_err());
+ }
}
diff --git a/crates/wasmtime/src/runtime/vm/sys/custom/mmap.rs b/crates/wasmtime/src/runtime/vm/sys/custom/mmap.rs
index bc2ff78d8b..30a9242b5e 100644
--- a/crates/wasmtime/src/runtime/vm/sys/custom/mmap.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/custom/mmap.rs
@@ -1,6 +1,6 @@
use super::cvt;
use crate::prelude::*;
-use crate::runtime::vm::sys::{capi, vm::MemoryImageSource};
+use crate::runtime::vm::sys::capi;
use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};
use core::ops::Range;
use core::ptr::{self, NonNull};
@@ -122,28 +122,6 @@ impl Mmap {
}
Ok(())
}
-
- pub unsafe fn map_image_at(
- &self,
- image_source: &MemoryImageSource,
- source_offset: u64,
- memory_offset: HostAlignedByteCount,
- memory_len: HostAlignedByteCount,
- ) -> Result<()> {
- assert_eq!(source_offset, 0);
- unsafe {
- let base = self
- .memory
- .as_ptr()
- .byte_add(memory_offset.byte_count())
- .cast();
- cvt(capi::wasmtime_memory_image_map_at(
- image_source.image_ptr().as_ptr(),
- base,
- memory_len.byte_count(),
- ))
- }
- }
}
impl Drop for Mmap {
diff --git a/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs b/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs
index 60321f5674..7febc5153e 100644
--- a/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs
@@ -95,9 +95,16 @@ impl MemoryImageSource {
}
}
- #[inline]
- pub(super) fn image_ptr(&self) -> SendSyncPtr<capi::wasmtime_memory_image> {
- self.data
+ /// See the documentation of the unix implementation of this method.
+ pub unsafe fn map_at(&self, base: *mut u8, source_offset: u64, len: usize) -> Result<()> {
+ assert_eq!(source_offset, 0);
+ unsafe {
+ cvt(capi::wasmtime_memory_image_map_at(
+ self.data.as_ptr(),
+ base.cast(),
+ len,
+ ))
+ }
}
pub unsafe fn remap_as_zeros_at(&self, base: *mut u8, len: usize) -> Result<()> {
diff --git a/crates/wasmtime/src/runtime/vm/sys/miri/mmap.rs b/crates/wasmtime/src/runtime/vm/sys/miri/mmap.rs
index 1afbc44f33..e33f236760 100644
--- a/crates/wasmtime/src/runtime/vm/sys/miri/mmap.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/miri/mmap.rs
@@ -6,7 +6,6 @@
//! but it's enough to get various tests running relying on memories and such.
use crate::prelude::*;
-use crate::runtime::vm::sys::vm::MemoryImageSource;
use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};
use std::alloc::{self, Layout};
use std::fs::File;
@@ -118,16 +117,6 @@ impl Mmap {
pub unsafe fn make_readwrite(&self, _range: Range<usize>) -> Result<()> {
Ok(())
}
-
- pub unsafe fn map_image_at(
- &self,
- image_source: &MemoryImageSource,
- _source_offset: u64,
- _memory_offset: HostAlignedByteCount,
- _memory_len: HostAlignedByteCount,
- ) -> Result<()> {
- match *image_source {}
- }
}
impl Drop for Mmap {
diff --git a/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs b/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs
index 00446aedf4..487a4cc3d7 100644
--- a/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs
@@ -69,6 +69,15 @@ impl MemoryImageSource {
Ok(None)
}
+ pub unsafe fn map_at(
+ &self,
+ _base: *mut u8,
+ _source_offset: u64,
+ _len: usize,
+ ) -> io::Result<()> {
+ match *self {}
+ }
+
pub unsafe fn remap_as_zeros_at(&self, _base: *mut u8, _len: usize) -> io::Result<()> {
match *self {}
}
diff --git a/crates/wasmtime/src/runtime/vm/sys/unix/mmap.rs b/crates/wasmtime/src/runtime/vm/sys/unix/mmap.rs
index 8d89bbb125..5b2301c760 100644
--- a/crates/wasmtime/src/runtime/vm/sys/unix/mmap.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/unix/mmap.rs
@@ -1,5 +1,4 @@
use crate::prelude::*;
-use crate::runtime::vm::sys::vm::MemoryImageSource;
use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};
use rustix::mm::{MprotectFlags, mprotect};
use std::ops::Range;
@@ -212,28 +211,6 @@ impl Mmap {
Ok(())
}
-
- pub unsafe fn map_image_at(
- &self,
- image_source: &MemoryImageSource,
- source_offset: u64,
- memory_offset: HostAlignedByteCount,
- memory_len: HostAlignedByteCount,
- ) -> Result<()> {
- unsafe {
- let map_base = self.memory.as_ptr().byte_add(memory_offset.byte_count());
- let ptr = rustix::mm::mmap(
- map_base.cast(),
- memory_len.byte_count(),
- rustix::mm::ProtFlags::READ | rustix::mm::ProtFlags::WRITE,
- rustix::mm::MapFlags::PRIVATE | rustix::mm::MapFlags::FIXED,
- image_source.as_file(),
- source_offset,
- )?;
- assert_eq!(map_base.cast(), ptr);
- };
- Ok(())
- }
}
impl Drop for Mmap {
diff --git a/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs b/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs
index 79ab69293c..b7db22041f 100644
--- a/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs
@@ -1,6 +1,6 @@
use crate::runtime::vm::sys::DecommitBehavior;
use rustix::fd::AsRawFd;
-use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect};
+use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap, mmap_anonymous, mprotect};
use std::fs::File;
use std::io;
#[cfg(feature = "std")]
@@ -172,7 +172,7 @@ impl MemoryImageSource {
Ok(Some(MemoryImageSource::Memfd(memfd)))
}
- pub(super) fn as_file(&self) -> &File {
+ fn as_file(&self) -> &File {
match *self {
#[cfg(feature = "std")]
MemoryImageSource::Mmap(ref file) => file,
@@ -181,6 +181,30 @@ impl MemoryImageSource {
}
}
+ /// Maps `len` bytes of this image, starting at `source_offset` within the
+ /// source, copy-on-write and read/write at `base`, replacing whatever was
+ /// mapped there before.
+ ///
+ /// # Safety
+ ///
+ /// `base` must be host-page-aligned and `base..base + len` must be a
+ /// region of address space that the caller exclusively owns and that may
+ /// have its pages replaced.
+ pub unsafe fn map_at(&self, base: *mut u8, source_offset: u64, len: usize) -> io::Result<()> {
+ let ptr = unsafe {
+ mmap(
+ base.cast(),
+ len,
+ ProtFlags::READ | ProtFlags::WRITE,
+ MapFlags::PRIVATE | MapFlags::FIXED,
+ self.as_file(),
+ source_offset,
+ )?
+ };
+ assert_eq!(base, ptr.cast());
+ Ok(())
+ }
+
pub unsafe fn remap_as_zeros_at(&self, base: *mut u8, len: usize) -> io::Result<()> {
let ptr = unsafe {
mmap_anonymous(
diff --git a/crates/wasmtime/src/runtime/vm/sys/windows/mmap.rs b/crates/wasmtime/src/runtime/vm/sys/windows/mmap.rs
index 137887600a..792946a292 100644
--- a/crates/wasmtime/src/runtime/vm/sys/windows/mmap.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/windows/mmap.rs
@@ -1,5 +1,4 @@
use crate::prelude::*;
-use crate::runtime::vm::sys::vm::MemoryImageSource;
use crate::runtime::vm::{HostAlignedByteCount, SendSyncPtr};
use std::fs::{File, OpenOptions};
use std::io;
@@ -259,16 +258,6 @@ impl Mmap {
}
Ok(())
}
-
- pub unsafe fn map_image_at(
- &self,
- image_source: &MemoryImageSource,
- _source_offset: u64,
- _memory_offset: HostAlignedByteCount,
- _memory_len: HostAlignedByteCount,
- ) -> Result<()> {
- match *image_source {}
- }
}
impl Drop for Mmap {
diff --git a/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs b/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs
index cf767135fa..d3bdce281a 100644
--- a/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs
@@ -80,6 +80,15 @@ impl MemoryImageSource {
Ok(None)
}
+ pub unsafe fn map_at(
+ &self,
+ _base: *mut u8,
+ _source_offset: u64,
+ _len: usize,
+ ) -> io::Result<()> {
+ match *self {}
+ }
+
pub unsafe fn remap_as_zeros_at(&self, _base: *mut u8, _len: usize) -> io::Result<()> {
match *self {}
}
diff --git a/tests/all/memory_creator.rs b/tests/all/memory_creator.rs
index 8d7a1fc8fe..340894cf85 100644
--- a/tests/all/memory_creator.rs
+++ b/tests/all/memory_creator.rs
@@ -1,193 +1,545 @@
-#[cfg(all(not(target_os = "windows"), not(miri)))]
-mod not_for_windows {
- use wasmtime::*;
+//! Tests of custom linear memories provided with `Config::with_host_memory`.
+// None of this works on miri since we use actual system mmap utilities.
+#![cfg(not(miri))]
- use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect, munmap};
+use std::ptr;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicUsize, Ordering::SeqCst};
+use wasmtime::*;
- use std::ptr::null_mut;
- use std::sync::{Arc, Mutex};
+const WASM_PAGE: usize = 0x10000;
+const GUARD_SIZE: usize = WASM_PAGE;
+const MAP_CAPACITY: usize = 10 << 20; // 10MiB as max cap for mmap.
- struct CustomMemory {
- mem: usize,
- size: usize,
- guard_size: usize,
- used_wasm_bytes: usize,
- glob_bytes_counter: Arc<Mutex<usize>>,
+/// What `CustomMemory` does when asked to initialize itself from a CoW memory
+/// image.
+#[derive(Clone, Copy, PartialEq)]
+#[cfg_attr(
+ not(target_os = "linux"),
+ allow(dead_code, reason = "CoW images are only built on Linux")
+)]
+enum ImageMode {
+ /// Map the image with `MemoryImage::map_at`, return true.
+ Map,
+ /// Do not map, return false, let Wasmtime copy the data.
+ ReturnFalse,
+ /// Return true *without* initializing the memory.
+ /// This is for tests to check that wasmtime does not manually copy anything.
+ Lie,
+ /// Fail initialization.
+ Fail,
+}
+
+/// Observability of calls for tests.
+#[derive(Default)]
+struct Counters {
+ num_memories_created: AtomicUsize,
+ num_bytes_accessible: AtomicUsize,
+ grow_to_calls: AtomicUsize,
+ init_from_image_calls: AtomicUsize,
+ images_mapped: AtomicUsize,
+}
+
+struct SendSyncPtr(*mut u8);
+unsafe impl Send for SendSyncPtr {}
+unsafe impl Sync for SendSyncPtr {}
+
+/// A linear memory backed by an mmap.
+/// `size` bytes are accessible, followed by inaccessible space up to `capacity` and a
+/// guard region of `GUARD_SIZE` bytes.
+struct CustomMemory {
+ base: SendSyncPtr,
+ capacity: usize,
+ size: usize,
+ /// Whether every `grow_to` moves the memory to a new reservation.
+ /// When false, growing beyond capacity fails (static memory).
+ move_on_grow: bool,
+ image_mode: ImageMode,
+ counters: Arc<Counters>,
+}
+
+impl CustomMemory {
+ fn new(
+ minimum: usize,
+ capacity: usize,
+ move_on_grow: bool,
+ image_mode: ImageMode,
+ counters: Arc<Counters>,
+ ) -> Self {
+ let capacity = capacity.max(minimum);
+ // SAFETY: the reservation is new and `minimum` bytes fit in it.
+ let base = unsafe {
+ let base = sys::reserve(capacity + GUARD_SIZE);
+ sys::make_available(base, minimum);
+ base
+ };
+ counters.num_bytes_accessible.fetch_add(minimum, SeqCst);
+ Self {
+ base: SendSyncPtr(base),
+ capacity,
+ size: minimum,
+ move_on_grow,
+ image_mode,
+ counters,
+ }
}
- impl CustomMemory {
- unsafe fn new(minimum: usize, maximum: usize, glob_counter: Arc<Mutex<usize>>) -> Self {
- let page_size = rustix::param::page_size();
- let guard_size = page_size;
- let size = maximum + guard_size;
- // We rely on the Wasm page size being multiple of host page size.
- assert_eq!(size % page_size, 0);
+ fn base(&self) -> *mut u8 {
+ self.base.0
+ }
+}
- let mem = unsafe {
- mmap_anonymous(null_mut(), size, ProtFlags::empty(), MapFlags::PRIVATE)
- .expect("mmap failed")
- };
+impl Drop for CustomMemory {
+ fn drop(&mut self) {
+ self.counters
+ .num_bytes_accessible
+ .fetch_sub(self.size, SeqCst);
+ // SAFETY: this memory owns its reservation, which nothing uses anymore.
+ unsafe { sys::release(self.base(), self.capacity + GUARD_SIZE) };
+ }
+}
+
+// SAFETY: the first `size` bytes at `base` are committed, readable, writable
+// and zero-initialized, and the rest of the reservation up to `capacity` plus
+// the guard region is inaccessible. Reservations are aligned to at least the
+// host page size. Without `move_on_grow` the memory never moves and refuses to
+// grow beyond its capacity. With `move_on_grow` every growth moves it to a new
+// reservation, which is only valid for growth beyond the capacity; growing
+// within the capacity then deliberately violates the base pointer stability
+// requirement, which one test uses to check that Wasmtime detects it.
+unsafe impl LinearMemory for CustomMemory {
+ fn byte_size(&self) -> usize {
+ self.size
+ }
- // NOTE: mmap_anonymous returns zero initialized memory, which is relied upon by this
- // API.
+ fn byte_capacity(&self) -> usize {
+ self.capacity
+ }
+ fn grow_to(&mut self, new_size: usize) -> Result<()> {
+ self.counters.grow_to_calls.fetch_add(1, SeqCst);
+ if self.move_on_grow {
+ let capacity = new_size.max(self.capacity);
+ // SAFETY: the new reservation is fresh, and the old one is owned
+ // by this memory and released after copying its contents.
unsafe {
- mprotect(mem, minimum, MprotectFlags::READ | MprotectFlags::WRITE)
- .expect("mprotect failed");
+ let base = sys::reserve(capacity + GUARD_SIZE);
+ sys::make_available(base, new_size);
+ ptr::copy_nonoverlapping(self.base(), base, self.size);
+ sys::release(self.base(), self.capacity + GUARD_SIZE);
+ self.base = SendSyncPtr(base);
}
- *glob_counter.lock().unwrap() += minimum;
-
- Self {
- mem: mem as usize,
- size,
- guard_size,
- used_wasm_bytes: minimum,
- glob_bytes_counter: glob_counter,
+ self.capacity = capacity;
+ } else {
+ if new_size > self.capacity {
+ bail!("custom memory cannot grow beyond its capacity");
}
+ // SAFETY: the new bytes lie within this memory's reservation. The
+ // size is a multiple of 64K, so the start is page aligned.
+ unsafe { sys::make_available(self.base().add(self.size), new_size - self.size) };
}
+ self.counters
+ .num_bytes_accessible
+ .fetch_add(new_size - self.size, SeqCst);
+ self.size = new_size;
+ Ok(())
}
- impl Drop for CustomMemory {
- fn drop(&mut self) {
- *self.glob_bytes_counter.lock().unwrap() -= self.used_wasm_bytes;
- unsafe { munmap(self.mem as *mut _, self.size).expect("munmap failed") };
- }
+ fn as_ptr(&self) -> *mut u8 {
+ self.base()
}
- unsafe impl LinearMemory for CustomMemory {
- fn byte_size(&self) -> usize {
- self.used_wasm_bytes
- }
-
- fn byte_capacity(&self) -> usize {
- self.size - self.guard_size
+ fn init_from_image(&mut self, image: &MemoryImage) -> Result<bool> {
+ self.counters.init_from_image_calls.fetch_add(1, SeqCst);
+ match self.image_mode {
+ ImageMode::Map => {
+ assert!(image.linear_memory_offset() + image.len() <= self.size);
+ // SAFETY: this memory owns its whole reservation and nothing
+ // has accessed it yet.
+ unsafe { image.map_at(self.base())? };
+ self.counters.images_mapped.fetch_add(1, SeqCst);
+ Ok(true)
+ }
+ ImageMode::ReturnFalse => Ok(false),
+ ImageMode::Lie => Ok(true),
+ ImageMode::Fail => bail!("custom memory refused its image"),
}
+ }
+}
- fn grow_to(&mut self, new_size: usize) -> wasmtime::Result<()> {
- println!("grow to {new_size:x}");
- let delta = new_size - self.used_wasm_bytes;
- unsafe {
- let start = (self.mem as *mut u8).add(self.used_wasm_bytes) as _;
- mprotect(start, delta, MprotectFlags::READ | MprotectFlags::WRITE)
- .expect("mprotect failed");
- }
+struct CustomMemoryCreator {
+ image_mode: ImageMode,
+ capacity_of_each: usize,
+ move_on_grow: bool,
+ counters: Arc<Counters>,
+}
- *self.glob_bytes_counter.lock().unwrap() += delta;
- self.used_wasm_bytes = new_size;
- Ok(())
+impl CustomMemoryCreator {
+ fn new(image_mode: ImageMode) -> Self {
+ Self {
+ image_mode,
+ capacity_of_each: MAP_CAPACITY,
+ move_on_grow: false,
+ counters: Arc::default(),
}
+ }
- fn as_ptr(&self) -> *mut u8 {
- self.mem as *mut u8
- }
+ fn with_capacity(mut self, capacity: usize) -> Self {
+ self.capacity_of_each = capacity;
+ self
}
- struct CustomMemoryCreator {
- pub num_created_memories: Mutex<usize>,
- pub num_total_bytes: Arc<Mutex<usize>>,
+ fn moving(mut self) -> Self {
+ self.move_on_grow = true;
+ self
}
+}
- impl CustomMemoryCreator {
- pub fn new() -> Self {
- Self {
- num_created_memories: Mutex::new(0),
- num_total_bytes: Arc::new(Mutex::new(0)),
- }
- }
+unsafe impl MemoryCreator for CustomMemoryCreator {
+ fn new_memory(
+ &self,
+ ty: MemoryType,
+ minimum: usize,
+ _maximum: Option<usize>,
+ reserved_size: Option<usize>,
+ guard_size: usize,
+ ) -> Result<Box<dyn LinearMemory>, String> {
+ assert_eq!(guard_size, 0);
+ assert_eq!(reserved_size, Some(0));
+ assert!(!ty.is_64());
+ let capacity = self.capacity_of_each;
+ self.counters.num_memories_created.fetch_add(1, SeqCst);
+ Ok(Box::new(CustomMemory::new(
+ minimum,
+ capacity,
+ self.move_on_grow,
+ self.image_mode,
+ self.counters.clone(),
+ )))
}
+}
- unsafe impl MemoryCreator for CustomMemoryCreator {
- fn new_memory(
- &self,
- ty: MemoryType,
- minimum: usize,
- maximum: Option<usize>,
- reserved_size: Option<usize>,
- guard_size: usize,
- ) -> Result<Box<dyn LinearMemory>, String> {
- assert_eq!(guard_size, 0);
- assert_eq!(reserved_size, Some(0));
- assert!(!ty.is_64());
- unsafe {
- // Cap the maximum at 10MiB to reduce the virtual memory
- // allocated by this test to execute on 32-bit platforms.
- let mem = Box::new(CustomMemory::new(
- minimum,
- maximum.unwrap_or(10 << 20),
- self.num_total_bytes.clone(),
- ));
- *self.num_created_memories.lock().unwrap() += 1;
- Ok(mem)
- }
- }
+/// Returns a store whose memories are created by `creator`, and the creator's
+/// counters. `configure` can further adjust the engine's configuration.
+fn store(
+ creator: CustomMemoryCreator,
+ configure: impl FnOnce(&mut Config),
+) -> (Store<()>, Arc<Counters>) {
+ let counters = creator.counters.clone();
+ let mut config = Config::new();
+ config
+ .with_host_memory(Arc::new(creator))
+ .memory_reservation(0)
+ .memory_guard_size(0);
+ configure(&mut config);
+ (Store::new(&Engine::new(&config).unwrap(), ()), counters)
+}
+
+/// A module whose memory is initialized by a data segment.
+const DATA_MODULE: &str = r#"
+ (module
+ (memory (export "memory") 1 4)
+ (data (i32.const 0) "hello")
+ (func (export "load") (param i32) (result i32)
+ (i32.load8_u (local.get 0)))
+ (func (export "store") (param i32 i32)
+ (i32.store8 (local.get 0) (local.get 1)))
+ (func (export "grow") (param i32) (result i32)
+ (memory.grow (local.get 0)))
+ )
+"#;
+
+/// This function tests that:
+/// - "hello" is accessible in the Wasm memory where it's expected to (in static data);
+/// - this stays true after growing the memory by a page.
+fn check_data_module(store: &mut Store<()>, instance: Instance) -> Result<()> {
+ let load = instance.get_typed_func::<i32, i32>(&mut *store, "load")?;
+ let grow = instance.get_typed_func::<i32, i32>(&mut *store, "grow")?;
+ for (i, byte) in b"hello\0".iter().enumerate() {
+ assert_eq!(load.call(&mut *store, i as i32)?, i32::from(*byte));
+ }
+ assert_eq!(grow.call(&mut *store, 1)?, 1);
+ for (i, byte) in b"hello\0".iter().enumerate() {
+ assert_eq!(load.call(&mut *store, i as i32)?, i32::from(*byte));
}
+ assert_eq!(load.call(&mut *store, WASM_PAGE as i32)?, 0);
+ Ok(())
+}
- fn config() -> (Store<()>, Arc<CustomMemoryCreator>) {
- let mem_creator = Arc::new(CustomMemoryCreator::new());
- let mut config = Config::new();
- config
- .with_host_memory(mem_creator.clone())
- .memory_reservation(0)
- .memory_guard_size(0);
- (Store::new(&Engine::new(&config).unwrap(), ()), mem_creator)
+// CoW of in-memory data requires memfd, which is linux-only.
+#[cfg(target_os = "linux")]
+mod linux_only {
+ use super::*;
+ // Regression test for [#10740](https://github.com/bytecodealliance/wasmtime/issues/10740):
+ // a custom memory that does not support CoW forces a copy from Wasmtime.
+ #[test]
+ #[cfg(target_os = "linux")]
+ fn host_memory_with_cow_image_when_impl_refuses_mapping() -> Result<()> {
+ let (mut store, counters) =
+ store(CustomMemoryCreator::new(ImageMode::ReturnFalse), |cfg| {
+ cfg.memory_init_cow(true);
+ });
+ let module = Module::new(store.engine(), DATA_MODULE)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ check_data_module(&mut store, instance)?;
+ assert_eq!(counters.num_memories_created.load(SeqCst), 1);
+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 1);
+ assert_eq!(counters.images_mapped.load(SeqCst), 0);
+ Ok(())
+ }
+
+ // When `init_from_image` returns `true`, Wasmtime trusts it and doesn't copy
+ // the module's data segments into the memory.
+ // This is relying on our CustomMemory breaking the contract (ImageMode::Lie) and not actually performing the mapping.
+ #[test]
+ #[cfg(target_os = "linux")]
+ fn host_memory_init_from_image_true_skips_data_segments() -> Result<()> {
+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Lie), |cfg| {
+ cfg.memory_init_cow(true);
+ });
+ let module = Module::new(store.engine(), DATA_MODULE)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let load = instance.get_typed_func::<i32, i32>(&mut store, "load")?;
+ // The memory claimed to hold the image but left itself zeroed.
+ assert_eq!(load.call(&mut store, 0)?, 0); // Would be 'h' if data was copied.
+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 1);
+ Ok(())
}
+ // This checks the CoW semantics: writing to one instantiated memory does not leak into memories
+ // later instantiated from the same image.
#[test]
- fn host_memory() -> wasmtime::Result<()> {
- let (mut store, mem_creator) = config();
- let module = Module::new(
- store.engine(),
- r#"
- (module
- (memory (export "memory") 1)
- )
- "#,
- )?;
- Instance::new(&mut store, &module, &[])?;
-
- assert_eq!(*mem_creator.num_created_memories.lock().unwrap(), 1);
+ #[cfg(target_os = "linux")]
+ fn host_memory_with_cow_image_when_impl_maps() -> Result<()> {
+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Map), |cfg| {
+ cfg.memory_init_cow(true);
+ });
+ let module = Module::new(store.engine(), DATA_MODULE)?;
+
+ let first_instance = Instance::new(&mut store, &module, &[])?;
+ check_data_module(&mut store, first_instance)?;
+
+ let first_store = first_instance.get_typed_func::<(i32, i32), ()>(&mut store, "store")?;
+ first_store.call(&mut store, (0, i32::from(b'x')))?;
+ let second_instance = Instance::new(&mut store, &module, &[])?;
+
+ let first_load = first_instance.get_typed_func::<i32, i32>(&mut store, "load")?;
+ assert_eq!(first_load.call(&mut store, 0)?, i32::from(b'x'));
+ let second_load = second_instance.get_typed_func::<i32, i32>(&mut store, "load")?;
+ assert_eq!(second_load.call(&mut store, 0)?, i32::from(b'h'));
+
+ assert_eq!(counters.num_memories_created.load(SeqCst), 2);
+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 2);
+ assert_eq!(counters.images_mapped.load(SeqCst), 2);
Ok(())
}
+ // Check that failures from implementations are propagated as-is (part of the public contract).
#[test]
- fn host_memory_grow() -> wasmtime::Result<()> {
- let (mut store, mem_creator) = config();
- let module = Module::new(
- store.engine(),
- r#"
- (module
- (func $f (drop (memory.grow (i32.const 1))))
- (memory (export "memory") 1 2)
- (start $f)
- )
- "#,
- )?;
-
- Instance::new(&mut store, &module, &[])?;
- let instance2 = Instance::new(&mut store, &module, &[])?;
-
- assert_eq!(*mem_creator.num_created_memories.lock().unwrap(), 2);
-
- assert_eq!(
- instance2
- .get_memory(&mut store, "memory")
- .unwrap()
- .size(&store),
- 2
+ #[cfg(target_os = "linux")]
+ fn host_memory_init_from_image_error_fails_instantiation() -> Result<()> {
+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Fail), |cfg| {
+ cfg.memory_init_cow(true);
+ });
+ let module = Module::new(store.engine(), DATA_MODULE)?;
+ let err = Instance::new(&mut store, &module, &[]).unwrap_err();
+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 1);
+ assert!(
+ format!("{err:?}").contains("custom memory refused its image"),
+ "unexpected error: {err:?}"
);
+ Ok(())
+ }
+}
- // we take the lock outside the assert, so it won't get poisoned on assert failure
- let tot_pages = *mem_creator.num_total_bytes.lock().unwrap();
- assert_eq!(
- tot_pages,
- (4 * wasmtime_environ::Memory::DEFAULT_PAGE_SIZE) as usize
- );
+// With CoW disabled `init_from_image` should not be called,
+// and the data segments should be copied into the custom memory.
+#[test]
+fn host_memory_without_cow_does_not_call_init_from_image() -> Result<()> {
+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Fail), |cfg| {
+ cfg.memory_init_cow(false);
+ });
+ let module = Module::new(store.engine(), DATA_MODULE)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ check_data_module(&mut store, instance)?;
+ assert_eq!(counters.init_from_image_calls.load(SeqCst), 0);
+ Ok(())
+}
- drop(store);
- let tot_pages = *mem_creator.num_total_bytes.lock().unwrap();
- assert_eq!(tot_pages, 0);
+// Regression test: memories created by the host with `Memory::new` used to
+// ignore the configured `MemoryCreator`.
+#[test]
+fn host_created_memory_uses_creator() -> Result<()> {
+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Fail), |_| {});
- Ok(())
+ let memory = Memory::new(&mut store, MemoryType::new(1, Some(3))?)?;
+ assert_eq!(counters.num_memories_created.load(SeqCst), 1);
+ assert_eq!(counters.num_bytes_accessible.load(SeqCst), WASM_PAGE);
+
+ // Growing from the host goes through `CustomMemory::grow_to`...
+ memory.grow(&mut store, 1)?;
+ assert_eq!(counters.num_bytes_accessible.load(SeqCst), 2 * WASM_PAGE);
+
+ // ... and so does growing from Wasm.
+ memory.data_mut(&mut store)[WASM_PAGE] = 42;
+ let module = Module::new(
+ store.engine(),
+ r#"
+ (module
+ (memory (import "" "memory") 1)
+ (func (export "grow") (result i32) (memory.grow (i32.const 1)))
+ (func (export "load") (param i32) (result i32)
+ (i32.load8_u (local.get 0)))
+ )
+ "#,
+ )?;
+ let instance = Instance::new(&mut store, &module, &[memory.into()])?;
+ let load = instance.get_typed_func::<i32, i32>(&mut store, "load")?;
+ let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?;
+
+ assert_eq!(load.call(&mut store, WASM_PAGE as i32)?, 42);
+ assert_eq!(grow.call(&mut store, ())?, 2);
+ assert_eq!(counters.num_bytes_accessible.load(SeqCst), 3 * WASM_PAGE);
+
+ // The importing instance didn't create a memory of its own.
+ assert_eq!(counters.num_memories_created.load(SeqCst), 1);
+
+ drop(store);
+ assert_eq!(counters.num_bytes_accessible.load(SeqCst), 0);
+
+ Ok(())
+}
+
+// A memory that moves when growing beyond its capacity keeps its contents,
+// and wasm code uses the new location afterwards.
+#[test]
+fn growth_that_moves_memory() -> Result<()> {
+ let creator = CustomMemoryCreator::new(ImageMode::ReturnFalse)
+ .with_capacity(WASM_PAGE)
+ .moving();
+ let (mut store, counters) = store(creator, |_| {});
+ let module = Module::new(store.engine(), DATA_MODULE)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let memory = instance.get_memory(&mut store, "memory").unwrap();
+ let load = instance.get_typed_func::<i32, i32>(&mut store, "load")?;
+ let store_byte = instance.get_typed_func::<(i32, i32), ()>(&mut store, "store")?;
+ let grow = instance.get_typed_func::<i32, i32>(&mut store, "grow")?;
+
+ store_byte.call(&mut store, (7, 42))?;
+ let base_before = memory.data_ptr(&store);
+
+ assert_eq!(grow.call(&mut store, 1)?, 1);
+ assert_eq!(counters.grow_to_calls.load(SeqCst), 1);
+ // Sanity check: memory moved somewhere else.
+ assert_ne!(memory.data_ptr(&store), base_before);
+ assert_eq!(memory.data_size(&store), 2 * WASM_PAGE);
+
+ assert_eq!(load.call(&mut store, 7)?, 42);
+ store_byte.call(&mut store, (WASM_PAGE as i32 + 7, 43))?;
+ assert_eq!(load.call(&mut store, WASM_PAGE as i32 + 7)?, 43);
+ assert_eq!(memory.data(&store)[WASM_PAGE + 7], 43);
+ Ok(())
+}
+
+// With `Config::memory_may_move(false)` a memory still grows within its
+// capacity, but growth beyond it is refused without asking the memory.
+#[test]
+fn growth_limited_to_capacity_when_memory_may_not_move() -> Result<()> {
+ let creator = CustomMemoryCreator::new(ImageMode::ReturnFalse).with_capacity(2 * WASM_PAGE);
+ let (mut store, counters) = store(creator, |config| {
+ config.memory_may_move(false);
+ });
+ let module = Module::new(store.engine(), DATA_MODULE)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let memory = instance.get_memory(&mut store, "memory").unwrap();
+ let grow = instance.get_typed_func::<i32, i32>(&mut store, "grow")?;
+ let base = memory.data_ptr(&store);
+ assert_eq!(memory.data_size(&store), WASM_PAGE);
+
+ assert_eq!(grow.call(&mut store, 1)?, 1);
+ assert_eq!(counters.grow_to_calls.load(SeqCst), 1);
+ assert_eq!(memory.data_size(&store), 2 * WASM_PAGE);
+ assert_eq!(memory.data_ptr(&store), base);
+
+ assert_eq!(grow.call(&mut store, 1)?, -1);
+ assert_eq!(counters.grow_to_calls.load(SeqCst), 1);
+ assert_eq!(memory.data_size(&store), 2 * WASM_PAGE);
+ assert_eq!(memory.data_ptr(&store), base);
+ Ok(())
+}
+
+// A memory that moves while growing within its capacity violates the `LinearMemory` contract.
+// Wasmtime does not guarantee that this is detected, but it is an easy check that prevents
+// potentially catastrophic errors, so we make sure it leads to a panic.
+#[test]
+#[should_panic(expected = "linear memory moved on grow when it must not have")]
+fn growth_within_capacity_that_moves_memory_panics() {
+ let creator = CustomMemoryCreator::new(ImageMode::ReturnFalse)
+ .with_capacity(2 * WASM_PAGE)
+ .moving();
+ let (mut store, _counters) = store(creator, |_| {});
+ let memory = Memory::new(&mut store, MemoryType::new(1, None).unwrap()).unwrap();
+ let _ = memory.grow(&mut store, 1);
+}
+
+// Mmap providers for unix and windows.
+mod sys {
+ #[cfg(unix)]
+ pub use unix::{make_available, release, reserve};
+ #[cfg(windows)]
+ pub use windows::{make_available, release, reserve};
+
+ #[cfg(unix)]
+ mod unix {
+ use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect, munmap};
+
+ pub unsafe fn reserve(size: usize) -> *mut u8 {
+ let ptr = unsafe {
+ mmap_anonymous(
+ std::ptr::null_mut(),
+ size,
+ ProtFlags::empty(),
+ MapFlags::PRIVATE,
+ )
+ };
+ ptr.expect("mmap failed").cast()
+ }
+
+ pub unsafe fn make_available(ptr: *mut u8, len: usize) {
+ unsafe { mprotect(ptr.cast(), len, MprotectFlags::READ | MprotectFlags::WRITE) }
+ .expect("mprotect failed");
+ }
+
+ pub unsafe fn release(ptr: *mut u8, size: usize) {
+ unsafe { munmap(ptr.cast(), size) }.expect("munmap failed");
+ }
+ }
+
+ #[cfg(windows)]
+ mod windows {
+ use windows_sys::Win32::System::Memory::{
+ MEM_COMMIT, MEM_RELEASE, MEM_RESERVE, PAGE_NOACCESS, PAGE_READWRITE, VirtualAlloc,
+ VirtualFree,
+ };
+
+ pub unsafe fn reserve(size: usize) -> *mut u8 {
+ let ptr = unsafe { VirtualAlloc(std::ptr::null(), size, MEM_RESERVE, PAGE_NOACCESS) };
+ assert!(!ptr.is_null(), "VirtualAlloc(MEM_RESERVE) failed");
+ ptr.cast()
+ }
+
+ pub unsafe fn make_available(ptr: *mut u8, len: usize) {
+ if len == 0 {
+ return;
+ }
+ let ret = unsafe { VirtualAlloc(ptr.cast(), len, MEM_COMMIT, PAGE_READWRITE) };
+ assert!(!ret.is_null(), "VirtualAlloc(MEM_COMMIT) failed");
+ }
+
+ pub unsafe fn release(ptr: *mut u8, _size: usize) {
+ let ret = unsafe { VirtualFree(ptr.cast(), 0, MEM_RELEASE) };
+ assert_ne!(ret, 0, "VirtualFree failed");
+ }
}
}