A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Remove `RuntimeMemoryCreator` and the proxy. (#15046)
Mat Gienieczko2a5abbef6132
diff --git a/crates/c-api/include/wasmtime/config.h b/crates/c-api/include/wasmtime/config.hindex d20b7916d4..a34d458933 100644--- a/crates/c-api/include/wasmtime/config.h+++ b/crates/c-api/include/wasmtime/config.h@@ -596,20 +596,81 @@ typedef struct wasmtime_linear_memory {void (*finalizer)(void *);} wasmtime_linear_memory_t;+/**+ * \brief The kind of memory a #wasmtime_new_memory_callback_t is asked to+ * create, values are in #wasmtime_memory_kind_enum.+ */+typedef uint8_t wasmtime_memory_kind_t;++/**+ * \brief Kinds of memory that a custom memory creator may be asked to create.+ *+ * For more information see the Rust documentation at+ * https://docs.wasmtime.dev/api/wasmtime/enum.MemoryKind.html+ */+enum wasmtime_memory_kind_enum { // MemoryKind+ /// A standard WebAssembly linear memory (module or host).+ WASMTIME_MEMORY_KIND_LINEAR_MEMORY,+ /// The memory backing a store's GC heap.+ ///+ /// It differs from standard linear memory:+ /// - It is created lazily, at most once per store: when a module that needs a+ /// GC heap is+ /// instantiated or when the first GC object is allocated. This might happen+ /// in the middle of+ /// execution of WebAssembly code.+ /// - It grows as GC objects are allocated, independently of `memory.grow`.+ /// - Its contents are Wasmtime's internal representation of GC objects, not+ /// data of the guest+ /// program. They are only meaningful to the store that created the GC heap,+ /// so they should not+ /// be shared, persisted, copied to other memories, etc.+ WASMTIME_MEMORY_KIND_GC_HEAP,+};++/**+ * \brief Parameters of a request to create a new linear memory.+ *+ * This structure is owned by Wasmtime and is only valid for the duration of+ * the #wasmtime_new_memory_callback_t invocation it is passed to. New fields+ * may be appended in future versions.+ *+ * For more information see the Rust documentation at+ * https://docs.wasmtime.dev/api/wasmtime/struct.MemoryRequest.html+ */+typedef struct wasmtime_memory_request {+ /// The type of memory being created.+ const wasm_memorytype_t *ty;+ /// Minimum size of the memory, in bytes.+ size_t minimum;+ /// Maximum size of the memory, in bytes, or `SIZE_MAX` if unbounded.+ size_t maximum;+ /// Size of the virtual memory reservation to make for this memory, in+ /// bytes. A multiple of the system page size.+ size_t reserved_size_in_bytes;+ /// Number of bytes after the memory allocation that must be left unmapped.+ /// A multiple of the system page size.+ size_t guard_size_in_bytes;+ /// The kind of memory being created, see #wasmtime_memory_kind_enum.+ wasmtime_memory_kind_t kind;+} wasmtime_memory_request_t;+/*** A callback to create a new LinearMemory from the specified parameters.** The result should be written to `memory_ret` and wasmtime will own the values* written into that struct.*+ * The memory returned must satisfy all properties of `request` and be zero+ * filled. `request` is only valid for the duration of this call.+ ** This callback must be thread-safe.** For more information about the parameters see the Rust documentation at* https://docs.wasmtime.dev/api/wasmtime/trait.MemoryCreator.html#tymethod.new_memory*/typedef wasmtime_error_t *(*wasmtime_new_memory_callback_t)(- void *env, const wasm_memorytype_t *ty, size_t minimum, size_t maximum,- size_t reserved_size_in_bytes, size_t guard_size_in_bytes,+ void *env, const wasmtime_memory_request_t *request,wasmtime_linear_memory_t *memory_ret);/**diff --git a/crates/c-api/include/wasmtime/config.hh b/crates/c-api/include/wasmtime/config.hhindex 03d5ee84aa..f862be5997 100644--- a/crates/c-api/include/wasmtime/config.hh+++ b/crates/c-api/include/wasmtime/config.hh@@ -69,6 +69,50 @@ enum RegallocAlgorithm {SinglePass = WASMTIME_REGALLOC_SINGLE_PASS,};+/// \brief The kind of memory a custom memory creator is asked to create.+///+/// https://docs.wasmtime.dev/api/wasmtime/enum.MemoryKind.html+enum class MemoryKind {+ /// A standard WebAssembly linear memory (module or host).+ LinearMemory = WASMTIME_MEMORY_KIND_LINEAR_MEMORY,+ /// The memory backing a store's GC heap.+ ///+ /// It differs from standard linear memory:+ /// - It is created lazily, at most once per store: when a module that needs a+ /// GC heap is instantiated or when the first GC object is allocated.+ /// This might happen in the middle of execution of WebAssembly code.+ /// - It grows as GC objects are allocated, independently of `memory.grow`.+ /// - Its contents are Wasmtime's internal representation of GC objects, not+ /// data of the guest program. They are only meaningful to the store+ /// that created the GC heap, so they should not be shared, persisted,+ /// copied to other memories, etc.+ GcHeap = WASMTIME_MEMORY_KIND_GC_HEAP,+};++/// \brief Parameters of a request to create a new linear memory, passed to a+/// custom memory creator. Only valid for the duration of that call.+///+/// https://docs.wasmtime.dev/api/wasmtime/struct.MemoryRequest.html+class MemoryRequest {+ const wasmtime_memory_request_t *req;++public:+ /// Wraps a raw C request.+ explicit MemoryRequest(const wasmtime_memory_request_t *req) : req(req) {}+ /// The type of memory being created.+ MemoryType::Ref ty() const { return MemoryType::Ref(req->ty); }+ /// Minimum size in bytes.+ size_t minimum() const { return req->minimum; }+ /// Maximum size in bytes, `SIZE_MAX` if unbounded.+ size_t maximum() const { return req->maximum; }+ /// Required virtual memory reservation in bytes.+ size_t reserved_size_in_bytes() const { return req->reserved_size_in_bytes; }+ /// Required unmapped guard region after the allocation, in bytes.+ size_t guard_size_in_bytes() const { return req->guard_size_in_bytes; }+ /// The kind of memory being created.+ MemoryKind kind() const { return static_cast<MemoryKind>(req->kind); }+};+#ifdef WASMTIME_FEATURE_POOLING_ALLOCATOR/*** \brief Pool allocation configuration for Wasmtime.@@ -643,15 +687,11 @@ private:template <typename T>static wasmtime_error_t *- raw_new_memory(void *env, const wasm_memorytype_t *ty, size_t minimum,- size_t maximum, size_t reserved_size_in_bytes,- size_t guard_size_in_bytes,+ raw_new_memory(void *env, const wasmtime_memory_request_t *request,wasmtime_linear_memory_t *memory_ret) {using Memory = typename T::Memory;T *creator = reinterpret_cast<T *>(env);- Result<Memory> result =- creator->new_memory(MemoryType::Ref(ty), minimum, maximum,- reserved_size_in_bytes, guard_size_in_bytes);+ Result<Memory> result = creator->new_memory(MemoryRequest(request));if (!result) {return result.err().capi_release();}diff --git a/crates/c-api/src/config.rs b/crates/c-api/src/config.rsindex e494f7786d..c889486366 100644--- a/crates/c-api/src/config.rs+++ b/crates/c-api/src/config.rs@@ -7,8 +7,8 @@ use std::os::raw::c_char;use std::ptr;use std::{ffi::CStr, sync::Arc};use wasmtime::{- Config, LinearMemory, MemoryCreator, OptLevel, ProfilingStrategy, RegallocAlgorithm, Result,- Strategy,+ Config, LinearMemory, MemoryCreator, MemoryKind, MemoryRequest, OptLevel, ProfilingStrategy,+ RegallocAlgorithm, Result, Strategy, bail,};#[cfg(feature = "pooling-allocator")]@@ -356,13 +356,26 @@ pub struct wasmtime_linear_memory_t {finalizer: Option<extern "C" fn(arg1: *mut std::ffi::c_void)>,}-pub type wasmtime_new_memory_callback_t = extern "C" fn(- env: *mut std::ffi::c_void,- ty: &wasm_memorytype_t,+#[repr(u8)]+#[derive(Clone, Copy)]+pub enum wasmtime_memory_kind_t {+ WASMTIME_MEMORY_KIND_LINEAR_MEMORY,+ WASMTIME_MEMORY_KIND_GC_HEAP,+}++#[repr(C)]+pub struct wasmtime_memory_request_t<'a> {+ ty: &'a wasm_memorytype_t,minimum: usize,maximum: usize,reserved_size_in_bytes: usize,guard_size_in_bytes: usize,+ kind: wasmtime_memory_kind_t,+}++pub type wasmtime_new_memory_callback_t = extern "C" fn(+ env: *mut std::ffi::c_void,+ request: &wasmtime_memory_request_t<'_>,memory_ret: *mut wasmtime_linear_memory_t,) -> Option<Box<wasmtime_error_t>>;@@ -419,14 +432,7 @@ unsafe impl Send for CHostMemoryCreator {}unsafe impl Sync for CHostMemoryCreator {}unsafe impl MemoryCreator for CHostMemoryCreator {- fn new_memory(- &self,- ty: wasmtime::MemoryType,- minimum: usize,- maximum: Option<usize>,- reserved_size_in_bytes: Option<usize>,- guard_size_in_bytes: usize,- ) -> Result<Box<dyn wasmtime::LinearMemory>, String> {+ fn new_memory(&self, request: &MemoryRequest<'_>) -> Result<Box<dyn wasmtime::LinearMemory>> {extern "C" fn panic_get_callback(_env: *mut std::ffi::c_void,_byte_size: &mut usize,@@ -440,6 +446,21 @@ unsafe impl MemoryCreator for CHostMemoryCreator {) -> Option<Box<wasmtime_error_t>> {panic!("a callback must be set");}+ let kind = match request.kind() {+ MemoryKind::LinearMemory => wasmtime_memory_kind_t::WASMTIME_MEMORY_KIND_LINEAR_MEMORY,+ MemoryKind::GcHeap => wasmtime_memory_kind_t::WASMTIME_MEMORY_KIND_GC_HEAP,+ // `MemoryKind` is `#[non_exhaustive]`+ kind => bail!("memory kind {kind:?} is not supported by the C API"),+ };+ let ty = wasm_memorytype_t::new(request.ty());+ let c_request = wasmtime_memory_request_t {+ ty: &ty,+ minimum: request.minimum(),+ maximum: request.maximum().unwrap_or(usize::MAX),+ reserved_size_in_bytes: request.reserved_size_in_bytes(),+ guard_size_in_bytes: request.guard_size_in_bytes(),+ kind,+ };let mut memory = wasmtime_linear_memory_t {env: ptr::null_mut(),get_memory: panic_get_callback,@@ -447,15 +468,7 @@ unsafe impl MemoryCreator for CHostMemoryCreator {finalizer: None,};let cb = self.new_memory;- let error = cb(- self.foreign.data,- &wasm_memorytype_t::new(ty),- minimum,- maximum.unwrap_or(usize::MAX),- reserved_size_in_bytes.unwrap_or(0),- guard_size_in_bytes,- &mut memory,- );+ let error = cb(self.foreign.data, &c_request, &mut memory);match error {None => {let foreign = crate::ForeignData {@@ -468,10 +481,7 @@ unsafe impl MemoryCreator for CHostMemoryCreator {grow_memory: memory.grow_memory,}))}- Some(err) => {- let err: wasmtime::Error = (*err).into();- Err(format!("{err}"))- }+ Some(err) => Err((*err).into()),}}}diff --git a/crates/c-api/tests/config.cc b/crates/c-api/tests/config.ccindex 371ae1ec2b..12e4423c73 100644--- a/crates/c-api/tests/config.cc+++ b/crates/c-api/tests/config.cc@@ -96,14 +96,13 @@ struct MyMemoryCreator {}};- Result<Memory> new_memory(const MemoryType::Ref &ty, size_t minimum,- size_t maximum, size_t reserved_size_in_bytes,- size_t guard_size_in_bytes) {- EXPECT_EQ(guard_size_in_bytes, 0);- EXPECT_EQ(reserved_size_in_bytes, 0);+ Result<Memory> new_memory(const MemoryRequest &request) {+ EXPECT_EQ(request.guard_size_in_bytes(), 0);+ EXPECT_EQ(request.reserved_size_in_bytes(), 0);+ EXPECT_EQ(request.kind(), MemoryKind::LinearMemory);Memory mem;- mem.grow_memory(minimum).unwrap();+ mem.grow_memory(request.minimum()).unwrap();return mem;}};diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rsindex e3bda88426..b21dd6733b 100644--- a/crates/environ/src/tunables.rs+++ b/crates/environ/src/tunables.rs@@ -428,6 +428,11 @@ impl<'a> MemoryTunables<'a> {}}+ /// Kind of memory these tunables are for.+ pub fn kind(&self) -> MemoryKind {+ self.kind+ }+/// Get the underlying tunables.////// This is ONLY for accessing tunable fields that DO NOT come in adiff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rsindex 254452409b..98ef130d5c 100644--- a/crates/wasmtime/src/config.rs+++ b/crates/wasmtime/src/config.rs@@ -17,11 +17,7 @@ use crate::memory::MemoryCreator;#[cfg(feature = "runtime")]use crate::profiling_agent::{self, ProfilingAgent};#[cfg(feature = "runtime")]-use crate::runtime::vm::{- GcRuntime, InstanceAllocator, OnDemandInstanceAllocator, RuntimeMemoryCreator,-};-#[cfg(feature = "runtime")]-use crate::trampoline::MemoryCreatorProxy;+use crate::runtime::vm::{GcRuntime, InstanceAllocator, OnDemandInstanceAllocator};#[cfg(feature = "async")]use crate::stack::{StackCreator, StackCreatorProxy};@@ -168,7 +164,7 @@ pub struct Config {#[cfg(feature = "cache")]pub(crate) cache: Option<Cache>,#[cfg(feature = "runtime")]- pub(crate) mem_creator: Option<Arc<dyn RuntimeMemoryCreator>>,+ pub(crate) mem_creator: Option<Arc<dyn MemoryCreator>>,#[cfg(feature = "runtime")]pub(crate) custom_code_memory: Option<Arc<dyn CustomCodeMemory>>,pub(crate) allocation_strategy: InstanceAllocationStrategy,@@ -1758,7 +1754,7 @@ impl Config {/// creating instance linear memories for the on-demand instance allocation strategy.#[cfg(feature = "runtime")]pub fn with_host_memory(&mut self, mem_creator: Arc<dyn MemoryCreator>) -> &mut Self {- self.mem_creator = Some(Arc::new(MemoryCreatorProxy(mem_creator)));+ self.mem_creator = Some(mem_creator);self}diff --git a/crates/wasmtime/src/runtime/memory.rs b/crates/wasmtime/src/runtime/memory.rsindex 73375272f3..ed7255359e 100644--- a/crates/wasmtime/src/runtime/memory.rs+++ b/crates/wasmtime/src/runtime/memory.rs@@ -12,7 +12,7 @@ use core::cell::UnsafeCell;use core::fmt;use core::slice;use core::time::Duration;-use wasmtime_environ::DefinedMemoryIndex;+use wasmtime_environ::{DefinedMemoryIndex, MemoryTunables};pub use crate::runtime::vm::WaitResult;@@ -980,45 +980,138 @@ impl core::hash::Hash for MemoryImage {/// Note that this is a relatively advanced feature and it is recommended to be/// familiar with Wasmtime runtime code to use it.pub unsafe trait MemoryCreator: Send + Sync {- /// Create a new `LinearMemory` object from the specified parameters.+ /// Create a new [`LinearMemory`] object from the specified parameters.///- /// The type of memory being created is specified by `ty` which indicates+ /// The returned memory must satisfy all of the properties of the [`MemoryRequest`]+ /// and be zero filled.+ fn new_memory(&self, request: &MemoryRequest<'_>) -> Result<Box<dyn LinearMemory>>;+}++/// The kind of memory that [`MemoryCreator`] is asked to create.+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]+#[non_exhaustive]+pub enum MemoryKind {+ /// A standard WebAssembly linear memory (module or host).+ LinearMemory,+ /// The memory backing a store's GC heap.+ ///+ /// It differs from standard linear memory:+ /// - It is created lazily, at most once per store: when a module that needs a GC heap is+ /// instantiated or when the first GC object is allocated. This might happen in the middle of+ /// execution of WebAssembly code.+ /// - It grows as GC objects are allocated, independently of `memory.grow`.+ /// - Its contents are Wasmtime's internal representation of GC objects, not data of the guest+ /// program. They are only meaningful to the store that created the GC heap, so they should not+ /// be shared, persisted, copied to other memories, etc.+ GcHeap,+}++impl From<wasmtime_environ::MemoryKind> for MemoryKind {+ fn from(value: wasmtime_environ::MemoryKind) -> Self {+ match value {+ wasmtime_environ::MemoryKind::LinearMemory => Self::LinearMemory,+ wasmtime_environ::MemoryKind::GcHeap => Self::GcHeap,+ }+ }+}++/// A request to a [`MemoryCreator::new_memory`].+pub struct MemoryRequest<'a> {+ ty: &'a wasmtime_environ::Memory,+ tunables: &'a MemoryTunables<'a>,+ minimum: usize,+ maximum: Option<usize>,+}++impl<'a> MemoryRequest<'a> {+ pub(crate) fn new(+ ty: &'a wasmtime_environ::Memory,+ tunables: &'a MemoryTunables<'a>,+ minimum: usize,+ maximum: Option<usize>,+ ) -> Self {+ Self {+ ty,+ tunables,+ minimum,+ maximum,+ }+ }++ /// The type of memory being created. Indicates/// both the minimum and maximum size, in wasm pages. The minimum and- /// maximum sizes, in bytes, are also specified as parameters to avoid- /// integer conversion if desired.- ///- /// The `reserved_size_in_bytes` value indicates the expected size of the- /// reservation that is to be made for this memory. If this value is `None`- /// than the implementation is free to allocate memory as it sees fit. If- /// the value is `Some`, however, then the implementation is expected to- /// reserve that many bytes for the memory's allocation, plus the guard+ /// maximum sizes, in bytes, are also specified as [`minimum`](Self::minimum) and [`maximum`](Self::maximum)+ /// to avoid integer conversion if desired.+ pub fn ty(&self) -> MemoryType {+ MemoryType::from_wasmtime_memory(self.ty)+ }++ /// Minimum size of the memory in bytes.+ /// Same as the minimum of [`ty`](Self::ty), but in bytes.+ pub fn minimum(&self) -> usize {+ self.minimum+ }++ /// Limit on the maximum size of the memory in bytes, if any.+ /// Same as the maximum of [`ty`](Self::ty), but in bytes.+ pub fn maximum(&self) -> Option<usize> {+ self.maximum+ }++ /// Indicates the expected size of the reservation that is to be made for this memory.+ /// The implementation must reserve this many bytes for the memory's allocation, plus the guard/// size at the end. Note that this reservation need only be a virtual/// memory reservation, physical memory does not need to be allocated- /// immediately. In this case `grow` should never move the base pointer and- /// the maximum size of `ty` is guaranteed to fit within- /// `reserved_size_in_bytes`.+ /// immediately.+ ///+ /// If this value is not zero, [`grow_to`](LinearMemory::grow_to) should never move the base pointer and+ /// the maximum size of [`ty`](Self::ty) is guaranteed to fit within `reserved_size_in_bytes`.///- /// The `guard_size_in_bytes` parameter indicates how many bytes of space,- /// after the memory allocation, is expected to be unmapped. JIT code will+ /// This value is tuned from the various [`Config`](crate::Config) methods about+ /// memory sizes/guards. It is guaranteed to be a multiple of the system page size.+ pub fn reserved_size_in_bytes(&self) -> usize {+ usize::try_from(self.tunables.reservation()).unwrap()+ }++ /// Indicates how many bytes of space after the memory allocation is expected to be unmapped. JIT code will/// elide bounds checks based on the `guard_size_in_bytes` provided, so for/// JIT code to work correctly the memory returned will need to be properly/// guarded with `guard_size_in_bytes` bytes left unmapped after the base/// allocation.///- /// Note that the `reserved_size_in_bytes` and `guard_size_in_bytes` options- /// are tuned from the various [`Config`](crate::Config) methods about- /// memory sizes/guards. Additionally these two values are guaranteed to be- /// multiples of the system page size.- ///- /// Memory created from this method should be zero filled.- fn new_memory(- &self,- ty: MemoryType,- minimum: usize,- maximum: Option<usize>,- reserved_size_in_bytes: Option<usize>,- guard_size_in_bytes: usize,- ) -> Result<Box<dyn LinearMemory>, String>;+ /// This value is tuned from the various [`Config`](crate::Config) methods about+ /// memory sizes/guards. It is guaranteed to be a multiple of the system page size.+ pub fn guard_size_in_bytes(&self) -> usize {+ usize::try_from(self.tunables.guard_size()).unwrap()+ }++ /// What kind of memory is requested.+ pub fn kind(&self) -> MemoryKind {+ self.tunables.kind().into()+ }++ /// Internal parameter: detailed internal representation of the memory being created.+ pub(crate) fn wasmtime_memory(&self) -> &'a wasmtime_environ::Memory {+ self.ty+ }++ /// Internal parameter: all the tunables, including those not exposed to the public API.+ pub(crate) fn tunables(&self) -> &'a MemoryTunables<'a> {+ self.tunables+ }+}++impl fmt::Debug for MemoryRequest<'_> {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("MemoryRequest")+ .field("ty", &self.ty())+ .field("minimum", &self.minimum)+ .field("maximum", &self.maximum)+ .field("reserved_size_in_bytes", &self.reserved_size_in_bytes())+ .field("guard_size_in_bytes", &self.guard_size_in_bytes())+ .field("kind", &self.kind())+ .finish()+ }}/// A constructor for externally-created shared memory.diff --git a/crates/wasmtime/src/runtime/trampoline.rs b/crates/wasmtime/src/runtime/trampoline.rsindex b465d3dc12..2a2e34e885 100644--- a/crates/wasmtime/src/runtime/trampoline.rs+++ b/crates/wasmtime/src/runtime/trampoline.rs@@ -6,7 +6,6 @@ mod table;mod tag;pub use self::global::*;-pub(crate) use memory::MemoryCreatorProxy;use self::memory::create_memory;use self::table::create_table;diff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rsindex b906b616dd..0476de97d0 100644--- a/crates/wasmtime/src/runtime/trampoline/memory.rs+++ b/crates/wasmtime/src/runtime/trampoline/memory.rs@@ -1,19 +1,17 @@use crate::MemoryType;-use crate::memory::{LinearMemory, MemoryCreator};use crate::prelude::*;use crate::runtime::vm::mpk::ProtectionKey;use crate::runtime::vm::{CompiledModuleId, InstanceAllocationRequest, InstanceAllocator, Memory, MemoryAllocationIndex,- ModuleRuntimeInfo, OnDemandInstanceAllocator, RuntimeMemoryCreator, SharedMemory, Table,- TableAllocationIndex,+ ModuleRuntimeInfo, OnDemandInstanceAllocator, SharedMemory, Table, TableAllocationIndex,};use crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque, StoreResourceLimiter};use alloc::sync::Arc;use core::future::Future;use core::pin::Pin;use wasmtime_environ::{- DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, MemoryKind, MemoryTunables,- Module, StaticModuleIndex, VMOffsets,+ DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, MemoryKind, Module,+ StaticModuleIndex, VMOffsets,};#[cfg(feature = "component-model")]@@ -74,30 +72,6 @@ pub async fn create_memory(}}-#[derive(Clone)]-pub(crate) struct MemoryCreatorProxy(pub Arc<dyn MemoryCreator>);--impl RuntimeMemoryCreator for MemoryCreatorProxy {- fn new_memory(- &self,- ty: &wasmtime_environ::Memory,- memory_tunables: &MemoryTunables<'_>,- minimum: usize,- maximum: Option<usize>,- ) -> Result<Box<dyn LinearMemory>> {- let reserved_size_in_bytes = Some(memory_tunables.reservation().try_into().unwrap());- self.0- .new_memory(- MemoryType::from_wasmtime_memory(ty),- minimum,- maximum,- reserved_size_in_bytes,- usize::try_from(memory_tunables.guard_size()).unwrap(),- )- .map_err(|e| format_err!(e))- }-}-struct SingleMemoryInstance<'a> {preallocation: Option<&'a SharedMemory>,ondemand: OnDemandInstanceAllocator,diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex 269486ef3f..05da5a9c0d 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -109,7 +109,7 @@ pub use crate::runtime::vm::instance::{PoolConcurrencyLimitError, PoolingAllocatorMetrics, PoolingInstanceAllocator,};pub use crate::runtime::vm::interpreter::*;-pub use crate::runtime::vm::memory::{Memory, RuntimeMemoryCreator, SharedMemory};+pub use crate::runtime::vm::memory::{Memory, SharedMemory};pub use crate::runtime::vm::mmap_vec::MmapVec;pub use crate::runtime::vm::provenance::*;pub use crate::runtime::vm::stack_switching::*;diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rsindex c512ae83bf..460ab810f9 100644--- a/crates/wasmtime/src/runtime/vm/instance.rs+++ b/crates/wasmtime/src/runtime/vm/instance.rs@@ -6,7 +6,7 @@ use crate::code::ModuleWithCode;use crate::module::ModuleRegistry;use crate::prelude::*;use crate::runtime::vm::export::{Export, ExportMemory};-use crate::runtime::vm::memory::{Memory, RuntimeMemoryCreator};+use crate::runtime::vm::memory::Memory;use crate::runtime::vm::table::{Table, TableElementType};use crate::runtime::vm::vmcontext::{VMBuiltinFunctionsArray, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition,diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rsindex 4a9ff0870c..4d5e29df07 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs@@ -1,9 +1,9 @@use super::{InstanceAllocationRequest, InstanceAllocator, MemoryAllocationIndex, TableAllocationIndex,};+use crate::MemoryCreator;use crate::prelude::*;use crate::runtime::vm::CompiledModuleId;-use crate::runtime::vm::instance::RuntimeMemoryCreator;use crate::runtime::vm::memory::{DefaultMemoryCreator, Memory};use crate::runtime::vm::mpk::ProtectionKey;use crate::runtime::vm::table::Table;@@ -29,7 +29,7 @@ use wasmtime_environ::{/// Represents the on-demand instance allocator.#[derive(Clone)]pub struct OnDemandInstanceAllocator {- mem_creator: Option<Arc<dyn RuntimeMemoryCreator>>,+ mem_creator: Option<Arc<dyn MemoryCreator>>,#[cfg(feature = "async")]stack_creator: Option<Arc<dyn RuntimeFiberStackCreator>>,#[cfg(feature = "async")]@@ -41,7 +41,7 @@ pub struct OnDemandInstanceAllocator {impl OnDemandInstanceAllocator {/// Creates a new on-demand instance allocator.pub fn new(- mem_creator: Option<Arc<dyn RuntimeMemoryCreator>>,+ mem_creator: Option<Arc<dyn MemoryCreator>>,stack_size: usize,stack_zeroing: bool,) -> Self {diff --git a/crates/wasmtime/src/runtime/vm/memory.rs b/crates/wasmtime/src/runtime/vm/memory.rsindex aa02d508c8..981fe43b21 100644--- a/crates/wasmtime/src/runtime/vm/memory.rs+++ b/crates/wasmtime/src/runtime/vm/memory.rs@@ -69,9 +69,6 @@//! `MemoryImage` is delegated to the `LinearMemory` itself through//! `LinearMemory::init_from_image`, and the pooling allocator's//! `StaticMemory` owns a `MemoryImageSlot` which is reused across instances.-//!-//! FIXME: don't have both RuntimeMemoryCreator and wasmtime::MemoryCreator,-//! they should be merged together.use crate::prelude::*;use crate::runtime::store::StoreResourceLimiter;@@ -79,7 +76,7 @@ use crate::runtime::vm::MemoryImage;use crate::runtime::vm::vmcontext::VMMemoryDefinition;#[cfg(feature = "pooling-allocator")]use crate::runtime::vm::{MemoryImageSlot, MmapOffset};-use crate::{Engine, LinearMemory};+use crate::{Engine, LinearMemory, MemoryCreator, MemoryRequest};use alloc::sync::Arc;use core::{ops::Range, ptr::NonNull};use wasmtime_environ::{MemoryKind, MemoryTunables};@@ -110,31 +107,21 @@ mod shared_memory_disabled;#[cfg(not(feature = "threads"))]pub use shared_memory_disabled::SharedMemory;-/// A memory allocator-pub trait RuntimeMemoryCreator: Send + Sync {- /// Create a new `LinearMemory`.- fn new_memory(- &self,- ty: &wasmtime_environ::Memory,- memory_tunables: &MemoryTunables<'_>,- minimum: usize,- maximum: Option<usize>,- ) -> Result<Box<dyn LinearMemory>>;-}--/// A default memory allocator used by Wasmtime+/// Default memory creator used by Wasmtime when no custom [`MemoryCreator`] implementation is+/// configured.+///+/// It chooses [`MmapMemory`] if available, falling back to [`MallocMemory`] otherwise.pub struct DefaultMemoryCreator;-impl RuntimeMemoryCreator for DefaultMemoryCreator {- /// Create new LinearMemory.- /// Chooses [`MmapMemory`] if it is available and tunables require it, falling back to [`MallocMemory`] otherwise.- fn new_memory(- &self,- ty: &wasmtime_environ::Memory,- memory_tunables: &MemoryTunables<'_>,- minimum: usize,- maximum: Option<usize>,- ) -> Result<Box<dyn LinearMemory>> {+// SAFETY: `MmapMemory` and `MallocMemory` own their allocations and we do not hold any references+// to them. Memory is zeroed automatically for mmap, and during growth in MallocMemory implementation.+unsafe impl MemoryCreator for DefaultMemoryCreator {+ fn new_memory(&self, request: &MemoryRequest<'_>) -> Result<Box<dyn LinearMemory>> {+ let ty = request.wasmtime_memory();+ let memory_tunables = request.tunables();+ let minimum = request.minimum();+ let maximum = request.maximum();+#[cfg(has_virtual_memory)]if memory_tunables.tunables().signals_based_traps|| memory_tunables.guard_size() > 0@@ -147,6 +134,10 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator {);}+ // MallocMemory does not have a maximum. This path is only possible when no initial+ // reservation was requested, in which case memory cannot move, or if virtual memory is+ // unavailable, in which case none of the usual optimizations are possible and this value+ // can be ignored.let _ = maximum;Ok(try_new::<Box<_>>(MallocMemory::new(ty, memory_tunables, minimum)?)?@@ -166,7 +157,7 @@ impl Memory {pub async fn new_dynamic(ty: &wasmtime_environ::Memory,engine: &Engine,- creator: &dyn RuntimeMemoryCreator,+ creator: &dyn MemoryCreator,memory_image: Option<&Arc<MemoryImage>>,limiter: Option<&mut StoreResourceLimiter<'_>>,kind: MemoryKind,@@ -174,7 +165,8 @@ impl Memory {let (minimum, maximum) = Self::limit_new(ty, limiter).await?;let tunables = engine.tunables();let memory_tunables = MemoryTunables::new(tunables, kind);- let allocation = creator.new_memory(ty, &memory_tunables, minimum, maximum)?;+ let request = MemoryRequest::new(ty, &memory_tunables, minimum, maximum);+ let allocation = creator.new_memory(&request)?;let memory = LocalMemory::new(ty, &memory_tunables, allocation, memory_image)?;Ok(if ty.shared {diff --git a/tests/all/memory_creator.rs b/tests/all/memory_creator.rsindex 340894cf85..2d07344370 100644--- a/tests/all/memory_creator.rs+++ b/tests/all/memory_creator.rs@@ -34,6 +34,7 @@ enum ImageMode {#[derive(Default)]struct Counters {num_memories_created: AtomicUsize,+ num_gc_heaps_created: AtomicUsize,num_bytes_accessible: AtomicUsize,grow_to_calls: AtomicUsize,init_from_image_calls: AtomicUsize,@@ -196,21 +197,17 @@ impl CustomMemoryCreator {}unsafe impl MemoryCreator for CustomMemoryCreator {- fn new_memory(- &self,- ty: MemoryType,- minimum: usize,- _maximum: Option<usize>,- reserved_size: Option<usize>,- guard_size: usize,- ) -> Result<Box<dyn LinearMemory>, String> {- assert_eq!(guard_size, 0);- assert_eq!(reserved_size, Some(0));- assert!(!ty.is_64());+ fn new_memory(&self, request: &MemoryRequest<'_>) -> Result<Box<dyn LinearMemory>> {+ assert_eq!(request.guard_size_in_bytes(), 0);+ assert_eq!(request.reserved_size_in_bytes(), 0);+ assert!(!request.ty().is_64());let capacity = self.capacity_of_each;self.counters.num_memories_created.fetch_add(1, SeqCst);+ if request.kind() == MemoryKind::GcHeap {+ self.counters.num_gc_heaps_created.fetch_add(1, SeqCst);+ }Ok(Box::new(CustomMemory::new(- minimum,+ request.minimum(),capacity,self.move_on_grow,self.image_mode,@@ -411,6 +408,29 @@ fn host_created_memory_uses_creator() -> Result<()> {Ok(())}+// The GC heaps are also created with custom creators and pass the correct kind.+#[test]+fn gc_heap_uses_creator() -> Result<()> {+ let (mut store, counters) = store(CustomMemoryCreator::new(ImageMode::Fail), |_| {});++ // A linear memory is not a GC heap.+ Memory::new(&mut store, MemoryType::new(1, None)?)?;+ assert_eq!(counters.num_memories_created.load(SeqCst), 1);+ assert_eq!(counters.num_gc_heaps_created.load(SeqCst), 0);++ // The first GC allocation creates the GC heap...+ ExternRef::new(&mut store, 42_u32)?;+ assert_eq!(counters.num_memories_created.load(SeqCst), 2);+ assert_eq!(counters.num_gc_heaps_created.load(SeqCst), 1);++ // ... that is reused for all GC operations.+ ExternRef::new(&mut store, 67_u32)?;+ assert_eq!(counters.num_memories_created.load(SeqCst), 2);+ assert_eq!(counters.num_gc_heaps_created.load(SeqCst), 1);++ Ok(())+}+// A memory that moves when growing beyond its capacity keeps its contents,// and wasm code uses the new location afterwards.#[test]