Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Use explicit boxing for InstanceAllocator async methods (#12849)

Nick Fitzgerald82ebbd5d4ef6
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index 4733dc6393..b8ea6a175b 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1929,7 +1929,7 @@ impl StoreOpaque {
let (mem_alloc_index, mem) = engine
.allocator()
- .allocate_memory(&mut request, &mem_ty, None)
+ .allocate_memory(&mut request, &mem_ty, None)?
.await?;
// Then, allocate the actual GC heap, passing in that memory
diff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rs
index 175e77b137..6d606f55e8 100644
--- a/crates/wasmtime/src/runtime/trampoline/memory.rs
+++ b/crates/wasmtime/src/runtime/trampoline/memory.rs
@@ -9,6 +9,8 @@ use crate::runtime::vm::{
};
use crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque, StoreResourceLimiter};
use alloc::sync::Arc;
+use core::future::Future;
+use core::pin::Pin;
use wasmtime_environ::{
DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, Module, StaticModuleIndex,
Tunables, VMOffsets,
@@ -126,7 +128,6 @@ struct SingleMemoryInstance<'a> {
ondemand: OnDemandInstanceAllocator,
}
-#[async_trait::async_trait]
unsafe impl InstanceAllocator for SingleMemoryInstance<'_> {
#[cfg(feature = "component-model")]
fn validate_component<'a>(
@@ -170,12 +171,15 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> {
self.ondemand.decrement_core_instance_count();
}
- async fn allocate_memory(
- &self,
- request: &mut InstanceAllocationRequest<'_, '_>,
- ty: &wasmtime_environ::Memory,
+ fn allocate_memory<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ request: &'a mut InstanceAllocationRequest<'b, 'c>,
+ ty: &'a wasmtime_environ::Memory,
memory_index: Option<DefinedMemoryIndex>,
- ) -> Result<(MemoryAllocationIndex, Memory)> {
+ ) -> Result<
+ Pin<Box<dyn Future<Output = Result<(MemoryAllocationIndex, Memory)>> + Send + 'a>>,
+ OutOfMemory,
+ > {
if cfg!(debug_assertions) {
let module = request.runtime_info.env_module();
let offsets = request.runtime_info.offsets();
@@ -184,15 +188,13 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> {
}
match self.preallocation {
- Some(shared_memory) => Ok((
- MemoryAllocationIndex::default(),
- shared_memory.clone().as_memory(),
- )),
- None => {
- self.ondemand
- .allocate_memory(request, ty, memory_index)
- .await
- }
+ Some(shared_memory) => Ok(Box::into_pin(try_new::<Box<_>>(async move {
+ Ok((
+ MemoryAllocationIndex::default(),
+ shared_memory.clone().as_memory(),
+ ))
+ })?)),
+ None => self.ondemand.allocate_memory(request, ty, memory_index),
}
}
@@ -208,13 +210,16 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> {
}
}
- async fn allocate_table(
- &self,
- req: &mut InstanceAllocationRequest<'_, '_>,
- ty: &wasmtime_environ::Table,
+ fn allocate_table<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ req: &'a mut InstanceAllocationRequest<'b, 'c>,
+ ty: &'a wasmtime_environ::Table,
table_index: DefinedTableIndex,
- ) -> Result<(TableAllocationIndex, Table)> {
- self.ondemand.allocate_table(req, ty, table_index).await
+ ) -> Result<
+ Pin<Box<dyn Future<Output = Result<(TableAllocationIndex, Table)>> + Send + 'a>>,
+ OutOfMemory,
+ > {
+ self.ondemand.allocate_table(req, ty, table_index)
}
unsafe fn deallocate_table(
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
index fd2a52b26e..888fe1ccae 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs
@@ -8,6 +8,8 @@ use crate::runtime::vm::table::Table;
use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo};
use crate::store::{Asyncness, InstanceId, StoreOpaque, StoreResourceLimiter};
use crate::{OpaqueRootScope, Val};
+use core::future::Future;
+use core::pin::Pin;
use core::{mem, ptr};
use wasmtime_environ::{
DefinedMemoryIndex, DefinedTableIndex, HostPtr, InitMemory, MemoryInitialization,
@@ -127,7 +129,6 @@ impl GcHeapAllocationIndex {
///
/// This trait is unsafe as it requires knowledge of Wasmtime's runtime
/// internals to implement correctly.
-#[async_trait::async_trait]
pub unsafe trait InstanceAllocator: Send + Sync {
/// Validate whether a component (including all of its contained core
/// modules) is allocatable by this instance allocator.
@@ -184,12 +185,18 @@ pub unsafe trait InstanceAllocator: Send + Sync {
fn decrement_core_instance_count(&self);
/// Allocate a memory for an instance.
- async fn allocate_memory(
- &self,
- request: &mut InstanceAllocationRequest<'_, '_>,
- ty: &wasmtime_environ::Memory,
+ ///
+ /// Returns `Err(OutOfMemory)` if boxing the future fails. The inner
+ /// `Result` covers other allocation errors (e.g. resource limits).
+ fn allocate_memory<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ request: &'a mut InstanceAllocationRequest<'b, 'c>,
+ ty: &'a wasmtime_environ::Memory,
memory_index: Option<DefinedMemoryIndex>,
- ) -> Result<(MemoryAllocationIndex, Memory)>;
+ ) -> Result<
+ Pin<Box<dyn Future<Output = Result<(MemoryAllocationIndex, Memory)>> + Send + 'a>>,
+ OutOfMemory,
+ >;
/// Deallocate an instance's previously allocated memory.
///
@@ -206,12 +213,18 @@ pub unsafe trait InstanceAllocator: Send + Sync {
);
/// Allocate a table for an instance.
- async fn allocate_table(
- &self,
- req: &mut InstanceAllocationRequest<'_, '_>,
- table: &wasmtime_environ::Table,
+ ///
+ /// Returns `Err(OutOfMemory)` if boxing the future fails. The inner
+ /// `Result` covers other allocation errors (e.g. resource limits).
+ fn allocate_table<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ req: &'a mut InstanceAllocationRequest<'b, 'c>,
+ table: &'a wasmtime_environ::Table,
table_index: DefinedTableIndex,
- ) -> Result<(TableAllocationIndex, Table)>;
+ ) -> Result<
+ Pin<Box<dyn Future<Output = Result<(TableAllocationIndex, Table)>> + Send + 'a>>,
+ OutOfMemory,
+ >;
/// Deallocate an instance's previously allocated table.
///
@@ -391,9 +404,9 @@ impl dyn InstanceAllocator + '_ {
/// Allocate the memories for the given instance allocation request, pushing
/// them into `memories`.
- async fn allocate_memories(
- &self,
- request: &mut InstanceAllocationRequest<'_, '_>,
+ async fn allocate_memories<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ request: &'a mut InstanceAllocationRequest<'b, 'c>,
memories: &mut TryPrimaryMap<DefinedMemoryIndex, (MemoryAllocationIndex, Memory)>,
) -> Result<()> {
let module = request.runtime_info.env_module();
@@ -409,7 +422,7 @@ impl dyn InstanceAllocator + '_ {
.expect("should be a defined memory since we skipped imported ones");
let memory = self
- .allocate_memory(request, ty, Some(memory_index))
+ .allocate_memory(request, ty, Some(memory_index))?
.await?;
memories.push(memory)?;
}
@@ -444,9 +457,9 @@ impl dyn InstanceAllocator + '_ {
/// Allocate tables for the given instance allocation request, pushing them
/// into `tables`.
- async fn allocate_tables(
- &self,
- request: &mut InstanceAllocationRequest<'_, '_>,
+ async fn allocate_tables<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ request: &'a mut InstanceAllocationRequest<'b, 'c>,
tables: &mut TryPrimaryMap<DefinedTableIndex, (TableAllocationIndex, Table)>,
) -> Result<()> {
let module = request.runtime_info.env_module();
@@ -461,7 +474,7 @@ impl dyn InstanceAllocator + '_ {
.defined_table_index(index)
.expect("should be a defined table since we skipped imported ones");
- let table = self.allocate_table(request, table, def_index).await?;
+ let table = self.allocate_table(request, table, def_index)?.await?;
tables.push(table)?;
}
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs
index d3015c70c1..5e9dae24f3 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs
@@ -8,6 +8,8 @@ use crate::runtime::vm::memory::{DefaultMemoryCreator, Memory};
use crate::runtime::vm::mpk::ProtectionKey;
use crate::runtime::vm::table::Table;
use alloc::sync::Arc;
+use core::future::Future;
+use core::pin::Pin;
use wasmtime_environ::{DefinedMemoryIndex, DefinedTableIndex, HostPtr, Module, VMOffsets};
#[cfg(feature = "gc")]
@@ -74,7 +76,6 @@ impl Default for OnDemandInstanceAllocator {
}
}
-#[async_trait::async_trait]
unsafe impl InstanceAllocator for OnDemandInstanceAllocator {
#[cfg(feature = "component-model")]
fn validate_component<'a>(
@@ -109,33 +110,38 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator {
fn decrement_core_instance_count(&self) {}
- async fn allocate_memory(
- &self,
- request: &mut InstanceAllocationRequest<'_, '_>,
- ty: &wasmtime_environ::Memory,
+ fn allocate_memory<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ request: &'a mut InstanceAllocationRequest<'b, 'c>,
+ ty: &'a wasmtime_environ::Memory,
memory_index: Option<DefinedMemoryIndex>,
- ) -> Result<(MemoryAllocationIndex, Memory)> {
+ ) -> Result<
+ Pin<Box<dyn Future<Output = Result<(MemoryAllocationIndex, Memory)>> + Send + 'a>>,
+ OutOfMemory,
+ > {
let creator = self
.mem_creator
.as_deref()
.unwrap_or_else(|| &DefaultMemoryCreator);
- let image = if let Some(memory_index) = memory_index {
- request.runtime_info.memory_image(memory_index)?
- } else {
- None
- };
+ Ok(Box::into_pin(try_new::<Box<_>>(async move {
+ let image = if let Some(memory_index) = memory_index {
+ request.runtime_info.memory_image(memory_index)?
+ } else {
+ None
+ };
- let allocation_index = MemoryAllocationIndex::default();
- let memory = Memory::new_dynamic(
- ty,
- request.store.engine(),
- creator,
- image,
- request.limiter.as_deref_mut(),
- )
- .await?;
- Ok((allocation_index, memory))
+ let allocation_index = MemoryAllocationIndex::default();
+ let memory = Memory::new_dynamic(
+ ty,
+ request.store.engine(),
+ creator,
+ image,
+ request.limiter.as_deref_mut(),
+ )
+ .await?;
+ Ok((allocation_index, memory))
+ })?))
}
unsafe fn deallocate_memory(
@@ -148,20 +154,25 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator {
// Normal destructors do all the necessary clean up.
}
- async fn allocate_table(
- &self,
- request: &mut InstanceAllocationRequest<'_, '_>,
- ty: &wasmtime_environ::Table,
+ fn allocate_table<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ request: &'a mut InstanceAllocationRequest<'b, 'c>,
+ ty: &'a wasmtime_environ::Table,
_table_index: DefinedTableIndex,
- ) -> Result<(TableAllocationIndex, Table)> {
- let allocation_index = TableAllocationIndex::default();
- let table = Table::new_dynamic(
- ty,
- request.store.engine().tunables(),
- request.limiter.as_deref_mut(),
- )
- .await?;
- Ok((allocation_index, table))
+ ) -> Result<
+ Pin<Box<dyn Future<Output = Result<(TableAllocationIndex, Table)>> + Send + 'a>>,
+ OutOfMemory,
+ > {
+ Ok(Box::into_pin(try_new::<Box<_>>(async move {
+ let allocation_index = TableAllocationIndex::default();
+ let table = Table::new_dynamic(
+ ty,
+ request.store.engine().tunables(),
+ request.limiter.as_deref_mut(),
+ )
+ .await?;
+ Ok((allocation_index, table))
+ })?))
}
unsafe fn deallocate_table(
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs
index ba6dd8d5cf..b3ddef64b0 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs
@@ -55,6 +55,8 @@ use crate::runtime::vm::{
mpk::{self, ProtectionKey, ProtectionMask},
sys::vm::PageMap,
};
+use core::future::Future;
+use core::pin::Pin;
use core::sync::atomic::AtomicUsize;
use std::borrow::Cow;
use std::fmt::Display;
@@ -553,7 +555,6 @@ impl PoolingInstanceAllocator {
}
}
-#[async_trait::async_trait]
unsafe impl InstanceAllocator for PoolingInstanceAllocator {
#[cfg(feature = "component-model")]
fn validate_component<'a>(
@@ -678,35 +679,40 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator {
self.live_core_instances.fetch_sub(1, Ordering::AcqRel);
}
- async fn allocate_memory(
- &self,
- request: &mut InstanceAllocationRequest<'_, '_>,
- ty: &wasmtime_environ::Memory,
+ fn allocate_memory<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ request: &'a mut InstanceAllocationRequest<'b, 'c>,
+ ty: &'a wasmtime_environ::Memory,
memory_index: Option<DefinedMemoryIndex>,
- ) -> Result<(MemoryAllocationIndex, Memory)> {
- async {
- // FIXME(rust-lang/rust#145127) this should ideally use a version of
- // `with_flush_and_retry` but adapted for async closures instead of only
- // sync closures. Right now that won't compile though so this is the
- // manually expanded version of the method.
- let e = match self.memories.allocate(request, ty, memory_index).await {
- Ok(result) => return Ok(result),
- Err(e) => e,
- };
-
- if e.is::<PoolConcurrencyLimitError>() {
- let queue = self.decommit_queue.lock().unwrap();
- if self.flush_decommit_queue(queue) {
- return self.memories.allocate(request, ty, memory_index).await;
+ ) -> Result<
+ Pin<Box<dyn Future<Output = Result<(MemoryAllocationIndex, Memory)>> + Send + 'a>>,
+ OutOfMemory,
+ > {
+ Ok(Box::into_pin(try_new::<Box<_>>(async move {
+ async {
+ // FIXME(rust-lang/rust#145127) this should ideally use a version of
+ // `with_flush_and_retry` but adapted for async closures instead of only
+ // sync closures. Right now that won't compile though so this is the
+ // manually expanded version of the method.
+ let e = match self.memories.allocate(request, ty, memory_index).await {
+ Ok(result) => return Ok(result),
+ Err(e) => e,
+ };
+
+ if e.is::<PoolConcurrencyLimitError>() {
+ let queue = self.decommit_queue.lock().unwrap();
+ if self.flush_decommit_queue(queue) {
+ return self.memories.allocate(request, ty, memory_index).await;
+ }
}
- }
- Err(e)
- }
- .await
- .inspect(|_| {
- self.live_memories.fetch_add(1, Ordering::Relaxed);
- })
+ Err(e)
+ }
+ .await
+ .inspect(|_| {
+ self.live_memories.fetch_add(1, Ordering::Relaxed);
+ })
+ })?))
}
unsafe fn deallocate_memory(
@@ -747,33 +753,38 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator {
self.merge_or_flush(queue);
}
- async fn allocate_table(
- &self,
- request: &mut InstanceAllocationRequest<'_, '_>,
- ty: &wasmtime_environ::Table,
+ fn allocate_table<'a, 'b: 'a, 'c: 'a>(
+ &'a self,
+ request: &'a mut InstanceAllocationRequest<'b, 'c>,
+ ty: &'a wasmtime_environ::Table,
_table_index: DefinedTableIndex,
- ) -> Result<(super::TableAllocationIndex, Table)> {
- async {
- // FIXME: see `allocate_memory` above for comments about duplication
- // with `with_flush_and_retry`.
- let e = match self.tables.allocate(request, ty).await {
- Ok(result) => return Ok(result),
- Err(e) => e,
- };
-
- if e.is::<PoolConcurrencyLimitError>() {
- let queue = self.decommit_queue.lock().unwrap();
- if self.flush_decommit_queue(queue) {
- return self.tables.allocate(request, ty).await;
+ ) -> Result<
+ Pin<Box<dyn Future<Output = Result<(super::TableAllocationIndex, Table)>> + Send + 'a>>,
+ OutOfMemory,
+ > {
+ Ok(Box::into_pin(try_new::<Box<_>>(async move {
+ async {
+ // FIXME: see `allocate_memory` above for comments about duplication
+ // with `with_flush_and_retry`.
+ let e = match self.tables.allocate(request, ty).await {
+ Ok(result) => return Ok(result),
+ Err(e) => e,
+ };
+
+ if e.is::<PoolConcurrencyLimitError>() {
+ let queue = self.decommit_queue.lock().unwrap();
+ if self.flush_decommit_queue(queue) {
+ return self.tables.allocate(request, ty).await;
+ }
}
- }
- Err(e)
- }
- .await
- .inspect(|_| {
- self.live_tables.fetch_add(1, Ordering::Relaxed);
- })
+ Err(e)
+ }
+ .await
+ .inspect(|_| {
+ self.live_tables.fetch_add(1, Ordering::Relaxed);
+ })
+ })?))
}
unsafe fn deallocate_table(