A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Use explicit boxing for InstanceAllocator async methods (#12849)
Nick Fitzgerald82ebbd5d4ef6
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 4733dc6393..b8ea6a175b 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -1929,7 +1929,7 @@ impl StoreOpaque {let (mem_alloc_index, mem) = engine.allocator()- .allocate_memory(&mut request, &mem_ty, None)+ .allocate_memory(&mut request, &mem_ty, None)?.await?;// Then, allocate the actual GC heap, passing in that memorydiff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rsindex 175e77b137..6d606f55e8 100644--- a/crates/wasmtime/src/runtime/trampoline/memory.rs+++ b/crates/wasmtime/src/runtime/trampoline/memory.rs@@ -9,6 +9,8 @@ use crate::runtime::vm::{};use crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque, StoreResourceLimiter};use alloc::sync::Arc;+use core::future::Future;+use core::pin::Pin;use wasmtime_environ::{DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, Module, StaticModuleIndex,Tunables, VMOffsets,@@ -126,7 +128,6 @@ struct SingleMemoryInstance<'a> {ondemand: OnDemandInstanceAllocator,}-#[async_trait::async_trait]unsafe impl InstanceAllocator for SingleMemoryInstance<'_> {#[cfg(feature = "component-model")]fn validate_component<'a>(@@ -170,12 +171,15 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> {self.ondemand.decrement_core_instance_count();}- async fn allocate_memory(- &self,- request: &mut InstanceAllocationRequest<'_, '_>,- ty: &wasmtime_environ::Memory,+ fn allocate_memory<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ request: &'a mut InstanceAllocationRequest<'b, 'c>,+ ty: &'a wasmtime_environ::Memory,memory_index: Option<DefinedMemoryIndex>,- ) -> Result<(MemoryAllocationIndex, Memory)> {+ ) -> Result<+ Pin<Box<dyn Future<Output = Result<(MemoryAllocationIndex, Memory)>> + Send + 'a>>,+ OutOfMemory,+ > {if cfg!(debug_assertions) {let module = request.runtime_info.env_module();let offsets = request.runtime_info.offsets();@@ -184,15 +188,13 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> {}match self.preallocation {- Some(shared_memory) => Ok((- MemoryAllocationIndex::default(),- shared_memory.clone().as_memory(),- )),- None => {- self.ondemand- .allocate_memory(request, ty, memory_index)- .await- }+ Some(shared_memory) => Ok(Box::into_pin(try_new::<Box<_>>(async move {+ Ok((+ MemoryAllocationIndex::default(),+ shared_memory.clone().as_memory(),+ ))+ })?)),+ None => self.ondemand.allocate_memory(request, ty, memory_index),}}@@ -208,13 +210,16 @@ unsafe impl InstanceAllocator for SingleMemoryInstance<'_> {}}- async fn allocate_table(- &self,- req: &mut InstanceAllocationRequest<'_, '_>,- ty: &wasmtime_environ::Table,+ fn allocate_table<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ req: &'a mut InstanceAllocationRequest<'b, 'c>,+ ty: &'a wasmtime_environ::Table,table_index: DefinedTableIndex,- ) -> Result<(TableAllocationIndex, Table)> {- self.ondemand.allocate_table(req, ty, table_index).await+ ) -> Result<+ Pin<Box<dyn Future<Output = Result<(TableAllocationIndex, Table)>> + Send + 'a>>,+ OutOfMemory,+ > {+ self.ondemand.allocate_table(req, ty, table_index)}unsafe fn deallocate_table(diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rsindex fd2a52b26e..888fe1ccae 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs@@ -8,6 +8,8 @@ use crate::runtime::vm::table::Table;use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo};use crate::store::{Asyncness, InstanceId, StoreOpaque, StoreResourceLimiter};use crate::{OpaqueRootScope, Val};+use core::future::Future;+use core::pin::Pin;use core::{mem, ptr};use wasmtime_environ::{DefinedMemoryIndex, DefinedTableIndex, HostPtr, InitMemory, MemoryInitialization,@@ -127,7 +129,6 @@ impl GcHeapAllocationIndex {////// This trait is unsafe as it requires knowledge of Wasmtime's runtime/// internals to implement correctly.-#[async_trait::async_trait]pub unsafe trait InstanceAllocator: Send + Sync {/// Validate whether a component (including all of its contained core/// modules) is allocatable by this instance allocator.@@ -184,12 +185,18 @@ pub unsafe trait InstanceAllocator: Send + Sync {fn decrement_core_instance_count(&self);/// Allocate a memory for an instance.- async fn allocate_memory(- &self,- request: &mut InstanceAllocationRequest<'_, '_>,- ty: &wasmtime_environ::Memory,+ ///+ /// Returns `Err(OutOfMemory)` if boxing the future fails. The inner+ /// `Result` covers other allocation errors (e.g. resource limits).+ fn allocate_memory<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ request: &'a mut InstanceAllocationRequest<'b, 'c>,+ ty: &'a wasmtime_environ::Memory,memory_index: Option<DefinedMemoryIndex>,- ) -> Result<(MemoryAllocationIndex, Memory)>;+ ) -> Result<+ Pin<Box<dyn Future<Output = Result<(MemoryAllocationIndex, Memory)>> + Send + 'a>>,+ OutOfMemory,+ >;/// Deallocate an instance's previously allocated memory.///@@ -206,12 +213,18 @@ pub unsafe trait InstanceAllocator: Send + Sync {);/// Allocate a table for an instance.- async fn allocate_table(- &self,- req: &mut InstanceAllocationRequest<'_, '_>,- table: &wasmtime_environ::Table,+ ///+ /// Returns `Err(OutOfMemory)` if boxing the future fails. The inner+ /// `Result` covers other allocation errors (e.g. resource limits).+ fn allocate_table<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ req: &'a mut InstanceAllocationRequest<'b, 'c>,+ table: &'a wasmtime_environ::Table,table_index: DefinedTableIndex,- ) -> Result<(TableAllocationIndex, Table)>;+ ) -> Result<+ Pin<Box<dyn Future<Output = Result<(TableAllocationIndex, Table)>> + Send + 'a>>,+ OutOfMemory,+ >;/// Deallocate an instance's previously allocated table.///@@ -391,9 +404,9 @@ impl dyn InstanceAllocator + '_ {/// Allocate the memories for the given instance allocation request, pushing/// them into `memories`.- async fn allocate_memories(- &self,- request: &mut InstanceAllocationRequest<'_, '_>,+ async fn allocate_memories<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ request: &'a mut InstanceAllocationRequest<'b, 'c>,memories: &mut TryPrimaryMap<DefinedMemoryIndex, (MemoryAllocationIndex, Memory)>,) -> Result<()> {let module = request.runtime_info.env_module();@@ -409,7 +422,7 @@ impl dyn InstanceAllocator + '_ {.expect("should be a defined memory since we skipped imported ones");let memory = self- .allocate_memory(request, ty, Some(memory_index))+ .allocate_memory(request, ty, Some(memory_index))?.await?;memories.push(memory)?;}@@ -444,9 +457,9 @@ impl dyn InstanceAllocator + '_ {/// Allocate tables for the given instance allocation request, pushing them/// into `tables`.- async fn allocate_tables(- &self,- request: &mut InstanceAllocationRequest<'_, '_>,+ async fn allocate_tables<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ request: &'a mut InstanceAllocationRequest<'b, 'c>,tables: &mut TryPrimaryMap<DefinedTableIndex, (TableAllocationIndex, Table)>,) -> Result<()> {let module = request.runtime_info.env_module();@@ -461,7 +474,7 @@ impl dyn InstanceAllocator + '_ {.defined_table_index(index).expect("should be a defined table since we skipped imported ones");- let table = self.allocate_table(request, table, def_index).await?;+ let table = self.allocate_table(request, table, def_index)?.await?;tables.push(table)?;}diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rsindex d3015c70c1..5e9dae24f3 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/on_demand.rs@@ -8,6 +8,8 @@ use crate::runtime::vm::memory::{DefaultMemoryCreator, Memory};use crate::runtime::vm::mpk::ProtectionKey;use crate::runtime::vm::table::Table;use alloc::sync::Arc;+use core::future::Future;+use core::pin::Pin;use wasmtime_environ::{DefinedMemoryIndex, DefinedTableIndex, HostPtr, Module, VMOffsets};#[cfg(feature = "gc")]@@ -74,7 +76,6 @@ impl Default for OnDemandInstanceAllocator {}}-#[async_trait::async_trait]unsafe impl InstanceAllocator for OnDemandInstanceAllocator {#[cfg(feature = "component-model")]fn validate_component<'a>(@@ -109,33 +110,38 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator {fn decrement_core_instance_count(&self) {}- async fn allocate_memory(- &self,- request: &mut InstanceAllocationRequest<'_, '_>,- ty: &wasmtime_environ::Memory,+ fn allocate_memory<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ request: &'a mut InstanceAllocationRequest<'b, 'c>,+ ty: &'a wasmtime_environ::Memory,memory_index: Option<DefinedMemoryIndex>,- ) -> Result<(MemoryAllocationIndex, Memory)> {+ ) -> Result<+ Pin<Box<dyn Future<Output = Result<(MemoryAllocationIndex, Memory)>> + Send + 'a>>,+ OutOfMemory,+ > {let creator = self.mem_creator.as_deref().unwrap_or_else(|| &DefaultMemoryCreator);- let image = if let Some(memory_index) = memory_index {- request.runtime_info.memory_image(memory_index)?- } else {- None- };+ Ok(Box::into_pin(try_new::<Box<_>>(async move {+ let image = if let Some(memory_index) = memory_index {+ request.runtime_info.memory_image(memory_index)?+ } else {+ None+ };- let allocation_index = MemoryAllocationIndex::default();- let memory = Memory::new_dynamic(- ty,- request.store.engine(),- creator,- image,- request.limiter.as_deref_mut(),- )- .await?;- Ok((allocation_index, memory))+ let allocation_index = MemoryAllocationIndex::default();+ let memory = Memory::new_dynamic(+ ty,+ request.store.engine(),+ creator,+ image,+ request.limiter.as_deref_mut(),+ )+ .await?;+ Ok((allocation_index, memory))+ })?))}unsafe fn deallocate_memory(@@ -148,20 +154,25 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator {// Normal destructors do all the necessary clean up.}- async fn allocate_table(- &self,- request: &mut InstanceAllocationRequest<'_, '_>,- ty: &wasmtime_environ::Table,+ fn allocate_table<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ request: &'a mut InstanceAllocationRequest<'b, 'c>,+ ty: &'a wasmtime_environ::Table,_table_index: DefinedTableIndex,- ) -> Result<(TableAllocationIndex, Table)> {- let allocation_index = TableAllocationIndex::default();- let table = Table::new_dynamic(- ty,- request.store.engine().tunables(),- request.limiter.as_deref_mut(),- )- .await?;- Ok((allocation_index, table))+ ) -> Result<+ Pin<Box<dyn Future<Output = Result<(TableAllocationIndex, Table)>> + Send + 'a>>,+ OutOfMemory,+ > {+ Ok(Box::into_pin(try_new::<Box<_>>(async move {+ let allocation_index = TableAllocationIndex::default();+ let table = Table::new_dynamic(+ ty,+ request.store.engine().tunables(),+ request.limiter.as_deref_mut(),+ )+ .await?;+ Ok((allocation_index, table))+ })?))}unsafe fn deallocate_table(diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rsindex ba6dd8d5cf..b3ddef64b0 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs@@ -55,6 +55,8 @@ use crate::runtime::vm::{mpk::{self, ProtectionKey, ProtectionMask},sys::vm::PageMap,};+use core::future::Future;+use core::pin::Pin;use core::sync::atomic::AtomicUsize;use std::borrow::Cow;use std::fmt::Display;@@ -553,7 +555,6 @@ impl PoolingInstanceAllocator {}}-#[async_trait::async_trait]unsafe impl InstanceAllocator for PoolingInstanceAllocator {#[cfg(feature = "component-model")]fn validate_component<'a>(@@ -678,35 +679,40 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator {self.live_core_instances.fetch_sub(1, Ordering::AcqRel);}- async fn allocate_memory(- &self,- request: &mut InstanceAllocationRequest<'_, '_>,- ty: &wasmtime_environ::Memory,+ fn allocate_memory<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ request: &'a mut InstanceAllocationRequest<'b, 'c>,+ ty: &'a wasmtime_environ::Memory,memory_index: Option<DefinedMemoryIndex>,- ) -> Result<(MemoryAllocationIndex, Memory)> {- async {- // FIXME(rust-lang/rust#145127) this should ideally use a version of- // `with_flush_and_retry` but adapted for async closures instead of only- // sync closures. Right now that won't compile though so this is the- // manually expanded version of the method.- let e = match self.memories.allocate(request, ty, memory_index).await {- Ok(result) => return Ok(result),- Err(e) => e,- };-- if e.is::<PoolConcurrencyLimitError>() {- let queue = self.decommit_queue.lock().unwrap();- if self.flush_decommit_queue(queue) {- return self.memories.allocate(request, ty, memory_index).await;+ ) -> Result<+ Pin<Box<dyn Future<Output = Result<(MemoryAllocationIndex, Memory)>> + Send + 'a>>,+ OutOfMemory,+ > {+ Ok(Box::into_pin(try_new::<Box<_>>(async move {+ async {+ // FIXME(rust-lang/rust#145127) this should ideally use a version of+ // `with_flush_and_retry` but adapted for async closures instead of only+ // sync closures. Right now that won't compile though so this is the+ // manually expanded version of the method.+ let e = match self.memories.allocate(request, ty, memory_index).await {+ Ok(result) => return Ok(result),+ Err(e) => e,+ };++ if e.is::<PoolConcurrencyLimitError>() {+ let queue = self.decommit_queue.lock().unwrap();+ if self.flush_decommit_queue(queue) {+ return self.memories.allocate(request, ty, memory_index).await;+ }}- }- Err(e)- }- .await- .inspect(|_| {- self.live_memories.fetch_add(1, Ordering::Relaxed);- })+ Err(e)+ }+ .await+ .inspect(|_| {+ self.live_memories.fetch_add(1, Ordering::Relaxed);+ })+ })?))}unsafe fn deallocate_memory(@@ -747,33 +753,38 @@ unsafe impl InstanceAllocator for PoolingInstanceAllocator {self.merge_or_flush(queue);}- async fn allocate_table(- &self,- request: &mut InstanceAllocationRequest<'_, '_>,- ty: &wasmtime_environ::Table,+ fn allocate_table<'a, 'b: 'a, 'c: 'a>(+ &'a self,+ request: &'a mut InstanceAllocationRequest<'b, 'c>,+ ty: &'a wasmtime_environ::Table,_table_index: DefinedTableIndex,- ) -> Result<(super::TableAllocationIndex, Table)> {- async {- // FIXME: see `allocate_memory` above for comments about duplication- // with `with_flush_and_retry`.- let e = match self.tables.allocate(request, ty).await {- Ok(result) => return Ok(result),- Err(e) => e,- };-- if e.is::<PoolConcurrencyLimitError>() {- let queue = self.decommit_queue.lock().unwrap();- if self.flush_decommit_queue(queue) {- return self.tables.allocate(request, ty).await;+ ) -> Result<+ Pin<Box<dyn Future<Output = Result<(super::TableAllocationIndex, Table)>> + Send + 'a>>,+ OutOfMemory,+ > {+ Ok(Box::into_pin(try_new::<Box<_>>(async move {+ async {+ // FIXME: see `allocate_memory` above for comments about duplication+ // with `with_flush_and_retry`.+ let e = match self.tables.allocate(request, ty).await {+ Ok(result) => return Ok(result),+ Err(e) => e,+ };++ if e.is::<PoolConcurrencyLimitError>() {+ let queue = self.decommit_queue.lock().unwrap();+ if self.flush_decommit_queue(queue) {+ return self.tables.allocate(request, ty).await;+ }}- }- Err(e)- }- .await- .inspect(|_| {- self.live_tables.fetch_add(1, Ordering::Relaxed);- })+ Err(e)+ }+ .await+ .inspect(|_| {+ self.live_tables.fetch_add(1, Ordering::Relaxed);+ })+ })?))}unsafe fn deallocate_table(