A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Refactor and document `release_resident_unused_memory` a bit (#14507)
Alex Crichton56f25ee84245
diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rsindex 649713fd48..150ffcb0f0 100644--- a/crates/wasmtime/src/engine.rs+++ b/crates/wasmtime/src/engine.rs@@ -821,6 +821,12 @@ impl Engine {/// slots. The tradeoff is that the next instantiation of any given module/// may be slower, because no cached memory mappings are present anymore.///+ /// Note that while this method is running the warm slots it is releasing+ /// are unavailable for allocation. Instantiations running concurrently+ /// with this method may therefore fail with a "maximum concurrent limit+ /// reached" error, even if few instances are live, when the pool's other+ /// free slots are exhausted.+ ////// Returns 0 if this engine is not using the pooling allocator, or if/// nothing was resident to release.///diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/index_allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/index_allocator.rsindex bc3833b6fc..5497f13078 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/index_allocator.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/index_allocator.rs@@ -207,6 +207,7 @@ struct Unused {enum AllocMode {ForceAffineAndClear,AnySlot,+ AnyWarm,}/// The division of an allocator's slot space and unused-warm-slot budget@@ -359,7 +360,8 @@ impl ModuleAffinityIndexAllocator {// stays on its own shard.for shard in super::shard_ids_from_home(self.shards.len()) {let mut inner = self.shard(shard).lock().unwrap();- if let Some(local) = Self::alloc_within(&mut inner, for_memory, AllocMode::AnySlot) {+ if let Some((local, _)) = Self::alloc_within(&mut inner, for_memory, AllocMode::AnySlot)+ {return Some(self.global_id(shard, local));}}@@ -382,7 +384,7 @@ impl ModuleAffinityIndexAllocator {// them all. This is a module-teardown path, not a hot path.for shard in self.shard_ids() {let mut inner = self.shard(shard).lock().unwrap();- if let Some(local) = Self::alloc_within(+ if let Some((local, _)) = Self::alloc_within(&mut inner,Some(MemoryInModule(module_id, memory_index)),AllocMode::ForceAffineAndClear,@@ -393,11 +395,13 @@ impl ModuleAffinityIndexAllocator {None}+ /// Allocates a slot within `inner`, returning the slot along with the+ /// number of bytes it had resident if it was previously a warm slot.fn alloc_within(inner: &mut Inner,for_memory: Option<MemoryInModule>,mode: AllocMode,- ) -> Option<SlotId> {+ ) -> Option<(SlotId, usize)> {// As a first-pass always attempt an affine allocation. This will// succeed if any slots are considered affine to `module_id` (if it's// specified). Failing that something else is attempted to be chosen.@@ -439,19 +443,28 @@ impl ModuleAffinityIndexAllocator {// `module_id` during module teardown. This means that there's// no consulting non-affine slots in this path.AllocMode::ForceAffineAndClear => None,++ // In this mode only warm slots are taken.+ AllocMode::AnyWarm => inner.pick_warm(),}})?;let slot = &mut inner.slot_state[slot_id.index()];- if let SlotState::UnusedWarm(Unused { bytes_resident, .. }) = slot {- inner.unused_bytes_resident -= *bytes_resident;- }+ let (affinity, bytes_resident) = match slot {+ SlotState::UnusedWarm(u) => (u.affinity, u.bytes_resident),+ _ => (None, 0),+ };+ inner.unused_bytes_resident -= bytes_resident;*slot = SlotState::Used(match mode {AllocMode::ForceAffineAndClear => None,AllocMode::AnySlot => for_memory,+ AllocMode::AnyWarm => {+ debug_assert!(for_memory.is_none());+ affinity+ }});- Some(slot_id)+ Some((slot_id, bytes_resident))}pub(crate) fn free(&self, index: SlotId, bytes_resident: usize) {@@ -579,8 +592,8 @@ impl ModuleAffinityIndexAllocator {.sum()}- /// Takes every warm slot that still has resident memory out of the free- /// lists, returning each slot and the number of bytes it has resident.+ /// Takes every warm slot out of the free lists, returning each slot and+ /// the number of bytes it has resident (which may be zero).////// The slots are marked used so that nothing can allocate them while/// their memory is being released; the caller frees each one again@@ -592,31 +605,9 @@ impl ModuleAffinityIndexAllocator {let mut taken = Vec::new();for (shard_index, shard) in self.shards.iter().enumerate() {let mut inner = shard.0.lock().unwrap();-- // Walk the warm list first and only then remove, because- // `remove` unlinks the very entries this walk is following.- let mut resident = Vec::new();- let mut next = inner.warm.head;- while let Some(slot) = next {- let unused = match &inner.slot_state[slot.index()] {- SlotState::UnusedWarm(u) => *u,- // The warm list only contains warm slots.- _ => unreachable!(),- };- next = unused.unused_list_link.next;- if unused.bytes_resident > 0 {- resident.push((slot, unused.bytes_resident));- }- }-- for (slot, bytes_resident) in resident {- let affinity = inner.slot_state[slot.index()].unwrap_unused().affinity;- inner.remove(slot);- inner.unused_bytes_resident -= bytes_resident;- // Keep the affinity: `free` reads it back out of this payload- // to re-file the slot on its module's affine list, so- // `Used(None)` here would un-affine every slot touched.- inner.slot_state[slot.index()] = SlotState::Used(affinity);+ while let Some((slot, bytes_resident)) =+ Self::alloc_within(&mut inner, None, AllocMode::AnyWarm)+ {taken.push((self.global_id(ShardId::from_index(shard_index), slot),bytes_resident,diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rsindex 00a4217af8..ca1425c63d 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs@@ -601,11 +601,14 @@ impl MemoryPool {let mut iov = Vec::with_capacity(taken.len());for (id, bytes_resident) in &taken {+ if *bytes_resident == 0 {+ continue;+ }let index = StripedAllocationIndex(id.0).as_unstriped_slot_index(stripe_index, self.stripes.len());iov.push(iovec {iov_base: self.get_base(index).as_mut_ptr().cast(),- iov_len: *bytes_resident,+ iov_len: self.layout.max_memory_bytes.byte_count(),});}