Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Refactor and document `release_resident_unused_memory` a bit (#14507)

Alex Crichton56f25ee84245
diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs
index 649713fd48..150ffcb0f0 100644
--- a/crates/wasmtime/src/engine.rs
+++ b/crates/wasmtime/src/engine.rs
@@ -821,6 +821,12 @@ impl Engine {
/// slots. The tradeoff is that the next instantiation of any given module
/// may be slower, because no cached memory mappings are present anymore.
///
+ /// Note that while this method is running the warm slots it is releasing
+ /// are unavailable for allocation. Instantiations running concurrently
+ /// with this method may therefore fail with a "maximum concurrent limit
+ /// reached" error, even if few instances are live, when the pool's other
+ /// free slots are exhausted.
+ ///
/// Returns 0 if this engine is not using the pooling allocator, or if
/// nothing was resident to release.
///
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/index_allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/index_allocator.rs
index bc3833b6fc..5497f13078 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/index_allocator.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/index_allocator.rs
@@ -207,6 +207,7 @@ struct Unused {
enum AllocMode {
ForceAffineAndClear,
AnySlot,
+ AnyWarm,
}
/// The division of an allocator's slot space and unused-warm-slot budget
@@ -359,7 +360,8 @@ impl ModuleAffinityIndexAllocator {
// stays on its own shard.
for shard in super::shard_ids_from_home(self.shards.len()) {
let mut inner = self.shard(shard).lock().unwrap();
- if let Some(local) = Self::alloc_within(&mut inner, for_memory, AllocMode::AnySlot) {
+ if let Some((local, _)) = Self::alloc_within(&mut inner, for_memory, AllocMode::AnySlot)
+ {
return Some(self.global_id(shard, local));
}
}
@@ -382,7 +384,7 @@ impl ModuleAffinityIndexAllocator {
// them all. This is a module-teardown path, not a hot path.
for shard in self.shard_ids() {
let mut inner = self.shard(shard).lock().unwrap();
- if let Some(local) = Self::alloc_within(
+ if let Some((local, _)) = Self::alloc_within(
&mut inner,
Some(MemoryInModule(module_id, memory_index)),
AllocMode::ForceAffineAndClear,
@@ -393,11 +395,13 @@ impl ModuleAffinityIndexAllocator {
None
}
+ /// Allocates a slot within `inner`, returning the slot along with the
+ /// number of bytes it had resident if it was previously a warm slot.
fn alloc_within(
inner: &mut Inner,
for_memory: Option<MemoryInModule>,
mode: AllocMode,
- ) -> Option<SlotId> {
+ ) -> Option<(SlotId, usize)> {
// As a first-pass always attempt an affine allocation. This will
// succeed if any slots are considered affine to `module_id` (if it's
// specified). Failing that something else is attempted to be chosen.
@@ -439,19 +443,28 @@ impl ModuleAffinityIndexAllocator {
// `module_id` during module teardown. This means that there's
// no consulting non-affine slots in this path.
AllocMode::ForceAffineAndClear => None,
+
+ // In this mode only warm slots are taken.
+ AllocMode::AnyWarm => inner.pick_warm(),
}
})?;
let slot = &mut inner.slot_state[slot_id.index()];
- if let SlotState::UnusedWarm(Unused { bytes_resident, .. }) = slot {
- inner.unused_bytes_resident -= *bytes_resident;
- }
+ let (affinity, bytes_resident) = match slot {
+ SlotState::UnusedWarm(u) => (u.affinity, u.bytes_resident),
+ _ => (None, 0),
+ };
+ inner.unused_bytes_resident -= bytes_resident;
*slot = SlotState::Used(match mode {
AllocMode::ForceAffineAndClear => None,
AllocMode::AnySlot => for_memory,
+ AllocMode::AnyWarm => {
+ debug_assert!(for_memory.is_none());
+ affinity
+ }
});
- Some(slot_id)
+ Some((slot_id, bytes_resident))
}
pub(crate) fn free(&self, index: SlotId, bytes_resident: usize) {
@@ -579,8 +592,8 @@ impl ModuleAffinityIndexAllocator {
.sum()
}
- /// Takes every warm slot that still has resident memory out of the free
- /// lists, returning each slot and the number of bytes it has resident.
+ /// Takes every warm slot out of the free lists, returning each slot and
+ /// the number of bytes it has resident (which may be zero).
///
/// The slots are marked used so that nothing can allocate them while
/// their memory is being released; the caller frees each one again
@@ -592,31 +605,9 @@ impl ModuleAffinityIndexAllocator {
let mut taken = Vec::new();
for (shard_index, shard) in self.shards.iter().enumerate() {
let mut inner = shard.0.lock().unwrap();
-
- // Walk the warm list first and only then remove, because
- // `remove` unlinks the very entries this walk is following.
- let mut resident = Vec::new();
- let mut next = inner.warm.head;
- while let Some(slot) = next {
- let unused = match &inner.slot_state[slot.index()] {
- SlotState::UnusedWarm(u) => *u,
- // The warm list only contains warm slots.
- _ => unreachable!(),
- };
- next = unused.unused_list_link.next;
- if unused.bytes_resident > 0 {
- resident.push((slot, unused.bytes_resident));
- }
- }
-
- for (slot, bytes_resident) in resident {
- let affinity = inner.slot_state[slot.index()].unwrap_unused().affinity;
- inner.remove(slot);
- inner.unused_bytes_resident -= bytes_resident;
- // Keep the affinity: `free` reads it back out of this payload
- // to re-file the slot on its module's affine list, so
- // `Used(None)` here would un-affine every slot touched.
- inner.slot_state[slot.index()] = SlotState::Used(affinity);
+ while let Some((slot, bytes_resident)) =
+ Self::alloc_within(&mut inner, None, AllocMode::AnyWarm)
+ {
taken.push((
self.global_id(ShardId::from_index(shard_index), slot),
bytes_resident,
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
index 00a4217af8..ca1425c63d 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
@@ -601,11 +601,14 @@ impl MemoryPool {
let mut iov = Vec::with_capacity(taken.len());
for (id, bytes_resident) in &taken {
+ if *bytes_resident == 0 {
+ continue;
+ }
let index = StripedAllocationIndex(id.0)
.as_unstriped_slot_index(stripe_index, self.stripes.len());
iov.push(iovec {
iov_base: self.get_base(index).as_mut_ptr().cast(),
- iov_len: *bytes_resident,
+ iov_len: self.layout.max_memory_bytes.byte_count(),
});
}