A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Bucket allocations by size in the free list (#12987)
Nick Fitzgeraldc26d8c7aa622
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rsindex d21d9026e4..604b7fcbda 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs@@ -1,5 +1,5 @@use crate::prelude::*;-use alloc::collections::BTreeMap;+use alloc::collections::{BTreeMap, BTreeSet, btree_map::Entry};use core::{alloc::Layout, num::NonZeroU32};/// A free list for use by our garbage collectors, using a sorted Vec of@@ -23,12 +23,22 @@ pub(crate) struct FreeList {/// even though it would have enough capacity for many allocations after/// enough iterations of the loop.capacity: usize,+/// Our free blocks, as a map from index to length of the free block at that/// index.free_block_index_to_len: BTreeMap<u32, u32>,++ /// A map from a block length to the set of free block indices for that+ /// length.+ ///+ /// This is necessary to avoid `O(n)` search for a block of a certain size+ /// in the worst case.+ free_block_len_to_indices: BTreeMap<u32, BTreeSet<u32>>,+/// Bump allocator: current position in the active free block./// Allocations bump this forward. When exhausted, refilled from blocks.bump_ptr: u32,+/// End of the current bump allocation region.bump_end: u32,}@@ -66,6 +76,7 @@ impl FreeList {let mut free_list = FreeList {capacity,free_block_index_to_len: BTreeMap::new(),+ free_block_len_to_indices: BTreeMap::new(),bump_ptr: 0,bump_end: 0,};@@ -257,11 +268,25 @@ impl FreeList {}// Find a block big enough.- let (&block_index, &block_len) = self- .free_block_index_to_len- .iter()- .find(|(_, len)| **len >= alloc_size)?;- self.free_block_index_to_len.remove(&block_index);+ let (block_index, block_len) = self+ .free_block_len_to_indices+ .range_mut(alloc_size..)+ .find_map(|(&block_len, indices)| {+ debug_assert!(block_len >= alloc_size);+ debug_assert_eq!(block_len % ALIGN_U32, 0);+ let block_index = indices.pop_first()?;+ Some((block_index, block_len))+ })?;++ let Entry::Occupied(entry) = self.free_block_len_to_indices.entry(block_len) else {+ unreachable!()+ };+ if entry.get().is_empty() {+ entry.remove();+ }++ let block_len2 = self.free_block_index_to_len.remove(&block_index);+ debug_assert_eq!(block_len, block_len2.unwrap());debug_assert_eq!(block_index % ALIGN_U32, 0);debug_assert_eq!(block_len % ALIGN_U32, 0);@@ -306,10 +331,15 @@ impl FreeList {// Also check if the last block in the list is now contiguous with// the extended bump region.- if let Some((&bi, &bl)) = self.free_block_index_to_len.last_key_value() {- if bi + bl == self.bump_ptr {- self.bump_ptr = bi;- self.free_block_index_to_len.pop_last();+ if let Some((&block_index, &block_len)) = self.free_block_index_to_len.last_key_value()+ {+ if block_index + block_len == self.bump_ptr {+ self.bump_ptr = block_index;++ let last = self.free_block_index_to_len.pop_last();+ debug_assert_eq!((block_index, block_len), last.unwrap());++ self.remove_from_block_len_to_index(block_index, block_len);}}@@ -324,6 +354,7 @@ impl FreeList {// Also check if the first block above the bump region is now// contiguous.if let Some(block_len) = self.free_block_index_to_len.remove(&self.bump_end) {+ self.remove_from_block_len_to_index(self.bump_end, block_len);self.bump_end += block_len;}@@ -358,10 +389,20 @@ impl FreeList {index_end = index + alloc_size,next_end = next_index + next_len,);- self.free_block_index_to_len.remove(&next_index);++ let next_len2 = self.free_block_index_to_len.remove(&next_index);+ debug_assert_eq!(next_len, next_len2.unwrap());+ self.remove_from_block_len_to_index(next_index, next_len);+let merged_block_len = next_index + next_len - prev_index;debug_assert_eq!(merged_block_len % ALIGN_U32, 0);*self.free_block_index_to_len.get_mut(&prev_index).unwrap() = merged_block_len;++ self.remove_from_block_len_to_index(prev_index, prev_len);+ self.free_block_len_to_indices+ .entry(merged_block_len)+ .or_default()+ .insert(prev_index);}// The prev and this blocks are contiguous: merge this into prev.@@ -373,9 +414,16 @@ impl FreeList {prev_end = prev_index + prev_len,index_end = index + alloc_size,);+let merged_block_len = index + alloc_size - prev_index;debug_assert_eq!(merged_block_len % ALIGN_U32, 0);*self.free_block_index_to_len.get_mut(&prev_index).unwrap() = merged_block_len;++ self.remove_from_block_len_to_index(prev_index, prev_len);+ self.free_block_len_to_indices+ .entry(merged_block_len)+ .or_default()+ .insert(prev_index);}// The this and next blocks are contiguous: merge next into this.@@ -387,10 +435,18 @@ impl FreeList {index_end = index + alloc_size,next_end = next_index + next_len,);- self.free_block_index_to_len.remove(&next_index);++ let next_len2 = self.free_block_index_to_len.remove(&next_index);+ debug_assert_eq!(next_len, next_len2.unwrap());+ self.remove_from_block_len_to_index(next_index, next_len);+let merged_block_len = next_index + next_len - index;debug_assert_eq!(merged_block_len % ALIGN_U32, 0);self.free_block_index_to_len.insert(index, merged_block_len);+ self.free_block_len_to_indices+ .entry(merged_block_len)+ .or_default()+ .insert(index);}// None of the blocks are contiguous: insert this block into the@@ -398,6 +454,10 @@ impl FreeList {(_, _) => {log::trace!("cannot merge blocks");self.free_block_index_to_len.insert(index, alloc_size);+ self.free_block_len_to_indices+ .entry(alloc_size)+ .or_default()+ .insert(index);}}@@ -406,7 +466,11 @@ impl FreeList {if let Some((&block_index, &block_len)) = self.free_block_index_to_len.last_key_value() {if block_index + block_len == self.bump_ptr {self.bump_ptr = block_index;- self.free_block_index_to_len.pop_last();++ let last = self.free_block_index_to_len.pop_last();+ debug_assert_eq!((block_index, block_len), last.unwrap());++ self.remove_from_block_len_to_index(block_index, block_len);}}@@ -415,6 +479,18 @@ impl FreeList {self.check_integrity();}+ #[track_caller]+ fn remove_from_block_len_to_index(&mut self, block_index: u32, block_len: u32) {+ let Entry::Occupied(mut entry) = self.free_block_len_to_indices.entry(block_len) else {+ unreachable!()+ };+ let was_present = entry.get_mut().remove(&block_index);+ debug_assert!(was_present);+ if entry.get().is_empty() {+ entry.remove();+ }+ }+/// Iterate over all free blocks as `(index, len)` pairs.pub fn iter_free_blocks(&self) -> impl Iterator<Item = (u32, u32)> + '_ {let bump = if self.bump_end > self.bump_ptr {@@ -472,9 +548,21 @@ impl FreeList {// (4)assert_eq!(len % ALIGN_U32, 0);+ // Check that this block is also in the correct size bucket.+ assert!(self.free_block_len_to_indices.contains_key(&len));+ assert!(self.free_block_len_to_indices[&len].contains(&index));+prev_end = Some(end);}+ // Check that every entry in our size buckets is correct.+ for (len, indices) in &self.free_block_len_to_indices {+ for idx in indices {+ assert!(self.free_block_index_to_len.contains_key(idx));+ assert_eq!(self.free_block_index_to_len[idx], *len);+ }+ }+// Check bump region validity.assert!(self.bump_ptr <= self.bump_end);if self.bump_ptr < self.bump_end {