A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Use bump allocation in DRC free list and other improvements (#12969)
Nick Fitzgerald009601394d24
diff --git a/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt b/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txtnew file mode 100644index 0000000000..c3a26b7b97--- /dev/null+++ b/crates/wasmtime/proptest-regressions/runtime/vm/gc/enabled/free_list.txt@@ -0,0 +1,7 @@+# Seeds for failure cases proptest has generated in the past. It is+# automatically read and these particular cases re-run before any+# novel cases are generated.+#+# It is recommended to check this file in to source control so that+# everyone who runs the test benefits from these saved cases.+cc b26e69fbaf46deb79652859039538e422818fd40b9afff63faa7aacbddecfd3d # shrinks to (capacity, ops) = (219544665809630458, [(10, Alloc(Layout { size: 193045289231815352, align: 8 (1 << 3) })), (10, Dealloc(Layout { size: 193045289231815352, align: 8 (1 << 3) }))])diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rsindex 29558ea308..753b27b046 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs@@ -176,17 +176,21 @@ impl DrcHeap {fn dealloc(&mut self, gc_ref: VMGcRef) {let drc_ref = drc_ref(&gc_ref);- let size = self.index(drc_ref).object_size();- let layout = FreeList::layout(size);+ let size = self.index(drc_ref).object_size;+ let alloc_size = FreeList::aligned_size(size);let index = gc_ref.as_heap_index().unwrap();// Poison the freed memory so that any stale access is detectable.if cfg!(gc_zeal) {let index = usize::try_from(index.get()).unwrap();- self.heap_slice_mut()[index..][..layout.size()].fill(POISON);+ let alloc_size = usize::try_from(alloc_size).unwrap();+ self.heap_slice_mut()[index..][..alloc_size].fill(POISON);}- self.free_list.as_mut().unwrap().dealloc(index, layout);+ self.free_list+ .as_mut()+ .unwrap()+ .dealloc_fast(index, alloc_size);}/// Increment the ref count for the associated object.@@ -920,6 +924,7 @@ unsafe impl GcHeap for DrcHeap {fn alloc_raw(&mut self, header: VMGcHeader, layout: Layout) -> Result<Result<VMGcRef, u64>> {debug_assert!(layout.size() >= core::mem::size_of::<VMDrcHeader>());debug_assert!(layout.align() >= core::mem::align_of::<VMDrcHeader>());+ debug_assert!(FreeList::can_align_to(layout.align()));debug_assert_eq!(header.reserved_u26(), 0);// We must have trace info for every GC type that we allocate in this@@ -933,8 +938,9 @@ unsafe impl GcHeap for DrcHeap {}let object_size = u32::try_from(layout.size()).unwrap();+ let alloc_size = FreeList::aligned_size(object_size);- let gc_ref = match self.free_list.as_mut().unwrap().alloc(layout)? {+ let gc_ref = match self.free_list.as_mut().unwrap().alloc_fast(alloc_size) {None => return Ok(Err(u64::try_from(layout.size()).unwrap())),Some(index) => VMGcRef::from_heap_index(index).unwrap(),};diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rsindex e638e5ea90..d21d9026e4 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/free_list.rs@@ -1,8 +1,9 @@use crate::prelude::*;use alloc::collections::BTreeMap;-use core::{alloc::Layout, num::NonZeroU32, ops::Bound};+use core::{alloc::Layout, num::NonZeroU32};-/// A very simple first-fit free list for use by our garbage collectors.+/// A free list for use by our garbage collectors, using a sorted Vec of+/// (index, length) pairs for cache-friendly operations.pub(crate) struct FreeList {/// The total capacity of the contiguous range of memory we are managing.///@@ -25,6 +26,11 @@ pub(crate) struct FreeList {/// Our free blocks, as a map from index to length of the free block at that/// index.free_block_index_to_len: BTreeMap<u32, u32>,+ /// Bump allocator: current position in the active free block.+ /// Allocations bump this forward. When exhausted, refilled from blocks.+ bump_ptr: u32,+ /// End of the current bump allocation region.+ bump_end: u32,}/// Our minimum and maximum supported alignment. Every allocation is aligned to@@ -40,6 +46,13 @@ impl FreeList {Layout::from_size_align(size, ALIGN_USIZE).unwrap()}+ /// Compute the aligned allocation size for a given byte size. Returns the+ /// size rounded up to this free list's alignment, as a u32.+ #[inline]+ pub fn aligned_size(size: u32) -> u32 {+ (size + ALIGN_U32 - 1) & !(ALIGN_U32 - 1)+ }+/// Get the current total capacity this free list manages.pub fn current_capacity(&self) -> usize {self.capacity@@ -53,6 +66,8 @@ impl FreeList {let mut free_list = FreeList {capacity,free_block_index_to_len: BTreeMap::new(),+ bump_ptr: 0,+ bump_end: 0,};let end = u32::try_from(free_list.capacity).unwrap_or_else(|_| {@@ -67,13 +82,11 @@ impl FreeList {let len = round_u32_down_to_pow2(end.saturating_sub(start), ALIGN_U32);- let entire_range = if len >= ALIGN_U32 {- Some((start, len))- } else {- None- };-- free_list.free_block_index_to_len.extend(entire_range);+ if len >= ALIGN_U32 {+ // Initialize bump allocator with the entire range.+ free_list.bump_ptr = start;+ free_list.bump_end = start + len;+ }free_list}@@ -146,11 +159,23 @@ impl FreeList {round_usize_down_to_pow2(cap.saturating_sub(ALIGN_USIZE), ALIGN_USIZE)}+ /// Total number of free blocks (including bump region if non-empty).+ #[cfg(test)]+ fn num_free_blocks(&self) -> usize {+ self.free_block_index_to_len.len() + if self.bump_end > self.bump_ptr { 1 } else { 0 }+ }++ /// Can this free list align allocations to the given value?+ pub fn can_align_to(align: usize) -> bool {+ debug_assert!(align.is_power_of_two());+ align <= ALIGN_USIZE+ }+/// Check the given layout for compatibility with this free list and return/// the actual block size we will use for this layout.fn check_layout(&self, layout: Layout) -> Result<u32> {ensure!(- layout.align() <= ALIGN_USIZE,+ Self::can_align_to(layout.align()),"requested allocation's alignment of {} is greater than max supported \alignment of {ALIGN_USIZE}",layout.align(),@@ -175,109 +200,146 @@ impl FreeList {})}- /// Find the first free block that can hold an allocation of the given size- /// and remove it from the free list.- fn first_fit(&mut self, alloc_size: u32) -> Option<(u32, u32)> {- debug_assert_eq!(alloc_size % ALIGN_U32, 0);-- let (&block_index, &block_len) = self- .free_block_index_to_len- .iter()- .find(|(_idx, len)| **len >= alloc_size)?;-- debug_assert_eq!(block_index % ALIGN_U32, 0);- debug_assert_eq!(block_len % ALIGN_U32, 0);-- let entry = self.free_block_index_to_len.remove(&block_index);- debug_assert!(entry.is_some());+ #[cfg(test)]+ pub fn alloc(&mut self, layout: Layout) -> Result<Option<NonZeroU32>> {+ log::trace!("FreeList::alloc({layout:?})");+ let alloc_size = self.check_layout(layout)?;+ Ok(self.alloc_impl(alloc_size))+ }- Some((block_index, block_len))+ /// Fast-path allocation with a pre-computed aligned size, as returned from+ /// `Self::aligned_size`.+ #[inline]+ pub fn alloc_fast(&mut self, alloc_size: u32) -> Option<NonZeroU32> {+ debug_assert_eq!(alloc_size % ALIGN_U32, 0);+ debug_assert!(alloc_size > 0);+ self.alloc_impl(alloc_size)}- /// If the given allocated block is large enough such that we can split it- /// and still have enough space left for future allocations, then split it.- ///- /// Returns the new length of the allocated block.- fn maybe_split(&mut self, alloc_size: u32, block_index: u32, block_len: u32) -> u32 {+ #[inline]+ fn alloc_impl(&mut self, alloc_size: u32) -> Option<NonZeroU32> {+ debug_assert_eq!(+ Self::layout(usize::try_from(alloc_size).unwrap()).size(),+ usize::try_from(alloc_size).unwrap()+ );debug_assert_eq!(alloc_size % ALIGN_U32, 0);- debug_assert_eq!(block_index % ALIGN_U32, 0);- debug_assert_eq!(block_len % ALIGN_U32, 0);- if block_len - alloc_size < ALIGN_U32 {- // The block is not large enough to split.- return block_len;- }+ // Fast path: bump allocate from the current region.+ let new_ptr = self.bump_ptr + alloc_size;+ if new_ptr <= self.bump_end {+ let result = self.bump_ptr;+ self.bump_ptr = new_ptr;+ debug_assert_ne!(result, 0);+ debug_assert_eq!(result % ALIGN_U32, 0);- // The block is large enough to split. Split the block at exactly the- // requested allocation size and put the tail back in the free list.- let new_block_len = alloc_size;- let split_start = block_index + alloc_size;- let split_len = block_len - alloc_size;+ self.check_integrity();- debug_assert_eq!(new_block_len % ALIGN_U32, 0);- debug_assert_eq!(split_start % ALIGN_U32, 0);- debug_assert_eq!(split_len % ALIGN_U32, 0);+ log::trace!("FreeList::alloc -> {result:#x}");+ return Some(unsafe { NonZeroU32::new_unchecked(result) });+ }- self.free_block_index_to_len.insert(split_start, split_len);+ // After we've mutated the free list, double check its integrity.+ self.check_integrity();- new_block_len+ // Slow path: find a block in the blocks list, then set it as bump region.+ self.alloc_slow(alloc_size)}- /// Allocate space for an object of the given layout.- ///- /// Returns:- ///- /// * `Ok(Some(_))`: Allocation succeeded.- ///- /// * `Ok(None)`: Can't currently fulfill the allocation request, but might- /// be able to if some stuff was reallocated.- ///- /// * `Err(_)`:- pub fn alloc(&mut self, layout: Layout) -> Result<Option<NonZeroU32>> {- log::trace!("FreeList::alloc({layout:?})");- let alloc_size = self.check_layout(layout)?;- debug_assert_eq!(alloc_size % ALIGN_U32, 0);+ #[cold]+ fn alloc_slow(&mut self, alloc_size: u32) -> Option<NonZeroU32> {+ // Put the remaining bump region back into blocks if non-empty.+ let remaining_ptr = self.bump_ptr;+ let remaining = self.bump_end - self.bump_ptr;+ self.bump_ptr = 0;+ self.bump_end = 0;+ if remaining >= ALIGN_U32 {+ self.insert_free_block(remaining_ptr, remaining);+ }++ // Find a block big enough.+ let (&block_index, &block_len) = self+ .free_block_index_to_len+ .iter()+ .find(|(_, len)| **len >= alloc_size)?;+ self.free_block_index_to_len.remove(&block_index);- let (block_index, block_len) = match self.first_fit(alloc_size) {- None => return Ok(None),- Some(tup) => tup,- };- debug_assert_ne!(block_index, 0);debug_assert_eq!(block_index % ALIGN_U32, 0);- debug_assert!(block_len >= alloc_size);debug_assert_eq!(block_len % ALIGN_U32, 0);- let block_len = self.maybe_split(alloc_size, block_index, block_len);- debug_assert!(block_len >= alloc_size);- debug_assert_eq!(block_len % ALIGN_U32, 0);+ // Set this block as the new bump region and allocate from it.+ self.bump_ptr = block_index + alloc_size;+ self.bump_end = block_index + block_len;- // After we've mutated the free list, double check its integrity.- #[cfg(debug_assertions)]+ debug_assert_ne!(block_index, 0);self.check_integrity();- log::trace!("FreeList::alloc({layout:?}) -> {block_index:#x}");- Ok(Some(unsafe { NonZeroU32::new_unchecked(block_index) }))+ Some(unsafe { NonZeroU32::new_unchecked(block_index) })}/// Deallocate an object with the given layout.pub fn dealloc(&mut self, index: NonZeroU32, layout: Layout) {log::trace!("FreeList::dealloc({index:#x}, {layout:?})");+ let alloc_size = self.check_layout(layout).unwrap();+ self.dealloc_impl(index.get(), alloc_size);+ }- let index = index.get();- debug_assert_eq!(index % ALIGN_U32, 0);+ /// Fast-path deallocation with a pre-computed aligned size.+ #[inline]+ pub fn dealloc_fast(&mut self, index: NonZeroU32, alloc_size: u32) {+ debug_assert_eq!(alloc_size % ALIGN_U32, 0);+ debug_assert_eq!(index.get() % ALIGN_U32, 0);+ self.dealloc_impl(index.get(), alloc_size);+ }- let alloc_size = self.check_layout(layout).unwrap();+ #[inline]+ fn dealloc_impl(&mut self, index: u32, alloc_size: u32) {+ debug_assert_eq!(+ Self::layout(usize::try_from(alloc_size).unwrap()).size(),+ usize::try_from(alloc_size).unwrap()+ );+ debug_assert_eq!(index % ALIGN_U32, 0);debug_assert_eq!(alloc_size % ALIGN_U32, 0);+ // Check if the freed block is directly below the bump region.+ if index + alloc_size == self.bump_ptr {+ self.bump_ptr = index;++ // Also check if the last block in the list is now contiguous with+ // the extended bump region.+ if let Some((&bi, &bl)) = self.free_block_index_to_len.last_key_value() {+ if bi + bl == self.bump_ptr {+ self.bump_ptr = bi;+ self.free_block_index_to_len.pop_last();+ }+ }++ self.check_integrity();+ return;+ }++ // Check if the freed block is directly above the bump region.+ if self.bump_end == index {+ self.bump_end = index + alloc_size;++ // Also check if the first block above the bump region is now+ // contiguous.+ if let Some(block_len) = self.free_block_index_to_len.remove(&self.bump_end) {+ self.bump_end += block_len;+ }++ self.check_integrity();+ return;+ }+let prev_block = self.free_block_index_to_len- .range((Bound::Unbounded, Bound::Excluded(index)))+ .range(..index).next_back().map(|(idx, len)| (*idx, *len));let next_block = self.free_block_index_to_len- .range((Bound::Excluded(index), Bound::Unbounded))+ .range(index + 1..).next().map(|(idx, len)| (*idx, *len));@@ -291,9 +353,9 @@ impl FreeList {&& blocks_are_contiguous(index, alloc_size, next_index) =>{log::trace!(- "merging blocks {prev_index:#x}..{prev_len:#x}, {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}",- prev_len = prev_index + prev_len,- index_end = index + u32::try_from(layout.size()).unwrap(),+ "merging blocks {prev_index:#x}..{prev_end:#x}, {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}",+ prev_end = prev_index + prev_len,+ index_end = index + alloc_size,next_end = next_index + next_len,);self.free_block_index_to_len.remove(&next_index);@@ -307,9 +369,9 @@ impl FreeList {if blocks_are_contiguous(prev_index, prev_len, index) =>{log::trace!(- "merging blocks {prev_index:#x}..{prev_len:#x}, {index:#x}..{index_end:#x}",- prev_len = prev_index + prev_len,- index_end = index + u32::try_from(layout.size()).unwrap(),+ "merging blocks {prev_index:#x}..{prev_end:#x}, {index:#x}..{index_end:#x}",+ prev_end = prev_index + prev_len,+ index_end = index + alloc_size,);let merged_block_len = index + alloc_size - prev_index;debug_assert_eq!(merged_block_len % ALIGN_U32, 0);@@ -322,7 +384,7 @@ impl FreeList {{log::trace!("merging blocks {index:#x}..{index_end:#x}, {next_index:#x}..{next_end:#x}",- index_end = index + u32::try_from(layout.size()).unwrap(),+ index_end = index + alloc_size,next_end = next_index + next_len,);self.free_block_index_to_len.remove(&next_index);@@ -339,15 +401,39 @@ impl FreeList {}}+ // After merge, check if the last block is now contiguous with the bump+ // region and absorb it.+ if let Some((&block_index, &block_len)) = self.free_block_index_to_len.last_key_value() {+ if block_index + block_len == self.bump_ptr {+ self.bump_ptr = block_index;+ self.free_block_index_to_len.pop_last();+ }+ }+// After we've added to/mutated the free list, double check its// integrity.- #[cfg(debug_assertions)]self.check_integrity();}/// Iterate over all free blocks as `(index, len)` pairs.pub fn iter_free_blocks(&self) -> impl Iterator<Item = (u32, u32)> + '_ {- self.free_block_index_to_len.iter().map(|(&i, &l)| (i, l))+ let bump = if self.bump_end > self.bump_ptr {+ Some((self.bump_ptr, self.bump_end - self.bump_ptr))+ } else {+ None+ };+ self.free_block_index_to_len+ .iter()+ .map(|(idx, len)| (*idx, *len))+ .chain(bump)+ }++ /// Insert a free block into the sorted blocks list with merging.+ fn insert_free_block(&mut self, index: u32, size: u32) {+ debug_assert_eq!(index % ALIGN_U32, 0);+ debug_assert_eq!(size % ALIGN_U32, 0);+ // Reuse dealloc_impl which handles insertion and merging.+ self.dealloc_impl(index, size);}/// Assert that the free list is valid:@@ -359,8 +445,11 @@ impl FreeList {/// 3. All blocks are aligned to `ALIGN`////// 4. All block sizes are a multiple of `ALIGN`- #[cfg(debug_assertions)]fn check_integrity(&self) {+ if !cfg!(gc_zeal) {+ return;+ }+let mut prev_end = None;for (&index, &len) in self.free_block_index_to_len.iter() {// (1)@@ -385,6 +474,26 @@ impl FreeList {prev_end = Some(end);}++ // Check bump region validity.+ assert!(self.bump_ptr <= self.bump_end);+ if self.bump_ptr < self.bump_end {+ assert_eq!(self.bump_ptr % ALIGN_U32, 0);+ assert_eq!(self.bump_end % ALIGN_U32, 0);+ assert!(usize::try_from(self.bump_end).unwrap() <= self.capacity);+ // Bump region should not overlap with any block.+ for (&index, &len) in self.free_block_index_to_len.iter() {+ let block_end = index + len;+ assert!(+ self.bump_end <= index || self.bump_ptr >= block_end,+ "bump region [{}, {}) overlaps with block [{}, {})",+ self.bump_ptr,+ self.bump_end,+ index,+ block_end+ );+ }+ }}}@@ -431,12 +540,15 @@ mod tests {use std::num::NonZeroUsize;fn free_list_block_len_and_size(free_list: &FreeList) -> (usize, Option<usize>) {- let len = free_list.free_block_index_to_len.len();- let size = free_list- .free_block_index_to_len- .values()- .next()- .map(|s| usize::try_from(*s).unwrap());+ let len = free_list.num_free_blocks();+ let size = if free_list.bump_end > free_list.bump_ptr {+ Some(usize::try_from(free_list.bump_end - free_list.bump_ptr).unwrap())+ } else {+ free_list+ .free_block_index_to_len+ .first_key_value()+ .map(|(_, &s)| usize::try_from(s).unwrap())+ };(len, size)}@@ -598,7 +710,7 @@ mod tests {// `ALIGN_U32`.let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 2);- assert_eq!(free_list.free_block_index_to_len.len(), 1);+ assert_eq!(free_list.num_free_blocks(), 1);assert_eq!(free_list.max_size(), ALIGN_USIZE * 2);// Allocate a block such that the remainder is not worth splitting.@@ -608,7 +720,7 @@ mod tests {.expect("have free space available for allocation");// Should not have split the block.- assert_eq!(free_list.free_block_index_to_len.len(), 0);+ assert_eq!(free_list.num_free_blocks(), 0);}#[test]@@ -617,7 +729,7 @@ mod tests {// `ALIGN_U32`.let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 3);- assert_eq!(free_list.free_block_index_to_len.len(), 1);+ assert_eq!(free_list.num_free_blocks(), 1);assert_eq!(free_list.max_size(), ALIGN_USIZE * 3);// Allocate a block such that the remainder is not worth splitting.@@ -627,7 +739,7 @@ mod tests {.expect("have free space available for allocation");// Should have split the block.- assert_eq!(free_list.free_block_index_to_len.len(), 1);+ assert_eq!(free_list.num_free_blocks(), 1);}#[test]@@ -636,7 +748,7 @@ mod tests {let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"initially one big free block");@@ -646,7 +758,7 @@ mod tests {.expect("allocation within 'static' free list limits").expect("have free space available for allocation");assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"should have split the block to allocate `a`");@@ -656,21 +768,21 @@ mod tests {.expect("allocation within 'static' free list limits").expect("have free space available for allocation");assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"should have split the block to allocate `b`");free_list.dealloc(a, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),2,"should have two non-contiguous free blocks after deallocating `a`");free_list.dealloc(b, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"should have merged `a` and `b` blocks with the rest to form a \single, contiguous free block after deallocating `b`"@@ -683,7 +795,7 @@ mod tests {let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"initially one big free block");@@ -701,21 +813,21 @@ mod tests {.expect("allocation within 'static' free list limits").expect("have free space available for allocation");assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"should have split the block to allocate `a`, `b`, and `c`");free_list.dealloc(a, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),2,"should have two non-contiguous free blocks after deallocating `a`");free_list.dealloc(b, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),2,"should have merged `a` and `b` blocks, but not merged with the \rest of the free space"@@ -730,7 +842,7 @@ mod tests {let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"initially one big free block");@@ -748,21 +860,21 @@ mod tests {.expect("allocation within 'static' free list limits").expect("have free space available for allocation");assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"should have split the block to allocate `a`, `b`, and `c`");free_list.dealloc(a, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),2,"should have two non-contiguous free blocks after deallocating `a`");free_list.dealloc(c, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),2,"should have merged `c` block with rest of the free space, but not \with `a` block"@@ -777,7 +889,7 @@ mod tests {let mut free_list = FreeList::new(ALIGN_USIZE + ALIGN_USIZE * 100);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"initially one big free block");@@ -799,21 +911,21 @@ mod tests {.expect("allocation within 'static' free list limits").expect("have free space available for allocation");assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"should have split the block to allocate `a`, `b`, `c`, and `d`");free_list.dealloc(a, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),2,"should have two non-contiguous free blocks after deallocating `a`");free_list.dealloc(c, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),3,"should not have merged `c` block `a` block or rest of the free \space"@@ -921,14 +1033,14 @@ mod tests {free_list.dealloc(a, layout);free_list.dealloc(b, layout);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"`dealloc` should merge blocks from different `add_capacity` calls together");free_list.add_capacity(ALIGN_USIZE);assert_eq!(- free_list.free_block_index_to_len.len(),+ free_list.num_free_blocks(),1,"`add_capacity` should eagerly merge new capacity into the last block \in the free list, when possible"