A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
bforest: Allocate all nodes upfront for insertion (#14637)
Adam Bratschi-Kayeb446f77a835d
diff --git a/cranelift/bforest/src/map.rs b/cranelift/bforest/src/map.rsindex 16e5450312..1c1b76184c 100644--- a/cranelift/bforest/src/map.rs+++ b/cranelift/bforest/src/map.rs@@ -1097,15 +1097,15 @@ mod tests {}let mut m = full(f);- // Verify geometry. Get get node2 as the root and leaves node0, 1, 3, ...+ // Verify geometry. Get get node1 as the root and leaves node0, 2, 5, ...m.verify(f, &());- assert_eq!(m.tpath(110, f, &()), "node2[0]--node0[0]");- assert_eq!(m.tpath(140, f, &()), "node2[0]--node0[3]");- assert_eq!(m.tpath(210, f, &()), "node2[1]--node1[0]");- assert_eq!(m.tpath(270, f, &()), "node2[1]--node1[6]");- assert_eq!(m.tpath(310, f, &()), "node2[2]--node3[0]");- assert_eq!(m.tpath(810, f, &()), "node2[7]--node8[0]");- assert_eq!(m.tpath(870, f, &()), "node2[7]--node8[6]");+ assert_eq!(m.tpath(110, f, &()), "node1[0]--node0[0]");+ assert_eq!(m.tpath(140, f, &()), "node1[0]--node0[3]");+ assert_eq!(m.tpath(210, f, &()), "node1[1]--node2[0]");+ assert_eq!(m.tpath(270, f, &()), "node1[1]--node2[6]");+ assert_eq!(m.tpath(310, f, &()), "node1[2]--node5[0]");+ assert_eq!(m.tpath(810, f, &()), "node1[7]--node10[0]");+ assert_eq!(m.tpath(870, f, &()), "node1[7]--node10[6]");{let mut c = m.cursor_mut(f, &());@@ -1210,19 +1210,19 @@ mod tests {// Verify geometry.m.verify(f, &());- assert_eq!(m.tpath(10, f, &()), "node2[0]--node0[0]");- assert_eq!(m.tpath(40, f, &()), "node2[0]--node0[3]");- assert_eq!(m.tpath(49, f, &()), "node2[0]--node0[4]");- assert_eq!(m.tpath(50, f, &()), "node2[1]--node1[0]");- assert_eq!(m.tpath(80, f, &()), "node2[1]--node1[3]");+ assert_eq!(m.tpath(10, f, &()), "node1[0]--node0[0]");+ assert_eq!(m.tpath(40, f, &()), "node1[0]--node0[3]");+ assert_eq!(m.tpath(49, f, &()), "node1[0]--node0[4]");+ assert_eq!(m.tpath(50, f, &()), "node1[1]--node2[0]");+ assert_eq!(m.tpath(80, f, &()), "node1[1]--node2[3]");// Remove the front entry from a node that stays healthy.assert_eq!(m.insert(55, 5.5, f, &()), None);assert_eq!(m.remove(50, f, &()), Some(5.0));m.verify(f, &());- assert_eq!(m.tpath(49, f, &()), "node2[0]--node0[4]");- assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[4]");- assert_eq!(m.tpath(55, f, &()), "node2[1]--node1[0]");+ assert_eq!(m.tpath(49, f, &()), "node1[0]--node0[4]");+ assert_eq!(m.tpath(50, f, &()), "node1[0]--node0[4]");+ assert_eq!(m.tpath(55, f, &()), "node1[1]--node2[0]");// Remove the front entry from the first leaf node: No critical key to update.assert_eq!(m.insert(15, 1.5, f, &()), None);@@ -1235,8 +1235,8 @@ mod tests {// No rebalancing for the right-most node. Still need critical key update.assert_eq!(m.remove(55, f, &()), Some(5.5));m.verify(f, &());- assert_eq!(m.tpath(55, f, &()), "node2[0]--node0[4]");- assert_eq!(m.tpath(60, f, &()), "node2[1]--node1[0]");+ assert_eq!(m.tpath(55, f, &()), "node1[0]--node0[4]");+ assert_eq!(m.tpath(60, f, &()), "node1[1]--node2[0]");// [ 15 20 30 40 ] [ 60 70 80 ]@@ -1244,8 +1244,8 @@ mod tests {assert_eq!(m.insert(90, 9.0, f, &()), None);assert_eq!(m.insert(100, 10.0, f, &()), None);m.verify(f, &());- assert_eq!(m.tpath(55, f, &()), "node2[0]--node0[4]");- assert_eq!(m.tpath(60, f, &()), "node2[1]--node1[0]");+ assert_eq!(m.tpath(55, f, &()), "node1[0]--node0[4]");+ assert_eq!(m.tpath(60, f, &()), "node1[1]--node2[0]");// [ 15 20 30 40 ] [ 60 70 80 90 100 ]@@ -1256,9 +1256,9 @@ mod tests {// [ 15 30 40 60 ] [ 70 80 90 100 ]// Check that the critical key was updated correctly.- assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[3]");- assert_eq!(m.tpath(60, f, &()), "node2[0]--node0[3]");- assert_eq!(m.tpath(70, f, &()), "node2[1]--node1[0]");+ assert_eq!(m.tpath(50, f, &()), "node1[0]--node0[3]");+ assert_eq!(m.tpath(60, f, &()), "node1[0]--node0[3]");+ assert_eq!(m.tpath(70, f, &()), "node1[1]--node2[0]");// Remove front entry from the left-most leaf node, underflowing.// This should cause two leaf nodes to be merged and the root node to go away.@@ -1280,8 +1280,8 @@ mod tests {m.verify(f, &());// [ 10 20 30 40 ] [ 70 ]- assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[4]");- assert_eq!(m.tpath(70, f, &()), "node2[1]--node1[0]");+ assert_eq!(m.tpath(50, f, &()), "node1[0]--node0[4]");+ assert_eq!(m.tpath(70, f, &()), "node1[1]--node2[0]");// Removing the last entry from the right leaf should cause a collapse.assert_eq!(m.remove(70, f, &()), Some(7.0));@@ -1306,28 +1306,28 @@ mod tests {m.verify(f, &());// Check geometry.- // Root: node11- // [ node2 170 node10 330 node16 490 node21 650 node26 810 node31 970 node36 1130 node41 ]- // L1: node11- assert_eq!(m.tpath(0, f, &()), "node11[0]--node2[0]--node0[0]");- assert_eq!(m.tpath(10000, f, &()), "node11[7]--node41[4]--node40[4]");+ // Root: node3+ // [ node1 170 node4 330 node14 490 node19 650 node24 810 node29 970 node34 1130 node39 ]+ // L1: node3+ assert_eq!(m.tpath(0, f, &()), "node3[0]--node1[0]--node0[0]");+ assert_eq!(m.tpath(10000, f, &()), "node3[7]--node39[4]--node43[4]");// 650 is a critical key in the middle of the root.- assert_eq!(m.tpath(640, f, &()), "node11[3]--node21[3]--node19[3]");- assert_eq!(m.tpath(650, f, &()), "node11[4]--node26[0]--node20[0]");+ assert_eq!(m.tpath(640, f, &()), "node3[3]--node19[3]--node22[3]");+ assert_eq!(m.tpath(650, f, &()), "node3[4]--node24[0]--node23[0]");- // Deleting 640 triggers a rebalance from node19 to node 20, cascading to n21 -> n26.+ // Deleting 640 triggers a rebalance from node22 to node 23, cascading to n19 -> n24.assert_eq!(m.remove(640, f, &()), Some(64.0));m.verify(f, &());- assert_eq!(m.tpath(650, f, &()), "node11[3]--node26[3]--node20[3]");+ assert_eq!(m.tpath(650, f, &()), "node3[3]--node24[3]--node23[3]");- // 1130 is in the first leaf of the last L1 node. Deleting it triggers a rebalance node35- // -> node37, but no rebalance above where there is no right sibling.- assert_eq!(m.tpath(1130, f, &()), "node11[6]--node41[0]--node35[0]");- assert_eq!(m.tpath(1140, f, &()), "node11[6]--node41[0]--node35[1]");+ // 1130 is in the first leaf of the last L1 node. Deleting it triggers a rebalance node38+ // -> node40, but no rebalance above where there is no right sibling.+ assert_eq!(m.tpath(1130, f, &()), "node3[6]--node39[0]--node38[0]");+ assert_eq!(m.tpath(1140, f, &()), "node3[6]--node39[0]--node38[1]");assert_eq!(m.remove(1130, f, &()), Some(113.0));m.verify(f, &());- assert_eq!(m.tpath(1140, f, &()), "node11[6]--node41[0]--node37[0]");+ assert_eq!(m.tpath(1140, f, &()), "node3[6]--node39[0]--node40[0]");}#[test]diff --git a/cranelift/bforest/src/path.rs b/cranelift/bforest/src/path.rsindex eff05175d3..ca7321e363 100644--- a/cranelift/bforest/src/path.rs+++ b/cranelift/bforest/src/path.rs@@ -4,7 +4,7 @@ use super::node::Removed;use super::{Comparator, Forest, MAX_PATH, Node, NodeData, NodePool, slice_insert, slice_shift};use core::borrow::Borrow;use core::marker::PhantomData;-use wasmtime_core::error::OutOfMemory;+use wasmtime_core::{alloc::PanicOnOom, error::OutOfMemory};#[cfg(test)]use core::fmt;@@ -296,6 +296,12 @@ impl<F: Forest> Path<F> {) -> Result<(), OutOfMemory> {let orig_root = self.node[0];+ // In the worst case this will perform one allocation for each iteration+ // of the loop and one additional allocation at the end. Reserving each+ // of these nodes up front guarantees that we'll either OOM now or the+ // full operation will succeed.+ pool.reserve(self.size + 1)?;+// Loop invariant: We need to split the node at `level` and then retry a failed insertion.// The items to insert are either `(key, ins_node)` or `(key, value)`.let mut ins_node = None;@@ -305,7 +311,7 @@ impl<F: Forest> Path<F> {let mut node = self.node[level];let mut entry = self.entry[level].into();split = pool[node].split(entry);- let rhs_node = pool.alloc_node(split.rhs_data)?;+ let rhs_node = pool.alloc_node(split.rhs_data).panic_on_oom();// Should the path be moved to the new RHS node?// Prefer the smaller node if we're right in the middle.@@ -365,7 +371,9 @@ impl<F: Forest> Path<F> {// If we get here we have split the original root node and need to add an extra level.let rhs_node = ins_node.expect("empty path");- let root = pool.alloc_node(NodeData::inner(orig_root, key, rhs_node))?;+ let root = pool+ .alloc_node(NodeData::inner(orig_root, key, rhs_node))+ .panic_on_oom();let entry = if self.node[0] == rhs_node { 1 } else { 0 };self.size += 1;slice_insert(&mut self.node[0..self.size], 0, root);diff --git a/cranelift/bforest/src/pool.rs b/cranelift/bforest/src/pool.rsindex a7889dafb3..35fc4090f9 100644--- a/cranelift/bforest/src/pool.rs+++ b/cranelift/bforest/src/pool.rs@@ -13,6 +13,8 @@ use wasmtime_core::error::OutOfMemory;pub(super) struct NodePool<F: Forest> {nodes: PrimaryMap<Node, NodeData<F>>,freelist: Option<Node>,+ /// Invariant: This tracks the length of the `freelist`.+ freelist_count: usize,}impl<F: Forest> NodePool<F> {@@ -21,6 +23,7 @@ impl<F: Forest> NodePool<F> {Self {nodes: PrimaryMap::new(),freelist: None,+ freelist_count: 0,}}@@ -28,6 +31,7 @@ impl<F: Forest> NodePool<F> {pub fn clear(&mut self) {self.nodes.clear();self.freelist = None;+ self.freelist_count = 0;}/// Allocate a new node containing `data`.@@ -37,7 +41,10 @@ impl<F: Forest> NodePool<F> {Some(node) => {// Remove this node from the free list.match self.nodes[node] {- NodeData::Free { next } => self.freelist = next,+ NodeData::Free { next } => {+ self.freelist = next;+ self.freelist_count -= 1;+ }_ => panic!("Invalid {node} on free list"),}self.nodes[node] = data;@@ -51,6 +58,25 @@ impl<F: Forest> NodePool<F> {}}+ /// Reserve at least `count` nodes. This guarantees that the next `count`+ /// calls to `alloc_node` will succeed.+ pub fn reserve(&mut self, count: usize) -> Result<(), OutOfMemory> {+ if count > self.freelist_count {+ self.nodes.try_reserve(count - self.freelist_count)?;+ let mut freelist_head = self.freelist;+ for _ in 0..count - self.freelist_count {+ freelist_head = Some(self.nodes.push(NodeData::Free {+ next: freelist_head,+ }));+ }+ // We inserted exactly enough nodes for the freelist to have size+ // `count`.+ self.freelist = freelist_head;+ self.freelist_count = count;+ }+ Ok(())+ }+/// Free a node.pub fn free_node(&mut self, node: Node) {// Quick check for a double free.@@ -59,6 +85,7 @@ impl<F: Forest> NodePool<F> {next: self.freelist,};self.freelist = Some(node);+ self.freelist_count += 1;}/// Free the entire tree rooted at `node`.diff --git a/cranelift/bforest/src/set.rs b/cranelift/bforest/src/set.rsindex a17e705c3d..ec896258ec 100644--- a/cranelift/bforest/src/set.rs+++ b/cranelift/bforest/src/set.rs@@ -462,7 +462,7 @@ mod tests {assert!(!c.is_empty());assert_eq!(c.goto_first(), Some(0));- assert_eq!(c.tpath(), "node2[0]--node0[0]");+ assert_eq!(c.tpath(), "node1[0]--node0[0]");assert_eq!(c.prev(), None);for i in 1..50 {@@ -509,7 +509,7 @@ mod tests {assert!(!c.is_empty());assert!(c.goto(0));- assert_eq!(c.tpath(), "node11[0]--node2[0]--node0[0]");+ assert_eq!(c.tpath(), "node3[0]--node1[0]--node0[0]");assert_eq!(c.prev(), None);for i in 1..150 {@@ -576,7 +576,7 @@ mod tests {// Peel off a whole sub-tree of the root by deleting from the front.// The 900 element is near the front of the second sub-tree.assert!(c.goto(900));- assert_eq!(c.tpath(), "node48[1]--node47[0]--node26[0]--node20[4]");+ assert_eq!(c.tpath(), "node12[1]--node13[0]--node24[0]--node23[4]");assert!(c.goto(0));for i in 0..900 {assert!(!c.is_empty());diff --git a/crates/fuzzing/tests/oom/bforest_map.rs b/crates/fuzzing/tests/oom/bforest_map.rsindex c3104822f0..23c736a50c 100644--- a/crates/fuzzing/tests/oom/bforest_map.rs+++ b/crates/fuzzing/tests/oom/bforest_map.rs@@ -17,3 +17,29 @@ fn bforest_map() -> Result<()> {Ok(())})}++#[test]+fn bforest_map_failed_insert_preserves_entries() -> Result<()> {+ OomTest::new()+ // Allow the assertion below to allocate its panic message.+ .allow_alloc_after_oom(true)+ .alloc_succeeds_after_oom(true)+ .test(|| {+ let mut forest = MapForest::new();+ let mut map = Map::new();+ for i in 0..1000 {+ if let Err(e) = map.try_insert(Key(i), i, &mut forest, &()) {+ // A failed insert must leave every existing entry in place.+ for j in 0..i {+ assert_eq!(+ map.get(Key(j), &forest, &()),+ Some(j),+ "key {j} lost after failed insert of key {i}"+ );+ }+ return Err(e.into());+ }+ }+ Ok(())+ })+}