Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

bforest: Allocate all nodes upfront for insertion (#14637)

Adam Bratschi-Kayeb446f77a835d
diff --git a/cranelift/bforest/src/map.rs b/cranelift/bforest/src/map.rs
index 16e5450312..1c1b76184c 100644
--- a/cranelift/bforest/src/map.rs
+++ b/cranelift/bforest/src/map.rs
@@ -1097,15 +1097,15 @@ mod tests {
}
let mut m = full(f);
- // Verify geometry. Get get node2 as the root and leaves node0, 1, 3, ...
+ // Verify geometry. Get get node1 as the root and leaves node0, 2, 5, ...
m.verify(f, &());
- assert_eq!(m.tpath(110, f, &()), "node2[0]--node0[0]");
- assert_eq!(m.tpath(140, f, &()), "node2[0]--node0[3]");
- assert_eq!(m.tpath(210, f, &()), "node2[1]--node1[0]");
- assert_eq!(m.tpath(270, f, &()), "node2[1]--node1[6]");
- assert_eq!(m.tpath(310, f, &()), "node2[2]--node3[0]");
- assert_eq!(m.tpath(810, f, &()), "node2[7]--node8[0]");
- assert_eq!(m.tpath(870, f, &()), "node2[7]--node8[6]");
+ assert_eq!(m.tpath(110, f, &()), "node1[0]--node0[0]");
+ assert_eq!(m.tpath(140, f, &()), "node1[0]--node0[3]");
+ assert_eq!(m.tpath(210, f, &()), "node1[1]--node2[0]");
+ assert_eq!(m.tpath(270, f, &()), "node1[1]--node2[6]");
+ assert_eq!(m.tpath(310, f, &()), "node1[2]--node5[0]");
+ assert_eq!(m.tpath(810, f, &()), "node1[7]--node10[0]");
+ assert_eq!(m.tpath(870, f, &()), "node1[7]--node10[6]");
{
let mut c = m.cursor_mut(f, &());
@@ -1210,19 +1210,19 @@ mod tests {
// Verify geometry.
m.verify(f, &());
- assert_eq!(m.tpath(10, f, &()), "node2[0]--node0[0]");
- assert_eq!(m.tpath(40, f, &()), "node2[0]--node0[3]");
- assert_eq!(m.tpath(49, f, &()), "node2[0]--node0[4]");
- assert_eq!(m.tpath(50, f, &()), "node2[1]--node1[0]");
- assert_eq!(m.tpath(80, f, &()), "node2[1]--node1[3]");
+ assert_eq!(m.tpath(10, f, &()), "node1[0]--node0[0]");
+ assert_eq!(m.tpath(40, f, &()), "node1[0]--node0[3]");
+ assert_eq!(m.tpath(49, f, &()), "node1[0]--node0[4]");
+ assert_eq!(m.tpath(50, f, &()), "node1[1]--node2[0]");
+ assert_eq!(m.tpath(80, f, &()), "node1[1]--node2[3]");
// Remove the front entry from a node that stays healthy.
assert_eq!(m.insert(55, 5.5, f, &()), None);
assert_eq!(m.remove(50, f, &()), Some(5.0));
m.verify(f, &());
- assert_eq!(m.tpath(49, f, &()), "node2[0]--node0[4]");
- assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[4]");
- assert_eq!(m.tpath(55, f, &()), "node2[1]--node1[0]");
+ assert_eq!(m.tpath(49, f, &()), "node1[0]--node0[4]");
+ assert_eq!(m.tpath(50, f, &()), "node1[0]--node0[4]");
+ assert_eq!(m.tpath(55, f, &()), "node1[1]--node2[0]");
// Remove the front entry from the first leaf node: No critical key to update.
assert_eq!(m.insert(15, 1.5, f, &()), None);
@@ -1235,8 +1235,8 @@ mod tests {
// No rebalancing for the right-most node. Still need critical key update.
assert_eq!(m.remove(55, f, &()), Some(5.5));
m.verify(f, &());
- assert_eq!(m.tpath(55, f, &()), "node2[0]--node0[4]");
- assert_eq!(m.tpath(60, f, &()), "node2[1]--node1[0]");
+ assert_eq!(m.tpath(55, f, &()), "node1[0]--node0[4]");
+ assert_eq!(m.tpath(60, f, &()), "node1[1]--node2[0]");
// [ 15 20 30 40 ] [ 60 70 80 ]
@@ -1244,8 +1244,8 @@ mod tests {
assert_eq!(m.insert(90, 9.0, f, &()), None);
assert_eq!(m.insert(100, 10.0, f, &()), None);
m.verify(f, &());
- assert_eq!(m.tpath(55, f, &()), "node2[0]--node0[4]");
- assert_eq!(m.tpath(60, f, &()), "node2[1]--node1[0]");
+ assert_eq!(m.tpath(55, f, &()), "node1[0]--node0[4]");
+ assert_eq!(m.tpath(60, f, &()), "node1[1]--node2[0]");
// [ 15 20 30 40 ] [ 60 70 80 90 100 ]
@@ -1256,9 +1256,9 @@ mod tests {
// [ 15 30 40 60 ] [ 70 80 90 100 ]
// Check that the critical key was updated correctly.
- assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[3]");
- assert_eq!(m.tpath(60, f, &()), "node2[0]--node0[3]");
- assert_eq!(m.tpath(70, f, &()), "node2[1]--node1[0]");
+ assert_eq!(m.tpath(50, f, &()), "node1[0]--node0[3]");
+ assert_eq!(m.tpath(60, f, &()), "node1[0]--node0[3]");
+ assert_eq!(m.tpath(70, f, &()), "node1[1]--node2[0]");
// Remove front entry from the left-most leaf node, underflowing.
// This should cause two leaf nodes to be merged and the root node to go away.
@@ -1280,8 +1280,8 @@ mod tests {
m.verify(f, &());
// [ 10 20 30 40 ] [ 70 ]
- assert_eq!(m.tpath(50, f, &()), "node2[0]--node0[4]");
- assert_eq!(m.tpath(70, f, &()), "node2[1]--node1[0]");
+ assert_eq!(m.tpath(50, f, &()), "node1[0]--node0[4]");
+ assert_eq!(m.tpath(70, f, &()), "node1[1]--node2[0]");
// Removing the last entry from the right leaf should cause a collapse.
assert_eq!(m.remove(70, f, &()), Some(7.0));
@@ -1306,28 +1306,28 @@ mod tests {
m.verify(f, &());
// Check geometry.
- // Root: node11
- // [ node2 170 node10 330 node16 490 node21 650 node26 810 node31 970 node36 1130 node41 ]
- // L1: node11
- assert_eq!(m.tpath(0, f, &()), "node11[0]--node2[0]--node0[0]");
- assert_eq!(m.tpath(10000, f, &()), "node11[7]--node41[4]--node40[4]");
+ // Root: node3
+ // [ node1 170 node4 330 node14 490 node19 650 node24 810 node29 970 node34 1130 node39 ]
+ // L1: node3
+ assert_eq!(m.tpath(0, f, &()), "node3[0]--node1[0]--node0[0]");
+ assert_eq!(m.tpath(10000, f, &()), "node3[7]--node39[4]--node43[4]");
// 650 is a critical key in the middle of the root.
- assert_eq!(m.tpath(640, f, &()), "node11[3]--node21[3]--node19[3]");
- assert_eq!(m.tpath(650, f, &()), "node11[4]--node26[0]--node20[0]");
+ assert_eq!(m.tpath(640, f, &()), "node3[3]--node19[3]--node22[3]");
+ assert_eq!(m.tpath(650, f, &()), "node3[4]--node24[0]--node23[0]");
- // Deleting 640 triggers a rebalance from node19 to node 20, cascading to n21 -> n26.
+ // Deleting 640 triggers a rebalance from node22 to node 23, cascading to n19 -> n24.
assert_eq!(m.remove(640, f, &()), Some(64.0));
m.verify(f, &());
- assert_eq!(m.tpath(650, f, &()), "node11[3]--node26[3]--node20[3]");
+ assert_eq!(m.tpath(650, f, &()), "node3[3]--node24[3]--node23[3]");
- // 1130 is in the first leaf of the last L1 node. Deleting it triggers a rebalance node35
- // -> node37, but no rebalance above where there is no right sibling.
- assert_eq!(m.tpath(1130, f, &()), "node11[6]--node41[0]--node35[0]");
- assert_eq!(m.tpath(1140, f, &()), "node11[6]--node41[0]--node35[1]");
+ // 1130 is in the first leaf of the last L1 node. Deleting it triggers a rebalance node38
+ // -> node40, but no rebalance above where there is no right sibling.
+ assert_eq!(m.tpath(1130, f, &()), "node3[6]--node39[0]--node38[0]");
+ assert_eq!(m.tpath(1140, f, &()), "node3[6]--node39[0]--node38[1]");
assert_eq!(m.remove(1130, f, &()), Some(113.0));
m.verify(f, &());
- assert_eq!(m.tpath(1140, f, &()), "node11[6]--node41[0]--node37[0]");
+ assert_eq!(m.tpath(1140, f, &()), "node3[6]--node39[0]--node40[0]");
}
#[test]
diff --git a/cranelift/bforest/src/path.rs b/cranelift/bforest/src/path.rs
index eff05175d3..ca7321e363 100644
--- a/cranelift/bforest/src/path.rs
+++ b/cranelift/bforest/src/path.rs
@@ -4,7 +4,7 @@ use super::node::Removed;
use super::{Comparator, Forest, MAX_PATH, Node, NodeData, NodePool, slice_insert, slice_shift};
use core::borrow::Borrow;
use core::marker::PhantomData;
-use wasmtime_core::error::OutOfMemory;
+use wasmtime_core::{alloc::PanicOnOom, error::OutOfMemory};
#[cfg(test)]
use core::fmt;
@@ -296,6 +296,12 @@ impl<F: Forest> Path<F> {
) -> Result<(), OutOfMemory> {
let orig_root = self.node[0];
+ // In the worst case this will perform one allocation for each iteration
+ // of the loop and one additional allocation at the end. Reserving each
+ // of these nodes up front guarantees that we'll either OOM now or the
+ // full operation will succeed.
+ pool.reserve(self.size + 1)?;
+
// Loop invariant: We need to split the node at `level` and then retry a failed insertion.
// The items to insert are either `(key, ins_node)` or `(key, value)`.
let mut ins_node = None;
@@ -305,7 +311,7 @@ impl<F: Forest> Path<F> {
let mut node = self.node[level];
let mut entry = self.entry[level].into();
split = pool[node].split(entry);
- let rhs_node = pool.alloc_node(split.rhs_data)?;
+ let rhs_node = pool.alloc_node(split.rhs_data).panic_on_oom();
// Should the path be moved to the new RHS node?
// Prefer the smaller node if we're right in the middle.
@@ -365,7 +371,9 @@ impl<F: Forest> Path<F> {
// If we get here we have split the original root node and need to add an extra level.
let rhs_node = ins_node.expect("empty path");
- let root = pool.alloc_node(NodeData::inner(orig_root, key, rhs_node))?;
+ let root = pool
+ .alloc_node(NodeData::inner(orig_root, key, rhs_node))
+ .panic_on_oom();
let entry = if self.node[0] == rhs_node { 1 } else { 0 };
self.size += 1;
slice_insert(&mut self.node[0..self.size], 0, root);
diff --git a/cranelift/bforest/src/pool.rs b/cranelift/bforest/src/pool.rs
index a7889dafb3..35fc4090f9 100644
--- a/cranelift/bforest/src/pool.rs
+++ b/cranelift/bforest/src/pool.rs
@@ -13,6 +13,8 @@ use wasmtime_core::error::OutOfMemory;
pub(super) struct NodePool<F: Forest> {
nodes: PrimaryMap<Node, NodeData<F>>,
freelist: Option<Node>,
+ /// Invariant: This tracks the length of the `freelist`.
+ freelist_count: usize,
}
impl<F: Forest> NodePool<F> {
@@ -21,6 +23,7 @@ impl<F: Forest> NodePool<F> {
Self {
nodes: PrimaryMap::new(),
freelist: None,
+ freelist_count: 0,
}
}
@@ -28,6 +31,7 @@ impl<F: Forest> NodePool<F> {
pub fn clear(&mut self) {
self.nodes.clear();
self.freelist = None;
+ self.freelist_count = 0;
}
/// Allocate a new node containing `data`.
@@ -37,7 +41,10 @@ impl<F: Forest> NodePool<F> {
Some(node) => {
// Remove this node from the free list.
match self.nodes[node] {
- NodeData::Free { next } => self.freelist = next,
+ NodeData::Free { next } => {
+ self.freelist = next;
+ self.freelist_count -= 1;
+ }
_ => panic!("Invalid {node} on free list"),
}
self.nodes[node] = data;
@@ -51,6 +58,25 @@ impl<F: Forest> NodePool<F> {
}
}
+ /// Reserve at least `count` nodes. This guarantees that the next `count`
+ /// calls to `alloc_node` will succeed.
+ pub fn reserve(&mut self, count: usize) -> Result<(), OutOfMemory> {
+ if count > self.freelist_count {
+ self.nodes.try_reserve(count - self.freelist_count)?;
+ let mut freelist_head = self.freelist;
+ for _ in 0..count - self.freelist_count {
+ freelist_head = Some(self.nodes.push(NodeData::Free {
+ next: freelist_head,
+ }));
+ }
+ // We inserted exactly enough nodes for the freelist to have size
+ // `count`.
+ self.freelist = freelist_head;
+ self.freelist_count = count;
+ }
+ Ok(())
+ }
+
/// Free a node.
pub fn free_node(&mut self, node: Node) {
// Quick check for a double free.
@@ -59,6 +85,7 @@ impl<F: Forest> NodePool<F> {
next: self.freelist,
};
self.freelist = Some(node);
+ self.freelist_count += 1;
}
/// Free the entire tree rooted at `node`.
diff --git a/cranelift/bforest/src/set.rs b/cranelift/bforest/src/set.rs
index a17e705c3d..ec896258ec 100644
--- a/cranelift/bforest/src/set.rs
+++ b/cranelift/bforest/src/set.rs
@@ -462,7 +462,7 @@ mod tests {
assert!(!c.is_empty());
assert_eq!(c.goto_first(), Some(0));
- assert_eq!(c.tpath(), "node2[0]--node0[0]");
+ assert_eq!(c.tpath(), "node1[0]--node0[0]");
assert_eq!(c.prev(), None);
for i in 1..50 {
@@ -509,7 +509,7 @@ mod tests {
assert!(!c.is_empty());
assert!(c.goto(0));
- assert_eq!(c.tpath(), "node11[0]--node2[0]--node0[0]");
+ assert_eq!(c.tpath(), "node3[0]--node1[0]--node0[0]");
assert_eq!(c.prev(), None);
for i in 1..150 {
@@ -576,7 +576,7 @@ mod tests {
// Peel off a whole sub-tree of the root by deleting from the front.
// The 900 element is near the front of the second sub-tree.
assert!(c.goto(900));
- assert_eq!(c.tpath(), "node48[1]--node47[0]--node26[0]--node20[4]");
+ assert_eq!(c.tpath(), "node12[1]--node13[0]--node24[0]--node23[4]");
assert!(c.goto(0));
for i in 0..900 {
assert!(!c.is_empty());
diff --git a/crates/fuzzing/tests/oom/bforest_map.rs b/crates/fuzzing/tests/oom/bforest_map.rs
index c3104822f0..23c736a50c 100644
--- a/crates/fuzzing/tests/oom/bforest_map.rs
+++ b/crates/fuzzing/tests/oom/bforest_map.rs
@@ -17,3 +17,29 @@ fn bforest_map() -> Result<()> {
Ok(())
})
}
+
+#[test]
+fn bforest_map_failed_insert_preserves_entries() -> Result<()> {
+ OomTest::new()
+ // Allow the assertion below to allocate its panic message.
+ .allow_alloc_after_oom(true)
+ .alloc_succeeds_after_oom(true)
+ .test(|| {
+ let mut forest = MapForest::new();
+ let mut map = Map::new();
+ for i in 0..1000 {
+ if let Err(e) = map.try_insert(Key(i), i, &mut forest, &()) {
+ // A failed insert must leave every existing entry in place.
+ for j in 0..i {
+ assert_eq!(
+ map.get(Key(j), &forest, &()),
+ Some(j),
+ "key {j} lost after failed insert of key {i}"
+ );
+ }
+ return Err(e.into());
+ }
+ }
+ Ok(())
+ })
+}