Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Fix LICM hoisting across separate loops (#15201)

Lancelot de Ferrière45abfe1ba2c8
diff --git a/cranelift/codegen/src/egraph/elaborate.rs b/cranelift/codegen/src/egraph/elaborate.rs
index c8907b2fd7..9cabc89f05 100644
--- a/cranelift/codegen/src/egraph/elaborate.rs
+++ b/cranelift/codegen/src/egraph/elaborate.rs
@@ -50,10 +50,10 @@ pub(crate) struct Elaborator<'a> {
/// Stack of loops whose headers dominate the current elaboration
/// path, outermost first.
loop_stack: SmallVec<[LoopStackEntry; 8]>,
- /// How many of the entries in `loop_stack` actually contain the current
- /// block in their loop.
+ /// Stack index just past the innermost loop containing the current block.
///
- /// Valid LICM hoisting targets are `loop_stack[..active_loop_depth]`.
+ /// Search `loop_stack[..active_loop_depth]` for LICM targets, checking loop
+ /// membership since earlier entries may be loops we have already exited.
///
/// This is not *strictly* necessary for correctness, and we could hoist to
/// anything in `loop_stack` because they all dominate the block we are
@@ -262,11 +262,10 @@ impl<'a> Elaborator<'a> {
/// Not every block dominated by a loop header is inside that loop: a block
/// that cannot reach the backedge, such as one that only traps or returns,
/// is dominated by the header but is not a member of the natural loop. This
- /// function finds how much of the loop stack actually contains this block;
- /// the rest is not available to hoist out of.
+ /// function finds the innermost loop on the stack containing this block.
///
- /// Returns `depth` where every loop in `self.loop_stack[..depth]` contains
- /// `block`.
+ /// Returns its stack index + 1, or zero if none contains `block`. Earlier
+ /// entries still need a membership check.
fn find_active_loop_depth(&self, block: Block) -> usize {
let mut lp = self.loop_analysis.innermost_loop(block);
while let Some(l) = lp {
@@ -634,8 +633,10 @@ impl<'a> Elaborator<'a> {
let hoist_level = self.loop_stack[..active_loop_depth]
.iter()
.position(|loop_entry| {
- self.domtree
- .block_dominates(value.in_block, loop_entry.hoist_block)
+ self.loop_analysis.is_in_loop(self.cur_block, loop_entry.lp)
+ && self
+ .domtree
+ .block_dominates(value.in_block, loop_entry.hoist_block)
})
.unwrap_or(active_loop_depth);
trace!(
diff --git a/cranelift/filetests/filetests/egraph/licm-separate-loops.clif b/cranelift/filetests/filetests/egraph/licm-separate-loops.clif
new file mode 100644
index 0000000000..56a9cd288f
--- /dev/null
+++ b/cranelift/filetests/filetests/egraph/licm-separate-loops.clif
@@ -0,0 +1,46 @@
+test optimize precise-output
+set opt_level=speed
+target x86_64
+
+function %separate_loops(i64) -> i64 {
+block0(v0: i64):
+ v6 = iconst.i64 1
+ jump block1(v0)
+
+block1(v1: i64):
+ v2 = isub v1, v6
+ brif v2, block1(v2), block2
+
+block2 cold:
+ jump block3(v0)
+
+block3(v3: i64) cold:
+ v4 = ushr v0, v0
+ v5 = isub v3, v4
+ brif v5, block3(v5), block4
+
+block4:
+ return v5
+}
+
+; function %separate_loops(i64) -> i64 fast {
+; block0(v0: i64):
+; v6 = iconst.i64 1
+; jump block1(v0)
+;
+; block1(v1: i64):
+; v7 = iconst.i64 1
+; v8 = isub v1, v7 ; v7 = 1
+; brif v8, block1(v8), block2
+;
+; block2 cold:
+; v4 = ushr.i64 v0, v0
+; jump block3(v0)
+;
+; block3(v3: i64) cold:
+; v5 = isub v3, v4
+; brif v5, block3(v5), block4
+;
+; block4:
+; return v5
+; }