Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift: Group blocks belonging to the same loop together during lowering (#15161)

Amanieu d'Antras6049b81e0356
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs
index 7d6026c97b..472f3d9db4 100644
--- a/cranelift/codegen/src/context.rs
+++ b/cranelift/codegen/src/context.rs
@@ -143,6 +143,7 @@ impl Context {
result = isa.compile_function(
&self.func,
&self.domtree,
+ &self.loop_analysis,
&mut self.regalloc_ctx,
self.want_disasm,
ctrl_plane,
@@ -390,6 +391,7 @@ impl Context {
// Branch optimizations can invalidate these; recompute them.
self.compute_cfg();
self.compute_domtree();
+ self.compute_loop_analysis();
self.verify_if(fisa)?;
diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rs
index f419633929..7ad618b030 100644
--- a/cranelift/codegen/src/isa/aarch64/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/mod.rs
@@ -6,6 +6,7 @@ use crate::isa::aarch64::settings as aarch64_settings;
#[cfg(feature = "unwind")]
use crate::isa::unwind::systemv;
use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, TargetIsa};
+use crate::loop_analysis::LoopAnalysis;
use crate::machinst::CompiledCode;
use crate::machinst::{
CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,
@@ -54,6 +55,7 @@ impl AArch64Backend {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<VCode<inst::Inst>> {
@@ -63,6 +65,7 @@ impl AArch64Backend {
compile::compile::<AArch64Backend>(
func,
domtree,
+ loop_analysis,
regalloc_ctx,
self,
abi,
@@ -78,11 +81,12 @@ impl TargetIsa for AArch64Backend {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
want_disasm: bool,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<CompiledCodeStencil> {
- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;
+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;
let emit_result = vcode.emit(&regalloc_ctx.output, want_disasm, &self.flags, ctrl_plane)?;
let value_labels_ranges = emit_result.value_labels_ranges;
diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs
index dfc5277d18..f6ea84668a 100644
--- a/cranelift/codegen/src/isa/mod.rs
+++ b/cranelift/codegen/src/isa/mod.rs
@@ -45,6 +45,7 @@
use crate::dominator_tree::DominatorTree;
pub use crate::isa::call_conv::CallConv;
+use crate::loop_analysis::LoopAnalysis;
use crate::CodegenResult;
use crate::ir::{self, Function, Type};
@@ -310,6 +311,7 @@ pub trait TargetIsa: fmt::Display + Send + Sync {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
want_disasm: bool,
ctrl_plane: &mut ControlPlane,
diff --git a/cranelift/codegen/src/isa/pulley_shared/mod.rs b/cranelift/codegen/src/isa/pulley_shared/mod.rs
index 4350d0d183..7da1536f33 100644
--- a/cranelift/codegen/src/isa/pulley_shared/mod.rs
+++ b/cranelift/codegen/src/isa/pulley_shared/mod.rs
@@ -12,6 +12,7 @@ use crate::{
dominator_tree::DominatorTree,
ir,
isa::{IsaFlagsHashKey, OwnedTargetIsa, TargetIsa},
+ loop_analysis::LoopAnalysis,
machinst::{self, CompiledCode, CompiledCodeStencil, MachInst, SigSet, VCode},
result::CodegenResult,
settings::{self as shared_settings, Flags},
@@ -122,6 +123,7 @@ where
&self,
func: &ir::Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<VCode<inst::InstAndKind<P>>> {
@@ -135,6 +137,7 @@ where
machinst::compile::<Self>(
func,
domtree,
+ loop_analysis,
regalloc_ctx,
self,
abi,
@@ -182,11 +185,12 @@ where
&self,
func: &ir::Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
want_disasm: bool,
ctrl_plane: &mut cranelift_control::ControlPlane,
) -> CodegenResult<CompiledCodeStencil> {
- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;
+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;
let want_disasm =
want_disasm || (cfg!(feature = "trace-log") && log::log_enabled!(log::Level::Debug));
diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rs
index a2c94c6cff..39ef5c2c0f 100644
--- a/cranelift/codegen/src/isa/riscv64/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/mod.rs
@@ -6,6 +6,7 @@ use crate::isa::riscv64::settings as riscv_settings;
use crate::isa::{
Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, OwnedTargetIsa, TargetIsa,
};
+use crate::loop_analysis::LoopAnalysis;
use crate::machinst::CompiledCode;
use crate::machinst::{
CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,
@@ -68,6 +69,7 @@ impl Riscv64Backend {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<VCode<inst::Inst>> {
@@ -77,6 +79,7 @@ impl Riscv64Backend {
compile::compile::<Riscv64Backend>(
func,
domtree,
+ loop_analysis,
regalloc_ctx,
self,
abi,
@@ -92,11 +95,12 @@ impl TargetIsa for Riscv64Backend {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
want_disasm: bool,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<CompiledCodeStencil> {
- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;
+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;
let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug);
let emit_result = vcode.emit(&regalloc_ctx.output, want_disasm, &self.flags, ctrl_plane)?;
diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rs
index 2f8200644e..cbbecf6790 100644
--- a/cranelift/codegen/src/isa/s390x/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/mod.rs
@@ -6,6 +6,7 @@ use crate::isa::s390x::settings as s390x_settings;
#[cfg(feature = "unwind")]
use crate::isa::unwind::systemv::RegisterMappingError;
use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, TargetIsa};
+use crate::loop_analysis::LoopAnalysis;
use crate::machinst::{
CompiledCode, CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet,
TextSectionBuilder, VCode, compile,
@@ -53,6 +54,7 @@ impl S390xBackend {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<VCode<inst::Inst>> {
@@ -62,6 +64,7 @@ impl S390xBackend {
compile::compile::<S390xBackend>(
func,
domtree,
+ loop_analysis,
regalloc_ctx,
self,
abi,
@@ -77,12 +80,13 @@ impl TargetIsa for S390xBackend {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
want_disasm: bool,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<CompiledCodeStencil> {
let flags = self.flags();
- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;
+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;
let emit_result = vcode.emit(&regalloc_ctx.output, want_disasm, flags, ctrl_plane)?;
let value_labels_ranges = emit_result.value_labels_ranges;
diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rs
index 3b1f9afd8d..7dc8765a79 100644
--- a/cranelift/codegen/src/isa/x64/mod.rs
+++ b/cranelift/codegen/src/isa/x64/mod.rs
@@ -10,6 +10,7 @@ use crate::ir::{self, Function, Type, types};
use crate::isa::unwind::systemv;
use crate::isa::x64::settings as x64_settings;
use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey};
+use crate::loop_analysis::LoopAnalysis;
use crate::machinst::{
CompiledCode, CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet,
TextSectionBuilder, VCode, compile,
@@ -61,6 +62,7 @@ impl X64Backend {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<VCode<inst::Inst>> {
@@ -72,6 +74,7 @@ impl X64Backend {
compile::compile::<Self>(
func,
domtree,
+ loop_analysis,
regalloc_ctx,
self,
abi,
@@ -87,11 +90,12 @@ impl TargetIsa for X64Backend {
&self,
func: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
want_disasm: bool,
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<CompiledCodeStencil> {
- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;
+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;
let emit_result = vcode.emit(&regalloc_ctx.output, want_disasm, &self.flags, ctrl_plane)?;
let value_labels_ranges = emit_result.value_labels_ranges;
diff --git a/cranelift/codegen/src/loop_analysis.rs b/cranelift/codegen/src/loop_analysis.rs
index 5d88c519af..f5b4b2069c 100644
--- a/cranelift/codegen/src/loop_analysis.rs
+++ b/cranelift/codegen/src/loop_analysis.rs
@@ -1,6 +1,8 @@
//! A loop analysis represented as mappings of loops to their header Block
//! and parent in the loop tree.
+use core::cmp::Ordering;
+
use crate::dominator_tree::DominatorTree;
use crate::entity::SecondaryMap;
use crate::entity::entity_impl;
@@ -13,7 +15,7 @@ use alloc::vec::Vec;
use smallvec::SmallVec;
/// A opaque reference to a code loop.
-#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct Loop(u32);
entity_impl!(Loop, "loop");
@@ -115,6 +117,20 @@ impl LoopAnalysis {
self.loops[lp].parent.expand()
}
+ /// Return the innermost common ancestor of two loops.
+ pub fn common_ancestor(&self, mut a: Loop, mut b: Loop) -> Option<Loop> {
+ loop {
+ // Because loops are ordered in RPO, a parent loop is guaranteed to
+ // have a lower index than a child loop. We can just walk up the
+ // tree until we find a common loop.
+ match a.cmp(&b) {
+ Ordering::Less => b = self.loop_parent(b)?,
+ Ordering::Equal => return Some(a),
+ Ordering::Greater => a = self.loop_parent(a)?,
+ }
+ }
+ }
+
/// Return the innermost loop for a given block.
pub fn innermost_loop(&self, block: Block) -> Option<Loop> {
self.block_loop_map[block].expand()
@@ -364,6 +380,18 @@ mod tests {
assert_eq!(loop_analysis.loop_level(block1).level(), 2);
assert_eq!(loop_analysis.loop_level(block2).level(), 2);
assert_eq!(loop_analysis.loop_level(block3).level(), 1);
+ assert_eq!(
+ loop_analysis.common_ancestor(loops[0], loops[1]),
+ Some(loops[0])
+ );
+ assert_eq!(
+ loop_analysis.common_ancestor(loops[1], loops[0]),
+ Some(loops[0])
+ );
+ assert_eq!(
+ loop_analysis.common_ancestor(loops[1], loops[1]),
+ Some(loops[1])
+ );
}
#[test]
@@ -428,5 +456,13 @@ mod tests {
assert_eq!(loop_analysis.loop_level(block3).level(), 2);
assert_eq!(loop_analysis.loop_level(block4).level(), 2);
assert_eq!(loop_analysis.loop_level(block5).level(), 1);
+ assert_eq!(
+ loop_analysis.common_ancestor(loops[1], loops[2]),
+ Some(loops[0])
+ );
+ assert_eq!(
+ loop_analysis.common_ancestor(loops[2], loops[1]),
+ Some(loops[0])
+ );
}
}
diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs
index 08c014ebd5..6e7c088c2b 100644
--- a/cranelift/codegen/src/machinst/blockorder.rs
+++ b/cranelift/codegen/src/machinst/blockorder.rs
@@ -61,9 +61,10 @@
//! the other redundancies that this scheme produces.
use crate::dominator_tree::DominatorTree;
-use crate::entity::SecondaryMap;
+use crate::entity::{EntityRef, SecondaryMap};
use crate::inst_predicates::visit_block_succs;
use crate::ir::{Block, Function, Inst, Opcode};
+use crate::loop_analysis::{Loop, LoopAnalysis};
use crate::{FxHashMap, FxHashSet};
use crate::{machinst::*, trace};
@@ -148,6 +149,7 @@ impl BlockLoweringOrder {
pub fn new(
f: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
ctrl_plane: &mut ControlPlane,
) -> BlockLoweringOrder {
trace!("BlockLoweringOrder: function body {:?}", f);
@@ -200,11 +202,8 @@ impl BlockLoweringOrder {
// lowering order, identifying critical edges to split along the way.
let mut lowered_order = Vec::new();
- let mut blockindex_by_block = SecondaryMap::with_default(BlockIndex::invalid());
for &block in domtree.cfg_rpo() {
- let idx = BlockIndex::new(lowered_order.len());
lowered_order.push(LoweredBlock::Orig { block });
- blockindex_by_block[block] = idx;
if block_out_count[block] > 1 {
let range = block_succ_range[block].clone();
@@ -232,6 +231,18 @@ impl BlockLoweringOrder {
}
}
+ // Step 3: group blocks belonging to natural loops together. This
+ // significantly helps register allocator that perform live range
+ // splitting only using the linear instruction order.
+ Self::group_loops(&mut lowered_order, loop_analysis);
+
+ // Step 4: create a mapping between blocks and lowered blocks.
+ let mut blockindex_by_block = SecondaryMap::with_default(BlockIndex::invalid());
+ for (i, lb) in lowered_order.iter().enumerate() {
+ if let LoweredBlock::Orig { block } = *lb {
+ blockindex_by_block[block] = BlockIndex::new(i);
+ }
+ }
let lb_to_bindex = FxHashMap::from_iter(
lowered_order
.iter()
@@ -239,7 +250,7 @@ impl BlockLoweringOrder {
.map(|(i, &lb)| (lb, BlockIndex::new(i))),
);
- // Step 3: build the successor tables given the lowering order. We can't perform this step
+ // Step 5: build the successor tables given the lowering order. We can't perform this step
// during the creation of `lowering_order`, as we need `lb_to_bindex` to be fully populated
// first.
let mut lowered_succ_indices = Vec::new();
@@ -310,6 +321,86 @@ impl BlockLoweringOrder {
result
}
+ /// Group basic blocks from natural loops contiguously, moving any exit
+ /// blocks after the main loop body. Since natural loops are dominated by
+ /// their loop header, this maintains the dominance order required by
+ /// lowering, which visits blocks in reverse order to see uses before defs.
+ fn group_loops(order: &mut Vec<LoweredBlock>, loops: &LoopAnalysis) {
+ if !loops.is_valid() || loops.loops().len() == 0 {
+ return;
+ }
+
+ #[derive(Clone, Copy)]
+ enum Item {
+ Block(LoweredBlock),
+ Loop(usize),
+ }
+
+ // Get a loop index for each loop, with index 0 representing blocks
+ // outside any loop.
+ let loop_index = |lp: Option<Loop>| match lp {
+ Some(lp) => lp.index() + 1,
+ None => 0,
+ };
+
+ // Collect the set of blocks belonging to each loop, while otherwise
+ // preserving the lowered block order.
+ let mut loop_blocks = vec![SmallVec::<[Item; 8]>::new(); loops.loops().len() + 1];
+ for node in order.drain(..) {
+ let innermost_loop = match node {
+ LoweredBlock::Orig { block } => {
+ let lp = loops.innermost_loop(block);
+ let loop_idx = loop_index(lp);
+
+ // The first block of a loop is always the loop header.
+ // Since loop analysis only records natural loops with a
+ // single entry point and not irreducible cycles, this is
+ // guaranteed to dominate all of the other blocks in the
+ // loop. When we encounter it, insert a placeholder for the
+ // loop in the parent loop.
+ if let Some(lp) = lp {
+ if loop_blocks[loop_idx].is_empty() {
+ debug_assert_eq!(loops.loop_header(lp), block);
+ loop_blocks[loop_index(loops.loop_parent(lp))]
+ .push(Item::Loop(loop_idx));
+ }
+ }
+
+ loop_idx
+ }
+ LoweredBlock::CriticalEdge {
+ pred,
+ succ,
+ succ_idx: _,
+ } => {
+ // An edge belongs to the common loop region of its
+ // endpoints. Exit stubs move outside the loop they leave;
+ // incoming stubs stay outside and backedge stubs stay in.
+ //
+ // A critical edge can never be a loop header.
+ match (loops.innermost_loop(pred), loops.innermost_loop(succ)) {
+ (Some(a), Some(b)) => loop_index(loops.common_ancestor(a, b)),
+ _ => 0,
+ }
+ }
+ };
+ loop_blocks[innermost_loop].push(Item::Block(node));
+ }
+
+ // Starting from the root loop, add all blocks in order, recursively
+ // expanding inner loops where `Item::Loop` is found.
+ let mut stack: SmallVec<[_; 8]> = smallvec![loop_blocks[0].iter()];
+ while let Some(iter) = stack.last_mut() {
+ match iter.next() {
+ Some(&Item::Block(block)) => order.push(block),
+ Some(&Item::Loop(child)) => stack.push(loop_blocks[child].iter()),
+ None => {
+ stack.pop();
+ }
+ }
+ }
+ }
+
/// Get the lowered order of blocks.
pub fn lowered_order(&self) -> &[LoweredBlock] {
&self.lowered_order[..]
@@ -394,8 +485,81 @@ mod test {
let mut cfg = ControlFlowGraph::new();
cfg.compute(&func);
let dom_tree = DominatorTree::with_function(&func, &cfg);
+ let mut loops = LoopAnalysis::new();
+ loops.compute(&func, &cfg, &dom_tree);
+
+ BlockLoweringOrder::new(&func, &dom_tree, &loops, &mut Default::default())
+ }
+
+ #[test]
+ fn multiple_latches_and_separate_exits() {
+ // 0
+ // |
+ // +---->1<----+
+ // | / \ |
+ // | 2 3 |
+ // +---| |---+
+ // | |
+ // 4 5
+ // \ /
+ // 6
+ //
+ // (2 -> 1 and 3 -> 1 are critical backedges and must be split.)
+ //
+ let order = build_test_func(
+ 7,
+ &[
+ (0, 1),
+ (1, 2),
+ (1, 3),
+ (2, 1),
+ (2, 4),
+ (3, 1),
+ (3, 5),
+ (4, 6),
+ (5, 6),
+ ],
+ );
- BlockLoweringOrder::new(&func, &dom_tree, &mut Default::default())
+ // Both latches and their split backedges stay with the loop; the
+ // separate exits follow the complete loop body.
+ use LoweredBlock::{CriticalEdge, Orig};
+ assert_eq!(
+ order.lowered_order,
+ [
+ Orig {
+ block: Block::new(0)
+ },
+ Orig {
+ block: Block::new(1)
+ },
+ Orig {
+ block: Block::new(3)
+ },
+ CriticalEdge {
+ pred: Block::new(3),
+ succ: Block::new(1),
+ succ_idx: 0,
+ },
+ Orig {
+ block: Block::new(2)
+ },
+ CriticalEdge {
+ pred: Block::new(2),
+ succ: Block::new(1),
+ succ_idx: 0,
+ },
+ Orig {
+ block: Block::new(5)
+ },
+ Orig {
+ block: Block::new(4)
+ },
+ Orig {
+ block: Block::new(6)
+ },
+ ]
+ );
}
#[test]
diff --git a/cranelift/codegen/src/machinst/compile.rs b/cranelift/codegen/src/machinst/compile.rs
index 72109af1a4..4d1a0a8fcf 100644
--- a/cranelift/codegen/src/machinst/compile.rs
+++ b/cranelift/codegen/src/machinst/compile.rs
@@ -3,6 +3,7 @@
use crate::dominator_tree::DominatorTree;
use crate::ir::Function;
use crate::isa::TargetIsa;
+use crate::loop_analysis::LoopAnalysis;
use crate::machinst::*;
use crate::settings::RegallocAlgorithm;
use crate::timing;
@@ -15,6 +16,7 @@ use regalloc2::{Algorithm, RegallocOptions};
pub fn compile<B: LowerBackend + TargetIsa>(
f: &Function,
domtree: &DominatorTree,
+ loop_analysis: &LoopAnalysis,
regalloc_ctx: &mut regalloc2::Ctx,
b: &B,
abi: Callee<<<B as LowerBackend>::MInst as MachInst>::ABIMachineSpec>,
@@ -23,7 +25,7 @@ pub fn compile<B: LowerBackend + TargetIsa>(
ctrl_plane: &mut ControlPlane,
) -> CodegenResult<VCode<B::MInst>> {
// Compute lowered block order.
- let block_order = BlockLoweringOrder::new(f, domtree, ctrl_plane);
+ let block_order = BlockLoweringOrder::new(f, domtree, loop_analysis, ctrl_plane);
// Build the lowering context.
let lower =