A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift: Group blocks belonging to the same loop together during lowering (#15161)
Amanieu d'Antras6049b81e0356
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rsindex 7d6026c97b..472f3d9db4 100644--- a/cranelift/codegen/src/context.rs+++ b/cranelift/codegen/src/context.rs@@ -143,6 +143,7 @@ impl Context {result = isa.compile_function(&self.func,&self.domtree,+ &self.loop_analysis,&mut self.regalloc_ctx,self.want_disasm,ctrl_plane,@@ -390,6 +391,7 @@ impl Context {// Branch optimizations can invalidate these; recompute them.self.compute_cfg();self.compute_domtree();+ self.compute_loop_analysis();self.verify_if(fisa)?;diff --git a/cranelift/codegen/src/isa/aarch64/mod.rs b/cranelift/codegen/src/isa/aarch64/mod.rsindex f419633929..7ad618b030 100644--- a/cranelift/codegen/src/isa/aarch64/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/mod.rs@@ -6,6 +6,7 @@ use crate::isa::aarch64::settings as aarch64_settings;#[cfg(feature = "unwind")]use crate::isa::unwind::systemv;use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, TargetIsa};+use crate::loop_analysis::LoopAnalysis;use crate::machinst::CompiledCode;use crate::machinst::{CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,@@ -54,6 +55,7 @@ impl AArch64Backend {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,ctrl_plane: &mut ControlPlane,) -> CodegenResult<VCode<inst::Inst>> {@@ -63,6 +65,7 @@ impl AArch64Backend {compile::compile::<AArch64Backend>(func,domtree,+ loop_analysis,regalloc_ctx,self,abi,@@ -78,11 +81,12 @@ impl TargetIsa for AArch64Backend {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,want_disasm: bool,ctrl_plane: &mut ControlPlane,) -> CodegenResult<CompiledCodeStencil> {- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;let emit_result = vcode.emit(®alloc_ctx.output, want_disasm, &self.flags, ctrl_plane)?;let value_labels_ranges = emit_result.value_labels_ranges;diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rsindex dfc5277d18..f6ea84668a 100644--- a/cranelift/codegen/src/isa/mod.rs+++ b/cranelift/codegen/src/isa/mod.rs@@ -45,6 +45,7 @@use crate::dominator_tree::DominatorTree;pub use crate::isa::call_conv::CallConv;+use crate::loop_analysis::LoopAnalysis;use crate::CodegenResult;use crate::ir::{self, Function, Type};@@ -310,6 +311,7 @@ pub trait TargetIsa: fmt::Display + Send + Sync {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,want_disasm: bool,ctrl_plane: &mut ControlPlane,diff --git a/cranelift/codegen/src/isa/pulley_shared/mod.rs b/cranelift/codegen/src/isa/pulley_shared/mod.rsindex 4350d0d183..7da1536f33 100644--- a/cranelift/codegen/src/isa/pulley_shared/mod.rs+++ b/cranelift/codegen/src/isa/pulley_shared/mod.rs@@ -12,6 +12,7 @@ use crate::{dominator_tree::DominatorTree,ir,isa::{IsaFlagsHashKey, OwnedTargetIsa, TargetIsa},+ loop_analysis::LoopAnalysis,machinst::{self, CompiledCode, CompiledCodeStencil, MachInst, SigSet, VCode},result::CodegenResult,settings::{self as shared_settings, Flags},@@ -122,6 +123,7 @@ where&self,func: &ir::Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,ctrl_plane: &mut ControlPlane,) -> CodegenResult<VCode<inst::InstAndKind<P>>> {@@ -135,6 +137,7 @@ wheremachinst::compile::<Self>(func,domtree,+ loop_analysis,regalloc_ctx,self,abi,@@ -182,11 +185,12 @@ where&self,func: &ir::Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,want_disasm: bool,ctrl_plane: &mut cranelift_control::ControlPlane,) -> CodegenResult<CompiledCodeStencil> {- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;let want_disasm =want_disasm || (cfg!(feature = "trace-log") && log::log_enabled!(log::Level::Debug));diff --git a/cranelift/codegen/src/isa/riscv64/mod.rs b/cranelift/codegen/src/isa/riscv64/mod.rsindex a2c94c6cff..39ef5c2c0f 100644--- a/cranelift/codegen/src/isa/riscv64/mod.rs+++ b/cranelift/codegen/src/isa/riscv64/mod.rs@@ -6,6 +6,7 @@ use crate::isa::riscv64::settings as riscv_settings;use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, OwnedTargetIsa, TargetIsa,};+use crate::loop_analysis::LoopAnalysis;use crate::machinst::CompiledCode;use crate::machinst::{CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet, TextSectionBuilder, VCode,@@ -68,6 +69,7 @@ impl Riscv64Backend {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,ctrl_plane: &mut ControlPlane,) -> CodegenResult<VCode<inst::Inst>> {@@ -77,6 +79,7 @@ impl Riscv64Backend {compile::compile::<Riscv64Backend>(func,domtree,+ loop_analysis,regalloc_ctx,self,abi,@@ -92,11 +95,12 @@ impl TargetIsa for Riscv64Backend {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,want_disasm: bool,ctrl_plane: &mut ControlPlane,) -> CodegenResult<CompiledCodeStencil> {- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;let want_disasm = want_disasm || log::log_enabled!(log::Level::Debug);let emit_result = vcode.emit(®alloc_ctx.output, want_disasm, &self.flags, ctrl_plane)?;diff --git a/cranelift/codegen/src/isa/s390x/mod.rs b/cranelift/codegen/src/isa/s390x/mod.rsindex 2f8200644e..cbbecf6790 100644--- a/cranelift/codegen/src/isa/s390x/mod.rs+++ b/cranelift/codegen/src/isa/s390x/mod.rs@@ -6,6 +6,7 @@ use crate::isa::s390x::settings as s390x_settings;#[cfg(feature = "unwind")]use crate::isa::unwind::systemv::RegisterMappingError;use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey, TargetIsa};+use crate::loop_analysis::LoopAnalysis;use crate::machinst::{CompiledCode, CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet,TextSectionBuilder, VCode, compile,@@ -53,6 +54,7 @@ impl S390xBackend {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,ctrl_plane: &mut ControlPlane,) -> CodegenResult<VCode<inst::Inst>> {@@ -62,6 +64,7 @@ impl S390xBackend {compile::compile::<S390xBackend>(func,domtree,+ loop_analysis,regalloc_ctx,self,abi,@@ -77,12 +80,13 @@ impl TargetIsa for S390xBackend {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,want_disasm: bool,ctrl_plane: &mut ControlPlane,) -> CodegenResult<CompiledCodeStencil> {let flags = self.flags();- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;let emit_result = vcode.emit(®alloc_ctx.output, want_disasm, flags, ctrl_plane)?;let value_labels_ranges = emit_result.value_labels_ranges;diff --git a/cranelift/codegen/src/isa/x64/mod.rs b/cranelift/codegen/src/isa/x64/mod.rsindex 3b1f9afd8d..7dc8765a79 100644--- a/cranelift/codegen/src/isa/x64/mod.rs+++ b/cranelift/codegen/src/isa/x64/mod.rs@@ -10,6 +10,7 @@ use crate::ir::{self, Function, Type, types};use crate::isa::unwind::systemv;use crate::isa::x64::settings as x64_settings;use crate::isa::{Builder as IsaBuilder, FunctionAlignment, IsaFlagsHashKey};+use crate::loop_analysis::LoopAnalysis;use crate::machinst::{CompiledCode, CompiledCodeStencil, MachInst, MachTextSectionBuilder, Reg, SigSet,TextSectionBuilder, VCode, compile,@@ -61,6 +62,7 @@ impl X64Backend {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,ctrl_plane: &mut ControlPlane,) -> CodegenResult<VCode<inst::Inst>> {@@ -72,6 +74,7 @@ impl X64Backend {compile::compile::<Self>(func,domtree,+ loop_analysis,regalloc_ctx,self,abi,@@ -87,11 +90,12 @@ impl TargetIsa for X64Backend {&self,func: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,want_disasm: bool,ctrl_plane: &mut ControlPlane,) -> CodegenResult<CompiledCodeStencil> {- let vcode = self.compile_vcode(func, domtree, regalloc_ctx, ctrl_plane)?;+ let vcode = self.compile_vcode(func, domtree, loop_analysis, regalloc_ctx, ctrl_plane)?;let emit_result = vcode.emit(®alloc_ctx.output, want_disasm, &self.flags, ctrl_plane)?;let value_labels_ranges = emit_result.value_labels_ranges;diff --git a/cranelift/codegen/src/loop_analysis.rs b/cranelift/codegen/src/loop_analysis.rsindex 5d88c519af..f5b4b2069c 100644--- a/cranelift/codegen/src/loop_analysis.rs+++ b/cranelift/codegen/src/loop_analysis.rs@@ -1,6 +1,8 @@//! A loop analysis represented as mappings of loops to their header Block//! and parent in the loop tree.+use core::cmp::Ordering;+use crate::dominator_tree::DominatorTree;use crate::entity::SecondaryMap;use crate::entity::entity_impl;@@ -13,7 +15,7 @@ use alloc::vec::Vec;use smallvec::SmallVec;/// A opaque reference to a code loop.-#[derive(Copy, Clone, PartialEq, Eq, Hash)]+#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]pub struct Loop(u32);entity_impl!(Loop, "loop");@@ -115,6 +117,20 @@ impl LoopAnalysis {self.loops[lp].parent.expand()}+ /// Return the innermost common ancestor of two loops.+ pub fn common_ancestor(&self, mut a: Loop, mut b: Loop) -> Option<Loop> {+ loop {+ // Because loops are ordered in RPO, a parent loop is guaranteed to+ // have a lower index than a child loop. We can just walk up the+ // tree until we find a common loop.+ match a.cmp(&b) {+ Ordering::Less => b = self.loop_parent(b)?,+ Ordering::Equal => return Some(a),+ Ordering::Greater => a = self.loop_parent(a)?,+ }+ }+ }+/// Return the innermost loop for a given block.pub fn innermost_loop(&self, block: Block) -> Option<Loop> {self.block_loop_map[block].expand()@@ -364,6 +380,18 @@ mod tests {assert_eq!(loop_analysis.loop_level(block1).level(), 2);assert_eq!(loop_analysis.loop_level(block2).level(), 2);assert_eq!(loop_analysis.loop_level(block3).level(), 1);+ assert_eq!(+ loop_analysis.common_ancestor(loops[0], loops[1]),+ Some(loops[0])+ );+ assert_eq!(+ loop_analysis.common_ancestor(loops[1], loops[0]),+ Some(loops[0])+ );+ assert_eq!(+ loop_analysis.common_ancestor(loops[1], loops[1]),+ Some(loops[1])+ );}#[test]@@ -428,5 +456,13 @@ mod tests {assert_eq!(loop_analysis.loop_level(block3).level(), 2);assert_eq!(loop_analysis.loop_level(block4).level(), 2);assert_eq!(loop_analysis.loop_level(block5).level(), 1);+ assert_eq!(+ loop_analysis.common_ancestor(loops[1], loops[2]),+ Some(loops[0])+ );+ assert_eq!(+ loop_analysis.common_ancestor(loops[2], loops[1]),+ Some(loops[0])+ );}}diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rsindex 08c014ebd5..6e7c088c2b 100644--- a/cranelift/codegen/src/machinst/blockorder.rs+++ b/cranelift/codegen/src/machinst/blockorder.rs@@ -61,9 +61,10 @@//! the other redundancies that this scheme produces.use crate::dominator_tree::DominatorTree;-use crate::entity::SecondaryMap;+use crate::entity::{EntityRef, SecondaryMap};use crate::inst_predicates::visit_block_succs;use crate::ir::{Block, Function, Inst, Opcode};+use crate::loop_analysis::{Loop, LoopAnalysis};use crate::{FxHashMap, FxHashSet};use crate::{machinst::*, trace};@@ -148,6 +149,7 @@ impl BlockLoweringOrder {pub fn new(f: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,ctrl_plane: &mut ControlPlane,) -> BlockLoweringOrder {trace!("BlockLoweringOrder: function body {:?}", f);@@ -200,11 +202,8 @@ impl BlockLoweringOrder {// lowering order, identifying critical edges to split along the way.let mut lowered_order = Vec::new();- let mut blockindex_by_block = SecondaryMap::with_default(BlockIndex::invalid());for &block in domtree.cfg_rpo() {- let idx = BlockIndex::new(lowered_order.len());lowered_order.push(LoweredBlock::Orig { block });- blockindex_by_block[block] = idx;if block_out_count[block] > 1 {let range = block_succ_range[block].clone();@@ -232,6 +231,18 @@ impl BlockLoweringOrder {}}+ // Step 3: group blocks belonging to natural loops together. This+ // significantly helps register allocator that perform live range+ // splitting only using the linear instruction order.+ Self::group_loops(&mut lowered_order, loop_analysis);++ // Step 4: create a mapping between blocks and lowered blocks.+ let mut blockindex_by_block = SecondaryMap::with_default(BlockIndex::invalid());+ for (i, lb) in lowered_order.iter().enumerate() {+ if let LoweredBlock::Orig { block } = *lb {+ blockindex_by_block[block] = BlockIndex::new(i);+ }+ }let lb_to_bindex = FxHashMap::from_iter(lowered_order.iter()@@ -239,7 +250,7 @@ impl BlockLoweringOrder {.map(|(i, &lb)| (lb, BlockIndex::new(i))),);- // Step 3: build the successor tables given the lowering order. We can't perform this step+ // Step 5: build the successor tables given the lowering order. We can't perform this step// during the creation of `lowering_order`, as we need `lb_to_bindex` to be fully populated// first.let mut lowered_succ_indices = Vec::new();@@ -310,6 +321,86 @@ impl BlockLoweringOrder {result}+ /// Group basic blocks from natural loops contiguously, moving any exit+ /// blocks after the main loop body. Since natural loops are dominated by+ /// their loop header, this maintains the dominance order required by+ /// lowering, which visits blocks in reverse order to see uses before defs.+ fn group_loops(order: &mut Vec<LoweredBlock>, loops: &LoopAnalysis) {+ if !loops.is_valid() || loops.loops().len() == 0 {+ return;+ }++ #[derive(Clone, Copy)]+ enum Item {+ Block(LoweredBlock),+ Loop(usize),+ }++ // Get a loop index for each loop, with index 0 representing blocks+ // outside any loop.+ let loop_index = |lp: Option<Loop>| match lp {+ Some(lp) => lp.index() + 1,+ None => 0,+ };++ // Collect the set of blocks belonging to each loop, while otherwise+ // preserving the lowered block order.+ let mut loop_blocks = vec![SmallVec::<[Item; 8]>::new(); loops.loops().len() + 1];+ for node in order.drain(..) {+ let innermost_loop = match node {+ LoweredBlock::Orig { block } => {+ let lp = loops.innermost_loop(block);+ let loop_idx = loop_index(lp);++ // The first block of a loop is always the loop header.+ // Since loop analysis only records natural loops with a+ // single entry point and not irreducible cycles, this is+ // guaranteed to dominate all of the other blocks in the+ // loop. When we encounter it, insert a placeholder for the+ // loop in the parent loop.+ if let Some(lp) = lp {+ if loop_blocks[loop_idx].is_empty() {+ debug_assert_eq!(loops.loop_header(lp), block);+ loop_blocks[loop_index(loops.loop_parent(lp))]+ .push(Item::Loop(loop_idx));+ }+ }++ loop_idx+ }+ LoweredBlock::CriticalEdge {+ pred,+ succ,+ succ_idx: _,+ } => {+ // An edge belongs to the common loop region of its+ // endpoints. Exit stubs move outside the loop they leave;+ // incoming stubs stay outside and backedge stubs stay in.+ //+ // A critical edge can never be a loop header.+ match (loops.innermost_loop(pred), loops.innermost_loop(succ)) {+ (Some(a), Some(b)) => loop_index(loops.common_ancestor(a, b)),+ _ => 0,+ }+ }+ };+ loop_blocks[innermost_loop].push(Item::Block(node));+ }++ // Starting from the root loop, add all blocks in order, recursively+ // expanding inner loops where `Item::Loop` is found.+ let mut stack: SmallVec<[_; 8]> = smallvec![loop_blocks[0].iter()];+ while let Some(iter) = stack.last_mut() {+ match iter.next() {+ Some(&Item::Block(block)) => order.push(block),+ Some(&Item::Loop(child)) => stack.push(loop_blocks[child].iter()),+ None => {+ stack.pop();+ }+ }+ }+ }+/// Get the lowered order of blocks.pub fn lowered_order(&self) -> &[LoweredBlock] {&self.lowered_order[..]@@ -394,8 +485,81 @@ mod test {let mut cfg = ControlFlowGraph::new();cfg.compute(&func);let dom_tree = DominatorTree::with_function(&func, &cfg);+ let mut loops = LoopAnalysis::new();+ loops.compute(&func, &cfg, &dom_tree);++ BlockLoweringOrder::new(&func, &dom_tree, &loops, &mut Default::default())+ }++ #[test]+ fn multiple_latches_and_separate_exits() {+ // 0+ // |+ // +---->1<----++ // | / \ |+ // | 2 3 |+ // +---| |---++ // | |+ // 4 5+ // \ /+ // 6+ //+ // (2 -> 1 and 3 -> 1 are critical backedges and must be split.)+ //+ let order = build_test_func(+ 7,+ &[+ (0, 1),+ (1, 2),+ (1, 3),+ (2, 1),+ (2, 4),+ (3, 1),+ (3, 5),+ (4, 6),+ (5, 6),+ ],+ );- BlockLoweringOrder::new(&func, &dom_tree, &mut Default::default())+ // Both latches and their split backedges stay with the loop; the+ // separate exits follow the complete loop body.+ use LoweredBlock::{CriticalEdge, Orig};+ assert_eq!(+ order.lowered_order,+ [+ Orig {+ block: Block::new(0)+ },+ Orig {+ block: Block::new(1)+ },+ Orig {+ block: Block::new(3)+ },+ CriticalEdge {+ pred: Block::new(3),+ succ: Block::new(1),+ succ_idx: 0,+ },+ Orig {+ block: Block::new(2)+ },+ CriticalEdge {+ pred: Block::new(2),+ succ: Block::new(1),+ succ_idx: 0,+ },+ Orig {+ block: Block::new(5)+ },+ Orig {+ block: Block::new(4)+ },+ Orig {+ block: Block::new(6)+ },+ ]+ );}#[test]diff --git a/cranelift/codegen/src/machinst/compile.rs b/cranelift/codegen/src/machinst/compile.rsindex 72109af1a4..4d1a0a8fcf 100644--- a/cranelift/codegen/src/machinst/compile.rs+++ b/cranelift/codegen/src/machinst/compile.rs@@ -3,6 +3,7 @@use crate::dominator_tree::DominatorTree;use crate::ir::Function;use crate::isa::TargetIsa;+use crate::loop_analysis::LoopAnalysis;use crate::machinst::*;use crate::settings::RegallocAlgorithm;use crate::timing;@@ -15,6 +16,7 @@ use regalloc2::{Algorithm, RegallocOptions};pub fn compile<B: LowerBackend + TargetIsa>(f: &Function,domtree: &DominatorTree,+ loop_analysis: &LoopAnalysis,regalloc_ctx: &mut regalloc2::Ctx,b: &B,abi: Callee<<<B as LowerBackend>::MInst as MachInst>::ABIMachineSpec>,@@ -23,7 +25,7 @@ pub fn compile<B: LowerBackend + TargetIsa>(ctrl_plane: &mut ControlPlane,) -> CodegenResult<VCode<B::MInst>> {// Compute lowered block order.- let block_order = BlockLoweringOrder::new(f, domtree, ctrl_plane);+ let block_order = BlockLoweringOrder::new(f, domtree, loop_analysis, ctrl_plane);// Build the lowering context.let lower =