A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: A few perf improvements for the dead-store elimination pass (#13947)
Nick Fitzgeraldd61adfecb9e2
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rsindex 5f224871df..aae88ec46a 100644--- a/cranelift/codegen/src/alias_analysis.rs+++ b/cranelift/codegen/src/alias_analysis.rs@@ -72,12 +72,13 @@ use crate::{FxHashMap, FxHashSet};use crate::{cursor::{Cursor, FuncCursor},dominator_tree::DominatorTree,+ flowgraph::ControlFlowGraph,inst_predicates::{inst_addr_offset_type, inst_store_data, visit_block_succs},ir::{AliasRegion, Block, Function, Inst, Opcode, Type, Value, immediates::Offset32},post_dominator_tree::PostDominatorTree,trace,};-use cranelift_entity::EntitySet;+use core::cmp::Ordering;use cranelift_entity::{EntityRef, SecondaryMap, packed_option::PackedOption};/// Determine whether this opcode behaves as a memory fence, i.e.,@@ -109,8 +110,8 @@ enum AliasRegionsObserved {}/// Which alias region(s) can an instruction observe?-fn alias_regions_observed(func: &Function, inst: Inst) -> AliasRegionsObserved {- let opcode = func.dfg.insts[inst].opcode();+fn alias_regions_observed(func: &Function, inst: Inst, opcode: Opcode) -> AliasRegionsObserved {+ debug_assert_eq!(func.dfg.insts[inst].opcode(), opcode);if opcode.is_return()|| opcode.is_call()|| opcode.can_trap()@@ -136,16 +137,24 @@ fn alias_regions_observed(func: &Function, inst: Inst) -> AliasRegionsObserved {}}+/// The last-store state for a single named alias region: the last store to the+/// region (if any) and whether that region has been observed since.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]+struct RegionState {+ /// Last store to this region.+ last_store: PackedOption<Inst>,++ /// Whether this region has been observed since it was last stored to.+ observed: bool,+}+/// For a given program point, the vector of last-store instruction/// indices for each disjoint category of abstract state.#[derive(Clone, Debug, Default, PartialEq, Eq)]pub struct LastStores {- /// Last store for each named alias region.- regions: SecondaryMap<AliasRegion, PackedOption<Inst>>,-- /// Whether each region has been observed or not since it was last stored- /// to.- observed_regions: EntitySet<AliasRegion>,+ /// Last store (and whether it has been observed) for each named alias+ /// region.+ regions: SecondaryMap<AliasRegion, RegionState>,/// Last store for memory accesses with no alias region.other: PackedOption<Inst>,@@ -186,8 +195,10 @@ impl LastStores {if let Some(memflags) = func.dfg.insts[inst].memflags() {match func.dfg.mem_flags[memflags].alias_region() {Some(region) => {- self.regions[region] = inst.into();- self.observed_regions.remove(region);+ self.regions[region] = RegionState {+ last_store: inst.into(),+ observed: false,+ };// And if this store can trap, then we need to observe// all other alias regions, to ensure that their state@@ -239,12 +250,12 @@ impl LastStores {// Everything else: determine which, if any, alias regions this// instruction observes.else {- match alias_regions_observed(func, inst) {+ match alias_regions_observed(func, inst, opcode) {AliasRegionsObserved::All => {self.observed_all = true;}AliasRegionsObserved::Just(region) => {- self.observed_regions.insert(region);+ self.regions[region].observed = true;// NB: Because stores without regions may alias any other// region, we have also observed the last-store in// `self.other`.@@ -260,9 +271,9 @@ impl LastStores {/// Mark all regions except for `excluding` (if given) as observed.fn observe_others(&mut self, excluding: Option<AliasRegion>) {- for (region, last_store) in self.regions.iter() {- if last_store.is_some() && excluding.is_none_or(|r| r != region) {- self.observed_regions.insert(region);+ for (region, state) in self.regions.iter_mut() {+ if state.last_store.is_some() && excluding.is_none_or(|r| r != region) {+ state.observed = true;}}self.observed_other = true;@@ -271,13 +282,14 @@ impl LastStores {/// Mark all regions whose last-store can trap as observed, except for/// `excluding`.fn observe_trapping_others(&mut self, excluding: AliasRegion, func: &Function) {- for (region, last_store) in self.regions.iter() {+ for (region, state) in self.regions.iter_mut() {if region != excluding- && last_store+ && state+ .last_store.expand().is_some_and(|s| func.dfg.insts[s].memflags_trap_code(&func.dfg).is_some()){- self.observed_regions.insert(region);+ state.observed = true;}}@@ -290,7 +302,6 @@ impl LastStores {/// Handle memory fence-like instructions by clearing all analysis data.fn fence(&mut self, inst: Inst) {self.regions.clear();- self.observed_regions.clear();self.other = inst.into();self.observed_other = false;self.last_fence = inst.into();@@ -304,15 +315,15 @@ impl LastStores {match func.dfg.mem_flags[memflags].alias_region() {None => return (self.other, self.observed_all || self.observed_other),Some(region) => {- let region_store = self.regions[region];+ let region_state = self.regions[region];// If the region has never been explicitly stored to,// fall back to the last fence (which affects all regions).- if region_store.is_none() {+ if region_state.last_store.is_none() {return (self.last_fence, self.observed_all);} else {return (- region_store,- self.observed_all || self.observed_regions.contains(region),+ region_state.last_store,+ self.observed_all || region_state.observed,);}}@@ -338,7 +349,6 @@ impl LastStores {// field.let LastStores {regions,- observed_regions,other,observed_other,last_fence,@@ -356,17 +366,6 @@ impl LastStores {old != new};- let union_set_entry = |a: &mut EntitySet<AliasRegion>,- b: &EntitySet<AliasRegion>,- region: AliasRegion|- -> bool {- if b.contains(region) {- a.insert(region)- } else {- false- }- };-let union_bool = |a: &mut bool, b: bool| -> bool {let old = *a;*a = *a || b;@@ -378,8 +377,12 @@ impl LastStores {let max_len = core::cmp::max(regions.keys().len(), rhs.regions.keys().len());for i in 0..max_len {let region = AliasRegion::new(i);- changed |= meet(&mut regions[region], rhs.regions[region]);- changed |= union_set_entry(observed_regions, &rhs.observed_regions, region);+ let rhs_state = rhs.regions[region];+ let state = &mut regions[region];+ changed |= meet(&mut state.last_store, rhs_state.last_store);+ // Union the observed bit: a region is observed after the meet if it+ // was observed on either incoming path.+ changed |= union_bool(&mut state.observed, rhs_state.observed);}changed |= meet(other, rhs.other);@@ -447,7 +450,15 @@ pub struct AliasAnalysis<'a> {domtree: &'a DominatorTree,/// The post-dominator tree for the function.- post_dom_tree: &'a PostDominatorTree,+ ///+ /// This is computed lazily, on the first cross-block dead-store candidate,+ /// because it is only ever consulted by dead-store elimination and only for+ /// candidates whose two stores live in different blocks (because we can+ /// easily test post-domination without building a post-dominator tree when+ /// the two instructions are in the same block). The majority of functions+ /// have no such dead-store candidates, so building it eagerly for every+ /// function is a waste.+ post_dom_tree: Option<PostDominatorTree>,/// Input state to a basic block.block_input: FxHashMap<Block, LastStores>,@@ -462,17 +473,12 @@ pub struct AliasAnalysis<'a> {impl<'a> AliasAnalysis<'a> {/// Perform an alias analysis pass.- pub fn new(- func: &Function,- domtree: &'a DominatorTree,- post_dom_tree: &'a PostDominatorTree,- ) -> AliasAnalysis<'a> {+ pub fn new(func: &Function, domtree: &'a DominatorTree) -> AliasAnalysis<'a> {trace!("alias analysis: input is:\n{:?}", func);assert!(domtree.is_valid());- assert!(post_dom_tree.is_valid());let mut analysis = AliasAnalysis {domtree,- post_dom_tree,+ post_dom_tree: None,block_input: FxHashMap::default(),mem_values: FxHashMap::default(),};@@ -481,6 +487,34 @@ impl<'a> AliasAnalysis<'a> {analysis}+ /// Does `overwriter` post-dominate `maybe_dead`?+ ///+ /// That is, does every path from `maybe_dead` out of the function pass+ /// through `overwriter`? Used as part of deciding whether `maybe_dead` is+ /// truly a dead store.+ fn post_dominates_maybe_dead_store(+ &mut self,+ func: &Function,+ cfg: &ControlFlowGraph,+ overwriter: Inst,+ maybe_dead: Inst,+ ) -> bool {+ let overwriter_block = func.layout.inst_block(overwriter).unwrap();+ let maybe_dead_block = func.layout.inst_block(maybe_dead).unwrap();++ // When our instructions are in the same block, we do not need to+ // force computation of the whole post-dominator tree: `overwriter`+ // post-dominates `maybe_dead` iff `overwriter` is at or after+ // `maybe_dead`.+ if overwriter_block == maybe_dead_block {+ return func.layout.pp_cmp(overwriter, maybe_dead) != Ordering::Less;+ }++ self.post_dom_tree+ .get_or_insert_with(|| PostDominatorTree::with_cfg(cfg))+ .post_dominates(overwriter, maybe_dead, &func.layout)+ }+fn compute_block_input_states(&mut self, func: &Function) {let mut queue = vec![];let mut queue_set = FxHashSet::default();@@ -538,6 +572,7 @@ impl<'a> AliasAnalysis<'a> {pub fn process_inst(&mut self,func: &mut Function,+ cfg: &ControlFlowGraph,state: &mut LastStores,inst: Inst,) -> OptResult {@@ -584,9 +619,7 @@ impl<'a> AliasAnalysis<'a> {// The last store is only dead if all paths// out of the function from it go through this// instruction.- && self- .post_dom_tree- .post_dominates(inst, last_store, &func.layout)+ && self.post_dominates_maybe_dead_store(func, cfg, inst, last_store){trace!(" --> discovered dead store at {last_store}: {}",@@ -709,13 +742,13 @@ impl<'a> AliasAnalysis<'a> {/// tracking because resolving some aliases may expose others/// (e.g. in cases of double-indirection with two separate chains/// of loads).- pub fn compute_and_update_aliases(&mut self, func: &mut Function) {+ pub fn compute_and_update_aliases(&mut self, func: &mut Function, cfg: &ControlFlowGraph) {let mut pos = FuncCursor::new(func);while let Some(block) = pos.next_block() {let mut state = self.block_starting_state(block);while let Some(inst) = pos.next_inst() {- match self.process_inst(pos.func, &mut state, inst) {+ match self.process_inst(pos.func, cfg, &mut state, inst) {OptResult::None => {}OptResult::AliasedLoad(replaced_result) => {let result = pos.func.dfg.inst_results(inst)[0];diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rsindex 7b7b390597..7d6026c97b 100644--- a/cranelift/codegen/src/context.rs+++ b/cranelift/codegen/src/context.rs@@ -19,7 +19,6 @@ use crate::isa::TargetIsa;use crate::loop_analysis::LoopAnalysis;use crate::machinst::{CompiledCode, CompiledCodeStencil};use crate::nan_canonicalization::do_nan_canonicalization;-use crate::post_dominator_tree::PostDominatorTree;use crate::remove_constant_phis::do_remove_constant_phis;use crate::result::{CodegenResult, CompileResult};use crate::settings::{FlagsOrIsa, OptLevel};@@ -47,9 +46,6 @@ pub struct Context {/// Dominator tree for `func`.pub domtree: DominatorTree,- /// Post-dominator tree for `func`.- pub post_dom_tree: PostDominatorTree,-/// Loop analysis of `func`.pub loop_analysis: LoopAnalysis,@@ -81,7 +77,6 @@ impl Context {func,cfg: ControlFlowGraph::new(),domtree: DominatorTree::new(),- post_dom_tree: PostDominatorTree::new(),loop_analysis: LoopAnalysis::new(),compiled_code: None,want_disasm: false,@@ -197,7 +192,6 @@ impl Context {self.func.dfg.resolve_all_aliases();if opt_level != OptLevel::None {- self.compute_post_dom_tree();self.egraph_pass(isa, ctrl_plane)?;}@@ -318,11 +312,6 @@ impl Context {self.domtree.compute(&self.func, &self.cfg);}- /// Compute the post-dominator tree.- pub fn compute_post_dom_tree(&mut self) {- self.post_dom_tree.compute(&self.func, &self.cfg);- }-/// Compute the loop analysis.pub fn compute_loop_analysis(&mut self) {self.loop_analysis@@ -353,8 +342,8 @@ impl Context {/// by a store instruction to the same instruction (so-called/// "store-to-load forwarding").pub fn replace_redundant_loads(&mut self) -> CodegenResult<()> {- let mut analysis = AliasAnalysis::new(&self.func, &self.domtree, &self.post_dom_tree);- analysis.compute_and_update_aliases(&mut self.func);+ let mut analysis = AliasAnalysis::new(&self.func, &self.domtree);+ analysis.compute_and_update_aliases(&mut self.func, &self.cfg);Ok(())}@@ -385,7 +374,7 @@ impl Context {);let fisa = fisa.into();self.compute_loop_analysis();- let mut alias_analysis = AliasAnalysis::new(&self.func, &self.domtree, &self.post_dom_tree);+ let mut alias_analysis = AliasAnalysis::new(&self.func, &self.domtree);let mut pass = EgraphPass::new(&mut self.func,&self.domtree,diff --git a/cranelift/codegen/src/egraph/mod.rs b/cranelift/codegen/src/egraph/mod.rsindex 64464ee64e..d4a2165a4b 100644--- a/cranelift/codegen/src/egraph/mod.rs+++ b/cranelift/codegen/src/egraph/mod.rs@@ -151,6 +151,9 @@ wherepub(crate) remat_values: &'opt mut FxHashSet<Value>,pub(crate) stats: &'opt mut Stats,domtree: &'opt DominatorTree,+ /// The pre-egraph control-flow graph, used by the alias analysis to lazily+ /// build a post-dominator tree for dead-store elimination.+ cfg: &'opt ControlFlowGraph,pub(crate) alias_analysis: &'opt mut AliasAnalysis<'analysis>,pub(crate) alias_analysis_state: &'opt mut LastStores,pub(crate) branch_to_trap_analysis: &'opt mut BranchToTrapAnalysis,@@ -548,7 +551,7 @@ where// stored value, or remove an idempotent store).match self.alias_analysis- .process_inst(self.func, self.alias_analysis_state, inst)+ .process_inst(self.func, self.cfg, self.alias_analysis_state, inst){OptResult::AliasedLoad(new_result) => {self.stats.alias_analysis_removed_load += 1;@@ -987,6 +990,7 @@ impl<'a> EgraphPass<'a> {remat_values: &mut self.remat_values,stats: &mut self.stats,domtree: &self.domtree,+ cfg: &*self.cfg,alias_analysis: self.alias_analysis,alias_analysis_state: &mut alias_analysis_state,branch_to_trap_analysis: &mut self.branch_to_trap_analysis,diff --git a/cranelift/codegen/src/flowgraph.rs b/cranelift/codegen/src/flowgraph.rsindex 30a79eedf8..17fc071b15 100644--- a/cranelift/codegen/src/flowgraph.rs+++ b/cranelift/codegen/src/flowgraph.rs@@ -167,6 +167,25 @@ impl ControlFlowGraph {self.data[block].successors.iter(&self.succ_forest)}+ /// Get an iterator over all blocks this control-flow graph has storage for,+ /// in block order.+ ///+ /// This may include blocks that are not in the function's layout (e.g. a+ /// block that was created but never inserted, or has since been removed);+ /// such blocks have no predecessors or successors in the graph.+ pub fn blocks(&self) -> impl ExactSizeIterator<Item = Block> + '_ {+ self.data.keys()+ }++ /// Get the number of blocks this control-flow graph has storage for.+ ///+ /// This equals `func.dfg.num_blocks()` for the function the CFG was+ /// computed from, and is an upper bound on the block indices appearing in+ /// the graph.+ pub fn num_blocks(&self) -> usize {+ self.blocks().len()+ }+/// Check if the CFG is in a valid state.////// Note that this doesn't perform any kind of validity checks. It simply checks if thediff --git a/cranelift/codegen/src/post_dominator_tree.rs b/cranelift/codegen/src/post_dominator_tree.rsindex a718c49ced..9c08cd6bb1 100644--- a/cranelift/codegen/src/post_dominator_tree.rs+++ b/cranelift/codegen/src/post_dominator_tree.rs@@ -23,20 +23,19 @@use crate::dominator_tree::{ChildIter, DomTreeGraph, DominatorTree};use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};-use crate::ir::{Block, Function, Layout, ProgramPoint};+use crate::ir::{Block, Layout, ProgramPoint};use core::cmp::Ordering;/// The reversed control-flow graph, augmented with a virtual sink above the/// function's exit blocks. Computing a `DominatorTree` over this graph yields/// the post-dominator tree.struct ReverseGraph<'a> {- func: &'a Function,cfg: &'a ControlFlowGraph,}impl DomTreeGraph for ReverseGraph<'_> {fn num_blocks(&self) -> usize {- self.func.dfg.num_blocks()+ self.cfg.num_blocks()}fn roots(&self) -> impl Iterator<Item = Block> {@@ -45,10 +44,14 @@ impl DomTreeGraph for ReverseGraph<'_> {// `return_call`, `trap`, etc...). These are exactly the blocks with no// CFG successors, and they are precisely the blocks with an edge to the// virtual sink.- self.func- .layout+ //+ // `cfg.blocks()` may include blocks not in the layout, but those are+ // isolated in the graph (no predecessors or successors), so they only+ // ever appear as trivial single-node roots and never affect the+ // post-domination of any real block.+ self.cfg.blocks()- .filter(|&block| self.func.block_successors(block).next().is_none())+ .filter(|&block| self.cfg.succ_iter(block).next().is_none())}fn successors(&self, block: Block) -> impl Iterator<Item = Block> {@@ -62,7 +65,7 @@ impl DomTreeGraph for ReverseGraph<'_> {fn predecessors(&self, block: Block) -> impl Iterator<Item = Block> {// Edges are reversed: a predecessor in the reversed graph is a// successor in the CFG.- self.func.block_successors(block)+ self.cfg.succ_iter(block)}}@@ -90,18 +93,17 @@ impl PostDominatorTree {}/// Allocate and compute a post-dominator tree.- pub fn with_function(func: &Function, cfg: &ControlFlowGraph) -> Self {+ pub fn with_cfg(cfg: &ControlFlowGraph) -> Self {let mut post_domtree = Self::new();- post_domtree.compute(func, cfg);+ post_domtree.compute(cfg);post_domtree}- /// Reset and compute the post-dominator tree for `func`, using the- /// control-flow graph `cfg`.- pub fn compute(&mut self, func: &Function, cfg: &ControlFlowGraph) {+ /// Reset and compute the post-dominator tree from the control-flow graph+ /// `cfg`.+ pub fn compute(&mut self, cfg: &ControlFlowGraph) {debug_assert!(cfg.is_valid());- self.dom_tree- .compute_from_graph(&ReverseGraph { func, cfg });+ self.dom_tree.compute_from_graph(&ReverseGraph { cfg });}/// Clear the data structures used to represent the post-dominator@@ -216,7 +218,7 @@ mod tests {use super::*;use crate::cursor::{Cursor, FuncCursor};use crate::ir::types::*;- use crate::ir::{InstBuilder, TrapCode};+ use crate::ir::{Function, InstBuilder, TrapCode};use alloc::string::String;use alloc::vec::Vec;use mutatis::{Mutate, check::Check, mutators as m};@@ -225,7 +227,7 @@ mod tests {fn empty() {let func = Function::new();let cfg = ControlFlowGraph::with_function(&func);- let pdt = PostDominatorTree::with_function(&func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);assert!(pdt.is_valid());}@@ -240,12 +242,12 @@ mod tests {let mut pdt = PostDominatorTree::new();assert!(!pdt.is_valid());- pdt.compute(cur.func, &cfg);+ pdt.compute(&cfg);assert!(pdt.is_valid());pdt.clear();assert!(!pdt.is_valid());// Recompute after clear.- pdt.compute(cur.func, &cfg);+ pdt.compute(&cfg);assert!(pdt.is_valid());}@@ -260,7 +262,7 @@ mod tests {cur.ins().return_(&[]);let cfg = ControlFlowGraph::with_function(cur.func);- let pdt = PostDominatorTree::with_function(cur.func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);// The single block is an exit, so it has no post-dominator and does not// diverge.@@ -300,7 +302,7 @@ mod tests {cur.ins().return_(&[]);let cfg = ControlFlowGraph::with_function(cur.func);- let pdt = PostDominatorTree::with_function(cur.func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);// Every path out of the function passes through `join`.assert_eq!(pdt.immediate_post_dominator(block0), Some(join));@@ -360,7 +362,7 @@ mod tests {cur.ins().return_(&[]);let cfg = ControlFlowGraph::with_function(cur.func);- let pdt = PostDominatorTree::with_function(cur.func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);assert_eq!(pdt.immediate_post_dominator(entry), Some(header));assert_eq!(pdt.immediate_post_dominator(header), Some(exit));@@ -387,7 +389,7 @@ mod tests {cur.ins().jump(block0, &[]);let cfg = ControlFlowGraph::with_function(cur.func);- let pdt = PostDominatorTree::with_function(cur.func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);// There is no exit block, so the function never returns: every block// diverges and has no post-dominator.@@ -421,7 +423,7 @@ mod tests {cur.ins().return_(&[]);let cfg = ControlFlowGraph::with_function(cur.func);- let pdt = PostDominatorTree::with_function(cur.func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);// Only `body` diverges.assert!(pdt.diverges(body));@@ -455,7 +457,7 @@ mod tests {cur.ins().return_(&[]);let cfg = ControlFlowGraph::with_function(cur.func);- let pdt = PostDominatorTree::with_function(cur.func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);// Two distinct exit blocks: neither post-dominates the entry, and the// entry's only post-dominator is the (virtual) sink.@@ -495,7 +497,7 @@ mod tests {cur.ins().trap(TrapCode::unwrap_user(1));let cfg = ControlFlowGraph::with_function(cur.func);- let pdt = PostDominatorTree::with_function(cur.func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);// A `trap` is a function exit, so `trap_block` is a root of the forest// and does not diverge.@@ -521,7 +523,7 @@ mod tests {cur.ins().return_(&[]);let cfg = ControlFlowGraph::with_function(cur.func);- let pdt = PostDominatorTree::with_function(cur.func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);let v1_def = cur.func.dfg.value_def(v1).unwrap_inst();let v2_def = cur.func.dfg.value_def(v2).unwrap_inst();@@ -674,7 +676,7 @@ mod tests {let (func, blocks) = graph.build();let cfg = ControlFlowGraph::with_function(&func);- let pdt = PostDominatorTree::with_function(&func, &cfg);+ let pdt = PostDominatorTree::with_cfg(&cfg);// The virtual sink is node `n`. Exit blocks have an edge to it.let sink = graph.blocks.len();diff --git a/cranelift/filetests/src/test_alias_analysis.rs b/cranelift/filetests/src/test_alias_analysis.rsindex 0bb0ccf652..33c9d3e208 100644--- a/cranelift/filetests/src/test_alias_analysis.rs+++ b/cranelift/filetests/src/test_alias_analysis.rs@@ -34,7 +34,7 @@ impl SubTest for TestAliasAnalysis {let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned());comp_ctx.flowgraph();- comp_ctx.compute_post_dom_tree();+ comp_ctx.compute_domtree();comp_ctx.replace_redundant_loads().map_err(|e| crate::pretty_anyhow_error(&comp_ctx.func, Into::into(e)))?;