A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Handle irreducible control flow in branch simplification (#13391)
Nick Fitzgeralddeedb95f3b39
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rsindex d83388501b..1991ffb2be 100644--- a/cranelift/codegen/src/context.rs+++ b/cranelift/codegen/src/context.rs@@ -333,7 +333,10 @@ impl Context {whereFOI: Into<FlagsOrIsa<'a>>,{- eliminate_unreachable_code(&mut self.func, &mut self.cfg, &self.domtree);+ let domtree = &self.domtree;+ eliminate_unreachable_code(&mut self.func, &mut self.cfg, |block| {+ domtree.is_reachable(block)+ });self.verify_if(fisa)}@@ -382,6 +385,7 @@ impl Context {&self.loop_analysis,&mut alias_analysis,ctrl_plane,+ &mut self.cfg,);pass.run();log::debug!("egraph stats: {:?}", pass.stats);diff --git a/cranelift/codegen/src/dominator_tree.rs b/cranelift/codegen/src/dominator_tree.rsindex 51deba8a1b..db8efa1d4e 100644--- a/cranelift/codegen/src/dominator_tree.rs+++ b/cranelift/codegen/src/dominator_tree.rs@@ -31,8 +31,6 @@ struct SpanningTreeNode {/// Immediate dominator value for the node./// Initialized to node's ancestor in the spanning tree.idom: u32,- /// Post-order number assigned during the CFG DFS traversal.- post_number: u32,}/// DFS preorder number for unvisited nodes and the virtual root in the spanning tree.@@ -90,7 +88,6 @@ impl SpanningTree {label: pre_number,semi: pre_number,idom: ancestor,- post_number: 0,});pre_number@@ -141,9 +138,6 @@ struct DominatorTreeNode {/// Maximum `dom_pre_number` for the sub-tree of the dominator tree that is rooted at this node./// This is always >= `dom_pre_number`.dom_pre_max: u32,-- /// Post-order number from the CFG DFS traversal. Zero for unreachable blocks.- post_number: u32,}/// The dominator tree for a single function,@@ -304,11 +298,6 @@ impl DominatorTree {na.dom_pre_number <= nb.dom_pre_number && na.dom_pre_max >= nb.dom_pre_max}- /// Returns the CFG post-order number for `block`.- pub fn post_number(&self, block: Block) -> u32 {- self.nodes[block].post_number- }-/// Get an iterator over the direct children of `block` in the dominator tree.////// These are the blocks whose immediate dominator is `block`, ordered by@@ -430,8 +419,6 @@ impl DominatorTree {);}Some(TraversalEvent::Exit(block)) => {- let pre_number = self.nodes[block].pre_number;- self.stree[pre_number].post_number = self.postorder.len() as u32;self.postorder.push(block);}None => break,@@ -525,12 +512,7 @@ impl DominatorTree {////// This populates child/sibling links and preorder numbers for fast dominance checks.fn compute_domtree_preorder(&mut self) {- // Step 0: Assign CFG post-order numbers to each block.- for (i, &block) in self.postorder.iter().enumerate() {- self.nodes[block].post_number = i as u32;- }-- // Step 1: Populate the child and sibling links.+ // Populate the child and sibling links.//// By following the CFG post-order and pushing to the front of the lists, we make sure that// sibling lists are ordered according to the CFG reverse post-order (i.e. decreasing CFG@@ -545,7 +527,7 @@ impl DominatorTree {}}- // Step 2. Assign pre-order numbers from a DFS of the dominator tree.+ // Assign pre-order numbers from a DFS of the dominator tree.debug_assert!(self.dfs_worklist.len() <= 1);let mut n = 0;while let Some(event) = self.dfs_worklist.pop() {@@ -563,7 +545,7 @@ impl DominatorTree {}}- // Step 3. Propagate the `dom_pre_max` numbers up the tree.+ // Propagate the `dom_pre_max` numbers up the tree.// The CFG post-order is topologically ordered w.r.t. dominance so a node comes after all// its dominator tree children.for &block in &self.postorder {diff --git a/cranelift/codegen/src/egraph/mod.rs b/cranelift/codegen/src/egraph/mod.rsindex 0ac2b9d036..5d56c1c292 100644--- a/cranelift/codegen/src/egraph/mod.rs+++ b/cranelift/codegen/src/egraph/mod.rs@@ -6,6 +6,7 @@ use crate::ctxhash::{CtxEq, CtxHash, NullCtx};use crate::cursor::{Cursor, CursorPosition, FuncCursor};use crate::dominator_tree::DominatorTree;use crate::egraph::elaborate::Elaborator;+use crate::flowgraph::ControlFlowGraph;use crate::inst_predicates::{is_mergeable_for_egraph, is_pure_for_egraph};use crate::ir::{Block, DataFlowGraph, Function, Inst, InstructionData, Type, Value, ValueDef, ValueListPool,@@ -31,160 +32,12 @@ mod elaborate;/// dominator tree, where each node's children are visited in order of/// decreasing CFG post-order number.///-/// This ordering satisfies two properties simultaneously:-///-/// 1. **Dominator-tree DFS pre-order**: The `ScopedHashMap` used for GVN-/// requires that we visit dominator-tree ancestors before their descendants,-/// and that we can "pop" scopes as we backtrack up the dominator tree. This-/// is a DFS pre-order traversal of the dominator tree: we process every-/// dominator before the blocks it dominates, and the scope stack always-/// mirrors the dominator-tree path from the root to the current block.-///-/// 2. **Non-back-edge predecessors before successors**: Branch optimization-/// (e.g. replacing a `brif` with a constant condition with a `jump`) can-/// make blocks unreachable. We track reachable blocks in an `EntitySet`,-/// inserting each successor block as we visit each block's terminator. For-/// this to work correctly, we must visit all of a block's non-back-edge CFG-/// predecessors before visiting the block itself, otherwise we might-/// incorrectly conclude that a block is unreachable because we just haven't-/// seen some predecessor that would otherwise mark it reachable yet.-///-/// Both properties hold when dominator-tree children are visited in order of-/// decreasing CFG post-order number. The full proof follows.-///-/// # Proof-///-/// Let `post(v)` be the post-order numbering of a node `v` assigned by the DFS-/// traversal of the original graph (the same traversal that produces the-/// spanning tree used for dominator computation).-///-/// The *lowest common ancestor* (LCA) of two nodes `x` and `y` in a rooted-/// tree is the deepest node that is an ancestor of both `x` and `y`.-///-/// Given a DFS traversal of the graph from the root, each edge `u -> v` falls-/// into exactly one of the following categories:-///-/// - *Back edge*: `v` is an ancestor of `u` in the DFS spanning tree (the edge-/// leads back up the current DFS call stack).-///-/// - *Tree edge*: `u -> v` is an edge of the DFS spanning tree itself (i.e.-/// `v` was first discovered via `u`).-///-/// - *Forward edge*: `v` is a descendant of `u` in the DFS spanning tree, but-/// `u -> v` is not a spanning tree edge (it skips over one or more tree-/// edges).-///-/// - *Cross edge*: `v` is neither an ancestor nor a descendant of `u` in the-/// DFS spanning tree; `v` was fully finished before `u` was even discovered.-///-/// ## Lemma 1-///-/// Given a spanning tree edge `a -> b`, then `post(a) > post(b)`.-///-/// ### Proof-///-/// `post(a) > post(b)`: by the definition of post-order numbering, where-/// successors are numbered before predecessors.-///-/// ## Lemma 2-///-/// Given that `a` strictly dominates `b`, then `post(a) > post(b)`.-///-/// ### Proof-///-/// - All spanning tree paths from `root` to `b` pass through `a`: by the-/// definition of strict dominance.-/// - `post(a) > post(b)`: by Lemma 1 applied to each successive tree edge on-/// the path from `a` to `b`, and transitivity.-///-/// ## Lemma 3-///-/// For any non-back edge `u -> v` (i.e. a tree, forward, or cross edge),-/// `post(u) > post(v)`.-///-/// ### Proof-///-/// - *Tree edge* `u -> v`: `v` is discovered from `u` and finishes before `u`-/// returns, so `post(v) < post(u)`.-/// - *Forward edge* `u -> v`: `v` is a descendant of `u` in the DFS tree and-/// is already finished before `u` revisits it, so `post(v) < post(u)`.-/// - *Cross edge* `u -> v`: `v` was in a subtree explored entirely before `u`-/// was discovered, so `post(v) < post(u)`.-///-/// ## Lemma 4-///-/// If `u -> v` is a non-back edge and `u` does not strictly dominate `v`, then-/// `v` is a direct child of `LCA(u, v)` in the dominator tree.-///-/// ### Proof-///-/// Let `p = LCA(u, v)`, `c_u` the child of `p` that is an ancestor-or-equal-/// of `u` in the dominator tree, and `c_v` the child of `p` that is an-/// ancestor-or-equal of `v`.-///-/// - `p != u`: `u` does not dominate `v`, so `u` is not an ancestor of `v` in-/// the dominator tree.-/// - `p != v`: if `v` dominated `u` then every graph path from `root` to `u`-/// would pass through `v`, making `u -> v` a back edge; but `u -> v` is-/// non-back.-/// - `c_u != c_v`: since `p != u` and `p != v`, the paths from `p` to `u` and-/// `v` diverge at distinct children of `p`.-///-/// Suppose for contradiction that `c_v != v`, so `c_v` strictly dominates `v`.-///-/// - `c_v` does not dominate `c_u`: `c_u` and `c_v` are different children of-/// `p`, so neither is an ancestor of the other in the dominator tree.-/// - `c_v` does not dominate `u`: the dominator tree path from `root` to `u`-/// passes through `p -> c_u -> ... -> u` and not through `c_v`.-/// - `c_v != u`: if `c_v = u` then `u` is a strict ancestor of `v` in the-/// dominator tree, meaning `u` strictly dominates `v`; but `u` does not-/// strictly dominate `v` by the precondition of the lemma.-/// - There is a graph path `root -> ... -> u` avoiding `c_v`: by the-/// definition of dominance, since `c_v` does not dominate `u`.-/// - There is a graph path `root -> ... -> u -> v` avoiding `c_v`: append the-/// edge `u -> v`; the path avoids `c_v` because `c_v` is not in-/// `root -> ... -> u` (previous step), `c_v != u` (above), and `c_v != v`-/// (contradiction assumption).-/// - Contradiction: `c_v` strictly dominates `v`, so every path from `root` to-/// `v` must pass through `c_v`.-///-/// Therefore `c_v = v`.-///-/// ## Theorem 1-///-/// A depth-first pre-order traversal of a graph's dominator tree, where each-/// child `c` of `idom(c)` is visited in order from greatest to least-/// `post(c)`, visits all of a node's non-back-edge predecessors before visiting-/// the node itself.-///-/// ### Proof-///-/// Let `u -> v` be any non-back edge. We show `u` is visited before `v`.-///-/// - Case 1: `u` strictly dominates `v`.-///-/// `u` is a proper ancestor of `v` in the dominator tree: by the definition-/// of strict dominance. `u` is visited before `v`: DFS pre-order visits every-/// ancestor before any of its descendants.-///-/// - Case 2: `u` does not strictly dominate `v` (and `u != v`)-///-/// Let `p = LCA(u, v)`, `c_u` the child of `p` that is an ancestor-or-equal-/// of `u`, and `c_v = v` (by Lemma 4).-///-/// `post(c_u) > post(c_v)`:-/// - if `c_u = u`: `post(c_u) = post(u) > post(v) = post(c_v)` by Lemma 3.-/// - if `c_u != u`: `post(c_u) > post(u) > post(v) = post(c_v)` by Lemma 2-/// then Lemma 3.-///-/// `c_u` is visited before `c_v` in the traversal: children of `p` are visited-/// in decreasing `post` order and `post(c_u) > post(c_v)`.-///-/// `u` is visited before `v`: DFS pre-order visits all of `c_u`'s subtree-/// before `c_v`, with `u` in `c_u`'s subtree and `v = c_v`.-///-/// Both cases establish that `u` is visited before `v` for every non-back edge-/// `u -> v`, which is exactly what the theorem claims.+/// The `ScopedHashMap` used for GVN requires that we visit dominator-tree+/// ancestors before their descendants, and that we can "pop" scopes as we+/// backtrack up the dominator tree. This is a DFS pre-order traversal of the+/// dominator tree: we process every dominator before the blocks it dominates,+/// and the scope stack always mirrors the dominator-tree path from the root to+/// the current block.struct EgraphBlockIter<'a> {domtree: &'a DominatorTree,stack: Vec<Block>,@@ -215,15 +68,8 @@ impl Iterator for EgraphBlockIter<'_> {self.children.clear();self.children.extend(self.domtree.children(block));- // Assert that `children()` yields in decreasing post-order number.- debug_assert!(- self.children- .windows(2)- .all(|w| { self.domtree.post_number(w[0]) > self.domtree.post_number(w[1]) })- );-- // Push in reverse so that the highest-post-number child ends up on top- // of the stack and is visited first.+ // Push in reverse so that the first child ends up on top of the stack+ // and is visited first.self.stack.extend(self.children.iter().rev().copied());Some(block)@@ -263,11 +109,9 @@ pub struct EgraphPass<'a> {/// Chaos-mode control-plane so we can test that we still get/// correct results when our heuristics make bad decisions.ctrl_plane: &'a mut ControlPlane,- /// The set of blocks that are reachable from the entry block given- /// the (potentially simplified/optimized) terminators. Used to- /// remove blocks that become unreachable after branch- /// simplification.- reachable_blocks: EntitySet<Block>,+ /// The control flow graph, used when eliminating unreachable code+ /// after branch simplification.+ cfg: &'a mut ControlFlowGraph,/// Which Values do we want to rematerialize in each block where/// they're used?remat_values: FxHashSet<Value>,@@ -734,9 +578,8 @@ wherefn simplify_skeleton_inst(&mut self, inst: Inst) -> Option<SkeletonInstSimplification> {// NB: we support simplifying branch terminators (e.g. `brif` with a// constant condition into `jump`). This can make blocks unreachable,- // but the caller handles that via the `reachable_blocks` set: blocks- // not in the set are removed from the layout (along with all their- // instructions and value uses).+ // but a separate `eliminate_unreachable_code` pass handles removing+ // them after the egraph pass completes.//// We do NOT yet support simplifying non-terminators into terminators// (e.g. `trapz` into `trap`) because that would introduce a@@ -876,16 +719,15 @@ impl<'a> EgraphPass<'a> {loop_analysis: &'a LoopAnalysis,alias_analysis: &'a mut AliasAnalysis<'a>,ctrl_plane: &'a mut ControlPlane,+ cfg: &'a mut ControlFlowGraph,) -> Self {- let mut reachable_blocks = EntitySet::with_capacity(func.dfg.num_blocks());- reachable_blocks.insert(func.layout.entry_block().unwrap());Self {func,domtree,loop_analysis,alias_analysis,ctrl_plane,- reachable_blocks,+ cfg,stats: Stats::default(),remat_values: FxHashSet::default(),}@@ -893,7 +735,7 @@ impl<'a> EgraphPass<'a> {/// Run the process.pub fn run(&mut self) {- self.remove_pure_and_optimize();+ let reachable_blocks = self.remove_pure_and_optimize();trace!("egraph built:\n{}\n", self.func.display());if cfg!(feature = "trace-log") {@@ -908,6 +750,10 @@ impl<'a> EgraphPass<'a> {}}+ crate::unreachable_code::eliminate_unreachable_code(self.func, self.cfg, |block| {+ reachable_blocks.contains(block)+ });+self.elaborate();log::trace!("stats: {:#?}", self.stats);@@ -932,8 +778,10 @@ impl<'a> EgraphPass<'a> {/// because the eclass can continue to be updated and we need to/// only refer to its subset that exists at this stage, to/// maintain acyclicity.)- fn remove_pure_and_optimize(&mut self) {+ fn remove_pure_and_optimize(&mut self) -> EntitySet<Block> {let mut cursor = FuncCursor::new(self.func);+ let mut reachable_blocks = EntitySet::<Block>::with_capacity(cursor.func.dfg.num_blocks());+ reachable_blocks.insert(cursor.func.layout.entry_block().unwrap());let mut value_to_opt_value: SecondaryMap<Value, Value> =SecondaryMap::with_default(Value::reserved_value());@@ -1003,6 +851,8 @@ impl<'a> EgraphPass<'a> {gvn_map.increment_depth();gvn_map_blocks.push(block);+ // Check that `gvn_map_blocks` is the path from this block up to the+ // root in the dominator tree.debug_assert_eq!(gvn_map_blocks, {let mut b = Some(block);let mut v = core::iter::from_fn(move || {@@ -1015,11 +865,6 @@ impl<'a> EgraphPass<'a> {v});- if !self.reachable_blocks.contains(block) {- cursor.func.layout.remove_block_and_insts(block);- continue;- }-trace!("Processing block {}", block);cursor.set_position(CursorPosition::Before(block));@@ -1097,15 +942,18 @@ impl<'a> EgraphPass<'a> {}}- let terminator_inst = cursor.func.layout.last_inst(block).unwrap();- for dest in cursor.func.dfg.insts[terminator_inst].branch_destination(- &cursor.func.dfg.jump_tables,- &cursor.func.dfg.exception_tables,- ) {- self.reachable_blocks- .insert(dest.block(&cursor.func.dfg.value_lists));+ if reachable_blocks.contains(block) {+ let terminator_inst = cursor.func.layout.last_inst(block).unwrap();+ for dest in cursor.func.dfg.insts[terminator_inst].branch_destination(+ &cursor.func.dfg.jump_tables,+ &cursor.func.dfg.exception_tables,+ ) {+ reachable_blocks.insert(dest.block(&cursor.func.dfg.value_lists));+ }}}++ reachable_blocks}/// Execute a simplification of an instruction in the side-effectfuldiff --git a/cranelift/codegen/src/unreachable_code.rs b/cranelift/codegen/src/unreachable_code.rsindex 68d7bd33fd..192afcef6b 100644--- a/cranelift/codegen/src/unreachable_code.rs+++ b/cranelift/codegen/src/unreachable_code.rs@@ -3,7 +3,6 @@use cranelift_entity::EntitySet;use crate::cursor::{Cursor, FuncCursor};-use crate::dominator_tree::DominatorTree;use crate::flowgraph::ControlFlowGraph;use crate::timing;use crate::{ir, trace};@@ -12,12 +11,10 @@ use crate::{ir, trace};////// This pass deletes whole blocks that can't be reached from the entry block. It does not delete/// individual instructions whose results are unused.-///-/// The reachability analysis is performed by the dominator tree analysis.pub fn eliminate_unreachable_code(func: &mut ir::Function,cfg: &mut ControlFlowGraph,- domtree: &DominatorTree,+ is_reachable: impl Fn(ir::Block) -> bool,) {let _tt = timing::unreachable_code();let mut pos = FuncCursor::new(func);@@ -25,7 +22,7 @@ pub fn eliminate_unreachable_code(let mut used_exception_tables =EntitySet::with_capacity(pos.func.stencil.dfg.exception_tables.len());while let Some(block) = pos.next_block() {- if domtree.is_reachable(block) {+ if is_reachable(block) {let inst = pos.func.layout.last_inst(block).unwrap();match pos.func.dfg.insts[inst] {ir::InstructionData::BranchTable { table, .. } => {diff --git a/cranelift/filetests/filetests/egraph/issue-13365.clif b/cranelift/filetests/filetests/egraph/issue-13365.clifnew file mode 100644index 0000000000..4c6addce2b--- /dev/null+++ b/cranelift/filetests/filetests/egraph/issue-13365.clif@@ -0,0 +1,29 @@+test optimize precise-output+set opt_level=speed+target x86_64++; Test branch simplifications and irreducible control flow.++function %f() {+block0:+ v0 = iconst.i8 0+ brif v0, block2, block1++block1:+ jump block2++block2:+ jump block1+}++; function %f() fast {+; block0:+; jump block1+;+; block1:+; jump block2+;+; block2:+; jump block1+; }+diff --git a/tests/disas/gc/drc/struct-new-default.wat b/tests/disas/gc/drc/struct-new-default.watindex f0886e21d1..64b21df0c9 100644--- a/tests/disas/gc/drc/struct-new-default.wat+++ b/tests/disas/gc/drc/struct-new-default.wat@@ -45,10 +45,10 @@;; jump block3;;;; block3:-;; v64 = iconst.i32 0+;; v65 = iconst.i32 0;; v43 = iconst.i64 32;; @0021 v19 = iadd.i64 v16, v43 ; v43 = 32-;; @0021 store user2 little v64, v19 ; v64 = 0+;; @0021 store user2 little v65, v19 ; v65 = 0;; @0024 jump block1;;;; block1:diff --git a/tests/disas/gc/struct-new-default.wat b/tests/disas/gc/struct-new-default.watindex dc44cadca0..7bf876a97b 100644--- a/tests/disas/gc/struct-new-default.wat+++ b/tests/disas/gc/struct-new-default.wat@@ -47,10 +47,10 @@;; jump block3;;;; block3:-;; v67 = iconst.i32 0+;; v68 = iconst.i32 0;; v46 = iconst.i64 32;; @0023 v20 = iadd.i64 v17, v46 ; v46 = 32-;; @0023 store user2 little v67, v20 ; v67 = 0+;; @0023 store user2 little v68, v20 ; v68 = 0;; @0023 v6 = vconst.i8x16 const0;; v38 = iconst.i64 48;; @0023 v37 = iadd.i64 v17, v38 ; v38 = 48