A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Alias analysis: Reuse the exit state of a block's single predecessor (#14425)
Nick Fitzgeralded04e6b5e0a9
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rsindex e6cb6fce46..c385ae5b9b 100644--- a/cranelift/codegen/src/alias_analysis.rs+++ b/cranelift/codegen/src/alias_analysis.rs@@ -273,9 +273,16 @@ impl LastStore {/// The instructions named here (`regions[r]` and `last_fence`) are *not*/// guaranteed to still be in the layout (unlike `mem_values`, this state does/// not maintain that invariant). A slot can name a store we already removed-/// because, e.g., `block_input` snapshots are computed once up front (in-/// `compute_block_input_states`) and can name a store a later-visited block-/// deletes.+/// because `block_input` snapshots are computed once up front (in+/// `compute_block_input_states`) and can name a dead store that a later-visited+/// block eliminates.+///+/// Note that this is only a hazard when processing blocks that read their state+/// from the initial snapshot. A block with exactly one reachable predecessor+/// instead inherits its predecessor's live exit state (see+/// `AliasAnalysis::push_scope` and `AliasAnalysis::finish_scope`), which names+/// only stores that are still in the layout; joins are the remaining source of+/// stale instructions here.////// We tolerate this, rather than enforce the invariant, because enforcing it/// would mean repairing a precomputed fixpoint's references across@@ -793,20 +800,32 @@ pub struct AliasAnalysis<'a> {/// its known value.mem_values: FxHashMap<MemoryLoc, KnownValue>,- /// The extent token for each scope currently on the dominator-tree path- /// we are walking.- ///- /// See `MemoryLoc::extent` for details.- extents: SmallVec<[u32; 8]>,+ /// The scopes currently on the dominator-tree path we are walking.+ scopes: SmallVec<[Scope; 8]>,/// Counter for minting fresh extent tokens.next_extent: u32,+}- /// The blocks whose scopes are currently on the stack, so that we can- /// assert that `push_scope` and `pop_scope` really are called in- /// dominator-tree pre-order.- #[cfg(debug_assertions)]- scope_blocks: SmallVec<[Block; 8]>,+/// One block's scope on the dominator-tree path currently being walked.+struct Scope {+ /// The block whose scope this is.+ block: Block,++ /// The extent token for this scope.+ ///+ /// See `MemoryLoc::extent` for details.+ extent: u32,++ /// This block's last-store state on exit, once we have finished walking its+ /// instructions.+ ///+ /// This is `None` between `push_scope` and `finish_scope`, and `Some` from+ /// `finish_scope` until this scope is popped. Therefore it is always `Some`+ /// for every scope strictly below the top of the stack, since we always+ /// finish a block before descending into any of its dominator-tree+ /// children.+ exit_state: Option<LastStores>,}impl<'a> AliasAnalysis<'a> {@@ -821,10 +840,8 @@ impl<'a> AliasAnalysis<'a> {observed_stores: FxHashMap::default(),block_input: FxHashMap::default(),mem_values: FxHashMap::default(),- extents: SmallVec::new(),+ scopes: SmallVec::new(),next_extent: 0,- #[cfg(debug_assertions)]- scope_blocks: SmallVec::new(),};analysis.compute_block_input_states(func);@@ -1053,48 +1070,94 @@ impl<'a> AliasAnalysis<'a> {/// block that this one does not dominate.pub fn push_scope(&mut self, cfg: &ControlFlowGraph, block: Block) -> MemoryState {debug_assert!(cfg.is_valid());+ debug_assert_eq!(+ self.domtree.idom(block),+ self.scopes.last().map(|s| s.block),+ "`push_scope` must be called in dominator-tree pre-order",+ );- #[cfg(debug_assertions)]- {- debug_assert_eq!(- self.domtree.idom(block),- self.scope_blocks.last().copied(),- "`push_scope` must be called in dominator-tree pre-order",- );- self.scope_blocks.push(block);- }-- let extent = match self.extents.last() {- // A block with exactly one reachable predecessor sees exactly the- // memory that its predecessor left behind, and so it continues its- // predecessor's extent.- Some(parent) if self.single_reachable_pred(cfg, block).is_some() => *parent,+ // A block with exactly one reachable predecessor sees exactly the+ // memory that its predecessor left behind.+ let only_pred = self.single_reachable_pred(cfg, block);++ let (stores, extent) = match (only_pred, self.scopes.last()) {+ // With exactly one reachable predecessor, this block's entry state+ // is exactly that predecessor's exit state, which we just finished+ // computing and is strictly more precise than the `block_input`+ // snapshot: that snapshot describes the pre-optimization function,+ // and so it can still name dead stores we have since eliminated.+ //+ // And, because it sees exactly the predecessor's memory state, it+ // also continues that predecessor's extent.+ (Some(only_pred), Some(parent)) if parent.block == only_pred => {+ // A sole predecessor must be this block's immediate dominator.+ // Since the scope stack is all of our ordered dominators, and+ // because no other dominator sits between a block and its+ // immediate dominator, the top of the scope stack is both this+ // block's sole predecessor and immediate dominator.+ debug_assert_eq!(self.domtree.idom(block), Some(parent.block));++ let stores = parent+ .exit_state+ .as_ref()+ .expect("`finish_scope` is always called before descending into a child scope")+ .clone();+ (stores, parent.extent)+ }- // Anything else (e.g. a control-flow join or the entry block)- // begins a new extent.+ // Anything else (e.g. a control-flow join, the entry block) falls+ // back to the initial fixpoint's snapshot, and begins a new extent._ => {self.next_extent += 1;- self.next_extent+ (self.block_input_stores(block), self.next_extent)}};- self.extents.push(extent);++ self.scopes.push(Scope {+ block,+ extent,+ exit_state: None,+ });MemoryState {- stores: self.block_input_stores(block),+ stores,extent,#[cfg(debug_assertions)]current_block: block,}}- /// Leave the scope of the block most recently passed to `push_scope`.- pub fn pop_scope(&mut self) {+ /// Record the given block's exit state, so that a successor for which it is+ /// the sole predecessor can inherit that state.+ ///+ /// Callers must call this exactly once per `push_scope`, after walking the+ /// block's instructions and before visiting any of its dominator-tree+ /// children.+ pub fn finish_scope(&mut self, block: Block, state: MemoryState) {+ let scope = self+ .scopes+ .last_mut()+ .expect("`finish_scope` without a matching `push_scope`");++ debug_assert_eq!(+ scope.block, block,+ "`finish_scope` must be called on the block most recently passed to `push_scope`",+ );+#[cfg(debug_assertions)]- self.scope_blocks- .pop()- .expect("`pop_scope` without a matching `push_scope`");+ debug_assert_eq!(state.current_block, block);++ debug_assert!(+ scope.exit_state.is_none(),+ "`finish_scope` must be called at most once per `push_scope`",+ );++ scope.exit_state = Some(state.stores);+ }- self.extents+ /// Leave the scope of the block most recently passed to `push_scope`.+ pub fn pop_scope(&mut self) {+ self.scopes.pop().expect("`pop_scope` without a matching `push_scope`");}@@ -1446,6 +1509,8 @@ impl<'a> AliasAnalysis<'a> {}}+ self.finish_scope(block, state);+let children = SmallVec::<[Block; 8]>::from_iter(domtree.children(block));stack.extend(children.into_iter().rev().map(BlockStackEntry::Visit));}diff --git a/cranelift/codegen/src/egraph/mod.rs b/cranelift/codegen/src/egraph/mod.rsindex 7b33d8d9e8..d0280f557b 100644--- a/cranelift/codegen/src/egraph/mod.rs+++ b/cranelift/codegen/src/egraph/mod.rs@@ -1022,6 +1022,13 @@ impl<'a> EgraphPass<'a> {}}}++ // Save this block's exit state, so that any of its successors for+ // which it is the sole predecessor can inherit that exit state+ // directly instead of falling back to the initial (and possibly+ // outdated, after optimizations) initial block-input snapshot.+ self.alias_analysis+ .finish_scope(block, alias_analysis_state);}cfg_maybe_changed}diff --git a/cranelift/filetests/filetests/alias/inherit-single-pred-exit-state.clif b/cranelift/filetests/filetests/alias/inherit-single-pred-exit-state.clifnew file mode 100644index 0000000000..67468b5ec7--- /dev/null+++ b/cranelift/filetests/filetests/alias/inherit-single-pred-exit-state.clif@@ -0,0 +1,140 @@+test optimize precise-output+set opt_level=speed+target x86_64++;; Everything in the same block does not rely on inheriting a sole predecessor's+;; exit state.+function %same_block(i64) {+ region0 = 0 "flags"++block0(v0: i64):+ ;; Initial flag load and trap if it is unset.+ v1 = load.i32 notrap aligned region0 v0+ trapz v1, user26++ ;; Unset the flag.+ v2 = iconst.i32 0+ store notrap aligned region0 v2, v0++ ;; Reset the flag, making the previous store dead, and then revealing that+ ;; this store is idempotent. Both stores should therefore get removed.+ store notrap aligned region0 v1, v0++ ;; Load the flag again and trap if it is unset again.+ ;;+ ;; We should forward the previous load of `v1` to `v3`, which should then+ ;; allow the `trapz` to be deduped by GVN.+ v3 = load.i32 notrap aligned region0 v0+ trapz v3, user26++ return+}++; function %same_block(i64) fast {+; region0 = 0 "flags"+;+; block0(v0: i64):+; v1 = load.i32 notrap aligned region0 v0+; trapz v1, user26+; return+; }++;; Same thing, but spread across a chain of single-predecessor blocks.+;;+;; The optimizations firing in this case relies upon reusing a block's sole+;; predecessor's exit state.+function %chain(i64) {+ region0 = 0 "flags"++block0(v0: i64):+ v1 = load.i32 notrap aligned region0 v0+ trapz v1, user26+ v2 = iconst.i32 0+ store notrap aligned region0 v2, v0+ store notrap aligned region0 v1, v0+ jump block1++block1:+ jump block2++block2:+ jump block3++block3:+ jump block4++block4:+ v3 = load.i32 notrap aligned region0 v0+ trapz v3, user26+ return+}++; function %chain(i64) fast {+; region0 = 0 "flags"+;+; block0(v0: i64):+; v1 = load.i32 notrap aligned region0 v0+; trapz v1, user26+; jump block1+;+; block1:+; jump block2+;+; block2:+; jump block3+;+; block3:+; jump block4+;+; block4:+; return+; }++;; Same thing, but instructions distributed cross the chain differently.+function %chain2(i64) {+ region0 = 0 "flags"++block0(v0: i64):+ v1 = load.i32 notrap aligned region0 v0+ jump block1++block1:+ trapz v1, user26+ jump block2++block2:+ v2 = iconst.i32 0+ store notrap aligned region0 v2, v0+ jump block3++block3:+ store notrap aligned region0 v1, v0+ jump block4++block4:+ v3 = load.i32 notrap aligned region0 v0+ trapz v3, user26++ return+}++; function %chain2(i64) fast {+; region0 = 0 "flags"+;+; block0(v0: i64):+; v1 = load.i32 notrap aligned region0 v0+; jump block1+;+; block1:+; trapz.i32 v1, user26+; jump block2+;+; block2:+; jump block3+;+; block3:+; jump block4+;+; block4:+; return+; }diff --git a/cranelift/filetests/filetests/alias/inherit-single-pred-siblings.clif b/cranelift/filetests/filetests/alias/inherit-single-pred-siblings.clifnew file mode 100644index 0000000000..9a213abbce--- /dev/null+++ b/cranelift/filetests/filetests/alias/inherit-single-pred-siblings.clif@@ -0,0 +1,81 @@+test optimize precise-output+set opt_level=speed+target x86_64++;; A parent's exit state must stay live across *all* of its dominator-tree+;; children, so that every sibling can inherit the parent's end-of-block state.++function %siblings(i64, i64, i32) -> i32, i32, i32, i32 {+ region0 = 0 "flags"+ region1 = 1 "R1"++block0(v0: i64, v1: i64, v2: i32):+ v3 = load.i32 notrap aligned region0 v0+ v4 = load.i32 notrap aligned region1 v1++ ;; Dead store.+ v5 = iconst.i32 0+ store notrap aligned region0 v5, v0++ ;; Idempotent store.+ store notrap aligned region0 v3, v0++ brif v2, block1, block4++;; The first child of `block0`, whose sole predecessor is `block0`. Should+;; inherit `block0`'s exit state.+block1:+ ;; Clobber `region0`.+ v6 = iconst.i32 1+ store notrap aligned region0 v6, v0+ jump block2++block2:+ ;; Redundant; should be replaced with `v4`.+ v7 = load.i32 notrap aligned region1 v1+ jump block3++block3:+ ;; Forwards to (a rematerialized) `v6` from the store in `block1`, not to+ ;; the load of `v3`.+ v8 = load.i32 notrap aligned region0 v0++ return v3, v4, v7, v8++;; The second child of `block0`, whose sole predecessor is `block0`. Sibling to+;; `block1`. Should inherit `block0`'s exit state.+block4:+ ;; Redundant; should be replaced with `v3`.+ v9 = load.i32 notrap aligned region0 v0++ ;; Redundant; should be replaced with `v4`.+ v10 = load.i32 notrap aligned region1 v1++ return v3, v4, v9, v10+}++; function %siblings(i64, i64, i32) -> i32, i32, i32, i32 fast {+; region0 = 0 "flags"+; region1 = 1 "R1"+;+; block0(v0: i64, v1: i64, v2: i32):+; v3 = load.i32 notrap aligned region0 v0+; v4 = load.i32 notrap aligned region1 v1+; brif v2, block1, block4+;+; block1:+; v6 = iconst.i32 1+; store notrap aligned region0 v6, v0 ; v6 = 1+; jump block2+;+; block2:+; jump block3+;+; block3:+; v11 = iconst.i32 1+; return v3, v4, v4, v11 ; v11 = 1+;+; block4:+; return v3, v4, v3, v4+; }+diff --git a/cranelift/filetests/filetests/alias/issue-13961.clif b/cranelift/filetests/filetests/alias/issue-13961.clifindex f30966499b..d00d97c120 100644--- a/cranelift/filetests/filetests/alias/issue-13961.clif+++ b/cranelift/filetests/filetests/alias/issue-13961.clif@@ -44,6 +44,6 @@ block1:; jump block1;; block1:-; store.i32 notrap region0 v1, v0; return v1; }+diff --git a/tests/disas/component-model/compile-time-builtins-x64.wat b/tests/disas/component-model/compile-time-builtins-x64.watindex 29c0dad0cd..e4753ecdf8 100644--- a/tests/disas/component-model/compile-time-builtins-x64.wat+++ b/tests/disas/component-model/compile-time-builtins-x64.wat@@ -180,33 +180,29 @@;; movq 0xe8(%rcx), %rax;; movl (%rax), %edx;; testl %edx, %edx-;; je 0x179+;; je 0x16d;; 119: movq 0x48(%rcx), %rcx;; movq 8(%rcx), %rcx;; movq 0x68(%rcx), %rcx;; movq 8(%rcx), %rdx;; xorq %rsi, %rsi;; cmpq %rdx, %rsi-;; jae 0x174+;; jae 0x168;; 135: movl (%rax), %edi;; testl %edi, %edi-;; je 0x17b+;; je 0x16f;; 13f: movq 8(%rcx), %rdi;; cmpq %rdi, %rsi-;; jae 0x17d+;; jae 0x171;; 14c: movq (%rcx), %rdi;; movzbq (%rdi, %rsi), %r8-;; movl (%rax), %r9d-;; testl %r9d, %r9d-;; je 0x17f-;; 160: addb $1, %r8b+;; addb $1, %r8b;; movb %r8b, (%rdi, %rsi);; addq $1, %rsi;; jmp 0x12c-;; 174: movq %rbp, %rsp+;; 168: movq %rbp, %rsp;; popq %rbp;; retq-;; 179: ud2-;; 17b: ud2-;; 17d: ud2-;; 17f: ud2+;; 16d: ud2+;; 16f: ud2+;; 171: ud2diff --git a/tests/disas/component-model/compile-time-builtins.wat b/tests/disas/component-model/compile-time-builtins.watindex b583fce136..962dcdd5a9 100644--- a/tests/disas/component-model/compile-time-builtins.wat+++ b/tests/disas/component-model/compile-time-builtins.wat@@ -417,12 +417,9 @@;; jump block51;;;; block51:-;; v87 = load.i32 notrap aligned region4 v23-;; trapz v87, user26;; jump block54;;;; block54:-;; v89 = load.i32 notrap aligned region5 v25;; jump block57;;;; block57: