Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Alias analysis: Reuse the exit state of a block's single predecessor (#14425)

Nick Fitzgeralded04e6b5e0a9
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rs
index e6cb6fce46..c385ae5b9b 100644
--- a/cranelift/codegen/src/alias_analysis.rs
+++ b/cranelift/codegen/src/alias_analysis.rs
@@ -273,9 +273,16 @@ impl LastStore {
/// The instructions named here (`regions[r]` and `last_fence`) are *not*
/// guaranteed to still be in the layout (unlike `mem_values`, this state does
/// not maintain that invariant). A slot can name a store we already removed
-/// because, e.g., `block_input` snapshots are computed once up front (in
-/// `compute_block_input_states`) and can name a store a later-visited block
-/// deletes.
+/// because `block_input` snapshots are computed once up front (in
+/// `compute_block_input_states`) and can name a dead store that a later-visited
+/// block eliminates.
+///
+/// Note that this is only a hazard when processing blocks that read their state
+/// from the initial snapshot. A block with exactly one reachable predecessor
+/// instead inherits its predecessor's live exit state (see
+/// `AliasAnalysis::push_scope` and `AliasAnalysis::finish_scope`), which names
+/// only stores that are still in the layout; joins are the remaining source of
+/// stale instructions here.
///
/// We tolerate this, rather than enforce the invariant, because enforcing it
/// would mean repairing a precomputed fixpoint's references across
@@ -793,20 +800,32 @@ pub struct AliasAnalysis<'a> {
/// its known value.
mem_values: FxHashMap<MemoryLoc, KnownValue>,
- /// The extent token for each scope currently on the dominator-tree path
- /// we are walking.
- ///
- /// See `MemoryLoc::extent` for details.
- extents: SmallVec<[u32; 8]>,
+ /// The scopes currently on the dominator-tree path we are walking.
+ scopes: SmallVec<[Scope; 8]>,
/// Counter for minting fresh extent tokens.
next_extent: u32,
+}
- /// The blocks whose scopes are currently on the stack, so that we can
- /// assert that `push_scope` and `pop_scope` really are called in
- /// dominator-tree pre-order.
- #[cfg(debug_assertions)]
- scope_blocks: SmallVec<[Block; 8]>,
+/// One block's scope on the dominator-tree path currently being walked.
+struct Scope {
+ /// The block whose scope this is.
+ block: Block,
+
+ /// The extent token for this scope.
+ ///
+ /// See `MemoryLoc::extent` for details.
+ extent: u32,
+
+ /// This block's last-store state on exit, once we have finished walking its
+ /// instructions.
+ ///
+ /// This is `None` between `push_scope` and `finish_scope`, and `Some` from
+ /// `finish_scope` until this scope is popped. Therefore it is always `Some`
+ /// for every scope strictly below the top of the stack, since we always
+ /// finish a block before descending into any of its dominator-tree
+ /// children.
+ exit_state: Option<LastStores>,
}
impl<'a> AliasAnalysis<'a> {
@@ -821,10 +840,8 @@ impl<'a> AliasAnalysis<'a> {
observed_stores: FxHashMap::default(),
block_input: FxHashMap::default(),
mem_values: FxHashMap::default(),
- extents: SmallVec::new(),
+ scopes: SmallVec::new(),
next_extent: 0,
- #[cfg(debug_assertions)]
- scope_blocks: SmallVec::new(),
};
analysis.compute_block_input_states(func);
@@ -1053,48 +1070,94 @@ impl<'a> AliasAnalysis<'a> {
/// block that this one does not dominate.
pub fn push_scope(&mut self, cfg: &ControlFlowGraph, block: Block) -> MemoryState {
debug_assert!(cfg.is_valid());
+ debug_assert_eq!(
+ self.domtree.idom(block),
+ self.scopes.last().map(|s| s.block),
+ "`push_scope` must be called in dominator-tree pre-order",
+ );
- #[cfg(debug_assertions)]
- {
- debug_assert_eq!(
- self.domtree.idom(block),
- self.scope_blocks.last().copied(),
- "`push_scope` must be called in dominator-tree pre-order",
- );
- self.scope_blocks.push(block);
- }
-
- let extent = match self.extents.last() {
- // A block with exactly one reachable predecessor sees exactly the
- // memory that its predecessor left behind, and so it continues its
- // predecessor's extent.
- Some(parent) if self.single_reachable_pred(cfg, block).is_some() => *parent,
+ // A block with exactly one reachable predecessor sees exactly the
+ // memory that its predecessor left behind.
+ let only_pred = self.single_reachable_pred(cfg, block);
+
+ let (stores, extent) = match (only_pred, self.scopes.last()) {
+ // With exactly one reachable predecessor, this block's entry state
+ // is exactly that predecessor's exit state, which we just finished
+ // computing and is strictly more precise than the `block_input`
+ // snapshot: that snapshot describes the pre-optimization function,
+ // and so it can still name dead stores we have since eliminated.
+ //
+ // And, because it sees exactly the predecessor's memory state, it
+ // also continues that predecessor's extent.
+ (Some(only_pred), Some(parent)) if parent.block == only_pred => {
+ // A sole predecessor must be this block's immediate dominator.
+ // Since the scope stack is all of our ordered dominators, and
+ // because no other dominator sits between a block and its
+ // immediate dominator, the top of the scope stack is both this
+ // block's sole predecessor and immediate dominator.
+ debug_assert_eq!(self.domtree.idom(block), Some(parent.block));
+
+ let stores = parent
+ .exit_state
+ .as_ref()
+ .expect("`finish_scope` is always called before descending into a child scope")
+ .clone();
+ (stores, parent.extent)
+ }
- // Anything else (e.g. a control-flow join or the entry block)
- // begins a new extent.
+ // Anything else (e.g. a control-flow join, the entry block) falls
+ // back to the initial fixpoint's snapshot, and begins a new extent.
_ => {
self.next_extent += 1;
- self.next_extent
+ (self.block_input_stores(block), self.next_extent)
}
};
- self.extents.push(extent);
+
+ self.scopes.push(Scope {
+ block,
+ extent,
+ exit_state: None,
+ });
MemoryState {
- stores: self.block_input_stores(block),
+ stores,
extent,
#[cfg(debug_assertions)]
current_block: block,
}
}
- /// Leave the scope of the block most recently passed to `push_scope`.
- pub fn pop_scope(&mut self) {
+ /// Record the given block's exit state, so that a successor for which it is
+ /// the sole predecessor can inherit that state.
+ ///
+ /// Callers must call this exactly once per `push_scope`, after walking the
+ /// block's instructions and before visiting any of its dominator-tree
+ /// children.
+ pub fn finish_scope(&mut self, block: Block, state: MemoryState) {
+ let scope = self
+ .scopes
+ .last_mut()
+ .expect("`finish_scope` without a matching `push_scope`");
+
+ debug_assert_eq!(
+ scope.block, block,
+ "`finish_scope` must be called on the block most recently passed to `push_scope`",
+ );
+
#[cfg(debug_assertions)]
- self.scope_blocks
- .pop()
- .expect("`pop_scope` without a matching `push_scope`");
+ debug_assert_eq!(state.current_block, block);
+
+ debug_assert!(
+ scope.exit_state.is_none(),
+ "`finish_scope` must be called at most once per `push_scope`",
+ );
+
+ scope.exit_state = Some(state.stores);
+ }
- self.extents
+ /// Leave the scope of the block most recently passed to `push_scope`.
+ pub fn pop_scope(&mut self) {
+ self.scopes
.pop()
.expect("`pop_scope` without a matching `push_scope`");
}
@@ -1446,6 +1509,8 @@ impl<'a> AliasAnalysis<'a> {
}
}
+ self.finish_scope(block, state);
+
let children = SmallVec::<[Block; 8]>::from_iter(domtree.children(block));
stack.extend(children.into_iter().rev().map(BlockStackEntry::Visit));
}
diff --git a/cranelift/codegen/src/egraph/mod.rs b/cranelift/codegen/src/egraph/mod.rs
index 7b33d8d9e8..d0280f557b 100644
--- a/cranelift/codegen/src/egraph/mod.rs
+++ b/cranelift/codegen/src/egraph/mod.rs
@@ -1022,6 +1022,13 @@ impl<'a> EgraphPass<'a> {
}
}
}
+
+ // Save this block's exit state, so that any of its successors for
+ // which it is the sole predecessor can inherit that exit state
+ // directly instead of falling back to the initial (and possibly
+ // outdated, after optimizations) initial block-input snapshot.
+ self.alias_analysis
+ .finish_scope(block, alias_analysis_state);
}
cfg_maybe_changed
}
diff --git a/cranelift/filetests/filetests/alias/inherit-single-pred-exit-state.clif b/cranelift/filetests/filetests/alias/inherit-single-pred-exit-state.clif
new file mode 100644
index 0000000000..67468b5ec7
--- /dev/null
+++ b/cranelift/filetests/filetests/alias/inherit-single-pred-exit-state.clif
@@ -0,0 +1,140 @@
+test optimize precise-output
+set opt_level=speed
+target x86_64
+
+;; Everything in the same block does not rely on inheriting a sole predecessor's
+;; exit state.
+function %same_block(i64) {
+ region0 = 0 "flags"
+
+block0(v0: i64):
+ ;; Initial flag load and trap if it is unset.
+ v1 = load.i32 notrap aligned region0 v0
+ trapz v1, user26
+
+ ;; Unset the flag.
+ v2 = iconst.i32 0
+ store notrap aligned region0 v2, v0
+
+ ;; Reset the flag, making the previous store dead, and then revealing that
+ ;; this store is idempotent. Both stores should therefore get removed.
+ store notrap aligned region0 v1, v0
+
+ ;; Load the flag again and trap if it is unset again.
+ ;;
+ ;; We should forward the previous load of `v1` to `v3`, which should then
+ ;; allow the `trapz` to be deduped by GVN.
+ v3 = load.i32 notrap aligned region0 v0
+ trapz v3, user26
+
+ return
+}
+
+; function %same_block(i64) fast {
+; region0 = 0 "flags"
+;
+; block0(v0: i64):
+; v1 = load.i32 notrap aligned region0 v0
+; trapz v1, user26
+; return
+; }
+
+;; Same thing, but spread across a chain of single-predecessor blocks.
+;;
+;; The optimizations firing in this case relies upon reusing a block's sole
+;; predecessor's exit state.
+function %chain(i64) {
+ region0 = 0 "flags"
+
+block0(v0: i64):
+ v1 = load.i32 notrap aligned region0 v0
+ trapz v1, user26
+ v2 = iconst.i32 0
+ store notrap aligned region0 v2, v0
+ store notrap aligned region0 v1, v0
+ jump block1
+
+block1:
+ jump block2
+
+block2:
+ jump block3
+
+block3:
+ jump block4
+
+block4:
+ v3 = load.i32 notrap aligned region0 v0
+ trapz v3, user26
+ return
+}
+
+; function %chain(i64) fast {
+; region0 = 0 "flags"
+;
+; block0(v0: i64):
+; v1 = load.i32 notrap aligned region0 v0
+; trapz v1, user26
+; jump block1
+;
+; block1:
+; jump block2
+;
+; block2:
+; jump block3
+;
+; block3:
+; jump block4
+;
+; block4:
+; return
+; }
+
+;; Same thing, but instructions distributed cross the chain differently.
+function %chain2(i64) {
+ region0 = 0 "flags"
+
+block0(v0: i64):
+ v1 = load.i32 notrap aligned region0 v0
+ jump block1
+
+block1:
+ trapz v1, user26
+ jump block2
+
+block2:
+ v2 = iconst.i32 0
+ store notrap aligned region0 v2, v0
+ jump block3
+
+block3:
+ store notrap aligned region0 v1, v0
+ jump block4
+
+block4:
+ v3 = load.i32 notrap aligned region0 v0
+ trapz v3, user26
+
+ return
+}
+
+; function %chain2(i64) fast {
+; region0 = 0 "flags"
+;
+; block0(v0: i64):
+; v1 = load.i32 notrap aligned region0 v0
+; jump block1
+;
+; block1:
+; trapz.i32 v1, user26
+; jump block2
+;
+; block2:
+; jump block3
+;
+; block3:
+; jump block4
+;
+; block4:
+; return
+; }
diff --git a/cranelift/filetests/filetests/alias/inherit-single-pred-siblings.clif b/cranelift/filetests/filetests/alias/inherit-single-pred-siblings.clif
new file mode 100644
index 0000000000..9a213abbce
--- /dev/null
+++ b/cranelift/filetests/filetests/alias/inherit-single-pred-siblings.clif
@@ -0,0 +1,81 @@
+test optimize precise-output
+set opt_level=speed
+target x86_64
+
+;; A parent's exit state must stay live across *all* of its dominator-tree
+;; children, so that every sibling can inherit the parent's end-of-block state.
+
+function %siblings(i64, i64, i32) -> i32, i32, i32, i32 {
+ region0 = 0 "flags"
+ region1 = 1 "R1"
+
+block0(v0: i64, v1: i64, v2: i32):
+ v3 = load.i32 notrap aligned region0 v0
+ v4 = load.i32 notrap aligned region1 v1
+
+ ;; Dead store.
+ v5 = iconst.i32 0
+ store notrap aligned region0 v5, v0
+
+ ;; Idempotent store.
+ store notrap aligned region0 v3, v0
+
+ brif v2, block1, block4
+
+;; The first child of `block0`, whose sole predecessor is `block0`. Should
+;; inherit `block0`'s exit state.
+block1:
+ ;; Clobber `region0`.
+ v6 = iconst.i32 1
+ store notrap aligned region0 v6, v0
+ jump block2
+
+block2:
+ ;; Redundant; should be replaced with `v4`.
+ v7 = load.i32 notrap aligned region1 v1
+ jump block3
+
+block3:
+ ;; Forwards to (a rematerialized) `v6` from the store in `block1`, not to
+ ;; the load of `v3`.
+ v8 = load.i32 notrap aligned region0 v0
+
+ return v3, v4, v7, v8
+
+;; The second child of `block0`, whose sole predecessor is `block0`. Sibling to
+;; `block1`. Should inherit `block0`'s exit state.
+block4:
+ ;; Redundant; should be replaced with `v3`.
+ v9 = load.i32 notrap aligned region0 v0
+
+ ;; Redundant; should be replaced with `v4`.
+ v10 = load.i32 notrap aligned region1 v1
+
+ return v3, v4, v9, v10
+}
+
+; function %siblings(i64, i64, i32) -> i32, i32, i32, i32 fast {
+; region0 = 0 "flags"
+; region1 = 1 "R1"
+;
+; block0(v0: i64, v1: i64, v2: i32):
+; v3 = load.i32 notrap aligned region0 v0
+; v4 = load.i32 notrap aligned region1 v1
+; brif v2, block1, block4
+;
+; block1:
+; v6 = iconst.i32 1
+; store notrap aligned region0 v6, v0 ; v6 = 1
+; jump block2
+;
+; block2:
+; jump block3
+;
+; block3:
+; v11 = iconst.i32 1
+; return v3, v4, v4, v11 ; v11 = 1
+;
+; block4:
+; return v3, v4, v3, v4
+; }
+
diff --git a/cranelift/filetests/filetests/alias/issue-13961.clif b/cranelift/filetests/filetests/alias/issue-13961.clif
index f30966499b..d00d97c120 100644
--- a/cranelift/filetests/filetests/alias/issue-13961.clif
+++ b/cranelift/filetests/filetests/alias/issue-13961.clif
@@ -44,6 +44,6 @@ block1:
; jump block1
;
; block1:
-; store.i32 notrap region0 v1, v0
; return v1
; }
+
diff --git a/tests/disas/component-model/compile-time-builtins-x64.wat b/tests/disas/component-model/compile-time-builtins-x64.wat
index 29c0dad0cd..e4753ecdf8 100644
--- a/tests/disas/component-model/compile-time-builtins-x64.wat
+++ b/tests/disas/component-model/compile-time-builtins-x64.wat
@@ -180,33 +180,29 @@
;; movq 0xe8(%rcx), %rax
;; movl (%rax), %edx
;; testl %edx, %edx
-;; je 0x179
+;; je 0x16d
;; 119: movq 0x48(%rcx), %rcx
;; movq 8(%rcx), %rcx
;; movq 0x68(%rcx), %rcx
;; movq 8(%rcx), %rdx
;; xorq %rsi, %rsi
;; cmpq %rdx, %rsi
-;; jae 0x174
+;; jae 0x168
;; 135: movl (%rax), %edi
;; testl %edi, %edi
-;; je 0x17b
+;; je 0x16f
;; 13f: movq 8(%rcx), %rdi
;; cmpq %rdi, %rsi
-;; jae 0x17d
+;; jae 0x171
;; 14c: movq (%rcx), %rdi
;; movzbq (%rdi, %rsi), %r8
-;; movl (%rax), %r9d
-;; testl %r9d, %r9d
-;; je 0x17f
-;; 160: addb $1, %r8b
+;; addb $1, %r8b
;; movb %r8b, (%rdi, %rsi)
;; addq $1, %rsi
;; jmp 0x12c
-;; 174: movq %rbp, %rsp
+;; 168: movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 179: ud2
-;; 17b: ud2
-;; 17d: ud2
-;; 17f: ud2
+;; 16d: ud2
+;; 16f: ud2
+;; 171: ud2
diff --git a/tests/disas/component-model/compile-time-builtins.wat b/tests/disas/component-model/compile-time-builtins.wat
index b583fce136..962dcdd5a9 100644
--- a/tests/disas/component-model/compile-time-builtins.wat
+++ b/tests/disas/component-model/compile-time-builtins.wat
@@ -417,12 +417,9 @@
;; jump block51
;;
;; block51:
-;; v87 = load.i32 notrap aligned region4 v23
-;; trapz v87, user26
;; jump block54
;;
;; block54:
-;; v89 = load.i32 notrap aligned region5 v25
;; jump block57
;;
;; block57: