A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Observe all stores we replace in `LastStores`; keep track of who observed a store (#14080)
Nick Fitzgerald0b0820c4416d
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rsindex 9532b39c6e..c15769fd70 100644--- a/cranelift/codegen/src/alias_analysis.rs+++ b/cranelift/codegen/src/alias_analysis.rs@@ -74,10 +74,9 @@//! up front, in `AliasAnalysis::observed_stores`, rather than tracking//! it as part of the per-block `LastStores` state.-use crate::cursor::CursorPosition;use crate::{FxHashMap, FxHashSet};use crate::{- cursor::{Cursor, FuncCursor},+ cursor::{Cursor, CursorPosition, FuncCursor},dominator_tree::DominatorTree,flowgraph::ControlFlowGraph,inst_predicates::{inst_addr_offset_type, inst_store_data, visit_block_succs},@@ -144,6 +143,48 @@ fn alias_regions_observed(func: &Function, inst: Inst, opcode: Opcode) -> AliasR}}+/// Who was the observer of some store instruction?+///+/// `Option<Observer>` -- where `None` is logically represented by the absense+/// of an entry in `AliasAnalysis::observed_stores` -- forms the following+/// lattice:+///+/// ```ignore+/// None+/// / | \ \ \+/// / | \ \ \+/// / | \ \ \+/// / | \ \ \+/// inst0 inst1 instN...+/// \ | / / /+/// \ | / / /+/// \ | / / /+/// \ | / / /+/// Many+/// ```+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(crate) enum Observer {+ /// There was exactly one observer: this instruction.+ One(Inst),+ /// There were many observers.+ Many,+}++impl Observer {+ fn meet(a: Self, b: Self) -> Self {+ match (a, b) {+ (Observer::Many, _) | (_, Observer::Many) => Observer::Many,+ (Observer::One(a), Observer::One(b)) => {+ if a == b {+ Observer::One(a)+ } else {+ Observer::Many+ }+ }+ }+ }+}+/// For a given program point, the last-store instruction for each disjoint/// category of abstract state.///@@ -176,14 +217,23 @@ pub struct LastStores {}/// Mark the store, if any, in the given last-store slot as observed.-fn observe(func: &Function, observed_stores: &mut FxHashSet<Inst>, last_store: PackedOption<Inst>) {- if let Some(inst) = last_store.expand() {+fn observe(+ func: &Function,+ observed_stores: &mut FxHashMap<Inst, Observer>,+ last_store: PackedOption<Inst>,+ observer: Inst,+) {+ if let Some(last_store) = last_store.expand() {// NB: last-store slots do not always hold stores; they can also hold// calls, fences, and the markers that `LastStores::meet_from` inserts// where two control-flow paths disagree. Only actual stores can be DSE// candidates, so don't bother recording other instructions as observed.- if func.dfg.insts[inst].opcode().can_store() {- observed_stores.insert(inst);+ if func.dfg.insts[last_store].opcode().can_store() {+ let entry = observed_stores+ .entry(last_store)+ .or_insert(Observer::One(observer));+ *entry = Observer::meet(*entry, Observer::One(observer));+ trace!(" observed_stores[{last_store:?}] = {entry:?}");}}}@@ -193,7 +243,7 @@ impl LastStores {&mut self,func: &Function,inst: Inst,- observed_stores: &mut FxHashSet<Inst>,+ observed_stores: &mut FxHashMap<Inst, Observer>,) {let opcode = func.dfg.insts[inst].opcode();@@ -207,7 +257,7 @@ impl LastStores {// state of memory on trap. We do this by marking every last-store as// observed, but not clearing our last-store information.else if opcode.can_trap() {- self.observe_others(func, observed_stores, None);+ self.observe_others(func, observed_stores, None, inst);}// Store instructions: update the last-store information for this// instruction's alias region, or, if it has no alias region, treat it@@ -216,30 +266,7 @@ impl LastStores {if let Some(memflags) = func.dfg.insts[inst].memflags() {match func.dfg.mem_flags[memflags].alias_region() {Some(region) => {- // NB: The old last-store instruction is *not* observed- // here, even though this new store instruction may not- // fully overwrite it. First, a new store in a block- // does not itself observe an old store in the same- // block. Second, the old store will never be an- // optimization candidate again from here on out:- //- // * We won't consider it again as we process the rest- // of this block, as it won't be in the last-store- // slot anymore.- //- // * What if we re-process this block in our initial- // fixed point loop? That implies this block is a- // member of a cycle in the CFG, but `meet_from` only- // propagates a store instruction when all- // predecessors agree on the same last-store- // instruction, but the predecessors already won't- // agree it is the old store since this block (which- // is on that path and therefore some kind of- // transitive predecessor) has already overridden it.- //- // Therefore, marking the old last-store as observed- // here is unnecessary (and, in fact, doing so would- // only inhibit optimization).+ observe(func, observed_stores, self.regions[region], inst);self.regions[region] = inst.into();// If this store can trap, then we need to observe@@ -282,9 +309,9 @@ impl LastStores {// incorrectly store to `v4+16`, when we otherwise// wouldn't have.if func.dfg.mem_flags[memflags].trap_code().is_some() {- self.observe_others(func, observed_stores, Some(region));+ self.observe_others(func, observed_stores, Some(region), inst);} else {- self.observe_trapping_others(func, observed_stores, region);+ self.observe_trapping_others(func, observed_stores, region, inst);}}None => {@@ -303,15 +330,22 @@ impl LastStores {// instruction observes.else {match alias_regions_observed(func, inst, opcode) {- AliasRegionsObserved::All => self.observe_others(func, observed_stores, None),+ AliasRegionsObserved::All => self.observe_others(func, observed_stores, None, inst),AliasRegionsObserved::Just(region) => {- observe(func, observed_stores, self.last_store_for_region(region));+ observe(+ func,+ observed_stores,+ self.last_store_for_region(region),+ inst,+ );// NB: Because stores without regions may alias any other// region, we have also observed the last such store, which// `self.last_fence` tracks.- observe(func, observed_stores, self.last_fence);+ observe(func, observed_stores, self.last_fence, inst);+ }+ AliasRegionsObserved::Other => {+ observe(func, observed_stores, self.last_fence, inst)}- AliasRegionsObserved::Other => observe(func, observed_stores, self.last_fence),AliasRegionsObserved::None => {}}}@@ -322,15 +356,16 @@ impl LastStores {fn observe_others(&self,func: &Function,- observed_stores: &mut FxHashSet<Inst>,+ observed_stores: &mut FxHashMap<Inst, Observer>,excluding: Option<AliasRegion>,+ observer: Inst,) {for (region, last_store) in self.regions.iter() {if excluding.is_none_or(|r| r != region) {- observe(func, observed_stores, *last_store);+ observe(func, observed_stores, *last_store, observer);}}- observe(func, observed_stores, self.last_fence);+ observe(func, observed_stores, self.last_fence, observer);}/// Mark the last store to every region whose last store can trap, except for@@ -338,8 +373,9 @@ impl LastStores {fn observe_trapping_others(&self,func: &Function,- observed_stores: &mut FxHashSet<Inst>,+ observed_stores: &mut FxHashMap<Inst, Observer>,excluding: AliasRegion,+ observer: Inst,) {let can_trap = |last_store: PackedOption<Inst>| {last_store@@ -349,21 +385,26 @@ impl LastStores {for (region, last_store) in self.regions.iter() {if region != excluding && can_trap(*last_store) {- observe(func, observed_stores, *last_store);+ observe(func, observed_stores, *last_store, observer);}}if can_trap(self.last_fence) {- observe(func, observed_stores, self.last_fence);+ observe(func, observed_stores, self.last_fence, observer);}}/// Handle memory fence-like instructions by clearing all analysis data.- fn fence(&mut self, func: &Function, inst: Inst, observed_stores: &mut FxHashSet<Inst>) {+ fn fence(+ &mut self,+ func: &Function,+ inst: Inst,+ observed_stores: &mut FxHashMap<Inst, Observer>,+ ) {// A fence can observe every region, so every store we are currently// tracking for a region becomes observed.for (_region, last_store) in self.regions.iter() {- observe(func, observed_stores, *last_store);+ observe(func, observed_stores, *last_store, inst);}self.regions.clear();@@ -409,7 +450,7 @@ impl LastStores {func: &Function,rhs: &LastStores,loc: Inst,- observed_stores: &mut FxHashSet<Inst>,+ observed_stores: &mut FxHashMap<Inst, Observer>,) -> bool {// NB: Destructure to make sure we don't accidentally forget a// field.@@ -418,7 +459,7 @@ impl LastStores {last_fence,} = self;- let meet = |observed_stores: &mut FxHashSet<Inst>,+ let meet = |observed_stores: &mut FxHashMap<Inst, Observer>,a: &mut PackedOption<Inst>,b: PackedOption<Inst>|-> bool {@@ -433,8 +474,8 @@ impl LastStores {// mark them both observed here. This keeps the// observed-stores set sound in the presence of loops and// control-flow join points.- observe(func, observed_stores, x.filter(|x| *x != loc).into());- observe(func, observed_stores, y.filter(|y| *y != loc).into());+ observe(func, observed_stores, x.filter(|x| *x != loc).into(), loc);+ observe(func, observed_stores, y.filter(|y| *y != loc).into(), loc);Some(loc)}};@@ -527,7 +568,7 @@ pub struct AliasAnalysis<'a> {/// Unlike the last-store state in `block_input`, this is *not* flow/// sensitive: a store is either observable somewhere in the function or it/// is not.- observed_stores: FxHashSet<Inst>,+ observed_stores: FxHashMap<Inst, Observer>,/// Input state to a basic block.block_input: FxHashMap<Block, LastStores>,@@ -543,12 +584,12 @@ pub struct AliasAnalysis<'a> {impl<'a> AliasAnalysis<'a> {/// Perform an alias analysis pass.pub fn new(func: &Function, domtree: &'a DominatorTree) -> AliasAnalysis<'a> {- trace!("alias analysis: input is:\n{:?}", func);+ trace!("alias analysis input is:\n{func:?}");assert!(domtree.is_valid());let mut analysis = AliasAnalysis {domtree,post_dom_tree: None,- observed_stores: FxHashSet::default(),+ observed_stores: FxHashMap::default(),block_input: FxHashMap::default(),mem_values: FxHashMap::default(),};@@ -604,15 +645,13 @@ impl<'a> AliasAnalysis<'a> {.or_insert_with(|| LastStores::default()).clone();- trace!(- "alias analysis: input to block{} is {:?}",- block.index(),- state- );+ trace!("analyzing {block:?}");+ trace!(" initial block state = {state:?}");for inst in func.layout.block_insts(block) {+ trace!(" analyzing {inst:?}: {}", func.dfg.display_inst(inst));state.update(func, inst, &mut self.observed_stores);- trace!("after inst{}: state is {:?}", inst.index(), state);+ trace!(" updated state = {state:?}");}visit_block_succs(func, block, |_inst, succ, _from_table| {@@ -635,6 +674,8 @@ impl<'a> AliasAnalysis<'a> {}});}++ trace!("final observed_stores = {:#?}", self.observed_stores);}/// Get the starting state for a block.@@ -673,8 +714,9 @@ impl<'a> AliasAnalysis<'a> {// Check whether this store makes the last store dead.if let Some(last_store) = last_store.expand() {- // A store can only be dead when unobserved.- if !self.observed_stores.contains(&last_store)+ // A store can only be dead when unobserved or only observed+ // by its overwriter.+ if self.observed_stores.get(&last_store).is_none_or(|o| *o == Observer::One(inst))// This instruction doesn't make the last// store dead if it itself is the last store.&& inst != last_store@@ -742,8 +784,13 @@ impl<'a> AliasAnalysis<'a> {// We are removing this idempotent store in favor of the// original, so if this idempotent store was observed,// then the original must now be observed as well.- if self.observed_stores.contains(&inst) {- observe(func, &mut self.observed_stores, last_store);+ if let Some(last_store) = last_store.expand() {+ if let Some(observer) = self.observed_stores.get(&inst).copied() {+ let entry =+ self.observed_stores.entry(last_store).or_insert(observer);+ *entry = Observer::meet(*entry, observer);+ trace!(" observed_stores[{last_store:?}] = {entry:?}");+ }}return OptResult::IdempotentStore;diff --git a/cranelift/filetests/filetests/alias/issue-14053.clif b/cranelift/filetests/filetests/alias/issue-14053.clifnew file mode 100644index 0000000000..501de5ec86--- /dev/null+++ b/cranelift/filetests/filetests/alias/issue-14053.clif@@ -0,0 +1,144 @@+test optimize precise-output+set opt_level=speed+target aarch64++;; A *divergent* block is one that can never reach a function exit, because it+;; is part of, or only leads to, an infinite loop. But a divergent path can+;; still leave the function via an implicit trap, which makes the memory state+;; at that point observable, so dead-store elimination may not treat it as if it+;; were unreachable.++;; The store in `block0` is overwritten by the store in `block1`, and `block1`+;; post-dominates `block0`. However, the `block2` path diverges into an infinite+;; loop which never reaches the exit, and it can trap, which must observe the+;; store from `block0`.+function %divergent_path_can_trap(i64, i32, i32, i32) {+ region0 = 0 "R0"++block0(v0: i64, v1: i32, v2: i32, v3: i32):+ store notrap aligned region0 v1, v0+ brif v2, block1, block2++block1:+ store notrap aligned region0 v2, v0+ return++block2:+ store user1 aligned region0 v3, v0+8+ jump block3++block3:+ jump block3+}++; function %divergent_path_can_trap(i64, i32, i32, i32) fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32, v2: i32, v3: i32):+; store notrap aligned region0 v1, v0+; brif v2, block1, block2+;+; block1:+; store.i32 notrap aligned region0 v2, v0+; return+;+; block2:+; store.i32 user1 aligned region0 v3, v0+8+; jump block3+;+; block3:+; jump block3+; }++;; Same as above, except that the divergent path stores to `region0` before it+;; can trap.+function %shadowed_before_trap_on_divergent_path(i64, i32, i32, i32) {+ region0 = 0 "R0"++block0(v0: i64, v1: i32, v2: i32, v3: i32):+ store notrap aligned region0 v1, v0+ brif v2, block1, block2++block1:+ store notrap aligned region0 v2, v0+ return++block2:+ store notrap aligned region0 v3, v0+8+ store user1 aligned region0 v3, v0+16+ jump block3++block3:+ jump block3+}++; function %shadowed_before_trap_on_divergent_path(i64, i32, i32, i32) fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32, v2: i32, v3: i32):+; store notrap aligned region0 v1, v0+; brif v2, block1, block2+;+; block1:+; store.i32 notrap aligned region0 v2, v0+; return+;+; block2:+; store.i32 notrap aligned region0 v3, v0+8+; store.i32 user1 aligned region0 v3, v0+16+; jump block3+;+; block3:+; jump block3+; }++;; Both stores are themselves in divergent blocks.+function %both_stores_in_divergent_blocks(i64, i32, i32, i32) {+ region0 = 0 "R0"++block0(v0: i64, v1: i32, v2: i32, v3: i32):+ brif v2, block1, block5++block1:+ store notrap aligned region0 v1, v0+ brif v3, block2, block3++block2:+ store user1 aligned region0 v3, v0+8+ jump block4++block3:+ store notrap aligned region0 v2, v0+ jump block4++block4:+ jump block4++block5:+ return+}++; function %both_stores_in_divergent_blocks(i64, i32, i32, i32) fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32, v2: i32, v3: i32):+; brif v2, block1, block5+;+; block1:+; store.i32 notrap aligned region0 v1, v0+; brif.i32 v3, block2, block3+;+; block2:+; store.i32 user1 aligned region0 v3, v0+8+; jump block4+;+; block3:+; store.i32 notrap aligned region0 v2, v0+; jump block4+;+; block4:+; jump block4+;+; block5:+; return+; }