A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: preserve unchanged alias-region extents across joins (#14797)
nv3ifu5d9cec06d500
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rsindex 97968fd65f..2b74efd761 100644--- a/cranelift/codegen/src/alias_analysis.rs+++ b/cranelift/codegen/src/alias_analysis.rs@@ -656,25 +656,6 @@ pub struct MemoryState {const NULL_EXTENT: u32 = 0;-impl MemoryState {- /// Get the extent token to key a `MemoryLoc` on, given the last store to- /// that location's alias region.- fn extent_for(&self, last_store: LastStore) -> u32 {- match last_store {- // Each of these already names a version of memory all on its own,- // function wide, and so doesn't need an additional extent token.- LastStore::NoStore | LastStore::Inst(_) => NULL_EXTENT,-- // This one, on the other hand, only names a version of memory- // relative to the control-flow join that lost track of it.- LastStore::Unknown => {- debug_assert_ne!(self.extent, NULL_EXTENT);- self.extent- }- }- }-}-/// A key identifying a unique memory location.////// For the result of a load to be equivalent to the result of another load, or@@ -688,12 +669,14 @@ struct MemoryLoc {/// `LastStore::Unknown` is only valid relative to its last control-flow/// join point.///- /// An extent is a control-flow join point and everything reachable from it- /// through single-predecessor edges. For `LastStore::Unknown`, this- /// identifies the area where the bytes in the associated alias region are- /// identical to when control entered the join point, and therefore exactly- /// the area within which two `LastStore::Unknown`-keyed memory locations- /// name the same value.+ /// For `LastStore::Unknown`, the token distinguishes unknown memory versions+ /// at different control-flow joins. Named regions can retain their token+ /// across a join when no path from the immediate dominator's exit writes+ /// the region and its effective last-store state agrees with the join's+ /// initial fixpoint snapshot. Otherwise, the region gets a fresh token.+ /// Different tokens do not imply different bytes, only that equivalence+ /// has not been established. Reusing a value still requires the full key+ /// to match and the known value's definition to dominate its use.////// This is necessary because on one hand, we need `LastStore::Unknown` to/// be a single bottom value so that `LastStore` and `LastStores` are proper@@ -825,6 +808,16 @@ pub struct AliasAnalysis<'a> {/// Counter for minting fresh extent tokens.next_extent: u32,++ /// Regions written by each block before optimization, so eliminated stores+ /// remain conservatively accounted for. `None` means a fence can clobber+ /// every region.+ block_clobbers: SecondaryMap<Block, Option<FxHashSet<AliasRegion>>>,++ /// Reusable set of regions written along the paths being searched.+ clobbers_scratch: FxHashSet<AliasRegion>,+ pending_scratch: SmallVec<[Block; 8]>,+ visited_scratch: FxHashSet<Block>,}/// One block's scope on the dominator-tree path currently being walked.@@ -837,6 +830,9 @@ struct Scope {/// See `MemoryLoc::extent` for details.extent: u32,+ /// Per-region tokens inherited by children; see `MemoryLoc::extent`.+ region_extents: SecondaryMap<AliasRegion, u32>,+/// This block's last-store state on exit, once we have finished walking its/// instructions.///@@ -854,6 +850,24 @@ impl<'a> AliasAnalysis<'a> {trace!("alias analysis input is:\n{func:?}");assert!(domtree.is_valid());+ let block_clobbers = func+ .layout+ .blocks()+ .map(|block| {+ let mut regions = FxHashSet::default();+ for inst in func.layout.block_insts(block) {+ match classify(func, inst) {+ InstEffect::Fence => return (block, None),+ InstEffect::Store { region, .. } => {+ regions.insert(region);+ }+ _ => {}+ }+ }+ (block, Some(regions))+ })+ .collect();+let mut analysis = AliasAnalysis {domtree,post_dom_tree: None,@@ -862,6 +876,10 @@ impl<'a> AliasAnalysis<'a> {mem_values: FxHashMap::default(),scopes: SmallVec::new(),next_extent: 0,+ block_clobbers,+ clobbers_scratch: FxHashSet::default(),+ pending_scratch: SmallVec::new(),+ visited_scratch: FxHashSet::default(),};analysis.compute_block_input_states(func);@@ -1085,6 +1103,25 @@ impl<'a> AliasAnalysis<'a> {self.observed_stores = observed_stores;}+ /// Get the extent token to key a `MemoryLoc` on for a memory-access instruction.+ fn extent_for(&self, state: &MemoryState, func: &Function, inst: Inst) -> u32 {+ match state.stores.get_last_store(func, inst) {+ // Each of these already names a version of memory all on its own,+ // function wide, and so doesn't need an additional extent token.+ LastStore::NoStore | LastStore::Inst(_) => NULL_EXTENT,++ // This one, on the other hand, only names a version of memory+ // relative to the control-flow join that lost track of it.+ LastStore::Unknown => {+ debug_assert_ne!(state.extent, NULL_EXTENT);+ let scope = self.scopes.last().expect("active alias-analysis scope");+ func.dfg.insts[inst]+ .alias_region(&func.dfg)+ .map_or(state.extent, |region| scope.region_extents[region])+ }+ }+ }+/// Get the solved last-store state on entry to the given block.fn block_input_stores(&self, block: Block) -> LastStores {match self.block_input.get(&block) {@@ -1093,6 +1130,98 @@ impl<'a> AliasAnalysis<'a> {}}+ /// Regions written on paths from `dominator`'s exit to `block`'s entry.+ /// Return `false` if a fence or missing effects prevent proving which regions+ /// are untouched. Also give up after 64 blocks to bound the analysis cost.+ fn clobbers_since(+ &self,+ cfg: &ControlFlowGraph,+ block: Block,+ dominator: Block,+ clobbers: &mut FxHashSet<AliasRegion>,+ pending: &mut SmallVec<[Block; 8]>,+ visited: &mut FxHashSet<Block>,+ ) -> bool {+ debug_assert!(clobbers.is_empty());+ debug_assert!(pending.is_empty());+ debug_assert!(visited.is_empty());+ pending.extend(cfg.pred_iter(block).map(|p| p.block));+ while let Some(pred) = pending.pop() {+ if pred == dominator || !self.block_input.contains_key(&pred) || !visited.insert(pred) {+ continue;+ }+ if visited.len() > 64 {+ return false;+ }+ let Some(Some(regions)) = self.block_clobbers.get(pred) else {+ return false;+ };+ clobbers.extend(regions.iter().copied());+ pending.extend(cfg.pred_iter(pred).map(|p| p.block));+ }+ true+ }++ /// Compute region tokens for a new extent, inheriting unchanged regions when safe.+ fn compute_region_extents(+ &mut self,+ cfg: &ControlFlowGraph,+ block: Block,+ stores: &LastStores,+ extent: u32,+ ) -> SecondaryMap<AliasRegion, u32> {+ let mut region_extents = SecondaryMap::with_default(extent);+ let mut clobbers = core::mem::take(&mut self.clobbers_scratch);+ let mut pending = core::mem::take(&mut self.pending_scratch);+ let mut visited = core::mem::take(&mut self.visited_scratch);+ clobbers.clear();+ pending.clear();+ visited.clear();+ if let Some(parent) = self.scopes.last() {+ let parent_stores = parent.exit_state.as_ref().expect("finished parent scope");+ // The parent's and join's fence states must match.+ // The parent must have an Unknown version.+ // The path search must identify the regions that may be written.+ if parent_stores.last_fence == stores.last_fence+ && (parent_stores.last_fence == LastStore::Unknown+ || parent_stores+ .regions+ .iter()+ .any(|(_, slot)| *slot == LastStore::Unknown))+ && self.clobbers_since(+ cfg,+ block,+ parent.block,+ &mut clobbers,+ &mut pending,+ &mut visited,+ )+ {+ region_extents = parent.region_extents.clone();+ for ®ion in &clobbers {+ region_extents[region] = extent;+ }+ let len = parent_stores+ .regions+ .keys()+ .len()+ .max(stores.regions.keys().len());+ for i in 0..len {+ let region = AliasRegion::new(i);+ if parent_stores.last_store_for_region(region)+ != stores.last_store_for_region(region)+ {+ region_extents[region] = extent;+ }+ }+ }+ }+ self.clobbers_scratch = clobbers;+ self.pending_scratch = pending;+ self.visited_scratch = visited;+ region_extents+ }+/// Enter the given block's scope, getting its initial memory state.////// Callers must walk the dominator tree in pre-order, entering each block's@@ -1110,7 +1239,7 @@ impl<'a> AliasAnalysis<'a> {// memory that its predecessor left behind.let only_pred = self.single_reachable_pred(cfg, block);- let (stores, extent) = match (only_pred, self.scopes.last()) {+ let (stores, extent, region_extents) = match (only_pred, self.scopes.last()) {// With exactly one reachable predecessor, this block's entry state// is exactly that predecessor's exit state, which we just finished// computing and is strictly more precise than the `block_input`@@ -1132,20 +1261,24 @@ impl<'a> AliasAnalysis<'a> {.as_ref().expect("`finish_scope` is always called before descending into a child scope").clone();- (stores, parent.extent)+ (stores, parent.extent, parent.region_extents.clone())}// Anything else (e.g. a control-flow join, the entry block) falls// back to the initial fixpoint's snapshot, and begins a new extent._ => {self.next_extent += 1;- (self.block_input_stores(block), self.next_extent)+ let extent = self.next_extent;+ let stores = self.block_input_stores(block);+ let region_extents = self.compute_region_extents(cfg, block, &stores, extent);+ (stores, extent, region_extents)}};self.scopes.push(Scope {block,extent,+ region_extents,exit_state: None,});@@ -1273,7 +1406,7 @@ impl<'a> AliasAnalysis<'a> {let dead_slot = LastStore::Inst(last_store);let dead_loc = MemoryLoc {last_store: dead_slot,- extent: state.extent_for(dead_slot),+ extent: NULL_EXTENT,address,offset,ty,@@ -1323,7 +1456,7 @@ impl<'a> AliasAnalysis<'a> {let check_loc = MemoryLoc {last_store,- extent: state.extent_for(last_store),+ extent: self.extent_for(state, func, inst),address,offset,ty,@@ -1367,7 +1500,7 @@ impl<'a> AliasAnalysis<'a> {let this_store = LastStore::Inst(inst);let mem_loc = MemoryLoc {last_store: this_store,- extent: state.extent_for(this_store),+ extent: NULL_EXTENT,address,offset,ty,@@ -1394,7 +1527,7 @@ impl<'a> AliasAnalysis<'a> {let load_result = func.dfg.inst_results(inst)[0];let mem_loc = MemoryLoc {last_store,- extent: state.extent_for(last_store),+ extent: self.extent_for(state, func, inst),address,offset,ty,diff --git a/cranelift/filetests/filetests/alias/issue-14312.clif b/cranelift/filetests/filetests/alias/issue-14312.clifnew file mode 100644index 0000000000..780b45cc3f--- /dev/null+++ b/cranelift/filetests/filetests/alias/issue-14312.clif@@ -0,0 +1,64 @@+test optimize precise-output+set opt_level=speed+target x86_64++;; Regression for #14312: a join that only writes another region must not+;; prevent reuse of a load with an unknown incoming memory version.+function %unchanged_region_join(i64, i64, i32) -> i32, i32 {+ region0 = 0 "read"+ region1 = 1 "write"+ sig0 = ()+ fn0 = %callee sig0++block0(v0: i64, v1: i64, v2: i32):+ brif v2, block1, block2+block1:+ call fn0()+ jump block3+block2:+ call fn0()+ jump block3+block3:+ v3 = load.i32 notrap region0 v0+ brif v2, block4, block5+block4:+ store notrap region1 v2, v1+ jump block6+block5:+ jump block6+block6:+ v4 = load.i32 notrap region0 v0+ return v3, v4+}++; function %unchanged_region_join(i64, i64, i32) -> i32, i32 fast {+; region0 = 0 "read"+; region1 = 1 "write"+; sig0 = () fast+; fn0 = %callee sig0+;+; block0(v0: i64, v1: i64, v2: i32):+; brif v2, block1, block2+;+; block1:+; call fn0()+; jump block3+;+; block2:+; call fn0()+; jump block3+;+; block3:+; v3 = load.i32 notrap region0 v0+; brif.i32 v2, block4, block5+;+; block4:+; store.i32 notrap region1 v2, v1+; jump block6+;+; block5:+; jump block6+;+; block6:+; return v3, v3+; }