Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: preserve unchanged alias-region extents across joins (#14797)

nv3ifu5d9cec06d500
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rs
index 97968fd65f..2b74efd761 100644
--- a/cranelift/codegen/src/alias_analysis.rs
+++ b/cranelift/codegen/src/alias_analysis.rs
@@ -656,25 +656,6 @@ pub struct MemoryState {
const NULL_EXTENT: u32 = 0;
-impl MemoryState {
- /// Get the extent token to key a `MemoryLoc` on, given the last store to
- /// that location's alias region.
- fn extent_for(&self, last_store: LastStore) -> u32 {
- match last_store {
- // Each of these already names a version of memory all on its own,
- // function wide, and so doesn't need an additional extent token.
- LastStore::NoStore | LastStore::Inst(_) => NULL_EXTENT,
-
- // This one, on the other hand, only names a version of memory
- // relative to the control-flow join that lost track of it.
- LastStore::Unknown => {
- debug_assert_ne!(self.extent, NULL_EXTENT);
- self.extent
- }
- }
- }
-}
-
/// A key identifying a unique memory location.
///
/// For the result of a load to be equivalent to the result of another load, or
@@ -688,12 +669,14 @@ struct MemoryLoc {
/// `LastStore::Unknown` is only valid relative to its last control-flow
/// join point.
///
- /// An extent is a control-flow join point and everything reachable from it
- /// through single-predecessor edges. For `LastStore::Unknown`, this
- /// identifies the area where the bytes in the associated alias region are
- /// identical to when control entered the join point, and therefore exactly
- /// the area within which two `LastStore::Unknown`-keyed memory locations
- /// name the same value.
+ /// For `LastStore::Unknown`, the token distinguishes unknown memory versions
+ /// at different control-flow joins. Named regions can retain their token
+ /// across a join when no path from the immediate dominator's exit writes
+ /// the region and its effective last-store state agrees with the join's
+ /// initial fixpoint snapshot. Otherwise, the region gets a fresh token.
+ /// Different tokens do not imply different bytes, only that equivalence
+ /// has not been established. Reusing a value still requires the full key
+ /// to match and the known value's definition to dominate its use.
///
/// This is necessary because on one hand, we need `LastStore::Unknown` to
/// be a single bottom value so that `LastStore` and `LastStores` are proper
@@ -825,6 +808,16 @@ pub struct AliasAnalysis<'a> {
/// Counter for minting fresh extent tokens.
next_extent: u32,
+
+ /// Regions written by each block before optimization, so eliminated stores
+ /// remain conservatively accounted for. `None` means a fence can clobber
+ /// every region.
+ block_clobbers: SecondaryMap<Block, Option<FxHashSet<AliasRegion>>>,
+
+ /// Reusable set of regions written along the paths being searched.
+ clobbers_scratch: FxHashSet<AliasRegion>,
+ pending_scratch: SmallVec<[Block; 8]>,
+ visited_scratch: FxHashSet<Block>,
}
/// One block's scope on the dominator-tree path currently being walked.
@@ -837,6 +830,9 @@ struct Scope {
/// See `MemoryLoc::extent` for details.
extent: u32,
+ /// Per-region tokens inherited by children; see `MemoryLoc::extent`.
+ region_extents: SecondaryMap<AliasRegion, u32>,
+
/// This block's last-store state on exit, once we have finished walking its
/// instructions.
///
@@ -854,6 +850,24 @@ impl<'a> AliasAnalysis<'a> {
trace!("alias analysis input is:\n{func:?}");
assert!(domtree.is_valid());
+ let block_clobbers = func
+ .layout
+ .blocks()
+ .map(|block| {
+ let mut regions = FxHashSet::default();
+ for inst in func.layout.block_insts(block) {
+ match classify(func, inst) {
+ InstEffect::Fence => return (block, None),
+ InstEffect::Store { region, .. } => {
+ regions.insert(region);
+ }
+ _ => {}
+ }
+ }
+ (block, Some(regions))
+ })
+ .collect();
+
let mut analysis = AliasAnalysis {
domtree,
post_dom_tree: None,
@@ -862,6 +876,10 @@ impl<'a> AliasAnalysis<'a> {
mem_values: FxHashMap::default(),
scopes: SmallVec::new(),
next_extent: 0,
+ block_clobbers,
+ clobbers_scratch: FxHashSet::default(),
+ pending_scratch: SmallVec::new(),
+ visited_scratch: FxHashSet::default(),
};
analysis.compute_block_input_states(func);
@@ -1085,6 +1103,25 @@ impl<'a> AliasAnalysis<'a> {
self.observed_stores = observed_stores;
}
+ /// Get the extent token to key a `MemoryLoc` on for a memory-access instruction.
+ fn extent_for(&self, state: &MemoryState, func: &Function, inst: Inst) -> u32 {
+ match state.stores.get_last_store(func, inst) {
+ // Each of these already names a version of memory all on its own,
+ // function wide, and so doesn't need an additional extent token.
+ LastStore::NoStore | LastStore::Inst(_) => NULL_EXTENT,
+
+ // This one, on the other hand, only names a version of memory
+ // relative to the control-flow join that lost track of it.
+ LastStore::Unknown => {
+ debug_assert_ne!(state.extent, NULL_EXTENT);
+ let scope = self.scopes.last().expect("active alias-analysis scope");
+ func.dfg.insts[inst]
+ .alias_region(&func.dfg)
+ .map_or(state.extent, |region| scope.region_extents[region])
+ }
+ }
+ }
+
/// Get the solved last-store state on entry to the given block.
fn block_input_stores(&self, block: Block) -> LastStores {
match self.block_input.get(&block) {
@@ -1093,6 +1130,98 @@ impl<'a> AliasAnalysis<'a> {
}
}
+ /// Regions written on paths from `dominator`'s exit to `block`'s entry.
+ /// Return `false` if a fence or missing effects prevent proving which regions
+ /// are untouched. Also give up after 64 blocks to bound the analysis cost.
+ fn clobbers_since(
+ &self,
+ cfg: &ControlFlowGraph,
+ block: Block,
+ dominator: Block,
+ clobbers: &mut FxHashSet<AliasRegion>,
+ pending: &mut SmallVec<[Block; 8]>,
+ visited: &mut FxHashSet<Block>,
+ ) -> bool {
+ debug_assert!(clobbers.is_empty());
+ debug_assert!(pending.is_empty());
+ debug_assert!(visited.is_empty());
+ pending.extend(cfg.pred_iter(block).map(|p| p.block));
+ while let Some(pred) = pending.pop() {
+ if pred == dominator || !self.block_input.contains_key(&pred) || !visited.insert(pred) {
+ continue;
+ }
+ if visited.len() > 64 {
+ return false;
+ }
+ let Some(Some(regions)) = self.block_clobbers.get(pred) else {
+ return false;
+ };
+ clobbers.extend(regions.iter().copied());
+ pending.extend(cfg.pred_iter(pred).map(|p| p.block));
+ }
+ true
+ }
+
+ /// Compute region tokens for a new extent, inheriting unchanged regions when safe.
+ fn compute_region_extents(
+ &mut self,
+ cfg: &ControlFlowGraph,
+ block: Block,
+ stores: &LastStores,
+ extent: u32,
+ ) -> SecondaryMap<AliasRegion, u32> {
+ let mut region_extents = SecondaryMap::with_default(extent);
+ let mut clobbers = core::mem::take(&mut self.clobbers_scratch);
+ let mut pending = core::mem::take(&mut self.pending_scratch);
+ let mut visited = core::mem::take(&mut self.visited_scratch);
+ clobbers.clear();
+ pending.clear();
+ visited.clear();
+ if let Some(parent) = self.scopes.last() {
+ let parent_stores = parent.exit_state.as_ref().expect("finished parent scope");
+ // The parent's and join's fence states must match.
+ // The parent must have an Unknown version.
+ // The path search must identify the regions that may be written.
+ if parent_stores.last_fence == stores.last_fence
+ && (parent_stores.last_fence == LastStore::Unknown
+ || parent_stores
+ .regions
+ .iter()
+ .any(|(_, slot)| *slot == LastStore::Unknown))
+ && self.clobbers_since(
+ cfg,
+ block,
+ parent.block,
+ &mut clobbers,
+ &mut pending,
+ &mut visited,
+ )
+ {
+ region_extents = parent.region_extents.clone();
+ for &region in &clobbers {
+ region_extents[region] = extent;
+ }
+ let len = parent_stores
+ .regions
+ .keys()
+ .len()
+ .max(stores.regions.keys().len());
+ for i in 0..len {
+ let region = AliasRegion::new(i);
+ if parent_stores.last_store_for_region(region)
+ != stores.last_store_for_region(region)
+ {
+ region_extents[region] = extent;
+ }
+ }
+ }
+ }
+ self.clobbers_scratch = clobbers;
+ self.pending_scratch = pending;
+ self.visited_scratch = visited;
+ region_extents
+ }
+
/// Enter the given block's scope, getting its initial memory state.
///
/// Callers must walk the dominator tree in pre-order, entering each block's
@@ -1110,7 +1239,7 @@ impl<'a> AliasAnalysis<'a> {
// memory that its predecessor left behind.
let only_pred = self.single_reachable_pred(cfg, block);
- let (stores, extent) = match (only_pred, self.scopes.last()) {
+ let (stores, extent, region_extents) = match (only_pred, self.scopes.last()) {
// With exactly one reachable predecessor, this block's entry state
// is exactly that predecessor's exit state, which we just finished
// computing and is strictly more precise than the `block_input`
@@ -1132,20 +1261,24 @@ impl<'a> AliasAnalysis<'a> {
.as_ref()
.expect("`finish_scope` is always called before descending into a child scope")
.clone();
- (stores, parent.extent)
+ (stores, parent.extent, parent.region_extents.clone())
}
// Anything else (e.g. a control-flow join, the entry block) falls
// back to the initial fixpoint's snapshot, and begins a new extent.
_ => {
self.next_extent += 1;
- (self.block_input_stores(block), self.next_extent)
+ let extent = self.next_extent;
+ let stores = self.block_input_stores(block);
+ let region_extents = self.compute_region_extents(cfg, block, &stores, extent);
+ (stores, extent, region_extents)
}
};
self.scopes.push(Scope {
block,
extent,
+ region_extents,
exit_state: None,
});
@@ -1273,7 +1406,7 @@ impl<'a> AliasAnalysis<'a> {
let dead_slot = LastStore::Inst(last_store);
let dead_loc = MemoryLoc {
last_store: dead_slot,
- extent: state.extent_for(dead_slot),
+ extent: NULL_EXTENT,
address,
offset,
ty,
@@ -1323,7 +1456,7 @@ impl<'a> AliasAnalysis<'a> {
let check_loc = MemoryLoc {
last_store,
- extent: state.extent_for(last_store),
+ extent: self.extent_for(state, func, inst),
address,
offset,
ty,
@@ -1367,7 +1500,7 @@ impl<'a> AliasAnalysis<'a> {
let this_store = LastStore::Inst(inst);
let mem_loc = MemoryLoc {
last_store: this_store,
- extent: state.extent_for(this_store),
+ extent: NULL_EXTENT,
address,
offset,
ty,
@@ -1394,7 +1527,7 @@ impl<'a> AliasAnalysis<'a> {
let load_result = func.dfg.inst_results(inst)[0];
let mem_loc = MemoryLoc {
last_store,
- extent: state.extent_for(last_store),
+ extent: self.extent_for(state, func, inst),
address,
offset,
ty,
diff --git a/cranelift/filetests/filetests/alias/issue-14312.clif b/cranelift/filetests/filetests/alias/issue-14312.clif
new file mode 100644
index 0000000000..780b45cc3f
--- /dev/null
+++ b/cranelift/filetests/filetests/alias/issue-14312.clif
@@ -0,0 +1,64 @@
+test optimize precise-output
+set opt_level=speed
+target x86_64
+
+;; Regression for #14312: a join that only writes another region must not
+;; prevent reuse of a load with an unknown incoming memory version.
+function %unchanged_region_join(i64, i64, i32) -> i32, i32 {
+ region0 = 0 "read"
+ region1 = 1 "write"
+ sig0 = ()
+ fn0 = %callee sig0
+
+block0(v0: i64, v1: i64, v2: i32):
+ brif v2, block1, block2
+block1:
+ call fn0()
+ jump block3
+block2:
+ call fn0()
+ jump block3
+block3:
+ v3 = load.i32 notrap region0 v0
+ brif v2, block4, block5
+block4:
+ store notrap region1 v2, v1
+ jump block6
+block5:
+ jump block6
+block6:
+ v4 = load.i32 notrap region0 v0
+ return v3, v4
+}
+
+; function %unchanged_region_join(i64, i64, i32) -> i32, i32 fast {
+; region0 = 0 "read"
+; region1 = 1 "write"
+; sig0 = () fast
+; fn0 = %callee sig0
+;
+; block0(v0: i64, v1: i64, v2: i32):
+; brif v2, block1, block2
+;
+; block1:
+; call fn0()
+; jump block3
+;
+; block2:
+; call fn0()
+; jump block3
+;
+; block3:
+; v3 = load.i32 notrap region0 v0
+; brif.i32 v2, block4, block5
+;
+; block4:
+; store.i32 notrap region1 v2, v1
+; jump block6
+;
+; block5:
+; jump block6
+;
+; block6:
+; return v3, v3
+; }