A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Ensure that `LastStores` is a proper lattice (#14230)
Nick Fitzgeraldb5b8f49a0486
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rsindex f07ab32162..cc45328f7c 100644--- a/cranelift/codegen/src/alias_analysis.rs+++ b/cranelift/codegen/src/alias_analysis.rs@@ -87,7 +87,8 @@ use crate::{trace,};use core::cmp::Ordering;-use cranelift_entity::{EntityRef, SecondaryMap, packed_option::PackedOption};+use cranelift_entity::{EntityRef, SecondaryMap};+use smallvec::SmallVec;/// Determine whether this opcode behaves as a memory fence, i.e.,/// prohibits any moving of memory accesses across it.@@ -187,8 +188,85 @@ impl Observer {}}-/// For a given program point, the last-store instruction for each disjoint-/// category of abstract state.+/// Mark the store, if any, named by the given slot value as observed.+fn observe(+ func: &Function,+ observed_stores: &mut FxHashMap<Inst, Observer>,+ slot: LastStore,+ observer: Inst,+) {+ let Some(last_store) = slot.inst() else {+ return;+ };++ // NB: the instruction is not necessarily a store: slots also hold calls,+ // fences, and atomics. Only actual stores can be DSE candidates, so don't+ // bother recording anything else as observed.+ if func.dfg.insts[last_store].opcode().can_store() {+ let entry = observed_stores+ .entry(last_store)+ .or_insert(Observer::One(observer));+ *entry = Observer::meet(*entry, Observer::One(observer));+ trace!(" observed_stores[{last_store:?}] = {entry:?}");+ }+}++/// Which instruction, if any, most recently wrote to a region.+///+/// This is a simple meet semi-lattice:+///+/// ```ignore+/// NoStore Inst(i)+/// \ /+/// \ /+/// Unknown+/// ```+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]+enum LastStore {+ /// Nothing has written to this region since the last fence (if any).+ ///+ /// This is _not_ the same as `Unknown`! This is *positive* information that+ /// nothing has been written since the last fence; in contrast, `Unknown` is+ /// the *absence* of information. It is sound to fall back to the last fence+ /// from `NoStore`; it is unsound to do that from `Unknown`.+ #[default]+ NoStore,++ /// `Inst` was the last store (or fence-like) instruction to write to this+ /// region.+ Inst(Inst),++ /// We know nothing about the contents of this memory region: the paths+ /// flowing into this program point disagreed.+ ///+ /// This is the semi-lattice's bottom.+ Unknown,+}++impl From<Inst> for LastStore {+ #[inline]+ fn from(inst: Inst) -> Self {+ Self::Inst(inst)+ }+}++impl LastStore {+ /// Meet two last-store values.+ fn meet(a: Self, b: Self) -> Self {+ if a == b { a } else { LastStore::Unknown }+ }++ /// The last instruction to write to this slot's associated region, if+ /// known.+ fn inst(self) -> Option<Inst> {+ match self {+ LastStore::Inst(i) => Some(i),+ LastStore::NoStore | LastStore::Unknown => None,+ }+ }+}++/// For a given program point, the last store to each disjoint memory region.////// ### Instructions In `LastStores` Might Not Be In The Function's `Layout`///@@ -205,151 +283,194 @@ impl Observer {/// code than simply checking whether an instruction is in the layout at a/// couple sites.#[derive(Clone, Debug, Default, PartialEq, Eq)]-pub struct LastStores {- /// Last store to each named alias region.- regions: SecondaryMap<AliasRegion, PackedOption<Inst>>,+struct LastStores {+ /// The last store to each named alias region.+ regions: SecondaryMap<AliasRegion, LastStore>,- /// Last instruction with fence semantics. This applies to ALL regions,- /// including ones not yet in the `regions` map.+ /// The store created by the last instruction with fence semantics.+ ///+ /// This applies to ALL regions, including ones not yet in the `regions`+ /// map.////// This is also the last store for memory accesses that have no alias- /// region: such a store may alias any region, and so is treated as a- /// fence, which means the two are always the same instruction.- last_fence: PackedOption<Inst>,+ /// region: such a store may alias any region, and so is treated as a fence.+ last_fence: LastStore,}-/// Mark the store, if any, in the given last-store slot as observed.-fn observe(- func: &Function,- observed_stores: &mut FxHashMap<Inst, Observer>,- last_store: PackedOption<Inst>,- observer: Inst,-) {- if let Some(last_store) = last_store.expand() {- // NB: last-store slots do not always hold stores; they can also hold- // calls, fences, and the markers that `LastStores::meet_from` inserts- // where two control-flow paths disagree. Only actual stores can be DSE- // candidates, so don't bother recording other instructions as observed.- if func.dfg.insts[last_store].opcode().can_store() {- let entry = observed_stores- .entry(last_store)- .or_insert(Observer::One(observer));- *entry = Observer::meet(*entry, Observer::One(observer));- trace!(" observed_stores[{last_store:?}] = {entry:?}");- }+/// What effect does an instruction have on the last-store state?+///+/// This factors out the shared logic between `LastStores::update` and+/// `LastStores::observe_inst` so they can't drift apart from each other.+enum InstEffect {+ /// A memory fence: it observes every region and then clobbers all of them.+ Fence,++ /// An explicitly-trapping instruction: it observes every region, but+ /// clobbers none of them.+ Trap,++ /// A store to a particular alias region.+ Store {+ /// The alias region this store writes.+ region: AliasRegion,+ /// Whether this store can trap.+ can_trap: bool,+ },++ /// An instruction that clobbers nothing and observes the given regions.+ Observes(AliasRegionsObserved),+}++/// Determine what effect the given instruction has on the last-store state.+fn classify(func: &Function, inst: Inst) -> InstEffect {+ let opcode = func.dfg.insts[inst].opcode();++ if has_memory_fence_semantics(opcode) {+ return InstEffect::Fence;+ }++ // Explicitly trapping instructions (`trap`, `trapz`, `udiv`,+ // `uadd_overflow_trap`, etc... but not loads/stores that can implicitly+ // trap): allow store-to-load forwarding across these instructions, but do+ // not eliminate dead stores across them, as that would change the state of+ // memory on trap. We do this by marking every last-store as observed, but+ // not clearing our last-store information.+ if opcode.can_trap() {+ return InstEffect::Trap;}++ // Store instructions: update the last-store information for this+ // instruction's alias region, or, if it has no alias region, treat it as a+ // fence.+ if opcode.can_store() {+ // A store with no memflags, and therefore no alias region, may alias+ // any region, so treat it like a fence. Ditto for a store whose+ // memflags name no alias region.+ let Some(memflags) = func.dfg.insts[inst].memflags() else {+ return InstEffect::Fence;+ };+ let Some(region) = func.dfg.mem_flags[memflags].alias_region() else {+ return InstEffect::Fence;+ };++ return InstEffect::Store {+ region,+ can_trap: func.dfg.mem_flags[memflags].trap_code().is_some(),+ };+ }++ // Everything else: determine which, if any, alias regions this instruction+ // observes.+ InstEffect::Observes(alias_regions_observed(func, inst, opcode))}impl LastStores {- pub(crate) fn update(- &mut self,- func: &Function,- inst: Inst,- observed_stores: &mut FxHashMap<Inst, Observer>,- ) {- let opcode = func.dfg.insts[inst].opcode();+ /// Advance this state across the given instruction.+ fn update(&mut self, func: &Function, inst: Inst) {+ match classify(func, inst) {+ InstEffect::Fence => {+ self.regions.clear();++ // NB: unlike every region slot, `self.last_fence` is *not*+ // observed by a fence; see `observe_inst`.+ self.last_fence = inst.into();+ }- if has_memory_fence_semantics(opcode) {- self.fence(func, inst, observed_stores);- }- // Explicitly trapping instructions (`trap`, `trapz`, `udiv`,- // `uadd_overflow_trap`, etc... but not loads/stores that can implicitly- // trap): allow store-to-load forwarding across these instructions, but- // do not eliminate dead stores across them, as that would change the- // state of memory on trap. We do this by marking every last-store as- // observed, but not clearing our last-store information.- else if opcode.can_trap() {- self.observe_others(func, observed_stores, None, inst);- }- // Store instructions: update the last-store information for this- // instruction's alias region, or, if it has no alias region, treat it- // as a fence.- else if opcode.can_store() {- if let Some(memflags) = func.dfg.insts[inst].memflags() {- match func.dfg.mem_flags[memflags].alias_region() {- Some(region) => {- observe(func, observed_stores, self.regions[region], inst);- self.regions[region] = inst.into();-- // If this store can trap, then we need to observe- // all other alias regions, to ensure that their state- // is preserved in the case that this store traps- // (similar to the `can_trap()` handling above).- //- // This prevents removing the first store in the- // following snippet, for example:- //- // store notrap region0 v0, v3+8- // store user42 region1 v1, v4+16- // store notrap region0 v2, v3+8- //- // ==/==>- //- // store user42 region1 v1, v4+16- // store notrap region0 v2, v3+8- //- // Removing it would be invalid because it drops a- // memory store to `v3+8` that would otherwise have been- // performed when writing to `v4+16` traps.- //- // On the other hand, if it cannot trap, then we need to- // observe all the regions whose last-store *can* trap- // so that we don't allow a non-trapping store to- // effectively be moved ahead of a trapping store:- //- // store user42 region0 v0, v3+8- // store notrap region1 v1, v4+16- // store user42 region0 v2, v3+8- //- // ==/==>- //- // store notrap region1 v1, v4+16- // store user42 region0 v2, v3+8- //- // In this case, removing the first store would mean- // that when writing to `v3+8` traps, we would- // incorrectly store to `v4+16`, when we otherwise- // wouldn't have.- if func.dfg.mem_flags[memflags].trap_code().is_some() {- self.observe_others(func, observed_stores, Some(region), inst);- } else {- self.observe_trapping_others(func, observed_stores, region, inst);- }- }- None => {- // A store with no alias region may alias any region, so- // treat it like a fence.- self.fence(func, inst, observed_stores);- }- }- } else {- // Store with no memflags (and therefore no region):- // treat it like a fence.- self.fence(func, inst, observed_stores);+ // Trapping instructions and pure observers leave every slot alone.+ InstEffect::Trap | InstEffect::Observes(_) => {}++ InstEffect::Store {+ region,+ can_trap: _,+ } => {+ self.regions[region] = inst.into();}}- // Everything else: determine which, if any, alias regions this- // instruction observes.- else {- match alias_regions_observed(func, inst, opcode) {- AliasRegionsObserved::All => self.observe_others(func, observed_stores, None, inst),- AliasRegionsObserved::Just(region) => {- observe(- func,- observed_stores,- self.last_store_for_region(region),- inst,- );- // NB: Because stores without regions may alias any other- // region, we have also observed the last such store, which- // `self.last_fence` tracks.- observe(func, observed_stores, self.last_fence, inst);+ }++ /// Record every store that the given instruction can observe into+ /// `observed`.+ ///+ /// This reads the state as of just before the instruction, so it must be+ /// called before `update` for the given instruction.+ fn observe_inst(&self, func: &Function, inst: Inst, observed: &mut FxHashMap<Inst, Observer>) {+ match classify(func, inst) {+ InstEffect::Fence => {+ // A fence can observe every region, so every store we are+ // currently tracking for a region becomes observed.+ for (_region, slot) in self.regions.iter() {+ observe(func, observed, *slot, inst);}- AliasRegionsObserved::Other => {- observe(func, observed_stores, self.last_fence, inst)++ // NB: `self.last_fence` is *not* observed here. Marking it+ // observed would, for example, prevent eliminating the first of+ // two adjacent stores that have no alias region.+ }++ InstEffect::Trap => self.observe_others(func, observed, None, inst),++ InstEffect::Store { region, can_trap } => {+ observe(func, observed, self.regions[region], inst);++ // If this store can trap, then we need to observe all other+ // alias regions, to ensure that their state is preserved in the+ // case that this store traps (similar to the `InstEffect::Trap`+ // handling above).+ //+ // This prevents removing the first store in the following+ // snippet, for example:+ //+ // store notrap region0 v0, v3+8+ // store user42 region1 v1, v4+16+ // store notrap region0 v2, v3+8+ //+ // ==/==>+ //+ // store user42 region1 v1, v4+16+ // store notrap region0 v2, v3+8+ //+ // Removing it would be invalid because it drops a memory store+ // to `v3+8` that would otherwise have been performed when+ // writing to `v4+16` traps.+ //+ // On the other hand, if it cannot trap, then we need to observe+ // all the regions whose last-store *can* trap so that we don't+ // allow a non-trapping store to effectively be moved ahead of a+ // trapping store:+ //+ // store user42 region0 v0, v3+8+ // store notrap region1 v1, v4+16+ // store user42 region0 v2, v3+8+ //+ // ==/==>+ //+ // store notrap region1 v1, v4+16+ // store user42 region0 v2, v3+8+ //+ // In this case, removing the first store would mean that when+ // writing to `v3+8` traps, we would incorrectly store to+ // `v4+16`, when we otherwise wouldn't have.+ if can_trap {+ self.observe_others(func, observed, Some(region), inst);+ } else {+ self.observe_trapping_others(func, observed, region, inst);}- AliasRegionsObserved::None => {}}++ InstEffect::Observes(AliasRegionsObserved::All) => {+ self.observe_others(func, observed, None, inst)+ }+ InstEffect::Observes(AliasRegionsObserved::Just(region)) => {+ observe(func, observed, self.last_store_for_region(region), inst);+ // NB: Because stores without regions may alias any other+ // region, we have also observed the last such store, which+ // `self.last_fence` tracks.+ observe(func, observed, self.last_fence, inst);+ }+ InstEffect::Observes(AliasRegionsObserved::Other) => {+ observe(func, observed, self.last_fence, inst)+ }+ InstEffect::Observes(AliasRegionsObserved::None) => {}}}@@ -362,9 +483,9 @@ impl LastStores {excluding: Option<AliasRegion>,observer: Inst,) {- for (region, last_store) in self.regions.iter() {+ for (region, slot) in self.regions.iter() {if excluding.is_none_or(|r| r != region) {- observe(func, observed_stores, *last_store, observer);+ observe(func, observed_stores, *slot, observer);}}observe(func, observed_stores, self.last_fence, observer);@@ -379,15 +500,14 @@ impl LastStores {excluding: AliasRegion,observer: Inst,) {- let can_trap = |last_store: PackedOption<Inst>| {- last_store- .expand()+ let can_trap = |slot: LastStore| {+ slot.inst().is_some_and(|s| func.dfg.insts[s].memflags_trap_code(&func.dfg).is_some())};- for (region, last_store) in self.regions.iter() {- if region != excluding && can_trap(*last_store) {- observe(func, observed_stores, *last_store, observer);+ for (region, slot) in self.regions.iter() {+ if region != excluding && can_trap(*slot) {+ observe(func, observed_stores, *slot, observer);}}@@ -396,33 +516,13 @@ impl LastStores {}}- /// Handle memory fence-like instructions by clearing all analysis data.- fn fence(- &mut self,- func: &Function,- inst: Inst,- observed_stores: &mut FxHashMap<Inst, Observer>,- ) {- // A fence can observe every region, so every store we are currently- // tracking for a region becomes observed.- for (_region, last_store) in self.regions.iter() {- observe(func, observed_stores, *last_store, inst);- }- self.regions.clear();-- // NB: `self.last_fence` is *not* observed here. See the comment in- // `LastStores::update`. Marking it observed would, for example, prevent- // eliminating the first of two adjacent stores that have no alias- // region.- self.last_fence = inst.into();- }-- /// Get the last store affecting the given alias region.- fn last_store_for_region(&self, region: AliasRegion) -> PackedOption<Inst> {- if self.regions[region].is_some() {- self.regions[region]- } else {- self.last_fence+ /// Get the last store to the given alias region.+ fn last_store_for_region(&self, region: AliasRegion) -> LastStore {+ match self.regions[region] {+ // Nothing has written this region since the last fence, so the+ // fence is what last (may have) written it.+ LastStore::NoStore => self.last_fence,+ slot => slot,}}@@ -430,26 +530,26 @@ impl LastStores {/// back to the last fence.////// Returns `None` when `inst` has no alias region.- fn raw_region_slot(&self, func: &Function, inst: Inst) -> Option<PackedOption<Inst>> {+ fn raw_region_slot(&self, func: &Function, inst: Inst) -> Option<LastStore> {let region = func.dfg.insts[inst].alias_region(&func.dfg)?;Some(self.regions[region])}- /// Roll this state back to the memory version from just before `dead`,- /// which is a store being removed from the function by dead-store+ /// Roll this state back to the last-store information from just before+ /// `dead`, which is a store being removed from the function by dead-store/// elimination.////// `prev_region_slot` must be what `dead`'s own alias-region slot held- /// immediately before `dead` overwrote it, as recorded by `region_slot`- /// when `dead` itself was processed (that is, it must not be the last-fence- /// fallback).+ /// immediately before `dead` overwrote it, as recorded by+ /// `raw_region_slot` when `dead` itself was processed (that is, it must not+ /// be the last-fence fallback).////// Only `dead`'s own alias-region slot is restored. A store with no alias/// region is treated as a fence by `update`, which clears *every* region/// slot, and we do not undo that; in that case, we leave this state/// alone. Similarly, stores marked observed while processing `dead` stay/// observed.- fn undo_store(&mut self, func: &Function, dead: Inst, prev_region_slot: PackedOption<Inst>) {+ fn undo_store(&mut self, func: &Function, dead: Inst, prev_region_slot: LastStore) {debug_assert!(func.dfg.insts[dead].opcode().can_store());let Some(region) = func.dfg.insts[dead].alias_region(&func.dfg) else {@@ -458,14 +558,13 @@ impl LastStores {// Only roll back if `dead` really is the current last store to its// region.- if self.regions[region].expand() == Some(dead) {+ if self.regions[region] == dead.into() {self.regions[region] = prev_region_slot;}}- /// Get the last-store instruction for the given `inst`'s alias region, if- /// any.- fn get_last_store(&self, func: &Function, inst: Inst) -> PackedOption<Inst> {+ /// Get the last store to the given `inst`'s alias region.+ fn get_last_store(&self, func: &Function, inst: Inst) -> LastStore {if let Some(memflags) = func.dfg.insts[inst].memflags() {return match func.dfg.mem_flags[memflags].alias_region() {None => self.last_fence,@@ -477,20 +576,14 @@ impl LastStores {if opcode.can_load() || opcode.can_store() {inst.into()} else {- None.into()+ LastStore::NoStore}}/// Meet `self` with `rhs`, placing the result in `self`.////// Returns `true` if `self` changed, `false` otherwise.- fn meet_from(- &mut self,- func: &Function,- rhs: &LastStores,- loc: Inst,- observed_stores: &mut FxHashMap<Inst, Observer>,- ) -> bool {+ fn meet_from(&mut self, rhs: &LastStores) -> bool {// NB: Destructure to make sure we don't accidentally forget a// field.let LastStores {@@ -498,61 +591,102 @@ impl LastStores {last_fence,} = self;- let meet = |observed_stores: &mut FxHashMap<Inst, Observer>,- a: &mut PackedOption<Inst>,- b: PackedOption<Inst>|- -> bool {- let old = a.expand();- let new = match (old, b.expand()) {- (None, None) => None,- (Some(a), Some(b)) if a == b => Some(a),- (x, y) => {- // The incoming paths disagree on the last store. Anything- // after the merge that observes this slot observes `loc`,- // not `x` or `y`, and, therefore, we must conservatively- // mark them both observed here. This keeps the- // observed-stores set sound in the presence of loops and- // control-flow join points.- observe(func, observed_stores, x.filter(|x| *x != loc).into(), loc);- observe(func, observed_stores, y.filter(|y| *y != loc).into(), loc);- Some(loc)- }- };- *a = new.into();- old != new- };-let mut changed = false;+ let mut meet = |a: &mut LastStore, b: LastStore| {+ let new = LastStore::meet(*a, b);+ changed |= new != *a;+ *a = new;+ };+let max_len = core::cmp::max(regions.keys().len(), rhs.regions.keys().len());for i in 0..max_len {let region = AliasRegion::new(i);- changed |= meet(observed_stores, &mut regions[region], rhs.regions[region]);+ meet(&mut regions[region], rhs.regions[region]);}- changed |= meet(observed_stores, last_fence, rhs.last_fence);+ meet(last_fence, rhs.last_fence);changed}}+/// The alias-analysis state as a walk over a single block advances it.+#[derive(Clone, Debug)]+pub struct MemoryState {+ /// The last store to each slot, as of the current program point.+ stores: LastStores,++ /// The token identifying this program point's extent.+ ///+ /// See `MemoryLoc::extent` for details.+ extent: u32,++ /// The current block we are walking.+ #[cfg(debug_assertions)]+ current_block: Block,+}++const NULL_EXTENT: u32 = 0;++impl MemoryState {+ /// Get the extent token to key a `MemoryLoc` on, given the last store to+ /// that location's alias region.+ fn extent_for(&self, last_store: LastStore) -> u32 {+ match last_store {+ // Each of these already names a version of memory all on its own,+ // function wide, and so doesn't need an additional extent token.+ LastStore::NoStore | LastStore::Inst(_) => NULL_EXTENT,++ // This one, on the other hand, only names a version of memory+ // relative to the control-flow join that lost track of it.+ LastStore::Unknown => {+ debug_assert_ne!(self.extent, NULL_EXTENT);+ self.extent+ }+ }+ }+}+/// A key identifying a unique memory location.///-/// For the result of a load to be equivalent to the result of another-/// load, or the store data from a store, we need for (i) the-/// "version" of memory (here ensured by having the same last store-/// instruction to touch the disjoint category of abstract state we're-/// accessing); (ii) the address must be the same (here ensured by-/// having the same SSA value, which doesn't change after computed);-/// (iii) the offset must be the same; (iv) the accessed type and-/// extension mode (e.g., 8-to-32, signed) must be the same; and (v)-/// the byte order of the two accesses must be the same.+/// For the result of a load to be equivalent to the result of another load, or+/// the store data from a store, we need all of these fields to match.#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]struct MemoryLoc {- last_store: PackedOption<Inst>,+ /// The last store to this location's alias region.+ last_store: LastStore,++ /// A token identifying the extent of our `LastStore`, since+ /// `LastStore::Unknown` is only valid relative to its last control-flow+ /// join point.+ ///+ /// An extent is a control-flow join point and everything reachable from it+ /// through single-predecessor edges. For `LastStore::Unknown`, this+ /// identifies the area where the bytes in the associated alias region are+ /// identical to when control entered the join point, and therefore exactly+ /// the area within which two `LastStore::Unknown`-keyed memory locations+ /// name the same value.+ ///+ /// This is necessary because on one hand, we need `LastStore::Unknown` to+ /// be a single bottom value so that `LastStore` and `LastStores` are proper+ /// lattices, but on the other hand we need identity to differentiate+ /// between instances of `LastStore::Unknown` to prevent, e.g., unsoundly+ /// store-to-load forwarding from a store in one control-flow join point to+ /// a load at a different control-flow join point.+ ///+ /// Extent is computed incrementally in `AliasAnalysis::push_scope`.+ extent: u32,++ /// The dynamic part of this memory location's address.address: Value,++ /// The static part of this memory location's address.offset: Offset32,++ /// The type being accessed at this memory location.ty: Type,+/// We keep the *opcode* of the instruction that produced the/// value we record at this key if the opcode is anything other/// than an ordinary load or store. This is needed when we@@ -565,6 +699,7 @@ struct MemoryLoc {/// in place of extending loads when we know the memory value, but/// we haven't yet done this.extending_opcode: Option<Opcode>,+/// The byte order of this access, as explicitly specified in its memory/// flags, or `None` when the access uses the target's native byte order.///@@ -596,15 +731,14 @@ struct KnownValue {/// When this entry was created by a store to a particular alias region,/// whatever that region's last-store slot held just *before* `def_inst`- /// overwrote it, as given by `LastStores::region_slot`.+ /// overwrote it, as given by `LastStores::raw_region_slot`.////// `None` means either the entry was created by a load or by a store with/// no alias region. Neither will ever undo `LastStores` state.///- /// `Some(maybe_inst)` contains the alias region slot's previous value, so- /// that it can be restored if `def_inst` is a dead store that gets- /// eliminated.- prev_region_slot: Option<PackedOption<Inst>>,+ /// `Some(slot)` contains the alias region slot's previous value, so that it+ /// can be restored if `def_inst` is a dead store that gets eliminated.+ prev_region_slot: Option<LastStore>,}/// The result of processing an instruction through alias analysis.@@ -653,9 +787,26 @@ pub struct AliasAnalysis<'a> {/// Input state to a basic block.block_input: FxHashMap<Block, LastStores>,- /// Known memory-value equivalences. This is the result of the- /// analysis. This is a mapping from a memory location to its known value.+ /// Known memory-value equivalences.+ ///+ /// This is the result of the analysis: a mapping from a memory location to+ /// its known value.mem_values: FxHashMap<MemoryLoc, KnownValue>,++ /// The extent token for each scope currently on the dominator-tree path+ /// we are walking.+ ///+ /// See `MemoryLoc::extent` for details.+ extents: SmallVec<[u32; 8]>,++ /// Counter for minting fresh extent tokens.+ next_extent: u32,++ /// The blocks whose scopes are currently on the stack, so that we can+ /// assert that `push_scope` and `pop_scope` really are called in+ /// dominator-tree pre-order.+ #[cfg(debug_assertions)]+ scope_blocks: SmallVec<[Block; 8]>,}impl<'a> AliasAnalysis<'a> {@@ -663,15 +814,23 @@ impl<'a> AliasAnalysis<'a> {pub fn new(func: &Function, domtree: &'a DominatorTree) -> AliasAnalysis<'a> {trace!("alias analysis input is:\n{func:?}");assert!(domtree.is_valid());+let mut analysis = AliasAnalysis {domtree,post_dom_tree: None,observed_stores: FxHashMap::default(),block_input: FxHashMap::default(),mem_values: FxHashMap::default(),+ extents: SmallVec::new(),+ next_extent: 0,+ #[cfg(debug_assertions)]+ scope_blocks: SmallVec::new(),};analysis.compute_block_input_states(func);+ analysis.verify_fixpoint(func);+ analysis.compute_observed_stores(func);+analysis}@@ -710,6 +869,7 @@ impl<'a> AliasAnalysis<'a> {fn compute_block_input_states(&mut self, func: &Function) {let mut queue = vec![];let mut queue_set = FxHashSet::default();+let entry = func.layout.entry_block().unwrap();queue.push(entry);queue_set.insert(entry);@@ -727,19 +887,13 @@ impl<'a> AliasAnalysis<'a> {for inst in func.layout.block_insts(block) {trace!(" analyzing {inst:?}: {}", func.dfg.display_inst(inst));- state.update(func, inst, &mut self.observed_stores);+ state.update(func, inst);trace!(" updated state = {state:?}");}visit_block_succs(func, block, |_inst, succ, _from_table| {- let succ_first_inst = func.layout.block_insts(succ).next().unwrap();let updated = match self.block_input.get_mut(&succ) {- Some(succ_state) => succ_state.meet_from(- func,- &state,- succ_first_inst,- &mut self.observed_stores,- ),+ Some(succ_state) => succ_state.meet_from(&state),None => {self.block_input.insert(succ, state.clone());true@@ -751,34 +905,212 @@ impl<'a> AliasAnalysis<'a> {}});}+ }++ /// Assert that `self.block_input` really is a fixed point.+ ///+ /// This should help catch regressions that don't result in miscompiles,+ /// only worse codegen and/or order-dependent results, both of which are+ /// otherwise hard to test against in filetests.+ fn verify_fixpoint(&self, func: &Function) {+ if !cfg!(debug_assertions) {+ return;+ }++ let entry = func.layout.entry_block().unwrap();++ let mut expected = FxHashMap::default();+ expected.insert(entry, LastStores::default());++ // Note that iterating through this hash map gives us ~nondeterministic+ // block ordering -- but that shouldn't matter if we _actually_ computed+ // an order-independent result! This is a convenient little foil to our+ // worklist's deterministic ordering.+ for (&block, input) in &self.block_input {+ // Compute the output state for this block.+ let mut state = input.clone();+ for inst in func.layout.block_insts(block) {+ state.update(func, inst);+ }++ // Meet that output state with the input state of each of this+ // block's successors.+ visit_block_succs(func, block, |_inst, succ, _from_table| {+ match expected.get_mut(&succ) {+ Some(succ_state) => {+ succ_state.meet_from(&state);+ }+ None => {+ expected.insert(succ, state.clone());+ }+ }+ });+ }++ // Our expected results should match our actual results.+ for (&block, input) in &self.block_input {+ assert_eq!(+ expected.get(&block),+ Some(input),+ "last-store analysis did not reach a fixpoint at {block}",+ );+ }+ assert_eq!(+ expected.len(),+ self.block_input.len(),+ "last-store analysis and its verifier disagree on which blocks are reachable",+ );+ }++ /// The single reachable predecessor of `block`, if it has exactly one.+ ///+ /// Returns `None` both when `block` has two or more distinct reachable+ /// predecessors and when it has none at all.+ fn single_reachable_pred(&self, cfg: &ControlFlowGraph, block: Block) -> Option<Block> {+ let mut preds = cfg+ .pred_iter(block)+ .map(|pred| pred.block)+ // Skip unreachable predecessors. Our fixpoint never propagates+ // state out of them.+ .filter(|pred| self.block_input.contains_key(pred));+ let first = preds.next()?;+ preds.all(|pred| pred == first).then_some(first)+ }++ /// Compute the set of stores that some instruction in this function can+ /// observe.+ ///+ /// This is deliberately not folded into the last-store fixpoint. While that+ /// fixpoint iterates, a slot can transiently name a store that the final+ /// solution does not, and observing it would make the resulting+ /// observed-stores set depend on the order in which the worklist visited+ /// blocks. Instead we walk the function once, in layout order, over the+ /// solved fixpoint.+ fn compute_observed_stores(&mut self, func: &Function) {+ let mut observed_stores = FxHashMap::default();++ for block in func.layout.blocks() {+ // Ignore unreachable blocks.+ if !self.block_input.contains_key(&block) {+ continue;+ }++ // Compute this block's final state.+ let mut state = self.block_input_stores(block);+ for inst in func.layout.block_insts(block) {+ state.observe_inst(func, inst, &mut observed_stores);+ state.update(func, inst);+ }++ // When a predecessor and successor disagree on the last store to a+ // region, we need to mark the predecessor's last store as observed,+ // so that dead-store elimination cannot remove it.+ visit_block_succs(func, block, |_inst, succ, _from_table| {+ let succ_input = self+ .block_input+ .get(&succ)+ .expect("successors of a reachable block are reachable");++ let observer = func.layout.block_insts(succ).next().unwrap();++ let max_len =+ core::cmp::max(state.regions.keys().len(), succ_input.regions.keys().len());+ for i in 0..max_len {+ let region = AliasRegion::new(i);+ if succ_input.regions[region] == LastStore::Unknown {+ observe(func, &mut observed_stores, state.regions[region], observer);+ }+ }++ if succ_input.last_fence == LastStore::Unknown {+ observe(func, &mut observed_stores, state.last_fence, observer);+ }+ });+ }++ trace!("final observed_stores = {observed_stores:#?}");+ self.observed_stores = observed_stores;+ }++ /// Get the solved last-store state on entry to the given block.+ fn block_input_stores(&self, block: Block) -> LastStores {+ match self.block_input.get(&block) {+ Some(input) => input.clone(),+ None => LastStores::default(),+ }+ }++ /// Enter the given block's scope, getting its initial memory state.+ ///+ /// Callers must walk the dominator tree in pre-order, entering each block's+ /// scope on the way down and calling `pop_scope` before moving on to any+ /// block that this one does not dominate.+ pub fn push_scope(&mut self, cfg: &ControlFlowGraph, block: Block) -> MemoryState {+ debug_assert!(cfg.is_valid());++ #[cfg(debug_assertions)]+ {+ debug_assert_eq!(+ self.domtree.idom(block),+ self.scope_blocks.last().copied(),+ "`push_scope` must be called in dominator-tree pre-order",+ );+ self.scope_blocks.push(block);+ }++ let extent = match self.extents.last() {+ // A block with exactly one reachable predecessor sees exactly the+ // memory that its predecessor left behind, and so it continues its+ // predecessor's extent.+ Some(parent) if self.single_reachable_pred(cfg, block).is_some() => *parent,++ // Anything else (e.g. a control-flow join or the entry block)+ // begins a new extent.+ _ => {+ self.next_extent += 1;+ self.next_extent+ }+ };+ self.extents.push(extent);- trace!("final observed_stores = {:#?}", self.observed_stores);+ MemoryState {+ stores: self.block_input_stores(block),+ extent,+ #[cfg(debug_assertions)]+ current_block: block,+ }}- /// Get the starting state for a block.- pub fn block_starting_state(&self, block: Block) -> LastStores {- self.block_input- .get(&block)- .cloned()- .unwrap_or_else(|| LastStores::default())+ /// Leave the scope of the block most recently passed to `push_scope`.+ pub fn pop_scope(&mut self) {+ #[cfg(debug_assertions)]+ self.scope_blocks+ .pop()+ .expect("`pop_scope` without a matching `push_scope`");++ self.extents+ .pop()+ .expect("`pop_scope` without a matching `push_scope`");}- /// Process one instruction. Meant to be invoked in program order- /// within a block, and ideally in RPO or at least some domtree- /// preorder for maximal reuse.+ /// Process one instruction.+ ///+ /// Must be invoked in program order within a block.pub fn process_inst(&mut self,func: &mut Function,cfg: &ControlFlowGraph,- state: &mut LastStores,+ state: &mut MemoryState,inst: Inst,) -> OptResult {trace!(- "process_inst: {inst}: {}\n\twith last stores: {state:?}\n\twith mem values = {:?}",+ "process_inst: {inst}: {}\n\twith last stores: {state:?}",func.dfg.display_inst(inst),- self.mem_values,);+ #[cfg(debug_assertions)]+ debug_assert_eq!(func.layout.inst_block(inst), Some(state.current_block));+let result = if let Some((address, offset, ty)) = inst_addr_offset_type(func, inst) {let address = func.dfg.resolve_aliases(address);let opcode = func.dfg.insts[inst].opcode();@@ -787,10 +1119,10 @@ impl<'a> AliasAnalysis<'a> {let store_data = inst_store_data(func, inst).unwrap();let store_data = func.dfg.resolve_aliases(store_data);- let last_store = state.get_last_store(func, inst);+ let last_store = state.stores.get_last_store(func, inst);// Check whether this store makes the last store dead.- if let Some(last_store) = last_store.expand() {+ if let LastStore::Inst(last_store) = last_store {// A store can only be dead when unobserved or only observed// by its overwriter.if self.observed_stores.get(&last_store).is_none_or(|o| *o == Observer::One(inst))@@ -830,8 +1162,10 @@ impl<'a> AliasAnalysis<'a> {// does not require to match this store's byte order.// Everything else in the key does have to match, so we// can take it from this store.+ let dead_slot = LastStore::Inst(last_store);let dead_loc = MemoryLoc {- last_store: last_store.into(),+ last_store: dead_slot,+ extent: state.extent_for(dead_slot),address,offset,ty,@@ -840,9 +1174,9 @@ impl<'a> AliasAnalysis<'a> {};let dead_entry = self.mem_values.remove(&dead_loc);- // Roll our last-store state back to the memory version- // just before the dead store, so that `state` describes- // memory as if the dead store had never happened.+ // Roll our last-store state back to just before the+ // dead store, so that `state` describes memory as if+ // the dead store had never happened.//// Our callers remove the dead store from the layout and// then reprocess this overwriting store. Without the@@ -865,8 +1199,11 @@ impl<'a> AliasAnalysis<'a> {// can come from a precomputed `block_input` snapshot,// for a predecessor block we have not walked yet) or it// has no alias region and therefore no slot of its own.- if let Some(prev) = dead_entry.and_then(|e| e.prev_region_slot) {- state.undo_store(func, last_store, prev);+ if let Some(prev) = dead_entry+ .filter(|e| self.domtree.dominates(e.def_inst, inst, &func.layout))+ .and_then(|e| e.prev_region_slot)+ {+ state.stores.undo_store(func, last_store, prev);}return OptResult::DeadStore {@@ -878,6 +1215,7 @@ impl<'a> AliasAnalysis<'a> {let check_loc = MemoryLoc {last_store,+ extent: state.extent_for(last_store),address,offset,ty,@@ -904,7 +1242,7 @@ impl<'a> AliasAnalysis<'a> {// We are removing this idempotent store in favor of the// original, so if this idempotent store was observed,// then the original must now be observed as well.- if let Some(last_store) = last_store.expand() {+ if let LastStore::Inst(last_store) = last_store {if let Some(observer) = self.observed_stores.get(&inst).copied() {let entry =self.observed_stores.entry(last_store).or_insert(observer);@@ -918,8 +1256,10 @@ impl<'a> AliasAnalysis<'a> {}// Otherwise, update our state to reflect this store.+ let this_store = LastStore::Inst(inst);let mem_loc = MemoryLoc {- last_store: inst.into(),+ last_store: this_store,+ extent: state.extent_for(this_store),address,offset,ty,@@ -936,16 +1276,17 @@ impl<'a> AliasAnalysis<'a> {// last-fence fallback, because we don't want to move an// instruction without a region into a region slot on// DSE rollback.- prev_region_slot: state.raw_region_slot(func, inst),+ prev_region_slot: state.stores.raw_region_slot(func, inst),},);OptResult::None} else if opcode.can_load() {- let last_store = state.get_last_store(func, inst);+ let last_store = state.stores.get_last_store(func, inst);let load_result = func.dfg.inst_results(inst)[0];let mem_loc = MemoryLoc {last_store,+ extent: state.extent_for(last_store),address,offset,ty,@@ -991,7 +1332,7 @@ impl<'a> AliasAnalysis<'a> {def_inst: inst,value: load_result,// A load does not advance the memory version, so- // there is no previous version to roll back to.+ // there is no previous slot value to roll back to.prev_region_slot: None,},);@@ -1014,14 +1355,23 @@ impl<'a> AliasAnalysis<'a> {OptResult::None};- let observed_stores_len = self.observed_stores.len();- state.update(func, inst, &mut self.observed_stores);- debug_assert_eq!(- observed_stores_len,- self.observed_stores.len(),- "`compute_block_input_states` should have already found all observed stores, \- but processing {inst} found a new one",- );+ // `compute_observed_stores` walks the whole function up front, so+ // processing an instruction here must never turn up a store that it did+ // not already find.+ #[cfg(debug_assertions)]+ {+ let mut new_observations = FxHashMap::default();+ state.stores.observe_inst(func, inst, &mut new_observations);+ for observed in new_observations.keys() {+ debug_assert!(+ self.observed_stores.contains_key(observed),+ "`compute_observed_stores` should have already found all observed \+ stores, but processing {inst} found that {observed} is observed",+ );+ }+ }++ state.stores.update(func, inst);result}@@ -1032,10 +1382,36 @@ impl<'a> AliasAnalysis<'a> {/// (e.g. in cases of double-indirection with two separate chains/// of loads).pub fn compute_and_update_aliases(&mut self, func: &mut Function, cfg: &ControlFlowGraph) {+ let domtree = self.domtree;let mut pos = FuncCursor::new(func);- while let Some(block) = pos.next_block() {- let mut state = self.block_starting_state(block);+ let Some(entry) = pos.func.layout.entry_block() else {+ return;+ };++ // Marks a block we have not walked yet, or the point at which we are+ // done with everything a block dominates and must leave its scope.+ enum BlockStackEntry {+ Visit(Block),+ Pop,+ }++ // Walk the dominator tree in pre-order. Unreachable blocks are not in+ // the dominator tree, and are simply skipped.+ let mut stack = vec![BlockStackEntry::Visit(entry)];+ while let Some(top) = stack.pop() {+ let block = match top {+ BlockStackEntry::Pop => {+ self.pop_scope();+ continue;+ }+ BlockStackEntry::Visit(block) => block,+ };++ stack.push(BlockStackEntry::Pop);+ let mut state = self.push_scope(cfg, block);++ pos.goto_top(block);while let Some(inst) = pos.next_inst() {match self.process_inst(pos.func, cfg, &mut state, inst) {OptResult::None => {}@@ -1063,6 +1439,9 @@ impl<'a> AliasAnalysis<'a> {}}}++ let children = SmallVec::<[Block; 8]>::from_iter(domtree.children(block));+ stack.extend(children.into_iter().rev().map(BlockStackEntry::Visit));}}}diff --git a/cranelift/codegen/src/egraph/mod.rs b/cranelift/codegen/src/egraph/mod.rsindex d4a2165a4b..7b33d8d9e8 100644--- a/cranelift/codegen/src/egraph/mod.rs+++ b/cranelift/codegen/src/egraph/mod.rs@@ -1,7 +1,7 @@//! Support for egraphs represented in the DataFlowGraph.use crate::FxHashSet;-use crate::alias_analysis::{AliasAnalysis, LastStores, OptResult};+use crate::alias_analysis::{AliasAnalysis, MemoryState, OptResult};use crate::branch_to_trap::BranchToTrapAnalysis;use crate::ctxhash::{CtxEq, CtxHash, NullCtx};use crate::cursor::{Cursor, CursorPosition, FuncCursor};@@ -155,7 +155,7 @@ where/// build a post-dominator tree for dead-store elimination.cfg: &'opt ControlFlowGraph,pub(crate) alias_analysis: &'opt mut AliasAnalysis<'analysis>,- pub(crate) alias_analysis_state: &'opt mut LastStores,+ pub(crate) alias_analysis_state: &'opt mut MemoryState,pub(crate) branch_to_trap_analysis: &'opt mut BranchToTrapAnalysis,ctrl_plane: &'opt mut ControlPlane,// Held locally during optimization of one node (recursively):@@ -923,11 +923,14 @@ impl<'a> EgraphPass<'a> {{gvn_map_blocks.pop();gvn_map.decrement_depth();+ self.alias_analysis.pop_scope();}gvn_map.increment_depth();gvn_map_blocks.push(block);+ let mut alias_analysis_state = self.alias_analysis.push_scope(&*self.cfg, block);+// Check that `gvn_map_blocks` is the path from this block up to the// root in the dominator tree.debug_assert_eq!(gvn_map_blocks, {@@ -945,8 +948,6 @@ impl<'a> EgraphPass<'a> {trace!("Processing block {}", block);cursor.set_position(CursorPosition::Before(block));- let mut alias_analysis_state = self.alias_analysis.block_starting_state(block);-for ¶m in cursor.func.dfg.block_params(block) {trace!("creating initial singleton eclass for blockparam {}", param);value_to_opt_value[param] = param;diff --git a/cranelift/filetests/filetests/alias/fence-fallback-across-join.clif b/cranelift/filetests/filetests/alias/fence-fallback-across-join.clifnew file mode 100644index 0000000000..548532ec85--- /dev/null+++ b/cranelift/filetests/filetests/alias/fence-fallback-across-join.clif@@ -0,0 +1,77 @@+test optimize precise-output+set opt_level=speed+target aarch64++function %fence_fallback_survives_join(i64, i32) -> i32, i32 {+ region0 = 0 "R0"++block0(v0: i64, v1: i32):+ v3 = load.i32 notrap aligned region0 v0+ brif v1, block1, block2++block1:+ jump block3++block2:+ jump block3++block3:+ v4 = load.i32 notrap aligned region0 v0+ return v3, v4+}++; function %fence_fallback_survives_join(i64, i32) -> i32, i32 fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32):+; v3 = load.i32 notrap aligned region0 v0+; brif v1, block1, block2+;+; block1:+; jump block3+;+; block2:+; jump block3+;+; block3:+; return v3, v3+; }++function %unknown_does_not_take_the_fence_fallback(i64, i32, i32) -> i32, i32 {+ region0 = 0 "R0"++block0(v0: i64, v1: i32, v2: i32):+ v3 = load.i32 notrap aligned region0 v0+ brif v1, block1, block2++block1:+ ;; This store prevents `block3` from reusing `v3`.+ store notrap aligned region0 v2, v0+ jump block3++block2:+ jump block3++block3:+ v4 = load.i32 notrap aligned region0 v0+ return v3, v4+}++; function %unknown_does_not_take_the_fence_fallback(i64, i32, i32) -> i32, i32 fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32, v2: i32):+; v3 = load.i32 notrap aligned region0 v0+; brif v1, block1, block2+;+; block1:+; store.i32 notrap aligned region0 v2, v0+; jump block3+;+; block2:+; jump block3+;+; block3:+; v4 = load.i32 notrap aligned region0 v0+; return v3, v4+; }diff --git a/cranelift/filetests/filetests/alias/join-order-independence.clif b/cranelift/filetests/filetests/alias/join-order-independence.clifnew file mode 100644index 0000000000..b108069708--- /dev/null+++ b/cranelift/filetests/filetests/alias/join-order-independence.clif@@ -0,0 +1,85 @@+test optimize precise-output+set opt_level=speed+target aarch64++;; The last-store analysis must not depend on the order in which the worklist+;; happens to visit blocks. These two functions differ only in the order of the+;; `brif`'s two arms, so they must optimize identically.++function %arms_in_order(i64, i32, i32) -> i32, i32 {+ region0 = 0 "R0"++block0(v0: i64, v1: i32, v2: i32):+ store notrap aligned region0 v1, v0+ brif v1, block3, block1++block1:+ store notrap aligned region0 v2, v0+ jump block3++block3:+ v3 = load.i32 notrap aligned region0 v0+ jump block4++block4:+ v4 = load.i32 notrap aligned region0 v0+ return v3, v4+}++; function %arms_in_order(i64, i32, i32) -> i32, i32 fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32, v2: i32):+; store notrap aligned region0 v1, v0+; brif v1, block3, block1+;+; block1:+; store.i32 notrap aligned region0 v2, v0+; jump block3+;+; block3:+; v3 = load.i32 notrap aligned region0 v0+; jump block4+;+; block4:+; return v3, v3+; }++function %arms_swapped(i64, i32, i32) -> i32, i32 {+ region0 = 0 "R0"++block0(v0: i64, v1: i32, v2: i32):+ store notrap aligned region0 v1, v0+ brif v1, block1, block3++block1:+ store notrap aligned region0 v2, v0+ jump block3++block3:+ v3 = load.i32 notrap aligned region0 v0+ jump block4++block4:+ v4 = load.i32 notrap aligned region0 v0+ return v3, v4+}++; function %arms_swapped(i64, i32, i32) -> i32, i32 fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32, v2: i32):+; store notrap aligned region0 v1, v0+; brif v1, block1, block3+;+; block1:+; store.i32 notrap aligned region0 v2, v0+; jump block3+;+; block3:+; v3 = load.i32 notrap aligned region0 v0+; jump block4+;+; block4:+; return v3, v3+; }diff --git a/cranelift/filetests/filetests/alias/join-single-predecessor.clif b/cranelift/filetests/filetests/alias/join-single-predecessor.clifnew file mode 100644index 0000000000..7c1278384b--- /dev/null+++ b/cranelift/filetests/filetests/alias/join-single-predecessor.clif@@ -0,0 +1,57 @@+test optimize precise-output+set opt_level=speed+target aarch64++function %single_pred_inherits(i64, i32, i32, i32) -> i32, i32 {+ region0 = 0 "R0"++block0(v0: i64, v1: i32, v2: i32, v3: i32):+ store notrap aligned region0 v1, v0+ jump block1++block1:+ brif v2, block2, block3++block3:+ jump block2++block2:+ v10 = load.i32 notrap aligned region0 v0+ jump block4++block4:+ ;; `block4` has exactly one predecessor, `block2`, so at the fixpoint its+ ;; region0 slot must equal `block2`'s outgoing value, and `v11` must+ ;; therefore be forwarded to `v10`.+ v11 = load.i32 notrap aligned region0 v0+ store notrap aligned region0 v3, v0+ brif v1, block1, block5++block5:+ return v10, v11+}++; function %single_pred_inherits(i64, i32, i32, i32) -> i32, i32 fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32, v2: i32, v3: i32):+; store notrap aligned region0 v1, v0+; jump block1+;+; block1:+; brif.i32 v2, block2, block3+;+; block3:+; jump block2+;+; block2:+; v10 = load.i32 notrap aligned region0 v0+; jump block4+;+; block4:+; store.i32 notrap aligned region0 v3, v0+; brif.i32 v1, block1, block5+;+; block5:+; return v10, v10+; }diff --git a/cranelift/filetests/filetests/alias/merge-token-identity.clif b/cranelift/filetests/filetests/alias/merge-token-identity.clifnew file mode 100644index 0000000000..98893abfcf--- /dev/null+++ b/cranelift/filetests/filetests/alias/merge-token-identity.clif@@ -0,0 +1,63 @@+test optimize precise-output+set opt_level=speed+target aarch64++;; `v3` and `v4` sit after two independent joins, with `block4`'s store in+;; between.+function %two_independent_joins(i64, i32, i32) -> i32, i32 {+ region0 = 0 "R0"++block0(v0: i64, v1: i32, v2: i32):+ brif v1, block1, block2++block1:+ store notrap aligned region0 v2, v0+ jump block3++block2:+ jump block3++block3:+ v3 = load.i32 notrap aligned region0 v0+ brif v1, block4, block5++block4:+ store notrap aligned region0 v1, v0+ jump block6++block5:+ jump block6++block6:+ v4 = load.i32 notrap aligned region0 v0+ return v3, v4+}++; function %two_independent_joins(i64, i32, i32) -> i32, i32 fast {+; region0 = 0 "R0"+;+; block0(v0: i64, v1: i32, v2: i32):+; brif v1, block1, block2+;+; block1:+; store.i32 notrap aligned region0 v2, v0+; jump block3+;+; block2:+; jump block3+;+; block3:+; v3 = load.i32 notrap aligned region0 v0+; brif.i32 v1, block4, block5+;+; block4:+; store.i32 notrap aligned region0 v1, v0+; jump block6+;+; block5:+; jump block6+;+; block6:+; v4 = load.i32 notrap aligned region0 v0+; return v3, v4+; }diff --git a/tests/disas/gc/array-copy-with-fuel.wat b/tests/disas/gc/array-copy-with-fuel.watindex 63ab28b9a6..40bc30174f 100644--- a/tests/disas/gc/array-copy-with-fuel.wat+++ b/tests/disas/gc/array-copy-with-fuel.wat@@ -93,8 +93,6 @@;; @002b brif.i32 v6, block4, block7;;;; block4:-;; v143 = load.i32 notrap aligned region6 v162-;; v145 = load.i32 notrap aligned region7 v163;; @002b v89 = icmp.i64 ult v45, v69;; @002b v94 = iadd.i64 v45, v170;; @002b v95 = iadd.i64 v69, v170