Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift: check for volatile loads late in alias analysis's classify (#15029)

TastyCZ4e9504d5ab5b
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rs
index 2b74efd761..0cbf5475ad 100644
--- a/cranelift/codegen/src/alias_analysis.rs
+++ b/cranelift/codegen/src/alias_analysis.rs
@@ -93,7 +93,8 @@ use crate::{
inst_addr_offset_type, inst_store_data, is_volatile_access, visit_block_succs,
},
ir::{
- AliasRegion, Block, Endianness, Function, Inst, Opcode, Type, Value, immediates::Offset32,
+ AliasRegion, Block, Endianness, Function, Inst, MemFlagsData, Opcode, Type, Value,
+ immediates::Offset32,
},
post_dominator_tree::PostDominatorTree,
trace,
@@ -131,24 +132,29 @@ enum AliasRegionsObserved {
}
/// Which alias region(s) can an instruction observe?
-fn alias_regions_observed(func: &Function, inst: Inst, opcode: Opcode) -> AliasRegionsObserved {
+///
+/// `memflags` must be the instruction's memflags, if it has any.
+fn alias_regions_observed(
+ func: &Function,
+ inst: Inst,
+ opcode: Opcode,
+ memflags: Option<MemFlagsData>,
+) -> AliasRegionsObserved {
debug_assert_eq!(func.dfg.insts[inst].opcode(), opcode);
+ debug_assert_eq!(func.dfg.insts[inst].memflags_data(&func.dfg), memflags);
if opcode.is_return()
|| opcode.is_call()
|| opcode.can_trap()
// NB: the `opcode.can_trap()` check above only covers explicitly
// trapping instructions (like `trap` and `trapz`), not loads/stores
// that can implicitly trap; we check those via their memflags.
- || func.dfg.insts[inst]
- .memflags_data(&func.dfg)
- .and_then(|flags| flags.trap_code())
- .is_some()
+ || memflags.and_then(|flags| flags.trap_code()).is_some()
{
return AliasRegionsObserved::All;
}
if opcode.can_load() {
- if let Some(region) = func.dfg.insts[inst].alias_region(&func.dfg) {
+ if let Some(region) = memflags.and_then(|flags| flags.alias_region()) {
AliasRegionsObserved::Just(region)
} else {
AliasRegionsObserved::Other
@@ -348,14 +354,6 @@ fn classify(func: &Function, inst: Inst) -> InstEffect {
return InstEffect::Fence;
}
- // A volatile access must stay as written, in order with the accesses
- // around it: treat it as a fence, which observes every earlier store (so
- // that none of them is removed as dead across it) and forgets everything
- // known about memory (so that nothing is forwarded across it).
- if is_volatile_access(func, inst) {
- return InstEffect::Fence;
- }
-
// Explicitly trapping instructions (`trap`, `trapz`, `udiv`,
// `uadd_overflow_trap`, etc... but not loads/stores that can implicitly
// trap): allow store-to-load forwarding across these instructions, but do
@@ -366,29 +364,46 @@ fn classify(func: &Function, inst: Inst) -> InstEffect {
return InstEffect::Trap;
}
+ // A volatile access must stay as written, in order with the accesses
+ // around it: treat it as a fence, which observes every earlier store (so
+ // that none of them is removed as dead across it) and forgets everything
+ // known about memory (so that nothing is forwarded across it). Only loads
+ // and stores can be volatile, so rather than looking up every
+ // instruction's memflags up here, we check for it below, where we already
+ // look them up.
+
// Store instructions: update the last-store information for this
// instruction's alias region, or, if it has no alias region, treat it as a
// fence.
if opcode.can_store() {
// A store with no memflags, and therefore no alias region, may alias
// any region, so treat it like a fence. Ditto for a store whose
- // memflags name no alias region.
+ // memflags name no alias region, and for a volatile store.
let Some(memflags) = func.dfg.insts[inst].memflags() else {
return InstEffect::Fence;
};
- let Some(region) = func.dfg.mem_flags[memflags].alias_region() else {
+ let memflags = func.dfg.mem_flags[memflags];
+ if memflags.volatile() {
+ return InstEffect::Fence;
+ }
+ let Some(region) = memflags.alias_region() else {
return InstEffect::Fence;
};
return InstEffect::Store {
region,
- can_trap: func.dfg.mem_flags[memflags].trap_code().is_some(),
+ can_trap: memflags.trap_code().is_some(),
};
}
- // Everything else: determine which, if any, alias regions this instruction
- // observes.
- InstEffect::Observes(alias_regions_observed(func, inst, opcode))
+ // Everything else, loads included. Look up the memflags once: a volatile
+ // load is a fence too, and otherwise they tell us which, if any, alias
+ // regions this instruction observes.
+ let memflags = func.dfg.insts[inst].memflags_data(&func.dfg);
+ if memflags.is_some_and(|flags| flags.volatile()) {
+ return InstEffect::Fence;
+ }
+ InstEffect::Observes(alias_regions_observed(func, inst, opcode, memflags))
}
impl LastStores {
diff --git a/cranelift/filetests/filetests/alias/volatile.clif b/cranelift/filetests/filetests/alias/volatile.clif
index 6ac644f546..3350b997d8 100644
--- a/cranelift/filetests/filetests/alias/volatile.clif
+++ b/cranelift/filetests/filetests/alias/volatile.clif
@@ -192,6 +192,33 @@ block0(v0: i64):
; return v4
; }
+;; A volatile store in an alias region is a fence too: what is known about
+;; other regions is not forwarded across it.
+function %no_forwarding_across_volatile_store_in_region(i64, i64) -> i32 {
+ region0 = 0 "heap"
+ region1 = 1 "mmio"
+block0(v0: i64, v1: i64):
+ v2 = iconst.i32 1
+ v3 = iconst.i32 2
+ store notrap aligned region0 v2, v0
+ store notrap aligned volatile region1 v3, v1
+ v4 = load.i32 notrap aligned region0 v0
+ return v4
+}
+
+; function %no_forwarding_across_volatile_store_in_region(i64, i64) -> i32 fast {
+; region0 = 0 "heap"
+; region1 = 1 "mmio"
+;
+; block0(v0: i64, v1: i64):
+; v2 = iconst.i32 1
+; store notrap aligned region0 v2, v0 ; v2 = 1
+; v3 = iconst.i32 2
+; store notrap aligned volatile region1 v3, v1 ; v3 = 2
+; v4 = load.i32 notrap aligned region0 v0
+; return v4
+; }
+
;; Narrow, wide and vector volatile accesses, and extending loads.
function %volatile_widths(i64) -> i8, i64, i128, i8x16, i32 {
block0(v0: i64):