Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: unwind last-store state after removing a dead store (#14111)

Nick Fitzgeralda9238c105b20
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rs
index c15769fd70..626455a778 100644
--- a/cranelift/codegen/src/alias_analysis.rs
+++ b/cranelift/codegen/src/alias_analysis.rs
@@ -424,6 +424,32 @@ impl LastStores {
}
}
+ /// Roll this state back to the memory version from just before `dead`,
+ /// which is a store being removed from the function by dead-store
+ /// elimination.
+ ///
+ /// `prev_last_store` must be the last-store instruction that immediately
+ /// preceded `dead`, as recorded when `dead` itself was processed.
+ ///
+ /// Only `dead`'s own alias-region slot is restored. A store with no alias
+ /// region is treated as a fence by `update`, which clears *every* region
+ /// slot, and we do not undo that; in that case, we leave this state
+ /// alone. Similarly, stores marked observed while processing `dead` stay
+ /// observed.
+ fn undo_store(&mut self, func: &Function, dead: Inst, prev_last_store: PackedOption<Inst>) {
+ debug_assert!(func.dfg.insts[dead].opcode().can_store());
+
+ let Some(region) = func.dfg.insts[dead].alias_region(&func.dfg) else {
+ return;
+ };
+
+ // Only roll back if `dead` really is the current last store to its
+ // region.
+ if self.regions[region].expand() == Some(dead) {
+ self.regions[region] = prev_last_store;
+ }
+ }
+
/// Get the last-store instruction for the given `inst`'s alias region, if
/// any.
fn get_last_store(&self, func: &Function, inst: Inst) -> PackedOption<Inst> {
@@ -527,6 +553,26 @@ struct MemoryLoc {
extending_opcode: Option<Opcode>,
}
+/// What is known to be in memory at an associated `MemoryLoc`.
+#[derive(Clone, Copy, Debug)]
+struct KnownValue {
+ /// The value held at the associated `MemoryLoc`.
+ value: Value,
+
+ /// The instruction that produced `value`: either the load that read it out
+ /// of memory or the store that wrote it there.
+ ///
+ /// Kept around for quick dominance checks.
+ def_inst: Inst,
+
+ /// When this entry was created by a store, the last-store instruction that
+ /// immediately preceded that store: that is, the memory version this
+ /// location was at just *before* `def_inst` overwrote it.
+ ///
+ /// `None` for entries created by loads.
+ prev_last_store: Option<PackedOption<Inst>>,
+}
+
/// The result of processing an instruction through alias analysis.
pub enum OptResult {
/// No optimization applied.
@@ -576,9 +622,7 @@ pub struct AliasAnalysis<'a> {
/// Known memory-value equivalences. This is the result of the
/// analysis. This is a mapping from (last store, address
/// expression, offset, type) to SSA `Value`.
- ///
- /// We keep the defining inst around for quick dominance checks.
- mem_values: FxHashMap<MemoryLoc, (Inst, Value)>,
+ mem_values: FxHashMap<MemoryLoc, KnownValue>,
}
impl<'a> AliasAnalysis<'a> {
@@ -754,7 +798,36 @@ impl<'a> AliasAnalysis<'a> {
ty,
extending_opcode: get_ext_opcode(opcode),
};
- self.mem_values.remove(&dead_loc);
+ let dead_entry = self.mem_values.remove(&dead_loc);
+
+ // Roll our last-store state back to the memory version
+ // just before the dead store, so that `state` describes
+ // memory as if the dead store had never happened.
+ //
+ // Our callers remove the dead store from the layout and
+ // then reprocess this overwriting store. Without the
+ // rollback, that reprocessing keys its `mem_values`
+ // lookup on the instruction we just removed, finds
+ // nothing, and so fails to notice that the overwriter
+ // has now become an idempotent store. Chains like
+ //
+ // v1 = load.i32 region0 v0
+ // store region0 v2, v0 ;; dead
+ // store region0 v1, v0 ;; idempotent, once the
+ // ;; dead store is gone
+ //
+ // would then need a whole additional pass over the
+ // function to collapse each link.
+ //
+ // A missing entry means we never processed the dead
+ // store as a store in this pass (it can come from a
+ // precomputed `block_input` snapshot, for a predecessor
+ // block we have not walked yet), so we have no previous
+ // version to roll back to and simply don't.
+ if let Some(prev) = dead_entry.and_then(|e| e.prev_last_store) {
+ state.undo_store(func, last_store, prev);
+ }
+
return OptResult::DeadStore {
dead: last_store,
overwriter: inst,
@@ -769,7 +842,12 @@ impl<'a> AliasAnalysis<'a> {
ty,
extending_opcode: get_ext_opcode(opcode),
};
- if let Some((def_inst, known_value)) = self.mem_values.get(&check_loc).cloned() {
+ if let Some(KnownValue {
+ def_inst,
+ value: known_value,
+ ..
+ }) = self.mem_values.get(&check_loc).cloned()
+ {
// Check for idempotent stores, where we are
// storing the exact same value back to a location
// that already has that value.
@@ -806,7 +884,14 @@ impl<'a> AliasAnalysis<'a> {
extending_opcode: get_ext_opcode(opcode),
};
trace!(" --> updating known values in memory: {mem_loc:?} = {store_data}");
- self.mem_values.insert(mem_loc, (inst, store_data));
+ self.mem_values.insert(
+ mem_loc,
+ KnownValue {
+ def_inst: inst,
+ value: store_data,
+ prev_last_store: Some(last_store),
+ },
+ );
OptResult::None
} else if opcode.can_load() {
@@ -831,8 +916,9 @@ impl<'a> AliasAnalysis<'a> {
// load (stores will always dominate though if
// their `last_store` survives through
// meet-points to this use-site).
- let aliased = if let Some((def_inst, value)) =
- self.mem_values.get(&mem_loc).cloned()
+ let aliased = if let Some(KnownValue {
+ def_inst, value, ..
+ }) = self.mem_values.get(&mem_loc).cloned()
{
trace!(" see known value {value} from {def_inst}");
if self.domtree.dominates(def_inst, inst, &func.layout) {
@@ -851,7 +937,16 @@ impl<'a> AliasAnalysis<'a> {
// as a new equivalent value.
if aliased.is_none() {
trace!(" --> inserting load result {load_result} at loc {mem_loc:?}");
- self.mem_values.insert(mem_loc, (inst, load_result));
+ self.mem_values.insert(
+ mem_loc,
+ KnownValue {
+ def_inst: inst,
+ value: load_result,
+ // A load does not advance the memory version, so
+ // there is no previous version to roll back to.
+ prev_last_store: None,
+ },
+ );
}
match aliased {
diff --git a/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif b/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif
index af926b0a0b..08188dd75c 100644
--- a/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif
+++ b/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif
@@ -21,7 +21,7 @@ block0(v0: i64, v1: i32):
; block0(v0: i64, v1: i32):
; v2 = load.i64 notrap aligned region0 v0
; trapz v2, user42
-; store notrap aligned region0 v2, v0
; v4 = iadd v1, v1
; return v4
; }
+
diff --git a/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clif b/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clif
new file mode 100644
index 0000000000..02e5ecd5b8
--- /dev/null
+++ b/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clif
@@ -0,0 +1,181 @@
+test optimize precise-output
+set opt_level=speed
+target x86_64
+
+;; Removing a dead store must expose the *previous* memory version to the store
+;; that overwrote it, so that a save/clear/restore sequence collapses entirely in
+;; a single pass rather than one link per pass.
+function %save_clear_restore(i64) {
+ region0 = 0 "flags"
+block0(v0: i64):
+ v1 = load.i32 notrap aligned region0 v0
+ v2 = iconst.i32 0
+ store notrap aligned region0 v2, v0
+ store notrap aligned region0 v1, v0
+ return
+}
+
+; function %save_clear_restore(i64) fast {
+; region0 = 0 "flags"
+;
+; block0(v0: i64):
+; v1 = load.i32 notrap aligned region0 v0
+; return
+; }
+
+;; The same, but with several dead stores between the load and the restore.
+function %save_clobber_many_restore(i64, i32, i32) {
+ region0 = 0 "flags"
+block0(v0: i64, v1: i32, v2: i32):
+ v3 = load.i32 notrap aligned region0 v0
+ store notrap aligned region0 v1, v0
+ store notrap aligned region0 v2, v0
+ store notrap aligned region0 v1, v0
+ store notrap aligned region0 v3, v0
+ return
+}
+
+; function %save_clobber_many_restore(i64, i32, i32) fast {
+; region0 = 0 "flags"
+;
+; block0(v0: i64, v1: i32, v2: i32):
+; v3 = load.i32 notrap aligned region0 v0
+; return
+; }
+
+;; Two independent flags, each in its own alias region, are both collapsed.
+;;
+;; Note that the accesses are interleaved: unwinding one region's dead store
+;; must not disturb the other region's last-store state.
+function %two_regions_interleaved(i64, i64) {
+ region0 = 0 "flags0"
+ region1 = 1 "flags1"
+block0(v0: i64, v1: i64):
+ v2 = load.i32 notrap aligned region0 v0
+ v3 = load.i32 notrap aligned region1 v1
+ v4 = iconst.i32 0
+ store notrap aligned region0 v4, v0
+ store notrap aligned region1 v4, v1
+ store notrap aligned region0 v2, v0
+ store notrap aligned region1 v3, v1
+ return
+}
+
+; function %two_regions_interleaved(i64, i64) fast {
+; region0 = 0 "flags0"
+; region1 = 1 "flags1"
+;
+; block0(v0: i64, v1: i64):
+; v2 = load.i32 notrap aligned region0 v0
+; v3 = load.i32 notrap aligned region1 v1
+; return
+; }
+
+;; The restore is folded across intervening blocks, so long as nothing in them
+;; observes the flag.
+function %save_clear_restore_cross_block(i64) {
+ region0 = 0 "flags"
+block0(v0: i64):
+ v1 = load.i32 notrap aligned region0 v0
+ v2 = iconst.i32 0
+ store notrap aligned region0 v2, v0
+ jump block1
+
+block1:
+ jump block2
+
+block2:
+ store notrap aligned region0 v1, v0
+ return
+}
+
+; function %save_clear_restore_cross_block(i64) fast {
+; region0 = 0 "flags"
+;
+; block0(v0: i64):
+; v1 = load.i32 notrap aligned region0 v0
+; jump block1
+;
+; block1:
+; jump block2
+;
+; block2:
+; return
+; }
+
+;; Negative test: a call between the clear and the restore observes the cleared
+;; flag, so neither store may be removed.
+function %call_observes_cleared_flag(i64) {
+ region0 = 0 "flags"
+ fn0 = %g(i64)
+block0(v0: i64):
+ v1 = load.i32 notrap aligned region0 v0
+ v2 = iconst.i32 0
+ store notrap aligned region0 v2, v0
+ call fn0(v0)
+ store notrap aligned region0 v1, v0
+ return
+}
+
+; function %call_observes_cleared_flag(i64) fast {
+; region0 = 0 "flags"
+; sig0 = (i64) fast
+; fn0 = %g sig0
+;
+; block0(v0: i64):
+; v1 = load.i32 notrap aligned region0 v0
+; v2 = iconst.i32 0
+; store notrap aligned region0 v2, v0 ; v2 = 0
+; call fn0(v0)
+; store notrap aligned region0 v1, v0
+; return
+; }
+
+;; Negative test: the final store writes a value other than the saved one, so it
+;; is not idempotent. Only the dead middle store is removed.
+function %restore_wrong_value(i64, i32) {
+ region0 = 0 "flags"
+block0(v0: i64, v1: i32):
+ v2 = load.i32 notrap aligned region0 v0
+ v3 = iconst.i32 0
+ store notrap aligned region0 v3, v0
+ store notrap aligned region0 v1, v0
+ return
+}
+
+; function %restore_wrong_value(i64, i32) fast {
+; region0 = 0 "flags"
+;
+; block0(v0: i64, v1: i32):
+; v2 = load.i32 notrap aligned region0 v0
+; store notrap aligned region0 v1, v0
+; return
+; }
+
+;; Negative test: rolling back to the previous memory version must not resurrect
+;; knowledge across a store to a *different* address in the same region. The
+;; region's last-store slot is per-region, not per-address, so after the store to
+;; `v0+8` the analysis no longer knows what is at `v0`, and the final store to
+;; `v0` cannot be proven idempotent.
+function %same_region_different_address(i64, i32) {
+ region0 = 0 "flags"
+block0(v0: i64, v1: i32):
+ v2 = load.i32 notrap aligned region0 v0
+ v3 = iconst.i32 0
+ store notrap aligned region0 v3, v0
+ store notrap aligned region0 v1, v0+8
+ store notrap aligned region0 v2, v0
+ return
+}
+
+; function %same_region_different_address(i64, i32) fast {
+; region0 = 0 "flags"
+;
+; block0(v0: i64, v1: i32):
+; v2 = load.i32 notrap aligned region0 v0
+; v3 = iconst.i32 0
+; store notrap aligned region0 v3, v0 ; v3 = 0
+; store notrap aligned region0 v1, v0+8
+; store notrap aligned region0 v2, v0
+; return
+; }
diff --git a/tests/disas/component-model/direct-adapter-calls-inlining.wat b/tests/disas/component-model/direct-adapter-calls-inlining.wat
index 04435c56b5..d7900b80b5 100644
--- a/tests/disas/component-model/direct-adapter-calls-inlining.wat
+++ b/tests/disas/component-model/direct-adapter-calls-inlining.wat
@@ -104,7 +104,6 @@
;; block9:
;; v11 = load.i64 notrap aligned readonly can_move region3 v3+112
;; v12 = load.i32 notrap aligned region4 v11
-;; store notrap aligned region4 v12, v11
;; jump block13
;;
;; block13:
diff --git a/tests/disas/component-model/direct-adapter-calls-x64.wat b/tests/disas/component-model/direct-adapter-calls-x64.wat
index cfa4fead7e..8831a79b6f 100644
--- a/tests/disas/component-model/direct-adapter-calls-x64.wat
+++ b/tests/disas/component-model/direct-adapter-calls-x64.wat
@@ -87,7 +87,7 @@
;; movq 0x18(%r10), %r10
;; addq $0x60, %r10
;; cmpq %rsp, %r10
-;; ja 0x147
+;; ja 0x13e
;; 79: subq $0x50, %rsp
;; movq %rbx, 0x20(%rsp)
;; movq %r12, 0x28(%rsp)
@@ -97,10 +97,10 @@
;; movq %rdi, (%rsp)
;; movq (%rsp), %rdi
;; movq 0x88(%rdi), %rcx
-;; movl (%rcx), %eax
+;; movl (%rcx), %esi
;; movq %rcx, 0x10(%rsp)
-;; testl %eax, %eax
-;; movq %rax, 8(%rsp)
+;; testl %esi, %esi
+;; movq %rsi, 8(%rsp)
;; jne 0xd5
;; b9: movq (%rsp), %rdi
;; movq 0x58(%rdi), %rax
@@ -109,24 +109,19 @@
;; movq (%rsp), %rsi
;; callq *%rax
;; ├─╼ exception frame offset: SP = FP - 0x50
-;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x132
-;; jmp 0x130
-;; d5: movq (%rsp), %rsi
-;; movq 0x70(%rsi), %rax
-;; movl (%rax), %ecx
-;; movl %ecx, (%rax)
-;; movq 0x48(%rsi), %rdi
+;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x121
+;; jmp 0x11f
+;; d5: movq (%rsp), %rcx
+;; movq 0x70(%rcx), %rax
+;; movl (%rax), %eax
+;; movq 0x48(%rcx), %rdi
+;; movq (%rsp), %rsi
;; callq 0
;; ├─╼ exception frame offset: SP = FP - 0x50
-;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0xef
-;; jmp 0xf7
-;; ef: movq %rax, %rdx
-;; jmp 0x132
-;; f7: movq %rax, %rdx
-;; movq 8(%rsp), %rcx
-;; movq 0x10(%rsp), %rax
-;; movl %ecx, (%rax)
-;; movq %rdx, %rax
+;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x121
+;; movq 0x10(%rsp), %rcx
+;; movq 8(%rsp), %rsi
+;; movl %esi, (%rcx)
;; movq 0x20(%rsp), %rbx
;; movq 0x28(%rsp), %r12
;; movq 0x30(%rsp), %r13
@@ -136,12 +131,14 @@
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 12b: jmp 0x132
-;; 130: ud2
-;; 132: movq (%rsp), %rsi
-;; 136: movq 0x58(%rsi), %rcx
-;; 13a: movq 0x68(%rsi), %rdi
-;; 13e: movl $0x31, %edx
-;; 143: callq *%rcx
-;; 145: ud2
-;; 147: ud2
+;; 11a: jmp 0x121
+;; 11f: ud2
+;; 121: movq (%rsp), %rcx
+;; 125: movq 0x58(%rcx), %rcx
+;; 129: movq (%rsp), %rax
+;; 12d: movq 0x68(%rax), %rdi
+;; 131: movl $0x31, %edx
+;; 136: movq (%rsp), %rsi
+;; 13a: callq *%rcx
+;; 13c: ud2
+;; 13e: ud2
diff --git a/tests/disas/component-model/direct-adapter-calls.wat b/tests/disas/component-model/direct-adapter-calls.wat
index e3ce501b14..870554c735 100644
--- a/tests/disas/component-model/direct-adapter-calls.wat
+++ b/tests/disas/component-model/direct-adapter-calls.wat
@@ -133,7 +133,6 @@
;; block7:
;; @008e v11 = load.i64 notrap aligned readonly can_move region2 v0+112
;; @008e v12 = load.i32 notrap aligned region3 v11
-;; @009a store notrap aligned region3 v12, v11
;; @009c v16 = load.i64 notrap aligned readonly can_move region4 v0+72
;; @009c try_call fn0(v16, v0, v2), sig1, block10(ret0), [ context v0, default: block6(exn0) ]
;;
diff --git a/tests/disas/component-model/sync-adapter-calls-x64.wat b/tests/disas/component-model/sync-adapter-calls-x64.wat
index 432e14bd40..a67c0ce87a 100644
--- a/tests/disas/component-model/sync-adapter-calls-x64.wat
+++ b/tests/disas/component-model/sync-adapter-calls-x64.wat
@@ -58,7 +58,7 @@
;; movq 0x18(%r10), %r10
;; addq $0x70, %r10
;; cmpq %rsp, %r10
-;; ja 0x156
+;; ja 0x154
;; 39: subq $0x60, %rsp
;; movq %rbx, 0x30(%rsp)
;; movq %r12, 0x38(%rsp)
@@ -76,8 +76,8 @@
;; movl $0x17, %edx
;; callq *%rax
;; ├─╼ exception frame offset: SP = FP - 0x60
-;; ╰─╼ exception handler: default handler, context at [SP+0x20], handler=0x140
-;; jmp 0x13e
+;; ╰─╼ exception handler: default handler, context at [SP+0x20], handler=0x13e
+;; jmp 0x13c
;; 84: movq 0xe0(%rsi), %rdx
;; movl (%rdx), %r8d
;; movl $0, (%rdx)
@@ -96,8 +96,7 @@
;; movl $0, 0x84(%rdi)
;; movq %rbx, 0x88(%rdi)
;; movq 0xb0(%rsi), %rsi
-;; movl (%rsi), %ebx
-;; movl %ebx, (%rsi)
+;; movl (%rsi), %esi
;; movq %r9, 0x88(%rdi)
;; movl %r10d, 0x80(%rdi)
;; movl %r11d, 0x84(%rdi)
@@ -113,11 +112,11 @@
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 13e: ud2
-;; 140: movq 0x20(%rsp), %rsi
-;; 145: movq 0x58(%rsi), %rax
-;; 149: movq 0x68(%rsi), %rdi
-;; 14d: movl $0x31, %edx
-;; 152: callq *%rax
+;; 13c: ud2
+;; 13e: movq 0x20(%rsp), %rsi
+;; 143: movq 0x58(%rsi), %rax
+;; 147: movq 0x68(%rsi), %rdi
+;; 14b: movl $0x31, %edx
+;; 150: callq *%rax
+;; 152: ud2
;; 154: ud2
-;; 156: ud2
diff --git a/tests/disas/component-model/sync-adapter-calls.wat b/tests/disas/component-model/sync-adapter-calls.wat
index f5c9398bc6..73df6c06ee 100644
--- a/tests/disas/component-model/sync-adapter-calls.wat
+++ b/tests/disas/component-model/sync-adapter-calls.wat
@@ -133,7 +133,6 @@
;; store notrap aligned region5 v19, v20+136
;; v26 = load.i64 notrap aligned readonly can_move region3 v3+176
;; v27 = load.i32 notrap aligned region4 v26
-;; store notrap aligned region4 v27, v26
;; jump block17
;;
;; block17: