A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: unwind last-store state after removing a dead store (#14111)
Nick Fitzgeralda9238c105b20
diff --git a/cranelift/codegen/src/alias_analysis.rs b/cranelift/codegen/src/alias_analysis.rsindex c15769fd70..626455a778 100644--- a/cranelift/codegen/src/alias_analysis.rs+++ b/cranelift/codegen/src/alias_analysis.rs@@ -424,6 +424,32 @@ impl LastStores {}}+ /// Roll this state back to the memory version from just before `dead`,+ /// which is a store being removed from the function by dead-store+ /// elimination.+ ///+ /// `prev_last_store` must be the last-store instruction that immediately+ /// preceded `dead`, as recorded when `dead` itself was processed.+ ///+ /// Only `dead`'s own alias-region slot is restored. A store with no alias+ /// region is treated as a fence by `update`, which clears *every* region+ /// slot, and we do not undo that; in that case, we leave this state+ /// alone. Similarly, stores marked observed while processing `dead` stay+ /// observed.+ fn undo_store(&mut self, func: &Function, dead: Inst, prev_last_store: PackedOption<Inst>) {+ debug_assert!(func.dfg.insts[dead].opcode().can_store());++ let Some(region) = func.dfg.insts[dead].alias_region(&func.dfg) else {+ return;+ };++ // Only roll back if `dead` really is the current last store to its+ // region.+ if self.regions[region].expand() == Some(dead) {+ self.regions[region] = prev_last_store;+ }+ }+/// Get the last-store instruction for the given `inst`'s alias region, if/// any.fn get_last_store(&self, func: &Function, inst: Inst) -> PackedOption<Inst> {@@ -527,6 +553,26 @@ struct MemoryLoc {extending_opcode: Option<Opcode>,}+/// What is known to be in memory at an associated `MemoryLoc`.+#[derive(Clone, Copy, Debug)]+struct KnownValue {+ /// The value held at the associated `MemoryLoc`.+ value: Value,++ /// The instruction that produced `value`: either the load that read it out+ /// of memory or the store that wrote it there.+ ///+ /// Kept around for quick dominance checks.+ def_inst: Inst,++ /// When this entry was created by a store, the last-store instruction that+ /// immediately preceded that store: that is, the memory version this+ /// location was at just *before* `def_inst` overwrote it.+ ///+ /// `None` for entries created by loads.+ prev_last_store: Option<PackedOption<Inst>>,+}+/// The result of processing an instruction through alias analysis.pub enum OptResult {/// No optimization applied.@@ -576,9 +622,7 @@ pub struct AliasAnalysis<'a> {/// Known memory-value equivalences. This is the result of the/// analysis. This is a mapping from (last store, address/// expression, offset, type) to SSA `Value`.- ///- /// We keep the defining inst around for quick dominance checks.- mem_values: FxHashMap<MemoryLoc, (Inst, Value)>,+ mem_values: FxHashMap<MemoryLoc, KnownValue>,}impl<'a> AliasAnalysis<'a> {@@ -754,7 +798,36 @@ impl<'a> AliasAnalysis<'a> {ty,extending_opcode: get_ext_opcode(opcode),};- self.mem_values.remove(&dead_loc);+ let dead_entry = self.mem_values.remove(&dead_loc);++ // Roll our last-store state back to the memory version+ // just before the dead store, so that `state` describes+ // memory as if the dead store had never happened.+ //+ // Our callers remove the dead store from the layout and+ // then reprocess this overwriting store. Without the+ // rollback, that reprocessing keys its `mem_values`+ // lookup on the instruction we just removed, finds+ // nothing, and so fails to notice that the overwriter+ // has now become an idempotent store. Chains like+ //+ // v1 = load.i32 region0 v0+ // store region0 v2, v0 ;; dead+ // store region0 v1, v0 ;; idempotent, once the+ // ;; dead store is gone+ //+ // would then need a whole additional pass over the+ // function to collapse each link.+ //+ // A missing entry means we never processed the dead+ // store as a store in this pass (it can come from a+ // precomputed `block_input` snapshot, for a predecessor+ // block we have not walked yet), so we have no previous+ // version to roll back to and simply don't.+ if let Some(prev) = dead_entry.and_then(|e| e.prev_last_store) {+ state.undo_store(func, last_store, prev);+ }+return OptResult::DeadStore {dead: last_store,overwriter: inst,@@ -769,7 +842,12 @@ impl<'a> AliasAnalysis<'a> {ty,extending_opcode: get_ext_opcode(opcode),};- if let Some((def_inst, known_value)) = self.mem_values.get(&check_loc).cloned() {+ if let Some(KnownValue {+ def_inst,+ value: known_value,+ ..+ }) = self.mem_values.get(&check_loc).cloned()+ {// Check for idempotent stores, where we are// storing the exact same value back to a location// that already has that value.@@ -806,7 +884,14 @@ impl<'a> AliasAnalysis<'a> {extending_opcode: get_ext_opcode(opcode),};trace!(" --> updating known values in memory: {mem_loc:?} = {store_data}");- self.mem_values.insert(mem_loc, (inst, store_data));+ self.mem_values.insert(+ mem_loc,+ KnownValue {+ def_inst: inst,+ value: store_data,+ prev_last_store: Some(last_store),+ },+ );OptResult::None} else if opcode.can_load() {@@ -831,8 +916,9 @@ impl<'a> AliasAnalysis<'a> {// load (stores will always dominate though if// their `last_store` survives through// meet-points to this use-site).- let aliased = if let Some((def_inst, value)) =- self.mem_values.get(&mem_loc).cloned()+ let aliased = if let Some(KnownValue {+ def_inst, value, ..+ }) = self.mem_values.get(&mem_loc).cloned(){trace!(" see known value {value} from {def_inst}");if self.domtree.dominates(def_inst, inst, &func.layout) {@@ -851,7 +937,16 @@ impl<'a> AliasAnalysis<'a> {// as a new equivalent value.if aliased.is_none() {trace!(" --> inserting load result {load_result} at loc {mem_loc:?}");- self.mem_values.insert(mem_loc, (inst, load_result));+ self.mem_values.insert(+ mem_loc,+ KnownValue {+ def_inst: inst,+ value: load_result,+ // A load does not advance the memory version, so+ // there is no previous version to roll back to.+ prev_last_store: None,+ },+ );}match aliased {diff --git a/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif b/cranelift/filetests/filetests/alias/check-unset-reset-flag.clifindex af926b0a0b..08188dd75c 100644--- a/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif+++ b/cranelift/filetests/filetests/alias/check-unset-reset-flag.clif@@ -21,7 +21,7 @@ block0(v0: i64, v1: i32):; block0(v0: i64, v1: i32):; v2 = load.i64 notrap aligned region0 v0; trapz v2, user42-; store notrap aligned region0 v2, v0; v4 = iadd v1, v1; return v4; }+diff --git a/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clif b/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clifnew file mode 100644index 0000000000..02e5ecd5b8--- /dev/null+++ b/cranelift/filetests/filetests/alias/dead-store-then-idempotent-store.clif@@ -0,0 +1,181 @@+test optimize precise-output+set opt_level=speed+target x86_64++;; Removing a dead store must expose the *previous* memory version to the store+;; that overwrote it, so that a save/clear/restore sequence collapses entirely in+;; a single pass rather than one link per pass.+function %save_clear_restore(i64) {+ region0 = 0 "flags"+block0(v0: i64):+ v1 = load.i32 notrap aligned region0 v0+ v2 = iconst.i32 0+ store notrap aligned region0 v2, v0+ store notrap aligned region0 v1, v0+ return+}++; function %save_clear_restore(i64) fast {+; region0 = 0 "flags"+;+; block0(v0: i64):+; v1 = load.i32 notrap aligned region0 v0+; return+; }++;; The same, but with several dead stores between the load and the restore.+function %save_clobber_many_restore(i64, i32, i32) {+ region0 = 0 "flags"+block0(v0: i64, v1: i32, v2: i32):+ v3 = load.i32 notrap aligned region0 v0+ store notrap aligned region0 v1, v0+ store notrap aligned region0 v2, v0+ store notrap aligned region0 v1, v0+ store notrap aligned region0 v3, v0+ return+}++; function %save_clobber_many_restore(i64, i32, i32) fast {+; region0 = 0 "flags"+;+; block0(v0: i64, v1: i32, v2: i32):+; v3 = load.i32 notrap aligned region0 v0+; return+; }++;; Two independent flags, each in its own alias region, are both collapsed.+;;+;; Note that the accesses are interleaved: unwinding one region's dead store+;; must not disturb the other region's last-store state.+function %two_regions_interleaved(i64, i64) {+ region0 = 0 "flags0"+ region1 = 1 "flags1"+block0(v0: i64, v1: i64):+ v2 = load.i32 notrap aligned region0 v0+ v3 = load.i32 notrap aligned region1 v1+ v4 = iconst.i32 0+ store notrap aligned region0 v4, v0+ store notrap aligned region1 v4, v1+ store notrap aligned region0 v2, v0+ store notrap aligned region1 v3, v1+ return+}++; function %two_regions_interleaved(i64, i64) fast {+; region0 = 0 "flags0"+; region1 = 1 "flags1"+;+; block0(v0: i64, v1: i64):+; v2 = load.i32 notrap aligned region0 v0+; v3 = load.i32 notrap aligned region1 v1+; return+; }++;; The restore is folded across intervening blocks, so long as nothing in them+;; observes the flag.+function %save_clear_restore_cross_block(i64) {+ region0 = 0 "flags"+block0(v0: i64):+ v1 = load.i32 notrap aligned region0 v0+ v2 = iconst.i32 0+ store notrap aligned region0 v2, v0+ jump block1++block1:+ jump block2++block2:+ store notrap aligned region0 v1, v0+ return+}++; function %save_clear_restore_cross_block(i64) fast {+; region0 = 0 "flags"+;+; block0(v0: i64):+; v1 = load.i32 notrap aligned region0 v0+; jump block1+;+; block1:+; jump block2+;+; block2:+; return+; }++;; Negative test: a call between the clear and the restore observes the cleared+;; flag, so neither store may be removed.+function %call_observes_cleared_flag(i64) {+ region0 = 0 "flags"+ fn0 = %g(i64)+block0(v0: i64):+ v1 = load.i32 notrap aligned region0 v0+ v2 = iconst.i32 0+ store notrap aligned region0 v2, v0+ call fn0(v0)+ store notrap aligned region0 v1, v0+ return+}++; function %call_observes_cleared_flag(i64) fast {+; region0 = 0 "flags"+; sig0 = (i64) fast+; fn0 = %g sig0+;+; block0(v0: i64):+; v1 = load.i32 notrap aligned region0 v0+; v2 = iconst.i32 0+; store notrap aligned region0 v2, v0 ; v2 = 0+; call fn0(v0)+; store notrap aligned region0 v1, v0+; return+; }++;; Negative test: the final store writes a value other than the saved one, so it+;; is not idempotent. Only the dead middle store is removed.+function %restore_wrong_value(i64, i32) {+ region0 = 0 "flags"+block0(v0: i64, v1: i32):+ v2 = load.i32 notrap aligned region0 v0+ v3 = iconst.i32 0+ store notrap aligned region0 v3, v0+ store notrap aligned region0 v1, v0+ return+}++; function %restore_wrong_value(i64, i32) fast {+; region0 = 0 "flags"+;+; block0(v0: i64, v1: i32):+; v2 = load.i32 notrap aligned region0 v0+; store notrap aligned region0 v1, v0+; return+; }++;; Negative test: rolling back to the previous memory version must not resurrect+;; knowledge across a store to a *different* address in the same region. The+;; region's last-store slot is per-region, not per-address, so after the store to+;; `v0+8` the analysis no longer knows what is at `v0`, and the final store to+;; `v0` cannot be proven idempotent.+function %same_region_different_address(i64, i32) {+ region0 = 0 "flags"+block0(v0: i64, v1: i32):+ v2 = load.i32 notrap aligned region0 v0+ v3 = iconst.i32 0+ store notrap aligned region0 v3, v0+ store notrap aligned region0 v1, v0+8+ store notrap aligned region0 v2, v0+ return+}++; function %same_region_different_address(i64, i32) fast {+; region0 = 0 "flags"+;+; block0(v0: i64, v1: i32):+; v2 = load.i32 notrap aligned region0 v0+; v3 = iconst.i32 0+; store notrap aligned region0 v3, v0 ; v3 = 0+; store notrap aligned region0 v1, v0+8+; store notrap aligned region0 v2, v0+; return+; }diff --git a/tests/disas/component-model/direct-adapter-calls-inlining.wat b/tests/disas/component-model/direct-adapter-calls-inlining.watindex 04435c56b5..d7900b80b5 100644--- a/tests/disas/component-model/direct-adapter-calls-inlining.wat+++ b/tests/disas/component-model/direct-adapter-calls-inlining.wat@@ -104,7 +104,6 @@;; block9:;; v11 = load.i64 notrap aligned readonly can_move region3 v3+112;; v12 = load.i32 notrap aligned region4 v11-;; store notrap aligned region4 v12, v11;; jump block13;;;; block13:diff --git a/tests/disas/component-model/direct-adapter-calls-x64.wat b/tests/disas/component-model/direct-adapter-calls-x64.watindex cfa4fead7e..8831a79b6f 100644--- a/tests/disas/component-model/direct-adapter-calls-x64.wat+++ b/tests/disas/component-model/direct-adapter-calls-x64.wat@@ -87,7 +87,7 @@;; movq 0x18(%r10), %r10;; addq $0x60, %r10;; cmpq %rsp, %r10-;; ja 0x147+;; ja 0x13e;; 79: subq $0x50, %rsp;; movq %rbx, 0x20(%rsp);; movq %r12, 0x28(%rsp)@@ -97,10 +97,10 @@;; movq %rdi, (%rsp);; movq (%rsp), %rdi;; movq 0x88(%rdi), %rcx-;; movl (%rcx), %eax+;; movl (%rcx), %esi;; movq %rcx, 0x10(%rsp)-;; testl %eax, %eax-;; movq %rax, 8(%rsp)+;; testl %esi, %esi+;; movq %rsi, 8(%rsp);; jne 0xd5;; b9: movq (%rsp), %rdi;; movq 0x58(%rdi), %rax@@ -109,24 +109,19 @@;; movq (%rsp), %rsi;; callq *%rax;; ├─╼ exception frame offset: SP = FP - 0x50-;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x132-;; jmp 0x130-;; d5: movq (%rsp), %rsi-;; movq 0x70(%rsi), %rax-;; movl (%rax), %ecx-;; movl %ecx, (%rax)-;; movq 0x48(%rsi), %rdi+;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x121+;; jmp 0x11f+;; d5: movq (%rsp), %rcx+;; movq 0x70(%rcx), %rax+;; movl (%rax), %eax+;; movq 0x48(%rcx), %rdi+;; movq (%rsp), %rsi;; callq 0;; ├─╼ exception frame offset: SP = FP - 0x50-;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0xef-;; jmp 0xf7-;; ef: movq %rax, %rdx-;; jmp 0x132-;; f7: movq %rax, %rdx-;; movq 8(%rsp), %rcx-;; movq 0x10(%rsp), %rax-;; movl %ecx, (%rax)-;; movq %rdx, %rax+;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x121+;; movq 0x10(%rsp), %rcx+;; movq 8(%rsp), %rsi+;; movl %esi, (%rcx);; movq 0x20(%rsp), %rbx;; movq 0x28(%rsp), %r12;; movq 0x30(%rsp), %r13@@ -136,12 +131,14 @@;; movq %rbp, %rsp;; popq %rbp;; retq-;; 12b: jmp 0x132-;; 130: ud2-;; 132: movq (%rsp), %rsi-;; 136: movq 0x58(%rsi), %rcx-;; 13a: movq 0x68(%rsi), %rdi-;; 13e: movl $0x31, %edx-;; 143: callq *%rcx-;; 145: ud2-;; 147: ud2+;; 11a: jmp 0x121+;; 11f: ud2+;; 121: movq (%rsp), %rcx+;; 125: movq 0x58(%rcx), %rcx+;; 129: movq (%rsp), %rax+;; 12d: movq 0x68(%rax), %rdi+;; 131: movl $0x31, %edx+;; 136: movq (%rsp), %rsi+;; 13a: callq *%rcx+;; 13c: ud2+;; 13e: ud2diff --git a/tests/disas/component-model/direct-adapter-calls.wat b/tests/disas/component-model/direct-adapter-calls.watindex e3ce501b14..870554c735 100644--- a/tests/disas/component-model/direct-adapter-calls.wat+++ b/tests/disas/component-model/direct-adapter-calls.wat@@ -133,7 +133,6 @@;; block7:;; @008e v11 = load.i64 notrap aligned readonly can_move region2 v0+112;; @008e v12 = load.i32 notrap aligned region3 v11-;; @009a store notrap aligned region3 v12, v11;; @009c v16 = load.i64 notrap aligned readonly can_move region4 v0+72;; @009c try_call fn0(v16, v0, v2), sig1, block10(ret0), [ context v0, default: block6(exn0) ];;diff --git a/tests/disas/component-model/sync-adapter-calls-x64.wat b/tests/disas/component-model/sync-adapter-calls-x64.watindex 432e14bd40..a67c0ce87a 100644--- a/tests/disas/component-model/sync-adapter-calls-x64.wat+++ b/tests/disas/component-model/sync-adapter-calls-x64.wat@@ -58,7 +58,7 @@;; movq 0x18(%r10), %r10;; addq $0x70, %r10;; cmpq %rsp, %r10-;; ja 0x156+;; ja 0x154;; 39: subq $0x60, %rsp;; movq %rbx, 0x30(%rsp);; movq %r12, 0x38(%rsp)@@ -76,8 +76,8 @@;; movl $0x17, %edx;; callq *%rax;; ├─╼ exception frame offset: SP = FP - 0x60-;; ╰─╼ exception handler: default handler, context at [SP+0x20], handler=0x140-;; jmp 0x13e+;; ╰─╼ exception handler: default handler, context at [SP+0x20], handler=0x13e+;; jmp 0x13c;; 84: movq 0xe0(%rsi), %rdx;; movl (%rdx), %r8d;; movl $0, (%rdx)@@ -96,8 +96,7 @@;; movl $0, 0x84(%rdi);; movq %rbx, 0x88(%rdi);; movq 0xb0(%rsi), %rsi-;; movl (%rsi), %ebx-;; movl %ebx, (%rsi)+;; movl (%rsi), %esi;; movq %r9, 0x88(%rdi);; movl %r10d, 0x80(%rdi);; movl %r11d, 0x84(%rdi)@@ -113,11 +112,11 @@;; movq %rbp, %rsp;; popq %rbp;; retq-;; 13e: ud2-;; 140: movq 0x20(%rsp), %rsi-;; 145: movq 0x58(%rsi), %rax-;; 149: movq 0x68(%rsi), %rdi-;; 14d: movl $0x31, %edx-;; 152: callq *%rax+;; 13c: ud2+;; 13e: movq 0x20(%rsp), %rsi+;; 143: movq 0x58(%rsi), %rax+;; 147: movq 0x68(%rsi), %rdi+;; 14b: movl $0x31, %edx+;; 150: callq *%rax+;; 152: ud2;; 154: ud2-;; 156: ud2diff --git a/tests/disas/component-model/sync-adapter-calls.wat b/tests/disas/component-model/sync-adapter-calls.watindex f5c9398bc6..73df6c06ee 100644--- a/tests/disas/component-model/sync-adapter-calls.wat+++ b/tests/disas/component-model/sync-adapter-calls.wat@@ -133,7 +133,6 @@;; store notrap aligned region5 v19, v20+136;; v26 = load.i64 notrap aligned readonly can_move region3 v3+176;; v27 = load.i32 notrap aligned region4 v26-;; store notrap aligned region4 v27, v26;; jump block17;;;; block17: