A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: merge readonly (but not notrap or can-move) loads. (#14972)
Chris Fallin38fb6c7eb9cf
diff --git a/cranelift/codegen/src/egraph/mod.rs b/cranelift/codegen/src/egraph/mod.rsindex d0280f557b..c0e44f21e9 100644--- a/cranelift/codegen/src/egraph/mod.rs+++ b/cranelift/codegen/src/egraph/mod.rs@@ -494,7 +494,13 @@ where// First, can we try to deduplicate? We need to keep some copy// of the instruction around because it's side-effecting, but// we may be able to reuse an earlier instance of it.- if is_mergeable_for_egraph(self.func, inst) {+ //+ // If we insert this instruction into the GVN map as a new+ // entry, remember its key: alias analysis below may still+ // remove it (e.g. a mergeable `readonly` load), in which case+ // the GVN entry must be updated to name the replacement value+ // rather than this instruction's (soon-removed) result.+ let gvn_key = if is_mergeable_for_egraph(self.func, inst) {let result = self.func.dfg.inst_results(inst).get(0).copied();trace!(" -> mergeable side-effecting op {}", inst);@@ -542,9 +548,12 @@ where}v.insert(result);trace!(" -> inserts as new (no GVN)");+ Some((ty, self.func.dfg.insts[inst]))}}- }+ } else {+ None+ };// Otherwise, if a load or store, process it with the alias analysis to// see if we can optimize it (rewrite in terms of an earlier load or@@ -562,6 +571,12 @@ where);self.value_to_opt_value[result] = new_result;self.available_block[result] = self.available_block[new_result];+ if let Some(key) = gvn_key {+ let depth = self.gvn_map.depth();+ debug_assert!(self.gvn_map.get(&NullCtx, &key).is_some());+ self.gvn_map+ .insert_with_depth(&NullCtx, key, Some(new_result), depth);+ }Some(SkeletonInstSimplification::Remove)}OptResult::IdempotentStore => {diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rsindex 08e4191b6b..993b2673a6 100644--- a/cranelift/codegen/src/inst_predicates.rs+++ b/cranelift/codegen/src/inst_predicates.rs@@ -1,6 +1,8 @@//! Instruction predicates/properties, shared by various analyses.use crate::ir::immediates::Offset32;-use crate::ir::{self, Block, DataFlowGraph, Function, Inst, InstructionData, Opcode, Type, Value};+use crate::ir::{+ self, Block, DataFlowGraph, Function, Inst, InstructionData, MemFlagsData, Opcode, Type, Value,+};/// Test whether the given opcode is unsafe to even consider as side-effect-free.#[inline(always)]@@ -55,25 +57,29 @@ fn has_side_effect(func: &Function, inst: Inst) -> bool {|| (opcode.can_load() && is_volatile_access(func, inst))}+fn load_mem_flags(func: &Function, inst: Inst) -> Option<MemFlagsData> {+ match func.dfg.insts[inst] {+ InstructionData::Load {+ opcode: Opcode::Load,+ flags,+ ..+ } => Some(func.dfg.mem_flags[flags]),+ _ => None,+ }+}+/// Does the given instruction behave as a "pure" node with respect to/// aegraph semantics?////// - Trivially pure nodes (bitwise arithmetic, etc)/// - Loads with the `readonly`, `notrap`, and `can_move` flags setpub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {- let is_pure_load = match func.dfg.insts[inst] {- InstructionData::Load {- opcode: Opcode::Load,- flags,- ..- } => {- let flags = func.dfg.mem_flags[flags];- // `readonly` and `can_move` each imply not `volatile` (the- // verifier rejects the combination), so such a load is never a- // volatile access.- flags.readonly() && flags.notrap() && flags.can_move()- }- _ => false,+ let is_pure_load = match load_mem_flags(func, inst) {+ // `readonly` and `can_move` each imply not `volatile` (the+ // verifier rejects the combination), so such a load is never a+ // volatile access.+ Some(flags) => flags.readonly() && flags.notrap() && flags.can_move(),+ None => false,};// Multi-value results do not play nicely with much of the egraph@@ -98,15 +104,34 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {/// result.pub fn is_mergeable_for_egraph(func: &Function, inst: Inst) -> bool {let op = func.dfg.insts[inst].opcode();+ let is_readonly_load = load_mem_flags(func, inst)+ .map(|flags| flags.readonly())+ .unwrap_or(false);// We can only merge zero- and one-result operators due to the way that GVN// is structured in the egraph implementation.func.dfg.inst_results(inst).len() <= 1// Loads/stores are handled by alias analysis and not// otherwise mergeable.- && !op.can_load()+ //+ // However, `readonly` loads *are* mergeable. Note that+ // `is_mergeable` is checked after `is_pure`, and readonly ++ // notrap + can_move loads are pure. Loads that are just+ // `readonly` are not pure (we must execute them, for their+ // potential trap side-effects, and for the same reason we+ // cannot hoist them arbitrarily), but one `readonly` load+ // subsuming another *is* a valid reason to make the second+ // reuse the first's result: `readonly` promises that once we+ // do the load, any subsequent load of the same address is+ // always the same.+ //+ // We don't let that case fall back to alias analysis because+ // its analysis is strictly more conservative: e.g. it will+ // not permit this merging across a call, while `readonly`+ // still promises that the value is constant across a call.+ && (!op.can_load() || is_readonly_load)&& !op.can_store()// Can only have idempotent side-effects.- && (!has_side_effect(func, inst) || op.side_effects_idempotent())+ && (!has_side_effect(func, inst) || op.side_effects_idempotent() || is_readonly_load)}/// Does the given instruction have any side-effect as per [has_side_effect], or else is a load,diff --git a/cranelift/filetests/filetests/egraph/readonly-loads-gvn.clif b/cranelift/filetests/filetests/egraph/readonly-loads-gvn.clifnew file mode 100644index 0000000000..54b5df7d9d--- /dev/null+++ b/cranelift/filetests/filetests/egraph/readonly-loads-gvn.clif@@ -0,0 +1,16 @@+test optimize+set opt_level=speed+target x86_64++function %f0(i64) -> i32, i32 {+ fn0 = colocated %ext(i64)++block0(v0: i64):+ v1 = load.i32 readonly v0+ call fn0(v0)+ v2 = load.i32 readonly v0+ return v1, v2+ ; check: v1 = load.i32 readonly v0+ ; nextln: call fn0(v0)+ ; nextln: return v1, v1+}diff --git a/tests/disas/funcref-readonly-loads.wat b/tests/disas/funcref-readonly-loads.watindex 3359932f81..22a06a2aa9 100644--- a/tests/disas/funcref-readonly-loads.wat+++ b/tests/disas/funcref-readonly-loads.wat@@ -23,9 +23,7 @@;; @001e v3 = load.i64 user16 aligned readonly region2 v2+8;; @001e v4 = load.i64 notrap aligned readonly region3 v2+24;; @001e call_indirect sig0, v3(v4, v0)-;; @0022 v5 = load.i64 user16 aligned readonly region2 v2+8-;; @0022 v6 = load.i64 notrap aligned readonly region3 v2+24-;; @0022 call_indirect sig0, v5(v6, v0)+;; @0022 call_indirect sig0, v3(v4, v0);; @0024 jump block1;;;; block1: