Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: fix multiplicity analysis of multi-def instructions. (#14324)

Chris Fallinb287430bd70f
diff --git a/cranelift/codegen/src/machinst/lower.rs b/cranelift/codegen/src/machinst/lower.rs
index fc681f137f..843cd6e021 100644
--- a/cranelift/codegen/src/machinst/lower.rs
+++ b/cranelift/codegen/src/machinst/lower.rs
@@ -308,33 +308,25 @@ pub struct Lower<'func, I: VCodeInst> {
/// actually merged" point. Instead, we compute a
/// transitive-uniqueness. That is what this enum represents.
///
-/// There is one final caveat as well to the result of this analysis. Notably,
-/// we define some instructions to be "root" instructions, which means that we
-/// assume they will always be codegen'd at the root of a matching tree, and not
-/// matched. (This comes with the caveat that we actually enforce this property
-/// by making them "opaque" to subtree matching in
-/// `get_value_as_source_or_const`). Because they will always be codegen'd once,
-/// they in some sense "reset" multiplicity: these root instructions can be used
-/// many times, but because their result(s) are only computed once, they only
-/// use their inputs once.
-///
-/// We currently define all multi-result instructions to be "root" instructions,
-/// because it is too complex to reason about matching through them, and they
-/// cause too-coarse-grained approximation of multiplicity otherwise: the
-/// analysis would have to assume (as it used to!) that they are always
-/// multiply-used, simply because they have multiple outputs even if those
-/// outputs are used only once.
-///
-/// In the future we could define other instructions to be "root" instructions
-/// as well, if we make the corresponding change to get_value_as_source_or_const
-/// as well.
+/// Note that this analysis is fundamentally about *instructions*
+/// being codegen'd more than once, even though it is tracked per
+/// value. For an instruction with multiple results, a use of any
+/// result is a use of the instruction: a lowering that merges the
+/// instruction by matching on one of its results (e.g., a `brif` that
+/// directly consumes the flags produced by a `uadd_overflow`)
+/// generates the whole instruction, including its operands. We
+/// account for this by treating a use of any result as a use of every
+/// result: as soon as a second result of the same instruction becomes
+/// used, all of its used results (and, transitively, its operand
+/// tree) become `Multiple`.
///
/// To define `ValueUseState` more plainly: a value is `Unused` if no references
/// exist to it; `Once` if only one other op refers to it, *and* that other op
-/// is `Unused` or `Once`; and `Multiple` otherwise. In other words, `Multiple`
-/// is contagious (except through root instructions): even if an op's result
-/// value is directly used only once in the CLIF, that value is `Multiple` if
-/// the op that uses it is itself used multiple times (hence could be codegen'd
+/// is `Unused` or `Once`, *and* no other result of the same instruction is
+/// used; and `Multiple` otherwise. In
+/// other words, `Multiple` is contagious: even if an op's result value is
+/// directly used only once in the CLIF, that value is `Multiple` if the op
+/// that uses it is itself used multiple times (hence could be codegen'd
/// multiple times). In brief, this analysis tells us whether, if every op
/// merged all of its operand tree, a given op could be codegen'd in more than
/// one place.
@@ -543,7 +535,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
);
for (i, param) in self.f.dfg.block_params(entry_bb).iter().enumerate() {
- if self.value_ir_uses[*param] == ValueUseState::Unused {
+ if self.value_use_state(*param) == ValueUseState::Unused {
continue;
}
let regs = writable_value_regs(self.value_regs[*param]);
@@ -614,7 +606,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
// for the benefit of debuginfo.
if self.f.dfg.values_labels.is_some() {
if let Some(vmctx_val) = self.f.special_param(ArgumentPurpose::VMContext) {
- if self.value_ir_uses[vmctx_val] != ValueUseState::Unused {
+ if self.value_use_state(vmctx_val) != ValueUseState::Unused {
let vmctx_reg = self.value_regs[vmctx_val].only_reg().unwrap();
self.emit(I::gen_dummy_use(vmctx_reg));
}
@@ -1347,11 +1339,13 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
Ok(vcode)
}
+ fn value_use_state(&self, val: Value) -> ValueUseState {
+ let val = map_to_first_result(self.f, val);
+ self.value_ir_uses[val]
+ }
+
pub fn value_is_unused(&self, val: Value) -> bool {
- match self.value_ir_uses[val] {
- ValueUseState::Unused => true,
- _ => false,
- }
+ self.value_lowered_uses[val] == 0
}
/// Does this value still have uses to serve at the current point in the
@@ -1375,6 +1369,17 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
}
}
+/// Map any `Value` to the first `Value` defined by its defining
+/// instruction. (Leaves values defined by blockparams
+/// identity-mapped.)
+fn map_to_first_result(f: &Function, val: Value) -> Value {
+ match f.dfg.value_def(val) {
+ ValueDef::Result(_, idx) if idx == 0 => val,
+ ValueDef::Result(inst, _) => f.dfg.inst_results(inst)[0],
+ _ => val,
+ }
+}
+
/// Pre-analysis: compute `value_ir_uses`. See comment on
/// `ValueUseState` for a description of what this analysis
/// computes.
@@ -1401,25 +1406,29 @@ fn compute_use_states(
let mut value_ir_uses = SecondaryMap::with_default(ValueUseState::Unused);
+ // Step 1: call `mark_use` on every use. When a value is used
+ // multiple times, it is pushed into `multiple_stack`, which we
+ // use to do the graph-walk that propagates multiplicity below.
+ let mut multiple_stack: SmallVec<[Value; 16]> = smallvec![];
+
if let Some(sret_param) = sret_param {
// There's an implicit use of the struct-return parameter in each
// copy of the function epilogue, which we count here.
value_ir_uses[sret_param] = ValueUseState::Multiple;
+ multiple_stack.push(sret_param);
}
- // Stack of iterators over Values as we do DFS to mark
- // Multiple-state subtrees. The iterator type is whatever is
- // returned by `uses` below.
- let mut stack: SmallVec<[_; 16]> = smallvec![];
+ let mut mark_use = |val: Value, multiple_stack: &mut SmallVec<[Value; 16]>| {
+ let val = map_to_first_result(f, val);
+ let old = value_ir_uses[val];
+ value_ir_uses[val].inc();
+ if value_ir_uses[val] == ValueUseState::Multiple && old != ValueUseState::Multiple {
+ multiple_stack.push(val);
+ }
+ };
// Find the args for the inst corresponding to the given value.
- //
- // Note that "root" instructions are skipped here. This means that multiple
- // uses of any result of a multi-result instruction are not considered
- // multiple uses of the operands of a multi-result instruction. This
- // requires tight coupling with `get_value_as_source_or_const` above which
- // is the consumer of the map that this function is producing.
- let uses = |value| {
+ let uses = |value: Value| {
trace!(" -> pushing args for {} onto stack", value);
if let ValueDef::Result(src_inst, _) = f.dfg.value_def(value) {
Some(f.dfg.inst_values(src_inst))
@@ -1439,43 +1448,18 @@ fn compute_use_states(
// additional use on each operand.
for arg in f.dfg.inst_values(inst) {
debug_assert!(f.dfg.value_is_real(arg));
- let old = value_ir_uses[arg];
- value_ir_uses[arg].inc();
- let new = value_ir_uses[arg];
- trace!("arg {} used, old state {:?}, new {:?}", arg, old, new);
+ mark_use(arg, &mut multiple_stack);
+ }
+ }
- // On transition to Multiple, do DFS.
- if old == ValueUseState::Multiple || new != ValueUseState::Multiple {
- continue;
- }
- if let Some(iter) = uses(arg) {
- stack.push(iter);
- }
- while let Some(iter) = stack.last_mut() {
- if let Some(value) = iter.next() {
- debug_assert!(f.dfg.value_is_real(value));
- trace!(" -> DFS reaches {}", value);
- if value_ir_uses[value] == ValueUseState::Multiple {
- // Truncate DFS here: no need to go further,
- // as whole subtree must already be Multiple.
- // With debug asserts, check one level of
- // that invariant at least.
- debug_assert!(uses(value).into_iter().flatten().all(|arg| {
- debug_assert!(f.dfg.value_is_real(arg));
- value_ir_uses[arg] == ValueUseState::Multiple
- }));
- continue;
- }
- value_ir_uses[value] = ValueUseState::Multiple;
- trace!(" -> became Multiple");
- if let Some(iter) = uses(value) {
- stack.push(iter);
- }
- } else {
- // Empty iterator, discard.
- stack.pop();
- }
- }
+ // Step 2: propagate multiplicity. `Multiple` is
+ // transitive/contagious: if we can lower any value `v` multiple
+ // times, that might involve lowering its uses multiple times, and
+ // their uses, and so on, depending on how deep a lowering rule
+ // matches.
+ while let Some(value) = multiple_stack.pop() {
+ for u in uses(value).into_iter().flatten() {
+ mark_use(u, &mut multiple_stack);
}
}
@@ -1649,17 +1633,7 @@ impl<'func, I: VCodeInst> Lower<'func, I> {
// OK to merge source instruction if we have a source
// instruction, and one of these two conditions hold:
//
- // - It has no side-effects and this instruction is not a "value-use
- // root" instruction. Instructions which are considered "roots"
- // for value-use calculations do not have accurate information
- // known about the `ValueUseState` of their operands. This is
- // currently done for multi-result instructions to prevent a use
- // of each result from forcing all operands of the multi-result
- // instruction to also be `Multiple`. This in turn means that the
- // `ValueUseState` for operands of a "root" instruction to be a
- // lie if pattern matching were to look through the multi-result
- // instruction. As a result the "look through this instruction"
- // logic only succeeds if it's not a root instruction.
+ // - It has no side-effects.
//
// - It has a side-effect, has one output value, that one
// output has only one use, directly or indirectly (so
@@ -1818,7 +1792,7 @@ mod tests {
use crate::ir::{Function, InstBuilder};
#[test]
- fn multi_result_use_once() {
+ fn multi_result_two_results_used_once_each() {
let mut func = Function::new();
let block0 = func.dfg.make_block();
let mut pos = FuncCursor::new(&mut func);
@@ -1831,10 +1805,33 @@ mod tests {
let func = pos.func;
let uses = super::compute_use_states(&func, None);
- assert_eq!(uses[v1], ValueUseState::Once);
+ assert_eq!(uses[v1], ValueUseState::Multiple);
+ assert_eq!(uses[v2], ValueUseState::Multiple);
+ assert_eq!(uses[v3], ValueUseState::Multiple);
+ assert_eq!(uses[v4], ValueUseState::Multiple);
+ assert_eq!(uses[v5], ValueUseState::Unused);
+ }
+
+ #[test]
+ fn multi_result_one_result_used() {
+ let mut func = Function::new();
+ let block0 = func.dfg.make_block();
+ let mut pos = FuncCursor::new(&mut func);
+ pos.insert_block(block0);
+ let v1 = pos.ins().iconst(types::I64, 0);
+ let v2 = pos.ins().iconst(types::I64, 1);
+ let v3 = pos.ins().iadd(v1, v2);
+ let (v4, v5) = pos.ins().uadd_overflow(v3, v1);
+ let v6 = pos.ins().uextend(types::I64, v5);
+ pos.ins().return_(&[v6]);
+ let func = pos.func;
+
+ let uses = super::compute_use_states(&func, None);
+ assert_eq!(uses[v1], ValueUseState::Multiple);
assert_eq!(uses[v2], ValueUseState::Once);
assert_eq!(uses[v3], ValueUseState::Once);
assert_eq!(uses[v4], ValueUseState::Once);
- assert_eq!(uses[v5], ValueUseState::Once);
+ assert_eq!(uses[v5], ValueUseState::Unused); // use counted against first def.
+ assert_eq!(uses[v6], ValueUseState::Once);
}
}
diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif
index 7340b8f5b1..1ab9b267dd 100644
--- a/cranelift/filetests/filetests/isa/x64/i128.clif
+++ b/cranelift/filetests/filetests/isa/x64/i128.clif
@@ -2056,10 +2056,11 @@ block0(v0: i64, v1: i64):
; pushq %rbp
; movq %rsp, %rbp
; block0:
+; movq (%rdi), %rdi
; uninit %rdx
; xorq %rdx, %rdx
; movq %rsi, %rax
-; addq (%rdi), %rax
+; addq %rdi, %rax
; adcq $0x0, %rdx
; movq %rbp, %rsp
; popq %rbp
@@ -2070,9 +2071,10 @@ block0(v0: i64, v1: i64):
; pushq %rbp
; movq %rsp, %rbp
; block1: ; offset 0x4
+; movq (%rdi), %rdi ; trap: heap_oob
; xorq %rdx, %rdx
; movq %rsi, %rax
-; addq (%rdi), %rax ; trap: heap_oob
+; addq %rdi, %rax
; adcq $0, %rdx
; movq %rbp, %rsp
; popq %rbp
diff --git a/cranelift/filetests/filetests/isa/x64/mul_overflow_flag_consumers.clif b/cranelift/filetests/filetests/isa/x64/mul_overflow_flag_consumers.clif
index d3b666994c..20bd60fda1 100644
--- a/cranelift/filetests/filetests/isa/x64/mul_overflow_flag_consumers.clif
+++ b/cranelift/filetests/filetests/isa/x64/mul_overflow_flag_consumers.clif
@@ -271,3 +271,137 @@ block0(v0: i16, v1: i16):
; popq %rbp
; retq
+
+function %smul_flags_select_load_product_stored(i64, i32, i64, i64) -> i64 {
+block0(v0: i64, v1: i32, v2: i64, v3: i64):
+ v4 = load.i32 v0
+ v5, v6 = smul_overflow v1, v4
+ store v5, v0
+ v7 = select v6, v2, v3
+ return v7
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movq %rdx, %r8
+; movl (%rdi), %r10d
+; movq %rsi, %rax
+; imull %r10d ;; implicit: %eax, %edx
+; movl %eax, (%rdi)
+; movq %rsi, %rax
+; imull %r10d ;; implicit: %eax, %edx
+; movq %r8, %rdx
+; movq %rcx, %rax
+; cmovoq %rdx, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movq %rdx, %r8
+; movl (%rdi), %r10d ; trap: heap_oob
+; movq %rsi, %rax
+; imull %r10d
+; movl %eax, (%rdi) ; trap: heap_oob
+; movq %rsi, %rax
+; imull %r10d
+; movq %r8, %rdx
+; movq %rcx, %rax
+; cmovoq %rdx, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %smul_flags_select_load_product_used_between(i64, i32, i64) -> i64 {
+block0(v0: i64, v1: i32, v2: i64):
+ v4 = load.i32 v0
+ v5, v6 = smul_overflow v1, v4
+ v8 = uextend.i64 v5
+ v7 = select v6, v2, v8
+ return v7
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movq %rdx, %rcx
+; movl (%rdi), %r10d
+; movq %rsi, %rax
+; imull %r10d ;; implicit: %eax, %edx
+; movl %eax, %eax
+; movq %rax, %rdx
+; movq %rsi, %rax
+; movq %rdx, %rsi
+; imull %r10d ;; implicit: %eax, %edx
+; movq %rsi, %rax
+; movq %rcx, %rdx
+; cmovoq %rdx, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movq %rdx, %rcx
+; movl (%rdi), %r10d ; trap: heap_oob
+; movq %rsi, %rax
+; imull %r10d
+; movl %eax, %eax
+; movq %rax, %rdx
+; movq %rsi, %rax
+; movq %rdx, %rsi
+; imull %r10d
+; movq %rsi, %rax
+; movq %rcx, %rdx
+; cmovoq %rdx, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
+function %smul_flags_select_load_product_unused(i64, i32, i64, i64) -> i64 {
+block0(v0: i64, v1: i32, v2: i64, v3: i64):
+ v4 = load.i32 v0
+ v5, v6 = smul_overflow v1, v4
+ v7 = select v6, v2, v3
+ return v7
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movq %rsi, %rax
+; movq %rdx, %r8
+; imull (%rdi) ;; implicit: %eax, %edx
+; movq %r8, %rdx
+; movq %rcx, %rax
+; cmovoq %rdx, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movq %rsi, %rax
+; movq %rdx, %r8
+; imull (%rdi) ; trap: heap_oob
+; movq %r8, %rdx
+; movq %rcx, %rax
+; cmovoq %rdx, %rax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/uadd_overflow_brif.clif b/cranelift/filetests/filetests/isa/x64/uadd_overflow_brif.clif
index ba59e25065..7d338b471d 100644
--- a/cranelift/filetests/filetests/isa/x64/uadd_overflow_brif.clif
+++ b/cranelift/filetests/filetests/isa/x64/uadd_overflow_brif.clif
@@ -378,3 +378,57 @@ block2:
; popq %rbp
; retq
+
+function %no_load_sinking_sum_used_between(i64, i32) -> i32 {
+block0(v0: i64, v1: i32):
+ v2 = load.i32 v0
+ v3, v4 = uadd_overflow v1, v2
+ v5 = iadd v3, v1
+ brif v4, block1(v5), block2
+
+block1(v6: i32):
+ return v6
+
+block2:
+ return v1
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movl (%rdi), %edi
+; movq %rsi, %r8
+; addl %edi, %r8d
+; movq %rsi, %rax
+; jb label2; j label1
+; block1:
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block2:
+; leal (%r8, %rax), %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movl (%rdi), %edi ; trap: heap_oob
+; movq %rsi, %r8
+; addl %edi, %r8d
+; movq %rsi, %rax
+; jb 0x1a
+; block2: ; offset 0x15
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; block3: ; offset 0x1a
+; addl %r8d, %eax
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+
diff --git a/cranelift/filetests/filetests/runtests/mul_overflow_flag_consumers.clif b/cranelift/filetests/filetests/runtests/mul_overflow_flag_consumers.clif
index 813caa779d..c1fefd7923 100644
--- a/cranelift/filetests/filetests/runtests/mul_overflow_flag_consumers.clif
+++ b/cranelift/filetests/filetests/runtests/mul_overflow_flag_consumers.clif
@@ -213,3 +213,41 @@ block2:
; run: %smulof_flags_brif_i32(0x7FFFFFFF, 2) == 1
; run: %smulof_flags_brif_i32(0x80000000, -1) == 1
; run: %smulof_flags_brif_i32(0x7FFFFFFF, -1) == 0
+
+function %smulof_flags_select_load_product_stored(i32, i32) -> i32, i8 {
+ ss0 = explicit_slot 8
+block0(v0: i32, v1: i32):
+ v2 = stack_addr.i64 ss0
+ store v1, v2
+ v3 = load.i32 v2
+ v4, v5 = smul_overflow v0, v3
+ v6 = stack_addr.i64 ss0+4
+ store v4, v6
+ v7 = iconst.i8 1
+ v8 = iconst.i8 0
+ v9 = select v5, v7, v8
+ v10 = load.i32 v6
+ return v10, v9
+}
+
+; run: %smulof_flags_select_load_product_stored(2, 3) == [6, 0]
+; run: %smulof_flags_select_load_product_stored(-2, 3) == [-6, 0]
+; run: %smulof_flags_select_load_product_stored(0x7FFFFFFF, 2) == [-2, 1]
+; run: %smulof_flags_select_load_product_stored(0x80000000, -1) == [0x80000000, 1]
+
+function %smulof_flags_select_load_product_used_between(i32, i32) -> i64 {
+ ss0 = explicit_slot 4
+block0(v0: i32, v1: i32):
+ v2 = stack_addr.i64 ss0
+ store v1, v2
+ v3 = load.i32 v2
+ v4, v5 = smul_overflow v0, v3
+ v6 = uextend.i64 v4
+ v7 = iconst.i64 -1
+ v8 = select v5, v7, v6
+ return v8
+}
+
+; run: %smulof_flags_select_load_product_used_between(2, 3) == 6
+; run: %smulof_flags_select_load_product_used_between(-2, 3) == 0xFFFFFFFA
+; run: %smulof_flags_select_load_product_used_between(0x7FFFFFFF, 2) == -1