A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: add `umul_overflow` / `smul_overflow` flags folding cases. (#14254)
Chris Fallind445cd7e071e
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex 17b5bafde9..f244f696b8 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -4957,6 +4957,108 @@(produces_flags_opportunistic_def producer sum_value)(Cond.Hs))))+;; Use of the flags output of `umul_overflow`/`smul_overflow` can produce a+;; `Cond` and give an opportunistic def of the other output. The multiply does+;; not set the condition flags; the overflow is detected by comparing the+;; product against itself extended from the original type (or, for 64-bit+;; values, by comparing the high half of the product against zero). Hence the+;; `ProducesFlagsOpportunisticDef2` chains for the narrow cases. For 64-bit+;; multiplies, the low product and overflow flag are lowered independently,+;; since the lowerings have no overlap/shared computation.++;; madd out, a_uxt, b_uxt+;; cmp out, out, uxt{b,h}+(rule 5+ (is_nonzero+ (second_result umul @ (umul_overflow (fits_in_16 ty) a b)))+ (if-let (first_result prod_value) umul)+ (let ((a_uext Reg (put_in_reg_zext32 a))+ (b_uext Reg (put_in_reg_zext32 b))+ (dst WritableReg (temp_writable_reg $I64))+ (producer ProducesFlags+ (produces_flags_opportunistic_def2+ (MInst.AluRRRR (ALUOp3.MAdd) (operand_size ty) dst a_uext b_uext (zero_reg))+ dst+ prod_value+ (MInst.AluRRRExtend (ALUOp.SubS) (OperandSize.Size32)+ (writable_zero_reg) dst dst+ (lower_extend_op ty (ArgumentExtension.Uext))))))+ (CondResult.Cond producer (Cond.Ne))))++;; umaddl out, a, b+;; cmp out, out, uxtw+(rule 6+ (is_nonzero+ (second_result umul @ (umul_overflow $I32 a b)))+ (if-let (first_result prod_value) umul)+ (let ((dst WritableReg (temp_writable_reg $I64))+ (producer ProducesFlags+ (produces_flags_opportunistic_def2+ (MInst.AluRRRR (ALUOp3.UMAddL) (operand_size $I32) dst a b (zero_reg))+ dst+ prod_value+ (MInst.AluRRRExtend (ALUOp.SubS) (OperandSize.Size64)+ (writable_zero_reg) dst dst+ (ExtendOp.UXTW)))))+ (CondResult.Cond producer (Cond.Ne))))++;; umulh tmp, a, b+;; cmp tmp, #0+(rule 7+ (is_nonzero+ (second_result umul @ (umul_overflow $I64 a b)))+ (let ((tmp Reg (umulh $I64 a b))+ (producer ProducesFlags (cmp64_imm tmp (u8_into_imm12 0))))+ (CondResult.Cond producer (Cond.Ne))))++;; madd out, a_sxt, b_sxt+;; cmp out, out, sxt{b,h}+(rule 8+ (is_nonzero+ (second_result smul @ (smul_overflow (fits_in_16 ty) a b)))+ (if-let (first_result prod_value) smul)+ (let ((a_sext Reg (put_in_reg_sext32 a))+ (b_sext Reg (put_in_reg_sext32 b))+ (dst WritableReg (temp_writable_reg $I64))+ (producer ProducesFlags+ (produces_flags_opportunistic_def2+ (MInst.AluRRRR (ALUOp3.MAdd) (operand_size ty) dst a_sext b_sext (zero_reg))+ dst+ prod_value+ (MInst.AluRRRExtend (ALUOp.SubS) (OperandSize.Size32)+ (writable_zero_reg) dst dst+ (lower_extend_op ty (ArgumentExtension.Sext))))))+ (CondResult.Cond producer (Cond.Ne))))++;; smaddl out, a, b+;; cmp out, out, sxtw+(rule 9+ (is_nonzero+ (second_result smul @ (smul_overflow $I32 a b)))+ (if-let (first_result prod_value) smul)+ (let ((dst WritableReg (temp_writable_reg $I64))+ (producer ProducesFlags+ (produces_flags_opportunistic_def2+ (MInst.AluRRRR (ALUOp3.SMAddL) (operand_size $I32) dst a b (zero_reg))+ dst+ prod_value+ (MInst.AluRRRExtend (ALUOp.SubS) (OperandSize.Size64)+ (writable_zero_reg) dst dst+ (ExtendOp.SXTW)))))+ (CondResult.Cond producer (Cond.Ne))))++;; smulh tmp, a, b+;; cmp tmp, out, asr #63+(rule 10+ (is_nonzero+ (second_result smul @ (smul_overflow $I64 a b)))+ (if-let (first_result prod_value) smul)+ (let ((prod Reg (put_in_reg prod_value))+ (tmp Reg (smulh $I64 a b))+ (producer ProducesFlags+ (cmp_rr_shift_asr (OperandSize.Size64) tmp prod 63)))+ (CondResult.Cond producer (Cond.Ne))))+(attr emit_icmp (veri chain))(decl emit_icmp (IntCC Value Value) CondResult)diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex c6a3e3679b..53abae090a 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -2970,8 +2970,7 @@;; bool (i.e., its only use(s) were folded into consumers such as a `brif` that;; consume the flags directly) does not need a `cset`; emit just the `adds`.(rule 3 (lower i @ (uadd_overflow (ty_32_or_64 ty) x y))- (if-let (second_result flags) i)- (if-let false (value_used flags))+ (if-let false (second_result_used i))(output_pair (add ty x y) (value_regs_invalid)));; For values smaller than a register, we do a normal `add` with both arguments@@ -3049,6 +3048,24 @@;;;; Rules for `umul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+;; The flags output of a `umul_overflow` that is not demanded as a+;; materialized bool (i.e., its only use(s) were folded into consumers such+;; as a `brif` that consume the flags directly) does not need the+;; `cmp`/`cset`; emit just the multiply.+(rule 5 (lower i @ (umul_overflow (fits_in_16 ty) a b))+ (if-let false (second_result_used i))+ (let ((a_uext Reg (put_in_reg_zext32 a))+ (b_uext Reg (put_in_reg_zext32 b)))+ (output_pair (madd ty a_uext b_uext (zero_reg)) (value_regs_invalid))))++(rule 4 (lower i @ (umul_overflow $I32 a b))+ (if-let false (second_result_used i))+ (output_pair (umaddl a b (zero_reg)) (value_regs_invalid)))++(rule 3 (lower i @ (umul_overflow $I64 a b))+ (if-let false (second_result_used i))+ (output_pair (madd $I64 a b (zero_reg)) (value_regs_invalid)))+;; uxt{b,h} a_ext, a;; uxt{b,h} b_ext, b;; mul out, a_ext, b_ext@@ -3097,6 +3114,22 @@;;;; Rules for `smul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+;; As for `umul_overflow` above: when the flags output is only consumed+;; folded (e.g., by a `brif`), emit just the multiply.+(rule 5 (lower i @ (smul_overflow (fits_in_16 ty) a b))+ (if-let false (second_result_used i))+ (let ((a_sext Reg (put_in_reg_sext32 a))+ (b_sext Reg (put_in_reg_sext32 b)))+ (output_pair (madd ty a_sext b_sext (zero_reg)) (value_regs_invalid))))++(rule 4 (lower i @ (smul_overflow $I32 a b))+ (if-let false (second_result_used i))+ (output_pair (smaddl a b (zero_reg)) (value_regs_invalid)))++(rule 3 (lower i @ (smul_overflow $I64 a b))+ (if-let false (second_result_used i))+ (output_pair (madd $I64 a b (zero_reg)) (value_regs_invalid)))+;; sxt{b,h} a_ext, a;; sxt{b,h} b_ext, b;; mul out, a_ext, b_extdiff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isleindex be4b4d6464..d1744469c8 100644--- a/cranelift/codegen/src/isa/x64/inst.isle+++ b/cranelift/codegen/src/isa/x64/inst.isle@@ -3886,6 +3886,46 @@(produces_flags_opportunistic_def producer sum_value)(CC.B))))+;; Using the flags output of `umul_overflow`/`smul_overflow` can produce a+;; `CC` and give an opportunistic def of the other output. The multiply+;; itself sets the overflow flag (OF for `imul`; OF==CF for the unsigned+;; `mul` of narrow types), so no extra compare is needed.+(rule 4+ (is_nonzero+ (second_result umul @ (umul_overflow _ x y @ (value_type (ty_int_ref_16_to_64 ty)))))+ (if-let (first_result prod_value) umul)+ (let ((producer ProducesFlags (x64_mul_lo_with_flags_paired ty false x y)))+ (CondResult.CC+ (produces_flags_opportunistic_def producer prod_value)+ (CC.O))))++(rule 5+ (is_nonzero+ (second_result umul @ (umul_overflow _ x y @ (value_type $I8))))+ (if-let (first_result prod_value) umul)+ (let ((producer ProducesFlags (x64_mul8_with_flags_paired false x y)))+ (CondResult.CC+ (produces_flags_opportunistic_def producer prod_value)+ (CC.O))))++(rule 6+ (is_nonzero+ (second_result smul @ (smul_overflow _ x y @ (value_type (ty_int_ref_16_to_64 ty)))))+ (if-let (first_result prod_value) smul)+ (let ((producer ProducesFlags (x64_mul_lo_with_flags_paired ty true x y)))+ (CondResult.CC+ (produces_flags_opportunistic_def producer prod_value)+ (CC.O))))++(rule 7+ (is_nonzero+ (second_result smul @ (smul_overflow _ x y @ (value_type $I8))))+ (if-let (first_result prod_value) smul)+ (let ((producer ProducesFlags (x64_mul8_with_flags_paired true x y)))+ (CondResult.CC+ (produces_flags_opportunistic_def producer prod_value)+ (CC.O))))+;; Like `is_nonzero` but with additional specializations for compare;; operators. We break this out from `is_nonzero` because we want to;; avoid unbounded recursion.diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex cabc44e5a8..ed9b8fac38 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -219,6 +219,20 @@;;;; Rules for `umul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+;; The flags output of a `umul_overflow` that is not demanded as a+;; materialized bool (i.e., its only use(s) were folded into consumers such+;; as a `brif` that consume the flags directly) does not need a `seto`;+;; emit just the `mul`.+(rule 4 (lower i @ (umul_overflow _ x y @ (value_type (ty_int_ref_16_to_64 ty))))+ (if-let (second_result flags) i)+ (if-let false (value_used flags))+ (output_pair (x64_mul ty false x y) (value_regs_invalid)))++(rule 5 (lower i @ (umul_overflow _ x y @ (value_type $I8)))+ (if-let (second_result flags) i)+ (if-let false (value_used flags))+ (output_pair (value_reg (x64_mul8 false x y)) (value_regs_invalid)))+(rule 2 (lower (umul_overflow _ x y @ (value_type $I8)))(construct_overflow_op (CC.O) (x64_mul8_with_flags_paired false x y)))@@ -227,6 +241,18 @@;;;; Rules for `smul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+;; As for `umul_overflow` above: when the flags output is only consumed+;; folded (e.g., by a `brif`), emit just the `imul`.+(rule 4 (lower i @ (smul_overflow _ x y @ (value_type (ty_int_ref_16_to_64 ty))))+ (if-let (second_result flags) i)+ (if-let false (value_used flags))+ (output_pair (x64_mul ty true x y) (value_regs_invalid)))++(rule 5 (lower i @ (smul_overflow _ x y @ (value_type $I8)))+ (if-let (second_result flags) i)+ (if-let false (value_used flags))+ (output_pair (value_reg (x64_mul8 true x y)) (value_regs_invalid)))+(rule 2 (lower (smul_overflow _ x y @ (value_type $I8)))(construct_overflow_op (CC.O) (x64_mul8_with_flags_paired true x y)))diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rsindex c05127ff0d..0d4288bd6d 100644--- a/cranelift/codegen/src/machinst/isle.rs+++ b/cranelift/codegen/src/machinst/isle.rs@@ -213,6 +213,12 @@ macro_rules! isle_lower_prelude_methods {.copied()}+ #[inline]+ fn second_result_used(&mut self, inst: Inst) -> bool {+ let second_result = self.lower_ctx.dfg().inst_results(inst).get(1).copied();+ second_result.is_some_and(|value| self.lower_ctx.value_lowered_used(value))+ }+#[inline]fn inst_data_value(&mut self, inst: Inst) -> (Type, InstructionData) {let ty = match self.first_result(inst) {diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isleindex 1eadf632f9..46ce41d47a 100644--- a/cranelift/codegen/src/prelude_lower.isle+++ b/cranelift/codegen/src/prelude_lower.isle@@ -64,6 +64,7 @@(extern constructor writable_value_regs writable_value_regs);; Construct an empty `ValueRegs` containing only invalid register sentinels.+(spec (value_regs_invalid) (provide true))(decl value_regs_invalid () ValueRegs)(extern constructor value_regs_invalid value_regs_invalid)@@ -86,6 +87,16 @@(extern constructor output output);; Construct a two-element `InstOutput`.+(spec (output_pair first second)+ (provide+ ; `lower` is specified in terms of the instruction's first result.+ ; The second result is returned for code generation but is not the+ ; semantic result of the lowering rule.+ (= (widthof (:lo first)) (widthof (:hi first)))+ (if (<= (widthof result) 64)+ (= result (conv_to (widthof result) (:lo first)))+ (let ((cat (concat (:hi first) (:lo first))))+ (= result (conv_to (widthof result) cat))))))(decl output_pair (ValueRegs ValueRegs) InstOutput)(extern constructor output_pair output_pair)@@ -281,6 +292,7 @@;; Returns whether the given value still has uses to serve in the;; lowering scan. If false, production of the given value may be;; skipped by the lowered instruction(s).+(spec (value_used value) (provide true))(decl pure value_used (Value) bool)(extern constructor value_used value_used)@@ -289,9 +301,18 @@(extern extractor first_result first_result);; Extract the second result value of the given instruction.+(spec (second_result value) (provide (= result value)))(decl second_result (Value) Inst)(extern extractor second_result second_result)+;; Returns whether an instruction's second result still has uses to serve in+;; the lowering scan. Overflow operations use this while lowering their first+;; (non-boolean) result.+(spec (second_result_used inst)+ (provide (> (widthof inst) 1)))+(decl pure second_result_used (Inst) bool)+(extern constructor second_result_used second_result_used)+;; Extract the `InstructionData` for an `Inst`.(decl inst_data_value (Type InstructionData) Inst)(extern extractor infallible inst_data_value inst_data_value)@@ -559,7 +580,10 @@;; uses of `value` (e.g., as its own operands) after the;; producer is constructed, and we want these to use the;; opportunistic def if they can, as well.- (ProducesFlagsOpportunisticDef (inst MInst) (reg Reg) (value Value))))+ (ProducesFlagsOpportunisticDef (inst MInst) (reg Reg) (value Value))+ ;; Like `ProducesFlagsOpportunisticDef`, with an+ ;; additional instruction.+ (ProducesFlagsOpportunisticDef2 (inst1 MInst) (reg Reg) (value Value) (inst2 MInst))))(spec (ProducesFlags.ProducesFlagsReturnsResultWithConsumer inst reg)(provide@@ -572,6 +596,18 @@(= (:flags result) (:flags_out inst))(not (:has_result result))))+(spec (ProducesFlags.ProducesFlagsOpportunisticDef2 inst1 reg value inst2)+ (provide+ ; Flags output from the first instruction pass to the second's input.+ (= (:flags_out inst1) (:flags_in inst2))++ ; Result flags of the ProducesFlags wrapper are from the second+ ; instruction's output.+ (= (:flags result) (:flags_out inst2))++ ; No value is produced.+ (not (:has_result result))))+(spec (ProducesFlags.ProducesFlagsSideEffect inst)(provide(= (:flags result) (:flags_out inst))@@ -667,6 +703,15 @@(ProducesFlags.ProducesFlagsReturnsResultWithConsumer inst reg) value)(ProducesFlags.ProducesFlagsOpportunisticDef inst reg value))+;; Wrap a pair of instructions -- `inst1`, which computes `value` into `reg`+;; but does not produce the flags of interest, followed by `inst2`, which+;; produces the flags (consuming the (irrelevant) flags of `inst1`) -- so+;; that, when they are emitted (via `with_flags` or `with_flags_side_effect`),+;; an opportunistic def of `value` is registered in `reg`.+(decl produces_flags_opportunistic_def2 (MInst Reg Value MInst) ProducesFlags)+(rule (produces_flags_opportunistic_def2 inst1 reg value inst2)+ (ProducesFlags.ProducesFlagsOpportunisticDef2 inst1 reg value inst2))+;; Helper for combining two flags-consumer instructions that return a;; single Reg, giving a ConsumesFlags that returns both values in a;; ValueRegs.@@ -838,6 +883,50 @@(_ Unit (opportunistic_def producer_value (value_reg producer_result))))consumer_result))+;; A pair of flag-producing side-effect instructions, where the first one also+;; computes `producer_value` in `producer_result` and registers an+;; opportunistic def of it, paired with a consumer that returns a single+;; register. (See `ProducesFlagsOpportunisticDef2`.)+(rule with_flags_opportunistic_def2_consumer_reg+ (with_flags (ProducesFlags.ProducesFlagsOpportunisticDef2 producer_inst1 producer_result producer_value producer_inst2)+ (ConsumesFlags.ConsumesFlagsReturnsReg consumer_inst consumer_result))+ (let ((_x Unit (emit producer_inst1))+ (_y Unit (emit producer_inst2))+ (_z Unit (emit consumer_inst))+ (_ Unit (opportunistic_def producer_value (value_reg producer_result))))+ (value_reg consumer_result)))++(rule with_flags_opportunistic_def2_consumer_twice+ (with_flags (ProducesFlags.ProducesFlagsOpportunisticDef2 producer_inst1 producer_result producer_value producer_inst2)+ (ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs consumer_inst_1+ consumer_inst_2+ consumer_result))+ ;; We must emit these instructions in order as the creator of+ ;; the ConsumesFlags may be relying on dataflow dependencies+ ;; amongst them.+ (let ((_x Unit (emit producer_inst1))+ (_y Unit (emit producer_inst2))+ (_z Unit (emit consumer_inst_1))+ (_w Unit (emit consumer_inst_2))+ (_ Unit (opportunistic_def producer_value (value_reg producer_result))))+ consumer_result))++(rule with_flags_opportunistic_def2_consumer_four+ (with_flags (ProducesFlags.ProducesFlagsOpportunisticDef2 producer_inst1 producer_result producer_value producer_inst2)+ (ConsumesFlags.ConsumesFlagsFourTimesReturnsValueRegs consumer_inst_1+ consumer_inst_2+ consumer_inst_3+ consumer_inst_4+ consumer_result))+ (let ((_x Unit (emit producer_inst1))+ (_y Unit (emit producer_inst2))+ (_z Unit (emit consumer_inst_1))+ (_w Unit (emit consumer_inst_2))+ (_v Unit (emit consumer_inst_3))+ (_u Unit (emit consumer_inst_4))+ (_ Unit (opportunistic_def producer_value (value_reg producer_result))))+ consumer_result))+(rule with_flags_producer_twice_reg(with_flags (ProducesFlags.ProducesFlagsTwiceSideEffect producer_inst1 producer_inst2)(ConsumesFlags.ConsumesFlagsReturnsReg consumer_inst consumer_result))@@ -907,6 +996,24 @@(let ((_ Unit (opportunistic_def producer_value (value_reg producer_result))))(SideEffectNoResult.Inst3 producer_inst consumer_inst_1 consumer_inst_2)))+;; The first instruction does not produce the flags of interest, so it+;; can be emitted eagerly; the flags producer and the consumer must+;; stay adjacent, so they are returned as a side-effect pair/triple to+;; be emitted together.+(rule (with_flags_side_effect+ (ProducesFlags.ProducesFlagsOpportunisticDef2 producer_inst1 producer_result producer_value producer_inst2)+ (ConsumesFlags.ConsumesFlagsSideEffect consumer_inst))+ (let ((_ Unit (emit producer_inst1))+ (_ Unit (opportunistic_def producer_value (value_reg producer_result))))+ (SideEffectNoResult.Inst2 producer_inst2 consumer_inst)))++(rule (with_flags_side_effect+ (ProducesFlags.ProducesFlagsOpportunisticDef2 producer_inst1 producer_result producer_value producer_inst2)+ (ConsumesFlags.ConsumesFlagsSideEffect2 consumer_inst_1 consumer_inst_2))+ (let ((_ Unit (emit producer_inst1))+ (_ Unit (opportunistic_def producer_value (value_reg producer_result))))+ (SideEffectNoResult.Inst3 producer_inst2 consumer_inst_1 consumer_inst_2)))+(rule (with_flags_side_effect(ProducesFlags.AlreadyExistingFlags)(ConsumesFlags.ConsumesFlagsSideEffect c))diff --git a/cranelift/codegen/src/spec/inst_specs.isle b/cranelift/codegen/src/spec/inst_specs.isleindex 9730858fa6..7eb9469a8e 100644--- a/cranelift/codegen/src/spec/inst_specs.isle+++ b/cranelift/codegen/src/spec/inst_specs.isle@@ -90,6 +90,19 @@((args (named Type) (bv 32) (bv 32)) (ret (bv 32)))((args (named Type) (bv 64) (bv 64)) (ret (bv 64))))+;; Binary operations whose first result has the operands' width and whose+;; second result is a one-bit flag.+(form+ bv_binary_8_to_64_with_flag+ ((args (named Type) (bv 8) (bv 8)) (ret (bv 8)))+ ((args (named Type) (bv 8) (bv 8)) (ret (bv 1)))+ ((args (named Type) (bv 16) (bv 16)) (ret (bv 16)))+ ((args (named Type) (bv 16) (bv 16)) (ret (bv 1)))+ ((args (named Type) (bv 32) (bv 32)) (ret (bv 32)))+ ((args (named Type) (bv 32) (bv 32)) (ret (bv 1)))+ ((args (named Type) (bv 64) (bv 64)) (ret (bv 64)))+ ((args (named Type) (bv 64) (bv 64)) (ret (bv 1))))+;; Shifts and rotates take their amount from a second operand whose integer type;; is independent of the type being shifted, so every combination of operand and;; amount width is a valid instantiation. Enumerating the full cross product is@@ -162,6 +175,36 @@(= (:bits ty) (widthof result))))(instantiate imul bv_binary_8_to_64)+(spec (umul_overflow ty x y)+ (provide+ (= (:bits ty) (widthof x))+ (= (widthof x) (widthof y))+ (let+ ((double (concat x x))+ (wide_width (widthof double))+ (product (bvmul (zero_ext wide_width x) (zero_ext wide_width y)))+ (low (conv_to (widthof x) product))+ (overflow (not (= product (zero_ext wide_width low)))))+ (if (= (widthof result) 1)+ (= result (zero_ext (widthof result) (if overflow #b1 #b0)))+ (= result (conv_to (widthof result) low))))))+(instantiate umul_overflow bv_binary_8_to_64_with_flag)++(spec (smul_overflow ty x y)+ (provide+ (= (:bits ty) (widthof x))+ (= (widthof x) (widthof y))+ (let+ ((double (concat x x))+ (wide_width (widthof double))+ (product (bvmul (sign_ext wide_width x) (sign_ext wide_width y)))+ (low (conv_to (widthof x) product))+ (overflow (not (= product (sign_ext wide_width low)))))+ (if (= (widthof result) 1)+ (= result (zero_ext (widthof result) (if overflow #b1 #b0)))+ (= result (conv_to (widthof result) low))))))+(instantiate smul_overflow bv_binary_8_to_64_with_flag)+(spec (smulhi ty x y)(provide(= (:bits ty) (widthof result))diff --git a/cranelift/filetests/filetests/isa/aarch64/mul_overflow_flag_consumers.clif b/cranelift/filetests/filetests/isa/aarch64/mul_overflow_flag_consumers.clifnew file mode 100644index 0000000000..5df6e78f96--- /dev/null+++ b/cranelift/filetests/filetests/isa/aarch64/mul_overflow_flag_consumers.clif@@ -0,0 +1,212 @@+test compile precise-output+set opt_level=speed+target aarch64++function %umul_flags_trapz(i64, i64, i64) -> i64 {+block0(v0: i64, v1: i64, v2: i64):+ v3, v4 = umul_overflow v0, v1+ trapz v4, user1+ v5 = iadd v3, v2+ return v5+}++; VCode:+; block0:+; madd x6, x0, x1, xzr+; umulh x7, x0, x1+; subs xzr, x7, #0+; b.eq #trap=user1+; add x0, x6, x2+; ret+;+; Disassembled:+; block0: ; offset 0x0+; mul x6, x0, x1+; umulh x7, x0, x1+; cmp x7, #0+; b.eq #0x18+; add x0, x6, x2+; ret+; udf #0xc11f ; trap: user1++function %umul_flags_trapz_i32(i32, i32, i32) -> i32 {+block0(v0: i32, v1: i32, v2: i32):+ v3, v4 = umul_overflow v0, v1+ trapz v4, user1+ v5 = iadd v3, v2+ return v5+}++; VCode:+; block0:+; umaddl x5, w0, w1, xzr+; subs xzr, x5, x5, UXTW+; b.eq #trap=user1+; add w0, w5, w2+; ret+;+; Disassembled:+; block0: ; offset 0x0+; umull x5, w0, w1+; cmp x5, w5, uxtw+; b.eq #0x14+; add w0, w5, w2+; ret+; udf #0xc11f ; trap: user1++function %smul_flags_trapnz(i64, i64, i64) -> i64 {+block0(v0: i64, v1: i64, v2: i64):+ v3, v4 = smul_overflow v0, v1+ trapnz v4, user1+ v5 = iadd v3, v2+ return v5+}++; VCode:+; block0:+; madd x6, x0, x1, xzr+; smulh x7, x0, x1+; subs xzr, x7, x6, ASR 63+; b.ne #trap=user1+; add x0, x6, x2+; ret+;+; Disassembled:+; block0: ; offset 0x0+; mul x6, x0, x1+; smulh x7, x0, x1+; cmp x7, x6, asr #63+; b.ne #0x18+; add x0, x6, x2+; ret+; udf #0xc11f ; trap: user1++function %smul_flags_select(i64, i64, i64) -> i64 {+block0(v0: i64, v1: i64, v2: i64):+ v3, v4 = smul_overflow v0, v1+ v5 = select v4, v3, v2+ return v5+}++; VCode:+; block0:+; madd x6, x0, x1, xzr+; smulh x5, x0, x1+; subs xzr, x5, x6, ASR 63+; csel x0, x6, x2, ne+; ret+;+; Disassembled:+; block0: ; offset 0x0+; mul x6, x0, x1+; smulh x5, x0, x1+; cmp x5, x6, asr #63+; csel x0, x6, x2, ne+; ret++function %umul_flags_select_spectre_guard(i64, i64, i64) -> i64 {+block0(v0: i64, v1: i64, v2: i64):+ v3, v4 = umul_overflow v0, v1+ v5 = select_spectre_guard v4, v3, v2+ return v5+}++; VCode:+; block0:+; madd x6, x0, x1, xzr+; umulh x5, x0, x1+; subs xzr, x5, #0+; csel x0, x6, x2, ne+; ret+;+; Disassembled:+; block0: ; offset 0x0+; mul x6, x0, x1+; umulh x5, x0, x1+; cmp x5, #0+; csel x0, x6, x2, ne+; ret++function %umul_flags_brif(i32, i32, i32) -> i32 {+block0(v0: i32, v1: i32, v2: i32):+ v4, v5 = umul_overflow v0, v1+ brif v5, block1, block2++block1:+ v6 = iadd v4, v4+ return v6++block2:+ return v2+}++; VCode:+; block0:+; umaddl x5, w0, w1, xzr+; subs xzr, x5, x5, UXTW+; b.ne label2 ; b label1+; block1:+; mov x0, x2+; ret+; block2:+; add w0, w5, w5+; ret+;+; Disassembled:+; block0: ; offset 0x0+; umull x5, w0, w1+; cmp x5, w5, uxtw+; b.ne #0x14+; block1: ; offset 0xc+; mov x0, x2+; ret+; block2: ; offset 0x14+; add w0, w5, w5+; ret++function %smul_flags_unused_i16(i16, i16) -> i16 {+block0(v0: i16, v1: i16):+ v2, v3 = smul_overflow v0, v1+ return v2+}++; VCode:+; block0:+; sxth w3, w0+; sxth w5, w1+; madd w0, w3, w5, wzr+; ret+;+; Disassembled:+; block0: ; offset 0x0+; sxth w3, w0+; sxth w5, w1+; mul w0, w3, w5+; ret++function %smul_flags_trapnz_i16(i16, i16) {+block0(v0: i16, v1: i16):+ v2, v3 = smul_overflow v0, v1+ trapnz v3, user1+ return+}++; VCode:+; block0:+; sxth w3, w0+; sxth w5, w1+; madd w7, w3, w5, wzr+; subs wzr, w7, w7, SXTH+; b.ne #trap=user1+; ret+;+; Disassembled:+; block0: ; offset 0x0+; sxth w3, w0+; sxth w5, w1+; mul w7, w3, w5+; cmp w7, w7, sxth+; b.ne #0x18+; ret+; udf #0xc11f ; trap: user1+diff --git a/cranelift/filetests/filetests/isa/x64/mul_overflow_flag_consumers.clif b/cranelift/filetests/filetests/isa/x64/mul_overflow_flag_consumers.clifnew file mode 100644index 0000000000..d3b666994c--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/mul_overflow_flag_consumers.clif@@ -0,0 +1,273 @@+test compile precise-output+set opt_level=speed+target x86_64++function %umul_flags_trapz(i64, i64, i64) -> i64 {+block0(v0: i64, v1: i64, v2: i64):+ v3, v4 = umul_overflow v0, v1+ trapz v4, user1+ v5 = iadd v3, v2+ return v5+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movq %rdi, %rax+; movq %rdx, %rdi+; mulq %rsi ;; implicit: %rax, %rdx+; jno #trap=user1+; movq %rdi, %rdx+; leaq (%rax, %rdx), %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movq %rdi, %rax+; movq %rdx, %rdi+; mulq %rsi+; jno 0x1e+; movq %rdi, %rdx+; addq %rdx, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+; ud2 ; trap: user1++function %umul_flags_trapz_i32(i32, i32, i32) -> i32 {+block0(v0: i32, v1: i32, v2: i32):+ v3, v4 = umul_overflow v0, v1+ trapz v4, user1+ v5 = iadd v3, v2+ return v5+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movq %rdi, %rax+; movq %rdx, %rdi+; mull %esi ;; implicit: %eax, %edx+; jno #trap=user1+; movq %rdi, %rdx+; leal (%rax, %rdx), %eax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movq %rdi, %rax+; movq %rdx, %rdi+; mull %esi+; jno 0x1c+; movq %rdi, %rdx+; addl %edx, %eax+; movq %rbp, %rsp+; popq %rbp+; retq+; ud2 ; trap: user1++function %smul_flags_trapnz(i64, i64, i64) -> i64 {+block0(v0: i64, v1: i64, v2: i64):+ v3, v4 = smul_overflow v0, v1+ trapnz v4, user1+ v5 = iadd v3, v2+ return v5+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movq %rdi, %rax+; movq %rdx, %rdi+; imulq %rsi ;; implicit: %rax, %rdx+; jo #trap=user1+; movq %rdi, %rdx+; leaq (%rax, %rdx), %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movq %rdi, %rax+; movq %rdx, %rdi+; imulq %rsi+; jo 0x1e+; movq %rdi, %rdx+; addq %rdx, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+; ud2 ; trap: user1++function %smul_flags_select(i64, i64, i64) -> i64 {+block0(v0: i64, v1: i64, v2: i64):+ v3, v4 = smul_overflow v0, v1+ v5 = select v4, v3, v2+ return v5+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movq %rdi, %rax+; movq %rdx, %r8+; imulq %rsi ;; implicit: %rax, %rdx+; movq %r8, %rdx+; movq %rax, %r8+; movq %rdx, %rax+; cmovoq %r8, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movq %rdi, %rax+; movq %rdx, %r8+; imulq %rsi+; movq %r8, %rdx+; movq %rax, %r8+; movq %rdx, %rax+; cmovoq %r8, %rax+; movq %rbp, %rsp+; popq %rbp+; retq++function %umul_flags_select_spectre_guard(i64, i64, i64) -> i64 {+block0(v0: i64, v1: i64, v2: i64):+ v3, v4 = umul_overflow v0, v1+ v5 = select_spectre_guard v4, v3, v2+ return v5+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movq %rdi, %rax+; movq %rdx, %r8+; mulq %rsi ;; implicit: %rax, %rdx+; movq %r8, %rdx+; movq %rax, %r8+; movq %rdx, %rax+; cmovoq %r8, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movq %rdi, %rax+; movq %rdx, %r8+; mulq %rsi+; movq %r8, %rdx+; movq %rax, %r8+; movq %rdx, %rax+; cmovoq %r8, %rax+; movq %rbp, %rsp+; popq %rbp+; retq++function %umul_flags_brif(i32, i32, i32) -> i32 {+block0(v0: i32, v1: i32, v2: i32):+ v4, v5 = umul_overflow v0, v1+ brif v5, block1, block2++block1:+ v6 = iadd v4, v4+ return v6++block2:+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movq %rdi, %rax+; movq %rdx, %r8+; mull %esi ;; implicit: %eax, %edx+; jo label2; j label1+; block1:+; movq %r8, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+; block2:+; leal (%rax, %rax), %eax+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movq %rdi, %rax+; movq %rdx, %r8+; mull %esi+; jo 0x1a+; block2: ; offset 0x12+; movq %r8, %rax+; movq %rbp, %rsp+; popq %rbp+; retq+; block3: ; offset 0x1a+; addl %eax, %eax+; movq %rbp, %rsp+; popq %rbp+; retq++function %smul_flags_unused_i16(i16, i16) -> i16 {+block0(v0: i16, v1: i16):+ v2, v3 = smul_overflow v0, v1+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movq %rdi, %rax+; imulw %si ;; implicit: %ax, %dx+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movq %rdi, %rax+; imulw %si+; movq %rbp, %rsp+; popq %rbp+; retq+diff --git a/cranelift/filetests/filetests/runtests/mul_overflow_flag_consumers.clif b/cranelift/filetests/filetests/runtests/mul_overflow_flag_consumers.clifnew file mode 100644index 0000000000..813caa779d--- /dev/null+++ b/cranelift/filetests/filetests/runtests/mul_overflow_flag_consumers.clif@@ -0,0 +1,215 @@+test run+test interpret+target x86_64+target aarch64++;; These tests execute the folded lowerings of `umul_overflow`/`smul_overflow`+;; where the flags output is consumed by a `select` or `brif` instead of being+;; materialized as a bool.++function %umulof_flags_select_i64(i64, i64) -> i8 {+block0(v0: i64, v1: i64):+ v2, v3 = umul_overflow v0, v1+ v4 = iconst.i8 1+ v5 = iconst.i8 0+ v6 = select v3, v4, v5+ return v6+}++; run: %umulof_flags_select_i64(0, 1) == 0+; run: %umulof_flags_select_i64(1, 1) == 0+; run: %umulof_flags_select_i64(-1, -1) == 1+; run: %umulof_flags_select_i64(-1, 2) == 1+; run: %umulof_flags_select_i64(0x7FFFFFFF_FFFFFFFF, 0x7FFFFFFF_FFFFFFFF) == 1+; run: %umulof_flags_select_i64(0x80000000_00000000, 0x7FFFFFFF_FFFFFFFF) == 1++function %umulof_flags_select_i32(i32, i32) -> i8 {+block0(v0: i32, v1: i32):+ v2, v3 = umul_overflow v0, v1+ v4 = iconst.i8 1+ v5 = iconst.i8 0+ v6 = select v3, v4, v5+ return v6+}++; run: %umulof_flags_select_i32(0, 1) == 0+; run: %umulof_flags_select_i32(1, 1) == 0+; run: %umulof_flags_select_i32(-1, -1) == 1+; run: %umulof_flags_select_i32(-1, 2) == 1+; run: %umulof_flags_select_i32(0x7FFFFFFF, 0x7FFFFFFF) == 1+; run: %umulof_flags_select_i32(0x80000000, 0x7FFFFFFF) == 1++function %umulof_flags_select_i16(i16, i16) -> i8 {+block0(v0: i16, v1: i16):+ v2, v3 = umul_overflow v0, v1+ v4 = iconst.i8 1+ v5 = iconst.i8 0+ v6 = select v3, v4, v5+ return v6+}++; run: %umulof_flags_select_i16(0, 1) == 0+; run: %umulof_flags_select_i16(1, 1) == 0+; run: %umulof_flags_select_i16(-1, -1) == 1+; run: %umulof_flags_select_i16(-1, 2) == 1+; run: %umulof_flags_select_i16(0x7FFF, 0x7FFF) == 1+; run: %umulof_flags_select_i16(0x8000, 0x7FFF) == 1++function %umulof_flags_select_i8(i8, i8) -> i8 {+block0(v0: i8, v1: i8):+ v2, v3 = umul_overflow v0, v1+ v4 = iconst.i8 1+ v5 = iconst.i8 0+ v6 = select v3, v4, v5+ return v6+}++; run: %umulof_flags_select_i8(0, 1) == 0+; run: %umulof_flags_select_i8(1, 1) == 0+; run: %umulof_flags_select_i8(-1, -1) == 1+; run: %umulof_flags_select_i8(-1, 2) == 1+; run: %umulof_flags_select_i8(0x7F, 0x7F) == 1+; run: %umulof_flags_select_i8(0x80, 0x7F) == 1++function %smulof_flags_select_i64(i64, i64) -> i8 {+block0(v0: i64, v1: i64):+ v2, v3 = smul_overflow v0, v1+ v4 = iconst.i8 1+ v5 = iconst.i8 0+ v6 = select v3, v4, v5+ return v6+}++; run: %smulof_flags_select_i64(0, 1) == 0+; run: %smulof_flags_select_i64(1, 1) == 0+; run: %smulof_flags_select_i64(-1, -1) == 0+; run: %smulof_flags_select_i64(-2, -2) == 0+; run: %smulof_flags_select_i64(0x7FFFFFFF_FFFFFFFF, 2) == 1+; run: %smulof_flags_select_i64(0x80000000_00000000, -1) == 1+; run: %smulof_flags_select_i64(0x7FFFFFFF_FFFFFFFF, 0x7FFFFFFF_FFFFFFFF) == 1++function %smulof_flags_select_i32(i32, i32) -> i8 {+block0(v0: i32, v1: i32):+ v2, v3 = smul_overflow v0, v1+ v4 = iconst.i8 1+ v5 = iconst.i8 0+ v6 = select v3, v4, v5+ return v6+}++; run: %smulof_flags_select_i32(0, 1) == 0+; run: %smulof_flags_select_i32(1, 1) == 0+; run: %smulof_flags_select_i32(-1, -1) == 0+; run: %smulof_flags_select_i32(-2, -2) == 0+; run: %smulof_flags_select_i32(0x7FFFFFFF, 2) == 1+; run: %smulof_flags_select_i32(0x80000000, -1) == 1+; run: %smulof_flags_select_i32(0x7FFFFFFF, 0x7FFFFFFF) == 1++function %smulof_flags_select_i16(i16, i16) -> i8 {+block0(v0: i16, v1: i16):+ v2, v3 = smul_overflow v0, v1+ v4 = iconst.i8 1+ v5 = iconst.i8 0+ v6 = select v3, v4, v5+ return v6+}++; run: %smulof_flags_select_i16(0, 1) == 0+; run: %smulof_flags_select_i16(1, 1) == 0+; run: %smulof_flags_select_i16(-1, -1) == 0+; run: %smulof_flags_select_i16(-2, -2) == 0+; run: %smulof_flags_select_i16(0x7FFF, 2) == 1+; run: %smulof_flags_select_i16(0x8000, -1) == 1+; run: %smulof_flags_select_i16(0x7FFF, 0x7FFF) == 1++function %smulof_flags_select_i8(i8, i8) -> i8 {+block0(v0: i8, v1: i8):+ v2, v3 = smul_overflow v0, v1+ v4 = iconst.i8 1+ v5 = iconst.i8 0+ v6 = select v3, v4, v5+ return v6+}++; run: %smulof_flags_select_i8(0, 1) == 0+; run: %smulof_flags_select_i8(1, 1) == 0+; run: %smulof_flags_select_i8(-1, -1) == 0+; run: %smulof_flags_select_i8(-2, -2) == 0+; run: %smulof_flags_select_i8(0x7F, 2) == 1+; run: %smulof_flags_select_i8(0x80, -1) == 1+; run: %smulof_flags_select_i8(0x7F, 0x7F) == 1++;; `brif` on the flags output. The `(0x7FFFFFFF..., -1)` cases do not overflow+;; (result is -0x7FFFFFFF...), but the high half of the 128-bit product is+;; nonzero, i.e., CF is set while OF is clear; these discriminate the overflow+;; flag from the carry flag.++function %umulof_flags_brif_i64(i64, i64) -> i8 {+block0(v0: i64, v1: i64):+ v2, v3 = umul_overflow v0, v1+ brif v3, block1, block2+block1:+ v4 = iconst.i8 1+ return v4+block2:+ v5 = iconst.i8 0+ return v5+}++; run: %umulof_flags_brif_i64(0, 1) == 0+; run: %umulof_flags_brif_i64(-1, -1) == 1+; run: %umulof_flags_brif_i64(-1, 2) == 1+; run: %umulof_flags_brif_i64(0x7FFFFFFF_FFFFFFFF, 0x7FFFFFFF_FFFFFFFF) == 1++function %umulof_flags_brif_i32(i32, i32) -> i8 {+block0(v0: i32, v1: i32):+ v2, v3 = umul_overflow v0, v1+ brif v3, block1, block2+block1:+ v4 = iconst.i8 1+ return v4+block2:+ v5 = iconst.i8 0+ return v5+}++; run: %umulof_flags_brif_i32(0, 1) == 0+; run: %umulof_flags_brif_i32(-1, -1) == 1+; run: %umulof_flags_brif_i32(-1, 2) == 1+; run: %umulof_flags_brif_i32(0x7FFFFFFF, 0x7FFFFFFF) == 1++function %smulof_flags_brif_i64(i64, i64) -> i8 {+block0(v0: i64, v1: i64):+ v2, v3 = smul_overflow v0, v1+ brif v3, block1, block2+block1:+ v4 = iconst.i8 1+ return v4+block2:+ v5 = iconst.i8 0+ return v5+}++; run: %smulof_flags_brif_i64(0, 1) == 0+; run: %smulof_flags_brif_i64(-1, -1) == 0+; run: %smulof_flags_brif_i64(0x7FFFFFFF_FFFFFFFF, 2) == 1+; run: %smulof_flags_brif_i64(0x80000000_00000000, -1) == 1+; run: %smulof_flags_brif_i64(0x7FFFFFFF_FFFFFFFF, -1) == 0++function %smulof_flags_brif_i32(i32, i32) -> i8 {+block0(v0: i32, v1: i32):+ v2, v3 = smul_overflow v0, v1+ brif v3, block1, block2+block1:+ v4 = iconst.i8 1+ return v4+block2:+ v5 = iconst.i8 0+ return v5+}++; run: %smulof_flags_brif_i32(0, 1) == 0+; run: %smulof_flags_brif_i32(-1, -1) == 0+; run: %smulof_flags_brif_i32(0x7FFFFFFF, 2) == 1+; run: %smulof_flags_brif_i32(0x80000000, -1) == 1+; run: %smulof_flags_brif_i32(0x7FFFFFFF, -1) == 0