A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: ISLE: add type parameters to most clif terms in lower (#12717)
Alexa VanHattumbc0ed8270014
diff --git a/cranelift/codegen/meta/src/gen_isle.rs b/cranelift/codegen/meta/src/gen_isle.rsindex 52bc7826df..e47a92ab4e 100644--- a/cranelift/codegen/meta/src/gen_isle.rs+++ b/cranelift/codegen/meta/src/gen_isle.rs@@ -230,6 +230,9 @@ fn gen_common_isle((IsleTarget::Opt, _, 1, false) => ("Value", true, "make_inst", "inst_data_value"),(IsleTarget::Opt, _, _, _) => ("Inst", false, "make_skeleton_inst", "inst_data"),+ (IsleTarget::Lower, false, r, _) if r >= 1 => {+ ("Inst", true, "make_inst", "inst_data_value")+ }(IsleTarget::Lower, _, _, _) => ("Inst", false, "make_inst", "inst_data_value"),};@@ -268,7 +271,13 @@ fn gen_common_isle(let mut s = format!("({inst_data_etor} {}(InstructionData.{} (Opcode.{})",- if ty_in_decl { "ty " } else { "" },+ if ty_in_decl {+ "ty "+ } else if isle_target == IsleTarget::Lower {+ "_ "+ } else {+ ""+ },inst.format.name,inst.camel_name);diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex 88b6d5a158..c4031ccef0 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -1956,7 +1956,7 @@(decl imm12_from_value (Imm12) Value)(extractor(imm12_from_value n)- (iconst (u64_from_imm64 (imm12_from_u64 n))))+ (iconst _ (u64_from_imm64 (imm12_from_u64 n))));; Conceptually the same as `imm12_from_value`, but tries negating the constant;; value (first sign-extending to handle narrow widths).(spec (imm12_from_negated_value arg)@@ -1980,7 +1980,7 @@)(decl pure partial imm12_from_negated_value (Value) Imm12)(rule imm12_from_negated_value- (imm12_from_negated_value (has_type ty (iconst n)))+ (imm12_from_negated_value (has_type ty (iconst _ n)))(if-let (imm12_from_u64 imm) (i64_cast_unsigned (i64_checked_neg (i64_sextend_imm64 ty n))))imm)@@ -3755,7 +3755,7 @@(MInst.TrapIf (cond_br_cond (Cond.Hs)) tc))))(decl sink_atomic_load (Inst) Reg)-(rule (sink_atomic_load x @ (atomic_load _ addr))+(rule (sink_atomic_load x @ (atomic_load _ _ addr))(let ((_ Unit (sink_inst x)))(put_in_reg addr)))@@ -3777,18 +3777,18 @@(alu_rrr op ty x y));; Special cases for when one operand is a constant.-(rule (alu_rs_imm_logic_commutative op ty x (iconst k))+(rule (alu_rs_imm_logic_commutative op ty x (iconst _ k))(if-let imm (imm_logic_from_imm64 ty k))(alu_rr_imm_logic op ty x imm))-(rule 1 (alu_rs_imm_logic_commutative op ty (iconst k) x)+(rule 1 (alu_rs_imm_logic_commutative op ty (iconst _ k) x)(if-let imm (imm_logic_from_imm64 ty k))(alu_rr_imm_logic op ty x imm));; Special cases for when one operand is shifted left by a constant.-(rule (alu_rs_imm_logic_commutative op ty x (ishl y (iconst k)))+(rule (alu_rs_imm_logic_commutative op ty x (ishl _ y (iconst _ k)))(if-let amt (lshl_from_imm64 ty k))(alu_rrr_shift op ty x y amt))-(rule 1 (alu_rs_imm_logic_commutative op ty (ishl x (iconst k)) y)+(rule 1 (alu_rs_imm_logic_commutative op ty (ishl _ x (iconst _ k)) y)(if-let amt (lshl_from_imm64 ty k))(alu_rrr_shift op ty y x amt))@@ -3805,10 +3805,10 @@(decl alu_rs_imm_logic (ALUOp Type Value Value) Reg)(rule -1 (alu_rs_imm_logic op ty x y)(alu_rrr op ty x y))-(rule (alu_rs_imm_logic op ty x (iconst k))+(rule (alu_rs_imm_logic op ty x (iconst _ k))(if-let imm (imm_logic_from_imm64 ty k))(alu_rr_imm_logic op ty x imm))-(rule (alu_rs_imm_logic op ty x (ishl y (iconst k)))+(rule (alu_rs_imm_logic op ty x (ishl _ y (iconst _ k)))(if-let amt (lshl_from_imm64 ty k))(alu_rrr_shift op ty x y amt))@@ -3905,15 +3905,15 @@(rule 0 (amode ty val offset)(amode_no_more_iconst ty val offset))-(rule 1 (amode ty (iadd x (i32_from_iconst y)) offset)+(rule 1 (amode ty (iadd _ x (i32_from_iconst y)) offset)(if-let new_offset (i32_checked_add y offset))(amode_no_more_iconst ty x new_offset))-(rule 2 (amode ty (iadd (i32_from_iconst x) y) offset)+(rule 2 (amode ty (iadd _ (i32_from_iconst x) y) offset)(if-let new_offset (i32_checked_add x offset))(amode_no_more_iconst ty y new_offset))(rule 3- (amode ty (stack_addr slot offset1) offset2)+ (amode ty (stack_addr _ slot offset1) offset2)(AMode.SlotOffset(abi_stackslot_offset_into_slot_region slot offset1 offset2)))@@ -3939,15 +3939,15 @@;; instructions. Constants on the other hand added to the amode represent only;; a single instruction folded in, so fewer instructions should be generated;; with these higher priority than the rules above.-(rule 3 (amode_no_more_iconst ty (iadd x y) offset)+(rule 3 (amode_no_more_iconst ty (iadd _ x y) offset)(AMode.RegReg (amode_add x offset) y))-(rule 4 (amode_no_more_iconst ty (iadd x (uextend y @ (value_type $I32))) offset)+(rule 4 (amode_no_more_iconst ty (iadd _ x (uextend _ y @ (value_type $I32))) offset)(AMode.RegExtended (amode_add x offset) y (ExtendOp.UXTW)))-(rule 4 (amode_no_more_iconst ty (iadd x (sextend y @ (value_type $I32))) offset)+(rule 4 (amode_no_more_iconst ty (iadd _ x (sextend _ y @ (value_type $I32))) offset)(AMode.RegExtended (amode_add x offset) y (ExtendOp.SXTW)))-(rule 5 (amode_no_more_iconst ty (iadd (uextend x @ (value_type $I32)) y) offset)+(rule 5 (amode_no_more_iconst ty (iadd _ (uextend _ x @ (value_type $I32)) y) offset)(AMode.RegExtended (amode_add y offset) x (ExtendOp.UXTW)))-(rule 5 (amode_no_more_iconst ty (iadd (sextend x @ (value_type $I32)) y) offset)+(rule 5 (amode_no_more_iconst ty (iadd _ (sextend _ x @ (value_type $I32)) y) offset)(AMode.RegExtended (amode_add y offset) x (ExtendOp.SXTW)));; `RegScaled*` rules where this matches an addition of an "index register" to a@@ -3957,19 +3957,19 @@;; Note that this can additionally bundle an extending operation but the;; extension must happen before the shift. This will pattern-match the shift;; first and then if that succeeds afterwards try to find an extend.-(rule 6 (amode_no_more_iconst ty (iadd x (ishl y (iconst (u64_from_imm64 n)))) offset)+(rule 6 (amode_no_more_iconst ty (iadd _ x (ishl _ y (iconst _ (u64_from_imm64 n)))) offset)(if-let true (u64_eq (ty_bytes ty) (u64_wrapping_shl 1 (shift_masked_imm ty n))))(amode_reg_scaled (amode_add x offset) y))-(rule 7 (amode_no_more_iconst ty (iadd (ishl y (iconst (u64_from_imm64 n))) x) offset)+(rule 7 (amode_no_more_iconst ty (iadd _ (ishl _ y (iconst _ (u64_from_imm64 n))) x) offset)(if-let true (u64_eq (ty_bytes ty) (u64_wrapping_shl 1 (shift_masked_imm ty n))))(amode_reg_scaled (amode_add x offset) y))(decl amode_reg_scaled (Reg Value) AMode)(rule 0 (amode_reg_scaled base index)(AMode.RegScaled base index))-(rule 1 (amode_reg_scaled base (uextend index @ (value_type $I32)))+(rule 1 (amode_reg_scaled base (uextend _ index @ (value_type $I32)))(AMode.RegScaledExtended base index (ExtendOp.UXTW)))-(rule 1 (amode_reg_scaled base (sextend index @ (value_type $I32)))+(rule 1 (amode_reg_scaled base (sextend _ index @ (value_type $I32)))(AMode.RegScaledExtended base index (ExtendOp.SXTW)));; Helper to add a 32-bit signed immediate to the register provided. This will@@ -4007,7 +4007,7 @@(decl sink_load_into_addr (Type Inst) Reg)-(rule (sink_load_into_addr ty x @ (load _ addr (offset32 offset)))+(rule (sink_load_into_addr ty x @ (load _ _ addr (offset32 offset)))(let ((_ Unit (sink_inst x)))(add_imm_to_addr addr (i64_cast_unsigned offset))))diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex dbd8b27fa0..3c2702e3e7 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -18,27 +18,27 @@;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule iconst (lower (has_type ty (iconst (u64_from_imm64 n))))+(rule iconst (lower (has_type ty (iconst _ (u64_from_imm64 n))))(imm ty (ImmExtend.Zero) n));;;; Rules for `f16const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f16const (u16_from_ieee16 n)))+(rule (lower (f16const _ (u16_from_ieee16 n)))(constant_f16 n));;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f32const (u32_from_ieee32 n)))+(rule (lower (f32const _ (u32_from_ieee32 n)))(constant_f32 n));;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f64const (u64_from_ieee64 n)))+(rule (lower (f64const _ (u64_from_ieee64 n)))(constant_f64 n));;;; Rules for `f128const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F128 (f128const (u128_from_constant n))))+(rule (lower (has_type $F128 (f128const _ (u128_from_constant n))))(constant_f128 n));;;; Rules for `nop` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -51,64 +51,64 @@;; `i64` and smaller;; Base case, simply adding things in registers.-(rule iadd_base_case -1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd x y)))+(rule iadd_base_case -1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x y)))(add ty x y));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule iadd_imm12_right 4 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd x (imm12_from_value y))))+(rule iadd_imm12_right 4 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (imm12_from_value y))))(add_imm ty x y))-(rule iadd_imm12_left 5 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd (imm12_from_value x) y)))+(rule iadd_imm12_left 5 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (imm12_from_value x) y)))(add_imm ty y x));; Same as the previous special cases, except we can switch the addition to a;; subtraction if the negated immediate fits in 12 bits.-(rule iadd_imm12_neg_right 2 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd x y)))+(rule iadd_imm12_neg_right 2 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x y)))(if-let imm12_neg (imm12_from_negated_value y))(sub_imm ty x imm12_neg))-(rule iadd_imm12_neg_left 3 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd x y)))+(rule iadd_imm12_neg_left 3 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x y)))(if-let imm12_neg (imm12_from_negated_value x))(sub_imm ty y imm12_neg));; Special cases for when we're adding an extended register where the extending;; operation can get folded into the add itself.-(rule iadd_extend_right 0 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd x (extended_value_from_value y))))+(rule iadd_extend_right 0 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (extended_value_from_value y))))(add_extend ty x y))-(rule iadd_extend_left 1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd (extended_value_from_value x) y)))+(rule iadd_extend_left 1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (extended_value_from_value x) y)))(add_extend ty y x));; Special cases for when we're adding the shift of a different;; register by a constant amount and the shift can get folded into the add.(rule iadd_ishl_right 7 (lower (has_type (ty_int_ref_scalar_64 ty)- (iadd x (ishl y (iconst k)))))+ (iadd _ x (ishl _ y (iconst _ k)))))(if-let amt (lshl_from_imm64 ty k))(add_shift ty x y amt))(rule iadd_ishl_left 6 (lower (has_type (ty_int_ref_scalar_64 ty)- (iadd (ishl x (iconst k)) y)))+ (iadd _ (ishl _ x (iconst _ k)) y)))(if-let amt (lshl_from_imm64 ty k))(add_shift ty y x amt));; Fold an `iadd` and `imul` combination into a `madd` instruction.-(rule iadd_imul_right 7 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd x (imul y z))))+(rule iadd_imul_right 7 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (imul _ y z))))(madd ty y z x))-(rule iadd_imul_left 6 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd (imul x y) z)))+(rule iadd_imul_left 6 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (imul _ x y) z)))(madd ty x y z));; Fold an `isub` and `imul` combination into a `msub` instruction.-(rule isub_imul (lower (has_type (ty_int_ref_scalar_64 ty) (isub x (imul y z))))+(rule isub_imul (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x (imul _ y z))))(msub ty y z x));; vectors-(rule -2 (lower (has_type ty @ (multi_lane _ _) (iadd x y)))+(rule -2 (lower (has_type ty @ (multi_lane _ _) (iadd _ x y)))(add_vec x y (vector_size ty)));; `i128`-(rule -3 (lower (has_type $I128 (iadd x y)))+(rule -3 (lower (has_type $I128 (iadd _ x y)))(let;; Get the high/low registers for `x`.((x_regs ValueRegs x)@@ -131,13 +131,13 @@;; lanes then the `dup` instruction can be used for this operation. Note that;; for now this only matches lane selection from the first vector `a`, but;; if necessary in the future rules can be added to select from `b` as well.-(rule 6 (lower (shuffle a b (shuffle_dup8_from_imm n)))+(rule 6 (lower (shuffle _ a b (shuffle_dup8_from_imm n)))(vec_dup_from_fpu a (VectorSize.Size8x16) n))-(rule 5 (lower (shuffle a b (shuffle_dup16_from_imm n)))+(rule 5 (lower (shuffle _ a b (shuffle_dup16_from_imm n)))(vec_dup_from_fpu a (VectorSize.Size16x8) n))-(rule 4 (lower (shuffle a b (shuffle_dup32_from_imm n)))+(rule 4 (lower (shuffle _ a b (shuffle_dup32_from_imm n)))(vec_dup_from_fpu a (VectorSize.Size32x4) n))-(rule 3 (lower (shuffle a b (shuffle_dup64_from_imm n)))+(rule 3 (lower (shuffle _ a b (shuffle_dup64_from_imm n)))(vec_dup_from_fpu a (VectorSize.Size64x2) n));; If the `Immediate` specified to the extractor looks like a duplication of the@@ -156,7 +156,7 @@;; When the shuffle looks like "concatenate `a` and `b` and shift right by n*8;; bytes", that's an `ext` instruction.-(rule 2 (lower (shuffle a b (vec_extract_imm4_from_immediate n)))+(rule 2 (lower (shuffle _ a b (vec_extract_imm4_from_immediate n)))(vec_extract a b n));; Attempts to extract `n` from the specified shuffle `Immediate` where each@@ -169,36 +169,36 @@;; Rules for the `uzp1` and `uzp2` instructions which gather even-numbered lanes;; or odd-numbered lanes-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1e1c_1a18_1614_1210_0e0c_0a08_0604_0200)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1e1c_1a18_1614_1210_0e0c_0a08_0604_0200)))(vec_uzp1 a b (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f1d_1b19_1715_1311_0f0d_0b09_0705_0301)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f1d_1b19_1715_1311_0f0d_0b09_0705_0301)))(vec_uzp2 a b (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1d1c_1918_1514_1110_0d0c_0908_0504_0100)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1d1c_1918_1514_1110_0d0c_0908_0504_0100)))(vec_uzp1 a b (VectorSize.Size16x8)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f1e_1b1a_1716_1312_0f0e_0b0a_0706_0302)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f1e_1b1a_1716_1312_0f0e_0b0a_0706_0302)))(vec_uzp2 a b (VectorSize.Size16x8)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1b1a1918_13121110_0b0a0908_03020100)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1b1a1918_13121110_0b0a0908_03020100)))(vec_uzp1 a b (VectorSize.Size32x4)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f1e1d1c_17161514_0f0e0d0c_07060504)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f1e1d1c_17161514_0f0e0d0c_07060504)))(vec_uzp2 a b (VectorSize.Size32x4)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1716151413121110_0706050403020100)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1716151413121110_0706050403020100)))(vec_uzp1 a b (VectorSize.Size64x2)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f1e1d1c1b1a1918_0f0e0d0c0b0a0908)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f1e1d1c1b1a1918_0f0e0d0c0b0a0908)))(vec_uzp2 a b (VectorSize.Size64x2)));; Rules for the `zip1` and `zip2` instructions which interleave lanes in the;; low or high halves of the two input vectors.-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1707_1606_1505_1404_1303_1202_1101_1000)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1707_1606_1505_1404_1303_1202_1101_1000)))(vec_zip1 a b (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f0f_1e0e_1d0d_1c0c_1b0b_1a0a_1909_1808)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f0f_1e0e_1d0d_1c0c_1b0b_1a0a_1909_1808)))(vec_zip2 a b (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1716_0706_1514_0504_1312_0302_1110_0100)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1716_0706_1514_0504_1312_0302_1110_0100)))(vec_zip1 a b (VectorSize.Size16x8)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f1e_0f0e_1d1c_0d0c_1b1a_0b0a_1918_0908)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f1e_0f0e_1d1c_0d0c_1b1a_0b0a_1918_0908)))(vec_zip2 a b (VectorSize.Size16x8)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x17161514_07060504_13121110_03020100)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x17161514_07060504_13121110_03020100)))(vec_zip1 a b (VectorSize.Size32x4)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f1e1d1c_0f0e0d0c_1b1a1918_0b0a0908)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f1e1d1c_0f0e0d0c_1b1a1918_0b0a0908)))(vec_zip2 a b (VectorSize.Size32x4)));; Note that zip1/zip2 for i64x2 vectors is omitted since it's already covered;; by the i64x2 cases of uzp1/uzp2 above where both zip and uzp have the same@@ -206,17 +206,17 @@;; Rules for the `trn1` and `trn2` instructions which interleave odd or even;; lanes in the two input vectors.-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1e0e_1c0c_1a0a_1808_1606_1404_1202_1000)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1e0e_1c0c_1a0a_1808_1606_1404_1202_1000)))(vec_trn1 a b (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f0f_1d0d_1b0b_1909_1707_1505_1303_1101)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f0f_1d0d_1b0b_1909_1707_1505_1303_1101)))(vec_trn2 a b (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1d1c_0d0c_1918_0908_1514_0504_1110_0100)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1d1c_0d0c_1918_0908_1514_0504_1110_0100)))(vec_trn1 a b (VectorSize.Size16x8)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f1e_0f0e_1b1a_0b0a_1716_0706_1312_0302)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f1e_0f0e_1b1a_0b0a_1716_0706_1312_0302)))(vec_trn2 a b (VectorSize.Size16x8)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1b1a1918_0b0a0908_13121110_03020100)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1b1a1918_0b0a0908_13121110_03020100)))(vec_trn1 a b (VectorSize.Size32x4)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x1f1e1d1c_0f0e0d0c_17161514_07060504)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x1f1e1d1c_0f0e0d0c_17161514_07060504)))(vec_trn2 a b (VectorSize.Size32x4)));; Note that trn1/trn2 for i64x2 vectors is omitted since it's already covered;; by the i64x2 cases of uzp1/uzp2 above where both trn and uzp have the same@@ -226,31 +226,31 @@;; the byte level, the 16-bit level, or 32-bit level. Note that all of these;; patterns only match reversals in the first operand, but they can;; theoretically be extended if necessary to reversals in the second operand.-(rule 1 (lower (shuffle a b (u128_from_immediate 0x0e0f_0c0d_0a0b_0809_0607_0405_0203_0001)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x0e0f_0c0d_0a0b_0809_0607_0405_0203_0001)))(rev16 a (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x0c0d0e0f_08090a0b_04050607_00010203)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x0c0d0e0f_08090a0b_04050607_00010203)))(rev32 a (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x0d0c0f0e_09080b0a_05040706_01000302)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x0d0c0f0e_09080b0a_05040706_01000302)))(rev32 a (VectorSize.Size16x8)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x08090a0b0c0d0e0f_0001020304050607)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x08090a0b0c0d0e0f_0001020304050607)))(rev64 a (VectorSize.Size8x16)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x09080b0a0d0c0f0e_0100030205040706)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x09080b0a0d0c0f0e_0100030205040706)))(rev64 a (VectorSize.Size16x8)))-(rule 1 (lower (shuffle a b (u128_from_immediate 0x0b0a09080f0e0d0c_0302010007060504)))+(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x0b0a09080f0e0d0c_0302010007060504)))(rev64 a (VectorSize.Size32x4)))-(rule (lower (has_type ty (shuffle rn rn2 (u128_from_immediate mask))))+(rule (lower (has_type ty (shuffle _ rn rn2 (u128_from_immediate mask))))(let ((mask_reg Reg (constant_f128 mask)))(vec_tbl2 rn rn2 mask_reg ty)));;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type vec_i128_ty (swizzle rn rm)))+(rule (lower (has_type vec_i128_ty (swizzle _ rn rm)))(vec_tbl rn rm));;;; Rules for `isplit` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (isplit x @ (value_type $I128)))+(rule (lower (isplit _ x @ (value_type $I128)))(let((x_regs ValueRegs x)(x_lo ValueRegs (value_regs_get x_regs 0))@@ -260,13 +260,13 @@;; Special-case the lowering of an `isplit` of a 128-bit multiply where the;; lower bits of the result are discarded and the operands are sign or zero;; extended. This maps directly to `umulh` and `smulh`.-(rule 1 (lower i @ (isplit (has_type $I128 (imul (uextend x) (uextend y)))))+(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (uextend _ x) (uextend _ y)))))(if-let (first_result lo) i)(if-let true (value_is_unused lo))(output_pair (invalid_reg)(umulh $I64 (put_in_reg_zext64 x) (put_in_reg_zext64 y))))-(rule 1 (lower i @ (isplit (has_type $I128 (imul (sextend x) (sextend y)))))+(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (sextend _ x) (sextend _ y)))))(if-let (first_result lo) i)(if-let true (value_is_unused lo))(output_pair (invalid_reg)@@ -274,21 +274,21 @@;;;; Rules for `iconcat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I128 (iconcat lo hi)))+(rule (lower (has_type $I128 (iconcat _ lo hi)))(output (value_regs lo hi)));;;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32X4 (scalar_to_vector x)))+(rule (lower (has_type $F32X4 (scalar_to_vector _ x)))(fpu_extend x (ScalarSize.Size32)))-(rule (lower (has_type $F64X2 (scalar_to_vector x)))+(rule (lower (has_type $F64X2 (scalar_to_vector _ x)))(fpu_extend x (ScalarSize.Size64)))-(rule -1 (lower (scalar_to_vector x @ (value_type $I64)))+(rule -1 (lower (scalar_to_vector _ x @ (value_type $I64)))(mov_to_fpu x (ScalarSize.Size64)))-(rule -2 (lower (scalar_to_vector x @ (value_type (int_fits_in_32 _))))+(rule -2 (lower (scalar_to_vector _ x @ (value_type (int_fits_in_32 _))))(mov_to_fpu (put_in_reg_zext32 x) (ScalarSize.Size32)));;;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -302,13 +302,13 @@;; 0 when all input elements are true, i.e. non-zero, or a NaN otherwise;; (either -1 or -2 when represented as an integer); NaNs are the only;; floating-point numbers that compare unequal to themselves.-(rule (lower (vall_true x @ (value_type (multi_lane 64 2))))+(rule (lower (vall_true _ x @ (value_type (multi_lane 64 2))))(let ((x1 Reg (cmeq0 x (VectorSize.Size64x2)))(x2 Reg (addp x1 x1 (VectorSize.Size64x2))))(with_flags (fpu_cmp (ScalarSize.Size64) x2 x2)(materialize_bool_result (Cond.Eq)))))-(rule (lower (vall_true x @ (value_type (multi_lane 32 2))))+(rule (lower (vall_true _ x @ (value_type (multi_lane 32 2))))(let ((x1 Reg (mov_from_vec x 0 (ScalarSize.Size64))))(with_flags (cmp_rr_shift (OperandSize.Size64) (zero_reg) x1 32)(ccmp_imm@@ -325,7 +325,7 @@;; mov xm, vn.d[0];; cmp xm, #0;; cset xm, ne-(rule -1 (lower (vall_true x @ (value_type (lane_fits_in_32 ty))))+(rule -1 (lower (vall_true _ x @ (value_type (lane_fits_in_32 ty))))(if (not_vec32x2 ty))(let ((x1 Reg (vec_lanes (VecLanesOp.Uminv) x (vector_size ty)))(x2 Reg (mov_from_vec x1 0 (ScalarSize.Size64))))@@ -334,40 +334,40 @@;;;; Rules for `vany_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (vany_true x @ (value_type in_ty)))+(rule (lower (vany_true _ x @ (value_type in_ty)))(with_flags (vanytrue x in_ty)(materialize_bool_result (Cond.Ne))));;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; special case for the `i16x8.extadd_pairwise_i8x16_s` wasm instruction-(rule (lower (has_type $I16X8 (iadd_pairwise (swiden_low x) (swiden_high x))))+(rule (lower (has_type $I16X8 (iadd_pairwise _ (swiden_low _ x) (swiden_high _ x))))(saddlp8 x));; special case for the `i32x4.extadd_pairwise_i16x8_s` wasm instruction-(rule (lower (has_type $I32X4 (iadd_pairwise (swiden_low x) (swiden_high x))))+(rule (lower (has_type $I32X4 (iadd_pairwise _ (swiden_low _ x) (swiden_high _ x))))(saddlp16 x));; special case for the `i16x8.extadd_pairwise_i8x16_u` wasm instruction-(rule (lower (has_type $I16X8 (iadd_pairwise (uwiden_low x) (uwiden_high x))))+(rule (lower (has_type $I16X8 (iadd_pairwise _ (uwiden_low _ x) (uwiden_high _ x))))(uaddlp8 x));; special case for the `i32x4.extadd_pairwise_i16x8_u` wasm instruction-(rule (lower (has_type $I32X4 (iadd_pairwise (uwiden_low x) (uwiden_high x))))+(rule (lower (has_type $I32X4 (iadd_pairwise _ (uwiden_low _ x) (uwiden_high _ x))))(uaddlp16 x))-(rule -1 (lower (has_type ty (iadd_pairwise x y)))+(rule -1 (lower (has_type ty (iadd_pairwise _ x y)))(addp x y (vector_size ty)));;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (iabs x)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (iabs _ x)))(vec_abs x (vector_size ty)))-(rule iabs_64 2 (lower (has_type $I64 (iabs x)))+(rule iabs_64 2 (lower (has_type $I64 (iabs _ x)))(abs (OperandSize.Size64) x))-(rule iabs_8_16_32 1 (lower (has_type (fits_in_32 ty) (iabs x)))+(rule iabs_8_16_32 1 (lower (has_type (fits_in_32 ty) (iabs _ x)))(abs (OperandSize.Size32) (put_in_reg_sext32 x))); `rustc` implementation.@@ -377,7 +377,7 @@; `x - 0`.; - if `x` is negative, the xor'd bits = ~x and the mask = -1, so we end up with; `~x - (-1) = ~x + 1`, which is exactly `abs(x)`.-(rule (lower (has_type $I128 (iabs x)))+(rule (lower (has_type $I128 (iabs _ x)))(let ((x_regs ValueRegs x)(x_lo Reg (value_regs_get x_regs 0))(x_hi Reg (value_regs_get x_regs 1))@@ -390,7 +390,7 @@;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I64X2 (avg_round x y)))+(rule (lower (has_type $I64X2 (avg_round _ x y)))(let ((one Reg (splat_const 1 (VectorSize.Size64x2)))(c Reg (orr_vec x y (VectorSize.Size64x2)))(c Reg (and_vec c one (VectorSize.Size64x2)))@@ -399,144 +399,144 @@(sum Reg (add_vec x y (VectorSize.Size64x2))))(add_vec c sum (VectorSize.Size64x2))))-(rule -1 (lower (has_type (lane_fits_in_32 ty) (avg_round x y)))+(rule -1 (lower (has_type (lane_fits_in_32 ty) (avg_round _ x y)))(vec_rrr (VecALUOp.Urhadd) x y (vector_size ty)));;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty @ (multi_lane _ _) (sqmul_round_sat x y)))+(rule (lower (has_type ty @ (multi_lane _ _) (sqmul_round_sat _ x y)))(vec_rrr (VecALUOp.Sqrdmulh) x y (vector_size ty)));;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fadd rn rm)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (fadd _ rn rm)))(vec_rrr (VecALUOp.Fadd) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fadd rn rm)))+(rule (lower (has_type (ty_scalar_float ty) (fadd _ rn rm)))(fpu_rrr (FPUOp2.Add) rn rm (scalar_size ty)));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fsub rn rm)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (fsub _ rn rm)))(vec_rrr (VecALUOp.Fsub) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fsub rn rm)))+(rule (lower (has_type (ty_scalar_float ty) (fsub _ rn rm)))(fpu_rrr (FPUOp2.Sub) rn rm (scalar_size ty)));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmul rn rm)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmul _ rn rm)))(vec_rrr (VecALUOp.Fmul) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fmul rn rm)))+(rule (lower (has_type (ty_scalar_float ty) (fmul _ rn rm)))(fpu_rrr (FPUOp2.Mul) rn rm (scalar_size ty)));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fdiv rn rm)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (fdiv _ rn rm)))(vec_rrr (VecALUOp.Fdiv) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fdiv rn rm)))+(rule (lower (has_type (ty_scalar_float ty) (fdiv _ rn rm)))(fpu_rrr (FPUOp2.Div) rn rm (scalar_size ty)));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmin rn rm)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmin _ rn rm)))(vec_rrr (VecALUOp.Fmin) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fmin rn rm)))+(rule (lower (has_type (ty_scalar_float ty) (fmin _ rn rm)))(fpu_rrr (FPUOp2.Min) rn rm (scalar_size ty)));;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmax rn rm)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmax _ rn rm)))(vec_rrr (VecALUOp.Fmax) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fmax rn rm)))+(rule (lower (has_type (ty_scalar_float ty) (fmax _ rn rm)))(fpu_rrr (FPUOp2.Max) rn rm (scalar_size ty)));;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (sqrt x)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (sqrt _ x)))(vec_misc (VecMisc2.Fsqrt) x (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (sqrt x)))+(rule (lower (has_type (ty_scalar_float ty) (sqrt _ x)))(fpu_rr (FPUOp1.Sqrt) x (scalar_size ty)));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fneg x)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (fneg _ x)))(vec_misc (VecMisc2.Fneg) x (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fneg x)))+(rule (lower (has_type (ty_scalar_float ty) (fneg _ x)))(fpu_rr (FPUOp1.Neg) x (scalar_size ty)));;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fabs x)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (fabs _ x)))(vec_misc (VecMisc2.Fabs) x (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fabs x)))+(rule (lower (has_type (ty_scalar_float ty) (fabs _ x)))(fpu_rr (FPUOp1.Abs) x (scalar_size ty)));;;; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F64 (fpromote x)))+(rule (lower (has_type $F64 (fpromote _ x)))(fpu_rr (FPUOp1.Cvt32To64) x (ScalarSize.Size32)));;;; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdemote x)))+(rule (lower (has_type $F32 (fdemote _ x)))(fpu_rr (FPUOp1.Cvt64To32) x (ScalarSize.Size64)));;;; Rules for `ceil` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (ceil x)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (ceil _ x)))(vec_misc (VecMisc2.Frintp) x (vector_size ty)))-(rule (lower (has_type $F32 (ceil x)))+(rule (lower (has_type $F32 (ceil _ x)))(fpu_round (FpuRoundMode.Plus32) x))-(rule (lower (has_type $F64 (ceil x)))+(rule (lower (has_type $F64 (ceil _ x)))(fpu_round (FpuRoundMode.Plus64) x));;;; Rules for `floor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (floor x)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (floor _ x)))(vec_misc (VecMisc2.Frintm) x (vector_size ty)))-(rule (lower (has_type $F32 (floor x)))+(rule (lower (has_type $F32 (floor _ x)))(fpu_round (FpuRoundMode.Minus32) x))-(rule (lower (has_type $F64 (floor x)))+(rule (lower (has_type $F64 (floor _ x)))(fpu_round (FpuRoundMode.Minus64) x));;;; Rules for `trunc` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (trunc x)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (trunc _ x)))(vec_misc (VecMisc2.Frintz) x (vector_size ty)))-(rule (lower (has_type $F32 (trunc x)))+(rule (lower (has_type $F32 (trunc _ x)))(fpu_round (FpuRoundMode.Zero32) x))-(rule (lower (has_type $F64 (trunc x)))+(rule (lower (has_type $F64 (trunc _ x)))(fpu_round (FpuRoundMode.Zero64) x));;;; Rules for `nearest` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (nearest x)))+(rule -1 (lower (has_type ty @ (multi_lane _ _) (nearest _ x)))(vec_misc (VecMisc2.Frintn) x (vector_size ty)))-(rule (lower (has_type $F32 (nearest x)))+(rule (lower (has_type $F32 (nearest _ x)))(fpu_round (FpuRoundMode.Nearest32) x))-(rule (lower (has_type $F64 (nearest x)))+(rule (lower (has_type $F64 (nearest _ x)))(fpu_round (FpuRoundMode.Nearest64) x));;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type ty (fma x y z))) (fmadd ty x y z))-(rule 2 (lower (has_type (ty_scalar_float ty) (fma x y (fneg z)))) (fnmsub ty x y z))+(rule 1 (lower (has_type ty (fma _ x y z))) (fmadd ty x y z))+(rule 2 (lower (has_type (ty_scalar_float ty) (fma _ x y (fneg _ z)))) (fnmsub ty x y z));; Constructors matching the scalar behavior of aarch64. If you're confused like;; I was reading over these, they are:@@ -552,10 +552,10 @@;; Switch `fm{add,sub}` to `fnm{add,sub}` if one of the operands are negated;; instruction instead-(rule 2 (fmadd ty (fneg x) y z) (fmsub ty x y z))-(rule 3 (fmadd ty x (fneg y) z) (fmsub ty x y z))-(rule 2 (fnmsub ty (fneg x) y z) (fnmadd ty x y z))-(rule 3 (fnmsub ty x (fneg y) z) (fnmadd ty x y z))+(rule 2 (fmadd ty (fneg _ x) y z) (fmsub ty x y z))+(rule 3 (fmadd ty x (fneg _ y) z) (fmsub ty x y z))+(rule 2 (fnmsub ty (fneg _ x) y z) (fnmadd ty x y z))+(rule 3 (fnmsub ty x (fneg _ y) z) (fnmadd ty x y z));; Scalar cases(rule 0 (fmadd (ty_scalar_float ty) x y z) (fpu_rrrr (FPUOp3.MAdd) (scalar_size ty) x y z))@@ -577,9 +577,9 @@;; Special case: if one of the multiplicands are a splat then the element-based;; fma can be used instead with 0 as the element index.-(rule 1 (lower_fmla op (splat x) y z size)+(rule 1 (lower_fmla op (splat _ x) y z size)(vec_fmla_elem op z y x size 0))-(rule 2 (lower_fmla op x (splat y) z size)+(rule 2 (lower_fmla op x (splat _ y) z size)(vec_fmla_elem op z x y size 0));; Special case: if one of the multiplicands is a shuffle to broadcast a@@ -589,130 +589,130 @@;; Note that in Cranelift shuffle always has i8x16 inputs and outputs so;; a `bitcast` is matched here explicitly since that's the main way a shuffle;; output will be fed into this instruction.-(rule 3 (lower_fmla op (bitcast _ (shuffle x x (shuffle32_from_imm n n n n))) y z size @ (VectorSize.Size32x4))+(rule 3 (lower_fmla op (bitcast _ _ (shuffle _ x x (shuffle32_from_imm n n n n))) y z size @ (VectorSize.Size32x4))(if-let true (u64_lt n 4))(vec_fmla_elem op z y x size n))-(rule 4 (lower_fmla op x (bitcast _ (shuffle y y (shuffle32_from_imm n n n n))) z size @ (VectorSize.Size32x4))+(rule 4 (lower_fmla op x (bitcast _ _ (shuffle _ y y (shuffle32_from_imm n n n n))) z size @ (VectorSize.Size32x4))(if-let true (u64_lt n 4))(vec_fmla_elem op z x y size n))-(rule 3 (lower_fmla op (bitcast _ (shuffle x x (shuffle64_from_imm n n))) y z size @ (VectorSize.Size64x2))+(rule 3 (lower_fmla op (bitcast _ _ (shuffle _ x x (shuffle64_from_imm n n))) y z size @ (VectorSize.Size64x2))(if-let true (u64_lt n 2))(vec_fmla_elem op z y x size n))-(rule 4 (lower_fmla op x (bitcast _ (shuffle y y (shuffle64_from_imm n n))) z size @ (VectorSize.Size64x2))+(rule 4 (lower_fmla op x (bitcast _ _ (shuffle _ y y (shuffle64_from_imm n n))) z size @ (VectorSize.Size64x2))(if-let true (u64_lt n 2))(vec_fmla_elem op z x y size n));;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (fcopysign x y)))+(rule (lower (has_type ty (fcopysign _ x y)))(fcopy_sign x y ty));;;; Rules for `fcvt_to_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint x @ (value_type $F32))))+(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint _ x @ (value_type $F32))))(fpu_to_int_cvt (FpuToIntOp.F32ToU32) x false $F32 out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_uint x @ (value_type $F32))))+(rule 1 (lower (has_type $I64 (fcvt_to_uint _ x @ (value_type $F32))))(fpu_to_int_cvt (FpuToIntOp.F32ToU64) x false $F32 $I64))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint x @ (value_type $F64))))+(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint _ x @ (value_type $F64))))(fpu_to_int_cvt (FpuToIntOp.F64ToU32) x false $F64 out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_uint x @ (value_type $F64))))+(rule 1 (lower (has_type $I64 (fcvt_to_uint _ x @ (value_type $F64))))(fpu_to_int_cvt (FpuToIntOp.F64ToU64) x false $F64 $I64));;;; Rules for `fcvt_to_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint x @ (value_type $F32))))+(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint _ x @ (value_type $F32))))(fpu_to_int_cvt (FpuToIntOp.F32ToI32) x true $F32 out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_sint x @ (value_type $F32))))+(rule 1 (lower (has_type $I64 (fcvt_to_sint _ x @ (value_type $F32))))(fpu_to_int_cvt (FpuToIntOp.F32ToI64) x true $F32 $I64))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint x @ (value_type $F64))))+(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint _ x @ (value_type $F64))))(fpu_to_int_cvt (FpuToIntOp.F64ToI32) x true $F64 out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_sint x @ (value_type $F64))))+(rule 1 (lower (has_type $I64 (fcvt_to_sint _ x @ (value_type $F64))))(fpu_to_int_cvt (FpuToIntOp.F64ToI64) x true $F64 $I64));;;; Rules for `fcvt_from_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_from_uint x @ (value_type (multi_lane 32 _)))))+(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_from_uint _ x @ (value_type (multi_lane 32 _)))))(vec_misc (VecMisc2.Ucvtf) x (vector_size ty)))-(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_from_uint x @ (value_type (multi_lane 64 _)))))+(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_from_uint _ x @ (value_type (multi_lane 64 _)))))(vec_misc (VecMisc2.Ucvtf) x (vector_size ty)))-(rule (lower (has_type $F32 (fcvt_from_uint x @ (value_type (fits_in_32 _)))))+(rule (lower (has_type $F32 (fcvt_from_uint _ x @ (value_type (fits_in_32 _)))))(int_to_fpu (IntToFpuOp.U32ToF32) (put_in_reg_zext32 x)))-(rule (lower (has_type $F64 (fcvt_from_uint x @ (value_type (fits_in_32 _)))))+(rule (lower (has_type $F64 (fcvt_from_uint _ x @ (value_type (fits_in_32 _)))))(int_to_fpu (IntToFpuOp.U32ToF64) (put_in_reg_zext32 x)))-(rule 1 (lower (has_type $F32 (fcvt_from_uint x @ (value_type $I64))))+(rule 1 (lower (has_type $F32 (fcvt_from_uint _ x @ (value_type $I64))))(int_to_fpu (IntToFpuOp.U64ToF32) x))-(rule 1 (lower (has_type $F64 (fcvt_from_uint x @ (value_type $I64))))+(rule 1 (lower (has_type $F64 (fcvt_from_uint _ x @ (value_type $I64))))(int_to_fpu (IntToFpuOp.U64ToF64) x));;;; Rules for `fcvt_from_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_from_sint x @ (value_type (multi_lane 32 _)))))+(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_from_sint _ x @ (value_type (multi_lane 32 _)))))(vec_misc (VecMisc2.Scvtf) x (vector_size ty)))-(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_from_sint x @ (value_type (multi_lane 64 _)))))+(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_from_sint _ x @ (value_type (multi_lane 64 _)))))(vec_misc (VecMisc2.Scvtf) x (vector_size ty)))-(rule (lower (has_type $F32 (fcvt_from_sint x @ (value_type (fits_in_32 _)))))+(rule (lower (has_type $F32 (fcvt_from_sint _ x @ (value_type (fits_in_32 _)))))(int_to_fpu (IntToFpuOp.I32ToF32) (put_in_reg_sext32 x)))-(rule (lower (has_type $F64 (fcvt_from_sint x @ (value_type (fits_in_32 _)))))+(rule (lower (has_type $F64 (fcvt_from_sint _ x @ (value_type (fits_in_32 _)))))(int_to_fpu (IntToFpuOp.I32ToF64) (put_in_reg_sext32 x)))-(rule 1 (lower (has_type $F32 (fcvt_from_sint x @ (value_type $I64))))+(rule 1 (lower (has_type $F32 (fcvt_from_sint _ x @ (value_type $I64))))(int_to_fpu (IntToFpuOp.I64ToF32) x))-(rule 1 (lower (has_type $F64 (fcvt_from_sint x @ (value_type $I64))))+(rule 1 (lower (has_type $F64 (fcvt_from_sint _ x @ (value_type $I64))))(int_to_fpu (IntToFpuOp.I64ToF64) x));;;; Rules for `fcvt_to_uint_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_to_uint_sat x @ (value_type (multi_lane 32 _)))))+(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_to_uint_sat _ x @ (value_type (multi_lane 32 _)))))(vec_misc (VecMisc2.Fcvtzu) x (vector_size ty)))-(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_to_uint_sat x @ (value_type (multi_lane 64 _)))))+(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_to_uint_sat _ x @ (value_type (multi_lane 64 _)))))(vec_misc (VecMisc2.Fcvtzu) x (vector_size ty)))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint_sat x @ (value_type $F32))))+(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint_sat _ x @ (value_type $F32))))(fpu_to_int_cvt_sat (FpuToIntOp.F32ToU32) x false out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_uint_sat x @ (value_type $F32))))+(rule 1 (lower (has_type $I64 (fcvt_to_uint_sat _ x @ (value_type $F32))))(fpu_to_int_cvt_sat (FpuToIntOp.F32ToU64) x false $I64))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint_sat x @ (value_type $F64))))+(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint_sat _ x @ (value_type $F64))))(fpu_to_int_cvt_sat (FpuToIntOp.F64ToU32) x false out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_uint_sat x @ (value_type $F64))))+(rule 1 (lower (has_type $I64 (fcvt_to_uint_sat _ x @ (value_type $F64))))(fpu_to_int_cvt_sat (FpuToIntOp.F64ToU64) x false $I64));;;; Rules for `fcvt_to_sint_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_to_sint_sat x @ (value_type (multi_lane 32 _)))))+(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_to_sint_sat _ x @ (value_type (multi_lane 32 _)))))(vec_misc (VecMisc2.Fcvtzs) x (vector_size ty)))-(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_to_sint_sat x @ (value_type (multi_lane 64 _)))))+(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_to_sint_sat _ x @ (value_type (multi_lane 64 _)))))(vec_misc (VecMisc2.Fcvtzs) x (vector_size ty)))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint_sat x @ (value_type $F32))))+(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint_sat _ x @ (value_type $F32))))(fpu_to_int_cvt_sat (FpuToIntOp.F32ToI32) x true out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_sint_sat x @ (value_type $F32))))+(rule 1 (lower (has_type $I64 (fcvt_to_sint_sat _ x @ (value_type $F32))))(fpu_to_int_cvt_sat (FpuToIntOp.F32ToI64) x true $I64))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint_sat x @ (value_type $F64))))+(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint_sat _ x @ (value_type $F64))))(fpu_to_int_cvt_sat (FpuToIntOp.F64ToI32) x true out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_sint_sat x @ (value_type $F64))))+(rule 1 (lower (has_type $I64 (fcvt_to_sint_sat _ x @ (value_type $F64))))(fpu_to_int_cvt_sat (FpuToIntOp.F64ToI64) x true $I64));;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -720,81 +720,81 @@;; `i64` and smaller;; Base case, simply subtracting things in registers.-(rule isub_base_case -4 (lower (has_type (ty_int_ref_scalar_64 ty) (isub x y)))+(rule isub_base_case -4 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x y)))(sub ty x y));; Special case for when one operand is an immediate that fits in 12 bits.-(rule isub_imm12 0 (lower (has_type (ty_int_ref_scalar_64 ty) (isub x (imm12_from_value y))))+(rule isub_imm12 0 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x (imm12_from_value y))))(sub_imm ty x y));; Same as the previous special case, except we can switch the subtraction to an;; addition if the negated immediate fits in 12 bits.-(rule isub_imm12_neg 2 (lower (has_type (ty_int_ref_scalar_64 ty) (isub x y)))+(rule isub_imm12_neg 2 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x y)))(if-let imm12_neg (imm12_from_negated_value y))(add_imm ty x imm12_neg));; Special cases for when we're subtracting an extended register where the;; extending operation can get folded into the sub itself.-(rule isub_extend 1 (lower (has_type (ty_int_ref_scalar_64 ty) (isub x (extended_value_from_value y))))+(rule isub_extend 1 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x (extended_value_from_value y))))(sub_extend ty x y));; Finally a special case for when we're subtracting the shift of a different;; register by a constant amount and the shift can get folded into the sub.(rule isub_ishl -3 (lower (has_type (ty_int_ref_scalar_64 ty)- (isub x (ishl y (iconst k)))))+ (isub _ x (ishl _ y (iconst _ k)))))(if-let amt (lshl_from_imm64 ty k))(sub_shift ty x y amt));; vectors-(rule -2 (lower (has_type ty @ (multi_lane _ _) (isub x y)))+(rule -2 (lower (has_type ty @ (multi_lane _ _) (isub _ x y)))(sub_vec x y (vector_size ty)));; `i128`-(rule -1 (lower (has_type $I128 (isub x y)))+(rule -1 (lower (has_type $I128 (isub _ x y)))(sub_i128 x y));;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 ty) (uadd_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (uadd_sat _ x y)))(uqadd x y (vector_size ty)));;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 ty) (sadd_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (sadd_sat _ x y)))(sqadd x y (vector_size ty)));;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 ty) (usub_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (usub_sat _ x y)))(uqsub x y (vector_size ty)));;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 ty) (ssub_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (ssub_sat _ x y)))(sqsub x y (vector_size ty)));;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule ineg_base_case 1 (lower (has_type (fits_in_64 ty) (ineg x)))+(rule ineg_base_case 1 (lower (has_type (fits_in_64 ty) (ineg _ x)))(sub ty (zero_reg) x));; `i128`-(rule 2 (lower (has_type $I128 (ineg x)))+(rule 2 (lower (has_type $I128 (ineg _ x)))(sub_i128 (value_regs_zero) x));; vectors.-(rule (lower (has_type (ty_vec128 ty) (ineg x)))+(rule (lower (has_type (ty_vec128 ty) (ineg _ x)))(neg x (vector_size ty)));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule imul_base_case -3 (lower (has_type (ty_int_ref_scalar_64 ty) (imul x y)))+(rule imul_base_case -3 (lower (has_type (ty_int_ref_scalar_64 ty) (imul _ x y)))(madd ty x y (zero_reg)));; `i128`.-(rule -1 (lower (has_type $I128 (imul x y)))+(rule -1 (lower (has_type $I128 (imul _ x y)))(let;; Get the high/low registers for `x`.((x_regs ValueRegs x)@@ -824,7 +824,7 @@;; Special cases where the upper bits are sign-or-zero extended of the lower bits;; so the calculation here is much simpler with just a `umulh` or `smulh`;; instead of the additions above as well.-(rule (lower (has_type $I128 (imul (uextend x) (uextend y))))+(rule (lower (has_type $I128 (imul _ (uextend _ x) (uextend _ y))))(let ((x Reg (put_in_reg_zext64 x))(y Reg (put_in_reg_zext64 y))@@ -832,7 +832,7 @@(value_regs(madd $I64 x y (zero_reg))(umulh $I64 x y))))-(rule (lower (has_type $I128 (imul (sextend x) (sextend y))))+(rule (lower (has_type $I128 (imul _ (sextend _ x) (sextend _ y))))(let ((x Reg (put_in_reg_sext64 x))(y Reg (put_in_reg_sext64 y))@@ -842,7 +842,7 @@(smulh $I64 x y))));; vectors (i8x8/i8x16/i16x4/i16x8/i32x2/i32x4)-(rule -2 (lower (has_type (lane_fits_in_32 ty @ (multi_lane _ _)) (imul x y)))+(rule -2 (lower (has_type (lane_fits_in_32 ty @ (multi_lane _ _)) (imul _ x y)))(mul x y (vector_size ty)));; Special lowering for i64x2.@@ -874,7 +874,7 @@;; xtn tmp2.2s, rm.2d;; shll rd.2d, rd.2s, #32;; umlal rd.2d, tmp2.2s, tmp1.2s-(rule -1 (lower (has_type $I64X2 (imul x y)))+(rule -1 (lower (has_type $I64X2 (imul _ x y)))(let ((rn Reg x)(rm Reg y);; Reverse the 32-bit elements in the 64-bit words.@@ -914,82 +914,82 @@;; Special case for `i16x8.extmul_low_i8x16_s`.(rule (lower (has_type $I16X8- (imul (swiden_low x @ (value_type $I8X16))- (swiden_low y @ (value_type $I8X16)))))+ (imul _ (swiden_low _ x @ (value_type $I8X16))+ (swiden_low _ y @ (value_type $I8X16)))))(smull8 x y false));; Special case for `i16x8.extmul_high_i8x16_s`.(rule (lower (has_type $I16X8- (imul (swiden_high x @ (value_type $I8X16))- (swiden_high y @ (value_type $I8X16)))))+ (imul _ (swiden_high _ x @ (value_type $I8X16))+ (swiden_high _ y @ (value_type $I8X16)))))(smull8 x y true));; Special case for `i16x8.extmul_low_i8x16_u`.(rule (lower (has_type $I16X8- (imul (uwiden_low x @ (value_type $I8X16))- (uwiden_low y @ (value_type $I8X16)))))+ (imul _ (uwiden_low _ x @ (value_type $I8X16))+ (uwiden_low _ y @ (value_type $I8X16)))))(umull8 x y false));; Special case for `i16x8.extmul_high_i8x16_u`.(rule (lower (has_type $I16X8- (imul (uwiden_high x @ (value_type $I8X16))- (uwiden_high y @ (value_type $I8X16)))))+ (imul _ (uwiden_high _ x @ (value_type $I8X16))+ (uwiden_high _ y @ (value_type $I8X16)))))(umull8 x y true));; Special case for `i32x4.extmul_low_i16x8_s`.(rule (lower (has_type $I32X4- (imul (swiden_low x @ (value_type $I16X8))- (swiden_low y @ (value_type $I16X8)))))+ (imul _ (swiden_low _ x @ (value_type $I16X8))+ (swiden_low _ y @ (value_type $I16X8)))))(smull16 x y false));; Special case for `i32x4.extmul_high_i16x8_s`.(rule (lower (has_type $I32X4- (imul (swiden_high x @ (value_type $I16X8))- (swiden_high y @ (value_type $I16X8)))))+ (imul _ (swiden_high _ x @ (value_type $I16X8))+ (swiden_high _ y @ (value_type $I16X8)))))(smull16 x y true));; Special case for `i32x4.extmul_low_i16x8_u`.(rule (lower (has_type $I32X4- (imul (uwiden_low x @ (value_type $I16X8))- (uwiden_low y @ (value_type $I16X8)))))+ (imul _ (uwiden_low _ x @ (value_type $I16X8))+ (uwiden_low _ y @ (value_type $I16X8)))))(umull16 x y false));; Special case for `i32x4.extmul_high_i16x8_u`.(rule (lower (has_type $I32X4- (imul (uwiden_high x @ (value_type $I16X8))- (uwiden_high y @ (value_type $I16X8)))))+ (imul _ (uwiden_high _ x @ (value_type $I16X8))+ (uwiden_high _ y @ (value_type $I16X8)))))(umull16 x y true));; Special case for `i64x2.extmul_low_i32x4_s`.(rule (lower (has_type $I64X2- (imul (swiden_low x @ (value_type $I32X4))- (swiden_low y @ (value_type $I32X4)))))+ (imul _ (swiden_low _ x @ (value_type $I32X4))+ (swiden_low _ y @ (value_type $I32X4)))))(smull32 x y false));; Special case for `i64x2.extmul_high_i32x4_s`.(rule (lower (has_type $I64X2- (imul (swiden_high x @ (value_type $I32X4))- (swiden_high y @ (value_type $I32X4)))))+ (imul _ (swiden_high _ x @ (value_type $I32X4))+ (swiden_high _ y @ (value_type $I32X4)))))(smull32 x y true));; Special case for `i64x2.extmul_low_i32x4_u`.(rule (lower (has_type $I64X2- (imul (uwiden_low x @ (value_type $I32X4))- (uwiden_low y @ (value_type $I32X4)))))+ (imul _ (uwiden_low _ x @ (value_type $I32X4))+ (uwiden_low _ y @ (value_type $I32X4)))))(umull32 x y false));; Special case for `i64x2.extmul_high_i32x4_u`.(rule (lower (has_type $I64X2- (imul (uwiden_high x @ (value_type $I32X4))- (uwiden_high y @ (value_type $I32X4)))))+ (imul _ (uwiden_high _ x @ (value_type $I32X4))+ (uwiden_high _ y @ (value_type $I32X4)))))(umull32 x y true));;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $I64 (smulhi x y)))+(rule 1 (lower (has_type $I64 (smulhi _ x y)))(smulh $I64 x y))-(rule (lower (has_type (fits_in_32 ty) (smulhi x y)))+(rule (lower (has_type (fits_in_32 ty) (smulhi _ x y)))(let ((x64 Reg (put_in_reg_sext64 x))(y64 Reg (put_in_reg_sext64 y))(mul Reg (madd $I64 x64 y64 (zero_reg)))@@ -998,10 +998,10 @@;;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $I64 (umulhi x y)))+(rule 1 (lower (has_type $I64 (umulhi _ x y)))(umulh $I64 x y))-(rule (lower (has_type (fits_in_32 ty) (umulhi x y)))+(rule (lower (has_type (fits_in_32 ty) (umulhi _ x y)))(let ((x64 Reg (put_in_reg_zext64 x))(y64 Reg (put_in_reg_zext64 y))@@ -1030,11 +1030,11 @@;; move it into a register.;; zero-extend non-zero constant-(rule (put_nonzero_in_reg (iconst (nonzero_u64_from_imm64 n)) (ExtType.Unsigned) ty)+(rule (put_nonzero_in_reg (iconst _ (nonzero_u64_from_imm64 n)) (ExtType.Unsigned) ty)(imm ty (ImmExtend.Zero) n));; sign-extend non-zero constant-(rule (put_nonzero_in_reg (iconst (nonzero_u64_from_imm64 n)) (ExtType.Signed) ty)+(rule (put_nonzero_in_reg (iconst _ (nonzero_u64_from_imm64 n)) (ExtType.Signed) ty)(imm ty (ImmExtend.Sign) n))(rule -1 (put_nonzero_in_reg val _ $I64)@@ -1049,10 +1049,10 @@;; Note that aarch64's `udiv` doesn't trap so to respect the semantics of;; CLIF's `udiv` the check for zero needs to be manually performed.-(rule udiv 1 (lower (has_type $I64 (udiv x y)))+(rule udiv 1 (lower (has_type $I64 (udiv _ x y)))(a64_udiv $I64 (put_in_reg x) (put_nonzero_in_reg y (ExtType.Unsigned) $I64)))-(rule udiv (lower (has_type (fits_in_32 ty) (udiv x y)))+(rule udiv (lower (has_type (fits_in_32 ty) (udiv _ x y)))(a64_udiv $I32 (put_in_reg_zext32 x) (put_nonzero_in_reg y (ExtType.Unsigned) ty)));;;; Rules for `sdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1078,7 +1078,7 @@;; TODO: if `y` is -1 then a check that `x` is not INT_MIN is all that's;; necessary, but right now `y` is checked to not be -1 as well.-(rule sdiv_base_case (lower (has_type $I64 (sdiv x y)))+(rule sdiv_base_case (lower (has_type $I64 (sdiv _ x y)))(let ((x64 Reg (put_in_reg_sext64 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Signed) $I64))(intmin_check_x Reg (intmin_check $I64 x64))@@ -1086,7 +1086,7 @@(result Reg (a64_sdiv $I64 valid_x64 y64)))result))-(rule sdiv_base_case -1 (lower (has_type (fits_in_32 ty) (sdiv x y)))+(rule sdiv_base_case -1 (lower (has_type (fits_in_32 ty) (sdiv _ x y)))(let ((x32 Reg (put_in_reg_sext32 x))(y32 Reg (put_nonzero_in_reg y (ExtType.Signed) ty))(intmin_check_x Reg (intmin_check ty x32))@@ -1096,11 +1096,11 @@;; Special case for `sdiv` where no checks are needed due to division by a;; constant meaning the checks are always passed.-(rule sdiv_safe_divisor 2 (lower (has_type $I64 (sdiv x (iconst imm))))+(rule sdiv_safe_divisor 2 (lower (has_type $I64 (sdiv _ x (iconst _ imm))))(if-let y (safe_divisor_from_imm64 $I64 imm))(a64_sdiv $I64 (put_in_reg_sext64 x) (imm $I64 (ImmExtend.Sign) y)))-(rule sdiv_safe_divisor 1 (lower (has_type (fits_in_32 ty) (sdiv x (iconst imm))))+(rule sdiv_safe_divisor 1 (lower (has_type (fits_in_32 ty) (sdiv _ x (iconst _ imm))))(if-let y (safe_divisor_from_imm64 ty imm))(a64_sdiv ty (put_in_reg_sext32 x) (imm ty (ImmExtend.Sign) y)))@@ -1122,28 +1122,28 @@;; TODO: we can avoid a 0 check, if the dividend is a non-0 constant-(rule urem (lower (has_type $I64 (urem x y)))+(rule urem (lower (has_type $I64 (urem _ x y)))(let ((x64 Reg (put_in_reg_zext64 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Unsigned) $I64))(div Reg (a64_udiv $I64 x64 y64))(result Reg (msub $I64 div y64 x64)))result))-(rule urem -1 (lower (has_type (fits_in_32 ty) (urem x y)))+(rule urem -1 (lower (has_type (fits_in_32 ty) (urem _ x y)))(let ((x64 Reg (put_in_reg_zext32 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Unsigned) ty))(div Reg (a64_udiv ty x64 y64))(result Reg (msub ty div y64 x64)))result))-(rule srem (lower (has_type $I64 (srem x y)))+(rule srem (lower (has_type $I64 (srem _ x y)))(let ((x64 Reg (put_in_reg_sext64 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Signed) $I64))(div Reg (a64_sdiv $I64 x64 y64))(result Reg (msub $I64 div y64 x64)))result))-(rule srem -1 (lower (has_type (fits_in_32 ty) (srem x y)))+(rule srem -1 (lower (has_type (fits_in_32 ty) (srem _ x y)))(let ((x64 Reg (put_in_reg_sext32 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Signed) ty))(div Reg (a64_sdiv ty x64 y64))@@ -1190,76 +1190,76 @@(with_flags_reg (cmp (operand_size ty) x y)(csel cc x y))))-(rule umin 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (umin x y)))+(rule umin 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (umin _ x y)))(cmp_and_choose ty (Cond.Lo) false x y))-(rule smin 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (smin x y)))+(rule smin 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (smin _ x y)))(cmp_and_choose ty (Cond.Lt) true x y))-(rule umax 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (umax x y)))+(rule umax 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (umax _ x y)))(cmp_and_choose ty (Cond.Hi) false x y))-(rule smax 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (smax x y)))+(rule smax 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (smax _ x y)))(cmp_and_choose ty (Cond.Gt) true x y));; Vector types.-(rule (lower (has_type ty @ (not_i64x2) (smin x y)))+(rule (lower (has_type ty @ (not_i64x2) (smin _ x y)))(vec_rrr (VecALUOp.Smin) x y (vector_size ty)))-(rule 1 (lower (has_type $I64X2 (smin x y)))+(rule 1 (lower (has_type $I64X2 (smin _ x y)))(bsl $I64X2 (vec_rrr (VecALUOp.Cmgt) y x (VectorSize.Size64x2)) x y))-(rule (lower (has_type ty @ (not_i64x2) (umin x y)))+(rule (lower (has_type ty @ (not_i64x2) (umin _ x y)))(vec_rrr (VecALUOp.Umin) x y (vector_size ty)))-(rule 1 (lower (has_type $I64X2 (umin x y)))+(rule 1 (lower (has_type $I64X2 (umin _ x y)))(bsl $I64X2 (vec_rrr (VecALUOp.Cmhi) y x (VectorSize.Size64x2)) x y))-(rule (lower (has_type ty @ (not_i64x2) (smax x y)))+(rule (lower (has_type ty @ (not_i64x2) (smax _ x y)))(vec_rrr (VecALUOp.Smax) x y (vector_size ty)))-(rule 1 (lower (has_type $I64X2 (smax x y)))+(rule 1 (lower (has_type $I64X2 (smax _ x y)))(bsl $I64X2 (vec_rrr (VecALUOp.Cmgt) x y (VectorSize.Size64x2)) x y))-(rule (lower (has_type ty @ (not_i64x2) (umax x y)))+(rule (lower (has_type ty @ (not_i64x2) (umax _ x y)))(vec_rrr (VecALUOp.Umax) x y (vector_size ty)))-(rule 1 (lower (has_type $I64X2 (umax x y)))+(rule 1 (lower (has_type $I64X2 (umax _ x y)))(bsl $I64X2 (vec_rrr (VecALUOp.Cmhi) x y (VectorSize.Size64x2)) x y));;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; General rule for extending input to an output which fits in a single;; register.-(rule uextend -2 (lower (has_type (fits_in_64 out) (uextend x @ (value_type in))))+(rule uextend -2 (lower (has_type (fits_in_64 out) (uextend _ x @ (value_type in))))(extend x false (ty_bits in) (ty_bits out)));; Extraction of a vector lane automatically extends as necessary, so we can;; skip an explicit extending instruction.(rule 1 (lower (has_type (fits_in_64 out)- (uextend (extractlane vec @ (value_type in)+ (uextend _ (extractlane _ vec @ (value_type in)(u8_from_uimm8 lane)))))(mov_from_vec (put_in_reg vec) lane (lane_size in)));; Atomic loads will also automatically zero their upper bits so the `uextend`;; instruction can effectively get skipped here.(rule 1 (lower (has_type (fits_in_64 out)- (uextend x @ (and (value_type in) (atomic_load (little_or_native_endian flags) _)))))+ (uextend _ x @ (and (value_type in) (atomic_load _ (little_or_native_endian flags) _)))))(if-let mem_op (is_sinkable_inst x))(load_acquire in flags (sink_atomic_load mem_op)));; Conversion to 128-bit needs a zero-extension of the lower bits and the upper;; bits are all zero.-(rule -1 (lower (has_type $I128 (uextend x)))+(rule -1 (lower (has_type $I128 (uextend _ x)))(value_regs (put_in_reg_zext64 x) (imm $I64 (ImmExtend.Zero) 0)));; Like above where vector extraction automatically zero-extends extending to;; i128 only requires generating a 0 constant for the upper bits.(rule (lower (has_type $I128- (uextend (extractlane vec @ (value_type in)+ (uextend _ (extractlane _ vec @ (value_type in)(u8_from_uimm8 lane)))))(value_regs (mov_from_vec (put_in_reg vec) lane (lane_size in)) (imm $I64 (ImmExtend.Zero) 0)));; Zero extensions from a load can be encoded in the load itself-(rule (lower (has_type (fits_in_64 _) (uextend x @ (has_type in_ty (load (little_or_native_endian flags) address offset)))))+(rule (lower (has_type (fits_in_64 _) (uextend _ x @ (has_type in_ty (load _ (little_or_native_endian flags) address offset)))))(if-let inst (is_sinkable_inst x))(let ((_ Unit (sink_inst inst)))(aarch64_uload in_ty (amode in_ty address offset) flags)))@@ -1273,13 +1273,13 @@;; General rule for extending input to an output which fits in a single;; register.-(rule sextend -4 (lower (has_type (fits_in_64 out) (sextend x @ (value_type in))))+(rule sextend -4 (lower (has_type (fits_in_64 out) (sextend _ x @ (value_type in))))(extend x true (ty_bits in) (ty_bits out)));; Extraction of a vector lane automatically extends as necessary, so we can;; skip an explicit extending instruction.(rule -3 (lower (has_type (fits_in_64 out)- (sextend (extractlane vec @ (value_type in)+ (sextend _ (extractlane _ vec @ (value_type in)(u8_from_uimm8 lane)))))(mov_from_vec_signed (put_in_reg vec)lane@@ -1287,7 +1287,7 @@(size_from_ty out)));; 64-bit to 128-bit only needs to sign-extend the input to the upper bits.-(rule -2 (lower (has_type $I128 (sextend x)))+(rule -2 (lower (has_type $I128 (sextend _ x)))(let ((lo Reg (put_in_reg_sext64 x))(hi Reg (asr_imm $I64 lo (imm_shift_from_u8 63))))(value_regs lo hi)))@@ -1298,7 +1298,7 @@;; Note that `mov_from_vec_signed` doesn't exist for i64x2, so that's;; specifically excluded here.(rule (lower (has_type $I128- (sextend (extractlane vec @ (value_type in @ (not_i64x2))+ (sextend _ (extractlane _ vec @ (value_type in @ (not_i64x2))(u8_from_uimm8 lane)))))(let ((lo Reg (mov_from_vec_signed (put_in_reg vec)lane@@ -1309,7 +1309,7 @@;; Extension from an extraction of i64x2 into i128.(rule -1 (lower (has_type $I128- (sextend (extractlane vec @ (value_type $I64X2)+ (sextend _ (extractlane _ vec @ (value_type $I64X2)(u8_from_uimm8 lane)))))(let ((lo Reg (mov_from_vec (put_in_reg vec)lane@@ -1318,7 +1318,7 @@(value_regs lo hi)));; Signed extensions from a load can be encoded in the load itself-(rule (lower (has_type (fits_in_64 _) (sextend x @ (has_type in_ty (load (little_or_native_endian flags) address offset)))))+(rule (lower (has_type (fits_in_64 _) (sextend _ x @ (has_type in_ty (load _ (little_or_native_endian flags) address offset)))))(if-let inst (is_sinkable_inst x))(let ((_ Unit (sink_inst inst)))(aarch64_sload in_ty (amode in_ty address offset) flags)))@@ -1335,21 +1335,21 @@;; Note that bitwise negation is implemented here as;;;; NOT rd, rm ==> ORR_NOT rd, zero, rm-(rule bnot_base_case -4 (lower (has_type (fits_in_64 (ty_int ty)) (bnot x)))+(rule bnot_base_case -4 (lower (has_type (fits_in_64 (ty_int ty)) (bnot _ x)))(orr_not ty (zero_reg) x));; Implementation of `bnot` for floats.-(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bnot x)))+(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bnot _ x)))(not x (float_vector_size_in_64 ty)));; Implementation of `bnot` for vector types.-(rule -2 (lower (has_type (ty_vec64 ty) (bnot x)))+(rule -2 (lower (has_type (ty_vec64 ty) (bnot _ x)))(not x (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (bnot x)))+(rule -1 (lower (has_type (ty_vec128 ty) (bnot _ x)))(not x (vector_size ty)));; Implementation of `bnot` for `i128`.-(rule (lower (has_type $I128 (bnot x)))+(rule (lower (has_type $I128 (bnot _ x)))(let ((x_regs ValueRegs x)(x_lo Reg (value_regs_get x_regs 0))(x_hi Reg (value_regs_get x_regs 1))@@ -1360,87 +1360,87 @@;; Special case to use `orr_not_shift` if it's a `bnot` of a const-left-shifted;; value.(rule bnot_ishl 1 (lower (has_type (fits_in_64 (ty_int ty))- (bnot (ishl x (iconst k)))))+ (bnot _ (ishl _ x (iconst _ k)))))(if-let amt (lshl_from_imm64 ty k))(orr_not_shift ty (zero_reg) x amt));; Special-cases for fusing a bnot with bxor-(rule 2 (lower (has_type (fits_in_64 ty) (bnot (bxor x y))))+(rule 2 (lower (has_type (fits_in_64 ty) (bnot _ (bxor _ x y))))(alu_rs_imm_logic (ALUOp.EorNot) ty x y))-(rule 3 (lower (has_type $I128 (bnot (bxor x y)))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y))+(rule 3 (lower (has_type $I128 (bnot _ (bxor _ x y)))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y));;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule band_fits_in_64 -5 (lower (has_type (fits_in_64 (ty_int ty)) (band x y)))+(rule band_fits_in_64 -5 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x y)))(alu_rs_imm_logic_commutative (ALUOp.And) ty x y))-(rule -4 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (band x y)))+(rule -4 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (band _ x y)))(and_vec x y (float_vector_size_in_64 ty)));; Implementation of `band` for vector types.-(rule -2 (lower (has_type (ty_vec64 ty) (band x y)))+(rule -2 (lower (has_type (ty_vec64 ty) (band _ x y)))(and_vec x y (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (band x y)))+(rule -1 (lower (has_type (ty_vec128 ty) (band _ x y)))(and_vec x y (vector_size ty)))-(rule (lower (has_type $I128 (band x y))) (i128_alu_bitop (ALUOp.And) $I64 x y))+(rule (lower (has_type $I128 (band _ x y))) (i128_alu_bitop (ALUOp.And) $I64 x y));; Specialized lowerings for `(band x (bnot y))` which is additionally produced;; by Cranelift's `band_not` instruction that is legalized into the simpler;; forms early on.-(rule band_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (band x (bnot y))))+(rule band_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x (bnot _ y))))(alu_rs_imm_logic (ALUOp.AndNot) ty x y))-(rule band_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (band (bnot y) x)))+(rule band_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (band _ (bnot _ y) x)))(alu_rs_imm_logic (ALUOp.AndNot) ty x y))-(rule 3 (lower (has_type $I128 (band x (bnot y)))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))-(rule 4 (lower (has_type $I128 (band (bnot y) x))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))+(rule 3 (lower (has_type $I128 (band _ x (bnot _ y)))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))+(rule 4 (lower (has_type $I128 (band _ (bnot _ y) x))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))-(rule 5 (lower (has_type (ty_vec64 ty) (band x (bnot y))))+(rule 5 (lower (has_type (ty_vec64 ty) (band _ x (bnot _ y))))(bic_vec x y (vector_size ty)))-(rule 6 (lower (has_type (ty_vec64 ty) (band (bnot y) x)))+(rule 6 (lower (has_type (ty_vec64 ty) (band _ (bnot _ y) x)))(bic_vec x y (vector_size ty)))-(rule 7 (lower (has_type (ty_vec128 ty) (band x (bnot y))))+(rule 7 (lower (has_type (ty_vec128 ty) (band _ x (bnot _ y))))(bic_vec x y (vector_size ty)))-(rule 8 (lower (has_type (ty_vec128 ty) (band (bnot y) x)))+(rule 8 (lower (has_type (ty_vec128 ty) (band _ (bnot _ y) x)))(bic_vec x y (vector_size ty)));;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule bor_fits_in_64 -4 (lower (has_type (fits_in_64 (ty_int ty)) (bor x y)))+(rule bor_fits_in_64 -4 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x y)))(alu_rs_imm_logic_commutative (ALUOp.Orr) ty x y))-(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bor x y)))+(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bor _ x y)))(orr_vec x y (float_vector_size_in_64 ty)));; Implementation of `bor` for vector types.-(rule -2 (lower (has_type (ty_vec64 ty) (bor x y)))+(rule -2 (lower (has_type (ty_vec64 ty) (bor _ x y)))(orr_vec x y (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (bor x y)))+(rule -1 (lower (has_type (ty_vec128 ty) (bor _ x y)))(orr_vec x y (vector_size ty)))-(rule (lower (has_type $I128 (bor x y))) (i128_alu_bitop (ALUOp.Orr) $I64 x y))+(rule (lower (has_type $I128 (bor _ x y))) (i128_alu_bitop (ALUOp.Orr) $I64 x y));; Specialized lowerings for `(bor x (bnot y))` which is additionally produced;; by Cranelift's `bor_not` instruction that is legalized into the simpler;; forms early on.-(rule bor_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (bor x (bnot y))))+(rule bor_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x (bnot _ y))))(alu_rs_imm_logic (ALUOp.OrrNot) ty x y))-(rule bor_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (bor (bnot y) x)))+(rule bor_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ (bnot _ y) x)))(alu_rs_imm_logic (ALUOp.OrrNot) ty x y))-(rule 3 (lower (has_type $I128 (bor x (bnot y)))) (i128_alu_bitop (ALUOp.OrrNot) $I64 x y))-(rule 4 (lower (has_type $I128 (bor (bnot y) x))) (i128_alu_bitop (ALUOp.OrrNot) $I64 x y))+(rule 3 (lower (has_type $I128 (bor _ x (bnot _ y)))) (i128_alu_bitop (ALUOp.OrrNot) $I64 x y))+(rule 4 (lower (has_type $I128 (bor _ (bnot _ y) x))) (i128_alu_bitop (ALUOp.OrrNot) $I64 x y))-(rule bor_not_right_vec64 5 (lower (has_type (ty_vec64 ty) (bor x (bnot y))))+(rule bor_not_right_vec64 5 (lower (has_type (ty_vec64 ty) (bor _ x (bnot _ y))))(orn_vec x y (vector_size ty)))-(rule bor_not_left_vec64 6 (lower (has_type (ty_vec64 ty) (bor (bnot y) x)))+(rule bor_not_left_vec64 6 (lower (has_type (ty_vec64 ty) (bor _ (bnot _ y) x)))(orn_vec x y (vector_size ty)))-(rule bor_not_right_vec128 7 (lower (has_type (ty_vec128 ty) (bor x (bnot y))))+(rule bor_not_right_vec128 7 (lower (has_type (ty_vec128 ty) (bor _ x (bnot _ y))))(orn_vec x y (vector_size ty)))-(rule bor_not_left_vec128 8 (lower (has_type (ty_vec128 ty) (bor (bnot y) x)))+(rule bor_not_left_vec128 8 (lower (has_type (ty_vec128 ty) (bor _ (bnot _ y) x)))(orn_vec x y (vector_size ty)))@@ -1456,54 +1456,54 @@;;;; (x << xs) | (y >> ys) if (xs + ys == widthof(ty)) => extr(x, y, ys)(rule 5 (lower (has_type (ty_32_or_64 ty)- (bor (ishl x (u8_from_iconst xs)) (ushr y (u8_from_iconst ys)))))+ (bor _ (ishl _ x (u8_from_iconst xs)) (ushr _ y (u8_from_iconst ys)))))(if-let true (u64_eq (ty_bits ty) (u64_wrapping_add xs ys)))(a64_extr ty x y (imm_shift_from_u8 ys)))(rule 5 (lower (has_type (ty_32_or_64 ty)- (bor (ushr y (u8_from_iconst ys)) (ishl x (u8_from_iconst xs)))))+ (bor _ (ushr _ y (u8_from_iconst ys)) (ishl _ x (u8_from_iconst xs)))))(if-let true (u64_eq (ty_bits ty) (u64_wrapping_add xs ys)))(a64_extr ty x y (imm_shift_from_u8 ys)));;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule bxor_fits_in_64 -4 (lower (has_type (fits_in_64 (ty_int ty)) (bxor x y)))+(rule bxor_fits_in_64 -4 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x y)))(alu_rs_imm_logic_commutative (ALUOp.Eor) ty x y))-(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bxor x y)))+(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bxor _ x y)))(eor_vec x y (float_vector_size_in_64 ty)));; Implementation of `bxor` for vector types.-(rule -2 (lower (has_type (ty_vec64 ty) (bxor x y)))+(rule -2 (lower (has_type (ty_vec64 ty) (bxor _ x y)))(eor_vec x y (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (bxor x y)))+(rule -1 (lower (has_type (ty_vec128 ty) (bxor _ x y)))(eor_vec x y (vector_size ty)))-(rule (lower (has_type $I128 (bxor x y))) (i128_alu_bitop (ALUOp.Eor) $I64 x y))+(rule (lower (has_type $I128 (bxor _ x y))) (i128_alu_bitop (ALUOp.Eor) $I64 x y));; Specialized lowerings for `(bxor x (bnot y))` which is additionally produced;; by Cranelift's `bxor_not` instruction that is legalized into the simpler;; forms early on.-(rule bxor_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor x (bnot y))))+(rule bxor_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x (bnot _ y))))(alu_rs_imm_logic (ALUOp.EorNot) ty x y))-(rule bxor_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (bxor (bnot y) x)))+(rule bxor_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ (bnot _ y) x)))(alu_rs_imm_logic (ALUOp.EorNot) ty x y))-(rule 3 (lower (has_type $I128 (bxor x (bnot y)))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y))-(rule 4 (lower (has_type $I128 (bxor (bnot y) x))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y))+(rule 3 (lower (has_type $I128 (bxor _ x (bnot _ y)))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y))+(rule 4 (lower (has_type $I128 (bxor _ (bnot _ y) x))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y));;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Shift for i8/i16/i32.-(rule ishl_fits_in_32 -1 (lower (has_type (fits_in_32 ty) (ishl x y)))+(rule ishl_fits_in_32 -1 (lower (has_type (fits_in_32 ty) (ishl _ x y)))(do_shift (ALUOp.Lsl) ty x y));; Shift for i64.-(rule ishl_64 (lower (has_type $I64 (ishl x y)))+(rule ishl_64 (lower (has_type $I64 (ishl _ x y)))(do_shift (ALUOp.Lsl) $I64 x y));; Shift for i128.-(rule (lower (has_type $I128 (ishl x y)))+(rule (lower (has_type $I128 (ishl _ x y)))(lower_shl128 x (value_regs_get y 0)));; lsl lo_lshift, src_lo, amt@@ -1533,12 +1533,12 @@(csel (Cond.Ne) lo_lshift maybe_hi)))));; Shift for vector types.-(rule -3 (lower (has_type (ty_vec128 ty) (ishl x y)))+(rule -3 (lower (has_type (ty_vec128 ty) (ishl _ x y)))(let ((size VectorSize (vector_size ty))(masked_shift_amt Reg (and_imm $I32 y (shift_mask ty)))(shift Reg (vec_dup masked_shift_amt size)))(sshl x shift size)))-(rule -2 (lower (has_type (ty_vec128 ty) (ishl x (iconst (u64_from_imm64 n)))))+(rule -2 (lower (has_type (ty_vec128 ty) (ishl _ x (iconst _ (u64_from_imm64 n)))))(ushl_vec_imm x (shift_masked_imm ty n) (vector_size ty)))(decl pure shift_masked_imm (Type u64) u8)@@ -1619,22 +1619,22 @@;; Note that this rule explicitly has a higher priority than the others;; to ensure it's attempted first, otherwise the type-based filters on the;; previous rules seem to take priority over this rule.-(rule do_shift_imm 1 (do_shift op ty x (iconst k))+(rule do_shift_imm 1 (do_shift op ty x (iconst _ k))(if-let shift (imm_shift_from_imm64 ty k))(alu_rr_imm_shift op ty x shift));;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Shift for i8/i16/i32.-(rule ushr_fits_in_32 -1 (lower (has_type (fits_in_32 ty) (ushr x y)))+(rule ushr_fits_in_32 -1 (lower (has_type (fits_in_32 ty) (ushr _ x y)))(do_shift (ALUOp.Lsr) ty (put_in_reg_zext32 x) y));; Shift for i64.-(rule ushr_64 (lower (has_type $I64 (ushr x y)))+(rule ushr_64 (lower (has_type $I64 (ushr _ x y)))(do_shift (ALUOp.Lsr) $I64 (put_in_reg_zext64 x) y));; Shift for i128.-(rule (lower (has_type $I128 (ushr x y)))+(rule (lower (has_type $I128 (ushr _ x y)))(lower_ushr128 x (value_regs_get y 0)));; Vector shifts.@@ -1642,14 +1642,14 @@;; Note that for constant shifts a 0-width shift can't be emitted so it's;; special cased to pass through the input as-is since a 0-shift doesn't modify;; the input anyway.-(rule -4 (lower (has_type (ty_vec128 ty) (ushr x y)))+(rule -4 (lower (has_type (ty_vec128 ty) (ushr _ x y)))(let ((size VectorSize (vector_size ty))(masked_shift_amt Reg (and_imm $I32 y (shift_mask ty)))(shift Reg (vec_dup (sub $I64 (zero_reg) masked_shift_amt) size)))(ushl x shift size)))-(rule -3 (lower (has_type (ty_vec128 ty) (ushr x (iconst (u64_from_imm64 n)))))+(rule -3 (lower (has_type (ty_vec128 ty) (ushr _ x (iconst _ (u64_from_imm64 n)))))(ushr_vec_imm x (shift_masked_imm ty n) (vector_size ty)))-(rule -2 (lower (has_type (ty_vec128 ty) (ushr x (iconst (u64_from_imm64 n)))))+(rule -2 (lower (has_type (ty_vec128 ty) (ushr _ x (iconst _ (u64_from_imm64 n)))))(if-let 0 (shift_masked_imm ty n))x)@@ -1683,15 +1683,15 @@;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Shift for i8/i16/i32.-(rule sshr_fits_in_32 -4 (lower (has_type (fits_in_32 ty) (sshr x y)))+(rule sshr_fits_in_32 -4 (lower (has_type (fits_in_32 ty) (sshr _ x y)))(do_shift (ALUOp.Asr) ty (put_in_reg_sext32 x) y));; Shift for i64.-(rule sshr_64 (lower (has_type $I64 (sshr x y)))+(rule sshr_64 (lower (has_type $I64 (sshr _ x y)))(do_shift (ALUOp.Asr) $I64 (put_in_reg_sext64 x) y));; Shift for i128.-(rule (lower (has_type $I128 (sshr x y)))+(rule (lower (has_type $I128 (sshr _ x y)))(lower_sshr128 x (value_regs_get y 0)));; Vector shifts.@@ -1700,14 +1700,14 @@;; that for constant shifts a 0-width shift can't be emitted so it's special;; cased to pass through the input as-is since a 0-shift doesn't modify the;; input anyway.-(rule -3 (lower (has_type (ty_vec128 ty) (sshr x y)))+(rule -3 (lower (has_type (ty_vec128 ty) (sshr _ x y)))(let ((size VectorSize (vector_size ty))(masked_shift_amt Reg (and_imm $I32 y (shift_mask ty)))(shift Reg (vec_dup (sub $I64 (zero_reg) masked_shift_amt) size)))(sshl x shift size)))-(rule -2 (lower (has_type (ty_vec128 ty) (sshr x (iconst (u64_from_imm64 n)))))+(rule -2 (lower (has_type (ty_vec128 ty) (sshr _ x (iconst _ (u64_from_imm64 n)))))(sshr_vec_imm x (shift_masked_imm ty n) (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (sshr x (iconst (u64_from_imm64 n)))))+(rule -1 (lower (has_type (ty_vec128 ty) (sshr _ x (iconst _ (u64_from_imm64 n)))))(if-let 0 (shift_masked_imm ty n))x)@@ -1743,13 +1743,13 @@;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; General 8/16-bit case.-(rule rotl_fits_in_16 -2 (lower (has_type (fits_in_16 ty) (rotl x y)))+(rule rotl_fits_in_16 -2 (lower (has_type (fits_in_16 ty) (rotl _ x y)))(let ((amt Reg (value_regs_get y 0))(neg_shift Reg (sub $I32 (zero_reg) amt)))(small_rotr ty (put_in_reg_zext32 x) neg_shift)));; Specialization for the 8/16-bit case when the rotation amount is an immediate.-(rule rotl_fits_in_16_imm -1 (lower (has_type (fits_in_16 ty) (rotl x (iconst k))))+(rule rotl_fits_in_16_imm -1 (lower (has_type (fits_in_16 ty) (rotl _ x (iconst _ k))))(if-let n (imm_shift_from_imm64 ty k))(small_rotr_imm ty (put_in_reg_zext32 x) (negate_imm_shift ty n)))@@ -1762,24 +1762,24 @@;; amount.;; General 32-bit case.-(rule rotl_32_base_case (lower (has_type $I32 (rotl x y)))+(rule rotl_32_base_case (lower (has_type $I32 (rotl _ x y)))(let ((amt Reg (value_regs_get y 0))(neg_shift Reg (sub $I32 (zero_reg) amt)))(a64_rotr $I32 x neg_shift)));; General 64-bit case.-(rule rotl_64_base_case (lower (has_type $I64 (rotl x y)))+(rule rotl_64_base_case (lower (has_type $I64 (rotl _ x y)))(let ((amt Reg (value_regs_get y 0))(neg_shift Reg (sub $I64 (zero_reg) amt)))(a64_rotr $I64 x neg_shift)));; Specialization for the 32-bit case when the rotation amount is an immediate.-(rule rotl_32_imm 1 (lower (has_type $I32 (rotl x (iconst k))))+(rule rotl_32_imm 1 (lower (has_type $I32 (rotl _ x (iconst _ k))))(if-let n (imm_shift_from_imm64 $I32 k))(a64_rotr_imm $I32 x (negate_imm_shift $I32 n)));; Specialization for the 64-bit case when the rotation amount is an immediate.-(rule rotl_64_imm 1 (lower (has_type $I64 (rotl x (iconst k))))+(rule rotl_64_imm 1 (lower (has_type $I64 (rotl _ x (iconst _ k))))(if-let n (imm_shift_from_imm64 $I64 k))(a64_rotr_imm $I64 x (negate_imm_shift $I64 n)))@@ -1798,7 +1798,7 @@;; General 128-bit case.;;;; TODO: much better codegen is possible with a constant amount.-(rule (lower (has_type $I128 (rotl x y)))+(rule (lower (has_type $I128 (rotl _ x y)))(let ((val ValueRegs x)(amt Reg (value_regs_get y 0))(neg_amt Reg (sub $I64 (imm $I64 (ImmExtend.Zero) 128) amt))@@ -1811,29 +1811,29 @@;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; General 8/16-bit case.-(rule rotr_fits_in_16 -3 (lower (has_type (fits_in_16 ty) (rotr x y)))+(rule rotr_fits_in_16 -3 (lower (has_type (fits_in_16 ty) (rotr _ x y)))(small_rotr ty (put_in_reg_zext32 x) (value_regs_get y 0)));; General 32-bit case.-(rule rotr_32_base_case -1 (lower (has_type $I32 (rotr x y)))+(rule rotr_32_base_case -1 (lower (has_type $I32 (rotr _ x y)))(a64_rotr $I32 x (value_regs_get y 0)));; General 64-bit case.-(rule rotr_64_base_case -1 (lower (has_type $I64 (rotr x y)))+(rule rotr_64_base_case -1 (lower (has_type $I64 (rotr _ x y)))(a64_rotr $I64 x (value_regs_get y 0)));; Specialization for the 8/16-bit case when the rotation amount is an immediate.-(rule rotr_fits_in_16_imm -2 (lower (has_type (fits_in_16 ty) (rotr x (iconst k))))+(rule rotr_fits_in_16_imm -2 (lower (has_type (fits_in_16 ty) (rotr _ x (iconst _ k))))(if-let n (imm_shift_from_imm64 ty k))(small_rotr_imm ty (put_in_reg_zext32 x) n));; Specialization for the 32-bit case when the rotation amount is an immediate.-(rule rotr_32_imm (lower (has_type $I32 (rotr x (iconst k))))+(rule rotr_32_imm (lower (has_type $I32 (rotr _ x (iconst _ k))))(if-let n (imm_shift_from_imm64 $I32 k))(a64_rotr_imm $I32 x n));; Specialization for the 64-bit case when the rotation amount is an immediate.-(rule rotr_64_imm (lower (has_type $I64 (rotr x (iconst k))))+(rule rotr_64_imm (lower (has_type $I64 (rotr _ x (iconst _ k))))(if-let n (imm_shift_from_imm64 $I64 k))(a64_rotr_imm $I64 x n))@@ -1914,7 +1914,7 @@;; General 128-bit case.;;;; TODO: much better codegen is possible with a constant amount.-(rule (lower (has_type $I128 (rotr x y)))+(rule (lower (has_type $I128 (rotr _ x y)))(let ((val ValueRegs x)(amt Reg (value_regs_get y 0))(neg_amt Reg (sub $I64 (imm $I64 (ImmExtend.Zero) 128) amt))@@ -1929,37 +1929,37 @@;; Reversing an 8-bit value with a 32-bit bitrev instruction will place;; the reversed result in the highest 8 bits, so we need to shift them down into;; place.-(rule (lower (has_type $I8 (bitrev x)))+(rule (lower (has_type $I8 (bitrev _ x)))(lsr_imm $I32 (rbit $I32 x) (imm_shift_from_u8 24)));; Reversing an 16-bit value with a 32-bit bitrev instruction will place;; the reversed result in the highest 16 bits, so we need to shift them down into;; place.-(rule (lower (has_type $I16 (bitrev x)))+(rule (lower (has_type $I16 (bitrev _ x)))(lsr_imm $I32 (rbit $I32 x) (imm_shift_from_u8 16)))-(rule (lower (has_type $I128 (bitrev x)))+(rule (lower (has_type $I128 (bitrev _ x)))(let ((val ValueRegs x)(lo_rev Reg (rbit $I64 (value_regs_get val 0)))(hi_rev Reg (rbit $I64 (value_regs_get val 1))))(value_regs hi_rev lo_rev)))-(rule -1 (lower (has_type ty (bitrev x)))+(rule -1 (lower (has_type ty (bitrev _ x)))(rbit ty x));;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule clz_8 (lower (has_type $I8 (clz x)))+(rule clz_8 (lower (has_type $I8 (clz _ x)))(sub_imm $I32 (a64_clz $I32 (put_in_reg_zext32 x)) (u8_into_imm12 24)))-(rule clz_16 (lower (has_type $I16 (clz x)))+(rule clz_16 (lower (has_type $I16 (clz _ x)))(sub_imm $I32 (a64_clz $I32 (put_in_reg_zext32 x)) (u8_into_imm12 16)))-(rule (lower (has_type $I128 (clz x)))+(rule (lower (has_type $I128 (clz _ x)))(lower_clz128 x))-(rule clz_32_64 -1 (lower (has_type ty (clz x)))+(rule clz_32_64 -1 (lower (has_type ty (clz _ x)))(a64_clz ty x));; clz hi_clz, hi@@ -1980,27 +1980,27 @@;; then using a `clz` instruction since the tail zeros are the same as the;; leading zeros of the reversed value.-(rule ctz_8 (lower (has_type $I8 (ctz x)))+(rule ctz_8 (lower (has_type $I8 (ctz _ x)))(a64_clz $I32 (orr_imm $I32 (rbit $I32 x) (u64_into_imm_logic $I32 0x800000))))-(rule ctz_16 (lower (has_type $I16 (ctz x)))+(rule ctz_16 (lower (has_type $I16 (ctz _ x)))(a64_clz $I32 (orr_imm $I32 (rbit $I32 x) (u64_into_imm_logic $I32 0x8000))))-(rule (lower (has_type $I128 (ctz x)))+(rule (lower (has_type $I128 (ctz _ x)))(let ((val ValueRegs x)(lo Reg (rbit $I64 (value_regs_get val 0)))(hi Reg (rbit $I64 (value_regs_get val 1))))(lower_clz128 (value_regs hi lo))))-(rule ctz_32_64 -1 (lower (has_type ty (ctz x)))+(rule ctz_32_64 -1 (lower (has_type ty (ctz _ x)))(a64_clz ty (rbit ty x)));;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule cls_8 (lower (has_type $I8 (cls x)))+(rule cls_8 (lower (has_type $I8 (cls _ x)))(sub_imm $I32 (a64_cls $I32 (put_in_reg_sext32 x)) (u8_into_imm12 24)))-(rule cls_16 (lower (has_type $I16 (cls x)))+(rule cls_16 (lower (has_type $I16 (cls _ x)))(sub_imm $I32 (a64_cls $I32 (put_in_reg_sext32 x)) (u8_into_imm12 16)));; cls lo_cls, lo@@ -2012,7 +2012,7 @@;; csel maybe_lo, lo_sign_bits, xzr, eq;; add out_lo, maybe_lo, hi_cls;; mov out_hi, 0-(rule (lower (has_type $I128 (cls x)))+(rule (lower (has_type $I128 (cls _ x)))(let ((val ValueRegs x)(lo Reg (value_regs_get val 0))(hi Reg (value_regs_get val 1))@@ -2026,21 +2026,21 @@(csel (Cond.Eq) lo_sign_bits (zero_reg)))))(value_regs (add $I64 maybe_lo hi_cls) (imm $I64 (ImmExtend.Zero) 0))))-(rule cls_32_64 -1 (lower (has_type ty (cls x)))+(rule cls_32_64 -1 (lower (has_type ty (cls _ x)))(a64_cls ty x));;;; Rules for `bswap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16 (bswap x)))+(rule (lower (has_type $I16 (bswap _ x)))(a64_rev16 $I16 x))-(rule (lower (has_type $I32 (bswap x)))+(rule (lower (has_type $I32 (bswap _ x)))(a64_rev32 $I32 x))-(rule (lower (has_type $I64 (bswap x)))+(rule (lower (has_type $I64 (bswap _ x)))(a64_rev64 $I64 x))-(rule (lower (has_type $I128 (bswap x)))+(rule (lower (has_type $I128 (bswap _ x)))(value_regs(a64_rev64 $I64 (value_regs_get x 1))(a64_rev64 $I64 (value_regs_get x 0))))@@ -2048,7 +2048,7 @@;;;; Rules for `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Bmask tests the value against zero, and uses `csetm` to assert the result.-(rule (lower (has_type out_ty (bmask x @ (value_type in_ty))))+(rule (lower (has_type out_ty (bmask _ x @ (value_type in_ty))))(lower_bmask out_ty in_ty x));;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2070,31 +2070,31 @@;; if ty == i128:;; mov out_hi, 0-(rule popcnt_8 (lower (has_type $I8 (popcnt x)))+(rule popcnt_8 (lower (has_type $I8 (popcnt _ x)))(let ((tmp Reg (mov_to_fpu x (ScalarSize.Size32)))(nbits Reg (vec_cnt tmp (VectorSize.Size8x8))))(mov_from_vec nbits 0 (ScalarSize.Size8))));; Note that this uses `addp` instead of `addv` as it's usually cheaper.-(rule popcnt_16 (lower (has_type $I16 (popcnt x)))+(rule popcnt_16 (lower (has_type $I16 (popcnt _ x)))(let ((tmp Reg (mov_to_fpu x (ScalarSize.Size32)))(nbits Reg (vec_cnt tmp (VectorSize.Size8x8)))(added Reg (addp nbits nbits (VectorSize.Size8x8))))(mov_from_vec added 0 (ScalarSize.Size8))))-(rule popcnt_32 (lower (has_type $I32 (popcnt x)))+(rule popcnt_32 (lower (has_type $I32 (popcnt _ x)))(let ((tmp Reg (mov_to_fpu x (ScalarSize.Size32)))(nbits Reg (vec_cnt tmp (VectorSize.Size8x8)))(added Reg (addv nbits (VectorSize.Size8x8))))(mov_from_vec added 0 (ScalarSize.Size8))))-(rule popcnt_64 (lower (has_type $I64 (popcnt x)))+(rule popcnt_64 (lower (has_type $I64 (popcnt _ x)))(let ((tmp Reg (mov_to_fpu x (ScalarSize.Size64)))(nbits Reg (vec_cnt tmp (VectorSize.Size8x8)))(added Reg (addv nbits (VectorSize.Size8x8))))(mov_from_vec added 0 (ScalarSize.Size8))))-(rule (lower (has_type $I128 (popcnt x)))+(rule (lower (has_type $I128 (popcnt _ x)))(let ((val ValueRegs x)(tmp_half Reg (mov_to_fpu (value_regs_get val 0) (ScalarSize.Size64)))(tmp Reg (mov_to_vec tmp_half (value_regs_get val 1) 1 (VectorSize.Size64x2)))@@ -2102,18 +2102,18 @@(added Reg (addv nbits (VectorSize.Size8x16))))(value_regs (mov_from_vec added 0 (ScalarSize.Size8)) (imm $I64 (ImmExtend.Zero) 0))))-(rule (lower (has_type $I8X16 (popcnt x)))+(rule (lower (has_type $I8X16 (popcnt _ x)))(vec_cnt x (VectorSize.Size8x16)));;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule bitselect (lower (has_type ty (bitselect c x y)))+(rule bitselect (lower (has_type ty (bitselect _ c x y)))(if (ty_int_ref_scalar_64 ty))(let ((tmp1 Reg (and_reg ty x c))(tmp2 Reg (bic ty y c)))(orr ty tmp1 tmp2)))-(rule 1 (lower (has_type (ty_vec128 ty) (bitselect c x y)))+(rule 1 (lower (has_type (ty_vec128 ty) (bitselect _ c x y)))(bsl ty c x y));;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2121,72 +2121,72 @@;; T -> I{64,32,16,8}: We can simply pass through the value: values;; are always stored with high bits undefined, so we can just leave;; them be.-(rule (lower (has_type ty (ireduce src)))+(rule (lower (has_type ty (ireduce _ src)))(if (ty_int_ref_scalar_64 ty))(value_regs_get src 0));;;; Rules for `fcmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 4 (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond_not_eq cond) x y)))+(rule 4 (lower (has_type ty @ (multi_lane _ _) (fcmp _ (fcmp_zero_cond_not_eq cond) x y)))(if (zero_value y))(let ((rn Reg x)(vec_size VectorSize (vector_size ty)))(value_reg (not (fcmeq0 rn vec_size) vec_size))))-(rule 3 (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond cond) x y)))+(rule 3 (lower (has_type ty @ (multi_lane _ _) (fcmp _ (fcmp_zero_cond cond) x y)))(if (zero_value y))(let ((rn Reg x)(vec_size VectorSize (vector_size ty)))(value_reg (float_cmp_zero cond rn vec_size))))-(rule 2 (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond_not_eq cond) x y)))+(rule 2 (lower (has_type ty @ (multi_lane _ _) (fcmp _ (fcmp_zero_cond_not_eq cond) x y)))(if (zero_value x))(let ((rn Reg y)(vec_size VectorSize (vector_size ty)))(value_reg (not (fcmeq0 rn vec_size) vec_size))))-(rule 1 (lower (has_type ty @ (multi_lane _ _) (fcmp (fcmp_zero_cond cond) x y)))+(rule 1 (lower (has_type ty @ (multi_lane _ _) (fcmp _ (fcmp_zero_cond cond) x y)))(if (zero_value x))(let ((rn Reg y)(vec_size VectorSize (vector_size ty)))(value_reg (float_cmp_zero_swap cond rn vec_size))))(rule 0 (lower (has_type out_ty- (fcmp cond x @ (value_type (ty_scalar_float in_ty)) y)))+ (fcmp _ cond x @ (value_type (ty_scalar_float in_ty)) y)))(with_flags (fpu_cmp (scalar_size in_ty) x y)(materialize_bool_result (fp_cond_code cond))))-(rule -1 (lower (has_type out_ty (fcmp cond x @ (value_type in_ty) y)))+(rule -1 (lower (has_type out_ty (fcmp _ cond x @ (value_type in_ty) y)))(if (ty_vector_float in_ty))(vec_cmp x y in_ty (fp_cond_code cond)));;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 3 (lower (has_type ty @ (multi_lane _ _) (icmp (icmp_zero_cond_not_eq cond) x y)))+(rule 3 (lower (has_type ty @ (multi_lane _ _) (icmp _ (icmp_zero_cond_not_eq cond) x y)))(if (zero_value y))(let ((rn Reg x)(vec_size VectorSize (vector_size ty)))(value_reg (not (cmeq0 rn vec_size) vec_size))))-(rule 2 (lower (has_type ty @ (multi_lane _ _) (icmp (icmp_zero_cond cond) x y)))+(rule 2 (lower (has_type ty @ (multi_lane _ _) (icmp _ (icmp_zero_cond cond) x y)))(if (zero_value y))(let ((rn Reg x)(vec_size VectorSize (vector_size ty)))(value_reg (int_cmp_zero cond rn vec_size))))-(rule 1 (lower (has_type ty @ (multi_lane _ _) (icmp (icmp_zero_cond_not_eq cond) x y)))+(rule 1 (lower (has_type ty @ (multi_lane _ _) (icmp _ (icmp_zero_cond_not_eq cond) x y)))(if (zero_value x))(let ((rn Reg y)(vec_size VectorSize (vector_size ty)))(value_reg (not (cmeq0 rn vec_size) vec_size))))-(rule 0 (lower (has_type ty @ (multi_lane _ _) (icmp (icmp_zero_cond cond) x y)))+(rule 0 (lower (has_type ty @ (multi_lane _ _) (icmp _ (icmp_zero_cond cond) x y)))(if (zero_value x))(let ((rn Reg y)(vec_size VectorSize (vector_size ty)))(value_reg (int_cmp_zero_swap cond rn vec_size))))-(rule icmp_8_16_32_64 -1 (lower (icmp cond x @ (value_type in_ty) y))+(rule icmp_8_16_32_64 -1 (lower (icmp _ cond x @ (value_type in_ty) y))(lower_icmp_into_reg cond x y in_ty $I8));;;; Rules for `trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2207,7 +2207,7 @@;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type ty- (select (maybe_uextend (icmp cc+ (select _ (maybe_uextend (icmp _ ccx @ (value_type in_ty)y))rn@@ -2220,7 +2220,7 @@rm)))(rule (lower (has_type ty- (select (maybe_uextend (fcmp cc x @ (value_type in_ty) y))+ (select _ (maybe_uextend (fcmp _ cc x @ (value_type in_ty) y))rnrm)))(let ((cond Cond (fp_cond_code cc)))@@ -2228,25 +2228,25 @@(fpu_cmp (scalar_size in_ty) x y)cond ty rn rm)))-(rule -1 (lower (has_type ty (select rcond @ (value_type $I8) rn rm)))+(rule -1 (lower (has_type ty (select _ rcond @ (value_type $I8) rn rm)))(let ((rcond Reg rcond))(lower_select(tst_imm $I32 rcond (u64_into_imm_logic $I32 255))(Cond.Ne) ty rn rm)))-(rule -2 (lower (has_type ty (select rcond @ (value_type (fits_in_32 _)) rn rm)))+(rule -2 (lower (has_type ty (select _ rcond @ (value_type (fits_in_32 _)) rn rm)))(let ((rcond Reg (put_in_reg_zext32 rcond)))(lower_select(cmp (OperandSize.Size32) rcond (zero_reg))(Cond.Ne) ty rn rm)))-(rule -3 (lower (has_type ty (select rcond @ (value_type (fits_in_64 _)) rn rm)))+(rule -3 (lower (has_type ty (select _ rcond @ (value_type (fits_in_64 _)) rn rm)))(let ((rcond Reg (put_in_reg_zext64 rcond)))(lower_select(cmp (OperandSize.Size64) rcond (zero_reg))(Cond.Ne) ty rn rm)))-(rule -4 (lower (has_type ty (select rcond @ (value_type $I128) rn rm)))+(rule -4 (lower (has_type ty (select _ rcond @ (value_type $I128) rn rm)))(let ((c ValueRegs (put_in_regs rcond))(c_lo Reg (value_regs_get c 0))(c_hi Reg (value_regs_get c 1))@@ -2258,7 +2258,7 @@;;;; Rules for `select_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type ty- (select_spectre_guard (maybe_uextend (icmp cc x @ (value_type in_ty) y))+ (select_spectre_guard _ (maybe_uextend (icmp _ cc x @ (value_type in_ty) y))if_trueif_false)))(let ((comparison FlagsAndCC (lower_icmp_into_flags cc x y in_ty))@@ -2271,13 +2271,13 @@(_ InstOutput (side_effect (csdb))))dst))-(rule -1 (lower (has_type ty (select_spectre_guard rcond @ (value_type (fits_in_64 _)) rn rm)))+(rule -1 (lower (has_type ty (select_spectre_guard _ rcond @ (value_type (fits_in_64 _)) rn rm)))(let ((rcond Reg (put_in_reg_zext64 rcond)))(lower_select(cmp (OperandSize.Size64) rcond (zero_reg))(Cond.Ne) ty rn rm)))-(rule -2 (lower (has_type ty (select_spectre_guard rcond @ (value_type $I128) rn rm)))+(rule -2 (lower (has_type ty (select_spectre_guard _ rcond @ (value_type $I128) rn rm)))(let ((c ValueRegs (put_in_regs rcond))(c_lo Reg (value_regs_get c 0))(c_hi Reg (value_regs_get c 1))@@ -2288,41 +2288,41 @@;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 _) (vconst (u128_from_constant x))))+(rule (lower (has_type (ty_vec128 _) (vconst _ (u128_from_constant x))))(constant_f128 x))-(rule 1 (lower (has_type ty (vconst (u64_from_constant x))))+(rule 1 (lower (has_type ty (vconst _ (u64_from_constant x))))(if (ty_vec64 ty))(constant_f64 x));;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty (splat x @ (value_type in_ty))))+(rule -1 (lower (has_type ty (splat _ x @ (value_type in_ty))))(if (ty_int_ref_scalar_64 in_ty))(vec_dup x (vector_size ty)))-(rule -2 (lower (has_type ty (splat x @ (value_type (ty_scalar_float _)))))+(rule -2 (lower (has_type ty (splat _ x @ (value_type (ty_scalar_float _)))))(vec_dup_from_fpu x (vector_size ty) 0))-(rule (lower (has_type ty (splat (f32const (u32_from_ieee32 n)))))+(rule (lower (has_type ty (splat _ (f32const _ (u32_from_ieee32 n)))))(splat_const n (vector_size ty)))-(rule (lower (has_type ty (splat (f64const (u64_from_ieee64 n)))))+(rule (lower (has_type ty (splat _ (f64const _ (u64_from_ieee64 n)))))(splat_const n (vector_size ty)))-(rule (lower (has_type ty (splat (iconst (u64_from_imm64 n)))))+(rule (lower (has_type ty (splat _ (iconst _ (u64_from_imm64 n)))))(splat_const n (vector_size ty)))-(rule (lower (has_type ty (splat (ireduce (iconst (u64_from_imm64 n))))))+(rule (lower (has_type ty (splat _ (ireduce _ (iconst _ (u64_from_imm64 n))))))(splat_const n (vector_size ty)))-(rule (lower (has_type ty (splat x @ (load flags _ _))))+(rule (lower (has_type ty (splat _ x @ (load _ flags _ _))))(if-let mem_op (is_sinkable_inst x))(let ((addr Reg (sink_load_into_addr (lane_type ty) mem_op)))(ld1r addr (vector_size ty) flags)));;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (valid_atomic_transaction ty) (atomic_load (little_or_native_endian flags) addr)))+(rule (lower (has_type (valid_atomic_transaction ty) (atomic_load _ (little_or_native_endian flags) addr)))(load_acquire ty flags addr))@@ -2336,116 +2336,116 @@(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Add) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Add) addr src))))(lse_atomic_rmw (AtomicRMWOp.Add) addr src ty flags))(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src))))(lse_atomic_rmw (AtomicRMWOp.Eor) addr src ty flags))(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Or) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Or) addr src))))(lse_atomic_rmw (AtomicRMWOp.Set) addr src ty flags))(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src))))(lse_atomic_rmw (AtomicRMWOp.Smax) addr src ty flags))(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src))))(lse_atomic_rmw (AtomicRMWOp.Smin) addr src ty flags))(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src))))(lse_atomic_rmw (AtomicRMWOp.Umax) addr src ty flags))(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src))))(lse_atomic_rmw (AtomicRMWOp.Umin) addr src ty flags))(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src))))(lse_atomic_rmw (AtomicRMWOp.Add) addr (sub ty (zero_reg) src) ty flags))(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.And) addr src))))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.And) addr src))))(lse_atomic_rmw (AtomicRMWOp.Clr) addr (eon ty src (zero_reg)) ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Add) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Add) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Add) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Sub) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.And) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.And) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.And) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Nand) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Nand) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Nand) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Or) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Or) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Orr) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Eor) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Smin) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Smax) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Umin) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Umax) addr src ty flags))(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Xchg) addr src)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xchg) addr src)))(atomic_rmw_loop (AtomicRMWLoopOp.Xchg) addr src ty flags));;;; Rules for `AtomicCAS` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule 1 (lower (and (use_lse)(has_type (valid_atomic_transaction ty)- (atomic_cas (little_or_native_endian flags) addr src1 src2))))+ (atomic_cas _ (little_or_native_endian flags) addr src1 src2))))(lse_atomic_cas addr src1 src2 ty flags))(rule (lower (and (has_type (valid_atomic_transaction ty)- (atomic_cas (little_or_native_endian flags) addr src1 src2))))+ (atomic_cas _ (little_or_native_endian flags) addr src1 src2))))(atomic_cas_loop addr src1 src2 ty flags));;;; Rules for 'fvdemote' ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (fvdemote x))+(rule (lower (fvdemote _ x))(fcvtn x (ScalarSize.Size32)));;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128_int ty) (snarrow x y)))+(rule 1 (lower (has_type (ty_vec128_int ty) (snarrow _ x y)))(if (zero_value y))(sqxtn x (lane_size ty)))-(rule 2 (lower (has_type (ty_vec64_int ty) (snarrow x y)))+(rule 2 (lower (has_type (ty_vec64_int ty) (snarrow _ x y)))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(sqxtn dst (lane_size ty))))-(rule 0 (lower (has_type (ty_vec128_int ty) (snarrow x y)))+(rule 0 (lower (has_type (ty_vec128_int ty) (snarrow _ x y)))(let ((low_half Reg (sqxtn x (lane_size ty)))(result Reg (sqxtn2 low_half y (lane_size ty))))result));;;; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128_int ty) (unarrow x y)))+(rule 1 (lower (has_type (ty_vec128_int ty) (unarrow _ x y)))(if (zero_value y))(sqxtun x (lane_size ty)))-(rule 2 (lower (has_type (ty_vec64_int ty) (unarrow x y)))+(rule 2 (lower (has_type (ty_vec64_int ty) (unarrow _ x y)))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(sqxtun dst (lane_size ty))))-(rule 0 (lower (has_type (ty_vec128_int ty) (unarrow x y)))+(rule 0 (lower (has_type (ty_vec128_int ty) (unarrow _ x y)))(let ((low_half Reg (sqxtun x (lane_size ty)))(result Reg (sqxtun2 low_half y (lane_size ty))))result))@@ -2453,45 +2453,45 @@;;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128_int ty) (uunarrow x y)))+(rule 1 (lower (has_type (ty_vec128_int ty) (uunarrow _ x y)))(if (zero_value y))(uqxtn x (lane_size ty)))-(rule 2 (lower (has_type (ty_vec64_int ty) (uunarrow x y)))+(rule 2 (lower (has_type (ty_vec64_int ty) (uunarrow _ x y)))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(uqxtn dst (lane_size ty))))-(rule 0 (lower (has_type (ty_vec128_int ty) (uunarrow x y)))+(rule 0 (lower (has_type (ty_vec128_int ty) (uunarrow _ x y)))(let ((low_half Reg (uqxtn x (lane_size ty)))(result Reg (uqxtn2 low_half y (lane_size ty))))result));;;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (swiden_low x)))+(rule (lower (has_type ty (swiden_low _ x)))(vec_extend (VecExtendOp.Sxtl) x false (lane_size ty)));;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128 ty) (swiden_high x)))+(rule 1 (lower (has_type (ty_vec128 ty) (swiden_high _ x)))(vec_extend (VecExtendOp.Sxtl) x true (lane_size ty)))-(rule (lower (has_type ty (swiden_high x)))+(rule (lower (has_type ty (swiden_high _ x)))(if (ty_vec64 ty))(let ((tmp Reg (fpu_move_from_vec x 1 (VectorSize.Size32x2))))(vec_extend (VecExtendOp.Sxtl) tmp false (lane_size ty))));;;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (uwiden_low x)))+(rule (lower (has_type ty (uwiden_low _ x)))(vec_extend (VecExtendOp.Uxtl) x false (lane_size ty)));;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128 ty) (uwiden_high x)))+(rule 1 (lower (has_type (ty_vec128 ty) (uwiden_high _ x)))(vec_extend (VecExtendOp.Uxtl) x true (lane_size ty)))-(rule (lower (has_type ty (uwiden_high x)))+(rule (lower (has_type ty (uwiden_high _ x)))(if (ty_vec64 ty))(let ((tmp Reg (fpu_move_from_vec x 1 (VectorSize.Size32x2))))(vec_extend (VecExtendOp.Uxtl) tmp false (lane_size ty))))@@ -2508,23 +2508,23 @@;;;; Rules for `func_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (func_addr (func_ref_data _ extname dist _)))+(rule (lower (func_addr _ (func_ref_data _ extname dist _)))(load_ext_name (box_external_name extname) 0 dist));;;; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (symbol_value (symbol_value_data extname dist offset)))+(rule (lower (symbol_value _ (symbol_value_data extname dist offset)))(load_ext_name (box_external_name extname) offset dist));;; Rules for `get_{frame,stack}_pointer` and `get_return_address` ;;;;;;;;;;;;;-(rule (lower (get_frame_pointer))+(rule (lower (get_frame_pointer _))(aarch64_fp))-(rule (lower (get_stack_pointer))+(rule (lower (get_stack_pointer _))(aarch64_sp))-(rule (lower (get_return_address))+(rule (lower (get_return_address _))(aarch64_link));; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2627,92 +2627,92 @@;;;; Rules for loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule load_i8_aarch64_uload8 (lower- (has_type $I8 (load (little_or_native_endian flags) address offset)))+ (has_type $I8 (load _ (little_or_native_endian flags) address offset)))(aarch64_uload8 (amode $I8 address offset) flags))(rule load_i16_aarch64_uload16 (lower- (has_type $I16 (load (little_or_native_endian flags) address offset)))+ (has_type $I16 (load _ (little_or_native_endian flags) address offset)))(aarch64_uload16 (amode $I16 address offset) flags))(rule load_i32_aarch64_uload32 (lower- (has_type $I32 (load (little_or_native_endian flags) address offset)))+ (has_type $I32 (load _ (little_or_native_endian flags) address offset)))(aarch64_uload32 (amode $I32 address offset) flags))(rule load_i64_aarch64_uload64 (lower- (has_type $I64 (load (little_or_native_endian flags) address offset)))+ (has_type $I64 (load _ (little_or_native_endian flags) address offset)))(aarch64_uload64 (amode $I64 address offset) flags))(rule (lower- (has_type $I128 (load (little_or_native_endian flags) address offset)))+ (has_type $I128 (load _ (little_or_native_endian flags) address offset)))(aarch64_loadp64 (pair_amode address offset) flags))(rule -1 (lower- (has_type (ty_float_or_vec (ty_16 _)) (load (little_or_native_endian flags) address offset)))+ (has_type (ty_float_or_vec (ty_16 _)) (load _ (little_or_native_endian flags) address offset)))(aarch64_fpuload16 (amode $F16 address offset) flags))(rule -2 (lower- (has_type (ty_float_or_vec (ty_32 _)) (load (little_or_native_endian flags) address offset)))+ (has_type (ty_float_or_vec (ty_32 _)) (load _ (little_or_native_endian flags) address offset)))(aarch64_fpuload32 (amode $F32 address offset) flags))(rule -3 (lower- (has_type (ty_float_or_vec (ty_64 _)) (load (little_or_native_endian flags) address offset)))+ (has_type (ty_float_or_vec (ty_64 _)) (load _ (little_or_native_endian flags) address offset)))(aarch64_fpuload64 (amode $F64 address offset) flags))(rule -4 (lower- (has_type (ty_float_or_vec (ty_128 _)) (load (little_or_native_endian flags) address offset)))+ (has_type (ty_float_or_vec (ty_128 _)) (load _ (little_or_native_endian flags) address offset)))(aarch64_fpuload128 (amode $F128 address offset) flags))(rule -5 (lower(has_type (ty_dyn_vec64 _)- (load (little_or_native_endian flags) address offset)))+ (load _ (little_or_native_endian flags) address offset)))(aarch64_fpuload64 (amode $F64 address offset) flags))(rule -6 (lower(has_type (ty_dyn_vec128 _)- (load (little_or_native_endian flags) address offset)))+ (load _ (little_or_native_endian flags) address offset)))(aarch64_fpuload128 (amode $I8X16 address offset) flags))(rule (lower- (uload8 (little_or_native_endian flags) address offset))+ (uload8 _ (little_or_native_endian flags) address offset))(aarch64_uload8 (amode $I8 address offset) flags))(rule (lower- (sload8 (little_or_native_endian flags) address offset))+ (sload8 _ (little_or_native_endian flags) address offset))(aarch64_sload8 (amode $I8 address offset) flags))(rule (lower- (uload16 (little_or_native_endian flags) address offset))+ (uload16 _ (little_or_native_endian flags) address offset))(aarch64_uload16 (amode $I16 address offset) flags))(rule (lower- (sload16 (little_or_native_endian flags) address offset))+ (sload16 _ (little_or_native_endian flags) address offset))(aarch64_sload16 (amode $I16 address offset) flags))(rule (lower- (uload32 (little_or_native_endian flags) address offset))+ (uload32 _ (little_or_native_endian flags) address offset))(aarch64_uload32 (amode $I32 address offset) flags))(rule (lower- (sload32 (little_or_native_endian flags) address offset))+ (sload32 _ (little_or_native_endian flags) address offset))(aarch64_sload32 (amode $I32 address offset) flags))(rule (lower- (sload8x8 (little_or_native_endian flags) address offset))+ (sload8x8 _ (little_or_native_endian flags) address offset))(vec_extend (VecExtendOp.Sxtl)(aarch64_fpuload64 (amode $F64 address offset) flags)false(ScalarSize.Size16)))(rule (lower- (uload8x8 (little_or_native_endian flags) address offset))+ (uload8x8 _ (little_or_native_endian flags) address offset))(vec_extend (VecExtendOp.Uxtl)(aarch64_fpuload64 (amode $F64 address offset) flags)false(ScalarSize.Size16)))(rule (lower- (sload16x4 (little_or_native_endian flags) address offset))+ (sload16x4 _ (little_or_native_endian flags) address offset))(vec_extend (VecExtendOp.Sxtl)(aarch64_fpuload64 (amode $F64 address offset) flags)false(ScalarSize.Size32)))(rule (lower- (uload16x4 (little_or_native_endian flags) address offset))+ (uload16x4 _ (little_or_native_endian flags) address offset))(vec_extend (VecExtendOp.Uxtl)(aarch64_fpuload64 (amode $F64 address offset) flags)false(ScalarSize.Size32)))(rule (lower- (sload32x2 (little_or_native_endian flags) address offset))+ (sload32x2 _ (little_or_native_endian flags) address offset))(vec_extend (VecExtendOp.Sxtl)(aarch64_fpuload64 (amode $F64 address offset) flags)false(ScalarSize.Size64)))(rule (lower- (uload32x2 (little_or_native_endian flags) address offset))+ (uload32x2 _ (little_or_native_endian flags) address offset))(vec_extend (VecExtendOp.Uxtl)(aarch64_fpuload64 (amode $F64 address offset) flags)false@@ -2785,7 +2785,7 @@;;; Rules for `{get,set}_pinned_reg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_pinned_reg))+(rule (lower (get_pinned_reg _))(mov_from_preg (preg_pinned)))(rule (lower (set_pinned_reg val))@@ -2794,66 +2794,66 @@;;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; SIMD&FP <=> SIMD&FP-(rule 7 (lower (has_type (ty_float_or_vec _) (bitcast _ x @ (value_type (ty_float_or_vec _)))))+(rule 7 (lower (has_type (ty_float_or_vec _) (bitcast _ _ x @ (value_type (ty_float_or_vec _)))))x); I128 => SIMD&FP-(rule 6 (lower (has_type (ty_float_or_vec _) (bitcast _ x @ (value_type $I128))))+(rule 6 (lower (has_type (ty_float_or_vec _) (bitcast _ _ x @ (value_type $I128))))(mov_to_vec (mov_to_fpu (value_regs_get x 0) (ScalarSize.Size64)) (value_regs_get x 1) 1 (VectorSize.Size64x2))); SIMD&FP => I128-(rule 5 (lower (has_type $I128 (bitcast _ x @ (value_type (ty_float_or_vec _)))))+(rule 5 (lower (has_type $I128 (bitcast _ _ x @ (value_type (ty_float_or_vec _)))))(value_regs (mov_from_vec x 0 (ScalarSize.Size64)) (mov_from_vec x 1 (ScalarSize.Size64)))); GPR => SIMD&FP-(rule 4 (lower (has_type (ty_float_or_vec _) (bitcast _ x @ (value_type in_ty))))+(rule 4 (lower (has_type (ty_float_or_vec _) (bitcast _ _ x @ (value_type in_ty))))(if (ty_int_ref_scalar_64 in_ty))(mov_to_fpu x (scalar_size in_ty))); SIMD&FP => GPR-(rule 3 (lower (has_type out_ty (bitcast _ x @ (value_type (fits_in_64 (ty_float_or_vec _))))))+(rule 3 (lower (has_type out_ty (bitcast _ _ x @ (value_type (fits_in_64 (ty_float_or_vec _))))))(if (ty_int_ref_scalar_64 out_ty))(mov_from_vec x 0 (scalar_size out_ty))); GPR <=> GPR-(rule 1 (lower (has_type out_ty (bitcast _ x @ (value_type in_ty))))+(rule 1 (lower (has_type out_ty (bitcast _ _ x @ (value_type in_ty))))(if (ty_int_ref_scalar_64 out_ty))(if (ty_int_ref_scalar_64 in_ty))x)-(rule 0 (lower (has_type $I128 (bitcast _ x @ (value_type $I128)))) x)+(rule 0 (lower (has_type $I128 (bitcast _ _ x @ (value_type $I128)))) x);;; Rules for `extractlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; extractlane with lane 0 can pass through the value unchanged; upper;; bits are undefined when a narrower type is in a wider register.-(rule 2 (lower (has_type (ty_scalar_float _) (extractlane val (u8_from_uimm8 0))))+(rule 2 (lower (has_type (ty_scalar_float _) (extractlane _ val (u8_from_uimm8 0))))val)(rule 0 (lower (has_type (ty_int ty)- (extractlane val+ (extractlane _ val(u8_from_uimm8 lane))))(mov_from_vec val lane (scalar_size ty)))(rule 1 (lower (has_type (ty_scalar_float ty)- (extractlane val @ (value_type vty)+ (extractlane _ val @ (value_type vty)(u8_from_uimm8 lane))))(fpu_move_from_vec val lane (vector_size vty)));;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (insertlane vec @ (value_type vty)+(rule 1 (lower (insertlane _ vec @ (value_type vty)val @ (value_type (ty_int _))(u8_from_uimm8 lane)))(mov_to_vec vec val lane (vector_size vty)))-(rule (lower (insertlane vec @ (value_type vty)+(rule (lower (insertlane _ vec @ (value_type vty)val @ (value_type (ty_scalar_float _))(u8_from_uimm8 lane)))(mov_vec_elem vec val lane 0 (vector_size vty)));;; Rules for `stack_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (stack_addr stack_slot offset))+(rule (lower (stack_addr _ stack_slot offset))(compute_stack_addr stack_slot offset));;; Rules for `vhigh_bits` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2867,7 +2867,7 @@;; in https://github.com/bytecodealliance/wasmtime/issues/2296,;; although they are not used here.-(rule (lower (vhigh_bits vec @ (value_type $I8X16)))+(rule (lower (vhigh_bits _ vec @ (value_type $I8X16)))(let (;; Replicate the MSB of each of the 16 byte lanes across;; the whole lane (sshr is an arithmetic right shift).@@ -2891,7 +2891,7 @@(result Reg (addv zipped (VectorSize.Size16x8))))(mov_from_vec result 0 (ScalarSize.Size16))))-(rule (lower (vhigh_bits vec @ (value_type $I16X8)))+(rule (lower (vhigh_bits _ vec @ (value_type $I16X8)))(let (;; Replicate the MSB of each of the 8 16-bit lanes across;; the whole lane (sshr is an arithmetic right shift).@@ -2904,7 +2904,7 @@(result Reg (addv anded (VectorSize.Size16x8))))(mov_from_vec result 0 (ScalarSize.Size16))))-(rule (lower (vhigh_bits vec @ (value_type $I32X4)))+(rule (lower (vhigh_bits _ vec @ (value_type $I32X4)))(let (;; Replicate the MSB of each of the 4 32-bit lanes across;; the whole lane (sshr is an arithmetic right shift).@@ -2917,7 +2917,7 @@(result Reg (addv anded (VectorSize.Size32x4))))(mov_from_vec result 0 (ScalarSize.Size32))))-(rule (lower (vhigh_bits vec @ (value_type $I64X2)))+(rule (lower (vhigh_bits _ vec @ (value_type $I64X2)))(let (;; Grab the MSB out of each of the lanes, right-shift to;; LSB, and add with a left-shift of upper lane's MSB back@@ -2931,7 +2931,7 @@;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap a b tc)))+(rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap _ a b tc)))(trap_if_overflow (add_with_flags_paired ty a b) tc));;;; Helpers for `*_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3011,15 +3011,15 @@;; For values smaller than a register, we do a normal `add` with both arguments;; zero extended. We then check if the output is the same as itself zero extended.-(rule 1 (lower (has_type (fits_in_16 ty) (uadd_overflow a b)))+(rule 1 (lower (has_type (fits_in_16 ty) (uadd_overflow _ a b)))(overflow_op_small ty a b (ArgumentExtension.Uext) (ALUOp.Add)));; For register sized add's we just emit a adds+cset, without further masking.-(rule 2 (lower (has_type (ty_32_or_64 ty) (uadd_overflow a b)))+(rule 2 (lower (has_type (ty_32_or_64 ty) (uadd_overflow _ a b)))(overflow_op_normal ty a b (ALUOp.AddS) (Cond.Hs)));; For 128bit integers we emit add+adcs+cset-(rule 0 (lower (has_type $I128 (uadd_overflow x y)))+(rule 0 (lower (has_type $I128 (uadd_overflow _ x y)))(overflow_op_128 x y (ALUOp.AddS) (ALUOp.AdcS) (Cond.Hs)));;;; Rules for `sadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3028,18 +3028,18 @@;; add out, a_ext, b, sxt{b,h};; cmp out, out, sxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (sadd_overflow a b)))+(rule 1 (lower (has_type (fits_in_16 ty) (sadd_overflow _ a b)))(overflow_op_small ty a b (ArgumentExtension.Sext) (ALUOp.Add)));; adds a, b;; cset of, vs-(rule 2 (lower (has_type (ty_32_or_64 ty) (sadd_overflow a b)))+(rule 2 (lower (has_type (ty_32_or_64 ty) (sadd_overflow _ a b)))(overflow_op_normal ty a b (ALUOp.AddS) (Cond.Vs)));; adds x_lo, y_lo;; addcs x_hi, y_hi;; cset of, vs-(rule 0 (lower (has_type $I128 (sadd_overflow x y)))+(rule 0 (lower (has_type $I128 (sadd_overflow _ x y)))(overflow_op_128 x y (ALUOp.AddS) (ALUOp.AdcS) (Cond.Vs)));;;; Rules for `usub_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3048,18 +3048,18 @@;; sub out, a_ext, b, ext{b,h};; cmp out, out, uxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (usub_overflow a b)))+(rule 1 (lower (has_type (fits_in_16 ty) (usub_overflow _ a b)))(overflow_op_small ty a b (ArgumentExtension.Uext) (ALUOp.Sub)));; subs a, b;; cset of, lo-(rule 2 (lower (has_type (ty_32_or_64 ty) (usub_overflow a b)))+(rule 2 (lower (has_type (ty_32_or_64 ty) (usub_overflow _ a b)))(overflow_op_normal ty a b (ALUOp.SubS) (Cond.Lo)));; subs x_lo, y_lo;; sbcs x_hi, y_hi;; cset of, lo-(rule 0 (lower (has_type $I128 (usub_overflow x y)))+(rule 0 (lower (has_type $I128 (usub_overflow _ x y)))(overflow_op_128 x y (ALUOp.SubS) (ALUOp.SbcS) (Cond.Lo)));;;; Rules for `ssub_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3068,18 +3068,18 @@;; sub out, a_ext, b, sxt{b,h};; cmp out, out, sxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (ssub_overflow a b)))+(rule 1 (lower (has_type (fits_in_16 ty) (ssub_overflow _ a b)))(overflow_op_small ty a b (ArgumentExtension.Sext) (ALUOp.Sub)));; subs a, b;; cset of, vs-(rule 2 (lower (has_type (ty_32_or_64 ty) (ssub_overflow a b)))+(rule 2 (lower (has_type (ty_32_or_64 ty) (ssub_overflow _ a b)))(overflow_op_normal ty a b (ALUOp.SubS) (Cond.Vs)));; subs x_lo, y_lo;; sbcs x_hi, y_hi;; cset of, vs-(rule 0 (lower (has_type $I128 (ssub_overflow x y)))+(rule 0 (lower (has_type $I128 (ssub_overflow _ x y)))(overflow_op_128 x y (ALUOp.SubS) (ALUOp.SbcS) (Cond.Vs)));;;; Rules for `umul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3089,7 +3089,7 @@;; mul out, a_ext, b_ext;; cmp out, out, uxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (umul_overflow a b)))+(rule 1 (lower (has_type (fits_in_16 ty) (umul_overflow _ a b)))(let ((extend ExtendOp (lower_extend_op ty (ArgumentExtension.Uext)))(a_uext Reg (put_in_reg_zext32 a))@@ -3105,7 +3105,7 @@;; umull out, a, b;; cmp out, out, uxtw;; cset of, ne-(rule 2 (lower (has_type $I32 (umul_overflow a b)))+(rule 2 (lower (has_type $I32 (umul_overflow _ a b)))(let ((out Reg (umaddl a b (zero_reg)))(out_of Reg (with_flags_reg@@ -3119,7 +3119,7 @@;; umulh tmp, a, b;; cmp tmp, #0;; cset of, ne-(rule 2 (lower (has_type $I64 (umul_overflow a b)))+(rule 2 (lower (has_type $I64 (umul_overflow _ a b)))(let ((out Reg (madd $I64 a b (zero_reg)))(tmp Reg (umulh $I64 a b))@@ -3137,7 +3137,7 @@;; mul out, a_ext, b_ext;; cmp out, out, sxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (smul_overflow a b)))+(rule 1 (lower (has_type (fits_in_16 ty) (smul_overflow _ a b)))(let ((extend ExtendOp (lower_extend_op ty (ArgumentExtension.Sext)))(a_sext Reg (put_in_reg_sext32 a))@@ -3153,7 +3153,7 @@;; smull out, a, b;; cmp out, out, sxtw;; cset of, ne-(rule 2 (lower (has_type $I32 (smul_overflow a b)))+(rule 2 (lower (has_type $I32 (smul_overflow _ a b)))(let ((out Reg (smaddl a b (zero_reg)))(out_of Reg (with_flags_reg@@ -3167,7 +3167,7 @@;; smulh tmp, a, b;; cmp tmp, out, ASR #63;; cset of, ne-(rule 2 (lower (has_type $I64 (smul_overflow a b)))+(rule 2 (lower (has_type $I64 (smul_overflow _ a b)))(let ((out Reg (madd $I64 a b (zero_reg)))(tmp Reg (smulh $I64 a b))@@ -3180,21 +3180,21 @@;;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value (symbol_value_data name _ _))))+(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value _ (symbol_value_data name _ _))))(elf_tls_get_addr name))-(rule (lower (has_type (tls_model (TlsModel.Macho)) (tls_value (symbol_value_data name _ _))))+(rule (lower (has_type (tls_model (TlsModel.Macho)) (tls_value _ (symbol_value_data name _ _))))(macho_tls_get_addr name));;; Rules for `fvpromote_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (fvpromote_low val))+(rule (lower (fvpromote_low _ val))(vec_rr_long (VecRRLongOp.Fcvtl32) val false));;; Rules for `brif` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `brif` following `icmp`-(rule (lower_branch (brif (maybe_uextend (icmp cc x @ (value_type ty) y)) _ _) (two_targets taken not_taken))+(rule (lower_branch (brif (maybe_uextend (icmp _ cc x @ (value_type ty) y)) _ _) (two_targets taken not_taken))(let ((comparison FlagsAndCC (lower_icmp_into_flags cc x y ty))(cond Cond (cond_code (flags_and_cc_cc comparison))))(emit_side_effect@@ -3204,7 +3204,7 @@(cond_br_cond cond))))));; `brif` following `fcmp`-(rule (lower_branch (brif (maybe_uextend (fcmp cc x @ (value_type (ty_scalar_float ty)) y)) _ _) (two_targets taken not_taken))+(rule (lower_branch (brif (maybe_uextend (fcmp _ cc x @ (value_type (ty_scalar_float ty)) y)) _ _) (two_targets taken not_taken))(let ((cond Cond (fp_cond_code cc)))(emit_side_effect(with_flags_side_effect (fpu_cmp (scalar_size ty) x y)@@ -3230,14 +3230,14 @@(cond_br taken not_taken (cond_br_not_zero rt (operand_size $I64)))))));; Special lowerings for `tbnz` - "Test bit and Branch if Nonzero"-(rule 1 (lower_branch (brif (band x @ (value_type ty) (u64_from_iconst n)) _ _)+(rule 1 (lower_branch (brif (band _ x @ (value_type ty) (u64_from_iconst n)) _ _)(two_targets taken not_taken))(if-let bit (test_and_compare_bit_const ty n))(emit_side_effect (tbnz taken not_taken x bit)));; Special lowering for `tbz` - "Test bit and Branch if Zero"-(rule 1 (lower_branch (brif (icmp (IntCC.Equal)- (band x @ (value_type (fits_in_64 ty))+(rule 1 (lower_branch (brif (icmp _ (IntCC.Equal)+ (band _ x @ (value_type (fits_in_64 ty))(u64_from_iconst n))(u64_from_iconst 0)) _ _)(two_targets taken not_taken))@@ -3265,7 +3265,7 @@;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))+(rule (lower (get_exception_handler_address _ (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(a64_label_address succ_label)))diff --git a/cranelift/codegen/src/isa/aarch64/lower_dynamic_neon.isle b/cranelift/codegen/src/isa/aarch64/lower_dynamic_neon.isleindex 64da0ba11f..8c9c21ed42 100644--- a/cranelift/codegen/src/isa/aarch64/lower_dynamic_neon.isle+++ b/cranelift/codegen/src/isa/aarch64/lower_dynamic_neon.isle@@ -1,98 +1,98 @@;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -4 (lower (has_type ty @ (dynamic_lane _ _) (iadd x y)))+(rule -4 (lower (has_type ty @ (dynamic_lane _ _) (iadd _ x y)))(value_reg (add_vec (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -5 (lower (has_type ty @ (dynamic_lane _ _) (isub x y)))+(rule -5 (lower (has_type ty @ (dynamic_lane _ _) (isub _ x y)))(value_reg (sub_vec (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -4 (lower (has_type (lane_fits_in_32 ty @ (dynamic_lane _ _)) (imul x y)))+(rule -4 (lower (has_type (lane_fits_in_32 ty @ (dynamic_lane _ _)) (imul _ x y)))(value_reg (vec_rrr (VecALUOp.Mul) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fadd x y)))+(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fadd _ x y)))(value_reg (vec_rrr (VecALUOp.Fadd) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fsub x y)))+(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fsub _ x y)))(value_reg (vec_rrr (VecALUOp.Fsub) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmul x y)))+(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmul _ x y)))(value_reg (vec_rrr (VecALUOp.Fmul) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fdiv x y)))+(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fdiv _ x y)))(value_reg (vec_rrr (VecALUOp.Fdiv) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmin x y)))+(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmin _ x y)))(value_reg (vec_rrr (VecALUOp.Fmin) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmax x y)))+(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmax _ x y)))(value_reg (vec_rrr (VecALUOp.Fmax) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type (ty_dyn128_int ty) (snarrow x y)))+(rule -2 (lower (has_type (ty_dyn128_int ty) (snarrow _ x y)))(if-let _ (zero_value y))(sqxtn x (lane_size ty)))-(rule -1 (lower (has_type (ty_dyn64_int ty) (snarrow x y)))+(rule -1 (lower (has_type (ty_dyn64_int ty) (snarrow _ x y)))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(sqxtn dst (lane_size ty))))-(rule -3 (lower (has_type (ty_dyn128_int ty) (snarrow x y)))+(rule -3 (lower (has_type (ty_dyn128_int ty) (snarrow _ x y)))(let ((low_half Reg (sqxtn x (lane_size ty)))(result Reg (sqxtn2 low_half y (lane_size ty))))result));;;; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type (ty_dyn128_int ty) (unarrow x y)))+(rule -2 (lower (has_type (ty_dyn128_int ty) (unarrow _ x y)))(if-let _ (zero_value y))(sqxtun x (lane_size ty)))-(rule -1 (lower (has_type (ty_dyn64_int ty) (unarrow x y)))+(rule -1 (lower (has_type (ty_dyn64_int ty) (unarrow _ x y)))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(sqxtun dst (lane_size ty))))-(rule -3 (lower (has_type (ty_dyn128_int ty) (unarrow x y)))+(rule -3 (lower (has_type (ty_dyn128_int ty) (unarrow _ x y)))(let ((low_half Reg (sqxtun x (lane_size ty)))(result Reg (sqxtun2 low_half y (lane_size ty))))result));;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type (ty_dyn128_int ty) (uunarrow x y)))+(rule -2 (lower (has_type (ty_dyn128_int ty) (uunarrow _ x y)))(if-let _ (zero_value y))(uqxtn x (lane_size ty)))-(rule -1 (lower (has_type (ty_dyn64_int ty) (uunarrow x y)))+(rule -1 (lower (has_type (ty_dyn64_int ty) (uunarrow _ x y)))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(uqxtn dst (lane_size ty))))-(rule -3 (lower (has_type (ty_dyn128_int ty) (uunarrow x y)))+(rule -3 (lower (has_type (ty_dyn128_int ty) (uunarrow _ x y)))(let ((low_half Reg (uqxtn x (lane_size ty)))(result Reg (uqxtn2 low_half y (lane_size ty))))result));;; Rules for `dynamic_stack_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (dynamic_stack_addr stack_slot))+(rule (lower (dynamic_stack_addr _ stack_slot))(let ((dst WritableReg (temp_writable_reg $I64))(_ Unit (emit (abi_dynamic_stackslot_addr dst stack_slot))))(value_reg dst)));;; Rules for `extract_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (extract_vector x 0))+(rule (lower (extract_vector _ x 0))(value_reg (put_in_reg x)));;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty (swiden_high x)))+(rule -1 (lower (has_type ty (swiden_high _ x)))(vec_extend (VecExtendOp.Sxtl) x true (lane_size ty)));;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty (uwiden_high x)))+(rule -1 (lower (has_type ty (uwiden_high _ x)))(vec_extend (VecExtendOp.Uxtl) x true (lane_size ty)))diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isleindex 53c71b2096..b172e42b78 100644--- a/cranelift/codegen/src/isa/pulley_shared/inst.isle+++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle@@ -182,7 +182,7 @@(type ValueOffset (enum (Both (value Value) (offset i32))))(decl pure amode_base (Value Offset32) ValueOffset)(rule (amode_base addr (offset32 offset)) (ValueOffset.Both addr offset))-(rule 1 (amode_base (iadd addr (i32_from_iconst b)) (offset32 offset))+(rule 1 (amode_base (iadd _ addr (i32_from_iconst b)) (offset32 offset))(if-let new_offset (i32_checked_add b offset))(ValueOffset.Both addr new_offset))@@ -270,7 +270,7 @@(type SinkableLoad (enum (Load (inst Inst) (ty Type) (addr Value) (offset u8))))(decl pure partial sinkable_load (Value) SinkableLoad)(rule (sinkable_load value @ (value_type ty))- (if-let inst @ (load flags addr (offset32 offset)) (is_sinkable_inst value))+ (if-let inst @ (load _ flags addr (offset32 offset)) (is_sinkable_inst value))(if-let true (is_native_endianness (endianness flags)))(if-let true (memflags_nontrapping flags))(if-let offset8 (i32_try_into_u8 offset))@@ -345,7 +345,7 @@;; used if the oob condition is true. Otherwise the raw address is used for the;; load/store.(decl pure partial wasm_g32 (Value Offset32 MemFlags Type) G32)-(rule (wasm_g32 (select oob (u64_from_iconst 0) raw_addr) (offset32 0) flags ty)+(rule (wasm_g32 (select _ oob (u64_from_iconst 0) raw_addr) (offset32 0) flags ty);; This must be a wasm load/store according to `MemFlags`, namely that it's;; got a "HEAP_OUT_OF_BOUNDS" trap code and it's little-endian.(if-let true (memflags_is_wasm flags))@@ -381,8 +381,8 @@(decl pure host_offset (Value) HostOffset)(type HostOffset (enum (All (a Value) (b i32))))(rule 0 (host_offset val) (HostOffset.All val 0))-(rule 1 (host_offset (iadd val @ (iadd _ _) (i32_from_iconst n))) (HostOffset.All val n))-(rule 2 (host_offset (iadd (i32_from_iconst n) val @ (iadd _ _))) (HostOffset.All val n))+(rule 1 (host_offset (iadd _ val @ (iadd _ _ _) (i32_from_iconst n))) (HostOffset.All val n))+(rule 2 (host_offset (iadd _ (i32_from_iconst n) val @ (iadd _ _ _))) (HostOffset.All val n));; Helper to test whether the first argument `oob` contains a condition for;; matching whether second argument `addr` is out-of-bounds. Searches for a@@ -392,23 +392,23 @@;; 32-bit hosts: search either side of the `iadd` for the base address within;; `oob` to see if it's a matching condition.-(rule 0 (wasm_oob_select oob (iadd base wasm_addr @ (value_type $I32)))+(rule 0 (wasm_oob_select oob (iadd _ base wasm_addr @ (value_type $I32)))(if-let (OobCond.All bound n) (wasm_oob_cond wasm_addr oob))(if-let (PointerWidth.PointerWidth32) (pointer_width))(OobSelect.All base bound wasm_addr n))-(rule 1 (wasm_oob_select oob (iadd wasm_addr @ (value_type $I32) base))+(rule 1 (wasm_oob_select oob (iadd _ wasm_addr @ (value_type $I32) base))(if-let (OobCond.All bound n) (wasm_oob_cond wasm_addr oob))(if-let (PointerWidth.PointerWidth32) (pointer_width))(OobSelect.All base bound wasm_addr n));; 64-bit hosts: also search either side, but the wasm address must also be;; a `uextend` from a 32-bit value.-(rule 0 (wasm_oob_select oob (iadd base wasm_addr_ext @ (value_type $I64)))+(rule 0 (wasm_oob_select oob (iadd _ base wasm_addr_ext @ (value_type $I64)))(if-let (OobCond.All bound n) (wasm_oob_cond wasm_addr_ext oob))(if-let wasm_addr (wasm32_addr_for_host64 wasm_addr_ext))(if-let (PointerWidth.PointerWidth64) (pointer_width))(OobSelect.All base bound wasm_addr n))-(rule 1 (wasm_oob_select oob (iadd wasm_addr_ext @ (value_type $I64) base))+(rule 1 (wasm_oob_select oob (iadd _ wasm_addr_ext @ (value_type $I64) base))(if-let (OobCond.All bound n) (wasm_oob_cond wasm_addr_ext oob))(if-let wasm_addr (wasm32_addr_for_host64 wasm_addr_ext))(if-let (PointerWidth.PointerWidth64) (pointer_width))@@ -419,7 +419,7 @@;; extended from 32-bits. That means that this only matches `(uextend ...)` or;; an `iconst` which already fits in 32 bits.(decl pure partial wasm32_addr_for_host64 (Value) Value)-(rule (wasm32_addr_for_host64 (uextend addr @ (value_type $I32))) addr)+(rule (wasm32_addr_for_host64 (uextend _ addr @ (value_type $I32))) addr)(rule (wasm32_addr_for_host64 val @ (u32_from_iconst _)) val);; Helper to search for the first argument, `wasm_addr`, within the oob@@ -427,9 +427,9 @@;; of the address against a particular bound.(decl pure partial wasm_oob_cond (Value Value) OobCond)(type OobCond (enum (All (a Value) (b u64))))-(rule (wasm_oob_cond wasm_addr (ugt wasm_addr (isub bound (u64_from_iconst n))))+(rule (wasm_oob_cond wasm_addr (ugt wasm_addr (isub _ bound (u64_from_iconst n))))(OobCond.All bound n))-(rule (wasm_oob_cond wasm_addr (ult (isub bound (u64_from_iconst n)) wasm_addr))+(rule (wasm_oob_cond wasm_addr (ult (isub _ bound (u64_from_iconst n)) wasm_addr))(OobCond.All bound n));;;; Newtypes for Different Register Classes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isleindex 00b0a2badf..d6df01f9df 100644--- a/cranelift/codegen/src/isa/pulley_shared/lower.isle+++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle@@ -20,10 +20,10 @@(Cond.IfXneq64I32 val 0));; Peel away explicit `uextend` values to take a look at the inner value.-(rule 2 (lower_cond (uextend val)) (lower_cond val))+(rule 2 (lower_cond (uextend _ val)) (lower_cond val));; Conditional branches on `icmp`s.-(rule 2 (lower_cond (icmp cc a b @ (value_type $I32))) (lower_cond_icmp32 cc a b))-(rule 2 (lower_cond (icmp cc a b @ (value_type $I64))) (lower_cond_icmp64 cc a b))+(rule 2 (lower_cond (icmp _ cc a b @ (value_type $I32))) (lower_cond_icmp32 cc a b))+(rule 2 (lower_cond (icmp _ cc a b @ (value_type $I64))) (lower_cond_icmp64 cc a b))(decl lower_cond_icmp32 (IntCC Value Value) Cond)(rule (lower_cond_icmp32 (IntCC.Equal) a b) (Cond.IfXeq32 a b))@@ -124,16 +124,16 @@;;;; Rules for `get_stack_pointer` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_stack_pointer))+(rule (lower (get_stack_pointer _))(pulley_get_special (sp_reg)));;;; Rules for `get_frame_pointer` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_frame_pointer)) (pulley_xmov_fp))+(rule (lower (get_frame_pointer _)) (pulley_xmov_fp));;;; Rules for `get_return_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_return_address)) (pulley_xmov_lr))+(rule (lower (get_return_address _)) (pulley_xmov_lr));;;; Rules for `return` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -232,43 +232,43 @@;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (iconst c))) (imm (i64_sextend_imm64 ty c)))+(rule (lower (has_type ty (iconst _ c))) (imm (i64_sextend_imm64 ty c)));;;; Rules for `f32const`;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f32const (u32_from_ieee32 x))) (pulley_fconst32 x))+(rule (lower (f32const _ (u32_from_ieee32 x))) (pulley_fconst32 x));;;; Rules for `f64const`;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f64const (u64_from_ieee64 x))) (pulley_fconst64 x))+(rule (lower (f64const _ (u64_from_ieee64 x))) (pulley_fconst64 x));;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_64 ty)) (iadd a b))) (pulley_xadd32 a b))-(rule 1 (lower (has_type $I64 (iadd a b))) (pulley_xadd64 a b))+(rule 0 (lower (has_type (ty_int (fits_in_64 ty)) (iadd _ a b))) (pulley_xadd32 a b))+(rule 1 (lower (has_type $I64 (iadd _ a b))) (pulley_xadd64 a b));; Fold constants into the instruction if possible-(rule 10 (lower (has_type (ty_int (fits_in_32 _)) (iadd a (u32_from_iconst b))))+(rule 10 (lower (has_type (ty_int (fits_in_32 _)) (iadd _ a (u32_from_iconst b))))(pulley_xadd32_u32 a b))-(rule 11 (lower (has_type (ty_int (fits_in_32 _)) (iadd a (u8_from_iconst b))))+(rule 11 (lower (has_type (ty_int (fits_in_32 _)) (iadd _ a (u8_from_iconst b))))(pulley_xadd32_u8 a b))-(rule 12 (lower (has_type $I64 (iadd a (u32_from_iconst b))))+(rule 12 (lower (has_type $I64 (iadd _ a (u32_from_iconst b))))(pulley_xadd64_u32 a b))-(rule 13 (lower (has_type $I64 (iadd a (u8_from_iconst b))))+(rule 13 (lower (has_type $I64 (iadd _ a (u8_from_iconst b))))(pulley_xadd64_u8 a b));; If the rhs is a constant and the negated version can fit within a smaller;; constant then switch this to a subtraction with the negated constant.-(rule 14 (lower (has_type (ty_int (fits_in_32 _)) (iadd a b)))+(rule 14 (lower (has_type (ty_int (fits_in_32 _)) (iadd _ a b)))(if-let c (u32_from_negated_iconst b))(pulley_xsub32_u32 a c))-(rule 15 (lower (has_type $I64 (iadd a b)))+(rule 15 (lower (has_type $I64 (iadd _ a b)))(if-let c (u32_from_negated_iconst b))(pulley_xsub64_u32 a c))-(rule 16 (lower (has_type (ty_int (fits_in_32 _)) (iadd a b)))+(rule 16 (lower (has_type (ty_int (fits_in_32 _)) (iadd _ a b)))(if-let c (u8_from_negated_iconst b))(pulley_xsub32_u8 a c))-(rule 17 (lower (has_type $I64 (iadd a b)))+(rule 17 (lower (has_type $I64 (iadd _ a b)))(if-let c (u8_from_negated_iconst b))(pulley_xsub64_u8 a c))@@ -289,7 +289,7 @@neg_u32);; 128-bit addition-(rule 1 (lower (has_type $I128 (iadd a b)))+(rule 1 (lower (has_type $I128 (iadd _ a b)))(let ((a ValueRegs a)(b ValueRegs b))(pulley_xadd128@@ -299,60 +299,60 @@(value_regs_get b 1))));; vector addition-(rule 1 (lower (has_type $I8X16 (iadd a b))) (pulley_vaddi8x16 a b))-(rule 1 (lower (has_type $I16X8 (iadd a b))) (pulley_vaddi16x8 a b))-(rule 1 (lower (has_type $I32X4 (iadd a b))) (pulley_vaddi32x4 a b))-(rule 1 (lower (has_type $I64X2 (iadd a b))) (pulley_vaddi64x2 a b))+(rule 1 (lower (has_type $I8X16 (iadd _ a b))) (pulley_vaddi8x16 a b))+(rule 1 (lower (has_type $I16X8 (iadd _ a b))) (pulley_vaddi16x8 a b))+(rule 1 (lower (has_type $I32X4 (iadd _ a b))) (pulley_vaddi32x4 a b))+(rule 1 (lower (has_type $I64X2 (iadd _ a b))) (pulley_vaddi64x2 a b))-(rule 1 (lower (has_type $I8X16 (sadd_sat a b))) (pulley_vaddi8x16_sat a b))-(rule 1 (lower (has_type $I8X16 (uadd_sat a b))) (pulley_vaddu8x16_sat a b))-(rule 1 (lower (has_type $I16X8 (sadd_sat a b))) (pulley_vaddi16x8_sat a b))-(rule 1 (lower (has_type $I16X8 (uadd_sat a b))) (pulley_vaddu16x8_sat a b))+(rule 1 (lower (has_type $I8X16 (sadd_sat _ a b))) (pulley_vaddi8x16_sat a b))+(rule 1 (lower (has_type $I8X16 (uadd_sat _ a b))) (pulley_vaddu8x16_sat a b))+(rule 1 (lower (has_type $I16X8 (sadd_sat _ a b))) (pulley_vaddi16x8_sat a b))+(rule 1 (lower (has_type $I16X8 (uadd_sat _ a b))) (pulley_vaddu16x8_sat a b));; Specialized lowerings for multiply-and-add-(rule 2 (lower (has_type $I32 (iadd (imul a b) c))) (pulley_xmadd32 a b c))-(rule 3 (lower (has_type $I32 (iadd c (imul a b)))) (pulley_xmadd32 a b c))-(rule 2 (lower (has_type $I64 (iadd (imul a b) c))) (pulley_xmadd64 a b c))-(rule 3 (lower (has_type $I64 (iadd c (imul a b)))) (pulley_xmadd64 a b c))+(rule 2 (lower (has_type $I32 (iadd _ (imul _ a b) c))) (pulley_xmadd32 a b c))+(rule 3 (lower (has_type $I32 (iadd _ c (imul _ a b)))) (pulley_xmadd32 a b c))+(rule 2 (lower (has_type $I64 (iadd _ (imul _ a b) c))) (pulley_xmadd64 a b c))+(rule 3 (lower (has_type $I64 (iadd _ c (imul _ a b)))) (pulley_xmadd64 a b c));;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16X8 (iadd_pairwise a b))) (pulley_vaddpairwisei16x8_s a b))-(rule (lower (has_type $I32X4 (iadd_pairwise a b))) (pulley_vaddpairwisei32x4_s a b))+(rule (lower (has_type $I16X8 (iadd_pairwise _ a b))) (pulley_vaddpairwisei16x8_s a b))+(rule (lower (has_type $I32X4 (iadd_pairwise _ a b))) (pulley_vaddpairwisei32x4_s a b));;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (isub a b))) (pulley_xsub32 a b))-(rule 1 (lower (has_type $I64 (isub a b))) (pulley_xsub64 a b))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a b))) (pulley_xsub32 a b))+(rule 1 (lower (has_type $I64 (isub _ a b))) (pulley_xsub64 a b));; Fold a rhs constant into the instruction if possible.-(rule 2 (lower (has_type (ty_int (fits_in_32 _)) (isub a (u32_from_iconst b))))+(rule 2 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a (u32_from_iconst b))))(pulley_xsub32_u32 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (isub a (u8_from_iconst b))))+(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a (u8_from_iconst b))))(pulley_xsub32_u8 a b))-(rule 4 (lower (has_type $I64 (isub a (u32_from_iconst b))))+(rule 4 (lower (has_type $I64 (isub _ a (u32_from_iconst b))))(pulley_xsub64_u32 a b))-(rule 5 (lower (has_type $I64 (isub a (u8_from_iconst b))))+(rule 5 (lower (has_type $I64 (isub _ a (u8_from_iconst b))))(pulley_xsub64_u8 a b));; If the rhs is a constant and the negated version can fit within a smaller;; constant then switch this to an addition with the negated constant.-(rule 6 (lower (has_type (ty_int (fits_in_32 _)) (isub a b)))+(rule 6 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a b)))(if-let c (u32_from_negated_iconst b))(pulley_xadd32_u32 a c))-(rule 7 (lower (has_type $I64 (isub a b)))+(rule 7 (lower (has_type $I64 (isub _ a b)))(if-let c (u32_from_negated_iconst b))(pulley_xadd64_u32 a c))-(rule 8 (lower (has_type (ty_int (fits_in_32 _)) (isub a b)))+(rule 8 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a b)))(if-let c (u8_from_negated_iconst b))(pulley_xadd32_u8 a c))-(rule 9 (lower (has_type $I64 (isub a b)))+(rule 9 (lower (has_type $I64 (isub _ a b)))(if-let c (u8_from_negated_iconst b))(pulley_xadd64_u8 a c));; 128-bit subtraction-(rule 1 (lower (has_type $I128 (isub a b)))+(rule 1 (lower (has_type $I128 (isub _ a b)))(let ((a ValueRegs a)(b ValueRegs b))(pulley_xsub128@@ -362,40 +362,40 @@(value_regs_get b 1))));; vector subtraction-(rule 1 (lower (has_type $I8X16 (isub a b))) (pulley_vsubi8x16 a b))-(rule 1 (lower (has_type $I16X8 (isub a b))) (pulley_vsubi16x8 a b))-(rule 1 (lower (has_type $I32X4 (isub a b))) (pulley_vsubi32x4 a b))-(rule 1 (lower (has_type $I64X2 (isub a b))) (pulley_vsubi64x2 a b))+(rule 1 (lower (has_type $I8X16 (isub _ a b))) (pulley_vsubi8x16 a b))+(rule 1 (lower (has_type $I16X8 (isub _ a b))) (pulley_vsubi16x8 a b))+(rule 1 (lower (has_type $I32X4 (isub _ a b))) (pulley_vsubi32x4 a b))+(rule 1 (lower (has_type $I64X2 (isub _ a b))) (pulley_vsubi64x2 a b))-(rule 1 (lower (has_type $I8X16 (ssub_sat a b))) (pulley_vsubi8x16_sat a b))-(rule 1 (lower (has_type $I8X16 (usub_sat a b))) (pulley_vsubu8x16_sat a b))-(rule 1 (lower (has_type $I16X8 (ssub_sat a b))) (pulley_vsubi16x8_sat a b))-(rule 1 (lower (has_type $I16X8 (usub_sat a b))) (pulley_vsubu16x8_sat a b))+(rule 1 (lower (has_type $I8X16 (ssub_sat _ a b))) (pulley_vsubi8x16_sat a b))+(rule 1 (lower (has_type $I8X16 (usub_sat _ a b))) (pulley_vsubu8x16_sat a b))+(rule 1 (lower (has_type $I16X8 (ssub_sat _ a b))) (pulley_vsubi16x8_sat a b))+(rule 1 (lower (has_type $I16X8 (usub_sat _ a b))) (pulley_vsubu16x8_sat a b));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (imul a b))) (pulley_xmul32 a b))-(rule (lower (has_type $I16 (imul a b))) (pulley_xmul32 a b))-(rule (lower (has_type $I32 (imul a b))) (pulley_xmul32 a b))-(rule (lower (has_type $I64 (imul a b))) (pulley_xmul64 a b))+(rule (lower (has_type $I8 (imul _ a b))) (pulley_xmul32 a b))+(rule (lower (has_type $I16 (imul _ a b))) (pulley_xmul32 a b))+(rule (lower (has_type $I32 (imul _ a b))) (pulley_xmul32 a b))+(rule (lower (has_type $I64 (imul _ a b))) (pulley_xmul64 a b))-(rule 1 (lower (has_type (ty_int (fits_in_32 _)) (imul a (i32_from_iconst b))))+(rule 1 (lower (has_type (ty_int (fits_in_32 _)) (imul _ a (i32_from_iconst b))))(pulley_xmul32_s32 a b))-(rule 2 (lower (has_type $I64 (imul a (i32_from_iconst b))))+(rule 2 (lower (has_type $I64 (imul _ a (i32_from_iconst b))))(pulley_xmul64_s32 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (imul a (i8_from_iconst b))))+(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (imul _ a (i8_from_iconst b))))(pulley_xmul32_s8 a b))-(rule 4 (lower (has_type $I64 (imul a (i8_from_iconst b))))+(rule 4 (lower (has_type $I64 (imul _ a (i8_from_iconst b))))(pulley_xmul64_s8 a b));; 128-bit (or wide) multiplication-(rule 4 (lower (has_type $I128 (imul (uextend a) (uextend b))))+(rule 4 (lower (has_type $I128 (imul _ (uextend _ a) (uextend _ b))))(pulley_xwidemul64_u (zext64 a) (zext64 b)))-(rule 4 (lower (has_type $I128 (imul (sextend a) (sextend b))))+(rule 4 (lower (has_type $I128 (imul _ (sextend _ a) (sextend _ b))))(pulley_xwidemul64_s (sext64 a) (sext64 b)));; for I128-(rule (lower (has_type $I128 (imul x y)))+(rule (lower (has_type $I128 (imul _ x y)))(let((x_regs ValueRegs x)(x_lo XReg (value_regs_get x_regs 0))@@ -420,106 +420,106 @@(value_regs wide_lo result_hi)));; vector multiplication-(rule (lower (has_type $I8X16 (imul a b))) (pulley_vmuli8x16 a b))-(rule (lower (has_type $I16X8 (imul a b))) (pulley_vmuli16x8 a b))-(rule (lower (has_type $I32X4 (imul a b))) (pulley_vmuli32x4 a b))-(rule (lower (has_type $I64X2 (imul a b))) (pulley_vmuli64x2 a b))+(rule (lower (has_type $I8X16 (imul _ a b))) (pulley_vmuli8x16 a b))+(rule (lower (has_type $I16X8 (imul _ a b))) (pulley_vmuli16x8 a b))+(rule (lower (has_type $I32X4 (imul _ a b))) (pulley_vmuli32x4 a b))+(rule (lower (has_type $I64X2 (imul _ a b))) (pulley_vmuli64x2 a b));;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (umulhi a b)))+(rule (lower (has_type $I8 (umulhi _ a b)))(if-let (u6_from_u8 shift) (u64_try_into_u8 8))(pulley_xshr32_u_u6 (pulley_xmul32 (zext32 a) (zext32 b)) shift))-(rule (lower (has_type $I16 (umulhi a b)))+(rule (lower (has_type $I16 (umulhi _ a b)))(if-let (u6_from_u8 shift) (u64_try_into_u8 16))(pulley_xshr32_u_u6 (pulley_xmul32 (zext32 a) (zext32 b)) shift))-(rule (lower (has_type $I32 (umulhi a b)))+(rule (lower (has_type $I32 (umulhi _ a b)))(if-let (u6_from_u8 shift) (u64_try_into_u8 32))(pulley_xshr64_u_u6 (pulley_xmul64 (zext64 a) (zext64 b)) shift))-(rule (lower (has_type $I64 (umulhi a b)))+(rule (lower (has_type $I64 (umulhi _ a b)))(pulley_xmulhi64_u a b));;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (smulhi a b)))+(rule (lower (has_type $I8 (smulhi _ a b)))(if-let (u6_from_u8 shift) (u64_try_into_u8 8))(pulley_xshr32_s_u6 (pulley_xmul32 (sext32 a) (sext32 b)) shift))-(rule (lower (has_type $I16 (smulhi a b)))+(rule (lower (has_type $I16 (smulhi _ a b)))(if-let (u6_from_u8 shift) (u64_try_into_u8 16))(pulley_xshr32_s_u6 (pulley_xmul32 (sext32 a) (sext32 b)) shift))-(rule (lower (has_type $I32 (smulhi a b)))+(rule (lower (has_type $I32 (smulhi _ a b)))(if-let (u6_from_u8 shift) (u64_try_into_u8 32))(pulley_xshr64_s_u6 (pulley_xmul64 (sext64 a) (sext64 b)) shift))-(rule (lower (has_type $I64 (smulhi a b)))+(rule (lower (has_type $I64 (smulhi _ a b)))(pulley_xmulhi64_s a b));;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16X8 (sqmul_round_sat a b))) (pulley_vqmulrsi16x8 a b))+(rule (lower (has_type $I16X8 (sqmul_round_sat _ a b))) (pulley_vqmulrsi16x8 a b));;;; Rules for `sdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (sdiv a b)))+(rule 0 (lower (has_type (fits_in_32 _) (sdiv _ a b)))(pulley_xdiv32_s (sext32 a) (sext32 b)))-(rule 1 (lower (has_type $I64 (sdiv a b))) (pulley_xdiv64_s a b))+(rule 1 (lower (has_type $I64 (sdiv _ a b))) (pulley_xdiv64_s a b));;;; Rules for `srem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (srem a b)))+(rule 0 (lower (has_type (fits_in_32 _) (srem _ a b)))(pulley_xrem32_s (sext32 a) (sext32 b)))-(rule 1 (lower (has_type $I64 (srem a b))) (pulley_xrem64_s a b))+(rule 1 (lower (has_type $I64 (srem _ a b))) (pulley_xrem64_s a b));;;; Rules for `udiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (udiv a b)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (udiv _ a b)))(pulley_xdiv32_u (zext32 a) (zext32 b)))-(rule 1 (lower (has_type $I64 (udiv a b))) (pulley_xdiv64_u a b))+(rule 1 (lower (has_type $I64 (udiv _ a b))) (pulley_xdiv64_u a b));;;; Rules for `urem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (urem a b)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (urem _ a b)))(pulley_xrem32_u (zext32 a) (zext32 b)))-(rule 1 (lower (has_type $I64 (urem a b))) (pulley_xrem64_u a b))+(rule 1 (lower (has_type $I64 (urem _ a b))) (pulley_xrem64_u a b));;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (avg_round a b))) (pulley_vavground8x16 a b))-(rule (lower (has_type $I16X8 (avg_round a b))) (pulley_vavground16x8 a b))+(rule (lower (has_type $I8X16 (avg_round _ a b))) (pulley_vavground8x16 a b))+(rule (lower (has_type $I16X8 (avg_round _ a b))) (pulley_vavground16x8 a b));;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (ishl a b)))+(rule (lower (has_type $I8 (ishl _ a b)))(pulley_xshl32 a (pulley_xband32_s8 b 7)))-(rule (lower (has_type $I16 (ishl a b)))+(rule (lower (has_type $I16 (ishl _ a b)))(pulley_xshl32 a (pulley_xband32_s8 b 15)))-(rule (lower (has_type $I32 (ishl a b)))+(rule (lower (has_type $I32 (ishl _ a b)))(pulley_xshl32 a b))-(rule (lower (has_type $I64 (ishl a b)))+(rule (lower (has_type $I64 (ishl _ a b)))(pulley_xshl64 a b));; Special-case constant shift amounts.-(rule 1 (lower (has_type $I32 (ishl a b)))+(rule 1 (lower (has_type $I32 (ishl _ a b)))(if-let n (u6_shift_from_iconst b))(pulley_xshl32_u6 a n))-(rule 1 (lower (has_type $I64 (ishl a b)))+(rule 1 (lower (has_type $I64 (ishl _ a b)))(if-let n (u6_shift_from_iconst b))(pulley_xshl64_u6 a n));; vector shifts-(rule (lower (has_type $I8X16 (ishl a b))) (pulley_vshli8x16 a b))-(rule (lower (has_type $I16X8 (ishl a b))) (pulley_vshli16x8 a b))-(rule (lower (has_type $I32X4 (ishl a b))) (pulley_vshli32x4 a b))-(rule (lower (has_type $I64X2 (ishl a b))) (pulley_vshli64x2 a b))+(rule (lower (has_type $I8X16 (ishl _ a b))) (pulley_vshli8x16 a b))+(rule (lower (has_type $I16X8 (ishl _ a b))) (pulley_vshli16x8 a b))+(rule (lower (has_type $I32X4 (ishl _ a b))) (pulley_vshli32x4 a b))+(rule (lower (has_type $I64X2 (ishl _ a b))) (pulley_vshli64x2 a b));; Helper to extract a constant from `Value`, mask it to 6 bits, and then make a;; `U6`.@@ -533,211 +533,211 @@;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (ushr a b)))+(rule (lower (has_type $I8 (ushr _ a b)))(pulley_xshr32_u (zext32 a) (pulley_xband32_s8 b 7)))-(rule (lower (has_type $I16 (ushr a b)))+(rule (lower (has_type $I16 (ushr _ a b)))(pulley_xshr32_u (zext32 a) (pulley_xband32_s8 b 15)))-(rule (lower (has_type $I32 (ushr a b)))+(rule (lower (has_type $I32 (ushr _ a b)))(pulley_xshr32_u a b))-(rule (lower (has_type $I64 (ushr a b)))+(rule (lower (has_type $I64 (ushr _ a b)))(pulley_xshr64_u a b));; Special-case constant shift amounts.-(rule 1 (lower (has_type $I32 (ushr a b)))+(rule 1 (lower (has_type $I32 (ushr _ a b)))(if-let n (u6_shift_from_iconst b))(pulley_xshr32_u_u6 a n))-(rule 1 (lower (has_type $I64 (ushr a b)))+(rule 1 (lower (has_type $I64 (ushr _ a b)))(if-let n (u6_shift_from_iconst b))(pulley_xshr64_u_u6 a n));; vector shifts-(rule (lower (has_type $I8X16 (ushr a b))) (pulley_vshri8x16_u a b))-(rule (lower (has_type $I16X8 (ushr a b))) (pulley_vshri16x8_u a b))-(rule (lower (has_type $I32X4 (ushr a b))) (pulley_vshri32x4_u a b))-(rule (lower (has_type $I64X2 (ushr a b))) (pulley_vshri64x2_u a b))+(rule (lower (has_type $I8X16 (ushr _ a b))) (pulley_vshri8x16_u a b))+(rule (lower (has_type $I16X8 (ushr _ a b))) (pulley_vshri16x8_u a b))+(rule (lower (has_type $I32X4 (ushr _ a b))) (pulley_vshri32x4_u a b))+(rule (lower (has_type $I64X2 (ushr _ a b))) (pulley_vshri64x2_u a b));;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (sshr a b)))+(rule (lower (has_type $I8 (sshr _ a b)))(pulley_xshr32_u (sext32 a) (pulley_xband32_s8 b 7)))-(rule (lower (has_type $I16 (sshr a b)))+(rule (lower (has_type $I16 (sshr _ a b)))(pulley_xshr32_u (sext32 a) (pulley_xband32_s8 b 15)))-(rule (lower (has_type $I32 (sshr a b)))+(rule (lower (has_type $I32 (sshr _ a b)))(pulley_xshr32_s a b))-(rule (lower (has_type $I64 (sshr a b)))+(rule (lower (has_type $I64 (sshr _ a b)))(pulley_xshr64_s a b));; Special-case constant shift amounts.-(rule 1 (lower (has_type $I32 (sshr a b)))+(rule 1 (lower (has_type $I32 (sshr _ a b)))(if-let n (u6_shift_from_iconst b))(pulley_xshr32_s_u6 a n))-(rule 1 (lower (has_type $I64 (sshr a b)))+(rule 1 (lower (has_type $I64 (sshr _ a b)))(if-let n (u6_shift_from_iconst b))(pulley_xshr64_s_u6 a n));; vector shifts-(rule (lower (has_type $I8X16 (sshr a b))) (pulley_vshri8x16_s a b))-(rule (lower (has_type $I16X8 (sshr a b))) (pulley_vshri16x8_s a b))-(rule (lower (has_type $I32X4 (sshr a b))) (pulley_vshri32x4_s a b))-(rule (lower (has_type $I64X2 (sshr a b))) (pulley_vshri64x2_s a b))+(rule (lower (has_type $I8X16 (sshr _ a b))) (pulley_vshri8x16_s a b))+(rule (lower (has_type $I16X8 (sshr _ a b))) (pulley_vshri16x8_s a b))+(rule (lower (has_type $I32X4 (sshr _ a b))) (pulley_vshri32x4_s a b))+(rule (lower (has_type $I64X2 (sshr _ a b))) (pulley_vshri64x2_s a b));;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (band a b))) (pulley_xband32 a b))-(rule 1 (lower (has_type $I64 (band a b))) (pulley_xband64 a b))+(rule 0 (lower (has_type (fits_in_32 _) (band _ a b))) (pulley_xband32 a b))+(rule 1 (lower (has_type $I64 (band _ a b))) (pulley_xband64 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (band a (i32_from_iconst b))))+(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (band _ a (i32_from_iconst b))))(pulley_xband32_s32 a b))-(rule 4 (lower (has_type $I64 (band a (i32_from_iconst b))))+(rule 4 (lower (has_type $I64 (band _ a (i32_from_iconst b))))(pulley_xband64_s32 a b))-(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (band a (i8_from_iconst b))))+(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (band _ a (i8_from_iconst b))))(pulley_xband32_s8 a b))-(rule 6 (lower (has_type $I64 (band a (i8_from_iconst b))))+(rule 6 (lower (has_type $I64 (band _ a (i8_from_iconst b))))(pulley_xband64_s8 a b))-(rule 2 (lower (has_type (ty_vec128 _) (band a b)))+(rule 2 (lower (has_type (ty_vec128 _) (band _ a b)))(pulley_vband128 a b));;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (bor a b))) (pulley_xbor32 a b))-(rule 1 (lower (has_type $I64 (bor a b))) (pulley_xbor64 a b))+(rule 0 (lower (has_type (fits_in_32 _) (bor _ a b))) (pulley_xbor32 a b))+(rule 1 (lower (has_type $I64 (bor _ a b))) (pulley_xbor64 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (bor a (i32_from_iconst b))))+(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (bor _ a (i32_from_iconst b))))(pulley_xbor32_s32 a b))-(rule 4 (lower (has_type $I64 (bor a (i32_from_iconst b))))+(rule 4 (lower (has_type $I64 (bor _ a (i32_from_iconst b))))(pulley_xbor64_s32 a b))-(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (bor a (i8_from_iconst b))))+(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (bor _ a (i8_from_iconst b))))(pulley_xbor32_s8 a b))-(rule 6 (lower (has_type $I64 (bor a (i8_from_iconst b))))+(rule 6 (lower (has_type $I64 (bor _ a (i8_from_iconst b))))(pulley_xbor64_s8 a b))-(rule 2 (lower (has_type (ty_vec128 _) (bor a b)))+(rule 2 (lower (has_type (ty_vec128 _) (bor _ a b)))(pulley_vbor128 a b));;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (bxor a b))) (pulley_xbxor32 a b))-(rule 1 (lower (has_type $I64 (bxor a b))) (pulley_xbxor64 a b))+(rule 0 (lower (has_type (fits_in_32 _) (bxor _ a b))) (pulley_xbxor32 a b))+(rule 1 (lower (has_type $I64 (bxor _ a b))) (pulley_xbxor64 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (bxor a (i32_from_iconst b))))+(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (bxor _ a (i32_from_iconst b))))(pulley_xbxor32_s32 a b))-(rule 4 (lower (has_type $I64 (bxor a (i32_from_iconst b))))+(rule 4 (lower (has_type $I64 (bxor _ a (i32_from_iconst b))))(pulley_xbxor64_s32 a b))-(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (bxor a (i8_from_iconst b))))+(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (bxor _ a (i8_from_iconst b))))(pulley_xbxor32_s8 a b))-(rule 6 (lower (has_type $I64 (bxor a (i8_from_iconst b))))+(rule 6 (lower (has_type $I64 (bxor _ a (i8_from_iconst b))))(pulley_xbxor64_s8 a b))-(rule 2 (lower (has_type (ty_vec128 _) (bxor a b)))+(rule 2 (lower (has_type (ty_vec128 _) (bxor _ a b)))(pulley_vbxor128 a b));;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (bnot a)))+(rule 0 (lower (has_type (fits_in_32 _) (bnot _ a)))(pulley_xbnot32 a))-(rule 1 (lower (has_type $I64 (bnot a)))+(rule 1 (lower (has_type $I64 (bnot _ a)))(pulley_xbnot64 a))-(rule 2 (lower (has_type (ty_vec128 _) (bnot a)))+(rule 2 (lower (has_type (ty_vec128 _) (bnot _ a)))(pulley_vbnot128 a));;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 _) (bitselect c x y)))+(rule (lower (has_type (ty_vec128 _) (bitselect _ c x y)))(pulley_vbitselect128 c x y));;;; Rules for `umin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (umin a b)))+(rule 0 (lower (has_type (fits_in_32 _) (umin _ a b)))(pulley_xmin32_u (zext32 a) (zext32 b)))-(rule 1 (lower (has_type $I64 (umin a b))) (pulley_xmin64_u a b))-(rule 1 (lower (has_type $I8X16 (umin a b))) (pulley_vmin8x16_u a b))-(rule 1 (lower (has_type $I16X8 (umin a b))) (pulley_vmin16x8_u a b))-(rule 1 (lower (has_type $I32X4 (umin a b))) (pulley_vmin32x4_u a b))+(rule 1 (lower (has_type $I64 (umin _ a b))) (pulley_xmin64_u a b))+(rule 1 (lower (has_type $I8X16 (umin _ a b))) (pulley_vmin8x16_u a b))+(rule 1 (lower (has_type $I16X8 (umin _ a b))) (pulley_vmin16x8_u a b))+(rule 1 (lower (has_type $I32X4 (umin _ a b))) (pulley_vmin32x4_u a b));;;; Rules for `smin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (smin a b)))+(rule 0 (lower (has_type (fits_in_32 _) (smin _ a b)))(pulley_xmin32_s (sext32 a) (sext32 b)))-(rule 1 (lower (has_type $I64 (smin a b))) (pulley_xmin64_s a b))-(rule 1 (lower (has_type $I8X16 (smin a b))) (pulley_vmin8x16_s a b))-(rule 1 (lower (has_type $I16X8 (smin a b))) (pulley_vmin16x8_s a b))-(rule 1 (lower (has_type $I32X4 (smin a b))) (pulley_vmin32x4_s a b))+(rule 1 (lower (has_type $I64 (smin _ a b))) (pulley_xmin64_s a b))+(rule 1 (lower (has_type $I8X16 (smin _ a b))) (pulley_vmin8x16_s a b))+(rule 1 (lower (has_type $I16X8 (smin _ a b))) (pulley_vmin16x8_s a b))+(rule 1 (lower (has_type $I32X4 (smin _ a b))) (pulley_vmin32x4_s a b));;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (umax a b)))+(rule 0 (lower (has_type (fits_in_32 _) (umax _ a b)))(pulley_xmax32_u (zext32 a) (zext32 b)))-(rule 1 (lower (has_type $I64 (umax a b))) (pulley_xmax64_u a b))-(rule 1 (lower (has_type $I8X16 (umax a b))) (pulley_vmax8x16_u a b))-(rule 1 (lower (has_type $I16X8 (umax a b))) (pulley_vmax16x8_u a b))-(rule 1 (lower (has_type $I32X4 (umax a b))) (pulley_vmax32x4_u a b))+(rule 1 (lower (has_type $I64 (umax _ a b))) (pulley_xmax64_u a b))+(rule 1 (lower (has_type $I8X16 (umax _ a b))) (pulley_vmax8x16_u a b))+(rule 1 (lower (has_type $I16X8 (umax _ a b))) (pulley_vmax16x8_u a b))+(rule 1 (lower (has_type $I32X4 (umax _ a b))) (pulley_vmax32x4_u a b));;;; Rules for `smax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (smax a b)))+(rule 0 (lower (has_type (fits_in_32 _) (smax _ a b)))(pulley_xmax32_s (sext32 a) (sext32 b)))-(rule 1 (lower (has_type $I64 (smax a b))) (pulley_xmax64_s a b))-(rule 1 (lower (has_type $I8X16 (smax a b))) (pulley_vmax8x16_s a b))-(rule 1 (lower (has_type $I16X8 (smax a b))) (pulley_vmax16x8_s a b))-(rule 1 (lower (has_type $I32X4 (smax a b))) (pulley_vmax32x4_s a b))+(rule 1 (lower (has_type $I64 (smax _ a b))) (pulley_xmax64_s a b))+(rule 1 (lower (has_type $I8X16 (smax _ a b))) (pulley_vmax8x16_s a b))+(rule 1 (lower (has_type $I16X8 (smax _ a b))) (pulley_vmax16x8_s a b))+(rule 1 (lower (has_type $I32X4 (smax _ a b))) (pulley_vmax32x4_s a b));;;; Rules for `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (bmask a @ (value_type (fits_in_32 _)))))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (bmask _ a @ (value_type (fits_in_32 _)))))(pulley_xbmask32 (zext32 a)))-(rule 1 (lower (has_type $I64 (bmask a)))+(rule 1 (lower (has_type $I64 (bmask _ a)))(pulley_xbmask64 (zext64 a)))-(rule 2 (lower (bmask a @ (value_type $I64)))+(rule 2 (lower (bmask _ a @ (value_type $I64)))(pulley_xbmask64 a));;;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (ctz a)))+(rule (lower (has_type $I8 (ctz _ a)))(pulley_xctz32 (pulley_xbor32_s32 a 0x100)))-(rule (lower (has_type $I16 (ctz a)))+(rule (lower (has_type $I16 (ctz _ a)))(pulley_xctz32 (pulley_xbor32_s32 a 0x10000)))-(rule (lower (has_type $I32 (ctz a))) (pulley_xctz32 a))-(rule (lower (has_type $I64 (ctz a))) (pulley_xctz64 a))+(rule (lower (has_type $I32 (ctz _ a))) (pulley_xctz32 a))+(rule (lower (has_type $I64 (ctz _ a))) (pulley_xctz64 a));;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (clz a)))+(rule (lower (has_type $I8 (clz _ a)))(pulley_xsub32_u8 (pulley_xclz32 (zext32 a)) 24))-(rule (lower (has_type $I16 (clz a)))+(rule (lower (has_type $I16 (clz _ a)))(pulley_xsub32_u8 (pulley_xclz32 (zext32 a)) 16))-(rule (lower (has_type $I32 (clz a))) (pulley_xclz32 a))-(rule (lower (has_type $I64 (clz a))) (pulley_xclz64 a))+(rule (lower (has_type $I32 (clz _ a))) (pulley_xclz32 a))+(rule (lower (has_type $I64 (clz _ a))) (pulley_xclz64 a));;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (popcnt a))) (pulley_xpopcnt32 (zext32 a)))-(rule 1 (lower (has_type $I64 (popcnt a))) (pulley_xpopcnt64 a))-(rule 1 (lower (has_type $I8X16 (popcnt a))) (pulley_vpopcnt8x16 a))+(rule 0 (lower (has_type (fits_in_32 _) (popcnt _ a))) (pulley_xpopcnt32 (zext32 a)))+(rule 1 (lower (has_type $I64 (popcnt _ a))) (pulley_xpopcnt64 a))+(rule 1 (lower (has_type $I8X16 (popcnt _ a))) (pulley_vpopcnt8x16 a));;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (rotl a b))) (pulley_xrotl32 a b))-(rule (lower (has_type $I64 (rotl a b))) (pulley_xrotl64 a b))+(rule (lower (has_type $I32 (rotl _ a b))) (pulley_xrotl32 a b))+(rule (lower (has_type $I64 (rotl _ a b))) (pulley_xrotl64 a b));;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (rotr a b))) (pulley_xrotr32 a b))-(rule (lower (has_type $I64 (rotr a b))) (pulley_xrotr64 a b))+(rule (lower (has_type $I32 (rotr _ a b))) (pulley_xrotr32 a b))+(rule (lower (has_type $I64 (rotr _ a b))) (pulley_xrotr64 a b));;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (icmp cc a b @ (value_type (ty_int ty))))+(rule (lower (icmp _ cc a b @ (value_type (ty_int ty))))(lower_icmp ty cc a b)); Recursion: bounded since only recursive rules swap condition code order from@@ -848,7 +848,7 @@;; vector comparisons-(rule 1 (lower (icmp cc a @ (value_type (ty_vec128 ty)) b))+(rule 1 (lower (icmp _ cc a @ (value_type (ty_vec128 ty)) b))(lower_vcmp ty cc a b)); Recursion: bounded since only recursive rules swap condition code order from@@ -891,10 +891,10 @@;;;; Rules for `fcmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (fcmp cc a b @ (value_type (ty_scalar_float ty))))+(rule (lower (fcmp _ cc a b @ (value_type (ty_scalar_float ty))))(lower_fcmp ty cc a b))-(rule 1 (lower (fcmp cc a b @ (value_type (ty_vec128 ty))))+(rule 1 (lower (fcmp _ cc a b @ (value_type (ty_vec128 ty))))(lower_vfcmp ty cc a b)); Recursion: bounded since recursive rules only implement certain condition@@ -960,73 +960,73 @@;;;; Rules for `load` and friends ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_int (fits_in_64 ty)) (load flags addr offset)))+(rule (lower (has_type (ty_int (fits_in_64 ty)) (load _ flags addr offset)))(gen_xload addr offset flags ty (ExtKind.None)))-(rule 1 (lower (has_type (ty_scalar_float ty) (load flags addr offset)))+(rule 1 (lower (has_type (ty_scalar_float ty) (load _ flags addr offset)))(gen_fload addr offset flags ty))-(rule 3 (lower (has_type $I128 (load flags addr offset)))+(rule 3 (lower (has_type $I128 (load _ flags addr offset)))(if-let offsetp8 (i32_checked_add offset 8))(let ((lo XReg (pulley_xload (amode addr offset) $I64 flags))(hi XReg (pulley_xload (amode addr offsetp8) $I64 flags)))(value_regs lo hi)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload8 flags addr offset)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload8 _ flags addr offset)))(gen_xload addr offset flags $I8 (ExtKind.Zero32)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload16 flags addr offset)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload16 _ flags addr offset)))(gen_xload addr offset flags $I16 (ExtKind.Zero32)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload32 flags addr offset)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload32 _ flags addr offset)))(gen_xload addr offset flags $I32 (ExtKind.None)))-(rule 1 (lower (has_type $I64 (uload8 flags addr offset)))+(rule 1 (lower (has_type $I64 (uload8 _ flags addr offset)))(gen_xload addr offset flags $I8 (ExtKind.Zero64)))-(rule 1 (lower (has_type $I64 (uload16 flags addr offset)))+(rule 1 (lower (has_type $I64 (uload16 _ flags addr offset)))(gen_xload addr offset flags $I16 (ExtKind.Zero64)))-(rule 1 (lower (has_type $I64 (uload32 flags addr offset)))+(rule 1 (lower (has_type $I64 (uload32 _ flags addr offset)))(gen_xload addr offset flags $I32 (ExtKind.Zero64)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload8 flags addr offset)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload8 _ flags addr offset)))(gen_xload addr offset flags $I8 (ExtKind.Sign32)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload16 flags addr offset)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload16 _ flags addr offset)))(gen_xload addr offset flags $I16 (ExtKind.Sign32)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload32 flags addr offset)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload32 _ flags addr offset)))(gen_xload addr offset flags $I32 (ExtKind.None)))-(rule 1 (lower (has_type $I64 (sload8 flags addr offset)))+(rule 1 (lower (has_type $I64 (sload8 _ flags addr offset)))(gen_xload addr offset flags $I8 (ExtKind.Sign64)))-(rule 1 (lower (has_type $I64 (sload16 flags addr offset)))+(rule 1 (lower (has_type $I64 (sload16 _ flags addr offset)))(gen_xload addr offset flags $I16 (ExtKind.Sign64)))-(rule 1 (lower (has_type $I64 (sload32 flags addr offset)))+(rule 1 (lower (has_type $I64 (sload32 _ flags addr offset)))(gen_xload addr offset flags $I32 (ExtKind.Sign64)))-(rule 2 (lower (has_type (ty_vec128 ty) (load flags addr offset)))+(rule 2 (lower (has_type (ty_vec128 ty) (load _ flags addr offset)))(gen_vload addr offset flags ty (VExtKind.None)))-(rule (lower (has_type ty (sload8x8 flags addr offset)))+(rule (lower (has_type ty (sload8x8 _ flags addr offset)))(gen_vload addr offset flags ty (VExtKind.S8x8)))-(rule (lower (has_type ty (uload8x8 flags addr offset)))+(rule (lower (has_type ty (uload8x8 _ flags addr offset)))(gen_vload addr offset flags ty (VExtKind.U8x8)))-(rule (lower (has_type ty (sload16x4 flags addr offset)))+(rule (lower (has_type ty (sload16x4 _ flags addr offset)))(gen_vload addr offset flags ty (VExtKind.S16x4)))-(rule (lower (has_type ty (uload16x4 flags addr offset)))+(rule (lower (has_type ty (uload16x4 _ flags addr offset)))(gen_vload addr offset flags ty (VExtKind.U16x4)))-(rule (lower (has_type ty (sload32x2 flags addr offset)))+(rule (lower (has_type ty (sload32x2 _ flags addr offset)))(gen_vload addr offset flags ty (VExtKind.S32x2)))-(rule (lower (has_type ty (uload32x2 flags addr offset)))+(rule (lower (has_type ty (uload32x2 _ flags addr offset)))(gen_vload addr offset flags ty (VExtKind.U32x2)));; Helper to generate an `xload*` instruction, of which there are many. This@@ -1293,7 +1293,7 @@;;;; Rules for `stack_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (stack_addr stack_slot offset))+(rule (lower (stack_addr _ stack_slot offset))(lower_stack_addr stack_slot offset))(decl lower_stack_addr (StackSlot Offset32) XReg)@@ -1304,24 +1304,24 @@;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (uextend val)))+(rule 0 (lower (has_type (fits_in_32 _) (uextend _ val)))(zext32 val))-(rule 1 (lower (has_type $I64 (uextend val)))+(rule 1 (lower (has_type $I64 (uextend _ val)))(zext64 val))-(rule 1 (lower (has_type $I128 (uextend val)))+(rule 1 (lower (has_type $I128 (uextend _ val)))(value_regs (zext64 val) (pulley_xzero)));;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (sextend val)))+(rule 0 (lower (has_type (fits_in_32 _) (sextend _ val)))(sext32 val))-(rule 1 (lower (has_type $I64 (sextend val)))+(rule 1 (lower (has_type $I64 (sextend _ val)))(sext64 val))-(rule 1 (lower (has_type $I128 (sextend val)))+(rule 1 (lower (has_type $I128 (sextend _ val)))(if-let (u6_from_u8 shift) (u64_try_into_u8 63))(let ((lo XReg (sext64 val))(hi XReg (pulley_xshr64_s_u6 lo shift)))@@ -1329,56 +1329,56 @@;;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 _ty) (ireduce src)))+(rule (lower (has_type (fits_in_64 _ty) (ireduce _ src)))src);;;; Rules for `iconcat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I128 (iconcat a b)))+(rule (lower (has_type $I128 (iconcat _ a b)))(value_regs a b));;;; Rules for `isplit` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (isplit x @ (value_type $I128)))+(rule (lower (isplit _ x @ (value_type $I128)))(let ((lo XReg (value_regs_get x 0))(hi XReg (value_regs_get x 1)))(output_pair lo hi)));;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (uadd_overflow_trap a b tc)))+(rule (lower (has_type $I32 (uadd_overflow_trap _ a b tc)))(pulley_xadd32_uoverflow_trap a b tc))-(rule (lower (has_type $I64 (uadd_overflow_trap a b tc)))+(rule (lower (has_type $I64 (uadd_overflow_trap _ a b tc)))(pulley_xadd64_uoverflow_trap a b tc));;;; Rules for `uadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (uadd_overflow a b)))+(rule (lower (has_type $I32 (uadd_overflow _ a b)))(let ((sum XReg (pulley_xadd32 a b))(is_overflow XReg (pulley_xult32 sum a)))(output_pair sum is_overflow)))-(rule (lower (has_type $I64 (uadd_overflow a b)))+(rule (lower (has_type $I64 (uadd_overflow _ a b)))(let ((sum XReg (pulley_xadd64 a b))(is_overflow XReg (pulley_xult64 sum a)))(output_pair sum is_overflow)));;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (select c a b)))+(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (select _ c a b)))(pulley_xselect32 (emit_cond (lower_cond c)) a b))-(rule 1 (lower (has_type $I64 (select c a b)))+(rule 1 (lower (has_type $I64 (select _ c a b)))(pulley_xselect64 (emit_cond (lower_cond c)) a b))-(rule 1 (lower (has_type $F32 (select c a b)))+(rule 1 (lower (has_type $F32 (select _ c a b)))(pulley_fselect32 (emit_cond (lower_cond c)) a b))-(rule 1 (lower (has_type $F64 (select c a b)))+(rule 1 (lower (has_type $F64 (select _ c a b)))(pulley_fselect64 (emit_cond (lower_cond c)) a b))-(rule 2 (lower (has_type (ty_vec128 _) (select c a b)))+(rule 2 (lower (has_type (ty_vec128 _) (select _ c a b)))(pulley_vselect (emit_cond (lower_cond c)) a b));; Helper to emit a conditional into a register itself.@@ -1426,388 +1426,388 @@;;;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (bitcast _flags val @ (value_type $I32))))+(rule (lower (has_type $F32 (bitcast _ _flags val @ (value_type $I32))))(pulley_bitcast_float_from_int_32 val))-(rule (lower (has_type $F64 (bitcast _flags val @ (value_type $I64))))+(rule (lower (has_type $F64 (bitcast _ _flags val @ (value_type $I64))))(pulley_bitcast_float_from_int_64 val))-(rule (lower (has_type $I32 (bitcast _flags val @ (value_type $F32))))+(rule (lower (has_type $I32 (bitcast _ _flags val @ (value_type $F32))))(pulley_bitcast_int_from_float_32 val))-(rule (lower (has_type $I64 (bitcast _flags val @ (value_type $F64))))+(rule (lower (has_type $I64 (bitcast _ _flags val @ (value_type $F64))))(pulley_bitcast_int_from_float_64 val))-(rule 1 (lower (has_type (ty_vec128 _) (bitcast _flags val @ (value_type (ty_vec128 _)))))+(rule 1 (lower (has_type (ty_vec128 _) (bitcast _ _flags val @ (value_type (ty_vec128 _)))))val)-(rule 2 (lower (has_type ty (bitcast _flags a @ (value_type ty)))) a)+(rule 2 (lower (has_type ty (bitcast _ _flags a @ (value_type ty)))) a);;;; Rules for `fcvt_to_{u,s}int` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (fcvt_to_uint val @ (value_type $F32))))+(rule (lower (has_type $I32 (fcvt_to_uint _ val @ (value_type $F32))))(pulley_x32_from_f32_u val))-(rule (lower (has_type $I32 (fcvt_to_uint val @ (value_type $F64))))+(rule (lower (has_type $I32 (fcvt_to_uint _ val @ (value_type $F64))))(pulley_x32_from_f64_u val))-(rule (lower (has_type $I64 (fcvt_to_uint val @ (value_type $F32))))+(rule (lower (has_type $I64 (fcvt_to_uint _ val @ (value_type $F32))))(pulley_x64_from_f32_u val))-(rule (lower (has_type $I64 (fcvt_to_uint val @ (value_type $F64))))+(rule (lower (has_type $I64 (fcvt_to_uint _ val @ (value_type $F64))))(pulley_x64_from_f64_u val))-(rule (lower (has_type $I32 (fcvt_to_sint val @ (value_type $F32))))+(rule (lower (has_type $I32 (fcvt_to_sint _ val @ (value_type $F32))))(pulley_x32_from_f32_s val))-(rule (lower (has_type $I32 (fcvt_to_sint val @ (value_type $F64))))+(rule (lower (has_type $I32 (fcvt_to_sint _ val @ (value_type $F64))))(pulley_x32_from_f64_s val))-(rule (lower (has_type $I64 (fcvt_to_sint val @ (value_type $F32))))+(rule (lower (has_type $I64 (fcvt_to_sint _ val @ (value_type $F32))))(pulley_x64_from_f32_s val))-(rule (lower (has_type $I64 (fcvt_to_sint val @ (value_type $F64))))+(rule (lower (has_type $I64 (fcvt_to_sint _ val @ (value_type $F64))))(pulley_x64_from_f64_s val));;;; Rules for `fcvt_from_{u,s}int` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type $F32 (fcvt_from_uint val @ (value_type (fits_in_32 _)))))+(rule 0 (lower (has_type $F32 (fcvt_from_uint _ val @ (value_type (fits_in_32 _)))))(pulley_f32_from_x32_u (zext32 val)))-(rule 1 (lower (has_type $F32 (fcvt_from_uint val @ (value_type $I64))))+(rule 1 (lower (has_type $F32 (fcvt_from_uint _ val @ (value_type $I64))))(pulley_f32_from_x64_u val))-(rule 0 (lower (has_type $F64 (fcvt_from_uint val @ (value_type (fits_in_32 _)))))+(rule 0 (lower (has_type $F64 (fcvt_from_uint _ val @ (value_type (fits_in_32 _)))))(pulley_f64_from_x32_u (zext32 val)))-(rule 1 (lower (has_type $F64 (fcvt_from_uint val @ (value_type $I64))))+(rule 1 (lower (has_type $F64 (fcvt_from_uint _ val @ (value_type $I64))))(pulley_f64_from_x64_u val))-(rule 0 (lower (has_type $F32 (fcvt_from_sint val @ (value_type (fits_in_32 _)))))+(rule 0 (lower (has_type $F32 (fcvt_from_sint _ val @ (value_type (fits_in_32 _)))))(pulley_f32_from_x32_s (sext32 val)))-(rule 1 (lower (has_type $F32 (fcvt_from_sint val @ (value_type $I64))))+(rule 1 (lower (has_type $F32 (fcvt_from_sint _ val @ (value_type $I64))))(pulley_f32_from_x64_s val))-(rule 0 (lower (has_type $F64 (fcvt_from_sint val @ (value_type (fits_in_32 _)))))+(rule 0 (lower (has_type $F64 (fcvt_from_sint _ val @ (value_type (fits_in_32 _)))))(pulley_f64_from_x32_s (sext32 val)))-(rule 1 (lower (has_type $F64 (fcvt_from_sint val @ (value_type $I64))))+(rule 1 (lower (has_type $F64 (fcvt_from_sint _ val @ (value_type $I64))))(pulley_f64_from_x64_s val))-(rule (lower (has_type $F32X4 (fcvt_from_sint val @ (value_type $I32X4))))+(rule (lower (has_type $F32X4 (fcvt_from_sint _ val @ (value_type $I32X4))))(pulley_vf32x4_from_i32x4_s val))-(rule (lower (has_type $F32X4 (fcvt_from_uint val @ (value_type $I32X4))))+(rule (lower (has_type $F32X4 (fcvt_from_uint _ val @ (value_type $I32X4))))(pulley_vf32x4_from_i32x4_u val))-(rule (lower (has_type $F64X2 (fcvt_from_sint val @ (value_type $I64X2))))+(rule (lower (has_type $F64X2 (fcvt_from_sint _ val @ (value_type $I64X2))))(pulley_vf64x2_from_i64x2_s val))-(rule (lower (has_type $F64X2 (fcvt_from_uint val @ (value_type $I64X2))))+(rule (lower (has_type $F64X2 (fcvt_from_uint _ val @ (value_type $I64X2))))(pulley_vf64x2_from_i64x2_u val));;;; Rules for `fcvt_to_{u,s}int_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (fcvt_to_uint_sat val @ (value_type $F32))))+(rule (lower (has_type $I32 (fcvt_to_uint_sat _ val @ (value_type $F32))))(pulley_x32_from_f32_u_sat val))-(rule (lower (has_type $I32 (fcvt_to_uint_sat val @ (value_type $F64))))+(rule (lower (has_type $I32 (fcvt_to_uint_sat _ val @ (value_type $F64))))(pulley_x32_from_f64_u_sat val))-(rule (lower (has_type $I64 (fcvt_to_uint_sat val @ (value_type $F32))))+(rule (lower (has_type $I64 (fcvt_to_uint_sat _ val @ (value_type $F32))))(pulley_x64_from_f32_u_sat val))-(rule (lower (has_type $I64 (fcvt_to_uint_sat val @ (value_type $F64))))+(rule (lower (has_type $I64 (fcvt_to_uint_sat _ val @ (value_type $F64))))(pulley_x64_from_f64_u_sat val))-(rule (lower (has_type $I32 (fcvt_to_sint_sat val @ (value_type $F32))))+(rule (lower (has_type $I32 (fcvt_to_sint_sat _ val @ (value_type $F32))))(pulley_x32_from_f32_s_sat val))-(rule (lower (has_type $I32 (fcvt_to_sint_sat val @ (value_type $F64))))+(rule (lower (has_type $I32 (fcvt_to_sint_sat _ val @ (value_type $F64))))(pulley_x32_from_f64_s_sat val))-(rule (lower (has_type $I64 (fcvt_to_sint_sat val @ (value_type $F32))))+(rule (lower (has_type $I64 (fcvt_to_sint_sat _ val @ (value_type $F32))))(pulley_x64_from_f32_s_sat val))-(rule (lower (has_type $I64 (fcvt_to_sint_sat val @ (value_type $F64))))+(rule (lower (has_type $I64 (fcvt_to_sint_sat _ val @ (value_type $F64))))(pulley_x64_from_f64_s_sat val))-(rule (lower (has_type $I32X4 (fcvt_to_sint_sat val @ (value_type $F32X4))))+(rule (lower (has_type $I32X4 (fcvt_to_sint_sat _ val @ (value_type $F32X4))))(pulley_vi32x4_from_f32x4_s val))-(rule (lower (has_type $I32X4 (fcvt_to_uint_sat val @ (value_type $F32X4))))+(rule (lower (has_type $I32X4 (fcvt_to_uint_sat _ val @ (value_type $F32X4))))(pulley_vi32x4_from_f32x4_u val))-(rule (lower (has_type $I64X2 (fcvt_to_sint_sat val @ (value_type $F64X2))))+(rule (lower (has_type $I64X2 (fcvt_to_sint_sat _ val @ (value_type $F64X2))))(pulley_vi64x2_from_f64x2_s val))-(rule (lower (has_type $I64X2 (fcvt_to_uint_sat val @ (value_type $F64X2))))+(rule (lower (has_type $I64X2 (fcvt_to_uint_sat _ val @ (value_type $F64X2))))(pulley_vi64x2_from_f64x2_u val));;;; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdemote val @ (value_type $F64))))+(rule (lower (has_type $F32 (fdemote _ val @ (value_type $F64))))(pulley_f32_from_f64 val));;;; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F64 (fpromote val @ (value_type $F32))))+(rule (lower (has_type $F64 (fpromote _ val @ (value_type $F32))))(pulley_f64_from_f32 val));;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fcopysign a b)))+(rule (lower (has_type $F32 (fcopysign _ a b)))(pulley_fcopysign32 a b))-(rule (lower (has_type $F64 (fcopysign a b)))+(rule (lower (has_type $F64 (fcopysign _ a b)))(pulley_fcopysign64 a b));;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fadd a b))) (pulley_fadd32 a b))-(rule (lower (has_type $F64 (fadd a b))) (pulley_fadd64 a b))-(rule (lower (has_type $F32X4 (fadd a b))) (pulley_vaddf32x4 a b))-(rule (lower (has_type $F64X2 (fadd a b))) (pulley_vaddf64x2 a b))+(rule (lower (has_type $F32 (fadd _ a b))) (pulley_fadd32 a b))+(rule (lower (has_type $F64 (fadd _ a b))) (pulley_fadd64 a b))+(rule (lower (has_type $F32X4 (fadd _ a b))) (pulley_vaddf32x4 a b))+(rule (lower (has_type $F64X2 (fadd _ a b))) (pulley_vaddf64x2 a b));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fsub a b))) (pulley_fsub32 a b))-(rule (lower (has_type $F64 (fsub a b))) (pulley_fsub64 a b))-(rule (lower (has_type $F32X4 (fsub a b))) (pulley_vsubf32x4 a b))-(rule (lower (has_type $F64X2 (fsub a b))) (pulley_vsubf64x2 a b))+(rule (lower (has_type $F32 (fsub _ a b))) (pulley_fsub32 a b))+(rule (lower (has_type $F64 (fsub _ a b))) (pulley_fsub64 a b))+(rule (lower (has_type $F32X4 (fsub _ a b))) (pulley_vsubf32x4 a b))+(rule (lower (has_type $F64X2 (fsub _ a b))) (pulley_vsubf64x2 a b));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmul a b))) (pulley_fmul32 a b))-(rule (lower (has_type $F64 (fmul a b))) (pulley_fmul64 a b))-(rule (lower (has_type $F32X4 (fmul a b))) (pulley_vmulf32x4 a b))-(rule (lower (has_type $F64X2 (fmul a b))) (pulley_vmulf64x2 a b))+(rule (lower (has_type $F32 (fmul _ a b))) (pulley_fmul32 a b))+(rule (lower (has_type $F64 (fmul _ a b))) (pulley_fmul64 a b))+(rule (lower (has_type $F32X4 (fmul _ a b))) (pulley_vmulf32x4 a b))+(rule (lower (has_type $F64X2 (fmul _ a b))) (pulley_vmulf64x2 a b));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdiv a b))) (pulley_fdiv32 a b))-(rule (lower (has_type $F64 (fdiv a b))) (pulley_fdiv64 a b))-(rule (lower (has_type $F32X4 (fdiv a b))) (pulley_vdivf32x4 a b))-(rule (lower (has_type $F64X2 (fdiv a b))) (pulley_vdivf64x2 a b))+(rule (lower (has_type $F32 (fdiv _ a b))) (pulley_fdiv32 a b))+(rule (lower (has_type $F64 (fdiv _ a b))) (pulley_fdiv64 a b))+(rule (lower (has_type $F32X4 (fdiv _ a b))) (pulley_vdivf32x4 a b))+(rule (lower (has_type $F64X2 (fdiv _ a b))) (pulley_vdivf64x2 a b));;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmax a b))) (pulley_fmaximum32 a b))-(rule (lower (has_type $F64 (fmax a b))) (pulley_fmaximum64 a b))-(rule (lower (has_type $F32X4 (fmax a b))) (pulley_vmaximumf32x4 a b))-(rule (lower (has_type $F64X2 (fmax a b))) (pulley_vmaximumf64x2 a b))+(rule (lower (has_type $F32 (fmax _ a b))) (pulley_fmaximum32 a b))+(rule (lower (has_type $F64 (fmax _ a b))) (pulley_fmaximum64 a b))+(rule (lower (has_type $F32X4 (fmax _ a b))) (pulley_vmaximumf32x4 a b))+(rule (lower (has_type $F64X2 (fmax _ a b))) (pulley_vmaximumf64x2 a b));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmin a b))) (pulley_fminimum32 a b))-(rule (lower (has_type $F64 (fmin a b))) (pulley_fminimum64 a b))-(rule (lower (has_type $F32X4 (fmin a b))) (pulley_vminimumf32x4 a b))-(rule (lower (has_type $F64X2 (fmin a b))) (pulley_vminimumf64x2 a b))+(rule (lower (has_type $F32 (fmin _ a b))) (pulley_fminimum32 a b))+(rule (lower (has_type $F64 (fmin _ a b))) (pulley_fminimum64 a b))+(rule (lower (has_type $F32X4 (fmin _ a b))) (pulley_vminimumf32x4 a b))+(rule (lower (has_type $F64X2 (fmin _ a b))) (pulley_vminimumf64x2 a b));;;; Rules for `trunc` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (trunc a))) (pulley_ftrunc32 a))-(rule (lower (has_type $F64 (trunc a))) (pulley_ftrunc64 a))-(rule (lower (has_type $F32X4 (trunc a))) (pulley_vtrunc32x4 a))-(rule (lower (has_type $F64X2 (trunc a))) (pulley_vtrunc64x2 a))+(rule (lower (has_type $F32 (trunc _ a))) (pulley_ftrunc32 a))+(rule (lower (has_type $F64 (trunc _ a))) (pulley_ftrunc64 a))+(rule (lower (has_type $F32X4 (trunc _ a))) (pulley_vtrunc32x4 a))+(rule (lower (has_type $F64X2 (trunc _ a))) (pulley_vtrunc64x2 a));;;; Rules for `floor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (floor a))) (pulley_ffloor32 a))-(rule (lower (has_type $F64 (floor a))) (pulley_ffloor64 a))-(rule (lower (has_type $F32X4 (floor a)))+(rule (lower (has_type $F32 (floor _ a))) (pulley_ffloor32 a))+(rule (lower (has_type $F64 (floor _ a))) (pulley_ffloor64 a))+(rule (lower (has_type $F32X4 (floor _ a)))(pulley_vfloor32x4 a))-(rule (lower (has_type $F64X2 (floor a)))+(rule (lower (has_type $F64X2 (floor _ a)))(pulley_vfloor64x2 a));;;; Rules for `ceil` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (ceil a))) (pulley_fceil32 a))-(rule (lower (has_type $F64 (ceil a))) (pulley_fceil64 a))-(rule (lower (has_type $F64X2 (ceil a)))+(rule (lower (has_type $F32 (ceil _ a))) (pulley_fceil32 a))+(rule (lower (has_type $F64 (ceil _ a))) (pulley_fceil64 a))+(rule (lower (has_type $F64X2 (ceil _ a)))(pulley_vceil64x2 a))-(rule (lower (has_type $F32X4 (ceil a)))+(rule (lower (has_type $F32X4 (ceil _ a)))(pulley_vceil32x4 a));;;; Rules for `nearest` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (nearest a))) (pulley_fnearest32 a))-(rule (lower (has_type $F64 (nearest a))) (pulley_fnearest64 a))-(rule (lower (has_type $F32X4 (nearest a)))+(rule (lower (has_type $F32 (nearest _ a))) (pulley_fnearest32 a))+(rule (lower (has_type $F64 (nearest _ a))) (pulley_fnearest64 a))+(rule (lower (has_type $F32X4 (nearest _ a)))(pulley_vnearest32x4 a))-(rule (lower (has_type $F64X2 (nearest a)))+(rule (lower (has_type $F64X2 (nearest _ a)))(pulley_vnearest64x2 a));;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (sqrt a))) (pulley_fsqrt32 a))-(rule (lower (has_type $F64 (sqrt a))) (pulley_fsqrt64 a))-(rule (lower (has_type $F32X4 (sqrt a)))+(rule (lower (has_type $F32 (sqrt _ a))) (pulley_fsqrt32 a))+(rule (lower (has_type $F64 (sqrt _ a))) (pulley_fsqrt64 a))+(rule (lower (has_type $F32X4 (sqrt _ a)))(pulley_vsqrt32x4 a))-(rule (lower (has_type $F64X2 (sqrt a)))+(rule (lower (has_type $F64X2 (sqrt _ a)))(pulley_vsqrt64x2 a));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fneg a))) (pulley_fneg32 a))-(rule (lower (has_type $F64 (fneg a))) (pulley_fneg64 a))-(rule (lower (has_type $F32X4 (fneg a))) (pulley_vnegf32x4 a))-(rule (lower (has_type $F64X2 (fneg a))) (pulley_vnegf64x2 a))+(rule (lower (has_type $F32 (fneg _ a))) (pulley_fneg32 a))+(rule (lower (has_type $F64 (fneg _ a))) (pulley_fneg64 a))+(rule (lower (has_type $F32X4 (fneg _ a))) (pulley_vnegf32x4 a))+(rule (lower (has_type $F64X2 (fneg _ a))) (pulley_vnegf64x2 a));;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (ineg a))) (pulley_xneg32 (sext32 a)))-(rule 1 (lower (has_type $I64 (ineg a))) (pulley_xneg64 a))+(rule 0 (lower (has_type (fits_in_32 _) (ineg _ a))) (pulley_xneg32 (sext32 a)))+(rule 1 (lower (has_type $I64 (ineg _ a))) (pulley_xneg64 a));; vector negation-(rule 1 (lower (has_type $I8X16 (ineg a))) (pulley_vneg8x16 a))-(rule 1 (lower (has_type $I16X8 (ineg a))) (pulley_vneg16x8 a))-(rule 1 (lower (has_type $I32X4 (ineg a))) (pulley_vneg32x4 a))-(rule 1 (lower (has_type $I64X2 (ineg a))) (pulley_vneg64x2 a))+(rule 1 (lower (has_type $I8X16 (ineg _ a))) (pulley_vneg8x16 a))+(rule 1 (lower (has_type $I16X8 (ineg _ a))) (pulley_vneg16x8 a))+(rule 1 (lower (has_type $I32X4 (ineg _ a))) (pulley_vneg32x4 a))+(rule 1 (lower (has_type $I64X2 (ineg _ a))) (pulley_vneg64x2 a));;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fabs a))) (pulley_fabs32 a))-(rule (lower (has_type $F64 (fabs a))) (pulley_fabs64 a))-(rule (lower (has_type $F32X4 (fabs a))) (pulley_vabsf32x4 a))-(rule (lower (has_type $F64X2 (fabs a))) (pulley_vabsf64x2 a))+(rule (lower (has_type $F32 (fabs _ a))) (pulley_fabs32 a))+(rule (lower (has_type $F64 (fabs _ a))) (pulley_fabs64 a))+(rule (lower (has_type $F32X4 (fabs _ a))) (pulley_vabsf32x4 a))+(rule (lower (has_type $F64X2 (fabs _ a))) (pulley_vabsf64x2 a));;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 _) (vconst (u128_from_constant a)))) (pulley_vconst128 a))+(rule (lower (has_type (ty_vec128 _) (vconst _ (u128_from_constant a)))) (pulley_vconst128 a));;;; Rules for `bswap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16 (bswap a)))+(rule (lower (has_type $I16 (bswap _ a)))(if-let (u6_from_u8 shift) (u64_try_into_u8 16))(pulley_xshr32_u_u6 (pulley_bswap32 a) shift))-(rule (lower (has_type $I32 (bswap a))) (pulley_bswap32 a))-(rule (lower (has_type $I64 (bswap a))) (pulley_bswap64 a))+(rule (lower (has_type $I32 (bswap _ a))) (pulley_bswap32 a))+(rule (lower (has_type $I64 (bswap _ a))) (pulley_bswap64 a));;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (iabs a))) (pulley_xabs32 (sext32 a)))-(rule 1 (lower (has_type $I64 (iabs a))) (pulley_xabs64 a))-(rule 1 (lower (has_type $I8X16 (iabs a))) (pulley_vabs8x16 a))-(rule 1 (lower (has_type $I16X8 (iabs a))) (pulley_vabs16x8 a))-(rule 1 (lower (has_type $I32X4 (iabs a))) (pulley_vabs32x4 a))-(rule 1 (lower (has_type $I64X2 (iabs a))) (pulley_vabs64x2 a))+(rule 0 (lower (has_type (fits_in_32 _) (iabs _ a))) (pulley_xabs32 (sext32 a)))+(rule 1 (lower (has_type $I64 (iabs _ a))) (pulley_xabs64 a))+(rule 1 (lower (has_type $I8X16 (iabs _ a))) (pulley_vabs8x16 a))+(rule 1 (lower (has_type $I16X8 (iabs _ a))) (pulley_vabs16x8 a))+(rule 1 (lower (has_type $I32X4 (iabs _ a))) (pulley_vabs32x4 a))+(rule 1 (lower (has_type $I64X2 (iabs _ a))) (pulley_vabs64x2 a));;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (splat a))) (pulley_vsplatx8 a))-(rule (lower (has_type $I16X8 (splat a))) (pulley_vsplatx16 a))-(rule (lower (has_type $I32X4 (splat a))) (pulley_vsplatx32 a))-(rule (lower (has_type $I64X2 (splat a))) (pulley_vsplatx64 a))-(rule (lower (has_type $F32X4 (splat a))) (pulley_vsplatf32 a))-(rule (lower (has_type $F64X2 (splat a))) (pulley_vsplatf64 a))+(rule (lower (has_type $I8X16 (splat _ a))) (pulley_vsplatx8 a))+(rule (lower (has_type $I16X8 (splat _ a))) (pulley_vsplatx16 a))+(rule (lower (has_type $I32X4 (splat _ a))) (pulley_vsplatx32 a))+(rule (lower (has_type $I64X2 (splat _ a))) (pulley_vsplatx64 a))+(rule (lower (has_type $F32X4 (splat _ a))) (pulley_vsplatf32 a))+(rule (lower (has_type $F64X2 (splat _ a))) (pulley_vsplatf64 a));;;; Rules for `vhigh_bits` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_32 _) (vhigh_bits a @ (value_type $I8X16))))+(rule (lower (has_type (fits_in_32 _) (vhigh_bits _ a @ (value_type $I8X16))))(pulley_vbitmask8x16 a))-(rule (lower (has_type (fits_in_32 _) (vhigh_bits a @ (value_type $I16X8))))+(rule (lower (has_type (fits_in_32 _) (vhigh_bits _ a @ (value_type $I16X8))))(pulley_vbitmask16x8 a))-(rule (lower (has_type (fits_in_32 _) (vhigh_bits a @ (value_type $I32X4))))+(rule (lower (has_type (fits_in_32 _) (vhigh_bits _ a @ (value_type $I32X4))))(pulley_vbitmask32x4 a))-(rule (lower (has_type (fits_in_32 _) (vhigh_bits a @ (value_type $I64X2))))+(rule (lower (has_type (fits_in_32 _) (vhigh_bits _ a @ (value_type $I64X2))))(pulley_vbitmask64x2 a));;;; Rules for `vall_true`; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (vall_true a @ (value_type $I8X16))) (pulley_valltrue8x16 a))-(rule (lower (vall_true a @ (value_type $I16X8))) (pulley_valltrue16x8 a))-(rule (lower (vall_true a @ (value_type $I32X4))) (pulley_valltrue32x4 a))-(rule (lower (vall_true a @ (value_type $I64X2))) (pulley_valltrue64x2 a))-(rule (lower (vall_true a @ (value_type $F32X4))) (pulley_valltrue32x4 a))-(rule (lower (vall_true a @ (value_type $F64X2))) (pulley_valltrue64x2 a))+(rule (lower (vall_true _ a @ (value_type $I8X16))) (pulley_valltrue8x16 a))+(rule (lower (vall_true _ a @ (value_type $I16X8))) (pulley_valltrue16x8 a))+(rule (lower (vall_true _ a @ (value_type $I32X4))) (pulley_valltrue32x4 a))+(rule (lower (vall_true _ a @ (value_type $I64X2))) (pulley_valltrue64x2 a))+(rule (lower (vall_true _ a @ (value_type $F32X4))) (pulley_valltrue32x4 a))+(rule (lower (vall_true _ a @ (value_type $F64X2))) (pulley_valltrue64x2 a));;;; Rules for `vany_true`; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (vany_true a @ (value_type $I8X16))) (pulley_vanytrue8x16 a))-(rule (lower (vany_true a @ (value_type $I16X8))) (pulley_vanytrue16x8 a))-(rule (lower (vany_true a @ (value_type $I32X4))) (pulley_vanytrue32x4 a))-(rule (lower (vany_true a @ (value_type $I64X2))) (pulley_vanytrue64x2 a))-(rule (lower (vany_true a @ (value_type $F32X4))) (pulley_vanytrue32x4 a))-(rule (lower (vany_true a @ (value_type $F64X2))) (pulley_vanytrue64x2 a))+(rule (lower (vany_true _ a @ (value_type $I8X16))) (pulley_vanytrue8x16 a))+(rule (lower (vany_true _ a @ (value_type $I16X8))) (pulley_vanytrue16x8 a))+(rule (lower (vany_true _ a @ (value_type $I32X4))) (pulley_vanytrue32x4 a))+(rule (lower (vany_true _ a @ (value_type $I64X2))) (pulley_vanytrue64x2 a))+(rule (lower (vany_true _ a @ (value_type $F32X4))) (pulley_vanytrue32x4 a))+(rule (lower (vany_true _ a @ (value_type $F64X2))) (pulley_vanytrue64x2 a));;;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (swiden_low a @ (value_type $I8X16))) (pulley_vwidenlow8x16_s a))-(rule (lower (swiden_low a @ (value_type $I16X8))) (pulley_vwidenlow16x8_s a))-(rule (lower (swiden_low a @ (value_type $I32X4))) (pulley_vwidenlow32x4_s a))+(rule (lower (swiden_low _ a @ (value_type $I8X16))) (pulley_vwidenlow8x16_s a))+(rule (lower (swiden_low _ a @ (value_type $I16X8))) (pulley_vwidenlow16x8_s a))+(rule (lower (swiden_low _ a @ (value_type $I32X4))) (pulley_vwidenlow32x4_s a));;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (swiden_high a @ (value_type $I8X16))) (pulley_vwidenhigh8x16_s a))-(rule (lower (swiden_high a @ (value_type $I16X8))) (pulley_vwidenhigh16x8_s a))-(rule (lower (swiden_high a @ (value_type $I32X4))) (pulley_vwidenhigh32x4_s a))+(rule (lower (swiden_high _ a @ (value_type $I8X16))) (pulley_vwidenhigh8x16_s a))+(rule (lower (swiden_high _ a @ (value_type $I16X8))) (pulley_vwidenhigh16x8_s a))+(rule (lower (swiden_high _ a @ (value_type $I32X4))) (pulley_vwidenhigh32x4_s a));;;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (uwiden_low a @ (value_type $I8X16))) (pulley_vwidenlow8x16_u a))-(rule (lower (uwiden_low a @ (value_type $I16X8))) (pulley_vwidenlow16x8_u a))-(rule (lower (uwiden_low a @ (value_type $I32X4))) (pulley_vwidenlow32x4_u a))+(rule (lower (uwiden_low _ a @ (value_type $I8X16))) (pulley_vwidenlow8x16_u a))+(rule (lower (uwiden_low _ a @ (value_type $I16X8))) (pulley_vwidenlow16x8_u a))+(rule (lower (uwiden_low _ a @ (value_type $I32X4))) (pulley_vwidenlow32x4_u a));;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (uwiden_high a @ (value_type $I8X16))) (pulley_vwidenhigh8x16_u a))-(rule (lower (uwiden_high a @ (value_type $I16X8))) (pulley_vwidenhigh16x8_u a))-(rule (lower (uwiden_high a @ (value_type $I32X4))) (pulley_vwidenhigh32x4_u a))+(rule (lower (uwiden_high _ a @ (value_type $I8X16))) (pulley_vwidenhigh8x16_u a))+(rule (lower (uwiden_high _ a @ (value_type $I16X8))) (pulley_vwidenhigh16x8_u a))+(rule (lower (uwiden_high _ a @ (value_type $I32X4))) (pulley_vwidenhigh32x4_u a));;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (snarrow a @ (value_type $I16X8) b)) (pulley_vnarrow16x8_s a b))-(rule (lower (snarrow a @ (value_type $I32X4) b)) (pulley_vnarrow32x4_s a b))-(rule (lower (snarrow a @ (value_type $I64X2) b)) (pulley_vnarrow64x2_s a b))+(rule (lower (snarrow _ a @ (value_type $I16X8) b)) (pulley_vnarrow16x8_s a b))+(rule (lower (snarrow _ a @ (value_type $I32X4) b)) (pulley_vnarrow32x4_s a b))+(rule (lower (snarrow _ a @ (value_type $I64X2) b)) (pulley_vnarrow64x2_s a b));;;; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (unarrow a @ (value_type $I16X8) b)) (pulley_vnarrow16x8_u a b))-(rule (lower (unarrow a @ (value_type $I32X4) b)) (pulley_vnarrow32x4_u a b))+(rule (lower (unarrow _ a @ (value_type $I16X8) b)) (pulley_vnarrow16x8_u a b))+(rule (lower (unarrow _ a @ (value_type $I32X4) b)) (pulley_vnarrow32x4_u a b));;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (uunarrow a @ (value_type $I64X2) b)) (pulley_vunarrow64x2_u a b))+(rule (lower (uunarrow _ a @ (value_type $I64X2) b)) (pulley_vunarrow64x2_u a b));;;; Rules for `fvpromote_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (fvpromote_low a @ (value_type $F32X4))) (pulley_vfpromotelow a))+(rule (lower (fvpromote_low _ a @ (value_type $F32X4))) (pulley_vfpromotelow a));;;; Rules for `fvdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (fvdemote a @ (value_type $F64X2))) (pulley_vfdemote a))+(rule (lower (fvdemote _ a @ (value_type $F64X2))) (pulley_vfdemote a));;;; Rules for `extractlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (extractlane a @ (value_type $I8X16) (u8_from_uimm8 lane)))+(rule (lower (extractlane _ a @ (value_type $I8X16) (u8_from_uimm8 lane)))(pulley_xextractv8x16 a lane))-(rule (lower (extractlane a @ (value_type $I16X8) (u8_from_uimm8 lane)))+(rule (lower (extractlane _ a @ (value_type $I16X8) (u8_from_uimm8 lane)))(pulley_xextractv16x8 a lane))-(rule (lower (extractlane a @ (value_type $I32X4) (u8_from_uimm8 lane)))+(rule (lower (extractlane _ a @ (value_type $I32X4) (u8_from_uimm8 lane)))(pulley_xextractv32x4 a lane))-(rule (lower (extractlane a @ (value_type $I64X2) (u8_from_uimm8 lane)))+(rule (lower (extractlane _ a @ (value_type $I64X2) (u8_from_uimm8 lane)))(pulley_xextractv64x2 a lane))-(rule (lower (extractlane a @ (value_type $F32X4) (u8_from_uimm8 lane)))+(rule (lower (extractlane _ a @ (value_type $F32X4) (u8_from_uimm8 lane)))(pulley_fextractv32x4 a lane))-(rule (lower (extractlane a @ (value_type $F64X2) (u8_from_uimm8 lane)))+(rule (lower (extractlane _ a @ (value_type $F64X2) (u8_from_uimm8 lane)))(pulley_fextractv64x2 a lane));;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (insertlane a @ (value_type $I8X16) b (u8_from_uimm8 lane)))+(rule (lower (insertlane _ a @ (value_type $I8X16) b (u8_from_uimm8 lane)))(pulley_vinsertx8 a b lane))-(rule (lower (insertlane a @ (value_type $I16X8) b (u8_from_uimm8 lane)))+(rule (lower (insertlane _ a @ (value_type $I16X8) b (u8_from_uimm8 lane)))(pulley_vinsertx16 a b lane))-(rule (lower (insertlane a @ (value_type $I32X4) b (u8_from_uimm8 lane)))+(rule (lower (insertlane _ a @ (value_type $I32X4) b (u8_from_uimm8 lane)))(pulley_vinsertx32 a b lane))-(rule (lower (insertlane a @ (value_type $I64X2) b (u8_from_uimm8 lane)))+(rule (lower (insertlane _ a @ (value_type $I64X2) b (u8_from_uimm8 lane)))(pulley_vinsertx64 a b lane))-(rule (lower (insertlane a @ (value_type $F32X4) b (u8_from_uimm8 lane)))+(rule (lower (insertlane _ a @ (value_type $F32X4) b (u8_from_uimm8 lane)))(pulley_vinsertf32 a b lane))-(rule (lower (insertlane a @ (value_type $F64X2) b (u8_from_uimm8 lane)))+(rule (lower (insertlane _ a @ (value_type $F64X2) b (u8_from_uimm8 lane)))(pulley_vinsertf64 a b lane));;;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1816,47 +1816,47 @@;; generates a bytecode-wise relatively inefficient lowering. Should be;; relatively easy to optimize if necessary in the future.-(rule (lower (scalar_to_vector a @ (value_type $I8)))+(rule (lower (scalar_to_vector _ a @ (value_type $I8)))(pulley_vinsertx8 (pulley_vconst128 0) a 0))-(rule (lower (scalar_to_vector a @ (value_type $I16)))+(rule (lower (scalar_to_vector _ a @ (value_type $I16)))(pulley_vinsertx16 (pulley_vconst128 0) a 0))-(rule (lower (scalar_to_vector a @ (value_type $I32)))+(rule (lower (scalar_to_vector _ a @ (value_type $I32)))(pulley_vinsertx32 (pulley_vconst128 0) a 0))-(rule (lower (scalar_to_vector a @ (value_type $I64)))+(rule (lower (scalar_to_vector _ a @ (value_type $I64)))(pulley_vinsertx64 (pulley_vconst128 0) a 0))-(rule (lower (scalar_to_vector a @ (value_type $F32)))+(rule (lower (scalar_to_vector _ a @ (value_type $F32)))(pulley_vinsertf32 (pulley_vconst128 0) a 0))-(rule (lower (scalar_to_vector a @ (value_type $F64)))+(rule (lower (scalar_to_vector _ a @ (value_type $F64)))(pulley_vinsertf64 (pulley_vconst128 0) a 0));;;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (shuffle a b (u128_from_immediate mask))))+(rule (lower (has_type $I8X16 (shuffle _ a b (u128_from_immediate mask))))(pulley_vshuffle a b mask));;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $I8X16 (swizzle a b))) (pulley_vswizzlei8x16 a b))+(rule 1 (lower (has_type $I8X16 (swizzle _ a b))) (pulley_vswizzlei8x16 a b));;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32X4 (fma a b c))) (pulley_vfma32x4 a b c))-(rule (lower (has_type $F64X2 (fma a b c))) (pulley_vfma64x2 a b c))+(rule (lower (has_type $F32X4 (fma _ a b c))) (pulley_vfma32x4 a b c))+(rule (lower (has_type $F64X2 (fma _ a b c))) (pulley_vfma64x2 a b c));; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (symbol_value (symbol_value_data extname dist offset)))+(rule (lower (symbol_value _ (symbol_value_data extname dist offset)))(load_ext_name extname offset dist));; Rules for `func_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (func_addr (func_ref_data _ extname dist _)))+(rule (lower (func_addr _ (func_ref_data _ extname dist _)))(load_ext_name extname 0 dist));; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))+(rule (lower (get_exception_handler_address _ (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(pulley_label_address succ_label)))diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isleindex 7633633b3a..80521217ed 100644--- a/cranelift/codegen/src/isa/riscv64/inst.isle+++ b/cranelift/codegen/src/isa/riscv64/inst.isle@@ -1983,7 +1983,7 @@;; value (first sign-extending to handle narrow widths).(decl pure partial imm12_from_negated_value (Value) Imm12)(rule- (imm12_from_negated_value (has_type ty (iconst n)))+ (imm12_from_negated_value (has_type ty (iconst _ n)))(if-let (imm12_from_u64 imm) (i64_cast_unsigned (i64_wrapping_neg (i64_sextend_imm64 ty n))))imm)@@ -2030,14 +2030,14 @@;; Like imm5_from_value, but first negates the `Value`.(decl pure partial imm5_from_negated_value (Value) Imm5)-(rule (imm5_from_negated_value (has_type ty (iconst n)))+(rule (imm5_from_negated_value (has_type ty (iconst _ n)))(if-let (imm5_from_i64 imm) (i64_wrapping_neg (i64_sextend_imm64 ty n)))imm);; Constructor that matches a `Value` equivalent to a replicated Imm5 on all lanes.(decl pure partial replicated_imm5 (Value) Imm5)-(rule (replicated_imm5 (splat (imm5_from_value n))) n)-(rule (replicated_imm5 (vconst (u128_from_constant n128)))+(rule (replicated_imm5 (splat _ (imm5_from_value n))) n)+(rule (replicated_imm5 (vconst _ (u128_from_constant n128)))(if-let (u128_replicated_u64 n64) n128)(if-let (u64_replicated_u32 n32) n64)(if-let (u32_replicated_u16 n16) n32)@@ -2047,10 +2047,10 @@;; Like replicated_imm5, but first negates the `Value`.(decl pure partial negated_replicated_imm5 (Value) Imm5)-(rule (negated_replicated_imm5 (splat n))+(rule (negated_replicated_imm5 (splat _ n))(if-let imm5 (imm5_from_negated_value n))imm5)-(rule (negated_replicated_imm5 (vconst (u128_from_constant n128)))+(rule (negated_replicated_imm5 (vconst _ (u128_from_constant n128)))(if-let (u128_replicated_u64 n64) n128)(if-let (u64_replicated_u32 n32) n64)(if-let (u32_replicated_u16 n16) n32)@@ -2062,8 +2062,8 @@;; Constructor that matches a `Value` equivalent to a replicated UImm5 on all lanes.(decl pure partial replicated_uimm5 (Value) UImm5)-(rule (replicated_uimm5 (splat (uimm5_from_value n))) n)-(rule 1 (replicated_uimm5 (vconst (u128_from_constant n128)))+(rule (replicated_uimm5 (splat _ (uimm5_from_value n))) n)+(rule 1 (replicated_uimm5 (vconst _ (u128_from_constant n128)))(if-let (u128_replicated_u64 n64) n128)(if-let (u64_replicated_u32 n32) n64)(if-let (u32_replicated_u16 n16) n32)@@ -2074,7 +2074,7 @@;; Helper to go directly from a `Value`, when it's an `iconst`, to an `UImm5`.(decl uimm5_from_value (UImm5) Value)(extractor (uimm5_from_value n)- (iconst (u64_from_imm64 (uimm5_from_u64 n))))+ (iconst _ (u64_from_imm64 (uimm5_from_u64 n))));; Extract a `UImm5` from an `u8`.(decl pure partial uimm5_from_u8 (UImm5) u8)@@ -2309,24 +2309,24 @@;; When extending our backend always extends to the full register width, so;; there's no need to extend-an-extend.-(rule 1 (val_already_extended (ExtendOp.Zero) (uextend _)) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (sextend _)) true)+(rule 1 (val_already_extended (ExtendOp.Zero) (uextend _ _)) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (sextend _ _)) true);; The result of `icmp`/`fcmp` is zero or one, meaning that it's already sign;; extended to the full register width.-(rule 1 (val_already_extended _ (icmp _ _ _)) true)-(rule 1 (val_already_extended _ (fcmp _ _ _)) true)+(rule 1 (val_already_extended _ (icmp _ _ _ _)) true)+(rule 1 (val_already_extended _ (fcmp _ _ _ _)) true);; The lowering for these operations always sign-extend their results due to the;; use of the `*w` instructions in RV64I. Note that this requires that the;; extension is from 32 to 64, 16/8-bit operations are explicitly excluded here.;; There are no native instructions for the 16/8 bit operations so they must;; fall through to actual sign extension above.-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (ishl _ _))) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (ushr _ _))) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (sshr _ _))) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (iadd _ _))) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (isub _ _))) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (ishl _ _ _))) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (ushr _ _ _))) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (sshr _ _ _))) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (iadd _ _ _))) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (isub _ _ _))) true)(type ExtendOp(enum@@ -2418,10 +2418,10 @@;;;; We can't recurse into `amode` again since that could cause stack overflows.;; See: https://github.com/bytecodealliance/wasmtime/pull/6968-(rule 1 (amode (iadd addr (i32_from_iconst y)) offset)+(rule 1 (amode (iadd _ addr (i32_from_iconst y)) offset)(if-let new_offset (i32_checked_add y offset))(amode_inner addr new_offset))-(rule 2 (amode (iadd (i32_from_iconst x) addr) offset)+(rule 2 (amode (iadd _ (i32_from_iconst x) addr) offset)(if-let new_offset (i32_checked_add x offset))(amode_inner addr new_offset))@@ -2434,15 +2434,15 @@(gen_reg_offset_amode r offset));; If the value is a `get_frame_pointer`, we can just use the offset from that.-(rule 1 (amode_inner (get_frame_pointer) offset)+(rule 1 (amode_inner (get_frame_pointer _) offset)(gen_fp_offset_amode offset));; If the value is a `get_stack_pointer`, we can just use the offset from that.-(rule 1 (amode_inner (get_stack_pointer) offset)+(rule 1 (amode_inner (get_stack_pointer _) offset)(gen_sp_offset_amode offset));; Similarly if the value is a `stack_addr` we can also turn that into an sp offset.-(rule 1 (amode_inner (stack_addr ss ss_offset) amode_offset)+(rule 1 (amode_inner (stack_addr _ ss ss_offset) amode_offset)(if-let combined_offset (i32_checked_add ss_offset amode_offset))(gen_stack_slot_amode ss combined_offset))@@ -2462,7 +2462,7 @@(decl sinkable_load (Inst Type MemFlags Value Offset32) Value)(extractor (sinkable_load inst ty flags addr offset)(and- (load (little_or_native_endian flags) addr offset)+ (load _ (little_or_native_endian flags) addr offset)(sinkable_inst (has_type ty inst))));; Returns a canonical type for a LoadOP. We only return I64 or F64.@@ -2933,9 +2933,9 @@;; If the input value is itself an `icmp` or `fcmp` we can avoid generating the;; result of the comparison and instead move the comparison directly into the;; `IntegerCompare` that's returned.-(rule 2 (is_nonzero_cmp (maybe_uextend (icmp cc a b @ (value_type (fits_in_64 _)))))+(rule 2 (is_nonzero_cmp (maybe_uextend (icmp _ cc a b @ (value_type (fits_in_64 _)))))(icmp_to_int_compare cc a b))-(rule 2 (is_nonzero_cmp (maybe_uextend (fcmp cc a @ (value_type ty) b)))+(rule 2 (is_nonzero_cmp (maybe_uextend (fcmp _ cc a @ (value_type ty) b)))(fcmp_to_float_compare cc ty a b));; Creates an `IntegerCompare` from an `icmp` node's parts. This will extend@@ -3163,8 +3163,8 @@;; be omitted because the result of the icmp or fcmp is a 0 or 1 directly. This;; means we can go straight to the `neg` instruction to produce the final;; result.-(rule 2 (gen_bmask val @ (maybe_uextend (icmp _ _ _))) (rv_neg val))-(rule 2 (gen_bmask val @ (maybe_uextend (fcmp _ _ _))) (rv_neg val))+(rule 2 (gen_bmask val @ (maybe_uextend (icmp _ _ _ _))) (rv_neg val))+(rule 2 (gen_bmask val @ (maybe_uextend (fcmp _ _ _ _))) (rv_neg val))(decl lower_bmask (Value Type) ValueRegs)(rule 0 (lower_bmask val (fits_in_64 _))diff --git a/cranelift/codegen/src/isa/riscv64/inst_vector.isle b/cranelift/codegen/src/isa/riscv64/inst_vector.isleindex 4070b8f33c..3c2d26950b 100644--- a/cranelift/codegen/src/isa/riscv64/inst_vector.isle+++ b/cranelift/codegen/src/isa/riscv64/inst_vector.isle@@ -1589,10 +1589,10 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.Equal) x y)(rv_vmseq_vv x y (unmasked) ty))-(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.Equal) x (splat y))+(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.Equal) x (splat _ y))(rv_vmseq_vx x y (unmasked) ty))-(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.Equal) (splat x) y)+(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.Equal) (splat _ x) y)(rv_vmseq_vx y x (unmasked) ty))(rule 3 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.Equal) x y)@@ -1608,10 +1608,10 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.NotEqual) x y)(rv_vmsne_vv x y (unmasked) ty))-(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.NotEqual) x (splat y))+(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.NotEqual) x (splat _ y))(rv_vmsne_vx x y (unmasked) ty))-(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.NotEqual) (splat x) y)+(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.NotEqual) (splat _ x) y)(rv_vmsne_vx y x (unmasked) ty))(rule 3 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.NotEqual) x y)@@ -1627,10 +1627,10 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThan) x y)(rv_vmsltu_vv x y (unmasked) ty))-(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThan) x (splat y))+(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThan) x (splat _ y))(rv_vmsltu_vx x y (unmasked) ty))-(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThan) (splat x) y)+(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThan) (splat _ x) y)(rv_vmsgtu_vx y x (unmasked) ty))(rule 4 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThan) x y)@@ -1642,10 +1642,10 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThan) x y)(rv_vmslt_vv x y (unmasked) ty))-(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThan) x (splat y))+(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThan) x (splat _ y))(rv_vmslt_vx x y (unmasked) ty))-(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThan) (splat x) y)+(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThan) (splat _ x) y)(rv_vmsgt_vx y x (unmasked) ty))(rule 4 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThan) x y)@@ -1657,7 +1657,7 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThanOrEqual) x y)(rv_vmsleu_vv x y (unmasked) ty))-(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThanOrEqual) x (splat y))+(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThanOrEqual) x (splat _ y))(rv_vmsleu_vx x y (unmasked) ty))(rule 3 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedLessThanOrEqual) x y)@@ -1669,7 +1669,7 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThanOrEqual) x y)(rv_vmsle_vv x y (unmasked) ty))-(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThanOrEqual) x (splat y))+(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThanOrEqual) x (splat _ y))(rv_vmsle_vx x y (unmasked) ty))(rule 3 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedLessThanOrEqual) x y)@@ -1681,10 +1681,10 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThan) x y)(rv_vmsgtu_vv x y (unmasked) ty))-(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThan) x (splat y))+(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThan) x (splat _ y))(rv_vmsgtu_vx x y (unmasked) ty))-(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThan) (splat x) y)+(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThan) (splat _ x) y)(rv_vmsltu_vx y x (unmasked) ty))(rule 3 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThan) x y)@@ -1696,10 +1696,10 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThan) x y)(rv_vmsgt_vv x y (unmasked) ty))-(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThan) x (splat y))+(rule 1 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThan) x (splat _ y))(rv_vmsgt_vx x y (unmasked) ty))-(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThan) (splat x) y)+(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThan) (splat _ x) y)(rv_vmslt_vx y x (unmasked) ty))(rule 3 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThan) x y)@@ -1711,7 +1711,7 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThanOrEqual) x y)(rv_vmsgeu_vv x y (unmasked) ty))-(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThanOrEqual) (splat x) y)+(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThanOrEqual) (splat _ x) y)(rv_vmsleu_vx y x (unmasked) ty))(rule 4 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.UnsignedGreaterThanOrEqual) x y)@@ -1723,7 +1723,7 @@(rule 0 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThanOrEqual) x y)(rv_vmsge_vv x y (unmasked) ty))-(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThanOrEqual) (splat x) y)+(rule 2 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThanOrEqual) (splat _ x) y)(rv_vmsle_vx y x (unmasked) ty))(rule 4 (gen_icmp_mask (ty_vec_fits_in_register ty) (IntCC.SignedGreaterThanOrEqual) x y)@@ -1743,10 +1743,10 @@(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Equal) x y)(rv_vmfeq_vv x y (unmasked) ty))-(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Equal) x (splat y))+(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Equal) x (splat _ y))(rv_vmfeq_vf x y (unmasked) ty))-(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Equal) (splat x) y)+(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.Equal) (splat _ x) y)(rv_vmfeq_vf y x (unmasked) ty));; FloatCC.NotEqual@@ -1755,10 +1755,10 @@(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.NotEqual) x y)(rv_vmfne_vv x y (unmasked) ty))-(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.NotEqual) x (splat y))+(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.NotEqual) x (splat _ y))(rv_vmfne_vf x y (unmasked) ty))-(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.NotEqual) (splat x) y)+(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.NotEqual) (splat _ x) y)(rv_vmfne_vf y x (unmasked) ty));; FloatCC.LessThan@@ -1766,10 +1766,10 @@(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThan) x y)(rv_vmflt_vv x y (unmasked) ty))-(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThan) x (splat y))+(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThan) x (splat _ y))(rv_vmflt_vf x y (unmasked) ty))-(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThan) (splat x) y)+(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThan) (splat _ x) y)(rv_vmfgt_vf y x (unmasked) ty));; FloatCC.LessThanOrEqual@@ -1777,10 +1777,10 @@(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThanOrEqual) x y)(rv_vmfle_vv x y (unmasked) ty))-(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThanOrEqual) x (splat y))+(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThanOrEqual) x (splat _ y))(rv_vmfle_vf x y (unmasked) ty))-(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThanOrEqual) (splat x) y)+(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.LessThanOrEqual) (splat _ x) y)(rv_vmfge_vf y x (unmasked) ty));; FloatCC.GreaterThan@@ -1788,10 +1788,10 @@(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThan) x y)(rv_vmfgt_vv x y (unmasked) ty))-(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThan) x (splat y))+(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThan) x (splat _ y))(rv_vmfgt_vf x y (unmasked) ty))-(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThan) (splat x) y)+(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThan) (splat _ x) y)(rv_vmflt_vf y x (unmasked) ty));; FloatCC.GreaterThanOrEqual@@ -1799,10 +1799,10 @@(rule 0 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThanOrEqual) x y)(rv_vmfge_vv x y (unmasked) ty))-(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThanOrEqual) x (splat y))+(rule 1 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThanOrEqual) x (splat _ y))(rv_vmfge_vf x y (unmasked) ty))-(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThanOrEqual) (splat x) y)+(rule 2 (gen_fcmp_mask (ty_vec_fits_in_register ty) (FloatCC.GreaterThanOrEqual) (splat _ x) y)(rv_vmfle_vf y x (unmasked) ty));; FloatCC.Ordereddiff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isleindex 49b56e3d09..efbae20af1 100644--- a/cranelift/codegen/src/isa/riscv64/lower.isle+++ b/cranelift/codegen/src/isa/riscv64/lower.isle@@ -6,61 +6,61 @@;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (iconst (u64_from_imm64 n))))+(rule (lower (has_type ty (iconst _ (u64_from_imm64 n))))(imm ty n));; ;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty) (vconst n)))+(rule (lower (has_type (ty_supported_vec ty) (vconst _ n)))(gen_constant ty (const_to_vconst n)));;;; Rules for `f16const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f16const (u16_from_ieee16 n)))+(rule (lower (f16const _ (u16_from_ieee16 n)))(imm $F16 n));;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f32const (u32_from_ieee32 n)))+(rule (lower (f32const _ (u32_from_ieee32 n)))(imm $F32 n));;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f64const (u64_from_ieee64 n)))+(rule (lower (f64const _ (u64_from_ieee64 n)))(imm $F64 n));;;; Rules for `f128const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f128const (u128_from_constant n)))+(rule (lower (f128const _ (u128_from_constant n)))(value_regs (imm $I64 (u128_low_bits n)) (imm $I64 (u128_high_bits n))))-(rule 1 (lower (f128const (u128_from_constant (u128_replicated_u64 n))))+(rule 1 (lower (f128const _ (u128_from_constant (u128_replicated_u64 n))))(let ((r Reg (imm $I64 n)))(value_regs r r)));;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Base case, simply adding things in registers.-(rule -1 (lower (has_type (fits_in_32 (ty_int ty)) (iadd x y)))+(rule -1 (lower (has_type (fits_in_32 (ty_int ty)) (iadd _ x y)))(rv_addw x y))-(rule 0 (lower (has_type $I64 (iadd x y)))+(rule 0 (lower (has_type $I64 (iadd _ x y)))(rv_add x y));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule 1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd x (imm12_from_value y))))+(rule 1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (imm12_from_value y))))(alu_rr_imm12 (select_addi ty) x y))-(rule 2 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd (imm12_from_value x) y)))+(rule 2 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (imm12_from_value x) y)))(alu_rr_imm12 (select_addi ty) y x));; Special case when one of the operands is uextended;; Needs `Zba`-(rule 3 (lower (has_type $I64 (iadd x (uextend y @ (value_type $I32)))))+(rule 3 (lower (has_type $I64 (iadd _ x (uextend _ y @ (value_type $I32)))))(if-let true (has_zba))(rv_adduw y x))-(rule 4 (lower (has_type $I64 (iadd (uextend x @ (value_type $I32)) y)))+(rule 4 (lower (has_type $I64 (iadd _ (uextend _ x @ (value_type $I32)) y)))(if-let true (has_zba))(rv_adduw x y))@@ -70,12 +70,12 @@(rule (match_shnadd (u64_from_imm64 2)) (AluOPRRR.Sh2add))(rule (match_shnadd (u64_from_imm64 3)) (AluOPRRR.Sh3add))-(rule 3 (lower (has_type $I64 (iadd x (ishl y (maybe_uextend (iconst n))))))+(rule 3 (lower (has_type $I64 (iadd _ x (ishl _ y (maybe_uextend (iconst _ n))))))(if-let true (has_zba))(if-let shnadd (match_shnadd n))(alu_rrr shnadd y x))-(rule 4 (lower (has_type $I64 (iadd (ishl x (maybe_uextend (iconst n))) y)))+(rule 4 (lower (has_type $I64 (iadd _ (ishl _ x (maybe_uextend (iconst _ n))) y)))(if-let true (has_zba))(if-let shnadd (match_shnadd n))(alu_rrr shnadd x y))@@ -92,18 +92,18 @@(rule (match_shnadd_uw (u64_from_imm64 2)) (AluOPRRR.Sh2adduw))(rule (match_shnadd_uw (u64_from_imm64 3)) (AluOPRRR.Sh3adduw))-(rule 5 (lower (has_type $I64 (iadd x (ishl (uextend y @ (value_type $I32)) (maybe_uextend (iconst n))))))+(rule 5 (lower (has_type $I64 (iadd _ x (ishl _ (uextend _ y @ (value_type $I32)) (maybe_uextend (iconst _ n))))))(if-let true (has_zba))(if-let shnadd_uw (match_shnadd_uw n))(alu_rrr shnadd_uw y x))-(rule 6 (lower (has_type $I64 (iadd (ishl (uextend x @ (value_type $I32)) (maybe_uextend (iconst n))) y)))+(rule 6 (lower (has_type $I64 (iadd _ (ishl _ (uextend _ x @ (value_type $I32)) (maybe_uextend (iconst _ n))) y)))(if-let true (has_zba))(if-let shnadd_uw (match_shnadd_uw n))(alu_rrr shnadd_uw x y));; I128 cases-(rule 7 (lower (has_type $I128 (iadd x y)))+(rule 7 (lower (has_type $I128 (iadd _ x y)))(let ((low XReg (rv_add (value_regs_get x 0) (value_regs_get y 0)));; compute carry.(carry XReg (rv_sltu low (value_regs_get y 0)))@@ -114,152 +114,152 @@(value_regs low high)));; SIMD Vectors-(rule 8 (lower (has_type (ty_supported_vec ty) (iadd x y)))+(rule 8 (lower (has_type (ty_supported_vec ty) (iadd _ x y)))(rv_vadd_vv x y (unmasked) ty))-(rule 9 (lower (has_type (ty_supported_vec ty) (iadd x (splat y))))+(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ y))))(rv_vadd_vx x y (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (iadd x (splat (sextend y @ (value_type sext_ty))))))+(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ (sextend _ y @ (value_type sext_ty))))))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) sext_ty))(rv_vwadd_wx x y (unmasked) (vstate_mf2 half_ty)))-(rule 10 (lower (has_type (ty_supported_vec ty) (iadd x (splat (uextend y @ (value_type uext_ty))))))+(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ (uextend _ y @ (value_type uext_ty))))))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) uext_ty))(rv_vwaddu_wx x y (unmasked) (vstate_mf2 half_ty)))-(rule 20 (lower (has_type (ty_supported_vec ty) (iadd x y)))+(rule 20 (lower (has_type (ty_supported_vec ty) (iadd _ x y)))(if-let y_imm (replicated_imm5 y))(rv_vadd_vi x y_imm (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (iadd (splat x) y)))+(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ x) y)))(rv_vadd_vx y x (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (iadd (splat (sextend x @ (value_type sext_ty))) y)))+(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty))) y)))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) sext_ty))(rv_vwadd_wx y x (unmasked) (vstate_mf2 half_ty)))-(rule 13 (lower (has_type (ty_supported_vec ty) (iadd (splat (uextend x @ (value_type uext_ty))) y)))+(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ (uextend _ x @ (value_type uext_ty))) y)))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) uext_ty))(rv_vwaddu_wx y x (unmasked) (vstate_mf2 half_ty)))-(rule 21 (lower (has_type (ty_supported_vec ty) (iadd x y)))+(rule 21 (lower (has_type (ty_supported_vec ty) (iadd _ x y)))(if-let x_imm (replicated_imm5 x))(rv_vadd_vi y x_imm (unmasked) ty));; Signed Widening Low Additions-(rule 9 (lower (has_type (ty_supported_vec _) (iadd x (swiden_low y @ (value_type in_ty)))))+(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (swiden_low _ y @ (value_type in_ty)))))(rv_vwadd_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 12 (lower (has_type (ty_supported_vec _) (iadd (swiden_low x @ (value_type in_ty)) y)))+(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty)) y)))(rv_vwadd_wv y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (swiden_low x @ (value_type in_ty))- (swiden_low y))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty))+ (swiden_low _ y))))(rv_vwadd_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (swiden_low x @ (value_type in_ty))- (splat (sextend y @ (value_type sext_ty))))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty))+ (splat _ (sextend _ y @ (value_type sext_ty))))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwadd_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 15 (lower (has_type (ty_supported_vec _) (iadd (splat (sextend x @ (value_type sext_ty)))- (swiden_low y @ (value_type in_ty)))))+(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty)))+ (swiden_low _ y @ (value_type in_ty)))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwadd_vx y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Signed Widening High Additions;; These are the same as the low additions, but we first slide down the inputs.-(rule 9 (lower (has_type (ty_supported_vec _) (iadd x (swiden_high y @ (value_type in_ty)))))+(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (swiden_high _ y @ (value_type in_ty)))))(rv_vwadd_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 12 (lower (has_type (ty_supported_vec _) (iadd (swiden_high x @ (value_type in_ty)) y)))+(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty)) y)))(rv_vwadd_wv y (gen_slidedown_half in_ty x) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (swiden_high x @ (value_type in_ty))- (swiden_high y))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty))+ (swiden_high _ y))))(rv_vwadd_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (swiden_high x @ (value_type in_ty))- (splat (sextend y @ (value_type sext_ty))))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty))+ (splat _ (sextend _ y @ (value_type sext_ty))))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwadd_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 15 (lower (has_type (ty_supported_vec _) (iadd (splat (sextend x @ (value_type sext_ty)))- (swiden_high y @ (value_type in_ty)))))+(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty)))+ (swiden_high _ y @ (value_type in_ty)))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwadd_vx (gen_slidedown_half in_ty y) x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening Low Additions-(rule 9 (lower (has_type (ty_supported_vec _) (iadd x (uwiden_low y @ (value_type in_ty)))))+(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (uwiden_low _ y @ (value_type in_ty)))))(rv_vwaddu_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 12 (lower (has_type (ty_supported_vec _) (iadd (uwiden_low x @ (value_type in_ty)) y)))+(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty)) y)))(rv_vwaddu_wv y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (uwiden_low x @ (value_type in_ty))- (uwiden_low y))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty))+ (uwiden_low _ y))))(rv_vwaddu_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (uwiden_low x @ (value_type in_ty))- (splat (uextend y @ (value_type uext_ty))))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty))+ (splat _ (uextend _ y @ (value_type uext_ty))))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwaddu_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 15 (lower (has_type (ty_supported_vec _) (iadd (splat (uextend x @ (value_type uext_ty)))- (uwiden_low y @ (value_type in_ty)))))+(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (uextend _ x @ (value_type uext_ty)))+ (uwiden_low _ y @ (value_type in_ty)))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwaddu_vx y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening High Additions;; These are the same as the low additions, but we first slide down the inputs.-(rule 9 (lower (has_type (ty_supported_vec _) (iadd x (uwiden_high y @ (value_type in_ty)))))+(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (uwiden_high _ y @ (value_type in_ty)))))(rv_vwaddu_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 12 (lower (has_type (ty_supported_vec _) (iadd (uwiden_high x @ (value_type in_ty)) y)))+(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty)) y)))(rv_vwaddu_wv y (gen_slidedown_half in_ty x) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (uwiden_high x @ (value_type in_ty))- (uwiden_high y))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty))+ (uwiden_high _ y))))(rv_vwaddu_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (uwiden_high x @ (value_type in_ty))- (splat (uextend y @ (value_type uext_ty))))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty))+ (splat _ (uextend _ y @ (value_type uext_ty))))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwaddu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 15 (lower (has_type (ty_supported_vec _) (iadd (splat (uextend y @ (value_type uext_ty)))- (uwiden_high x @ (value_type in_ty)))))+(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (uextend _ y @ (value_type uext_ty)))+ (uwiden_high _ x @ (value_type in_ty)))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwaddu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Signed Widening Mixed High/Low Additions-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (swiden_low x @ (value_type in_ty))- (swiden_high y))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty))+ (swiden_high _ y))))(rv_vwadd_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (swiden_high x @ (value_type in_ty))- (swiden_low y))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty))+ (swiden_low _ y))))(rv_vwadd_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening Mixed High/Low Additions-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (uwiden_low x @ (value_type in_ty))- (uwiden_high y))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty))+ (uwiden_high _ y))))(rv_vwaddu_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd (uwiden_high x @ (value_type in_ty))- (uwiden_low y))))+(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty))+ (uwiden_low _ y))))(rv_vwaddu_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Fused Multiply Accumulate Rules `vmacc`@@ -268,46 +268,46 @@;; register instead of the addition one. The actual pattern matched seems to be;; exactly the same.-(rule 9 (lower (has_type (ty_supported_vec ty) (iadd x (imul y z))))+(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ y z))))(rv_vmacc_vv x y z (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (iadd x (imul y (splat z)))))+(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ y (splat _ z)))))(rv_vmacc_vx x y z (unmasked) ty))-(rule 11 (lower (has_type (ty_supported_vec ty) (iadd x (imul (splat y) z))))+(rule 11 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ (splat _ y) z))))(rv_vmacc_vx x z y (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (iadd (imul x y) z)))+(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ x y) z)))(rv_vmacc_vv z x y (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (iadd (imul x (splat y)) z)))+(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ x (splat _ y)) z)))(rv_vmacc_vx z x y (unmasked) ty))-(rule 14 (lower (has_type (ty_supported_vec ty) (iadd (imul (splat x) y) z)))+(rule 14 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ (splat _ x) y) z)))(rv_vmacc_vx z y x (unmasked) ty));; Fused Multiply Subtract Rules `vnmsac`-(rule 9 (lower (has_type (ty_supported_vec ty) (iadd x (ineg (imul y z)))))+(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ y z)))))(rv_vnmsac_vv x y z (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (iadd x (ineg (imul y (splat z))))))+(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ y (splat _ z))))))(rv_vnmsac_vx x y z (unmasked) ty))-(rule 11 (lower (has_type (ty_supported_vec ty) (iadd x (ineg (imul (splat y) z)))))+(rule 11 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ (splat _ y) z)))))(rv_vnmsac_vx x z y (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (iadd (ineg (imul x y)) z)))+(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ x y)) z)))(rv_vnmsac_vv z x y (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (iadd (ineg (imul x (splat y))) z)))+(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ x (splat _ y))) z)))(rv_vnmsac_vx z x y (unmasked) ty))-(rule 14 (lower (has_type (ty_supported_vec ty) (iadd (ineg (imul (splat x) y)) z)))+(rule 14 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ (splat _ x) y)) z)))(rv_vnmsac_vx z y x (unmasked) ty));;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 ty) (uadd_overflow_trap x y tc)))+(rule 0 (lower (has_type (fits_in_32 ty) (uadd_overflow_trap _ x y tc)))(let ((tmp_x XReg (zext x))(tmp_y XReg (zext y))(sum XReg (rv_add tmp_x tmp_y))@@ -315,7 +315,7 @@(_ InstOutput (gen_trapnz test tc)))sum))-(rule 1 (lower (has_type $I64 (uadd_overflow_trap x y tc)))+(rule 1 (lower (has_type $I64 (uadd_overflow_trap _ x y tc)))(let ((tmp XReg (rv_add x y))(_ InstOutput (gen_trapif (IntCC.UnsignedLessThan) tmp x tc)))tmp))@@ -323,20 +323,20 @@;;;; Rules for uadd_overflow ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; For i64, we can use the fact that if a + b < a, then overflow occurred-(rule 0 (lower (has_type $I64 (uadd_overflow x y)))+(rule 0 (lower (has_type $I64 (uadd_overflow _ x y)))(let ((sum XReg (rv_add x y))(overflow XReg (rv_sltu sum x)))(output_pair sum overflow)));; i32 case (on RV64 use addw to detect 32-bit overflow correctly)-(rule 1 (lower (has_type $I32 (uadd_overflow x y)))+(rule 1 (lower (has_type $I32 (uadd_overflow _ x y)))(let ((x64 XReg (zext x))(sum XReg (rv_addw x y))(overflow XReg (rv_sltu sum x64)))(output_pair sum overflow)));; For i128, we need to handle the high and low parts separately-(rule 2 (lower (has_type $I128 (uadd_overflow x y)))+(rule 2 (lower (has_type $I128 (uadd_overflow _ x y)))(let ((x_regs ValueRegs x)(y_regs ValueRegs y)(x_lo XReg (value_regs_get x_regs 0))@@ -354,147 +354,147 @@;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Base case, simply subtracting things in registers.-(rule 0 (lower (has_type (fits_in_32 (ty_int ty)) (isub x y)))+(rule 0 (lower (has_type (fits_in_32 (ty_int ty)) (isub _ x y)))(rv_subw x y))-(rule 1 (lower (has_type $I64 (isub x y)))+(rule 1 (lower (has_type $I64 (isub _ x y)))(rv_sub x y))-(rule 2 (lower (has_type $I128 (isub x y)))+(rule 2 (lower (has_type $I128 (isub _ x y)))(sub_i128 x y));; Switch to an `addi` by a negative if we can fit the value in an `imm12`.-(rule 3 (lower (has_type (ty_int_ref_scalar_64 ty) (isub x y)))+(rule 3 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x y)))(if-let imm12_neg (imm12_from_negated_value y))(alu_rr_imm12 (select_addi ty) x imm12_neg));; SIMD Vectors-(rule 4 (lower (has_type (ty_supported_vec ty) (isub x y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (isub _ x y)))(rv_vsub_vv x y (unmasked) ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (isub x (splat y))))+(rule 5 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ y))))(rv_vsub_vx x y (unmasked) ty))-(rule 6 (lower (has_type (ty_supported_vec ty) (isub x (splat (sextend y @ (value_type sext_ty))))))+(rule 6 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ (sextend _ y @ (value_type sext_ty))))))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) sext_ty))(rv_vwsub_wx x y (unmasked) (vstate_mf2 half_ty)))-(rule 6 (lower (has_type (ty_supported_vec ty) (isub x (splat (uextend y @ (value_type uext_ty))))))+(rule 6 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ (uextend _ y @ (value_type uext_ty))))))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) uext_ty))(rv_vwsubu_wx x y (unmasked) (vstate_mf2 half_ty)))-(rule 7 (lower (has_type (ty_supported_vec ty) (isub (splat x) y)))+(rule 7 (lower (has_type (ty_supported_vec ty) (isub _ (splat _ x) y)))(rv_vrsub_vx y x (unmasked) ty))-(rule 8 (lower (has_type (ty_supported_vec ty) (isub x y)))+(rule 8 (lower (has_type (ty_supported_vec ty) (isub _ x y)))(if-let imm5_neg (negated_replicated_imm5 y))(rv_vadd_vi x imm5_neg (unmasked) ty))-(rule 9 (lower (has_type (ty_supported_vec ty) (isub x y)))+(rule 9 (lower (has_type (ty_supported_vec ty) (isub _ x y)))(if-let x_imm (replicated_imm5 x))(rv_vrsub_vi y x_imm (unmasked) ty));; Signed Widening Low Subtractions-(rule 6 (lower (has_type (ty_supported_vec _) (isub x (swiden_low y @ (value_type in_ty)))))+(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (swiden_low _ y @ (value_type in_ty)))))(rv_vwsub_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (swiden_low x @ (value_type in_ty))- (swiden_low y))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty))+ (swiden_low _ y))))(rv_vwsub_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (swiden_low x @ (value_type in_ty))- (splat (sextend y @ (value_type sext_ty))))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty))+ (splat _ (sextend _ y @ (value_type sext_ty))))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwsub_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Signed Widening High Subtractions;; These are the same as the low widenings, but we first slide down the inputs.-(rule 6 (lower (has_type (ty_supported_vec _) (isub x (swiden_high y @ (value_type in_ty)))))+(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (swiden_high _ y @ (value_type in_ty)))))(rv_vwsub_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (swiden_high x @ (value_type in_ty))- (swiden_high y))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty))+ (swiden_high _ y))))(rv_vwsub_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (swiden_high x @ (value_type in_ty))- (splat (sextend y @ (value_type sext_ty))))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty))+ (splat _ (sextend _ y @ (value_type sext_ty))))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwsub_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening Low Subtractions-(rule 6 (lower (has_type (ty_supported_vec _) (isub x (uwiden_low y @ (value_type in_ty)))))+(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (uwiden_low _ y @ (value_type in_ty)))))(rv_vwsubu_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (uwiden_low x @ (value_type in_ty))- (uwiden_low y))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty))+ (uwiden_low _ y))))(rv_vwsubu_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (uwiden_low x @ (value_type in_ty))- (splat (uextend y @ (value_type uext_ty))))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty))+ (splat _ (uextend _ y @ (value_type uext_ty))))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwsubu_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening High Subtractions;; These are the same as the low widenings, but we first slide down the inputs.-(rule 6 (lower (has_type (ty_supported_vec _) (isub x (uwiden_high y @ (value_type in_ty)))))+(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (uwiden_high _ y @ (value_type in_ty)))))(rv_vwsubu_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (uwiden_high x @ (value_type in_ty))- (uwiden_high y))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty))+ (uwiden_high _ y))))(rv_vwsubu_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (uwiden_high x @ (value_type in_ty))- (splat (uextend y @ (value_type uext_ty))))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty))+ (splat _ (uextend _ y @ (value_type uext_ty))))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwsubu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Signed Widening Mixed High/Low Subtractions-(rule 10 (lower (has_type (ty_supported_vec _) (isub (swiden_low x @ (value_type in_ty))- (swiden_high y))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty))+ (swiden_high _ y))))(rv_vwsub_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (swiden_high x @ (value_type in_ty))- (swiden_low y))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty))+ (swiden_low _ y))))(rv_vwsub_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening Mixed High/Low Subtractions-(rule 10 (lower (has_type (ty_supported_vec _) (isub (uwiden_low x @ (value_type in_ty))- (uwiden_high y))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty))+ (uwiden_high _ y))))(rv_vwsubu_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub (uwiden_high x @ (value_type in_ty))- (uwiden_low y))))+(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty))+ (uwiden_low _ y))))(rv_vwsubu_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_int ty) (ineg val)))+(rule (lower (has_type (ty_int ty) (ineg _ val)))(neg ty val))-(rule 1 (lower (has_type (ty_supported_vec ty) (ineg x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (ineg _ x)))(rv_vneg_v x (unmasked) ty));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (imul x y)))+(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (imul _ x y)))(rv_mul x y))-(rule 1 (lower (has_type (fits_in_32 (ty_int ty)) (imul x y)))+(rule 1 (lower (has_type (fits_in_32 (ty_int ty)) (imul _ x y)))(rv_mulw x y));; for I128-(rule 2 (lower (has_type $I128 (imul x y)))+(rule 2 (lower (has_type $I128 (imul _ x y)))(let((x_regs ValueRegs x)(x_lo XReg (value_regs_get x_regs 0))@@ -522,82 +522,82 @@;; Special case 128-bit multiplication where the operands are extended since;; that maps directly to the `mulhu` and `mulh` instructions.-(rule 6 (lower (has_type $I128 (imul (uextend x) (uextend y))))+(rule 6 (lower (has_type $I128 (imul _ (uextend _ x) (uextend _ y))))(let ((x XReg (zext x))(y XReg (zext y)))(value_regs (rv_mul x y) (rv_mulhu x y))))-(rule 6 (lower (has_type $I128 (imul (sextend x) (sextend y))))+(rule 6 (lower (has_type $I128 (imul _ (sextend _ x) (sextend _ y))))(let ((x XReg (sext x))(y XReg (sext y)))(value_regs (rv_mul x y) (rv_mulh x y))));; Vector multiplication-(rule 3 (lower (has_type (ty_supported_vec ty) (imul x y)))+(rule 3 (lower (has_type (ty_supported_vec ty) (imul _ x y)))(rv_vmul_vv x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (imul (splat x) y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (imul _ (splat _ x) y)))(rv_vmul_vx y x (unmasked) ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (imul x (splat y))))+(rule 5 (lower (has_type (ty_supported_vec ty) (imul _ x (splat _ y))))(rv_vmul_vx x y (unmasked) ty));;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (smulhi x y)))+(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (smulhi _ x y)))(lower_smlhi ty (sext x) (sext y)))-(rule 1 (lower (has_type (ty_supported_vec ty) (smulhi x y)))+(rule 1 (lower (has_type (ty_supported_vec ty) (smulhi _ x y)))(rv_vmulh_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (smulhi (splat x) y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (smulhi _ (splat _ x) y)))(rv_vmulh_vx y x (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (smulhi x (splat y))))+(rule 3 (lower (has_type (ty_supported_vec ty) (smulhi _ x (splat _ y))))(rv_vmulh_vx x y (unmasked) ty));;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 ty) (umulhi x y)))+(rule 0 (lower (has_type (fits_in_32 ty) (umulhi _ x y)))(let ((tmp XReg (rv_mul (zext x) (zext y))))(rv_srli tmp (imm12_const (ty_bits ty)))))-(rule 1 (lower (has_type $I64 (umulhi x y)))+(rule 1 (lower (has_type $I64 (umulhi _ x y)))(rv_mulhu x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (umulhi x y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (umulhi _ x y)))(rv_vmulhu_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (umulhi (splat x) y)))+(rule 3 (lower (has_type (ty_supported_vec ty) (umulhi _ (splat _ x) y)))(rv_vmulhu_vx y x (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (umulhi x (splat y))))+(rule 4 (lower (has_type (ty_supported_vec ty) (umulhi _ x (splat _ y))))(rv_vmulhu_vx x y (unmasked) ty));;;; Rules for `udiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_16 ty) (udiv x y)))+(rule 0 (lower (has_type (fits_in_16 ty) (udiv _ x y)))(if-let true (has_m))(rv_divuw (zext x) (nonzero_divisor (zext y))))-(rule 1 (lower (has_type (fits_in_16 ty) (udiv x y @ (iconst imm))))+(rule 1 (lower (has_type (fits_in_16 ty) (udiv _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 ty imm))(rv_divuw (zext x) (zext y)))-(rule 2 (lower (has_type $I32 (udiv x y)))+(rule 2 (lower (has_type $I32 (udiv _ x y)))(if-let true (has_m))(rv_divuw x (nonzero_divisor (zext y))))-(rule 3 (lower (has_type $I32 (udiv x y @ (iconst imm))))+(rule 3 (lower (has_type $I32 (udiv _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 $I32 imm))(rv_divuw x y))-(rule 2 (lower (has_type $I64 (udiv x y)))+(rule 2 (lower (has_type $I64 (udiv _ x y)))(if-let true (has_m))(rv_divu x (nonzero_divisor y)))-(rule 3 (lower (has_type $I64 (udiv x y @ (iconst imm))))+(rule 3 (lower (has_type $I64 (udiv _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 $I64 imm))(rv_divu x y))@@ -610,31 +610,31 @@;;;; Rules for `sdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_16 ty) (sdiv x y)))+(rule 0 (lower (has_type (fits_in_16 ty) (sdiv _ x y)))(if-let true (has_m))(let ((x XReg (sext x)))(rv_divw x (safe_sdiv_divisor ty x (sext y)))))-(rule 1 (lower (has_type (fits_in_16 ty) (sdiv x y @ (iconst imm))))+(rule 1 (lower (has_type (fits_in_16 ty) (sdiv _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 ty imm))(rv_divw (sext x) (sext y)))-(rule 2 (lower (has_type $I32 (sdiv x y)))+(rule 2 (lower (has_type $I32 (sdiv _ x y)))(if-let true (has_m))(let ((x XReg (sext x)))(rv_divw x (safe_sdiv_divisor $I32 x (sext y)))))-(rule 3 (lower (has_type $I32 (sdiv x y @ (iconst imm))))+(rule 3 (lower (has_type $I32 (sdiv _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 $I32 imm))(rv_divw x y))-(rule 2 (lower (has_type $I64 (sdiv x y)))+(rule 2 (lower (has_type $I64 (sdiv _ x y)))(if-let true (has_m))(rv_div x (safe_sdiv_divisor $I64 x y)))-(rule 3 (lower (has_type $I64 (sdiv x y @ (iconst imm))))+(rule 3 (lower (has_type $I64 (sdiv _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 $I64 imm))(rv_div x y))@@ -660,84 +660,84 @@;;;; Rules for `urem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_16 ty) (urem x y)))+(rule 0 (lower (has_type (fits_in_16 ty) (urem _ x y)))(if-let true (has_m))(rv_remuw (zext x) (nonzero_divisor (zext y))))-(rule 1 (lower (has_type (fits_in_16 ty) (urem x y @ (iconst imm))))+(rule 1 (lower (has_type (fits_in_16 ty) (urem _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 ty imm))(rv_remuw (zext x) (zext y)))-(rule 2 (lower (has_type $I32 (urem x y)))+(rule 2 (lower (has_type $I32 (urem _ x y)))(if-let true (has_m))(rv_remuw x (nonzero_divisor (zext y))))-(rule 3 (lower (has_type $I32 (urem x y @ (iconst imm))))+(rule 3 (lower (has_type $I32 (urem _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 $I32 imm))(rv_remuw x y))-(rule 2 (lower (has_type $I64 (urem x y)))+(rule 2 (lower (has_type $I64 (urem _ x y)))(if-let true (has_m))(rv_remu x (nonzero_divisor y)))-(rule 3 (lower (has_type $I64 (urem x y @ (iconst imm))))+(rule 3 (lower (has_type $I64 (urem _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 $I64 imm))(rv_remu x y));;;; Rules for `srem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_16 ty) (srem x y)))+(rule 0 (lower (has_type (fits_in_16 ty) (srem _ x y)))(if-let true (has_m))(rv_remw (sext x) (nonzero_divisor (sext y))))-(rule 1 (lower (has_type (fits_in_16 ty) (srem x y @ (iconst imm))))+(rule 1 (lower (has_type (fits_in_16 ty) (srem _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 ty imm))(rv_remw (sext x) (sext y)))-(rule 2 (lower (has_type $I32 (srem x y)))+(rule 2 (lower (has_type $I32 (srem _ x y)))(if-let true (has_m))(rv_remw x (nonzero_divisor (sext y))))-(rule 3 (lower (has_type $I32 (srem x y @ (iconst imm))))+(rule 3 (lower (has_type $I32 (srem _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 $I32 imm))(rv_remw x y))-(rule 2 (lower (has_type $I64 (srem x y)))+(rule 2 (lower (has_type $I64 (srem _ x y)))(if-let true (has_m))(rv_rem x (nonzero_divisor y)))-(rule 3 (lower (has_type $I64 (srem x y @ (iconst imm))))+(rule 3 (lower (has_type $I64 (srem _ x y @ (iconst _ imm))))(if-let true (has_m))(if (safe_divisor_from_imm64 $I64 imm))(rv_rem x y));;;; Rules for `and` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type (fits_in_64 ty) (band x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (band _ x y)))(rv_and x y))-(rule 0 (lower (has_type (ty_reg_pair _) (band x y)))+(rule 0 (lower (has_type (ty_reg_pair _) (band _ x y)))(value_regs(rv_and (value_regs_get x 0) (value_regs_get y 0))(rv_and (value_regs_get x 1) (value_regs_get y 1))));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (band x (imm12_from_value y))))+(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x (imm12_from_value y))))(rv_andi x y))-(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (band (imm12_from_value x) y)))+(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (band _ (imm12_from_value x) y)))(rv_andi y x))-(rule 3 (lower (has_type (ty_supported_float_size ty) (band x y)))+(rule 3 (lower (has_type (ty_supported_float_size ty) (band _ x y)))(lower_float_binary (AluOPRRR.And) x y ty));; No need to NaN-box when moving back to the floating point register as the high;; bits will already be set.-(rule 4 (lower (has_type (ty_supported_float_size $F16) (band x y)))+(rule 4 (lower (has_type (ty_supported_float_size $F16) (band _ x y)))(if-let false (has_zfhmin))(lower_float_binary (AluOPRRR.And) x y $F32))@@ -745,66 +745,66 @@;; by Cranelift's `band_not` instruction that is legalized into the simpler;; forms early on.-(rule 5 (lower (has_type (fits_in_64 (ty_int ty)) (band x (bnot y))))+(rule 5 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x (bnot _ y))))(if-let true (has_zbb))(rv_andn x y))-(rule 6 (lower (has_type (fits_in_64 (ty_int ty)) (band (bnot y) x)))+(rule 6 (lower (has_type (fits_in_64 (ty_int ty)) (band _ (bnot _ y) x)))(if-let true (has_zbb))(rv_andn x y))-(rule 7 (lower (has_type (ty_reg_pair _) (band x (bnot y))))+(rule 7 (lower (has_type (ty_reg_pair _) (band _ x (bnot _ y))))(if-let true (has_zbb))(let ((low XReg (rv_andn (value_regs_get x 0) (value_regs_get y 0)))(high XReg (rv_andn (value_regs_get x 1) (value_regs_get y 1))))(value_regs low high)))-(rule 8 (lower (has_type (ty_reg_pair _) (band (bnot y) x)))+(rule 8 (lower (has_type (ty_reg_pair _) (band _ (bnot _ y) x)))(if-let true (has_zbb))(let ((low XReg (rv_andn (value_regs_get x 0) (value_regs_get y 0)))(high XReg (rv_andn (value_regs_get x 1) (value_regs_get y 1))))(value_regs low high)))-(rule 9 (lower (has_type (ty_supported_vec ty) (band x y)))+(rule 9 (lower (has_type (ty_supported_vec ty) (band _ x y)))(rv_vand_vv x y (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (band x (splat y))))+(rule 10 (lower (has_type (ty_supported_vec ty) (band _ x (splat _ y))))(if (ty_vector_not_float ty))(rv_vand_vx x y (unmasked) ty))-(rule 11 (lower (has_type (ty_supported_vec ty) (band (splat x) y)))+(rule 11 (lower (has_type (ty_supported_vec ty) (band _ (splat _ x) y)))(if (ty_vector_not_float ty))(rv_vand_vx y x (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (band x y)))+(rule 12 (lower (has_type (ty_supported_vec ty) (band _ x y)))(if-let y_imm (replicated_imm5 y))(rv_vand_vi x y_imm (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (band x y)))+(rule 13 (lower (has_type (ty_supported_vec ty) (band _ x y)))(if-let x_imm (replicated_imm5 x))(rv_vand_vi y x_imm (unmasked) ty));; `bclr{,i}` specializations from `zbs`-(rule 14 (lower (has_type (fits_in_32 ty) (band x (bnot (ishl (i64_from_iconst 1) y)))))+(rule 14 (lower (has_type (fits_in_32 ty) (band _ x (bnot _ (ishl _ (i64_from_iconst 1) y)))))(if-let true (has_zbs))(rv_bclr x (rv_andi y (imm12_const (u8_wrapping_sub (ty_bits ty) 1)))))-(rule 15 (lower (has_type (fits_in_32 ty) (band (bnot (ishl (i64_from_iconst 1) y)) x)))+(rule 15 (lower (has_type (fits_in_32 ty) (band _ (bnot _ (ishl _ (i64_from_iconst 1) y)) x)))(if-let true (has_zbs))(rv_bclr x (rv_andi y (imm12_const (u8_wrapping_sub (ty_bits ty) 1)))))-(rule 16 (lower (has_type $I64 (band x (bnot (ishl (i64_from_iconst 1) y)))))+(rule 16 (lower (has_type $I64 (band _ x (bnot _ (ishl _ (i64_from_iconst 1) y)))))(if-let true (has_zbs))(rv_bclr x y))-(rule 17 (lower (has_type $I64 (band (bnot (ishl (i64_from_iconst 1) y)) x)))+(rule 17 (lower (has_type $I64 (band _ (bnot _ (ishl _ (i64_from_iconst 1) y)) x)))(if-let true (has_zbs))(rv_bclr x y))-(rule 18 (lower (has_type (fits_in_64 ty) (band x (u64_from_iconst n))))+(rule 18 (lower (has_type (fits_in_64 ty) (band _ x (u64_from_iconst n))))(if-let true (has_zbs))(if-let imm (bclr_imm ty n))(rv_bclri x imm))-(rule 19 (lower (has_type (fits_in_64 ty) (band (u64_from_iconst n) x)))+(rule 19 (lower (has_type (fits_in_64 ty) (band _ (u64_from_iconst n) x)))(if-let true (has_zbs))(if-let imm (bclr_imm ty n))(rv_bclri x imm))@@ -814,65 +814,65 @@;; `bext{,i}` specializations from `zbs`-(rule 20 (lower (has_type $I32 (band (ushr x y) (u64_from_iconst 1))))+(rule 20 (lower (has_type $I32 (band _ (ushr _ x y) (u64_from_iconst 1))))(if-let true (has_zbs))(rv_bext x (rv_andi y (imm12_const 31))))-(rule 20 (lower (has_type $I32 (band (sshr x y) (u64_from_iconst 1))))+(rule 20 (lower (has_type $I32 (band _ (sshr _ x y) (u64_from_iconst 1))))(if-let true (has_zbs))(rv_bext x (rv_andi y (imm12_const 31))))-(rule 20 (lower (has_type $I32 (band (u64_from_iconst 1) (ushr x y))))+(rule 20 (lower (has_type $I32 (band _ (u64_from_iconst 1) (ushr _ x y))))(if-let true (has_zbs))(rv_bext x (rv_andi y (imm12_const 31))))-(rule 20 (lower (has_type $I32 (band (u64_from_iconst 1) (sshr x y))))+(rule 20 (lower (has_type $I32 (band _ (u64_from_iconst 1) (sshr _ x y))))(if-let true (has_zbs))(rv_bext x (rv_andi y (imm12_const 31))))-(rule 20 (lower (has_type $I64 (band (ushr x y) (u64_from_iconst 1))))+(rule 20 (lower (has_type $I64 (band _ (ushr _ x y) (u64_from_iconst 1))))(if-let true (has_zbs))(rv_bext x y))-(rule 20 (lower (has_type $I64 (band (sshr x y) (u64_from_iconst 1))))+(rule 20 (lower (has_type $I64 (band _ (sshr _ x y) (u64_from_iconst 1))))(if-let true (has_zbs))(rv_bext x y))-(rule 20 (lower (has_type $I64 (band (u64_from_iconst 1) (ushr x y))))+(rule 20 (lower (has_type $I64 (band _ (u64_from_iconst 1) (ushr _ x y))))(if-let true (has_zbs))(rv_bext x y))-(rule 20 (lower (has_type $I64 (band (u64_from_iconst 1) (sshr x y))))+(rule 20 (lower (has_type $I64 (band _ (u64_from_iconst 1) (sshr _ x y))))(if-let true (has_zbs))(rv_bext x y))-(rule 21 (lower (has_type $I32 (band (ushr x (imm12_from_value y)) (u64_from_iconst 1))))+(rule 21 (lower (has_type $I32 (band _ (ushr _ x (imm12_from_value y)) (u64_from_iconst 1))))(if-let true (has_zbs))(rv_bexti x (imm12_and y 31)))-(rule 21 (lower (has_type $I32 (band (sshr x (imm12_from_value y)) (u64_from_iconst 1))))+(rule 21 (lower (has_type $I32 (band _ (sshr _ x (imm12_from_value y)) (u64_from_iconst 1))))(if-let true (has_zbs))(rv_bexti x (imm12_and y 31)))-(rule 21 (lower (has_type $I64 (band (ushr x (imm12_from_value y)) (u64_from_iconst 1))))+(rule 21 (lower (has_type $I64 (band _ (ushr _ x (imm12_from_value y)) (u64_from_iconst 1))))(if-let true (has_zbs))(rv_bexti x (imm12_and y 63)))-(rule 21 (lower (has_type $I64 (band (sshr x (imm12_from_value y)) (u64_from_iconst 1))))+(rule 21 (lower (has_type $I64 (band _ (sshr _ x (imm12_from_value y)) (u64_from_iconst 1))))(if-let true (has_zbs))(rv_bexti x (imm12_and y 63)));;;; Rules for `or` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type (ty_int ty) (bor x y)))+(rule -1 (lower (has_type (ty_int ty) (bor _ x y)))(gen_or ty x y))-(rule 0 (lower (has_type $F128 (bor x y)))+(rule 0 (lower (has_type $F128 (bor _ x y)))(gen_or $I128 x y));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bor x (imm12_from_value y))))+(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x (imm12_from_value y))))(rv_ori x y))-(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bor (imm12_from_value x) y)))+(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ (imm12_from_value x) y)))(rv_ori y x))-(rule 3 (lower (has_type (ty_supported_float_size ty) (bor x y)))+(rule 3 (lower (has_type (ty_supported_float_size ty) (bor _ x y)))(lower_float_binary (AluOPRRR.Or) x y ty));; No need to NaN-box when moving back to the floating point register as the high;; bits will already be set.-(rule 4 (lower (has_type (ty_supported_float_size $F16) (bor x y)))+(rule 4 (lower (has_type (ty_supported_float_size $F16) (bor _ x y)))(if-let false (has_zfhmin))(lower_float_binary (AluOPRRR.Or) x y $F32))@@ -880,66 +880,66 @@;; by Cranelift's `bor_not` instruction that is legalized into the simpler;; forms early on.-(rule 5 (lower (has_type (fits_in_64 (ty_int ty)) (bor x (bnot y))))+(rule 5 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x (bnot _ y))))(if-let true (has_zbb))(rv_orn x y))-(rule 6 (lower (has_type (fits_in_64 (ty_int ty)) (bor (bnot y) x)))+(rule 6 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ (bnot _ y) x)))(if-let true (has_zbb))(rv_orn x y))-(rule 7 (lower (has_type (ty_reg_pair _) (bor x (bnot y))))+(rule 7 (lower (has_type (ty_reg_pair _) (bor _ x (bnot _ y))))(if-let true (has_zbb))(let ((low XReg (rv_orn (value_regs_get x 0) (value_regs_get y 0)))(high XReg (rv_orn (value_regs_get x 1) (value_regs_get y 1))))(value_regs low high)))-(rule 8 (lower (has_type (ty_reg_pair _) (bor (bnot y) x)))+(rule 8 (lower (has_type (ty_reg_pair _) (bor _ (bnot _ y) x)))(if-let true (has_zbb))(let ((low XReg (rv_orn (value_regs_get x 0) (value_regs_get y 0)))(high XReg (rv_orn (value_regs_get x 1) (value_regs_get y 1))))(value_regs low high)))-(rule 9 (lower (has_type (ty_supported_vec ty) (bor x y)))+(rule 9 (lower (has_type (ty_supported_vec ty) (bor _ x y)))(rv_vor_vv x y (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (bor x (splat y))))+(rule 10 (lower (has_type (ty_supported_vec ty) (bor _ x (splat _ y))))(if (ty_vector_not_float ty))(rv_vor_vx x y (unmasked) ty))-(rule 11 (lower (has_type (ty_supported_vec ty) (bor (splat x) y)))+(rule 11 (lower (has_type (ty_supported_vec ty) (bor _ (splat _ x) y)))(if (ty_vector_not_float ty))(rv_vor_vx y x (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (bor x y)))+(rule 12 (lower (has_type (ty_supported_vec ty) (bor _ x y)))(if-let y_imm (replicated_imm5 y))(rv_vor_vi x y_imm (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (bor x y)))+(rule 13 (lower (has_type (ty_supported_vec ty) (bor _ x y)))(if-let x_imm (replicated_imm5 x))(rv_vor_vi y x_imm (unmasked) ty));; `bset{,i}` specializations from `zbs`-(rule 14 (lower (has_type $I32 (bor x (ishl (i64_from_iconst 1) y))))+(rule 14 (lower (has_type $I32 (bor _ x (ishl _ (i64_from_iconst 1) y))))(if-let true (has_zbs))(rv_bset x (rv_andi y (imm12_const 31))))-(rule 15 (lower (has_type $I32 (bor (ishl (i64_from_iconst 1) y) x)))+(rule 15 (lower (has_type $I32 (bor _ (ishl _ (i64_from_iconst 1) y) x)))(if-let true (has_zbs))(rv_bset x (rv_andi y (imm12_const 31))))-(rule 14 (lower (has_type $I64 (bor x (ishl (i64_from_iconst 1) y))))+(rule 14 (lower (has_type $I64 (bor _ x (ishl _ (i64_from_iconst 1) y))))(if-let true (has_zbs))(rv_bset x y))-(rule 15 (lower (has_type $I64 (bor (ishl (i64_from_iconst 1) y) x)))+(rule 15 (lower (has_type $I64 (bor _ (ishl _ (i64_from_iconst 1) y) x)))(if-let true (has_zbs))(rv_bset x y))-(rule 16 (lower (has_type (fits_in_64 _) (bor x (u64_from_iconst n))))+(rule 16 (lower (has_type (fits_in_64 _) (bor _ x (u64_from_iconst n))))(if-let true (has_zbs))(if-let imm (bseti_imm n))(rv_bseti x imm))-(rule 17 (lower (has_type (fits_in_64 _) (bor (u64_from_iconst n) x)))+(rule 17 (lower (has_type (fits_in_64 _) (bor _ (u64_from_iconst n) x)))(if-let true (has_zbs))(if-let imm (bseti_imm n))(rv_bseti x imm))@@ -948,62 +948,62 @@(extern constructor bseti_imm bseti_imm);;;; Rules for `xor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 (ty_int ty)) (bxor x y)))+(rule 0 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x y)))(rv_xor x y));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor x (imm12_from_value y))))+(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x (imm12_from_value y))))(rv_xori x y))-(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bxor (imm12_from_value x) y)))+(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ (imm12_from_value x) y)))(rv_xori y x))-(rule 3 (lower (has_type (ty_reg_pair _) (bxor x y)))+(rule 3 (lower (has_type (ty_reg_pair _) (bxor _ x y)))(lower_b128_binary (AluOPRRR.Xor) x y))-(rule 4 (lower (has_type (ty_supported_float_size ty) (bxor x y)))+(rule 4 (lower (has_type (ty_supported_float_size ty) (bxor _ x y)))(lower_float_binary (AluOPRRR.Xor) x y ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (bxor x y)))+(rule 5 (lower (has_type (ty_supported_vec ty) (bxor _ x y)))(rv_vxor_vv x y (unmasked) ty))-(rule 6 (lower (has_type (ty_supported_vec ty) (bxor x (splat y))))+(rule 6 (lower (has_type (ty_supported_vec ty) (bxor _ x (splat _ y))))(if (ty_vector_not_float ty))(rv_vxor_vx x y (unmasked) ty))-(rule 7 (lower (has_type (ty_supported_vec ty) (bxor (splat x) y)))+(rule 7 (lower (has_type (ty_supported_vec ty) (bxor _ (splat _ x) y)))(if (ty_vector_not_float ty))(rv_vxor_vx y x (unmasked) ty))-(rule 8 (lower (has_type (ty_supported_vec ty) (bxor x y)))+(rule 8 (lower (has_type (ty_supported_vec ty) (bxor _ x y)))(if-let y_imm (replicated_imm5 y))(rv_vxor_vi x y_imm (unmasked) ty))-(rule 9 (lower (has_type (ty_supported_vec ty) (bxor x y)))+(rule 9 (lower (has_type (ty_supported_vec ty) (bxor _ x y)))(if-let x_imm (replicated_imm5 x))(rv_vxor_vi y x_imm (unmasked) ty));; `binv{,i}` specializations from `zbs`-(rule 13 (lower (has_type $I32 (bxor x (ishl (i64_from_iconst 1) y))))+(rule 13 (lower (has_type $I32 (bxor _ x (ishl _ (i64_from_iconst 1) y))))(if-let true (has_zbs))(rv_binv x (rv_andi y (imm12_const 31))))-(rule 14 (lower (has_type $I32 (bxor (ishl (i64_from_iconst 1) y) x)))+(rule 14 (lower (has_type $I32 (bxor _ (ishl _ (i64_from_iconst 1) y) x)))(if-let true (has_zbs))(rv_binv x (rv_andi y (imm12_const 31))))-(rule 13 (lower (has_type $I64 (bxor x (ishl (i64_from_iconst 1) y))))+(rule 13 (lower (has_type $I64 (bxor _ x (ishl _ (i64_from_iconst 1) y))))(if-let true (has_zbs))(rv_binv x y))-(rule 14 (lower (has_type $I64 (bxor (ishl (i64_from_iconst 1) y) x)))+(rule 14 (lower (has_type $I64 (bxor _ (ishl _ (i64_from_iconst 1) y) x)))(if-let true (has_zbs))(rv_binv x y))-(rule 15 (lower (has_type (fits_in_64 _) (bxor x (u64_from_iconst n))))+(rule 15 (lower (has_type (fits_in_64 _) (bxor _ x (u64_from_iconst n))))(if-let true (has_zbs))(if-let imm (binvi_imm n))(rv_binvi x imm))-(rule 16 (lower (has_type (fits_in_64 _) (bxor (u64_from_iconst n) x)))+(rule 16 (lower (has_type (fits_in_64 _) (bxor _ (u64_from_iconst n) x)))(if-let true (has_zbs))(if-let imm (binvi_imm n))(rv_binvi x imm))@@ -1013,30 +1013,30 @@;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int_ref_scalar_64 _) (bnot x)))+(rule 0 (lower (has_type (ty_int_ref_scalar_64 _) (bnot _ x)))(rv_not x))-(rule 1 (lower (has_type (ty_supported_float_size ty) (bnot x)))+(rule 1 (lower (has_type (ty_supported_float_size ty) (bnot _ x)))(move_x_to_f (rv_not (move_f_to_x x ty)) ty))-(rule 2 (lower (has_type (ty_reg_pair _) (bnot x)))+(rule 2 (lower (has_type (ty_reg_pair _) (bnot _ x)))(value_regs(rv_not (value_regs_get x 0))(rv_not (value_regs_get x 1))))-(rule 3 (lower (has_type (ty_supported_vec ty) (bnot x)))+(rule 3 (lower (has_type (ty_supported_vec ty) (bnot _ x)))(rv_vnot_v x (unmasked) ty))-(rule 4 (lower (has_type (ty_int_ref_scalar_64 _) (bnot (bxor x y))))+(rule 4 (lower (has_type (ty_int_ref_scalar_64 _) (bnot _ (bxor _ x y))))(if-let true (has_zbb))(rv_xnor x y));;;; Rules for `bit_reverse` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (bitrev x)))+(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (bitrev _ x)))(gen_bitrev ty x))-(rule 1 (lower (has_type $I128 (bitrev x)))+(rule 1 (lower (has_type $I128 (bitrev _ x)))(value_regs(gen_bitrev $I64 (value_regs_get x 1))(gen_bitrev $I64 (value_regs_get x 0))))@@ -1061,10 +1061,10 @@;;;; Rules for `bswap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bswap x)))+(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bswap _ x)))(gen_bswap ty x))-(rule 2 (lower (has_type $I128 (bswap x)))+(rule 2 (lower (has_type $I128 (bswap _ x)))(value_regs(gen_bswap $I64 (value_regs_get x 1))(gen_bswap $I64 (value_regs_get x 0))))@@ -1107,10 +1107,10 @@(rv_rev8 x));;;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (ctz x)))+(rule (lower (has_type (fits_in_64 ty) (ctz _ x)))(lower_ctz ty x))-(rule 1 (lower (has_type $I128 (ctz x)))+(rule 1 (lower (has_type $I128 (ctz _ x)))(let ((x_lo XReg (value_regs_get x 0))(x_hi XReg (value_regs_get x 1));; Count both halves@@ -1122,10 +1122,10 @@(value_regs result (imm $I64 0))));;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (clz x)))+(rule 0 (lower (has_type (fits_in_64 ty) (clz _ x)))(gen_cltz true x ty))-(rule 1 (lower (has_type $I128 (clz x)))+(rule 1 (lower (has_type $I128 (clz _ x)))(let ((x_lo XReg (value_regs_get x 0))(x_hi XReg (value_regs_get x 1));; Count both halves@@ -1135,18 +1135,18 @@(low XReg (gen_select_xreg (cmp_eqz x_hi) low (zero_reg))))(value_regs (rv_add high low) (imm $I64 0))))-(rule 2 (lower (has_type (fits_in_16 ty) (clz x)))+(rule 2 (lower (has_type (fits_in_16 ty) (clz _ x)))(if-let true (has_zbb))(let ((tmp XReg (zext x))(count XReg (rv_clz tmp)));; We always do the operation on the full 64-bit register, so subtract 64 from the result.(rv_addi count (imm12_const_add (ty_bits ty) -64))))-(rule 3 (lower (has_type $I32 (clz x)))+(rule 3 (lower (has_type $I32 (clz _ x)))(if-let true (has_zbb))(rv_clzw x))-(rule 3 (lower (has_type $I64 (clz x)))+(rule 3 (lower (has_type $I64 (clz _ x)))(if-let true (has_zbb))(rv_clz x))@@ -1159,7 +1159,7 @@;;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (cls x)))+(rule (lower (has_type (fits_in_64 ty) (cls _ x)))(let ((tmp XReg (sext x))(tmp2 XReg (gen_select_xreg (cmp_ltz tmp) (rv_not tmp) tmp))(tmp3 XReg (gen_clz tmp2)))@@ -1170,7 +1170,7 @@;; If the sign bit is set, we count the leading zeros of the inverted value.;; Otherwise we can just count the leading zeros of the original value.;; Subtract 1 since the sign bit does not count.-(rule 1 (lower (has_type $I128 (cls x)))+(rule 1 (lower (has_type $I128 (cls _ x)))(let ((low XReg (value_regs_get x 0))(high XReg (value_regs_get x 1))(low XReg (gen_select_xreg (cmp_ltz high) (rv_not low) low))@@ -1187,14 +1187,14 @@;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 _) (uextend val)))+(rule 0 (lower (has_type (fits_in_64 _) (uextend _ val)))(zext val))-(rule 1 (lower (has_type $I128 (uextend val)))+(rule 1 (lower (has_type $I128 (uextend _ val)))(value_regs (zext val) (imm $I64 0)));; When the source of an `uextend` is a load, we can merge both ops-(rule 2 (lower (has_type (fits_in_64 _) (uextend (sinkable_load inst ty flags addr offset))))+(rule 2 (lower (has_type (fits_in_64 _) (uextend _ (sinkable_load inst ty flags addr offset))))(gen_sunk_load inst (amode addr offset) (uextend_load_op ty) flags))(decl pure uextend_load_op (Type) LoadOP)@@ -1203,15 +1203,15 @@(rule (uextend_load_op $I32) (LoadOP.Lwu));;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 _) (sextend val @ (value_type in_ty))))+(rule 0 (lower (has_type (fits_in_64 _) (sextend _ val @ (value_type in_ty))))(sext val))-(rule 1 (lower (has_type $I128 (sextend val @ (value_type in_ty))))+(rule 1 (lower (has_type $I128 (sextend _ val @ (value_type in_ty))))(let ((lo XReg (sext val)))(value_regs lo (rv_srai lo (imm12_const 63)))));; When the source of an `sextend` is a load, we can merge both ops-(rule 2 (lower (has_type (fits_in_64 _) (sextend (sinkable_load inst ty flags addr offset))))+(rule 2 (lower (has_type (fits_in_64 _) (sextend _ (sinkable_load inst ty flags addr offset))))(gen_sunk_load inst (amode addr offset) (sextend_load_op ty) flags))(decl pure sextend_load_op (Type) LoadOP)@@ -1221,10 +1221,10 @@;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 _) (popcnt x)))+(rule 0 (lower (has_type (fits_in_64 _) (popcnt _ x)))(gen_popcnt (zext x)))-(rule 1 (lower (has_type $I128 (popcnt x)))+(rule 1 (lower (has_type $I128 (popcnt _ x)))(let((x ValueRegs x)(low XReg (gen_popcnt (value_regs_get x 0)))@@ -1232,15 +1232,15 @@(result XReg (rv_add low high)))(value_regs result (imm $I64 0))))-(rule 2 (lower (has_type (fits_in_64 _) (popcnt x)))+(rule 2 (lower (has_type (fits_in_64 _) (popcnt _ x)))(if-let true (has_zbb))(rv_cpop (zext x)))-(rule 3 (lower (has_type $I32 (popcnt x)))+(rule 3 (lower (has_type $I32 (popcnt _ x)))(if-let true (has_zbb))(rv_cpopw x))-(rule 3 (lower (has_type $I128 (popcnt x)))+(rule 3 (lower (has_type $I128 (popcnt _ x)))(if-let true (has_zbb))(let((x ValueRegs x)@@ -1261,7 +1261,7 @@;;;; TODO: LLVM generates a much better implementation for I8X16. See: https://godbolt.org/z/qr6vf9Gr3;; For the other types it seems to be largely the same.-(rule 4 (lower (has_type (ty_supported_vec ty) (popcnt x)))+(rule 4 (lower (has_type (ty_supported_vec ty) (popcnt _ x)))(if-let one (u64_to_uimm5 1))(if-let two (u64_to_uimm5 2))(if-let four (u64_to_uimm5 4))@@ -1296,34 +1296,34 @@;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8/16 bit types need a mask on the shift amount-(rule 0 (lower (has_type (ty_int (ty_8_or_16 ty)) (ishl x y)))+(rule 0 (lower (has_type (ty_int (ty_8_or_16 ty)) (ishl _ x y)))(if-let mask (u64_to_imm12 (ty_shift_mask ty)))(rv_sllw x (rv_andi (value_regs_get y 0) mask)));; Using the 32bit version of `sll` automatically masks the shift amount.-(rule 1 (lower (has_type $I32 (ishl x y)))+(rule 1 (lower (has_type $I32 (ishl _ x y)))(rv_sllw x (value_regs_get y 0)));; Similarly, the 64bit version does the right thing.-(rule 1 (lower (has_type $I64 (ishl x y)))+(rule 1 (lower (has_type $I64 (ishl _ x y)))(rv_sll x (value_regs_get y 0)));; If the shift amount is known. We can mask it and encode it in the instruction.-(rule 2 (lower (has_type (int_fits_in_32 ty) (ishl x (maybe_uextend (imm12_from_value y)))))+(rule 2 (lower (has_type (int_fits_in_32 ty) (ishl _ x (maybe_uextend (imm12_from_value y)))))(rv_slliw x (imm12_and y (ty_shift_mask ty))));; We technically don't need to mask the shift amount here. The instruction;; does the right thing. But it's neater when pretty printing it.-(rule 3 (lower (has_type ty @ $I64 (ishl x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (has_type ty @ $I64 (ishl _ x (maybe_uextend (imm12_from_value y)))))(rv_slli x (imm12_and y (ty_shift_mask ty))));; With `Zba` we have a shift that zero extends the LHS argument.-(rule 4 (lower (has_type $I64 (ishl (uextend x @ (value_type $I32)) (maybe_uextend (imm12_from_value y)))))+(rule 4 (lower (has_type $I64 (ishl _ (uextend _ x @ (value_type $I32)) (maybe_uextend (imm12_from_value y)))))(if-let true (has_zba))(rv_slliuw x y));; I128 cases-(rule 4 (lower (has_type $I128 (ishl x y)))+(rule 4 (lower (has_type $I128 (ishl _ x y)))(let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))(len_sub_shamt XReg (value_regs_get tmp 1))@@ -1346,39 +1346,39 @@;; SIMD Cases;; We don't need to mask anything since it is done by the instruction according to SEW.-(rule 5 (lower (has_type (ty_supported_vec ty) (ishl x y)))+(rule 5 (lower (has_type (ty_supported_vec ty) (ishl _ x y)))(rv_vsll_vx x (value_regs_get y 0) (unmasked) ty))-(rule 6 (lower (has_type (ty_supported_vec ty) (ishl x (maybe_uextend (uimm5_from_value y)))))+(rule 6 (lower (has_type (ty_supported_vec ty) (ishl _ x (maybe_uextend (uimm5_from_value y)))))(rv_vsll_vi x y (unmasked) ty));;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be;; zero extended.-(rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x y)))+(rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (ushr _ x y)))(if-let mask (u64_to_imm12 (ty_shift_mask ty)))(rv_srlw (zext x) (rv_andi (value_regs_get y 0) mask)));; Using the 32bit version of `srl` automatically masks the shift amount.-(rule 1 (lower (has_type $I32 (ushr x y)))+(rule 1 (lower (has_type $I32 (ushr _ x y)))(rv_srlw x (value_regs_get y 0)));; Similarly, the 64bit version does the right thing.-(rule 1 (lower (has_type $I64 (ushr x y)))+(rule 1 (lower (has_type $I64 (ushr _ x y)))(rv_srl x (value_regs_get y 0)));; When the RHS is known we can just encode it in the instruction.-(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x (maybe_uextend (imm12_from_value y)))))+(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr _ x (maybe_uextend (imm12_from_value y)))))(rv_srliw (zext x) (imm12_and y (ty_shift_mask ty))))-(rule 3 (lower (has_type $I32 (ushr x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (has_type $I32 (ushr _ x (maybe_uextend (imm12_from_value y)))))(rv_srliw x y))-(rule 3 (lower (has_type $I64 (ushr x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (has_type $I64 (ushr _ x (maybe_uextend (imm12_from_value y)))))(rv_srli x y))-(rule 3 (lower (has_type $I128 (ushr x y)))+(rule 3 (lower (has_type $I128 (ushr _ x y)))(let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))(len_sub_shamt XReg (value_regs_get tmp 1))@@ -1401,39 +1401,39 @@;; SIMD Cases;; We don't need to mask or extend anything since it is done by the instruction according to SEW.-(rule 4 (lower (has_type (ty_supported_vec ty) (ushr x y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (ushr _ x y)))(rv_vsrl_vx x (value_regs_get y 0) (unmasked) ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (ushr x (maybe_uextend (uimm5_from_value y)))))+(rule 5 (lower (has_type (ty_supported_vec ty) (ushr _ x (maybe_uextend (uimm5_from_value y)))))(rv_vsrl_vi x y (unmasked) ty));;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be;; zero extended.-(rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x y)))+(rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (sshr _ x y)))(if-let mask (u64_to_imm12 (ty_shift_mask ty)))(rv_sraw (sext x) (rv_andi (value_regs_get y 0) mask)));; Using the 32bit version of `sra` automatically masks the shift amount.-(rule 1 (lower (has_type $I32 (sshr x y)))+(rule 1 (lower (has_type $I32 (sshr _ x y)))(rv_sraw x (value_regs_get y 0)));; Similarly, the 64bit version does the right thing.-(rule 1 (lower (has_type $I64 (sshr x y)))+(rule 1 (lower (has_type $I64 (sshr _ x y)))(rv_sra x (value_regs_get y 0)));; When the RHS is known we can just encode it in the instruction.-(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x (maybe_uextend (imm12_from_value y)))))+(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr _ x (maybe_uextend (imm12_from_value y)))))(rv_sraiw (sext x) (imm12_and y (ty_shift_mask ty))))-(rule 3 (lower (has_type $I32 (sshr x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (has_type $I32 (sshr _ x (maybe_uextend (imm12_from_value y)))))(rv_sraiw x y))-(rule 3 (lower (has_type $I64 (sshr x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (has_type $I64 (sshr _ x (maybe_uextend (imm12_from_value y)))))(rv_srai x y))-(rule 3 (lower (has_type $I128 (sshr x y)))+(rule 3 (lower (has_type $I128 (sshr _ x y)))(let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))(len_sub_shamt XReg (value_regs_get tmp 1))@@ -1461,16 +1461,16 @@;; SIMD Cases;; We don't need to mask or extend anything since it is done by the instruction according to SEW.-(rule 4 (lower (has_type (ty_supported_vec ty) (sshr x y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (sshr _ x y)))(rv_vsra_vx x (value_regs_get y 0) (unmasked) ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (sshr x (maybe_uextend (uimm5_from_value y)))))+(rule 5 (lower (has_type (ty_supported_vec ty) (sshr _ x (maybe_uextend (uimm5_from_value y)))))(rv_vsra_vi x y (unmasked) ty));;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (rotl rs amount)))+(rule 0 (lower (has_type (fits_in_64 ty) (rotl _ rs amount)))(let((rs XReg (zext rs))(amount XReg (value_regs_get amount 0))@@ -1482,25 +1482,25 @@(part3 Reg (gen_select_xreg (cmp_eqz shamt) (zero_reg) part2)))(rv_or part1 part3)))-(rule 1 (lower (has_type $I32 (rotl rs amount)))+(rule 1 (lower (has_type $I32 (rotl _ rs amount)))(if-let true (has_zbb))(rv_rolw rs (value_regs_get amount 0)))-(rule 2 (lower (has_type $I32 (rotl rs (u64_from_iconst n))))+(rule 2 (lower (has_type $I32 (rotl _ rs (u64_from_iconst n))))(if-let true (has_zbb))(if-let (imm12_from_u64 imm) (u64_wrapping_sub 32 (u64_and n 31)))(rv_roriw rs imm))-(rule 1 (lower (has_type $I64 (rotl rs amount)))+(rule 1 (lower (has_type $I64 (rotl _ rs amount)))(if-let true (has_zbb))(rv_rol rs (value_regs_get amount 0)))-(rule 2 (lower (has_type $I64 (rotl rs (u64_from_iconst n))))+(rule 2 (lower (has_type $I64 (rotl _ rs (u64_from_iconst n))))(if-let true (has_zbb))(if-let (imm12_from_u64 imm) (u64_wrapping_sub 64 (u64_and n 63)))(rv_rori rs imm))-(rule 1 (lower (has_type $I128 (rotl x y)))+(rule 1 (lower (has_type $I128 (rotl _ x y)))(let((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))@@ -1526,7 +1526,7 @@;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (rotr rs amount)))+(rule (lower (has_type (fits_in_64 ty) (rotr _ rs amount)))(let((rs XReg (zext rs))(amount XReg (value_regs_get amount 0))@@ -1538,23 +1538,23 @@(part3 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) part2)))(rv_or part1 part3)))-(rule 1 (lower (has_type $I32 (rotr rs amount)))+(rule 1 (lower (has_type $I32 (rotr _ rs amount)))(if-let true (has_zbb))(rv_rorw rs (value_regs_get amount 0)))-(rule 2 (lower (has_type $I32 (rotr rs (imm12_from_value n))))+(rule 2 (lower (has_type $I32 (rotr _ rs (imm12_from_value n))))(if-let true (has_zbb))(rv_roriw rs n))-(rule 1 (lower (has_type $I64 (rotr rs amount)))+(rule 1 (lower (has_type $I64 (rotr _ rs amount)))(if-let true (has_zbb))(rv_ror rs (value_regs_get amount 0)))-(rule 2 (lower (has_type $I64 (rotr rs (imm12_from_value n))))+(rule 2 (lower (has_type $I64 (rotr _ rs (imm12_from_value n))))(if-let true (has_zbb))(rv_rori rs n))-(rule 1 (lower (has_type $I128 (rotr x y)))+(rule 1 (lower (has_type $I128 (rotr _ x y)))(let((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))@@ -1579,27 +1579,27 @@)));;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fabs x)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fabs _ x)))(rv_fabs ty x))-(rule 1 (lower (has_type (ty_supported_vec ty) (fabs x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (fabs _ x)))(rv_vfabs_v x (unmasked) ty));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fneg x)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fneg _ x)))(rv_fneg ty x))-(rule 1 (lower (has_type (ty_supported_vec ty) (fneg x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (fneg _ x)))(rv_vfneg_v x (unmasked) ty));;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fcopysign x y)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fcopysign _ x y)))(rv_fsgnj ty x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fcopysign x y)))+(rule 1 (lower (has_type (ty_supported_vec ty) (fcopysign _ x y)))(rv_vfsgnj_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fcopysign x (splat y))))+(rule 2 (lower (has_type (ty_supported_vec ty) (fcopysign _ x (splat _ y))))(rv_vfsgnj_vf x y (unmasked) ty));;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1626,7 +1626,7 @@(type IsFneg (enum (Result (negate u64) (value Value))))(decl pure is_fneg (Value) IsFneg)-(rule 1 (is_fneg (fneg x)) (IsFneg.Result 1 x))+(rule 1 (is_fneg (fneg _ x)) (IsFneg.Result 1 x))(rule 0 (is_fneg x) (IsFneg.Result 0 x))(decl pure is_fneg_neg (IsFneg) u64)@@ -1635,7 +1635,7 @@(decl pure get_fneg_value (IsFneg) Value)(rule (get_fneg_value (IsFneg.Result _ v)) v)-(rule (lower (has_type ty (fma x_src y_src z_src)))+(rule (lower (has_type ty (fma _ x_src y_src z_src)))(let((x_res IsFneg (is_fneg x_src))(y_res IsFneg (is_fneg y_src))@@ -1655,54 +1655,54 @@(rule 1 (rv_fma (ty_supported_vec ty) 0 1 x y z) (rv_vfmsac_vv z y x (unmasked) ty))(rule 1 (rv_fma (ty_supported_vec ty) 1 0 x y z) (rv_vfnmsac_vv z y x (unmasked) ty))(rule 1 (rv_fma (ty_supported_vec ty) 1 1 x y z) (rv_vfnmacc_vv z y x (unmasked) ty))-(rule 2 (rv_fma (ty_supported_vec ty) 0 0 (splat x) y z) (rv_vfmacc_vf z y x (unmasked) ty))-(rule 2 (rv_fma (ty_supported_vec ty) 0 1 (splat x) y z) (rv_vfmsac_vf z y x (unmasked) ty))-(rule 2 (rv_fma (ty_supported_vec ty) 1 0 (splat x) y z) (rv_vfnmsac_vf z y x (unmasked) ty))-(rule 2 (rv_fma (ty_supported_vec ty) 1 1 (splat x) y z) (rv_vfnmacc_vf z y x (unmasked) ty))-(rule 3 (rv_fma (ty_supported_vec ty) 0 0 x (splat y) z) (rv_vfmacc_vf z x y (unmasked) ty))-(rule 3 (rv_fma (ty_supported_vec ty) 0 1 x (splat y) z) (rv_vfmsac_vf z x y (unmasked) ty))-(rule 3 (rv_fma (ty_supported_vec ty) 1 0 x (splat y) z) (rv_vfnmsac_vf z x y (unmasked) ty))-(rule 3 (rv_fma (ty_supported_vec ty) 1 1 x (splat y) z) (rv_vfnmacc_vf z x y (unmasked) ty))+(rule 2 (rv_fma (ty_supported_vec ty) 0 0 (splat _ x) y z) (rv_vfmacc_vf z y x (unmasked) ty))+(rule 2 (rv_fma (ty_supported_vec ty) 0 1 (splat _ x) y z) (rv_vfmsac_vf z y x (unmasked) ty))+(rule 2 (rv_fma (ty_supported_vec ty) 1 0 (splat _ x) y z) (rv_vfnmsac_vf z y x (unmasked) ty))+(rule 2 (rv_fma (ty_supported_vec ty) 1 1 (splat _ x) y z) (rv_vfnmacc_vf z y x (unmasked) ty))+(rule 3 (rv_fma (ty_supported_vec ty) 0 0 x (splat _ y) z) (rv_vfmacc_vf z x y (unmasked) ty))+(rule 3 (rv_fma (ty_supported_vec ty) 0 1 x (splat _ y) z) (rv_vfmsac_vf z x y (unmasked) ty))+(rule 3 (rv_fma (ty_supported_vec ty) 1 0 x (splat _ y) z) (rv_vfnmsac_vf z x y (unmasked) ty))+(rule 3 (rv_fma (ty_supported_vec ty) 1 1 x (splat _ y) z) (rv_vfnmacc_vf z x y (unmasked) ty));;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (sqrt x)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (sqrt _ x)))(rv_fsqrt ty (FRM.RNE) x))-(rule 1 (lower (has_type (ty_supported_vec ty) (sqrt x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (sqrt _ x)))(rv_vfsqrt_v x (unmasked) ty));;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule -1;;(lower- (has_type (valid_atomic_transaction ty) (atomic_rmw (little_or_native_endian flags) op addr x)))+ (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) op addr x)))(gen_atomic (get_atomic_rmw_op ty op) addr x (atomic_amo)));;; for I8 and I16(rule 1(lower- (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw (little_or_native_endian flags) op addr x)))+ (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) op addr x)))(gen_atomic_rmw_loop op ty addr x));;;special for I8 and I16 max min etc.;;;because I need uextend or sextend the value.(rule 2(lower- (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw (little_or_native_endian flags) (is_atomic_rmw_max_etc op true) addr x)))+ (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) (is_atomic_rmw_max_etc op true) addr x)))(gen_atomic_rmw_loop op ty addr (sext x)))(rule 2;;(lower- (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw (little_or_native_endian flags) (is_atomic_rmw_max_etc op false) addr x)))+ (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) (is_atomic_rmw_max_etc op false) addr x)));;(gen_atomic_rmw_loop op ty addr (zext x)));;;;; Rules for `AtomicRmwOp.Sub`(rule(lower- (has_type (valid_atomic_transaction ty) (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Sub) addr x)))+ (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) addr x)))(let((tmp WritableReg (temp_writable_reg ty))(x2 Reg (rv_neg x)))@@ -1720,7 +1720,7 @@;;;;; Rules for `AtomicRmwOp.Nand`(rule(lower- (has_type (valid_atomic_transaction ty) (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Nand) addr x)))+ (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Nand) addr x)))(gen_atomic_rmw_loop (AtomicRmwOp.Nand) ty addr x))(decl is_atomic_rmw_max_etc (AtomicRmwOp bool) AtomicRmwOp)@@ -1728,7 +1728,7 @@;;;;; Rules for `atomic load`;;;;;;;;;;;;;;;;;(rule- (lower (has_type (valid_atomic_transaction ty) (atomic_load (little_or_native_endian flags) p)))+ (lower (has_type (valid_atomic_transaction ty) (atomic_load _ (little_or_native_endian flags) p)))(gen_atomic_load p ty))@@ -1754,7 +1754,7 @@;;;;; Rules for `atomic cas`;;;;;;;;;;;;;;;;;(rule- (lower (has_type (valid_atomic_transaction ty) (atomic_cas (little_or_native_endian flags) p e x)))+ (lower (has_type (valid_atomic_transaction ty) (atomic_cas _ (little_or_native_endian flags) p e x)))(let((t0 WritableReg (temp_writable_reg ty))(dst WritableReg (temp_writable_reg ty))@@ -1763,28 +1763,28 @@;;;;; Rules for `ireduce`;;;;;;;;;;;;;;;;;(rule- (lower (has_type ty (ireduce x)))+ (lower (has_type ty (ireduce _ x)))(value_regs_get x 0));;;;; Rules for `fpromote`;;;;;;;;;;;;;;;;;-(rule (lower (fpromote x))+(rule (lower (fpromote _ x))(rv_fcvtds x));;;;; Rules for `fvpromote_low`;;;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty) (fvpromote_low x)))+(rule (lower (has_type (ty_supported_vec ty) (fvpromote_low _ x)))(if-let half_ty (ty_half_width ty))(rv_vfwcvt_f_f_v x (unmasked) (vstate_mf2 half_ty)));;;;; Rules for `fdemote`;;;;;;;;;;;;;;;;;;-(rule (lower (fdemote x))+(rule (lower (fdemote _ x))(rv_fcvtsd (FRM.RNE) x));;;;; Rules for `fvdemote`;;;;;;;;;;;;;;;;;;; `vfncvt...` leaves the upper bits of the register undefined so;; we need to zero them out.-(rule (lower (has_type (ty_supported_vec ty @ $F32X4) (fvdemote x)))+(rule (lower (has_type (ty_supported_vec ty @ $F32X4) (fvdemote _ x)))(if-let zero (i8_to_imm5 0))(let ((narrow VReg (rv_vfncvt_f_f_w x (unmasked) (vstate_mf2 ty)))(mask VReg (gen_vec_mask 0xC)))@@ -1796,57 +1796,57 @@;;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fadd x y)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fadd _ x y)))(rv_fadd ty (FRM.RNE) x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fadd x y)))+(rule 1 (lower (has_type (ty_supported_vec ty) (fadd _ x y)))(rv_vfadd_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fadd x (splat y))))+(rule 2 (lower (has_type (ty_supported_vec ty) (fadd _ x (splat _ y))))(rv_vfadd_vf x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (fadd (splat x) y)))+(rule 3 (lower (has_type (ty_supported_vec ty) (fadd _ (splat _ x) y)))(rv_vfadd_vf y x (unmasked) ty));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fsub x y)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fsub _ x y)))(rv_fsub ty (FRM.RNE) x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fsub x y)))+(rule 1 (lower (has_type (ty_supported_vec ty) (fsub _ x y)))(rv_vfsub_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fsub x (splat y))))+(rule 2 (lower (has_type (ty_supported_vec ty) (fsub _ x (splat _ y))))(rv_vfsub_vf x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (fsub (splat x) y)))+(rule 3 (lower (has_type (ty_supported_vec ty) (fsub _ (splat _ x) y)))(rv_vfrsub_vf y x (unmasked) ty));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fmul x y)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fmul _ x y)))(rv_fmul ty (FRM.RNE) x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fmul x y)))+(rule 1 (lower (has_type (ty_supported_vec ty) (fmul _ x y)))(rv_vfmul_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fmul x (splat y))))+(rule 2 (lower (has_type (ty_supported_vec ty) (fmul _ x (splat _ y))))(rv_vfmul_vf x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (fmul (splat x) y)))+(rule 3 (lower (has_type (ty_supported_vec ty) (fmul _ (splat _ x) y)))(rv_vfmul_vf y x (unmasked) ty));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fdiv x y)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fdiv _ x y)))(rv_fdiv ty (FRM.RNE) x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fdiv x y)))+(rule 1 (lower (has_type (ty_supported_vec ty) (fdiv _ x y)))(rv_vfdiv_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fdiv x (splat y))))+(rule 2 (lower (has_type (ty_supported_vec ty) (fdiv _ x (splat _ y))))(rv_vfdiv_vf x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (fdiv (splat x) y)))+(rule 3 (lower (has_type (ty_supported_vec ty) (fdiv _ (splat _ x) y)))(rv_vfrdiv_vf y x (unmasked) ty));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1854,7 +1854,7 @@;; RISC-V's `fmin` instruction returns the number input if one of inputs is a;; NaN. We handle this by manually checking if one of the inputs is a NaN;; and selecting based on that result.-(rule 0 (lower (has_type (ty_supported_float_full ty) (fmin x y)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fmin _ x y)))(let (;; Check if both inputs are not nan.(is_ordered FloatCompare (fcmp_to_float_compare (FloatCC.Ordered) ty x y))@@ -1865,7 +1865,7 @@;; With Zfa we can use the special `fminm` that precisely matches the expected;; NaN behavior.-(rule 1 (lower (has_type (ty_supported_float_full ty) (fmin x y)))+(rule 1 (lower (has_type (ty_supported_float_full ty) (fmin _ x y)))(if-let true (has_zfa))(rv_fminm ty x y))@@ -1875,7 +1875,7 @@;;;; TODO: We can improve this by using a masked `fmin` instruction that modifies;; the canonical nan register. That way we could avoid the `vmerge.vv` instruction.-(rule 2 (lower (has_type (ty_supported_vec ty) (fmin x y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (fmin _ x y)))(let ((is_not_nan VReg (gen_fcmp_mask ty (FloatCC.Ordered) x y))(nan XReg (imm $I64 (canonical_nan_u64 (lane_type ty))))(vec_nan VReg (rv_vmv_vx nan ty))@@ -1887,7 +1887,7 @@;; RISC-V's `fmax` instruction returns the number input if one of inputs is a;; NaN. We handle this by manually checking if one of the inputs is a NaN;; and selecting based on that result.-(rule 0 (lower (has_type (ty_supported_float_full ty) (fmax x y)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (fmax _ x y)))(let (;; Check if both inputs are not nan.(is_ordered FloatCompare (fcmp_to_float_compare (FloatCC.Ordered) ty x y))@@ -1898,7 +1898,7 @@;; With Zfa we can use the special `fmaxm` that precisely matches the expected;; NaN behavior.-(rule 1 (lower (has_type (ty_supported_float_full ty) (fmax x y)))+(rule 1 (lower (has_type (ty_supported_float_full ty) (fmax _ x y)))(if-let true (has_zfa))(rv_fmaxm ty x y))@@ -1908,7 +1908,7 @@;;;; TODO: We can improve this by using a masked `fmax` instruction that modifies;; the canonical nan register. That way we could avoid the `vmerge.vv` instruction.-(rule 2 (lower (has_type (ty_supported_vec ty) (fmax x y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (fmax _ x y)))(let ((is_not_nan VReg (gen_fcmp_mask ty (FloatCC.Ordered) x y))(nan XReg (imm $I64 (canonical_nan_u64 (lane_type ty))))(vec_nan VReg (rv_vmv_vx nan ty))@@ -1917,7 +1917,7 @@;;;;; Rules for `stack_addr`;;;;;;;;;(rule- (lower (stack_addr ss offset))+ (lower (stack_addr _ ss offset))(gen_stack_addr ss offset));;;;; Rules for `select`;;;;;;;;;@@ -1926,28 +1926,28 @@;; of the iconst rule because that runs into regalloc issues. gen_select_xreg;; has some optimizations based on the use of the zero register so we have to;; manually match it here.-(rule 5 (lower (has_type (ty_int_ref_scalar_64 _) (select c (i64_from_iconst 0) y)))+(rule 5 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c (i64_from_iconst 0) y)))(gen_select_xreg (is_nonzero_cmp c) (zero_reg) y))-(rule 4 (lower (has_type (ty_int_ref_scalar_64 _) (select c x (i64_from_iconst 0))))+(rule 4 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c x (i64_from_iconst 0))))(gen_select_xreg (is_nonzero_cmp c) x (zero_reg)))-(rule 3 (lower (has_type (ty_int_ref_scalar_64 _) (select c x y)))+(rule 3 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c x y)))(gen_select_xreg (is_nonzero_cmp c) x y))-(rule 2 (lower (has_type (ty_reg_pair _) (select c x y)))+(rule 2 (lower (has_type (ty_reg_pair _) (select _ c x y)))(gen_select_regs (is_nonzero_cmp c) x y))-(rule 1 (lower (has_type (ty_supported_vec _) (select c x y)))+(rule 1 (lower (has_type (ty_supported_vec _) (select _ c x y)))(gen_select_vreg (is_nonzero_cmp c) x y))-(rule 0 (lower (has_type (ty_supported_float_size _) (select c x y)))+(rule 0 (lower (has_type (ty_supported_float_size _) (select _ c x y)))(gen_select_freg (is_nonzero_cmp c) x y));;;;; Rules for `bitselect`;;;;;;;;;;; Do a (c & x) | (~c & y) operation.-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (bitselect c x y)))+(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (bitselect _ c x y)))(let ((tmp_x XReg (rv_and c x))(c_inverse XReg (rv_not c))(tmp_y XReg (rv_and c_inverse y)))@@ -1958,7 +1958,7 @@;; using the type of the inputs so that we avoid emitting unnecessary;; `vsetvl` instructions. it's likely that the vector unit is already;; configured for that type.-(rule 1 (lower (has_type (ty_supported_vec ty) (bitselect c x y)))+(rule 1 (lower (has_type (ty_supported_vec ty) (bitselect _ c x y)))(let ((tmp_x VReg (rv_vand_vv c x (unmasked) ty))(c_inverse VReg (rv_vnot_v c (unmasked) ty))(tmp_y VReg (rv_vand_vv c_inverse y (unmasked) ty)))@@ -1976,26 +1976,26 @@;;;; See: https://github.com/bytecodealliance/wasmtime/issues/8131-(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect (icmp cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y)))+(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (icmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y)))(let ((mask VReg (gen_icmp_mask cmp_ty cc a b)))(rv_vmerge_vvm y x mask cmp_ty)))-(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect (fcmp cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y)))+(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (fcmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y)))(let ((mask VReg (gen_fcmp_mask cmp_ty cc a b)))(rv_vmerge_vvm y x mask cmp_ty)))-(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect (bitcast _ (fcmp cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y)))+(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (bitcast _ _ (fcmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y)))(let ((mask VReg (gen_fcmp_mask cmp_ty cc a b)))(rv_vmerge_vvm y x mask cmp_ty)))-(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect (bitcast _ (icmp cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y)))+(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (bitcast _ _ (icmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y)))(let ((mask VReg (gen_icmp_mask cmp_ty cc a b)))(rv_vmerge_vvm y x mask cmp_ty)));;;;; Rules for `isplit`;;;;;;;;;(rule- (lower (isplit x))+ (lower (isplit _ x))(let((t1 XReg (value_regs_get x 0))(t2 XReg (value_regs_get x 1)))@@ -2003,7 +2003,7 @@;;;;; Rules for `iconcat`;;;;;;;;;(rule- (lower (has_type $I128 (iconcat x y)))+ (lower (has_type $I128 (iconcat _ x y)))(let((t1 XReg x)(t2 XReg y))@@ -2012,90 +2012,90 @@;; Special-case the lowering of an `isplit` of a 128-bit multiply where the;; lower bits of the result are discarded and the operands are sign or zero;; extended. This maps directly to `umulh` and `smulh`.-(rule 1 (lower i @ (isplit (has_type $I128 (imul (uextend x) (uextend y)))))+(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (uextend _ x) (uextend _ y)))))(if-let (first_result lo) i)(if-let true (value_is_unused lo))(output_pair (invalid_reg) (rv_mulhu (zext x) (zext y))))-(rule 1 (lower i @ (isplit (has_type $I128 (imul (sextend x) (sextend y)))))+(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (sextend _ x) (sextend _ y)))))(if-let (first_result lo) i)(if-let true (value_is_unused lo))(output_pair (invalid_reg) (rv_mulh (sext x) (sext y))));;;;; Rules for `smax`;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (smax x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (smax _ x y)))(let ((x XReg (sext x))(y XReg (sext y)))(gen_select_xreg (cmp_gt x y) x y)))-(rule 1 (lower (has_type $I128 (smax x y)))+(rule 1 (lower (has_type $I128 (smax _ x y)))(gen_select_regs (icmp_to_int_compare (IntCC.SignedGreaterThan) x y) x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (smax x y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (smax _ x y)))(rv_vmax_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (smax x (splat y))))+(rule 3 (lower (has_type (ty_supported_vec ty) (smax _ x (splat _ y))))(rv_vmax_vx x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (smax (splat x) y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (smax _ (splat _ x) y)))(rv_vmax_vx y x (unmasked) ty));;;;; Rules for `smin`;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (smin x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (smin _ x y)))(let ((x XReg (sext x))(y XReg (sext y)))(gen_select_xreg (cmp_lt x y) x y)))-(rule 1 (lower (has_type $I128 (smin x y)))+(rule 1 (lower (has_type $I128 (smin _ x y)))(gen_select_regs (icmp_to_int_compare (IntCC.SignedLessThan) x y) x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (smin x y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (smin _ x y)))(rv_vmin_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (smin x (splat y))))+(rule 3 (lower (has_type (ty_supported_vec ty) (smin _ x (splat _ y))))(rv_vmin_vx x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (smin (splat x) y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (smin _ (splat _ x) y)))(rv_vmin_vx y x (unmasked) ty));;;;; Rules for `umax`;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (umax x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (umax _ x y)))(let ((x XReg (zext x))(y XReg (zext y)))(gen_select_xreg (cmp_gtu x y) x y)))-(rule 1 (lower (has_type $I128 (umax x y)))+(rule 1 (lower (has_type $I128 (umax _ x y)))(gen_select_regs (icmp_to_int_compare (IntCC.UnsignedGreaterThan) x y) x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (umax x y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (umax _ x y)))(rv_vmaxu_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (umax x (splat y))))+(rule 3 (lower (has_type (ty_supported_vec ty) (umax _ x (splat _ y))))(rv_vmaxu_vx x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (umax (splat x) y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (umax _ (splat _ x) y)))(rv_vmaxu_vx y x (unmasked) ty));;;;; Rules for `umin`;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (umin x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (umin _ x y)))(let ((x XReg (zext x))(y XReg (zext y)))(gen_select_xreg (cmp_ltu x y) x y)))-(rule 1 (lower (has_type $I128 (umin x y)))+(rule 1 (lower (has_type $I128 (umin _ x y)))(gen_select_regs (icmp_to_int_compare (IntCC.UnsignedLessThan) x y) x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (umin x y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (umin _ x y)))(rv_vminu_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (umin x (splat y))))+(rule 3 (lower (has_type (ty_supported_vec ty) (umin _ x (splat _ y))))(rv_vminu_vx x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (umin (splat x) y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (umin _ (splat _ x) y)))(rv_vminu_vx y x (unmasked) ty))@@ -2124,7 +2124,7 @@(gen_trapif_val_i128 (ZeroCond.Zero) value code)); fold icmp + trapz-(rule 2 (lower (trapz (icmp cc x @ (value_type (fits_in_64 _)) y) code))+(rule 2 (lower (trapz (icmp _ cc x @ (value_type (fits_in_64 _)) y) code))(gen_trapif (intcc_complement cc) x y code));;;;; Rules for `trapnz`;;;;;;;;;@@ -2137,44 +2137,44 @@(gen_trapif_val_i128 (ZeroCond.NonZero) value code)); fold icmp + trapnz-(rule 2 (lower (trapnz (icmp cc x @ (value_type (fits_in_64 _)) y) code))+(rule 2 (lower (trapnz (icmp _ cc x @ (value_type (fits_in_64 _)) y) code))(gen_trapif cc x y code));;;;; Rules for `uload8`;;;;;;;;;-(rule (lower (uload8 (little_or_native_endian flags) addr offset))+(rule (lower (uload8 _ (little_or_native_endian flags) addr offset))(gen_load (amode addr offset) (LoadOP.Lbu) flags));;;;; Rules for `sload8`;;;;;;;;;-(rule (lower (sload8 (little_or_native_endian flags) addr offset))+(rule (lower (sload8 _ (little_or_native_endian flags) addr offset))(gen_load (amode addr offset) (LoadOP.Lb) flags));;;;; Rules for `uload16`;;;;;;;;;-(rule (lower (uload16 (little_or_native_endian flags) addr offset))+(rule (lower (uload16 _ (little_or_native_endian flags) addr offset))(gen_load (amode addr offset) (LoadOP.Lhu) flags));;;;; Rules for `iload16`;;;;;;;;;-(rule (lower (sload16 (little_or_native_endian flags) addr offset))+(rule (lower (sload16 _ (little_or_native_endian flags) addr offset))(gen_load (amode addr offset) (LoadOP.Lh) flags));;;;; Rules for `uload32`;;;;;;;;;-(rule (lower (uload32 (little_or_native_endian flags) addr offset))+(rule (lower (uload32 _ (little_or_native_endian flags) addr offset))(gen_load (amode addr offset) (LoadOP.Lwu) flags));;;;; Rules for `sload32`;;;;;;;;;-(rule (lower (sload32 (little_or_native_endian flags) addr offset))+(rule (lower (sload32 _ (little_or_native_endian flags) addr offset))(gen_load (amode addr offset) (LoadOP.Lw) flags));;;;; Rules for `load`;;;;;;;;;-(rule (lower (has_type ty (load (little_or_native_endian flags) addr offset)))+(rule (lower (has_type ty (load _ (little_or_native_endian flags) addr offset)))(gen_load (amode addr offset) (load_op ty) flags))-(rule 1 (lower (has_type (ty_reg_pair _) (load (little_or_native_endian flags) addr offset)))+(rule 1 (lower (has_type (ty_reg_pair _) (load _ (little_or_native_endian flags) addr offset)))(if-let offset_plus_8 (i32_checked_add offset 8))(let ((lo XReg (gen_load (amode addr offset) (LoadOP.Ld) flags))(hi XReg (gen_load (amode addr offset_plus_8) (LoadOP.Ld) flags)))(value_regs lo hi)))-(rule 2 (lower (has_type (ty_supported_vec ty) (load (little_or_native_endian flags) addr offset)))+(rule 2 (lower (has_type (ty_supported_vec ty) (load _ (little_or_native_endian flags) addr offset)))(let ((eew VecElementWidth (element_width_from_type ty))(amode AMode (amode addr offset)))(vec_load eew (VecAMode.UnitStride amode) flags (unmasked) ty)))@@ -2201,27 +2201,27 @@(rv_vzext_vf2 loaded (unmasked) ty)));;;;; Rules for `uload8x8`;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I16X8) (uload8x8 (little_or_native_endian flags) addr offset)))+(rule (lower (has_type (ty_supported_vec ty @ $I16X8) (uload8x8 _ (little_or_native_endian flags) addr offset)))(gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)));;;;; Rules for `uload16x4`;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I32X4) (uload16x4 (little_or_native_endian flags) addr offset)))+(rule (lower (has_type (ty_supported_vec ty @ $I32X4) (uload16x4 _ (little_or_native_endian flags) addr offset)))(gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)));;;;; Rules for `uload32x2`;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I64X2) (uload32x2 (little_or_native_endian flags) addr offset)))+(rule (lower (has_type (ty_supported_vec ty @ $I64X2) (uload32x2 _ (little_or_native_endian flags) addr offset)))(gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)));;;;; Rules for `sload8x8`;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I16X8) (sload8x8 (little_or_native_endian flags) addr offset)))+(rule (lower (has_type (ty_supported_vec ty @ $I16X8) (sload8x8 _ (little_or_native_endian flags) addr offset)))(gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)));;;;; Rules for `sload16x4`;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I32X4) (sload16x4 (little_or_native_endian flags) addr offset)))+(rule (lower (has_type (ty_supported_vec ty @ $I32X4) (sload16x4 _ (little_or_native_endian flags) addr offset)))(gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)));;;;; Rules for `sload32x2`;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I64X2) (sload32x2 (little_or_native_endian flags) addr offset)))+(rule (lower (has_type (ty_supported_vec ty @ $I64X2) (sload32x2 _ (little_or_native_endian flags) addr offset)))(gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)));;;;; Rules for `istore8`;;;;;;;;;@@ -2264,7 +2264,7 @@;; materializing that, but before that happens try to match some;; constant-related patterns-(rule 0 (lower (icmp cc x @ (value_type (fits_in_64 ty)) y))+(rule 0 (lower (icmp _ cc x @ (value_type (fits_in_64 ty)) y))(lower_icmp cc x y)); Recursion: at most once to implement >= in terms of <.@@ -2354,7 +2354,7 @@;; Currently only `==`, `!=`, and `<` are implemented, and everything else;; delegates to one of those.-(rule 20 (lower (icmp cc x @ (value_type $I128) y))+(rule 20 (lower (icmp _ cc x @ (value_type $I128) y))(lower_icmp_i128 cc x y)); Recursion: at most once to implement some conditions in terms of a smaller primitive set.@@ -2400,23 +2400,23 @@;; vector icmp comparisons-(rule 30 (lower (icmp cc x @ (value_type (ty_supported_vec ty)) y))+(rule 30 (lower (icmp _ cc x @ (value_type (ty_supported_vec ty)) y))(gen_expand_mask ty (gen_icmp_mask ty cc x y)));;;;; Rules for `fcmp`;;;;;;;;;-(rule 0 (lower (fcmp cc x @ (value_type (ty_supported_float_full ty)) y))+(rule 0 (lower (fcmp _ cc x @ (value_type (ty_supported_float_full ty)) y))(lower_float_compare (fcmp_to_float_compare cc ty x y)))(decl lower_float_compare (FloatCompare) XReg)(rule (lower_float_compare (FloatCompare.One r)) r)(rule (lower_float_compare (FloatCompare.Zero r)) (rv_seqz r))-(rule 1 (lower (fcmp cc x @ (value_type (ty_supported_vec ty)) y))+(rule 1 (lower (fcmp _ cc x @ (value_type (ty_supported_vec ty)) y))(gen_expand_mask ty (gen_fcmp_mask ty cc x y)));;;;; Rules for `func_addr`;;;;;;;;;(rule- (lower (func_addr (func_ref_data _ name dist _)))+ (lower (func_addr _ (func_ref_data _ name dist _)))(load_ext_name name 0 dist));;;;; Rules for `fcvt_to_uint`;;;;;;;;;@@ -2428,7 +2428,7 @@;; TODO: could this perhaps be more optimal through inspection of the `fcsr`?;; Unsure whether that needs to be preserved across function calls and/or would;; cause other problems. Also unsure whether it's actually more performant.-(rule (lower (has_type ity (fcvt_to_uint v @ (value_type fty))))+(rule (lower (has_type ity (fcvt_to_uint _ v @ (value_type fty))))(let ((_ InstOutput (gen_trapz (rv_feq fty v v) (TrapCode.BAD_CONVERSION_TO_INTEGER)))(min FReg (imm fty (fcvt_umin_bound fty false)))(_ InstOutput (gen_trapnz (rv_fle fty v min) (TrapCode.INTEGER_OVERFLOW)))@@ -2445,7 +2445,7 @@;;;;; Rules for `fcvt_to_sint`;;;;;;;;;;; NB: see above with `fcvt_to_uint` as this is similar-(rule (lower (has_type ity (fcvt_to_sint v @ (value_type fty))))+(rule (lower (has_type ity (fcvt_to_sint _ v @ (value_type fty))))(let ((_ InstOutput (gen_trapz (rv_feq fty v v) (TrapCode.BAD_CONVERSION_TO_INTEGER)))(min FReg (imm fty (fcvt_smin_bound fty ity false)))(_ InstOutput (gen_trapnz (rv_fle fty v min) (TrapCode.INTEGER_OVERFLOW)))@@ -2461,7 +2461,7 @@;;;;; Rules for `fcvt_to_sint_sat`;;;;;;;;;-(rule 0 (lower (has_type to (fcvt_to_sint_sat v @ (value_type (ty_supported_float_full from)))))+(rule 0 (lower (has_type to (fcvt_to_sint_sat _ v @ (value_type (ty_supported_float_full from)))))(handle_fcvt_to_int_nan from v (lower_fcvt_to_sint_sat from to v)));; Lowers to a `rv_fcvt*` instruction but handles 8/16-bit cases where the@@ -2496,7 +2496,7 @@(not_nan_mask XReg (rv_neg is_not_nan)))(rv_and xreg not_nan_mask)))-(rule 1 (lower (has_type (ty_supported_vec _) (fcvt_to_sint_sat v @ (value_type from_ty))))+(rule 1 (lower (has_type (ty_supported_vec _) (fcvt_to_sint_sat _ v @ (value_type from_ty))))(if-let zero (i8_to_imm5 0))(let ((is_nan VReg (rv_vmfne_vv v v (unmasked) from_ty))(cvt VReg (rv_vfcvt_rtz_x_f_v v (unmasked) from_ty)))@@ -2504,7 +2504,7 @@;;;;; Rules for `fcvt_to_uint_sat`;;;;;;;;;-(rule 0 (lower (has_type to (fcvt_to_uint_sat v @ (value_type (ty_supported_float_full from)))))+(rule 0 (lower (has_type to (fcvt_to_uint_sat _ v @ (value_type (ty_supported_float_full from)))))(handle_fcvt_to_int_nan from v (lower_fcvt_to_uint_sat from to v)));; Lowers to a `rv_fcvt*` instruction but handles 8/16-bit cases where the@@ -2523,78 +2523,78 @@(decl fcvt_umin_bound (Type bool) u64)(extern constructor fcvt_umin_bound fcvt_umin_bound)-(rule 1 (lower (has_type (ty_supported_vec _) (fcvt_to_uint_sat v @ (value_type from_ty))))+(rule 1 (lower (has_type (ty_supported_vec _) (fcvt_to_uint_sat _ v @ (value_type from_ty))))(if-let zero (i8_to_imm5 0))(let ((is_nan VReg (rv_vmfne_vv v v (unmasked) from_ty))(cvt VReg (rv_vfcvt_rtz_xu_f_v v (unmasked) from_ty)))(rv_vmerge_vim cvt zero is_nan from_ty)));;;;; Rules for `fcvt_from_sint`;;;;;;;;;-(rule 0 (lower (has_type $F32 (fcvt_from_sint v @ (value_type (fits_in_16 ty)))))+(rule 0 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type (fits_in_16 ty)))))(rv_fcvtsl (FRM.RNE) (sext v)))-(rule 1 (lower (has_type $F32 (fcvt_from_sint v @ (value_type $I32))))+(rule 1 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type $I32))))(rv_fcvtsw (FRM.RNE) v))-(rule 1 (lower (has_type $F32 (fcvt_from_sint v @ (value_type $I64))))+(rule 1 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type $I64))))(rv_fcvtsl (FRM.RNE) v))-(rule 0 (lower (has_type $F64 (fcvt_from_sint v @ (value_type (fits_in_16 ty)))))+(rule 0 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type (fits_in_16 ty)))))(rv_fcvtdl (FRM.RNE) (sext v)))-(rule 1 (lower (has_type $F64 (fcvt_from_sint v @ (value_type $I32))))+(rule 1 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type $I32))))(rv_fcvtdw v))-(rule 1 (lower (has_type $F64 (fcvt_from_sint v @ (value_type $I64))))+(rule 1 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type $I64))))(rv_fcvtdl (FRM.RNE) v))-(rule 2 (lower (has_type (ty_supported_vec _) (fcvt_from_sint v @ (value_type from_ty))))+(rule 2 (lower (has_type (ty_supported_vec _) (fcvt_from_sint _ v @ (value_type from_ty))))(rv_vfcvt_f_x_v v (unmasked) from_ty));;;;; Rules for `fcvt_from_uint`;;;;;;;;;-(rule 0 (lower (has_type $F32 (fcvt_from_uint v @ (value_type (fits_in_16 ty)))))+(rule 0 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type (fits_in_16 ty)))))(rv_fcvtslu (FRM.RNE) (zext v)))-(rule 1 (lower (has_type $F32 (fcvt_from_uint v @ (value_type $I32))))+(rule 1 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type $I32))))(rv_fcvtswu (FRM.RNE) v))-(rule 1 (lower (has_type $F32 (fcvt_from_uint v @ (value_type $I64))))+(rule 1 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type $I64))))(rv_fcvtslu (FRM.RNE) v))-(rule 0 (lower (has_type $F64 (fcvt_from_uint v @ (value_type (fits_in_16 ty)))))+(rule 0 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type (fits_in_16 ty)))))(rv_fcvtdlu (FRM.RNE) (zext v)))-(rule 1 (lower (has_type $F64 (fcvt_from_uint v @ (value_type $I32))))+(rule 1 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type $I32))))(rv_fcvtdwu v))-(rule 1 (lower (has_type $F64 (fcvt_from_uint v @ (value_type $I64))))+(rule 1 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type $I64))))(rv_fcvtdlu (FRM.RNE) v))-(rule 2 (lower (has_type (ty_supported_vec _) (fcvt_from_uint v @ (value_type from_ty))))+(rule 2 (lower (has_type (ty_supported_vec _) (fcvt_from_uint _ v @ (value_type from_ty))))(rv_vfcvt_f_xu_v v (unmasked) from_ty));;;;; Rules for `symbol_value`;;;;;;;;;(rule- (lower (symbol_value (symbol_value_data name dist offset)))+ (lower (symbol_value _ (symbol_value_data name dist offset)))(load_ext_name name offset dist));;;;; Rules for `tls_value` ;;;;;;;;;;;;;;-(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value (symbol_value_data name _ _))))+(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value _ (symbol_value_data name _ _))))(elf_tls_get_addr name));;;;; Rules for `bitcast`;;;;;;;;;;; These rules should probably be handled in `gen_bitcast`, but it's convenient to have that return;; a single register, instead of a `ValueRegs`-(rule 3 (lower (has_type (ty_reg_pair _) (bitcast _ v @ (value_type (ty_supported_vec _)))))+(rule 3 (lower (has_type (ty_reg_pair _) (bitcast _ _ v @ (value_type (ty_supported_vec _)))))(value_regs(gen_extractlane $I64X2 v 0)(gen_extractlane $I64X2 v 1)));; Move the high half into a vector register, and then use vslide1up to move it up and;; insert the lower half in one instruction.-(rule 2 (lower (has_type (ty_supported_vec _) (bitcast _ v @ (value_type (ty_reg_pair _)))))+(rule 2 (lower (has_type (ty_supported_vec _) (bitcast _ _ v @ (value_type (ty_reg_pair _)))))(let ((lo XReg (value_regs_get v 0))(hi XReg (value_regs_get v 1))(vstate VState (vstate_from_type $I64X2))@@ -2603,38 +2603,38 @@;; `gen_bitcast` below only works with single register values, so handle I128;; and F128 specially here.-(rule 1 (lower (has_type (ty_reg_pair _) (bitcast _ v @ (value_type (ty_reg_pair _)))))+(rule 1 (lower (has_type (ty_reg_pair _) (bitcast _ _ v @ (value_type (ty_reg_pair _)))))v)-(rule 0 (lower (has_type out_ty (bitcast _ v @ (value_type in_ty))))+(rule 0 (lower (has_type out_ty (bitcast _ _ v @ (value_type in_ty))))(gen_bitcast v in_ty out_ty));;;;; Rules for `ceil`;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (ceil x)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (ceil _ x)))(gen_float_round (FRM.RUP) x ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (ceil x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (ceil _ x)))(gen_vec_round x (FRM.RUP) ty));;;;; Rules for `floor`;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (floor x)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (floor _ x)))(gen_float_round (FRM.RDN) x ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (floor x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (floor _ x)))(gen_vec_round x (FRM.RDN) ty));;;;; Rules for `trunc`;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (trunc x)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (trunc _ x)))(gen_float_round (FRM.RTZ) x ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (trunc x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (trunc _ x)))(gen_vec_round x (FRM.RTZ) ty));;;;; Rules for `nearest`;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (nearest x)))+(rule 0 (lower (has_type (ty_supported_float_full ty) (nearest _ x)))(gen_float_round (FRM.RNE) x ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (nearest x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (nearest _ x)))(gen_vec_round x (FRM.RNE) ty))@@ -2648,10 +2648,10 @@;; Base case: use `gen_bmask` to generate a 0 mask or -1 mask from the value of;; `cmp`. This is then used with some bit twiddling to produce the final result.-(rule 0 (lower (has_type (fits_in_64 _) (select_spectre_guard cmp x y)))+(rule 0 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp x y)))(let ((mask XReg (gen_bmask cmp)))(rv_or (rv_and mask x) (rv_andn y mask))))-(rule 1 (lower (has_type $I128 (select_spectre_guard cmp x y)))+(rule 1 (lower (has_type $I128 (select_spectre_guard _ cmp x y)))(let ((mask XReg (gen_bmask cmp)))(value_regs(rv_or (rv_and mask (value_regs_get x 0)) (rv_andn (value_regs_get y 0) mask))@@ -2659,14 +2659,14 @@;; Special case when an argument is the constant zero as some ands and ors;; can be folded away.-(rule 2 (lower (has_type (fits_in_64 _) (select_spectre_guard cmp (i64_from_iconst 0) y)))+(rule 2 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp (i64_from_iconst 0) y)))(rv_andn y (gen_bmask cmp)))-(rule 3 (lower (has_type (fits_in_64 _) (select_spectre_guard cmp x (i64_from_iconst 0))))+(rule 3 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp x (i64_from_iconst 0))))(rv_and x (gen_bmask cmp)));;;;; Rules for `bmask`;;;;;;;;;(rule- (lower (has_type oty (bmask x)))+ (lower (has_type oty (bmask _ x)))(lower_bmask x oty));; N.B.: the Ret itself is generated by the ABI.@@ -2675,13 +2675,13 @@;;; Rules for `get_{frame,stack}_pointer` and `get_return_address` ;;;;;;;;;;;;;-(rule (lower (get_frame_pointer))+(rule (lower (get_frame_pointer _))(gen_mov_from_preg (fp_reg)))-(rule (lower (get_stack_pointer))+(rule (lower (get_stack_pointer _))(gen_mov_from_preg (sp_reg)))-(rule (lower (get_return_address))+(rule (lower (get_return_address _))(load_ra));;; Rules for `iabs` ;;;;;;;;;;;;;@@ -2691,7 +2691,7 @@;; sext.{b,h,w} a0, a0;; neg a1, a0;; max a0, a0, a1-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (iabs x)))+(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (iabs _ x)))(let ((extended XReg (sext x))(negated XReg (rv_neg extended)))(gen_select_xreg (cmp_gt extended negated) extended negated)))@@ -2699,7 +2699,7 @@;; For vectors we generate the same code, but with vector instructions;; we can skip the sign extension, since the vector unit will only process;; Element Sized chunks.-(rule 1 (lower (has_type (ty_supported_vec ty) (iabs x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (iabs _ x)))(let ((negated VReg (rv_vneg_v x (unmasked) ty)))(rv_vmax_vv x negated (unmasked) ty)))@@ -2796,7 +2796,7 @@;;;; Rules for `extractlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (extractlane x @ (value_type ty) (u8_from_uimm8 idx)))+(rule (lower (extractlane _ x @ (value_type ty) (u8_from_uimm8 idx)))(gen_extractlane ty x idx));;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2804,14 +2804,14 @@;; We can insert a lane by using a masked splat from an X register.;; Build a mask that is only enabled in the lane we want to insert.;; Then use a masked splat (vmerge) to insert the value.-(rule 0 (lower (insertlane vec @ (value_type (ty_supported_vec ty))+(rule 0 (lower (insertlane _ vec @ (value_type (ty_supported_vec ty))val @ (value_type (ty_int _))(u8_from_uimm8 lane)))(let ((mask VReg (gen_vec_mask (u64_wrapping_shl 1 lane))))(rv_vmerge_vxm vec val mask ty)));; Similar to above, but using the float variants of the instructions.-(rule 1 (lower (insertlane vec @ (value_type (ty_supported_vec ty))+(rule 1 (lower (insertlane _ vec @ (value_type (ty_supported_vec ty))val @ (value_type (ty_supported_float_full _))(u8_from_uimm8 lane)))(let ((mask VReg (gen_vec_mask (u64_wrapping_shl 1 lane))))@@ -2819,7 +2819,7 @@;; If we are inserting from an Imm5 const we can use the immediate;; variant of vmerge.-(rule 2 (lower (insertlane vec @ (value_type (ty_supported_vec ty))+(rule 2 (lower (insertlane _ vec @ (value_type (ty_supported_vec ty))(i64_from_iconst (imm5_from_i64 imm))(u8_from_uimm8 lane)))(let ((mask VReg (gen_vec_mask (u64_wrapping_shl 1 lane))))@@ -2827,13 +2827,13 @@;;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type ty (splat n @ (value_type (ty_supported_float_full _)))))+(rule 0 (lower (has_type ty (splat _ n @ (value_type (ty_supported_float_full _)))))(rv_vfmv_vf n ty))-(rule 1 (lower (has_type ty (splat n @ (value_type (ty_int_ref_scalar_64 _)))))+(rule 1 (lower (has_type ty (splat _ n @ (value_type (ty_int_ref_scalar_64 _)))))(rv_vmv_vx n ty))-(rule 2 (lower (has_type ty (splat (iconst (u64_from_imm64 (imm5_from_u64 imm))))))+(rule 2 (lower (has_type ty (splat _ (iconst _ (u64_from_imm64 (imm5_from_u64 imm))))))(rv_vmv_vi imm ty));; TODO: We can splat out more patterns by using for example a vmv.v.i i8x16 for@@ -2844,56 +2844,56 @@;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (uadd_sat x y)))+(rule 0 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y)))(rv_vsaddu_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (uadd_sat x (splat y))))+(rule 1 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x (splat _ y))))(rv_vsaddu_vx x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (uadd_sat (splat x) y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (uadd_sat _ (splat _ x) y)))(rv_vsaddu_vx y x (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (uadd_sat x y)))+(rule 3 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y)))(if-let y_imm (replicated_imm5 y))(rv_vsaddu_vi x y_imm (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (uadd_sat x y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y)))(if-let x_imm (replicated_imm5 x))(rv_vsaddu_vi y x_imm (unmasked) ty));;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (sadd_sat x y)))+(rule 0 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y)))(rv_vsadd_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (sadd_sat x (splat y))))+(rule 1 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x (splat _ y))))(rv_vsadd_vx x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (sadd_sat (splat x) y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (sadd_sat _ (splat _ x) y)))(rv_vsadd_vx y x (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (sadd_sat x y)))+(rule 3 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y)))(if-let y_imm (replicated_imm5 y))(rv_vsadd_vi x y_imm (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (sadd_sat x y)))+(rule 4 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y)))(if-let x_imm (replicated_imm5 x))(rv_vsadd_vi y x_imm (unmasked) ty));;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (usub_sat x y)))+(rule 0 (lower (has_type (ty_supported_vec ty) (usub_sat _ x y)))(rv_vssubu_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (usub_sat x (splat y))))+(rule 1 (lower (has_type (ty_supported_vec ty) (usub_sat _ x (splat _ y))))(rv_vssubu_vx x y (unmasked) ty));;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (ssub_sat x y)))+(rule 0 (lower (has_type (ty_supported_vec ty) (ssub_sat _ x y)))(rv_vssub_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (ssub_sat x (splat y))))+(rule 1 (lower (has_type (ty_supported_vec ty) (ssub_sat _ x (splat _ y))))(rv_vssub_vx x y (unmasked) ty));;;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2904,7 +2904,7 @@;; be a 1.;; The reduce operation leaves the result in the lowest lane, we then move it;; into the destination X register.-(rule (lower (vall_true x @ (value_type (ty_supported_vec ty))))+(rule (lower (vall_true _ x @ (value_type (ty_supported_vec ty))))(if-let one (i8_to_imm5 1));; We don't need to broadcast the immediate into all lanes, only into lane 0.;; I did it this way since it uses one less instruction than with a vmv.s.x.@@ -2918,7 +2918,7 @@;; Here we do a Vector Reduce operation. Get the unsigned maximum value of the;; input vector register. Move the max to an X register, and do a `snez` on it;; to ensure its either 1 or 0.-(rule (lower (vany_true x @ (value_type (ty_supported_vec ty))))+(rule (lower (vany_true _ x @ (value_type (ty_supported_vec ty))))(let ((max VReg (rv_vredmaxu_vs x x (unmasked) ty))(x_max XReg (rv_vmv_xs max ty)))(rv_snez x_max)))@@ -2934,7 +2934,7 @@;; to hold the full mask. Additionally, in some cases (e.g. i64x2) we are going;; to read some tail bits. These are undefined, so we need to further mask them;; off.-(rule (lower (vhigh_bits x @ (value_type (ty_supported_vec ty))))+(rule (lower (vhigh_bits _ x @ (value_type (ty_supported_vec ty))))(let ((mask VReg (rv_vmslt_vx x (zero_reg) (unmasked) ty));; Here we only need I64X1, but emit an AVL of 2 since it;; saves one vector state change in the case of I64X2.@@ -2946,13 +2946,13 @@;;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (swizzle x y)))+(rule 0 (lower (has_type (ty_supported_vec ty) (swizzle _ x y)))(rv_vrgather_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (swizzle x (splat y))))+(rule 1 (lower (has_type (ty_supported_vec ty) (swizzle _ x (splat _ y))))(rv_vrgather_vx x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (swizzle x y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (swizzle _ x y)))(if-let y_imm (replicated_uimm5 y))(rv_vrgather_vi x y_imm (unmasked) ty))@@ -2963,7 +2963,7 @@;;;; vrgather will insert a 0 for lanes that are out of bounds, so we can let it load;; negative and out of bounds indexes.-(rule (lower (has_type (ty_supported_vec ty @ $I8X16) (shuffle x y (vconst_from_immediate mask))))+(rule (lower (has_type (ty_supported_vec ty @ $I8X16) (shuffle _ x y (vconst_from_immediate mask))))(if-let neg16 (i8_to_imm5 -16))(let ((x_mask VReg (gen_constant ty mask))(x_lanes VReg (rv_vrgather_vv x x_mask (unmasked) ty))@@ -2974,51 +2974,51 @@;;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Slide down half the vector, and do a signed extension.-(rule 0 (lower (has_type (ty_supported_vec out_ty) (swiden_high x @ (value_type in_ty))))+(rule 0 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ x @ (value_type in_ty))))(rv_vsext_vf2 (gen_slidedown_half in_ty x) (unmasked) out_ty))-(rule 1 (lower (has_type (ty_supported_vec out_ty) (swiden_high (swiden_high x @ (value_type in_ty)))))+(rule 1 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ (swiden_high _ x @ (value_type in_ty)))))(if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty)))(rv_vsext_vf4 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty))-(rule 2 (lower (has_type (ty_supported_vec out_ty) (swiden_high (swiden_high (swiden_high x @ (value_type in_ty))))))+(rule 2 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ (swiden_high _ (swiden_high _ x @ (value_type in_ty))))))(if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty)))(rv_vsext_vf8 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty));;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Slide down half the vector, and do a zero extension.-(rule 0 (lower (has_type (ty_supported_vec out_ty) (uwiden_high x @ (value_type in_ty))))+(rule 0 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ x @ (value_type in_ty))))(rv_vzext_vf2 (gen_slidedown_half in_ty x) (unmasked) out_ty))-(rule 1 (lower (has_type (ty_supported_vec out_ty) (uwiden_high (uwiden_high x @ (value_type in_ty)))))+(rule 1 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ (uwiden_high _ x @ (value_type in_ty)))))(if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty)))(rv_vzext_vf4 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty))-(rule 2 (lower (has_type (ty_supported_vec out_ty) (uwiden_high (uwiden_high (uwiden_high x @ (value_type in_ty))))))+(rule 2 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ (uwiden_high _ (uwiden_high _ x @ (value_type in_ty))))))(if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty)))(rv_vzext_vf8 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty));;;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec out_ty) (swiden_low x)))+(rule 0 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ x)))(rv_vsext_vf2 x (unmasked) out_ty))-(rule 1 (lower (has_type (ty_supported_vec out_ty) (swiden_low (swiden_low x))))+(rule 1 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ (swiden_low _ x))))(rv_vsext_vf4 x (unmasked) out_ty))-(rule 2 (lower (has_type (ty_supported_vec out_ty) (swiden_low (swiden_low (swiden_low x)))))+(rule 2 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ (swiden_low _ (swiden_low _ x)))))(rv_vsext_vf8 x (unmasked) out_ty));;;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec out_ty) (uwiden_low x)))+(rule 0 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ x)))(rv_vzext_vf2 x (unmasked) out_ty))-(rule 1 (lower (has_type (ty_supported_vec out_ty) (uwiden_low (uwiden_low x))))+(rule 1 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ (uwiden_low _ x))))(rv_vzext_vf4 x (unmasked) out_ty))-(rule 2 (lower (has_type (ty_supported_vec out_ty) (uwiden_low (uwiden_low (uwiden_low x)))))+(rule 2 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ (uwiden_low _ (uwiden_low _ x)))))(rv_vzext_vf8 x (unmasked) out_ty));;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3036,7 +3036,7 @@;; However V8 does something better. They use 2 vcompresses using LMUL2, that means;; that they can do the whole thing in 3 instructions (2 vcompress + vadd). We don't;; support LMUL > 1, so we can't do that.-(rule (lower (has_type (ty_supported_vec ty) (iadd_pairwise x y)))+(rule (lower (has_type (ty_supported_vec ty) (iadd_pairwise _ x y)))(if-let half_size (u64_to_uimm5 (u64_checked_div (ty_lane_count ty) 2)))(let ((odd_mask VReg (gen_vec_mask 0x5555555555555555))(lhs_lo VReg (rv_vcompress_vm x odd_mask ty))@@ -3067,7 +3067,7 @@;; Right Logical instruction using the `vxrm` fixed-point rounding mode. The;; default rounding mode is `rnu` (round-to-nearest-up (add +0.5 LSB)).;; Which is coincidentally the rounding mode we want for `avg_round`.-(rule (lower (has_type (ty_supported_vec ty) (avg_round x y)))+(rule (lower (has_type (ty_supported_vec ty) (avg_round _ x y)))(if-let one (u64_to_uimm5 1))(let ((lhs VReg (rv_vand_vv x y (unmasked) ty))(xor VReg (rv_vxor_vv x y (unmasked) ty))@@ -3076,38 +3076,38 @@;;;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (scalar_to_vector x)))+(rule 0 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ x)))(if (ty_vector_float ty))(let ((zero VReg (rv_vmv_vx (zero_reg) ty))(elem VReg (rv_vfmv_sf x ty))(mask VReg (gen_vec_mask 1)))(rv_vmerge_vvm zero elem mask ty)))-(rule 1 (lower (has_type (ty_supported_vec ty) (scalar_to_vector x)))+(rule 1 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ x)))(if (ty_vector_not_float ty))(let ((zero VReg (rv_vmv_vx (zero_reg) ty))(mask VReg (gen_vec_mask 1)))(rv_vmerge_vxm zero x mask ty)))-(rule 2 (lower (has_type (ty_supported_vec ty) (scalar_to_vector (imm5_from_value x))))+(rule 2 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ (imm5_from_value x))))(let ((zero VReg (rv_vmv_vx (zero_reg) ty))(mask VReg (gen_vec_mask 1)))(rv_vmerge_vim zero x mask ty)));;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat x y)))+(rule 0 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ x y)))(rv_vsmul_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat x (splat y))))+(rule 1 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ x (splat _ y))))(rv_vsmul_vx x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat (splat x) y)))+(rule 2 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ (splat _ x) y)))(rv_vsmul_vx y x (unmasked) ty));;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec out_ty) (snarrow x @ (value_type in_ty) y)))+(rule (lower (has_type (ty_supported_vec out_ty) (snarrow _ x @ (value_type in_ty) y)))(if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2)))(if-let zero (u64_to_uimm5 0))(let ((x_clip VReg (rv_vnclip_wi x zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty))))@@ -3116,7 +3116,7 @@;;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec out_ty) (uunarrow x @ (value_type in_ty) y)))+(rule (lower (has_type (ty_supported_vec out_ty) (uunarrow _ x @ (value_type in_ty) y)))(if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2)))(if-let zero (u64_to_uimm5 0))(let ((x_clip VReg (rv_vnclipu_wi x zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty))))@@ -3129,7 +3129,7 @@;; To correct for this we just remove negative values using `vmax` and then use the normal;; unsigned to unsigned narrowing instruction.-(rule (lower (has_type (ty_supported_vec out_ty) (unarrow x @ (value_type in_ty) y)))+(rule (lower (has_type (ty_supported_vec out_ty) (unarrow _ x @ (value_type in_ty) y)))(if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2)))(if-let zero (u64_to_uimm5 0))(let ((x_pos VReg (rv_vmax_vx x (zero_reg) (unmasked) in_ty))@@ -3141,7 +3141,7 @@;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))+(rule (lower (get_exception_handler_address _ (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(rv64_label_address succ_label)))diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isleindex 41830d57bd..99b16344a2 100644--- a/cranelift/codegen/src/isa/s390x/inst.isle+++ b/cranelift/codegen/src/isa/s390x/inst.isle@@ -1918,11 +1918,11 @@(rule (lower_address flags addr @ (value_type (ty_addr64 _)) (i64_from_offset offset))(memarg_reg_plus_off addr offset 0 flags))-(rule 1 (lower_address flags (has_type (ty_addr64 _) (iadd x y)) (memarg_imm_from_offset z))+(rule 1 (lower_address flags (has_type (ty_addr64 _) (iadd _ x y)) (memarg_imm_from_offset z))(memarg_reg_plus_reg_plus_off x y z flags))(rule 1 (lower_address flags- (symbol_value (symbol_value_data name (RelocDistance.Near) sym_offset))+ (symbol_value _ (symbol_value_data name (RelocDistance.Near) sym_offset))(i64_from_offset offset))(if-let final_offset (memarg_symbol_offset_sum offset sym_offset))(memarg_symbol name final_offset flags))@@ -1935,7 +1935,7 @@(rule (lower_address_bias flags addr @ (value_type $I64) (i64_from_offset offset) bias)(memarg_reg_plus_off addr offset bias flags))-(rule 1 (lower_address_bias flags (has_type $I64 (iadd x y)) z bias)+(rule 1 (lower_address_bias flags (has_type $I64 (iadd _ x y)) z bias)(if-let final_offset (memarg_imm_from_offset_plus_bias z bias))(memarg_reg_plus_reg_plus_off x y final_offset flags))@@ -1944,7 +1944,7 @@(decl pure partial load_sym (Inst) Inst)(rule (load_sym inst)- (if-let (load _ (symbol_value (symbol_value_data _ (RelocDistance.Near) sym_offset))+ (if-let (load _ _ (symbol_value _ (symbol_value_data _ (RelocDistance.Near) sym_offset))(i64_from_offset load_offset))inst)(if (memarg_symbol_offset_sum sym_offset load_offset))@@ -1952,7 +1952,7 @@(decl pure partial uload16_sym (Inst) Inst)(rule (uload16_sym inst)- (if-let (uload16 _ (symbol_value (symbol_value_data _ (RelocDistance.Near) sym_offset))+ (if-let (uload16 _ _ (symbol_value _ (symbol_value_data _ (RelocDistance.Near) sym_offset))(i64_from_offset load_offset))inst)(if (memarg_symbol_offset_sum sym_offset load_offset))@@ -1972,11 +1972,11 @@;; A value that is the result of a sign-extend from a 32-bit value.(decl sext32_value (Value) Value)-(extractor (sext32_value x) (sextend (and x (value_type $I32))))+(extractor (sext32_value x) (sextend _ (and x (value_type $I32))));; A value that is the result of a zero-extend from a 32-bit value.(decl zext32_value (Value) Value)-(extractor (zext32_value x) (uextend (and x (value_type $I32))))+(extractor (zext32_value x) (uextend _ (and x (value_type $I32))));; Helpers for sinkable loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1988,7 +1988,7 @@;; Sinkable big-endian load instruction.(decl sinkable_load (Inst) Value)(extractor (sinkable_load inst)- (sinkable_inst (and inst (load (bigendian) _addr _offset))))+ (sinkable_inst (and inst (load _ (bigendian) _addr _offset))));; Sinkable big-endian load instruction (32/64-bit types only).(decl sinkable_load_32_64 (Inst) Value)@@ -2003,60 +2003,60 @@;; Sinkable little-endian load instruction.(decl sinkable_load_little (Inst) Value)(extractor (sinkable_load_little inst)- (sinkable_inst (and inst (load (littleendian) _addr _offset))))+ (sinkable_inst (and inst (load _ (littleendian) _addr _offset))));; Sinkable big-endian sload16 instruction.(decl sinkable_sload16 (Inst) Value)(extractor (sinkable_sload16 inst)- (sinkable_inst (and inst (sload16 (bigendian) _addr _offset))))+ (sinkable_inst (and inst (sload16 _ (bigendian) _addr _offset))));; Sinkable big-endian sload32 instruction.(decl sinkable_sload32 (Inst) Value)(extractor (sinkable_sload32 inst)- (sinkable_inst (and inst (sload32 (bigendian) _addr _offset))))+ (sinkable_inst (and inst (sload32 _ (bigendian) _addr _offset))));; Sinkable big-endian uload16 instruction.(decl sinkable_uload16 (Inst) Value)(extractor (sinkable_uload16 inst)- (sinkable_inst (and inst (uload16 (bigendian) _addr _offset))))+ (sinkable_inst (and inst (uload16 _ (bigendian) _addr _offset))));; Sinkable big-endian uload32 instruction.(decl sinkable_uload32 (Inst) Value)(extractor (sinkable_uload32 inst)- (sinkable_inst (and inst (uload32 (bigendian) _addr _offset))))+ (sinkable_inst (and inst (uload32 _ (bigendian) _addr _offset))));; Sink a load instruction, returning its address as `MemArg`.;; This is a side-effectful operation, see `sink_inst`.(decl sink_load (Inst) MemArg)-(rule (sink_load inst @ (load flags addr offset))+(rule (sink_load inst @ (load _ flags addr offset))(let ((_ Unit (sink_inst inst)))(lower_address flags addr offset)));; Sink a sload16 instruction, returning its address as `MemArg`.;; This is a side-effectful operation, see `sink_inst`.(decl sink_sload16 (Inst) MemArg)-(rule (sink_sload16 inst @ (sload16 flags addr offset))+(rule (sink_sload16 inst @ (sload16 _ flags addr offset))(let ((_ Unit (sink_inst inst)))(lower_address flags addr offset)));; Sink a sload32 instruction, returning its address as `MemArg`.;; This is a side-effectful operation, see `sink_inst`.(decl sink_sload32 (Inst) MemArg)-(rule (sink_sload32 inst @ (sload32 flags addr offset))+(rule (sink_sload32 inst @ (sload32 _ flags addr offset))(let ((_ Unit (sink_inst inst)))(lower_address flags addr offset)));; Sink a uload16 instruction, returning its address as `MemArg`.;; This is a side-effectful operation, see `sink_inst`.(decl sink_uload16 (Inst) MemArg)-(rule (sink_uload16 inst @ (uload16 flags addr offset))+(rule (sink_uload16 inst @ (uload16 _ flags addr offset))(let ((_ Unit (sink_inst inst)))(lower_address flags addr offset)));; Sink a uload32 instruction, returning its address as `MemArg`.;; This is a side-effectful operation, see `sink_inst`.(decl sink_uload32 (Inst) MemArg)-(rule (sink_uload32 inst @ (uload32 flags addr offset))+(rule (sink_uload32 inst @ (uload32 _ flags addr offset))(let ((_ Unit (sink_inst inst)))(lower_address flags addr offset)))diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isleindex 670ca6b2cc..583c0a24a6 100644--- a/cranelift/codegen/src/isa/s390x/lower.isle+++ b/cranelift/codegen/src/isa/s390x/lower.isle@@ -12,37 +12,37 @@;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (iconst (u64_from_imm64 n))))+(rule (lower (has_type ty (iconst _ (u64_from_imm64 n))))(imm ty n));;;; Rules for `f16const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f16const (u16_from_ieee16 x)))+(rule (lower (f16const _ (u16_from_ieee16 x)))(imm $F16 x));;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f32const (u32_from_ieee32 x)))+(rule (lower (f32const _ (u32_from_ieee32 x)))(imm $F32 x));;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f64const (u64_from_ieee64 x)))+(rule (lower (f64const _ (u64_from_ieee64 x)))(imm $F64 x));;;; Rules for `f128const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f128const (u128_from_constant x)))+(rule (lower (f128const _ (u128_from_constant x)))(vec_imm $F128 (be_vec_const $F128 x)));;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (vconst (u128_from_constant x))))+(rule (lower (has_type ty (vconst _ (u128_from_constant x))))(vec_imm ty (be_vec_const ty x)))@@ -54,13 +54,13 @@;;;; Rules for `iconcat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (vr128_ty ty) (iconcat x y)))+(rule (lower (has_type (vr128_ty ty) (iconcat _ x y)))(mov_to_vec128 ty y x));;;; Rules for `isplit` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (isplit x @ (value_type $I128)))+(rule (lower (isplit _ x @ (value_type $I128)))(let ((x_reg Reg x)(x_hi Reg (vec_extract_lane $I64X2 x_reg 0 (zero_reg)))(x_lo Reg (vec_extract_lane $I64X2 x_reg 1 (zero_reg))))@@ -70,56 +70,56 @@;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Add two registers.-(rule 0 (lower (has_type (fits_in_64 ty) (iadd x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (iadd _ x y)))(add_reg ty x y));; Add a register and a sign-extended register.-(rule 8 (lower (has_type (fits_in_64 ty) (iadd x (sext32_value y))))+(rule 8 (lower (has_type (fits_in_64 ty) (iadd _ x (sext32_value y))))(add_reg_sext32 ty x y))-(rule 15 (lower (has_type (fits_in_64 ty) (iadd (sext32_value x) y)))+(rule 15 (lower (has_type (fits_in_64 ty) (iadd _ (sext32_value x) y)))(add_reg_sext32 ty y x));; Add a register and an immediate.-(rule 7 (lower (has_type (fits_in_64 ty) (iadd x (i16_from_value y))))+(rule 7 (lower (has_type (fits_in_64 ty) (iadd _ x (i16_from_value y))))(add_simm16 ty x y))-(rule 14 (lower (has_type (fits_in_64 ty) (iadd (i16_from_value x) y)))+(rule 14 (lower (has_type (fits_in_64 ty) (iadd _ (i16_from_value x) y)))(add_simm16 ty y x))-(rule 6 (lower (has_type (fits_in_64 ty) (iadd x (i32_from_value y))))+(rule 6 (lower (has_type (fits_in_64 ty) (iadd _ x (i32_from_value y))))(add_simm32 ty x y))-(rule 13 (lower (has_type (fits_in_64 ty) (iadd (i32_from_value x) y)))+(rule 13 (lower (has_type (fits_in_64 ty) (iadd _ (i32_from_value x) y)))(add_simm32 ty y x));; Add a register and memory (32/64-bit types).-(rule 5 (lower (has_type (fits_in_64 ty) (iadd x (sinkable_load_32_64 y))))+(rule 5 (lower (has_type (fits_in_64 ty) (iadd _ x (sinkable_load_32_64 y))))(add_mem ty x (sink_load y)))-(rule 12 (lower (has_type (fits_in_64 ty) (iadd (sinkable_load_32_64 x) y)))+(rule 12 (lower (has_type (fits_in_64 ty) (iadd _ (sinkable_load_32_64 x) y)))(add_mem ty y (sink_load x)));; Add a register and memory (16-bit types).-(rule 4 (lower (has_type (fits_in_64 ty) (iadd x (sinkable_load_16 y))))+(rule 4 (lower (has_type (fits_in_64 ty) (iadd _ x (sinkable_load_16 y))))(add_mem_sext16 ty x (sink_load y)))-(rule 11 (lower (has_type (fits_in_64 ty) (iadd (sinkable_load_16 x) y)))+(rule 11 (lower (has_type (fits_in_64 ty) (iadd _ (sinkable_load_16 x) y)))(add_mem_sext16 ty y (sink_load x)));; Add a register and sign-extended memory.-(rule 3 (lower (has_type (fits_in_64 ty) (iadd x (sinkable_sload16 y))))+(rule 3 (lower (has_type (fits_in_64 ty) (iadd _ x (sinkable_sload16 y))))(add_mem_sext16 ty x (sink_sload16 y)))-(rule 10 (lower (has_type (fits_in_64 ty) (iadd (sinkable_sload16 x) y)))+(rule 10 (lower (has_type (fits_in_64 ty) (iadd _ (sinkable_sload16 x) y)))(add_mem_sext16 ty y (sink_sload16 x)))-(rule 2 (lower (has_type (fits_in_64 ty) (iadd x (sinkable_sload32 y))))+(rule 2 (lower (has_type (fits_in_64 ty) (iadd _ x (sinkable_sload32 y))))(add_mem_sext32 ty x (sink_sload32 y)))-(rule 9 (lower (has_type (fits_in_64 ty) (iadd (sinkable_sload32 x) y)))+(rule 9 (lower (has_type (fits_in_64 ty) (iadd _ (sinkable_sload32 x) y)))(add_mem_sext32 ty y (sink_sload32 x)));; Add two vector registers.-(rule 1 (lower (has_type (vr128_ty ty) (iadd x y)))+(rule 1 (lower (has_type (vr128_ty ty) (iadd _ x y)))(vec_add ty x y));;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Add (saturate unsigned) two vector registers.-(rule (lower (has_type (ty_vec128 ty) (uadd_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (uadd_sat _ x y)))(let ((sum Reg (vec_add ty x y)))(vec_or ty sum (vec_cmphl ty x sum))))@@ -127,7 +127,7 @@;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Add (saturate signed) two vector registers. $I64X2 not supported.-(rule (lower (has_type (ty_vec128 ty) (sadd_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (sadd_sat _ x y)))(vec_pack_ssat (vec_widen_type ty)(vec_add (vec_widen_type ty) (vec_unpacks_high ty x)(vec_unpacks_high ty y))@@ -138,7 +138,7 @@;;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Lane-wise integer pairwise addition for 8-/16/32-bit vector registers.-(rule (lower (has_type ty @ (multi_lane bits _) (iadd_pairwise x y)))+(rule (lower (has_type ty @ (multi_lane bits _) (iadd_pairwise _ x y)))(let ((size Reg (vec_imm_splat $I8X16 bits)))(vec_pack_lane_order (vec_widen_type ty)(vec_add ty x (vec_lshr_by_byte x size))@@ -146,9 +146,9 @@;; special case for the `i32x4.dot_i16x8_s` wasm instruction(rule 1 (lower- (has_type dst_ty (iadd_pairwise- (imul (swiden_low x @ (value_type src_ty)) (swiden_low y))- (imul (swiden_high x) (swiden_high y)))))+ (has_type dst_ty (iadd_pairwise _+ (imul _ (swiden_low _ x @ (value_type src_ty)) (swiden_low _ y))+ (imul _ (swiden_high _ x) (swiden_high _ y)))))(vec_add dst_ty (vec_smul_even src_ty x y)(vec_smul_odd src_ty x y)))@@ -156,49 +156,49 @@;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Sub two registers.-(rule 0 (lower (has_type (fits_in_64 ty) (isub x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (isub _ x y)))(sub_reg ty x y));; Sub a register and a sign-extended register.-(rule 8 (lower (has_type (fits_in_64 ty) (isub x (sext32_value y))))+(rule 8 (lower (has_type (fits_in_64 ty) (isub _ x (sext32_value y))))(sub_reg_sext32 ty x y));; Sub a register and an immediate (using add of the negated value).-(rule 7 (lower (has_type (fits_in_64 ty) (isub x (i16_from_negated_value y))))+(rule 7 (lower (has_type (fits_in_64 ty) (isub _ x (i16_from_negated_value y))))(add_simm16 ty x y))-(rule 6 (lower (has_type (fits_in_64 ty) (isub x (i32_from_negated_value y))))+(rule 6 (lower (has_type (fits_in_64 ty) (isub _ x (i32_from_negated_value y))))(add_simm32 ty x y));; Sub a register and memory (32/64-bit types).-(rule 5 (lower (has_type (fits_in_64 ty) (isub x (sinkable_load_32_64 y))))+(rule 5 (lower (has_type (fits_in_64 ty) (isub _ x (sinkable_load_32_64 y))))(sub_mem ty x (sink_load y)));; Sub a register and memory (16-bit types).-(rule 4 (lower (has_type (fits_in_64 ty) (isub x (sinkable_load_16 y))))+(rule 4 (lower (has_type (fits_in_64 ty) (isub _ x (sinkable_load_16 y))))(sub_mem_sext16 ty x (sink_load y)));; Sub a register and sign-extended memory.-(rule 3 (lower (has_type (fits_in_64 ty) (isub x (sinkable_sload16 y))))+(rule 3 (lower (has_type (fits_in_64 ty) (isub _ x (sinkable_sload16 y))))(sub_mem_sext16 ty x (sink_sload16 y)))-(rule 2 (lower (has_type (fits_in_64 ty) (isub x (sinkable_sload32 y))))+(rule 2 (lower (has_type (fits_in_64 ty) (isub _ x (sinkable_sload32 y))))(sub_mem_sext32 ty x (sink_sload32 y)));; Sub two vector registers.-(rule 1 (lower (has_type (vr128_ty ty) (isub x y)))+(rule 1 (lower (has_type (vr128_ty ty) (isub _ x y)))(vec_sub ty x y));;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Add (saturate unsigned) two vector registers.-(rule (lower (has_type (ty_vec128 ty) (usub_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (usub_sat _ x y)))(vec_and ty (vec_sub ty x y) (vec_cmphl ty x y)));;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Add (saturate signed) two vector registers. $I64X2 not supported.-(rule (lower (has_type (ty_vec128 ty) (ssub_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (ssub_sat _ x y)))(vec_pack_ssat (vec_widen_type ty)(vec_sub (vec_widen_type ty) (vec_unpacks_high ty x)(vec_unpacks_high ty y))@@ -210,23 +210,23 @@;; Absolute value of a register.;; For types smaller than 32-bit, the input value must be sign-extended.-(rule 2 (lower (has_type (fits_in_64 ty) (iabs x)))+(rule 2 (lower (has_type (fits_in_64 ty) (iabs _ x)))(abs_reg (ty_ext32 ty) (put_in_reg_sext32 x)));; Absolute value of a sign-extended register.-(rule 3 (lower (has_type (fits_in_64 ty) (iabs (sext32_value x))))+(rule 3 (lower (has_type (fits_in_64 ty) (iabs _ (sext32_value x))))(abs_reg_sext32 ty x));; Absolute value of a vector register.-(rule 1 (lower (has_type (ty_vec128 ty) (iabs x)))+(rule 1 (lower (has_type (ty_vec128 ty) (iabs _ x)))(vec_abs ty x));; Absolute value of a 128-bit integer on z17.-(rule 4 (lower (has_type (and (vxrs_ext3_enabled) $I128) (iabs x)))+(rule 4 (lower (has_type (and (vxrs_ext3_enabled) $I128) (iabs _ x)))(vec_abs $I128 x));; Absolute value of a 128-bit integer pre-z17.-(rule 0 (lower (has_type (and (vxrs_ext3_disabled) $I128) (iabs x)))+(rule 0 (lower (has_type (and (vxrs_ext3_disabled) $I128) (iabs _ x)))(let ((zero Reg (vec_imm $I128 0))(pos Reg x)(neg Reg (vec_sub $I128 zero pos))@@ -238,30 +238,30 @@;;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Negate a register.-(rule 2 (lower (has_type (fits_in_64 ty) (ineg x)))+(rule 2 (lower (has_type (fits_in_64 ty) (ineg _ x)))(neg_reg ty x));; Negate a sign-extended register.-(rule 3 (lower (has_type (fits_in_64 ty) (ineg (sext32_value x))))+(rule 3 (lower (has_type (fits_in_64 ty) (ineg _ (sext32_value x))))(neg_reg_sext32 ty x));; Negate a vector register.-(rule 1 (lower (has_type (ty_vec128 ty) (ineg x)))+(rule 1 (lower (has_type (ty_vec128 ty) (ineg _ x)))(vec_neg ty x));; Negate a 128-bit integer on z17.-(rule 4 (lower (has_type (and (vxrs_ext3_enabled) $I128) (ineg x)))+(rule 4 (lower (has_type (and (vxrs_ext3_enabled) $I128) (ineg _ x)))(vec_neg $I128 x));; Negate a 128-bit integer pre-z17.-(rule 0 (lower (has_type (and (vxrs_ext3_disabled) $I128) (ineg x)))+(rule 0 (lower (has_type (and (vxrs_ext3_disabled) $I128) (ineg _ x)))(vec_sub $I128 (vec_imm $I128 0) x));;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Unsigned maximum of two scalar integers - expand to icmp + select.-(rule 2 (lower (has_type (fits_in_64 ty) (umax x y)))+(rule 2 (lower (has_type (fits_in_64 ty) (umax _ x y)))(let ((x_ext Reg (put_in_reg_zext32 x))(y_ext Reg (put_in_reg_zext32 y))(cond ProducesBool (bool (icmpu_reg (ty_ext32 ty) x_ext y_ext)@@ -269,25 +269,25 @@(select_bool_reg ty cond y_ext x_ext)));; Unsigned maximum of two 128-bit integers pre-z17 - expand to icmp + select.-(rule 1 (lower (has_type (and (vxrs_ext3_disabled) $I128) (umax x y)))+(rule 1 (lower (has_type (and (vxrs_ext3_disabled) $I128) (umax _ x y)))(let ((x_reg Reg (put_in_reg x))(y_reg Reg (put_in_reg y))(cond ProducesBool (vec_int128_ucmphi y_reg x_reg)))(select_bool_reg $I128 cond y_reg x_reg)));; Unsigned maximum of two 128-bit integers on z17.-(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (umax x y)))+(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (umax _ x y)))(vec_umax $I128 x y));; Unsigned maximum of two vector registers.-(rule 0 (lower (has_type (ty_vec128 ty) (umax x y)))+(rule 0 (lower (has_type (ty_vec128 ty) (umax _ x y)))(vec_umax ty x y));;;; Rules for `umin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Unsigned minimum of two scalar integers - expand to icmp + select.-(rule 2 (lower (has_type (fits_in_64 ty) (umin x y)))+(rule 2 (lower (has_type (fits_in_64 ty) (umin _ x y)))(let ((x_ext Reg (put_in_reg_zext32 x))(y_ext Reg (put_in_reg_zext32 y))(cond ProducesBool (bool (icmpu_reg (ty_ext32 ty) x_ext y_ext)@@ -295,25 +295,25 @@(select_bool_reg ty cond y_ext x_ext)));; Unsigned maximum of two 128-bit integers pre-z17 - expand to icmp + select.-(rule 1 (lower (has_type (and (vxrs_ext3_disabled) $I128) (umin x y)))+(rule 1 (lower (has_type (and (vxrs_ext3_disabled) $I128) (umin _ x y)))(let ((x_reg Reg (put_in_reg x))(y_reg Reg (put_in_reg y))(cond ProducesBool (vec_int128_ucmphi x_reg y_reg)))(select_bool_reg $I128 cond y_reg x_reg)));; Unsigned minimum of two 128-bit integers on z17.-(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (umin x y)))+(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (umin _ x y)))(vec_umin $I128 x y));; Unsigned minimum of two vector registers.-(rule 0 (lower (has_type (ty_vec128 ty) (umin x y)))+(rule 0 (lower (has_type (ty_vec128 ty) (umin _ x y)))(vec_umin ty x y));;;; Rules for `smax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Signed maximum of two scalar integers - expand to icmp + select.-(rule 2 (lower (has_type (fits_in_64 ty) (smax x y)))+(rule 2 (lower (has_type (fits_in_64 ty) (smax _ x y)))(let ((x_ext Reg (put_in_reg_sext32 x))(y_ext Reg (put_in_reg_sext32 y))(cond ProducesBool (bool (icmps_reg (ty_ext32 ty) x_ext y_ext)@@ -321,25 +321,25 @@(select_bool_reg ty cond y_ext x_ext)));; Signed maximum of two 128-bit integers pre-z17 - expand to icmp + select.-(rule 1 (lower (has_type (and (vxrs_ext3_disabled) $I128) (smax x y)))+(rule 1 (lower (has_type (and (vxrs_ext3_disabled) $I128) (smax _ x y)))(let ((x_reg Reg (put_in_reg x))(y_reg Reg (put_in_reg y))(cond ProducesBool (vec_int128_scmphi y_reg x_reg)))(select_bool_reg $I128 cond y_reg x_reg)));; Signed maximum of two 128-bit integers on z17.-(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (smax x y)))+(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (smax _ x y)))(vec_smax $I128 x y));; Signed maximum of two vector registers.-(rule (lower (has_type (ty_vec128 ty) (smax x y)))+(rule (lower (has_type (ty_vec128 ty) (smax _ x y)))(vec_smax ty x y));;;; Rules for `smin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Signed minimum of two scalar integers - expand to icmp + select.-(rule 2 (lower (has_type (fits_in_64 ty) (smin x y)))+(rule 2 (lower (has_type (fits_in_64 ty) (smin _ x y)))(let ((x_ext Reg (put_in_reg_sext32 x))(y_ext Reg (put_in_reg_sext32 y))(cond ProducesBool (bool (icmps_reg (ty_ext32 ty) x_ext y_ext)@@ -347,75 +347,75 @@(select_bool_reg ty cond y_ext x_ext)));; Signed maximum of two 128-bit integers pre-z17 - expand to icmp + select.-(rule 1 (lower (has_type (and (vxrs_ext3_disabled) $I128) (smin x y)))+(rule 1 (lower (has_type (and (vxrs_ext3_disabled) $I128) (smin _ x y)))(let ((x_reg Reg (put_in_reg x))(y_reg Reg (put_in_reg y))(cond ProducesBool (vec_int128_scmphi x_reg y_reg)))(select_bool_reg $I128 cond y_reg x_reg)));; Signed minimum of two 128-bit integers on z17.-(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (smin x y)))+(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (smin _ x y)))(vec_smin $I128 x y));; Signed minimum of two vector registers.-(rule (lower (has_type (ty_vec128 ty) (smin x y)))+(rule (lower (has_type (ty_vec128 ty) (smin _ x y)))(vec_smin ty x y));;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Unsigned average of two vector registers.-(rule (lower (has_type (ty_vec128 ty) (avg_round x y)))+(rule (lower (has_type (ty_vec128 ty) (avg_round _ x y)))(vec_uavg ty x y));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Multiply two registers.-(rule 0 (lower (has_type (fits_in_64 ty) (imul x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (imul _ x y)))(mul_reg ty x y));; Multiply a register and a sign-extended register.-(rule 8 (lower (has_type (fits_in_64 ty) (imul x (sext32_value y))))+(rule 8 (lower (has_type (fits_in_64 ty) (imul _ x (sext32_value y))))(mul_reg_sext32 ty x y))-(rule 15 (lower (has_type (fits_in_64 ty) (imul (sext32_value x) y)))+(rule 15 (lower (has_type (fits_in_64 ty) (imul _ (sext32_value x) y)))(mul_reg_sext32 ty y x));; Multiply a register and an immediate.-(rule 7 (lower (has_type (fits_in_64 ty) (imul x (i16_from_value y))))+(rule 7 (lower (has_type (fits_in_64 ty) (imul _ x (i16_from_value y))))(mul_simm16 ty x y))-(rule 14 (lower (has_type (fits_in_64 ty) (imul (i16_from_value x) y)))+(rule 14 (lower (has_type (fits_in_64 ty) (imul _ (i16_from_value x) y)))(mul_simm16 ty y x))-(rule 6 (lower (has_type (fits_in_64 ty) (imul x (i32_from_value y))))+(rule 6 (lower (has_type (fits_in_64 ty) (imul _ x (i32_from_value y))))(mul_simm32 ty x y))-(rule 13 (lower (has_type (fits_in_64 ty) (imul (i32_from_value x) y)))+(rule 13 (lower (has_type (fits_in_64 ty) (imul _ (i32_from_value x) y)))(mul_simm32 ty y x));; Multiply a register and memory (32/64-bit types).-(rule 5 (lower (has_type (fits_in_64 ty) (imul x (sinkable_load_32_64 y))))+(rule 5 (lower (has_type (fits_in_64 ty) (imul _ x (sinkable_load_32_64 y))))(mul_mem ty x (sink_load y)))-(rule 12 (lower (has_type (fits_in_64 ty) (imul (sinkable_load_32_64 x) y)))+(rule 12 (lower (has_type (fits_in_64 ty) (imul _ (sinkable_load_32_64 x) y)))(mul_mem ty y (sink_load x)));; Multiply a register and memory (16-bit types).-(rule 4 (lower (has_type (fits_in_64 ty) (imul x (sinkable_load_16 y))))+(rule 4 (lower (has_type (fits_in_64 ty) (imul _ x (sinkable_load_16 y))))(mul_mem_sext16 ty x (sink_load y)))-(rule 11 (lower (has_type (fits_in_64 ty) (imul (sinkable_load_16 x) y)))+(rule 11 (lower (has_type (fits_in_64 ty) (imul _ (sinkable_load_16 x) y)))(mul_mem_sext16 ty y (sink_load x)));; Multiply a register and sign-extended memory.-(rule 3 (lower (has_type (fits_in_64 ty) (imul x (sinkable_sload16 y))))+(rule 3 (lower (has_type (fits_in_64 ty) (imul _ x (sinkable_sload16 y))))(mul_mem_sext16 ty x (sink_sload16 y)))-(rule 10 (lower (has_type (fits_in_64 ty) (imul (sinkable_sload16 x) y)))+(rule 10 (lower (has_type (fits_in_64 ty) (imul _ (sinkable_sload16 x) y)))(mul_mem_sext16 ty y (sink_sload16 x)))-(rule 2 (lower (has_type (fits_in_64 ty) (imul x (sinkable_sload32 y))))+(rule 2 (lower (has_type (fits_in_64 ty) (imul _ x (sinkable_sload32 y))))(mul_mem_sext32 ty x (sink_sload32 y)))-(rule 9 (lower (has_type (fits_in_64 ty) (imul (sinkable_sload32 x) y)))+(rule 9 (lower (has_type (fits_in_64 ty) (imul _ (sinkable_sload32 x) y)))(mul_mem_sext32 ty y (sink_sload32 x)));; Multiply two vector registers, using a helper.(decl vec_mul_impl (Type Reg Reg) Reg)-(rule 1 (lower (has_type (vr128_ty ty) (imul x y)))+(rule 1 (lower (has_type (vr128_ty ty) (imul _ x y)))(vec_mul_impl ty x y));; Multiply two vector registers - byte, halfword, and word.@@ -453,46 +453,46 @@;; Special-case the lowering of a 128-bit multiply where the operands are sign;; or zero extended. This maps directly to `umul_wide` and `smul_wide`.-(rule 16 (lower (has_type $I128 (imul (uextend x) (uextend y))))+(rule 16 (lower (has_type $I128 (imul _ (uextend _ x) (uextend _ y))))(let ((pair RegPair (umul_wide (put_in_reg_zext64 x) (put_in_reg_zext64 y))))(mov_to_vec128 $I64X2 (regpair_hi pair) (regpair_lo pair))))-(rule 16 (lower (has_type $I128 (imul (sextend x) (sextend y))))+(rule 16 (lower (has_type $I128 (imul _ (sextend _ x) (sextend _ y))))(let ((pair RegPair (smul_wide (put_in_reg_sext64 x) (put_in_reg_sext64 y))))(mov_to_vec128 $I64X2 (regpair_hi pair) (regpair_lo pair))));;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Multiply high part unsigned, 8-bit or 16-bit types. (Uses 32-bit multiply.)-(rule -1 (lower (has_type (ty_8_or_16 ty) (umulhi x y)))+(rule -1 (lower (has_type (ty_8_or_16 ty) (umulhi _ x y)))(let ((ext_reg_x Reg (put_in_reg_zext32 x))(ext_reg_y Reg (put_in_reg_zext32 y))(ext_mul Reg (mul_reg $I32 ext_reg_x ext_reg_y)))(lshr_imm $I32 ext_mul (ty_bits ty))));; Multiply high part unsigned, 32-bit types. (Uses 64-bit multiply.)-(rule (lower (has_type $I32 (umulhi x y)))+(rule (lower (has_type $I32 (umulhi _ x y)))(let ((ext_reg_x Reg (put_in_reg_zext64 x))(ext_reg_y Reg (put_in_reg_zext64 y))(ext_mul Reg (mul_reg $I64 ext_reg_x ext_reg_y)))(lshr_imm $I64 ext_mul 32)));; Multiply high part unsigned, 64-bit types. (Uses umul_wide.)-(rule (lower (has_type $I64 (umulhi x y)))+(rule (lower (has_type $I64 (umulhi _ x y)))(let ((pair RegPair (umul_wide x y)))(regpair_hi pair)));; Multiply high part unsigned, vector types with 8-, 16-, or 32-bit elements.-(rule (lower (has_type $I8X16 (umulhi x y))) (vec_umulhi $I8X16 x y))-(rule (lower (has_type $I16X8 (umulhi x y))) (vec_umulhi $I16X8 x y))-(rule (lower (has_type $I32X4 (umulhi x y))) (vec_umulhi $I32X4 x y))+(rule (lower (has_type $I8X16 (umulhi _ x y))) (vec_umulhi $I8X16 x y))+(rule (lower (has_type $I16X8 (umulhi _ x y))) (vec_umulhi $I16X8 x y))+(rule (lower (has_type $I32X4 (umulhi _ x y))) (vec_umulhi $I32X4 x y));; Multiply high part unsigned, vector types with 64-bit elements on z17.-(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I64X2) (umulhi x y))) (vec_umulhi $I64X2 x y))+(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I64X2) (umulhi _ x y))) (vec_umulhi $I64X2 x y));; Multiply high part unsigned, vector types with 64-bit elements pre-z17.;; Has to be scalarized.-(rule (lower (has_type (and (vxrs_ext3_disabled) $I64X2) (umulhi x y)))+(rule (lower (has_type (and (vxrs_ext3_disabled) $I64X2) (umulhi _ x y)))(let ((pair_0 RegPair (umul_wide (vec_extract_lane $I64X2 x 0 (zero_reg))(vec_extract_lane $I64X2 y 0 (zero_reg))))(res_0 Reg (regpair_hi pair_0))@@ -505,35 +505,35 @@;;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Multiply high part signed, 8-bit or 16-bit types. (Uses 32-bit multiply.)-(rule -1 (lower (has_type (ty_8_or_16 ty) (smulhi x y)))+(rule -1 (lower (has_type (ty_8_or_16 ty) (smulhi _ x y)))(let ((ext_reg_x Reg (put_in_reg_sext32 x))(ext_reg_y Reg (put_in_reg_sext32 y))(ext_mul Reg (mul_reg $I32 ext_reg_x ext_reg_y)))(ashr_imm $I32 ext_mul (ty_bits ty))));; Multiply high part signed, 32-bit types. (Uses 64-bit multiply.)-(rule (lower (has_type $I32 (smulhi x y)))+(rule (lower (has_type $I32 (smulhi _ x y)))(let ((ext_reg_x Reg (put_in_reg_sext64 x))(ext_reg_y Reg (put_in_reg_sext64 y))(ext_mul Reg (mul_reg $I64 ext_reg_x ext_reg_y)))(ashr_imm $I64 ext_mul 32)));; Multiply high part signed, 64-bit types. (Uses smul_wide.)-(rule (lower (has_type $I64 (smulhi x y)))+(rule (lower (has_type $I64 (smulhi _ x y)))(let ((pair RegPair (smul_wide x y)))(regpair_hi pair)));; Multiply high part signed, vector types with 8-, 16-, or 32-bit elements.-(rule (lower (has_type $I8X16 (smulhi x y))) (vec_smulhi $I8X16 x y))-(rule (lower (has_type $I16X8 (smulhi x y))) (vec_smulhi $I16X8 x y))-(rule (lower (has_type $I32X4 (smulhi x y))) (vec_smulhi $I32X4 x y))+(rule (lower (has_type $I8X16 (smulhi _ x y))) (vec_smulhi $I8X16 x y))+(rule (lower (has_type $I16X8 (smulhi _ x y))) (vec_smulhi $I16X8 x y))+(rule (lower (has_type $I32X4 (smulhi _ x y))) (vec_smulhi $I32X4 x y));; Multiply high part signed, vector types with 64-bit elements on z17.-(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I64X2) (smulhi x y))) (vec_smulhi $I64X2 x y))+(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I64X2) (smulhi _ x y))) (vec_smulhi $I64X2 x y));; Multiply high part signed, vector types with 64-bit elements pre-z17.;; Has to be scalarized.-(rule (lower (has_type (and (vxrs_ext3_disabled) $I64X2) (smulhi x y)))+(rule (lower (has_type (and (vxrs_ext3_disabled) $I64X2) (smulhi _ x y)))(let ((pair_0 RegPair (smul_wide (vec_extract_lane $I64X2 x 0 (zero_reg))(vec_extract_lane $I64X2 y 0 (zero_reg))))(res_0 Reg (copy_reg $I64 (regpair_hi pair_0)))@@ -546,7 +546,7 @@;;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Fixed-point multiplication of two vector registers.-(rule (lower (has_type (ty_vec128 ty) (sqmul_round_sat x y)))+(rule (lower (has_type (ty_vec128 ty) (sqmul_round_sat _ x y)))(vec_pack_ssat (vec_widen_type ty)(sqmul_impl (vec_widen_type ty)(vec_unpacks_high ty x)@@ -584,7 +584,7 @@;; and the second variant for 64-bit input types.;; Implement `udiv`.-(rule (lower (has_type (fits_in_64 ty) (udiv x y)))+(rule (lower (has_type (fits_in_64 ty) (udiv _ x y)))(let (;; Look at the divisor to determine whether we need to generate;; an explicit division-by zero check.@@ -603,7 +603,7 @@;; Implement `urem`. Same as `udiv`, but finds the remainder in;; the high half of the result register pair instead.-(rule (lower (has_type (fits_in_64 ty) (urem x y)))+(rule (lower (has_type (fits_in_64 ty) (urem _ x y)))(let ((ext_x RegPair (regpair (imm (ty_ext32 ty) 0)(put_in_reg_zext32 x)))(ext_y Reg (put_in_reg_zext32 y))@@ -612,11 +612,11 @@(regpair_hi pair)));; Implement `udiv` for 128-bit integers on z17 (only).-(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I128) (udiv x y)))+(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I128) (udiv _ x y)))(vec_udiv $I128 x y));; Implement `urem` for 128-bit integers on z17 (only).-(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I128) (urem x y)))+(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I128) (urem _ x y)))(vec_urem $I128 x y))@@ -637,7 +637,7 @@;; and the second variant for 64-bit input types.;; Implement `sdiv`.-(rule (lower (has_type (fits_in_64 ty) (sdiv x y)))+(rule (lower (has_type (fits_in_64 ty) (sdiv _ x y)))(let (;; Look at the divisor to determine whether we need to generate;; explicit division-by-zero and/or integer-overflow checks.@@ -657,7 +657,7 @@;; Implement `srem`. Same as `sdiv`, but finds the remainder in;; the high half of the result register pair instead. Also, handle;; the integer overflow case differently, see below.-(rule (lower (has_type (fits_in_64 ty) (srem x y)))+(rule (lower (has_type (fits_in_64 ty) (srem _ x y)))(let ((OFcheck bool (div_overflow_check_needed y))(ext_x Reg (put_in_reg_sext64 x))(ext_y Reg (put_in_reg_sext32 y))@@ -756,13 +756,13 @@;; Implement `sdiv` for 128-bit integers on z17 (only).-(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I128) (sdiv x y)))+(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I128) (sdiv _ x y)))(let ((OFcheck bool (div_overflow_check_needed y))(_ Reg (maybe_trap_if_sdiv_overflow OFcheck $I128 $I128 x y)))(vec_sdiv $I128 x y)));; Implement `srem` for 128-bit integers on z17 (only).-(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I128) (srem x y)))+(rule 1 (lower (has_type (and (vxrs_ext3_enabled) $I128) (srem _ x y)))(let ((OFcheck bool (div_overflow_check_needed y))(top Reg (maybe_avoid_srem_overflow OFcheck $I128 x y)))(vec_srem $I128 top y)))@@ -771,26 +771,26 @@;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Shift left, shift amount in register.-(rule 0 (lower (has_type (fits_in_64 ty) (ishl x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (ishl _ x y)))(let ((masked_amt Reg (mask_amt_reg ty (amt_reg y))))(lshl_reg ty x masked_amt)));; Shift left, immediate shift amount.-(rule 1 (lower (has_type (fits_in_64 ty) (ishl x (i64_from_value y))))+(rule 1 (lower (has_type (fits_in_64 ty) (ishl _ x (i64_from_value y))))(let ((masked_amt u8 (mask_amt_imm ty y)))(lshl_imm ty x masked_amt)));; Vector shift left, shift amount in register.-(rule 2 (lower (has_type (ty_vec128 ty) (ishl x y)))+(rule 2 (lower (has_type (ty_vec128 ty) (ishl _ x y)))(vec_lshl_reg ty x (amt_reg y)));; Vector shift left, immediate shift amount.-(rule 3 (lower (has_type (ty_vec128 ty) (ishl x (i64_from_value y))))+(rule 3 (lower (has_type (ty_vec128 ty) (ishl _ x (i64_from_value y))))(let ((masked_amt u8 (mask_amt_imm ty y)))(vec_lshl_imm ty x masked_amt)));; 128-bit vector shift left.-(rule 4 (lower (has_type $I128 (ishl x y)))+(rule 4 (lower (has_type $I128 (ishl _ x y)))(let ((amt Reg (amt_vr y)))(vec_lshl_by_bit (vec_lshl_by_byte x amt) amt)))@@ -799,29 +799,29 @@;; Shift right logical, shift amount in register.;; For types smaller than 32-bit, the input value must be zero-extended.-(rule 0 (lower (has_type (fits_in_64 ty) (ushr x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (ushr _ x y)))(let ((ext_reg Reg (put_in_reg_zext32 x))(masked_amt Reg (mask_amt_reg ty (amt_reg y))))(lshr_reg (ty_ext32 ty) ext_reg masked_amt)));; Shift right logical, immediate shift amount.;; For types smaller than 32-bit, the input value must be zero-extended.-(rule 1 (lower (has_type (fits_in_64 ty) (ushr x (i64_from_value y))))+(rule 1 (lower (has_type (fits_in_64 ty) (ushr _ x (i64_from_value y))))(let ((ext_reg Reg (put_in_reg_zext32 x))(masked_amt u8 (mask_amt_imm ty y)))(lshr_imm (ty_ext32 ty) ext_reg masked_amt)));; Vector shift right logical, shift amount in register.-(rule 2 (lower (has_type (ty_vec128 ty) (ushr x y)))+(rule 2 (lower (has_type (ty_vec128 ty) (ushr _ x y)))(vec_lshr_reg ty x (amt_reg y)));; Vector shift right logical, immediate shift amount.-(rule 3 (lower (has_type (ty_vec128 ty) (ushr x (i64_from_value y))))+(rule 3 (lower (has_type (ty_vec128 ty) (ushr _ x (i64_from_value y))))(let ((masked_amt u8 (mask_amt_imm ty y)))(vec_lshr_imm ty x masked_amt)));; 128-bit vector shift right logical.-(rule 4 (lower (has_type $I128 (ushr x y)))+(rule 4 (lower (has_type $I128 (ushr _ x y)))(let ((amt Reg (amt_vr y)))(vec_lshr_by_bit (vec_lshr_by_byte x amt) amt)))@@ -830,29 +830,29 @@;; Shift right arithmetic, shift amount in register.;; For types smaller than 32-bit, the input value must be sign-extended.-(rule 0 (lower (has_type (fits_in_64 ty) (sshr x y)))+(rule 0 (lower (has_type (fits_in_64 ty) (sshr _ x y)))(let ((ext_reg Reg (put_in_reg_sext32 x))(masked_amt Reg (mask_amt_reg ty (amt_reg y))))(ashr_reg (ty_ext32 ty) ext_reg masked_amt)));; Shift right arithmetic, immediate shift amount.;; For types smaller than 32-bit, the input value must be sign-extended.-(rule 1 (lower (has_type (fits_in_64 ty) (sshr x (i64_from_value y))))+(rule 1 (lower (has_type (fits_in_64 ty) (sshr _ x (i64_from_value y))))(let ((ext_reg Reg (put_in_reg_sext32 x))(masked_amt u8 (mask_amt_imm ty y)))(ashr_imm (ty_ext32 ty) ext_reg masked_amt)));; Vector shift right arithmetic, shift amount in register.-(rule 2 (lower (has_type (ty_vec128 ty) (sshr x y)))+(rule 2 (lower (has_type (ty_vec128 ty) (sshr _ x y)))(vec_ashr_reg ty x (amt_reg y)));; Vector shift right arithmetic, immediate shift amount.-(rule 3 (lower (has_type (ty_vec128 ty) (sshr x (i64_from_value y))))+(rule 3 (lower (has_type (ty_vec128 ty) (sshr _ x (i64_from_value y))))(let ((masked_amt u8 (mask_amt_imm ty y)))(vec_ashr_imm ty x masked_amt)));; 128-bit vector shift right arithmetic.-(rule 4 (lower (has_type $I128 (sshr x y)))+(rule 4 (lower (has_type $I128 (sshr _ x y)))(let ((amt Reg (amt_vr y)))(vec_ashr_by_bit (vec_ashr_by_byte x amt) amt)))@@ -860,17 +860,17 @@;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Rotate left, shift amount in register. 32-bit or 64-bit types.-(rule 0 (lower (has_type (ty_32_or_64 ty) (rotl x y)))+(rule 0 (lower (has_type (ty_32_or_64 ty) (rotl _ x y)))(rot_reg ty x (amt_reg y)));; Rotate left arithmetic, immediate shift amount. 32-bit or 64-bit types.-(rule 1 (lower (has_type (ty_32_or_64 ty) (rotl x (i64_from_value y))))+(rule 1 (lower (has_type (ty_32_or_64 ty) (rotl _ x (i64_from_value y))))(let ((masked_amt u8 (mask_amt_imm ty y)))(rot_imm ty x masked_amt)));; Rotate left, shift amount in register. 8-bit or 16-bit types.;; Implemented via a pair of 32-bit shifts on the zero-extended input.-(rule 2 (lower (has_type (ty_8_or_16 ty) (rotl x y)))+(rule 2 (lower (has_type (ty_8_or_16 ty) (rotl _ x y)))(let ((ext_reg Reg (put_in_reg_zext32 x))(ext_ty Type (ty_ext32 ty))(pos_amt Reg (amt_reg y))@@ -882,7 +882,7 @@;; Rotate left, immediate shift amount. 8-bit or 16-bit types.;; Implemented via a pair of 32-bit shifts on the zero-extended input.-(rule 3 (lower (has_type (ty_8_or_16 ty) (rotl x (and (i64_from_value pos_amt)+(rule 3 (lower (has_type (ty_8_or_16 ty) (rotl _ x (and (i64_from_value pos_amt)(i64_from_negated_value neg_amt)))))(let ((ext_reg Reg (put_in_reg_zext32 x))(ext_ty Type (ty_ext32 ty))@@ -892,17 +892,17 @@(lshr_imm ext_ty ext_reg masked_neg_amt))));; Vector rotate left, shift amount in register.-(rule 4 (lower (has_type (ty_vec128 ty) (rotl x y)))+(rule 4 (lower (has_type (ty_vec128 ty) (rotl _ x y)))(vec_rot_reg ty x (amt_reg y)));; Vector rotate left, immediate shift amount.-(rule 5 (lower (has_type (ty_vec128 ty) (rotl x (i64_from_value y))))+(rule 5 (lower (has_type (ty_vec128 ty) (rotl _ x (i64_from_value y))))(let ((masked_amt u8 (mask_amt_imm ty y)))(vec_rot_imm ty x masked_amt)));; 128-bit full vector rotate left.;; Implemented via a pair of 128-bit full vector shifts.-(rule 6 (lower (has_type $I128 (rotl x y)))+(rule 6 (lower (has_type $I128 (rotl _ x y)))(let ((x_reg Reg x)(pos_amt Reg (amt_vr y))(neg_amt Reg (vec_neg $I8X16 pos_amt)))@@ -915,19 +915,19 @@;; Rotate right, shift amount in register. 32-bit or 64-bit types.;; Implemented as rotate left with negated rotate amount.-(rule 0 (lower (has_type (ty_32_or_64 ty) (rotr x y)))+(rule 0 (lower (has_type (ty_32_or_64 ty) (rotr _ x y)))(let ((negated_amt Reg (neg_reg $I32 (amt_reg y))))(rot_reg ty x negated_amt)));; Rotate right arithmetic, immediate shift amount. 32-bit or 64-bit types.;; Implemented as rotate left with negated rotate amount.-(rule 1 (lower (has_type (ty_32_or_64 ty) (rotr x (i64_from_negated_value y))))+(rule 1 (lower (has_type (ty_32_or_64 ty) (rotr _ x (i64_from_negated_value y))))(let ((negated_amt u8 (mask_amt_imm ty y)))(rot_imm ty x negated_amt)));; Rotate right, shift amount in register. 8-bit or 16-bit types.;; Implemented as rotate left with negated rotate amount.-(rule 2 (lower (has_type (ty_8_or_16 ty) (rotr x y)))+(rule 2 (lower (has_type (ty_8_or_16 ty) (rotr _ x y)))(let ((ext_reg Reg (put_in_reg_zext32 x))(ext_ty Type (ty_ext32 ty))(pos_amt Reg (amt_reg y))@@ -939,7 +939,7 @@;; Rotate right, immediate shift amount. 8-bit or 16-bit types.;; Implemented as rotate left with negated rotate amount.-(rule 3 (lower (has_type (ty_8_or_16 ty) (rotr x (and (i64_from_value pos_amt)+(rule 3 (lower (has_type (ty_8_or_16 ty) (rotr _ x (and (i64_from_value pos_amt)(i64_from_negated_value neg_amt)))))(let ((ext_reg Reg (put_in_reg_zext32 x))(ext_ty Type (ty_ext32 ty))@@ -950,19 +950,19 @@;; Vector rotate right, shift amount in register.;; Implemented as rotate left with negated rotate amount.-(rule 4 (lower (has_type (ty_vec128 ty) (rotr x y)))+(rule 4 (lower (has_type (ty_vec128 ty) (rotr _ x y)))(let ((negated_amt Reg (neg_reg $I32 (amt_reg y))))(vec_rot_reg ty x negated_amt)));; Vector rotate right, immediate shift amount.;; Implemented as rotate left with negated rotate amount.-(rule 5 (lower (has_type (ty_vec128 ty) (rotr x (i64_from_negated_value y))))+(rule 5 (lower (has_type (ty_vec128 ty) (rotr _ x (i64_from_negated_value y))))(let ((negated_amt u8 (mask_amt_imm ty y)))(vec_rot_imm ty x negated_amt)));; 128-bit full vector rotate right.;; Implemented via a pair of 128-bit full vector shifts.-(rule 6 (lower (has_type $I128 (rotr x y)))+(rule 6 (lower (has_type $I128 (rotr _ x y)))(let ((x_reg Reg x)(pos_amt Reg (amt_vr y))(neg_amt Reg (vec_neg $I8X16 pos_amt)))@@ -974,156 +974,156 @@;;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Up to 64-bit source type: Always a no-op.-(rule 1 (lower (ireduce x @ (value_type (fits_in_64 _ty))))+(rule 1 (lower (ireduce _ x @ (value_type (fits_in_64 _ty))))x);; 128-bit source type: Extract the low half.-(rule (lower (ireduce x @ (value_type (vr128_ty _ty))))+(rule (lower (ireduce _ x @ (value_type (vr128_ty _ty))))(vec_extract_lane $I64X2 x 1 (zero_reg)));;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16- or 32-bit target types.-(rule 1 (lower (has_type (gpr32_ty _ty) (uextend x)))+(rule 1 (lower (has_type (gpr32_ty _ty) (uextend _ x)))(put_in_reg_zext32 x));; 64-bit target types.-(rule 2 (lower (has_type (gpr64_ty _ty) (uextend x)))+(rule 2 (lower (has_type (gpr64_ty _ty) (uextend _ x)))(put_in_reg_zext64 x));; 128-bit target types.-(rule (lower (has_type $I128 (uextend x @ (value_type $I8))))+(rule (lower (has_type $I128 (uextend _ x @ (value_type $I8))))(vec_insert_lane $I8X16 (vec_imm $I128 0) x 15 (zero_reg)))-(rule (lower (has_type $I128 (uextend x @ (value_type $I16))))+(rule (lower (has_type $I128 (uextend _ x @ (value_type $I16))))(vec_insert_lane $I16X8 (vec_imm $I128 0) x 7 (zero_reg)))-(rule (lower (has_type $I128 (uextend x @ (value_type $I32))))+(rule (lower (has_type $I128 (uextend _ x @ (value_type $I32))))(vec_insert_lane $I32X4 (vec_imm $I128 0) x 3 (zero_reg)))-(rule (lower (has_type $I128 (uextend x @ (value_type $I64))))+(rule (lower (has_type $I128 (uextend _ x @ (value_type $I64))))(vec_insert_lane $I64X2 (vec_imm $I128 0) x 1 (zero_reg)));;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16- or 32-bit target types.-(rule 1 (lower (has_type (gpr32_ty _ty) (sextend x)))+(rule 1 (lower (has_type (gpr32_ty _ty) (sextend _ x)))(put_in_reg_sext32 x));; 64-bit target types.-(rule 2 (lower (has_type (gpr64_ty _ty) (sextend x)))+(rule 2 (lower (has_type (gpr64_ty _ty) (sextend _ x)))(put_in_reg_sext64 x));; 128-bit target types.-(rule (lower (has_type $I128 (sextend x)))+(rule (lower (has_type $I128 (sextend _ x)))(let ((x_ext Reg (put_in_reg_sext64 x)))(mov_to_vec128 $I128 (ashr_imm $I64 x_ext 63) x_ext)));;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (snarrow x @ (value_type (ty_vec128 ty)) y))+(rule (lower (snarrow _ x @ (value_type (ty_vec128 ty)) y))(vec_pack_ssat_lane_order ty x y));;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (uunarrow x @ (value_type (ty_vec128 ty)) y))+(rule (lower (uunarrow _ x @ (value_type (ty_vec128 ty)) y))(vec_pack_usat_lane_order ty x y));;;; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (unarrow x @ (value_type (ty_vec128 ty)) y))+(rule (lower (unarrow _ x @ (value_type (ty_vec128 ty)) y))(let ((zero Reg (vec_imm ty 0)))(vec_pack_usat_lane_order ty (vec_smax ty x zero) (vec_smax ty y zero))));;;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (swiden_low x @ (value_type (ty_vec128 ty))))+(rule (lower (swiden_low _ x @ (value_type (ty_vec128 ty))))(vec_unpacks_low_lane_order ty x));;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (swiden_high x @ (value_type (ty_vec128 ty))))+(rule (lower (swiden_high _ x @ (value_type (ty_vec128 ty))))(vec_unpacks_high_lane_order ty x));;;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (uwiden_low x @ (value_type (ty_vec128 ty))))+(rule (lower (uwiden_low _ x @ (value_type (ty_vec128 ty))))(vec_unpacku_low_lane_order ty x));;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (uwiden_high x @ (value_type (ty_vec128 ty))))+(rule (lower (uwiden_high _ x @ (value_type (ty_vec128 ty))))(vec_unpacku_high_lane_order ty x));;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; z15 version using a single instruction (NOR).-(rule 2 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bnot x)))+(rule 2 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bnot _ x)))(let ((rx Reg x))(not_or_reg ty rx rx)));; z14 version using XOR with -1.-(rule 1 (lower (has_type (and (mie3_disabled) (fits_in_64 ty)) (bnot x)))+(rule 1 (lower (has_type (and (mie3_disabled) (fits_in_64 ty)) (bnot _ x)))(not_reg ty x));; Vector version using vector NOR.-(rule (lower (has_type (vr128_ty ty) (bnot x)))+(rule (lower (has_type (vr128_ty ty) (bnot _ x)))(vec_not ty x));; Float version using vector NOR.-(rule 5 (lower (has_type (ty_scalar_float _) (bnot x)))+(rule 5 (lower (has_type (ty_scalar_float _) (bnot _ x)))(vec_not $F64X2 x));; With z15 (bnot (bxor ...)) can be a single instruction, similar to the;; (bxor _ (bnot _)) lowering.-(rule 3 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bnot (bxor x y))))+(rule 3 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bnot _ (bxor _ x y))))(not_xor_reg ty x y));; Combine a not/xor operation of vector types into one.-(rule 4 (lower (has_type (vr128_ty ty) (bnot (bxor x y))))+(rule 4 (lower (has_type (vr128_ty ty) (bnot _ (bxor _ x y))))(vec_not_xor ty x y));;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; And two registers.-(rule -1 (lower (has_type (fits_in_64 ty) (band x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (band _ x y)))(and_reg ty x y));; And a register and an immediate.-(rule 5 (lower (has_type (fits_in_64 ty) (band x (uimm16shifted_from_inverted_value y))))+(rule 5 (lower (has_type (fits_in_64 ty) (band _ x (uimm16shifted_from_inverted_value y))))(and_uimm16shifted ty x y))-(rule 6 (lower (has_type (fits_in_64 ty) (band (uimm16shifted_from_inverted_value x) y)))+(rule 6 (lower (has_type (fits_in_64 ty) (band _ (uimm16shifted_from_inverted_value x) y)))(and_uimm16shifted ty y x))-(rule 3 (lower (has_type (fits_in_64 ty) (band x (uimm32shifted_from_inverted_value y))))+(rule 3 (lower (has_type (fits_in_64 ty) (band _ x (uimm32shifted_from_inverted_value y))))(and_uimm32shifted ty x y))-(rule 4 (lower (has_type (fits_in_64 ty) (band (uimm32shifted_from_inverted_value x) y)))+(rule 4 (lower (has_type (fits_in_64 ty) (band _ (uimm32shifted_from_inverted_value x) y)))(and_uimm32shifted ty y x));; And a register and memory (32/64-bit types).-(rule 1 (lower (has_type (fits_in_64 ty) (band x (sinkable_load_32_64 y))))+(rule 1 (lower (has_type (fits_in_64 ty) (band _ x (sinkable_load_32_64 y))))(and_mem ty x (sink_load y)))-(rule 2 (lower (has_type (fits_in_64 ty) (band (sinkable_load_32_64 x) y)))+(rule 2 (lower (has_type (fits_in_64 ty) (band _ (sinkable_load_32_64 x) y)))(and_mem ty y (sink_load x)));; And two vector registers.-(rule 0 (lower (has_type (vr128_ty ty) (band x y)))+(rule 0 (lower (has_type (vr128_ty ty) (band _ x y)))(vec_and ty x y));; And two float registers, using vector overlay.-(rule 11 (lower (has_type (ty_scalar_float _) (band x y)))+(rule 11 (lower (has_type (ty_scalar_float _) (band _ x y)))(vec_and $F64X2 x y))-(rule 12 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (band x y) z)))+(rule 12 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (band _ x y) z)))(vec_eval ty 0b00000001 x y z))-(rule 13 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band x (band y z))))+(rule 13 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (band _ y z))))(vec_eval ty 0b00000001 x y z));; Specialized lowerings for `(band x (bnot y))` which is additionally produced@@ -1131,101 +1131,101 @@;; forms early on.;; z15 version using a single instruction.-(rule 7 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (band x (bnot y))))+(rule 7 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (band _ x (bnot _ y))))(and_not_reg ty x y))-(rule 8 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (band (bnot y) x)))+(rule 8 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (band _ (bnot _ y) x)))(and_not_reg ty x y));; And-not two vector registers.-(rule 9 (lower (has_type (vr128_ty ty) (band x (bnot y))))+(rule 9 (lower (has_type (vr128_ty ty) (band _ x (bnot _ y))))(vec_and_not ty x y))-(rule 10 (lower (has_type (vr128_ty ty) (band (bnot y) x)))+(rule 10 (lower (has_type (vr128_ty ty) (band _ (bnot _ y) x)))(vec_and_not ty x y));; And-not three vector registers.-(rule 14 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (band (bnot x) y) z)))+(rule 14 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (band _ (bnot _ x) y) z)))(vec_eval ty 0b00010000 x y z))-(rule 15 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (bnot x) (band y z))))+(rule 15 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (bnot _ x) (band _ y z))))(vec_eval ty 0b00010000 x y z))-(rule 16 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (band x y) (bnot z))))+(rule 16 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (band _ x y) (bnot _ z))))(vec_eval ty 0b00000010 x y z))-(rule 17 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band x (band y (bnot z)))))+(rule 17 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (band _ y (bnot _ z)))))(vec_eval ty 0b00000010 x y z))-(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (band x (bnot y)) z)))+(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (band _ x (bnot _ y)) z)))(vec_eval ty 0b00000100 y x z))-(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band x (band (bnot y) z))))+(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (band _ (bnot _ y) z))))(vec_eval ty 0b00000100 z x y));; Not-and three vector registers-(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (band (band x y) z))))+(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (band _ (band _ x y) z))))(vec_eval ty 0b11111110 x y z))-(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (band x (band y z)))))+(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (band _ x (band _ y z)))))(vec_eval ty 0b11111110 x y z));; And-Nand three vector registers-(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (bnot (band x y)) z)))+(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (bnot _ (band _ x y)) z)))(vec_eval ty 0b01010100 x y z))-(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band x (bnot (band y z)))))+(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (bnot _ (band _ y z)))))(vec_eval ty 0b00001110 x y z));; And-Or 3 vector registers-(rule 22 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band x (bor y z))))+(rule 22 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (bor _ y z))))(vec_eval ty 0b00000111 x y z))-(rule 23 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (bor x y) z)))+(rule 23 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (bor _ x y) z)))(vec_eval ty 0b00010101 x y z));; And-Nor 3 vector registers-(rule 24 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band x (bnot (bor y z)))))+(rule 24 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (bnot _ (bor _ y z)))))(vec_eval ty 0b00001000 x y z))-(rule 25 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (bnot (bor x y)) z)))+(rule 25 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (bnot _ (bor _ x y)) z)))(vec_eval ty 0b01000000 x y z));; And-Xor 3 vector registers-(rule 26 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band x (bxor y z))))+(rule 26 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (bxor _ y z))))(vec_eval ty 0b00000110 x y z))-(rule 27 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (bxor x y) z)))+(rule 27 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (bxor _ x y) z)))(vec_eval ty 0b00010100 x y z));; And-Nxor 3 vector registers-(rule 28 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band x (bnot (bxor y z)))))+(rule 28 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (bnot _ (bxor _ y z)))))(vec_eval ty 0b00001001 x y z))-(rule 29 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band (bnot (bxor x y)) z)))+(rule 29 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (bnot _ (bxor _ x y)) z)))(vec_eval ty 0b01000001 x y z));;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Or two registers.-(rule -1 (lower (has_type (fits_in_64 ty) (bor x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (bor _ x y)))(or_reg ty x y));; Or a register and an immediate.-(rule 5 (lower (has_type (fits_in_64 ty) (bor x (uimm16shifted_from_value y))))+(rule 5 (lower (has_type (fits_in_64 ty) (bor _ x (uimm16shifted_from_value y))))(or_uimm16shifted ty x y))-(rule 6 (lower (has_type (fits_in_64 ty) (bor (uimm16shifted_from_value x) y)))+(rule 6 (lower (has_type (fits_in_64 ty) (bor _ (uimm16shifted_from_value x) y)))(or_uimm16shifted ty y x))-(rule 3 (lower (has_type (fits_in_64 ty) (bor x (uimm32shifted_from_value y))))+(rule 3 (lower (has_type (fits_in_64 ty) (bor _ x (uimm32shifted_from_value y))))(or_uimm32shifted ty x y))-(rule 4 (lower (has_type (fits_in_64 ty) (bor (uimm32shifted_from_value x) y)))+(rule 4 (lower (has_type (fits_in_64 ty) (bor _ (uimm32shifted_from_value x) y)))(or_uimm32shifted ty y x));; Or a register and memory (32/64-bit types).-(rule 1 (lower (has_type (fits_in_64 ty) (bor x (sinkable_load_32_64 y))))+(rule 1 (lower (has_type (fits_in_64 ty) (bor _ x (sinkable_load_32_64 y))))(or_mem ty x (sink_load y)))-(rule 2 (lower (has_type (fits_in_64 ty) (bor (sinkable_load_32_64 x) y)))+(rule 2 (lower (has_type (fits_in_64 ty) (bor _ (sinkable_load_32_64 x) y)))(or_mem ty y (sink_load x)));; Or two vector registers.-(rule 0 (lower (has_type (vr128_ty ty) (bor x y)))+(rule 0 (lower (has_type (vr128_ty ty) (bor _ x y)))(vec_or ty x y));; Or two floating registers, using vector overlay-(rule 11 (lower (has_type (ty_scalar_float _) (bor x y)))+(rule 11 (lower (has_type (ty_scalar_float _) (bor _ x y)))(vec_or $F64X2 x y));; Or three vector registers.-(rule 12 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bor x y) z)))+(rule 12 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bor _ x y) z)))(vec_eval ty 0b01111111 x y z))-(rule 13 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor x (bor y z))))+(rule 13 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bor _ y z))))(vec_eval ty 0b01111111 x y z));; Specialized lowerings for `(bor x (bnot y))` which is additionally produced@@ -1233,115 +1233,115 @@;; forms early on.;; z15 version using a single instruction.-(rule 7 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bor x (bnot y))))+(rule 7 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bor _ x (bnot _ y))))(or_not_reg ty x y))-(rule 8 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bor (bnot y) x)))+(rule 8 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bor _ (bnot _ y) x)))(or_not_reg ty x y));; Or-not two vector registers.-(rule 9 (lower (has_type (vr128_ty ty) (bor x (bnot y))))+(rule 9 (lower (has_type (vr128_ty ty) (bor _ x (bnot _ y))))(vec_or_not ty x y))-(rule 10 (lower (has_type (vr128_ty ty) (bor (bnot y) x)))+(rule 10 (lower (has_type (vr128_ty ty) (bor _ (bnot _ y) x)))(vec_or_not ty x y));; 3-input bor with a single not-(rule 14 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bor (bnot x) y) z)))+(rule 14 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bor _ (bnot _ x) y) z)))(vec_eval ty 0b11110111 x y z))-(rule 15 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bnot x) (bor y z))))+(rule 15 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bnot _ x) (bor _ y z))))(vec_eval ty 0b11110111 x y z))-(rule 16 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bor x (bnot y)) z)))+(rule 16 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bor _ x (bnot _ y)) z)))(vec_eval ty 0b11011111 x y z))-(rule 17 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor x (bor (bnot y) z))))+(rule 17 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bor _ (bnot _ y) z))))(vec_eval ty 0b11011111 x y z))-(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bor x y) (bnot z))))+(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bor _ x y) (bnot _ z))))(vec_eval ty 0b10111111 x y z))-(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor x (bor y (bnot z)))))+(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bor _ y (bnot _ z)))))(vec_eval ty 0b10111111 x y z));; 3-input bnor-(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor (bor x y) z))))+(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ (bor _ x y) z))))(vec_eval ty 0b10000000 x y z))-(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor x (bor y z)))))+(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ x (bor _ y z)))))(vec_eval ty 0b10000000 x y z));; Or-Nor-(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bnot (bor x y)) z)))+(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bnot _ (bor _ x y)) z)))(vec_eval ty 0b11010101 x y z))-(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor x (bnot (bor y z)))))+(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bnot _ (bor _ y z)))))(vec_eval ty 0b10001111 x y z));; Or-And-(rule 22 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (band x y) z)))+(rule 22 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (band _ x y) z)))(vec_eval ty 0b01010111 x y z))-(rule 23 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor x (band y z))))+(rule 23 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (band _ y z))))(vec_eval ty 0b00011111 x y z));; Or-Nand-(rule 24 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bnot (band x y)) z)))+(rule 24 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bnot _ (band _ x y)) z)))(vec_eval ty 0b11111101 x y z))-(rule 25 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor x (bnot (band y z)))))+(rule 25 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bnot _ (band _ y z)))))(vec_eval ty 0b11101111 x y z));; Or-Xor-(rule 26 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bxor x y) z)))+(rule 26 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bxor _ x y) z)))(vec_eval ty 0b01111101 x y z))-(rule 27 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor x (bxor y z))))+(rule 27 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bxor _ y z))))(vec_eval ty 0b01101111 x y z));; Or-Nxor-(rule 28 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor (bnot (bxor x y)) z)))+(rule 28 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bnot _ (bxor _ x y)) z)))(vec_eval ty 0b11010111 x y z))-(rule 29 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor x (bnot (bxor y z)))))+(rule 29 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bnot _ (bxor _ y z)))))(vec_eval ty 0b10011111 x y z));; Nor-And-(rule 30 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor (band x y) z))))+(rule 30 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ (band _ x y) z))))(vec_eval ty 0b10101000 x y z))-(rule 31 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor x (band y z)))))+(rule 31 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ x (band _ y z)))))(vec_eval ty 0b11100000 x y z));; Nor-Nand-(rule 32 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor (bnot (band x y)) z))))+(rule 32 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ (bnot _ (band _ x y)) z))))(vec_eval ty 0b00000010 x y z))-(rule 33 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor x (bnot (band y z))))))+(rule 33 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ x (bnot _ (band _ y z))))))(vec_eval ty 0b00010000 x y z));; Nor-Xor-(rule 34 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor (bxor x y) z))))+(rule 34 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ (bxor _ x y) z))))(vec_eval ty 0b10000010 x y z))-(rule 35 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor x (bxor y z)))))+(rule 35 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ x (bxor _ y z)))))(vec_eval ty 0b10010000 x y z));; Nor-Nxor-(rule 36 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor (bnot (bxor x y)) z))))+(rule 36 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ (bnot _ (bxor _ x y)) z))))(vec_eval ty 0b00101000 x y z))-(rule 37 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bor x (bnot (bxor y z))))))+(rule 37 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ x (bnot _ (bxor _ y z))))))(vec_eval ty 0b01100000 x y z));;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Xor two registers.-(rule -1 (lower (has_type (fits_in_64 ty) (bxor x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (bxor _ x y)))(xor_reg ty x y));; Xor a register and an immediate.-(rule 3 (lower (has_type (fits_in_64 ty) (bxor x (uimm32shifted_from_value y))))+(rule 3 (lower (has_type (fits_in_64 ty) (bxor _ x (uimm32shifted_from_value y))))(xor_uimm32shifted ty x y))-(rule 4 (lower (has_type (fits_in_64 ty) (bxor (uimm32shifted_from_value x) y)))+(rule 4 (lower (has_type (fits_in_64 ty) (bxor _ (uimm32shifted_from_value x) y)))(xor_uimm32shifted ty y x));; Xor a register and memory (32/64-bit types).-(rule 1 (lower (has_type (fits_in_64 ty) (bxor x (sinkable_load_32_64 y))))+(rule 1 (lower (has_type (fits_in_64 ty) (bxor _ x (sinkable_load_32_64 y))))(xor_mem ty x (sink_load y)))-(rule 2 (lower (has_type (fits_in_64 ty) (bxor (sinkable_load_32_64 x) y)))+(rule 2 (lower (has_type (fits_in_64 ty) (bxor _ (sinkable_load_32_64 x) y)))(xor_mem ty y (sink_load x)));; Xor two vector registers.-(rule 0 (lower (has_type (vr128_ty ty) (bxor x y)))+(rule 0 (lower (has_type (vr128_ty ty) (bxor _ x y)))(vec_xor ty x y));; Xor two floating registers, using vector overlay-(rule 9 (lower (has_type (ty_scalar_float _) (bxor x y)))+(rule 9 (lower (has_type (ty_scalar_float _) (bxor _ x y)))(vec_xor $F64X2 x y));; Specialized lowerings for `(bxor x (bnot y))` which is additionally produced@@ -1349,134 +1349,134 @@;; forms early on.;; z15 version using a single instruction.-(rule 5 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bxor x (bnot y))))+(rule 5 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bxor _ x (bnot _ y))))(not_xor_reg ty x y))-(rule 6 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bxor (bnot y) x)))+(rule 6 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bxor _ (bnot _ y) x)))(not_xor_reg ty x y));; Xor-not two vector registers.-(rule 7 (lower (has_type (vr128_ty ty) (bxor x (bnot y))))+(rule 7 (lower (has_type (vr128_ty ty) (bxor _ x (bnot _ y))))(vec_not_xor ty x y))-(rule 8 (lower (has_type (vr128_ty ty) (bxor (bnot y) x)))+(rule 8 (lower (has_type (vr128_ty ty) (bxor _ (bnot _ y) x)))(vec_not_xor ty x y));; 3-input Xor-(rule 10 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor (bxor x y) z)))+(rule 10 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bxor _ x y) z)))(vec_eval ty 0b01101001 x y z))-(rule 11 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor x (bxor y z))))+(rule 11 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bxor _ y z))))(vec_eval ty 0b01101001 x y z));; Xor-And-(rule 12 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor (band x y) z)))+(rule 12 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (band _ x y) z)))(vec_eval ty 0b01010110 x y z))-(rule 13 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor x (band y z))))+(rule 13 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (band _ y z))))(vec_eval ty 0b00011110 x y z));; Xor-Nand-(rule 14 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor (bnot (band x y)) z)))+(rule 14 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bnot _ (band _ x y)) z)))(vec_eval ty 0b10101001 x y z))-(rule 15 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor x (bnot (band y z)))))+(rule 15 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bnot _ (band _ y z)))))(vec_eval ty 0b11100001 x y z));; Xor-Or-(rule 16 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor (bor x y) z)))+(rule 16 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bor _ x y) z)))(vec_eval ty 0b01101010 x y z))-(rule 17 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor x (bor y z))))+(rule 17 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bor _ y z))))(vec_eval ty 0b01111000 x y z));; Xor-Nor-(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor (bnot (bor x y)) z)))+(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bnot _ (bor _ x y)) z)))(vec_eval ty 0b10010101 x y z))-(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor x (bnot (bor y z)))))+(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bnot _ (bor _ y z)))))(vec_eval ty 0b10000111 x y z));; Xor-Nxor-(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor (bnot (bxor x y)) z)))+(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bnot _ (bxor _ x y)) z)))(vec_eval ty 0b10010110 x y z))-(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor x (bnot (bxor y z)))))+(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bnot _ (bxor _ y z)))))(vec_eval ty 0b10010110 x y z));; 3-input Nxor-(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor (bxor x y) z))))+(rule 20 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ (bxor _ x y) z))))(vec_eval ty 0b10010110 x y z))-(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor x (bxor y z)))))+(rule 21 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x (bxor _ y z)))))(vec_eval ty 0b10010110 x y z));; Nxor-And-(rule 22 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor (band x y) z))))+(rule 22 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ (band _ x y) z))))(vec_eval ty 0b10101001 x y z))-(rule 23 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor x (band y z)))))+(rule 23 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x (band _ y z)))))(vec_eval ty 0b11100001 x y z));; Nxor-Nand-(rule 24 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor (bnot (band x y)) z))))+(rule 24 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ (bnot _ (band _ x y)) z))))(vec_eval ty 0b01010110 x y z))-(rule 25 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor x (bnot (band y z))))))+(rule 25 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x (bnot _ (band _ y z))))))(vec_eval ty 0b00011110 x y z));; Nxor-Or-(rule 26 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor (bor x y) z))))+(rule 26 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ (bor _ x y) z))))(vec_eval ty 0b10010101 x y z))-(rule 27 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor x (bor y z)))))+(rule 27 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x (bor _ y z)))))(vec_eval ty 0b10000111 x y z));; Nxor-Nor-(rule 28 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor (bnot (bor x y)) z))))+(rule 28 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ (bnot _ (bor _ x y)) z))))(vec_eval ty 0b01101010 x y z))-(rule 29 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor x (bnot (bor y z))))))+(rule 29 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x (bnot _ (bor _ y z))))))(vec_eval ty 0b01111000 x y z));; Xor-Nxor-(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor (bnot (bxor x y)) z))))+(rule 18 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ (bnot _ (bxor _ x y)) z))))(vec_eval ty 0b01101001 x y z))-(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot (bxor x (bnot (bxor y z))))))+(rule 19 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x (bnot _ (bxor _ y z))))))(vec_eval ty 0b01101001 x y z));;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; z15 version using a NAND instruction.-(rule 2 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bitselect x y z)))+(rule 2 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (bitselect _ x y z)))(let ((rx Reg x)(if_true Reg (and_reg ty y rx))(if_false Reg (and_not_reg ty z rx)))(or_reg ty if_false if_true)));; z14 version using XOR with -1.-(rule 1 (lower (has_type (and (mie3_disabled) (fits_in_64 ty)) (bitselect x y z)))+(rule 1 (lower (has_type (and (mie3_disabled) (fits_in_64 ty)) (bitselect _ x y z)))(let ((rx Reg x)(if_true Reg (and_reg ty y rx))(if_false Reg (and_reg ty z (not_reg ty rx))))(or_reg ty if_false if_true)));; Bitselect vector registers.-(rule (lower (has_type (vr128_ty ty) (bitselect x y z)))+(rule (lower (has_type (vr128_ty ty) (bitselect _ x y z)))(vec_select ty y z x));; Bitselect-not vector registers.-(rule 5 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bitselect x (bnot y) z)))+(rule 5 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bitselect _ x (bnot _ y) z)))(vec_eval ty 0b01011100 x y z))-(rule 6 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bitselect x y (bnot z))))+(rule 6 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bitselect _ x y (bnot _ z))))(vec_eval ty 0b10100011 x y z))-(rule 7 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bitselect x (bnot y) (bnot z))))+(rule 7 (lower (has_type (and (vxrs_ext3_enabled) (vr128_ty ty)) (bitselect _ x (bnot _ y) (bnot _ z))))(vec_eval ty 0b10101100 x y z));; Special-case some float-selection instructions for min/max-(rule 3 (lower (has_type (ty_vec128 ty) (bitselect (bitcast _ (fcmp (FloatCC.LessThan) x y)) x y)))+(rule 3 (lower (has_type (ty_vec128 ty) (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) x y)) x y)))(fmin_pseudo_reg ty y x))-(rule 4 (lower (has_type (ty_vec128 ty) (bitselect (bitcast _ (fcmp (FloatCC.LessThan) y x)) x y)))+(rule 4 (lower (has_type (ty_vec128 ty) (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) y x)) x y)))(fmax_pseudo_reg ty y x));;;; Rules for `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (bmask x)))+(rule (lower (has_type ty (bmask _ x)))(lower_bool_to_mask ty (value_nonzero x)));;;; Rules for `bitrev` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (bitrev x)))+(rule (lower (has_type ty (bitrev _ x)))(bitrev_bytes ty(bitrev_bits 4 0xf0f0_f0f0_f0f0_f0f0 ty(bitrev_bits 2 0xcccc_cccc_cccc_cccc ty@@ -1511,7 +1511,7 @@;;;; Rules for `bswap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (bswap x)))+(rule (lower (has_type ty (bswap _ x)))(bitrev_bytes ty x));;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1527,7 +1527,7 @@;; Count leading zeros pre-z17, via FLOGR on an input zero-extended to 64 bits,;; with the result compensated for the extra bits.-(rule 1 (lower (has_type (and (mie4_disabled) (fits_in_64 ty)) (clz x)))+(rule 1 (lower (has_type (and (mie4_disabled) (fits_in_64 ty)) (clz _ x)))(let ((ext_reg Reg (put_in_reg_zext64 x));; Ask for a value of 64 in the all-zero 64-bit input case.;; After compensation this will match the expected semantics.@@ -1535,7 +1535,7 @@(clz_offset ty clz)));; Count leading zeros, 128-bit full vector pre-z17.-(rule (lower (has_type (and (vxrs_ext3_disabled) $I128) (clz x)))+(rule (lower (has_type (and (vxrs_ext3_disabled) $I128) (clz _ x)))(let ((clz_vec Reg (vec_clz $I64X2 x))(zero Reg (vec_imm $I64X2 0))(clz_hi Reg (vec_permute_dw_imm $I64X2 zero 0 clz_vec 0))@@ -1546,13 +1546,13 @@;; Count leading zeros on z17, via CLZG on an input zero-extended to 64 bits,;; with the result compensated for the extra bits.-(rule 3 (lower (has_type (and (mie4_enabled) (fits_in_64 ty)) (clz x)))+(rule 3 (lower (has_type (and (mie4_enabled) (fits_in_64 ty)) (clz _ x)))(let ((ext_reg Reg (put_in_reg_zext64 x))(clz Reg (clz_reg ext_reg)))(clz_offset ty clz)));; Count leading zeros, 128-bit full vector on z17.-(rule 2 (lower (has_type (and (vxrs_ext3_enabled) $I128) (clz x)))+(rule 2 (lower (has_type (and (vxrs_ext3_enabled) $I128) (clz _ x)))(vec_clz $I128 x));;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1571,7 +1571,7 @@;; i.e. computing;; cls(x) == clz(x ^ (x >> 63)) - 1;; where x is the sign-extended input.-(rule 1 (lower (has_type (and (mie4_disabled) (fits_in_64 ty)) (cls x)))+(rule 1 (lower (has_type (and (mie4_disabled) (fits_in_64 ty)) (cls _ x)))(let ((ext_reg Reg (put_in_reg_sext64 x))(signbit_copies Reg (ashr_imm $I64 ext_reg 63))(inv_reg Reg (xor_reg $I64 ext_reg signbit_copies))@@ -1579,7 +1579,7 @@(cls_offset ty clz)));; Count leading sign-bit copies, 128-bit full vector pre-z17.-(rule (lower (has_type (and (vxrs_ext3_disabled) $I128) (cls x)))+(rule (lower (has_type (and (vxrs_ext3_disabled) $I128) (cls _ x)))(let ((x_reg Reg x)(ones Reg (vec_imm_splat $I8X16 255))(signbit_copies Reg (vec_ashr_by_bit (vec_ashr_by_byte x_reg ones) ones))@@ -1593,7 +1593,7 @@(vec_add $I128 (vec_select $I128 clz_sum clz_hi mask) ones)));; Count leading sign-bit copies on z17, similar to above.-(rule 3 (lower (has_type (and (mie4_enabled) (fits_in_64 ty)) (cls x)))+(rule 3 (lower (has_type (and (mie4_enabled) (fits_in_64 ty)) (cls _ x)))(let ((ext_reg Reg (put_in_reg_sext64 x))(signbit_copies Reg (ashr_imm $I64 ext_reg 63))(inv_reg Reg (xor_reg $I64 ext_reg signbit_copies))@@ -1601,7 +1601,7 @@(cls_offset ty clz)));; Count leading sign-bit copies, 128-bit full vector on z17.-(rule 2 (lower (has_type (and (vxrs_ext3_enabled) $I128) (cls x)))+(rule 2 (lower (has_type (and (vxrs_ext3_enabled) $I128) (cls _ x)))(let ((x_reg Reg x)(ones Reg (vec_imm_splat $I8X16 255))(signbit_copies Reg (vec_ashr_by_bit (vec_ashr_by_byte x_reg ones) ones))@@ -1624,7 +1624,7 @@;; never zero by setting a "guard bit" in the position corresponding to;; the input type size. This way the 64-bit algorithm above will handle;; that case correctly automatically.-(rule 2 (lower (has_type (and (mie4_disabled) (gpr32_ty ty)) (ctz x)))+(rule 2 (lower (has_type (and (mie4_disabled) (gpr32_ty ty)) (ctz _ x)))(let ((rx Reg (or_uimm16shifted $I64 x (ctz_guardbit ty)))(lastbit Reg (and_reg $I64 rx (neg_reg $I64 rx)))(clz Reg (clz_flogr_reg 64 lastbit)))@@ -1639,14 +1639,14 @@;; via its condition code. We check for that and replace the instruction;; result with the value -1 via a conditional move, which will then lead to;; the correct result after the final subtraction from 63.-(rule 1 (lower (has_type (and (mie4_disabled) (gpr64_ty _ty)) (ctz x)))+(rule 1 (lower (has_type (and (mie4_disabled) (gpr64_ty _ty)) (ctz _ x)))(let ((rx Reg x)(lastbit Reg (and_reg $I64 rx (neg_reg $I64 rx)))(clz Reg (clz_flogr_reg -1 lastbit)))(sub_reg $I64 (imm $I64 63) clz)));; Count trailing zeros, 128-bit full vector pre-z17.-(rule 0 (lower (has_type (and (vxrs_ext3_disabled) $I128) (ctz x)))+(rule 0 (lower (has_type (and (vxrs_ext3_disabled) $I128) (ctz _ x)))(let ((ctz_vec Reg (vec_ctz $I64X2 x))(zero Reg (vec_imm $I64X2 0))(ctz_hi Reg (vec_permute_dw_imm $I64X2 zero 0 ctz_vec 0))@@ -1657,27 +1657,27 @@;; Count leading zeros on z17, via CTZG on types smaller than 64-bit,;; using the same guard bit mechanism as above.-(rule 5 (lower (has_type (and (mie4_enabled) (gpr32_ty ty)) (ctz x)))+(rule 5 (lower (has_type (and (mie4_enabled) (gpr32_ty ty)) (ctz _ x)))(ctz_reg (or_uimm16shifted $I64 x (ctz_guardbit ty))));; Count leading zeros on z17, via CTZG directly on 64-bit types.-(rule 4 (lower (has_type (and (mie4_enabled) (gpr64_ty _ty)) (ctz x)))+(rule 4 (lower (has_type (and (mie4_enabled) (gpr64_ty _ty)) (ctz _ x)))(ctz_reg x));; Count trailing zeros, 128-bit full vector on z17.-(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (ctz x)))+(rule 3 (lower (has_type (and (vxrs_ext3_enabled) $I128) (ctz _ x)))(vec_ctz $I128 x));;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Population count for 8-bit types is supported by the POPCNT instruction.-(rule (lower (has_type $I8 (popcnt x)))+(rule (lower (has_type $I8 (popcnt _ x)))(popcnt_byte x));; On z15, the POPCNT instruction has a variant to compute a full 64-bit;; population count, which we also use for 16- and 32-bit types.-(rule -1 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (popcnt x)))+(rule -1 (lower (has_type (and (mie3_enabled) (fits_in_64 ty)) (popcnt _ x)))(popcnt_reg (put_in_reg_zext64 x)));; On z14, we use the regular POPCNT, which computes the population count@@ -1688,18 +1688,18 @@;; $I16, where we instead accumulate in the low byte and clear high bits;; via an explicit and operation.)-(rule (lower (has_type (and (mie3_disabled) $I16) (popcnt x)))+(rule (lower (has_type (and (mie3_disabled) $I16) (popcnt _ x)))(let ((cnt2 Reg (popcnt_byte x))(cnt1 Reg (add_reg $I32 cnt2 (lshr_imm $I32 cnt2 8))))(and_uimm16shifted $I32 cnt1 (uimm16shifted 255 0))))-(rule (lower (has_type (and (mie3_disabled) $I32) (popcnt x)))+(rule (lower (has_type (and (mie3_disabled) $I32) (popcnt _ x)))(let ((cnt4 Reg (popcnt_byte x))(cnt2 Reg (add_reg $I32 cnt4 (lshl_imm $I32 cnt4 16)))(cnt1 Reg (add_reg $I32 cnt2 (lshl_imm $I32 cnt2 8))))(lshr_imm $I32 cnt1 24)))-(rule (lower (has_type (and (mie3_disabled) $I64) (popcnt x)))+(rule (lower (has_type (and (mie3_disabled) $I64) (popcnt _ x)))(let ((cnt8 Reg (popcnt_byte x))(cnt4 Reg (add_reg $I64 cnt8 (lshl_imm $I64 cnt8 32)))(cnt2 Reg (add_reg $I64 cnt4 (lshl_imm $I64 cnt4 16)))@@ -1707,11 +1707,11 @@(lshr_imm $I64 cnt1 56)));; Population count for vector types.-(rule 1 (lower (has_type (ty_vec128 ty) (popcnt x)))+(rule 1 (lower (has_type (ty_vec128 ty) (popcnt _ x)))(vec_popcnt ty x));; Population count, 128-bit full vector.-(rule (lower (has_type $I128 (popcnt x)))+(rule (lower (has_type $I128 (popcnt _ x)))(let ((popcnt_vec Reg (vec_popcnt $I64X2 x))(zero Reg (vec_imm $I64X2 0))(popcnt_hi Reg (vec_permute_dw_imm $I64X2 zero 0 popcnt_vec 0))@@ -1722,141 +1722,141 @@;;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Add two registers.-(rule (lower (has_type ty (fadd x y)))+(rule (lower (has_type ty (fadd _ x y)))(fadd_reg ty x y));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Subtract two registers.-(rule (lower (has_type ty (fsub x y)))+(rule (lower (has_type ty (fsub _ x y)))(fsub_reg ty x y));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Multiply two registers.-(rule (lower (has_type ty (fmul x y)))+(rule (lower (has_type ty (fmul _ x y)))(fmul_reg ty x y));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Divide two registers.-(rule (lower (has_type ty (fdiv x y)))+(rule (lower (has_type ty (fdiv _ x y)))(fdiv_reg ty x y));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Minimum of two registers.-(rule (lower (has_type ty (fmin x y)))+(rule (lower (has_type ty (fmin _ x y)))(fmin_reg ty x y));;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Maximum of two registers.-(rule (lower (has_type ty (fmax x y)))+(rule (lower (has_type ty (fmax _ x y)))(fmax_reg ty x y));;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Copysign of two registers.-(rule (lower (has_type $F32 (fcopysign x y)))+(rule (lower (has_type $F32 (fcopysign _ x y)))(vec_select $F32 x y (imm $F32 2147483647)))-(rule (lower (has_type $F64 (fcopysign x y)))+(rule (lower (has_type $F64 (fcopysign _ x y)))(vec_select $F64 x y (imm $F64 9223372036854775807)))-(rule (lower (has_type $F128 (fcopysign x y)))+(rule (lower (has_type $F128 (fcopysign _ x y)))(vec_select $F128 x y (vec_imm $F128 (imm8x16 127 255 255 255 255 255 255 255255 255 255 255 255 255 255 255))))-(rule (lower (has_type $F32X4 (fcopysign x y)))+(rule (lower (has_type $F32X4 (fcopysign _ x y)))(vec_select $F32X4 x y (vec_imm_bit_mask $F32X4 1 31)))-(rule (lower (has_type $F64X2 (fcopysign x y)))+(rule (lower (has_type $F64X2 (fcopysign _ x y)))(vec_select $F64X2 x y (vec_imm_bit_mask $F64X2 1 63)));;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Multiply-and-add of three registers.-(rule (lower (has_type ty (fma x y z)))+(rule (lower (has_type ty (fma _ x y z)))(fma_reg ty x y z));;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Square root of a register.-(rule (lower (has_type ty (sqrt x)))+(rule (lower (has_type ty (sqrt _ x)))(sqrt_reg ty x));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Negated value of a register.-(rule (lower (has_type ty (fneg x)))+(rule (lower (has_type ty (fneg _ x)))(fneg_reg ty x));;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Absolute value of a register.-(rule (lower (has_type ty (fabs x)))+(rule (lower (has_type ty (fabs _ x)))(fabs_reg ty x));;;; Rules for `ceil` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Round value in a register towards positive infinity.-(rule (lower (has_type ty (ceil x)))+(rule (lower (has_type ty (ceil _ x)))(ceil_reg ty x));;;; Rules for `floor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Round value in a register towards negative infinity.-(rule (lower (has_type ty (floor x)))+(rule (lower (has_type ty (floor _ x)))(floor_reg ty x));;;; Rules for `trunc` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Round value in a register towards zero.-(rule (lower (has_type ty (trunc x)))+(rule (lower (has_type ty (trunc _ x)))(trunc_reg ty x));;;; Rules for `nearest` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Round value in a register towards nearest.-(rule (lower (has_type ty (nearest x)))+(rule (lower (has_type ty (nearest _ x)))(nearest_reg ty x));;;; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Promote a register.-(rule (lower (has_type dst_ty (fpromote x @ (value_type src_ty))))+(rule (lower (has_type dst_ty (fpromote _ x @ (value_type src_ty))))(fpromote_reg dst_ty src_ty x))-(rule 1 (lower (has_type $F128 (fpromote x @ (value_type $F32))))+(rule 1 (lower (has_type $F128 (fpromote _ x @ (value_type $F32))))(fpromote_reg $F128 $F64 (fpromote_reg $F64 $F32 x)));;;; Rules for `fvpromote_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Promote a register.-(rule (lower (has_type $F64X2 (fvpromote_low x @ (value_type $F32X4))))+(rule (lower (has_type $F64X2 (fvpromote_low _ x @ (value_type $F32X4))))(fpromote_reg $F64X2 $F32X4 (vec_merge_low_lane_order $I32X4 x x)));;;; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Demote a register.-(rule (lower (has_type dst_ty (fdemote x @ (value_type src_ty))))+(rule (lower (has_type dst_ty (fdemote _ x @ (value_type src_ty))))(fdemote_reg dst_ty src_ty (FpuRoundMode.Current) x))-(rule 1 (lower (has_type $F32 (fdemote x @ (value_type $F128))))+(rule 1 (lower (has_type $F32 (fdemote _ x @ (value_type $F128))))(fdemote_reg $F32 $F64 (FpuRoundMode.Current)(fdemote_reg $F64 $F128 (FpuRoundMode.ShorterPrecision) x)))@@ -1864,7 +1864,7 @@;;;; Rules for `fvdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Demote a register.-(rule (lower (has_type $F32X4 (fvdemote x @ (value_type $F64X2))))+(rule (lower (has_type $F32X4 (fvdemote _ x @ (value_type $F64X2))))(let ((dst Reg (fdemote_reg $F32X4 $F64X2 (FpuRoundMode.Current) x)))(vec_pack_lane_order $I64X2 (vec_lshr_imm $I64X2 dst 32)(vec_imm $I64X2 0))))@@ -1874,39 +1874,39 @@;; Convert a 32-bit or smaller unsigned integer to $F32 (z15 instruction).(rule 1 (lower (has_type $F32- (fcvt_from_uint x @ (value_type (and (vxrs_ext2_enabled) (fits_in_32 ty))))))+ (fcvt_from_uint _ x @ (value_type (and (vxrs_ext2_enabled) (fits_in_32 ty))))))(fcvt_from_uint_reg $F32 $I32 (FpuRoundMode.ToNearestTiesToEven)(put_in_reg_zext32 x)));; Convert a 64-bit or smaller unsigned integer to $F32, via an intermediate $F64.-(rule (lower (has_type $F32 (fcvt_from_uint x @ (value_type (fits_in_64 ty)))))+(rule (lower (has_type $F32 (fcvt_from_uint _ x @ (value_type (fits_in_64 ty)))))(fdemote_reg $F32 $F64 (FpuRoundMode.ToNearestTiesToEven)(fcvt_from_uint_reg $F64 $I64 (FpuRoundMode.ShorterPrecision)(put_in_reg_zext64 x))));; Convert a 64-bit or smaller unsigned integer to $F64.-(rule (lower (has_type $F64 (fcvt_from_uint x @ (value_type (fits_in_64 ty)))))+(rule (lower (has_type $F64 (fcvt_from_uint _ x @ (value_type (fits_in_64 ty)))))(fcvt_from_uint_reg $F64 $I64 (FpuRoundMode.ToNearestTiesToEven)(put_in_reg_zext64 x)));; Convert a 32-bit or smaller unsigned integer to $F128.-(rule 1 (lower (has_type $F128 (fcvt_from_uint x @ (value_type (fits_in_32 ty)))))+(rule 1 (lower (has_type $F128 (fcvt_from_uint _ x @ (value_type (fits_in_32 ty)))))(fcvt_from_uint_reg $F128 $I32 (FpuRoundMode.ToNearestTiesToEven)(put_in_reg_zext32 x)));; Convert a 64-bit or smaller unsigned integer to $F128.-(rule (lower (has_type $F128 (fcvt_from_uint x @ (value_type (fits_in_64 ty)))))+(rule (lower (has_type $F128 (fcvt_from_uint _ x @ (value_type (fits_in_64 ty)))))(fcvt_from_uint_reg $F128 $I64 (FpuRoundMode.ToNearestTiesToEven)(put_in_reg_zext64 x)));; Convert $I32X4 to $F32X4 (z15 instruction).(rule 1 (lower (has_type (and (vxrs_ext2_enabled) $F32X4)- (fcvt_from_uint x @ (value_type $I32X4))))+ (fcvt_from_uint _ x @ (value_type $I32X4))))(fcvt_from_uint_reg $F32X4 $I32X4 (FpuRoundMode.ToNearestTiesToEven) x));; Convert $I32X4 to $F32X4 (via two $F64X2 on z14).(rule (lower (has_type (and (vxrs_ext2_disabled) $F32X4)- (fcvt_from_uint x @ (value_type $I32X4))))+ (fcvt_from_uint _ x @ (value_type $I32X4))))(vec_permute $F32X4(fdemote_reg $F32X4 $F64X2 (FpuRoundMode.ToNearestTiesToEven)(fcvt_from_uint_reg $F64X2 $I64X2 (FpuRoundMode.ShorterPrecision)@@ -1917,7 +1917,7 @@(vec_imm $I8X16 (imm8x16 0 1 2 3 8 9 10 11 16 17 18 19 24 25 26 27))));; Convert $I64X2 to $F64X2.-(rule (lower (has_type $F64X2 (fcvt_from_uint x @ (value_type $I64X2))))+(rule (lower (has_type $F64X2 (fcvt_from_uint _ x @ (value_type $I64X2))))(fcvt_from_uint_reg $F64X2 $I64X2 (FpuRoundMode.ToNearestTiesToEven) x))@@ -1925,39 +1925,39 @@;; Convert a 32-bit or smaller signed integer to $F32 (z15 instruction).(rule 1 (lower (has_type $F32- (fcvt_from_sint x @ (value_type (and (vxrs_ext2_enabled) (fits_in_32 ty))))))+ (fcvt_from_sint _ x @ (value_type (and (vxrs_ext2_enabled) (fits_in_32 ty))))))(fcvt_from_sint_reg $F32 $I32 (FpuRoundMode.ToNearestTiesToEven)(put_in_reg_sext32 x)));; Convert a 64-bit or smaller signed integer to $F32, via an intermediate $F64.-(rule (lower (has_type $F32 (fcvt_from_sint x @ (value_type (fits_in_64 ty)))))+(rule (lower (has_type $F32 (fcvt_from_sint _ x @ (value_type (fits_in_64 ty)))))(fdemote_reg $F32 $F64 (FpuRoundMode.ToNearestTiesToEven)(fcvt_from_sint_reg $F64 $I64 (FpuRoundMode.ShorterPrecision)(put_in_reg_sext64 x))));; Convert a 64-bit or smaller signed integer to $F64.-(rule (lower (has_type $F64 (fcvt_from_sint x @ (value_type (fits_in_64 ty)))))+(rule (lower (has_type $F64 (fcvt_from_sint _ x @ (value_type (fits_in_64 ty)))))(fcvt_from_sint_reg $F64 $I64 (FpuRoundMode.ToNearestTiesToEven)(put_in_reg_sext64 x)));; Convert a 32-bit or smaller signed integer to $F128.-(rule 1 (lower (has_type $F128 (fcvt_from_sint x @ (value_type (fits_in_32 ty)))))+(rule 1 (lower (has_type $F128 (fcvt_from_sint _ x @ (value_type (fits_in_32 ty)))))(fcvt_from_sint_reg $F128 $I32 (FpuRoundMode.ToNearestTiesToEven)(put_in_reg_sext32 x)));; Convert a 64-bit or smaller signed integer to $F128.-(rule (lower (has_type $F128 (fcvt_from_sint x @ (value_type (fits_in_64 ty)))))+(rule (lower (has_type $F128 (fcvt_from_sint _ x @ (value_type (fits_in_64 ty)))))(fcvt_from_sint_reg $F128 $I64 (FpuRoundMode.ToNearestTiesToEven)(put_in_reg_sext64 x)));; Convert $I32X4 to $F32X4 (z15 instruction).(rule 1 (lower (has_type (and (vxrs_ext2_enabled) $F32X4)- (fcvt_from_sint x @ (value_type $I32X4))))+ (fcvt_from_sint _ x @ (value_type $I32X4))))(fcvt_from_sint_reg $F32X4 $I32X4 (FpuRoundMode.ToNearestTiesToEven) x));; Convert $I32X4 to $F32X4 (via two $F64X2 on z14).(rule (lower (has_type (and (vxrs_ext2_disabled) $F32X4)- (fcvt_from_sint x @ (value_type $I32X4))))+ (fcvt_from_sint _ x @ (value_type $I32X4))))(vec_permute $F32X4(fdemote_reg $F32X4 $F64X2 (FpuRoundMode.ToNearestTiesToEven)(fcvt_from_sint_reg $F64X2 $I64X2 (FpuRoundMode.ShorterPrecision)@@ -1968,7 +1968,7 @@(vec_imm $I8X16 (imm8x16 0 1 2 3 8 9 10 11 16 17 18 19 24 25 26 27))));; Convert $I64X2 to $F64X2.-(rule (lower (has_type $F64X2 (fcvt_from_sint x @ (value_type $I64X2))))+(rule (lower (has_type $F64X2 (fcvt_from_sint _ x @ (value_type $I64X2))))(fcvt_from_sint_reg $F64X2 $I64X2 (FpuRoundMode.ToNearestTiesToEven) x))@@ -1977,7 +1977,7 @@;; Convert a scalar floating-point value in a register to an unsigned integer.;; Traps if the input cannot be represented in the output type.(rule (lower (has_type (fits_in_64 dst_ty)- (fcvt_to_uint x @ (value_type src_ty))))+ (fcvt_to_uint _ x @ (value_type src_ty))))(let ((src Reg (put_in_reg x));; First, check whether the input is a NaN, and trap if so.(_ Reg (trap_if (fcmp_reg src_ty src src)@@ -2002,7 +2002,7 @@;; Convert a scalar floating-point value in a register to a signed integer.;; Traps if the input cannot be represented in the output type.(rule (lower (has_type (fits_in_64 dst_ty)- (fcvt_to_sint x @ (value_type src_ty))))+ (fcvt_to_sint _ x @ (value_type src_ty))))(let ((src Reg (put_in_reg x));; First, check whether the input is a NaN, and trap if so.(_ Reg (trap_if (fcmp_reg src_ty src src)@@ -2027,7 +2027,7 @@;; Convert a scalar floating-point value in a register to an unsigned integer.(rule -1 (lower (has_type (fits_in_64 dst_ty)- (fcvt_to_uint_sat x @ (value_type src_ty))))+ (fcvt_to_uint_sat _ x @ (value_type src_ty))))(let ((src Reg (put_in_reg x));; Perform the conversion using the larger type size.(flt_ty Type (fcvt_flt_ty dst_ty src_ty))@@ -2039,12 +2039,12 @@;; Convert $F32X4 to $I32X4 (z15 instruction).(rule 1 (lower (has_type (and (vxrs_ext2_enabled) $I32X4)- (fcvt_to_uint_sat x @ (value_type $F32X4))))+ (fcvt_to_uint_sat _ x @ (value_type $F32X4))))(fcvt_to_uint_reg $I32X4 $F32X4 (FpuRoundMode.ToZero) x));; Convert $F32X4 to $I32X4 (via two $F64X2 on z14).(rule (lower (has_type (and (vxrs_ext2_disabled) $I32X4)- (fcvt_to_uint_sat x @ (value_type $F32X4))))+ (fcvt_to_uint_sat _ x @ (value_type $F32X4))))(vec_pack_usat $I64X2(fcvt_to_uint_reg $I64X2 $F64X2 (FpuRoundMode.ToZero)(fpromote_reg $F64X2 $F32X4 (vec_merge_high $I32X4 x x)))@@ -2052,7 +2052,7 @@(fpromote_reg $F64X2 $F32X4 (vec_merge_low $I32X4 x x)))));; Convert $F64X2 to $I64X2.-(rule (lower (has_type $I64X2 (fcvt_to_uint_sat x @ (value_type $F64X2))))+(rule (lower (has_type $I64X2 (fcvt_to_uint_sat _ x @ (value_type $F64X2))))(fcvt_to_uint_reg $I64X2 $F64X2 (FpuRoundMode.ToZero) x))@@ -2060,7 +2060,7 @@;; Convert a scalar floating-point value in a register to a signed integer.(rule -1 (lower (has_type (fits_in_64 dst_ty)- (fcvt_to_sint_sat x @ (value_type src_ty))))+ (fcvt_to_sint_sat _ x @ (value_type src_ty))))(let ((src Reg (put_in_reg x));; Perform the conversion using the larger type size.(flt_ty Type (fcvt_flt_ty dst_ty src_ty))@@ -2079,7 +2079,7 @@;; Convert $F32X4 to $I32X4 (z15 instruction).(rule 1 (lower (has_type (and (vxrs_ext2_enabled) $I32X4)- (fcvt_to_sint_sat src @ (value_type $F32X4))))+ (fcvt_to_sint_sat _ src @ (value_type $F32X4))));; See above for why we need to handle NaNs specially.(vec_select $I32X4(fcvt_to_sint_reg $I32X4 $F32X4 (FpuRoundMode.ToZero) src)@@ -2087,7 +2087,7 @@;; Convert $F32X4 to $I32X4 (via two $F64X2 on z14).(rule (lower (has_type (and (vxrs_ext2_disabled) $I32X4)- (fcvt_to_sint_sat src @ (value_type $F32X4))))+ (fcvt_to_sint_sat _ src @ (value_type $F32X4))));; See above for why we need to handle NaNs specially.(vec_select $I32X4(vec_pack_ssat $I64X2@@ -2098,7 +2098,7 @@(vec_imm $I32X4 0) (vec_fcmpeq $F32X4 src src)));; Convert $F64X2 to $I64X2.-(rule (lower (has_type $I64X2 (fcvt_to_sint_sat src @ (value_type $F64X2))))+(rule (lower (has_type $I64X2 (fcvt_to_sint_sat _ src @ (value_type $F64X2))));; See above for why we need to handle NaNs specially.(vec_select $I64X2(fcvt_to_sint_reg $I64X2 $F64X2 (FpuRoundMode.ToZero) src)@@ -2108,45 +2108,45 @@;;;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Reinterpret a 64-bit integer value as floating-point.-(rule (lower (has_type $F64 (bitcast _ x @ (value_type $I64))))+(rule (lower (has_type $F64 (bitcast _ _ x @ (value_type $I64))))(vec_insert_lane_undef $F64X2 x 0 (zero_reg)));; Reinterpret a 64-bit floating-point value as integer.-(rule (lower (has_type $I64 (bitcast _ x @ (value_type $F64))))+(rule (lower (has_type $I64 (bitcast _ _ x @ (value_type $F64))))(vec_extract_lane $F64X2 x 0 (zero_reg)));; Reinterpret a 32-bit integer value as floating-point.-(rule (lower (has_type $F32 (bitcast _ x @ (value_type $I32))))+(rule (lower (has_type $F32 (bitcast _ _ x @ (value_type $I32))))(vec_insert_lane_undef $F32X4 x 0 (zero_reg)));; Reinterpret a 32-bit floating-point value as integer.-(rule (lower (has_type $I32 (bitcast _ x @ (value_type $F32))))+(rule (lower (has_type $I32 (bitcast _ _ x @ (value_type $F32))))(vec_extract_lane $F32X4 x 0 (zero_reg)));; Reinterpret a 16-bit integer value as floating-point.-(rule (lower (has_type $F16 (bitcast _ x @ (value_type $I16))))+(rule (lower (has_type $F16 (bitcast _ _ x @ (value_type $I16))))(vec_insert_lane_undef $F16X8 x 0 (zero_reg)));; Reinterpret a 16-bit floating-point value as integer.-(rule (lower (has_type $I16 (bitcast _ x @ (value_type $F16))))+(rule (lower (has_type $I16 (bitcast _ _ x @ (value_type $F16))))(vec_extract_lane $F16X8 x 0 (zero_reg)));; Bitcast between types residing in GPRs is a no-op.(rule 1 (lower (has_type (gpr32_ty _)- (bitcast _ x @ (value_type (gpr32_ty _)))))+ (bitcast _ _ x @ (value_type (gpr32_ty _)))))x)(rule 2 (lower (has_type (gpr64_ty _)- (bitcast _ x @ (value_type (gpr64_ty _)))))+ (bitcast _ _ x @ (value_type (gpr64_ty _)))))x);; Bitcast between types residing in FPRs is a no-op.(rule 3 (lower (has_type (ty_scalar_float _)- (bitcast _ x @ (value_type (ty_scalar_float _)))))+ (bitcast _ _ x @ (value_type (ty_scalar_float _)))))x);; Bitcast between types residing in VRs is a no-op if lane count is unchanged.(rule 5 (lower (has_type (multi_lane bits count)- (bitcast _ x @ (value_type (multi_lane bits count)))))+ (bitcast _ _ x @ (value_type (multi_lane bits count)))))x);; Bitcast between types residing in VRs with different lane counts is a@@ -2157,7 +2157,7 @@;; choice of lane order depending on the current function ABI, this case will;; currently never arise in practice.(rule 4 (lower (has_type (vr128_ty out_ty)- (bitcast flags x @ (value_type (vr128_ty in_ty)))))+ (bitcast _ flags x @ (value_type (vr128_ty in_ty)))))(abi_vec_elt_rev (lane_order_from_memflags flags) out_ty(abi_vec_elt_rev (lane_order_from_memflags flags) in_ty x)))@@ -2171,37 +2171,37 @@;;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Insert vector lane from general-purpose register.-(rule 1 (lower (insertlane x @ (value_type ty)+(rule 1 (lower (insertlane _ x @ (value_type ty)y @ (value_type in_ty)(u8_from_uimm8 idx)))(if (ty_int_ref_scalar_64 in_ty))(vec_insert_lane ty x y (be_lane_idx ty idx) (zero_reg)));; Insert vector lane from floating-point register.-(rule 0 (lower (insertlane x @ (value_type ty)+(rule 0 (lower (insertlane _ x @ (value_type ty)y @ (value_type (ty_scalar_float _))(u8_from_uimm8 idx)))(vec_move_lane_and_insert ty x (be_lane_idx ty idx) y 0));; Insert vector lane from another vector lane.-(rule 2 (lower (insertlane x @ (value_type ty)- (extractlane y (u8_from_uimm8 src_idx))+(rule 2 (lower (insertlane _ x @ (value_type ty)+ (extractlane _ y (u8_from_uimm8 src_idx))(u8_from_uimm8 dst_idx)))(vec_move_lane_and_insert ty x (be_lane_idx ty dst_idx)y (be_lane_idx ty src_idx)));; Insert vector lane from signed 16-bit immediate.-(rule 3 (lower (insertlane x @ (value_type ty) (i16_from_value y)+(rule 3 (lower (insertlane _ x @ (value_type ty) (i16_from_value y)(u8_from_uimm8 idx)))(vec_insert_lane_imm ty x y (be_lane_idx ty idx)));; Insert vector lane from big-endian memory.-(rule 4 (lower (insertlane x @ (value_type ty) (sinkable_load y)+(rule 4 (lower (insertlane _ x @ (value_type ty) (sinkable_load y)(u8_from_uimm8 idx)))(vec_load_lane ty x (sink_load y) (be_lane_idx ty idx)));; Insert vector lane from little-endian memory.-(rule 5 (lower (insertlane x @ (value_type ty) (sinkable_load_little y)+(rule 5 (lower (insertlane _ x @ (value_type ty) (sinkable_load_little y)(u8_from_uimm8 idx)))(vec_load_lane_little ty x (sink_load y) (be_lane_idx ty idx)))@@ -2289,25 +2289,25 @@;; Extract vector lane to general-purpose register.(rule 1 (lower (has_type out_ty- (extractlane x @ (value_type ty) (u8_from_uimm8 idx))))+ (extractlane _ x @ (value_type ty) (u8_from_uimm8 idx))))(if (ty_int_ref_scalar_64 out_ty))(vec_extract_lane ty x (be_lane_idx ty idx) (zero_reg)));; Extract vector lane to floating-point register.(rule 0 (lower (has_type (ty_scalar_float _)- (extractlane x @ (value_type ty) (u8_from_uimm8 idx))))+ (extractlane _ x @ (value_type ty) (u8_from_uimm8 idx))))(vec_replicate_lane ty x (be_lane_idx ty idx)));; Extract vector lane and store to big-endian memory.(rule 6 (lower (store flags @ (bigendian)- (extractlane x @ (value_type ty) (u8_from_uimm8 idx))+ (extractlane _ x @ (value_type ty) (u8_from_uimm8 idx))addr offset))(side_effect (vec_store_lane ty x(lower_address flags addr offset) (be_lane_idx ty idx))));; Extract vector lane and store to little-endian memory.(rule 5 (lower (store flags @ (littleendian)- (extractlane x @ (value_type ty) (u8_from_uimm8 idx))+ (extractlane _ x @ (value_type ty) (u8_from_uimm8 idx))addr offset))(side_effect (vec_store_lane_little ty x(lower_address flags addr offset) (be_lane_idx ty idx))))@@ -2346,29 +2346,29 @@;;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load replicated value from general-purpose register.-(rule 1 (lower (has_type ty (splat x @ (value_type in_ty))))+(rule 1 (lower (has_type ty (splat _ x @ (value_type in_ty))))(if (ty_int_ref_scalar_64 in_ty))(vec_replicate_lane ty (vec_insert_lane_undef ty x 0 (zero_reg)) 0));; Load replicated value from floating-point register.-(rule 0 (lower (has_type ty (splat+(rule 0 (lower (has_type ty (splat _x @ (value_type (ty_scalar_float _)))))(vec_replicate_lane ty x 0));; Load replicated value from vector lane.-(rule 2 (lower (has_type ty (splat (extractlane x (u8_from_uimm8 idx)))))+(rule 2 (lower (has_type ty (splat _ (extractlane _ x (u8_from_uimm8 idx)))))(vec_replicate_lane ty x (be_lane_idx ty idx)));; Load replicated 16-bit immediate value.-(rule 3 (lower (has_type ty (splat (i16_from_value x))))+(rule 3 (lower (has_type ty (splat _ (i16_from_value x))))(vec_imm_replicate ty x));; Load replicated value from big-endian memory.-(rule 4 (lower (has_type ty (splat (sinkable_load x))))+(rule 4 (lower (has_type ty (splat _ (sinkable_load x))))(vec_load_replicate ty (sink_load x)));; Load replicated value from little-endian memory.-(rule 5 (lower (has_type ty (splat (sinkable_load_little x))))+(rule 5 (lower (has_type ty (splat _ (sinkable_load_little x))))(vec_load_replicate_little ty (sink_load x)))@@ -2405,31 +2405,31 @@;;;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load scalar value from general-purpose register.-(rule 1 (lower (has_type ty (scalar_to_vector+(rule 1 (lower (has_type ty (scalar_to_vector _x @ (value_type in_ty))))(if (ty_int_ref_scalar_64 in_ty))(vec_insert_lane ty (vec_imm ty 0) x (be_lane_idx ty 0) (zero_reg)));; Load scalar value from floating-point register.-(rule 0 (lower (has_type ty (scalar_to_vector+(rule 0 (lower (has_type ty (scalar_to_vector _x @ (value_type (ty_scalar_float _)))))(vec_move_lane_and_zero ty (be_lane_idx ty 0) x 0));; Load scalar value from vector lane.-(rule 2 (lower (has_type ty (scalar_to_vector- (extractlane x (u8_from_uimm8 idx)))))+(rule 2 (lower (has_type ty (scalar_to_vector _+ (extractlane _ x (u8_from_uimm8 idx)))))(vec_move_lane_and_zero ty (be_lane_idx ty 0) x (be_lane_idx ty idx)));; Load scalar 16-bit immediate value.-(rule 3 (lower (has_type ty (scalar_to_vector (i16_from_value x))))+(rule 3 (lower (has_type ty (scalar_to_vector _ (i16_from_value x))))(vec_insert_lane_imm ty (vec_imm ty 0) x (be_lane_idx ty 0)));; Load scalar value from big-endian memory.-(rule 4 (lower (has_type ty (scalar_to_vector (sinkable_load x))))+(rule 4 (lower (has_type ty (scalar_to_vector _ (sinkable_load x))))(vec_load_lane ty (vec_imm ty 0) (sink_load x) (be_lane_idx ty 0)));; Load scalar value lane from little-endian memory.-(rule 5 (lower (has_type ty (scalar_to_vector (sinkable_load_little x))))+(rule 5 (lower (has_type ty (scalar_to_vector _ (sinkable_load_little x))))(vec_load_lane_little ty (vec_imm ty 0) (sink_load x) (be_lane_idx ty 0)))@@ -2456,157 +2456,157 @@;;;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; General case: use vec_permute and then mask off zero lanes.-(rule -2 (lower (shuffle x y (shuffle_mask permute_mask and_mask)))+(rule -2 (lower (shuffle _ x y (shuffle_mask permute_mask and_mask)))(vec_and $I8X16 (vec_imm_byte_mask $I8X16 and_mask)(vec_permute $I8X16 x y (vec_imm $I8X16 permute_mask))));; If the pattern has no zero lanes, just a vec_permute suffices.-(rule -1 (lower (shuffle x y (shuffle_mask permute_mask 65535)))+(rule -1 (lower (shuffle _ x y (shuffle_mask permute_mask 65535)))(vec_permute $I8X16 x y (vec_imm $I8X16 permute_mask)));; Special patterns that can be implemented via MERGE HIGH.-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23) 65535)))(vec_merge_high $I64X2 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 1 2 3 16 17 18 19 4 5 6 7 20 21 22 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 1 2 3 16 17 18 19 4 5 6 7 20 21 22 23) 65535)))(vec_merge_high $I32X4 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 1 16 17 2 3 18 19 4 5 20 21 6 7 22 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 1 16 17 2 3 18 19 4 5 20 21 6 7 22 23) 65535)))(vec_merge_high $I16X8 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 16 1 17 2 18 3 19 4 20 5 21 6 22 7 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 16 1 17 2 18 3 19 4 20 5 21 6 22 7 23) 65535)))(vec_merge_high $I8X16 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 17 18 19 20 21 22 23 0 1 2 3 4 5 6 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 17 18 19 20 21 22 23 0 1 2 3 4 5 6 7) 65535)))(vec_merge_high $I64X2 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 17 18 19 0 1 2 3 20 21 22 23 4 5 6 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 17 18 19 0 1 2 3 20 21 22 23 4 5 6 7) 65535)))(vec_merge_high $I32X4 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 17 0 1 18 19 2 3 20 21 4 5 22 23 6 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 17 0 1 18 19 2 3 20 21 4 5 22 23 6 7) 65535)))(vec_merge_high $I16X8 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 0 17 1 18 2 19 3 20 4 21 5 22 6 23 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 0 17 1 18 2 19 3 20 4 21 5 22 6 23 7) 65535)))(vec_merge_high $I8X16 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7) 65535)))(vec_merge_high $I64X2 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 1 2 3 0 1 2 3 4 5 6 7 4 5 6 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 1 2 3 0 1 2 3 4 5 6 7 4 5 6 7) 65535)))(vec_merge_high $I32X4 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 1 0 1 2 3 2 3 4 5 4 5 6 7 6 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 1 0 1 2 3 2 3 4 5 4 5 6 7 6 7) 65535)))(vec_merge_high $I16X8 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 0 1 1 2 2 3 3 4 4 5 5 6 6 7 7) 65535)))(vec_merge_high $I8X16 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 17 18 19 20 21 22 23 16 17 18 19 20 21 22 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 17 18 19 20 21 22 23 16 17 18 19 20 21 22 23) 65535)))(vec_merge_high $I64X2 y y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 17 18 19 16 17 18 19 20 21 22 23 20 21 22 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 17 18 19 16 17 18 19 20 21 22 23 20 21 22 23) 65535)))(vec_merge_high $I32X4 y y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 17 16 17 18 19 18 19 20 21 20 21 22 23 22 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 17 16 17 18 19 18 19 20 21 20 21 22 23 22 23) 65535)))(vec_merge_high $I16X8 y y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 16 17 17 18 18 19 19 20 20 21 21 22 22 23 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 16 17 17 18 18 19 19 20 20 21 21 22 22 23 23) 65535)))(vec_merge_high $I8X16 y y));; Special patterns that can be implemented via MERGE LOW.-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31) 65535)))(vec_merge_low $I64X2 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 9 10 11 24 25 26 27 12 13 14 15 28 29 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 9 10 11 24 25 26 27 12 13 14 15 28 29 30 31) 65535)))(vec_merge_low $I32X4 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 9 24 25 10 11 26 27 12 13 28 29 14 15 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 9 24 25 10 11 26 27 12 13 28 29 14 15 30 31) 65535)))(vec_merge_low $I16X8 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 24 9 25 10 26 11 27 12 28 13 29 14 30 15 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 24 9 25 10 26 11 27 12 28 13 29 14 30 15 31) 65535)))(vec_merge_low $I8X16 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 25 26 27 28 29 30 31 8 9 10 11 12 13 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 25 26 27 28 29 30 31 8 9 10 11 12 13 14 15) 65535)))(vec_merge_low $I64X2 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 25 26 27 8 9 10 11 28 29 30 31 12 13 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 25 26 27 8 9 10 11 28 29 30 31 12 13 14 15) 65535)))(vec_merge_low $I32X4 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 25 8 9 26 27 10 11 28 29 12 13 30 31 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 25 8 9 26 27 10 11 28 29 12 13 30 31 14 15) 65535)))(vec_merge_low $I16X8 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 8 25 9 26 10 27 11 28 12 29 13 30 14 31 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 8 25 9 26 10 27 11 28 12 29 13 30 14 31 15) 65535)))(vec_merge_low $I8X16 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 9 10 11 12 13 14 15 8 9 10 11 12 13 14 15) 65535)))(vec_merge_low $I64X2 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 9 10 11 8 9 10 11 12 13 14 15 12 13 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 9 10 11 8 9 10 11 12 13 14 15 12 13 14 15) 65535)))(vec_merge_low $I32X4 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 9 8 9 10 11 10 11 12 13 12 13 14 15 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 9 8 9 10 11 10 11 12 13 12 13 14 15 14 15) 65535)))(vec_merge_low $I16X8 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 8 9 9 10 10 11 11 12 12 13 13 14 14 15 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 8 9 9 10 10 11 11 12 12 13 13 14 14 15 15) 65535)))(vec_merge_low $I8X16 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 25 26 27 28 29 30 31 24 25 26 27 28 29 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 25 26 27 28 29 30 31 24 25 26 27 28 29 30 31) 65535)))(vec_merge_low $I64X2 y y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 25 26 27 24 25 26 27 28 29 30 31 28 29 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 25 26 27 24 25 26 27 28 29 30 31 28 29 30 31) 65535)))(vec_merge_low $I32X4 y y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 25 24 25 26 27 26 27 28 29 28 29 30 31 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 25 24 25 26 27 26 27 28 29 28 29 30 31 30 31) 65535)))(vec_merge_low $I16X8 y y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 24 25 25 26 26 27 27 28 28 29 29 30 30 31 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 24 25 25 26 26 27 27 28 28 29 29 30 30 31 31) 65535)))(vec_merge_low $I8X16 y y));; Special patterns that can be implemented via PACK.-(rule (lower (shuffle x y (shuffle_mask (imm8x16 4 5 6 7 12 13 14 15 20 21 22 23 28 29 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 4 5 6 7 12 13 14 15 20 21 22 23 28 29 30 31) 65535)))(vec_pack $I64X2 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 2 3 6 7 10 11 14 15 18 19 22 23 26 27 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 2 3 6 7 10 11 14 15 18 19 22 23 26 27 30 31) 65535)))(vec_pack $I32X4 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 1 3 5 7 9 11 13 15 17 19 21 23 25 27 29 31) 65535)))(vec_pack $I16X8 x y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 20 21 22 23 28 29 30 31 4 5 6 7 12 13 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 20 21 22 23 28 29 30 31 4 5 6 7 12 13 14 15) 65535)))(vec_pack $I64X2 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 18 19 22 23 26 27 30 31 2 3 6 7 10 11 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 18 19 22 23 26 27 30 31 2 3 6 7 10 11 14 15) 65535)))(vec_pack $I32X4 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 17 19 21 23 25 27 29 31 1 3 5 7 9 11 13 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 17 19 21 23 25 27 29 31 1 3 5 7 9 11 13 15) 65535)))(vec_pack $I16X8 y x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 4 5 6 7 12 13 14 15 4 5 6 7 12 13 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 4 5 6 7 12 13 14 15 4 5 6 7 12 13 14 15) 65535)))(vec_pack $I64X2 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 2 3 6 7 10 11 14 15 2 3 6 7 10 11 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 2 3 6 7 10 11 14 15 2 3 6 7 10 11 14 15) 65535)))(vec_pack $I32X4 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 1 3 5 7 9 11 13 15 1 3 5 7 9 11 13 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 1 3 5 7 9 11 13 15 1 3 5 7 9 11 13 15) 65535)))(vec_pack $I16X8 x x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 20 21 22 23 28 29 30 31 20 21 22 23 28 29 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 20 21 22 23 28 29 30 31 20 21 22 23 28 29 30 31) 65535)))(vec_pack $I64X2 y y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 18 19 22 23 26 27 30 31 18 19 22 23 26 27 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 18 19 22 23 26 27 30 31 18 19 22 23 26 27 30 31) 65535)))(vec_pack $I32X4 y y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 17 19 21 23 25 27 29 31 17 19 21 23 25 27 29 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 17 19 21 23 25 27 29 31 17 19 21 23 25 27 29 31) 65535)))(vec_pack $I16X8 y y));; Special patterns that can be implemented via UNPACK HIGH.-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ _ _ _ 0 1 2 3 _ _ _ _ 4 5 6 7) 3855)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ _ _ _ 0 1 2 3 _ _ _ _ 4 5 6 7) 3855)))(vec_unpacku_high $I32X4 x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ _ 0 1 _ _ 2 3 _ _ 4 5 _ _ 6 7) 13107)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ _ 0 1 _ _ 2 3 _ _ 4 5 _ _ 6 7) 13107)))(vec_unpacku_high $I16X8 x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ 0 _ 1 _ 2 _ 3 _ 4 _ 5 _ 6 _ 7) 21845)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ 0 _ 1 _ 2 _ 3 _ 4 _ 5 _ 6 _ 7) 21845)))(vec_unpacku_high $I8X16 x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ _ _ _ 16 17 18 19 _ _ _ _ 20 21 22 23) 3855)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ _ _ _ 16 17 18 19 _ _ _ _ 20 21 22 23) 3855)))(vec_unpacku_high $I32X4 y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ _ 16 17 _ _ 18 19 _ _ 20 21 _ _ 22 23) 13107)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ _ 16 17 _ _ 18 19 _ _ 20 21 _ _ 22 23) 13107)))(vec_unpacku_high $I16X8 y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ 16 _ 17 _ 18 _ 19 _ 20 _ 21 _ 22 _ 23) 21845)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ 16 _ 17 _ 18 _ 19 _ 20 _ 21 _ 22 _ 23) 21845)))(vec_unpacku_high $I8X16 y));; Special patterns that can be implemented via UNPACK LOW.-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ _ _ _ 8 9 10 11 _ _ _ _ 12 13 14 15) 3855)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ _ _ _ 8 9 10 11 _ _ _ _ 12 13 14 15) 3855)))(vec_unpacku_low $I32X4 x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ _ 8 9 _ _ 10 11 _ _ 12 13 _ _ 14 15) 13107)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ _ 8 9 _ _ 10 11 _ _ 12 13 _ _ 14 15) 13107)))(vec_unpacku_low $I16X8 x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ 8 _ 9 _ 10 _ 11 _ 12 _ 13 _ 14 _ 15) 21845)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ 8 _ 9 _ 10 _ 11 _ 12 _ 13 _ 14 _ 15) 21845)))(vec_unpacku_low $I8X16 x))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ _ _ _ 24 25 26 27 _ _ _ _ 28 29 30 31) 3855)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ _ _ _ 24 25 26 27 _ _ _ _ 28 29 30 31) 3855)))(vec_unpacku_low $I32X4 y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ _ 24 25 _ _ 26 27 _ _ 28 29 _ _ 30 31) 13107)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ _ 24 25 _ _ 26 27 _ _ 28 29 _ _ 30 31) 13107)))(vec_unpacku_low $I16X8 y))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 _ 24 _ 25 _ 26 _ 27 _ 28 _ 29 _ 30 _ 31) 21845)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 _ 24 _ 25 _ 26 _ 27 _ 28 _ 29 _ 30 _ 31) 21845)))(vec_unpacku_low $I8X16 y));; Special patterns that can be implemented via PERMUTE DOUBLEWORD IMMEDIATE.-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 1 2 3 4 5 6 7 24 25 26 27 28 29 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 1 2 3 4 5 6 7 24 25 26 27 28 29 30 31) 65535)))(vec_permute_dw_imm $I8X16 x 0 y 1))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23) 65535)))(vec_permute_dw_imm $I8X16 x 1 y 0))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 17 18 19 20 21 22 23 8 9 10 11 12 13 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 17 18 19 20 21 22 23 8 9 10 11 12 13 14 15) 65535)))(vec_permute_dw_imm $I8X16 y 0 x 1))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 25 26 27 28 29 30 31 0 1 2 3 4 5 6 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 25 26 27 28 29 30 31 0 1 2 3 4 5 6 7) 65535)))(vec_permute_dw_imm $I8X16 y 1 x 0))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15) 65535)))(vec_permute_dw_imm $I8X16 x 0 x 1))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 8 9 10 11 12 13 14 15 0 1 2 3 4 5 6 7) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 8 9 10 11 12 13 14 15 0 1 2 3 4 5 6 7) 65535)))(vec_permute_dw_imm $I8X16 x 1 x 0))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31) 65535)))(vec_permute_dw_imm $I8X16 y 0 y 1))-(rule (lower (shuffle x y (shuffle_mask (imm8x16 24 25 26 27 28 29 30 31 16 17 18 19 20 21 22 23) 65535)))+(rule (lower (shuffle _ x y (shuffle_mask (imm8x16 24 25 26 27 28 29 30 31 16 17 18 19 20 21 22 23) 65535)))(vec_permute_dw_imm $I8X16 y 1 y 0));;;; Rules for `blendv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (and (ty_vec128 ty) (vxrs_ext3_enabled)) (blendv p x y)))+(rule 1 (lower (has_type (and (ty_vec128 ty) (vxrs_ext3_enabled)) (blendv _ p x y)))(vec_blend ty p x y))@@ -2619,7 +2619,7 @@;; permute-lane-element := umin (16, swizzle-lane-element);; and pass a zero vector as second operand to the permute instruction.-(rule 1 (lower (has_type (ty_vec128 ty) (swizzle x y)))+(rule 1 (lower (has_type (ty_vec128 ty) (swizzle _ x y)))(if-let (LaneOrder.BigEndian) (lane_order))(vec_permute ty x (vec_imm ty 0)(vec_umin $I8X16 (vec_imm_splat $I8X16 16) y)))@@ -2638,7 +2638,7 @@;; the input vector as second operand (covering lanes 16 .. 31);; to implement the required swizzle semantics.-(rule (lower (has_type (ty_vec128 ty) (swizzle x y)))+(rule (lower (has_type (ty_vec128 ty) (swizzle _ x y)))(if-let (LaneOrder.LittleEndian) (lane_order))(vec_permute ty (vec_imm ty 0) x(vec_umax $I8X16 (vec_imm_splat $I8X16 239)@@ -2648,38 +2648,38 @@;;;; Rules for `stack_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load the address of a stack slot.-(rule (lower (has_type ty (stack_addr stack_slot offset)))+(rule (lower (has_type ty (stack_addr _ stack_slot offset)))(stack_addr_impl ty stack_slot offset));;;; Rules for `func_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load the address of a function, target reachable via PC-relative instruction.-(rule 1 (lower (func_addr (func_ref_data _ name (RelocDistance.Near) _)))+(rule 1 (lower (func_addr _ (func_ref_data _ name (RelocDistance.Near) _)))(load_addr (memarg_symbol name 0 (memflags_trusted))));; Load the address of a function, general case.-(rule (lower (func_addr (func_ref_data _ name _ _)))+(rule (lower (func_addr _ (func_ref_data _ name _ _)))(load_symbol_reloc (SymbolReloc.Absolute name 0)));;;; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load the address of a symbol, target reachable via PC-relative instruction.-(rule 1 (lower (symbol_value (symbol_value_data name (RelocDistance.Near)+(rule 1 (lower (symbol_value _ (symbol_value_data name (RelocDistance.Near)off)))(if-let offset (memarg_symbol_offset off))(load_addr (memarg_symbol name offset (memflags_trusted))));; Load the address of a symbol, general case.-(rule (lower (symbol_value (symbol_value_data name _ offset)))+(rule (lower (symbol_value _ (symbol_value_data name _ offset)))(load_symbol_reloc (SymbolReloc.Absolute name offset)));;;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load the address of a TLS symbol (ELF general-dynamic model).-(rule (lower (tls_value (symbol_value_data name _ 0)))+(rule (lower (tls_value _ (symbol_value_data name _ 0)))(if (tls_model_is_elf_gd))(let ((symbol SymbolReloc (SymbolReloc.TlsGd name))(got Reg (load_addr (memarg_got)))@@ -2707,74 +2707,74 @@;;;; Rules for `load` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load 8-bit integers.-(rule (lower (has_type $I8 (load flags addr offset)))+(rule (lower (has_type $I8 (load _ flags addr offset)))(zext32_mem $I8 (lower_address flags addr offset)));; Load 16-bit big-endian integers.-(rule (lower (has_type $I16 (load flags @ (bigendian) addr offset)))+(rule (lower (has_type $I16 (load _ flags @ (bigendian) addr offset)))(zext32_mem $I16 (lower_address flags addr offset)));; Load 16-bit little-endian integers.-(rule -1 (lower (has_type $I16 (load flags @ (littleendian) addr offset)))+(rule -1 (lower (has_type $I16 (load _ flags @ (littleendian) addr offset)))(loadrev16 (lower_address flags addr offset)));; Load 32-bit big-endian integers.-(rule (lower (has_type $I32 (load flags @ (bigendian) addr offset)))+(rule (lower (has_type $I32 (load _ flags @ (bigendian) addr offset)))(load32 (lower_address flags addr offset)));; Load 32-bit little-endian integers.-(rule -1 (lower (has_type $I32 (load flags @ (littleendian) addr offset)))+(rule -1 (lower (has_type $I32 (load _ flags @ (littleendian) addr offset)))(loadrev32 (lower_address flags addr offset)));; Load 64-bit big-endian integers.-(rule (lower (has_type $I64 (load flags @ (bigendian) addr offset)))+(rule (lower (has_type $I64 (load _ flags @ (bigendian) addr offset)))(load64 (lower_address flags addr offset)));; Load 64-bit little-endian integers.-(rule -1 (lower (has_type $I64 (load flags @ (littleendian) addr offset)))+(rule -1 (lower (has_type $I64 (load _ flags @ (littleendian) addr offset)))(loadrev64 (lower_address flags addr offset)));; Load 16-bit big-endian floating-point values (as vector lane).-(rule (lower (has_type $F16 (load flags @ (bigendian) addr offset)))+(rule (lower (has_type $F16 (load _ flags @ (bigendian) addr offset)))(vec_load_lane_undef $F16X8 (lower_address flags addr offset) 0));; Load 16-bit little-endian floating-point values (as vector lane).-(rule -1 (lower (has_type $F16 (load flags @ (littleendian) addr offset)))+(rule -1 (lower (has_type $F16 (load _ flags @ (littleendian) addr offset)))(vec_load_lane_little_undef $F16X8 (lower_address flags addr offset) 0));; Load 32-bit big-endian floating-point values (as vector lane).-(rule (lower (has_type $F32 (load flags @ (bigendian) addr offset)))+(rule (lower (has_type $F32 (load _ flags @ (bigendian) addr offset)))(vec_load_lane_undef $F32X4 (lower_address flags addr offset) 0));; Load 32-bit little-endian floating-point values (as vector lane).-(rule -1 (lower (has_type $F32 (load flags @ (littleendian) addr offset)))+(rule -1 (lower (has_type $F32 (load _ flags @ (littleendian) addr offset)))(vec_load_lane_little_undef $F32X4 (lower_address flags addr offset) 0));; Load 64-bit big-endian floating-point values (as vector lane).-(rule (lower (has_type $F64 (load flags @ (bigendian) addr offset)))+(rule (lower (has_type $F64 (load _ flags @ (bigendian) addr offset)))(vec_load_lane_undef $F64X2 (lower_address flags addr offset) 0));; Load 64-bit little-endian floating-point values (as vector lane).-(rule -1 (lower (has_type $F64 (load flags @ (littleendian) addr offset)))+(rule -1 (lower (has_type $F64 (load _ flags @ (littleendian) addr offset)))(vec_load_lane_little_undef $F64X2 (lower_address flags addr offset) 0));; Load 128-bit big-endian vector values, BE lane order - direct load.-(rule 4 (lower (has_type (vr128_ty ty) (load flags @ (bigendian) addr offset)))+(rule 4 (lower (has_type (vr128_ty ty) (load _ flags @ (bigendian) addr offset)))(if-let (LaneOrder.BigEndian) (lane_order))(vec_load ty (lower_address flags addr offset)));; Load 128-bit little-endian vector values, BE lane order - byte-reversed load.-(rule 3 (lower (has_type (vr128_ty ty) (load flags @ (littleendian) addr offset)))+(rule 3 (lower (has_type (vr128_ty ty) (load _ flags @ (littleendian) addr offset)))(if-let (LaneOrder.BigEndian) (lane_order))(vec_load_byte_rev ty flags addr offset));; Load 128-bit big-endian vector values, LE lane order - element-reversed load.-(rule 2 (lower (has_type (vr128_ty ty) (load flags @ (bigendian) addr offset)))+(rule 2 (lower (has_type (vr128_ty ty) (load _ flags @ (bigendian) addr offset)))(if-let (LaneOrder.LittleEndian) (lane_order))(vec_load_elt_rev ty flags addr offset));; Load 128-bit little-endian vector values, LE lane order - fully-reversed load.-(rule 1 (lower (has_type (vr128_ty ty) (load flags @ (littleendian) addr offset)))+(rule 1 (lower (has_type (vr128_ty ty) (load _ flags @ (littleendian) addr offset)))(if-let (LaneOrder.LittleEndian) (lane_order))(vec_load_full_rev ty flags addr offset))@@ -2875,22 +2875,22 @@;;;; Rules for `uload8` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16- or 32-bit target types.-(rule (lower (has_type (gpr32_ty _ty) (uload8 flags addr offset)))+(rule (lower (has_type (gpr32_ty _ty) (uload8 _ flags addr offset)))(zext32_mem $I8 (lower_address flags addr offset)));; 64-bit target types.-(rule 1 (lower (has_type (gpr64_ty _ty) (uload8 flags addr offset)))+(rule 1 (lower (has_type (gpr64_ty _ty) (uload8 _ flags addr offset)))(zext64_mem $I8 (lower_address flags addr offset)));;;; Rules for `sload8` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 16- or 32-bit target types.-(rule (lower (has_type (gpr32_ty _ty) (sload8 flags addr offset)))+(rule (lower (has_type (gpr32_ty _ty) (sload8 _ flags addr offset)))(sext32_mem $I8 (lower_address flags addr offset)));; 64-bit target types.-(rule 1 (lower (has_type (gpr64_ty _ty) (sload8 flags addr offset)))+(rule 1 (lower (has_type (gpr64_ty _ty) (sload8 _ flags addr offset)))(sext64_mem $I8 (lower_address flags addr offset)))@@ -2898,23 +2898,23 @@;; 32-bit target type, big-endian source value.(rule 3 (lower (has_type (gpr32_ty _ty)- (uload16 flags @ (bigendian) addr offset)))+ (uload16 _ flags @ (bigendian) addr offset)))(zext32_mem $I16 (lower_address flags addr offset)));; 32-bit target type, little-endian source value (via explicit extension).(rule 1 (lower (has_type (gpr32_ty _ty)- (uload16 flags @ (littleendian) addr offset)))+ (uload16 _ flags @ (littleendian) addr offset)))(let ((reg16 Reg (loadrev16 (lower_address flags addr offset))))(zext32_reg $I16 reg16)));; 64-bit target type, big-endian source value.(rule 4 (lower (has_type (gpr64_ty _ty)- (uload16 flags @ (bigendian) addr offset)))+ (uload16 _ flags @ (bigendian) addr offset)))(zext64_mem $I16 (lower_address flags addr offset)));; 64-bit target type, little-endian source value (via explicit extension).(rule 2 (lower (has_type (gpr64_ty _ty)- (uload16 flags @ (littleendian) addr offset)))+ (uload16 _ flags @ (littleendian) addr offset)))(let ((reg16 Reg (loadrev16 (lower_address flags addr offset))))(zext64_reg $I16 reg16)))@@ -2923,23 +2923,23 @@;; 32-bit target type, big-endian source value.(rule 2 (lower (has_type (gpr32_ty _ty)- (sload16 flags @ (bigendian) addr offset)))+ (sload16 _ flags @ (bigendian) addr offset)))(sext32_mem $I16 (lower_address flags addr offset)));; 32-bit target type, little-endian source value (via explicit extension).(rule 0 (lower (has_type (gpr32_ty _ty)- (sload16 flags @ (littleendian) addr offset)))+ (sload16 _ flags @ (littleendian) addr offset)))(let ((reg16 Reg (loadrev16 (lower_address flags addr offset))))(sext32_reg $I16 reg16)));; 64-bit target type, big-endian source value.(rule 3 (lower (has_type (gpr64_ty _ty)- (sload16 flags @ (bigendian) addr offset)))+ (sload16 _ flags @ (bigendian) addr offset)))(sext64_mem $I16 (lower_address flags addr offset)));; 64-bit target type, little-endian source value (via explicit extension).(rule 1 (lower (has_type (gpr64_ty _ty)- (sload16 flags @ (littleendian) addr offset)))+ (sload16 _ flags @ (littleendian) addr offset)))(let ((reg16 Reg (loadrev16 (lower_address flags addr offset))))(sext64_reg $I16 reg16)))@@ -2948,12 +2948,12 @@;; 64-bit target type, big-endian source value.(rule 1 (lower (has_type (gpr64_ty _ty)- (uload32 flags @ (bigendian) addr offset)))+ (uload32 _ flags @ (bigendian) addr offset)))(zext64_mem $I32 (lower_address flags addr offset)));; 64-bit target type, little-endian source value (via explicit extension).(rule (lower (has_type (gpr64_ty _ty)- (uload32 flags @ (littleendian) addr offset)))+ (uload32 _ flags @ (littleendian) addr offset)))(let ((reg32 Reg (loadrev32 (lower_address flags addr offset))))(zext64_reg $I32 reg32)))@@ -2962,12 +2962,12 @@;; 64-bit target type, big-endian source value.(rule 1 (lower (has_type (gpr64_ty _ty)- (sload32 flags @ (bigendian) addr offset)))+ (sload32 _ flags @ (bigendian) addr offset)))(sext64_mem $I32 (lower_address flags addr offset)));; 64-bit target type, little-endian source value (via explicit extension).(rule (lower (has_type (gpr64_ty _ty)- (sload32 flags @ (littleendian) addr offset)))+ (sload32 _ flags @ (littleendian) addr offset)))(let ((reg32 Reg (loadrev32 (lower_address flags addr offset))))(sext64_reg $I32 reg32)))@@ -2975,27 +2975,27 @@;;;; Rules for `uloadNxM` and `sloadNxM` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Unsigned 8->16 bit extension.-(rule (lower (has_type $I16X8 (uload8x8 flags addr offset)))+(rule (lower (has_type $I16X8 (uload8x8 _ flags addr offset)))(vec_unpacku_high $I8X16 (load_v64 $I8X16 flags addr offset)));; Signed 8->16 bit extension.-(rule (lower (has_type $I16X8 (sload8x8 flags addr offset)))+(rule (lower (has_type $I16X8 (sload8x8 _ flags addr offset)))(vec_unpacks_high $I8X16 (load_v64 $I8X16 flags addr offset)));; Unsigned 16->32 bit extension.-(rule (lower (has_type $I32X4 (uload16x4 flags addr offset)))+(rule (lower (has_type $I32X4 (uload16x4 _ flags addr offset)))(vec_unpacku_high $I16X8 (load_v64 $I16X8 flags addr offset)));; Signed 16->32 bit extension.-(rule (lower (has_type $I32X4 (sload16x4 flags addr offset)))+(rule (lower (has_type $I32X4 (sload16x4 _ flags addr offset)))(vec_unpacks_high $I16X8 (load_v64 $I16X8 flags addr offset)));; Unsigned 32->64 bit extension.-(rule (lower (has_type $I64X2 (uload32x2 flags addr offset)))+(rule (lower (has_type $I64X2 (uload32x2 _ flags addr offset)))(vec_unpacku_high $I32X4 (load_v64 $I32X4 flags addr offset)));; Signed 32->64 bit extension.-(rule (lower (has_type $I64X2 (sload32x2 flags addr offset)))+(rule (lower (has_type $I64X2 (sload32x2 _ flags addr offset)))(vec_unpacks_high $I32X4 (load_v64 $I32X4 flags addr offset)))@@ -3312,49 +3312,49 @@;; Atomic AND for 32/64-bit big-endian types, using a single instruction.(rule 1 (lower (has_type (ty_32_or_64 ty)- (atomic_rmw flags @ (bigendian) (AtomicRmwOp.And) addr src)))+ (atomic_rmw _ flags @ (bigendian) (AtomicRmwOp.And) addr src)))(atomic_rmw_and ty (put_in_reg src)(lower_address flags addr (zero_offset))));; Atomic AND for 32/64-bit big-endian types, using byte-swapped input/output.(rule (lower (has_type (ty_32_or_64 ty)- (atomic_rmw flags @ (littleendian) (AtomicRmwOp.And) addr src)))+ (atomic_rmw _ flags @ (littleendian) (AtomicRmwOp.And) addr src)))(bswap_reg ty (atomic_rmw_and ty (bswap_reg ty (put_in_reg src))(lower_address flags addr (zero_offset)))));; Atomic OR for 32/64-bit big-endian types, using a single instruction.(rule 1 (lower (has_type (ty_32_or_64 ty)- (atomic_rmw flags @ (bigendian) (AtomicRmwOp.Or) addr src)))+ (atomic_rmw _ flags @ (bigendian) (AtomicRmwOp.Or) addr src)))(atomic_rmw_or ty (put_in_reg src)(lower_address flags addr (zero_offset))));; Atomic OR for 32/64-bit little-endian types, using byte-swapped input/output.(rule (lower (has_type (ty_32_or_64 ty)- (atomic_rmw flags @ (littleendian) (AtomicRmwOp.Or) addr src)))+ (atomic_rmw _ flags @ (littleendian) (AtomicRmwOp.Or) addr src)))(bswap_reg ty (atomic_rmw_or ty (bswap_reg ty (put_in_reg src))(lower_address flags addr (zero_offset)))));; Atomic XOR for 32/64-bit big-endian types, using a single instruction.(rule 1 (lower (has_type (ty_32_or_64 ty)- (atomic_rmw flags @ (bigendian) (AtomicRmwOp.Xor) addr src)))+ (atomic_rmw _ flags @ (bigendian) (AtomicRmwOp.Xor) addr src)))(atomic_rmw_xor ty (put_in_reg src)(lower_address flags addr (zero_offset))));; Atomic XOR for 32/64-bit little-endian types, using byte-swapped input/output.(rule (lower (has_type (ty_32_or_64 ty)- (atomic_rmw flags @ (littleendian) (AtomicRmwOp.Xor) addr src)))+ (atomic_rmw _ flags @ (littleendian) (AtomicRmwOp.Xor) addr src)))(bswap_reg ty (atomic_rmw_xor ty (bswap_reg ty (put_in_reg src))(lower_address flags addr (zero_offset)))));; Atomic ADD for 32/64-bit big-endian types, using a single instruction.(rule (lower (has_type (ty_32_or_64 ty)- (atomic_rmw flags @ (bigendian) (AtomicRmwOp.Add) addr src)))+ (atomic_rmw _ flags @ (bigendian) (AtomicRmwOp.Add) addr src)))(atomic_rmw_add ty (put_in_reg src)(lower_address flags addr (zero_offset))));; Atomic SUB for 32/64-bit big-endian types, using atomic ADD with negated input.(rule (lower (has_type (ty_32_or_64 ty)- (atomic_rmw flags @ (bigendian) (AtomicRmwOp.Sub) addr src)))+ (atomic_rmw _ flags @ (bigendian) (AtomicRmwOp.Sub) addr src)))(atomic_rmw_add ty (neg_reg ty (put_in_reg src))(lower_address flags addr (zero_offset))))@@ -3362,7 +3362,7 @@;; Atomic operations that require a compare-and-swap loop.;; Operations for 32/64-bit types can use a fullword compare-and-swap loop.-(rule -1 (lower (has_type (ty_32_or_64 ty) (atomic_rmw flags op addr src)))+(rule -1 (lower (has_type (ty_32_or_64 ty) (atomic_rmw _ flags op addr src)))(let ((src_reg Reg (put_in_reg src))(addr_reg Reg (put_in_reg addr));; Create body of compare-and-swap loop.@@ -3374,7 +3374,7 @@(casloop ib ty flags addr_reg val1)));; Operations for 8/16-bit types must operate on the surrounding aligned word.-(rule -2 (lower (has_type (ty_8_or_16 ty) (atomic_rmw flags op addr src)))+(rule -2 (lower (has_type (ty_8_or_16 ty) (atomic_rmw _ flags op addr src)))(let ((src_reg Reg (put_in_reg src))(addr_reg Reg (put_in_reg addr));; Prepare access to surrounding aligned word.@@ -3568,20 +3568,20 @@;; 32/64-bit big-endian atomic compare-and-swap instruction.(rule 2 (lower (has_type (ty_32_or_64 ty)- (atomic_cas flags @ (bigendian) addr src1 src2)))+ (atomic_cas _ flags @ (bigendian) addr src1 src2)))(atomic_cas_impl ty (put_in_reg src1) (put_in_reg src2)(lower_address flags addr (zero_offset))));; 32/64-bit little-endian atomic compare-and-swap instruction.;; Implemented by byte-swapping old/new inputs and the output.(rule 1 (lower (has_type (ty_32_or_64 ty)- (atomic_cas flags @ (littleendian) addr src1 src2)))+ (atomic_cas _ flags @ (littleendian) addr src1 src2)))(bswap_reg ty (atomic_cas_impl ty (bswap_reg ty (put_in_reg src1))(bswap_reg ty (put_in_reg src2))(lower_address flags addr (zero_offset)))));; 8/16-bit atomic compare-and-swap implemented via loop.-(rule (lower (has_type (ty_8_or_16 ty) (atomic_cas flags addr src1 src2)))+(rule (lower (has_type (ty_8_or_16 ty) (atomic_cas _ flags addr src1 src2)))(let ((src1_reg Reg (put_in_reg src1))(src2_reg Reg (put_in_reg src2))(addr_reg Reg (put_in_reg addr))@@ -3638,31 +3638,31 @@;; Atomic loads can be implemented via regular loads on this platform.;; 8-bit atomic load.-(rule (lower (has_type $I8 (atomic_load flags addr)))+(rule (lower (has_type $I8 (atomic_load _ flags addr)))(zext32_mem $I8 (lower_address flags addr (zero_offset))));; 16-bit big-endian atomic load.-(rule 1 (lower (has_type $I16 (atomic_load flags @ (bigendian) addr)))+(rule 1 (lower (has_type $I16 (atomic_load _ flags @ (bigendian) addr)))(zext32_mem $I16 (lower_address flags addr (zero_offset))));; 16-bit little-endian atomic load.-(rule (lower (has_type $I16 (atomic_load flags @ (littleendian) addr)))+(rule (lower (has_type $I16 (atomic_load _ flags @ (littleendian) addr)))(loadrev16 (lower_address flags addr (zero_offset))));; 32-bit big-endian atomic load.-(rule 1 (lower (has_type $I32 (atomic_load flags @ (bigendian) addr)))+(rule 1 (lower (has_type $I32 (atomic_load _ flags @ (bigendian) addr)))(load32 (lower_address flags addr (zero_offset))));; 32-bit little-endian atomic load.-(rule (lower (has_type $I32 (atomic_load flags @ (littleendian) addr)))+(rule (lower (has_type $I32 (atomic_load _ flags @ (littleendian) addr)))(loadrev32 (lower_address flags addr (zero_offset))));; 64-bit big-endian atomic load.-(rule 1 (lower (has_type $I64 (atomic_load flags @ (bigendian) addr)))+(rule 1 (lower (has_type $I64 (atomic_load _ flags @ (bigendian) addr)))(load64 (lower_address flags addr (zero_offset))));; 64-bit little-endian atomic load.-(rule (lower (has_type $I64 (atomic_load flags @ (littleendian) addr)))+(rule (lower (has_type $I64 (atomic_load _ flags @ (littleendian) addr)))(loadrev64 (lower_address flags addr (zero_offset))))@@ -3727,7 +3727,7 @@;; Main `icmp` entry point. Generate a `ProducesBool` capturing the;; integer comparison and immediately lower it to a 0/1 integer result.;; In this case, it is safe to sink memory loads.-(rule -1 (lower (has_type (fits_in_64 ty) (icmp int_cc x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (icmp _ int_cc x y)))(lower_bool ty (icmp_val true int_cc x y)))@@ -3857,25 +3857,25 @@;; Vector `icmp` produces a boolean vector.;; We need to handle the various IntCC flags separately here.-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.Equal) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.Equal) x y)))(vec_cmpeq ty x y))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.NotEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.NotEqual) x y)))(vec_not ty (vec_cmpeq ty x y)))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.SignedGreaterThan) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.SignedGreaterThan) x y)))(vec_cmph ty x y))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.SignedLessThanOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.SignedLessThanOrEqual) x y)))(vec_not ty (vec_cmph ty x y)))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.SignedLessThan) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.SignedLessThan) x y)))(vec_cmph ty y x))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.SignedGreaterThanOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.SignedGreaterThanOrEqual) x y)))(vec_not ty (vec_cmph ty y x)))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.UnsignedGreaterThan) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.UnsignedGreaterThan) x y)))(vec_cmphl ty x y))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.UnsignedLessThanOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.UnsignedLessThanOrEqual) x y)))(vec_not ty (vec_cmphl ty x y)))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.UnsignedLessThan) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.UnsignedLessThan) x y)))(vec_cmphl ty y x))-(rule (lower (has_type (ty_vec128 ty) (icmp (IntCC.UnsignedGreaterThanOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (icmp _ (IntCC.UnsignedGreaterThanOrEqual) x y)))(vec_not ty (vec_cmphl ty y x)))@@ -3883,7 +3883,7 @@;; Main `fcmp` entry point. Generate a `ProducesBool` capturing the;; integer comparison and immediately lower it to a 0/1 integer result.-(rule -1 (lower (has_type (fits_in_64 ty) (fcmp float_cc x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (fcmp _ float_cc x y)))(lower_bool ty (fcmp_val float_cc x y)));; Return a `ProducesBool` to implement any floating-point comparison.@@ -3895,33 +3895,33 @@;; Vector `fcmp` produces a boolean vector.;; We need to handle the various FloatCC flags separately here.-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.Equal) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.Equal) x y)))(vec_fcmpeq ty x y))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.NotEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.NotEqual) x y)))(vec_not ty (vec_fcmpeq ty x y)))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.GreaterThan) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.GreaterThan) x y)))(vec_fcmph ty x y))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrLessThanOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.UnorderedOrLessThanOrEqual) x y)))(vec_not ty (vec_fcmph ty x y)))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.GreaterThanOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.GreaterThanOrEqual) x y)))(vec_fcmphe ty x y))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrLessThan) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.UnorderedOrLessThan) x y)))(vec_not ty (vec_fcmphe ty x y)))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.LessThan) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.LessThan) x y)))(vec_fcmph ty y x))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))(vec_not ty (vec_fcmph ty y x)))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.LessThanOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.LessThanOrEqual) x y)))(vec_fcmphe ty y x))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrGreaterThan) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.UnorderedOrGreaterThan) x y)))(vec_not ty (vec_fcmphe ty y x)))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.Ordered) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.Ordered) x y)))(vec_or ty (vec_fcmphe ty x y) (vec_fcmphe ty y x)))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.Unordered) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.Unordered) x y)))(vec_not_or ty (vec_fcmphe ty x y) (vec_fcmphe ty y x)))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.OrderedNotEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.OrderedNotEqual) x y)))(vec_or ty (vec_fcmph ty x y) (vec_fcmph ty y x)))-(rule (lower (has_type (ty_vec128 ty) (fcmp (FloatCC.UnorderedOrEqual) x y)))+(rule (lower (has_type (ty_vec128 ty) (fcmp _ (FloatCC.UnorderedOrEqual) x y)))(vec_not_or ty (vec_fcmph ty x y) (vec_fcmph ty y x)))@@ -3929,7 +3929,7 @@;; Main `vall_true` entry point. Generate a `ProducesBool` capturing the;; comparison and immediately lower it to a 0/1 integer result.-(rule (lower (has_type (fits_in_64 ty) (vall_true x)))+(rule (lower (has_type (fits_in_64 ty) (vall_true _ x)))(lower_bool ty (vall_true_val x)));; Return a `ProducesBool` to implement `vall_true`.@@ -3939,66 +3939,66 @@(floatcc_as_cond (FloatCC.Unordered))));; Short-circuit `vall_true` on the result of a `icmp`.-(rule (vall_true_val (has_type ty (icmp (IntCC.Equal) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.Equal) x y)))(bool (vec_cmpeqs ty x y)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (icmp (IntCC.NotEqual) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.NotEqual) x y)))(bool (vec_cmpeqs ty x y)(floatcc_as_cond (FloatCC.Unordered))))-(rule (vall_true_val (has_type ty (icmp (IntCC.SignedGreaterThan) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.SignedGreaterThan) x y)))(bool (vec_cmphs ty x y)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (icmp (IntCC.SignedLessThanOrEqual) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.SignedLessThanOrEqual) x y)))(bool (vec_cmphs ty x y)(floatcc_as_cond (FloatCC.Unordered))))-(rule (vall_true_val (has_type ty (icmp (IntCC.SignedLessThan) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.SignedLessThan) x y)))(bool (vec_cmphs ty y x)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (icmp (IntCC.SignedGreaterThanOrEqual) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.SignedGreaterThanOrEqual) x y)))(bool (vec_cmphs ty y x)(floatcc_as_cond (FloatCC.Unordered))))-(rule (vall_true_val (has_type ty (icmp (IntCC.UnsignedGreaterThan) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.UnsignedGreaterThan) x y)))(bool (vec_cmphls ty x y)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (icmp (IntCC.UnsignedLessThanOrEqual) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.UnsignedLessThanOrEqual) x y)))(bool (vec_cmphls ty x y)(floatcc_as_cond (FloatCC.Unordered))))-(rule (vall_true_val (has_type ty (icmp (IntCC.UnsignedLessThan) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.UnsignedLessThan) x y)))(bool (vec_cmphls ty y x)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (icmp (IntCC.UnsignedGreaterThanOrEqual) x y)))+(rule (vall_true_val (has_type ty (icmp _ (IntCC.UnsignedGreaterThanOrEqual) x y)))(bool (vec_cmphls ty y x)(floatcc_as_cond (FloatCC.Unordered))));; Short-circuit `vall_true` on the result of a `fcmp` where possible.-(rule (vall_true_val (has_type ty (fcmp (FloatCC.Equal) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.Equal) x y)))(bool (vec_fcmpeqs ty x y)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.NotEqual) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.NotEqual) x y)))(bool (vec_fcmpeqs ty x y)(floatcc_as_cond (FloatCC.Unordered))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.GreaterThan) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.GreaterThan) x y)))(bool (vec_fcmphs ty x y)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.UnorderedOrLessThanOrEqual) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.UnorderedOrLessThanOrEqual) x y)))(bool (vec_fcmphs ty x y)(floatcc_as_cond (FloatCC.Unordered))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.GreaterThanOrEqual) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.GreaterThanOrEqual) x y)))(bool (vec_fcmphes ty x y)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.UnorderedOrLessThan) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.UnorderedOrLessThan) x y)))(bool (vec_fcmphes ty x y)(floatcc_as_cond (FloatCC.Unordered))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.LessThan) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.LessThan) x y)))(bool (vec_fcmphs ty y x)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))(bool (vec_fcmphs ty y x)(floatcc_as_cond (FloatCC.Unordered))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.LessThanOrEqual) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.LessThanOrEqual) x y)))(bool (vec_fcmphes ty y x)(floatcc_as_cond (FloatCC.Equal))))-(rule (vall_true_val (has_type ty (fcmp (FloatCC.UnorderedOrGreaterThan) x y)))+(rule (vall_true_val (has_type ty (fcmp _ (FloatCC.UnorderedOrGreaterThan) x y)))(bool (vec_fcmphes ty y x)(floatcc_as_cond (FloatCC.Unordered))))@@ -4007,7 +4007,7 @@;; Main `vany_true` entry point. Generate a `ProducesBool` capturing the;; comparison and immediately lower it to a 0/1 integer result.-(rule (lower (has_type (fits_in_64 ty) (vany_true x)))+(rule (lower (has_type (fits_in_64 ty) (vany_true _ x)))(lower_bool ty (vany_true_val x)));; Return a `ProducesBool` to implement `vany_true`.@@ -4017,111 +4017,111 @@(floatcc_as_cond (FloatCC.NotEqual))));; Short-circuit `vany_true` on the result of a `icmp`.-(rule (vany_true_val (has_type ty (icmp (IntCC.Equal) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.Equal) x y)))(bool (vec_cmpeqs ty x y)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (icmp (IntCC.NotEqual) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.NotEqual) x y)))(bool (vec_cmpeqs ty x y)(floatcc_as_cond (FloatCC.NotEqual))))-(rule (vany_true_val (has_type ty (icmp (IntCC.SignedGreaterThan) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.SignedGreaterThan) x y)))(bool (vec_cmphs ty x y)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (icmp (IntCC.SignedLessThanOrEqual) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.SignedLessThanOrEqual) x y)))(bool (vec_cmphs ty x y)(floatcc_as_cond (FloatCC.NotEqual))))-(rule (vany_true_val (has_type ty (icmp (IntCC.SignedLessThan) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.SignedLessThan) x y)))(bool (vec_cmphs ty y x)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (icmp (IntCC.SignedGreaterThanOrEqual) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.SignedGreaterThanOrEqual) x y)))(bool (vec_cmphs ty y x)(floatcc_as_cond (FloatCC.NotEqual))))-(rule (vany_true_val (has_type ty (icmp (IntCC.UnsignedGreaterThan) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.UnsignedGreaterThan) x y)))(bool (vec_cmphls ty x y)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (icmp (IntCC.UnsignedLessThanOrEqual) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.UnsignedLessThanOrEqual) x y)))(bool (vec_cmphls ty x y)(floatcc_as_cond (FloatCC.NotEqual))))-(rule (vany_true_val (has_type ty (icmp (IntCC.UnsignedLessThan) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.UnsignedLessThan) x y)))(bool (vec_cmphls ty y x)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (icmp (IntCC.UnsignedGreaterThanOrEqual) x y)))+(rule (vany_true_val (has_type ty (icmp _ (IntCC.UnsignedGreaterThanOrEqual) x y)))(bool (vec_cmphls ty y x)(floatcc_as_cond (FloatCC.NotEqual))));; Short-circuit `vany_true` on the result of a `fcmp` where possible.-(rule (vany_true_val (has_type ty (fcmp (FloatCC.Equal) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.Equal) x y)))(bool (vec_fcmpeqs ty x y)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.NotEqual) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.NotEqual) x y)))(bool (vec_fcmpeqs ty x y)(floatcc_as_cond (FloatCC.NotEqual))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.GreaterThan) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.GreaterThan) x y)))(bool (vec_fcmphs ty x y)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.UnorderedOrLessThanOrEqual) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.UnorderedOrLessThanOrEqual) x y)))(bool (vec_fcmphs ty x y)(floatcc_as_cond (FloatCC.NotEqual))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.GreaterThanOrEqual) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.GreaterThanOrEqual) x y)))(bool (vec_fcmphes ty x y)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.UnorderedOrLessThan) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.UnorderedOrLessThan) x y)))(bool (vec_fcmphes ty x y)(floatcc_as_cond (FloatCC.NotEqual))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.LessThan) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.LessThan) x y)))(bool (vec_fcmphs ty y x)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.UnorderedOrGreaterThanOrEqual) x y)))(bool (vec_fcmphs ty y x)(floatcc_as_cond (FloatCC.NotEqual))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.LessThanOrEqual) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.LessThanOrEqual) x y)))(bool (vec_fcmphes ty y x)(floatcc_as_cond (FloatCC.Ordered))))-(rule (vany_true_val (has_type ty (fcmp (FloatCC.UnorderedOrGreaterThan) x y)))+(rule (vany_true_val (has_type ty (fcmp _ (FloatCC.UnorderedOrGreaterThan) x y)))(bool (vec_fcmphes ty y x)(floatcc_as_cond (FloatCC.NotEqual))));;;; Rules for `vhigh_bits` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (vhigh_bits x @ (value_type (multi_lane 8 16))))+(rule (lower (vhigh_bits _ x @ (value_type (multi_lane 8 16))))(if-let (LaneOrder.LittleEndian) (lane_order))(let ((mask Reg (vec_imm $I8X16 (imm8x16 0 8 16 24 32 40 48 5664 72 80 88 96 104 112 120))))(vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))-(rule 1 (lower (vhigh_bits x @ (value_type (multi_lane 8 16))))+(rule 1 (lower (vhigh_bits _ x @ (value_type (multi_lane 8 16))))(if-let (LaneOrder.BigEndian) (lane_order))(let ((mask Reg (vec_imm $I8X16 (imm8x16 120 112 104 96 88 80 72 6456 48 40 32 24 16 8 0))))(vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))-(rule (lower (vhigh_bits x @ (value_type (multi_lane 16 8))))+(rule (lower (vhigh_bits _ x @ (value_type (multi_lane 16 8))))(if-let (LaneOrder.LittleEndian) (lane_order))(let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 1280 16 32 48 64 80 96 112))))(vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))-(rule 1 (lower (vhigh_bits x @ (value_type (multi_lane 16 8))))+(rule 1 (lower (vhigh_bits _ x @ (value_type (multi_lane 16 8))))(if-let (LaneOrder.BigEndian) (lane_order))(let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 128112 96 80 64 48 32 16 0))))(vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))-(rule (lower (vhigh_bits x @ (value_type (multi_lane 32 4))))+(rule (lower (vhigh_bits _ x @ (value_type (multi_lane 32 4))))(if-let (LaneOrder.LittleEndian) (lane_order))(let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 128128 128 128 128 0 32 64 96))))(vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))-(rule 1 (lower (vhigh_bits x @ (value_type (multi_lane 32 4))))+(rule 1 (lower (vhigh_bits _ x @ (value_type (multi_lane 32 4))))(if-let (LaneOrder.BigEndian) (lane_order))(let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 128128 128 128 128 96 64 32 0))))(vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))-(rule (lower (vhigh_bits x @ (value_type (multi_lane 64 2))))+(rule (lower (vhigh_bits _ x @ (value_type (multi_lane 64 2))))(if-let (LaneOrder.LittleEndian) (lane_order))(let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 128128 128 128 128 128 128 0 64))))(vec_extract_lane $I64X2 (vec_bitpermute x mask) 0 (zero_reg))))-(rule 1 (lower (vhigh_bits x @ (value_type (multi_lane 64 2))))+(rule 1 (lower (vhigh_bits _ x @ (value_type (multi_lane 64 2))))(if-let (LaneOrder.BigEndian) (lane_order))(let ((mask Reg (vec_imm $I8X16 (imm8x16 128 128 128 128 128 128 128 128128 128 128 128 128 128 64 0))))@@ -4135,8 +4135,8 @@;; instruction, directly use that compare. Note that it is not safe to sink;; memory loads here, see the `icmp` comment.(decl value_nonzero (Value) ProducesBool)-(rule (value_nonzero (icmp int_cc x y)) (icmp_val false int_cc x y))-(rule (value_nonzero (fcmp float_cc x y)) (fcmp_val float_cc x y))+(rule (value_nonzero (icmp _ int_cc x y)) (icmp_val false int_cc x y))+(rule (value_nonzero (fcmp _ float_cc x y)) (fcmp_val float_cc x y))(rule -1 (value_nonzero val @ (value_type (gpr32_ty ty)))(bool (icmps_simm16 $I32 (put_in_reg_sext32 val) 0)(intcc_as_cond (IntCC.NotEqual))))@@ -4148,14 +4148,14 @@(floatcc_as_cond (FloatCC.NotEqual))));; Main `select` entry point. Lower the `value_nonzero` result.-(rule (lower (has_type ty (select val_cond val_true val_false)))+(rule (lower (has_type ty (select _ val_cond val_true val_false)))(select_bool_reg ty (value_nonzero val_cond)(put_in_reg val_true) (put_in_reg val_false)));; Special-case some float-selection instructions for min/max-(rule 1 (lower (has_type (ty_scalar_float ty) (select (maybe_uextend (fcmp (FloatCC.LessThan) x y)) x y)))+(rule 1 (lower (has_type (ty_scalar_float ty) (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) x y)) x y)))(fmin_pseudo_reg ty y x))-(rule 2 (lower (has_type (ty_scalar_float ty) (select (maybe_uextend (fcmp (FloatCC.LessThan) y x)) x y)))+(rule 2 (lower (has_type (ty_scalar_float ty) (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) y x)) x y)))(fmax_pseudo_reg ty y x))@@ -4164,7 +4164,7 @@;; We need to guarantee a conditional move instruction. But on this platform;; this is already the best way to implement select in general, so the;; implementation of `select_spectre_guard` is identical to `select`.-(rule (lower (has_type ty (select_spectre_guard+(rule (lower (has_type ty (select_spectre_guard _val_cond val_true val_false)))(select_bool_reg ty (value_nonzero val_cond)(put_in_reg val_true) (put_in_reg val_false)))@@ -4256,62 +4256,62 @@;; remap the IntCC::UnsignedGreaterThan value that we have here as result;; of the unsigned_add_overflow_condition call to the correct mask.-(rule 0 (lower (has_type (fits_in_64 ty) (uadd_overflow_trap x y tc)))+(rule 0 (lower (has_type (fits_in_64 ty) (uadd_overflow_trap _ x y tc)))(with_flags(add_logical_reg_with_flags_paired ty x y)(trap_if_impl (mask_as_cond 3) tc)));; Add a register an a zero-extended register.(rule 4 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap x (zext32_value y) tc)))+ (uadd_overflow_trap _ x (zext32_value y) tc)))(with_flags(add_logical_reg_zext32_with_flags_paired ty x y)(trap_if_impl (mask_as_cond 3) tc)))(rule 8 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap (zext32_value x) y tc)))+ (uadd_overflow_trap _ (zext32_value x) y tc)))(with_flags(add_logical_reg_zext32_with_flags_paired ty y x)(trap_if_impl (mask_as_cond 3) tc)));; Add a register and an immediate(rule 3 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap x (u32_from_value y) tc)))+ (uadd_overflow_trap _ x (u32_from_value y) tc)))(with_flags(add_logical_zimm32_with_flags_paired ty x y)(trap_if_impl (mask_as_cond 3) tc)))(rule 7 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap (u32_from_value x) y tc)))+ (uadd_overflow_trap _ (u32_from_value x) y tc)))(with_flags(add_logical_zimm32_with_flags_paired ty y x)(trap_if_impl (mask_as_cond 3) tc)));; Add a register and memory (32/64-bit types).(rule 2 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap x (sinkable_load_32_64 y) tc)))+ (uadd_overflow_trap _ x (sinkable_load_32_64 y) tc)))(with_flags(add_logical_mem_with_flags_paired ty x (sink_load y))(trap_if_impl (mask_as_cond 3) tc)))(rule 6 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap (sinkable_load_32_64 x) y tc)))+ (uadd_overflow_trap _ (sinkable_load_32_64 x) y tc)))(with_flags(add_logical_mem_with_flags_paired ty y (sink_load x))(trap_if_impl (mask_as_cond 3) tc)));; Add a register and zero-extended memory.(rule 1 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap x (sinkable_uload32 y) tc)))+ (uadd_overflow_trap _ x (sinkable_uload32 y) tc)))(with_flags(add_logical_mem_zext32_with_flags_paired ty x (sink_uload32 y))(trap_if_impl (mask_as_cond 3) tc)))(rule 5 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap (sinkable_uload32 x) y tc)))+ (uadd_overflow_trap _ (sinkable_uload32 x) y tc)))(with_flags(add_logical_mem_zext32_with_flags_paired ty y (sink_uload32 x))(trap_if_impl (mask_as_cond 3) tc)));;;; Rules for `uadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_32_or_64 ty) (uadd_overflow x y)))+(rule 0 (lower (has_type (ty_32_or_64 ty) (uadd_overflow _ x y)))(let ((sum Reg (add_reg ty x y))(overflow Reg(lower_bool $I8@@ -4428,18 +4428,18 @@;;;; Rules for `get_{frame,stack}_pointer` and `get_return_address` ;;;;;;;;;;;;-(rule (lower (get_stack_pointer))+(rule (lower (get_stack_pointer _))(sp))-(rule (lower (get_frame_pointer))+(rule (lower (get_frame_pointer _))(load64 (memarg_frame_pointer_offset)))-(rule (lower (get_return_address))+(rule (lower (get_return_address _))(load64 (memarg_return_address_offset)));; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))+(rule (lower (get_exception_handler_address _ (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(s390x_label_address succ_label)))diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isleindex ef058fb988..813d087a72 100644--- a/cranelift/codegen/src/isa/x64/inst.isle+++ b/cranelift/codegen/src/isa/x64/inst.isle@@ -509,11 +509,11 @@(decl to_amode (MemFlags Value Offset32) SyntheticAmode)(rule 0 (to_amode flags base offset)(amode_imm_reg flags base offset))-(rule 1 (to_amode flags (iadd x y) offset)+(rule 1 (to_amode flags (iadd _ x y) offset)(to_amode_add flags x y offset))(rule 2- (to_amode flags (stack_addr slot offset1) offset2)+ (to_amode flags (stack_addr _ slot offset1) offset2)(stackslot_amode slot offset1 offset2));; Same as `to_amode`, except that the base address is computed via the addition@@ -542,16 +542,16 @@(rule to_amode_add_const_lhs 2 (to_amode_add flags (i32_from_iconst c) x offset)(if-let sum (i32_checked_add offset c))(amode_imm_reg flags x sum))-(rule to_amode_add_const_fold_iadd_lhs_rhs 3 (to_amode_add flags (iadd x (i32_from_iconst c)) y offset)+(rule to_amode_add_const_fold_iadd_lhs_rhs 3 (to_amode_add flags (iadd _ x (i32_from_iconst c)) y offset)(if-let sum (i32_checked_add offset c))(amode_imm_reg_reg_shift flags x y sum))-(rule to_amode_add_const_fold_iadd_lhs_lhs 4 (to_amode_add flags (iadd (i32_from_iconst c) x) y offset)+(rule to_amode_add_const_fold_iadd_lhs_lhs 4 (to_amode_add flags (iadd _ (i32_from_iconst c) x) y offset)(if-let sum (i32_checked_add offset c))(amode_imm_reg_reg_shift flags x y sum))-(rule to_amode_add_const_fold_iadd_rhs_rhs 5 (to_amode_add flags x (iadd y (i32_from_iconst c)) offset)+(rule to_amode_add_const_fold_iadd_rhs_rhs 5 (to_amode_add flags x (iadd _ y (i32_from_iconst c)) offset)(if-let sum (i32_checked_add offset c))(amode_imm_reg_reg_shift flags x y sum))-(rule to_amode_add_const_fold_iadd_rhs_lhs 6 (to_amode_add flags x (iadd (i32_from_iconst c) y) offset)+(rule to_amode_add_const_fold_iadd_rhs_lhs 6 (to_amode_add flags x (iadd _ (i32_from_iconst c) y) offset)(if-let sum (i32_checked_add offset c))(amode_imm_reg_reg_shift flags x y sum))@@ -565,7 +565,7 @@(decl amode_imm_reg (MemFlags Value Offset32) Amode)(rule amode_imm_reg_base 0 (amode_imm_reg flags base offset)(Amode.ImmReg offset base flags))-(rule amode_imm_reg_iadd 1 (amode_imm_reg flags (iadd x y) offset)+(rule amode_imm_reg_iadd 1 (amode_imm_reg flags (iadd _ x y) offset)(amode_imm_reg_reg_shift flags x y offset))(spec (amode_imm_reg_reg_shift flags x y offset)@@ -579,10 +579,10 @@(decl amode_imm_reg_reg_shift (MemFlags Value Value Offset32) Amode)(rule amode_imm_reg_reg_shift_no_shift 0 (amode_imm_reg_reg_shift flags x y offset)(Amode.ImmRegRegShift offset x y 0 flags)) ;; 0 == y<<0 == "no shift"-(rule amode_imm_reg_reg_shift_shl_rhs 1 (amode_imm_reg_reg_shift flags x (ishl y (iconst (uimm8 shift))) offset)+(rule amode_imm_reg_reg_shift_shl_rhs 1 (amode_imm_reg_reg_shift flags x (ishl _ y (iconst _ (uimm8 shift))) offset)(if-let true (u32_lt_eq shift 3))(Amode.ImmRegRegShift offset x y shift flags))-(rule amode_imm_reg_reg_shift_shl_lhs 2 (amode_imm_reg_reg_shift flags (ishl y (iconst (uimm8 shift))) x offset)+(rule amode_imm_reg_reg_shift_shl_lhs 2 (amode_imm_reg_reg_shift flags (ishl _ y (iconst _ (uimm8 shift))) x offset)(if-let true (u32_lt_eq shift 3))(Amode.ImmRegRegShift offset x y shift flags))@@ -1258,15 +1258,15 @@;; be a 32-bit operation, will automatically zero the upper 32-bits of the;; destination register that `Value` is placed in.(decl pure value32_zeros_upper32 (Value) bool)-(rule (value32_zeros_upper32 (iadd _ _)) true)-(rule (value32_zeros_upper32 (isub _ _)) true)-(rule (value32_zeros_upper32 (imul _ _)) true)-(rule (value32_zeros_upper32 (band _ _)) true)-(rule (value32_zeros_upper32 (bor _ _)) true)-(rule (value32_zeros_upper32 (bxor _ _)) true)-(rule (value32_zeros_upper32 (ishl _ _)) true)-(rule (value32_zeros_upper32 (ushr _ _)) true)-(rule (value32_zeros_upper32 (uload32 _ _ _)) true)+(rule (value32_zeros_upper32 (iadd _ _ _)) true)+(rule (value32_zeros_upper32 (isub _ _ _)) true)+(rule (value32_zeros_upper32 (imul _ _ _)) true)+(rule (value32_zeros_upper32 (band _ _ _)) true)+(rule (value32_zeros_upper32 (bor _ _ _)) true)+(rule (value32_zeros_upper32 (bxor _ _ _)) true)+(rule (value32_zeros_upper32 (ishl _ _ _)) true)+(rule (value32_zeros_upper32 (ushr _ _ _)) true)+(rule (value32_zeros_upper32 (uload32 _ _ _ _)) true)(rule -1 (value32_zeros_upper32 _) false);;;; Helpers for Working SSE tidbits ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3442,11 +3442,11 @@;; Special case some instructions where lowerings directly produce condition;; codes.-(rule 2 (is_nonzero (vall_true vec)) (is_vall_true vec))-(rule 2 (is_nonzero (vany_true vec)) (is_vany_true vec))-(rule 2 (is_nonzero (uextend (vall_true vec))) (is_vall_true vec))-(rule 2 (is_nonzero (uextend (vany_true vec))) (is_vany_true vec))-(rule 2 (is_nonzero (band a @ (value_type (ty_int (fits_in_64 ty))) b))+(rule 2 (is_nonzero (vall_true _ vec)) (is_vall_true vec))+(rule 2 (is_nonzero (vany_true _ vec)) (is_vany_true vec))+(rule 2 (is_nonzero (uextend _ (vall_true _ vec))) (is_vall_true vec))+(rule 2 (is_nonzero (uextend _ (vany_true _ vec))) (is_vany_true vec))+(rule 2 (is_nonzero (band _ a @ (value_type (ty_int (fits_in_64 ty))) b))(is_nonzero_band ty a b))@@ -3455,10 +3455,10 @@;; avoid unbounded recursion.(decl is_nonzero_cmp (Value) CondResult)-(rule 1 (is_nonzero_cmp (fcmp cc a b)) (emit_fcmp cc a b))-(rule 1 (is_nonzero_cmp (icmp cc a b)) (emit_cmp cc a b))-(rule 1 (is_nonzero_cmp (uextend (fcmp cc a b))) (emit_fcmp cc a b))-(rule 1 (is_nonzero_cmp (uextend (icmp cc a b))) (emit_cmp cc a b))+(rule 1 (is_nonzero_cmp (fcmp _ cc a b)) (emit_fcmp cc a b))+(rule 1 (is_nonzero_cmp (icmp _ cc a b)) (emit_cmp cc a b))+(rule 1 (is_nonzero_cmp (uextend _ (fcmp _ cc a b))) (emit_fcmp cc a b))+(rule 1 (is_nonzero_cmp (uextend _ (icmp _ cc a b))) (emit_cmp cc a b))(rule 0 (is_nonzero_cmp val) (is_nonzero val))(decl is_nonzero_band (Type Value Value) CondResult)@@ -3469,7 +3469,7 @@;; same semantics this requires that `a` is in a register which forces `bt` to;; use modulo semantics for the second operand `b`, thus `put_in_gpr` is;; manually used.-(rule 1 (is_nonzero_band (ty_32_or_64 ty) a (ishl (u64_from_iconst 1) b))+(rule 1 (is_nonzero_band (ty_32_or_64 ty) a (ishl _ (u64_from_iconst 1) b))(CondResult.CC (x64_bt ty (put_in_gpr a) b) (CC.B)));; If a value is and'd one shifted by a variable value that matches `bt` asdiff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex 3d09b9931d..1ce3df5753 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -15,36 +15,36 @@;; `i64` and smaller.(rule (lower (has_type (fits_in_64 ty)- (iconst (u64_from_imm64 x))))+ (iconst _ (u64_from_imm64 x))))(imm ty x));; `i128`(rule 1 (lower (has_type $I128- (iconst (u64_from_imm64 x))))+ (iconst _ (u64_from_imm64 x))))(value_regs (imm $I64 x)(imm $I64 0)));;;; Rules for `f16const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f16const (u16_from_ieee16 x)))+(rule (lower (f16const _ (u16_from_ieee16 x)))(imm $F16 x));;;; Rules for `f32const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f32const (u32_from_ieee32 x)))+(rule (lower (f32const _ (u32_from_ieee32 x)))(imm $F32 x));;;; Rules for `f64const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f64const (u64_from_ieee64 x)))+(rule (lower (f64const _ (u64_from_ieee64 x)))(imm $F64 x));;;; Rules for `f128const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (f128const const))+(rule (lower (f128const _ const));; TODO use Inst::gen_constant() instead.(x64_xmm_load_const $F128 (const_to_vconst const)))-(rule 1 (lower (f128const (u128_from_constant 0)))+(rule 1 (lower (f128const _ (u128_from_constant 0)))(xmm_zero $F128));;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -53,7 +53,7 @@;; Base case for 8 and 16-bit types(rule -6 (lower (has_type (fits_in_16 ty)- (iadd x y)))+ (iadd _ x y)))(x64_add ty x y));; Base case for 32 and 64-bit types which might end up using the `lea`@@ -63,39 +63,39 @@;; but the actual instruction emitted might be an `add` if it's equivalent.;; For more details on this see the `emit.rs` logic to emit;; `LoadEffectiveAddress`.-(rule iadd_base_case_32_or_64_lea -5 (lower (has_type (ty_32_or_64 ty) (iadd x y)))+(rule iadd_base_case_32_or_64_lea -5 (lower (has_type (ty_32_or_64 ty) (iadd _ x y)))(x64_lea ty (to_amode_add (mem_flags_trusted) x y (zero_offset))));; Higher-priority cases than the previous two where a load can be sunk into;; the add instruction itself. Note that both operands are tested for;; sink-ability since addition is commutative(rule -4 (lower (has_type (fits_in_64 ty)- (iadd x (sinkable_load y))))+ (iadd _ x (sinkable_load y))))(x64_add ty x y))(rule -3 (lower (has_type (fits_in_64 ty)- (iadd (sinkable_load x) y)))+ (iadd _ (sinkable_load x) y)))(x64_add ty y x));; SSE.(rule (lower (has_type (multi_lane 8 16)- (iadd x y)))+ (iadd _ x y)))(x64_paddb x y))(rule (lower (has_type (multi_lane 16 8)- (iadd x y)))+ (iadd _ x y)))(x64_paddw x y))(rule (lower (has_type (multi_lane 32 4)- (iadd x y)))+ (iadd _ x y)))(x64_paddd x y))(rule (lower (has_type (multi_lane 64 2)- (iadd x y)))+ (iadd _ x y)))(x64_paddq x y));; `i128`-(rule 1 (lower (has_type $I128 (iadd x y)))+(rule 1 (lower (has_type $I128 (iadd _ x y)));; Get the high/low registers for `x`.(let ((x_regs ValueRegs x)(y_regs ValueRegs y))@@ -104,10 +104,10 @@(value_regs_get_gpr x_regs 1)(value_regs_get_gpr y_regs 0)(value_regs_get_gpr y_regs 1))))-(rule 2 (lower (has_type $I128 (iadd x (iconcat y_lo y_hi))))+(rule 2 (lower (has_type $I128 (iadd _ x (iconcat _ y_lo y_hi))))(let ((x_regs ValueRegs x))(iadd128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1) y_lo y_hi)))-(rule 3 (lower (has_type $I128 (iadd x (uextend y @ (value_type $I64)))))+(rule 3 (lower (has_type $I128 (iadd _ x (uextend _ y @ (value_type $I64)))))(let ((x_regs ValueRegs x))(iadd128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1)y (RegMemImm.Imm 0))))@@ -115,7 +115,7 @@;; Specialized lowering rule for `iadd` of two 64-bit unsigned integers, meaning;; that we can skip the `adc` and instead use `setb`. This is in some sense a;; way of modeling `uadd_overflow`.-(rule 4 (lower (has_type $I128 (iadd (uextend x) (uextend y @ (value_type $I64)))))+(rule 4 (lower (has_type $I128 (iadd _ (uextend _ x) (uextend _ y @ (value_type $I64)))))(let ((x Gpr (extend_to_gpr x $I64 (ExtendKind.Zero)))(ret ValueRegs (with_flags (x64_add_with_flags_paired $I64 x y)@@ -173,71 +173,71 @@;;;; Rules for `uadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (uadd_overflow x y @ (value_type (fits_in_64 ty))))+(rule 1 (lower (uadd_overflow _ x y @ (value_type (fits_in_64 ty))))(construct_overflow_op_alu ty (CC.B) (ProduceFlagsOp.Add) x y));; i128 gets lowered into adc and add-(rule 0 (lower (uadd_overflow x y @ (value_type $I128)))+(rule 0 (lower (uadd_overflow _ x y @ (value_type $I128)))(construct_overflow_op_alu_128 (CC.B) (ProduceFlagsOp.Add) (ChainFlagsOp.Adc) x y));;;; Rules for `sadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (sadd_overflow x y @ (value_type (fits_in_64 ty))))+(rule 1 (lower (sadd_overflow _ x y @ (value_type (fits_in_64 ty))))(construct_overflow_op_alu ty (CC.O) (ProduceFlagsOp.Add) x y))-(rule 0 (lower (sadd_overflow x y @ (value_type $I128)))+(rule 0 (lower (sadd_overflow _ x y @ (value_type $I128)))(construct_overflow_op_alu_128 (CC.O) (ProduceFlagsOp.Add) (ChainFlagsOp.Adc) x y));;;; Rules for `usub_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (usub_overflow x y @ (value_type (fits_in_64 ty))))+(rule 1 (lower (usub_overflow _ x y @ (value_type (fits_in_64 ty))))(construct_overflow_op_alu ty (CC.B) (ProduceFlagsOp.Sub) x y))-(rule 0 (lower (usub_overflow x y @ (value_type $I128)))+(rule 0 (lower (usub_overflow _ x y @ (value_type $I128)))(construct_overflow_op_alu_128 (CC.B) (ProduceFlagsOp.Sub) (ChainFlagsOp.Sbb) x y));;;; Rules for `ssub_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (ssub_overflow x y @ (value_type (fits_in_64 ty))))+(rule 1 (lower (ssub_overflow _ x y @ (value_type (fits_in_64 ty))))(construct_overflow_op_alu ty (CC.O) (ProduceFlagsOp.Sub) x y))-(rule 0 (lower (ssub_overflow x y @ (value_type $I128)))+(rule 0 (lower (ssub_overflow _ x y @ (value_type $I128)))(construct_overflow_op_alu_128 (CC.O) (ProduceFlagsOp.Sub) (ChainFlagsOp.Sbb) x y));;;; Rules for `umul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (umul_overflow x y @ (value_type $I8)))+(rule 2 (lower (umul_overflow _ x y @ (value_type $I8)))(construct_overflow_op (CC.O) (x64_mul8_with_flags_paired false x y)))-(rule 3 (lower (umul_overflow x y @ (value_type (ty_int_ref_16_to_64 ty))))+(rule 3 (lower (umul_overflow _ x y @ (value_type (ty_int_ref_16_to_64 ty))))(construct_overflow_op (CC.O) (x64_mul_lo_with_flags_paired ty false x y)));;;; Rules for `smul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (smul_overflow x y @ (value_type $I8)))+(rule 2 (lower (smul_overflow _ x y @ (value_type $I8)))(construct_overflow_op (CC.O) (x64_mul8_with_flags_paired true x y)))-(rule 3 (lower (smul_overflow x y @ (value_type (ty_int_ref_16_to_64 ty))))+(rule 3 (lower (smul_overflow _ x y @ (value_type (ty_int_ref_16_to_64 ty))))(construct_overflow_op (CC.O) (x64_mul_lo_with_flags_paired ty true x y)));;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type (multi_lane 8 16)- (sadd_sat x y)))+ (sadd_sat _ x y)))(x64_paddsb x y))(rule (lower (has_type (multi_lane 16 8)- (sadd_sat x y)))+ (sadd_sat _ x y)))(x64_paddsw x y));;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type (multi_lane 8 16)- (uadd_sat x y)))+ (uadd_sat _ x y)))(x64_paddusb x y))(rule (lower (has_type (multi_lane 16 8)- (uadd_sat x y)))+ (uadd_sat _ x y)))(x64_paddusw x y));;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -246,29 +246,29 @@;; Sub two registers.(rule -3 (lower (has_type (fits_in_64 ty)- (isub x y)))+ (isub _ x y)))(x64_sub ty x y));; SSE.(rule (lower (has_type (multi_lane 8 16)- (isub x y)))+ (isub _ x y)))(x64_psubb x y))(rule (lower (has_type (multi_lane 16 8)- (isub x y)))+ (isub _ x y)))(x64_psubw x y))(rule (lower (has_type (multi_lane 32 4)- (isub x y)))+ (isub _ x y)))(x64_psubd x y))(rule (lower (has_type (multi_lane 64 2)- (isub x y)))+ (isub _ x y)))(x64_psubq x y));; `i128`-(rule 1 (lower (has_type $I128 (isub x y)))+(rule 1 (lower (has_type $I128 (isub _ x y)));; Get the high/low registers for `x`.(let ((x_regs ValueRegs x)(y_regs ValueRegs y))@@ -277,10 +277,10 @@(value_regs_get_gpr x_regs 1)(value_regs_get_gpr y_regs 0)(value_regs_get_gpr y_regs 1))))-(rule 2 (lower (has_type $I128 (isub x (iconcat y_lo y_hi))))+(rule 2 (lower (has_type $I128 (isub _ x (iconcat _ y_lo y_hi))))(let ((x_regs ValueRegs x))(isub128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1) y_lo y_hi)))-(rule 3 (lower (has_type $I128 (isub x (uextend y @ (value_type $I64)))))+(rule 3 (lower (has_type $I128 (isub _ x (uextend _ y @ (value_type $I64)))))(let ((x_regs ValueRegs x))(isub128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1)y (RegMemImm.Imm 0))))@@ -296,21 +296,21 @@;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type (multi_lane 8 16)- (ssub_sat x y)))+ (ssub_sat _ x y)))(x64_psubsb x y))(rule (lower (has_type (multi_lane 16 8)- (ssub_sat x y)))+ (ssub_sat _ x y)))(x64_psubsw x y));;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type (multi_lane 8 16)- (usub_sat x y)))+ (usub_sat _ x y)))(x64_psubusb x y))(rule (lower (has_type (multi_lane 16 8)- (usub_sat x y)))+ (usub_sat _ x y)))(x64_psubusw x y));;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -318,24 +318,24 @@;; `{i,b}64` and smaller.;; And two registers.-(rule 0 (lower (has_type ty (band x y)))+(rule 0 (lower (has_type ty (band _ x y)))(if (ty_int_ref_scalar_64 ty))(x64_and ty x y));; The above case automatically handles when the rhs is an immediate or a;; sinkable load, but additionally handle the lhs here.-(rule 1 (lower (has_type ty (band (sinkable_load x) y)))+(rule 1 (lower (has_type ty (band _ (sinkable_load x) y)))(if (ty_int_ref_scalar_64 ty))(x64_and ty y x))-(rule 2 (lower (has_type ty (band (simm32_from_value x) y)))+(rule 2 (lower (has_type ty (band _ (simm32_from_value x) y)))(if (ty_int_ref_scalar_64 ty))(x64_and ty y x));; f32 and f64-(rule 5 (lower (has_type (ty_scalar_float ty) (band x y)))+(rule 5 (lower (has_type (ty_scalar_float ty) (band _ x y)))(sse_and ty x y));; SSE.@@ -348,7 +348,7 @@(rule -1 (sse_and (multi_lane _bits _lanes) x y) (x64_pand x y))(rule 6 (lower (has_type ty @ (multi_lane _bits _lanes)- (band x y)))+ (band _ x y)))(sse_and ty x y));; `i128`.@@ -364,7 +364,7 @@(value_gprs (x64_and $I64 x_lo y_lo)(x64_and $I64 x_hi y_hi))))-(rule 7 (lower (has_type $I128 (band x y)))+(rule 7 (lower (has_type $I128 (band _ x y)))(and_i128 x y));; Specialized lowerings for `(band x (bnot y))` which is additionally produced@@ -383,17 +383,17 @@;; while x86 does;;;; pandn(x, y) = and(not(x), y)-(rule 8 (lower (has_type ty @ (multi_lane _bits _lane) (band x (bnot y))))+(rule 8 (lower (has_type ty @ (multi_lane _bits _lane) (band _ x (bnot _ y))))(sse_and_not ty y x))-(rule 9 (lower (has_type ty @ (multi_lane _bits _lane) (band (bnot y) x)))+(rule 9 (lower (has_type ty @ (multi_lane _bits _lane) (band _ (bnot _ y) x)))(sse_and_not ty y x))-(rule 10 (lower (has_type ty (band x (bnot y))))+(rule 10 (lower (has_type ty (band _ x (bnot _ y))))(if (ty_int_ref_scalar_64 ty))(if-let true (has_bmi1));; the first argument is the one that gets inverted with andn(x64_andn ty y x))-(rule 11 (lower (has_type ty (band (bnot y) x)))+(rule 11 (lower (has_type ty (band _ (bnot _ y) x)))(if (ty_int_ref_scalar_64 ty))(if-let true (has_bmi1))(x64_andn ty y x))@@ -401,25 +401,25 @@;; Specialization of `blsr` for BMI1(decl pure partial val_minus_one (Value) Value)-(rule 0 (val_minus_one (isub x (u64_from_iconst 1))) x)-(rule 0 (val_minus_one (iadd x (i64_from_iconst -1))) x)-(rule 1 (val_minus_one (iadd (i64_from_iconst -1) x)) x)+(rule 0 (val_minus_one (isub _ x (u64_from_iconst 1))) x)+(rule 0 (val_minus_one (iadd _ x (i64_from_iconst -1))) x)+(rule 1 (val_minus_one (iadd _ (i64_from_iconst -1) x)) x)-(rule 12 (lower (has_type (ty_32_or_64 ty) (band x y)))+(rule 12 (lower (has_type (ty_32_or_64 ty) (band _ x y)))(if-let true (has_bmi1))(if-let x (val_minus_one y))(x64_blsr ty x))-(rule 13 (lower (has_type (ty_32_or_64 ty) (band y x)))+(rule 13 (lower (has_type (ty_32_or_64 ty) (band _ y x)))(if-let true (has_bmi1))(if-let x (val_minus_one y))(x64_blsr ty x));; Specialization of `blsi` for BMI1-(rule 14 (lower (has_type (ty_32_or_64 ty) (band (ineg x) x)))+(rule 14 (lower (has_type (ty_32_or_64 ty) (band _ (ineg _ x) x)))(if-let true (has_bmi1))(x64_blsi ty x))-(rule 15 (lower (has_type (ty_32_or_64 ty) (band x (ineg x))))+(rule 15 (lower (has_type (ty_32_or_64 ty) (band _ x (ineg _ x))))(if-let true (has_bmi1))(x64_blsi ty x))@@ -432,9 +432,9 @@;; `ishl`, so an `and` instruction is required to mask the index to match the;; semantics of Cranelift's `ishl`.-(rule 16 (lower (has_type (ty_32_or_64 ty) (band x y)))+(rule 16 (lower (has_type (ty_32_or_64 ty) (band _ x y)))(if-let true (has_bmi2))- (if-let (ishl (u64_from_iconst 1) index) (val_minus_one y))+ (if-let (ishl _ (u64_from_iconst 1) index) (val_minus_one y))(x64_bzhi ty x (x64_and ty index (RegMemImm.Imm (u32_wrapping_sub (ty_bits ty) 1)))));;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -442,24 +442,24 @@;; `{i,b}64` and smaller.;; Or two registers.-(rule 0 (lower (has_type ty (bor x y)))+(rule 0 (lower (has_type ty (bor _ x y)))(if (ty_int_ref_scalar_64 ty))(x64_or ty x y));; Handle immediates/sinkable loads on the lhs in addition to the automatic;; handling of the rhs above-(rule 1 (lower (has_type ty (bor (sinkable_load x) y)))+(rule 1 (lower (has_type ty (bor _ (sinkable_load x) y)))(if (ty_int_ref_scalar_64 ty))(x64_or ty y x))-(rule 2 (lower (has_type ty (bor (simm32_from_value x) y)))+(rule 2 (lower (has_type ty (bor _ (simm32_from_value x) y)))(if (ty_int_ref_scalar_64 ty))(x64_or ty y x));; f32 and f64-(rule 5 (lower (has_type (ty_scalar_float ty) (bor x y)))+(rule 5 (lower (has_type (ty_scalar_float ty) (bor _ x y)))(sse_or ty x y));; SSE.@@ -472,7 +472,7 @@(rule -1 (sse_or (multi_lane _bits _lanes) x y) (x64_por x y))(rule 6 (lower (has_type ty @ (multi_lane _bits _lanes)- (bor x y)))+ (bor _ x y)))(sse_or ty x y));; `{i,b}128`.@@ -486,7 +486,7 @@(value_gprs (x64_or $I64 x_lo y_lo)(x64_or $I64 x_hi y_hi))))-(rule 7 (lower (has_type $I128 (bor x y)))+(rule 7 (lower (has_type $I128 (bor _ x y)))(or_i128 x y));; Specialized lowerings to generate the `shld` instruction.@@ -495,13 +495,13 @@;; different register. Pattern-match doing this with bit-ops and shifts to;; generate a `shld` instruction.(rule 8 (lower (has_type (ty_int_ref_16_to_64 ty)- (bor (ishl x (u8_from_iconst xs)) (ushr y (u8_from_iconst ys)))))+ (bor _ (ishl _ x (u8_from_iconst xs)) (ushr _ y (u8_from_iconst ys)))))(if-let true (u64_eq (ty_bits ty) (u64_wrapping_add xs ys)))(if-let true (u64_gt xs 0))(if-let true (u64_gt ys 0))(x64_shld ty x y xs))(rule 8 (lower (has_type (ty_int_ref_16_to_64 ty)- (bor (ushr y (u8_from_iconst ys)) (ishl x (u8_from_iconst xs)))))+ (bor _ (ushr _ y (u8_from_iconst ys)) (ishl _ x (u8_from_iconst xs)))))(if-let true (u64_eq (ty_bits ty) (u64_wrapping_add xs ys)))(if-let true (u64_gt xs 0))(if-let true (u64_gt ys 0))@@ -513,34 +513,34 @@;; `{i,b}64` and smaller.;; Xor two registers.-(rule 0 (lower (has_type ty (bxor x y)))+(rule 0 (lower (has_type ty (bxor _ x y)))(if (ty_int_ref_scalar_64 ty))(x64_xor ty x y));; Handle xor with lhs immediates/sinkable loads in addition to the automatic;; handling of the rhs above.-(rule 1 (lower (has_type ty (bxor (sinkable_load x) y)))+(rule 1 (lower (has_type ty (bxor _ (sinkable_load x) y)))(if (ty_int_ref_scalar_64 ty))(x64_xor ty y x))-(rule 4 (lower (has_type ty (bxor (simm32_from_value x) y)))+(rule 4 (lower (has_type ty (bxor _ (simm32_from_value x) y)))(if (ty_int_ref_scalar_64 ty))(x64_xor ty y x));; f32 and f64-(rule 5 (lower (has_type (ty_scalar_float ty) (bxor x y)))+(rule 5 (lower (has_type (ty_scalar_float ty) (bxor _ x y)))(x64_xor_vector ty x y));; SSE.-(rule 6 (lower (has_type ty @ (multi_lane _bits _lanes) (bxor x y)))+(rule 6 (lower (has_type ty @ (multi_lane _bits _lanes) (bxor _ x y)))(x64_xor_vector ty x y));; `{i,b}128`.-(rule 7 (lower (has_type $I128 (bxor x y)))+(rule 7 (lower (has_type $I128 (bxor _ x y)))(let ((x_regs ValueRegs x)(x_lo Gpr (value_regs_get_gpr x_regs 0))(x_hi Gpr (value_regs_get_gpr x_regs 1))@@ -552,11 +552,11 @@;; Specialization of `blsmsk` for BMI1-(rule 8 (lower (has_type (ty_32_or_64 ty) (bxor x y)))+(rule 8 (lower (has_type (ty_32_or_64 ty) (bxor _ x y)))(if-let true (has_bmi1))(if-let x (val_minus_one y))(x64_blsmsk ty x))-(rule 9 (lower (has_type (ty_32_or_64 ty) (bxor y x)))+(rule 9 (lower (has_type (ty_32_or_64 ty) (bxor _ y x)))(if-let true (has_bmi1))(if-let x (val_minus_one y))(x64_blsmsk ty x))@@ -565,7 +565,7 @@;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (ishl src amt)))+(rule -1 (lower (has_type (fits_in_64 ty) (ishl _ src amt)))(x64_shl ty src (put_masked_in_imm8_gpr amt ty)));; `i128`.@@ -602,7 +602,7 @@(cmove $I64 (CC.Z) lo_shifted zero)(cmove $I64 (CC.Z) hi_shifted_ lo_shifted)))))-(rule (lower (has_type $I128 (ishl src amt)))+(rule (lower (has_type $I128 (ishl _ src amt)));; NB: Only the low bits of `amt` matter since we logically mask the shift;; amount to the value's bit width.(let ((amt_ Gpr (lo_gpr amt)))@@ -614,7 +614,7 @@;; in higher feature sets like AVX), we lower the `ishl.i8x16` to a sequence of;; instructions. The basic idea, whether the amount to shift by is an immediate;; or not, is to use a 16x8 shift and then mask off the incorrect bits to 0s.-(rule (lower (has_type ty @ $I8X16 (ishl src amt)))+(rule (lower (has_type ty @ $I8X16 (ishl _ src amt)))(let (;; Mask the amount to ensure wrapping behaviour(masked_amt RegMemImm (mask_xmm_shift ty amt))@@ -660,20 +660,20 @@;; 16x8, 32x4, and 64x2 shifts can each use a single instruction, once the shift amount is masked.-(rule (lower (has_type ty @ $I16X8 (ishl src amt)))+(rule (lower (has_type ty @ $I16X8 (ishl _ src amt)))(x64_psllw src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I32X4 (ishl src amt)))+(rule (lower (has_type ty @ $I32X4 (ishl _ src amt)))(x64_pslld src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I64X2 (ishl src amt)))+(rule (lower (has_type ty @ $I64X2 (ishl _ src amt)))(x64_psllq src (mov_rmi_to_xmm (mask_xmm_shift ty amt))));;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (ushr src amt)))+(rule -1 (lower (has_type (fits_in_64 ty) (ushr _ src amt)))(let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Zero))))(x64_shr ty src_ (put_masked_in_imm8_gpr amt ty))))@@ -710,7 +710,7 @@(cmove $I64 (CC.Z) lo_shifted_ hi_shifted)(cmove $I64 (CC.Z) hi_shifted zero)))))-(rule (lower (has_type $I128 (ushr src amt)))+(rule (lower (has_type $I128 (ushr _ src amt)));; NB: Only the low bits of `amt` matter since we logically mask the shift;; amount to the value's bit width.(let ((amt_ Gpr (lo_gpr amt)))@@ -720,7 +720,7 @@;; There are no 8x16 shifts in x64. Do the same 16x8-shift-and-mask thing we do;; with 8x16 `ishl`.-(rule (lower (has_type ty @ $I8X16 (ushr src amt)))+(rule (lower (has_type ty @ $I8X16 (ushr _ src amt)))(let (;; Mask the amount to ensure wrapping behaviour(masked_amt RegMemImm (mask_xmm_shift ty amt))@@ -766,26 +766,26 @@;; 16x8, 32x4, and 64x2 shifts can each use a single instruction, once the shift amount is masked.-(rule (lower (has_type ty @ $I16X8 (ushr src amt)))+(rule (lower (has_type ty @ $I16X8 (ushr _ src amt)))(x64_psrlw src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I32X4 (ushr src amt)))+(rule (lower (has_type ty @ $I32X4 (ushr _ src amt)))(x64_psrld src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I64X2 (ushr src amt)))+(rule (lower (has_type ty @ $I64X2 (ushr _ src amt)))(x64_psrlq src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))(decl mask_xmm_shift (Type Value) RegMemImm)(rule (mask_xmm_shift ty amt)(gpr_to_reg (x64_and $I64 amt (RegMemImm.Imm (shift_mask ty)))))-(rule 1 (mask_xmm_shift ty (iconst n))+(rule 1 (mask_xmm_shift ty (iconst _ n))(RegMemImm.Imm (shift_amount_masked ty n)));;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (sshr src amt)))+(rule -1 (lower (has_type (fits_in_64 ty) (sshr _ src amt)))(let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Sign))))(x64_sar ty src_ (put_masked_in_imm8_gpr amt ty))))@@ -822,7 +822,7 @@(cmove $I64 (CC.Z) lo_shifted_ hi_shifted)(cmove $I64 (CC.Z) hi_shifted sign_bits)))))-(rule (lower (has_type $I128 (sshr src amt)))+(rule (lower (has_type $I128 (sshr _ src amt)));; NB: Only the low bits of `amt` matter since we logically mask the shift;; amount to the value's bit width.(let ((amt_ Gpr (lo_gpr amt)))@@ -843,7 +843,7 @@;; hi.i16x8 = [(s8, s8), (s9, s9), ..., (s15, s15)];; shifted_hi.i16x8 = shift each lane of `high`;; result = [s0'', s1'', ..., s15'']-(rule (lower (has_type ty @ $I8X16 (sshr src amt @ (value_type amt_ty))))+(rule (lower (has_type ty @ $I8X16 (sshr _ src amt @ (value_type amt_ty))))(let ((src_ Xmm (put_in_xmm src));; Mask the amount to ensure wrapping behaviour(masked_amt RegMemImm (mask_xmm_shift ty amt))@@ -872,30 +872,30 @@;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure;; that if the shift amount is in a register, it is in an XMM register.-(rule (lower (has_type ty @ $I16X8 (sshr src amt)))+(rule (lower (has_type ty @ $I16X8 (sshr _ src amt)))(x64_psraw src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I32X4 (sshr src amt)))+(rule (lower (has_type ty @ $I32X4 (sshr _ src amt)))(x64_psrad src (mov_rmi_to_xmm (mask_xmm_shift ty amt))));; The `sshr.i64x2` CLIF instruction has no single x86 instruction in the older;; feature sets. To remedy this, a small dance is done with an unsigned right;; shift plus some extra ops.-(rule 3 (lower (has_type ty @ $I64X2 (sshr src (iconst n))))+(rule 3 (lower (has_type ty @ $I64X2 (sshr _ src (iconst _ n))))(if-let true (has_avx512vl))(if-let true (has_avx512f))(x64_vpsraq_imm src (shift_amount_masked ty n)))-(rule 2 (lower (has_type ty @ $I64X2 (sshr src amt)))+(rule 2 (lower (has_type ty @ $I64X2 (sshr _ src amt)))(if-let true (has_avx512vl))(if-let true (has_avx512f))(let ((masked Gpr (x64_and $I64 amt (RegMemImm.Imm (shift_mask ty)))))(x64_vpsraq src (x64_movd_to_xmm masked))))-(rule 1 (lower (has_type $I64X2 (sshr src (u32_from_iconst amt))))+(rule 1 (lower (has_type $I64X2 (sshr _ src (u32_from_iconst amt))))(lower_i64x2_sshr_imm src (u32_and amt 63)))-(rule (lower (has_type $I64X2 (sshr src amt)))+(rule (lower (has_type $I64X2 (sshr _ src amt)))(lower_i64x2_sshr_gpr src (x64_and $I64 amt (RegMemImm.Imm 63))))(decl lower_i64x2_sshr_imm (Xmm u32) Xmm)@@ -955,13 +955,13 @@;; `i64` and smaller: we can rely on x86's rotate-amount masking since;; we operate on the whole register. For const's we mask the constant.-(rule -1 (lower (has_type (fits_in_64 ty) (rotl src amt)))+(rule -1 (lower (has_type (fits_in_64 ty) (rotl _ src amt)))(x64_rotl ty src (put_masked_in_imm8_gpr amt ty)));; `i128`.-(rule (lower (has_type $I128 (rotl src amt)))+(rule (lower (has_type $I128 (rotl _ src amt)))(let ((src_ ValueRegs src);; NB: Only the low bits of `amt` matter since we logically mask the;; rotation amount to the value's bit width.@@ -976,13 +976,13 @@;; `i64` and smaller: we can rely on x86's rotate-amount masking since;; we operate on the whole register. For const's we mask the constant.-(rule -1 (lower (has_type (fits_in_64 ty) (rotr src amt)))+(rule -1 (lower (has_type (fits_in_64 ty) (rotr _ src amt)))(x64_rotr ty src (put_masked_in_imm8_gpr amt ty)));; `i128`.-(rule (lower (has_type $I128 (rotr src amt)))+(rule (lower (has_type $I128 (rotr _ src amt)))(let ((src_ ValueRegs src);; NB: Only the low bits of `amt` matter since we logically mask the;; rotation amount to the value's bit width.@@ -996,10 +996,10 @@;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (ineg x)))+(rule -1 (lower (has_type (fits_in_64 ty) (ineg _ x)))(x64_neg ty x))-(rule -2 (lower (has_type $I128 (ineg x)))+(rule -2 (lower (has_type $I128 (ineg _ x)));; Get the high/low registers for `x`.(let ((regs ValueRegs x)(lo Gpr (value_regs_get_gpr regs 0))@@ -1010,26 +1010,26 @@;; SSE.-(rule (lower (has_type $I8X16 (ineg x)))+(rule (lower (has_type $I8X16 (ineg _ x)))(x64_psubb (imm $I8X16 0) x))-(rule (lower (has_type $I16X8 (ineg x)))+(rule (lower (has_type $I16X8 (ineg _ x)))(x64_psubw (imm $I16X8 0) x))-(rule (lower (has_type $I32X4 (ineg x)))+(rule (lower (has_type $I32X4 (ineg _ x)))(x64_psubd (imm $I32X4 0) x))-(rule (lower (has_type $I64X2 (ineg x)))+(rule (lower (has_type $I64X2 (ineg _ x)))(x64_psubq (imm $I64X2 0) x));;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type (multi_lane 8 16)- (avg_round x y)))+ (avg_round _ x y)))(x64_pavgb x y))(rule (lower (has_type (multi_lane 16 8)- (avg_round x y)))+ (avg_round _ x y)))(x64_pavgw x y));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1038,32 +1038,32 @@;; 8-bit base case, needs a special instruction encoding and additionally;; move sinkable loads to the right.-(rule -8 (lower (has_type $I8 (imul x y))) (x64_mul8 false x y))-(rule -7 (lower (has_type $I8 (imul (sinkable_load x) y))) (x64_mul8 false y x))+(rule -8 (lower (has_type $I8 (imul _ x y))) (x64_mul8 false x y))+(rule -7 (lower (has_type $I8 (imul _ (sinkable_load x) y))) (x64_mul8 false y x));; 16-to-64-bit base cases, same as above by moving sinkable loads to the right.-(rule -6 (lower (has_type (ty_int_ref_16_to_64 ty) (imul x y)))+(rule -6 (lower (has_type (ty_int_ref_16_to_64 ty) (imul _ x y)))(x64_imul ty x y))-(rule -5 (lower (has_type (ty_int_ref_16_to_64 ty) (imul (sinkable_load x) y)))+(rule -5 (lower (has_type (ty_int_ref_16_to_64 ty) (imul _ (sinkable_load x) y)))(x64_imul ty y x));; Lift out constants to use 3-operand form.-(rule -4 (lower (has_type (ty_int_ref_16_to_64 ty) (imul x (i32_from_iconst y))))+(rule -4 (lower (has_type (ty_int_ref_16_to_64 ty) (imul _ x (i32_from_iconst y))))(x64_imul_imm ty x y))-(rule -3 (lower (has_type (ty_int_ref_16_to_64 ty) (imul (i32_from_iconst x) y)))+(rule -3 (lower (has_type (ty_int_ref_16_to_64 ty) (imul _ (i32_from_iconst x) y)))(x64_imul_imm ty y x));; Special case widening multiplication from 8-to-16-bits with a single;; instruction since the 8-bit-multiply places both the high and low halves in;; the same register-(rule -2 (lower (has_type $I16 (imul (sextend x) (sextend y))))+(rule -2 (lower (has_type $I16 (imul _ (sextend _ x) (sextend _ y))))(x64_mul8 true x y))-(rule -2 (lower (has_type $I16 (imul (uextend x) (uextend y))))+(rule -2 (lower (has_type $I16 (imul _ (uextend _ x) (uextend _ y))))(x64_mul8 false x y));; `i128`.-(rule 2 (lower (has_type $I128 (imul x y)))+(rule 2 (lower (has_type $I128 (imul _ x y)))(let ((x_regs ValueRegs x)(y_regs ValueRegs y))(imul128@@ -1072,7 +1072,7 @@(value_regs_get_gpr y_regs 0)(value_regs_get_gpr y_regs 1))))-(rule 4 (lower (has_type $I128 (imul (iconcat x_lo x_hi) (iconcat y_lo y_hi))))+(rule 4 (lower (has_type $I128 (imul _ (iconcat _ x_lo x_hi) (iconcat _ y_lo y_hi))))(imul128 x_lo x_hi y_lo y_hi));; Helper for lowering 128-bit multiplication with the 64-bit halves of the@@ -1114,28 +1114,28 @@;; operands and producing a 128-bit result, which exactly matches the semantics;; of widening 64-bit inputs to 128-bit and then multiplying them. That means;; that these cases can get some some simpler codegen.-(rule 5 (lower (has_type $I128 (imul (uextend x @ (value_type $I64))- (uextend y @ (value_type $I64)))))+(rule 5 (lower (has_type $I128 (imul _ (uextend _ x @ (value_type $I64))+ (uextend _ y @ (value_type $I64)))))(x64_mul $I64 false x y))-(rule 5 (lower (has_type $I128 (imul (sextend x @ (value_type $I64))- (sextend y @ (value_type $I64)))))+(rule 5 (lower (has_type $I128 (imul _ (sextend _ x @ (value_type $I64))+ (sextend _ y @ (value_type $I64)))))(x64_mul $I64 true x y));; SSE.;; (No i8x16 multiply.)-(rule (lower (has_type (multi_lane 16 8) (imul x y)))+(rule (lower (has_type (multi_lane 16 8) (imul _ x y)))(x64_pmullw x y))-(rule (lower (has_type (multi_lane 32 4) (imul x y)))+(rule (lower (has_type (multi_lane 32 4) (imul _ x y)))(if-let true (has_sse41))(x64_pmulld x y));; Without `pmulld` the `pmuludq` instruction is used instead which performs;; 32-bit multiplication storing the 64-bit result. The 64-bit result is;; truncated to 32-bits and everything else is woven into place.-(rule -1 (lower (has_type (multi_lane 32 4) (imul x y)))+(rule -1 (lower (has_type (multi_lane 32 4) (imul _ x y)))(let ((x Xmm x)(y Xmm y)@@ -1148,7 +1148,7 @@;; With AVX-512 we can implement `i64x2` multiplication with a single;; instruction.-(rule 3 (lower (has_type (multi_lane 64 2) (imul x y)))+(rule 3 (lower (has_type (multi_lane 64 2) (imul _ x y)))(if-let true (has_avx512vl))(if-let true (has_avx512dq))(x64_vpmullq x y))@@ -1174,7 +1174,7 @@;; the lower 32-bits, however, we will need to use shifts to isolate the high;; 32-bits when doing calculations, i.e., `Ah == A >> 32`.(rule (lower (has_type (multi_lane 64 2)- (imul a b)))+ (imul _ a b)))(let ((a0 Xmm a)(b0 Xmm b);; a_hi = A >> 32@@ -1196,9 +1196,9 @@;; Special case for `i32x4.extmul_high_i16x8_s`.(rule 1 (lower (has_type (multi_lane 32 4)- (imul (swiden_high (and (value_type (multi_lane 16 8))+ (imul _ (swiden_high _ (and (value_type (multi_lane 16 8))x))- (swiden_high (and (value_type (multi_lane 16 8))+ (swiden_high _ (and (value_type (multi_lane 16 8))y)))))(let ((x2 Xmm x)(y2 Xmm y)@@ -1208,9 +1208,9 @@;; Special case for `i64x2.extmul_high_i32x4_s`.(rule 1 (lower (has_type (multi_lane 64 2)- (imul (swiden_high (and (value_type (multi_lane 32 4))+ (imul _ (swiden_high _ (and (value_type (multi_lane 32 4))x))- (swiden_high (and (value_type (multi_lane 32 4))+ (swiden_high _ (and (value_type (multi_lane 32 4))y)))))(if-let true (has_sse41))(let ((x2 Xmm (x64_pshufd x 0xFA))@@ -1219,9 +1219,9 @@;; Special case for `i32x4.extmul_low_i16x8_s`.(rule 1 (lower (has_type (multi_lane 32 4)- (imul (swiden_low (and (value_type (multi_lane 16 8))+ (imul _ (swiden_low _ (and (value_type (multi_lane 16 8))x))- (swiden_low (and (value_type (multi_lane 16 8))+ (swiden_low _ (and (value_type (multi_lane 16 8))y)))))(let ((x2 Xmm x)(y2 Xmm y)@@ -1231,9 +1231,9 @@;; Special case for `i64x2.extmul_low_i32x4_s`.(rule 1 (lower (has_type (multi_lane 64 2)- (imul (swiden_low (and (value_type (multi_lane 32 4))+ (imul _ (swiden_low _ (and (value_type (multi_lane 32 4))x))- (swiden_low (and (value_type (multi_lane 32 4))+ (swiden_low _ (and (value_type (multi_lane 32 4))y)))))(if-let true (has_sse41))(let ((x2 Xmm (x64_pshufd x 0x50))@@ -1242,9 +1242,9 @@;; Special case for `i32x4.extmul_high_i16x8_u`.(rule 1 (lower (has_type (multi_lane 32 4)- (imul (uwiden_high (and (value_type (multi_lane 16 8))+ (imul _ (uwiden_high _ (and (value_type (multi_lane 16 8))x))- (uwiden_high (and (value_type (multi_lane 16 8))+ (uwiden_high _ (and (value_type (multi_lane 16 8))y)))))(let ((x2 Xmm x)(y2 Xmm y)@@ -1254,9 +1254,9 @@;; Special case for `i64x2.extmul_high_i32x4_u`.(rule 1 (lower (has_type (multi_lane 64 2)- (imul (uwiden_high (and (value_type (multi_lane 32 4))+ (imul _ (uwiden_high _ (and (value_type (multi_lane 32 4))x))- (uwiden_high (and (value_type (multi_lane 32 4))+ (uwiden_high _ (and (value_type (multi_lane 32 4))y)))))(let ((x2 Xmm (x64_pshufd x 0xFA))(y2 Xmm (x64_pshufd y 0xFA)))@@ -1264,9 +1264,9 @@;; Special case for `i32x4.extmul_low_i16x8_u`.(rule 1 (lower (has_type (multi_lane 32 4)- (imul (uwiden_low (and (value_type (multi_lane 16 8))+ (imul _ (uwiden_low _ (and (value_type (multi_lane 16 8))x))- (uwiden_low (and (value_type (multi_lane 16 8))+ (uwiden_low _ (and (value_type (multi_lane 16 8))y)))))(let ((x2 Xmm x)(y2 Xmm y)@@ -1276,9 +1276,9 @@;; Special case for `i64x2.extmul_low_i32x4_u`.(rule 1 (lower (has_type (multi_lane 64 2)- (imul (uwiden_low (and (value_type (multi_lane 32 4))+ (imul _ (uwiden_low _ (and (value_type (multi_lane 32 4))x))- (uwiden_low (and (value_type (multi_lane 32 4))+ (uwiden_low _ (and (value_type (multi_lane 32 4))y)))))(let ((x2 Xmm (x64_pshufd x 0x50))(y2 Xmm (x64_pshufd y 0x50)))@@ -1286,32 +1286,32 @@;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $I8X16 (iabs x)))+(rule 1 (lower (has_type $I8X16 (iabs _ x)))(if-let true (has_ssse3))(x64_pabsb_a_or_avx x));; Note the use of `pminub` with signed inputs will produce the positive signed;; result which is what is desired here. The `pmaxub` isn't available until;; SSE4.1 in which case the single-instruction above lowering would apply.-(rule (lower (has_type $I8X16 (iabs x)))+(rule (lower (has_type $I8X16 (iabs _ x)))(let ((x Xmm x)(negated Xmm (x64_psubb (xmm_zero $I8X16) x)))(x64_pminub_a x negated)))-(rule 1 (lower (has_type $I16X8 (iabs x)))+(rule 1 (lower (has_type $I16X8 (iabs _ x)))(if-let true (has_ssse3))(x64_pabsw_a_or_avx x))-(rule (lower (has_type $I16X8 (iabs x)))+(rule (lower (has_type $I16X8 (iabs _ x)))(let ((x Xmm x)(negated Xmm (x64_psubw (xmm_zero $I16X8) x)))(x64_pmaxsw_a x negated)))-(rule 1 (lower (has_type $I32X4 (iabs x)))+(rule 1 (lower (has_type $I32X4 (iabs _ x)))(if-let true (has_ssse3))(x64_pabsd_a_or_avx x))@@ -1321,7 +1321,7 @@;; subtracting the mask this subtracts 0 for positive lanes (does nothing) or;; ends up adding one for negative lanes. This means that for a negative lane;; `x` the result is `!x + 1` which is the result of negating it.-(rule (lower (has_type $I32X4 (iabs x)))+(rule (lower (has_type $I32X4 (iabs _ x)))(let ((x Xmm x)(negative_mask Xmm (x64_psrad x (xmi_imm 31)))@@ -1330,7 +1330,7 @@(x64_psubd flipped_if_negative negative_mask)));; When AVX512 is available, we can use a single `vpabsq` instruction.-(rule 2 (lower (has_type $I64X2 (iabs x)))+(rule 2 (lower (has_type $I64X2 (iabs _ x)))(if-let true (has_avx512vl))(if-let true (has_avx512f))(x64_vpabsq x))@@ -1339,7 +1339,7 @@;; x` and then blend in those results with `blendvpd` if the MSB of `neg` was;; set to 1 (i.e. if `neg` was negative or, conversely, if `x` was originally;; positive).-(rule 1 (lower (has_type $I64X2 (iabs x)))+(rule 1 (lower (has_type $I64X2 (iabs _ x)))(if-let true (has_sse41))(let ((rx Xmm x)(neg Xmm (x64_psubq (imm $I64X2 0) rx)))@@ -1348,7 +1348,7 @@;; and if `blendvpd` isn't available then perform a shift/shuffle to generate a;; mask of which lanes are negative, followed by flipping bits/sub to make both;; positive.-(rule (lower (has_type $I64X2 (iabs x)))+(rule (lower (has_type $I64X2 (iabs _ x)))(let ((x Xmm x)(signs Xmm (x64_psrad x (RegMemImm.Imm 31)))(signs Xmm (x64_pshufd signs 0b11_11_01_01))@@ -1357,7 +1357,7 @@;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (iabs x)))+(rule -1 (lower (has_type (fits_in_64 ty) (iabs _ x)))(let ((src Gpr x)(neg ProducesFlags (x64_neg_paired ty src));; Manually extract the result from the neg, then ignore@@ -1370,7 +1370,7 @@(with_flags_reg (produces_flags_ignore neg) cmove)));; `i128`. Negate the low bits, `adc` to the higher bits, then negate high bits.-(rule (lower (has_type $I128 (iabs x)))+(rule (lower (has_type $I128 (iabs _ x)));; Get the high/low registers for `x`.(let ((x_regs ValueRegs x)(x_lo Gpr (value_regs_get_gpr x_regs 0))@@ -1392,35 +1392,35 @@;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fabs x)))+(rule (lower (has_type $F32 (fabs _ x)))(x64_andps x (imm $F32 0x7fffffff)))-(rule (lower (has_type $F64 (fabs x)))+(rule (lower (has_type $F64 (fabs _ x)))(x64_andpd x (imm $F64 0x7fffffffffffffff)));; Special case for `f32x4.abs`.-(rule (lower (has_type $F32X4 (fabs x)))+(rule (lower (has_type $F32X4 (fabs _ x)))(x64_andps x(x64_psrld (vector_all_ones) (xmi_imm 1))));; Special case for `f64x2.abs`.-(rule (lower (has_type $F64X2 (fabs x)))+(rule (lower (has_type $F64X2 (fabs _ x)))(x64_andpd x(x64_psrlq (vector_all_ones) (xmi_imm 1))));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fneg x)))+(rule (lower (has_type $F32 (fneg _ x)))(x64_xorps x (imm $F32 0x80000000)))-(rule (lower (has_type $F64 (fneg x)))+(rule (lower (has_type $F64 (fneg _ x)))(x64_xorpd x (imm $F64 0x8000000000000000)))-(rule (lower (has_type $F32X4 (fneg x)))+(rule (lower (has_type $F32X4 (fneg _ x)))(x64_xorps x(x64_pslld (vector_all_ones) (xmi_imm 31))))-(rule (lower (has_type $F64X2 (fneg x)))+(rule (lower (has_type $F64X2 (fneg _ x)))(x64_xorpd x(x64_psllq (vector_all_ones) (xmi_imm 63))))@@ -1466,14 +1466,14 @@;; Call the lower_bmask rule that does all the procssing-(rule (lower (has_type out_ty (bmask x @ (value_type in_ty))))+(rule (lower (has_type out_ty (bmask _ x @ (value_type in_ty))))(lower_bmask out_ty in_ty x));;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule -2 (lower (has_type ty (bnot x)))+(rule -2 (lower (has_type ty (bnot _ x)))(if (ty_int_ref_scalar_64 ty))(x64_not ty x))@@ -1488,23 +1488,23 @@(value_gprs (x64_not $I64 x_lo)(x64_not $I64 x_hi))))-(rule (lower (has_type $I128 (bnot x)))+(rule (lower (has_type $I128 (bnot _ x)))(not_i128 x));; f32 and f64-(rule -3 (lower (has_type (ty_scalar_float ty) (bnot x)))+(rule -3 (lower (has_type (ty_scalar_float ty) (bnot _ x)))(x64_xor_vector ty x (vector_all_ones)));; Special case for vector-types where bit-negation is an xor against an;; all-one value-(rule -1 (lower (has_type ty @ (multi_lane _bits _lanes) (bnot x)))+(rule -1 (lower (has_type ty @ (multi_lane _bits _lanes) (bnot _ x)))(x64_xor_vector ty x (vector_all_ones)));;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type ty @ (multi_lane _bits _lanes)- (bitselect condition+ (bitselect _ conditionif_trueif_false)));; a = and if_true, condition@@ -1518,7 +1518,7 @@;; If every byte of the condition is guaranteed to be all ones or all zeroes,;; we can use x64_blend.(rule 1 (lower (has_type ty @ (multi_lane _bits _lanes)- (bitselect condition+ (bitselect _ conditionif_trueif_false)))(if-let true (has_sse41))@@ -1526,10 +1526,10 @@(x64_pblendvb if_false if_true condition))(decl pure partial all_ones_or_all_zeros (Value) bool)-(rule (all_ones_or_all_zeros (and (icmp _ _ _) (value_type (multi_lane _ _)))) true)-(rule (all_ones_or_all_zeros (and (fcmp _ _ _) (value_type (multi_lane _ _)))) true)-(rule (all_ones_or_all_zeros (and (bitcast _ (fcmp _ _ _)) (value_type (multi_lane _ _)))) true)-(rule (all_ones_or_all_zeros (vconst (vconst_all_ones_or_all_zeros))) true)+(rule (all_ones_or_all_zeros (and (icmp _ _ _ _) (value_type (multi_lane _ _)))) true)+(rule (all_ones_or_all_zeros (and (fcmp _ _ _ _) (value_type (multi_lane _ _)))) true)+(rule (all_ones_or_all_zeros (and (bitcast _ _ (fcmp _ _ _ _)) (value_type (multi_lane _ _)))) true)+(rule (all_ones_or_all_zeros (vconst _ (vconst_all_ones_or_all_zeros))) true)(decl pure vconst_all_ones_or_all_zeros () Constant)(extern extractor vconst_all_ones_or_all_zeros vconst_all_ones_or_all_zeros)@@ -1539,29 +1539,29 @@;; instructions and how they're lowered into CLIF. Note the careful ordering;; of all the operands here to ensure that the input CLIF matched is implemented;; by the corresponding x64 instruction.-(rule 2 (lower (has_type $F32X4 (bitselect (bitcast _ (fcmp (FloatCC.LessThan) x y)) x y)))+(rule 2 (lower (has_type $F32X4 (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) x y)) x y)))(x64_minps x y))-(rule 2 (lower (has_type $F64X2 (bitselect (bitcast _ (fcmp (FloatCC.LessThan) x y)) x y)))+(rule 2 (lower (has_type $F64X2 (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) x y)) x y)))(x64_minpd x y))-(rule 3 (lower (has_type $F32X4 (bitselect (bitcast _ (fcmp (FloatCC.LessThan) y x)) x y)))+(rule 3 (lower (has_type $F32X4 (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) y x)) x y)))(x64_maxps x y))-(rule 3 (lower (has_type $F64X2 (bitselect (bitcast _ (fcmp (FloatCC.LessThan) y x)) x y)))+(rule 3 (lower (has_type $F64X2 (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) y x)) x y)))(x64_maxpd x y));; Scalar rules-(rule 3 (lower (has_type $I128 (bitselect c t f)))+(rule 3 (lower (has_type $I128 (bitselect _ c t f)))(let ((a ValueRegs (and_i128 c t))(b ValueRegs (and_i128 (not_i128 c) f)))(or_i128 a b)))-(rule 4 (lower (has_type (ty_int_ref_scalar_64 ty) (bitselect c t f)))+(rule 4 (lower (has_type (ty_int_ref_scalar_64 ty) (bitselect _ c t f)))(let ((a Gpr (x64_and ty c t))(b Gpr (x64_and ty (x64_not ty c) f)))(x64_or ty a b)))-(rule 5 (lower (has_type (ty_scalar_float ty) (bitselect c t f)))+(rule 5 (lower (has_type (ty_scalar_float ty) (bitselect _ c t f)))(let ((a Xmm (sse_and ty c t))(c_neg Xmm (x64_xor_vector ty c (vector_all_ones)))(b Xmm (sse_and ty c_neg f)))@@ -1570,26 +1570,26 @@;;;; Rules for `blendv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type $I8X16- (blendv condition if_true if_false)))+ (blendv _ condition if_true if_false)))(if-let true (has_sse41))(x64_pblendvb if_false if_true condition))(rule (lower (has_type $I32X4- (blendv condition if_true if_false)))+ (blendv _ condition if_true if_false)))(if-let true (has_sse41))(x64_blendvps if_false if_true condition))(rule (lower (has_type $I64X2- (blendv condition if_true if_false)))+ (blendv _ condition if_true if_false)))(if-let true (has_sse41))(x64_blendvpd if_false if_true condition));;;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (insertlane vec @ (value_type $I8X16) val (u8_from_uimm8 idx)))+(rule 1 (lower (insertlane _ vec @ (value_type $I8X16) val (u8_from_uimm8 idx)))(if-let true (has_sse41))(x64_pinsrb vec val idx))-(rule 2 (lower (insertlane vec @ (value_type $I8X16) (sinkable_load_exact val) (u8_from_uimm8 idx)))+(rule 2 (lower (insertlane _ vec @ (value_type $I8X16) (sinkable_load_exact val) (u8_from_uimm8 idx)))(if-let true (has_sse41))(x64_pinsrb vec val idx))@@ -1617,7 +1617,7 @@;; This all, laboriously, gets the `val` into the desired lane so it's then;; `por`'d with the original vec-with-a-hole to produce the final result of the;; insertion.-(rule (lower (insertlane vec @ (value_type $I8X16) val (u8_from_uimm8 n)))+(rule (lower (insertlane _ vec @ (value_type $I8X16) val (u8_from_uimm8 n)))(let ((vec_with_hole Xmm (x64_pand vec (insert_i8x16_lane_hole n)))(val Gpr (x64_movzx (ExtMode.BL) val))(val Gpr (x64_shll_mi val (u8_wrapping_shl (u8_and n 3) 3)))@@ -1635,62 +1635,62 @@;; i16x8.replace_lane-(rule (lower (insertlane vec @ (value_type $I16X8) val (u8_from_uimm8 idx)))+(rule (lower (insertlane _ vec @ (value_type $I16X8) val (u8_from_uimm8 idx)))(x64_pinsrw vec val idx))-(rule 1 (lower (insertlane vec @ (value_type $I16X8) (sinkable_load_exact val) (u8_from_uimm8 idx)))+(rule 1 (lower (insertlane _ vec @ (value_type $I16X8) (sinkable_load_exact val) (u8_from_uimm8 idx)))(x64_pinsrw vec val idx));; i32x4.replace_lane-(rule 1 (lower (insertlane vec @ (value_type $I32X4) val (u8_from_uimm8 idx)))+(rule 1 (lower (insertlane _ vec @ (value_type $I32X4) val (u8_from_uimm8 idx)))(if-let true (has_sse41))(x64_pinsrd vec val idx))-(rule (lower (insertlane vec @ (value_type $I32X4) val (u8_from_uimm8 0)))+(rule (lower (insertlane _ vec @ (value_type $I32X4) val (u8_from_uimm8 0)))(x64_movss_regmove vec (x64_movd_to_xmm val)));; tmp = [ vec[1] vec[0] val[1] val[0] ];; result = [ vec[3] vec[2] tmp[0] tmp[2] ]-(rule (lower (insertlane vec @ (value_type $I32X4) val (u8_from_uimm8 1)))+(rule (lower (insertlane _ vec @ (value_type $I32X4) val (u8_from_uimm8 1)))(let ((val Xmm (x64_movd_to_xmm val))(vec Xmm vec))(x64_shufps (x64_punpcklqdq val vec) vec 0b11_10_00_10)));; tmp = [ vec[0] vec[3] val[0] val[0] ];; result = [ tmp[2] tmp[0] vec[1] vec[0] ]-(rule (lower (insertlane vec @ (value_type $I32X4) val (u8_from_uimm8 2)))+(rule (lower (insertlane _ vec @ (value_type $I32X4) val (u8_from_uimm8 2)))(let ((val Xmm (x64_movd_to_xmm val))(vec Xmm vec))(x64_shufps vec (x64_shufps val vec 0b00_11_00_00) 0b10_00_01_00)));; tmp = [ vec[3] vec[2] val[1] val[0] ];; result = [ tmp[0] tmp[2] vec[1] vec[0] ]-(rule (lower (insertlane vec @ (value_type $I32X4) val (u8_from_uimm8 3)))+(rule (lower (insertlane _ vec @ (value_type $I32X4) val (u8_from_uimm8 3)))(let ((val Xmm (x64_movd_to_xmm val))(vec Xmm vec))(x64_shufps vec (x64_shufps val vec 0b11_10_01_00) 0b00_10_01_00)));; i64x2.replace_lane-(rule 1 (lower (insertlane vec @ (value_type $I64X2) val (u8_from_uimm8 idx)))+(rule 1 (lower (insertlane _ vec @ (value_type $I64X2) val (u8_from_uimm8 idx)))(if-let true (has_sse41))(x64_pinsrq vec val idx))-(rule (lower (insertlane vec @ (value_type $I64X2) val (u8_from_uimm8 0)))+(rule (lower (insertlane _ vec @ (value_type $I64X2) val (u8_from_uimm8 0)))(x64_movsd_regmove vec (x64_movq_to_xmm val)))-(rule (lower (insertlane vec @ (value_type $I64X2) val (u8_from_uimm8 1)))+(rule (lower (insertlane _ vec @ (value_type $I64X2) val (u8_from_uimm8 1)))(x64_punpcklqdq vec (x64_movq_to_xmm val)));; (i64x2.replace_lane 1) with a splat as source for lane 0 -- we can elide;; the splat and just do a move. This turns out to be a common pattern when;; constructing an i64x2 out of two i64s.-(rule 3 (lower (insertlane (has_type $I64X2 (splat lane0))+(rule 3 (lower (insertlane _ (has_type $I64X2 (splat _ lane0))lane1(u8_from_uimm8 1)))(if-let true (has_sse41))(x64_pinsrq (bitcast_gpr_to_xmm 64 lane0) lane1 1))-(rule 1 (lower (insertlane vec @ (value_type $F32X4) (sinkable_load val) (u8_from_uimm8 idx)))+(rule 1 (lower (insertlane _ vec @ (value_type $F32X4) (sinkable_load val) (u8_from_uimm8 idx)))(if-let true (has_sse41))(x64_insertps vec val (sse_insertps_lane_imm idx)))-(rule (lower (insertlane vec @ (value_type $F32X4) val (u8_from_uimm8 idx)))+(rule (lower (insertlane _ vec @ (value_type $F32X4) val (u8_from_uimm8 idx)))(f32x4_insertlane vec val idx));; Helper function used below for `insertlane` but also here for other@@ -1735,7 +1735,7 @@;; Here the `movsd` instruction is used specifically to specialize moving;; into the fist lane where unlike above cases we're not using the lane;; immediate as an immediate to the instruction itself.-(rule (lower (insertlane vec @ (value_type $F64X2) val (u8_from_uimm8 0)))+(rule (lower (insertlane _ vec @ (value_type $F64X2) val (u8_from_uimm8 0)))(x64_movsd_regmove vec val));; f64x2.replace_lane 1@@ -1743,7 +1743,7 @@;; Here the `movlhps` instruction is used specifically to specialize moving;; into the second lane where unlike above cases we're not using the lane;; immediate as an immediate to the instruction itself.-(rule (lower (insertlane vec @ (value_type $F64X2) val (u8_from_uimm8 1)))+(rule (lower (insertlane _ vec @ (value_type $F64X2) val (u8_from_uimm8 1)))(x64_movlhps vec val));;;; Rules for `smin`, `smax`, `umin`, `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1761,16 +1761,16 @@(with_flags_reg (x64_cmp ty y_reg x_reg)(cmove ty cc y_reg x_reg))))-(rule -1 (lower (has_type (fits_in_64 ty) (umin x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (umin _ x y)))(cmp_and_choose ty (CC.B) x y))-(rule -1 (lower (has_type (fits_in_64 ty) (umax x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (umax _ x y)))(cmp_and_choose ty (CC.NB) x y))-(rule -1 (lower (has_type (fits_in_64 ty) (smin x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (smin _ x y)))(cmp_and_choose ty (CC.L) x y))-(rule -1 (lower (has_type (fits_in_64 ty) (smax x y)))+(rule -1 (lower (has_type (fits_in_64 ty) (smax _ x y)))(cmp_and_choose ty (CC.NL) x y));; SSE helpers for determining if single-instruction lowerings are available.@@ -1797,7 +1797,7 @@;; SSE `smax`.-(rule (lower (has_type (ty_vec128 ty) (smax x y)))+(rule (lower (has_type (ty_vec128 ty) (smax _ x y)))(lower_vec_smax ty x y))(decl lower_vec_smax (Type Xmm Xmm) Xmm)@@ -1817,11 +1817,11 @@;; SSE `smin`.-(rule 1 (lower (has_type (ty_vec128 ty) (smin x y)))+(rule 1 (lower (has_type (ty_vec128 ty) (smin _ x y)))(if-let true (has_pmins ty))(x64_pmins ty x y))-(rule (lower (has_type (ty_vec128 ty) (smin x y)))+(rule (lower (has_type (ty_vec128 ty) (smin _ x y)))(let ((x Xmm x)(y Xmm y)@@ -1833,20 +1833,20 @@;; SSE `umax`.-(rule 2 (lower (has_type (ty_vec128 ty) (umax x y)))+(rule 2 (lower (has_type (ty_vec128 ty) (umax _ x y)))(if-let true (has_pmaxu ty))(x64_pmaxu ty x y));; If y < x then the saturating subtraction will be zero, otherwise when added;; back to x it'll return y.-(rule 1 (lower (has_type $I16X8 (umax x y)))+(rule 1 (lower (has_type $I16X8 (umax _ x y)))(let ((x Xmm x))(x64_paddw x (x64_psubusw y x))));; Flip the upper bits of each lane so the signed comparison has the same;; result as a signed comparison, and then select the results with the output;; mask. See `pcmpgt` lowering for info on flipping the upper bit.-(rule (lower (has_type (ty_vec128 ty) (umax x y)))+(rule (lower (has_type (ty_vec128 ty) (umax _ x y)))(let ((x Xmm x)(y Xmm y)@@ -1869,18 +1869,18 @@;; SSE `umin`.-(rule 2 (lower (has_type (ty_vec128 ty) (umin x y)))+(rule 2 (lower (has_type (ty_vec128 ty) (umin _ x y)))(if-let true (has_pminu ty))(x64_pminu ty x y));; If x < y then the saturating subtraction will be 0. Otherwise if x > y then;; the saturated result, when subtracted again, will go back to `y`.-(rule 1 (lower (has_type $I16X8 (umin x y)))+(rule 1 (lower (has_type $I16X8 (umin _ x y)))(let ((x Xmm x))(x64_psubw x (x64_psubusw x y))));; Same as `umax`, and see `pcmpgt` for docs on flipping the upper bit.-(rule (lower (has_type (ty_vec128 ty) (umin x y)))+(rule (lower (has_type (ty_vec128 ty) (umin _ x y)))(let ((x Xmm x)(y Xmm y)@@ -1918,7 +1918,7 @@;;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap a b tc)))+(rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap _ a b tc)))(with_flags(x64_add_with_flags_paired ty a b)(trap_if (CC.B) tc)))@@ -1927,13 +1927,13 @@;; handling of above.(rule 1 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap (simm32_from_value a) b tc)))+ (uadd_overflow_trap _ (simm32_from_value a) b tc)))(with_flags(x64_add_with_flags_paired ty b a)(trap_if (CC.B) tc)))(rule 2 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap (sinkable_load a) b tc)))+ (uadd_overflow_trap _ (sinkable_load a) b tc)))(with_flags(x64_add_with_flags_paired ty b a)(trap_if (CC.B) tc)))@@ -1946,42 +1946,42 @@;;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (icmp cc a @ (value_type (fits_in_64 ty)) b))+(rule -2 (lower (icmp _ cc a @ (value_type (fits_in_64 ty)) b))(lower_cond_bool (emit_cmp cc a b)))-(rule -1 (lower (icmp cc a @ (value_type $I128) b))+(rule -1 (lower (icmp _ cc a @ (value_type $I128) b))(lower_cond_bool (emit_cmp cc a b)));; Peephole optimization for `x < 0`, when x is a signed 64 bit value-(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedLessThan) x @ (value_type $I64) (u64_from_iconst 0))))+(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedLessThan) x @ (value_type $I64) (u64_from_iconst 0))))(x64_shrq_mi x 63));; Peephole optimization for `0 > x`, when x is a signed 64 bit value-(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I64))))+(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I64))))(x64_shrq_mi x 63));; Peephole optimization for `0 <= x`, when x is a signed 64 bit value-(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I64))))+(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I64))))(x64_shrq_mi (x64_not $I64 x) 63));; Peephole optimization for `x >= 0`, when x is a signed 64 bit value-(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I64) (u64_from_iconst 0))))+(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I64) (u64_from_iconst 0))))(x64_shrq_mi (x64_not $I64 x) 63));; Peephole optimization for `x < 0`, when x is a signed 32 bit value-(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedLessThan) x @ (value_type $I32) (u64_from_iconst 0))))+(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedLessThan) x @ (value_type $I32) (u64_from_iconst 0))))(x64_shrl_mi x 31));; Peephole optimization for `0 > x`, when x is a signed 32 bit value-(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I32))))+(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I32))))(x64_shrl_mi x 31));; Peephole optimization for `0 <= x`, when x is a signed 32 bit value-(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I32))))+(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I32))))(x64_shrl_mi (x64_not $I64 x) 31));; Peephole optimization for `x >= 0`, when x is a signed 32 bit value-(rule 2 (lower (has_type $I8 (icmp (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I32) (u64_from_iconst 0))))+(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I32) (u64_from_iconst 0))))(x64_shrl_mi (x64_not $I64 x) 31));; For XMM-held values, we lower to `PCMP*` instructions, sometimes more than@@ -1989,31 +1989,31 @@;; lane will be filled with all 1s or all 0s according to the comparison,;; whereas for GPR-held values, the result will be simply 0 or 1 (upper bits;; unset).-(rule (lower (icmp (IntCC.Equal) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.Equal) a @ (value_type (ty_vec128 ty)) b))(x64_pcmpeq ty a b));; To lower a not-equals comparison, we perform an equality comparison;; (PCMPEQ*) and then invert the bits (PXOR with all 1s).-(rule (lower (icmp (IntCC.NotEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.NotEqual) a @ (value_type (ty_vec128 ty)) b))(let ((checked Xmm (x64_pcmpeq ty a b))(all_ones Xmm (vector_all_ones)))(x64_pxor checked all_ones)));; SSE `sgt`-(rule (lower (icmp (IntCC.SignedGreaterThan) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.SignedGreaterThan) a @ (value_type (ty_vec128 ty)) b))(x64_pcmpgt ty a b));; SSE `slt`-(rule (lower (icmp (IntCC.SignedLessThan) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.SignedLessThan) a @ (value_type (ty_vec128 ty)) b))(x64_pcmpgt ty b a));; SSE `ugt`;; N.B.: we must manually prevent load coalescing operands; the;; register allocator gets confused otherwise.-(rule 1 (lower (icmp (IntCC.UnsignedGreaterThan) a @ (value_type (ty_vec128 ty)) b))+(rule 1 (lower (icmp _ (IntCC.UnsignedGreaterThan) a @ (value_type (ty_vec128 ty)) b))(if-let true (has_pmaxu ty))(let ((a Xmm a)(b Xmm b)@@ -2023,7 +2023,7 @@;; Flip the upper bit of each lane so the result of a signed comparison is the;; same as the result of an unsigned comparison (see docs on `pcmpgt` for more)-(rule (lower (icmp (IntCC.UnsignedGreaterThan) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.UnsignedGreaterThan) a @ (value_type (ty_vec128 ty)) b))(let ((mask Xmm (flip_high_bit_mask ty))(a_masked Xmm (x64_pxor a mask))(b_masked Xmm (x64_pxor b mask)))@@ -2031,7 +2031,7 @@;; SSE `ult`-(rule 1 (lower (icmp (IntCC.UnsignedLessThan) a @ (value_type (ty_vec128 ty)) b))+(rule 1 (lower (icmp _ (IntCC.UnsignedLessThan) a @ (value_type (ty_vec128 ty)) b))(if-let true (has_pminu ty));; N.B.: see note above.(let ((a Xmm a)@@ -2042,7 +2042,7 @@;; Flip the upper bit of `a` and `b` so the signed comparison result will;; be the same as the unsigned comparison result (see docs on `pcmpgt` for more).-(rule (lower (icmp (IntCC.UnsignedLessThan) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.UnsignedLessThan) a @ (value_type (ty_vec128 ty)) b))(let ((mask Xmm (flip_high_bit_mask ty))(a_masked Xmm (x64_pxor a mask))(b_masked Xmm (x64_pxor b mask)))@@ -2051,41 +2051,41 @@;; SSE `sge`;; Use `pmaxs*` and compare the result to `a` to see if it's `>= b`.-(rule 1 (lower (icmp (IntCC.SignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule 1 (lower (icmp _ (IntCC.SignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(if-let true (has_pmaxs ty))(x64_pcmpeq ty a (x64_pmaxs ty a b)));; Without `pmaxs*` use a `pcmpgt*` with reversed operands and invert the;; result.-(rule (lower (icmp (IntCC.SignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.SignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(x64_pxor (x64_pcmpgt ty b a) (vector_all_ones)));; SSE `sle`;; With `pmins*` use that and compare the result to `a`.-(rule 1 (lower (icmp (IntCC.SignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule 1 (lower (icmp _ (IntCC.SignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(if-let true (has_pmins ty))(x64_pcmpeq ty a (x64_pmins ty a b)));; Without `pmins*` perform a greater-than test and invert the result.-(rule (lower (icmp (IntCC.SignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.SignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(x64_pxor (x64_pcmpgt ty a b) (vector_all_ones)));; SSE `uge`-(rule 2 (lower (icmp (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule 2 (lower (icmp _ (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(if-let true (has_pmaxu ty))(x64_pcmpeq ty a (x64_pmaxu ty a b)));; Perform a saturating subtract of `a` from `b` and if the result is zero then;; `a` is greater or equal.-(rule 1 (lower (icmp (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type $I16X8) b))+(rule 1 (lower (icmp _ (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type $I16X8) b))(x64_pcmpeqw (x64_psubusw b a) (xmm_zero $I16X8)));; Flip the upper bit of each lane so the signed comparison is the same as;; an unsigned one and then invert the result. See docs on `pcmpgt` for why;; flipping the upper bit works.-(rule (lower (icmp (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.UnsignedGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(let ((mask Xmm (flip_high_bit_mask ty))(a_masked Xmm (x64_pxor a mask))@@ -2096,14 +2096,14 @@;; SSE `ule`-(rule 2 (lower (icmp (IntCC.UnsignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule 2 (lower (icmp _ (IntCC.UnsignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(if-let true (has_pminu ty))(x64_pcmpeq ty a (x64_pminu ty a b)));; A saturating subtraction will produce zeros if `a` is less than `b`, so;; compare that result to an all-zeros result to figure out lanes of `a` that;; are <= to the lanes in `b`-(rule 1 (lower (icmp (IntCC.UnsignedLessThanOrEqual) a @ (value_type $I16X8) b))+(rule 1 (lower (icmp _ (IntCC.UnsignedLessThanOrEqual) a @ (value_type $I16X8) b))(let ((zeros_if_a_is_min Xmm (x64_psubusw a b)))(x64_pcmpeqw zeros_if_a_is_min (xmm_zero $I8X16))))@@ -2111,7 +2111,7 @@;; produces the same result as an unsigned comparison. Then test test for `gt`;; and invert the result to get the `le` that is desired here. See docs on;; `pcmpgt` for why flipping the upper bit works.-(rule (lower (icmp (IntCC.UnsignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (icmp _ (IntCC.UnsignedLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(let ((mask Xmm (flip_high_bit_mask ty))(a_masked Xmm (x64_pxor a mask))@@ -2139,40 +2139,40 @@;; - less than assigns Z = 0, P = 0, C = 1;; - equal assigns Z = 1, P = 0, C = 0-(rule -1 (lower (fcmp cc a @ (value_type (ty_scalar_float ty)) b))+(rule -1 (lower (fcmp _ cc a @ (value_type (ty_scalar_float ty)) b))(lower_cond_bool (emit_fcmp cc a b)));; For vector lowerings, we use `CMPP*` instructions with a 3-bit operand that;; determines the comparison to make. Note that comparisons that succeed will;; fill the lane with 1s; comparisons that do not will fill the lane with 0s.-(rule (lower (fcmp (FloatCC.Equal) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.Equal) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty a b (FcmpImm.Equal)))-(rule (lower (fcmp (FloatCC.NotEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.NotEqual) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty a b (FcmpImm.NotEqual)))-(rule (lower (fcmp (FloatCC.LessThan) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.LessThan) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty a b (FcmpImm.LessThan)))-(rule (lower (fcmp (FloatCC.LessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.LessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty a b (FcmpImm.LessThanOrEqual)))-(rule (lower (fcmp (FloatCC.Ordered) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.Ordered) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty a b (FcmpImm.Ordered)))-(rule (lower (fcmp (FloatCC.Unordered) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.Unordered) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty a b (FcmpImm.Unordered)))-(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThan) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.UnorderedOrGreaterThan) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty a b (FcmpImm.UnorderedOrGreaterThan)))-(rule (lower (fcmp (FloatCC.UnorderedOrGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.UnorderedOrGreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty a b (FcmpImm.UnorderedOrGreaterThanOrEqual)));; Some vector lowerings rely on flipping the operands and using a reversed;; comparison code.-(rule (lower (fcmp (FloatCC.GreaterThan) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.GreaterThan) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty b a (FcmpImm.LessThan)))-(rule (lower (fcmp (FloatCC.GreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.GreaterThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty b a (FcmpImm.LessThanOrEqual)))-(rule (lower (fcmp (FloatCC.UnorderedOrLessThan) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.UnorderedOrLessThan) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty b a (FcmpImm.UnorderedOrGreaterThan)))-(rule (lower (fcmp (FloatCC.UnorderedOrLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))+(rule (lower (fcmp _ (FloatCC.UnorderedOrLessThanOrEqual) a @ (value_type (ty_vec128 ty)) b))(x64_cmpp ty b a (FcmpImm.UnorderedOrGreaterThanOrEqual)));; Some vector lowerings are simply not supported for certain codes:@@ -2181,7 +2181,7 @@;;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (select cond x y)) (lower_select (is_nonzero_cmp cond) x y))+(rule (lower (select _ cond x y)) (lower_select (is_nonzero_cmp cond) x y)); Recursion: at most once to swap the And case for an Or.(decl rec lower_select (CondResult Value Value) InstOutput)@@ -2249,21 +2249,21 @@;; Specializations for floating-point compares to generate a `mins*` or a;; `maxs*` instruction. These are equivalent to the "pseudo-m{in,ax}";; specializations for vectors.-(rule 3 (lower (has_type $F32 (select (maybe_uextend (fcmp (FloatCC.LessThan) x y)) x y)))+(rule 3 (lower (has_type $F32 (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) x y)) x y)))(x64_minss x y))-(rule 3 (lower (has_type $F64 (select (maybe_uextend (fcmp (FloatCC.LessThan) x y)) x y)))+(rule 3 (lower (has_type $F64 (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) x y)) x y)))(x64_minsd x y))-(rule 4 (lower (has_type $F32 (select (maybe_uextend (fcmp (FloatCC.LessThan) y x)) x y)))+(rule 4 (lower (has_type $F32 (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) y x)) x y)))(x64_maxss x y))-(rule 4 (lower (has_type $F64 (select (maybe_uextend (fcmp (FloatCC.LessThan) y x)) x y)))+(rule 4 (lower (has_type $F64 (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) y x)) x y)))(x64_maxsd x y));; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (has_type (ty_32_or_64 ty) (clz src)))+(rule 2 (lower (has_type (ty_32_or_64 ty) (clz _ src)))(do_clz ty ty src))-(rule 1 (lower (has_type (ty_8_or_16 ty) (clz src)))+(rule 1 (lower (has_type (ty_8_or_16 ty) (clz _ src)))(let ((extended Gpr (extend_to_gpr src $I64 (ExtendKind.Zero)))(clz Gpr (do_clz $I64 $I64 extended)))(x64_sub $I64 clz (RegMemImm.Imm (u32_wrapping_sub 64 (ty_bits ty))))))@@ -2271,7 +2271,7 @@(rule 0 (lower(has_type $I128- (clz src)))+ (clz _ src)))(let ((upper Gpr (do_clz $I64 $I64 (value_regs_get_gpr src 1)))(lower Gpr (x64_add $I64(do_clz $I64 $I64 (value_regs_get_gpr src 0))@@ -2300,17 +2300,17 @@;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (has_type (ty_32_or_64 ty) (ctz src)))+(rule 2 (lower (has_type (ty_32_or_64 ty) (ctz _ src)))(do_ctz ty ty src))-(rule 1 (lower (has_type (ty_8_or_16 ty) (ctz src)))+(rule 1 (lower (has_type (ty_8_or_16 ty) (ctz _ src)))(let ((extended Gpr (extend_to_gpr src $I32 (ExtendKind.Zero)))(stopbit Gpr (x64_or $I32 extended (RegMemImm.Imm (u32_wrapping_shl 1 (ty_bits ty))))))(do_ctz $I32 ty stopbit)))(rule 0 (lower(has_type $I128- (ctz src)))+ (ctz _ src)))(let ((lower Gpr (do_ctz $I64 $I64 (value_regs_get_gpr src 0)))(upper Gpr (x64_add $I64(do_ctz $I64 $I64 (value_regs_get_gpr src 1))@@ -2334,17 +2334,17 @@;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (has_type (ty_32_or_64 ty) (cls src)))+(rule 2 (lower (has_type (ty_32_or_64 ty) (cls _ src)))(do_cls ty src))-(rule 1 (lower (has_type (ty_8_or_16 ty) (cls src)))+(rule 1 (lower (has_type (ty_8_or_16 ty) (cls _ src)))(let ((extended Gpr (extend_to_gpr src $I32 (ExtendKind.Sign)))(cls Gpr (do_cls $I32 extended)))(x64_sub $I32 cls (RegMemImm.Imm (u32_wrapping_sub 32 (ty_bits ty))))))(rule 0 (lower(has_type $I128- (cls src)))+ (cls _ src)))(let ((upper Gpr (do_cls $I64 (value_regs_get_gpr src 1)))(sign_fill Gpr (x64_sarq_mi (value_regs_get_gpr src 1) 63))(xored Gpr (x64_xor $I64 (value_regs_get_gpr src 0) sign_fill))@@ -2369,15 +2369,15 @@;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 4 (lower (has_type (ty_32_or_64 ty) (popcnt src)))+(rule 4 (lower (has_type (ty_32_or_64 ty) (popcnt _ src)))(if-let true (use_popcnt))(x64_popcnt ty src))-(rule 3 (lower (has_type (ty_8_or_16 ty) (popcnt src)))+(rule 3 (lower (has_type (ty_8_or_16 ty) (popcnt _ src)))(if-let true (use_popcnt))(x64_popcnt $I32 (extend_to_gpr src $I32 (ExtendKind.Zero))))-(rule 1 (lower (has_type $I128 (popcnt src)))+(rule 1 (lower (has_type $I128 (popcnt _ src)))(if-let true (use_popcnt))(let ((lo_count Gpr (x64_popcnt $I64 (value_regs_get_gpr src 0)))(hi_count Gpr (x64_popcnt $I64 (value_regs_get_gpr src 1))))@@ -2385,17 +2385,17 @@(rule -1 (lower(has_type (ty_32_or_64 ty)- (popcnt src)))+ (popcnt _ src)))(do_popcnt ty src))(rule -2 (lower(has_type (ty_8_or_16 ty)- (popcnt src)))+ (popcnt _ src)))(do_popcnt $I32 (extend_to_gpr src $I32 (ExtendKind.Zero))))(rule (lower(has_type $I128- (popcnt src)))+ (popcnt _ src)))(let ((lo_count Gpr (do_popcnt $I64 (value_regs_get_gpr src 0)))(hi_count Gpr (do_popcnt $I64 (value_regs_get_gpr src 1))))(value_regs (x64_add $I64 lo_count hi_count) (imm $I64 0))))@@ -2462,7 +2462,7 @@final))-(rule 2 (lower (has_type $I8X16 (popcnt src)))+(rule 2 (lower (has_type $I8X16 (popcnt _ src)))(if-let true (has_avx512vl))(if-let true (has_avx512bitalg))(x64_vpopcntb src))@@ -2488,7 +2488,7 @@;; __m128i lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);-(rule 1 (lower (has_type $I8X16 (popcnt src)))+(rule 1 (lower (has_type $I8X16 (popcnt _ src)))(if-let true (has_ssse3))(let ((low_mask XmmMem (emit_u128_le_const 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f))(low_nibbles Xmm (sse_and $I8X16 src low_mask))@@ -2504,7 +2504,7 @@(x64_paddb bit_counts_low bit_counts_high)));; A modified version of the popcnt method from Hacker's Delight.-(rule (lower (has_type $I8X16 (popcnt src)))+(rule (lower (has_type $I8X16 (popcnt _ src)))(let ((mask1 XmmMem (emit_u128_le_const 0x77777777777777777777777777777777))(src Xmm src)(shifted Xmm (x64_pand (x64_psrlq src (xmi_imm 1)) mask1))@@ -2518,19 +2518,19 @@;; Rules for `bitrev` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (bitrev src)))+(rule (lower (has_type $I8 (bitrev _ src)))(do_bitrev8 $I32 src))-(rule (lower (has_type $I16 (bitrev src)))+(rule (lower (has_type $I16 (bitrev _ src)))(do_bitrev16 $I32 src))-(rule (lower (has_type $I32 (bitrev src)))+(rule (lower (has_type $I32 (bitrev _ src)))(do_bitrev32 $I32 src))-(rule (lower (has_type $I64 (bitrev src)))+(rule (lower (has_type $I64 (bitrev _ src)))(do_bitrev64 $I64 src))-(rule (lower (has_type $I128 (bitrev src)))+(rule (lower (has_type $I128 (bitrev _ src)))(value_regs(do_bitrev64 $I64 (value_regs_get_gpr src 1))(do_bitrev64 $I64 (value_regs_get_gpr src 0))))@@ -2597,16 +2597,16 @@;; x64 bswap instruction is only for 32- or 64-bit swaps;; implement the 16-bit swap as a rotl by 8-(rule (lower (has_type $I16 (bswap src)))+(rule (lower (has_type $I16 (bswap _ src)))(x64_rolw_mi src 8))-(rule (lower (has_type $I32 (bswap src)))+(rule (lower (has_type $I32 (bswap _ src)))(x64_bswap $I32 src))-(rule (lower (has_type $I64 (bswap src)))+(rule (lower (has_type $I64 (bswap _ src)))(x64_bswap $I64 src))-(rule (lower (has_type $I128 (bswap src)))+(rule (lower (has_type $I128 (bswap _ src)))(value_regs(x64_bswap $I64 (value_regs_get_gpr src 1))(x64_bswap $I64 (value_regs_get_gpr src 0))))@@ -2614,16 +2614,16 @@;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; I{8,16,32,64} -> I128.-(rule (lower (has_type $I128 (uextend src)))+(rule (lower (has_type $I128 (uextend _ src)))(value_regs (extend_to_gpr src $I64 (ExtendKind.Zero)) (imm $I64 0)));; I{8,16,32} -> I64.-(rule (lower (has_type $I64 (uextend src)))+(rule (lower (has_type $I64 (uextend _ src)))(extend_to_gpr src $I64 (ExtendKind.Zero)));; I{8,16} -> I32;; I8 -> I16-(rule -1 (lower (has_type (fits_in_32 _) (uextend src)))+(rule -1 (lower (has_type (fits_in_32 _) (uextend _ src)))(extend_to_gpr src $I32 (ExtendKind.Zero)));; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2632,30 +2632,30 @@;;;; Produce upper 64 bits sign-extended from lower 64: shift right by;; 63 bits to spread the sign bit across the result.-(rule (lower (has_type $I128 (sextend src)))+(rule (lower (has_type $I128 (sextend _ src)))(let ((lo Gpr (extend_to_gpr src $I64 (ExtendKind.Sign)))(hi Gpr (x64_sarq_mi lo 63)))(value_regs lo hi)));; I{8,16,32} -> I64.-(rule (lower (has_type $I64 (sextend src)))+(rule (lower (has_type $I64 (sextend _ src)))(extend_to_gpr src $I64 (ExtendKind.Sign)));; I{8,16} -> I32;; I8 -> I16-(rule -1 (lower (has_type (fits_in_32 _) (sextend src)))+(rule -1 (lower (has_type (fits_in_32 _) (sextend _ src)))(extend_to_gpr src $I32 (ExtendKind.Sign)));; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; T -> T is always a no-op, even I128 -> I128.-(rule (lower (has_type ty (ireduce src @ (value_type ty))))+(rule (lower (has_type ty (ireduce _ src @ (value_type ty))))src);; T -> I{64,32,16,8}: We can simply pass through the value: values;; are always stored with high bits undefined, so we can just leave;; them be.-(rule 1 (lower (has_type (fits_in_64 ty) (ireduce src)))+(rule 1 (lower (has_type (fits_in_64 ty) (ireduce _ src)))(value_regs_get_gpr src 0));; Rules for `debugtrap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2665,107 +2665,107 @@;; Rules for `x86_pmaddubsw` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16X8 (x86_pmaddubsw x y)))+(rule (lower (has_type $I16X8 (x86_pmaddubsw _ x y)))(if-let true (has_ssse3))(x64_pmaddubsw y x));; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fadd x y)))+(rule (lower (has_type $F32 (fadd _ x y)))(x64_addss x y))-(rule (lower (has_type $F64 (fadd x y)))+(rule (lower (has_type $F64 (fadd _ x y)))(x64_addsd x y))-(rule (lower (has_type $F32X4 (fadd x y)))+(rule (lower (has_type $F32X4 (fadd _ x y)))(x64_addps x y))-(rule (lower (has_type $F64X2 (fadd x y)))+(rule (lower (has_type $F64X2 (fadd _ x y)))(x64_addpd x y));; The above rules automatically sink loads for rhs operands, so additionally;; add rules for sinking loads with lhs operands.-(rule 1 (lower (has_type $F32 (fadd (sinkable_load x) y)))+(rule 1 (lower (has_type $F32 (fadd _ (sinkable_load x) y)))(x64_addss y x))-(rule 1 (lower (has_type $F64 (fadd (sinkable_load x) y)))+(rule 1 (lower (has_type $F64 (fadd _ (sinkable_load x) y)))(x64_addsd y x))-(rule 1 (lower (has_type $F32X4 (fadd (sinkable_load x) y)))+(rule 1 (lower (has_type $F32X4 (fadd _ (sinkable_load x) y)))(x64_addps y x))-(rule 1 (lower (has_type $F64X2 (fadd (sinkable_load x) y)))+(rule 1 (lower (has_type $F64X2 (fadd _ (sinkable_load x) y)))(x64_addpd y x));; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fsub x y)))+(rule (lower (has_type $F32 (fsub _ x y)))(x64_subss x y))-(rule (lower (has_type $F64 (fsub x y)))+(rule (lower (has_type $F64 (fsub _ x y)))(x64_subsd x y))-(rule (lower (has_type $F32X4 (fsub x y)))+(rule (lower (has_type $F32X4 (fsub _ x y)))(x64_subps x y))-(rule (lower (has_type $F64X2 (fsub x y)))+(rule (lower (has_type $F64X2 (fsub _ x y)))(x64_subpd x y));; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmul x y)))+(rule (lower (has_type $F32 (fmul _ x y)))(x64_mulss x y))-(rule (lower (has_type $F64 (fmul x y)))+(rule (lower (has_type $F64 (fmul _ x y)))(x64_mulsd x y))-(rule (lower (has_type $F32X4 (fmul x y)))+(rule (lower (has_type $F32X4 (fmul _ x y)))(x64_mulps x y))-(rule (lower (has_type $F64X2 (fmul x y)))+(rule (lower (has_type $F64X2 (fmul _ x y)))(x64_mulpd x y));; The above rules automatically sink loads for rhs operands, so additionally;; add rules for sinking loads with lhs operands.-(rule 1 (lower (has_type $F32 (fmul (sinkable_load x) y)))+(rule 1 (lower (has_type $F32 (fmul _ (sinkable_load x) y)))(x64_mulss y x))-(rule 1 (lower (has_type $F64 (fmul (sinkable_load x) y)))+(rule 1 (lower (has_type $F64 (fmul _ (sinkable_load x) y)))(x64_mulsd y x))-(rule 1 (lower (has_type $F32X4 (fmul (sinkable_load x) y)))+(rule 1 (lower (has_type $F32X4 (fmul _ (sinkable_load x) y)))(x64_mulps y x))-(rule 1 (lower (has_type $F64X2 (fmul (sinkable_load x) y)))+(rule 1 (lower (has_type $F64X2 (fmul _ (sinkable_load x) y)))(x64_mulpd y x));; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdiv x y)))+(rule (lower (has_type $F32 (fdiv _ x y)))(x64_divss x y))-(rule (lower (has_type $F64 (fdiv x y)))+(rule (lower (has_type $F64 (fdiv _ x y)))(x64_divsd x y))-(rule (lower (has_type $F32X4 (fdiv x y)))+(rule (lower (has_type $F32X4 (fdiv _ x y)))(x64_divps x y))-(rule (lower (has_type $F64X2 (fdiv x y)))+(rule (lower (has_type $F64X2 (fdiv _ x y)))(x64_divpd x y));; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (sqrt x)))+(rule (lower (has_type $F32 (sqrt _ x)))(x64_sqrtss (xmm_zero $F32X4) x))-(rule (lower (has_type $F64 (sqrt x)))+(rule (lower (has_type $F64 (sqrt _ x)))(x64_sqrtsd (xmm_zero $F64X2) x))-(rule (lower (has_type $F32X4 (sqrt x)))+(rule (lower (has_type $F32X4 (sqrt _ x)))(x64_sqrtps x))-(rule (lower (has_type $F64X2 (sqrt x)))+(rule (lower (has_type $F64X2 (sqrt _ x)))(x64_sqrtpd x));; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F64 (fpromote x)))+(rule (lower (has_type $F64 (fpromote _ x)))(x64_cvtss2sd (xmm_zero $F64X2) x));; Rules for `fvpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F64X2 (fvpromote_low x)))+(rule (lower (has_type $F64X2 (fvpromote_low _ x)))(x64_cvtps2pd (put_in_xmm x)));; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdemote x)))+(rule (lower (has_type $F32 (fdemote _ x)))(x64_cvtsd2ss (xmm_zero $F32X4) x));; Rules for `fvdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32X4 (fvdemote x)))+(rule (lower (has_type $F32X4 (fvdemote _ x)))(x64_cvtpd2ps x));; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmin x y)))+(rule (lower (has_type $F32 (fmin _ x y)))(xmm_min_max_seq $F32 true x y))-(rule (lower (has_type $F64 (fmin x y)))+(rule (lower (has_type $F64 (fmin _ x y)))(xmm_min_max_seq $F64 true x y));; Vector-typed version. We don't use single pseudoinstructions as@@ -2780,7 +2780,7 @@;; require that fmin(NaN, _) = fmin(_, NaN) = NaN, and fmin(+0, -0) =;; fmin(-0, +0) = -0.-(rule (lower (has_type $F32X4 (fmin x y)))+(rule (lower (has_type $F32X4 (fmin _ x y)));; Compute min(x, y) and min(y, x) with native;; instructions. These will differ in one of the edge cases;; above that we have to handle properly. (Conversely, if they@@ -2825,7 +2825,7 @@;; Likewise for F64 lanes, except that the right-shift is by 13 bits;; (1 sign, 11 exponent, 1 QNaN bit).-(rule (lower (has_type $F64X2 (fmin x y)))+(rule (lower (has_type $F64X2 (fmin _ x y)))(let ((x Xmm x) ;; force x/y into registers and disallow load sinking(y Xmm y)(min1 Xmm (x64_minpd x y))@@ -2839,15 +2839,15 @@;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmax x y)))+(rule (lower (has_type $F32 (fmax _ x y)))(xmm_min_max_seq $F32 false x y))-(rule (lower (has_type $F64 (fmax x y)))+(rule (lower (has_type $F64 (fmax _ x y)))(xmm_min_max_seq $F64 false x y));; The vector version of fmax here is a dual to the fmin sequence;; above, almost, with a few differences.-(rule (lower (has_type $F32X4 (fmax x y)))+(rule (lower (has_type $F32X4 (fmax _ x y)));; Compute max(x, y) and max(y, x) with native;; instructions. These will differ in one of the edge cases;; above that we have to handle properly. (Conversely, if they@@ -2892,7 +2892,7 @@(final Xmm (x64_andnps nan_fraction_mask max_blended_nan_positive)))final))-(rule (lower (has_type $F64X2 (fmax x y)))+(rule (lower (has_type $F64X2 (fmax _ x y)));; Compute max(x, y) and max(y, x) with native;; instructions. These will differ in one of the edge cases;; above that we have to handle properly. (Conversely, if they@@ -2943,12 +2943,12 @@;; Base case for fma is to call out to one of two libcalls. For vectors they;; need to be decomposed, handle each element individually, and then recomposed.-(rule (lower (has_type $F32 (fma x y z)))+(rule (lower (has_type $F32 (fma _ x y z)))(libcall_3 (LibCall.FmaF32) x y z))-(rule (lower (has_type $F64 (fma x y z)))+(rule (lower (has_type $F64 (fma _ x y z)))(libcall_3 (LibCall.FmaF64) x y z))-(rule (lower (has_type $F32X4 (fma x y z)))+(rule (lower (has_type $F32X4 (fma _ x y z)))(let ((x Xmm (put_in_xmm x))(y Xmm (put_in_xmm y))@@ -2972,7 +2972,7 @@(tmp Xmm (f32x4_insertlane tmp x3 3)))tmp))-(rule (lower (has_type $F64X2 (fma x y z)))+(rule (lower (has_type $F64X2 (fma _ x y z)))(let ((x Xmm (put_in_xmm x))(y Xmm (put_in_xmm y))@@ -2988,7 +2988,7 @@;; Special case for when the `fma` feature is active and a native instruction;; can be used.-(rule 1 (lower (has_type ty (fma x y z)))+(rule 1 (lower (has_type ty (fma _ x y z)))(if-let true (use_fma))(fmadd ty x y z))@@ -3007,15 +3007,15 @@;; If one of the values being multiplied is negated then use a `vfnmadd*`;; instruction instead-(rule 3 (fmadd ty (fneg x) y z) (fnmadd ty x y z))-(rule 4 (fmadd ty x (fneg y) z) (fnmadd ty x y z))+(rule 3 (fmadd ty (fneg _ x) y z) (fnmadd ty x y z))+(rule 4 (fmadd ty x (fneg _ y) z) (fnmadd ty x y z))(rule (fnmadd ty x y z) (x64_vfnmadd213 ty x y z))(rule 1 (fnmadd ty (sinkable_load x) y z) (x64_vfnmadd132 ty y z x))(rule 2 (fnmadd ty x (sinkable_load y) z) (x64_vfnmadd132 ty x z y))-(rule 2 (lower (has_type ty (fma x y (fneg z))))+(rule 2 (lower (has_type ty (fma _ x y (fneg _ z))))(if-let true (use_fma))(fmsub ty x y z))@@ -3034,8 +3034,8 @@;; If one of the values being multiplied is negated then use a `vfnmsub*`;; instruction instead-(rule 3 (fmsub ty (fneg x) y z) (fnmsub ty x y z))-(rule 4 (fmsub ty x (fneg y) z) (fnmsub ty x y z))+(rule 3 (fmsub ty (fneg _ x) y z) (fnmsub ty x y z))+(rule 4 (fmsub ty x (fneg _ y) z) (fnmsub ty x y z))(rule (fnmsub ty x y z) (x64_vfnmsub213 ty x y z))(rule 1 (fnmsub ty (sinkable_load x) y z) (x64_vfnmsub132 ty y z x))@@ -3051,48 +3051,48 @@;;;; By default, we zero-extend all sub-64-bit loads to a GPR.(rule load_sub64_x64_movzx -4 (lower (has_type (and (fits_in_32 ty) (is_gpr_type _))- (load (little_or_native_endian flags) address offset)))+ (load _ (little_or_native_endian flags) address offset)))(x64_movzx (ext_mode (ty_bits_u16 ty) 64) (to_amode flags address offset)));; But if we know that both the `from` and `to` are 64 bits, we simply load with;; no extension.-(rule load_64_x64_movzx -1 (lower (has_type (ty_int_ref_64 ty) (load (little_or_native_endian flags) address offset)))+(rule load_64_x64_movzx -1 (lower (has_type (ty_int_ref_64 ty) (load _ (little_or_native_endian flags) address offset)))(x64_mov (to_amode flags address offset)));; Also, certain scalar loads have a specific `from` width and extension kind;; (signed -> `sx`, zeroed -> `zx`). We overwrite the high bits of the 64-bit;; GPR even if the `to` type is smaller (e.g., 16-bits).-(rule (lower (has_type (is_gpr_type ty) (uload8 (little_or_native_endian flags) address offset)))+(rule (lower (has_type (is_gpr_type ty) (uload8 _ (little_or_native_endian flags) address offset)))(x64_movzx (ExtMode.BQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (sload8 (little_or_native_endian flags) address offset)))+(rule (lower (has_type (is_gpr_type ty) (sload8 _ (little_or_native_endian flags) address offset)))(x64_movsx (ExtMode.BQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (uload16 (little_or_native_endian flags) address offset)))+(rule (lower (has_type (is_gpr_type ty) (uload16 _ (little_or_native_endian flags) address offset)))(x64_movzx (ExtMode.WQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (sload16 (little_or_native_endian flags) address offset)))+(rule (lower (has_type (is_gpr_type ty) (sload16 _ (little_or_native_endian flags) address offset)))(x64_movsx (ExtMode.WQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (uload32 (little_or_native_endian flags) address offset)))+(rule (lower (has_type (is_gpr_type ty) (uload32 _ (little_or_native_endian flags) address offset)))(x64_movzx (ExtMode.LQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (sload32 (little_or_native_endian flags) address offset)))+(rule (lower (has_type (is_gpr_type ty) (sload32 _ (little_or_native_endian flags) address offset)))(x64_movsx (ExtMode.LQ) (to_amode flags address offset)));; To load to XMM registers, we use the x64-specific instructions for each type.;; For `$F32` and `$F64` this is important--we only want to load 32 or 64 bits.;; But for the 128-bit types, this is not strictly necessary for performance but;; might help with clarity during disassembly.-(rule 4 (lower (has_type (is_xmm_type (ty_16 _)) (load (little_or_native_endian flags) address offset)))+(rule 4 (lower (has_type (is_xmm_type (ty_16 _)) (load _ (little_or_native_endian flags) address offset)))(x64_pinsrw (xmm_uninit_value) (to_amode flags address offset) 0))-(rule 3 (lower (has_type (is_xmm_type (ty_32 _)) (load (little_or_native_endian flags) address offset)))+(rule 3 (lower (has_type (is_xmm_type (ty_32 _)) (load _ (little_or_native_endian flags) address offset)))(x64_movss_load (to_amode flags address offset)))-(rule 2 (lower (has_type (is_xmm_type (ty_64 _)) (load (little_or_native_endian flags) address offset)))+(rule 2 (lower (has_type (is_xmm_type (ty_64 _)) (load _ (little_or_native_endian flags) address offset)))(x64_movsd_load (to_amode flags address offset)))-(rule 1 (lower (has_type $F32X4 (load (little_or_native_endian flags) address offset)))+(rule 1 (lower (has_type $F32X4 (load _ (little_or_native_endian flags) address offset)))(x64_movups_load (to_amode flags address offset)))-(rule 1 (lower (has_type $F64X2 (load (little_or_native_endian flags) address offset)))+(rule 1 (lower (has_type $F64X2 (load _ (little_or_native_endian flags) address offset)))(x64_movupd_load (to_amode flags address offset)))-(rule 0 (lower (has_type (is_xmm_type (ty_128 _)) (load (little_or_native_endian flags) address offset)))+(rule 0 (lower (has_type (is_xmm_type (ty_128 _)) (load _ (little_or_native_endian flags) address offset)))(x64_movdqu_load (to_amode flags address offset)));; We can load an I128 by doing two 64-bit loads.(rule -3 (lower (has_type $I128- (load (little_or_native_endian flags) address offset)))+ (load _ (little_or_native_endian flags) address offset)))(let ((addr_lo SyntheticAmode (to_amode flags address offset))(addr_hi SyntheticAmode (amode_offset addr_lo 8))(value_lo Reg (x64_mov addr_lo))@@ -3101,36 +3101,36 @@;; We also include widening vector loads; these sign- or zero-extend each lane;; to the next wider width (e.g., 16x4 -> 32x4).-(rule 1 (lower (has_type $I16X8 (sload8x8 (little_or_native_endian flags) address offset)))+(rule 1 (lower (has_type $I16X8 (sload8x8 _ (little_or_native_endian flags) address offset)))(if-let true (has_sse41))(x64_pmovsxbw (to_amode flags address offset)))-(rule 1 (lower (has_type $I16X8 (uload8x8 (little_or_native_endian flags) address offset)))+(rule 1 (lower (has_type $I16X8 (uload8x8 _ (little_or_native_endian flags) address offset)))(if-let true (has_sse41))(x64_pmovzxbw (to_amode flags address offset)))-(rule 1 (lower (has_type $I32X4 (sload16x4 (little_or_native_endian flags) address offset)))+(rule 1 (lower (has_type $I32X4 (sload16x4 _ (little_or_native_endian flags) address offset)))(if-let true (has_sse41))(x64_pmovsxwd (to_amode flags address offset)))-(rule 1 (lower (has_type $I32X4 (uload16x4 (little_or_native_endian flags) address offset)))+(rule 1 (lower (has_type $I32X4 (uload16x4 _ (little_or_native_endian flags) address offset)))(if-let true (has_sse41))(x64_pmovzxwd (to_amode flags address offset)))-(rule 1 (lower (has_type $I64X2 (sload32x2 (little_or_native_endian flags) address offset)))+(rule 1 (lower (has_type $I64X2 (sload32x2 _ (little_or_native_endian flags) address offset)))(if-let true (has_sse41))(x64_pmovsxdq (to_amode flags address offset)))-(rule 1 (lower (has_type $I64X2 (uload32x2 (little_or_native_endian flags) address offset)))+(rule 1 (lower (has_type $I64X2 (uload32x2 _ (little_or_native_endian flags) address offset)))(if-let true (has_sse41))(x64_pmovzxdq (to_amode flags address offset)))-(rule (lower (has_type $I16X8 (sload8x8 (little_or_native_endian flags) address offset)))+(rule (lower (has_type $I16X8 (sload8x8 _ (little_or_native_endian flags) address offset)))(lower_swiden_low $I16X8 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I16X8 (uload8x8 (little_or_native_endian flags) address offset)))+(rule (lower (has_type $I16X8 (uload8x8 _ (little_or_native_endian flags) address offset)))(lower_uwiden_low $I16X8 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I32X4 (sload16x4 (little_or_native_endian flags) address offset)))+(rule (lower (has_type $I32X4 (sload16x4 _ (little_or_native_endian flags) address offset)))(lower_swiden_low $I32X4 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I32X4 (uload16x4 (little_or_native_endian flags) address offset)))+(rule (lower (has_type $I32X4 (uload16x4 _ (little_or_native_endian flags) address offset)))(lower_uwiden_low $I32X4 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I64X2 (sload32x2 (little_or_native_endian flags) address offset)))+(rule (lower (has_type $I64X2 (sload32x2 _ (little_or_native_endian flags) address offset)))(lower_swiden_low $I64X2 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I64X2 (uload32x2 (little_or_native_endian flags) address offset)))+(rule (lower (has_type $I64X2 (uload32x2 _ (little_or_native_endian flags) address offset)))(lower_uwiden_low $I64X2 (x64_movq_to_xmm (to_amode flags address offset))));; Rules for `store*` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3237,40 +3237,40 @@;; f32 or f64 despite the source perhaps being an integer vector since the;; result of the instruction is the same.(rule 2 (lower (store (little_or_native_endian flags)- (has_type $F32 (extractlane value (u8_from_uimm8 0)))+ (has_type $F32 (extractlane _ value (u8_from_uimm8 0)))addressoffset))(side_effect(x64_movss_store (to_amode flags address offset) value)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $F64 (extractlane value (u8_from_uimm8 0)))+ (has_type $F64 (extractlane _ value (u8_from_uimm8 0)))addressoffset))(side_effect(x64_movsd_store (to_amode flags address offset) value)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $I8 (extractlane value (u8_from_uimm8 n)))+ (has_type $I8 (extractlane _ value (u8_from_uimm8 n)))addressoffset))(if-let true (has_sse41))(side_effect(x64_pextrb_store (to_amode flags address offset) value n)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $I16 (extractlane value (u8_from_uimm8 n)))+ (has_type $I16 (extractlane _ value (u8_from_uimm8 n)))addressoffset))(if-let true (has_sse41))(side_effect(x64_pextrw_store (to_amode flags address offset) value n)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $I32 (extractlane value (u8_from_uimm8 n)))+ (has_type $I32 (extractlane _ value (u8_from_uimm8 n)))addressoffset))(if-let true (has_sse41))(side_effect(x64_pextrd_store (to_amode flags address offset) value n)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $I64 (extractlane value (u8_from_uimm8 n)))+ (has_type $I64 (extractlane _ value (u8_from_uimm8 n)))addressoffset))(if-let true (has_sse41))@@ -3283,9 +3283,9 @@(rule store_x64_add_mem 3 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (iadd (and+ (iadd _ (and(sinkable_load sink)- (load flags addr offset))+ (load _ flags addr offset))src2))addroffset))@@ -3297,10 +3297,10 @@(rule 2 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (iadd src2+ (iadd _ src2(and(sinkable_load sink)- (load flags addr offset))))+ (load _ flags addr offset))))addroffset))(let ((_ RegMemImm sink))@@ -3311,9 +3311,9 @@(rule 2 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (isub (and+ (isub _ (and(sinkable_load sink)- (load flags addr offset))+ (load _ flags addr offset))src2))addroffset))@@ -3325,9 +3325,9 @@(rule 3 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (band (and+ (band _ (and(sinkable_load sink)- (load flags addr offset))+ (load _ flags addr offset))src2))addroffset))@@ -3339,10 +3339,10 @@(rule 2 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (band src2+ (band _ src2(and(sinkable_load sink)- (load flags addr offset))))+ (load _ flags addr offset))))addroffset))(let ((_ RegMemImm sink))@@ -3353,9 +3353,9 @@(rule 3 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (bor (and+ (bor _ (and(sinkable_load sink)- (load flags addr offset))+ (load _ flags addr offset))src2))addroffset))@@ -3367,10 +3367,10 @@(rule 2 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (bor src2+ (bor _ src2(and(sinkable_load sink)- (load flags addr offset))))+ (load _ flags addr offset))))addroffset))(let ((_ RegMemImm sink))@@ -3381,9 +3381,9 @@(rule 3 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (bxor (and+ (bxor _ (and(sinkable_load sink)- (load flags addr offset))+ (load _ flags addr offset))src2))addroffset))@@ -3395,10 +3395,10 @@(rule 2 (lower(store (little_or_native_endian flags)(has_type (ty_32_or_64 ty)- (bxor src2+ (bxor _ src2(and(sinkable_load sink)- (load flags addr offset))))+ (load _ flags addr offset))))addroffset))(let ((_ RegMemImm sink))@@ -3412,12 +3412,12 @@;; Rules for `func_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (func_addr (func_ref_data _ extname dist _)))+(rule (lower (func_addr _ (func_ref_data _ extname dist _)))(load_ext_name extname 0 dist));; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (symbol_value (symbol_value_data extname dist offset)))+(rule (lower (symbol_value _ (symbol_value_data extname dist offset)))(load_ext_name extname offset dist));; Rules for `atomic_load` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3428,12 +3428,12 @@;;;; This lowering is only valid for I8, I16, I32, and I64. The sub-64-bit types;; are zero extended, as with a normal load.-(rule 1 (lower (has_type $I64 (atomic_load (little_or_native_endian flags) address)))+(rule 1 (lower (has_type $I64 (atomic_load _ (little_or_native_endian flags) address)))(x64_mov (to_amode flags address (zero_offset))))-(rule (lower (has_type (and (fits_in_32 ty) (ty_int _)) (atomic_load (little_or_native_endian flags) address)))+(rule (lower (has_type (and (fits_in_32 ty) (ty_int _)) (atomic_load _ (little_or_native_endian flags) address)))(x64_movzx (ext_mode (ty_bits_u16 ty) 64) (to_amode flags address (zero_offset))));; Lower 128-bit `atomic_load` using `cmpxchg16b`.-(rule 1 (lower (has_type $I128 (atomic_load (little_or_native_endian flags) address)))+(rule 1 (lower (has_type $I128 (atomic_load _ (little_or_native_endian flags) address)))(if-let true (has_cmpxchg16b))(x64_cmpxchg16b (value_regs (imm $I64 0) (imm $I64 0)) (value_regs (imm $I64 0) (imm $I64 0)) (to_amode flags address (zero_offset))))@@ -3455,9 +3455,9 @@;; Rules for `atomic_cas` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_cas (little_or_native_endian flags) address expected replacement)))+ (atomic_cas _ (little_or_native_endian flags) address expected replacement)))(x64_cmpxchg ty expected replacement (to_amode flags address (zero_offset))))-(rule 1 (lower (has_type $I128 (atomic_cas flags address expected replacement)))+(rule 1 (lower (has_type $I128 (atomic_cas _ flags address expected replacement)))(if-let true (has_cmpxchg16b))(x64_cmpxchg16b expected replacement (to_amode flags address (zero_offset))))@@ -3466,51 +3466,51 @@;; This is a simple, general-case atomic update, based on a loop involving;; `cmpxchg`.(rule (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_rmw (little_or_native_endian flags) op address input)))+ (atomic_rmw _ (little_or_native_endian flags) op address input)))(x64_atomic_rmw_seq ty (atomic_rmw_seq_op op) (to_amode flags address (zero_offset)) input));; `Add` and `Sub` can use `lock xadd`(rule 1 (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Add) address input)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Add) address input)))(x64_xadd ty (to_amode flags address (zero_offset)) input))(rule 1 (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Sub) address input)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) address input)))(x64_xadd ty (to_amode flags address (zero_offset)) (x64_neg ty input)));; `Xchg` can use `xchg`(rule 1 (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Xchg) address input)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xchg) address input)))(x64_xchg ty (to_amode flags address (zero_offset)) input));; `Add`, `Sub`, `And`, `Or` and `Xor` can use `lock`-prefixed instructions if;; the old value is not required.(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Add) address input)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Add) address input)))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_add (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)))(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Sub) address input)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) address input)))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_sub (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)))(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.And) address input)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.And) address input)))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_and (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)))(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Or) address input)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Or) address input)))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_or (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)))(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw (little_or_native_endian flags) (AtomicRmwOp.Xor) address input)))+ (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xor) address input)))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_xor (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)));; 128-bit integers always use a `lock cmpxchg16b` loop.-(rule 3 (lower (has_type $I128 (atomic_rmw (little_or_native_endian flags) op address input)))+(rule 3 (lower (has_type $I128 (atomic_rmw _ (little_or_native_endian flags) op address input)))(if-let true (has_cmpxchg16b))(x64_atomic_128_rmw_seq op (to_amode flags address (zero_offset)) input))@@ -3608,7 +3608,7 @@;; Rules for `stack_switch` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; currently, only the Basic model is supported-(rule (lower (stack_switch store_context_ptr load_context_ptr in_payload0))+(rule (lower (stack_switch _ store_context_ptr load_context_ptr in_payload0))(if-let (StackSwitchModel.Basic) (stack_switch_model))(let ((store_context_ptr Gpr (put_in_gpr store_context_ptr))(load_context_ptr Gpr (put_in_gpr load_context_ptr))@@ -3617,13 +3617,13 @@;;;; Rules for `get_{frame,stack}_pointer` and `get_return_address` ;;;;;;;;;;;;-(rule (lower (get_frame_pointer))+(rule (lower (get_frame_pointer _))(x64_rbp))-(rule (lower (get_stack_pointer))+(rule (lower (get_stack_pointer _))(x64_rsp))-(rule (lower (get_return_address))+(rule (lower (get_return_address _))(x64_load $I64(Amode.ImmReg 8 (x64_rbp) (mem_flags_trusted))(ExtKind.None)))@@ -3651,13 +3651,13 @@;; Rules for `select_spectre_guard` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (select_spectre_guard cond x y))+(rule (lower (select_spectre_guard _ cond x y))(lower_select (is_nonzero_cmp cond) x y));; Note that for GPR-based spectre guards everything is forced into a register;; not `GprMem`. The `lower_select_spectre_gpr` helper below handles "and";; conditions which the `lower_select_gpr` helper does not.-(rule 1 (lower (has_type (is_single_register_gpr_type ty) (select_spectre_guard cond x y)))+(rule 1 (lower (has_type (is_single_register_gpr_type ty) (select_spectre_guard _ cond x y)))(lower_select_spectre_gpr ty (is_nonzero_cmp cond) (put_in_gpr x) y))(decl lower_select_spectre_gpr (Type CondResult Gpr Gpr) Gpr)@@ -3684,31 +3684,31 @@;; `fcvt_from_sint` to an x86-specific opcode using a zero constant which would;; be subject to normal LICM, but that's not feasible today.-(rule 2 (lower (has_type $F32 (fcvt_from_sint a @ (value_type $I8))))+(rule 2 (lower (has_type $F32 (fcvt_from_sint _ a @ (value_type $I8))))(x64_cvtsi2ss $I32 (xmm_zero $F32X4) (extend_to_gpr a $I32 (ExtendKind.Sign))))-(rule 2 (lower (has_type $F32 (fcvt_from_sint a @ (value_type $I16))))+(rule 2 (lower (has_type $F32 (fcvt_from_sint _ a @ (value_type $I16))))(x64_cvtsi2ss $I32 (xmm_zero $F32X4) (extend_to_gpr a $I32 (ExtendKind.Sign))))-(rule 1 (lower (has_type $F32 (fcvt_from_sint a @ (value_type (ty_int (fits_in_64 ty))))))+(rule 1 (lower (has_type $F32 (fcvt_from_sint _ a @ (value_type (ty_int (fits_in_64 ty))))))(x64_cvtsi2ss ty (xmm_zero $F32X4) a))-(rule 2 (lower (has_type $F64 (fcvt_from_sint a @ (value_type $I8))))+(rule 2 (lower (has_type $F64 (fcvt_from_sint _ a @ (value_type $I8))))(x64_cvtsi2sd $I32 (xmm_zero $F64X2) (extend_to_gpr a $I32 (ExtendKind.Sign))))-(rule 2 (lower (has_type $F64 (fcvt_from_sint a @ (value_type $I16))))+(rule 2 (lower (has_type $F64 (fcvt_from_sint _ a @ (value_type $I16))))(x64_cvtsi2sd $I32 (xmm_zero $F64X2) (extend_to_gpr a $I32 (ExtendKind.Sign))))-(rule 1 (lower (has_type $F64 (fcvt_from_sint a @ (value_type (ty_int (fits_in_64 ty))))))+(rule 1 (lower (has_type $F64 (fcvt_from_sint _ a @ (value_type (ty_int (fits_in_64 ty))))))(x64_cvtsi2sd ty (xmm_zero $F64X2) a))-(rule 0 (lower (fcvt_from_sint a @ (value_type $I32X4)))+(rule 0 (lower (fcvt_from_sint _ a @ (value_type $I32X4)))(x64_cvtdq2ps a));; Base case: decompose the i64x2 input into two scalar registers and convert;; each of those into a float. Afterwards re-pack the two results into the final;; destination.-(rule 0 (lower (fcvt_from_sint a @ (value_type $I64X2)))+(rule 0 (lower (fcvt_from_sint _ a @ (value_type $I64X2)))(let ((a Xmm a)(zero Xmm (xmm_zero $F64X2))@@ -3717,18 +3717,18 @@)(x64_unpcklpd f0 f1)))-(rule 1 (lower (has_type $F64X2 (fcvt_from_sint (swiden_low a @ (value_type $I32X4)))))+(rule 1 (lower (has_type $F64X2 (fcvt_from_sint _ (swiden_low _ a @ (value_type $I32X4)))))(x64_cvtdq2pd a));; Rules for `fcvt_from_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $F32 (fcvt_from_uint val @ (value_type (fits_in_32 (ty_int ty))))))+(rule 1 (lower (has_type $F32 (fcvt_from_uint _ val @ (value_type (fits_in_32 (ty_int ty))))))(x64_cvtsi2ss $I64 (xmm_zero $F32X4) (extend_to_gpr val $I64 (ExtendKind.Zero))))-(rule 1 (lower (has_type $F64 (fcvt_from_uint val @ (value_type (fits_in_32 (ty_int ty))))))+(rule 1 (lower (has_type $F64 (fcvt_from_uint _ val @ (value_type (fits_in_32 (ty_int ty))))))(x64_cvtsi2sd $I64 (xmm_zero $F64X2) (extend_to_gpr val $I64 (ExtendKind.Zero))))-(rule (lower (has_type ty (fcvt_from_uint val @ (value_type $I64))))+(rule (lower (has_type ty (fcvt_from_uint _ val @ (value_type $I64))))(cvt_u64_to_float_seq ty val));; Base case of u64x2 being converted to f64x2. No native instruction for this@@ -3759,7 +3759,7 @@;; number. The final step in combining via float arithmetic will chop off the;; leading `1.` at the start of the float that we constructed, one for the low;; half and one for the upper half.-(rule -1 (lower (has_type $F64X2 (fcvt_from_uint val @ (value_type $I64X2))))+(rule -1 (lower (has_type $F64X2 (fcvt_from_uint _ val @ (value_type $I64X2))))(let ((low32_mask XmmMem (emit_u128_le_const 0x00000000ffffffff_00000000ffffffff))(float_1p52 XmmMem (emit_u128_le_const 0x4330000000000000_4330000000000000))(float_1p84 XmmMem (emit_u128_le_const 0x4530000000000000_4530000000000000))@@ -3774,7 +3774,7 @@;; 0x1.0p52 + double(src). 0x1.0p52 is unique because at this exponent;; every value of the mantissa represents a corresponding uint32 number.;; When we subtract 0x1.0p52 we are left with double(src).-(rule 1 (lower (has_type $F64X2 (fcvt_from_uint (uwiden_low val @ (value_type $I32X4)))))+(rule 1 (lower (has_type $F64X2 (fcvt_from_uint _ (uwiden_low _ val @ (value_type $I32X4)))))(let ((uint_mask XmmMem (emit_u128_le_const 0x43300000_43300000))(res Xmm (x64_unpcklps val uint_mask))(uint_mask_high XmmMem (emit_u128_le_const 0x4330000000000000_4330000000000000)))@@ -3782,7 +3782,7 @@;; When AVX512VL and AVX512F are available,;; `fcvt_from_uint` can be lowered to a single instruction.-(rule 2 (lower (has_type $F32X4 (fcvt_from_uint src)))+(rule 2 (lower (has_type $F32X4 (fcvt_from_uint _ src)))(if-let true (has_avx512vl))(if-let true (has_avx512f))(x64_vcvtudq2ps src))@@ -3810,7 +3810,7 @@;; -> Convert(Ah) // Convert .. with no loss of significant digits from previous shift;; -> Ah = Ah + Ah // Double Ah to account for shift right before the conversion.;; -> dst = Ah + Al // Add the two floats together-(rule 1 (lower (has_type $F32X4 (fcvt_from_uint val)))+(rule 1 (lower (has_type $F32X4 (fcvt_from_uint _ val)))(let ((a Xmm val);; get the low 16 bits@@ -3834,20 +3834,20 @@;; Rules for `fcvt_to_uint` and `fcvt_to_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type out_ty (fcvt_to_uint val @ (value_type (ty_scalar_float _)))))+(rule (lower (has_type out_ty (fcvt_to_uint _ val @ (value_type (ty_scalar_float _)))))(cvt_float_to_uint_seq out_ty val false))-(rule (lower (has_type out_ty (fcvt_to_uint_sat val @ (value_type (ty_scalar_float _)))))+(rule (lower (has_type out_ty (fcvt_to_uint_sat _ val @ (value_type (ty_scalar_float _)))))(cvt_float_to_uint_seq out_ty val true))-(rule (lower (has_type out_ty (fcvt_to_sint val @ (value_type (ty_scalar_float _)))))+(rule (lower (has_type out_ty (fcvt_to_sint _ val @ (value_type (ty_scalar_float _)))))(cvt_float_to_sint_seq out_ty val false))-(rule (lower (has_type out_ty (fcvt_to_sint_sat val @ (value_type (ty_scalar_float _)))))+(rule (lower (has_type out_ty (fcvt_to_sint_sat _ val @ (value_type (ty_scalar_float _)))))(cvt_float_to_sint_seq out_ty val true));; The x64 backend currently only supports these two type combinations.-(rule 1 (lower (has_type $I32X4 (fcvt_to_sint_sat val @ (value_type $F32X4))))+(rule 1 (lower (has_type $I32X4 (fcvt_to_sint_sat _ val @ (value_type $F32X4))))(let ((src Xmm val);; Sets tmp to zero if float is NaN@@ -3918,7 +3918,7 @@;;;; | Step 6 | Step 7 |;; | (0-(INT_MAX+1))..(UINT_MAX-(INT_MAX+1))(w/overflow) | ((INT_MAX+1)-(INT_MAX+1))..(INT_MAX+1) |-(rule 1 (lower (has_type $I32X4 (fcvt_to_uint_sat val @ (value_type $F32X4))))+(rule 1 (lower (has_type $I32X4 (fcvt_to_uint_sat _ val @ (value_type $F32X4))))(let ((src Xmm val);; Converting to unsigned int so if float src is negative or NaN@@ -3965,12 +3965,12 @@;; Rules for `x86_cvtt2dq` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32X4 (x86_cvtt2dq val @ (value_type $F32X4))))+(rule (lower (has_type $I32X4 (x86_cvtt2dq _ val @ (value_type $F32X4))))(x64_cvttps2dq val));; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (iadd_pairwise x y)))+(rule (lower (has_type $I8X16 (iadd_pairwise _ x y)))(let (;; Shuffle all the even lanes of `x` and `y` into one register(even_lane_mask Xmm (x64_movdqu_load (emit_u128_le_const 0x00ff_00ff_00ff_00ff_00ff_00ff_00ff_00ff)))@@ -3986,11 +3986,11 @@(x64_paddb evens odds)))-(rule 1 (lower (has_type $I16X8 (iadd_pairwise x y)))+(rule 1 (lower (has_type $I16X8 (iadd_pairwise _ x y)))(if-let true (has_ssse3))(x64_phaddw x y))-(rule (lower (has_type $I16X8 (iadd_pairwise x y)))+(rule (lower (has_type $I16X8 (iadd_pairwise _ x y)))(let ((x Xmm x)(y Xmm y)@@ -4016,11 +4016,11 @@)(x64_paddw evens odds)))-(rule 1 (lower (has_type $I32X4 (iadd_pairwise x y)))+(rule 1 (lower (has_type $I32X4 (iadd_pairwise _ x y)))(if-let true (has_ssse3))(x64_phaddd x y))-(rule (lower (has_type $I32X4 (iadd_pairwise x y)))+(rule (lower (has_type $I32X4 (iadd_pairwise _ x y)))(let ((x Xmm x)(y Xmm y)@@ -4033,9 +4033,9 @@;; special case for the `i16x8.extadd_pairwise_i8x16_s` wasm instruction(rule 2 (lower- (has_type $I16X8 (iadd_pairwise- (swiden_low val @ (value_type $I8X16))- (swiden_high val))))+ (has_type $I16X8 (iadd_pairwise _+ (swiden_low _ val @ (value_type $I8X16))+ (swiden_high _ val))))(if-let true (has_ssse3))(let ((mul_const Xmm (x64_xmm_load_const $I8X16(emit_u128_le_const 0x01010101010101010101010101010101))))@@ -4043,26 +4043,26 @@;; special case for the `i32x4.extadd_pairwise_i16x8_s` wasm instruction(rule 2 (lower- (has_type $I32X4 (iadd_pairwise- (swiden_low val @ (value_type $I16X8))- (swiden_high val))))+ (has_type $I32X4 (iadd_pairwise _+ (swiden_low _ val @ (value_type $I16X8))+ (swiden_high _ val))))(let ((mul_const XmmMem (emit_u128_le_const 0x0001_0001_0001_0001_0001_0001_0001_0001)))(x64_pmaddwd val mul_const)));; special case for the `i16x8.extadd_pairwise_i8x16_u` wasm instruction(rule 2 (lower- (has_type $I16X8 (iadd_pairwise- (uwiden_low val @ (value_type $I8X16))- (uwiden_high val))))+ (has_type $I16X8 (iadd_pairwise _+ (uwiden_low _ val @ (value_type $I8X16))+ (uwiden_high _ val))))(if-let true (has_ssse3))(let ((mul_const XmmMem (emit_u128_le_const 0x01010101010101010101010101010101)))(x64_pmaddubsw val mul_const)));; special case for the `i32x4.extadd_pairwise_i16x8_u` wasm instruction(rule 2 (lower- (has_type $I32X4 (iadd_pairwise- (uwiden_low val @ (value_type $I16X8))- (uwiden_high val))))+ (has_type $I32X4 (iadd_pairwise _+ (uwiden_low _ val @ (value_type $I16X8))+ (uwiden_high _ val))))(let ((xor_const XmmMem (emit_u128_le_const 0x8000_8000_8000_8000_8000_8000_8000_8000))(dst Xmm (x64_pxor val xor_const))@@ -4074,25 +4074,25 @@;; special case for the `i32x4.dot_i16x8_s` wasm instruction(rule 2 (lower- (has_type $I32X4 (iadd_pairwise- (imul (swiden_low x) (swiden_low y))- (imul (swiden_high x) (swiden_high y)))))+ (has_type $I32X4 (iadd_pairwise _+ (imul _ (swiden_low _ x) (swiden_low _ y))+ (imul _ (swiden_high _ x) (swiden_high _ y)))))(x64_pmaddwd x y));; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; With SSE4.1 use the `pmovsx*` instructions for this-(rule 1 (lower (has_type $I16X8 (swiden_low val @ (value_type $I8X16))))+(rule 1 (lower (has_type $I16X8 (swiden_low _ val @ (value_type $I8X16))))(if-let true (has_sse41))(x64_pmovsxbw val))-(rule 1 (lower (has_type $I32X4 (swiden_low val @ (value_type $I16X8))))+(rule 1 (lower (has_type $I32X4 (swiden_low _ val @ (value_type $I16X8))))(if-let true (has_sse41))(x64_pmovsxwd val))-(rule 1 (lower (has_type $I64X2 (swiden_low val @ (value_type $I32X4))))+(rule 1 (lower (has_type $I64X2 (swiden_low _ val @ (value_type $I32X4))))(if-let true (has_sse41))(x64_pmovsxdq val))-(rule (lower (has_type ty (swiden_low val))) (lower_swiden_low ty val))+(rule (lower (has_type ty (swiden_low _ val))) (lower_swiden_low ty val))(decl lower_swiden_low (Type Xmm) Xmm)@@ -4115,31 +4115,31 @@;; Similar to `swiden_low` with SSE4.1 except that the upper lanes are moved;; to the lower lanes first.-(rule 1 (lower (has_type $I16X8 (swiden_high val @ (value_type $I8X16))))+(rule 1 (lower (has_type $I16X8 (swiden_high _ val @ (value_type $I8X16))))(if-let true (has_sse41))(if-let true (has_ssse3))(let ((x Xmm val))(x64_pmovsxbw (x64_palignr x x 8))))-(rule 1 (lower (has_type $I32X4 (swiden_high val @ (value_type $I16X8))))+(rule 1 (lower (has_type $I32X4 (swiden_high _ val @ (value_type $I16X8))))(if-let true (has_sse41))(if-let true (has_ssse3))(let ((x Xmm val))(x64_pmovsxwd (x64_palignr x x 8))))-(rule 1 (lower (has_type $I64X2 (swiden_high val @ (value_type $I32X4))))+(rule 1 (lower (has_type $I64X2 (swiden_high _ val @ (value_type $I32X4))))(if-let true (has_sse41))(x64_pmovsxdq (x64_pshufd val 0b11_10_11_10)));; Similar to `swiden_low` versions but using `punpckh*` instructions to;; pair the high lanes next to each other.-(rule (lower (has_type $I16X8 (swiden_high val @ (value_type $I8X16))))+(rule (lower (has_type $I16X8 (swiden_high _ val @ (value_type $I8X16))))(let ((val Xmm val))(x64_psraw (x64_punpckhbw val val) (xmi_imm 8))))-(rule (lower (has_type $I32X4 (swiden_high val @ (value_type $I16X8))))+(rule (lower (has_type $I32X4 (swiden_high _ val @ (value_type $I16X8))))(let ((val Xmm val))(x64_psrad (x64_punpckhwd val val) (xmi_imm 16))));; Same as `swiden_low`, but `val` has its high lanes moved down.-(rule (lower (has_type $I64X2 (swiden_high val @ (value_type $I32X4))))+(rule (lower (has_type $I64X2 (swiden_high _ val @ (value_type $I32X4))))(let ((val Xmm (x64_pshufd val 0b00_00_11_10))(tmp Xmm (x64_pcmpgtd_a (xmm_zero $I32X4) val)))(x64_punpckldq val tmp)))@@ -4147,17 +4147,17 @@;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; With SSE4.1 use the `pmovzx*` instructions for this-(rule 1 (lower (has_type $I16X8 (uwiden_low val @ (value_type $I8X16))))+(rule 1 (lower (has_type $I16X8 (uwiden_low _ val @ (value_type $I8X16))))(if-let true (has_sse41))(x64_pmovzxbw val))-(rule 1 (lower (has_type $I32X4 (uwiden_low val @ (value_type $I16X8))))+(rule 1 (lower (has_type $I32X4 (uwiden_low _ val @ (value_type $I16X8))))(if-let true (has_sse41))(x64_pmovzxwd val))-(rule 1 (lower (has_type $I64X2 (uwiden_low val @ (value_type $I32X4))))+(rule 1 (lower (has_type $I64X2 (uwiden_low _ val @ (value_type $I32X4))))(if-let true (has_sse41))(x64_pmovzxdq val))-(rule (lower (has_type ty (uwiden_low val))) (lower_uwiden_low ty val))+(rule (lower (has_type ty (uwiden_low _ val))) (lower_uwiden_low ty val));; Interleave an all-zero register with the low lanes to produce zero-extended;; results.@@ -4172,19 +4172,19 @@;;;; Note that according to `llvm-mca` at least these instructions are faster;; than using `pmovzx*` in terms of cycles, even if SSE4.1 is available.-(rule (lower (has_type $I16X8 (uwiden_high val @ (value_type $I8X16))))+(rule (lower (has_type $I16X8 (uwiden_high _ val @ (value_type $I8X16))))(x64_punpckhbw val (xmm_zero $I8X16)))-(rule (lower (has_type $I32X4 (uwiden_high val @ (value_type $I16X8))))+(rule (lower (has_type $I32X4 (uwiden_high _ val @ (value_type $I16X8))))(x64_punpckhwd val (xmm_zero $I8X16)))-(rule (lower (has_type $I64X2 (uwiden_high val @ (value_type $I32X4))))+(rule (lower (has_type $I64X2 (uwiden_high _ val @ (value_type $I32X4))))(x64_unpckhps val (xmm_zero $F32X4)));; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (snarrow a @ (value_type $I16X8) b)))+(rule (lower (has_type $I8X16 (snarrow _ a @ (value_type $I16X8) b)))(x64_packsswb a b))-(rule (lower (has_type $I16X8 (snarrow a @ (value_type $I32X4) b)))+(rule (lower (has_type $I16X8 (snarrow _ a @ (value_type $I32X4) b)))(x64_packssdw a b));; We're missing a `snarrow` case for $I64X2@@ -4193,8 +4193,8 @@;; This rule is a special case for handling the translation of the wasm op;; `i32x4.trunc_sat_f64x2_s_zero`. It can be removed once we have an;; implementation of `snarrow` for `I64X2`.-(rule (lower (has_type $I32X4 (snarrow (has_type $I64X2 (fcvt_to_sint_sat val))- (vconst (u128_from_constant 0)))))+(rule (lower (has_type $I32X4 (snarrow _ (has_type $I64X2 (fcvt_to_sint_sat _ val))+ (vconst _ (u128_from_constant 0)))))(let ((a Xmm val);; y = i32x4.trunc_sat_f64x2_s_zero(x) is lowered to:@@ -4217,16 +4217,16 @@;; This rule is a special case for handling the translation of the wasm op;; `i32x4.relaxed_trunc_f64x2_s_zero`.-(rule (lower (has_type $I32X4 (snarrow (has_type $I64X2 (x86_cvtt2dq val))- (vconst (u128_from_constant 0)))))+(rule (lower (has_type $I32X4 (snarrow _ (has_type $I64X2 (x86_cvtt2dq _ val))+ (vconst _ (u128_from_constant 0)))))(x64_cvttpd2dq val));; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (unarrow a @ (value_type $I16X8) b)))+(rule (lower (has_type $I8X16 (unarrow _ a @ (value_type $I16X8) b)))(x64_packuswb a b))-(rule 1 (lower (has_type $I16X8 (unarrow a @ (value_type $I32X4) b)))+(rule 1 (lower (has_type $I16X8 (unarrow _ a @ (value_type $I32X4) b)))(if-let true (has_sse41))(x64_packusdw a b))@@ -4236,7 +4236,7 @@;;;; If this is performance sensitive then it's probably best to upgrade the CPU;; to get the above single-instruction lowering.-(rule (lower (has_type $I16X8 (unarrow a @ (value_type $I32X4) b)))+(rule (lower (has_type $I16X8 (unarrow _ a @ (value_type $I32X4) b)))(let ((a Xmm (unarrow_i32x4_lanes_to_low_u16_lanes a))(b Xmm (unarrow_i32x4_lanes_to_low_u16_lanes b))@@ -4273,31 +4273,31 @@;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -3 (lower (has_type (is_gpr_type (fits_in_64 ty)) (bitcast _ src @ (value_type (is_xmm_type _)))))+(rule -3 (lower (has_type (is_gpr_type (fits_in_64 ty)) (bitcast _ _ src @ (value_type (is_xmm_type _)))))(bitcast_xmm_to_gpr (ty_bits ty) src))-(rule -2 (lower (has_type (is_xmm_type (fits_in_64 ty)) (bitcast _ src @ (value_type (is_gpr_type _)))))+(rule -2 (lower (has_type (is_xmm_type (fits_in_64 ty)) (bitcast _ _ src @ (value_type (is_gpr_type _)))))(bitcast_gpr_to_xmm (ty_bits ty) src))-(rule -1 (lower (has_type $I128 (bitcast _ src @ (value_type (is_xmm_type _)))))+(rule -1 (lower (has_type $I128 (bitcast _ _ src @ (value_type (is_xmm_type _)))))(bitcast_xmm_to_gprs src))-(rule 0 (lower (has_type (is_xmm_type _) (bitcast _ src @ (value_type $I128))))+(rule 0 (lower (has_type (is_xmm_type _) (bitcast _ _ src @ (value_type $I128))))(bitcast_gprs_to_xmm src));; Bitcast between types residing in GPR registers is a no-op.(rule 1 (lower (has_type (is_gpr_type _)- (bitcast _ x @ (value_type (is_gpr_type _)))))+ (bitcast _ _ x @ (value_type (is_gpr_type _)))))x);; Bitcast between types residing in XMM registers is a no-op.(rule 3 (lower (has_type (is_xmm_type _)- (bitcast _ x @ (value_type (is_xmm_type _)))))+ (bitcast _ _ x @ (value_type (is_xmm_type _)))))x);; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fcopysign a @ (value_type $F32) b)))+(rule (lower (has_type $F32 (fcopysign _ a @ (value_type $F32) b)))(let ((sign_bit Xmm (imm $F32 0x80000000))(a Xmm a) ;; force into reg so we don't sink a 128-bit load.(b Xmm b))@@ -4305,7 +4305,7 @@(x64_andnps sign_bit a)(x64_andps sign_bit b))))-(rule (lower (has_type $F64 (fcopysign a @ (value_type $F64) b)))+(rule (lower (has_type $F64 (fcopysign _ a @ (value_type $F64) b)))(let ((sign_bit Xmm (imm $F64 0x8000000000000000))(a Xmm a) ;; force into reg so we don't sink a 128-bit load.(b Xmm b))@@ -4369,27 +4369,27 @@;; Rules for `ceil` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (ceil a @ (value_type ty)))+(rule (lower (ceil _ a @ (value_type ty)))(x64_round ty a (RoundImm.RoundUp)));; Rules for `floor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (floor a @ (value_type ty)))+(rule (lower (floor _ a @ (value_type ty)))(x64_round ty a (RoundImm.RoundDown)));; Rules for `nearest` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (nearest a @ (value_type ty)))+(rule (lower (nearest _ a @ (value_type ty)))(x64_round ty a (RoundImm.RoundNearest)));; Rules for `trunc` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (trunc a @ (value_type ty)))+(rule (lower (trunc _ a @ (value_type ty)))(x64_round ty a (RoundImm.RoundZero)));; Rules for `stack_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (stack_addr stack_slot offset))+(rule (lower (stack_addr _ stack_slot offset))(stack_addr_impl stack_slot offset));; Rules for `udiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -4403,7 +4403,7 @@;; The inputs to the `div` instruction are different for 8-bit division so;; it needs a special case here since the instruction being crafted has a;; different shape.-(rule 2 (lower (udiv a @ (value_type $I8) b))+(rule 2 (lower (udiv _ a @ (value_type $I8) b))(x64_divb_m (extend_to_gpr a $I32 (ExtendKind.Zero))(put_in_gpr b)(TrapCode.INTEGER_DIVISION_BY_ZERO)))@@ -4411,19 +4411,19 @@;; 16-to-64-bit division is all done with a similar instruction and the only;; tricky requirement here is that when div traps are disallowed the divisor;; must not be zero.-(rule 1 (lower (udiv a @ (value_type (fits_in_64 ty)) b))+(rule 1 (lower (udiv _ a @ (value_type (fits_in_64 ty)) b))(value_regs_get(x64_div ty a (imm $I64 0) (put_in_gpr b) (TrapCode.INTEGER_DIVISION_BY_ZERO))0));; Rules for `sdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (sdiv a @ (value_type $I8) b))+(rule 2 (lower (sdiv _ a @ (value_type $I8) b))(x64_idivb_m (x64_cbtw_zo a)(nonzero_sdiv_divisor $I8 b)(TrapCode.INTEGER_OVERFLOW)))-(rule 1 (lower (sdiv a @ (value_type (fits_in_64 ty)) b))+(rule 1 (lower (sdiv _ a @ (value_type (fits_in_64 ty)) b))(let ((a Gpr a))(value_regs_get(x64_idiv ty a (repeat_sign_bit ty a) (nonzero_sdiv_divisor ty b) (TrapCode.INTEGER_OVERFLOW))@@ -4445,7 +4445,7 @@;; This is required to differentiate the divide-by-zero trap from the;; integer-overflow trap, the two trapping conditions of signed division.(decl nonzero_sdiv_divisor (Type Value) Reg)-(rule 1 (nonzero_sdiv_divisor ty (iconst imm))+(rule 1 (nonzero_sdiv_divisor ty (iconst _ imm))(if-let n (safe_divisor_from_imm64 ty imm))(imm ty n))(rule 0 (nonzero_sdiv_divisor ty val)@@ -4461,7 +4461,7 @@;; The remainder is in AH, so take the result of the division and right-shift;; by 8.-(rule 2 (lower (urem a @ (value_type $I8) b))+(rule 2 (lower (urem _ a @ (value_type $I8) b))(let ((result Gpr (x64_divb_m (extend_to_gpr a $I32 (ExtendKind.Zero))(put_in_gpr b) ;; see `udiv` for why not `gpr_mem`@@ -4469,7 +4469,7 @@)(x64_shrq_mi result 8)))-(rule 1 (lower (urem a @ (value_type (fits_in_64 ty)) b))+(rule 1 (lower (urem _ a @ (value_type (fits_in_64 ty)) b))(value_regs_get(x64_div ty a (imm $I64 0) (put_in_gpr b) (TrapCode.INTEGER_DIVISION_BY_ZERO))1))@@ -4481,7 +4481,7 @@;;;; Note that like `urem` for i8 types the result is in AH so to get the result;; it's right-shifted down.-(rule 3 (lower (srem a @ (value_type $I8) (iconst imm)))+(rule 3 (lower (srem _ a @ (value_type $I8) (iconst _ imm)))(if-let n (safe_divisor_from_imm64 $I8 imm))(let ((a Gpr (x64_cbtw_zo a))@@ -4490,7 +4490,7 @@(x64_shrq_mi result 8)));; Same as the above rule but for 16-to-64 bit types.-(rule 2 (lower (srem a @ (value_type ty) (iconst imm)))+(rule 2 (lower (srem _ a @ (value_type ty) (iconst _ imm)))(if-let n (safe_divisor_from_imm64 ty imm))(let ((a Gpr a)@@ -4500,10 +4500,10 @@(x64_idiv ty a (repeat_sign_bit ty a) (imm ty n) (TrapCode.INTEGER_DIVISION_BY_ZERO))1)))-(rule 1 (lower (srem a @ (value_type $I8) b))+(rule 1 (lower (srem _ a @ (value_type $I8) b))(x64_shrq_mi (x64_checked_srem_seq8 (x64_cbtw_zo a) b) 8))-(rule (lower (srem a @ (value_type ty) b))+(rule (lower (srem _ a @ (value_type ty) b))(let ((a Gpr a)(size OperandSize (raw_operand_size_of_type ty))@@ -4514,31 +4514,31 @@;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (umulhi a @ (value_type $I8) b))+(rule 0 (lower (umulhi _ a @ (value_type $I8) b))(x64_shrw_mi (x64_mul8 false a b) 8))-(rule 1 (lower (umulhi a @ (value_type (ty_int_ref_16_to_64 ty)) b))+(rule 1 (lower (umulhi _ a @ (value_type (ty_int_ref_16_to_64 ty)) b))(value_regs_get_gpr (x64_mul ty false a b) 1));; The BMI2 instruction set introduced `mulx` which defines two registers but;; if the two registers are the same then it only defines the upper bits. This;; helps slightly reduce register pressure by ensuring only one register here is;; clobbered.-(rule 2 (lower (umulhi a @ (value_type (ty_32_or_64 ty)) b))+(rule 2 (lower (umulhi _ a @ (value_type (ty_32_or_64 ty)) b))(if-let true (has_bmi2))(x64_mulx_hi ty a b));; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (smulhi a @ (value_type $I8) b))+(rule 0 (lower (smulhi _ a @ (value_type $I8) b))(x64_sarw_mi (x64_mul8 true a b) 8))-(rule 1 (lower (smulhi a @ (value_type (ty_int_ref_16_to_64 ty)) b))+(rule 1 (lower (smulhi _ a @ (value_type (ty_int_ref_16_to_64 ty)) b))(value_regs_get_gpr (x64_mul ty true a b) 1));; Rules for `get_pinned_reg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_pinned_reg))+(rule (lower (get_pinned_reg _))(read_pinned_gpr));; Rules for `set_pinned_reg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -4548,13 +4548,13 @@;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (vconst const)))+(rule (lower (has_type ty (vconst _ const)));; TODO use Inst::gen_constant() instead.(x64_xmm_load_const ty (const_to_vconst const)));; Special cases for known constant patterns to skip a 16-byte load.-(rule 1 (lower (has_type ty (vconst (u128_from_constant 0)))) (xmm_zero ty))-(rule 1 (lower (has_type ty (vconst (u128_from_constant -1)))) (vector_all_ones))+(rule 1 (lower (has_type ty (vconst _ (u128_from_constant 0)))) (xmm_zero ty))+(rule 1 (lower (has_type ty (vconst _ (u128_from_constant -1)))) (vector_all_ones));; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -4562,7 +4562,7 @@;; indicates which lane of the two operands is chosen for the output. A bit of;; 0 chooses the corresponding 16-it lane from `a` and a bit of 1 chooses the;; corresponding 16-bit lane from `b`.-(rule 14 (lower (shuffle a b (pblendw_imm n)))+(rule 14 (lower (shuffle _ a b (pblendw_imm n)))(if-let true (has_sse41))(x64_pblendw a b n))(decl pblendw_imm (u8) Immediate)@@ -4573,7 +4573,7 @@;; swapped in the instruction here. The `palignr` instruction uses the second;; operand as the low-order bytes and the first operand as high-order bytes,;; so put `a` second.-(rule 13 (lower (shuffle a b (palignr_imm_from_immediate n)))+(rule 13 (lower (shuffle _ a b (palignr_imm_from_immediate n)))(if-let true (has_ssse3))(x64_palignr b a n))(decl palignr_imm_from_immediate (u8) Immediate)@@ -4585,13 +4585,13 @@;; extractors here implemented in Rust and note that there's two cases for each;; instruction here to match when either the first or second shuffle operand is;; used.-(rule 12 (lower (shuffle x y (pshuflw_lhs_imm imm)))+(rule 12 (lower (shuffle _ x y (pshuflw_lhs_imm imm)))(x64_pshuflw x imm))-(rule 11 (lower (shuffle x y (pshuflw_rhs_imm imm)))+(rule 11 (lower (shuffle _ x y (pshuflw_rhs_imm imm)))(x64_pshuflw y imm))-(rule 10 (lower (shuffle x y (pshufhw_lhs_imm imm)))+(rule 10 (lower (shuffle _ x y (pshufhw_lhs_imm imm)))(x64_pshufhw x imm))-(rule 9 (lower (shuffle x y (pshufhw_rhs_imm imm)))+(rule 9 (lower (shuffle _ x y (pshufhw_rhs_imm imm)))(x64_pshufhw y imm))(decl pshuflw_lhs_imm (u8) Immediate)@@ -4608,9 +4608,9 @@;; selects 32-bit values from either `x` or `y`, but not both. This means;; there's one rule for selecting from `x` and another rule for selecting from;; `y`.-(rule 8 (lower (shuffle x y (pshufd_lhs_imm imm)))+(rule 8 (lower (shuffle _ x y (pshufd_lhs_imm imm)))(x64_pshufd x imm))-(rule 7 (lower (shuffle x y (pshufd_rhs_imm imm)))+(rule 7 (lower (shuffle _ x y (pshufd_rhs_imm imm)))(x64_pshufd y imm))(decl pshufd_lhs_imm (u8) Immediate)@@ -4619,27 +4619,27 @@(extern extractor pshufd_rhs_imm pshufd_rhs_imm);; Special case for i8-level interleaving of upper/low bytes.-(rule 6 (lower (shuffle a b (u128_from_immediate 0x1f0f_1e0e_1d0d_1c0c_1b0b_1a0a_1909_1808)))+(rule 6 (lower (shuffle _ a b (u128_from_immediate 0x1f0f_1e0e_1d0d_1c0c_1b0b_1a0a_1909_1808)))(x64_punpckhbw a b))-(rule 6 (lower (shuffle a b (u128_from_immediate 0x1707_1606_1505_1404_1303_1202_1101_1000)))+(rule 6 (lower (shuffle _ a b (u128_from_immediate 0x1707_1606_1505_1404_1303_1202_1101_1000)))(x64_punpcklbw a b));; Special case for i16-level interleaving of upper/low bytes.-(rule 6 (lower (shuffle a b (u128_from_immediate 0x1f1e_0f0e_1d1c_0d0c_1b1a_0b0a_1918_0908)))+(rule 6 (lower (shuffle _ a b (u128_from_immediate 0x1f1e_0f0e_1d1c_0d0c_1b1a_0b0a_1918_0908)))(x64_punpckhwd a b))-(rule 6 (lower (shuffle a b (u128_from_immediate 0x1716_0706_1514_0504_1312_0302_1110_0100)))+(rule 6 (lower (shuffle _ a b (u128_from_immediate 0x1716_0706_1514_0504_1312_0302_1110_0100)))(x64_punpcklwd a b));; Special case for i32-level interleaving of upper/low bytes.-(rule 6 (lower (shuffle a b (u128_from_immediate 0x1f1e1d1c_0f0e0d0c_1b1a1918_0b0a0908)))+(rule 6 (lower (shuffle _ a b (u128_from_immediate 0x1f1e1d1c_0f0e0d0c_1b1a1918_0b0a0908)))(x64_punpckhdq a b))-(rule 6 (lower (shuffle a b (u128_from_immediate 0x17161514_07060504_13121110_03020100)))+(rule 6 (lower (shuffle _ a b (u128_from_immediate 0x17161514_07060504_13121110_03020100)))(x64_punpckldq a b));; Special case for i64-level interleaving of upper/low bytes.-(rule 6 (lower (shuffle a b (u128_from_immediate 0x1f1e1d1c1b1a1918_0f0e0d0c0b0a0908)))+(rule 6 (lower (shuffle _ a b (u128_from_immediate 0x1f1e1d1c1b1a1918_0f0e0d0c0b0a0908)))(x64_punpckhqdq a b))-(rule 6 (lower (shuffle a b (u128_from_immediate 0x1716151413121110_0706050403020100)))+(rule 6 (lower (shuffle _ a b (u128_from_immediate 0x1716151413121110_0706050403020100)))(x64_punpcklqdq a b));; If the vector shift mask is all 0s then that means the first byte of the@@ -4647,7 +4647,7 @@;; all-zeros constant from a rip-relative location but it should be slightly;; more efficient to execute the `pshufb` here-and-now with an xor'd-to-be-zero;; register.-(rule 6 (lower (shuffle a _ (u128_from_immediate 0)))+(rule 6 (lower (shuffle _ a _ (u128_from_immediate 0)))(if-let true (has_ssse3))(x64_pshufb a (xmm_zero $I8X16)))@@ -4661,9 +4661,9 @@;; hypothesized that the dedicated instructions are better than `shufps`.;; Someone with more knowledge about x86 timings should perhaps reorder the;; rules here eventually though.-(rule 5 (lower (shuffle x y (shufps_imm imm)))+(rule 5 (lower (shuffle _ x y (shufps_imm imm)))(x64_shufps x y imm))-(rule 4 (lower (shuffle x y (shufps_rev_imm imm)))+(rule 4 (lower (shuffle _ x y (shufps_rev_imm imm)))(x64_shufps y x imm))(decl shufps_imm(u8) Immediate)@@ -4676,21 +4676,21 @@;; register. We statically build `constructed_mask` to zero out any unknown lane;; indices (may not be completely necessary: verification could fail incorrect;; mask values) and fix the indexes to all point to the `dst` vector.-(rule 3 (lower (shuffle a a (vec_mask_from_immediate mask)))+(rule 3 (lower (shuffle _ a a (vec_mask_from_immediate mask)))(if-let true (has_ssse3))(x64_pshufb a (shuffle_0_31_mask mask)));; For the case where the shuffle mask contains out-of-bounds values (values;; greater than 31) we must mask off those resulting values in the result of;; `vpermi2b`.-(rule 2 (lower (shuffle a b (vec_mask_from_immediate (perm_from_mask_with_zeros mask zeros))))+(rule 2 (lower (shuffle _ a b (vec_mask_from_immediate (perm_from_mask_with_zeros mask zeros))))(if-let true (has_avx512vl))(if-let true (has_avx512vbmi))(x64_andps (x64_vpermi2b (x64_xmm_load_const $I8X16 mask) a b) zeros));; However, if the shuffle mask contains no out-of-bounds values, we can use;; `vpermi2b` without any masking.-(rule 1 (lower (shuffle a b (vec_mask_from_immediate mask)))+(rule 1 (lower (shuffle _ a b (vec_mask_from_immediate mask)))(if-let true (has_avx512vl))(if-let true (has_avx512vbmi))(x64_vpermi2b (x64_xmm_load_const $I8X16 (perm_from_mask mask)) a b))@@ -4698,7 +4698,7 @@;; If `lhs` and `rhs` are different, we must shuffle each separately and then OR;; them together. This is necessary due to PSHUFB semantics. As in the case;; above, we build the `constructed_mask` for each case statically.-(rule (lower (shuffle a b (vec_mask_from_immediate mask)))+(rule (lower (shuffle _ a b (vec_mask_from_immediate mask)))(x64_por(lower_pshufb a (shuffle_0_15_mask mask))(lower_pshufb b (shuffle_16_31_mask mask))))@@ -4711,13 +4711,13 @@;; https://github.com/WebAssembly/simd/issues/93. The CLIF semantics match the;; Wasm SIMD semantics for this instruction. The instruction format maps to;; variables like: %dst = swizzle %src, %mask-(rule (lower (swizzle src mask))+(rule (lower (swizzle _ src mask))(let ((mask Xmm (x64_paddusb mask (emit_u128_le_const 0x70707070707070707070707070707070))))(lower_pshufb src mask)));; Rules for `x86_pshufb` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (x86_pshufb src mask))+(rule (lower (x86_pshufb _ src mask))(if-let true (has_ssse3))(x64_pshufb src mask))@@ -4742,56 +4742,56 @@;; Remove the extractlane instruction, leaving the float where it is. The upper;; bits will remain unchanged; for correctness, this relies on Cranelift type;; checking to avoid using those bits.-(rule 3 (lower (has_type (ty_scalar_float _) (extractlane val 0)))+(rule 3 (lower (has_type (ty_scalar_float _) (extractlane _ val 0)))val);; `f32x4.extract_lane N` where `N != 0`-(rule 1 (lower (extractlane val @ (value_type $F32X4) (u8_from_uimm8 lane)))+(rule 1 (lower (extractlane _ val @ (value_type $F32X4) (u8_from_uimm8 lane)))(x64_pshufd val lane));; `f64x2.extract_lane N` where `N != 0` (aka N == 1)-(rule (lower (extractlane val @ (value_type $F64X2) 1))+(rule (lower (extractlane _ val @ (value_type $F64X2) 1))(x64_pshufd val 0b11_10_11_10));; `i8x16.extract_lane N`;;;; Note that without SSE4.1 a 16-bit lane extraction is performed and then;; the result is updated if the desired index is either odd or even.-(rule 2 (lower (extractlane val @ (value_type ty @ $I8X16) (u8_from_uimm8 lane)))+(rule 2 (lower (extractlane _ val @ (value_type ty @ $I8X16) (u8_from_uimm8 lane)))(if-let true (has_sse41))(x64_pextrb val lane));; extracting an odd lane has an extra shift-right-(rule 1 (lower (extractlane val @ (value_type ty @ $I8X16) (u8_from_uimm8 lane)))+(rule 1 (lower (extractlane _ val @ (value_type ty @ $I8X16) (u8_from_uimm8 lane)))(if-let 1 (u8_and lane 1))(x64_shrw_mi (x64_pextrw val (u8_wrapping_shr lane 1)) 8));; Extracting an even lane already has the desired lane in the lower bits. Note;; that having arbitrary upper bits in the returned register should be ok since;; all operators on the resulting `i8` type should work correctly regardless of;; the bits in the rest of the register.-(rule (lower (extractlane val @ (value_type ty @ $I8X16) (u8_from_uimm8 lane)))+(rule (lower (extractlane _ val @ (value_type ty @ $I8X16) (u8_from_uimm8 lane)))(if-let 0 (u8_and lane 1))(x64_pextrw val (u8_wrapping_shr lane 1)));; `i16x8.extract_lane N`-(rule (lower (extractlane val @ (value_type ty @ $I16X8) (u8_from_uimm8 lane)))+(rule (lower (extractlane _ val @ (value_type ty @ $I16X8) (u8_from_uimm8 lane)))(x64_pextrw val lane));; `i32x4.extract_lane N`-(rule 2 (lower (extractlane val @ (value_type ty @ $I32X4) (u8_from_uimm8 lane)))+(rule 2 (lower (extractlane _ val @ (value_type ty @ $I32X4) (u8_from_uimm8 lane)))(if-let true (has_sse41))(x64_pextrd val lane))-(rule 1 (lower (extractlane val @ (value_type $I32X4) 0))+(rule 1 (lower (extractlane _ val @ (value_type $I32X4) 0))(x64_movd_to_gpr val))-(rule (lower (extractlane val @ (value_type $I32X4) (u8_from_uimm8 n)))+(rule (lower (extractlane _ val @ (value_type $I32X4) (u8_from_uimm8 n)))(x64_movd_to_gpr (x64_pshufd val n)));; `i64x2.extract_lane N`-(rule 1 (lower (extractlane val @ (value_type $I64X2) (u8_from_uimm8 lane)))+(rule 1 (lower (extractlane _ val @ (value_type $I64X2) (u8_from_uimm8 lane)))(if-let true (has_sse41))(x64_pextrq val lane))-(rule (lower (extractlane val @ (value_type $I64X2) 0))+(rule (lower (extractlane _ val @ (value_type $I64X2) 0))(x64_movq_to_gpr val))-(rule (lower (extractlane val @ (value_type $I64X2) 1))+(rule (lower (extractlane _ val @ (value_type $I64X2) 1))(x64_movq_to_gpr (x64_pshufd val 0b00_00_11_10)));; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -4800,21 +4800,21 @@;; xmm-register-to-xmm-register semantics are used to modify only the low bits;; of a guaranteed-zero register. This ensures that the upper bits are cleared;; as the upper bits of `src` in a register are otherwise undefined.-(rule 1 (lower (scalar_to_vector src @ (value_type $F32)))+(rule 1 (lower (scalar_to_vector _ src @ (value_type $F32)))(x64_movss_regmove (xmm_zero $F32X4) src))-(rule 1 (lower (scalar_to_vector src @ (value_type $F64)))+(rule 1 (lower (scalar_to_vector _ src @ (value_type $F64)))(x64_movsd_regmove (xmm_zero $F64X2) src));; Case 2: when moving a scalar value of any other type, use MOVD to zero;; the upper lanes.-(rule (lower (scalar_to_vector src @ (value_type ty)))+(rule (lower (scalar_to_vector _ src @ (value_type ty)))(bitcast_gpr_to_xmm (ty_bits ty) src));; Case 3: when presented with `load + scalar_to_vector`, coalesce into a single;; MOVSS/MOVSD instruction.-(rule 2 (lower (scalar_to_vector (and (sinkable_load src) (value_type (ty_32 _)))))+(rule 2 (lower (scalar_to_vector _ (and (sinkable_load src) (value_type (ty_32 _)))))(x64_movss_load src))-(rule 3 (lower (scalar_to_vector (and (sinkable_load src) (value_type (ty_64 _)))))+(rule 3 (lower (scalar_to_vector _ (and (sinkable_load src) (value_type (ty_64 _)))))(x64_movsd_load src));; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -4830,20 +4830,20 @@;; i8x16 splats: use `vpbroadcastb` on AVX2 and otherwise `pshufb` broadcasts;; with a mask of zero which is calculated with an xor-against-itself register.-(rule 0 (lower (has_type $I8X16 (splat src)))+(rule 0 (lower (has_type $I8X16 (splat _ src)))(let ((src Xmm (x64_movd_to_xmm src)))(x64_pshufd (x64_pshuflw (x64_punpcklbw src src) 0) 0)))-(rule 1 (lower (has_type $I8X16 (splat src)))+(rule 1 (lower (has_type $I8X16 (splat _ src)))(if-let true (has_ssse3))(x64_pshufb (bitcast_gpr_to_xmm 32 src) (xmm_zero $I8X16)))-(rule 2 (lower (has_type $I8X16 (splat src)))+(rule 2 (lower (has_type $I8X16 (splat _ src)))(if-let true (use_avx2))(x64_vpbroadcastb (bitcast_gpr_to_xmm 32 src)))-(rule 3 (lower (has_type $I8X16 (splat (sinkable_load_exact addr))))+(rule 3 (lower (has_type $I8X16 (splat _ (sinkable_load_exact addr))))(if-let true (has_sse41))(if-let true (has_ssse3))(x64_pshufb (x64_pinsrb (xmm_uninit_value) addr 0) (xmm_zero $I8X16)))-(rule 4 (lower (has_type $I8X16 (splat (sinkable_load_exact addr))))+(rule 4 (lower (has_type $I8X16 (splat _ (sinkable_load_exact addr))))(if-let true (use_avx2))(x64_vpbroadcastb addr))@@ -4852,14 +4852,14 @@;; to the low four lanes, and `pshufd` broadcasts the low 32-bit lane (which;; at that point is two of the 16-bit values we want to broadcast) to all the;; lanes.-(rule 0 (lower (has_type $I16X8 (splat src)))+(rule 0 (lower (has_type $I16X8 (splat _ src)))(x64_pshufd (x64_pshuflw (bitcast_gpr_to_xmm 32 src) 0) 0))-(rule 1 (lower (has_type $I16X8 (splat src)))+(rule 1 (lower (has_type $I16X8 (splat _ src)))(if-let true (use_avx2))(x64_vpbroadcastw (bitcast_gpr_to_xmm 32 src)))-(rule 2 (lower (has_type $I16X8 (splat (sinkable_load_exact addr))))+(rule 2 (lower (has_type $I16X8 (splat _ (sinkable_load_exact addr))))(x64_pshufd (x64_pshuflw (x64_pinsrw (xmm_uninit_value) addr 0) 0) 0))-(rule 3 (lower (has_type $I16X8 (splat (sinkable_load_exact addr))))+(rule 3 (lower (has_type $I16X8 (splat _ (sinkable_load_exact addr))))(if-let true (use_avx2))(x64_vpbroadcastw addr))@@ -4867,19 +4867,19 @@;; used to broadcast the low lane to all other lanes.;;;; Note that sinkable-load cases come later-(rule 0 (lower (has_type $I32X4 (splat src)))+(rule 0 (lower (has_type $I32X4 (splat _ src)))(x64_pshufd (bitcast_gpr_to_xmm 32 src) 0))-(rule 1 (lower (has_type $I32X4 (splat src)))+(rule 1 (lower (has_type $I32X4 (splat _ src)))(if-let true (use_avx2))(x64_vpbroadcastd (bitcast_gpr_to_xmm 32 src)));; f32x4.splat - the source is already in an xmm register so `shufps` is all;; that's necessary to complete the splat. This is specialized to `vbroadcastss`;; on AVX2 to leverage that specific instruction for this operation.-(rule 0 (lower (has_type $F32X4 (splat src)))+(rule 0 (lower (has_type $F32X4 (splat _ src)))(let ((tmp Xmm src))(x64_shufps src src 0)))-(rule 1 (lower (has_type $F32X4 (splat src)))+(rule 1 (lower (has_type $F32X4 (splat _ src)))(if-let true (use_avx2))(x64_vbroadcastss src))@@ -4892,27 +4892,27 @@;; the register-based encoding is only available with AVX2. With the;; `sinkable_load` extractor this should be guaranteed to use the memory-based;; encoding hence the `has_avx` test.-(rule 5 (lower (has_type (multi_lane 32 4) (splat (sinkable_load addr))))+(rule 5 (lower (has_type (multi_lane 32 4) (splat _ (sinkable_load addr))))(let ((tmp Xmm (x64_movss_load addr)))(x64_shufps tmp tmp 0)))-(rule 6 (lower (has_type (multi_lane 32 4) (splat (sinkable_load addr))))+(rule 6 (lower (has_type (multi_lane 32 4) (splat _ (sinkable_load addr))))(if-let true (has_avx))(x64_vbroadcastss addr));; t64x2.splat - use `pshufd` to broadcast the lower 64-bit lane to the upper;; lane. A minor specialization for sinkable loads to avoid going through a gpr;; for i64 splats is used as well when `movddup` is available.-(rule 0 (lower (has_type $I64X2 (splat src)))+(rule 0 (lower (has_type $I64X2 (splat _ src)))(x64_pshufd (bitcast_gpr_to_xmm 64 src) 0b01_00_01_00))-(rule 0 (lower (has_type $F64X2 (splat src)))+(rule 0 (lower (has_type $F64X2 (splat _ src)))(x64_pshufd src 0b01_00_01_00))-(rule 6 (lower (has_type (multi_lane 64 2) (splat (sinkable_load addr))))+(rule 6 (lower (has_type (multi_lane 64 2) (splat _ (sinkable_load addr))))(if-let true (has_sse3))(x64_movddup addr));; Rules for `vany_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (vany_true val)) (lower_cond_bool (is_vany_true val)))+(rule (lower (vany_true _ val)) (lower_cond_bool (is_vany_true val)));; Any nonzero byte in `val` means that any lane is true. Compare `val` with a;; zeroed register and extract the high bits to a gpr mask. If the mask is@@ -4932,7 +4932,7 @@;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (vall_true val)) (lower_cond_bool (is_vall_true val)))+(rule (lower (vall_true _ val)) (lower_cond_bool (is_vall_true val)))(decl is_vall_true (Value) CondResult)(rule 1 (is_vall_true val @ (value_type ty))@@ -4961,13 +4961,13 @@;; for setting/clearing REX.W) as we need at most 16 bits of output for;; `vhigh_bits`.-(rule (lower (vhigh_bits val @ (value_type (multi_lane 8 16))))+(rule (lower (vhigh_bits _ val @ (value_type (multi_lane 8 16))))(x64_pmovmskb val))-(rule (lower (vhigh_bits val @ (value_type (multi_lane 32 4))))+(rule (lower (vhigh_bits _ val @ (value_type (multi_lane 32 4))))(x64_movmskps val))-(rule (lower (vhigh_bits val @ (value_type (multi_lane 64 2))))+(rule (lower (vhigh_bits _ val @ (value_type (multi_lane 64 2))))(x64_movmskpd val));; There is no x86 instruction for extracting the high bit of 16-bit lanes so@@ -4976,7 +4976,7 @@;; PACKSSWB([x1, x2, ...], [x1, x2, ...]) = [x1', x2', ..., x1', x2', ...];; - use PMOVMSKB to gather the high bits; now we have duplicates, though;; - shift away the bottom 8 high bits to remove the duplicates.-(rule (lower (vhigh_bits val @ (value_type (multi_lane 16 8))))+(rule (lower (vhigh_bits _ val @ (value_type (multi_lane 16 8))))(let ((src Xmm val)(tmp Xmm (x64_packsswb src src))(tmp Gpr (x64_pmovmskb tmp)))@@ -4984,12 +4984,12 @@;; Rules for `iconcat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (iconcat lo @ (value_type $I64) hi))+(rule (lower (iconcat _ lo @ (value_type $I64) hi))(value_regs lo hi));; Rules for `isplit` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (isplit val @ (value_type $I128)))+(rule (lower (isplit _ val @ (value_type $I128)))(let ((regs ValueRegs val)(lo Reg (value_regs_get regs 0))(hi Reg (value_regs_get regs 1)))@@ -4997,18 +4997,18 @@;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value (symbol_value_data name _ _))))+(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value _ (symbol_value_data name _ _))))(elf_tls_get_addr name))-(rule (lower (has_type (tls_model (TlsModel.Macho)) (tls_value (symbol_value_data name _ _))))+(rule (lower (has_type (tls_model (TlsModel.Macho)) (tls_value _ (symbol_value_data name _ _))))(macho_tls_get_addr name))-(rule (lower (has_type (tls_model (TlsModel.Coff)) (tls_value (symbol_value_data name _ _))))+(rule (lower (has_type (tls_model (TlsModel.Coff)) (tls_value _ (symbol_value_data name _ _))))(coff_tls_get_addr name));; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (sqmul_round_sat qx @ (value_type $I16X8) qy))+(rule 1 (lower (sqmul_round_sat _ qx @ (value_type $I16X8) qy))(if-let true (has_ssse3))(let ((src1 Xmm qx)(src2 Xmm qy)@@ -5024,7 +5024,7 @@;;;; so perform all those operations here manually with a lack of the native;; instruction.-(rule (lower (sqmul_round_sat qx @ (value_type $I16X8) qy))+(rule (lower (sqmul_round_sat _ qx @ (value_type $I16X8) qy))(let ((qx Xmm qx)(qy Xmm qy)@@ -5050,7 +5050,7 @@;; Rules for `x86_pmulhrsw` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (x86_pmulhrsw qx @ (value_type $I16X8) qy))+(rule (lower (x86_pmulhrsw _ qx @ (value_type $I16X8) qy))(if-let true (has_ssse3))(x64_pmulhrsw qx qy))@@ -5068,8 +5068,8 @@;; ROUNDPD xmm_y, xmm_y, 0x0B;; ADDPD xmm_y, [wasm_f64x2_splat(0x1.0p+52)];; SHUFPS xmm_y, xmm_xmp, 0x88-(rule (lower (uunarrow (fcvt_to_uint_sat src @ (value_type $F64X2))- (vconst (u128_from_constant 0))))+(rule (lower (uunarrow _ (fcvt_to_uint_sat _ src @ (value_type $F64X2))+ (vconst _ (u128_from_constant 0))))(let ((src Xmm src);; MOVAPD xmm_y, xmm_x@@ -5096,7 +5096,7 @@;; Rules for `get_exception_handler_address` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (get_exception_handler_address (u64_from_imm64 idx) block))+(rule (lower (get_exception_handler_address _ (u64_from_imm64 idx) block))(let ((succ_label MachLabel (block_exn_successor_label block idx)))(x64_label_address succ_label)))diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rsindex f73ede5b25..53660767c5 100644--- a/cranelift/codegen/src/machinst/isle.rs+++ b/cranelift/codegen/src/machinst/isle.rs@@ -203,8 +203,13 @@ macro_rules! isle_lower_prelude_methods {}#[inline]- fn inst_data_value(&mut self, inst: Inst) -> InstructionData {- self.lower_ctx.dfg().insts[inst]+ fn inst_data_value(&mut self, inst: Inst) -> (Type, InstructionData) {+ let ty = match self.first_result(inst) {+ Some(v) => self.value_type(v),+ None => types::INVALID,+ };+ let data = self.lower_ctx.dfg().insts[inst];+ (ty, data)}#[inline]diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isleindex f1f492bddb..deb916f13e 100644--- a/cranelift/codegen/src/prelude_lower.isle+++ b/cranelift/codegen/src/prelude_lower.isle@@ -272,7 +272,7 @@(extern extractor first_result first_result);; Extract the `InstructionData` for an `Inst`.-(decl inst_data_value (InstructionData) Inst)+(decl inst_data_value (Type InstructionData) Inst)(extern extractor infallible inst_data_value inst_data_value);; Extract the type of the instruction's first result.@@ -306,7 +306,7 @@(spec (u64_from_iconst arg) (provide (= arg (zero_ext 64 result))))(decl u64_from_iconst (u64) Value)(extractor (u64_from_iconst x)- (def_inst (iconst (u64_from_imm64 x))))+ (def_inst (iconst _ (u64_from_imm64 x))))(decl i8_from_iconst (i8) Value)(extractor (i8_from_iconst x)@@ -358,13 +358,13 @@(extractor (ne x y) (icmp (IntCC.NotEqual) x y))(decl ult (Value Value) Value)-(extractor (ult x y) (icmp (IntCC.UnsignedLessThan) x y))+(extractor (ult x y) (icmp _ (IntCC.UnsignedLessThan) x y))(decl ule (Value Value) Value)(extractor (ule x y) (icmp (IntCC.UnsignedLessThanOrEqual) x y))(decl ugt (Value Value) Value)-(extractor (ugt x y) (icmp (IntCC.UnsignedGreaterThan) x y))+(extractor (ugt x y) (icmp _ (IntCC.UnsignedGreaterThan) x y))(decl uge (Value Value) Value)(extractor (uge x y) (icmp (IntCC.UnsignedGreaterThanOrEqual) x y))