A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Remove most usage of `has_type` in ISLE backends (#13655)
Alex Crichton439e52e67f0e
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex 04f91685d8..a451bde035 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -1989,7 +1989,7 @@)(decl pure partial imm12_from_negated_value (Value) Imm12)(rule imm12_from_negated_value- (imm12_from_negated_value (has_type ty (iconst _ n)))+ (imm12_from_negated_value (iconst ty n))(if-let (imm12_from_u64 imm) (i64_cast_unsigned (i64_checked_neg (i64_sextend_imm64 ty n))))imm)diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex a81f40bf46..e7e9aef38e 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -18,7 +18,7 @@;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule iconst (lower (has_type ty (iconst _ (u64_from_imm64 n))))+(rule iconst (lower (iconst ty (u64_from_imm64 n)))(imm ty (ImmExtend.Zero) n));;;; Rules for `f16const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -38,7 +38,7 @@;;;; Rules for `f128const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F128 (f128const _ (u128_from_constant n))))+(rule (lower (f128const $F128 (u128_from_constant n)))(constant_f128 n));;;; Rules for `nop` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -51,64 +51,62 @@;; `i64` and smaller;; Base case, simply adding things in registers.-(rule iadd_base_case -1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x y)))+(rule iadd_base_case -1 (lower (iadd (ty_int_ref_scalar_64 ty) x y))(add ty x y));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule iadd_imm12_right 4 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (imm12_from_value y))))+(rule iadd_imm12_right 4 (lower (iadd (ty_int_ref_scalar_64 ty) x (imm12_from_value y)))(add_imm ty x y))-(rule iadd_imm12_left 5 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (imm12_from_value x) y)))+(rule iadd_imm12_left 5 (lower (iadd (ty_int_ref_scalar_64 ty) (imm12_from_value x) y))(add_imm ty y x));; Same as the previous special cases, except we can switch the addition to a;; subtraction if the negated immediate fits in 12 bits.-(rule iadd_imm12_neg_right 2 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x y)))+(rule iadd_imm12_neg_right 2 (lower (iadd (ty_int_ref_scalar_64 ty) x y))(if-let imm12_neg (imm12_from_negated_value y))(sub_imm ty x imm12_neg))-(rule iadd_imm12_neg_left 3 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x y)))+(rule iadd_imm12_neg_left 3 (lower (iadd (ty_int_ref_scalar_64 ty) x y))(if-let imm12_neg (imm12_from_negated_value x))(sub_imm ty y imm12_neg));; Special cases for when we're adding an extended register where the extending;; operation can get folded into the add itself.-(rule iadd_extend_right 0 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (extended_value_from_value y))))+(rule iadd_extend_right 0 (lower (iadd (ty_int_ref_scalar_64 ty) x (extended_value_from_value y)))(add_extend ty x y))-(rule iadd_extend_left 1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (extended_value_from_value x) y)))+(rule iadd_extend_left 1 (lower (iadd (ty_int_ref_scalar_64 ty) (extended_value_from_value x) y))(add_extend ty y x));; Special cases for when we're adding the shift of a different;; register by a constant amount and the shift can get folded into the add.-(rule iadd_ishl_right 7 (lower (has_type (ty_int_ref_scalar_64 ty)- (iadd _ x (ishl _ y (iconst _ k)))))+(rule iadd_ishl_right 7 (lower (iadd (ty_int_ref_scalar_64 ty) x (ishl _ y (iconst _ k))))(if-let amt (lshl_from_imm64 ty k))(add_shift ty x y amt))-(rule iadd_ishl_left 6 (lower (has_type (ty_int_ref_scalar_64 ty)- (iadd _ (ishl _ x (iconst _ k)) y)))+(rule iadd_ishl_left 6 (lower (iadd (ty_int_ref_scalar_64 ty) (ishl _ x (iconst _ k)) y))(if-let amt (lshl_from_imm64 ty k))(add_shift ty y x amt));; Fold an `iadd` and `imul` combination into a `madd` instruction.-(rule iadd_imul_right 7 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (imul _ y z))))+(rule iadd_imul_right 7 (lower (iadd (ty_int_ref_scalar_64 ty) x (imul _ y z)))(madd ty y z x))-(rule iadd_imul_left 6 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (imul _ x y) z)))+(rule iadd_imul_left 6 (lower (iadd (ty_int_ref_scalar_64 ty) (imul _ x y) z))(madd ty x y z));; Fold an `isub` and `imul` combination into a `msub` instruction.-(rule isub_imul (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x (imul _ y z))))+(rule isub_imul (lower (isub (ty_int_ref_scalar_64 ty) x (imul _ y z)))(msub ty y z x));; vectors-(rule -2 (lower (has_type ty @ (multi_lane _ _) (iadd _ x y)))+(rule -2 (lower (iadd ty @ (multi_lane _ _) x y))(add_vec x y (vector_size ty)));; `i128`-(rule -3 (lower (has_type $I128 (iadd _ x y)))+(rule -3 (lower (iadd $I128 x y))(let;; Get the high/low registers for `x`.((x_regs ValueRegs x)@@ -239,13 +237,13 @@(rule 1 (lower (shuffle _ a b (u128_from_immediate 0x0b0a09080f0e0d0c_0302010007060504)))(rev64 a (VectorSize.Size32x4)))-(rule (lower (has_type ty (shuffle _ rn rn2 (u128_from_immediate mask))))+(rule (lower (shuffle ty rn rn2 (u128_from_immediate mask)))(let ((mask_reg Reg (constant_f128 mask)))(vec_tbl2 rn rn2 mask_reg ty)));;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type vec_i128_ty (swizzle _ rn rm)))+(rule (lower (swizzle vec_i128_ty rn rm))(vec_tbl rn rm));;;; Rules for `isplit` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -260,13 +258,13 @@;; Special-case the lowering of an `isplit` of a 128-bit multiply where the;; lower bits of the result are discarded and the operands are sign or zero;; extended. This maps directly to `umulh` and `smulh`.-(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (uextend _ x) (uextend _ y)))))+(rule 1 (lower i @ (isplit _ (imul $I128 (uextend _ x) (uextend _ y))))(if-let (first_result lo) i)(if-let true (value_is_unused lo))(output_pair (invalid_reg)(umulh $I64 (put_in_reg_zext64 x) (put_in_reg_zext64 y))))-(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (sextend _ x) (sextend _ y)))))+(rule 1 (lower i @ (isplit _ (imul $I128 (sextend _ x) (sextend _ y))))(if-let (first_result lo) i)(if-let true (value_is_unused lo))(output_pair (invalid_reg)@@ -274,15 +272,15 @@;;;; Rules for `iconcat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I128 (iconcat _ lo hi)))+(rule (lower (iconcat $I128 lo hi))(output (value_regs lo hi)));;;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32X4 (scalar_to_vector _ x)))+(rule (lower (scalar_to_vector $F32X4 x))(fpu_extend x (ScalarSize.Size32)))-(rule (lower (has_type $F64X2 (scalar_to_vector _ x)))+(rule (lower (scalar_to_vector $F64X2 x))(fpu_extend x (ScalarSize.Size64)))(rule -1 (lower (scalar_to_vector _ x @ (value_type $I64)))@@ -341,22 +339,22 @@;;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; special case for the `i16x8.extadd_pairwise_i8x16_s` wasm instruction-(rule (lower (has_type $I16X8 (iadd_pairwise _ (swiden_low _ x) (swiden_high _ x))))+(rule (lower (iadd_pairwise $I16X8 (swiden_low _ x) (swiden_high _ x)))(saddlp8 x));; special case for the `i32x4.extadd_pairwise_i16x8_s` wasm instruction-(rule (lower (has_type $I32X4 (iadd_pairwise _ (swiden_low _ x) (swiden_high _ x))))+(rule (lower (iadd_pairwise $I32X4 (swiden_low _ x) (swiden_high _ x)))(saddlp16 x));; special case for the `i16x8.extadd_pairwise_i8x16_u` wasm instruction-(rule (lower (has_type $I16X8 (iadd_pairwise _ (uwiden_low _ x) (uwiden_high _ x))))+(rule (lower (iadd_pairwise $I16X8 (uwiden_low _ x) (uwiden_high _ x)))(uaddlp8 x));; special case for the `i32x4.extadd_pairwise_i16x8_u` wasm instruction-(rule (lower (has_type $I32X4 (iadd_pairwise _ (uwiden_low _ x) (uwiden_high _ x))))+(rule (lower (iadd_pairwise $I32X4 (uwiden_low _ x) (uwiden_high _ x)))(uaddlp16 x))-(rule -1 (lower (has_type ty (iadd_pairwise _ x y)))+(rule -1 (lower (iadd_pairwise ty x y))(addp x y (vector_size ty)));; With FEAT_DotProd, fold the `swiden`/`imul`/`iadd_pairwise`/`iadd` tree that@@ -378,13 +376,13 @@;;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (iabs _ x)))+(rule -1 (lower (iabs ty @ (multi_lane _ _) x))(vec_abs x (vector_size ty)))-(rule iabs_64 2 (lower (has_type $I64 (iabs _ x)))+(rule iabs_64 2 (lower (iabs $I64 x))(abs (OperandSize.Size64) x))-(rule iabs_8_16_32 1 (lower (has_type (fits_in_32 ty) (iabs _ x)))+(rule iabs_8_16_32 1 (lower (iabs (fits_in_32 ty) x))(abs (OperandSize.Size32) (put_in_reg_sext32 x))); `rustc` implementation.@@ -394,7 +392,7 @@; `x - 0`.; - if `x` is negative, the xor'd bits = ~x and the mask = -1, so we end up with; `~x - (-1) = ~x + 1`, which is exactly `abs(x)`.-(rule (lower (has_type $I128 (iabs _ x)))+(rule (lower (iabs $I128 x))(let ((x_regs ValueRegs x)(x_lo Reg (value_regs_get x_regs 0))(x_hi Reg (value_regs_get x_regs 1))@@ -407,7 +405,7 @@;;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I64X2 (avg_round _ x y)))+(rule (lower (avg_round $I64X2 x y))(let ((one Reg (splat_const 1 (VectorSize.Size64x2)))(c Reg (orr_vec x y (VectorSize.Size64x2)))(c Reg (and_vec c one (VectorSize.Size64x2)))@@ -416,144 +414,144 @@(sum Reg (add_vec x y (VectorSize.Size64x2))))(add_vec c sum (VectorSize.Size64x2))))-(rule -1 (lower (has_type (lane_fits_in_32 ty) (avg_round _ x y)))+(rule -1 (lower (avg_round (lane_fits_in_32 ty) x y))(vec_rrr (VecALUOp.Urhadd) x y (vector_size ty)));;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty @ (multi_lane _ _) (sqmul_round_sat _ x y)))+(rule (lower (sqmul_round_sat ty @ (multi_lane _ _) x y))(vec_rrr (VecALUOp.Sqrdmulh) x y (vector_size ty)));;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fadd _ rn rm)))+(rule -1 (lower (fadd ty @ (multi_lane _ _) rn rm))(vec_rrr (VecALUOp.Fadd) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fadd _ rn rm)))+(rule (lower (fadd (ty_scalar_float ty) rn rm))(fpu_rrr (FPUOp2.Add) rn rm (scalar_size ty)));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fsub _ rn rm)))+(rule -1 (lower (fsub ty @ (multi_lane _ _) rn rm))(vec_rrr (VecALUOp.Fsub) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fsub _ rn rm)))+(rule (lower (fsub (ty_scalar_float ty) rn rm))(fpu_rrr (FPUOp2.Sub) rn rm (scalar_size ty)));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmul _ rn rm)))+(rule -1 (lower (fmul ty @ (multi_lane _ _) rn rm))(vec_rrr (VecALUOp.Fmul) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fmul _ rn rm)))+(rule (lower (fmul (ty_scalar_float ty) rn rm))(fpu_rrr (FPUOp2.Mul) rn rm (scalar_size ty)));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fdiv _ rn rm)))+(rule -1 (lower (fdiv ty @ (multi_lane _ _) rn rm))(vec_rrr (VecALUOp.Fdiv) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fdiv _ rn rm)))+(rule (lower (fdiv (ty_scalar_float ty) rn rm))(fpu_rrr (FPUOp2.Div) rn rm (scalar_size ty)));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmin _ rn rm)))+(rule -1 (lower (fmin ty @ (multi_lane _ _) rn rm))(vec_rrr (VecALUOp.Fmin) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fmin _ rn rm)))+(rule (lower (fmin (ty_scalar_float ty) rn rm))(fpu_rrr (FPUOp2.Min) rn rm (scalar_size ty)));;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fmax _ rn rm)))+(rule -1 (lower (fmax ty @ (multi_lane _ _) rn rm))(vec_rrr (VecALUOp.Fmax) rn rm (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fmax _ rn rm)))+(rule (lower (fmax (ty_scalar_float ty) rn rm))(fpu_rrr (FPUOp2.Max) rn rm (scalar_size ty)));;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (sqrt _ x)))+(rule -1 (lower (sqrt ty @ (multi_lane _ _) x))(vec_misc (VecMisc2.Fsqrt) x (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (sqrt _ x)))+(rule (lower (sqrt (ty_scalar_float ty) x))(fpu_rr (FPUOp1.Sqrt) x (scalar_size ty)));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fneg _ x)))+(rule -1 (lower (fneg ty @ (multi_lane _ _) x))(vec_misc (VecMisc2.Fneg) x (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fneg _ x)))+(rule (lower (fneg (ty_scalar_float ty) x))(fpu_rr (FPUOp1.Neg) x (scalar_size ty)));;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (fabs _ x)))+(rule -1 (lower (fabs ty @ (multi_lane _ _) x))(vec_misc (VecMisc2.Fabs) x (vector_size ty)))-(rule (lower (has_type (ty_scalar_float ty) (fabs _ x)))+(rule (lower (fabs (ty_scalar_float ty) x))(fpu_rr (FPUOp1.Abs) x (scalar_size ty)));;;; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F64 (fpromote _ x)))+(rule (lower (fpromote $F64 x))(fpu_rr (FPUOp1.Cvt32To64) x (ScalarSize.Size32)));;;; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdemote _ x)))+(rule (lower (fdemote $F32 x))(fpu_rr (FPUOp1.Cvt64To32) x (ScalarSize.Size64)));;;; Rules for `ceil` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (ceil _ x)))+(rule -1 (lower (ceil ty @ (multi_lane _ _) x))(vec_misc (VecMisc2.Frintp) x (vector_size ty)))-(rule (lower (has_type $F32 (ceil _ x)))+(rule (lower (ceil $F32 x))(fpu_round (FpuRoundMode.Plus32) x))-(rule (lower (has_type $F64 (ceil _ x)))+(rule (lower (ceil $F64 x))(fpu_round (FpuRoundMode.Plus64) x));;;; Rules for `floor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (floor _ x)))+(rule -1 (lower (floor ty @ (multi_lane _ _) x))(vec_misc (VecMisc2.Frintm) x (vector_size ty)))-(rule (lower (has_type $F32 (floor _ x)))+(rule (lower (floor $F32 x))(fpu_round (FpuRoundMode.Minus32) x))-(rule (lower (has_type $F64 (floor _ x)))+(rule (lower (floor $F64 x))(fpu_round (FpuRoundMode.Minus64) x));;;; Rules for `trunc` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (trunc _ x)))+(rule -1 (lower (trunc ty @ (multi_lane _ _) x))(vec_misc (VecMisc2.Frintz) x (vector_size ty)))-(rule (lower (has_type $F32 (trunc _ x)))+(rule (lower (trunc $F32 x))(fpu_round (FpuRoundMode.Zero32) x))-(rule (lower (has_type $F64 (trunc _ x)))+(rule (lower (trunc $F64 x))(fpu_round (FpuRoundMode.Zero64) x));;;; Rules for `nearest` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane _ _) (nearest _ x)))+(rule -1 (lower (nearest ty @ (multi_lane _ _) x))(vec_misc (VecMisc2.Frintn) x (vector_size ty)))-(rule (lower (has_type $F32 (nearest _ x)))+(rule (lower (nearest $F32 x))(fpu_round (FpuRoundMode.Nearest32) x))-(rule (lower (has_type $F64 (nearest _ x)))+(rule (lower (nearest $F64 x))(fpu_round (FpuRoundMode.Nearest64) x));;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type ty (fma _ x y z))) (fmadd ty x y z))-(rule 2 (lower (has_type (ty_scalar_float ty) (fma _ x y (fneg _ z)))) (fnmsub ty x y z))+(rule 1 (lower (fma ty x y z)) (fmadd ty x y z))+(rule 2 (lower (fma (ty_scalar_float ty) x y (fneg _ z))) (fnmsub ty x y z));; Constructors matching the scalar behavior of aarch64. If you're confused like;; I was reading over these, they are:@@ -621,115 +619,115 @@;;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (fcopysign _ x y)))+(rule (lower (fcopysign ty x y))(fcopy_sign x y ty));;;; Rules for `fcvt_to_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint _ x @ (value_type $F32))))+(rule (lower (fcvt_to_uint (fits_in_32 out_ty) x @ (value_type $F32)))(fpu_to_int_cvt (FpuToIntOp.F32ToU32) x false $F32 out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_uint _ x @ (value_type $F32))))+(rule 1 (lower (fcvt_to_uint $I64 x @ (value_type $F32)))(fpu_to_int_cvt (FpuToIntOp.F32ToU64) x false $F32 $I64))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint _ x @ (value_type $F64))))+(rule (lower (fcvt_to_uint (fits_in_32 out_ty) x @ (value_type $F64)))(fpu_to_int_cvt (FpuToIntOp.F64ToU32) x false $F64 out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_uint _ x @ (value_type $F64))))+(rule 1 (lower (fcvt_to_uint $I64 x @ (value_type $F64)))(fpu_to_int_cvt (FpuToIntOp.F64ToU64) x false $F64 $I64));;;; Rules for `fcvt_to_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint _ x @ (value_type $F32))))+(rule (lower (fcvt_to_sint (fits_in_32 out_ty) x @ (value_type $F32)))(fpu_to_int_cvt (FpuToIntOp.F32ToI32) x true $F32 out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_sint _ x @ (value_type $F32))))+(rule 1 (lower (fcvt_to_sint $I64 x @ (value_type $F32)))(fpu_to_int_cvt (FpuToIntOp.F32ToI64) x true $F32 $I64))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint _ x @ (value_type $F64))))+(rule (lower (fcvt_to_sint (fits_in_32 out_ty) x @ (value_type $F64)))(fpu_to_int_cvt (FpuToIntOp.F64ToI32) x true $F64 out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_sint _ x @ (value_type $F64))))+(rule 1 (lower (fcvt_to_sint $I64 x @ (value_type $F64)))(fpu_to_int_cvt (FpuToIntOp.F64ToI64) x true $F64 $I64));;;; Rules for `fcvt_from_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_from_uint _ x @ (value_type (multi_lane 32 _)))))+(rule -1 (lower (fcvt_from_uint ty @ (multi_lane 32 _) x @ (value_type (multi_lane 32 _))))(vec_misc (VecMisc2.Ucvtf) x (vector_size ty)))-(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_from_uint _ x @ (value_type (multi_lane 64 _)))))+(rule -1 (lower (fcvt_from_uint ty @ (multi_lane 64 _) x @ (value_type (multi_lane 64 _))))(vec_misc (VecMisc2.Ucvtf) x (vector_size ty)))-(rule (lower (has_type $F32 (fcvt_from_uint _ x @ (value_type (fits_in_32 _)))))+(rule (lower (fcvt_from_uint $F32 x @ (value_type (fits_in_32 _))))(int_to_fpu (IntToFpuOp.U32ToF32) (put_in_reg_zext32 x)))-(rule (lower (has_type $F64 (fcvt_from_uint _ x @ (value_type (fits_in_32 _)))))+(rule (lower (fcvt_from_uint $F64 x @ (value_type (fits_in_32 _))))(int_to_fpu (IntToFpuOp.U32ToF64) (put_in_reg_zext32 x)))-(rule 1 (lower (has_type $F32 (fcvt_from_uint _ x @ (value_type $I64))))+(rule 1 (lower (fcvt_from_uint $F32 x @ (value_type $I64)))(int_to_fpu (IntToFpuOp.U64ToF32) x))-(rule 1 (lower (has_type $F64 (fcvt_from_uint _ x @ (value_type $I64))))+(rule 1 (lower (fcvt_from_uint $F64 x @ (value_type $I64)))(int_to_fpu (IntToFpuOp.U64ToF64) x));;;; Rules for `fcvt_from_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_from_sint _ x @ (value_type (multi_lane 32 _)))))+(rule -1 (lower (fcvt_from_sint ty @ (multi_lane 32 _) x @ (value_type (multi_lane 32 _))))(vec_misc (VecMisc2.Scvtf) x (vector_size ty)))-(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_from_sint _ x @ (value_type (multi_lane 64 _)))))+(rule -1 (lower (fcvt_from_sint ty @ (multi_lane 64 _) x @ (value_type (multi_lane 64 _))))(vec_misc (VecMisc2.Scvtf) x (vector_size ty)))-(rule (lower (has_type $F32 (fcvt_from_sint _ x @ (value_type (fits_in_32 _)))))+(rule (lower (fcvt_from_sint $F32 x @ (value_type (fits_in_32 _))))(int_to_fpu (IntToFpuOp.I32ToF32) (put_in_reg_sext32 x)))-(rule (lower (has_type $F64 (fcvt_from_sint _ x @ (value_type (fits_in_32 _)))))+(rule (lower (fcvt_from_sint $F64 x @ (value_type (fits_in_32 _))))(int_to_fpu (IntToFpuOp.I32ToF64) (put_in_reg_sext32 x)))-(rule 1 (lower (has_type $F32 (fcvt_from_sint _ x @ (value_type $I64))))+(rule 1 (lower (fcvt_from_sint $F32 x @ (value_type $I64)))(int_to_fpu (IntToFpuOp.I64ToF32) x))-(rule 1 (lower (has_type $F64 (fcvt_from_sint _ x @ (value_type $I64))))+(rule 1 (lower (fcvt_from_sint $F64 x @ (value_type $I64)))(int_to_fpu (IntToFpuOp.I64ToF64) x));;;; Rules for `fcvt_to_uint_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_to_uint_sat _ x @ (value_type (multi_lane 32 _)))))+(rule -1 (lower (fcvt_to_uint_sat ty @ (multi_lane 32 _) x @ (value_type (multi_lane 32 _))))(vec_misc (VecMisc2.Fcvtzu) x (vector_size ty)))-(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_to_uint_sat _ x @ (value_type (multi_lane 64 _)))))+(rule -1 (lower (fcvt_to_uint_sat ty @ (multi_lane 64 _) x @ (value_type (multi_lane 64 _))))(vec_misc (VecMisc2.Fcvtzu) x (vector_size ty)))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint_sat _ x @ (value_type $F32))))+(rule (lower (fcvt_to_uint_sat (fits_in_32 out_ty) x @ (value_type $F32)))(fpu_to_int_cvt_sat (FpuToIntOp.F32ToU32) x false out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_uint_sat _ x @ (value_type $F32))))+(rule 1 (lower (fcvt_to_uint_sat $I64 x @ (value_type $F32)))(fpu_to_int_cvt_sat (FpuToIntOp.F32ToU64) x false $I64))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_uint_sat _ x @ (value_type $F64))))+(rule (lower (fcvt_to_uint_sat (fits_in_32 out_ty) x @ (value_type $F64)))(fpu_to_int_cvt_sat (FpuToIntOp.F64ToU32) x false out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_uint_sat _ x @ (value_type $F64))))+(rule 1 (lower (fcvt_to_uint_sat $I64 x @ (value_type $F64)))(fpu_to_int_cvt_sat (FpuToIntOp.F64ToU64) x false $I64));;;; Rules for `fcvt_to_sint_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty @ (multi_lane 32 _) (fcvt_to_sint_sat _ x @ (value_type (multi_lane 32 _)))))+(rule -1 (lower (fcvt_to_sint_sat ty @ (multi_lane 32 _) x @ (value_type (multi_lane 32 _))))(vec_misc (VecMisc2.Fcvtzs) x (vector_size ty)))-(rule -1 (lower (has_type ty @ (multi_lane 64 _) (fcvt_to_sint_sat _ x @ (value_type (multi_lane 64 _)))))+(rule -1 (lower (fcvt_to_sint_sat ty @ (multi_lane 64 _) x @ (value_type (multi_lane 64 _))))(vec_misc (VecMisc2.Fcvtzs) x (vector_size ty)))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint_sat _ x @ (value_type $F32))))+(rule (lower (fcvt_to_sint_sat (fits_in_32 out_ty) x @ (value_type $F32)))(fpu_to_int_cvt_sat (FpuToIntOp.F32ToI32) x true out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_sint_sat _ x @ (value_type $F32))))+(rule 1 (lower (fcvt_to_sint_sat $I64 x @ (value_type $F32)))(fpu_to_int_cvt_sat (FpuToIntOp.F32ToI64) x true $I64))-(rule (lower (has_type (fits_in_32 out_ty) (fcvt_to_sint_sat _ x @ (value_type $F64))))+(rule (lower (fcvt_to_sint_sat (fits_in_32 out_ty) x @ (value_type $F64)))(fpu_to_int_cvt_sat (FpuToIntOp.F64ToI32) x true out_ty))-(rule 1 (lower (has_type $I64 (fcvt_to_sint_sat _ x @ (value_type $F64))))+(rule 1 (lower (fcvt_to_sint_sat $I64 x @ (value_type $F64)))(fpu_to_int_cvt_sat (FpuToIntOp.F64ToI64) x true $I64));;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -737,81 +735,80 @@;; `i64` and smaller;; Base case, simply subtracting things in registers.-(rule isub_base_case -4 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x y)))+(rule isub_base_case -4 (lower (isub (ty_int_ref_scalar_64 ty) x y))(sub ty x y));; Special case for when one operand is an immediate that fits in 12 bits.-(rule isub_imm12 0 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x (imm12_from_value y))))+(rule isub_imm12 0 (lower (isub (ty_int_ref_scalar_64 ty) x (imm12_from_value y)))(sub_imm ty x y));; Same as the previous special case, except we can switch the subtraction to an;; addition if the negated immediate fits in 12 bits.-(rule isub_imm12_neg 2 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x y)))+(rule isub_imm12_neg 2 (lower (isub (ty_int_ref_scalar_64 ty) x y))(if-let imm12_neg (imm12_from_negated_value y))(add_imm ty x imm12_neg));; Special cases for when we're subtracting an extended register where the;; extending operation can get folded into the sub itself.-(rule isub_extend 1 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x (extended_value_from_value y))))+(rule isub_extend 1 (lower (isub (ty_int_ref_scalar_64 ty) x (extended_value_from_value y)))(sub_extend ty x y));; Finally a special case for when we're subtracting the shift of a different;; register by a constant amount and the shift can get folded into the sub.-(rule isub_ishl -3 (lower (has_type (ty_int_ref_scalar_64 ty)- (isub _ x (ishl _ y (iconst _ k)))))+(rule isub_ishl -3 (lower (isub (ty_int_ref_scalar_64 ty) x (ishl _ y (iconst _ k))))(if-let amt (lshl_from_imm64 ty k))(sub_shift ty x y amt));; vectors-(rule -2 (lower (has_type ty @ (multi_lane _ _) (isub _ x y)))+(rule -2 (lower (isub ty @ (multi_lane _ _) x y))(sub_vec x y (vector_size ty)));; `i128`-(rule -1 (lower (has_type $I128 (isub _ x y)))+(rule -1 (lower (isub $I128 x y))(sub_i128 x y));;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 ty) (uadd_sat _ x y)))+(rule (lower (uadd_sat (ty_vec128 ty) x y))(uqadd x y (vector_size ty)));;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 ty) (sadd_sat _ x y)))+(rule (lower (sadd_sat (ty_vec128 ty) x y))(sqadd x y (vector_size ty)));;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 ty) (usub_sat _ x y)))+(rule (lower (usub_sat (ty_vec128 ty) x y))(uqsub x y (vector_size ty)));;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 ty) (ssub_sat _ x y)))+(rule (lower (ssub_sat (ty_vec128 ty) x y))(sqsub x y (vector_size ty)));;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule ineg_base_case 1 (lower (has_type (fits_in_64 ty) (ineg _ x)))+(rule ineg_base_case 1 (lower (ineg (fits_in_64 ty) x))(sub ty (zero_reg) x));; `i128`-(rule 2 (lower (has_type $I128 (ineg _ x)))+(rule 2 (lower (ineg $I128 x))(sub_i128 (value_regs_zero) x));; vectors.-(rule (lower (has_type (ty_vec128 ty) (ineg _ x)))+(rule (lower (ineg (ty_vec128 ty) x))(neg x (vector_size ty)));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule imul_base_case -3 (lower (has_type (ty_int_ref_scalar_64 ty) (imul _ x y)))+(rule imul_base_case -3 (lower (imul (ty_int_ref_scalar_64 ty) x y))(madd ty x y (zero_reg)));; `i128`.-(rule -1 (lower (has_type $I128 (imul _ x y)))+(rule -1 (lower (imul $I128 x y))(let;; Get the high/low registers for `x`.((x_regs ValueRegs x)@@ -841,7 +838,7 @@;; Special cases where the upper bits are sign-or-zero extended of the lower bits;; so the calculation here is much simpler with just a `umulh` or `smulh`;; instead of the additions above as well.-(rule (lower (has_type $I128 (imul _ (uextend _ x) (uextend _ y))))+(rule (lower (imul $I128 (uextend _ x) (uextend _ y)))(let ((x Reg (put_in_reg_zext64 x))(y Reg (put_in_reg_zext64 y))@@ -849,7 +846,7 @@(value_regs(madd $I64 x y (zero_reg))(umulh $I64 x y))))-(rule (lower (has_type $I128 (imul _ (sextend _ x) (sextend _ y))))+(rule (lower (imul $I128 (sextend _ x) (sextend _ y)))(let ((x Reg (put_in_reg_sext64 x))(y Reg (put_in_reg_sext64 y))@@ -859,7 +856,7 @@(smulh $I64 x y))));; vectors (i8x8/i8x16/i16x4/i16x8/i32x2/i32x4)-(rule -2 (lower (has_type (lane_fits_in_32 ty @ (multi_lane _ _)) (imul _ x y)))+(rule -2 (lower (imul (lane_fits_in_32 ty @ (multi_lane _ _)) x y))(mul x y (vector_size ty)));; Special lowering for i64x2.@@ -891,7 +888,7 @@;; xtn tmp2.2s, rm.2d;; shll rd.2d, rd.2s, #32;; umlal rd.2d, tmp2.2s, tmp1.2s-(rule -1 (lower (has_type $I64X2 (imul _ x y)))+(rule -1 (lower (imul $I64X2 x y))(let ((rn Reg x)(rm Reg y);; Reverse the 32-bit elements in the 64-bit words.@@ -930,83 +927,71 @@result));; Special case for `i16x8.extmul_low_i8x16_s`.-(rule (lower (has_type $I16X8- (imul _ (swiden_low _ x @ (value_type $I8X16))- (swiden_low _ y @ (value_type $I8X16)))))+(rule (lower (imul $I16X8 (swiden_low _ x @ (value_type $I8X16))+ (swiden_low _ y @ (value_type $I8X16))))(smull8 x y false));; Special case for `i16x8.extmul_high_i8x16_s`.-(rule (lower (has_type $I16X8- (imul _ (swiden_high _ x @ (value_type $I8X16))- (swiden_high _ y @ (value_type $I8X16)))))+(rule (lower (imul $I16X8 (swiden_high _ x @ (value_type $I8X16))+ (swiden_high _ y @ (value_type $I8X16))))(smull8 x y true));; Special case for `i16x8.extmul_low_i8x16_u`.-(rule (lower (has_type $I16X8- (imul _ (uwiden_low _ x @ (value_type $I8X16))- (uwiden_low _ y @ (value_type $I8X16)))))+(rule (lower (imul $I16X8 (uwiden_low _ x @ (value_type $I8X16))+ (uwiden_low _ y @ (value_type $I8X16))))(umull8 x y false));; Special case for `i16x8.extmul_high_i8x16_u`.-(rule (lower (has_type $I16X8- (imul _ (uwiden_high _ x @ (value_type $I8X16))- (uwiden_high _ y @ (value_type $I8X16)))))+(rule (lower (imul $I16X8 (uwiden_high _ x @ (value_type $I8X16))+ (uwiden_high _ y @ (value_type $I8X16))))(umull8 x y true));; Special case for `i32x4.extmul_low_i16x8_s`.-(rule (lower (has_type $I32X4- (imul _ (swiden_low _ x @ (value_type $I16X8))- (swiden_low _ y @ (value_type $I16X8)))))+(rule (lower (imul $I32X4 (swiden_low _ x @ (value_type $I16X8))+ (swiden_low _ y @ (value_type $I16X8))))(smull16 x y false));; Special case for `i32x4.extmul_high_i16x8_s`.-(rule (lower (has_type $I32X4- (imul _ (swiden_high _ x @ (value_type $I16X8))- (swiden_high _ y @ (value_type $I16X8)))))+(rule (lower (imul $I32X4 (swiden_high _ x @ (value_type $I16X8))+ (swiden_high _ y @ (value_type $I16X8))))(smull16 x y true));; Special case for `i32x4.extmul_low_i16x8_u`.-(rule (lower (has_type $I32X4- (imul _ (uwiden_low _ x @ (value_type $I16X8))- (uwiden_low _ y @ (value_type $I16X8)))))+(rule (lower (imul $I32X4 (uwiden_low _ x @ (value_type $I16X8))+ (uwiden_low _ y @ (value_type $I16X8))))(umull16 x y false));; Special case for `i32x4.extmul_high_i16x8_u`.-(rule (lower (has_type $I32X4- (imul _ (uwiden_high _ x @ (value_type $I16X8))- (uwiden_high _ y @ (value_type $I16X8)))))+(rule (lower (imul $I32X4 (uwiden_high _ x @ (value_type $I16X8))+ (uwiden_high _ y @ (value_type $I16X8))))(umull16 x y true));; Special case for `i64x2.extmul_low_i32x4_s`.-(rule (lower (has_type $I64X2- (imul _ (swiden_low _ x @ (value_type $I32X4))- (swiden_low _ y @ (value_type $I32X4)))))+(rule (lower (imul $I64X2 (swiden_low _ x @ (value_type $I32X4))+ (swiden_low _ y @ (value_type $I32X4))))(smull32 x y false));; Special case for `i64x2.extmul_high_i32x4_s`.-(rule (lower (has_type $I64X2- (imul _ (swiden_high _ x @ (value_type $I32X4))- (swiden_high _ y @ (value_type $I32X4)))))+(rule (lower (imul $I64X2 (swiden_high _ x @ (value_type $I32X4))+ (swiden_high _ y @ (value_type $I32X4))))(smull32 x y true));; Special case for `i64x2.extmul_low_i32x4_u`.-(rule (lower (has_type $I64X2- (imul _ (uwiden_low _ x @ (value_type $I32X4))- (uwiden_low _ y @ (value_type $I32X4)))))+(rule (lower (imul $I64X2 (uwiden_low _ x @ (value_type $I32X4))+ (uwiden_low _ y @ (value_type $I32X4))))(umull32 x y false));; Special case for `i64x2.extmul_high_i32x4_u`.-(rule (lower (has_type $I64X2- (imul _ (uwiden_high _ x @ (value_type $I32X4))- (uwiden_high _ y @ (value_type $I32X4)))))+(rule (lower (imul $I64X2 (uwiden_high _ x @ (value_type $I32X4))+ (uwiden_high _ y @ (value_type $I32X4))))(umull32 x y true));;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $I64 (smulhi _ x y)))+(rule 1 (lower (smulhi $I64 x y))(smulh $I64 x y))-(rule (lower (has_type (fits_in_32 ty) (smulhi _ x y)))+(rule (lower (smulhi (fits_in_32 ty) x y))(let ((x64 Reg (put_in_reg_sext64 x))(y64 Reg (put_in_reg_sext64 y))(mul Reg (madd $I64 x64 y64 (zero_reg)))@@ -1015,10 +1000,10 @@;;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $I64 (umulhi _ x y)))+(rule 1 (lower (umulhi $I64 x y))(umulh $I64 x y))-(rule (lower (has_type (fits_in_32 ty) (umulhi _ x y)))+(rule (lower (umulhi (fits_in_32 ty) x y))(let ((x64 Reg (put_in_reg_zext64 x))(y64 Reg (put_in_reg_zext64 y))@@ -1066,10 +1051,10 @@;; Note that aarch64's `udiv` doesn't trap so to respect the semantics of;; CLIF's `udiv` the check for zero needs to be manually performed.-(rule udiv 1 (lower (has_type $I64 (udiv _ x y)))+(rule udiv 1 (lower (udiv $I64 x y))(a64_udiv $I64 (put_in_reg x) (put_nonzero_in_reg y (ExtType.Unsigned) $I64)))-(rule udiv (lower (has_type (fits_in_32 ty) (udiv _ x y)))+(rule udiv (lower (udiv (fits_in_32 ty) x y))(a64_udiv $I32 (put_in_reg_zext32 x) (put_nonzero_in_reg y (ExtType.Unsigned) ty)));;;; Rules for `sdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1095,7 +1080,7 @@;; TODO: if `y` is -1 then a check that `x` is not INT_MIN is all that's;; necessary, but right now `y` is checked to not be -1 as well.-(rule sdiv_base_case (lower (has_type $I64 (sdiv _ x y)))+(rule sdiv_base_case (lower (sdiv $I64 x y))(let ((x64 Reg (put_in_reg_sext64 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Signed) $I64))(intmin_check_x Reg (intmin_check $I64 x64))@@ -1103,7 +1088,7 @@(result Reg (a64_sdiv $I64 valid_x64 y64)))result))-(rule sdiv_base_case -1 (lower (has_type (fits_in_32 ty) (sdiv _ x y)))+(rule sdiv_base_case -1 (lower (sdiv (fits_in_32 ty) x y))(let ((x32 Reg (put_in_reg_sext32 x))(y32 Reg (put_nonzero_in_reg y (ExtType.Signed) ty))(intmin_check_x Reg (intmin_check ty x32))@@ -1113,11 +1098,11 @@;; Special case for `sdiv` where no checks are needed due to division by a;; constant meaning the checks are always passed.-(rule sdiv_safe_divisor 2 (lower (has_type $I64 (sdiv _ x (iconst _ imm))))+(rule sdiv_safe_divisor 2 (lower (sdiv $I64 x (iconst _ imm)))(if-let y (safe_divisor_from_imm64 $I64 imm))(a64_sdiv $I64 (put_in_reg_sext64 x) (imm $I64 (ImmExtend.Sign) y)))-(rule sdiv_safe_divisor 1 (lower (has_type (fits_in_32 ty) (sdiv _ x (iconst _ imm))))+(rule sdiv_safe_divisor 1 (lower (sdiv (fits_in_32 ty) x (iconst _ imm)))(if-let y (safe_divisor_from_imm64 ty imm))(a64_sdiv ty (put_in_reg_sext32 x) (imm ty (ImmExtend.Sign) y)))@@ -1139,28 +1124,28 @@;; TODO: we can avoid a 0 check, if the dividend is a non-0 constant-(rule urem (lower (has_type $I64 (urem _ x y)))+(rule urem (lower (urem $I64 x y))(let ((x64 Reg (put_in_reg_zext64 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Unsigned) $I64))(div Reg (a64_udiv $I64 x64 y64))(result Reg (msub $I64 div y64 x64)))result))-(rule urem -1 (lower (has_type (fits_in_32 ty) (urem _ x y)))+(rule urem -1 (lower (urem (fits_in_32 ty) x y))(let ((x64 Reg (put_in_reg_zext32 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Unsigned) ty))(div Reg (a64_udiv ty x64 y64))(result Reg (msub ty div y64 x64)))result))-(rule srem (lower (has_type $I64 (srem _ x y)))+(rule srem (lower (srem $I64 x y))(let ((x64 Reg (put_in_reg_sext64 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Signed) $I64))(div Reg (a64_sdiv $I64 x64 y64))(result Reg (msub $I64 div y64 x64)))result))-(rule srem -1 (lower (has_type (fits_in_32 ty) (srem _ x y)))+(rule srem -1 (lower (srem (fits_in_32 ty) x y))(let ((x64 Reg (put_in_reg_sext32 x))(y64 Reg (put_nonzero_in_reg y (ExtType.Signed) ty))(div Reg (a64_sdiv ty x64 y64))@@ -1207,76 +1192,73 @@(with_flags_reg (cmp (operand_size ty) x y)(csel cc x y))))-(rule umin 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (umin _ x y)))+(rule umin 2 (lower (umin (and (fits_in_64 ty) (ty_int _)) x y))(cmp_and_choose ty (Cond.Lo) false x y))-(rule smin 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (smin _ x y)))+(rule smin 2 (lower (smin (and (fits_in_64 ty) (ty_int _)) x y))(cmp_and_choose ty (Cond.Lt) true x y))-(rule umax 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (umax _ x y)))+(rule umax 2 (lower (umax (and (fits_in_64 ty) (ty_int _)) x y))(cmp_and_choose ty (Cond.Hi) false x y))-(rule smax 2 (lower (has_type (and (fits_in_64 ty) (ty_int _)) (smax _ x y)))+(rule smax 2 (lower (smax (and (fits_in_64 ty) (ty_int _)) x y))(cmp_and_choose ty (Cond.Gt) true x y));; Vector types.-(rule (lower (has_type ty @ (not_i64x2) (smin _ x y)))+(rule (lower (smin ty @ (not_i64x2) x y))(vec_rrr (VecALUOp.Smin) x y (vector_size ty)))-(rule 1 (lower (has_type $I64X2 (smin _ x y)))+(rule 1 (lower (smin $I64X2 x y))(bsl $I64X2 (vec_rrr (VecALUOp.Cmgt) y x (VectorSize.Size64x2)) x y))-(rule (lower (has_type ty @ (not_i64x2) (umin _ x y)))+(rule (lower (umin ty @ (not_i64x2) x y))(vec_rrr (VecALUOp.Umin) x y (vector_size ty)))-(rule 1 (lower (has_type $I64X2 (umin _ x y)))+(rule 1 (lower (umin $I64X2 x y))(bsl $I64X2 (vec_rrr (VecALUOp.Cmhi) y x (VectorSize.Size64x2)) x y))-(rule (lower (has_type ty @ (not_i64x2) (smax _ x y)))+(rule (lower (smax ty @ (not_i64x2) x y))(vec_rrr (VecALUOp.Smax) x y (vector_size ty)))-(rule 1 (lower (has_type $I64X2 (smax _ x y)))+(rule 1 (lower (smax $I64X2 x y))(bsl $I64X2 (vec_rrr (VecALUOp.Cmgt) x y (VectorSize.Size64x2)) x y))-(rule (lower (has_type ty @ (not_i64x2) (umax _ x y)))+(rule (lower (umax ty @ (not_i64x2) x y))(vec_rrr (VecALUOp.Umax) x y (vector_size ty)))-(rule 1 (lower (has_type $I64X2 (umax _ x y)))+(rule 1 (lower (umax $I64X2 x y))(bsl $I64X2 (vec_rrr (VecALUOp.Cmhi) x y (VectorSize.Size64x2)) x y));;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; General rule for extending input to an output which fits in a single;; register.-(rule uextend -2 (lower (has_type (fits_in_64 out) (uextend _ x @ (value_type in))))+(rule uextend -2 (lower (uextend (fits_in_64 out) x @ (value_type in)))(extend x false (ty_bits in) (ty_bits out)));; Extraction of a vector lane automatically extends as necessary, so we can;; skip an explicit extending instruction.-(rule 1 (lower (has_type (fits_in_64 out)- (uextend _ (extractlane _ vec @ (value_type in)- (u8_from_uimm8 lane)))))+(rule 1 (lower (uextend (fits_in_64 out) (extractlane _ vec @ (value_type in)+ (u8_from_uimm8 lane))))(mov_from_vec (put_in_reg vec) lane (lane_size in)));; Atomic loads will also automatically zero their upper bits so the `uextend`;; instruction can effectively get skipped here.-(rule 1 (lower (has_type (fits_in_64 out)- (uextend _ x @ (and (value_type in) (atomic_load _ (little_or_native_endian flags) _)))))+(rule 1 (lower (uextend (fits_in_64 out) x @ (and (value_type in) (atomic_load _ (little_or_native_endian flags) _))))(if-let mem_op (is_sinkable_inst x))(load_acquire in flags (sink_atomic_load mem_op)));; Conversion to 128-bit needs a zero-extension of the lower bits and the upper;; bits are all zero.-(rule -1 (lower (has_type $I128 (uextend _ x)))+(rule -1 (lower (uextend $I128 x))(value_regs (put_in_reg_zext64 x) (imm $I64 (ImmExtend.Zero) 0)));; Like above where vector extraction automatically zero-extends extending to;; i128 only requires generating a 0 constant for the upper bits.-(rule (lower (has_type $I128- (uextend _ (extractlane _ vec @ (value_type in)- (u8_from_uimm8 lane)))))+(rule (lower (uextend $I128 (extractlane _ vec @ (value_type in)+ (u8_from_uimm8 lane))))(value_regs (mov_from_vec (put_in_reg vec) lane (lane_size in)) (imm $I64 (ImmExtend.Zero) 0)));; Zero extensions from a load can be encoded in the load itself-(rule (lower (has_type (fits_in_64 _) (uextend _ x @ (has_type in_ty (load _ (little_or_native_endian flags) address offset)))))+(rule (lower (uextend (fits_in_64 _) x @ (load in_ty (little_or_native_endian flags) address offset)))(if-let inst (is_sinkable_inst x))(let ((_ Unit (sink_inst inst)))(aarch64_uload in_ty (amode in_ty address offset) flags)))@@ -1290,21 +1272,20 @@;; General rule for extending input to an output which fits in a single;; register.-(rule sextend -4 (lower (has_type (fits_in_64 out) (sextend _ x @ (value_type in))))+(rule sextend -4 (lower (sextend (fits_in_64 out) x @ (value_type in)))(extend x true (ty_bits in) (ty_bits out)));; Extraction of a vector lane automatically extends as necessary, so we can;; skip an explicit extending instruction.-(rule -3 (lower (has_type (fits_in_64 out)- (sextend _ (extractlane _ vec @ (value_type in)- (u8_from_uimm8 lane)))))+(rule -3 (lower (sextend (fits_in_64 out) (extractlane _ vec @ (value_type in)+ (u8_from_uimm8 lane))))(mov_from_vec_signed (put_in_reg vec)lane(vector_size in)(size_from_ty out)));; 64-bit to 128-bit only needs to sign-extend the input to the upper bits.-(rule -2 (lower (has_type $I128 (sextend _ x)))+(rule -2 (lower (sextend $I128 x))(let ((lo Reg (put_in_reg_sext64 x))(hi Reg (asr_imm $I64 lo (imm_shift_from_u8 63))))(value_regs lo hi)))@@ -1314,9 +1295,8 @@;;;; Note that `mov_from_vec_signed` doesn't exist for i64x2, so that's;; specifically excluded here.-(rule (lower (has_type $I128- (sextend _ (extractlane _ vec @ (value_type in @ (not_i64x2))- (u8_from_uimm8 lane)))))+(rule (lower (sextend $I128 (extractlane _ vec @ (value_type in @ (not_i64x2))+ (u8_from_uimm8 lane))))(let ((lo Reg (mov_from_vec_signed (put_in_reg vec)lane(vector_size in)@@ -1325,9 +1305,8 @@(value_regs lo hi)));; Extension from an extraction of i64x2 into i128.-(rule -1 (lower (has_type $I128- (sextend _ (extractlane _ vec @ (value_type $I64X2)- (u8_from_uimm8 lane)))))+(rule -1 (lower (sextend $I128 (extractlane _ vec @ (value_type $I64X2)+ (u8_from_uimm8 lane))))(let ((lo Reg (mov_from_vec (put_in_reg vec)lane(ScalarSize.Size64)))@@ -1335,7 +1314,7 @@(value_regs lo hi)));; Signed extensions from a load can be encoded in the load itself-(rule (lower (has_type (fits_in_64 _) (sextend _ x @ (has_type in_ty (load _ (little_or_native_endian flags) address offset)))))+(rule (lower (sextend (fits_in_64 _) x @ (load in_ty (little_or_native_endian flags) address offset)))(if-let inst (is_sinkable_inst x))(let ((_ Unit (sink_inst inst)))(aarch64_sload in_ty (amode in_ty address offset) flags)))@@ -1352,21 +1331,21 @@;; Note that bitwise negation is implemented here as;;;; NOT rd, rm ==> ORR_NOT rd, zero, rm-(rule bnot_base_case -4 (lower (has_type (fits_in_64 (ty_int ty)) (bnot _ x)))+(rule bnot_base_case -4 (lower (bnot (fits_in_64 (ty_int ty)) x))(orr_not ty (zero_reg) x));; Implementation of `bnot` for floats.-(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bnot _ x)))+(rule -3 (lower (bnot (fits_in_64 (ty_scalar_float ty)) x))(not x (float_vector_size_in_64 ty)));; Implementation of `bnot` for vector types.-(rule -2 (lower (has_type (ty_vec64 ty) (bnot _ x)))+(rule -2 (lower (bnot (ty_vec64 ty) x))(not x (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (bnot _ x)))+(rule -1 (lower (bnot (ty_vec128 ty) x))(not x (vector_size ty)));; Implementation of `bnot` for `i128`.-(rule (lower (has_type $I128 (bnot _ x)))+(rule (lower (bnot $I128 x))(let ((x_regs ValueRegs x)(x_lo Reg (value_regs_get x_regs 0))(x_hi Reg (value_regs_get x_regs 1))@@ -1376,88 +1355,87 @@;; Special case to use `orr_not_shift` if it's a `bnot` of a const-left-shifted;; value.-(rule bnot_ishl 1 (lower (has_type (fits_in_64 (ty_int ty))- (bnot _ (ishl _ x (iconst _ k)))))+(rule bnot_ishl 1 (lower (bnot (fits_in_64 (ty_int ty)) (ishl _ x (iconst _ k))))(if-let amt (lshl_from_imm64 ty k))(orr_not_shift ty (zero_reg) x amt));; Special-cases for fusing a bnot with bxor-(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bnot _ (bxor _ x y))))+(rule 2 (lower (bnot (fits_in_64 (ty_int ty)) (bxor _ x y)))(alu_rs_imm_logic (ALUOp.EorNot) ty x y))-(rule 3 (lower (has_type $I128 (bnot _ (bxor _ x y)))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y))+(rule 3 (lower (bnot $I128 (bxor _ x y))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y));;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule band_fits_in_64 -5 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x y)))+(rule band_fits_in_64 -5 (lower (band (fits_in_64 (ty_int ty)) x y))(alu_rs_imm_logic_commutative (ALUOp.And) ty x y))-(rule -4 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (band _ x y)))+(rule -4 (lower (band (fits_in_64 (ty_scalar_float ty)) x y))(and_vec x y (float_vector_size_in_64 ty)));; Implementation of `band` for vector types.-(rule -2 (lower (has_type (ty_vec64 ty) (band _ x y)))+(rule -2 (lower (band (ty_vec64 ty) x y))(and_vec x y (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (band _ x y)))+(rule -1 (lower (band (ty_vec128 ty) x y))(and_vec x y (vector_size ty)))-(rule (lower (has_type $I128 (band _ x y))) (i128_alu_bitop (ALUOp.And) $I64 x y))+(rule (lower (band $I128 x y)) (i128_alu_bitop (ALUOp.And) $I64 x y));; Specialized lowerings for `(band x (bnot y))` which is additionally produced;; by Cranelift's `band_not` builder method, which emits the simpler;; forms early on.-(rule band_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x (bnot _ y))))+(rule band_not_right 1 (lower (band (fits_in_64 (ty_int ty)) x (bnot _ y)))(alu_rs_imm_logic (ALUOp.AndNot) ty x y))-(rule band_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (band _ (bnot _ y) x)))+(rule band_not_left 2 (lower (band (fits_in_64 (ty_int ty)) (bnot _ y) x))(alu_rs_imm_logic (ALUOp.AndNot) ty x y))-(rule 3 (lower (has_type $I128 (band _ x (bnot _ y)))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))-(rule 4 (lower (has_type $I128 (band _ (bnot _ y) x))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))+(rule 3 (lower (band $I128 x (bnot _ y))) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))+(rule 4 (lower (band $I128 (bnot _ y) x)) (i128_alu_bitop (ALUOp.AndNot) $I64 x y))-(rule 5 (lower (has_type (ty_vec64 ty) (band _ x (bnot _ y))))+(rule 5 (lower (band (ty_vec64 ty) x (bnot _ y)))(bic_vec x y (vector_size ty)))-(rule 6 (lower (has_type (ty_vec64 ty) (band _ (bnot _ y) x)))+(rule 6 (lower (band (ty_vec64 ty) (bnot _ y) x))(bic_vec x y (vector_size ty)))-(rule 7 (lower (has_type (ty_vec128 ty) (band _ x (bnot _ y))))+(rule 7 (lower (band (ty_vec128 ty) x (bnot _ y)))(bic_vec x y (vector_size ty)))-(rule 8 (lower (has_type (ty_vec128 ty) (band _ (bnot _ y) x)))+(rule 8 (lower (band (ty_vec128 ty) (bnot _ y) x))(bic_vec x y (vector_size ty)));;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule bor_fits_in_64 -4 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x y)))+(rule bor_fits_in_64 -4 (lower (bor (fits_in_64 (ty_int ty)) x y))(alu_rs_imm_logic_commutative (ALUOp.Orr) ty x y))-(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bor _ x y)))+(rule -3 (lower (bor (fits_in_64 (ty_scalar_float ty)) x y))(orr_vec x y (float_vector_size_in_64 ty)));; Implementation of `bor` for vector types.-(rule -2 (lower (has_type (ty_vec64 ty) (bor _ x y)))+(rule -2 (lower (bor (ty_vec64 ty) x y))(orr_vec x y (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (bor _ x y)))+(rule -1 (lower (bor (ty_vec128 ty) x y))(orr_vec x y (vector_size ty)))-(rule (lower (has_type $I128 (bor _ x y))) (i128_alu_bitop (ALUOp.Orr) $I64 x y))+(rule (lower (bor $I128 x y)) (i128_alu_bitop (ALUOp.Orr) $I64 x y));; Specialized lowerings for `(bor x (bnot y))` which is additionally produced;; by Cranelift's `bor_not` builder method, which emits the simpler;; forms early on.-(rule bor_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x (bnot _ y))))+(rule bor_not_right 1 (lower (bor (fits_in_64 (ty_int ty)) x (bnot _ y)))(alu_rs_imm_logic (ALUOp.OrrNot) ty x y))-(rule bor_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ (bnot _ y) x)))+(rule bor_not_left 2 (lower (bor (fits_in_64 (ty_int ty)) (bnot _ y) x))(alu_rs_imm_logic (ALUOp.OrrNot) ty x y))-(rule 3 (lower (has_type $I128 (bor _ x (bnot _ y)))) (i128_alu_bitop (ALUOp.OrrNot) $I64 x y))-(rule 4 (lower (has_type $I128 (bor _ (bnot _ y) x))) (i128_alu_bitop (ALUOp.OrrNot) $I64 x y))+(rule 3 (lower (bor $I128 x (bnot _ y))) (i128_alu_bitop (ALUOp.OrrNot) $I64 x y))+(rule 4 (lower (bor $I128 (bnot _ y) x)) (i128_alu_bitop (ALUOp.OrrNot) $I64 x y))-(rule bor_not_right_vec64 5 (lower (has_type (ty_vec64 ty) (bor _ x (bnot _ y))))+(rule bor_not_right_vec64 5 (lower (bor (ty_vec64 ty) x (bnot _ y)))(orn_vec x y (vector_size ty)))-(rule bor_not_left_vec64 6 (lower (has_type (ty_vec64 ty) (bor _ (bnot _ y) x)))+(rule bor_not_left_vec64 6 (lower (bor (ty_vec64 ty) (bnot _ y) x))(orn_vec x y (vector_size ty)))-(rule bor_not_right_vec128 7 (lower (has_type (ty_vec128 ty) (bor _ x (bnot _ y))))+(rule bor_not_right_vec128 7 (lower (bor (ty_vec128 ty) x (bnot _ y)))(orn_vec x y (vector_size ty)))-(rule bor_not_left_vec128 8 (lower (has_type (ty_vec128 ty) (bor _ (bnot _ y) x)))+(rule bor_not_left_vec128 8 (lower (bor (ty_vec128 ty) (bnot _ y) x))(orn_vec x y (vector_size ty)))@@ -1477,14 +1455,12 @@;; they sum to the type width then it means that the shifts don't actually do;; anything CLIF-wise and this should compile down to a `bor` operation. Leave;; that edge case to the mid-end and only lower to `extr` here.-(rule 5 (lower (has_type (ty_32_or_64 ty)- (bor _ (ishl _ x (u8_from_iconst xs)) (ushr _ y (u8_from_iconst ys)))))+(rule 5 (lower (bor (ty_32_or_64 ty) (ishl _ x (u8_from_iconst xs)) (ushr _ y (u8_from_iconst ys))))(if-let true (u64_eq (ty_bits ty) (u64_wrapping_add xs ys)))(if-let true (u64_gt xs 0))(if-let true (u64_gt ys 0))(a64_extr ty x y (imm_shift_from_u8 ys)))-(rule 5 (lower (has_type (ty_32_or_64 ty)- (bor _ (ushr _ y (u8_from_iconst ys)) (ishl _ x (u8_from_iconst xs)))))+(rule 5 (lower (bor (ty_32_or_64 ty) (ushr _ y (u8_from_iconst ys)) (ishl _ x (u8_from_iconst xs))))(if-let true (u64_eq (ty_bits ty) (u64_wrapping_add xs ys)))(if-let true (u64_gt xs 0))(if-let true (u64_gt ys 0))@@ -1492,44 +1468,44 @@;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule bxor_fits_in_64 -4 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x y)))+(rule bxor_fits_in_64 -4 (lower (bxor (fits_in_64 (ty_int ty)) x y))(alu_rs_imm_logic_commutative (ALUOp.Eor) ty x y))-(rule -3 (lower (has_type (fits_in_64 (ty_scalar_float ty)) (bxor _ x y)))+(rule -3 (lower (bxor (fits_in_64 (ty_scalar_float ty)) x y))(eor_vec x y (float_vector_size_in_64 ty)));; Implementation of `bxor` for vector types.-(rule -2 (lower (has_type (ty_vec64 ty) (bxor _ x y)))+(rule -2 (lower (bxor (ty_vec64 ty) x y))(eor_vec x y (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (bxor _ x y)))+(rule -1 (lower (bxor (ty_vec128 ty) x y))(eor_vec x y (vector_size ty)))-(rule (lower (has_type $I128 (bxor _ x y))) (i128_alu_bitop (ALUOp.Eor) $I64 x y))+(rule (lower (bxor $I128 x y)) (i128_alu_bitop (ALUOp.Eor) $I64 x y));; Specialized lowerings for `(bxor x (bnot y))` which is additionally produced;; by Cranelift's `bxor_not` builder method, which emits the simpler;; forms early on.-(rule bxor_not_right 1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x (bnot _ y))))+(rule bxor_not_right 1 (lower (bxor (fits_in_64 (ty_int ty)) x (bnot _ y)))(alu_rs_imm_logic (ALUOp.EorNot) ty x y))-(rule bxor_not_left 2 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ (bnot _ y) x)))+(rule bxor_not_left 2 (lower (bxor (fits_in_64 (ty_int ty)) (bnot _ y) x))(alu_rs_imm_logic (ALUOp.EorNot) ty x y))-(rule 3 (lower (has_type $I128 (bxor _ x (bnot _ y)))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y))-(rule 4 (lower (has_type $I128 (bxor _ (bnot _ y) x))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y))+(rule 3 (lower (bxor $I128 x (bnot _ y))) (i128_alu_bitop (ALUOp.EorNot) $I64 x y))+(rule 4 (lower (bxor $I128 (bnot _ y) x)) (i128_alu_bitop (ALUOp.EorNot) $I64 x y));;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Shift for i8/i16/i32.-(rule ishl_fits_in_32 -1 (lower (has_type (fits_in_32 ty) (ishl _ x y)))+(rule ishl_fits_in_32 -1 (lower (ishl (fits_in_32 ty) x y))(do_shift (ALUOp.Lsl) ty x y));; Shift for i64.-(rule ishl_64 (lower (has_type $I64 (ishl _ x y)))+(rule ishl_64 (lower (ishl $I64 x y))(do_shift (ALUOp.Lsl) $I64 x y));; Shift for i128.-(rule (lower (has_type $I128 (ishl _ x y)))+(rule (lower (ishl $I128 x y))(lower_shl128 x (value_regs_get y 0)));; lsl lo_lshift, src_lo, amt@@ -1559,12 +1535,12 @@(csel (Cond.Ne) lo_lshift maybe_hi)))));; Shift for vector types.-(rule -3 (lower (has_type (ty_vec128 ty) (ishl _ x y)))+(rule -3 (lower (ishl (ty_vec128 ty) x y))(let ((size VectorSize (vector_size ty))(masked_shift_amt Reg (and_imm $I32 y (shift_mask ty)))(shift Reg (vec_dup masked_shift_amt size)))(sshl x shift size)))-(rule -2 (lower (has_type (ty_vec128 ty) (ishl _ x (iconst _ (u64_from_imm64 n)))))+(rule -2 (lower (ishl (ty_vec128 ty) x (iconst _ (u64_from_imm64 n))))(ushl_vec_imm x (shift_masked_imm ty n) (vector_size ty)))(decl pure shift_masked_imm (Type u64) u8)@@ -1652,15 +1628,15 @@;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Shift for i8/i16/i32.-(rule ushr_fits_in_32 -1 (lower (has_type (fits_in_32 ty) (ushr _ x y)))+(rule ushr_fits_in_32 -1 (lower (ushr (fits_in_32 ty) x y))(do_shift (ALUOp.Lsr) ty (put_in_reg_zext32 x) y));; Shift for i64.-(rule ushr_64 (lower (has_type $I64 (ushr _ x y)))+(rule ushr_64 (lower (ushr $I64 x y))(do_shift (ALUOp.Lsr) $I64 (put_in_reg_zext64 x) y));; Shift for i128.-(rule (lower (has_type $I128 (ushr _ x y)))+(rule (lower (ushr $I128 x y))(lower_ushr128 x (value_regs_get y 0)));; Vector shifts.@@ -1668,14 +1644,14 @@;; Note that for constant shifts a 0-width shift can't be emitted so it's;; special cased to pass through the input as-is since a 0-shift doesn't modify;; the input anyway.-(rule -4 (lower (has_type (ty_vec128 ty) (ushr _ x y)))+(rule -4 (lower (ushr (ty_vec128 ty) x y))(let ((size VectorSize (vector_size ty))(masked_shift_amt Reg (and_imm $I32 y (shift_mask ty)))(shift Reg (vec_dup (sub $I64 (zero_reg) masked_shift_amt) size)))(ushl x shift size)))-(rule -3 (lower (has_type (ty_vec128 ty) (ushr _ x (iconst _ (u64_from_imm64 n)))))+(rule -3 (lower (ushr (ty_vec128 ty) x (iconst _ (u64_from_imm64 n))))(ushr_vec_imm x (shift_masked_imm ty n) (vector_size ty)))-(rule -2 (lower (has_type (ty_vec128 ty) (ushr _ x (iconst _ (u64_from_imm64 n)))))+(rule -2 (lower (ushr (ty_vec128 ty) x (iconst _ (u64_from_imm64 n))))(if-let 0 (shift_masked_imm ty n))x)@@ -1709,15 +1685,15 @@;;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Shift for i8/i16/i32.-(rule sshr_fits_in_32 -4 (lower (has_type (fits_in_32 ty) (sshr _ x y)))+(rule sshr_fits_in_32 -4 (lower (sshr (fits_in_32 ty) x y))(do_shift (ALUOp.Asr) ty (put_in_reg_sext32 x) y));; Shift for i64.-(rule sshr_64 (lower (has_type $I64 (sshr _ x y)))+(rule sshr_64 (lower (sshr $I64 x y))(do_shift (ALUOp.Asr) $I64 (put_in_reg_sext64 x) y));; Shift for i128.-(rule (lower (has_type $I128 (sshr _ x y)))+(rule (lower (sshr $I128 x y))(lower_sshr128 x (value_regs_get y 0)));; Vector shifts.@@ -1726,14 +1702,14 @@;; that for constant shifts a 0-width shift can't be emitted so it's special;; cased to pass through the input as-is since a 0-shift doesn't modify the;; input anyway.-(rule -3 (lower (has_type (ty_vec128 ty) (sshr _ x y)))+(rule -3 (lower (sshr (ty_vec128 ty) x y))(let ((size VectorSize (vector_size ty))(masked_shift_amt Reg (and_imm $I32 y (shift_mask ty)))(shift Reg (vec_dup (sub $I64 (zero_reg) masked_shift_amt) size)))(sshl x shift size)))-(rule -2 (lower (has_type (ty_vec128 ty) (sshr _ x (iconst _ (u64_from_imm64 n)))))+(rule -2 (lower (sshr (ty_vec128 ty) x (iconst _ (u64_from_imm64 n))))(sshr_vec_imm x (shift_masked_imm ty n) (vector_size ty)))-(rule -1 (lower (has_type (ty_vec128 ty) (sshr _ x (iconst _ (u64_from_imm64 n)))))+(rule -1 (lower (sshr (ty_vec128 ty) x (iconst _ (u64_from_imm64 n))))(if-let 0 (shift_masked_imm ty n))x)@@ -1769,13 +1745,13 @@;;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; General 8/16-bit case.-(rule rotl_fits_in_16 -2 (lower (has_type (fits_in_16 ty) (rotl _ x y)))+(rule rotl_fits_in_16 -2 (lower (rotl (fits_in_16 ty) x y))(let ((amt Reg (value_regs_get y 0))(neg_shift Reg (sub $I32 (zero_reg) amt)))(small_rotr ty (put_in_reg_zext32 x) neg_shift)));; Specialization for the 8/16-bit case when the rotation amount is an immediate.-(rule rotl_fits_in_16_imm -1 (lower (has_type (fits_in_16 ty) (rotl _ x (iconst _ k))))+(rule rotl_fits_in_16_imm -1 (lower (rotl (fits_in_16 ty) x (iconst _ k)))(if-let n (imm_shift_from_imm64 ty k))(small_rotr_imm ty (put_in_reg_zext32 x) (negate_imm_shift ty n)))@@ -1788,24 +1764,24 @@;; amount.;; General 32-bit case.-(rule rotl_32_base_case (lower (has_type $I32 (rotl _ x y)))+(rule rotl_32_base_case (lower (rotl $I32 x y))(let ((amt Reg (value_regs_get y 0))(neg_shift Reg (sub $I32 (zero_reg) amt)))(a64_rotr $I32 x neg_shift)));; General 64-bit case.-(rule rotl_64_base_case (lower (has_type $I64 (rotl _ x y)))+(rule rotl_64_base_case (lower (rotl $I64 x y))(let ((amt Reg (value_regs_get y 0))(neg_shift Reg (sub $I64 (zero_reg) amt)))(a64_rotr $I64 x neg_shift)));; Specialization for the 32-bit case when the rotation amount is an immediate.-(rule rotl_32_imm 1 (lower (has_type $I32 (rotl _ x (iconst _ k))))+(rule rotl_32_imm 1 (lower (rotl $I32 x (iconst _ k)))(if-let n (imm_shift_from_imm64 $I32 k))(a64_rotr_imm $I32 x (negate_imm_shift $I32 n)));; Specialization for the 64-bit case when the rotation amount is an immediate.-(rule rotl_64_imm 1 (lower (has_type $I64 (rotl _ x (iconst _ k))))+(rule rotl_64_imm 1 (lower (rotl $I64 x (iconst _ k)))(if-let n (imm_shift_from_imm64 $I64 k))(a64_rotr_imm $I64 x (negate_imm_shift $I64 n)))@@ -1824,7 +1800,7 @@;; General 128-bit case.;;;; TODO: much better codegen is possible with a constant amount.-(rule (lower (has_type $I128 (rotl _ x y)))+(rule (lower (rotl $I128 x y))(let ((val ValueRegs x)(amt Reg (value_regs_get y 0))(neg_amt Reg (sub $I64 (imm $I64 (ImmExtend.Zero) 128) amt))@@ -1837,29 +1813,29 @@;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; General 8/16-bit case.-(rule rotr_fits_in_16 -3 (lower (has_type (fits_in_16 ty) (rotr _ x y)))+(rule rotr_fits_in_16 -3 (lower (rotr (fits_in_16 ty) x y))(small_rotr ty (put_in_reg_zext32 x) (value_regs_get y 0)));; General 32-bit case.-(rule rotr_32_base_case -1 (lower (has_type $I32 (rotr _ x y)))+(rule rotr_32_base_case -1 (lower (rotr $I32 x y))(a64_rotr $I32 x (value_regs_get y 0)));; General 64-bit case.-(rule rotr_64_base_case -1 (lower (has_type $I64 (rotr _ x y)))+(rule rotr_64_base_case -1 (lower (rotr $I64 x y))(a64_rotr $I64 x (value_regs_get y 0)));; Specialization for the 8/16-bit case when the rotation amount is an immediate.-(rule rotr_fits_in_16_imm -2 (lower (has_type (fits_in_16 ty) (rotr _ x (iconst _ k))))+(rule rotr_fits_in_16_imm -2 (lower (rotr (fits_in_16 ty) x (iconst _ k)))(if-let n (imm_shift_from_imm64 ty k))(small_rotr_imm ty (put_in_reg_zext32 x) n));; Specialization for the 32-bit case when the rotation amount is an immediate.-(rule rotr_32_imm (lower (has_type $I32 (rotr _ x (iconst _ k))))+(rule rotr_32_imm (lower (rotr $I32 x (iconst _ k)))(if-let n (imm_shift_from_imm64 $I32 k))(a64_rotr_imm $I32 x n));; Specialization for the 64-bit case when the rotation amount is an immediate.-(rule rotr_64_imm (lower (has_type $I64 (rotr _ x (iconst _ k))))+(rule rotr_64_imm (lower (rotr $I64 x (iconst _ k)))(if-let n (imm_shift_from_imm64 $I64 k))(a64_rotr_imm $I64 x n))@@ -1940,7 +1916,7 @@;; General 128-bit case.;;;; TODO: much better codegen is possible with a constant amount.-(rule (lower (has_type $I128 (rotr _ x y)))+(rule (lower (rotr $I128 x y))(let ((val ValueRegs x)(amt Reg (value_regs_get y 0))(neg_amt Reg (sub $I64 (imm $I64 (ImmExtend.Zero) 128) amt))@@ -1955,37 +1931,37 @@;; Reversing an 8-bit value with a 32-bit bitrev instruction will place;; the reversed result in the highest 8 bits, so we need to shift them down into;; place.-(rule (lower (has_type $I8 (bitrev _ x)))+(rule (lower (bitrev $I8 x))(lsr_imm $I32 (rbit $I32 x) (imm_shift_from_u8 24)));; Reversing an 16-bit value with a 32-bit bitrev instruction will place;; the reversed result in the highest 16 bits, so we need to shift them down into;; place.-(rule (lower (has_type $I16 (bitrev _ x)))+(rule (lower (bitrev $I16 x))(lsr_imm $I32 (rbit $I32 x) (imm_shift_from_u8 16)))-(rule (lower (has_type $I128 (bitrev _ x)))+(rule (lower (bitrev $I128 x))(let ((val ValueRegs x)(lo_rev Reg (rbit $I64 (value_regs_get val 0)))(hi_rev Reg (rbit $I64 (value_regs_get val 1))))(value_regs hi_rev lo_rev)))-(rule -1 (lower (has_type ty (bitrev _ x)))+(rule -1 (lower (bitrev ty x))(rbit ty x));;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule clz_8 (lower (has_type $I8 (clz _ x)))+(rule clz_8 (lower (clz $I8 x))(sub_imm $I32 (a64_clz $I32 (put_in_reg_zext32 x)) (u8_into_imm12 24)))-(rule clz_16 (lower (has_type $I16 (clz _ x)))+(rule clz_16 (lower (clz $I16 x))(sub_imm $I32 (a64_clz $I32 (put_in_reg_zext32 x)) (u8_into_imm12 16)))-(rule (lower (has_type $I128 (clz _ x)))+(rule (lower (clz $I128 x))(lower_clz128 x))-(rule clz_32_64 -1 (lower (has_type ty (clz _ x)))+(rule clz_32_64 -1 (lower (clz ty x))(a64_clz ty x));; clz hi_clz, hi@@ -2006,27 +1982,27 @@;; then using a `clz` instruction since the tail zeros are the same as the;; leading zeros of the reversed value.-(rule ctz_8 (lower (has_type $I8 (ctz _ x)))+(rule ctz_8 (lower (ctz $I8 x))(a64_clz $I32 (orr_imm $I32 (rbit $I32 x) (u64_into_imm_logic $I32 0x800000))))-(rule ctz_16 (lower (has_type $I16 (ctz _ x)))+(rule ctz_16 (lower (ctz $I16 x))(a64_clz $I32 (orr_imm $I32 (rbit $I32 x) (u64_into_imm_logic $I32 0x8000))))-(rule (lower (has_type $I128 (ctz _ x)))+(rule (lower (ctz $I128 x))(let ((val ValueRegs x)(lo Reg (rbit $I64 (value_regs_get val 0)))(hi Reg (rbit $I64 (value_regs_get val 1))))(lower_clz128 (value_regs hi lo))))-(rule ctz_32_64 -1 (lower (has_type ty (ctz _ x)))+(rule ctz_32_64 -1 (lower (ctz ty x))(a64_clz ty (rbit ty x)));;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule cls_8 (lower (has_type $I8 (cls _ x)))+(rule cls_8 (lower (cls $I8 x))(sub_imm $I32 (a64_cls $I32 (put_in_reg_sext32 x)) (u8_into_imm12 24)))-(rule cls_16 (lower (has_type $I16 (cls _ x)))+(rule cls_16 (lower (cls $I16 x))(sub_imm $I32 (a64_cls $I32 (put_in_reg_sext32 x)) (u8_into_imm12 16)));; cls lo_cls, lo@@ -2038,7 +2014,7 @@;; csel maybe_lo, lo_sign_bits, xzr, eq;; add out_lo, maybe_lo, hi_cls;; mov out_hi, 0-(rule (lower (has_type $I128 (cls _ x)))+(rule (lower (cls $I128 x))(let ((val ValueRegs x)(lo Reg (value_regs_get val 0))(hi Reg (value_regs_get val 1))@@ -2052,21 +2028,21 @@(csel (Cond.Eq) lo_sign_bits (zero_reg)))))(value_regs (add $I64 maybe_lo hi_cls) (imm $I64 (ImmExtend.Zero) 0))))-(rule cls_32_64 -1 (lower (has_type ty (cls _ x)))+(rule cls_32_64 -1 (lower (cls ty x))(a64_cls ty x));;;; Rules for `bswap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16 (bswap _ x)))+(rule (lower (bswap $I16 x))(a64_rev16 $I16 x))-(rule (lower (has_type $I32 (bswap _ x)))+(rule (lower (bswap $I32 x))(a64_rev32 $I32 x))-(rule (lower (has_type $I64 (bswap _ x)))+(rule (lower (bswap $I64 x))(a64_rev64 $I64 x))-(rule (lower (has_type $I128 (bswap _ x)))+(rule (lower (bswap $I128 x))(value_regs(a64_rev64 $I64 (value_regs_get x 1))(a64_rev64 $I64 (value_regs_get x 0))))@@ -2074,7 +2050,7 @@;;;; Rules for `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Bmask tests the value against zero, and uses `csetm` to assert the result.-(rule (lower (has_type out_ty (bmask _ x @ (value_type in_ty))))+(rule (lower (bmask out_ty x @ (value_type in_ty)))(lower_bmask out_ty in_ty x));;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2096,31 +2072,31 @@;; if ty == i128:;; mov out_hi, 0-(rule popcnt_8 (lower (has_type $I8 (popcnt _ x)))+(rule popcnt_8 (lower (popcnt $I8 x))(let ((tmp Reg (mov_to_fpu x (ScalarSize.Size32)))(nbits Reg (vec_cnt tmp (VectorSize.Size8x8))))(mov_from_vec nbits 0 (ScalarSize.Size8))));; Note that this uses `addp` instead of `addv` as it's usually cheaper.-(rule popcnt_16 (lower (has_type $I16 (popcnt _ x)))+(rule popcnt_16 (lower (popcnt $I16 x))(let ((tmp Reg (mov_to_fpu x (ScalarSize.Size32)))(nbits Reg (vec_cnt tmp (VectorSize.Size8x8)))(added Reg (addp nbits nbits (VectorSize.Size8x8))))(mov_from_vec added 0 (ScalarSize.Size8))))-(rule popcnt_32 (lower (has_type $I32 (popcnt _ x)))+(rule popcnt_32 (lower (popcnt $I32 x))(let ((tmp Reg (mov_to_fpu x (ScalarSize.Size32)))(nbits Reg (vec_cnt tmp (VectorSize.Size8x8)))(added Reg (addv nbits (VectorSize.Size8x8))))(mov_from_vec added 0 (ScalarSize.Size8))))-(rule popcnt_64 (lower (has_type $I64 (popcnt _ x)))+(rule popcnt_64 (lower (popcnt $I64 x))(let ((tmp Reg (mov_to_fpu x (ScalarSize.Size64)))(nbits Reg (vec_cnt tmp (VectorSize.Size8x8)))(added Reg (addv nbits (VectorSize.Size8x8))))(mov_from_vec added 0 (ScalarSize.Size8))))-(rule (lower (has_type $I128 (popcnt _ x)))+(rule (lower (popcnt $I128 x))(let ((val ValueRegs x)(tmp_half Reg (mov_to_fpu (value_regs_get val 0) (ScalarSize.Size64)))(tmp Reg (mov_to_vec tmp_half (value_regs_get val 1) 1 (VectorSize.Size64x2)))@@ -2128,18 +2104,18 @@(added Reg (addv nbits (VectorSize.Size8x16))))(value_regs (mov_from_vec added 0 (ScalarSize.Size8)) (imm $I64 (ImmExtend.Zero) 0))))-(rule (lower (has_type $I8X16 (popcnt _ x)))+(rule (lower (popcnt $I8X16 x))(vec_cnt x (VectorSize.Size8x16)));;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule bitselect (lower (has_type ty (bitselect _ c x y)))+(rule bitselect (lower (bitselect ty c x y))(if (ty_int_ref_scalar_64 ty))(let ((tmp1 Reg (and_reg ty x c))(tmp2 Reg (bic ty y c)))(orr ty tmp1 tmp2)))-(rule 1 (lower (has_type (ty_vec128 ty) (bitselect _ c x y)))+(rule 1 (lower (bitselect (ty_vec128 ty) c x y))(bsl ty c x y));;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2147,31 +2123,31 @@;; T -> I{64,32,16,8}: We can simply pass through the value: values;; are always stored with high bits undefined, so we can just leave;; them be.-(rule (lower (has_type ty (ireduce _ src)))+(rule (lower (ireduce ty src))(if (ty_int_ref_scalar_64 ty))(value_regs_get src 0));;;; Rules for `fcmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 4 (lower (has_type ty @ (multi_lane _ _) (fcmp _ (fcmp_zero_cond_not_eq cond) x y)))+(rule 4 (lower (fcmp ty @ (multi_lane _ _) (fcmp_zero_cond_not_eq cond) x y))(if (zero_value y))(let ((rn Reg x)(vec_size VectorSize (vector_size ty)))(value_reg (not (fcmeq0 rn vec_size) vec_size))))-(rule 3 (lower (has_type ty @ (multi_lane _ _) (fcmp _ (fcmp_zero_cond cond) x y)))+(rule 3 (lower (fcmp ty @ (multi_lane _ _) (fcmp_zero_cond cond) x y))(if (zero_value y))(let ((rn Reg x)(vec_size VectorSize (vector_size ty)))(value_reg (float_cmp_zero cond rn vec_size))))-(rule 2 (lower (has_type ty @ (multi_lane _ _) (fcmp _ (fcmp_zero_cond_not_eq cond) x y)))+(rule 2 (lower (fcmp ty @ (multi_lane _ _) (fcmp_zero_cond_not_eq cond) x y))(if (zero_value x))(let ((rn Reg y)(vec_size VectorSize (vector_size ty)))(value_reg (not (fcmeq0 rn vec_size) vec_size))))-(rule 1 (lower (has_type ty @ (multi_lane _ _) (fcmp _ (fcmp_zero_cond cond) x y)))+(rule 1 (lower (fcmp ty @ (multi_lane _ _) (fcmp_zero_cond cond) x y))(if (zero_value x))(let ((rn Reg y)(vec_size VectorSize (vector_size ty)))@@ -2186,25 +2162,25 @@;;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 3 (lower (has_type ty @ (multi_lane _ _) (icmp _ (icmp_zero_cond_not_eq cond) x y)))+(rule 3 (lower (icmp ty @ (multi_lane _ _) (icmp_zero_cond_not_eq cond) x y))(if (zero_value y))(let ((rn Reg x)(vec_size VectorSize (vector_size ty)))(value_reg (not (cmeq0 rn vec_size) vec_size))))-(rule 2 (lower (has_type ty @ (multi_lane _ _) (icmp _ (icmp_zero_cond cond) x y)))+(rule 2 (lower (icmp ty @ (multi_lane _ _) (icmp_zero_cond cond) x y))(if (zero_value y))(let ((rn Reg x)(vec_size VectorSize (vector_size ty)))(value_reg (int_cmp_zero cond rn vec_size))))-(rule 1 (lower (has_type ty @ (multi_lane _ _) (icmp _ (icmp_zero_cond_not_eq cond) x y)))+(rule 1 (lower (icmp ty @ (multi_lane _ _) (icmp_zero_cond_not_eq cond) x y))(if (zero_value x))(let ((rn Reg y)(vec_size VectorSize (vector_size ty)))(value_reg (not (cmeq0 rn vec_size) vec_size))))-(rule 0 (lower (has_type ty @ (multi_lane _ _) (icmp _ (icmp_zero_cond cond) x y)))+(rule 0 (lower (icmp ty @ (multi_lane _ _) (icmp_zero_cond cond) x y))(if (zero_value x))(let ((rn Reg y)(vec_size VectorSize (vector_size ty)))@@ -2284,38 +2260,38 @@;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 _) (vconst _ (u128_from_constant x))))+(rule (lower (vconst (ty_vec128 _) (u128_from_constant x)))(constant_f128 x))-(rule 1 (lower (has_type ty (vconst _ (u64_from_constant x))))+(rule 1 (lower (vconst ty (u64_from_constant x)))(if (ty_vec64 ty))(constant_f64 x));;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty (splat _ x @ (value_type in_ty))))+(rule -1 (lower (splat ty x @ (value_type in_ty)))(if (ty_int_ref_scalar_64 in_ty))(vec_dup x (vector_size ty)))-(rule -2 (lower (has_type ty (splat _ x @ (value_type (ty_scalar_float _)))))+(rule -2 (lower (splat ty x @ (value_type (ty_scalar_float _))))(vec_dup_from_fpu x (vector_size ty) 0))-(rule (lower (has_type ty (splat _ (f32const _ (u32_from_ieee32 n)))))+(rule (lower (splat ty (f32const _ (u32_from_ieee32 n))))(splat_const n (vector_size ty)))-(rule (lower (has_type ty (splat _ (f64const _ (u64_from_ieee64 n)))))+(rule (lower (splat ty (f64const _ (u64_from_ieee64 n))))(splat_const n (vector_size ty)))-(rule (lower (has_type ty (splat _ (iconst _ (u64_from_imm64 n)))))+(rule (lower (splat ty (iconst _ (u64_from_imm64 n))))(splat_const n (vector_size ty)))-(rule (lower (has_type ty (splat _ x @ (load _ (mem_flags_data flags) _ _))))+(rule (lower (splat ty x @ (load _ (mem_flags_data flags) _ _)))(if-let mem_op (is_sinkable_inst x))(let ((addr Reg (sink_load_into_addr (lane_type ty) mem_op)))(ld1r addr (vector_size ty) flags)));;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (valid_atomic_transaction ty) (atomic_load _ (little_or_native_endian flags) addr)))+(rule (lower (atomic_load (valid_atomic_transaction ty) (little_or_native_endian flags) addr))(load_acquire ty flags addr))@@ -2328,85 +2304,63 @@;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Add) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Add) addr src)))(lse_atomic_rmw (AtomicRMWOp.Add) addr src ty flags))(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src)))(lse_atomic_rmw (AtomicRMWOp.Eor) addr src ty flags))(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Or) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Or) addr src)))(lse_atomic_rmw (AtomicRMWOp.Set) addr src ty flags))(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src)))(lse_atomic_rmw (AtomicRMWOp.Smax) addr src ty flags))(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src)))(lse_atomic_rmw (AtomicRMWOp.Smin) addr src ty flags))(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src)))(lse_atomic_rmw (AtomicRMWOp.Umax) addr src ty flags))(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src)))(lse_atomic_rmw (AtomicRMWOp.Umin) addr src ty flags))(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src)))(lse_atomic_rmw (AtomicRMWOp.Add) addr (sub ty (zero_reg) src) ty flags))(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.And) addr src))))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.And) addr src)))(lse_atomic_rmw (AtomicRMWOp.Clr) addr (eon ty src (zero_reg)) ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Add) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Add) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Add) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Sub) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.And) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.And) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.And) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Nand) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Nand) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Nand) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Or) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Or) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Orr) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Eor) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Smin) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Smax) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Umin) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Umax) addr src ty flags))-(rule (lower (has_type (valid_atomic_transaction ty)- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xchg) addr src)))+(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Xchg) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Xchg) addr src ty flags));;;; Rules for `AtomicCAS` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule 1 (lower (and (use_lse)- (has_type (valid_atomic_transaction ty)- (atomic_cas _ (little_or_native_endian flags) addr src1 src2))))+ (atomic_cas (valid_atomic_transaction ty) (little_or_native_endian flags) addr src1 src2)))(lse_atomic_cas addr src1 src2 ty flags))-(rule (lower (and (has_type (valid_atomic_transaction ty)- (atomic_cas _ (little_or_native_endian flags) addr src1 src2))))+(rule (lower (and (atomic_cas (valid_atomic_transaction ty) (little_or_native_endian flags) addr src1 src2)))(atomic_cas_loop addr src1 src2 ty flags));;;; Rules for 'fvdemote' ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2415,30 +2369,30 @@;;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128_int ty) (snarrow _ x y)))+(rule 1 (lower (snarrow (ty_vec128_int ty) x y))(if (zero_value y))(sqxtn x (lane_size ty)))-(rule 2 (lower (has_type (ty_vec64_int ty) (snarrow _ x y)))+(rule 2 (lower (snarrow (ty_vec64_int ty) x y))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(sqxtn dst (lane_size ty))))-(rule 0 (lower (has_type (ty_vec128_int ty) (snarrow _ x y)))+(rule 0 (lower (snarrow (ty_vec128_int ty) x y))(let ((low_half Reg (sqxtn x (lane_size ty)))(result Reg (sqxtn2 low_half y (lane_size ty))))result));;;; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128_int ty) (unarrow _ x y)))+(rule 1 (lower (unarrow (ty_vec128_int ty) x y))(if (zero_value y))(sqxtun x (lane_size ty)))-(rule 2 (lower (has_type (ty_vec64_int ty) (unarrow _ x y)))+(rule 2 (lower (unarrow (ty_vec64_int ty) x y))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(sqxtun dst (lane_size ty))))-(rule 0 (lower (has_type (ty_vec128_int ty) (unarrow _ x y)))+(rule 0 (lower (unarrow (ty_vec128_int ty) x y))(let ((low_half Reg (sqxtun x (lane_size ty)))(result Reg (sqxtun2 low_half y (lane_size ty))))result))@@ -2446,45 +2400,45 @@;;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128_int ty) (uunarrow _ x y)))+(rule 1 (lower (uunarrow (ty_vec128_int ty) x y))(if (zero_value y))(uqxtn x (lane_size ty)))-(rule 2 (lower (has_type (ty_vec64_int ty) (uunarrow _ x y)))+(rule 2 (lower (uunarrow (ty_vec64_int ty) x y))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(uqxtn dst (lane_size ty))))-(rule 0 (lower (has_type (ty_vec128_int ty) (uunarrow _ x y)))+(rule 0 (lower (uunarrow (ty_vec128_int ty) x y))(let ((low_half Reg (uqxtn x (lane_size ty)))(result Reg (uqxtn2 low_half y (lane_size ty))))result));;;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (swiden_low _ x)))+(rule (lower (swiden_low ty x))(vec_extend (VecExtendOp.Sxtl) x false (lane_size ty)));;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128 ty) (swiden_high _ x)))+(rule 1 (lower (swiden_high (ty_vec128 ty) x))(vec_extend (VecExtendOp.Sxtl) x true (lane_size ty)))-(rule (lower (has_type ty (swiden_high _ x)))+(rule (lower (swiden_high ty x))(if (ty_vec64 ty))(let ((tmp Reg (fpu_move_from_vec x 1 (VectorSize.Size32x2))))(vec_extend (VecExtendOp.Sxtl) tmp false (lane_size ty))));;;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (uwiden_low _ x)))+(rule (lower (uwiden_low ty x))(vec_extend (VecExtendOp.Uxtl) x false (lane_size ty)));;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (ty_vec128 ty) (uwiden_high _ x)))+(rule 1 (lower (uwiden_high (ty_vec128 ty) x))(vec_extend (VecExtendOp.Uxtl) x true (lane_size ty)))-(rule (lower (has_type ty (uwiden_high _ x)))+(rule (lower (uwiden_high ty x))(if (ty_vec64 ty))(let ((tmp Reg (fpu_move_from_vec x 1 (VectorSize.Size32x2))))(vec_extend (VecExtendOp.Uxtl) tmp false (lane_size ty))))@@ -2620,39 +2574,37 @@;;;; Rules for loads ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule load_i8_aarch64_uload8 (lower- (has_type $I8 (load _ (little_or_native_endian flags) address offset)))+ (load $I8 (little_or_native_endian flags) address offset))(aarch64_uload8 (amode $I8 address offset) flags))(rule load_i16_aarch64_uload16 (lower- (has_type $I16 (load _ (little_or_native_endian flags) address offset)))+ (load $I16 (little_or_native_endian flags) address offset))(aarch64_uload16 (amode $I16 address offset) flags))(rule load_i32_aarch64_uload32 (lower- (has_type $I32 (load _ (little_or_native_endian flags) address offset)))+ (load $I32 (little_or_native_endian flags) address offset))(aarch64_uload32 (amode $I32 address offset) flags))(rule load_i64_aarch64_uload64 (lower- (has_type $I64 (load _ (little_or_native_endian flags) address offset)))+ (load $I64 (little_or_native_endian flags) address offset))(aarch64_uload64 (amode $I64 address offset) flags))(rule (lower- (has_type $I128 (load _ (little_or_native_endian flags) address offset)))+ (load $I128 (little_or_native_endian flags) address offset))(aarch64_loadp64 (pair_amode address offset) flags))(rule -1 (lower- (has_type (ty_float_or_vec (ty_16 _)) (load _ (little_or_native_endian flags) address offset)))+ (load (ty_float_or_vec (ty_16 _)) (little_or_native_endian flags) address offset))(aarch64_fpuload16 (amode $F16 address offset) flags))(rule -2 (lower- (has_type (ty_float_or_vec (ty_32 _)) (load _ (little_or_native_endian flags) address offset)))+ (load (ty_float_or_vec (ty_32 _)) (little_or_native_endian flags) address offset))(aarch64_fpuload32 (amode $F32 address offset) flags))(rule -3 (lower- (has_type (ty_float_or_vec (ty_64 _)) (load _ (little_or_native_endian flags) address offset)))+ (load (ty_float_or_vec (ty_64 _)) (little_or_native_endian flags) address offset))(aarch64_fpuload64 (amode $F64 address offset) flags))(rule -4 (lower- (has_type (ty_float_or_vec (ty_128 _)) (load _ (little_or_native_endian flags) address offset)))+ (load (ty_float_or_vec (ty_128 _)) (little_or_native_endian flags) address offset))(aarch64_fpuload128 (amode $F128 address offset) flags))(rule -5 (lower- (has_type (ty_dyn_vec64 _)- (load _ (little_or_native_endian flags) address offset)))+ (load (ty_dyn_vec64 _) (little_or_native_endian flags) address offset))(aarch64_fpuload64 (amode $F64 address offset) flags))(rule -6 (lower- (has_type (ty_dyn_vec128 _)- (load _ (little_or_native_endian flags) address offset)))+ (load (ty_dyn_vec128 _) (little_or_native_endian flags) address offset))(aarch64_fpuload128 (amode $I8X16 address offset) flags))(rule (lower@@ -2787,49 +2739,47 @@;;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; SIMD&FP <=> SIMD&FP-(rule 7 (lower (has_type (ty_float_or_vec _) (bitcast _ _ x @ (value_type (ty_float_or_vec _)))))+(rule 7 (lower (bitcast (ty_float_or_vec _) _ x @ (value_type (ty_float_or_vec _))))x); I128 => SIMD&FP-(rule 6 (lower (has_type (ty_float_or_vec _) (bitcast _ _ x @ (value_type $I128))))+(rule 6 (lower (bitcast (ty_float_or_vec _) _ x @ (value_type $I128)))(mov_to_vec (mov_to_fpu (value_regs_get x 0) (ScalarSize.Size64)) (value_regs_get x 1) 1 (VectorSize.Size64x2))); SIMD&FP => I128-(rule 5 (lower (has_type $I128 (bitcast _ _ x @ (value_type (ty_float_or_vec _)))))+(rule 5 (lower (bitcast $I128 _ x @ (value_type (ty_float_or_vec _))))(value_regs (mov_from_vec x 0 (ScalarSize.Size64)) (mov_from_vec x 1 (ScalarSize.Size64)))); GPR => SIMD&FP-(rule 4 (lower (has_type (ty_float_or_vec _) (bitcast _ _ x @ (value_type in_ty))))+(rule 4 (lower (bitcast (ty_float_or_vec _) _ x @ (value_type in_ty)))(if (ty_int_ref_scalar_64 in_ty))(mov_to_fpu x (scalar_size in_ty))); SIMD&FP => GPR-(rule 3 (lower (has_type out_ty (bitcast _ _ x @ (value_type (fits_in_64 (ty_float_or_vec _))))))+(rule 3 (lower (bitcast out_ty _ x @ (value_type (fits_in_64 (ty_float_or_vec _)))))(if (ty_int_ref_scalar_64 out_ty))(mov_from_vec x 0 (scalar_size out_ty))); GPR <=> GPR-(rule 1 (lower (has_type out_ty (bitcast _ _ x @ (value_type in_ty))))+(rule 1 (lower (bitcast out_ty _ x @ (value_type in_ty)))(if (ty_int_ref_scalar_64 out_ty))(if (ty_int_ref_scalar_64 in_ty))x)-(rule 0 (lower (has_type $I128 (bitcast _ _ x @ (value_type $I128)))) x)+(rule 0 (lower (bitcast $I128 _ x @ (value_type $I128))) x);;; Rules for `extractlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; extractlane with lane 0 can pass through the value unchanged; upper;; bits are undefined when a narrower type is in a wider register.-(rule 2 (lower (has_type (ty_scalar_float _) (extractlane _ val (u8_from_uimm8 0))))+(rule 2 (lower (extractlane (ty_scalar_float _) val (u8_from_uimm8 0)))val)-(rule 0 (lower (has_type (ty_int ty)- (extractlane _ val- (u8_from_uimm8 lane))))+(rule 0 (lower (extractlane (ty_int ty) val+ (u8_from_uimm8 lane)))(mov_from_vec val lane (scalar_size ty)))-(rule 1 (lower (has_type (ty_scalar_float ty)- (extractlane _ val @ (value_type vty)- (u8_from_uimm8 lane))))+(rule 1 (lower (extractlane (ty_scalar_float ty) val @ (value_type vty)+ (u8_from_uimm8 lane)))(fpu_move_from_vec val lane (vector_size vty)));;; Rules for `insertlane` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2924,7 +2874,7 @@;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap _ a b tc)))+(rule (lower (uadd_overflow_trap (fits_in_64 ty) a b tc))(trap_if_overflow (add_with_flags_paired ty a b) tc));;;; Helpers for `*_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3004,15 +2954,15 @@;; For values smaller than a register, we do a normal `add` with both arguments;; zero extended. We then check if the output is the same as itself zero extended.-(rule 1 (lower (has_type (fits_in_16 ty) (uadd_overflow _ a b)))+(rule 1 (lower (uadd_overflow (fits_in_16 ty) a b))(overflow_op_small ty a b (ArgumentExtension.Uext) (ALUOp.Add)));; For register sized add's we just emit a adds+cset, without further masking.-(rule 2 (lower (has_type (ty_32_or_64 ty) (uadd_overflow _ a b)))+(rule 2 (lower (uadd_overflow (ty_32_or_64 ty) a b))(overflow_op_normal ty a b (ALUOp.AddS) (Cond.Hs)));; For 128bit integers we emit add+adcs+cset-(rule 0 (lower (has_type $I128 (uadd_overflow _ x y)))+(rule 0 (lower (uadd_overflow $I128 x y))(overflow_op_128 x y (ALUOp.AddS) (ALUOp.AdcS) (Cond.Hs)));;;; Rules for `sadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3021,18 +2971,18 @@;; add out, a_ext, b, sxt{b,h};; cmp out, out, sxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (sadd_overflow _ a b)))+(rule 1 (lower (sadd_overflow (fits_in_16 ty) a b))(overflow_op_small ty a b (ArgumentExtension.Sext) (ALUOp.Add)));; adds a, b;; cset of, vs-(rule 2 (lower (has_type (ty_32_or_64 ty) (sadd_overflow _ a b)))+(rule 2 (lower (sadd_overflow (ty_32_or_64 ty) a b))(overflow_op_normal ty a b (ALUOp.AddS) (Cond.Vs)));; adds x_lo, y_lo;; addcs x_hi, y_hi;; cset of, vs-(rule 0 (lower (has_type $I128 (sadd_overflow _ x y)))+(rule 0 (lower (sadd_overflow $I128 x y))(overflow_op_128 x y (ALUOp.AddS) (ALUOp.AdcS) (Cond.Vs)));;;; Rules for `usub_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3041,18 +2991,18 @@;; sub out, a_ext, b, ext{b,h};; cmp out, out, uxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (usub_overflow _ a b)))+(rule 1 (lower (usub_overflow (fits_in_16 ty) a b))(overflow_op_small ty a b (ArgumentExtension.Uext) (ALUOp.Sub)));; subs a, b;; cset of, lo-(rule 2 (lower (has_type (ty_32_or_64 ty) (usub_overflow _ a b)))+(rule 2 (lower (usub_overflow (ty_32_or_64 ty) a b))(overflow_op_normal ty a b (ALUOp.SubS) (Cond.Lo)));; subs x_lo, y_lo;; sbcs x_hi, y_hi;; cset of, lo-(rule 0 (lower (has_type $I128 (usub_overflow _ x y)))+(rule 0 (lower (usub_overflow $I128 x y))(overflow_op_128 x y (ALUOp.SubS) (ALUOp.SbcS) (Cond.Lo)));;;; Rules for `ssub_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3061,18 +3011,18 @@;; sub out, a_ext, b, sxt{b,h};; cmp out, out, sxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (ssub_overflow _ a b)))+(rule 1 (lower (ssub_overflow (fits_in_16 ty) a b))(overflow_op_small ty a b (ArgumentExtension.Sext) (ALUOp.Sub)));; subs a, b;; cset of, vs-(rule 2 (lower (has_type (ty_32_or_64 ty) (ssub_overflow _ a b)))+(rule 2 (lower (ssub_overflow (ty_32_or_64 ty) a b))(overflow_op_normal ty a b (ALUOp.SubS) (Cond.Vs)));; subs x_lo, y_lo;; sbcs x_hi, y_hi;; cset of, vs-(rule 0 (lower (has_type $I128 (ssub_overflow _ x y)))+(rule 0 (lower (ssub_overflow $I128 x y))(overflow_op_128 x y (ALUOp.SubS) (ALUOp.SbcS) (Cond.Vs)));;;; Rules for `umul_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3082,7 +3032,7 @@;; mul out, a_ext, b_ext;; cmp out, out, uxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (umul_overflow _ a b)))+(rule 1 (lower (umul_overflow (fits_in_16 ty) a b))(let ((extend ExtendOp (lower_extend_op ty (ArgumentExtension.Uext)))(a_uext Reg (put_in_reg_zext32 a))@@ -3098,7 +3048,7 @@;; umull out, a, b;; cmp out, out, uxtw;; cset of, ne-(rule 2 (lower (has_type $I32 (umul_overflow _ a b)))+(rule 2 (lower (umul_overflow $I32 a b))(let ((out Reg (umaddl a b (zero_reg)))(out_of Reg (with_flags_reg@@ -3112,7 +3062,7 @@;; umulh tmp, a, b;; cmp tmp, #0;; cset of, ne-(rule 2 (lower (has_type $I64 (umul_overflow _ a b)))+(rule 2 (lower (umul_overflow $I64 a b))(let ((out Reg (madd $I64 a b (zero_reg)))(tmp Reg (umulh $I64 a b))@@ -3130,7 +3080,7 @@;; mul out, a_ext, b_ext;; cmp out, out, sxt{b,h};; cset of, ne-(rule 1 (lower (has_type (fits_in_16 ty) (smul_overflow _ a b)))+(rule 1 (lower (smul_overflow (fits_in_16 ty) a b))(let ((extend ExtendOp (lower_extend_op ty (ArgumentExtension.Sext)))(a_sext Reg (put_in_reg_sext32 a))@@ -3146,7 +3096,7 @@;; smull out, a, b;; cmp out, out, sxtw;; cset of, ne-(rule 2 (lower (has_type $I32 (smul_overflow _ a b)))+(rule 2 (lower (smul_overflow $I32 a b))(let ((out Reg (smaddl a b (zero_reg)))(out_of Reg (with_flags_reg@@ -3160,7 +3110,7 @@;; smulh tmp, a, b;; cmp tmp, out, ASR #63;; cset of, ne-(rule 2 (lower (has_type $I64 (smul_overflow _ a b)))+(rule 2 (lower (smul_overflow $I64 a b))(let ((out Reg (madd $I64 a b (zero_reg)))(tmp Reg (smulh $I64 a b))@@ -3173,10 +3123,10 @@;;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value _ (symbol_value_data name _ _))))+(rule (lower (tls_value (tls_model (TlsModel.ElfGd)) (symbol_value_data name _ _)))(elf_tls_get_addr name))-(rule (lower (has_type (tls_model (TlsModel.Macho)) (tls_value _ (symbol_value_data name _ _))))+(rule (lower (tls_value (tls_model (TlsModel.Macho)) (symbol_value_data name _ _)))(macho_tls_get_addr name));;; Rules for `fvpromote_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/codegen/src/isa/aarch64/lower_dynamic_neon.isle b/cranelift/codegen/src/isa/aarch64/lower_dynamic_neon.isleindex 8c9c21ed42..8c85246fd9 100644--- a/cranelift/codegen/src/isa/aarch64/lower_dynamic_neon.isle+++ b/cranelift/codegen/src/isa/aarch64/lower_dynamic_neon.isle@@ -1,78 +1,78 @@;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -4 (lower (has_type ty @ (dynamic_lane _ _) (iadd _ x y)))+(rule -4 (lower (iadd ty @ (dynamic_lane _ _) x y))(value_reg (add_vec (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -5 (lower (has_type ty @ (dynamic_lane _ _) (isub _ x y)))+(rule -5 (lower (isub ty @ (dynamic_lane _ _) x y))(value_reg (sub_vec (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -4 (lower (has_type (lane_fits_in_32 ty @ (dynamic_lane _ _)) (imul _ x y)))+(rule -4 (lower (imul (lane_fits_in_32 ty @ (dynamic_lane _ _)) x y))(value_reg (vec_rrr (VecALUOp.Mul) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fadd _ x y)))+(rule -2 (lower (fadd ty @ (dynamic_lane _ _) x y))(value_reg (vec_rrr (VecALUOp.Fadd) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fsub _ x y)))+(rule -2 (lower (fsub ty @ (dynamic_lane _ _) x y))(value_reg (vec_rrr (VecALUOp.Fsub) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmul _ x y)))+(rule -2 (lower (fmul ty @ (dynamic_lane _ _) x y))(value_reg (vec_rrr (VecALUOp.Fmul) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fdiv _ x y)))+(rule -2 (lower (fdiv ty @ (dynamic_lane _ _) x y))(value_reg (vec_rrr (VecALUOp.Fdiv) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmin _ x y)))+(rule -2 (lower (fmin ty @ (dynamic_lane _ _) x y))(value_reg (vec_rrr (VecALUOp.Fmin) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type ty @ (dynamic_lane _ _) (fmax _ x y)))+(rule -2 (lower (fmax ty @ (dynamic_lane _ _) x y))(value_reg (vec_rrr (VecALUOp.Fmax) (put_in_reg x) (put_in_reg y) (vector_size ty))));;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type (ty_dyn128_int ty) (snarrow _ x y)))+(rule -2 (lower (snarrow (ty_dyn128_int ty) x y))(if-let _ (zero_value y))(sqxtn x (lane_size ty)))-(rule -1 (lower (has_type (ty_dyn64_int ty) (snarrow _ x y)))+(rule -1 (lower (snarrow (ty_dyn64_int ty) x y))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(sqxtn dst (lane_size ty))))-(rule -3 (lower (has_type (ty_dyn128_int ty) (snarrow _ x y)))+(rule -3 (lower (snarrow (ty_dyn128_int ty) x y))(let ((low_half Reg (sqxtn x (lane_size ty)))(result Reg (sqxtn2 low_half y (lane_size ty))))result));;;; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type (ty_dyn128_int ty) (unarrow _ x y)))+(rule -2 (lower (unarrow (ty_dyn128_int ty) x y))(if-let _ (zero_value y))(sqxtun x (lane_size ty)))-(rule -1 (lower (has_type (ty_dyn64_int ty) (unarrow _ x y)))+(rule -1 (lower (unarrow (ty_dyn64_int ty) x y))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(sqxtun dst (lane_size ty))))-(rule -3 (lower (has_type (ty_dyn128_int ty) (unarrow _ x y)))+(rule -3 (lower (unarrow (ty_dyn128_int ty) x y))(let ((low_half Reg (sqxtun x (lane_size ty)))(result Reg (sqxtun2 low_half y (lane_size ty))))result));;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -2 (lower (has_type (ty_dyn128_int ty) (uunarrow _ x y)))+(rule -2 (lower (uunarrow (ty_dyn128_int ty) x y))(if-let _ (zero_value y))(uqxtn x (lane_size ty)))-(rule -1 (lower (has_type (ty_dyn64_int ty) (uunarrow _ x y)))+(rule -1 (lower (uunarrow (ty_dyn64_int ty) x y))(let ((dst Reg (mov_vec_elem x y 1 0 (VectorSize.Size64x2))))(uqxtn dst (lane_size ty))))-(rule -3 (lower (has_type (ty_dyn128_int ty) (uunarrow _ x y)))+(rule -3 (lower (uunarrow (ty_dyn128_int ty) x y))(let ((low_half Reg (uqxtn x (lane_size ty)))(result Reg (uqxtn2 low_half y (lane_size ty))))result))@@ -89,10 +89,10 @@;;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty (swiden_high _ x)))+(rule -1 (lower (swiden_high ty x))(vec_extend (VecExtendOp.Sxtl) x true (lane_size ty)));;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type ty (uwiden_high _ x)))+(rule -1 (lower (uwiden_high ty x))(vec_extend (VecExtendOp.Uxtl) x true (lane_size ty)))diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isleindex b907eb9d35..dcd583798d 100644--- a/cranelift/codegen/src/isa/pulley_shared/lower.isle+++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle@@ -232,7 +232,7 @@;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (iconst _ c))) (imm (i64_sextend_imm64 ty c)))+(rule (lower (iconst ty c)) (imm (i64_sextend_imm64 ty c)));;;; Rules for `f32const`;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -244,31 +244,31 @@;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_64 ty)) (iadd _ a b))) (pulley_xadd32 a b))-(rule 1 (lower (has_type $I64 (iadd _ a b))) (pulley_xadd64 a b))+(rule 0 (lower (iadd (ty_int (fits_in_64 ty)) a b)) (pulley_xadd32 a b))+(rule 1 (lower (iadd $I64 a b)) (pulley_xadd64 a b));; Fold constants into the instruction if possible-(rule 10 (lower (has_type (ty_int (fits_in_32 _)) (iadd _ a (u32_from_iconst b))))+(rule 10 (lower (iadd (ty_int (fits_in_32 _)) a (u32_from_iconst b)))(pulley_xadd32_u32 a b))-(rule 11 (lower (has_type (ty_int (fits_in_32 _)) (iadd _ a (u8_from_iconst b))))+(rule 11 (lower (iadd (ty_int (fits_in_32 _)) a (u8_from_iconst b)))(pulley_xadd32_u8 a b))-(rule 12 (lower (has_type $I64 (iadd _ a (u32_from_iconst b))))+(rule 12 (lower (iadd $I64 a (u32_from_iconst b)))(pulley_xadd64_u32 a b))-(rule 13 (lower (has_type $I64 (iadd _ a (u8_from_iconst b))))+(rule 13 (lower (iadd $I64 a (u8_from_iconst b)))(pulley_xadd64_u8 a b));; If the rhs is a constant and the negated version can fit within a smaller;; constant then switch this to a subtraction with the negated constant.-(rule 14 (lower (has_type (ty_int (fits_in_32 _)) (iadd _ a b)))+(rule 14 (lower (iadd (ty_int (fits_in_32 _)) a b))(if-let c (u32_from_negated_iconst b))(pulley_xsub32_u32 a c))-(rule 15 (lower (has_type $I64 (iadd _ a b)))+(rule 15 (lower (iadd $I64 a b))(if-let c (u32_from_negated_iconst b))(pulley_xsub64_u32 a c))-(rule 16 (lower (has_type (ty_int (fits_in_32 _)) (iadd _ a b)))+(rule 16 (lower (iadd (ty_int (fits_in_32 _)) a b))(if-let c (u8_from_negated_iconst b))(pulley_xsub32_u8 a c))-(rule 17 (lower (has_type $I64 (iadd _ a b)))+(rule 17 (lower (iadd $I64 a b))(if-let c (u8_from_negated_iconst b))(pulley_xsub64_u8 a c))@@ -289,7 +289,7 @@neg_u32);; 128-bit addition-(rule 1 (lower (has_type $I128 (iadd _ a b)))+(rule 1 (lower (iadd $I128 a b))(let ((a ValueRegs a)(b ValueRegs b))(pulley_xadd128@@ -299,60 +299,60 @@(value_regs_get b 1))));; vector addition-(rule 1 (lower (has_type $I8X16 (iadd _ a b))) (pulley_vaddi8x16 a b))-(rule 1 (lower (has_type $I16X8 (iadd _ a b))) (pulley_vaddi16x8 a b))-(rule 1 (lower (has_type $I32X4 (iadd _ a b))) (pulley_vaddi32x4 a b))-(rule 1 (lower (has_type $I64X2 (iadd _ a b))) (pulley_vaddi64x2 a b))+(rule 1 (lower (iadd $I8X16 a b)) (pulley_vaddi8x16 a b))+(rule 1 (lower (iadd $I16X8 a b)) (pulley_vaddi16x8 a b))+(rule 1 (lower (iadd $I32X4 a b)) (pulley_vaddi32x4 a b))+(rule 1 (lower (iadd $I64X2 a b)) (pulley_vaddi64x2 a b))-(rule 1 (lower (has_type $I8X16 (sadd_sat _ a b))) (pulley_vaddi8x16_sat a b))-(rule 1 (lower (has_type $I8X16 (uadd_sat _ a b))) (pulley_vaddu8x16_sat a b))-(rule 1 (lower (has_type $I16X8 (sadd_sat _ a b))) (pulley_vaddi16x8_sat a b))-(rule 1 (lower (has_type $I16X8 (uadd_sat _ a b))) (pulley_vaddu16x8_sat a b))+(rule 1 (lower (sadd_sat $I8X16 a b)) (pulley_vaddi8x16_sat a b))+(rule 1 (lower (uadd_sat $I8X16 a b)) (pulley_vaddu8x16_sat a b))+(rule 1 (lower (sadd_sat $I16X8 a b)) (pulley_vaddi16x8_sat a b))+(rule 1 (lower (uadd_sat $I16X8 a b)) (pulley_vaddu16x8_sat a b));; Specialized lowerings for multiply-and-add-(rule 2 (lower (has_type $I32 (iadd _ (imul _ a b) c))) (pulley_xmadd32 a b c))-(rule 3 (lower (has_type $I32 (iadd _ c (imul _ a b)))) (pulley_xmadd32 a b c))-(rule 2 (lower (has_type $I64 (iadd _ (imul _ a b) c))) (pulley_xmadd64 a b c))-(rule 3 (lower (has_type $I64 (iadd _ c (imul _ a b)))) (pulley_xmadd64 a b c))+(rule 2 (lower (iadd $I32 (imul _ a b) c)) (pulley_xmadd32 a b c))+(rule 3 (lower (iadd $I32 c (imul _ a b))) (pulley_xmadd32 a b c))+(rule 2 (lower (iadd $I64 (imul _ a b) c)) (pulley_xmadd64 a b c))+(rule 3 (lower (iadd $I64 c (imul _ a b))) (pulley_xmadd64 a b c));;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16X8 (iadd_pairwise _ a b))) (pulley_vaddpairwisei16x8_s a b))-(rule (lower (has_type $I32X4 (iadd_pairwise _ a b))) (pulley_vaddpairwisei32x4_s a b))+(rule (lower (iadd_pairwise $I16X8 a b)) (pulley_vaddpairwisei16x8_s a b))+(rule (lower (iadd_pairwise $I32X4 a b)) (pulley_vaddpairwisei32x4_s a b));;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a b))) (pulley_xsub32 a b))-(rule 1 (lower (has_type $I64 (isub _ a b))) (pulley_xsub64 a b))+(rule 0 (lower (isub (ty_int (fits_in_32 _)) a b)) (pulley_xsub32 a b))+(rule 1 (lower (isub $I64 a b)) (pulley_xsub64 a b));; Fold a rhs constant into the instruction if possible.-(rule 2 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a (u32_from_iconst b))))+(rule 2 (lower (isub (ty_int (fits_in_32 _)) a (u32_from_iconst b)))(pulley_xsub32_u32 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a (u8_from_iconst b))))+(rule 3 (lower (isub (ty_int (fits_in_32 _)) a (u8_from_iconst b)))(pulley_xsub32_u8 a b))-(rule 4 (lower (has_type $I64 (isub _ a (u32_from_iconst b))))+(rule 4 (lower (isub $I64 a (u32_from_iconst b)))(pulley_xsub64_u32 a b))-(rule 5 (lower (has_type $I64 (isub _ a (u8_from_iconst b))))+(rule 5 (lower (isub $I64 a (u8_from_iconst b)))(pulley_xsub64_u8 a b));; If the rhs is a constant and the negated version can fit within a smaller;; constant then switch this to an addition with the negated constant.-(rule 6 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a b)))+(rule 6 (lower (isub (ty_int (fits_in_32 _)) a b))(if-let c (u32_from_negated_iconst b))(pulley_xadd32_u32 a c))-(rule 7 (lower (has_type $I64 (isub _ a b)))+(rule 7 (lower (isub $I64 a b))(if-let c (u32_from_negated_iconst b))(pulley_xadd64_u32 a c))-(rule 8 (lower (has_type (ty_int (fits_in_32 _)) (isub _ a b)))+(rule 8 (lower (isub (ty_int (fits_in_32 _)) a b))(if-let c (u8_from_negated_iconst b))(pulley_xadd32_u8 a c))-(rule 9 (lower (has_type $I64 (isub _ a b)))+(rule 9 (lower (isub $I64 a b))(if-let c (u8_from_negated_iconst b))(pulley_xadd64_u8 a c));; 128-bit subtraction-(rule 1 (lower (has_type $I128 (isub _ a b)))+(rule 1 (lower (isub $I128 a b))(let ((a ValueRegs a)(b ValueRegs b))(pulley_xsub128@@ -362,40 +362,40 @@(value_regs_get b 1))));; vector subtraction-(rule 1 (lower (has_type $I8X16 (isub _ a b))) (pulley_vsubi8x16 a b))-(rule 1 (lower (has_type $I16X8 (isub _ a b))) (pulley_vsubi16x8 a b))-(rule 1 (lower (has_type $I32X4 (isub _ a b))) (pulley_vsubi32x4 a b))-(rule 1 (lower (has_type $I64X2 (isub _ a b))) (pulley_vsubi64x2 a b))+(rule 1 (lower (isub $I8X16 a b)) (pulley_vsubi8x16 a b))+(rule 1 (lower (isub $I16X8 a b)) (pulley_vsubi16x8 a b))+(rule 1 (lower (isub $I32X4 a b)) (pulley_vsubi32x4 a b))+(rule 1 (lower (isub $I64X2 a b)) (pulley_vsubi64x2 a b))-(rule 1 (lower (has_type $I8X16 (ssub_sat _ a b))) (pulley_vsubi8x16_sat a b))-(rule 1 (lower (has_type $I8X16 (usub_sat _ a b))) (pulley_vsubu8x16_sat a b))-(rule 1 (lower (has_type $I16X8 (ssub_sat _ a b))) (pulley_vsubi16x8_sat a b))-(rule 1 (lower (has_type $I16X8 (usub_sat _ a b))) (pulley_vsubu16x8_sat a b))+(rule 1 (lower (ssub_sat $I8X16 a b)) (pulley_vsubi8x16_sat a b))+(rule 1 (lower (usub_sat $I8X16 a b)) (pulley_vsubu8x16_sat a b))+(rule 1 (lower (ssub_sat $I16X8 a b)) (pulley_vsubi16x8_sat a b))+(rule 1 (lower (usub_sat $I16X8 a b)) (pulley_vsubu16x8_sat a b));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (imul _ a b))) (pulley_xmul32 a b))-(rule (lower (has_type $I16 (imul _ a b))) (pulley_xmul32 a b))-(rule (lower (has_type $I32 (imul _ a b))) (pulley_xmul32 a b))-(rule (lower (has_type $I64 (imul _ a b))) (pulley_xmul64 a b))+(rule (lower (imul $I8 a b)) (pulley_xmul32 a b))+(rule (lower (imul $I16 a b)) (pulley_xmul32 a b))+(rule (lower (imul $I32 a b)) (pulley_xmul32 a b))+(rule (lower (imul $I64 a b)) (pulley_xmul64 a b))-(rule 1 (lower (has_type (ty_int (fits_in_32 _)) (imul _ a (i32_from_iconst b))))+(rule 1 (lower (imul (ty_int (fits_in_32 _)) a (i32_from_iconst b)))(pulley_xmul32_s32 a b))-(rule 2 (lower (has_type $I64 (imul _ a (i32_from_iconst b))))+(rule 2 (lower (imul $I64 a (i32_from_iconst b)))(pulley_xmul64_s32 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (imul _ a (i8_from_iconst b))))+(rule 3 (lower (imul (ty_int (fits_in_32 _)) a (i8_from_iconst b)))(pulley_xmul32_s8 a b))-(rule 4 (lower (has_type $I64 (imul _ a (i8_from_iconst b))))+(rule 4 (lower (imul $I64 a (i8_from_iconst b)))(pulley_xmul64_s8 a b));; 128-bit (or wide) multiplication-(rule 4 (lower (has_type $I128 (imul _ (uextend _ a) (uextend _ b))))+(rule 4 (lower (imul $I128 (uextend _ a) (uextend _ b)))(pulley_xwidemul64_u (zext64 a) (zext64 b)))-(rule 4 (lower (has_type $I128 (imul _ (sextend _ a) (sextend _ b))))+(rule 4 (lower (imul $I128 (sextend _ a) (sextend _ b)))(pulley_xwidemul64_s (sext64 a) (sext64 b)));; for I128-(rule (lower (has_type $I128 (imul _ x y)))+(rule (lower (imul $I128 x y))(let((x_regs ValueRegs x)(x_lo XReg (value_regs_get x_regs 0))@@ -420,106 +420,106 @@(value_regs wide_lo result_hi)));; vector multiplication-(rule (lower (has_type $I8X16 (imul _ a b))) (pulley_vmuli8x16 a b))-(rule (lower (has_type $I16X8 (imul _ a b))) (pulley_vmuli16x8 a b))-(rule (lower (has_type $I32X4 (imul _ a b))) (pulley_vmuli32x4 a b))-(rule (lower (has_type $I64X2 (imul _ a b))) (pulley_vmuli64x2 a b))+(rule (lower (imul $I8X16 a b)) (pulley_vmuli8x16 a b))+(rule (lower (imul $I16X8 a b)) (pulley_vmuli16x8 a b))+(rule (lower (imul $I32X4 a b)) (pulley_vmuli32x4 a b))+(rule (lower (imul $I64X2 a b)) (pulley_vmuli64x2 a b));;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (umulhi _ a b)))+(rule (lower (umulhi $I8 a b))(if-let (u6_from_u8 shift) (u64_try_into_u8 8))(pulley_xshr32_u_u6 (pulley_xmul32 (zext32 a) (zext32 b)) shift))-(rule (lower (has_type $I16 (umulhi _ a b)))+(rule (lower (umulhi $I16 a b))(if-let (u6_from_u8 shift) (u64_try_into_u8 16))(pulley_xshr32_u_u6 (pulley_xmul32 (zext32 a) (zext32 b)) shift))-(rule (lower (has_type $I32 (umulhi _ a b)))+(rule (lower (umulhi $I32 a b))(if-let (u6_from_u8 shift) (u64_try_into_u8 32))(pulley_xshr64_u_u6 (pulley_xmul64 (zext64 a) (zext64 b)) shift))-(rule (lower (has_type $I64 (umulhi _ a b)))+(rule (lower (umulhi $I64 a b))(pulley_xmulhi64_u a b));;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (smulhi _ a b)))+(rule (lower (smulhi $I8 a b))(if-let (u6_from_u8 shift) (u64_try_into_u8 8))(pulley_xshr32_s_u6 (pulley_xmul32 (sext32 a) (sext32 b)) shift))-(rule (lower (has_type $I16 (smulhi _ a b)))+(rule (lower (smulhi $I16 a b))(if-let (u6_from_u8 shift) (u64_try_into_u8 16))(pulley_xshr32_s_u6 (pulley_xmul32 (sext32 a) (sext32 b)) shift))-(rule (lower (has_type $I32 (smulhi _ a b)))+(rule (lower (smulhi $I32 a b))(if-let (u6_from_u8 shift) (u64_try_into_u8 32))(pulley_xshr64_s_u6 (pulley_xmul64 (sext64 a) (sext64 b)) shift))-(rule (lower (has_type $I64 (smulhi _ a b)))+(rule (lower (smulhi $I64 a b))(pulley_xmulhi64_s a b));;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16X8 (sqmul_round_sat _ a b))) (pulley_vqmulrsi16x8 a b))+(rule (lower (sqmul_round_sat $I16X8 a b)) (pulley_vqmulrsi16x8 a b));;;; Rules for `sdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (sdiv _ a b)))+(rule 0 (lower (sdiv (fits_in_32 _) a b))(pulley_xdiv32_s (sext32 a) (sext32 b)))-(rule 1 (lower (has_type $I64 (sdiv _ a b))) (pulley_xdiv64_s a b))+(rule 1 (lower (sdiv $I64 a b)) (pulley_xdiv64_s a b));;;; Rules for `srem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (srem _ a b)))+(rule 0 (lower (srem (fits_in_32 _) a b))(pulley_xrem32_s (sext32 a) (sext32 b)))-(rule 1 (lower (has_type $I64 (srem _ a b))) (pulley_xrem64_s a b))+(rule 1 (lower (srem $I64 a b)) (pulley_xrem64_s a b));;;; Rules for `udiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (udiv _ a b)))+(rule 0 (lower (udiv (ty_int (fits_in_32 _)) a b))(pulley_xdiv32_u (zext32 a) (zext32 b)))-(rule 1 (lower (has_type $I64 (udiv _ a b))) (pulley_xdiv64_u a b))+(rule 1 (lower (udiv $I64 a b)) (pulley_xdiv64_u a b));;;; Rules for `urem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (urem _ a b)))+(rule 0 (lower (urem (ty_int (fits_in_32 _)) a b))(pulley_xrem32_u (zext32 a) (zext32 b)))-(rule 1 (lower (has_type $I64 (urem _ a b))) (pulley_xrem64_u a b))+(rule 1 (lower (urem $I64 a b)) (pulley_xrem64_u a b));;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (avg_round _ a b))) (pulley_vavground8x16 a b))-(rule (lower (has_type $I16X8 (avg_round _ a b))) (pulley_vavground16x8 a b))+(rule (lower (avg_round $I8X16 a b)) (pulley_vavground8x16 a b))+(rule (lower (avg_round $I16X8 a b)) (pulley_vavground16x8 a b));;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (ishl _ a b)))+(rule (lower (ishl $I8 a b))(pulley_xshl32 a (pulley_xband32_s8 b 7)))-(rule (lower (has_type $I16 (ishl _ a b)))+(rule (lower (ishl $I16 a b))(pulley_xshl32 a (pulley_xband32_s8 b 15)))-(rule (lower (has_type $I32 (ishl _ a b)))+(rule (lower (ishl $I32 a b))(pulley_xshl32 a b))-(rule (lower (has_type $I64 (ishl _ a b)))+(rule (lower (ishl $I64 a b))(pulley_xshl64 a b));; Special-case constant shift amounts.-(rule 1 (lower (has_type $I32 (ishl _ a b)))+(rule 1 (lower (ishl $I32 a b))(if-let n (u6_shift_from_iconst b))(pulley_xshl32_u6 a n))-(rule 1 (lower (has_type $I64 (ishl _ a b)))+(rule 1 (lower (ishl $I64 a b))(if-let n (u6_shift_from_iconst b))(pulley_xshl64_u6 a n));; vector shifts-(rule (lower (has_type $I8X16 (ishl _ a b))) (pulley_vshli8x16 a b))-(rule (lower (has_type $I16X8 (ishl _ a b))) (pulley_vshli16x8 a b))-(rule (lower (has_type $I32X4 (ishl _ a b))) (pulley_vshli32x4 a b))-(rule (lower (has_type $I64X2 (ishl _ a b))) (pulley_vshli64x2 a b))+(rule (lower (ishl $I8X16 a b)) (pulley_vshli8x16 a b))+(rule (lower (ishl $I16X8 a b)) (pulley_vshli16x8 a b))+(rule (lower (ishl $I32X4 a b)) (pulley_vshli32x4 a b))+(rule (lower (ishl $I64X2 a b)) (pulley_vshli64x2 a b));; Helper to extract a constant from `Value`, mask it to 6 bits, and then make a;; `U6`.@@ -533,207 +533,207 @@;;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (ushr _ a b)))+(rule (lower (ushr $I8 a b))(pulley_xshr32_u (zext32 a) (pulley_xband32_s8 b 7)))-(rule (lower (has_type $I16 (ushr _ a b)))+(rule (lower (ushr $I16 a b))(pulley_xshr32_u (zext32 a) (pulley_xband32_s8 b 15)))-(rule (lower (has_type $I32 (ushr _ a b)))+(rule (lower (ushr $I32 a b))(pulley_xshr32_u a b))-(rule (lower (has_type $I64 (ushr _ a b)))+(rule (lower (ushr $I64 a b))(pulley_xshr64_u a b));; Special-case constant shift amounts.-(rule 1 (lower (has_type $I32 (ushr _ a b)))+(rule 1 (lower (ushr $I32 a b))(if-let n (u6_shift_from_iconst b))(pulley_xshr32_u_u6 a n))-(rule 1 (lower (has_type $I64 (ushr _ a b)))+(rule 1 (lower (ushr $I64 a b))(if-let n (u6_shift_from_iconst b))(pulley_xshr64_u_u6 a n));; vector shifts-(rule (lower (has_type $I8X16 (ushr _ a b))) (pulley_vshri8x16_u a b))-(rule (lower (has_type $I16X8 (ushr _ a b))) (pulley_vshri16x8_u a b))-(rule (lower (has_type $I32X4 (ushr _ a b))) (pulley_vshri32x4_u a b))-(rule (lower (has_type $I64X2 (ushr _ a b))) (pulley_vshri64x2_u a b))+(rule (lower (ushr $I8X16 a b)) (pulley_vshri8x16_u a b))+(rule (lower (ushr $I16X8 a b)) (pulley_vshri16x8_u a b))+(rule (lower (ushr $I32X4 a b)) (pulley_vshri32x4_u a b))+(rule (lower (ushr $I64X2 a b)) (pulley_vshri64x2_u a b));;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (sshr _ a b)))+(rule (lower (sshr $I8 a b))(pulley_xshr32_u (sext32 a) (pulley_xband32_s8 b 7)))-(rule (lower (has_type $I16 (sshr _ a b)))+(rule (lower (sshr $I16 a b))(pulley_xshr32_u (sext32 a) (pulley_xband32_s8 b 15)))-(rule (lower (has_type $I32 (sshr _ a b)))+(rule (lower (sshr $I32 a b))(pulley_xshr32_s a b))-(rule (lower (has_type $I64 (sshr _ a b)))+(rule (lower (sshr $I64 a b))(pulley_xshr64_s a b));; Special-case constant shift amounts.-(rule 1 (lower (has_type $I32 (sshr _ a b)))+(rule 1 (lower (sshr $I32 a b))(if-let n (u6_shift_from_iconst b))(pulley_xshr32_s_u6 a n))-(rule 1 (lower (has_type $I64 (sshr _ a b)))+(rule 1 (lower (sshr $I64 a b))(if-let n (u6_shift_from_iconst b))(pulley_xshr64_s_u6 a n));; vector shifts-(rule (lower (has_type $I8X16 (sshr _ a b))) (pulley_vshri8x16_s a b))-(rule (lower (has_type $I16X8 (sshr _ a b))) (pulley_vshri16x8_s a b))-(rule (lower (has_type $I32X4 (sshr _ a b))) (pulley_vshri32x4_s a b))-(rule (lower (has_type $I64X2 (sshr _ a b))) (pulley_vshri64x2_s a b))+(rule (lower (sshr $I8X16 a b)) (pulley_vshri8x16_s a b))+(rule (lower (sshr $I16X8 a b)) (pulley_vshri16x8_s a b))+(rule (lower (sshr $I32X4 a b)) (pulley_vshri32x4_s a b))+(rule (lower (sshr $I64X2 a b)) (pulley_vshri64x2_s a b));;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (band _ a b))) (pulley_xband32 a b))-(rule 1 (lower (has_type $I64 (band _ a b))) (pulley_xband64 a b))+(rule 0 (lower (band (fits_in_32 _) a b)) (pulley_xband32 a b))+(rule 1 (lower (band $I64 a b)) (pulley_xband64 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (band _ a (i32_from_iconst b))))+(rule 3 (lower (band (ty_int (fits_in_32 _)) a (i32_from_iconst b)))(pulley_xband32_s32 a b))-(rule 4 (lower (has_type $I64 (band _ a (i32_from_iconst b))))+(rule 4 (lower (band $I64 a (i32_from_iconst b)))(pulley_xband64_s32 a b))-(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (band _ a (i8_from_iconst b))))+(rule 5 (lower (band (ty_int (fits_in_32 _)) a (i8_from_iconst b)))(pulley_xband32_s8 a b))-(rule 6 (lower (has_type $I64 (band _ a (i8_from_iconst b))))+(rule 6 (lower (band $I64 a (i8_from_iconst b)))(pulley_xband64_s8 a b))-(rule 2 (lower (has_type (ty_vec128 _) (band _ a b)))+(rule 2 (lower (band (ty_vec128 _) a b))(pulley_vband128 a b));;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (bor _ a b))) (pulley_xbor32 a b))-(rule 1 (lower (has_type $I64 (bor _ a b))) (pulley_xbor64 a b))+(rule 0 (lower (bor (fits_in_32 _) a b)) (pulley_xbor32 a b))+(rule 1 (lower (bor $I64 a b)) (pulley_xbor64 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (bor _ a (i32_from_iconst b))))+(rule 3 (lower (bor (ty_int (fits_in_32 _)) a (i32_from_iconst b)))(pulley_xbor32_s32 a b))-(rule 4 (lower (has_type $I64 (bor _ a (i32_from_iconst b))))+(rule 4 (lower (bor $I64 a (i32_from_iconst b)))(pulley_xbor64_s32 a b))-(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (bor _ a (i8_from_iconst b))))+(rule 5 (lower (bor (ty_int (fits_in_32 _)) a (i8_from_iconst b)))(pulley_xbor32_s8 a b))-(rule 6 (lower (has_type $I64 (bor _ a (i8_from_iconst b))))+(rule 6 (lower (bor $I64 a (i8_from_iconst b)))(pulley_xbor64_s8 a b))-(rule 2 (lower (has_type (ty_vec128 _) (bor _ a b)))+(rule 2 (lower (bor (ty_vec128 _) a b))(pulley_vbor128 a b));;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (bxor _ a b))) (pulley_xbxor32 a b))-(rule 1 (lower (has_type $I64 (bxor _ a b))) (pulley_xbxor64 a b))+(rule 0 (lower (bxor (fits_in_32 _) a b)) (pulley_xbxor32 a b))+(rule 1 (lower (bxor $I64 a b)) (pulley_xbxor64 a b))-(rule 3 (lower (has_type (ty_int (fits_in_32 _)) (bxor _ a (i32_from_iconst b))))+(rule 3 (lower (bxor (ty_int (fits_in_32 _)) a (i32_from_iconst b)))(pulley_xbxor32_s32 a b))-(rule 4 (lower (has_type $I64 (bxor _ a (i32_from_iconst b))))+(rule 4 (lower (bxor $I64 a (i32_from_iconst b)))(pulley_xbxor64_s32 a b))-(rule 5 (lower (has_type (ty_int (fits_in_32 _)) (bxor _ a (i8_from_iconst b))))+(rule 5 (lower (bxor (ty_int (fits_in_32 _)) a (i8_from_iconst b)))(pulley_xbxor32_s8 a b))-(rule 6 (lower (has_type $I64 (bxor _ a (i8_from_iconst b))))+(rule 6 (lower (bxor $I64 a (i8_from_iconst b)))(pulley_xbxor64_s8 a b))-(rule 2 (lower (has_type (ty_vec128 _) (bxor _ a b)))+(rule 2 (lower (bxor (ty_vec128 _) a b))(pulley_vbxor128 a b));;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (bnot _ a)))+(rule 0 (lower (bnot (fits_in_32 _) a))(pulley_xbnot32 a))-(rule 1 (lower (has_type $I64 (bnot _ a)))+(rule 1 (lower (bnot $I64 a))(pulley_xbnot64 a))-(rule 2 (lower (has_type (ty_vec128 _) (bnot _ a)))+(rule 2 (lower (bnot (ty_vec128 _) a))(pulley_vbnot128 a));;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 _) (bitselect _ c x y)))+(rule (lower (bitselect (ty_vec128 _) c x y))(pulley_vbitselect128 c x y));;;; Rules for `umin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (umin _ a b)))+(rule 0 (lower (umin (fits_in_32 _) a b))(pulley_xmin32_u (zext32 a) (zext32 b)))-(rule 1 (lower (has_type $I64 (umin _ a b))) (pulley_xmin64_u a b))-(rule 1 (lower (has_type $I8X16 (umin _ a b))) (pulley_vmin8x16_u a b))-(rule 1 (lower (has_type $I16X8 (umin _ a b))) (pulley_vmin16x8_u a b))-(rule 1 (lower (has_type $I32X4 (umin _ a b))) (pulley_vmin32x4_u a b))+(rule 1 (lower (umin $I64 a b)) (pulley_xmin64_u a b))+(rule 1 (lower (umin $I8X16 a b)) (pulley_vmin8x16_u a b))+(rule 1 (lower (umin $I16X8 a b)) (pulley_vmin16x8_u a b))+(rule 1 (lower (umin $I32X4 a b)) (pulley_vmin32x4_u a b));;;; Rules for `smin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (smin _ a b)))+(rule 0 (lower (smin (fits_in_32 _) a b))(pulley_xmin32_s (sext32 a) (sext32 b)))-(rule 1 (lower (has_type $I64 (smin _ a b))) (pulley_xmin64_s a b))-(rule 1 (lower (has_type $I8X16 (smin _ a b))) (pulley_vmin8x16_s a b))-(rule 1 (lower (has_type $I16X8 (smin _ a b))) (pulley_vmin16x8_s a b))-(rule 1 (lower (has_type $I32X4 (smin _ a b))) (pulley_vmin32x4_s a b))+(rule 1 (lower (smin $I64 a b)) (pulley_xmin64_s a b))+(rule 1 (lower (smin $I8X16 a b)) (pulley_vmin8x16_s a b))+(rule 1 (lower (smin $I16X8 a b)) (pulley_vmin16x8_s a b))+(rule 1 (lower (smin $I32X4 a b)) (pulley_vmin32x4_s a b));;;; Rules for `umax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (umax _ a b)))+(rule 0 (lower (umax (fits_in_32 _) a b))(pulley_xmax32_u (zext32 a) (zext32 b)))-(rule 1 (lower (has_type $I64 (umax _ a b))) (pulley_xmax64_u a b))-(rule 1 (lower (has_type $I8X16 (umax _ a b))) (pulley_vmax8x16_u a b))-(rule 1 (lower (has_type $I16X8 (umax _ a b))) (pulley_vmax16x8_u a b))-(rule 1 (lower (has_type $I32X4 (umax _ a b))) (pulley_vmax32x4_u a b))+(rule 1 (lower (umax $I64 a b)) (pulley_xmax64_u a b))+(rule 1 (lower (umax $I8X16 a b)) (pulley_vmax8x16_u a b))+(rule 1 (lower (umax $I16X8 a b)) (pulley_vmax16x8_u a b))+(rule 1 (lower (umax $I32X4 a b)) (pulley_vmax32x4_u a b));;;; Rules for `smax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (smax _ a b)))+(rule 0 (lower (smax (fits_in_32 _) a b))(pulley_xmax32_s (sext32 a) (sext32 b)))-(rule 1 (lower (has_type $I64 (smax _ a b))) (pulley_xmax64_s a b))-(rule 1 (lower (has_type $I8X16 (smax _ a b))) (pulley_vmax8x16_s a b))-(rule 1 (lower (has_type $I16X8 (smax _ a b))) (pulley_vmax16x8_s a b))-(rule 1 (lower (has_type $I32X4 (smax _ a b))) (pulley_vmax32x4_s a b))+(rule 1 (lower (smax $I64 a b)) (pulley_xmax64_s a b))+(rule 1 (lower (smax $I8X16 a b)) (pulley_vmax8x16_s a b))+(rule 1 (lower (smax $I16X8 a b)) (pulley_vmax16x8_s a b))+(rule 1 (lower (smax $I32X4 a b)) (pulley_vmax32x4_s a b));;;; Rules for `bmask` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (bmask _ a @ (value_type (fits_in_32 _)))))+(rule 0 (lower (bmask (ty_int (fits_in_32 _)) a @ (value_type (fits_in_32 _))))(pulley_xbmask32 (zext32 a)))-(rule 1 (lower (has_type $I64 (bmask _ a)))+(rule 1 (lower (bmask $I64 a))(pulley_xbmask64 (zext64 a)))(rule 2 (lower (bmask _ a @ (value_type $I64)))(pulley_xbmask64 a));;;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (ctz _ a)))+(rule (lower (ctz $I8 a))(pulley_xctz32 (pulley_xbor32_s32 a 0x100)))-(rule (lower (has_type $I16 (ctz _ a)))+(rule (lower (ctz $I16 a))(pulley_xctz32 (pulley_xbor32_s32 a 0x10000)))-(rule (lower (has_type $I32 (ctz _ a))) (pulley_xctz32 a))-(rule (lower (has_type $I64 (ctz _ a))) (pulley_xctz64 a))+(rule (lower (ctz $I32 a)) (pulley_xctz32 a))+(rule (lower (ctz $I64 a)) (pulley_xctz64 a));;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (clz _ a)))+(rule (lower (clz $I8 a))(pulley_xsub32_u8 (pulley_xclz32 (zext32 a)) 24))-(rule (lower (has_type $I16 (clz _ a)))+(rule (lower (clz $I16 a))(pulley_xsub32_u8 (pulley_xclz32 (zext32 a)) 16))-(rule (lower (has_type $I32 (clz _ a))) (pulley_xclz32 a))-(rule (lower (has_type $I64 (clz _ a))) (pulley_xclz64 a))+(rule (lower (clz $I32 a)) (pulley_xclz32 a))+(rule (lower (clz $I64 a)) (pulley_xclz64 a));;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (popcnt _ a))) (pulley_xpopcnt32 (zext32 a)))-(rule 1 (lower (has_type $I64 (popcnt _ a))) (pulley_xpopcnt64 a))-(rule 1 (lower (has_type $I8X16 (popcnt _ a))) (pulley_vpopcnt8x16 a))+(rule 0 (lower (popcnt (fits_in_32 _) a)) (pulley_xpopcnt32 (zext32 a)))+(rule 1 (lower (popcnt $I64 a)) (pulley_xpopcnt64 a))+(rule 1 (lower (popcnt $I8X16 a)) (pulley_vpopcnt8x16 a));;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (rotl _ a b))) (pulley_xrotl32 a b))-(rule (lower (has_type $I64 (rotl _ a b))) (pulley_xrotl64 a b))+(rule (lower (rotl $I32 a b)) (pulley_xrotl32 a b))+(rule (lower (rotl $I64 a b)) (pulley_xrotl64 a b));;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (rotr _ a b))) (pulley_xrotr32 a b))-(rule (lower (has_type $I64 (rotr _ a b))) (pulley_xrotr64 a b))+(rule (lower (rotr $I32 a b)) (pulley_xrotr32 a b))+(rule (lower (rotr $I64 a b)) (pulley_xrotr64 a b));;;; Rules for `icmp` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -960,73 +960,73 @@;;;; Rules for `load` and friends ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_int (fits_in_64 ty)) (load _ (mem_flags_data flags) addr offset)))+(rule (lower (load (ty_int (fits_in_64 ty)) (mem_flags_data flags) addr offset))(gen_xload addr offset flags ty (ExtKind.None)))-(rule 1 (lower (has_type (ty_scalar_float ty) (load _ (mem_flags_data flags) addr offset)))+(rule 1 (lower (load (ty_scalar_float ty) (mem_flags_data flags) addr offset))(gen_fload addr offset flags ty))-(rule 3 (lower (has_type $I128 (load _ (mem_flags_data flags) addr offset)))+(rule 3 (lower (load $I128 (mem_flags_data flags) addr offset))(if-let offsetp8 (i32_checked_add offset 8))(let ((lo XReg (pulley_xload (amode addr offset) $I64 flags))(hi XReg (pulley_xload (amode addr offsetp8) $I64 flags)))(value_regs lo hi)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload8 _ (mem_flags_data flags) addr offset)))+(rule 0 (lower (uload8 (ty_int (fits_in_32 _)) (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I8 (ExtKind.Zero32)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload16 _ (mem_flags_data flags) addr offset)))+(rule 0 (lower (uload16 (ty_int (fits_in_32 _)) (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I16 (ExtKind.Zero32)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (uload32 _ (mem_flags_data flags) addr offset)))+(rule 0 (lower (uload32 (ty_int (fits_in_32 _)) (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I32 (ExtKind.None)))-(rule 1 (lower (has_type $I64 (uload8 _ (mem_flags_data flags) addr offset)))+(rule 1 (lower (uload8 $I64 (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I8 (ExtKind.Zero64)))-(rule 1 (lower (has_type $I64 (uload16 _ (mem_flags_data flags) addr offset)))+(rule 1 (lower (uload16 $I64 (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I16 (ExtKind.Zero64)))-(rule 1 (lower (has_type $I64 (uload32 _ (mem_flags_data flags) addr offset)))+(rule 1 (lower (uload32 $I64 (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I32 (ExtKind.Zero64)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload8 _ (mem_flags_data flags) addr offset)))+(rule 0 (lower (sload8 (ty_int (fits_in_32 _)) (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I8 (ExtKind.Sign32)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload16 _ (mem_flags_data flags) addr offset)))+(rule 0 (lower (sload16 (ty_int (fits_in_32 _)) (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I16 (ExtKind.Sign32)))-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (sload32 _ (mem_flags_data flags) addr offset)))+(rule 0 (lower (sload32 (ty_int (fits_in_32 _)) (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I32 (ExtKind.None)))-(rule 1 (lower (has_type $I64 (sload8 _ (mem_flags_data flags) addr offset)))+(rule 1 (lower (sload8 $I64 (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I8 (ExtKind.Sign64)))-(rule 1 (lower (has_type $I64 (sload16 _ (mem_flags_data flags) addr offset)))+(rule 1 (lower (sload16 $I64 (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I16 (ExtKind.Sign64)))-(rule 1 (lower (has_type $I64 (sload32 _ (mem_flags_data flags) addr offset)))+(rule 1 (lower (sload32 $I64 (mem_flags_data flags) addr offset))(gen_xload addr offset flags $I32 (ExtKind.Sign64)))-(rule 2 (lower (has_type (ty_vec128 ty) (load _ (mem_flags_data flags) addr offset)))+(rule 2 (lower (load (ty_vec128 ty) (mem_flags_data flags) addr offset))(gen_vload addr offset flags ty (VExtKind.None)))-(rule (lower (has_type ty (sload8x8 _ (mem_flags_data flags) addr offset)))+(rule (lower (sload8x8 ty (mem_flags_data flags) addr offset))(gen_vload addr offset flags ty (VExtKind.S8x8)))-(rule (lower (has_type ty (uload8x8 _ (mem_flags_data flags) addr offset)))+(rule (lower (uload8x8 ty (mem_flags_data flags) addr offset))(gen_vload addr offset flags ty (VExtKind.U8x8)))-(rule (lower (has_type ty (sload16x4 _ (mem_flags_data flags) addr offset)))+(rule (lower (sload16x4 ty (mem_flags_data flags) addr offset))(gen_vload addr offset flags ty (VExtKind.S16x4)))-(rule (lower (has_type ty (uload16x4 _ (mem_flags_data flags) addr offset)))+(rule (lower (uload16x4 ty (mem_flags_data flags) addr offset))(gen_vload addr offset flags ty (VExtKind.U16x4)))-(rule (lower (has_type ty (sload32x2 _ (mem_flags_data flags) addr offset)))+(rule (lower (sload32x2 ty (mem_flags_data flags) addr offset))(gen_vload addr offset flags ty (VExtKind.S32x2)))-(rule (lower (has_type ty (uload32x2 _ (mem_flags_data flags) addr offset)))+(rule (lower (uload32x2 ty (mem_flags_data flags) addr offset))(gen_vload addr offset flags ty (VExtKind.U32x2)));; Helper to generate an `xload*` instruction, of which there are many. This@@ -1367,24 +1367,24 @@;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (uextend _ val)))+(rule 0 (lower (uextend (fits_in_32 _) val))(zext32 val))-(rule 1 (lower (has_type $I64 (uextend _ val)))+(rule 1 (lower (uextend $I64 val))(zext64 val))-(rule 1 (lower (has_type $I128 (uextend _ val)))+(rule 1 (lower (uextend $I128 val))(value_regs (zext64 val) (pulley_xzero)));;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (sextend _ val)))+(rule 0 (lower (sextend (fits_in_32 _) val))(sext32 val))-(rule 1 (lower (has_type $I64 (sextend _ val)))+(rule 1 (lower (sextend $I64 val))(sext64 val))-(rule 1 (lower (has_type $I128 (sextend _ val)))+(rule 1 (lower (sextend $I128 val))(if-let (u6_from_u8 shift) (u64_try_into_u8 63))(let ((lo XReg (sext64 val))(hi XReg (pulley_xshr64_s_u6 lo shift)))@@ -1392,12 +1392,12 @@;;;; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 _ty) (ireduce _ src)))+(rule (lower (ireduce (fits_in_64 _ty) src))src);;;; Rules for `iconcat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I128 (iconcat _ a b)))+(rule (lower (iconcat $I128 a b))(value_regs a b));;;; Rules for `isplit` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1409,39 +1409,39 @@;;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (uadd_overflow_trap _ a b tc)))+(rule (lower (uadd_overflow_trap $I32 a b tc))(pulley_xadd32_uoverflow_trap a b tc))-(rule (lower (has_type $I64 (uadd_overflow_trap _ a b tc)))+(rule (lower (uadd_overflow_trap $I64 a b tc))(pulley_xadd64_uoverflow_trap a b tc));;;; Rules for `uadd_overflow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (uadd_overflow _ a b)))+(rule (lower (uadd_overflow $I32 a b))(let ((sum XReg (pulley_xadd32 a b))(is_overflow XReg (pulley_xult32 sum a)))(output_pair sum is_overflow)))-(rule (lower (has_type $I64 (uadd_overflow _ a b)))+(rule (lower (uadd_overflow $I64 a b))(let ((sum XReg (pulley_xadd64 a b))(is_overflow XReg (pulley_xult64 sum a)))(output_pair sum is_overflow)));;;; Rules for `select` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int (fits_in_32 _)) (select _ c a b)))+(rule 0 (lower (select (ty_int (fits_in_32 _)) c a b))(pulley_xselect32 (emit_cond (lower_cond c)) a b))-(rule 1 (lower (has_type $I64 (select _ c a b)))+(rule 1 (lower (select $I64 c a b))(pulley_xselect64 (emit_cond (lower_cond c)) a b))-(rule 1 (lower (has_type $F32 (select _ c a b)))+(rule 1 (lower (select $F32 c a b))(pulley_fselect32 (emit_cond (lower_cond c)) a b))-(rule 1 (lower (has_type $F64 (select _ c a b)))+(rule 1 (lower (select $F64 c a b))(pulley_fselect64 (emit_cond (lower_cond c)) a b))-(rule 2 (lower (has_type (ty_vec128 _) (select _ c a b)))+(rule 2 (lower (select (ty_vec128 _) c a b))(pulley_vselect (emit_cond (lower_cond c)) a b));; Helper to emit a conditional into a register itself.@@ -1489,293 +1489,293 @@;;;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (bitcast _ _flags val @ (value_type $I32))))+(rule (lower (bitcast $F32 _flags val @ (value_type $I32)))(pulley_bitcast_float_from_int_32 val))-(rule (lower (has_type $F64 (bitcast _ _flags val @ (value_type $I64))))+(rule (lower (bitcast $F64 _flags val @ (value_type $I64)))(pulley_bitcast_float_from_int_64 val))-(rule (lower (has_type $I32 (bitcast _ _flags val @ (value_type $F32))))+(rule (lower (bitcast $I32 _flags val @ (value_type $F32)))(pulley_bitcast_int_from_float_32 val))-(rule (lower (has_type $I64 (bitcast _ _flags val @ (value_type $F64))))+(rule (lower (bitcast $I64 _flags val @ (value_type $F64)))(pulley_bitcast_int_from_float_64 val))-(rule 1 (lower (has_type (ty_vec128 _) (bitcast _ _flags val @ (value_type (ty_vec128 _)))))+(rule 1 (lower (bitcast (ty_vec128 _) _flags val @ (value_type (ty_vec128 _))))val)-(rule 2 (lower (has_type ty (bitcast _ _flags a @ (value_type ty)))) a)+(rule 2 (lower (bitcast ty _flags a @ (value_type ty))) a);;;; Rules for `fcvt_to_{u,s}int` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (fcvt_to_uint _ val @ (value_type $F32))))+(rule (lower (fcvt_to_uint $I32 val @ (value_type $F32)))(pulley_x32_from_f32_u val))-(rule (lower (has_type $I32 (fcvt_to_uint _ val @ (value_type $F64))))+(rule (lower (fcvt_to_uint $I32 val @ (value_type $F64)))(pulley_x32_from_f64_u val))-(rule (lower (has_type $I64 (fcvt_to_uint _ val @ (value_type $F32))))+(rule (lower (fcvt_to_uint $I64 val @ (value_type $F32)))(pulley_x64_from_f32_u val))-(rule (lower (has_type $I64 (fcvt_to_uint _ val @ (value_type $F64))))+(rule (lower (fcvt_to_uint $I64 val @ (value_type $F64)))(pulley_x64_from_f64_u val))-(rule (lower (has_type $I32 (fcvt_to_sint _ val @ (value_type $F32))))+(rule (lower (fcvt_to_sint $I32 val @ (value_type $F32)))(pulley_x32_from_f32_s val))-(rule (lower (has_type $I32 (fcvt_to_sint _ val @ (value_type $F64))))+(rule (lower (fcvt_to_sint $I32 val @ (value_type $F64)))(pulley_x32_from_f64_s val))-(rule (lower (has_type $I64 (fcvt_to_sint _ val @ (value_type $F32))))+(rule (lower (fcvt_to_sint $I64 val @ (value_type $F32)))(pulley_x64_from_f32_s val))-(rule (lower (has_type $I64 (fcvt_to_sint _ val @ (value_type $F64))))+(rule (lower (fcvt_to_sint $I64 val @ (value_type $F64)))(pulley_x64_from_f64_s val));;;; Rules for `fcvt_from_{u,s}int` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type $F32 (fcvt_from_uint _ val @ (value_type (fits_in_32 _)))))+(rule 0 (lower (fcvt_from_uint $F32 val @ (value_type (fits_in_32 _))))(pulley_f32_from_x32_u (zext32 val)))-(rule 1 (lower (has_type $F32 (fcvt_from_uint _ val @ (value_type $I64))))+(rule 1 (lower (fcvt_from_uint $F32 val @ (value_type $I64)))(pulley_f32_from_x64_u val))-(rule 0 (lower (has_type $F64 (fcvt_from_uint _ val @ (value_type (fits_in_32 _)))))+(rule 0 (lower (fcvt_from_uint $F64 val @ (value_type (fits_in_32 _))))(pulley_f64_from_x32_u (zext32 val)))-(rule 1 (lower (has_type $F64 (fcvt_from_uint _ val @ (value_type $I64))))+(rule 1 (lower (fcvt_from_uint $F64 val @ (value_type $I64)))(pulley_f64_from_x64_u val))-(rule 0 (lower (has_type $F32 (fcvt_from_sint _ val @ (value_type (fits_in_32 _)))))+(rule 0 (lower (fcvt_from_sint $F32 val @ (value_type (fits_in_32 _))))(pulley_f32_from_x32_s (sext32 val)))-(rule 1 (lower (has_type $F32 (fcvt_from_sint _ val @ (value_type $I64))))+(rule 1 (lower (fcvt_from_sint $F32 val @ (value_type $I64)))(pulley_f32_from_x64_s val))-(rule 0 (lower (has_type $F64 (fcvt_from_sint _ val @ (value_type (fits_in_32 _)))))+(rule 0 (lower (fcvt_from_sint $F64 val @ (value_type (fits_in_32 _))))(pulley_f64_from_x32_s (sext32 val)))-(rule 1 (lower (has_type $F64 (fcvt_from_sint _ val @ (value_type $I64))))+(rule 1 (lower (fcvt_from_sint $F64 val @ (value_type $I64)))(pulley_f64_from_x64_s val))-(rule (lower (has_type $F32X4 (fcvt_from_sint _ val @ (value_type $I32X4))))+(rule (lower (fcvt_from_sint $F32X4 val @ (value_type $I32X4)))(pulley_vf32x4_from_i32x4_s val))-(rule (lower (has_type $F32X4 (fcvt_from_uint _ val @ (value_type $I32X4))))+(rule (lower (fcvt_from_uint $F32X4 val @ (value_type $I32X4)))(pulley_vf32x4_from_i32x4_u val))-(rule (lower (has_type $F64X2 (fcvt_from_sint _ val @ (value_type $I64X2))))+(rule (lower (fcvt_from_sint $F64X2 val @ (value_type $I64X2)))(pulley_vf64x2_from_i64x2_s val))-(rule (lower (has_type $F64X2 (fcvt_from_uint _ val @ (value_type $I64X2))))+(rule (lower (fcvt_from_uint $F64X2 val @ (value_type $I64X2)))(pulley_vf64x2_from_i64x2_u val));;;; Rules for `fcvt_to_{u,s}int_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32 (fcvt_to_uint_sat _ val @ (value_type $F32))))+(rule (lower (fcvt_to_uint_sat $I32 val @ (value_type $F32)))(pulley_x32_from_f32_u_sat val))-(rule (lower (has_type $I32 (fcvt_to_uint_sat _ val @ (value_type $F64))))+(rule (lower (fcvt_to_uint_sat $I32 val @ (value_type $F64)))(pulley_x32_from_f64_u_sat val))-(rule (lower (has_type $I64 (fcvt_to_uint_sat _ val @ (value_type $F32))))+(rule (lower (fcvt_to_uint_sat $I64 val @ (value_type $F32)))(pulley_x64_from_f32_u_sat val))-(rule (lower (has_type $I64 (fcvt_to_uint_sat _ val @ (value_type $F64))))+(rule (lower (fcvt_to_uint_sat $I64 val @ (value_type $F64)))(pulley_x64_from_f64_u_sat val))-(rule (lower (has_type $I32 (fcvt_to_sint_sat _ val @ (value_type $F32))))+(rule (lower (fcvt_to_sint_sat $I32 val @ (value_type $F32)))(pulley_x32_from_f32_s_sat val))-(rule (lower (has_type $I32 (fcvt_to_sint_sat _ val @ (value_type $F64))))+(rule (lower (fcvt_to_sint_sat $I32 val @ (value_type $F64)))(pulley_x32_from_f64_s_sat val))-(rule (lower (has_type $I64 (fcvt_to_sint_sat _ val @ (value_type $F32))))+(rule (lower (fcvt_to_sint_sat $I64 val @ (value_type $F32)))(pulley_x64_from_f32_s_sat val))-(rule (lower (has_type $I64 (fcvt_to_sint_sat _ val @ (value_type $F64))))+(rule (lower (fcvt_to_sint_sat $I64 val @ (value_type $F64)))(pulley_x64_from_f64_s_sat val))-(rule (lower (has_type $I32X4 (fcvt_to_sint_sat _ val @ (value_type $F32X4))))+(rule (lower (fcvt_to_sint_sat $I32X4 val @ (value_type $F32X4)))(pulley_vi32x4_from_f32x4_s val))-(rule (lower (has_type $I32X4 (fcvt_to_uint_sat _ val @ (value_type $F32X4))))+(rule (lower (fcvt_to_uint_sat $I32X4 val @ (value_type $F32X4)))(pulley_vi32x4_from_f32x4_u val))-(rule (lower (has_type $I64X2 (fcvt_to_sint_sat _ val @ (value_type $F64X2))))+(rule (lower (fcvt_to_sint_sat $I64X2 val @ (value_type $F64X2)))(pulley_vi64x2_from_f64x2_s val))-(rule (lower (has_type $I64X2 (fcvt_to_uint_sat _ val @ (value_type $F64X2))))+(rule (lower (fcvt_to_uint_sat $I64X2 val @ (value_type $F64X2)))(pulley_vi64x2_from_f64x2_u val));;;; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdemote _ val @ (value_type $F64))))+(rule (lower (fdemote $F32 val @ (value_type $F64)))(pulley_f32_from_f64 val));;;; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F64 (fpromote _ val @ (value_type $F32))))+(rule (lower (fpromote $F64 val @ (value_type $F32)))(pulley_f64_from_f32 val));;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fcopysign _ a b)))+(rule (lower (fcopysign $F32 a b))(pulley_fcopysign32 a b))-(rule (lower (has_type $F64 (fcopysign _ a b)))+(rule (lower (fcopysign $F64 a b))(pulley_fcopysign64 a b));;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fadd _ a b))) (pulley_fadd32 a b))-(rule (lower (has_type $F64 (fadd _ a b))) (pulley_fadd64 a b))-(rule (lower (has_type $F32X4 (fadd _ a b))) (pulley_vaddf32x4 a b))-(rule (lower (has_type $F64X2 (fadd _ a b))) (pulley_vaddf64x2 a b))+(rule (lower (fadd $F32 a b)) (pulley_fadd32 a b))+(rule (lower (fadd $F64 a b)) (pulley_fadd64 a b))+(rule (lower (fadd $F32X4 a b)) (pulley_vaddf32x4 a b))+(rule (lower (fadd $F64X2 a b)) (pulley_vaddf64x2 a b));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fsub _ a b))) (pulley_fsub32 a b))-(rule (lower (has_type $F64 (fsub _ a b))) (pulley_fsub64 a b))-(rule (lower (has_type $F32X4 (fsub _ a b))) (pulley_vsubf32x4 a b))-(rule (lower (has_type $F64X2 (fsub _ a b))) (pulley_vsubf64x2 a b))+(rule (lower (fsub $F32 a b)) (pulley_fsub32 a b))+(rule (lower (fsub $F64 a b)) (pulley_fsub64 a b))+(rule (lower (fsub $F32X4 a b)) (pulley_vsubf32x4 a b))+(rule (lower (fsub $F64X2 a b)) (pulley_vsubf64x2 a b));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmul _ a b))) (pulley_fmul32 a b))-(rule (lower (has_type $F64 (fmul _ a b))) (pulley_fmul64 a b))-(rule (lower (has_type $F32X4 (fmul _ a b))) (pulley_vmulf32x4 a b))-(rule (lower (has_type $F64X2 (fmul _ a b))) (pulley_vmulf64x2 a b))+(rule (lower (fmul $F32 a b)) (pulley_fmul32 a b))+(rule (lower (fmul $F64 a b)) (pulley_fmul64 a b))+(rule (lower (fmul $F32X4 a b)) (pulley_vmulf32x4 a b))+(rule (lower (fmul $F64X2 a b)) (pulley_vmulf64x2 a b));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdiv _ a b))) (pulley_fdiv32 a b))-(rule (lower (has_type $F64 (fdiv _ a b))) (pulley_fdiv64 a b))-(rule (lower (has_type $F32X4 (fdiv _ a b))) (pulley_vdivf32x4 a b))-(rule (lower (has_type $F64X2 (fdiv _ a b))) (pulley_vdivf64x2 a b))+(rule (lower (fdiv $F32 a b)) (pulley_fdiv32 a b))+(rule (lower (fdiv $F64 a b)) (pulley_fdiv64 a b))+(rule (lower (fdiv $F32X4 a b)) (pulley_vdivf32x4 a b))+(rule (lower (fdiv $F64X2 a b)) (pulley_vdivf64x2 a b));;;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmax _ a b))) (pulley_fmaximum32 a b))-(rule (lower (has_type $F64 (fmax _ a b))) (pulley_fmaximum64 a b))-(rule (lower (has_type $F32X4 (fmax _ a b))) (pulley_vmaximumf32x4 a b))-(rule (lower (has_type $F64X2 (fmax _ a b))) (pulley_vmaximumf64x2 a b))+(rule (lower (fmax $F32 a b)) (pulley_fmaximum32 a b))+(rule (lower (fmax $F64 a b)) (pulley_fmaximum64 a b))+(rule (lower (fmax $F32X4 a b)) (pulley_vmaximumf32x4 a b))+(rule (lower (fmax $F64X2 a b)) (pulley_vmaximumf64x2 a b));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmin _ a b))) (pulley_fminimum32 a b))-(rule (lower (has_type $F64 (fmin _ a b))) (pulley_fminimum64 a b))-(rule (lower (has_type $F32X4 (fmin _ a b))) (pulley_vminimumf32x4 a b))-(rule (lower (has_type $F64X2 (fmin _ a b))) (pulley_vminimumf64x2 a b))+(rule (lower (fmin $F32 a b)) (pulley_fminimum32 a b))+(rule (lower (fmin $F64 a b)) (pulley_fminimum64 a b))+(rule (lower (fmin $F32X4 a b)) (pulley_vminimumf32x4 a b))+(rule (lower (fmin $F64X2 a b)) (pulley_vminimumf64x2 a b));;;; Rules for `trunc` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (trunc _ a))) (pulley_ftrunc32 a))-(rule (lower (has_type $F64 (trunc _ a))) (pulley_ftrunc64 a))-(rule (lower (has_type $F32X4 (trunc _ a))) (pulley_vtrunc32x4 a))-(rule (lower (has_type $F64X2 (trunc _ a))) (pulley_vtrunc64x2 a))+(rule (lower (trunc $F32 a)) (pulley_ftrunc32 a))+(rule (lower (trunc $F64 a)) (pulley_ftrunc64 a))+(rule (lower (trunc $F32X4 a)) (pulley_vtrunc32x4 a))+(rule (lower (trunc $F64X2 a)) (pulley_vtrunc64x2 a));;;; Rules for `floor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (floor _ a))) (pulley_ffloor32 a))-(rule (lower (has_type $F64 (floor _ a))) (pulley_ffloor64 a))-(rule (lower (has_type $F32X4 (floor _ a)))+(rule (lower (floor $F32 a)) (pulley_ffloor32 a))+(rule (lower (floor $F64 a)) (pulley_ffloor64 a))+(rule (lower (floor $F32X4 a))(pulley_vfloor32x4 a))-(rule (lower (has_type $F64X2 (floor _ a)))+(rule (lower (floor $F64X2 a))(pulley_vfloor64x2 a));;;; Rules for `ceil` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (ceil _ a))) (pulley_fceil32 a))-(rule (lower (has_type $F64 (ceil _ a))) (pulley_fceil64 a))-(rule (lower (has_type $F64X2 (ceil _ a)))+(rule (lower (ceil $F32 a)) (pulley_fceil32 a))+(rule (lower (ceil $F64 a)) (pulley_fceil64 a))+(rule (lower (ceil $F64X2 a))(pulley_vceil64x2 a))-(rule (lower (has_type $F32X4 (ceil _ a)))+(rule (lower (ceil $F32X4 a))(pulley_vceil32x4 a));;;; Rules for `nearest` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (nearest _ a))) (pulley_fnearest32 a))-(rule (lower (has_type $F64 (nearest _ a))) (pulley_fnearest64 a))-(rule (lower (has_type $F32X4 (nearest _ a)))+(rule (lower (nearest $F32 a)) (pulley_fnearest32 a))+(rule (lower (nearest $F64 a)) (pulley_fnearest64 a))+(rule (lower (nearest $F32X4 a))(pulley_vnearest32x4 a))-(rule (lower (has_type $F64X2 (nearest _ a)))+(rule (lower (nearest $F64X2 a))(pulley_vnearest64x2 a));;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (sqrt _ a))) (pulley_fsqrt32 a))-(rule (lower (has_type $F64 (sqrt _ a))) (pulley_fsqrt64 a))-(rule (lower (has_type $F32X4 (sqrt _ a)))+(rule (lower (sqrt $F32 a)) (pulley_fsqrt32 a))+(rule (lower (sqrt $F64 a)) (pulley_fsqrt64 a))+(rule (lower (sqrt $F32X4 a))(pulley_vsqrt32x4 a))-(rule (lower (has_type $F64X2 (sqrt _ a)))+(rule (lower (sqrt $F64X2 a))(pulley_vsqrt64x2 a));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fneg _ a))) (pulley_fneg32 a))-(rule (lower (has_type $F64 (fneg _ a))) (pulley_fneg64 a))-(rule (lower (has_type $F32X4 (fneg _ a))) (pulley_vnegf32x4 a))-(rule (lower (has_type $F64X2 (fneg _ a))) (pulley_vnegf64x2 a))+(rule (lower (fneg $F32 a)) (pulley_fneg32 a))+(rule (lower (fneg $F64 a)) (pulley_fneg64 a))+(rule (lower (fneg $F32X4 a)) (pulley_vnegf32x4 a))+(rule (lower (fneg $F64X2 a)) (pulley_vnegf64x2 a));;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (ineg _ a))) (pulley_xneg32 (sext32 a)))-(rule 1 (lower (has_type $I64 (ineg _ a))) (pulley_xneg64 a))+(rule 0 (lower (ineg (fits_in_32 _) a)) (pulley_xneg32 (sext32 a)))+(rule 1 (lower (ineg $I64 a)) (pulley_xneg64 a));; vector negation-(rule 1 (lower (has_type $I8X16 (ineg _ a))) (pulley_vneg8x16 a))-(rule 1 (lower (has_type $I16X8 (ineg _ a))) (pulley_vneg16x8 a))-(rule 1 (lower (has_type $I32X4 (ineg _ a))) (pulley_vneg32x4 a))-(rule 1 (lower (has_type $I64X2 (ineg _ a))) (pulley_vneg64x2 a))+(rule 1 (lower (ineg $I8X16 a)) (pulley_vneg8x16 a))+(rule 1 (lower (ineg $I16X8 a)) (pulley_vneg16x8 a))+(rule 1 (lower (ineg $I32X4 a)) (pulley_vneg32x4 a))+(rule 1 (lower (ineg $I64X2 a)) (pulley_vneg64x2 a));;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fabs _ a))) (pulley_fabs32 a))-(rule (lower (has_type $F64 (fabs _ a))) (pulley_fabs64 a))-(rule (lower (has_type $F32X4 (fabs _ a))) (pulley_vabsf32x4 a))-(rule (lower (has_type $F64X2 (fabs _ a))) (pulley_vabsf64x2 a))+(rule (lower (fabs $F32 a)) (pulley_fabs32 a))+(rule (lower (fabs $F64 a)) (pulley_fabs64 a))+(rule (lower (fabs $F32X4 a)) (pulley_vabsf32x4 a))+(rule (lower (fabs $F64X2 a)) (pulley_vabsf64x2 a));;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_vec128 _) (vconst _ (u128_from_constant a)))) (pulley_vconst128 a))+(rule (lower (vconst (ty_vec128 _) (u128_from_constant a))) (pulley_vconst128 a));;;; Rules for `bswap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16 (bswap _ a)))+(rule (lower (bswap $I16 a))(if-let (u6_from_u8 shift) (u64_try_into_u8 16))(pulley_xshr32_u_u6 (pulley_bswap32 a) shift))-(rule (lower (has_type $I32 (bswap _ a))) (pulley_bswap32 a))-(rule (lower (has_type $I64 (bswap _ a))) (pulley_bswap64 a))+(rule (lower (bswap $I32 a)) (pulley_bswap32 a))+(rule (lower (bswap $I64 a)) (pulley_bswap64 a));;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 _) (iabs _ a))) (pulley_xabs32 (sext32 a)))-(rule 1 (lower (has_type $I64 (iabs _ a))) (pulley_xabs64 a))-(rule 1 (lower (has_type $I8X16 (iabs _ a))) (pulley_vabs8x16 a))-(rule 1 (lower (has_type $I16X8 (iabs _ a))) (pulley_vabs16x8 a))-(rule 1 (lower (has_type $I32X4 (iabs _ a))) (pulley_vabs32x4 a))-(rule 1 (lower (has_type $I64X2 (iabs _ a))) (pulley_vabs64x2 a))+(rule 0 (lower (iabs (fits_in_32 _) a)) (pulley_xabs32 (sext32 a)))+(rule 1 (lower (iabs $I64 a)) (pulley_xabs64 a))+(rule 1 (lower (iabs $I8X16 a)) (pulley_vabs8x16 a))+(rule 1 (lower (iabs $I16X8 a)) (pulley_vabs16x8 a))+(rule 1 (lower (iabs $I32X4 a)) (pulley_vabs32x4 a))+(rule 1 (lower (iabs $I64X2 a)) (pulley_vabs64x2 a));;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (splat _ a))) (pulley_vsplatx8 a))-(rule (lower (has_type $I16X8 (splat _ a))) (pulley_vsplatx16 a))-(rule (lower (has_type $I32X4 (splat _ a))) (pulley_vsplatx32 a))-(rule (lower (has_type $I64X2 (splat _ a))) (pulley_vsplatx64 a))-(rule (lower (has_type $F32X4 (splat _ a))) (pulley_vsplatf32 a))-(rule (lower (has_type $F64X2 (splat _ a))) (pulley_vsplatf64 a))+(rule (lower (splat $I8X16 a)) (pulley_vsplatx8 a))+(rule (lower (splat $I16X8 a)) (pulley_vsplatx16 a))+(rule (lower (splat $I32X4 a)) (pulley_vsplatx32 a))+(rule (lower (splat $I64X2 a)) (pulley_vsplatx64 a))+(rule (lower (splat $F32X4 a)) (pulley_vsplatf32 a))+(rule (lower (splat $F64X2 a)) (pulley_vsplatf64 a));;;; Rules for `vhigh_bits` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_32 _) (vhigh_bits _ a @ (value_type $I8X16))))+(rule (lower (vhigh_bits (fits_in_32 _) a @ (value_type $I8X16)))(pulley_vbitmask8x16 a))-(rule (lower (has_type (fits_in_32 _) (vhigh_bits _ a @ (value_type $I16X8))))+(rule (lower (vhigh_bits (fits_in_32 _) a @ (value_type $I16X8)))(pulley_vbitmask16x8 a))-(rule (lower (has_type (fits_in_32 _) (vhigh_bits _ a @ (value_type $I32X4))))+(rule (lower (vhigh_bits (fits_in_32 _) a @ (value_type $I32X4)))(pulley_vbitmask32x4 a))-(rule (lower (has_type (fits_in_32 _) (vhigh_bits _ a @ (value_type $I64X2))))+(rule (lower (vhigh_bits (fits_in_32 _) a @ (value_type $I64X2)))(pulley_vbitmask64x2 a));;;; Rules for `vall_true`; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1894,17 +1894,17 @@;;;; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (shuffle _ a b (u128_from_immediate mask))))+(rule (lower (shuffle $I8X16 a b (u128_from_immediate mask)))(pulley_vshuffle a b mask));;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $I8X16 (swizzle _ a b))) (pulley_vswizzlei8x16 a b))+(rule 1 (lower (swizzle $I8X16 a b)) (pulley_vswizzlei8x16 a b));;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32X4 (fma _ a b c))) (pulley_vfma32x4 a b c))-(rule (lower (has_type $F64X2 (fma _ a b c))) (pulley_vfma64x2 a b c))+(rule (lower (fma $F32X4 a b c)) (pulley_vfma32x4 a b c))+(rule (lower (fma $F64X2 a b c)) (pulley_vfma64x2 a b c));; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isleindex 80d812aaf2..8bf905ea51 100644--- a/cranelift/codegen/src/isa/riscv64/inst.isle+++ b/cranelift/codegen/src/isa/riscv64/inst.isle@@ -1981,7 +1981,7 @@;; value (first sign-extending to handle narrow widths).(decl pure partial imm12_from_negated_value (Value) Imm12)(rule- (imm12_from_negated_value (has_type ty (iconst _ n)))+ (imm12_from_negated_value (iconst ty n))(if-let (imm12_from_u64 imm) (i64_cast_unsigned (i64_wrapping_neg (i64_sextend_imm64 ty n))))imm)@@ -2034,7 +2034,7 @@;; Like imm5_from_value, but first negates the `Value`.(decl pure partial imm5_from_negated_value (Value) Imm5)-(rule (imm5_from_negated_value (has_type ty (iconst _ n)))+(rule (imm5_from_negated_value (iconst ty n))(if-let (imm5_from_i64 imm) (i64_wrapping_neg (i64_sextend_imm64 ty n)))imm)@@ -2344,11 +2344,11 @@;; extension is from 32 to 64, 16/8-bit operations are explicitly excluded here.;; There are no native instructions for the 16/8 bit operations so they must;; fall through to actual sign extension above.-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (ishl _ _ _))) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (ushr _ _ _))) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (sshr _ _ _))) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (iadd _ _ _))) true)-(rule 1 (val_already_extended (ExtendOp.Signed) (has_type $I32 (isub _ _ _))) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (ishl $I32 _ _)) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (ushr $I32 _ _)) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (sshr $I32 _ _)) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (iadd $I32 _ _)) true)+(rule 1 (val_already_extended (ExtendOp.Signed) (isub $I32 _ _)) true)(type ExtendOp(enum@@ -2485,7 +2485,7 @@(extractor (sinkable_load inst ty flags addr offset)(and(load _ (little_or_native_endian flags) addr offset)- (sinkable_inst (has_type ty inst))))+ (sinkable_inst (and (result_type ty) inst))));; Returns a canonical type for a LoadOP. We only return I64 or F64.(decl load_op_reg_type (LoadOP) Type)diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isleindex 3f198de859..5566c3ba60 100644--- a/cranelift/codegen/src/isa/riscv64/lower.isle+++ b/cranelift/codegen/src/isa/riscv64/lower.isle@@ -6,12 +6,12 @@;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (iconst _ (u64_from_imm64 n))))+(rule (lower (iconst ty (u64_from_imm64 n)))(imm ty n));; ;;;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty) (vconst _ n)))+(rule (lower (vconst (ty_supported_vec ty) n))(gen_constant ty (const_to_vconst n)));;;; Rules for `f16const` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -41,26 +41,26 @@;;;; Rules for `iadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Base case, simply adding things in registers.-(rule -1 (lower (has_type (fits_in_32 (ty_int ty)) (iadd _ x y)))+(rule -1 (lower (iadd (fits_in_32 (ty_int ty)) x y))(rv_addw x y))-(rule 0 (lower (has_type $I64 (iadd _ x y)))+(rule 0 (lower (iadd $I64 x y))(rv_add x y));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule 1 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ x (imm12_from_value y))))+(rule 1 (lower (iadd (ty_int_ref_scalar_64 ty) x (imm12_from_value y)))(alu_rr_imm12 (select_addi ty) x y))-(rule 2 (lower (has_type (ty_int_ref_scalar_64 ty) (iadd _ (imm12_from_value x) y)))+(rule 2 (lower (iadd (ty_int_ref_scalar_64 ty) (imm12_from_value x) y))(alu_rr_imm12 (select_addi ty) y x));; Special case when one of the operands is uextended;; Needs `Zba`-(rule 3 (lower (has_type $I64 (iadd _ x (uextend _ y @ (value_type $I32)))))+(rule 3 (lower (iadd $I64 x (uextend _ y @ (value_type $I32))))(if-let true (has_zba))(rv_adduw y x))-(rule 4 (lower (has_type $I64 (iadd _ (uextend _ x @ (value_type $I32)) y)))+(rule 4 (lower (iadd $I64 (uextend _ x @ (value_type $I32)) y))(if-let true (has_zba))(rv_adduw x y))@@ -70,12 +70,12 @@(rule (match_shnadd (u64_from_imm64 2)) (AluOPRRR.Sh2add))(rule (match_shnadd (u64_from_imm64 3)) (AluOPRRR.Sh3add))-(rule 3 (lower (has_type $I64 (iadd _ x (ishl _ y (maybe_uextend (iconst _ n))))))+(rule 3 (lower (iadd $I64 x (ishl _ y (maybe_uextend (iconst _ n)))))(if-let true (has_zba))(if-let shnadd (match_shnadd n))(alu_rrr shnadd y x))-(rule 4 (lower (has_type $I64 (iadd _ (ishl _ x (maybe_uextend (iconst _ n))) y)))+(rule 4 (lower (iadd $I64 (ishl _ x (maybe_uextend (iconst _ n))) y))(if-let true (has_zba))(if-let shnadd (match_shnadd n))(alu_rrr shnadd x y))@@ -92,18 +92,18 @@(rule (match_shnadd_uw (u64_from_imm64 2)) (AluOPRRR.Sh2adduw))(rule (match_shnadd_uw (u64_from_imm64 3)) (AluOPRRR.Sh3adduw))-(rule 5 (lower (has_type $I64 (iadd _ x (ishl _ (uextend _ y @ (value_type $I32)) (maybe_uextend (iconst _ n))))))+(rule 5 (lower (iadd $I64 x (ishl _ (uextend _ y @ (value_type $I32)) (maybe_uextend (iconst _ n)))))(if-let true (has_zba))(if-let shnadd_uw (match_shnadd_uw n))(alu_rrr shnadd_uw y x))-(rule 6 (lower (has_type $I64 (iadd _ (ishl _ (uextend _ x @ (value_type $I32)) (maybe_uextend (iconst _ n))) y)))+(rule 6 (lower (iadd $I64 (ishl _ (uextend _ x @ (value_type $I32)) (maybe_uextend (iconst _ n))) y))(if-let true (has_zba))(if-let shnadd_uw (match_shnadd_uw n))(alu_rrr shnadd_uw x y));; I128 cases-(rule 7 (lower (has_type $I128 (iadd _ x y)))+(rule 7 (lower (iadd $I128 x y))(let ((low XReg (rv_add (value_regs_get x 0) (value_regs_get y 0)));; compute carry.(carry XReg (rv_sltu low (value_regs_get y 0)))@@ -114,152 +114,152 @@(value_regs low high)));; SIMD Vectors-(rule 8 (lower (has_type (ty_supported_vec ty) (iadd _ x y)))+(rule 8 (lower (iadd (ty_supported_vec ty) x y))(rv_vadd_vv x y (unmasked) ty))-(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ y))))+(rule 9 (lower (iadd (ty_supported_vec ty) x (splat _ y)))(rv_vadd_vx x y (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ (sextend _ y @ (value_type sext_ty))))))+(rule 10 (lower (iadd (ty_supported_vec ty) x (splat _ (sextend _ y @ (value_type sext_ty)))))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) sext_ty))(rv_vwadd_wx x y (unmasked) (vstate_mf2 half_ty)))-(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (splat _ (uextend _ y @ (value_type uext_ty))))))+(rule 10 (lower (iadd (ty_supported_vec ty) x (splat _ (uextend _ y @ (value_type uext_ty)))))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) uext_ty))(rv_vwaddu_wx x y (unmasked) (vstate_mf2 half_ty)))-(rule 20 (lower (has_type (ty_supported_vec ty) (iadd _ x y)))+(rule 20 (lower (iadd (ty_supported_vec ty) x y))(if-let y_imm (replicated_imm5 y))(rv_vadd_vi x y_imm (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ x) y)))+(rule 12 (lower (iadd (ty_supported_vec ty) (splat _ x) y))(rv_vadd_vx y x (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty))) y)))+(rule 13 (lower (iadd (ty_supported_vec ty) (splat _ (sextend _ x @ (value_type sext_ty))) y))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) sext_ty))(rv_vwadd_wx y x (unmasked) (vstate_mf2 half_ty)))-(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (splat _ (uextend _ x @ (value_type uext_ty))) y)))+(rule 13 (lower (iadd (ty_supported_vec ty) (splat _ (uextend _ x @ (value_type uext_ty))) y))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) uext_ty))(rv_vwaddu_wx y x (unmasked) (vstate_mf2 half_ty)))-(rule 21 (lower (has_type (ty_supported_vec ty) (iadd _ x y)))+(rule 21 (lower (iadd (ty_supported_vec ty) x y))(if-let x_imm (replicated_imm5 x))(rv_vadd_vi y x_imm (unmasked) ty));; Signed Widening Low Additions-(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (swiden_low _ y @ (value_type in_ty)))))+(rule 9 (lower (iadd (ty_supported_vec _) x (swiden_low _ y @ (value_type in_ty))))(rv_vwadd_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty)) y)))+(rule 12 (lower (iadd (ty_supported_vec _) (swiden_low _ x @ (value_type in_ty)) y))(rv_vwadd_wv y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty))- (swiden_low _ y))))+(rule 13 (lower (iadd (ty_supported_vec _) (swiden_low _ x @ (value_type in_ty))+ (swiden_low _ y)))(rv_vwadd_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty))- (splat _ (sextend _ y @ (value_type sext_ty))))))+(rule 13 (lower (iadd (ty_supported_vec _) (swiden_low _ x @ (value_type in_ty))+ (splat _ (sextend _ y @ (value_type sext_ty)))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwadd_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty)))- (swiden_low _ y @ (value_type in_ty)))))+(rule 15 (lower (iadd (ty_supported_vec _) (splat _ (sextend _ x @ (value_type sext_ty)))+ (swiden_low _ y @ (value_type in_ty))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwadd_vx y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Signed Widening High Additions;; These are the same as the low additions, but we first slide down the inputs.-(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (swiden_high _ y @ (value_type in_ty)))))+(rule 9 (lower (iadd (ty_supported_vec _) x (swiden_high _ y @ (value_type in_ty))))(rv_vwadd_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty)) y)))+(rule 12 (lower (iadd (ty_supported_vec _) (swiden_high _ x @ (value_type in_ty)) y))(rv_vwadd_wv y (gen_slidedown_half in_ty x) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty))- (swiden_high _ y))))+(rule 13 (lower (iadd (ty_supported_vec _) (swiden_high _ x @ (value_type in_ty))+ (swiden_high _ y)))(rv_vwadd_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty))- (splat _ (sextend _ y @ (value_type sext_ty))))))+(rule 13 (lower (iadd (ty_supported_vec _) (swiden_high _ x @ (value_type in_ty))+ (splat _ (sextend _ y @ (value_type sext_ty)))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwadd_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (sextend _ x @ (value_type sext_ty)))- (swiden_high _ y @ (value_type in_ty)))))+(rule 15 (lower (iadd (ty_supported_vec _) (splat _ (sextend _ x @ (value_type sext_ty)))+ (swiden_high _ y @ (value_type in_ty))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwadd_vx (gen_slidedown_half in_ty y) x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening Low Additions-(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (uwiden_low _ y @ (value_type in_ty)))))+(rule 9 (lower (iadd (ty_supported_vec _) x (uwiden_low _ y @ (value_type in_ty))))(rv_vwaddu_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty)) y)))+(rule 12 (lower (iadd (ty_supported_vec _) (uwiden_low _ x @ (value_type in_ty)) y))(rv_vwaddu_wv y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty))- (uwiden_low _ y))))+(rule 13 (lower (iadd (ty_supported_vec _) (uwiden_low _ x @ (value_type in_ty))+ (uwiden_low _ y)))(rv_vwaddu_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty))- (splat _ (uextend _ y @ (value_type uext_ty))))))+(rule 13 (lower (iadd (ty_supported_vec _) (uwiden_low _ x @ (value_type in_ty))+ (splat _ (uextend _ y @ (value_type uext_ty)))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwaddu_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (uextend _ x @ (value_type uext_ty)))- (uwiden_low _ y @ (value_type in_ty)))))+(rule 15 (lower (iadd (ty_supported_vec _) (splat _ (uextend _ x @ (value_type uext_ty)))+ (uwiden_low _ y @ (value_type in_ty))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwaddu_vx y x (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening High Additions;; These are the same as the low additions, but we first slide down the inputs.-(rule 9 (lower (has_type (ty_supported_vec _) (iadd _ x (uwiden_high _ y @ (value_type in_ty)))))+(rule 9 (lower (iadd (ty_supported_vec _) x (uwiden_high _ y @ (value_type in_ty))))(rv_vwaddu_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 12 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty)) y)))+(rule 12 (lower (iadd (ty_supported_vec _) (uwiden_high _ x @ (value_type in_ty)) y))(rv_vwaddu_wv y (gen_slidedown_half in_ty x) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty))- (uwiden_high _ y))))+(rule 13 (lower (iadd (ty_supported_vec _) (uwiden_high _ x @ (value_type in_ty))+ (uwiden_high _ y)))(rv_vwaddu_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty))- (splat _ (uextend _ y @ (value_type uext_ty))))))+(rule 13 (lower (iadd (ty_supported_vec _) (uwiden_high _ x @ (value_type in_ty))+ (splat _ (uextend _ y @ (value_type uext_ty)))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwaddu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 15 (lower (has_type (ty_supported_vec _) (iadd _ (splat _ (uextend _ y @ (value_type uext_ty)))- (uwiden_high _ x @ (value_type in_ty)))))+(rule 15 (lower (iadd (ty_supported_vec _) (splat _ (uextend _ y @ (value_type uext_ty)))+ (uwiden_high _ x @ (value_type in_ty))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwaddu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Signed Widening Mixed High/Low Additions-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_low _ x @ (value_type in_ty))- (swiden_high _ y))))+(rule 13 (lower (iadd (ty_supported_vec _) (swiden_low _ x @ (value_type in_ty))+ (swiden_high _ y)))(rv_vwadd_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (swiden_high _ x @ (value_type in_ty))- (swiden_low _ y))))+(rule 13 (lower (iadd (ty_supported_vec _) (swiden_high _ x @ (value_type in_ty))+ (swiden_low _ y)))(rv_vwadd_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening Mixed High/Low Additions-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_low _ x @ (value_type in_ty))- (uwiden_high _ y))))+(rule 13 (lower (iadd (ty_supported_vec _) (uwiden_low _ x @ (value_type in_ty))+ (uwiden_high _ y)))(rv_vwaddu_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 13 (lower (has_type (ty_supported_vec _) (iadd _ (uwiden_high _ x @ (value_type in_ty))- (uwiden_low _ y))))+(rule 13 (lower (iadd (ty_supported_vec _) (uwiden_high _ x @ (value_type in_ty))+ (uwiden_low _ y)))(rv_vwaddu_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Fused Multiply Accumulate Rules `vmacc`@@ -268,46 +268,46 @@;; register instead of the addition one. The actual pattern matched seems to be;; exactly the same.-(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ y z))))+(rule 9 (lower (iadd (ty_supported_vec ty) x (imul _ y z)))(rv_vmacc_vv x y z (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ y (splat _ z)))))+(rule 10 (lower (iadd (ty_supported_vec ty) x (imul _ y (splat _ z))))(rv_vmacc_vx x y z (unmasked) ty))-(rule 11 (lower (has_type (ty_supported_vec ty) (iadd _ x (imul _ (splat _ y) z))))+(rule 11 (lower (iadd (ty_supported_vec ty) x (imul _ (splat _ y) z)))(rv_vmacc_vx x z y (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ x y) z)))+(rule 12 (lower (iadd (ty_supported_vec ty) (imul _ x y) z))(rv_vmacc_vv z x y (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ x (splat _ y)) z)))+(rule 13 (lower (iadd (ty_supported_vec ty) (imul _ x (splat _ y)) z))(rv_vmacc_vx z x y (unmasked) ty))-(rule 14 (lower (has_type (ty_supported_vec ty) (iadd _ (imul _ (splat _ x) y) z)))+(rule 14 (lower (iadd (ty_supported_vec ty) (imul _ (splat _ x) y) z))(rv_vmacc_vx z y x (unmasked) ty));; Fused Multiply Subtract Rules `vnmsac`-(rule 9 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ y z)))))+(rule 9 (lower (iadd (ty_supported_vec ty) x (ineg _ (imul _ y z))))(rv_vnmsac_vv x y z (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ y (splat _ z))))))+(rule 10 (lower (iadd (ty_supported_vec ty) x (ineg _ (imul _ y (splat _ z)))))(rv_vnmsac_vx x y z (unmasked) ty))-(rule 11 (lower (has_type (ty_supported_vec ty) (iadd _ x (ineg _ (imul _ (splat _ y) z)))))+(rule 11 (lower (iadd (ty_supported_vec ty) x (ineg _ (imul _ (splat _ y) z))))(rv_vnmsac_vx x z y (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ x y)) z)))+(rule 12 (lower (iadd (ty_supported_vec ty) (ineg _ (imul _ x y)) z))(rv_vnmsac_vv z x y (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ x (splat _ y))) z)))+(rule 13 (lower (iadd (ty_supported_vec ty) (ineg _ (imul _ x (splat _ y))) z))(rv_vnmsac_vx z x y (unmasked) ty))-(rule 14 (lower (has_type (ty_supported_vec ty) (iadd _ (ineg _ (imul _ (splat _ x) y)) z)))+(rule 14 (lower (iadd (ty_supported_vec ty) (ineg _ (imul _ (splat _ x) y)) z))(rv_vnmsac_vx z y x (unmasked) ty));;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 ty) (uadd_overflow_trap _ x y tc)))+(rule 0 (lower (uadd_overflow_trap (fits_in_32 ty) x y tc))(let ((tmp_x XReg (zext x))(tmp_y XReg (zext y))(sum XReg (rv_add tmp_x tmp_y))@@ -315,7 +315,7 @@(_ InstOutput (gen_trapif (cmp_nez test) tc)))sum))-(rule 1 (lower (has_type $I64 (uadd_overflow_trap _ x y tc)))+(rule 1 (lower (uadd_overflow_trap $I64 x y tc))(let ((tmp XReg (rv_add x y))(_ InstOutput (gen_trapif (cmp_ltu tmp x) tc)))tmp))@@ -323,13 +323,13 @@;;;; Rules for uadd_overflow ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; For i64, we can use the fact that if a + b < a, then overflow occurred-(rule 0 (lower (has_type $I64 (uadd_overflow _ x y)))+(rule 0 (lower (uadd_overflow $I64 x y))(let ((sum XReg (rv_add x y))(overflow XReg (rv_sltu sum x)))(output_pair sum overflow)));; i32 case (on RV64 use addw to detect 32-bit overflow correctly)-(rule 1 (lower (has_type $I32 (uadd_overflow _ x y)))+(rule 1 (lower (uadd_overflow $I32 x y))(let ((x64 XReg (zext x))(y64 XReg (zext y))(sum XReg (rv_add x64 y64))@@ -337,7 +337,7 @@(output_pair sum overflow)));; For i128, we need to handle the high and low parts separately-(rule 2 (lower (has_type $I128 (uadd_overflow _ x y)))+(rule 2 (lower (uadd_overflow $I128 x y))(let ((x_regs ValueRegs x)(y_regs ValueRegs y)(x_lo XReg (value_regs_get x_regs 0))@@ -355,147 +355,147 @@;;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Base case, simply subtracting things in registers.-(rule 0 (lower (has_type (fits_in_32 (ty_int ty)) (isub _ x y)))+(rule 0 (lower (isub (fits_in_32 (ty_int ty)) x y))(rv_subw x y))-(rule 1 (lower (has_type $I64 (isub _ x y)))+(rule 1 (lower (isub $I64 x y))(rv_sub x y))-(rule 2 (lower (has_type $I128 (isub _ x y)))+(rule 2 (lower (isub $I128 x y))(sub_i128 x y));; Switch to an `addi` by a negative if we can fit the value in an `imm12`.-(rule 3 (lower (has_type (ty_int_ref_scalar_64 ty) (isub _ x y)))+(rule 3 (lower (isub (ty_int_ref_scalar_64 ty) x y))(if-let imm12_neg (imm12_from_negated_value y))(alu_rr_imm12 (select_addi ty) x imm12_neg));; SIMD Vectors-(rule 4 (lower (has_type (ty_supported_vec ty) (isub _ x y)))+(rule 4 (lower (isub (ty_supported_vec ty) x y))(rv_vsub_vv x y (unmasked) ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ y))))+(rule 5 (lower (isub (ty_supported_vec ty) x (splat _ y)))(rv_vsub_vx x y (unmasked) ty))-(rule 6 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ (sextend _ y @ (value_type sext_ty))))))+(rule 6 (lower (isub (ty_supported_vec ty) x (splat _ (sextend _ y @ (value_type sext_ty)))))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) sext_ty))(rv_vwsub_wx x y (unmasked) (vstate_mf2 half_ty)))-(rule 6 (lower (has_type (ty_supported_vec ty) (isub _ x (splat _ (uextend _ y @ (value_type uext_ty))))))+(rule 6 (lower (isub (ty_supported_vec ty) x (splat _ (uextend _ y @ (value_type uext_ty)))))(if-let half_ty (ty_half_width ty))(if-let true (ty_equal (lane_type half_ty) uext_ty))(rv_vwsubu_wx x y (unmasked) (vstate_mf2 half_ty)))-(rule 7 (lower (has_type (ty_supported_vec ty) (isub _ (splat _ x) y)))+(rule 7 (lower (isub (ty_supported_vec ty) (splat _ x) y))(rv_vrsub_vx y x (unmasked) ty))-(rule 8 (lower (has_type (ty_supported_vec ty) (isub _ x y)))+(rule 8 (lower (isub (ty_supported_vec ty) x y))(if-let imm5_neg (negated_replicated_imm5 y))(rv_vadd_vi x imm5_neg (unmasked) ty))-(rule 9 (lower (has_type (ty_supported_vec ty) (isub _ x y)))+(rule 9 (lower (isub (ty_supported_vec ty) x y))(if-let x_imm (replicated_imm5 x))(rv_vrsub_vi y x_imm (unmasked) ty));; Signed Widening Low Subtractions-(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (swiden_low _ y @ (value_type in_ty)))))+(rule 6 (lower (isub (ty_supported_vec _) x (swiden_low _ y @ (value_type in_ty))))(rv_vwsub_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty))- (swiden_low _ y))))+(rule 10 (lower (isub (ty_supported_vec _) (swiden_low _ x @ (value_type in_ty))+ (swiden_low _ y)))(rv_vwsub_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty))- (splat _ (sextend _ y @ (value_type sext_ty))))))+(rule 10 (lower (isub (ty_supported_vec _) (swiden_low _ x @ (value_type in_ty))+ (splat _ (sextend _ y @ (value_type sext_ty)))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwsub_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Signed Widening High Subtractions;; These are the same as the low widenings, but we first slide down the inputs.-(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (swiden_high _ y @ (value_type in_ty)))))+(rule 6 (lower (isub (ty_supported_vec _) x (swiden_high _ y @ (value_type in_ty))))(rv_vwsub_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty))- (swiden_high _ y))))+(rule 10 (lower (isub (ty_supported_vec _) (swiden_high _ x @ (value_type in_ty))+ (swiden_high _ y)))(rv_vwsub_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty))- (splat _ (sextend _ y @ (value_type sext_ty))))))+(rule 10 (lower (isub (ty_supported_vec _) (swiden_high _ x @ (value_type in_ty))+ (splat _ (sextend _ y @ (value_type sext_ty)))))(if-let true (ty_equal (lane_type in_ty) sext_ty))(rv_vwsub_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening Low Subtractions-(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (uwiden_low _ y @ (value_type in_ty)))))+(rule 6 (lower (isub (ty_supported_vec _) x (uwiden_low _ y @ (value_type in_ty))))(rv_vwsubu_wv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty))- (uwiden_low _ y))))+(rule 10 (lower (isub (ty_supported_vec _) (uwiden_low _ x @ (value_type in_ty))+ (uwiden_low _ y)))(rv_vwsubu_vv x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty))- (splat _ (uextend _ y @ (value_type uext_ty))))))+(rule 10 (lower (isub (ty_supported_vec _) (uwiden_low _ x @ (value_type in_ty))+ (splat _ (uextend _ y @ (value_type uext_ty)))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwsubu_vx x y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening High Subtractions;; These are the same as the low widenings, but we first slide down the inputs.-(rule 6 (lower (has_type (ty_supported_vec _) (isub _ x (uwiden_high _ y @ (value_type in_ty)))))+(rule 6 (lower (isub (ty_supported_vec _) x (uwiden_high _ y @ (value_type in_ty))))(rv_vwsubu_wv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty))- (uwiden_high _ y))))+(rule 10 (lower (isub (ty_supported_vec _) (uwiden_high _ x @ (value_type in_ty))+ (uwiden_high _ y)))(rv_vwsubu_vv (gen_slidedown_half in_ty x) (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty))- (splat _ (uextend _ y @ (value_type uext_ty))))))+(rule 10 (lower (isub (ty_supported_vec _) (uwiden_high _ x @ (value_type in_ty))+ (splat _ (uextend _ y @ (value_type uext_ty)))))(if-let true (ty_equal (lane_type in_ty) uext_ty))(rv_vwsubu_vx (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Signed Widening Mixed High/Low Subtractions-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_low _ x @ (value_type in_ty))- (swiden_high _ y))))+(rule 10 (lower (isub (ty_supported_vec _) (swiden_low _ x @ (value_type in_ty))+ (swiden_high _ y)))(rv_vwsub_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (swiden_high _ x @ (value_type in_ty))- (swiden_low _ y))))+(rule 10 (lower (isub (ty_supported_vec _) (swiden_high _ x @ (value_type in_ty))+ (swiden_low _ y)))(rv_vwsub_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));; Unsigned Widening Mixed High/Low Subtractions-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_low _ x @ (value_type in_ty))- (uwiden_high _ y))))+(rule 10 (lower (isub (ty_supported_vec _) (uwiden_low _ x @ (value_type in_ty))+ (uwiden_high _ y)))(rv_vwsubu_vv x (gen_slidedown_half in_ty y) (unmasked) (vstate_mf2 (ty_half_lanes in_ty))))-(rule 10 (lower (has_type (ty_supported_vec _) (isub _ (uwiden_high _ x @ (value_type in_ty))- (uwiden_low _ y))))+(rule 10 (lower (isub (ty_supported_vec _) (uwiden_high _ x @ (value_type in_ty))+ (uwiden_low _ y)))(rv_vwsubu_vv (gen_slidedown_half in_ty x) y (unmasked) (vstate_mf2 (ty_half_lanes in_ty))));;;; Rules for `ineg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_int ty) (ineg _ val)))+(rule (lower (ineg (ty_int ty) val))(neg ty val))-(rule 1 (lower (has_type (ty_supported_vec ty) (ineg _ x)))+(rule 1 (lower (ineg (ty_supported_vec ty) x))(rv_vneg_v x (unmasked) ty));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (imul _ x y)))+(rule 0 (lower (imul (ty_int_ref_scalar_64 ty) x y))(rv_mul x y))-(rule 1 (lower (has_type (fits_in_32 (ty_int ty)) (imul _ x y)))+(rule 1 (lower (imul (fits_in_32 (ty_int ty)) x y))(rv_mulw x y));; for I128-(rule 2 (lower (has_type $I128 (imul _ x y)))+(rule 2 (lower (imul $I128 x y))(let((x_regs ValueRegs x)(x_lo XReg (value_regs_get x_regs 0))@@ -523,82 +523,82 @@;; Special case 128-bit multiplication where the operands are extended since;; that maps directly to the `mulhu` and `mulh` instructions.-(rule 6 (lower (has_type $I128 (imul _ (uextend _ x) (uextend _ y))))+(rule 6 (lower (imul $I128 (uextend _ x) (uextend _ y)))(let ((x XReg (zext x))(y XReg (zext y)))(value_regs (rv_mul x y) (rv_mulhu x y))))-(rule 6 (lower (has_type $I128 (imul _ (sextend _ x) (sextend _ y))))+(rule 6 (lower (imul $I128 (sextend _ x) (sextend _ y)))(let ((x XReg (sext x))(y XReg (sext y)))(value_regs (rv_mul x y) (rv_mulh x y))));; Vector multiplication-(rule 3 (lower (has_type (ty_supported_vec ty) (imul _ x y)))+(rule 3 (lower (imul (ty_supported_vec ty) x y))(rv_vmul_vv x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (imul _ (splat _ x) y)))+(rule 4 (lower (imul (ty_supported_vec ty) (splat _ x) y))(rv_vmul_vx y x (unmasked) ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (imul _ x (splat _ y))))+(rule 5 (lower (imul (ty_supported_vec ty) x (splat _ y)))(rv_vmul_vx x y (unmasked) ty));;;; Rules for `smulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (smulhi _ x y)))+(rule 0 (lower (smulhi (ty_int_ref_scalar_64 ty) x y))(lower_smlhi ty (sext x) (sext y)))-(rule 1 (lower (has_type (ty_supported_vec ty) (smulhi _ x y)))+(rule 1 (lower (smulhi (ty_supported_vec ty) x y))(rv_vmulh_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (smulhi _ (splat _ x) y)))+(rule 2 (lower (smulhi (ty_supported_vec ty) (splat _ x) y))(rv_vmulh_vx y x (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (smulhi _ x (splat _ y))))+(rule 3 (lower (smulhi (ty_supported_vec ty) x (splat _ y)))(rv_vmulh_vx x y (unmasked) ty));;;; Rules for `umulhi` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_32 ty) (umulhi _ x y)))+(rule 0 (lower (umulhi (fits_in_32 ty) x y))(let ((tmp XReg (rv_mul (zext x) (zext y))))(rv_srli tmp (imm12_const (ty_bits ty)))))-(rule 1 (lower (has_type $I64 (umulhi _ x y)))+(rule 1 (lower (umulhi $I64 x y))(rv_mulhu x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (umulhi _ x y)))+(rule 2 (lower (umulhi (ty_supported_vec ty) x y))(rv_vmulhu_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (umulhi _ (splat _ x) y)))+(rule 3 (lower (umulhi (ty_supported_vec ty) (splat _ x) y))(rv_vmulhu_vx y x (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (umulhi _ x (splat _ y))))+(rule 4 (lower (umulhi (ty_supported_vec ty) x (splat _ y)))(rv_vmulhu_vx x y (unmasked) ty));;;; Rules for `udiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_16 ty) (udiv _ x y)))+(rule 0 (lower (udiv (fits_in_16 ty) x y))(if-let true (has_m))(rv_divuw (zext x) (nonzero_divisor (zext y))))-(rule 1 (lower (has_type (fits_in_16 ty) (udiv _ x y @ (iconst _ imm))))+(rule 1 (lower (udiv (fits_in_16 ty) x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 ty imm))(rv_divuw (zext x) (zext y)))-(rule 2 (lower (has_type $I32 (udiv _ x y)))+(rule 2 (lower (udiv $I32 x y))(if-let true (has_m))(rv_divuw x (nonzero_divisor (zext y))))-(rule 3 (lower (has_type $I32 (udiv _ x y @ (iconst _ imm))))+(rule 3 (lower (udiv $I32 x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 $I32 imm))(rv_divuw x y))-(rule 2 (lower (has_type $I64 (udiv _ x y)))+(rule 2 (lower (udiv $I64 x y))(if-let true (has_m))(rv_divu x (nonzero_divisor y)))-(rule 3 (lower (has_type $I64 (udiv _ x y @ (iconst _ imm))))+(rule 3 (lower (udiv $I64 x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 $I64 imm))(rv_divu x y))@@ -611,31 +611,31 @@;;;; Rules for `sdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_16 ty) (sdiv _ x y)))+(rule 0 (lower (sdiv (fits_in_16 ty) x y))(if-let true (has_m))(let ((x XReg (sext x)))(rv_divw x (safe_sdiv_divisor ty x (sext y)))))-(rule 1 (lower (has_type (fits_in_16 ty) (sdiv _ x y @ (iconst _ imm))))+(rule 1 (lower (sdiv (fits_in_16 ty) x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 ty imm))(rv_divw (sext x) (sext y)))-(rule 2 (lower (has_type $I32 (sdiv _ x y)))+(rule 2 (lower (sdiv $I32 x y))(if-let true (has_m))(let ((x XReg (sext x)))(rv_divw x (safe_sdiv_divisor $I32 x (sext y)))))-(rule 3 (lower (has_type $I32 (sdiv _ x y @ (iconst _ imm))))+(rule 3 (lower (sdiv $I32 x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 $I32 imm))(rv_divw x y))-(rule 2 (lower (has_type $I64 (sdiv _ x y)))+(rule 2 (lower (sdiv $I64 x y))(if-let true (has_m))(rv_div x (safe_sdiv_divisor $I64 x y)))-(rule 3 (lower (has_type $I64 (sdiv _ x y @ (iconst _ imm))))+(rule 3 (lower (sdiv $I64 x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 $I64 imm))(rv_div x y))@@ -660,84 +660,84 @@;;;; Rules for `urem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_16 ty) (urem _ x y)))+(rule 0 (lower (urem (fits_in_16 ty) x y))(if-let true (has_m))(rv_remuw (zext x) (nonzero_divisor (zext y))))-(rule 1 (lower (has_type (fits_in_16 ty) (urem _ x y @ (iconst _ imm))))+(rule 1 (lower (urem (fits_in_16 ty) x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 ty imm))(rv_remuw (zext x) (zext y)))-(rule 2 (lower (has_type $I32 (urem _ x y)))+(rule 2 (lower (urem $I32 x y))(if-let true (has_m))(rv_remuw x (nonzero_divisor (zext y))))-(rule 3 (lower (has_type $I32 (urem _ x y @ (iconst _ imm))))+(rule 3 (lower (urem $I32 x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 $I32 imm))(rv_remuw x y))-(rule 2 (lower (has_type $I64 (urem _ x y)))+(rule 2 (lower (urem $I64 x y))(if-let true (has_m))(rv_remu x (nonzero_divisor y)))-(rule 3 (lower (has_type $I64 (urem _ x y @ (iconst _ imm))))+(rule 3 (lower (urem $I64 x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 $I64 imm))(rv_remu x y));;;; Rules for `srem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_16 ty) (srem _ x y)))+(rule 0 (lower (srem (fits_in_16 ty) x y))(if-let true (has_m))(rv_remw (sext x) (nonzero_divisor (sext y))))-(rule 1 (lower (has_type (fits_in_16 ty) (srem _ x y @ (iconst _ imm))))+(rule 1 (lower (srem (fits_in_16 ty) x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 ty imm))(rv_remw (sext x) (sext y)))-(rule 2 (lower (has_type $I32 (srem _ x y)))+(rule 2 (lower (srem $I32 x y))(if-let true (has_m))(rv_remw x (nonzero_divisor (sext y))))-(rule 3 (lower (has_type $I32 (srem _ x y @ (iconst _ imm))))+(rule 3 (lower (srem $I32 x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 $I32 imm))(rv_remw x y))-(rule 2 (lower (has_type $I64 (srem _ x y)))+(rule 2 (lower (srem $I64 x y))(if-let true (has_m))(rv_rem x (nonzero_divisor y)))-(rule 3 (lower (has_type $I64 (srem _ x y @ (iconst _ imm))))+(rule 3 (lower (srem $I64 x y @ (iconst _ imm)))(if-let true (has_m))(if (safe_divisor_from_imm64 $I64 imm))(rv_rem x y));;;; Rules for `and` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type (fits_in_64 ty) (band _ x y)))+(rule -1 (lower (band (fits_in_64 ty) x y))(rv_and x y))-(rule 0 (lower (has_type (ty_reg_pair _) (band _ x y)))+(rule 0 (lower (band (ty_reg_pair _) x y))(value_regs(rv_and (value_regs_get x 0) (value_regs_get y 0))(rv_and (value_regs_get x 1) (value_regs_get y 1))));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x (imm12_from_value y))))+(rule 1 (lower (band (fits_in_64 (ty_int ty)) x (imm12_from_value y)))(rv_andi x y))-(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (band _ (imm12_from_value x) y)))+(rule 2 (lower (band (fits_in_64 (ty_int ty)) (imm12_from_value x) y))(rv_andi y x))-(rule 3 (lower (has_type (ty_supported_float_size ty) (band _ x y)))+(rule 3 (lower (band (ty_supported_float_size ty) x y))(lower_float_binary (AluOPRRR.And) x y ty));; No need to NaN-box when moving back to the floating point register as the high;; bits will already be set.-(rule 4 (lower (has_type (ty_supported_float_size $F16) (band _ x y)))+(rule 4 (lower (band (ty_supported_float_size $F16) x y))(if-let false (has_zfhmin))(lower_float_binary (AluOPRRR.And) x y $F32))@@ -745,66 +745,66 @@;; by Cranelift's `band_not` builder method, which emits the simpler;; forms early on.-(rule 5 (lower (has_type (fits_in_64 (ty_int ty)) (band _ x (bnot _ y))))+(rule 5 (lower (band (fits_in_64 (ty_int ty)) x (bnot _ y)))(if-let true (has_zbb))(rv_andn x y))-(rule 6 (lower (has_type (fits_in_64 (ty_int ty)) (band _ (bnot _ y) x)))+(rule 6 (lower (band (fits_in_64 (ty_int ty)) (bnot _ y) x))(if-let true (has_zbb))(rv_andn x y))-(rule 7 (lower (has_type (ty_reg_pair _) (band _ x (bnot _ y))))+(rule 7 (lower (band (ty_reg_pair _) x (bnot _ y)))(if-let true (has_zbb))(let ((low XReg (rv_andn (value_regs_get x 0) (value_regs_get y 0)))(high XReg (rv_andn (value_regs_get x 1) (value_regs_get y 1))))(value_regs low high)))-(rule 8 (lower (has_type (ty_reg_pair _) (band _ (bnot _ y) x)))+(rule 8 (lower (band (ty_reg_pair _) (bnot _ y) x))(if-let true (has_zbb))(let ((low XReg (rv_andn (value_regs_get x 0) (value_regs_get y 0)))(high XReg (rv_andn (value_regs_get x 1) (value_regs_get y 1))))(value_regs low high)))-(rule 9 (lower (has_type (ty_supported_vec ty) (band _ x y)))+(rule 9 (lower (band (ty_supported_vec ty) x y))(rv_vand_vv x y (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (band _ x (splat _ y))))+(rule 10 (lower (band (ty_supported_vec ty) x (splat _ y)))(if (ty_vector_not_float ty))(rv_vand_vx x y (unmasked) ty))-(rule 11 (lower (has_type (ty_supported_vec ty) (band _ (splat _ x) y)))+(rule 11 (lower (band (ty_supported_vec ty) (splat _ x) y))(if (ty_vector_not_float ty))(rv_vand_vx y x (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (band _ x y)))+(rule 12 (lower (band (ty_supported_vec ty) x y))(if-let y_imm (replicated_imm5 y))(rv_vand_vi x y_imm (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (band _ x y)))+(rule 13 (lower (band (ty_supported_vec ty) x y))(if-let x_imm (replicated_imm5 x))(rv_vand_vi y x_imm (unmasked) ty));; `bclr{,i}` specializations from `zbs`-(rule 14 (lower (has_type (fits_in_32 ty) (band _ x (bnot _ (ishl _ (i64_from_iconst 1) y)))))+(rule 14 (lower (band (fits_in_32 ty) x (bnot _ (ishl _ (i64_from_iconst 1) y))))(if-let true (has_zbs))(rv_bclr x (rv_andi y (imm12_const (u8_wrapping_sub (ty_bits ty) 1)))))-(rule 15 (lower (has_type (fits_in_32 ty) (band _ (bnot _ (ishl _ (i64_from_iconst 1) y)) x)))+(rule 15 (lower (band (fits_in_32 ty) (bnot _ (ishl _ (i64_from_iconst 1) y)) x))(if-let true (has_zbs))(rv_bclr x (rv_andi y (imm12_const (u8_wrapping_sub (ty_bits ty) 1)))))-(rule 16 (lower (has_type $I64 (band _ x (bnot _ (ishl _ (i64_from_iconst 1) y)))))+(rule 16 (lower (band $I64 x (bnot _ (ishl _ (i64_from_iconst 1) y))))(if-let true (has_zbs))(rv_bclr x y))-(rule 17 (lower (has_type $I64 (band _ (bnot _ (ishl _ (i64_from_iconst 1) y)) x)))+(rule 17 (lower (band $I64 (bnot _ (ishl _ (i64_from_iconst 1) y)) x))(if-let true (has_zbs))(rv_bclr x y))-(rule 18 (lower (has_type (fits_in_64 ty) (band _ x (u64_from_iconst n))))+(rule 18 (lower (band (fits_in_64 ty) x (u64_from_iconst n)))(if-let true (has_zbs))(if-let imm (bclr_imm ty n))(rv_bclri x imm))-(rule 19 (lower (has_type (fits_in_64 ty) (band _ (u64_from_iconst n) x)))+(rule 19 (lower (band (fits_in_64 ty) (u64_from_iconst n) x))(if-let true (has_zbs))(if-let imm (bclr_imm ty n))(rv_bclri x imm))@@ -814,65 +814,65 @@;; `bext{,i}` specializations from `zbs`-(rule 20 (lower (has_type $I32 (band _ (ushr _ x y) (u64_from_iconst 1))))+(rule 20 (lower (band $I32 (ushr _ x y) (u64_from_iconst 1)))(if-let true (has_zbs))(rv_bext x (rv_andi y (imm12_const 31))))-(rule 20 (lower (has_type $I32 (band _ (sshr _ x y) (u64_from_iconst 1))))+(rule 20 (lower (band $I32 (sshr _ x y) (u64_from_iconst 1)))(if-let true (has_zbs))(rv_bext x (rv_andi y (imm12_const 31))))-(rule 20 (lower (has_type $I32 (band _ (u64_from_iconst 1) (ushr _ x y))))+(rule 20 (lower (band $I32 (u64_from_iconst 1) (ushr _ x y)))(if-let true (has_zbs))(rv_bext x (rv_andi y (imm12_const 31))))-(rule 20 (lower (has_type $I32 (band _ (u64_from_iconst 1) (sshr _ x y))))+(rule 20 (lower (band $I32 (u64_from_iconst 1) (sshr _ x y)))(if-let true (has_zbs))(rv_bext x (rv_andi y (imm12_const 31))))-(rule 20 (lower (has_type $I64 (band _ (ushr _ x y) (u64_from_iconst 1))))+(rule 20 (lower (band $I64 (ushr _ x y) (u64_from_iconst 1)))(if-let true (has_zbs))(rv_bext x y))-(rule 20 (lower (has_type $I64 (band _ (sshr _ x y) (u64_from_iconst 1))))+(rule 20 (lower (band $I64 (sshr _ x y) (u64_from_iconst 1)))(if-let true (has_zbs))(rv_bext x y))-(rule 20 (lower (has_type $I64 (band _ (u64_from_iconst 1) (ushr _ x y))))+(rule 20 (lower (band $I64 (u64_from_iconst 1) (ushr _ x y)))(if-let true (has_zbs))(rv_bext x y))-(rule 20 (lower (has_type $I64 (band _ (u64_from_iconst 1) (sshr _ x y))))+(rule 20 (lower (band $I64 (u64_from_iconst 1) (sshr _ x y)))(if-let true (has_zbs))(rv_bext x y))-(rule 21 (lower (has_type $I32 (band _ (ushr _ x (imm12_from_value y)) (u64_from_iconst 1))))+(rule 21 (lower (band $I32 (ushr _ x (imm12_from_value y)) (u64_from_iconst 1)))(if-let true (has_zbs))(rv_bexti x (imm12_and y 31)))-(rule 21 (lower (has_type $I32 (band _ (sshr _ x (imm12_from_value y)) (u64_from_iconst 1))))+(rule 21 (lower (band $I32 (sshr _ x (imm12_from_value y)) (u64_from_iconst 1)))(if-let true (has_zbs))(rv_bexti x (imm12_and y 31)))-(rule 21 (lower (has_type $I64 (band _ (ushr _ x (imm12_from_value y)) (u64_from_iconst 1))))+(rule 21 (lower (band $I64 (ushr _ x (imm12_from_value y)) (u64_from_iconst 1)))(if-let true (has_zbs))(rv_bexti x (imm12_and y 63)))-(rule 21 (lower (has_type $I64 (band _ (sshr _ x (imm12_from_value y)) (u64_from_iconst 1))))+(rule 21 (lower (band $I64 (sshr _ x (imm12_from_value y)) (u64_from_iconst 1)))(if-let true (has_zbs))(rv_bexti x (imm12_and y 63)));;;; Rules for `or` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -1 (lower (has_type (ty_int ty) (bor _ x y)))+(rule -1 (lower (bor (ty_int ty) x y))(gen_or ty x y))-(rule 0 (lower (has_type $F128 (bor _ x y)))+(rule 0 (lower (bor $F128 x y))(gen_or $I128 x y));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x (imm12_from_value y))))+(rule 1 (lower (bor (fits_in_64 (ty_int ty)) x (imm12_from_value y)))(rv_ori x y))-(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ (imm12_from_value x) y)))+(rule 2 (lower (bor (fits_in_64 (ty_int ty)) (imm12_from_value x) y))(rv_ori y x))-(rule 3 (lower (has_type (ty_supported_float_size ty) (bor _ x y)))+(rule 3 (lower (bor (ty_supported_float_size ty) x y))(lower_float_binary (AluOPRRR.Or) x y ty));; No need to NaN-box when moving back to the floating point register as the high;; bits will already be set.-(rule 4 (lower (has_type (ty_supported_float_size $F16) (bor _ x y)))+(rule 4 (lower (bor (ty_supported_float_size $F16) x y))(if-let false (has_zfhmin))(lower_float_binary (AluOPRRR.Or) x y $F32))@@ -880,66 +880,66 @@;; by Cranelift's `bor_not` builder method, which emits the simpler;; forms early on.-(rule 5 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ x (bnot _ y))))+(rule 5 (lower (bor (fits_in_64 (ty_int ty)) x (bnot _ y)))(if-let true (has_zbb))(rv_orn x y))-(rule 6 (lower (has_type (fits_in_64 (ty_int ty)) (bor _ (bnot _ y) x)))+(rule 6 (lower (bor (fits_in_64 (ty_int ty)) (bnot _ y) x))(if-let true (has_zbb))(rv_orn x y))-(rule 7 (lower (has_type (ty_reg_pair _) (bor _ x (bnot _ y))))+(rule 7 (lower (bor (ty_reg_pair _) x (bnot _ y)))(if-let true (has_zbb))(let ((low XReg (rv_orn (value_regs_get x 0) (value_regs_get y 0)))(high XReg (rv_orn (value_regs_get x 1) (value_regs_get y 1))))(value_regs low high)))-(rule 8 (lower (has_type (ty_reg_pair _) (bor _ (bnot _ y) x)))+(rule 8 (lower (bor (ty_reg_pair _) (bnot _ y) x))(if-let true (has_zbb))(let ((low XReg (rv_orn (value_regs_get x 0) (value_regs_get y 0)))(high XReg (rv_orn (value_regs_get x 1) (value_regs_get y 1))))(value_regs low high)))-(rule 9 (lower (has_type (ty_supported_vec ty) (bor _ x y)))+(rule 9 (lower (bor (ty_supported_vec ty) x y))(rv_vor_vv x y (unmasked) ty))-(rule 10 (lower (has_type (ty_supported_vec ty) (bor _ x (splat _ y))))+(rule 10 (lower (bor (ty_supported_vec ty) x (splat _ y)))(if (ty_vector_not_float ty))(rv_vor_vx x y (unmasked) ty))-(rule 11 (lower (has_type (ty_supported_vec ty) (bor _ (splat _ x) y)))+(rule 11 (lower (bor (ty_supported_vec ty) (splat _ x) y))(if (ty_vector_not_float ty))(rv_vor_vx y x (unmasked) ty))-(rule 12 (lower (has_type (ty_supported_vec ty) (bor _ x y)))+(rule 12 (lower (bor (ty_supported_vec ty) x y))(if-let y_imm (replicated_imm5 y))(rv_vor_vi x y_imm (unmasked) ty))-(rule 13 (lower (has_type (ty_supported_vec ty) (bor _ x y)))+(rule 13 (lower (bor (ty_supported_vec ty) x y))(if-let x_imm (replicated_imm5 x))(rv_vor_vi y x_imm (unmasked) ty));; `bset{,i}` specializations from `zbs`-(rule 14 (lower (has_type $I32 (bor _ x (ishl _ (i64_from_iconst 1) y))))+(rule 14 (lower (bor $I32 x (ishl _ (i64_from_iconst 1) y)))(if-let true (has_zbs))(rv_bset x (rv_andi y (imm12_const 31))))-(rule 15 (lower (has_type $I32 (bor _ (ishl _ (i64_from_iconst 1) y) x)))+(rule 15 (lower (bor $I32 (ishl _ (i64_from_iconst 1) y) x))(if-let true (has_zbs))(rv_bset x (rv_andi y (imm12_const 31))))-(rule 14 (lower (has_type $I64 (bor _ x (ishl _ (i64_from_iconst 1) y))))+(rule 14 (lower (bor $I64 x (ishl _ (i64_from_iconst 1) y)))(if-let true (has_zbs))(rv_bset x y))-(rule 15 (lower (has_type $I64 (bor _ (ishl _ (i64_from_iconst 1) y) x)))+(rule 15 (lower (bor $I64 (ishl _ (i64_from_iconst 1) y) x))(if-let true (has_zbs))(rv_bset x y))-(rule 16 (lower (has_type (fits_in_64 _) (bor _ x (u64_from_iconst n))))+(rule 16 (lower (bor (fits_in_64 _) x (u64_from_iconst n)))(if-let true (has_zbs))(if-let imm (bseti_imm n))(rv_bseti x imm))-(rule 17 (lower (has_type (fits_in_64 _) (bor _ (u64_from_iconst n) x)))+(rule 17 (lower (bor (fits_in_64 _) (u64_from_iconst n) x))(if-let true (has_zbs))(if-let imm (bseti_imm n))(rv_bseti x imm))@@ -948,62 +948,62 @@(extern constructor bseti_imm bseti_imm);;;; Rules for `xor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x y)))+(rule 0 (lower (bxor (fits_in_64 (ty_int ty)) x y))(rv_xor x y));; Special cases for when one operand is an immediate that fits in 12 bits.-(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ x (imm12_from_value y))))+(rule 1 (lower (bxor (fits_in_64 (ty_int ty)) x (imm12_from_value y)))(rv_xori x y))-(rule 2 (lower (has_type (fits_in_64 (ty_int ty)) (bxor _ (imm12_from_value x) y)))+(rule 2 (lower (bxor (fits_in_64 (ty_int ty)) (imm12_from_value x) y))(rv_xori y x))-(rule 3 (lower (has_type (ty_reg_pair _) (bxor _ x y)))+(rule 3 (lower (bxor (ty_reg_pair _) x y))(lower_b128_binary (AluOPRRR.Xor) x y))-(rule 4 (lower (has_type (ty_supported_float_size ty) (bxor _ x y)))+(rule 4 (lower (bxor (ty_supported_float_size ty) x y))(lower_float_binary (AluOPRRR.Xor) x y ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (bxor _ x y)))+(rule 5 (lower (bxor (ty_supported_vec ty) x y))(rv_vxor_vv x y (unmasked) ty))-(rule 6 (lower (has_type (ty_supported_vec ty) (bxor _ x (splat _ y))))+(rule 6 (lower (bxor (ty_supported_vec ty) x (splat _ y)))(if (ty_vector_not_float ty))(rv_vxor_vx x y (unmasked) ty))-(rule 7 (lower (has_type (ty_supported_vec ty) (bxor _ (splat _ x) y)))+(rule 7 (lower (bxor (ty_supported_vec ty) (splat _ x) y))(if (ty_vector_not_float ty))(rv_vxor_vx y x (unmasked) ty))-(rule 8 (lower (has_type (ty_supported_vec ty) (bxor _ x y)))+(rule 8 (lower (bxor (ty_supported_vec ty) x y))(if-let y_imm (replicated_imm5 y))(rv_vxor_vi x y_imm (unmasked) ty))-(rule 9 (lower (has_type (ty_supported_vec ty) (bxor _ x y)))+(rule 9 (lower (bxor (ty_supported_vec ty) x y))(if-let x_imm (replicated_imm5 x))(rv_vxor_vi y x_imm (unmasked) ty));; `binv{,i}` specializations from `zbs`-(rule 13 (lower (has_type $I32 (bxor _ x (ishl _ (i64_from_iconst 1) y))))+(rule 13 (lower (bxor $I32 x (ishl _ (i64_from_iconst 1) y)))(if-let true (has_zbs))(rv_binv x (rv_andi y (imm12_const 31))))-(rule 14 (lower (has_type $I32 (bxor _ (ishl _ (i64_from_iconst 1) y) x)))+(rule 14 (lower (bxor $I32 (ishl _ (i64_from_iconst 1) y) x))(if-let true (has_zbs))(rv_binv x (rv_andi y (imm12_const 31))))-(rule 13 (lower (has_type $I64 (bxor _ x (ishl _ (i64_from_iconst 1) y))))+(rule 13 (lower (bxor $I64 x (ishl _ (i64_from_iconst 1) y)))(if-let true (has_zbs))(rv_binv x y))-(rule 14 (lower (has_type $I64 (bxor _ (ishl _ (i64_from_iconst 1) y) x)))+(rule 14 (lower (bxor $I64 (ishl _ (i64_from_iconst 1) y) x))(if-let true (has_zbs))(rv_binv x y))-(rule 15 (lower (has_type (fits_in_64 _) (bxor _ x (u64_from_iconst n))))+(rule 15 (lower (bxor (fits_in_64 _) x (u64_from_iconst n)))(if-let true (has_zbs))(if-let imm (binvi_imm n))(rv_binvi x imm))-(rule 16 (lower (has_type (fits_in_64 _) (bxor _ (u64_from_iconst n) x)))+(rule 16 (lower (bxor (fits_in_64 _) (u64_from_iconst n) x))(if-let true (has_zbs))(if-let imm (binvi_imm n))(rv_binvi x imm))@@ -1013,30 +1013,30 @@;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int_ref_scalar_64 _) (bnot _ x)))+(rule 0 (lower (bnot (ty_int_ref_scalar_64 _) x))(rv_not x))-(rule 1 (lower (has_type (ty_supported_float_size ty) (bnot _ x)))+(rule 1 (lower (bnot (ty_supported_float_size ty) x))(move_x_to_f (rv_not (move_f_to_x x ty)) ty))-(rule 2 (lower (has_type (ty_reg_pair _) (bnot _ x)))+(rule 2 (lower (bnot (ty_reg_pair _) x))(value_regs(rv_not (value_regs_get x 0))(rv_not (value_regs_get x 1))))-(rule 3 (lower (has_type (ty_supported_vec ty) (bnot _ x)))+(rule 3 (lower (bnot (ty_supported_vec ty) x))(rv_vnot_v x (unmasked) ty))-(rule 4 (lower (has_type (ty_int_ref_scalar_64 _) (bnot _ (bxor _ x y))))+(rule 4 (lower (bnot (ty_int_ref_scalar_64 _) (bxor _ x y)))(if-let true (has_zbb))(rv_xnor x y));;;; Rules for `bit_reverse` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (bitrev _ x)))+(rule 0 (lower (bitrev (ty_int_ref_scalar_64 ty) x))(gen_bitrev ty x))-(rule 1 (lower (has_type $I128 (bitrev _ x)))+(rule 1 (lower (bitrev $I128 x))(value_regs(gen_bitrev $I64 (value_regs_get x 1))(gen_bitrev $I64 (value_regs_get x 0))))@@ -1061,10 +1061,10 @@;;;; Rules for `bswap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type (fits_in_64 (ty_int ty)) (bswap _ x)))+(rule 1 (lower (bswap (fits_in_64 (ty_int ty)) x))(gen_bswap ty x))-(rule 2 (lower (has_type $I128 (bswap _ x)))+(rule 2 (lower (bswap $I128 x))(value_regs(gen_bswap $I64 (value_regs_get x 1))(gen_bswap $I64 (value_regs_get x 0))))@@ -1107,10 +1107,10 @@(rv_rev8 x));;;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (ctz _ x)))+(rule (lower (ctz (fits_in_64 ty) x))(lower_ctz ty x))-(rule 1 (lower (has_type $I128 (ctz _ x)))+(rule 1 (lower (ctz $I128 x))(let ((x_lo XReg (value_regs_get x 0))(x_hi XReg (value_regs_get x 1));; Count both halves@@ -1122,10 +1122,10 @@(value_regs result (imm $I64 0))));;;; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (clz _ x)))+(rule 0 (lower (clz (fits_in_64 ty) x))(gen_cltz true x ty))-(rule 1 (lower (has_type $I128 (clz _ x)))+(rule 1 (lower (clz $I128 x))(let ((x_lo XReg (value_regs_get x 0))(x_hi XReg (value_regs_get x 1));; Count both halves@@ -1135,18 +1135,18 @@(low XReg (gen_select_xreg (cmp_eqz x_hi) low (zero_reg))))(value_regs (rv_add high low) (imm $I64 0))))-(rule 2 (lower (has_type (fits_in_16 ty) (clz _ x)))+(rule 2 (lower (clz (fits_in_16 ty) x))(if-let true (has_zbb))(let ((tmp XReg (zext x))(count XReg (rv_clz tmp)));; We always do the operation on the full 64-bit register, so subtract 64 from the result.(rv_addi count (imm12_const_add (ty_bits ty) -64))))-(rule 3 (lower (has_type $I32 (clz _ x)))+(rule 3 (lower (clz $I32 x))(if-let true (has_zbb))(rv_clzw x))-(rule 3 (lower (has_type $I64 (clz _ x)))+(rule 3 (lower (clz $I64 x))(if-let true (has_zbb))(rv_clz x))@@ -1159,7 +1159,7 @@;;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (cls _ x)))+(rule (lower (cls (fits_in_64 ty) x))(let ((tmp XReg (sext x))(tmp2 XReg (gen_select_xreg (cmp_ltz tmp) (rv_not tmp) tmp))(tmp3 XReg (gen_clz tmp2)))@@ -1170,7 +1170,7 @@;; If the sign bit is set, we count the leading zeros of the inverted value.;; Otherwise we can just count the leading zeros of the original value.;; Subtract 1 since the sign bit does not count.-(rule 1 (lower (has_type $I128 (cls _ x)))+(rule 1 (lower (cls $I128 x))(let ((low XReg (value_regs_get x 0))(high XReg (value_regs_get x 1))(low XReg (gen_select_xreg (cmp_ltz high) (rv_not low) low))@@ -1187,14 +1187,14 @@;;;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 _) (uextend _ val)))+(rule 0 (lower (uextend (fits_in_64 _) val))(zext val))-(rule 1 (lower (has_type $I128 (uextend _ val)))+(rule 1 (lower (uextend $I128 val))(value_regs (zext val) (imm $I64 0)));; When the source of an `uextend` is a load, we can merge both ops-(rule 2 (lower (has_type (fits_in_64 _) (uextend _ (sinkable_load inst ty flags addr offset))))+(rule 2 (lower (uextend (fits_in_64 _) (sinkable_load inst ty flags addr offset)))(gen_sunk_load inst (amode addr offset) (uextend_load_op ty) flags))(decl pure uextend_load_op (Type) LoadOP)@@ -1203,15 +1203,15 @@(rule (uextend_load_op $I32) (LoadOP.Lwu));;;; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 _) (sextend _ val @ (value_type in_ty))))+(rule 0 (lower (sextend (fits_in_64 _) val @ (value_type in_ty)))(sext val))-(rule 1 (lower (has_type $I128 (sextend _ val @ (value_type in_ty))))+(rule 1 (lower (sextend $I128 val @ (value_type in_ty)))(let ((lo XReg (sext val)))(value_regs lo (rv_srai lo (imm12_const 63)))));; When the source of an `sextend` is a load, we can merge both ops-(rule 2 (lower (has_type (fits_in_64 _) (sextend _ (sinkable_load inst ty flags addr offset))))+(rule 2 (lower (sextend (fits_in_64 _) (sinkable_load inst ty flags addr offset)))(gen_sunk_load inst (amode addr offset) (sextend_load_op ty) flags))(decl pure sextend_load_op (Type) LoadOP)@@ -1221,10 +1221,10 @@;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 _) (popcnt _ x)))+(rule 0 (lower (popcnt (fits_in_64 _) x))(gen_popcnt (zext x)))-(rule 1 (lower (has_type $I128 (popcnt _ x)))+(rule 1 (lower (popcnt $I128 x))(let((x ValueRegs x)(low XReg (gen_popcnt (value_regs_get x 0)))@@ -1232,15 +1232,15 @@(result XReg (rv_add low high)))(value_regs result (imm $I64 0))))-(rule 2 (lower (has_type (fits_in_64 _) (popcnt _ x)))+(rule 2 (lower (popcnt (fits_in_64 _) x))(if-let true (has_zbb))(rv_cpop (zext x)))-(rule 3 (lower (has_type $I32 (popcnt _ x)))+(rule 3 (lower (popcnt $I32 x))(if-let true (has_zbb))(rv_cpopw x))-(rule 3 (lower (has_type $I128 (popcnt _ x)))+(rule 3 (lower (popcnt $I128 x))(if-let true (has_zbb))(let((x ValueRegs x)@@ -1261,7 +1261,7 @@;;;; TODO: LLVM generates a much better implementation for I8X16. See: https://godbolt.org/z/qr6vf9Gr3;; For the other types it seems to be largely the same.-(rule 4 (lower (has_type (ty_supported_vec ty) (popcnt _ x)))+(rule 4 (lower (popcnt (ty_supported_vec ty) x))(if-let one (u64_to_uimm5 1))(if-let two (u64_to_uimm5 2))(if-let four (u64_to_uimm5 4))@@ -1296,34 +1296,34 @@;;;; Rules for `ishl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8/16 bit types need a mask on the shift amount-(rule 0 (lower (has_type (ty_int (ty_8_or_16 ty)) (ishl _ x y)))+(rule 0 (lower (ishl (ty_int (ty_8_or_16 ty)) x y))(if-let mask (u64_to_imm12 (ty_shift_mask ty)))(rv_sllw x (rv_andi (value_regs_get y 0) mask)));; Using the 32bit version of `sll` automatically masks the shift amount.-(rule 1 (lower (has_type $I32 (ishl _ x y)))+(rule 1 (lower (ishl $I32 x y))(rv_sllw x (value_regs_get y 0)));; Similarly, the 64bit version does the right thing.-(rule 1 (lower (has_type $I64 (ishl _ x y)))+(rule 1 (lower (ishl $I64 x y))(rv_sll x (value_regs_get y 0)));; If the shift amount is known. We can mask it and encode it in the instruction.-(rule 2 (lower (has_type (int_fits_in_32 ty) (ishl _ x (maybe_uextend (imm12_from_value y)))))+(rule 2 (lower (ishl (int_fits_in_32 ty) x (maybe_uextend (imm12_from_value y))))(rv_slliw x (imm12_and y (ty_shift_mask ty))));; We technically don't need to mask the shift amount here. The instruction;; does the right thing. But it's neater when pretty printing it.-(rule 3 (lower (has_type ty @ $I64 (ishl _ x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (ishl ty @ $I64 x (maybe_uextend (imm12_from_value y))))(rv_slli x (imm12_and y (ty_shift_mask ty))));; With `Zba` we have a shift that zero extends the LHS argument.-(rule 4 (lower (has_type $I64 (ishl _ (uextend _ x @ (value_type $I32)) (maybe_uextend (imm12_from_value y)))))+(rule 4 (lower (ishl $I64 (uextend _ x @ (value_type $I32)) (maybe_uextend (imm12_from_value y))))(if-let true (has_zba))(rv_slliuw x y));; I128 cases-(rule 4 (lower (has_type $I128 (ishl _ x y)))+(rule 4 (lower (ishl $I128 x y))(let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))(len_sub_shamt XReg (value_regs_get tmp 1))@@ -1346,39 +1346,39 @@;; SIMD Cases;; We don't need to mask anything since it is done by the instruction according to SEW.-(rule 5 (lower (has_type (ty_supported_vec ty) (ishl _ x y)))+(rule 5 (lower (ishl (ty_supported_vec ty) x y))(rv_vsll_vx x (value_regs_get y 0) (unmasked) ty))-(rule 6 (lower (has_type (ty_supported_vec ty) (ishl _ x (maybe_uextend (uimm5_from_value y)))))+(rule 6 (lower (ishl (ty_supported_vec ty) x (maybe_uextend (uimm5_from_value y))))(rv_vsll_vi x y (unmasked) ty));;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be;; zero extended.-(rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (ushr _ x y)))+(rule 0 (lower (ushr (ty_int (fits_in_16 ty)) x y))(if-let mask (u64_to_imm12 (ty_shift_mask ty)))(rv_srlw (zext x) (rv_andi (value_regs_get y 0) mask)));; Using the 32bit version of `srl` automatically masks the shift amount.-(rule 1 (lower (has_type $I32 (ushr _ x y)))+(rule 1 (lower (ushr $I32 x y))(rv_srlw x (value_regs_get y 0)));; Similarly, the 64bit version does the right thing.-(rule 1 (lower (has_type $I64 (ushr _ x y)))+(rule 1 (lower (ushr $I64 x y))(rv_srl x (value_regs_get y 0)));; When the RHS is known we can just encode it in the instruction.-(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr _ x (maybe_uextend (imm12_from_value y)))))+(rule 2 (lower (ushr (ty_int (fits_in_16 ty)) x (maybe_uextend (imm12_from_value y))))(rv_srliw (zext x) (imm12_and y (ty_shift_mask ty))))-(rule 3 (lower (has_type $I32 (ushr _ x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (ushr $I32 x (maybe_uextend (imm12_from_value y))))(rv_srliw x y))-(rule 3 (lower (has_type $I64 (ushr _ x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (ushr $I64 x (maybe_uextend (imm12_from_value y))))(rv_srli x y))-(rule 3 (lower (has_type $I128 (ushr _ x y)))+(rule 3 (lower (ushr $I128 x y))(let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))(len_sub_shamt XReg (value_regs_get tmp 1))@@ -1401,39 +1401,39 @@;; SIMD Cases;; We don't need to mask or extend anything since it is done by the instruction according to SEW.-(rule 4 (lower (has_type (ty_supported_vec ty) (ushr _ x y)))+(rule 4 (lower (ushr (ty_supported_vec ty) x y))(rv_vsrl_vx x (value_regs_get y 0) (unmasked) ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (ushr _ x (maybe_uextend (uimm5_from_value y)))))+(rule 5 (lower (ushr (ty_supported_vec ty) x (maybe_uextend (uimm5_from_value y))))(rv_vsrl_vi x y (unmasked) ty));;;; Rules for `sshr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be;; zero extended.-(rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (sshr _ x y)))+(rule 0 (lower (sshr (ty_int (fits_in_16 ty)) x y))(if-let mask (u64_to_imm12 (ty_shift_mask ty)))(rv_sraw (sext x) (rv_andi (value_regs_get y 0) mask)));; Using the 32bit version of `sra` automatically masks the shift amount.-(rule 1 (lower (has_type $I32 (sshr _ x y)))+(rule 1 (lower (sshr $I32 x y))(rv_sraw x (value_regs_get y 0)));; Similarly, the 64bit version does the right thing.-(rule 1 (lower (has_type $I64 (sshr _ x y)))+(rule 1 (lower (sshr $I64 x y))(rv_sra x (value_regs_get y 0)));; When the RHS is known we can just encode it in the instruction.-(rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr _ x (maybe_uextend (imm12_from_value y)))))+(rule 2 (lower (sshr (ty_int (fits_in_16 ty)) x (maybe_uextend (imm12_from_value y))))(rv_sraiw (sext x) (imm12_and y (ty_shift_mask ty))))-(rule 3 (lower (has_type $I32 (sshr _ x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (sshr $I32 x (maybe_uextend (imm12_from_value y))))(rv_sraiw x y))-(rule 3 (lower (has_type $I64 (sshr _ x (maybe_uextend (imm12_from_value y)))))+(rule 3 (lower (sshr $I64 x (maybe_uextend (imm12_from_value y))))(rv_srai x y))-(rule 3 (lower (has_type $I128 (sshr _ x y)))+(rule 3 (lower (sshr $I128 x y))(let ((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))(len_sub_shamt XReg (value_regs_get tmp 1))@@ -1461,16 +1461,16 @@;; SIMD Cases;; We don't need to mask or extend anything since it is done by the instruction according to SEW.-(rule 4 (lower (has_type (ty_supported_vec ty) (sshr _ x y)))+(rule 4 (lower (sshr (ty_supported_vec ty) x y))(rv_vsra_vx x (value_regs_get y 0) (unmasked) ty))-(rule 5 (lower (has_type (ty_supported_vec ty) (sshr _ x (maybe_uextend (uimm5_from_value y)))))+(rule 5 (lower (sshr (ty_supported_vec ty) x (maybe_uextend (uimm5_from_value y))))(rv_vsra_vi x y (unmasked) ty));;;; Rules for `rotl` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (rotl _ rs amount)))+(rule 0 (lower (rotl (fits_in_64 ty) rs amount))(let((rs XReg (zext rs))(amount XReg (value_regs_get amount 0))@@ -1482,25 +1482,25 @@(part3 Reg (gen_select_xreg (cmp_eqz shamt) (zero_reg) part2)))(rv_or part1 part3)))-(rule 1 (lower (has_type $I32 (rotl _ rs amount)))+(rule 1 (lower (rotl $I32 rs amount))(if-let true (has_zbb))(rv_rolw rs (value_regs_get amount 0)))-(rule 2 (lower (has_type $I32 (rotl _ rs (u64_from_iconst n))))+(rule 2 (lower (rotl $I32 rs (u64_from_iconst n)))(if-let true (has_zbb))(if-let (imm12_from_u64 imm) (u64_wrapping_sub 32 (u64_and n 31)))(rv_roriw rs imm))-(rule 1 (lower (has_type $I64 (rotl _ rs amount)))+(rule 1 (lower (rotl $I64 rs amount))(if-let true (has_zbb))(rv_rol rs (value_regs_get amount 0)))-(rule 2 (lower (has_type $I64 (rotl _ rs (u64_from_iconst n))))+(rule 2 (lower (rotl $I64 rs (u64_from_iconst n)))(if-let true (has_zbb))(if-let (imm12_from_u64 imm) (u64_wrapping_sub 64 (u64_and n 63)))(rv_rori rs imm))-(rule 1 (lower (has_type $I128 (rotl _ x y)))+(rule 1 (lower (rotl $I128 x y))(let((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))@@ -1526,7 +1526,7 @@;;;; Rules for `rotr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (rotr _ rs amount)))+(rule (lower (rotr (fits_in_64 ty) rs amount))(let((rs XReg (zext rs))(amount XReg (value_regs_get amount 0))@@ -1538,23 +1538,23 @@(part3 XReg (gen_select_xreg (cmp_eqz shamt) (zero_reg) part2)))(rv_or part1 part3)))-(rule 1 (lower (has_type $I32 (rotr _ rs amount)))+(rule 1 (lower (rotr $I32 rs amount))(if-let true (has_zbb))(rv_rorw rs (value_regs_get amount 0)))-(rule 2 (lower (has_type $I32 (rotr _ rs (imm12_from_value n))))+(rule 2 (lower (rotr $I32 rs (imm12_from_value n)))(if-let true (has_zbb))(rv_roriw rs n))-(rule 1 (lower (has_type $I64 (rotr _ rs amount)))+(rule 1 (lower (rotr $I64 rs amount))(if-let true (has_zbb))(rv_ror rs (value_regs_get amount 0)))-(rule 2 (lower (has_type $I64 (rotr _ rs (imm12_from_value n))))+(rule 2 (lower (rotr $I64 rs (imm12_from_value n)))(if-let true (has_zbb))(rv_rori rs n))-(rule 1 (lower (has_type $I128 (rotr _ x y)))+(rule 1 (lower (rotr $I128 x y))(let((tmp ValueRegs (gen_shamt $I128 (value_regs_get y 0)))(shamt XReg (value_regs_get tmp 0))@@ -1579,27 +1579,27 @@)));;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fabs _ x)))+(rule 0 (lower (fabs (ty_supported_float_full ty) x))(rv_fabs ty x))-(rule 1 (lower (has_type (ty_supported_vec ty) (fabs _ x)))+(rule 1 (lower (fabs (ty_supported_vec ty) x))(rv_vfabs_v x (unmasked) ty));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fneg _ x)))+(rule 0 (lower (fneg (ty_supported_float_full ty) x))(rv_fneg ty x))-(rule 1 (lower (has_type (ty_supported_vec ty) (fneg _ x)))+(rule 1 (lower (fneg (ty_supported_vec ty) x))(rv_vfneg_v x (unmasked) ty));;;; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fcopysign _ x y)))+(rule 0 (lower (fcopysign (ty_supported_float_full ty) x y))(rv_fsgnj ty x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fcopysign _ x y)))+(rule 1 (lower (fcopysign (ty_supported_vec ty) x y))(rv_vfsgnj_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fcopysign _ x (splat _ y))))+(rule 2 (lower (fcopysign (ty_supported_vec ty) x (splat _ y)))(rv_vfsgnj_vf x y (unmasked) ty));;;; Rules for `fma` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1635,7 +1635,7 @@(decl pure get_fneg_value (IsFneg) Value)(rule (get_fneg_value (IsFneg.Result _ v)) v)-(rule (lower (has_type ty (fma _ x_src y_src z_src)))+(rule (lower (fma ty x_src y_src z_src))(let((x_res IsFneg (is_fneg x_src))(y_res IsFneg (is_fneg y_src))@@ -1665,44 +1665,44 @@(rule 3 (rv_fma (ty_supported_vec ty) 1 1 x (splat _ y) z) (rv_vfnmacc_vf z x y (unmasked) ty));;;; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (sqrt _ x)))+(rule 0 (lower (sqrt (ty_supported_float_full ty) x))(rv_fsqrt ty (FRM.RNE) x))-(rule 1 (lower (has_type (ty_supported_vec ty) (sqrt _ x)))+(rule 1 (lower (sqrt (ty_supported_vec ty) x))(rv_vfsqrt_v x (unmasked) ty));;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule -1;;(lower- (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) op addr x)))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) op addr x))(gen_atomic (get_atomic_rmw_op ty op) addr x (atomic_amo)));;; for I8 and I16(rule 1(lower- (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) op addr x)))+ (atomic_rmw (valid_atomic_transaction (fits_in_16 ty)) (little_or_native_endian flags) op addr x))(gen_atomic_rmw_loop op ty addr x));;;special for I8 and I16 max min etc.;;;because I need uextend or sextend the value.(rule 2(lower- (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) (is_atomic_rmw_max_etc op true) addr x)))+ (atomic_rmw (valid_atomic_transaction (fits_in_16 ty)) (little_or_native_endian flags) (is_atomic_rmw_max_etc op true) addr x))(gen_atomic_rmw_loop op ty addr (sext x)))(rule 2;;(lower- (has_type (valid_atomic_transaction (fits_in_16 ty)) (atomic_rmw _ (little_or_native_endian flags) (is_atomic_rmw_max_etc op false) addr x)))+ (atomic_rmw (valid_atomic_transaction (fits_in_16 ty)) (little_or_native_endian flags) (is_atomic_rmw_max_etc op false) addr x));;(gen_atomic_rmw_loop op ty addr (zext x)));;;;; Rules for `AtomicRmwOp.Sub`(rule(lower- (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) addr x)))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Sub) addr x))(let((tmp WritableReg (temp_writable_reg ty))(x2 Reg (rv_neg x)))@@ -1720,7 +1720,7 @@;;;;; Rules for `AtomicRmwOp.Nand`(rule(lower- (has_type (valid_atomic_transaction ty) (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Nand) addr x)))+ (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Nand) addr x))(gen_atomic_rmw_loop (AtomicRmwOp.Nand) ty addr x))(decl is_atomic_rmw_max_etc (AtomicRmwOp bool) AtomicRmwOp)@@ -1728,7 +1728,7 @@;;;;; Rules for `atomic load`;;;;;;;;;;;;;;;;;(rule- (lower (has_type (valid_atomic_transaction ty) (atomic_load _ (little_or_native_endian flags) p)))+ (lower (atomic_load (valid_atomic_transaction ty) (little_or_native_endian flags) p))(gen_atomic_load p ty))@@ -1754,7 +1754,7 @@;;;;; Rules for `atomic cas`;;;;;;;;;;;;;;;;;(rule- (lower (has_type (valid_atomic_transaction ty) (atomic_cas _ (little_or_native_endian flags) p e x)))+ (lower (atomic_cas (valid_atomic_transaction ty) (little_or_native_endian flags) p e x))(let((t0 WritableReg (temp_writable_reg ty))(dst WritableReg (temp_writable_reg ty))@@ -1763,7 +1763,7 @@;;;;; Rules for `ireduce`;;;;;;;;;;;;;;;;;(rule- (lower (has_type ty (ireduce _ x)))+ (lower (ireduce ty x))(value_regs_get x 0));;;;; Rules for `fpromote`;;;;;;;;;;;;;;;;;@@ -1772,7 +1772,7 @@;;;;; Rules for `fvpromote_low`;;;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty) (fvpromote_low _ x)))+(rule (lower (fvpromote_low (ty_supported_vec ty) x))(if-let half_ty (ty_half_width ty))(rv_vfwcvt_f_f_v x (unmasked) (vstate_mf2 half_ty)))@@ -1784,7 +1784,7 @@;; `vfncvt...` leaves the upper bits of the register undefined so;; we need to zero them out.-(rule (lower (has_type (ty_supported_vec ty @ $F32X4) (fvdemote _ x)))+(rule (lower (fvdemote (ty_supported_vec ty @ $F32X4) x))(if-let zero (i8_to_imm5 0))(let ((narrow VReg (rv_vfncvt_f_f_w x (unmasked) (vstate_mf2 ty)))(mask VReg (gen_vec_mask 0xC)))@@ -1796,57 +1796,57 @@;;;; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fadd _ x y)))+(rule 0 (lower (fadd (ty_supported_float_full ty) x y))(rv_fadd ty (FRM.RNE) x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fadd _ x y)))+(rule 1 (lower (fadd (ty_supported_vec ty) x y))(rv_vfadd_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fadd _ x (splat _ y))))+(rule 2 (lower (fadd (ty_supported_vec ty) x (splat _ y)))(rv_vfadd_vf x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (fadd _ (splat _ x) y)))+(rule 3 (lower (fadd (ty_supported_vec ty) (splat _ x) y))(rv_vfadd_vf y x (unmasked) ty));;;; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fsub _ x y)))+(rule 0 (lower (fsub (ty_supported_float_full ty) x y))(rv_fsub ty (FRM.RNE) x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fsub _ x y)))+(rule 1 (lower (fsub (ty_supported_vec ty) x y))(rv_vfsub_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fsub _ x (splat _ y))))+(rule 2 (lower (fsub (ty_supported_vec ty) x (splat _ y)))(rv_vfsub_vf x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (fsub _ (splat _ x) y)))+(rule 3 (lower (fsub (ty_supported_vec ty) (splat _ x) y))(rv_vfrsub_vf y x (unmasked) ty));;;; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fmul _ x y)))+(rule 0 (lower (fmul (ty_supported_float_full ty) x y))(rv_fmul ty (FRM.RNE) x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fmul _ x y)))+(rule 1 (lower (fmul (ty_supported_vec ty) x y))(rv_vfmul_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fmul _ x (splat _ y))))+(rule 2 (lower (fmul (ty_supported_vec ty) x (splat _ y)))(rv_vfmul_vf x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (fmul _ (splat _ x) y)))+(rule 3 (lower (fmul (ty_supported_vec ty) (splat _ x) y))(rv_vfmul_vf y x (unmasked) ty));;;; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (fdiv _ x y)))+(rule 0 (lower (fdiv (ty_supported_float_full ty) x y))(rv_fdiv ty (FRM.RNE) x y))-(rule 1 (lower (has_type (ty_supported_vec ty) (fdiv _ x y)))+(rule 1 (lower (fdiv (ty_supported_vec ty) x y))(rv_vfdiv_vv x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (fdiv _ x (splat _ y))))+(rule 2 (lower (fdiv (ty_supported_vec ty) x (splat _ y)))(rv_vfdiv_vf x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (fdiv _ (splat _ x) y)))+(rule 3 (lower (fdiv (ty_supported_vec ty) (splat _ x) y))(rv_vfrdiv_vf y x (unmasked) ty));;;; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1854,7 +1854,7 @@;; RISC-V's `fmin` instruction returns the number input if one of inputs is a;; NaN. We handle this by manually checking if one of the inputs is a NaN;; and selecting based on that result.-(rule 0 (lower (has_type (ty_supported_float_full ty) (fmin _ x y)))+(rule 0 (lower (fmin (ty_supported_float_full ty) x y))(let (;; Check if both inputs are not nan.(is_ordered FloatCompare (fcmp_to_float_compare (FloatCC.Ordered) ty x y))@@ -1865,7 +1865,7 @@;; With Zfa we can use the special `fminm` that precisely matches the expected;; NaN behavior.-(rule 1 (lower (has_type (ty_supported_float_full ty) (fmin _ x y)))+(rule 1 (lower (fmin (ty_supported_float_full ty) x y))(if-let true (has_zfa))(rv_fminm ty x y))@@ -1875,7 +1875,7 @@;;;; TODO: We can improve this by using a masked `fmin` instruction that modifies;; the canonical nan register. That way we could avoid the `vmerge.vv` instruction.-(rule 2 (lower (has_type (ty_supported_vec ty) (fmin _ x y)))+(rule 2 (lower (fmin (ty_supported_vec ty) x y))(let ((is_not_nan VReg (gen_fcmp_mask ty (FloatCC.Ordered) x y))(nan XReg (imm $I64 (canonical_nan_u64 (lane_type ty))))(vec_nan VReg (rv_vmv_vx nan ty))@@ -1887,7 +1887,7 @@;; RISC-V's `fmax` instruction returns the number input if one of inputs is a;; NaN. We handle this by manually checking if one of the inputs is a NaN;; and selecting based on that result.-(rule 0 (lower (has_type (ty_supported_float_full ty) (fmax _ x y)))+(rule 0 (lower (fmax (ty_supported_float_full ty) x y))(let (;; Check if both inputs are not nan.(is_ordered FloatCompare (fcmp_to_float_compare (FloatCC.Ordered) ty x y))@@ -1898,7 +1898,7 @@;; With Zfa we can use the special `fmaxm` that precisely matches the expected;; NaN behavior.-(rule 1 (lower (has_type (ty_supported_float_full ty) (fmax _ x y)))+(rule 1 (lower (fmax (ty_supported_float_full ty) x y))(if-let true (has_zfa))(rv_fmaxm ty x y))@@ -1908,7 +1908,7 @@;;;; TODO: We can improve this by using a masked `fmax` instruction that modifies;; the canonical nan register. That way we could avoid the `vmerge.vv` instruction.-(rule 2 (lower (has_type (ty_supported_vec ty) (fmax _ x y)))+(rule 2 (lower (fmax (ty_supported_vec ty) x y))(let ((is_not_nan VReg (gen_fcmp_mask ty (FloatCC.Ordered) x y))(nan XReg (imm $I64 (canonical_nan_u64 (lane_type ty))))(vec_nan VReg (rv_vmv_vx nan ty))@@ -1926,28 +1926,28 @@;; of the iconst rule because that runs into regalloc issues. gen_select_xreg;; has some optimizations based on the use of the zero register so we have to;; manually match it here.-(rule 5 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c (i64_from_iconst 0) y)))+(rule 5 (lower (select (ty_int_ref_scalar_64 _) c (i64_from_iconst 0) y))(gen_select_xreg (is_nonzero_cmp c) (zero_reg) y))-(rule 4 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c x (i64_from_iconst 0))))+(rule 4 (lower (select (ty_int_ref_scalar_64 _) c x (i64_from_iconst 0)))(gen_select_xreg (is_nonzero_cmp c) x (zero_reg)))-(rule 3 (lower (has_type (ty_int_ref_scalar_64 _) (select _ c x y)))+(rule 3 (lower (select (ty_int_ref_scalar_64 _) c x y))(gen_select_xreg (is_nonzero_cmp c) x y))-(rule 2 (lower (has_type (ty_reg_pair _) (select _ c x y)))+(rule 2 (lower (select (ty_reg_pair _) c x y))(gen_select_regs (is_nonzero_cmp c) x y))-(rule 1 (lower (has_type (ty_supported_vec _) (select _ c x y)))+(rule 1 (lower (select (ty_supported_vec _) c x y))(gen_select_vreg (is_nonzero_cmp c) x y))-(rule 0 (lower (has_type (ty_supported_float_size _) (select _ c x y)))+(rule 0 (lower (select (ty_supported_float_size _) c x y))(gen_select_freg (is_nonzero_cmp c) x y));;;;; Rules for `bitselect`;;;;;;;;;;; Do a (c & x) | (~c & y) operation.-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (bitselect _ c x y)))+(rule 0 (lower (bitselect (ty_int_ref_scalar_64 ty) c x y))(let ((tmp_x XReg (rv_and c x))(c_inverse XReg (rv_not c))(tmp_y XReg (rv_and c_inverse y)))@@ -1958,7 +1958,7 @@;; using the type of the inputs so that we avoid emitting unnecessary;; `vsetvl` instructions. it's likely that the vector unit is already;; configured for that type.-(rule 1 (lower (has_type (ty_supported_vec ty) (bitselect _ c x y)))+(rule 1 (lower (bitselect (ty_supported_vec ty) c x y))(let ((tmp_x VReg (rv_vand_vv c x (unmasked) ty))(c_inverse VReg (rv_vnot_v c (unmasked) ty))(tmp_y VReg (rv_vand_vv c_inverse y (unmasked) ty)))@@ -1976,19 +1976,19 @@;;;; See: https://github.com/bytecodealliance/wasmtime/issues/8131-(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (icmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y)))+(rule 2 (lower (bitselect (ty_supported_vec _ty) (icmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y))(let ((mask VReg (gen_icmp_mask cmp_ty cc a b)))(rv_vmerge_vvm y x mask cmp_ty)))-(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (fcmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y)))+(rule 2 (lower (bitselect (ty_supported_vec _ty) (fcmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b) x y))(let ((mask VReg (gen_fcmp_mask cmp_ty cc a b)))(rv_vmerge_vvm y x mask cmp_ty)))-(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (bitcast _ _ (fcmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y)))+(rule 2 (lower (bitselect (ty_supported_vec _ty) (bitcast _ _ (fcmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y))(let ((mask VReg (gen_fcmp_mask cmp_ty cc a b)))(rv_vmerge_vvm y x mask cmp_ty)))-(rule 2 (lower (has_type (ty_supported_vec _ty) (bitselect _ (bitcast _ _ (icmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y)))+(rule 2 (lower (bitselect (ty_supported_vec _ty) (bitcast _ _ (icmp _ cc a @ (value_type (ty_supported_vec cmp_ty)) b)) x y))(let ((mask VReg (gen_icmp_mask cmp_ty cc a b)))(rv_vmerge_vvm y x mask cmp_ty)))@@ -2003,7 +2003,7 @@;;;;; Rules for `iconcat`;;;;;;;;;(rule- (lower (has_type $I128 (iconcat _ x y)))+ (lower (iconcat $I128 x y))(let((t1 XReg x)(t2 XReg y))@@ -2012,90 +2012,90 @@;; Special-case the lowering of an `isplit` of a 128-bit multiply where the;; lower bits of the result are discarded and the operands are sign or zero;; extended. This maps directly to `umulh` and `smulh`.-(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (uextend _ x) (uextend _ y)))))+(rule 1 (lower i @ (isplit _ (imul $I128 (uextend _ x) (uextend _ y))))(if-let (first_result lo) i)(if-let true (value_is_unused lo))(output_pair (invalid_reg) (rv_mulhu (zext x) (zext y))))-(rule 1 (lower i @ (isplit _ (has_type $I128 (imul _ (sextend _ x) (sextend _ y)))))+(rule 1 (lower i @ (isplit _ (imul $I128 (sextend _ x) (sextend _ y))))(if-let (first_result lo) i)(if-let true (value_is_unused lo))(output_pair (invalid_reg) (rv_mulh (sext x) (sext y))));;;;; Rules for `smax`;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (smax _ x y)))+(rule 0 (lower (smax (fits_in_64 ty) x y))(let ((x XReg (sext x))(y XReg (sext y)))(gen_select_xreg (cmp_gt x y) x y)))-(rule 1 (lower (has_type $I128 (smax _ x y)))+(rule 1 (lower (smax $I128 x y))(gen_select_regs (icmp_to_int_compare (IntCC.SignedGreaterThan) x y) x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (smax _ x y)))+(rule 2 (lower (smax (ty_supported_vec ty) x y))(rv_vmax_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (smax _ x (splat _ y))))+(rule 3 (lower (smax (ty_supported_vec ty) x (splat _ y)))(rv_vmax_vx x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (smax _ (splat _ x) y)))+(rule 4 (lower (smax (ty_supported_vec ty) (splat _ x) y))(rv_vmax_vx y x (unmasked) ty));;;;; Rules for `smin`;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (smin _ x y)))+(rule 0 (lower (smin (fits_in_64 ty) x y))(let ((x XReg (sext x))(y XReg (sext y)))(gen_select_xreg (cmp_lt x y) x y)))-(rule 1 (lower (has_type $I128 (smin _ x y)))+(rule 1 (lower (smin $I128 x y))(gen_select_regs (icmp_to_int_compare (IntCC.SignedLessThan) x y) x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (smin _ x y)))+(rule 2 (lower (smin (ty_supported_vec ty) x y))(rv_vmin_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (smin _ x (splat _ y))))+(rule 3 (lower (smin (ty_supported_vec ty) x (splat _ y)))(rv_vmin_vx x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (smin _ (splat _ x) y)))+(rule 4 (lower (smin (ty_supported_vec ty) (splat _ x) y))(rv_vmin_vx y x (unmasked) ty));;;;; Rules for `umax`;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (umax _ x y)))+(rule 0 (lower (umax (fits_in_64 ty) x y))(let ((x XReg (zext x))(y XReg (zext y)))(gen_select_xreg (cmp_gtu x y) x y)))-(rule 1 (lower (has_type $I128 (umax _ x y)))+(rule 1 (lower (umax $I128 x y))(gen_select_regs (icmp_to_int_compare (IntCC.UnsignedGreaterThan) x y) x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (umax _ x y)))+(rule 2 (lower (umax (ty_supported_vec ty) x y))(rv_vmaxu_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (umax _ x (splat _ y))))+(rule 3 (lower (umax (ty_supported_vec ty) x (splat _ y)))(rv_vmaxu_vx x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (umax _ (splat _ x) y)))+(rule 4 (lower (umax (ty_supported_vec ty) (splat _ x) y))(rv_vmaxu_vx y x (unmasked) ty));;;;; Rules for `umin`;;;;;;;;;-(rule 0 (lower (has_type (fits_in_64 ty) (umin _ x y)))+(rule 0 (lower (umin (fits_in_64 ty) x y))(let ((x XReg (zext x))(y XReg (zext y)))(gen_select_xreg (cmp_ltu x y) x y)))-(rule 1 (lower (has_type $I128 (umin _ x y)))+(rule 1 (lower (umin $I128 x y))(gen_select_regs (icmp_to_int_compare (IntCC.UnsignedLessThan) x y) x y))-(rule 2 (lower (has_type (ty_supported_vec ty) (umin _ x y)))+(rule 2 (lower (umin (ty_supported_vec ty) x y))(rv_vminu_vv x y (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (umin _ x (splat _ y))))+(rule 3 (lower (umin (ty_supported_vec ty) x (splat _ y)))(rv_vminu_vx x y (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (umin _ (splat _ x) y)))+(rule 4 (lower (umin (ty_supported_vec ty) (splat _ x) y))(rv_vminu_vx y x (unmasked) ty))@@ -2147,16 +2147,16 @@(gen_load (amode addr offset) (LoadOP.Lw) flags));;;;; Rules for `load`;;;;;;;;;-(rule (lower (has_type ty (load _ (little_or_native_endian flags) addr offset)))+(rule (lower (load ty (little_or_native_endian flags) addr offset))(gen_load (amode addr offset) (load_op ty) flags))-(rule 1 (lower (has_type (ty_reg_pair _) (load _ (little_or_native_endian flags) addr offset)))+(rule 1 (lower (load (ty_reg_pair _) (little_or_native_endian flags) addr offset))(if-let offset_plus_8 (i32_checked_add offset 8))(let ((lo XReg (gen_load (amode addr offset) (LoadOP.Ld) flags))(hi XReg (gen_load (amode addr offset_plus_8) (LoadOP.Ld) flags)))(value_regs lo hi)))-(rule 2 (lower (has_type (ty_supported_vec ty) (load _ (little_or_native_endian flags) addr offset)))+(rule 2 (lower (load (ty_supported_vec ty) (little_or_native_endian flags) addr offset))(let ((eew VecElementWidth (element_width_from_type ty))(amode AMode (amode addr offset)))(vec_load eew (VecAMode.UnitStride amode) flags (unmasked) ty)))@@ -2183,27 +2183,27 @@(rv_vzext_vf2 loaded (unmasked) ty)));;;;; Rules for `uload8x8`;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I16X8) (uload8x8 _ (little_or_native_endian flags) addr offset)))+(rule (lower (uload8x8 (ty_supported_vec ty @ $I16X8) (little_or_native_endian flags) addr offset))(gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)));;;;; Rules for `uload16x4`;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I32X4) (uload16x4 _ (little_or_native_endian flags) addr offset)))+(rule (lower (uload16x4 (ty_supported_vec ty @ $I32X4) (little_or_native_endian flags) addr offset))(gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)));;;;; Rules for `uload32x2`;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I64X2) (uload32x2 _ (little_or_native_endian flags) addr offset)))+(rule (lower (uload32x2 (ty_supported_vec ty @ $I64X2) (little_or_native_endian flags) addr offset))(gen_load64_extend ty (ExtendOp.Zero) flags (amode addr offset)));;;;; Rules for `sload8x8`;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I16X8) (sload8x8 _ (little_or_native_endian flags) addr offset)))+(rule (lower (sload8x8 (ty_supported_vec ty @ $I16X8) (little_or_native_endian flags) addr offset))(gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)));;;;; Rules for `sload16x4`;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I32X4) (sload16x4 _ (little_or_native_endian flags) addr offset)))+(rule (lower (sload16x4 (ty_supported_vec ty @ $I32X4) (little_or_native_endian flags) addr offset))(gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)));;;;; Rules for `sload32x2`;;;;;;;;;-(rule (lower (has_type (ty_supported_vec ty @ $I64X2) (sload32x2 _ (little_or_native_endian flags) addr offset)))+(rule (lower (sload32x2 (ty_supported_vec ty @ $I64X2) (little_or_native_endian flags) addr offset))(gen_load64_extend ty (ExtendOp.Signed) flags (amode addr offset)));;;;; Rules for `istore8`;;;;;;;;;@@ -2410,7 +2410,7 @@;; TODO: could this perhaps be more optimal through inspection of the `fcsr`?;; Unsure whether that needs to be preserved across function calls and/or would;; cause other problems. Also unsure whether it's actually more performant.-(rule (lower (has_type ity (fcvt_to_uint _ v @ (value_type fty))))+(rule (lower (fcvt_to_uint ity v @ (value_type fty)))(let ((_ InstOutput (gen_trapif (cmp_eqz (rv_feq fty v v)) (TrapCode.BAD_CONVERSION_TO_INTEGER)))(min FReg (imm fty (fcvt_umin_bound fty false)))(_ InstOutput (gen_trapif (cmp_nez (rv_fle fty v min)) (TrapCode.INTEGER_OVERFLOW)))@@ -2427,7 +2427,7 @@;;;;; Rules for `fcvt_to_sint`;;;;;;;;;;; NB: see above with `fcvt_to_uint` as this is similar-(rule (lower (has_type ity (fcvt_to_sint _ v @ (value_type fty))))+(rule (lower (fcvt_to_sint ity v @ (value_type fty)))(let ((_ InstOutput (gen_trapif (cmp_eqz (rv_feq fty v v)) (TrapCode.BAD_CONVERSION_TO_INTEGER)))(min FReg (imm fty (fcvt_smin_bound fty ity false)))(_ InstOutput (gen_trapif (cmp_nez (rv_fle fty v min)) (TrapCode.INTEGER_OVERFLOW)))@@ -2443,7 +2443,7 @@;;;;; Rules for `fcvt_to_sint_sat`;;;;;;;;;-(rule 0 (lower (has_type to (fcvt_to_sint_sat _ v @ (value_type (ty_supported_float_full from)))))+(rule 0 (lower (fcvt_to_sint_sat to v @ (value_type (ty_supported_float_full from))))(handle_fcvt_to_int_nan from v (lower_fcvt_to_sint_sat from to v)));; Lowers to a `rv_fcvt*` instruction but handles 8/16-bit cases where the@@ -2478,7 +2478,7 @@(not_nan_mask XReg (rv_neg is_not_nan)))(rv_and xreg not_nan_mask)))-(rule 1 (lower (has_type (ty_supported_vec _) (fcvt_to_sint_sat _ v @ (value_type from_ty))))+(rule 1 (lower (fcvt_to_sint_sat (ty_supported_vec _) v @ (value_type from_ty)))(if-let zero (i8_to_imm5 0))(let ((is_nan VReg (rv_vmfne_vv v v (unmasked) from_ty))(cvt VReg (rv_vfcvt_rtz_x_f_v v (unmasked) from_ty)))@@ -2486,7 +2486,7 @@;;;;; Rules for `fcvt_to_uint_sat`;;;;;;;;;-(rule 0 (lower (has_type to (fcvt_to_uint_sat _ v @ (value_type (ty_supported_float_full from)))))+(rule 0 (lower (fcvt_to_uint_sat to v @ (value_type (ty_supported_float_full from))))(handle_fcvt_to_int_nan from v (lower_fcvt_to_uint_sat from to v)));; Lowers to a `rv_fcvt*` instruction but handles 8/16-bit cases where the@@ -2505,54 +2505,54 @@(decl fcvt_umin_bound (Type bool) u64)(extern constructor fcvt_umin_bound fcvt_umin_bound)-(rule 1 (lower (has_type (ty_supported_vec _) (fcvt_to_uint_sat _ v @ (value_type from_ty))))+(rule 1 (lower (fcvt_to_uint_sat (ty_supported_vec _) v @ (value_type from_ty)))(if-let zero (i8_to_imm5 0))(let ((is_nan VReg (rv_vmfne_vv v v (unmasked) from_ty))(cvt VReg (rv_vfcvt_rtz_xu_f_v v (unmasked) from_ty)))(rv_vmerge_vim cvt zero is_nan from_ty)));;;;; Rules for `fcvt_from_sint`;;;;;;;;;-(rule 0 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type (fits_in_16 ty)))))+(rule 0 (lower (fcvt_from_sint $F32 v @ (value_type (fits_in_16 ty))))(rv_fcvtsl (FRM.RNE) (sext v)))-(rule 1 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type $I32))))+(rule 1 (lower (fcvt_from_sint $F32 v @ (value_type $I32)))(rv_fcvtsw (FRM.RNE) v))-(rule 1 (lower (has_type $F32 (fcvt_from_sint _ v @ (value_type $I64))))+(rule 1 (lower (fcvt_from_sint $F32 v @ (value_type $I64)))(rv_fcvtsl (FRM.RNE) v))-(rule 0 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type (fits_in_16 ty)))))+(rule 0 (lower (fcvt_from_sint $F64 v @ (value_type (fits_in_16 ty))))(rv_fcvtdl (FRM.RNE) (sext v)))-(rule 1 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type $I32))))+(rule 1 (lower (fcvt_from_sint $F64 v @ (value_type $I32)))(rv_fcvtdw v))-(rule 1 (lower (has_type $F64 (fcvt_from_sint _ v @ (value_type $I64))))+(rule 1 (lower (fcvt_from_sint $F64 v @ (value_type $I64)))(rv_fcvtdl (FRM.RNE) v))-(rule 2 (lower (has_type (ty_supported_vec _) (fcvt_from_sint _ v @ (value_type from_ty))))+(rule 2 (lower (fcvt_from_sint (ty_supported_vec _) v @ (value_type from_ty)))(rv_vfcvt_f_x_v v (unmasked) from_ty));;;;; Rules for `fcvt_from_uint`;;;;;;;;;-(rule 0 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type (fits_in_16 ty)))))+(rule 0 (lower (fcvt_from_uint $F32 v @ (value_type (fits_in_16 ty))))(rv_fcvtslu (FRM.RNE) (zext v)))-(rule 1 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type $I32))))+(rule 1 (lower (fcvt_from_uint $F32 v @ (value_type $I32)))(rv_fcvtswu (FRM.RNE) v))-(rule 1 (lower (has_type $F32 (fcvt_from_uint _ v @ (value_type $I64))))+(rule 1 (lower (fcvt_from_uint $F32 v @ (value_type $I64)))(rv_fcvtslu (FRM.RNE) v))-(rule 0 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type (fits_in_16 ty)))))+(rule 0 (lower (fcvt_from_uint $F64 v @ (value_type (fits_in_16 ty))))(rv_fcvtdlu (FRM.RNE) (zext v)))-(rule 1 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type $I32))))+(rule 1 (lower (fcvt_from_uint $F64 v @ (value_type $I32)))(rv_fcvtdwu v))-(rule 1 (lower (has_type $F64 (fcvt_from_uint _ v @ (value_type $I64))))+(rule 1 (lower (fcvt_from_uint $F64 v @ (value_type $I64)))(rv_fcvtdlu (FRM.RNE) v))-(rule 2 (lower (has_type (ty_supported_vec _) (fcvt_from_uint _ v @ (value_type from_ty))))+(rule 2 (lower (fcvt_from_uint (ty_supported_vec _) v @ (value_type from_ty)))(rv_vfcvt_f_xu_v v (unmasked) from_ty));;;;; Rules for `symbol_value`;;;;;;;;;@@ -2562,21 +2562,21 @@;;;;; Rules for `tls_value` ;;;;;;;;;;;;;;-(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value _ (symbol_value_data name _ _))))+(rule (lower (tls_value (tls_model (TlsModel.ElfGd)) (symbol_value_data name _ _)))(elf_tls_get_addr name));;;;; Rules for `bitcast`;;;;;;;;;;; These rules should probably be handled in `gen_bitcast`, but it's convenient to have that return;; a single register, instead of a `ValueRegs`-(rule 3 (lower (has_type (ty_reg_pair _) (bitcast _ _ v @ (value_type (ty_supported_vec _)))))+(rule 3 (lower (bitcast (ty_reg_pair _) _ v @ (value_type (ty_supported_vec _))))(value_regs(gen_extractlane $I64X2 v 0)(gen_extractlane $I64X2 v 1)));; Move the high half into a vector register, and then use vslide1up to move it up and;; insert the lower half in one instruction.-(rule 2 (lower (has_type (ty_supported_vec _) (bitcast _ _ v @ (value_type (ty_reg_pair _)))))+(rule 2 (lower (bitcast (ty_supported_vec _) _ v @ (value_type (ty_reg_pair _))))(let ((lo XReg (value_regs_get v 0))(hi XReg (value_regs_get v 1))(vstate VState (vstate_from_type $I64X2))@@ -2585,38 +2585,38 @@;; `gen_bitcast` below only works with single register values, so handle I128;; and F128 specially here.-(rule 1 (lower (has_type (ty_reg_pair _) (bitcast _ _ v @ (value_type (ty_reg_pair _)))))+(rule 1 (lower (bitcast (ty_reg_pair _) _ v @ (value_type (ty_reg_pair _))))v)-(rule 0 (lower (has_type out_ty (bitcast _ _ v @ (value_type in_ty))))+(rule 0 (lower (bitcast out_ty _ v @ (value_type in_ty)))(gen_bitcast v in_ty out_ty));;;;; Rules for `ceil`;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (ceil _ x)))+(rule 0 (lower (ceil (ty_supported_float_full ty) x))(gen_float_round (FRM.RUP) x ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (ceil _ x)))+(rule 1 (lower (ceil (ty_supported_vec ty) x))(gen_vec_round x (FRM.RUP) ty));;;;; Rules for `floor`;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (floor _ x)))+(rule 0 (lower (floor (ty_supported_float_full ty) x))(gen_float_round (FRM.RDN) x ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (floor _ x)))+(rule 1 (lower (floor (ty_supported_vec ty) x))(gen_vec_round x (FRM.RDN) ty));;;;; Rules for `trunc`;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (trunc _ x)))+(rule 0 (lower (trunc (ty_supported_float_full ty) x))(gen_float_round (FRM.RTZ) x ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (trunc _ x)))+(rule 1 (lower (trunc (ty_supported_vec ty) x))(gen_vec_round x (FRM.RTZ) ty));;;;; Rules for `nearest`;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_float_full ty) (nearest _ x)))+(rule 0 (lower (nearest (ty_supported_float_full ty) x))(gen_float_round (FRM.RNE) x ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (nearest _ x)))+(rule 1 (lower (nearest (ty_supported_vec ty) x))(gen_vec_round x (FRM.RNE) ty))@@ -2630,10 +2630,10 @@;; Base case: use `gen_bmask` to generate a 0 mask or -1 mask from the value of;; `cmp`. This is then used with some bit twiddling to produce the final result.-(rule 0 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp x y)))+(rule 0 (lower (select_spectre_guard (fits_in_64 _) cmp x y))(let ((mask XReg (gen_bmask cmp)))(rv_or (rv_and mask x) (rv_andn y mask))))-(rule 1 (lower (has_type $I128 (select_spectre_guard _ cmp x y)))+(rule 1 (lower (select_spectre_guard $I128 cmp x y))(let ((mask XReg (gen_bmask cmp)))(value_regs(rv_or (rv_and mask (value_regs_get x 0)) (rv_andn (value_regs_get y 0) mask))@@ -2641,14 +2641,14 @@;; Special case when an argument is the constant zero as some ands and ors;; can be folded away.-(rule 2 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp (i64_from_iconst 0) y)))+(rule 2 (lower (select_spectre_guard (fits_in_64 _) cmp (i64_from_iconst 0) y))(rv_andn y (gen_bmask cmp)))-(rule 3 (lower (has_type (fits_in_64 _) (select_spectre_guard _ cmp x (i64_from_iconst 0))))+(rule 3 (lower (select_spectre_guard (fits_in_64 _) cmp x (i64_from_iconst 0)))(rv_and x (gen_bmask cmp)));;;;; Rules for `bmask`;;;;;;;;;(rule- (lower (has_type oty (bmask _ x)))+ (lower (bmask oty x))(lower_bmask x oty));; N.B.: the Ret itself is generated by the ABI.@@ -2673,7 +2673,7 @@;; sext.{b,h,w} a0, a0;; neg a1, a0;; max a0, a0, a1-(rule 0 (lower (has_type (ty_int_ref_scalar_64 ty) (iabs _ x)))+(rule 0 (lower (iabs (ty_int_ref_scalar_64 ty) x))(let ((extended XReg (sext x))(negated XReg (rv_neg extended)))(gen_select_xreg (cmp_gt extended negated) extended negated)))@@ -2681,7 +2681,7 @@;; For vectors we generate the same code, but with vector instructions;; we can skip the sign extension, since the vector unit will only process;; Element Sized chunks.-(rule 1 (lower (has_type (ty_supported_vec ty) (iabs _ x)))+(rule 1 (lower (iabs (ty_supported_vec ty) x))(let ((negated VReg (rv_vneg_v x (unmasked) ty)))(rv_vmax_vv x negated (unmasked) ty)))@@ -2809,13 +2809,13 @@;;;; Rules for `splat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type ty (splat _ n @ (value_type (ty_supported_float_full _)))))+(rule 0 (lower (splat ty n @ (value_type (ty_supported_float_full _))))(rv_vfmv_vf n ty))-(rule 1 (lower (has_type ty (splat _ n @ (value_type (ty_int_ref_scalar_64 _)))))+(rule 1 (lower (splat ty n @ (value_type (ty_int_ref_scalar_64 _))))(rv_vmv_vx n ty))-(rule 2 (lower (has_type ty (splat _ (iconst _ (u64_from_imm64 (imm5_from_u64 imm))))))+(rule 2 (lower (splat ty (iconst _ (u64_from_imm64 (imm5_from_u64 imm)))))(rv_vmv_vi imm ty));; TODO: We can splat out more patterns by using for example a vmv.v.i i8x16 for@@ -2826,56 +2826,56 @@;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y)))+(rule 0 (lower (uadd_sat (ty_supported_vec ty) x y))(rv_vsaddu_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x (splat _ y))))+(rule 1 (lower (uadd_sat (ty_supported_vec ty) x (splat _ y)))(rv_vsaddu_vx x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (uadd_sat _ (splat _ x) y)))+(rule 2 (lower (uadd_sat (ty_supported_vec ty) (splat _ x) y))(rv_vsaddu_vx y x (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y)))+(rule 3 (lower (uadd_sat (ty_supported_vec ty) x y))(if-let y_imm (replicated_imm5 y))(rv_vsaddu_vi x y_imm (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (uadd_sat _ x y)))+(rule 4 (lower (uadd_sat (ty_supported_vec ty) x y))(if-let x_imm (replicated_imm5 x))(rv_vsaddu_vi y x_imm (unmasked) ty));;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y)))+(rule 0 (lower (sadd_sat (ty_supported_vec ty) x y))(rv_vsadd_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x (splat _ y))))+(rule 1 (lower (sadd_sat (ty_supported_vec ty) x (splat _ y)))(rv_vsadd_vx x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (sadd_sat _ (splat _ x) y)))+(rule 2 (lower (sadd_sat (ty_supported_vec ty) (splat _ x) y))(rv_vsadd_vx y x (unmasked) ty))-(rule 3 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y)))+(rule 3 (lower (sadd_sat (ty_supported_vec ty) x y))(if-let y_imm (replicated_imm5 y))(rv_vsadd_vi x y_imm (unmasked) ty))-(rule 4 (lower (has_type (ty_supported_vec ty) (sadd_sat _ x y)))+(rule 4 (lower (sadd_sat (ty_supported_vec ty) x y))(if-let x_imm (replicated_imm5 x))(rv_vsadd_vi y x_imm (unmasked) ty));;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (usub_sat _ x y)))+(rule 0 (lower (usub_sat (ty_supported_vec ty) x y))(rv_vssubu_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (usub_sat _ x (splat _ y))))+(rule 1 (lower (usub_sat (ty_supported_vec ty) x (splat _ y)))(rv_vssubu_vx x y (unmasked) ty));;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (ssub_sat _ x y)))+(rule 0 (lower (ssub_sat (ty_supported_vec ty) x y))(rv_vssub_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (ssub_sat _ x (splat _ y))))+(rule 1 (lower (ssub_sat (ty_supported_vec ty) x (splat _ y)))(rv_vssub_vx x y (unmasked) ty));;;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2928,13 +2928,13 @@;;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (swizzle _ x y)))+(rule 0 (lower (swizzle (ty_supported_vec ty) x y))(rv_vrgather_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (swizzle _ x (splat _ y))))+(rule 1 (lower (swizzle (ty_supported_vec ty) x (splat _ y)))(rv_vrgather_vx x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (swizzle _ x y)))+(rule 2 (lower (swizzle (ty_supported_vec ty) x y))(if-let y_imm (replicated_uimm5 y))(rv_vrgather_vi x y_imm (unmasked) ty))@@ -2945,7 +2945,7 @@;;;; vrgather will insert a 0 for lanes that are out of bounds, so we can let it load;; negative and out of bounds indexes.-(rule (lower (has_type (ty_supported_vec ty @ $I8X16) (shuffle _ x y (vconst_from_immediate mask))))+(rule (lower (shuffle (ty_supported_vec ty @ $I8X16) x y (vconst_from_immediate mask)))(if-let neg16 (i8_to_imm5 -16))(let ((x_mask VReg (gen_constant ty mask))(x_lanes VReg (rv_vrgather_vv x x_mask (unmasked) ty))@@ -2956,51 +2956,51 @@;;;; Rules for `swiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Slide down half the vector, and do a signed extension.-(rule 0 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ x @ (value_type in_ty))))+(rule 0 (lower (swiden_high (ty_supported_vec out_ty) x @ (value_type in_ty)))(rv_vsext_vf2 (gen_slidedown_half in_ty x) (unmasked) out_ty))-(rule 1 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ (swiden_high _ x @ (value_type in_ty)))))+(rule 1 (lower (swiden_high (ty_supported_vec out_ty) (swiden_high _ x @ (value_type in_ty))))(if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty)))(rv_vsext_vf4 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty))-(rule 2 (lower (has_type (ty_supported_vec out_ty) (swiden_high _ (swiden_high _ (swiden_high _ x @ (value_type in_ty))))))+(rule 2 (lower (swiden_high (ty_supported_vec out_ty) (swiden_high _ (swiden_high _ x @ (value_type in_ty)))))(if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty)))(rv_vsext_vf8 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty));;;; Rules for `uwiden_high` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Slide down half the vector, and do a zero extension.-(rule 0 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ x @ (value_type in_ty))))+(rule 0 (lower (uwiden_high (ty_supported_vec out_ty) x @ (value_type in_ty)))(rv_vzext_vf2 (gen_slidedown_half in_ty x) (unmasked) out_ty))-(rule 1 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ (uwiden_high _ x @ (value_type in_ty)))))+(rule 1 (lower (uwiden_high (ty_supported_vec out_ty) (uwiden_high _ x @ (value_type in_ty))))(if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty)))(rv_vzext_vf4 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty))-(rule 2 (lower (has_type (ty_supported_vec out_ty) (uwiden_high _ (uwiden_high _ (uwiden_high _ x @ (value_type in_ty))))))+(rule 2 (lower (uwiden_high (ty_supported_vec out_ty) (uwiden_high _ (uwiden_high _ x @ (value_type in_ty)))))(if-let (uimm5_from_u64 amt) (u64_wrapping_sub (ty_lane_count in_ty) (ty_lane_count out_ty)))(rv_vzext_vf8 (rv_vslidedown_vi x amt (unmasked) in_ty) (unmasked) out_ty));;;; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ x)))+(rule 0 (lower (swiden_low (ty_supported_vec out_ty) x))(rv_vsext_vf2 x (unmasked) out_ty))-(rule 1 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ (swiden_low _ x))))+(rule 1 (lower (swiden_low (ty_supported_vec out_ty) (swiden_low _ x)))(rv_vsext_vf4 x (unmasked) out_ty))-(rule 2 (lower (has_type (ty_supported_vec out_ty) (swiden_low _ (swiden_low _ (swiden_low _ x)))))+(rule 2 (lower (swiden_low (ty_supported_vec out_ty) (swiden_low _ (swiden_low _ x))))(rv_vsext_vf8 x (unmasked) out_ty));;;; Rules for `uwiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ x)))+(rule 0 (lower (uwiden_low (ty_supported_vec out_ty) x))(rv_vzext_vf2 x (unmasked) out_ty))-(rule 1 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ (uwiden_low _ x))))+(rule 1 (lower (uwiden_low (ty_supported_vec out_ty) (uwiden_low _ x)))(rv_vzext_vf4 x (unmasked) out_ty))-(rule 2 (lower (has_type (ty_supported_vec out_ty) (uwiden_low _ (uwiden_low _ (uwiden_low _ x)))))+(rule 2 (lower (uwiden_low (ty_supported_vec out_ty) (uwiden_low _ (uwiden_low _ x))))(rv_vzext_vf8 x (unmasked) out_ty));;;; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3018,7 +3018,7 @@;; However V8 does something better. They use 2 vcompresses using LMUL2, that means;; that they can do the whole thing in 3 instructions (2 vcompress + vadd). We don't;; support LMUL > 1, so we can't do that.-(rule (lower (has_type (ty_supported_vec ty) (iadd_pairwise _ x y)))+(rule (lower (iadd_pairwise (ty_supported_vec ty) x y))(if-let half_size (u64_to_uimm5 (u64_checked_div (ty_lane_count ty) 2)))(let ((odd_mask VReg (gen_vec_mask 0x5555555555555555))(lhs_lo VReg (rv_vcompress_vm x odd_mask ty))@@ -3049,7 +3049,7 @@;; Right Logical instruction using the `vxrm` fixed-point rounding mode. The;; default rounding mode is `rnu` (round-to-nearest-up (add +0.5 LSB)).;; Which is coincidentally the rounding mode we want for `avg_round`.-(rule (lower (has_type (ty_supported_vec ty) (avg_round _ x y)))+(rule (lower (avg_round (ty_supported_vec ty) x y))(if-let one (u64_to_uimm5 1))(let ((lhs VReg (rv_vand_vv x y (unmasked) ty))(xor VReg (rv_vxor_vv x y (unmasked) ty))@@ -3058,38 +3058,38 @@;;;; Rules for `scalar_to_vector` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ x)))+(rule 0 (lower (scalar_to_vector (ty_supported_vec ty) x))(if (ty_vector_float ty))(let ((zero VReg (rv_vmv_vx (zero_reg) ty))(elem VReg (rv_vfmv_sf x ty))(mask VReg (gen_vec_mask 1)))(rv_vmerge_vvm zero elem mask ty)))-(rule 1 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ x)))+(rule 1 (lower (scalar_to_vector (ty_supported_vec ty) x))(if (ty_vector_not_float ty))(let ((zero VReg (rv_vmv_vx (zero_reg) ty))(mask VReg (gen_vec_mask 1)))(rv_vmerge_vxm zero x mask ty)))-(rule 2 (lower (has_type (ty_supported_vec ty) (scalar_to_vector _ (imm5_from_value x))))+(rule 2 (lower (scalar_to_vector (ty_supported_vec ty) (imm5_from_value x)))(let ((zero VReg (rv_vmv_vx (zero_reg) ty))(mask VReg (gen_vec_mask 1)))(rv_vmerge_vim zero x mask ty)));;;; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 0 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ x y)))+(rule 0 (lower (sqmul_round_sat (ty_supported_vec ty) x y))(rv_vsmul_vv x y (unmasked) ty))-(rule 1 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ x (splat _ y))))+(rule 1 (lower (sqmul_round_sat (ty_supported_vec ty) x (splat _ y)))(rv_vsmul_vx x y (unmasked) ty))-(rule 2 (lower (has_type (ty_supported_vec ty) (sqmul_round_sat _ (splat _ x) y)))+(rule 2 (lower (sqmul_round_sat (ty_supported_vec ty) (splat _ x) y))(rv_vsmul_vx y x (unmasked) ty));;;; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec out_ty) (snarrow _ x @ (value_type in_ty) y)))+(rule (lower (snarrow (ty_supported_vec out_ty) x @ (value_type in_ty) y))(if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2)))(if-let zero (u64_to_uimm5 0))(let ((x_clip VReg (rv_vnclip_wi x zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty))))@@ -3098,7 +3098,7 @@;;;; Rules for `uunarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (ty_supported_vec out_ty) (uunarrow _ x @ (value_type in_ty) y)))+(rule (lower (uunarrow (ty_supported_vec out_ty) x @ (value_type in_ty) y))(if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2)))(if-let zero (u64_to_uimm5 0))(let ((x_clip VReg (rv_vnclipu_wi x zero (unmasked) (vstate_mf2 (ty_half_lanes out_ty))))@@ -3111,7 +3111,7 @@;; To correct for this we just remove negative values using `vmax` and then use the normal;; unsigned to unsigned narrowing instruction.-(rule (lower (has_type (ty_supported_vec out_ty) (unarrow _ x @ (value_type in_ty) y)))+(rule (lower (unarrow (ty_supported_vec out_ty) x @ (value_type in_ty) y))(if-let lane_diff (u64_to_uimm5 (u64_checked_div (ty_lane_count out_ty) 2)))(if-let zero (u64_to_uimm5 0))(let ((x_pos VReg (rv_vmax_vx x (zero_reg) (unmasked) in_ty))diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex 6e337571c9..a42493bfbc 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -14,13 +14,11 @@;;;; Rules for `iconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule (lower (has_type (fits_in_64 ty)- (iconst _ (u64_from_imm64 x))))+(rule (lower (iconst (fits_in_64 ty) (u64_from_imm64 x)))(imm ty x));; `i128`-(rule 1 (lower (has_type $I128- (iconst _ (u64_from_imm64 x))))+(rule 1 (lower (iconst $I128 (u64_from_imm64 x)))(value_regs (imm $I64 x)(imm $I64 0)))@@ -52,8 +50,7 @@;; `i64` and smaller.;; Base case for 8 and 16-bit types-(rule -6 (lower (has_type (fits_in_16 ty)- (iadd _ x y)))+(rule -6 (lower (iadd (fits_in_16 ty) x y))(x64_add ty x y));; Base case for 32 and 64-bit types which might end up using the `lea`@@ -63,39 +60,33 @@;; but the actual instruction emitted might be an `add` if it's equivalent.;; For more details on this see the `emit.rs` logic to emit;; `LoadEffectiveAddress`.-(rule iadd_base_case_32_or_64_lea -5 (lower (has_type (ty_32_or_64 ty) (iadd _ x y)))+(rule iadd_base_case_32_or_64_lea -5 (lower (iadd (ty_32_or_64 ty) x y))(x64_lea ty (to_amode_add (mem_flags_trusted_data) x y (zero_offset))));; Higher-priority cases than the previous two where a load can be sunk into;; the add instruction itself. Note that both operands are tested for;; sink-ability since addition is commutative-(rule -4 (lower (has_type (fits_in_64 ty)- (iadd _ x (sinkable_load y))))+(rule -4 (lower (iadd (fits_in_64 ty) x (sinkable_load y)))(x64_add ty x y))-(rule -3 (lower (has_type (fits_in_64 ty)- (iadd _ (sinkable_load x) y)))+(rule -3 (lower (iadd (fits_in_64 ty) (sinkable_load x) y))(x64_add ty y x));; SSE.-(rule (lower (has_type (multi_lane 8 16)- (iadd _ x y)))+(rule (lower (iadd (multi_lane 8 16) x y))(x64_paddb x y))-(rule (lower (has_type (multi_lane 16 8)- (iadd _ x y)))+(rule (lower (iadd (multi_lane 16 8) x y))(x64_paddw x y))-(rule (lower (has_type (multi_lane 32 4)- (iadd _ x y)))+(rule (lower (iadd (multi_lane 32 4) x y))(x64_paddd x y))-(rule (lower (has_type (multi_lane 64 2)- (iadd _ x y)))+(rule (lower (iadd (multi_lane 64 2) x y))(x64_paddq x y));; `i128`-(rule 1 (lower (has_type $I128 (iadd _ x y)))+(rule 1 (lower (iadd $I128 x y));; Get the high/low registers for `x`.(let ((x_regs ValueRegs x)(y_regs ValueRegs y))@@ -104,10 +95,10 @@(value_regs_get_gpr x_regs 1)(value_regs_get_gpr y_regs 0)(value_regs_get_gpr y_regs 1))))-(rule 2 (lower (has_type $I128 (iadd _ x (iconcat _ y_lo y_hi))))+(rule 2 (lower (iadd $I128 x (iconcat _ y_lo y_hi)))(let ((x_regs ValueRegs x))(iadd128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1) y_lo y_hi)))-(rule 3 (lower (has_type $I128 (iadd _ x (uextend _ y @ (value_type $I64)))))+(rule 3 (lower (iadd $I128 x (uextend _ y @ (value_type $I64))))(let ((x_regs ValueRegs x))(iadd128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1)y (gpr_mem_imm_64_imm 0))))@@ -115,7 +106,7 @@;; Specialized lowering rule for `iadd` of two 64-bit unsigned integers, meaning;; that we can skip the `adc` and instead use `setb`. This is in some sense a;; way of modeling `uadd_overflow`.-(rule 4 (lower (has_type $I128 (iadd _ (uextend _ x) (uextend _ y @ (value_type $I64)))))+(rule 4 (lower (iadd $I128 (uextend _ x) (uextend _ y @ (value_type $I64))))(let ((x Gpr (extend_to_gpr x $I64 (ExtendKind.Zero)))(ret ValueRegs (with_flags (x64_add_with_flags_paired $I64 x y)@@ -222,22 +213,18 @@;;;; Rules for `sadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (multi_lane 8 16)- (sadd_sat _ x y)))+(rule (lower (sadd_sat (multi_lane 8 16) x y))(x64_paddsb x y))-(rule (lower (has_type (multi_lane 16 8)- (sadd_sat _ x y)))+(rule (lower (sadd_sat (multi_lane 16 8) x y))(x64_paddsw x y));;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (multi_lane 8 16)- (uadd_sat _ x y)))+(rule (lower (uadd_sat (multi_lane 8 16) x y))(x64_paddusb x y))-(rule (lower (has_type (multi_lane 16 8)- (uadd_sat _ x y)))+(rule (lower (uadd_sat (multi_lane 16 8) x y))(x64_paddusw x y));;;; Rules for `isub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -245,30 +232,25 @@;; `i64` and smaller.;; Sub two registers.-(rule -3 (lower (has_type (fits_in_64 ty)- (isub _ x y)))+(rule -3 (lower (isub (fits_in_64 ty) x y))(x64_sub ty x y));; SSE.-(rule (lower (has_type (multi_lane 8 16)- (isub _ x y)))+(rule (lower (isub (multi_lane 8 16) x y))(x64_psubb x y))-(rule (lower (has_type (multi_lane 16 8)- (isub _ x y)))+(rule (lower (isub (multi_lane 16 8) x y))(x64_psubw x y))-(rule (lower (has_type (multi_lane 32 4)- (isub _ x y)))+(rule (lower (isub (multi_lane 32 4) x y))(x64_psubd x y))-(rule (lower (has_type (multi_lane 64 2)- (isub _ x y)))+(rule (lower (isub (multi_lane 64 2) x y))(x64_psubq x y));; `i128`-(rule 1 (lower (has_type $I128 (isub _ x y)))+(rule 1 (lower (isub $I128 x y));; Get the high/low registers for `x`.(let ((x_regs ValueRegs x)(y_regs ValueRegs y))@@ -277,10 +259,10 @@(value_regs_get_gpr x_regs 1)(value_regs_get_gpr y_regs 0)(value_regs_get_gpr y_regs 1))))-(rule 2 (lower (has_type $I128 (isub _ x (iconcat _ y_lo y_hi))))+(rule 2 (lower (isub $I128 x (iconcat _ y_lo y_hi)))(let ((x_regs ValueRegs x))(isub128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1) y_lo y_hi)))-(rule 3 (lower (has_type $I128 (isub _ x (uextend _ y @ (value_type $I64)))))+(rule 3 (lower (isub $I128 x (uextend _ y @ (value_type $I64))))(let ((x_regs ValueRegs x))(isub128 (value_regs_get_gpr x 0) (value_regs_get_gpr x 1)y (gpr_mem_imm_64_imm 0))))@@ -295,22 +277,18 @@;;;; Rules for `ssub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (multi_lane 8 16)- (ssub_sat _ x y)))+(rule (lower (ssub_sat (multi_lane 8 16) x y))(x64_psubsb x y))-(rule (lower (has_type (multi_lane 16 8)- (ssub_sat _ x y)))+(rule (lower (ssub_sat (multi_lane 16 8) x y))(x64_psubsw x y));;;; Rules for `usub_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (multi_lane 8 16)- (usub_sat _ x y)))+(rule (lower (usub_sat (multi_lane 8 16) x y))(x64_psubusb x y))-(rule (lower (has_type (multi_lane 16 8)- (usub_sat _ x y)))+(rule (lower (usub_sat (multi_lane 16 8) x y))(x64_psubusw x y));;;; Rules for `band` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -318,18 +296,18 @@;; `{i,b}64` and smaller.;; And two registers.-(rule 0 (lower (has_type ty (band _ x y)))+(rule 0 (lower (band ty x y))(if (ty_int_ref_scalar_64 ty))(x64_and ty x y));; The above case automatically handles when the rhs is an immediate or a;; sinkable load, but additionally handle the lhs here.-(rule 1 (lower (has_type ty (band _ (sinkable_load x) y)))+(rule 1 (lower (band ty (sinkable_load x) y))(if (ty_int_ref_scalar_64 ty))(x64_and ty y x))-(rule 2 (lower (has_type ty (band _ (simm32_from_value x) y)))+(rule 2 (lower (band ty (simm32_from_value x) y))(if (ty_int_ref_scalar_64 ty))(x64_and ty y x))@@ -338,7 +316,7 @@;; Note that `x` and `y` are both constrained to being in registers here because;; x86 instructions with a memory operand will load 128 bits, not the desired;; bit width of 32 or 64 bits here.-(rule 5 (lower (has_type (ty_scalar_float ty) (band _ x y)))+(rule 5 (lower (band (ty_scalar_float ty) x y))(sse_and ty x (put_in_xmm y)));; SSE.@@ -350,8 +328,7 @@(rule (sse_and $F64 x y) (x64_andpd x y))(rule -1 (sse_and (multi_lane _bits _lanes) x y) (x64_pand x y))-(rule 6 (lower (has_type ty @ (multi_lane _bits _lanes)- (band _ x y)))+(rule 6 (lower (band ty @ (multi_lane _bits _lanes) x y))(sse_and ty x y));; `i128`.@@ -367,7 +344,7 @@(value_gprs (x64_and $I64 x_lo y_lo)(x64_and $I64 x_hi y_hi))))-(rule 7 (lower (has_type $I128 (band _ x y)))+(rule 7 (lower (band $I128 x y))(and_i128 x y));; Specialized lowerings for `(band x (bnot y))` which is additionally produced@@ -386,17 +363,17 @@;; while x86 does;;;; pandn(x, y) = and(not(x), y)-(rule 8 (lower (has_type ty @ (multi_lane _bits _lane) (band _ x (bnot _ y))))+(rule 8 (lower (band ty @ (multi_lane _bits _lane) x (bnot _ y)))(sse_and_not ty y x))-(rule 9 (lower (has_type ty @ (multi_lane _bits _lane) (band _ (bnot _ y) x)))+(rule 9 (lower (band ty @ (multi_lane _bits _lane) (bnot _ y) x))(sse_and_not ty y x))-(rule 10 (lower (has_type ty (band _ x (bnot _ y))))+(rule 10 (lower (band ty x (bnot _ y)))(if (ty_int_ref_scalar_64 ty))(if-let true (has_bmi1));; the first argument is the one that gets inverted with andn(x64_andn ty y x))-(rule 11 (lower (has_type ty (band _ (bnot _ y) x)))+(rule 11 (lower (band ty (bnot _ y) x))(if (ty_int_ref_scalar_64 ty))(if-let true (has_bmi1))(x64_andn ty y x))@@ -408,21 +385,21 @@(rule 0 (val_minus_one (iadd _ x (i64_from_iconst -1))) x)(rule 1 (val_minus_one (iadd _ (i64_from_iconst -1) x)) x)-(rule 12 (lower (has_type (ty_32_or_64 ty) (band _ x y)))+(rule 12 (lower (band (ty_32_or_64 ty) x y))(if-let true (has_bmi1))(if-let x (val_minus_one y))(x64_blsr ty x))-(rule 13 (lower (has_type (ty_32_or_64 ty) (band _ y x)))+(rule 13 (lower (band (ty_32_or_64 ty) y x))(if-let true (has_bmi1))(if-let x (val_minus_one y))(x64_blsr ty x));; Specialization of `blsi` for BMI1-(rule 14 (lower (has_type (ty_32_or_64 ty) (band _ (ineg _ x) x)))+(rule 14 (lower (band (ty_32_or_64 ty) (ineg _ x) x))(if-let true (has_bmi1))(x64_blsi ty x))-(rule 15 (lower (has_type (ty_32_or_64 ty) (band _ x (ineg _ x))))+(rule 15 (lower (band (ty_32_or_64 ty) x (ineg _ x)))(if-let true (has_bmi1))(x64_blsi ty x))@@ -435,7 +412,7 @@;; `ishl`, so an `and` instruction is required to mask the index to match the;; semantics of Cranelift's `ishl`.-(rule 16 (lower (has_type (ty_32_or_64 ty) (band _ x y)))+(rule 16 (lower (band (ty_32_or_64 ty) x y))(if-let true (has_bmi2))(if-let (ishl _ (u64_from_iconst 1) index) (val_minus_one y))(x64_bzhi ty x (x64_and ty index (RegMemImm.Imm (u32_wrapping_sub (ty_bits ty) 1)))))@@ -445,18 +422,18 @@;; `{i,b}64` and smaller.;; Or two registers.-(rule 0 (lower (has_type ty (bor _ x y)))+(rule 0 (lower (bor ty x y))(if (ty_int_ref_scalar_64 ty))(x64_or ty x y));; Handle immediates/sinkable loads on the lhs in addition to the automatic;; handling of the rhs above-(rule 1 (lower (has_type ty (bor _ (sinkable_load x) y)))+(rule 1 (lower (bor ty (sinkable_load x) y))(if (ty_int_ref_scalar_64 ty))(x64_or ty y x))-(rule 2 (lower (has_type ty (bor _ (simm32_from_value x) y)))+(rule 2 (lower (bor ty (simm32_from_value x) y))(if (ty_int_ref_scalar_64 ty))(x64_or ty y x))@@ -465,7 +442,7 @@;; Note that `x` and `y` are both constrained to being in registers here because;; x86 instructions with a memory operand will load 128 bits, not the desired;; bit width of 32 or 64 bits here.-(rule 5 (lower (has_type (ty_scalar_float ty) (bor _ x y)))+(rule 5 (lower (bor (ty_scalar_float ty) x y))(sse_or ty x (put_in_xmm y)));; SSE.@@ -478,8 +455,7 @@(rule (sse_or $F64 x y) (x64_orpd x y))(rule -1 (sse_or (multi_lane _bits _lanes) x y) (x64_por x y))-(rule 6 (lower (has_type ty @ (multi_lane _bits _lanes)- (bor _ x y)))+(rule 6 (lower (bor ty @ (multi_lane _bits _lanes) x y))(sse_or ty x y));; `{i,b}128`.@@ -493,7 +469,7 @@(value_gprs (x64_or $I64 x_lo y_lo)(x64_or $I64 x_hi y_hi))))-(rule 7 (lower (has_type $I128 (bor _ x y)))+(rule 7 (lower (bor $I128 x y))(or_i128 x y));; Specialized lowerings to generate the `shld` instruction.@@ -501,14 +477,12 @@;; The `shld` instruction will shift a value left and shift-in bits from a;; different register. Pattern-match doing this with bit-ops and shifts to;; generate a `shld` instruction.-(rule 8 (lower (has_type (ty_int_ref_16_to_64 ty)- (bor _ (ishl _ x (u8_from_iconst xs)) (ushr _ y (u8_from_iconst ys)))))+(rule 8 (lower (bor (ty_int_ref_16_to_64 ty) (ishl _ x (u8_from_iconst xs)) (ushr _ y (u8_from_iconst ys))))(if-let true (u64_eq (ty_bits ty) (u64_wrapping_add xs ys)))(if-let true (u64_gt xs 0))(if-let true (u64_gt ys 0))(x64_shld ty x y xs))-(rule 8 (lower (has_type (ty_int_ref_16_to_64 ty)- (bor _ (ushr _ y (u8_from_iconst ys)) (ishl _ x (u8_from_iconst xs)))))+(rule 8 (lower (bor (ty_int_ref_16_to_64 ty) (ushr _ y (u8_from_iconst ys)) (ishl _ x (u8_from_iconst xs))))(if-let true (u64_eq (ty_bits ty) (u64_wrapping_add xs ys)))(if-let true (u64_gt xs 0))(if-let true (u64_gt ys 0))@@ -520,18 +494,18 @@;; `{i,b}64` and smaller.;; Xor two registers.-(rule 0 (lower (has_type ty (bxor _ x y)))+(rule 0 (lower (bxor ty x y))(if (ty_int_ref_scalar_64 ty))(x64_xor ty x y));; Handle xor with lhs immediates/sinkable loads in addition to the automatic;; handling of the rhs above.-(rule 1 (lower (has_type ty (bxor _ (sinkable_load x) y)))+(rule 1 (lower (bxor ty (sinkable_load x) y))(if (ty_int_ref_scalar_64 ty))(x64_xor ty y x))-(rule 4 (lower (has_type ty (bxor _ (simm32_from_value x) y)))+(rule 4 (lower (bxor ty (simm32_from_value x) y))(if (ty_int_ref_scalar_64 ty))(x64_xor ty y x))@@ -540,17 +514,17 @@;; Note that `x` and `y` are both constrained to being in registers here because;; x86 instructions with a memory operand will load 128 bits, not the desired;; bit width of 32 or 64 bits here.-(rule 5 (lower (has_type (ty_scalar_float ty) (bxor _ x y)))+(rule 5 (lower (bxor (ty_scalar_float ty) x y))(x64_xor_vector ty x (put_in_xmm y)));; SSE.-(rule 6 (lower (has_type ty @ (multi_lane _bits _lanes) (bxor _ x y)))+(rule 6 (lower (bxor ty @ (multi_lane _bits _lanes) x y))(x64_xor_vector ty x y));; `{i,b}128`.-(rule 7 (lower (has_type $I128 (bxor _ x y)))+(rule 7 (lower (bxor $I128 x y))(let ((x_regs ValueRegs x)(x_lo Gpr (value_regs_get_gpr x_regs 0))(x_hi Gpr (value_regs_get_gpr x_regs 1))@@ -562,11 +536,11 @@;; Specialization of `blsmsk` for BMI1-(rule 8 (lower (has_type (ty_32_or_64 ty) (bxor _ x y)))+(rule 8 (lower (bxor (ty_32_or_64 ty) x y))(if-let true (has_bmi1))(if-let x (val_minus_one y))(x64_blsmsk ty x))-(rule 9 (lower (has_type (ty_32_or_64 ty) (bxor _ y x)))+(rule 9 (lower (bxor (ty_32_or_64 ty) y x))(if-let true (has_bmi1))(if-let x (val_minus_one y))(x64_blsmsk ty x))@@ -575,7 +549,7 @@;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (ishl _ src amt)))+(rule -1 (lower (ishl (fits_in_64 ty) src amt))(x64_shl ty src (put_masked_in_imm8_gpr amt ty)));; `i128`.@@ -612,7 +586,7 @@(cmove $I64 (CC.Z) lo_shifted zero)(cmove $I64 (CC.Z) hi_shifted_ lo_shifted)))))-(rule (lower (has_type $I128 (ishl _ src amt)))+(rule (lower (ishl $I128 src amt));; NB: Only the low bits of `amt` matter since we logically mask the shift;; amount to the value's bit width.(let ((amt_ Gpr (lo_gpr amt)))@@ -624,7 +598,7 @@;; in higher feature sets like AVX), we lower the `ishl.i8x16` to a sequence of;; instructions. The basic idea, whether the amount to shift by is an immediate;; or not, is to use a 16x8 shift and then mask off the incorrect bits to 0s.-(rule (lower (has_type ty @ $I8X16 (ishl _ src amt)))+(rule (lower (ishl ty @ $I8X16 src amt))(let (;; Mask the amount to ensure wrapping behaviour(masked_amt RegMemImm (mask_xmm_shift ty amt))@@ -670,20 +644,20 @@;; 16x8, 32x4, and 64x2 shifts can each use a single instruction, once the shift amount is masked.-(rule (lower (has_type ty @ $I16X8 (ishl _ src amt)))+(rule (lower (ishl ty @ $I16X8 src amt))(x64_psllw src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I32X4 (ishl _ src amt)))+(rule (lower (ishl ty @ $I32X4 src amt))(x64_pslld src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I64X2 (ishl _ src amt)))+(rule (lower (ishl ty @ $I64X2 src amt))(x64_psllq src (mov_rmi_to_xmm (mask_xmm_shift ty amt))));;;; Rules for `ushr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (ushr _ src amt)))+(rule -1 (lower (ushr (fits_in_64 ty) src amt))(let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Zero))))(x64_shr ty src_ (put_masked_in_imm8_gpr amt ty))))@@ -720,7 +694,7 @@(cmove $I64 (CC.Z) lo_shifted_ hi_shifted)(cmove $I64 (CC.Z) hi_shifted zero)))))-(rule (lower (has_type $I128 (ushr _ src amt)))+(rule (lower (ushr $I128 src amt));; NB: Only the low bits of `amt` matter since we logically mask the shift;; amount to the value's bit width.(let ((amt_ Gpr (lo_gpr amt)))@@ -730,7 +704,7 @@;; There are no 8x16 shifts in x64. Do the same 16x8-shift-and-mask thing we do;; with 8x16 `ishl`.-(rule (lower (has_type ty @ $I8X16 (ushr _ src amt)))+(rule (lower (ushr ty @ $I8X16 src amt))(let (;; Mask the amount to ensure wrapping behaviour(masked_amt RegMemImm (mask_xmm_shift ty amt))@@ -774,13 +748,13 @@;; 16x8, 32x4, and 64x2 shifts can each use a single instruction, once the shift amount is masked.-(rule (lower (has_type ty @ $I16X8 (ushr _ src amt)))+(rule (lower (ushr ty @ $I16X8 src amt))(x64_psrlw src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I32X4 (ushr _ src amt)))+(rule (lower (ushr ty @ $I32X4 src amt))(x64_psrld src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I64X2 (ushr _ src amt)))+(rule (lower (ushr ty @ $I64X2 src amt))(x64_psrlq src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))(decl mask_xmm_shift (Type Value) RegMemImm)@@ -793,7 +767,7 @@;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (sshr _ src amt)))+(rule -1 (lower (sshr (fits_in_64 ty) src amt))(let ((src_ Gpr (extend_to_gpr src ty (ExtendKind.Sign))))(x64_sar ty src_ (put_masked_in_imm8_gpr amt ty))))@@ -830,7 +804,7 @@(cmove $I64 (CC.Z) lo_shifted_ hi_shifted)(cmove $I64 (CC.Z) hi_shifted sign_bits)))))-(rule (lower (has_type $I128 (sshr _ src amt)))+(rule (lower (sshr $I128 src amt));; NB: Only the low bits of `amt` matter since we logically mask the shift;; amount to the value's bit width.(let ((amt_ Gpr (lo_gpr amt)))@@ -851,7 +825,7 @@;; hi.i16x8 = [(s8, s8), (s9, s9), ..., (s15, s15)];; shifted_hi.i16x8 = shift each lane of `high`;; result = [s0'', s1'', ..., s15'']-(rule (lower (has_type ty @ $I8X16 (sshr _ src amt @ (value_type amt_ty))))+(rule (lower (sshr ty @ $I8X16 src amt @ (value_type amt_ty)))(let ((src_ Xmm (put_in_xmm src));; Mask the amount to ensure wrapping behaviour(masked_amt RegMemImm (mask_xmm_shift ty amt))@@ -880,30 +854,30 @@;; `sshr.{i16x8,i32x4}` can be a simple `psra{w,d}`, we just have to make sure;; that if the shift amount is in a register, it is in an XMM register.-(rule (lower (has_type ty @ $I16X8 (sshr _ src amt)))+(rule (lower (sshr ty @ $I16X8 src amt))(x64_psraw src (mov_rmi_to_xmm (mask_xmm_shift ty amt))))-(rule (lower (has_type ty @ $I32X4 (sshr _ src amt)))+(rule (lower (sshr ty @ $I32X4 src amt))(x64_psrad src (mov_rmi_to_xmm (mask_xmm_shift ty amt))));; The `sshr.i64x2` CLIF instruction has no single x86 instruction in the older;; feature sets. To remedy this, a small dance is done with an unsigned right;; shift plus some extra ops.-(rule 3 (lower (has_type ty @ $I64X2 (sshr _ src (iconst _ n))))+(rule 3 (lower (sshr ty @ $I64X2 src (iconst _ n)))(if-let true (has_avx512vl))(if-let true (has_avx512f))(x64_vpsraq_imm src (shift_amount_masked ty n)))-(rule 2 (lower (has_type ty @ $I64X2 (sshr _ src amt)))+(rule 2 (lower (sshr ty @ $I64X2 src amt))(if-let true (has_avx512vl))(if-let true (has_avx512f))(let ((masked Gpr (x64_and $I64 amt (RegMemImm.Imm (shift_mask ty)))))(x64_vpsraq src (x64_movd_to_xmm masked))))-(rule 1 (lower (has_type $I64X2 (sshr _ src (u32_from_iconst amt))))+(rule 1 (lower (sshr $I64X2 src (u32_from_iconst amt)))(lower_i64x2_sshr_imm src (u32_and amt 63)))-(rule (lower (has_type $I64X2 (sshr _ src amt)))+(rule (lower (sshr $I64X2 src amt))(lower_i64x2_sshr_gpr src (x64_and $I64 amt (RegMemImm.Imm 63))))(decl lower_i64x2_sshr_imm (Xmm u32) Xmm)@@ -963,13 +937,13 @@;; `i64` and smaller: we can rely on x86's rotate-amount masking since;; we operate on the whole register. For const's we mask the constant.-(rule -1 (lower (has_type (fits_in_64 ty) (rotl _ src amt)))+(rule -1 (lower (rotl (fits_in_64 ty) src amt))(x64_rotl ty src (put_masked_in_imm8_gpr amt ty)));; `i128`.-(rule (lower (has_type $I128 (rotl _ src amt)))+(rule (lower (rotl $I128 src amt))(let ((src_ ValueRegs src);; NB: Only the low bits of `amt` matter since we logically mask the;; rotation amount to the value's bit width.@@ -984,13 +958,13 @@;; `i64` and smaller: we can rely on x86's rotate-amount masking since;; we operate on the whole register. For const's we mask the constant.-(rule -1 (lower (has_type (fits_in_64 ty) (rotr _ src amt)))+(rule -1 (lower (rotr (fits_in_64 ty) src amt))(x64_rotr ty src (put_masked_in_imm8_gpr amt ty)));; `i128`.-(rule (lower (has_type $I128 (rotr _ src amt)))+(rule (lower (rotr $I128 src amt))(let ((src_ ValueRegs src);; NB: Only the low bits of `amt` matter since we logically mask the;; rotation amount to the value's bit width.@@ -1004,10 +978,10 @@;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (ineg _ x)))+(rule -1 (lower (ineg (fits_in_64 ty) x))(x64_neg ty x))-(rule -2 (lower (has_type $I128 (ineg _ x)))+(rule -2 (lower (ineg $I128 x));; Get the high/low registers for `x`.(let ((regs ValueRegs x)(lo Gpr (value_regs_get_gpr regs 0))@@ -1018,26 +992,24 @@;; SSE.-(rule (lower (has_type $I8X16 (ineg _ x)))+(rule (lower (ineg $I8X16 x))(x64_psubb (imm $I8X16 0) x))-(rule (lower (has_type $I16X8 (ineg _ x)))+(rule (lower (ineg $I16X8 x))(x64_psubw (imm $I16X8 0) x))-(rule (lower (has_type $I32X4 (ineg _ x)))+(rule (lower (ineg $I32X4 x))(x64_psubd (imm $I32X4 0) x))-(rule (lower (has_type $I64X2 (ineg _ x)))+(rule (lower (ineg $I64X2 x))(x64_psubq (imm $I64X2 0) x));;;; Rules for `avg_round` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (multi_lane 8 16)- (avg_round _ x y)))+(rule (lower (avg_round (multi_lane 8 16) x y))(x64_pavgb x y))-(rule (lower (has_type (multi_lane 16 8)- (avg_round _ x y)))+(rule (lower (avg_round (multi_lane 16 8) x y))(x64_pavgw x y));;;; Rules for `imul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -1046,32 +1018,32 @@;; 8-bit base case, needs a special instruction encoding and additionally;; move sinkable loads to the right.-(rule -8 (lower (has_type $I8 (imul _ x y))) (x64_mul8 false x y))-(rule -7 (lower (has_type $I8 (imul _ (sinkable_load x) y))) (x64_mul8 false y x))+(rule -8 (lower (imul $I8 x y)) (x64_mul8 false x y))+(rule -7 (lower (imul $I8 (sinkable_load x) y)) (x64_mul8 false y x));; 16-to-64-bit base cases, same as above by moving sinkable loads to the right.-(rule -6 (lower (has_type (ty_int_ref_16_to_64 ty) (imul _ x y)))+(rule -6 (lower (imul (ty_int_ref_16_to_64 ty) x y))(x64_imul ty x y))-(rule -5 (lower (has_type (ty_int_ref_16_to_64 ty) (imul _ (sinkable_load x) y)))+(rule -5 (lower (imul (ty_int_ref_16_to_64 ty) (sinkable_load x) y))(x64_imul ty y x));; Lift out constants to use 3-operand form.-(rule -4 (lower (has_type (ty_int_ref_16_to_64 ty) (imul _ x (i32_from_iconst y))))+(rule -4 (lower (imul (ty_int_ref_16_to_64 ty) x (i32_from_iconst y)))(x64_imul_imm ty x y))-(rule -3 (lower (has_type (ty_int_ref_16_to_64 ty) (imul _ (i32_from_iconst x) y)))+(rule -3 (lower (imul (ty_int_ref_16_to_64 ty) (i32_from_iconst x) y))(x64_imul_imm ty y x));; Special case widening multiplication from 8-to-16-bits with a single;; instruction since the 8-bit-multiply places both the high and low halves in;; the same register-(rule -2 (lower (has_type $I16 (imul _ (sextend _ x) (sextend _ y))))+(rule -2 (lower (imul $I16 (sextend _ x) (sextend _ y)))(x64_mul8 true x y))-(rule -2 (lower (has_type $I16 (imul _ (uextend _ x) (uextend _ y))))+(rule -2 (lower (imul $I16 (uextend _ x) (uextend _ y)))(x64_mul8 false x y));; `i128`.-(rule 2 (lower (has_type $I128 (imul _ x y)))+(rule 2 (lower (imul $I128 x y))(let ((x_regs ValueRegs x)(y_regs ValueRegs y))(imul128@@ -1080,7 +1052,7 @@(value_regs_get_gpr y_regs 0)(value_regs_get_gpr y_regs 1))))-(rule 4 (lower (has_type $I128 (imul _ (iconcat _ x_lo x_hi) (iconcat _ y_lo y_hi))))+(rule 4 (lower (imul $I128 (iconcat _ x_lo x_hi) (iconcat _ y_lo y_hi)))(imul128 x_lo x_hi y_lo y_hi));; Helper for lowering 128-bit multiplication with the 64-bit halves of the@@ -1122,28 +1094,28 @@;; operands and producing a 128-bit result, which exactly matches the semantics;; of widening 64-bit inputs to 128-bit and then multiplying them. That means;; that these cases can get some some simpler codegen.-(rule 5 (lower (has_type $I128 (imul _ (uextend _ x @ (value_type $I64))- (uextend _ y @ (value_type $I64)))))+(rule 5 (lower (imul $I128 (uextend _ x @ (value_type $I64))+ (uextend _ y @ (value_type $I64))))(x64_mul $I64 false x y))-(rule 5 (lower (has_type $I128 (imul _ (sextend _ x @ (value_type $I64))- (sextend _ y @ (value_type $I64)))))+(rule 5 (lower (imul $I128 (sextend _ x @ (value_type $I64))+ (sextend _ y @ (value_type $I64))))(x64_mul $I64 true x y));; SSE.;; (No i8x16 multiply.)-(rule (lower (has_type (multi_lane 16 8) (imul _ x y)))+(rule (lower (imul (multi_lane 16 8) x y))(x64_pmullw x y))-(rule (lower (has_type (multi_lane 32 4) (imul _ x y)))+(rule (lower (imul (multi_lane 32 4) x y))(if-let true (has_sse41))(x64_pmulld x y));; Without `pmulld` the `pmuludq` instruction is used instead which performs;; 32-bit multiplication storing the 64-bit result. The 64-bit result is;; truncated to 32-bits and everything else is woven into place.-(rule -1 (lower (has_type (multi_lane 32 4) (imul _ x y)))+(rule -1 (lower (imul (multi_lane 32 4) x y))(let ((x Xmm x)(y Xmm y)@@ -1156,7 +1128,7 @@;; With AVX-512 we can implement `i64x2` multiplication with a single;; instruction.-(rule 3 (lower (has_type (multi_lane 64 2) (imul _ x y)))+(rule 3 (lower (imul (multi_lane 64 2) x y))(if-let true (has_avx512vl))(if-let true (has_avx512dq))(x64_vpmullq x y))@@ -1181,8 +1153,7 @@;; the lane of the destination. For this reason we don't need shifts to isolate;; the lower 32-bits, however, we will need to use shifts to isolate the high;; 32-bits when doing calculations, i.e., `Ah == A >> 32`.-(rule (lower (has_type (multi_lane 64 2)- (imul _ a b)))+(rule (lower (imul (multi_lane 64 2) a b))(let ((a0 Xmm a)(b0 Xmm b);; a_hi = A >> 32@@ -1203,11 +1174,10 @@(x64_paddq al_bl aa_bb_shifted)));; Special case for `i32x4.extmul_high_i16x8_s`.-(rule 1 (lower (has_type (multi_lane 32 4)- (imul _ (swiden_high _ (and (value_type (multi_lane 16 8))+(rule 1 (lower (imul (multi_lane 32 4) (swiden_high _ (and (value_type (multi_lane 16 8))x))(swiden_high _ (and (value_type (multi_lane 16 8))- y)))))+ y))))(let ((x2 Xmm x)(y2 Xmm y)(lo Xmm (x64_pmullw x2 y2))@@ -1215,22 +1185,20 @@(x64_punpckhwd lo hi)));; Special case for `i64x2.extmul_high_i32x4_s`.-(rule 1 (lower (has_type (multi_lane 64 2)- (imul _ (swiden_high _ (and (value_type (multi_lane 32 4))+(rule 1 (lower (imul (multi_lane 64 2) (swiden_high _ (and (value_type (multi_lane 32 4))x))(swiden_high _ (and (value_type (multi_lane 32 4))- y)))))+ y))))(if-let true (has_sse41))(let ((x2 Xmm (x64_pshufd x 0xFA))(y2 Xmm (x64_pshufd y 0xFA)))(x64_pmuldq x2 y2)));; Special case for `i32x4.extmul_low_i16x8_s`.-(rule 1 (lower (has_type (multi_lane 32 4)- (imul _ (swiden_low _ (and (value_type (multi_lane 16 8))+(rule 1 (lower (imul (multi_lane 32 4) (swiden_low _ (and (value_type (multi_lane 16 8))x))(swiden_low _ (and (value_type (multi_lane 16 8))- y)))))+ y))))(let ((x2 Xmm x)(y2 Xmm y)(lo Xmm (x64_pmullw x2 y2))@@ -1238,22 +1206,20 @@(x64_punpcklwd lo hi)));; Special case for `i64x2.extmul_low_i32x4_s`.-(rule 1 (lower (has_type (multi_lane 64 2)- (imul _ (swiden_low _ (and (value_type (multi_lane 32 4))+(rule 1 (lower (imul (multi_lane 64 2) (swiden_low _ (and (value_type (multi_lane 32 4))x))(swiden_low _ (and (value_type (multi_lane 32 4))- y)))))+ y))))(if-let true (has_sse41))(let ((x2 Xmm (x64_pshufd x 0x50))(y2 Xmm (x64_pshufd y 0x50)))(x64_pmuldq x2 y2)));; Special case for `i32x4.extmul_high_i16x8_u`.-(rule 1 (lower (has_type (multi_lane 32 4)- (imul _ (uwiden_high _ (and (value_type (multi_lane 16 8))+(rule 1 (lower (imul (multi_lane 32 4) (uwiden_high _ (and (value_type (multi_lane 16 8))x))(uwiden_high _ (and (value_type (multi_lane 16 8))- y)))))+ y))))(let ((x2 Xmm x)(y2 Xmm y)(lo Xmm (x64_pmullw x2 y2))@@ -1261,21 +1227,19 @@(x64_punpckhwd lo hi)));; Special case for `i64x2.extmul_high_i32x4_u`.-(rule 1 (lower (has_type (multi_lane 64 2)- (imul _ (uwiden_high _ (and (value_type (multi_lane 32 4))+(rule 1 (lower (imul (multi_lane 64 2) (uwiden_high _ (and (value_type (multi_lane 32 4))x))(uwiden_high _ (and (value_type (multi_lane 32 4))- y)))))+ y))))(let ((x2 Xmm (x64_pshufd x 0xFA))(y2 Xmm (x64_pshufd y 0xFA)))(x64_pmuludq x2 y2)));; Special case for `i32x4.extmul_low_i16x8_u`.-(rule 1 (lower (has_type (multi_lane 32 4)- (imul _ (uwiden_low _ (and (value_type (multi_lane 16 8))+(rule 1 (lower (imul (multi_lane 32 4) (uwiden_low _ (and (value_type (multi_lane 16 8))x))(uwiden_low _ (and (value_type (multi_lane 16 8))- y)))))+ y))))(let ((x2 Xmm x)(y2 Xmm y)(lo Xmm (x64_pmullw x2 y2))@@ -1283,43 +1247,42 @@(x64_punpcklwd lo hi)));; Special case for `i64x2.extmul_low_i32x4_u`.-(rule 1 (lower (has_type (multi_lane 64 2)- (imul _ (uwiden_low _ (and (value_type (multi_lane 32 4))+(rule 1 (lower (imul (multi_lane 64 2) (uwiden_low _ (and (value_type (multi_lane 32 4))x))(uwiden_low _ (and (value_type (multi_lane 32 4))- y)))))+ y))))(let ((x2 Xmm (x64_pshufd x 0x50))(y2 Xmm (x64_pshufd y 0x50)))(x64_pmuludq x2 y2)));;;; Rules for `iabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $I8X16 (iabs _ x)))+(rule 1 (lower (iabs $I8X16 x))(if-let true (has_ssse3))(x64_pabsb_a_or_avx x));; Note the use of `pminub` with signed inputs will produce the positive signed;; result which is what is desired here. The `pmaxub` isn't available until;; SSE4.1 in which case the single-instruction above lowering would apply.-(rule (lower (has_type $I8X16 (iabs _ x)))+(rule (lower (iabs $I8X16 x))(let ((x Xmm x)(negated Xmm (x64_psubb (xmm_zero $I8X16) x)))(x64_pminub_a x negated)))-(rule 1 (lower (has_type $I16X8 (iabs _ x)))+(rule 1 (lower (iabs $I16X8 x))(if-let true (has_ssse3))(x64_pabsw_a_or_avx x))-(rule (lower (has_type $I16X8 (iabs _ x)))+(rule (lower (iabs $I16X8 x))(let ((x Xmm x)(negated Xmm (x64_psubw (xmm_zero $I16X8) x)))(x64_pmaxsw_a x negated)))-(rule 1 (lower (has_type $I32X4 (iabs _ x)))+(rule 1 (lower (iabs $I32X4 x))(if-let true (has_ssse3))(x64_pabsd_a_or_avx x))@@ -1329,7 +1292,7 @@;; subtracting the mask this subtracts 0 for positive lanes (does nothing) or;; ends up adding one for negative lanes. This means that for a negative lane;; `x` the result is `!x + 1` which is the result of negating it.-(rule (lower (has_type $I32X4 (iabs _ x)))+(rule (lower (iabs $I32X4 x))(let ((x Xmm x)(negative_mask Xmm (x64_psrad x (xmi_imm 31)))@@ -1338,7 +1301,7 @@(x64_psubd flipped_if_negative negative_mask)));; When AVX512 is available, we can use a single `vpabsq` instruction.-(rule 2 (lower (has_type $I64X2 (iabs _ x)))+(rule 2 (lower (iabs $I64X2 x))(if-let true (has_avx512vl))(if-let true (has_avx512f))(x64_vpabsq x))@@ -1347,7 +1310,7 @@;; x` and then blend in those results with `blendvpd` if the MSB of `neg` was;; set to 1 (i.e. if `neg` was negative or, conversely, if `x` was originally;; positive).-(rule 1 (lower (has_type $I64X2 (iabs _ x)))+(rule 1 (lower (iabs $I64X2 x))(if-let true (has_sse41))(let ((rx Xmm x)(neg Xmm (x64_psubq (imm $I64X2 0) rx)))@@ -1356,7 +1319,7 @@;; and if `blendvpd` isn't available then perform a shift/shuffle to generate a;; mask of which lanes are negative, followed by flipping bits/sub to make both;; positive.-(rule (lower (has_type $I64X2 (iabs _ x)))+(rule (lower (iabs $I64X2 x))(let ((x Xmm x)(signs Xmm (x64_psrad x (RegMemImm.Imm 31)))(signs Xmm (x64_pshufd signs 0b11_11_01_01))@@ -1365,7 +1328,7 @@;; `i64` and smaller.-(rule -1 (lower (has_type (fits_in_64 ty) (iabs _ x)))+(rule -1 (lower (iabs (fits_in_64 ty) x))(let ((src Gpr x)(neg ProducesFlags (x64_neg_paired ty src));; Manually extract the result from the neg, then ignore@@ -1378,7 +1341,7 @@(with_flags_reg (produces_flags_ignore neg) cmove)));; `i128`. Negate the low bits, `adc` to the higher bits, then negate high bits.-(rule (lower (has_type $I128 (iabs _ x)))+(rule (lower (iabs $I128 x));; Get the high/low registers for `x`.(let ((x_regs ValueRegs x)(x_lo Gpr (value_regs_get_gpr x_regs 0))@@ -1400,35 +1363,35 @@;;;; Rules for `fabs` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fabs _ x)))+(rule (lower (fabs $F32 x))(x64_andps x (xmm_new (imm $F32 0x7fffffff))))-(rule (lower (has_type $F64 (fabs _ x)))+(rule (lower (fabs $F64 x))(x64_andpd x (xmm_new (imm $F64 0x7fffffffffffffff))));; Special case for `f32x4.abs`.-(rule (lower (has_type $F32X4 (fabs _ x)))+(rule (lower (fabs $F32X4 x))(x64_andps x(x64_psrld (vector_all_ones) (xmi_imm 1))));; Special case for `f64x2.abs`.-(rule (lower (has_type $F64X2 (fabs _ x)))+(rule (lower (fabs $F64X2 x))(x64_andpd x(x64_psrlq (vector_all_ones) (xmi_imm 1))));;;; Rules for `fneg` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fneg _ x)))+(rule (lower (fneg $F32 x))(x64_xorps x (xmm_new (imm $F32 0x80000000))))-(rule (lower (has_type $F64 (fneg _ x)))+(rule (lower (fneg $F64 x))(x64_xorpd x (xmm_new (imm $F64 0x8000000000000000))))-(rule (lower (has_type $F32X4 (fneg _ x)))+(rule (lower (fneg $F32X4 x))(x64_xorps x(x64_pslld (vector_all_ones) (xmi_imm 31))))-(rule (lower (has_type $F64X2 (fneg _ x)))+(rule (lower (fneg $F64X2 x))(x64_xorpd x(x64_psllq (vector_all_ones) (xmi_imm 63))))@@ -1474,14 +1437,14 @@;; Call the lower_bmask rule that does all the procssing-(rule (lower (has_type out_ty (bmask _ x @ (value_type in_ty))))+(rule (lower (bmask out_ty x @ (value_type in_ty)))(lower_bmask out_ty in_ty x));;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; `i64` and smaller.-(rule -2 (lower (has_type ty (bnot _ x)))+(rule -2 (lower (bnot ty x))(if (ty_int_ref_scalar_64 ty))(x64_not ty x))@@ -1496,25 +1459,24 @@(value_gprs (x64_not $I64 x_lo)(x64_not $I64 x_hi))))-(rule (lower (has_type $I128 (bnot _ x)))+(rule (lower (bnot $I128 x))(not_i128 x));; f32 and f64-(rule -3 (lower (has_type (ty_scalar_float ty) (bnot _ x)))+(rule -3 (lower (bnot (ty_scalar_float ty) x))(x64_xor_vector ty x (vector_all_ones)));; Special case for vector-types where bit-negation is an xor against an;; all-one value-(rule -1 (lower (has_type ty @ (multi_lane _bits _lanes) (bnot _ x)))+(rule -1 (lower (bnot ty @ (multi_lane _bits _lanes) x))(x64_xor_vector ty x (vector_all_ones)));;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty @ (multi_lane _bits _lanes)- (bitselect _ condition+(rule (lower (bitselect ty @ (multi_lane _bits _lanes) conditionif_true- if_false)))+ if_false));; a = and if_true, condition;; b = and_not condition, if_false;; or b, a@@ -1525,10 +1487,9 @@;; If every byte of the condition is guaranteed to be all ones or all zeroes,;; we can use x64_blend.-(rule 1 (lower (has_type ty @ (multi_lane _bits _lanes)- (bitselect _ condition+(rule 1 (lower (bitselect ty @ (multi_lane _bits _lanes) conditionif_true- if_false)))+ if_false))(if-let true (has_sse41))(if (all_ones_or_all_zeros condition))(x64_pblendvb if_false if_true condition))@@ -1547,29 +1508,29 @@;; instructions and how they're lowered into CLIF. Note the careful ordering;; of all the operands here to ensure that the input CLIF matched is implemented;; by the corresponding x64 instruction.-(rule 2 (lower (has_type $F32X4 (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) x y)) x y)))+(rule 2 (lower (bitselect $F32X4 (bitcast _ _ (fcmp _ (FloatCC.LessThan) x y)) x y))(x64_minps x y))-(rule 2 (lower (has_type $F64X2 (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) x y)) x y)))+(rule 2 (lower (bitselect $F64X2 (bitcast _ _ (fcmp _ (FloatCC.LessThan) x y)) x y))(x64_minpd x y))-(rule 3 (lower (has_type $F32X4 (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) y x)) x y)))+(rule 3 (lower (bitselect $F32X4 (bitcast _ _ (fcmp _ (FloatCC.LessThan) y x)) x y))(x64_maxps x y))-(rule 3 (lower (has_type $F64X2 (bitselect _ (bitcast _ _ (fcmp _ (FloatCC.LessThan) y x)) x y)))+(rule 3 (lower (bitselect $F64X2 (bitcast _ _ (fcmp _ (FloatCC.LessThan) y x)) x y))(x64_maxpd x y));; Scalar rules-(rule 3 (lower (has_type $I128 (bitselect _ c t f)))+(rule 3 (lower (bitselect $I128 c t f))(let ((a ValueRegs (and_i128 c t))(b ValueRegs (and_i128 (not_i128 c) f)))(or_i128 a b)))-(rule 4 (lower (has_type (ty_int_ref_scalar_64 ty) (bitselect _ c t f)))+(rule 4 (lower (bitselect (ty_int_ref_scalar_64 ty) c t f))(let ((a Gpr (x64_and ty c t))(b Gpr (x64_and ty (x64_not ty c) f)))(x64_or ty a b)))-(rule 5 (lower (has_type (ty_scalar_float ty) (bitselect _ c t f)))+(rule 5 (lower (bitselect (ty_scalar_float ty) c t f));; The SSE bitwise ops act on the full 128 bits of an XMM register, so;; force each scalar float operand into an XMM register first -- we;; must not fold a 4/8-byte load into an operand that the instruction@@ -1584,18 +1545,15 @@;;;; Rules for `blendv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16- (blendv _ condition if_true if_false)))+(rule (lower (blendv $I8X16 condition if_true if_false))(if-let true (has_sse41))(x64_pblendvb if_false if_true condition))-(rule (lower (has_type $I32X4- (blendv _ condition if_true if_false)))+(rule (lower (blendv $I32X4 condition if_true if_false))(if-let true (has_sse41))(x64_blendvps if_false if_true condition))-(rule (lower (has_type $I64X2- (blendv _ condition if_true if_false)))+(rule (lower (blendv $I64X2 condition if_true if_false))(if-let true (has_sse41))(x64_blendvpd if_false if_true condition))@@ -1696,7 +1654,7 @@;; (i64x2.replace_lane 1) with a splat as source for lane 0 -- we can elide;; the splat and just do a move. This turns out to be a common pattern when;; constructing an i64x2 out of two i64s.-(rule 3 (lower (insertlane _ (has_type $I64X2 (splat _ lane0))+(rule 3 (lower (insertlane _ (splat $I64X2 lane0)lane1(u8_from_uimm8 1)))(if-let true (has_sse41))@@ -1776,16 +1734,16 @@(with_flags_reg (x64_cmp ty y_reg x_reg)(cmove ty cc y_reg x_reg))))-(rule -1 (lower (has_type (fits_in_64 ty) (umin _ x y)))+(rule -1 (lower (umin (fits_in_64 ty) x y))(cmp_and_choose ty (CC.B) x y))-(rule -1 (lower (has_type (fits_in_64 ty) (umax _ x y)))+(rule -1 (lower (umax (fits_in_64 ty) x y))(cmp_and_choose ty (CC.NB) x y))-(rule -1 (lower (has_type (fits_in_64 ty) (smin _ x y)))+(rule -1 (lower (smin (fits_in_64 ty) x y))(cmp_and_choose ty (CC.L) x y))-(rule -1 (lower (has_type (fits_in_64 ty) (smax _ x y)))+(rule -1 (lower (smax (fits_in_64 ty) x y))(cmp_and_choose ty (CC.NL) x y));; SSE helpers for determining if single-instruction lowerings are available.@@ -1812,7 +1770,7 @@;; SSE `smax`.-(rule (lower (has_type (ty_vec128 ty) (smax _ x y)))+(rule (lower (smax (ty_vec128 ty) x y))(lower_vec_smax ty x y))(decl lower_vec_smax (Type Xmm Xmm) Xmm)@@ -1832,11 +1790,11 @@;; SSE `smin`.-(rule 1 (lower (has_type (ty_vec128 ty) (smin _ x y)))+(rule 1 (lower (smin (ty_vec128 ty) x y))(if-let true (has_pmins ty))(x64_pmins ty x y))-(rule (lower (has_type (ty_vec128 ty) (smin _ x y)))+(rule (lower (smin (ty_vec128 ty) x y))(let ((x Xmm x)(y Xmm y)@@ -1848,20 +1806,20 @@;; SSE `umax`.-(rule 2 (lower (has_type (ty_vec128 ty) (umax _ x y)))+(rule 2 (lower (umax (ty_vec128 ty) x y))(if-let true (has_pmaxu ty))(x64_pmaxu ty x y));; If y < x then the saturating subtraction will be zero, otherwise when added;; back to x it'll return y.-(rule 1 (lower (has_type $I16X8 (umax _ x y)))+(rule 1 (lower (umax $I16X8 x y))(let ((x Xmm x))(x64_paddw x (x64_psubusw y x))));; Flip the upper bits of each lane so the signed comparison has the same;; result as a signed comparison, and then select the results with the output;; mask. See `pcmpgt` lowering for info on flipping the upper bit.-(rule (lower (has_type (ty_vec128 ty) (umax _ x y)))+(rule (lower (umax (ty_vec128 ty) x y))(let ((x Xmm x)(y Xmm y)@@ -1884,18 +1842,18 @@;; SSE `umin`.-(rule 2 (lower (has_type (ty_vec128 ty) (umin _ x y)))+(rule 2 (lower (umin (ty_vec128 ty) x y))(if-let true (has_pminu ty))(x64_pminu ty x y));; If x < y then the saturating subtraction will be 0. Otherwise if x > y then;; the saturated result, when subtracted again, will go back to `y`.-(rule 1 (lower (has_type $I16X8 (umin _ x y)))+(rule 1 (lower (umin $I16X8 x y))(let ((x Xmm x))(x64_psubw x (x64_psubusw x y))));; Same as `umax`, and see `pcmpgt` for docs on flipping the upper bit.-(rule (lower (has_type (ty_vec128 ty) (umin _ x y)))+(rule (lower (umin (ty_vec128 ty) x y))(let ((x Xmm x)(y Xmm y)@@ -1933,7 +1891,7 @@;;;; Rules for `uadd_overflow_trap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (fits_in_64 ty) (uadd_overflow_trap _ a b tc)))+(rule (lower (uadd_overflow_trap (fits_in_64 ty) a b tc))(with_flags(x64_add_with_flags_paired ty a b)(trap_if (CC.B) tc)))@@ -1941,14 +1899,12 @@;; Handle lhs immediates/sinkable loads in addition to the automatic rhs;; handling of above.-(rule 1 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap _ (simm32_from_value a) b tc)))+(rule 1 (lower (uadd_overflow_trap (fits_in_64 ty) (simm32_from_value a) b tc))(with_flags(x64_add_with_flags_paired ty b a)(trap_if (CC.B) tc)))-(rule 2 (lower (has_type (fits_in_64 ty)- (uadd_overflow_trap _ (sinkable_load a) b tc)))+(rule 2 (lower (uadd_overflow_trap (fits_in_64 ty) (sinkable_load a) b tc))(with_flags(x64_add_with_flags_paired ty b a)(trap_if (CC.B) tc)))@@ -1968,35 +1924,35 @@(lower_cond_bool (emit_cmp cc a b)));; Peephole optimization for `x < 0`, when x is a signed 64 bit value-(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedLessThan) x @ (value_type $I64) (u64_from_iconst 0))))+(rule 2 (lower (icmp $I8 (IntCC.SignedLessThan) x @ (value_type $I64) (u64_from_iconst 0)))(x64_shrq_mi x 63));; Peephole optimization for `0 > x`, when x is a signed 64 bit value-(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I64))))+(rule 2 (lower (icmp $I8 (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I64)))(x64_shrq_mi x 63));; Peephole optimization for `0 <= x`, when x is a signed 64 bit value-(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I64))))+(rule 2 (lower (icmp $I8 (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I64)))(x64_shrq_mi (x64_not $I64 x) 63));; Peephole optimization for `x >= 0`, when x is a signed 64 bit value-(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I64) (u64_from_iconst 0))))+(rule 2 (lower (icmp $I8 (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I64) (u64_from_iconst 0)))(x64_shrq_mi (x64_not $I64 x) 63));; Peephole optimization for `x < 0`, when x is a signed 32 bit value-(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedLessThan) x @ (value_type $I32) (u64_from_iconst 0))))+(rule 2 (lower (icmp $I8 (IntCC.SignedLessThan) x @ (value_type $I32) (u64_from_iconst 0)))(x64_shrl_mi x 31));; Peephole optimization for `0 > x`, when x is a signed 32 bit value-(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I32))))+(rule 2 (lower (icmp $I8 (IntCC.SignedGreaterThan) (u64_from_iconst 0) x @ (value_type $I32)))(x64_shrl_mi x 31));; Peephole optimization for `0 <= x`, when x is a signed 32 bit value-(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I32))))+(rule 2 (lower (icmp $I8 (IntCC.SignedLessThanOrEqual) (u64_from_iconst 0) x @ (value_type $I32)))(x64_shrl_mi (x64_not $I32 x) 31));; Peephole optimization for `x >= 0`, when x is a signed 32 bit value-(rule 2 (lower (has_type $I8 (icmp _ (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I32) (u64_from_iconst 0))))+(rule 2 (lower (icmp $I8 (IntCC.SignedGreaterThanOrEqual) x @ (value_type $I32) (u64_from_iconst 0)))(x64_shrl_mi (x64_not $I32 x) 31));; For XMM-held values, we lower to `PCMP*` instructions, sometimes more than@@ -2264,29 +2220,28 @@;; Specializations for floating-point compares to generate a `mins*` or a;; `maxs*` instruction. These are equivalent to the "pseudo-m{in,ax}";; specializations for vectors.-(rule 3 (lower (has_type $F32 (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) x y)) x y)))+(rule 3 (lower (select $F32 (maybe_uextend (fcmp _ (FloatCC.LessThan) x y)) x y))(x64_minss x y))-(rule 3 (lower (has_type $F64 (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) x y)) x y)))+(rule 3 (lower (select $F64 (maybe_uextend (fcmp _ (FloatCC.LessThan) x y)) x y))(x64_minsd x y))-(rule 4 (lower (has_type $F32 (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) y x)) x y)))+(rule 4 (lower (select $F32 (maybe_uextend (fcmp _ (FloatCC.LessThan) y x)) x y))(x64_maxss x y))-(rule 4 (lower (has_type $F64 (select _ (maybe_uextend (fcmp _ (FloatCC.LessThan) y x)) x y)))+(rule 4 (lower (select $F64 (maybe_uextend (fcmp _ (FloatCC.LessThan) y x)) x y))(x64_maxsd x y));; Rules for `clz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (has_type (ty_32_or_64 ty) (clz _ src)))+(rule 2 (lower (clz (ty_32_or_64 ty) src))(do_clz ty ty src))-(rule 1 (lower (has_type (ty_8_or_16 ty) (clz _ src)))+(rule 1 (lower (clz (ty_8_or_16 ty) src))(let ((extended Gpr (extend_to_gpr src $I64 (ExtendKind.Zero)))(clz Gpr (do_clz $I64 $I64 extended)))(x64_sub $I64 clz (RegMemImm.Imm (u32_wrapping_sub 64 (ty_bits ty))))))(rule 0 (lower- (has_type $I128- (clz _ src)))+ (clz $I128 src))(let ((upper Gpr (do_clz $I64 $I64 (value_regs_get_gpr src 1)))(lower Gpr (x64_add $I64(do_clz $I64 $I64 (value_regs_get_gpr src 0))@@ -2315,17 +2270,16 @@;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (has_type (ty_32_or_64 ty) (ctz _ src)))+(rule 2 (lower (ctz (ty_32_or_64 ty) src))(do_ctz ty ty src))-(rule 1 (lower (has_type (ty_8_or_16 ty) (ctz _ src)))+(rule 1 (lower (ctz (ty_8_or_16 ty) src))(let ((extended Gpr (extend_to_gpr src $I32 (ExtendKind.Zero)))(stopbit Gpr (x64_or $I32 extended (RegMemImm.Imm (u32_wrapping_shl 1 (ty_bits ty))))))(do_ctz $I32 ty stopbit)))(rule 0 (lower- (has_type $I128- (ctz _ src)))+ (ctz $I128 src))(let ((lower Gpr (do_ctz $I64 $I64 (value_regs_get_gpr src 0)))(upper Gpr (x64_add $I64(do_ctz $I64 $I64 (value_regs_get_gpr src 1))@@ -2349,17 +2303,16 @@;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 2 (lower (has_type (ty_32_or_64 ty) (cls _ src)))+(rule 2 (lower (cls (ty_32_or_64 ty) src))(do_cls ty src))-(rule 1 (lower (has_type (ty_8_or_16 ty) (cls _ src)))+(rule 1 (lower (cls (ty_8_or_16 ty) src))(let ((extended Gpr (extend_to_gpr src $I32 (ExtendKind.Sign)))(cls Gpr (do_cls $I32 extended)))(x64_sub $I32 cls (RegMemImm.Imm (u32_wrapping_sub 32 (ty_bits ty))))))(rule 0 (lower- (has_type $I128- (cls _ src)))+ (cls $I128 src))(let ((upper Gpr (do_cls $I64 (value_regs_get_gpr src 1)))(sign_fill Gpr (x64_sarq_mi (value_regs_get_gpr src 1) 63))(xored Gpr (x64_xor $I64 (value_regs_get_gpr src 0) sign_fill))@@ -2384,33 +2337,30 @@;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 4 (lower (has_type (ty_32_or_64 ty) (popcnt _ src)))+(rule 4 (lower (popcnt (ty_32_or_64 ty) src))(if-let true (use_popcnt))(x64_popcnt ty src))-(rule 3 (lower (has_type (ty_8_or_16 ty) (popcnt _ src)))+(rule 3 (lower (popcnt (ty_8_or_16 ty) src))(if-let true (use_popcnt))(x64_popcnt $I32 (extend_to_gpr src $I32 (ExtendKind.Zero))))-(rule 1 (lower (has_type $I128 (popcnt _ src)))+(rule 1 (lower (popcnt $I128 src))(if-let true (use_popcnt))(let ((lo_count Gpr (x64_popcnt $I64 (value_regs_get_gpr src 0)))(hi_count Gpr (x64_popcnt $I64 (value_regs_get_gpr src 1))))(value_regs (x64_add $I64 lo_count hi_count) (imm $I64 0))))(rule -1 (lower- (has_type (ty_32_or_64 ty)- (popcnt _ src)))+ (popcnt (ty_32_or_64 ty) src))(do_popcnt ty src))(rule -2 (lower- (has_type (ty_8_or_16 ty)- (popcnt _ src)))+ (popcnt (ty_8_or_16 ty) src))(do_popcnt $I32 (extend_to_gpr src $I32 (ExtendKind.Zero))))(rule (lower- (has_type $I128- (popcnt _ src)))+ (popcnt $I128 src))(let ((lo_count Gpr (do_popcnt $I64 (value_regs_get_gpr src 0)))(hi_count Gpr (do_popcnt $I64 (value_regs_get_gpr src 1))))(value_regs (x64_add $I64 lo_count hi_count) (imm $I64 0))))@@ -2477,7 +2427,7 @@final))-(rule 2 (lower (has_type $I8X16 (popcnt _ src)))+(rule 2 (lower (popcnt $I8X16 src))(if-let true (has_avx512vl))(if-let true (has_avx512bitalg))(x64_vpopcntb src))@@ -2503,7 +2453,7 @@;; __m128i lookup = _mm_setr_epi8(0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4);-(rule 1 (lower (has_type $I8X16 (popcnt _ src)))+(rule 1 (lower (popcnt $I8X16 src))(if-let true (has_ssse3))(let ((low_mask XmmMem128 (emit_xmm_mem_128_le_const 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f))(low_nibbles Xmm (x64_pand src low_mask))@@ -2519,7 +2469,7 @@(x64_paddb bit_counts_low bit_counts_high)));; A modified version of the popcnt method from Hacker's Delight.-(rule (lower (has_type $I8X16 (popcnt _ src)))+(rule (lower (popcnt $I8X16 src))(let ((mask1 XmmMem128 (emit_xmm_mem_128_le_const 0x77777777777777777777777777777777))(src Xmm src)(shifted Xmm (x64_pand (x64_psrlq src (xmi_imm 1)) mask1))@@ -2533,19 +2483,19 @@;; Rules for `bitrev` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8 (bitrev _ src)))+(rule (lower (bitrev $I8 src))(do_bitrev8 $I32 src))-(rule (lower (has_type $I16 (bitrev _ src)))+(rule (lower (bitrev $I16 src))(do_bitrev16 $I32 src))-(rule (lower (has_type $I32 (bitrev _ src)))+(rule (lower (bitrev $I32 src))(do_bitrev32 $I32 src))-(rule (lower (has_type $I64 (bitrev _ src)))+(rule (lower (bitrev $I64 src))(do_bitrev64 $I64 src))-(rule (lower (has_type $I128 (bitrev _ src)))+(rule (lower (bitrev $I128 src))(value_regs(do_bitrev64 $I64 (value_regs_get_gpr src 1))(do_bitrev64 $I64 (value_regs_get_gpr src 0))))@@ -2612,16 +2562,16 @@;; x64 bswap instruction is only for 32- or 64-bit swaps;; implement the 16-bit swap as a rotl by 8-(rule (lower (has_type $I16 (bswap _ src)))+(rule (lower (bswap $I16 src))(x64_rolw_mi src 8))-(rule (lower (has_type $I32 (bswap _ src)))+(rule (lower (bswap $I32 src))(x64_bswap $I32 src))-(rule (lower (has_type $I64 (bswap _ src)))+(rule (lower (bswap $I64 src))(x64_bswap $I64 src))-(rule (lower (has_type $I128 (bswap _ src)))+(rule (lower (bswap $I128 src))(value_regs(x64_bswap $I64 (value_regs_get_gpr src 1))(x64_bswap $I64 (value_regs_get_gpr src 0))))@@ -2629,16 +2579,16 @@;; Rules for `uextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; I{8,16,32,64} -> I128.-(rule (lower (has_type $I128 (uextend _ src)))+(rule (lower (uextend $I128 src))(value_regs (extend_to_gpr src $I64 (ExtendKind.Zero)) (imm $I64 0)));; I{8,16,32} -> I64.-(rule (lower (has_type $I64 (uextend _ src)))+(rule (lower (uextend $I64 src))(extend_to_gpr src $I64 (ExtendKind.Zero)));; I{8,16} -> I32;; I8 -> I16-(rule -1 (lower (has_type (fits_in_32 _) (uextend _ src)))+(rule -1 (lower (uextend (fits_in_32 _) src))(extend_to_gpr src $I32 (ExtendKind.Zero)));; Rules for `sextend` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2647,30 +2597,30 @@;;;; Produce upper 64 bits sign-extended from lower 64: shift right by;; 63 bits to spread the sign bit across the result.-(rule (lower (has_type $I128 (sextend _ src)))+(rule (lower (sextend $I128 src))(let ((lo Gpr (extend_to_gpr src $I64 (ExtendKind.Sign)))(hi Gpr (x64_sarq_mi lo 63)))(value_regs lo hi)));; I{8,16,32} -> I64.-(rule (lower (has_type $I64 (sextend _ src)))+(rule (lower (sextend $I64 src))(extend_to_gpr src $I64 (ExtendKind.Sign)));; I{8,16} -> I32;; I8 -> I16-(rule -1 (lower (has_type (fits_in_32 _) (sextend _ src)))+(rule -1 (lower (sextend (fits_in_32 _) src))(extend_to_gpr src $I32 (ExtendKind.Sign)));; Rules for `ireduce` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; T -> T is always a no-op, even I128 -> I128.-(rule (lower (has_type ty (ireduce _ src @ (value_type ty))))+(rule (lower (ireduce ty src @ (value_type ty)))src);; T -> I{64,32,16,8}: We can simply pass through the value: values;; are always stored with high bits undefined, so we can just leave;; them be.-(rule 1 (lower (has_type (fits_in_64 ty) (ireduce _ src)))+(rule 1 (lower (ireduce (fits_in_64 ty) src))(value_regs_get_gpr src 0));; Rules for `debugtrap` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2680,107 +2630,107 @@;; Rules for `x86_pmaddubsw` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I16X8 (x86_pmaddubsw _ x y)))+(rule (lower (x86_pmaddubsw $I16X8 x y))(if-let true (has_ssse3))(x64_pmaddubsw y x));; Rules for `fadd` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fadd _ x y)))+(rule (lower (fadd $F32 x y))(x64_addss x y))-(rule (lower (has_type $F64 (fadd _ x y)))+(rule (lower (fadd $F64 x y))(x64_addsd x y))-(rule (lower (has_type $F32X4 (fadd _ x y)))+(rule (lower (fadd $F32X4 x y))(x64_addps x y))-(rule (lower (has_type $F64X2 (fadd _ x y)))+(rule (lower (fadd $F64X2 x y))(x64_addpd x y));; The above rules automatically sink loads for rhs operands, so additionally;; add rules for sinking loads with lhs operands.-(rule 1 (lower (has_type $F32 (fadd _ (sinkable_load_32 x) y)))+(rule 1 (lower (fadd $F32 (sinkable_load_32 x) y))(x64_addss y x))-(rule 1 (lower (has_type $F64 (fadd _ (sinkable_load_64 x) y)))+(rule 1 (lower (fadd $F64 (sinkable_load_64 x) y))(x64_addsd y x))-(rule 1 (lower (has_type $F32X4 (fadd _ (sinkable_load_128 x) y)))+(rule 1 (lower (fadd $F32X4 (sinkable_load_128 x) y))(x64_addps y x))-(rule 1 (lower (has_type $F64X2 (fadd _ (sinkable_load_128 x) y)))+(rule 1 (lower (fadd $F64X2 (sinkable_load_128 x) y))(x64_addpd y x));; Rules for `fsub` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fsub _ x y)))+(rule (lower (fsub $F32 x y))(x64_subss x y))-(rule (lower (has_type $F64 (fsub _ x y)))+(rule (lower (fsub $F64 x y))(x64_subsd x y))-(rule (lower (has_type $F32X4 (fsub _ x y)))+(rule (lower (fsub $F32X4 x y))(x64_subps x y))-(rule (lower (has_type $F64X2 (fsub _ x y)))+(rule (lower (fsub $F64X2 x y))(x64_subpd x y));; Rules for `fmul` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmul _ x y)))+(rule (lower (fmul $F32 x y))(x64_mulss x y))-(rule (lower (has_type $F64 (fmul _ x y)))+(rule (lower (fmul $F64 x y))(x64_mulsd x y))-(rule (lower (has_type $F32X4 (fmul _ x y)))+(rule (lower (fmul $F32X4 x y))(x64_mulps x y))-(rule (lower (has_type $F64X2 (fmul _ x y)))+(rule (lower (fmul $F64X2 x y))(x64_mulpd x y));; The above rules automatically sink loads for rhs operands, so additionally;; add rules for sinking loads with lhs operands.-(rule 1 (lower (has_type $F32 (fmul _ (sinkable_load_32 x) y)))+(rule 1 (lower (fmul $F32 (sinkable_load_32 x) y))(x64_mulss y x))-(rule 1 (lower (has_type $F64 (fmul _ (sinkable_load_64 x) y)))+(rule 1 (lower (fmul $F64 (sinkable_load_64 x) y))(x64_mulsd y x))-(rule 1 (lower (has_type $F32X4 (fmul _ (sinkable_load_128 x) y)))+(rule 1 (lower (fmul $F32X4 (sinkable_load_128 x) y))(x64_mulps y x))-(rule 1 (lower (has_type $F64X2 (fmul _ (sinkable_load_128 x) y)))+(rule 1 (lower (fmul $F64X2 (sinkable_load_128 x) y))(x64_mulpd y x));; Rules for `fdiv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdiv _ x y)))+(rule (lower (fdiv $F32 x y))(x64_divss x y))-(rule (lower (has_type $F64 (fdiv _ x y)))+(rule (lower (fdiv $F64 x y))(x64_divsd x y))-(rule (lower (has_type $F32X4 (fdiv _ x y)))+(rule (lower (fdiv $F32X4 x y))(x64_divps x y))-(rule (lower (has_type $F64X2 (fdiv _ x y)))+(rule (lower (fdiv $F64X2 x y))(x64_divpd x y));; Rules for `sqrt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (sqrt _ x)))+(rule (lower (sqrt $F32 x))(x64_sqrtss (xmm_zero $F32X4) x))-(rule (lower (has_type $F64 (sqrt _ x)))+(rule (lower (sqrt $F64 x))(x64_sqrtsd (xmm_zero $F64X2) x))-(rule (lower (has_type $F32X4 (sqrt _ x)))+(rule (lower (sqrt $F32X4 x))(x64_sqrtps x))-(rule (lower (has_type $F64X2 (sqrt _ x)))+(rule (lower (sqrt $F64X2 x))(x64_sqrtpd x));; Rules for `fpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F64 (fpromote _ x)))+(rule (lower (fpromote $F64 x))(x64_cvtss2sd (xmm_zero $F64X2) x));; Rules for `fvpromote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F64X2 (fvpromote_low _ x)))+(rule (lower (fvpromote_low $F64X2 x))(x64_cvtps2pd (put_in_xmm x)));; Rules for `fdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fdemote _ x)))+(rule (lower (fdemote $F32 x))(x64_cvtsd2ss (xmm_zero $F32X4) x));; Rules for `fvdemote` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32X4 (fvdemote _ x)))+(rule (lower (fvdemote $F32X4 x))(x64_cvtpd2ps x));; Rules for `fmin` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmin _ x y)))+(rule (lower (fmin $F32 x y))(xmm_min_max_seq $F32 true x y))-(rule (lower (has_type $F64 (fmin _ x y)))+(rule (lower (fmin $F64 x y))(xmm_min_max_seq $F64 true x y));; Vector-typed version. We don't use single pseudoinstructions as@@ -2795,7 +2745,7 @@;; require that fmin(NaN, _) = fmin(_, NaN) = NaN, and fmin(+0, -0) =;; fmin(-0, +0) = -0.-(rule (lower (has_type $F32X4 (fmin _ x y)))+(rule (lower (fmin $F32X4 x y));; Compute min(x, y) and min(y, x) with native;; instructions. These will differ in one of the edge cases;; above that we have to handle properly. (Conversely, if they@@ -2840,7 +2790,7 @@;; Likewise for F64 lanes, except that the right-shift is by 13 bits;; (1 sign, 11 exponent, 1 QNaN bit).-(rule (lower (has_type $F64X2 (fmin _ x y)))+(rule (lower (fmin $F64X2 x y))(let ((x Xmm x) ;; force x/y into registers and disallow load sinking(y Xmm y)(min1 Xmm (x64_minpd x y))@@ -2854,15 +2804,15 @@;; Rules for `fmax` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fmax _ x y)))+(rule (lower (fmax $F32 x y))(xmm_min_max_seq $F32 false x y))-(rule (lower (has_type $F64 (fmax _ x y)))+(rule (lower (fmax $F64 x y))(xmm_min_max_seq $F64 false x y));; The vector version of fmax here is a dual to the fmin sequence;; above, almost, with a few differences.-(rule (lower (has_type $F32X4 (fmax _ x y)))+(rule (lower (fmax $F32X4 x y));; Compute max(x, y) and max(y, x) with native;; instructions. These will differ in one of the edge cases;; above that we have to handle properly. (Conversely, if they@@ -2907,7 +2857,7 @@(final Xmm (x64_andnps nan_fraction_mask max_blended_nan_positive)))final))-(rule (lower (has_type $F64X2 (fmax _ x y)))+(rule (lower (fmax $F64X2 x y));; Compute max(x, y) and max(y, x) with native;; instructions. These will differ in one of the edge cases;; above that we have to handle properly. (Conversely, if they@@ -2958,12 +2908,12 @@;; Base case for fma is to call out to one of two libcalls. For vectors they;; need to be decomposed, handle each element individually, and then recomposed.-(rule (lower (has_type $F32 (fma _ x y z)))+(rule (lower (fma $F32 x y z))(libcall_3 (LibCall.FmaF32) x y z))-(rule (lower (has_type $F64 (fma _ x y z)))+(rule (lower (fma $F64 x y z))(libcall_3 (LibCall.FmaF64) x y z))-(rule (lower (has_type $F32X4 (fma _ x y z)))+(rule (lower (fma $F32X4 x y z))(let ((x Xmm (put_in_xmm x))(y Xmm (put_in_xmm y))@@ -2987,7 +2937,7 @@(tmp Xmm (f32x4_insertlane tmp x3 3)))tmp))-(rule (lower (has_type $F64X2 (fma _ x y z)))+(rule (lower (fma $F64X2 x y z))(let ((x Xmm (put_in_xmm x))(y Xmm (put_in_xmm y))@@ -3003,7 +2953,7 @@;; Special case for when the `fma` feature is active and a native instruction;; can be used.-(rule 1 (lower (has_type ty (fma _ x y z)))+(rule 1 (lower (fma ty x y z))(if-let true (use_fma))(fmadd ty x y z))@@ -3032,7 +2982,7 @@(rule 2 (fnmadd ty x y @ (sinkable_load _) z) (x64_vfnmadd132 ty x z y))-(rule 2 (lower (has_type ty (fma _ x y (fneg _ z))))+(rule 2 (lower (fma ty x y (fneg _ z)))(if-let true (use_fma))(fmsub ty x y z))@@ -3067,49 +3017,47 @@;; 8-bit loads.;;;; By default, we zero-extend all sub-64-bit loads to a GPR.-(rule load_sub64_x64_movzx -4 (lower (has_type (and (fits_in_32 ty) (is_gpr_type _))- (load _ (little_or_native_endian flags) address offset)))+(rule load_sub64_x64_movzx -4 (lower (load (and (fits_in_32 ty) (is_gpr_type _)) (little_or_native_endian flags) address offset))(x64_movzx (ext_mode (ty_bits_u16 ty) 64) (to_amode flags address offset)));; But if we know that both the `from` and `to` are 64 bits, we simply load with;; no extension.-(rule load_64_x64_movzx -1 (lower (has_type (ty_int_ref_64 ty) (load _ (little_or_native_endian flags) address offset)))+(rule load_64_x64_movzx -1 (lower (load (ty_int_ref_64 ty) (little_or_native_endian flags) address offset))(x64_mov (to_amode flags address offset)));; Also, certain scalar loads have a specific `from` width and extension kind;; (signed -> `sx`, zeroed -> `zx`). We overwrite the high bits of the 64-bit;; GPR even if the `to` type is smaller (e.g., 16-bits).-(rule (lower (has_type (is_gpr_type ty) (uload8 _ (little_or_native_endian flags) address offset)))+(rule (lower (uload8 (is_gpr_type ty) (little_or_native_endian flags) address offset))(x64_movzx (ExtMode.BQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (sload8 _ (little_or_native_endian flags) address offset)))+(rule (lower (sload8 (is_gpr_type ty) (little_or_native_endian flags) address offset))(x64_movsx (ExtMode.BQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (uload16 _ (little_or_native_endian flags) address offset)))+(rule (lower (uload16 (is_gpr_type ty) (little_or_native_endian flags) address offset))(x64_movzx (ExtMode.WQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (sload16 _ (little_or_native_endian flags) address offset)))+(rule (lower (sload16 (is_gpr_type ty) (little_or_native_endian flags) address offset))(x64_movsx (ExtMode.WQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (uload32 _ (little_or_native_endian flags) address offset)))+(rule (lower (uload32 (is_gpr_type ty) (little_or_native_endian flags) address offset))(x64_movzx (ExtMode.LQ) (to_amode flags address offset)))-(rule (lower (has_type (is_gpr_type ty) (sload32 _ (little_or_native_endian flags) address offset)))+(rule (lower (sload32 (is_gpr_type ty) (little_or_native_endian flags) address offset))(x64_movsx (ExtMode.LQ) (to_amode flags address offset)));; To load to XMM registers, we use the x64-specific instructions for each type.;; For `$F32` and `$F64` this is important--we only want to load 32 or 64 bits.;; But for the 128-bit types, this is not strictly necessary for performance but;; might help with clarity during disassembly.-(rule 4 (lower (has_type (is_xmm_type (ty_16 _)) (load _ (little_or_native_endian flags) address offset)))+(rule 4 (lower (load (is_xmm_type (ty_16 _)) (little_or_native_endian flags) address offset))(x64_pinsrw (xmm_uninit_value) (to_amode flags address offset) 0))-(rule 3 (lower (has_type (is_xmm_type (ty_32 _)) (load _ (little_or_native_endian flags) address offset)))+(rule 3 (lower (load (is_xmm_type (ty_32 _)) (little_or_native_endian flags) address offset))(x64_movss_load (to_amode flags address offset)))-(rule 2 (lower (has_type (is_xmm_type (ty_64 _)) (load _ (little_or_native_endian flags) address offset)))+(rule 2 (lower (load (is_xmm_type (ty_64 _)) (little_or_native_endian flags) address offset))(x64_movsd_load (to_amode flags address offset)))-(rule 1 (lower (has_type $F32X4 (load _ (little_or_native_endian flags) address offset)))+(rule 1 (lower (load $F32X4 (little_or_native_endian flags) address offset))(x64_movups_load (to_amode flags address offset)))-(rule 1 (lower (has_type $F64X2 (load _ (little_or_native_endian flags) address offset)))+(rule 1 (lower (load $F64X2 (little_or_native_endian flags) address offset))(x64_movupd_load (to_amode flags address offset)))-(rule 0 (lower (has_type (is_xmm_type (ty_128 _)) (load _ (little_or_native_endian flags) address offset)))+(rule 0 (lower (load (is_xmm_type (ty_128 _)) (little_or_native_endian flags) address offset))(x64_movdqu_load (synthetic_amode_to_xmm_mem_128 (to_amode flags address offset))));; We can load an I128 by doing two 64-bit loads.-(rule -3 (lower (has_type $I128- (load _ (little_or_native_endian flags) address offset)))+(rule -3 (lower (load $I128 (little_or_native_endian flags) address offset))(let ((addr_lo SyntheticAmode (to_amode flags address offset))(addr_hi SyntheticAmode (amode_offset addr_lo flags 8))(value_lo Reg (x64_mov addr_lo))@@ -3118,36 +3066,36 @@;; We also include widening vector loads; these sign- or zero-extend each lane;; to the next wider width (e.g., 16x4 -> 32x4).-(rule 1 (lower (has_type $I16X8 (sload8x8 _ (little_or_native_endian flags) address offset)))+(rule 1 (lower (sload8x8 $I16X8 (little_or_native_endian flags) address offset))(if-let true (has_sse41))(x64_pmovsxbw (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))-(rule 1 (lower (has_type $I16X8 (uload8x8 _ (little_or_native_endian flags) address offset)))+(rule 1 (lower (uload8x8 $I16X8 (little_or_native_endian flags) address offset))(if-let true (has_sse41))(x64_pmovzxbw (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))-(rule 1 (lower (has_type $I32X4 (sload16x4 _ (little_or_native_endian flags) address offset)))+(rule 1 (lower (sload16x4 $I32X4 (little_or_native_endian flags) address offset))(if-let true (has_sse41))(x64_pmovsxwd (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))-(rule 1 (lower (has_type $I32X4 (uload16x4 _ (little_or_native_endian flags) address offset)))+(rule 1 (lower (uload16x4 $I32X4 (little_or_native_endian flags) address offset))(if-let true (has_sse41))(x64_pmovzxwd (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))-(rule 1 (lower (has_type $I64X2 (sload32x2 _ (little_or_native_endian flags) address offset)))+(rule 1 (lower (sload32x2 $I64X2 (little_or_native_endian flags) address offset))(if-let true (has_sse41))(x64_pmovsxdq (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))-(rule 1 (lower (has_type $I64X2 (uload32x2 _ (little_or_native_endian flags) address offset)))+(rule 1 (lower (uload32x2 $I64X2 (little_or_native_endian flags) address offset))(if-let true (has_sse41))(x64_pmovzxdq (synthetic_amode_to_xmm_mem_64 (to_amode flags address offset))))-(rule (lower (has_type $I16X8 (sload8x8 _ (little_or_native_endian flags) address offset)))+(rule (lower (sload8x8 $I16X8 (little_or_native_endian flags) address offset))(lower_swiden_low $I16X8 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I16X8 (uload8x8 _ (little_or_native_endian flags) address offset)))+(rule (lower (uload8x8 $I16X8 (little_or_native_endian flags) address offset))(lower_uwiden_low $I16X8 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I32X4 (sload16x4 _ (little_or_native_endian flags) address offset)))+(rule (lower (sload16x4 $I32X4 (little_or_native_endian flags) address offset))(lower_swiden_low $I32X4 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I32X4 (uload16x4 _ (little_or_native_endian flags) address offset)))+(rule (lower (uload16x4 $I32X4 (little_or_native_endian flags) address offset))(lower_uwiden_low $I32X4 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I64X2 (sload32x2 _ (little_or_native_endian flags) address offset)))+(rule (lower (sload32x2 $I64X2 (little_or_native_endian flags) address offset))(lower_swiden_low $I64X2 (x64_movq_to_xmm (to_amode flags address offset))))-(rule (lower (has_type $I64X2 (uload32x2 _ (little_or_native_endian flags) address offset)))+(rule (lower (uload32x2 $I64X2 (little_or_native_endian flags) address offset))(lower_uwiden_low $I64X2 (x64_movq_to_xmm (to_amode flags address offset))));; Rules for `store*` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3254,40 +3202,40 @@;; f32 or f64 despite the source perhaps being an integer vector since the;; result of the instruction is the same.(rule 2 (lower (store (little_or_native_endian flags)- (has_type $F32 (extractlane _ value (u8_from_uimm8 0)))+ (extractlane $F32 value (u8_from_uimm8 0))addressoffset))(side_effect(x64_movss_store (to_amode flags address offset) value)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $F64 (extractlane _ value (u8_from_uimm8 0)))+ (extractlane $F64 value (u8_from_uimm8 0))addressoffset))(side_effect(x64_movsd_store (to_amode flags address offset) value)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $I8 (extractlane _ value (u8_from_uimm8 n)))+ (extractlane $I8 value (u8_from_uimm8 n))addressoffset))(if-let true (has_sse41))(side_effect(x64_pextrb_store (to_amode flags address offset) value n)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $I16 (extractlane _ value (u8_from_uimm8 n)))+ (extractlane $I16 value (u8_from_uimm8 n))addressoffset))(if-let true (has_sse41))(side_effect(x64_pextrw_store (to_amode flags address offset) value n)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $I32 (extractlane _ value (u8_from_uimm8 n)))+ (extractlane $I32 value (u8_from_uimm8 n))addressoffset))(if-let true (has_sse41))(side_effect(x64_pextrd_store (to_amode flags address offset) value n)))(rule 2 (lower (store (little_or_native_endian flags)- (has_type $I64 (extractlane _ value (u8_from_uimm8 n)))+ (extractlane $I64 value (u8_from_uimm8 n))addressoffset))(if-let true (has_sse41))@@ -3299,11 +3247,10 @@;; `add mem, {reg,imm}`(rule store_x64_add_mem 3 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (iadd _ (and+ (iadd (ty_32_or_64 ty) (and(sinkable_load sink)(load _ (mem_flags_data flags) addr offset))- src2))+ src2)addroffset))(let ((_ RegMemImm sink))@@ -3313,11 +3260,10 @@;; `add mem, {reg,imm}` with args swapped(rule 2 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (iadd _ src2+ (iadd (ty_32_or_64 ty) src2(and(sinkable_load sink)- (load _ (mem_flags_data flags) addr offset))))+ (load _ (mem_flags_data flags) addr offset)))addroffset))(let ((_ RegMemImm sink))@@ -3327,11 +3273,10 @@;; `sub mem, {reg,imm}`(rule 2 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (isub _ (and+ (isub (ty_32_or_64 ty) (and(sinkable_load sink)(load _ (mem_flags_data flags) addr offset))- src2))+ src2)addroffset))(let ((_ RegMemImm sink))@@ -3341,11 +3286,10 @@;; `and mem, {reg,imm}`(rule 3 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (band _ (and+ (band (ty_32_or_64 ty) (and(sinkable_load sink)(load _ (mem_flags_data flags) addr offset))- src2))+ src2)addroffset))(let ((_ RegMemImm sink))@@ -3355,11 +3299,10 @@;; `and mem, {reg,imm}` with args swapped(rule 2 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (band _ src2+ (band (ty_32_or_64 ty) src2(and(sinkable_load sink)- (load _ (mem_flags_data flags) addr offset))))+ (load _ (mem_flags_data flags) addr offset)))addroffset))(let ((_ RegMemImm sink))@@ -3369,11 +3312,10 @@;; `or mem, {reg,imm}`(rule 3 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (bor _ (and+ (bor (ty_32_or_64 ty) (and(sinkable_load sink)(load _ (mem_flags_data flags) addr offset))- src2))+ src2)addroffset))(let ((_ RegMemImm sink))@@ -3383,11 +3325,10 @@;; `or mem, {reg,imm}` with args swapped(rule 2 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (bor _ src2+ (bor (ty_32_or_64 ty) src2(and(sinkable_load sink)- (load _ (mem_flags_data flags) addr offset))))+ (load _ (mem_flags_data flags) addr offset)))addroffset))(let ((_ RegMemImm sink))@@ -3397,11 +3338,10 @@;; Xor mem, reg(rule 3 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (bxor _ (and+ (bxor (ty_32_or_64 ty) (and(sinkable_load sink)(load _ (mem_flags_data flags) addr offset))- src2))+ src2)addroffset))(let ((_ RegMemImm sink))@@ -3411,11 +3351,10 @@;; Xor mem, reg with args swapped(rule 2 (lower(store (little_or_native_endian flags)- (has_type (ty_32_or_64 ty)- (bxor _ src2+ (bxor (ty_32_or_64 ty) src2(and(sinkable_load sink)- (load _ (mem_flags_data flags) addr offset))))+ (load _ (mem_flags_data flags) addr offset)))addroffset))(let ((_ RegMemImm sink))@@ -3445,12 +3384,12 @@;;;; This lowering is only valid for I8, I16, I32, and I64. The sub-64-bit types;; are zero extended, as with a normal load.-(rule 1 (lower (has_type $I64 (atomic_load _ (little_or_native_endian flags) address)))+(rule 1 (lower (atomic_load $I64 (little_or_native_endian flags) address))(x64_mov (to_amode flags address (zero_offset))))-(rule (lower (has_type (and (fits_in_32 ty) (ty_int _)) (atomic_load _ (little_or_native_endian flags) address)))+(rule (lower (atomic_load (and (fits_in_32 ty) (ty_int _)) (little_or_native_endian flags) address))(x64_movzx (ext_mode (ty_bits_u16 ty) 64) (to_amode flags address (zero_offset))));; Lower 128-bit `atomic_load` using `cmpxchg16b`.-(rule 1 (lower (has_type $I128 (atomic_load _ (little_or_native_endian flags) address)))+(rule 1 (lower (atomic_load $I128 (little_or_native_endian flags) address))(if-let true (has_cmpxchg16b))(x64_cmpxchg16b (value_regs (imm $I64 0) (imm $I64 0)) (value_regs (imm $I64 0) (imm $I64 0)) (to_amode flags address (zero_offset))))@@ -3471,10 +3410,9 @@;; Rules for `atomic_cas` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_cas _ (little_or_native_endian flags) address expected replacement)))+(rule (lower (atomic_cas (and (fits_in_64 ty) (ty_int _)) (little_or_native_endian flags) address expected replacement))(x64_cmpxchg ty expected replacement (to_amode flags address (zero_offset))))-(rule 1 (lower (has_type $I128 (atomic_cas _ (mem_flags_data flags) address expected replacement)))+(rule 1 (lower (atomic_cas $I128 (mem_flags_data flags) address expected replacement))(if-let true (has_cmpxchg16b))(x64_cmpxchg16b expected replacement (to_amode flags address (zero_offset))))@@ -3482,52 +3420,43 @@;; This is a simple, general-case atomic update, based on a loop involving;; `cmpxchg`.-(rule (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_rmw _ (little_or_native_endian flags) op address input)))+(rule (lower (atomic_rmw (and (fits_in_64 ty) (ty_int _)) (little_or_native_endian flags) op address input))(x64_atomic_rmw_seq ty (atomic_rmw_seq_op op) (to_amode flags address (zero_offset)) input));; `Add` and `Sub` can use `lock xadd`-(rule 1 (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Add) address input)))+(rule 1 (lower (atomic_rmw (and (fits_in_64 ty) (ty_int _)) (little_or_native_endian flags) (AtomicRmwOp.Add) address input))(x64_xadd ty (to_amode flags address (zero_offset)) input))-(rule 1 (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) address input)))+(rule 1 (lower (atomic_rmw (and (fits_in_64 ty) (ty_int _)) (little_or_native_endian flags) (AtomicRmwOp.Sub) address input))(x64_xadd ty (to_amode flags address (zero_offset)) (x64_neg ty input)));; `Xchg` can use `xchg`-(rule 1 (lower (has_type (and (fits_in_64 ty) (ty_int _))- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xchg) address input)))+(rule 1 (lower (atomic_rmw (and (fits_in_64 ty) (ty_int _)) (little_or_native_endian flags) (AtomicRmwOp.Xchg) address input))(x64_xchg ty (to_amode flags address (zero_offset)) input));; `Add`, `Sub`, `And`, `Or` and `Xor` can use `lock`-prefixed instructions if;; the old value is not required.-(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Add) address input)))+(rule 2 (lower i @ (atomic_rmw (fits_in_64 (ty_int ty)) (little_or_native_endian flags) (AtomicRmwOp.Add) address input))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_add (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)))-(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Sub) address input)))+(rule 2 (lower i @ (atomic_rmw (fits_in_64 (ty_int ty)) (little_or_native_endian flags) (AtomicRmwOp.Sub) address input))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_sub (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)))-(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.And) address input)))+(rule 2 (lower i @ (atomic_rmw (fits_in_64 (ty_int ty)) (little_or_native_endian flags) (AtomicRmwOp.And) address input))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_and (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)))-(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Or) address input)))+(rule 2 (lower i @ (atomic_rmw (fits_in_64 (ty_int ty)) (little_or_native_endian flags) (AtomicRmwOp.Or) address input))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_or (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)))-(rule 2 (lower i @ (has_type (fits_in_64 (ty_int ty))- (atomic_rmw _ (little_or_native_endian flags) (AtomicRmwOp.Xor) address input)))+(rule 2 (lower i @ (atomic_rmw (fits_in_64 (ty_int ty)) (little_or_native_endian flags) (AtomicRmwOp.Xor) address input))(if-let (first_result res) i)(if-let true (value_is_unused res))(side_effect_as_invalid (x64_lock_xor (raw_operand_size_of_type ty) (to_amode flags address (zero_offset)) input)));; 128-bit integers always use a `lock cmpxchg16b` loop.-(rule 3 (lower (has_type $I128 (atomic_rmw _ (little_or_native_endian flags) op address input)))+(rule 3 (lower (atomic_rmw $I128 (little_or_native_endian flags) op address input))(if-let true (has_cmpxchg16b))(x64_atomic_128_rmw_seq op (to_amode flags address (zero_offset)) flags input))@@ -3672,7 +3601,7 @@;; Note that for GPR-based spectre guards everything is forced into a register;; not `GprMem`. The `lower_select_spectre_gpr` helper below handles "and";; conditions which the `lower_select_gpr` helper does not.-(rule 1 (lower (has_type (is_single_register_gpr_type ty) (select_spectre_guard _ cond x y)))+(rule 1 (lower (select_spectre_guard (is_single_register_gpr_type ty) cond x y))(lower_select_spectre_gpr ty (is_nonzero_cmp cond) (put_in_gpr x) y))(decl lower_select_spectre_gpr (Type CondResult Gpr Gpr) Gpr)@@ -3699,22 +3628,22 @@;; `fcvt_from_sint` to an x86-specific opcode using a zero constant which would;; be subject to normal LICM, but that's not feasible today.-(rule 2 (lower (has_type $F32 (fcvt_from_sint _ a @ (value_type $I8))))+(rule 2 (lower (fcvt_from_sint $F32 a @ (value_type $I8)))(x64_cvtsi2ss $I32 (xmm_zero $F32X4) (extend_to_gpr a $I32 (ExtendKind.Sign))))-(rule 2 (lower (has_type $F32 (fcvt_from_sint _ a @ (value_type $I16))))+(rule 2 (lower (fcvt_from_sint $F32 a @ (value_type $I16)))(x64_cvtsi2ss $I32 (xmm_zero $F32X4) (extend_to_gpr a $I32 (ExtendKind.Sign))))-(rule 1 (lower (has_type $F32 (fcvt_from_sint _ a @ (value_type (ty_int (fits_in_64 ty))))))+(rule 1 (lower (fcvt_from_sint $F32 a @ (value_type (ty_int (fits_in_64 ty)))))(x64_cvtsi2ss ty (xmm_zero $F32X4) a))-(rule 2 (lower (has_type $F64 (fcvt_from_sint _ a @ (value_type $I8))))+(rule 2 (lower (fcvt_from_sint $F64 a @ (value_type $I8)))(x64_cvtsi2sd $I32 (xmm_zero $F64X2) (extend_to_gpr a $I32 (ExtendKind.Sign))))-(rule 2 (lower (has_type $F64 (fcvt_from_sint _ a @ (value_type $I16))))+(rule 2 (lower (fcvt_from_sint $F64 a @ (value_type $I16)))(x64_cvtsi2sd $I32 (xmm_zero $F64X2) (extend_to_gpr a $I32 (ExtendKind.Sign))))-(rule 1 (lower (has_type $F64 (fcvt_from_sint _ a @ (value_type (ty_int (fits_in_64 ty))))))+(rule 1 (lower (fcvt_from_sint $F64 a @ (value_type (ty_int (fits_in_64 ty)))))(x64_cvtsi2sd ty (xmm_zero $F64X2) a))(rule 0 (lower (fcvt_from_sint _ a @ (value_type $I32X4)))@@ -3732,7 +3661,7 @@)(x64_unpcklpd f0 f1)))-(rule 1 (lower (has_type $F64X2 (fcvt_from_sint _ (swiden_low _ a @ (value_type $I32X4)))))+(rule 1 (lower (fcvt_from_sint $F64X2 (swiden_low _ a @ (value_type $I32X4))));; `cvtdq2pd` only reads the low 8 bytes of its source operand; force;; `a` into an XMM register so we don't fold a 128-bit load into an;; instruction that only wants 64 bits.@@ -3740,13 +3669,13 @@;; Rules for `fcvt_from_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule 1 (lower (has_type $F32 (fcvt_from_uint _ val @ (value_type (fits_in_32 (ty_int ty))))))+(rule 1 (lower (fcvt_from_uint $F32 val @ (value_type (fits_in_32 (ty_int ty)))))(x64_cvtsi2ss $I64 (xmm_zero $F32X4) (extend_to_gpr val $I64 (ExtendKind.Zero))))-(rule 1 (lower (has_type $F64 (fcvt_from_uint _ val @ (value_type (fits_in_32 (ty_int ty))))))+(rule 1 (lower (fcvt_from_uint $F64 val @ (value_type (fits_in_32 (ty_int ty)))))(x64_cvtsi2sd $I64 (xmm_zero $F64X2) (extend_to_gpr val $I64 (ExtendKind.Zero))))-(rule (lower (has_type ty (fcvt_from_uint _ val @ (value_type $I64))))+(rule (lower (fcvt_from_uint ty val @ (value_type $I64)))(cvt_u64_to_float_seq ty val));; Base case of u64x2 being converted to f64x2. No native instruction for this@@ -3777,7 +3706,7 @@;; number. The final step in combining via float arithmetic will chop off the;; leading `1.` at the start of the float that we constructed, one for the low;; half and one for the upper half.-(rule -1 (lower (has_type $F64X2 (fcvt_from_uint _ val @ (value_type $I64X2))))+(rule -1 (lower (fcvt_from_uint $F64X2 val @ (value_type $I64X2)))(let ((low32_mask XmmMem128 (emit_xmm_mem_128_le_const 0x00000000ffffffff_00000000ffffffff))(float_1p52 XmmMem128 (emit_xmm_mem_128_le_const 0x4330000000000000_4330000000000000))(float_1p84 XmmMem128 (emit_xmm_mem_128_le_const 0x4530000000000000_4530000000000000))@@ -3792,7 +3721,7 @@;; 0x1.0p52 + double(src). 0x1.0p52 is unique because at this exponent;; every value of the mantissa represents a corresponding uint32 number.;; When we subtract 0x1.0p52 we are left with double(src).-(rule 1 (lower (has_type $F64X2 (fcvt_from_uint _ (uwiden_low _ val @ (value_type $I32X4)))))+(rule 1 (lower (fcvt_from_uint $F64X2 (uwiden_low _ val @ (value_type $I32X4))))(let ((uint_mask XmmMem128 (emit_xmm_mem_128_le_const 0x43300000_43300000))(res Xmm (x64_unpcklps val uint_mask))(uint_mask_high XmmMem128 (emit_xmm_mem_128_le_const 0x4330000000000000_4330000000000000)))@@ -3800,7 +3729,7 @@;; When AVX512VL and AVX512F are available,;; `fcvt_from_uint` can be lowered to a single instruction.-(rule 2 (lower (has_type $F32X4 (fcvt_from_uint _ src)))+(rule 2 (lower (fcvt_from_uint $F32X4 src))(if-let true (has_avx512vl))(if-let true (has_avx512f))(x64_vcvtudq2ps src))@@ -3828,7 +3757,7 @@;; -> Convert(Ah) // Convert .. with no loss of significant digits from previous shift;; -> Ah = Ah + Ah // Double Ah to account for shift right before the conversion.;; -> dst = Ah + Al // Add the two floats together-(rule 1 (lower (has_type $F32X4 (fcvt_from_uint _ val)))+(rule 1 (lower (fcvt_from_uint $F32X4 val))(let ((a Xmm val);; get the low 16 bits@@ -3852,20 +3781,20 @@;; Rules for `fcvt_to_uint` and `fcvt_to_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type out_ty (fcvt_to_uint _ val @ (value_type (ty_scalar_float _)))))+(rule (lower (fcvt_to_uint out_ty val @ (value_type (ty_scalar_float _))))(cvt_float_to_uint_seq out_ty val false))-(rule (lower (has_type out_ty (fcvt_to_uint_sat _ val @ (value_type (ty_scalar_float _)))))+(rule (lower (fcvt_to_uint_sat out_ty val @ (value_type (ty_scalar_float _))))(cvt_float_to_uint_seq out_ty val true))-(rule (lower (has_type out_ty (fcvt_to_sint _ val @ (value_type (ty_scalar_float _)))))+(rule (lower (fcvt_to_sint out_ty val @ (value_type (ty_scalar_float _))))(cvt_float_to_sint_seq out_ty val false))-(rule (lower (has_type out_ty (fcvt_to_sint_sat _ val @ (value_type (ty_scalar_float _)))))+(rule (lower (fcvt_to_sint_sat out_ty val @ (value_type (ty_scalar_float _))))(cvt_float_to_sint_seq out_ty val true));; The x64 backend currently only supports these two type combinations.-(rule 1 (lower (has_type $I32X4 (fcvt_to_sint_sat _ val @ (value_type $F32X4))))+(rule 1 (lower (fcvt_to_sint_sat $I32X4 val @ (value_type $F32X4)))(let ((src Xmm val);; Sets tmp to zero if float is NaN@@ -3936,7 +3865,7 @@;;;; | Step 6 | Step 7 |;; | (0-(INT_MAX+1))..(UINT_MAX-(INT_MAX+1))(w/overflow) | ((INT_MAX+1)-(INT_MAX+1))..(INT_MAX+1) |-(rule 1 (lower (has_type $I32X4 (fcvt_to_uint_sat _ val @ (value_type $F32X4))))+(rule 1 (lower (fcvt_to_uint_sat $I32X4 val @ (value_type $F32X4)))(let ((src Xmm val);; Converting to unsigned int so if float src is negative or NaN@@ -3983,12 +3912,12 @@;; Rules for `x86_cvtt2dq` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I32X4 (x86_cvtt2dq _ val @ (value_type $F32X4))))+(rule (lower (x86_cvtt2dq $I32X4 val @ (value_type $F32X4)))(x64_cvttps2dq val));; Rules for `iadd_pairwise` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (iadd_pairwise _ x y)))+(rule (lower (iadd_pairwise $I8X16 x y))(let (;; Shuffle all the even lanes of `x` and `y` into one register(even_lane_mask Xmm (x64_movdqu_load (emit_xmm_mem_128_le_const 0x00ff_00ff_00ff_00ff_00ff_00ff_00ff_00ff)))@@ -4004,11 +3933,11 @@(x64_paddb evens odds)))-(rule 1 (lower (has_type $I16X8 (iadd_pairwise _ x y)))+(rule 1 (lower (iadd_pairwise $I16X8 x y))(if-let true (has_ssse3))(x64_phaddw x y))-(rule (lower (has_type $I16X8 (iadd_pairwise _ x y)))+(rule (lower (iadd_pairwise $I16X8 x y))(let ((x Xmm x)(y Xmm y)@@ -4034,11 +3963,11 @@)(x64_paddw evens odds)))-(rule 1 (lower (has_type $I32X4 (iadd_pairwise _ x y)))+(rule 1 (lower (iadd_pairwise $I32X4 x y))(if-let true (has_ssse3))(x64_phaddd x y))-(rule (lower (has_type $I32X4 (iadd_pairwise _ x y)))+(rule (lower (iadd_pairwise $I32X4 x y))(let ((x Xmm x)(y Xmm y)@@ -4051,9 +3980,9 @@;; special case for the `i16x8.extadd_pairwise_i8x16_s` wasm instruction(rule 2 (lower- (has_type $I16X8 (iadd_pairwise _+ (iadd_pairwise $I16X8(swiden_low _ val @ (value_type $I8X16))- (swiden_high _ val))))+ (swiden_high _ val)))(if-let true (has_ssse3))(let ((mul_const Xmm (x64_xmm_load_const $I8X16(emit_u128_le_const 0x01010101010101010101010101010101))))@@ -4061,26 +3990,26 @@;; special case for the `i32x4.extadd_pairwise_i16x8_s` wasm instruction(rule 2 (lower- (has_type $I32X4 (iadd_pairwise _+ (iadd_pairwise $I32X4(swiden_low _ val @ (value_type $I16X8))- (swiden_high _ val))))+ (swiden_high _ val)))(let ((mul_const XmmMem128 (emit_xmm_mem_128_le_const 0x0001_0001_0001_0001_0001_0001_0001_0001)))(x64_pmaddwd val mul_const)));; special case for the `i16x8.extadd_pairwise_i8x16_u` wasm instruction(rule 2 (lower- (has_type $I16X8 (iadd_pairwise _+ (iadd_pairwise $I16X8(uwiden_low _ val @ (value_type $I8X16))- (uwiden_high _ val))))+ (uwiden_high _ val)))(if-let true (has_ssse3))(let ((mul_const XmmMem128 (emit_xmm_mem_128_le_const 0x01010101010101010101010101010101)))(x64_pmaddubsw val mul_const)));; special case for the `i32x4.extadd_pairwise_i16x8_u` wasm instruction(rule 2 (lower- (has_type $I32X4 (iadd_pairwise _+ (iadd_pairwise $I32X4(uwiden_low _ val @ (value_type $I16X8))- (uwiden_high _ val))))+ (uwiden_high _ val)))(let ((xor_const XmmMem128 (emit_xmm_mem_128_le_const 0x8000_8000_8000_8000_8000_8000_8000_8000))(dst Xmm (x64_pxor val xor_const))@@ -4092,9 +4021,9 @@;; special case for the `i32x4.dot_i16x8_s` wasm instruction(rule 2 (lower- (has_type $I32X4 (iadd_pairwise _+ (iadd_pairwise $I32X4(imul _ (swiden_low _ x) (swiden_low _ y))- (imul _ (swiden_high _ x) (swiden_high _ y)))))+ (imul _ (swiden_high _ x) (swiden_high _ y))))(x64_pmaddwd x y));; Rules for `swiden_low` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -4104,17 +4033,17 @@;; `val` here is a 128-bit vector value; force it into an XMM register so;; we don't accidentally fold a full 16-byte load in place of the 8-byte;; one.-(rule 1 (lower (has_type $I16X8 (swiden_low _ val @ (value_type $I8X16))))+(rule 1 (lower (swiden_low $I16X8 val @ (value_type $I8X16)))(if-let true (has_sse41))(x64_pmovsxbw (put_in_xmm val)))-(rule 1 (lower (has_type $I32X4 (swiden_low _ val @ (value_type $I16X8))))+(rule 1 (lower (swiden_low $I32X4 val @ (value_type $I16X8)))(if-let true (has_sse41))(x64_pmovsxwd (put_in_xmm val)))-(rule 1 (lower (has_type $I64X2 (swiden_low _ val @ (value_type $I32X4))))+(rule 1 (lower (swiden_low $I64X2 val @ (value_type $I32X4)))(if-let true (has_sse41))(x64_pmovsxdq (put_in_xmm val)))-(rule (lower (has_type ty (swiden_low _ val))) (lower_swiden_low ty val))+(rule (lower (swiden_low ty val)) (lower_swiden_low ty val))(decl lower_swiden_low (Type Xmm) Xmm)@@ -4137,31 +4066,31 @@;; Similar to `swiden_low` with SSE4.1 except that the upper lanes are moved;; to the lower lanes first.-(rule 1 (lower (has_type $I16X8 (swiden_high _ val @ (value_type $I8X16))))+(rule 1 (lower (swiden_high $I16X8 val @ (value_type $I8X16)))(if-let true (has_sse41))(if-let true (has_ssse3))(let ((x Xmm val))(x64_pmovsxbw (x64_palignr x x 8))))-(rule 1 (lower (has_type $I32X4 (swiden_high _ val @ (value_type $I16X8))))+(rule 1 (lower (swiden_high $I32X4 val @ (value_type $I16X8)))(if-let true (has_sse41))(if-let true (has_ssse3))(let ((x Xmm val))(x64_pmovsxwd (x64_palignr x x 8))))-(rule 1 (lower (has_type $I64X2 (swiden_high _ val @ (value_type $I32X4))))+(rule 1 (lower (swiden_high $I64X2 val @ (value_type $I32X4)))(if-let true (has_sse41))(x64_pmovsxdq (x64_pshufd val 0b11_10_11_10)));; Similar to `swiden_low` versions but using `punpckh*` instructions to;; pair the high lanes next to each other.-(rule (lower (has_type $I16X8 (swiden_high _ val @ (value_type $I8X16))))+(rule (lower (swiden_high $I16X8 val @ (value_type $I8X16)))(let ((val Xmm val))(x64_psraw (x64_punpckhbw val val) (xmi_imm 8))))-(rule (lower (has_type $I32X4 (swiden_high _ val @ (value_type $I16X8))))+(rule (lower (swiden_high $I32X4 val @ (value_type $I16X8)))(let ((val Xmm val))(x64_psrad (x64_punpckhwd val val) (xmi_imm 16))));; Same as `swiden_low`, but `val` has its high lanes moved down.-(rule (lower (has_type $I64X2 (swiden_high _ val @ (value_type $I32X4))))+(rule (lower (swiden_high $I64X2 val @ (value_type $I32X4)))(let ((val Xmm (x64_pshufd val 0b00_00_11_10))(tmp Xmm (x64_pcmpgtd_a (xmm_zero $I32X4) val)))(x64_punpckldq val tmp)))@@ -4170,17 +4099,17 @@;; With SSE4.1 use the `pmovzx*` instructions for this. Same register-forcing;; rationale as the `pmovsx*` case above.-(rule 1 (lower (has_type $I16X8 (uwiden_low _ val @ (value_type $I8X16))))+(rule 1 (lower (uwiden_low $I16X8 val @ (value_type $I8X16)))(if-let true (has_sse41))(x64_pmovzxbw (put_in_xmm val)))-(rule 1 (lower (has_type $I32X4 (uwiden_low _ val @ (value_type $I16X8))))+(rule 1 (lower (uwiden_low $I32X4 val @ (value_type $I16X8)))(if-let true (has_sse41))(x64_pmovzxwd (put_in_xmm val)))-(rule 1 (lower (has_type $I64X2 (uwiden_low _ val @ (value_type $I32X4))))+(rule 1 (lower (uwiden_low $I64X2 val @ (value_type $I32X4)))(if-let true (has_sse41))(x64_pmovzxdq (put_in_xmm val)))-(rule (lower (has_type ty (uwiden_low _ val))) (lower_uwiden_low ty val))+(rule (lower (uwiden_low ty val)) (lower_uwiden_low ty val));; Interleave an all-zero register with the low lanes to produce zero-extended;; results.@@ -4195,19 +4124,19 @@;;;; Note that according to `llvm-mca` at least these instructions are faster;; than using `pmovzx*` in terms of cycles, even if SSE4.1 is available.-(rule (lower (has_type $I16X8 (uwiden_high _ val @ (value_type $I8X16))))+(rule (lower (uwiden_high $I16X8 val @ (value_type $I8X16)))(x64_punpckhbw val (xmm_zero $I8X16)))-(rule (lower (has_type $I32X4 (uwiden_high _ val @ (value_type $I16X8))))+(rule (lower (uwiden_high $I32X4 val @ (value_type $I16X8)))(x64_punpckhwd val (xmm_zero $I8X16)))-(rule (lower (has_type $I64X2 (uwiden_high _ val @ (value_type $I32X4))))+(rule (lower (uwiden_high $I64X2 val @ (value_type $I32X4)))(x64_unpckhps val (xmm_zero $F32X4)));; Rules for `snarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (snarrow _ a @ (value_type $I16X8) b)))+(rule (lower (snarrow $I8X16 a @ (value_type $I16X8) b))(x64_packsswb a b))-(rule (lower (has_type $I16X8 (snarrow _ a @ (value_type $I32X4) b)))+(rule (lower (snarrow $I16X8 a @ (value_type $I32X4) b))(x64_packssdw a b));; We're missing a `snarrow` case for $I64X2@@ -4216,8 +4145,8 @@;; This rule is a special case for handling the translation of the wasm op;; `i32x4.trunc_sat_f64x2_s_zero`. It can be removed once we have an;; implementation of `snarrow` for `I64X2`.-(rule (lower (has_type $I32X4 (snarrow _ (has_type $I64X2 (fcvt_to_sint_sat _ val))- (vconst _ (u128_from_constant 0)))))+(rule (lower (snarrow $I32X4 (fcvt_to_sint_sat $I64X2 val)+ (vconst _ (u128_from_constant 0))))(let ((a Xmm val);; y = i32x4.trunc_sat_f64x2_s_zero(x) is lowered to:@@ -4240,16 +4169,16 @@;; This rule is a special case for handling the translation of the wasm op;; `i32x4.relaxed_trunc_f64x2_s_zero`.-(rule (lower (has_type $I32X4 (snarrow _ (has_type $I64X2 (x86_cvtt2dq _ val))- (vconst _ (u128_from_constant 0)))))+(rule (lower (snarrow $I32X4 (x86_cvtt2dq $I64X2 val)+ (vconst _ (u128_from_constant 0))))(x64_cvttpd2dq val));; Rules for `unarrow` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $I8X16 (unarrow _ a @ (value_type $I16X8) b)))+(rule (lower (unarrow $I8X16 a @ (value_type $I16X8) b))(x64_packuswb a b))-(rule 1 (lower (has_type $I16X8 (unarrow _ a @ (value_type $I32X4) b)))+(rule 1 (lower (unarrow $I16X8 a @ (value_type $I32X4) b))(if-let true (has_sse41))(x64_packusdw a b))@@ -4259,7 +4188,7 @@;;;; If this is performance sensitive then it's probably best to upgrade the CPU;; to get the above single-instruction lowering.-(rule (lower (has_type $I16X8 (unarrow _ a @ (value_type $I32X4) b)))+(rule (lower (unarrow $I16X8 a @ (value_type $I32X4) b))(let ((a Xmm (unarrow_i32x4_lanes_to_low_u16_lanes a))(b Xmm (unarrow_i32x4_lanes_to_low_u16_lanes b))@@ -4296,31 +4225,29 @@;; Rules for `bitcast` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule -3 (lower (has_type (is_gpr_type (fits_in_64 ty)) (bitcast _ _ src @ (value_type (is_xmm_type _)))))+(rule -3 (lower (bitcast (is_gpr_type (fits_in_64 ty)) _ src @ (value_type (is_xmm_type _))))(bitcast_xmm_to_gpr (ty_bits ty) src))-(rule -2 (lower (has_type (is_xmm_type (fits_in_64 ty)) (bitcast _ _ src @ (value_type (is_gpr_type _)))))+(rule -2 (lower (bitcast (is_xmm_type (fits_in_64 ty)) _ src @ (value_type (is_gpr_type _))))(bitcast_gpr_to_xmm (ty_bits ty) src))-(rule -1 (lower (has_type $I128 (bitcast _ _ src @ (value_type (is_xmm_type _)))))+(rule -1 (lower (bitcast $I128 _ src @ (value_type (is_xmm_type _))))(bitcast_xmm_to_gprs src))-(rule 0 (lower (has_type (is_xmm_type _) (bitcast _ _ src @ (value_type $I128))))+(rule 0 (lower (bitcast (is_xmm_type _) _ src @ (value_type $I128)))(bitcast_gprs_to_xmm src));; Bitcast between types residing in GPR registers is a no-op.-(rule 1 (lower (has_type (is_gpr_type _)- (bitcast _ _ x @ (value_type (is_gpr_type _)))))+(rule 1 (lower (bitcast (is_gpr_type _) _ x @ (value_type (is_gpr_type _))))x);; Bitcast between types residing in XMM registers is a no-op.-(rule 3 (lower (has_type (is_xmm_type _)- (bitcast _ _ x @ (value_type (is_xmm_type _)))))+(rule 3 (lower (bitcast (is_xmm_type _) _ x @ (value_type (is_xmm_type _))))x);; Rules for `fcopysign` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type $F32 (fcopysign _ a @ (value_type $F32) b)))+(rule (lower (fcopysign $F32 a @ (value_type $F32) b))(let ((sign_bit Xmm (imm $F32 0x80000000))(a Xmm a) ;; force into reg so we don't sink a 128-bit load.(b Xmm b))@@ -4328,7 +4255,7 @@(x64_andnps sign_bit a)(x64_andps sign_bit b))))-(rule (lower (has_type $F64 (fcopysign _ a @ (value_type $F64) b)))+(rule (lower (fcopysign $F64 a @ (value_type $F64) b))(let ((sign_bit Xmm (imm $F64 0x8000000000000000))(a Xmm a) ;; force into reg so we don't sink a 128-bit load.(b Xmm b))@@ -4590,13 +4517,13 @@;; Rules for `vconst` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type ty (vconst _ const)))+(rule (lower (vconst ty const));; TODO use Inst::gen_constant() instead.(x64_xmm_load_const ty (const_to_vconst const)));; Special cases for known constant patterns to skip a 16-byte load.-(rule 1 (lower (has_type ty (vconst _ (u128_from_constant 0)))) (xmm_zero ty))-(rule 1 (lower (has_type ty (vconst _ (u128_from_constant -1)))) (vector_all_ones))+(rule 1 (lower (vconst ty (u128_from_constant 0))) (xmm_zero ty))+(rule 1 (lower (vconst ty (u128_from_constant -1))) (vector_all_ones));; Rules for `shuffle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -4784,7 +4711,7 @@;; Remove the extractlane instruction, leaving the float where it is. The upper;; bits will remain unchanged; for correctness, this relies on Cranelift type;; checking to avoid using those bits.-(rule 3 (lower (has_type (ty_scalar_float _) (extractlane _ val 0)))+(rule 3 (lower (extractlane (ty_scalar_float _) val 0))val);; `f32x4.extract_lane N` where `N != 0`@@ -4872,21 +4799,21 @@;; i8x16 splats: use `vpbroadcastb` on AVX2 and otherwise `pshufb` broadcasts;; with a mask of zero which is calculated with an xor-against-itself register.-(rule 0 (lower (has_type $I8X16 (splat _ src)))+(rule 0 (lower (splat $I8X16 src))(let ((src Gpr src)(src Xmm (x64_movd_to_xmm src)))(x64_pshufd (x64_pshuflw (x64_punpcklbw src src) 0) 0)))-(rule 1 (lower (has_type $I8X16 (splat _ src)))+(rule 1 (lower (splat $I8X16 src))(if-let true (has_ssse3))(x64_pshufb (bitcast_gpr_to_xmm 32 src) (xmm_zero $I8X16)))-(rule 2 (lower (has_type $I8X16 (splat _ src)))+(rule 2 (lower (splat $I8X16 src))(if-let true (use_avx2))(x64_vpbroadcastb (bitcast_gpr_to_xmm 32 src)))-(rule 3 (lower (has_type $I8X16 (splat _ (sinkable_load_exact addr))))+(rule 3 (lower (splat $I8X16 (sinkable_load_exact addr)))(if-let true (has_sse41))(if-let true (has_ssse3))(x64_pshufb (x64_pinsrb (xmm_uninit_value) addr 0) (xmm_zero $I8X16)))-(rule 4 (lower (has_type $I8X16 (splat _ (sinkable_load_8 addr))))+(rule 4 (lower (splat $I8X16 (sinkable_load_8 addr)))(if-let true (use_avx2))(x64_vpbroadcastb addr))@@ -4895,14 +4822,14 @@;; to the low four lanes, and `pshufd` broadcasts the low 32-bit lane (which;; at that point is two of the 16-bit values we want to broadcast) to all the;; lanes.-(rule 0 (lower (has_type $I16X8 (splat _ src)))+(rule 0 (lower (splat $I16X8 src))(x64_pshufd (x64_pshuflw (bitcast_gpr_to_xmm 32 src) 0) 0))-(rule 1 (lower (has_type $I16X8 (splat _ src)))+(rule 1 (lower (splat $I16X8 src))(if-let true (use_avx2))(x64_vpbroadcastw (bitcast_gpr_to_xmm 32 src)))-(rule 2 (lower (has_type $I16X8 (splat _ (sinkable_load_exact addr))))+(rule 2 (lower (splat $I16X8 (sinkable_load_exact addr)))(x64_pshufd (x64_pshuflw (x64_pinsrw (xmm_uninit_value) addr 0) 0) 0))-(rule 3 (lower (has_type $I16X8 (splat _ (sinkable_load_16 addr))))+(rule 3 (lower (splat $I16X8 (sinkable_load_16 addr)))(if-let true (use_avx2))(x64_vpbroadcastw addr))@@ -4910,19 +4837,19 @@;; used to broadcast the low lane to all other lanes.;;;; Note that sinkable-load cases come later-(rule 0 (lower (has_type $I32X4 (splat _ src)))+(rule 0 (lower (splat $I32X4 src))(x64_pshufd (bitcast_gpr_to_xmm 32 src) 0))-(rule 1 (lower (has_type $I32X4 (splat _ src)))+(rule 1 (lower (splat $I32X4 src))(if-let true (use_avx2))(x64_vpbroadcastd (bitcast_gpr_to_xmm 32 src)));; f32x4.splat - the source is already in an xmm register so `shufps` is all;; that's necessary to complete the splat. This is specialized to `vbroadcastss`;; on AVX2 to leverage that specific instruction for this operation.-(rule 0 (lower (has_type $F32X4 (splat _ src)))+(rule 0 (lower (splat $F32X4 src))(let ((tmp Xmm src))(x64_shufps tmp tmp 0)))-(rule 1 (lower (has_type $F32X4 (splat _ src)))+(rule 1 (lower (splat $F32X4 src))(if-let true (use_avx2))(x64_vbroadcastss src))@@ -4935,21 +4862,21 @@;; the register-based encoding is only available with AVX2. With the;; `sinkable_load` extractor this should be guaranteed to use the memory-based;; encoding hence the `has_avx` test.-(rule 5 (lower (has_type (multi_lane 32 4) (splat _ (sinkable_load addr))))+(rule 5 (lower (splat (multi_lane 32 4) (sinkable_load addr)))(let ((tmp Xmm (x64_movss_load addr)))(x64_shufps tmp tmp 0)))-(rule 6 (lower (has_type (multi_lane 32 4) (splat _ (sinkable_load_32 addr))))+(rule 6 (lower (splat (multi_lane 32 4) (sinkable_load_32 addr)))(if-let true (has_avx))(x64_vbroadcastss addr));; t64x2.splat - use `pshufd` to broadcast the lower 64-bit lane to the upper;; lane. A minor specialization for sinkable loads to avoid going through a gpr;; for i64 splats is used as well when `movddup` is available.-(rule 0 (lower (has_type $I64X2 (splat _ src)))+(rule 0 (lower (splat $I64X2 src))(x64_pshufd (bitcast_gpr_to_xmm 64 src) 0b01_00_01_00))-(rule 0 (lower (has_type $F64X2 (splat _ src)))+(rule 0 (lower (splat $F64X2 src))(x64_pshufd (put_in_xmm src) 0b01_00_01_00))-(rule 6 (lower (has_type (multi_lane 64 2) (splat _ (sinkable_load_64 addr))))+(rule 6 (lower (splat (multi_lane 64 2) (sinkable_load_64 addr)))(if-let true (has_sse3))(x64_movddup addr))@@ -5085,13 +5012,13 @@;; Rules for `tls_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (has_type (tls_model (TlsModel.ElfGd)) (tls_value _ (symbol_value_data name _ _))))+(rule (lower (tls_value (tls_model (TlsModel.ElfGd)) (symbol_value_data name _ _)))(elf_tls_get_addr name))-(rule (lower (has_type (tls_model (TlsModel.Macho)) (tls_value _ (symbol_value_data name _ _))))+(rule (lower (tls_value (tls_model (TlsModel.Macho)) (symbol_value_data name _ _)))(macho_tls_get_addr name))-(rule (lower (has_type (tls_model (TlsModel.Coff)) (tls_value _ (symbol_value_data name _ _))))+(rule (lower (tls_value (tls_model (TlsModel.Coff)) (symbol_value_data name _ _)))(coff_tls_get_addr name));; Rules for `sqmul_round_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;