Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

s390x: replace type-based isa checks with if-let (#13171)

Jimmy Brissone5cdeb4b96fa
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index 17e027fc01..c78297bce8 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -1580,23 +1580,15 @@
;; Helpers for querying enabled ISA extensions ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(decl mie3_enabled () Type)
-(extern extractor mie3_enabled mie3_enabled)
-(decl mie3_disabled () Type)
-(extern extractor mie3_disabled mie3_disabled)
-(decl mie4_enabled () Type)
-(extern extractor mie4_enabled mie4_enabled)
-(decl mie4_disabled () Type)
-(extern extractor mie4_disabled mie4_disabled)
-
-(decl vxrs_ext2_enabled () Type)
-(extern extractor vxrs_ext2_enabled vxrs_ext2_enabled)
-(decl vxrs_ext2_disabled () Type)
-(extern extractor vxrs_ext2_disabled vxrs_ext2_disabled)
-(decl vxrs_ext3_enabled () Type)
-(extern extractor vxrs_ext3_enabled vxrs_ext3_enabled)
-(decl vxrs_ext3_disabled () Type)
-(extern extractor vxrs_ext3_disabled vxrs_ext3_disabled)
+(decl pure has_mie3 () bool)
+(extern constructor has_mie3 has_mie3)
+(decl pure has_mie4 () bool)
+(extern constructor has_mie4 has_mie4)
+
+(decl pure has_vxrs_ext2 () bool)
+(extern constructor has_vxrs_ext2 has_vxrs_ext2)
+(decl pure has_vxrs_ext3 () bool)
+(extern constructor has_vxrs_ext3 has_vxrs_ext3)
;; Helpers for SIMD lane number operations ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1945,27 +1937,37 @@
(memarg_reg_plus_reg_plus_off x y z flags))
(rule 1 (lower_address flags
- (symbol_value _ (symbol_value_data name (RelocDistance.Near) sym_offset))
- (i64_from_offset offset))
+ (symbol_value _ (symbol_value_data name (RelocDistance.Near) sym_offset))
+ (i64_from_offset offset))
(if-let final_offset (memarg_symbol_offset_sum offset sym_offset))
(memarg_symbol name final_offset flags))
-(rule 2 (lower_address flags (has_type (mie4_enabled)
+(rule 2 (lower_address flags
(iadd $I64 (ishl $I64 (uextend $I64 (iadd $I32 x (simm20_from_value z)))
- (u8_from_value shift)) y)) (i64_from_offset offset))
+ (u8_from_value shift)) y)
+ (i64_from_offset offset))
+ (if-let true (has_mie4))
(memarg_reg_plus_off (load_logical_indexed_addr x y z shift) offset 0 flags))
-(rule 3 (lower_address flags (has_type (mie4_enabled)
+(rule 3 (lower_address flags
(iadd $I64 y (ishl $I64 (uextend $I64 (iadd $I32 x (simm20_from_value z)))
- (u8_from_value shift)))) (i64_from_offset offset))
+ (u8_from_value shift)))
+ (i64_from_offset offset))
+ (if-let true (has_mie4))
(memarg_reg_plus_off (load_logical_indexed_addr y x z shift) offset 0 flags))
-(rule 4 (lower_address flags (has_type (mie4_enabled)
- (iadd $I64 (ishl $I64 (sextend $I64 (iadd $I32 x (simm20_from_value z))) (u8_from_value shift)) y)) (i64_from_offset offset))
+(rule 4 (lower_address flags
+ (iadd $I64 (ishl $I64 (sextend $I64 (iadd $I32 x (simm20_from_value z)))
+ (u8_from_value shift)) y)
+ (i64_from_offset offset))
+ (if-let true (has_mie4))
(memarg_reg_plus_off (load_indexed_addr x y z shift) offset 0 flags))
-(rule 5 (lower_address flags (has_type (mie4_enabled)
- (iadd $I64 y (ishl $I64 (sextend $I64 (iadd $I32 x (simm20_from_value z))) (u8_from_value shift)))) (i64_from_offset offset))
+(rule 5 (lower_address flags
+ (iadd $I64 y (ishl $I64 (sextend $I64 (iadd $I32 x (simm20_from_value z)))
+ (u8_from_value shift)))
+ (i64_from_offset offset))
+ (if-let true (has_mie4))
(memarg_reg_plus_off (load_indexed_addr y x z shift) offset 0 flags))
;; Lower an address plus a small bias into a `MemArg`.
@@ -4970,13 +4972,17 @@
;; Helpers for generating `fcvt_to_[us]int` instructions ;;;;;;;;;;;;;;;;;;;;;;;
(decl fcvt_flt_ty (Type Type) Type)
-(rule 1 (fcvt_flt_ty (fits_in_32 ty) (and (vxrs_ext2_enabled) $F32)) $F32)
+(rule 1 (fcvt_flt_ty (fits_in_32 ty) $F32)
+ (if-let true (has_vxrs_ext2))
+ $F32)
(rule (fcvt_flt_ty (fits_in_64 ty) $F32) $F64)
(rule (fcvt_flt_ty (fits_in_64 ty) $F64) $F64)
(rule (fcvt_flt_ty (fits_in_64 ty) $F128) $F128)
(decl fcvt_int_ty (Type Type) Type)
-(rule 1 (fcvt_int_ty (fits_in_32 ty) (and (vxrs_ext2_enabled) $F32)) $I32)
+(rule 1 (fcvt_int_ty (fits_in_32 ty) $F32)
+ (if-let true (has_vxrs_ext2))
+ $I32)
(rule (fcvt_int_ty (fits_in_64 ty) $F32) $I64)
(rule (fcvt_int_ty (fits_in_64 ty) $F64) $I64)
(rule 1 (fcvt_int_ty (fits_in_32 ty) $F128) $I32)
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index 84d9af11d9..dc634d0ece 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -115,24 +115,28 @@
(rule 1 (lower (iadd (vr128_ty ty) x y))
(vec_add ty x y))
-(rule 16 (lower (iadd (and (mie4_enabled) $I64)
+(rule 16 (lower (iadd $I64
(ishl $I64 (uextend $I64 (iadd $I32 x (simm20_from_value z))) (u8_from_value shift))
y))
+ (if-let true (has_mie4))
(load_logical_indexed_addr x y z shift))
-(rule 17 (lower (iadd (and (mie4_enabled) $I64)
+(rule 17 (lower (iadd $I64
y
(ishl $I64 (uextend $I64 (iadd $I32 x (simm20_from_value z))) (u8_from_value shift))))
+ (if-let true (has_mie4))
(load_logical_indexed_addr y x z shift))
-(rule 18 (lower (iadd (and (mie4_enabled) $I64)
+(rule 18 (lower (iadd $I64
(ishl $I64 (sextend $I64 (iadd $I32 x (simm20_from_value z))) (u8_from_value shift))
y))
+ (if-let true (has_mie4))
(load_indexed_addr x y z shift))
-(rule 19 (lower (iadd (and (mie4_enabled) $I64)
+(rule 19 (lower (iadd $I64
y
(ishl $I64 (sextend $I64 (iadd $I32 x (simm20_from_value z))) (u8_from_value shift))))
+ (if-let true (has_mie4))
(load_indexed_addr y x z shift))
;;;; Rules for `uadd_sat` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -165,7 +169,7 @@
;; special case for the `i32x4.dot_i16x8_s` wasm instruction
(rule 1 (lower
- (iadd_pairwise dst_ty
+ (iadd_pairwise dst_ty
(imul _ (swiden_low _ x @ (value_type src_ty)) (swiden_low _ y))
(imul _ (swiden_high _ x) (swiden_high _ y))))
(vec_add dst_ty (vec_smul_even src_ty x y)
@@ -241,11 +245,13 @@
(vec_abs ty x))
;; Absolute value of a 128-bit integer on z17.
-(rule 4 (lower (iabs (and (vxrs_ext3_enabled) $I128) x))
+(rule 4 (lower (iabs $I128 x))
+ (if-let true (has_vxrs_ext3))
(vec_abs $I128 x))
;; Absolute value of a 128-bit integer pre-z17.
-(rule 0 (lower (iabs (and (vxrs_ext3_disabled) $I128) x))
+(rule 0 (lower (iabs $I128 x))
+ (if-let false (has_vxrs_ext3))
(let ((zero Reg (vec_imm $I128 0))
(pos Reg x)
(neg Reg (vec_sub $I128 zero pos))
@@ -269,11 +275,13 @@
(vec_neg ty x))
;; Negate a 128-bit integer on z17.
-(rule 4 (lower (ineg (and (vxrs_ext3_enabled) $I128) x))
+(rule 4 (lower (ineg $I128 x))
+ (if-let true (has_vxrs_ext3))
(vec_neg $I128 x))
;; Negate a 128-bit integer pre-z17.
-(rule 0 (lower (ineg (and (vxrs_ext3_disabled) $I128) x))
+(rule 0 (lower (ineg $I128 x))
+ (if-let false (has_vxrs_ext3))
(vec_sub $I128 (vec_imm $I128 0) x))
@@ -288,14 +296,16 @@
(select_bool_reg ty cond y_ext x_ext)))
;; Unsigned maximum of two 128-bit integers pre-z17 - expand to icmp + select.
-(rule 1 (lower (umax (and (vxrs_ext3_disabled) $I128) x y))
+(rule 1 (lower (umax $I128 x y))
+ (if-let false (has_vxrs_ext3))
(let ((x_reg Reg (put_in_reg x))
(y_reg Reg (put_in_reg y))
(cond ProducesBool (vec_int128_ucmphi y_reg x_reg)))
(select_bool_reg $I128 cond y_reg x_reg)))
;; Unsigned maximum of two 128-bit integers on z17.
-(rule 3 (lower (umax (and (vxrs_ext3_enabled) $I128) x y))
+(rule 3 (lower (umax $I128 x y))
+ (if-let true (has_vxrs_ext3))
(vec_umax $I128 x y))
;; Unsigned maximum of two vector registers.
@@ -314,14 +324,16 @@
(select_bool_reg ty cond y_ext x_ext)))
;; Unsigned maximum of two 128-bit integers pre-z17 - expand to icmp + select.
-(rule 1 (lower (umin (and (vxrs_ext3_disabled) $I128) x y))
+(rule 1 (lower (umin $I128 x y))
+ (if-let false (has_vxrs_ext3))
(let ((x_reg Reg (put_in_reg x))
(y_reg Reg (put_in_reg y))
(cond ProducesBool (vec_int128_ucmphi x_reg y_reg)))
(select_bool_reg $I128 cond y_reg x_reg)))
;; Unsigned minimum of two 128-bit integers on z17.
-(rule 3 (lower (umin (and (vxrs_ext3_enabled) $I128) x y))
+(rule 3 (lower (umin $I128 x y))
+ (if-let true (has_vxrs_ext3))
(vec_umin $I128 x y))
;; Unsigned minimum of two vector registers.
@@ -340,14 +352,16 @@
(select_bool_reg ty cond y_ext x_ext)))
;; Signed maximum of two 128-bit integers pre-z17 - expand to icmp + select.
-(rule 1 (lower (smax (and (vxrs_ext3_disabled) $I128) x y))
+(rule 1 (lower (smax $I128 x y))
+ (if-let false (has_vxrs_ext3))
(let ((x_reg Reg (put_in_reg x))
(y_reg Reg (put_in_reg y))
(cond ProducesBool (vec_int128_scmphi y_reg x_reg)))
(select_bool_reg $I128 cond y_reg x_reg)))
;; Signed maximum of two 128-bit integers on z17.
-(rule 3 (lower (smax (and (vxrs_ext3_enabled) $I128) x y))
+(rule 3 (lower (smax $I128 x y))
+ (if-let true (has_vxrs_ext3))
(vec_smax $I128 x y))
;; Signed maximum of two vector registers.
@@ -366,14 +380,16 @@
(select_bool_reg ty cond y_ext x_ext)))
;; Signed maximum of two 128-bit integers pre-z17 - expand to icmp + select.
-(rule 1 (lower (smin (and (vxrs_ext3_disabled) $I128) x y))
+(rule 1 (lower (smin $I128 x y))
+ (if-let false (has_vxrs_ext3))
(let ((x_reg Reg (put_in_reg x))
(y_reg Reg (put_in_reg y))
(cond ProducesBool (vec_int128_scmphi x_reg y_reg)))
(select_bool_reg $I128 cond y_reg x_reg)))
;; Signed minimum of two 128-bit integers on z17.
-(rule 3 (lower (smin (and (vxrs_ext3_enabled) $I128) x y))
+(rule 3 (lower (smin $I128 x y))
+ (if-let true (has_vxrs_ext3))
(vec_smin $I128 x y))
;; Signed minimum of two vector registers.
@@ -443,10 +459,13 @@
(rule (vec_mul_impl $I32X4 x y) (vec_mul $I32X4 x y))
;; Multiply two vector registers - doubleword on z17.
-(rule 1 (vec_mul_impl (and (vxrs_ext3_enabled) $I64X2) x y) (vec_mul $I64X2 x y))
+(rule 1 (vec_mul_impl $I64X2 x y)
+ (if-let true (has_vxrs_ext3))
+ (vec_mul $I64X2 x y))
;; Multiply two vector registers - doubleword pre-z17. Has to be scalarized.
-(rule (vec_mul_impl (and (vxrs_ext3_disabled) $I64X2) x y)
+(rule (vec_mul_impl $I64X2 x y)
+ (if-let false (has_vxrs_ext3))
(mov_to_vec128 $I64X2
(mul_reg $I64 (vec_extract_lane $I64X2 x 0 (zero_reg))
(vec_extract_lane $I64X2 y 0 (zero_reg)))
@@ -454,10 +473,13 @@
(vec_extract_lane $I64X2 y 1 (zero_reg)))))
;; Multiply two vector registers - quadword on z17.
-(rule 1 (vec_mul_impl (and (vxrs_ext3_enabled) $I128) x y) (vec_mul $I128 x y))
+(rule 1 (vec_mul_impl $I128 x y)
+ (if-let true (has_vxrs_ext3))
+ (vec_mul $I128 x y))
;; Multiply two vector registers - quadword pre-z17.
-(rule (vec_mul_impl (and (vxrs_ext3_disabled) $I128) x y)
+(rule (vec_mul_impl $I128 x y)
+ (if-let false (has_vxrs_ext3))
(let ((x_hi Reg (vec_extract_lane $I64X2 x 0 (zero_reg)))
(x_lo Reg (vec_extract_lane $I64X2 x 1 (zero_reg)))
(y_hi Reg (vec_extract_lane $I64X2 y 0 (zero_reg)))
@@ -507,11 +529,14 @@
(rule (lower (umulhi $I32X4 x y)) (vec_umulhi $I32X4 x y))
;; Multiply high part unsigned, vector types with 64-bit elements on z17.
-(rule 1 (lower (umulhi (and (vxrs_ext3_enabled) $I64X2) x y)) (vec_umulhi $I64X2 x y))
+(rule 1 (lower (umulhi $I64X2 x y))
+ (if-let true (has_vxrs_ext3))
+ (vec_umulhi $I64X2 x y))
;; Multiply high part unsigned, vector types with 64-bit elements pre-z17.
;; Has to be scalarized.
-(rule (lower (umulhi (and (vxrs_ext3_disabled) $I64X2) x y))
+(rule (lower (umulhi $I64X2 x y))
+ (if-let false (has_vxrs_ext3))
(let ((pair_0 RegPair (umul_wide (vec_extract_lane $I64X2 x 0 (zero_reg))
(vec_extract_lane $I64X2 y 0 (zero_reg))))
(res_0 Reg (regpair_hi pair_0))
@@ -548,11 +573,14 @@
(rule (lower (smulhi $I32X4 x y)) (vec_smulhi $I32X4 x y))
;; Multiply high part signed, vector types with 64-bit elements on z17.
-(rule 1 (lower (smulhi (and (vxrs_ext3_enabled) $I64X2) x y)) (vec_smulhi $I64X2 x y))
+(rule 1 (lower (smulhi $I64X2 x y))
+ (if-let true (has_vxrs_ext3))
+ (vec_smulhi $I64X2 x y))
;; Multiply high part signed, vector types with 64-bit elements pre-z17.
;; Has to be scalarized.
-(rule (lower (smulhi (and (vxrs_ext3_disabled) $I64X2) x y))
+(rule (lower (smulhi $I64X2 x y))
+ (if-let false (has_vxrs_ext3))
(let ((pair_0 RegPair (smul_wide (vec_extract_lane $I64X2 x 0 (zero_reg))
(vec_extract_lane $I64X2 y 0 (zero_reg))))
(res_0 Reg (copy_reg $I64 (regpair_hi pair_0)))
@@ -631,12 +659,14 @@
(regpair_hi pair)))
;; Implement `udiv` for 128-bit integers on z17 (only).
-(rule 1 (lower (udiv (and (vxrs_ext3_enabled) $I128) x y))
- (vec_udiv $I128 x y))
+(rule 1 (lower (udiv $I128 x y))
+ (if-let true (has_vxrs_ext3))
+ (vec_udiv $I128 x y))
;; Implement `urem` for 128-bit integers on z17 (only).
-(rule 1 (lower (urem (and (vxrs_ext3_enabled) $I128) x y))
- (vec_urem $I128 x y))
+(rule 1 (lower (urem $I128 x y))
+ (if-let true (has_vxrs_ext3))
+ (vec_urem $I128 x y))
;;;; Rules for `sdiv` and `srem` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -775,15 +805,17 @@
;; Implement `sdiv` for 128-bit integers on z17 (only).
-(rule 1 (lower (sdiv (and (vxrs_ext3_enabled) $I128) x y))
- (let ((OFcheck bool (div_overflow_check_needed y))
- (_ Reg (maybe_trap_if_sdiv_overflow OFcheck $I128 $I128 x y)))
+(rule 1 (lower (sdiv $I128 x y))
+ (if-let true (has_vxrs_ext3))
+ (let ((OFcheck bool (div_overflow_check_needed y))
+ (_ Reg (maybe_trap_if_sdiv_overflow OFcheck $I128 $I128 x y)))
(vec_sdiv $I128 x y)))
;; Implement `srem` for 128-bit integers on z17 (only).
-(rule 1 (lower (srem (and (vxrs_ext3_enabled) $I128) x y))
- (let ((OFcheck bool (div_overflow_check_needed y))
- (top Reg (maybe_avoid_srem_overflow OFcheck $I128 x y)))
+(rule 1 (lower (srem $I128 x y))
+ (if-let true (has_vxrs_ext3))
+ (let ((OFcheck bool (div_overflow_check_needed y))
+ (top Reg (maybe_avoid_srem_overflow OFcheck $I128 x y)))
(vec_srem $I128 top y)))
@@ -1084,12 +1116,14 @@
;;;; Rules for `bnot` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; z15 version using a single instruction (NOR).
-(rule 2 (lower (bnot (and (mie3_enabled) (fits_in_64 ty)) x))
+(rule 2 (lower (bnot (fits_in_64 ty) x))
+ (if-let true (has_mie3))
(let ((rx Reg x))
(not_or_reg ty rx rx)))
;; z14 version using XOR with -1.
-(rule 1 (lower (bnot (and (mie3_disabled) (fits_in_64 ty)) x))
+(rule 1 (lower (bnot (fits_in_64 ty) x))
+ (if-let false (has_mie3))
(not_reg ty x))
;; Vector version using vector NOR.
@@ -1102,7 +1136,8 @@
;; With z15 (bnot (bxor ...)) can be a single instruction, similar to the
;; (bxor _ (bnot _)) lowering.
-(rule 3 (lower (bnot (and (mie3_enabled) (fits_in_64 ty)) (bxor _ x y)))
+(rule 3 (lower (bnot (fits_in_64 ty) (bxor _ x y)))
+ (if-let true (has_mie3))
(not_xor_reg ty x y))
;; Combine a not/xor operation of vector types into one.
@@ -1140,9 +1175,11 @@
(rule 11 (lower (band (ty_scalar_float _) x y))
(vec_and $F64X2 x y))
-(rule 12 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x y) z))
+(rule 12 (lower (band (vr128_ty ty) (band _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000001 x y z))
-(rule 13 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) x (band _ y z)))
+(rule 13 (lower (band (vr128_ty ty) x (band _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000001 x y z))
;; Specialized lowerings for `(band x (bnot y))` which is additionally produced
@@ -1150,9 +1187,11 @@
;; forms early on.
;; z15 version using a single instruction.
-(rule 7 (lower (band (and (mie3_enabled) (fits_in_64 ty)) x (bnot _ y)))
+(rule 7 (lower (band (fits_in_64 ty) x (bnot _ y)))
+ (if-let true (has_mie3))
(and_not_reg ty x y))
-(rule 8 (lower (band (and (mie3_enabled) (fits_in_64 ty)) (bnot _ y) x))
+(rule 8 (lower (band (fits_in_64 ty) (bnot _ y) x))
+ (if-let true (has_mie3))
(and_not_reg ty x y))
;; And-not two vector registers.
@@ -1162,53 +1201,71 @@
(vec_and_not ty x y))
;; And-not three vector registers.
-(rule 14 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (bnot _ x) y) z))
+(rule 14 (lower (band (vr128_ty ty) (band _ (bnot _ x) y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00010000 x y z))
-(rule 15 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ x) (band _ y z)))
+(rule 15 (lower (band (vr128_ty ty) (bnot _ x) (band _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00010000 x y z))
-(rule 16 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x y) (bnot _ z)))
+(rule 16 (lower (band (vr128_ty ty) (band _ x y) (bnot _ z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000010 x y z))
-(rule 17 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) x (band _ y (bnot _ z))))
+(rule 17 (lower (band (vr128_ty ty) x (band _ y (bnot _ z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000010 x y z))
-(rule 18 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (bnot _ y)) z))
+(rule 18 (lower (band (vr128_ty ty) (band _ x (bnot _ y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000100 x y z))
-(rule 19 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) x (band _ (bnot _ y) z)))
+(rule 19 (lower (band (vr128_ty ty) x (band _ (bnot _ y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000100 x y z))
;; Not-and three vector registers
-(rule 20 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ (band _ x y) z)))
+(rule 20 (lower (bnot (vr128_ty ty) (band _ (band _ x y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11111110 x y z))
-(rule 21 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x (band _ y z))))
+(rule 21 (lower (bnot (vr128_ty ty) (band _ x (band _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11111110 x y z))
;; And-Nand three vector registers
-(rule 20 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (band _ x y)) z))
+(rule 20 (lower (band (vr128_ty ty) (bnot _ (band _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01010100 x y z))
-(rule 21 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (band _ y z))))
+(rule 21 (lower (band (vr128_ty ty) x (bnot _ (band _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00001110 x y z))
;; And-Or 3 vector registers
-(rule 22 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bor _ y z)))
+(rule 22 (lower (band (vr128_ty ty) x (bor _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000111 x y z))
-(rule 23 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x y) z))
+(rule 23 (lower (band (vr128_ty ty) (bor _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00010101 x y z))
;; And-Nor 3 vector registers
-(rule 24 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (bor _ y z))))
+(rule 24 (lower (band (vr128_ty ty) x (bnot _ (bor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00001000 x y z))
-(rule 25 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ x y)) z))
+(rule 25 (lower (band (vr128_ty ty) (bnot _ (bor _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01000000 x y z))
;; And-Xor 3 vector registers
-(rule 26 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bxor _ y z)))
+(rule 26 (lower (band (vr128_ty ty) x (bxor _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000110 x y z))
-(rule 27 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x y) z))
+(rule 27 (lower (band (vr128_ty ty) (bxor _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00010100 x y z))
;; And-Nxor 3 vector registers
-(rule 28 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (bxor _ y z))))
+(rule 28 (lower (band (vr128_ty ty) x (bnot _ (bxor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00001001 x y z))
-(rule 29 (lower (band (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x y)) z))
+(rule 29 (lower (band (vr128_ty ty) (bnot _ (bxor _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01000001 x y z))
;;;; Rules for `bor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1242,9 +1299,11 @@
(vec_or $F64X2 x y))
;; Or three vector registers.
-(rule 12 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x y) z))
+(rule 12 (lower (bor (vr128_ty ty) (bor _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01111111 x y z))
-(rule 13 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bor _ y z)))
+(rule 13 (lower (bor (vr128_ty ty) x (bor _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01111111 x y z))
;; Specialized lowerings for `(bor x (bnot y))` which is additionally produced
@@ -1252,9 +1311,11 @@
;; forms early on.
;; z15 version using a single instruction.
-(rule 7 (lower (bor (and (mie3_enabled) (fits_in_64 ty)) x (bnot _ y)))
+(rule 7 (lower (bor (fits_in_64 ty) x (bnot _ y)))
+ (if-let true (has_mie3))
(or_not_reg ty x y))
-(rule 8 (lower (bor (and (mie3_enabled) (fits_in_64 ty)) (bnot _ y) x))
+(rule 8 (lower (bor (fits_in_64 ty) (bnot _ y) x))
+ (if-let true (has_mie3))
(or_not_reg ty x y))
;; Or-not two vector registers.
@@ -1264,77 +1325,103 @@
(vec_or_not ty x y))
;; 3-input bor with a single not
-(rule 14 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bnot _ x) y) z))
+(rule 14 (lower (bor (vr128_ty ty) (bor _ (bnot _ x) y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11110111 x y z))
-(rule 15 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ x) (bor _ y z)))
+(rule 15 (lower (bor (vr128_ty ty) (bnot _ x) (bor _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11110111 x y z))
-(rule 16 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bnot _ y)) z))
+(rule 16 (lower (bor (vr128_ty ty) (bor _ x (bnot _ y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11011111 x y z))
-(rule 17 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bor _ (bnot _ y) z)))
+(rule 17 (lower (bor (vr128_ty ty) x (bor _ (bnot _ y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11011111 x y z))
-(rule 18 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x y) (bnot _ z)))
+(rule 18 (lower (bor (vr128_ty ty) (bor _ x y) (bnot _ z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10111111 x y z))
-(rule 19 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bor _ y (bnot _ z))))
+(rule 19 (lower (bor (vr128_ty ty) x (bor _ y (bnot _ z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10111111 x y z))
;; 3-input bnor
-(rule 20 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bor _ x y) z)))
+(rule 20 (lower (bnot (vr128_ty ty) (bor _ (bor _ x y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10000000 x y z))
-(rule 21 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bor _ y z))))
+(rule 21 (lower (bnot (vr128_ty ty) (bor _ x (bor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10000000 x y z))
;; Or-Nor
-(rule 20 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ x y)) z))
+(rule 20 (lower (bor (vr128_ty ty) (bnot _ (bor _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11010101 x y z))
-(rule 21 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (bor _ y z))))
+(rule 21 (lower (bor (vr128_ty ty) x (bnot _ (bor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10001111 x y z))
;; Or-And
-(rule 22 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x y) z))
+(rule 22 (lower (bor (vr128_ty ty) (band _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01010111 x y z))
-(rule 23 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (band _ y z)))
+(rule 23 (lower (bor (vr128_ty ty) x (band _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00011111 x y z))
;; Or-Nand
-(rule 24 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (band _ x y)) z))
+(rule 24 (lower (bor (vr128_ty ty) (bnot _ (band _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11111101 x y z))
-(rule 25 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (band _ y z))))
+(rule 25 (lower (bor (vr128_ty ty) x (bnot _ (band _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11101111 x y z))
;; Or-Xor
-(rule 26 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x y) z))
+(rule 26 (lower (bor (vr128_ty ty) (bxor _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01111101 x y z))
-(rule 27 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bxor _ y z)))
+(rule 27 (lower (bor (vr128_ty ty) x (bxor _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01101111 x y z))
;; Or-Nxor
-(rule 28 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x y)) z))
+(rule 28 (lower (bor (vr128_ty ty) (bnot _ (bxor _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11010111 x y z))
-(rule 29 (lower (bor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (bxor _ y z))))
+(rule 29 (lower (bor (vr128_ty ty) x (bnot _ (bxor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10011111 x y z))
;; Nor-And
-(rule 30 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (band _ x y) z)))
+(rule 30 (lower (bnot (vr128_ty ty) (bor _ (band _ x y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10101000 x y z))
-(rule 31 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (band _ y z))))
+(rule 31 (lower (bnot (vr128_ty ty) (bor _ x (band _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11100000 x y z))
;; Nor-Nand
-(rule 32 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bnot _ (band _ x y)) z)))
+(rule 32 (lower (bnot (vr128_ty ty) (bor _ (bnot _ (band _ x y)) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00000010 x y z))
-(rule 33 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bnot _ (band _ y z)))))
+(rule 33 (lower (bnot (vr128_ty ty) (bor _ x (bnot _ (band _ y z)))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00010000 x y z))
;; Nor-Xor
-(rule 34 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bxor _ x y) z)))
+(rule 34 (lower (bnot (vr128_ty ty) (bor _ (bxor _ x y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10000010 x y z))
-(rule 35 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bxor _ y z))))
+(rule 35 (lower (bnot (vr128_ty ty) (bor _ x (bxor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10010000 x y z))
;; Nor-Nxor
-(rule 36 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ (bnot _ (bxor _ x y)) z)))
+(rule 36 (lower (bnot (vr128_ty ty) (bor _ (bnot _ (bxor _ x y)) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00101000 x y z))
-(rule 37 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x (bnot _ (bxor _ y z)))))
+(rule 37 (lower (bnot (vr128_ty ty) (bor _ x (bnot _ (bxor _ y z)))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01100000 x y z))
;;;; Rules for `bxor` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1368,9 +1455,11 @@
;; forms early on.
;; z15 version using a single instruction.
-(rule 5 (lower (bxor (and (mie3_enabled) (fits_in_64 ty)) x (bnot _ y)))
+(rule 5 (lower (bxor (fits_in_64 ty) x (bnot _ y)))
+ (if-let true (has_mie3))
(not_xor_reg ty x y))
-(rule 6 (lower (bxor (and (mie3_enabled) (fits_in_64 ty)) (bnot _ y) x))
+(rule 6 (lower (bxor (fits_in_64 ty) (bnot _ y) x))
+ (if-let true (has_mie3))
(not_xor_reg ty x y))
;; Xor-not two vector registers.
@@ -1380,88 +1469,114 @@
(vec_not_xor ty x y))
;; 3-input Xor
-(rule 10 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x y) z))
+(rule 10 (lower (bxor (vr128_ty ty) (bxor _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01101001 x y z))
-(rule 11 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bxor _ y z)))
+(rule 11 (lower (bxor (vr128_ty ty) x (bxor _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01101001 x y z))
;; Xor-And
-(rule 12 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) (band _ x y) z))
+(rule 12 (lower (bxor (vr128_ty ty) (band _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01010110 x y z))
-(rule 13 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (band _ y z)))
+(rule 13 (lower (bxor (vr128_ty ty) x (band _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00011110 x y z))
;; Xor-Nand
-(rule 14 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (band _ x y)) z))
+(rule 14 (lower (bxor (vr128_ty ty) (bnot _ (band _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10101001 x y z))
-(rule 15 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (band _ y z))))
+(rule 15 (lower (bxor (vr128_ty ty) x (bnot _ (band _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11100001 x y z))
;; Xor-Or
-(rule 16 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bor _ x y) z))
+(rule 16 (lower (bxor (vr128_ty ty) (bor _ x y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01101010 x y z))
-(rule 17 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bor _ y z)))
+(rule 17 (lower (bxor (vr128_ty ty) x (bor _ y z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01111000 x y z))
;; Xor-Nor
-(rule 18 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bor _ x y)) z))
+(rule 18 (lower (bxor (vr128_ty ty) (bnot _ (bor _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10010101 x y z))
-(rule 19 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (bor _ y z))))
+(rule 19 (lower (bxor (vr128_ty ty) x (bnot _ (bor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10000111 x y z))
;; Xor-Nxor
-(rule 18 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) (bnot _ (bxor _ x y)) z))
+(rule 18 (lower (bxor (vr128_ty ty) (bnot _ (bxor _ x y)) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10010110 x y z))
-(rule 19 (lower (bxor (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ (bxor _ y z))))
+(rule 19 (lower (bxor (vr128_ty ty) x (bnot _ (bxor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10010110 x y z))
;; 3-input Nxor
-(rule 20 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bxor _ x y) z)))
+(rule 20 (lower (bnot (vr128_ty ty) (bxor _ (bxor _ x y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10010110 x y z))
-(rule 21 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bxor _ y z))))
+(rule 21 (lower (bnot (vr128_ty ty) (bxor _ x (bxor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10010110 x y z))
;; Nxor-And
-(rule 22 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (band _ x y) z)))
+(rule 22 (lower (bnot (vr128_ty ty) (bxor _ (band _ x y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10101001 x y z))
-(rule 23 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (band _ y z))))
+(rule 23 (lower (bnot (vr128_ty ty) (bxor _ x (band _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b11100001 x y z))
;; Nxor-Nand
-(rule 24 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bnot _ (band _ x y)) z)))
+(rule 24 (lower (bnot (vr128_ty ty) (bxor _ (bnot _ (band _ x y)) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01010110 x y z))
-(rule 25 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bnot _ (band _ y z)))))
+(rule 25 (lower (bnot (vr128_ty ty) (bxor _ x (bnot _ (band _ y z)))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b00011110 x y z))
;; Nxor-Or
-(rule 26 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bor _ x y) z)))
+(rule 26 (lower (bnot (vr128_ty ty) (bxor _ (bor _ x y) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10010101 x y z))
-(rule 27 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bor _ y z))))
+(rule 27 (lower (bnot (vr128_ty ty) (bxor _ x (bor _ y z))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10000111 x y z))
;; Nxor-Nor
-(rule 28 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bnot _ (bor _ x y)) z)))
+(rule 28 (lower (bnot (vr128_ty ty) (bxor _ (bnot _ (bor _ x y)) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01101010 x y z))
-(rule 29 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bnot _ (bor _ y z)))))
+(rule 29 (lower (bnot (vr128_ty ty) (bxor _ x (bnot _ (bor _ y z)))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01111000 x y z))
;; Xor-Nxor
-(rule 18 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ (bnot _ (bxor _ x y)) z)))
+(rule 18 (lower (bnot (vr128_ty ty) (bxor _ (bnot _ (bxor _ x y)) z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01101001 x y z))
-(rule 19 (lower (bnot (and (vxrs_ext3_enabled) (vr128_ty ty)) (bxor _ x (bnot _ (bxor _ y z)))))
+(rule 19 (lower (bnot (vr128_ty ty) (bxor _ x (bnot _ (bxor _ y z)))))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01101001 x y z))
;;;; Rules for `bitselect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; z15 version using a NAND instruction.
-(rule 2 (lower (bitselect (and (mie3_enabled) (fits_in_64 ty)) x y z))
+(rule 2 (lower (bitselect (fits_in_64 ty) x y z))
+ (if-let true (has_mie3))
(let ((rx Reg x)
(if_true Reg (and_reg ty y rx))
(if_false Reg (and_not_reg ty z rx)))
(or_reg ty if_false if_true)))
;; z14 version using XOR with -1.
-(rule 1 (lower (bitselect (and (mie3_disabled) (fits_in_64 ty)) x y z))
+(rule 1 (lower (bitselect (fits_in_64 ty) x y z))
+ (if-let false (has_mie3))
(let ((rx Reg x)
(if_true Reg (and_reg ty y rx))
(if_false Reg (and_reg ty z (not_reg ty rx))))
@@ -1472,11 +1587,14 @@
(vec_select ty y z x))
;; Bitselect-not vector registers.
-(rule 5 (lower (bitselect (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ y) z))
+(rule 5 (lower (bitselect (vr128_ty ty) x (bnot _ y) z))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b01011100 x y z))
-(rule 6 (lower (bitselect (and (vxrs_ext3_enabled) (vr128_ty ty)) x y (bnot _ z)))
+(rule 6 (lower (bitselect (vr128_ty ty) x y (bnot _ z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10100011 x y z))
-(rule 7 (lower (bitselect (and (vxrs_ext3_enabled) (vr128_ty ty)) x (bnot _ y) (bnot _ z)))
+(rule 7 (lower (bitselect (vr128_ty ty) x (bnot _ y) (bnot _ z)))
+ (if-let true (has_vxrs_ext3))
(vec_eval ty 0b10101100 x y z))
;; Special-case some float-selection instructions for min/max
@@ -1546,7 +1664,8 @@
;; Count leading zeros pre-z17, via FLOGR on an input zero-extended to 64 bits,
;; with the result compensated for the extra bits.
-(rule 1 (lower (clz (and (mie4_disabled) (fits_in_64 ty)) x))
+(rule 1 (lower (clz (fits_in_64 ty) x))
+ (if-let false (has_mie4))
(let ((ext_reg Reg (put_in_reg_zext64 x))
;; Ask for a value of 64 in the all-zero 64-bit input case.
;; After compensation this will match the expected semantics.
@@ -1554,7 +1673,8 @@
(clz_offset ty clz)))
;; Count leading zeros, 128-bit full vector pre-z17.
-(rule (lower (clz (and (vxrs_ext3_disabled) $I128) x))
+(rule (lower (clz $I128 x))
+ (if-let false (has_vxrs_ext3))
(let ((clz_vec Reg (vec_clz $I64X2 x))
(zero Reg (vec_imm $I64X2 0))
(clz_hi Reg (vec_permute_dw_imm $I64X2 zero 0 clz_vec 0))
@@ -1565,14 +1685,16 @@
;; Count leading zeros on z17, via CLZG on an input zero-extended to 64 bits,
;; with the result compensated for the extra bits.
-(rule 3 (lower (clz (and (mie4_enabled) (fits_in_64 ty)) x))
+(rule 3 (lower (clz (fits_in_64 ty) x))
+ (if-let true (has_mie4))
(let ((ext_reg Reg (put_in_reg_zext64 x))
(clz Reg (clz_reg ext_reg)))
(clz_offset ty clz)))
;; Count leading zeros, 128-bit full vector on z17.
-(rule 2 (lower (clz (and (vxrs_ext3_enabled) $I128) x))
- (vec_clz $I128 x))
+(rule 2 (lower (clz $I128 x))
+ (if-let true (has_vxrs_ext3))
+ (vec_clz $I128 x))
;;;; Rules for `cls` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1590,7 +1712,8 @@
;; i.e. computing
;; cls(x) == clz(x ^ (x >> 63)) - 1
;; where x is the sign-extended input.
-(rule 1 (lower (cls (and (mie4_disabled) (fits_in_64 ty)) x))
+(rule 1 (lower (cls (fits_in_64 ty) x))
+ (if-let false (has_mie4))
(let ((ext_reg Reg (put_in_reg_sext64 x))
(signbit_copies Reg (ashr_imm $I64 ext_reg 63))
(inv_reg Reg (xor_reg $I64 ext_reg signbit_copies))
@@ -1598,7 +1721,8 @@
(cls_offset ty clz)))
;; Count leading sign-bit copies, 128-bit full vector pre-z17.
-(rule (lower (cls (and (vxrs_ext3_disabled) $I128) x))
+(rule (lower (cls $I128 x))
+ (if-let false (has_vxrs_ext3))
(let ((x_reg Reg x)
(ones Reg (vec_imm_splat $I8X16 255))
(signbit_copies Reg (vec_ashr_by_bit (vec_ashr_by_byte x_reg ones) ones))
@@ -1612,21 +1736,23 @@
(vec_add $I128 (vec_select $I128 clz_sum clz_hi mask) ones)))
;; Count leading sign-bit copies on z17, similar to above.
-(rule 3 (lower (cls (and (mie4_enabled) (fits_in_64 ty)) x))
- (let ((ext_reg Reg (put_in_reg_sext64 x))
- (signbit_copies Reg (ashr_imm $I64 ext_reg 63))
- (inv_reg Reg (xor_reg $I64 ext_reg signbit_copies))
- (clz Reg (clz_reg inv_reg)))
- (cls_offset ty clz)))
+(rule 3 (lower (cls (fits_in_64 ty) x))
+ (if-let true (has_mie4))
+ (let ((ext_reg Reg (put_in_reg_sext64 x))
+ (signbit_copies Reg (ashr_imm $I64 ext_reg 63))
+ (inv_reg Reg (xor_reg $I64 ext_reg signbit_copies))
+ (clz Reg (clz_reg inv_reg)))
+ (cls_offset ty clz)))
;; Count leading sign-bit copies, 128-bit full vector on z17.
-(rule 2 (lower (cls (and (vxrs_ext3_enabled) $I128) x))
- (let ((x_reg Reg x)
- (ones Reg (vec_imm_splat $I8X16 255))
- (signbit_copies Reg (vec_ashr_by_bit (vec_ashr_by_byte x_reg ones) ones))
- (inv_reg Reg (vec_xor $I128 x_reg signbit_copies))
- (clz_vec Reg (vec_clz $I128 inv_reg)))
- (vec_add $I128 clz_vec ones)))
+(rule 2 (lower (cls $I128 x))
+ (if-let true (has_vxrs_ext3))
+ (let ((x_reg Reg x)
+ (ones Reg (vec_imm_splat $I8X16 255))
+ (signbit_copies Reg (vec_ashr_by_bit (vec_ashr_by_byte x_reg ones) ones))
+ (inv_reg Reg (vec_xor $I128 x_reg signbit_copies))
+ (clz_vec Reg (vec_clz $I128 inv_reg)))
+ (vec_add $I128 clz_vec ones)))
;;;; Rules for `ctz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1643,7 +1769,8 @@
;; never zero by setting a "guard bit" in the position corresponding to
;; the input type size. This way the 64-bit algorithm above will handle
;; that case correctly automatically.
-(rule 2 (lower (ctz (and (mie4_disabled) (gpr32_ty ty)) x))
+(rule 2 (lower (ctz (gpr32_ty ty) x))
+ (if-let false (has_mie4))
(let ((rx Reg (or_uimm16shifted $I64 x (ctz_guardbit ty)))
(lastbit Reg (and_reg $I64 rx (neg_reg $I64 rx)))
(clz Reg (clz_flogr_reg 64 lastbit)))
@@ -1658,14 +1785,16 @@
;; via its condition code. We check for that and replace the instruction
;; result with the value -1 via a conditional move, which will then lead to
;; the correct result after the final subtraction from 63.
-(rule 1 (lower (ctz (and (mie4_disabled) (gpr64_ty _ty)) x))
+(rule 1 (lower (ctz (gpr64_ty _ty) x))
+ (if-let false (has_mie4))
(let ((rx Reg x)
(lastbit Reg (and_reg $I64 rx (neg_reg $I64 rx)))
(clz Reg (clz_flogr_reg -1 lastbit)))
(sub_reg $I64 (imm $I64 63) clz)))
;; Count trailing zeros, 128-bit full vector pre-z17.
-(rule 0 (lower (ctz (and (vxrs_ext3_disabled) $I128) x))
+ (rule 0 (lower (ctz $I128 x))
+ (if-let false (has_vxrs_ext3))
(let ((ctz_vec Reg (vec_ctz $I64X2 x))
(zero Reg (vec_imm $I64X2 0))
(ctz_hi Reg (vec_permute_dw_imm $I64X2 zero 0 ctz_vec 0))
@@ -1676,16 +1805,19 @@
;; Count leading zeros on z17, via CTZG on types smaller than 64-bit,
;; using the same guard bit mechanism as above.
-(rule 5 (lower (ctz (and (mie4_enabled) (gpr32_ty ty)) x))
- (ctz_reg (or_uimm16shifted $I64 x (ctz_guardbit ty))))
+(rule 5 (lower (ctz (gpr32_ty ty) x))
+ (if-let true (has_mie4))
+ (ctz_reg (or_uimm16shifted $I64 x (ctz_guardbit ty))))
;; Count leading zeros on z17, via CTZG directly on 64-bit types.
-(rule 4 (lower (ctz (and (mie4_enabled) (gpr64_ty _)) x))
- (ctz_reg x))
+(rule 4 (lower (ctz (gpr64_ty _) x))
+ (if-let true (has_mie4))
+ (ctz_reg x))
;; Count trailing zeros, 128-bit full vector on z17.
-(rule 3 (lower (ctz (and (vxrs_ext3_enabled) $I128) x))
- (vec_ctz $I128 x))
+(rule 3 (lower (ctz $I128 x))
+ (if-let true (has_vxrs_ext3))
+ (vec_ctz $I128 x))
;;;; Rules for `popcnt` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1696,7 +1828,8 @@
;; On z15, the POPCNT instruction has a variant to compute a full 64-bit
;; population count, which we also use for 16- and 32-bit types.
-(rule -1 (lower (popcnt (and (mie3_enabled) (fits_in_64 ty)) x))
+(rule -1 (lower (popcnt (fits_in_64 ty) x))
+ (if-let true (has_mie3))
(popcnt_reg (put_in_reg_zext64 x)))
;; On z14, we use the regular POPCNT, which computes the population count
@@ -1707,18 +1840,21 @@
;; $I16, where we instead accumulate in the low byte and clear high bits
;; via an explicit and operation.)
-(rule (lower (popcnt (and (mie3_disabled) $I16) x))
+(rule (lower (popcnt $I16 x))
+ (if-let false (has_mie3))
(let ((cnt2 Reg (popcnt_byte x))
(cnt1 Reg (add_reg $I32 cnt2 (lshr_imm $I32 cnt2 8))))
(and_uimm16shifted $I32 cnt1 (uimm16shifted 255 0))))
-(rule (lower (popcnt (and (mie3_disabled) $I32) x))
+(rule (lower (popcnt $I32 x))
+ (if-let false (has_mie3))
(let ((cnt4 Reg (popcnt_byte x))
(cnt2 Reg (add_reg $I32 cnt4 (lshl_imm $I32 cnt4 16)))
(cnt1 Reg (add_reg $I32 cnt2 (lshl_imm $I32 cnt2 8))))
(lshr_imm $I32 cnt1 24)))
-(rule (lower (popcnt (and (mie3_disabled) $I64) x))
+(rule (lower (popcnt $I64 x))
+ (if-let false (has_mie3))
(let ((cnt8 Reg (popcnt_byte x))
(cnt4 Reg (add_reg $I64 cnt8 (lshl_imm $I64 cnt8 32)))
(cnt2 Reg (add_reg $I64 cnt4 (lshl_imm $I64 cnt4 16)))
@@ -1892,8 +2028,8 @@
;;;; Rules for `fcvt_from_uint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Convert a 32-bit or smaller unsigned integer to $F32 (z15 instruction).
-(rule 1 (lower
- (fcvt_from_uint $F32 x @ (value_type (and (vxrs_ext2_enabled) (fits_in_32 ty)))))
+(rule 1 (lower (fcvt_from_uint $F32 x @ (value_type (fits_in_32 ty))))
+ (if-let true (has_vxrs_ext2))
(fcvt_from_uint_reg $F32 $I32 (FpuRoundMode.ToNearestTiesToEven)
(put_in_reg_zext32 x)))
@@ -1919,11 +2055,13 @@
(put_in_reg_zext64 x)))
;; Convert $I32X4 to $F32X4 (z15 instruction).
-(rule 1 (lower (fcvt_from_uint (and (vxrs_ext2_enabled) $F32X4) x @ (value_type $I32X4)))
+(rule 1 (lower (fcvt_from_uint $F32X4 x @ (value_type $I32X4)))
+ (if-let true (has_vxrs_ext2))
(fcvt_from_uint_reg $F32X4 $I32X4 (FpuRoundMode.ToNearestTiesToEven) x))
;; Convert $I32X4 to $F32X4 (via two $F64X2 on z14).
-(rule (lower (fcvt_from_uint (and (vxrs_ext2_disabled) $F32X4) x @ (value_type $I32X4)))
+(rule (lower (fcvt_from_uint $F32X4 x @ (value_type $I32X4)))
+ (if-let false (has_vxrs_ext2))
(vec_permute $F32X4
(fdemote_reg $F32X4 $F64X2 (FpuRoundMode.ToNearestTiesToEven)
(fcvt_from_uint_reg $F64X2 $I64X2 (FpuRoundMode.ShorterPrecision)
@@ -1941,8 +2079,8 @@
;;;; Rules for `fcvt_from_sint` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Convert a 32-bit or smaller signed integer to $F32 (z15 instruction).
-(rule 1 (lower
- (fcvt_from_sint $F32 x @ (value_type (and (vxrs_ext2_enabled) (fits_in_32 ty)))))
+(rule 1 (lower (fcvt_from_sint $F32 x @ (value_type (fits_in_32 ty))))
+ (if-let true (has_vxrs_ext2))
(fcvt_from_sint_reg $F32 $I32 (FpuRoundMode.ToNearestTiesToEven)
(put_in_reg_sext32 x)))
@@ -1968,11 +2106,13 @@
(put_in_reg_sext64 x)))
;; Convert $I32X4 to $F32X4 (z15 instruction).
-(rule 1 (lower (fcvt_from_sint (and (vxrs_ext2_enabled) $F32X4) x @ (value_type $I32X4)))
+(rule 1 (lower (fcvt_from_sint $F32X4 x @ (value_type $I32X4)))
+ (if-let true (has_vxrs_ext2))
(fcvt_from_sint_reg $F32X4 $I32X4 (FpuRoundMode.ToNearestTiesToEven) x))
;; Convert $I32X4 to $F32X4 (via two $F64X2 on z14).
-(rule (lower (fcvt_from_sint (and (vxrs_ext2_disabled) $F32X4) x @ (value_type $I32X4)))
+(rule (lower (fcvt_from_sint $F32X4 x @ (value_type $I32X4)))
+ (if-let false (has_vxrs_ext2))
(vec_permute $F32X4
(fdemote_reg $F32X4 $F64X2 (FpuRoundMode.ToNearestTiesToEven)
(fcvt_from_sint_reg $F64X2 $I64X2 (FpuRoundMode.ShorterPrecision)
@@ -2050,11 +2190,13 @@
(uint_sat_reg dst_ty int_ty dst)))
;; Convert $F32X4 to $I32X4 (z15 instruction).
-(rule 1 (lower (fcvt_to_uint_sat (and (vxrs_ext2_enabled) $I32X4) x @ (value_type $F32X4)))
+(rule 1 (lower (fcvt_to_uint_sat $I32X4 x @ (value_type $F32X4)))
+ (if-let true (has_vxrs_ext2))
(fcvt_to_uint_reg $I32X4 $F32X4 (FpuRoundMode.ToZero) x))
;; Convert $F32X4 to $I32X4 (via two $F64X2 on z14).
-(rule (lower (fcvt_to_uint_sat (and (vxrs_ext2_disabled) $I32X4) x @ (value_type $F32X4)))
+(rule (lower (fcvt_to_uint_sat $I32X4 x @ (value_type $F32X4)))
+ (if-let false (has_vxrs_ext2))
(vec_pack_usat $I64X2
(fcvt_to_uint_reg $I64X2 $F64X2 (FpuRoundMode.ToZero)
(fpromote_reg $F64X2 $F32X4 (vec_merge_high $I32X4 x x)))
@@ -2087,14 +2229,16 @@
(sint_sat_reg dst_ty int_ty sat)))
;; Convert $F32X4 to $I32X4 (z15 instruction).
-(rule 1 (lower (fcvt_to_sint_sat (and (vxrs_ext2_enabled) $I32X4) src @ (value_type $F32X4)))
+(rule 1 (lower (fcvt_to_sint_sat $I32X4 src @ (value_type $F32X4)))
+ (if-let true (has_vxrs_ext2))
;; See above for why we need to handle NaNs specially.
(vec_select $I32X4
(fcvt_to_sint_reg $I32X4 $F32X4 (FpuRoundMode.ToZero) src)
(vec_imm $I32X4 0) (vec_fcmpeq $F32X4 src src)))
;; Convert $F32X4 to $I32X4 (via two $F64X2 on z14).
-(rule (lower (fcvt_to_sint_sat (and (vxrs_ext2_disabled) $I32X4) src @ (value_type $F32X4)))
+(rule (lower (fcvt_to_sint_sat $I32X4 src @ (value_type $F32X4)))
+ (if-let false (has_vxrs_ext2))
;; See above for why we need to handle NaNs specially.
(vec_select $I32X4
(vec_pack_ssat $I64X2
@@ -2236,25 +2380,25 @@
(vec_load_lane ty dst addr lane_imm))
;; On z15, we have instructions to perform little-endian loads.
-(rule 1 (vec_load_lane_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 16 _)) dst addr lane_imm)
+(rule 1 (vec_load_lane_little ty @ (multi_lane 16 _) dst addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_load_lane_rev ty dst addr lane_imm))
-(rule 1 (vec_load_lane_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 32 _)) dst addr lane_imm)
+(rule 1 (vec_load_lane_little ty @ (multi_lane 32 _) dst addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_load_lane_rev ty dst addr lane_imm))
-(rule 1 (vec_load_lane_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 64 _)) dst addr lane_imm)
+(rule 1 (vec_load_lane_little ty @ (multi_lane 64 _) dst addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_load_lane_rev ty dst addr lane_imm))
;; On z14, use a little-endian load to GPR followed by vec_insert_lane.
-(rule (vec_load_lane_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 16 _)) dst addr lane_imm)
+(rule (vec_load_lane_little ty @ (multi_lane 16 _) dst addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(vec_insert_lane ty dst (loadrev16 addr) lane_imm (zero_reg)))
-(rule (vec_load_lane_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 32 _)) dst addr lane_imm)
+(rule (vec_load_lane_little ty @ (multi_lane 32 _) dst addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(vec_insert_lane ty dst (loadrev32 addr) lane_imm (zero_reg)))
-(rule (vec_load_lane_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 64 _)) dst addr lane_imm)
+(rule (vec_load_lane_little ty @ (multi_lane 64 _) dst addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(vec_insert_lane ty dst (loadrev64 addr) lane_imm (zero_reg)))
;; Helper to implement a generic little-endian variant of vec_load_lane_undef.
@@ -2265,25 +2409,25 @@
(vec_load_lane_undef ty addr lane_imm))
;; On z15, we have instructions to perform little-endian loads.
-(rule 1 (vec_load_lane_little_undef (and (vxrs_ext2_enabled)
- ty @ (multi_lane 16 _)) addr lane_imm)
+(rule 1 (vec_load_lane_little_undef ty @ (multi_lane 16 _) addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_load_lane_rev_undef ty addr lane_imm))
-(rule 1 (vec_load_lane_little_undef (and (vxrs_ext2_enabled)
- ty @ (multi_lane 32 _)) addr lane_imm)
+(rule 1 (vec_load_lane_little_undef ty @ (multi_lane 32 _) addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_load_lane_rev_undef ty addr lane_imm))
-(rule 1 (vec_load_lane_little_undef (and (vxrs_ext2_enabled)
- ty @ (multi_lane 64 _)) addr lane_imm)
+(rule 1 (vec_load_lane_little_undef ty @ (multi_lane 64 _) addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_load_lane_rev_undef ty addr lane_imm))
;; On z14, use a little-endian load to GPR followed by vec_insert_lane_undef.
-(rule (vec_load_lane_little_undef (and (vxrs_ext2_disabled)
- ty @ (multi_lane 16 _)) addr lane_imm)
+(rule (vec_load_lane_little_undef ty @ (multi_lane 16 _) addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(vec_insert_lane_undef ty (loadrev16 addr) lane_imm (zero_reg)))
-(rule (vec_load_lane_little_undef (and (vxrs_ext2_disabled)
- ty @ (multi_lane 32 _)) addr lane_imm)
+(rule (vec_load_lane_little_undef ty @ (multi_lane 32 _) addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(vec_insert_lane_undef ty (loadrev32 addr) lane_imm (zero_reg)))
-(rule (vec_load_lane_little_undef (and (vxrs_ext2_disabled)
- ty @ (multi_lane 64 _)) addr lane_imm)
+(rule (vec_load_lane_little_undef ty @ (multi_lane 64 _) addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(vec_insert_lane_undef ty (loadrev64 addr) lane_imm (zero_reg)))
@@ -2321,25 +2465,25 @@
(vec_store_lane ty src addr lane_imm))
;; On z15, we have instructions to perform little-endian stores.
-(rule 1 (vec_store_lane_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 16 _)) src addr lane_imm)
+(rule 1 (vec_store_lane_little ty @ (multi_lane 16 _) src addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_store_lane_rev ty src addr lane_imm))
-(rule 1 (vec_store_lane_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 32 _)) src addr lane_imm)
+(rule 1 (vec_store_lane_little ty @ (multi_lane 32 _) src addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_store_lane_rev ty src addr lane_imm))
-(rule 1 (vec_store_lane_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 64 _)) src addr lane_imm)
+(rule 1 (vec_store_lane_little ty @ (multi_lane 64 _) src addr lane_imm)
+ (if-let true (has_vxrs_ext2))
(vec_store_lane_rev ty src addr lane_imm))
;; On z14, use vec_extract_lane followed by a little-endian store from GPR.
-(rule (vec_store_lane_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 16 _)) src addr lane_imm)
+(rule (vec_store_lane_little ty @ (multi_lane 16 _) src addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(storerev16 (vec_extract_lane ty src lane_imm (zero_reg)) addr))
-(rule (vec_store_lane_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 32 _)) src addr lane_imm)
+(rule (vec_store_lane_little ty @ (multi_lane 32 _) src addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(storerev32 (vec_extract_lane ty src lane_imm (zero_reg)) addr))
-(rule (vec_store_lane_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 64 _)) src addr lane_imm)
+(rule (vec_store_lane_little ty @ (multi_lane 64 _) src addr lane_imm)
+ (if-let false (has_vxrs_ext2))
(storerev64 (vec_extract_lane ty src lane_imm (zero_reg)) addr))
@@ -2379,25 +2523,25 @@
(vec_load_replicate ty addr))
;; On z15, we have instructions to perform little-endian loads.
-(rule 1 (vec_load_replicate_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 16 _)) addr)
+(rule 1 (vec_load_replicate_little ty @ (multi_lane 16 _) addr)
+ (if-let true (has_vxrs_ext2))
(vec_load_replicate_rev ty addr))
-(rule 1 (vec_load_replicate_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 32 _)) addr)
+(rule 1 (vec_load_replicate_little ty @ (multi_lane 32 _) addr)
+ (if-let true (has_vxrs_ext2))
(vec_load_replicate_rev ty addr))
-(rule 1 (vec_load_replicate_little (and (vxrs_ext2_enabled)
- ty @ (multi_lane 64 _)) addr)
+(rule 1 (vec_load_replicate_little ty @ (multi_lane 64 _) addr)
+ (if-let true (has_vxrs_ext2))
(vec_load_replicate_rev ty addr))
;; On z14, use a little-endian load (via GPR) and replicate.
-(rule (vec_load_replicate_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 16 _)) addr)
+(rule (vec_load_replicate_little ty @ (multi_lane 16 _) addr)
+ (if-let false (has_vxrs_ext2))
(vec_replicate_lane ty (vec_load_lane_little_undef ty addr 0) 0))
-(rule (vec_load_replicate_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 32 _)) addr)
+(rule (vec_load_replicate_little ty @ (multi_lane 32 _) addr)
+ (if-let false (has_vxrs_ext2))
(vec_replicate_lane ty (vec_load_lane_little_undef ty addr 0) 0))
-(rule (vec_load_replicate_little (and (vxrs_ext2_disabled)
- ty @ (multi_lane 64 _)) addr)
+(rule (vec_load_replicate_little ty @ (multi_lane 64 _) addr)
+ (if-let false (has_vxrs_ext2))
(vec_replicate_lane ty (vec_load_lane_little_undef ty addr 0) 0))
@@ -2602,8 +2746,9 @@
;;;; Rules for `blendv` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-(rule 1 (lower (blendv (and (ty_vec128 ty) (vxrs_ext3_enabled)) p x y))
- (vec_blend ty p x y))
+(rule 1 (lower (blendv (ty_vec128 ty) p x y))
+ (if-let true (has_vxrs_ext3))
+ (vec_blend ty p x y))
;;;; Rules for `swizzle` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2779,11 +2924,13 @@
(decl vec_load_full_rev (Type MemFlags Value Offset32) Reg)
;; Full-vector byte-reversed load via single instruction on z15.
-(rule 1 (vec_load_full_rev (and (vxrs_ext2_enabled) (vr128_ty ty)) flags addr offset)
+ (rule 1 (vec_load_full_rev (vr128_ty ty) flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_loadrev ty (lower_address flags addr offset)))
;; Full-vector byte-reversed load via GPRs on z14.
-(rule (vec_load_full_rev (and (vxrs_ext2_disabled) (vr128_ty ty)) flags addr offset)
+(rule (vec_load_full_rev (vr128_ty ty) flags addr offset)
+ (if-let false (has_vxrs_ext2))
(let ((lo_addr MemArg (lower_address_bias flags addr offset 0))
(hi_addr MemArg (lower_address_bias flags addr offset 8))
(lo_val Reg (loadrev64 lo_addr))
@@ -2807,25 +2954,25 @@
(vec_load ty (lower_address flags addr offset)))
;; Element-wise byte-reversed load via single instruction on z15.
-(rule 1 (vec_load_byte_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 64 2))
- flags addr offset)
+(rule 1 (vec_load_byte_rev ty @ (multi_lane 64 2) flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_load_byte64rev ty (lower_address flags addr offset)))
-(rule 1 (vec_load_byte_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 32 4))
- flags addr offset)
+(rule 1 (vec_load_byte_rev ty @ (multi_lane 32 4) flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_load_byte32rev ty (lower_address flags addr offset)))
-(rule 1 (vec_load_byte_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 16 8))
- flags addr offset)
+(rule 1 (vec_load_byte_rev ty @ (multi_lane 16 8) flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_load_byte16rev ty (lower_address flags addr offset)))
;; Element-wise byte-reversed load as element-swapped byte-reversed load on z14.
-(rule (vec_load_byte_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 64 2))
- flags addr offset)
+(rule (vec_load_byte_rev ty @ (multi_lane 64 2) flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_elt_rev ty (vec_load_full_rev ty flags addr offset)))
-(rule (vec_load_byte_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 32 4))
- flags addr offset)
+(rule (vec_load_byte_rev ty @ (multi_lane 32 4) flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_elt_rev ty (vec_load_full_rev ty flags addr offset)))
-(rule (vec_load_byte_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 16 8))
- flags addr offset)
+(rule (vec_load_byte_rev ty @ (multi_lane 16 8) flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_elt_rev ty (vec_load_full_rev ty flags addr offset)))
@@ -2846,25 +2993,25 @@
(vec_load_full_rev ty flags addr offset))
;; Element-reversed load via single instruction on z15.
-(rule 1 (vec_load_elt_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 64 2))
- flags addr offset)
+(rule 1 (vec_load_elt_rev ty @ (multi_lane 64 2) flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_load_elt64rev ty (lower_address flags addr offset)))
-(rule 1 (vec_load_elt_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 32 4))
- flags addr offset)
+(rule 1 (vec_load_elt_rev ty @ (multi_lane 32 4) flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_load_elt32rev ty (lower_address flags addr offset)))
-(rule 1 (vec_load_elt_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 16 8))
- flags addr offset)
+(rule 1 (vec_load_elt_rev ty @ (multi_lane 16 8) flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_load_elt16rev ty (lower_address flags addr offset)))
;; Element-reversed load as element-swapped direct load on z14.
-(rule (vec_load_elt_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 64 2))
- flags addr offset)
+(rule (vec_load_elt_rev ty @ (multi_lane 64 2) flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_elt_rev ty (vec_load ty (lower_address flags addr offset))))
-(rule (vec_load_elt_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 32 4))
- flags addr offset)
+(rule (vec_load_elt_rev ty @ (multi_lane 32 4) flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_elt_rev ty (vec_load ty (lower_address flags addr offset))))
-(rule (vec_load_elt_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 16 8))
- flags addr offset)
+(rule (vec_load_elt_rev ty @ (multi_lane 16 8) flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_elt_rev ty (vec_load ty (lower_address flags addr offset))))
@@ -3117,11 +3264,13 @@
(decl vec_store_full_rev (Type Reg MemFlags Value Offset32) SideEffectNoResult)
;; Full-vector byte-reversed store via single instruction on z15.
-(rule 1 (vec_store_full_rev (vxrs_ext2_enabled) val flags addr offset)
+(rule 1 (vec_store_full_rev _ val flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_storerev val (lower_address flags addr offset)))
;; Full-vector byte-reversed store via GPRs on z14.
-(rule (vec_store_full_rev (vxrs_ext2_disabled) val flags addr offset)
+(rule (vec_store_full_rev _ val flags addr offset)
+ (if-let false (has_vxrs_ext2))
(let ((lo_addr MemArg (lower_address_bias flags addr offset 0))
(hi_addr MemArg (lower_address_bias flags addr offset 8))
(lo_val Reg (vec_extract_lane $I64X2 val 1 (zero_reg)))
@@ -3146,25 +3295,25 @@
(vec_store val (lower_address flags addr offset)))
;; Element-wise byte-reversed store via single instruction on z15.
-(rule 1 (vec_store_byte_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 64 2))
- val flags addr offset)
+(rule 1 (vec_store_byte_rev ty @ (multi_lane 64 2) val flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_store_byte64rev val (lower_address flags addr offset)))
-(rule 1 (vec_store_byte_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 32 4))
- val flags addr offset)
+(rule 1 (vec_store_byte_rev ty @ (multi_lane 32 4) val flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_store_byte32rev val (lower_address flags addr offset)))
-(rule 1 (vec_store_byte_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 16 8))
- val flags addr offset)
+(rule 1 (vec_store_byte_rev ty @ (multi_lane 16 8) val flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_store_byte16rev val (lower_address flags addr offset)))
;; Element-wise byte-reversed load as element-swapped byte-reversed store on z14.
-(rule (vec_store_byte_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 64 2))
- val flags addr offset)
+(rule (vec_store_byte_rev ty @ (multi_lane 64 2) val flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_store_full_rev ty (vec_elt_rev ty val) flags addr offset))
-(rule (vec_store_byte_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 32 4))
- val flags addr offset)
+(rule (vec_store_byte_rev ty @ (multi_lane 32 4) val flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_store_full_rev ty (vec_elt_rev ty val) flags addr offset))
-(rule (vec_store_byte_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 16 8))
- val flags addr offset)
+(rule (vec_store_byte_rev ty @ (multi_lane 16 8) val flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_store_full_rev ty (vec_elt_rev ty val) flags addr offset))
@@ -3184,25 +3333,25 @@
(vec_store_full_rev ty val flags addr offset))
;; Element-reversed store via single instruction on z15.
-(rule 1 (vec_store_elt_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 64 2))
- val flags addr offset)
+(rule 1 (vec_store_elt_rev ty @ (multi_lane 64 2) val flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_store_elt64rev val (lower_address flags addr offset)))
-(rule 1 (vec_store_elt_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 32 4))
- val flags addr offset)
+(rule 1 (vec_store_elt_rev ty @ (multi_lane 32 4) val flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_store_elt32rev val (lower_address flags addr offset)))
-(rule 1 (vec_store_elt_rev (and (vxrs_ext2_enabled) ty @ (multi_lane 16 8))
- val flags addr offset)
+(rule 1 (vec_store_elt_rev ty @ (multi_lane 16 8) val flags addr offset)
+ (if-let true (has_vxrs_ext2))
(vec_store_elt16rev val (lower_address flags addr offset)))
;; Element-reversed store as element-swapped direct store on z14.
-(rule (vec_store_elt_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 64 2))
- val flags addr offset)
+(rule (vec_store_elt_rev ty @ (multi_lane 64 2) val flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_store (vec_elt_rev ty val) (lower_address flags addr offset)))
-(rule (vec_store_elt_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 32 4))
- val flags addr offset)
+(rule (vec_store_elt_rev ty @ (multi_lane 32 4) val flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_store (vec_elt_rev ty val) (lower_address flags addr offset)))
-(rule (vec_store_elt_rev (and (vxrs_ext2_disabled) ty @ (multi_lane 16 8))
- val flags addr offset)
+(rule (vec_store_elt_rev ty @ (multi_lane 16 8) val flags addr offset)
+ (if-let false (has_vxrs_ext2))
(vec_store (vec_elt_rev ty val) (lower_address flags addr offset)))
@@ -3383,18 +3532,22 @@
;; On z15 this can use the NN(G)RK instruction. On z14, perform an And
;; operation and invert the result. In the little-endian case, we can
;; simply byte-swap the source operand.
-(rule 4 (atomic_rmw_body ib (and (mie3_enabled) (ty_32_or_64 ty)) (bigendian)
+(rule 4 (atomic_rmw_body ib (ty_32_or_64 ty) (bigendian)
(AtomicRmwOp.Nand) tmp val src)
+ (if-let true (has_mie3))
(push_alu_reg ib (aluop_not_and ty) tmp val src))
-(rule 3 (atomic_rmw_body ib (and (mie3_enabled) (ty_32_or_64 ty)) (littleendian)
+(rule 3 (atomic_rmw_body ib (ty_32_or_64 ty) (littleendian)
(AtomicRmwOp.Nand) tmp val src)
+ (if-let true (has_mie3))
(push_alu_reg ib (aluop_not_and ty) tmp val (bswap_reg ty src)))
-(rule 2 (atomic_rmw_body ib (and (mie3_disabled) (ty_32_or_64 ty)) (bigendian)
+(rule 2 (atomic_rmw_body ib (ty_32_or_64 ty) (bigendian)
(AtomicRmwOp.Nand) tmp val src)
+ (if-let false (has_mie3))
(push_not_reg ib ty tmp
(push_alu_reg ib (aluop_and ty) tmp val src)))
-(rule 1 (atomic_rmw_body ib (and (mie3_disabled) (ty_32_or_64 ty)) (littleendian)
+(rule 1 (atomic_rmw_body ib (ty_32_or_64 ty) (littleendian)
(AtomicRmwOp.Nand) tmp val src)
+ (if-let false (has_mie3))
(push_not_reg ib ty tmp
(push_alu_reg ib (aluop_and ty) tmp val (bswap_reg ty src))))
@@ -3789,42 +3942,54 @@
;; Compare (signed) 128-bit integers on z17.
-(rule 2 (icmp_val _ int_cc @ (signed) x @ (value_type (and (vxrs_ext3_enabled) (vr128_ty _))) y)
- (bool (vec_elt_icmps x y) (intcc_as_cond int_cc)))
+(rule 2 (icmp_val _ int_cc @ (signed) x @ (value_type (vr128_ty _)) y)
+ (if-let true (has_vxrs_ext3))
+ (bool (vec_elt_icmps x y) (intcc_as_cond int_cc)))
;; Compare (unsigned) 128-bit integers on z17.
-(rule 1 (icmp_val _ int_cc @ (unsigned) x @ (value_type (and (vxrs_ext3_enabled) (vr128_ty _))) y)
- (bool (vec_elt_icmpu x y) (intcc_as_cond int_cc)))
+(rule 1 (icmp_val _ int_cc @ (unsigned) x @ (value_type (vr128_ty _)) y)
+ (if-let true (has_vxrs_ext3))
+ (bool (vec_elt_icmpu x y) (intcc_as_cond int_cc)))
;; Compare 128-bit integers for equality pre-z17.
;; Implemented via element-wise comparison using the all-element true CC flag.
-(rule (icmp_val _ (IntCC.Equal) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty _))) y)
+(rule (icmp_val _ (IntCC.Equal) x @ (value_type (vr128_ty _)) y)
+ (if-let false (has_vxrs_ext3))
(bool (vec_cmpeqs $I64X2 x y)
(floatcc_as_cond (FloatCC.Equal))))
-(rule (icmp_val _ (IntCC.NotEqual) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty _))) y)
+(rule (icmp_val _ (IntCC.NotEqual) x @ (value_type (vr128_ty _)) y)
+ (if-let false (has_vxrs_ext3))
(bool (vec_cmpeqs $I64X2 x y)
(floatcc_as_cond (FloatCC.NotEqual))))
;; Compare (signed) 128-bit integers for relational inequality pre-z17.
;; Implemented via synthetic instruction using VECG and VCHLGS.
-(rule (icmp_val _ (IntCC.SignedGreaterThan) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty ty))) y)
+(rule (icmp_val _ (IntCC.SignedGreaterThan) x @ (value_type (vr128_ty ty)) y)
+ (if-let false (has_vxrs_ext3))
(vec_int128_scmphi x y))
-(rule (icmp_val _ (IntCC.SignedLessThan) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty ty))) y)
+(rule (icmp_val _ (IntCC.SignedLessThan) x @ (value_type (vr128_ty ty)) y)
+ (if-let false (has_vxrs_ext3))
(vec_int128_scmphi y x))
-(rule (icmp_val _ (IntCC.SignedGreaterThanOrEqual) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty ty))) y)
+(rule (icmp_val _ (IntCC.SignedGreaterThanOrEqual) x @ (value_type (vr128_ty ty)) y)
+ (if-let false (has_vxrs_ext3))
(invert_bool (vec_int128_scmphi y x)))
-(rule (icmp_val _ (IntCC.SignedLessThanOrEqual) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty ty))) y)
+(rule (icmp_val _ (IntCC.SignedLessThanOrEqual) x @ (value_type (vr128_ty ty)) y)
+ (if-let false (has_vxrs_ext3))
(invert_bool (vec_int128_scmphi x y)))
;; Compare (unsigned) 128-bit integers for relational inequality pre-z17.
;; Implemented via synthetic instruction using VECLG and VCHLGS.
-(rule (icmp_val _ (IntCC.UnsignedGreaterThan) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty ty))) y)
+(rule (icmp_val _ (IntCC.UnsignedGreaterThan) x @ (value_type (vr128_ty ty)) y)
+ (if-let false (has_vxrs_ext3))
(vec_int128_ucmphi x y))
-(rule (icmp_val _ (IntCC.UnsignedLessThan) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty ty))) y)
+(rule (icmp_val _ (IntCC.UnsignedLessThan) x @ (value_type (vr128_ty ty)) y)
+ (if-let false (has_vxrs_ext3))
(vec_int128_ucmphi y x))
-(rule (icmp_val _ (IntCC.UnsignedGreaterThanOrEqual) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty ty))) y)
+(rule (icmp_val _ (IntCC.UnsignedGreaterThanOrEqual) x @ (value_type (vr128_ty ty)) y)
+ (if-let false (has_vxrs_ext3))
(invert_bool (vec_int128_ucmphi y x)))
-(rule (icmp_val _ (IntCC.UnsignedLessThanOrEqual) x @ (value_type (and (vxrs_ext3_disabled) (vr128_ty ty))) y)
+(rule (icmp_val _ (IntCC.UnsignedLessThanOrEqual) x @ (value_type (vr128_ty ty)) y)
+ (if-let false (has_vxrs_ext3))
(invert_bool (vec_int128_ucmphi x y)))
diff --git a/cranelift/codegen/src/isa/s390x/lower/isle.rs b/cranelift/codegen/src/isa/s390x/lower/isle.rs
index 75ea159196..bededcef31 100644
--- a/cranelift/codegen/src/isa/s390x/lower/isle.rs
+++ b/cranelift/codegen/src/isa/s390x/lower/isle.rs
@@ -213,75 +213,23 @@ impl generated_code::Context for IsleContext<'_, '_, MInst, S390xBackend> {
}
#[inline]
- fn mie3_enabled(&mut self, _: Type) -> Option<()> {
- if self.backend.isa_flags.has_mie3() {
- Some(())
- } else {
- None
- }
- }
-
- #[inline]
- fn mie3_disabled(&mut self, _: Type) -> Option<()> {
- if !self.backend.isa_flags.has_mie3() {
- Some(())
- } else {
- None
- }
- }
-
- #[inline]
- fn mie4_enabled(&mut self, _: Type) -> Option<()> {
- if self.backend.isa_flags.has_mie4() {
- Some(())
- } else {
- None
- }
- }
-
- #[inline]
- fn mie4_disabled(&mut self, _: Type) -> Option<()> {
- if !self.backend.isa_flags.has_mie4() {
- Some(())
- } else {
- None
- }
+ fn has_mie3(&mut self) -> bool {
+ self.backend.isa_flags.has_mie3()
}
#[inline]
- fn vxrs_ext2_enabled(&mut self, _: Type) -> Option<()> {
- if self.backend.isa_flags.has_vxrs_ext2() {
- Some(())
- } else {
- None
- }
- }
-
- #[inline]
- fn vxrs_ext2_disabled(&mut self, _: Type) -> Option<()> {
- if !self.backend.isa_flags.has_vxrs_ext2() {
- Some(())
- } else {
- None
- }
+ fn has_mie4(&mut self) -> bool {
+ self.backend.isa_flags.has_mie4()
}
#[inline]
- fn vxrs_ext3_enabled(&mut self, _: Type) -> Option<()> {
- if self.backend.isa_flags.has_vxrs_ext3() {
- Some(())
- } else {
- None
- }
+ fn has_vxrs_ext2(&mut self) -> bool {
+ self.backend.isa_flags.has_vxrs_ext2()
}
#[inline]
- fn vxrs_ext3_disabled(&mut self, _: Type) -> Option<()> {
- if !self.backend.isa_flags.has_vxrs_ext3() {
- Some(())
- } else {
- None
- }
+ fn has_vxrs_ext3(&mut self) -> bool {
+ self.backend.isa_flags.has_vxrs_ext3()
}
#[inline]