Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

x64: Optimize some lowerings using `ptest` (#13462)

Alex Crichton1b9e6b35236e
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isle
index 7324c34731..1543bbf73a 100644
--- a/cranelift/codegen/src/isa/pulley_shared/inst.isle
+++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle
@@ -430,9 +430,9 @@
;; of the address against a particular bound.
(decl pure partial wasm_oob_cond (Value Value) OobCond)
(type OobCond (enum (All (a Value) (b u64))))
-(rule (wasm_oob_cond wasm_addr (ugt wasm_addr (isub _ bound (u64_from_iconst n))))
+(rule (wasm_oob_cond wasm_addr (ugt _ wasm_addr (isub _ bound (u64_from_iconst n))))
(OobCond.All bound n))
-(rule (wasm_oob_cond wasm_addr (ult (isub _ bound (u64_from_iconst n)) wasm_addr))
+(rule (wasm_oob_cond wasm_addr (ult _ (isub _ bound (u64_from_iconst n)) wasm_addr))
(OobCond.All bound n))
;;;; Newtypes for Different Register Classes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 1a9d4b5692..9bee238f0c 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -4968,22 +4968,47 @@
(mask Gpr (x64_pmovmskb any_byte_zero))
)
(CondResult.CC (x64_cmpl_mi mask 0xffff) (CC.NZ))))
+
+;; With `ptest` that'll set `ZF` based on whether `val & val` has any nonzero
+;; bit, which is equivalent to saying that anything is true (or not equal to
+;; zero).
(rule 1 (is_vany_true val)
(if-let true (has_sse41))
(let ((val Xmm val))
(CondResult.CC (x64_ptest val val) (CC.NZ))))
+;; A `band` operation can get folded into `ptest` directly since it already does
+;; the operation for us.
+(rule 2 (is_vany_true (band (ty_vec128 _) a b))
+ (if-let true (has_sse41))
+ (CondResult.CC (x64_ptest a b) (CC.NZ)))
+
+;; In addition to `band`, the `ptest` instruction also performs "andnot" and
+;; puts the results in the `C` flag. This means that `(band a (bnot b))` can
+;; also be folded in. Note that bitwise negation happens in the first operand.
+(rule 3 (is_vany_true (band (ty_vec128 _) a (bnot _ b)))
+ (if-let true (has_sse41))
+ (CondResult.CC (x64_ptest b a) (CC.NB)))
+(rule 4 (is_vany_true (band (ty_vec128 _) (bnot _ a) b))
+ (if-let true (has_sse41))
+ (CondResult.CC (x64_ptest a b) (CC.NB)))
+
+;; Vector inequality can be a bit more efficient with `ptest`. When testing
+;; this property of vectors the inputs are xor'd together and the result is
+;; passed to `ptest`. The logical operation here is: is any lane of `a` not
+;; equal to the corresponding lane in `b`? With `pxor` we can reduce that to
+;; if any bit in the xor is set, and that's exactly that testing the `ZF` flag
+;; after `ptest` will tell us.
+(rule 2 (is_vany_true (ne (ty_vec128 ty) a b))
+ (if-let true (has_sse41))
+ (let ((tmp Xmm (x64_pxor a b)))
+ (CondResult.CC (x64_ptest tmp tmp) (CC.NZ))))
+
;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (vall_true _ val)) (lower_cond_bool (is_vall_true val)))
(decl is_vall_true (Value) CondResult)
-(rule 1 (is_vall_true val @ (value_type ty))
- (if-let true (has_sse41))
- (let ((src Xmm val)
- (zeros Xmm (xmm_zero ty))
- (cmp Xmm (x64_pcmpeq (vec_int_type ty) src zeros)))
- (CondResult.CC (x64_ptest cmp cmp) (CC.Z))))
;; Perform an appropriately-sized lane-wise comparison with zero. If the
;; result is all 0s then all of them are true because nothing was equal to
@@ -4993,6 +5018,26 @@
(mask Gpr (x64_pmovmskb lanes_with_zero)))
(CondResult.CC (x64_testl_mr mask mask) (CC.Z))))
+;; With `ptest` this can be a bit smaller. Compare `val` to an all-zeros mask,
+;; then `ptest` that comparison with itself. If any lane in `val` is zero then
+;; the result will have some 1s, otherwise it's all zeros.
+(rule 1 (is_vall_true val @ (value_type ty))
+ (if-let true (has_sse41))
+ (let ((src Xmm val)
+ (zeros Xmm (xmm_zero ty))
+ (cmp Xmm (x64_pcmpeq (vec_int_type ty) src zeros)))
+ (CondResult.CC (x64_ptest cmp cmp) (CC.Z))))
+
+;; This is similar to `(is_vany_true (ne _ a b))` above. The logical operation
+;; here is: is every lane of `a` equal to the corresponding lane of `b`. With
+;; `a ^ b` this property is only true if every single bit in the output is 0.
+;; That is exactly what `ptest` tells us, so look for the `ZF` flag on the
+;; xor'd result.
+(rule 2 (is_vall_true (eq (ty_vec128 ty) a b))
+ (if-let true (has_sse41))
+ (let ((tmp Xmm (x64_pxor a b)))
+ (CondResult.CC (x64_ptest tmp tmp) (CC.Z))))
+
;; Rules for `vhigh_bits` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; The Intel specification allows using both 32-bit and 64-bit GPRs as
diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isle
index 8200c814c9..baae79a3df 100644
--- a/cranelift/codegen/src/prelude_lower.isle
+++ b/cranelift/codegen/src/prelude_lower.isle
@@ -341,35 +341,35 @@
(decl uimm8 (u8) Imm64)
(extern extractor uimm8 uimm8)
-(decl eq (Value Value) Value)
-(extractor (eq x y) (icmp (IntCC.Equal) x y))
+(decl eq (Type Value Value) Value)
+(extractor (eq ty x y) (icmp ty (IntCC.Equal) x y))
-(decl ne (Value Value) Value)
-(extractor (ne x y) (icmp (IntCC.NotEqual) x y))
+(decl ne (Type Value Value) Value)
+(extractor (ne ty x y) (icmp ty (IntCC.NotEqual) x y))
-(decl ult (Value Value) Value)
-(extractor (ult x y) (icmp _ (IntCC.UnsignedLessThan) x y))
+(decl ult (Type Value Value) Value)
+(extractor (ult ty x y) (icmp ty (IntCC.UnsignedLessThan) x y))
-(decl ule (Value Value) Value)
-(extractor (ule x y) (icmp (IntCC.UnsignedLessThanOrEqual) x y))
+(decl ule (Type Value Value) Value)
+(extractor (ule ty x y) (icmp ty (IntCC.UnsignedLessThanOrEqual) x y))
-(decl ugt (Value Value) Value)
-(extractor (ugt x y) (icmp _ (IntCC.UnsignedGreaterThan) x y))
+(decl ugt (Type Value Value) Value)
+(extractor (ugt ty x y) (icmp ty (IntCC.UnsignedGreaterThan) x y))
-(decl uge (Value Value) Value)
-(extractor (uge x y) (icmp (IntCC.UnsignedGreaterThanOrEqual) x y))
+(decl uge (Type Value Value) Value)
+(extractor (uge ty x y) (icmp ty (IntCC.UnsignedGreaterThanOrEqual) x y))
-(decl slt (Value Value) Value)
-(extractor (slt x y) (icmp (IntCC.SignedLessThan) x y))
+(decl slt (Type Value Value) Value)
+(extractor (slt ty x y) (icmp ty (IntCC.SignedLessThan) x y))
-(decl sle (Value Value) Value)
-(extractor (sle x y) (icmp (IntCC.SignedLessThanOrEqual) x y))
+(decl sle (Type Value Value) Value)
+(extractor (sle ty x y) (icmp ty (IntCC.SignedLessThanOrEqual) x y))
-(decl sgt (Value Value) Value)
-(extractor (sgt x y) (icmp (IntCC.SignedGreaterThan) x y))
+(decl sgt (Type Value Value) Value)
+(extractor (sgt ty x y) (icmp ty (IntCC.SignedGreaterThan) x y))
-(decl sge (Value Value) Value)
-(extractor (sge x y) (icmp (IntCC.SignedGreaterThanOrEqual) x y))
+(decl sge (Type Value Value) Value)
+(extractor (sge ty x y) (icmp ty (IntCC.SignedGreaterThanOrEqual) x y))
;; Get the MachLabel for a particular CLIF block's indexed exception-handling successor.
(decl block_exn_successor_label (Block u64) MachLabel)
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs
index 9698fd4921..03c340ea29 100644
--- a/crates/test-util/src/wast.rs
+++ b/crates/test-util/src/wast.rs
@@ -655,6 +655,7 @@ impl WastTest {
"misc_testsuite/winch/issue-10331.wast",
"misc_testsuite/winch/replace_lane.wast",
"misc_testsuite/simd/riscv64-replicated-imm5-works.wast",
+ "misc_testsuite/simd/v128-equal.wast",
"spec_testsuite/simd_align.wast",
"spec_testsuite/simd_boolean.wast",
"spec_testsuite/simd_conversions.wast",
diff --git a/tests/disas/x64-vptest.wat b/tests/disas/x64-vptest.wat
new file mode 100644
index 0000000000..ffd6cd020e
--- /dev/null
+++ b/tests/disas/x64-vptest.wat
@@ -0,0 +1,72 @@
+;;! target = 'x86_64'
+;;! test = 'compile'
+;;! flags = '-Ccranelift-has-sse41 -Ccranelift-has-avx'
+
+(module
+ (func $e1 (param v128 v128) (result i32)
+ local.get 0
+ local.get 1
+ i8x16.eq
+ i8x16.all_true)
+
+ (func $e2 (param v128 v128) (result i32)
+ local.get 0
+ local.get 1
+ i8x16.ne
+ v128.any_true
+ i32.eqz)
+
+ (func $band (param v128 v128) (result i32)
+ local.get 0
+ local.get 1
+ v128.and
+ v128.any_true)
+
+ (func $band_not (param v128 v128) (result i32)
+ local.get 0
+ local.get 1
+ v128.not
+ v128.and
+ v128.any_true)
+)
+;; wasm[0]::function[0]::e1:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; vpxor %xmm1, %xmm0, %xmm5
+;; vptest %xmm5, %xmm5
+;; sete %r8b
+;; movzbl %r8b, %eax
+;; movq %rbp, %rsp
+;; popq %rbp
+;; retq
+;;
+;; wasm[0]::function[1]::e2:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; vpxor %xmm1, %xmm0, %xmm5
+;; vptest %xmm5, %xmm5
+;; sete %r8b
+;; movzbl %r8b, %eax
+;; movq %rbp, %rsp
+;; popq %rbp
+;; retq
+;;
+;; wasm[0]::function[2]::band:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; vptest %xmm1, %xmm0
+;; setne %sil
+;; movzbl %sil, %eax
+;; movq %rbp, %rsp
+;; popq %rbp
+;; retq
+;;
+;; wasm[0]::function[3]::band_not:
+;; pushq %rbp
+;; movq %rsp, %rbp
+;; vptest %xmm0, %xmm1
+;; setae %sil
+;; movzbl %sil, %eax
+;; movq %rbp, %rsp
+;; popq %rbp
+;; retq
diff --git a/tests/misc_testsuite/simd/v128-equal.wast b/tests/misc_testsuite/simd/v128-equal.wast
new file mode 100644
index 0000000000..411d6e6141
--- /dev/null
+++ b/tests/misc_testsuite/simd/v128-equal.wast
@@ -0,0 +1,40 @@
+;;! simd = true
+
+(module
+ (func (export "e1") (param v128 v128) (result i32)
+ local.get 0
+ local.get 1
+ i8x16.eq
+ i8x16.all_true)
+
+ (func (export "e2") (param v128 v128) (result i32)
+ local.get 0
+ local.get 1
+ i8x16.ne
+ v128.any_true
+ i32.eqz)
+
+ (func (export "band") (param v128 v128) (result i32)
+ local.get 0
+ local.get 1
+ v128.and
+ v128.any_true)
+
+ (func (export "band_not") (param v128 v128) (result i32)
+ local.get 0
+ local.get 1
+ v128.not
+ v128.and
+ v128.any_true)
+)
+
+(assert_return (invoke "e1" (v128.const i32x4 0 0 0 0) (v128.const i32x4 0 0 0 0)) (i32.const 1))
+(assert_return (invoke "e1" (v128.const i32x4 0 0 0 0) (v128.const i32x4 0 0 0 1)) (i32.const 0))
+
+(assert_return (invoke "e2" (v128.const i32x4 0 0 0 0) (v128.const i32x4 0 0 0 0)) (i32.const 1))
+(assert_return (invoke "e2" (v128.const i32x4 0 0 0 0) (v128.const i32x4 0 0 0 1)) (i32.const 0))
+
+(assert_return (invoke "band" (v128.const i32x4 1 0 0 0) (v128.const i32x4 2 0 0 0)) (i32.const 0))
+(assert_return (invoke "band" (v128.const i32x4 1 0 0 0) (v128.const i32x4 1 0 0 0)) (i32.const 1))
+(assert_return (invoke "band_not" (v128.const i32x4 1 0 0 0) (v128.const i32x4 1 0 0 0)) (i32.const 0))
+(assert_return (invoke "band_not" (v128.const i32x4 1 0 0 0) (v128.const i32x4 0 0 0 0)) (i32.const 1))