A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
x64: Optimize some lowerings using `ptest` (#13462)
Alex Crichton1b9e6b35236e
diff --git a/cranelift/codegen/src/isa/pulley_shared/inst.isle b/cranelift/codegen/src/isa/pulley_shared/inst.isleindex 7324c34731..1543bbf73a 100644--- a/cranelift/codegen/src/isa/pulley_shared/inst.isle+++ b/cranelift/codegen/src/isa/pulley_shared/inst.isle@@ -430,9 +430,9 @@;; of the address against a particular bound.(decl pure partial wasm_oob_cond (Value Value) OobCond)(type OobCond (enum (All (a Value) (b u64))))-(rule (wasm_oob_cond wasm_addr (ugt wasm_addr (isub _ bound (u64_from_iconst n))))+(rule (wasm_oob_cond wasm_addr (ugt _ wasm_addr (isub _ bound (u64_from_iconst n))))(OobCond.All bound n))-(rule (wasm_oob_cond wasm_addr (ult (isub _ bound (u64_from_iconst n)) wasm_addr))+(rule (wasm_oob_cond wasm_addr (ult _ (isub _ bound (u64_from_iconst n)) wasm_addr))(OobCond.All bound n));;;; Newtypes for Different Register Classes ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex 1a9d4b5692..9bee238f0c 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -4968,22 +4968,47 @@(mask Gpr (x64_pmovmskb any_byte_zero)))(CondResult.CC (x64_cmpl_mi mask 0xffff) (CC.NZ))))++;; With `ptest` that'll set `ZF` based on whether `val & val` has any nonzero+;; bit, which is equivalent to saying that anything is true (or not equal to+;; zero).(rule 1 (is_vany_true val)(if-let true (has_sse41))(let ((val Xmm val))(CondResult.CC (x64_ptest val val) (CC.NZ))))+;; A `band` operation can get folded into `ptest` directly since it already does+;; the operation for us.+(rule 2 (is_vany_true (band (ty_vec128 _) a b))+ (if-let true (has_sse41))+ (CondResult.CC (x64_ptest a b) (CC.NZ)))++;; In addition to `band`, the `ptest` instruction also performs "andnot" and+;; puts the results in the `C` flag. This means that `(band a (bnot b))` can+;; also be folded in. Note that bitwise negation happens in the first operand.+(rule 3 (is_vany_true (band (ty_vec128 _) a (bnot _ b)))+ (if-let true (has_sse41))+ (CondResult.CC (x64_ptest b a) (CC.NB)))+(rule 4 (is_vany_true (band (ty_vec128 _) (bnot _ a) b))+ (if-let true (has_sse41))+ (CondResult.CC (x64_ptest a b) (CC.NB)))++;; Vector inequality can be a bit more efficient with `ptest`. When testing+;; this property of vectors the inputs are xor'd together and the result is+;; passed to `ptest`. The logical operation here is: is any lane of `a` not+;; equal to the corresponding lane in `b`? With `pxor` we can reduce that to+;; if any bit in the xor is set, and that's exactly that testing the `ZF` flag+;; after `ptest` will tell us.+(rule 2 (is_vany_true (ne (ty_vec128 ty) a b))+ (if-let true (has_sse41))+ (let ((tmp Xmm (x64_pxor a b)))+ (CondResult.CC (x64_ptest tmp tmp) (CC.NZ))))+;; Rules for `vall_true` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (vall_true _ val)) (lower_cond_bool (is_vall_true val)))(decl is_vall_true (Value) CondResult)-(rule 1 (is_vall_true val @ (value_type ty))- (if-let true (has_sse41))- (let ((src Xmm val)- (zeros Xmm (xmm_zero ty))- (cmp Xmm (x64_pcmpeq (vec_int_type ty) src zeros)))- (CondResult.CC (x64_ptest cmp cmp) (CC.Z))));; Perform an appropriately-sized lane-wise comparison with zero. If the;; result is all 0s then all of them are true because nothing was equal to@@ -4993,6 +5018,26 @@(mask Gpr (x64_pmovmskb lanes_with_zero)))(CondResult.CC (x64_testl_mr mask mask) (CC.Z))))+;; With `ptest` this can be a bit smaller. Compare `val` to an all-zeros mask,+;; then `ptest` that comparison with itself. If any lane in `val` is zero then+;; the result will have some 1s, otherwise it's all zeros.+(rule 1 (is_vall_true val @ (value_type ty))+ (if-let true (has_sse41))+ (let ((src Xmm val)+ (zeros Xmm (xmm_zero ty))+ (cmp Xmm (x64_pcmpeq (vec_int_type ty) src zeros)))+ (CondResult.CC (x64_ptest cmp cmp) (CC.Z))))++;; This is similar to `(is_vany_true (ne _ a b))` above. The logical operation+;; here is: is every lane of `a` equal to the corresponding lane of `b`. With+;; `a ^ b` this property is only true if every single bit in the output is 0.+;; That is exactly what `ptest` tells us, so look for the `ZF` flag on the+;; xor'd result.+(rule 2 (is_vall_true (eq (ty_vec128 ty) a b))+ (if-let true (has_sse41))+ (let ((tmp Xmm (x64_pxor a b)))+ (CondResult.CC (x64_ptest tmp tmp) (CC.Z))))+;; Rules for `vhigh_bits` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; The Intel specification allows using both 32-bit and 64-bit GPRs asdiff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isleindex 8200c814c9..baae79a3df 100644--- a/cranelift/codegen/src/prelude_lower.isle+++ b/cranelift/codegen/src/prelude_lower.isle@@ -341,35 +341,35 @@(decl uimm8 (u8) Imm64)(extern extractor uimm8 uimm8)-(decl eq (Value Value) Value)-(extractor (eq x y) (icmp (IntCC.Equal) x y))+(decl eq (Type Value Value) Value)+(extractor (eq ty x y) (icmp ty (IntCC.Equal) x y))-(decl ne (Value Value) Value)-(extractor (ne x y) (icmp (IntCC.NotEqual) x y))+(decl ne (Type Value Value) Value)+(extractor (ne ty x y) (icmp ty (IntCC.NotEqual) x y))-(decl ult (Value Value) Value)-(extractor (ult x y) (icmp _ (IntCC.UnsignedLessThan) x y))+(decl ult (Type Value Value) Value)+(extractor (ult ty x y) (icmp ty (IntCC.UnsignedLessThan) x y))-(decl ule (Value Value) Value)-(extractor (ule x y) (icmp (IntCC.UnsignedLessThanOrEqual) x y))+(decl ule (Type Value Value) Value)+(extractor (ule ty x y) (icmp ty (IntCC.UnsignedLessThanOrEqual) x y))-(decl ugt (Value Value) Value)-(extractor (ugt x y) (icmp _ (IntCC.UnsignedGreaterThan) x y))+(decl ugt (Type Value Value) Value)+(extractor (ugt ty x y) (icmp ty (IntCC.UnsignedGreaterThan) x y))-(decl uge (Value Value) Value)-(extractor (uge x y) (icmp (IntCC.UnsignedGreaterThanOrEqual) x y))+(decl uge (Type Value Value) Value)+(extractor (uge ty x y) (icmp ty (IntCC.UnsignedGreaterThanOrEqual) x y))-(decl slt (Value Value) Value)-(extractor (slt x y) (icmp (IntCC.SignedLessThan) x y))+(decl slt (Type Value Value) Value)+(extractor (slt ty x y) (icmp ty (IntCC.SignedLessThan) x y))-(decl sle (Value Value) Value)-(extractor (sle x y) (icmp (IntCC.SignedLessThanOrEqual) x y))+(decl sle (Type Value Value) Value)+(extractor (sle ty x y) (icmp ty (IntCC.SignedLessThanOrEqual) x y))-(decl sgt (Value Value) Value)-(extractor (sgt x y) (icmp (IntCC.SignedGreaterThan) x y))+(decl sgt (Type Value Value) Value)+(extractor (sgt ty x y) (icmp ty (IntCC.SignedGreaterThan) x y))-(decl sge (Value Value) Value)-(extractor (sge x y) (icmp (IntCC.SignedGreaterThanOrEqual) x y))+(decl sge (Type Value Value) Value)+(extractor (sge ty x y) (icmp ty (IntCC.SignedGreaterThanOrEqual) x y));; Get the MachLabel for a particular CLIF block's indexed exception-handling successor.(decl block_exn_successor_label (Block u64) MachLabel)diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex 9698fd4921..03c340ea29 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -655,6 +655,7 @@ impl WastTest {"misc_testsuite/winch/issue-10331.wast","misc_testsuite/winch/replace_lane.wast","misc_testsuite/simd/riscv64-replicated-imm5-works.wast",+ "misc_testsuite/simd/v128-equal.wast","spec_testsuite/simd_align.wast","spec_testsuite/simd_boolean.wast","spec_testsuite/simd_conversions.wast",diff --git a/tests/disas/x64-vptest.wat b/tests/disas/x64-vptest.watnew file mode 100644index 0000000000..ffd6cd020e--- /dev/null+++ b/tests/disas/x64-vptest.wat@@ -0,0 +1,72 @@+;;! target = 'x86_64'+;;! test = 'compile'+;;! flags = '-Ccranelift-has-sse41 -Ccranelift-has-avx'++(module+ (func $e1 (param v128 v128) (result i32)+ local.get 0+ local.get 1+ i8x16.eq+ i8x16.all_true)++ (func $e2 (param v128 v128) (result i32)+ local.get 0+ local.get 1+ i8x16.ne+ v128.any_true+ i32.eqz)++ (func $band (param v128 v128) (result i32)+ local.get 0+ local.get 1+ v128.and+ v128.any_true)++ (func $band_not (param v128 v128) (result i32)+ local.get 0+ local.get 1+ v128.not+ v128.and+ v128.any_true)+)+;; wasm[0]::function[0]::e1:+;; pushq %rbp+;; movq %rsp, %rbp+;; vpxor %xmm1, %xmm0, %xmm5+;; vptest %xmm5, %xmm5+;; sete %r8b+;; movzbl %r8b, %eax+;; movq %rbp, %rsp+;; popq %rbp+;; retq+;;+;; wasm[0]::function[1]::e2:+;; pushq %rbp+;; movq %rsp, %rbp+;; vpxor %xmm1, %xmm0, %xmm5+;; vptest %xmm5, %xmm5+;; sete %r8b+;; movzbl %r8b, %eax+;; movq %rbp, %rsp+;; popq %rbp+;; retq+;;+;; wasm[0]::function[2]::band:+;; pushq %rbp+;; movq %rsp, %rbp+;; vptest %xmm1, %xmm0+;; setne %sil+;; movzbl %sil, %eax+;; movq %rbp, %rsp+;; popq %rbp+;; retq+;;+;; wasm[0]::function[3]::band_not:+;; pushq %rbp+;; movq %rsp, %rbp+;; vptest %xmm0, %xmm1+;; setae %sil+;; movzbl %sil, %eax+;; movq %rbp, %rsp+;; popq %rbp+;; retqdiff --git a/tests/misc_testsuite/simd/v128-equal.wast b/tests/misc_testsuite/simd/v128-equal.wastnew file mode 100644index 0000000000..411d6e6141--- /dev/null+++ b/tests/misc_testsuite/simd/v128-equal.wast@@ -0,0 +1,40 @@+;;! simd = true++(module+ (func (export "e1") (param v128 v128) (result i32)+ local.get 0+ local.get 1+ i8x16.eq+ i8x16.all_true)++ (func (export "e2") (param v128 v128) (result i32)+ local.get 0+ local.get 1+ i8x16.ne+ v128.any_true+ i32.eqz)++ (func (export "band") (param v128 v128) (result i32)+ local.get 0+ local.get 1+ v128.and+ v128.any_true)++ (func (export "band_not") (param v128 v128) (result i32)+ local.get 0+ local.get 1+ v128.not+ v128.and+ v128.any_true)+)++(assert_return (invoke "e1" (v128.const i32x4 0 0 0 0) (v128.const i32x4 0 0 0 0)) (i32.const 1))+(assert_return (invoke "e1" (v128.const i32x4 0 0 0 0) (v128.const i32x4 0 0 0 1)) (i32.const 0))++(assert_return (invoke "e2" (v128.const i32x4 0 0 0 0) (v128.const i32x4 0 0 0 0)) (i32.const 1))+(assert_return (invoke "e2" (v128.const i32x4 0 0 0 0) (v128.const i32x4 0 0 0 1)) (i32.const 0))++(assert_return (invoke "band" (v128.const i32x4 1 0 0 0) (v128.const i32x4 2 0 0 0)) (i32.const 0))+(assert_return (invoke "band" (v128.const i32x4 1 0 0 0) (v128.const i32x4 1 0 0 0)) (i32.const 1))+(assert_return (invoke "band_not" (v128.const i32x4 1 0 0 0) (v128.const i32x4 1 0 0 0)) (i32.const 0))+(assert_return (invoke "band_not" (v128.const i32x4 1 0 0 0) (v128.const i32x4 0 0 0 0)) (i32.const 1))