A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
x64: Lower single-source `shuffle` of distinct operands to one `pshufb` (#14849)
Evan Carrolle6487ad97303
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex 3e4b325335..10c885289e 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -4843,17 +4843,33 @@(if-let true (has_ssse3))(x64_pshufb a (const_to_xmm_mem_128 (shuffle_0_31_mask mask))))+;; If the mask only selects lanes from one operand then only that operand is+;; shuffled, even when `lhs` and `rhs` are different values. The rules below+;; would otherwise read the unused operand too, either as a second `pshufb` plus+;; `por` or as an input of `vpermi2b`, which creates a false dependency on it.+(rule 2 (lower (shuffle _ a _ (vec_mask_from_immediate (shuffle_lhs_only mask))))+ (if-let true (has_ssse3))+ (x64_pshufb a (const_to_xmm_mem_128 (shuffle_0_15_mask mask))))+(rule 1 (lower (shuffle _ _ b (vec_mask_from_immediate (shuffle_rhs_only mask))))+ (if-let true (has_ssse3))+ (x64_pshufb b (const_to_xmm_mem_128 (shuffle_16_31_mask mask))))++(decl shuffle_lhs_only (VecMask) VecMask)+(extern extractor shuffle_lhs_only shuffle_lhs_only)+(decl shuffle_rhs_only (VecMask) VecMask)+(extern extractor shuffle_rhs_only shuffle_rhs_only)+;; For the case where the shuffle mask contains out-of-bounds values (values;; greater than 31) we must mask off those resulting values in the result of;; `vpermi2b`.-(rule 2 (lower (shuffle _ a b (vec_mask_from_immediate (perm_from_mask_with_zeros mask zeros))))+(rule 0 (lower (shuffle _ a b (vec_mask_from_immediate (perm_from_mask_with_zeros mask zeros))))(if-let true (has_avx512vl))(if-let true (has_avx512vbmi))(x64_andps (x64_vpermi2b (x64_xmm_load_const $I8X16 mask) a b) (const_to_xmm_mem_128 zeros)));; However, if the shuffle mask contains no out-of-bounds values, we can use;; `vpermi2b` without any masking.-(rule 1 (lower (shuffle _ a b (vec_mask_from_immediate mask)))+(rule -1 (lower (shuffle _ a b (vec_mask_from_immediate mask)))(if-let true (has_avx512vl))(if-let true (has_avx512vbmi))(x64_vpermi2b (x64_xmm_load_const $I8X16 (perm_from_mask mask)) a b))@@ -4861,7 +4877,7 @@;; If `lhs` and `rhs` are different, we must shuffle each separately and then OR;; them together. This is necessary due to PSHUFB semantics. As in the case;; above, we build the `constructed_mask` for each case statically.-(rule (lower (shuffle _ a b (vec_mask_from_immediate mask)))+(rule -2 (lower (shuffle _ a b (vec_mask_from_immediate mask)))(x64_por(lower_pshufb a (const_to_xmm_mem_128 (shuffle_0_15_mask mask)))(lower_pshufb b (const_to_xmm_mem_128 (shuffle_16_31_mask mask)))))diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rsindex c5b33e1623..f0a43025f1 100644--- a/cranelift/codegen/src/isa/x64/lower/isle.rs+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs@@ -1508,6 +1508,24 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {.use_constant(VCodeConstantData::Generated(mask))}+ #[inline]+ fn shuffle_lhs_only(&mut self, mask: &VecMask) -> Option<VecMask> {+ if mask.iter().all(|&b| b < 16) {+ Some(mask.clone())+ } else {+ None+ }+ }++ #[inline]+ fn shuffle_rhs_only(&mut self, mask: &VecMask) -> Option<VecMask> {+ if mask.iter().all(|&b| (16..32).contains(&b)) {+ Some(mask.clone())+ } else {+ None+ }+ }+#[inline]fn perm_from_mask_with_zeros(&mut self,diff --git a/cranelift/filetests/filetests/isa/x64/shuffle-avx512-single-source.clif b/cranelift/filetests/filetests/isa/x64/shuffle-avx512-single-source.clifnew file mode 100644index 0000000000..72de74dfcc--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/shuffle-avx512-single-source.clif@@ -0,0 +1,75 @@+test compile precise-output+target x86_64 ssse3 has_avx512vl has_avx512vbmi++function %shuffle_lhs_only(i8x16, i8x16) -> i8x16 {+block0(v0: i8x16, v1: i8x16):+ v2 = shuffle v0, v1, [0 4 1 5 2 6 3 7 8 12 9 13 10 14 11 15]+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; pshufb (%rip), %xmm0+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; pshufb 0x13(%rip), %xmm0+; movq %rbp, %rsp+; popq %rbp+; retq+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rcx, %rax)+; addl $0x7030602, %eax+; orb %cl, (%rcx, %rcx)+; orl $0xf0b0e0a, %eax++function %shuffle_rhs_only(i8x16, i8x16) -> i8x16 {+block0(v0: i8x16, v1: i8x16):+ v2 = shuffle v0, v1, [16 20 17 21 18 22 19 23 24 28 25 29 26 30 27 31]+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movdqa %xmm1, %xmm0+; pshufb (%rip), %xmm0+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movdqa %xmm1, %xmm0+; pshufb 0xf(%rip), %xmm0+; movq %rbp, %rsp+; popq %rbp+; retq+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rcx, %rax)+; addl $0x7030602, %eax+; orb %cl, (%rcx, %rcx)+; orl $0xf0b0e0a, %eax+diff --git a/cranelift/filetests/filetests/isa/x64/shuffle.clif b/cranelift/filetests/filetests/isa/x64/shuffle.clifindex 75465db88d..81eb8c8cc7 100644--- a/cranelift/filetests/filetests/isa/x64/shuffle.clif+++ b/cranelift/filetests/filetests/isa/x64/shuffle.clif@@ -808,3 +808,76 @@ block0(v0: i16x8, v1: i16x8):; popq %rbp; retq++function %shuffle_lhs_only(i8x16, i8x16) -> i8x16 {+block0(v0: i8x16, v1: i8x16):+ v2 = shuffle v0, v1, [0 4 1 5 2 6 3 7 8 12 9 13 10 14 11 15]+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; pshufb (%rip), %xmm0+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; pshufb 0x13(%rip), %xmm0+; movq %rbp, %rsp+; popq %rbp+; retq+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rcx, %rax)+; addl $0x7030602, %eax+; orb %cl, (%rcx, %rcx)+; orl $0xf0b0e0a, %eax++function %shuffle_rhs_only(i8x16, i8x16) -> i8x16 {+block0(v0: i8x16, v1: i8x16):+ v2 = shuffle v0, v1, [16 20 17 21 18 22 19 23 24 28 25 29 26 30 27 31]+ return v2+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; movdqa %xmm1, %xmm0+; pshufb (%rip), %xmm0+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; movdqa %xmm1, %xmm0+; pshufb 0xf(%rip), %xmm0+; movq %rbp, %rsp+; popq %rbp+; retq+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rax)+; addb %al, (%rcx, %rax)+; addl $0x7030602, %eax+; orb %cl, (%rcx, %rcx)+; orl $0xf0b0e0a, %eax+diff --git a/cranelift/filetests/filetests/runtests/simd-shuffle.clif b/cranelift/filetests/filetests/runtests/simd-shuffle.clifindex 72a48857ba..393961da52 100644--- a/cranelift/filetests/filetests/runtests/simd-shuffle.clif+++ b/cranelift/filetests/filetests/runtests/simd-shuffle.clif@@ -572,3 +572,17 @@ block0(v0: i16x8, v1: i16x8):return v5}; run: %pblendw_0b10011001([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [9 2 3 12 13 6 7 16]++function %shuffle_lhs_only(i8x16, i8x16) -> i8x16 {+block0(v0: i8x16, v1: i8x16):+ v2 = shuffle v0, v1, [0 4 1 5 2 6 3 7 8 12 9 13 10 14 11 15]+ return v2+}+; run: %shuffle_lhs_only([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32]) == [1 5 2 6 3 7 4 8 9 13 10 14 11 15 12 16]++function %shuffle_rhs_only(i8x16, i8x16) -> i8x16 {+block0(v0: i8x16, v1: i8x16):+ v2 = shuffle v0, v1, [16 20 17 21 18 22 19 23 24 28 25 29 26 30 27 31]+ return v2+}+; run: %shuffle_rhs_only([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32]) == [17 21 18 22 19 23 20 24 25 29 26 30 27 31 28 32]