Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

x64: Lower single-source `shuffle` of distinct operands to one `pshufb` (#14849)

Evan Carrolle6487ad97303
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 3e4b325335..10c885289e 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -4843,17 +4843,33 @@
(if-let true (has_ssse3))
(x64_pshufb a (const_to_xmm_mem_128 (shuffle_0_31_mask mask))))
+;; If the mask only selects lanes from one operand then only that operand is
+;; shuffled, even when `lhs` and `rhs` are different values. The rules below
+;; would otherwise read the unused operand too, either as a second `pshufb` plus
+;; `por` or as an input of `vpermi2b`, which creates a false dependency on it.
+(rule 2 (lower (shuffle _ a _ (vec_mask_from_immediate (shuffle_lhs_only mask))))
+ (if-let true (has_ssse3))
+ (x64_pshufb a (const_to_xmm_mem_128 (shuffle_0_15_mask mask))))
+(rule 1 (lower (shuffle _ _ b (vec_mask_from_immediate (shuffle_rhs_only mask))))
+ (if-let true (has_ssse3))
+ (x64_pshufb b (const_to_xmm_mem_128 (shuffle_16_31_mask mask))))
+
+(decl shuffle_lhs_only (VecMask) VecMask)
+(extern extractor shuffle_lhs_only shuffle_lhs_only)
+(decl shuffle_rhs_only (VecMask) VecMask)
+(extern extractor shuffle_rhs_only shuffle_rhs_only)
+
;; For the case where the shuffle mask contains out-of-bounds values (values
;; greater than 31) we must mask off those resulting values in the result of
;; `vpermi2b`.
-(rule 2 (lower (shuffle _ a b (vec_mask_from_immediate (perm_from_mask_with_zeros mask zeros))))
+(rule 0 (lower (shuffle _ a b (vec_mask_from_immediate (perm_from_mask_with_zeros mask zeros))))
(if-let true (has_avx512vl))
(if-let true (has_avx512vbmi))
(x64_andps (x64_vpermi2b (x64_xmm_load_const $I8X16 mask) a b) (const_to_xmm_mem_128 zeros)))
;; However, if the shuffle mask contains no out-of-bounds values, we can use
;; `vpermi2b` without any masking.
-(rule 1 (lower (shuffle _ a b (vec_mask_from_immediate mask)))
+(rule -1 (lower (shuffle _ a b (vec_mask_from_immediate mask)))
(if-let true (has_avx512vl))
(if-let true (has_avx512vbmi))
(x64_vpermi2b (x64_xmm_load_const $I8X16 (perm_from_mask mask)) a b))
@@ -4861,7 +4877,7 @@
;; If `lhs` and `rhs` are different, we must shuffle each separately and then OR
;; them together. This is necessary due to PSHUFB semantics. As in the case
;; above, we build the `constructed_mask` for each case statically.
-(rule (lower (shuffle _ a b (vec_mask_from_immediate mask)))
+(rule -2 (lower (shuffle _ a b (vec_mask_from_immediate mask)))
(x64_por
(lower_pshufb a (const_to_xmm_mem_128 (shuffle_0_15_mask mask)))
(lower_pshufb b (const_to_xmm_mem_128 (shuffle_16_31_mask mask)))))
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index c5b33e1623..f0a43025f1 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -1508,6 +1508,24 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
.use_constant(VCodeConstantData::Generated(mask))
}
+ #[inline]
+ fn shuffle_lhs_only(&mut self, mask: &VecMask) -> Option<VecMask> {
+ if mask.iter().all(|&b| b < 16) {
+ Some(mask.clone())
+ } else {
+ None
+ }
+ }
+
+ #[inline]
+ fn shuffle_rhs_only(&mut self, mask: &VecMask) -> Option<VecMask> {
+ if mask.iter().all(|&b| (16..32).contains(&b)) {
+ Some(mask.clone())
+ } else {
+ None
+ }
+ }
+
#[inline]
fn perm_from_mask_with_zeros(
&mut self,
diff --git a/cranelift/filetests/filetests/isa/x64/shuffle-avx512-single-source.clif b/cranelift/filetests/filetests/isa/x64/shuffle-avx512-single-source.clif
new file mode 100644
index 0000000000..72de74dfcc
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/shuffle-avx512-single-source.clif
@@ -0,0 +1,75 @@
+test compile precise-output
+target x86_64 ssse3 has_avx512vl has_avx512vbmi
+
+function %shuffle_lhs_only(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle v0, v1, [0 4 1 5 2 6 3 7 8 12 9 13 10 14 11 15]
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; pshufb (%rip), %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; pshufb 0x13(%rip), %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rcx, %rax)
+; addl $0x7030602, %eax
+; orb %cl, (%rcx, %rcx)
+; orl $0xf0b0e0a, %eax
+
+function %shuffle_rhs_only(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle v0, v1, [16 20 17 21 18 22 19 23 24 28 25 29 26 30 27 31]
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqa %xmm1, %xmm0
+; pshufb (%rip), %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqa %xmm1, %xmm0
+; pshufb 0xf(%rip), %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rcx, %rax)
+; addl $0x7030602, %eax
+; orb %cl, (%rcx, %rcx)
+; orl $0xf0b0e0a, %eax
+
diff --git a/cranelift/filetests/filetests/isa/x64/shuffle.clif b/cranelift/filetests/filetests/isa/x64/shuffle.clif
index 75465db88d..81eb8c8cc7 100644
--- a/cranelift/filetests/filetests/isa/x64/shuffle.clif
+++ b/cranelift/filetests/filetests/isa/x64/shuffle.clif
@@ -808,3 +808,76 @@ block0(v0: i16x8, v1: i16x8):
; popq %rbp
; retq
+
+function %shuffle_lhs_only(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle v0, v1, [0 4 1 5 2 6 3 7 8 12 9 13 10 14 11 15]
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; pshufb (%rip), %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; pshufb 0x13(%rip), %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rcx, %rax)
+; addl $0x7030602, %eax
+; orb %cl, (%rcx, %rcx)
+; orl $0xf0b0e0a, %eax
+
+function %shuffle_rhs_only(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle v0, v1, [16 20 17 21 18 22 19 23 24 28 25 29 26 30 27 31]
+ return v2
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; movdqa %xmm1, %xmm0
+; pshufb (%rip), %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; movdqa %xmm1, %xmm0
+; pshufb 0xf(%rip), %xmm0
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rax)
+; addb %al, (%rcx, %rax)
+; addl $0x7030602, %eax
+; orb %cl, (%rcx, %rcx)
+; orl $0xf0b0e0a, %eax
+
diff --git a/cranelift/filetests/filetests/runtests/simd-shuffle.clif b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
index 72a48857ba..393961da52 100644
--- a/cranelift/filetests/filetests/runtests/simd-shuffle.clif
+++ b/cranelift/filetests/filetests/runtests/simd-shuffle.clif
@@ -572,3 +572,17 @@ block0(v0: i16x8, v1: i16x8):
return v5
}
; run: %pblendw_0b10011001([1 2 3 4 5 6 7 8], [9 10 11 12 13 14 15 16]) == [9 2 3 12 13 6 7 16]
+
+function %shuffle_lhs_only(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle v0, v1, [0 4 1 5 2 6 3 7 8 12 9 13 10 14 11 15]
+ return v2
+}
+; run: %shuffle_lhs_only([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32]) == [1 5 2 6 3 7 4 8 9 13 10 14 11 15 12 16]
+
+function %shuffle_rhs_only(i8x16, i8x16) -> i8x16 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = shuffle v0, v1, [16 20 17 21 18 22 19 23 24 28 25 29 26 30 27 31]
+ return v2
+}
+; run: %shuffle_rhs_only([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32]) == [17 21 18 22 19 23 20 24 25 29 26 30 27 31 28 32]