Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Fix deterministic behavior of `i32x4.relaxed_dot_i8x16_i7x16_add_s` (#13702)

Alex Crichtone37e3d8ca1c4
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 44829926d9..ffffd57f95 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -390,20 +390,25 @@
(rule -1 (lower (iadd_pairwise ty x y))
(addp x y (vector_size ty)))
-;; With FEAT_DotProd, fold the `swiden`/`imul`/`iadd_pairwise`/`iadd` tree that
+;; With FEAT_DotProd, fold the exact-i32 dot-product tree that
;; `i32x4.relaxed_dot_i8x16_i7x16_add_s` decomposes into (there is no dot CLIF
-;; opcode) back into a single `sdot`. This is bit-identical to the otherwise
-;; emitted smull/saddlp fallback over the in-i7 input range the op guarantees.
+;; opcode) back into a single `sdot`. `sdot` accumulates the four signed `i8 *
+;; i8` products of each 32-bit lane directly in i32, which is exactly what this
+;; tree computes: `lo`/`hi` are the i16 products of the low/high input halves,
+;; each widened to i32 and reduced pairwise entirely in i32 (no narrowing), then
+;; added to the accumulator `c`.
;; Priority 8 stays above the scalar `iadd` rules whose opaque
;; `ty_int_ref_scalar_64` guard the overlap checker can't prove disjoint here.
(rule 8 (lower (and (use_dotprod)
(has_type $I32X4
(iadd _
(iadd_pairwise _
- (swiden_low _ dot @ (iadd_pairwise _
- (imul _ (swiden_low _ a) (swiden_low _ b))
- (imul _ (swiden_high _ a) (swiden_high _ b))))
- (swiden_high _ dot))
+ (iadd_pairwise _
+ (swiden_low _ lo @ (imul _ (swiden_low _ a) (swiden_low _ b)))
+ (swiden_high _ lo))
+ (iadd_pairwise _
+ (swiden_low _ hi @ (imul _ (swiden_high _ a) (swiden_high _ b)))
+ (swiden_high _ hi)))
c))))
(sdot c a b))
diff --git a/cranelift/filetests/filetests/isa/aarch64/simd-sdot.clif b/cranelift/filetests/filetests/isa/aarch64/simd-sdot.clif
index 10fff0680c..c52023874c 100644
--- a/cranelift/filetests/filetests/isa/aarch64/simd-sdot.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/simd-sdot.clif
@@ -2,7 +2,8 @@ test compile precise-output
set unwind_info=false
target aarch64 has_dotprod
-;; Tests the aarch64 `sdot` lowering: the i8 dot-product tree contracts to one `sdot`.
+;; Tests the aarch64 `sdot` lowering: the exact-i32 dot-product tree that
+;; `i32x4.relaxed_dot_i8x16_i7x16_add_s` decomposes into contracts to one `sdot`.
function %sdot_i8x16(i8x16, i8x16, i32x4) -> i32x4 {
block0(v0: i8x16, v1: i8x16, v2: i32x4):
v3 = swiden_low v0
@@ -11,12 +12,15 @@ block0(v0: i8x16, v1: i8x16, v2: i32x4):
v6 = swiden_high v0
v7 = swiden_high v1
v8 = imul v6, v7
- v9 = iadd_pairwise v5, v8
- v10 = swiden_low v9
- v11 = swiden_high v9
- v12 = iadd_pairwise v10, v11
- v13 = iadd v12, v2
- return v13
+ v9 = swiden_low v5
+ v10 = swiden_high v5
+ v11 = iadd_pairwise v9, v10
+ v12 = swiden_low v8
+ v13 = swiden_high v8
+ v14 = iadd_pairwise v12, v13
+ v15 = iadd_pairwise v11, v14
+ v16 = iadd v15, v2
+ return v16
}
; VCode:
diff --git a/cranelift/filetests/filetests/runtests/simd-relaxed-dot-i16x8.clif b/cranelift/filetests/filetests/runtests/simd-relaxed-dot-i16x8.clif
new file mode 100644
index 0000000000..e39fe0065f
--- /dev/null
+++ b/cranelift/filetests/filetests/runtests/simd-relaxed-dot-i16x8.clif
@@ -0,0 +1,21 @@
+test interpret
+test run
+target aarch64
+target aarch64 has_dotprod
+target x86_64 has_sse3 has_ssse3 has_sse41
+target s390x
+
+;; Test the wasm `i16x8.relaxed_dot_i8x16_i7x16_s` deterministic lowering
+function %relaxed_dot_i16x8(i8x16, i8x16) -> i16x8 {
+block0(v0: i8x16, v1: i8x16):
+ v2 = swiden_low v0
+ v3 = swiden_low v1
+ v4 = imul v2, v3
+ v5 = swiden_high v0
+ v6 = swiden_high v1
+ v7 = imul v5, v6
+ v8 = iadd_pairwise v4, v7
+ return v8
+}
+; run: %relaxed_dot_i16x8([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1]) == [3 7 11 15 19 23 27 31]
+; run: %relaxed_dot_i16x8([-128 -128 -128 -128 127 127 127 127 0 0 0 0 0 0 0 0], [-128 -128 -128 -128 127 127 127 127 0 0 0 0 0 0 0 0]) == [-32768 -32768 32258 32258 0 0 0 0]
diff --git a/cranelift/filetests/filetests/runtests/simd-sdot.clif b/cranelift/filetests/filetests/runtests/simd-sdot.clif
index fb73ac2e31..98de6e1aa9 100644
--- a/cranelift/filetests/filetests/runtests/simd-sdot.clif
+++ b/cranelift/filetests/filetests/runtests/simd-sdot.clif
@@ -5,7 +5,8 @@ target aarch64 has_dotprod
target x86_64 has_sse3 has_ssse3 has_sse41
target s390x
-;; Tests the aarch64 `sdot` lowering of the i8 4-way dot product.
+;; Tests the deterministic lowering of the wasm relaxed-simd instruction
+;; `i32x4.relaxed_dot_i8x16_i7x16_add_s`
function %sdot_i8x16(i8x16, i8x16, i32x4) -> i32x4 {
block0(v0: i8x16, v1: i8x16, v2: i32x4):
v3 = swiden_low v0
@@ -14,14 +15,18 @@ block0(v0: i8x16, v1: i8x16, v2: i32x4):
v6 = swiden_high v0
v7 = swiden_high v1
v8 = imul v6, v7
- v9 = iadd_pairwise v5, v8
- v10 = swiden_low v9
- v11 = swiden_high v9
- v12 = iadd_pairwise v10, v11
- v13 = iadd v12, v2
- return v13
+ v9 = swiden_low v5
+ v10 = swiden_high v5
+ v11 = iadd_pairwise v9, v10
+ v12 = swiden_low v8
+ v13 = swiden_high v8
+ v14 = iadd_pairwise v12, v13
+ v15 = iadd_pairwise v11, v14
+ v16 = iadd v15, v2
+ return v16
}
-; each i32x4 lane i = c[i] + sum_{j=4i..4i+3} a[j]*b[j]
+
; run: %sdot_i8x16([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [0 0 0 0]) == [10 26 42 58]
; run: %sdot_i8x16([-1 -2 -3 -4 1 2 3 4 5 5 5 5 -5 -5 -5 -5], [2 2 2 2 3 3 3 3 1 1 1 1 4 4 4 4], [100 200 300 400]) == [80 230 320 320]
; run: %sdot_i8x16([127 127 127 127 -128 -128 -128 -128 100 -100 100 -100 0 0 0 0], [127 127 127 127 127 127 127 127 63 63 63 63 1 2 3 4], [0 0 0 0]) == [64516 -65024 0 0]
+; run: %sdot_i8x16([-128 -128 0 0 -128 -128 -128 -128 127 127 127 127 0 0 0 0], [-128 -128 0 0 -128 -128 -128 -128 127 127 127 127 0 0 0 0], [0 0 0 0]) == [32768 65536 64516 0]
diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs
index f62943cc5b..1035afabfb 100644
--- a/crates/cranelift/src/translate/code_translator.rs
+++ b/crates/cranelift/src/translate/code_translator.rs
@@ -2605,24 +2605,32 @@ pub fn translate_operator(
Operator::I32x4RelaxedDotI8x16I7x16AddS => {
let c = pop1_with_bitcast(environ, I32X4, builder);
let (a, b) = pop2_with_bitcast(environ, I8X16, builder);
- let dot =
+ let dot32 =
if environ.relaxed_simd_deterministic() || !environ.use_x86_pmaddubsw_for_dot() {
- // Deterministic semantics are to treat both operands as
- // signed integers and perform the dot product.
+ // Deterministic semantics, or the lowering on any non-x86
+ // target. Each i32 lane is the *exact* sum of the four
+ // signed `i8 * i8` products covering it.
let alo = builder.ins().swiden_low(a);
let blo = builder.ins().swiden_low(b);
let lo = builder.ins().imul(alo, blo);
let ahi = builder.ins().swiden_high(a);
let bhi = builder.ins().swiden_high(b);
let hi = builder.ins().imul(ahi, bhi);
- builder.ins().iadd_pairwise(lo, hi)
+ let lo_lo = builder.ins().swiden_low(lo);
+ let lo_hi = builder.ins().swiden_high(lo);
+ let hi_lo = builder.ins().swiden_low(hi);
+ let hi_hi = builder.ins().swiden_high(hi);
+ let s_lo = builder.ins().iadd_pairwise(lo_lo, lo_hi);
+ let s_hi = builder.ins().iadd_pairwise(hi_lo, hi_hi);
+ builder.ins().iadd_pairwise(s_lo, s_hi)
} else {
- builder.ins().x86_pmaddubsw(a, b)
+ let dot = builder.ins().x86_pmaddubsw(a, b);
+ let dotlo = builder.ins().swiden_low(dot);
+ let dothi = builder.ins().swiden_high(dot);
+ builder.ins().iadd_pairwise(dotlo, dothi)
};
- let dotlo = builder.ins().swiden_low(dot);
- let dothi = builder.ins().swiden_high(dot);
- let dot32 = builder.ins().iadd_pairwise(dotlo, dothi);
- environ.stacks.push1(builder.ins().iadd(dot32, c));
+ let result = builder.ins().iadd(dot32, c);
+ environ.stacks.push1(result);
}
Operator::BrOnNull { relative_depth } => {
diff --git a/tests/disas/aarch64-relaxed-simd.wat b/tests/disas/aarch64-relaxed-simd.wat
index d63e668f70..52369596b1 100644
--- a/tests/disas/aarch64-relaxed-simd.wat
+++ b/tests/disas/aarch64-relaxed-simd.wat
@@ -79,9 +79,10 @@
;; stp x29, x30, [sp, #-0x10]!
;; mov x29, sp
;; smull v16.8h, v0.8b, v1.8b
-;; smull2 v17.8h, v0.16b, v1.16b
-;; addp v16.8h, v16.8h, v17.8h
;; saddlp v16.4s, v16.8h
+;; smull2 v17.8h, v0.16b, v1.16b
+;; saddlp v17.4s, v17.8h
+;; addp v16.4s, v16.4s, v17.4s
;; add v0.4s, v16.4s, v2.4s
;; ldp x29, x30, [sp], #0x10
;; ret
diff --git a/tests/disas/x64-relaxed-simd-deterministic.wat b/tests/disas/x64-relaxed-simd-deterministic.wat
index 62456ff76d..9962cba19f 100644
--- a/tests/disas/x64-relaxed-simd-deterministic.wat
+++ b/tests/disas/x64-relaxed-simd-deterministic.wat
@@ -124,21 +124,18 @@
;; vpmovsxbw %xmm0, %xmm3
;; vpmovsxbw %xmm1, %xmm4
;; vpmullw %xmm4, %xmm3, %xmm3
+;; vpmaddwd 0x36(%rip), %xmm3, %xmm3
;; vpalignr $8, %xmm0, %xmm0, %xmm0
;; vpmovsxbw %xmm0, %xmm0
;; vpalignr $8, %xmm1, %xmm1, %xmm1
;; vpmovsxbw %xmm1, %xmm1
;; vpmullw %xmm1, %xmm0, %xmm0
-;; vphaddw %xmm0, %xmm3, %xmm0
-;; vpmaddwd 0x17(%rip), %xmm0, %xmm0
+;; vpmaddwd 0x24(%rip), %xmm0, %xmm0
+;; vphaddd %xmm0, %xmm3, %xmm0
;; vpaddd %xmm2, %xmm0, %xmm0
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 1a2: addb %al, (%rax)
-;; 1a4: addb %al, (%rax)
-;; 1a6: addb %al, (%rax)
-;; 1a8: addb %al, (%rax)
;; 1aa: addb %al, (%rax)
;; 1ac: addb %al, (%rax)
;; 1ae: addb %al, (%rax)
@@ -150,3 +147,11 @@
;; 1ba: addl %eax, (%rax)
;; 1bc: addl %eax, (%rax)
;; 1be: addl %eax, (%rax)
+;; 1c0: addl %eax, (%rax)
+;; 1c2: addl %eax, (%rax)
+;; 1c4: addl %eax, (%rax)
+;; 1c6: addl %eax, (%rax)
+;; 1c8: addl %eax, (%rax)
+;; 1ca: addl %eax, (%rax)
+;; 1cc: addl %eax, (%rax)
+;; 1ce: addl %eax, (%rax)