A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Fix deterministic behavior of `i32x4.relaxed_dot_i8x16_i7x16_add_s` (#13702)
Alex Crichtone37e3d8ca1c4
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex 44829926d9..ffffd57f95 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -390,20 +390,25 @@(rule -1 (lower (iadd_pairwise ty x y))(addp x y (vector_size ty)))-;; With FEAT_DotProd, fold the `swiden`/`imul`/`iadd_pairwise`/`iadd` tree that+;; With FEAT_DotProd, fold the exact-i32 dot-product tree that;; `i32x4.relaxed_dot_i8x16_i7x16_add_s` decomposes into (there is no dot CLIF-;; opcode) back into a single `sdot`. This is bit-identical to the otherwise-;; emitted smull/saddlp fallback over the in-i7 input range the op guarantees.+;; opcode) back into a single `sdot`. `sdot` accumulates the four signed `i8 *+;; i8` products of each 32-bit lane directly in i32, which is exactly what this+;; tree computes: `lo`/`hi` are the i16 products of the low/high input halves,+;; each widened to i32 and reduced pairwise entirely in i32 (no narrowing), then+;; added to the accumulator `c`.;; Priority 8 stays above the scalar `iadd` rules whose opaque;; `ty_int_ref_scalar_64` guard the overlap checker can't prove disjoint here.(rule 8 (lower (and (use_dotprod)(has_type $I32X4(iadd _(iadd_pairwise _- (swiden_low _ dot @ (iadd_pairwise _- (imul _ (swiden_low _ a) (swiden_low _ b))- (imul _ (swiden_high _ a) (swiden_high _ b))))- (swiden_high _ dot))+ (iadd_pairwise _+ (swiden_low _ lo @ (imul _ (swiden_low _ a) (swiden_low _ b)))+ (swiden_high _ lo))+ (iadd_pairwise _+ (swiden_low _ hi @ (imul _ (swiden_high _ a) (swiden_high _ b)))+ (swiden_high _ hi)))c))))(sdot c a b))diff --git a/cranelift/filetests/filetests/isa/aarch64/simd-sdot.clif b/cranelift/filetests/filetests/isa/aarch64/simd-sdot.clifindex 10fff0680c..c52023874c 100644--- a/cranelift/filetests/filetests/isa/aarch64/simd-sdot.clif+++ b/cranelift/filetests/filetests/isa/aarch64/simd-sdot.clif@@ -2,7 +2,8 @@ test compile precise-outputset unwind_info=falsetarget aarch64 has_dotprod-;; Tests the aarch64 `sdot` lowering: the i8 dot-product tree contracts to one `sdot`.+;; Tests the aarch64 `sdot` lowering: the exact-i32 dot-product tree that+;; `i32x4.relaxed_dot_i8x16_i7x16_add_s` decomposes into contracts to one `sdot`.function %sdot_i8x16(i8x16, i8x16, i32x4) -> i32x4 {block0(v0: i8x16, v1: i8x16, v2: i32x4):v3 = swiden_low v0@@ -11,12 +12,15 @@ block0(v0: i8x16, v1: i8x16, v2: i32x4):v6 = swiden_high v0v7 = swiden_high v1v8 = imul v6, v7- v9 = iadd_pairwise v5, v8- v10 = swiden_low v9- v11 = swiden_high v9- v12 = iadd_pairwise v10, v11- v13 = iadd v12, v2- return v13+ v9 = swiden_low v5+ v10 = swiden_high v5+ v11 = iadd_pairwise v9, v10+ v12 = swiden_low v8+ v13 = swiden_high v8+ v14 = iadd_pairwise v12, v13+ v15 = iadd_pairwise v11, v14+ v16 = iadd v15, v2+ return v16}; VCode:diff --git a/cranelift/filetests/filetests/runtests/simd-relaxed-dot-i16x8.clif b/cranelift/filetests/filetests/runtests/simd-relaxed-dot-i16x8.clifnew file mode 100644index 0000000000..e39fe0065f--- /dev/null+++ b/cranelift/filetests/filetests/runtests/simd-relaxed-dot-i16x8.clif@@ -0,0 +1,21 @@+test interpret+test run+target aarch64+target aarch64 has_dotprod+target x86_64 has_sse3 has_ssse3 has_sse41+target s390x++;; Test the wasm `i16x8.relaxed_dot_i8x16_i7x16_s` deterministic lowering+function %relaxed_dot_i16x8(i8x16, i8x16) -> i16x8 {+block0(v0: i8x16, v1: i8x16):+ v2 = swiden_low v0+ v3 = swiden_low v1+ v4 = imul v2, v3+ v5 = swiden_high v0+ v6 = swiden_high v1+ v7 = imul v5, v6+ v8 = iadd_pairwise v4, v7+ return v8+}+; run: %relaxed_dot_i16x8([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1]) == [3 7 11 15 19 23 27 31]+; run: %relaxed_dot_i16x8([-128 -128 -128 -128 127 127 127 127 0 0 0 0 0 0 0 0], [-128 -128 -128 -128 127 127 127 127 0 0 0 0 0 0 0 0]) == [-32768 -32768 32258 32258 0 0 0 0]diff --git a/cranelift/filetests/filetests/runtests/simd-sdot.clif b/cranelift/filetests/filetests/runtests/simd-sdot.clifindex fb73ac2e31..98de6e1aa9 100644--- a/cranelift/filetests/filetests/runtests/simd-sdot.clif+++ b/cranelift/filetests/filetests/runtests/simd-sdot.clif@@ -5,7 +5,8 @@ target aarch64 has_dotprodtarget x86_64 has_sse3 has_ssse3 has_sse41target s390x-;; Tests the aarch64 `sdot` lowering of the i8 4-way dot product.+;; Tests the deterministic lowering of the wasm relaxed-simd instruction+;; `i32x4.relaxed_dot_i8x16_i7x16_add_s`function %sdot_i8x16(i8x16, i8x16, i32x4) -> i32x4 {block0(v0: i8x16, v1: i8x16, v2: i32x4):v3 = swiden_low v0@@ -14,14 +15,18 @@ block0(v0: i8x16, v1: i8x16, v2: i32x4):v6 = swiden_high v0v7 = swiden_high v1v8 = imul v6, v7- v9 = iadd_pairwise v5, v8- v10 = swiden_low v9- v11 = swiden_high v9- v12 = iadd_pairwise v10, v11- v13 = iadd v12, v2- return v13+ v9 = swiden_low v5+ v10 = swiden_high v5+ v11 = iadd_pairwise v9, v10+ v12 = swiden_low v8+ v13 = swiden_high v8+ v14 = iadd_pairwise v12, v13+ v15 = iadd_pairwise v11, v14+ v16 = iadd v15, v2+ return v16}-; each i32x4 lane i = c[i] + sum_{j=4i..4i+3} a[j]*b[j]+; run: %sdot_i8x16([1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16], [1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1], [0 0 0 0]) == [10 26 42 58]; run: %sdot_i8x16([-1 -2 -3 -4 1 2 3 4 5 5 5 5 -5 -5 -5 -5], [2 2 2 2 3 3 3 3 1 1 1 1 4 4 4 4], [100 200 300 400]) == [80 230 320 320]; run: %sdot_i8x16([127 127 127 127 -128 -128 -128 -128 100 -100 100 -100 0 0 0 0], [127 127 127 127 127 127 127 127 63 63 63 63 1 2 3 4], [0 0 0 0]) == [64516 -65024 0 0]+; run: %sdot_i8x16([-128 -128 0 0 -128 -128 -128 -128 127 127 127 127 0 0 0 0], [-128 -128 0 0 -128 -128 -128 -128 127 127 127 127 0 0 0 0], [0 0 0 0]) == [32768 65536 64516 0]diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rsindex f62943cc5b..1035afabfb 100644--- a/crates/cranelift/src/translate/code_translator.rs+++ b/crates/cranelift/src/translate/code_translator.rs@@ -2605,24 +2605,32 @@ pub fn translate_operator(Operator::I32x4RelaxedDotI8x16I7x16AddS => {let c = pop1_with_bitcast(environ, I32X4, builder);let (a, b) = pop2_with_bitcast(environ, I8X16, builder);- let dot =+ let dot32 =if environ.relaxed_simd_deterministic() || !environ.use_x86_pmaddubsw_for_dot() {- // Deterministic semantics are to treat both operands as- // signed integers and perform the dot product.+ // Deterministic semantics, or the lowering on any non-x86+ // target. Each i32 lane is the *exact* sum of the four+ // signed `i8 * i8` products covering it.let alo = builder.ins().swiden_low(a);let blo = builder.ins().swiden_low(b);let lo = builder.ins().imul(alo, blo);let ahi = builder.ins().swiden_high(a);let bhi = builder.ins().swiden_high(b);let hi = builder.ins().imul(ahi, bhi);- builder.ins().iadd_pairwise(lo, hi)+ let lo_lo = builder.ins().swiden_low(lo);+ let lo_hi = builder.ins().swiden_high(lo);+ let hi_lo = builder.ins().swiden_low(hi);+ let hi_hi = builder.ins().swiden_high(hi);+ let s_lo = builder.ins().iadd_pairwise(lo_lo, lo_hi);+ let s_hi = builder.ins().iadd_pairwise(hi_lo, hi_hi);+ builder.ins().iadd_pairwise(s_lo, s_hi)} else {- builder.ins().x86_pmaddubsw(a, b)+ let dot = builder.ins().x86_pmaddubsw(a, b);+ let dotlo = builder.ins().swiden_low(dot);+ let dothi = builder.ins().swiden_high(dot);+ builder.ins().iadd_pairwise(dotlo, dothi)};- let dotlo = builder.ins().swiden_low(dot);- let dothi = builder.ins().swiden_high(dot);- let dot32 = builder.ins().iadd_pairwise(dotlo, dothi);- environ.stacks.push1(builder.ins().iadd(dot32, c));+ let result = builder.ins().iadd(dot32, c);+ environ.stacks.push1(result);}Operator::BrOnNull { relative_depth } => {diff --git a/tests/disas/aarch64-relaxed-simd.wat b/tests/disas/aarch64-relaxed-simd.watindex d63e668f70..52369596b1 100644--- a/tests/disas/aarch64-relaxed-simd.wat+++ b/tests/disas/aarch64-relaxed-simd.wat@@ -79,9 +79,10 @@;; stp x29, x30, [sp, #-0x10]!;; mov x29, sp;; smull v16.8h, v0.8b, v1.8b-;; smull2 v17.8h, v0.16b, v1.16b-;; addp v16.8h, v16.8h, v17.8h;; saddlp v16.4s, v16.8h+;; smull2 v17.8h, v0.16b, v1.16b+;; saddlp v17.4s, v17.8h+;; addp v16.4s, v16.4s, v17.4s;; add v0.4s, v16.4s, v2.4s;; ldp x29, x30, [sp], #0x10;; retdiff --git a/tests/disas/x64-relaxed-simd-deterministic.wat b/tests/disas/x64-relaxed-simd-deterministic.watindex 62456ff76d..9962cba19f 100644--- a/tests/disas/x64-relaxed-simd-deterministic.wat+++ b/tests/disas/x64-relaxed-simd-deterministic.wat@@ -124,21 +124,18 @@;; vpmovsxbw %xmm0, %xmm3;; vpmovsxbw %xmm1, %xmm4;; vpmullw %xmm4, %xmm3, %xmm3+;; vpmaddwd 0x36(%rip), %xmm3, %xmm3;; vpalignr $8, %xmm0, %xmm0, %xmm0;; vpmovsxbw %xmm0, %xmm0;; vpalignr $8, %xmm1, %xmm1, %xmm1;; vpmovsxbw %xmm1, %xmm1;; vpmullw %xmm1, %xmm0, %xmm0-;; vphaddw %xmm0, %xmm3, %xmm0-;; vpmaddwd 0x17(%rip), %xmm0, %xmm0+;; vpmaddwd 0x24(%rip), %xmm0, %xmm0+;; vphaddd %xmm0, %xmm3, %xmm0;; vpaddd %xmm2, %xmm0, %xmm0;; movq %rbp, %rsp;; popq %rbp;; retq-;; 1a2: addb %al, (%rax)-;; 1a4: addb %al, (%rax)-;; 1a6: addb %al, (%rax)-;; 1a8: addb %al, (%rax);; 1aa: addb %al, (%rax);; 1ac: addb %al, (%rax);; 1ae: addb %al, (%rax)@@ -150,3 +147,11 @@;; 1ba: addl %eax, (%rax);; 1bc: addl %eax, (%rax);; 1be: addl %eax, (%rax)+;; 1c0: addl %eax, (%rax)+;; 1c2: addl %eax, (%rax)+;; 1c4: addl %eax, (%rax)+;; 1c6: addl %eax, (%rax)+;; 1c8: addl %eax, (%rax)+;; 1ca: addl %eax, (%rax)+;; 1cc: addl %eax, (%rax)+;; 1ce: addl %eax, (%rax)