Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

winch(aarch64): v128 mul, dot, and q15mulr ops (#13965)

Daniel Macovei89ed11286410
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs
index bfd1ba3959..81d4cc43b7 100644
--- a/crates/test-util/src/wast.rs
+++ b/crates/test-util/src/wast.rs
@@ -530,6 +530,7 @@ impl WastTest {
"misc_testsuite/simd/load_splat_out_of_bounds.wast",
"misc_testsuite/simd/edge-of-memory.wast",
"misc_testsuite/issue6562.wast",
+ "misc_testsuite/simd/almost-extmul.wast",
];
if now_supported.iter().any(|part| self.path.ends_with(part)) {
return false;
diff --git a/tests/disas/winch/aarch64/v128_ops/f32x4_mul.wat b/tests/disas/winch/aarch64/v128_ops/f32x4_mul.wat
new file mode 100644
index 0000000000..4620de3c88
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/f32x4_mul.wat
@@ -0,0 +1,39 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))
+
+ (func (export "f32x4_mul") (param v128) (param v128) (result v128)
+ (f32x4.mul (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x70
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; fmul v1.4s, v1.4s, v0.4s
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 70: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_ops/f64x2_mul.wat b/tests/disas/winch/aarch64/v128_ops/f64x2_mul.wat
new file mode 100644
index 0000000000..ef7f4122fb
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/f64x2_mul.wat
@@ -0,0 +1,39 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))
+
+ (func (export "f64x2_mul") (param v128) (param v128) (result v128)
+ (f64x2.mul (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x70
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; fmul v1.2d, v1.2d, v0.2d
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 70: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_ops/i16x8_mul.wat b/tests/disas/winch/aarch64/v128_ops/i16x8_mul.wat
new file mode 100644
index 0000000000..e584ae4006
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i16x8_mul.wat
@@ -0,0 +1,39 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))
+
+ (func (export "i16x8_mul") (param v128) (param v128) (result v128)
+ (i16x8.mul (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x70
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; mul v1.8h, v1.8h, v0.8h
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 70: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_ops/i16x8_q15mulr_sat_s.wat b/tests/disas/winch/aarch64/v128_ops/i16x8_q15mulr_sat_s.wat
new file mode 100644
index 0000000000..6d6128b4d1
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i16x8_q15mulr_sat_s.wat
@@ -0,0 +1,39 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))
+
+ (func (export "i16x8_q15mulr_sat_s") (param v128) (param v128) (result v128)
+ (i16x8.q15mulr_sat_s (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x70
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; sqrdmulh v1.8h, v1.8h, v0.8h
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 70: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_ops/i32x4_dot_i16x8_s.wat b/tests/disas/winch/aarch64/v128_ops/i32x4_dot_i16x8_s.wat
new file mode 100644
index 0000000000..c9742a1213
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i32x4_dot_i16x8_s.wat
@@ -0,0 +1,41 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))
+
+ (func (export "i32x4_dot_i16x8_s") (param v128) (param v128) (result v128)
+ (i32x4.dot_i16x8_s (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x78
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; smull v31.4s, v1.4h, v0.4h
+;; smull2 v1.4s, v1.8h, v0.8h
+;; addp v1.4s, v31.4s, v1.4s
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 78: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_ops/i32x4_mul.wat b/tests/disas/winch/aarch64/v128_ops/i32x4_mul.wat
new file mode 100644
index 0000000000..a2cb2d1652
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i32x4_mul.wat
@@ -0,0 +1,39 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))
+
+ (func (export "i32x4_mul") (param v128) (param v128) (result v128)
+ (i32x4.mul (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x70
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; mul v1.4s, v1.4s, v0.4s
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 70: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_ops/i64x2_mul.wat b/tests/disas/winch/aarch64/v128_ops/i64x2_mul.wat
new file mode 100644
index 0000000000..9e530a558d
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i64x2_mul.wat
@@ -0,0 +1,46 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))
+
+ (func (export "i64x2_mul") (param v128) (param v128) (result v128)
+ (i64x2.mul (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x8c
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; rev64 v31.4s, v0.4s
+;; mul v31.4s, v31.4s, v1.4s
+;; addp v31.4s, v31.4s, v31.4s
+;; xtn v2.2s, v1.2d
+;; xtn v1.2s, v0.2d
+;; shll v31.2d, v31.2s, #32
+;; umlal v31.2d, v1.2s, v2.2s
+;; mov v1.16b, v31.16b
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 8c: udf #0xc11f
diff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rs
index b364eb9f1a..e2f3f767ba 100644
--- a/winch/codegen/src/isa/aarch64/asm.rs
+++ b/winch/codegen/src/isa/aarch64/asm.rs
@@ -16,7 +16,7 @@ use cranelift_codegen::isa::aarch64;
use cranelift_codegen::isa::aarch64::inst::emit::{enc_arith_rrr, enc_move_wide, enc_movk};
use cranelift_codegen::isa::aarch64::inst::{
ASIMDFPModImm, FpuToIntOp, MoveWideConst, NZCV, UImm5, VecALUModOp, VecExtendOp, VecRRLongOp,
- VecRRNarrowOp, VecRRPairLongOp, VecRRRLongOp,
+ VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp,
};
use cranelift_codegen::{
Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,
@@ -568,6 +568,26 @@ impl Assembler {
});
}
+ /// Vector widening multiply accumulate: multiply the low or high halves of
+ /// `rn` and `rm`, adding the result to lanes of twice the source lane width.
+ pub fn vec_rrrr_long(
+ &mut self,
+ alu_op: VecRRRLongModOp,
+ rn: Reg,
+ rm: Reg,
+ rd: WritableReg,
+ high_half: bool,
+ ) {
+ self.emit(Inst::VecRRRLongMod {
+ alu_op,
+ rd: rd.map(Into::into),
+ ri: rd.to_reg().into(),
+ rn: rn.into(),
+ rm: rm.into(),
+ high_half,
+ });
+ }
+
/// Vector pairwise widening add: sum adjacent lane pairs of `rn` into
/// lanes of twice the source lane width.
pub fn vec_rr_pair_long(&mut self, op: VecRRPairLongOp, rn: Reg, rd: WritableReg) {
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 419252ab16..656ab4af2c 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -36,7 +36,7 @@ use cranelift_codegen::{
isa::aarch64::inst::{
self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, ScalarSize,
VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecRRLongOp, VecRRNarrowOp,
- VecRRPairLongOp, VecRRRLongOp, VectorSize,
+ VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VectorSize,
},
settings,
};
@@ -1898,10 +1898,133 @@ impl Masm for MacroAssembler {
fn v128_mul(
&mut self,
- _context: &mut CodeGenContext<Emission>,
- _kind: V128MulKind,
+ context: &mut CodeGenContext<Emission>,
+ kind: V128MulKind,
) -> Result<()> {
- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))
+ let rhs = context.pop_to_reg(self, None)?;
+ let lhs = context.pop_to_reg(self, None)?;
+
+ match kind {
+ V128MulKind::F32x4 => self.asm.vec_rrr(
+ VecALUOp::Fmul,
+ lhs.reg,
+ rhs.reg,
+ writable!(lhs.reg),
+ VectorSize::Size32x4,
+ ),
+ V128MulKind::F64x2 => self.asm.vec_rrr(
+ VecALUOp::Fmul,
+ lhs.reg,
+ rhs.reg,
+ writable!(lhs.reg),
+ VectorSize::Size64x2,
+ ),
+ V128MulKind::I16x8 => self.asm.vec_rrr(
+ VecALUOp::Mul,
+ lhs.reg,
+ rhs.reg,
+ writable!(lhs.reg),
+ VectorSize::Size16x8,
+ ),
+ V128MulKind::I32x4 => self.asm.vec_rrr(
+ VecALUOp::Mul,
+ lhs.reg,
+ rhs.reg,
+ writable!(lhs.reg),
+ VectorSize::Size32x4,
+ ),
+ // aarch64 has no 64-bit lane `mul`, so the multiplication is
+ // performed with 32-bit operations (following the lowering in
+ // cranelift's `isa/aarch64/lower.isle`).
+ //
+ // Each 64-bit lane of `lhs` and `rhs` is split into 32-bit
+ // halves:
+ //
+ // x = a + 2^32(b) pictured as |b|a| (high half | low half)
+ // y = c + 2^32(d) pictured as |d|c|
+ //
+ // making each lane of the product:
+ //
+ // x * y = ac + 2^32(ad + bc) + 2^64(bd)
+ //
+ // The `2^64(bd)` term is entirely above bit 63, so it is
+ // discarded, same as the wrapping 64-bit multiplication.
+ V128MulKind::I64x2 => {
+ let tmp = context.any_fpr(self)?;
+ self.with_scratch::<FloatScratch, _>(|masm, hi| {
+ // Swap the halves of each lane of `rhs` so each half
+ // lines up with the opposite half of `lhs`:
+ // hi = |c|d|
+ masm.asm.vec_misc(
+ VecMisc2::Rev64,
+ rhs.reg,
+ hi.writable(),
+ VectorSize::Size32x4,
+ );
+ // A single 32-bit multiply against `lhs = |b|a|` now
+ // computes both cross terms of every lane:
+ // hi = |bc|ad|
+ // The products' upper bits are discarded, matching the
+ // shift past 2^64 they would receive below.
+ masm.asm.vec_rrr(
+ VecALUOp::Mul,
+ hi.inner(),
+ lhs.reg,
+ hi.writable(),
+ VectorSize::Size32x4,
+ );
+ // Sum adjacent elements, collapsing each lane to:
+ // hi = |ad + bc|
+ masm.asm.vec_rrr(
+ VecALUOp::Addp,
+ hi.inner(),
+ hi.inner(),
+ hi.writable(),
+ VectorSize::Size32x4,
+ );
+ // Extract the low halves of each lane:
+ // tmp = |a1|a0|
+ // lhs = |c1|c0|
+ masm.asm.vec_narrow(
+ VecRRNarrowOp::Xtn,
+ lhs.reg,
+ writable!(tmp),
+ false,
+ ScalarSize::Size32,
+ );
+ masm.asm.vec_narrow(
+ VecRRNarrowOp::Xtn,
+ rhs.reg,
+ writable!(lhs.reg),
+ false,
+ ScalarSize::Size32,
+ );
+ // Widen the sums back to 64-bit lanes, shifted into the
+ // high half:
+ // hi = |(ad + bc) << 32|
+ masm.asm
+ .vec_rr_long(VecRRLongOp::Shll32, hi.inner(), hi.writable(), false);
+ // Multiply the low halves into exact 64-bit products and
+ // accumulate, completing each lane:
+ // hi = |ac + 2^32(ad + bc)|
+ masm.asm.vec_rrrr_long(
+ VecRRRLongModOp::Umlal32,
+ lhs.reg,
+ tmp,
+ hi.writable(),
+ false,
+ );
+ masm.asm
+ .fmov_rr(hi.inner(), writable!(lhs.reg), OperandSize::S128);
+ });
+ context.free_reg(tmp);
+ }
+ }
+
+ context.stack.push(lhs.into());
+ context.free_reg(rhs);
+
+ Ok(())
}
fn v128_abs(&mut self, src: Reg, dst: WritableReg, kind: V128AbsKind) -> Result<()> {
@@ -1981,12 +2104,19 @@ impl Masm for MacroAssembler {
fn v128_q15mulr_sat_s(
&mut self,
- _lhs: Reg,
- _rhs: Reg,
- _dst: WritableReg,
- _size: OperandSize,
+ lhs: Reg,
+ rhs: Reg,
+ dst: WritableReg,
+ size: OperandSize,
) -> Result<()> {
- bail!(CodeGenError::unimplemented_masm_instruction())
+ self.asm.vec_rrr(
+ VecALUOp::Sqrdmulh,
+ lhs,
+ rhs,
+ dst,
+ VectorSize::from_lane_size(size.into(), true),
+ );
+ Ok(())
}
fn v128_all_true(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
@@ -2168,8 +2298,22 @@ impl Masm for MacroAssembler {
Ok(())
}
- fn v128_dot(&mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg) -> Result<()> {
- bail!(CodeGenError::unimplemented_masm_instruction())
+ fn v128_dot(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg) -> Result<()> {
+ self.with_scratch::<FloatScratch, _>(|masm, low| {
+ // Multiply each half into 32-bit lanes, then sum adjacent pairs.
+ masm.asm
+ .vec_rrr_long(VecRRRLongOp::Smull16, lhs, rhs, low.writable(), false);
+ masm.asm
+ .vec_rrr_long(VecRRRLongOp::Smull16, lhs, rhs, dst, true);
+ masm.asm.vec_rrr(
+ VecALUOp::Addp,
+ low.inner(),
+ dst.to_reg(),
+ dst,
+ VectorSize::Size32x4,
+ );
+ });
+ Ok(())
}
fn v128_popcnt(&mut self, context: &mut CodeGenContext<Emission>) -> Result<()> {