A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch(aarch64): v128 mul, dot, and q15mulr ops (#13965)
Daniel Macovei89ed11286410
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex bfd1ba3959..81d4cc43b7 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -530,6 +530,7 @@ impl WastTest {"misc_testsuite/simd/load_splat_out_of_bounds.wast","misc_testsuite/simd/edge-of-memory.wast","misc_testsuite/issue6562.wast",+ "misc_testsuite/simd/almost-extmul.wast",];if now_supported.iter().any(|part| self.path.ends_with(part)) {return false;diff --git a/tests/disas/winch/aarch64/v128_ops/f32x4_mul.wat b/tests/disas/winch/aarch64/v128_ops/f32x4_mul.watnew file mode 100644index 0000000000..4620de3c88--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/f32x4_mul.wat@@ -0,0 +1,39 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))++ (func (export "f32x4_mul") (param v128) (param v128) (result v128)+ (f32x4.mul (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x70+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; fmul v1.4s, v1.4s, v0.4s+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 70: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_ops/f64x2_mul.wat b/tests/disas/winch/aarch64/v128_ops/f64x2_mul.watnew file mode 100644index 0000000000..ef7f4122fb--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/f64x2_mul.wat@@ -0,0 +1,39 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))++ (func (export "f64x2_mul") (param v128) (param v128) (result v128)+ (f64x2.mul (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x70+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; fmul v1.2d, v1.2d, v0.2d+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 70: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_ops/i16x8_mul.wat b/tests/disas/winch/aarch64/v128_ops/i16x8_mul.watnew file mode 100644index 0000000000..e584ae4006--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i16x8_mul.wat@@ -0,0 +1,39 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))++ (func (export "i16x8_mul") (param v128) (param v128) (result v128)+ (i16x8.mul (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x70+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; mul v1.8h, v1.8h, v0.8h+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 70: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_ops/i16x8_q15mulr_sat_s.wat b/tests/disas/winch/aarch64/v128_ops/i16x8_q15mulr_sat_s.watnew file mode 100644index 0000000000..6d6128b4d1--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i16x8_q15mulr_sat_s.wat@@ -0,0 +1,39 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))++ (func (export "i16x8_q15mulr_sat_s") (param v128) (param v128) (result v128)+ (i16x8.q15mulr_sat_s (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x70+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; sqrdmulh v1.8h, v1.8h, v0.8h+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 70: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_ops/i32x4_dot_i16x8_s.wat b/tests/disas/winch/aarch64/v128_ops/i32x4_dot_i16x8_s.watnew file mode 100644index 0000000000..c9742a1213--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i32x4_dot_i16x8_s.wat@@ -0,0 +1,41 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))++ (func (export "i32x4_dot_i16x8_s") (param v128) (param v128) (result v128)+ (i32x4.dot_i16x8_s (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x78+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; smull v31.4s, v1.4h, v0.4h+;; smull2 v1.4s, v1.8h, v0.8h+;; addp v1.4s, v31.4s, v1.4s+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 78: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_ops/i32x4_mul.wat b/tests/disas/winch/aarch64/v128_ops/i32x4_mul.watnew file mode 100644index 0000000000..a2cb2d1652--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i32x4_mul.wat@@ -0,0 +1,39 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))++ (func (export "i32x4_mul") (param v128) (param v128) (result v128)+ (i32x4.mul (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x70+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; mul v1.4s, v1.4s, v0.4s+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 70: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_ops/i64x2_mul.wat b/tests/disas/winch/aarch64/v128_ops/i64x2_mul.watnew file mode 100644index 0000000000..9e530a558d--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i64x2_mul.wat@@ -0,0 +1,46 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory (data "\00\00\00\00\00\00\00\00\00\00\00\00\00\00\a0\7f"))++ (func (export "i64x2_mul") (param v128) (param v128) (result v128)+ (i64x2.mul (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x8c+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; rev64 v31.4s, v0.4s+;; mul v31.4s, v31.4s, v1.4s+;; addp v31.4s, v31.4s, v31.4s+;; xtn v2.2s, v1.2d+;; xtn v1.2s, v0.2d+;; shll v31.2d, v31.2s, #32+;; umlal v31.2d, v1.2s, v2.2s+;; mov v1.16b, v31.16b+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 8c: udf #0xc11fdiff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rsindex b364eb9f1a..e2f3f767ba 100644--- a/winch/codegen/src/isa/aarch64/asm.rs+++ b/winch/codegen/src/isa/aarch64/asm.rs@@ -16,7 +16,7 @@ use cranelift_codegen::isa::aarch64;use cranelift_codegen::isa::aarch64::inst::emit::{enc_arith_rrr, enc_move_wide, enc_movk};use cranelift_codegen::isa::aarch64::inst::{ASIMDFPModImm, FpuToIntOp, MoveWideConst, NZCV, UImm5, VecALUModOp, VecExtendOp, VecRRLongOp,- VecRRNarrowOp, VecRRPairLongOp, VecRRRLongOp,+ VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp,};use cranelift_codegen::{Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,@@ -568,6 +568,26 @@ impl Assembler {});}+ /// Vector widening multiply accumulate: multiply the low or high halves of+ /// `rn` and `rm`, adding the result to lanes of twice the source lane width.+ pub fn vec_rrrr_long(+ &mut self,+ alu_op: VecRRRLongModOp,+ rn: Reg,+ rm: Reg,+ rd: WritableReg,+ high_half: bool,+ ) {+ self.emit(Inst::VecRRRLongMod {+ alu_op,+ rd: rd.map(Into::into),+ ri: rd.to_reg().into(),+ rn: rn.into(),+ rm: rm.into(),+ high_half,+ });+ }+/// Vector pairwise widening add: sum adjacent lane pairs of `rn` into/// lanes of twice the source lane width.pub fn vec_rr_pair_long(&mut self, op: VecRRPairLongOp, rn: Reg, rd: WritableReg) {diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex 419252ab16..656ab4af2c 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -36,7 +36,7 @@ use cranelift_codegen::{isa::aarch64::inst::{self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, ScalarSize,VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecRRLongOp, VecRRNarrowOp,- VecRRPairLongOp, VecRRRLongOp, VectorSize,+ VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VectorSize,},settings,};@@ -1898,10 +1898,133 @@ impl Masm for MacroAssembler {fn v128_mul(&mut self,- _context: &mut CodeGenContext<Emission>,- _kind: V128MulKind,+ context: &mut CodeGenContext<Emission>,+ kind: V128MulKind,) -> Result<()> {- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))+ let rhs = context.pop_to_reg(self, None)?;+ let lhs = context.pop_to_reg(self, None)?;++ match kind {+ V128MulKind::F32x4 => self.asm.vec_rrr(+ VecALUOp::Fmul,+ lhs.reg,+ rhs.reg,+ writable!(lhs.reg),+ VectorSize::Size32x4,+ ),+ V128MulKind::F64x2 => self.asm.vec_rrr(+ VecALUOp::Fmul,+ lhs.reg,+ rhs.reg,+ writable!(lhs.reg),+ VectorSize::Size64x2,+ ),+ V128MulKind::I16x8 => self.asm.vec_rrr(+ VecALUOp::Mul,+ lhs.reg,+ rhs.reg,+ writable!(lhs.reg),+ VectorSize::Size16x8,+ ),+ V128MulKind::I32x4 => self.asm.vec_rrr(+ VecALUOp::Mul,+ lhs.reg,+ rhs.reg,+ writable!(lhs.reg),+ VectorSize::Size32x4,+ ),+ // aarch64 has no 64-bit lane `mul`, so the multiplication is+ // performed with 32-bit operations (following the lowering in+ // cranelift's `isa/aarch64/lower.isle`).+ //+ // Each 64-bit lane of `lhs` and `rhs` is split into 32-bit+ // halves:+ //+ // x = a + 2^32(b) pictured as |b|a| (high half | low half)+ // y = c + 2^32(d) pictured as |d|c|+ //+ // making each lane of the product:+ //+ // x * y = ac + 2^32(ad + bc) + 2^64(bd)+ //+ // The `2^64(bd)` term is entirely above bit 63, so it is+ // discarded, same as the wrapping 64-bit multiplication.+ V128MulKind::I64x2 => {+ let tmp = context.any_fpr(self)?;+ self.with_scratch::<FloatScratch, _>(|masm, hi| {+ // Swap the halves of each lane of `rhs` so each half+ // lines up with the opposite half of `lhs`:+ // hi = |c|d|+ masm.asm.vec_misc(+ VecMisc2::Rev64,+ rhs.reg,+ hi.writable(),+ VectorSize::Size32x4,+ );+ // A single 32-bit multiply against `lhs = |b|a|` now+ // computes both cross terms of every lane:+ // hi = |bc|ad|+ // The products' upper bits are discarded, matching the+ // shift past 2^64 they would receive below.+ masm.asm.vec_rrr(+ VecALUOp::Mul,+ hi.inner(),+ lhs.reg,+ hi.writable(),+ VectorSize::Size32x4,+ );+ // Sum adjacent elements, collapsing each lane to:+ // hi = |ad + bc|+ masm.asm.vec_rrr(+ VecALUOp::Addp,+ hi.inner(),+ hi.inner(),+ hi.writable(),+ VectorSize::Size32x4,+ );+ // Extract the low halves of each lane:+ // tmp = |a1|a0|+ // lhs = |c1|c0|+ masm.asm.vec_narrow(+ VecRRNarrowOp::Xtn,+ lhs.reg,+ writable!(tmp),+ false,+ ScalarSize::Size32,+ );+ masm.asm.vec_narrow(+ VecRRNarrowOp::Xtn,+ rhs.reg,+ writable!(lhs.reg),+ false,+ ScalarSize::Size32,+ );+ // Widen the sums back to 64-bit lanes, shifted into the+ // high half:+ // hi = |(ad + bc) << 32|+ masm.asm+ .vec_rr_long(VecRRLongOp::Shll32, hi.inner(), hi.writable(), false);+ // Multiply the low halves into exact 64-bit products and+ // accumulate, completing each lane:+ // hi = |ac + 2^32(ad + bc)|+ masm.asm.vec_rrrr_long(+ VecRRRLongModOp::Umlal32,+ lhs.reg,+ tmp,+ hi.writable(),+ false,+ );+ masm.asm+ .fmov_rr(hi.inner(), writable!(lhs.reg), OperandSize::S128);+ });+ context.free_reg(tmp);+ }+ }++ context.stack.push(lhs.into());+ context.free_reg(rhs);++ Ok(())}fn v128_abs(&mut self, src: Reg, dst: WritableReg, kind: V128AbsKind) -> Result<()> {@@ -1981,12 +2104,19 @@ impl Masm for MacroAssembler {fn v128_q15mulr_sat_s(&mut self,- _lhs: Reg,- _rhs: Reg,- _dst: WritableReg,- _size: OperandSize,+ lhs: Reg,+ rhs: Reg,+ dst: WritableReg,+ size: OperandSize,) -> Result<()> {- bail!(CodeGenError::unimplemented_masm_instruction())+ self.asm.vec_rrr(+ VecALUOp::Sqrdmulh,+ lhs,+ rhs,+ dst,+ VectorSize::from_lane_size(size.into(), true),+ );+ Ok(())}fn v128_all_true(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {@@ -2168,8 +2298,22 @@ impl Masm for MacroAssembler {Ok(())}- fn v128_dot(&mut self, _lhs: Reg, _rhs: Reg, _dst: WritableReg) -> Result<()> {- bail!(CodeGenError::unimplemented_masm_instruction())+ fn v128_dot(&mut self, lhs: Reg, rhs: Reg, dst: WritableReg) -> Result<()> {+ self.with_scratch::<FloatScratch, _>(|masm, low| {+ // Multiply each half into 32-bit lanes, then sum adjacent pairs.+ masm.asm+ .vec_rrr_long(VecRRRLongOp::Smull16, lhs, rhs, low.writable(), false);+ masm.asm+ .vec_rrr_long(VecRRRLongOp::Smull16, lhs, rhs, dst, true);+ masm.asm.vec_rrr(+ VecALUOp::Addp,+ low.inner(),+ dst.to_reg(),+ dst,+ VectorSize::Size32x4,+ );+ });+ Ok(())}fn v128_popcnt(&mut self, context: &mut CodeGenContext<Emission>) -> Result<()> {