A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch(aarch64): v128 shuffle, swizzle, and bitmask ops (#13966)
Daniel Macoveieac68408fb58
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex 81d4cc43b7..983b315ee2 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -531,6 +531,8 @@ impl WastTest {"misc_testsuite/simd/edge-of-memory.wast","misc_testsuite/issue6562.wast","misc_testsuite/simd/almost-extmul.wast",+ "misc_testsuite/simd/sse-cannot-fold-unaligned-loads.wast",+ "misc_testsuite/simd/issue_3327_bnot_lowering.wast",];if now_supported.iter().any(|part| self.path.ends_with(part)) {return false;diff --git a/tests/disas/winch/aarch64/v128_ops/i16x8_bitmask.wat b/tests/disas/winch/aarch64/v128_ops/i16x8_bitmask.watnew file mode 100644index 0000000000..01529033f6--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i16x8_bitmask.wat@@ -0,0 +1,44 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func (export "i16x8_bitmask") (param v128) (result i32)+ (i16x8.bitmask (local.get 0)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x20+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x74+;; 2c: mov x9, x0+;; sub x28, x28, #0x20+;; mov sp, x28+;; stur x0, [x28, #0x18]+;; stur x1, [x28, #0x10]+;; stur q0, [x28]+;; ldur q0, [x28]+;; sshr v0.8h, v0.8h, #0xf+;; ldr q31, #0x80+;; and v0.16b, v0.16b, v31.16b+;; addv h0, v0.8h+;; umov w0, v0.h[0]+;; add x28, x28, #0x20+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 74: udf #0xc11f+;; 78: udf #0+;; 7c: udf #0+;; 80: .byte 0x01, 0x00, 0x02, 0x00+;; 84: .byte 0x04, 0x00, 0x08, 0x00+;; 88: .byte 0x10, 0x00, 0x20, 0x00+;; 8c: .byte 0x40, 0x00, 0x80, 0x00diff --git a/tests/disas/winch/aarch64/v128_ops/i32x4_bitmask.wat b/tests/disas/winch/aarch64/v128_ops/i32x4_bitmask.watnew file mode 100644index 0000000000..8433b77f1e--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i32x4_bitmask.wat@@ -0,0 +1,44 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func (export "i32x4_bitmask") (param v128) (result i32)+ (i32x4.bitmask (local.get 0)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x20+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x74+;; 2c: mov x9, x0+;; sub x28, x28, #0x20+;; mov sp, x28+;; stur x0, [x28, #0x18]+;; stur x1, [x28, #0x10]+;; stur q0, [x28]+;; ldur q0, [x28]+;; sshr v0.4s, v0.4s, #0x1f+;; ldr q31, #0x80+;; and v0.16b, v0.16b, v31.16b+;; addv s0, v0.4s+;; mov w0, v0.s[0]+;; add x28, x28, #0x20+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 74: udf #0xc11f+;; 78: udf #0+;; 7c: udf #0+;; 80: udf #1+;; 84: udf #2+;; 88: udf #4+;; 8c: udf #8diff --git a/tests/disas/winch/aarch64/v128_ops/i64x2_bitmask.wat b/tests/disas/winch/aarch64/v128_ops/i64x2_bitmask.watnew file mode 100644index 0000000000..4e436c1014--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i64x2_bitmask.wat@@ -0,0 +1,39 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func (export "i64x2_bitmask") (param v128) (result i32)+ (i64x2.bitmask (local.get 0)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x20+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x78+;; 2c: mov x9, x0+;; sub x28, x28, #0x20+;; mov sp, x28+;; stur x0, [x28, #0x18]+;; stur x1, [x28, #0x10]+;; stur q0, [x28]+;; ldur q0, [x28]+;; mov x16, v0.d[1]+;; mov x0, v0.d[0]+;; lsr x16, x16, #0x3f+;; lsr x0, x0, #0x3f+;; lsl x16, x16, #1+;; add x0, x0, x16, uxtx+;; add x28, x28, #0x20+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 78: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_ops/i8x16_bitmask.wat b/tests/disas/winch/aarch64/v128_ops/i8x16_bitmask.watnew file mode 100644index 0000000000..cdaee58945--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i8x16_bitmask.wat@@ -0,0 +1,44 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func (export "i8x16_bitmask") (param v128) (result i32)+ (i8x16.bitmask (local.get 0)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x20+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x7c+;; 2c: mov x9, x0+;; sub x28, x28, #0x20+;; mov sp, x28+;; stur x0, [x28, #0x18]+;; stur x1, [x28, #0x10]+;; stur q0, [x28]+;; ldur q0, [x28]+;; sshr v0.16b, v0.16b, #7+;; ldr q31, #0x80+;; and v0.16b, v0.16b, v31.16b+;; ext v31.16b, v0.16b, v0.16b, #8+;; zip1 v0.16b, v0.16b, v31.16b+;; addv h0, v0.8h+;; umov w0, v0.h[0]+;; add x28, x28, #0x20+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 7c: udf #0xc11f+;; 80: stxrb w4, w1, [x16]+;; 84: .byte 0x10, 0x20, 0x40, 0x80+;; 88: stxrb w4, w1, [x16]+;; 8c: .byte 0x10, 0x20, 0x40, 0x80diff --git a/tests/disas/winch/aarch64/v128_ops/i8x16_shuffle.wat b/tests/disas/winch/aarch64/v128_ops/i8x16_shuffle.watnew file mode 100644index 0000000000..997ad24d1c--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i8x16_shuffle.wat@@ -0,0 +1,49 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func (export "i8x16_shuffle") (param v128) (param v128) (result v128)+ (i8x16.shuffle 0 17 2 19 4 21 6 23 8 25 10 27 12 29 14 31+ (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x7c+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; ldr q31, #0x80+;; tbl v1.16b, {v1.16b}, v31.16b+;; ldr q31, #0x90+;; tbx v1.16b, {v0.16b}, v31.16b+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 7c: udf #0xc11f+;; 80: sbfx w0, w8, #2, #3+;; 84: b #0xfffffffffc185494+;; 88: madd w8, w8, w10, w6+;; 8c: fmadd s12, s8, s14, s7+;; 90: .byte 0xf0, 0x01, 0xf2, 0x03+;; 94: .byte 0xf4, 0x05, 0xf6, 0x07+;; 98: .byte 0xf8, 0x09, 0xfa, 0x0b+;; 9c: .byte 0xfc, 0x0d, 0xfe, 0x0fdiff --git a/tests/disas/winch/aarch64/v128_ops/i8x16_swizzle.wat b/tests/disas/winch/aarch64/v128_ops/i8x16_swizzle.watnew file mode 100644index 0000000000..85502be958--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/i8x16_swizzle.wat@@ -0,0 +1,37 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func (export "i8x16_swizzle") (param v128) (param v128) (result v128)+ (i8x16.swizzle (local.get 0) (local.get 1)))+)+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x70+;; 2c: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; stur q0, [x28, #0x10]+;; stur q1, [x28]+;; ldur q0, [x28]+;; ldur q1, [x28, #0x10]+;; tbl v1.16b, {v1.16b}, v0.16b+;; mov v0.16b, v1.16b+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 70: udf #0xc11fdiff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rsindex e2f3f767ba..3c7d57fd50 100644--- a/winch/codegen/src/isa/aarch64/asm.rs+++ b/winch/codegen/src/isa/aarch64/asm.rs@@ -16,7 +16,7 @@ use cranelift_codegen::isa::aarch64;use cranelift_codegen::isa::aarch64::inst::emit::{enc_arith_rrr, enc_move_wide, enc_movk};use cranelift_codegen::isa::aarch64::inst::{ASIMDFPModImm, FpuToIntOp, MoveWideConst, NZCV, UImm5, VecALUModOp, VecExtendOp, VecRRLongOp,- VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp,+ VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp,};use cranelift_codegen::{Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,@@ -637,6 +637,65 @@ impl Assembler {});}+ /// Vector table lookup: index `rn`'s bytes by `rm`, zeroing out of range+ /// lanes.+ pub fn vec_tbl(&mut self, rn: Reg, rm: Reg, rd: WritableReg) {+ self.emit(Inst::VecTbl {+ rd: rd.map(Into::into),+ rn: rn.into(),+ rm: rm.into(),+ });+ }++ /// Vector table lookup, leaving out of range lanes of `rd` unmodified.+ pub fn vec_tbl_ext(&mut self, rn: Reg, rm: Reg, rd: WritableReg) {+ self.emit(Inst::VecTblExt {+ rd: rd.map(Into::into),+ ri: rd.to_reg().into(),+ rn: rn.into(),+ rm: rm.into(),+ });+ }++ /// Vector shift by immediate.+ pub fn vec_shift_imm(+ &mut self,+ op: VecShiftImmOp,+ imm: u8,+ rn: Reg,+ rd: WritableReg,+ size: VectorSize,+ ) {+ self.emit(Inst::VecShiftImm {+ op,+ rd: rd.map(Into::into),+ rn: rn.into(),+ size,+ imm,+ });+ }++ /// Extract a vector from a pair of vectors, starting at byte `imm4`.+ pub fn vec_extract(&mut self, rn: Reg, rm: Reg, rd: WritableReg, imm4: u8) {+ self.emit(Inst::VecExtract {+ rd: rd.map(Into::into),+ rn: rn.into(),+ rm: rm.into(),+ imm4,+ });+ }++ /// Load a 128-bit constant into a vector register.+ pub fn vec_load_const(&mut self, constant: &[u8; 16], rd: WritableReg) {+ let handle = self.add_constant(constant);+ self.uload(+ AMode::Const { addr: handle },+ rd,+ OperandSize::S128,+ TRUSTED_FLAGS,+ );+ }+/// Vector reduction across lanespub fn vec_lanes(&mut self, op: VecLanesOp, rn: Reg, rd: WritableReg, size: VectorSize) {self.emit(Inst::VecLanes {diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex 656ab4af2c..255ad6fc69 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -36,7 +36,7 @@ use cranelift_codegen::{isa::aarch64::inst::{self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, ScalarSize,VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecRRLongOp, VecRRNarrowOp,- VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VectorSize,+ VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp, VectorSize,},settings,};@@ -1388,12 +1388,43 @@ impl Masm for MacroAssembler {Ok(())}- fn shuffle(&mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _lanes: [u8; 16]) -> Result<()> {- bail!(CodeGenError::unimplemented_masm_instruction())+ fn shuffle(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, lanes: [u8; 16]) -> Result<()> {+ // Use `tbl` with `lanes` to select the lanes in `lhs` and `rhs`+ // separately, then combine them with `tbx` into `dst`.+ //+ // A single-register `tbl` sets lanes whose index is out of range+ // (above 15) to 0, so each source is indexed by a version of `lanes`+ // that puts the other source's indices out of range: `lhs` is indexed+ // by `lanes` as-is (its indices are 0..=15; `rhs`'s 16..=31 fall out+ // of range), while for `rhs` every index is shifted down by 16+ // (mapping 16..=31 to 0..=15, and wrapping `lhs`'s 0..=15 to 240..).+ //+ // `tbx`, unlike `tbl`, leaves out of range lanes of the destination+ // unmodified, so the second lookup fills in `rhs`'s lanes while+ // preserving the lanes already selected from `lhs`, avoiding a+ // separate combining instruction.+ let mut lhs_mask = [0u8; 16];+ let mut rhs_mask = [0u8; 16];+ for (i, lane) in lanes.iter().enumerate() {+ lhs_mask[i] = *lane;+ rhs_mask[i] = lane.wrapping_sub(16);+ }++ self.with_scratch::<FloatScratch, _>(|masm, mask| {+ masm.asm.vec_load_const(&lhs_mask, mask.writable());+ masm.asm.vec_tbl(lhs, mask.inner(), dst);+ masm.asm.vec_load_const(&rhs_mask, mask.writable());+ masm.asm.vec_tbl_ext(rhs, mask.inner(), dst);+ });+ Ok(())}- fn swizzle(&mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg) -> Result<()> {- bail!(CodeGenError::unimplemented_masm_instruction())+ fn swizzle(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg) -> Result<()> {+ // `tbl` sets lanes whose index is out of range (above 15) to 0, which+ // is exactly the behavior the Wasm instruction requires, so no+ // clamping of the indices in `rhs` is needed.+ self.asm.vec_tbl(lhs, rhs, dst);+ Ok(())}fn atomic_rmw(@@ -2158,8 +2189,99 @@ impl Masm for MacroAssembler {Ok(())}- fn v128_bitmask(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()> {- bail!(CodeGenError::unimplemented_masm_instruction())+ fn v128_bitmask(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {+ // Each sequence replicates the sign bit across its lane, masks in the+ // bit position that lane contributes, and sums the lanes together,+ // following the `vhigh_bits` lowerings in cranelift's+ // `isa/aarch64/lower.isle`, which use the same masks.+ let (shift, mask, vector_size) = match size {+ OperandSize::S8 => (+ 7,+ [+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10,+ 0x20, 0x40, 0x80,+ ],+ VectorSize::Size8x16,+ ),+ OperandSize::S16 => (+ 15,+ [+ 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x08, 0x00, 0x10, 0x00, 0x20, 0x00, 0x40,+ 0x00, 0x80, 0x00,+ ],+ VectorSize::Size16x8,+ ),+ OperandSize::S32 => (+ 31,+ [+ 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08,+ 0x00, 0x00, 0x00,+ ],+ VectorSize::Size32x4,+ ),+ // With only two lanes the sign bits are cheaper to move out and+ // combine in general purpose registers.+ OperandSize::S64 => {+ let sign_bit = ImmShift::maybe_from_u64(63).unwrap();+ self.with_scratch::<IntScratch, _>(|masm, upper| {+ masm.asm+ .mov_from_vec(src, upper.writable(), 1, OperandSize::S64);+ masm.asm.mov_from_vec(src, dst, 0, OperandSize::S64);+ masm.asm.shift_ir(+ sign_bit,+ upper.inner(),+ upper.writable(),+ ShiftKind::ShrU,+ OperandSize::S64,+ );+ masm.asm.shift_ir(+ sign_bit,+ dst.to_reg(),+ dst,+ ShiftKind::ShrU,+ OperandSize::S64,+ );+ masm.asm.shift_ir(+ ImmShift::maybe_from_u64(1).unwrap(),+ upper.inner(),+ upper.writable(),+ ShiftKind::Shl,+ OperandSize::S64,+ );+ masm.asm+ .add_rrr(upper.inner(), dst.to_reg(), dst, OperandSize::S64);+ });+ return Ok(());+ }+ _ => bail!(CodeGenError::unexpected_operand_size()),+ };++ // `src` is dead after this operation, so it doubles as a temporary.+ let acc = writable!(src);+ self.with_scratch::<FloatScratch, _>(|masm, tmp| {+ masm.asm+ .vec_shift_imm(VecShiftImmOp::Sshr, shift, src, acc, vector_size);+ masm.asm.vec_load_const(&mask, tmp.writable());+ masm.asm+ .vec_rrr(VecALUOp::And, src, tmp.inner(), acc, vector_size);+ if let VectorSize::Size8x16 = vector_size {+ // Interleave the two halves so that each 16-bit lane holds one+ // byte's worth of bits from each half, letting a single+ // reduction gather all sixteen.+ masm.asm.vec_extract(src, src, tmp.writable(), 8);+ masm.asm+ .vec_rrr(VecALUOp::Zip1, src, tmp.inner(), acc, VectorSize::Size8x16);+ }+ let (reduce_size, lane_size) = match vector_size {+ VectorSize::Size8x16 | VectorSize::Size16x8 => {+ (VectorSize::Size16x8, OperandSize::S16)+ }+ other => (other, OperandSize::S32),+ };+ masm.asm.vec_lanes(VecLanesOp::Addv, src, acc, reduce_size);+ masm.asm.mov_from_vec(src, dst, 0, lane_size);+ });+ Ok(())}fn v128_trunc(