Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

winch(aarch64): v128 shuffle, swizzle, and bitmask ops (#13966)

Daniel Macoveieac68408fb58
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs
index 81d4cc43b7..983b315ee2 100644
--- a/crates/test-util/src/wast.rs
+++ b/crates/test-util/src/wast.rs
@@ -531,6 +531,8 @@ impl WastTest {
"misc_testsuite/simd/edge-of-memory.wast",
"misc_testsuite/issue6562.wast",
"misc_testsuite/simd/almost-extmul.wast",
+ "misc_testsuite/simd/sse-cannot-fold-unaligned-loads.wast",
+ "misc_testsuite/simd/issue_3327_bnot_lowering.wast",
];
if now_supported.iter().any(|part| self.path.ends_with(part)) {
return false;
diff --git a/tests/disas/winch/aarch64/v128_ops/i16x8_bitmask.wat b/tests/disas/winch/aarch64/v128_ops/i16x8_bitmask.wat
new file mode 100644
index 0000000000..01529033f6
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i16x8_bitmask.wat
@@ -0,0 +1,44 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (func (export "i16x8_bitmask") (param v128) (result i32)
+ (i16x8.bitmask (local.get 0)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x20
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x74
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x20
+;; mov sp, x28
+;; stur x0, [x28, #0x18]
+;; stur x1, [x28, #0x10]
+;; stur q0, [x28]
+;; ldur q0, [x28]
+;; sshr v0.8h, v0.8h, #0xf
+;; ldr q31, #0x80
+;; and v0.16b, v0.16b, v31.16b
+;; addv h0, v0.8h
+;; umov w0, v0.h[0]
+;; add x28, x28, #0x20
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 74: udf #0xc11f
+;; 78: udf #0
+;; 7c: udf #0
+;; 80: .byte 0x01, 0x00, 0x02, 0x00
+;; 84: .byte 0x04, 0x00, 0x08, 0x00
+;; 88: .byte 0x10, 0x00, 0x20, 0x00
+;; 8c: .byte 0x40, 0x00, 0x80, 0x00
diff --git a/tests/disas/winch/aarch64/v128_ops/i32x4_bitmask.wat b/tests/disas/winch/aarch64/v128_ops/i32x4_bitmask.wat
new file mode 100644
index 0000000000..8433b77f1e
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i32x4_bitmask.wat
@@ -0,0 +1,44 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (func (export "i32x4_bitmask") (param v128) (result i32)
+ (i32x4.bitmask (local.get 0)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x20
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x74
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x20
+;; mov sp, x28
+;; stur x0, [x28, #0x18]
+;; stur x1, [x28, #0x10]
+;; stur q0, [x28]
+;; ldur q0, [x28]
+;; sshr v0.4s, v0.4s, #0x1f
+;; ldr q31, #0x80
+;; and v0.16b, v0.16b, v31.16b
+;; addv s0, v0.4s
+;; mov w0, v0.s[0]
+;; add x28, x28, #0x20
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 74: udf #0xc11f
+;; 78: udf #0
+;; 7c: udf #0
+;; 80: udf #1
+;; 84: udf #2
+;; 88: udf #4
+;; 8c: udf #8
diff --git a/tests/disas/winch/aarch64/v128_ops/i64x2_bitmask.wat b/tests/disas/winch/aarch64/v128_ops/i64x2_bitmask.wat
new file mode 100644
index 0000000000..4e436c1014
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i64x2_bitmask.wat
@@ -0,0 +1,39 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (func (export "i64x2_bitmask") (param v128) (result i32)
+ (i64x2.bitmask (local.get 0)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x20
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x78
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x20
+;; mov sp, x28
+;; stur x0, [x28, #0x18]
+;; stur x1, [x28, #0x10]
+;; stur q0, [x28]
+;; ldur q0, [x28]
+;; mov x16, v0.d[1]
+;; mov x0, v0.d[0]
+;; lsr x16, x16, #0x3f
+;; lsr x0, x0, #0x3f
+;; lsl x16, x16, #1
+;; add x0, x0, x16, uxtx
+;; add x28, x28, #0x20
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 78: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_ops/i8x16_bitmask.wat b/tests/disas/winch/aarch64/v128_ops/i8x16_bitmask.wat
new file mode 100644
index 0000000000..cdaee58945
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i8x16_bitmask.wat
@@ -0,0 +1,44 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (func (export "i8x16_bitmask") (param v128) (result i32)
+ (i8x16.bitmask (local.get 0)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x20
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x7c
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x20
+;; mov sp, x28
+;; stur x0, [x28, #0x18]
+;; stur x1, [x28, #0x10]
+;; stur q0, [x28]
+;; ldur q0, [x28]
+;; sshr v0.16b, v0.16b, #7
+;; ldr q31, #0x80
+;; and v0.16b, v0.16b, v31.16b
+;; ext v31.16b, v0.16b, v0.16b, #8
+;; zip1 v0.16b, v0.16b, v31.16b
+;; addv h0, v0.8h
+;; umov w0, v0.h[0]
+;; add x28, x28, #0x20
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 7c: udf #0xc11f
+;; 80: stxrb w4, w1, [x16]
+;; 84: .byte 0x10, 0x20, 0x40, 0x80
+;; 88: stxrb w4, w1, [x16]
+;; 8c: .byte 0x10, 0x20, 0x40, 0x80
diff --git a/tests/disas/winch/aarch64/v128_ops/i8x16_shuffle.wat b/tests/disas/winch/aarch64/v128_ops/i8x16_shuffle.wat
new file mode 100644
index 0000000000..997ad24d1c
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i8x16_shuffle.wat
@@ -0,0 +1,49 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (func (export "i8x16_shuffle") (param v128) (param v128) (result v128)
+ (i8x16.shuffle 0 17 2 19 4 21 6 23 8 25 10 27 12 29 14 31
+ (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x7c
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; ldr q31, #0x80
+;; tbl v1.16b, {v1.16b}, v31.16b
+;; ldr q31, #0x90
+;; tbx v1.16b, {v0.16b}, v31.16b
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 7c: udf #0xc11f
+;; 80: sbfx w0, w8, #2, #3
+;; 84: b #0xfffffffffc185494
+;; 88: madd w8, w8, w10, w6
+;; 8c: fmadd s12, s8, s14, s7
+;; 90: .byte 0xf0, 0x01, 0xf2, 0x03
+;; 94: .byte 0xf4, 0x05, 0xf6, 0x07
+;; 98: .byte 0xf8, 0x09, 0xfa, 0x0b
+;; 9c: .byte 0xfc, 0x0d, 0xfe, 0x0f
diff --git a/tests/disas/winch/aarch64/v128_ops/i8x16_swizzle.wat b/tests/disas/winch/aarch64/v128_ops/i8x16_swizzle.wat
new file mode 100644
index 0000000000..85502be958
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_ops/i8x16_swizzle.wat
@@ -0,0 +1,37 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (func (export "i8x16_swizzle") (param v128) (param v128) (result v128)
+ (i8x16.swizzle (local.get 0) (local.get 1)))
+)
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x70
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; stur q0, [x28, #0x10]
+;; stur q1, [x28]
+;; ldur q0, [x28]
+;; ldur q1, [x28, #0x10]
+;; tbl v1.16b, {v1.16b}, v0.16b
+;; mov v0.16b, v1.16b
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 70: udf #0xc11f
diff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rs
index e2f3f767ba..3c7d57fd50 100644
--- a/winch/codegen/src/isa/aarch64/asm.rs
+++ b/winch/codegen/src/isa/aarch64/asm.rs
@@ -16,7 +16,7 @@ use cranelift_codegen::isa::aarch64;
use cranelift_codegen::isa::aarch64::inst::emit::{enc_arith_rrr, enc_move_wide, enc_movk};
use cranelift_codegen::isa::aarch64::inst::{
ASIMDFPModImm, FpuToIntOp, MoveWideConst, NZCV, UImm5, VecALUModOp, VecExtendOp, VecRRLongOp,
- VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp,
+ VecRRNarrowOp, VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp,
};
use cranelift_codegen::{
Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,
@@ -637,6 +637,65 @@ impl Assembler {
});
}
+ /// Vector table lookup: index `rn`'s bytes by `rm`, zeroing out of range
+ /// lanes.
+ pub fn vec_tbl(&mut self, rn: Reg, rm: Reg, rd: WritableReg) {
+ self.emit(Inst::VecTbl {
+ rd: rd.map(Into::into),
+ rn: rn.into(),
+ rm: rm.into(),
+ });
+ }
+
+ /// Vector table lookup, leaving out of range lanes of `rd` unmodified.
+ pub fn vec_tbl_ext(&mut self, rn: Reg, rm: Reg, rd: WritableReg) {
+ self.emit(Inst::VecTblExt {
+ rd: rd.map(Into::into),
+ ri: rd.to_reg().into(),
+ rn: rn.into(),
+ rm: rm.into(),
+ });
+ }
+
+ /// Vector shift by immediate.
+ pub fn vec_shift_imm(
+ &mut self,
+ op: VecShiftImmOp,
+ imm: u8,
+ rn: Reg,
+ rd: WritableReg,
+ size: VectorSize,
+ ) {
+ self.emit(Inst::VecShiftImm {
+ op,
+ rd: rd.map(Into::into),
+ rn: rn.into(),
+ size,
+ imm,
+ });
+ }
+
+ /// Extract a vector from a pair of vectors, starting at byte `imm4`.
+ pub fn vec_extract(&mut self, rn: Reg, rm: Reg, rd: WritableReg, imm4: u8) {
+ self.emit(Inst::VecExtract {
+ rd: rd.map(Into::into),
+ rn: rn.into(),
+ rm: rm.into(),
+ imm4,
+ });
+ }
+
+ /// Load a 128-bit constant into a vector register.
+ pub fn vec_load_const(&mut self, constant: &[u8; 16], rd: WritableReg) {
+ let handle = self.add_constant(constant);
+ self.uload(
+ AMode::Const { addr: handle },
+ rd,
+ OperandSize::S128,
+ TRUSTED_FLAGS,
+ );
+ }
+
/// Vector reduction across lanes
pub fn vec_lanes(&mut self, op: VecLanesOp, rn: Reg, rd: WritableReg, size: VectorSize) {
self.emit(Inst::VecLanes {
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 656ab4af2c..255ad6fc69 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -36,7 +36,7 @@ use cranelift_codegen::{
isa::aarch64::inst::{
self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, ScalarSize,
VecALUModOp, VecALUOp, VecExtendOp, VecLanesOp, VecMisc2, VecRRLongOp, VecRRNarrowOp,
- VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VectorSize,
+ VecRRPairLongOp, VecRRRLongModOp, VecRRRLongOp, VecShiftImmOp, VectorSize,
},
settings,
};
@@ -1388,12 +1388,43 @@ impl Masm for MacroAssembler {
Ok(())
}
- fn shuffle(&mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg, _lanes: [u8; 16]) -> Result<()> {
- bail!(CodeGenError::unimplemented_masm_instruction())
+ fn shuffle(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg, lanes: [u8; 16]) -> Result<()> {
+ // Use `tbl` with `lanes` to select the lanes in `lhs` and `rhs`
+ // separately, then combine them with `tbx` into `dst`.
+ //
+ // A single-register `tbl` sets lanes whose index is out of range
+ // (above 15) to 0, so each source is indexed by a version of `lanes`
+ // that puts the other source's indices out of range: `lhs` is indexed
+ // by `lanes` as-is (its indices are 0..=15; `rhs`'s 16..=31 fall out
+ // of range), while for `rhs` every index is shifted down by 16
+ // (mapping 16..=31 to 0..=15, and wrapping `lhs`'s 0..=15 to 240..).
+ //
+ // `tbx`, unlike `tbl`, leaves out of range lanes of the destination
+ // unmodified, so the second lookup fills in `rhs`'s lanes while
+ // preserving the lanes already selected from `lhs`, avoiding a
+ // separate combining instruction.
+ let mut lhs_mask = [0u8; 16];
+ let mut rhs_mask = [0u8; 16];
+ for (i, lane) in lanes.iter().enumerate() {
+ lhs_mask[i] = *lane;
+ rhs_mask[i] = lane.wrapping_sub(16);
+ }
+
+ self.with_scratch::<FloatScratch, _>(|masm, mask| {
+ masm.asm.vec_load_const(&lhs_mask, mask.writable());
+ masm.asm.vec_tbl(lhs, mask.inner(), dst);
+ masm.asm.vec_load_const(&rhs_mask, mask.writable());
+ masm.asm.vec_tbl_ext(rhs, mask.inner(), dst);
+ });
+ Ok(())
}
- fn swizzle(&mut self, _dst: WritableReg, _lhs: Reg, _rhs: Reg) -> Result<()> {
- bail!(CodeGenError::unimplemented_masm_instruction())
+ fn swizzle(&mut self, dst: WritableReg, lhs: Reg, rhs: Reg) -> Result<()> {
+ // `tbl` sets lanes whose index is out of range (above 15) to 0, which
+ // is exactly the behavior the Wasm instruction requires, so no
+ // clamping of the indices in `rhs` is needed.
+ self.asm.vec_tbl(lhs, rhs, dst);
+ Ok(())
}
fn atomic_rmw(
@@ -2158,8 +2189,99 @@ impl Masm for MacroAssembler {
Ok(())
}
- fn v128_bitmask(&mut self, _src: Reg, _dst: WritableReg, _size: OperandSize) -> Result<()> {
- bail!(CodeGenError::unimplemented_masm_instruction())
+ fn v128_bitmask(&mut self, src: Reg, dst: WritableReg, size: OperandSize) -> Result<()> {
+ // Each sequence replicates the sign bit across its lane, masks in the
+ // bit position that lane contributes, and sums the lanes together,
+ // following the `vhigh_bits` lowerings in cranelift's
+ // `isa/aarch64/lower.isle`, which use the same masks.
+ let (shift, mask, vector_size) = match size {
+ OperandSize::S8 => (
+ 7,
+ [
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x01, 0x02, 0x04, 0x08, 0x10,
+ 0x20, 0x40, 0x80,
+ ],
+ VectorSize::Size8x16,
+ ),
+ OperandSize::S16 => (
+ 15,
+ [
+ 0x01, 0x00, 0x02, 0x00, 0x04, 0x00, 0x08, 0x00, 0x10, 0x00, 0x20, 0x00, 0x40,
+ 0x00, 0x80, 0x00,
+ ],
+ VectorSize::Size16x8,
+ ),
+ OperandSize::S32 => (
+ 31,
+ [
+ 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08,
+ 0x00, 0x00, 0x00,
+ ],
+ VectorSize::Size32x4,
+ ),
+ // With only two lanes the sign bits are cheaper to move out and
+ // combine in general purpose registers.
+ OperandSize::S64 => {
+ let sign_bit = ImmShift::maybe_from_u64(63).unwrap();
+ self.with_scratch::<IntScratch, _>(|masm, upper| {
+ masm.asm
+ .mov_from_vec(src, upper.writable(), 1, OperandSize::S64);
+ masm.asm.mov_from_vec(src, dst, 0, OperandSize::S64);
+ masm.asm.shift_ir(
+ sign_bit,
+ upper.inner(),
+ upper.writable(),
+ ShiftKind::ShrU,
+ OperandSize::S64,
+ );
+ masm.asm.shift_ir(
+ sign_bit,
+ dst.to_reg(),
+ dst,
+ ShiftKind::ShrU,
+ OperandSize::S64,
+ );
+ masm.asm.shift_ir(
+ ImmShift::maybe_from_u64(1).unwrap(),
+ upper.inner(),
+ upper.writable(),
+ ShiftKind::Shl,
+ OperandSize::S64,
+ );
+ masm.asm
+ .add_rrr(upper.inner(), dst.to_reg(), dst, OperandSize::S64);
+ });
+ return Ok(());
+ }
+ _ => bail!(CodeGenError::unexpected_operand_size()),
+ };
+
+ // `src` is dead after this operation, so it doubles as a temporary.
+ let acc = writable!(src);
+ self.with_scratch::<FloatScratch, _>(|masm, tmp| {
+ masm.asm
+ .vec_shift_imm(VecShiftImmOp::Sshr, shift, src, acc, vector_size);
+ masm.asm.vec_load_const(&mask, tmp.writable());
+ masm.asm
+ .vec_rrr(VecALUOp::And, src, tmp.inner(), acc, vector_size);
+ if let VectorSize::Size8x16 = vector_size {
+ // Interleave the two halves so that each 16-bit lane holds one
+ // byte's worth of bits from each half, letting a single
+ // reduction gather all sixteen.
+ masm.asm.vec_extract(src, src, tmp.writable(), 8);
+ masm.asm
+ .vec_rrr(VecALUOp::Zip1, src, tmp.inner(), acc, VectorSize::Size8x16);
+ }
+ let (reduce_size, lane_size) = match vector_size {
+ VectorSize::Size8x16 | VectorSize::Size16x8 => {
+ (VectorSize::Size16x8, OperandSize::S16)
+ }
+ other => (other, OperandSize::S32),
+ };
+ masm.asm.vec_lanes(VecLanesOp::Addv, src, acc, reduce_size);
+ masm.asm.mov_from_vec(src, dst, 0, lane_size);
+ });
+ Ok(())
}
fn v128_trunc(