A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch(aarch64): v128 bitwise ops (#13921)
Daniel Macovei67f21c20e039
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex b724334d8d..623177037e 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -513,7 +513,10 @@ impl WastTest {}if config.compiler == Compiler::Winch && cfg!(target_arch = "aarch64") {- let now_supported = ["misc_testsuite/winch/v128_load_lane_invalid_address.wast"];+ let now_supported = [+ "misc_testsuite/winch/v128_load_lane_invalid_address.wast",+ "misc_testsuite/simd/issue4807.wast",+ ];if now_supported.iter().any(|part| self.path.ends_with(part)) {return false;}@@ -557,7 +560,6 @@ impl WastTest {"misc_testsuite/simd/cvt-from-uint.wast","misc_testsuite/simd/edge-of-memory.wast","misc_testsuite/simd/interesting-float-splat.wast",- "misc_testsuite/simd/issue4807.wast","misc_testsuite/simd/issue6725-no-egraph-panic.wast","misc_testsuite/simd/issue_3173_select_v128.wast","misc_testsuite/simd/issue_3327_bnot_lowering.wast",diff --git a/tests/disas/winch/aarch64/v128_ops/and.wat b/tests/disas/winch/aarch64/v128_ops/and.watnew file mode 100644index 0000000000..afacb83c28--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/and.wat@@ -0,0 +1,51 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory 1)+ (func (export "run")+ (v128.store (i32.const 0)+ (v128.and+ (v128.const i64x2 0 -1)+ (v128.const i64x2 -1 0)))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x10+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x74+;; 2c: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; ldr q0, #0x80+;; ldr q1, #0x90+;; and v1.16b, v1.16b, v0.16b+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; stur q1, [x1]+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 74: udf #0xc11f+;; 78: udf #0+;; 7c: udf #0+;; 80: .byte 0xff, 0xff, 0xff, 0xff+;; 84: .byte 0xff, 0xff, 0xff, 0xff+;; 88: udf #0+;; 8c: udf #0+;; 90: udf #0+;; 94: udf #0+;; 98: .byte 0xff, 0xff, 0xff, 0xff+;; 9c: .byte 0xff, 0xff, 0xff, 0xffdiff --git a/tests/disas/winch/aarch64/v128_ops/andnot.wat b/tests/disas/winch/aarch64/v128_ops/andnot.watnew file mode 100644index 0000000000..a410d73229--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/andnot.wat@@ -0,0 +1,51 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory 1)+ (func (export "run")+ (v128.store (i32.const 0)+ (v128.andnot+ (v128.const i64x2 0 -1)+ (v128.const i64x2 -1 0)))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x10+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x74+;; 2c: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; ldr q0, #0x80+;; ldr q1, #0x90+;; bic v1.16b, v1.16b, v0.16b+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; stur q1, [x1]+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 74: udf #0xc11f+;; 78: udf #0+;; 7c: udf #0+;; 80: .byte 0xff, 0xff, 0xff, 0xff+;; 84: .byte 0xff, 0xff, 0xff, 0xff+;; 88: udf #0+;; 8c: udf #0+;; 90: udf #0+;; 94: udf #0+;; 98: .byte 0xff, 0xff, 0xff, 0xff+;; 9c: .byte 0xff, 0xff, 0xff, 0xffdiff --git a/tests/disas/winch/aarch64/v128_ops/any_true.wat b/tests/disas/winch/aarch64/v128_ops/any_true.watnew file mode 100644index 0000000000..9d40901f8a--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/any_true.wat@@ -0,0 +1,39 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func (export "run") (result i32)+ (v128.any_true (v128.const i64x2 0 1))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x10+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x6c+;; 2c: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; ldr q0, #0x70+;; umaxp v31.4s, v0.4s, v0.4s+;; mov x0, v31.d[0]+;; cmp x0, #0+;; cset x0, ne+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 6c: udf #0xc11f+;; 70: udf #0+;; 74: udf #0+;; 78: udf #1+;; 7c: udf #0diff --git a/tests/disas/winch/aarch64/v128_ops/bitselect.wat b/tests/disas/winch/aarch64/v128_ops/bitselect.watnew file mode 100644index 0000000000..e1fb3a6b73--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/bitselect.wat@@ -0,0 +1,56 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory 1)+ (func (export "run")+ (v128.store (i32.const 0)+ (v128.bitselect+ (v128.const i64x2 1 2)+ (v128.const i64x2 3 4)+ (v128.const i64x2 -1 0)))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x10+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x7c+;; 2c: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; ldr q0, #0x80+;; ldr q1, #0x90+;; ldr q2, #0xa0+;; mov v3.16b, v0.16b+;; bsl v3.16b, v2.16b, v1.16b+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; stur q3, [x1]+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 7c: udf #0xc11f+;; 80: .byte 0xff, 0xff, 0xff, 0xff+;; 84: .byte 0xff, 0xff, 0xff, 0xff+;; 88: udf #0+;; 8c: udf #0+;; 90: udf #3+;; 94: udf #0+;; 98: udf #4+;; 9c: udf #0+;; a0: udf #1+;; a4: udf #0+;; a8: udf #2+;; ac: udf #0diff --git a/tests/disas/winch/aarch64/v128_ops/not.wat b/tests/disas/winch/aarch64/v128_ops/not.watnew file mode 100644index 0000000000..17621a624d--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/not.wat@@ -0,0 +1,45 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory 1)+ (func (export "run")+ (v128.store (i32.const 0)+ (v128.not (v128.const i32x4 1 2 3 4)))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x10+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x70+;; 2c: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; ldr q0, #0x80+;; mvn v0.16b, v0.16b+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; stur q0, [x1]+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 70: udf #0xc11f+;; 74: udf #0+;; 78: udf #0+;; 7c: udf #0+;; 80: udf #1+;; 84: udf #2+;; 88: udf #3+;; 8c: udf #4diff --git a/tests/disas/winch/aarch64/v128_ops/or.wat b/tests/disas/winch/aarch64/v128_ops/or.watnew file mode 100644index 0000000000..341d7f3e6b--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/or.wat@@ -0,0 +1,51 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory 1)+ (func (export "run")+ (v128.store (i32.const 0)+ (v128.or+ (v128.const i64x2 0 -1)+ (v128.const i64x2 -1 0)))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x10+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x74+;; 2c: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; ldr q0, #0x80+;; ldr q1, #0x90+;; orr v1.16b, v0.16b, v1.16b+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; stur q1, [x1]+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 74: udf #0xc11f+;; 78: udf #0+;; 7c: udf #0+;; 80: .byte 0xff, 0xff, 0xff, 0xff+;; 84: .byte 0xff, 0xff, 0xff, 0xff+;; 88: udf #0+;; 8c: udf #0+;; 90: udf #0+;; 94: udf #0+;; 98: .byte 0xff, 0xff, 0xff, 0xff+;; 9c: .byte 0xff, 0xff, 0xff, 0xffdiff --git a/tests/disas/winch/aarch64/v128_ops/xor.wat b/tests/disas/winch/aarch64/v128_ops/xor.watnew file mode 100644index 0000000000..0702999068--- /dev/null+++ b/tests/disas/winch/aarch64/v128_ops/xor.wat@@ -0,0 +1,51 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory 1)+ (func (export "run")+ (v128.store (i32.const 0)+ (v128.xor+ (v128.const i64x2 0 -1)+ (v128.const i64x2 -1 0)))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x10+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x74+;; 2c: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; ldr q0, #0x80+;; ldr q1, #0x90+;; eor v1.16b, v0.16b, v1.16b+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; stur q1, [x1]+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 74: udf #0xc11f+;; 78: udf #0+;; 7c: udf #0+;; 80: .byte 0xff, 0xff, 0xff, 0xff+;; 84: .byte 0xff, 0xff, 0xff, 0xff+;; 88: udf #0+;; 8c: udf #0+;; 90: udf #0+;; 94: udf #0+;; 98: .byte 0xff, 0xff, 0xff, 0xff+;; 9c: .byte 0xff, 0xff, 0xff, 0xffdiff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rsindex 85284ddb26..d461456de6 100644--- a/winch/codegen/src/isa/aarch64/asm.rs+++ b/winch/codegen/src/isa/aarch64/asm.rs@@ -15,7 +15,7 @@ use crate::{use cranelift_codegen::isa::aarch64;use cranelift_codegen::isa::aarch64::inst::emit::{enc_arith_rrr, enc_move_wide, enc_movk};use cranelift_codegen::isa::aarch64::inst::{- ASIMDFPModImm, FpuToIntOp, MoveWideConst, NZCV, UImm5,+ ASIMDFPModImm, FpuToIntOp, MoveWideConst, NZCV, UImm5, VecALUModOp,};use cranelift_codegen::{Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, MachInstEmitState, MachLabel,@@ -26,7 +26,7 @@ use cranelift_codegen::{FPULeftShiftImm, FPUOp1, FPUOp2,FPUOpRI::{self, UShr32, UShr64},FPUOpRIMod, FPURightShiftImm, FpuRoundMode, Imm12, ImmLogic, ImmShift, Inst, IntToFpuOp,- PairAMode, ScalarSize, VecLanesOp, VecMisc2, VectorSize,+ PairAMode, ScalarSize, VecALUOp, VecLanesOp, VecMisc2, VectorSize,emit::{EmitInfo, EmitState},},settings,@@ -358,6 +358,10 @@ impl Assembler {rd: writable,rn: rn.into(),},+ OperandSize::S128 => Inst::FpuMove128 {+ rd: writable,+ rn: rn.into(),+ },_ => unreachable!(),};@@ -419,6 +423,53 @@ impl Assembler {self.alu_rrr(ALUOp::Adc, rm, rn, rd, size);}+ /// Vector three-register ALU operation whose destination is also an input+ pub fn vec_rrr_mod(+ &mut self,+ alu_op: VecALUModOp,+ rn: Reg,+ rm: Reg,+ rd: WritableReg,+ size: VectorSize,+ ) {+ self.emit(Inst::VecRRRMod {+ alu_op,+ rd: rd.map(Into::into),+ ri: rd.to_reg().into(),+ rn: rn.into(),+ rm: rm.into(),+ size,+ });+ }++ /// Vector two register miscellaneous instruction.+ pub fn vec_misc(&mut self, op: VecMisc2, rn: Reg, rd: WritableReg, size: VectorSize) {+ self.emit(Inst::VecMisc {+ op,+ rd: rd.map(Into::into),+ rn: rn.into(),+ size,+ });+ }++ /// Vector ALU op+ pub fn vec_rrr(+ &mut self,+ alu_op: VecALUOp,+ rn: Reg,+ rm: Reg,+ rd: WritableReg,+ size: VectorSize,+ ) {+ self.emit(Inst::VecRRR {+ alu_op,+ rd: rd.map(Into::into),+ rn: rn.into(),+ rm: rm.into(),+ size,+ });+ }+/// Add across Vector.pub fn addv(&mut self, rn: Reg, rd: WritableReg, size: VectorSize) {self.emit(Inst::VecLanes {diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex e9a7d51d9c..b4e83b385b 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -33,7 +33,8 @@ use cranelift_codegen::{ir::{MemFlagsData, RelSourceLoc, SourceLoc, types},isa::aarch64,isa::aarch64::inst::{- self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, VectorSize,+ self, Cond, ExtendOp, Imm12, ImmLogic, ImmShift, SImm7Scaled, SImm9, VecALUModOp, VecALUOp,+ VecMisc2, VectorSize,},settings,};@@ -1430,42 +1431,65 @@ impl Masm for MacroAssembler {bail!(CodeGenError::unimplemented_masm_instruction())}- fn v128_not(&mut self, _dst: WritableReg) -> Result<()> {- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))+ fn v128_not(&mut self, dst: WritableReg) -> Result<()> {+ self.asm+ .vec_misc(VecMisc2::Not, dst.to_reg(), dst, VectorSize::Size32x4);+ Ok(())}fn fence(&mut self) -> Result<()> {Err(format_err!(CodeGenError::unimplemented_masm_instruction()))}- fn v128_and(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()> {- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))+ fn v128_and(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()> {+ self.asm+ .vec_rrr(VecALUOp::And, src1, src2, dst, VectorSize::Size32x4);+ Ok(())}- fn v128_and_not(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()> {- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))+ fn v128_and_not(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()> {+ self.asm+ .vec_rrr(VecALUOp::Bic, src2, src1, dst, VectorSize::Size32x4);+ Ok(())}- fn v128_or(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()> {- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))+ fn v128_or(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()> {+ self.asm+ .vec_rrr(VecALUOp::Orr, src1, src2, dst, VectorSize::Size32x4);+ Ok(())}- fn v128_xor(&mut self, _src1: Reg, _src2: Reg, _dst: WritableReg) -> Result<()> {- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))+ fn v128_xor(&mut self, src1: Reg, src2: Reg, dst: WritableReg) -> Result<()> {+ self.asm+ .vec_rrr(VecALUOp::Eor, src1, src2, dst, VectorSize::Size32x4);+ Ok(())}- fn v128_bitselect(- &mut self,- _src1: Reg,- _src2: Reg,- _mask: Reg,- _dst: WritableReg,- ) -> Result<()> {- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))+ fn v128_bitselect(&mut self, src1: Reg, src2: Reg, mask: Reg, dst: WritableReg) -> Result<()> {+ self.asm.fmov_rr(mask, dst, OperandSize::S128);+ self.asm+ .vec_rrr_mod(VecALUModOp::Bsl, src1, src2, dst, VectorSize::Size32x4);+ Ok(())}- fn v128_any_true(&mut self, _src: Reg, _dst: WritableReg) -> Result<()> {- Err(format_err!(CodeGenError::unimplemented_masm_instruction()))+ fn v128_any_true(&mut self, src: Reg, dst: WritableReg) -> Result<()> {+ self.with_scratch::<FloatScratch, _>(|masm, tmp| {+ masm.asm.vec_rrr(+ VecALUOp::Umaxp,+ src,+ src,+ tmp.writable(),+ VectorSize::Size32x4,+ );+ masm.asm.mov_from_vec(tmp.inner(), dst, 0, OperandSize::S64);+ });+ self.asm.subs_ir(+ Imm12::maybe_from_u64(0).unwrap(),+ dst.to_reg(),+ OperandSize::S64,+ );+ self.asm.cset(dst, Cond::Ne);+ Ok(())}fn v128_convert(&mut self, _src: Reg, _dst: WritableReg, _kind: V128ConvertKind) -> Result<()> {