A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch(aarch64): v128 ABI and value plumbing (#13948)
Daniel Macovei74d98bc56a74
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rsindex ceaa45d550..ae1e531bbf 100644--- a/crates/test-util/src/wast.rs+++ b/crates/test-util/src/wast.rs@@ -416,7 +416,7 @@ impl Compiler {}if cfg!(target_arch = "aarch64") {- return (config.simd() && !config.spec_test()) || config.threads();+ return config.threads();}!cfg!(target_arch = "x86_64")@@ -518,10 +518,29 @@ impl WastTest {"misc_testsuite/simd/issue4807.wast","misc_testsuite/simd/replace-lane-preserve.wast","misc_testsuite/simd/spillslot-size-fuzzbug.wast",+ "misc_testsuite/simd/interesting-float-splat.wast",+ "misc_testsuite/simd/issue_3173_select_v128.wast",+ "misc_testsuite/simd/v128-select.wast",+ "misc_testsuite/winch/issue-10460.wast",+ "misc_testsuite/winch/issue-10357.wast",+ "misc_testsuite/winch/simd_multivalue.wast",+ "misc_testsuite/simd/unaligned-load.wast",+ "misc_testsuite/simd/riscv64-replicated-imm5-works.wast",+ "misc_testsuite/simd/issue6725-no-egraph-panic.wast",+ "misc_testsuite/winch/replace_lane.wast",+ "misc_testsuite/simd/v128-equal.wast",+ "misc_testsuite/winch/issue-10331.wast",];if now_supported.iter().any(|part| self.path.ends_with(part)) {return false;}++ // SIMD support on winch aarch64 is still incomplete+ // (bytecodealliance/wasmtime#9925); non-spec simd tests are+ // expected to fail unless listed in `now_supported` above.+ if self.config.simd() && !self.config.spec_test() {+ return true;+ }}if config.compiler.should_fail(&self.config) {@@ -570,6 +589,11 @@ impl WastTest {"misc_testsuite/simd/spillslot-size-fuzzbug.wast","misc_testsuite/simd/sse-cannot-fold-unaligned-loads.wast","misc_testsuite/simd/unaligned-load.wast",+ "misc_testsuite/simd/riscv64-replicated-imm5-works.wast",+ "misc_testsuite/simd/issue6725-no-egraph-panic.wast",+ "misc_testsuite/winch/replace_lane.wast",+ "misc_testsuite/simd/v128-equal.wast",+ "misc_testsuite/winch/issue-10331.wast","misc_testsuite/simd/v128-select.wast","misc_testsuite/winch/issue-10331.wast","misc_testsuite/winch/issue-10357.wast",@@ -666,6 +690,7 @@ impl WastTest {"misc_testsuite/winch/replace_lane.wast","misc_testsuite/simd/riscv64-replicated-imm5-works.wast","misc_testsuite/simd/v128-equal.wast",+ "misc_testsuite/winch/issue-10331.wast","spec_testsuite/simd_align.wast","spec_testsuite/simd_boolean.wast","spec_testsuite/simd_conversions.wast",@@ -702,6 +727,11 @@ impl WastTest {"spec_testsuite/simd_bitwise.wast","misc_testsuite/simd/load_splat_out_of_bounds.wast","misc_testsuite/simd/unaligned-load.wast",+ "misc_testsuite/simd/riscv64-replicated-imm5-works.wast",+ "misc_testsuite/simd/issue6725-no-egraph-panic.wast",+ "misc_testsuite/winch/replace_lane.wast",+ "misc_testsuite/simd/v128-equal.wast",+ "misc_testsuite/winch/issue-10331.wast","multi-memory/simd_memory-multi.wast","misc_testsuite/simd/issue4807.wast","spec_testsuite/simd_const.wast",diff --git a/tests/disas/winch/aarch64/v128_multi_results.wat b/tests/disas/winch/aarch64/v128_multi_results.watnew file mode 100644index 0000000000..293041b708--- /dev/null+++ b/tests/disas/winch/aarch64/v128_multi_results.wat@@ -0,0 +1,96 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func $mr (result v128 i32)+ (v128.const i64x2 1 2)+ (i32.const 5))+ (func (export "drive") (result i32) (local $x i32) (local $v v128)+ (call $mr)+ (local.set $x)+ (local.set $v)+ (local.get $x)))+;; wasm[0]::function[0]::mr:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x1, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x28+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x84+;; 2c: mov x9, x1+;; sub x28, x28, #0x18+;; mov sp, x28+;; stur x1, [x28, #0x10]+;; stur x2, [x28, #8]+;; stur x0, [x28]+;; mov x0, #5+;; sub x28, x28, #0x10+;; mov sp, x28+;; ldr q31, #0x90+;; stur q31, [x28]+;; ldur x1, [x28, #0x10]+;; ldur q31, [x28]+;; add x28, x28, #0x10+;; mov sp, x28+;; stur q31, [x1]+;; add x28, x28, #0x18+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 84: udf #0xc11f+;; 88: udf #0+;; 8c: udf #0+;; 90: udf #1+;; 94: udf #0+;; 98: udf #2+;; 9c: udf #0+;;+;; wasm[0]::function[1]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x40+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x140+;; cc: mov x9, x0+;; sub x28, x28, #0x30+;; mov sp, x28+;; stur x0, [x28, #0x28]+;; stur x1, [x28, #0x20]+;; mov x16, #0+;; stur x16, [x28, #0x18]+;; stur x16, [x28, #0x10]+;; stur x16, [x28, #8]+;; stur x16, [x28]+;; sub x28, x28, #0x10+;; mov sp, x28+;; mov x1, x9+;; mov x2, x9+;; add x0, x28, #0+;; bl #0+;; 10c: ldur x9, [x28, #0x38]+;; stur w0, [x28, #0x2c]+;; ldur q0, [x28]+;; add x28, x28, #0x10+;; mov sp, x28+;; stur q0, [x28]+;; ldur w0, [x28, #0x1c]+;; add x28, x28, #0x30+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 140: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_param.wat b/tests/disas/winch/aarch64/v128_param.watnew file mode 100644index 0000000000..00e072bb6f--- /dev/null+++ b/tests/disas/winch/aarch64/v128_param.wat@@ -0,0 +1,32 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (func (export "f") (param v128) (result i32)+ (i32.const 7)))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x20+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x60+;; 2c: mov x9, x0+;; sub x28, x28, #0x20+;; mov sp, x28+;; stur x0, [x28, #0x18]+;; stur x1, [x28, #0x10]+;; stur q0, [x28]+;; mov x0, #7+;; add x28, x28, #0x20+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 60: udf #0xc11fdiff --git a/tests/disas/winch/aarch64/v128_select.wat b/tests/disas/winch/aarch64/v128_select.watnew file mode 100644index 0000000000..9e7ea833a5--- /dev/null+++ b/tests/disas/winch/aarch64/v128_select.wat@@ -0,0 +1,57 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory 1)+ (func (export "f") (param i32)+ (v128.store (i32.const 0)+ (select (v128.const i64x2 1 1) (v128.const i64x2 2 2) (local.get 0)))))+;; wasm[0]::function[0]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x18+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x94+;; 2c: mov x9, x0+;; sub x28, x28, #0x18+;; mov sp, x28+;; stur x0, [x28, #0x10]+;; stur x1, [x28, #8]+;; stur w2, [x28, #4]+;; ldur w0, [x28, #4]+;; ldr q0, #0xa0+;; ldr q1, #0xb0+;; cmp w0, #0+;; b.ne #0x60+;; 58: mov v0.16b, v0.16b+;; b #0x64+;; 60: mov v0.16b, v1.16b+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; sub sp, x28, #8+;; stur q0, [x1]+;; mov sp, x28+;; add x28, x28, #0x18+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 94: udf #0xc11f+;; 98: udf #0+;; 9c: udf #0+;; a0: udf #2+;; a4: udf #0+;; a8: udf #2+;; ac: udf #0+;; b0: udf #1+;; b4: udf #0+;; b8: udf #1+;; bc: udf #0diff --git a/tests/disas/winch/aarch64/v128_stack_params.wat b/tests/disas/winch/aarch64/v128_stack_params.watnew file mode 100644index 0000000000..00e6fa7e31--- /dev/null+++ b/tests/disas/winch/aarch64/v128_stack_params.wat@@ -0,0 +1,147 @@+;;! target = "aarch64"+;;! test = "winch"++(module+ (memory 1)+ (func $take (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (result i32)+ (v128.store (i32.const 0) (local.get 9))+ (i32.load (i32.const 0)))+ (func (export "drive") (result i32)+ (call $take+ (v128.const i64x2 0 0)+ (v128.const i64x2 1 1)+ (v128.const i64x2 2 2)+ (v128.const i64x2 3 3)+ (v128.const i64x2 4 4)+ (v128.const i64x2 5 5)+ (v128.const i64x2 6 6)+ (v128.const i64x2 7 7)+ (v128.const i64x2 8 8)+ (v128.const i64x2 9 9))))+;; wasm[0]::function[0]::take:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x90+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x9c+;; 2c: mov x9, x0+;; sub x28, x28, #0x90+;; mov sp, x28+;; stur x0, [x28, #0x88]+;; stur x1, [x28, #0x80]+;; stur q0, [x28, #0x70]+;; stur q1, [x28, #0x60]+;; stur q2, [x28, #0x50]+;; stur q3, [x28, #0x40]+;; stur q4, [x28, #0x30]+;; stur q5, [x28, #0x20]+;; stur q6, [x28, #0x10]+;; stur q7, [x28]+;; ldur q0, [x29, #0x20]+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; stur q0, [x1]+;; mov x0, #0+;; ldur x1, [x9, #0x38]+;; add x1, x1, w0, uxtw+;; ldur w0, [x1]+;; add x28, x28, #0x90+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 9c: udf #0xc11f+;;+;; wasm[0]::function[1]:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x30+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x148+;; cc: mov x9, x0+;; sub x28, x28, #0x10+;; mov sp, x28+;; stur x0, [x28, #8]+;; stur x1, [x28]+;; sub x28, x28, #0x20+;; mov sp, x28+;; mov x0, x9+;; mov x1, x9+;; ldr q0, #0x150+;; ldr q1, #0x160+;; ldr q2, #0x170+;; ldr q3, #0x180+;; ldr q4, #0x190+;; ldr q5, #0x1a0+;; ldr q6, #0x1b0+;; ldr q7, #0x1c0+;; ldr q31, #0x1d0+;; stur q31, [x28]+;; ldr q31, #0x1e0+;; stur q31, [x28, #0x10]+;; bl #0+;; 124: add x28, x28, #0x20+;; mov sp, x28+;; ldur x9, [x28, #8]+;; add x28, x28, #0x10+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; ret+;; 148: udf #0xc11f+;; 14c: udf #0+;; 150: udf #0+;; 154: udf #0+;; 158: udf #0+;; 15c: udf #0+;; 160: udf #1+;; 164: udf #0+;; 168: udf #1+;; 16c: udf #0+;; 170: udf #2+;; 174: udf #0+;; 178: udf #2+;; 17c: udf #0+;; 180: udf #3+;; 184: udf #0+;; 188: udf #3+;; 18c: udf #0+;; 190: udf #4+;; 194: udf #0+;; 198: udf #4+;; 19c: udf #0+;; 1a0: udf #5+;; 1a4: udf #0+;; 1a8: udf #5+;; 1ac: udf #0+;; 1b0: udf #6+;; 1b4: udf #0+;; 1b8: udf #6+;; 1bc: udf #0+;; 1c0: udf #7+;; 1c4: udf #0+;; 1c8: udf #7+;; 1cc: udf #0+;; 1d0: udf #8+;; 1d4: udf #0+;; 1d8: udf #8+;; 1dc: udf #0+;; 1e0: udf #9+;; 1e4: udf #0+;; 1e8: udf #9+;; 1ec: udf #0diff --git a/winch/codegen/src/isa/aarch64/abi.rs b/winch/codegen/src/isa/aarch64/abi.rsindex 200e837f28..ad933eea20 100644--- a/winch/codegen/src/isa/aarch64/abi.rs+++ b/winch/codegen/src/isa/aarch64/abi.rs@@ -153,7 +153,8 @@ impl Aarch64ABI {(index_env.next_gpr().map(regs::xreg), ty)}- ty @ (WasmValType::F32 | WasmValType::F64) => {+ // v128 is passed in a vector register, like f32 and f64.+ ty @ (WasmValType::F32 | WasmValType::F64 | WasmValType::V128) => {(index_env.next_fpr().map(regs::vreg), ty)}@@ -163,28 +164,31 @@ impl Aarch64ABI {}_ => bail!(CodeGenError::unsupported_wasm_type()),},-- _ => bail!(CodeGenError::unsupported_wasm_type()),};let ty_size = <Self as ABI>::sizeof(wasm_arg);let default = || {- let arg = ABIOperand::stack_offset(stack_offset, *ty, ty_size as u32);- let slot_size = Self::stack_slot_size();// Stack slots for parameters are aligned to a fixed slot size,- // in the case of Aarch64, 8 bytes.+ // 8 bytes if the type size is 8 or less and type-sized aligned+ // if the type size is greater than 8 bytes.// For the non-default calling convention, stack slots for// return values are type-sized aligned.// For the default calling convention, we don't type-size align,// given that results on the stack must match spills generated// from within the compiler, which are not type-size aligned.- let next_stack = if params_or_returns == ParamsOrReturns::Params {- align_to(stack_offset, slot_size as u32) + (slot_size as u32)+ let (arg_offset, next_stack) = if params_or_returns == ParamsOrReturns::Params {+ let slot_size = (Self::stack_slot_size() as u32).max(ty_size as u32);+ let offset = align_to(stack_offset, slot_size);+ (offset, offset + slot_size)} else if call_conv.is_default() {- stack_offset + (ty_size as u32)+ (stack_offset, stack_offset + (ty_size as u32))} else {- align_to(stack_offset, ty_size as u32) + (ty_size as u32)+ (+ stack_offset,+ align_to(stack_offset, ty_size as u32) + (ty_size as u32),+ )};+ let arg = ABIOperand::stack_offset(arg_offset, *ty, ty_size as u32);(arg, next_stack)};Ok(reg.map_or_else(default, |reg| {diff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rsindex ec9cb39766..aa2a2912bc 100644--- a/winch/codegen/src/isa/aarch64/asm.rs+++ b/winch/codegen/src/isa/aarch64/asm.rs@@ -1144,6 +1144,14 @@ impl Assembler {cond,});}+ OperandSize::S128 => {+ self.emit(Inst::VecCSel {+ rd: rd.map(Into::into),+ rn: rn.into(),+ rm: rm.into(),+ cond,+ });+ }_ => todo!(),}}diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex d747f8ae72..7bba6eaa1a 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -364,7 +364,7 @@ impl Masm for MacroAssembler {})})?;}- imm @ (I::F32(_) | I::F64(_)) => {+ imm @ (I::F32(_) | I::F64(_) | I::V128(_)) => {self.with_scratch::<FloatScratch, _>(|masm, scratch| -> Result<()> {masm.asm.mov_ir(scratch.writable(), imm, imm.size());dst.to_addressing_mode(masm, size, |masm, mem| {@@ -373,7 +373,6 @@ impl Masm for MacroAssembler {})})?;}- _ => bail!(CodeGenError::unsupported_wasm_type()),};Ok(())}@@ -444,15 +443,9 @@ impl Masm for MacroAssembler {fn wasm_load(&mut self, src: Self::Address, dst: WritableReg, kind: LoadKind) -> Result<()> {let size = kind.derive_operand_size();self.with_aligned_sp(|masm| match &kind {- LoadKind::Operand(_) => {- if size == OperandSize::S128 {- bail!(CodeGenError::UnimplementedWasmLoadKind)- } else {- src.to_addressing_mode(masm, size, |masm, mem| {- Ok(masm.asm.uload(mem, dst, size, UNTRUSTED_FLAGS))- })- }- }+ LoadKind::Operand(_) => src.to_addressing_mode(masm, size, |masm, mem| {+ Ok(masm.asm.uload(mem, dst, size, UNTRUSTED_FLAGS))+ }),LoadKind::Splat(_) => bail!(CodeGenError::UnimplementedWasmLoadKind),LoadKind::ScalarExtend(extend_kind) => {if extend_kind.signed() {