Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

winch(aarch64): v128 ABI and value plumbing (#13948)

Daniel Macovei74d98bc56a74
diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs
index ceaa45d550..ae1e531bbf 100644
--- a/crates/test-util/src/wast.rs
+++ b/crates/test-util/src/wast.rs
@@ -416,7 +416,7 @@ impl Compiler {
}
if cfg!(target_arch = "aarch64") {
- return (config.simd() && !config.spec_test()) || config.threads();
+ return config.threads();
}
!cfg!(target_arch = "x86_64")
@@ -518,10 +518,29 @@ impl WastTest {
"misc_testsuite/simd/issue4807.wast",
"misc_testsuite/simd/replace-lane-preserve.wast",
"misc_testsuite/simd/spillslot-size-fuzzbug.wast",
+ "misc_testsuite/simd/interesting-float-splat.wast",
+ "misc_testsuite/simd/issue_3173_select_v128.wast",
+ "misc_testsuite/simd/v128-select.wast",
+ "misc_testsuite/winch/issue-10460.wast",
+ "misc_testsuite/winch/issue-10357.wast",
+ "misc_testsuite/winch/simd_multivalue.wast",
+ "misc_testsuite/simd/unaligned-load.wast",
+ "misc_testsuite/simd/riscv64-replicated-imm5-works.wast",
+ "misc_testsuite/simd/issue6725-no-egraph-panic.wast",
+ "misc_testsuite/winch/replace_lane.wast",
+ "misc_testsuite/simd/v128-equal.wast",
+ "misc_testsuite/winch/issue-10331.wast",
];
if now_supported.iter().any(|part| self.path.ends_with(part)) {
return false;
}
+
+ // SIMD support on winch aarch64 is still incomplete
+ // (bytecodealliance/wasmtime#9925); non-spec simd tests are
+ // expected to fail unless listed in `now_supported` above.
+ if self.config.simd() && !self.config.spec_test() {
+ return true;
+ }
}
if config.compiler.should_fail(&self.config) {
@@ -570,6 +589,11 @@ impl WastTest {
"misc_testsuite/simd/spillslot-size-fuzzbug.wast",
"misc_testsuite/simd/sse-cannot-fold-unaligned-loads.wast",
"misc_testsuite/simd/unaligned-load.wast",
+ "misc_testsuite/simd/riscv64-replicated-imm5-works.wast",
+ "misc_testsuite/simd/issue6725-no-egraph-panic.wast",
+ "misc_testsuite/winch/replace_lane.wast",
+ "misc_testsuite/simd/v128-equal.wast",
+ "misc_testsuite/winch/issue-10331.wast",
"misc_testsuite/simd/v128-select.wast",
"misc_testsuite/winch/issue-10331.wast",
"misc_testsuite/winch/issue-10357.wast",
@@ -666,6 +690,7 @@ impl WastTest {
"misc_testsuite/winch/replace_lane.wast",
"misc_testsuite/simd/riscv64-replicated-imm5-works.wast",
"misc_testsuite/simd/v128-equal.wast",
+ "misc_testsuite/winch/issue-10331.wast",
"spec_testsuite/simd_align.wast",
"spec_testsuite/simd_boolean.wast",
"spec_testsuite/simd_conversions.wast",
@@ -702,6 +727,11 @@ impl WastTest {
"spec_testsuite/simd_bitwise.wast",
"misc_testsuite/simd/load_splat_out_of_bounds.wast",
"misc_testsuite/simd/unaligned-load.wast",
+ "misc_testsuite/simd/riscv64-replicated-imm5-works.wast",
+ "misc_testsuite/simd/issue6725-no-egraph-panic.wast",
+ "misc_testsuite/winch/replace_lane.wast",
+ "misc_testsuite/simd/v128-equal.wast",
+ "misc_testsuite/winch/issue-10331.wast",
"multi-memory/simd_memory-multi.wast",
"misc_testsuite/simd/issue4807.wast",
"spec_testsuite/simd_const.wast",
diff --git a/tests/disas/winch/aarch64/v128_multi_results.wat b/tests/disas/winch/aarch64/v128_multi_results.wat
new file mode 100644
index 0000000000..293041b708
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_multi_results.wat
@@ -0,0 +1,96 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (func $mr (result v128 i32)
+ (v128.const i64x2 1 2)
+ (i32.const 5))
+ (func (export "drive") (result i32) (local $x i32) (local $v v128)
+ (call $mr)
+ (local.set $x)
+ (local.set $v)
+ (local.get $x)))
+;; wasm[0]::function[0]::mr:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x1, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x28
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x84
+;; 2c: mov x9, x1
+;; sub x28, x28, #0x18
+;; mov sp, x28
+;; stur x1, [x28, #0x10]
+;; stur x2, [x28, #8]
+;; stur x0, [x28]
+;; mov x0, #5
+;; sub x28, x28, #0x10
+;; mov sp, x28
+;; ldr q31, #0x90
+;; stur q31, [x28]
+;; ldur x1, [x28, #0x10]
+;; ldur q31, [x28]
+;; add x28, x28, #0x10
+;; mov sp, x28
+;; stur q31, [x1]
+;; add x28, x28, #0x18
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 84: udf #0xc11f
+;; 88: udf #0
+;; 8c: udf #0
+;; 90: udf #1
+;; 94: udf #0
+;; 98: udf #2
+;; 9c: udf #0
+;;
+;; wasm[0]::function[1]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x40
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x140
+;; cc: mov x9, x0
+;; sub x28, x28, #0x30
+;; mov sp, x28
+;; stur x0, [x28, #0x28]
+;; stur x1, [x28, #0x20]
+;; mov x16, #0
+;; stur x16, [x28, #0x18]
+;; stur x16, [x28, #0x10]
+;; stur x16, [x28, #8]
+;; stur x16, [x28]
+;; sub x28, x28, #0x10
+;; mov sp, x28
+;; mov x1, x9
+;; mov x2, x9
+;; add x0, x28, #0
+;; bl #0
+;; 10c: ldur x9, [x28, #0x38]
+;; stur w0, [x28, #0x2c]
+;; ldur q0, [x28]
+;; add x28, x28, #0x10
+;; mov sp, x28
+;; stur q0, [x28]
+;; ldur w0, [x28, #0x1c]
+;; add x28, x28, #0x30
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 140: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_param.wat b/tests/disas/winch/aarch64/v128_param.wat
new file mode 100644
index 0000000000..00e072bb6f
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_param.wat
@@ -0,0 +1,32 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (func (export "f") (param v128) (result i32)
+ (i32.const 7)))
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x20
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x60
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x20
+;; mov sp, x28
+;; stur x0, [x28, #0x18]
+;; stur x1, [x28, #0x10]
+;; stur q0, [x28]
+;; mov x0, #7
+;; add x28, x28, #0x20
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 60: udf #0xc11f
diff --git a/tests/disas/winch/aarch64/v128_select.wat b/tests/disas/winch/aarch64/v128_select.wat
new file mode 100644
index 0000000000..9e7ea833a5
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_select.wat
@@ -0,0 +1,57 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory 1)
+ (func (export "f") (param i32)
+ (v128.store (i32.const 0)
+ (select (v128.const i64x2 1 1) (v128.const i64x2 2 2) (local.get 0)))))
+;; wasm[0]::function[0]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x18
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x94
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x18
+;; mov sp, x28
+;; stur x0, [x28, #0x10]
+;; stur x1, [x28, #8]
+;; stur w2, [x28, #4]
+;; ldur w0, [x28, #4]
+;; ldr q0, #0xa0
+;; ldr q1, #0xb0
+;; cmp w0, #0
+;; b.ne #0x60
+;; 58: mov v0.16b, v0.16b
+;; b #0x64
+;; 60: mov v0.16b, v1.16b
+;; mov x0, #0
+;; ldur x1, [x9, #0x38]
+;; add x1, x1, w0, uxtw
+;; sub sp, x28, #8
+;; stur q0, [x1]
+;; mov sp, x28
+;; add x28, x28, #0x18
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 94: udf #0xc11f
+;; 98: udf #0
+;; 9c: udf #0
+;; a0: udf #2
+;; a4: udf #0
+;; a8: udf #2
+;; ac: udf #0
+;; b0: udf #1
+;; b4: udf #0
+;; b8: udf #1
+;; bc: udf #0
diff --git a/tests/disas/winch/aarch64/v128_stack_params.wat b/tests/disas/winch/aarch64/v128_stack_params.wat
new file mode 100644
index 0000000000..00e6fa7e31
--- /dev/null
+++ b/tests/disas/winch/aarch64/v128_stack_params.wat
@@ -0,0 +1,147 @@
+;;! target = "aarch64"
+;;! test = "winch"
+
+(module
+ (memory 1)
+ (func $take (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (param v128) (result i32)
+ (v128.store (i32.const 0) (local.get 9))
+ (i32.load (i32.const 0)))
+ (func (export "drive") (result i32)
+ (call $take
+ (v128.const i64x2 0 0)
+ (v128.const i64x2 1 1)
+ (v128.const i64x2 2 2)
+ (v128.const i64x2 3 3)
+ (v128.const i64x2 4 4)
+ (v128.const i64x2 5 5)
+ (v128.const i64x2 6 6)
+ (v128.const i64x2 7 7)
+ (v128.const i64x2 8 8)
+ (v128.const i64x2 9 9))))
+;; wasm[0]::function[0]::take:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x90
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x9c
+;; 2c: mov x9, x0
+;; sub x28, x28, #0x90
+;; mov sp, x28
+;; stur x0, [x28, #0x88]
+;; stur x1, [x28, #0x80]
+;; stur q0, [x28, #0x70]
+;; stur q1, [x28, #0x60]
+;; stur q2, [x28, #0x50]
+;; stur q3, [x28, #0x40]
+;; stur q4, [x28, #0x30]
+;; stur q5, [x28, #0x20]
+;; stur q6, [x28, #0x10]
+;; stur q7, [x28]
+;; ldur q0, [x29, #0x20]
+;; mov x0, #0
+;; ldur x1, [x9, #0x38]
+;; add x1, x1, w0, uxtw
+;; stur q0, [x1]
+;; mov x0, #0
+;; ldur x1, [x9, #0x38]
+;; add x1, x1, w0, uxtw
+;; ldur w0, [x1]
+;; add x28, x28, #0x90
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 9c: udf #0xc11f
+;;
+;; wasm[0]::function[1]:
+;; stp x29, x30, [sp, #-0x10]!
+;; mov x29, sp
+;; str x28, [sp, #-0x10]!
+;; mov x28, sp
+;; ldur x16, [x0, #8]
+;; ldur x16, [x16, #0x18]
+;; mov x17, #0
+;; movk x17, #0x30
+;; add x16, x16, x17
+;; cmp sp, x16
+;; b.lo #0x148
+;; cc: mov x9, x0
+;; sub x28, x28, #0x10
+;; mov sp, x28
+;; stur x0, [x28, #8]
+;; stur x1, [x28]
+;; sub x28, x28, #0x20
+;; mov sp, x28
+;; mov x0, x9
+;; mov x1, x9
+;; ldr q0, #0x150
+;; ldr q1, #0x160
+;; ldr q2, #0x170
+;; ldr q3, #0x180
+;; ldr q4, #0x190
+;; ldr q5, #0x1a0
+;; ldr q6, #0x1b0
+;; ldr q7, #0x1c0
+;; ldr q31, #0x1d0
+;; stur q31, [x28]
+;; ldr q31, #0x1e0
+;; stur q31, [x28, #0x10]
+;; bl #0
+;; 124: add x28, x28, #0x20
+;; mov sp, x28
+;; ldur x9, [x28, #8]
+;; add x28, x28, #0x10
+;; mov sp, x28
+;; mov sp, x28
+;; ldr x28, [sp], #0x10
+;; ldp x29, x30, [sp], #0x10
+;; ret
+;; 148: udf #0xc11f
+;; 14c: udf #0
+;; 150: udf #0
+;; 154: udf #0
+;; 158: udf #0
+;; 15c: udf #0
+;; 160: udf #1
+;; 164: udf #0
+;; 168: udf #1
+;; 16c: udf #0
+;; 170: udf #2
+;; 174: udf #0
+;; 178: udf #2
+;; 17c: udf #0
+;; 180: udf #3
+;; 184: udf #0
+;; 188: udf #3
+;; 18c: udf #0
+;; 190: udf #4
+;; 194: udf #0
+;; 198: udf #4
+;; 19c: udf #0
+;; 1a0: udf #5
+;; 1a4: udf #0
+;; 1a8: udf #5
+;; 1ac: udf #0
+;; 1b0: udf #6
+;; 1b4: udf #0
+;; 1b8: udf #6
+;; 1bc: udf #0
+;; 1c0: udf #7
+;; 1c4: udf #0
+;; 1c8: udf #7
+;; 1cc: udf #0
+;; 1d0: udf #8
+;; 1d4: udf #0
+;; 1d8: udf #8
+;; 1dc: udf #0
+;; 1e0: udf #9
+;; 1e4: udf #0
+;; 1e8: udf #9
+;; 1ec: udf #0
diff --git a/winch/codegen/src/isa/aarch64/abi.rs b/winch/codegen/src/isa/aarch64/abi.rs
index 200e837f28..ad933eea20 100644
--- a/winch/codegen/src/isa/aarch64/abi.rs
+++ b/winch/codegen/src/isa/aarch64/abi.rs
@@ -153,7 +153,8 @@ impl Aarch64ABI {
(index_env.next_gpr().map(regs::xreg), ty)
}
- ty @ (WasmValType::F32 | WasmValType::F64) => {
+ // v128 is passed in a vector register, like f32 and f64.
+ ty @ (WasmValType::F32 | WasmValType::F64 | WasmValType::V128) => {
(index_env.next_fpr().map(regs::vreg), ty)
}
@@ -163,28 +164,31 @@ impl Aarch64ABI {
}
_ => bail!(CodeGenError::unsupported_wasm_type()),
},
-
- _ => bail!(CodeGenError::unsupported_wasm_type()),
};
let ty_size = <Self as ABI>::sizeof(wasm_arg);
let default = || {
- let arg = ABIOperand::stack_offset(stack_offset, *ty, ty_size as u32);
- let slot_size = Self::stack_slot_size();
// Stack slots for parameters are aligned to a fixed slot size,
- // in the case of Aarch64, 8 bytes.
+ // 8 bytes if the type size is 8 or less and type-sized aligned
+ // if the type size is greater than 8 bytes.
// For the non-default calling convention, stack slots for
// return values are type-sized aligned.
// For the default calling convention, we don't type-size align,
// given that results on the stack must match spills generated
// from within the compiler, which are not type-size aligned.
- let next_stack = if params_or_returns == ParamsOrReturns::Params {
- align_to(stack_offset, slot_size as u32) + (slot_size as u32)
+ let (arg_offset, next_stack) = if params_or_returns == ParamsOrReturns::Params {
+ let slot_size = (Self::stack_slot_size() as u32).max(ty_size as u32);
+ let offset = align_to(stack_offset, slot_size);
+ (offset, offset + slot_size)
} else if call_conv.is_default() {
- stack_offset + (ty_size as u32)
+ (stack_offset, stack_offset + (ty_size as u32))
} else {
- align_to(stack_offset, ty_size as u32) + (ty_size as u32)
+ (
+ stack_offset,
+ align_to(stack_offset, ty_size as u32) + (ty_size as u32),
+ )
};
+ let arg = ABIOperand::stack_offset(arg_offset, *ty, ty_size as u32);
(arg, next_stack)
};
Ok(reg.map_or_else(default, |reg| {
diff --git a/winch/codegen/src/isa/aarch64/asm.rs b/winch/codegen/src/isa/aarch64/asm.rs
index ec9cb39766..aa2a2912bc 100644
--- a/winch/codegen/src/isa/aarch64/asm.rs
+++ b/winch/codegen/src/isa/aarch64/asm.rs
@@ -1144,6 +1144,14 @@ impl Assembler {
cond,
});
}
+ OperandSize::S128 => {
+ self.emit(Inst::VecCSel {
+ rd: rd.map(Into::into),
+ rn: rn.into(),
+ rm: rm.into(),
+ cond,
+ });
+ }
_ => todo!(),
}
}
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index d747f8ae72..7bba6eaa1a 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -364,7 +364,7 @@ impl Masm for MacroAssembler {
})
})?;
}
- imm @ (I::F32(_) | I::F64(_)) => {
+ imm @ (I::F32(_) | I::F64(_) | I::V128(_)) => {
self.with_scratch::<FloatScratch, _>(|masm, scratch| -> Result<()> {
masm.asm.mov_ir(scratch.writable(), imm, imm.size());
dst.to_addressing_mode(masm, size, |masm, mem| {
@@ -373,7 +373,6 @@ impl Masm for MacroAssembler {
})
})?;
}
- _ => bail!(CodeGenError::unsupported_wasm_type()),
};
Ok(())
}
@@ -444,15 +443,9 @@ impl Masm for MacroAssembler {
fn wasm_load(&mut self, src: Self::Address, dst: WritableReg, kind: LoadKind) -> Result<()> {
let size = kind.derive_operand_size();
self.with_aligned_sp(|masm| match &kind {
- LoadKind::Operand(_) => {
- if size == OperandSize::S128 {
- bail!(CodeGenError::UnimplementedWasmLoadKind)
- } else {
- src.to_addressing_mode(masm, size, |masm, mem| {
- Ok(masm.asm.uload(mem, dst, size, UNTRUSTED_FLAGS))
- })
- }
- }
+ LoadKind::Operand(_) => src.to_addressing_mode(masm, size, |masm, mem| {
+ Ok(masm.asm.uload(mem, dst, size, UNTRUSTED_FLAGS))
+ }),
LoadKind::Splat(_) => bail!(CodeGenError::UnimplementedWasmLoadKind),
LoadKind::ScalarExtend(extend_kind) => {
if extend_kind.signed() {