A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: aarch64: fix `preserve-all` to save full vector registers. (#12944)
Chris Fallin5b5e3573d7de
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rsindex 6bab37ac03..cd312d43e7 100644--- a/cranelift/codegen/src/isa/aarch64/abi.rs+++ b/cranelift/codegen/src/isa/aarch64/abi.rs@@ -38,7 +38,10 @@ impl From<StackAMode> for AMode {// Returns the size of stack space needed to store the// `clobbered_callee_saved` registers.-fn compute_clobber_size(clobbered_callee_saves: &[Writable<RealReg>]) -> u32 {+fn compute_clobber_size(+ call_conv: isa::CallConv,+ clobbered_callee_saves: &[Writable<RealReg>],+) -> u32 {let mut int_regs = 0;let mut vec_regs = 0;for ® in clobbered_callee_saves {@@ -55,16 +58,22 @@ fn compute_clobber_size(clobbered_callee_saves: &[Writable<RealReg>]) -> u32 {// Round up to multiple of 2, to keep 16-byte stack alignment.let int_save_bytes = (int_regs + (int_regs & 1)) * 8;- // The Procedure Call Standard for the Arm 64-bit Architecture- // (AAPCS64, including several related ABIs such as the one used by- // Windows) mandates saving only the bottom 8 bytes of the vector- // registers, so we round up the number of registers to ensure- // proper stack alignment (similarly to the situation with- // `int_reg`).- let vec_reg_size = 8;- let vec_save_padding = vec_regs & 1;- // FIXME: SVE: ABI is different to Neon, so do we treat all vec regs as Z-regs?- let vec_save_bytes = (vec_regs + vec_save_padding) * vec_reg_size;+ let vec_save_bytes = if call_conv == isa::CallConv::PreserveAll {+ // In the PreserveAll ABI, we save the entire vector register,+ // i.e., all 128 bits.+ vec_regs * 16+ } else {+ // The Procedure Call Standard for the Arm 64-bit Architecture+ // (AAPCS64, including several related ABIs such as the one used by+ // Windows) mandates saving only the bottom 8 bytes of the vector+ // registers, so we round up the number of registers to ensure+ // proper stack alignment (similarly to the situation with+ // `int_reg`).+ let vec_reg_size = 8;+ let vec_save_padding = vec_regs & 1;+ // FIXME: SVE: ABI is different to Neon, so do we treat all vec regs as Z-regs?+ (vec_regs + vec_save_padding) * vec_reg_size+ };int_save_bytes + vec_save_bytes}@@ -714,7 +723,7 @@ impl ABIMachineSpec for AArch64MachineDeps {}fn gen_clobber_save(- _call_conv: isa::CallConv,+ call_conv: isa::CallConv,flags: &settings::Flags,frame_layout: &FrameLayout,) -> SmallVec<[Inst; 16]> {@@ -857,74 +866,91 @@ impl ABIMachineSpec for AArch64MachineDeps {}}- let store_vec_reg = |rd| Inst::FpuStore64 {- rd,- mem: AMode::SPPreIndexed {- simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(),- },- flags: MemFlags::trusted(),- };- let iter = clobbered_vec.chunks_exact(2);+ if call_conv == isa::CallConv::PreserveAll {+ // Store full vector registers in PreserveAll convention.+ for reg in clobbered_vec.iter().rev() {+ let inst = Inst::FpuStore128 {+ rd: reg.to_reg().into(),+ mem: AMode::SPPreIndexed {+ simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(),+ },+ flags: MemFlags::trusted(),+ };+ insts.push(inst);+ // N.B.: no unwind info: we don't have a way to+ // represent "full register" anyway.+ }+ } else {+ let store_vec_reg_half = |rd| Inst::FpuStore64 {+ rd,+ mem: AMode::SPPreIndexed {+ simm9: SImm9::maybe_from_i64(-clobber_offset_change).unwrap(),+ },+ flags: MemFlags::trusted(),+ };+ let iter = clobbered_vec.chunks_exact(2);- if let [rd] = iter.remainder() {- let rd: Reg = rd.to_reg().into();+ if let [rd] = iter.remainder() {+ let rd: Reg = rd.to_reg().into();- debug_assert_eq!(rd.class(), RegClass::Float);- insts.push(store_vec_reg(rd));+ debug_assert_eq!(rd.class(), RegClass::Float);+ insts.push(store_vec_reg_half(rd));- if flags.unwind_info() {- clobber_offset -= clobber_offset_change as u32;- insts.push(Inst::Unwind {- inst: UnwindInst::SaveReg {- clobber_offset,- reg: rd.to_real_reg().unwrap(),- },- });+ if flags.unwind_info() {+ clobber_offset -= clobber_offset_change as u32;+ insts.push(Inst::Unwind {+ inst: UnwindInst::SaveReg {+ clobber_offset,+ reg: rd.to_real_reg().unwrap(),+ },+ });+ }}- }- let store_vec_reg_pair = |rt, rt2| {- let clobber_offset_change = 16;+ let store_vec_reg_half_pair = |rt, rt2| {+ let clobber_offset_change = 16;- (- Inst::FpuStoreP64 {- rt,- rt2,- mem: PairAMode::SPPreIndexed {- simm7: SImm7Scaled::maybe_from_i64(-clobber_offset_change, F64).unwrap(),+ (+ Inst::FpuStoreP64 {+ rt,+ rt2,+ mem: PairAMode::SPPreIndexed {+ simm7: SImm7Scaled::maybe_from_i64(-clobber_offset_change, F64)+ .unwrap(),+ },+ flags: MemFlags::trusted(),},- flags: MemFlags::trusted(),- },- clobber_offset_change as u32,- )- };- let mut iter = iter.rev();+ clobber_offset_change as u32,+ )+ };+ let mut iter = iter.rev();- while let Some([rt, rt2]) = iter.next() {- let rt: Reg = rt.to_reg().into();- let rt2: Reg = rt2.to_reg().into();+ while let Some([rt, rt2]) = iter.next() {+ let rt: Reg = rt.to_reg().into();+ let rt2: Reg = rt2.to_reg().into();- debug_assert_eq!(rt.class(), RegClass::Float);- debug_assert_eq!(rt2.class(), RegClass::Float);+ debug_assert_eq!(rt.class(), RegClass::Float);+ debug_assert_eq!(rt2.class(), RegClass::Float);- let (inst, clobber_offset_change) = store_vec_reg_pair(rt, rt2);+ let (inst, clobber_offset_change) = store_vec_reg_half_pair(rt, rt2);- insts.push(inst);+ insts.push(inst);- if flags.unwind_info() {- clobber_offset -= clobber_offset_change;- insts.push(Inst::Unwind {- inst: UnwindInst::SaveReg {- clobber_offset,- reg: rt.to_real_reg().unwrap(),- },- });- insts.push(Inst::Unwind {- inst: UnwindInst::SaveReg {- clobber_offset: clobber_offset + clobber_offset_change / 2,- reg: rt2.to_real_reg().unwrap(),- },- });+ if flags.unwind_info() {+ clobber_offset -= clobber_offset_change;+ insts.push(Inst::Unwind {+ inst: UnwindInst::SaveReg {+ clobber_offset,+ reg: rt.to_real_reg().unwrap(),+ },+ });+ insts.push(Inst::Unwind {+ inst: UnwindInst::SaveReg {+ clobber_offset: clobber_offset + clobber_offset_change / 2,+ reg: rt2.to_real_reg().unwrap(),+ },+ });+ }}}@@ -943,7 +969,7 @@ impl ABIMachineSpec for AArch64MachineDeps {}fn gen_clobber_restore(- _call_conv: isa::CallConv,+ call_conv: isa::CallConv,_flags: &settings::Flags,frame_layout: &FrameLayout,) -> SmallVec<[Inst; 16]> {@@ -956,40 +982,55 @@ impl ABIMachineSpec for AArch64MachineDeps {insts.extend(Self::gen_sp_reg_adjust(stack_size as i32));}- let load_vec_reg = |rd| Inst::FpuLoad64 {- rd,- mem: AMode::SPPostIndexed {- simm9: SImm9::maybe_from_i64(16).unwrap(),- },- flags: MemFlags::trusted(),- };- let load_vec_reg_pair = |rt, rt2| Inst::FpuLoadP64 {- rt,- rt2,- mem: PairAMode::SPPostIndexed {- simm7: SImm7Scaled::maybe_from_i64(16, F64).unwrap(),- },- flags: MemFlags::trusted(),- };+ if call_conv == isa::CallConv::PreserveAll {+ for reg in clobbered_vec.iter() {+ let inst = Inst::FpuLoad128 {+ rd: reg.map(|r| r.into()),+ mem: AMode::SPPostIndexed {+ simm9: SImm9::maybe_from_i64(16).unwrap(),+ },+ flags: MemFlags::trusted(),+ };+ insts.push(inst);+ // N.B.: no unwind info; we don't have a way to+ // represent "full vector register saved" anyway.+ }+ } else {+ let load_vec_reg_half = |rd| Inst::FpuLoad64 {+ rd,+ mem: AMode::SPPostIndexed {+ simm9: SImm9::maybe_from_i64(16).unwrap(),+ },+ flags: MemFlags::trusted(),+ };+ let load_vec_reg_half_pair = |rt, rt2| Inst::FpuLoadP64 {+ rt,+ rt2,+ mem: PairAMode::SPPostIndexed {+ simm7: SImm7Scaled::maybe_from_i64(16, F64).unwrap(),+ },+ flags: MemFlags::trusted(),+ };- let mut iter = clobbered_vec.chunks_exact(2);+ let mut iter = clobbered_vec.chunks_exact(2);- while let Some([rt, rt2]) = iter.next() {- let rt: Writable<Reg> = rt.map(|r| r.into());- let rt2: Writable<Reg> = rt2.map(|r| r.into());+ while let Some([rt, rt2]) = iter.next() {+ let rt: Writable<Reg> = rt.map(|r| r.into());+ let rt2: Writable<Reg> = rt2.map(|r| r.into());- debug_assert_eq!(rt.to_reg().class(), RegClass::Float);- debug_assert_eq!(rt2.to_reg().class(), RegClass::Float);- insts.push(load_vec_reg_pair(rt, rt2));- }+ debug_assert_eq!(rt.to_reg().class(), RegClass::Float);+ debug_assert_eq!(rt2.to_reg().class(), RegClass::Float);+ insts.push(load_vec_reg_half_pair(rt, rt2));+ }- debug_assert!(iter.remainder().len() <= 1);+ debug_assert!(iter.remainder().len() <= 1);- if let [rd] = iter.remainder() {- let rd: Writable<Reg> = rd.map(|r| r.into());+ if let [rd] = iter.remainder() {+ let rd: Writable<Reg> = rd.map(|r| r.into());- debug_assert_eq!(rd.to_reg().class(), RegClass::Float);- insts.push(load_vec_reg(rd));+ debug_assert_eq!(rd.to_reg().class(), RegClass::Float);+ insts.push(load_vec_reg_half(rd));+ }}let mut iter = clobbered_int.chunks_exact(2);@@ -1150,7 +1191,7 @@ impl ABIMachineSpec for AArch64MachineDeps {regs.sort_unstable();// Compute clobber size.- let clobber_size = compute_clobber_size(®s);+ let clobber_size = compute_clobber_size(call_conv, ®s);// Compute linkage frame size.let setup_area_size = if flags.preserve_frame_pointers()diff --git a/cranelift/filetests/filetests/isa/aarch64/preserve-all.clif b/cranelift/filetests/filetests/isa/aarch64/preserve-all.clifindex f361163f98..2f3672baf4 100644--- a/cranelift/filetests/filetests/isa/aarch64/preserve-all.clif+++ b/cranelift/filetests/filetests/isa/aarch64/preserve-all.clif@@ -53,41 +53,73 @@ block0(v0: i64):; stp x4, x5, [sp, #-16]!; stp x2, x3, [sp, #-16]!; stp x0, x1, [sp, #-16]!-; stp d30, d31, [sp, #-16]!-; stp d28, d29, [sp, #-16]!-; stp d26, d27, [sp, #-16]!-; stp d24, d25, [sp, #-16]!-; stp d22, d23, [sp, #-16]!-; stp d20, d21, [sp, #-16]!-; stp d18, d19, [sp, #-16]!-; stp d16, d17, [sp, #-16]!-; stp d14, d15, [sp, #-16]!-; stp d12, d13, [sp, #-16]!-; stp d10, d11, [sp, #-16]!-; stp d8, d9, [sp, #-16]!-; stp d6, d7, [sp, #-16]!-; stp d4, d5, [sp, #-16]!-; stp d2, d3, [sp, #-16]!-; stp d0, d1, [sp, #-16]!+; str q31, [sp, #-16]!+; str q30, [sp, #-16]!+; str q29, [sp, #-16]!+; str q28, [sp, #-16]!+; str q27, [sp, #-16]!+; str q26, [sp, #-16]!+; str q25, [sp, #-16]!+; str q24, [sp, #-16]!+; str q23, [sp, #-16]!+; str q22, [sp, #-16]!+; str q21, [sp, #-16]!+; str q20, [sp, #-16]!+; str q19, [sp, #-16]!+; str q18, [sp, #-16]!+; str q17, [sp, #-16]!+; str q16, [sp, #-16]!+; str q15, [sp, #-16]!+; str q14, [sp, #-16]!+; str q13, [sp, #-16]!+; str q12, [sp, #-16]!+; str q11, [sp, #-16]!+; str q10, [sp, #-16]!+; str q9, [sp, #-16]!+; str q8, [sp, #-16]!+; str q7, [sp, #-16]!+; str q6, [sp, #-16]!+; str q5, [sp, #-16]!+; str q4, [sp, #-16]!+; str q3, [sp, #-16]!+; str q2, [sp, #-16]!+; str q1, [sp, #-16]!+; str q0, [sp, #-16]!; block0:; load_ext_name_far x2, TestCase(%libcall)+0; blr x2-; ldp d0, d1, [sp], #16-; ldp d2, d3, [sp], #16-; ldp d4, d5, [sp], #16-; ldp d6, d7, [sp], #16-; ldp d8, d9, [sp], #16-; ldp d10, d11, [sp], #16-; ldp d12, d13, [sp], #16-; ldp d14, d15, [sp], #16-; ldp d16, d17, [sp], #16-; ldp d18, d19, [sp], #16-; ldp d20, d21, [sp], #16-; ldp d22, d23, [sp], #16-; ldp d24, d25, [sp], #16-; ldp d26, d27, [sp], #16-; ldp d28, d29, [sp], #16-; ldp d30, d31, [sp], #16+; ldr q0, [sp], #16+; ldr q1, [sp], #16+; ldr q2, [sp], #16+; ldr q3, [sp], #16+; ldr q4, [sp], #16+; ldr q5, [sp], #16+; ldr q6, [sp], #16+; ldr q7, [sp], #16+; ldr q8, [sp], #16+; ldr q9, [sp], #16+; ldr q10, [sp], #16+; ldr q11, [sp], #16+; ldr q12, [sp], #16+; ldr q13, [sp], #16+; ldr q14, [sp], #16+; ldr q15, [sp], #16+; ldr q16, [sp], #16+; ldr q17, [sp], #16+; ldr q18, [sp], #16+; ldr q19, [sp], #16+; ldr q20, [sp], #16+; ldr q21, [sp], #16+; ldr q22, [sp], #16+; ldr q23, [sp], #16+; ldr q24, [sp], #16+; ldr q25, [sp], #16+; ldr q26, [sp], #16+; ldr q27, [sp], #16+; ldr q28, [sp], #16+; ldr q29, [sp], #16+; ldr q30, [sp], #16+; ldr q31, [sp], #16; ldp x0, x1, [sp], #16; ldp x2, x3, [sp], #16; ldp x4, x5, [sp], #16@@ -113,44 +145,76 @@ block0(v0: i64):; stp x4, x5, [sp, #-0x10]!; stp x2, x3, [sp, #-0x10]!; stp x0, x1, [sp, #-0x10]!-; stp d30, d31, [sp, #-0x10]!-; stp d28, d29, [sp, #-0x10]!-; stp d26, d27, [sp, #-0x10]!-; stp d24, d25, [sp, #-0x10]!-; stp d22, d23, [sp, #-0x10]!-; stp d20, d21, [sp, #-0x10]!-; stp d18, d19, [sp, #-0x10]!-; stp d16, d17, [sp, #-0x10]!-; stp d14, d15, [sp, #-0x10]!-; stp d12, d13, [sp, #-0x10]!-; stp d10, d11, [sp, #-0x10]!-; stp d8, d9, [sp, #-0x10]!-; stp d6, d7, [sp, #-0x10]!-; stp d4, d5, [sp, #-0x10]!-; stp d2, d3, [sp, #-0x10]!-; stp d0, d1, [sp, #-0x10]!-; block1: ; offset 0x6c-; ldr x2, #0x74-; b #0x7c+; str q31, [sp, #-0x10]!+; str q30, [sp, #-0x10]!+; str q29, [sp, #-0x10]!+; str q28, [sp, #-0x10]!+; str q27, [sp, #-0x10]!+; str q26, [sp, #-0x10]!+; str q25, [sp, #-0x10]!+; str q24, [sp, #-0x10]!+; str q23, [sp, #-0x10]!+; str q22, [sp, #-0x10]!+; str q21, [sp, #-0x10]!+; str q20, [sp, #-0x10]!+; str q19, [sp, #-0x10]!+; str q18, [sp, #-0x10]!+; str q17, [sp, #-0x10]!+; str q16, [sp, #-0x10]!+; str q15, [sp, #-0x10]!+; str q14, [sp, #-0x10]!+; str q13, [sp, #-0x10]!+; str q12, [sp, #-0x10]!+; str q11, [sp, #-0x10]!+; str q10, [sp, #-0x10]!+; str q9, [sp, #-0x10]!+; str q8, [sp, #-0x10]!+; str q7, [sp, #-0x10]!+; str q6, [sp, #-0x10]!+; str q5, [sp, #-0x10]!+; str q4, [sp, #-0x10]!+; str q3, [sp, #-0x10]!+; str q2, [sp, #-0x10]!+; str q1, [sp, #-0x10]!+; str q0, [sp, #-0x10]!+; block1: ; offset 0xac+; ldr x2, #0xb4+; b #0xbc; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %libcall 0; .byte 0x00, 0x00, 0x00, 0x00; blr x2-; ldp d0, d1, [sp], #0x10-; ldp d2, d3, [sp], #0x10-; ldp d4, d5, [sp], #0x10-; ldp d6, d7, [sp], #0x10-; ldp d8, d9, [sp], #0x10-; ldp d10, d11, [sp], #0x10-; ldp d12, d13, [sp], #0x10-; ldp d14, d15, [sp], #0x10-; ldp d16, d17, [sp], #0x10-; ldp d18, d19, [sp], #0x10-; ldp d20, d21, [sp], #0x10-; ldp d22, d23, [sp], #0x10-; ldp d24, d25, [sp], #0x10-; ldp d26, d27, [sp], #0x10-; ldp d28, d29, [sp], #0x10-; ldp d30, d31, [sp], #0x10+; ldr q0, [sp], #0x10+; ldr q1, [sp], #0x10+; ldr q2, [sp], #0x10+; ldr q3, [sp], #0x10+; ldr q4, [sp], #0x10+; ldr q5, [sp], #0x10+; ldr q6, [sp], #0x10+; ldr q7, [sp], #0x10+; ldr q8, [sp], #0x10+; ldr q9, [sp], #0x10+; ldr q10, [sp], #0x10+; ldr q11, [sp], #0x10+; ldr q12, [sp], #0x10+; ldr q13, [sp], #0x10+; ldr q14, [sp], #0x10+; ldr q15, [sp], #0x10+; ldr q16, [sp], #0x10+; ldr q17, [sp], #0x10+; ldr q18, [sp], #0x10+; ldr q19, [sp], #0x10+; ldr q20, [sp], #0x10+; ldr q21, [sp], #0x10+; ldr q22, [sp], #0x10+; ldr q23, [sp], #0x10+; ldr q24, [sp], #0x10+; ldr q25, [sp], #0x10+; ldr q26, [sp], #0x10+; ldr q27, [sp], #0x10+; ldr q28, [sp], #0x10+; ldr q29, [sp], #0x10+; ldr q30, [sp], #0x10+; ldr q31, [sp], #0x10; ldp x0, x1, [sp], #0x10; ldp x2, x3, [sp], #0x10; ldp x4, x5, [sp], #0x10