A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
winch: restore aarch64 frame pointer before advancing SP (#14516)
Daniel Macovei6e5693ab3c07
diff --git a/tests/disas/winch/aarch64/return_call/empty_callee.wat b/tests/disas/winch/aarch64/return_call/empty_callee.watnew file mode 100644index 0000000000..42494f881e--- /dev/null+++ b/tests/disas/winch/aarch64/return_call/empty_callee.wat@@ -0,0 +1,139 @@+;;! target = "aarch64"+;;! test = "winch"+;;! flags = "-W tail-call=y"+;;! filter = "tail_caller"++;; The caller has stack arguments, but the tail callee has none. Restore the+;; saved FP before advancing SP past the old frame (issue #14503).+(module+ (func $leaf (param i32) (result i32) local.get 0)+ (func $tail_caller (param i32 i32 i32 i32 i32 i32 i32 i32 i32 i32 i32 i32)+ (result i32)+ local.get 0+ return_call $leaf)++ ;; The entry-SP offset exceeds the 12-bit add-immediate range.+ (func $tail_caller_large (param+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64 i64+ ) (result i32)+ i32.const 42+ return_call $leaf))+;; wasm[0]::function[1]::tail_caller:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x2c+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x128+;; ac: mov x9, x0+;; sub x28, x28, #0x28+;; mov sp, x28+;; stur x0, [x28, #0x20]+;; stur x1, [x28, #0x18]+;; stur w2, [x28, #0x14]+;; stur w3, [x28, #0x10]+;; stur w4, [x28, #0xc]+;; stur w5, [x28, #8]+;; stur w6, [x28, #4]+;; stur w7, [x28]+;; ldur w16, [x28, #0x14]+;; sub x28, x28, #4+;; mov sp, x28+;; stur w16, [x28]+;; mov x0, x9+;; mov x1, x9+;; ldur w2, [x28]+;; ldur x28, [x29, #-0x10]+;; ldur x30, [x29, #8]+;; add x16, x29, #0x40+;; ldur x29, [x29]+;; mov sp, x16+;; b #0+;; 10c: add x28, x28, #0x28+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; add sp, sp, #0x30+;; ret+;; 128: udf #0xc11f+;;+;; wasm[0]::function[2]::tail_caller_large:+;; stp x29, x30, [sp, #-0x10]!+;; mov x29, sp+;; str x28, [sp, #-0x10]!+;; mov x28, sp+;; ldur x16, [x0, #8]+;; ldur x16, [x16, #0x18]+;; mov x17, #0+;; movk x17, #0x40+;; add x16, x16, x17+;; cmp sp, x16+;; b.lo #0x1e0+;; 16c: mov x9, x0+;; sub x28, x28, #0x40+;; mov sp, x28+;; stur x0, [x28, #0x38]+;; stur x1, [x28, #0x30]+;; stur x2, [x28, #0x28]+;; stur x3, [x28, #0x20]+;; stur x4, [x28, #0x18]+;; stur x5, [x28, #0x10]+;; stur x6, [x28, #8]+;; stur x7, [x28]+;; mov x0, x9+;; mov x1, x9+;; mov x2, #0x2a+;; ldur x28, [x29, #-0x10]+;; ldur x30, [x29, #8]+;; mov x16, #0x1060+;; add x16, x16, x29, uxtx+;; ldur x29, [x29]+;; mov sp, x16+;; b #0+;; 1c0: add x28, x28, #0x40+;; mov sp, x28+;; mov sp, x28+;; ldr x28, [sp], #0x10+;; ldp x29, x30, [sp], #0x10+;; mov x16, #0x1050+;; add sp, sp, x16+;; ret+;; 1e0: udf #0xc11fdiff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex 5868651288..86cd295238 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -142,6 +142,46 @@ impl MacroAssembler {Ok(res)}++ /// Calculate the tail callee's entry SP, restore the old frame, and then+ /// update SP. All old-frame reads must precede the SP update so signal+ /// handlers cannot overwrite the frame before it is restored.+ fn finish_tail_call_frame(+ &mut self,+ callee_args_from_fp: i64,+ restore_frame: impl FnOnce(&mut Self) -> Result<()>,+ ) -> Result<()> {+ self.with_scratch::<IntScratch, _>(|masm, work| {+ // Calculate the entry SP while the current FP is available.+ let magnitude = callee_args_from_fp.unsigned_abs();+ if let Some(imm) = Imm12::maybe_from_u64(magnitude) {+ if callee_args_from_fp >= 0 {+ masm.asm+ .add_ir(imm, regs::fp(), work.writable(), OperandSize::S64);+ } else {+ masm.asm+ .sub_ir(imm, regs::fp(), work.writable(), OperandSize::S64);+ }+ } else {+ masm.asm+ .mov_ir(work.writable(), I::I64(magnitude), OperandSize::S64);+ if callee_args_from_fp >= 0 {+ masm.asm+ .add_rrr(regs::fp(), work.inner(), work.writable(), OperandSize::S64);+ } else {+ masm.asm+ .sub_rrr(work.inner(), regs::fp(), work.writable(), OperandSize::S64);+ }+ }++ restore_frame(masm)?;++ let zero = Imm12::maybe_from_u64(0).unwrap();+ masm.asm+ .add_ir(zero, work.inner(), writable!(regs::sp()), OperandSize::S64);+ Ok(())+ })+ }}impl Masm for MacroAssembler {@@ -525,25 +565,10 @@ impl Masm for MacroAssembler {writable!(regs::lr()),)?;- let entry_sp_offset = plan.callee_args_from_fp.unsigned_abs();- if let Some(imm) = Imm12::maybe_from_u64(entry_sp_offset) {- assert!(plan.callee_args_from_fp >= 0);- self.asm- .add_ir(imm, regs::fp(), writable!(regs::sp()), OperandSize::S64);- } else {- self.with_scratch::<IntScratch, _>(|masm, work| {- masm.asm- .mov_ir(work.writable(), I::I64(entry_sp_offset), OperandSize::S64);- masm.asm.add_rrr(- regs::fp(),- work.inner(),- writable!(regs::sp()),- OperandSize::S64,- );- });- }-- self.load_ptr(Address::offset(regs::fp(), 0), writable!(regs::fp()))+ assert!(plan.callee_args_from_fp >= 0);+ self.finish_tail_call_frame(plan.callee_args_from_fp, |masm| {+ masm.load_ptr(Address::offset(regs::fp(), 0), writable!(regs::fp()))+ })}fn with_tail_call_resize(@@ -579,41 +604,13 @@ impl Masm for MacroAssembler {move_args(self, scratch_size)?;- self.with_scratch::<IntScratch, _>(|masm, work| {- // Calculate the callee's entry SP while the current FP is available.- let magnitude = plan.callee_args_from_fp.unsigned_abs();- if let Some(imm) = Imm12::maybe_from_u64(magnitude) {- if plan.callee_args_from_fp >= 0 {- masm.asm- .add_ir(imm, regs::fp(), work.writable(), OperandSize::S64);- } else {- masm.asm- .sub_ir(imm, regs::fp(), work.writable(), OperandSize::S64);- }- } else {- masm.asm- .mov_ir(work.writable(), I::I64(magnitude), OperandSize::S64);- if plan.callee_args_from_fp >= 0 {- masm.asm- .add_rrr(work.inner(), regs::fp(), work.writable(), OperandSize::S64);- } else {- masm.asm- .sub_rrr(work.inner(), regs::fp(), work.writable(), OperandSize::S64);- }- }-- // Finish all old-frame reads before advancing SP. The tail callee- // will save FP, LR, and x28 into its replacement frame.+ self.finish_tail_call_frame(plan.callee_args_from_fp, |masm| {+ // The tail callee will save FP, LR, and x28 into its replacement frame.masm.load_ptr(Address::from_shadow_sp(0), writable!(regs::fp()))?;masm.load_ptr(Address::from_shadow_sp(i64::from(word_bytes)),writable!(regs::shadow_sp()),- )?;-- let zero = Imm12::maybe_from_u64(0).unwrap();- masm.asm- .add_ir(zero, work.inner(), writable!(regs::sp()), OperandSize::S64);- wasmtime_environ::error::Ok(())+ )})}