A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
feat: optimize frame layout for tail-call-only functions (#11608)
Paul Nodeta591e3be4375
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rsindex 51b91b3d1d..84f5df6d16 100644--- a/cranelift/codegen/src/isa/aarch64/abi.rs+++ b/cranelift/codegen/src/isa/aarch64/abi.rs@@ -1199,15 +1199,17 @@ impl ABIMachineSpec for AArch64MachineDeps {// Compute clobber size.let clobber_size = compute_clobber_size(call_conv, ®s);+ let needs_linkage_frame = flags.preserve_frame_pointers()+ // The function arguments that are passed on the stack are addressed+ // relative to the Frame Pointer.+ || incoming_args_size > 0+ || tail_args_size > incoming_args_size+ || clobber_size > 0+ || fixed_frame_storage_size > 0+ || outgoing_args_size > 0;+// Compute linkage frame size.- let setup_area_size = if flags.preserve_frame_pointers()- || function_calls != FunctionCalls::None- // The function arguments that are passed on the stack are addressed- // relative to the Frame Pointer.- || incoming_args_size > 0- || clobber_size > 0- || fixed_frame_storage_size > 0- {+ let setup_area_size = if needs_linkage_frame || function_calls == FunctionCalls::Regular {16 // FP, LR} else {0diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rsindex 51776985ac..93ddd22e7f 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs@@ -3692,7 +3692,9 @@ fn emit_return_call_common_sequence<T>(}}- if let Some(key) = info.key {+ if (setup_area_size > 0 || info.sign_return_address_all)+ && let Some(key) = info.key+ {sink.put4(key.enc_auti_hint());}}diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rsindex 90ba2ef0c7..e1403f0030 100644--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs@@ -88,6 +88,8 @@ pub struct ReturnCallInfo<T> {pub new_stack_arg_size: u32,/// API key to use to restore the return address, if any.pub key: Option<APIKey>,+ /// Whether pointer-auth return addresses are signed even without frame setup.+ pub sign_return_address_all: bool,}fn count_zero_half_words(mut value: u64, num_half_words: u8) -> usize {diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rsindex b7f6b84e94..184bd0d23a 100644--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs@@ -135,6 +135,7 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {dest,uses,key,+ sign_return_address_all: self.backend.isa_flags.sign_return_address_all(),new_stack_arg_size,})}@@ -157,6 +158,7 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {dest,uses,key,+ sign_return_address_all: self.backend.isa_flags.sign_return_address_all(),new_stack_arg_size,})}diff --git a/cranelift/filetests/filetests/isa/aarch64/call-pauth-bkey.clif b/cranelift/filetests/filetests/isa/aarch64/call-pauth-bkey.clifindex 589b4b4992..f707a1accd 100644--- a/cranelift/filetests/filetests/isa/aarch64/call-pauth-bkey.clif+++ b/cranelift/filetests/filetests/isa/aarch64/call-pauth-bkey.clif@@ -95,24 +95,15 @@ block0(v0: i64):}; VCode:-; pacibz-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_far x1, TestCase(%g)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:; block0: ; offset 0x0-; pacibz-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0xc-; ldr x1, #0x14-; b #0x1c+; ldr x1, #8+; b #0x10; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0; .byte 0x00, 0x00, 0x00, 0x00-; ldp x29, x30, [sp], #0x10-; autibz; br x1diff --git a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clifindex 73792bee0d..5ad09e6d3a 100644--- a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif+++ b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif@@ -96,24 +96,15 @@ block0(v0: i64):}; VCode:-; paciaz-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_far x1, TestCase(%g)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:; block0: ; offset 0x0-; paciaz-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0xc-; ldr x1, #0x14-; b #0x1c+; ldr x1, #8+; b #0x10; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0; .byte 0x00, 0x00, 0x00, 0x00-; ldp x29, x30, [sp], #0x10-; autiaz; br x1diff --git a/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clifindex 9c746fad53..d08bf21901 100644--- a/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif+++ b/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif@@ -30,22 +30,16 @@ block0(v0: i64):}; VCode:-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_far x1, TestCase(%callee_i64)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:; block0: ; offset 0x0-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0x8-; ldr x1, #0x10-; b #0x18+; ldr x1, #8+; b #0x10; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_i64 0; .byte 0x00, 0x00, 0x00, 0x00-; ldp x29, x30, [sp], #0x10; br x1;;;; Test colocated tail calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -60,20 +54,14 @@ block0(v0: i64):}; VCode:-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_near x1, TestCase(%callee_i64)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:; block0: ; offset 0x0-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0x8; adrp x1, #0 ; reloc_external Aarch64AdrPrelPgHi21 %callee_i64 0; add x1, x1, #0 ; reloc_external Aarch64AddAbsLo12Nc %callee_i64 0-; ldp x29, x30, [sp], #0x10; br x1;;;; Test passing `f64`s ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -107,22 +95,16 @@ block0(v0: f64):}; VCode:-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_far x1, TestCase(%callee_f64)+0; return_call_ind x1 new_stack_arg_size:0 v0=v0;; Disassembled:; block0: ; offset 0x0-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0x8-; ldr x1, #0x10-; b #0x18+; ldr x1, #8+; b #0x10; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_f64 0; .byte 0x00, 0x00, 0x00, 0x00-; ldp x29, x30, [sp], #0x10; br x1;;;; Test passing `i8`s ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -158,22 +140,16 @@ block0(v0: i8):}; VCode:-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_far x1, TestCase(%callee_i8)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:; block0: ; offset 0x0-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0x8-; ldr x1, #0x10-; b #0x18+; ldr x1, #8+; b #0x10; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_i8 0; .byte 0x00, 0x00, 0x00, 0x00-; ldp x29, x30, [sp], #0x10; br x1;;;; Test passing many arguments on stack ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/filetests/filetests/isa/aarch64/return-call.clif b/cranelift/filetests/filetests/isa/aarch64/return-call.clifindex ffb8ee2a38..4cd83166ca 100644--- a/cranelift/filetests/filetests/isa/aarch64/return-call.clif+++ b/cranelift/filetests/filetests/isa/aarch64/return-call.clif@@ -28,22 +28,16 @@ block0(v0: i64):}; VCode:-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_far x1, TestCase(%callee_i64)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:; block0: ; offset 0x0-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0x8-; ldr x1, #0x10-; b #0x18+; ldr x1, #8+; b #0x10; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_i64 0; .byte 0x00, 0x00, 0x00, 0x00-; ldp x29, x30, [sp], #0x10; br x1;;;; Test colocated tail calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -56,18 +50,12 @@ block0(v0: i64):}; VCode:-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; return_call TestCase(%callee_i64) new_stack_arg_size:0 x2=x2;; Disassembled:; block0: ; offset 0x0-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0x8-; ldp x29, x30, [sp], #0x10-; b #0xc ; reloc_external Call %callee_i64 0+; b #0 ; reloc_external Call %callee_i64 0;;;; Test passing `f64`s ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -98,22 +86,16 @@ block0(v0: f64):}; VCode:-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_far x1, TestCase(%callee_f64)+0; return_call_ind x1 new_stack_arg_size:0 v0=v0;; Disassembled:; block0: ; offset 0x0-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0x8-; ldr x1, #0x10-; b #0x18+; ldr x1, #8+; b #0x10; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_f64 0; .byte 0x00, 0x00, 0x00, 0x00-; ldp x29, x30, [sp], #0x10; br x1;;;; Test passing `i8`s ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -147,22 +129,16 @@ block0(v0: i8):}; VCode:-; stp fp, lr, [sp, #-16]!-; mov fp, sp; block0:; load_ext_name_far x1, TestCase(%callee_i8)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:; block0: ; offset 0x0-; stp x29, x30, [sp, #-0x10]!-; mov x29, sp-; block1: ; offset 0x8-; ldr x1, #0x10-; b #0x18+; ldr x1, #8+; b #0x10; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_i8 0; .byte 0x00, 0x00, 0x00, 0x00-; ldp x29, x30, [sp], #0x10; br x1;;;; Test passing many arguments on stack ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/filetests/filetests/isa/aarch64/tail-call-frame-optimization-unwind.clif b/cranelift/filetests/filetests/isa/aarch64/tail-call-frame-optimization-unwind.clifnew file mode 100644index 0000000000..0d4249f346--- /dev/null+++ b/cranelift/filetests/filetests/isa/aarch64/tail-call-frame-optimization-unwind.clif@@ -0,0 +1,37 @@+test compile precise-output+set unwind_info=true+set preserve_frame_pointers=false+target aarch64++;; With unwind enabled, tail-only functions that do not need frame storage+;; should still avoid FP/LR frame setup.+function %tail_only_with_unwind() -> i64 tail {+ fn0 = colocated %target_func() -> i64 tail++block0:+ return_call fn0()+}++; VCode:+; block0:+; return_call TestCase(%target_func) new_stack_arg_size:0+;+; Disassembled:+; block0: ; offset 0x0+; b #0 ; reloc_external Call %target_func 0++function %target_func() -> i64 {+block0:+ v0 = iconst.i64 42+ return v0+}++; VCode:+; block0:+; movz x0, #42+; ret+;+; Disassembled:+; block0: ; offset 0x0+; mov x0, #0x2a+; retdiff --git a/cranelift/filetests/filetests/isa/aarch64/tail-call-frame-optimization.clif b/cranelift/filetests/filetests/isa/aarch64/tail-call-frame-optimization.clifnew file mode 100644index 0000000000..4b29cf2a80--- /dev/null+++ b/cranelift/filetests/filetests/isa/aarch64/tail-call-frame-optimization.clif@@ -0,0 +1,117 @@+test compile precise-output+set unwind_info=false+set preserve_frame_pointers=false+target aarch64++;; Test 1: Tail calling convention with tail-only calls+;; FunctionCalls::TailOnly → setup_area_size = 0 → no frame setup+function %tail_only_function() -> i64 tail {+ fn0 = colocated %target_func() -> i64 tail++block0:+ return_call fn0()+}++; VCode:+; block0:+; return_call TestCase(%target_func) new_stack_arg_size:0+;+; Disassembled:+; block0: ; offset 0x0+; b #0 ; reloc_external Call %target_func 0++;; Test 2: SystemV calling convention with regular calls+;; FunctionCalls::Regular → setup_area_size = 16 → standard frame+function %systemv_regular_function() -> i64 system_v {+ fn0 = colocated %target_func() -> i64 system_v++block0:+ v0 = call fn0()+ return v0+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; block0:+; bl 0+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; block1: ; offset 0x8+; bl #8 ; reloc_external Call %target_func 0+; ldp x29, x30, [sp], #0x10+; ret++;; Test 3: Tail calling convention with conditional tail calls+;; Multiple return_call instructions still → FunctionCalls::TailOnly → no frame setup+function %tail_only_conditional(i64) -> i64 tail {+ fn0 = colocated %target_func() -> i64 tail+ fn1 = colocated %other_func() -> i64 tail++block0(v0: i64):+ v1 = iconst.i64 0+ v2 = icmp sgt v0, v1+ brif v2, block1, block2++block1:+ return_call fn0()++block2:+ return_call fn1()+}++; VCode:+; block0:+; subs xzr, x2, #0+; b.gt label2 ; b label1+; block1:+; return_call TestCase(%other_func) new_stack_arg_size:0+; block2:+; return_call TestCase(%target_func) new_stack_arg_size:0+;+; Disassembled:+; block0: ; offset 0x0+; cmp x2, #0+; b.gt #0xc+; block1: ; offset 0x8+; b #8 ; reloc_external Call %other_func 0+; block2: ; offset 0xc+; b #0xc ; reloc_external Call %target_func 0++;; Target functions for testing+function %target_func() -> i64 {+block0:+ v0 = iconst.i64 42+ return v0+}++; VCode:+; block0:+; movz x0, #42+; ret+;+; Disassembled:+; block0: ; offset 0x0+; mov x0, #0x2a+; ret++function %other_func() -> i64 {+block0:+ v0 = iconst.i64 24+ return v0+}++; VCode:+; block0:+; movz x0, #24+; ret+;+; Disassembled:+; block0: ; offset 0x0+; mov x0, #0x18+; ret