Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: fix unwind info around tail calls. (#14642)

Chris Fallin2cb329d23ba3
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs
index 8659561704..3556efef15 100644
--- a/cranelift/codegen/src/isa/aarch64/abi.rs
+++ b/cranelift/codegen/src/isa/aarch64/abi.rs
@@ -784,6 +784,22 @@ impl ABIMachineSpec for AArch64MachineDeps {
let mut insts = SmallVec::new();
let setup_frame = frame_layout.setup_area_size > 0;
+ if flags.unwind_info() && setup_frame {
+ // The *unwind* frame (but not the actual frame) starts at the
+ // clobbers, just below the saved FP/LR pair.
+ //
+ // This is emitted before the incoming argument area is resized
+ // below so that the CFA is defined in terms of FP while the saved
+ // FP/LR pair is being moved: FP keeps pointing at the original pair
+ // until it has been copied, and then points at the new copy.
+ insts.push(Inst::Unwind {
+ inst: UnwindInst::DefineNewFrame {
+ offset_downward_to_clobbers: frame_layout.clobber_size,
+ offset_upward_to_caller_sp: frame_layout.setup_area_size,
+ },
+ });
+ }
+
// When a return_call within this function required more stack arguments than we have
// present, resize the incoming argument area of the frame to accommodate those arguments.
let incoming_args_diff = frame_layout.tail_args_size - frame_layout.incoming_args_size;
@@ -800,18 +816,20 @@ impl ABIMachineSpec for AArch64MachineDeps {
// Move fp and lr down.
if setup_frame {
- // Reload the frame pointer from the stack.
+ // Reload the saved frame pointer from the stack. This uses a
+ // temporary rather than FP itself so that FP keeps pointing at
+ // a valid frame record until the copy below is complete.
insts.push(Inst::ULoad64 {
- rd: regs::writable_fp_reg(),
+ rd: writable_spilltmp_reg(),
mem: AMode::SPOffset {
off: i64::from(incoming_args_diff),
},
flags: MemFlagsData::trusted(),
});
- // Store the frame pointer and link register again at the new SP
+ // Store the saved frame pointer and link register again at the new SP
insts.push(Inst::StoreP64 {
- rt: fp_reg(),
+ rt: spilltmp_reg(),
rt2: link_reg(),
mem: PairAMode::SignedOffset {
reg: regs::stack_reg(),
@@ -829,17 +847,6 @@ impl ABIMachineSpec for AArch64MachineDeps {
}
}
- if flags.unwind_info() && setup_frame {
- // The *unwind* frame (but not the actual frame) starts at the
- // clobbers, just below the saved FP/LR pair.
- insts.push(Inst::Unwind {
- inst: UnwindInst::DefineNewFrame {
- offset_downward_to_clobbers: frame_layout.clobber_size,
- offset_upward_to_caller_sp: frame_layout.setup_area_size,
- },
- });
- }
-
// We use pre-indexed addressing modes here, rather than the possibly
// more efficient "subtract sp once then used fixed offsets" scheme,
// because (i) we cannot necessarily guarantee that the offset of a
diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs
index 8b130715cd..0fe5398a67 100644
--- a/cranelift/codegen/src/isa/riscv64/abi.rs
+++ b/cranelift/codegen/src/isa/riscv64/abi.rs
@@ -437,6 +437,22 @@ impl ABIMachineSpec for Riscv64MachineDeps {
let mut insts = SmallVec::new();
let setup_frame = frame_layout.setup_area_size > 0;
+ if flags.unwind_info() && setup_frame {
+ // The *unwind* frame (but not the actual frame) starts at the
+ // clobbers, just below the saved FP/LR pair.
+ //
+ // This is emitted before the incoming argument area is resized
+ // below so that the CFA is defined in terms of FP while the saved
+ // FP/LR pair is being moved: FP keeps pointing at the original pair
+ // until it has been copied, and then points at the new copy.
+ insts.push(Inst::Unwind {
+ inst: UnwindInst::DefineNewFrame {
+ offset_downward_to_clobbers: frame_layout.clobber_size,
+ offset_upward_to_caller_sp: frame_layout.setup_area_size,
+ },
+ });
+ }
+
let incoming_args_diff = frame_layout.tail_args_size - frame_layout.incoming_args_size;
if incoming_args_diff > 0 {
// Decrement SP by the amount of additional incoming argument space we need
@@ -451,15 +467,18 @@ impl ABIMachineSpec for Riscv64MachineDeps {
I64,
MemFlagsData::trusted(),
));
+ // Copy the saved frame pointer down. This uses a temporary
+ // rather than FP itself so that FP keeps pointing at a valid
+ // frame record until the copy is complete.
insts.push(Inst::gen_load(
- writable_fp_reg(),
+ writable_spilltmp_reg(),
AMode::SPOffset(i64::from(incoming_args_diff)),
I64,
MemFlagsData::trusted(),
));
insts.push(Inst::gen_store(
AMode::SPOffset(0),
- fp_reg(),
+ spilltmp_reg(),
I64,
MemFlagsData::trusted(),
));
@@ -469,17 +488,6 @@ impl ABIMachineSpec for Riscv64MachineDeps {
}
}
- if flags.unwind_info() && setup_frame {
- // The *unwind* frame (but not the actual frame) starts at the
- // clobbers, just below the saved FP/LR pair.
- insts.push(Inst::Unwind {
- inst: UnwindInst::DefineNewFrame {
- offset_downward_to_clobbers: frame_layout.clobber_size,
- offset_upward_to_caller_sp: frame_layout.setup_area_size,
- },
- });
- }
-
// Adjust the stack pointer downward for clobbers, the function fixed
// frame (spillslots and storage slots), and outgoing arguments.
let stack_size = frame_layout.clobber_size
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index 88117cdd8b..3dde295c91 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -656,6 +656,26 @@ impl ABIMachineSpec for X64ABIMachineSpec {
) -> SmallVec<[Self::I; 16]> {
let mut insts = SmallVec::new();
+ // Emit unwind info: start the frame. The frame (from unwind consumers'
+ // point of view) starts at clobbers, just below the FP and return
+ // address. Spill slots and stack slots are part of our actual frame but
+ // do not concern the unwinder.
+ //
+ // This is emitted before the incoming argument area is resized below so
+ // that the CFA is defined in terms of RBP while the saved RBP and return
+ // address are being moved: RBP keeps pointing at the original copies
+ // until both have been moved, and then points at the new copies. In both
+ // cases the caller's frame is `setup_area_size` above RBP, and the
+ // clobbers are just below RBP.
+ if flags.unwind_info() && frame_layout.setup_area_size > 0 {
+ insts.push(Inst::Unwind {
+ inst: UnwindInst::DefineNewFrame {
+ offset_downward_to_clobbers: frame_layout.clobber_size,
+ offset_upward_to_caller_sp: frame_layout.setup_area_size,
+ },
+ });
+ }
+
// When a return_call within this function required more stack arguments than we have
// present, resize the incoming argument area of the frame to accommodate those arguments.
let incoming_args_diff = frame_layout.tail_args_size - frame_layout.incoming_args_size;
@@ -668,14 +688,6 @@ impl ABIMachineSpec for X64ABIMachineSpec {
.expect("`incoming_args_diff` is too large to fit in a 32-bit immediate"),
));
- // Make sure to keep the frame pointer and stack pointer in sync at
- // this point.
- let rbp = Gpr::RBP;
- let rsp = Gpr::RSP;
- insts.push(Inst::External {
- inst: asm::inst::movq_mr::new(Writable::from_reg(rbp), rsp).into(),
- });
-
let incoming_args_diff = i32::try_from(incoming_args_diff).unwrap();
// Move the saved frame pointer down by `incoming_args_diff`.
@@ -692,22 +704,13 @@ impl ABIMachineSpec for X64ABIMachineSpec {
insts.push(Inst::External { inst });
let inst = asm::inst::movq_mr::new(Amode::imm_reg(8, regs::rsp()), r11.to_reg()).into();
insts.push(Inst::External { inst });
- }
- // We need to factor `incoming_args_diff` into the offset upward here, as we have grown
- // the argument area -- `setup_area_size` alone will not be the correct offset up to the
- // original caller's SP.
- let offset_upward_to_caller_sp = frame_layout.setup_area_size + incoming_args_diff;
- if flags.unwind_info() && offset_upward_to_caller_sp > 0 {
- // Emit unwind info: start the frame. The frame (from unwind
- // consumers' point of view) starts at clobbbers, just below
- // the FP and return address. Spill slots and stack slots are
- // part of our actual frame but do not concern the unwinder.
- insts.push(Inst::Unwind {
- inst: UnwindInst::DefineNewFrame {
- offset_downward_to_clobbers: frame_layout.clobber_size,
- offset_upward_to_caller_sp,
- },
+ // Now that the saved frame pointer and return address have been
+ // moved, point the frame pointer at the new frame record.
+ let rbp = Gpr::RBP;
+ let rsp = Gpr::RSP;
+ insts.push(Inst::External {
+ inst: asm::inst::movq_mr::new(Writable::from_reg(rbp), rsp).into(),
});
}
diff --git a/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif
index 748baf8f0b..a9def1013f 100644
--- a/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif
@@ -191,8 +191,8 @@ block0:
; stp fp, lr, [sp, #-16]!
; mov fp, sp
; sub sp, sp, #160
-; ldr fp, [sp, #160]
-; stp fp, lr, [sp]
+; ldr x16, [sp, #160]
+; stp x16, lr, [sp]
; mov fp, sp
; stp x27, x28, [sp, #-16]!
; stp x25, x26, [sp, #-16]!
@@ -258,8 +258,8 @@ block0:
; stp x29, x30, [sp, #-0x10]!
; mov x29, sp
; sub sp, sp, #0xa0
-; ldur x29, [sp, #0xa0]
-; stp x29, x30, [sp]
+; ldur x16, [sp, #0xa0]
+; stp x16, x30, [sp]
; mov x29, sp
; stp x27, x28, [sp, #-0x10]!
; stp x25, x26, [sp, #-0x10]!
diff --git a/cranelift/filetests/filetests/isa/aarch64/return-call.clif b/cranelift/filetests/filetests/isa/aarch64/return-call.clif
index b52c38acb9..21df4307bf 100644
--- a/cranelift/filetests/filetests/isa/aarch64/return-call.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/return-call.clif
@@ -203,8 +203,8 @@ block0:
; stp fp, lr, [sp, #-16]!
; mov fp, sp
; sub sp, sp, #160
-; ldr fp, [sp, #160]
-; stp fp, lr, [sp]
+; ldr x16, [sp, #160]
+; stp x16, lr, [sp]
; mov fp, sp
; stp x27, x28, [sp, #-16]!
; stp x25, x26, [sp, #-16]!
@@ -266,8 +266,8 @@ block0:
; stp x29, x30, [sp, #-0x10]!
; mov x29, sp
; sub sp, sp, #0xa0
-; ldur x29, [sp, #0xa0]
-; stp x29, x30, [sp]
+; ldur x16, [sp, #0xa0]
+; stp x16, x30, [sp]
; mov x29, sp
; stp x27, x28, [sp, #-0x10]!
; stp x25, x26, [sp, #-0x10]!
@@ -428,8 +428,8 @@ block2:
; stp fp, lr, [sp, #-16]!
; mov fp, sp
; sub sp, sp, #176
-; ldr fp, [sp, #176]
-; stp fp, lr, [sp]
+; ldr x16, [sp, #176]
+; stp x16, lr, [sp]
; mov fp, sp
; stp x27, x28, [sp, #-16]!
; stp x25, x26, [sp, #-16]!
@@ -522,8 +522,8 @@ block2:
; stp x29, x30, [sp, #-0x10]!
; mov x29, sp
; sub sp, sp, #0xb0
-; ldur x29, [sp, #0xb0]
-; stp x29, x30, [sp]
+; ldur x16, [sp, #0xb0]
+; stp x16, x30, [sp]
; mov x29, sp
; stp x27, x28, [sp, #-0x10]!
; stp x25, x26, [sp, #-0x10]!
diff --git a/cranelift/filetests/filetests/isa/aarch64/tail-call-grow-frame-unwind.clif b/cranelift/filetests/filetests/isa/aarch64/tail-call-grow-frame-unwind.clif
new file mode 100644
index 0000000000..1c5e886652
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/tail-call-grow-frame-unwind.clif
@@ -0,0 +1,51 @@
+; A `tail` function that grows its incoming argument area for a
+; `return_call` with more stack arguments than it received. The CFI for the
+; saved callee-saved registers must be relative to the moved frame record.
+test unwind
+set unwind_info=true
+set preserve_frame_pointers=true
+target aarch64
+
+function %f(i64) -> i64 tail {
+ fn0 = %g(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 tail
+ fn1 = %h() -> i64 tail
+block0(v0: i64):
+ v1 = call fn1()
+ v2 = call fn1()
+ v3 = iadd v0, v1
+ v4 = call fn1()
+ v5 = iadd v3, v2
+ v6 = iadd v5, v4
+ return_call fn0(v6, v0, v1, v2, v3, v4, v5, v6, v0, v1)
+}
+
+; sameln: 0x00000000: CIE
+; nextln: length: 0x0000000c
+; nextln: version: 0x01
+; nextln: code_align: 4
+; nextln: data_align: -8
+; nextln: ra_register: 0x1e
+; nextln: DW_CFA_def_cfa (r31, 0)
+; nextln: Instructions: Init State:
+; check: 0x00000010: FDE
+; nextln: length: 0x0000002c
+; nextln: CIE_pointer: 0x00000000
+; nextln: start_addr: 0x0000000000000000
+; nextln: range_size: 0x00000000000000c8 (end_addr = 0x00000000000000c8)
+; nextln: Instructions:
+; nextln: DW_CFA_advance_loc (1)
+; nextln: DW_CFA_def_cfa_offset (16)
+; nextln: DW_CFA_offset (r29, 2)
+; nextln: DW_CFA_offset (r30, 1)
+; nextln: DW_CFA_advance_loc (1)
+; nextln: DW_CFA_def_cfa_register (r29)
+; nextln: DW_CFA_advance_loc (5)
+; nextln: DW_CFA_offset (r21, 4)
+; nextln: DW_CFA_offset (r22, 3)
+; nextln: DW_CFA_advance_loc (1)
+; nextln: DW_CFA_offset (r19, 6)
+; nextln: DW_CFA_offset (r20, 5)
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
diff --git a/cranelift/filetests/filetests/isa/aarch64/tail-call-grow-frame.clif b/cranelift/filetests/filetests/isa/aarch64/tail-call-grow-frame.clif
new file mode 100644
index 0000000000..f48079adb9
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/tail-call-grow-frame.clif
@@ -0,0 +1,121 @@
+; A `tail` function that grows its incoming argument area for a
+; `return_call` with more stack arguments than it received. The frame is
+; defined for unwinding before the saved FP and return address are moved,
+; and FP is only updated once both have been moved.
+test compile precise-output
+set unwind_info=true
+set preserve_frame_pointers=true
+target aarch64
+
+function %f(i64) -> i64 tail {
+ fn0 = %g(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 tail
+ fn1 = %h() -> i64 tail
+block0(v0: i64):
+ v1 = call fn1()
+ v2 = call fn1()
+ v3 = iadd v0, v1
+ v4 = call fn1()
+ v5 = iadd v3, v2
+ v6 = iadd v5, v4
+ return_call fn0(v6, v0, v1, v2, v3, v4, v5, v6, v0, v1)
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; unwind PushFrameRegs { offset_upward_to_caller_sp: 16 }
+; mov fp, sp
+; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 32 }
+; sub sp, sp, #32
+; unwind StackAlloc { size: 32 }
+; ldr x16, [sp, #32]
+; stp x16, lr, [sp]
+; mov fp, sp
+; stp x21, x22, [sp, #-16]!
+; unwind SaveReg { clobber_offset: 16, reg: p21i }
+; unwind SaveReg { clobber_offset: 24, reg: p22i }
+; stp x19, x20, [sp, #-16]!
+; unwind SaveReg { clobber_offset: 0, reg: p19i }
+; unwind SaveReg { clobber_offset: 8, reg: p20i }
+; block0:
+; mov x19, x2
+; load_ext_name_far x11, TestCase(%h)+0
+; blr x11
+; mov x22, x2
+; load_ext_name_far x11, TestCase(%h)+0
+; blr x11
+; mov x21, x2
+; mov x2, x22
+; add x20, x19, x2
+; load_ext_name_far x11, TestCase(%h)+0
+; blr x11
+; mov x0, x21
+; add x12, x20, x0
+; add x11, x12, x2
+; mov x7, x2
+; str x12, [sp, #48]
+; str x11, [sp, #56]
+; str x19, [sp, #64]
+; mov x2, x22
+; str x2, [sp, #72]
+; load_ext_name_far x1, TestCase(%g)+0
+; mov x6, x20
+; mov x5, x0
+; mov x4, x2
+; mov x3, x19
+; mov x2, x11
+; return_call_ind x1 new_stack_arg_size:32 x2=x2 x3=x3 x4=x4 x5=x5 x6=x6 x7=x7
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; sub sp, sp, #0x20
+; ldur x16, [sp, #0x20]
+; stp x16, x30, [sp]
+; mov x29, sp
+; stp x21, x22, [sp, #-0x10]!
+; stp x19, x20, [sp, #-0x10]!
+; block1: ; offset 0x20
+; mov x19, x2
+; ldr x11, #0x2c
+; b #0x34
+; udf #0 ; reloc_external Abs8 %h 0
+; udf #0
+; blr x11
+; mov x22, x2
+; ldr x11, #0x44
+; b #0x4c
+; udf #0 ; reloc_external Abs8 %h 0
+; udf #0
+; blr x11
+; mov x21, x2
+; mov x2, x22
+; add x20, x19, x2
+; ldr x11, #0x64
+; b #0x6c
+; udf #0 ; reloc_external Abs8 %h 0
+; udf #0
+; blr x11
+; mov x0, x21
+; add x12, x20, x0
+; add x11, x12, x2
+; mov x7, x2
+; stur x12, [sp, #0x30]
+; stur x11, [sp, #0x38]
+; stur x19, [sp, #0x40]
+; mov x2, x22
+; stur x2, [sp, #0x48]
+; ldr x1, #0x9c
+; b #0xa4
+; udf #0 ; reloc_external Abs8 %g 0
+; udf #0
+; mov x6, x20
+; mov x5, x0
+; mov x4, x2
+; mov x3, x19
+; mov x2, x11
+; ldp x19, x20, [sp], #0x10
+; ldp x21, x22, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; br x1
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
index 56885db4f9..e9117453c2 100644
--- a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
@@ -249,8 +249,8 @@ block0:
; mv fp,sp
; addi sp,sp,-144
; sd ra,8(sp)
-; ld fp,144(sp)
-; sd fp,0(sp)
+; ld t6,144(sp)
+; sd t6,0(sp)
; mv fp,sp
; addi sp,sp,-128
; sd s1,120(sp)
@@ -327,8 +327,8 @@ block0:
; mv s0, sp
; addi sp, sp, -0x90
; sd ra, 8(sp)
-; ld s0, 0x90(sp)
-; sd s0, 0(sp)
+; ld t6, 0x90(sp)
+; sd t6, 0(sp)
; mv s0, sp
; addi sp, sp, -0x80
; sd s1, 0x78(sp)
diff --git a/cranelift/filetests/filetests/isa/riscv64/return-call.clif b/cranelift/filetests/filetests/isa/riscv64/return-call.clif
index dee0225cd5..60d2fcd868 100644
--- a/cranelift/filetests/filetests/isa/riscv64/return-call.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/return-call.clif
@@ -270,8 +270,8 @@ block0:
; mv fp,sp
; addi sp,sp,-144
; sd ra,8(sp)
-; ld fp,144(sp)
-; sd fp,0(sp)
+; ld t6,144(sp)
+; sd t6,0(sp)
; mv fp,sp
; addi sp,sp,-112
; sd s1,104(sp)
@@ -345,8 +345,8 @@ block0:
; mv s0, sp
; addi sp, sp, -0x90
; sd ra, 8(sp)
-; ld s0, 0x90(sp)
-; sd s0, 0(sp)
+; ld t6, 0x90(sp)
+; sd t6, 0(sp)
; mv s0, sp
; addi sp, sp, -0x70
; sd s1, 0x68(sp)
@@ -544,8 +544,8 @@ block2:
; mv fp,sp
; addi sp,sp,-160
; sd ra,8(sp)
-; ld fp,160(sp)
-; sd fp,0(sp)
+; ld t6,160(sp)
+; sd t6,0(sp)
; mv fp,sp
; addi sp,sp,-128
; sd s1,120(sp)
@@ -649,8 +649,8 @@ block2:
; mv s0, sp
; addi sp, sp, -0xa0
; sd ra, 8(sp)
-; ld s0, 0xa0(sp)
-; sd s0, 0(sp)
+; ld t6, 0xa0(sp)
+; sd t6, 0(sp)
; mv s0, sp
; addi sp, sp, -0x80
; sd s1, 0x78(sp)
diff --git a/cranelift/filetests/filetests/isa/riscv64/tail-call-grow-frame-unwind.clif b/cranelift/filetests/filetests/isa/riscv64/tail-call-grow-frame-unwind.clif
new file mode 100644
index 0000000000..a889cc55ee
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/tail-call-grow-frame-unwind.clif
@@ -0,0 +1,51 @@
+; A `tail` function that grows its incoming argument area for a
+; `return_call` with more stack arguments than it received. The CFI for the
+; saved callee-saved registers must be relative to the moved frame record.
+test unwind
+set unwind_info=true
+set preserve_frame_pointers=true
+target riscv64
+
+function %f(i64) -> i64 tail {
+ fn0 = %g(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 tail
+ fn1 = %h() -> i64 tail
+block0(v0: i64):
+ v1 = call fn1()
+ v2 = call fn1()
+ v3 = iadd v0, v1
+ v4 = call fn1()
+ v5 = iadd v3, v2
+ v6 = iadd v5, v4
+ return_call fn0(v6, v0, v1, v2, v3, v4, v5, v6, v0, v1)
+}
+
+; sameln: 0x00000000: CIE
+; nextln: length: 0x0000000c
+; nextln: version: 0x01
+; nextln: code_align: 2
+; nextln: data_align: -8
+; nextln: ra_register: 0x1
+; nextln: DW_CFA_def_cfa (r2, 0)
+; nextln: Instructions: Init State:
+; check: 0x00000010: FDE
+; nextln: length: 0x0000002c
+; nextln: CIE_pointer: 0x00000000
+; nextln: start_addr: 0x0000000000000000
+; nextln: range_size: 0x00000000000000f4 (end_addr = 0x00000000000000f4)
+; nextln: Instructions:
+; nextln: DW_CFA_advance_loc (6)
+; nextln: DW_CFA_def_cfa_offset (16)
+; nextln: DW_CFA_offset (r8, 2)
+; nextln: DW_CFA_offset (r1, 1)
+; nextln: DW_CFA_advance_loc (2)
+; nextln: DW_CFA_def_cfa_register (r8)
+; nextln: DW_CFA_advance_loc (14)
+; nextln: DW_CFA_offset (r18, 3)
+; nextln: DW_CFA_advance_loc (2)
+; nextln: DW_CFA_offset (r19, 4)
+; nextln: DW_CFA_advance_loc (2)
+; nextln: DW_CFA_offset (r20, 5)
+; nextln: DW_CFA_advance_loc (2)
+; nextln: DW_CFA_offset (r21, 6)
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
diff --git a/cranelift/filetests/filetests/isa/riscv64/tail-call-grow-frame.clif b/cranelift/filetests/filetests/isa/riscv64/tail-call-grow-frame.clif
new file mode 100644
index 0000000000..8ab294c8aa
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/tail-call-grow-frame.clif
@@ -0,0 +1,134 @@
+; A `tail` function that grows its incoming argument area for a
+; `return_call` with more stack arguments than it received. The frame is
+; defined for unwinding before the saved FP and return address are moved,
+; and FP is only updated once both have been moved.
+test compile precise-output
+set unwind_info=true
+set preserve_frame_pointers=true
+target riscv64
+
+function %f(i64) -> i64 tail {
+ fn0 = %g(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 tail
+ fn1 = %h() -> i64 tail
+block0(v0: i64):
+ v1 = call fn1()
+ v2 = call fn1()
+ v3 = iadd v0, v1
+ v4 = call fn1()
+ v5 = iadd v3, v2
+ v6 = iadd v5, v4
+ return_call fn0(v6, v0, v1, v2, v3, v4, v5, v6, v0, v1)
+}
+
+; VCode:
+; addi sp,sp,-16
+; sd ra,8(sp)
+; sd fp,0(sp)
+; unwind PushFrameRegs { offset_upward_to_caller_sp: 16 }
+; mv fp,sp
+; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 32 }
+; addi sp,sp,-16
+; sd ra,8(sp)
+; ld t6,16(sp)
+; sd t6,0(sp)
+; mv fp,sp
+; addi sp,sp,-32
+; sd s2,24(sp)
+; unwind SaveReg { clobber_offset: 24, reg: p18i }
+; sd s3,16(sp)
+; unwind SaveReg { clobber_offset: 16, reg: p19i }
+; sd s4,8(sp)
+; unwind SaveReg { clobber_offset: 8, reg: p20i }
+; sd s5,0(sp)
+; unwind SaveReg { clobber_offset: 0, reg: p21i }
+; block0:
+; mv s2,a0
+; load_ext_name_far a1,%h+0
+; callind a1
+; mv s5,a0
+; load_ext_name_far a1,%h+0
+; callind a1
+; mv s4,a0
+; mv a0,s5
+; add s3,s2,a0
+; load_ext_name_far a1,%h+0
+; callind a1
+; mv a1,s4
+; add a6,s3,a1
+; add a7,a6,a0
+; sd s2,-16(incoming_arg)
+; mv a2,s5
+; sd a2,-8(incoming_arg)
+; load_ext_name_far t0,%g+0
+; mv a5,a0
+; mv a4,s3
+; mv a3,a1
+; mv a1,s2
+; mv a0,a7
+; return_call_ind t0 new_stack_arg_size:16 a0=a0 a1=a1 a2=a2 a3=a3 a4=a4 a5=a5 a6=a6 a7=a7
+;
+; Disassembled:
+; block0: ; offset 0x0
+; addi sp, sp, -0x10
+; sd ra, 8(sp)
+; sd s0, 0(sp)
+; mv s0, sp
+; addi sp, sp, -0x10
+; sd ra, 8(sp)
+; ld t6, 0x10(sp)
+; sd t6, 0(sp)
+; mv s0, sp
+; addi sp, sp, -0x20
+; sd s2, 0x18(sp)
+; sd s3, 0x10(sp)
+; sd s4, 8(sp)
+; sd s5, 0(sp)
+; block1: ; offset 0x38
+; mv s2, a0
+; auipc a1, 0
+; ld a1, 0xc(a1)
+; j 0xc
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %h 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; jalr a1
+; mv s5, a0
+; auipc a1, 0
+; ld a1, 0xc(a1)
+; j 0xc
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %h 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; jalr a1
+; mv s4, a0
+; mv a0, s5
+; add s3, s2, a0
+; auipc a1, 0
+; ld a1, 0xc(a1)
+; j 0xc
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %h 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; jalr a1
+; mv a1, s4
+; add a6, s3, a1
+; add a7, a6, a0
+; sd s2, 0x30(sp)
+; mv a2, s5
+; sd a2, 0x38(sp)
+; auipc t0, 0
+; ld t0, 0xc(t0)
+; j 0xc
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %g 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; mv a5, a0
+; mv a4, s3
+; mv a3, a1
+; mv a1, s2
+; mv a0, a7
+; ld s2, 0x18(sp)
+; ld s3, 0x10(sp)
+; ld s4, 8(sp)
+; ld s5, 0(sp)
+; ld ra, 0x28(sp)
+; ld s0, 0x20(sp)
+; addi sp, sp, 0x30
+; jr t0
+
diff --git a/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
index b6cd768835..2c454e7cb1 100644
--- a/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
+++ b/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
@@ -249,11 +249,11 @@ block0:
; pushq %rbp
; movq %rsp, %rbp
; subq $0xa0, %rsp
-; movq %rsp, %rbp
; movq 0xa0(%rsp), %r11
; movq %r11, (%rsp)
; movq 0xa8(%rsp), %r11
; movq %r11, 8(%rsp)
+; movq %rsp, %rbp
; subq $0xa0, %rsp
; movq %rbx, 0x70(%rsp)
; movq %r12, 0x78(%rsp)
@@ -343,11 +343,11 @@ block0:
; pushq %rbp
; movq %rsp, %rbp
; subq $0xa0, %rsp
-; movq %rsp, %rbp
; movq 0xa0(%rsp), %r11
; movq %r11, (%rsp)
; movq 0xa8(%rsp), %r11
; movq %r11, 8(%rsp)
+; movq %rsp, %rbp
; subq $0xa0, %rsp
; movq %rbx, 0x70(%rsp)
; movq %r12, 0x78(%rsp)
diff --git a/cranelift/filetests/filetests/isa/x64/return-call.clif b/cranelift/filetests/filetests/isa/x64/return-call.clif
index 095ad21a9c..14d5bc8d2d 100644
--- a/cranelift/filetests/filetests/isa/x64/return-call.clif
+++ b/cranelift/filetests/filetests/isa/x64/return-call.clif
@@ -310,11 +310,11 @@ block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32):
; pushq %rbp
; movq %rsp, %rbp
; subq $0x10, %rsp
-; movq %rsp, %rbp
; movq 0x10(%rsp), %r11
; movq %r11, (%rsp)
; movq 0x18(%rsp), %r11
; movq %r11, 8(%rsp)
+; movq %rsp, %rbp
; block0:
; movq %r9, %rax
; movq <offset:0>+-0x10(%rbp), %r9
@@ -333,11 +333,11 @@ block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32):
; pushq %rbp
; movq %rsp, %rbp
; subq $0x10, %rsp
-; movq %rsp, %rbp
; movq 0x10(%rsp), %r11
; movq %r11, (%rsp)
; movq 0x18(%rsp), %r11
; movq %r11, 8(%rsp)
+; movq %rsp, %rbp
; block1: ; offset 0x1e
; movq %r9, %rax
; movq 0x20(%rbp), %r9
@@ -416,11 +416,11 @@ block0:
; pushq %rbp
; movq %rsp, %rbp
; subq $0xa0, %rsp
-; movq %rsp, %rbp
; movq 0xa0(%rsp), %r11
; movq %r11, (%rsp)
; movq 0xa8(%rsp), %r11
; movq %r11, 8(%rsp)
+; movq %rsp, %rbp
; subq $0xa0, %rsp
; movq %rbx, 0x70(%rsp)
; movq %r12, 0x78(%rsp)
@@ -508,11 +508,11 @@ block0:
; pushq %rbp
; movq %rsp, %rbp
; subq $0xa0, %rsp
-; movq %rsp, %rbp
; movq 0xa0(%rsp), %r11
; movq %r11, (%rsp)
; movq 0xa8(%rsp), %r11
; movq %r11, 8(%rsp)
+; movq %rsp, %rbp
; subq $0xa0, %rsp
; movq %rbx, 0x70(%rsp)
; movq %r12, 0x78(%rsp)
diff --git a/cranelift/filetests/filetests/isa/x64/tail-call-grow-frame-unwind.clif b/cranelift/filetests/filetests/isa/x64/tail-call-grow-frame-unwind.clif
new file mode 100644
index 0000000000..a4f686a9ef
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/tail-call-grow-frame-unwind.clif
@@ -0,0 +1,58 @@
+; A `tail` function that grows its incoming argument area for a
+; `return_call` with more stack arguments than it received. The CFI for the
+; saved callee-saved registers must be relative to the moved frame record.
+test unwind
+set unwind_info=true
+set preserve_frame_pointers=true
+target x86_64-unknown-linux-gnu
+
+function %f(i64) -> i64 tail {
+ fn0 = %g(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 tail
+ fn1 = %h() -> i64 tail
+block0(v0: i64):
+ v1 = call fn1()
+ v2 = call fn1()
+ v3 = iadd v0, v1
+ v4 = call fn1()
+ v5 = iadd v3, v2
+ v6 = iadd v5, v4
+ return_call fn0(v6, v0, v1, v2, v3, v4, v5, v6, v0, v1)
+}
+
+; sameln: 0x00000000: CIE
+; nextln: length: 0x00000014
+; nextln: version: 0x01
+; nextln: code_align: 1
+; nextln: data_align: -8
+; nextln: ra_register: 0x10
+; nextln: DW_CFA_def_cfa (r7, 8)
+; nextln: DW_CFA_offset (r16, 1)
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
+; nextln: DW_CFA_nop
+; nextln: Instructions: Init State:
+; check: 0x00000018: FDE
+; nextln: length: 0x0000002c
+; nextln: CIE_pointer: 0x00000000
+; nextln: start_addr: 0x0000000000000000
+; nextln: range_size: 0x00000000000000c5 (end_addr = 0x00000000000000c5)
+; nextln: Instructions:
+; nextln: DW_CFA_advance_loc (1)
+; nextln: DW_CFA_def_cfa_offset (16)
+; nextln: DW_CFA_offset (r6, 2)
+; nextln: DW_CFA_advance_loc (3)
+; nextln: DW_CFA_def_cfa_register (r6)
+; nextln: DW_CFA_advance_loc (34)
+; nextln: DW_CFA_offset (r3, 8)
+; nextln: DW_CFA_advance_loc (5)
+; nextln: DW_CFA_offset (r12, 7)
+; nextln: DW_CFA_advance_loc (5)
+; nextln: DW_CFA_offset (r13, 6)
+; nextln: DW_CFA_advance_loc (5)
+; nextln: DW_CFA_offset (r14, 5)
+; nextln: DW_CFA_advance_loc (5)
+; nextln: DW_CFA_offset (r15, 4)
+; nextln: DW_CFA_nop
diff --git a/cranelift/filetests/filetests/isa/x64/tail-call-grow-frame.clif b/cranelift/filetests/filetests/isa/x64/tail-call-grow-frame.clif
new file mode 100644
index 0000000000..27355464cd
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/tail-call-grow-frame.clif
@@ -0,0 +1,122 @@
+; A `tail` function that grows its incoming argument area for a
+; `return_call` with more stack arguments than it received. The frame is
+; defined for unwinding before the saved RBP and return address are moved,
+; and RBP is only updated once both have been moved.
+test compile precise-output
+set unwind_info=true
+set preserve_frame_pointers=true
+target x86_64-unknown-linux-gnu
+
+function %f(i64) -> i64 tail {
+ fn0 = %g(i64, i64, i64, i64, i64, i64, i64, i64, i64, i64) -> i64 tail
+ fn1 = %h() -> i64 tail
+block0(v0: i64):
+ v1 = call fn1()
+ v2 = call fn1()
+ v3 = iadd v0, v1
+ v4 = call fn1()
+ v5 = iadd v3, v2
+ v6 = iadd v5, v4
+ return_call fn0(v6, v0, v1, v2, v3, v4, v5, v6, v0, v1)
+}
+
+; VCode:
+; pushq %rbp
+; unwind PushFrameRegs { offset_upward_to_caller_sp: 16 }
+; movq %rsp, %rbp
+; unwind DefineNewFrame { offset_upward_to_caller_sp: 16, offset_downward_to_clobbers: 48 }
+; subq $0x20, %rsp
+; movq 0x20(%rsp), %r11
+; movq %r11, (%rsp)
+; movq 0x28(%rsp), %r11
+; movq %r11, 8(%rsp)
+; movq %rsp, %rbp
+; subq $0x30, %rsp
+; movq %rbx, (%rsp)
+; unwind SaveReg { clobber_offset: 0, reg: p3i }
+; movq %r12, 8(%rsp)
+; unwind SaveReg { clobber_offset: 8, reg: p12i }
+; movq %r13, 0x10(%rsp)
+; unwind SaveReg { clobber_offset: 16, reg: p13i }
+; movq %r14, 0x18(%rsp)
+; unwind SaveReg { clobber_offset: 24, reg: p14i }
+; movq %r15, 0x20(%rsp)
+; unwind SaveReg { clobber_offset: 32, reg: p15i }
+; block0:
+; movq %rdi, %r14
+; load_ext_name %h+0, %rax
+; call *%rax
+; movq %rax, %r15
+; load_ext_name %h+0, %rcx
+; call *%rcx
+; movq %rax, %r13
+; movq %r14, %rbx
+; movq %r15, %rax
+; leaq (%rbx, %rax), %r12
+; load_ext_name %h+0, %rax
+; call *%rax
+; movq %r13, %rcx
+; leaq (%r12, %rcx), %rdx
+; leaq (%rdx, %rax), %rdi
+; movq %rdx, <offset:0>+-0x20(%rbp)
+; movq %rdi, <offset:0>+-0x18(%rbp)
+; movq %rbx, <offset:0>+-0x10(%rbp)
+; movq %r15, %rdx
+; movq %rdx, <offset:0>+-8(%rbp)
+; load_ext_name %g+0, %r10
+; movq %rbx, %rsi
+; movq %r12, %r8
+; movq %rax, %r9
+; return_call_unknown %r10 (32) tmp=%r11 %rdi=%rdi %rsi=%rsi %rdx=%rdx %rcx=%rcx %r8=%r8 %r9=%r9
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; subq $0x20, %rsp
+; movq 0x20(%rsp), %r11
+; movq %r11, (%rsp)
+; movq 0x28(%rsp), %r11
+; movq %r11, 8(%rsp)
+; movq %rsp, %rbp
+; subq $0x30, %rsp
+; movq %rbx, (%rsp)
+; movq %r12, 8(%rsp)
+; movq %r13, 0x10(%rsp)
+; movq %r14, 0x18(%rsp)
+; movq %r15, 0x20(%rsp)
+; block1: ; offset 0x3a
+; movq %rdi, %r14
+; movabsq $0, %rax ; reloc_external Abs8 %h 0
+; callq *%rax
+; movq %rax, %r15
+; movabsq $0, %rcx ; reloc_external Abs8 %h 0
+; callq *%rcx
+; movq %rax, %r13
+; movq %r14, %rbx
+; movq %r15, %rax
+; leaq (%rbx, %rax), %r12
+; movabsq $0, %rax ; reloc_external Abs8 %h 0
+; callq *%rax
+; movq %r13, %rcx
+; leaq (%r12, %rcx), %rdx
+; leaq (%rdx, %rax), %rdi
+; movq %rdx, 0x10(%rbp)
+; movq %rdi, 0x18(%rbp)
+; movq %rbx, 0x20(%rbp)
+; movq %r15, %rdx
+; movq %rdx, 0x28(%rbp)
+; movabsq $0, %r10 ; reloc_external Abs8 %g 0
+; movq %rbx, %rsi
+; movq %r12, %r8
+; movq %rax, %r9
+; movq (%rsp), %rbx
+; movq 8(%rsp), %r12
+; movq 0x10(%rsp), %r13
+; movq 0x18(%rsp), %r14
+; movq 0x20(%rsp), %r15
+; addq $0x30, %rsp
+; movq %rbp, %rsp
+; popq %rbp
+; jmpq *%r10
+