Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cranelift: s390x: fix patchable call ABI to not clobber outside clobber-save area. (#12148)

Chris Fallin99ef35d1477c
diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs
index e1f2c040d0..2cec91bbdd 100644
--- a/cranelift/codegen/src/isa/s390x/abi.rs
+++ b/cranelift/codegen/src/isa/s390x/abi.rs
@@ -794,28 +794,65 @@ impl ABIMachineSpec for S390xMachineDeps {
});
}
- // Save FPRs.
- for (i, reg) in get_clobbered_fprs(frame_layout).iter().enumerate() {
- insts.push(Inst::VecStoreLane {
- size: 64,
- rd: reg.to_reg().into(),
- mem: MemArg::reg_plus_off(
- stack_reg(),
- (i * 8) as i64
- + frame_layout.outgoing_args_size as i64
- + frame_layout.fixed_frame_storage_size as i64,
- MemFlags::trusted(),
- ),
- lane_imm: 0,
- });
- if flags.unwind_info() {
- insts.push(Inst::Unwind {
- inst: UnwindInst::SaveReg {
- clobber_offset: (i * 8) as u32,
- reg: reg.to_reg(),
- },
+ // Write the dedicated clobber region.
+ match call_conv {
+ isa::CallConv::Patchable => {
+ // Explicitly save full v0-v31. Save r0-r5; r6-r15 are
+ // in the ABI-defined register save area.
+ //
+ // N.B.: unwind metadata doesn't support a full vector
+ // reg clobber-save, and we don't expect an unwinder
+ // to try to unwind the patchable ABI anyway, so we
+ // omit unwind insts.
+ for i in 0..32 {
+ insts.push(Inst::VecStore {
+ rd: regs::vr(i),
+ mem: MemArg::reg_plus_off(
+ stack_reg(),
+ 6 * 8
+ + (i as i64) * 16
+ + frame_layout.outgoing_args_size as i64
+ + frame_layout.fixed_frame_storage_size as i64,
+ MemFlags::trusted(),
+ ),
+ });
+ }
+ insts.push(Inst::StoreMultiple64 {
+ rt: regs::gpr(0),
+ rt2: regs::gpr(5),
+ mem: MemArg::reg_plus_off(
+ stack_reg(),
+ frame_layout.outgoing_args_size as i64
+ + frame_layout.fixed_frame_storage_size as i64,
+ MemFlags::trusted(),
+ ),
});
}
+ _ => {
+ // Save FPRs.
+ for (i, reg) in get_clobbered_fprs(frame_layout).iter().enumerate() {
+ insts.push(Inst::VecStoreLane {
+ size: 64,
+ rd: reg.to_reg().into(),
+ mem: MemArg::reg_plus_off(
+ stack_reg(),
+ (i * 8) as i64
+ + frame_layout.outgoing_args_size as i64
+ + frame_layout.fixed_frame_storage_size as i64,
+ MemFlags::trusted(),
+ ),
+ lane_imm: 0,
+ });
+ if flags.unwind_info() {
+ insts.push(Inst::Unwind {
+ inst: UnwindInst::SaveReg {
+ clobber_offset: (i * 8) as u32,
+ reg: reg.to_reg(),
+ },
+ });
+ }
+ }
+ }
}
insts
@@ -828,8 +865,15 @@ impl ABIMachineSpec for S390xMachineDeps {
) -> SmallVec<[Inst; 16]> {
let mut insts = SmallVec::new();
- // Restore FPRs.
- insts.extend(gen_restore_fprs(frame_layout));
+ match call_conv {
+ isa::CallConv::Patchable => {
+ insts.extend(gen_restore_patchable(frame_layout));
+ }
+ _ => {
+ // Restore FPRs.
+ insts.extend(gen_restore_fprs(frame_layout));
+ }
+ }
// Restore GPRs (including SP).
insts.extend(gen_restore_gprs(call_conv, frame_layout, 0));
@@ -942,17 +986,28 @@ impl ABIMachineSpec for S390xMachineDeps {
// sort because the registers will be unique (there are no dups).
regs.sort_unstable();
- // Compute clobber size. We only need to count FPR save slots.
- let mut clobber_size = 0;
- for reg in &regs {
- match reg.to_reg().class() {
- RegClass::Int => {}
- RegClass::Float => {
- clobber_size += 8;
+ // Compute clobber size. If we are in the patchable ABI, we
+ // know we save r0-r5 and v0-v31 in the separate clobber area,
+ // and use the register-save area for r6-r15; otherwise we
+ // don't count GPRs and only save the FPR part of vector regs.
+ let mut clobber_size = match call_conv {
+ isa::CallConv::Patchable => 8 * 6 + 16 * 32,
+ _ => {
+ let mut clobber_size = 0;
+ for reg in &regs {
+ match reg.to_reg().class() {
+ RegClass::Int => {}
+ RegClass::Float => {
+ // We only need to count FPR save slots in the
+ // ordinary (SysV and tail-call) ABIs.
+ clobber_size += 8;
+ }
+ RegClass::Vector => unreachable!(),
+ }
}
- RegClass::Vector => unreachable!(),
+ clobber_size
}
- }
+ };
// Common code assumes that tail-call arguments are part of the caller's
// frame. This is not correct for our tail-call convention. To ensure
@@ -1131,6 +1186,12 @@ fn get_clobbered_gprs(frame_layout: &FrameLayout) -> Option<(u8, u8)> {
let last = clobbered_gpr.last().unwrap().to_reg().hw_enc();
debug_assert!(clobbered_gpr.iter().all(|r| r.to_reg().hw_enc() >= first));
debug_assert!(clobbered_gpr.iter().all(|r| r.to_reg().hw_enc() <= last));
+
+ // Explicitly clamp `first` at r6 at a minimum; that is what the
+ // reg-save area has space for. If we're in the patchable ABI with
+ // all regs clobber-saved, we save r0-r5 separately.
+ let first = core::cmp::max(first, 6);
+
Some((first, last))
}
@@ -1227,6 +1288,35 @@ fn gen_restore_fprs(frame_layout: &FrameLayout) -> SmallVec<[Inst; 16]> {
insts
}
+fn gen_restore_patchable(frame_layout: &FrameLayout) -> SmallVec<[Inst; 16]> {
+ let mut insts = SmallVec::new();
+
+ for i in 0..32 {
+ insts.push(Inst::VecLoad {
+ rd: regs::writable_vr(i),
+ mem: MemArg::reg_plus_off(
+ stack_reg(),
+ 6 * 8
+ + (i as i64) * 16
+ + frame_layout.outgoing_args_size as i64
+ + frame_layout.fixed_frame_storage_size as i64,
+ MemFlags::trusted(),
+ ),
+ });
+ }
+ insts.push(Inst::LoadMultiple64 {
+ rt: regs::writable_gpr(0),
+ rt2: regs::writable_gpr(5),
+ mem: MemArg::reg_plus_off(
+ stack_reg(),
+ frame_layout.outgoing_args_size as i64 + frame_layout.fixed_frame_storage_size as i64,
+ MemFlags::trusted(),
+ ),
+ });
+
+ insts
+}
+
const fn sysv_clobbers() -> PRegSet {
PRegSet::empty()
.with(gpr_preg(0))
diff --git a/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif b/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif
index 8ecdf5c1f5..25fb7f3949 100644
--- a/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif
+++ b/cranelift/filetests/filetests/isa/s390x/patchable-abi.clif
@@ -42,155 +42,159 @@ block0(v0: i64):
}
; VCode:
-; stmg %r0, %r15, 0(%r15)
-; aghi %r15, -416
-; std %f0, 160(%r15)
-; std %f1, 168(%r15)
-; std %f2, 176(%r15)
-; std %f3, 184(%r15)
-; std %f4, 192(%r15)
-; std %f5, 200(%r15)
-; std %f6, 208(%r15)
-; std %f7, 216(%r15)
-; std %f8, 224(%r15)
-; std %f9, 232(%r15)
-; std %f10, 240(%r15)
-; std %f11, 248(%r15)
-; std %f12, 256(%r15)
-; std %f13, 264(%r15)
-; std %f14, 272(%r15)
-; std %f15, 280(%r15)
-; vsteg %v16, 288(%r15), 0
-; vsteg %v17, 296(%r15), 0
-; vsteg %v18, 304(%r15), 0
-; vsteg %v19, 312(%r15), 0
-; vsteg %v20, 320(%r15), 0
-; vsteg %v21, 328(%r15), 0
-; vsteg %v22, 336(%r15), 0
-; vsteg %v23, 344(%r15), 0
-; vsteg %v24, 352(%r15), 0
-; vsteg %v25, 360(%r15), 0
-; vsteg %v26, 368(%r15), 0
-; vsteg %v27, 376(%r15), 0
-; vsteg %v28, 384(%r15), 0
-; vsteg %v29, 392(%r15), 0
-; vsteg %v30, 400(%r15), 0
-; vsteg %v31, 408(%r15), 0
+; stmg %r6, %r15, 48(%r15)
+; aghi %r15, -720
+; vst %v0, 208(%r15)
+; vst %v1, 224(%r15)
+; vst %v2, 240(%r15)
+; vst %v3, 256(%r15)
+; vst %v4, 272(%r15)
+; vst %v5, 288(%r15)
+; vst %v6, 304(%r15)
+; vst %v7, 320(%r15)
+; vst %v8, 336(%r15)
+; vst %v9, 352(%r15)
+; vst %v10, 368(%r15)
+; vst %v11, 384(%r15)
+; vst %v12, 400(%r15)
+; vst %v13, 416(%r15)
+; vst %v14, 432(%r15)
+; vst %v15, 448(%r15)
+; vst %v16, 464(%r15)
+; vst %v17, 480(%r15)
+; vst %v18, 496(%r15)
+; vst %v19, 512(%r15)
+; vst %v20, 528(%r15)
+; vst %v21, 544(%r15)
+; vst %v22, 560(%r15)
+; vst %v23, 576(%r15)
+; vst %v24, 592(%r15)
+; vst %v25, 608(%r15)
+; vst %v26, 624(%r15)
+; vst %v27, 640(%r15)
+; vst %v28, 656(%r15)
+; vst %v29, 672(%r15)
+; vst %v30, 688(%r15)
+; vst %v31, 704(%r15)
+; stmg %r0, %r5, 160(%r15)
; block0:
; bras %r1, 12 ; data %libcall + 0 ; lg %r4, 0(%r1)
; basr %r14, %r4
-; ld %f0, 160(%r15)
-; ld %f1, 168(%r15)
-; ld %f2, 176(%r15)
-; ld %f3, 184(%r15)
-; ld %f4, 192(%r15)
-; ld %f5, 200(%r15)
-; ld %f6, 208(%r15)
-; ld %f7, 216(%r15)
-; ld %f8, 224(%r15)
-; ld %f9, 232(%r15)
-; ld %f10, 240(%r15)
-; ld %f11, 248(%r15)
-; ld %f12, 256(%r15)
-; ld %f13, 264(%r15)
-; ld %f14, 272(%r15)
-; ld %f15, 280(%r15)
-; vleg %v16, 288(%r15), 0
-; vleg %v17, 296(%r15), 0
-; vleg %v18, 304(%r15), 0
-; vleg %v19, 312(%r15), 0
-; vleg %v20, 320(%r15), 0
-; vleg %v21, 328(%r15), 0
-; vleg %v22, 336(%r15), 0
-; vleg %v23, 344(%r15), 0
-; vleg %v24, 352(%r15), 0
-; vleg %v25, 360(%r15), 0
-; vleg %v26, 368(%r15), 0
-; vleg %v27, 376(%r15), 0
-; vleg %v28, 384(%r15), 0
-; vleg %v29, 392(%r15), 0
-; vleg %v30, 400(%r15), 0
-; vleg %v31, 408(%r15), 0
-; lmg %r0, %r15, 416(%r15)
+; vl %v0, 208(%r15)
+; vl %v1, 224(%r15)
+; vl %v2, 240(%r15)
+; vl %v3, 256(%r15)
+; vl %v4, 272(%r15)
+; vl %v5, 288(%r15)
+; vl %v6, 304(%r15)
+; vl %v7, 320(%r15)
+; vl %v8, 336(%r15)
+; vl %v9, 352(%r15)
+; vl %v10, 368(%r15)
+; vl %v11, 384(%r15)
+; vl %v12, 400(%r15)
+; vl %v13, 416(%r15)
+; vl %v14, 432(%r15)
+; vl %v15, 448(%r15)
+; vl %v16, 464(%r15)
+; vl %v17, 480(%r15)
+; vl %v18, 496(%r15)
+; vl %v19, 512(%r15)
+; vl %v20, 528(%r15)
+; vl %v21, 544(%r15)
+; vl %v22, 560(%r15)
+; vl %v23, 576(%r15)
+; vl %v24, 592(%r15)
+; vl %v25, 608(%r15)
+; vl %v26, 624(%r15)
+; vl %v27, 640(%r15)
+; vl %v28, 656(%r15)
+; vl %v29, 672(%r15)
+; vl %v30, 688(%r15)
+; vl %v31, 704(%r15)
+; lmg %r0, %r5, 160(%r15)
+; lmg %r6, %r15, 768(%r15)
; br %r14
;
; Disassembled:
; block0: ; offset 0x0
-; stmg %r0, %r15, 0(%r15)
-; aghi %r15, -0x1a0
-; std %f0, 0xa0(%r15)
-; std %f1, 0xa8(%r15)
-; std %f2, 0xb0(%r15)
-; std %f3, 0xb8(%r15)
-; std %f4, 0xc0(%r15)
-; std %f5, 0xc8(%r15)
-; std %f6, 0xd0(%r15)
-; std %f7, 0xd8(%r15)
-; std %f8, 0xe0(%r15)
-; std %f9, 0xe8(%r15)
-; std %f10, 0xf0(%r15)
-; std %f11, 0xf8(%r15)
-; std %f12, 0x100(%r15)
-; std %f13, 0x108(%r15)
-; std %f14, 0x110(%r15)
-; std %f15, 0x118(%r15)
-; vsteg %v16, 0x120(%r15), 0
-; vsteg %v17, 0x128(%r15), 0
-; vsteg %v18, 0x130(%r15), 0
-; vsteg %v19, 0x138(%r15), 0
-; vsteg %v20, 0x140(%r15), 0
-; vsteg %v21, 0x148(%r15), 0
-; vsteg %v22, 0x150(%r15), 0
-; vsteg %v23, 0x158(%r15), 0
-; vsteg %v24, 0x160(%r15), 0
-; vsteg %v25, 0x168(%r15), 0
-; vsteg %v26, 0x170(%r15), 0
-; vsteg %v27, 0x178(%r15), 0
-; vsteg %v28, 0x180(%r15), 0
-; vsteg %v29, 0x188(%r15), 0
-; vsteg %v30, 0x190(%r15), 0
-; vsteg %v31, 0x198(%r15), 0
-; block1: ; offset 0xaa
-; bras %r1, 0xb6
+; stmg %r6, %r15, 0x30(%r15)
+; aghi %r15, -0x2d0
+; vst %v0, 0xd0(%r15)
+; vst %v1, 0xe0(%r15)
+; vst %v2, 0xf0(%r15)
+; vst %v3, 0x100(%r15)
+; vst %v4, 0x110(%r15)
+; vst %v5, 0x120(%r15)
+; vst %v6, 0x130(%r15)
+; vst %v7, 0x140(%r15)
+; vst %v8, 0x150(%r15)
+; vst %v9, 0x160(%r15)
+; vst %v10, 0x170(%r15)
+; vst %v11, 0x180(%r15)
+; vst %v12, 0x190(%r15)
+; vst %v13, 0x1a0(%r15)
+; vst %v14, 0x1b0(%r15)
+; vst %v15, 0x1c0(%r15)
+; vst %v16, 0x1d0(%r15)
+; vst %v17, 0x1e0(%r15)
+; vst %v18, 0x1f0(%r15)
+; vst %v19, 0x200(%r15)
+; vst %v20, 0x210(%r15)
+; vst %v21, 0x220(%r15)
+; vst %v22, 0x230(%r15)
+; vst %v23, 0x240(%r15)
+; vst %v24, 0x250(%r15)
+; vst %v25, 0x260(%r15)
+; vst %v26, 0x270(%r15)
+; vst %v27, 0x280(%r15)
+; vst %v28, 0x290(%r15)
+; vst %v29, 0x2a0(%r15)
+; vst %v30, 0x2b0(%r15)
+; vst %v31, 0x2c0(%r15)
+; stmg %r0, %r5, 0xa0(%r15)
+; block1: ; offset 0xd0
+; bras %r1, 0xdc
; .byte 0x00, 0x00 ; reloc_external Abs8 %libcall 0
; .byte 0x00, 0x00
; .byte 0x00, 0x00
; .byte 0x00, 0x00
; lg %r4, 0(%r1)
; basr %r14, %r4
-; ld %f0, 0xa0(%r15)
-; ld %f1, 0xa8(%r15)
-; ld %f2, 0xb0(%r15)
-; ld %f3, 0xb8(%r15)
-; ld %f4, 0xc0(%r15)
-; ld %f5, 0xc8(%r15)
-; ld %f6, 0xd0(%r15)
-; ld %f7, 0xd8(%r15)
-; ld %f8, 0xe0(%r15)
-; ld %f9, 0xe8(%r15)
-; ld %f10, 0xf0(%r15)
-; ld %f11, 0xf8(%r15)
-; ld %f12, 0x100(%r15)
-; ld %f13, 0x108(%r15)
-; ld %f14, 0x110(%r15)
-; ld %f15, 0x118(%r15)
-; vleg %v16, 0x120(%r15), 0
-; vleg %v17, 0x128(%r15), 0
-; vleg %v18, 0x130(%r15), 0
-; vleg %v19, 0x138(%r15), 0
-; vleg %v20, 0x140(%r15), 0
-; vleg %v21, 0x148(%r15), 0
-; vleg %v22, 0x150(%r15), 0
-; vleg %v23, 0x158(%r15), 0
-; vleg %v24, 0x160(%r15), 0
-; vleg %v25, 0x168(%r15), 0
-; vleg %v26, 0x170(%r15), 0
-; vleg %v27, 0x178(%r15), 0
-; vleg %v28, 0x180(%r15), 0
-; vleg %v29, 0x188(%r15), 0
-; vleg %v30, 0x190(%r15), 0
-; vleg %v31, 0x198(%r15), 0
-; lmg %r0, %r15, 0x1a0(%r15)
+; vl %v0, 0xd0(%r15)
+; vl %v1, 0xe0(%r15)
+; vl %v2, 0xf0(%r15)
+; vl %v3, 0x100(%r15)
+; vl %v4, 0x110(%r15)
+; vl %v5, 0x120(%r15)
+; vl %v6, 0x130(%r15)
+; vl %v7, 0x140(%r15)
+; vl %v8, 0x150(%r15)
+; vl %v9, 0x160(%r15)
+; vl %v10, 0x170(%r15)
+; vl %v11, 0x180(%r15)
+; vl %v12, 0x190(%r15)
+; vl %v13, 0x1a0(%r15)
+; vl %v14, 0x1b0(%r15)
+; vl %v15, 0x1c0(%r15)
+; vl %v16, 0x1d0(%r15)
+; vl %v17, 0x1e0(%r15)
+; vl %v18, 0x1f0(%r15)
+; vl %v19, 0x200(%r15)
+; vl %v20, 0x210(%r15)
+; vl %v21, 0x220(%r15)
+; vl %v22, 0x230(%r15)
+; vl %v23, 0x240(%r15)
+; vl %v24, 0x250(%r15)
+; vl %v25, 0x260(%r15)
+; vl %v26, 0x270(%r15)
+; vl %v27, 0x280(%r15)
+; vl %v28, 0x290(%r15)
+; vl %v29, 0x2a0(%r15)
+; vl %v30, 0x2b0(%r15)
+; vl %v31, 0x2c0(%r15)
+; lmg %r0, %r5, 0xa0(%r15)
+; lmg %r6, %r15, 0x300(%r15)
; br %r14
function %patchable_call(i64) system_v {