Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift: Implement 128-bit atomic RMW for s390x (#14809)

Aelin00954ef8e7db
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index 1a385f805e..fa5869e977 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -225,6 +225,17 @@
(rn Reg)
(mem MemArg))
+ ;; An atomic read-modify-write operation with a memory in-/out operand,
+ ;; a register destination, and a register source implemented via an
+ ;; atomic compare-and-swap loop.
+ (AtomicRmw128Loop
+ (op AtomicRmwOp)
+ (rd WritableReg)
+ (rn Reg)
+ (scratch1 WritableRegPair)
+ (scratch2 WritableRegPair)
+ (mem MemArg))
+
;; A 32-bit atomic compare-and-swap operation.
(AtomicCas32
(rd WritableReg)
@@ -2346,6 +2357,18 @@
(_ Unit (emit (MInst.AtomicRmw op dst src mem))))
dst))
+;; Helper for emitting `MInst.AtomicRmw128Loop` instructions.
+(decl atomic_rmw_128_loop (AtomicRmwOp Reg MemArg) Reg)
+(rule (atomic_rmw_128_loop op src mem)
+ (let ((dst WritableReg (temp_writable_reg $I128))
+ (_ Unit (emit (MInst.AtomicRmw128Loop op
+ dst
+ src
+ (temp_writable_regpair $I64)
+ (temp_writable_regpair $I64)
+ mem))))
+ dst))
+
;; Helper for emitting `MInst.AtomicCas32` instructions.
(decl atomic_cas32 (Reg Reg MemArg) Reg)
(rule (atomic_cas32 src1 src2 mem)
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index 28903b2390..bf73306d9d 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -5,6 +5,7 @@ use crate::isa::CallConv;
use crate::isa::s390x::abi::REG_SAVE_AREA_SIZE;
use crate::isa::s390x::inst::*;
use crate::isa::s390x::settings as s390x_settings;
+use crate::opts::AtomicRmwOp;
use cranelift_control::ControlPlane;
/// Debug macro for testing that a regpair is valid: that the high register is even, and the low
@@ -2127,6 +2128,250 @@ impl Inst {
state,
);
}
+ &Inst::AtomicRmw128Loop {
+ op,
+ rd,
+ rn,
+ scratch1,
+ scratch2,
+ ref mem,
+ } => {
+ // rd = VR for previous value
+ // rn = operand value
+ // scratch1: regpair for previous value in GPRs
+ // scratch2: regpair for new value in GPRs
+
+ let loop_label = sink.get_label();
+
+ let scratch1_hi = scratch1.hi;
+ let scratch1_lo = scratch1.lo;
+ debug_assert_valid_regpair!(scratch1_hi.to_reg(), scratch1_lo.to_reg());
+ let scratch2_hi = scratch2.hi;
+ let scratch2_lo = scratch2.lo;
+ debug_assert_valid_regpair!(scratch2_hi.to_reg(), scratch2_lo.to_reg());
+
+ // Do an atomic load of the previous value into a VR
+ Inst::VecLoad {
+ rd,
+ mem: mem.clone(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+
+ // Load the previous value into a pair of GPRs
+ Inst::VecExtractLane {
+ size: 64,
+ rd: scratch1.hi,
+ rn: rd.to_reg(),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ Inst::VecExtractLane {
+ size: 64,
+ rd: scratch1.lo,
+ rn: rd.to_reg(),
+ lane_imm: 1,
+ lane_reg: zero_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+
+ // This is where the compare-and-swap loop starts
+ sink.bind_label(loop_label, &mut state.ctrl_plane);
+
+ // Optionally, do some math to calculate the new value
+ match op {
+ AtomicRmwOp::Add
+ | AtomicRmwOp::Sub
+ | AtomicRmwOp::And
+ | AtomicRmwOp::Nand
+ | AtomicRmwOp::Or
+ | AtomicRmwOp::Xor
+ | AtomicRmwOp::Smax
+ | AtomicRmwOp::Smin
+ | AtomicRmwOp::Umax
+ | AtomicRmwOp::Umin => {
+ let has_vxrs_ext3 = emit_info.isa_flags.has_vxrs_ext3();
+ let is_minmax = matches!(
+ op,
+ AtomicRmwOp::Smax
+ | AtomicRmwOp::Smin
+ | AtomicRmwOp::Umax
+ | AtomicRmwOp::Umin
+ );
+
+ if !is_minmax || has_vxrs_ext3 {
+ let binary_op = match op {
+ AtomicRmwOp::Add => VecBinaryOp::Add128,
+ AtomicRmwOp::Sub => VecBinaryOp::Sub128,
+ AtomicRmwOp::And => VecBinaryOp::And128,
+ AtomicRmwOp::Nand => VecBinaryOp::NotAnd128,
+ AtomicRmwOp::Or => VecBinaryOp::Orr128,
+ AtomicRmwOp::Xor => VecBinaryOp::Xor128,
+ // These require VXRS_EXT3, so use them if possible and otherwise
+ // go through a fallback.
+ AtomicRmwOp::Smax => VecBinaryOp::SMax128,
+ AtomicRmwOp::Smin => VecBinaryOp::SMin128,
+ AtomicRmwOp::Umax => VecBinaryOp::UMax128,
+ AtomicRmwOp::Umin => VecBinaryOp::UMin128,
+ _ => unreachable!(),
+ };
+
+ Inst::VecRRR {
+ op: binary_op,
+ rd,
+ rn: rd.to_reg(),
+ rm: rn,
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+
+ // Load the new value into a pair of GPRs
+ Inst::VecExtractLane {
+ size: 64,
+ rd: scratch2_hi,
+ rn: rd.to_reg(),
+ lane_imm: 0,
+ lane_reg: zero_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ Inst::VecExtractLane {
+ size: 64,
+ rd: scratch2_lo,
+ rn: rd.to_reg(),
+ lane_imm: 1,
+ lane_reg: zero_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ } else {
+ // minmax fallbacks without VXRS_EXT3
+ let lg_label = sink.get_label();
+ let cmp_end_label = sink.get_label();
+
+ let (a, b) = match op {
+ AtomicRmwOp::Umax | AtomicRmwOp::Smax => (rd.to_reg(), rn),
+ AtomicRmwOp::Umin | AtomicRmwOp::Smin => (rn, rd.to_reg()),
+ _ => unreachable!(),
+ };
+
+ // Sets CC 1 if previous value should be kept
+ match op {
+ AtomicRmwOp::Umin | AtomicRmwOp::Umax => {
+ Inst::VecInt128UCmpHi {
+ tmp: rd,
+ rn: a,
+ rm: b,
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ }
+ AtomicRmwOp::Smin | AtomicRmwOp::Smax => {
+ Inst::VecInt128SCmpHi {
+ tmp: rd,
+ rn: a,
+ rm: b,
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ }
+ _ => unreachable!(),
+ };
+
+ // If CC 1, branch to larger than
+ let opcode = 0xc04; // BCRL
+ sink.use_label_at_offset(
+ sink.cur_offset(),
+ lg_label,
+ LabelUse::BranchRIL,
+ );
+ put(sink, &enc_ril_c(opcode, 4, 0));
+
+ // If CC not 1, write new value
+
+ // Load the new value into a pair of GPRs
+ Inst::VecExtractLane {
+ size: 64,
+ rd: scratch2_hi,
+ rn,
+ lane_imm: 0,
+ lane_reg: zero_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ Inst::VecExtractLane {
+ size: 64,
+ rd: scratch2_lo,
+ rn,
+ lane_imm: 1,
+ lane_reg: zero_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+
+ // Now jump past the other branch
+ let opcode = 0xc04; // BCRL
+ sink.use_label_at_offset(
+ sink.cur_offset(),
+ cmp_end_label,
+ LabelUse::BranchRIL,
+ );
+ put(sink, &enc_ril_c(opcode, 15, 0));
+
+ sink.bind_label(lg_label, &mut state.ctrl_plane);
+
+ // If CC 1, keep previous value
+
+ Inst::Mov64 {
+ rd: scratch2_hi,
+ rm: scratch1_hi.to_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ Inst::Mov64 {
+ rd: scratch2_lo,
+ rm: scratch1_lo.to_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+
+ sink.bind_label(cmp_end_label, &mut state.ctrl_plane);
+ }
+ }
+ AtomicRmwOp::Xchg => {
+ // Load the new value into a pair of GPRs
+ Inst::VecExtractLane {
+ size: 64,
+ rd: scratch2_hi,
+ rn,
+ lane_imm: 0,
+ lane_reg: zero_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ Inst::VecExtractLane {
+ size: 64,
+ rd: scratch2_lo,
+ rn,
+ lane_imm: 1,
+ lane_reg: zero_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+ }
+ };
+
+ // Perform the CAS
+ Inst::AtomicCas128 {
+ rd: scratch1,
+ ri: scratch1.to_regpair(),
+ rn: scratch2.to_regpair(),
+ mem: mem.clone(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+
+ // Move the old value into a VR
+ Inst::MovToVec128 {
+ rd,
+ rn: scratch1_hi.to_reg(),
+ rm: scratch1_lo.to_reg(),
+ }
+ .emit_with_alloc_consumer(sink, emit_info, state);
+
+ // If the comparison was not equal, try again
+ let opcode = 0xc04; // BCRL
+ sink.use_label_at_offset(sink.cur_offset(), loop_label, LabelUse::BranchRIL);
+ put(sink, &enc_ril_c(opcode, 4, 0));
+ }
&Inst::Loop { ref body, cond } => {
// This sequence is *one* instruction in the vcode, and is expanded only here at
// emission time, because it requires branching to internal labels.
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
index 18919d885f..bc5346f71d 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
@@ -3340,6 +3340,74 @@ fn test_s390x_binemit() {
"EB456FFF7FE7",
"laxg %r4, %r5, 524287(%r6)",
));
+ insns.push((
+ Inst::AtomicRmw128Loop {
+ op: AtomicRmwOp::Add,
+ rd: writable_vr(1),
+ rn: vr(2),
+ scratch1: w_regpair,
+ scratch2: w_regpair,
+ mem: MemArg::BXD20 {
+ base: zero_reg(),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(-524288).unwrap(),
+ flags: MachMemFlags::trusted(),
+ },
+ },
+ "E31000008071E71010000006B3CD0021E73100013021E711200040F3B3CD0021E73100013021EB220000803EE71230000062C044FFFFFFF2",
+ "atomic_rmw_128_loop_add %v1, %v2, -524288, scratch1=%r2, scratch2=%r2",
+ ));
+ insns.push((
+ Inst::AtomicRmw128Loop {
+ op: AtomicRmwOp::Sub,
+ rd: writable_vr(1),
+ rn: vr(2),
+ scratch1: w_regpair,
+ scratch2: w_regpair,
+ mem: MemArg::BXD20 {
+ base: zero_reg(),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(524287).unwrap(),
+ flags: MachMemFlags::trusted(),
+ },
+ },
+ "E3100FFF7F71E71010000006B3CD0021E73100013021E711200040F7B3CD0021E73100013021EB220FFF7F3EE71230000062C044FFFFFFF2",
+ "atomic_rmw_128_loop_sub %v1, %v2, 524287, scratch1=%r2, scratch2=%r2",
+ ));
+ insns.push((
+ Inst::AtomicRmw128Loop {
+ op: AtomicRmwOp::Or,
+ rd: writable_vr(1),
+ rn: vr(2),
+ scratch1: w_regpair,
+ scratch2: w_regpair,
+ mem: MemArg::BXD20 {
+ base: gpr(6),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(-524288).unwrap(),
+ flags: MachMemFlags::trusted(),
+ },
+ },
+ "E31060008071E71010000006B3CD0021E73100013021E7112000006AB3CD0021E73100013021EB226000803EE71230000062C044FFFFFFF2",
+ "atomic_rmw_128_loop_or %v1, %v2, -524288(%r6), scratch1=%r2, scratch2=%r2",
+ ));
+ insns.push((
+ Inst::AtomicRmw128Loop {
+ op: AtomicRmwOp::Umin,
+ rd: writable_vr(1),
+ rn: vr(2),
+ scratch1: w_regpair,
+ scratch2: w_regpair,
+ mem: MemArg::BXD20 {
+ base: gpr(6),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(524287).unwrap(),
+ flags: MachMemFlags::trusted(),
+ },
+ },
+ "E3106FFF7F71E71010000006B3CD0021E73100013021E711200040FCB3CD0021E73100013021EB226FFF7F3EE71230000062C044FFFFFFF2",
+ "atomic_rmw_128_loop_umin %v1, %v2, 524287(%r6), scratch1=%r2, scratch2=%r2",
+ ));
insns.push((
Inst::AtomicCas32 {
rd: writable_gpr(4),
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index 1c0840f25c..ef6b417ecd 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -1,7 +1,7 @@
//! This module defines s390x-specific machine instruction types.
use crate::binemit::{Addend, CodeOffset, Reloc};
-use crate::ir::{ExternalName, MemFlagsData, Type, types};
+use crate::ir::{AtomicRmwOp, ExternalName, MemFlagsData, Type, types};
use crate::isa::s390x::abi::S390xMachineDeps;
use crate::isa::{CallConv, FunctionAlignment};
use crate::machinst::*;
@@ -56,7 +56,7 @@ pub struct ReturnCallInfo<T> {
fn inst_size_test() {
// This test will help with unintentionally growing the size
// of the Inst enum.
- assert_eq!(40, core::mem::size_of::<Inst>());
+ assert_eq!(48, core::mem::size_of::<Inst>());
}
/// A register pair. Enum so it can be destructured in ISLE.
@@ -132,6 +132,7 @@ impl Inst {
| Inst::CmpTrapRSImm16 { .. }
| Inst::CmpTrapRUImm16 { .. }
| Inst::AtomicRmw { .. }
+ | Inst::AtomicRmw128Loop { .. }
| Inst::AtomicCas32 { .. }
| Inst::AtomicCas64 { .. }
| Inst::AtomicCas128 { .. }
@@ -586,6 +587,22 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl Ope
collector.reg_use(rn);
memarg_operands(mem, collector);
}
+ Inst::AtomicRmw128Loop {
+ rd,
+ rn,
+ scratch1,
+ scratch2,
+ mem,
+ ..
+ } => {
+ collector.reg_early_def(rd);
+ collector.reg_use(rn);
+ collector.reg_fixed_early_def(&mut scratch1.hi, gpr(2));
+ collector.reg_fixed_early_def(&mut scratch1.lo, gpr(3));
+ collector.reg_fixed_early_def(&mut scratch2.hi, gpr(4));
+ collector.reg_fixed_early_def(&mut scratch2.lo, gpr(5));
+ memarg_operands(mem, collector);
+ }
Inst::AtomicCas32 {
rd, ri, rn, mem, ..
}
@@ -1859,6 +1876,49 @@ impl Inst {
let mem = mem.pretty_print_default();
format!("{mem_str}{op} {rd}, {rn}, {mem}")
}
+ &Inst::AtomicRmw128Loop {
+ op,
+ rd,
+ rn,
+ scratch1,
+ scratch2,
+ ref mem,
+ } => {
+ let op = match op {
+ AtomicRmwOp::Add => "add",
+ AtomicRmwOp::Sub => "sub",
+ AtomicRmwOp::And => "and",
+ AtomicRmwOp::Nand => "nand",
+ AtomicRmwOp::Or => "or",
+ AtomicRmwOp::Xor => "xor",
+ AtomicRmwOp::Smax => "smax",
+ AtomicRmwOp::Smin => "smin",
+ AtomicRmwOp::Umax => "umax",
+ AtomicRmwOp::Umin => "umin",
+ AtomicRmwOp::Xchg => "xchg",
+ };
+
+ let rd = pretty_print_reg(rd.to_reg());
+ let rn = pretty_print_reg(rn);
+ let scratch1 = pretty_print_regpair(scratch1.to_regpair());
+ let scratch2 = pretty_print_regpair(scratch2.to_regpair());
+ let mem = mem.clone();
+ let (mem_str, mem) = mem_finalize_for_show(
+ &mem,
+ state,
+ MemInstType {
+ have_d12: false,
+ have_d20: true,
+ have_pcrel: false,
+ have_unaligned_pcrel: false,
+ have_index: false,
+ },
+ );
+ let mem = mem.pretty_print_default();
+ format!(
+ "{mem_str}atomic_rmw_128_loop_{op} {rd}, {rn}, {mem}, scratch1={scratch1}, scratch2={scratch2}"
+ )
+ }
&Inst::AtomicCas32 {
rd,
ri,
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index 6ce4894aad..467fb3a757 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -3506,8 +3506,12 @@
;; Atomic operations that require a compare-and-swap loop.
+;; Operations for 128-bit types use a compare-and-swap loop not implementable in ISLE
+(rule -1 (lower (atomic_rmw $I128 (mem_flags_data flags) op addr src))
+ (atomic_rmw_128_loop op src (lower_address flags addr (zero_offset))))
+
;; Operations for 32/64-bit types can use a fullword compare-and-swap loop.
-(rule -1 (lower (atomic_rmw (ty_32_or_64 ty) (mem_flags_data flags) op addr src))
+(rule -2 (lower (atomic_rmw (ty_32_or_64 ty) (mem_flags_data flags) op addr src))
(let ((src_reg Reg (put_in_reg src))
(addr_reg Reg (put_in_reg addr))
;; Create body of compare-and-swap loop.
@@ -3519,7 +3523,7 @@
(casloop ib ty flags addr_reg val1)))
;; Operations for 8/16-bit types must operate on the surrounding aligned word.
-(rule -2 (lower (atomic_rmw (ty_8_or_16 ty) (mem_flags_data flags) op addr src))
+(rule -3 (lower (atomic_rmw (ty_8_or_16 ty) (mem_flags_data flags) op addr src))
(let ((src_reg Reg (put_in_reg src))
(addr_reg Reg (put_in_reg addr))
;; Prepare access to surrounding aligned word.
diff --git a/cranelift/codegen/src/machinst/reg.rs b/cranelift/codegen/src/machinst/reg.rs
index 0c004483cc..29218857af 100644
--- a/cranelift/codegen/src/machinst/reg.rs
+++ b/cranelift/codegen/src/machinst/reg.rs
@@ -469,6 +469,15 @@ pub trait OperandVisitorImpl: OperandVisitor {
self.reg_fixed(reg.reg.as_mut(), rreg, OperandKind::Def, OperandPos::Late);
}
+ /// Add a register "early fixed def", which ties a vreg to a particular
+ /// RealReg and logically occurs at the beginning of the instruction,
+ /// alongside all uses. Use this when the def may be written before
+ /// all uses are read; the regalloc will ensure that it does not overwrite
+ /// any uses.
+ fn reg_fixed_early_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>, rreg: Reg) {
+ self.reg_fixed(reg.reg.as_mut(), rreg, OperandKind::Def, OperandPos::Early);
+ }
+
/// Add an operand tying a virtual register to a physical register.
fn reg_fixed(&mut self, reg: &mut Reg, rreg: Reg, kind: OperandKind, pos: OperandPos) {
debug_assert!(reg.is_virtual());
diff --git a/cranelift/filetests/filetests/isa/s390x/atomic_rmw.clif b/cranelift/filetests/filetests/isa/s390x/atomic_rmw.clif
index 2792bd61f2..ad48a4605c 100644
--- a/cranelift/filetests/filetests/isa/s390x/atomic_rmw.clif
+++ b/cranelift/filetests/filetests/isa/s390x/atomic_rmw.clif
@@ -1,10 +1,49 @@
test compile precise-output
+set enable_multi_ret_implicit_sret
target s390x
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ATOMIC_RMW (XCHG)
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+function %atomic_rmw_xchg_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 xchg v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_xchg %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; lgdr %r4, %f2
+; vlgvg %r5, %v2, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_xchg_i64(i64, i64, i64) -> i64 {
block0(v0: i64, v1: i64, v2: i64):
v3 = atomic_rmw.i64 xchg v1, v2
@@ -109,6 +148,45 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r4)
; br %r14
+function %atomic_rmw_add_i128(i64, i128) -> i128 {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 add v0, v1
+ return v2
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; atomic_rmw_128_loop_add %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vaq %v5, %v5, %v2
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_add_i64(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 add v0, v1
@@ -205,6 +283,45 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r5)
; br %r14
+function %atomic_rmw_sub_i128(i64, i128) -> i128 {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 sub v0, v1
+ return v2
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; atomic_rmw_128_loop_sub %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vsq %v5, %v5, %v2
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_sub_i64(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 sub v0, v1
@@ -305,6 +422,45 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r5)
; br %r14
+function %atomic_rmw_and_i128(i64, i128) -> i128 {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 and v0, v1
+ return v2
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; atomic_rmw_128_loop_and %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vn %v5, %v5, %v2
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_and_i64(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 and v0, v1
@@ -399,6 +555,45 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r4)
; br %r14
+function %atomic_rmw_or_i128(i64, i128) -> i128 {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 or v0, v1
+ return v2
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; atomic_rmw_128_loop_or %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vo %v5, %v5, %v2
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_or_i64(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 or v0, v1
@@ -493,6 +688,45 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r4)
; br %r14
+function %atomic_rmw_xor_i128(i64, i128) -> i128 {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 xor v0, v1
+ return v2
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; atomic_rmw_128_loop_xor %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r3
+; lgr %r12, %r2
+; vl %v2, 0(%r4)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vx %v5, %v5, %v2
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_xor_i64(i64, i64) -> i64 {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 xor v0, v1
@@ -587,6 +821,45 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r4)
; br %r14
+function %atomic_rmw_nand_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 nand v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_nand %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vnn %v5, %v5, %v2
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_nand_i64(i64, i64, i64) -> i64 {
block0(v0: i64, v1: i64, v2: i64):
v3 = atomic_rmw.i64 nand v1, v2
@@ -698,6 +971,51 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r4)
; br %r14
+function %atomic_rmw_smin_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 smin v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_smin %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vecg %v5, %v2
+; jne 0x34
+; vchlgs %v5, %v2, %v5
+; jgl 0x4a
+; lgdr %r4, %f2
+; vlgvg %r5, %v2, 1
+; jg 0x52
+; lgr %r4, %r2
+; lgr %r5, %r3
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_smin_i64(i64, i64, i64) -> i64 {
block0(v0: i64, v1: i64, v2: i64):
v3 = atomic_rmw.i64 smin v1, v2
@@ -812,6 +1130,51 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r5)
; br %r14
+function %atomic_rmw_smax_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 smax v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_smax %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vecg %v2, %v5
+; jne 0x34
+; vchlgs %v5, %v5, %v2
+; jgl 0x4a
+; lgdr %r4, %f2
+; vlgvg %r5, %v2, 1
+; jg 0x52
+; lgr %r4, %r2
+; lgr %r5, %r3
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_smax_i64(i64, i64, i64) -> i64 {
block0(v0: i64, v1: i64, v2: i64):
v3 = atomic_rmw.i64 smax v1, v2
@@ -926,6 +1289,51 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r5)
; br %r14
+function %atomic_rmw_umin_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 umin v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_umin %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; veclg %v5, %v2
+; jne 0x34
+; vchlgs %v5, %v2, %v5
+; jgl 0x4a
+; lgdr %r4, %f2
+; vlgvg %r5, %v2, 1
+; jg 0x52
+; lgr %r4, %r2
+; lgr %r5, %r3
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_umin_i64(i64, i64, i64) -> i64 {
block0(v0: i64, v1: i64, v2: i64):
v3 = atomic_rmw.i64 umin v1, v2
@@ -1040,6 +1448,51 @@ block0(v0: i64, v1: i64, v2: i8):
; rll %r2, %r0, 8(%r5)
; br %r14
+function %atomic_rmw_umax_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 umax v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_umax %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; veclg %v2, %v5
+; jne 0x34
+; vchlgs %v5, %v5, %v2
+; jgl 0x4a
+; lgdr %r4, %f2
+; vlgvg %r5, %v2, 1
+; jg 0x52
+; lgr %r4, %r2
+; lgr %r5, %r3
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
function %atomic_rmw_umax_i64(i64, i64, i64) -> i64 {
block0(v0: i64, v1: i64, v2: i64):
v3 = atomic_rmw.i64 umax v1, v2
diff --git a/cranelift/filetests/filetests/isa/s390x/atomic_rmw_128-vxrs_ext3.clif b/cranelift/filetests/filetests/isa/s390x/atomic_rmw_128-vxrs_ext3.clif
new file mode 100644
index 0000000000..79bfe585b4
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/atomic_rmw_128-vxrs_ext3.clif
@@ -0,0 +1,160 @@
+test compile precise-output
+set enable_multi_ret_implicit_sret
+target s390x has_vxrs_ext3
+
+function %atomic_rmw_smin_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 smin v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_smin %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vmn %v5, %v5, %v2, 4
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
+function %atomic_rmw_smax_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 smax v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_smax %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vmx %v5, %v5, %v2, 4
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
+function %atomic_rmw_umin_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 umin v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_umin %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vmnl %v5, %v5, %v2, 4
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
+function %atomic_rmw_umax_i128(i64, i64, i128) -> i128 {
+block0(v0: i64, v1: i64, v2: i128):
+ v3 = atomic_rmw.i128 umax v1, v2
+ return v3
+}
+
+; VCode:
+; stmg %r11, %r15, 88(%r15)
+; block0:
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; atomic_rmw_128_loop_umax %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 88(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r11, %r15, 0x58(%r15)
+; block1: ; offset 0x6
+; lgr %r11, %r4
+; lgr %r12, %r2
+; vl %v2, 0(%r5)
+; vl %v5, 0(%r11) ; trap: heap_oob
+; lgdr %r2, %f5
+; vlgvg %r3, %v5, 1
+; vmxl %v5, %v5, %v2, 4
+; lgdr %r4, %f5
+; vlgvg %r5, %v5, 1
+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob
+; vlvgp %v5, %r2, %r3
+; jgl 0x24
+; lgr %r2, %r12
+; vst %v5, 0(%r2)
+; lmg %r11, %r15, 0x58(%r15)
+; br %r14
+
diff --git a/cranelift/filetests/filetests/runtests/atomic-128.clif b/cranelift/filetests/filetests/runtests/atomic-128.clif
index 0019952714..35449a406b 100644
--- a/cranelift/filetests/filetests/runtests/atomic-128.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-128.clif
@@ -4,16 +4,23 @@ set enable_llvm_abi_extensions
set enable_multi_ret_implicit_sret
target x86_64 has_cmpxchg16b
target aarch64
+target s390x
function %atomic_load(i128) -> i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128):
- v3 = stack_addr.i64 ss0
- store.i128 notrap v0, v3
+ ; Dynamically align the slot to 16
v1 = stack_addr.i64 ss0
- v2 = atomic_load.i128 v1
- return v2
+ v2 = iconst.i64 15
+ v3 = band v1, v2
+ v4 = iconst.i64 16
+ v5 = isub v4, v3
+ v6 = iadd v1, v5
+
+ store.i128 notrap v0, v6
+ v7 = atomic_load.i128 v6
+ return v7
}
; run: %atomic_load(0) == 0
; run: %atomic_load(-1) == -1
@@ -25,14 +32,20 @@ block0(v0: i128):
function %atomic_store(i128) -> i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128):
+ ; Dynamically align the slot to 16
v1 = stack_addr.i64 ss0
- atomic_store.i128 v0, v1
- v3 = stack_addr.i64 ss0
- v2 = load.i128 notrap v3
- return v2
+ v2 = iconst.i64 15
+ v3 = band v1, v2
+ v4 = iconst.i64 16
+ v5 = isub v4, v3
+ v6 = iadd v1, v5
+
+ atomic_store.i128 v0, v6
+ v7 = load.i128 notrap v6
+ return v7
}
; run: %atomic_store(0) == 0
; run: %atomic_store(-1) == -1
@@ -44,16 +57,21 @@ block0(v0: i128):
function %atomic_cas(i128, i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128, v2: i128):
- v6 = stack_addr.i64 ss0
- store.i128 notrap v0, v6
+ ; Dynamically align the slot to 16
v3 = stack_addr.i64 ss0
- v4 = atomic_cas.i128 v3, v1, v2
- v7 = stack_addr.i64 ss0
- v5 = load.i128 notrap v7
- return v5, v4
+ v4 = iconst.i64 15
+ v5 = band v3, v4
+ v6 = iconst.i64 16
+ v7 = isub v6, v5
+ v8 = iadd v3, v7
+
+ store.i128 notrap v0, v8
+ v9 = atomic_cas.i128 v8, v1, v2
+ v10 = load.i128 notrap v8
+ return v10, v9
}
; run: %atomic_cas(0, 0, 2) == [2, 0]
@@ -64,16 +82,21 @@ block0(v0: i128, v1: i128, v2: i128):
function %atomic_add(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 add v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 add v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_add(0, 0) == [0, 0]
@@ -85,16 +108,21 @@ block0(v0: i128, v1: i128):
function %atomic_sub(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 sub v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 sub v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_sub(0, 0) == [0, 0]
@@ -106,16 +134,21 @@ block0(v0: i128, v1: i128):
function %atomic_and(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 and v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 and v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_and(0, 0) == [0, 0]
@@ -127,16 +160,21 @@ block0(v0: i128, v1: i128):
function %atomic_nand(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 nand v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 nand v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_nand(0, 0) == [-1, 0]
@@ -148,16 +186,21 @@ block0(v0: i128, v1: i128):
function %atomic_or(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 or v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 or v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_or(0, 0) == [0, 0]
@@ -169,16 +212,21 @@ block0(v0: i128, v1: i128):
function %atomic_xor(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 xor v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 xor v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_xor(0, 0) == [0, 0]
@@ -190,16 +238,21 @@ block0(v0: i128, v1: i128):
function %atomic_xchg(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 xchg v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 xchg v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_xchg(0, 0) == [0, 0]
@@ -211,16 +264,21 @@ block0(v0: i128, v1: i128):
function %atomic_umin(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 umin v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 umin v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_umin(0, 0) == [0, 0]
@@ -234,16 +292,21 @@ block0(v0: i128, v1: i128):
function %atomic_umax(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 umax v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 umax v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_umax(0, 0) == [0, 0]
@@ -257,16 +320,21 @@ block0(v0: i128, v1: i128):
function %atomic_smin(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 smin v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 smin v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_smin(0, 0) == [0, 0]
@@ -280,16 +348,21 @@ block0(v0: i128, v1: i128):
function %atomic_smax(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16, align=16
+ ss0 = explicit_slot 32, align=8
block0(v0: i128, v1: i128):
- v5 = stack_addr.i64 ss0
- store.i128 notrap v0, v5
+ ; Dynamically align the slot to 16
v2 = stack_addr.i64 ss0
- v3 = atomic_rmw.i128 smax v2, v1
- v6 = stack_addr.i64 ss0
- v4 = load.i128 notrap v6
- return v4, v3
+ v3 = iconst.i64 15
+ v4 = band v2, v3
+ v5 = iconst.i64 16
+ v6 = isub v5, v4
+ v7 = iadd v2, v6
+
+ store.i128 notrap v0, v7
+ v8 = atomic_rmw.i128 smax v7, v1
+ v9 = load.i128 notrap v7
+ return v9, v8
}
; run: %atomic_smax(0, 0) == [0, 0]