A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift: Implement 128-bit atomic RMW for s390x (#14809)
Aelin00954ef8e7db
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isleindex 1a385f805e..fa5869e977 100644--- a/cranelift/codegen/src/isa/s390x/inst.isle+++ b/cranelift/codegen/src/isa/s390x/inst.isle@@ -225,6 +225,17 @@(rn Reg)(mem MemArg))+ ;; An atomic read-modify-write operation with a memory in-/out operand,+ ;; a register destination, and a register source implemented via an+ ;; atomic compare-and-swap loop.+ (AtomicRmw128Loop+ (op AtomicRmwOp)+ (rd WritableReg)+ (rn Reg)+ (scratch1 WritableRegPair)+ (scratch2 WritableRegPair)+ (mem MemArg))+;; A 32-bit atomic compare-and-swap operation.(AtomicCas32(rd WritableReg)@@ -2346,6 +2357,18 @@(_ Unit (emit (MInst.AtomicRmw op dst src mem))))dst))+;; Helper for emitting `MInst.AtomicRmw128Loop` instructions.+(decl atomic_rmw_128_loop (AtomicRmwOp Reg MemArg) Reg)+(rule (atomic_rmw_128_loop op src mem)+ (let ((dst WritableReg (temp_writable_reg $I128))+ (_ Unit (emit (MInst.AtomicRmw128Loop op+ dst+ src+ (temp_writable_regpair $I64)+ (temp_writable_regpair $I64)+ mem))))+ dst))+;; Helper for emitting `MInst.AtomicCas32` instructions.(decl atomic_cas32 (Reg Reg MemArg) Reg)(rule (atomic_cas32 src1 src2 mem)diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rsindex 28903b2390..bf73306d9d 100644--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs@@ -5,6 +5,7 @@ use crate::isa::CallConv;use crate::isa::s390x::abi::REG_SAVE_AREA_SIZE;use crate::isa::s390x::inst::*;use crate::isa::s390x::settings as s390x_settings;+use crate::opts::AtomicRmwOp;use cranelift_control::ControlPlane;/// Debug macro for testing that a regpair is valid: that the high register is even, and the low@@ -2127,6 +2128,250 @@ impl Inst {state,);}+ &Inst::AtomicRmw128Loop {+ op,+ rd,+ rn,+ scratch1,+ scratch2,+ ref mem,+ } => {+ // rd = VR for previous value+ // rn = operand value+ // scratch1: regpair for previous value in GPRs+ // scratch2: regpair for new value in GPRs++ let loop_label = sink.get_label();++ let scratch1_hi = scratch1.hi;+ let scratch1_lo = scratch1.lo;+ debug_assert_valid_regpair!(scratch1_hi.to_reg(), scratch1_lo.to_reg());+ let scratch2_hi = scratch2.hi;+ let scratch2_lo = scratch2.lo;+ debug_assert_valid_regpair!(scratch2_hi.to_reg(), scratch2_lo.to_reg());++ // Do an atomic load of the previous value into a VR+ Inst::VecLoad {+ rd,+ mem: mem.clone(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);++ // Load the previous value into a pair of GPRs+ Inst::VecExtractLane {+ size: 64,+ rd: scratch1.hi,+ rn: rd.to_reg(),+ lane_imm: 0,+ lane_reg: zero_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ Inst::VecExtractLane {+ size: 64,+ rd: scratch1.lo,+ rn: rd.to_reg(),+ lane_imm: 1,+ lane_reg: zero_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);++ // This is where the compare-and-swap loop starts+ sink.bind_label(loop_label, &mut state.ctrl_plane);++ // Optionally, do some math to calculate the new value+ match op {+ AtomicRmwOp::Add+ | AtomicRmwOp::Sub+ | AtomicRmwOp::And+ | AtomicRmwOp::Nand+ | AtomicRmwOp::Or+ | AtomicRmwOp::Xor+ | AtomicRmwOp::Smax+ | AtomicRmwOp::Smin+ | AtomicRmwOp::Umax+ | AtomicRmwOp::Umin => {+ let has_vxrs_ext3 = emit_info.isa_flags.has_vxrs_ext3();+ let is_minmax = matches!(+ op,+ AtomicRmwOp::Smax+ | AtomicRmwOp::Smin+ | AtomicRmwOp::Umax+ | AtomicRmwOp::Umin+ );++ if !is_minmax || has_vxrs_ext3 {+ let binary_op = match op {+ AtomicRmwOp::Add => VecBinaryOp::Add128,+ AtomicRmwOp::Sub => VecBinaryOp::Sub128,+ AtomicRmwOp::And => VecBinaryOp::And128,+ AtomicRmwOp::Nand => VecBinaryOp::NotAnd128,+ AtomicRmwOp::Or => VecBinaryOp::Orr128,+ AtomicRmwOp::Xor => VecBinaryOp::Xor128,+ // These require VXRS_EXT3, so use them if possible and otherwise+ // go through a fallback.+ AtomicRmwOp::Smax => VecBinaryOp::SMax128,+ AtomicRmwOp::Smin => VecBinaryOp::SMin128,+ AtomicRmwOp::Umax => VecBinaryOp::UMax128,+ AtomicRmwOp::Umin => VecBinaryOp::UMin128,+ _ => unreachable!(),+ };++ Inst::VecRRR {+ op: binary_op,+ rd,+ rn: rd.to_reg(),+ rm: rn,+ }+ .emit_with_alloc_consumer(sink, emit_info, state);++ // Load the new value into a pair of GPRs+ Inst::VecExtractLane {+ size: 64,+ rd: scratch2_hi,+ rn: rd.to_reg(),+ lane_imm: 0,+ lane_reg: zero_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ Inst::VecExtractLane {+ size: 64,+ rd: scratch2_lo,+ rn: rd.to_reg(),+ lane_imm: 1,+ lane_reg: zero_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ } else {+ // minmax fallbacks without VXRS_EXT3+ let lg_label = sink.get_label();+ let cmp_end_label = sink.get_label();++ let (a, b) = match op {+ AtomicRmwOp::Umax | AtomicRmwOp::Smax => (rd.to_reg(), rn),+ AtomicRmwOp::Umin | AtomicRmwOp::Smin => (rn, rd.to_reg()),+ _ => unreachable!(),+ };++ // Sets CC 1 if previous value should be kept+ match op {+ AtomicRmwOp::Umin | AtomicRmwOp::Umax => {+ Inst::VecInt128UCmpHi {+ tmp: rd,+ rn: a,+ rm: b,+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ }+ AtomicRmwOp::Smin | AtomicRmwOp::Smax => {+ Inst::VecInt128SCmpHi {+ tmp: rd,+ rn: a,+ rm: b,+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ }+ _ => unreachable!(),+ };++ // If CC 1, branch to larger than+ let opcode = 0xc04; // BCRL+ sink.use_label_at_offset(+ sink.cur_offset(),+ lg_label,+ LabelUse::BranchRIL,+ );+ put(sink, &enc_ril_c(opcode, 4, 0));++ // If CC not 1, write new value++ // Load the new value into a pair of GPRs+ Inst::VecExtractLane {+ size: 64,+ rd: scratch2_hi,+ rn,+ lane_imm: 0,+ lane_reg: zero_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ Inst::VecExtractLane {+ size: 64,+ rd: scratch2_lo,+ rn,+ lane_imm: 1,+ lane_reg: zero_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);++ // Now jump past the other branch+ let opcode = 0xc04; // BCRL+ sink.use_label_at_offset(+ sink.cur_offset(),+ cmp_end_label,+ LabelUse::BranchRIL,+ );+ put(sink, &enc_ril_c(opcode, 15, 0));++ sink.bind_label(lg_label, &mut state.ctrl_plane);++ // If CC 1, keep previous value++ Inst::Mov64 {+ rd: scratch2_hi,+ rm: scratch1_hi.to_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ Inst::Mov64 {+ rd: scratch2_lo,+ rm: scratch1_lo.to_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);++ sink.bind_label(cmp_end_label, &mut state.ctrl_plane);+ }+ }+ AtomicRmwOp::Xchg => {+ // Load the new value into a pair of GPRs+ Inst::VecExtractLane {+ size: 64,+ rd: scratch2_hi,+ rn,+ lane_imm: 0,+ lane_reg: zero_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ Inst::VecExtractLane {+ size: 64,+ rd: scratch2_lo,+ rn,+ lane_imm: 1,+ lane_reg: zero_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);+ }+ };++ // Perform the CAS+ Inst::AtomicCas128 {+ rd: scratch1,+ ri: scratch1.to_regpair(),+ rn: scratch2.to_regpair(),+ mem: mem.clone(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);++ // Move the old value into a VR+ Inst::MovToVec128 {+ rd,+ rn: scratch1_hi.to_reg(),+ rm: scratch1_lo.to_reg(),+ }+ .emit_with_alloc_consumer(sink, emit_info, state);++ // If the comparison was not equal, try again+ let opcode = 0xc04; // BCRL+ sink.use_label_at_offset(sink.cur_offset(), loop_label, LabelUse::BranchRIL);+ put(sink, &enc_ril_c(opcode, 4, 0));+ }&Inst::Loop { ref body, cond } => {// This sequence is *one* instruction in the vcode, and is expanded only here at// emission time, because it requires branching to internal labels.diff --git a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rsindex 18919d885f..bc5346f71d 100644--- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs+++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs@@ -3340,6 +3340,74 @@ fn test_s390x_binemit() {"EB456FFF7FE7","laxg %r4, %r5, 524287(%r6)",));+ insns.push((+ Inst::AtomicRmw128Loop {+ op: AtomicRmwOp::Add,+ rd: writable_vr(1),+ rn: vr(2),+ scratch1: w_regpair,+ scratch2: w_regpair,+ mem: MemArg::BXD20 {+ base: zero_reg(),+ index: zero_reg(),+ disp: SImm20::maybe_from_i64(-524288).unwrap(),+ flags: MachMemFlags::trusted(),+ },+ },+ "E31000008071E71010000006B3CD0021E73100013021E711200040F3B3CD0021E73100013021EB220000803EE71230000062C044FFFFFFF2",+ "atomic_rmw_128_loop_add %v1, %v2, -524288, scratch1=%r2, scratch2=%r2",+ ));+ insns.push((+ Inst::AtomicRmw128Loop {+ op: AtomicRmwOp::Sub,+ rd: writable_vr(1),+ rn: vr(2),+ scratch1: w_regpair,+ scratch2: w_regpair,+ mem: MemArg::BXD20 {+ base: zero_reg(),+ index: zero_reg(),+ disp: SImm20::maybe_from_i64(524287).unwrap(),+ flags: MachMemFlags::trusted(),+ },+ },+ "E3100FFF7F71E71010000006B3CD0021E73100013021E711200040F7B3CD0021E73100013021EB220FFF7F3EE71230000062C044FFFFFFF2",+ "atomic_rmw_128_loop_sub %v1, %v2, 524287, scratch1=%r2, scratch2=%r2",+ ));+ insns.push((+ Inst::AtomicRmw128Loop {+ op: AtomicRmwOp::Or,+ rd: writable_vr(1),+ rn: vr(2),+ scratch1: w_regpair,+ scratch2: w_regpair,+ mem: MemArg::BXD20 {+ base: gpr(6),+ index: zero_reg(),+ disp: SImm20::maybe_from_i64(-524288).unwrap(),+ flags: MachMemFlags::trusted(),+ },+ },+ "E31060008071E71010000006B3CD0021E73100013021E7112000006AB3CD0021E73100013021EB226000803EE71230000062C044FFFFFFF2",+ "atomic_rmw_128_loop_or %v1, %v2, -524288(%r6), scratch1=%r2, scratch2=%r2",+ ));+ insns.push((+ Inst::AtomicRmw128Loop {+ op: AtomicRmwOp::Umin,+ rd: writable_vr(1),+ rn: vr(2),+ scratch1: w_regpair,+ scratch2: w_regpair,+ mem: MemArg::BXD20 {+ base: gpr(6),+ index: zero_reg(),+ disp: SImm20::maybe_from_i64(524287).unwrap(),+ flags: MachMemFlags::trusted(),+ },+ },+ "E3106FFF7F71E71010000006B3CD0021E73100013021E711200040FCB3CD0021E73100013021EB226FFF7F3EE71230000062C044FFFFFFF2",+ "atomic_rmw_128_loop_umin %v1, %v2, 524287(%r6), scratch1=%r2, scratch2=%r2",+ ));insns.push((Inst::AtomicCas32 {rd: writable_gpr(4),diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rsindex 1c0840f25c..ef6b417ecd 100644--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs@@ -1,7 +1,7 @@//! This module defines s390x-specific machine instruction types.use crate::binemit::{Addend, CodeOffset, Reloc};-use crate::ir::{ExternalName, MemFlagsData, Type, types};+use crate::ir::{AtomicRmwOp, ExternalName, MemFlagsData, Type, types};use crate::isa::s390x::abi::S390xMachineDeps;use crate::isa::{CallConv, FunctionAlignment};use crate::machinst::*;@@ -56,7 +56,7 @@ pub struct ReturnCallInfo<T> {fn inst_size_test() {// This test will help with unintentionally growing the size// of the Inst enum.- assert_eq!(40, core::mem::size_of::<Inst>());+ assert_eq!(48, core::mem::size_of::<Inst>());}/// A register pair. Enum so it can be destructured in ISLE.@@ -132,6 +132,7 @@ impl Inst {| Inst::CmpTrapRSImm16 { .. }| Inst::CmpTrapRUImm16 { .. }| Inst::AtomicRmw { .. }+ | Inst::AtomicRmw128Loop { .. }| Inst::AtomicCas32 { .. }| Inst::AtomicCas64 { .. }| Inst::AtomicCas128 { .. }@@ -586,6 +587,22 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl Opecollector.reg_use(rn);memarg_operands(mem, collector);}+ Inst::AtomicRmw128Loop {+ rd,+ rn,+ scratch1,+ scratch2,+ mem,+ ..+ } => {+ collector.reg_early_def(rd);+ collector.reg_use(rn);+ collector.reg_fixed_early_def(&mut scratch1.hi, gpr(2));+ collector.reg_fixed_early_def(&mut scratch1.lo, gpr(3));+ collector.reg_fixed_early_def(&mut scratch2.hi, gpr(4));+ collector.reg_fixed_early_def(&mut scratch2.lo, gpr(5));+ memarg_operands(mem, collector);+ }Inst::AtomicCas32 {rd, ri, rn, mem, ..}@@ -1859,6 +1876,49 @@ impl Inst {let mem = mem.pretty_print_default();format!("{mem_str}{op} {rd}, {rn}, {mem}")}+ &Inst::AtomicRmw128Loop {+ op,+ rd,+ rn,+ scratch1,+ scratch2,+ ref mem,+ } => {+ let op = match op {+ AtomicRmwOp::Add => "add",+ AtomicRmwOp::Sub => "sub",+ AtomicRmwOp::And => "and",+ AtomicRmwOp::Nand => "nand",+ AtomicRmwOp::Or => "or",+ AtomicRmwOp::Xor => "xor",+ AtomicRmwOp::Smax => "smax",+ AtomicRmwOp::Smin => "smin",+ AtomicRmwOp::Umax => "umax",+ AtomicRmwOp::Umin => "umin",+ AtomicRmwOp::Xchg => "xchg",+ };++ let rd = pretty_print_reg(rd.to_reg());+ let rn = pretty_print_reg(rn);+ let scratch1 = pretty_print_regpair(scratch1.to_regpair());+ let scratch2 = pretty_print_regpair(scratch2.to_regpair());+ let mem = mem.clone();+ let (mem_str, mem) = mem_finalize_for_show(+ &mem,+ state,+ MemInstType {+ have_d12: false,+ have_d20: true,+ have_pcrel: false,+ have_unaligned_pcrel: false,+ have_index: false,+ },+ );+ let mem = mem.pretty_print_default();+ format!(+ "{mem_str}atomic_rmw_128_loop_{op} {rd}, {rn}, {mem}, scratch1={scratch1}, scratch2={scratch2}"+ )+ }&Inst::AtomicCas32 {rd,ri,diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isleindex 6ce4894aad..467fb3a757 100644--- a/cranelift/codegen/src/isa/s390x/lower.isle+++ b/cranelift/codegen/src/isa/s390x/lower.isle@@ -3506,8 +3506,12 @@;; Atomic operations that require a compare-and-swap loop.+;; Operations for 128-bit types use a compare-and-swap loop not implementable in ISLE+(rule -1 (lower (atomic_rmw $I128 (mem_flags_data flags) op addr src))+ (atomic_rmw_128_loop op src (lower_address flags addr (zero_offset))))+;; Operations for 32/64-bit types can use a fullword compare-and-swap loop.-(rule -1 (lower (atomic_rmw (ty_32_or_64 ty) (mem_flags_data flags) op addr src))+(rule -2 (lower (atomic_rmw (ty_32_or_64 ty) (mem_flags_data flags) op addr src))(let ((src_reg Reg (put_in_reg src))(addr_reg Reg (put_in_reg addr));; Create body of compare-and-swap loop.@@ -3519,7 +3523,7 @@(casloop ib ty flags addr_reg val1)));; Operations for 8/16-bit types must operate on the surrounding aligned word.-(rule -2 (lower (atomic_rmw (ty_8_or_16 ty) (mem_flags_data flags) op addr src))+(rule -3 (lower (atomic_rmw (ty_8_or_16 ty) (mem_flags_data flags) op addr src))(let ((src_reg Reg (put_in_reg src))(addr_reg Reg (put_in_reg addr));; Prepare access to surrounding aligned word.diff --git a/cranelift/codegen/src/machinst/reg.rs b/cranelift/codegen/src/machinst/reg.rsindex 0c004483cc..29218857af 100644--- a/cranelift/codegen/src/machinst/reg.rs+++ b/cranelift/codegen/src/machinst/reg.rs@@ -469,6 +469,15 @@ pub trait OperandVisitorImpl: OperandVisitor {self.reg_fixed(reg.reg.as_mut(), rreg, OperandKind::Def, OperandPos::Late);}+ /// Add a register "early fixed def", which ties a vreg to a particular+ /// RealReg and logically occurs at the beginning of the instruction,+ /// alongside all uses. Use this when the def may be written before+ /// all uses are read; the regalloc will ensure that it does not overwrite+ /// any uses.+ fn reg_fixed_early_def(&mut self, reg: &mut Writable<impl AsMut<Reg>>, rreg: Reg) {+ self.reg_fixed(reg.reg.as_mut(), rreg, OperandKind::Def, OperandPos::Early);+ }+/// Add an operand tying a virtual register to a physical register.fn reg_fixed(&mut self, reg: &mut Reg, rreg: Reg, kind: OperandKind, pos: OperandPos) {debug_assert!(reg.is_virtual());diff --git a/cranelift/filetests/filetests/isa/s390x/atomic_rmw.clif b/cranelift/filetests/filetests/isa/s390x/atomic_rmw.clifindex 2792bd61f2..ad48a4605c 100644--- a/cranelift/filetests/filetests/isa/s390x/atomic_rmw.clif+++ b/cranelift/filetests/filetests/isa/s390x/atomic_rmw.clif@@ -1,10 +1,49 @@test compile precise-output+set enable_multi_ret_implicit_srettarget s390x;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ATOMIC_RMW (XCHG);;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+function %atomic_rmw_xchg_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 xchg v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_xchg %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; lgdr %r4, %f2+; vlgvg %r5, %v2, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_xchg_i64(i64, i64, i64) -> i64 {block0(v0: i64, v1: i64, v2: i64):v3 = atomic_rmw.i64 xchg v1, v2@@ -109,6 +148,45 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r4); br %r14+function %atomic_rmw_add_i128(i64, i128) -> i128 {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 add v0, v1+ return v2+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; atomic_rmw_128_loop_add %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vaq %v5, %v5, %v2+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_add_i64(i64, i64) -> i64 {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 add v0, v1@@ -205,6 +283,45 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r5); br %r14+function %atomic_rmw_sub_i128(i64, i128) -> i128 {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 sub v0, v1+ return v2+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; atomic_rmw_128_loop_sub %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vsq %v5, %v5, %v2+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_sub_i64(i64, i64) -> i64 {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 sub v0, v1@@ -305,6 +422,45 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r5); br %r14+function %atomic_rmw_and_i128(i64, i128) -> i128 {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 and v0, v1+ return v2+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; atomic_rmw_128_loop_and %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vn %v5, %v5, %v2+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_and_i64(i64, i64) -> i64 {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 and v0, v1@@ -399,6 +555,45 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r4); br %r14+function %atomic_rmw_or_i128(i64, i128) -> i128 {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 or v0, v1+ return v2+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; atomic_rmw_128_loop_or %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vo %v5, %v5, %v2+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_or_i64(i64, i64) -> i64 {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 or v0, v1@@ -493,6 +688,45 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r4); br %r14+function %atomic_rmw_xor_i128(i64, i128) -> i128 {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 xor v0, v1+ return v2+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; atomic_rmw_128_loop_xor %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r3+; lgr %r12, %r2+; vl %v2, 0(%r4)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vx %v5, %v5, %v2+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_xor_i64(i64, i64) -> i64 {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 xor v0, v1@@ -587,6 +821,45 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r4); br %r14+function %atomic_rmw_nand_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 nand v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_nand %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vnn %v5, %v5, %v2+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_nand_i64(i64, i64, i64) -> i64 {block0(v0: i64, v1: i64, v2: i64):v3 = atomic_rmw.i64 nand v1, v2@@ -698,6 +971,51 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r4); br %r14+function %atomic_rmw_smin_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 smin v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_smin %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vecg %v5, %v2+; jne 0x34+; vchlgs %v5, %v2, %v5+; jgl 0x4a+; lgdr %r4, %f2+; vlgvg %r5, %v2, 1+; jg 0x52+; lgr %r4, %r2+; lgr %r5, %r3+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_smin_i64(i64, i64, i64) -> i64 {block0(v0: i64, v1: i64, v2: i64):v3 = atomic_rmw.i64 smin v1, v2@@ -812,6 +1130,51 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r5); br %r14+function %atomic_rmw_smax_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 smax v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_smax %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vecg %v2, %v5+; jne 0x34+; vchlgs %v5, %v5, %v2+; jgl 0x4a+; lgdr %r4, %f2+; vlgvg %r5, %v2, 1+; jg 0x52+; lgr %r4, %r2+; lgr %r5, %r3+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_smax_i64(i64, i64, i64) -> i64 {block0(v0: i64, v1: i64, v2: i64):v3 = atomic_rmw.i64 smax v1, v2@@ -926,6 +1289,51 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r5); br %r14+function %atomic_rmw_umin_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 umin v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_umin %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; veclg %v5, %v2+; jne 0x34+; vchlgs %v5, %v2, %v5+; jgl 0x4a+; lgdr %r4, %f2+; vlgvg %r5, %v2, 1+; jg 0x52+; lgr %r4, %r2+; lgr %r5, %r3+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_umin_i64(i64, i64, i64) -> i64 {block0(v0: i64, v1: i64, v2: i64):v3 = atomic_rmw.i64 umin v1, v2@@ -1040,6 +1448,51 @@ block0(v0: i64, v1: i64, v2: i8):; rll %r2, %r0, 8(%r5); br %r14+function %atomic_rmw_umax_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 umax v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_umax %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; veclg %v2, %v5+; jne 0x34+; vchlgs %v5, %v5, %v2+; jgl 0x4a+; lgdr %r4, %f2+; vlgvg %r5, %v2, 1+; jg 0x52+; lgr %r4, %r2+; lgr %r5, %r3+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+function %atomic_rmw_umax_i64(i64, i64, i64) -> i64 {block0(v0: i64, v1: i64, v2: i64):v3 = atomic_rmw.i64 umax v1, v2diff --git a/cranelift/filetests/filetests/isa/s390x/atomic_rmw_128-vxrs_ext3.clif b/cranelift/filetests/filetests/isa/s390x/atomic_rmw_128-vxrs_ext3.clifnew file mode 100644index 0000000000..79bfe585b4--- /dev/null+++ b/cranelift/filetests/filetests/isa/s390x/atomic_rmw_128-vxrs_ext3.clif@@ -0,0 +1,160 @@+test compile precise-output+set enable_multi_ret_implicit_sret+target s390x has_vxrs_ext3++function %atomic_rmw_smin_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 smin v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_smin %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vmn %v5, %v5, %v2, 4+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14++function %atomic_rmw_smax_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 smax v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_smax %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vmx %v5, %v5, %v2, 4+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14++function %atomic_rmw_umin_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 umin v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_umin %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vmnl %v5, %v5, %v2, 4+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14++function %atomic_rmw_umax_i128(i64, i64, i128) -> i128 {+block0(v0: i64, v1: i64, v2: i128):+ v3 = atomic_rmw.i128 umax v1, v2+ return v3+}++; VCode:+; stmg %r11, %r15, 88(%r15)+; block0:+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; atomic_rmw_128_loop_umax %v5, %v2, 0(%r11), scratch1=%r2, scratch2=%r4+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 88(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r11, %r15, 0x58(%r15)+; block1: ; offset 0x6+; lgr %r11, %r4+; lgr %r12, %r2+; vl %v2, 0(%r5)+; vl %v5, 0(%r11) ; trap: heap_oob+; lgdr %r2, %f5+; vlgvg %r3, %v5, 1+; vmxl %v5, %v5, %v2, 4+; lgdr %r4, %f5+; vlgvg %r5, %v5, 1+; cdsg %r2, %r4, 0(%r11) ; trap: heap_oob+; vlvgp %v5, %r2, %r3+; jgl 0x24+; lgr %r2, %r12+; vst %v5, 0(%r2)+; lmg %r11, %r15, 0x58(%r15)+; br %r14+diff --git a/cranelift/filetests/filetests/runtests/atomic-128.clif b/cranelift/filetests/filetests/runtests/atomic-128.clifindex 0019952714..35449a406b 100644--- a/cranelift/filetests/filetests/runtests/atomic-128.clif+++ b/cranelift/filetests/filetests/runtests/atomic-128.clif@@ -4,16 +4,23 @@ set enable_llvm_abi_extensionsset enable_multi_ret_implicit_srettarget x86_64 has_cmpxchg16btarget aarch64+target s390xfunction %atomic_load(i128) -> i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128):- v3 = stack_addr.i64 ss0- store.i128 notrap v0, v3+ ; Dynamically align the slot to 16v1 = stack_addr.i64 ss0- v2 = atomic_load.i128 v1- return v2+ v2 = iconst.i64 15+ v3 = band v1, v2+ v4 = iconst.i64 16+ v5 = isub v4, v3+ v6 = iadd v1, v5++ store.i128 notrap v0, v6+ v7 = atomic_load.i128 v6+ return v7}; run: %atomic_load(0) == 0; run: %atomic_load(-1) == -1@@ -25,14 +32,20 @@ block0(v0: i128):function %atomic_store(i128) -> i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128):+ ; Dynamically align the slot to 16v1 = stack_addr.i64 ss0- atomic_store.i128 v0, v1- v3 = stack_addr.i64 ss0- v2 = load.i128 notrap v3- return v2+ v2 = iconst.i64 15+ v3 = band v1, v2+ v4 = iconst.i64 16+ v5 = isub v4, v3+ v6 = iadd v1, v5++ atomic_store.i128 v0, v6+ v7 = load.i128 notrap v6+ return v7}; run: %atomic_store(0) == 0; run: %atomic_store(-1) == -1@@ -44,16 +57,21 @@ block0(v0: i128):function %atomic_cas(i128, i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128, v2: i128):- v6 = stack_addr.i64 ss0- store.i128 notrap v0, v6+ ; Dynamically align the slot to 16v3 = stack_addr.i64 ss0- v4 = atomic_cas.i128 v3, v1, v2- v7 = stack_addr.i64 ss0- v5 = load.i128 notrap v7- return v5, v4+ v4 = iconst.i64 15+ v5 = band v3, v4+ v6 = iconst.i64 16+ v7 = isub v6, v5+ v8 = iadd v3, v7++ store.i128 notrap v0, v8+ v9 = atomic_cas.i128 v8, v1, v2+ v10 = load.i128 notrap v8+ return v10, v9}; run: %atomic_cas(0, 0, 2) == [2, 0]@@ -64,16 +82,21 @@ block0(v0: i128, v1: i128, v2: i128):function %atomic_add(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 add v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 add v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_add(0, 0) == [0, 0]@@ -85,16 +108,21 @@ block0(v0: i128, v1: i128):function %atomic_sub(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 sub v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 sub v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_sub(0, 0) == [0, 0]@@ -106,16 +134,21 @@ block0(v0: i128, v1: i128):function %atomic_and(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 and v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 and v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_and(0, 0) == [0, 0]@@ -127,16 +160,21 @@ block0(v0: i128, v1: i128):function %atomic_nand(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 nand v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 nand v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_nand(0, 0) == [-1, 0]@@ -148,16 +186,21 @@ block0(v0: i128, v1: i128):function %atomic_or(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 or v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 or v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_or(0, 0) == [0, 0]@@ -169,16 +212,21 @@ block0(v0: i128, v1: i128):function %atomic_xor(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 xor v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 xor v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_xor(0, 0) == [0, 0]@@ -190,16 +238,21 @@ block0(v0: i128, v1: i128):function %atomic_xchg(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 xchg v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 xchg v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_xchg(0, 0) == [0, 0]@@ -211,16 +264,21 @@ block0(v0: i128, v1: i128):function %atomic_umin(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 umin v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 umin v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_umin(0, 0) == [0, 0]@@ -234,16 +292,21 @@ block0(v0: i128, v1: i128):function %atomic_umax(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 umax v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 umax v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_umax(0, 0) == [0, 0]@@ -257,16 +320,21 @@ block0(v0: i128, v1: i128):function %atomic_smin(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 smin v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 smin v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_smin(0, 0) == [0, 0]@@ -280,16 +348,21 @@ block0(v0: i128, v1: i128):function %atomic_smax(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16, align=16+ ss0 = explicit_slot 32, align=8block0(v0: i128, v1: i128):- v5 = stack_addr.i64 ss0- store.i128 notrap v0, v5+ ; Dynamically align the slot to 16v2 = stack_addr.i64 ss0- v3 = atomic_rmw.i128 smax v2, v1- v6 = stack_addr.i64 ss0- v4 = load.i128 notrap v6- return v4, v3+ v3 = iconst.i64 15+ v4 = band v2, v3+ v5 = iconst.i64 16+ v6 = isub v5, v4+ v7 = iadd v2, v6++ store.i128 notrap v0, v7+ v8 = atomic_rmw.i128 smax v7, v1+ v9 = load.i128 notrap v7+ return v9, v8}; run: %atomic_smax(0, 0) == [0, 0]