Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift: Add 128-bit atomics support for AArch64 (#14475)

Aelin89b18a7b3fab
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index ea1e985d98..2ff7dbbcaf 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -292,6 +292,33 @@
(scratch1 WritableReg)
(scratch2 WritableReg))
+ ;; A synthetic insn, which is a 128-bit load-linked store-conditional loop, that has the
+ ;; overall effect of atomically modifying a memory location in a particular way. Because we
+ ;; have no way to explain to the regalloc about earlyclobber registers, this instruction has
+ ;; completely fixed operand registers, and we rely on the RA's coalescing to remove copies
+ ;; in the surrounding code to the extent it can. Load- and store-exclusive instructions,
+ ;; with acquire-release semantics, are used to access memory. The operand conventions are:
+ ;;
+ ;; x25 (rd) address
+ ;; x26 (rd) low bytes of second operand for `op`
+ ;; x22 (rd) high bytes of second operand for `op`
+ ;; x27 (wr) low bytes of old value
+ ;; x23 (wr) high bytes of old value
+ ;; x24 (wr) scratch reg; value afterwards has no meaning
+ ;; x28 (wr) scratch reg; value afterwards has no meaning
+ ;; x21 (wr) scratch reg; value afterwards has no meaning
+ (AtomicRMW128Loop
+ (op AtomicRMWLoopOp)
+ (flags MemFlagsData)
+ (addr Reg)
+ (operand_lo Reg)
+ (operand_hi Reg)
+ (oldval_lo WritableReg)
+ (oldval_hi WritableReg)
+ (scratch1 WritableReg)
+ (scratch2 WritableReg)
+ (scratch3 WritableReg))
+
;; Similar to AtomicRMWLoop, a compare-and-swap operation implemented using a load-linked
;; store-conditional loop, with acquire-release semantics.
;; Note that the operand conventions, although very similar to AtomicRMWLoop, are different:
@@ -310,6 +337,29 @@
(oldval WritableReg)
(scratch WritableReg))
+ ;; Similar to AtomicRMW128Loop, a 128-bit compare-and-swap operation implemented using a load-linked
+ ;; store-conditional loop, with acquire-release semantics.
+ ;; Note that the operand conventions, although very similar to AtomicRMW128Loop, are different:
+ ;;
+ ;; x25 (rd) address
+ ;; x26 (rd) low bytes of expected value
+ ;; x23 (rd) high bytes of expected value
+ ;; x28 (rd) low bytes of replacement value
+ ;; x22 (rd) high bytes of replacement value
+ ;; x27 (wr) low bytes of old value
+ ;; x21 (wr) high bytes of old value
+ ;; x24 (wr) scratch reg; value afterwards has no meaning
+ (AtomicCAS128Loop
+ (flags MemFlagsData)
+ (addr Reg)
+ (expected_lo Reg)
+ (expected_hi Reg)
+ (replacement_lo Reg)
+ (replacement_hi Reg)
+ (oldval_lo WritableReg)
+ (oldval_hi WritableReg)
+ (scratch WritableReg))
+
;; An atomic read-modify-write operation. These instructions require the
;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have
;; acquire-release semantics.
@@ -347,6 +397,16 @@
(rn Reg)
(flags MemFlagsData))
+ ;; Read 128 bits from address `rn` and put the lower and upper halves in `rt1` and `rt2`
+ ;; respectively.
+ ;; This instruction is sequentially consistent.
+ (LoadAcquire128
+ (rt1 WritableReg)
+ (rt2 WritableReg)
+ (rn Reg)
+ (scratch WritableReg)
+ (flags MemFlagsData))
+
;; Write the lowest `ty` bits of `rt` to address `rn`.
;; This instruction is sequentially consistent.
(StoreRelease
@@ -355,6 +415,15 @@
(rn Reg)
(flags MemFlagsData))
+ ;; Write 128 bits from `rt1` and `rt2` to address `rn`.
+ ;; This instruction is sequentially consistent.
+ (StoreRelease128
+ (rt1 Reg)
+ (rt2 Reg)
+ (rn Reg)
+ (scratch WritableReg)
+ (flags MemFlagsData))
+
;; A memory fence. This must provide ordering to ensure that, at a minimum, neither loads
;; nor stores may move forwards or backwards across the fence. Currently emitted as "dmb
;; ish". This instruction is sequentially consistent.
@@ -3030,11 +3099,30 @@
(_ Unit (emit (MInst.LoadAcquire ty dst addr flags))))
dst))
+;; Helper for emitting `MInst.LoadAcquire128` instructions.
+(decl load_acquire_128 (MemFlagsData Reg) ValueRegs)
+(rule (load_acquire_128 flags addr)
+ (let ((dst_lo WritableReg (temp_writable_reg $I64))
+ (dst_hi WritableReg (temp_writable_reg $I64))
+ (scratch WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.LoadAcquire128 dst_lo dst_hi addr scratch flags))))
+ (value_regs dst_lo dst_hi)))
+
;; Helper for emitting `MInst.StoreRelease` instructions.
(decl store_release (Type MemFlagsData Reg Reg) SideEffectNoResult)
(rule (store_release ty flags src addr)
(SideEffectNoResult.Inst (MInst.StoreRelease ty src addr flags)))
+;; Helper for emitting `MInst.StoreRelease128` instructions.
+(decl store_release_128 (MemFlagsData ValueRegs Reg) SideEffectNoResult)
+(rule (store_release_128 flags src addr)
+ (let ((scratch WritableReg (temp_writable_reg $I64)))
+ (SideEffectNoResult.Inst (MInst.StoreRelease128 (value_regs_get src 0)
+ (value_regs_get src 1)
+ addr
+ scratch
+ flags))))
+
;; Helper for generating a `tst` instruction.
;;
;; Produces a `ProducesFlags` rather than a register or emitted instruction
@@ -4522,6 +4610,31 @@
(_ Unit (emit (MInst.AtomicRMWLoop ty op flags addr operand dst scratch1 scratch2))))
dst))
+;; Helper for emitting `MInst.AtomicRMW128Loop` instructions.
+;; - Make sure that both args are in virtual regs, since in effect
+;; we have to do a parallel copy to get them safely to the AtomicRMW input
+;; regs, and that's not guaranteed safe if either is in a real reg.
+;; - Move the args to the preordained AtomicRMW input regs
+;; - And finally, copy the preordained AtomicRMW output reg to its destination.
+(decl atomic_rmw_128_loop (AtomicRMWLoopOp Reg ValueRegs MemFlagsData) ValueRegs)
+(rule (atomic_rmw_128_loop op addr operand flags)
+ (let ((dst_lo WritableReg (temp_writable_reg $I64))
+ (dst_hi WritableReg (temp_writable_reg $I64))
+ (scratch1 WritableReg (temp_writable_reg $I64))
+ (scratch2 WritableReg (temp_writable_reg $I64))
+ (scratch3 WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.AtomicRMW128Loop op
+ flags
+ addr
+ (value_regs_get operand 0)
+ (value_regs_get operand 1)
+ dst_lo
+ dst_hi
+ scratch1
+ scratch2
+ scratch3))))
+ (value_regs dst_lo dst_hi)))
+
;; Helper for emitting `MInst.AtomicCASLoop` instructions.
;; This is very similar to, but not identical to, the AtomicRmw case. Note
;; that the AtomicCASLoop sequence does its own masking, so we don't need to worry
@@ -4535,6 +4648,26 @@
(_ Unit (emit (MInst.AtomicCASLoop ty flags addr expect replace dst scratch))))
dst))
+;; Helper for emitting `MInst.AtomicCASLoop` instructions.
+;; This is very similar to, but not identical to, the AtomicRmw case.
+;; Make sure that all three args are in virtual regs. See corresponding comment
+;; for `atomic_rmw_128_loop` above.
+(decl atomic_cas_128_loop (Reg ValueRegs ValueRegs MemFlagsData) ValueRegs)
+(rule (atomic_cas_128_loop addr expect replace flags)
+ (let ((dst_lo WritableReg (temp_writable_reg $I64))
+ (dst_hi WritableReg (temp_writable_reg $I64))
+ (scratch WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.AtomicCAS128Loop flags
+ addr
+ (value_regs_get expect 0)
+ (value_regs_get expect 1)
+ (value_regs_get replace 0)
+ (value_regs_get replace 1)
+ dst_lo
+ dst_hi
+ scratch))))
+ (value_regs dst_lo dst_hi)))
+
;; Helper for emitting `MInst.MovPReg` instructions.
(attr mov_from_preg (veri chain))
(decl mov_from_preg (PReg) Reg)
@@ -5835,6 +5968,10 @@
(attr MInst.VecLoadReplicate (tag vector))
(attr MInst.VecCSel (tag vector))
(attr MInst.LoadAcquire (tag atomics))
+(attr MInst.LoadAcquire128 (tag atomics))
+(attr MInst.StoreRelease128 (tag atomics))
+(attr MInst.AtomicRMW128Loop (tag atomics))
+(attr MInst.AtomicCAS128Loop (tag atomics))
(attr extract_vector (tag vector))
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 66d6cd1fe0..66484a3c86 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -630,6 +630,32 @@ fn enc_ldaxr(ty: Type, rt: Writable<Reg>, rn: Reg) -> u32 {
| machreg_to_gpr(rt.to_reg())
}
+fn enc_ldaxp(ty: Type, rt: Writable<Reg>, rt2: Writable<Reg>, rn: Reg) -> u32 {
+ let sz = match ty {
+ I64 => 0b1,
+ I32 => 0b0,
+ _ => unreachable!(),
+ };
+ 0b10_0010000_1_1_11111_1_00000_00000_00000
+ | (sz << 30)
+ | (machreg_to_gpr(rt2.to_reg()) << 10)
+ | (machreg_to_gpr(rn) << 5)
+ | machreg_to_gpr(rt.to_reg())
+}
+
+fn enc_ldxp(ty: Type, rt: Writable<Reg>, rt2: Writable<Reg>, rn: Reg) -> u32 {
+ let sz = match ty {
+ I64 => 0b1,
+ I32 => 0b0,
+ _ => unreachable!(),
+ };
+ 0b10_0010000_1_1_11111_0_00000_00000_00000
+ | (sz << 30)
+ | (machreg_to_gpr(rt2.to_reg()) << 10)
+ | (machreg_to_gpr(rn) << 5)
+ | machreg_to_gpr(rt.to_reg())
+}
+
fn enc_stlxr(ty: Type, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
let sz = match ty {
I64 => 0b11,
@@ -645,6 +671,34 @@ fn enc_stlxr(ty: Type, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
| machreg_to_gpr(rt)
}
+fn enc_stlxp(ty: Type, rs: Writable<Reg>, rt: Reg, rt2: Reg, rn: Reg) -> u32 {
+ let sz = match ty {
+ I64 => 0b1,
+ I32 => 0b0,
+ _ => unreachable!(),
+ };
+ 0b10_0010000_0_1_00000_1_00000_00000_00000
+ | (sz << 30)
+ | (machreg_to_gpr(rs.to_reg()) << 16)
+ | (machreg_to_gpr(rt2) << 10)
+ | (machreg_to_gpr(rn) << 5)
+ | machreg_to_gpr(rt)
+}
+
+fn enc_stxp(ty: Type, rs: Writable<Reg>, rt: Reg, rt2: Reg, rn: Reg) -> u32 {
+ let sz = match ty {
+ I64 => 0b1,
+ I32 => 0b0,
+ _ => unreachable!(),
+ };
+ 0b10_0010000_0_1_00000_0_00000_00000_00000
+ | (sz << 30)
+ | (machreg_to_gpr(rs.to_reg()) << 16)
+ | (machreg_to_gpr(rt2) << 10)
+ | (machreg_to_gpr(rn) << 5)
+ | machreg_to_gpr(rt)
+}
+
fn enc_cas(size: u32, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {
debug_assert_eq!(size & 0b11, size);
@@ -1662,6 +1716,188 @@ impl MachInstEmit for Inst {
));
sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);
}
+ &Inst::AtomicRMW128Loop { op, flags, .. } => {
+ /* Emit this:
+ again:
+ ldaxp x27, x23, [x25]
+ op x28, x21, x27, x23, x26, x22 // op is adds,subc,and,orr,eor
+ stlxp w24, x28, [x25]
+ cbnz x24, again
+
+ Operand conventions:
+ IN: x25 (addr), x26 (low bytes of op), x22 (high bytes of op)
+ OUT: x27 (low bytes of old value), x23 (high bytes of old value), x24 (trashed), x28 (trashed), x21 (trashed)
+
+ It is unfortunate that, per the ARM documentation, x28 cannot be used for
+ both the store-data and success-flag operands of stlxr. This causes the
+ instruction's behaviour to be "CONSTRAINED UNPREDICTABLE", so we use x24
+ instead for the success-flag.
+ */
+ // TODO: We should not hardcode registers here, a better idea would be to
+ // pass some scratch registers in the AtomicRMWLoop pseudo-instruction, and use those
+ let xzr = zero_reg();
+ let x21 = xreg(21);
+ let x22 = xreg(22);
+ let x23 = xreg(23);
+ let x24 = xreg(24);
+ let x25 = xreg(25);
+ let x26 = xreg(26);
+ let x27 = xreg(27);
+ let x28 = xreg(28);
+ let x21wr = writable_xreg(21);
+ let x23wr = writable_xreg(23);
+ let x24wr = writable_xreg(24);
+ let x27wr = writable_xreg(27);
+ let x28wr = writable_xreg(28);
+ let again_label = sink.get_label();
+
+ // again:
+ sink.bind_label(again_label, &mut state.ctrl_plane);
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ sink.put4(enc_ldaxp(I64, x27wr, x23wr, x25)); // ldaxp x27, x23, [x25]
+
+ match op {
+ AtomicRMWLoopOp::Xchg => {} // do nothing
+ AtomicRMWLoopOp::Smax
+ | AtomicRMWLoopOp::Umax
+ | AtomicRMWLoopOp::Smin
+ | AtomicRMWLoopOp::Umin => {
+ // cmp x27, x26
+ // sbcs xzr, x23, x22
+ // csel.op x21, x22, x23
+ // csel.op x28, x26, x27
+
+ let cond = match op {
+ AtomicRMWLoopOp::Smax => Cond::Lt,
+ AtomicRMWLoopOp::Umax => Cond::Lo,
+ AtomicRMWLoopOp::Smin => Cond::Ge,
+ AtomicRMWLoopOp::Umin => Cond::Hs,
+ _ => unreachable!(),
+ };
+
+ Inst::AluRRR {
+ alu_op: ALUOp::SubS,
+ size: OperandSize::Size64,
+ rd: writable_zero_reg(),
+ rn: x27,
+ rm: x26,
+ }
+ .emit(sink, emit_info, state);
+
+ Inst::AluRRR {
+ alu_op: ALUOp::SbcS,
+ size: OperandSize::Size64,
+ rd: writable_zero_reg(),
+ rn: x23,
+ rm: x22,
+ }
+ .emit(sink, emit_info, state);
+
+ Inst::CSel {
+ cond,
+ rd: x21wr,
+ rn: x22,
+ rm: x23,
+ }
+ .emit(sink, emit_info, state);
+
+ Inst::CSel {
+ cond,
+ rd: x28wr,
+ rn: x26,
+ rm: x27,
+ }
+ .emit(sink, emit_info, state);
+ }
+ _ => {
+ let (op_lo, op_hi) = match op {
+ // adds x28, x27, x26
+ // adc x21, x23, x22
+ AtomicRMWLoopOp::Add => (ALUOp::AddS, ALUOp::Adc),
+ // subs x28, x27, x26
+ // sbc x21, x23, x22
+ AtomicRMWLoopOp::Sub => (ALUOp::SubS, ALUOp::Sbc),
+ // and x28, x27, x26
+ // and x21, x23, x22
+ AtomicRMWLoopOp::And | AtomicRMWLoopOp::Nand => {
+ (ALUOp::And, ALUOp::And)
+ }
+ // orr x28, x27, x26
+ // orr x21, x23, x22
+ AtomicRMWLoopOp::Orr => (ALUOp::Orr, ALUOp::Orr),
+ // eor x28, x27, x26
+ // eor x21, x23, x22
+ AtomicRMWLoopOp::Eor => (ALUOp::Eor, ALUOp::Eor),
+ _ => unreachable!(),
+ };
+
+ Inst::AluRRR {
+ alu_op: op_lo,
+ size: OperandSize::Size64,
+ rd: x28wr,
+ rn: x27,
+ rm: x26,
+ }
+ .emit(sink, emit_info, state);
+
+ Inst::AluRRR {
+ alu_op: op_hi,
+ size: OperandSize::Size64,
+ rd: x21wr,
+ rn: x23,
+ rm: x22,
+ }
+ .emit(sink, emit_info, state);
+
+ if op == AtomicRMWLoopOp::Nand {
+ // mvn x28, x28
+ // mvn x21, x21
+
+ Inst::AluRRR {
+ alu_op: ALUOp::OrrNot,
+ size: OperandSize::Size64,
+ rd: x28wr,
+ rn: xzr,
+ rm: x28,
+ }
+ .emit(sink, emit_info, state);
+
+ Inst::AluRRR {
+ alu_op: ALUOp::OrrNot,
+ size: OperandSize::Size64,
+ rd: x21wr,
+ rn: xzr,
+ rm: x21,
+ }
+ .emit(sink, emit_info, state);
+ }
+ }
+ }
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ if op == AtomicRMWLoopOp::Xchg {
+ sink.put4(enc_stlxp(I64, x24wr, x26, x22, x25)); // stlxp w24, x26, x22, [x25]
+ } else {
+ sink.put4(enc_stlxp(I64, x24wr, x28, x21, x25)); // stlxp w24, x28, x21, [x25]
+ }
+
+ // cbnz w24, again
+ // Note, we're actually testing x24, and relying on the default zero-high-half
+ // rule in the assignment that `stlxp` does.
+ let br_offset = sink.cur_offset();
+ sink.put4(enc_conditional_br(
+ BranchTarget::Label(again_label),
+ CondBrKind::NotZero(x24, OperandSize::Size64),
+ ));
+ sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);
+ }
&Inst::AtomicCAS {
rd,
rs,
@@ -1782,6 +2018,132 @@ impl MachInstEmit for Inst {
// out:
sink.bind_label(out_label, &mut state.ctrl_plane);
}
+ &Inst::AtomicCAS128Loop { flags, .. } => {
+ /* Emit this:
+ again:
+ ldaxp x27, x21, [x25]
+ cmp x27, x26
+ b.ne keep
+ cmp x21, x23
+ b.ne keep
+ stlxp w24, x28, x22, [x25]
+ cbnz w24, again
+ b out
+ keep:
+ stlxp w24, x27, x21, [x25]
+ cbnz w24, again
+ out:
+
+ Operand conventions:
+ IN: x25 (addr), x26 (low bytes of expected value), x23 (high bytes of expected value), x28 (low bytes of replacement value), x22 (high bytes of replacement value)
+ OUT: x27 (low bytes of old value), x21 (high bytes of old value), x24 (trashed)
+ */
+ let x21 = xreg(21);
+ let x22 = xreg(22);
+ let x23 = xreg(23);
+ let x24 = xreg(24);
+ let x25 = xreg(25);
+ let x26 = xreg(26);
+ let x27 = xreg(27);
+ let x28 = xreg(28);
+ let xzrwr = writable_zero_reg();
+ let x21wr = writable_xreg(21);
+ let x24wr = writable_xreg(24);
+ let x27wr = writable_xreg(27);
+ let again_label = sink.get_label();
+ let keep_label = sink.get_label();
+ let out_label = sink.get_label();
+
+ // again:
+ sink.bind_label(again_label, &mut state.ctrl_plane);
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ // ldaxp x27, x21, [x25]
+ sink.put4(enc_ldaxp(I64, x27wr, x21wr, x25));
+
+ // cmp x27, x26
+ Inst::AluRRR {
+ alu_op: ALUOp::SubS,
+ size: OperandSize::Size64,
+ rd: xzrwr,
+ rn: x27,
+ rm: x26,
+ }
+ .emit(sink, emit_info, state);
+
+ // b.ne keep
+ let br_keep_offset = sink.cur_offset();
+ sink.put4(enc_conditional_br(
+ BranchTarget::Label(keep_label),
+ CondBrKind::Cond(Cond::Ne),
+ ));
+ sink.use_label_at_offset(br_keep_offset, keep_label, LabelUse::Branch19);
+
+ // cmp x21, x23
+ Inst::AluRRR {
+ alu_op: ALUOp::SubS,
+ size: OperandSize::Size64,
+ rd: xzrwr,
+ rn: x21,
+ rm: x23,
+ }
+ .emit(sink, emit_info, state);
+
+ // b.ne keep
+ let br_keep_offset = sink.cur_offset();
+ sink.put4(enc_conditional_br(
+ BranchTarget::Label(keep_label),
+ CondBrKind::Cond(Cond::Ne),
+ ));
+ sink.use_label_at_offset(br_keep_offset, keep_label, LabelUse::Branch19);
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ // stlxp w24, x28, x22, [x25]
+ sink.put4(enc_stlxp(I64, x24wr, x28, x22, x25));
+
+ // cbnz w24, again
+ let br_again_offset = sink.cur_offset();
+ sink.put4(enc_conditional_br(
+ BranchTarget::Label(again_label),
+ CondBrKind::NotZero(x24, OperandSize::Size64),
+ ));
+ sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
+
+ // b out
+ let b_out_offset = sink.cur_offset();
+ sink.put4(enc_jump26(
+ 0b000101,
+ BranchTarget::Label(out_label).as_offset26_or_zero(),
+ ));
+ sink.use_label_at_offset(b_out_offset, out_label, LabelUse::Branch26);
+
+ // keep:
+ sink.bind_label(keep_label, &mut state.ctrl_plane);
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ // stlxp w24, x27, x21, [x25]
+ sink.put4(enc_stlxp(I64, x24wr, x27, x21, x25));
+
+ // cbnz w24, again
+ let br_again_offset = sink.cur_offset();
+ sink.put4(enc_conditional_br(
+ BranchTarget::Label(again_label),
+ CondBrKind::NotZero(x24, OperandSize::Size64),
+ ));
+ sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
+
+ // out:
+ sink.bind_label(out_label, &mut state.ctrl_plane);
+ }
&Inst::LoadAcquire {
access_ty,
rt,
@@ -1794,6 +2156,40 @@ impl MachInstEmit for Inst {
sink.put4(enc_ldar(access_ty, rt, rn));
}
+ &Inst::LoadAcquire128 {
+ rt1,
+ rt2,
+ rn,
+ scratch,
+ flags,
+ } => {
+ let again_label = sink.get_label();
+
+ // again:
+ sink.bind_label(again_label, &mut state.ctrl_plane);
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ // ldaxp rt1, rt2, [rn]
+ sink.put4(enc_ldaxp(I64, rt1, rt2, rn));
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ // stxp scratch, rt1, rt2, [rn]
+ sink.put4(enc_stxp(I64, scratch, rt1.to_reg(), rt2.to_reg(), rn));
+
+ // cbnz scratch, again.
+ let br_again_offset = sink.cur_offset();
+ sink.put4(enc_conditional_br(
+ BranchTarget::Label(again_label),
+ CondBrKind::NotZero(scratch.to_reg(), OperandSize::Size64),
+ ));
+ sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
+ }
&Inst::StoreRelease {
access_ty,
rt,
@@ -1806,6 +2202,40 @@ impl MachInstEmit for Inst {
sink.put4(enc_stlr(access_ty, rt, rn));
}
+ &Inst::StoreRelease128 {
+ rt1,
+ rt2,
+ rn,
+ scratch,
+ flags,
+ } => {
+ let again_label = sink.get_label();
+
+ // again:
+ sink.bind_label(again_label, &mut state.ctrl_plane);
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ // ldxp xzr, scratch, [rn]
+ sink.put4(enc_ldxp(I64, writable_zero_reg(), scratch, rn));
+
+ if let Some(trap_code) = flags.trap_code() {
+ sink.add_trap(trap_code);
+ }
+
+ // stlxp scratch, rt1, rt2, [rn]
+ sink.put4(enc_stlxp(I64, scratch, rt1, rt2, rn));
+
+ // cbnz scratch, again.
+ let br_again_offset = sink.cur_offset();
+ sink.put4(enc_conditional_br(
+ BranchTarget::Label(again_label),
+ CondBrKind::NotZero(scratch.to_reg(), OperandSize::Size64),
+ ));
+ sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);
+ }
&Inst::Fence {} => {
sink.put4(enc_dmb_ish()); // dmb ish
}
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
index 04d7e77def..a3c07b6e92 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs
@@ -7302,6 +7302,71 @@ fn test_aarch64_binemit() {
"atomic_rmw_loop_umax_16 addr=x25 operand=x26 oldval=x27 scratch1=x24 scratch2=x28",
));
+ insns.push((
+ Inst::AtomicRMW128Loop {
+ op: AtomicRMWLoopOp::Add,
+ flags: MemFlagsData::trusted(),
+ addr: xreg(25),
+ operand_lo: xreg(26),
+ operand_hi: xreg(27),
+ oldval_lo: writable_xreg(13),
+ oldval_hi: writable_xreg(14),
+ scratch1: writable_xreg(15),
+ scratch2: writable_xreg(16),
+ scratch3: writable_xreg(17),
+ },
+ "3BDF7FC87C031AABF502169A3CD738C898FFFFB5",
+ "atomic_rmw_128_loop_add addr=x25 operand_lo=x26 operand_hi=x27 oldval_lo=x13 oldval_hi=x14 scratch1=x15 scratch2=x16 scratch3=x17",
+ ));
+ insns.push((
+ Inst::AtomicRMW128Loop {
+ op: AtomicRMWLoopOp::Xchg,
+ flags: MemFlagsData::trusted(),
+ addr: xreg(25),
+ operand_lo: xreg(26),
+ operand_hi: xreg(27),
+ oldval_lo: writable_xreg(13),
+ oldval_hi: writable_xreg(14),
+ scratch1: writable_xreg(15),
+ scratch2: writable_xreg(16),
+ scratch3: writable_xreg(17),
+ },
+ "3BDF7FC83ADB38C8D8FFFFB5",
+ "atomic_rmw_128_loop_xchg addr=x25 operand_lo=x26 operand_hi=x27 oldval_lo=x13 oldval_hi=x14 scratch1=x15 scratch2=x16 scratch3=x17",
+ ));
+ insns.push((
+ Inst::AtomicRMW128Loop {
+ op: AtomicRMWLoopOp::Umin,
+ flags: MemFlagsData::trusted(),
+ addr: xreg(25),
+ operand_lo: xreg(26),
+ operand_hi: xreg(27),
+ oldval_lo: writable_xreg(13),
+ oldval_hi: writable_xreg(14),
+ scratch1: writable_xreg(15),
+ scratch2: writable_xreg(16),
+ scratch3: writable_xreg(17),
+ },
+ "3BDF7FC87F031AEBFF0216FAD522979A5C239B9A3CD738C858FFFFB5",
+ "atomic_rmw_128_loop_umin addr=x25 operand_lo=x26 operand_hi=x27 oldval_lo=x13 oldval_hi=x14 scratch1=x15 scratch2=x16 scratch3=x17",
+ ));
+ insns.push((
+ Inst::AtomicRMW128Loop {
+ op: AtomicRMWLoopOp::Nand,
+ flags: MemFlagsData::trusted(),
+ addr: xreg(25),
+ operand_lo: xreg(26),
+ operand_hi: xreg(27),
+ oldval_lo: writable_xreg(13),
+ oldval_hi: writable_xreg(14),
+ scratch1: writable_xreg(15),
+ scratch2: writable_xreg(16),
+ scratch3: writable_xreg(17),
+ },
+ "3BDF7FC87C031A8AF502168AFC033CAAF50335AA3CD738C858FFFFB5",
+ "atomic_rmw_128_loop_nand addr=x25 operand_lo=x26 operand_hi=x27 oldval_lo=x13 oldval_hi=x14 scratch1=x15 scratch2=x16 scratch3=x17",
+ ));
+
insns.push((
Inst::AtomicRMW {
ty: I8,
@@ -7839,6 +7904,22 @@ fn test_aarch64_binemit() {
"atomic_cas_loop_64 addr=x25, expect=x26, replacement=x28, oldval=x27, scratch=x24",
));
+ insns.push((
+ Inst::AtomicCAS128Loop {
+ flags: MemFlagsData::trusted(),
+ addr: xreg(25),
+ expected_lo: xreg(26),
+ expected_hi: xreg(24),
+ replacement_lo: xreg(28),
+ replacement_hi: xreg(23),
+ oldval_lo: writable_xreg(27),
+ oldval_hi: writable_xreg(22),
+ scratch: writable_xreg(21),
+ },
+ "3BD77FC87F031AEBC1000054BF0217EB810000543CDB38C858FFFFB5030000143BD738C8F8FEFFB5",
+ "atomic_cas_128_loop addr=x25, expected_lo=x26, expected_hi=x24, replacement_lo=x28, replacement_hi=x23, oldval_lo=x27, oldval_hi=x22, scratch=x21",
+ ));
+
insns.push((
Inst::LoadAcquire {
access_ty: I8,
@@ -7883,6 +7964,18 @@ fn test_aarch64_binemit() {
"ldar x28, [x7]",
));
+ insns.push((
+ Inst::LoadAcquire128 {
+ rt1: writable_xreg(28),
+ rt2: writable_xreg(27),
+ rn: xreg(7),
+ scratch: writable_xreg(26),
+ flags: MemFlagsData::trusted(),
+ },
+ "FCEC7FC8FC6C3AC8DAFFFFB5",
+ "load_acquire_128 x28, x27, [x7], scratch=x26",
+ ));
+
insns.push((
Inst::StoreRelease {
access_ty: I8,
@@ -7927,6 +8020,18 @@ fn test_aarch64_binemit() {
"stlr x28, [x7]",
));
+ insns.push((
+ Inst::StoreRelease128 {
+ rt1: xreg(28),
+ rt2: xreg(27),
+ rn: xreg(7),
+ scratch: writable_xreg(26),
+ flags: MemFlagsData::trusted(),
+ },
+ "FF687FC8FCEC3AC8DAFFFFB5",
+ "store_release_128 x28, x27, [x7], scratch=x26",
+ ));
+
insns.push((Inst::Fence {}, "BF3B03D5", "dmb ish"));
let flags = settings::Flags::new(settings::builder());
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index 230cc2abb4..9fa282db3c 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -487,6 +487,29 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_fixed_def(scratch2, xreg(28));
}
}
+ Inst::AtomicRMW128Loop {
+ op,
+ addr,
+ operand_lo,
+ operand_hi,
+ oldval_lo,
+ oldval_hi,
+ scratch1,
+ scratch2,
+ scratch3,
+ ..
+ } => {
+ collector.reg_fixed_use(addr, xreg(25));
+ collector.reg_fixed_use(operand_lo, xreg(26));
+ collector.reg_fixed_use(operand_hi, xreg(22));
+ collector.reg_fixed_def(oldval_lo, xreg(27));
+ collector.reg_fixed_def(oldval_hi, xreg(23));
+ collector.reg_fixed_def(scratch1, xreg(24));
+ if *op != AtomicRMWLoopOp::Xchg {
+ collector.reg_fixed_def(scratch2, xreg(28));
+ collector.reg_fixed_def(scratch3, xreg(21));
+ }
+ }
Inst::AtomicRMW { rs, rt, rn, .. } => {
collector.reg_use(rs);
collector.reg_def(rt);
@@ -533,14 +556,58 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_fixed_def(oldval, xreg(27));
collector.reg_fixed_def(scratch, xreg(24));
}
+ Inst::AtomicCAS128Loop {
+ addr,
+ expected_lo,
+ expected_hi,
+ replacement_lo,
+ replacement_hi,
+ oldval_lo,
+ oldval_hi,
+ scratch,
+ ..
+ } => {
+ collector.reg_fixed_use(addr, xreg(25));
+ collector.reg_fixed_use(expected_lo, xreg(26));
+ collector.reg_fixed_use(expected_hi, xreg(23));
+ collector.reg_fixed_use(replacement_lo, xreg(28));
+ collector.reg_fixed_use(replacement_hi, xreg(22));
+ collector.reg_fixed_def(oldval_lo, xreg(27));
+ collector.reg_fixed_def(oldval_hi, xreg(21));
+ collector.reg_fixed_def(scratch, xreg(24));
+ }
Inst::LoadAcquire { rt, rn, .. } => {
collector.reg_use(rn);
collector.reg_def(rt);
}
+ Inst::LoadAcquire128 {
+ rt1,
+ rt2,
+ rn,
+ scratch,
+ ..
+ } => {
+ collector.reg_use(rn);
+ collector.reg_def(rt1);
+ collector.reg_def(rt2);
+ collector.reg_def(scratch);
+ }
Inst::StoreRelease { rt, rn, .. } => {
collector.reg_use(rn);
collector.reg_use(rt);
}
+ Inst::StoreRelease128 {
+ rt1,
+ rt2,
+ rn,
+ scratch,
+ ..
+ } => {
+ collector.reg_use(rn);
+ collector.reg_use(rt1);
+ collector.reg_use(rt2);
+ collector.reg_early_def(scratch);
+ }
Inst::Fence {} | Inst::Csdb {} => {}
Inst::FpuMove32 { rd, rn } => {
collector.reg_def(rd);
@@ -1653,6 +1720,43 @@ impl Inst {
scratch2,
)
}
+ &Inst::AtomicRMW128Loop {
+ op,
+ addr,
+ operand_lo,
+ operand_hi,
+ oldval_lo,
+ oldval_hi,
+ scratch1,
+ scratch2,
+ scratch3,
+ ..
+ } => {
+ let op = match op {
+ AtomicRMWLoopOp::Add => "add",
+ AtomicRMWLoopOp::Sub => "sub",
+ AtomicRMWLoopOp::Eor => "eor",
+ AtomicRMWLoopOp::Orr => "orr",
+ AtomicRMWLoopOp::And => "and",
+ AtomicRMWLoopOp::Nand => "nand",
+ AtomicRMWLoopOp::Smin => "smin",
+ AtomicRMWLoopOp::Smax => "smax",
+ AtomicRMWLoopOp::Umin => "umin",
+ AtomicRMWLoopOp::Umax => "umax",
+ AtomicRMWLoopOp::Xchg => "xchg",
+ };
+ let addr = pretty_print_ireg(addr, OperandSize::Size64);
+ let operand_lo = pretty_print_ireg(operand_lo, OperandSize::Size64);
+ let operand_hi = pretty_print_ireg(operand_hi, OperandSize::Size64);
+ let oldval_lo = pretty_print_ireg(oldval_lo.to_reg(), OperandSize::Size64);
+ let oldval_hi = pretty_print_ireg(oldval_hi.to_reg(), OperandSize::Size64);
+ let scratch1 = pretty_print_ireg(scratch1.to_reg(), OperandSize::Size64);
+ let scratch2 = pretty_print_ireg(scratch2.to_reg(), OperandSize::Size64);
+ let scratch3 = pretty_print_ireg(scratch3.to_reg(), OperandSize::Size64);
+ format!(
+ "atomic_rmw_128_loop_{op} addr={addr} operand_lo={operand_lo} operand_hi={operand_hi} oldval_lo={oldval_lo} oldval_hi={oldval_hi} scratch1={scratch1} scratch2={scratch2} scratch3={scratch3}",
+ )
+ }
&Inst::AtomicCAS {
rd, rs, rt, rn, ty, ..
} => {
@@ -1716,6 +1820,29 @@ impl Inst {
scratch,
)
}
+ &Inst::AtomicCAS128Loop {
+ addr,
+ expected_lo,
+ expected_hi,
+ replacement_lo,
+ replacement_hi,
+ oldval_lo,
+ oldval_hi,
+ scratch,
+ ..
+ } => {
+ let addr = pretty_print_ireg(addr, OperandSize::Size64);
+ let expected_lo = pretty_print_ireg(expected_lo, OperandSize::Size64);
+ let expected_hi = pretty_print_ireg(expected_hi, OperandSize::Size64);
+ let replacement_lo = pretty_print_ireg(replacement_lo, OperandSize::Size64);
+ let replacement_hi = pretty_print_ireg(replacement_hi, OperandSize::Size64);
+ let oldval_lo = pretty_print_ireg(oldval_lo.to_reg(), OperandSize::Size64);
+ let oldval_hi = pretty_print_ireg(oldval_hi.to_reg(), OperandSize::Size64);
+ let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
+ format!(
+ "atomic_cas_128_loop addr={addr}, expected_lo={expected_lo}, expected_hi={expected_hi}, replacement_lo={replacement_lo}, replacement_hi={replacement_hi}, oldval_lo={oldval_lo}, oldval_hi={oldval_hi}, scratch={scratch}",
+ )
+ }
&Inst::LoadAcquire {
access_ty, rt, rn, ..
} => {
@@ -1731,6 +1858,19 @@ impl Inst {
let rt = pretty_print_ireg(rt.to_reg(), size);
format!("{op} {rt}, [{rn}]")
}
+ &Inst::LoadAcquire128 {
+ rt1,
+ rt2,
+ rn,
+ scratch,
+ ..
+ } => {
+ let rt1 = pretty_print_ireg(rt1.to_reg(), OperandSize::Size64);
+ let rt2 = pretty_print_ireg(rt2.to_reg(), OperandSize::Size64);
+ let rn = pretty_print_ireg(rn, OperandSize::Size64);
+ let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
+ format!("load_acquire_128 {rt1}, {rt2}, [{rn}], scratch={scratch}")
+ }
&Inst::StoreRelease {
access_ty, rt, rn, ..
} => {
@@ -1746,6 +1886,19 @@ impl Inst {
let rt = pretty_print_ireg(rt, size);
format!("{op} {rt}, [{rn}]")
}
+ &Inst::StoreRelease128 {
+ rt1,
+ rt2,
+ rn,
+ scratch,
+ ..
+ } => {
+ let rt1 = pretty_print_ireg(rt1, OperandSize::Size64);
+ let rt2 = pretty_print_ireg(rt2, OperandSize::Size64);
+ let rn = pretty_print_ireg(rn, OperandSize::Size64);
+ let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);
+ format!("store_release_128 {rt1}, {rt2}, [{rn}], scratch={scratch}")
+ }
&Inst::Fence {} => {
format!("dmb ish")
}
@@ -3203,14 +3356,10 @@ impl MachInstLabelUse for LabelUse {
}
}
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn inst_size_test() {
- // This test will help with unintentionally growing the size
- // of the Inst enum.
- assert_eq!(32, core::mem::size_of::<Inst>());
- }
+#[test]
+#[cfg(target_pointer_width = "64")]
+fn inst_size_test() {
+ // This test will help with unintentionally growing the size
+ // of the Inst enum.
+ assert_eq!(48, core::mem::size_of::<Inst>());
}
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index de5a653354..26c09e8fbf 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -2384,13 +2384,20 @@
;;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (atomic_load (valid_atomic_transaction ty) (little_or_native_endian flags) addr))
(load_acquire ty flags addr))
-
+;; Lower 128-bit `atomic_load` through a special case
+(rule 1 (lower (atomic_load $I128 (little_or_native_endian flags) addr))
+ (load_acquire_128 flags addr))
;;;; Rules for `AtomicStore` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule (lower (atomic_store (little_or_native_endian flags)
src @ (value_type (valid_atomic_transaction ty))
addr))
(side_effect (store_release ty flags src addr)))
+;; Lower 128-bit `atomic_store` through a special case
+(rule 1 (lower (atomic_store (little_or_native_endian flags)
+ src @ (value_type $I128)
+ addr))
+ (side_effect (store_release_128 flags src addr)))
;;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -2446,19 +2453,45 @@
(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Xchg) addr src))
(atomic_rmw_loop (AtomicRMWLoopOp.Xchg) addr src ty flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Add) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Add) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Sub) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.And) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.And) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Nand) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Nand) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Or) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Orr) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Eor) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Smin) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Smax) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Umin) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Umax) addr src flags))
+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Xchg) addr src))
+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Xchg) addr src flags))
+
;;;; Rules for `AtomicCAS` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(rule 1 (lower (and (use_lse)
(atomic_cas (valid_atomic_transaction ty) (little_or_native_endian flags) addr src1 src2)))
(lse_atomic_cas addr src1 src2 ty flags))
(attr rule lse_atomic_cas_i128 (tag atomics))
-(rule lse_atomic_cas_i128 2 (lower (and (use_lse)
+(rule lse_atomic_cas_i128 3 (lower (and (use_lse)
(atomic_cas $I128 (little_or_native_endian flags) addr expect replace)))
(lse_atomic_cas_128 addr expect replace flags))
(rule (lower (and (atomic_cas (valid_atomic_transaction ty) (little_or_native_endian flags) addr src1 src2)))
(atomic_cas_loop addr src1 src2 ty flags))
+(rule 2 (lower (and (atomic_cas $I128 (little_or_native_endian flags) addr expect replace)))
+ (atomic_cas_128_loop addr expect replace flags))
+
;;;; Rules for 'fvdemote' ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(attr fvdemote (tag vector))
(rule (lower (fvdemote _ x))
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
index a2abe420a3..bed9449208 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
@@ -342,7 +342,7 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
/// This is target-word-size dependent. And it excludes booleans and reftypes.
fn valid_atomic_transaction(&mut self, ty: Type) -> Option<Type> {
match ty {
- I8 | I16 | I32 | I64 => Some(ty),
+ I8 | I16 | I32 | I64 | I128 => Some(ty),
_ => None,
}
}
diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic-cas-128.clif b/cranelift/filetests/filetests/isa/aarch64/atomic-cas-128.clif
new file mode 100644
index 0000000000..bcbb77fe8b
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/atomic-cas-128.clif
@@ -0,0 +1,130 @@
+test compile precise-output
+target aarch64
+
+function %atomic_cas_i128(i64, i128, i128) -> i128 {
+block0(v0: i64, v1: i128, v2: i128):
+ v3 = atomic_cas.i128 v0, v1, v2
+ return v3
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x5
+; mov x23, x3
+; mov x25, x0
+; mov x26, x2
+; mov x28, x4
+; atomic_cas_128_loop addr=x25, expected_lo=x26, expected_hi=x23, replacement_lo=x28, replacement_hi=x22, oldval_lo=x27, oldval_hi=x21, scratch=x24
+; mov x0, x27
+; mov x1, x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x5
+; mov x23, x3
+; mov x25, x0
+; mov x26, x2
+; mov x28, x4
+; ldaxp x27, x21, [x25] ; trap: heap_oob
+; cmp x27, x26
+; b.ne #0x4c
+; cmp x21, x23
+; b.ne #0x4c
+; stlxp w24, x28, x22, [x25] ; trap: heap_oob
+; cbnz x24, #0x2c
+; b #0x54
+; stlxp w24, x27, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x2c
+; mov x0, x27
+; mov x1, x21
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
+function %atomic_cas_i128_success(i64, i128, i128) -> i8 {
+block0(v0: i64, v1: i128, v2: i128):
+ v3 = atomic_cas.i128 v0, v1, v2
+ v4 = icmp eq v3, v1
+ return v4
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x5
+; mov x23, x3
+; mov x25, x0
+; mov x26, x2
+; mov x28, x4
+; atomic_cas_128_loop addr=x25, expected_lo=x26, expected_hi=x23, replacement_lo=x28, replacement_hi=x22, oldval_lo=x27, oldval_hi=x21, scratch=x24
+; subs xzr, x27, x26
+; ccmp x21, x23, #nzcv, eq
+; cset x0, eq
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x5
+; mov x23, x3
+; mov x25, x0
+; mov x26, x2
+; mov x28, x4
+; ldaxp x27, x21, [x25] ; trap: heap_oob
+; cmp x27, x26
+; b.ne #0x4c
+; cmp x21, x23
+; b.ne #0x4c
+; stlxp w24, x28, x22, [x25] ; trap: heap_oob
+; cbnz x24, #0x2c
+; b #0x54
+; stlxp w24, x27, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x2c
+; cmp x27, x26
+; ccmp x21, x23, #0, eq
+; cset x0, eq
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif b/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif
index 29cd380c0f..926ab9a1ea 100644
--- a/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif
@@ -1,6 +1,55 @@
test compile precise-output
target aarch64
+function %atomic_rmw_add_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 add v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_add addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; adds x28, x27, x26
+; adc x21, x23, x22
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_add_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 add v0, v1
@@ -169,6 +218,55 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_sub_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 sub v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_sub addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; subs x28, x27, x26
+; sbc x21, x23, x22
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_sub_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 sub v0, v1
@@ -337,6 +435,55 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_and_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 and v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_and addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; and x28, x27, x26
+; and x21, x23, x22
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_and_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 and v0, v1
@@ -505,6 +652,57 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_nand_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 nand v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_nand addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; and x28, x27, x26
+; and x21, x23, x22
+; mvn x28, x28
+; mvn x21, x21
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_nand_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 nand v0, v1
@@ -677,6 +875,55 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_or_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 or v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_orr addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; orr x28, x27, x26
+; orr x21, x23, x22
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_or_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 or v0, v1
@@ -845,6 +1092,55 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_xor_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 xor v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_eor addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; eor x28, x27, x26
+; eor x21, x23, x22
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_xor_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 xor v0, v1
@@ -1013,6 +1309,57 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_smax_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 smax v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_smax addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; cmp x27, x26
+; sbcs xzr, x23, x22
+; csel x21, x22, x23, lt
+; csel x28, x26, x27, lt
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_smax_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 smax v0, v1
@@ -1187,6 +1534,57 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_umax_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 umax v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_umax addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; cmp x27, x26
+; sbcs xzr, x23, x22
+; csel x21, x22, x23, lo
+; csel x28, x26, x27, lo
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_umax_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 umax v0, v1
@@ -1359,6 +1757,57 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_smin_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 smin v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_smin addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; cmp x27, x26
+; sbcs xzr, x23, x22
+; csel x21, x22, x23, ge
+; csel x28, x26, x27, ge
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_smin_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 smin v0, v1
@@ -1533,6 +1982,57 @@ block0(v0: i64, v1: i8):
; ldp x29, x30, [sp], #0x10
; ret
+function %atomic_rmw_umin_i128(i64, i128) {
+block0(v0: i64, v1: i128):
+ v2 = atomic_rmw.i128 umin v0, v1
+ return
+}
+
+; VCode:
+; stp fp, lr, [sp, #-16]!
+; mov fp, sp
+; stp x27, x28, [sp, #-16]!
+; stp x25, x26, [sp, #-16]!
+; stp x23, x24, [sp, #-16]!
+; stp x21, x22, [sp, #-16]!
+; block0:
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; atomic_rmw_128_loop_umin addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21
+; ldp x21, x22, [sp], #16
+; ldp x23, x24, [sp], #16
+; ldp x25, x26, [sp], #16
+; ldp x27, x28, [sp], #16
+; ldp fp, lr, [sp], #16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stp x29, x30, [sp, #-0x10]!
+; mov x29, sp
+; stp x27, x28, [sp, #-0x10]!
+; stp x25, x26, [sp, #-0x10]!
+; stp x23, x24, [sp, #-0x10]!
+; stp x21, x22, [sp, #-0x10]!
+; block1: ; offset 0x18
+; mov x22, x3
+; mov x25, x0
+; mov x26, x2
+; ldaxp x27, x23, [x25] ; trap: heap_oob
+; cmp x27, x26
+; sbcs xzr, x23, x22
+; csel x21, x22, x23, hs
+; csel x28, x26, x27, hs
+; stlxp w24, x28, x21, [x25] ; trap: heap_oob
+; cbnz x24, #0x24
+; ldp x21, x22, [sp], #0x10
+; ldp x23, x24, [sp], #0x10
+; ldp x25, x26, [sp], #0x10
+; ldp x27, x28, [sp], #0x10
+; ldp x29, x30, [sp], #0x10
+; ret
+
function %atomic_rmw_umin_i64(i64, i64) {
block0(v0: i64, v1: i64):
v2 = atomic_rmw.i64 umin v0, v1
diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic_load.clif b/cranelift/filetests/filetests/isa/aarch64/atomic_load.clif
index 044cf0f141..ef894f8add 100644
--- a/cranelift/filetests/filetests/isa/aarch64/atomic_load.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/atomic_load.clif
@@ -1,6 +1,24 @@
test compile precise-output
target aarch64
+function %atomic_load_i128(i64) -> i128 {
+block0(v0: i64):
+ v1 = atomic_load.i128 v0
+ return v1
+}
+
+; VCode:
+; block0:
+; load_acquire_128 x0, x1, [x0], scratch=x4
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; ldaxp x0, x1, [x0] ; trap: heap_oob
+; stxp w4, x0, x1, [x0] ; trap: heap_oob
+; cbnz x4, #0
+; ret
+
function %atomic_load_i64(i64) -> i64 {
block0(v0: i64):
v1 = atomic_load.i64 v0
diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic_store.clif b/cranelift/filetests/filetests/isa/aarch64/atomic_store.clif
index 97927059cf..282f002066 100644
--- a/cranelift/filetests/filetests/isa/aarch64/atomic_store.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/atomic_store.clif
@@ -1,6 +1,24 @@
test compile precise-output
target aarch64
+function %atomic_store_i128(i128, i64) {
+block0(v0: i128, v1: i64):
+ atomic_store.i128 v0, v1
+ return
+}
+
+; VCode:
+; block0:
+; store_release_128 x0, x1, [x2], scratch=x4
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; ldxp xzr, x4, [x2] ; trap: heap_oob
+; stlxp w4, x0, x1, [x2] ; trap: heap_oob
+; cbnz x4, #0
+; ret
+
function %atomic_store_i64(i64, i64) {
block0(v0: i64, v1: i64):
atomic_store.i64 v0, v1
diff --git a/cranelift/filetests/filetests/runtests/atomic-128.clif b/cranelift/filetests/filetests/runtests/atomic-128.clif
index cdf018dbf9..0019952714 100644
--- a/cranelift/filetests/filetests/runtests/atomic-128.clif
+++ b/cranelift/filetests/filetests/runtests/atomic-128.clif
@@ -3,9 +3,10 @@ test run
set enable_llvm_abi_extensions
set enable_multi_ret_implicit_sret
target x86_64 has_cmpxchg16b
+target aarch64
function %atomic_load(i128) -> i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128):
v3 = stack_addr.i64 ss0
@@ -24,7 +25,7 @@ block0(v0: i128):
function %atomic_store(i128) -> i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128):
v1 = stack_addr.i64 ss0
@@ -43,7 +44,7 @@ block0(v0: i128):
function %atomic_cas(i128, i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128, v2: i128):
v6 = stack_addr.i64 ss0
@@ -63,7 +64,7 @@ block0(v0: i128, v1: i128, v2: i128):
function %atomic_add(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -84,7 +85,7 @@ block0(v0: i128, v1: i128):
function %atomic_sub(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -105,7 +106,7 @@ block0(v0: i128, v1: i128):
function %atomic_and(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -126,7 +127,7 @@ block0(v0: i128, v1: i128):
function %atomic_nand(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -147,7 +148,7 @@ block0(v0: i128, v1: i128):
function %atomic_or(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -168,7 +169,7 @@ block0(v0: i128, v1: i128):
function %atomic_xor(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -189,7 +190,7 @@ block0(v0: i128, v1: i128):
function %atomic_xchg(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -210,7 +211,7 @@ block0(v0: i128, v1: i128):
function %atomic_umin(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -233,7 +234,7 @@ block0(v0: i128, v1: i128):
function %atomic_umax(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -256,7 +257,7 @@ block0(v0: i128, v1: i128):
function %atomic_smin(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0
@@ -279,7 +280,7 @@ block0(v0: i128, v1: i128):
function %atomic_smax(i128, i128) -> i128, i128 {
- ss0 = explicit_slot 16
+ ss0 = explicit_slot 16, align=16
block0(v0: i128, v1: i128):
v5 = stack_addr.i64 ss0