A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift: Add 128-bit atomics support for AArch64 (#14475)
Aelin89b18a7b3fab
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex ea1e985d98..2ff7dbbcaf 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -292,6 +292,33 @@(scratch1 WritableReg)(scratch2 WritableReg))+ ;; A synthetic insn, which is a 128-bit load-linked store-conditional loop, that has the+ ;; overall effect of atomically modifying a memory location in a particular way. Because we+ ;; have no way to explain to the regalloc about earlyclobber registers, this instruction has+ ;; completely fixed operand registers, and we rely on the RA's coalescing to remove copies+ ;; in the surrounding code to the extent it can. Load- and store-exclusive instructions,+ ;; with acquire-release semantics, are used to access memory. The operand conventions are:+ ;;+ ;; x25 (rd) address+ ;; x26 (rd) low bytes of second operand for `op`+ ;; x22 (rd) high bytes of second operand for `op`+ ;; x27 (wr) low bytes of old value+ ;; x23 (wr) high bytes of old value+ ;; x24 (wr) scratch reg; value afterwards has no meaning+ ;; x28 (wr) scratch reg; value afterwards has no meaning+ ;; x21 (wr) scratch reg; value afterwards has no meaning+ (AtomicRMW128Loop+ (op AtomicRMWLoopOp)+ (flags MemFlagsData)+ (addr Reg)+ (operand_lo Reg)+ (operand_hi Reg)+ (oldval_lo WritableReg)+ (oldval_hi WritableReg)+ (scratch1 WritableReg)+ (scratch2 WritableReg)+ (scratch3 WritableReg))+;; Similar to AtomicRMWLoop, a compare-and-swap operation implemented using a load-linked;; store-conditional loop, with acquire-release semantics.;; Note that the operand conventions, although very similar to AtomicRMWLoop, are different:@@ -310,6 +337,29 @@(oldval WritableReg)(scratch WritableReg))+ ;; Similar to AtomicRMW128Loop, a 128-bit compare-and-swap operation implemented using a load-linked+ ;; store-conditional loop, with acquire-release semantics.+ ;; Note that the operand conventions, although very similar to AtomicRMW128Loop, are different:+ ;;+ ;; x25 (rd) address+ ;; x26 (rd) low bytes of expected value+ ;; x23 (rd) high bytes of expected value+ ;; x28 (rd) low bytes of replacement value+ ;; x22 (rd) high bytes of replacement value+ ;; x27 (wr) low bytes of old value+ ;; x21 (wr) high bytes of old value+ ;; x24 (wr) scratch reg; value afterwards has no meaning+ (AtomicCAS128Loop+ (flags MemFlagsData)+ (addr Reg)+ (expected_lo Reg)+ (expected_hi Reg)+ (replacement_lo Reg)+ (replacement_hi Reg)+ (oldval_lo WritableReg)+ (oldval_hi WritableReg)+ (scratch WritableReg))+;; An atomic read-modify-write operation. These instructions require the;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have;; acquire-release semantics.@@ -347,6 +397,16 @@(rn Reg)(flags MemFlagsData))+ ;; Read 128 bits from address `rn` and put the lower and upper halves in `rt1` and `rt2`+ ;; respectively.+ ;; This instruction is sequentially consistent.+ (LoadAcquire128+ (rt1 WritableReg)+ (rt2 WritableReg)+ (rn Reg)+ (scratch WritableReg)+ (flags MemFlagsData))+;; Write the lowest `ty` bits of `rt` to address `rn`.;; This instruction is sequentially consistent.(StoreRelease@@ -355,6 +415,15 @@(rn Reg)(flags MemFlagsData))+ ;; Write 128 bits from `rt1` and `rt2` to address `rn`.+ ;; This instruction is sequentially consistent.+ (StoreRelease128+ (rt1 Reg)+ (rt2 Reg)+ (rn Reg)+ (scratch WritableReg)+ (flags MemFlagsData))+;; A memory fence. This must provide ordering to ensure that, at a minimum, neither loads;; nor stores may move forwards or backwards across the fence. Currently emitted as "dmb;; ish". This instruction is sequentially consistent.@@ -3030,11 +3099,30 @@(_ Unit (emit (MInst.LoadAcquire ty dst addr flags))))dst))+;; Helper for emitting `MInst.LoadAcquire128` instructions.+(decl load_acquire_128 (MemFlagsData Reg) ValueRegs)+(rule (load_acquire_128 flags addr)+ (let ((dst_lo WritableReg (temp_writable_reg $I64))+ (dst_hi WritableReg (temp_writable_reg $I64))+ (scratch WritableReg (temp_writable_reg $I64))+ (_ Unit (emit (MInst.LoadAcquire128 dst_lo dst_hi addr scratch flags))))+ (value_regs dst_lo dst_hi)))+;; Helper for emitting `MInst.StoreRelease` instructions.(decl store_release (Type MemFlagsData Reg Reg) SideEffectNoResult)(rule (store_release ty flags src addr)(SideEffectNoResult.Inst (MInst.StoreRelease ty src addr flags)))+;; Helper for emitting `MInst.StoreRelease128` instructions.+(decl store_release_128 (MemFlagsData ValueRegs Reg) SideEffectNoResult)+(rule (store_release_128 flags src addr)+ (let ((scratch WritableReg (temp_writable_reg $I64)))+ (SideEffectNoResult.Inst (MInst.StoreRelease128 (value_regs_get src 0)+ (value_regs_get src 1)+ addr+ scratch+ flags))))+;; Helper for generating a `tst` instruction.;;;; Produces a `ProducesFlags` rather than a register or emitted instruction@@ -4522,6 +4610,31 @@(_ Unit (emit (MInst.AtomicRMWLoop ty op flags addr operand dst scratch1 scratch2))))dst))+;; Helper for emitting `MInst.AtomicRMW128Loop` instructions.+;; - Make sure that both args are in virtual regs, since in effect+;; we have to do a parallel copy to get them safely to the AtomicRMW input+;; regs, and that's not guaranteed safe if either is in a real reg.+;; - Move the args to the preordained AtomicRMW input regs+;; - And finally, copy the preordained AtomicRMW output reg to its destination.+(decl atomic_rmw_128_loop (AtomicRMWLoopOp Reg ValueRegs MemFlagsData) ValueRegs)+(rule (atomic_rmw_128_loop op addr operand flags)+ (let ((dst_lo WritableReg (temp_writable_reg $I64))+ (dst_hi WritableReg (temp_writable_reg $I64))+ (scratch1 WritableReg (temp_writable_reg $I64))+ (scratch2 WritableReg (temp_writable_reg $I64))+ (scratch3 WritableReg (temp_writable_reg $I64))+ (_ Unit (emit (MInst.AtomicRMW128Loop op+ flags+ addr+ (value_regs_get operand 0)+ (value_regs_get operand 1)+ dst_lo+ dst_hi+ scratch1+ scratch2+ scratch3))))+ (value_regs dst_lo dst_hi)))+;; Helper for emitting `MInst.AtomicCASLoop` instructions.;; This is very similar to, but not identical to, the AtomicRmw case. Note;; that the AtomicCASLoop sequence does its own masking, so we don't need to worry@@ -4535,6 +4648,26 @@(_ Unit (emit (MInst.AtomicCASLoop ty flags addr expect replace dst scratch))))dst))+;; Helper for emitting `MInst.AtomicCASLoop` instructions.+;; This is very similar to, but not identical to, the AtomicRmw case.+;; Make sure that all three args are in virtual regs. See corresponding comment+;; for `atomic_rmw_128_loop` above.+(decl atomic_cas_128_loop (Reg ValueRegs ValueRegs MemFlagsData) ValueRegs)+(rule (atomic_cas_128_loop addr expect replace flags)+ (let ((dst_lo WritableReg (temp_writable_reg $I64))+ (dst_hi WritableReg (temp_writable_reg $I64))+ (scratch WritableReg (temp_writable_reg $I64))+ (_ Unit (emit (MInst.AtomicCAS128Loop flags+ addr+ (value_regs_get expect 0)+ (value_regs_get expect 1)+ (value_regs_get replace 0)+ (value_regs_get replace 1)+ dst_lo+ dst_hi+ scratch))))+ (value_regs dst_lo dst_hi)))+;; Helper for emitting `MInst.MovPReg` instructions.(attr mov_from_preg (veri chain))(decl mov_from_preg (PReg) Reg)@@ -5835,6 +5968,10 @@(attr MInst.VecLoadReplicate (tag vector))(attr MInst.VecCSel (tag vector))(attr MInst.LoadAcquire (tag atomics))+(attr MInst.LoadAcquire128 (tag atomics))+(attr MInst.StoreRelease128 (tag atomics))+(attr MInst.AtomicRMW128Loop (tag atomics))+(attr MInst.AtomicCAS128Loop (tag atomics))(attr extract_vector (tag vector))diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rsindex 66d6cd1fe0..66484a3c86 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs@@ -630,6 +630,32 @@ fn enc_ldaxr(ty: Type, rt: Writable<Reg>, rn: Reg) -> u32 {| machreg_to_gpr(rt.to_reg())}+fn enc_ldaxp(ty: Type, rt: Writable<Reg>, rt2: Writable<Reg>, rn: Reg) -> u32 {+ let sz = match ty {+ I64 => 0b1,+ I32 => 0b0,+ _ => unreachable!(),+ };+ 0b10_0010000_1_1_11111_1_00000_00000_00000+ | (sz << 30)+ | (machreg_to_gpr(rt2.to_reg()) << 10)+ | (machreg_to_gpr(rn) << 5)+ | machreg_to_gpr(rt.to_reg())+}++fn enc_ldxp(ty: Type, rt: Writable<Reg>, rt2: Writable<Reg>, rn: Reg) -> u32 {+ let sz = match ty {+ I64 => 0b1,+ I32 => 0b0,+ _ => unreachable!(),+ };+ 0b10_0010000_1_1_11111_0_00000_00000_00000+ | (sz << 30)+ | (machreg_to_gpr(rt2.to_reg()) << 10)+ | (machreg_to_gpr(rn) << 5)+ | machreg_to_gpr(rt.to_reg())+}+fn enc_stlxr(ty: Type, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {let sz = match ty {I64 => 0b11,@@ -645,6 +671,34 @@ fn enc_stlxr(ty: Type, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {| machreg_to_gpr(rt)}+fn enc_stlxp(ty: Type, rs: Writable<Reg>, rt: Reg, rt2: Reg, rn: Reg) -> u32 {+ let sz = match ty {+ I64 => 0b1,+ I32 => 0b0,+ _ => unreachable!(),+ };+ 0b10_0010000_0_1_00000_1_00000_00000_00000+ | (sz << 30)+ | (machreg_to_gpr(rs.to_reg()) << 16)+ | (machreg_to_gpr(rt2) << 10)+ | (machreg_to_gpr(rn) << 5)+ | machreg_to_gpr(rt)+}++fn enc_stxp(ty: Type, rs: Writable<Reg>, rt: Reg, rt2: Reg, rn: Reg) -> u32 {+ let sz = match ty {+ I64 => 0b1,+ I32 => 0b0,+ _ => unreachable!(),+ };+ 0b10_0010000_0_1_00000_0_00000_00000_00000+ | (sz << 30)+ | (machreg_to_gpr(rs.to_reg()) << 16)+ | (machreg_to_gpr(rt2) << 10)+ | (machreg_to_gpr(rn) << 5)+ | machreg_to_gpr(rt)+}+fn enc_cas(size: u32, rs: Writable<Reg>, rt: Reg, rn: Reg) -> u32 {debug_assert_eq!(size & 0b11, size);@@ -1662,6 +1716,188 @@ impl MachInstEmit for Inst {));sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);}+ &Inst::AtomicRMW128Loop { op, flags, .. } => {+ /* Emit this:+ again:+ ldaxp x27, x23, [x25]+ op x28, x21, x27, x23, x26, x22 // op is adds,subc,and,orr,eor+ stlxp w24, x28, [x25]+ cbnz x24, again++ Operand conventions:+ IN: x25 (addr), x26 (low bytes of op), x22 (high bytes of op)+ OUT: x27 (low bytes of old value), x23 (high bytes of old value), x24 (trashed), x28 (trashed), x21 (trashed)++ It is unfortunate that, per the ARM documentation, x28 cannot be used for+ both the store-data and success-flag operands of stlxr. This causes the+ instruction's behaviour to be "CONSTRAINED UNPREDICTABLE", so we use x24+ instead for the success-flag.+ */+ // TODO: We should not hardcode registers here, a better idea would be to+ // pass some scratch registers in the AtomicRMWLoop pseudo-instruction, and use those+ let xzr = zero_reg();+ let x21 = xreg(21);+ let x22 = xreg(22);+ let x23 = xreg(23);+ let x24 = xreg(24);+ let x25 = xreg(25);+ let x26 = xreg(26);+ let x27 = xreg(27);+ let x28 = xreg(28);+ let x21wr = writable_xreg(21);+ let x23wr = writable_xreg(23);+ let x24wr = writable_xreg(24);+ let x27wr = writable_xreg(27);+ let x28wr = writable_xreg(28);+ let again_label = sink.get_label();++ // again:+ sink.bind_label(again_label, &mut state.ctrl_plane);++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ sink.put4(enc_ldaxp(I64, x27wr, x23wr, x25)); // ldaxp x27, x23, [x25]++ match op {+ AtomicRMWLoopOp::Xchg => {} // do nothing+ AtomicRMWLoopOp::Smax+ | AtomicRMWLoopOp::Umax+ | AtomicRMWLoopOp::Smin+ | AtomicRMWLoopOp::Umin => {+ // cmp x27, x26+ // sbcs xzr, x23, x22+ // csel.op x21, x22, x23+ // csel.op x28, x26, x27++ let cond = match op {+ AtomicRMWLoopOp::Smax => Cond::Lt,+ AtomicRMWLoopOp::Umax => Cond::Lo,+ AtomicRMWLoopOp::Smin => Cond::Ge,+ AtomicRMWLoopOp::Umin => Cond::Hs,+ _ => unreachable!(),+ };++ Inst::AluRRR {+ alu_op: ALUOp::SubS,+ size: OperandSize::Size64,+ rd: writable_zero_reg(),+ rn: x27,+ rm: x26,+ }+ .emit(sink, emit_info, state);++ Inst::AluRRR {+ alu_op: ALUOp::SbcS,+ size: OperandSize::Size64,+ rd: writable_zero_reg(),+ rn: x23,+ rm: x22,+ }+ .emit(sink, emit_info, state);++ Inst::CSel {+ cond,+ rd: x21wr,+ rn: x22,+ rm: x23,+ }+ .emit(sink, emit_info, state);++ Inst::CSel {+ cond,+ rd: x28wr,+ rn: x26,+ rm: x27,+ }+ .emit(sink, emit_info, state);+ }+ _ => {+ let (op_lo, op_hi) = match op {+ // adds x28, x27, x26+ // adc x21, x23, x22+ AtomicRMWLoopOp::Add => (ALUOp::AddS, ALUOp::Adc),+ // subs x28, x27, x26+ // sbc x21, x23, x22+ AtomicRMWLoopOp::Sub => (ALUOp::SubS, ALUOp::Sbc),+ // and x28, x27, x26+ // and x21, x23, x22+ AtomicRMWLoopOp::And | AtomicRMWLoopOp::Nand => {+ (ALUOp::And, ALUOp::And)+ }+ // orr x28, x27, x26+ // orr x21, x23, x22+ AtomicRMWLoopOp::Orr => (ALUOp::Orr, ALUOp::Orr),+ // eor x28, x27, x26+ // eor x21, x23, x22+ AtomicRMWLoopOp::Eor => (ALUOp::Eor, ALUOp::Eor),+ _ => unreachable!(),+ };++ Inst::AluRRR {+ alu_op: op_lo,+ size: OperandSize::Size64,+ rd: x28wr,+ rn: x27,+ rm: x26,+ }+ .emit(sink, emit_info, state);++ Inst::AluRRR {+ alu_op: op_hi,+ size: OperandSize::Size64,+ rd: x21wr,+ rn: x23,+ rm: x22,+ }+ .emit(sink, emit_info, state);++ if op == AtomicRMWLoopOp::Nand {+ // mvn x28, x28+ // mvn x21, x21++ Inst::AluRRR {+ alu_op: ALUOp::OrrNot,+ size: OperandSize::Size64,+ rd: x28wr,+ rn: xzr,+ rm: x28,+ }+ .emit(sink, emit_info, state);++ Inst::AluRRR {+ alu_op: ALUOp::OrrNot,+ size: OperandSize::Size64,+ rd: x21wr,+ rn: xzr,+ rm: x21,+ }+ .emit(sink, emit_info, state);+ }+ }+ }++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ if op == AtomicRMWLoopOp::Xchg {+ sink.put4(enc_stlxp(I64, x24wr, x26, x22, x25)); // stlxp w24, x26, x22, [x25]+ } else {+ sink.put4(enc_stlxp(I64, x24wr, x28, x21, x25)); // stlxp w24, x28, x21, [x25]+ }++ // cbnz w24, again+ // Note, we're actually testing x24, and relying on the default zero-high-half+ // rule in the assignment that `stlxp` does.+ let br_offset = sink.cur_offset();+ sink.put4(enc_conditional_br(+ BranchTarget::Label(again_label),+ CondBrKind::NotZero(x24, OperandSize::Size64),+ ));+ sink.use_label_at_offset(br_offset, again_label, LabelUse::Branch19);+ }&Inst::AtomicCAS {rd,rs,@@ -1782,6 +2018,132 @@ impl MachInstEmit for Inst {// out:sink.bind_label(out_label, &mut state.ctrl_plane);}+ &Inst::AtomicCAS128Loop { flags, .. } => {+ /* Emit this:+ again:+ ldaxp x27, x21, [x25]+ cmp x27, x26+ b.ne keep+ cmp x21, x23+ b.ne keep+ stlxp w24, x28, x22, [x25]+ cbnz w24, again+ b out+ keep:+ stlxp w24, x27, x21, [x25]+ cbnz w24, again+ out:++ Operand conventions:+ IN: x25 (addr), x26 (low bytes of expected value), x23 (high bytes of expected value), x28 (low bytes of replacement value), x22 (high bytes of replacement value)+ OUT: x27 (low bytes of old value), x21 (high bytes of old value), x24 (trashed)+ */+ let x21 = xreg(21);+ let x22 = xreg(22);+ let x23 = xreg(23);+ let x24 = xreg(24);+ let x25 = xreg(25);+ let x26 = xreg(26);+ let x27 = xreg(27);+ let x28 = xreg(28);+ let xzrwr = writable_zero_reg();+ let x21wr = writable_xreg(21);+ let x24wr = writable_xreg(24);+ let x27wr = writable_xreg(27);+ let again_label = sink.get_label();+ let keep_label = sink.get_label();+ let out_label = sink.get_label();++ // again:+ sink.bind_label(again_label, &mut state.ctrl_plane);++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ // ldaxp x27, x21, [x25]+ sink.put4(enc_ldaxp(I64, x27wr, x21wr, x25));++ // cmp x27, x26+ Inst::AluRRR {+ alu_op: ALUOp::SubS,+ size: OperandSize::Size64,+ rd: xzrwr,+ rn: x27,+ rm: x26,+ }+ .emit(sink, emit_info, state);++ // b.ne keep+ let br_keep_offset = sink.cur_offset();+ sink.put4(enc_conditional_br(+ BranchTarget::Label(keep_label),+ CondBrKind::Cond(Cond::Ne),+ ));+ sink.use_label_at_offset(br_keep_offset, keep_label, LabelUse::Branch19);++ // cmp x21, x23+ Inst::AluRRR {+ alu_op: ALUOp::SubS,+ size: OperandSize::Size64,+ rd: xzrwr,+ rn: x21,+ rm: x23,+ }+ .emit(sink, emit_info, state);++ // b.ne keep+ let br_keep_offset = sink.cur_offset();+ sink.put4(enc_conditional_br(+ BranchTarget::Label(keep_label),+ CondBrKind::Cond(Cond::Ne),+ ));+ sink.use_label_at_offset(br_keep_offset, keep_label, LabelUse::Branch19);++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ // stlxp w24, x28, x22, [x25]+ sink.put4(enc_stlxp(I64, x24wr, x28, x22, x25));++ // cbnz w24, again+ let br_again_offset = sink.cur_offset();+ sink.put4(enc_conditional_br(+ BranchTarget::Label(again_label),+ CondBrKind::NotZero(x24, OperandSize::Size64),+ ));+ sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);++ // b out+ let b_out_offset = sink.cur_offset();+ sink.put4(enc_jump26(+ 0b000101,+ BranchTarget::Label(out_label).as_offset26_or_zero(),+ ));+ sink.use_label_at_offset(b_out_offset, out_label, LabelUse::Branch26);++ // keep:+ sink.bind_label(keep_label, &mut state.ctrl_plane);++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ // stlxp w24, x27, x21, [x25]+ sink.put4(enc_stlxp(I64, x24wr, x27, x21, x25));++ // cbnz w24, again+ let br_again_offset = sink.cur_offset();+ sink.put4(enc_conditional_br(+ BranchTarget::Label(again_label),+ CondBrKind::NotZero(x24, OperandSize::Size64),+ ));+ sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);++ // out:+ sink.bind_label(out_label, &mut state.ctrl_plane);+ }&Inst::LoadAcquire {access_ty,rt,@@ -1794,6 +2156,40 @@ impl MachInstEmit for Inst {sink.put4(enc_ldar(access_ty, rt, rn));}+ &Inst::LoadAcquire128 {+ rt1,+ rt2,+ rn,+ scratch,+ flags,+ } => {+ let again_label = sink.get_label();++ // again:+ sink.bind_label(again_label, &mut state.ctrl_plane);++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ // ldaxp rt1, rt2, [rn]+ sink.put4(enc_ldaxp(I64, rt1, rt2, rn));++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ // stxp scratch, rt1, rt2, [rn]+ sink.put4(enc_stxp(I64, scratch, rt1.to_reg(), rt2.to_reg(), rn));++ // cbnz scratch, again.+ let br_again_offset = sink.cur_offset();+ sink.put4(enc_conditional_br(+ BranchTarget::Label(again_label),+ CondBrKind::NotZero(scratch.to_reg(), OperandSize::Size64),+ ));+ sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);+ }&Inst::StoreRelease {access_ty,rt,@@ -1806,6 +2202,40 @@ impl MachInstEmit for Inst {sink.put4(enc_stlr(access_ty, rt, rn));}+ &Inst::StoreRelease128 {+ rt1,+ rt2,+ rn,+ scratch,+ flags,+ } => {+ let again_label = sink.get_label();++ // again:+ sink.bind_label(again_label, &mut state.ctrl_plane);++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ // ldxp xzr, scratch, [rn]+ sink.put4(enc_ldxp(I64, writable_zero_reg(), scratch, rn));++ if let Some(trap_code) = flags.trap_code() {+ sink.add_trap(trap_code);+ }++ // stlxp scratch, rt1, rt2, [rn]+ sink.put4(enc_stlxp(I64, scratch, rt1, rt2, rn));++ // cbnz scratch, again.+ let br_again_offset = sink.cur_offset();+ sink.put4(enc_conditional_br(+ BranchTarget::Label(again_label),+ CondBrKind::NotZero(scratch.to_reg(), OperandSize::Size64),+ ));+ sink.use_label_at_offset(br_again_offset, again_label, LabelUse::Branch19);+ }&Inst::Fence {} => {sink.put4(enc_dmb_ish()); // dmb ish}diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rsindex 04d7e77def..a3c07b6e92 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit_tests.rs@@ -7302,6 +7302,71 @@ fn test_aarch64_binemit() {"atomic_rmw_loop_umax_16 addr=x25 operand=x26 oldval=x27 scratch1=x24 scratch2=x28",));+ insns.push((+ Inst::AtomicRMW128Loop {+ op: AtomicRMWLoopOp::Add,+ flags: MemFlagsData::trusted(),+ addr: xreg(25),+ operand_lo: xreg(26),+ operand_hi: xreg(27),+ oldval_lo: writable_xreg(13),+ oldval_hi: writable_xreg(14),+ scratch1: writable_xreg(15),+ scratch2: writable_xreg(16),+ scratch3: writable_xreg(17),+ },+ "3BDF7FC87C031AABF502169A3CD738C898FFFFB5",+ "atomic_rmw_128_loop_add addr=x25 operand_lo=x26 operand_hi=x27 oldval_lo=x13 oldval_hi=x14 scratch1=x15 scratch2=x16 scratch3=x17",+ ));+ insns.push((+ Inst::AtomicRMW128Loop {+ op: AtomicRMWLoopOp::Xchg,+ flags: MemFlagsData::trusted(),+ addr: xreg(25),+ operand_lo: xreg(26),+ operand_hi: xreg(27),+ oldval_lo: writable_xreg(13),+ oldval_hi: writable_xreg(14),+ scratch1: writable_xreg(15),+ scratch2: writable_xreg(16),+ scratch3: writable_xreg(17),+ },+ "3BDF7FC83ADB38C8D8FFFFB5",+ "atomic_rmw_128_loop_xchg addr=x25 operand_lo=x26 operand_hi=x27 oldval_lo=x13 oldval_hi=x14 scratch1=x15 scratch2=x16 scratch3=x17",+ ));+ insns.push((+ Inst::AtomicRMW128Loop {+ op: AtomicRMWLoopOp::Umin,+ flags: MemFlagsData::trusted(),+ addr: xreg(25),+ operand_lo: xreg(26),+ operand_hi: xreg(27),+ oldval_lo: writable_xreg(13),+ oldval_hi: writable_xreg(14),+ scratch1: writable_xreg(15),+ scratch2: writable_xreg(16),+ scratch3: writable_xreg(17),+ },+ "3BDF7FC87F031AEBFF0216FAD522979A5C239B9A3CD738C858FFFFB5",+ "atomic_rmw_128_loop_umin addr=x25 operand_lo=x26 operand_hi=x27 oldval_lo=x13 oldval_hi=x14 scratch1=x15 scratch2=x16 scratch3=x17",+ ));+ insns.push((+ Inst::AtomicRMW128Loop {+ op: AtomicRMWLoopOp::Nand,+ flags: MemFlagsData::trusted(),+ addr: xreg(25),+ operand_lo: xreg(26),+ operand_hi: xreg(27),+ oldval_lo: writable_xreg(13),+ oldval_hi: writable_xreg(14),+ scratch1: writable_xreg(15),+ scratch2: writable_xreg(16),+ scratch3: writable_xreg(17),+ },+ "3BDF7FC87C031A8AF502168AFC033CAAF50335AA3CD738C858FFFFB5",+ "atomic_rmw_128_loop_nand addr=x25 operand_lo=x26 operand_hi=x27 oldval_lo=x13 oldval_hi=x14 scratch1=x15 scratch2=x16 scratch3=x17",+ ));+insns.push((Inst::AtomicRMW {ty: I8,@@ -7839,6 +7904,22 @@ fn test_aarch64_binemit() {"atomic_cas_loop_64 addr=x25, expect=x26, replacement=x28, oldval=x27, scratch=x24",));+ insns.push((+ Inst::AtomicCAS128Loop {+ flags: MemFlagsData::trusted(),+ addr: xreg(25),+ expected_lo: xreg(26),+ expected_hi: xreg(24),+ replacement_lo: xreg(28),+ replacement_hi: xreg(23),+ oldval_lo: writable_xreg(27),+ oldval_hi: writable_xreg(22),+ scratch: writable_xreg(21),+ },+ "3BD77FC87F031AEBC1000054BF0217EB810000543CDB38C858FFFFB5030000143BD738C8F8FEFFB5",+ "atomic_cas_128_loop addr=x25, expected_lo=x26, expected_hi=x24, replacement_lo=x28, replacement_hi=x23, oldval_lo=x27, oldval_hi=x22, scratch=x21",+ ));+insns.push((Inst::LoadAcquire {access_ty: I8,@@ -7883,6 +7964,18 @@ fn test_aarch64_binemit() {"ldar x28, [x7]",));+ insns.push((+ Inst::LoadAcquire128 {+ rt1: writable_xreg(28),+ rt2: writable_xreg(27),+ rn: xreg(7),+ scratch: writable_xreg(26),+ flags: MemFlagsData::trusted(),+ },+ "FCEC7FC8FC6C3AC8DAFFFFB5",+ "load_acquire_128 x28, x27, [x7], scratch=x26",+ ));+insns.push((Inst::StoreRelease {access_ty: I8,@@ -7927,6 +8020,18 @@ fn test_aarch64_binemit() {"stlr x28, [x7]",));+ insns.push((+ Inst::StoreRelease128 {+ rt1: xreg(28),+ rt2: xreg(27),+ rn: xreg(7),+ scratch: writable_xreg(26),+ flags: MemFlagsData::trusted(),+ },+ "FF687FC8FCEC3AC8DAFFFFB5",+ "store_release_128 x28, x27, [x7], scratch=x26",+ ));+insns.push((Inst::Fence {}, "BF3B03D5", "dmb ish"));let flags = settings::Flags::new(settings::builder());diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rsindex 230cc2abb4..9fa282db3c 100644--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs@@ -487,6 +487,29 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_fixed_def(scratch2, xreg(28));}}+ Inst::AtomicRMW128Loop {+ op,+ addr,+ operand_lo,+ operand_hi,+ oldval_lo,+ oldval_hi,+ scratch1,+ scratch2,+ scratch3,+ ..+ } => {+ collector.reg_fixed_use(addr, xreg(25));+ collector.reg_fixed_use(operand_lo, xreg(26));+ collector.reg_fixed_use(operand_hi, xreg(22));+ collector.reg_fixed_def(oldval_lo, xreg(27));+ collector.reg_fixed_def(oldval_hi, xreg(23));+ collector.reg_fixed_def(scratch1, xreg(24));+ if *op != AtomicRMWLoopOp::Xchg {+ collector.reg_fixed_def(scratch2, xreg(28));+ collector.reg_fixed_def(scratch3, xreg(21));+ }+ }Inst::AtomicRMW { rs, rt, rn, .. } => {collector.reg_use(rs);collector.reg_def(rt);@@ -533,14 +556,58 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_fixed_def(oldval, xreg(27));collector.reg_fixed_def(scratch, xreg(24));}+ Inst::AtomicCAS128Loop {+ addr,+ expected_lo,+ expected_hi,+ replacement_lo,+ replacement_hi,+ oldval_lo,+ oldval_hi,+ scratch,+ ..+ } => {+ collector.reg_fixed_use(addr, xreg(25));+ collector.reg_fixed_use(expected_lo, xreg(26));+ collector.reg_fixed_use(expected_hi, xreg(23));+ collector.reg_fixed_use(replacement_lo, xreg(28));+ collector.reg_fixed_use(replacement_hi, xreg(22));+ collector.reg_fixed_def(oldval_lo, xreg(27));+ collector.reg_fixed_def(oldval_hi, xreg(21));+ collector.reg_fixed_def(scratch, xreg(24));+ }Inst::LoadAcquire { rt, rn, .. } => {collector.reg_use(rn);collector.reg_def(rt);}+ Inst::LoadAcquire128 {+ rt1,+ rt2,+ rn,+ scratch,+ ..+ } => {+ collector.reg_use(rn);+ collector.reg_def(rt1);+ collector.reg_def(rt2);+ collector.reg_def(scratch);+ }Inst::StoreRelease { rt, rn, .. } => {collector.reg_use(rn);collector.reg_use(rt);}+ Inst::StoreRelease128 {+ rt1,+ rt2,+ rn,+ scratch,+ ..+ } => {+ collector.reg_use(rn);+ collector.reg_use(rt1);+ collector.reg_use(rt2);+ collector.reg_early_def(scratch);+ }Inst::Fence {} | Inst::Csdb {} => {}Inst::FpuMove32 { rd, rn } => {collector.reg_def(rd);@@ -1653,6 +1720,43 @@ impl Inst {scratch2,)}+ &Inst::AtomicRMW128Loop {+ op,+ addr,+ operand_lo,+ operand_hi,+ oldval_lo,+ oldval_hi,+ scratch1,+ scratch2,+ scratch3,+ ..+ } => {+ let op = match op {+ AtomicRMWLoopOp::Add => "add",+ AtomicRMWLoopOp::Sub => "sub",+ AtomicRMWLoopOp::Eor => "eor",+ AtomicRMWLoopOp::Orr => "orr",+ AtomicRMWLoopOp::And => "and",+ AtomicRMWLoopOp::Nand => "nand",+ AtomicRMWLoopOp::Smin => "smin",+ AtomicRMWLoopOp::Smax => "smax",+ AtomicRMWLoopOp::Umin => "umin",+ AtomicRMWLoopOp::Umax => "umax",+ AtomicRMWLoopOp::Xchg => "xchg",+ };+ let addr = pretty_print_ireg(addr, OperandSize::Size64);+ let operand_lo = pretty_print_ireg(operand_lo, OperandSize::Size64);+ let operand_hi = pretty_print_ireg(operand_hi, OperandSize::Size64);+ let oldval_lo = pretty_print_ireg(oldval_lo.to_reg(), OperandSize::Size64);+ let oldval_hi = pretty_print_ireg(oldval_hi.to_reg(), OperandSize::Size64);+ let scratch1 = pretty_print_ireg(scratch1.to_reg(), OperandSize::Size64);+ let scratch2 = pretty_print_ireg(scratch2.to_reg(), OperandSize::Size64);+ let scratch3 = pretty_print_ireg(scratch3.to_reg(), OperandSize::Size64);+ format!(+ "atomic_rmw_128_loop_{op} addr={addr} operand_lo={operand_lo} operand_hi={operand_hi} oldval_lo={oldval_lo} oldval_hi={oldval_hi} scratch1={scratch1} scratch2={scratch2} scratch3={scratch3}",+ )+ }&Inst::AtomicCAS {rd, rs, rt, rn, ty, ..} => {@@ -1716,6 +1820,29 @@ impl Inst {scratch,)}+ &Inst::AtomicCAS128Loop {+ addr,+ expected_lo,+ expected_hi,+ replacement_lo,+ replacement_hi,+ oldval_lo,+ oldval_hi,+ scratch,+ ..+ } => {+ let addr = pretty_print_ireg(addr, OperandSize::Size64);+ let expected_lo = pretty_print_ireg(expected_lo, OperandSize::Size64);+ let expected_hi = pretty_print_ireg(expected_hi, OperandSize::Size64);+ let replacement_lo = pretty_print_ireg(replacement_lo, OperandSize::Size64);+ let replacement_hi = pretty_print_ireg(replacement_hi, OperandSize::Size64);+ let oldval_lo = pretty_print_ireg(oldval_lo.to_reg(), OperandSize::Size64);+ let oldval_hi = pretty_print_ireg(oldval_hi.to_reg(), OperandSize::Size64);+ let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);+ format!(+ "atomic_cas_128_loop addr={addr}, expected_lo={expected_lo}, expected_hi={expected_hi}, replacement_lo={replacement_lo}, replacement_hi={replacement_hi}, oldval_lo={oldval_lo}, oldval_hi={oldval_hi}, scratch={scratch}",+ )+ }&Inst::LoadAcquire {access_ty, rt, rn, ..} => {@@ -1731,6 +1858,19 @@ impl Inst {let rt = pretty_print_ireg(rt.to_reg(), size);format!("{op} {rt}, [{rn}]")}+ &Inst::LoadAcquire128 {+ rt1,+ rt2,+ rn,+ scratch,+ ..+ } => {+ let rt1 = pretty_print_ireg(rt1.to_reg(), OperandSize::Size64);+ let rt2 = pretty_print_ireg(rt2.to_reg(), OperandSize::Size64);+ let rn = pretty_print_ireg(rn, OperandSize::Size64);+ let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);+ format!("load_acquire_128 {rt1}, {rt2}, [{rn}], scratch={scratch}")+ }&Inst::StoreRelease {access_ty, rt, rn, ..} => {@@ -1746,6 +1886,19 @@ impl Inst {let rt = pretty_print_ireg(rt, size);format!("{op} {rt}, [{rn}]")}+ &Inst::StoreRelease128 {+ rt1,+ rt2,+ rn,+ scratch,+ ..+ } => {+ let rt1 = pretty_print_ireg(rt1, OperandSize::Size64);+ let rt2 = pretty_print_ireg(rt2, OperandSize::Size64);+ let rn = pretty_print_ireg(rn, OperandSize::Size64);+ let scratch = pretty_print_ireg(scratch.to_reg(), OperandSize::Size64);+ format!("store_release_128 {rt1}, {rt2}, [{rn}], scratch={scratch}")+ }&Inst::Fence {} => {format!("dmb ish")}@@ -3203,14 +3356,10 @@ impl MachInstLabelUse for LabelUse {}}-#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn inst_size_test() {- // This test will help with unintentionally growing the size- // of the Inst enum.- assert_eq!(32, core::mem::size_of::<Inst>());- }+#[test]+#[cfg(target_pointer_width = "64")]+fn inst_size_test() {+ // This test will help with unintentionally growing the size+ // of the Inst enum.+ assert_eq!(48, core::mem::size_of::<Inst>());}diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex de5a653354..26c09e8fbf 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -2384,13 +2384,20 @@;;;; Rules for `AtomicLoad` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (atomic_load (valid_atomic_transaction ty) (little_or_native_endian flags) addr))(load_acquire ty flags addr))-+;; Lower 128-bit `atomic_load` through a special case+(rule 1 (lower (atomic_load $I128 (little_or_native_endian flags) addr))+ (load_acquire_128 flags addr));;;; Rules for `AtomicStore` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule (lower (atomic_store (little_or_native_endian flags)src @ (value_type (valid_atomic_transaction ty))addr))(side_effect (store_release ty flags src addr)))+;; Lower 128-bit `atomic_store` through a special case+(rule 1 (lower (atomic_store (little_or_native_endian flags)+ src @ (value_type $I128)+ addr))+ (side_effect (store_release_128 flags src addr)));;;; Rules for `AtomicRMW` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -2446,19 +2453,45 @@(rule (lower (atomic_rmw (valid_atomic_transaction ty) (little_or_native_endian flags) (AtomicRmwOp.Xchg) addr src))(atomic_rmw_loop (AtomicRMWLoopOp.Xchg) addr src ty flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Add) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Add) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Sub) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Sub) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.And) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.And) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Nand) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Nand) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Or) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Orr) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Xor) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Eor) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Smin) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Smin) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Smax) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Smax) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Umin) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Umin) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Umax) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Umax) addr src flags))+(rule 2 (lower (atomic_rmw $I128 (little_or_native_endian flags) (AtomicRmwOp.Xchg) addr src))+ (atomic_rmw_128_loop (AtomicRMWLoopOp.Xchg) addr src flags))+;;;; Rules for `AtomicCAS` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(rule 1 (lower (and (use_lse)(atomic_cas (valid_atomic_transaction ty) (little_or_native_endian flags) addr src1 src2)))(lse_atomic_cas addr src1 src2 ty flags))(attr rule lse_atomic_cas_i128 (tag atomics))-(rule lse_atomic_cas_i128 2 (lower (and (use_lse)+(rule lse_atomic_cas_i128 3 (lower (and (use_lse)(atomic_cas $I128 (little_or_native_endian flags) addr expect replace)))(lse_atomic_cas_128 addr expect replace flags))(rule (lower (and (atomic_cas (valid_atomic_transaction ty) (little_or_native_endian flags) addr src1 src2)))(atomic_cas_loop addr src1 src2 ty flags))+(rule 2 (lower (and (atomic_cas $I128 (little_or_native_endian flags) addr expect replace)))+ (atomic_cas_128_loop addr expect replace flags))+;;;; Rules for 'fvdemote' ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;(attr fvdemote (tag vector))(rule (lower (fvdemote _ x))diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rsindex a2abe420a3..bed9449208 100644--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs@@ -342,7 +342,7 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {/// This is target-word-size dependent. And it excludes booleans and reftypes.fn valid_atomic_transaction(&mut self, ty: Type) -> Option<Type> {match ty {- I8 | I16 | I32 | I64 => Some(ty),+ I8 | I16 | I32 | I64 | I128 => Some(ty),_ => None,}}diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic-cas-128.clif b/cranelift/filetests/filetests/isa/aarch64/atomic-cas-128.clifnew file mode 100644index 0000000000..bcbb77fe8b--- /dev/null+++ b/cranelift/filetests/filetests/isa/aarch64/atomic-cas-128.clif@@ -0,0 +1,130 @@+test compile precise-output+target aarch64++function %atomic_cas_i128(i64, i128, i128) -> i128 {+block0(v0: i64, v1: i128, v2: i128):+ v3 = atomic_cas.i128 v0, v1, v2+ return v3+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x5+; mov x23, x3+; mov x25, x0+; mov x26, x2+; mov x28, x4+; atomic_cas_128_loop addr=x25, expected_lo=x26, expected_hi=x23, replacement_lo=x28, replacement_hi=x22, oldval_lo=x27, oldval_hi=x21, scratch=x24+; mov x0, x27+; mov x1, x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x5+; mov x23, x3+; mov x25, x0+; mov x26, x2+; mov x28, x4+; ldaxp x27, x21, [x25] ; trap: heap_oob+; cmp x27, x26+; b.ne #0x4c+; cmp x21, x23+; b.ne #0x4c+; stlxp w24, x28, x22, [x25] ; trap: heap_oob+; cbnz x24, #0x2c+; b #0x54+; stlxp w24, x27, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x2c+; mov x0, x27+; mov x1, x21+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret++function %atomic_cas_i128_success(i64, i128, i128) -> i8 {+block0(v0: i64, v1: i128, v2: i128):+ v3 = atomic_cas.i128 v0, v1, v2+ v4 = icmp eq v3, v1+ return v4+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x5+; mov x23, x3+; mov x25, x0+; mov x26, x2+; mov x28, x4+; atomic_cas_128_loop addr=x25, expected_lo=x26, expected_hi=x23, replacement_lo=x28, replacement_hi=x22, oldval_lo=x27, oldval_hi=x21, scratch=x24+; subs xzr, x27, x26+; ccmp x21, x23, #nzcv, eq+; cset x0, eq+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x5+; mov x23, x3+; mov x25, x0+; mov x26, x2+; mov x28, x4+; ldaxp x27, x21, [x25] ; trap: heap_oob+; cmp x27, x26+; b.ne #0x4c+; cmp x21, x23+; b.ne #0x4c+; stlxp w24, x28, x22, [x25] ; trap: heap_oob+; cbnz x24, #0x2c+; b #0x54+; stlxp w24, x27, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x2c+; cmp x27, x26+; ccmp x21, x23, #0, eq+; cset x0, eq+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif b/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clifindex 29cd380c0f..926ab9a1ea 100644--- a/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif+++ b/cranelift/filetests/filetests/isa/aarch64/atomic-rmw.clif@@ -1,6 +1,55 @@test compile precise-outputtarget aarch64+function %atomic_rmw_add_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 add v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_add addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; adds x28, x27, x26+; adc x21, x23, x22+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_add_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 add v0, v1@@ -169,6 +218,55 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_sub_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 sub v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_sub addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; subs x28, x27, x26+; sbc x21, x23, x22+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_sub_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 sub v0, v1@@ -337,6 +435,55 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_and_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 and v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_and addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; and x28, x27, x26+; and x21, x23, x22+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_and_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 and v0, v1@@ -505,6 +652,57 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_nand_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 nand v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_nand addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; and x28, x27, x26+; and x21, x23, x22+; mvn x28, x28+; mvn x21, x21+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_nand_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 nand v0, v1@@ -677,6 +875,55 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_or_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 or v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_orr addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; orr x28, x27, x26+; orr x21, x23, x22+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_or_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 or v0, v1@@ -845,6 +1092,55 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_xor_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 xor v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_eor addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; eor x28, x27, x26+; eor x21, x23, x22+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_xor_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 xor v0, v1@@ -1013,6 +1309,57 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_smax_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 smax v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_smax addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; cmp x27, x26+; sbcs xzr, x23, x22+; csel x21, x22, x23, lt+; csel x28, x26, x27, lt+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_smax_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 smax v0, v1@@ -1187,6 +1534,57 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_umax_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 umax v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_umax addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; cmp x27, x26+; sbcs xzr, x23, x22+; csel x21, x22, x23, lo+; csel x28, x26, x27, lo+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_umax_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 umax v0, v1@@ -1359,6 +1757,57 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_smin_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 smin v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_smin addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; cmp x27, x26+; sbcs xzr, x23, x22+; csel x21, x22, x23, ge+; csel x28, x26, x27, ge+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_smin_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 smin v0, v1@@ -1533,6 +1982,57 @@ block0(v0: i64, v1: i8):; ldp x29, x30, [sp], #0x10; ret+function %atomic_rmw_umin_i128(i64, i128) {+block0(v0: i64, v1: i128):+ v2 = atomic_rmw.i128 umin v0, v1+ return+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; stp x27, x28, [sp, #-16]!+; stp x25, x26, [sp, #-16]!+; stp x23, x24, [sp, #-16]!+; stp x21, x22, [sp, #-16]!+; block0:+; mov x22, x3+; mov x25, x0+; mov x26, x2+; atomic_rmw_128_loop_umin addr=x25 operand_lo=x26 operand_hi=x22 oldval_lo=x27 oldval_hi=x23 scratch1=x24 scratch2=x28 scratch3=x21+; ldp x21, x22, [sp], #16+; ldp x23, x24, [sp], #16+; ldp x25, x26, [sp], #16+; ldp x27, x28, [sp], #16+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; stp x27, x28, [sp, #-0x10]!+; stp x25, x26, [sp, #-0x10]!+; stp x23, x24, [sp, #-0x10]!+; stp x21, x22, [sp, #-0x10]!+; block1: ; offset 0x18+; mov x22, x3+; mov x25, x0+; mov x26, x2+; ldaxp x27, x23, [x25] ; trap: heap_oob+; cmp x27, x26+; sbcs xzr, x23, x22+; csel x21, x22, x23, hs+; csel x28, x26, x27, hs+; stlxp w24, x28, x21, [x25] ; trap: heap_oob+; cbnz x24, #0x24+; ldp x21, x22, [sp], #0x10+; ldp x23, x24, [sp], #0x10+; ldp x25, x26, [sp], #0x10+; ldp x27, x28, [sp], #0x10+; ldp x29, x30, [sp], #0x10+; ret+function %atomic_rmw_umin_i64(i64, i64) {block0(v0: i64, v1: i64):v2 = atomic_rmw.i64 umin v0, v1diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic_load.clif b/cranelift/filetests/filetests/isa/aarch64/atomic_load.clifindex 044cf0f141..ef894f8add 100644--- a/cranelift/filetests/filetests/isa/aarch64/atomic_load.clif+++ b/cranelift/filetests/filetests/isa/aarch64/atomic_load.clif@@ -1,6 +1,24 @@test compile precise-outputtarget aarch64+function %atomic_load_i128(i64) -> i128 {+block0(v0: i64):+ v1 = atomic_load.i128 v0+ return v1+}++; VCode:+; block0:+; load_acquire_128 x0, x1, [x0], scratch=x4+; ret+;+; Disassembled:+; block0: ; offset 0x0+; ldaxp x0, x1, [x0] ; trap: heap_oob+; stxp w4, x0, x1, [x0] ; trap: heap_oob+; cbnz x4, #0+; ret+function %atomic_load_i64(i64) -> i64 {block0(v0: i64):v1 = atomic_load.i64 v0diff --git a/cranelift/filetests/filetests/isa/aarch64/atomic_store.clif b/cranelift/filetests/filetests/isa/aarch64/atomic_store.clifindex 97927059cf..282f002066 100644--- a/cranelift/filetests/filetests/isa/aarch64/atomic_store.clif+++ b/cranelift/filetests/filetests/isa/aarch64/atomic_store.clif@@ -1,6 +1,24 @@test compile precise-outputtarget aarch64+function %atomic_store_i128(i128, i64) {+block0(v0: i128, v1: i64):+ atomic_store.i128 v0, v1+ return+}++; VCode:+; block0:+; store_release_128 x0, x1, [x2], scratch=x4+; ret+;+; Disassembled:+; block0: ; offset 0x0+; ldxp xzr, x4, [x2] ; trap: heap_oob+; stlxp w4, x0, x1, [x2] ; trap: heap_oob+; cbnz x4, #0+; ret+function %atomic_store_i64(i64, i64) {block0(v0: i64, v1: i64):atomic_store.i64 v0, v1diff --git a/cranelift/filetests/filetests/runtests/atomic-128.clif b/cranelift/filetests/filetests/runtests/atomic-128.clifindex cdf018dbf9..0019952714 100644--- a/cranelift/filetests/filetests/runtests/atomic-128.clif+++ b/cranelift/filetests/filetests/runtests/atomic-128.clif@@ -3,9 +3,10 @@ test runset enable_llvm_abi_extensionsset enable_multi_ret_implicit_srettarget x86_64 has_cmpxchg16b+target aarch64function %atomic_load(i128) -> i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128):v3 = stack_addr.i64 ss0@@ -24,7 +25,7 @@ block0(v0: i128):function %atomic_store(i128) -> i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128):v1 = stack_addr.i64 ss0@@ -43,7 +44,7 @@ block0(v0: i128):function %atomic_cas(i128, i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128, v2: i128):v6 = stack_addr.i64 ss0@@ -63,7 +64,7 @@ block0(v0: i128, v1: i128, v2: i128):function %atomic_add(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -84,7 +85,7 @@ block0(v0: i128, v1: i128):function %atomic_sub(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -105,7 +106,7 @@ block0(v0: i128, v1: i128):function %atomic_and(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -126,7 +127,7 @@ block0(v0: i128, v1: i128):function %atomic_nand(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -147,7 +148,7 @@ block0(v0: i128, v1: i128):function %atomic_or(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -168,7 +169,7 @@ block0(v0: i128, v1: i128):function %atomic_xor(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -189,7 +190,7 @@ block0(v0: i128, v1: i128):function %atomic_xchg(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -210,7 +211,7 @@ block0(v0: i128, v1: i128):function %atomic_umin(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -233,7 +234,7 @@ block0(v0: i128, v1: i128):function %atomic_umax(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -256,7 +257,7 @@ block0(v0: i128, v1: i128):function %atomic_smin(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0@@ -279,7 +280,7 @@ block0(v0: i128, v1: i128):function %atomic_smax(i128, i128) -> i128, i128 {- ss0 = explicit_slot 16+ ss0 = explicit_slot 16, align=16block0(v0: i128, v1: i128):v5 = stack_addr.i64 ss0