A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift: Implement 128-bit atomic CAS for s390x (#14771)
Aelin281af9952f38
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isleindex 7f194c42df..1a385f805e 100644--- a/cranelift/codegen/src/isa/s390x/inst.isle+++ b/cranelift/codegen/src/isa/s390x/inst.isle@@ -239,6 +239,13 @@(rn Reg)(mem MemArg))+ ;; A 128-bit atomic compare-and-swap operation.+ (AtomicCas128+ (rd WritableRegPair)+ (ri RegPair)+ (rn RegPair)+ (mem MemArg))+;; A memory fence operation.(Fence)@@ -2353,6 +2360,13 @@(_ Unit (emit (MInst.AtomicCas64 dst src1 src2 mem))))dst))+;; Helper for emitting `MInst.AtomicCas128` instructions.+(decl atomic_cas128 (RegPair RegPair MemArg) RegPair)+(rule (atomic_cas128 src1 src2 mem)+ (let ((dst WritableRegPair (temp_writable_regpair $I64))+ (_ Unit (emit (MInst.AtomicCas128 dst src1 src2 mem))))+ dst))+;; Helper for emitting `MInst.Fence` instructions.(decl fence_impl () SideEffectNoResult)(rule (fence_impl)diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rsindex 10b4a9afff..28903b2390 100644--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs@@ -2186,6 +2186,39 @@ impl Inst {rd, rn, &mem, opcode_rs, opcode_rsy, true, sink, emit_info, state,);}+ &Inst::AtomicCas128 {+ rd,+ ri,+ rn,+ ref mem,+ } => {+ let rd1 = rd.hi;+ let rd2 = rd.lo;+ debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());+ let ri1 = ri.hi;+ let ri2 = ri.lo;+ debug_assert_valid_regpair!(ri1, ri2);+ let rn1 = rn.hi;+ let rn2 = rn.lo;+ debug_assert_valid_regpair!(rn1, rn2);++ debug_assert_eq!(rd1.to_reg(), ri1);+ debug_assert_eq!(rd2.to_reg(), ri2);+ let mem = mem.clone();++ // CDSG+ mem_rs_emit(+ rd1.to_reg(),+ rn1,+ &mem,+ None,+ Some(0xeb3e),+ true,+ sink,+ emit_info,+ state,+ );+ }&Inst::Fence => {put(sink, &enc_e(0x07e0));}diff --git a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rsindex e2ac640ffb..18919d885f 100644--- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs+++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs@@ -3520,6 +3520,66 @@ fn test_s390x_binemit() {"EB456FFF7F30","csg %r4, %r5, 524287(%r6)",));+ insns.push((+ Inst::AtomicCas128 {+ rd: w_regpair,+ ri: regpair,+ rn: regpair,+ mem: MemArg::BXD20 {+ base: zero_reg(),+ index: zero_reg(),+ disp: SImm20::maybe_from_i64(-524287).unwrap(),+ flags: MachMemFlags::trusted(),+ },+ },+ "EB220001803E",+ "cdsg %r2, %r2, -524287",+ ));+ insns.push((+ Inst::AtomicCas128 {+ rd: w_regpair,+ ri: regpair,+ rn: regpair,+ mem: MemArg::BXD20 {+ base: zero_reg(),+ index: zero_reg(),+ disp: SImm20::maybe_from_i64(524287).unwrap(),+ flags: MachMemFlags::trusted(),+ },+ },+ "EB220FFF7F3E",+ "cdsg %r2, %r2, 524287",+ ));+ insns.push((+ Inst::AtomicCas128 {+ rd: w_regpair,+ ri: regpair,+ rn: regpair,+ mem: MemArg::BXD20 {+ base: gpr(6),+ index: zero_reg(),+ disp: SImm20::maybe_from_i64(-524287).unwrap(),+ flags: MachMemFlags::trusted(),+ },+ },+ "EB226001803E",+ "cdsg %r2, %r2, -524287(%r6)",+ ));+ insns.push((+ Inst::AtomicCas128 {+ rd: w_regpair,+ ri: regpair,+ rn: regpair,+ mem: MemArg::BXD20 {+ base: gpr(6),+ index: zero_reg(),+ disp: SImm20::maybe_from_i64(524287).unwrap(),+ flags: MachMemFlags::trusted(),+ },+ },+ "EB226FFF7F3E",+ "cdsg %r2, %r2, 524287(%r6)",+ ));insns.push((Inst::Fence, "07E0", "bcr 14, 0"));insns.push((diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rsindex 7d14622677..1c0840f25c 100644--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs@@ -52,10 +52,11 @@ pub struct ReturnCallInfo<T> {}#[test]+#[cfg(target_pointer_width = "64")]fn inst_size_test() {// This test will help with unintentionally growing the size// of the Inst enum.- assert_eq!(32, core::mem::size_of::<Inst>());+ assert_eq!(40, core::mem::size_of::<Inst>());}/// A register pair. Enum so it can be destructured in ISLE.@@ -133,6 +134,7 @@ impl Inst {| Inst::AtomicRmw { .. }| Inst::AtomicCas32 { .. }| Inst::AtomicCas64 { .. }+ | Inst::AtomicCas128 { .. }| Inst::Fence| Inst::Load32 { .. }| Inst::Load32ZExt8 { .. }@@ -595,6 +597,15 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl Opecollector.reg_use(rn);memarg_operands(mem, collector);}+ Inst::AtomicCas128 { rd, ri, rn, mem } => {+ collector.reg_fixed_def(&mut rd.hi, gpr(2)); // `rd.hi` == `ri.hi`+ collector.reg_fixed_def(&mut rd.lo, gpr(3)); // `rd.lo` == `ri.lo`+ collector.reg_fixed_use(&mut ri.hi, gpr(2));+ collector.reg_fixed_use(&mut ri.lo, gpr(3));+ collector.reg_fixed_use(&mut rn.hi, gpr(4));+ collector.reg_fixed_use(&mut rn.lo, gpr(5));+ memarg_operands(mem, collector);+ }Inst::Fence => {}Inst::Load32 { rd, mem, .. }| Inst::Load32ZExt8 { rd, mem, .. }@@ -1889,6 +1900,30 @@ impl Inst {format!("{}{} {}, {}, {}", mem_str, op.unwrap(), rd, rn, mem)}+ &Inst::AtomicCas128 {+ rd,+ ri,+ rn,+ ref mem,+ } => {+ let rd = pretty_print_regpair_mod(rd, ri);+ let rn = pretty_print_regpair(rn);+ let mem = mem.clone();+ let (mem_str, mem) = mem_finalize_for_show(+ &mem,+ state,+ MemInstType {+ have_d12: false,+ have_d20: true,+ have_pcrel: false,+ have_unaligned_pcrel: false,+ have_index: false,+ },+ );+ let mem = mem.pretty_print_default();++ format!("{mem_str}cdsg {rd}, {rn}, {mem}")+ }&Inst::Fence => "bcr 14, 0".to_string(),&Inst::Load32 { rd, ref mem }| &Inst::Load32ZExt8 { rd, ref mem }diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isleindex 2f3e9ed973..41c06aa63d 100644--- a/cranelift/codegen/src/isa/s390x/lower.isle+++ b/cranelift/codegen/src/isa/s390x/lower.isle@@ -3709,6 +3709,29 @@;;;; Rules for `atomic_cas` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+;; Helper to move a 128-bit value from a VR to a RegPair of GPRs+(decl vr_to_regpair (Reg) RegPair)+(rule (vr_to_regpair src)+ (let ((src_hi Reg (vec_extract_lane $I64X2 src 0 (zero_reg)))+ (src_lo Reg (vec_extract_lane $I64X2 src 1 (zero_reg))))+ (regpair src_hi src_lo)))++;; Helper to move a 128-bit Value into a RegPair+(decl i128_to_regpair (Value) RegPair)+(rule (i128_to_regpair val)+ (let ((src Reg val))+ (vr_to_regpair src)))++;; Helper to move a RegPair into a single vector register+(decl regpair_to_vr (RegPair) Reg)+(rule (regpair_to_vr pair)+ (mov_to_vec128 $I64X2 (regpair_hi pair) (regpair_lo pair)))++;; 128-bit big-endian atomic compare-and-swap instruction.+(rule 3 (lower (atomic_cas $I128 (mem_flags_data flags @ (bigendian)) addr src1 src2))+ (regpair_to_vr (atomic_cas128 (i128_to_regpair src1) (i128_to_regpair src2)+ (lower_address flags addr (zero_offset)))))+;; 32/64-bit big-endian atomic compare-and-swap instruction.(rule 2 (lower (atomic_cas (ty_32_or_64 ty) (mem_flags_data flags @ (bigendian)) addr src1 src2))(atomic_cas_impl ty (put_in_reg src1) (put_in_reg src2)diff --git a/cranelift/filetests/filetests/isa/s390x/atomic_cas.clif b/cranelift/filetests/filetests/isa/s390x/atomic_cas.clifindex b1e781a42d..7b190366a8 100644--- a/cranelift/filetests/filetests/isa/s390x/atomic_cas.clif+++ b/cranelift/filetests/filetests/isa/s390x/atomic_cas.clif@@ -1,10 +1,54 @@test compile precise-output+set enable_multi_ret_implicit_srettarget s390x;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ATOMIC_CAS;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;+function %atomic_cas_i128(i128, i128, i64) -> i128 {+block0(v0: i128, v1: i128, v2: i64):+ v3 = atomic_cas.i128 v2, v0, v1+ return v3+}++; VCode:+; stmg %r6, %r15, 48(%r15)+; block0:+; lgr %r6, %r5+; lgr %r14, %r2+; vl %v1, 0(%r3)+; vl %v3, 0(%r4)+; lgdr %r2, %f1+; vlgvg %r3, %v1, 1+; lgdr %r4, %f3+; vlgvg %r5, %v3, 1+; cdsg %r2, %r4, 0(%r6)+; vlvgp %v18, %r2, %r3+; lgr %r2, %r14+; vst %v18, 0(%r2)+; lmg %r6, %r15, 48(%r15)+; br %r14+;+; Disassembled:+; block0: ; offset 0x0+; stmg %r6, %r15, 0x30(%r15)+; block1: ; offset 0x6+; lgr %r6, %r5+; lgr %r14, %r2+; vl %v1, 0(%r3)+; vl %v3, 0(%r4)+; lgdr %r2, %f1+; vlgvg %r3, %v1, 1+; lgdr %r4, %f3+; vlgvg %r5, %v3, 1+; cdsg %r2, %r4, 0(%r6) ; trap: heap_oob+; vlvgp %v18, %r2, %r3+; lgr %r2, %r14+; vst %v18, 0(%r2)+; lmg %r6, %r15, 0x30(%r15)+; br %r14+function %atomic_cas_i64(i64, i64, i64) -> i64 {block0(v0: i64, v1: i64, v2: i64):v3 = atomic_cas.i64 v2, v0, v1