Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift: Implement 128-bit atomic CAS for s390x (#14771)

Aelin281af9952f38
diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isle
index 7f194c42df..1a385f805e 100644
--- a/cranelift/codegen/src/isa/s390x/inst.isle
+++ b/cranelift/codegen/src/isa/s390x/inst.isle
@@ -239,6 +239,13 @@
(rn Reg)
(mem MemArg))
+ ;; A 128-bit atomic compare-and-swap operation.
+ (AtomicCas128
+ (rd WritableRegPair)
+ (ri RegPair)
+ (rn RegPair)
+ (mem MemArg))
+
;; A memory fence operation.
(Fence)
@@ -2353,6 +2360,13 @@
(_ Unit (emit (MInst.AtomicCas64 dst src1 src2 mem))))
dst))
+;; Helper for emitting `MInst.AtomicCas128` instructions.
+(decl atomic_cas128 (RegPair RegPair MemArg) RegPair)
+(rule (atomic_cas128 src1 src2 mem)
+ (let ((dst WritableRegPair (temp_writable_regpair $I64))
+ (_ Unit (emit (MInst.AtomicCas128 dst src1 src2 mem))))
+ dst))
+
;; Helper for emitting `MInst.Fence` instructions.
(decl fence_impl () SideEffectNoResult)
(rule (fence_impl)
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit.rs b/cranelift/codegen/src/isa/s390x/inst/emit.rs
index 10b4a9afff..28903b2390 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit.rs
@@ -2186,6 +2186,39 @@ impl Inst {
rd, rn, &mem, opcode_rs, opcode_rsy, true, sink, emit_info, state,
);
}
+ &Inst::AtomicCas128 {
+ rd,
+ ri,
+ rn,
+ ref mem,
+ } => {
+ let rd1 = rd.hi;
+ let rd2 = rd.lo;
+ debug_assert_valid_regpair!(rd1.to_reg(), rd2.to_reg());
+ let ri1 = ri.hi;
+ let ri2 = ri.lo;
+ debug_assert_valid_regpair!(ri1, ri2);
+ let rn1 = rn.hi;
+ let rn2 = rn.lo;
+ debug_assert_valid_regpair!(rn1, rn2);
+
+ debug_assert_eq!(rd1.to_reg(), ri1);
+ debug_assert_eq!(rd2.to_reg(), ri2);
+ let mem = mem.clone();
+
+ // CDSG
+ mem_rs_emit(
+ rd1.to_reg(),
+ rn1,
+ &mem,
+ None,
+ Some(0xeb3e),
+ true,
+ sink,
+ emit_info,
+ state,
+ );
+ }
&Inst::Fence => {
put(sink, &enc_e(0x07e0));
}
diff --git a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
index e2ac640ffb..18919d885f 100644
--- a/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/emit_tests.rs
@@ -3520,6 +3520,66 @@ fn test_s390x_binemit() {
"EB456FFF7F30",
"csg %r4, %r5, 524287(%r6)",
));
+ insns.push((
+ Inst::AtomicCas128 {
+ rd: w_regpair,
+ ri: regpair,
+ rn: regpair,
+ mem: MemArg::BXD20 {
+ base: zero_reg(),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(-524287).unwrap(),
+ flags: MachMemFlags::trusted(),
+ },
+ },
+ "EB220001803E",
+ "cdsg %r2, %r2, -524287",
+ ));
+ insns.push((
+ Inst::AtomicCas128 {
+ rd: w_regpair,
+ ri: regpair,
+ rn: regpair,
+ mem: MemArg::BXD20 {
+ base: zero_reg(),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(524287).unwrap(),
+ flags: MachMemFlags::trusted(),
+ },
+ },
+ "EB220FFF7F3E",
+ "cdsg %r2, %r2, 524287",
+ ));
+ insns.push((
+ Inst::AtomicCas128 {
+ rd: w_regpair,
+ ri: regpair,
+ rn: regpair,
+ mem: MemArg::BXD20 {
+ base: gpr(6),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(-524287).unwrap(),
+ flags: MachMemFlags::trusted(),
+ },
+ },
+ "EB226001803E",
+ "cdsg %r2, %r2, -524287(%r6)",
+ ));
+ insns.push((
+ Inst::AtomicCas128 {
+ rd: w_regpair,
+ ri: regpair,
+ rn: regpair,
+ mem: MemArg::BXD20 {
+ base: gpr(6),
+ index: zero_reg(),
+ disp: SImm20::maybe_from_i64(524287).unwrap(),
+ flags: MachMemFlags::trusted(),
+ },
+ },
+ "EB226FFF7F3E",
+ "cdsg %r2, %r2, 524287(%r6)",
+ ));
insns.push((Inst::Fence, "07E0", "bcr 14, 0"));
insns.push((
diff --git a/cranelift/codegen/src/isa/s390x/inst/mod.rs b/cranelift/codegen/src/isa/s390x/inst/mod.rs
index 7d14622677..1c0840f25c 100644
--- a/cranelift/codegen/src/isa/s390x/inst/mod.rs
+++ b/cranelift/codegen/src/isa/s390x/inst/mod.rs
@@ -52,10 +52,11 @@ pub struct ReturnCallInfo<T> {
}
#[test]
+#[cfg(target_pointer_width = "64")]
fn inst_size_test() {
// This test will help with unintentionally growing the size
// of the Inst enum.
- assert_eq!(32, core::mem::size_of::<Inst>());
+ assert_eq!(40, core::mem::size_of::<Inst>());
}
/// A register pair. Enum so it can be destructured in ISLE.
@@ -133,6 +134,7 @@ impl Inst {
| Inst::AtomicRmw { .. }
| Inst::AtomicCas32 { .. }
| Inst::AtomicCas64 { .. }
+ | Inst::AtomicCas128 { .. }
| Inst::Fence
| Inst::Load32 { .. }
| Inst::Load32ZExt8 { .. }
@@ -595,6 +597,15 @@ fn s390x_get_operands(inst: &mut Inst, collector: &mut DenyReuseVisitor<impl Ope
collector.reg_use(rn);
memarg_operands(mem, collector);
}
+ Inst::AtomicCas128 { rd, ri, rn, mem } => {
+ collector.reg_fixed_def(&mut rd.hi, gpr(2)); // `rd.hi` == `ri.hi`
+ collector.reg_fixed_def(&mut rd.lo, gpr(3)); // `rd.lo` == `ri.lo`
+ collector.reg_fixed_use(&mut ri.hi, gpr(2));
+ collector.reg_fixed_use(&mut ri.lo, gpr(3));
+ collector.reg_fixed_use(&mut rn.hi, gpr(4));
+ collector.reg_fixed_use(&mut rn.lo, gpr(5));
+ memarg_operands(mem, collector);
+ }
Inst::Fence => {}
Inst::Load32 { rd, mem, .. }
| Inst::Load32ZExt8 { rd, mem, .. }
@@ -1889,6 +1900,30 @@ impl Inst {
format!("{}{} {}, {}, {}", mem_str, op.unwrap(), rd, rn, mem)
}
+ &Inst::AtomicCas128 {
+ rd,
+ ri,
+ rn,
+ ref mem,
+ } => {
+ let rd = pretty_print_regpair_mod(rd, ri);
+ let rn = pretty_print_regpair(rn);
+ let mem = mem.clone();
+ let (mem_str, mem) = mem_finalize_for_show(
+ &mem,
+ state,
+ MemInstType {
+ have_d12: false,
+ have_d20: true,
+ have_pcrel: false,
+ have_unaligned_pcrel: false,
+ have_index: false,
+ },
+ );
+ let mem = mem.pretty_print_default();
+
+ format!("{mem_str}cdsg {rd}, {rn}, {mem}")
+ }
&Inst::Fence => "bcr 14, 0".to_string(),
&Inst::Load32 { rd, ref mem }
| &Inst::Load32ZExt8 { rd, ref mem }
diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isle
index 2f3e9ed973..41c06aa63d 100644
--- a/cranelift/codegen/src/isa/s390x/lower.isle
+++ b/cranelift/codegen/src/isa/s390x/lower.isle
@@ -3709,6 +3709,29 @@
;;;; Rules for `atomic_cas` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Helper to move a 128-bit value from a VR to a RegPair of GPRs
+(decl vr_to_regpair (Reg) RegPair)
+(rule (vr_to_regpair src)
+ (let ((src_hi Reg (vec_extract_lane $I64X2 src 0 (zero_reg)))
+ (src_lo Reg (vec_extract_lane $I64X2 src 1 (zero_reg))))
+ (regpair src_hi src_lo)))
+
+;; Helper to move a 128-bit Value into a RegPair
+(decl i128_to_regpair (Value) RegPair)
+(rule (i128_to_regpair val)
+ (let ((src Reg val))
+ (vr_to_regpair src)))
+
+;; Helper to move a RegPair into a single vector register
+(decl regpair_to_vr (RegPair) Reg)
+(rule (regpair_to_vr pair)
+ (mov_to_vec128 $I64X2 (regpair_hi pair) (regpair_lo pair)))
+
+;; 128-bit big-endian atomic compare-and-swap instruction.
+(rule 3 (lower (atomic_cas $I128 (mem_flags_data flags @ (bigendian)) addr src1 src2))
+ (regpair_to_vr (atomic_cas128 (i128_to_regpair src1) (i128_to_regpair src2)
+ (lower_address flags addr (zero_offset)))))
+
;; 32/64-bit big-endian atomic compare-and-swap instruction.
(rule 2 (lower (atomic_cas (ty_32_or_64 ty) (mem_flags_data flags @ (bigendian)) addr src1 src2))
(atomic_cas_impl ty (put_in_reg src1) (put_in_reg src2)
diff --git a/cranelift/filetests/filetests/isa/s390x/atomic_cas.clif b/cranelift/filetests/filetests/isa/s390x/atomic_cas.clif
index b1e781a42d..7b190366a8 100644
--- a/cranelift/filetests/filetests/isa/s390x/atomic_cas.clif
+++ b/cranelift/filetests/filetests/isa/s390x/atomic_cas.clif
@@ -1,10 +1,54 @@
test compile precise-output
+set enable_multi_ret_implicit_sret
target s390x
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; ATOMIC_CAS
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+function %atomic_cas_i128(i128, i128, i64) -> i128 {
+block0(v0: i128, v1: i128, v2: i64):
+ v3 = atomic_cas.i128 v2, v0, v1
+ return v3
+}
+
+; VCode:
+; stmg %r6, %r15, 48(%r15)
+; block0:
+; lgr %r6, %r5
+; lgr %r14, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; lgdr %r2, %f1
+; vlgvg %r3, %v1, 1
+; lgdr %r4, %f3
+; vlgvg %r5, %v3, 1
+; cdsg %r2, %r4, 0(%r6)
+; vlvgp %v18, %r2, %r3
+; lgr %r2, %r14
+; vst %v18, 0(%r2)
+; lmg %r6, %r15, 48(%r15)
+; br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+; stmg %r6, %r15, 0x30(%r15)
+; block1: ; offset 0x6
+; lgr %r6, %r5
+; lgr %r14, %r2
+; vl %v1, 0(%r3)
+; vl %v3, 0(%r4)
+; lgdr %r2, %f1
+; vlgvg %r3, %v1, 1
+; lgdr %r4, %f3
+; vlgvg %r5, %v3, 1
+; cdsg %r2, %r4, 0(%r6) ; trap: heap_oob
+; vlvgp %v18, %r2, %r3
+; lgr %r2, %r14
+; vst %v18, 0(%r2)
+; lmg %r6, %r15, 0x30(%r15)
+; br %r14
+
function %atomic_cas_i64(i64, i64, i64) -> i64 {
block0(v0: i64, v1: i64, v2: i64):
v3 = atomic_cas.i64 v2, v0, v1