Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Fix double `lr.w` in sub-word `atomic_rmw` loops (#13399)

Suraj Kumar71c5b1e1bbc5
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index fa922a6b6e..483d93c8ad 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -1824,7 +1824,9 @@ impl Inst {
amo: AMO::SeqCst,
}
.emit(sink, emit_info, state);
- //
+ // For sub-word ops the merge step needs the original full word.
+ // Stash it in spilltmp2 before `extract` clobbers `dst`; reusing
+ // the value avoids a second LR that would cancel the reservation.
let store_value: Reg = match op {
crate::ir::AtomicRmwOp::Add
@@ -1832,6 +1834,8 @@ impl Inst {
| crate::ir::AtomicRmwOp::And
| crate::ir::AtomicRmwOp::Or
| crate::ir::AtomicRmwOp::Xor => {
+ Inst::gen_move(writable_spilltmp_reg2(), dst.to_reg(), I64)
+ .emit(sink, emit_info, state);
AtomicOP::extract(dst, offset, dst.to_reg(), ty)
.iter()
.for_each(|i| i.emit(sink, emit_info, state));
@@ -1849,14 +1853,6 @@ impl Inst {
rs2: x,
}
.emit(sink, emit_info, state);
- Inst::Atomic {
- op: AtomicOP::load_op(ty),
- rd: writable_spilltmp_reg2(),
- addr: p,
- src: zero_reg(),
- amo: AMO::SeqCst,
- }
- .emit(sink, emit_info, state);
AtomicOP::merge(
writable_spilltmp_reg2(),
writable_spilltmp_reg(),
@@ -1870,6 +1866,8 @@ impl Inst {
}
crate::ir::AtomicRmwOp::Nand => {
if ty.bits() < 32 {
+ Inst::gen_move(writable_spilltmp_reg2(), dst.to_reg(), I64)
+ .emit(sink, emit_info, state);
AtomicOP::extract(dst, offset, dst.to_reg(), ty)
.iter()
.for_each(|i| i.emit(sink, emit_info, state));
@@ -1883,14 +1881,6 @@ impl Inst {
.emit(sink, emit_info, state);
Inst::construct_bit_not(t0, t0.to_reg()).emit(sink, emit_info, state);
if ty.bits() < 32 {
- Inst::Atomic {
- op: AtomicOP::load_op(ty),
- rd: writable_spilltmp_reg2(),
- addr: p,
- src: zero_reg(),
- amo: AMO::SeqCst,
- }
- .emit(sink, emit_info, state);
AtomicOP::merge(
writable_spilltmp_reg2(),
writable_spilltmp_reg(),
@@ -1912,6 +1902,8 @@ impl Inst {
| crate::ir::AtomicRmwOp::Smax => {
let label_select_dst = sink.get_label();
let label_select_done = sink.get_label();
+ Inst::gen_move(writable_spilltmp_reg2(), dst.to_reg(), I64)
+ .emit(sink, emit_info, state);
if op == crate::ir::AtomicRmwOp::Umin || op == crate::ir::AtomicRmwOp::Umax
{
AtomicOP::extract(dst, offset, dst.to_reg(), ty)
@@ -1943,14 +1935,6 @@ impl Inst {
sink.bind_label(label_select_dst, &mut state.ctrl_plane);
Inst::gen_move(t0, dst.to_reg(), I64).emit(sink, emit_info, state);
sink.bind_label(label_select_done, &mut state.ctrl_plane);
- Inst::Atomic {
- op: AtomicOP::load_op(ty),
- rd: writable_spilltmp_reg2(),
- addr: p,
- src: zero_reg(),
- amo: AMO::SeqCst,
- }
- .emit(sink, emit_info, state);
AtomicOP::merge(
writable_spilltmp_reg2(),
writable_spilltmp_reg(),
@@ -1963,17 +1947,11 @@ impl Inst {
spilltmp_reg2()
}
crate::ir::AtomicRmwOp::Xchg => {
+ Inst::gen_move(writable_spilltmp_reg2(), dst.to_reg(), I64)
+ .emit(sink, emit_info, state);
AtomicOP::extract(dst, offset, dst.to_reg(), ty)
.iter()
.for_each(|i| i.emit(sink, emit_info, state));
- Inst::Atomic {
- op: AtomicOP::load_op(ty),
- rd: writable_spilltmp_reg2(),
- addr: p,
- src: zero_reg(),
- amo: AMO::SeqCst,
- }
- .emit(sink, emit_info, state);
AtomicOP::merge(
writable_spilltmp_reg2(),
writable_spilltmp_reg(),
diff --git a/cranelift/filetests/filetests/isa/riscv64/issue-13076.clif b/cranelift/filetests/filetests/isa/riscv64/issue-13076.clif
new file mode 100644
index 0000000000..d405748c01
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/issue-13076.clif
@@ -0,0 +1,194 @@
+test compile precise-output
+set unwind_info=false
+target riscv64
+
+function %atomic_rmw_add_i8(i64, i8) {
+block0(v0: i64, v1: i8):
+ v2 = atomic_rmw.i8 add v0, v1
+ return
+}
+
+; VCode:
+; block0:
+; andi a2,a0,3
+; slli a2,a2,3
+; andi a0,a0,-4
+; atomic_rmw.i8 add a4,a1,(a0)##t0=a3 offset=a2
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; andi a2, a0, 3
+; slli a2, a2, 3
+; andi a0, a0, -4
+; lr.w.aqrl a4, (a0) ; trap: heap_oob
+; mv t5, a4
+; srl a4, a4, a2
+; andi a4, a4, 0xff
+; add a3, a4, a1
+; addi t6, zero, 0xff
+; sll t6, t6, a2
+; not t6, t6
+; and t5, t5, t6
+; andi t6, a3, 0xff
+; sll t6, t6, a2
+; or t5, t5, t6
+; sc.w.aqrl a3, t5, (a0) ; trap: heap_oob
+; bnez a3, -0x34
+; ret
+
+function %atomic_rmw_sub_i16(i64, i16) {
+block0(v0: i64, v1: i16):
+ v2 = atomic_rmw.i16 sub v0, v1
+ return
+}
+
+; VCode:
+; block0:
+; andi a2,a0,3
+; slli a2,a2,3
+; andi a0,a0,-4
+; atomic_rmw.i16 sub a4,a1,(a0)##t0=a3 offset=a2
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; andi a2, a0, 3
+; slli a2, a2, 3
+; andi a0, a0, -4
+; lr.w.aqrl a4, (a0) ; trap: heap_oob
+; mv t5, a4
+; srl a4, a4, a2
+; slli a4, a4, 0x30
+; srli a4, a4, 0x30
+; sub a3, a4, a1
+; addi t6, zero, -1
+; slli t6, t6, 0x30
+; srli t6, t6, 0x30
+; sll t6, t6, a2
+; not t6, t6
+; and t5, t5, t6
+; slli t6, a3, 0x30
+; srli t6, t6, 0x30
+; sll t6, t6, a2
+; or t5, t5, t6
+; sc.w.aqrl a3, t5, (a0) ; trap: heap_oob
+; bnez a3, -0x44
+; ret
+
+function %atomic_rmw_xchg_i8(i64, i8) {
+block0(v0: i64, v1: i8):
+ v2 = atomic_rmw.i8 xchg v0, v1
+ return
+}
+
+; VCode:
+; block0:
+; andi a2,a0,3
+; slli a2,a2,3
+; andi a0,a0,-4
+; atomic_rmw.i8 xchg a4,a1,(a0)##t0=a3 offset=a2
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; andi a2, a0, 3
+; slli a2, a2, 3
+; andi a0, a0, -4
+; lr.w.aqrl a4, (a0) ; trap: heap_oob
+; mv t5, a4
+; srl a4, a4, a2
+; andi a4, a4, 0xff
+; addi t6, zero, 0xff
+; sll t6, t6, a2
+; not t6, t6
+; and t5, t5, t6
+; andi t6, a1, 0xff
+; sll t6, t6, a2
+; or t5, t5, t6
+; sc.w.aqrl a3, t5, (a0) ; trap: heap_oob
+; bnez a3, -0x30
+; ret
+
+function %atomic_rmw_nand_i8(i64, i8) {
+block0(v0: i64, v1: i8):
+ v2 = atomic_rmw.i8 nand v0, v1
+ return
+}
+
+; VCode:
+; block0:
+; andi a2,a0,3
+; slli a2,a2,3
+; andi a0,a0,-4
+; atomic_rmw.i8 nand a4,a1,(a0)##t0=a3 offset=a2
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; andi a2, a0, 3
+; slli a2, a2, 3
+; andi a0, a0, -4
+; lr.w.aqrl a4, (a0) ; trap: heap_oob
+; mv t5, a4
+; srl a4, a4, a2
+; andi a4, a4, 0xff
+; and a3, a1, a4
+; not a3, a3
+; addi t6, zero, 0xff
+; sll t6, t6, a2
+; not t6, t6
+; and t5, t5, t6
+; andi t6, a3, 0xff
+; sll t6, t6, a2
+; or t5, t5, t6
+; sc.w.aqrl a3, t5, (a0) ; trap: heap_oob
+; bnez a3, -0x38
+; ret
+
+function %atomic_rmw_umin_i16(i64, i16) {
+block0(v0: i64, v1: i16):
+ v2 = atomic_rmw.i16 umin v0, v1
+ return
+}
+
+; VCode:
+; block0:
+; slli a1,a1,48
+; srli a1,a1,48
+; andi a2,a0,3
+; slli a2,a2,3
+; andi a3,a0,-4
+; atomic_rmw.i16 umin a0,a1,(a3)##t0=a4 offset=a2
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; slli a1, a1, 0x30
+; srli a1, a1, 0x30
+; andi a2, a0, 3
+; slli a2, a2, 3
+; andi a3, a0, -4
+; lr.w.aqrl a0, (a3) ; trap: heap_oob
+; mv t5, a0
+; srl a0, a0, a2
+; slli a0, a0, 0x30
+; srli a0, a0, 0x30
+; bltu a0, a1, 0xc
+; mv a4, a1
+; j 8
+; mv a4, a0
+; addi t6, zero, -1
+; slli t6, t6, 0x30
+; srli t6, t6, 0x30
+; sll t6, t6, a2
+; not t6, t6
+; and t5, t5, t6
+; slli t6, a4, 0x30
+; srli t6, t6, 0x30
+; sll t6, t6, a2
+; or t5, t5, t6
+; sc.w.aqrl a4, t5, (a3) ; trap: heap_oob
+; bnez a4, -0x50
+; ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/issue-6954.clif b/cranelift/filetests/filetests/isa/riscv64/issue-6954.clif
index 902bf9fe8d..7e1f19baec 100644
--- a/cranelift/filetests/filetests/isa/riscv64/issue-6954.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/issue-6954.clif
@@ -384,10 +384,10 @@ block0(v0: i16, v1: f32, v2: f64x2, v3: i32, v4: i8, v5: i64x2, v6: i8, v7: f32x
; slli a2, a2, 3
; andi a3, a0, -4
; lr.w.aqrl a0, (a3) ; trap: heap_oob
+; mv t5, a0
; srl a0, a0, a2
; andi a0, a0, 0xff
; and a4, a0, a6
-; lr.w.aqrl t5, (a3) ; trap: heap_oob
; addi t6, zero, 0xff
; sll t6, t6, a2
; not t6, t6