Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

x64: Fix possible overflow in `Amode::offset` (#12949)

Alex Crichton2a50190f9457
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isle
index 1e42f0ffc9..1140773e81 100644
--- a/cranelift/codegen/src/isa/x64/inst.isle
+++ b/cranelift/codegen/src/isa/x64/inst.isle
@@ -236,20 +236,15 @@
;;
;; For `AtomicRmwOp::Xchg`, use `Atomic128XchgSeq` instead.
;;
+ ;; This requires that `mem_high` addresses 8 bytes beyond `mem_low`.
+ ;;
;; This instruction sequence has fixed register uses as follows:
- ;; - %rax (low), %rdx (high) (written) the old value at `mem`
+ ;; - %rax (low), %rdx (high) (written) the old value at `mem_low`
;; - %rbx (low), %rcx (high) (written) used as temp registers to hold
;; the replacement value
;; - %rflags is written. Do not assume anything about it after the
;; instruction.
- (Atomic128RmwSeq (op Atomic128RmwSeqOp)
- (mem BoxSyntheticAmode)
- (operand_low Gpr)
- (operand_high Gpr)
- (temp_low WritableGpr)
- (temp_high WritableGpr)
- (dst_old_low WritableGpr)
- (dst_old_high WritableGpr))
+ (Atomic128RmwSeq (args BoxAtomic128RmwSeqArgs))
;; A synthetic instruction, based on a loop around a native `lock
;; cmpxchg16b` instruction.
@@ -258,16 +253,14 @@
;; replacement value is the same every time, this instruction doesn't
;; require any temporary registers.
;;
+ ;; This requires that `mem_high` addresses 8 bytes beyond `mem_low`.
+ ;;
;; This instruction sequence has fixed register uses as follows:
- ;; - %rax (low), %rdx (high) (written) the old value at `mem`
+ ;; - %rax (low), %rdx (high) (written) the old value at `mem_low`
;; - %rbx (low), %rcx (high) (read) the replacement value
;; - %rflags is written. Do not assume anything about it after the
;; instruction.
- (Atomic128XchgSeq (mem SyntheticAmode)
- (operand_low Gpr)
- (operand_high Gpr)
- (dst_old_low WritableGpr)
- (dst_old_high WritableGpr))
+ (Atomic128XchgSeq (args BoxAtomic128XchgSeqArgs))
;; =========================================
;; Meta-instructions generating no code.
@@ -338,11 +331,8 @@
(type BoxCallIndInfo extern (enum))
(type BoxReturnCallInfo extern (enum))
(type BoxReturnCallIndInfo extern (enum))
-(type BoxSyntheticAmode extern (enum))
-
-(decl pure box_synthetic_amode (SyntheticAmode) BoxSyntheticAmode)
-(extern constructor box_synthetic_amode box_synthetic_amode)
-(convert SyntheticAmode BoxSyntheticAmode box_synthetic_amode)
+(type BoxAtomic128RmwSeqArgs extern (enum))
+(type BoxAtomic128XchgSeqArgs extern (enum))
;; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits.
(decl operand_size_of_type_32_64 (Type) OperandSize)
@@ -352,6 +342,16 @@
(decl raw_operand_size_of_type (Type) OperandSize)
(extern constructor raw_operand_size_of_type raw_operand_size_of_type)
+(decl atomic128_rmw_seq_args
+ (Atomic128RmwSeqOp SyntheticAmode SyntheticAmode Gpr Gpr WritableGpr WritableGpr WritableGpr WritableGpr)
+ BoxAtomic128RmwSeqArgs)
+(extern constructor atomic128_rmw_seq_args atomic128_rmw_seq_args)
+
+(decl atomic128_xchg_seq_args
+ (SyntheticAmode SyntheticAmode Gpr Gpr WritableGpr WritableGpr)
+ BoxAtomic128XchgSeqArgs)
+(extern constructor atomic128_xchg_seq_args atomic128_xchg_seq_args)
+
;; Get the bit width of an `OperandSize`.
(decl operand_size_bits (OperandSize) u16)
(rule (operand_size_bits (OperandSize.Size8)) 8)
@@ -588,8 +588,19 @@
;; Offsetting an Amode. Used when we need to do consecutive
;; loads/stores to adjacent addresses.
-(decl amode_offset (SyntheticAmode i32) SyntheticAmode)
-(extern constructor amode_offset amode_offset)
+;;
+;; Note that the input `Amode` might have a static offset that's too large to
+;; add the specified `i32` to so in the case of overflow a new `Amode` is
+;; generated entirely using `lea`
+(decl amode_offset (SyntheticAmode MemFlags i32) SyntheticAmode)
+(rule 1 (amode_offset base _flags offset)
+ (if-let new (amode_try_offset base offset))
+ new)
+(rule 0 (amode_offset base flags offset)
+ (Amode.ImmReg offset (x64_lea $I64 base) flags))
+
+(decl pure partial amode_try_offset (SyntheticAmode i32) SyntheticAmode)
+(extern constructor amode_try_offset amode_try_offset)
;; Return a zero offset as an `Offset32`.
(spec (zero_offset) (provide (= result #x00000000)))
@@ -3693,32 +3704,38 @@
(_ Unit (emit (MInst.AtomicRmwSeq ty op mem input tmp dst))))
dst))
-(decl x64_atomic_128_rmw_seq (AtomicRmwOp SyntheticAmode ValueRegs) ValueRegs)
-(rule (x64_atomic_128_rmw_seq op mem input)
+(decl x64_atomic_128_rmw_seq (AtomicRmwOp SyntheticAmode MemFlags ValueRegs) ValueRegs)
+(rule (x64_atomic_128_rmw_seq op mem_low flags input)
(let ((dst_low WritableGpr (temp_writable_gpr))
(dst_high WritableGpr (temp_writable_gpr))
(tmp_low WritableGpr (temp_writable_gpr))
(tmp_high WritableGpr (temp_writable_gpr))
(input_low Gpr (value_regs_get_gpr input 0))
(input_high Gpr (value_regs_get_gpr input 1))
- (_ Unit (emit (MInst.Atomic128RmwSeq (atomic_128_rmw_seq_op op) mem input_low input_high tmp_low tmp_high dst_low dst_high))))
+ (mem_high SyntheticAmode (amode_offset mem_low flags 8))
+ (args BoxAtomic128RmwSeqArgs (atomic128_rmw_seq_args (atomic_128_rmw_seq_op op) mem_low mem_high input_low input_high tmp_low tmp_high dst_low dst_high))
+ (_ Unit (emit (MInst.Atomic128RmwSeq args))))
(value_regs dst_low dst_high)))
-(rule 1 (x64_atomic_128_rmw_seq (AtomicRmwOp.Xchg) mem input)
+(rule 1 (x64_atomic_128_rmw_seq (AtomicRmwOp.Xchg) mem_low flags input)
(let ((dst_low WritableGpr (temp_writable_gpr))
(dst_high WritableGpr (temp_writable_gpr))
(input_low Gpr (value_regs_get_gpr input 0))
(input_high Gpr (value_regs_get_gpr input 1))
- (_ Unit (emit (MInst.Atomic128XchgSeq mem input_low input_high dst_low dst_high))))
+ (mem_high SyntheticAmode (amode_offset mem_low flags 8))
+ (args BoxAtomic128XchgSeqArgs (atomic128_xchg_seq_args mem_low mem_high input_low input_high dst_low dst_high))
+ (_ Unit (emit (MInst.Atomic128XchgSeq args))))
(value_regs dst_low dst_high)))
-(decl x64_atomic_128_store_seq (SyntheticAmode ValueRegs) SideEffectNoResult)
-(rule (x64_atomic_128_store_seq mem input)
+(decl x64_atomic_128_store_seq (SyntheticAmode MemFlags ValueRegs) SideEffectNoResult)
+(rule (x64_atomic_128_store_seq mem_low flags input)
(let ((dst_low WritableGpr (temp_writable_gpr))
(dst_high WritableGpr (temp_writable_gpr))
(input_low Gpr (value_regs_get_gpr input 0))
- (input_high Gpr (value_regs_get_gpr input 1)))
- (SideEffectNoResult.Inst (MInst.Atomic128XchgSeq mem input_low input_high dst_low dst_high))))
+ (input_high Gpr (value_regs_get_gpr input 1))
+ (mem_high SyntheticAmode (amode_offset mem_low flags 8))
+ (args BoxAtomic128XchgSeqArgs (atomic128_xchg_seq_args mem_low mem_high input_low input_high dst_low dst_high)))
+ (SideEffectNoResult.Inst (MInst.Atomic128XchgSeq args))))
(type AtomicRmwSeqOp
diff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rs
index 233299966a..7e3b7394b9 100644
--- a/cranelift/codegen/src/isa/x64/inst/args.rs
+++ b/cranelift/codegen/src/isa/x64/inst/args.rs
@@ -439,14 +439,14 @@ impl Amode {
}
/// Offset the amode by a fixed offset.
- pub(crate) fn offset(&self, offset: i32) -> Self {
+ pub(crate) fn offset(&self, offset: i32) -> Option<Self> {
let mut ret = self.clone();
- match &mut ret {
- &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
- &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
+ let simm32 = match &mut ret {
+ Amode::ImmReg { simm32, .. } | Amode::ImmRegRegShift { simm32, .. } => simm32,
_ => panic!("Cannot offset amode: {self:?}"),
- }
- ret
+ };
+ *simm32 = simm32.checked_add(offset)?;
+ Some(ret)
}
pub(crate) fn aligned(&self) -> bool {
@@ -575,18 +575,18 @@ impl SyntheticAmode {
}
/// Offset the synthetic amode by a fixed offset.
- pub(crate) fn offset(&self, offset: i32) -> Self {
+ pub(crate) fn offset(&self, offset: i32) -> Option<Self> {
let mut ret = self.clone();
match &mut ret {
- SyntheticAmode::Real(amode) => *amode = amode.offset(offset),
- SyntheticAmode::SlotOffset { simm32 } => *simm32 += offset,
+ SyntheticAmode::Real(amode) => *amode = amode.offset(offset)?,
+ SyntheticAmode::SlotOffset { simm32 } => *simm32 = simm32.checked_add(offset)?,
// `amode_offset` is used only in i128.load/store which
// takes a synthetic amode from `to_amode`; `to_amode` can
// only produce Real or SlotOffset amodes, never
// IncomingArg or ConstantOffset.
_ => panic!("Cannot offset SyntheticAmode: {self:?}"),
}
- ret
+ Some(ret)
}
}
@@ -1061,3 +1061,34 @@ impl OperandSize {
self.to_bytes() * 8
}
}
+
+pub use crate::isa::x64::lower::isle::generated_code::Atomic128RmwSeqOp;
+
+/// "Package" of the arguments for the instruction `Atomic128RmwSeq` to avoid
+/// making the `Inst` enum massive.
+#[derive(Debug, Clone)]
+#[expect(missing_docs, reason = "self-describing fields")]
+pub struct Atomic128RmwSeqArgs {
+ pub op: Atomic128RmwSeqOp,
+ pub mem_low: SyntheticAmode,
+ pub mem_high: SyntheticAmode,
+ pub operand_low: Gpr,
+ pub operand_high: Gpr,
+ pub temp_low: WritableGpr,
+ pub temp_high: WritableGpr,
+ pub dst_old_low: WritableGpr,
+ pub dst_old_high: WritableGpr,
+}
+
+/// "Package" of the arguments for the instruction `Atomic128XchgSeq` to avoid
+/// making the `Inst` enum massive.
+#[derive(Debug, Clone)]
+#[expect(missing_docs, reason = "self-describing fields")]
+pub struct Atomic128XchgSeqArgs {
+ pub mem_low: SyntheticAmode,
+ pub mem_high: SyntheticAmode,
+ pub operand_low: Gpr,
+ pub operand_high: Gpr,
+ pub dst_old_low: WritableGpr,
+ pub dst_old_high: WritableGpr,
+}
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index de0fecb829..7f6f1aeb76 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1544,16 +1544,18 @@ pub(crate) fn emit(
one_way_jmp(sink, CC::NZ, again_label);
}
- Inst::Atomic128RmwSeq {
- op,
- mem,
- operand_low,
- operand_high,
- temp_low,
- temp_high,
- dst_old_low,
- dst_old_high,
- } => {
+ Inst::Atomic128RmwSeq { args } => {
+ let Atomic128RmwSeqArgs {
+ op,
+ mem_low,
+ mem_high,
+ operand_low,
+ operand_high,
+ temp_low,
+ temp_high,
+ dst_old_low,
+ dst_old_high,
+ } = &**args;
let operand_low = *operand_low;
let operand_high = *operand_high;
let temp_low = *temp_low;
@@ -1564,13 +1566,14 @@ pub(crate) fn emit(
debug_assert_eq!(temp_high.to_reg(), regs::rcx());
debug_assert_eq!(dst_old_low.to_reg(), regs::rax());
debug_assert_eq!(dst_old_high.to_reg(), regs::rdx());
- let mem = mem.finalize(state.frame_layout(), sink).clone();
+ let mem_low = mem_low.finalize(state.frame_layout(), sink).clone();
+ let mem_high = mem_high.finalize(state.frame_layout(), sink).clone();
let again_label = sink.get_label();
// Load the initial value.
- asm::inst::movq_rm::new(dst_old_low, mem.clone()).emit(sink, info, state);
- asm::inst::movq_rm::new(dst_old_high, mem.offset(8)).emit(sink, info, state);
+ asm::inst::movq_rm::new(dst_old_low, mem_low.clone()).emit(sink, info, state);
+ asm::inst::movq_rm::new(dst_old_high, mem_high).emit(sink, info, state);
// again:
sink.bind_label(again_label, state.ctrl_plane_mut());
@@ -1656,7 +1659,7 @@ pub(crate) fn emit(
PairedGpr::from(dst_old_high),
temp_low.to_reg(),
temp_high.to_reg(),
- mem,
+ mem_low,
)
.emit(sink, info, state);
@@ -1664,13 +1667,15 @@ pub(crate) fn emit(
one_way_jmp(sink, CC::NZ, again_label);
}
- Inst::Atomic128XchgSeq {
- mem,
- operand_low,
- operand_high,
- dst_old_low,
- dst_old_high,
- } => {
+ Inst::Atomic128XchgSeq { args } => {
+ let Atomic128XchgSeqArgs {
+ mem_low,
+ mem_high,
+ operand_low,
+ operand_high,
+ dst_old_low,
+ dst_old_high,
+ } = &**args;
let operand_low = *operand_low;
let operand_high = *operand_high;
let dst_old_low = *dst_old_low;
@@ -1679,13 +1684,14 @@ pub(crate) fn emit(
debug_assert_eq!(operand_high, regs::rcx());
debug_assert_eq!(dst_old_low.to_reg(), regs::rax());
debug_assert_eq!(dst_old_high.to_reg(), regs::rdx());
- let mem = mem.finalize(state.frame_layout(), sink).clone();
+ let mem_low = mem_low.finalize(state.frame_layout(), sink).clone();
+ let mem_high = mem_high.finalize(state.frame_layout(), sink).clone();
let again_label = sink.get_label();
// Load the initial value.
- asm::inst::movq_rm::new(dst_old_low, mem.clone()).emit(sink, info, state);
- asm::inst::movq_rm::new(dst_old_high, mem.offset(8)).emit(sink, info, state);
+ asm::inst::movq_rm::new(dst_old_low, mem_low.clone()).emit(sink, info, state);
+ asm::inst::movq_rm::new(dst_old_high, mem_high).emit(sink, info, state);
// again:
sink.bind_label(again_label, state.ctrl_plane_mut());
@@ -1696,7 +1702,7 @@ pub(crate) fn emit(
PairedGpr::from(dst_old_high),
operand_low,
operand_high,
- mem,
+ mem_low,
)
.emit(sink, info, state);
diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
index 1bd8b0f6b2..77d53ba384 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs
@@ -244,72 +244,70 @@ fn test_x64_emit() {
));
// Atomic128RmwSeq
+ let rmw_args = Atomic128RmwSeqArgs {
+ op: Atomic128RmwSeqOp::Or,
+ mem_low: am3.clone(),
+ mem_high: am3.offset(8).unwrap(),
+ operand_low: Gpr::unwrap_new(r10),
+ operand_high: Gpr::unwrap_new(r11),
+ temp_low: w_rbx.map(Gpr::unwrap_new),
+ temp_high: w_rcx.map(Gpr::unwrap_new),
+ dst_old_low: w_rax.map(Gpr::unwrap_new),
+ dst_old_high: w_rdx.map(Gpr::unwrap_new),
+ };
insns.push((
Inst::Atomic128RmwSeq {
- op: Atomic128RmwSeqOp::Or,
- mem: Box::new(am3.clone()),
- operand_low: Gpr::unwrap_new(r10),
- operand_high: Gpr::unwrap_new(r11),
- temp_low: w_rbx.map(Gpr::unwrap_new),
- temp_high: w_rcx.map(Gpr::unwrap_new),
- dst_old_low: w_rax.map(Gpr::unwrap_new),
- dst_old_high: w_rdx.map(Gpr::unwrap_new),
+ args: Box::new(Atomic128RmwSeqArgs {
+ op: Atomic128RmwSeqOp::Or,
+ ..rmw_args.clone()
+ }),
},
"498B01498B51084889C34889D1490BDA490BCBF0490FC7090F85E9FFFFFF",
- "atomically { %rdx:%rax = 0(%r9); %rcx:%rbx = %rdx:%rax Or %r11:%r10; 0(%r9) = %rcx:%rbx }",
+ "atomically { %rdx:%rax = 0(%r9):8(%r9); %rcx:%rbx = %rdx:%rax Or %r11:%r10; 0(%r9):8(%r9) = %rcx:%rbx }"
));
insns.push((
Inst::Atomic128RmwSeq {
- op: Atomic128RmwSeqOp::And,
- mem: Box::new(am3.clone()),
- operand_low: Gpr::unwrap_new(r10),
- operand_high: Gpr::unwrap_new(r11),
- temp_low: w_rbx.map(Gpr::unwrap_new),
- temp_high: w_rcx.map(Gpr::unwrap_new),
- dst_old_low: w_rax.map(Gpr::unwrap_new),
- dst_old_high: w_rdx.map(Gpr::unwrap_new),
+ args: Box::new(Atomic128RmwSeqArgs {
+ op: Atomic128RmwSeqOp::And,
+ ..rmw_args.clone()
+ }),
},
"498B01498B51084889C34889D14923DA4923CBF0490FC7090F85E9FFFFFF",
- "atomically { %rdx:%rax = 0(%r9); %rcx:%rbx = %rdx:%rax And %r11:%r10; 0(%r9) = %rcx:%rbx }"
+ "atomically { %rdx:%rax = 0(%r9):8(%r9); %rcx:%rbx = %rdx:%rax And %r11:%r10; 0(%r9):8(%r9) = %rcx:%rbx }"
));
insns.push((
Inst::Atomic128RmwSeq {
- op: Atomic128RmwSeqOp::Umin,
- mem: Box::new(am3.clone()),
- operand_low: Gpr::unwrap_new(r10),
- operand_high: Gpr::unwrap_new(r11),
- temp_low: w_rbx.map(Gpr::unwrap_new),
- temp_high: w_rcx.map(Gpr::unwrap_new),
- dst_old_low: w_rax.map(Gpr::unwrap_new),
- dst_old_high: w_rdx.map(Gpr::unwrap_new),
+ args: Box::new(Atomic128RmwSeqArgs {
+ op: Atomic128RmwSeqOp::Umin,
+ ..rmw_args.clone()
+ }),
},
"498B01498B51084889C34889D14C39D3491BCB4889D1490F43DA490F43CBF0490FC7090F85DEFFFFFF",
- "atomically { %rdx:%rax = 0(%r9); %rcx:%rbx = %rdx:%rax Umin %r11:%r10; 0(%r9) = %rcx:%rbx }"
+ "atomically { %rdx:%rax = 0(%r9):8(%r9); %rcx:%rbx = %rdx:%rax Umin %r11:%r10; 0(%r9):8(%r9) = %rcx:%rbx }"
));
insns.push((
Inst::Atomic128RmwSeq {
- op: Atomic128RmwSeqOp::Add,
- mem: Box::new(am3.clone()),
- operand_low: Gpr::unwrap_new(r10),
- operand_high: Gpr::unwrap_new(r11),
- temp_low: w_rbx.map(Gpr::unwrap_new),
- temp_high: w_rcx.map(Gpr::unwrap_new),
- dst_old_low: w_rax.map(Gpr::unwrap_new),
- dst_old_high: w_rdx.map(Gpr::unwrap_new),
+ args: Box::new(Atomic128RmwSeqArgs {
+ op: Atomic128RmwSeqOp::Add,
+ ..rmw_args.clone()
+ }),
},
"498B01498B51084889C34889D14903DA4913CBF0490FC7090F85E9FFFFFF",
- "atomically { %rdx:%rax = 0(%r9); %rcx:%rbx = %rdx:%rax Add %r11:%r10; 0(%r9) = %rcx:%rbx }"
+ "atomically { %rdx:%rax = 0(%r9):8(%r9); %rcx:%rbx = %rdx:%rax Add %r11:%r10; 0(%r9):8(%r9) = %rcx:%rbx }",
));
insns.push((
Inst::Atomic128XchgSeq {
- mem: am3.clone(),
- operand_low: Gpr::unwrap_new(rbx),
- operand_high: Gpr::unwrap_new(rcx),
- dst_old_low: w_rax.map(Gpr::unwrap_new),
- dst_old_high: w_rdx.map(Gpr::unwrap_new),
+ args: Box::new(Atomic128XchgSeqArgs {
+ mem_low: am3.clone(),
+ mem_high: am3.offset(8).unwrap(),
+ operand_low: Gpr::unwrap_new(rbx),
+ operand_high: Gpr::unwrap_new(rcx),
+ dst_old_low: w_rax.map(Gpr::unwrap_new),
+ dst_old_high: w_rdx.map(Gpr::unwrap_new),
+ }),
},
"498B01498B5108F0490FC7090F85F5FFFFFF",
- "atomically { %rdx:%rax = 0(%r9); 0(%r9) = %rcx:%rbx }",
+ "atomically { %rdx:%rax = 0(%r9):8(%r9); 0(%r9):8(%r9) = %rcx:%rbx }",
));
// ========================================================
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index c5759979bb..3dc1e4614c 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -757,42 +757,48 @@ impl PrettyPrint for Inst {
)
}
- Inst::Atomic128RmwSeq {
- op,
- mem,
- operand_low,
- operand_high,
- temp_low,
- temp_high,
- dst_old_low,
- dst_old_high,
- } => {
+ Inst::Atomic128RmwSeq { args } => {
+ let Atomic128RmwSeqArgs {
+ op,
+ mem_low,
+ mem_high,
+ operand_low,
+ operand_high,
+ temp_low,
+ temp_high,
+ dst_old_low,
+ dst_old_high,
+ } = &**args;
let operand_low = pretty_print_reg(**operand_low, 8);
let operand_high = pretty_print_reg(**operand_high, 8);
let temp_low = pretty_print_reg(*temp_low.to_reg(), 8);
let temp_high = pretty_print_reg(*temp_high.to_reg(), 8);
let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);
let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);
- let mem = mem.pretty_print(16);
+ let mem_low = mem_low.pretty_print(16);
+ let mem_high = mem_high.pretty_print(16);
format!(
- "atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem} = {temp_high}:{temp_low} }}"
+ "atomically {{ {dst_old_high}:{dst_old_low} = {mem_low}:{mem_high}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem_low}:{mem_high} = {temp_high}:{temp_low} }}"
)
}
- Inst::Atomic128XchgSeq {
- mem,
- operand_low,
- operand_high,
- dst_old_low,
- dst_old_high,
- } => {
+ Inst::Atomic128XchgSeq { args } => {
+ let Atomic128XchgSeqArgs {
+ mem_low,
+ mem_high,
+ operand_low,
+ operand_high,
+ dst_old_low,
+ dst_old_high,
+ } = &**args;
let operand_low = pretty_print_reg(**operand_low, 8);
let operand_high = pretty_print_reg(**operand_high, 8);
let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);
let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);
- let mem = mem.pretty_print(16);
+ let mem_low = mem_low.pretty_print(16);
+ let mem_high = mem_high.pretty_print(16);
format!(
- "atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {mem} = {operand_high}:{operand_low} }}"
+ "atomically {{ {dst_old_high}:{dst_old_low} = {mem_low}:{mem_high}; {mem_low}:{mem_high} = {operand_high}:{operand_low} }}"
)
}
@@ -1105,16 +1111,18 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
mem.get_operands_late(collector)
}
- Inst::Atomic128RmwSeq {
- operand_low,
- operand_high,
- temp_low,
- temp_high,
- dst_old_low,
- dst_old_high,
- mem,
- ..
- } => {
+ Inst::Atomic128RmwSeq { args } => {
+ let Atomic128RmwSeqArgs {
+ mem_low,
+ mem_high,
+ operand_low,
+ operand_high,
+ temp_low,
+ temp_high,
+ dst_old_low,
+ dst_old_high,
+ op: _,
+ } = &mut **args;
// All registers are collected in the `Late` position so that they don't overlap.
collector.reg_late_use(operand_low);
collector.reg_late_use(operand_high);
@@ -1122,23 +1130,26 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_fixed_def(temp_high, regs::rcx());
collector.reg_fixed_def(dst_old_low, regs::rax());
collector.reg_fixed_def(dst_old_high, regs::rdx());
- mem.get_operands_late(collector)
+ mem_low.get_operands_late(collector);
+ mem_high.get_operands_late(collector);
}
- Inst::Atomic128XchgSeq {
- operand_low,
- operand_high,
- dst_old_low,
- dst_old_high,
- mem,
- ..
- } => {
+ Inst::Atomic128XchgSeq { args } => {
+ let Atomic128XchgSeqArgs {
+ mem_low,
+ mem_high,
+ operand_low,
+ operand_high,
+ dst_old_low,
+ dst_old_high,
+ } = &mut **args;
// All registers are collected in the `Late` position so that they don't overlap.
collector.reg_fixed_late_use(operand_low, regs::rbx());
collector.reg_fixed_late_use(operand_high, regs::rcx());
collector.reg_fixed_def(dst_old_low, regs::rax());
collector.reg_fixed_def(dst_old_high, regs::rdx());
- mem.get_operands_late(collector)
+ mem_low.get_operands_late(collector);
+ mem_high.get_operands_late(collector);
}
Inst::Args { args } => {
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index e4fc7a8218..9819ef8c87 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -3094,7 +3094,7 @@
(rule -3 (lower (has_type $I128
(load _ (little_or_native_endian flags) address offset)))
(let ((addr_lo SyntheticAmode (to_amode flags address offset))
- (addr_hi SyntheticAmode (amode_offset addr_lo 8))
+ (addr_hi SyntheticAmode (amode_offset addr_lo flags 8))
(value_lo Reg (x64_mov addr_lo))
(value_hi Reg (x64_mov addr_hi)))
(value_regs value_lo value_hi)))
@@ -3225,7 +3225,7 @@
(value_lo Gpr (value_regs_get_gpr value_reg 0))
(value_hi Gpr (value_regs_get_gpr value_reg 1))
(addr_lo SyntheticAmode (to_amode flags address offset))
- (addr_hi SyntheticAmode (amode_offset addr_lo 8)))
+ (addr_hi SyntheticAmode (amode_offset addr_lo flags 8)))
(side_effect
(side_effect_concat
(x64_movrm $I64 addr_lo value_lo)
@@ -3450,7 +3450,7 @@
;; Lower 128-bit `atomic_store` using `cmpxchg16b`.
(rule 1 (lower (atomic_store (little_or_native_endian flags) value @ (value_type $I128) address))
(if-let true (has_cmpxchg16b))
- (side_effect (x64_atomic_128_store_seq (to_amode flags address (zero_offset)) value)))
+ (side_effect (x64_atomic_128_store_seq (to_amode flags address (zero_offset)) flags value)))
;; Rules for `atomic_cas` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -3512,7 +3512,7 @@
;; 128-bit integers always use a `lock cmpxchg16b` loop.
(rule 3 (lower (has_type $I128 (atomic_rmw _ (little_or_native_endian flags) op address input)))
(if-let true (has_cmpxchg16b))
- (x64_atomic_128_rmw_seq op (to_amode flags address (zero_offset)) input))
+ (x64_atomic_128_rmw_seq op (to_amode flags address (zero_offset)) flags input))
;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rs
index 57bfa06ee7..60d50df4ea 100644
--- a/cranelift/codegen/src/isa/x64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs
@@ -36,7 +36,8 @@ type BoxCallIndInfo = Box<CallInfo<RegMem>>;
type BoxReturnCallInfo = Box<ReturnCallInfo<ExternalName>>;
type BoxReturnCallIndInfo = Box<ReturnCallInfo<Reg>>;
type VecArgPair = Vec<ArgPair>;
-type BoxSyntheticAmode = Box<SyntheticAmode>;
+type BoxAtomic128RmwSeqArgs = Box<Atomic128RmwSeqArgs>;
+type BoxAtomic128XchgSeqArgs = Box<Atomic128XchgSeqArgs>;
/// When interacting with the external assembler (see `external.rs`), we
/// need to fix the types we'll use.
@@ -639,7 +640,7 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
}
#[inline]
- fn amode_offset(&mut self, addr: &SyntheticAmode, offset: i32) -> SyntheticAmode {
+ fn amode_try_offset(&mut self, addr: &SyntheticAmode, offset: i32) -> Option<SyntheticAmode> {
addr.offset(offset)
}
@@ -970,8 +971,48 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {
WritableGpr::from_reg(reg)
}
- fn box_synthetic_amode(&mut self, amode: &SyntheticAmode) -> BoxSyntheticAmode {
- Box::new(amode.clone())
+ fn atomic128_rmw_seq_args(
+ &mut self,
+ op: &Atomic128RmwSeqOp,
+ mem_low: &SyntheticAmode,
+ mem_high: &SyntheticAmode,
+ operand_low: Gpr,
+ operand_high: Gpr,
+ temp_low: WritableGpr,
+ temp_high: WritableGpr,
+ dst_old_low: WritableGpr,
+ dst_old_high: WritableGpr,
+ ) -> BoxAtomic128RmwSeqArgs {
+ Box::new(Atomic128RmwSeqArgs {
+ op: *op,
+ mem_low: mem_low.clone(),
+ mem_high: mem_high.clone(),
+ operand_low,
+ operand_high,
+ temp_low,
+ temp_high,
+ dst_old_low,
+ dst_old_high,
+ })
+ }
+
+ fn atomic128_xchg_seq_args(
+ &mut self,
+ mem_low: &SyntheticAmode,
+ mem_high: &SyntheticAmode,
+ operand_low: Gpr,
+ operand_high: Gpr,
+ dst_old_low: WritableGpr,
+ dst_old_high: WritableGpr,
+ ) -> BoxAtomic128XchgSeqArgs {
+ Box::new(Atomic128XchgSeqArgs {
+ mem_low: mem_low.clone(),
+ mem_high: mem_high.clone(),
+ operand_low,
+ operand_high,
+ dst_old_low,
+ dst_old_high,
+ })
}
////////////////////////////////////////////////////////////////////////////
diff --git a/cranelift/filetests/filetests/isa/x64/atomic-128.clif b/cranelift/filetests/filetests/isa/x64/atomic-128.clif
index fd5796f5b9..261e19b768 100644
--- a/cranelift/filetests/filetests/isa/x64/atomic-128.clif
+++ b/cranelift/filetests/filetests/isa/x64/atomic-128.clif
@@ -62,7 +62,7 @@ block0(v0: i128, v1: i64):
; movq %rsi, %rcx
; movq %rdi, %rbx
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%r8); 0(%r8) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%r8):8(%r8); 0(%r8):8(%r8) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -141,7 +141,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Add %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Add %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -183,7 +183,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Sub %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Sub %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -225,7 +225,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax And %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax And %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -267,7 +267,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Nand %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Nand %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -311,7 +311,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Or %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Or %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -353,7 +353,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Xor %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Xor %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -396,7 +396,7 @@ block0(v0: i64, v1: i128):
; block0:
; movq %rdx, %rcx
; movq %rsi, %rbx
-; atomically { %rdx:%rax = 0(%rdi); 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -435,7 +435,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Umin %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Umin %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -480,7 +480,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Umax %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Umax %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -525,7 +525,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Smin %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Smin %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
@@ -570,7 +570,7 @@ block0(v0: i64, v1: i128):
; movq %rbx, (%rsp)
; block0:
; movq %rdx, %r8
-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Smax %r8:%rsi; 0(%rdi) = %rcx:%rbx }
+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Smax %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }
; movq (%rsp), %rbx
; addq $0x10, %rsp
; movq %rbp, %rsp
diff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clif
index 51777c75c2..3b601d4d24 100644
--- a/cranelift/filetests/filetests/isa/x64/i128.clif
+++ b/cranelift/filetests/filetests/isa/x64/i128.clif
@@ -2225,3 +2225,33 @@ block0(v0: i64, v1: i64):
; popq %rbp
; retq
+function %load_huge_offset(i64) -> i64, i64 {
+block0(v0: i64):
+ v4 = load.i128 v0+0x7fffffff
+ v5, v6 = isplit v4
+ return v5, v6
+}
+
+; VCode:
+; pushq %rbp
+; movq %rsp, %rbp
+; block0:
+; leaq 0x7fffffff(%rdi), %rdx
+; movq 0x7fffffff(%rdi), %rax
+; movq 8(%rdx), %rdx
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+;
+; Disassembled:
+; block0: ; offset 0x0
+; pushq %rbp
+; movq %rsp, %rbp
+; block1: ; offset 0x4
+; leaq 0x7fffffff(%rdi), %rdx ; trap: heap_oob
+; movq 0x7fffffff(%rdi), %rax ; trap: heap_oob
+; movq 8(%rdx), %rdx ; trap: heap_oob
+; movq %rbp, %rsp
+; popq %rbp
+; retq
+