A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
x64: Fix possible overflow in `Amode::offset` (#12949)
Alex Crichton2a50190f9457
diff --git a/cranelift/codegen/src/isa/x64/inst.isle b/cranelift/codegen/src/isa/x64/inst.isleindex 1e42f0ffc9..1140773e81 100644--- a/cranelift/codegen/src/isa/x64/inst.isle+++ b/cranelift/codegen/src/isa/x64/inst.isle@@ -236,20 +236,15 @@;;;; For `AtomicRmwOp::Xchg`, use `Atomic128XchgSeq` instead.;;+ ;; This requires that `mem_high` addresses 8 bytes beyond `mem_low`.+ ;;;; This instruction sequence has fixed register uses as follows:- ;; - %rax (low), %rdx (high) (written) the old value at `mem`+ ;; - %rax (low), %rdx (high) (written) the old value at `mem_low`;; - %rbx (low), %rcx (high) (written) used as temp registers to hold;; the replacement value;; - %rflags is written. Do not assume anything about it after the;; instruction.- (Atomic128RmwSeq (op Atomic128RmwSeqOp)- (mem BoxSyntheticAmode)- (operand_low Gpr)- (operand_high Gpr)- (temp_low WritableGpr)- (temp_high WritableGpr)- (dst_old_low WritableGpr)- (dst_old_high WritableGpr))+ (Atomic128RmwSeq (args BoxAtomic128RmwSeqArgs));; A synthetic instruction, based on a loop around a native `lock;; cmpxchg16b` instruction.@@ -258,16 +253,14 @@;; replacement value is the same every time, this instruction doesn't;; require any temporary registers.;;+ ;; This requires that `mem_high` addresses 8 bytes beyond `mem_low`.+ ;;;; This instruction sequence has fixed register uses as follows:- ;; - %rax (low), %rdx (high) (written) the old value at `mem`+ ;; - %rax (low), %rdx (high) (written) the old value at `mem_low`;; - %rbx (low), %rcx (high) (read) the replacement value;; - %rflags is written. Do not assume anything about it after the;; instruction.- (Atomic128XchgSeq (mem SyntheticAmode)- (operand_low Gpr)- (operand_high Gpr)- (dst_old_low WritableGpr)- (dst_old_high WritableGpr))+ (Atomic128XchgSeq (args BoxAtomic128XchgSeqArgs));; =========================================;; Meta-instructions generating no code.@@ -338,11 +331,8 @@(type BoxCallIndInfo extern (enum))(type BoxReturnCallInfo extern (enum))(type BoxReturnCallIndInfo extern (enum))-(type BoxSyntheticAmode extern (enum))--(decl pure box_synthetic_amode (SyntheticAmode) BoxSyntheticAmode)-(extern constructor box_synthetic_amode box_synthetic_amode)-(convert SyntheticAmode BoxSyntheticAmode box_synthetic_amode)+(type BoxAtomic128RmwSeqArgs extern (enum))+(type BoxAtomic128XchgSeqArgs extern (enum));; Get the `OperandSize` for a given `Type`, rounding smaller types up to 32 bits.(decl operand_size_of_type_32_64 (Type) OperandSize)@@ -352,6 +342,16 @@(decl raw_operand_size_of_type (Type) OperandSize)(extern constructor raw_operand_size_of_type raw_operand_size_of_type)+(decl atomic128_rmw_seq_args+ (Atomic128RmwSeqOp SyntheticAmode SyntheticAmode Gpr Gpr WritableGpr WritableGpr WritableGpr WritableGpr)+ BoxAtomic128RmwSeqArgs)+(extern constructor atomic128_rmw_seq_args atomic128_rmw_seq_args)++(decl atomic128_xchg_seq_args+ (SyntheticAmode SyntheticAmode Gpr Gpr WritableGpr WritableGpr)+ BoxAtomic128XchgSeqArgs)+(extern constructor atomic128_xchg_seq_args atomic128_xchg_seq_args)+;; Get the bit width of an `OperandSize`.(decl operand_size_bits (OperandSize) u16)(rule (operand_size_bits (OperandSize.Size8)) 8)@@ -588,8 +588,19 @@;; Offsetting an Amode. Used when we need to do consecutive;; loads/stores to adjacent addresses.-(decl amode_offset (SyntheticAmode i32) SyntheticAmode)-(extern constructor amode_offset amode_offset)+;;+;; Note that the input `Amode` might have a static offset that's too large to+;; add the specified `i32` to so in the case of overflow a new `Amode` is+;; generated entirely using `lea`+(decl amode_offset (SyntheticAmode MemFlags i32) SyntheticAmode)+(rule 1 (amode_offset base _flags offset)+ (if-let new (amode_try_offset base offset))+ new)+(rule 0 (amode_offset base flags offset)+ (Amode.ImmReg offset (x64_lea $I64 base) flags))++(decl pure partial amode_try_offset (SyntheticAmode i32) SyntheticAmode)+(extern constructor amode_try_offset amode_try_offset);; Return a zero offset as an `Offset32`.(spec (zero_offset) (provide (= result #x00000000)))@@ -3693,32 +3704,38 @@(_ Unit (emit (MInst.AtomicRmwSeq ty op mem input tmp dst))))dst))-(decl x64_atomic_128_rmw_seq (AtomicRmwOp SyntheticAmode ValueRegs) ValueRegs)-(rule (x64_atomic_128_rmw_seq op mem input)+(decl x64_atomic_128_rmw_seq (AtomicRmwOp SyntheticAmode MemFlags ValueRegs) ValueRegs)+(rule (x64_atomic_128_rmw_seq op mem_low flags input)(let ((dst_low WritableGpr (temp_writable_gpr))(dst_high WritableGpr (temp_writable_gpr))(tmp_low WritableGpr (temp_writable_gpr))(tmp_high WritableGpr (temp_writable_gpr))(input_low Gpr (value_regs_get_gpr input 0))(input_high Gpr (value_regs_get_gpr input 1))- (_ Unit (emit (MInst.Atomic128RmwSeq (atomic_128_rmw_seq_op op) mem input_low input_high tmp_low tmp_high dst_low dst_high))))+ (mem_high SyntheticAmode (amode_offset mem_low flags 8))+ (args BoxAtomic128RmwSeqArgs (atomic128_rmw_seq_args (atomic_128_rmw_seq_op op) mem_low mem_high input_low input_high tmp_low tmp_high dst_low dst_high))+ (_ Unit (emit (MInst.Atomic128RmwSeq args))))(value_regs dst_low dst_high)))-(rule 1 (x64_atomic_128_rmw_seq (AtomicRmwOp.Xchg) mem input)+(rule 1 (x64_atomic_128_rmw_seq (AtomicRmwOp.Xchg) mem_low flags input)(let ((dst_low WritableGpr (temp_writable_gpr))(dst_high WritableGpr (temp_writable_gpr))(input_low Gpr (value_regs_get_gpr input 0))(input_high Gpr (value_regs_get_gpr input 1))- (_ Unit (emit (MInst.Atomic128XchgSeq mem input_low input_high dst_low dst_high))))+ (mem_high SyntheticAmode (amode_offset mem_low flags 8))+ (args BoxAtomic128XchgSeqArgs (atomic128_xchg_seq_args mem_low mem_high input_low input_high dst_low dst_high))+ (_ Unit (emit (MInst.Atomic128XchgSeq args))))(value_regs dst_low dst_high)))-(decl x64_atomic_128_store_seq (SyntheticAmode ValueRegs) SideEffectNoResult)-(rule (x64_atomic_128_store_seq mem input)+(decl x64_atomic_128_store_seq (SyntheticAmode MemFlags ValueRegs) SideEffectNoResult)+(rule (x64_atomic_128_store_seq mem_low flags input)(let ((dst_low WritableGpr (temp_writable_gpr))(dst_high WritableGpr (temp_writable_gpr))(input_low Gpr (value_regs_get_gpr input 0))- (input_high Gpr (value_regs_get_gpr input 1)))- (SideEffectNoResult.Inst (MInst.Atomic128XchgSeq mem input_low input_high dst_low dst_high))))+ (input_high Gpr (value_regs_get_gpr input 1))+ (mem_high SyntheticAmode (amode_offset mem_low flags 8))+ (args BoxAtomic128XchgSeqArgs (atomic128_xchg_seq_args mem_low mem_high input_low input_high dst_low dst_high)))+ (SideEffectNoResult.Inst (MInst.Atomic128XchgSeq args))))(type AtomicRmwSeqOpdiff --git a/cranelift/codegen/src/isa/x64/inst/args.rs b/cranelift/codegen/src/isa/x64/inst/args.rsindex 233299966a..7e3b7394b9 100644--- a/cranelift/codegen/src/isa/x64/inst/args.rs+++ b/cranelift/codegen/src/isa/x64/inst/args.rs@@ -439,14 +439,14 @@ impl Amode {}/// Offset the amode by a fixed offset.- pub(crate) fn offset(&self, offset: i32) -> Self {+ pub(crate) fn offset(&self, offset: i32) -> Option<Self> {let mut ret = self.clone();- match &mut ret {- &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,- &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,+ let simm32 = match &mut ret {+ Amode::ImmReg { simm32, .. } | Amode::ImmRegRegShift { simm32, .. } => simm32,_ => panic!("Cannot offset amode: {self:?}"),- }- ret+ };+ *simm32 = simm32.checked_add(offset)?;+ Some(ret)}pub(crate) fn aligned(&self) -> bool {@@ -575,18 +575,18 @@ impl SyntheticAmode {}/// Offset the synthetic amode by a fixed offset.- pub(crate) fn offset(&self, offset: i32) -> Self {+ pub(crate) fn offset(&self, offset: i32) -> Option<Self> {let mut ret = self.clone();match &mut ret {- SyntheticAmode::Real(amode) => *amode = amode.offset(offset),- SyntheticAmode::SlotOffset { simm32 } => *simm32 += offset,+ SyntheticAmode::Real(amode) => *amode = amode.offset(offset)?,+ SyntheticAmode::SlotOffset { simm32 } => *simm32 = simm32.checked_add(offset)?,// `amode_offset` is used only in i128.load/store which// takes a synthetic amode from `to_amode`; `to_amode` can// only produce Real or SlotOffset amodes, never// IncomingArg or ConstantOffset._ => panic!("Cannot offset SyntheticAmode: {self:?}"),}- ret+ Some(ret)}}@@ -1061,3 +1061,34 @@ impl OperandSize {self.to_bytes() * 8}}++pub use crate::isa::x64::lower::isle::generated_code::Atomic128RmwSeqOp;++/// "Package" of the arguments for the instruction `Atomic128RmwSeq` to avoid+/// making the `Inst` enum massive.+#[derive(Debug, Clone)]+#[expect(missing_docs, reason = "self-describing fields")]+pub struct Atomic128RmwSeqArgs {+ pub op: Atomic128RmwSeqOp,+ pub mem_low: SyntheticAmode,+ pub mem_high: SyntheticAmode,+ pub operand_low: Gpr,+ pub operand_high: Gpr,+ pub temp_low: WritableGpr,+ pub temp_high: WritableGpr,+ pub dst_old_low: WritableGpr,+ pub dst_old_high: WritableGpr,+}++/// "Package" of the arguments for the instruction `Atomic128XchgSeq` to avoid+/// making the `Inst` enum massive.+#[derive(Debug, Clone)]+#[expect(missing_docs, reason = "self-describing fields")]+pub struct Atomic128XchgSeqArgs {+ pub mem_low: SyntheticAmode,+ pub mem_high: SyntheticAmode,+ pub operand_low: Gpr,+ pub operand_high: Gpr,+ pub dst_old_low: WritableGpr,+ pub dst_old_high: WritableGpr,+}diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rsindex de0fecb829..7f6f1aeb76 100644--- a/cranelift/codegen/src/isa/x64/inst/emit.rs+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs@@ -1544,16 +1544,18 @@ pub(crate) fn emit(one_way_jmp(sink, CC::NZ, again_label);}- Inst::Atomic128RmwSeq {- op,- mem,- operand_low,- operand_high,- temp_low,- temp_high,- dst_old_low,- dst_old_high,- } => {+ Inst::Atomic128RmwSeq { args } => {+ let Atomic128RmwSeqArgs {+ op,+ mem_low,+ mem_high,+ operand_low,+ operand_high,+ temp_low,+ temp_high,+ dst_old_low,+ dst_old_high,+ } = &**args;let operand_low = *operand_low;let operand_high = *operand_high;let temp_low = *temp_low;@@ -1564,13 +1566,14 @@ pub(crate) fn emit(debug_assert_eq!(temp_high.to_reg(), regs::rcx());debug_assert_eq!(dst_old_low.to_reg(), regs::rax());debug_assert_eq!(dst_old_high.to_reg(), regs::rdx());- let mem = mem.finalize(state.frame_layout(), sink).clone();+ let mem_low = mem_low.finalize(state.frame_layout(), sink).clone();+ let mem_high = mem_high.finalize(state.frame_layout(), sink).clone();let again_label = sink.get_label();// Load the initial value.- asm::inst::movq_rm::new(dst_old_low, mem.clone()).emit(sink, info, state);- asm::inst::movq_rm::new(dst_old_high, mem.offset(8)).emit(sink, info, state);+ asm::inst::movq_rm::new(dst_old_low, mem_low.clone()).emit(sink, info, state);+ asm::inst::movq_rm::new(dst_old_high, mem_high).emit(sink, info, state);// again:sink.bind_label(again_label, state.ctrl_plane_mut());@@ -1656,7 +1659,7 @@ pub(crate) fn emit(PairedGpr::from(dst_old_high),temp_low.to_reg(),temp_high.to_reg(),- mem,+ mem_low,).emit(sink, info, state);@@ -1664,13 +1667,15 @@ pub(crate) fn emit(one_way_jmp(sink, CC::NZ, again_label);}- Inst::Atomic128XchgSeq {- mem,- operand_low,- operand_high,- dst_old_low,- dst_old_high,- } => {+ Inst::Atomic128XchgSeq { args } => {+ let Atomic128XchgSeqArgs {+ mem_low,+ mem_high,+ operand_low,+ operand_high,+ dst_old_low,+ dst_old_high,+ } = &**args;let operand_low = *operand_low;let operand_high = *operand_high;let dst_old_low = *dst_old_low;@@ -1679,13 +1684,14 @@ pub(crate) fn emit(debug_assert_eq!(operand_high, regs::rcx());debug_assert_eq!(dst_old_low.to_reg(), regs::rax());debug_assert_eq!(dst_old_high.to_reg(), regs::rdx());- let mem = mem.finalize(state.frame_layout(), sink).clone();+ let mem_low = mem_low.finalize(state.frame_layout(), sink).clone();+ let mem_high = mem_high.finalize(state.frame_layout(), sink).clone();let again_label = sink.get_label();// Load the initial value.- asm::inst::movq_rm::new(dst_old_low, mem.clone()).emit(sink, info, state);- asm::inst::movq_rm::new(dst_old_high, mem.offset(8)).emit(sink, info, state);+ asm::inst::movq_rm::new(dst_old_low, mem_low.clone()).emit(sink, info, state);+ asm::inst::movq_rm::new(dst_old_high, mem_high).emit(sink, info, state);// again:sink.bind_label(again_label, state.ctrl_plane_mut());@@ -1696,7 +1702,7 @@ pub(crate) fn emit(PairedGpr::from(dst_old_high),operand_low,operand_high,- mem,+ mem_low,).emit(sink, info, state);diff --git a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs b/cranelift/codegen/src/isa/x64/inst/emit_tests.rsindex 1bd8b0f6b2..77d53ba384 100644--- a/cranelift/codegen/src/isa/x64/inst/emit_tests.rs+++ b/cranelift/codegen/src/isa/x64/inst/emit_tests.rs@@ -244,72 +244,70 @@ fn test_x64_emit() {));// Atomic128RmwSeq+ let rmw_args = Atomic128RmwSeqArgs {+ op: Atomic128RmwSeqOp::Or,+ mem_low: am3.clone(),+ mem_high: am3.offset(8).unwrap(),+ operand_low: Gpr::unwrap_new(r10),+ operand_high: Gpr::unwrap_new(r11),+ temp_low: w_rbx.map(Gpr::unwrap_new),+ temp_high: w_rcx.map(Gpr::unwrap_new),+ dst_old_low: w_rax.map(Gpr::unwrap_new),+ dst_old_high: w_rdx.map(Gpr::unwrap_new),+ };insns.push((Inst::Atomic128RmwSeq {- op: Atomic128RmwSeqOp::Or,- mem: Box::new(am3.clone()),- operand_low: Gpr::unwrap_new(r10),- operand_high: Gpr::unwrap_new(r11),- temp_low: w_rbx.map(Gpr::unwrap_new),- temp_high: w_rcx.map(Gpr::unwrap_new),- dst_old_low: w_rax.map(Gpr::unwrap_new),- dst_old_high: w_rdx.map(Gpr::unwrap_new),+ args: Box::new(Atomic128RmwSeqArgs {+ op: Atomic128RmwSeqOp::Or,+ ..rmw_args.clone()+ }),},"498B01498B51084889C34889D1490BDA490BCBF0490FC7090F85E9FFFFFF",- "atomically { %rdx:%rax = 0(%r9); %rcx:%rbx = %rdx:%rax Or %r11:%r10; 0(%r9) = %rcx:%rbx }",+ "atomically { %rdx:%rax = 0(%r9):8(%r9); %rcx:%rbx = %rdx:%rax Or %r11:%r10; 0(%r9):8(%r9) = %rcx:%rbx }"));insns.push((Inst::Atomic128RmwSeq {- op: Atomic128RmwSeqOp::And,- mem: Box::new(am3.clone()),- operand_low: Gpr::unwrap_new(r10),- operand_high: Gpr::unwrap_new(r11),- temp_low: w_rbx.map(Gpr::unwrap_new),- temp_high: w_rcx.map(Gpr::unwrap_new),- dst_old_low: w_rax.map(Gpr::unwrap_new),- dst_old_high: w_rdx.map(Gpr::unwrap_new),+ args: Box::new(Atomic128RmwSeqArgs {+ op: Atomic128RmwSeqOp::And,+ ..rmw_args.clone()+ }),},"498B01498B51084889C34889D14923DA4923CBF0490FC7090F85E9FFFFFF",- "atomically { %rdx:%rax = 0(%r9); %rcx:%rbx = %rdx:%rax And %r11:%r10; 0(%r9) = %rcx:%rbx }"+ "atomically { %rdx:%rax = 0(%r9):8(%r9); %rcx:%rbx = %rdx:%rax And %r11:%r10; 0(%r9):8(%r9) = %rcx:%rbx }"));insns.push((Inst::Atomic128RmwSeq {- op: Atomic128RmwSeqOp::Umin,- mem: Box::new(am3.clone()),- operand_low: Gpr::unwrap_new(r10),- operand_high: Gpr::unwrap_new(r11),- temp_low: w_rbx.map(Gpr::unwrap_new),- temp_high: w_rcx.map(Gpr::unwrap_new),- dst_old_low: w_rax.map(Gpr::unwrap_new),- dst_old_high: w_rdx.map(Gpr::unwrap_new),+ args: Box::new(Atomic128RmwSeqArgs {+ op: Atomic128RmwSeqOp::Umin,+ ..rmw_args.clone()+ }),},"498B01498B51084889C34889D14C39D3491BCB4889D1490F43DA490F43CBF0490FC7090F85DEFFFFFF",- "atomically { %rdx:%rax = 0(%r9); %rcx:%rbx = %rdx:%rax Umin %r11:%r10; 0(%r9) = %rcx:%rbx }"+ "atomically { %rdx:%rax = 0(%r9):8(%r9); %rcx:%rbx = %rdx:%rax Umin %r11:%r10; 0(%r9):8(%r9) = %rcx:%rbx }"));insns.push((Inst::Atomic128RmwSeq {- op: Atomic128RmwSeqOp::Add,- mem: Box::new(am3.clone()),- operand_low: Gpr::unwrap_new(r10),- operand_high: Gpr::unwrap_new(r11),- temp_low: w_rbx.map(Gpr::unwrap_new),- temp_high: w_rcx.map(Gpr::unwrap_new),- dst_old_low: w_rax.map(Gpr::unwrap_new),- dst_old_high: w_rdx.map(Gpr::unwrap_new),+ args: Box::new(Atomic128RmwSeqArgs {+ op: Atomic128RmwSeqOp::Add,+ ..rmw_args.clone()+ }),},"498B01498B51084889C34889D14903DA4913CBF0490FC7090F85E9FFFFFF",- "atomically { %rdx:%rax = 0(%r9); %rcx:%rbx = %rdx:%rax Add %r11:%r10; 0(%r9) = %rcx:%rbx }"+ "atomically { %rdx:%rax = 0(%r9):8(%r9); %rcx:%rbx = %rdx:%rax Add %r11:%r10; 0(%r9):8(%r9) = %rcx:%rbx }",));insns.push((Inst::Atomic128XchgSeq {- mem: am3.clone(),- operand_low: Gpr::unwrap_new(rbx),- operand_high: Gpr::unwrap_new(rcx),- dst_old_low: w_rax.map(Gpr::unwrap_new),- dst_old_high: w_rdx.map(Gpr::unwrap_new),+ args: Box::new(Atomic128XchgSeqArgs {+ mem_low: am3.clone(),+ mem_high: am3.offset(8).unwrap(),+ operand_low: Gpr::unwrap_new(rbx),+ operand_high: Gpr::unwrap_new(rcx),+ dst_old_low: w_rax.map(Gpr::unwrap_new),+ dst_old_high: w_rdx.map(Gpr::unwrap_new),+ }),},"498B01498B5108F0490FC7090F85F5FFFFFF",- "atomically { %rdx:%rax = 0(%r9); 0(%r9) = %rcx:%rbx }",+ "atomically { %rdx:%rax = 0(%r9):8(%r9); 0(%r9):8(%r9) = %rcx:%rbx }",));// ========================================================diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex c5759979bb..3dc1e4614c 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -757,42 +757,48 @@ impl PrettyPrint for Inst {)}- Inst::Atomic128RmwSeq {- op,- mem,- operand_low,- operand_high,- temp_low,- temp_high,- dst_old_low,- dst_old_high,- } => {+ Inst::Atomic128RmwSeq { args } => {+ let Atomic128RmwSeqArgs {+ op,+ mem_low,+ mem_high,+ operand_low,+ operand_high,+ temp_low,+ temp_high,+ dst_old_low,+ dst_old_high,+ } = &**args;let operand_low = pretty_print_reg(**operand_low, 8);let operand_high = pretty_print_reg(**operand_high, 8);let temp_low = pretty_print_reg(*temp_low.to_reg(), 8);let temp_high = pretty_print_reg(*temp_high.to_reg(), 8);let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);- let mem = mem.pretty_print(16);+ let mem_low = mem_low.pretty_print(16);+ let mem_high = mem_high.pretty_print(16);format!(- "atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem} = {temp_high}:{temp_low} }}"+ "atomically {{ {dst_old_high}:{dst_old_low} = {mem_low}:{mem_high}; {temp_high}:{temp_low} = {dst_old_high}:{dst_old_low} {op:?} {operand_high}:{operand_low}; {mem_low}:{mem_high} = {temp_high}:{temp_low} }}")}- Inst::Atomic128XchgSeq {- mem,- operand_low,- operand_high,- dst_old_low,- dst_old_high,- } => {+ Inst::Atomic128XchgSeq { args } => {+ let Atomic128XchgSeqArgs {+ mem_low,+ mem_high,+ operand_low,+ operand_high,+ dst_old_low,+ dst_old_high,+ } = &**args;let operand_low = pretty_print_reg(**operand_low, 8);let operand_high = pretty_print_reg(**operand_high, 8);let dst_old_low = pretty_print_reg(*dst_old_low.to_reg(), 8);let dst_old_high = pretty_print_reg(*dst_old_high.to_reg(), 8);- let mem = mem.pretty_print(16);+ let mem_low = mem_low.pretty_print(16);+ let mem_high = mem_high.pretty_print(16);format!(- "atomically {{ {dst_old_high}:{dst_old_low} = {mem}; {mem} = {operand_high}:{operand_low} }}"+ "atomically {{ {dst_old_high}:{dst_old_low} = {mem_low}:{mem_high}; {mem_low}:{mem_high} = {operand_high}:{operand_low} }}")}@@ -1105,16 +1111,18 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {mem.get_operands_late(collector)}- Inst::Atomic128RmwSeq {- operand_low,- operand_high,- temp_low,- temp_high,- dst_old_low,- dst_old_high,- mem,- ..- } => {+ Inst::Atomic128RmwSeq { args } => {+ let Atomic128RmwSeqArgs {+ mem_low,+ mem_high,+ operand_low,+ operand_high,+ temp_low,+ temp_high,+ dst_old_low,+ dst_old_high,+ op: _,+ } = &mut **args;// All registers are collected in the `Late` position so that they don't overlap.collector.reg_late_use(operand_low);collector.reg_late_use(operand_high);@@ -1122,23 +1130,26 @@ fn x64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_fixed_def(temp_high, regs::rcx());collector.reg_fixed_def(dst_old_low, regs::rax());collector.reg_fixed_def(dst_old_high, regs::rdx());- mem.get_operands_late(collector)+ mem_low.get_operands_late(collector);+ mem_high.get_operands_late(collector);}- Inst::Atomic128XchgSeq {- operand_low,- operand_high,- dst_old_low,- dst_old_high,- mem,- ..- } => {+ Inst::Atomic128XchgSeq { args } => {+ let Atomic128XchgSeqArgs {+ mem_low,+ mem_high,+ operand_low,+ operand_high,+ dst_old_low,+ dst_old_high,+ } = &mut **args;// All registers are collected in the `Late` position so that they don't overlap.collector.reg_fixed_late_use(operand_low, regs::rbx());collector.reg_fixed_late_use(operand_high, regs::rcx());collector.reg_fixed_def(dst_old_low, regs::rax());collector.reg_fixed_def(dst_old_high, regs::rdx());- mem.get_operands_late(collector)+ mem_low.get_operands_late(collector);+ mem_high.get_operands_late(collector);}Inst::Args { args } => {diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex e4fc7a8218..9819ef8c87 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -3094,7 +3094,7 @@(rule -3 (lower (has_type $I128(load _ (little_or_native_endian flags) address offset)))(let ((addr_lo SyntheticAmode (to_amode flags address offset))- (addr_hi SyntheticAmode (amode_offset addr_lo 8))+ (addr_hi SyntheticAmode (amode_offset addr_lo flags 8))(value_lo Reg (x64_mov addr_lo))(value_hi Reg (x64_mov addr_hi)))(value_regs value_lo value_hi)))@@ -3225,7 +3225,7 @@(value_lo Gpr (value_regs_get_gpr value_reg 0))(value_hi Gpr (value_regs_get_gpr value_reg 1))(addr_lo SyntheticAmode (to_amode flags address offset))- (addr_hi SyntheticAmode (amode_offset addr_lo 8)))+ (addr_hi SyntheticAmode (amode_offset addr_lo flags 8)))(side_effect(side_effect_concat(x64_movrm $I64 addr_lo value_lo)@@ -3450,7 +3450,7 @@;; Lower 128-bit `atomic_store` using `cmpxchg16b`.(rule 1 (lower (atomic_store (little_or_native_endian flags) value @ (value_type $I128) address))(if-let true (has_cmpxchg16b))- (side_effect (x64_atomic_128_store_seq (to_amode flags address (zero_offset)) value)))+ (side_effect (x64_atomic_128_store_seq (to_amode flags address (zero_offset)) flags value)));; Rules for `atomic_cas` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;@@ -3512,7 +3512,7 @@;; 128-bit integers always use a `lock cmpxchg16b` loop.(rule 3 (lower (has_type $I128 (atomic_rmw _ (little_or_native_endian flags) op address input)))(if-let true (has_cmpxchg16b))- (x64_atomic_128_rmw_seq op (to_amode flags address (zero_offset)) input))+ (x64_atomic_128_rmw_seq op (to_amode flags address (zero_offset)) flags input));; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;diff --git a/cranelift/codegen/src/isa/x64/lower/isle.rs b/cranelift/codegen/src/isa/x64/lower/isle.rsindex 57bfa06ee7..60d50df4ea 100644--- a/cranelift/codegen/src/isa/x64/lower/isle.rs+++ b/cranelift/codegen/src/isa/x64/lower/isle.rs@@ -36,7 +36,8 @@ type BoxCallIndInfo = Box<CallInfo<RegMem>>;type BoxReturnCallInfo = Box<ReturnCallInfo<ExternalName>>;type BoxReturnCallIndInfo = Box<ReturnCallInfo<Reg>>;type VecArgPair = Vec<ArgPair>;-type BoxSyntheticAmode = Box<SyntheticAmode>;+type BoxAtomic128RmwSeqArgs = Box<Atomic128RmwSeqArgs>;+type BoxAtomic128XchgSeqArgs = Box<Atomic128XchgSeqArgs>;/// When interacting with the external assembler (see `external.rs`), we/// need to fix the types we'll use.@@ -639,7 +640,7 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {}#[inline]- fn amode_offset(&mut self, addr: &SyntheticAmode, offset: i32) -> SyntheticAmode {+ fn amode_try_offset(&mut self, addr: &SyntheticAmode, offset: i32) -> Option<SyntheticAmode> {addr.offset(offset)}@@ -970,8 +971,48 @@ impl Context for IsleContext<'_, '_, MInst, X64Backend> {WritableGpr::from_reg(reg)}- fn box_synthetic_amode(&mut self, amode: &SyntheticAmode) -> BoxSyntheticAmode {- Box::new(amode.clone())+ fn atomic128_rmw_seq_args(+ &mut self,+ op: &Atomic128RmwSeqOp,+ mem_low: &SyntheticAmode,+ mem_high: &SyntheticAmode,+ operand_low: Gpr,+ operand_high: Gpr,+ temp_low: WritableGpr,+ temp_high: WritableGpr,+ dst_old_low: WritableGpr,+ dst_old_high: WritableGpr,+ ) -> BoxAtomic128RmwSeqArgs {+ Box::new(Atomic128RmwSeqArgs {+ op: *op,+ mem_low: mem_low.clone(),+ mem_high: mem_high.clone(),+ operand_low,+ operand_high,+ temp_low,+ temp_high,+ dst_old_low,+ dst_old_high,+ })+ }++ fn atomic128_xchg_seq_args(+ &mut self,+ mem_low: &SyntheticAmode,+ mem_high: &SyntheticAmode,+ operand_low: Gpr,+ operand_high: Gpr,+ dst_old_low: WritableGpr,+ dst_old_high: WritableGpr,+ ) -> BoxAtomic128XchgSeqArgs {+ Box::new(Atomic128XchgSeqArgs {+ mem_low: mem_low.clone(),+ mem_high: mem_high.clone(),+ operand_low,+ operand_high,+ dst_old_low,+ dst_old_high,+ })}////////////////////////////////////////////////////////////////////////////diff --git a/cranelift/filetests/filetests/isa/x64/atomic-128.clif b/cranelift/filetests/filetests/isa/x64/atomic-128.clifindex fd5796f5b9..261e19b768 100644--- a/cranelift/filetests/filetests/isa/x64/atomic-128.clif+++ b/cranelift/filetests/filetests/isa/x64/atomic-128.clif@@ -62,7 +62,7 @@ block0(v0: i128, v1: i64):; movq %rsi, %rcx; movq %rdi, %rbx; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%r8); 0(%r8) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%r8):8(%r8); 0(%r8):8(%r8) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -141,7 +141,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Add %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Add %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -183,7 +183,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Sub %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Sub %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -225,7 +225,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax And %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax And %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -267,7 +267,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Nand %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Nand %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -311,7 +311,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Or %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Or %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -353,7 +353,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Xor %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Xor %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -396,7 +396,7 @@ block0(v0: i64, v1: i128):; block0:; movq %rdx, %rcx; movq %rsi, %rbx-; atomically { %rdx:%rax = 0(%rdi); 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -435,7 +435,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Umin %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Umin %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -480,7 +480,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Umax %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Umax %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -525,7 +525,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Smin %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Smin %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rsp@@ -570,7 +570,7 @@ block0(v0: i64, v1: i128):; movq %rbx, (%rsp); block0:; movq %rdx, %r8-; atomically { %rdx:%rax = 0(%rdi); %rcx:%rbx = %rdx:%rax Smax %r8:%rsi; 0(%rdi) = %rcx:%rbx }+; atomically { %rdx:%rax = 0(%rdi):8(%rdi); %rcx:%rbx = %rdx:%rax Smax %r8:%rsi; 0(%rdi):8(%rdi) = %rcx:%rbx }; movq (%rsp), %rbx; addq $0x10, %rsp; movq %rbp, %rspdiff --git a/cranelift/filetests/filetests/isa/x64/i128.clif b/cranelift/filetests/filetests/isa/x64/i128.clifindex 51777c75c2..3b601d4d24 100644--- a/cranelift/filetests/filetests/isa/x64/i128.clif+++ b/cranelift/filetests/filetests/isa/x64/i128.clif@@ -2225,3 +2225,33 @@ block0(v0: i64, v1: i64):; popq %rbp; retq+function %load_huge_offset(i64) -> i64, i64 {+block0(v0: i64):+ v4 = load.i128 v0+0x7fffffff+ v5, v6 = isplit v4+ return v5, v6+}++; VCode:+; pushq %rbp+; movq %rsp, %rbp+; block0:+; leaq 0x7fffffff(%rdi), %rdx+; movq 0x7fffffff(%rdi), %rax+; movq 8(%rdx), %rdx+; movq %rbp, %rsp+; popq %rbp+; retq+;+; Disassembled:+; block0: ; offset 0x0+; pushq %rbp+; movq %rsp, %rbp+; block1: ; offset 0x4+; leaq 0x7fffffff(%rdi), %rdx ; trap: heap_oob+; movq 0x7fffffff(%rdi), %rax ; trap: heap_oob+; movq 8(%rdx), %rdx ; trap: heap_oob+; movq %rbp, %rsp+; popq %rbp+; retq+