A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
aarch64: Add support for "near" in `LoadExtName` (#11570)
Alex Crichton10d2cbc59d35
diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rsindex 39490849a8..d8f769c7a8 100644--- a/cranelift/codegen/src/binemit/mod.rs+++ b/cranelift/codegen/src/binemit/mod.rs@@ -82,6 +82,11 @@ pub enum Reloc {/// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)Aarch64AdrGotPage21,+ /// Equivalent of `R_AARCH64_ADR_PREL_PG_HI21`.+ Aarch64AdrPrelPgHi21,+ /// Equivalent of `R_AARCH64_ADD_ABS_LO12_NC`.+ Aarch64AddAbsLo12Nc,+/// AArch64 GOT Low bits/// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0@@ -155,6 +160,8 @@ impl fmt::Display for Reloc {Self::Aarch64TlsDescCall => write!(f, "Aarch64TlsDescCall"),Self::Aarch64AdrGotPage21 => write!(f, "Aarch64AdrGotPage21"),Self::Aarch64Ld64GotLo12Nc => write!(f, "Aarch64AdrGotLo12Nc"),+ Self::Aarch64AdrPrelPgHi21 => write!(f, "Aarch64AdrPrelPgHi21"),+ Self::Aarch64AddAbsLo12Nc => write!(f, "Aarch64AddAbsLo12Nc"),Self::S390xTlsGd64 => write!(f, "TlsGd64"),Self::S390xTlsGdCall => write!(f, "TlsGdCall"),Self::PulleyCallIndirectHost => write!(f, "PulleyCallIndirectHost"),diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isleindex 7caaacefbf..0baa5e3235 100644--- a/cranelift/codegen/src/isa/aarch64/inst.isle+++ b/cranelift/codegen/src/isa/aarch64/inst.isle@@ -918,7 +918,14 @@(rtmp2 WritableReg));; Load an inline symbol reference.- (LoadExtName+ (LoadExtNameGot+ (rd WritableReg)+ (name BoxExternalName))+ (LoadExtNameNear+ (rd WritableReg)+ (name BoxExternalName)+ (offset i64))+ (LoadExtNameFar(rd WritableReg)(name BoxExternalName)(offset i64))@@ -3808,11 +3815,51 @@(_ Unit (emit (MInst.VecLoadReplicate dst src size flags))))dst))-;; Helper for emitting `MInst.LoadExtName` instructions.-(decl load_ext_name (BoxExternalName i64) Reg)-(rule (load_ext_name extname offset)+(decl pure is_pic () bool)+(extern constructor is_pic is_pic)++;; Helper loading an external name into a register via `MInst.LoadExt*`+(decl load_ext_name (BoxExternalName i64 RelocDistance) Reg)++;; When `is_pic` is true all names are referenced through the GOT. Note that+;; the relocation is applied to the address of the GOT itself so the offset+;; requested must be manually added in (or skipped if it's 0).+(rule (load_ext_name name offset _)+ (if-let true (is_pic))+ (add $I64 (load_ext_name_got name) (imm $I64 (ImmExtend.Zero) (i64_cast_unsigned offset))))+(rule 1 (load_ext_name name 0 _)+ (if-let true (is_pic))+ (load_ext_name_got name))++;; Relocations that are "near" do an `adr;add` combo.+(rule 1 (load_ext_name name offset (RelocDistance.Near))+ (if-let false (is_pic))+ (load_ext_name_near name offset))++;; Relocations that are "far" do an `Abs8` relocation.+(rule 1 (load_ext_name name offset (RelocDistance.Far))+ (if-let false (is_pic))+ (load_ext_name_far name offset))++;; Helper for emitting `MInst.LoadExtNameGot` instructions.+(decl load_ext_name_got (BoxExternalName) Reg)+(rule (load_ext_name_got extname)+ (let ((dst WritableReg (temp_writable_reg $I64))+ (_ Unit (emit (MInst.LoadExtNameGot dst extname))))+ dst))++;; Helper for emitting `MInst.LoadExtNameNear` instructions.+(decl load_ext_name_near (BoxExternalName i64) Reg)+(rule (load_ext_name_near extname offset)+ (let ((dst WritableReg (temp_writable_reg $I64))+ (_ Unit (emit (MInst.LoadExtNameNear dst extname offset))))+ dst))++;; Helper for emitting `MInst.LoadExtNameFar` instructions.+(decl load_ext_name_far (BoxExternalName i64) Reg)+(rule (load_ext_name_far extname offset)(let ((dst WritableReg (temp_writable_reg $I64))- (_ Unit (emit (MInst.LoadExtName dst extname offset))))+ (_ Unit (emit (MInst.LoadExtNameFar dst extname offset))))dst));; Lower the address of a load or a store.diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rsindex 3d621c0c91..13ffccb38f 100644--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs@@ -3230,56 +3230,82 @@ impl MachInstEmit for Inst {// disable the worst-case-size check in this case.start_off = sink.cur_offset();}- &Inst::LoadExtName {+ &Inst::LoadExtNameGot { rd, ref name } => {+ // See this CE Example for the variations of this with and without BTI & PAUTH+ // https://godbolt.org/z/ncqjbbvvn+ //+ // Emit the following code:+ // adrp rd, :got:X+ // ldr rd, [rd, :got_lo12:X]++ // adrp rd, symbol+ sink.add_reloc(Reloc::Aarch64AdrGotPage21, &**name, 0);+ let inst = Inst::Adrp { rd, off: 0 };+ inst.emit(sink, emit_info, state);++ // ldr rd, [rd, :got_lo12:X]+ sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, &**name, 0);+ let inst = Inst::ULoad64 {+ rd,+ mem: AMode::reg(rd.to_reg()),+ flags: MemFlags::trusted(),+ };+ inst.emit(sink, emit_info, state);+ }+ &Inst::LoadExtNameNear {rd,ref name,offset,} => {- if emit_info.0.is_pic() {- // See this CE Example for the variations of this with and without BTI & PAUTH- // https://godbolt.org/z/ncqjbbvvn- //- // Emit the following code:- // adrp rd, :got:X- // ldr rd, [rd, :got_lo12:X]-- // adrp rd, symbol- sink.add_reloc(Reloc::Aarch64AdrGotPage21, &**name, 0);- let inst = Inst::Adrp { rd, off: 0 };- inst.emit(sink, emit_info, state);+ // Emit the following code:+ // adrp rd, X+ // add rd, rd, :lo12:X+ //+ // See https://godbolt.org/z/855KEvM5r for an example.- // ldr rd, [rd, :got_lo12:X]- sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, &**name, 0);- let inst = Inst::ULoad64 {- rd,- mem: AMode::reg(rd.to_reg()),- flags: MemFlags::trusted(),- };- inst.emit(sink, emit_info, state);- } else {- // With absolute offsets we set up a load from a preallocated space, and then jump- // over it.- //- // Emit the following code:- // ldr rd, #8- // b #0x10- // <8 byte space>-- let inst = Inst::ULoad64 {- rd,- mem: AMode::Label {- label: MemLabel::PCRel(8),- },- flags: MemFlags::trusted(),- };- inst.emit(sink, emit_info, state);- let inst = Inst::Jump {- dest: BranchTarget::ResolvedOffset(12),- };- inst.emit(sink, emit_info, state);- sink.add_reloc(Reloc::Abs8, &**name, offset);- sink.put8(0);- }+ // adrp rd, symbol+ sink.add_reloc(Reloc::Aarch64AdrPrelPgHi21, &**name, offset);+ let inst = Inst::Adrp { rd, off: 0 };+ inst.emit(sink, emit_info, state);++ // add rd, rd, :lo12:X+ sink.add_reloc(Reloc::Aarch64AddAbsLo12Nc, &**name, offset);+ let inst = Inst::AluRRImm12 {+ alu_op: ALUOp::Add,+ size: OperandSize::Size64,+ rd,+ rn: rd.to_reg(),+ imm12: Imm12::ZERO,+ };+ inst.emit(sink, emit_info, state);+ }+ &Inst::LoadExtNameFar {+ rd,+ ref name,+ offset,+ } => {+ // With absolute offsets we set up a load from a preallocated space, and then jump+ // over it.+ //+ // Emit the following code:+ // ldr rd, #8+ // b #0x10+ // <8 byte space>++ let inst = Inst::ULoad64 {+ rd,+ mem: AMode::Label {+ label: MemLabel::PCRel(8),+ },+ flags: MemFlags::trusted(),+ };+ inst.emit(sink, emit_info, state);+ let inst = Inst::Jump {+ dest: BranchTarget::ResolvedOffset(12),+ };+ inst.emit(sink, emit_info, state);+ sink.add_reloc(Reloc::Abs8, &**name, offset);+ sink.put8(0);}&Inst::LoadAddr { rd, ref mem } => {let mem = mem.clone();diff --git a/cranelift/codegen/src/isa/aarch64/inst/imms.rs b/cranelift/codegen/src/isa/aarch64/inst/imms.rsindex e90b622ec7..e408f26985 100644--- a/cranelift/codegen/src/isa/aarch64/inst/imms.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/imms.rs@@ -270,6 +270,12 @@ pub struct Imm12 {}impl Imm12 {+ /// Handy 0-value constant.+ pub const ZERO: Imm12 = Imm12 {+ bits: 0,+ shift12: false,+ };+/// Compute a Imm12 from raw bits, if possible.pub fn maybe_from_u64(val: u64) -> Option<Imm12> {if val & !0xfff == 0 {diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rsindex e0fddfa109..d582accf47 100644--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs@@ -879,7 +879,9 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_early_def(rtmp1);collector.reg_early_def(rtmp2);}- Inst::LoadExtName { rd, .. } => {+ Inst::LoadExtNameGot { rd, .. }+ | Inst::LoadExtNameNear { rd, .. }+ | Inst::LoadExtNameFar { rd, .. } => {collector.reg_def(rd);}Inst::LoadAddr { rd, mem } => {@@ -2746,13 +2748,25 @@ impl Inst {targets)}- &Inst::LoadExtName {+ &Inst::LoadExtNameGot { rd, ref name } => {+ let rd = pretty_print_reg(rd.to_reg());+ format!("load_ext_name_got {rd}, {name:?}")+ }+ &Inst::LoadExtNameNear {+ rd,+ ref name,+ offset,+ } => {+ let rd = pretty_print_reg(rd.to_reg());+ format!("load_ext_name_near {rd}, {name:?}+{offset}")+ }+ &Inst::LoadExtNameFar {rd,ref name,offset,} => {let rd = pretty_print_reg(rd.to_reg());- format!("load_ext_name {rd}, {name:?}+{offset}")+ format!("load_ext_name_far {rd}, {name:?}+{offset}")}&Inst::LoadAddr { rd, ref mem } => {// TODO: we really should find a better way to avoid duplication ofdiff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isleindex 694ad6283f..8c43493354 100644--- a/cranelift/codegen/src/isa/aarch64/lower.isle+++ b/cranelift/codegen/src/isa/aarch64/lower.isle@@ -2484,13 +2484,13 @@;;;; Rules for `func_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (func_addr (func_ref_data _ extname _)))- (load_ext_name (box_external_name extname) 0))+(rule (lower (func_addr (func_ref_data _ extname dist)))+ (load_ext_name (box_external_name extname) 0 dist));;;; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;-(rule (lower (symbol_value (symbol_value_data extname _ offset)))- (load_ext_name (box_external_name extname) offset))+(rule (lower (symbol_value (symbol_value_data extname dist offset)))+ (load_ext_name (box_external_name extname) offset dist));;; Rules for `get_{frame,stack}_pointer` and `get_return_address` ;;;;;;;;;;;;;@@ -2506,7 +2506,7 @@;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((output ValueRegsVec (gen_call_output sig_ref))(abi Sig (abi_sig sig_ref))(uses CallArgList (gen_call_args abi args))@@ -2516,12 +2516,12 @@output));; Direct call to an out-of-range function (implicitly via pointer).-(rule (lower (call (func_ref_data sig_ref name _) args))+(rule (lower (call (func_ref_data sig_ref name dist) args))(let ((output ValueRegsVec (gen_call_output sig_ref))(abi Sig (abi_sig sig_ref))(uses CallArgList (gen_call_args abi args))(defs CallRetList (gen_call_rets abi output))- (target Reg (load_ext_name name 0))+ (target Reg (load_ext_name name 0 dist))(info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none)))(_ Unit (emit_side_effect (call_ind_impl info))))output))@@ -2540,7 +2540,7 @@;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (reloc_distance_near)) args et) targets)+(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (RelocDistance.Near)) args et) targets)(let ((abi Sig (abi_sig sig_ref))(trycall OptionTryCallInfo (try_call_info et targets))(uses CallArgList (gen_call_args abi args))@@ -2549,12 +2549,12 @@(emit_side_effect (call_impl info))));; Direct call to an out-of-range function (implicitly via pointer).-(rule (lower_branch (try_call (func_ref_data sig_ref name _) args et) targets)+(rule (lower_branch (try_call (func_ref_data sig_ref name dist) args et) targets)(let ((abi Sig (abi_sig sig_ref))(trycall OptionTryCallInfo (try_call_info et targets))(uses CallArgList (gen_call_args abi args))(defs CallRetList (gen_try_call_rets abi))- (target Reg (load_ext_name name 0))+ (target Reg (load_ext_name name 0 dist))(info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall)))(emit_side_effect (call_ind_impl info))))@@ -2578,17 +2578,17 @@;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower (return_call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (return_call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((abi Sig (abi_sig sig_ref))(uses CallArgList (gen_return_call_args abi args))(info BoxReturnCallInfo (gen_return_call_info abi name uses)))(side_effect (return_call_impl info))));; Direct call to an out-of-range function (implicitly via pointer).-(rule (lower (return_call (func_ref_data sig_ref name _) args))+(rule (lower (return_call (func_ref_data sig_ref name dist) args))(let ((abi Sig (abi_sig sig_ref))(uses CallArgList (gen_return_call_args abi args))- (target Reg (load_ext_name name 0))+ (target Reg (load_ext_name name 0 dist))(info BoxReturnCallIndInfo (gen_return_call_ind_info abi target uses)))(side_effect (return_call_ind_impl info))))diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rsindex a79d245bb7..00d6bf2cba 100644--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs@@ -852,4 +852,8 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {ShiftOpShiftImm::maybe_from_shift(shift.value().into()).unwrap(),)}++ fn is_pic(&mut self) -> bool {+ self.backend.flags.is_pic()+ }}diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isleindex c415f08f3a..bca4058a1c 100644--- a/cranelift/codegen/src/isa/pulley_shared/lower.isle+++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle@@ -141,7 +141,7 @@;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to a pulley function.-(rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((output ValueRegsVec (gen_call_output sig_ref))(abi Sig (abi_sig sig_ref))(uses CallArgList (gen_call_args abi args))@@ -174,7 +174,7 @@;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to a pulley function.-(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (reloc_distance_near)) args et) targets)+(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (RelocDistance.Near)) args et) targets)(let ((abi Sig (abi_sig sig_ref))(trycall OptionTryCallInfo (try_call_info et targets))(uses CallArgList (gen_call_args abi args))@@ -205,7 +205,7 @@;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to a pulley function.-(rule 1 (lower (return_call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (return_call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((abi Sig (abi_sig sig_ref))(uses CallArgList (gen_return_call_args abi args))(info BoxReturnCallInfo (gen_return_call_info abi name uses)))diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isleindex c5bfe51580..39d95e8978 100644--- a/cranelift/codegen/src/isa/riscv64/lower.isle+++ b/cranelift/codegen/src/isa/riscv64/lower.isle@@ -2669,7 +2669,7 @@;;;; Rules for calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((output ValueRegsVec (gen_call_output sig_ref))(abi Sig (abi_sig sig_ref))(uses CallArgList (gen_call_args abi args))@@ -2703,7 +2703,7 @@;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (reloc_distance_near)) args et) targets)+(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (RelocDistance.Near)) args et) targets)(let ((abi Sig (abi_sig sig_ref))(trycall OptionTryCallInfo (try_call_info et targets))(uses CallArgList (gen_call_args abi args))@@ -2735,7 +2735,7 @@;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower (return_call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (return_call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((abi Sig (abi_sig sig_ref))(uses CallArgList (gen_return_call_args abi args))(info BoxReturnCallInfo (gen_return_call_info abi name uses)))diff --git a/cranelift/codegen/src/isa/s390x/inst.isle b/cranelift/codegen/src/isa/s390x/inst.isleindex f53401ea63..57f70f65d1 100644--- a/cranelift/codegen/src/isa/s390x/inst.isle+++ b/cranelift/codegen/src/isa/s390x/inst.isle@@ -1825,7 +1825,7 @@(memarg_reg_plus_reg x y 0 flags))(rule 1 (lower_address flags- (symbol_value (symbol_value_data name (reloc_distance_near) sym_offset))+ (symbol_value (symbol_value_data name (RelocDistance.Near) sym_offset))(i64_from_offset offset))(if-let final_offset (memarg_symbol_offset_sum offset sym_offset))(memarg_symbol name final_offset flags))@@ -1846,7 +1846,7 @@(decl pure partial load_sym (Inst) Inst)(rule (load_sym inst)- (if-let (load _ (symbol_value (symbol_value_data _ (reloc_distance_near) sym_offset))+ (if-let (load _ (symbol_value (symbol_value_data _ (RelocDistance.Near) sym_offset))(i64_from_offset load_offset))inst)(if (memarg_symbol_offset_sum sym_offset load_offset))@@ -1854,7 +1854,7 @@(decl pure partial uload16_sym (Inst) Inst)(rule (uload16_sym inst)- (if-let (uload16 _ (symbol_value (symbol_value_data _ (reloc_distance_near) sym_offset))+ (if-let (uload16 _ (symbol_value (symbol_value_data _ (RelocDistance.Near) sym_offset))(i64_from_offset load_offset))inst)(if (memarg_symbol_offset_sum sym_offset load_offset))diff --git a/cranelift/codegen/src/isa/s390x/lower.isle b/cranelift/codegen/src/isa/s390x/lower.isleindex 3675079634..026cf88e69 100644--- a/cranelift/codegen/src/isa/s390x/lower.isle+++ b/cranelift/codegen/src/isa/s390x/lower.isle@@ -2302,7 +2302,7 @@;;;; Rules for `func_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load the address of a function, target reachable via PC-relative instruction.-(rule 1 (lower (func_addr (func_ref_data _ name (reloc_distance_near))))+(rule 1 (lower (func_addr (func_ref_data _ name (RelocDistance.Near))))(load_addr (memarg_symbol name 0 (memflags_trusted))));; Load the address of a function, general case.@@ -2313,7 +2313,7 @@;;;; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Load the address of a symbol, target reachable via PC-relative instruction.-(rule 1 (lower (symbol_value (symbol_value_data name (reloc_distance_near)+(rule 1 (lower (symbol_value (symbol_value_data name (RelocDistance.Near)off)))(if-let offset (memarg_symbol_offset off))(load_addr (memarg_symbol name offset (memflags_trusted))))@@ -3957,7 +3957,7 @@;;;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((output ValueRegsVec (gen_call_output sig_ref))(abi Sig (abi_sig sig_ref))(_ Unit (abi_emit_call_adjust_stack abi))@@ -3995,7 +3995,7 @@;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;; Direct tail call to an in-range function.-(rule 1 (lower (return_call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (return_call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((abi Sig (abi_sig sig_ref))(_ Unit (abi_emit_return_call_adjust_stack abi))(uses CallArgList (gen_return_call_args abi (abi_prepare_args abi args)))@@ -4024,7 +4024,7 @@;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (reloc_distance_near)) args et) targets)+(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (RelocDistance.Near)) args et) targets)(let ((abi Sig (abi_sig sig_ref))(trycall OptionTryCallInfo (try_call_info et targets))(_ Unit (abi_emit_call_adjust_stack abi))diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isleindex 7008f2fc21..6a5834610e 100644--- a/cranelift/codegen/src/isa/x64/lower.isle+++ b/cranelift/codegen/src/isa/x64/lower.isle@@ -3480,7 +3480,7 @@;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((output ValueRegsVec (gen_call_output sig_ref))(abi Sig (abi_sig sig_ref))(uses CallArgList (gen_call_args abi args))@@ -3514,7 +3514,7 @@;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower (return_call (func_ref_data sig_ref name (reloc_distance_near)) args))+(rule 1 (lower (return_call (func_ref_data sig_ref name (RelocDistance.Near)) args))(let ((abi Sig (abi_sig sig_ref))(uses CallArgList (gen_return_call_args abi args))(info BoxReturnCallInfo (gen_return_call_info abi name uses)))@@ -3539,7 +3539,7 @@;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Direct call to an in-range function.-(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (reloc_distance_near)) args et) targets)+(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (RelocDistance.Near)) args et) targets)(let ((abi Sig (abi_sig sig_ref))(trycall OptionTryCallInfo (try_call_info et targets))(uses CallArgList (gen_call_args abi args))diff --git a/cranelift/codegen/src/machinst/isle.rs b/cranelift/codegen/src/machinst/isle.rsindex a9646a5863..36421a7c43 100644--- a/cranelift/codegen/src/machinst/isle.rs+++ b/cranelift/codegen/src/machinst/isle.rs@@ -358,15 +358,6 @@ macro_rules! isle_lower_prelude_methods {Some((name.clone(), reloc, offset))}- #[inline]- fn reloc_distance_near(&mut self, dist: RelocDistance) -> Option<()> {- if dist == RelocDistance::Near {- Some(())- } else {- None- }- }-#[inline]fn u128_from_immediate(&mut self, imm: Immediate) -> Option<u128> {let bytes = self.lower_ctx.get_immediate_data(imm).as_slice();diff --git a/cranelift/codegen/src/prelude_lower.isle b/cranelift/codegen/src/prelude_lower.isleindex b2c384e2b9..069594b940 100644--- a/cranelift/codegen/src/prelude_lower.isle+++ b/cranelift/codegen/src/prelude_lower.isle@@ -196,7 +196,7 @@(type UnwindInst (primitive UnwindInst))(type ExternalName (primitive ExternalName))(type BoxExternalName (primitive BoxExternalName))-(type RelocDistance (primitive RelocDistance))+(type RelocDistance extern (enum (Near) (Far)))(type VecArgPair extern (enum))(type VecRetPair extern (enum))(type CallArgList (primitive CallArgList))@@ -995,11 +995,6 @@(decl symbol_value_data (ExternalName RelocDistance i64) GlobalValue)(extern extractor symbol_value_data symbol_value_data)-;; Accessor for `RelocDistance`.--(decl reloc_distance_near () RelocDistance)-(extern extractor reloc_distance_near reloc_distance_near)-;; Accessor for `Immediate` as a vector of u8 values.(decl vec_mask_from_immediate (VecMask) Immediate)diff --git a/cranelift/filetests/filetests/isa/aarch64/bti.clif b/cranelift/filetests/filetests/isa/aarch64/bti.clifindex d5e7841bf0..f2ba08c270 100644--- a/cranelift/filetests/filetests/isa/aarch64/bti.clif+++ b/cranelift/filetests/filetests/isa/aarch64/bti.clif@@ -154,7 +154,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x3, TestCase(%g)+0+; load_ext_name_far x3, TestCase(%g)+0; blr x3; ldp fp, lr, [sp], #16; retdiff --git a/cranelift/filetests/filetests/isa/aarch64/call-pauth-bkey.clif b/cranelift/filetests/filetests/isa/aarch64/call-pauth-bkey.clifindex 9b37ca1521..589b4b4992 100644--- a/cranelift/filetests/filetests/isa/aarch64/call-pauth-bkey.clif+++ b/cranelift/filetests/filetests/isa/aarch64/call-pauth-bkey.clif@@ -15,7 +15,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x3, TestCase(%g)+0+; load_ext_name_far x3, TestCase(%g)+0; blr x3; ldp fp, lr, [sp], #16; retabsp@@ -63,7 +63,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x3, TestCase(%g)+0+; load_ext_name_far x3, TestCase(%g)+0; mov x0, x2; blr x3; mov x2, x0@@ -99,7 +99,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%g)+0+; load_ext_name_far x1, TestCase(%g)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:diff --git a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clifindex 736cafe2e0..73792bee0d 100644--- a/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif+++ b/cranelift/filetests/filetests/isa/aarch64/call-pauth.clif@@ -15,7 +15,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x3, TestCase(%g)+0+; load_ext_name_far x3, TestCase(%g)+0; blr x3; ldp fp, lr, [sp], #16; autiasp ; ret@@ -64,7 +64,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x3, TestCase(%g)+0+; load_ext_name_far x3, TestCase(%g)+0; mov x0, x2; blr x3; mov x2, x0@@ -100,7 +100,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%g)+0+; load_ext_name_far x1, TestCase(%g)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:diff --git a/cranelift/filetests/filetests/isa/aarch64/call.clif b/cranelift/filetests/filetests/isa/aarch64/call.clifindex abc709702d..dae6137124 100644--- a/cranelift/filetests/filetests/isa/aarch64/call.clif+++ b/cranelift/filetests/filetests/isa/aarch64/call.clif@@ -17,7 +17,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x3, TestCase(%g)+0+; load_ext_name_far x3, TestCase(%g)+0; blr x3; ldp fp, lr, [sp], #16; ret@@ -47,7 +47,7 @@ block0(v0: i32):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x3, TestCase(%g)+0+; load_ext_name_far x3, TestCase(%g)+0; blr x3; ldp fp, lr, [sp], #16; ret@@ -90,7 +90,7 @@ block0(v0: i32):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x3, TestCase(%g)+0+; load_ext_name_far x3, TestCase(%g)+0; blr x3; ldp fp, lr, [sp], #16; ret@@ -137,7 +137,7 @@ block0(v0: i8):; block0:; movz w7, #42; strb w0, [sp]-; load_ext_name x8, TestCase(%g)+0+; load_ext_name_far x8, TestCase(%g)+0; mov x0, x7; mov x1, x7; mov x2, x7@@ -229,24 +229,24 @@ block0:; mov fp, sp; sub sp, sp, #48; block0:-; load_ext_name x9, TestCase(%g0)+0+; load_ext_name_far x9, TestCase(%g0)+0; blr x9; str q0, [sp, #32]-; load_ext_name x9, TestCase(%g1)+0+; load_ext_name_far x9, TestCase(%g1)+0; blr x9; str q0, [sp, #16]-; load_ext_name x9, TestCase(%g1)+0+; load_ext_name_far x9, TestCase(%g1)+0; blr x9; str q0, [sp]-; load_ext_name x9, TestCase(%g2)+0+; load_ext_name_far x9, TestCase(%g2)+0; blr x9-; load_ext_name x10, TestCase(%g3)+0+; load_ext_name_far x10, TestCase(%g3)+0; ldr q0, [sp, #32]; blr x10-; load_ext_name x11, TestCase(%g4)+0+; load_ext_name_far x11, TestCase(%g4)+0; ldr q0, [sp, #16]; blr x11-; load_ext_name x12, TestCase(%g4)+0+; load_ext_name_far x12, TestCase(%g4)+0; ldr q0, [sp]; blr x12; add sp, sp, #48@@ -325,24 +325,24 @@ block0:; mov fp, sp; sub sp, sp, #48; block0:-; load_ext_name x9, TestCase(%g0)+0+; load_ext_name_far x9, TestCase(%g0)+0; blr x9; str q0, [sp, #32]-; load_ext_name x9, TestCase(%g0)+0+; load_ext_name_far x9, TestCase(%g0)+0; blr x9; str q0, [sp, #16]-; load_ext_name x9, TestCase(%g0)+0+; load_ext_name_far x9, TestCase(%g0)+0; blr x9; str q0, [sp]-; load_ext_name x9, TestCase(%g1)+0+; load_ext_name_far x9, TestCase(%g1)+0; blr x9-; load_ext_name x10, TestCase(%g2)+0+; load_ext_name_far x10, TestCase(%g2)+0; ldr q0, [sp, #32]; blr x10-; load_ext_name x11, TestCase(%g2)+0+; load_ext_name_far x11, TestCase(%g2)+0; ldr q0, [sp, #16]; blr x11-; load_ext_name x12, TestCase(%g2)+0+; load_ext_name_far x12, TestCase(%g2)+0; ldr q0, [sp]; blr x12; add sp, sp, #48@@ -425,24 +425,24 @@ block0:; mov fp, sp; sub sp, sp, #48; block0:-; load_ext_name x9, TestCase(%g0)+0+; load_ext_name_far x9, TestCase(%g0)+0; blr x9; str q0, [sp, #32]-; load_ext_name x9, TestCase(%g1)+0+; load_ext_name_far x9, TestCase(%g1)+0; blr x9; str q0, [sp, #16]-; load_ext_name x9, TestCase(%g2)+0+; load_ext_name_far x9, TestCase(%g2)+0; blr x9; str q0, [sp]-; load_ext_name x9, TestCase(%g3)+0+; load_ext_name_far x9, TestCase(%g3)+0; blr x9-; load_ext_name x10, TestCase(%g4)+0+; load_ext_name_far x10, TestCase(%g4)+0; ldr q0, [sp, #32]; blr x10-; load_ext_name x11, TestCase(%g5)+0+; load_ext_name_far x11, TestCase(%g5)+0; ldr q0, [sp, #16]; blr x11-; load_ext_name x12, TestCase(%g6)+0+; load_ext_name_far x12, TestCase(%g6)+0; ldr q0, [sp]; blr x12; add sp, sp, #48@@ -531,7 +531,7 @@ block0(v0: i64):; mov fp, sp; block0:; movz x2, #42-; load_ext_name x4, TestCase(%f11)+0+; load_ext_name_far x4, TestCase(%f11)+0; mov x1, x0; mov x0, x2; blr x4@@ -585,7 +585,7 @@ block0(v0: i64):; mov fp, sp; block0:; movz x3, #42-; load_ext_name x4, TestCase(%f12)+0+; load_ext_name_far x4, TestCase(%f12)+0; mov x2, x0; mov x0, x3; blr x4@@ -639,7 +639,7 @@ block0(v0: i64):; mov fp, sp; block0:; movz x2, #42-; load_ext_name x4, TestCase(%f13)+0+; load_ext_name_far x4, TestCase(%f13)+0; mov x1, x0; mov x0, x2; blr x4@@ -701,7 +701,7 @@ block0(v0: i128, v1: i64):; block0:; str x0, [sp]; str x1, [sp, #8]-; load_ext_name x8, TestCase(%f14)+0+; load_ext_name_far x8, TestCase(%f14)+0; mov x4, x0; mov x5, x1; mov x6, x2@@ -773,7 +773,7 @@ block0(v0: i128, v1: i64):; block0:; str x0, [sp]; str x1, [sp, #8]-; load_ext_name x8, TestCase(%f15)+0+; load_ext_name_far x8, TestCase(%f15)+0; mov x4, x0; mov x5, x1; mov x6, x2@@ -839,7 +839,7 @@ block0(v0: i64):; str x24, [sp, #-16]!; block0:; mov x24, x0-; load_ext_name x2, TestCase(%g)+0+; load_ext_name_far x2, TestCase(%g)+0; mov x8, x24; blr x2; mov x0, x24@@ -879,7 +879,7 @@ block0(v0: i64):; str x24, [sp, #-16]!; block0:; mov x24, x8-; load_ext_name x2, TestCase(%g)+0+; load_ext_name_far x2, TestCase(%g)+0; blr x2; mov x8, x24; ldr x24, [sp], #16@@ -915,7 +915,7 @@ block0:; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x0, TestCase(%g)+0+; load_ext_name_far x0, TestCase(%g)+0; blr x0; ldp fp, lr, [sp], #16; retdiff --git a/cranelift/filetests/filetests/isa/aarch64/exceptions.clif b/cranelift/filetests/filetests/isa/aarch64/exceptions.clifindex 670173d57e..91c029e1cd 100644--- a/cranelift/filetests/filetests/isa/aarch64/exceptions.clif+++ b/cranelift/filetests/filetests/isa/aarch64/exceptions.clif@@ -157,7 +157,7 @@ function %f2(i32) -> i32, f32, f64 {; block0:; fmov d1, #1; str q1, [sp]-; load_ext_name x11, TestCase(%g)+0+; load_ext_name_far x11, TestCase(%g)+0; mov x2, x0; blr x11; b MachLabel(1); catch [default: MachLabel(2)]; block1:@@ -286,7 +286,7 @@ function %f4(i64, i32) -> i32, f32, f64 {; str x0, [sp, #8]; fmov d1, #1; str q1, [sp, #16]-; load_ext_name x12, TestCase(%g)+0+; load_ext_name_far x12, TestCase(%g)+0; mov x2, x1; str x1, [sp]; blr x12; b MachLabel(3); catch [context stack1, tag0: MachLabel(1), tag1: MachLabel(2), context stack0, tag0: MachLabel(4)]diff --git a/cranelift/filetests/filetests/isa/aarch64/fuzzbug-60035.clif b/cranelift/filetests/filetests/isa/aarch64/fuzzbug-60035.clifindex 40fd6ccfa4..12e6787c6d 100644--- a/cranelift/filetests/filetests/isa/aarch64/fuzzbug-60035.clif+++ b/cranelift/filetests/filetests/isa/aarch64/fuzzbug-60035.clif@@ -17,7 +17,7 @@ block0:; mov fp, sp; str x19, [sp, #-16]!; block0:-; load_ext_name x19, User(userextname0)+0+; load_ext_name_far x19, User(userextname0)+0; blr x19; blr x19; ldr x19, [sp], #16diff --git a/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clifindex ca0c1fcd48..d8ca4f4486 100644--- a/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif+++ b/cranelift/filetests/filetests/isa/aarch64/return-call-indirect.clif@@ -33,7 +33,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%callee_i64)+0+; load_ext_name_far x1, TestCase(%callee_i64)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:@@ -63,7 +63,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%callee_i64)+0+; load_ext_name_near x1, TestCase(%callee_i64)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:@@ -71,10 +71,8 @@ block0(v0: i64):; stp x29, x30, [sp, #-0x10]!; mov x29, sp; block1: ; offset 0x8-; ldr x1, #0x10-; b #0x18-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_i64 0-; .byte 0x00, 0x00, 0x00, 0x00+; adrp x1, #0 ; reloc_external Aarch64AdrPrelPgHi21 %callee_i64 0+; add x1, x1, #0 ; reloc_external Aarch64AddAbsLo12Nc %callee_i64 0; ldp x29, x30, [sp], #0x10; br x1@@ -112,7 +110,7 @@ block0(v0: f64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%callee_f64)+0+; load_ext_name_far x1, TestCase(%callee_f64)+0; return_call_ind x1 new_stack_arg_size:0 v0=v0;; Disassembled:@@ -163,7 +161,7 @@ block0(v0: i8):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%callee_i8)+0+; load_ext_name_far x1, TestCase(%callee_i8)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:@@ -255,7 +253,7 @@ block0:; movz x23, #125; movz x24, #130; movz x25, #135-; load_ext_name x2, TestCase(%tail_callee_stack_args)+0+; load_ext_name_far x2, TestCase(%tail_callee_stack_args)+0; str x10, [sp, #112]; str x11, [sp, #120]; str x12, [sp, #128]diff --git a/cranelift/filetests/filetests/isa/aarch64/return-call.clif b/cranelift/filetests/filetests/isa/aarch64/return-call.clifindex 1e018b3c4a..96b36885ef 100644--- a/cranelift/filetests/filetests/isa/aarch64/return-call.clif+++ b/cranelift/filetests/filetests/isa/aarch64/return-call.clif@@ -31,7 +31,7 @@ block0(v0: i64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%callee_i64)+0+; load_ext_name_far x1, TestCase(%callee_i64)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:@@ -101,7 +101,7 @@ block0(v0: f64):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%callee_f64)+0+; load_ext_name_far x1, TestCase(%callee_f64)+0; return_call_ind x1 new_stack_arg_size:0 v0=v0;; Disassembled:@@ -150,7 +150,7 @@ block0(v0: i8):; stp fp, lr, [sp, #-16]!; mov fp, sp; block0:-; load_ext_name x1, TestCase(%callee_i8)+0+; load_ext_name_far x1, TestCase(%callee_i8)+0; return_call_ind x1 new_stack_arg_size:0 x2=x2;; Disassembled:@@ -283,7 +283,7 @@ block0:; str x23, [sp, #232]; str x24, [sp, #240]; str x25, [sp, #248]-; load_ext_name x1, TestCase(%tail_callee_stack_args)+0+; load_ext_name_far x1, TestCase(%tail_callee_stack_args)+0; return_call_ind x1 new_stack_arg_size:160 x2=x2 x3=x3 x4=x4 x5=x5 x6=x6 x7=x7;; Disassembled:@@ -514,7 +514,7 @@ block2:; str x15, [sp, #256]; str x14, [sp, #264]; str x2, [sp, #272]-; load_ext_name x1, TestCase(%different_callee2)+0+; load_ext_name_far x1, TestCase(%different_callee2)+0; ldr x2, [sp]; return_call_ind x1 new_stack_arg_size:176 x2=x2 x3=x3 x4=x4 x5=x5 x6=x6 x7=x7; block2:@@ -539,7 +539,7 @@ block2:; str x0, [sp, #264]; str x15, [sp, #272]; str x14, [sp, #280]-; load_ext_name x1, TestCase(%different_callee1)+0+; load_ext_name_far x1, TestCase(%different_callee1)+0; return_call_ind x1 new_stack_arg_size:160 x2=x2 x3=x3 x4=x4 x5=x5 x6=x6 x7=x7;; Disassembled:diff --git a/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif b/cranelift/filetests/filetests/isa/aarch64/stack-limit.clifindex 49d4963f6b..26081172cf 100644--- a/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif+++ b/cranelift/filetests/filetests/isa/aarch64/stack-limit.clif@@ -52,7 +52,7 @@ block0(v0: i64):; subs xzr, sp, x16, UXTX; b.lo #trap=stk_ovf; block0:-; load_ext_name x0, TestCase(%foo)+0+; load_ext_name_far x0, TestCase(%foo)+0; blr x0; ldp fp, lr, [sp], #16; retdiff --git a/cranelift/filetests/filetests/isa/aarch64/symbol-value-pic.clif b/cranelift/filetests/filetests/isa/aarch64/symbol-value-pic.clifindex 44c5df4065..ba48387357 100644--- a/cranelift/filetests/filetests/isa/aarch64/symbol-value-pic.clif+++ b/cranelift/filetests/filetests/isa/aarch64/symbol-value-pic.clif@@ -3,22 +3,185 @@ set unwind_info=falseset is_pictarget aarch64-function %f() -> i64 {- gv0 = symbol %my_global+function %func_addr() -> i64 {+ fn0 = %func0(i64) -> i64block0:- v0 = symbol_value.i64 gv0- return v0+ v0 = func_addr.i64 fn0+ return v0}; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp; block0:-; load_ext_name x0, TestCase(%my_global)+0+; load_ext_name_got x0, TestCase(%func0)+; ldp fp, lr, [sp], #16; ret;; Disassembled:; block0: ; offset 0x0-; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %my_global 0-; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %my_global 0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; block1: ; offset 0x8+; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %func0 0+; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %func0 0+; ldp x29, x30, [sp], #0x10+; ret++function %colocated_func_addr() -> i64 {+ fn0 = colocated %func0(i64) -> i64++block0:+ v0 = func_addr.i64 fn0+ return v0+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; block0:+; load_ext_name_got x0, TestCase(%func0)+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; block1: ; offset 0x8+; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %func0 0+; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %func0 0+; ldp x29, x30, [sp], #0x10+; ret++function %symbol_value() -> i64 {+ gv0 = symbol %global0++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_got x0, TestCase(%global0)+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %global0 0+; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %global0 0+; ret++function %symbol_value_plus_offset() -> i64 {+ gv0 = symbol %global0+123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_got x0, TestCase(%global0)+; movz x2, #123+; add x0, x0, x2+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %global0 0+; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %global0 0+; mov x2, #0x7b+; add x0, x0, x2+; ret++function %symbol_value_minus_offset() -> i64 {+ gv0 = symbol %global0-123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_got x0, TestCase(%global0)+; movn x2, #122+; add x0, x0, x2+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %global0 0+; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %global0 0+; mov x2, #-0x7b+; add x0, x0, x2+; ret++function %colocated_symbol_value() -> i64 {+ gv0 = symbol colocated %global0++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_got x0, TestCase(%global0)+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %global0 0+; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %global0 0+; ret++function %colocated_symbol_value_plus_offset() -> i64 {+ gv0 = symbol colocated %global0+123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_got x0, TestCase(%global0)+; movz x2, #123+; add x0, x0, x2+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %global0 0+; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %global0 0+; mov x2, #0x7b+; add x0, x0, x2+; ret++function %colocated_symbol_value_minus_offset() -> i64 {+ gv0 = symbol colocated %global0-123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_got x0, TestCase(%global0)+; movn x2, #122+; add x0, x0, x2+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrGotPage21 %global0 0+; ldr x0, [x0] ; reloc_external Aarch64AdrGotLo12Nc %global0 0+; mov x2, #-0x7b+; add x0, x0, x2; retdiff --git a/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif b/cranelift/filetests/filetests/isa/aarch64/symbol-value.clifindex 9410eed1f4..60ec6779af 100644--- a/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif+++ b/cranelift/filetests/filetests/isa/aarch64/symbol-value.clif@@ -2,24 +2,177 @@ test compile precise-outputset unwind_info=falsetarget aarch64-function %f() -> i64 {- gv0 = symbol %my_global+function %func_addr() -> i64 {+ fn0 = %func0(i64) -> i64block0:- v0 = symbol_value.i64 gv0- return v0+ v0 = func_addr.i64 fn0+ return v0}; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp; block0:-; load_ext_name x0, TestCase(%my_global)+0+; load_ext_name_far x0, TestCase(%func0)+0+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; block1: ; offset 0x8+; ldr x0, #0x10+; b #0x18+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %func0 0+; .byte 0x00, 0x00, 0x00, 0x00+; ldp x29, x30, [sp], #0x10+; ret++function %colocated_func_addr() -> i64 {+ fn0 = colocated %func0(i64) -> i64++block0:+ v0 = func_addr.i64 fn0+ return v0+}++; VCode:+; stp fp, lr, [sp, #-16]!+; mov fp, sp+; block0:+; load_ext_name_near x0, TestCase(%func0)+0+; ldp fp, lr, [sp], #16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; stp x29, x30, [sp, #-0x10]!+; mov x29, sp+; block1: ; offset 0x8+; adrp x0, #0 ; reloc_external Aarch64AdrPrelPgHi21 %func0 0+; add x0, x0, #0 ; reloc_external Aarch64AddAbsLo12Nc %func0 0+; ldp x29, x30, [sp], #0x10+; ret++function %symbol_value() -> i64 {+ gv0 = symbol %global0++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_far x0, TestCase(%global0)+0; ret;; Disassembled:; block0: ; offset 0x0; ldr x0, #8; b #0x10-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %my_global 0+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 0; .byte 0x00, 0x00, 0x00, 0x00; ret+function %symbol_value_plus_offset() -> i64 {+ gv0 = symbol %global0+123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_far x0, TestCase(%global0)+123+; ret+;+; Disassembled:+; block0: ; offset 0x0+; ldr x0, #8+; b #0x10+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 123+; .byte 0x00, 0x00, 0x00, 0x00+; ret++function %symbol_value_minus_offset() -> i64 {+ gv0 = symbol %global0-123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_far x0, TestCase(%global0)+-123+; ret+;+; Disassembled:+; block0: ; offset 0x0+; ldr x0, #8+; b #0x10+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 -123+; .byte 0x00, 0x00, 0x00, 0x00+; ret++function %colocated_symbol_value() -> i64 {+ gv0 = symbol colocated %global0++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_near x0, TestCase(%global0)+0+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrPrelPgHi21 %global0 0+; add x0, x0, #0 ; reloc_external Aarch64AddAbsLo12Nc %global0 0+; ret++function %colocated_symbol_value_plus_offset() -> i64 {+ gv0 = symbol colocated %global0+123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_near x0, TestCase(%global0)+123+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrPrelPgHi21 %global0 123+; add x0, x0, #0 ; reloc_external Aarch64AddAbsLo12Nc %global0 123+; ret++function %colocated_symbol_value_minus_offset() -> i64 {+ gv0 = symbol colocated %global0-123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_near x0, TestCase(%global0)+-123+; ret+;+; Disassembled:+; block0: ; offset 0x0+; adrp x0, #0 ; reloc_external Aarch64AdrPrelPgHi21 %global0 -123+; add x0, x0, #0 ; reloc_external Aarch64AddAbsLo12Nc %global0 -123+; ret+diff --git a/cranelift/filetests/filetests/isa/aarch64/tail-call-conv.clif b/cranelift/filetests/filetests/isa/aarch64/tail-call-conv.clifindex 9ba81c4ec6..c456afa553 100644--- a/cranelift/filetests/filetests/isa/aarch64/tail-call-conv.clif+++ b/cranelift/filetests/filetests/isa/aarch64/tail-call-conv.clif@@ -118,7 +118,7 @@ block0:; str x24, [sp, #136]; str x25, [sp, #144]; str x26, [sp, #152]-; load_ext_name x15, TestCase(%tail_callee_stack_args)+0+; load_ext_name_far x15, TestCase(%tail_callee_stack_args)+0; blr x15; add sp, sp, #160; ldp x19, x20, [sp], #16@@ -388,7 +388,7 @@ block0:; sub sp, sp, #240; block0:; mov x8, sp-; load_ext_name x12, TestCase(%tail_callee_stack_rets)+0+; load_ext_name_far x12, TestCase(%tail_callee_stack_rets)+0; blr x12; ldr x2, [sp, #232]; add sp, sp, #240@@ -681,7 +681,7 @@ block0:; str x20, [sp, #144]; str x22, [sp, #152]; add x8, sp, #160-; load_ext_name x10, TestCase(%tail_callee_stack_args_and_rets)+0+; load_ext_name_far x10, TestCase(%tail_callee_stack_args_and_rets)+0; blr x10; ldr x2, [sp, #392]; add sp, sp, #400diff --git a/cranelift/filetests/filetests/isa/aarch64/uext-sext-handling.clif b/cranelift/filetests/filetests/isa/aarch64/uext-sext-handling.clifindex 473f3b7f16..7b081a4697 100644--- a/cranelift/filetests/filetests/isa/aarch64/uext-sext-handling.clif+++ b/cranelift/filetests/filetests/isa/aarch64/uext-sext-handling.clif@@ -15,7 +15,7 @@ block0(v0: i8):; nextln: mov fp, sp; nextln: block0:; check-not: uxtb w0, w0-; nextln: load_ext_name x2, User(userextname0)+0+; nextln: load_ext_name_far x2, User(userextname0)+0; nextln: blr x2; The aaple aarch64 call conv respects the uext and sext flags@@ -32,5 +32,5 @@ block0(v0: i8):; nextln: mov fp, sp; nextln: block0:; nextln: uxtb w0, w0-; nextln: load_ext_name x4, User(userextname0)+0+; nextln: load_ext_name_far x4, User(userextname0)+0; nextln: blr x4diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rsindex 157d64e2a9..cf1e1e0376 100644--- a/cranelift/jit/src/compiled_blob.rs+++ b/cranelift/jit/src/compiled_blob.rs@@ -94,6 +94,26 @@ impl CompiledBlob {Reloc::Aarch64Ld64GotLo12Nc => {panic!("GOT relocation shouldn't be generated when !is_pic");}+ Reloc::Aarch64AdrPrelPgHi21 => {+ let base = get_address(name);+ let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };+ let get_page = |x| x & (!0xfff);+ let pcrel = i32::try_from(get_page(what as isize) - get_page(at as isize))+ .unwrap()+ .cast_unsigned();+ let iptr = at as *mut u32;+ let hi21 = pcrel >> 12;+ let lo = (hi21 & 0x3) << 29;+ let hi = (hi21 & 0x1ffffc) << 3;+ unsafe { modify_inst32(iptr, |inst| inst | lo | hi) };+ }+ Reloc::Aarch64AddAbsLo12Nc => {+ let base = get_address(name);+ let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };+ let iptr = at as *mut u32;+ let imm12 = (what.addr() as u32 & 0xfff) << 10;+ unsafe { modify_inst32(iptr, |inst| inst | imm12) };+ }Reloc::RiscvCallPlt => {// A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.// It is the equivalent of two relocations: