A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
riscv64: Implement near relocations (#11585)
Alex Crichton13b300deac44
diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rsindex d1236a923c..cd34c0b3ff 100644--- a/cranelift/codegen/src/binemit/mod.rs+++ b/cranelift/codegen/src/binemit/mod.rs@@ -120,6 +120,12 @@ pub enum Reloc {/// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>RiscvGotHi20,+ /// High 20 bits of a 32-bit PC-relative offset relocation+ ///+ /// This is the `R_RISCV_PCREL_HI20` relocation from the RISC-V ELF psABI document.+ /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>+ RiscvPCRelHi20,+/// s390x TLS GD64 - 64-bit offset of tls_index for GD symbol in GOTS390xTlsGd64,/// s390x TLS GDCall - marker to enable optimization of TLS calls@@ -152,6 +158,7 @@ impl fmt::Display for Reloc {Self::RiscvCallPlt => write!(f, "RiscvCallPlt"),Self::RiscvTlsGdHi20 => write!(f, "RiscvTlsGdHi20"),Self::RiscvGotHi20 => write!(f, "RiscvGotHi20"),+ Self::RiscvPCRelHi20 => write!(f, "RiscvPCRelHi20"),Self::RiscvPCRelLo12I => write!(f, "RiscvPCRelLo12I"),Self::ElfX86_64TlsGd => write!(f, "ElfX86_64TlsGd"),Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isleindex 08283480e5..6e020d6a24 100644--- a/cranelift/codegen/src/isa/riscv64/inst.isle+++ b/cranelift/codegen/src/isa/riscv64/inst.isle@@ -139,7 +139,14 @@(kind IntegerCompare));; Load an inline symbol reference.- (LoadExtName+ (LoadExtNameGot+ (rd WritableReg)+ (name BoxExternalName))+ (LoadExtNameNear+ (rd WritableReg)+ (name BoxExternalName)+ (offset i64))+ (LoadExtNameFar(rd WritableReg)(name BoxExternalName)(offset i64))@@ -2750,8 +2757,43 @@;;;; load extern name-(decl load_ext_name (ExternalName i64) Reg)-(extern constructor load_ext_name load_ext_name)+(decl load_ext_name (ExternalName i64 RelocDistance) Reg)+(rule (load_ext_name name offset _dist)+ (if-let true (is_pic))+ (rv_add (load_ext_name_got name) (imm $I64 (i64_cast_unsigned offset))))+(rule 1 (load_ext_name name 0 _dist)+ (if-let true (is_pic))+ (load_ext_name_got name))+(rule (load_ext_name name offset (RelocDistance.Near))+ (if-let false (is_pic))+ (load_ext_name_near name offset))+(rule (load_ext_name name offset (RelocDistance.Far))+ (if-let false (is_pic))+ (load_ext_name_far name offset))++(decl pure is_pic () bool)+(extern constructor is_pic is_pic)++;; Helper for emitting `MInst.LoadExtNameGot` instructions.+(decl load_ext_name_got (BoxExternalName) Reg)+(rule (load_ext_name_got extname)+ (let ((dst WritableReg (temp_writable_reg $I64))+ (_ Unit (emit (MInst.LoadExtNameGot dst extname))))+ dst))++;; Helper for emitting `MInst.LoadExtNameNear` instructions.+(decl load_ext_name_near (BoxExternalName i64) Reg)+(rule (load_ext_name_near extname offset)+ (let ((dst WritableReg (temp_writable_reg $I64))+ (_ Unit (emit (MInst.LoadExtNameNear dst extname offset))))+ dst))++;; Helper for emitting `MInst.LoadExtNameFar` instructions.+(decl load_ext_name_far (BoxExternalName i64) Reg)+(rule (load_ext_name_far extname offset)+ (let ((dst WritableReg (temp_writable_reg $I64))+ (_ Unit (emit (MInst.LoadExtNameFar dst extname offset))))+ dst))(decl elf_tls_get_addr (ExternalName) Reg)(rule (elf_tls_get_addr name)diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rsindex 8dc2a95149..cfc062dc10 100644--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs@@ -8,6 +8,7 @@ use crate::isa::riscv64::lower::isle::generated_code::{use cranelift_control::ControlPlane;pub struct EmitInfo {+ #[expect(dead_code, reason = "may want to be used in the future")]shared_flag: settings::Flags,isa_flags: super::super::riscv_settings::Flags,}@@ -170,7 +171,9 @@ impl Inst {| Inst::ReturnCallInd { .. }| Inst::Jal { .. }| Inst::CondBr { .. }- | Inst::LoadExtName { .. }+ | Inst::LoadExtNameGot { .. }+ | Inst::LoadExtNameNear { .. }+ | Inst::LoadExtNameFar { .. }| Inst::ElfTlsGetAddr { .. }| Inst::LoadAddr { .. }| Inst::Mov { .. }@@ -1999,76 +2002,108 @@ impl Inst {.emit(sink, emit_info, state);}- &Inst::LoadExtName {+ &Inst::LoadExtNameGot { rd, ref name } => {+ // Load a PC-relative address into a register.+ // RISC-V does this slightly differently from other arches. We emit a relocation+ // with a label, instead of the symbol itself.+ //+ // See: https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses+ //+ // Emit the following code:+ // label:+ // auipc rd, 0 # R_RISCV_GOT_HI20 (symbol_name)+ // ld rd, rd, 0 # R_RISCV_PCREL_LO12_I (label)++ // Create the label that is going to be published to the final binary object.+ let auipc_label = sink.get_label();+ sink.bind_label(auipc_label, &mut state.ctrl_plane);++ // Get the current PC.+ sink.add_reloc(Reloc::RiscvGotHi20, &**name, 0);+ Inst::Auipc {+ rd,+ imm: Imm20::from_i32(0),+ }+ .emit_uncompressed(sink, emit_info, state, start_off);++ // The `ld` here, points to the `auipc` label instead of directly to the symbol.+ sink.add_reloc(Reloc::RiscvPCRelLo12I, &auipc_label, 0);+ Inst::Load {+ rd,+ op: LoadOP::Ld,+ flags: MemFlags::trusted(),+ from: AMode::RegOffset(rd.to_reg(), 0),+ }+ .emit_uncompressed(sink, emit_info, state, start_off);+ }++ &Inst::LoadExtNameFar {rd,ref name,offset,} => {- if emit_info.shared_flag.is_pic() {- // Load a PC-relative address into a register.- // RISC-V does this slightly differently from other arches. We emit a relocation- // with a label, instead of the symbol itself.- //- // See: https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses- //- // Emit the following code:- // label:- // auipc rd, 0 # R_RISCV_GOT_HI20 (symbol_name)- // ld rd, rd, 0 # R_RISCV_PCREL_LO12_I (label)-- // Create the label that is going to be published to the final binary object.- let auipc_label = sink.get_label();- sink.bind_label(auipc_label, &mut state.ctrl_plane);-- // Get the current PC.- sink.add_reloc(Reloc::RiscvGotHi20, &**name, 0);- Inst::Auipc {- rd,- imm: Imm20::from_i32(0),- }- .emit_uncompressed(sink, emit_info, state, start_off);+ // In the non PIC sequence we relocate the absolute address into+ // a preallocated space, load it into a register and jump over+ // it.+ //+ // Emit the following code:+ // ld rd, label_data+ // j label_end+ // label_data:+ // <8 byte space> # ABS8+ // label_end:- // The `ld` here, points to the `auipc` label instead of directly to the symbol.- sink.add_reloc(Reloc::RiscvPCRelLo12I, &auipc_label, 0);- Inst::Load {- rd,- op: LoadOP::Ld,- flags: MemFlags::trusted(),- from: AMode::RegOffset(rd.to_reg(), 0),- }- .emit_uncompressed(sink, emit_info, state, start_off);- } else {- // In the non PIC sequence we relocate the absolute address into- // a prealocatted space, load it into a register and jump over it.- //- // Emit the following code:- // ld rd, label_data- // j label_end- // label_data:- // <8 byte space> # ABS8- // label_end:-- let label_data = sink.get_label();- let label_end = sink.get_label();-- // Load the value from a label- Inst::Load {- rd,- op: LoadOP::Ld,- flags: MemFlags::trusted(),- from: AMode::Label(label_data),- }- .emit(sink, emit_info, state);+ let label_data = sink.get_label();+ let label_end = sink.get_label();- // Jump over the data- Inst::gen_jump(label_end).emit(sink, emit_info, state);+ // Load the value from a label+ Inst::Load {+ rd,+ op: LoadOP::Ld,+ flags: MemFlags::trusted(),+ from: AMode::Label(label_data),+ }+ .emit(sink, emit_info, state);- sink.bind_label(label_data, &mut state.ctrl_plane);- sink.add_reloc(Reloc::Abs8, name.as_ref(), offset);- sink.put8(0);+ // Jump over the data+ Inst::gen_jump(label_end).emit(sink, emit_info, state);- sink.bind_label(label_end, &mut state.ctrl_plane);+ sink.bind_label(label_data, &mut state.ctrl_plane);+ sink.add_reloc(Reloc::Abs8, name.as_ref(), offset);+ sink.put8(0);++ sink.bind_label(label_end, &mut state.ctrl_plane);+ }++ &Inst::LoadExtNameNear {+ rd,+ ref name,+ offset,+ } => {+ // Emit the following code:+ // label:+ // auipc rd, 0 # R_RISCV_PCREL_HI20 (symbol_name)+ // ld rd, rd, 0 # R_RISCV_PCREL_LO12_I (label)++ let auipc_label = sink.get_label();+ sink.bind_label(auipc_label, &mut state.ctrl_plane);++ // Get the current PC.+ sink.add_reloc(Reloc::RiscvPCRelHi20, &**name, offset);+ Inst::Auipc {+ rd,+ imm: Imm20::from_i32(0),+ }+ .emit_uncompressed(sink, emit_info, state, start_off);++ sink.add_reloc(Reloc::RiscvPCRelLo12I, &auipc_label, 0);+ Inst::AluRRImm12 {+ alu_op: AluOPRRI::Addi,+ rd,+ rs: rd.to_reg(),+ imm12: Imm12::ZERO,}+ .emit_uncompressed(sink, emit_info, state, start_off);}&Inst::ElfTlsGetAddr { rd, ref name } => {diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rsindex 6b74ed2af7..070da931be 100644--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs@@ -390,7 +390,9 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {collector.reg_use(rs1);collector.reg_use(rs2);}- Inst::LoadExtName { rd, .. } => {+ Inst::LoadExtNameGot { rd, .. }+ | Inst::LoadExtNameNear { rd, .. }+ | Inst::LoadExtNameFar { rd, .. } => {collector.reg_def(rd);}Inst::ElfTlsGetAddr { rd, .. } => {@@ -1411,13 +1413,25 @@ impl Inst {format!("{op_name} {rd},{src},({addr})")}}- &MInst::LoadExtName {+ &MInst::LoadExtNameGot { rd, ref name } => {+ let rd = format_reg(rd.to_reg());+ format!("load_ext_name_got {rd},{}", name.display(None))+ }+ &MInst::LoadExtNameNear {+ rd,+ ref name,+ offset,+ } => {+ let rd = format_reg(rd.to_reg());+ format!("load_ext_name_near {rd},{}{offset:+}", name.display(None))+ }+ &MInst::LoadExtNameFar {rd,ref name,offset,} => {let rd = format_reg(rd.to_reg());- format!("load_sym {},{}{:+}", rd, name.display(None), offset)+ format!("load_ext_name_far {rd},{}{offset:+}", name.display(None))}&Inst::ElfTlsGetAddr { rd, ref name } => {let rd = format_reg(rd.to_reg());diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isleindex 39d95e8978..65e81c53b4 100644--- a/cranelift/codegen/src/isa/riscv64/lower.isle+++ b/cranelift/codegen/src/isa/riscv64/lower.isle@@ -2379,8 +2379,8 @@;;;;; Rules for `func_addr`;;;;;;;;;(rule- (lower (func_addr (func_ref_data _ name _)))- (load_ext_name name 0))+ (lower (func_addr (func_ref_data _ name dist)))+ (load_ext_name name 0 dist));;;;; Rules for `fcvt_to_uint`;;;;;;;;;@@ -2538,8 +2538,8 @@;;;;; Rules for `symbol_value`;;;;;;;;;(rule- (lower (symbol_value (symbol_value_data name _ offset)))- (load_ext_name name offset))+ (lower (symbol_value (symbol_value_data name dist offset)))+ (load_ext_name name offset dist));;;;; Rules for `tls_value` ;;;;;;;;;;;;;;@@ -2679,12 +2679,12 @@output));; Direct call to an out-of-range function (implicitly via pointer).-(rule (lower (call (func_ref_data sig_ref name _) args))+(rule (lower (call (func_ref_data sig_ref name dist) args))(let ((output ValueRegsVec (gen_call_output sig_ref))(abi Sig (abi_sig sig_ref))(uses CallArgList (gen_call_args abi args))(defs CallRetList (gen_call_rets abi output))- (target Reg (load_ext_name name 0))+ (target Reg (load_ext_name name 0 dist))(info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none)))(_ Unit (emit_side_effect (call_ind_impl info))))output))@@ -2712,12 +2712,12 @@(emit_side_effect (call_impl info))));; Direct call to an out-of-range function (implicitly via pointer).-(rule (lower_branch (try_call (func_ref_data sig_ref name _) args et) targets)+(rule (lower_branch (try_call (func_ref_data sig_ref name dist) args et) targets)(let ((abi Sig (abi_sig sig_ref))(trycall OptionTryCallInfo (try_call_info et targets))(uses CallArgList (gen_call_args abi args))(defs CallRetList (gen_try_call_rets abi))- (target Reg (load_ext_name name 0))+ (target Reg (load_ext_name name 0 dist))(info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall)))(emit_side_effect (call_ind_impl info))))@@ -2742,10 +2742,10 @@(side_effect (return_call_impl info))));; Direct call to an out-of-range function (implicitly via pointer).-(rule (lower (return_call (func_ref_data sig_ref name _) args))+(rule (lower (return_call (func_ref_data sig_ref name dist) args))(let ((abi Sig (abi_sig sig_ref))(uses CallArgList (gen_return_call_args abi args))- (target Reg (load_ext_name name 0))+ (target Reg (load_ext_name name 0 dist))(info BoxReturnCallIndInfo (gen_return_call_ind_info abi target uses)))(side_effect (return_call_ind_impl info))))diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rsindex 2c17862c0c..fab00cff25 100644--- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs+++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs@@ -575,15 +575,6 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>fn store_op(&mut self, ty: Type) -> StoreOP {StoreOP::from_type(ty)}- fn load_ext_name(&mut self, name: ExternalName, offset: i64) -> Reg {- let tmp = self.temp_writable_reg(I64);- self.emit(&MInst::LoadExtName {- rd: tmp,- name: Box::new(name),- offset,- });- tmp.to_reg()- }fn gen_stack_addr(&mut self, slot: StackSlot, offset: Offset32) -> Reg {let result = self.temp_writable_reg(I64);@@ -727,6 +718,10 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>_ => unimplemented!(),}}++ fn is_pic(&mut self) -> bool {+ self.backend.flags.is_pic()+ }}/// The main entry point for lowering with ISLE.diff --git a/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif b/cranelift/filetests/filetests/isa/riscv64/atomic_store.clifindex f3024e0727..3f69cb1a9d 100644--- a/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif+++ b/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif@@ -29,17 +29,14 @@ block0(v0: i64):; VCode:; block0:-; load_sym a2,%sym+0+; load_ext_name_near a2,%sym+0; atomic_store.i64 a0,(a2); ret;; Disassembled:; block0: ; offset 0x0-; auipc a2, 0-; ld a2, 0xc(a2)-; j 0xc-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %sym 0-; .byte 0x00, 0x00, 0x00, 0x00+; auipc a2, 0 ; reloc_external RiscvPCRelHi20 %sym 0+; mv a2, a2 ; reloc_external RiscvPCRelLo12I func+0 0; fence rw, w; sd a0, 0(a2) ; trap: heap_oob; ret@@ -93,17 +90,14 @@ block0(v0: i32):; VCode:; block0:-; load_sym a2,%sym+0+; load_ext_name_near a2,%sym+0; atomic_store.i32 a0,(a2); ret;; Disassembled:; block0: ; offset 0x0-; auipc a2, 0-; ld a2, 0xc(a2)-; j 0xc-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %sym 0-; .byte 0x00, 0x00, 0x00, 0x00+; auipc a2, 0 ; reloc_external RiscvPCRelHi20 %sym 0+; mv a2, a2 ; reloc_external RiscvPCRelLo12I func+0 0; fence rw, w; sw a0, 0(a2) ; trap: heap_oob; retdiff --git a/cranelift/filetests/filetests/isa/riscv64/call.clif b/cranelift/filetests/filetests/isa/riscv64/call.clifindex ee0690f89a..2acf93320b 100644--- a/cranelift/filetests/filetests/isa/riscv64/call.clif+++ b/cranelift/filetests/filetests/isa/riscv64/call.clif@@ -17,7 +17,7 @@ block0(v0: i64):; sd fp,0(sp); mv fp,sp; block0:-; load_sym a3,%g+0+; load_ext_name_far a3,%g+0; callind a3; ld ra,8(sp); ld fp,0(sp)@@ -57,7 +57,7 @@ block0(v0: i32):; mv fp,sp; block0:; slli a0,a0,32; srli a0,a0,32-; load_sym a5,%g+0+; load_ext_name_far a5,%g+0; callind a5; ld ra,8(sp); ld fp,0(sp)@@ -115,7 +115,7 @@ block0(v0: i32):; mv fp,sp; block0:; slli a0,a0,32; srai a0,a0,32-; load_sym a5,%g+0+; load_ext_name_far a5,%g+0; callind a5; ld ra,8(sp); ld fp,0(sp)@@ -178,7 +178,7 @@ block0(v0: i8):; li a7,42; slli a4,a0,56; srai a4,a4,56; sd a4,0(sp)-; load_sym s1,%g+0+; load_ext_name_far s1,%g+0; mv a0,a7; mv a1,a7; mv a2,a7@@ -294,24 +294,24 @@ block0:; fsd fs2,16(sp); fsd fs4,8(sp); block0:-; load_sym a3,%g0+0+; load_ext_name_far a3,%g0+0; callind a3; fmv.d fs0,fa0-; load_sym a3,%g1+0+; load_ext_name_far a3,%g1+0; callind a3; fmv.d fs2,fa0-; load_sym a3,%g1+0+; load_ext_name_far a3,%g1+0; callind a3; fmv.d fs4,fa0-; load_sym a3,%g2+0+; load_ext_name_far a3,%g2+0; callind a3-; load_sym a4,%g3+0+; load_ext_name_far a4,%g3+0; fmv.d fa0,fs0; callind a4-; load_sym a5,%g4+0+; load_ext_name_far a5,%g4+0; fmv.d fa0,fs2; callind a5-; load_sym a0,%g4+0+; load_ext_name_far a0,%g4+0; fmv.d fa0,fs4; callind a0; fld fs0,24(sp)@@ -424,7 +424,7 @@ block0(v0: i64):; mv fp,sp; block0:; li a2,42-; load_sym a4,%f11+0+; load_ext_name_far a4,%f11+0; mv a1,a0; mv a0,a2; callind a4@@ -487,7 +487,7 @@ block0(v0: i64):; mv fp,sp; block0:; li a2,42-; load_sym a4,%f12+0+; load_ext_name_far a4,%f12+0; mv a1,a0; mv a0,a2; callind a4@@ -550,7 +550,7 @@ block0(v0: i64):; mv fp,sp; block0:; li a2,42-; load_sym a4,%f13+0+; load_ext_name_far a4,%f13+0; mv a1,a0; mv a0,a2; callind a4@@ -630,7 +630,7 @@ block0(v0: i128, v1: i64):; sd s3,16(sp); block0:; sd a1,0(sp)-; load_sym s1,%f14+0+; load_ext_name_far s1,%f14+0; mv a5,a1; mv a6,a2; mv a7,a0@@ -729,7 +729,7 @@ block0(v0: i128, v1: i64):; sd s3,16(sp); block0:; sd a1,0(sp)-; load_sym s1,%f15+0+; load_ext_name_far s1,%f15+0; mv a5,a1; mv a6,a2; mv a7,a0diff --git a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif b/cranelift/filetests/filetests/isa/riscv64/exceptions.clifindex e80453fc37..b4b172ee8d 100644--- a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif+++ b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif@@ -274,7 +274,7 @@ function %f2(i32) -> i32, f32, f64 {; slli a1,a5,40; fmv.d.x fa1,a1; fsd fa1,0(slot)-; load_sym a1,%g+0+; load_ext_name_far a1,%g+0; callind a1; j MachLabel(1); catch [default: MachLabel(2)]; block1:; li a0,1@@ -502,7 +502,7 @@ function %f4(i64, i32) -> i32, f32, f64 {; slli a2,a0,40; fmv.d.x fa1,a2; fsd fa1,16(slot)-; load_sym a2,%g+0+; load_ext_name_far a2,%g+0; mv a0,a1; sd a1,0(slot); callind a2; j MachLabel(3); catch [context stack1, tag0: MachLabel(1), tag1: MachLabel(2), context stack0, tag0: MachLabel(4)]diff --git a/cranelift/filetests/filetests/isa/riscv64/fuzzbug-60035.clif b/cranelift/filetests/filetests/isa/riscv64/fuzzbug-60035.clifindex 8c5d54cea4..107082b8f1 100644--- a/cranelift/filetests/filetests/isa/riscv64/fuzzbug-60035.clif+++ b/cranelift/filetests/filetests/isa/riscv64/fuzzbug-60035.clif@@ -20,7 +20,7 @@ block0:; addi sp,sp,-16; sd s1,8(sp); block0:-; load_sym s1,userextname0+0+; load_ext_name_far s1,userextname0+0; callind s1; callind s1; ld s1,8(sp)diff --git a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clifindex d5d1d7b574..6bbd6bdf82 100644--- a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif+++ b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif@@ -35,7 +35,7 @@ block0(v0: i64):; sd fp,0(sp); mv fp,sp; block0:-; load_sym t0,%callee_i64+0+; load_ext_name_far t0,%callee_i64+0; return_call_ind t0 new_stack_arg_size:0 a0=a0;; Disassembled:@@ -72,7 +72,7 @@ block0(v0: i64):; sd fp,0(sp); mv fp,sp; block0:-; load_sym t0,%callee_i64+0+; load_ext_name_near t0,%callee_i64+0; return_call_ind t0 new_stack_arg_size:0 a0=a0;; Disassembled:@@ -82,11 +82,8 @@ block0(v0: i64):; sd s0, 0(sp); mv s0, sp; block1: ; offset 0x10-; auipc t0, 0-; ld t0, 0xc(t0)-; j 0xc-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_i64 0-; .byte 0x00, 0x00, 0x00, 0x00+; auipc t0, 0 ; reloc_external RiscvPCRelHi20 %callee_i64 0+; mv t0, t0 ; reloc_external RiscvPCRelLo12I func+16 0; ld ra, 8(sp); ld s0, 0(sp); addi sp, sp, 0x10@@ -132,7 +129,7 @@ block0(v0: f64):; sd fp,0(sp); mv fp,sp; block0:-; load_sym t0,%callee_f64+0+; load_ext_name_far t0,%callee_f64+0; return_call_ind t0 new_stack_arg_size:0 fa0=fa0;; Disassembled:@@ -188,7 +185,7 @@ block0(v0: i8):; sd fp,0(sp); mv fp,sp; block0:-; load_sym t0,%callee_i8+0+; load_ext_name_far t0,%callee_i8+0; return_call_ind t0 new_stack_arg_size:0 a0=a0;; Disassembled:@@ -296,7 +293,7 @@ block0:; li s1,125; li a0,130; li a1,135-; load_sym a2,%tail_callee_stack_args+0+; load_ext_name_far a2,%tail_callee_stack_args+0; sd s2,-144(incoming_arg); sd s3,-136(incoming_arg); sd s4,-128(incoming_arg)diff --git a/cranelift/filetests/filetests/isa/riscv64/return-call.clif b/cranelift/filetests/filetests/isa/riscv64/return-call.clifindex 2a47ad837a..3252d60973 100644--- a/cranelift/filetests/filetests/isa/riscv64/return-call.clif+++ b/cranelift/filetests/filetests/isa/riscv64/return-call.clif@@ -33,7 +33,7 @@ block0(v0: i64):; sd fp,0(sp); mv fp,sp; block0:-; load_sym t0,%callee_i64+0+; load_ext_name_far t0,%callee_i64+0; return_call_ind t0 new_stack_arg_size:0 a0=a0;; Disassembled:@@ -121,7 +121,7 @@ block0(v0: f64):; sd fp,0(sp); mv fp,sp; block0:-; load_sym t0,%callee_f64+0+; load_ext_name_far t0,%callee_f64+0; return_call_ind t0 new_stack_arg_size:0 fa0=fa0;; Disassembled:@@ -175,7 +175,7 @@ block0(v0: i8):; sd fp,0(sp); mv fp,sp; block0:-; load_sym t0,%callee_i8+0+; load_ext_name_far t0,%callee_i8+0; return_call_ind t0 new_stack_arg_size:0 a0=a0;; Disassembled:@@ -331,7 +331,7 @@ block0:; sd s1,-24(incoming_arg); sd a0,-16(incoming_arg); sd a1,-8(incoming_arg)-; load_sym t0,%tail_callee_stack_args+0+; load_ext_name_far t0,%tail_callee_stack_args+0; ld a0,8(slot); ld a1,0(slot); return_call_ind t0 new_stack_arg_size:144 a0=a0 a1=a1 a2=a2 a3=a3 a4=a4 a5=a5 a6=a6 a7=a7@@ -610,7 +610,7 @@ block2:; sd s7,-32(incoming_arg); sd s6,-24(incoming_arg); sd a0,-16(incoming_arg)-; load_sym t0,%different_callee2+0+; load_ext_name_far t0,%different_callee2+0; ld a0,16(slot); ld a1,8(slot); ld a2,0(slot)@@ -635,7 +635,7 @@ block2:; sd s8,-24(incoming_arg); sd s7,-16(incoming_arg); sd s6,-8(incoming_arg)-; load_sym t0,%different_callee1+0+; load_ext_name_far t0,%different_callee1+0; ld a1,8(slot); ld a2,0(slot); return_call_ind t0 new_stack_arg_size:144 a0=a0 a1=a1 a2=a2 a3=a3 a4=a4 a5=a5 a6=a6 a7=a7diff --git a/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif b/cranelift/filetests/filetests/isa/riscv64/stack-limit.clifindex fe934fc7cf..4f494f48bf 100644--- a/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif+++ b/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif@@ -54,7 +54,7 @@ block0(v0: i64):; addi t6,t6,16; trap_if stk_ovf##(sp ult t6); block0:-; load_sym a0,%foo+0+; load_ext_name_far a0,%foo+0; callind a0; ld ra,8(sp); ld fp,0(sp)diff --git a/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif b/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clifindex 049e9717d5..1e4d60a141 100644--- a/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif+++ b/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif@@ -13,7 +13,7 @@ block0:; VCode:; block0:-; load_sym a0,%my_global+0+; load_ext_name_got a0,%my_global; ret;; Disassembled:diff --git a/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif b/cranelift/filetests/filetests/isa/riscv64/symbol-value.clifindex a43878ae8a..31f25bac43 100644--- a/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif+++ b/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif@@ -2,25 +2,197 @@ test compile precise-outputset unwind_info=falsetarget riscv64-function %f() -> i64 {- gv0 = symbol %my_global+function %func_addr() -> i64 {+ fn0 = %func0(i64) -> i64block0:- v0 = symbol_value.i64 gv0- return v0+ v0 = func_addr.i64 fn0+ return v0}; VCode:+; addi sp,sp,-16+; sd ra,8(sp)+; sd fp,0(sp)+; mv fp,sp; block0:-; load_sym a0,%my_global+0+; load_ext_name_far a0,%func0+0+; ld ra,8(sp)+; ld fp,0(sp)+; addi sp,sp,16; ret;; Disassembled:; block0: ; offset 0x0+; addi sp, sp, -0x10+; sd ra, 8(sp)+; sd s0, 0(sp)+; mv s0, sp+; block1: ; offset 0x10; auipc a0, 0; ld a0, 0xc(a0); j 0xc-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %my_global 0+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %func0 0; .byte 0x00, 0x00, 0x00, 0x00+; ld ra, 8(sp)+; ld s0, 0(sp)+; addi sp, sp, 0x10+; ret++function %colocated_func_addr() -> i64 {+ fn0 = colocated %func0(i64) -> i64++block0:+ v0 = func_addr.i64 fn0+ return v0+}++; VCode:+; addi sp,sp,-16+; sd ra,8(sp)+; sd fp,0(sp)+; mv fp,sp+; block0:+; load_ext_name_near a0,%func0+0+; ld ra,8(sp)+; ld fp,0(sp)+; addi sp,sp,16+; ret+;+; Disassembled:+; block0: ; offset 0x0+; addi sp, sp, -0x10+; sd ra, 8(sp)+; sd s0, 0(sp)+; mv s0, sp+; block1: ; offset 0x10+; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %func0 0+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+16 0+; ld ra, 8(sp)+; ld s0, 0(sp)+; addi sp, sp, 0x10+; ret++function %symbol_value() -> i64 {+ gv0 = symbol %global0++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_far a0,%global0+0+; ret+;+; Disassembled:+; block0: ; offset 0x0+; auipc a0, 0+; ld a0, 0xc(a0)+; j 0xc+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 0+; .byte 0x00, 0x00, 0x00, 0x00+; ret++function %symbol_value_plus_offset() -> i64 {+ gv0 = symbol %global0+123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_far a0,%global0+123+; ret+;+; Disassembled:+; block0: ; offset 0x0+; auipc a0, 0+; ld a0, 0xc(a0)+; j 0xc+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 123+; .byte 0x00, 0x00, 0x00, 0x00+; ret++function %symbol_value_minus_offset() -> i64 {+ gv0 = symbol %global0-123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_far a0,%global0-123+; ret+;+; Disassembled:+; block0: ; offset 0x0+; auipc a0, 0+; ld a0, 0xc(a0)+; j 0xc+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 -123+; .byte 0x00, 0x00, 0x00, 0x00+; ret++function %colocated_symbol_value() -> i64 {+ gv0 = symbol colocated %global0++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_near a0,%global0+0+; ret+;+; Disassembled:+; block0: ; offset 0x0+; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 0+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0+; ret++function %colocated_symbol_value_plus_offset() -> i64 {+ gv0 = symbol colocated %global0+123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_near a0,%global0+123+; ret+;+; Disassembled:+; block0: ; offset 0x0+; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 123+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0+; ret++function %colocated_symbol_value_minus_offset() -> i64 {+ gv0 = symbol colocated %global0-123++block0:+ v0 = symbol_value.i64 gv0+ return v0+}++; VCode:+; block0:+; load_ext_name_near a0,%global0-123+; ret+;+; Disassembled:+; block0: ; offset 0x0+; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 -123+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0; retdiff --git a/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif b/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clifindex 2becbf9aff..0e67ebbfc4 100644--- a/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif+++ b/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif@@ -134,7 +134,7 @@ block0:; sd s2,120(sp); sd a0,128(sp); sd a1,136(sp)-; load_sym s3,%tail_callee_stack_args+0+; load_ext_name_far s3,%tail_callee_stack_args+0; ld a0,8(slot); ld a1,0(slot); callind s3@@ -489,7 +489,7 @@ block0:; sd s11,312(sp); block0:; load_addr a0,0(sp)-; load_sym a4,%tail_callee_stack_rets+0+; load_ext_name_far a4,%tail_callee_stack_rets+0; callind a4; ld a0,96(slot); ld s1,392(sp)@@ -869,7 +869,7 @@ block0:; sd a1,136(sp); sd a2,144(sp); load_addr a0,160(sp)-; load_sym t1,%tail_callee_stack_args_and_rets+0+; load_ext_name_far t1,%tail_callee_stack_args_and_rets+0; ld a1,0(slot); ld a2,96(slot); callind t1diff --git a/cranelift/filetests/filetests/isa/riscv64/zca.clif b/cranelift/filetests/filetests/isa/riscv64/zca.clifindex 2311ff1636..e61b377432 100644--- a/cranelift/filetests/filetests/isa/riscv64/zca.clif+++ b/cranelift/filetests/filetests/isa/riscv64/zca.clif@@ -183,7 +183,7 @@ block0(v0: i8):; sd fp,0(sp); mv fp,sp; block0:-; load_sym t0,%callee_i8+0+; load_ext_name_far t0,%callee_i8+0; return_call_ind t0 new_stack_arg_size:0 a0=a0;; Disassembled:diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rsindex 4a477be0a4..18aaf44f5b 100644--- a/cranelift/jit/src/backend.rs+++ b/cranelift/jit/src/backend.rs@@ -212,7 +212,7 @@ impl JITModule {}fn get_address(&self, name: &ModuleRelocTarget) -> *const u8 {- match *name {+ match name {ModuleRelocTarget::User { .. } => {let (name, linkage) = if ModuleDeclarations::is_function(name) {let func_id = FuncId::from_name(name);@@ -244,12 +244,18 @@ impl JITModule {panic!("can't resolve symbol {name}");}}- ModuleRelocTarget::LibCall(ref libcall) => {+ ModuleRelocTarget::LibCall(libcall) => {let sym = (self.libcall_names)(*libcall);self.lookup_symbol(&sym).unwrap_or_else(|| panic!("can't resolve libcall {sym}"))}- _ => panic!("invalid name"),+ ModuleRelocTarget::FunctionOffset(func_id, offset) => {+ match &self.compiled_functions[*func_id] {+ Some(compiled) => return compiled.ptr.wrapping_add(*offset as usize),+ None => todo!(),+ }+ }+ name => panic!("invalid name {name:?}"),}}diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rsindex b25917fdea..395bce41bc 100644--- a/cranelift/jit/src/compiled_blob.rs+++ b/cranelift/jit/src/compiled_blob.rs@@ -28,12 +28,15 @@ impl CompiledBlob {) {use std::ptr::write_unaligned;- for &ModuleReloc {- kind,- offset,- ref name,- addend,- } in &self.relocs+ for (+ i,+ &ModuleReloc {+ kind,+ offset,+ ref name,+ addend,+ },+ ) in self.relocs.iter().enumerate(){debug_assert!((offset as usize) < self.size);let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) };@@ -159,6 +162,43 @@ impl CompiledBlob {at.write_unaligned(at.read_unaligned().wrapping_add(pcrel));}}++ // See <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>+ // for why `0x800` is added here.+ Reloc::RiscvPCRelHi20 => {+ let base = get_address(name);+ let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };+ let pcrel = i32::try_from((what as isize) - (at as isize) + 0x800)+ .unwrap()+ .cast_unsigned();+ let at = at as *mut u32;+ unsafe {+ modify_inst32(at, |i| i | (pcrel & 0xfffff000));+ }+ }++ // The target of this relocation is the `auipc` preceding this+ // instruction which should be `RiscvPCRelHi20`, and the actual+ // target that we're relocating against is the target of that+ // relocation. Assume for now that the previous relocation is+ // the target of this relocation, and then use that.+ Reloc::RiscvPCRelLo12I => {+ let prev_reloc = &self.relocs[i - 1];+ assert_eq!(prev_reloc.kind, Reloc::RiscvPCRelHi20);+ let lo_target = get_address(name);+ let hi_address =+ unsafe { self.ptr.offset(isize::try_from(prev_reloc.offset).unwrap()) };+ assert_eq!(lo_target, hi_address);+ let hi_target = get_address(&prev_reloc.name);+ let pcrel = i32::try_from((hi_target as isize) - (hi_address as isize))+ .unwrap()+ .cast_unsigned();+ let at = at as *mut u32;+ unsafe {+ modify_inst32(at, |i| i | ((pcrel & 0xfff) << 20));+ }+ }+other => unimplemented!("unimplemented reloc {other:?}"),}}