Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

riscv64: Implement near relocations (#11585)

Alex Crichton13b300deac44
diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs
index d1236a923c..cd34c0b3ff 100644
--- a/cranelift/codegen/src/binemit/mod.rs
+++ b/cranelift/codegen/src/binemit/mod.rs
@@ -120,6 +120,12 @@ pub enum Reloc {
/// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
RiscvGotHi20,
+ /// High 20 bits of a 32-bit PC-relative offset relocation
+ ///
+ /// This is the `R_RISCV_PCREL_HI20` relocation from the RISC-V ELF psABI document.
+ /// <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
+ RiscvPCRelHi20,
+
/// s390x TLS GD64 - 64-bit offset of tls_index for GD symbol in GOT
S390xTlsGd64,
/// s390x TLS GDCall - marker to enable optimization of TLS calls
@@ -152,6 +158,7 @@ impl fmt::Display for Reloc {
Self::RiscvCallPlt => write!(f, "RiscvCallPlt"),
Self::RiscvTlsGdHi20 => write!(f, "RiscvTlsGdHi20"),
Self::RiscvGotHi20 => write!(f, "RiscvGotHi20"),
+ Self::RiscvPCRelHi20 => write!(f, "RiscvPCRelHi20"),
Self::RiscvPCRelLo12I => write!(f, "RiscvPCRelLo12I"),
Self::ElfX86_64TlsGd => write!(f, "ElfX86_64TlsGd"),
Self::MachOX86_64Tlv => write!(f, "MachOX86_64Tlv"),
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 08283480e5..6e020d6a24 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -139,7 +139,14 @@
(kind IntegerCompare))
;; Load an inline symbol reference.
- (LoadExtName
+ (LoadExtNameGot
+ (rd WritableReg)
+ (name BoxExternalName))
+ (LoadExtNameNear
+ (rd WritableReg)
+ (name BoxExternalName)
+ (offset i64))
+ (LoadExtNameFar
(rd WritableReg)
(name BoxExternalName)
(offset i64))
@@ -2750,8 +2757,43 @@
;;;; load extern name
-(decl load_ext_name (ExternalName i64) Reg)
-(extern constructor load_ext_name load_ext_name)
+(decl load_ext_name (ExternalName i64 RelocDistance) Reg)
+(rule (load_ext_name name offset _dist)
+ (if-let true (is_pic))
+ (rv_add (load_ext_name_got name) (imm $I64 (i64_cast_unsigned offset))))
+(rule 1 (load_ext_name name 0 _dist)
+ (if-let true (is_pic))
+ (load_ext_name_got name))
+(rule (load_ext_name name offset (RelocDistance.Near))
+ (if-let false (is_pic))
+ (load_ext_name_near name offset))
+(rule (load_ext_name name offset (RelocDistance.Far))
+ (if-let false (is_pic))
+ (load_ext_name_far name offset))
+
+(decl pure is_pic () bool)
+(extern constructor is_pic is_pic)
+
+;; Helper for emitting `MInst.LoadExtNameGot` instructions.
+(decl load_ext_name_got (BoxExternalName) Reg)
+(rule (load_ext_name_got extname)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.LoadExtNameGot dst extname))))
+ dst))
+
+;; Helper for emitting `MInst.LoadExtNameNear` instructions.
+(decl load_ext_name_near (BoxExternalName i64) Reg)
+(rule (load_ext_name_near extname offset)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.LoadExtNameNear dst extname offset))))
+ dst))
+
+;; Helper for emitting `MInst.LoadExtNameFar` instructions.
+(decl load_ext_name_far (BoxExternalName i64) Reg)
+(rule (load_ext_name_far extname offset)
+ (let ((dst WritableReg (temp_writable_reg $I64))
+ (_ Unit (emit (MInst.LoadExtNameFar dst extname offset))))
+ dst))
(decl elf_tls_get_addr (ExternalName) Reg)
(rule (elf_tls_get_addr name)
diff --git a/cranelift/codegen/src/isa/riscv64/inst/emit.rs b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
index 8dc2a95149..cfc062dc10 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/emit.rs
@@ -8,6 +8,7 @@ use crate::isa::riscv64::lower::isle::generated_code::{
use cranelift_control::ControlPlane;
pub struct EmitInfo {
+ #[expect(dead_code, reason = "may want to be used in the future")]
shared_flag: settings::Flags,
isa_flags: super::super::riscv_settings::Flags,
}
@@ -170,7 +171,9 @@ impl Inst {
| Inst::ReturnCallInd { .. }
| Inst::Jal { .. }
| Inst::CondBr { .. }
- | Inst::LoadExtName { .. }
+ | Inst::LoadExtNameGot { .. }
+ | Inst::LoadExtNameNear { .. }
+ | Inst::LoadExtNameFar { .. }
| Inst::ElfTlsGetAddr { .. }
| Inst::LoadAddr { .. }
| Inst::Mov { .. }
@@ -1999,76 +2002,108 @@ impl Inst {
.emit(sink, emit_info, state);
}
- &Inst::LoadExtName {
+ &Inst::LoadExtNameGot { rd, ref name } => {
+ // Load a PC-relative address into a register.
+ // RISC-V does this slightly differently from other arches. We emit a relocation
+ // with a label, instead of the symbol itself.
+ //
+ // See: https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
+ //
+ // Emit the following code:
+ // label:
+ // auipc rd, 0 # R_RISCV_GOT_HI20 (symbol_name)
+ // ld rd, rd, 0 # R_RISCV_PCREL_LO12_I (label)
+
+ // Create the label that is going to be published to the final binary object.
+ let auipc_label = sink.get_label();
+ sink.bind_label(auipc_label, &mut state.ctrl_plane);
+
+ // Get the current PC.
+ sink.add_reloc(Reloc::RiscvGotHi20, &**name, 0);
+ Inst::Auipc {
+ rd,
+ imm: Imm20::from_i32(0),
+ }
+ .emit_uncompressed(sink, emit_info, state, start_off);
+
+ // The `ld` here, points to the `auipc` label instead of directly to the symbol.
+ sink.add_reloc(Reloc::RiscvPCRelLo12I, &auipc_label, 0);
+ Inst::Load {
+ rd,
+ op: LoadOP::Ld,
+ flags: MemFlags::trusted(),
+ from: AMode::RegOffset(rd.to_reg(), 0),
+ }
+ .emit_uncompressed(sink, emit_info, state, start_off);
+ }
+
+ &Inst::LoadExtNameFar {
rd,
ref name,
offset,
} => {
- if emit_info.shared_flag.is_pic() {
- // Load a PC-relative address into a register.
- // RISC-V does this slightly differently from other arches. We emit a relocation
- // with a label, instead of the symbol itself.
- //
- // See: https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
- //
- // Emit the following code:
- // label:
- // auipc rd, 0 # R_RISCV_GOT_HI20 (symbol_name)
- // ld rd, rd, 0 # R_RISCV_PCREL_LO12_I (label)
-
- // Create the label that is going to be published to the final binary object.
- let auipc_label = sink.get_label();
- sink.bind_label(auipc_label, &mut state.ctrl_plane);
-
- // Get the current PC.
- sink.add_reloc(Reloc::RiscvGotHi20, &**name, 0);
- Inst::Auipc {
- rd,
- imm: Imm20::from_i32(0),
- }
- .emit_uncompressed(sink, emit_info, state, start_off);
+ // In the non PIC sequence we relocate the absolute address into
+ // a preallocated space, load it into a register and jump over
+ // it.
+ //
+ // Emit the following code:
+ // ld rd, label_data
+ // j label_end
+ // label_data:
+ // <8 byte space> # ABS8
+ // label_end:
- // The `ld` here, points to the `auipc` label instead of directly to the symbol.
- sink.add_reloc(Reloc::RiscvPCRelLo12I, &auipc_label, 0);
- Inst::Load {
- rd,
- op: LoadOP::Ld,
- flags: MemFlags::trusted(),
- from: AMode::RegOffset(rd.to_reg(), 0),
- }
- .emit_uncompressed(sink, emit_info, state, start_off);
- } else {
- // In the non PIC sequence we relocate the absolute address into
- // a prealocatted space, load it into a register and jump over it.
- //
- // Emit the following code:
- // ld rd, label_data
- // j label_end
- // label_data:
- // <8 byte space> # ABS8
- // label_end:
-
- let label_data = sink.get_label();
- let label_end = sink.get_label();
-
- // Load the value from a label
- Inst::Load {
- rd,
- op: LoadOP::Ld,
- flags: MemFlags::trusted(),
- from: AMode::Label(label_data),
- }
- .emit(sink, emit_info, state);
+ let label_data = sink.get_label();
+ let label_end = sink.get_label();
- // Jump over the data
- Inst::gen_jump(label_end).emit(sink, emit_info, state);
+ // Load the value from a label
+ Inst::Load {
+ rd,
+ op: LoadOP::Ld,
+ flags: MemFlags::trusted(),
+ from: AMode::Label(label_data),
+ }
+ .emit(sink, emit_info, state);
- sink.bind_label(label_data, &mut state.ctrl_plane);
- sink.add_reloc(Reloc::Abs8, name.as_ref(), offset);
- sink.put8(0);
+ // Jump over the data
+ Inst::gen_jump(label_end).emit(sink, emit_info, state);
- sink.bind_label(label_end, &mut state.ctrl_plane);
+ sink.bind_label(label_data, &mut state.ctrl_plane);
+ sink.add_reloc(Reloc::Abs8, name.as_ref(), offset);
+ sink.put8(0);
+
+ sink.bind_label(label_end, &mut state.ctrl_plane);
+ }
+
+ &Inst::LoadExtNameNear {
+ rd,
+ ref name,
+ offset,
+ } => {
+ // Emit the following code:
+ // label:
+ // auipc rd, 0 # R_RISCV_PCREL_HI20 (symbol_name)
+ // ld rd, rd, 0 # R_RISCV_PCREL_LO12_I (label)
+
+ let auipc_label = sink.get_label();
+ sink.bind_label(auipc_label, &mut state.ctrl_plane);
+
+ // Get the current PC.
+ sink.add_reloc(Reloc::RiscvPCRelHi20, &**name, offset);
+ Inst::Auipc {
+ rd,
+ imm: Imm20::from_i32(0),
+ }
+ .emit_uncompressed(sink, emit_info, state, start_off);
+
+ sink.add_reloc(Reloc::RiscvPCRelLo12I, &auipc_label, 0);
+ Inst::AluRRImm12 {
+ alu_op: AluOPRRI::Addi,
+ rd,
+ rs: rd.to_reg(),
+ imm12: Imm12::ZERO,
}
+ .emit_uncompressed(sink, emit_info, state, start_off);
}
&Inst::ElfTlsGetAddr { rd, ref name } => {
diff --git a/cranelift/codegen/src/isa/riscv64/inst/mod.rs b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
index 6b74ed2af7..070da931be 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/mod.rs
@@ -390,7 +390,9 @@ fn riscv64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
collector.reg_use(rs1);
collector.reg_use(rs2);
}
- Inst::LoadExtName { rd, .. } => {
+ Inst::LoadExtNameGot { rd, .. }
+ | Inst::LoadExtNameNear { rd, .. }
+ | Inst::LoadExtNameFar { rd, .. } => {
collector.reg_def(rd);
}
Inst::ElfTlsGetAddr { rd, .. } => {
@@ -1411,13 +1413,25 @@ impl Inst {
format!("{op_name} {rd},{src},({addr})")
}
}
- &MInst::LoadExtName {
+ &MInst::LoadExtNameGot { rd, ref name } => {
+ let rd = format_reg(rd.to_reg());
+ format!("load_ext_name_got {rd},{}", name.display(None))
+ }
+ &MInst::LoadExtNameNear {
+ rd,
+ ref name,
+ offset,
+ } => {
+ let rd = format_reg(rd.to_reg());
+ format!("load_ext_name_near {rd},{}{offset:+}", name.display(None))
+ }
+ &MInst::LoadExtNameFar {
rd,
ref name,
offset,
} => {
let rd = format_reg(rd.to_reg());
- format!("load_sym {},{}{:+}", rd, name.display(None), offset)
+ format!("load_ext_name_far {rd},{}{offset:+}", name.display(None))
}
&Inst::ElfTlsGetAddr { rd, ref name } => {
let rd = format_reg(rd.to_reg());
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 39d95e8978..65e81c53b4 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -2379,8 +2379,8 @@
;;;;; Rules for `func_addr`;;;;;;;;;
(rule
- (lower (func_addr (func_ref_data _ name _)))
- (load_ext_name name 0))
+ (lower (func_addr (func_ref_data _ name dist)))
+ (load_ext_name name 0 dist))
;;;;; Rules for `fcvt_to_uint`;;;;;;;;;
@@ -2538,8 +2538,8 @@
;;;;; Rules for `symbol_value`;;;;;;;;;
(rule
- (lower (symbol_value (symbol_value_data name _ offset)))
- (load_ext_name name offset))
+ (lower (symbol_value (symbol_value_data name dist offset)))
+ (load_ext_name name offset dist))
;;;;; Rules for `tls_value` ;;;;;;;;;;;;;;
@@ -2679,12 +2679,12 @@
output))
;; Direct call to an out-of-range function (implicitly via pointer).
-(rule (lower (call (func_ref_data sig_ref name _) args))
+(rule (lower (call (func_ref_data sig_ref name dist) args))
(let ((output ValueRegsVec (gen_call_output sig_ref))
(abi Sig (abi_sig sig_ref))
(uses CallArgList (gen_call_args abi args))
(defs CallRetList (gen_call_rets abi output))
- (target Reg (load_ext_name name 0))
+ (target Reg (load_ext_name name 0 dist))
(info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none)))
(_ Unit (emit_side_effect (call_ind_impl info))))
output))
@@ -2712,12 +2712,12 @@
(emit_side_effect (call_impl info))))
;; Direct call to an out-of-range function (implicitly via pointer).
-(rule (lower_branch (try_call (func_ref_data sig_ref name _) args et) targets)
+(rule (lower_branch (try_call (func_ref_data sig_ref name dist) args et) targets)
(let ((abi Sig (abi_sig sig_ref))
(trycall OptionTryCallInfo (try_call_info et targets))
(uses CallArgList (gen_call_args abi args))
(defs CallRetList (gen_try_call_rets abi))
- (target Reg (load_ext_name name 0))
+ (target Reg (load_ext_name name 0 dist))
(info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall)))
(emit_side_effect (call_ind_impl info))))
@@ -2742,10 +2742,10 @@
(side_effect (return_call_impl info))))
;; Direct call to an out-of-range function (implicitly via pointer).
-(rule (lower (return_call (func_ref_data sig_ref name _) args))
+(rule (lower (return_call (func_ref_data sig_ref name dist) args))
(let ((abi Sig (abi_sig sig_ref))
(uses CallArgList (gen_return_call_args abi args))
- (target Reg (load_ext_name name 0))
+ (target Reg (load_ext_name name 0 dist))
(info BoxReturnCallIndInfo (gen_return_call_ind_info abi target uses)))
(side_effect (return_call_ind_impl info))))
diff --git a/cranelift/codegen/src/isa/riscv64/lower/isle.rs b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
index 2c17862c0c..fab00cff25 100644
--- a/cranelift/codegen/src/isa/riscv64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/riscv64/lower/isle.rs
@@ -575,15 +575,6 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>
fn store_op(&mut self, ty: Type) -> StoreOP {
StoreOP::from_type(ty)
}
- fn load_ext_name(&mut self, name: ExternalName, offset: i64) -> Reg {
- let tmp = self.temp_writable_reg(I64);
- self.emit(&MInst::LoadExtName {
- rd: tmp,
- name: Box::new(name),
- offset,
- });
- tmp.to_reg()
- }
fn gen_stack_addr(&mut self, slot: StackSlot, offset: Offset32) -> Reg {
let result = self.temp_writable_reg(I64);
@@ -727,6 +718,10 @@ impl generated_code::Context for RV64IsleContext<'_, '_, MInst, Riscv64Backend>
_ => unimplemented!(),
}
}
+
+ fn is_pic(&mut self) -> bool {
+ self.backend.flags.is_pic()
+ }
}
/// The main entry point for lowering with ISLE.
diff --git a/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif b/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif
index f3024e0727..3f69cb1a9d 100644
--- a/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/atomic_store.clif
@@ -29,17 +29,14 @@ block0(v0: i64):
; VCode:
; block0:
-; load_sym a2,%sym+0
+; load_ext_name_near a2,%sym+0
; atomic_store.i64 a0,(a2)
; ret
;
; Disassembled:
; block0: ; offset 0x0
-; auipc a2, 0
-; ld a2, 0xc(a2)
-; j 0xc
-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %sym 0
-; .byte 0x00, 0x00, 0x00, 0x00
+; auipc a2, 0 ; reloc_external RiscvPCRelHi20 %sym 0
+; mv a2, a2 ; reloc_external RiscvPCRelLo12I func+0 0
; fence rw, w
; sd a0, 0(a2) ; trap: heap_oob
; ret
@@ -93,17 +90,14 @@ block0(v0: i32):
; VCode:
; block0:
-; load_sym a2,%sym+0
+; load_ext_name_near a2,%sym+0
; atomic_store.i32 a0,(a2)
; ret
;
; Disassembled:
; block0: ; offset 0x0
-; auipc a2, 0
-; ld a2, 0xc(a2)
-; j 0xc
-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %sym 0
-; .byte 0x00, 0x00, 0x00, 0x00
+; auipc a2, 0 ; reloc_external RiscvPCRelHi20 %sym 0
+; mv a2, a2 ; reloc_external RiscvPCRelLo12I func+0 0
; fence rw, w
; sw a0, 0(a2) ; trap: heap_oob
; ret
diff --git a/cranelift/filetests/filetests/isa/riscv64/call.clif b/cranelift/filetests/filetests/isa/riscv64/call.clif
index ee0690f89a..2acf93320b 100644
--- a/cranelift/filetests/filetests/isa/riscv64/call.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/call.clif
@@ -17,7 +17,7 @@ block0(v0: i64):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym a3,%g+0
+; load_ext_name_far a3,%g+0
; callind a3
; ld ra,8(sp)
; ld fp,0(sp)
@@ -57,7 +57,7 @@ block0(v0: i32):
; mv fp,sp
; block0:
; slli a0,a0,32; srli a0,a0,32
-; load_sym a5,%g+0
+; load_ext_name_far a5,%g+0
; callind a5
; ld ra,8(sp)
; ld fp,0(sp)
@@ -115,7 +115,7 @@ block0(v0: i32):
; mv fp,sp
; block0:
; slli a0,a0,32; srai a0,a0,32
-; load_sym a5,%g+0
+; load_ext_name_far a5,%g+0
; callind a5
; ld ra,8(sp)
; ld fp,0(sp)
@@ -178,7 +178,7 @@ block0(v0: i8):
; li a7,42
; slli a4,a0,56; srai a4,a4,56
; sd a4,0(sp)
-; load_sym s1,%g+0
+; load_ext_name_far s1,%g+0
; mv a0,a7
; mv a1,a7
; mv a2,a7
@@ -294,24 +294,24 @@ block0:
; fsd fs2,16(sp)
; fsd fs4,8(sp)
; block0:
-; load_sym a3,%g0+0
+; load_ext_name_far a3,%g0+0
; callind a3
; fmv.d fs0,fa0
-; load_sym a3,%g1+0
+; load_ext_name_far a3,%g1+0
; callind a3
; fmv.d fs2,fa0
-; load_sym a3,%g1+0
+; load_ext_name_far a3,%g1+0
; callind a3
; fmv.d fs4,fa0
-; load_sym a3,%g2+0
+; load_ext_name_far a3,%g2+0
; callind a3
-; load_sym a4,%g3+0
+; load_ext_name_far a4,%g3+0
; fmv.d fa0,fs0
; callind a4
-; load_sym a5,%g4+0
+; load_ext_name_far a5,%g4+0
; fmv.d fa0,fs2
; callind a5
-; load_sym a0,%g4+0
+; load_ext_name_far a0,%g4+0
; fmv.d fa0,fs4
; callind a0
; fld fs0,24(sp)
@@ -424,7 +424,7 @@ block0(v0: i64):
; mv fp,sp
; block0:
; li a2,42
-; load_sym a4,%f11+0
+; load_ext_name_far a4,%f11+0
; mv a1,a0
; mv a0,a2
; callind a4
@@ -487,7 +487,7 @@ block0(v0: i64):
; mv fp,sp
; block0:
; li a2,42
-; load_sym a4,%f12+0
+; load_ext_name_far a4,%f12+0
; mv a1,a0
; mv a0,a2
; callind a4
@@ -550,7 +550,7 @@ block0(v0: i64):
; mv fp,sp
; block0:
; li a2,42
-; load_sym a4,%f13+0
+; load_ext_name_far a4,%f13+0
; mv a1,a0
; mv a0,a2
; callind a4
@@ -630,7 +630,7 @@ block0(v0: i128, v1: i64):
; sd s3,16(sp)
; block0:
; sd a1,0(sp)
-; load_sym s1,%f14+0
+; load_ext_name_far s1,%f14+0
; mv a5,a1
; mv a6,a2
; mv a7,a0
@@ -729,7 +729,7 @@ block0(v0: i128, v1: i64):
; sd s3,16(sp)
; block0:
; sd a1,0(sp)
-; load_sym s1,%f15+0
+; load_ext_name_far s1,%f15+0
; mv a5,a1
; mv a6,a2
; mv a7,a0
diff --git a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif
index e80453fc37..b4b172ee8d 100644
--- a/cranelift/filetests/filetests/isa/riscv64/exceptions.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/exceptions.clif
@@ -274,7 +274,7 @@ function %f2(i32) -> i32, f32, f64 {
; slli a1,a5,40
; fmv.d.x fa1,a1
; fsd fa1,0(slot)
-; load_sym a1,%g+0
+; load_ext_name_far a1,%g+0
; callind a1; j MachLabel(1); catch [default: MachLabel(2)]
; block1:
; li a0,1
@@ -502,7 +502,7 @@ function %f4(i64, i32) -> i32, f32, f64 {
; slli a2,a0,40
; fmv.d.x fa1,a2
; fsd fa1,16(slot)
-; load_sym a2,%g+0
+; load_ext_name_far a2,%g+0
; mv a0,a1
; sd a1,0(slot)
; callind a2; j MachLabel(3); catch [context stack1, tag0: MachLabel(1), tag1: MachLabel(2), context stack0, tag0: MachLabel(4)]
diff --git a/cranelift/filetests/filetests/isa/riscv64/fuzzbug-60035.clif b/cranelift/filetests/filetests/isa/riscv64/fuzzbug-60035.clif
index 8c5d54cea4..107082b8f1 100644
--- a/cranelift/filetests/filetests/isa/riscv64/fuzzbug-60035.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/fuzzbug-60035.clif
@@ -20,7 +20,7 @@ block0:
; addi sp,sp,-16
; sd s1,8(sp)
; block0:
-; load_sym s1,userextname0+0
+; load_ext_name_far s1,userextname0+0
; callind s1
; callind s1
; ld s1,8(sp)
diff --git a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
index d5d1d7b574..6bbd6bdf82 100644
--- a/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/return-call-indirect.clif
@@ -35,7 +35,7 @@ block0(v0: i64):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym t0,%callee_i64+0
+; load_ext_name_far t0,%callee_i64+0
; return_call_ind t0 new_stack_arg_size:0 a0=a0
;
; Disassembled:
@@ -72,7 +72,7 @@ block0(v0: i64):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym t0,%callee_i64+0
+; load_ext_name_near t0,%callee_i64+0
; return_call_ind t0 new_stack_arg_size:0 a0=a0
;
; Disassembled:
@@ -82,11 +82,8 @@ block0(v0: i64):
; sd s0, 0(sp)
; mv s0, sp
; block1: ; offset 0x10
-; auipc t0, 0
-; ld t0, 0xc(t0)
-; j 0xc
-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %callee_i64 0
-; .byte 0x00, 0x00, 0x00, 0x00
+; auipc t0, 0 ; reloc_external RiscvPCRelHi20 %callee_i64 0
+; mv t0, t0 ; reloc_external RiscvPCRelLo12I func+16 0
; ld ra, 8(sp)
; ld s0, 0(sp)
; addi sp, sp, 0x10
@@ -132,7 +129,7 @@ block0(v0: f64):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym t0,%callee_f64+0
+; load_ext_name_far t0,%callee_f64+0
; return_call_ind t0 new_stack_arg_size:0 fa0=fa0
;
; Disassembled:
@@ -188,7 +185,7 @@ block0(v0: i8):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym t0,%callee_i8+0
+; load_ext_name_far t0,%callee_i8+0
; return_call_ind t0 new_stack_arg_size:0 a0=a0
;
; Disassembled:
@@ -296,7 +293,7 @@ block0:
; li s1,125
; li a0,130
; li a1,135
-; load_sym a2,%tail_callee_stack_args+0
+; load_ext_name_far a2,%tail_callee_stack_args+0
; sd s2,-144(incoming_arg)
; sd s3,-136(incoming_arg)
; sd s4,-128(incoming_arg)
diff --git a/cranelift/filetests/filetests/isa/riscv64/return-call.clif b/cranelift/filetests/filetests/isa/riscv64/return-call.clif
index 2a47ad837a..3252d60973 100644
--- a/cranelift/filetests/filetests/isa/riscv64/return-call.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/return-call.clif
@@ -33,7 +33,7 @@ block0(v0: i64):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym t0,%callee_i64+0
+; load_ext_name_far t0,%callee_i64+0
; return_call_ind t0 new_stack_arg_size:0 a0=a0
;
; Disassembled:
@@ -121,7 +121,7 @@ block0(v0: f64):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym t0,%callee_f64+0
+; load_ext_name_far t0,%callee_f64+0
; return_call_ind t0 new_stack_arg_size:0 fa0=fa0
;
; Disassembled:
@@ -175,7 +175,7 @@ block0(v0: i8):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym t0,%callee_i8+0
+; load_ext_name_far t0,%callee_i8+0
; return_call_ind t0 new_stack_arg_size:0 a0=a0
;
; Disassembled:
@@ -331,7 +331,7 @@ block0:
; sd s1,-24(incoming_arg)
; sd a0,-16(incoming_arg)
; sd a1,-8(incoming_arg)
-; load_sym t0,%tail_callee_stack_args+0
+; load_ext_name_far t0,%tail_callee_stack_args+0
; ld a0,8(slot)
; ld a1,0(slot)
; return_call_ind t0 new_stack_arg_size:144 a0=a0 a1=a1 a2=a2 a3=a3 a4=a4 a5=a5 a6=a6 a7=a7
@@ -610,7 +610,7 @@ block2:
; sd s7,-32(incoming_arg)
; sd s6,-24(incoming_arg)
; sd a0,-16(incoming_arg)
-; load_sym t0,%different_callee2+0
+; load_ext_name_far t0,%different_callee2+0
; ld a0,16(slot)
; ld a1,8(slot)
; ld a2,0(slot)
@@ -635,7 +635,7 @@ block2:
; sd s8,-24(incoming_arg)
; sd s7,-16(incoming_arg)
; sd s6,-8(incoming_arg)
-; load_sym t0,%different_callee1+0
+; load_ext_name_far t0,%different_callee1+0
; ld a1,8(slot)
; ld a2,0(slot)
; return_call_ind t0 new_stack_arg_size:144 a0=a0 a1=a1 a2=a2 a3=a3 a4=a4 a5=a5 a6=a6 a7=a7
diff --git a/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif b/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif
index fe934fc7cf..4f494f48bf 100644
--- a/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/stack-limit.clif
@@ -54,7 +54,7 @@ block0(v0: i64):
; addi t6,t6,16
; trap_if stk_ovf##(sp ult t6)
; block0:
-; load_sym a0,%foo+0
+; load_ext_name_far a0,%foo+0
; callind a0
; ld ra,8(sp)
; ld fp,0(sp)
diff --git a/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif b/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif
index 049e9717d5..1e4d60a141 100644
--- a/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/symbol-value-pic.clif
@@ -13,7 +13,7 @@ block0:
; VCode:
; block0:
-; load_sym a0,%my_global+0
+; load_ext_name_got a0,%my_global
; ret
;
; Disassembled:
diff --git a/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif b/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif
index a43878ae8a..31f25bac43 100644
--- a/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/symbol-value.clif
@@ -2,25 +2,197 @@ test compile precise-output
set unwind_info=false
target riscv64
-function %f() -> i64 {
- gv0 = symbol %my_global
+function %func_addr() -> i64 {
+ fn0 = %func0(i64) -> i64
block0:
- v0 = symbol_value.i64 gv0
- return v0
+ v0 = func_addr.i64 fn0
+ return v0
}
; VCode:
+; addi sp,sp,-16
+; sd ra,8(sp)
+; sd fp,0(sp)
+; mv fp,sp
; block0:
-; load_sym a0,%my_global+0
+; load_ext_name_far a0,%func0+0
+; ld ra,8(sp)
+; ld fp,0(sp)
+; addi sp,sp,16
; ret
;
; Disassembled:
; block0: ; offset 0x0
+; addi sp, sp, -0x10
+; sd ra, 8(sp)
+; sd s0, 0(sp)
+; mv s0, sp
+; block1: ; offset 0x10
; auipc a0, 0
; ld a0, 0xc(a0)
; j 0xc
-; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %my_global 0
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %func0 0
; .byte 0x00, 0x00, 0x00, 0x00
+; ld ra, 8(sp)
+; ld s0, 0(sp)
+; addi sp, sp, 0x10
+; ret
+
+function %colocated_func_addr() -> i64 {
+ fn0 = colocated %func0(i64) -> i64
+
+block0:
+ v0 = func_addr.i64 fn0
+ return v0
+}
+
+; VCode:
+; addi sp,sp,-16
+; sd ra,8(sp)
+; sd fp,0(sp)
+; mv fp,sp
+; block0:
+; load_ext_name_near a0,%func0+0
+; ld ra,8(sp)
+; ld fp,0(sp)
+; addi sp,sp,16
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; addi sp, sp, -0x10
+; sd ra, 8(sp)
+; sd s0, 0(sp)
+; mv s0, sp
+; block1: ; offset 0x10
+; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %func0 0
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+16 0
+; ld ra, 8(sp)
+; ld s0, 0(sp)
+; addi sp, sp, 0x10
+; ret
+
+function %symbol_value() -> i64 {
+ gv0 = symbol %global0
+
+block0:
+ v0 = symbol_value.i64 gv0
+ return v0
+}
+
+; VCode:
+; block0:
+; load_ext_name_far a0,%global0+0
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; auipc a0, 0
+; ld a0, 0xc(a0)
+; j 0xc
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 0
+; .byte 0x00, 0x00, 0x00, 0x00
+; ret
+
+function %symbol_value_plus_offset() -> i64 {
+ gv0 = symbol %global0+123
+
+block0:
+ v0 = symbol_value.i64 gv0
+ return v0
+}
+
+; VCode:
+; block0:
+; load_ext_name_far a0,%global0+123
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; auipc a0, 0
+; ld a0, 0xc(a0)
+; j 0xc
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 123
+; .byte 0x00, 0x00, 0x00, 0x00
+; ret
+
+function %symbol_value_minus_offset() -> i64 {
+ gv0 = symbol %global0-123
+
+block0:
+ v0 = symbol_value.i64 gv0
+ return v0
+}
+
+; VCode:
+; block0:
+; load_ext_name_far a0,%global0-123
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; auipc a0, 0
+; ld a0, 0xc(a0)
+; j 0xc
+; .byte 0x00, 0x00, 0x00, 0x00 ; reloc_external Abs8 %global0 -123
+; .byte 0x00, 0x00, 0x00, 0x00
+; ret
+
+function %colocated_symbol_value() -> i64 {
+ gv0 = symbol colocated %global0
+
+block0:
+ v0 = symbol_value.i64 gv0
+ return v0
+}
+
+; VCode:
+; block0:
+; load_ext_name_near a0,%global0+0
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 0
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0
+; ret
+
+function %colocated_symbol_value_plus_offset() -> i64 {
+ gv0 = symbol colocated %global0+123
+
+block0:
+ v0 = symbol_value.i64 gv0
+ return v0
+}
+
+; VCode:
+; block0:
+; load_ext_name_near a0,%global0+123
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 123
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0
+; ret
+
+function %colocated_symbol_value_minus_offset() -> i64 {
+ gv0 = symbol colocated %global0-123
+
+block0:
+ v0 = symbol_value.i64 gv0
+ return v0
+}
+
+; VCode:
+; block0:
+; load_ext_name_near a0,%global0-123
+; ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+; auipc a0, 0 ; reloc_external RiscvPCRelHi20 %global0 -123
+; mv a0, a0 ; reloc_external RiscvPCRelLo12I func+0 0
; ret
diff --git a/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif b/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif
index 2becbf9aff..0e67ebbfc4 100644
--- a/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/tail-call-conv.clif
@@ -134,7 +134,7 @@ block0:
; sd s2,120(sp)
; sd a0,128(sp)
; sd a1,136(sp)
-; load_sym s3,%tail_callee_stack_args+0
+; load_ext_name_far s3,%tail_callee_stack_args+0
; ld a0,8(slot)
; ld a1,0(slot)
; callind s3
@@ -489,7 +489,7 @@ block0:
; sd s11,312(sp)
; block0:
; load_addr a0,0(sp)
-; load_sym a4,%tail_callee_stack_rets+0
+; load_ext_name_far a4,%tail_callee_stack_rets+0
; callind a4
; ld a0,96(slot)
; ld s1,392(sp)
@@ -869,7 +869,7 @@ block0:
; sd a1,136(sp)
; sd a2,144(sp)
; load_addr a0,160(sp)
-; load_sym t1,%tail_callee_stack_args_and_rets+0
+; load_ext_name_far t1,%tail_callee_stack_args_and_rets+0
; ld a1,0(slot)
; ld a2,96(slot)
; callind t1
diff --git a/cranelift/filetests/filetests/isa/riscv64/zca.clif b/cranelift/filetests/filetests/isa/riscv64/zca.clif
index 2311ff1636..e61b377432 100644
--- a/cranelift/filetests/filetests/isa/riscv64/zca.clif
+++ b/cranelift/filetests/filetests/isa/riscv64/zca.clif
@@ -183,7 +183,7 @@ block0(v0: i8):
; sd fp,0(sp)
; mv fp,sp
; block0:
-; load_sym t0,%callee_i8+0
+; load_ext_name_far t0,%callee_i8+0
; return_call_ind t0 new_stack_arg_size:0 a0=a0
;
; Disassembled:
diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs
index 4a477be0a4..18aaf44f5b 100644
--- a/cranelift/jit/src/backend.rs
+++ b/cranelift/jit/src/backend.rs
@@ -212,7 +212,7 @@ impl JITModule {
}
fn get_address(&self, name: &ModuleRelocTarget) -> *const u8 {
- match *name {
+ match name {
ModuleRelocTarget::User { .. } => {
let (name, linkage) = if ModuleDeclarations::is_function(name) {
let func_id = FuncId::from_name(name);
@@ -244,12 +244,18 @@ impl JITModule {
panic!("can't resolve symbol {name}");
}
}
- ModuleRelocTarget::LibCall(ref libcall) => {
+ ModuleRelocTarget::LibCall(libcall) => {
let sym = (self.libcall_names)(*libcall);
self.lookup_symbol(&sym)
.unwrap_or_else(|| panic!("can't resolve libcall {sym}"))
}
- _ => panic!("invalid name"),
+ ModuleRelocTarget::FunctionOffset(func_id, offset) => {
+ match &self.compiled_functions[*func_id] {
+ Some(compiled) => return compiled.ptr.wrapping_add(*offset as usize),
+ None => todo!(),
+ }
+ }
+ name => panic!("invalid name {name:?}"),
}
}
diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs
index b25917fdea..395bce41bc 100644
--- a/cranelift/jit/src/compiled_blob.rs
+++ b/cranelift/jit/src/compiled_blob.rs
@@ -28,12 +28,15 @@ impl CompiledBlob {
) {
use std::ptr::write_unaligned;
- for &ModuleReloc {
- kind,
- offset,
- ref name,
- addend,
- } in &self.relocs
+ for (
+ i,
+ &ModuleReloc {
+ kind,
+ offset,
+ ref name,
+ addend,
+ },
+ ) in self.relocs.iter().enumerate()
{
debug_assert!((offset as usize) < self.size);
let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) };
@@ -159,6 +162,43 @@ impl CompiledBlob {
at.write_unaligned(at.read_unaligned().wrapping_add(pcrel));
}
}
+
+ // See <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
+ // for why `0x800` is added here.
+ Reloc::RiscvPCRelHi20 => {
+ let base = get_address(name);
+ let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
+ let pcrel = i32::try_from((what as isize) - (at as isize) + 0x800)
+ .unwrap()
+ .cast_unsigned();
+ let at = at as *mut u32;
+ unsafe {
+ modify_inst32(at, |i| i | (pcrel & 0xfffff000));
+ }
+ }
+
+ // The target of this relocation is the `auipc` preceding this
+ // instruction which should be `RiscvPCRelHi20`, and the actual
+ // target that we're relocating against is the target of that
+ // relocation. Assume for now that the previous relocation is
+ // the target of this relocation, and then use that.
+ Reloc::RiscvPCRelLo12I => {
+ let prev_reloc = &self.relocs[i - 1];
+ assert_eq!(prev_reloc.kind, Reloc::RiscvPCRelHi20);
+ let lo_target = get_address(name);
+ let hi_address =
+ unsafe { self.ptr.offset(isize::try_from(prev_reloc.offset).unwrap()) };
+ assert_eq!(lo_target, hi_address);
+ let hi_target = get_address(&prev_reloc.name);
+ let pcrel = i32::try_from((hi_target as isize) - (hi_address as isize))
+ .unwrap()
+ .cast_unsigned();
+ let at = at as *mut u32;
+ unsafe {
+ modify_inst32(at, |i| i | ((pcrel & 0xfff) << 20));
+ }
+ }
+
other => unimplemented!("unimplemented reloc {other:?}"),
}
}