Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift-jit: handle out-of-range x86_64 calls and symbol addresses (#13975)

Gleb Pomykalov6901aa7cee05
diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs
index cd34c0b3ff..5ffa00efe1 100644
--- a/cranelift/codegen/src/binemit/mod.rs
+++ b/cranelift/codegen/src/binemit/mod.rs
@@ -26,7 +26,13 @@ pub enum Reloc {
Abs8,
/// x86 PC-relative 4-byte
X86PCRel4,
- /// x86 call to PC-relative 4-byte
+ /// x86 call to PC-relative 4-byte.
+ ///
+ /// This relocation is only ever applied to the displacement of a `call`
+ /// or `jmp` instruction, never to address materialization (e.g. `lea`,
+ /// which uses [`Reloc::X86PCRel4`] instead). Consumers may therefore
+ /// redirect the control transfer through a veneer when the target is out
+ /// of range of the 32-bit displacement.
X86CallPCRel4,
/// x86 call to PLT-relative 4-byte
X86CallPLTRel4,
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index b3104ffaa7..eb7244e00b 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -1419,9 +1419,16 @@ pub(crate) fn emit(
// If we know the distance to the name is within 2GB (e.g., a
// module-local function), we can generate a RIP-relative
// address, with a relocation.
+ //
+ // Note that this is `X86PCRel4` rather than `X86CallPCRel4`:
+ // the latter is reserved for the displacement of `call`/`jmp`
+ // instructions, which relocation consumers may redirect through
+ // a veneer if the target turns out to be out of range. This `lea`
+ // computes the address of the symbol itself, so no such
+ // redirection is possible.
asm::inst::leaq_rm::new(*dst, riprel).emit(sink, info, state);
let cur = sink.cur_offset();
- sink.add_reloc_at_offset(cur - 4, Reloc::X86CallPCRel4, name, *offset - 4);
+ sink.add_reloc_at_offset(cur - 4, Reloc::X86PCRel4, name, *offset - 4);
} else {
// The full address can be encoded in the register, with a
// relocation.
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 8134eb5dc6..3f2b6aa4ac 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -1689,7 +1689,7 @@ impl MachInstLabelUse for LabelUse {
fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {
match (reloc, addend) {
- (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
+ (Reloc::X86PCRel4 | Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),
_ => None,
}
}
diff --git a/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
index 3883012afd..b6cd768835 100644
--- a/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
+++ b/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif
@@ -79,7 +79,7 @@ block0(v0: i64):
; pushq %rbp
; movq %rsp, %rbp
; block1: ; offset 0x4
-; leaq (%rip), %r10 ; reloc_external CallPCRel4 %callee_i64 -4
+; leaq (%rip), %r10 ; reloc_external PCRel4 %callee_i64 -4
; movq %rbp, %rsp
; popq %rbp
; jmpq *%r10
diff --git a/cranelift/filetests/filetests/isa/x64/symbols.clif b/cranelift/filetests/filetests/isa/x64/symbols.clif
index c708f12570..f950c4b4ec 100644
--- a/cranelift/filetests/filetests/isa/x64/symbols.clif
+++ b/cranelift/filetests/filetests/isa/x64/symbols.clif
@@ -50,7 +50,7 @@ block0:
; pushq %rbp
; movq %rsp, %rbp
; block1: ; offset 0x4
-; leaq (%rip), %rax ; reloc_external CallPCRel4 %func0 -4
+; leaq (%rip), %rax ; reloc_external PCRel4 %func0 -4
; movq %rbp, %rsp
; popq %rbp
; retq
@@ -158,7 +158,7 @@ block0:
; pushq %rbp
; movq %rsp, %rbp
; block1: ; offset 0x4
-; leaq (%rip), %rax ; reloc_external CallPCRel4 %global0 -4
+; leaq (%rip), %rax ; reloc_external PCRel4 %global0 -4
; movq %rbp, %rsp
; popq %rbp
; retq
@@ -185,7 +185,7 @@ block0:
; pushq %rbp
; movq %rsp, %rbp
; block1: ; offset 0x4
-; leaq (%rip), %rax ; reloc_external CallPCRel4 %global0 119
+; leaq (%rip), %rax ; reloc_external PCRel4 %global0 119
; movq %rbp, %rsp
; popq %rbp
; retq
@@ -212,7 +212,7 @@ block0:
; pushq %rbp
; movq %rsp, %rbp
; block1: ; offset 0x4
-; leaq (%rip), %rax ; reloc_external CallPCRel4 %global0 -127
+; leaq (%rip), %rax ; reloc_external PCRel4 %global0 -127
; movq %rbp, %rsp
; popq %rbp
; retq
diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs
index 82cfed4422..7d50bebbe9 100644
--- a/cranelift/jit/src/backend.rs
+++ b/cranelift/jit/src/backend.rs
@@ -20,7 +20,7 @@ use std::collections::BTreeMap;
use std::collections::HashMap;
use std::ffi::CString;
use std::io::Write;
-use target_lexicon::PointerWidth;
+use target_lexicon::{Architecture, PointerWidth};
const WRITABLE_DATA_ALIGNMENT: u64 = 0x8;
const READONLY_DATA_ALIGNMENT: u64 = 0x1;
@@ -66,7 +66,18 @@ impl JITBuilder {
// which might not reach all definitions; we can't handle that here, so
// we require long-range relocation types.
flag_builder.set("use_colocated_libcalls", "false").unwrap();
- flag_builder.set("is_pic", "false").unwrap();
+ // On x86_64, enable PIC so that symbol addresses are materialized with
+ // GOT-relative loads rather than RIP-relative `lea`s, which assume the
+ // symbol is within ±2 GiB of the code. The JIT memory provider makes
+ // no such promise, so `CompiledBlob::perform_relocations` resolves the
+ // loads through per-blob GOT entries, relaxing them back to `lea` when
+ // the symbol does turn out to be in range.
+ let is_pic = if cfg!(target_arch = "x86_64") {
+ "true"
+ } else {
+ "false"
+ };
+ flag_builder.set("is_pic", is_pic).unwrap();
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {msg}");
});
@@ -357,8 +368,9 @@ impl JITModule {
/// Create a new `JITModule`.
pub fn new(builder: JITBuilder) -> Self {
assert!(
- !builder.isa.flags().is_pic(),
- "cranelift-jit needs is_pic=false"
+ !builder.isa.flags().is_pic()
+ || builder.isa.triple().architecture == Architecture::X86_64,
+ "cranelift-jit only supports is_pic=true on x86_64"
);
let memory = builder
diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs
index db88301fa7..a873f7a914 100644
--- a/cranelift/jit/src/compiled_blob.rs
+++ b/cranelift/jit/src/compiled_blob.rs
@@ -6,7 +6,16 @@ use cranelift_module::{ModuleError, ModuleReloc, ModuleRelocTarget, ModuleResult
use crate::JITMemoryProvider;
use crate::memory::JITMemoryKind;
-const VENEER_SIZE: usize = 24; // ldr + br + pointer
+/// Size reserved per veneer. This is exactly the size of the largest veneer:
+/// 16 bytes on AArch64 (`ldr` + `br` + 8-byte target address). The x86_64
+/// veneer is 14 bytes (`jmp qword ptr [rip]` + 8-byte target address).
+///
+/// Veneers are placed directly after the compiled code, followed by the GOT
+/// entries: `[code][veneers][GOT entries]`.
+const VENEER_SIZE: usize = 16;
+
+/// Size of a GOT entry: the absolute 8-byte address of a symbol.
+const GOT_ENTRY_SIZE: usize = 8;
/// Reads a 32bit instruction at `iptr`, and writes it again after
/// being altered by `modifier`
@@ -22,6 +31,7 @@ pub(crate) struct CompiledBlob {
size: usize,
relocs: Vec<ModuleReloc>,
veneer_count: usize,
+ got_count: usize,
#[cfg(feature = "wasmtime-unwinder")]
wasmtime_exception_data: Option<Vec<u8>>,
}
@@ -37,17 +47,28 @@ impl CompiledBlob {
#[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option<Vec<u8>>,
kind: JITMemoryKind,
) -> ModuleResult<Self> {
- // Reserve veneers for all function calls just in case
+ // Reserve a worst-case veneer slot for every branch relocation and a
+ // GOT entry for every GOT-relative load in case its target is out of
+ // range.
let mut veneer_count = 0;
+ let mut got_count = 0;
for reloc in &relocs {
match reloc.kind {
- Reloc::Arm64Call => veneer_count += 1,
+ Reloc::Arm64Call | Reloc::X86CallPCRel4 | Reloc::X86CallPLTRel4 => {
+ veneer_count += 1
+ }
+ Reloc::X86GOTPCRel4 => got_count += 1,
_ => {}
}
}
+ let mut alloc_size = data.len() + veneer_count * VENEER_SIZE;
+ if got_count > 0 {
+ alloc_size = alloc_size.next_multiple_of(GOT_ENTRY_SIZE) + got_count * GOT_ENTRY_SIZE;
+ }
+
let ptr = memory
- .allocate(data.len() + veneer_count * VENEER_SIZE, align, kind)
+ .allocate(alloc_size, align, kind)
.map_err(|e| ModuleError::Allocation { err: e })?;
unsafe {
@@ -59,6 +80,7 @@ impl CompiledBlob {
size: data.len(),
relocs,
veneer_count,
+ got_count,
#[cfg(feature = "wasmtime-unwinder")]
wasmtime_exception_data,
})
@@ -83,6 +105,7 @@ impl CompiledBlob {
size,
relocs,
veneer_count: 0,
+ got_count: 0,
#[cfg(feature = "wasmtime-unwinder")]
wasmtime_exception_data,
})
@@ -101,6 +124,12 @@ impl CompiledBlob {
self.wasmtime_exception_data.as_deref()
}
+ /// Offset of the GOT entries within the allocation: after the code and
+ /// the veneers, aligned to the entry size.
+ fn got_offset(&self) -> usize {
+ (self.size + self.veneer_count * VENEER_SIZE).next_multiple_of(GOT_ENTRY_SIZE)
+ }
+
pub(crate) fn perform_relocations(
&self,
get_address: impl Fn(&ModuleRelocTarget) -> *const u8,
@@ -108,6 +137,7 @@ impl CompiledBlob {
use std::ptr::write_unaligned;
let mut next_veneer_idx = 0;
+ let mut next_got_idx = 0;
let relocation_target_addr = |name: &ModuleRelocTarget, addend: Addend| {
let addend = isize::try_from(addend).unwrap();
get_address(name)
@@ -137,16 +167,88 @@ impl CompiledBlob {
let what = relocation_target_addr(name, addend);
unsafe { write_unaligned(at as *mut u64, u64::try_from(what).unwrap()) };
}
- Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
+ Reloc::X86PCRel4 => {
let what = relocation_target_addr(name, addend);
let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
unsafe { write_unaligned(at as *mut i32, pcrel) };
}
- Reloc::X86GOTPCRel4 => {
- panic!("GOT relocation shouldn't be generated when !is_pic");
+ Reloc::X86CallPCRel4 | Reloc::X86CallPLTRel4 => {
+ // These relocations are applied to the 32-bit displacement of a `call`
+ // or `jmp` instruction, which is relative to the end of the
+ // instruction, i.e. 4 bytes past `at`. The addend (always -4 as
+ // emitted by the x64 backend) accounts for this, so the instruction
+ // transfers control to `what + 4`. The PLT form additionally permits
+ // resolution through a stub, which the veneer below provides.
+ let what = relocation_target_addr(name, addend);
+ if let Ok(pcrel) = i32::try_from((what as isize) - (at as isize)) {
+ unsafe { write_unaligned(at as *mut i32, pcrel) };
+ } else {
+ // The target is out of range for the 32-bit displacement, so
+ // redirect the call through a veneer at the end of the function,
+ // just like for `Arm64Call` below: `jmp qword ptr [rip]` followed
+ // by the absolute target address.
+ let veneer_idx = next_veneer_idx;
+ next_veneer_idx += 1;
+ assert!(veneer_idx < self.veneer_count);
+ let veneer =
+ unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };
+
+ let target = u64::try_from(what.checked_add(4).unwrap()).unwrap();
+ unsafe {
+ ptr::copy_nonoverlapping([0xff, 0x25, 0, 0, 0, 0].as_ptr(), veneer, 6);
+ write_unaligned(veneer.byte_add(6).cast::<u64>(), target);
+ }
+
+ // Point the original instruction at the veneer instead; the
+ // displacement is again relative to the end of the 4-byte field.
+ let pcrel = i32::try_from((veneer as isize) - (at as isize) - 4).unwrap();
+ unsafe { write_unaligned(at as *mut i32, pcrel) };
+ }
}
- Reloc::X86CallPLTRel4 => {
- panic!("PLT relocation shouldn't be generated when !is_pic");
+ Reloc::X86GOTPCRel4 => {
+ // This relocation is applied to the 32-bit displacement of a
+ // `mov reg, qword ptr [rip + disp]` instruction that loads the
+ // address of a symbol from its GOT entry. The addend (always -4 as
+ // emitted by the x64 backend) accounts for the displacement being
+ // relative to the end of the instruction, and any symbol offset is
+ // added by a separately emitted instruction, so the GOT entry holds
+ // the address of the symbol itself: `what + 4`.
+ let what = relocation_target_addr(name, addend);
+ let symbol_addr = u64::try_from(what.checked_add(4).unwrap()).unwrap();
+
+ // If the symbol is within displacement range, relax the load to a
+ // `lea` computing its address directly, the same way linkers relax
+ // `R_X86_64_REX_GOTPCRELX`. The instruction is expected to be a REX
+ // prefix, the `mov` opcode 0x8b, and a ModRM byte with mod=0b00 and
+ // r/m=0b101 (RIP-relative); only rewrite it if it matches.
+ let insn = (offset >= 3)
+ .then(|| unsafe { at.cast_const().sub(3).cast::<[u8; 3]>().read() });
+ let is_rip_relative_mov = insn.is_some_and(
+ |insn| matches!(insn, [0x48..=0x4f, 0x8b, modrm] if modrm & 0xc7 == 0x05),
+ );
+ let direct_pcrel = i32::try_from((what as isize) - (at as isize));
+ match (is_rip_relative_mov, direct_pcrel) {
+ (true, Ok(pcrel)) => unsafe {
+ at.sub(2).write(0x8d); // mov -> lea
+ write_unaligned(at as *mut i32, pcrel);
+ },
+ _ => {
+ // Resolve through a GOT entry at the end of the allocation,
+ // which is always in range of the load.
+ let got_idx = next_got_idx;
+ next_got_idx += 1;
+ assert!(got_idx < self.got_count);
+ let entry = unsafe {
+ self.ptr
+ .byte_add(self.got_offset() + got_idx * GOT_ENTRY_SIZE)
+ };
+ unsafe { write_unaligned(entry.cast::<u64>(), symbol_addr) };
+
+ let pcrel =
+ i32::try_from((entry as isize) - (at as isize) - 4).unwrap();
+ unsafe { write_unaligned(at as *mut i32, pcrel) };
+ }
+ }
}
Reloc::S390xPCRel32Dbl | Reloc::S390xPLTRel32Dbl => {
let what = relocation_target_addr(name, addend);
@@ -176,7 +278,7 @@ impl CompiledBlob {
// end of the function.
let veneer_idx = next_veneer_idx;
next_veneer_idx += 1;
- assert!(veneer_idx <= self.veneer_count);
+ assert!(veneer_idx < self.veneer_count);
let veneer =
unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };
diff --git a/cranelift/jit/tests/basic.rs b/cranelift/jit/tests/basic.rs
index 064ab51646..4a5bfbfea9 100644
--- a/cranelift/jit/tests/basic.rs
+++ b/cranelift/jit/tests/basic.rs
@@ -10,8 +10,14 @@ use cranelift_module::*;
fn isa() -> Option<OwnedTargetIsa> {
let mut flag_builder = settings::builder();
flag_builder.set("use_colocated_libcalls", "false").unwrap();
- // FIXME set back to true once the x64 backend supports it.
- flag_builder.set("is_pic", "false").unwrap();
+ // PIC is only supported by the JIT on x86_64, where it routes symbol
+ // address materialization through per-blob GOT entries.
+ let is_pic = if cfg!(target_arch = "x86_64") {
+ "true"
+ } else {
+ "false"
+ };
+ flag_builder.set("is_pic", is_pic).unwrap();
let isa_builder = cranelift_native::builder().ok()?;
isa_builder.finish(settings::Flags::new(flag_builder)).ok()
}
@@ -219,3 +225,682 @@ fn empty_data_object() {
data.define(Box::new([]));
module.define_data(data_id, &data).unwrap();
}
+
+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
+mod far_x86_memory {
+ use std::collections::VecDeque;
+ use std::io;
+ use std::ptr;
+ use std::sync::{Arc, Mutex};
+
+ use cranelift_jit::{BranchProtection, JITMemoryKind, JITMemoryProvider};
+ use cranelift_module::ModuleResult;
+
+ pub(super) const VENEER_SIZE: usize = 16;
+
+ struct ReservedAddressSpace {
+ base: usize,
+ len: usize,
+ page_size: usize,
+ }
+
+ impl ReservedAddressSpace {
+ fn new() -> Self {
+ let page_size = usize::try_from(unsafe { libc::sysconf(libc::_SC_PAGESIZE) }).unwrap();
+ let len = (i32::MAX as usize)
+ .checked_add(16 * 1024 * 1024)
+ .unwrap()
+ .next_multiple_of(page_size);
+ let mapping = unsafe {
+ libc::mmap(
+ ptr::null_mut(),
+ len,
+ libc::PROT_NONE,
+ libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_NORESERVE,
+ -1,
+ 0,
+ )
+ };
+ assert_ne!(mapping, libc::MAP_FAILED);
+ Self {
+ base: mapping.addr(),
+ len,
+ page_size,
+ }
+ }
+ }
+
+ impl Drop for ReservedAddressSpace {
+ fn drop(&mut self) {
+ let result = unsafe { libc::munmap(self.base as *mut libc::c_void, self.len) };
+ assert_eq!(result, 0);
+ }
+ }
+
+ #[derive(Clone, Copy)]
+ pub(super) enum Placement {
+ Low,
+ High,
+ }
+
+ #[derive(Clone, Copy)]
+ enum FinalProtection {
+ ReadExecute,
+ ReadOnly,
+ ReadWrite,
+ }
+
+ struct Allocation {
+ addr: usize,
+ len: usize,
+ final_protection: FinalProtection,
+ }
+
+ /// Places JIT allocations at predetermined ends of a virtual address
+ /// reservation so tests can deterministically exceed `rel32` range.
+ pub(super) struct FarMemoryProvider {
+ space: ReservedAddressSpace,
+ executable_placements: VecDeque<Placement>,
+ readonly_placements: VecDeque<Placement>,
+ writable_placements: VecDeque<Placement>,
+ low_offset: usize,
+ high_offset: usize,
+ allocations: Vec<Allocation>,
+ requested_exec_sizes: Arc<Mutex<Vec<usize>>>,
+ }
+
+ impl FarMemoryProvider {
+ pub(super) fn new(
+ executable_placements: impl IntoIterator<Item = Placement>,
+ readonly_placements: impl IntoIterator<Item = Placement>,
+ ) -> (Self, Arc<Mutex<Vec<usize>>>) {
+ let space = ReservedAddressSpace::new();
+ let high_offset = space.len;
+ let requested_exec_sizes = Arc::new(Mutex::new(Vec::new()));
+ (
+ Self {
+ space,
+ executable_placements: executable_placements.into_iter().collect(),
+ readonly_placements: readonly_placements.into_iter().collect(),
+ writable_placements: VecDeque::new(),
+ low_offset: 0,
+ high_offset,
+ allocations: Vec::new(),
+ requested_exec_sizes: Arc::clone(&requested_exec_sizes),
+ },
+ requested_exec_sizes,
+ )
+ }
+
+ fn allocate_at(
+ &mut self,
+ size: usize,
+ align: u64,
+ placement: Placement,
+ final_protection: FinalProtection,
+ ) -> io::Result<*mut u8> {
+ assert!(usize::try_from(align).unwrap() <= self.space.page_size);
+ let len = size
+ .next_multiple_of(self.space.page_size)
+ .max(self.space.page_size);
+ let addr = match placement {
+ Placement::Low => {
+ let addr = self.space.base + self.low_offset;
+ self.low_offset += len;
+ addr
+ }
+ Placement::High => {
+ self.high_offset -= len;
+ self.space.base + self.high_offset
+ }
+ };
+ assert!(self.low_offset <= self.high_offset);
+ let result = unsafe {
+ libc::mprotect(
+ addr as *mut libc::c_void,
+ len,
+ libc::PROT_READ | libc::PROT_WRITE,
+ )
+ };
+ assert_eq!(result, 0);
+ self.allocations.push(Allocation {
+ addr,
+ len,
+ final_protection,
+ });
+ Ok(addr as *mut u8)
+ }
+ }
+
+ impl JITMemoryProvider for FarMemoryProvider {
+ fn allocate(
+ &mut self,
+ size: usize,
+ align: u64,
+ kind: JITMemoryKind,
+ ) -> io::Result<*mut u8> {
+ let (placement, final_protection) = match kind {
+ JITMemoryKind::Executable => {
+ self.requested_exec_sizes.lock().unwrap().push(size);
+ (
+ self.executable_placements
+ .pop_front()
+ .expect("placement for every executable allocation"),
+ FinalProtection::ReadExecute,
+ )
+ }
+ JITMemoryKind::Writable => (
+ self.writable_placements
+ .pop_front()
+ .expect("placement for every writable allocation"),
+ FinalProtection::ReadWrite,
+ ),
+ JITMemoryKind::ReadOnly => (
+ self.readonly_placements
+ .pop_front()
+ .expect("placement for every read-only allocation"),
+ FinalProtection::ReadOnly,
+ ),
+ };
+ self.allocate_at(size, align, placement, final_protection)
+ }
+
+ unsafe fn free_memory(&mut self) {
+ // All allocations are freed together when `space` is unmapped.
+ self.allocations.clear();
+ }
+
+ fn finalize(&mut self, _branch_protection: BranchProtection) -> ModuleResult<()> {
+ for allocation in &self.allocations {
+ let protection = match allocation.final_protection {
+ FinalProtection::ReadExecute => libc::PROT_READ | libc::PROT_EXEC,
+ FinalProtection::ReadOnly => libc::PROT_READ,
+ FinalProtection::ReadWrite => libc::PROT_READ | libc::PROT_WRITE,
+ };
+ let result = unsafe {
+ libc::mprotect(
+ allocation.addr as *mut libc::c_void,
+ allocation.len,
+ protection,
+ )
+ };
+ assert_eq!(result, 0);
+ }
+ Ok(())
+ }
+ }
+}
+
+/// Reproduces a bug where a `call` or `jmp` between two functions of the same
+/// module that happen to be placed further than ±2 GiB apart panicked while
+/// applying the `X86CallPCRel4` relocation, instead of routing the control
+/// transfer through a veneer like AArch64 already did for `Arm64Call`.
+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
+#[test]
+fn far_x86_control_transfers_use_veneers() {
+ use std::mem;
+
+ use cranelift_codegen::binemit::Reloc;
+
+ use far_x86_memory::{FarMemoryProvider, Placement, VENEER_SIZE};
+
+ let (memory, requested_exec_sizes) = FarMemoryProvider::new(
+ [
+ Placement::Low,
+ Placement::High,
+ Placement::Low,
+ Placement::High,
+ Placement::Low,
+ Placement::Low,
+ Placement::High,
+ Placement::High,
+ ],
+ [],
+ );
+ let mut builder = JITBuilder::new(default_libcall_names()).unwrap();
+ builder.memory_provider(Box::new(memory));
+
+ let mut module = JITModule::new(builder);
+ let mut signature = module.make_signature();
+ signature.returns.push(AbiParam::new(types::I32));
+ let low_callee = module
+ .declare_function("low_callee", Linkage::Local, &signature)
+ .unwrap();
+ let high_caller = module
+ .declare_function("high_caller", Linkage::Local, &signature)
+ .unwrap();
+ let low_tail_caller = module
+ .declare_function("low_tail_caller", Linkage::Local, &signature)
+ .unwrap();
+ let high_tail_callee = module
+ .declare_function("high_tail_callee", Linkage::Local, &signature)
+ .unwrap();
+
+ // low_callee: mov eax, 42; ret
+ module
+ .define_function_bytes(low_callee, 1, &[0xb8, 42, 0, 0, 0, 0xc3], &[])
+ .unwrap();
+
+ // Compile high_caller normally so this test also checks the relocation
+ // emitted for a compiler-generated direct call.
+ let mut ctx = module.make_context();
+ ctx.func.name = UserFuncName::user(0, high_caller.as_u32());
+ ctx.func.signature = signature.clone();
+ let low_callee_ref = module.declare_func_in_func(low_callee, &mut ctx.func);
+ let mut func_ctx = FunctionBuilderContext::new();
+ {
+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
+ let block = bcx.create_block();
+ bcx.switch_to_block(block);
+ let call = bcx.ins().call(low_callee_ref, &[]);
+ let result = bcx.inst_results(call)[0];
+ bcx.ins().return_(&[result]);
+ }
+ module.define_function(high_caller, &mut ctx).unwrap();
+ let (high_caller_code_len, high_caller_reloc_offset) = {
+ let compiled = ctx.compiled_code().unwrap();
+ let reloc = compiled
+ .buffer
+ .relocs()
+ .iter()
+ .find(|reloc| reloc.kind == Reloc::X86CallPCRel4)
+ .unwrap();
+ (compiled.code_buffer().len(), reloc.offset as usize)
+ };
+
+ // low_tail_caller: jmp high_tail_callee
+ let relocations = [ModuleReloc {
+ offset: 1,
+ kind: Reloc::X86CallPCRel4,
+ name: ModuleRelocTarget::user(0, high_tail_callee.as_u32()),
+ addend: -4,
+ }];
+ module
+ .define_function_bytes(low_tail_caller, 1, &[0xe9, 0, 0, 0, 0], &relocations)
+ .unwrap();
+
+ // high_tail_callee: mov eax, 84; ret
+ module
+ .define_function_bytes(high_tail_callee, 1, &[0xb8, 84, 0, 0, 0, 0xc3], &[])
+ .unwrap();
+
+ // A caller with one near target and two far targets checks that only the
+ // far calls use distinct, densely packed veneers.
+ let mixed_caller = module
+ .declare_function("mixed_caller", Linkage::Local, &signature)
+ .unwrap();
+ let near = module
+ .declare_function("near", Linkage::Local, &signature)
+ .unwrap();
+ let far_a = module
+ .declare_function("far_a", Linkage::Local, &signature)
+ .unwrap();
+ let far_b = module
+ .declare_function("far_b", Linkage::Local, &signature)
+ .unwrap();
+ let mixed_caller_code = [
+ 0x48, 0x83, 0xec, 0x08, // sub rsp, 8
+ 0xe8, 0, 0, 0, 0, // call near
+ 0xe8, 0, 0, 0, 0, // call far_a
+ 0xe8, 0, 0, 0, 0, // call far_b
+ 0x48, 0x83, 0xc4, 0x08, // add rsp, 8
+ 0xc3, // ret
+ ];
+ let mixed_relocations = [
+ ModuleReloc {
+ offset: 5,
+ kind: Reloc::X86CallPCRel4,
+ name: ModuleRelocTarget::user(0, near.as_u32()),
+ addend: -4,
+ },
+ ModuleReloc {
+ offset: 10,
+ kind: Reloc::X86CallPCRel4,
+ name: ModuleRelocTarget::user(0, far_a.as_u32()),
+ addend: -4,
+ },
+ ModuleReloc {
+ offset: 15,
+ kind: Reloc::X86CallPCRel4,
+ name: ModuleRelocTarget::user(0, far_b.as_u32()),
+ addend: -4,
+ },
+ ];
+ module
+ .define_function_bytes(mixed_caller, 1, &mixed_caller_code, &mixed_relocations)
+ .unwrap();
+ module
+ .define_function_bytes(near, 1, &[0xb8, 1, 0, 0, 0, 0xc3], &[])
+ .unwrap();
+ module
+ .define_function_bytes(far_a, 1, &[0xb8, 2, 0, 0, 0, 0xc3], &[])
+ .unwrap();
+ module
+ .define_function_bytes(far_b, 1, &[0xb8, 42, 0, 0, 0, 0xc3], &[])
+ .unwrap();
+
+ module.finalize_definitions().unwrap();
+
+ let low_callee_ptr = module.get_finalized_function(low_callee);
+ let high_caller_ptr = module.get_finalized_function(high_caller);
+ let low_tail_caller_ptr = module.get_finalized_function(low_tail_caller);
+ let high_tail_callee_ptr = module.get_finalized_function(high_tail_callee);
+ let mixed_caller_ptr = module.get_finalized_function(mixed_caller);
+ let near_ptr = module.get_finalized_function(near);
+ let far_a_ptr = module.get_finalized_function(far_a);
+ let far_b_ptr = module.get_finalized_function(far_b);
+
+ fn branch_destination(caller: *const u8, reloc_offset: usize) -> *const u8 {
+ let at = caller.wrapping_byte_add(reloc_offset);
+ let displacement = unsafe { at.cast::<i32>().read_unaligned() };
+ at.wrapping_byte_add(4)
+ .wrapping_byte_offset(displacement as isize)
+ }
+
+ fn assert_veneer(
+ caller: *const u8,
+ code_len: usize,
+ reloc_offset: usize,
+ expected_target: *const u8,
+ ) -> *const u8 {
+ assert!(caller.addr().abs_diff(expected_target.addr()) > i32::MAX as usize);
+
+ let veneer = branch_destination(caller, reloc_offset);
+ assert_eq!(veneer, caller.wrapping_byte_add(code_len));
+
+ let veneer_code = unsafe { std::slice::from_raw_parts(veneer, 6) };
+ assert_eq!(veneer_code, [0xff, 0x25, 0, 0, 0, 0]);
+ let veneer_target = unsafe { veneer.byte_add(6).cast::<u64>().read_unaligned() };
+ assert_eq!(veneer_target, expected_target.addr() as u64);
+ veneer
+ }
+
+ // Check a negative-distance call and a positive-distance tail jump.
+ assert!(high_caller_ptr.addr() > low_callee_ptr.addr());
+ assert_veneer(
+ high_caller_ptr,
+ high_caller_code_len,
+ high_caller_reloc_offset,
+ low_callee_ptr,
+ );
+ let high_caller_fn: extern "C" fn() -> u32 = unsafe { mem::transmute(high_caller_ptr) };
+ assert_eq!(high_caller_fn(), 42);
+
+ assert!(low_tail_caller_ptr.addr() < high_tail_callee_ptr.addr());
+ assert_veneer(low_tail_caller_ptr, 5, 1, high_tail_callee_ptr);
+ let low_tail_caller_fn: extern "C" fn() -> u32 = unsafe { mem::transmute(low_tail_caller_ptr) };
+ assert_eq!(low_tail_caller_fn(), 84);
+
+ assert!(mixed_caller_ptr.addr().abs_diff(near_ptr.addr()) <= i32::MAX as usize);
+ assert!(mixed_caller_ptr.addr().abs_diff(far_a_ptr.addr()) > i32::MAX as usize);
+ assert!(mixed_caller_ptr.addr().abs_diff(far_b_ptr.addr()) > i32::MAX as usize);
+ assert_eq!(branch_destination(mixed_caller_ptr, 5), near_ptr);
+ let veneer_a = assert_veneer(mixed_caller_ptr, mixed_caller_code.len(), 10, far_a_ptr);
+ let veneer_b = assert_veneer(
+ mixed_caller_ptr,
+ mixed_caller_code.len() + VENEER_SIZE,
+ 15,
+ far_b_ptr,
+ );
+ assert_eq!(veneer_b, veneer_a.wrapping_byte_add(VENEER_SIZE));
+ let mixed_caller_fn: extern "C" fn() -> u32 = unsafe { mem::transmute(mixed_caller_ptr) };
+ assert_eq!(mixed_caller_fn(), 42);
+
+ // One worst-case slot is reserved for every branch relocation, including
+ // the call that ultimately resolves directly.
+ let requested_exec_sizes = requested_exec_sizes.lock().unwrap();
+ assert_eq!(requested_exec_sizes[1], high_caller_code_len + VENEER_SIZE);
+ assert_eq!(requested_exec_sizes[2], 5 + VENEER_SIZE);
+ assert_eq!(
+ requested_exec_sizes[4],
+ mixed_caller_code.len() + mixed_relocations.len() * VENEER_SIZE
+ );
+ drop(requested_exec_sizes);
+
+ unsafe { module.free_memory() };
+}
+
+/// Reproduces the Roto failure: JIT code materializes the address of an
+/// anonymous read-only data object allocated more than ±2 GiB away. The
+/// GOT-relative load cannot be relaxed to a `lea` and reads the address from
+/// a GOT entry at the end of the function's allocation.
+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
+#[test]
+fn far_x86_data_address_resolves_through_got() {
+ use cranelift_codegen::binemit::Reloc;
+
+ use far_x86_memory::{FarMemoryProvider, Placement};
+
+ let (memory, _) = FarMemoryProvider::new([Placement::Low], [Placement::High]);
+ let mut builder = JITBuilder::new(default_libcall_names()).unwrap();
+ builder.memory_provider(Box::new(memory));
+ let mut module = JITModule::new(builder);
+
+ let data_id = module.declare_anonymous_data(false, false).unwrap();
+ let mut data = DataDescription::new();
+ data.define(vec![42].into_boxed_slice());
+ module.define_data(data_id, &data).unwrap();
+
+ let mut signature = module.make_signature();
+ signature
+ .returns
+ .push(AbiParam::new(module.target_config().pointer_type()));
+ let address_func = module
+ .declare_function("data_address", Linkage::Local, &signature)
+ .unwrap();
+ let mut ctx = module.make_context();
+ ctx.func.name = UserFuncName::user(0, address_func.as_u32());
+ ctx.func.signature = signature;
+ let data_ref = module.declare_data_in_func(data_id, &mut ctx.func);
+ let mut func_ctx = FunctionBuilderContext::new();
+ {
+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
+ let block = bcx.create_block();
+ bcx.switch_to_block(block);
+ let address = bcx
+ .ins()
+ .symbol_value(module.target_config().pointer_type(), data_ref);
+ bcx.ins().return_(&[address]);
+ bcx.seal_all_blocks();
+ bcx.finalize(module.target_config());
+ }
+ module.define_function(address_func, &mut ctx).unwrap();
+
+ // Before the fix, finalization panics here while applying an out-of-range
+ // X86PCRel4 relocation.
+ module.finalize_definitions().unwrap();
+
+ let relocs = ctx.compiled_code().unwrap().buffer.relocs();
+ assert_eq!(relocs.len(), 1);
+ assert_eq!(relocs[0].kind, Reloc::X86GOTPCRel4);
+ let reloc_offset = usize::try_from(relocs[0].offset).unwrap();
+
+ let (data_ptr, _) = module.get_finalized_data(data_id);
+ let address_ptr = module.get_finalized_function(address_func);
+ assert!(address_ptr.addr().abs_diff(data_ptr.addr()) > i32::MAX as usize);
+
+ // The load stays a `mov` and reads the data object's address from a GOT
+ // entry at the end of the function's allocation.
+ assert_eq!(
+ unsafe { address_ptr.byte_add(reloc_offset - 2).read() },
+ 0x8b
+ );
+ let displacement = unsafe {
+ address_ptr
+ .byte_add(reloc_offset)
+ .cast::<i32>()
+ .read_unaligned()
+ };
+ let got_entry = address_ptr
+ .wrapping_byte_add(reloc_offset + 4)
+ .wrapping_byte_offset(displacement as isize);
+ assert!(got_entry.addr() > address_ptr.addr());
+ assert_eq!(
+ unsafe { got_entry.cast::<u64>().read_unaligned() },
+ data_ptr.addr() as u64
+ );
+
+ let get_address: extern "C" fn() -> usize = unsafe { std::mem::transmute(address_ptr) };
+ assert_eq!(get_address(), data_ptr.addr());
+
+ unsafe { module.free_memory() };
+}
+
+/// When the memory provider does place a symbol within displacement range,
+/// the GOT-relative load is relaxed to a `lea` computing the address
+/// directly, so near symbol accesses don't pay for a GOT indirection.
+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
+#[test]
+fn near_x86_data_address_relaxes_got_load_to_lea() {
+ use cranelift_codegen::binemit::Reloc;
+
+ use far_x86_memory::{FarMemoryProvider, Placement};
+
+ let (memory, _) = FarMemoryProvider::new([Placement::Low], [Placement::Low]);
+ let mut builder = JITBuilder::new(default_libcall_names()).unwrap();
+ builder.memory_provider(Box::new(memory));
+ let mut module = JITModule::new(builder);
+
+ let data_id = module.declare_anonymous_data(false, false).unwrap();
+ let mut data = DataDescription::new();
+ data.define(vec![42].into_boxed_slice());
+ module.define_data(data_id, &data).unwrap();
+
+ let mut signature = module.make_signature();
+ signature
+ .returns
+ .push(AbiParam::new(module.target_config().pointer_type()));
+ let address_func = module
+ .declare_function("data_address", Linkage::Local, &signature)
+ .unwrap();
+ let mut ctx = module.make_context();
+ ctx.func.name = UserFuncName::user(0, address_func.as_u32());
+ ctx.func.signature = signature;
+ let data_ref = module.declare_data_in_func(data_id, &mut ctx.func);
+ let mut func_ctx = FunctionBuilderContext::new();
+ {
+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
+ let block = bcx.create_block();
+ bcx.switch_to_block(block);
+ let address = bcx
+ .ins()
+ .symbol_value(module.target_config().pointer_type(), data_ref);
+ bcx.ins().return_(&[address]);
+ bcx.seal_all_blocks();
+ bcx.finalize(module.target_config());
+ }
+ module.define_function(address_func, &mut ctx).unwrap();
+ module.finalize_definitions().unwrap();
+
+ let relocs = ctx.compiled_code().unwrap().buffer.relocs();
+ assert_eq!(relocs.len(), 1);
+ assert_eq!(relocs[0].kind, Reloc::X86GOTPCRel4);
+ let reloc_offset = usize::try_from(relocs[0].offset).unwrap();
+
+ let (data_ptr, _) = module.get_finalized_data(data_id);
+ let address_ptr = module.get_finalized_function(address_func);
+ assert!(address_ptr.addr().abs_diff(data_ptr.addr()) <= i32::MAX as usize);
+
+ // The load was relaxed to a `lea` pointing directly at the data object.
+ assert_eq!(
+ unsafe { address_ptr.byte_add(reloc_offset - 2).read() },
+ 0x8d
+ );
+ let displacement = unsafe {
+ address_ptr
+ .byte_add(reloc_offset)
+ .cast::<i32>()
+ .read_unaligned()
+ };
+ assert_eq!(
+ address_ptr
+ .wrapping_byte_add(reloc_offset + 4)
+ .wrapping_byte_offset(displacement as isize),
+ data_ptr
+ );
+
+ let get_address: extern "C" fn() -> usize = unsafe { std::mem::transmute(address_ptr) };
+ assert_eq!(get_address(), data_ptr.addr());
+
+ unsafe { module.free_memory() };
+}
+
+/// A materialized function pointer has the same range issue as a data pointer,
+/// but direct calls to that function must remain eligible for call veneers.
+/// The GOT entry holds the function's real address, preserving pointer
+/// identity with `get_finalized_function`.
+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
+#[test]
+fn far_x86_function_address_resolves_through_got() {
+ use cranelift_codegen::binemit::Reloc;
+
+ use far_x86_memory::{FarMemoryProvider, Placement};
+
+ let (memory, _) = FarMemoryProvider::new([Placement::High, Placement::Low], []);
+ let mut builder = JITBuilder::new(default_libcall_names()).unwrap();
+ builder.memory_provider(Box::new(memory));
+ let mut module = JITModule::new(builder);
+
+ let target_signature = module.make_signature();
+ let target = module
+ .declare_function("address_target", Linkage::Local, &target_signature)
+ .unwrap();
+ module
+ .define_function_bytes(target, 1, &[0xc3], &[])
+ .unwrap();
+
+ let mut address_signature = module.make_signature();
+ address_signature
+ .returns
+ .push(AbiParam::new(module.target_config().pointer_type()));
+ let address_func = module
+ .declare_function("function_address", Linkage::Local, &address_signature)
+ .unwrap();
+ let mut ctx = module.make_context();
+ ctx.func.name = UserFuncName::user(0, address_func.as_u32());
+ ctx.func.signature = address_signature;
+ let target_ref = module.declare_func_in_func(target, &mut ctx.func);
+ let mut func_ctx = FunctionBuilderContext::new();
+ {
+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
+ let block = bcx.create_block();
+ bcx.switch_to_block(block);
+ bcx.ins().call(target_ref, &[]);
+ let address = bcx
+ .ins()
+ .func_addr(module.target_config().pointer_type(), target_ref);
+ bcx.ins().return_(&[address]);
+ bcx.seal_all_blocks();
+ bcx.finalize(module.target_config());
+ }
+ module.define_function(address_func, &mut ctx).unwrap();
+
+ // Before the fix, finalization panics here while applying an out-of-range
+ // X86PCRel4 relocation.
+ module.finalize_definitions().unwrap();
+
+ let relocs = ctx.compiled_code().unwrap().buffer.relocs();
+ assert_eq!(relocs.len(), 2);
+ assert_eq!(
+ relocs
+ .iter()
+ .filter(|reloc| reloc.kind == Reloc::X86CallPCRel4)
+ .count(),
+ 1
+ );
+ assert_eq!(
+ relocs
+ .iter()
+ .filter(|reloc| reloc.kind == Reloc::X86GOTPCRel4)
+ .count(),
+ 1
+ );
+
+ let target_ptr = module.get_finalized_function(target);
+ let address_ptr = module.get_finalized_function(address_func);
+ assert!(address_ptr.addr().abs_diff(target_ptr.addr()) > i32::MAX as usize);
+ let get_address: extern "C" fn() -> usize = unsafe { std::mem::transmute(address_ptr) };
+ assert_eq!(get_address(), target_ptr.addr());
+
+ unsafe { module.free_memory() };
+}
diff --git a/cranelift/object/tests/basic.rs b/cranelift/object/tests/basic.rs
index 7af6b0731e..76ac29b6ec 100644
--- a/cranelift/object/tests/basic.rs
+++ b/cranelift/object/tests/basic.rs
@@ -314,6 +314,107 @@ fn aarch64_colocated_data_symbol_reloc() {
product.emit().expect("emit object file");
}
+#[test]
+fn x86_64_colocated_symbol_address_uses_non_branch_relocation() {
+ use object::{Object as _, ObjectSection as _, ObjectSymbol as _};
+
+ for (triple, expected_encoding) in [
+ (
+ "x86_64-unknown-linux-gnu",
+ object::RelocationEncoding::Generic,
+ ),
+ (
+ "x86_64-apple-darwin",
+ object::RelocationEncoding::X86RipRelative,
+ ),
+ (
+ "x86_64-pc-windows-msvc",
+ object::RelocationEncoding::Generic,
+ ),
+ ] {
+ let flag_builder = settings::builder();
+ let isa = cranelift_codegen::isa::lookup_by_name(triple)
+ .unwrap()
+ .finish(settings::Flags::new(flag_builder))
+ .unwrap();
+ let mut module =
+ ObjectModule::new(ObjectBuilder::new(isa, "test", default_libcall_names()).unwrap());
+
+ let data_id = module
+ .declare_data("my_data", Linkage::Local, true, false)
+ .unwrap();
+ let mut data_desc = DataDescription::new();
+ data_desc.define_zeroinit(8);
+ module.define_data(data_id, &data_desc).unwrap();
+
+ let sig = Signature {
+ params: vec![],
+ returns: vec![AbiParam::new(types::I64)],
+ call_conv: module.target_config().default_call_conv,
+ };
+ let func_id = module
+ .declare_function("data_address", Linkage::Local, &sig)
+ .unwrap();
+ let mut ctx = Context::new();
+ ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig);
+ let mut func_ctx = FunctionBuilderContext::new();
+ {
+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);
+ let block = bcx.create_block();
+ bcx.switch_to_block(block);
+ let data = module.declare_data_in_func(data_id, &mut bcx.func);
+ let address = bcx
+ .ins()
+ .symbol_value(module.target_config().pointer_type(), data);
+ bcx.ins().return_(&[address]);
+ bcx.seal_all_blocks();
+ bcx.finalize(module.target_config());
+ }
+ module.define_function(func_id, &mut ctx).unwrap();
+
+ let bytes = module.finish().emit().expect("emit object file");
+ let file = object::File::parse(&*bytes).expect("parse emitted object");
+ let data_symbol = file
+ .symbols()
+ .find(|symbol| {
+ symbol
+ .name()
+ .is_ok_and(|name| name.trim_start_matches('_') == "my_data")
+ })
+ .unwrap_or_else(|| panic!("{triple}: data symbol present in object"))
+ .index();
+ let function = file
+ .symbols()
+ .find(|symbol| {
+ symbol
+ .name()
+ .is_ok_and(|name| name.trim_start_matches('_') == "data_address")
+ })
+ .unwrap_or_else(|| panic!("{triple}: function symbol present in object"));
+ let section = file
+ .section_by_index(function.section_index().unwrap())
+ .expect("function section present");
+ let (_, relocation) = section
+ .relocations()
+ .find(|(_, relocation)| {
+ relocation.target() == object::read::RelocationTarget::Symbol(data_symbol)
+ })
+ .expect("address materialization relocation present");
+
+ assert_eq!(
+ relocation.kind(),
+ object::RelocationKind::Relative,
+ "{triple}"
+ );
+ assert_eq!(
+ relocation.encoding(),
+ expected_encoding,
+ "{triple}: an address materialization must not use a branch relocation"
+ );
+ assert_eq!(relocation.size(), 32, "{triple}");
+ }
+}
+
// ---------- `.eh_frame` emission tests ----------
#[cfg(feature = "unwind")]