A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift-jit: handle out-of-range x86_64 calls and symbol addresses (#13975)
Gleb Pomykalov6901aa7cee05
diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rsindex cd34c0b3ff..5ffa00efe1 100644--- a/cranelift/codegen/src/binemit/mod.rs+++ b/cranelift/codegen/src/binemit/mod.rs@@ -26,7 +26,13 @@ pub enum Reloc {Abs8,/// x86 PC-relative 4-byteX86PCRel4,- /// x86 call to PC-relative 4-byte+ /// x86 call to PC-relative 4-byte.+ ///+ /// This relocation is only ever applied to the displacement of a `call`+ /// or `jmp` instruction, never to address materialization (e.g. `lea`,+ /// which uses [`Reloc::X86PCRel4`] instead). Consumers may therefore+ /// redirect the control transfer through a veneer when the target is out+ /// of range of the 32-bit displacement.X86CallPCRel4,/// x86 call to PLT-relative 4-byteX86CallPLTRel4,diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rsindex b3104ffaa7..eb7244e00b 100644--- a/cranelift/codegen/src/isa/x64/inst/emit.rs+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs@@ -1419,9 +1419,16 @@ pub(crate) fn emit(// If we know the distance to the name is within 2GB (e.g., a// module-local function), we can generate a RIP-relative// address, with a relocation.+ //+ // Note that this is `X86PCRel4` rather than `X86CallPCRel4`:+ // the latter is reserved for the displacement of `call`/`jmp`+ // instructions, which relocation consumers may redirect through+ // a veneer if the target turns out to be out of range. This `lea`+ // computes the address of the symbol itself, so no such+ // redirection is possible.asm::inst::leaq_rm::new(*dst, riprel).emit(sink, info, state);let cur = sink.cur_offset();- sink.add_reloc_at_offset(cur - 4, Reloc::X86CallPCRel4, name, *offset - 4);+ sink.add_reloc_at_offset(cur - 4, Reloc::X86PCRel4, name, *offset - 4);} else {// The full address can be encoded in the register, with a// relocation.diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex 8134eb5dc6..3f2b6aa4ac 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -1689,7 +1689,7 @@ impl MachInstLabelUse for LabelUse {fn from_reloc(reloc: Reloc, addend: Addend) -> Option<Self> {match (reloc, addend) {- (Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),+ (Reloc::X86PCRel4 | Reloc::X86CallPCRel4, -4) => Some(LabelUse::JmpRel32),_ => None,}}diff --git a/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif b/cranelift/filetests/filetests/isa/x64/return-call-indirect.clifindex 3883012afd..b6cd768835 100644--- a/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif+++ b/cranelift/filetests/filetests/isa/x64/return-call-indirect.clif@@ -79,7 +79,7 @@ block0(v0: i64):; pushq %rbp; movq %rsp, %rbp; block1: ; offset 0x4-; leaq (%rip), %r10 ; reloc_external CallPCRel4 %callee_i64 -4+; leaq (%rip), %r10 ; reloc_external PCRel4 %callee_i64 -4; movq %rbp, %rsp; popq %rbp; jmpq *%r10diff --git a/cranelift/filetests/filetests/isa/x64/symbols.clif b/cranelift/filetests/filetests/isa/x64/symbols.clifindex c708f12570..f950c4b4ec 100644--- a/cranelift/filetests/filetests/isa/x64/symbols.clif+++ b/cranelift/filetests/filetests/isa/x64/symbols.clif@@ -50,7 +50,7 @@ block0:; pushq %rbp; movq %rsp, %rbp; block1: ; offset 0x4-; leaq (%rip), %rax ; reloc_external CallPCRel4 %func0 -4+; leaq (%rip), %rax ; reloc_external PCRel4 %func0 -4; movq %rbp, %rsp; popq %rbp; retq@@ -158,7 +158,7 @@ block0:; pushq %rbp; movq %rsp, %rbp; block1: ; offset 0x4-; leaq (%rip), %rax ; reloc_external CallPCRel4 %global0 -4+; leaq (%rip), %rax ; reloc_external PCRel4 %global0 -4; movq %rbp, %rsp; popq %rbp; retq@@ -185,7 +185,7 @@ block0:; pushq %rbp; movq %rsp, %rbp; block1: ; offset 0x4-; leaq (%rip), %rax ; reloc_external CallPCRel4 %global0 119+; leaq (%rip), %rax ; reloc_external PCRel4 %global0 119; movq %rbp, %rsp; popq %rbp; retq@@ -212,7 +212,7 @@ block0:; pushq %rbp; movq %rsp, %rbp; block1: ; offset 0x4-; leaq (%rip), %rax ; reloc_external CallPCRel4 %global0 -127+; leaq (%rip), %rax ; reloc_external PCRel4 %global0 -127; movq %rbp, %rsp; popq %rbp; retqdiff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rsindex 82cfed4422..7d50bebbe9 100644--- a/cranelift/jit/src/backend.rs+++ b/cranelift/jit/src/backend.rs@@ -20,7 +20,7 @@ use std::collections::BTreeMap;use std::collections::HashMap;use std::ffi::CString;use std::io::Write;-use target_lexicon::PointerWidth;+use target_lexicon::{Architecture, PointerWidth};const WRITABLE_DATA_ALIGNMENT: u64 = 0x8;const READONLY_DATA_ALIGNMENT: u64 = 0x1;@@ -66,7 +66,18 @@ impl JITBuilder {// which might not reach all definitions; we can't handle that here, so// we require long-range relocation types.flag_builder.set("use_colocated_libcalls", "false").unwrap();- flag_builder.set("is_pic", "false").unwrap();+ // On x86_64, enable PIC so that symbol addresses are materialized with+ // GOT-relative loads rather than RIP-relative `lea`s, which assume the+ // symbol is within ±2 GiB of the code. The JIT memory provider makes+ // no such promise, so `CompiledBlob::perform_relocations` resolves the+ // loads through per-blob GOT entries, relaxing them back to `lea` when+ // the symbol does turn out to be in range.+ let is_pic = if cfg!(target_arch = "x86_64") {+ "true"+ } else {+ "false"+ };+ flag_builder.set("is_pic", is_pic).unwrap();let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {panic!("host machine is not supported: {msg}");});@@ -357,8 +368,9 @@ impl JITModule {/// Create a new `JITModule`.pub fn new(builder: JITBuilder) -> Self {assert!(- !builder.isa.flags().is_pic(),- "cranelift-jit needs is_pic=false"+ !builder.isa.flags().is_pic()+ || builder.isa.triple().architecture == Architecture::X86_64,+ "cranelift-jit only supports is_pic=true on x86_64");let memory = builderdiff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rsindex db88301fa7..a873f7a914 100644--- a/cranelift/jit/src/compiled_blob.rs+++ b/cranelift/jit/src/compiled_blob.rs@@ -6,7 +6,16 @@ use cranelift_module::{ModuleError, ModuleReloc, ModuleRelocTarget, ModuleResultuse crate::JITMemoryProvider;use crate::memory::JITMemoryKind;-const VENEER_SIZE: usize = 24; // ldr + br + pointer+/// Size reserved per veneer. This is exactly the size of the largest veneer:+/// 16 bytes on AArch64 (`ldr` + `br` + 8-byte target address). The x86_64+/// veneer is 14 bytes (`jmp qword ptr [rip]` + 8-byte target address).+///+/// Veneers are placed directly after the compiled code, followed by the GOT+/// entries: `[code][veneers][GOT entries]`.+const VENEER_SIZE: usize = 16;++/// Size of a GOT entry: the absolute 8-byte address of a symbol.+const GOT_ENTRY_SIZE: usize = 8;/// Reads a 32bit instruction at `iptr`, and writes it again after/// being altered by `modifier`@@ -22,6 +31,7 @@ pub(crate) struct CompiledBlob {size: usize,relocs: Vec<ModuleReloc>,veneer_count: usize,+ got_count: usize,#[cfg(feature = "wasmtime-unwinder")]wasmtime_exception_data: Option<Vec<u8>>,}@@ -37,17 +47,28 @@ impl CompiledBlob {#[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option<Vec<u8>>,kind: JITMemoryKind,) -> ModuleResult<Self> {- // Reserve veneers for all function calls just in case+ // Reserve a worst-case veneer slot for every branch relocation and a+ // GOT entry for every GOT-relative load in case its target is out of+ // range.let mut veneer_count = 0;+ let mut got_count = 0;for reloc in &relocs {match reloc.kind {- Reloc::Arm64Call => veneer_count += 1,+ Reloc::Arm64Call | Reloc::X86CallPCRel4 | Reloc::X86CallPLTRel4 => {+ veneer_count += 1+ }+ Reloc::X86GOTPCRel4 => got_count += 1,_ => {}}}+ let mut alloc_size = data.len() + veneer_count * VENEER_SIZE;+ if got_count > 0 {+ alloc_size = alloc_size.next_multiple_of(GOT_ENTRY_SIZE) + got_count * GOT_ENTRY_SIZE;+ }+let ptr = memory- .allocate(data.len() + veneer_count * VENEER_SIZE, align, kind)+ .allocate(alloc_size, align, kind).map_err(|e| ModuleError::Allocation { err: e })?;unsafe {@@ -59,6 +80,7 @@ impl CompiledBlob {size: data.len(),relocs,veneer_count,+ got_count,#[cfg(feature = "wasmtime-unwinder")]wasmtime_exception_data,})@@ -83,6 +105,7 @@ impl CompiledBlob {size,relocs,veneer_count: 0,+ got_count: 0,#[cfg(feature = "wasmtime-unwinder")]wasmtime_exception_data,})@@ -101,6 +124,12 @@ impl CompiledBlob {self.wasmtime_exception_data.as_deref()}+ /// Offset of the GOT entries within the allocation: after the code and+ /// the veneers, aligned to the entry size.+ fn got_offset(&self) -> usize {+ (self.size + self.veneer_count * VENEER_SIZE).next_multiple_of(GOT_ENTRY_SIZE)+ }+pub(crate) fn perform_relocations(&self,get_address: impl Fn(&ModuleRelocTarget) -> *const u8,@@ -108,6 +137,7 @@ impl CompiledBlob {use std::ptr::write_unaligned;let mut next_veneer_idx = 0;+ let mut next_got_idx = 0;let relocation_target_addr = |name: &ModuleRelocTarget, addend: Addend| {let addend = isize::try_from(addend).unwrap();get_address(name)@@ -137,16 +167,88 @@ impl CompiledBlob {let what = relocation_target_addr(name, addend);unsafe { write_unaligned(at as *mut u64, u64::try_from(what).unwrap()) };}- Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {+ Reloc::X86PCRel4 => {let what = relocation_target_addr(name, addend);let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();unsafe { write_unaligned(at as *mut i32, pcrel) };}- Reloc::X86GOTPCRel4 => {- panic!("GOT relocation shouldn't be generated when !is_pic");+ Reloc::X86CallPCRel4 | Reloc::X86CallPLTRel4 => {+ // These relocations are applied to the 32-bit displacement of a `call`+ // or `jmp` instruction, which is relative to the end of the+ // instruction, i.e. 4 bytes past `at`. The addend (always -4 as+ // emitted by the x64 backend) accounts for this, so the instruction+ // transfers control to `what + 4`. The PLT form additionally permits+ // resolution through a stub, which the veneer below provides.+ let what = relocation_target_addr(name, addend);+ if let Ok(pcrel) = i32::try_from((what as isize) - (at as isize)) {+ unsafe { write_unaligned(at as *mut i32, pcrel) };+ } else {+ // The target is out of range for the 32-bit displacement, so+ // redirect the call through a veneer at the end of the function,+ // just like for `Arm64Call` below: `jmp qword ptr [rip]` followed+ // by the absolute target address.+ let veneer_idx = next_veneer_idx;+ next_veneer_idx += 1;+ assert!(veneer_idx < self.veneer_count);+ let veneer =+ unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };++ let target = u64::try_from(what.checked_add(4).unwrap()).unwrap();+ unsafe {+ ptr::copy_nonoverlapping([0xff, 0x25, 0, 0, 0, 0].as_ptr(), veneer, 6);+ write_unaligned(veneer.byte_add(6).cast::<u64>(), target);+ }++ // Point the original instruction at the veneer instead; the+ // displacement is again relative to the end of the 4-byte field.+ let pcrel = i32::try_from((veneer as isize) - (at as isize) - 4).unwrap();+ unsafe { write_unaligned(at as *mut i32, pcrel) };+ }}- Reloc::X86CallPLTRel4 => {- panic!("PLT relocation shouldn't be generated when !is_pic");+ Reloc::X86GOTPCRel4 => {+ // This relocation is applied to the 32-bit displacement of a+ // `mov reg, qword ptr [rip + disp]` instruction that loads the+ // address of a symbol from its GOT entry. The addend (always -4 as+ // emitted by the x64 backend) accounts for the displacement being+ // relative to the end of the instruction, and any symbol offset is+ // added by a separately emitted instruction, so the GOT entry holds+ // the address of the symbol itself: `what + 4`.+ let what = relocation_target_addr(name, addend);+ let symbol_addr = u64::try_from(what.checked_add(4).unwrap()).unwrap();++ // If the symbol is within displacement range, relax the load to a+ // `lea` computing its address directly, the same way linkers relax+ // `R_X86_64_REX_GOTPCRELX`. The instruction is expected to be a REX+ // prefix, the `mov` opcode 0x8b, and a ModRM byte with mod=0b00 and+ // r/m=0b101 (RIP-relative); only rewrite it if it matches.+ let insn = (offset >= 3)+ .then(|| unsafe { at.cast_const().sub(3).cast::<[u8; 3]>().read() });+ let is_rip_relative_mov = insn.is_some_and(+ |insn| matches!(insn, [0x48..=0x4f, 0x8b, modrm] if modrm & 0xc7 == 0x05),+ );+ let direct_pcrel = i32::try_from((what as isize) - (at as isize));+ match (is_rip_relative_mov, direct_pcrel) {+ (true, Ok(pcrel)) => unsafe {+ at.sub(2).write(0x8d); // mov -> lea+ write_unaligned(at as *mut i32, pcrel);+ },+ _ => {+ // Resolve through a GOT entry at the end of the allocation,+ // which is always in range of the load.+ let got_idx = next_got_idx;+ next_got_idx += 1;+ assert!(got_idx < self.got_count);+ let entry = unsafe {+ self.ptr+ .byte_add(self.got_offset() + got_idx * GOT_ENTRY_SIZE)+ };+ unsafe { write_unaligned(entry.cast::<u64>(), symbol_addr) };++ let pcrel =+ i32::try_from((entry as isize) - (at as isize) - 4).unwrap();+ unsafe { write_unaligned(at as *mut i32, pcrel) };+ }+ }}Reloc::S390xPCRel32Dbl | Reloc::S390xPLTRel32Dbl => {let what = relocation_target_addr(name, addend);@@ -176,7 +278,7 @@ impl CompiledBlob {// end of the function.let veneer_idx = next_veneer_idx;next_veneer_idx += 1;- assert!(veneer_idx <= self.veneer_count);+ assert!(veneer_idx < self.veneer_count);let veneer =unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };diff --git a/cranelift/jit/tests/basic.rs b/cranelift/jit/tests/basic.rsindex 064ab51646..4a5bfbfea9 100644--- a/cranelift/jit/tests/basic.rs+++ b/cranelift/jit/tests/basic.rs@@ -10,8 +10,14 @@ use cranelift_module::*;fn isa() -> Option<OwnedTargetIsa> {let mut flag_builder = settings::builder();flag_builder.set("use_colocated_libcalls", "false").unwrap();- // FIXME set back to true once the x64 backend supports it.- flag_builder.set("is_pic", "false").unwrap();+ // PIC is only supported by the JIT on x86_64, where it routes symbol+ // address materialization through per-blob GOT entries.+ let is_pic = if cfg!(target_arch = "x86_64") {+ "true"+ } else {+ "false"+ };+ flag_builder.set("is_pic", is_pic).unwrap();let isa_builder = cranelift_native::builder().ok()?;isa_builder.finish(settings::Flags::new(flag_builder)).ok()}@@ -219,3 +225,682 @@ fn empty_data_object() {data.define(Box::new([]));module.define_data(data_id, &data).unwrap();}++#[cfg(all(target_arch = "x86_64", target_os = "linux"))]+mod far_x86_memory {+ use std::collections::VecDeque;+ use std::io;+ use std::ptr;+ use std::sync::{Arc, Mutex};++ use cranelift_jit::{BranchProtection, JITMemoryKind, JITMemoryProvider};+ use cranelift_module::ModuleResult;++ pub(super) const VENEER_SIZE: usize = 16;++ struct ReservedAddressSpace {+ base: usize,+ len: usize,+ page_size: usize,+ }++ impl ReservedAddressSpace {+ fn new() -> Self {+ let page_size = usize::try_from(unsafe { libc::sysconf(libc::_SC_PAGESIZE) }).unwrap();+ let len = (i32::MAX as usize)+ .checked_add(16 * 1024 * 1024)+ .unwrap()+ .next_multiple_of(page_size);+ let mapping = unsafe {+ libc::mmap(+ ptr::null_mut(),+ len,+ libc::PROT_NONE,+ libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_NORESERVE,+ -1,+ 0,+ )+ };+ assert_ne!(mapping, libc::MAP_FAILED);+ Self {+ base: mapping.addr(),+ len,+ page_size,+ }+ }+ }++ impl Drop for ReservedAddressSpace {+ fn drop(&mut self) {+ let result = unsafe { libc::munmap(self.base as *mut libc::c_void, self.len) };+ assert_eq!(result, 0);+ }+ }++ #[derive(Clone, Copy)]+ pub(super) enum Placement {+ Low,+ High,+ }++ #[derive(Clone, Copy)]+ enum FinalProtection {+ ReadExecute,+ ReadOnly,+ ReadWrite,+ }++ struct Allocation {+ addr: usize,+ len: usize,+ final_protection: FinalProtection,+ }++ /// Places JIT allocations at predetermined ends of a virtual address+ /// reservation so tests can deterministically exceed `rel32` range.+ pub(super) struct FarMemoryProvider {+ space: ReservedAddressSpace,+ executable_placements: VecDeque<Placement>,+ readonly_placements: VecDeque<Placement>,+ writable_placements: VecDeque<Placement>,+ low_offset: usize,+ high_offset: usize,+ allocations: Vec<Allocation>,+ requested_exec_sizes: Arc<Mutex<Vec<usize>>>,+ }++ impl FarMemoryProvider {+ pub(super) fn new(+ executable_placements: impl IntoIterator<Item = Placement>,+ readonly_placements: impl IntoIterator<Item = Placement>,+ ) -> (Self, Arc<Mutex<Vec<usize>>>) {+ let space = ReservedAddressSpace::new();+ let high_offset = space.len;+ let requested_exec_sizes = Arc::new(Mutex::new(Vec::new()));+ (+ Self {+ space,+ executable_placements: executable_placements.into_iter().collect(),+ readonly_placements: readonly_placements.into_iter().collect(),+ writable_placements: VecDeque::new(),+ low_offset: 0,+ high_offset,+ allocations: Vec::new(),+ requested_exec_sizes: Arc::clone(&requested_exec_sizes),+ },+ requested_exec_sizes,+ )+ }++ fn allocate_at(+ &mut self,+ size: usize,+ align: u64,+ placement: Placement,+ final_protection: FinalProtection,+ ) -> io::Result<*mut u8> {+ assert!(usize::try_from(align).unwrap() <= self.space.page_size);+ let len = size+ .next_multiple_of(self.space.page_size)+ .max(self.space.page_size);+ let addr = match placement {+ Placement::Low => {+ let addr = self.space.base + self.low_offset;+ self.low_offset += len;+ addr+ }+ Placement::High => {+ self.high_offset -= len;+ self.space.base + self.high_offset+ }+ };+ assert!(self.low_offset <= self.high_offset);+ let result = unsafe {+ libc::mprotect(+ addr as *mut libc::c_void,+ len,+ libc::PROT_READ | libc::PROT_WRITE,+ )+ };+ assert_eq!(result, 0);+ self.allocations.push(Allocation {+ addr,+ len,+ final_protection,+ });+ Ok(addr as *mut u8)+ }+ }++ impl JITMemoryProvider for FarMemoryProvider {+ fn allocate(+ &mut self,+ size: usize,+ align: u64,+ kind: JITMemoryKind,+ ) -> io::Result<*mut u8> {+ let (placement, final_protection) = match kind {+ JITMemoryKind::Executable => {+ self.requested_exec_sizes.lock().unwrap().push(size);+ (+ self.executable_placements+ .pop_front()+ .expect("placement for every executable allocation"),+ FinalProtection::ReadExecute,+ )+ }+ JITMemoryKind::Writable => (+ self.writable_placements+ .pop_front()+ .expect("placement for every writable allocation"),+ FinalProtection::ReadWrite,+ ),+ JITMemoryKind::ReadOnly => (+ self.readonly_placements+ .pop_front()+ .expect("placement for every read-only allocation"),+ FinalProtection::ReadOnly,+ ),+ };+ self.allocate_at(size, align, placement, final_protection)+ }++ unsafe fn free_memory(&mut self) {+ // All allocations are freed together when `space` is unmapped.+ self.allocations.clear();+ }++ fn finalize(&mut self, _branch_protection: BranchProtection) -> ModuleResult<()> {+ for allocation in &self.allocations {+ let protection = match allocation.final_protection {+ FinalProtection::ReadExecute => libc::PROT_READ | libc::PROT_EXEC,+ FinalProtection::ReadOnly => libc::PROT_READ,+ FinalProtection::ReadWrite => libc::PROT_READ | libc::PROT_WRITE,+ };+ let result = unsafe {+ libc::mprotect(+ allocation.addr as *mut libc::c_void,+ allocation.len,+ protection,+ )+ };+ assert_eq!(result, 0);+ }+ Ok(())+ }+ }+}++/// Reproduces a bug where a `call` or `jmp` between two functions of the same+/// module that happen to be placed further than ±2 GiB apart panicked while+/// applying the `X86CallPCRel4` relocation, instead of routing the control+/// transfer through a veneer like AArch64 already did for `Arm64Call`.+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]+#[test]+fn far_x86_control_transfers_use_veneers() {+ use std::mem;++ use cranelift_codegen::binemit::Reloc;++ use far_x86_memory::{FarMemoryProvider, Placement, VENEER_SIZE};++ let (memory, requested_exec_sizes) = FarMemoryProvider::new(+ [+ Placement::Low,+ Placement::High,+ Placement::Low,+ Placement::High,+ Placement::Low,+ Placement::Low,+ Placement::High,+ Placement::High,+ ],+ [],+ );+ let mut builder = JITBuilder::new(default_libcall_names()).unwrap();+ builder.memory_provider(Box::new(memory));++ let mut module = JITModule::new(builder);+ let mut signature = module.make_signature();+ signature.returns.push(AbiParam::new(types::I32));+ let low_callee = module+ .declare_function("low_callee", Linkage::Local, &signature)+ .unwrap();+ let high_caller = module+ .declare_function("high_caller", Linkage::Local, &signature)+ .unwrap();+ let low_tail_caller = module+ .declare_function("low_tail_caller", Linkage::Local, &signature)+ .unwrap();+ let high_tail_callee = module+ .declare_function("high_tail_callee", Linkage::Local, &signature)+ .unwrap();++ // low_callee: mov eax, 42; ret+ module+ .define_function_bytes(low_callee, 1, &[0xb8, 42, 0, 0, 0, 0xc3], &[])+ .unwrap();++ // Compile high_caller normally so this test also checks the relocation+ // emitted for a compiler-generated direct call.+ let mut ctx = module.make_context();+ ctx.func.name = UserFuncName::user(0, high_caller.as_u32());+ ctx.func.signature = signature.clone();+ let low_callee_ref = module.declare_func_in_func(low_callee, &mut ctx.func);+ let mut func_ctx = FunctionBuilderContext::new();+ {+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);+ let block = bcx.create_block();+ bcx.switch_to_block(block);+ let call = bcx.ins().call(low_callee_ref, &[]);+ let result = bcx.inst_results(call)[0];+ bcx.ins().return_(&[result]);+ }+ module.define_function(high_caller, &mut ctx).unwrap();+ let (high_caller_code_len, high_caller_reloc_offset) = {+ let compiled = ctx.compiled_code().unwrap();+ let reloc = compiled+ .buffer+ .relocs()+ .iter()+ .find(|reloc| reloc.kind == Reloc::X86CallPCRel4)+ .unwrap();+ (compiled.code_buffer().len(), reloc.offset as usize)+ };++ // low_tail_caller: jmp high_tail_callee+ let relocations = [ModuleReloc {+ offset: 1,+ kind: Reloc::X86CallPCRel4,+ name: ModuleRelocTarget::user(0, high_tail_callee.as_u32()),+ addend: -4,+ }];+ module+ .define_function_bytes(low_tail_caller, 1, &[0xe9, 0, 0, 0, 0], &relocations)+ .unwrap();++ // high_tail_callee: mov eax, 84; ret+ module+ .define_function_bytes(high_tail_callee, 1, &[0xb8, 84, 0, 0, 0, 0xc3], &[])+ .unwrap();++ // A caller with one near target and two far targets checks that only the+ // far calls use distinct, densely packed veneers.+ let mixed_caller = module+ .declare_function("mixed_caller", Linkage::Local, &signature)+ .unwrap();+ let near = module+ .declare_function("near", Linkage::Local, &signature)+ .unwrap();+ let far_a = module+ .declare_function("far_a", Linkage::Local, &signature)+ .unwrap();+ let far_b = module+ .declare_function("far_b", Linkage::Local, &signature)+ .unwrap();+ let mixed_caller_code = [+ 0x48, 0x83, 0xec, 0x08, // sub rsp, 8+ 0xe8, 0, 0, 0, 0, // call near+ 0xe8, 0, 0, 0, 0, // call far_a+ 0xe8, 0, 0, 0, 0, // call far_b+ 0x48, 0x83, 0xc4, 0x08, // add rsp, 8+ 0xc3, // ret+ ];+ let mixed_relocations = [+ ModuleReloc {+ offset: 5,+ kind: Reloc::X86CallPCRel4,+ name: ModuleRelocTarget::user(0, near.as_u32()),+ addend: -4,+ },+ ModuleReloc {+ offset: 10,+ kind: Reloc::X86CallPCRel4,+ name: ModuleRelocTarget::user(0, far_a.as_u32()),+ addend: -4,+ },+ ModuleReloc {+ offset: 15,+ kind: Reloc::X86CallPCRel4,+ name: ModuleRelocTarget::user(0, far_b.as_u32()),+ addend: -4,+ },+ ];+ module+ .define_function_bytes(mixed_caller, 1, &mixed_caller_code, &mixed_relocations)+ .unwrap();+ module+ .define_function_bytes(near, 1, &[0xb8, 1, 0, 0, 0, 0xc3], &[])+ .unwrap();+ module+ .define_function_bytes(far_a, 1, &[0xb8, 2, 0, 0, 0, 0xc3], &[])+ .unwrap();+ module+ .define_function_bytes(far_b, 1, &[0xb8, 42, 0, 0, 0, 0xc3], &[])+ .unwrap();++ module.finalize_definitions().unwrap();++ let low_callee_ptr = module.get_finalized_function(low_callee);+ let high_caller_ptr = module.get_finalized_function(high_caller);+ let low_tail_caller_ptr = module.get_finalized_function(low_tail_caller);+ let high_tail_callee_ptr = module.get_finalized_function(high_tail_callee);+ let mixed_caller_ptr = module.get_finalized_function(mixed_caller);+ let near_ptr = module.get_finalized_function(near);+ let far_a_ptr = module.get_finalized_function(far_a);+ let far_b_ptr = module.get_finalized_function(far_b);++ fn branch_destination(caller: *const u8, reloc_offset: usize) -> *const u8 {+ let at = caller.wrapping_byte_add(reloc_offset);+ let displacement = unsafe { at.cast::<i32>().read_unaligned() };+ at.wrapping_byte_add(4)+ .wrapping_byte_offset(displacement as isize)+ }++ fn assert_veneer(+ caller: *const u8,+ code_len: usize,+ reloc_offset: usize,+ expected_target: *const u8,+ ) -> *const u8 {+ assert!(caller.addr().abs_diff(expected_target.addr()) > i32::MAX as usize);++ let veneer = branch_destination(caller, reloc_offset);+ assert_eq!(veneer, caller.wrapping_byte_add(code_len));++ let veneer_code = unsafe { std::slice::from_raw_parts(veneer, 6) };+ assert_eq!(veneer_code, [0xff, 0x25, 0, 0, 0, 0]);+ let veneer_target = unsafe { veneer.byte_add(6).cast::<u64>().read_unaligned() };+ assert_eq!(veneer_target, expected_target.addr() as u64);+ veneer+ }++ // Check a negative-distance call and a positive-distance tail jump.+ assert!(high_caller_ptr.addr() > low_callee_ptr.addr());+ assert_veneer(+ high_caller_ptr,+ high_caller_code_len,+ high_caller_reloc_offset,+ low_callee_ptr,+ );+ let high_caller_fn: extern "C" fn() -> u32 = unsafe { mem::transmute(high_caller_ptr) };+ assert_eq!(high_caller_fn(), 42);++ assert!(low_tail_caller_ptr.addr() < high_tail_callee_ptr.addr());+ assert_veneer(low_tail_caller_ptr, 5, 1, high_tail_callee_ptr);+ let low_tail_caller_fn: extern "C" fn() -> u32 = unsafe { mem::transmute(low_tail_caller_ptr) };+ assert_eq!(low_tail_caller_fn(), 84);++ assert!(mixed_caller_ptr.addr().abs_diff(near_ptr.addr()) <= i32::MAX as usize);+ assert!(mixed_caller_ptr.addr().abs_diff(far_a_ptr.addr()) > i32::MAX as usize);+ assert!(mixed_caller_ptr.addr().abs_diff(far_b_ptr.addr()) > i32::MAX as usize);+ assert_eq!(branch_destination(mixed_caller_ptr, 5), near_ptr);+ let veneer_a = assert_veneer(mixed_caller_ptr, mixed_caller_code.len(), 10, far_a_ptr);+ let veneer_b = assert_veneer(+ mixed_caller_ptr,+ mixed_caller_code.len() + VENEER_SIZE,+ 15,+ far_b_ptr,+ );+ assert_eq!(veneer_b, veneer_a.wrapping_byte_add(VENEER_SIZE));+ let mixed_caller_fn: extern "C" fn() -> u32 = unsafe { mem::transmute(mixed_caller_ptr) };+ assert_eq!(mixed_caller_fn(), 42);++ // One worst-case slot is reserved for every branch relocation, including+ // the call that ultimately resolves directly.+ let requested_exec_sizes = requested_exec_sizes.lock().unwrap();+ assert_eq!(requested_exec_sizes[1], high_caller_code_len + VENEER_SIZE);+ assert_eq!(requested_exec_sizes[2], 5 + VENEER_SIZE);+ assert_eq!(+ requested_exec_sizes[4],+ mixed_caller_code.len() + mixed_relocations.len() * VENEER_SIZE+ );+ drop(requested_exec_sizes);++ unsafe { module.free_memory() };+}++/// Reproduces the Roto failure: JIT code materializes the address of an+/// anonymous read-only data object allocated more than ±2 GiB away. The+/// GOT-relative load cannot be relaxed to a `lea` and reads the address from+/// a GOT entry at the end of the function's allocation.+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]+#[test]+fn far_x86_data_address_resolves_through_got() {+ use cranelift_codegen::binemit::Reloc;++ use far_x86_memory::{FarMemoryProvider, Placement};++ let (memory, _) = FarMemoryProvider::new([Placement::Low], [Placement::High]);+ let mut builder = JITBuilder::new(default_libcall_names()).unwrap();+ builder.memory_provider(Box::new(memory));+ let mut module = JITModule::new(builder);++ let data_id = module.declare_anonymous_data(false, false).unwrap();+ let mut data = DataDescription::new();+ data.define(vec![42].into_boxed_slice());+ module.define_data(data_id, &data).unwrap();++ let mut signature = module.make_signature();+ signature+ .returns+ .push(AbiParam::new(module.target_config().pointer_type()));+ let address_func = module+ .declare_function("data_address", Linkage::Local, &signature)+ .unwrap();+ let mut ctx = module.make_context();+ ctx.func.name = UserFuncName::user(0, address_func.as_u32());+ ctx.func.signature = signature;+ let data_ref = module.declare_data_in_func(data_id, &mut ctx.func);+ let mut func_ctx = FunctionBuilderContext::new();+ {+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);+ let block = bcx.create_block();+ bcx.switch_to_block(block);+ let address = bcx+ .ins()+ .symbol_value(module.target_config().pointer_type(), data_ref);+ bcx.ins().return_(&[address]);+ bcx.seal_all_blocks();+ bcx.finalize(module.target_config());+ }+ module.define_function(address_func, &mut ctx).unwrap();++ // Before the fix, finalization panics here while applying an out-of-range+ // X86PCRel4 relocation.+ module.finalize_definitions().unwrap();++ let relocs = ctx.compiled_code().unwrap().buffer.relocs();+ assert_eq!(relocs.len(), 1);+ assert_eq!(relocs[0].kind, Reloc::X86GOTPCRel4);+ let reloc_offset = usize::try_from(relocs[0].offset).unwrap();++ let (data_ptr, _) = module.get_finalized_data(data_id);+ let address_ptr = module.get_finalized_function(address_func);+ assert!(address_ptr.addr().abs_diff(data_ptr.addr()) > i32::MAX as usize);++ // The load stays a `mov` and reads the data object's address from a GOT+ // entry at the end of the function's allocation.+ assert_eq!(+ unsafe { address_ptr.byte_add(reloc_offset - 2).read() },+ 0x8b+ );+ let displacement = unsafe {+ address_ptr+ .byte_add(reloc_offset)+ .cast::<i32>()+ .read_unaligned()+ };+ let got_entry = address_ptr+ .wrapping_byte_add(reloc_offset + 4)+ .wrapping_byte_offset(displacement as isize);+ assert!(got_entry.addr() > address_ptr.addr());+ assert_eq!(+ unsafe { got_entry.cast::<u64>().read_unaligned() },+ data_ptr.addr() as u64+ );++ let get_address: extern "C" fn() -> usize = unsafe { std::mem::transmute(address_ptr) };+ assert_eq!(get_address(), data_ptr.addr());++ unsafe { module.free_memory() };+}++/// When the memory provider does place a symbol within displacement range,+/// the GOT-relative load is relaxed to a `lea` computing the address+/// directly, so near symbol accesses don't pay for a GOT indirection.+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]+#[test]+fn near_x86_data_address_relaxes_got_load_to_lea() {+ use cranelift_codegen::binemit::Reloc;++ use far_x86_memory::{FarMemoryProvider, Placement};++ let (memory, _) = FarMemoryProvider::new([Placement::Low], [Placement::Low]);+ let mut builder = JITBuilder::new(default_libcall_names()).unwrap();+ builder.memory_provider(Box::new(memory));+ let mut module = JITModule::new(builder);++ let data_id = module.declare_anonymous_data(false, false).unwrap();+ let mut data = DataDescription::new();+ data.define(vec![42].into_boxed_slice());+ module.define_data(data_id, &data).unwrap();++ let mut signature = module.make_signature();+ signature+ .returns+ .push(AbiParam::new(module.target_config().pointer_type()));+ let address_func = module+ .declare_function("data_address", Linkage::Local, &signature)+ .unwrap();+ let mut ctx = module.make_context();+ ctx.func.name = UserFuncName::user(0, address_func.as_u32());+ ctx.func.signature = signature;+ let data_ref = module.declare_data_in_func(data_id, &mut ctx.func);+ let mut func_ctx = FunctionBuilderContext::new();+ {+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);+ let block = bcx.create_block();+ bcx.switch_to_block(block);+ let address = bcx+ .ins()+ .symbol_value(module.target_config().pointer_type(), data_ref);+ bcx.ins().return_(&[address]);+ bcx.seal_all_blocks();+ bcx.finalize(module.target_config());+ }+ module.define_function(address_func, &mut ctx).unwrap();+ module.finalize_definitions().unwrap();++ let relocs = ctx.compiled_code().unwrap().buffer.relocs();+ assert_eq!(relocs.len(), 1);+ assert_eq!(relocs[0].kind, Reloc::X86GOTPCRel4);+ let reloc_offset = usize::try_from(relocs[0].offset).unwrap();++ let (data_ptr, _) = module.get_finalized_data(data_id);+ let address_ptr = module.get_finalized_function(address_func);+ assert!(address_ptr.addr().abs_diff(data_ptr.addr()) <= i32::MAX as usize);++ // The load was relaxed to a `lea` pointing directly at the data object.+ assert_eq!(+ unsafe { address_ptr.byte_add(reloc_offset - 2).read() },+ 0x8d+ );+ let displacement = unsafe {+ address_ptr+ .byte_add(reloc_offset)+ .cast::<i32>()+ .read_unaligned()+ };+ assert_eq!(+ address_ptr+ .wrapping_byte_add(reloc_offset + 4)+ .wrapping_byte_offset(displacement as isize),+ data_ptr+ );++ let get_address: extern "C" fn() -> usize = unsafe { std::mem::transmute(address_ptr) };+ assert_eq!(get_address(), data_ptr.addr());++ unsafe { module.free_memory() };+}++/// A materialized function pointer has the same range issue as a data pointer,+/// but direct calls to that function must remain eligible for call veneers.+/// The GOT entry holds the function's real address, preserving pointer+/// identity with `get_finalized_function`.+#[cfg(all(target_arch = "x86_64", target_os = "linux"))]+#[test]+fn far_x86_function_address_resolves_through_got() {+ use cranelift_codegen::binemit::Reloc;++ use far_x86_memory::{FarMemoryProvider, Placement};++ let (memory, _) = FarMemoryProvider::new([Placement::High, Placement::Low], []);+ let mut builder = JITBuilder::new(default_libcall_names()).unwrap();+ builder.memory_provider(Box::new(memory));+ let mut module = JITModule::new(builder);++ let target_signature = module.make_signature();+ let target = module+ .declare_function("address_target", Linkage::Local, &target_signature)+ .unwrap();+ module+ .define_function_bytes(target, 1, &[0xc3], &[])+ .unwrap();++ let mut address_signature = module.make_signature();+ address_signature+ .returns+ .push(AbiParam::new(module.target_config().pointer_type()));+ let address_func = module+ .declare_function("function_address", Linkage::Local, &address_signature)+ .unwrap();+ let mut ctx = module.make_context();+ ctx.func.name = UserFuncName::user(0, address_func.as_u32());+ ctx.func.signature = address_signature;+ let target_ref = module.declare_func_in_func(target, &mut ctx.func);+ let mut func_ctx = FunctionBuilderContext::new();+ {+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);+ let block = bcx.create_block();+ bcx.switch_to_block(block);+ bcx.ins().call(target_ref, &[]);+ let address = bcx+ .ins()+ .func_addr(module.target_config().pointer_type(), target_ref);+ bcx.ins().return_(&[address]);+ bcx.seal_all_blocks();+ bcx.finalize(module.target_config());+ }+ module.define_function(address_func, &mut ctx).unwrap();++ // Before the fix, finalization panics here while applying an out-of-range+ // X86PCRel4 relocation.+ module.finalize_definitions().unwrap();++ let relocs = ctx.compiled_code().unwrap().buffer.relocs();+ assert_eq!(relocs.len(), 2);+ assert_eq!(+ relocs+ .iter()+ .filter(|reloc| reloc.kind == Reloc::X86CallPCRel4)+ .count(),+ 1+ );+ assert_eq!(+ relocs+ .iter()+ .filter(|reloc| reloc.kind == Reloc::X86GOTPCRel4)+ .count(),+ 1+ );++ let target_ptr = module.get_finalized_function(target);+ let address_ptr = module.get_finalized_function(address_func);+ assert!(address_ptr.addr().abs_diff(target_ptr.addr()) > i32::MAX as usize);+ let get_address: extern "C" fn() -> usize = unsafe { std::mem::transmute(address_ptr) };+ assert_eq!(get_address(), target_ptr.addr());++ unsafe { module.free_memory() };+}diff --git a/cranelift/object/tests/basic.rs b/cranelift/object/tests/basic.rsindex 7af6b0731e..76ac29b6ec 100644--- a/cranelift/object/tests/basic.rs+++ b/cranelift/object/tests/basic.rs@@ -314,6 +314,107 @@ fn aarch64_colocated_data_symbol_reloc() {product.emit().expect("emit object file");}+#[test]+fn x86_64_colocated_symbol_address_uses_non_branch_relocation() {+ use object::{Object as _, ObjectSection as _, ObjectSymbol as _};++ for (triple, expected_encoding) in [+ (+ "x86_64-unknown-linux-gnu",+ object::RelocationEncoding::Generic,+ ),+ (+ "x86_64-apple-darwin",+ object::RelocationEncoding::X86RipRelative,+ ),+ (+ "x86_64-pc-windows-msvc",+ object::RelocationEncoding::Generic,+ ),+ ] {+ let flag_builder = settings::builder();+ let isa = cranelift_codegen::isa::lookup_by_name(triple)+ .unwrap()+ .finish(settings::Flags::new(flag_builder))+ .unwrap();+ let mut module =+ ObjectModule::new(ObjectBuilder::new(isa, "test", default_libcall_names()).unwrap());++ let data_id = module+ .declare_data("my_data", Linkage::Local, true, false)+ .unwrap();+ let mut data_desc = DataDescription::new();+ data_desc.define_zeroinit(8);+ module.define_data(data_id, &data_desc).unwrap();++ let sig = Signature {+ params: vec![],+ returns: vec![AbiParam::new(types::I64)],+ call_conv: module.target_config().default_call_conv,+ };+ let func_id = module+ .declare_function("data_address", Linkage::Local, &sig)+ .unwrap();+ let mut ctx = Context::new();+ ctx.func = Function::with_name_signature(UserFuncName::user(0, func_id.as_u32()), sig);+ let mut func_ctx = FunctionBuilderContext::new();+ {+ let mut bcx = FunctionBuilder::new(&mut ctx.func, &mut func_ctx);+ let block = bcx.create_block();+ bcx.switch_to_block(block);+ let data = module.declare_data_in_func(data_id, &mut bcx.func);+ let address = bcx+ .ins()+ .symbol_value(module.target_config().pointer_type(), data);+ bcx.ins().return_(&[address]);+ bcx.seal_all_blocks();+ bcx.finalize(module.target_config());+ }+ module.define_function(func_id, &mut ctx).unwrap();++ let bytes = module.finish().emit().expect("emit object file");+ let file = object::File::parse(&*bytes).expect("parse emitted object");+ let data_symbol = file+ .symbols()+ .find(|symbol| {+ symbol+ .name()+ .is_ok_and(|name| name.trim_start_matches('_') == "my_data")+ })+ .unwrap_or_else(|| panic!("{triple}: data symbol present in object"))+ .index();+ let function = file+ .symbols()+ .find(|symbol| {+ symbol+ .name()+ .is_ok_and(|name| name.trim_start_matches('_') == "data_address")+ })+ .unwrap_or_else(|| panic!("{triple}: function symbol present in object"));+ let section = file+ .section_by_index(function.section_index().unwrap())+ .expect("function section present");+ let (_, relocation) = section+ .relocations()+ .find(|(_, relocation)| {+ relocation.target() == object::read::RelocationTarget::Symbol(data_symbol)+ })+ .expect("address materialization relocation present");++ assert_eq!(+ relocation.kind(),+ object::RelocationKind::Relative,+ "{triple}"+ );+ assert_eq!(+ relocation.encoding(),+ expected_encoding,+ "{triple}: an address materialization must not use a branch relocation"+ );+ assert_eq!(relocation.size(), 32, "{triple}");+ }+}+// ---------- `.eh_frame` emission tests ----------#[cfg(feature = "unwind")]