A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
//! Defines `JITModule`.
use crate::{
compiled_blob::CompiledBlob,
memory::{BranchProtection, JITMemoryKind, JITMemoryProvider, SystemMemoryProvider},
};
use cranelift_codegen::binemit::Reloc;
use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa};
use cranelift_codegen::settings::Configurable;
use cranelift_codegen::{ir, settings};
use cranelift_control::ControlPlane;
use cranelift_entity::SecondaryMap;
use cranelift_module::{
DataDescription, DataId, FuncId, Init, Linkage, Module, ModuleDeclarations, ModuleError,
ModuleReloc, ModuleRelocTarget, ModuleResult,
};
use log::info;
use std::cell::RefCell;
use std::collections::BTreeMap;
use std::collections::HashMap;
use std::ffi::CString;
use std::io::Write;
use target_lexicon::{Architecture, PointerWidth};
const WRITABLE_DATA_ALIGNMENT: u64 = 0x8;
const READONLY_DATA_ALIGNMENT: u64 = 0x1;
/// A builder for `JITModule`.
pub struct JITBuilder {
isa: OwnedTargetIsa,
symbols: HashMap<String, SendWrapper<*const u8>>,
lookup_symbols: Vec<Box<dyn Fn(&str) -> Option<*const u8> + Send>>,
libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
memory: Option<Box<dyn JITMemoryProvider + Send>>,
}
impl JITBuilder {
/// Create a new `JITBuilder`.
///
/// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall`
/// enum to symbols. LibCalls are inserted in the IR as part of the legalization for certain
/// floating point instructions, and for stack probes. If you don't know what to use for this
/// argument, use `cranelift_module::default_libcall_names()`.
pub fn new(
libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
) -> ModuleResult<Self> {
Self::with_flags(&[], libcall_names)
}
/// Create a new `JITBuilder` with the given flags.
///
/// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall`
/// enum to symbols. LibCalls are inserted in the IR as part of the legalization for certain
/// floating point instructions, and for stack probes. If you don't know what to use for this
/// argument, use `cranelift_module::default_libcall_names()`.
pub fn with_flags(
flags: &[(&str, &str)],
libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
) -> ModuleResult<Self> {
let mut flag_builder = settings::builder();
for (name, value) in flags {
flag_builder.set(name, value)?;
}
// On at least AArch64, "colocated" calls use shorter-range relocations,
// which might not reach all definitions; we can't handle that here, so
// we require long-range relocation types.
flag_builder.set("use_colocated_libcalls", "false").unwrap();
// On x86_64, enable PIC so that symbol addresses are materialized with
// GOT-relative loads rather than RIP-relative `lea`s, which assume the
// symbol is within ±2 GiB of the code. The JIT memory provider makes
// no such promise, so `CompiledBlob::perform_relocations` resolves the
// loads through per-blob GOT entries, relaxing them back to `lea` when
// the symbol does turn out to be in range.
let is_pic = if cfg!(target_arch = "x86_64") {
"true"
} else {
"false"
};
flag_builder.set("is_pic", is_pic).unwrap();
let isa_builder = cranelift_native::builder().unwrap_or_else(|msg| {
panic!("host machine is not supported: {msg}");
});
let isa = isa_builder.finish(settings::Flags::new(flag_builder))?;
Ok(Self::with_isa(isa, libcall_names))
}
/// Create a new `JITBuilder` with an arbitrary target. This is mainly
/// useful for testing.
///
/// To create a `JITBuilder` for native use, use the `new` or `with_flags`
/// constructors instead.
///
/// The `libcall_names` function provides a way to translate `cranelift_codegen`'s `ir::LibCall`
/// enum to symbols. LibCalls are inserted in the IR as part of the legalization for certain
/// floating point instructions, and for stack probes. If you don't know what to use for this
/// argument, use `cranelift_module::default_libcall_names()`.
pub fn with_isa(
isa: OwnedTargetIsa,
libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
) -> Self {
let symbols = HashMap::new();
let lookup_symbols = vec![Box::new(lookup_with_dlsym) as Box<_>];
Self {
isa,
symbols,
lookup_symbols,
libcall_names,
memory: None,
}
}
/// Define a symbol in the internal symbol table.
///
/// The JIT will use the symbol table to resolve names that are declared,
/// but not defined, in the module being compiled. A common example is
/// external functions. With this method, functions and data can be exposed
/// to the code being compiled which are defined by the host.
///
/// If a symbol is defined more than once, the most recent definition will
/// be retained.
///
/// If the JIT fails to find a symbol in its internal table, it will fall
/// back to a platform-specific search (this typically involves searching
/// the current process for public symbols, followed by searching the
/// platform's C runtime).
pub fn symbol<K>(&mut self, name: K, ptr: *const u8) -> &mut Self
where
K: Into<String>,
{
self.symbols.insert(name.into(), SendWrapper(ptr));
self
}
/// Define multiple symbols in the internal symbol table.
///
/// Using this is equivalent to calling `symbol` on each element.
pub fn symbols<It, K>(&mut self, symbols: It) -> &mut Self
where
It: IntoIterator<Item = (K, *const u8)>,
K: Into<String>,
{
for (name, ptr) in symbols {
self.symbols.insert(name.into(), SendWrapper(ptr));
}
self
}
/// Add a symbol lookup fn.
///
/// Symbol lookup fn's are used to lookup symbols when they couldn't be found in the internal
/// symbol table. Symbol lookup fn's are called in reverse of the order in which they were added.
pub fn symbol_lookup_fn(
&mut self,
symbol_lookup_fn: Box<dyn Fn(&str) -> Option<*const u8> + Send>,
) -> &mut Self {
self.lookup_symbols.push(symbol_lookup_fn);
self
}
/// Set the memory provider for the module.
///
/// If unset, defaults to [`SystemMemoryProvider`].
pub fn memory_provider(&mut self, provider: Box<dyn JITMemoryProvider + Send>) -> &mut Self {
self.memory = Some(provider);
self
}
}
/// A wrapper that impls Send for the contents.
///
/// SAFETY: This must not be used for any types where it would be UB for them to be Send
#[derive(Copy, Clone)]
struct SendWrapper<T>(T);
unsafe impl<T> Send for SendWrapper<T> {}
/// A `JITModule` implements `Module` and emits code and data into memory where it can be
/// directly called and accessed.
///
/// See the `JITBuilder` for a convenient way to construct `JITModule` instances.
pub struct JITModule {
isa: OwnedTargetIsa,
symbols: RefCell<HashMap<String, SendWrapper<*const u8>>>,
lookup_symbols: Vec<Box<dyn Fn(&str) -> Option<*const u8> + Send>>,
libcall_names: Box<dyn Fn(ir::LibCall) -> String + Send + Sync>,
memory: Box<dyn JITMemoryProvider + Send>,
declarations: ModuleDeclarations,
compiled_functions: SecondaryMap<FuncId, Option<CompiledBlob>>,
compiled_data_objects: SecondaryMap<DataId, Option<CompiledBlob>>,
/// Map from a function's start address to its (end address, FuncId), used
/// to resolve a PC back to its function for exception unwinding.
code_ranges: BTreeMap<usize, (usize, FuncId)>,
functions_to_finalize: Vec<FuncId>,
data_objects_to_finalize: Vec<DataId>,
}
impl JITModule {
/// Free memory allocated for code and data segments of compiled functions.
///
/// # Safety
///
/// Because this function invalidates any pointers retrieved from the
/// corresponding module, it should only be used when none of the functions
/// from that module are currently executing and none of the `fn` pointers
/// are called afterwards.
pub unsafe fn free_memory(mut self) {
self.memory.free_memory();
}
fn lookup_symbol(&self, name: &str) -> Option<*const u8> {
match self.symbols.borrow_mut().entry(name.to_owned()) {
std::collections::hash_map::Entry::Occupied(occ) => Some(occ.get().0),
std::collections::hash_map::Entry::Vacant(vac) => {
let ptr = self
.lookup_symbols
.iter()
.rev() // Try last lookup function first
.find_map(|lookup| lookup(name));
if let Some(ptr) = ptr {
vac.insert(SendWrapper(ptr));
}
ptr
}
}
}
/// Get the address that the given [`ModuleRelocTarget`] would resolve to.
///
/// For locally defined functions and data objects this is equivalent to
/// [`Self::get_finalized_function`] cq [`Self::get_finalized_data`], while
/// for external functions and data objects this will iterate through the
/// registered symbol lookup functions until one matches.
pub fn get_address(&self, name: &ModuleRelocTarget) -> *const u8 {
match name {
ModuleRelocTarget::User { .. } => {
let (name, linkage) = if ModuleDeclarations::is_function(name) {
let func_id = FuncId::from_name(name);
match &self.compiled_functions[func_id] {
Some(compiled) => return compiled.ptr(),
None => {
let decl = self.declarations.get_function_decl(func_id);
(&decl.name, decl.linkage)
}
}
} else {
let data_id = DataId::from_name(name);
match &self.compiled_data_objects[data_id] {
Some(compiled) => return compiled.ptr(),
None => {
let decl = self.declarations.get_data_decl(data_id);
(&decl.name, decl.linkage)
}
}
};
let name = name
.as_ref()
.expect("anonymous symbol must be defined locally");
if let Some(ptr) = self.lookup_symbol(name) {
ptr
} else if linkage == Linkage::Preemptible {
0 as *const u8
} else {
panic!("can't resolve symbol {name}");
}
}
ModuleRelocTarget::LibCall(libcall) => {
let sym = (self.libcall_names)(*libcall);
self.lookup_symbol(&sym)
.unwrap_or_else(|| panic!("can't resolve libcall {sym}"))
}
ModuleRelocTarget::FunctionOffset(func_id, offset) => {
match &self.compiled_functions[*func_id] {
Some(compiled) => return compiled.ptr().wrapping_add(*offset as usize),
None => todo!(),
}
}
name => panic!("invalid name {name:?}"),
}
}
/// Returns the address of a finalized function.
///
/// The pointer remains valid until either [`JITModule::free_memory`] is called or in the future
/// some way of deallocating this individual function is used.
pub fn get_finalized_function(&self, func_id: FuncId) -> *const u8 {
let info = &self.compiled_functions[func_id];
assert!(
!self.functions_to_finalize.iter().any(|x| *x == func_id),
"function not yet finalized"
);
info.as_ref()
.expect("function must be compiled before it can be finalized")
.ptr()
}
/// Returns the address and size of a finalized data object.
///
/// The pointer remains valid until either [`JITModule::free_memory`] is called or in the future
/// some way of deallocating this individual data object is used.
pub fn get_finalized_data(&self, data_id: DataId) -> (*const u8, usize) {
let info = &self.compiled_data_objects[data_id];
assert!(
!self.data_objects_to_finalize.iter().any(|x| *x == data_id),
"data object not yet finalized"
);
let compiled = info
.as_ref()
.expect("data object must be compiled before it can be finalized");
(compiled.ptr(), compiled.size())
}
fn record_function_for_perf(&self, ptr: *const u8, size: usize, name: &str) {
// The Linux perf tool supports JIT code via a /tmp/perf-$PID.map file,
// which contains memory regions and their associated names. If we
// are profiling with perf and saving binaries to PERF_BUILDID_DIR
// for post-profile analysis, write information about each function
// we define.
if cfg!(unix) && ::std::env::var_os("PERF_BUILDID_DIR").is_some() {
let mut map_file = ::std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(format!("/tmp/perf-{}.map", ::std::process::id()))
.unwrap();
let _ = writeln!(map_file, "{:x} {:x} {}", ptr as usize, size, name);
}
}
/// Finalize all functions and data objects that are defined but not yet finalized.
/// All symbols referenced in their bodies that are declared as needing a definition
/// must be defined by this point.
///
/// Use `get_finalized_function` and `get_finalized_data` to obtain the final
/// artifacts.
///
/// Returns ModuleError in case of allocation or syscall failure
pub fn finalize_definitions(&mut self) -> ModuleResult<()> {
for func in std::mem::take(&mut self.functions_to_finalize) {
let decl = self.declarations.get_function_decl(func);
assert!(decl.linkage.is_definable());
let func = self.compiled_functions[func]
.as_ref()
.expect("function must be compiled before it can be finalized");
func.perform_relocations(|name| self.get_address(name));
}
for data in std::mem::take(&mut self.data_objects_to_finalize) {
let decl = self.declarations.get_data_decl(data);
assert!(decl.linkage.is_definable());
let data = self.compiled_data_objects[data]
.as_ref()
.expect("data object must be compiled before it can be finalized");
data.perform_relocations(|name| self.get_address(name));
}
// Now that we're done patching, prepare the memory for execution!
let branch_protection = if cfg!(target_arch = "aarch64") && use_bti(&self.isa.isa_flags()) {
BranchProtection::BTI
} else {
BranchProtection::None
};
self.memory.finalize(branch_protection)?;
Ok(())
}
/// Create a new `JITModule`.
pub fn new(builder: JITBuilder) -> Self {
assert!(
!builder.isa.flags().is_pic()
|| builder.isa.triple().architecture == Architecture::X86_64,
"cranelift-jit only supports is_pic=true on x86_64"
);
let memory = builder
.memory
.unwrap_or_else(|| Box::new(SystemMemoryProvider::new()));
Self {
isa: builder.isa,
symbols: RefCell::new(builder.symbols),
lookup_symbols: builder.lookup_symbols,
libcall_names: builder.libcall_names,
memory,
declarations: ModuleDeclarations::default(),
compiled_functions: SecondaryMap::new(),
compiled_data_objects: SecondaryMap::new(),
code_ranges: BTreeMap::new(),
functions_to_finalize: Vec::new(),
data_objects_to_finalize: Vec::new(),
}
}
/// Look up the Wasmtime unwind ExceptionTable and corresponding
/// base PC, if any, for a given PC that may be within one of the
/// CompiledBlobs in this module.
#[cfg(feature = "wasmtime-unwinder")]
pub fn lookup_wasmtime_exception_data<'a>(
&'a self,
pc: usize,
) -> Option<(usize, wasmtime_unwinder::ExceptionTable<'a>)> {
let (&start, &(end, func)) = self.code_ranges.range(..=pc).next_back()?;
if pc >= end {
return None;
}
// Get the ExceptionTable. The "parse" here simply reads two
// u32s for lengths and constructs borrowed slices, so it's
// cheap.
let data = self.compiled_functions[func]
.as_ref()
.unwrap()
.wasmtime_exception_data()?;
let exception_table = wasmtime_unwinder::ExceptionTable::parse(data).ok()?;
Some((start, exception_table))
}
}
impl Module for JITModule {
fn isa(&self) -> &dyn TargetIsa {
&*self.isa
}
fn declarations(&self) -> &ModuleDeclarations {
&self.declarations
}
fn declare_function(
&mut self,
name: &str,
linkage: Linkage,
signature: &ir::Signature,
) -> ModuleResult<FuncId> {
let (id, _linkage) = self
.declarations
.declare_function(name, linkage, signature)?;
Ok(id)
}
fn declare_anonymous_function(&mut self, signature: &ir::Signature) -> ModuleResult<FuncId> {
let id = self.declarations.declare_anonymous_function(signature)?;
Ok(id)
}
fn declare_data(
&mut self,
name: &str,
linkage: Linkage,
writable: bool,
tls: bool,
) -> ModuleResult<DataId> {
assert!(!tls, "JIT doesn't yet support TLS");
let (id, _linkage) = self
.declarations
.declare_data(name, linkage, writable, tls)?;
Ok(id)
}
fn declare_anonymous_data(&mut self, writable: bool, tls: bool) -> ModuleResult<DataId> {
assert!(!tls, "JIT doesn't yet support TLS");
let id = self.declarations.declare_anonymous_data(writable, tls)?;
Ok(id)
}
fn define_function_with_control_plane(
&mut self,
id: FuncId,
ctx: &mut cranelift_codegen::Context,
ctrl_plane: &mut ControlPlane,
) -> ModuleResult<()> {
info!("defining function {}: {}", id, ctx.func.display());
let decl = self.declarations.get_function_decl(id);
if !decl.linkage.is_definable() {
return Err(ModuleError::InvalidImportDefinition(
decl.linkage_name(id).into_owned(),
));
}
if !self.compiled_functions[id].is_none() {
return Err(ModuleError::DuplicateDefinition(
decl.linkage_name(id).into_owned(),
));
}
// work around borrow-checker to allow reuse of ctx below
let res = ctx.compile(self.isa(), ctrl_plane)?;
let alignment = res.buffer.min_alignment as u64;
let compiled_code = ctx.compiled_code().unwrap();
let align = alignment
.max(self.isa.function_alignment().minimum as u64)
.max(self.isa.symbol_alignment());
let relocs = compiled_code
.buffer
.relocs()
.iter()
.map(|reloc| ModuleReloc::from_mach_reloc(reloc, &ctx.func, id))
.collect();
#[cfg(feature = "wasmtime-unwinder")]
let wasmtime_exception_data = {
let mut exception_builder = wasmtime_unwinder::ExceptionTableBuilder::default();
exception_builder
.add_func(0, compiled_code.buffer.call_sites())
.map_err(|_| {
ModuleError::Compilation(cranelift_codegen::CodegenError::Unsupported(
"Invalid exception data".into(),
))
})?;
Some(exception_builder.to_vec())
};
let blob = self.compiled_functions[id].insert(CompiledBlob::new(
&mut *self.memory,
compiled_code.code_buffer(),
align,
relocs,
#[cfg(feature = "wasmtime-unwinder")]
wasmtime_exception_data,
JITMemoryKind::Executable,
)?);
let (ptr, size) = (blob.ptr(), blob.size());
self.record_function_for_perf(ptr, size, &decl.linkage_name(id));
let range_start = ptr.addr();
let range_end = range_start + size;
self.code_ranges.insert(range_start, (range_end, id));
self.functions_to_finalize.push(id);
Ok(())
}
fn define_function_bytes(
&mut self,
id: FuncId,
alignment: u64,
bytes: &[u8],
relocs: &[ModuleReloc],
) -> ModuleResult<()> {
info!("defining function {id} with bytes");
let decl = self.declarations.get_function_decl(id);
if !decl.linkage.is_definable() {
return Err(ModuleError::InvalidImportDefinition(
decl.linkage_name(id).into_owned(),
));
}
if !self.compiled_functions[id].is_none() {
return Err(ModuleError::DuplicateDefinition(
decl.linkage_name(id).into_owned(),
));
}
let align = alignment
.max(self.isa.function_alignment().minimum as u64)
.max(self.isa.symbol_alignment());
let blob = self.compiled_functions[id].insert(CompiledBlob::new(
&mut *self.memory,
bytes,
align,
relocs.to_owned(),
#[cfg(feature = "wasmtime-unwinder")]
None,
JITMemoryKind::Executable,
)?);
let (ptr, size) = (blob.ptr(), blob.size());
self.record_function_for_perf(ptr, size, &decl.linkage_name(id));
self.functions_to_finalize.push(id);
Ok(())
}
fn define_data(&mut self, id: DataId, data: &DataDescription) -> ModuleResult<()> {
let decl = self.declarations.get_data_decl(id);
if !decl.linkage.is_definable() {
return Err(ModuleError::InvalidImportDefinition(
decl.linkage_name(id).into_owned(),
));
}
if !self.compiled_data_objects[id].is_none() {
return Err(ModuleError::DuplicateDefinition(
decl.linkage_name(id).into_owned(),
));
}
assert!(!decl.tls, "JIT doesn't yet support TLS");
let &DataDescription {
ref init,
function_decls: _,
data_decls: _,
function_relocs: _,
data_relocs: _,
custom_section: _,
align,
used: _,
} = data;
let (align, kind) = if decl.writable {
(
align.unwrap_or(WRITABLE_DATA_ALIGNMENT),
JITMemoryKind::Writable,
)
} else {
(
align.unwrap_or(READONLY_DATA_ALIGNMENT),
JITMemoryKind::ReadOnly,
)
};
let pointer_reloc = match self.isa.triple().pointer_width().unwrap() {
PointerWidth::U16 => panic!(),
PointerWidth::U32 => Reloc::Abs4,
PointerWidth::U64 => Reloc::Abs8,
};
let relocs = data.all_relocs(pointer_reloc).collect::<Vec<_>>();
self.compiled_data_objects[id] = Some(match *init {
Init::Uninitialized => {
panic!("data is not initialized yet");
}
Init::Zeros { size } => CompiledBlob::new_zeroed(
&mut *self.memory,
size.max(1),
align,
relocs,
#[cfg(feature = "wasmtime-unwinder")]
None,
kind,
)?,
Init::Bytes { ref contents } => CompiledBlob::new(
&mut *self.memory,
if contents.is_empty() {
// Make sure to allocate at least 1 byte. Allocating 0 bytes is UB. Previously
// a dummy value was used, however as it turns out this will cause pc-relative
// relocations to fail on architectures where pc-relative offsets are range
// restricted as the dummy value is not close enough to the code that has the
// pc-relative relocation.
&[0]
} else {
&contents[..]
},
align,
relocs,
#[cfg(feature = "wasmtime-unwinder")]
None,
kind,
)?,
});
self.data_objects_to_finalize.push(id);
Ok(())
}
}
#[cfg(not(windows))]
fn lookup_with_dlsym(name: &str) -> Option<*const u8> {
let c_str = CString::new(name).unwrap();
let c_str_ptr = c_str.as_ptr();
let sym = unsafe { libc::dlsym(libc::RTLD_DEFAULT, c_str_ptr) };
if sym.is_null() {
None
} else {
Some(sym as *const u8)
}
}
#[cfg(windows)]
fn lookup_with_dlsym(name: &str) -> Option<*const u8> {
use std::os::windows::io::RawHandle;
use std::ptr;
use windows_sys::Win32::Foundation::HMODULE;
use windows_sys::Win32::System::LibraryLoader;
const UCRTBASE: &[u8] = b"ucrtbase.dll\0";
let c_str = CString::new(name).unwrap();
let c_str_ptr = c_str.as_ptr();
unsafe {
let handles = [
// try to find the searched symbol in the currently running executable
ptr::null_mut(),
// try to find the searched symbol in local c runtime
LibraryLoader::GetModuleHandleA(UCRTBASE.as_ptr()) as RawHandle,
];
for handle in &handles {
let addr = LibraryLoader::GetProcAddress(*handle as HMODULE, c_str_ptr.cast());
match addr {
None => continue,
Some(addr) => return Some(addr as *const u8),
}
}
None
}
}
fn use_bti(isa_flags: &Vec<settings::Value>) -> bool {
isa_flags
.iter()
.find(|&f| f.name == "use_bti")
.map_or(false, |f| f.as_bool().unwrap_or(false))
}
const _ASSERT_JIT_MODULE_IS_SEND: () = {
const fn assert_is_send<T: Send>() {}
assert_is_send::<JITModule>();
};