A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Use bulk libunwind section registration when available (#14304)
Daniel Macoveicb091c33cece
diff --git a/crates/wasmtime/src/runtime/vm/helpers.c b/crates/wasmtime/src/runtime/vm/helpers.cindex b083721fa1..3eff066dbb 100644--- a/crates/wasmtime/src/runtime/vm/helpers.c+++ b/crates/wasmtime/src/runtime/vm/helpers.c@@ -39,12 +39,29 @@ void VERSIONED_SYMBOL(wasmtime_debug_builtins_init)() {#endif // FEATURE_DEBUG_BUILTINS// For more information about this see `unix/unwind.rs` and the-// `using_libunwind` function. The basic idea is that weak symbols aren't stable+// `Libunwind` struct. The basic idea is that weak symbols aren't stable// in Rust so we use a bit of C to work around that.#ifndef CFG_TARGET_OS_windows-__attribute__((weak)) extern void __unw_add_dynamic_fde();+struct Libunwind {+ void (*add_dynamic_fde)(uintptr_t);+ void (*add_dynamic_eh_frame_section)(uintptr_t);+ void (*remove_dynamic_eh_frame_section)(uintptr_t);+};-bool VERSIONED_SYMBOL(wasmtime_using_libunwind)() {- return __unw_add_dynamic_fde != NULL;+__attribute__((weak)) extern void __unw_add_dynamic_fde(uintptr_t);+#ifndef CFG_TARGET_OS_macos+__attribute__((weak)) extern void __unw_add_dynamic_eh_frame_section(uintptr_t);+__attribute__((weak)) extern void+ __unw_remove_dynamic_eh_frame_section(uintptr_t);+#endif++struct Libunwind VERSIONED_SYMBOL(wasmtime_libunwind)(void) {+ return (struct Libunwind){+ .add_dynamic_fde = __unw_add_dynamic_fde,+#ifndef CFG_TARGET_OS_macos+ .add_dynamic_eh_frame_section = __unw_add_dynamic_eh_frame_section,+ .remove_dynamic_eh_frame_section = __unw_remove_dynamic_eh_frame_section,+#endif+ };}#endifdiff --git a/crates/wasmtime/src/runtime/vm/sys/unix/unwind.rs b/crates/wasmtime/src/runtime/vm/sys/unix/unwind.rsindex 6c500f0064..ce824e6be1 100644--- a/crates/wasmtime/src/runtime/vm/sys/unix/unwind.rs+++ b/crates/wasmtime/src/runtime/vm/sys/unix/unwind.rs@@ -2,11 +2,31 @@use crate::prelude::*;use crate::runtime::vm::SendSyncPtr;+use crate::sync::OnceLock;use core::ptr::NonNull;+type UnwindFn = unsafe extern "C" fn(usize);+/// Represents a registration of function unwind information for System V ABI.-pub struct UnwindRegistration {- registrations: TryVec<SendSyncPtr<u8>>,+pub enum UnwindRegistration {+ /// libgcc registers the whole table with one call to `__register_frame`.+ SingleFrame(SendSyncPtr<u8>),+ /// Older libunwind versions register each FDE separately.+ MultipleFrames(TryVec<SendSyncPtr<u8>>),+ /// Newer libunwind versions can register an entire `.eh_frame` section.+ LibunwindSection {+ section: SendSyncPtr<u8>,+ deregister: UnwindFn,+ },+}++// Keep this layout in sync with `struct Libunwind` in helpers.c.+#[repr(C)]+#[derive(Default)]+struct Libunwind {+ add_dynamic_fde: Option<UnwindFn>,+ add_dynamic_eh_frame_section: Option<UnwindFn>,+ remove_dynamic_eh_frame_section: Option<UnwindFn>,}cfg_select! {@@ -20,42 +40,114 @@ cfg_select! {target_arch = "arm" => {unsafe extern "C" fn __register_frame(_: *const u8) {}unsafe extern "C" fn __deregister_frame(_: *const u8) {}- unsafe extern "C" fn wasmtime_using_libunwind() -> bool {- false+ unsafe fn wasmtime_libunwind() -> Libunwind {+ Libunwind::default()}}_ => {unsafe extern "C" {- // libunwind importfn __register_frame(fde: *const u8);fn __deregister_frame(fde: *const u8);#[wasmtime_versioned_export_macros::versioned_link]- fn wasmtime_using_libunwind() -> bool;+ fn wasmtime_libunwind() -> Libunwind;}}}-/// There are two primary unwinders on Unix platforms: libunwind and libgcc.-///-/// Unfortunately their interface to `__register_frame` is different. The-/// libunwind library takes a pointer to an individual FDE while libgcc takes a-/// null-terminated list of FDEs. This means we need to know what unwinder-/// is being used at runtime.-///-/// This detection is done currently by looking for a libunwind-specific symbol.-/// This specific symbol was somewhat recommended by LLVM's-/// "RTDyldMemoryManager.cpp" file which says:-///-/// > We use the presence of __unw_add_dynamic_fde to detect libunwind.-///-/// I'll note that there's also a different libunwind project at-/// https://www.nongnu.org/libunwind/ but that doesn't appear to have-/// `__register_frame` so I don't think that interacts with this.-fn using_libunwind() -> bool {- // On macOS the libgcc interface is never used so libunwind is always used.- // Otherwise delegate to `helpers.c` since weak symbols can't be used from- // Rust at this time.- cfg!(target_os = "macos") || unsafe { wasmtime_using_libunwind() }+impl Libunwind {+ fn get() -> &'static Self {+ static LIBUNWIND: OnceLock<Libunwind> = OnceLock::new();+ LIBUNWIND.get_or_init(|| {+ // SAFETY: the C helper returns nullable function pointers with the+ // signatures and layout declared above.+ let api = unsafe { wasmtime_libunwind() };+ #[cfg(target_os = "macos")]+ let api = {+ let mut api = api;+ if let Some((add, remove)) = load_unwind_section_api() {+ api.add_dynamic_eh_frame_section = Some(add);+ api.remove_dynamic_eh_frame_section = Some(remove);+ }+ api+ };+ api+ })+ }++ /// There are two primary unwinders on Unix platforms: libunwind and libgcc.+ ///+ /// Unfortunately their interface to `__register_frame` is different. The+ /// libunwind library takes a pointer to an individual FDE while libgcc takes a+ /// null-terminated list of FDEs. This means we need to know what unwinder+ /// is being used at runtime.+ ///+ /// This detection is done currently by looking for a libunwind-specific symbol.+ /// This specific symbol was somewhat recommended by LLVM's+ /// "RTDyldMemoryManager.cpp" file which says:+ ///+ /// > We use the presence of __unw_add_dynamic_fde to detect libunwind.+ ///+ /// I'll note that there's also a different libunwind project at+ /// https://www.nongnu.org/libunwind/ but that doesn't appear to have+ /// `__register_frame` so I don't think that interacts with this.+ fn is_libunwind(&self) -> bool {+ // macOS always uses the libunwind convention.+ cfg!(target_os = "macos") || self.add_dynamic_fde.is_some()+ }+}++#[cfg(target_os = "macos")]+fn load_unwind_section_api() -> Option<(UnwindFn, UnwindFn)> {+ // Weak imports tolerate symbols missing at runtime, but the linker still+ // needs their exports in the build SDK. The macOS 11.3 SDK has the older+ // __unw_add_dynamic_fde export but lacks the section APIs, so resolve those+ // at runtime to keep building with older SDKs.+ //+ // SAFETY: dladdr initializes info on success. The returned path is a C+ // string owned by the loaded image containing our linked registration API.+ unsafe {+ let mut info = core::mem::MaybeUninit::<libc::Dl_info>::uninit();+ if libc::dladdr((__register_frame as *const ()).cast(), info.as_mut_ptr()) == 0 {+ return None;+ }+ let info = info.assume_init();+ if info.dli_fname.is_null() {+ return None;+ }++ // Use the same provider as the linked individual-FDE API. RTLD_FIRST+ // prevents borrowing a missing API from a dependency's separate registry.+ let lib = libc::dlopen(+ info.dli_fname,+ libc::RTLD_LAZY | libc::RTLD_LOCAL | libc::RTLD_FIRST,+ );+ if lib.is_null() {+ return None;+ }++ // Search overrides or interposition can change which library is opened.+ if libc::dlsym(lib, c"__register_frame".as_ptr())+ != __register_frame as *const () as *mut libc::c_void+ || libc::dlsym(lib, c"__deregister_frame".as_ptr())+ != __deregister_frame as *const () as *mut libc::c_void+ {+ libc::dlclose(lib);+ return None;+ }+ let add = libc::dlsym(lib, c"__unw_add_dynamic_eh_frame_section".as_ptr());+ let remove = libc::dlsym(lib, c"__unw_remove_dynamic_eh_frame_section".as_ptr());+ if add.is_null() || remove.is_null() {+ libc::dlclose(lib);+ return None;+ }++ // Both symbols have the UnwindFn ABI. Keep the library handle open for+ // the lifetime of the function pointers cached in Libunwind::get.+ Some((+ core::mem::transmute::<*mut libc::c_void, UnwindFn>(add),+ core::mem::transmute::<*mut libc::c_void, UnwindFn>(remove),+ ))+ }}impl UnwindRegistration {@@ -79,58 +171,68 @@ impl UnwindRegistration {"The unwind info must always be aligned to a page");- let mut registrations = TryVec::new();+ let api = Libunwind::get();+ let info = SendSyncPtr::new(NonNull::new(unwind_info.cast_mut()).unwrap());unsafe {- if using_libunwind() {- // For libunwind, `__register_frame` takes a pointer to a single- // FDE. Note that we subtract 4 from the length of unwind info since- // wasmtime-encode .eh_frame sections always have a trailing 32-bit- // zero for the platforms above.- let start = unwind_info;- let end = start.add(unwind_len - 4);- let mut current = start;-- // Walk all of the entries in the frame table and register them- while current < end {- let len = current.cast::<u32>().read_unaligned() as usize;-- // Skip over the CIE- if current != start {- __register_frame(current);- let cur = NonNull::new(current.cast_mut()).unwrap();- registrations.push(SendSyncPtr::new(cur))?;- }+ if !api.is_libunwind() {+ // libgcc walks the table until an entry of length zero.+ __register_frame(unwind_info);+ return Ok(Self::SingleFrame(info));+ }++ // Require both APIs so removal uses the same registration grouping.+ if let (Some(add), Some(remove)) = (+ api.add_dynamic_eh_frame_section,+ api.remove_dynamic_eh_frame_section,+ ) {+ add(unwind_info as usize);+ return Ok(Self::LibunwindSection {+ section: info,+ deregister: remove,+ });+ }- // Move to the next table entry (+4 because the length itself is- // not inclusive)- current = current.add(len + 4);+ // libunwind's __register_frame takes a single FDE. Our .eh_frame+ // section ends with a four-byte zero terminator.+ let mut registration = Self::MultipleFrames(TryVec::new());+ let Self::MultipleFrames(frames) = &mut registration else {+ unreachable!();+ };+ let end = unwind_info.add(unwind_len - 4);+ let mut current = unwind_info;+ while current < end {+ let len = current.cast::<u32>().read_unaligned() as usize;+ // Skip the CIE at the beginning of the section.+ if current != unwind_info {+ // Record the pointer before registering it, so an allocation+ // failure unregisters all previous entries via Drop.+ frames.push(SendSyncPtr::new(NonNull::new(current.cast_mut()).unwrap()))?;+ __register_frame(current);}- } else {- // On gnu (libgcc), `__register_frame` will walk the FDEs until an- // entry of length 0- __register_frame(unwind_info);- let info = NonNull::new(unwind_info.cast_mut()).unwrap();- registrations.push(SendSyncPtr::new(info))?;+ current = current.add(len + 4);}+ Ok(registration)}-- Ok(UnwindRegistration { registrations })}}impl Drop for UnwindRegistration {fn drop(&mut self) {unsafe {- // libgcc stores the frame entries as a linked list in decreasing- // sort order based on the PC value of the registered entry.- //- // As we store the registrations in increasing order, it would be- // O(N^2) to deregister in that order.- //- // To ensure that we just pop off the first element in the list upon- // every deregistration, walk our list of registrations backwards.- for fde in self.registrations.iter().rev() {- __deregister_frame(fde.as_ptr());+ match self {+ Self::SingleFrame(frame) => __deregister_frame(frame.as_ptr()),+ Self::MultipleFrames(frames) => {+ // Preserve reverse registration order when removing FDEs.+ for frame in frames.iter().rev() {+ __deregister_frame(frame.as_ptr());+ }+ }+ Self::LibunwindSection {+ section,+ deregister,+ } => {+ deregister(section.as_ptr() as usize);+ }}}}diff --git a/tests/all/module.rs b/tests/all/module.rsindex 6a4daff1a7..2920aadb13 100644--- a/tests/all/module.rs+++ b/tests/all/module.rs@@ -650,3 +650,75 @@ fn deserialize_raw_fails_for_native() {);}}++#[cfg(all(+ any(target_os = "macos", target_os = "linux"),+ target_arch = "x86_64",+ not(miri)+))]+mod native_unwind {+ use wasmtime::{Config, Engine, Module, Result};++ // On macOS aarch64, the system unwinder currently stops before reaching the+ // guest, including without bulk registration. Exercise the host/guest transition+ // on x86-64 macOS and Linux until that separate process-ABI issue is fixed.+ #[test]+ fn native_unwind_walks_nested_wasm_frames() -> Result<()> {+ use wasmtime::{Caller, Func, Inlining, Instance, Store, Strategy};++ let mut config = Config::new();+ config+ .strategy(Strategy::Cranelift)+ .native_unwind_info(true)+ .compiler_inlining(Inlining::No);+ let engine = Engine::new(&config)?;+ // Post-call arithmetic keeps each guest caller live instead of tail calling.+ let module = Module::new(+ &engine,+ r#"(module+ (import "" "capture" (func $capture (result i32)))+ (func $inner (result i32) call $capture i32.const 1 i32.add)+ (func $middle (result i32) call $inner i32.const 2 i32.add)+ (func $run (export "run") (result i32) call $middle i32.const 4 i32.add))"#,+ )?;+ let start = module.text().as_ptr() as usize;+ let functions = module+ .functions()+ .map(|f| (f.name, start + f.offset, f.len))+ .collect::<Vec<_>>();+ let mut store = Store::new(&engine, Vec::<usize>::with_capacity(128));+ let capture = Func::wrap(&mut store, |mut caller: Caller<'_, Vec<usize>>| {+ backtrace::trace(|frame| {+ let frames = caller.data_mut();+ // Stop before growing the buffer to avoid allocating in the callback.+ if frames.len() == frames.capacity() {+ return false;+ }+ frames.push(frame.ip() as usize);+ true+ });+ 10_i32+ });+ let instance = Instance::new(&mut store, &module, &[capture.into()])?;+ assert_eq!(+ instance+ .get_typed_func::<(), i32>(&mut store, "run")?+ .call(&mut store, ())?,+ 17+ );+ let names = store+ .data()+ .iter()+ .filter_map(|pc| {+ // Return addresses point after the call instruction. Match actual+ // guest functions, not any trampoline inside the module's text.+ functions+ .iter()+ .find(|(_, start, len)| *pc > *start && *pc <= start + len)+ .map(|(name, _, _)| name.as_deref().unwrap_or_default())+ })+ .collect::<Vec<_>>();+ assert_eq!(names, ["inner", "middle", "run"]);+ Ok(())+ }+}diff --git a/tests/native_unwind_registration.rs b/tests/native_unwind_registration.rsnew file mode 100644index 0000000000..571fbfb494--- /dev/null+++ b/tests/native_unwind_registration.rs@@ -0,0 +1,96 @@+//! This test needs its own executable because it queries unwind info after+//! dropping modules. Another test in the same process could reuse a dropped+//! module's code addresses and make the removal check find the new module's+//! unwind info instead.++#![cfg(all(+ any(target_os = "macos", all(target_os = "linux", target_arch = "x86_64")),+ not(miri)+))]++use wasmtime::{Config, Engine, Module, Result};++#[repr(C)]+#[derive(Default)]+struct DwarfEhBases {+ tbase: usize,+ dbase: usize,+ func: usize,+}++unsafe extern "C" {+ fn _Unwind_Find_FDE(pc: *const u8, bases: *mut DwarfEhBases) -> *const u8;+}++fn find_fde(pc: usize) -> (usize, usize) {+ let mut bases = DwarfEhBases::default();+ // SAFETY: the API only looks up the address, and bases is a valid out pointer.+ let fde = unsafe { _Unwind_Find_FDE(pc as *const u8, &mut bases) as usize };+ (fde, bases.func)+}++fn check_module(module: &Module) -> Vec<usize> {+ let start = module.text().as_ptr() as usize;+ let image = module.image_range();+ let pcs = module+ .functions()+ .map(|f| start + f.offset + 1)+ .collect::<Vec<_>>();+ assert_eq!(pcs.len(), 3, "expected all three defined functions");+ for pc in &pcs {+ let (fde, func) = find_fde(*pc);+ assert!(fde >= image.start as usize && fde < image.end as usize);+ assert_eq!(func, pc - 1);+ }++ pcs+}++#[test]+fn native_unwind_sections_survive_non_lifo_removal() -> Result<()> {+ let mut config = Config::new();+ config.native_unwind_info(true);+ let engine = Engine::new(&config)?;+ let wasm = wat::parse_str(+ r#"(module+ (import "" "capture" (func $capture))+ (func $inner call $capture)+ (func $middle call $inner)+ (func (export "run") call $middle))"#,+ )?;+ let make_module = || Module::from_binary(&engine, &wasm);+ let first = make_module()?;+ let middle = make_module()?;+ let last = make_module()?;+ check_module(&first);+ let middle_pcs = check_module(&middle);+ check_module(&last);++ drop(middle);+ for pc in middle_pcs {+ assert_eq!(+ find_fde(pc).0,+ 0,+ "dropped module still has registered unwind info"+ );+ }+ let first_pcs = check_module(&first);+ check_module(&last);+ drop(first);+ for pc in first_pcs {+ assert_eq!(find_fde(pc).0, 0);+ }+ let last_pcs = check_module(&last);+ drop(last);+ for pc in last_pcs {+ assert_eq!(find_fde(pc).0, 0);+ }+ // Exercise registration again after the registry has removed every section.+ let fresh = make_module()?;+ let fresh_pcs = check_module(&fresh);+ drop(fresh);+ for pc in fresh_pcs {+ assert_eq!(find_fde(pc).0, 0);+ }+ Ok(())+}