Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
crashes: Capture glibc's abort diagnostic in crash reports (#60624)
Richard Feldman6b9f448ffc1d
diff --git a/Cargo.lock b/Cargo.lockindex a44cdb5271c..e6ef165bba8 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4347,6 +4347,7 @@ version = "0.1.0"dependencies = ["async-process","crash-handler",+ "libc","log","mach2 0.5.0","minidumper",diff --git a/crates/crashes/Cargo.toml b/crates/crashes/Cargo.tomlindex f8b898112c1..1b118097cb4 100644--- a/crates/crashes/Cargo.toml+++ b/crates/crashes/Cargo.toml@@ -16,6 +16,9 @@ serde_json.workspace = truesystem_specs.workspace = truezstd.workspace = true+[target.'cfg(target_os = "linux")'.dependencies]+libc.workspace = true+[target.'cfg(target_os = "macos")'.dependencies]mach2.workspace = truediff --git a/crates/crashes/src/crashes.rs b/crates/crashes/src/crashes.rsindex 28217496cad..5d3fc954d30 100644--- a/crates/crashes/src/crashes.rs+++ b/crates/crashes/src/crashes.rs@@ -138,6 +138,17 @@ whereinfo!("crash signal handlers installed");send_crash_server_message(&client, CrashServerMessage::Init(crash_init));+ #[cfg(all(target_os = "linux", target_env = "gnu"))]+ if let Some(address) = abort_message_address() {+ send_crash_server_message(+ &client,+ CrashServerMessage::AbortMessageLocation(AbortMessageLocation {+ pid: process::id(),+ address,+ }),+ );+ }+#[cfg(target_os = "linux")]handler.set_ptracer(Some(_crash_handler.id()));@@ -170,6 +181,7 @@ pub struct CrashServer {panic_info: Mutex<Option<CrashPanic>>,active_gpu: Mutex<Option<system_specs::GpuSpecs>>,user_info: Mutex<Option<UserInfo>>,+ abort_message_location: Mutex<Option<AbortMessageLocation>>,has_connection: Arc<AtomicBool>,logs_dir: PathBuf,}@@ -179,11 +191,27 @@ pub struct CrashInfo {pub init: InitCrashHandler,pub panic: Option<CrashPanic>,pub minidump_error: Option<String>,+ /// The diagnostic the C runtime recorded before aborting the process, e.g.+ /// glibc's "free(): invalid pointer". Only present when the crash was a+ /// runtime-initiated abort rather than a signal like SIGSEGV or a panic.+ #[serde(default)]+ pub abort_message: Option<String>,pub gpus: Vec<system_specs::GpuInfo>,pub active_gpu: Option<system_specs::GpuSpecs>,pub user_info: Option<UserInfo>,}+/// Where to find the C runtime's abort diagnostic in the crashed process's+/// memory. Sent by the client at startup so that after a crash the server can+/// recover the message with `process_vm_readv`; the crashed process itself+/// can't safely do this work, since its heap may be corrupt and its allocator+/// locks may be held by the crashed thread.+#[derive(Debug, Deserialize, Serialize, Clone, Copy)]+pub struct AbortMessageLocation {+ pub pid: u32,+ pub address: u64,+}+#[derive(Debug, Deserialize, Serialize, Clone)]pub struct InitCrashHandler {pub session_id: String,@@ -233,6 +261,110 @@ enum CrashServerMessage {Panic(CrashPanic),GPUInfo(GpuSpecs),UserInfo(UserInfo),+ AbortMessageLocation(AbortMessageLocation),+}++/// glibc records the diagnostic it prints just before aborting (malloc integrity+/// failures like "free(): invalid pointer", assertion failures, stack-smashing+/// reports) in the private global `__abort_msg`, specifically so it can be+/// recovered post-mortem. Resolve its address here, in a safe context at startup.+/// The symbol is only exported at the GLIBC_PRIVATE version, which plain `dlsym`+/// won't resolve, and it has no stability guarantee, so a null result (e.g. musl,+/// or a future glibc removing it) just disables this diagnostic.+#[cfg(all(target_os = "linux", target_env = "gnu"))]+fn abort_message_address() -> Option<u64> {+ let ptr = unsafe {+ libc::dlvsym(+ libc::RTLD_DEFAULT,+ c"__abort_msg".as_ptr(),+ c"GLIBC_PRIVATE".as_ptr(),+ )+ };+ std::ptr::NonNull::new(ptr).map(|ptr| ptr.as_ptr() as u64)+}++/// Read the crashed process's abort diagnostic. `__abort_msg` points to a+/// `struct abort_msg_s { unsigned int size; char msg[]; }` that glibc allocates+/// with mmap so that it stays intact even when the heap is corrupt. `size` is+/// the total byte size of that mapping (header included, rounded up to whole+/// pages), not the message length; the message itself is NUL-terminated.+#[cfg(target_os = "linux")]+fn read_abort_message(location: AbortMessageLocation) -> Option<String> {+ let pointer_bytes = read_process_memory(location.pid, location.address, size_of::<usize>())?;+ let message_address = usize::from_ne_bytes(pointer_bytes.try_into().ok()?) as u64;+ if message_address == 0 {+ return None;+ }+ let size_bytes = read_process_memory(location.pid, message_address, size_of::<u32>())?;+ let size = u32::from_ne_bytes(size_bytes.try_into().ok()?);+ let message_bytes = read_process_memory(+ location.pid,+ message_address + size_of::<u32>() as u64,+ abort_message_read_len(size)?,+ )?;+ parse_abort_message(&message_bytes)+}++/// How many message bytes to read given the `size` field of glibc's+/// `abort_msg_s`. `size` holds the total size of the mmap'd allocation, so a+/// value that isn't a whole number of pages means the layout has changed and+/// we shouldn't trust it. Reading is capped at (one page minus the header),+/// which both bounds the work and ensures the read never extends past the end+/// of the mapping.+#[cfg(any(target_os = "linux", test))]+fn abort_message_read_len(size: u32) -> Option<usize> {+ // Every Linux page size (4 KiB, 16 KiB, 64 KiB, ...) is a multiple of 4 KiB.+ const PAGE_MULTIPLE: usize = 4096;+ const MAX_READ: usize = 4096;++ let size = size as usize;+ if size == 0 || !size.is_multiple_of(PAGE_MULTIPLE) {+ log::warn!("__abort_msg size field {size} is not page-rounded; layout may have changed");+ return None;+ }+ Some(size.min(MAX_READ) - size_of::<u32>())+}++/// The message is NUL-terminated inside a zero-filled mapping, so truncate at+/// the first NUL; `trim` alone would keep the padding, since NUL is not+/// whitespace.+#[cfg(any(target_os = "linux", test))]+fn parse_abort_message(bytes: &[u8]) -> Option<String> {+ let len = bytes+ .iter()+ .position(|&byte| byte == 0)+ .unwrap_or(bytes.len());+ let message = String::from_utf8_lossy(&bytes[..len]).trim().to_string();+ (!message.is_empty()).then_some(message)+}++#[cfg(target_os = "linux")]+fn read_process_memory(pid: u32, address: u64, len: usize) -> Option<Vec<u8>> {+ let mut buffer = vec![0u8; len];+ let local = libc::iovec {+ iov_base: buffer.as_mut_ptr().cast(),+ iov_len: len,+ };+ let remote = libc::iovec {+ iov_base: address as *mut libc::c_void,+ iov_len: len,+ };+ let bytes_read =+ unsafe { libc::process_vm_readv(pid as libc::pid_t, &local, 1, &remote, 1, 0) };+ if bytes_read < 0 {+ log::warn!(+ "process_vm_readv of {len} bytes at {address:#x} in pid {pid} failed: {}",+ io::Error::last_os_error()+ );+ return None;+ }+ if bytes_read as usize != len {+ log::warn!(+ "process_vm_readv short read at {address:#x} in pid {pid}: {bytes_read} of {len} bytes"+ );+ return None;+ }+ Some(buffer)}impl minidumper::ServerHandler for CrashServer {@@ -281,6 +413,14 @@ impl minidumper::ServerHandler for CrashServer {}};+ // The crashed process is still alive at this point: it stays parked in+ // its signal handler until the server acknowledges the dump request,+ // which happens after this callback returns.+ #[cfg(target_os = "linux")]+ let abort_message = (*self.abort_message_location.lock()).and_then(read_abort_message);+ #[cfg(not(target_os = "linux"))]+ let abort_message = None;+let crash_info = CrashInfo {init: self.initialization_params@@ -289,6 +429,7 @@ impl minidumper::ServerHandler for CrashServer {.expect("not initialized"),panic: self.panic_info.lock().clone(),minidump_error,+ abort_message,active_gpu: self.active_gpu.lock().clone(),gpus,user_info: self.user_info.lock().clone(),@@ -320,6 +461,9 @@ impl minidumper::ServerHandler for CrashServer {CrashServerMessage::UserInfo(user_info) => {self.user_info.lock().replace(user_info);}+ CrashServerMessage::AbortMessageLocation(location) => {+ self.abort_message_location.lock().replace(location);+ }}}@@ -523,6 +667,7 @@ pub fn crash_server(socket: &Path, logs_dir: PathBuf) {initialization_params: Mutex::default(),panic_info: Mutex::default(),user_info: Mutex::default(),+ abort_message_location: Mutex::default(),has_connection,active_gpu: Mutex::default(),logs_dir,@@ -532,3 +677,99 @@ pub fn crash_server(socket: &Path, logs_dir: PathBuf) {).expect("failed to run server");}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn abort_message_read_len_requires_page_rounded_total() {+ assert_eq!(abort_message_read_len(0), None);+ // A message length rather than a mapping total means the glibc layout+ // has changed out from under us.+ assert_eq!(abort_message_read_len(23), None);+ assert_eq!(abort_message_read_len(4097), None);+ // The read must stay within the mapping: one page minus the header.+ assert_eq!(abort_message_read_len(4096), Some(4092));+ // Larger totals (long messages, larger page sizes) are clamped.+ assert_eq!(abort_message_read_len(8192), Some(4092));+ assert_eq!(abort_message_read_len(65536), Some(4092));+ }++ #[test]+ fn parse_abort_message_truncates_at_nul() {+ let mut buffer = b"free(): invalid pointer\n\0".to_vec();+ buffer.resize(4092, 0);+ assert_eq!(+ parse_abort_message(&buffer),+ Some("free(): invalid pointer".to_string())+ );+ }++ #[test]+ fn parse_abort_message_handles_missing_nul() {+ assert_eq!(+ parse_abort_message(b"double free or corruption (out)"),+ Some("double free or corruption (out)".to_string())+ );+ }++ #[test]+ fn parse_abort_message_rejects_empty() {+ assert_eq!(parse_abort_message(&[]), None);+ assert_eq!(parse_abort_message(&[0; 16]), None);+ assert_eq!(parse_abort_message(b"\n \0garbage after nul"), None);+ }++ /// End-to-end check of `read_abort_message` against a synthetic+ /// `abort_msg_s` in this very process (`process_vm_readv` may always read+ /// one's own memory). The message page is followed by a `PROT_NONE` guard+ /// page so the test fails if the read ever extends past the mapping glibc+ /// would have allocated.+ #[cfg(target_os = "linux")]+ #[test]+ fn read_abort_message_reads_glibc_layout_from_a_live_process() {+ let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;+ unsafe {+ let mapping = libc::mmap(+ std::ptr::null_mut(),+ 2 * page_size,+ libc::PROT_READ | libc::PROT_WRITE,+ libc::MAP_ANON | libc::MAP_PRIVATE,+ -1,+ 0,+ );+ assert_ne!(mapping, libc::MAP_FAILED);+ assert_eq!(+ libc::mprotect(+ mapping.cast::<u8>().add(page_size).cast(),+ page_size,+ libc::PROT_NONE+ ),+ 0+ );++ mapping.cast::<u32>().write(page_size as u32);+ let message = b"free(): invalid pointer\n\0";+ std::ptr::copy_nonoverlapping(+ message.as_ptr(),+ mapping.cast::<u8>().add(size_of::<u32>()),+ message.len(),+ );++ // Stands in for the `__abort_msg` global: a pointer variable whose+ // address we hand to the reader.+ let abort_msg: *mut libc::c_void = mapping;+ let location = AbortMessageLocation {+ pid: process::id(),+ address: (&raw const abort_msg) as u64,+ };+ assert_eq!(+ read_abort_message(location),+ Some("free(): invalid pointer".to_string())+ );++ libc::munmap(mapping, 2 * page_size);+ }+ }+}diff --git a/crates/zed/src/reliability.rs b/crates/zed/src/reliability.rsindex b4a06b2629d..00917069d20 100644--- a/crates/zed/src/reliability.rs+++ b/crates/zed/src/reliability.rs@@ -220,6 +220,21 @@ async fn upload_minidump(if let Some(minidump_error) = metadata.minidump_error.clone() {form = form.text("minidump_error", minidump_error);}+ if let Some(abort_message) = metadata.abort_message.as_ref() {+ // Sentry tag values are limited to 200 characters on a single line, so+ // put a searchable prefix in the tag (which grouping rules also match+ // on) and the full message in a context.+ let tag: String = abort_message+ .lines()+ .next()+ .unwrap_or_default()+ .chars()+ .take(200)+ .collect();+ form = form+ .text("sentry[tags][abort_message]", tag)+ .text("sentry[contexts][abort][message]", abort_message.clone());+ }if let Some(is_staff) = &metadata.user_info