A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Insert veneers on arm64 in cranelift-jit where necessary (#12239)
bjorn373ef034ae314
diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rsindex e320ba75f6..cbed2c3217 100644--- a/cranelift/jit/src/backend.rs+++ b/cranelift/jit/src/backend.rs@@ -2,7 +2,7 @@use crate::{compiled_blob::CompiledBlob,- memory::{BranchProtection, JITMemoryProvider, SystemMemoryProvider},+ memory::{BranchProtection, JITMemoryKind, JITMemoryProvider, SystemMemoryProvider},};use cranelift_codegen::binemit::Reloc;use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa};@@ -19,7 +19,6 @@ use std::cell::RefCell;use std::collections::HashMap;use std::ffi::CString;use std::io::Write;-use std::ptr;use target_lexicon::PointerWidth;const WRITABLE_DATA_ALIGNMENT: u64 = 0x8;@@ -217,7 +216,7 @@ impl JITModule {let (name, linkage) = if ModuleDeclarations::is_function(name) {let func_id = FuncId::from_name(name);match &self.compiled_functions[func_id] {- Some(compiled) => return compiled.ptr,+ Some(compiled) => return compiled.ptr(),None => {let decl = self.declarations.get_function_decl(func_id);(&decl.name, decl.linkage)@@ -226,7 +225,7 @@ impl JITModule {} else {let data_id = DataId::from_name(name);match &self.compiled_data_objects[data_id] {- Some(compiled) => return compiled.ptr,+ Some(compiled) => return compiled.ptr(),None => {let decl = self.declarations.get_data_decl(data_id);(&decl.name, decl.linkage)@@ -251,7 +250,7 @@ impl JITModule {}ModuleRelocTarget::FunctionOffset(func_id, offset) => {match &self.compiled_functions[*func_id] {- Some(compiled) => return compiled.ptr.wrapping_add(*offset as usize),+ Some(compiled) => return compiled.ptr().wrapping_add(*offset as usize),None => todo!(),}}@@ -271,7 +270,7 @@ impl JITModule {);info.as_ref().expect("function must be compiled before it can be finalized")- .ptr+ .ptr()}/// Returns the address and size of a finalized data object.@@ -288,10 +287,10 @@ impl JITModule {.as_ref().expect("data object must be compiled before it can be finalized");- (compiled.ptr, compiled.size)+ (compiled.ptr(), compiled.size())}- fn record_function_for_perf(&self, ptr: *mut u8, size: usize, name: &str) {+ fn record_function_for_perf(&self, ptr: *const u8, size: usize, name: &str) {// The Linux perf tool supports JIT code via a /tmp/perf-$PID.map file,// which contains memory regions and their associated names. If we// are profiling with perf and saving binaries to PERF_BUILDID_DIR@@ -407,8 +406,7 @@ impl JITModule {let data = self.compiled_functions[func].as_ref().unwrap()- .exception_data- .as_ref()?;+ .wasmtime_exception_data()?;let exception_table = wasmtime_unwinder::ExceptionTable::parse(data).ok()?;Some((start, exception_table))}@@ -485,22 +483,9 @@ impl Module for JITModule {let alignment = res.buffer.alignment as u64;let compiled_code = ctx.compiled_code().unwrap();- let size = compiled_code.code_info().total_size as usize;let align = alignment.max(self.isa.function_alignment().minimum as u64).max(self.isa.symbol_alignment());- let ptr =- self.memory- .allocate_readexec(size, align)- .map_err(|e| ModuleError::Allocation {- message: "unable to alloc function",- err: e,- })?;-- {- let mem = unsafe { std::slice::from_raw_parts_mut(ptr, size) };- mem.copy_from_slice(compiled_code.code_buffer());- }let relocs = compiled_code.buffer@@ -509,22 +494,8 @@ impl Module for JITModule {.map(|reloc| ModuleReloc::from_mach_reloc(reloc, &ctx.func, id)).collect();- self.record_function_for_perf(ptr, size, &decl.linkage_name(id));- self.compiled_functions[id] = Some(CompiledBlob {- ptr,- size,- relocs,- #[cfg(feature = "wasmtime-unwinder")]- exception_data: None,- });-- let range_start = ptr as usize;- let range_end = range_start + size;- // These will be sorted when we finalize.- self.code_ranges.push((range_start, range_end, id));-#[cfg(feature = "wasmtime-unwinder")]- {+ let wasmtime_exception_data = {let mut exception_builder = wasmtime_unwinder::ExceptionTableBuilder::default();exception_builder.add_func(0, compiled_code.buffer.call_sites())@@ -533,9 +504,25 @@ impl Module for JITModule {"Invalid exception data".into(),))})?;- self.compiled_functions[id].as_mut().unwrap().exception_data =- Some(exception_builder.to_vec());- }+ Some(exception_builder.to_vec())+ };++ let blob = self.compiled_functions[id].insert(CompiledBlob::new(+ &mut *self.memory,+ compiled_code.code_buffer(),+ align,+ relocs,+ #[cfg(feature = "wasmtime-unwinder")]+ wasmtime_exception_data,+ JITMemoryKind::Executable,+ )?);+ let (ptr, size) = (blob.ptr(), blob.size());+ self.record_function_for_perf(ptr, size, &decl.linkage_name(id));++ let range_start = ptr.addr();+ let range_end = range_start + size;+ // These will be sorted when we finalize.+ self.code_ranges.push((range_start, range_end, id));self.functions_to_finalize.push(id);@@ -563,30 +550,21 @@ impl Module for JITModule {));}- let size = bytes.len();let align = alignment.max(self.isa.function_alignment().minimum as u64).max(self.isa.symbol_alignment());- let ptr =- self.memory- .allocate_readexec(size, align)- .map_err(|e| ModuleError::Allocation {- message: "unable to alloc function bytes",- err: e,- })?;-- unsafe {- ptr::copy_nonoverlapping(bytes.as_ptr(), ptr, size);- }- self.record_function_for_perf(ptr, size, &decl.linkage_name(id));- self.compiled_functions[id] = Some(CompiledBlob {- ptr,- size,- relocs: relocs.to_owned(),+ let blob = self.compiled_functions[id].insert(CompiledBlob::new(+ &mut *self.memory,+ bytes,+ align,+ relocs.to_owned(),#[cfg(feature = "wasmtime-unwinder")]- exception_data: None,- });+ None,+ JITMemoryKind::Executable,+ )?);+ let (ptr, size) = (blob.ptr(), blob.size());+ self.record_function_for_perf(ptr, size, &decl.linkage_name(id));self.functions_to_finalize.push(id);@@ -620,52 +598,18 @@ impl Module for JITModule {used: _,} = data;- // Make sure to allocate at least 1 byte. Allocating 0 bytes is UB. Previously a dummy- // value was used, however as it turns out this will cause pc-relative relocations to- // fail on architectures where pc-relative offsets are range restricted as the dummy- // value is not close enough to the code that has the pc-relative relocation.- let alloc_size = std::cmp::max(init.size(), 1);-- let ptr = if decl.writable {- self.memory- .allocate_readwrite(alloc_size, align.unwrap_or(WRITABLE_DATA_ALIGNMENT))- .map_err(|e| ModuleError::Allocation {- message: "unable to alloc writable data",- err: e,- })?+ let (align, kind) = if decl.writable {+ (+ align.unwrap_or(WRITABLE_DATA_ALIGNMENT),+ JITMemoryKind::Writable,+ )} else {- self.memory- .allocate_readonly(alloc_size, align.unwrap_or(READONLY_DATA_ALIGNMENT))- .map_err(|e| ModuleError::Allocation {- message: "unable to alloc readonly data",- err: e,- })?+ (+ align.unwrap_or(READONLY_DATA_ALIGNMENT),+ JITMemoryKind::ReadOnly,+ )};- if ptr.is_null() {- // FIXME pass a Layout to allocate and only compute the layout once.- std::alloc::handle_alloc_error(- std::alloc::Layout::from_size_align(- alloc_size,- align.unwrap_or(READONLY_DATA_ALIGNMENT).try_into().unwrap(),- )- .unwrap(),- );- }-- match *init {- Init::Uninitialized => {- panic!("data is not initialized yet");- }- Init::Zeros { size } => {- unsafe { ptr::write_bytes(ptr, 0, size) };- }- Init::Bytes { ref contents } => {- let src = contents.as_ptr();- unsafe { ptr::copy_nonoverlapping(src, ptr, contents.len()) };- }- }-let pointer_reloc = match self.isa.triple().pointer_width().unwrap() {PointerWidth::U16 => panic!(),PointerWidth::U32 => Reloc::Abs4,@@ -673,44 +617,43 @@ impl Module for JITModule {};let relocs = data.all_relocs(pointer_reloc).collect::<Vec<_>>();- self.compiled_data_objects[id] = Some(CompiledBlob {- ptr,- size: init.size(),- relocs,- #[cfg(feature = "wasmtime-unwinder")]- exception_data: None,+ self.compiled_data_objects[id] = Some(match *init {+ Init::Uninitialized => {+ panic!("data is not initialized yet");+ }+ Init::Zeros { size } => CompiledBlob::new_zeroed(+ &mut *self.memory,+ size.max(1),+ align,+ relocs,+ #[cfg(feature = "wasmtime-unwinder")]+ None,+ kind,+ )?,+ Init::Bytes { ref contents } => CompiledBlob::new(+ &mut *self.memory,+ if contents.is_empty() {+ // Make sure to allocate at least 1 byte. Allocating 0 bytes is UB. Previously+ // a dummy value was used, however as it turns out this will cause pc-relative+ // relocations to fail on architectures where pc-relative offsets are range+ // restricted as the dummy value is not close enough to the code that has the+ // pc-relative relocation.+ &[0]+ } else {+ &contents[..]+ },+ align,+ relocs,+ #[cfg(feature = "wasmtime-unwinder")]+ None,+ kind,+ )?,});+self.data_objects_to_finalize.push(id);Ok(())}-- fn get_name(&self, name: &str) -> Option<cranelift_module::FuncOrDataId> {- self.declarations().get_name(name)- }-- fn target_config(&self) -> cranelift_codegen::isa::TargetFrontendConfig {- self.isa().frontend_config()- }-- fn make_context(&self) -> cranelift_codegen::Context {- let mut ctx = cranelift_codegen::Context::new();- ctx.func.signature.call_conv = self.isa().default_call_conv();- ctx- }-- fn clear_context(&self, ctx: &mut cranelift_codegen::Context) {- ctx.clear();- ctx.func.signature.call_conv = self.isa().default_call_conv();- }-- fn make_signature(&self) -> ir::Signature {- ir::Signature::new(self.isa().default_call_conv())- }-- fn clear_signature(&self, sig: &mut ir::Signature) {- sig.clear(self.isa().default_call_conv());- }}#[cfg(not(windows))]@@ -728,6 +671,7 @@ fn lookup_with_dlsym(name: &str) -> Option<*const u8> {#[cfg(windows)]fn lookup_with_dlsym(name: &str) -> Option<*const u8> {use std::os::windows::io::RawHandle;+ use std::ptr;use windows_sys::Win32::Foundation::HMODULE;use windows_sys::Win32::System::LibraryLoader;@@ -763,13 +707,7 @@ fn use_bti(isa_flags: &Vec<settings::Value>) -> bool {.map_or(false, |f| f.as_bool().unwrap_or(false))}-#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_jit_module_is_send() {- fn assert_is_send<T: Send>() {}- assert_is_send::<JITModule>();- }-}+const _ASSERT_JIT_MODULE_IS_SEND: () = {+ const fn assert_is_send<T: Send>() {}+ assert_is_send::<JITModule>();+};diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rsindex 003d6f00da..81e1bc707d 100644--- a/cranelift/jit/src/compiled_blob.rs+++ b/cranelift/jit/src/compiled_blob.rs@@ -1,6 +1,12 @@+use std::ptr;+use cranelift_codegen::binemit::Reloc;-use cranelift_module::ModuleReloc;-use cranelift_module::ModuleRelocTarget;+use cranelift_module::{ModuleError, ModuleReloc, ModuleRelocTarget, ModuleResult};++use crate::JITMemoryProvider;+use crate::memory::JITMemoryKind;++const VENEER_SIZE: usize = 24; // ldr + br + pointer/// Reads a 32bit instruction at `iptr`, and writes it again after/// being altered by `modifier`@@ -12,22 +18,97 @@ unsafe fn modify_inst32(iptr: *mut u32, modifier: impl FnOnce(u32) -> u32) {#[derive(Clone)]pub(crate) struct CompiledBlob {- pub(crate) ptr: *mut u8,- pub(crate) size: usize,- pub(crate) relocs: Vec<ModuleReloc>,+ ptr: *mut u8,+ size: usize,+ relocs: Vec<ModuleReloc>,+ veneer_count: usize,#[cfg(feature = "wasmtime-unwinder")]- pub(crate) exception_data: Option<Vec<u8>>,+ wasmtime_exception_data: Option<Vec<u8>>,}unsafe impl Send for CompiledBlob {}impl CompiledBlob {+ pub(crate) fn new(+ memory: &mut dyn JITMemoryProvider,+ data: &[u8],+ align: u64,+ relocs: Vec<ModuleReloc>,+ #[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option<Vec<u8>>,+ kind: JITMemoryKind,+ ) -> ModuleResult<Self> {+ // Reserve veneers for all function calls just in case+ let mut veneer_count = 0;+ for reloc in &relocs {+ match reloc.kind {+ Reloc::Arm64Call => veneer_count += 1,+ _ => {}+ }+ }++ let ptr = memory+ .allocate(data.len() + veneer_count * VENEER_SIZE, align, kind)+ .map_err(|e| ModuleError::Allocation { err: e })?;++ unsafe {+ ptr::copy_nonoverlapping(data.as_ptr(), ptr, data.len());+ }++ Ok(CompiledBlob {+ ptr,+ size: data.len(),+ relocs,+ veneer_count,+ #[cfg(feature = "wasmtime-unwinder")]+ wasmtime_exception_data,+ })+ }++ pub(crate) fn new_zeroed(+ memory: &mut dyn JITMemoryProvider,+ size: usize,+ align: u64,+ relocs: Vec<ModuleReloc>,+ #[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option<Vec<u8>>,+ kind: JITMemoryKind,+ ) -> ModuleResult<Self> {+ let ptr = memory+ .allocate(size, align, kind)+ .map_err(|e| ModuleError::Allocation { err: e })?;++ unsafe { ptr::write_bytes(ptr, 0, size) };++ Ok(CompiledBlob {+ ptr,+ size,+ relocs,+ veneer_count: 0,+ #[cfg(feature = "wasmtime-unwinder")]+ wasmtime_exception_data,+ })+ }++ pub(crate) fn ptr(&self) -> *const u8 {+ self.ptr+ }++ pub(crate) fn size(&self) -> usize {+ self.size+ }++ #[cfg(feature = "wasmtime-unwinder")]+ pub(crate) fn wasmtime_exception_data(&self) -> Option<&[u8]> {+ self.wasmtime_exception_data.as_deref()+ }+pub(crate) fn perform_relocations(&self,get_address: impl Fn(&ModuleRelocTarget) -> *const u8,) {use std::ptr::write_unaligned;+ let mut next_veneer_idx = 0;+for (i,&ModuleReloc {@@ -75,21 +156,53 @@ impl CompiledBlob {}Reloc::Arm64Call => {let base = get_address(name);+ let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };// The instruction is 32 bits long.let iptr = at as *mut u32;+// The offset encoded in the `bl` instruction is the// number of bytes divided by 4.- let diff = ((base as isize) - (at as isize)) >> 2;+ let diff = ((what as isize) - (at as isize)) >> 2;// Sign propagating right shift disposes of the// included bits, so the result is expected to be- // either all sign bits or 0, depending on if the original- // value was negative or positive.- assert!((diff >> 26 == -1) || (diff >> 26 == 0));- // The lower 26 bits of the `bl` instruction form the- // immediate offset argument.- let chop = 32 - 26;- let imm26 = (diff as u32) << chop >> chop;- unsafe { modify_inst32(iptr, |inst| inst | imm26) };+ // either all sign bits or 0 when in-range, depending+ // on if the original value was negative or positive.+ if (diff >> 25 == -1) || (diff >> 25 == 0) {+ // The lower 26 bits of the `bl` instruction form the+ // immediate offset argument.+ let chop = 32 - 26;+ let imm26 = (diff as u32) << chop >> chop;+ unsafe { modify_inst32(iptr, |inst| inst | imm26) };+ } else {+ // If the target is out of range for a direct call, insert a veneer at the+ // end of the function.+ let veneer_idx = next_veneer_idx;+ next_veneer_idx += 1;+ assert!(veneer_idx <= self.veneer_count);+ let veneer =+ unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };++ // Write the veneer+ // x16 is reserved as scratch register to be used by veneers and PLT entries+ unsafe {+ write_unaligned(+ veneer.cast::<u32>(),+ 0x58000050, // ldr x16, 0x8+ );+ write_unaligned(+ veneer.byte_add(4).cast::<u32>(),+ 0xd61f0200, // br x16+ );+ write_unaligned(veneer.byte_add(8).cast::<u64>(), what.addr() as u64);+ };++ // Set the veneer as target of the call+ let diff = ((veneer as isize) - (at as isize)) >> 2;+ assert!((diff >> 25 == -1) || (diff >> 25 == 0));+ let chop = 32 - 26;+ let imm26 = (diff as u32) << chop >> chop;+ unsafe { modify_inst32(iptr, |inst| inst | imm26) };+ }}Reloc::Aarch64AdrGotPage21 => {panic!("GOT relocation shouldn't be generated when !is_pic");diff --git a/cranelift/jit/src/memory/arena.rs b/cranelift/jit/src/memory/arena.rsindex 1afb93f7cf..00fdbbc88c 100644--- a/cranelift/jit/src/memory/arena.rs+++ b/cranelift/jit/src/memory/arena.rs@@ -4,7 +4,7 @@ use std::ptr;use cranelift_module::ModuleResult;-use super::{BranchProtection, JITMemoryProvider};+use super::{BranchProtection, JITMemoryKind, JITMemoryProvider};fn align_up(addr: usize, align: usize) -> usize {debug_assert!(align.is_power_of_two());@@ -123,7 +123,7 @@ impl ArenaMemoryProvider {})}- fn allocate(+ fn allocate_inner(&mut self,size: usize,align: u64,@@ -221,16 +221,16 @@ impl Drop for ArenaMemoryProvider {}impl JITMemoryProvider for ArenaMemoryProvider {- fn allocate_readexec(&mut self, size: usize, align: u64) -> io::Result<*mut u8> {- self.allocate(size, align, region::Protection::READ_EXECUTE)- }-- fn allocate_readwrite(&mut self, size: usize, align: u64) -> io::Result<*mut u8> {- self.allocate(size, align, region::Protection::READ_WRITE)- }-- fn allocate_readonly(&mut self, size: usize, align: u64) -> io::Result<*mut u8> {- self.allocate(size, align, region::Protection::READ)+ fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8> {+ self.allocate_inner(+ size,+ align,+ match kind {+ JITMemoryKind::Executable => region::Protection::READ_EXECUTE,+ JITMemoryKind::Writable => region::Protection::READ_WRITE,+ JITMemoryKind::ReadOnly => region::Protection::READ,+ },+ )}unsafe fn free_memory(&mut self) {@@ -254,7 +254,9 @@ mod tests {for align_log2 in 0..8 {let align = 1usize << align_log2;for size in 1..128 {- let ptr = arena.allocate_readwrite(size, align as u64).unwrap();+ let ptr = arena+ .allocate(size, align as u64, JITMemoryKind::Writable)+ .unwrap();// assert!(ptr.is_aligned_to(align));assert_eq!(ptr.addr() % align, 0);}@@ -269,7 +271,7 @@ mod tests {// platforms. Physical memory should be committed as we go.let reserve_size = 1 << 40;let mut arena = ArenaMemoryProvider::new_with_size(reserve_size).unwrap();- let ptr = arena.allocate_readwrite(1, 1).unwrap();+ let ptr = arena.allocate(1, 1, JITMemoryKind::Writable).unwrap();assert_eq!(ptr.addr(), arena.ptr.addr());arena.finalize(BranchProtection::None);unsafe { ptr.write_volatile(42) };@@ -280,8 +282,10 @@ mod tests {fn over_capacity() {let mut arena = ArenaMemoryProvider::new_with_size(1 << 20).unwrap(); // 1 MB- let _ = arena.allocate_readwrite(900_000, 1).unwrap();- let _ = arena.allocate_readwrite(200_000, 1).unwrap_err();+ let _ = arena.allocate(900_000, 1, JITMemoryKind::Writable).unwrap();+ let _ = arena+ .allocate(200_000, 1, JITMemoryKind::Writable)+ .unwrap_err();}#[test]diff --git a/cranelift/jit/src/memory/mod.rs b/cranelift/jit/src/memory/mod.rsindex 9052d2da56..7a9e8b77eb 100644--- a/cranelift/jit/src/memory/mod.rs+++ b/cranelift/jit/src/memory/mod.rs@@ -16,14 +16,19 @@ pub enum BranchProtection {BTI,}-/// A provider of memory for the JIT.-pub trait JITMemoryProvider {+pub enum JITMemoryKind {/// Allocate memory that will be executable once finalized.- fn allocate_readexec(&mut self, size: usize, align: u64) -> io::Result<*mut u8>;+ Executable,/// Allocate writable memory.- fn allocate_readwrite(&mut self, size: usize, align: u64) -> io::Result<*mut u8>;+ Writable,/// Allocate memory that will be read-only once finalized.- fn allocate_readonly(&mut self, size: usize, align: u64) -> io::Result<*mut u8>;+ ReadOnly,+}++/// A provider of memory for the JIT.+pub trait JITMemoryProvider {+ /// Allocate memory+ fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8>;/// Free the memory region.unsafe fn free_memory(&mut self);diff --git a/cranelift/jit/src/memory/system.rs b/cranelift/jit/src/memory/system.rsindex 08f8f0fd0a..bd845aef4a 100644--- a/cranelift/jit/src/memory/system.rs+++ b/cranelift/jit/src/memory/system.rs@@ -9,8 +9,7 @@ use std::io;use std::mem;use std::ptr;-use super::BranchProtection;-use super::JITMemoryProvider;+use super::{BranchProtection, JITMemoryKind, JITMemoryProvider};/// A simple struct consisting of a pointer and length.struct PtrLen {@@ -263,15 +262,11 @@ impl JITMemoryProvider for SystemMemoryProvider {self.code.set_readable_and_executable(branch_protection)}- fn allocate_readexec(&mut self, size: usize, align: u64) -> io::Result<*mut u8> {- self.code.allocate(size, align)- }-- fn allocate_readwrite(&mut self, size: usize, align: u64) -> io::Result<*mut u8> {- self.writable.allocate(size, align)- }-- fn allocate_readonly(&mut self, size: usize, align: u64) -> io::Result<*mut u8> {- self.readonly.allocate(size, align)+ fn allocate(&mut self, size: usize, align: u64, kind: JITMemoryKind) -> io::Result<*mut u8> {+ match kind {+ JITMemoryKind::Executable => self.code.allocate(size, align),+ JITMemoryKind::Writable => self.writable.allocate(size, align),+ JITMemoryKind::ReadOnly => self.readonly.allocate(size, align),+ }}}diff --git a/cranelift/module/src/module.rs b/cranelift/module/src/module.rsindex 79dcc1efec..526f998561 100644--- a/cranelift/module/src/module.rs+++ b/cranelift/module/src/module.rs@@ -291,8 +291,6 @@ pub enum ModuleError {/// Memory allocation failure from a backendAllocation {- /// Tell where the allocation came from- message: &'static str,/// Io error the allocation failed witherr: std::io::Error,},@@ -321,7 +319,7 @@ impl std::error::Error for ModuleError {| Self::DuplicateDefinition { .. }| Self::InvalidImportDefinition { .. } => None,Self::Compilation(source) => Some(source),- Self::Allocation { err: source, .. } => Some(source),+ Self::Allocation { err: source } => Some(source),Self::Backend(source) => Some(&**source),Self::Flag(source) => Some(source),}@@ -355,8 +353,8 @@ impl std::fmt::Display for ModuleError {Self::Compilation(err) => {write!(f, "Compilation error: {err}")}- Self::Allocation { message, err } => {- write!(f, "Allocation error: {message}: {err}")+ Self::Allocation { err } => {+ write!(f, "Allocation error: {err}")}Self::Backend(err) => write!(f, "Backend error: {err}"),Self::Flag(err) => write!(f, "Flag error: {err}"),