A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Debug: create private code memories per store when debugging is enabled. (#12051)
Chris Fallin99ecf728d6fe
diff --git a/crates/c-api/include/wasmtime/profiling.h b/crates/c-api/include/wasmtime/profiling.hindex cc64023644..885fec56ae 100644--- a/crates/c-api/include/wasmtime/profiling.h+++ b/crates/c-api/include/wasmtime/profiling.h@@ -56,6 +56,7 @@ typedef struct wasmtime_guestprofiler_modules {/*** \brief Begin profiling a new guest.*+ * \param engine engine in which to perform the profiling* \param module_name name recorded in the profile* \param interval_nanos intended sampling interval in nanoseconds recorded in* the profile@@ -71,8 +72,9 @@ typedef struct wasmtime_guestprofiler_modules {* https://docs.wasmtime.dev/api/wasmtime/struct.GuestProfiler.html#method.new*/WASM_API_EXTERN /* own */ wasmtime_guestprofiler_t *wasmtime_guestprofiler_new(- const wasm_name_t *module_name, uint64_t interval_nanos,- const wasmtime_guestprofiler_modules_t *modules, size_t modules_len);+ const wasm_engine_t *engine, const wasm_name_t *module_name,+ uint64_t interval_nanos, const wasmtime_guestprofiler_modules_t *modules,+ size_t modules_len);/*** \brief Add a sample to the profile.diff --git a/crates/c-api/src/profiling.rs b/crates/c-api/src/profiling.rsindex cda09c4e33..307b434f80 100644--- a/crates/c-api/src/profiling.rs+++ b/crates/c-api/src/profiling.rs@@ -1,4 +1,7 @@-use crate::{wasm_byte_vec_t, wasm_name_t, wasmtime_error_t, wasmtime_module_t, wasmtime_store_t};+use crate::{+ wasm_byte_vec_t, wasm_engine_t, wasm_name_t, wasmtime_error_t, wasmtime_module_t,+ wasmtime_store_t,+};use std::slice;use std::str::from_utf8;use std::time::Duration;@@ -18,6 +21,7 @@ pub struct wasmtime_guestprofiler_modules_t<'a> {#[unsafe(no_mangle)]pub unsafe extern "C" fn wasmtime_guestprofiler_new(+ engine: &wasm_engine_t,module_name: &wasm_name_t,interval_nanos: u64,modules: *const wasmtime_guestprofiler_modules_t,@@ -36,7 +40,13 @@ pub unsafe extern "C" fn wasmtime_guestprofiler_new(}).collect::<Vec<_>>();Box::new(wasmtime_guestprofiler_t {- guest_profiler: GuestProfiler::new(module_name, Duration::from_nanos(interval_nanos), list),+ guest_profiler: GuestProfiler::new(+ &engine.engine,+ module_name,+ Duration::from_nanos(interval_nanos),+ list,+ )+ .unwrap(),})}diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rsindex 27becc5991..190b0979ab 100644--- a/crates/environ/src/key.rs+++ b/crates/environ/src/key.rs@@ -515,4 +515,29 @@ impl FuncKey {_ => panic!("`FuncKey::unwrap_resource_drop_trampoline` called on {self:?}"),}}++ /// Is this "Store-invariant"? This allows us to execute+ /// EngineCode directly rather than StoreCode.+ ///+ /// Any function that is either directly from Wasm code, or calls+ /// it directly (not indirected through a runtime-provided+ /// function pointer), is "store-variant": we need to use a+ /// StoreCode-specific version of the code to hit any patching+ /// that our specific instantiations may have (due to debugging+ /// breakpoints, etc). Trampolines into the runtime cannot be+ /// patched and so can use EngineCode instead. This allows for+ /// less complex plumbing in some places where we can avoid+ /// looking up the StoreCode (or having access to the Store).+ pub fn is_store_invariant(&self) -> bool {+ match self {+ Self::DefinedWasmFunction(..) | Self::ArrayToWasmTrampoline(..) => false,+ Self::WasmToArrayTrampoline(..)+ | Self::WasmToBuiltinTrampoline(..)+ | Self::PulleyHostCall(..) => true,+ #[cfg(feature = "component-model")]+ Self::ComponentTrampoline(..)+ | Self::ResourceDropTrampoline+ | Self::UnsafeIntrinsic(..) => true,+ }+ }}diff --git a/crates/wasmtime/src/runtime/code.rs b/crates/wasmtime/src/runtime/code.rsindex 952f44f779..4fabb195db 100644--- a/crates/wasmtime/src/runtime/code.rs+++ b/crates/wasmtime/src/runtime/code.rs@@ -1,25 +1,114 @@+use crate::Engine;+use crate::Module;+use crate::module::ModuleRegistry;+use crate::vm::ModuleMemoryImageSource;use crate::{code_memory::CodeMemory, type_registry::TypeCollection};+#[cfg(feature = "debug")]+use alloc::boxed::Box;use alloc::sync::Arc;+use anyhow::Result;+use core::ops::{Add, Range, Sub};+use wasmtime_environ::DefinedFuncIndex;use wasmtime_environ::ModuleTypes;#[cfg(feature = "component-model")]use wasmtime_environ::component::ComponentTypes;-/// Metadata in Wasmtime about a loaded compiled artifact in memory which is-/// ready to execute.+macro_rules! define_pc_kind {+ ($ty:ident) => {+ #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+ pub struct $ty(usize);++ impl Add<usize> for $ty {+ type Output = $ty;+ fn add(self, other: usize) -> $ty {+ $ty(self.0.wrapping_add(other))+ }+ }+ impl Sub<usize> for $ty {+ type Output = $ty;+ fn sub(self, other: usize) -> $ty {+ $ty(self.0.wrapping_sub(other))+ }+ }++ impl $ty {+ /// Is the given PC within this range? Give the relative+ /// offset if so.+ pub fn offset_of(range: Range<$ty>, pc: usize) -> Option<usize> {+ if pc >= range.start.0 && pc < range.end.0 {+ Some(pc.wrapping_sub(range.start.0))+ } else {+ None+ }+ }++ /// Get the raw PC value.+ pub fn raw(&self) -> usize {+ self.0+ }+ }+ };+}++// An address in "engine code": the original (canonical) copy of code+// for a compiled module.+//+// See [`EngineCode`] for more details.+define_pc_kind!(EngineCodePC);++// An address in "store code": the copy of code used when executing+// instances of a module in a given store.+//+// May or may not be the same as the engine code address -- a store+// is allowed to make a private copy of engine code as its store+// code.+//+// See [`StoreCode`] for more details.+define_pc_kind!(StoreCodePC);++impl StoreCodePC {+ /// Construct a StoreCodePC for search purposes from a raw PC+ /// observed during execution.+ pub fn from_raw(pc: usize) -> StoreCodePC {+ StoreCodePC(pc)+ }+}++/// Metadata about, and original machine code for, a loaded compiled+/// artifact in memory which is ready for Wasmtime to execute.////// This structure is used in both `Module` and `Component`. For components it's/// notably shared amongst the core wasm modules within a component and the/// component itself. For core wasm modules this is uniquely owned within a/// `Module`.-pub struct CodeObject {- /// Actual underlying mmap which is executable and contains other compiled- /// information.+///+/// The `EngineCode` is "static", like a module: there is one copy per+/// unit of code. Instantiation of a module containing an `EngineCode`+/// into a `Store` produces a `StoreCode`. The latter is the owner of+/// the actual machine code that runs. This machine code *may* be a+/// read-only-shared copy of the original, or may be a private copy+/// that we have patched. The latter is useful for some kinds of+/// debugging/instrumentation, which is always scoped per-Store.+///+/// The `EngineCode` does *not* expose its underlying `CodeMemory`, to+/// guard against the original-copy being used in an inappropriate+/// (store-specific) context. Instead, it has accessors that provide+/// the instance-agnostic metadata from the `CodeMemory`, and+/// instance-specific data and code pointers can be obtained from the+/// `StoreCode`.+pub struct EngineCode {+ /// Actual underlying code which is executable and contains other+ /// compiled information.////// Note the `Arc` here is used to share this with `CompiledModule` and the/// global module registry of traps. While probably not strictly necessary/// and could be avoided with some refactorings is a hopefully a relatively/// minor `Arc` for now.- mmap: Arc<CodeMemory>,+ ///+ /// As noted above, this is the *original* copy of the code, and+ /// may be used directly, but may also be deep-cloned to a private+ /// copy in a `StoreCode`.+ original_code: Arc<CodeMemory>,/// Registered shared signature for the loaded object.///@@ -35,23 +124,19 @@ pub struct CodeObject {types: Types,}-impl CodeObject {- pub fn new(mmap: Arc<CodeMemory>, signatures: TypeCollection, types: Types) -> CodeObject {+impl EngineCode {+ pub fn new(mmap: Arc<CodeMemory>, signatures: TypeCollection, types: Types) -> EngineCode {// The corresponding unregister for this is below in `Drop for- // CodeObject`.- crate::module::register_code(&mmap);+ // EngineCode`.+ crate::module::register_code(&mmap, mmap.raw_addr_range());- CodeObject {- mmap,+ EngineCode {+ original_code: mmap,signatures,types,}}- pub fn code_memory(&self) -> &Arc<CodeMemory> {- &self.mmap- }-#[cfg(feature = "component-model")]pub fn types(&self) -> &Types {&self.types@@ -64,11 +149,111 @@ impl CodeObject {pub fn signatures(&self) -> &TypeCollection {&self.signatures}++ pub fn text_size(&self) -> usize {+ self.original_code.text().len()+ }++ /// Give the range of engine-code PCs in this code.+ pub fn text_range(&self) -> Range<EngineCodePC> {+ let raw = self.original_code.raw_addr_range();+ EngineCodePC(raw.start)..EngineCodePC(raw.end)+ }++ // For all accessors to code object sections below: we can use the+ // original (uncloned) code image for these: they are just slices+ // of bytes that we interpret independent of location.+ //+ // We hide the raw `CodeMemory` in `original_code` (*not* a public+ // field) because we do not want the EngineCode copy used+ // inadvertently for execution of Wasm functions.+ //+ // We thus *cannot* add an accessor below for anything that+ // actually interprets the location of the bytes as a code+ // pointer to a Wasm function.+ //+ // Note that Wasm-to-array trampolines, in contrast, are fair game+ // to execute directly from the EngineCode: these are not specific+ // to Wasm functions, but instead call builtins, and we never+ // patch trampolines in StoreCode, so we can freely mix in+ // EngineCode variants.++ /// Get the Wasm-to-array trampoline for the given raw range in+ /// the text segment.+ pub(crate) fn raw_wasm_to_array_trampoline_data(&self, range: Range<usize>) -> &[u8] {+ &self.original_code.text()[range]+ }++ /// Provide the address-map data for this EngineCode.+ pub fn address_map_data(&self) -> &[u8] {+ // We can use the original (uncloned) code image for this: it+ // is just a slice of bytes that we interpret independent of+ // location.+ self.original_code.address_map_data()+ }++ /// Provide the stack-map data for this EngineCode.+ pub fn stack_map_data(&self) -> &[u8] {+ // We can use the original (uncloned) code image for this: it+ // is just a slice of bytes that we interpret independent of+ // location.+ self.original_code.stack_map_data()+ }++ /// Returns the encoded exception-tables section to pass to+ /// `wasmtime_unwinder::ExceptionTable::parse`.+ pub fn exception_tables(&self) -> &[u8] {+ // We can use the original (uncloned) code image for this: it+ // is just a slice of bytes that we interpret independent of+ // location.+ self.original_code.exception_tables()+ }++ /// Returns the encoded frame-tables section to pass to+ /// `wasmtime_environ::FrameTable::parse`.+ pub fn frame_tables(&self) -> &[u8] {+ self.original_code.frame_tables()+ }++ /// Returns the data in the `ELF_NAME_DATA` section.+ #[inline]+ pub fn func_name_data(&self) -> &[u8] {+ self.original_code.func_name_data()+ }++ /// Returns the contents of the `ELF_WASMTIME_DWARF` section.+ #[inline]+ pub fn wasm_dwarf(&self) -> &[u8] {+ self.original_code.wasm_dwarf()+ }++ /// Returns the raw image as bytes (in our internal image format).+ pub fn image(&self) -> &[u8] {+ &self.original_code.mmap()[..]+ }++ pub fn text(&self) -> &[u8] {+ &self.original_code.text()+ }++ /// Returns the concatenated list of all data associated with this wasm+ /// module.+ ///+ /// This is used for initialization of memories and all data ranges stored+ /// in a `Module` are relative to the slice returned here.+ #[inline]+ pub fn wasm_data(&self) -> &[u8] {+ self.original_code.wasm_data()+ }++ pub(crate) fn module_memory_image_source(&self) -> &Arc<impl ModuleMemoryImageSource> {+ &self.original_code+ }}-impl Drop for CodeObject {+impl Drop for EngineCode {fn drop(&mut self) {- crate::module::unregister_code(&self.mmap);+ crate::module::unregister_code(self.original_code.raw_addr_range());}}@@ -100,3 +285,193 @@ impl From<Arc<ComponentTypes>> for Types {Types::Component(types)}}++/// A `Store`-local instance of code.+///+/// This type encapsulates executable code within the context of+/// instantiation in a single store. It may be an unmodified pointer+/// to the read-only original code, or it may be locally patched for+/// this store due to debugging or instrumentation settings.+///+/// Most things that the runtime will want to do with a module's image+/// will require the `EngineCode` -- all metadata lives there. The+/// `StoreCode` is solely responsible for the executable machine code.+///+/// This type is designed to be uniquely owned, unlike `EngineCode`+/// above. The runtime data structures will have scattered `Arc`s to+/// the `EngineCode` but only one+pub struct StoreCode(StoreCodeStorage);++enum StoreCodeStorage {+ Shared(Arc<CodeMemory>),+ /// Private copy of the given code memory.+ ///+ /// This is the only reference to this CodeMemory. The StoreCode+ /// is owned directly by the Store's ModuleRegistry.+ #[cfg(feature = "debug")]+ Private(Box<CodeMemory>),+}++impl StoreCode {+ /// Create a new StoreCode for a given Store from a given+ /// EngineCode, given the engine's settings (tunables).+ pub fn new(engine: &Engine, engine_code: &Arc<EngineCode>) -> Result<Self> {+ // Enabled guest-debugging causes us to allocate private+ // copies of code in every store, to allow individual enabling+ // of breakpoints (by code patching) independently in each+ // one.+ #[cfg(feature = "debug")]+ let code = if engine.tunables().debug_guest {+ // TODO(#12104): we should be able to clone only `.text`;+ // this clones the whole image.+ let mut private_copy = engine_code.original_code.deep_clone(engine)?;+ private_copy.publish()?;+ crate::module::register_code(&engine_code.original_code, private_copy.raw_addr_range());+ StoreCodeStorage::Private(Box::new(private_copy))+ } else {+ StoreCodeStorage::Shared(engine_code.original_code.clone())+ };++ #[cfg(not(feature = "debug"))]+ let code = StoreCodeStorage::Shared(engine_code.original_code.clone());+ // Avoid unused-variable warning in build without debugging+ // support.+ let _ = engine;++ Ok(StoreCode(code))+ }++ /// Provide the underlying CodeMemory.+ pub fn code_memory(&self) -> &CodeMemory {+ match &self.0 {+ StoreCodeStorage::Shared(m) => m,+ #[cfg(feature = "debug")]+ StoreCodeStorage::Private(m) => m,+ }+ }++ /// Provide a mutable reference to a CodeMemory that is privately+ /// owned only by this StoreCode.+ #[cfg(feature = "debug")]+ pub fn code_memory_mut(&mut self) -> Option<&mut CodeMemory> {+ match &mut self.0 {+ StoreCodeStorage::Shared(_) => None,+ StoreCodeStorage::Private(m) => Some(m),+ }+ }++ /// Provide the address range for this StoreCode.+ pub fn text_range(&self) -> Range<StoreCodePC> {+ let raw = self.code_memory().raw_addr_range();+ StoreCodePC(raw.start)..StoreCodePC(raw.end)+ }++ /// Provide the actual text segment for this StoreCode.+ pub fn text(&self) -> &[u8] {+ self.code_memory().text()+ }+}++impl Drop for StoreCode {+ fn drop(&mut self) {+ match &self.0 {+ StoreCodeStorage::Shared(_) => {+ // Drop impl for EngineCode will de-register (see+ // above).+ }+ #[cfg(feature = "debug")]+ StoreCodeStorage::Private(mem) => {+ crate::module::unregister_code(mem.raw_addr_range());+ }+ }+ }+}++/// A wrapper for a Module together with a StoreCode. Allows fetching+/// code pointers, ready to call.+pub struct ModuleWithCode<'a> {+ module: &'a Module,+ store_code: &'a StoreCode,+}++impl<'a> ModuleWithCode<'a> {+ /// Find the StoreCode in a given store for a module and wrap it+ /// up with that module, ready to compute code pointers.+ pub fn in_store(+ registry: &'a ModuleRegistry,+ module: &'a Module,+ ) -> Option<ModuleWithCode<'a>> {+ let store_code = registry.store_code(module.engine_code())?;+ Some(ModuleWithCode { module, store_code })+ }++ pub(crate) fn from_raw(module: &'a Module, store_code: &'a StoreCode) -> ModuleWithCode<'a> {+ ModuleWithCode { module, store_code }+ }++ /// Provide the Module wrapped in this tuple.+ pub fn module(&self) -> &'a Module {+ self.module+ }++ /// Provide the StoreCode wrapped in this tuple.+ pub fn store_code(&self) -> &'a StoreCode {+ self.store_code+ }++ /// Returns an iterator over all functions defined within this module with+ /// their index and their raw pointer.+ #[inline]+ pub fn finished_functions(+ &self,+ ) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, &[u8])> + '_ {+ self.module+ .env_module()+ .defined_func_indices()+ .map(|i| (i, self.finished_function(i)))+ }++ /// Returns the slice in the text section of the function that+ /// `index` points to.+ #[inline]+ pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] {+ let range = self+ .module+ .compiled_module()+ .finished_function_range(def_func_index);+ &self.store_code.text()[range]+ }++ /// Get the array-to-Wasm trampoline for the function `index`+ /// points to, as a slice of raw code that can be converted to a+ /// callable function pointer.+ ///+ /// If the function `index` points to does not escape, then `None` is+ /// returned.+ ///+ /// These trampolines are used for array callers (e.g. `Func::new`)+ /// calling Wasm callees.+ pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> {+ let range = self+ .module+ .compiled_module()+ .array_to_wasm_trampoline_range(def_func_index)?;+ Some(&self.store_code.text()[range])+ }++ /// Get the text offset (relative PC) for a given absolute PC in+ /// this module.+ #[cfg(any(feature = "gc", feature = "debug"))]+ pub(crate) fn text_offset(&self, pc: usize) -> Option<u32> {+ StoreCodePC::offset_of(self.store_code.text_range(), pc)+ .map(|offset| u32::try_from(offset).expect("Module larger than 4GiB"))+ }++ /// Lookup the stack map at a program counter value.+ #[cfg(feature = "gc")]+ pub(crate) fn lookup_stack_map(&self, pc: usize) -> Option<wasmtime_environ::StackMap<'_>> {+ let text_offset = self.text_offset(pc)?;+ let info = self.module.engine_code().stack_map_data();+ wasmtime_environ::StackMap::lookup(text_offset, info)+ }+}diff --git a/crates/wasmtime/src/runtime/code_memory.rs b/crates/wasmtime/src/runtime/code_memory.rsindex 6777790b18..4659ec85f2 100644--- a/crates/wasmtime/src/runtime/code_memory.rs+++ b/crates/wasmtime/src/runtime/code_memory.rs@@ -441,6 +441,23 @@ impl CodeMemory {pub fn lookup_trap_code(&self, text_offset: usize) -> Option<Trap> {lookup_trap_code(self.trap_data(), text_offset)}++ /// Get the raw address range of this CodeMemory.+ pub(crate) fn raw_addr_range(&self) -> Range<usize> {+ let start = self.text().as_ptr().addr();+ let end = start + self.text().len();+ start..end+ }++ /// Create a "deep clone": a separate CodeMemory for the same code+ /// that can be patched or mutated independently. Also returns a+ /// "metadata and location" handle that can be registered with the+ /// global module registry and used for trap metadata lookups.+ #[cfg(feature = "debug")]+ pub(crate) fn deep_clone(self: &Arc<Self>, engine: &Engine) -> Result<CodeMemory> {+ let mmap = self.mmap.deep_clone()?;+ Self::new(engine, mmap)+ }}/// Returns the range of `inner` within `outer`, such that `outer[range]` is thediff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rsindex 2990248d9e..c4d5812723 100644--- a/crates/wasmtime/src/runtime/component/component.rs+++ b/crates/wasmtime/src/runtime/component/component.rs@@ -6,7 +6,7 @@ use crate::prelude::*;use crate::runtime::vm::open_file_for_mmap;use crate::runtime::vm::{CompiledModuleId, VMArrayCallFunction, VMFuncRef, VMWasmCallFunction};use crate::{- Engine, Module, ResourcesRequired, code::CodeObject, code_memory::CodeMemory,+ Engine, Module, ResourcesRequired, code::EngineCode, code_memory::CodeMemory,type_registry::TypeCollection,};use crate::{FuncType, ValType};@@ -81,7 +81,7 @@ struct ComponentInner {////// Note that the `Arc` here is used to share this allocation with internal/// modules.- code: Arc<CodeObject>,+ code: Arc<EngineCode>,/// Metadata produced during compilation.info: CompiledComponentInfo,@@ -430,10 +430,10 @@ impl Component {);types.canonicalize_for_runtime_usage(&mut |idx| signatures.shared_type(idx).unwrap());- // Assemble the `CodeObject` artifact which is shared by all core wasm+ // Assemble the `EngineCode` artifact which is shared by all core wasm// modules as well as the final component.let types = Arc::new(types);- let code = Arc::new(CodeObject::new(code_memory, signatures, types.into()));+ let code = Arc::new(EngineCode::new(code_memory, signatures, types.into()));// Convert all information about static core wasm modules into actual// `Module` instances by converting each `CompiledModuleInfo`, the@@ -496,17 +496,13 @@ impl Component {self.inner.code.signatures()}- pub(crate) fn text(&self) -> &[u8] {- self.inner.code.code_memory().text()- }-pub(crate) fn trampoline_ptrs(&self, index: TrampolineIndex) -> AllCallFuncPointers {let wasm_call = self- .func(FuncKey::ComponentTrampoline(Abi::Wasm, index))+ .store_invariant_func(FuncKey::ComponentTrampoline(Abi::Wasm, index)).unwrap().cast();let array_call = self- .func(FuncKey::ComponentTrampoline(Abi::Array, index))+ .store_invariant_func(FuncKey::ComponentTrampoline(Abi::Array, index)).unwrap().cast();AllCallFuncPointers {@@ -520,10 +516,10 @@ impl Component {intrinsic: UnsafeIntrinsic,) -> Option<AllCallFuncPointers> {let wasm_call = self- .func(FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic))?+ .store_invariant_func(FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic))?.cast();let array_call = self- .func(FuncKey::UnsafeIntrinsic(Abi::Array, intrinsic))?+ .store_invariant_func(FuncKey::UnsafeIntrinsic(Abi::Array, intrinsic))?.cast();Some(AllCallFuncPointers {wasm_call,@@ -532,7 +528,11 @@ impl Component {}/// Look up a function in this component's text section by `FuncKey`.- fn func(&self, key: FuncKey) -> Option<NonNull<u8>> {+ ///+ /// This supports only `FuncKey`s that do not invoke Wasm code,+ /// i.e., code that is potentially Store-specific.+ fn store_invariant_func(&self, key: FuncKey) -> Option<NonNull<u8>> {+ assert!(key.is_store_invariant());let loc = self.inner.index.func_loc(key)?;Some(self.func_loc_to_pointer(loc))}@@ -540,14 +540,16 @@ impl Component {/// Given a function location within this component's text section, get a/// pointer to the function.///+ /// This works only for Store-invariant functions.+ ////// Panics on out-of-bounds function locations.fn func_loc_to_pointer(&self, loc: &FunctionLoc) -> NonNull<u8> {- let text = self.text();+ let text = self.engine_code().text();let trampoline = &text[loc.start as usize..][..loc.length as usize];NonNull::from(trampoline).cast()}- pub(crate) fn code_object(&self) -> &Arc<CodeObject> {+ pub(crate) fn engine_code(&self) -> &Arc<EngineCode> {&self.inner.code}@@ -560,7 +562,7 @@ impl Component {/// [`Module::serialize`]: crate::Module::serialize/// [`Module`]: crate::Modulepub fn serialize(&self) -> Result<Vec<u8>> {- Ok(self.code_object().code_memory().mmap().to_vec())+ Ok(self.engine_code().image().to_vec())}/// Creates a new `VMFuncRef` with all fields filled out for the destructor@@ -577,7 +579,7 @@ impl Component {// then this can't be called by the component, so it's ok to leave it// blank.let wasm_call = self- .func(FuncKey::ResourceDropTrampoline)+ .store_invariant_func(FuncKey::ResourceDropTrampoline).map(|f| f.cast().into());VMFuncRef {@@ -680,7 +682,10 @@ impl Component {/// For more information see/// [`Module::image_range`](crate::Module::image_range).pub fn image_range(&self) -> Range<*const u8> {- self.inner.code.code_memory().mmap().image_range()+ let range = self.inner.code.text_range();+ let start = range.start.raw() as *const u8;+ let end = range.end.raw() as *const u8;+ start..end}/// Force initialization of copy-on-write images to happen here-and-nowdiff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex 2d746ed171..07de7d8c76 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -2928,9 +2928,9 @@ impl Instance {log::trace!("creating new thread");let start_func_ty = FuncType::new(store.engine(), [ValType::I32], []);- let instance = self.id().get_mut(store.0);+ let (instance, registry) = self.id().get_mut_and_registry(store.0);let callee = instance- .index_runtime_func_table(start_func_table_idx, start_func_idx as u64)?+ .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?.ok_or_else(|| {anyhow!("the start function index points to an uninitialized function")})?;diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rsindex fb1e99a09c..a803e6040c 100644--- a/crates/wasmtime/src/runtime/component/instance.rs+++ b/crates/wasmtime/src/runtime/component/instance.rs@@ -632,7 +632,7 @@ where.store_data_mut().component_instance_mut(id).instance(item.instance);- let instance = store.instance_mut(id);+ let (instance, registry) = store.instance_and_module_registry_mut(id);let idx = match &item.item {ExportItem::Index(idx) => (*idx).into(),@@ -649,7 +649,7 @@ where};// SAFETY: the `store_id` owns this instance and all exports contained// within.- unsafe { instance.get_export_by_index_mut(store_id, idx) }+ unsafe { instance.get_export_by_index_mut(registry, store_id, idx) }}fn resource_tables<'a>(@@ -742,9 +742,9 @@ impl<'a> Instantiator<'a> {component: &'a Component,store: &mut StoreOpaque,imports: &'a Arc<PrimaryMap<RuntimeImportIndex, RuntimeImport>>,- ) -> Instantiator<'a> {+ ) -> Result<Instantiator<'a>> {let env_component = component.env_component();- store.modules_mut().register_component(component);+ store.register_component(component)?;let imported_resources: ImportedResources =PrimaryMap::with_capacity(env_component.imported_resources.len());@@ -757,12 +757,12 @@ impl<'a> Instantiator<'a> {);let id = store.store_data_mut().push_component_instance(instance);- Instantiator {+ Ok(Instantiator {component,imports,core_imports: OwnedImports::empty(),id,- }+ })}async fn run<T>(&mut self, store: &mut StoreContextMut<'_, T>) -> Result<()> {@@ -1165,7 +1165,7 @@ impl<T: 'static> InstancePre<T> {.engine().allocator().increment_component_instance_count()?;- let mut instantiator = Instantiator::new(&self.component, store.0, &self.imports);+ let mut instantiator = Instantiator::new(&self.component, store.0, &self.imports)?;instantiator.run(&mut store).await.map_err(|e| {store.engine()diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rsindex 762c031706..94817a52c8 100644--- a/crates/wasmtime/src/runtime/component/store.rs+++ b/crates/wasmtime/src/runtime/component/store.rs@@ -145,6 +145,25 @@ impl StoreComponentInstanceId {self.from_data_get_mut(store.store_data_mut())}+ /// Return a mutable `ComponentInstance` and a `ModuleRegistry`+ /// from the store.+ ///+ /// # Panics+ ///+ /// Panics if `self` does not belong to `store`.+ #[cfg(feature = "component-model-async")]+ pub(crate) fn get_mut_and_registry<'a>(+ &self,+ store: &'a mut StoreOpaque,+ ) -> (+ Pin<&'a mut ComponentInstance>,+ &'a crate::module::ModuleRegistry,+ ) {+ let (store_data, registry) = store.store_data_mut_and_registry();+ let instance = self.from_data_get_mut(store_data);+ (instance, registry)+ }+/// Same as `get_mut`, but borrows less of a store.pub(crate) fn from_data_get_mut<'a>(&self,diff --git a/crates/wasmtime/src/runtime/debug.rs b/crates/wasmtime/src/runtime/debug.rsindex 70bcef9526..f6b4730bf4 100644--- a/crates/wasmtime/src/runtime/debug.rs+++ b/crates/wasmtime/src/runtime/debug.rs@@ -244,12 +244,11 @@ impl VirtualFrame {/// Return virtual frames corresponding to a physical frame, from/// outermost to innermost.fn decode(store: &mut StoreOpaque, frame: Frame, is_trapping_frame: bool) -> Vec<VirtualFrame> {- let module = store+ let (module_with_code, pc) = store.modules()- .lookup_module_by_pc(frame.pc())+ .module_and_code_by_pc(frame.pc()).expect("Wasm frame PC does not correspond to a module");- let base = module.code_object().code_memory().text().as_ptr() as usize;- let pc = frame.pc().wrapping_sub(base);+ let module = module_with_code.module();let table = module.frame_table().unwrap();let pc = u32::try_from(pc).expect("PC offset too large");let pos = if is_trapping_frame {diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rsindex eaca5ccf46..7d524184e8 100644--- a/crates/wasmtime/src/runtime/externals/table.rs+++ b/crates/wasmtime/src/runtime/externals/table.rs@@ -154,10 +154,11 @@ impl Table {lazy_init_range: impl IntoIterator<Item = u64>,) -> (&'a mut vm::Table, Option<&'a mut GcStore>) {self.instance.assert_belongs_to(store.id());- let (store, instance) = store.optional_gc_store_and_instance_mut(self.instance.instance());+ let (store, registry, instance) =+ store.optional_gc_store_and_registry_and_instance_mut(self.instance.instance());(- instance.get_defined_table_with_lazy_init(self.index, lazy_init_range),+ instance.get_defined_table_with_lazy_init(registry, self.index, lazy_init_range),store,)}diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rsindex 3445a4a401..5c4d6b2664 100644--- a/crates/wasmtime/src/runtime/instance.rs+++ b/crates/wasmtime/src/runtime/instance.rs@@ -231,7 +231,7 @@ impl Instance {// Note that under normal operation this shouldn't do much as the list// of funcs-with-holes should generally be empty. As a result the// process of filling this out is not super optimized at this point.- store.modules_mut().register_module(module);+ store.register_module(module)?;let (funcrefs, modules) = store.func_refs_and_modules();funcrefs.fill(modules);@@ -308,7 +308,7 @@ impl Instance {// Register the module just before instantiation to ensure we keep the module// properly referenced while in use by the store.- let module_id = store.modules_mut().register_module(module);+ let module_id = store.register_module(module)?;// The first thing we do is issue an instance allocation request// to the instance allocator. This, on success, will give us an@@ -371,10 +371,14 @@ impl Instance {// If a start function is present, invoke it. Make sure we use all the// trap-handling configuration in `store` as well.let store_id = store.0.id();- let mut instance = self.id.get_mut(store.0);+ let (mut instance, registry) = self.id.get_mut_and_module_registry(store.0);// SAFETY: the `store_id` is the id of the store that owns this// instance and any function stored within the instance.- let f = unsafe { instance.as_mut().get_exported_func(store_id, start) };+ let f = unsafe {+ instance+ .as_mut()+ .get_exported_func(registry, store_id, start)+ };let caller_vmctx = instance.vmctx();unsafe {let funcref = f.vm_func_ref(store.0);@@ -479,7 +483,10 @@ impl Instance {let id = store.id();// SAFETY: the store `id` owns this instance and all exports contained// within.- let export = unsafe { self.id.get_mut(store).get_export_by_index_mut(id, entity) };+ let export = unsafe {+ let (instance, registry) = self.id.get_mut_and_module_registry(store);+ instance.get_export_by_index_mut(registry, id, entity)+ };Extern::from_wasmtime_export(export, store)}@@ -922,7 +929,7 @@ fn pre_instantiate_raw() -> Result<OwnedImports> {// Register this module and use it to fill out any funcref wasm_call holes// we can. For more comments on this see `typecheck_externs`.- store.modules_mut().register_module(module);+ store.register_module(module)?;let (funcrefs, modules) = store.func_refs_and_modules();funcrefs.fill(modules);diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rsindex 5fb1130f09..cbb3f4022b 100644--- a/crates/wasmtime/src/runtime/instantiate.rs+++ b/crates/wasmtime/src/runtime/instantiate.rs@@ -3,10 +3,12 @@//! `CompiledModule` to allow compiling and instantiating to be done as separate//! steps.+use crate::code::EngineCode;use crate::prelude::*;-use crate::runtime::vm::{CompiledModuleId, MmapVec};-use crate::{code_memory::CodeMemory, profiling_agent::ProfilingAgent};+use crate::profiling_agent::ProfilingAgent;+use crate::runtime::vm::CompiledModuleId;use alloc::sync::Arc;+use core::ops::Range;use core::str;use wasmtime_environ::{CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, EntityRef, FilePos, FuncIndex,@@ -18,7 +20,7 @@ use wasmtime_environ::{pub struct CompiledModule {/// A unique ID used to register this module with the engine.unique_id: CompiledModuleId,- code_memory: Arc<CodeMemory>,+ engine_code: Arc<EngineCode>,module: Arc<Module>,meta: Metadata,index: Arc<CompiledFunctionsTable>,@@ -29,10 +31,11 @@ pub struct CompiledModule {impl CompiledModule {/// Creates `CompiledModule` directly from a precompiled artifact.///- /// The `code_memory` argument is expected to be the result of a previous- /// call to `ObjectBuilder::finish` above. This is an ELF image, at this- /// time, which contains all necessary information to create a- /// `CompiledModule` from a compilation.+ /// The `engine_code` argument is expected to be an EngineCode+ /// wrapper around a CodeMemory containing the result of a+ /// previous call to `ObjectBuilder::finish` above. This is an ELF+ /// image, at this time, which contains all necessary information+ /// to create a `CompiledModule` from a compilation.////// This method also takes `info`, an optionally-provided deserialization/// of the artifacts' compilation metadata section. If this information is@@ -44,14 +47,14 @@ impl CompiledModule {/// The `profiler` argument here is used to inform JIT profiling runtimes/// about new code that is loaded.pub fn from_artifacts(- code_memory: Arc<CodeMemory>,+ engine_code: Arc<EngineCode>,info: CompiledModuleInfo,index: Arc<CompiledFunctionsTable>,profiler: &dyn ProfilingAgent,) -> Result<Self> {let mut ret = Self {unique_id: CompiledModuleId::new(),- code_memory,+ engine_code,module: Arc::new(info.module),meta: info.meta,index,@@ -65,7 +68,8 @@ impl CompiledModule {fn register_profiling(&mut self, profiler: &dyn ProfilingAgent) -> Result<()> {// TODO-Bug?: "code_memory" is not exclusive for this module in the case of components,// so we may be registering the same code range multiple times here.- profiler.register_module(&self.code_memory.mmap()[..], &|addr| {++ profiler.register_module(self.engine_code.image(), &|addr| {let idx = self.func_by_text_offset(addr)?;let idx = self.module.func_index(idx);let name = self.func_name(idx)?;@@ -82,25 +86,6 @@ impl CompiledModule {self.unique_id}- /// Returns the underlying memory which contains the compiled module's- /// image.- pub fn mmap(&self) -> &MmapVec {- self.code_memory.mmap()- }-- /// Returns the underlying owned mmap of this compiled image.- pub fn code_memory(&self) -> &Arc<CodeMemory> {- &self.code_memory- }-- /// Returns the text section of the ELF image for this compiled module.- ///- /// This memory should have the read/execute permissions.- #[inline]- pub fn text(&self) -> &[u8] {- self.code_memory.text()- }-/// Return a reference-counting pointer to a module.pub fn module(&self) -> &Arc<Module> {&self.module@@ -121,43 +106,52 @@ impl CompiledModule {// `from_utf8_unchecked` if we really wanted since this section is// guaranteed to only have valid utf-8 data. Until it's a problem it's// probably best to double-check this though.- let data = self.code_memory().func_name_data();+ let data = self.engine_code.func_name_data();Some(str::from_utf8(&data[name.offset as usize..][..name.len as usize]).unwrap())}/// Returns an iterator over all functions defined within this module with- /// their index and their body in memory.+ /// their index and their offset in the underlying code image.#[inline]- pub fn finished_functions(+ pub fn finished_function_ranges(&self,- ) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, &[u8])> + '_ {+ ) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, Range<usize>)> + '_ {self.module.defined_func_indices()- .map(|i| (i, self.finished_function(i)))+ .map(|i| (i, self.finished_function_range(i)))}- /// Returns the body of the function that `index` points to.+ /// Returns the offset in the text section of the function that `index` points to.#[inline]- pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] {+ pub fn finished_function_range(&self, def_func_index: DefinedFuncIndex) -> Range<usize> {let loc = self.func_loc(def_func_index);- &self.text()[loc.start as usize..][..loc.length as usize]+ let start = usize::try_from(loc.start).unwrap();+ let end = usize::try_from(loc.start + loc.length).unwrap();+ start..end}- /// Get the array-to-Wasm trampoline for the function `index` points to.+ /// Get the array-to-Wasm trampoline for the function `index`+ /// points to, as a range in the text segment.////// If the function `index` points to does not escape, then `None` is/// returned.////// These trampolines are used for array callers (e.g. `Func::new`)/// calling Wasm callees.- pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> {+ pub fn array_to_wasm_trampoline_range(+ &self,+ def_func_index: DefinedFuncIndex,+ ) -> Option<Range<usize>> {assert!(def_func_index.index() < self.module.num_defined_funcs());let key = FuncKey::ArrayToWasmTrampoline(self.module_index(), def_func_index);let loc = self.index.func_loc(key)?;- Some(&self.text()[loc.start as usize..][..loc.length as usize])+ let start = usize::try_from(loc.start).unwrap();+ let end = usize::try_from(loc.start + loc.length).unwrap();+ Some(start..end)}- /// Get the Wasm-to-array trampoline for the given signature.+ /// Get the Wasm-to-array trampoline for the given signature, as a+ /// range in the text segment.////// These trampolines are used for filling in/// `VMFuncRef::wasm_call` for `Func::wrap`-style host funcrefs@@ -165,7 +159,12 @@ impl CompiledModule {pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> Option<&[u8]> {let key = FuncKey::WasmToArrayTrampoline(signature);let loc = self.index.func_loc(key)?;- Some(&self.text()[loc.start as usize..][..loc.length as usize])+ let start = usize::try_from(loc.start).unwrap();+ let end = usize::try_from(loc.start + loc.length).unwrap();+ Some(+ self.engine_code+ .raw_wasm_to_array_trampoline_data(start..end),+ )}/// Lookups a defined function by a program counter value.@@ -229,7 +228,7 @@ impl CompiledModule {let (_, range) = &self.meta.dwarf[i];let start = range.start.try_into().ok()?;let end = range.end.try_into().ok()?;- self.code_memory().wasm_dwarf().get(start..end)+ self.engine_code.wasm_dwarf().get(start..end)}).unwrap_or(&[]);Ok(EndianSlice::new(data, gimli::LittleEndian))@@ -254,7 +253,7 @@ impl CompiledModule {/// If this function returns `false` then `lookup_file_pos` will always/// return `None`.pub fn has_address_map(&self) -> bool {- !self.code_memory.address_map_data().is_empty()+ !self.engine_code.address_map_data().is_empty()}}diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rsindex f498eca796..36366a5387 100644--- a/crates/wasmtime/src/runtime/module.rs+++ b/crates/wasmtime/src/runtime/module.rs@@ -5,7 +5,7 @@ use crate::runtime::vm::{CompiledModuleId, MmapVec, ModuleMemoryImages, VMWasmCause crate::sync::OnceLock;use crate::{Engine,- code::CodeObject,+ code::EngineCode,code_memory::CodeMemory,instantiate::CompiledModule,resources::ResourcesRequired,@@ -144,7 +144,7 @@ struct ModuleInner {/// Note that this `Arc` is used to share information between compiled/// modules within a component. For bare core wasm modules created with/// `Module::new`, for example, this is a uniquely owned `Arc`.- code: Arc<CodeObject>,+ code: Arc<EngineCode>,/// A set of initialization images for memories, if any.///@@ -518,26 +518,21 @@ impl Module {let signatures =engine.register_and_canonicalize_types(&mut types, core::iter::once(&mut info.module));- // Package up all our data into a `CodeObject` and delegate to the final+ // Package up all our data into an `EngineCode` and delegate to the final// step of module compilation.- let code = Arc::new(CodeObject::new(code_memory, signatures, types.into()));+ let code = Arc::new(EngineCode::new(code_memory, signatures, types.into()));let index = Arc::new(index);Module::from_parts_raw(engine, code, info, index, true)}pub(crate) fn from_parts_raw(engine: &Engine,- code: Arc<CodeObject>,+ code: Arc<EngineCode>,info: CompiledModuleInfo,index: Arc<CompiledFunctionsTable>,serializable: bool,) -> Result<Self> {- let module = CompiledModule::from_artifacts(- code.code_memory().clone(),- info,- index,- engine.profiler(),- )?;+ let module = CompiledModule::from_artifacts(code.clone(), info, index, engine.profiler())?;// Validate the module can be used with the current instance allocator.let offsets = VMOffsets::new(HostPtr, module.module());@@ -636,14 +631,14 @@ impl Module {if !self.inner.serializable {bail!("cannot serialize a module exported from a component");}- Ok(self.compiled_module().mmap().to_vec())+ Ok(self.engine_code().image().to_vec())}pub(crate) fn compiled_module(&self) -> &CompiledModule {&self.inner.module}- pub(crate) fn code_object(&self) -> &Arc<CodeObject> {+ pub(crate) fn engine_code(&self) -> &Arc<EngineCode> {&self.inner.code}@@ -973,8 +968,14 @@ impl Module {////// It is not safe to modify the memory in this range, nor is it safe to/// modify the protections of memory in this range.+ ///+ /// Note that depending on the engine configuration, this image+ /// range may not actually be the code that is directly executed.pub fn image_range(&self) -> Range<*const u8> {- self.compiled_module().mmap().image_range()+ let range = self.engine_code().text_range();+ let start = range.start.raw() as *const u8;+ let end = range.end.raw() as *const u8;+ start..end}/// Force initialization of copy-on-write images to happen here-and-now@@ -1043,17 +1044,15 @@ impl Module {/// ```pub fn address_map<'a>(&'a self) -> Option<impl Iterator<Item = (usize, Option<u32>)> + 'a> {Some(- wasmtime_environ::iterate_address_map(- self.code_object().code_memory().address_map_data(),- )?- .map(|(offset, file_pos)| (offset as usize, file_pos.file_offset())),+ wasmtime_environ::iterate_address_map(self.engine_code().address_map_data())?+ .map(|(offset, file_pos)| (offset as usize, file_pos.file_offset())),)}/// Get this module's code object's `.text` section, containing its compiled/// executable code.pub fn text(&self) -> &[u8] {- self.code_object().code_memory().text()+ self.engine_code().text()}/// Get information about functions in this module's `.text` section: their@@ -1062,7 +1061,7 @@ impl Module {/// Results are yielded in a ModuleFunction struct.pub fn functions<'a>(&'a self) -> impl ExactSizeIterator<Item = ModuleFunction> + 'a {let module = self.compiled_module();- module.finished_functions().map(|(idx, _)| {+ self.env_module().defined_func_indices().map(|idx| {let loc = module.func_loc(idx);let idx = module.module().func_index(idx);ModuleFunction {@@ -1084,6 +1083,11 @@ impl Module {/// Return the address, in memory, of the trampoline that allows Wasm to/// call a array function of the given signature.+ ///+ /// Note that unlike all other code-pointer-returning functions,+ /// this *can* be present on `Module` (without a `StoreCode`)+ /// because we can execute the `EngineCode` for trampolines that+ /// leave the store to call the host.pub(crate) fn wasm_to_array_trampoline(&self,signature: VMSharedTypeIndex,@@ -1130,25 +1134,10 @@ impl Module {Ok(images)}- /// Get the text offset (relative PC) for a given absolute PC in- /// this module.- #[cfg(feature = "gc")]- pub(crate) fn text_offset(&self, pc: usize) -> u32 {- u32::try_from(pc - self.inner.module.text().as_ptr() as usize).unwrap()- }-- /// Lookup the stack map at a program counter value.- #[cfg(feature = "gc")]- pub(crate) fn lookup_stack_map(&self, pc: usize) -> Option<wasmtime_environ::StackMap<'_>> {- let text_offset = self.text_offset(pc);- let info = self.inner.code.code_memory().stack_map_data();- wasmtime_environ::StackMap::lookup(text_offset, info)- }-/// Obtain an exception-table parser on this module's exception metadata.#[cfg(feature = "gc")]pub(crate) fn exception_table<'a>(&'a self) -> ExceptionTable<'a> {- ExceptionTable::parse(self.inner.code.code_memory().exception_tables())+ ExceptionTable::parse(self.inner.code.exception_tables()).expect("Exception tables were validated on module load")}@@ -1156,7 +1145,7 @@ impl Module {/// (debug instrumentation) metadata.#[cfg(feature = "debug")]pub(crate) fn frame_table<'a>(&'a self) -> Option<FrameTable<'a>> {- let data = self.inner.code.code_memory().frame_tables();+ let data = self.inner.code.frame_tables();if data.is_empty() {None} else {@@ -1213,7 +1202,7 @@ fn memory_images(inner: &Arc<ModuleInner>) -> Result<Option<ModuleMemoryImages>>ModuleMemoryImages::new(&inner.engine,inner.module.module(),- inner.code.code_memory(),+ inner.code.module_memory_image_source(),)}diff --git a/crates/wasmtime/src/runtime/module/registry.rs b/crates/wasmtime/src/runtime/module/registry.rsindex a7836b4e3b..eba6e9a833 100644--- a/crates/wasmtime/src/runtime/module/registry.rs+++ b/crates/wasmtime/src/runtime/module/registry.rs@@ -1,173 +1,198 @@//! Implements a registry of modules for a store.-use crate::code::CodeObject;+use crate::code::{EngineCode, EngineCodePC, ModuleWithCode, StoreCode, StoreCodePC};#[cfg(feature = "component-model")]use crate::component::Component;-use crate::prelude::*;use crate::runtime::vm::VMWasmCallFunction;use crate::sync::{OnceLock, RwLock};+use crate::vm::CompiledModuleId;+use crate::{Engine, prelude::*};use crate::{FrameInfo, Module, code_memory::CodeMemory};use alloc::collections::btree_map::{BTreeMap, Entry};use alloc::sync::Arc;+use core::ops::Range;use core::ptr::NonNull;use wasmtime_environ::VMSharedTypeIndex;/// Used for registering modules with a store.///-/// Note that the primary reason for this registry is to ensure that everything-/// in `Module` is kept alive for the duration of a `Store`. At this time we-/// need "basically everything" within a `Module` to stay alive once it's-/// instantiated within a store. While there's some smaller portions that could-/// theoretically be omitted as they're not needed by the store they're-/// currently small enough to not worry much about.+/// There are two basic purposes that this registry serves:+///+/// - It keeps all modules and their metadata alive as long as the+/// store exists.+/// - It owns the [`StoreCode`], i.e. possibly-private-copy of machine+/// code, for all modules that execute in this store.+///+/// The registry allows for translation of EngineCode to StoreCode,+/// deduplicating by the start address of the EngineCode; and allows+/// for looking up modules by "registered module ID", and looking up+/// StoreCode and Modules by PC.+///+/// Note that multiple modules may be backed by a single+/// `StoreCode`. This is specifically the case for components in+/// general. When a component is first instantiated, the component+/// itself is registered (which loads the StoreCode into the+/// registry), then each individual module within that component is+/// registered and added to the data structures.+///+/// A brief overview of the kinds of compiled object and their+/// relationships:+///+/// - `Module` is a Wasm module. It owns a `CompiledModule`.+/// - `CompiledModule` contains metadata about the module (e.g., a map+/// from Wasm function indices to locations in the machine code),+/// and also owns an `EngineCode`.+/// - `EngineCode` holds an `Arc` to a `CodeMemory` with the canonical+/// copy of machine code, as well as some lower-level metadata+/// (signatures and types). It is instantiated by this registry into+/// `StoreCode`.+/// - `StoreCode` owns either another `Arc` to the same `CodeMemory`+/// as `EngineCode`, or if guest debugging is enabled and causes us+/// to clone private copies of code for patching per store, owns its+/// own private `CodeMemory` at a different address.+/// - Instances hold a `RegisteredModuleId` to be able to look up their modules.#[derive(Default)]pub struct ModuleRegistry {- // Keyed by the end address of a `CodeObject`.- //- // The value here is the start address and the information about what's- // loaded at that address.- loaded_code: BTreeMap<usize, (usize, LoadedCode)>,-- // Preserved for keeping data segments alive or similar- modules_without_code: Vec<Module>,+ /// StoreCode and Modules associated with it.+ ///+ /// Keyed by the start address of the `StoreCode`. We maintain the+ /// invariant of no overlaps on insertion. We use a range query to+ /// find the StoreCode for a given PC: take the range `0..=pc`,+ /// then take the last element of the range. That picks the+ /// highest start address <= the query, and we can check whether+ /// it contains the address.+ loaded_code: BTreeMap<StoreCodePC, LoadedCode>,++ /// Map from EngineCodePC start to StoreCodePC start. We use this+ /// to memoize the store-code creation process: each EngineCode is+ /// instantiated to a StoreCode only once per store.+ store_code: BTreeMap<EngineCodePC, StoreCodePC>,++ /// Modules instantiated in this registry.+ ///+ /// Every module is placed in this map, but not every module will+ /// be in a LoadedCode entry, because the module may have no text.+ modules: BTreeMap<RegisteredModuleId, Module>,}struct LoadedCode {- /// Kept alive here in the store to have a strong reference to keep the- /// relevant code mapped while the store is alive.- _code: Arc<CodeObject>,-- /// Modules found within `self.code`, keyed by start address here of the- /// address of the first function in the module.- modules: BTreeMap<usize, Module>,+ /// The StoreCode in this range.+ code: StoreCode,- /// These modules have no functions inside of them but they can still be- /// used for trampoline lookup.- modules_with_only_trampolines: Vec<Module>,+ /// Map by starting text offset of Modules in this code region.+ modules: BTreeMap<usize, RegisteredModuleId>,}/// An identifier of a module that has previously been inserted into a/// `ModuleRegistry`.-#[derive(Clone, Copy, Debug)]-pub enum RegisteredModuleId {- /// Index into `ModuleRegistry::modules_without_code`.- WithoutCode(usize),- /// Start address of the module's code so that we can get it again via- /// `ModuleRegistry::lookup_module`.- LoadedCode(usize),+///+/// This is just a newtype around `CompiledModuleId`, which is unique+/// within the Engine.+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+pub struct RegisteredModuleId(CompiledModuleId);++fn assert_no_overlap(loaded_code: &BTreeMap<StoreCodePC, LoadedCode>, range: Range<StoreCodePC>) {+ if let Some((start, _)) = loaded_code.range(range.start..).next() {+ assert!(*start >= range.end);+ }+ if let Some((_, code)) = loaded_code.range(..range.end).next_back() {+ assert!(code.code.text_range().end <= range.start);+ }}impl ModuleRegistry {/// Get a previously-registered module by id.- pub fn lookup_module_by_id(&self, id: RegisteredModuleId) -> Option<&Module> {- match id {- RegisteredModuleId::WithoutCode(idx) => self.modules_without_code.get(idx),- RegisteredModuleId::LoadedCode(pc) => {- let (module, _) = self.module_and_offset(pc)?;- Some(module)- }- }- }-- /// Fetches a registered module given a program counter value.- #[cfg(any(feature = "gc", feature = "debug"))]- pub fn lookup_module_by_pc(&self, pc: usize) -> Option<&Module> {- let (module, _) = self.module_and_offset(pc)?;- Some(module)+ pub fn module_by_id(&self, id: RegisteredModuleId) -> Option<&Module> {+ self.modules.get(&id)}- fn code(&self, pc: usize) -> Option<(&LoadedCode, usize)> {- let (end, (start, code)) = self.loaded_code.range(pc..).next()?;- if pc < *start || *end < pc {- return None;- }- Some((code, pc - *start))+ /// Fetches a registered StoreCode and module and an offset within+ /// it given a program counter value.+ pub fn module_and_code_by_pc<'a>(&'a self, pc: usize) -> Option<(ModuleWithCode<'a>, usize)> {+ let (_, code) = self+ .loaded_code+ .range(..=StoreCodePC::from_raw(pc))+ .next_back()?;+ let offset = StoreCodePC::offset_of(code.code.text_range(), pc)?;+ let (_, module_id) = code.modules.range(..=offset).next_back()?;+ let module = self.modules.get(&module_id)?;+ Some((ModuleWithCode::from_raw(module, &code.code), offset))}- fn module_and_offset(&self, pc: usize) -> Option<(&Module, usize)> {- let (code, offset) = self.code(pc)?;- Some((code.module(pc)?, offset))+ /// Fetches the `StoreCode` for a given `EngineCode`.+ pub fn store_code(&self, engine_code: &EngineCode) -> Option<&StoreCode> {+ let store_code_pc = *self.store_code.get(&engine_code.text_range().start)?;+ let (_, code) = self.loaded_code.range(store_code_pc..).next()?;+ Some(&code.code)}/// Gets an iterator over all modules in the registry.#[cfg(feature = "coredump")]pub fn all_modules(&self) -> impl Iterator<Item = &'_ Module> + '_ {- self.loaded_code- .values()- .flat_map(|(_, code)| code.modules.values())- .chain(self.modules_without_code.iter())+ self.modules.values()}/// Registers a new module with the registry.- pub fn register_module(&mut self, module: &Module) -> RegisteredModuleId {- self.register(module.code_object(), Some(module)).unwrap()+ pub fn register_module(+ &mut self,+ module: &Module,+ engine: &Engine,+ ) -> Result<RegisteredModuleId> {+ self.register(module.id(), module.engine_code(), Some(module), engine)+ .map(|id| id.unwrap())}#[cfg(feature = "component-model")]- pub fn register_component(&mut self, component: &Component) {- self.register(component.code_object(), None);+ pub fn register_component(&mut self, component: &Component, engine: &Engine) -> Result<()> {+ self.register(component.id(), component.engine_code(), None, engine)?;+ Ok(())}/// Registers a new module with the registry.fn register(&mut self,- code: &Arc<CodeObject>,+ compiled_id: CompiledModuleId,+ code: &Arc<EngineCode>,module: Option<&Module>,- ) -> Option<RegisteredModuleId> {- let text = code.code_memory().text();-- // If there's not actually any functions in this module then we may- // still need to preserve it for its data segments. Instances of this- // module will hold a pointer to the data stored in the module itself,- // and for schemes that perform lazy initialization which could use the- // module in the future. For that reason we continue to register empty- // modules and retain them.- if text.is_empty() {- return module.map(|module| {- let id = RegisteredModuleId::WithoutCode(self.modules_without_code.len());- self.modules_without_code.push(module.clone());- id- });- }-- // The module code range is exclusive for end, so make it inclusive as- // it may be a valid PC value- let start_addr = text.as_ptr() as usize;- let end_addr = start_addr + text.len() - 1;- let id = module.map(|_| RegisteredModuleId::LoadedCode(start_addr));-- // If this module is already present in the registry then that means- // it's either an overlapping image, for example for two modules- // found within a component, or it's a second instantiation of the same- // module. Delegate to `push_module` to find out.- if let Some((other_start, prev)) = self.loaded_code.get_mut(&end_addr) {- assert_eq!(*other_start, start_addr);- if let Some(module) = module {- prev.push_module(module);+ engine: &Engine,+ ) -> Result<Option<RegisteredModuleId>> {+ // Register the module, if any.+ let id = module.map(|module| {+ let id = RegisteredModuleId(compiled_id);+ self.modules.entry(id).or_insert_with(|| module.clone());+ id+ });++ // Create a StoreCode if one does not already exist.+ let store_code_pc = match self.store_code.entry(code.text_range().start) {+ Entry::Vacant(v) => {+ let store_code = StoreCode::new(engine, code)?;+ let store_code_pc = store_code.text_range().start;+ assert_no_overlap(&self.loaded_code, store_code.text_range());+ self.loaded_code.insert(+ store_code_pc,+ LoadedCode {+ code: store_code,+ modules: BTreeMap::default(),+ },+ );+ *v.insert(store_code_pc)}- return id;- }+ Entry::Occupied(o) => *o.get(),+ };- // Assert that this module's code doesn't collide with any other- // registered modules- if let Some((_, (prev_start, _))) = self.loaded_code.range(start_addr..).next() {- assert!(*prev_start > end_addr);- }- if let Some((prev_end, _)) = self.loaded_code.range(..=start_addr).next_back() {- assert!(*prev_end < start_addr);+ // Add this module to the LoadedCode if not present.+ if let (Some(module), Some(id)) = (module, id) {+ if let Some((_, range)) = module.compiled_module().finished_function_ranges().next() {+ let loaded_code = self+ .loaded_code+ .get_mut(&store_code_pc)+ .expect("loaded_code must have entry for StoreCodePC");+ loaded_code.modules.insert(range.start, id);+ }}- let mut item = LoadedCode {- _code: code.clone(),- modules: Default::default(),- modules_with_only_trampolines: Vec::new(),- };- if let Some(module) = module {- item.push_module(module);- }- let prev = self.loaded_code.insert(end_addr, (start_addr, item));- assert!(prev.is_none());- id+ Ok(id)}/// Fetches frame information about a program counter in a backtrace.@@ -178,10 +203,23 @@ impl ModuleRegistry {/// debug information due to the compiler's configuration. The second/// boolean indicates whether the engine used to compile this module is/// using environment variables to control debuginfo parsing.- pub(crate) fn lookup_frame_info(&self, pc: usize) -> Option<(FrameInfo, &Module)> {- let (module, offset) = self.module_and_offset(pc)?;- let info = FrameInfo::new(module.clone(), offset)?;- Some((info, module))+ pub(crate) fn lookup_frame_info<'a>(+ &'a self,+ pc: usize,+ ) -> Option<(FrameInfo, ModuleWithCode<'a>)> {+ let (_, code) = self+ .loaded_code+ .range(..=StoreCodePC::from_raw(pc))+ .next_back()?;+ let text_offset = StoreCodePC::offset_of(code.code.text_range(), pc)?;+ let (_, module_id) = code.modules.range(..=text_offset).next_back()?;+ let module = self+ .modules+ .get(&module_id)+ .expect("referenced module ID not found");+ let info = FrameInfo::new(module.clone(), text_offset)?;+ let module_with_code = ModuleWithCode::from_raw(module, &code.code);+ Some((info, module_with_code))}pub fn wasm_to_array_trampoline(@@ -195,59 +233,15 @@ impl ModuleRegistry {// `VMSharedSignatureIndex` to `SignatureIndex` map.//// See also the comment in `ModuleInner::wasm_to_native_trampoline`.- for (_, code) in self.loaded_code.values() {- for module in code- .modules- .values()- .chain(&code.modules_with_only_trampolines)- {- if let Some(trampoline) = module.wasm_to_array_trampoline(sig) {- return Some(trampoline);- }+ for module in self.modules.values() {+ if let Some(trampoline) = module.wasm_to_array_trampoline(sig) {+ return Some(trampoline);}}None}}-impl LoadedCode {- fn push_module(&mut self, module: &Module) {- let func = match module.compiled_module().finished_functions().next() {- Some((_, func)) => func,- // There are no compiled functions in this module so there's no- // need to push onto `self.modules` which is only used for frame- // information lookup for a trap which only symbolicates defined- // functions.- None => {- self.modules_with_only_trampolines.push(module.clone());- return;- }- };- let start = func.as_ptr() as usize;-- match self.modules.entry(start) {- // This module is already present, and it should be the same as- // `module`.- Entry::Occupied(m) => {- debug_assert!(Arc::ptr_eq(&module.inner, &m.get().inner));- }- // This module was not already present, so now it's time to insert.- Entry::Vacant(v) => {- v.insert(module.clone());- }- }- }-- fn module(&self, pc: usize) -> Option<&Module> {- // The `modules` map is keyed on the start address of the first- // function in the module, so find the first module whose start address- // is less than the `pc`. That may be the wrong module but lookup- // within the module should fail in that case.- let (_start, module) = self.modules.range(..=pc).next_back()?;- Some(module)- }-}-// This is the global code registry that stores information for all loaded code// objects that are currently in use by any `Store` in the current process.//@@ -285,26 +279,24 @@ pub fn lookup_code(pc: usize) -> Option<(Arc<CodeMemory>, usize)> {/// This is required to enable traps to work correctly since the signal handler/// will lookup in the `GLOBAL_CODE` list to determine which a particular pc/// is a trap or not.-pub fn register_code(code: &Arc<CodeMemory>) {- let text = code.text();- if text.is_empty() {+pub fn register_code(image: &Arc<CodeMemory>, address: Range<usize>) {+ if address.is_empty() {return;}- let start = text.as_ptr() as usize;- let end = start + text.len() - 1;- let prev = global_code().write().insert(end, (start, code.clone()));+ let start = address.start;+ let end = address.end - 1;+ let prev = global_code().write().insert(end, (start, image.clone()));assert!(prev.is_none());}/// Unregisters a code mmap from the global map.////// Must have been previously registered with `register`.-pub fn unregister_code(code: &Arc<CodeMemory>) {- let text = code.text();- if text.is_empty() {+pub fn unregister_code(address: Range<usize>) {+ if address.is_empty() {return;}- let end = (text.as_ptr() as usize) + text.len() - 1;+ let end = address.end - 1;let code = global_code().write().remove(&end);assert!(code.is_some());}@@ -332,12 +324,12 @@ fn test_frame_info() -> Result<(), anyhow::Error> {// Create an instance to ensure the frame information is registered.Instance::new(&mut store, &module, &[])?;- for (i, alloc) in module.compiled_module().finished_functions() {- let (start, end) = {- let ptr = alloc.as_ptr();- let len = alloc.len();- (ptr as usize, ptr as usize + len)- };+ // Look for frame info for each function. Assume that StoreCode+ // does not actually clone in the default configuration.+ for (i, range) in module.compiled_module().finished_function_ranges() {+ let base = module.engine_code().text_range().start.raw();+ let start = base + range.start;+ let end = base + range.end;for pc in start..end {let (frame, _) = store.as_context()diff --git a/crates/wasmtime/src/runtime/profiling.rs b/crates/wasmtime/src/runtime/profiling.rsindex 2b183a68b7..6baac6064a 100644--- a/crates/wasmtime/src/runtime/profiling.rs+++ b/crates/wasmtime/src/runtime/profiling.rs@@ -1,9 +1,8 @@#[cfg(feature = "component-model")]use crate::component::Component;-use crate::instantiate::CompiledModule;-use crate::prelude::*;use crate::runtime::vm::Backtrace;use crate::{AsContext, CallHook, Module};+use crate::{Engine, prelude::*};use core::cmp::Ordering;use fxprof_processed_profile::debugid::DebugId;use fxprof_processed_profile::{@@ -109,10 +108,20 @@ impl GuestProfiler {/// "Security" section of the [`GuestProfiler`] documentation for guidance/// on what modules should not be included in this list.pub fn new(+ engine: &Engine,module_name: &str,interval: Duration,modules: impl IntoIterator<Item = (String, Module)>,- ) -> Self {+ ) -> Result<Self> {+ // Check that guest debugging is not enabled. The+ // instrumentation would make profiling results unreliable,+ // but more fundamentally, it means that code is cloned per+ // instantiation (for breakpoint patching) so the logic below+ // is incorrect.+ if engine.tunables().debug_guest {+ anyhow::bail!("Profiling cannot be performed when guest-debugging is enabled.");+ }+let zero = ReferenceTimestamp::from_millis_since_unix_epoch(0.0);let mut profile = Profile::new(module_name, zero, interval.into());@@ -121,18 +130,21 @@ impl GuestProfiler {let mut modules: Vec<_> = modules.into_iter().filter_map(|(name, module)| {+ assert!(Engine::same(module.engine(), engine));let compiled = module.compiled_module();let text_range = {// Assumption: within text, the code for a given module is packed linearly and// is non-overlapping; if this is violated, it should be safe but might result// in incorrect profiling results.- let start =- compiled.finished_functions().next()?.1.as_ptr_range().start as usize;- let end = compiled.finished_functions().last()?.1.as_ptr_range().end as usize;+ let start = compiled.finished_function_ranges().next()?.1.start;+ let end = compiled.finished_function_ranges().last()?.1.end;++ let start = (module.engine_code().text_range().start + start).raw();+ let end = (module.engine_code().text_range().start + end).raw();start..end};- module_symbols(name, compiled).map(|lib| {+ module_symbols(name, &module).map(|lib| {let libhandle = profile.add_lib(lib);ProfiledModule {module,@@ -150,14 +162,14 @@ impl GuestProfiler {let thread = profile.add_thread(process, 0, Timestamp::from_nanos_since_reference(0), true);let start = Instant::now();let marker = CallMarker::new(&mut profile);- Self {+ Ok(Self {profile,modules,process,thread,start,marker,- }+ })}/// Create a new profiler for the provided component@@ -167,16 +179,17 @@ impl GuestProfiler {/// instrumentation to track calls in each of its constituent modules.#[cfg(feature = "component-model")]pub fn new_component(+ engine: &Engine,component_name: &str,interval: Duration,component: Component,extra_modules: impl IntoIterator<Item = (String, Module)>,- ) -> Self {+ ) -> Result<Self> {let modules = component.static_modules().map(|m| (m.name().unwrap_or("<unknown>").to_string(), m.clone())).chain(extra_modules);- Self::new(component_name, interval, modules)+ Self::new(engine, component_name, interval, modules)}/// Add a sample to the profile. This function collects a backtrace from@@ -248,23 +261,29 @@ impl GuestProfiler {}}-fn module_symbols(name: String, compiled: &CompiledModule) -> Option<LibraryInfo> {- let symbols = Vec::from_iter(compiled.finished_functions().map(|(defined_idx, _)| {- let loc = compiled.func_loc(defined_idx);- let func_idx = compiled.module().func_index(defined_idx);- let mut name = String::new();- demangle_function_name_or_index(- &mut name,- compiled.func_name(func_idx),- defined_idx.as_u32() as usize,- )- .unwrap();- Symbol {- address: loc.start,- size: Some(loc.length),- name,- }- }));+fn module_symbols(name: String, module: &Module) -> Option<LibraryInfo> {+ let compiled = module.compiled_module();+ let symbols = Vec::from_iter(+ module+ .env_module()+ .defined_func_indices()+ .map(|defined_idx| {+ let loc = compiled.func_loc(defined_idx);+ let func_idx = compiled.module().func_index(defined_idx);+ let mut name = String::new();+ demangle_function_name_or_index(+ &mut name,+ compiled.func_name(func_idx),+ defined_idx.as_u32() as usize,+ )+ .unwrap();+ Symbol {+ address: loc.start,+ size: Some(loc.length),+ name,+ }+ }),+ );if symbols.is_empty() {return None;}diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 710ff8ca68..f2f00b7fd2 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -1616,14 +1616,25 @@ impl StoreOpaque {&mut self.store_data}+ pub fn store_data_mut_and_registry(&mut self) -> (&mut StoreData, &ModuleRegistry) {+ (&mut self.store_data, &self.modules)+ }+#[inline]pub(crate) fn modules(&self) -> &ModuleRegistry {&self.modules}- #[inline]- pub(crate) fn modules_mut(&mut self) -> &mut ModuleRegistry {- &mut self.modules+ pub(crate) fn register_module(&mut self, module: &Module) -> Result<RegisteredModuleId> {+ self.modules.register_module(module, &self.engine)+ }++ #[cfg(feature = "component-model")]+ pub(crate) fn register_component(+ &mut self,+ component: &crate::component::Component,+ ) -> Result<()> {+ self.modules.register_component(component, &self.engine)}pub(crate) fn func_refs_and_modules(&mut self) -> (&mut FuncRefs, &ModuleRegistry) {@@ -1649,7 +1660,7 @@ impl StoreOpaque {StoreInstanceKind::Real { module_id } => {let module = self.modules()- .lookup_module_by_id(module_id)+ .module_by_id(module_id).expect("should always have a registered module for real instances");Some(module)}@@ -1676,6 +1687,16 @@ impl StoreOpaque {self.instances[id].handle.get_mut()}+ /// Accessor from `InstanceId` to both `Pin<&mut vm::Instance>`+ /// and `&ModuleRegistry`.+ #[inline]+ pub fn instance_and_module_registry_mut(+ &mut self,+ id: InstanceId,+ ) -> (Pin<&mut vm::Instance>, &ModuleRegistry) {+ (self.instances[id].handle.get_mut(), &self.modules)+ }+/// Access multiple instances specified via `ids`.////// # Panics@@ -1710,6 +1731,23 @@ impl StoreOpaque {(self.gc_store.as_mut(), self.instances[id].handle.get_mut())}+ /// Tuple of `Self::optional_gc_store_mut`, `Self::modules`, and+ /// `Self::instance_mut`.+ pub fn optional_gc_store_and_registry_and_instance_mut(+ &mut self,+ id: InstanceId,+ ) -> (+ Option<&mut GcStore>,+ &ModuleRegistry,+ Pin<&mut vm::Instance>,+ ) {+ (+ self.gc_store.as_mut(),+ &self.modules,+ self.instances[id].handle.get_mut(),+ )+ }+/// Get all instances (ignoring dummy instances) within this store.pub fn all_instances<'a>(&'a mut self) -> impl ExactSizeIterator<Item = Instance> + 'a {let instances = self@@ -2036,12 +2074,12 @@ impl StoreOpaque {"we should always get a valid frame pointer for Wasm frames");- let module_info = self+ let (module_with_code, _offset) = self.modules()- .lookup_module_by_pc(pc)+ .module_and_code_by_pc(pc).expect("should have module info for Wasm frame");- if let Some(stack_map) = module_info.lookup_stack_map(pc) {+ if let Some(stack_map) = module_with_code.lookup_stack_map(pc) {log::trace!("We have a stack map that maps {} bytes in this Wasm frame",stack_map.frame_size()@@ -2056,8 +2094,10 @@ impl StoreOpaque {}#[cfg(feature = "debug")]- if let Some(frame_table) = module_info.frame_table() {- let relpc = module_info.text_offset(pc);+ if let Some(frame_table) = module_with_code.module().frame_table() {+ let relpc = module_with_code+ .text_offset(pc)+ .expect("PC should be within module");for stack_slot in super::debug::gc_refs_in_frame(frame_table, relpc, fp) {unsafe {self.trace_wasm_stack_slot(gc_roots_list, stack_slot);diff --git a/crates/wasmtime/src/runtime/store/data.rs b/crates/wasmtime/src/runtime/store/data.rsindex 71599f8794..265e7a1ce8 100644--- a/crates/wasmtime/src/runtime/store/data.rs+++ b/crates/wasmtime/src/runtime/store/data.rs@@ -1,3 +1,4 @@+use crate::module::ModuleRegistry;use crate::runtime::vm::{self, GcStore, VMStore};use crate::store::StoreOpaque;use crate::{StoreContext, StoreContextMut};@@ -253,6 +254,21 @@ impl StoreInstanceId {store.instance_mut(self.instance)}+ /// Get both an instance handle and a borrow to the module+ /// registry in the store.+ ///+ /// # Panics+ ///+ /// Panics if `self` does not belong to `store`.+ #[inline]+ pub(crate) fn get_mut_and_module_registry<'a>(+ &self,+ store: &'a mut StoreOpaque,+ ) -> (Pin<&'a mut vm::Instance>, &'a ModuleRegistry) {+ self.assert_belongs_to(store.id());+ store.instance_and_module_registry_mut(self.instance)+ }+/// Same as [`Self::get_mut`], but also returns the `GcStore`.#[inline]pub(crate) fn get_with_gc_store_mut<'a>(diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rsindex 3089e4c4ef..fbe103cb2d 100644--- a/crates/wasmtime/src/runtime/trap.rs+++ b/crates/wasmtime/src/runtime/trap.rs@@ -325,7 +325,8 @@ impl WasmBacktrace {// do this then we will print out a helpful note in// `Display` to indicate that more detailed information// in a trap may be available.- let has_unparsed_debuginfo = module.compiled_module().has_unparsed_debuginfo();+ let has_unparsed_debuginfo =+ module.module().compiled_module().has_unparsed_debuginfo();if has_unparsed_debuginfo&& wasm_backtrace_details_env_used&& cfg!(feature = "addr2line")@@ -439,10 +440,8 @@ impl FrameInfo {let compiled_module = module.compiled_module();let index = compiled_module.func_by_text_offset(text_offset)?;let func_start = compiled_module.func_start_srcloc(index);- let instr = wasmtime_environ::lookup_file_pos(- compiled_module.code_memory().address_map_data(),- text_offset,- );+ let instr =+ wasmtime_environ::lookup_file_pos(module.engine_code().address_map_data(), text_offset);let index = compiled_module.module().func_index(index);let func_index = index.as_u32();let func_name = compiled_module.func_name(index).map(|s| s.to_string());diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex 247cea7ac1..a7c597affe 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -44,9 +44,7 @@ use core::pin::pin;use core::ptr::NonNull;use core::sync::atomic::{AtomicUsize, Ordering};use core::task::{Context, Poll, Waker};-use wasmtime_environ::{- DefinedFuncIndex, DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex,-};+use wasmtime_environ::{DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex};#[cfg(feature = "gc")]use wasmtime_environ::ModuleInternedTypeIndex;@@ -124,10 +122,12 @@ pub use crate::runtime::vm::table::{Table, TableElementType};#[cfg(feature = "gc")]pub use crate::runtime::vm::throw::*;pub use crate::runtime::vm::traphandlers::*;+#[cfg(feature = "component-model")]+pub use crate::runtime::vm::vmcontext::VMArrayCallFunction;pub use crate::runtime::vm::vmcontext::{- VMArrayCallFunction, VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport,- VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport,- VMOpaqueContext, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,+ VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition,+ VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext,+ VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,};#[cfg(has_custom_sync)]pub(crate) use sys::capi;@@ -344,7 +344,7 @@ impl ModuleRuntimeInfo {fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {match self {ModuleRuntimeInfo::Module(m) => m- .code_object()+ .engine_code().signatures().shared_type(module_index).expect("bad module-level interned type index"),@@ -352,38 +352,6 @@ impl ModuleRuntimeInfo {}}- /// Returns the address, in memory, that the function `index` resides at.- fn function(&self, index: DefinedFuncIndex) -> NonNull<VMWasmCallFunction> {- let module = match self {- ModuleRuntimeInfo::Module(m) => m,- ModuleRuntimeInfo::Bare(_) => unreachable!(),- };- let ptr = module- .compiled_module()- .finished_function(index)- .as_ptr()- .cast::<VMWasmCallFunction>()- .cast_mut();- NonNull::new(ptr).unwrap()- }-- /// Returns the address, in memory, of the trampoline that allows the given- /// defined Wasm function to be called by the array calling convention.- ///- /// Returns `None` for Wasm functions which do not escape, and therefore are- /// not callable from outside the Wasm module itself.- fn array_to_wasm_trampoline(- &self,- index: DefinedFuncIndex,- ) -> Option<NonNull<VMArrayCallFunction>> {- let m = match self {- ModuleRuntimeInfo::Module(m) => m,- ModuleRuntimeInfo::Bare(_) => unreachable!(),- };- let ptr = NonNull::from(m.compiled_module().array_to_wasm_trampoline(index)?);- Some(ptr.cast())- }-/// Returns the `MemoryImage` structure used for copy-on-write/// initialization of the memory, if it's applicable.fn memory_image(@@ -413,7 +381,7 @@ impl ModuleRuntimeInfo {/// A slice pointing to all data that is referenced by this instance.fn wasm_data(&self) -> &[u8] {match self {- ModuleRuntimeInfo::Module(m) => m.compiled_module().code_memory().wasm_data(),+ ModuleRuntimeInfo::Module(m) => m.engine_code().wasm_data(),ModuleRuntimeInfo::Bare(_) => &[],}}@@ -423,7 +391,7 @@ impl ModuleRuntimeInfo {fn type_ids(&self) -> &[VMSharedTypeIndex] {match self {ModuleRuntimeInfo::Module(m) => m- .code_object()+ .engine_code().signatures().as_module_map().values()diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rsindex 94a0889c22..07148acac3 100644--- a/crates/wasmtime/src/runtime/vm/component.rs+++ b/crates/wasmtime/src/runtime/vm/component.rs@@ -7,6 +7,7 @@//! cranelift-compiled adapters, will use this `VMComponentContext` as well.use crate::component::{Component, Instance, InstancePre, ResourceType, RuntimeImport};+use crate::module::ModuleRegistry;use crate::runtime::component::ComponentInstanceId;use crate::runtime::vm::instance::{InstanceLayout, OwnedInstance, OwnedVMContext};use crate::runtime::vm::vmcontext::VMFunctionBody;@@ -368,6 +369,7 @@ impl ComponentInstance {/// Returns the `Func` at index `func_idx` in the funcref table at `table_idx`.pub fn index_runtime_func_table(&self,+ registry: &ModuleRegistry,table_idx: RuntimeTableIndex,func_idx: u64,) -> Result<Option<Func>> {@@ -381,7 +383,8 @@ impl ComponentInstance {// SAFETY: We just constructed `instance_ptr` from a valid pointer. This pointer won't leave// this call, so we don't need a lifetime to bind it to.let instance = Pin::new_unchecked(instance_ptr.as_mut());- let table = instance.get_defined_table_with_lazy_init(table.index, [func_idx]);+ let table =+ instance.get_defined_table_with_lazy_init(registry, table.index, [func_idx]);let func = table.get_func(func_idx)?.map(|funcref| Func::from_vm_func_ref(store.id(), funcref));diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rsindex ae4cd0d638..4716588088 100644--- a/crates/wasmtime/src/runtime/vm/const_expr.rs+++ b/crates/wasmtime/src/runtime/vm/const_expr.rs@@ -51,13 +51,10 @@ impl ConstEvalContext {fn ref_func(&mut self, store: &mut StoreOpaque, index: FuncIndex) -> Result<Val> {let id = store.id();+ let (instance, registry) = store.instance_and_module_registry_mut(self.instance);// SAFETY: `id` is the correct store-owner of the function being looked// up- let func = unsafe {- store- .instance_mut(self.instance)- .get_exported_func(id, index)- };+ let func = unsafe { instance.get_exported_func(registry, id, index) };Ok(func.into())}diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rsindex dedf27b79e..38b9df44bb 100644--- a/crates/wasmtime/src/runtime/vm/instance.rs+++ b/crates/wasmtime/src/runtime/vm/instance.rs@@ -3,6 +3,8 @@//! `InstanceHandle` is a reference-counting handle for an `Instance`.use crate::OpaqueRootScope;+use crate::code::ModuleWithCode;+use crate::module::ModuleRegistry;use crate::prelude::*;use crate::runtime::vm::const_expr::{ConstEvalContext, ConstExprEvaluator};use crate::runtime::vm::export::{Export, ExportMemory};@@ -18,6 +20,7 @@ use crate::runtime::vm::{VMStoreRawPtr, VmPtr, VmSafe, WasmFault, catch_unwind_and_record_trap,};use crate::store::{InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter};+use crate::vm::VMWasmCallFunction;use alloc::sync::Arc;use core::alloc::Layout;use core::marker;@@ -346,7 +349,6 @@ impl Instance {self.runtime_info.env_module()}- #[cfg(any(feature = "gc", feature = "debug"))]pub(crate) fn runtime_module(&self) -> Option<&crate::Module> {match &self.runtime_info {ModuleRuntimeInfo::Module(m) => Some(m),@@ -550,10 +552,11 @@ impl Instance {/// functions that this instance can reference.pub unsafe fn get_exported_func(self: Pin<&mut Self>,+ registry: &ModuleRegistry,store: StoreId,index: FuncIndex,) -> crate::Func {- let func_ref = self.get_func_ref(index).unwrap();+ let func_ref = self.get_func_ref(registry, index).unwrap();// SAFETY: the validity of `func_ref` is guaranteed by the validity of// `self`, and the contract that `store` must own `func_ref` is a@@ -780,18 +783,38 @@ impl Instance {/// This functions requires that `into` is a valid pointer.unsafe fn construct_func_ref(self: Pin<&mut Self>,+ registry: &ModuleRegistry,index: FuncIndex,type_index: VMSharedTypeIndex,into: *mut VMFuncRef,) {+ let module_with_code = ModuleWithCode::in_store(+ registry,+ self.runtime_module()+ .expect("funcref impossible in fake module"),+ )+ .expect("module not in store");+let func_ref = if let Some(def_index) = self.env_module().defined_func_index(index) {VMFuncRef {- array_call: self- .runtime_info- .array_to_wasm_trampoline(def_index)- .expect("should have array-to-Wasm trampoline for escaping function")+ array_call: NonNull::from(+ module_with_code+ .array_to_wasm_trampoline(def_index)+ .expect("should have array-to-Wasm trampoline for escaping function"),+ )+ .cast()+ .into(),+ wasm_call: Some(+ NonNull::new(+ module_with_code+ .finished_function(def_index)+ .as_ptr()+ .cast::<VMWasmCallFunction>()+ .cast_mut(),+ )+ .unwrap().into(),- wasm_call: Some(self.runtime_info.function(def_index).into()),+ ),vmctx: VMOpaqueContext::from_vmcontext(self.vmctx()).into(),type_index,}@@ -820,6 +843,7 @@ impl Instance {/// be passed into JIT code.pub(crate) fn get_func_ref(self: Pin<&mut Self>,+ registry: &ModuleRegistry,index: FuncIndex,) -> Option<NonNull<VMFuncRef>> {if index == FuncIndex::reserved_value() {@@ -862,7 +886,7 @@ impl Instance {// SAFETY: the `func_ref` ptr should be valid as it's within our// `VMContext` area.unsafe {- self.construct_func_ref(index, sig, func_ref.as_ptr());+ self.construct_func_ref(registry, index, sig, func_ref.as_ptr());}Some(func_ref)@@ -978,13 +1002,11 @@ impl Instance {.and_then(|s| s.get(..len)).ok_or(Trap::TableOutOfBounds)?;for (i, func_idx) in positions.zip(elements) {+ let (instance, registry) =+ store.instance_and_module_registry_mut(elements_instance_id);// SAFETY: the `store_id` passed to `get_exported_func` is// indeed the store that owns the function.- let func = unsafe {- store- .instance_mut(elements_instance_id)- .get_exported_func(store_id, *func_idx)- };+ let func = unsafe { instance.get_exported_func(registry, store_id, *func_idx) };table.set_(store, i, func.into()).unwrap();}}@@ -1196,11 +1218,12 @@ impl Instance {/// works correctly.pub(crate) fn get_table_with_lazy_init(self: Pin<&mut Self>,+ registry: &ModuleRegistry,table_index: TableIndex,range: impl Iterator<Item = u64>,) -> &mut Table {let (idx, instance) = self.defined_table_index_and_instance(table_index);- instance.get_defined_table_with_lazy_init(idx, range)+ instance.get_defined_table_with_lazy_init(registry, idx, range)}/// Gets the raw runtime table data structure owned by this instance@@ -1209,6 +1232,7 @@ impl Instance {/// The `range` specified is eagerly initialized for funcref tables.pub fn get_defined_table_with_lazy_init(mut self: Pin<&mut Self>,+ registry: &ModuleRegistry,idx: DefinedTableIndex,range: impl IntoIterator<Item = u64>,) -> &mut Table {@@ -1241,8 +1265,8 @@ impl Instance {};// Panicking here helps catch bugs rather than silently truncating by accident.let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned();- let func_ref =- func_index.and_then(|func_index| self.as_mut().get_func_ref(func_index));+ let func_ref = func_index+ .and_then(|func_index| self.as_mut().get_func_ref(registry, func_index));self.as_mut().tables_mut()[idx].1.set_func(i, func_ref)@@ -1529,6 +1553,7 @@ impl Instance {/// instance.pub unsafe fn get_export_by_index_mut(self: Pin<&mut Self>,+ registry: &ModuleRegistry,store: StoreId,export: EntityIndex,) -> Export {@@ -1536,7 +1561,7 @@ impl Instance {// SAFETY: the contract of `store` owning the this instance is a// safety requirement of this function itself.EntityIndex::Function(i) => {- Export::Function(unsafe { self.get_exported_func(store, i) })+ Export::Function(unsafe { self.get_exported_func(registry, store, i) })}EntityIndex::Global(i) => Export::Global(self.get_exported_global(store, i)),EntityIndex::Table(i) => Export::Table(self.get_exported_table(store, i)),diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex 5c384f9b20..70d5856f43 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -562,9 +562,9 @@ fn memory_init(// Implementation of `ref.func`.fn ref_func(store: &mut dyn VMStore, instance: InstanceId, func_index: u32) -> NonNull<u8> {- store- .instance_mut(instance)- .get_func_ref(FuncIndex::from_u32(func_index))+ let (instance, registry) = store.instance_and_module_registry_mut(instance);+ instance+ .get_func_ref(registry, FuncIndex::from_u32(func_index)).expect("ref_func: funcref should always be available for given func index").cast()}@@ -583,9 +583,8 @@ fn table_get_lazy_init_func_ref(index: u64,) -> *mut u8 {let table_index = TableIndex::from_u32(table_index);- let table = store- .instance_mut(instance)- .get_table_with_lazy_init(table_index, core::iter::once(index));+ let (instance, registry) = store.instance_and_module_registry_mut(instance);+ let table = instance.get_table_with_lazy_init(registry, table_index, core::iter::once(index));let elem = table.get_func(index).expect("table access already bounds-checked");@@ -962,14 +961,14 @@ fn array_new_elem(match elements {TableSegmentElements::Functions(fs) => {let store_id = store.id();- let mut instance = store.instance_mut(instance_id);+ let (mut instance, registry) = store.instance_and_module_registry_mut(instance_id);vals.extend(fs.get(src..).and_then(|s| s.get(..len)).ok_or_else(|| Trap::TableOutOfBounds)?.iter().map(|f| {- let raw_func_ref = instance.as_mut().get_func_ref(*f);+ let raw_func_ref = instance.as_mut().get_func_ref(registry, *f);let func = unsafe {raw_func_ref.map(|p| Func::from_vm_func_ref(store_id, p))};@@ -1052,7 +1051,7 @@ fn array_init_elem(// Get the passive element segment.let mut storage = None;let store_id = store.id();- let mut instance = store.instance_mut(instance);+ let (mut instance, registry) = store.instance_and_module_registry_mut(instance);let elements = instance.passive_element_segment(&mut storage, elem_index);// Convert array offsets into `usize`s.@@ -1067,7 +1066,7 @@ fn array_init_elem(.ok_or_else(|| Trap::TableOutOfBounds)?.iter().map(|f| {- let raw_func_ref = instance.as_mut().get_func_ref(*f);+ let raw_func_ref = instance.as_mut().get_func_ref(registry, *f);let func = unsafe { raw_func_ref.map(|p| Func::from_vm_func_ref(store_id, p)) };Val::FuncRef(func)})diff --git a/crates/wasmtime/src/runtime/vm/mmap_vec.rs b/crates/wasmtime/src/runtime/vm/mmap_vec.rsindex f64de50ab8..9bebbf4d72 100644--- a/crates/wasmtime/src/runtime/vm/mmap_vec.rs+++ b/crates/wasmtime/src/runtime/vm/mmap_vec.rs@@ -270,6 +270,33 @@ impl MmapVec {MmapVec::Mmap { mmap, len } => unsafe { mmap.slice_mut(0..*len) },}}++ /// Create a copy of this `MmapVec` that can be separately+ /// mutated.+ #[cfg(feature = "debug")]+ pub(crate) fn deep_clone(&self) -> Result<MmapVec> {+ match self {+ #[cfg(not(has_virtual_memory))]+ MmapVec::Alloc { layout, .. } => {+ MmapVec::from_slice_with_alignment(&self[..], layout.align())+ }+ MmapVec::ExternallyOwned { .. } => {+ anyhow::bail!("Cannot clone an externally-owned code memory.");+ }+ #[cfg(has_virtual_memory)]+ MmapVec::Mmap { mmap, len } => {+ if let Some(original_file) = mmap.original_file() {+ let mmap = Mmap::from_file(original_file.clone())?;+ Ok(MmapVec::Mmap { mmap, len: *len })+ } else {+ MmapVec::from_slice_with_alignment(+ &self[..],+ crate::runtime::vm::host_page_size(),+ )+ }+ }+ }+ }}impl Deref for MmapVec {diff --git a/crates/wasmtime/src/runtime/vm/throw.rs b/crates/wasmtime/src/runtime/vm/throw.rsindex ba5105fe11..5463e930b4 100644--- a/crates/wasmtime/src/runtime/vm/throw.rs+++ b/crates/wasmtime/src/runtime/vm/throw.rs@@ -54,11 +54,9 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {frame.fp(),frame.pc());- let module = nogc.modules().lookup_module_by_pc(frame.pc())?;- let base = module.code_object().code_memory().text().as_ptr() as usize;- let rel_pc = u32::try_from(frame.pc().wrapping_sub(base)).expect("Module larger than 4GiB");- let et = module.exception_table();- let (frame_offset, handlers) = et.lookup_pc(rel_pc);+ let (module, rel_pc) = nogc.modules().module_and_code_by_pc(frame.pc())?;+ let et = module.module().exception_table();+ let (frame_offset, handlers) = et.lookup_pc(u32::try_from(rel_pc).unwrap());let fp_to_sp = frame_offset.map(|frame_offset| -isize::try_from(frame_offset).unwrap());for handler in handlers {log::trace!("-> checking handler: {handler:?}");@@ -112,9 +110,10 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {if is_match {let fp_to_sp = fp_to_sp.expect("frame offset must be known if we found a handler");return Some((- base.wrapping_add(- usize::try_from(handler.handler_offset).expect("Module larger than usize"),- ),+ (module.store_code().text_range().start+ + usize::try_from(handler.handler_offset)+ .expect("Module larger than usize"))+ .raw(),frame.fp().wrapping_add_signed(fp_to_sp),));}diff --git a/src/commands/run.rs b/src/commands/run.rsindex bfe2fb99f4..3f4e2196a5 100644--- a/src/commands/run.rs+++ b/src/commands/run.rs@@ -355,7 +355,7 @@ impl RunCommand {profiled_modules,path,*interval,- ));+ )?);#[cfg(not(feature = "profiling"))]{let _ = (profiled_modules, path, interval, main_target);@@ -383,22 +383,24 @@ impl RunCommand {profiled_modules: Vec<(String, Module)>,path: &str,interval: std::time::Duration,- ) -> Box<dyn FnOnce(&mut Store<Host>)> {+ ) -> Result<Box<dyn FnOnce(&mut Store<Host>)>> {use wasmtime::{AsContext, GuestProfiler, StoreContext, StoreContextMut, UpdateDeadline};let module_name = self.module_and_args[0].to_str().unwrap_or("<main module>");store.data_mut().guest_profiler = match main_target {RunTarget::Core(_m) => Some(Arc::new(GuestProfiler::new(+ store.engine(),module_name,interval,profiled_modules,- ))),+ )?)),RunTarget::Component(component) => Some(Arc::new(GuestProfiler::new_component(+ store.engine(),module_name,interval,component.clone(),profiled_modules,- ))),+ )?)),};fn sample(@@ -450,7 +452,7 @@ impl RunCommand {});let path = path.to_string();- return Box::new(move |store| {+ Ok(Box::new(move |store| {let profiler = Arc::try_unwrap(store.data_mut().guest_profiler.take().unwrap()).expect("profiling doesn't support threads yet");if let Err(e) = std::fs::File::create(&path)@@ -463,7 +465,7 @@ impl RunCommand {eprintln!("Profile written to: {path}");eprintln!("View this profile at https://profiler.firefox.com/.");}- });+ }))}async fn load_main_module(diff --git a/src/commands/serve.rs b/src/commands/serve.rsindex 00f5e2a89c..e97a1769ad 100644--- a/src/commands/serve.rs+++ b/src/commands/serve.rs@@ -769,11 +769,12 @@ fn setup_guest_profiler(let module_name = "<main>";store.data_mut().guest_profiler = Some(Arc::new(GuestProfiler::new_component(+ store.engine(),module_name,interval,component,std::iter::empty(),- )));+ )?));fn sample(mut store: StoreContextMut<Host>,