Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Debug: create private code memories per store when debugging is enabled. (#12051)

Chris Fallin99ecf728d6fe
diff --git a/crates/c-api/include/wasmtime/profiling.h b/crates/c-api/include/wasmtime/profiling.h
index cc64023644..885fec56ae 100644
--- a/crates/c-api/include/wasmtime/profiling.h
+++ b/crates/c-api/include/wasmtime/profiling.h
@@ -56,6 +56,7 @@ typedef struct wasmtime_guestprofiler_modules {
/**
* \brief Begin profiling a new guest.
*
+ * \param engine engine in which to perform the profiling
* \param module_name name recorded in the profile
* \param interval_nanos intended sampling interval in nanoseconds recorded in
* the profile
@@ -71,8 +72,9 @@ typedef struct wasmtime_guestprofiler_modules {
* https://docs.wasmtime.dev/api/wasmtime/struct.GuestProfiler.html#method.new
*/
WASM_API_EXTERN /* own */ wasmtime_guestprofiler_t *wasmtime_guestprofiler_new(
- const wasm_name_t *module_name, uint64_t interval_nanos,
- const wasmtime_guestprofiler_modules_t *modules, size_t modules_len);
+ const wasm_engine_t *engine, const wasm_name_t *module_name,
+ uint64_t interval_nanos, const wasmtime_guestprofiler_modules_t *modules,
+ size_t modules_len);
/**
* \brief Add a sample to the profile.
diff --git a/crates/c-api/src/profiling.rs b/crates/c-api/src/profiling.rs
index cda09c4e33..307b434f80 100644
--- a/crates/c-api/src/profiling.rs
+++ b/crates/c-api/src/profiling.rs
@@ -1,4 +1,7 @@
-use crate::{wasm_byte_vec_t, wasm_name_t, wasmtime_error_t, wasmtime_module_t, wasmtime_store_t};
+use crate::{
+ wasm_byte_vec_t, wasm_engine_t, wasm_name_t, wasmtime_error_t, wasmtime_module_t,
+ wasmtime_store_t,
+};
use std::slice;
use std::str::from_utf8;
use std::time::Duration;
@@ -18,6 +21,7 @@ pub struct wasmtime_guestprofiler_modules_t<'a> {
#[unsafe(no_mangle)]
pub unsafe extern "C" fn wasmtime_guestprofiler_new(
+ engine: &wasm_engine_t,
module_name: &wasm_name_t,
interval_nanos: u64,
modules: *const wasmtime_guestprofiler_modules_t,
@@ -36,7 +40,13 @@ pub unsafe extern "C" fn wasmtime_guestprofiler_new(
})
.collect::<Vec<_>>();
Box::new(wasmtime_guestprofiler_t {
- guest_profiler: GuestProfiler::new(module_name, Duration::from_nanos(interval_nanos), list),
+ guest_profiler: GuestProfiler::new(
+ &engine.engine,
+ module_name,
+ Duration::from_nanos(interval_nanos),
+ list,
+ )
+ .unwrap(),
})
}
diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rs
index 27becc5991..190b0979ab 100644
--- a/crates/environ/src/key.rs
+++ b/crates/environ/src/key.rs
@@ -515,4 +515,29 @@ impl FuncKey {
_ => panic!("`FuncKey::unwrap_resource_drop_trampoline` called on {self:?}"),
}
}
+
+ /// Is this "Store-invariant"? This allows us to execute
+ /// EngineCode directly rather than StoreCode.
+ ///
+ /// Any function that is either directly from Wasm code, or calls
+ /// it directly (not indirected through a runtime-provided
+ /// function pointer), is "store-variant": we need to use a
+ /// StoreCode-specific version of the code to hit any patching
+ /// that our specific instantiations may have (due to debugging
+ /// breakpoints, etc). Trampolines into the runtime cannot be
+ /// patched and so can use EngineCode instead. This allows for
+ /// less complex plumbing in some places where we can avoid
+ /// looking up the StoreCode (or having access to the Store).
+ pub fn is_store_invariant(&self) -> bool {
+ match self {
+ Self::DefinedWasmFunction(..) | Self::ArrayToWasmTrampoline(..) => false,
+ Self::WasmToArrayTrampoline(..)
+ | Self::WasmToBuiltinTrampoline(..)
+ | Self::PulleyHostCall(..) => true,
+ #[cfg(feature = "component-model")]
+ Self::ComponentTrampoline(..)
+ | Self::ResourceDropTrampoline
+ | Self::UnsafeIntrinsic(..) => true,
+ }
+ }
}
diff --git a/crates/wasmtime/src/runtime/code.rs b/crates/wasmtime/src/runtime/code.rs
index 952f44f779..4fabb195db 100644
--- a/crates/wasmtime/src/runtime/code.rs
+++ b/crates/wasmtime/src/runtime/code.rs
@@ -1,25 +1,114 @@
+use crate::Engine;
+use crate::Module;
+use crate::module::ModuleRegistry;
+use crate::vm::ModuleMemoryImageSource;
use crate::{code_memory::CodeMemory, type_registry::TypeCollection};
+#[cfg(feature = "debug")]
+use alloc::boxed::Box;
use alloc::sync::Arc;
+use anyhow::Result;
+use core::ops::{Add, Range, Sub};
+use wasmtime_environ::DefinedFuncIndex;
use wasmtime_environ::ModuleTypes;
#[cfg(feature = "component-model")]
use wasmtime_environ::component::ComponentTypes;
-/// Metadata in Wasmtime about a loaded compiled artifact in memory which is
-/// ready to execute.
+macro_rules! define_pc_kind {
+ ($ty:ident) => {
+ #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+ pub struct $ty(usize);
+
+ impl Add<usize> for $ty {
+ type Output = $ty;
+ fn add(self, other: usize) -> $ty {
+ $ty(self.0.wrapping_add(other))
+ }
+ }
+ impl Sub<usize> for $ty {
+ type Output = $ty;
+ fn sub(self, other: usize) -> $ty {
+ $ty(self.0.wrapping_sub(other))
+ }
+ }
+
+ impl $ty {
+ /// Is the given PC within this range? Give the relative
+ /// offset if so.
+ pub fn offset_of(range: Range<$ty>, pc: usize) -> Option<usize> {
+ if pc >= range.start.0 && pc < range.end.0 {
+ Some(pc.wrapping_sub(range.start.0))
+ } else {
+ None
+ }
+ }
+
+ /// Get the raw PC value.
+ pub fn raw(&self) -> usize {
+ self.0
+ }
+ }
+ };
+}
+
+// An address in "engine code": the original (canonical) copy of code
+// for a compiled module.
+//
+// See [`EngineCode`] for more details.
+define_pc_kind!(EngineCodePC);
+
+// An address in "store code": the copy of code used when executing
+// instances of a module in a given store.
+//
+// May or may not be the same as the engine code address -- a store
+// is allowed to make a private copy of engine code as its store
+// code.
+//
+// See [`StoreCode`] for more details.
+define_pc_kind!(StoreCodePC);
+
+impl StoreCodePC {
+ /// Construct a StoreCodePC for search purposes from a raw PC
+ /// observed during execution.
+ pub fn from_raw(pc: usize) -> StoreCodePC {
+ StoreCodePC(pc)
+ }
+}
+
+/// Metadata about, and original machine code for, a loaded compiled
+/// artifact in memory which is ready for Wasmtime to execute.
///
/// This structure is used in both `Module` and `Component`. For components it's
/// notably shared amongst the core wasm modules within a component and the
/// component itself. For core wasm modules this is uniquely owned within a
/// `Module`.
-pub struct CodeObject {
- /// Actual underlying mmap which is executable and contains other compiled
- /// information.
+///
+/// The `EngineCode` is "static", like a module: there is one copy per
+/// unit of code. Instantiation of a module containing an `EngineCode`
+/// into a `Store` produces a `StoreCode`. The latter is the owner of
+/// the actual machine code that runs. This machine code *may* be a
+/// read-only-shared copy of the original, or may be a private copy
+/// that we have patched. The latter is useful for some kinds of
+/// debugging/instrumentation, which is always scoped per-Store.
+///
+/// The `EngineCode` does *not* expose its underlying `CodeMemory`, to
+/// guard against the original-copy being used in an inappropriate
+/// (store-specific) context. Instead, it has accessors that provide
+/// the instance-agnostic metadata from the `CodeMemory`, and
+/// instance-specific data and code pointers can be obtained from the
+/// `StoreCode`.
+pub struct EngineCode {
+ /// Actual underlying code which is executable and contains other
+ /// compiled information.
///
/// Note the `Arc` here is used to share this with `CompiledModule` and the
/// global module registry of traps. While probably not strictly necessary
/// and could be avoided with some refactorings is a hopefully a relatively
/// minor `Arc` for now.
- mmap: Arc<CodeMemory>,
+ ///
+ /// As noted above, this is the *original* copy of the code, and
+ /// may be used directly, but may also be deep-cloned to a private
+ /// copy in a `StoreCode`.
+ original_code: Arc<CodeMemory>,
/// Registered shared signature for the loaded object.
///
@@ -35,23 +124,19 @@ pub struct CodeObject {
types: Types,
}
-impl CodeObject {
- pub fn new(mmap: Arc<CodeMemory>, signatures: TypeCollection, types: Types) -> CodeObject {
+impl EngineCode {
+ pub fn new(mmap: Arc<CodeMemory>, signatures: TypeCollection, types: Types) -> EngineCode {
// The corresponding unregister for this is below in `Drop for
- // CodeObject`.
- crate::module::register_code(&mmap);
+ // EngineCode`.
+ crate::module::register_code(&mmap, mmap.raw_addr_range());
- CodeObject {
- mmap,
+ EngineCode {
+ original_code: mmap,
signatures,
types,
}
}
- pub fn code_memory(&self) -> &Arc<CodeMemory> {
- &self.mmap
- }
-
#[cfg(feature = "component-model")]
pub fn types(&self) -> &Types {
&self.types
@@ -64,11 +149,111 @@ impl CodeObject {
pub fn signatures(&self) -> &TypeCollection {
&self.signatures
}
+
+ pub fn text_size(&self) -> usize {
+ self.original_code.text().len()
+ }
+
+ /// Give the range of engine-code PCs in this code.
+ pub fn text_range(&self) -> Range<EngineCodePC> {
+ let raw = self.original_code.raw_addr_range();
+ EngineCodePC(raw.start)..EngineCodePC(raw.end)
+ }
+
+ // For all accessors to code object sections below: we can use the
+ // original (uncloned) code image for these: they are just slices
+ // of bytes that we interpret independent of location.
+ //
+ // We hide the raw `CodeMemory` in `original_code` (*not* a public
+ // field) because we do not want the EngineCode copy used
+ // inadvertently for execution of Wasm functions.
+ //
+ // We thus *cannot* add an accessor below for anything that
+ // actually interprets the location of the bytes as a code
+ // pointer to a Wasm function.
+ //
+ // Note that Wasm-to-array trampolines, in contrast, are fair game
+ // to execute directly from the EngineCode: these are not specific
+ // to Wasm functions, but instead call builtins, and we never
+ // patch trampolines in StoreCode, so we can freely mix in
+ // EngineCode variants.
+
+ /// Get the Wasm-to-array trampoline for the given raw range in
+ /// the text segment.
+ pub(crate) fn raw_wasm_to_array_trampoline_data(&self, range: Range<usize>) -> &[u8] {
+ &self.original_code.text()[range]
+ }
+
+ /// Provide the address-map data for this EngineCode.
+ pub fn address_map_data(&self) -> &[u8] {
+ // We can use the original (uncloned) code image for this: it
+ // is just a slice of bytes that we interpret independent of
+ // location.
+ self.original_code.address_map_data()
+ }
+
+ /// Provide the stack-map data for this EngineCode.
+ pub fn stack_map_data(&self) -> &[u8] {
+ // We can use the original (uncloned) code image for this: it
+ // is just a slice of bytes that we interpret independent of
+ // location.
+ self.original_code.stack_map_data()
+ }
+
+ /// Returns the encoded exception-tables section to pass to
+ /// `wasmtime_unwinder::ExceptionTable::parse`.
+ pub fn exception_tables(&self) -> &[u8] {
+ // We can use the original (uncloned) code image for this: it
+ // is just a slice of bytes that we interpret independent of
+ // location.
+ self.original_code.exception_tables()
+ }
+
+ /// Returns the encoded frame-tables section to pass to
+ /// `wasmtime_environ::FrameTable::parse`.
+ pub fn frame_tables(&self) -> &[u8] {
+ self.original_code.frame_tables()
+ }
+
+ /// Returns the data in the `ELF_NAME_DATA` section.
+ #[inline]
+ pub fn func_name_data(&self) -> &[u8] {
+ self.original_code.func_name_data()
+ }
+
+ /// Returns the contents of the `ELF_WASMTIME_DWARF` section.
+ #[inline]
+ pub fn wasm_dwarf(&self) -> &[u8] {
+ self.original_code.wasm_dwarf()
+ }
+
+ /// Returns the raw image as bytes (in our internal image format).
+ pub fn image(&self) -> &[u8] {
+ &self.original_code.mmap()[..]
+ }
+
+ pub fn text(&self) -> &[u8] {
+ &self.original_code.text()
+ }
+
+ /// Returns the concatenated list of all data associated with this wasm
+ /// module.
+ ///
+ /// This is used for initialization of memories and all data ranges stored
+ /// in a `Module` are relative to the slice returned here.
+ #[inline]
+ pub fn wasm_data(&self) -> &[u8] {
+ self.original_code.wasm_data()
+ }
+
+ pub(crate) fn module_memory_image_source(&self) -> &Arc<impl ModuleMemoryImageSource> {
+ &self.original_code
+ }
}
-impl Drop for CodeObject {
+impl Drop for EngineCode {
fn drop(&mut self) {
- crate::module::unregister_code(&self.mmap);
+ crate::module::unregister_code(self.original_code.raw_addr_range());
}
}
@@ -100,3 +285,193 @@ impl From<Arc<ComponentTypes>> for Types {
Types::Component(types)
}
}
+
+/// A `Store`-local instance of code.
+///
+/// This type encapsulates executable code within the context of
+/// instantiation in a single store. It may be an unmodified pointer
+/// to the read-only original code, or it may be locally patched for
+/// this store due to debugging or instrumentation settings.
+///
+/// Most things that the runtime will want to do with a module's image
+/// will require the `EngineCode` -- all metadata lives there. The
+/// `StoreCode` is solely responsible for the executable machine code.
+///
+/// This type is designed to be uniquely owned, unlike `EngineCode`
+/// above. The runtime data structures will have scattered `Arc`s to
+/// the `EngineCode` but only one
+pub struct StoreCode(StoreCodeStorage);
+
+enum StoreCodeStorage {
+ Shared(Arc<CodeMemory>),
+ /// Private copy of the given code memory.
+ ///
+ /// This is the only reference to this CodeMemory. The StoreCode
+ /// is owned directly by the Store's ModuleRegistry.
+ #[cfg(feature = "debug")]
+ Private(Box<CodeMemory>),
+}
+
+impl StoreCode {
+ /// Create a new StoreCode for a given Store from a given
+ /// EngineCode, given the engine's settings (tunables).
+ pub fn new(engine: &Engine, engine_code: &Arc<EngineCode>) -> Result<Self> {
+ // Enabled guest-debugging causes us to allocate private
+ // copies of code in every store, to allow individual enabling
+ // of breakpoints (by code patching) independently in each
+ // one.
+ #[cfg(feature = "debug")]
+ let code = if engine.tunables().debug_guest {
+ // TODO(#12104): we should be able to clone only `.text`;
+ // this clones the whole image.
+ let mut private_copy = engine_code.original_code.deep_clone(engine)?;
+ private_copy.publish()?;
+ crate::module::register_code(&engine_code.original_code, private_copy.raw_addr_range());
+ StoreCodeStorage::Private(Box::new(private_copy))
+ } else {
+ StoreCodeStorage::Shared(engine_code.original_code.clone())
+ };
+
+ #[cfg(not(feature = "debug"))]
+ let code = StoreCodeStorage::Shared(engine_code.original_code.clone());
+ // Avoid unused-variable warning in build without debugging
+ // support.
+ let _ = engine;
+
+ Ok(StoreCode(code))
+ }
+
+ /// Provide the underlying CodeMemory.
+ pub fn code_memory(&self) -> &CodeMemory {
+ match &self.0 {
+ StoreCodeStorage::Shared(m) => m,
+ #[cfg(feature = "debug")]
+ StoreCodeStorage::Private(m) => m,
+ }
+ }
+
+ /// Provide a mutable reference to a CodeMemory that is privately
+ /// owned only by this StoreCode.
+ #[cfg(feature = "debug")]
+ pub fn code_memory_mut(&mut self) -> Option<&mut CodeMemory> {
+ match &mut self.0 {
+ StoreCodeStorage::Shared(_) => None,
+ StoreCodeStorage::Private(m) => Some(m),
+ }
+ }
+
+ /// Provide the address range for this StoreCode.
+ pub fn text_range(&self) -> Range<StoreCodePC> {
+ let raw = self.code_memory().raw_addr_range();
+ StoreCodePC(raw.start)..StoreCodePC(raw.end)
+ }
+
+ /// Provide the actual text segment for this StoreCode.
+ pub fn text(&self) -> &[u8] {
+ self.code_memory().text()
+ }
+}
+
+impl Drop for StoreCode {
+ fn drop(&mut self) {
+ match &self.0 {
+ StoreCodeStorage::Shared(_) => {
+ // Drop impl for EngineCode will de-register (see
+ // above).
+ }
+ #[cfg(feature = "debug")]
+ StoreCodeStorage::Private(mem) => {
+ crate::module::unregister_code(mem.raw_addr_range());
+ }
+ }
+ }
+}
+
+/// A wrapper for a Module together with a StoreCode. Allows fetching
+/// code pointers, ready to call.
+pub struct ModuleWithCode<'a> {
+ module: &'a Module,
+ store_code: &'a StoreCode,
+}
+
+impl<'a> ModuleWithCode<'a> {
+ /// Find the StoreCode in a given store for a module and wrap it
+ /// up with that module, ready to compute code pointers.
+ pub fn in_store(
+ registry: &'a ModuleRegistry,
+ module: &'a Module,
+ ) -> Option<ModuleWithCode<'a>> {
+ let store_code = registry.store_code(module.engine_code())?;
+ Some(ModuleWithCode { module, store_code })
+ }
+
+ pub(crate) fn from_raw(module: &'a Module, store_code: &'a StoreCode) -> ModuleWithCode<'a> {
+ ModuleWithCode { module, store_code }
+ }
+
+ /// Provide the Module wrapped in this tuple.
+ pub fn module(&self) -> &'a Module {
+ self.module
+ }
+
+ /// Provide the StoreCode wrapped in this tuple.
+ pub fn store_code(&self) -> &'a StoreCode {
+ self.store_code
+ }
+
+ /// Returns an iterator over all functions defined within this module with
+ /// their index and their raw pointer.
+ #[inline]
+ pub fn finished_functions(
+ &self,
+ ) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, &[u8])> + '_ {
+ self.module
+ .env_module()
+ .defined_func_indices()
+ .map(|i| (i, self.finished_function(i)))
+ }
+
+ /// Returns the slice in the text section of the function that
+ /// `index` points to.
+ #[inline]
+ pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] {
+ let range = self
+ .module
+ .compiled_module()
+ .finished_function_range(def_func_index);
+ &self.store_code.text()[range]
+ }
+
+ /// Get the array-to-Wasm trampoline for the function `index`
+ /// points to, as a slice of raw code that can be converted to a
+ /// callable function pointer.
+ ///
+ /// If the function `index` points to does not escape, then `None` is
+ /// returned.
+ ///
+ /// These trampolines are used for array callers (e.g. `Func::new`)
+ /// calling Wasm callees.
+ pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> {
+ let range = self
+ .module
+ .compiled_module()
+ .array_to_wasm_trampoline_range(def_func_index)?;
+ Some(&self.store_code.text()[range])
+ }
+
+ /// Get the text offset (relative PC) for a given absolute PC in
+ /// this module.
+ #[cfg(any(feature = "gc", feature = "debug"))]
+ pub(crate) fn text_offset(&self, pc: usize) -> Option<u32> {
+ StoreCodePC::offset_of(self.store_code.text_range(), pc)
+ .map(|offset| u32::try_from(offset).expect("Module larger than 4GiB"))
+ }
+
+ /// Lookup the stack map at a program counter value.
+ #[cfg(feature = "gc")]
+ pub(crate) fn lookup_stack_map(&self, pc: usize) -> Option<wasmtime_environ::StackMap<'_>> {
+ let text_offset = self.text_offset(pc)?;
+ let info = self.module.engine_code().stack_map_data();
+ wasmtime_environ::StackMap::lookup(text_offset, info)
+ }
+}
diff --git a/crates/wasmtime/src/runtime/code_memory.rs b/crates/wasmtime/src/runtime/code_memory.rs
index 6777790b18..4659ec85f2 100644
--- a/crates/wasmtime/src/runtime/code_memory.rs
+++ b/crates/wasmtime/src/runtime/code_memory.rs
@@ -441,6 +441,23 @@ impl CodeMemory {
pub fn lookup_trap_code(&self, text_offset: usize) -> Option<Trap> {
lookup_trap_code(self.trap_data(), text_offset)
}
+
+ /// Get the raw address range of this CodeMemory.
+ pub(crate) fn raw_addr_range(&self) -> Range<usize> {
+ let start = self.text().as_ptr().addr();
+ let end = start + self.text().len();
+ start..end
+ }
+
+ /// Create a "deep clone": a separate CodeMemory for the same code
+ /// that can be patched or mutated independently. Also returns a
+ /// "metadata and location" handle that can be registered with the
+ /// global module registry and used for trap metadata lookups.
+ #[cfg(feature = "debug")]
+ pub(crate) fn deep_clone(self: &Arc<Self>, engine: &Engine) -> Result<CodeMemory> {
+ let mmap = self.mmap.deep_clone()?;
+ Self::new(engine, mmap)
+ }
}
/// Returns the range of `inner` within `outer`, such that `outer[range]` is the
diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rs
index 2990248d9e..c4d5812723 100644
--- a/crates/wasmtime/src/runtime/component/component.rs
+++ b/crates/wasmtime/src/runtime/component/component.rs
@@ -6,7 +6,7 @@ use crate::prelude::*;
use crate::runtime::vm::open_file_for_mmap;
use crate::runtime::vm::{CompiledModuleId, VMArrayCallFunction, VMFuncRef, VMWasmCallFunction};
use crate::{
- Engine, Module, ResourcesRequired, code::CodeObject, code_memory::CodeMemory,
+ Engine, Module, ResourcesRequired, code::EngineCode, code_memory::CodeMemory,
type_registry::TypeCollection,
};
use crate::{FuncType, ValType};
@@ -81,7 +81,7 @@ struct ComponentInner {
///
/// Note that the `Arc` here is used to share this allocation with internal
/// modules.
- code: Arc<CodeObject>,
+ code: Arc<EngineCode>,
/// Metadata produced during compilation.
info: CompiledComponentInfo,
@@ -430,10 +430,10 @@ impl Component {
);
types.canonicalize_for_runtime_usage(&mut |idx| signatures.shared_type(idx).unwrap());
- // Assemble the `CodeObject` artifact which is shared by all core wasm
+ // Assemble the `EngineCode` artifact which is shared by all core wasm
// modules as well as the final component.
let types = Arc::new(types);
- let code = Arc::new(CodeObject::new(code_memory, signatures, types.into()));
+ let code = Arc::new(EngineCode::new(code_memory, signatures, types.into()));
// Convert all information about static core wasm modules into actual
// `Module` instances by converting each `CompiledModuleInfo`, the
@@ -496,17 +496,13 @@ impl Component {
self.inner.code.signatures()
}
- pub(crate) fn text(&self) -> &[u8] {
- self.inner.code.code_memory().text()
- }
-
pub(crate) fn trampoline_ptrs(&self, index: TrampolineIndex) -> AllCallFuncPointers {
let wasm_call = self
- .func(FuncKey::ComponentTrampoline(Abi::Wasm, index))
+ .store_invariant_func(FuncKey::ComponentTrampoline(Abi::Wasm, index))
.unwrap()
.cast();
let array_call = self
- .func(FuncKey::ComponentTrampoline(Abi::Array, index))
+ .store_invariant_func(FuncKey::ComponentTrampoline(Abi::Array, index))
.unwrap()
.cast();
AllCallFuncPointers {
@@ -520,10 +516,10 @@ impl Component {
intrinsic: UnsafeIntrinsic,
) -> Option<AllCallFuncPointers> {
let wasm_call = self
- .func(FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic))?
+ .store_invariant_func(FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic))?
.cast();
let array_call = self
- .func(FuncKey::UnsafeIntrinsic(Abi::Array, intrinsic))?
+ .store_invariant_func(FuncKey::UnsafeIntrinsic(Abi::Array, intrinsic))?
.cast();
Some(AllCallFuncPointers {
wasm_call,
@@ -532,7 +528,11 @@ impl Component {
}
/// Look up a function in this component's text section by `FuncKey`.
- fn func(&self, key: FuncKey) -> Option<NonNull<u8>> {
+ ///
+ /// This supports only `FuncKey`s that do not invoke Wasm code,
+ /// i.e., code that is potentially Store-specific.
+ fn store_invariant_func(&self, key: FuncKey) -> Option<NonNull<u8>> {
+ assert!(key.is_store_invariant());
let loc = self.inner.index.func_loc(key)?;
Some(self.func_loc_to_pointer(loc))
}
@@ -540,14 +540,16 @@ impl Component {
/// Given a function location within this component's text section, get a
/// pointer to the function.
///
+ /// This works only for Store-invariant functions.
+ ///
/// Panics on out-of-bounds function locations.
fn func_loc_to_pointer(&self, loc: &FunctionLoc) -> NonNull<u8> {
- let text = self.text();
+ let text = self.engine_code().text();
let trampoline = &text[loc.start as usize..][..loc.length as usize];
NonNull::from(trampoline).cast()
}
- pub(crate) fn code_object(&self) -> &Arc<CodeObject> {
+ pub(crate) fn engine_code(&self) -> &Arc<EngineCode> {
&self.inner.code
}
@@ -560,7 +562,7 @@ impl Component {
/// [`Module::serialize`]: crate::Module::serialize
/// [`Module`]: crate::Module
pub fn serialize(&self) -> Result<Vec<u8>> {
- Ok(self.code_object().code_memory().mmap().to_vec())
+ Ok(self.engine_code().image().to_vec())
}
/// Creates a new `VMFuncRef` with all fields filled out for the destructor
@@ -577,7 +579,7 @@ impl Component {
// then this can't be called by the component, so it's ok to leave it
// blank.
let wasm_call = self
- .func(FuncKey::ResourceDropTrampoline)
+ .store_invariant_func(FuncKey::ResourceDropTrampoline)
.map(|f| f.cast().into());
VMFuncRef {
@@ -680,7 +682,10 @@ impl Component {
/// For more information see
/// [`Module::image_range`](crate::Module::image_range).
pub fn image_range(&self) -> Range<*const u8> {
- self.inner.code.code_memory().mmap().image_range()
+ let range = self.inner.code.text_range();
+ let start = range.start.raw() as *const u8;
+ let end = range.end.raw() as *const u8;
+ start..end
}
/// Force initialization of copy-on-write images to happen here-and-now
diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rs
index 2d746ed171..07de7d8c76 100644
--- a/crates/wasmtime/src/runtime/component/concurrent.rs
+++ b/crates/wasmtime/src/runtime/component/concurrent.rs
@@ -2928,9 +2928,9 @@ impl Instance {
log::trace!("creating new thread");
let start_func_ty = FuncType::new(store.engine(), [ValType::I32], []);
- let instance = self.id().get_mut(store.0);
+ let (instance, registry) = self.id().get_mut_and_registry(store.0);
let callee = instance
- .index_runtime_func_table(start_func_table_idx, start_func_idx as u64)?
+ .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
.ok_or_else(|| {
anyhow!("the start function index points to an uninitialized function")
})?;
diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rs
index fb1e99a09c..a803e6040c 100644
--- a/crates/wasmtime/src/runtime/component/instance.rs
+++ b/crates/wasmtime/src/runtime/component/instance.rs
@@ -632,7 +632,7 @@ where
.store_data_mut()
.component_instance_mut(id)
.instance(item.instance);
- let instance = store.instance_mut(id);
+ let (instance, registry) = store.instance_and_module_registry_mut(id);
let idx = match &item.item {
ExportItem::Index(idx) => (*idx).into(),
@@ -649,7 +649,7 @@ where
};
// SAFETY: the `store_id` owns this instance and all exports contained
// within.
- unsafe { instance.get_export_by_index_mut(store_id, idx) }
+ unsafe { instance.get_export_by_index_mut(registry, store_id, idx) }
}
fn resource_tables<'a>(
@@ -742,9 +742,9 @@ impl<'a> Instantiator<'a> {
component: &'a Component,
store: &mut StoreOpaque,
imports: &'a Arc<PrimaryMap<RuntimeImportIndex, RuntimeImport>>,
- ) -> Instantiator<'a> {
+ ) -> Result<Instantiator<'a>> {
let env_component = component.env_component();
- store.modules_mut().register_component(component);
+ store.register_component(component)?;
let imported_resources: ImportedResources =
PrimaryMap::with_capacity(env_component.imported_resources.len());
@@ -757,12 +757,12 @@ impl<'a> Instantiator<'a> {
);
let id = store.store_data_mut().push_component_instance(instance);
- Instantiator {
+ Ok(Instantiator {
component,
imports,
core_imports: OwnedImports::empty(),
id,
- }
+ })
}
async fn run<T>(&mut self, store: &mut StoreContextMut<'_, T>) -> Result<()> {
@@ -1165,7 +1165,7 @@ impl<T: 'static> InstancePre<T> {
.engine()
.allocator()
.increment_component_instance_count()?;
- let mut instantiator = Instantiator::new(&self.component, store.0, &self.imports);
+ let mut instantiator = Instantiator::new(&self.component, store.0, &self.imports)?;
instantiator.run(&mut store).await.map_err(|e| {
store
.engine()
diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rs
index 762c031706..94817a52c8 100644
--- a/crates/wasmtime/src/runtime/component/store.rs
+++ b/crates/wasmtime/src/runtime/component/store.rs
@@ -145,6 +145,25 @@ impl StoreComponentInstanceId {
self.from_data_get_mut(store.store_data_mut())
}
+ /// Return a mutable `ComponentInstance` and a `ModuleRegistry`
+ /// from the store.
+ ///
+ /// # Panics
+ ///
+ /// Panics if `self` does not belong to `store`.
+ #[cfg(feature = "component-model-async")]
+ pub(crate) fn get_mut_and_registry<'a>(
+ &self,
+ store: &'a mut StoreOpaque,
+ ) -> (
+ Pin<&'a mut ComponentInstance>,
+ &'a crate::module::ModuleRegistry,
+ ) {
+ let (store_data, registry) = store.store_data_mut_and_registry();
+ let instance = self.from_data_get_mut(store_data);
+ (instance, registry)
+ }
+
/// Same as `get_mut`, but borrows less of a store.
pub(crate) fn from_data_get_mut<'a>(
&self,
diff --git a/crates/wasmtime/src/runtime/debug.rs b/crates/wasmtime/src/runtime/debug.rs
index 70bcef9526..f6b4730bf4 100644
--- a/crates/wasmtime/src/runtime/debug.rs
+++ b/crates/wasmtime/src/runtime/debug.rs
@@ -244,12 +244,11 @@ impl VirtualFrame {
/// Return virtual frames corresponding to a physical frame, from
/// outermost to innermost.
fn decode(store: &mut StoreOpaque, frame: Frame, is_trapping_frame: bool) -> Vec<VirtualFrame> {
- let module = store
+ let (module_with_code, pc) = store
.modules()
- .lookup_module_by_pc(frame.pc())
+ .module_and_code_by_pc(frame.pc())
.expect("Wasm frame PC does not correspond to a module");
- let base = module.code_object().code_memory().text().as_ptr() as usize;
- let pc = frame.pc().wrapping_sub(base);
+ let module = module_with_code.module();
let table = module.frame_table().unwrap();
let pc = u32::try_from(pc).expect("PC offset too large");
let pos = if is_trapping_frame {
diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs
index eaca5ccf46..7d524184e8 100644
--- a/crates/wasmtime/src/runtime/externals/table.rs
+++ b/crates/wasmtime/src/runtime/externals/table.rs
@@ -154,10 +154,11 @@ impl Table {
lazy_init_range: impl IntoIterator<Item = u64>,
) -> (&'a mut vm::Table, Option<&'a mut GcStore>) {
self.instance.assert_belongs_to(store.id());
- let (store, instance) = store.optional_gc_store_and_instance_mut(self.instance.instance());
+ let (store, registry, instance) =
+ store.optional_gc_store_and_registry_and_instance_mut(self.instance.instance());
(
- instance.get_defined_table_with_lazy_init(self.index, lazy_init_range),
+ instance.get_defined_table_with_lazy_init(registry, self.index, lazy_init_range),
store,
)
}
diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs
index 3445a4a401..5c4d6b2664 100644
--- a/crates/wasmtime/src/runtime/instance.rs
+++ b/crates/wasmtime/src/runtime/instance.rs
@@ -231,7 +231,7 @@ impl Instance {
// Note that under normal operation this shouldn't do much as the list
// of funcs-with-holes should generally be empty. As a result the
// process of filling this out is not super optimized at this point.
- store.modules_mut().register_module(module);
+ store.register_module(module)?;
let (funcrefs, modules) = store.func_refs_and_modules();
funcrefs.fill(modules);
@@ -308,7 +308,7 @@ impl Instance {
// Register the module just before instantiation to ensure we keep the module
// properly referenced while in use by the store.
- let module_id = store.modules_mut().register_module(module);
+ let module_id = store.register_module(module)?;
// The first thing we do is issue an instance allocation request
// to the instance allocator. This, on success, will give us an
@@ -371,10 +371,14 @@ impl Instance {
// If a start function is present, invoke it. Make sure we use all the
// trap-handling configuration in `store` as well.
let store_id = store.0.id();
- let mut instance = self.id.get_mut(store.0);
+ let (mut instance, registry) = self.id.get_mut_and_module_registry(store.0);
// SAFETY: the `store_id` is the id of the store that owns this
// instance and any function stored within the instance.
- let f = unsafe { instance.as_mut().get_exported_func(store_id, start) };
+ let f = unsafe {
+ instance
+ .as_mut()
+ .get_exported_func(registry, store_id, start)
+ };
let caller_vmctx = instance.vmctx();
unsafe {
let funcref = f.vm_func_ref(store.0);
@@ -479,7 +483,10 @@ impl Instance {
let id = store.id();
// SAFETY: the store `id` owns this instance and all exports contained
// within.
- let export = unsafe { self.id.get_mut(store).get_export_by_index_mut(id, entity) };
+ let export = unsafe {
+ let (instance, registry) = self.id.get_mut_and_module_registry(store);
+ instance.get_export_by_index_mut(registry, id, entity)
+ };
Extern::from_wasmtime_export(export, store)
}
@@ -922,7 +929,7 @@ fn pre_instantiate_raw(
) -> Result<OwnedImports> {
// Register this module and use it to fill out any funcref wasm_call holes
// we can. For more comments on this see `typecheck_externs`.
- store.modules_mut().register_module(module);
+ store.register_module(module)?;
let (funcrefs, modules) = store.func_refs_and_modules();
funcrefs.fill(modules);
diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rs
index 5fb1130f09..cbb3f4022b 100644
--- a/crates/wasmtime/src/runtime/instantiate.rs
+++ b/crates/wasmtime/src/runtime/instantiate.rs
@@ -3,10 +3,12 @@
//! `CompiledModule` to allow compiling and instantiating to be done as separate
//! steps.
+use crate::code::EngineCode;
use crate::prelude::*;
-use crate::runtime::vm::{CompiledModuleId, MmapVec};
-use crate::{code_memory::CodeMemory, profiling_agent::ProfilingAgent};
+use crate::profiling_agent::ProfilingAgent;
+use crate::runtime::vm::CompiledModuleId;
use alloc::sync::Arc;
+use core::ops::Range;
use core::str;
use wasmtime_environ::{
CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, EntityRef, FilePos, FuncIndex,
@@ -18,7 +20,7 @@ use wasmtime_environ::{
pub struct CompiledModule {
/// A unique ID used to register this module with the engine.
unique_id: CompiledModuleId,
- code_memory: Arc<CodeMemory>,
+ engine_code: Arc<EngineCode>,
module: Arc<Module>,
meta: Metadata,
index: Arc<CompiledFunctionsTable>,
@@ -29,10 +31,11 @@ pub struct CompiledModule {
impl CompiledModule {
/// Creates `CompiledModule` directly from a precompiled artifact.
///
- /// The `code_memory` argument is expected to be the result of a previous
- /// call to `ObjectBuilder::finish` above. This is an ELF image, at this
- /// time, which contains all necessary information to create a
- /// `CompiledModule` from a compilation.
+ /// The `engine_code` argument is expected to be an EngineCode
+ /// wrapper around a CodeMemory containing the result of a
+ /// previous call to `ObjectBuilder::finish` above. This is an ELF
+ /// image, at this time, which contains all necessary information
+ /// to create a `CompiledModule` from a compilation.
///
/// This method also takes `info`, an optionally-provided deserialization
/// of the artifacts' compilation metadata section. If this information is
@@ -44,14 +47,14 @@ impl CompiledModule {
/// The `profiler` argument here is used to inform JIT profiling runtimes
/// about new code that is loaded.
pub fn from_artifacts(
- code_memory: Arc<CodeMemory>,
+ engine_code: Arc<EngineCode>,
info: CompiledModuleInfo,
index: Arc<CompiledFunctionsTable>,
profiler: &dyn ProfilingAgent,
) -> Result<Self> {
let mut ret = Self {
unique_id: CompiledModuleId::new(),
- code_memory,
+ engine_code,
module: Arc::new(info.module),
meta: info.meta,
index,
@@ -65,7 +68,8 @@ impl CompiledModule {
fn register_profiling(&mut self, profiler: &dyn ProfilingAgent) -> Result<()> {
// TODO-Bug?: "code_memory" is not exclusive for this module in the case of components,
// so we may be registering the same code range multiple times here.
- profiler.register_module(&self.code_memory.mmap()[..], &|addr| {
+
+ profiler.register_module(self.engine_code.image(), &|addr| {
let idx = self.func_by_text_offset(addr)?;
let idx = self.module.func_index(idx);
let name = self.func_name(idx)?;
@@ -82,25 +86,6 @@ impl CompiledModule {
self.unique_id
}
- /// Returns the underlying memory which contains the compiled module's
- /// image.
- pub fn mmap(&self) -> &MmapVec {
- self.code_memory.mmap()
- }
-
- /// Returns the underlying owned mmap of this compiled image.
- pub fn code_memory(&self) -> &Arc<CodeMemory> {
- &self.code_memory
- }
-
- /// Returns the text section of the ELF image for this compiled module.
- ///
- /// This memory should have the read/execute permissions.
- #[inline]
- pub fn text(&self) -> &[u8] {
- self.code_memory.text()
- }
-
/// Return a reference-counting pointer to a module.
pub fn module(&self) -> &Arc<Module> {
&self.module
@@ -121,43 +106,52 @@ impl CompiledModule {
// `from_utf8_unchecked` if we really wanted since this section is
// guaranteed to only have valid utf-8 data. Until it's a problem it's
// probably best to double-check this though.
- let data = self.code_memory().func_name_data();
+ let data = self.engine_code.func_name_data();
Some(str::from_utf8(&data[name.offset as usize..][..name.len as usize]).unwrap())
}
/// Returns an iterator over all functions defined within this module with
- /// their index and their body in memory.
+ /// their index and their offset in the underlying code image.
#[inline]
- pub fn finished_functions(
+ pub fn finished_function_ranges(
&self,
- ) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, &[u8])> + '_ {
+ ) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, Range<usize>)> + '_ {
self.module
.defined_func_indices()
- .map(|i| (i, self.finished_function(i)))
+ .map(|i| (i, self.finished_function_range(i)))
}
- /// Returns the body of the function that `index` points to.
+ /// Returns the offset in the text section of the function that `index` points to.
#[inline]
- pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] {
+ pub fn finished_function_range(&self, def_func_index: DefinedFuncIndex) -> Range<usize> {
let loc = self.func_loc(def_func_index);
- &self.text()[loc.start as usize..][..loc.length as usize]
+ let start = usize::try_from(loc.start).unwrap();
+ let end = usize::try_from(loc.start + loc.length).unwrap();
+ start..end
}
- /// Get the array-to-Wasm trampoline for the function `index` points to.
+ /// Get the array-to-Wasm trampoline for the function `index`
+ /// points to, as a range in the text segment.
///
/// If the function `index` points to does not escape, then `None` is
/// returned.
///
/// These trampolines are used for array callers (e.g. `Func::new`)
/// calling Wasm callees.
- pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> {
+ pub fn array_to_wasm_trampoline_range(
+ &self,
+ def_func_index: DefinedFuncIndex,
+ ) -> Option<Range<usize>> {
assert!(def_func_index.index() < self.module.num_defined_funcs());
let key = FuncKey::ArrayToWasmTrampoline(self.module_index(), def_func_index);
let loc = self.index.func_loc(key)?;
- Some(&self.text()[loc.start as usize..][..loc.length as usize])
+ let start = usize::try_from(loc.start).unwrap();
+ let end = usize::try_from(loc.start + loc.length).unwrap();
+ Some(start..end)
}
- /// Get the Wasm-to-array trampoline for the given signature.
+ /// Get the Wasm-to-array trampoline for the given signature, as a
+ /// range in the text segment.
///
/// These trampolines are used for filling in
/// `VMFuncRef::wasm_call` for `Func::wrap`-style host funcrefs
@@ -165,7 +159,12 @@ impl CompiledModule {
pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> Option<&[u8]> {
let key = FuncKey::WasmToArrayTrampoline(signature);
let loc = self.index.func_loc(key)?;
- Some(&self.text()[loc.start as usize..][..loc.length as usize])
+ let start = usize::try_from(loc.start).unwrap();
+ let end = usize::try_from(loc.start + loc.length).unwrap();
+ Some(
+ self.engine_code
+ .raw_wasm_to_array_trampoline_data(start..end),
+ )
}
/// Lookups a defined function by a program counter value.
@@ -229,7 +228,7 @@ impl CompiledModule {
let (_, range) = &self.meta.dwarf[i];
let start = range.start.try_into().ok()?;
let end = range.end.try_into().ok()?;
- self.code_memory().wasm_dwarf().get(start..end)
+ self.engine_code.wasm_dwarf().get(start..end)
})
.unwrap_or(&[]);
Ok(EndianSlice::new(data, gimli::LittleEndian))
@@ -254,7 +253,7 @@ impl CompiledModule {
/// If this function returns `false` then `lookup_file_pos` will always
/// return `None`.
pub fn has_address_map(&self) -> bool {
- !self.code_memory.address_map_data().is_empty()
+ !self.engine_code.address_map_data().is_empty()
}
}
diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rs
index f498eca796..36366a5387 100644
--- a/crates/wasmtime/src/runtime/module.rs
+++ b/crates/wasmtime/src/runtime/module.rs
@@ -5,7 +5,7 @@ use crate::runtime::vm::{CompiledModuleId, MmapVec, ModuleMemoryImages, VMWasmCa
use crate::sync::OnceLock;
use crate::{
Engine,
- code::CodeObject,
+ code::EngineCode,
code_memory::CodeMemory,
instantiate::CompiledModule,
resources::ResourcesRequired,
@@ -144,7 +144,7 @@ struct ModuleInner {
/// Note that this `Arc` is used to share information between compiled
/// modules within a component. For bare core wasm modules created with
/// `Module::new`, for example, this is a uniquely owned `Arc`.
- code: Arc<CodeObject>,
+ code: Arc<EngineCode>,
/// A set of initialization images for memories, if any.
///
@@ -518,26 +518,21 @@ impl Module {
let signatures =
engine.register_and_canonicalize_types(&mut types, core::iter::once(&mut info.module));
- // Package up all our data into a `CodeObject` and delegate to the final
+ // Package up all our data into an `EngineCode` and delegate to the final
// step of module compilation.
- let code = Arc::new(CodeObject::new(code_memory, signatures, types.into()));
+ let code = Arc::new(EngineCode::new(code_memory, signatures, types.into()));
let index = Arc::new(index);
Module::from_parts_raw(engine, code, info, index, true)
}
pub(crate) fn from_parts_raw(
engine: &Engine,
- code: Arc<CodeObject>,
+ code: Arc<EngineCode>,
info: CompiledModuleInfo,
index: Arc<CompiledFunctionsTable>,
serializable: bool,
) -> Result<Self> {
- let module = CompiledModule::from_artifacts(
- code.code_memory().clone(),
- info,
- index,
- engine.profiler(),
- )?;
+ let module = CompiledModule::from_artifacts(code.clone(), info, index, engine.profiler())?;
// Validate the module can be used with the current instance allocator.
let offsets = VMOffsets::new(HostPtr, module.module());
@@ -636,14 +631,14 @@ impl Module {
if !self.inner.serializable {
bail!("cannot serialize a module exported from a component");
}
- Ok(self.compiled_module().mmap().to_vec())
+ Ok(self.engine_code().image().to_vec())
}
pub(crate) fn compiled_module(&self) -> &CompiledModule {
&self.inner.module
}
- pub(crate) fn code_object(&self) -> &Arc<CodeObject> {
+ pub(crate) fn engine_code(&self) -> &Arc<EngineCode> {
&self.inner.code
}
@@ -973,8 +968,14 @@ impl Module {
///
/// It is not safe to modify the memory in this range, nor is it safe to
/// modify the protections of memory in this range.
+ ///
+ /// Note that depending on the engine configuration, this image
+ /// range may not actually be the code that is directly executed.
pub fn image_range(&self) -> Range<*const u8> {
- self.compiled_module().mmap().image_range()
+ let range = self.engine_code().text_range();
+ let start = range.start.raw() as *const u8;
+ let end = range.end.raw() as *const u8;
+ start..end
}
/// Force initialization of copy-on-write images to happen here-and-now
@@ -1043,17 +1044,15 @@ impl Module {
/// ```
pub fn address_map<'a>(&'a self) -> Option<impl Iterator<Item = (usize, Option<u32>)> + 'a> {
Some(
- wasmtime_environ::iterate_address_map(
- self.code_object().code_memory().address_map_data(),
- )?
- .map(|(offset, file_pos)| (offset as usize, file_pos.file_offset())),
+ wasmtime_environ::iterate_address_map(self.engine_code().address_map_data())?
+ .map(|(offset, file_pos)| (offset as usize, file_pos.file_offset())),
)
}
/// Get this module's code object's `.text` section, containing its compiled
/// executable code.
pub fn text(&self) -> &[u8] {
- self.code_object().code_memory().text()
+ self.engine_code().text()
}
/// Get information about functions in this module's `.text` section: their
@@ -1062,7 +1061,7 @@ impl Module {
/// Results are yielded in a ModuleFunction struct.
pub fn functions<'a>(&'a self) -> impl ExactSizeIterator<Item = ModuleFunction> + 'a {
let module = self.compiled_module();
- module.finished_functions().map(|(idx, _)| {
+ self.env_module().defined_func_indices().map(|idx| {
let loc = module.func_loc(idx);
let idx = module.module().func_index(idx);
ModuleFunction {
@@ -1084,6 +1083,11 @@ impl Module {
/// Return the address, in memory, of the trampoline that allows Wasm to
/// call a array function of the given signature.
+ ///
+ /// Note that unlike all other code-pointer-returning functions,
+ /// this *can* be present on `Module` (without a `StoreCode`)
+ /// because we can execute the `EngineCode` for trampolines that
+ /// leave the store to call the host.
pub(crate) fn wasm_to_array_trampoline(
&self,
signature: VMSharedTypeIndex,
@@ -1130,25 +1134,10 @@ impl Module {
Ok(images)
}
- /// Get the text offset (relative PC) for a given absolute PC in
- /// this module.
- #[cfg(feature = "gc")]
- pub(crate) fn text_offset(&self, pc: usize) -> u32 {
- u32::try_from(pc - self.inner.module.text().as_ptr() as usize).unwrap()
- }
-
- /// Lookup the stack map at a program counter value.
- #[cfg(feature = "gc")]
- pub(crate) fn lookup_stack_map(&self, pc: usize) -> Option<wasmtime_environ::StackMap<'_>> {
- let text_offset = self.text_offset(pc);
- let info = self.inner.code.code_memory().stack_map_data();
- wasmtime_environ::StackMap::lookup(text_offset, info)
- }
-
/// Obtain an exception-table parser on this module's exception metadata.
#[cfg(feature = "gc")]
pub(crate) fn exception_table<'a>(&'a self) -> ExceptionTable<'a> {
- ExceptionTable::parse(self.inner.code.code_memory().exception_tables())
+ ExceptionTable::parse(self.inner.code.exception_tables())
.expect("Exception tables were validated on module load")
}
@@ -1156,7 +1145,7 @@ impl Module {
/// (debug instrumentation) metadata.
#[cfg(feature = "debug")]
pub(crate) fn frame_table<'a>(&'a self) -> Option<FrameTable<'a>> {
- let data = self.inner.code.code_memory().frame_tables();
+ let data = self.inner.code.frame_tables();
if data.is_empty() {
None
} else {
@@ -1213,7 +1202,7 @@ fn memory_images(inner: &Arc<ModuleInner>) -> Result<Option<ModuleMemoryImages>>
ModuleMemoryImages::new(
&inner.engine,
inner.module.module(),
- inner.code.code_memory(),
+ inner.code.module_memory_image_source(),
)
}
diff --git a/crates/wasmtime/src/runtime/module/registry.rs b/crates/wasmtime/src/runtime/module/registry.rs
index a7836b4e3b..eba6e9a833 100644
--- a/crates/wasmtime/src/runtime/module/registry.rs
+++ b/crates/wasmtime/src/runtime/module/registry.rs
@@ -1,173 +1,198 @@
//! Implements a registry of modules for a store.
-use crate::code::CodeObject;
+use crate::code::{EngineCode, EngineCodePC, ModuleWithCode, StoreCode, StoreCodePC};
#[cfg(feature = "component-model")]
use crate::component::Component;
-use crate::prelude::*;
use crate::runtime::vm::VMWasmCallFunction;
use crate::sync::{OnceLock, RwLock};
+use crate::vm::CompiledModuleId;
+use crate::{Engine, prelude::*};
use crate::{FrameInfo, Module, code_memory::CodeMemory};
use alloc::collections::btree_map::{BTreeMap, Entry};
use alloc::sync::Arc;
+use core::ops::Range;
use core::ptr::NonNull;
use wasmtime_environ::VMSharedTypeIndex;
/// Used for registering modules with a store.
///
-/// Note that the primary reason for this registry is to ensure that everything
-/// in `Module` is kept alive for the duration of a `Store`. At this time we
-/// need "basically everything" within a `Module` to stay alive once it's
-/// instantiated within a store. While there's some smaller portions that could
-/// theoretically be omitted as they're not needed by the store they're
-/// currently small enough to not worry much about.
+/// There are two basic purposes that this registry serves:
+///
+/// - It keeps all modules and their metadata alive as long as the
+/// store exists.
+/// - It owns the [`StoreCode`], i.e. possibly-private-copy of machine
+/// code, for all modules that execute in this store.
+///
+/// The registry allows for translation of EngineCode to StoreCode,
+/// deduplicating by the start address of the EngineCode; and allows
+/// for looking up modules by "registered module ID", and looking up
+/// StoreCode and Modules by PC.
+///
+/// Note that multiple modules may be backed by a single
+/// `StoreCode`. This is specifically the case for components in
+/// general. When a component is first instantiated, the component
+/// itself is registered (which loads the StoreCode into the
+/// registry), then each individual module within that component is
+/// registered and added to the data structures.
+///
+/// A brief overview of the kinds of compiled object and their
+/// relationships:
+///
+/// - `Module` is a Wasm module. It owns a `CompiledModule`.
+/// - `CompiledModule` contains metadata about the module (e.g., a map
+/// from Wasm function indices to locations in the machine code),
+/// and also owns an `EngineCode`.
+/// - `EngineCode` holds an `Arc` to a `CodeMemory` with the canonical
+/// copy of machine code, as well as some lower-level metadata
+/// (signatures and types). It is instantiated by this registry into
+/// `StoreCode`.
+/// - `StoreCode` owns either another `Arc` to the same `CodeMemory`
+/// as `EngineCode`, or if guest debugging is enabled and causes us
+/// to clone private copies of code for patching per store, owns its
+/// own private `CodeMemory` at a different address.
+/// - Instances hold a `RegisteredModuleId` to be able to look up their modules.
#[derive(Default)]
pub struct ModuleRegistry {
- // Keyed by the end address of a `CodeObject`.
- //
- // The value here is the start address and the information about what's
- // loaded at that address.
- loaded_code: BTreeMap<usize, (usize, LoadedCode)>,
-
- // Preserved for keeping data segments alive or similar
- modules_without_code: Vec<Module>,
+ /// StoreCode and Modules associated with it.
+ ///
+ /// Keyed by the start address of the `StoreCode`. We maintain the
+ /// invariant of no overlaps on insertion. We use a range query to
+ /// find the StoreCode for a given PC: take the range `0..=pc`,
+ /// then take the last element of the range. That picks the
+ /// highest start address <= the query, and we can check whether
+ /// it contains the address.
+ loaded_code: BTreeMap<StoreCodePC, LoadedCode>,
+
+ /// Map from EngineCodePC start to StoreCodePC start. We use this
+ /// to memoize the store-code creation process: each EngineCode is
+ /// instantiated to a StoreCode only once per store.
+ store_code: BTreeMap<EngineCodePC, StoreCodePC>,
+
+ /// Modules instantiated in this registry.
+ ///
+ /// Every module is placed in this map, but not every module will
+ /// be in a LoadedCode entry, because the module may have no text.
+ modules: BTreeMap<RegisteredModuleId, Module>,
}
struct LoadedCode {
- /// Kept alive here in the store to have a strong reference to keep the
- /// relevant code mapped while the store is alive.
- _code: Arc<CodeObject>,
-
- /// Modules found within `self.code`, keyed by start address here of the
- /// address of the first function in the module.
- modules: BTreeMap<usize, Module>,
+ /// The StoreCode in this range.
+ code: StoreCode,
- /// These modules have no functions inside of them but they can still be
- /// used for trampoline lookup.
- modules_with_only_trampolines: Vec<Module>,
+ /// Map by starting text offset of Modules in this code region.
+ modules: BTreeMap<usize, RegisteredModuleId>,
}
/// An identifier of a module that has previously been inserted into a
/// `ModuleRegistry`.
-#[derive(Clone, Copy, Debug)]
-pub enum RegisteredModuleId {
- /// Index into `ModuleRegistry::modules_without_code`.
- WithoutCode(usize),
- /// Start address of the module's code so that we can get it again via
- /// `ModuleRegistry::lookup_module`.
- LoadedCode(usize),
+///
+/// This is just a newtype around `CompiledModuleId`, which is unique
+/// within the Engine.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct RegisteredModuleId(CompiledModuleId);
+
+fn assert_no_overlap(loaded_code: &BTreeMap<StoreCodePC, LoadedCode>, range: Range<StoreCodePC>) {
+ if let Some((start, _)) = loaded_code.range(range.start..).next() {
+ assert!(*start >= range.end);
+ }
+ if let Some((_, code)) = loaded_code.range(..range.end).next_back() {
+ assert!(code.code.text_range().end <= range.start);
+ }
}
impl ModuleRegistry {
/// Get a previously-registered module by id.
- pub fn lookup_module_by_id(&self, id: RegisteredModuleId) -> Option<&Module> {
- match id {
- RegisteredModuleId::WithoutCode(idx) => self.modules_without_code.get(idx),
- RegisteredModuleId::LoadedCode(pc) => {
- let (module, _) = self.module_and_offset(pc)?;
- Some(module)
- }
- }
- }
-
- /// Fetches a registered module given a program counter value.
- #[cfg(any(feature = "gc", feature = "debug"))]
- pub fn lookup_module_by_pc(&self, pc: usize) -> Option<&Module> {
- let (module, _) = self.module_and_offset(pc)?;
- Some(module)
+ pub fn module_by_id(&self, id: RegisteredModuleId) -> Option<&Module> {
+ self.modules.get(&id)
}
- fn code(&self, pc: usize) -> Option<(&LoadedCode, usize)> {
- let (end, (start, code)) = self.loaded_code.range(pc..).next()?;
- if pc < *start || *end < pc {
- return None;
- }
- Some((code, pc - *start))
+ /// Fetches a registered StoreCode and module and an offset within
+ /// it given a program counter value.
+ pub fn module_and_code_by_pc<'a>(&'a self, pc: usize) -> Option<(ModuleWithCode<'a>, usize)> {
+ let (_, code) = self
+ .loaded_code
+ .range(..=StoreCodePC::from_raw(pc))
+ .next_back()?;
+ let offset = StoreCodePC::offset_of(code.code.text_range(), pc)?;
+ let (_, module_id) = code.modules.range(..=offset).next_back()?;
+ let module = self.modules.get(&module_id)?;
+ Some((ModuleWithCode::from_raw(module, &code.code), offset))
}
- fn module_and_offset(&self, pc: usize) -> Option<(&Module, usize)> {
- let (code, offset) = self.code(pc)?;
- Some((code.module(pc)?, offset))
+ /// Fetches the `StoreCode` for a given `EngineCode`.
+ pub fn store_code(&self, engine_code: &EngineCode) -> Option<&StoreCode> {
+ let store_code_pc = *self.store_code.get(&engine_code.text_range().start)?;
+ let (_, code) = self.loaded_code.range(store_code_pc..).next()?;
+ Some(&code.code)
}
/// Gets an iterator over all modules in the registry.
#[cfg(feature = "coredump")]
pub fn all_modules(&self) -> impl Iterator<Item = &'_ Module> + '_ {
- self.loaded_code
- .values()
- .flat_map(|(_, code)| code.modules.values())
- .chain(self.modules_without_code.iter())
+ self.modules.values()
}
/// Registers a new module with the registry.
- pub fn register_module(&mut self, module: &Module) -> RegisteredModuleId {
- self.register(module.code_object(), Some(module)).unwrap()
+ pub fn register_module(
+ &mut self,
+ module: &Module,
+ engine: &Engine,
+ ) -> Result<RegisteredModuleId> {
+ self.register(module.id(), module.engine_code(), Some(module), engine)
+ .map(|id| id.unwrap())
}
#[cfg(feature = "component-model")]
- pub fn register_component(&mut self, component: &Component) {
- self.register(component.code_object(), None);
+ pub fn register_component(&mut self, component: &Component, engine: &Engine) -> Result<()> {
+ self.register(component.id(), component.engine_code(), None, engine)?;
+ Ok(())
}
/// Registers a new module with the registry.
fn register(
&mut self,
- code: &Arc<CodeObject>,
+ compiled_id: CompiledModuleId,
+ code: &Arc<EngineCode>,
module: Option<&Module>,
- ) -> Option<RegisteredModuleId> {
- let text = code.code_memory().text();
-
- // If there's not actually any functions in this module then we may
- // still need to preserve it for its data segments. Instances of this
- // module will hold a pointer to the data stored in the module itself,
- // and for schemes that perform lazy initialization which could use the
- // module in the future. For that reason we continue to register empty
- // modules and retain them.
- if text.is_empty() {
- return module.map(|module| {
- let id = RegisteredModuleId::WithoutCode(self.modules_without_code.len());
- self.modules_without_code.push(module.clone());
- id
- });
- }
-
- // The module code range is exclusive for end, so make it inclusive as
- // it may be a valid PC value
- let start_addr = text.as_ptr() as usize;
- let end_addr = start_addr + text.len() - 1;
- let id = module.map(|_| RegisteredModuleId::LoadedCode(start_addr));
-
- // If this module is already present in the registry then that means
- // it's either an overlapping image, for example for two modules
- // found within a component, or it's a second instantiation of the same
- // module. Delegate to `push_module` to find out.
- if let Some((other_start, prev)) = self.loaded_code.get_mut(&end_addr) {
- assert_eq!(*other_start, start_addr);
- if let Some(module) = module {
- prev.push_module(module);
+ engine: &Engine,
+ ) -> Result<Option<RegisteredModuleId>> {
+ // Register the module, if any.
+ let id = module.map(|module| {
+ let id = RegisteredModuleId(compiled_id);
+ self.modules.entry(id).or_insert_with(|| module.clone());
+ id
+ });
+
+ // Create a StoreCode if one does not already exist.
+ let store_code_pc = match self.store_code.entry(code.text_range().start) {
+ Entry::Vacant(v) => {
+ let store_code = StoreCode::new(engine, code)?;
+ let store_code_pc = store_code.text_range().start;
+ assert_no_overlap(&self.loaded_code, store_code.text_range());
+ self.loaded_code.insert(
+ store_code_pc,
+ LoadedCode {
+ code: store_code,
+ modules: BTreeMap::default(),
+ },
+ );
+ *v.insert(store_code_pc)
}
- return id;
- }
+ Entry::Occupied(o) => *o.get(),
+ };
- // Assert that this module's code doesn't collide with any other
- // registered modules
- if let Some((_, (prev_start, _))) = self.loaded_code.range(start_addr..).next() {
- assert!(*prev_start > end_addr);
- }
- if let Some((prev_end, _)) = self.loaded_code.range(..=start_addr).next_back() {
- assert!(*prev_end < start_addr);
+ // Add this module to the LoadedCode if not present.
+ if let (Some(module), Some(id)) = (module, id) {
+ if let Some((_, range)) = module.compiled_module().finished_function_ranges().next() {
+ let loaded_code = self
+ .loaded_code
+ .get_mut(&store_code_pc)
+ .expect("loaded_code must have entry for StoreCodePC");
+ loaded_code.modules.insert(range.start, id);
+ }
}
- let mut item = LoadedCode {
- _code: code.clone(),
- modules: Default::default(),
- modules_with_only_trampolines: Vec::new(),
- };
- if let Some(module) = module {
- item.push_module(module);
- }
- let prev = self.loaded_code.insert(end_addr, (start_addr, item));
- assert!(prev.is_none());
- id
+ Ok(id)
}
/// Fetches frame information about a program counter in a backtrace.
@@ -178,10 +203,23 @@ impl ModuleRegistry {
/// debug information due to the compiler's configuration. The second
/// boolean indicates whether the engine used to compile this module is
/// using environment variables to control debuginfo parsing.
- pub(crate) fn lookup_frame_info(&self, pc: usize) -> Option<(FrameInfo, &Module)> {
- let (module, offset) = self.module_and_offset(pc)?;
- let info = FrameInfo::new(module.clone(), offset)?;
- Some((info, module))
+ pub(crate) fn lookup_frame_info<'a>(
+ &'a self,
+ pc: usize,
+ ) -> Option<(FrameInfo, ModuleWithCode<'a>)> {
+ let (_, code) = self
+ .loaded_code
+ .range(..=StoreCodePC::from_raw(pc))
+ .next_back()?;
+ let text_offset = StoreCodePC::offset_of(code.code.text_range(), pc)?;
+ let (_, module_id) = code.modules.range(..=text_offset).next_back()?;
+ let module = self
+ .modules
+ .get(&module_id)
+ .expect("referenced module ID not found");
+ let info = FrameInfo::new(module.clone(), text_offset)?;
+ let module_with_code = ModuleWithCode::from_raw(module, &code.code);
+ Some((info, module_with_code))
}
pub fn wasm_to_array_trampoline(
@@ -195,59 +233,15 @@ impl ModuleRegistry {
// `VMSharedSignatureIndex` to `SignatureIndex` map.
//
// See also the comment in `ModuleInner::wasm_to_native_trampoline`.
- for (_, code) in self.loaded_code.values() {
- for module in code
- .modules
- .values()
- .chain(&code.modules_with_only_trampolines)
- {
- if let Some(trampoline) = module.wasm_to_array_trampoline(sig) {
- return Some(trampoline);
- }
+ for module in self.modules.values() {
+ if let Some(trampoline) = module.wasm_to_array_trampoline(sig) {
+ return Some(trampoline);
}
}
None
}
}
-impl LoadedCode {
- fn push_module(&mut self, module: &Module) {
- let func = match module.compiled_module().finished_functions().next() {
- Some((_, func)) => func,
- // There are no compiled functions in this module so there's no
- // need to push onto `self.modules` which is only used for frame
- // information lookup for a trap which only symbolicates defined
- // functions.
- None => {
- self.modules_with_only_trampolines.push(module.clone());
- return;
- }
- };
- let start = func.as_ptr() as usize;
-
- match self.modules.entry(start) {
- // This module is already present, and it should be the same as
- // `module`.
- Entry::Occupied(m) => {
- debug_assert!(Arc::ptr_eq(&module.inner, &m.get().inner));
- }
- // This module was not already present, so now it's time to insert.
- Entry::Vacant(v) => {
- v.insert(module.clone());
- }
- }
- }
-
- fn module(&self, pc: usize) -> Option<&Module> {
- // The `modules` map is keyed on the start address of the first
- // function in the module, so find the first module whose start address
- // is less than the `pc`. That may be the wrong module but lookup
- // within the module should fail in that case.
- let (_start, module) = self.modules.range(..=pc).next_back()?;
- Some(module)
- }
-}
-
// This is the global code registry that stores information for all loaded code
// objects that are currently in use by any `Store` in the current process.
//
@@ -285,26 +279,24 @@ pub fn lookup_code(pc: usize) -> Option<(Arc<CodeMemory>, usize)> {
/// This is required to enable traps to work correctly since the signal handler
/// will lookup in the `GLOBAL_CODE` list to determine which a particular pc
/// is a trap or not.
-pub fn register_code(code: &Arc<CodeMemory>) {
- let text = code.text();
- if text.is_empty() {
+pub fn register_code(image: &Arc<CodeMemory>, address: Range<usize>) {
+ if address.is_empty() {
return;
}
- let start = text.as_ptr() as usize;
- let end = start + text.len() - 1;
- let prev = global_code().write().insert(end, (start, code.clone()));
+ let start = address.start;
+ let end = address.end - 1;
+ let prev = global_code().write().insert(end, (start, image.clone()));
assert!(prev.is_none());
}
/// Unregisters a code mmap from the global map.
///
/// Must have been previously registered with `register`.
-pub fn unregister_code(code: &Arc<CodeMemory>) {
- let text = code.text();
- if text.is_empty() {
+pub fn unregister_code(address: Range<usize>) {
+ if address.is_empty() {
return;
}
- let end = (text.as_ptr() as usize) + text.len() - 1;
+ let end = address.end - 1;
let code = global_code().write().remove(&end);
assert!(code.is_some());
}
@@ -332,12 +324,12 @@ fn test_frame_info() -> Result<(), anyhow::Error> {
// Create an instance to ensure the frame information is registered.
Instance::new(&mut store, &module, &[])?;
- for (i, alloc) in module.compiled_module().finished_functions() {
- let (start, end) = {
- let ptr = alloc.as_ptr();
- let len = alloc.len();
- (ptr as usize, ptr as usize + len)
- };
+ // Look for frame info for each function. Assume that StoreCode
+ // does not actually clone in the default configuration.
+ for (i, range) in module.compiled_module().finished_function_ranges() {
+ let base = module.engine_code().text_range().start.raw();
+ let start = base + range.start;
+ let end = base + range.end;
for pc in start..end {
let (frame, _) = store
.as_context()
diff --git a/crates/wasmtime/src/runtime/profiling.rs b/crates/wasmtime/src/runtime/profiling.rs
index 2b183a68b7..6baac6064a 100644
--- a/crates/wasmtime/src/runtime/profiling.rs
+++ b/crates/wasmtime/src/runtime/profiling.rs
@@ -1,9 +1,8 @@
#[cfg(feature = "component-model")]
use crate::component::Component;
-use crate::instantiate::CompiledModule;
-use crate::prelude::*;
use crate::runtime::vm::Backtrace;
use crate::{AsContext, CallHook, Module};
+use crate::{Engine, prelude::*};
use core::cmp::Ordering;
use fxprof_processed_profile::debugid::DebugId;
use fxprof_processed_profile::{
@@ -109,10 +108,20 @@ impl GuestProfiler {
/// "Security" section of the [`GuestProfiler`] documentation for guidance
/// on what modules should not be included in this list.
pub fn new(
+ engine: &Engine,
module_name: &str,
interval: Duration,
modules: impl IntoIterator<Item = (String, Module)>,
- ) -> Self {
+ ) -> Result<Self> {
+ // Check that guest debugging is not enabled. The
+ // instrumentation would make profiling results unreliable,
+ // but more fundamentally, it means that code is cloned per
+ // instantiation (for breakpoint patching) so the logic below
+ // is incorrect.
+ if engine.tunables().debug_guest {
+ anyhow::bail!("Profiling cannot be performed when guest-debugging is enabled.");
+ }
+
let zero = ReferenceTimestamp::from_millis_since_unix_epoch(0.0);
let mut profile = Profile::new(module_name, zero, interval.into());
@@ -121,18 +130,21 @@ impl GuestProfiler {
let mut modules: Vec<_> = modules
.into_iter()
.filter_map(|(name, module)| {
+ assert!(Engine::same(module.engine(), engine));
let compiled = module.compiled_module();
let text_range = {
// Assumption: within text, the code for a given module is packed linearly and
// is non-overlapping; if this is violated, it should be safe but might result
// in incorrect profiling results.
- let start =
- compiled.finished_functions().next()?.1.as_ptr_range().start as usize;
- let end = compiled.finished_functions().last()?.1.as_ptr_range().end as usize;
+ let start = compiled.finished_function_ranges().next()?.1.start;
+ let end = compiled.finished_function_ranges().last()?.1.end;
+
+ let start = (module.engine_code().text_range().start + start).raw();
+ let end = (module.engine_code().text_range().start + end).raw();
start..end
};
- module_symbols(name, compiled).map(|lib| {
+ module_symbols(name, &module).map(|lib| {
let libhandle = profile.add_lib(lib);
ProfiledModule {
module,
@@ -150,14 +162,14 @@ impl GuestProfiler {
let thread = profile.add_thread(process, 0, Timestamp::from_nanos_since_reference(0), true);
let start = Instant::now();
let marker = CallMarker::new(&mut profile);
- Self {
+ Ok(Self {
profile,
modules,
process,
thread,
start,
marker,
- }
+ })
}
/// Create a new profiler for the provided component
@@ -167,16 +179,17 @@ impl GuestProfiler {
/// instrumentation to track calls in each of its constituent modules.
#[cfg(feature = "component-model")]
pub fn new_component(
+ engine: &Engine,
component_name: &str,
interval: Duration,
component: Component,
extra_modules: impl IntoIterator<Item = (String, Module)>,
- ) -> Self {
+ ) -> Result<Self> {
let modules = component
.static_modules()
.map(|m| (m.name().unwrap_or("<unknown>").to_string(), m.clone()))
.chain(extra_modules);
- Self::new(component_name, interval, modules)
+ Self::new(engine, component_name, interval, modules)
}
/// Add a sample to the profile. This function collects a backtrace from
@@ -248,23 +261,29 @@ impl GuestProfiler {
}
}
-fn module_symbols(name: String, compiled: &CompiledModule) -> Option<LibraryInfo> {
- let symbols = Vec::from_iter(compiled.finished_functions().map(|(defined_idx, _)| {
- let loc = compiled.func_loc(defined_idx);
- let func_idx = compiled.module().func_index(defined_idx);
- let mut name = String::new();
- demangle_function_name_or_index(
- &mut name,
- compiled.func_name(func_idx),
- defined_idx.as_u32() as usize,
- )
- .unwrap();
- Symbol {
- address: loc.start,
- size: Some(loc.length),
- name,
- }
- }));
+fn module_symbols(name: String, module: &Module) -> Option<LibraryInfo> {
+ let compiled = module.compiled_module();
+ let symbols = Vec::from_iter(
+ module
+ .env_module()
+ .defined_func_indices()
+ .map(|defined_idx| {
+ let loc = compiled.func_loc(defined_idx);
+ let func_idx = compiled.module().func_index(defined_idx);
+ let mut name = String::new();
+ demangle_function_name_or_index(
+ &mut name,
+ compiled.func_name(func_idx),
+ defined_idx.as_u32() as usize,
+ )
+ .unwrap();
+ Symbol {
+ address: loc.start,
+ size: Some(loc.length),
+ name,
+ }
+ }),
+ );
if symbols.is_empty() {
return None;
}
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index 710ff8ca68..f2f00b7fd2 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1616,14 +1616,25 @@ impl StoreOpaque {
&mut self.store_data
}
+ pub fn store_data_mut_and_registry(&mut self) -> (&mut StoreData, &ModuleRegistry) {
+ (&mut self.store_data, &self.modules)
+ }
+
#[inline]
pub(crate) fn modules(&self) -> &ModuleRegistry {
&self.modules
}
- #[inline]
- pub(crate) fn modules_mut(&mut self) -> &mut ModuleRegistry {
- &mut self.modules
+ pub(crate) fn register_module(&mut self, module: &Module) -> Result<RegisteredModuleId> {
+ self.modules.register_module(module, &self.engine)
+ }
+
+ #[cfg(feature = "component-model")]
+ pub(crate) fn register_component(
+ &mut self,
+ component: &crate::component::Component,
+ ) -> Result<()> {
+ self.modules.register_component(component, &self.engine)
}
pub(crate) fn func_refs_and_modules(&mut self) -> (&mut FuncRefs, &ModuleRegistry) {
@@ -1649,7 +1660,7 @@ impl StoreOpaque {
StoreInstanceKind::Real { module_id } => {
let module = self
.modules()
- .lookup_module_by_id(module_id)
+ .module_by_id(module_id)
.expect("should always have a registered module for real instances");
Some(module)
}
@@ -1676,6 +1687,16 @@ impl StoreOpaque {
self.instances[id].handle.get_mut()
}
+ /// Accessor from `InstanceId` to both `Pin<&mut vm::Instance>`
+ /// and `&ModuleRegistry`.
+ #[inline]
+ pub fn instance_and_module_registry_mut(
+ &mut self,
+ id: InstanceId,
+ ) -> (Pin<&mut vm::Instance>, &ModuleRegistry) {
+ (self.instances[id].handle.get_mut(), &self.modules)
+ }
+
/// Access multiple instances specified via `ids`.
///
/// # Panics
@@ -1710,6 +1731,23 @@ impl StoreOpaque {
(self.gc_store.as_mut(), self.instances[id].handle.get_mut())
}
+ /// Tuple of `Self::optional_gc_store_mut`, `Self::modules`, and
+ /// `Self::instance_mut`.
+ pub fn optional_gc_store_and_registry_and_instance_mut(
+ &mut self,
+ id: InstanceId,
+ ) -> (
+ Option<&mut GcStore>,
+ &ModuleRegistry,
+ Pin<&mut vm::Instance>,
+ ) {
+ (
+ self.gc_store.as_mut(),
+ &self.modules,
+ self.instances[id].handle.get_mut(),
+ )
+ }
+
/// Get all instances (ignoring dummy instances) within this store.
pub fn all_instances<'a>(&'a mut self) -> impl ExactSizeIterator<Item = Instance> + 'a {
let instances = self
@@ -2036,12 +2074,12 @@ impl StoreOpaque {
"we should always get a valid frame pointer for Wasm frames"
);
- let module_info = self
+ let (module_with_code, _offset) = self
.modules()
- .lookup_module_by_pc(pc)
+ .module_and_code_by_pc(pc)
.expect("should have module info for Wasm frame");
- if let Some(stack_map) = module_info.lookup_stack_map(pc) {
+ if let Some(stack_map) = module_with_code.lookup_stack_map(pc) {
log::trace!(
"We have a stack map that maps {} bytes in this Wasm frame",
stack_map.frame_size()
@@ -2056,8 +2094,10 @@ impl StoreOpaque {
}
#[cfg(feature = "debug")]
- if let Some(frame_table) = module_info.frame_table() {
- let relpc = module_info.text_offset(pc);
+ if let Some(frame_table) = module_with_code.module().frame_table() {
+ let relpc = module_with_code
+ .text_offset(pc)
+ .expect("PC should be within module");
for stack_slot in super::debug::gc_refs_in_frame(frame_table, relpc, fp) {
unsafe {
self.trace_wasm_stack_slot(gc_roots_list, stack_slot);
diff --git a/crates/wasmtime/src/runtime/store/data.rs b/crates/wasmtime/src/runtime/store/data.rs
index 71599f8794..265e7a1ce8 100644
--- a/crates/wasmtime/src/runtime/store/data.rs
+++ b/crates/wasmtime/src/runtime/store/data.rs
@@ -1,3 +1,4 @@
+use crate::module::ModuleRegistry;
use crate::runtime::vm::{self, GcStore, VMStore};
use crate::store::StoreOpaque;
use crate::{StoreContext, StoreContextMut};
@@ -253,6 +254,21 @@ impl StoreInstanceId {
store.instance_mut(self.instance)
}
+ /// Get both an instance handle and a borrow to the module
+ /// registry in the store.
+ ///
+ /// # Panics
+ ///
+ /// Panics if `self` does not belong to `store`.
+ #[inline]
+ pub(crate) fn get_mut_and_module_registry<'a>(
+ &self,
+ store: &'a mut StoreOpaque,
+ ) -> (Pin<&'a mut vm::Instance>, &'a ModuleRegistry) {
+ self.assert_belongs_to(store.id());
+ store.instance_and_module_registry_mut(self.instance)
+ }
+
/// Same as [`Self::get_mut`], but also returns the `GcStore`.
#[inline]
pub(crate) fn get_with_gc_store_mut<'a>(
diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rs
index 3089e4c4ef..fbe103cb2d 100644
--- a/crates/wasmtime/src/runtime/trap.rs
+++ b/crates/wasmtime/src/runtime/trap.rs
@@ -325,7 +325,8 @@ impl WasmBacktrace {
// do this then we will print out a helpful note in
// `Display` to indicate that more detailed information
// in a trap may be available.
- let has_unparsed_debuginfo = module.compiled_module().has_unparsed_debuginfo();
+ let has_unparsed_debuginfo =
+ module.module().compiled_module().has_unparsed_debuginfo();
if has_unparsed_debuginfo
&& wasm_backtrace_details_env_used
&& cfg!(feature = "addr2line")
@@ -439,10 +440,8 @@ impl FrameInfo {
let compiled_module = module.compiled_module();
let index = compiled_module.func_by_text_offset(text_offset)?;
let func_start = compiled_module.func_start_srcloc(index);
- let instr = wasmtime_environ::lookup_file_pos(
- compiled_module.code_memory().address_map_data(),
- text_offset,
- );
+ let instr =
+ wasmtime_environ::lookup_file_pos(module.engine_code().address_map_data(), text_offset);
let index = compiled_module.module().func_index(index);
let func_index = index.as_u32();
let func_name = compiled_module.func_name(index).map(|s| s.to_string());
diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs
index 247cea7ac1..a7c597affe 100644
--- a/crates/wasmtime/src/runtime/vm.rs
+++ b/crates/wasmtime/src/runtime/vm.rs
@@ -44,9 +44,7 @@ use core::pin::pin;
use core::ptr::NonNull;
use core::sync::atomic::{AtomicUsize, Ordering};
use core::task::{Context, Poll, Waker};
-use wasmtime_environ::{
- DefinedFuncIndex, DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex,
-};
+use wasmtime_environ::{DefinedMemoryIndex, HostPtr, VMOffsets, VMSharedTypeIndex};
#[cfg(feature = "gc")]
use wasmtime_environ::ModuleInternedTypeIndex;
@@ -124,10 +122,12 @@ pub use crate::runtime::vm::table::{Table, TableElementType};
#[cfg(feature = "gc")]
pub use crate::runtime::vm::throw::*;
pub use crate::runtime::vm::traphandlers::*;
+#[cfg(feature = "component-model")]
+pub use crate::runtime::vm::vmcontext::VMArrayCallFunction;
pub use crate::runtime::vm::vmcontext::{
- VMArrayCallFunction, VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport,
- VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport,
- VMOpaqueContext, VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,
+ VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition,
+ VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext,
+ VMStoreContext, VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,
};
#[cfg(has_custom_sync)]
pub(crate) use sys::capi;
@@ -344,7 +344,7 @@ impl ModuleRuntimeInfo {
fn engine_type_index(&self, module_index: ModuleInternedTypeIndex) -> VMSharedTypeIndex {
match self {
ModuleRuntimeInfo::Module(m) => m
- .code_object()
+ .engine_code()
.signatures()
.shared_type(module_index)
.expect("bad module-level interned type index"),
@@ -352,38 +352,6 @@ impl ModuleRuntimeInfo {
}
}
- /// Returns the address, in memory, that the function `index` resides at.
- fn function(&self, index: DefinedFuncIndex) -> NonNull<VMWasmCallFunction> {
- let module = match self {
- ModuleRuntimeInfo::Module(m) => m,
- ModuleRuntimeInfo::Bare(_) => unreachable!(),
- };
- let ptr = module
- .compiled_module()
- .finished_function(index)
- .as_ptr()
- .cast::<VMWasmCallFunction>()
- .cast_mut();
- NonNull::new(ptr).unwrap()
- }
-
- /// Returns the address, in memory, of the trampoline that allows the given
- /// defined Wasm function to be called by the array calling convention.
- ///
- /// Returns `None` for Wasm functions which do not escape, and therefore are
- /// not callable from outside the Wasm module itself.
- fn array_to_wasm_trampoline(
- &self,
- index: DefinedFuncIndex,
- ) -> Option<NonNull<VMArrayCallFunction>> {
- let m = match self {
- ModuleRuntimeInfo::Module(m) => m,
- ModuleRuntimeInfo::Bare(_) => unreachable!(),
- };
- let ptr = NonNull::from(m.compiled_module().array_to_wasm_trampoline(index)?);
- Some(ptr.cast())
- }
-
/// Returns the `MemoryImage` structure used for copy-on-write
/// initialization of the memory, if it's applicable.
fn memory_image(
@@ -413,7 +381,7 @@ impl ModuleRuntimeInfo {
/// A slice pointing to all data that is referenced by this instance.
fn wasm_data(&self) -> &[u8] {
match self {
- ModuleRuntimeInfo::Module(m) => m.compiled_module().code_memory().wasm_data(),
+ ModuleRuntimeInfo::Module(m) => m.engine_code().wasm_data(),
ModuleRuntimeInfo::Bare(_) => &[],
}
}
@@ -423,7 +391,7 @@ impl ModuleRuntimeInfo {
fn type_ids(&self) -> &[VMSharedTypeIndex] {
match self {
ModuleRuntimeInfo::Module(m) => m
- .code_object()
+ .engine_code()
.signatures()
.as_module_map()
.values()
diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs
index 94a0889c22..07148acac3 100644
--- a/crates/wasmtime/src/runtime/vm/component.rs
+++ b/crates/wasmtime/src/runtime/vm/component.rs
@@ -7,6 +7,7 @@
//! cranelift-compiled adapters, will use this `VMComponentContext` as well.
use crate::component::{Component, Instance, InstancePre, ResourceType, RuntimeImport};
+use crate::module::ModuleRegistry;
use crate::runtime::component::ComponentInstanceId;
use crate::runtime::vm::instance::{InstanceLayout, OwnedInstance, OwnedVMContext};
use crate::runtime::vm::vmcontext::VMFunctionBody;
@@ -368,6 +369,7 @@ impl ComponentInstance {
/// Returns the `Func` at index `func_idx` in the funcref table at `table_idx`.
pub fn index_runtime_func_table(
&self,
+ registry: &ModuleRegistry,
table_idx: RuntimeTableIndex,
func_idx: u64,
) -> Result<Option<Func>> {
@@ -381,7 +383,8 @@ impl ComponentInstance {
// SAFETY: We just constructed `instance_ptr` from a valid pointer. This pointer won't leave
// this call, so we don't need a lifetime to bind it to.
let instance = Pin::new_unchecked(instance_ptr.as_mut());
- let table = instance.get_defined_table_with_lazy_init(table.index, [func_idx]);
+ let table =
+ instance.get_defined_table_with_lazy_init(registry, table.index, [func_idx]);
let func = table
.get_func(func_idx)?
.map(|funcref| Func::from_vm_func_ref(store.id(), funcref));
diff --git a/crates/wasmtime/src/runtime/vm/const_expr.rs b/crates/wasmtime/src/runtime/vm/const_expr.rs
index ae4cd0d638..4716588088 100644
--- a/crates/wasmtime/src/runtime/vm/const_expr.rs
+++ b/crates/wasmtime/src/runtime/vm/const_expr.rs
@@ -51,13 +51,10 @@ impl ConstEvalContext {
fn ref_func(&mut self, store: &mut StoreOpaque, index: FuncIndex) -> Result<Val> {
let id = store.id();
+ let (instance, registry) = store.instance_and_module_registry_mut(self.instance);
// SAFETY: `id` is the correct store-owner of the function being looked
// up
- let func = unsafe {
- store
- .instance_mut(self.instance)
- .get_exported_func(id, index)
- };
+ let func = unsafe { instance.get_exported_func(registry, id, index) };
Ok(func.into())
}
diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs
index dedf27b79e..38b9df44bb 100644
--- a/crates/wasmtime/src/runtime/vm/instance.rs
+++ b/crates/wasmtime/src/runtime/vm/instance.rs
@@ -3,6 +3,8 @@
//! `InstanceHandle` is a reference-counting handle for an `Instance`.
use crate::OpaqueRootScope;
+use crate::code::ModuleWithCode;
+use crate::module::ModuleRegistry;
use crate::prelude::*;
use crate::runtime::vm::const_expr::{ConstEvalContext, ConstExprEvaluator};
use crate::runtime::vm::export::{Export, ExportMemory};
@@ -18,6 +20,7 @@ use crate::runtime::vm::{
VMStoreRawPtr, VmPtr, VmSafe, WasmFault, catch_unwind_and_record_trap,
};
use crate::store::{InstanceId, StoreId, StoreInstanceId, StoreOpaque, StoreResourceLimiter};
+use crate::vm::VMWasmCallFunction;
use alloc::sync::Arc;
use core::alloc::Layout;
use core::marker;
@@ -346,7 +349,6 @@ impl Instance {
self.runtime_info.env_module()
}
- #[cfg(any(feature = "gc", feature = "debug"))]
pub(crate) fn runtime_module(&self) -> Option<&crate::Module> {
match &self.runtime_info {
ModuleRuntimeInfo::Module(m) => Some(m),
@@ -550,10 +552,11 @@ impl Instance {
/// functions that this instance can reference.
pub unsafe fn get_exported_func(
self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
store: StoreId,
index: FuncIndex,
) -> crate::Func {
- let func_ref = self.get_func_ref(index).unwrap();
+ let func_ref = self.get_func_ref(registry, index).unwrap();
// SAFETY: the validity of `func_ref` is guaranteed by the validity of
// `self`, and the contract that `store` must own `func_ref` is a
@@ -780,18 +783,38 @@ impl Instance {
/// This functions requires that `into` is a valid pointer.
unsafe fn construct_func_ref(
self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
index: FuncIndex,
type_index: VMSharedTypeIndex,
into: *mut VMFuncRef,
) {
+ let module_with_code = ModuleWithCode::in_store(
+ registry,
+ self.runtime_module()
+ .expect("funcref impossible in fake module"),
+ )
+ .expect("module not in store");
+
let func_ref = if let Some(def_index) = self.env_module().defined_func_index(index) {
VMFuncRef {
- array_call: self
- .runtime_info
- .array_to_wasm_trampoline(def_index)
- .expect("should have array-to-Wasm trampoline for escaping function")
+ array_call: NonNull::from(
+ module_with_code
+ .array_to_wasm_trampoline(def_index)
+ .expect("should have array-to-Wasm trampoline for escaping function"),
+ )
+ .cast()
+ .into(),
+ wasm_call: Some(
+ NonNull::new(
+ module_with_code
+ .finished_function(def_index)
+ .as_ptr()
+ .cast::<VMWasmCallFunction>()
+ .cast_mut(),
+ )
+ .unwrap()
.into(),
- wasm_call: Some(self.runtime_info.function(def_index).into()),
+ ),
vmctx: VMOpaqueContext::from_vmcontext(self.vmctx()).into(),
type_index,
}
@@ -820,6 +843,7 @@ impl Instance {
/// be passed into JIT code.
pub(crate) fn get_func_ref(
self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
index: FuncIndex,
) -> Option<NonNull<VMFuncRef>> {
if index == FuncIndex::reserved_value() {
@@ -862,7 +886,7 @@ impl Instance {
// SAFETY: the `func_ref` ptr should be valid as it's within our
// `VMContext` area.
unsafe {
- self.construct_func_ref(index, sig, func_ref.as_ptr());
+ self.construct_func_ref(registry, index, sig, func_ref.as_ptr());
}
Some(func_ref)
@@ -978,13 +1002,11 @@ impl Instance {
.and_then(|s| s.get(..len))
.ok_or(Trap::TableOutOfBounds)?;
for (i, func_idx) in positions.zip(elements) {
+ let (instance, registry) =
+ store.instance_and_module_registry_mut(elements_instance_id);
// SAFETY: the `store_id` passed to `get_exported_func` is
// indeed the store that owns the function.
- let func = unsafe {
- store
- .instance_mut(elements_instance_id)
- .get_exported_func(store_id, *func_idx)
- };
+ let func = unsafe { instance.get_exported_func(registry, store_id, *func_idx) };
table.set_(store, i, func.into()).unwrap();
}
}
@@ -1196,11 +1218,12 @@ impl Instance {
/// works correctly.
pub(crate) fn get_table_with_lazy_init(
self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
table_index: TableIndex,
range: impl Iterator<Item = u64>,
) -> &mut Table {
let (idx, instance) = self.defined_table_index_and_instance(table_index);
- instance.get_defined_table_with_lazy_init(idx, range)
+ instance.get_defined_table_with_lazy_init(registry, idx, range)
}
/// Gets the raw runtime table data structure owned by this instance
@@ -1209,6 +1232,7 @@ impl Instance {
/// The `range` specified is eagerly initialized for funcref tables.
pub fn get_defined_table_with_lazy_init(
mut self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
idx: DefinedTableIndex,
range: impl IntoIterator<Item = u64>,
) -> &mut Table {
@@ -1241,8 +1265,8 @@ impl Instance {
};
// Panicking here helps catch bugs rather than silently truncating by accident.
let func_index = precomputed.get(usize::try_from(i).unwrap()).cloned();
- let func_ref =
- func_index.and_then(|func_index| self.as_mut().get_func_ref(func_index));
+ let func_ref = func_index
+ .and_then(|func_index| self.as_mut().get_func_ref(registry, func_index));
self.as_mut().tables_mut()[idx]
.1
.set_func(i, func_ref)
@@ -1529,6 +1553,7 @@ impl Instance {
/// instance.
pub unsafe fn get_export_by_index_mut(
self: Pin<&mut Self>,
+ registry: &ModuleRegistry,
store: StoreId,
export: EntityIndex,
) -> Export {
@@ -1536,7 +1561,7 @@ impl Instance {
// SAFETY: the contract of `store` owning the this instance is a
// safety requirement of this function itself.
EntityIndex::Function(i) => {
- Export::Function(unsafe { self.get_exported_func(store, i) })
+ Export::Function(unsafe { self.get_exported_func(registry, store, i) })
}
EntityIndex::Global(i) => Export::Global(self.get_exported_global(store, i)),
EntityIndex::Table(i) => Export::Table(self.get_exported_table(store, i)),
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 5c384f9b20..70d5856f43 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -562,9 +562,9 @@ fn memory_init(
// Implementation of `ref.func`.
fn ref_func(store: &mut dyn VMStore, instance: InstanceId, func_index: u32) -> NonNull<u8> {
- store
- .instance_mut(instance)
- .get_func_ref(FuncIndex::from_u32(func_index))
+ let (instance, registry) = store.instance_and_module_registry_mut(instance);
+ instance
+ .get_func_ref(registry, FuncIndex::from_u32(func_index))
.expect("ref_func: funcref should always be available for given func index")
.cast()
}
@@ -583,9 +583,8 @@ fn table_get_lazy_init_func_ref(
index: u64,
) -> *mut u8 {
let table_index = TableIndex::from_u32(table_index);
- let table = store
- .instance_mut(instance)
- .get_table_with_lazy_init(table_index, core::iter::once(index));
+ let (instance, registry) = store.instance_and_module_registry_mut(instance);
+ let table = instance.get_table_with_lazy_init(registry, table_index, core::iter::once(index));
let elem = table
.get_func(index)
.expect("table access already bounds-checked");
@@ -962,14 +961,14 @@ fn array_new_elem(
match elements {
TableSegmentElements::Functions(fs) => {
let store_id = store.id();
- let mut instance = store.instance_mut(instance_id);
+ let (mut instance, registry) = store.instance_and_module_registry_mut(instance_id);
vals.extend(
fs.get(src..)
.and_then(|s| s.get(..len))
.ok_or_else(|| Trap::TableOutOfBounds)?
.iter()
.map(|f| {
- let raw_func_ref = instance.as_mut().get_func_ref(*f);
+ let raw_func_ref = instance.as_mut().get_func_ref(registry, *f);
let func = unsafe {
raw_func_ref.map(|p| Func::from_vm_func_ref(store_id, p))
};
@@ -1052,7 +1051,7 @@ fn array_init_elem(
// Get the passive element segment.
let mut storage = None;
let store_id = store.id();
- let mut instance = store.instance_mut(instance);
+ let (mut instance, registry) = store.instance_and_module_registry_mut(instance);
let elements = instance.passive_element_segment(&mut storage, elem_index);
// Convert array offsets into `usize`s.
@@ -1067,7 +1066,7 @@ fn array_init_elem(
.ok_or_else(|| Trap::TableOutOfBounds)?
.iter()
.map(|f| {
- let raw_func_ref = instance.as_mut().get_func_ref(*f);
+ let raw_func_ref = instance.as_mut().get_func_ref(registry, *f);
let func = unsafe { raw_func_ref.map(|p| Func::from_vm_func_ref(store_id, p)) };
Val::FuncRef(func)
})
diff --git a/crates/wasmtime/src/runtime/vm/mmap_vec.rs b/crates/wasmtime/src/runtime/vm/mmap_vec.rs
index f64de50ab8..9bebbf4d72 100644
--- a/crates/wasmtime/src/runtime/vm/mmap_vec.rs
+++ b/crates/wasmtime/src/runtime/vm/mmap_vec.rs
@@ -270,6 +270,33 @@ impl MmapVec {
MmapVec::Mmap { mmap, len } => unsafe { mmap.slice_mut(0..*len) },
}
}
+
+ /// Create a copy of this `MmapVec` that can be separately
+ /// mutated.
+ #[cfg(feature = "debug")]
+ pub(crate) fn deep_clone(&self) -> Result<MmapVec> {
+ match self {
+ #[cfg(not(has_virtual_memory))]
+ MmapVec::Alloc { layout, .. } => {
+ MmapVec::from_slice_with_alignment(&self[..], layout.align())
+ }
+ MmapVec::ExternallyOwned { .. } => {
+ anyhow::bail!("Cannot clone an externally-owned code memory.");
+ }
+ #[cfg(has_virtual_memory)]
+ MmapVec::Mmap { mmap, len } => {
+ if let Some(original_file) = mmap.original_file() {
+ let mmap = Mmap::from_file(original_file.clone())?;
+ Ok(MmapVec::Mmap { mmap, len: *len })
+ } else {
+ MmapVec::from_slice_with_alignment(
+ &self[..],
+ crate::runtime::vm::host_page_size(),
+ )
+ }
+ }
+ }
+ }
}
impl Deref for MmapVec {
diff --git a/crates/wasmtime/src/runtime/vm/throw.rs b/crates/wasmtime/src/runtime/vm/throw.rs
index ba5105fe11..5463e930b4 100644
--- a/crates/wasmtime/src/runtime/vm/throw.rs
+++ b/crates/wasmtime/src/runtime/vm/throw.rs
@@ -54,11 +54,9 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {
frame.fp(),
frame.pc()
);
- let module = nogc.modules().lookup_module_by_pc(frame.pc())?;
- let base = module.code_object().code_memory().text().as_ptr() as usize;
- let rel_pc = u32::try_from(frame.pc().wrapping_sub(base)).expect("Module larger than 4GiB");
- let et = module.exception_table();
- let (frame_offset, handlers) = et.lookup_pc(rel_pc);
+ let (module, rel_pc) = nogc.modules().module_and_code_by_pc(frame.pc())?;
+ let et = module.module().exception_table();
+ let (frame_offset, handlers) = et.lookup_pc(u32::try_from(rel_pc).unwrap());
let fp_to_sp = frame_offset.map(|frame_offset| -isize::try_from(frame_offset).unwrap());
for handler in handlers {
log::trace!("-> checking handler: {handler:?}");
@@ -112,9 +110,10 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {
if is_match {
let fp_to_sp = fp_to_sp.expect("frame offset must be known if we found a handler");
return Some((
- base.wrapping_add(
- usize::try_from(handler.handler_offset).expect("Module larger than usize"),
- ),
+ (module.store_code().text_range().start
+ + usize::try_from(handler.handler_offset)
+ .expect("Module larger than usize"))
+ .raw(),
frame.fp().wrapping_add_signed(fp_to_sp),
));
}
diff --git a/src/commands/run.rs b/src/commands/run.rs
index bfe2fb99f4..3f4e2196a5 100644
--- a/src/commands/run.rs
+++ b/src/commands/run.rs
@@ -355,7 +355,7 @@ impl RunCommand {
profiled_modules,
path,
*interval,
- ));
+ )?);
#[cfg(not(feature = "profiling"))]
{
let _ = (profiled_modules, path, interval, main_target);
@@ -383,22 +383,24 @@ impl RunCommand {
profiled_modules: Vec<(String, Module)>,
path: &str,
interval: std::time::Duration,
- ) -> Box<dyn FnOnce(&mut Store<Host>)> {
+ ) -> Result<Box<dyn FnOnce(&mut Store<Host>)>> {
use wasmtime::{AsContext, GuestProfiler, StoreContext, StoreContextMut, UpdateDeadline};
let module_name = self.module_and_args[0].to_str().unwrap_or("<main module>");
store.data_mut().guest_profiler = match main_target {
RunTarget::Core(_m) => Some(Arc::new(GuestProfiler::new(
+ store.engine(),
module_name,
interval,
profiled_modules,
- ))),
+ )?)),
RunTarget::Component(component) => Some(Arc::new(GuestProfiler::new_component(
+ store.engine(),
module_name,
interval,
component.clone(),
profiled_modules,
- ))),
+ )?)),
};
fn sample(
@@ -450,7 +452,7 @@ impl RunCommand {
});
let path = path.to_string();
- return Box::new(move |store| {
+ Ok(Box::new(move |store| {
let profiler = Arc::try_unwrap(store.data_mut().guest_profiler.take().unwrap())
.expect("profiling doesn't support threads yet");
if let Err(e) = std::fs::File::create(&path)
@@ -463,7 +465,7 @@ impl RunCommand {
eprintln!("Profile written to: {path}");
eprintln!("View this profile at https://profiler.firefox.com/.");
}
- });
+ }))
}
async fn load_main_module(
diff --git a/src/commands/serve.rs b/src/commands/serve.rs
index 00f5e2a89c..e97a1769ad 100644
--- a/src/commands/serve.rs
+++ b/src/commands/serve.rs
@@ -769,11 +769,12 @@ fn setup_guest_profiler(
let module_name = "<main>";
store.data_mut().guest_profiler = Some(Arc::new(GuestProfiler::new_component(
+ store.engine(),
module_name,
interval,
component,
std::iter::empty(),
- )));
+ )?));
fn sample(
mut store: StoreContextMut<Host>,