A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Refactor `ModuleRegistry` to not rely on image layout (#13483)
Alex Crichton6687e5265877
diff --git a/crates/wasmtime/src/runtime/code.rs b/crates/wasmtime/src/runtime/code.rsindex 484fa9373e..3d6af317c3 100644--- a/crates/wasmtime/src/runtime/code.rs+++ b/crates/wasmtime/src/runtime/code.rs@@ -215,12 +215,6 @@ impl EngineCode {self.original_code.exception_tables()}- /// Returns the encoded frame-tables section to pass to- /// `wasmtime_environ::FrameTable::parse`.- pub fn frame_tables(&self) -> &[u8] {- self.original_code.frame_tables()- }-/// Returns the data in the `ELF_NAME_DATA` section.#[inline]pub fn func_name_data(&self) -> &[u8] {@@ -262,6 +256,12 @@ impl EngineCode {pub(crate) fn module_memory_image_source(&self) -> &Arc<impl ModuleMemoryImageSource> {&self.original_code}++ /// See [`CodeMemory::frame_table`].+ #[cfg(feature = "debug")]+ pub(crate) fn frame_table<'a>(&'a self) -> Option<wasmtime_environ::FrameTable<'a>> {+ self.original_code.frame_table()+ }}impl Drop for EngineCode {@@ -430,23 +430,6 @@ impl<'a> ModuleWithCode<'a> {self.module}- /// Provide the StoreCode wrapped in this tuple.- pub fn store_code(&self) -> &'a StoreCode {- self.store_code- }-- /// Returns an iterator over all functions defined within this module with- /// their index and their raw pointer.- #[inline]- pub fn finished_functions(- &self,- ) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, &[u8])> + '_ {- self.module- .env_module()- .defined_func_indices()- .map(|i| (i, self.finished_function(i)))- }-/// Returns the slice in the text section of the function that/// `index` points to.#[inline]@@ -474,20 +457,4 @@ impl<'a> ModuleWithCode<'a> {.array_to_wasm_trampoline_range(def_func_index)?;Some(&self.store_code.text()[range])}-- /// Get the text offset (relative PC) for a given absolute PC in- /// this module.- #[cfg(feature = "gc")]- pub(crate) fn text_offset(&self, pc: usize) -> Option<u32> {- StoreCodePC::offset_of(self.store_code.text_range(), pc)- .map(|offset| u32::try_from(offset).expect("Module larger than 4GiB"))- }-- /// Lookup the stack map at a program counter value.- #[cfg(feature = "gc")]- pub(crate) fn lookup_stack_map(&self, pc: usize) -> Option<wasmtime_environ::StackMap<'_>> {- let text_offset = self.text_offset(pc)?;- let info = self.module.engine_code().stack_map_data();- wasmtime_environ::StackMap::lookup(text_offset, info)- }}diff --git a/crates/wasmtime/src/runtime/code_memory.rs b/crates/wasmtime/src/runtime/code_memory.rsindex 3aee69d991..6078b5091c 100644--- a/crates/wasmtime/src/runtime/code_memory.rs+++ b/crates/wasmtime/src/runtime/code_memory.rs@@ -633,6 +633,22 @@ impl CodeMemory {let mmap = self.mmap.deep_clone()?;Self::new(engine, mmap)}++ /// Obtain a frame-table parser on this module's frame state slot+ /// (debug instrumentation) metadata.+ #[cfg(feature = "debug")]+ pub(crate) fn frame_table(&self) -> Option<wasmtime_environ::FrameTable<'_>> {+ let data = self.frame_tables();+ if data.is_empty() {+ None+ } else {+ let orig_text = self.text();+ Some(+ wasmtime_environ::FrameTable::parse(data, orig_text)+ .expect("Frame tables were validated on module load"),+ )+ }+ }}fn section_name<'a>(diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rsindex 00f519bb70..489caf0a8e 100644--- a/crates/wasmtime/src/runtime/component/component.rs+++ b/crates/wasmtime/src/runtime/component/component.rs@@ -933,6 +933,10 @@ impl Component {_ => unreachable!(),}}++ pub(crate) fn index(&self) -> &Arc<CompiledFunctionsTable> {+ &self.inner.index+ }}/// A value which represents a known export of a component.diff --git a/crates/wasmtime/src/runtime/debug.rs b/crates/wasmtime/src/runtime/debug.rsindex c345de75c4..08cb703b89 100644--- a/crates/wasmtime/src/runtime/debug.rs+++ b/crates/wasmtime/src/runtime/debug.rs@@ -645,10 +645,10 @@ impl FrameDataCache {// module that actually contains the physical PC// (i.e., the outermost function that inlined the// others).- let (module, frames) = VirtualFrame::decode(registry, frame.pc());+ let (store_code, frames) = VirtualFrame::decode(registry, frame.pc());let frames = frames.into_iter()- .map(|frame| FrameData::compute(frame, &module))+ .map(|frame| FrameData::compute(frame, store_code)).collect::<Vec<_>>();v.insert(frames)}@@ -672,18 +672,17 @@ struct VirtualFrame {impl VirtualFrame {/// Return virtual frames corresponding to a physical frame, from/// outermost to innermost.- fn decode(registry: &ModuleRegistry, pc: usize) -> (Module, Vec<VirtualFrame>) {- let (module_with_code, pc) = registry- .module_and_code_by_pc(pc)+ fn decode(registry: &ModuleRegistry, pc: usize) -> (&StoreCode, Vec<VirtualFrame>) {+ let (store_code, pc) = registry+ .store_code_by_pc(pc).expect("Wasm frame PC does not correspond to a module");- let module = module_with_code.module();- let table = module.frame_table().unwrap();+ let table = store_code.code_memory().frame_table().unwrap();let pc = u32::try_from(pc).expect("PC offset too large");let program_points = table.find_program_point(pc, FrameInstPos::Post).expect("There must be a program point record in every frame when debug instrumentation is enabled");(- module.clone(),+ store_code,program_points.map(|(wasm_pc, frame_descriptor, stack_shape)| VirtualFrame {wasm_pc,@@ -719,8 +718,8 @@ struct FrameData {}impl FrameData {- fn compute(frame: VirtualFrame, module: &Module) -> Self {- let frame_table = module.frame_table().unwrap();+ fn compute(frame: VirtualFrame, store_code: &StoreCode) -> Self {+ let frame_table = store_code.code_memory().frame_table().unwrap();// Parse the frame descriptor.let (data, slot_to_fp_offset) = frame_table.frame_descriptor(frame.frame_descriptor)diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rsindex d935a7a169..188d005e1f 100644--- a/crates/wasmtime/src/runtime/instantiate.rs+++ b/crates/wasmtime/src/runtime/instantiate.rs@@ -267,6 +267,10 @@ impl CompiledModule {self.engine_code.wasm_bytecode_for_module(self.module.module_index)}++ pub(crate) fn index(&self) -> &Arc<CompiledFunctionsTable> {+ &self.index+ }}#[cfg(feature = "addr2line")]diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rsindex bc6bcfcb4e..9b4294f34f 100644--- a/crates/wasmtime/src/runtime/module.rs+++ b/crates/wasmtime/src/runtime/module.rs@@ -18,14 +18,10 @@ use core::ptr::NonNull;#[cfg(feature = "std")]use std::{fs::File, path::Path};use wasmparser::{Parser, ValidPayload, Validator};-#[cfg(feature = "debug")]-use wasmtime_environ::FrameTable;use wasmtime_environ::{CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind,StaticModuleIndex, TypeTrace, VMOffsets, VMSharedTypeIndex, WasmChecksum,};-#[cfg(feature = "gc")]-use wasmtime_unwinder::ExceptionTable;mod registry;pub use registry::*;@@ -1178,27 +1174,10 @@ impl Module {Ok(images)}- /// Obtain an exception-table parser on this module's exception metadata.- #[cfg(feature = "gc")]- pub(crate) fn exception_table<'a>(&'a self) -> ExceptionTable<'a> {- ExceptionTable::parse(self.inner.code.exception_tables())- .expect("Exception tables were validated on module load")- }-- /// Obtain a frame-table parser on this module's frame state slot- /// (debug instrumentation) metadata.+ /// See [`CodeMemory::frame_table`].#[cfg(feature = "debug")]- pub(crate) fn frame_table<'a>(&'a self) -> Option<FrameTable<'a>> {- let data = self.inner.code.frame_tables();- if data.is_empty() {- None- } else {- let orig_text = self.inner.code.text();- Some(- FrameTable::parse(data, orig_text)- .expect("Frame tables were validated on module load"),- )- }+ pub(crate) fn frame_table<'a>(&'a self) -> Option<wasmtime_environ::FrameTable<'a>> {+ self.inner.code.frame_table()}/// Is this `Module` the same as another?@@ -1216,6 +1195,10 @@ impl Module {pub fn same(a: &Module, b: &Module) -> bool {Arc::ptr_eq(&a.inner, &b.inner)}++ pub(crate) fn index(&self) -> &Arc<CompiledFunctionsTable> {+ &self.inner.module.index()+ }}/// Describes a function for a given module.diff --git a/crates/wasmtime/src/runtime/module/registry.rs b/crates/wasmtime/src/runtime/module/registry.rsindex c16d12cb6c..cd05f1b44b 100644--- a/crates/wasmtime/src/runtime/module/registry.rs+++ b/crates/wasmtime/src/runtime/module/registry.rs@@ -12,7 +12,10 @@ use alloc::sync::Arc;use core::marker::PhantomData;use core::ops::Range;use core::ptr::NonNull;-use wasmtime_environ::{VMSharedTypeIndex, collections::btree_map::Entry};+use wasmtime_environ::{+ CompiledFunctionsTable, FuncKey, StaticModuleIndex, VMSharedTypeIndex,+ collections::btree_map::Entry,+};/// Used for registering modules with a store.///@@ -79,8 +82,19 @@ struct LoadedCode {/// The StoreCode in this range.code: StoreCode,- /// Map by starting text offset of Modules in this code region.- modules: TryBTreeMap<usize, RegisteredModuleId>,+ /// The index of compiled functions in `code`.+ ///+ /// This index is used to map from program counters back to a `FuncKey`.+ /// Primarily used in [`ModuleRegistry::lookup_frame_info`] at this time+ /// below.+ index: Arc<CompiledFunctionsTable>,++ /// A mapping from the [`StaticModuleIndex`] values returned by the `index`+ /// field above to a module within this registry.+ ///+ /// This is lazily populated as modules are registered and instantiated from+ /// a component.+ modules: TrySecondaryMap<StaticModuleIndex, Option<RegisteredModuleId>>,}/// An identifier of a module that has previously been inserted into a@@ -119,6 +133,12 @@ impl<'a> RegisterBreakpointState<'a> {}}+enum ModuleOrComponent<'a> {+ Module(&'a Module),+ #[cfg(feature = "component-model")]+ Component(&'a Component),+}+impl ModuleRegistry {/// Get a previously-registered module by id.pub fn module_by_id(&self, id: RegisteredModuleId) -> Option<&Module> {@@ -130,17 +150,29 @@ impl ModuleRegistry {self.modules.get(RegisteredModuleId(id))}- /// Fetches a registered StoreCode and module and an offset within- /// it given a program counter value.- pub fn module_and_code_by_pc<'a>(&'a self, pc: usize) -> Option<(ModuleWithCode<'a>, usize)> {+ /// Looks up `pc`, an absolute program counter address, to see if it+ /// corresponds to any loaded code within this [`ModuleRegistry`].+ ///+ /// Returns `None` if nothing is found, and otherwise returns the+ /// corresponding [`LoadedCode`] as well as a relative pc from the start of+ /// the code's text section if found.+ fn loaded_code_by_pc(&self, pc: usize) -> Option<(&LoadedCode, usize)> {let (_, code) = self.loaded_code.range(..=StoreCodePC::from_raw(pc)).next_back()?;let offset = StoreCodePC::offset_of(code.code.text_range(), pc)?;- let (_, module_id) = code.modules.range(..=offset).next_back()?;- let module = self.modules.get(*module_id)?;- Some((ModuleWithCode::from_raw(module, &code.code), offset))+ Some((code, offset))+ }++ /// Consults this [`ModuleRegistry`] to see if `pc` corrseponds to any+ /// previously-registered block of code.+ ///+ /// Upon success returns the [`StoreCode`] as well as a relative pc from the+ /// start of the text section.+ pub fn store_code_by_pc(&self, pc: usize) -> Option<(&StoreCode, usize)> {+ let (code, pc) = self.loaded_code_by_pc(pc)?;+ Some((&code.code, pc))}/// Fetches the `StoreCode` for a given `EngineCode`.@@ -190,14 +222,8 @@ impl ModuleRegistry {engine: &Engine,breakpoint_state: RegisterBreakpointState,) -> Result<RegisteredModuleId> {- self.register(- module.id(),- module.engine_code(),- Some(module),- engine,- breakpoint_state,- )- .map(|id| id.unwrap())+ self.register(ModuleOrComponent::Module(module), engine, breakpoint_state)+ .map(|id| id.unwrap())}#[cfg(feature = "component-model")]@@ -208,9 +234,7 @@ impl ModuleRegistry {breakpoint_state: RegisterBreakpointState,) -> Result<()> {self.register(- component.id(),- component.engine_code(),- None,+ ModuleOrComponent::Component(component),engine,breakpoint_state,)?;@@ -220,19 +244,29 @@ impl ModuleRegistry {/// Registers a new module with the registry.fn register(&mut self,- compiled_id: CompiledModuleId,- code: &Arc<EngineCode>,- module: Option<&Module>,+ module_or_component: ModuleOrComponent<'_>,engine: &Engine,breakpoint_state: RegisterBreakpointState,) -> Result<Option<RegisteredModuleId>> {+ let compiled_id = match module_or_component {+ ModuleOrComponent::Module(module) => module.id(),+ #[cfg(feature = "component-model")]+ ModuleOrComponent::Component(component) => component.id(),+ };+ let code = match module_or_component {+ ModuleOrComponent::Module(module) => module.engine_code(),+ #[cfg(feature = "component-model")]+ ModuleOrComponent::Component(component) => component.engine_code(),+ };// Register the module, if any.- let id = if let Some(module) = module {- let id = RegisteredModuleId(compiled_id);- self.modules.entry(id).or_insert_with(|| module.clone())?;- Some(id)- } else {- None+ let id = match module_or_component {+ ModuleOrComponent::Module(module) => {+ let id = RegisteredModuleId(compiled_id);+ self.modules.entry(id).or_insert_with(|| module.clone())?;+ Some(id)+ }+ #[cfg(feature = "component-model")]+ ModuleOrComponent::Component(_) => None,};// Create a StoreCode if one does not already exist.@@ -241,11 +275,17 @@ impl ModuleRegistry {let store_code = StoreCode::new(engine, code)?;let store_code_pc = store_code.text_range().start;assert_no_overlap(&self.loaded_code, store_code.text_range());+ let index = match module_or_component {+ ModuleOrComponent::Module(module) => module.index(),+ #[cfg(feature = "component-model")]+ ModuleOrComponent::Component(component) => component.index(),+ };self.loaded_code.insert(store_code_pc,LoadedCode {code: store_code,- modules: TryBTreeMap::default(),+ index: index.clone(),+ modules: Default::default(),},)?;*v.insert(store_code_pc)?@@ -254,15 +294,15 @@ impl ModuleRegistry {};// Add this module to the LoadedCode if not present.- if let (Some(module), Some(id)) = (module, id) {- if let Some((_, range)) = module.compiled_module().finished_function_ranges().next() {- let loaded_code = self- .loaded_code- .get_mut(store_code_pc)- .expect("loaded_code must have entry for StoreCodePC");- loaded_code.modules.insert(range.start, id)?;- breakpoint_state.update(&mut loaded_code.code, module)?;- }+ if let (ModuleOrComponent::Module(module), Some(id)) = (module_or_component, id) {+ let loaded_code = self+ .loaded_code+ .get_mut(store_code_pc)+ .expect("loaded_code must have entry for StoreCodePC");+ loaded_code+ .modules+ .insert(module.env_module().module_index, Some(id))?;+ breakpoint_state.update(&mut loaded_code.code, module)?;}Ok(id)@@ -280,15 +320,18 @@ impl ModuleRegistry {&'a self,pc: usize,) -> Option<(FrameInfo, ModuleWithCode<'a>)> {- let (_, code) = self- .loaded_code- .range(..=StoreCodePC::from_raw(pc))- .next_back()?;- let text_offset = StoreCodePC::offset_of(code.code.text_range(), pc)?;- let (_, module_id) = code.modules.range(..=text_offset).next_back()?;+ let (code, text_offset) = self.loaded_code_by_pc(pc)?;+ let module_index = match code+ .index+ .func_by_text_offset(u32::try_from(text_offset).ok()?)?+ {+ FuncKey::DefinedWasmFunction(module, _) => module,+ _ => return None,+ };+ let module_id = (*code.modules.get(module_index)?)?;let module = self.modules- .get(*module_id)+ .get(module_id).expect("referenced module ID not found");let info = FrameInfo::new(module.clone(), text_offset)?;let module_with_code = ModuleWithCode::from_raw(module, &code.code);diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rsindex 320824211e..63fa437b82 100644--- a/crates/wasmtime/src/runtime/store/gc.rs+++ b/crates/wasmtime/src/runtime/store/gc.rs@@ -641,12 +641,16 @@ impl StoreOpaque {"we should always get a valid frame pointer for Wasm frames");- let (module_with_code, _offset) = self+ let (store_code, offset) = self.modules()- .module_and_code_by_pc(pc)- .expect("should have module info for Wasm frame");+ .store_code_by_pc(pc)+ .expect("should have store code for Wasm frame");+ let offset = u32::try_from(offset).unwrap();- if let Some(stack_map) = module_with_code.lookup_stack_map(pc) {+ let stack_map =+ wasmtime_environ::StackMap::lookup(offset, store_code.code_memory().stack_map_data());++ if let Some(stack_map) = stack_map {log::trace!("We have a stack map that maps {} bytes in this Wasm frame",stack_map.frame_size()@@ -661,11 +665,8 @@ impl StoreOpaque {}#[cfg(feature = "debug")]- if let Some(frame_table) = module_with_code.module().frame_table() {- let relpc = module_with_code- .text_offset(pc)- .expect("PC should be within module");- for stack_slot in crate::debug::gc_refs_in_frame(frame_table, relpc, fp) {+ if let Some(frame_table) = store_code.code_memory().frame_table() {+ for stack_slot in crate::debug::gc_refs_in_frame(frame_table, offset, fp) {unsafe {self.trace_wasm_stack_slot(gc_roots_list, stack_slot);}diff --git a/crates/wasmtime/src/runtime/vm/throw.rs b/crates/wasmtime/src/runtime/vm/throw.rsindex a2b24f9da2..e64b74da8e 100644--- a/crates/wasmtime/src/runtime/vm/throw.rs+++ b/crates/wasmtime/src/runtime/vm/throw.rs@@ -7,7 +7,7 @@ use crate::{};use core::ptr::NonNull;use wasmtime_environ::{DefinedTagIndex, TagIndex};-use wasmtime_unwinder::{Frame, Handler};+use wasmtime_unwinder::{ExceptionTable, Frame, Handler};/// Compute the target of the pending exception on the store.///@@ -50,8 +50,9 @@ pub unsafe fn compute_handler(frame.fp(),frame.pc());- let (module, rel_pc) = store.modules().module_and_code_by_pc(frame.pc())?;- let et = module.module().exception_table();+ let (store_code, rel_pc) = store.modules().store_code_by_pc(frame.pc())?;+ let et = ExceptionTable::parse(store_code.code_memory().exception_tables())+ .expect("Exception tables were validated on module load");let (frame_offset, handlers) = et.lookup_pc(u32::try_from(rel_pc).unwrap());let fp_to_sp = frame_offset.map(|frame_offset| -isize::try_from(frame_offset).unwrap());for handler in handlers {@@ -106,7 +107,7 @@ pub unsafe fn compute_handler(if is_match {let fp_to_sp = fp_to_sp.expect("frame offset must be known if we found a handler");return Some((- (module.store_code().text_range().start+ (store_code.text_range().start+ usize::try_from(handler.handler_offset).expect("Module larger than usize")).raw(),