A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Lookup functions in the text section by `FuncKey` at runtime (#11630)
Nick Fitzgerald1806c26562ec
diff --git a/cranelift/entity/src/primary.rs b/cranelift/entity/src/primary.rsindex cd01993246..6c1dbc18ad 100644--- a/cranelift/entity/src/primary.rs+++ b/cranelift/entity/src/primary.rs@@ -67,6 +67,11 @@ whereself.elems.get(k.index())}+ /// Get the slice of values associated with the given range of keys, if any.+ pub fn get_range(&self, range: core::ops::Range<K>) -> Option<&[V]> {+ self.elems.get(range.start.index()..range.end.index())+ }+/// Get the element at `k` if it exists, mutable version.pub fn get_mut(&mut self, k: K) -> Option<&mut V> {self.elems.get_mut(k.index())@@ -97,6 +102,11 @@ whereself.elems.iter_mut()}+ /// Get this map's underlying values as a slice.+ pub fn as_values_slice(&self) -> &[V] {+ &self.elems+ }+/// Iterate over all the keys and values in this map.pub fn iter(&self) -> Iter<'_, K, V> {Iter::new(self.elems.iter())@@ -294,6 +304,18 @@ where}}+impl<K, V> Extend<V> for PrimaryMap<K, V>+where+ K: EntityRef,+{+ fn extend<T>(&mut self, iter: T)+ where+ T: IntoIterator<Item = V>,+ {+ self.elems.extend(iter);+ }+}+impl<K, V> From<Vec<V>> for PrimaryMap<K, V>whereK: EntityRef,diff --git a/crates/environ/src/address_map.rs b/crates/environ/src/address_map.rsindex fe69dcebc6..c9cafcaa8d 100644--- a/crates/environ/src/address_map.rs+++ b/crates/environ/src/address_map.rs@@ -30,10 +30,22 @@ impl FilePos {FilePos(pos)}+ /// Get the null file position.+ pub fn none() -> FilePos {+ FilePos(u32::MAX)+ }++ /// Is this the null file position?+ #[inline]+ pub fn is_none(&self) -> bool {+ *self == FilePos::none()+ }+/// Returns the offset that this offset was created with.///- /// Note that the `Default` implementation will return `None` here, whereas- /// positions created with `FilePos::new` will return `Some`.+ /// Note that positions created with `FilePos::none` and the `Default`+ /// implementation will return `None` here, whereas positions created with+ /// `FilePos::new` will return `Some`.pub fn file_offset(self) -> Option<u32> {if self.0 == u32::MAX {None@@ -45,7 +57,7 @@ impl FilePos {impl Default for FilePos {fn default() -> FilePos {- FilePos(u32::MAX)+ FilePos::none()}}diff --git a/crates/environ/src/compile/mod.rs b/crates/environ/src/compile/mod.rsindex 5a1f2c516a..0bec4bed58 100644--- a/crates/environ/src/compile/mod.rs+++ b/crates/environ/src/compile/mod.rs@@ -3,10 +3,9 @@use crate::prelude::*;use crate::{- DefinedFuncIndex, FlagValue, FunctionLoc, ObjectKind, PrimaryMap, StaticModuleIndex, TripleExt,- WasmError, WasmFuncType,+ DefinedFuncIndex, FlagValue, FuncKey, FunctionLoc, ObjectKind, PrimaryMap, StaticModuleIndex,+ TripleExt, Tunables, WasmError, WasmFuncType, obj,};-use crate::{Tunables, obj};use anyhow::Result;use object::write::{Object, SymbolId};use object::{Architecture, BinaryFormat, FileFlags};@@ -17,7 +16,6 @@ use std::path;use std::sync::Arc;mod address_map;-mod key;mod module_artifacts;mod module_environ;mod module_types;@@ -25,7 +23,6 @@ mod stack_maps;mod trap_encoding;pub use self::address_map::*;-pub use self::key::*;pub use self::module_artifacts::*;pub use self::module_environ::*;pub use self::module_types::*;diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rsindex 4fc3a3792a..a4301a1fed 100644--- a/crates/environ/src/compile/module_artifacts.rs+++ b/crates/environ/src/compile/module_artifacts.rs@@ -3,9 +3,8 @@use crate::prelude::*;use crate::{- CompiledFunctionInfo, CompiledModuleInfo, DebugInfoData, DefinedFuncIndex, FunctionLoc,- FunctionName, MemoryInitialization, Metadata, ModuleInternedTypeIndex, ModuleTranslation,- PrimaryMap, Tunables, obj,+ CompiledModuleInfo, DebugInfoData, FunctionName, MemoryInitialization, Metadata,+ ModuleTranslation, Tunables, obj,};use anyhow::{Result, bail};use object::SectionKind;@@ -113,12 +112,7 @@ impl<'a> ObjectBuilder<'a> {/// Returns the `CompiledModuleInfo` corresponding to this core Wasm module/// as a result of this append operation. This is then serialized into the/// final artifact by the caller.- pub fn append(- &mut self,- translation: ModuleTranslation<'_>,- funcs: PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo>,- wasm_to_array_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>,- ) -> Result<CompiledModuleInfo> {+ pub fn append(&mut self, translation: ModuleTranslation<'_>) -> Result<CompiledModuleInfo> {let ModuleTranslation {mut module,debuginfo,@@ -219,8 +213,6 @@ impl<'a> ObjectBuilder<'a> {Ok(CompiledModuleInfo {module,- funcs,- wasm_to_array_trampolines,func_names,meta: Metadata {has_unparsed_debuginfo,diff --git a/crates/environ/src/component/artifacts.rs b/crates/environ/src/component/artifacts.rsindex 2aabb21ef3..22b9c2ce79 100644--- a/crates/environ/src/component/artifacts.rs+++ b/crates/environ/src/component/artifacts.rs@@ -2,8 +2,8 @@//! which are serialized with `bincode` into output ELF files.use crate::{- CompiledModuleInfo, FunctionLoc, PrimaryMap, StaticModuleIndex,- component::{Component, ComponentTypes, TrampolineIndex, TypeComponentIndex},+ CompiledFunctionsTable, CompiledModuleInfo, PrimaryMap, StaticModuleIndex,+ component::{Component, ComponentTypes, TypeComponentIndex},};use serde_derive::{Deserialize, Serialize};@@ -14,6 +14,8 @@ pub struct ComponentArtifacts {pub ty: TypeComponentIndex,/// Information all kept available at runtime as-is.pub info: CompiledComponentInfo,+ /// The index of every compiled function's location in the text section.+ pub table: CompiledFunctionsTable,/// Type information for this component and all contained modules.pub types: ComponentTypes,/// Serialized metadata about all included core wasm modules.@@ -25,16 +27,4 @@ pub struct ComponentArtifacts {pub struct CompiledComponentInfo {/// Type information calculated during translation about this component.pub component: Component,-- /// Where lowered function wasm-call trampolines that end up in a- /// `VMFuncRef` are located within the `text` section of `code_memory`.- pub wasm_call_trampolines: PrimaryMap<TrampolineIndex, FunctionLoc>,-- /// Where lowered function array-call trampolines that end up in a- /// `VMFuncRef` are located within the `text` section of `code_memory`.- pub array_call_trampolines: PrimaryMap<TrampolineIndex, FunctionLoc>,-- /// The location of the wasm-to-array trampoline for the `resource.drop`- /// intrinsic.- pub resource_drop_wasm_to_array_trampoline: Option<FunctionLoc>,}diff --git a/crates/environ/src/compile/key.rs b/crates/environ/src/key.rssimilarity index 99%rename from crates/environ/src/compile/key.rsrename to crates/environ/src/key.rsindex 58c43ecb4b..3c438be6f3 100644--- a/crates/environ/src/compile/key.rs+++ b/crates/environ/src/key.rs@@ -230,22 +230,8 @@ impl FuncKey {kind << Self::KIND_OFFSET}- /// Get this key's kind.- pub fn kind(self) -> FuncKeyKind {- self.namespace().kind()- }-- /// Get this key's namespace.- pub fn namespace(self) -> FuncKeyNamespace {- self.into_parts().0- }-- /// Get this key's index.- pub fn index(self) -> FuncKeyIndex {- self.into_parts().1- }-/// Split this key into its namespace and index halves.+ #[inline]pub fn into_parts(self) -> (FuncKeyNamespace, FuncKeyIndex) {let (namespace, index) = match self {FuncKey::DefinedWasmFunction(module, def_func) => {@@ -294,16 +280,19 @@ impl FuncKey {(FuncKeyNamespace(namespace), FuncKeyIndex(index))}- /// Create a key from its namespace and index parts.- ///- /// Should only be called with namespaces and indices that are ultimately- /// derived from the same key. For example, if you attempt to pair an index- /// and namespace that come from different keys, that may panic. If it- /// happens not to panic, you'll end up with a valid key that names an- /// arbitrary function in the given namespace, but that function probably- /// does not actually exist in the compilation artifact.- pub fn from_parts(namespace: FuncKeyNamespace, index: FuncKeyIndex) -> Self {- Self::from_raw_parts(namespace.into_raw(), index.into_raw())+ /// Get this key's kind.+ pub fn kind(self) -> FuncKeyKind {+ self.namespace().kind()+ }++ /// Get this key's namespace.+ pub fn namespace(self) -> FuncKeyNamespace {+ self.into_parts().0+ }++ /// Get this key's index.+ pub fn index(self) -> FuncKeyIndex {+ self.into_parts().1}/// Get the raw, underlying `(namespace, index)` representation of this@@ -320,6 +309,18 @@ impl FuncKey {(ns.into_raw(), index.into_raw())}+ /// Create a key from its namespace and index parts.+ ///+ /// Should only be called with namespaces and indices that are ultimately+ /// derived from the same key. For example, if you attempt to pair an index+ /// and namespace that come from different keys, that may panic. If it+ /// happens not to panic, you'll end up with a valid key that names an+ /// arbitrary function in the given namespace, but that function probably+ /// does not actually exist in the compilation artifact.+ pub fn from_parts(namespace: FuncKeyNamespace, index: FuncKeyIndex) -> Self {+ Self::from_raw_parts(namespace.into_raw(), index.into_raw())+ }+/// Create a key from its raw, underlying representation.////// Should only be given the results of a previous call todiff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rsindex 43501dd940..19bff1c498 100644--- a/crates/environ/src/lib.rs+++ b/crates/environ/src/lib.rs@@ -26,6 +26,7 @@ mod error;mod ext;mod gc;mod hostcall;+mod key;mod module;mod module_artifacts;mod module_types;@@ -46,6 +47,7 @@ pub use crate::demangling::*;pub use crate::error::*;pub use crate::gc::*;pub use crate::hostcall::*;+pub use crate::key::*;pub use crate::module::*;pub use crate::module_artifacts::*;pub use crate::module_types::*;diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rsindex 974b4f0fcc..73481147b3 100644--- a/crates/environ/src/module_artifacts.rs+++ b/crates/environ/src/module_artifacts.rs@@ -2,27 +2,16 @@//! with `bincode` as part of a module's compilation process.use crate::prelude::*;-use crate::{DefinedFuncIndex, FilePos, FuncIndex, Module, ModuleInternedTypeIndex, PrimaryMap};-use core::fmt;+use crate::{FilePos, FuncIndex, FuncKey, FuncKeyIndex, FuncKeyKind, FuncKeyNamespace, Module};use core::ops::Range;-use core::str;+use core::{fmt, u32};+use core::{iter, str};+use cranelift_entity::{EntityRef, PrimaryMap};use serde_derive::{Deserialize, Serialize};-/// Secondary in-memory results of function compilation.-#[derive(Clone, Serialize, Deserialize)]-pub struct CompiledFunctionInfo {- /// Where this function was found in the original wasm file.- pub start_srcloc: FilePos,- /// The [`FunctionLoc`] indicating the location of this function in the text- /// section of the competition artifact.- pub wasm_func_loc: FunctionLoc,- /// A trampoline for array callers (e.g. `Func::new`) calling into this function (if needed).- pub array_to_wasm_trampoline: Option<FunctionLoc>,-}-/// Description of where a function is located in the text section of a/// compiled image.-#[derive(Copy, Clone, Debug, Serialize, Deserialize)]+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]pub struct FunctionLoc {/// The byte offset from the start of the text section where this/// function starts.@@ -31,6 +20,504 @@ pub struct FunctionLoc {pub length: u32,}+impl FunctionLoc {+ /// Is this an empty function location?+ #[inline]+ pub fn is_empty(&self) -> bool {+ self.length == 0+ }+}++/// A builder for a `CompiledFunctionsTable`.+pub struct CompiledFunctionsTableBuilder {+ inner: CompiledFunctionsTable,+}++impl CompiledFunctionsTableBuilder {+ /// Create a new builder.+ pub fn new() -> Self {+ Self {+ inner: CompiledFunctionsTable {+ namespaces: PrimaryMap::new(),+ func_loc_starts: PrimaryMap::new(),+ sparse_starts: PrimaryMap::new(),+ src_loc_starts: PrimaryMap::new(),+ sparse_indices: PrimaryMap::new(),+ func_locs: PrimaryMap::new(),+ src_locs: PrimaryMap::new(),+ },+ }+ }++ fn last_namespace(&self) -> Option<FuncKeyNamespace> {+ let (_, &ns) = self.inner.namespaces.last()?;+ Some(ns)+ }++ fn last_key_index(&self) -> Option<FuncKeyIndex> {+ let (ns_idx, ns) = self.inner.namespaces.last()?;+ let start = self.inner.func_loc_starts[ns_idx];+ if CompiledFunctionsTable::is_dense(ns.kind()) {+ let len = self.inner.func_locs.len();+ let len = u32::try_from(len).unwrap();+ let key_index = len - start.as_u32();+ let key_index = FuncKeyIndex::from_raw(key_index);+ Some(key_index)+ } else {+ let sparse_start = self.inner.sparse_starts[ns_idx];+ if self.inner.sparse_indices.len() > sparse_start.index() {+ let (_, &key_index) = self.inner.sparse_indices.last().unwrap();+ Some(key_index)+ } else {+ None+ }+ }+ }++ fn last_func_loc(&self) -> Option<FunctionLoc> {+ let (_, &loc) = self.inner.func_locs.last()?;+ Some(loc)+ }++ /// Push a new entry into this builder.+ ///+ /// Panics if the key or function location is out of order.+ pub fn push_func(+ &mut self,+ key: FuncKey,+ func_loc: FunctionLoc,+ src_loc: FilePos,+ ) -> &mut Self {+ let (key_ns, key_index) = key.into_parts();++ assert!(+ self.last_namespace().is_none_or(|ns| ns <= key_ns),+ "`FuncKey`s pushed out of order"+ );+ assert!(+ self.last_key_index().is_none_or(+ |i| i <= key_index || self.last_namespace().is_some_and(|ns| ns != key_ns)+ ),+ "`FuncKey`s pushed out of order"+ );+ assert!(+ self.last_func_loc()+ .is_none_or(|l| l.start + l.length <= func_loc.start),+ "`FunctionLoc`s pushed out of order"+ );++ // Make sure that there is a `kind` entry for this key's kind.+ let kind_start_index = self+ .inner+ .namespaces+ .last()+ .and_then(|(ns_idx, ns)| {+ if *ns == key_ns {+ Some(self.inner.func_loc_starts[ns_idx])+ } else {+ None+ }+ })+ .unwrap_or_else(|| {+ let start = self.inner.func_locs.next_key();+ let ns_idx = self.inner.namespaces.push(key_ns);+ let ns_idx2 = self.inner.func_loc_starts.push(start);+ let ns_idx3 = self+ .inner+ .sparse_starts+ .push(self.inner.sparse_indices.next_key());+ let ns_idx4 = self+ .inner+ .src_loc_starts+ .push(self.inner.src_locs.next_key());+ debug_assert_eq!(ns_idx, ns_idx2);+ debug_assert_eq!(ns_idx, ns_idx3);+ debug_assert_eq!(ns_idx, ns_idx4);+ start+ });++ if CompiledFunctionsTable::is_dense(key.kind()) {+ // Figure out the index within `func_locs` for this key's entry.+ let index = kind_start_index.as_u32() + key_index.into_raw();+ let index = FuncLocIndex::from_u32(index);+ debug_assert!(self.inner.func_locs.get(index).is_none());++ // Fill in null entries for any key indices that have been omitted.+ //+ // Note that we need a null `FunctionLoc`, but we also need+ // `func_locs` to be sorted so that we support reverse+ // lookups. Therefore, we take care to create an empty function+ // location that starts at the text offset that the previous one (if+ // any) ends at, and use that as our null entry.+ let null_func_loc = FunctionLoc {+ start: self+ .last_func_loc()+ .map(|l| l.start + l.length)+ .unwrap_or_default(),+ length: 0,+ };+ let gap = index.index() - self.inner.func_locs.len();+ self.inner+ .func_locs+ .extend(iter::repeat(null_func_loc).take(gap));+ debug_assert_eq!(index, self.inner.func_locs.next_key());++ if CompiledFunctionsTable::has_src_locs(key_ns.kind()) {+ self.inner+ .src_locs+ .extend(iter::repeat(FilePos::none()).take(gap));+ }+ } else {+ debug_assert!(+ src_loc.is_none(),+ "sparse keys do not have source locations"+ );+ self.inner.sparse_indices.push(key_index);+ }++ // And finally, we push this entry.+ self.inner.func_locs.push(func_loc);+ if CompiledFunctionsTable::has_src_locs(key_ns.kind()) {+ self.inner.src_locs.push(src_loc);+ } else {+ debug_assert!(src_loc.is_none());+ }++ self+ }++ /// Finish construction of the `CompiledFunctionsTable`.+ pub fn finish(self) -> CompiledFunctionsTable {+ self.inner+ }+}++#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]+struct NamespaceIndex(u32);+cranelift_entity::entity_impl!(NamespaceIndex);++#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]+struct FuncLocIndex(u32);+cranelift_entity::entity_impl!(FuncLocIndex);++#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]+struct SparseIndex(u32);+cranelift_entity::entity_impl!(SparseIndex);++#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]+struct SrcLocIndex(u32);+cranelift_entity::entity_impl!(SrcLocIndex);++/// A table describing the set of functions compiled into an artifact, their+/// locations within the text section, and etc...+///+/// Logically, this type is a map from a `FuncKey` to the associated function's+///+/// * location within the associated text section, and+/// * optional source location.+///+/// How this map is *actually* implemented is with a series of lookup and binary+/// search tables, split out in a data-oriented, struct-of-arrays style. We+/// organize the data in this way is service of three goals:+///+/// 1. Provide fast look ups: We need to look up the metadata for a function by+/// its key at runtime. During instantiation, for example, we need to create+/// `VMFuncRef`s for escaping functions and this requires looking up the+/// locations of those Wasm functions and their associated array-to-Wasm+/// trampolines.+///+/// 2. Keep memory overheads low and code size small: This type is serialized+/// into all of our ELF artifacts and deserialized into all `Module`s and+/// `Component`s at runtime.+///+/// 3. Be generic over any kind of function (whether defined Wasm function,+/// trampoline, or etc...) that we compile: Adding a new kind of trampoline,+/// for example, should not require updating this structure to add a new+/// table of the function locations for just trampolines of that new kind. We+/// should be able to store and query all kinds of functions uniformly.+//+// TODO: This structure could be directly encoded as raw ELF sections, instead+// of a `struct` containing a bunch of `PrimaryMap`s, which would allow us to+// avoid the serialize/deserialize runtime costs.+#[derive(Debug, Serialize, Deserialize)]+pub struct CompiledFunctionsTable {+ /// A binary-search index for this table, mapping raw `FuncKeyNamespace`s to+ /// their associated `NamespaceIndex`. That `NamespaceIndex` can then be+ /// used to find the range of other entity indices that are specific to that+ /// namespace.+ namespaces: PrimaryMap<NamespaceIndex, FuncKeyNamespace>,++ /// `self.func_loc_starts[i]..self.func_loc_starts[i+1]` describes the range+ /// within `self.func_locs` whose entries are associated with the namespace+ /// `self.index[i]`.+ ///+ /// When `self.func_loc_starts[i+1]` is out of bounds, then the range is to+ /// the end of `self.func_locs`.+ func_loc_starts: PrimaryMap<NamespaceIndex, FuncLocIndex>,++ /// `self.sparse_starts[i]..self.sparse_starts[i+1]` describes the range+ /// within `self.sparse_indices` whose entries are associated with the+ /// namespace `self.index[i]`.+ ///+ /// When `self.sparse_starts[i+1]` is out of bounds, then the range is to+ /// the end of `self.sparse_indices`.+ ///+ /// Entries are only valid for sparse, non-dense namespaces.+ sparse_starts: PrimaryMap<NamespaceIndex, SparseIndex>,++ /// `self.src_loc_starts[i]..self.src_loc_starts[i+1]` describes the range+ /// within `self.src_loc_indices` whose entries are associated with the+ /// namespace `self.index[i]`.+ ///+ /// When `self.src_loc_starts[i+1]` is out of bounds, then the range is to+ /// the end of `self.src_locs`.+ ///+ /// Entries are only valid for namespaces whose functions have source+ /// locations.+ src_loc_starts: PrimaryMap<NamespaceIndex, SrcLocIndex>,++ /// `self.sparse_indices[i]` contains the index part of+ /// `FuncKey::from_parts(ns, index)` where `ns` is determined by+ /// `self.sparse_starts` and is a sparse, non-dense key kind. (Note that for+ /// dense keys, this information is implicitly encoded in their offset from+ /// the namespace's start index.)+ ///+ /// This is sorted to allow for binary searches.+ sparse_indices: PrimaryMap<SparseIndex, FuncKeyIndex>,++ /// `self.func_locs[i]` contains the location within the text section of+ /// `FuncKey::from_parts(self.namespaces[ns], i - start)`'s function, where+ /// `ns` and `start` are determined by `self.func_loc_starts`.+ ///+ /// Values are sorted by function location to support reverse queries from+ /// function location back to `FuncKey`.+ ///+ /// The absence of a function location (for gaps in dense namespaces) is+ /// represented with `FunctionLoc::none()`.+ func_locs: PrimaryMap<FuncLocIndex, FunctionLoc>,++ /// `self.src_locs[i]` contains the initial source location of+ /// `FuncKey::from_parts(self.namespaces[ns], i - start)`'s function, where+ /// `ns` and `start` are determined by `self.src_loc_starts`.+ ///+ /// The absence of a source location is represented by `FilePos::none()`.+ src_locs: PrimaryMap<SrcLocIndex, FilePos>,+}++impl CompiledFunctionsTable {+ #[inline]+ fn namespace_index(&self, namespace: FuncKeyNamespace) -> Option<NamespaceIndex> {+ const LINEAR_SEARCH_LIMIT: usize = 32;+ if self.namespaces.len() <= LINEAR_SEARCH_LIMIT {+ self.namespaces+ .iter()+ .find_map(|(idx, ns)| if *ns == namespace { Some(idx) } else { None })+ } else {+ self.namespaces+ .binary_search_values_by_key(&namespace, |ns| *ns)+ .ok()+ }+ }++ #[inline]+ fn func_loc_range(&self, ns_idx: NamespaceIndex) -> Range<FuncLocIndex> {+ let start = self.func_loc_starts[ns_idx];+ let next_ns_idx = NamespaceIndex::from_u32(ns_idx.as_u32() + 1);+ let end = self+ .func_loc_starts+ .get(next_ns_idx)+ .copied()+ .unwrap_or_else(|| self.func_locs.next_key());+ start..end+ }++ fn sparse_range(&self, ns_idx: NamespaceIndex) -> Range<SparseIndex> {+ debug_assert!(!Self::is_dense(self.namespaces[ns_idx].kind()));+ let start = self.sparse_starts[ns_idx];+ let next_ns_idx = NamespaceIndex::from_u32(ns_idx.as_u32() + 1);+ let end = self+ .sparse_starts+ .get(next_ns_idx)+ .copied()+ .unwrap_or_else(|| self.sparse_indices.next_key());+ start..end+ }++ fn src_loc_range(&self, ns_idx: NamespaceIndex) -> Range<SrcLocIndex> {+ debug_assert!(Self::has_src_locs(self.namespaces[ns_idx].kind()));+ let start = self.src_loc_starts[ns_idx];+ let next_ns_idx = NamespaceIndex::from_u32(ns_idx.as_u32() + 1);+ let end = self+ .src_loc_starts+ .get(next_ns_idx)+ .copied()+ .unwrap_or_else(|| self.src_locs.next_key());+ start..end+ }++ /// Get the index within `self.{func_locs,src_locs}` that is associated with+ /// the given `key`, if any.+ #[inline]+ fn func_loc_index(&self, key: FuncKey) -> Option<FuncLocIndex> {+ let (key_ns, key_index) = key.into_parts();+ let ns_idx = self.namespace_index(key_ns)?;+ let Range { start, end } = self.func_loc_range(ns_idx);++ let index = if Self::is_dense(key.kind()) {+ let index = start.as_u32().checked_add(key_index.into_raw())?;+ FuncLocIndex::from_u32(index)+ } else {+ let sparse_range = self.sparse_range(ns_idx);+ let sparse_subslice = self.sparse_indices.get_range(sparse_range).unwrap();+ match sparse_subslice.binary_search(&key_index) {+ Ok(i) => FuncLocIndex::new(start.index() + i),+ Err(_) => return None,+ }+ };++ if index < end { Some(index) } else { None }+ }++ /// Get the location of the function associated with the given `key` inside+ /// the text section, if any.+ #[inline]+ pub fn func_loc(&self, key: FuncKey) -> Option<&FunctionLoc> {+ let index = self.func_loc_index(key)?;+ let loc = &self.func_locs[index];+ if loc.is_empty() { None } else { Some(loc) }+ }++ fn src_loc_index(&self, key: FuncKey) -> Option<SrcLocIndex> {+ let (key_ns, key_index) = key.into_parts();+ if !Self::has_src_locs(key_ns.kind()) {+ return None;+ }++ let ns_idx = self.namespace_index(key_ns)?;+ let Range { start, end } = self.src_loc_range(ns_idx);++ debug_assert!(Self::is_dense(key_ns.kind()));+ let index = start.as_u32().checked_add(key_index.into_raw())?;+ let index = SrcLocIndex::from_u32(index);+ if index >= end {+ return None;+ }++ Some(index)+ }++ /// Get the initial source location of the function associated with the+ /// given `key`, if any.+ pub fn src_loc(&self, key: FuncKey) -> Option<FilePos> {+ let index = self.src_loc_index(key)?;+ let loc = self.src_locs[index];+ if loc.is_none() { None } else { Some(loc) }+ }++ /// Given an offset into the text section, get the key for its associated+ /// function and its offset within that function.+ pub fn func_by_text_offset(&self, text_offset: u32) -> Option<FuncKey> {+ let index = match self.func_locs.as_values_slice().binary_search_by(|loc| {+ if loc.is_empty() {+ loc.start+ .cmp(&text_offset)+ .then_with(|| core::cmp::Ordering::Less)+ } else {+ if loc.start > text_offset {+ core::cmp::Ordering::Greater+ } else if loc.start + loc.length <= text_offset {+ core::cmp::Ordering::Less+ } else {+ debug_assert!(loc.start <= text_offset);+ debug_assert!(text_offset < loc.start + loc.length);+ core::cmp::Ordering::Equal+ }+ }+ }) {+ // Exact match, the offset is at the end of this function.+ Ok(k) => k,+ // Not an exact match: `k` is where the offset would be+ // "inserted". Since we key based on the end, function `k` might+ // contain the offset, so we'll validate on the range check+ // below.+ Err(k) => k,+ };+ let index = FuncLocIndex::new(index);++ // Make sure that the text offset is actually within this function.+ // Non-exact binary search results can either be because we have a text+ // offset within a function but not exactly at its inclusive end, or+ // because the text offset is not within any of our functions. We filter+ // that latter case out with this check.+ let loc = self.func_locs.get(index)?;+ let start = loc.start;+ let end = start + loc.length;+ if text_offset < start || end < text_offset {+ return None;+ }++ let ns_idx = match self+ .func_loc_starts+ .binary_search_values_by_key(&index, |s| *s)+ {+ // Exact match: `i` is the entry's index.+ Ok(i) => i,+ // Not an exact match: the index, if it were the start of a+ // namespace's range, would be at `i`. Therefore, our namespace+ // entry is actually at index `i - 1`.+ Err(i) => {+ let i = i.as_u32();+ assert_ne!(i, 0);+ NamespaceIndex::from_u32(i - 1)+ }+ };+ let key_ns = self.namespaces[ns_idx];+ let start = self.func_loc_starts[ns_idx];++ let key_index = if Self::is_dense(key_ns.kind()) {+ let key_index = index.as_u32() - start.as_u32();+ FuncKeyIndex::from_raw(key_index)+ } else {+ let sparse_index = index.as_u32() - start.as_u32();+ let sparse_index = SparseIndex::from_u32(sparse_index);+ self.sparse_indices[sparse_index]+ };+ let key = FuncKey::from_parts(key_ns, key_index);++ Some(key)+ }++ /// Whether the given kind's index space is (generally) densely populated+ /// and therefore we should densely pack them in the table for `O(1)`+ /// lookups; otherwise, we should avoid code size bloat by using the sparse+ /// table indirection and `O(log n)` binary search lookups.+ fn is_dense(kind: FuncKeyKind) -> bool {+ match kind {+ FuncKeyKind::DefinedWasmFunction+ | FuncKeyKind::WasmToArrayTrampoline+ | FuncKeyKind::PulleyHostCall => true,++ FuncKeyKind::ArrayToWasmTrampoline | FuncKeyKind::WasmToBuiltinTrampoline => false,++ #[cfg(feature = "component-model")]+ FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline => true,+ }+ }++ /// Whether the given function kind has source locations or not.+ fn has_src_locs(kind: FuncKeyKind) -> bool {+ match kind {+ FuncKeyKind::DefinedWasmFunction => true,+ FuncKeyKind::ArrayToWasmTrampoline+ | FuncKeyKind::WasmToArrayTrampoline+ | FuncKeyKind::WasmToBuiltinTrampoline+ | FuncKeyKind::PulleyHostCall => false,+ #[cfg(feature = "component-model")]+ FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline => false,+ }+ }+}+/// Secondary in-memory results of module compilation.////// This opaque structure can be optionally passed back to@@ -40,18 +527,11 @@ pub struct CompiledModuleInfo {/// Type information about the compiled WebAssembly module.pub module: Module,- /// Metadata about each compiled function.- pub funcs: PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo>,+ /// General compilation metadata.+ pub meta: Metadata,/// Sorted list, by function index, of names we have for this module.pub func_names: Vec<FunctionName>,-- /// Metadata about wasm-to-array trampolines. Used when exposing a native- /// callee (e.g. `Func::wrap`) to a Wasm caller. Sorted by signature index.- pub wasm_to_array_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>,-- /// General compilation metadata.- pub meta: Metadata,}/// The name of a function stored in the@@ -117,3 +597,139 @@ pub enum ObjectKind {/// A component compilation artifactComponent,}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::{DefinedFuncIndex, StaticModuleIndex};++ fn func_loc(range: Range<u32>) -> FunctionLoc {+ FunctionLoc {+ start: range.start,+ length: range.end - range.start,+ }+ }++ fn def_func_key(m: u32, f: u32) -> FuncKey {+ FuncKey::DefinedWasmFunction(+ StaticModuleIndex::from_u32(m),+ DefinedFuncIndex::from_u32(f),+ )+ }++ fn array_to_wasm_tramp_key(m: u32, f: u32) -> FuncKey {+ FuncKey::ArrayToWasmTrampoline(+ StaticModuleIndex::from_u32(m),+ DefinedFuncIndex::from_u32(f),+ )+ }++ fn make_test_table() -> CompiledFunctionsTable {+ let mut builder = CompiledFunctionsTableBuilder::new();++ builder+ // ========= Dense =========+ .push_func(def_func_key(0, 0), func_loc(0..10), FilePos::new(111))+ .push_func(def_func_key(0, 1), func_loc(10..20), FilePos::new(222))+ .push_func(def_func_key(0, 2), func_loc(20..30), FilePos::none())+ // Gap in dense keys!+ .push_func(def_func_key(0, 5), func_loc(30..40), FilePos::new(333))+ // ========= Sparse =========+ .push_func(+ array_to_wasm_tramp_key(0, 1),+ func_loc(100..110),+ FilePos::none(),+ )+ .push_func(+ array_to_wasm_tramp_key(0, 2),+ func_loc(110..120),+ FilePos::none(),+ )+ .push_func(+ array_to_wasm_tramp_key(0, 5),+ func_loc(120..130),+ FilePos::none(),+ );++ builder.finish()+ }++ #[test]+ fn src_locs() {+ let table = make_test_table();++ for (key, expected) in [+ (def_func_key(0, 0), Some(FilePos::new(111))),+ (def_func_key(0, 1), Some(FilePos::new(222))),+ (def_func_key(0, 2), None),+ (def_func_key(0, 3), None),+ (def_func_key(0, 4), None),+ (def_func_key(0, 5), Some(FilePos::new(333))),+ (array_to_wasm_tramp_key(0, 0), None),+ (array_to_wasm_tramp_key(0, 1), None),+ (array_to_wasm_tramp_key(0, 2), None),+ (array_to_wasm_tramp_key(0, 3), None),+ (array_to_wasm_tramp_key(0, 4), None),+ (array_to_wasm_tramp_key(0, 5), None),+ ] {+ eprintln!("Checking key {key:?}");+ let actual = table.src_loc(key);+ assert_eq!(expected, actual);+ }+ }++ #[test]+ fn func_locs() {+ let table = make_test_table();++ for (key, expected) in [+ (def_func_key(0, 0), Some(0)),+ (def_func_key(0, 1), Some(10)),+ (def_func_key(0, 2), Some(20)),+ (def_func_key(0, 3), None),+ (def_func_key(0, 4), None),+ (def_func_key(0, 5), Some(30)),+ (array_to_wasm_tramp_key(0, 0), None),+ (array_to_wasm_tramp_key(0, 1), Some(100)),+ (array_to_wasm_tramp_key(0, 2), Some(110)),+ (array_to_wasm_tramp_key(0, 3), None),+ (array_to_wasm_tramp_key(0, 4), None),+ (array_to_wasm_tramp_key(0, 5), Some(120)),+ ] {+ let actual = table.func_loc(key);+ match (expected, actual) {+ (None, None) => {}+ (Some(expected), Some(actual)) => assert_eq!(expected, actual.start),+ (None, Some(actual)) => {+ panic!("expected no function location for {key:?}, got {actual:?}")+ }+ (Some(_), None) => {+ panic!("expected a function location for {key:?}, but got nothing")+ }+ }+ }+ }++ #[test]+ fn reverse_func_locs() {+ let table = make_test_table();++ for (range, expected) in [+ (0..10, Some(def_func_key(0, 0))),+ (10..20, Some(def_func_key(0, 1))),+ (20..30, Some(def_func_key(0, 2))),+ (30..40, Some(def_func_key(0, 5))),+ (40..100, None),+ (100..110, Some(array_to_wasm_tramp_key(0, 1))),+ (110..120, Some(array_to_wasm_tramp_key(0, 2))),+ (120..130, Some(array_to_wasm_tramp_key(0, 5))),+ (140..150, None),+ ] {+ for i in range {+ eprintln!("Checking offset {i}");+ let actual = table.func_by_text_offset(i);+ assert_eq!(expected, actual);+ }+ }+ }+}diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rsindex 87a3252c95..5d210bd58c 100644--- a/crates/wasmtime/src/compile.rs+++ b/crates/wasmtime/src/compile.rs@@ -26,22 +26,17 @@ use crate::Engine;use crate::hash_map::HashMap;use crate::hash_set::HashSet;use crate::prelude::*;-use std::{- any::Any,- borrow::Cow,- collections::{BTreeMap, BTreeSet},- mem,- ops::Range,-};+use std::{any::Any, borrow::Cow, collections::BTreeMap, mem, ops::Range};use call_graph::CallGraph;#[cfg(feature = "component-model")]use wasmtime_environ::component::Translator;use wasmtime_environ::{- BuiltinFunctionIndex, CompiledFunctionBody, CompiledFunctionInfo, CompiledModuleInfo, Compiler,- DefinedFuncIndex, FilePos, FinishedObject, FuncKey, FunctionBodyData, InliningCompiler,- IntraModuleInlining, ModuleEnvironment, ModuleTranslation, ModuleTypes, ModuleTypesBuilder,- ObjectKind, PrimaryMap, SecondaryMap, StaticModuleIndex, Tunables,+ BuiltinFunctionIndex, CompiledFunctionBody, CompiledFunctionsTable,+ CompiledFunctionsTableBuilder, CompiledModuleInfo, Compiler, DefinedFuncIndex, FilePos,+ FinishedObject, FuncKey, FunctionBodyData, InliningCompiler, IntraModuleInlining,+ ModuleEnvironment, ModuleTranslation, ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap,+ StaticModuleIndex, Tunables,};mod call_graph;@@ -71,7 +66,10 @@ pub(crate) fn build_artifacts<T: FinishedObject>(wasm: &[u8],dwarf_package: Option<&[u8]>,obj_state: &T::State,-) -> Result<(T, Option<(CompiledModuleInfo, ModuleTypes)>)> {+) -> Result<(+ T,+ Option<(CompiledModuleInfo, CompiledFunctionsTable, ModuleTypes)>,+)> {let tunables = engine.tunables();// First a `ModuleEnvironment` is created which records type information@@ -117,7 +115,6 @@ pub(crate) fn build_artifacts<T: FinishedObject>(engine.append_bti(&mut object);let (mut object, compilation_artifacts) = indices.link_and_append_code(- &types,object,engine,compiled_funcs,@@ -125,12 +122,12 @@ pub(crate) fn build_artifacts<T: FinishedObject>(dwarf_package,)?;- let info = compilation_artifacts.unwrap_as_module_info();+ let (info, index) = compilation_artifacts.unwrap_as_module_info();let types = types.finish();- object.serialize_info(&(&info, &types));+ object.serialize_info(&(&info, &index, &types));let result = T::finish_object(object, obj_state)?;- Ok((result, Some((info, types))))+ Ok((result, Some((info, index, types))))}/// Performs the compilation phase for a component, translating and@@ -188,7 +185,6 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(engine.append_bti(&mut object);let (mut object, compilation_artifacts) = indices.link_and_append_code(- types.module_types_builder(),object,engine,compiled_funcs,@@ -199,12 +195,10 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(let info = CompiledComponentInfo {component: component.component,- wasm_call_trampolines: compilation_artifacts.wasm_call_trampolines,- array_call_trampolines: compilation_artifacts.array_call_trampolines,- resource_drop_wasm_to_array_trampoline: compilation_artifacts.resource_drop_trampoline,};let artifacts = ComponentArtifacts {info,+ table: compilation_artifacts.table,ty,types,static_modules: compilation_artifacts.modules,@@ -874,7 +868,7 @@ impl UnlinkedCompileOutputs<'_> {/// indices within it.fn pre_link(self) -> PreLinkOutput {// We must ensure that `compiled_funcs` contains the function bodies- // sorted by their `FuncKey`, as `CompiledFunctionsIndex` relies on that+ // sorted by their `FuncKey`, as `CompiledFunctionsTable` relies on that// property.//// Furthermore, note that, because the order functions end up in@@ -898,7 +892,7 @@ impl UnlinkedCompileOutputs<'_> {let index = compiled_funcs.len();compiled_funcs.push((output.symbol, output.function.code));- if output.start_srcloc != FilePos::default() {+ if output.start_srcloc != FilePos::none() {indices.start_srclocs.insert(output.key, output.start_srcloc);@@ -941,7 +935,6 @@ impl FunctionIndices {/// them to the given ELF file.fn link_and_append_code<'a>(self,- types: &ModuleTypesBuilder,mut obj: object::write::Object<'static>,engine: &'a Engine,compiled_funcs: Vec<(String, Box<dyn Any + Send + Sync>)>,@@ -981,103 +974,21 @@ impl FunctionIndices {)?;}- let mut def_funcs = SecondaryMap::<- StaticModuleIndex,- PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo>,- >::new();-- #[cfg(feature = "component-model")]- let mut wasm_call_trampolines = PrimaryMap::<- wasmtime_environ::component::TrampolineIndex,- wasmtime_environ::FunctionLoc,- >::new();-- #[cfg(feature = "component-model")]- let mut array_call_trampolines = PrimaryMap::<- wasmtime_environ::component::TrampolineIndex,- wasmtime_environ::FunctionLoc,- >::new();-- #[cfg(feature = "component-model")]- let mut resource_drop_trampoline = None;-- for (key, index) in &self.indices {- // NB: exhaustively match on function keys to make sure that we are- // remembering to handle everything we are compiling when doing this- // final linking and metadata-collection step.- match *key {- FuncKey::DefinedWasmFunction(module, def_func) => {- let (_, wasm_func_loc) = symbol_ids_and_locs[*index];- let start_srcloc = self.start_srclocs[key];-- let array_to_wasm_trampoline = self- .indices- .get(&FuncKey::ArrayToWasmTrampoline(module, def_func))- .map(|index| {- let (_, loc) = symbol_ids_and_locs[*index];- loc- });-- debug_assert!(def_funcs[module].get(def_func).is_none());- let def_func2 = def_funcs[module].push(CompiledFunctionInfo {- start_srcloc,- wasm_func_loc,- array_to_wasm_trampoline,- });- debug_assert_eq!(def_func, def_func2);- }-- FuncKey::ArrayToWasmTrampoline(module, def_func) => {- // These are handled by the `DefinedWasmFunction` arm above.- debug_assert!(def_funcs[module].get(def_func).is_some());- }-- FuncKey::WasmToArrayTrampoline(_) => {- // These are handled in `modules` creation below.- }-- FuncKey::WasmToBuiltinTrampoline(_) => {- // Nothing we need to do for these: they are only called by- // Wasm functions, and we never create `funcref`s containing- // them, so we don't need to keep any metadata for them or- // anything like that.- }-- FuncKey::PulleyHostCall(_) => {- unreachable!("we don't compile any artifacts for Pulley host calls")- }-- #[cfg(feature = "component-model")]- FuncKey::ComponentTrampoline(abi, trampoline) => {- let (_, loc) = symbol_ids_and_locs[*index];- match abi {- wasmtime_environ::Abi::Wasm => {- debug_assert!(wasm_call_trampolines.get(trampoline).is_none());- let trampoline2 = wasm_call_trampolines.push(loc);- debug_assert_eq!(trampoline, trampoline2);- }- wasmtime_environ::Abi::Array => {- debug_assert!(array_call_trampolines.get(trampoline).is_none());- let trampoline2 = array_call_trampolines.push(loc);- debug_assert_eq!(trampoline, trampoline2);- }- }- }-- #[cfg(feature = "component-model")]- FuncKey::ResourceDropTrampoline => {- let (_, loc) = symbol_ids_and_locs[*index];- resource_drop_trampoline = Some(loc);- }- }+ let mut table_builder = CompiledFunctionsTableBuilder::new();+ for (key, compiled_func_index) in &self.indices {+ let (_, func_loc) = symbol_ids_and_locs[*compiled_func_index];+ let src_loc = self+ .start_srclocs+ .get(key)+ .copied()+ .unwrap_or_else(FilePos::none);+ table_builder.push_func(*key, func_loc, src_loc);}let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables);let modules = translations.into_iter()- .map(|(module, mut translation)| {- let def_funcs = mem::take(&mut def_funcs[module]);-+ .map(|(_, mut translation)| {// If configured attempt to use static memory initialization// which can either at runtime be implemented as a single memcpy// to initialize memory or otherwise enabling@@ -1095,38 +1006,13 @@ impl FunctionIndices {translation.try_func_table_init();}- // Gather this module's trampolines.- let unique_and_sorted_trampoline_sigs = translation- .module- .types- .iter()- .map(|(_, ty)| ty.unwrap_module_type_index())- .filter(|idx| types[*idx].is_func())- .map(|idx| types.trampoline_type(idx))- .collect::<BTreeSet<_>>();- let wasm_to_array_trampolines = unique_and_sorted_trampoline_sigs- .into_iter()- .map(|ty| {- debug_assert_eq!(ty, types.trampoline_type(ty));- let key = FuncKey::WasmToArrayTrampoline(ty);- let index = self.indices[&key];- let (_, loc) = symbol_ids_and_locs[index];- (ty, loc)- })- .collect();-- obj.append(translation, def_funcs, wasm_to_array_trampolines)+ obj.append(translation)}).collect::<Result<PrimaryMap<_, _>>>()?;let artifacts = Artifacts {modules,- #[cfg(feature = "component-model")]- wasm_call_trampolines,- #[cfg(feature = "component-model")]- array_call_trampolines,- #[cfg(feature = "component-model")]- resource_drop_trampoline,+ table: table_builder.finish(),};Ok((obj, artifacts))@@ -1135,32 +1021,19 @@ impl FunctionIndices {/// The artifacts necessary for finding and calling Wasm functions at runtime,/// to be serialized into an ELF file.-#[derive(Default)]struct Artifacts {modules: PrimaryMap<StaticModuleIndex, CompiledModuleInfo>,- #[cfg(feature = "component-model")]- wasm_call_trampolines:- PrimaryMap<wasmtime_environ::component::TrampolineIndex, wasmtime_environ::FunctionLoc>,- #[cfg(feature = "component-model")]- array_call_trampolines:- PrimaryMap<wasmtime_environ::component::TrampolineIndex, wasmtime_environ::FunctionLoc>,- #[cfg(feature = "component-model")]- resource_drop_trampoline: Option<wasmtime_environ::FunctionLoc>,+ table: CompiledFunctionsTable,}impl Artifacts {/// Assuming this compilation was for a single core Wasm module, get the/// resulting `CompiledModuleInfo`.- fn unwrap_as_module_info(self) -> CompiledModuleInfo {+ fn unwrap_as_module_info(self) -> (CompiledModuleInfo, CompiledFunctionsTable) {assert_eq!(self.modules.len(), 1);-- #[cfg(feature = "component-model")]- {- assert!(self.wasm_call_trampolines.is_empty());- assert!(self.array_call_trampolines.is_empty());- }-- self.modules.into_iter().next().unwrap().1+ let info = self.modules.into_iter().next().unwrap().1;+ let table = self.table;+ (info, table)}}diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rsindex 222dccdb63..eb83c44852 100644--- a/crates/wasmtime/src/runtime/component/component.rs+++ b/crates/wasmtime/src/runtime/component/component.rs@@ -15,12 +15,12 @@ use core::ops::Range;use core::ptr::NonNull;#[cfg(feature = "std")]use std::path::Path;-use wasmtime_environ::TypeTrace;use wasmtime_environ::component::{CompiledComponentInfo, ComponentArtifacts, ComponentTypes, CoreDef, Export, ExportIndex,GlobalInitializer, InstantiateModule, NameMapNoIntern, OptionsIndex, StaticModuleIndex,TrampolineIndex, TypeComponentIndex, TypeFuncIndex, VMComponentOffsets,};+use wasmtime_environ::{Abi, CompiledFunctionsTable, FuncKey, TypeTrace};use wasmtime_environ::{FunctionLoc, HostPtr, ObjectKind, PrimaryMap};/// A compiled WebAssembly Component.@@ -86,6 +86,10 @@ struct ComponentInner {/// Metadata produced during compilation.info: CompiledComponentInfo,+ /// The index of compiled functions and their locations in the text section+ /// for this component.+ index: Arc<CompiledFunctionsTable>,+/// A cached handle to the `wasmtime::FuncType` for the canonical ABI's/// `realloc`, to avoid the need to look up types in the registry and take/// locks when calling `realloc` via `TypedFunc::call_raw`.@@ -400,12 +404,14 @@ impl Component {let ComponentArtifacts {ty,info,+ table: index,mut types,mut static_modules,} = match artifacts {Some(artifacts) => artifacts,None => postcard::from_bytes(code_memory.wasmtime_info())?,};+ let index = Arc::new(index);// Validate that the component can be used with the current instance// allocator.@@ -434,7 +440,9 @@ impl Component {// `types` type information, and the code memory to a runtime object.let static_modules = static_modules.into_iter()- .map(|(_, info)| Module::from_parts_raw(engine, code.clone(), info, false))+ .map(|(_, info)| {+ Module::from_parts_raw(engine, code.clone(), info, index.clone(), false)+ }).collect::<Result<_>>()?;let realloc_func_type = Arc::new(FuncType::new(@@ -451,6 +459,7 @@ impl Component {static_modules,code,info,+ index,realloc_func_type,}),})@@ -492,8 +501,16 @@ impl Component {}pub(crate) fn trampoline_ptrs(&self, index: TrampolineIndex) -> AllCallFuncPointers {- let wasm_call = &self.inner.info.wasm_call_trampolines[index];- let array_call = &self.inner.info.array_call_trampolines[index];+ let wasm_call = self+ .inner+ .index+ .func_loc(FuncKey::ComponentTrampoline(Abi::Wasm, index))+ .unwrap();+ let array_call = self+ .inner+ .index+ .func_loc(FuncKey::ComponentTrampoline(Abi::Array, index))+ .unwrap();AllCallFuncPointers {wasm_call: self.func(wasm_call).cast(),array_call: self.func(array_call).cast(),@@ -537,10 +554,10 @@ impl Component {// blank.let wasm_call = self.inner- .info- .resource_drop_wasm_to_array_trampoline- .as_ref()- .map(|i| self.func(i).cast().into());+ .index+ .func_loc(FuncKey::ResourceDropTrampoline)+ .map(|loc| self.func(loc).cast().into());+VMFuncRef {wasm_call,..*dtor.func_ref()diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rsindex 9d6be04d5c..5fb1130f09 100644--- a/crates/wasmtime/src/runtime/instantiate.rs+++ b/crates/wasmtime/src/runtime/instantiate.rs@@ -9,19 +9,20 @@ use crate::{code_memory::CodeMemory, profiling_agent::ProfilingAgent};use alloc::sync::Arc;use core::str;use wasmtime_environ::{- CompiledFunctionInfo, CompiledModuleInfo, DefinedFuncIndex, FilePos, FuncIndex, FunctionLoc,- FunctionName, Metadata, Module, ModuleInternedTypeIndex, PrimaryMap,+ CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, EntityRef, FilePos, FuncIndex,+ FuncKey, FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex,+ StaticModuleIndex,};/// A compiled wasm module, ready to be instantiated.pub struct CompiledModule {- module: Arc<Module>,- funcs: PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo>,- wasm_to_array_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>,- meta: Metadata,- code_memory: Arc<CodeMemory>,/// A unique ID used to register this module with the engine.unique_id: CompiledModuleId,+ code_memory: Arc<CodeMemory>,+ module: Arc<Module>,+ meta: Metadata,+ index: Arc<CompiledFunctionsTable>,+ /// Sorted list, by function index, of names we have for this module.func_names: Vec<FunctionName>,}@@ -45,15 +46,15 @@ impl CompiledModule {pub fn from_artifacts(code_memory: Arc<CodeMemory>,info: CompiledModuleInfo,+ index: Arc<CompiledFunctionsTable>,profiler: &dyn ProfilingAgent,) -> Result<Self> {let mut ret = Self {- module: Arc::new(info.module),- funcs: info.funcs,- wasm_to_array_trampolines: info.wasm_to_array_trampolines,+ unique_id: CompiledModuleId::new(),code_memory,+ module: Arc::new(info.module),meta: info.meta,- unique_id: CompiledModuleId::new(),+ index,func_names: info.func_names,};ret.register_profiling(profiler)?;@@ -65,7 +66,7 @@ impl CompiledModule {// TODO-Bug?: "code_memory" is not exclusive for this module in the case of components,// so we may be registering the same code range multiple times here.profiler.register_module(&self.code_memory.mmap()[..], &|addr| {- let (idx, _) = self.func_by_text_offset(addr)?;+ let idx = self.func_by_text_offset(addr)?;let idx = self.module.func_index(idx);let name = self.func_name(idx)?;let mut demangled = String::new();@@ -105,6 +106,10 @@ impl CompiledModule {&self.module}+ fn module_index(&self) -> StaticModuleIndex {+ self.module.module_index+ }+/// Looks up the `name` section name for the function index `idx`, if one/// was specified in the original wasm module.pub fn func_name(&self, idx: FuncIndex) -> Option<&str> {@@ -126,15 +131,15 @@ impl CompiledModule {pub fn finished_functions(&self,) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, &[u8])> + '_ {- self.funcs- .iter()- .map(move |(i, _)| (i, self.finished_function(i)))+ self.module+ .defined_func_indices()+ .map(|i| (i, self.finished_function(i)))}/// Returns the body of the function that `index` points to.#[inline]- pub fn finished_function(&self, index: DefinedFuncIndex) -> &[u8] {- let loc = self.funcs[index].wasm_func_loc;+ pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] {+ let loc = self.func_loc(def_func_index);&self.text()[loc.start as usize..][..loc.length as usize]}@@ -145,8 +150,10 @@ impl CompiledModule {////// These trampolines are used for array callers (e.g. `Func::new`)/// calling Wasm callees.- pub fn array_to_wasm_trampoline(&self, index: DefinedFuncIndex) -> Option<&[u8]> {- let loc = self.funcs[index].array_to_wasm_trampoline?;+ pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> {+ assert!(def_func_index.index() < self.module.num_defined_funcs());+ let key = FuncKey::ArrayToWasmTrampoline(self.module_index(), def_func_index);+ let loc = self.index.func_loc(key)?;Some(&self.text()[loc.start as usize..][..loc.length as usize])}@@ -155,67 +162,45 @@ impl CompiledModule {/// These trampolines are used for filling in/// `VMFuncRef::wasm_call` for `Func::wrap`-style host funcrefs/// that don't have access to a compiler when created.- pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> &[u8] {- let idx = match self- .wasm_to_array_trampolines- .binary_search_by_key(&signature, |entry| entry.0)- {- Ok(idx) => idx,- Err(_) => panic!("missing trampoline for {signature:?}"),- };-- let (_, loc) = self.wasm_to_array_trampolines[idx];- &self.text()[loc.start as usize..][..loc.length as usize]+ pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> Option<&[u8]> {+ let key = FuncKey::WasmToArrayTrampoline(signature);+ let loc = self.index.func_loc(key)?;+ Some(&self.text()[loc.start as usize..][..loc.length as usize])}/// Lookups a defined function by a program counter value.////// Returns the defined function index and the relative address of/// `text_offset` within the function itself.- pub fn func_by_text_offset(&self, text_offset: usize) -> Option<(DefinedFuncIndex, u32)> {+ pub fn func_by_text_offset(&self, text_offset: usize) -> Option<DefinedFuncIndex> {let text_offset = u32::try_from(text_offset).unwrap();-- let index = match self.funcs.binary_search_values_by_key(&text_offset, |e| {- debug_assert!(e.wasm_func_loc.length > 0);- // Return the inclusive "end" of the function- e.wasm_func_loc.start + e.wasm_func_loc.length - 1- }) {- Ok(k) => {- // Exact match, pc is at the end of this function- k- }- Err(k) => {- // Not an exact match, k is where `pc` would be "inserted"- // Since we key based on the end, function `k` might contain `pc`,- // so we'll validate on the range check below- k+ let key = self.index.func_by_text_offset(text_offset)?;+ match key {+ FuncKey::DefinedWasmFunction(module, def_func_index) => {+ debug_assert_eq!(module, self.module_index());+ Some(def_func_index)}- };-- let CompiledFunctionInfo { wasm_func_loc, .. } = self.funcs.get(index)?;- let start = wasm_func_loc.start;- let end = wasm_func_loc.start + wasm_func_loc.length;-- if text_offset < start || end < text_offset {- return None;+ _ => None,}-- Some((index, text_offset - wasm_func_loc.start))}/// Gets the function location information for a given function index.- pub fn func_loc(&self, index: DefinedFuncIndex) -> &FunctionLoc {- &self- .funcs- .get(index)+ pub fn func_loc(&self, def_func_index: DefinedFuncIndex) -> &FunctionLoc {+ assert!(def_func_index.index() < self.module.num_defined_funcs());+ let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index);+ self.index+ .func_loc(key).expect("defined function should be present")- .wasm_func_loc}/// Returns the original binary offset in the file that `index` was defined/// at.- pub fn func_start_srcloc(&self, index: DefinedFuncIndex) -> FilePos {- self.funcs[index].start_srcloc+ pub fn func_start_srcloc(&self, def_func_index: DefinedFuncIndex) -> FilePos {+ assert!(def_func_index.index() < self.module.num_defined_funcs());+ let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index);+ self.index+ .src_loc(key)+ .expect("defined function should be present")}/// Creates a new symbolication context which can be used to furtherdiff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rsindex 0420f7ee4a..35b7214058 100644--- a/crates/wasmtime/src/runtime/module.rs+++ b/crates/wasmtime/src/runtime/module.rs@@ -19,8 +19,8 @@ use core::ptr::NonNull;use std::{fs::File, path::Path};use wasmparser::{Parser, ValidPayload, Validator};use wasmtime_environ::{- CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind, TypeTrace, VMOffsets,- VMSharedTypeIndex,+ CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind,+ TypeTrace, VMOffsets, VMSharedTypeIndex,};#[cfg(feature = "gc")]use wasmtime_unwinder::ExceptionTable;@@ -495,13 +495,13 @@ impl Module {pub(crate) fn from_parts(engine: &Engine,code_memory: Arc<CodeMemory>,- info_and_types: Option<(CompiledModuleInfo, ModuleTypes)>,+ info_and_types: Option<(CompiledModuleInfo, CompiledFunctionsTable, ModuleTypes)>,) -> Result<Self> {// Acquire this module's metadata and type information, deserializing// it from the provided artifact if it wasn't otherwise provided// already.- let (mut info, mut types) = match info_and_types {- Some((info, types)) => (info, types),+ let (mut info, index, mut types) = match info_and_types {+ Some((info, index, types)) => (info, index, types),None => postcard::from_bytes(code_memory.wasmtime_info())?,};@@ -519,17 +519,23 @@ impl Module {// Package up all our data into a `CodeObject` and delegate to the final// step of module compilation.let code = Arc::new(CodeObject::new(code_memory, signatures, types.into()));- Module::from_parts_raw(engine, code, info, true)+ let index = Arc::new(index);+ Module::from_parts_raw(engine, code, info, index, true)}pub(crate) fn from_parts_raw(engine: &Engine,code: Arc<CodeObject>,info: CompiledModuleInfo,+ index: Arc<CompiledFunctionsTable>,serializable: bool,) -> Result<Self> {- let module =- CompiledModule::from_artifacts(code.code_memory().clone(), info, engine.profiler())?;+ let module = CompiledModule::from_artifacts(+ code.code_memory().clone(),+ info,+ index,+ engine.profiler(),+ )?;// Validate the module can be used with the current instance allocator.let offsets = VMOffsets::new(HostPtr, module.module());@@ -1106,6 +1112,7 @@ impl Module {let ptr = self.compiled_module().wasm_to_array_trampoline(trampoline_module_ty)+ .expect("always have a trampoline for the trampoline type").as_ptr().cast::<VMWasmCallFunction>().cast_mut();diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rsindex 8ac3d38d33..3089e4c4ef 100644--- a/crates/wasmtime/src/runtime/trap.rs+++ b/crates/wasmtime/src/runtime/trap.rs@@ -437,7 +437,7 @@ impl FrameInfo {/// if no information can be found.pub(crate) fn new(module: Module, text_offset: usize) -> Option<FrameInfo> {let compiled_module = module.compiled_module();- let (index, _func_offset) = compiled_module.func_by_text_offset(text_offset)?;+ let index = compiled_module.func_by_text_offset(text_offset)?;let func_start = compiled_module.func_start_srcloc(index);let instr = wasmtime_environ::lookup_file_pos(compiled_module.code_memory().address_map_data(),diff --git a/tests/all/coredump.rs b/tests/all/coredump.rsindex 945322fe78..53351b7663 100644--- a/tests/all/coredump.rs+++ b/tests/all/coredump.rs@@ -37,6 +37,7 @@ fn coredump_attached_to_error() -> Result<()> {#[test]#[cfg_attr(miri, ignore)]fn coredump_has_stack() -> Result<()> {+ let _ = env_logger::try_init();let mut config = Config::default();config.coredump_on_trap(true);let engine = Engine::new(&config).unwrap();