Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Lookup functions in the text section by `FuncKey` at runtime (#11630)

Nick Fitzgerald1806c26562ec
diff --git a/cranelift/entity/src/primary.rs b/cranelift/entity/src/primary.rs
index cd01993246..6c1dbc18ad 100644
--- a/cranelift/entity/src/primary.rs
+++ b/cranelift/entity/src/primary.rs
@@ -67,6 +67,11 @@ where
self.elems.get(k.index())
}
+ /// Get the slice of values associated with the given range of keys, if any.
+ pub fn get_range(&self, range: core::ops::Range<K>) -> Option<&[V]> {
+ self.elems.get(range.start.index()..range.end.index())
+ }
+
/// Get the element at `k` if it exists, mutable version.
pub fn get_mut(&mut self, k: K) -> Option<&mut V> {
self.elems.get_mut(k.index())
@@ -97,6 +102,11 @@ where
self.elems.iter_mut()
}
+ /// Get this map's underlying values as a slice.
+ pub fn as_values_slice(&self) -> &[V] {
+ &self.elems
+ }
+
/// Iterate over all the keys and values in this map.
pub fn iter(&self) -> Iter<'_, K, V> {
Iter::new(self.elems.iter())
@@ -294,6 +304,18 @@ where
}
}
+impl<K, V> Extend<V> for PrimaryMap<K, V>
+where
+ K: EntityRef,
+{
+ fn extend<T>(&mut self, iter: T)
+ where
+ T: IntoIterator<Item = V>,
+ {
+ self.elems.extend(iter);
+ }
+}
+
impl<K, V> From<Vec<V>> for PrimaryMap<K, V>
where
K: EntityRef,
diff --git a/crates/environ/src/address_map.rs b/crates/environ/src/address_map.rs
index fe69dcebc6..c9cafcaa8d 100644
--- a/crates/environ/src/address_map.rs
+++ b/crates/environ/src/address_map.rs
@@ -30,10 +30,22 @@ impl FilePos {
FilePos(pos)
}
+ /// Get the null file position.
+ pub fn none() -> FilePos {
+ FilePos(u32::MAX)
+ }
+
+ /// Is this the null file position?
+ #[inline]
+ pub fn is_none(&self) -> bool {
+ *self == FilePos::none()
+ }
+
/// Returns the offset that this offset was created with.
///
- /// Note that the `Default` implementation will return `None` here, whereas
- /// positions created with `FilePos::new` will return `Some`.
+ /// Note that positions created with `FilePos::none` and the `Default`
+ /// implementation will return `None` here, whereas positions created with
+ /// `FilePos::new` will return `Some`.
pub fn file_offset(self) -> Option<u32> {
if self.0 == u32::MAX {
None
@@ -45,7 +57,7 @@ impl FilePos {
impl Default for FilePos {
fn default() -> FilePos {
- FilePos(u32::MAX)
+ FilePos::none()
}
}
diff --git a/crates/environ/src/compile/mod.rs b/crates/environ/src/compile/mod.rs
index 5a1f2c516a..0bec4bed58 100644
--- a/crates/environ/src/compile/mod.rs
+++ b/crates/environ/src/compile/mod.rs
@@ -3,10 +3,9 @@
use crate::prelude::*;
use crate::{
- DefinedFuncIndex, FlagValue, FunctionLoc, ObjectKind, PrimaryMap, StaticModuleIndex, TripleExt,
- WasmError, WasmFuncType,
+ DefinedFuncIndex, FlagValue, FuncKey, FunctionLoc, ObjectKind, PrimaryMap, StaticModuleIndex,
+ TripleExt, Tunables, WasmError, WasmFuncType, obj,
};
-use crate::{Tunables, obj};
use anyhow::Result;
use object::write::{Object, SymbolId};
use object::{Architecture, BinaryFormat, FileFlags};
@@ -17,7 +16,6 @@ use std::path;
use std::sync::Arc;
mod address_map;
-mod key;
mod module_artifacts;
mod module_environ;
mod module_types;
@@ -25,7 +23,6 @@ mod stack_maps;
mod trap_encoding;
pub use self::address_map::*;
-pub use self::key::*;
pub use self::module_artifacts::*;
pub use self::module_environ::*;
pub use self::module_types::*;
diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rs
index 4fc3a3792a..a4301a1fed 100644
--- a/crates/environ/src/compile/module_artifacts.rs
+++ b/crates/environ/src/compile/module_artifacts.rs
@@ -3,9 +3,8 @@
use crate::prelude::*;
use crate::{
- CompiledFunctionInfo, CompiledModuleInfo, DebugInfoData, DefinedFuncIndex, FunctionLoc,
- FunctionName, MemoryInitialization, Metadata, ModuleInternedTypeIndex, ModuleTranslation,
- PrimaryMap, Tunables, obj,
+ CompiledModuleInfo, DebugInfoData, FunctionName, MemoryInitialization, Metadata,
+ ModuleTranslation, Tunables, obj,
};
use anyhow::{Result, bail};
use object::SectionKind;
@@ -113,12 +112,7 @@ impl<'a> ObjectBuilder<'a> {
/// Returns the `CompiledModuleInfo` corresponding to this core Wasm module
/// as a result of this append operation. This is then serialized into the
/// final artifact by the caller.
- pub fn append(
- &mut self,
- translation: ModuleTranslation<'_>,
- funcs: PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo>,
- wasm_to_array_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>,
- ) -> Result<CompiledModuleInfo> {
+ pub fn append(&mut self, translation: ModuleTranslation<'_>) -> Result<CompiledModuleInfo> {
let ModuleTranslation {
mut module,
debuginfo,
@@ -219,8 +213,6 @@ impl<'a> ObjectBuilder<'a> {
Ok(CompiledModuleInfo {
module,
- funcs,
- wasm_to_array_trampolines,
func_names,
meta: Metadata {
has_unparsed_debuginfo,
diff --git a/crates/environ/src/component/artifacts.rs b/crates/environ/src/component/artifacts.rs
index 2aabb21ef3..22b9c2ce79 100644
--- a/crates/environ/src/component/artifacts.rs
+++ b/crates/environ/src/component/artifacts.rs
@@ -2,8 +2,8 @@
//! which are serialized with `bincode` into output ELF files.
use crate::{
- CompiledModuleInfo, FunctionLoc, PrimaryMap, StaticModuleIndex,
- component::{Component, ComponentTypes, TrampolineIndex, TypeComponentIndex},
+ CompiledFunctionsTable, CompiledModuleInfo, PrimaryMap, StaticModuleIndex,
+ component::{Component, ComponentTypes, TypeComponentIndex},
};
use serde_derive::{Deserialize, Serialize};
@@ -14,6 +14,8 @@ pub struct ComponentArtifacts {
pub ty: TypeComponentIndex,
/// Information all kept available at runtime as-is.
pub info: CompiledComponentInfo,
+ /// The index of every compiled function's location in the text section.
+ pub table: CompiledFunctionsTable,
/// Type information for this component and all contained modules.
pub types: ComponentTypes,
/// Serialized metadata about all included core wasm modules.
@@ -25,16 +27,4 @@ pub struct ComponentArtifacts {
pub struct CompiledComponentInfo {
/// Type information calculated during translation about this component.
pub component: Component,
-
- /// Where lowered function wasm-call trampolines that end up in a
- /// `VMFuncRef` are located within the `text` section of `code_memory`.
- pub wasm_call_trampolines: PrimaryMap<TrampolineIndex, FunctionLoc>,
-
- /// Where lowered function array-call trampolines that end up in a
- /// `VMFuncRef` are located within the `text` section of `code_memory`.
- pub array_call_trampolines: PrimaryMap<TrampolineIndex, FunctionLoc>,
-
- /// The location of the wasm-to-array trampoline for the `resource.drop`
- /// intrinsic.
- pub resource_drop_wasm_to_array_trampoline: Option<FunctionLoc>,
}
diff --git a/crates/environ/src/compile/key.rs b/crates/environ/src/key.rs
similarity index 99%
rename from crates/environ/src/compile/key.rs
rename to crates/environ/src/key.rs
index 58c43ecb4b..3c438be6f3 100644
--- a/crates/environ/src/compile/key.rs
+++ b/crates/environ/src/key.rs
@@ -230,22 +230,8 @@ impl FuncKey {
kind << Self::KIND_OFFSET
}
- /// Get this key's kind.
- pub fn kind(self) -> FuncKeyKind {
- self.namespace().kind()
- }
-
- /// Get this key's namespace.
- pub fn namespace(self) -> FuncKeyNamespace {
- self.into_parts().0
- }
-
- /// Get this key's index.
- pub fn index(self) -> FuncKeyIndex {
- self.into_parts().1
- }
-
/// Split this key into its namespace and index halves.
+ #[inline]
pub fn into_parts(self) -> (FuncKeyNamespace, FuncKeyIndex) {
let (namespace, index) = match self {
FuncKey::DefinedWasmFunction(module, def_func) => {
@@ -294,16 +280,19 @@ impl FuncKey {
(FuncKeyNamespace(namespace), FuncKeyIndex(index))
}
- /// Create a key from its namespace and index parts.
- ///
- /// Should only be called with namespaces and indices that are ultimately
- /// derived from the same key. For example, if you attempt to pair an index
- /// and namespace that come from different keys, that may panic. If it
- /// happens not to panic, you'll end up with a valid key that names an
- /// arbitrary function in the given namespace, but that function probably
- /// does not actually exist in the compilation artifact.
- pub fn from_parts(namespace: FuncKeyNamespace, index: FuncKeyIndex) -> Self {
- Self::from_raw_parts(namespace.into_raw(), index.into_raw())
+ /// Get this key's kind.
+ pub fn kind(self) -> FuncKeyKind {
+ self.namespace().kind()
+ }
+
+ /// Get this key's namespace.
+ pub fn namespace(self) -> FuncKeyNamespace {
+ self.into_parts().0
+ }
+
+ /// Get this key's index.
+ pub fn index(self) -> FuncKeyIndex {
+ self.into_parts().1
}
/// Get the raw, underlying `(namespace, index)` representation of this
@@ -320,6 +309,18 @@ impl FuncKey {
(ns.into_raw(), index.into_raw())
}
+ /// Create a key from its namespace and index parts.
+ ///
+ /// Should only be called with namespaces and indices that are ultimately
+ /// derived from the same key. For example, if you attempt to pair an index
+ /// and namespace that come from different keys, that may panic. If it
+ /// happens not to panic, you'll end up with a valid key that names an
+ /// arbitrary function in the given namespace, but that function probably
+ /// does not actually exist in the compilation artifact.
+ pub fn from_parts(namespace: FuncKeyNamespace, index: FuncKeyIndex) -> Self {
+ Self::from_raw_parts(namespace.into_raw(), index.into_raw())
+ }
+
/// Create a key from its raw, underlying representation.
///
/// Should only be given the results of a previous call to
diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs
index 43501dd940..19bff1c498 100644
--- a/crates/environ/src/lib.rs
+++ b/crates/environ/src/lib.rs
@@ -26,6 +26,7 @@ mod error;
mod ext;
mod gc;
mod hostcall;
+mod key;
mod module;
mod module_artifacts;
mod module_types;
@@ -46,6 +47,7 @@ pub use crate::demangling::*;
pub use crate::error::*;
pub use crate::gc::*;
pub use crate::hostcall::*;
+pub use crate::key::*;
pub use crate::module::*;
pub use crate::module_artifacts::*;
pub use crate::module_types::*;
diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs
index 974b4f0fcc..73481147b3 100644
--- a/crates/environ/src/module_artifacts.rs
+++ b/crates/environ/src/module_artifacts.rs
@@ -2,27 +2,16 @@
//! with `bincode` as part of a module's compilation process.
use crate::prelude::*;
-use crate::{DefinedFuncIndex, FilePos, FuncIndex, Module, ModuleInternedTypeIndex, PrimaryMap};
-use core::fmt;
+use crate::{FilePos, FuncIndex, FuncKey, FuncKeyIndex, FuncKeyKind, FuncKeyNamespace, Module};
use core::ops::Range;
-use core::str;
+use core::{fmt, u32};
+use core::{iter, str};
+use cranelift_entity::{EntityRef, PrimaryMap};
use serde_derive::{Deserialize, Serialize};
-/// Secondary in-memory results of function compilation.
-#[derive(Clone, Serialize, Deserialize)]
-pub struct CompiledFunctionInfo {
- /// Where this function was found in the original wasm file.
- pub start_srcloc: FilePos,
- /// The [`FunctionLoc`] indicating the location of this function in the text
- /// section of the competition artifact.
- pub wasm_func_loc: FunctionLoc,
- /// A trampoline for array callers (e.g. `Func::new`) calling into this function (if needed).
- pub array_to_wasm_trampoline: Option<FunctionLoc>,
-}
-
/// Description of where a function is located in the text section of a
/// compiled image.
-#[derive(Copy, Clone, Debug, Serialize, Deserialize)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct FunctionLoc {
/// The byte offset from the start of the text section where this
/// function starts.
@@ -31,6 +20,504 @@ pub struct FunctionLoc {
pub length: u32,
}
+impl FunctionLoc {
+ /// Is this an empty function location?
+ #[inline]
+ pub fn is_empty(&self) -> bool {
+ self.length == 0
+ }
+}
+
+/// A builder for a `CompiledFunctionsTable`.
+pub struct CompiledFunctionsTableBuilder {
+ inner: CompiledFunctionsTable,
+}
+
+impl CompiledFunctionsTableBuilder {
+ /// Create a new builder.
+ pub fn new() -> Self {
+ Self {
+ inner: CompiledFunctionsTable {
+ namespaces: PrimaryMap::new(),
+ func_loc_starts: PrimaryMap::new(),
+ sparse_starts: PrimaryMap::new(),
+ src_loc_starts: PrimaryMap::new(),
+ sparse_indices: PrimaryMap::new(),
+ func_locs: PrimaryMap::new(),
+ src_locs: PrimaryMap::new(),
+ },
+ }
+ }
+
+ fn last_namespace(&self) -> Option<FuncKeyNamespace> {
+ let (_, &ns) = self.inner.namespaces.last()?;
+ Some(ns)
+ }
+
+ fn last_key_index(&self) -> Option<FuncKeyIndex> {
+ let (ns_idx, ns) = self.inner.namespaces.last()?;
+ let start = self.inner.func_loc_starts[ns_idx];
+ if CompiledFunctionsTable::is_dense(ns.kind()) {
+ let len = self.inner.func_locs.len();
+ let len = u32::try_from(len).unwrap();
+ let key_index = len - start.as_u32();
+ let key_index = FuncKeyIndex::from_raw(key_index);
+ Some(key_index)
+ } else {
+ let sparse_start = self.inner.sparse_starts[ns_idx];
+ if self.inner.sparse_indices.len() > sparse_start.index() {
+ let (_, &key_index) = self.inner.sparse_indices.last().unwrap();
+ Some(key_index)
+ } else {
+ None
+ }
+ }
+ }
+
+ fn last_func_loc(&self) -> Option<FunctionLoc> {
+ let (_, &loc) = self.inner.func_locs.last()?;
+ Some(loc)
+ }
+
+ /// Push a new entry into this builder.
+ ///
+ /// Panics if the key or function location is out of order.
+ pub fn push_func(
+ &mut self,
+ key: FuncKey,
+ func_loc: FunctionLoc,
+ src_loc: FilePos,
+ ) -> &mut Self {
+ let (key_ns, key_index) = key.into_parts();
+
+ assert!(
+ self.last_namespace().is_none_or(|ns| ns <= key_ns),
+ "`FuncKey`s pushed out of order"
+ );
+ assert!(
+ self.last_key_index().is_none_or(
+ |i| i <= key_index || self.last_namespace().is_some_and(|ns| ns != key_ns)
+ ),
+ "`FuncKey`s pushed out of order"
+ );
+ assert!(
+ self.last_func_loc()
+ .is_none_or(|l| l.start + l.length <= func_loc.start),
+ "`FunctionLoc`s pushed out of order"
+ );
+
+ // Make sure that there is a `kind` entry for this key's kind.
+ let kind_start_index = self
+ .inner
+ .namespaces
+ .last()
+ .and_then(|(ns_idx, ns)| {
+ if *ns == key_ns {
+ Some(self.inner.func_loc_starts[ns_idx])
+ } else {
+ None
+ }
+ })
+ .unwrap_or_else(|| {
+ let start = self.inner.func_locs.next_key();
+ let ns_idx = self.inner.namespaces.push(key_ns);
+ let ns_idx2 = self.inner.func_loc_starts.push(start);
+ let ns_idx3 = self
+ .inner
+ .sparse_starts
+ .push(self.inner.sparse_indices.next_key());
+ let ns_idx4 = self
+ .inner
+ .src_loc_starts
+ .push(self.inner.src_locs.next_key());
+ debug_assert_eq!(ns_idx, ns_idx2);
+ debug_assert_eq!(ns_idx, ns_idx3);
+ debug_assert_eq!(ns_idx, ns_idx4);
+ start
+ });
+
+ if CompiledFunctionsTable::is_dense(key.kind()) {
+ // Figure out the index within `func_locs` for this key's entry.
+ let index = kind_start_index.as_u32() + key_index.into_raw();
+ let index = FuncLocIndex::from_u32(index);
+ debug_assert!(self.inner.func_locs.get(index).is_none());
+
+ // Fill in null entries for any key indices that have been omitted.
+ //
+ // Note that we need a null `FunctionLoc`, but we also need
+ // `func_locs` to be sorted so that we support reverse
+ // lookups. Therefore, we take care to create an empty function
+ // location that starts at the text offset that the previous one (if
+ // any) ends at, and use that as our null entry.
+ let null_func_loc = FunctionLoc {
+ start: self
+ .last_func_loc()
+ .map(|l| l.start + l.length)
+ .unwrap_or_default(),
+ length: 0,
+ };
+ let gap = index.index() - self.inner.func_locs.len();
+ self.inner
+ .func_locs
+ .extend(iter::repeat(null_func_loc).take(gap));
+ debug_assert_eq!(index, self.inner.func_locs.next_key());
+
+ if CompiledFunctionsTable::has_src_locs(key_ns.kind()) {
+ self.inner
+ .src_locs
+ .extend(iter::repeat(FilePos::none()).take(gap));
+ }
+ } else {
+ debug_assert!(
+ src_loc.is_none(),
+ "sparse keys do not have source locations"
+ );
+ self.inner.sparse_indices.push(key_index);
+ }
+
+ // And finally, we push this entry.
+ self.inner.func_locs.push(func_loc);
+ if CompiledFunctionsTable::has_src_locs(key_ns.kind()) {
+ self.inner.src_locs.push(src_loc);
+ } else {
+ debug_assert!(src_loc.is_none());
+ }
+
+ self
+ }
+
+ /// Finish construction of the `CompiledFunctionsTable`.
+ pub fn finish(self) -> CompiledFunctionsTable {
+ self.inner
+ }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
+struct NamespaceIndex(u32);
+cranelift_entity::entity_impl!(NamespaceIndex);
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
+struct FuncLocIndex(u32);
+cranelift_entity::entity_impl!(FuncLocIndex);
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
+struct SparseIndex(u32);
+cranelift_entity::entity_impl!(SparseIndex);
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
+struct SrcLocIndex(u32);
+cranelift_entity::entity_impl!(SrcLocIndex);
+
+/// A table describing the set of functions compiled into an artifact, their
+/// locations within the text section, and etc...
+///
+/// Logically, this type is a map from a `FuncKey` to the associated function's
+///
+/// * location within the associated text section, and
+/// * optional source location.
+///
+/// How this map is *actually* implemented is with a series of lookup and binary
+/// search tables, split out in a data-oriented, struct-of-arrays style. We
+/// organize the data in this way is service of three goals:
+///
+/// 1. Provide fast look ups: We need to look up the metadata for a function by
+/// its key at runtime. During instantiation, for example, we need to create
+/// `VMFuncRef`s for escaping functions and this requires looking up the
+/// locations of those Wasm functions and their associated array-to-Wasm
+/// trampolines.
+///
+/// 2. Keep memory overheads low and code size small: This type is serialized
+/// into all of our ELF artifacts and deserialized into all `Module`s and
+/// `Component`s at runtime.
+///
+/// 3. Be generic over any kind of function (whether defined Wasm function,
+/// trampoline, or etc...) that we compile: Adding a new kind of trampoline,
+/// for example, should not require updating this structure to add a new
+/// table of the function locations for just trampolines of that new kind. We
+/// should be able to store and query all kinds of functions uniformly.
+//
+// TODO: This structure could be directly encoded as raw ELF sections, instead
+// of a `struct` containing a bunch of `PrimaryMap`s, which would allow us to
+// avoid the serialize/deserialize runtime costs.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct CompiledFunctionsTable {
+ /// A binary-search index for this table, mapping raw `FuncKeyNamespace`s to
+ /// their associated `NamespaceIndex`. That `NamespaceIndex` can then be
+ /// used to find the range of other entity indices that are specific to that
+ /// namespace.
+ namespaces: PrimaryMap<NamespaceIndex, FuncKeyNamespace>,
+
+ /// `self.func_loc_starts[i]..self.func_loc_starts[i+1]` describes the range
+ /// within `self.func_locs` whose entries are associated with the namespace
+ /// `self.index[i]`.
+ ///
+ /// When `self.func_loc_starts[i+1]` is out of bounds, then the range is to
+ /// the end of `self.func_locs`.
+ func_loc_starts: PrimaryMap<NamespaceIndex, FuncLocIndex>,
+
+ /// `self.sparse_starts[i]..self.sparse_starts[i+1]` describes the range
+ /// within `self.sparse_indices` whose entries are associated with the
+ /// namespace `self.index[i]`.
+ ///
+ /// When `self.sparse_starts[i+1]` is out of bounds, then the range is to
+ /// the end of `self.sparse_indices`.
+ ///
+ /// Entries are only valid for sparse, non-dense namespaces.
+ sparse_starts: PrimaryMap<NamespaceIndex, SparseIndex>,
+
+ /// `self.src_loc_starts[i]..self.src_loc_starts[i+1]` describes the range
+ /// within `self.src_loc_indices` whose entries are associated with the
+ /// namespace `self.index[i]`.
+ ///
+ /// When `self.src_loc_starts[i+1]` is out of bounds, then the range is to
+ /// the end of `self.src_locs`.
+ ///
+ /// Entries are only valid for namespaces whose functions have source
+ /// locations.
+ src_loc_starts: PrimaryMap<NamespaceIndex, SrcLocIndex>,
+
+ /// `self.sparse_indices[i]` contains the index part of
+ /// `FuncKey::from_parts(ns, index)` where `ns` is determined by
+ /// `self.sparse_starts` and is a sparse, non-dense key kind. (Note that for
+ /// dense keys, this information is implicitly encoded in their offset from
+ /// the namespace's start index.)
+ ///
+ /// This is sorted to allow for binary searches.
+ sparse_indices: PrimaryMap<SparseIndex, FuncKeyIndex>,
+
+ /// `self.func_locs[i]` contains the location within the text section of
+ /// `FuncKey::from_parts(self.namespaces[ns], i - start)`'s function, where
+ /// `ns` and `start` are determined by `self.func_loc_starts`.
+ ///
+ /// Values are sorted by function location to support reverse queries from
+ /// function location back to `FuncKey`.
+ ///
+ /// The absence of a function location (for gaps in dense namespaces) is
+ /// represented with `FunctionLoc::none()`.
+ func_locs: PrimaryMap<FuncLocIndex, FunctionLoc>,
+
+ /// `self.src_locs[i]` contains the initial source location of
+ /// `FuncKey::from_parts(self.namespaces[ns], i - start)`'s function, where
+ /// `ns` and `start` are determined by `self.src_loc_starts`.
+ ///
+ /// The absence of a source location is represented by `FilePos::none()`.
+ src_locs: PrimaryMap<SrcLocIndex, FilePos>,
+}
+
+impl CompiledFunctionsTable {
+ #[inline]
+ fn namespace_index(&self, namespace: FuncKeyNamespace) -> Option<NamespaceIndex> {
+ const LINEAR_SEARCH_LIMIT: usize = 32;
+ if self.namespaces.len() <= LINEAR_SEARCH_LIMIT {
+ self.namespaces
+ .iter()
+ .find_map(|(idx, ns)| if *ns == namespace { Some(idx) } else { None })
+ } else {
+ self.namespaces
+ .binary_search_values_by_key(&namespace, |ns| *ns)
+ .ok()
+ }
+ }
+
+ #[inline]
+ fn func_loc_range(&self, ns_idx: NamespaceIndex) -> Range<FuncLocIndex> {
+ let start = self.func_loc_starts[ns_idx];
+ let next_ns_idx = NamespaceIndex::from_u32(ns_idx.as_u32() + 1);
+ let end = self
+ .func_loc_starts
+ .get(next_ns_idx)
+ .copied()
+ .unwrap_or_else(|| self.func_locs.next_key());
+ start..end
+ }
+
+ fn sparse_range(&self, ns_idx: NamespaceIndex) -> Range<SparseIndex> {
+ debug_assert!(!Self::is_dense(self.namespaces[ns_idx].kind()));
+ let start = self.sparse_starts[ns_idx];
+ let next_ns_idx = NamespaceIndex::from_u32(ns_idx.as_u32() + 1);
+ let end = self
+ .sparse_starts
+ .get(next_ns_idx)
+ .copied()
+ .unwrap_or_else(|| self.sparse_indices.next_key());
+ start..end
+ }
+
+ fn src_loc_range(&self, ns_idx: NamespaceIndex) -> Range<SrcLocIndex> {
+ debug_assert!(Self::has_src_locs(self.namespaces[ns_idx].kind()));
+ let start = self.src_loc_starts[ns_idx];
+ let next_ns_idx = NamespaceIndex::from_u32(ns_idx.as_u32() + 1);
+ let end = self
+ .src_loc_starts
+ .get(next_ns_idx)
+ .copied()
+ .unwrap_or_else(|| self.src_locs.next_key());
+ start..end
+ }
+
+ /// Get the index within `self.{func_locs,src_locs}` that is associated with
+ /// the given `key`, if any.
+ #[inline]
+ fn func_loc_index(&self, key: FuncKey) -> Option<FuncLocIndex> {
+ let (key_ns, key_index) = key.into_parts();
+ let ns_idx = self.namespace_index(key_ns)?;
+ let Range { start, end } = self.func_loc_range(ns_idx);
+
+ let index = if Self::is_dense(key.kind()) {
+ let index = start.as_u32().checked_add(key_index.into_raw())?;
+ FuncLocIndex::from_u32(index)
+ } else {
+ let sparse_range = self.sparse_range(ns_idx);
+ let sparse_subslice = self.sparse_indices.get_range(sparse_range).unwrap();
+ match sparse_subslice.binary_search(&key_index) {
+ Ok(i) => FuncLocIndex::new(start.index() + i),
+ Err(_) => return None,
+ }
+ };
+
+ if index < end { Some(index) } else { None }
+ }
+
+ /// Get the location of the function associated with the given `key` inside
+ /// the text section, if any.
+ #[inline]
+ pub fn func_loc(&self, key: FuncKey) -> Option<&FunctionLoc> {
+ let index = self.func_loc_index(key)?;
+ let loc = &self.func_locs[index];
+ if loc.is_empty() { None } else { Some(loc) }
+ }
+
+ fn src_loc_index(&self, key: FuncKey) -> Option<SrcLocIndex> {
+ let (key_ns, key_index) = key.into_parts();
+ if !Self::has_src_locs(key_ns.kind()) {
+ return None;
+ }
+
+ let ns_idx = self.namespace_index(key_ns)?;
+ let Range { start, end } = self.src_loc_range(ns_idx);
+
+ debug_assert!(Self::is_dense(key_ns.kind()));
+ let index = start.as_u32().checked_add(key_index.into_raw())?;
+ let index = SrcLocIndex::from_u32(index);
+ if index >= end {
+ return None;
+ }
+
+ Some(index)
+ }
+
+ /// Get the initial source location of the function associated with the
+ /// given `key`, if any.
+ pub fn src_loc(&self, key: FuncKey) -> Option<FilePos> {
+ let index = self.src_loc_index(key)?;
+ let loc = self.src_locs[index];
+ if loc.is_none() { None } else { Some(loc) }
+ }
+
+ /// Given an offset into the text section, get the key for its associated
+ /// function and its offset within that function.
+ pub fn func_by_text_offset(&self, text_offset: u32) -> Option<FuncKey> {
+ let index = match self.func_locs.as_values_slice().binary_search_by(|loc| {
+ if loc.is_empty() {
+ loc.start
+ .cmp(&text_offset)
+ .then_with(|| core::cmp::Ordering::Less)
+ } else {
+ if loc.start > text_offset {
+ core::cmp::Ordering::Greater
+ } else if loc.start + loc.length <= text_offset {
+ core::cmp::Ordering::Less
+ } else {
+ debug_assert!(loc.start <= text_offset);
+ debug_assert!(text_offset < loc.start + loc.length);
+ core::cmp::Ordering::Equal
+ }
+ }
+ }) {
+ // Exact match, the offset is at the end of this function.
+ Ok(k) => k,
+ // Not an exact match: `k` is where the offset would be
+ // "inserted". Since we key based on the end, function `k` might
+ // contain the offset, so we'll validate on the range check
+ // below.
+ Err(k) => k,
+ };
+ let index = FuncLocIndex::new(index);
+
+ // Make sure that the text offset is actually within this function.
+ // Non-exact binary search results can either be because we have a text
+ // offset within a function but not exactly at its inclusive end, or
+ // because the text offset is not within any of our functions. We filter
+ // that latter case out with this check.
+ let loc = self.func_locs.get(index)?;
+ let start = loc.start;
+ let end = start + loc.length;
+ if text_offset < start || end < text_offset {
+ return None;
+ }
+
+ let ns_idx = match self
+ .func_loc_starts
+ .binary_search_values_by_key(&index, |s| *s)
+ {
+ // Exact match: `i` is the entry's index.
+ Ok(i) => i,
+ // Not an exact match: the index, if it were the start of a
+ // namespace's range, would be at `i`. Therefore, our namespace
+ // entry is actually at index `i - 1`.
+ Err(i) => {
+ let i = i.as_u32();
+ assert_ne!(i, 0);
+ NamespaceIndex::from_u32(i - 1)
+ }
+ };
+ let key_ns = self.namespaces[ns_idx];
+ let start = self.func_loc_starts[ns_idx];
+
+ let key_index = if Self::is_dense(key_ns.kind()) {
+ let key_index = index.as_u32() - start.as_u32();
+ FuncKeyIndex::from_raw(key_index)
+ } else {
+ let sparse_index = index.as_u32() - start.as_u32();
+ let sparse_index = SparseIndex::from_u32(sparse_index);
+ self.sparse_indices[sparse_index]
+ };
+ let key = FuncKey::from_parts(key_ns, key_index);
+
+ Some(key)
+ }
+
+ /// Whether the given kind's index space is (generally) densely populated
+ /// and therefore we should densely pack them in the table for `O(1)`
+ /// lookups; otherwise, we should avoid code size bloat by using the sparse
+ /// table indirection and `O(log n)` binary search lookups.
+ fn is_dense(kind: FuncKeyKind) -> bool {
+ match kind {
+ FuncKeyKind::DefinedWasmFunction
+ | FuncKeyKind::WasmToArrayTrampoline
+ | FuncKeyKind::PulleyHostCall => true,
+
+ FuncKeyKind::ArrayToWasmTrampoline | FuncKeyKind::WasmToBuiltinTrampoline => false,
+
+ #[cfg(feature = "component-model")]
+ FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline => true,
+ }
+ }
+
+ /// Whether the given function kind has source locations or not.
+ fn has_src_locs(kind: FuncKeyKind) -> bool {
+ match kind {
+ FuncKeyKind::DefinedWasmFunction => true,
+ FuncKeyKind::ArrayToWasmTrampoline
+ | FuncKeyKind::WasmToArrayTrampoline
+ | FuncKeyKind::WasmToBuiltinTrampoline
+ | FuncKeyKind::PulleyHostCall => false,
+ #[cfg(feature = "component-model")]
+ FuncKeyKind::ComponentTrampoline | FuncKeyKind::ResourceDropTrampoline => false,
+ }
+ }
+}
+
/// Secondary in-memory results of module compilation.
///
/// This opaque structure can be optionally passed back to
@@ -40,18 +527,11 @@ pub struct CompiledModuleInfo {
/// Type information about the compiled WebAssembly module.
pub module: Module,
- /// Metadata about each compiled function.
- pub funcs: PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo>,
+ /// General compilation metadata.
+ pub meta: Metadata,
/// Sorted list, by function index, of names we have for this module.
pub func_names: Vec<FunctionName>,
-
- /// Metadata about wasm-to-array trampolines. Used when exposing a native
- /// callee (e.g. `Func::wrap`) to a Wasm caller. Sorted by signature index.
- pub wasm_to_array_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>,
-
- /// General compilation metadata.
- pub meta: Metadata,
}
/// The name of a function stored in the
@@ -117,3 +597,139 @@ pub enum ObjectKind {
/// A component compilation artifact
Component,
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::{DefinedFuncIndex, StaticModuleIndex};
+
+ fn func_loc(range: Range<u32>) -> FunctionLoc {
+ FunctionLoc {
+ start: range.start,
+ length: range.end - range.start,
+ }
+ }
+
+ fn def_func_key(m: u32, f: u32) -> FuncKey {
+ FuncKey::DefinedWasmFunction(
+ StaticModuleIndex::from_u32(m),
+ DefinedFuncIndex::from_u32(f),
+ )
+ }
+
+ fn array_to_wasm_tramp_key(m: u32, f: u32) -> FuncKey {
+ FuncKey::ArrayToWasmTrampoline(
+ StaticModuleIndex::from_u32(m),
+ DefinedFuncIndex::from_u32(f),
+ )
+ }
+
+ fn make_test_table() -> CompiledFunctionsTable {
+ let mut builder = CompiledFunctionsTableBuilder::new();
+
+ builder
+ // ========= Dense =========
+ .push_func(def_func_key(0, 0), func_loc(0..10), FilePos::new(111))
+ .push_func(def_func_key(0, 1), func_loc(10..20), FilePos::new(222))
+ .push_func(def_func_key(0, 2), func_loc(20..30), FilePos::none())
+ // Gap in dense keys!
+ .push_func(def_func_key(0, 5), func_loc(30..40), FilePos::new(333))
+ // ========= Sparse =========
+ .push_func(
+ array_to_wasm_tramp_key(0, 1),
+ func_loc(100..110),
+ FilePos::none(),
+ )
+ .push_func(
+ array_to_wasm_tramp_key(0, 2),
+ func_loc(110..120),
+ FilePos::none(),
+ )
+ .push_func(
+ array_to_wasm_tramp_key(0, 5),
+ func_loc(120..130),
+ FilePos::none(),
+ );
+
+ builder.finish()
+ }
+
+ #[test]
+ fn src_locs() {
+ let table = make_test_table();
+
+ for (key, expected) in [
+ (def_func_key(0, 0), Some(FilePos::new(111))),
+ (def_func_key(0, 1), Some(FilePos::new(222))),
+ (def_func_key(0, 2), None),
+ (def_func_key(0, 3), None),
+ (def_func_key(0, 4), None),
+ (def_func_key(0, 5), Some(FilePos::new(333))),
+ (array_to_wasm_tramp_key(0, 0), None),
+ (array_to_wasm_tramp_key(0, 1), None),
+ (array_to_wasm_tramp_key(0, 2), None),
+ (array_to_wasm_tramp_key(0, 3), None),
+ (array_to_wasm_tramp_key(0, 4), None),
+ (array_to_wasm_tramp_key(0, 5), None),
+ ] {
+ eprintln!("Checking key {key:?}");
+ let actual = table.src_loc(key);
+ assert_eq!(expected, actual);
+ }
+ }
+
+ #[test]
+ fn func_locs() {
+ let table = make_test_table();
+
+ for (key, expected) in [
+ (def_func_key(0, 0), Some(0)),
+ (def_func_key(0, 1), Some(10)),
+ (def_func_key(0, 2), Some(20)),
+ (def_func_key(0, 3), None),
+ (def_func_key(0, 4), None),
+ (def_func_key(0, 5), Some(30)),
+ (array_to_wasm_tramp_key(0, 0), None),
+ (array_to_wasm_tramp_key(0, 1), Some(100)),
+ (array_to_wasm_tramp_key(0, 2), Some(110)),
+ (array_to_wasm_tramp_key(0, 3), None),
+ (array_to_wasm_tramp_key(0, 4), None),
+ (array_to_wasm_tramp_key(0, 5), Some(120)),
+ ] {
+ let actual = table.func_loc(key);
+ match (expected, actual) {
+ (None, None) => {}
+ (Some(expected), Some(actual)) => assert_eq!(expected, actual.start),
+ (None, Some(actual)) => {
+ panic!("expected no function location for {key:?}, got {actual:?}")
+ }
+ (Some(_), None) => {
+ panic!("expected a function location for {key:?}, but got nothing")
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn reverse_func_locs() {
+ let table = make_test_table();
+
+ for (range, expected) in [
+ (0..10, Some(def_func_key(0, 0))),
+ (10..20, Some(def_func_key(0, 1))),
+ (20..30, Some(def_func_key(0, 2))),
+ (30..40, Some(def_func_key(0, 5))),
+ (40..100, None),
+ (100..110, Some(array_to_wasm_tramp_key(0, 1))),
+ (110..120, Some(array_to_wasm_tramp_key(0, 2))),
+ (120..130, Some(array_to_wasm_tramp_key(0, 5))),
+ (140..150, None),
+ ] {
+ for i in range {
+ eprintln!("Checking offset {i}");
+ let actual = table.func_by_text_offset(i);
+ assert_eq!(expected, actual);
+ }
+ }
+ }
+}
diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs
index 87a3252c95..5d210bd58c 100644
--- a/crates/wasmtime/src/compile.rs
+++ b/crates/wasmtime/src/compile.rs
@@ -26,22 +26,17 @@ use crate::Engine;
use crate::hash_map::HashMap;
use crate::hash_set::HashSet;
use crate::prelude::*;
-use std::{
- any::Any,
- borrow::Cow,
- collections::{BTreeMap, BTreeSet},
- mem,
- ops::Range,
-};
+use std::{any::Any, borrow::Cow, collections::BTreeMap, mem, ops::Range};
use call_graph::CallGraph;
#[cfg(feature = "component-model")]
use wasmtime_environ::component::Translator;
use wasmtime_environ::{
- BuiltinFunctionIndex, CompiledFunctionBody, CompiledFunctionInfo, CompiledModuleInfo, Compiler,
- DefinedFuncIndex, FilePos, FinishedObject, FuncKey, FunctionBodyData, InliningCompiler,
- IntraModuleInlining, ModuleEnvironment, ModuleTranslation, ModuleTypes, ModuleTypesBuilder,
- ObjectKind, PrimaryMap, SecondaryMap, StaticModuleIndex, Tunables,
+ BuiltinFunctionIndex, CompiledFunctionBody, CompiledFunctionsTable,
+ CompiledFunctionsTableBuilder, CompiledModuleInfo, Compiler, DefinedFuncIndex, FilePos,
+ FinishedObject, FuncKey, FunctionBodyData, InliningCompiler, IntraModuleInlining,
+ ModuleEnvironment, ModuleTranslation, ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap,
+ StaticModuleIndex, Tunables,
};
mod call_graph;
@@ -71,7 +66,10 @@ pub(crate) fn build_artifacts<T: FinishedObject>(
wasm: &[u8],
dwarf_package: Option<&[u8]>,
obj_state: &T::State,
-) -> Result<(T, Option<(CompiledModuleInfo, ModuleTypes)>)> {
+) -> Result<(
+ T,
+ Option<(CompiledModuleInfo, CompiledFunctionsTable, ModuleTypes)>,
+)> {
let tunables = engine.tunables();
// First a `ModuleEnvironment` is created which records type information
@@ -117,7 +115,6 @@ pub(crate) fn build_artifacts<T: FinishedObject>(
engine.append_bti(&mut object);
let (mut object, compilation_artifacts) = indices.link_and_append_code(
- &types,
object,
engine,
compiled_funcs,
@@ -125,12 +122,12 @@ pub(crate) fn build_artifacts<T: FinishedObject>(
dwarf_package,
)?;
- let info = compilation_artifacts.unwrap_as_module_info();
+ let (info, index) = compilation_artifacts.unwrap_as_module_info();
let types = types.finish();
- object.serialize_info(&(&info, &types));
+ object.serialize_info(&(&info, &index, &types));
let result = T::finish_object(object, obj_state)?;
- Ok((result, Some((info, types))))
+ Ok((result, Some((info, index, types))))
}
/// Performs the compilation phase for a component, translating and
@@ -188,7 +185,6 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
engine.append_bti(&mut object);
let (mut object, compilation_artifacts) = indices.link_and_append_code(
- types.module_types_builder(),
object,
engine,
compiled_funcs,
@@ -199,12 +195,10 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(
let info = CompiledComponentInfo {
component: component.component,
- wasm_call_trampolines: compilation_artifacts.wasm_call_trampolines,
- array_call_trampolines: compilation_artifacts.array_call_trampolines,
- resource_drop_wasm_to_array_trampoline: compilation_artifacts.resource_drop_trampoline,
};
let artifacts = ComponentArtifacts {
info,
+ table: compilation_artifacts.table,
ty,
types,
static_modules: compilation_artifacts.modules,
@@ -874,7 +868,7 @@ impl UnlinkedCompileOutputs<'_> {
/// indices within it.
fn pre_link(self) -> PreLinkOutput {
// We must ensure that `compiled_funcs` contains the function bodies
- // sorted by their `FuncKey`, as `CompiledFunctionsIndex` relies on that
+ // sorted by their `FuncKey`, as `CompiledFunctionsTable` relies on that
// property.
//
// Furthermore, note that, because the order functions end up in
@@ -898,7 +892,7 @@ impl UnlinkedCompileOutputs<'_> {
let index = compiled_funcs.len();
compiled_funcs.push((output.symbol, output.function.code));
- if output.start_srcloc != FilePos::default() {
+ if output.start_srcloc != FilePos::none() {
indices
.start_srclocs
.insert(output.key, output.start_srcloc);
@@ -941,7 +935,6 @@ impl FunctionIndices {
/// them to the given ELF file.
fn link_and_append_code<'a>(
self,
- types: &ModuleTypesBuilder,
mut obj: object::write::Object<'static>,
engine: &'a Engine,
compiled_funcs: Vec<(String, Box<dyn Any + Send + Sync>)>,
@@ -981,103 +974,21 @@ impl FunctionIndices {
)?;
}
- let mut def_funcs = SecondaryMap::<
- StaticModuleIndex,
- PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo>,
- >::new();
-
- #[cfg(feature = "component-model")]
- let mut wasm_call_trampolines = PrimaryMap::<
- wasmtime_environ::component::TrampolineIndex,
- wasmtime_environ::FunctionLoc,
- >::new();
-
- #[cfg(feature = "component-model")]
- let mut array_call_trampolines = PrimaryMap::<
- wasmtime_environ::component::TrampolineIndex,
- wasmtime_environ::FunctionLoc,
- >::new();
-
- #[cfg(feature = "component-model")]
- let mut resource_drop_trampoline = None;
-
- for (key, index) in &self.indices {
- // NB: exhaustively match on function keys to make sure that we are
- // remembering to handle everything we are compiling when doing this
- // final linking and metadata-collection step.
- match *key {
- FuncKey::DefinedWasmFunction(module, def_func) => {
- let (_, wasm_func_loc) = symbol_ids_and_locs[*index];
- let start_srcloc = self.start_srclocs[key];
-
- let array_to_wasm_trampoline = self
- .indices
- .get(&FuncKey::ArrayToWasmTrampoline(module, def_func))
- .map(|index| {
- let (_, loc) = symbol_ids_and_locs[*index];
- loc
- });
-
- debug_assert!(def_funcs[module].get(def_func).is_none());
- let def_func2 = def_funcs[module].push(CompiledFunctionInfo {
- start_srcloc,
- wasm_func_loc,
- array_to_wasm_trampoline,
- });
- debug_assert_eq!(def_func, def_func2);
- }
-
- FuncKey::ArrayToWasmTrampoline(module, def_func) => {
- // These are handled by the `DefinedWasmFunction` arm above.
- debug_assert!(def_funcs[module].get(def_func).is_some());
- }
-
- FuncKey::WasmToArrayTrampoline(_) => {
- // These are handled in `modules` creation below.
- }
-
- FuncKey::WasmToBuiltinTrampoline(_) => {
- // Nothing we need to do for these: they are only called by
- // Wasm functions, and we never create `funcref`s containing
- // them, so we don't need to keep any metadata for them or
- // anything like that.
- }
-
- FuncKey::PulleyHostCall(_) => {
- unreachable!("we don't compile any artifacts for Pulley host calls")
- }
-
- #[cfg(feature = "component-model")]
- FuncKey::ComponentTrampoline(abi, trampoline) => {
- let (_, loc) = symbol_ids_and_locs[*index];
- match abi {
- wasmtime_environ::Abi::Wasm => {
- debug_assert!(wasm_call_trampolines.get(trampoline).is_none());
- let trampoline2 = wasm_call_trampolines.push(loc);
- debug_assert_eq!(trampoline, trampoline2);
- }
- wasmtime_environ::Abi::Array => {
- debug_assert!(array_call_trampolines.get(trampoline).is_none());
- let trampoline2 = array_call_trampolines.push(loc);
- debug_assert_eq!(trampoline, trampoline2);
- }
- }
- }
-
- #[cfg(feature = "component-model")]
- FuncKey::ResourceDropTrampoline => {
- let (_, loc) = symbol_ids_and_locs[*index];
- resource_drop_trampoline = Some(loc);
- }
- }
+ let mut table_builder = CompiledFunctionsTableBuilder::new();
+ for (key, compiled_func_index) in &self.indices {
+ let (_, func_loc) = symbol_ids_and_locs[*compiled_func_index];
+ let src_loc = self
+ .start_srclocs
+ .get(key)
+ .copied()
+ .unwrap_or_else(FilePos::none);
+ table_builder.push_func(*key, func_loc, src_loc);
}
let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables);
let modules = translations
.into_iter()
- .map(|(module, mut translation)| {
- let def_funcs = mem::take(&mut def_funcs[module]);
-
+ .map(|(_, mut translation)| {
// If configured attempt to use static memory initialization
// which can either at runtime be implemented as a single memcpy
// to initialize memory or otherwise enabling
@@ -1095,38 +1006,13 @@ impl FunctionIndices {
translation.try_func_table_init();
}
- // Gather this module's trampolines.
- let unique_and_sorted_trampoline_sigs = translation
- .module
- .types
- .iter()
- .map(|(_, ty)| ty.unwrap_module_type_index())
- .filter(|idx| types[*idx].is_func())
- .map(|idx| types.trampoline_type(idx))
- .collect::<BTreeSet<_>>();
- let wasm_to_array_trampolines = unique_and_sorted_trampoline_sigs
- .into_iter()
- .map(|ty| {
- debug_assert_eq!(ty, types.trampoline_type(ty));
- let key = FuncKey::WasmToArrayTrampoline(ty);
- let index = self.indices[&key];
- let (_, loc) = symbol_ids_and_locs[index];
- (ty, loc)
- })
- .collect();
-
- obj.append(translation, def_funcs, wasm_to_array_trampolines)
+ obj.append(translation)
})
.collect::<Result<PrimaryMap<_, _>>>()?;
let artifacts = Artifacts {
modules,
- #[cfg(feature = "component-model")]
- wasm_call_trampolines,
- #[cfg(feature = "component-model")]
- array_call_trampolines,
- #[cfg(feature = "component-model")]
- resource_drop_trampoline,
+ table: table_builder.finish(),
};
Ok((obj, artifacts))
@@ -1135,32 +1021,19 @@ impl FunctionIndices {
/// The artifacts necessary for finding and calling Wasm functions at runtime,
/// to be serialized into an ELF file.
-#[derive(Default)]
struct Artifacts {
modules: PrimaryMap<StaticModuleIndex, CompiledModuleInfo>,
- #[cfg(feature = "component-model")]
- wasm_call_trampolines:
- PrimaryMap<wasmtime_environ::component::TrampolineIndex, wasmtime_environ::FunctionLoc>,
- #[cfg(feature = "component-model")]
- array_call_trampolines:
- PrimaryMap<wasmtime_environ::component::TrampolineIndex, wasmtime_environ::FunctionLoc>,
- #[cfg(feature = "component-model")]
- resource_drop_trampoline: Option<wasmtime_environ::FunctionLoc>,
+ table: CompiledFunctionsTable,
}
impl Artifacts {
/// Assuming this compilation was for a single core Wasm module, get the
/// resulting `CompiledModuleInfo`.
- fn unwrap_as_module_info(self) -> CompiledModuleInfo {
+ fn unwrap_as_module_info(self) -> (CompiledModuleInfo, CompiledFunctionsTable) {
assert_eq!(self.modules.len(), 1);
-
- #[cfg(feature = "component-model")]
- {
- assert!(self.wasm_call_trampolines.is_empty());
- assert!(self.array_call_trampolines.is_empty());
- }
-
- self.modules.into_iter().next().unwrap().1
+ let info = self.modules.into_iter().next().unwrap().1;
+ let table = self.table;
+ (info, table)
}
}
diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rs
index 222dccdb63..eb83c44852 100644
--- a/crates/wasmtime/src/runtime/component/component.rs
+++ b/crates/wasmtime/src/runtime/component/component.rs
@@ -15,12 +15,12 @@ use core::ops::Range;
use core::ptr::NonNull;
#[cfg(feature = "std")]
use std::path::Path;
-use wasmtime_environ::TypeTrace;
use wasmtime_environ::component::{
CompiledComponentInfo, ComponentArtifacts, ComponentTypes, CoreDef, Export, ExportIndex,
GlobalInitializer, InstantiateModule, NameMapNoIntern, OptionsIndex, StaticModuleIndex,
TrampolineIndex, TypeComponentIndex, TypeFuncIndex, VMComponentOffsets,
};
+use wasmtime_environ::{Abi, CompiledFunctionsTable, FuncKey, TypeTrace};
use wasmtime_environ::{FunctionLoc, HostPtr, ObjectKind, PrimaryMap};
/// A compiled WebAssembly Component.
@@ -86,6 +86,10 @@ struct ComponentInner {
/// Metadata produced during compilation.
info: CompiledComponentInfo,
+ /// The index of compiled functions and their locations in the text section
+ /// for this component.
+ index: Arc<CompiledFunctionsTable>,
+
/// A cached handle to the `wasmtime::FuncType` for the canonical ABI's
/// `realloc`, to avoid the need to look up types in the registry and take
/// locks when calling `realloc` via `TypedFunc::call_raw`.
@@ -400,12 +404,14 @@ impl Component {
let ComponentArtifacts {
ty,
info,
+ table: index,
mut types,
mut static_modules,
} = match artifacts {
Some(artifacts) => artifacts,
None => postcard::from_bytes(code_memory.wasmtime_info())?,
};
+ let index = Arc::new(index);
// Validate that the component can be used with the current instance
// allocator.
@@ -434,7 +440,9 @@ impl Component {
// `types` type information, and the code memory to a runtime object.
let static_modules = static_modules
.into_iter()
- .map(|(_, info)| Module::from_parts_raw(engine, code.clone(), info, false))
+ .map(|(_, info)| {
+ Module::from_parts_raw(engine, code.clone(), info, index.clone(), false)
+ })
.collect::<Result<_>>()?;
let realloc_func_type = Arc::new(FuncType::new(
@@ -451,6 +459,7 @@ impl Component {
static_modules,
code,
info,
+ index,
realloc_func_type,
}),
})
@@ -492,8 +501,16 @@ impl Component {
}
pub(crate) fn trampoline_ptrs(&self, index: TrampolineIndex) -> AllCallFuncPointers {
- let wasm_call = &self.inner.info.wasm_call_trampolines[index];
- let array_call = &self.inner.info.array_call_trampolines[index];
+ let wasm_call = self
+ .inner
+ .index
+ .func_loc(FuncKey::ComponentTrampoline(Abi::Wasm, index))
+ .unwrap();
+ let array_call = self
+ .inner
+ .index
+ .func_loc(FuncKey::ComponentTrampoline(Abi::Array, index))
+ .unwrap();
AllCallFuncPointers {
wasm_call: self.func(wasm_call).cast(),
array_call: self.func(array_call).cast(),
@@ -537,10 +554,10 @@ impl Component {
// blank.
let wasm_call = self
.inner
- .info
- .resource_drop_wasm_to_array_trampoline
- .as_ref()
- .map(|i| self.func(i).cast().into());
+ .index
+ .func_loc(FuncKey::ResourceDropTrampoline)
+ .map(|loc| self.func(loc).cast().into());
+
VMFuncRef {
wasm_call,
..*dtor.func_ref()
diff --git a/crates/wasmtime/src/runtime/instantiate.rs b/crates/wasmtime/src/runtime/instantiate.rs
index 9d6be04d5c..5fb1130f09 100644
--- a/crates/wasmtime/src/runtime/instantiate.rs
+++ b/crates/wasmtime/src/runtime/instantiate.rs
@@ -9,19 +9,20 @@ use crate::{code_memory::CodeMemory, profiling_agent::ProfilingAgent};
use alloc::sync::Arc;
use core::str;
use wasmtime_environ::{
- CompiledFunctionInfo, CompiledModuleInfo, DefinedFuncIndex, FilePos, FuncIndex, FunctionLoc,
- FunctionName, Metadata, Module, ModuleInternedTypeIndex, PrimaryMap,
+ CompiledFunctionsTable, CompiledModuleInfo, DefinedFuncIndex, EntityRef, FilePos, FuncIndex,
+ FuncKey, FunctionLoc, FunctionName, Metadata, Module, ModuleInternedTypeIndex,
+ StaticModuleIndex,
};
/// A compiled wasm module, ready to be instantiated.
pub struct CompiledModule {
- module: Arc<Module>,
- funcs: PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo>,
- wasm_to_array_trampolines: Vec<(ModuleInternedTypeIndex, FunctionLoc)>,
- meta: Metadata,
- code_memory: Arc<CodeMemory>,
/// A unique ID used to register this module with the engine.
unique_id: CompiledModuleId,
+ code_memory: Arc<CodeMemory>,
+ module: Arc<Module>,
+ meta: Metadata,
+ index: Arc<CompiledFunctionsTable>,
+ /// Sorted list, by function index, of names we have for this module.
func_names: Vec<FunctionName>,
}
@@ -45,15 +46,15 @@ impl CompiledModule {
pub fn from_artifacts(
code_memory: Arc<CodeMemory>,
info: CompiledModuleInfo,
+ index: Arc<CompiledFunctionsTable>,
profiler: &dyn ProfilingAgent,
) -> Result<Self> {
let mut ret = Self {
- module: Arc::new(info.module),
- funcs: info.funcs,
- wasm_to_array_trampolines: info.wasm_to_array_trampolines,
+ unique_id: CompiledModuleId::new(),
code_memory,
+ module: Arc::new(info.module),
meta: info.meta,
- unique_id: CompiledModuleId::new(),
+ index,
func_names: info.func_names,
};
ret.register_profiling(profiler)?;
@@ -65,7 +66,7 @@ impl CompiledModule {
// TODO-Bug?: "code_memory" is not exclusive for this module in the case of components,
// so we may be registering the same code range multiple times here.
profiler.register_module(&self.code_memory.mmap()[..], &|addr| {
- let (idx, _) = self.func_by_text_offset(addr)?;
+ let idx = self.func_by_text_offset(addr)?;
let idx = self.module.func_index(idx);
let name = self.func_name(idx)?;
let mut demangled = String::new();
@@ -105,6 +106,10 @@ impl CompiledModule {
&self.module
}
+ fn module_index(&self) -> StaticModuleIndex {
+ self.module.module_index
+ }
+
/// Looks up the `name` section name for the function index `idx`, if one
/// was specified in the original wasm module.
pub fn func_name(&self, idx: FuncIndex) -> Option<&str> {
@@ -126,15 +131,15 @@ impl CompiledModule {
pub fn finished_functions(
&self,
) -> impl ExactSizeIterator<Item = (DefinedFuncIndex, &[u8])> + '_ {
- self.funcs
- .iter()
- .map(move |(i, _)| (i, self.finished_function(i)))
+ self.module
+ .defined_func_indices()
+ .map(|i| (i, self.finished_function(i)))
}
/// Returns the body of the function that `index` points to.
#[inline]
- pub fn finished_function(&self, index: DefinedFuncIndex) -> &[u8] {
- let loc = self.funcs[index].wasm_func_loc;
+ pub fn finished_function(&self, def_func_index: DefinedFuncIndex) -> &[u8] {
+ let loc = self.func_loc(def_func_index);
&self.text()[loc.start as usize..][..loc.length as usize]
}
@@ -145,8 +150,10 @@ impl CompiledModule {
///
/// These trampolines are used for array callers (e.g. `Func::new`)
/// calling Wasm callees.
- pub fn array_to_wasm_trampoline(&self, index: DefinedFuncIndex) -> Option<&[u8]> {
- let loc = self.funcs[index].array_to_wasm_trampoline?;
+ pub fn array_to_wasm_trampoline(&self, def_func_index: DefinedFuncIndex) -> Option<&[u8]> {
+ assert!(def_func_index.index() < self.module.num_defined_funcs());
+ let key = FuncKey::ArrayToWasmTrampoline(self.module_index(), def_func_index);
+ let loc = self.index.func_loc(key)?;
Some(&self.text()[loc.start as usize..][..loc.length as usize])
}
@@ -155,67 +162,45 @@ impl CompiledModule {
/// These trampolines are used for filling in
/// `VMFuncRef::wasm_call` for `Func::wrap`-style host funcrefs
/// that don't have access to a compiler when created.
- pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> &[u8] {
- let idx = match self
- .wasm_to_array_trampolines
- .binary_search_by_key(&signature, |entry| entry.0)
- {
- Ok(idx) => idx,
- Err(_) => panic!("missing trampoline for {signature:?}"),
- };
-
- let (_, loc) = self.wasm_to_array_trampolines[idx];
- &self.text()[loc.start as usize..][..loc.length as usize]
+ pub fn wasm_to_array_trampoline(&self, signature: ModuleInternedTypeIndex) -> Option<&[u8]> {
+ let key = FuncKey::WasmToArrayTrampoline(signature);
+ let loc = self.index.func_loc(key)?;
+ Some(&self.text()[loc.start as usize..][..loc.length as usize])
}
/// Lookups a defined function by a program counter value.
///
/// Returns the defined function index and the relative address of
/// `text_offset` within the function itself.
- pub fn func_by_text_offset(&self, text_offset: usize) -> Option<(DefinedFuncIndex, u32)> {
+ pub fn func_by_text_offset(&self, text_offset: usize) -> Option<DefinedFuncIndex> {
let text_offset = u32::try_from(text_offset).unwrap();
-
- let index = match self.funcs.binary_search_values_by_key(&text_offset, |e| {
- debug_assert!(e.wasm_func_loc.length > 0);
- // Return the inclusive "end" of the function
- e.wasm_func_loc.start + e.wasm_func_loc.length - 1
- }) {
- Ok(k) => {
- // Exact match, pc is at the end of this function
- k
- }
- Err(k) => {
- // Not an exact match, k is where `pc` would be "inserted"
- // Since we key based on the end, function `k` might contain `pc`,
- // so we'll validate on the range check below
- k
+ let key = self.index.func_by_text_offset(text_offset)?;
+ match key {
+ FuncKey::DefinedWasmFunction(module, def_func_index) => {
+ debug_assert_eq!(module, self.module_index());
+ Some(def_func_index)
}
- };
-
- let CompiledFunctionInfo { wasm_func_loc, .. } = self.funcs.get(index)?;
- let start = wasm_func_loc.start;
- let end = wasm_func_loc.start + wasm_func_loc.length;
-
- if text_offset < start || end < text_offset {
- return None;
+ _ => None,
}
-
- Some((index, text_offset - wasm_func_loc.start))
}
/// Gets the function location information for a given function index.
- pub fn func_loc(&self, index: DefinedFuncIndex) -> &FunctionLoc {
- &self
- .funcs
- .get(index)
+ pub fn func_loc(&self, def_func_index: DefinedFuncIndex) -> &FunctionLoc {
+ assert!(def_func_index.index() < self.module.num_defined_funcs());
+ let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index);
+ self.index
+ .func_loc(key)
.expect("defined function should be present")
- .wasm_func_loc
}
/// Returns the original binary offset in the file that `index` was defined
/// at.
- pub fn func_start_srcloc(&self, index: DefinedFuncIndex) -> FilePos {
- self.funcs[index].start_srcloc
+ pub fn func_start_srcloc(&self, def_func_index: DefinedFuncIndex) -> FilePos {
+ assert!(def_func_index.index() < self.module.num_defined_funcs());
+ let key = FuncKey::DefinedWasmFunction(self.module_index(), def_func_index);
+ self.index
+ .src_loc(key)
+ .expect("defined function should be present")
}
/// Creates a new symbolication context which can be used to further
diff --git a/crates/wasmtime/src/runtime/module.rs b/crates/wasmtime/src/runtime/module.rs
index 0420f7ee4a..35b7214058 100644
--- a/crates/wasmtime/src/runtime/module.rs
+++ b/crates/wasmtime/src/runtime/module.rs
@@ -19,8 +19,8 @@ use core::ptr::NonNull;
use std::{fs::File, path::Path};
use wasmparser::{Parser, ValidPayload, Validator};
use wasmtime_environ::{
- CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind, TypeTrace, VMOffsets,
- VMSharedTypeIndex,
+ CompiledFunctionsTable, CompiledModuleInfo, EntityIndex, HostPtr, ModuleTypes, ObjectKind,
+ TypeTrace, VMOffsets, VMSharedTypeIndex,
};
#[cfg(feature = "gc")]
use wasmtime_unwinder::ExceptionTable;
@@ -495,13 +495,13 @@ impl Module {
pub(crate) fn from_parts(
engine: &Engine,
code_memory: Arc<CodeMemory>,
- info_and_types: Option<(CompiledModuleInfo, ModuleTypes)>,
+ info_and_types: Option<(CompiledModuleInfo, CompiledFunctionsTable, ModuleTypes)>,
) -> Result<Self> {
// Acquire this module's metadata and type information, deserializing
// it from the provided artifact if it wasn't otherwise provided
// already.
- let (mut info, mut types) = match info_and_types {
- Some((info, types)) => (info, types),
+ let (mut info, index, mut types) = match info_and_types {
+ Some((info, index, types)) => (info, index, types),
None => postcard::from_bytes(code_memory.wasmtime_info())?,
};
@@ -519,17 +519,23 @@ impl Module {
// Package up all our data into a `CodeObject` and delegate to the final
// step of module compilation.
let code = Arc::new(CodeObject::new(code_memory, signatures, types.into()));
- Module::from_parts_raw(engine, code, info, true)
+ let index = Arc::new(index);
+ Module::from_parts_raw(engine, code, info, index, true)
}
pub(crate) fn from_parts_raw(
engine: &Engine,
code: Arc<CodeObject>,
info: CompiledModuleInfo,
+ index: Arc<CompiledFunctionsTable>,
serializable: bool,
) -> Result<Self> {
- let module =
- CompiledModule::from_artifacts(code.code_memory().clone(), info, engine.profiler())?;
+ let module = CompiledModule::from_artifacts(
+ code.code_memory().clone(),
+ info,
+ index,
+ engine.profiler(),
+ )?;
// Validate the module can be used with the current instance allocator.
let offsets = VMOffsets::new(HostPtr, module.module());
@@ -1106,6 +1112,7 @@ impl Module {
let ptr = self
.compiled_module()
.wasm_to_array_trampoline(trampoline_module_ty)
+ .expect("always have a trampoline for the trampoline type")
.as_ptr()
.cast::<VMWasmCallFunction>()
.cast_mut();
diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rs
index 8ac3d38d33..3089e4c4ef 100644
--- a/crates/wasmtime/src/runtime/trap.rs
+++ b/crates/wasmtime/src/runtime/trap.rs
@@ -437,7 +437,7 @@ impl FrameInfo {
/// if no information can be found.
pub(crate) fn new(module: Module, text_offset: usize) -> Option<FrameInfo> {
let compiled_module = module.compiled_module();
- let (index, _func_offset) = compiled_module.func_by_text_offset(text_offset)?;
+ let index = compiled_module.func_by_text_offset(text_offset)?;
let func_start = compiled_module.func_start_srcloc(index);
let instr = wasmtime_environ::lookup_file_pos(
compiled_module.code_memory().address_map_data(),
diff --git a/tests/all/coredump.rs b/tests/all/coredump.rs
index 945322fe78..53351b7663 100644
--- a/tests/all/coredump.rs
+++ b/tests/all/coredump.rs
@@ -37,6 +37,7 @@ fn coredump_attached_to_error() -> Result<()> {
#[test]
#[cfg_attr(miri, ignore)]
fn coredump_has_stack() -> Result<()> {
+ let _ = env_logger::try_init();
let mut config = Config::default();
config.coredump_on_trap(true);
let engine = Engine::new(&config).unwrap();