Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Make SCCs and graph traversals reusable (#12590)

Nick Fitzgerald59d23a356484
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 6e538cdde3..0e55cba034 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -21,5 +21,6 @@ extern crate std;
pub mod alloc;
pub mod error;
pub mod math;
+pub mod non_max;
pub mod slab;
pub mod undo;
diff --git a/crates/core/src/non_max.rs b/crates/core/src/non_max.rs
new file mode 100644
index 0000000000..51e1d33413
--- /dev/null
+++ b/crates/core/src/non_max.rs
@@ -0,0 +1,94 @@
+//! Integer newtypes that are guaranteed not to be their maximum value.
+
+macro_rules! define_non_max {
+ ( $(
+ $( #[ $attr:meta ] )*
+ pub struct $non_max:ident($non_zero:ty) : $prim:ty;
+ )* ) => {
+ $(
+ /// A
+ #[doc = concat!("`", stringify!($prim), "`")]
+ /// value that is known not to be
+ #[doc = concat!("`", stringify!($prim), "::MAX`.")]
+ ///
+ /// This enables some memory layout optimizations. For example,
+ #[doc = concat!("`Option<", stringify!($prim), ">`")]
+ /// is the same size as
+ #[doc = concat!("`", stringify!($prim), "`:")]
+ ///
+ /// ```
+ /// # use wasmtime_internal_core::non_max::NonMaxU32;
+ /// assert_eq!(
+ /// core::mem::size_of::<u32>(),
+ /// core::mem::size_of::<Option<NonMaxU32>>(),
+ /// );
+ /// ```
+ #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+ pub struct $non_max($non_zero);
+
+ impl Default for $non_max {
+ fn default() -> Self {
+ Self::new(0).unwrap()
+ }
+ }
+
+ impl core::fmt::Debug for $non_max {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ f.debug_tuple(stringify!($non_max))
+ .field(&self.get())
+ .finish()
+ }
+ }
+
+ impl $non_max {
+ /// Construct a new
+ #[doc = concat!("`", stringify!($prim), "`")]
+ /// value.
+ ///
+ /// Returns `None` when given
+ #[doc = concat!("`", stringify!($prim), "::MAX`.")]
+ #[inline]
+ pub fn new(x: $prim) -> Option<Self> {
+ if x == <$prim>::MAX {
+ None
+ } else {
+ // Safety: `x != $prim::MAX`.
+ Some(unsafe { Self::new_unchecked(x) })
+ }
+ }
+
+ /// Unsafely construct a new
+ #[doc = concat!("`", stringify!($prim), "`")]
+ /// value.
+ ///
+ /// # Safety
+ ///
+ /// The given value must not be
+ #[doc = concat!("`", stringify!($prim), "::MAX`.")]
+ #[inline]
+ pub unsafe fn new_unchecked(x: $prim) -> Self {
+ debug_assert_ne!(x, <$prim>::MAX);
+ // Safety: We know that `x+1` is non-zero because it will not
+ // overflow because `x` is not `$prim::MAX`.
+ Self(unsafe { <$non_zero>::new_unchecked(x + 1) })
+ }
+
+ /// Get the underlying
+ #[doc = concat!("`", stringify!($prim), "`")]
+ /// value.
+ #[inline]
+ pub fn get(&self) -> $prim {
+ self.0.get() - 1
+ }
+ }
+ )*
+ };
+}
+
+define_non_max! {
+ pub struct NonMaxU8(core::num::NonZeroU8) : u8;
+ pub struct NonMaxU16(core::num::NonZeroU16) : u16;
+ pub struct NonMaxU32(core::num::NonZeroU32) : u32;
+ pub struct NonMaxU64(core::num::NonZeroU64) : u64;
+ pub struct NonMaxUsize(core::num::NonZeroUsize) : usize;
+}
diff --git a/crates/environ/src/graphs.rs b/crates/environ/src/graphs.rs
new file mode 100644
index 0000000000..5c9be856d5
--- /dev/null
+++ b/crates/environ/src/graphs.rs
@@ -0,0 +1,134 @@
+//! Generic graph traits, traversals, and data structures for use in Wasmtime.
+
+mod dfs;
+mod entity_graph;
+mod scc;
+use core::{fmt, iter};
+
+pub use dfs::*;
+pub use entity_graph::*;
+pub use scc::*;
+
+use crate::prelude::*;
+
+/// A trait for any kind of graph data structure.
+pub trait Graph<Node> {
+ /// The iterator type returned by `Nodes::nodes`.
+ type NodesIter<'a>: Iterator<Item = Node>
+ where
+ Self: 'a;
+
+ /// Iterate over the nodes in this graph.
+ fn nodes(&self) -> Self::NodesIter<'_>;
+
+ /// The iterator type returned by `Successors::successors`.
+ type SuccessorsIter<'a>: Iterator<Item = Node>
+ where
+ Self: 'a;
+
+ /// Iterate over the successors of the given `node`.
+ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_>;
+
+ // Provided Methods.
+
+ /// Like `Iterator::by_ref` but for `Graph`.
+ fn by_ref(&self) -> &Self {
+ self
+ }
+
+ /// Use the given predicate to filter out certain nodes from the graph.
+ fn filter_nodes<F>(self, predicate: F) -> FilterNodes<Self, F>
+ where
+ Self: Sized,
+ F: Fn(&Node) -> bool,
+ {
+ FilterNodes {
+ graph: self,
+ predicate,
+ }
+ }
+}
+
+impl<T, Node> Graph<Node> for &'_ T
+where
+ T: ?Sized + Graph<Node>,
+{
+ type NodesIter<'a>
+ = T::NodesIter<'a>
+ where
+ Self: 'a;
+
+ fn nodes(&self) -> Self::NodesIter<'_> {
+ (*self).nodes()
+ }
+
+ type SuccessorsIter<'a>
+ = T::SuccessorsIter<'a>
+ where
+ Self: 'a;
+
+ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_> {
+ (*self).successors(node)
+ }
+}
+
+/// A graph whose nodes are being filtered by the predicate `F`.
+///
+/// Created by the `Graph::filter_nodes` trait method.
+pub struct FilterNodes<G, F> {
+ graph: G,
+ predicate: F,
+}
+
+impl<G, F> fmt::Debug for FilterNodes<G, F>
+where
+ G: fmt::Debug,
+{
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("Filter")
+ .field("graph", &self.graph)
+ .field("predicate", &"..")
+ .finish()
+ }
+}
+
+impl<G, F, Node> Graph<Node> for FilterNodes<G, F>
+where
+ G: Graph<Node>,
+ F: Fn(&Node) -> bool,
+{
+ type NodesIter<'a>
+ = iter::Filter<G::NodesIter<'a>, &'a F>
+ where
+ Self: 'a;
+
+ fn nodes(&self) -> Self::NodesIter<'_> {
+ self.graph.nodes().filter(&self.predicate)
+ }
+
+ type SuccessorsIter<'a>
+ = iter::Filter<G::SuccessorsIter<'a>, &'a F>
+ where
+ Self: 'a;
+
+ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_> {
+ self.graph.successors(node).filter(&self.predicate)
+ }
+}
+
+/// Extend `dest` with `items` and return the range of indices in `dest` where
+/// they ended up.
+fn extend_with_range<T>(
+ dest: &mut Vec<T>,
+ items: impl IntoIterator<Item = T>,
+) -> core::ops::Range<u32> {
+ let start = dest.len();
+ let start = u32::try_from(start).unwrap();
+
+ dest.extend(items);
+
+ let end = dest.len();
+ let end = u32::try_from(end).unwrap();
+
+ start..end
+}
diff --git a/crates/environ/src/graphs/dfs.rs b/crates/environ/src/graphs/dfs.rs
new file mode 100644
index 0000000000..6265c7f314
--- /dev/null
+++ b/crates/environ/src/graphs/dfs.rs
@@ -0,0 +1,91 @@
+use super::*;
+
+/// An iterative depth-first traversal.
+pub struct Dfs<Node> {
+ stack: Vec<DfsEvent<Node>>,
+}
+
+impl<Node> Default for Dfs<Node> {
+ fn default() -> Self {
+ Self {
+ stack: Default::default(),
+ }
+ }
+}
+
+impl<Node> Dfs<Node> {
+ /// Create a new DFS traversal, starting at the given roots.
+ pub fn new(roots: impl IntoIterator<Item = Node>) -> Self {
+ let mut dfs = Self::default();
+ dfs.add_roots(roots);
+ dfs
+ }
+
+ /// Add a single new root to this traversal, to be visited immediately.
+ pub fn add_root(&mut self, root: Node) {
+ self.stack.push(DfsEvent::Pre(root));
+ }
+
+ /// Add multiple new roots to this traversal, to be visited immediately.
+ pub fn add_roots(&mut self, roots: impl IntoIterator<Item = Node>) {
+ self.stack
+ .extend(roots.into_iter().map(|v| DfsEvent::Pre(v)));
+ }
+}
+
+/// An event during a DFS traversal.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum DfsEvent<Node> {
+ /// The first time seeing this node.
+ Pre(Node),
+
+ /// After having just visited the given edge.
+ AfterEdge(Node, Node),
+
+ /// Finished visiting this node and all of its successors.
+ Post(Node),
+}
+
+impl<Node> Dfs<Node>
+where
+ Node: Copy,
+{
+ /// Pump the traversal, yielding the next `DfsEvent`.
+ ///
+ /// Returns `None` when the traversal is complete.
+ pub fn next<G>(&mut self, graph: G, seen: impl Fn(Node) -> bool) -> Option<DfsEvent<Node>>
+ where
+ G: Graph<Node>,
+ {
+ loop {
+ let event = self.stack.pop()?;
+
+ if let DfsEvent::Pre(node) = event {
+ if seen(node) {
+ continue;
+ }
+
+ let successors = graph.successors(node);
+
+ let (min, max) = successors.size_hint();
+ let estimated_successors_len = max.unwrap_or_else(|| 2 * min);
+ self.stack.reserve(
+ // We push an after-edge and pre event for each successor.
+ 2 * estimated_successors_len
+ // And we push one post event for this node.
+ + 1,
+ );
+
+ self.stack.push(DfsEvent::Post(node));
+ for succ in successors {
+ self.stack.push(DfsEvent::AfterEdge(node, succ));
+ if !seen(succ) {
+ self.stack.push(DfsEvent::Pre(succ));
+ }
+ }
+ }
+
+ return Some(event);
+ }
+ }
+}
diff --git a/crates/environ/src/graphs/entity_graph.rs b/crates/environ/src/graphs/entity_graph.rs
new file mode 100644
index 0000000000..659d439441
--- /dev/null
+++ b/crates/environ/src/graphs/entity_graph.rs
@@ -0,0 +1,102 @@
+use super::*;
+use crate::{EntityRef, SecondaryMap, prelude::*};
+use core::{
+ fmt::{self, Debug},
+ iter,
+ ops::Range,
+};
+
+/// A graph of `EntityRef` nodes reified into a densely packed representation.
+pub struct EntityGraph<Node>
+where
+ Node: EntityRef,
+{
+ /// A map from each node to the subslice of `self.edge_elems` that are its
+ /// edges.
+ edges: SecondaryMap<Node, Range<u32>>,
+
+ /// Densely packed edge elements for `self.edges`.
+ edge_elems: Vec<Node>,
+}
+
+impl<Node> Debug for EntityGraph<Node>
+where
+ Node: EntityRef + Debug,
+{
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ struct Edges<'a, Node: EntityRef + Debug>(&'a EntityGraph<Node>);
+
+ impl<'a, Node: EntityRef + Debug> Debug for Edges<'a, Node> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_map()
+ .entries(
+ self.0
+ .nodes()
+ .map(|n| (n, self.0.successors(n).collect::<Box<[_]>>())),
+ )
+ .finish()
+ }
+ }
+
+ f.debug_struct("Graph")
+ .field("edges", &Edges(self))
+ .finish()
+ }
+}
+
+impl<Node> EntityGraph<Node>
+where
+ Node: EntityRef + Debug,
+{
+ /// Construct a new, concrete `EntityGraph`.
+ pub fn new<E>(
+ nodes: impl IntoIterator<Item = Node>,
+ mut successors: impl FnMut(Node, &mut Vec<Node>) -> Result<(), E>,
+ ) -> Result<Self, E> {
+ let nodes = nodes.into_iter();
+
+ let (min, max) = nodes.size_hint();
+ let capacity = max.unwrap_or_else(|| 2 * min);
+
+ let mut edges = SecondaryMap::with_capacity(capacity);
+ let mut edge_elems = vec![];
+
+ let mut succs = vec![];
+ for v in nodes {
+ debug_assert!(succs.is_empty());
+ successors(v, &mut succs)?;
+
+ debug_assert_eq!(edges[v], Range::default());
+ edges[v] = extend_with_range(&mut edge_elems, succs.drain(..));
+ }
+
+ Ok(EntityGraph { edges, edge_elems })
+ }
+}
+
+impl<Node> Graph<Node> for EntityGraph<Node>
+where
+ Node: EntityRef,
+{
+ type NodesIter<'a>
+ = cranelift_entity::Keys<Node>
+ where
+ Self: 'a;
+
+ #[inline]
+ fn nodes(&self) -> Self::NodesIter<'_> {
+ self.edges.keys()
+ }
+
+ type SuccessorsIter<'a>
+ = iter::Copied<core::slice::Iter<'a, Node>>
+ where
+ Self: 'a;
+
+ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_> {
+ let Range { start, end } = self.edges[node].clone();
+ let start = usize::try_from(start).unwrap();
+ let end = usize::try_from(end).unwrap();
+ self.edge_elems[start..end].iter().copied()
+ }
+}
diff --git a/crates/wasmtime/src/compile/scc.rs b/crates/environ/src/graphs/scc.rs
similarity index 84%
rename from crates/wasmtime/src/compile/scc.rs
rename to crates/environ/src/graphs/scc.rs
index 025f2e3f53..37072db83b 100644
--- a/crates/wasmtime/src/compile/scc.rs
+++ b/crates/environ/src/graphs/scc.rs
@@ -12,20 +12,23 @@
//! [Tarjan's algorithm]: https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm
use super::*;
-use std::{
- collections::BTreeSet,
+use crate::{
+ EntityRef, EntitySet, PrimaryMap, SecondaryMap, non_max::NonMaxU32,
+ packed_option::PackedOption, prelude::*,
+};
+use alloc::collections::BTreeSet;
+use core::{
+ cmp,
fmt::{self, Debug},
hash::Hash,
+ iter,
ops::Range,
};
-use wasmtime_environ::{
- EntityRef, EntitySet, PrimaryMap, SecondaryMap, packed_option::PackedOption,
-};
/// A strongly-connected component.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Scc(u32);
-wasmtime_environ::entity_impl!(Scc);
+crate::entity_impl!(Scc);
/// The set of strongly-connected components for a graph of `Node`s.
pub struct StronglyConnectedComponents<Node>
@@ -64,16 +67,14 @@ where
impl<Node> StronglyConnectedComponents<Node>
where
- Node: EntityRef + std::fmt::Debug,
+ Node: EntityRef + Debug,
{
/// Find the strongly-connected for the given graph.
- pub fn new<I, F, S>(nodes: I, successors: F) -> Self
+ pub fn new<G>(graph: G) -> Self
where
- I: IntoIterator<Item = Node>,
- F: Fn(Node) -> S,
- S: Iterator<Item = Node>,
+ G: Graph<Node>,
{
- let nodes = nodes.into_iter();
+ let nodes = graph.nodes();
// The resulting components and their nodes.
let mut component_nodes = vec![];
@@ -97,7 +98,7 @@ where
let mut dfs = Dfs::new(nodes);
while let Some(event) = dfs.next(
- &successors,
+ &graph,
// We have seen the node before if we have assigned it a DFS index.
|node| indices[node].is_some(),
) {
@@ -134,10 +135,8 @@ where
// part of the same SCC as this node, so propagate its
// lowlink to this node's lowlink.
if on_stack.contains(succ) {
- lowlinks[node] = Some(std::cmp::min(
- lowlinks[node].unwrap(),
- lowlinks[succ].unwrap(),
- ));
+ lowlinks[node] =
+ Some(cmp::min(lowlinks[node].unwrap(), lowlinks[succ].unwrap()));
}
}
@@ -153,7 +152,7 @@ where
let mut done = false;
components.push(extend_with_range(
&mut component_nodes,
- std::iter::from_fn(|| {
+ iter::from_fn(|| {
if done {
return None;
}
@@ -236,7 +235,7 @@ where
/// In the following diagram, the solid lines are the input graph and the
/// dotted lines show its condensation:
///
- /// ```ignore
+ /// ```text
/// ...............
/// : :
/// : ,-----. :
@@ -302,10 +301,9 @@ where
/// question "given that I've finished processing calls in `scc({e,f,...})`
/// for inlining, which other SCCs are now potentially ready for
/// processing?".
- pub fn evaporation<F, S>(&self, successors: F) -> Evaporation
+ pub fn evaporation<G>(&self, graph: G) -> Evaporation
where
- F: Fn(Node) -> S,
- S: Iterator<Item = Node>,
+ G: Graph<Node>,
{
log::trace!("Creating the evaporation of {self:#?}");
@@ -328,7 +326,7 @@ where
let mut reverse_edge_set = BTreeSet::<(Scc, Scc)>::new();
for (from_component, nodes) in self.iter() {
for node in nodes {
- for succ in successors(*node) {
+ for succ in graph.successors(*node) {
let to_component = node_to_component[succ].unwrap();
if to_component == from_component {
continue;
@@ -408,120 +406,13 @@ impl Evaporation {
}
}
-/// An iterative depth-first traversal.
-struct Dfs<Node> {
- stack: Vec<DfsEvent<Node>>,
-}
-
-impl<Node> Dfs<Node> {
- fn new(roots: impl IntoIterator<Item = Node>) -> Self {
- Self {
- stack: roots.into_iter().map(|v| DfsEvent::Pre(v)).collect(),
- }
- }
-}
-
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-enum DfsEvent<Node> {
- /// The first time seeing this node.
- Pre(Node),
-
- /// After having just visited the given edge.
- AfterEdge(Node, Node),
-
- /// Finished visiting this node and all of its successors.
- Post(Node),
-}
-
-impl<Node> Dfs<Node>
-where
- Node: Copy + std::fmt::Debug,
-{
- /// Pump the traversal, yielding the next `DfsEvent`.
- fn next<S>(
- &mut self,
- successors: impl Fn(Node) -> S,
- seen: impl Fn(Node) -> bool,
- ) -> Option<DfsEvent<Node>>
- where
- S: Iterator<Item = Node>,
- {
- loop {
- let event = self.stack.pop()?;
-
- if let DfsEvent::Pre(node) = event {
- if seen(node) {
- continue;
- }
-
- let successors = successors(node);
-
- let (min, max) = successors.size_hint();
- let estimated_successors_len = max.unwrap_or_else(|| 2 * min);
- self.stack.reserve(
- // We push an after-edge and pre event for each successor.
- 2 * estimated_successors_len
- // And we push one post event for this node.
- + 1,
- );
-
- self.stack.push(DfsEvent::Post(node));
- for succ in successors {
- self.stack.push(DfsEvent::AfterEdge(node, succ));
- if !seen(succ) {
- self.stack.push(DfsEvent::Pre(succ));
- }
- }
- }
-
- return Some(event);
- }
- }
-}
-
-mod non_max {
- use std::num::NonZeroU32;
-
- #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
- pub struct NonMaxU32(NonZeroU32);
-
- impl Default for NonMaxU32 {
- fn default() -> Self {
- Self::new(0).unwrap()
- }
- }
-
- impl core::fmt::Debug for NonMaxU32 {
- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
- f.debug_tuple("NonMaxU32").field(&self.get()).finish()
- }
- }
-
- impl NonMaxU32 {
- pub fn new(x: u32) -> Option<Self> {
- if x == u32::MAX {
- None
- } else {
- // Safety: We know that `x+1` is non-zero because it will not
- // overflow because `x` is not `u32::MAX`.
- Some(Self(unsafe { NonZeroU32::new_unchecked(x + 1) }))
- }
- }
-
- pub fn get(&self) -> u32 {
- self.0.get() - 1
- }
- }
-}
-use non_max::*;
-
#[cfg(test)]
mod tests {
use super::*;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct Node(u32);
- wasmtime_environ::entity_impl!(Node);
+ crate::entity_impl!(Node);
#[derive(Debug)]
struct Graph {
@@ -543,14 +434,30 @@ mod tests {
self.edges[Node::from_u32(node)].extend(edges.into_iter().map(|v| Node::from_u32(v)));
self
}
+ }
+
+ impl super::Graph<Node> for Graph {
+ type NodesIter<'a>
+ = cranelift_entity::Keys<Node>
+ where
+ Self: 'a;
+
+ fn nodes(&self) -> Self::NodesIter<'_> {
+ self.edges.keys()
+ }
+
+ type SuccessorsIter<'a>
+ = core::iter::Copied<core::slice::Iter<'a, Node>>
+ where
+ Self: 'a;
- fn edges(&self, node: Node) -> impl Iterator<Item = Node> {
+ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_> {
self.edges[node].iter().copied()
}
}
fn components(graph: &Graph) -> Vec<Vec<u32>> {
- let components = StronglyConnectedComponents::new(graph.edges.keys(), |v| graph.edges(v));
+ let components = StronglyConnectedComponents::new(graph);
components
.values()
.map(|vs| vs.iter().map(|v| v.as_u32()).collect::<Vec<_>>())
diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs
index f7bd814eae..2ff579a09b 100644
--- a/crates/environ/src/lib.rs
+++ b/crates/environ/src/lib.rs
@@ -17,6 +17,7 @@ extern crate std;
extern crate alloc;
pub mod collections;
+pub mod graphs;
pub mod prelude;
mod address_map;
@@ -90,7 +91,7 @@ pub use wasmtime_core::error;
#[cfg(feature = "anyhow")]
pub use self::error::ToWasmtimeResult;
-pub use wasmtime_core::{alloc::PanicOnOom, undo::Undo};
+pub use wasmtime_core::{alloc::PanicOnOom, non_max, undo::Undo};
// Only for use with `bindgen!`-generated code.
#[doc(hidden)]
diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs
index 658d182fe7..a4985f3219 100644
--- a/crates/wasmtime/src/compile.rs
+++ b/crates/wasmtime/src/compile.rs
@@ -27,19 +27,16 @@ use crate::hash_map::HashMap;
use crate::hash_set::HashSet;
use crate::prelude::*;
use std::{any::Any, borrow::Cow, collections::BTreeMap, mem, ops::Range};
-
-use call_graph::CallGraph;
use wasmtime_environ::{
Abi, CompiledFunctionBody, CompiledFunctionsTable, CompiledFunctionsTableBuilder,
CompiledModuleInfo, Compiler, DefinedFuncIndex, FilePos, FinishedObject, FuncKey,
FunctionBodyData, InliningCompiler, IntraModuleInlining, ModuleEnvironment, ModuleTranslation,
ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap, StaticModuleIndex, Tunables,
+ graphs::{EntityGraph, Graph as _},
};
#[cfg(feature = "component-model")]
use wasmtime_environ::{WasmChecksum, component::Translator};
-mod call_graph;
-mod scc;
mod stratify;
mod code_builder;
@@ -598,61 +595,11 @@ the use case.
compiler: &dyn Compiler,
inlining_compiler: &dyn InliningCompiler,
) -> Result<Vec<CompileOutput<'a>>, Error> {
- /// The index of a function (of any kind: Wasm function, trampoline, or
- /// etc...) in our list of unlinked outputs.
- #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
- struct OutputIndex(u32);
- wasmtime_environ::entity_impl!(OutputIndex);
-
// Our list of unlinked outputs.
let mut outputs = PrimaryMap::<OutputIndex, Option<CompileOutput<'_>>>::from(
engine.run_maybe_parallel(self.inputs, |f| f(compiler).map(Some))?,
);
- /// Whether a function (as described by the given `FuncKey`) can
- /// participate in inlining or not (either as a candidate for being
- /// inlined into a caller or having a callee inlined into a callsite
- /// within itself).
- fn is_inlining_function(key: FuncKey) -> bool {
- match key {
- // Wasm functions can both be inlined into other functions and
- // have other functions inlined into them.
- FuncKey::DefinedWasmFunction(..) => true,
-
- // Intrinsics can be inlined into other functions.
- #[cfg(feature = "component-model")]
- FuncKey::UnsafeIntrinsic(..) => true,
-
- // Trampolines cannot participate in inlining since our
- // unwinding and exceptions infrastructure relies on them being
- // in their own call frames.
- FuncKey::ArrayToWasmTrampoline(..)
- | FuncKey::WasmToArrayTrampoline(..)
- | FuncKey::WasmToBuiltinTrampoline(..)
- | FuncKey::PatchableToBuiltinTrampoline(..) => false,
- #[cfg(feature = "component-model")]
- FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,
-
- FuncKey::PulleyHostCall(_) => {
- unreachable!("we don't compile artifacts for Pulley host calls")
- }
- }
- }
-
- /// Get just the output indices of the functions that can participate in
- /// inlining from our unlinked outputs.
- fn inlining_functions<'a>(
- outputs: &'a PrimaryMap<OutputIndex, Option<CompileOutput<'_>>>,
- ) -> impl Iterator<Item = OutputIndex> + 'a {
- outputs.iter().filter_map(|(index, output)| {
- if is_inlining_function(output.as_ref().unwrap().key) {
- Some(index)
- } else {
- None
- }
- })
- }
-
// A map from a `FuncKey` to its index in our unlinked outputs.
//
// We will generally just be working with `OutputIndex`es, but
@@ -672,13 +619,11 @@ the use case.
// We only inline Wasm functions, not trampolines, because we rely on
// trampolines being in their own stack frame when we save the entry and
// exit SP, FP, and PC for backtraces in trampolines.
- let call_graph = CallGraph::<OutputIndex>::new(inlining_functions(&outputs), {
+ let call_graph = EntityGraph::<OutputIndex>::new(inlining_functions(&outputs), {
let mut func_keys = IndexSet::default();
let outputs = &outputs;
let key_to_output = &key_to_output;
move |output_index, calls| {
- debug_assert!(calls.is_empty());
-
let output = outputs[output_index].as_ref().unwrap();
debug_assert!(is_inlining_function(output.key));
@@ -688,18 +633,20 @@ the use case.
// Translate each of those to keys to output indices, which is
// what we actually need.
+ debug_assert!(calls.is_empty());
calls.extend(
func_keys
.iter()
.copied()
- .filter_map(|key| key_to_output.get(&key)),
+ .filter_map(|key| key_to_output.get(&key).copied()),
);
log::trace!(
"call graph edges for {output_index:?} = {:?}: {calls:?}",
output.key
);
- Ok(())
+
+ crate::error::Ok(())
}
})?;
@@ -708,8 +655,10 @@ the use case.
// (because they either do not call each other or are part of a
// mutual-recursion cycle; either way we won't inline members of the
// same layer into each other).
- let strata =
- stratify::Strata::<OutputIndex>::new(inlining_functions(&outputs), &call_graph);
+ let strata = stratify::Strata::<OutputIndex>::new(&call_graph.filter_nodes(|f| {
+ let key = outputs[*f].as_ref().unwrap().key;
+ is_inlining_function(key)
+ }));
let mut layer_outputs = vec![];
for layer in strata.layers() {
// Temporarily take this layer's outputs out of our unlinked outputs
@@ -915,6 +864,56 @@ the use case.
}
}
+/// The index of a function (of any kind: Wasm function, trampoline, or
+/// etc...) in our list of unlinked outputs.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
+struct OutputIndex(u32);
+wasmtime_environ::entity_impl!(OutputIndex);
+
+/// Whether a function (as described by the given `FuncKey`) can
+/// participate in inlining or not (either as a candidate for being
+/// inlined into a caller or having a callee inlined into a callsite
+/// within itself).
+fn is_inlining_function(key: FuncKey) -> bool {
+ match key {
+ // Wasm functions can both be inlined into other functions and
+ // have other functions inlined into them.
+ FuncKey::DefinedWasmFunction(..) => true,
+
+ // Intrinsics can be inlined into other functions.
+ #[cfg(feature = "component-model")]
+ FuncKey::UnsafeIntrinsic(..) => true,
+
+ // Trampolines cannot participate in inlining since our
+ // unwinding and exceptions infrastructure relies on them being
+ // in their own call frames.
+ FuncKey::ArrayToWasmTrampoline(..)
+ | FuncKey::WasmToArrayTrampoline(..)
+ | FuncKey::WasmToBuiltinTrampoline(..)
+ | FuncKey::PatchableToBuiltinTrampoline(..) => false,
+ #[cfg(feature = "component-model")]
+ FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,
+
+ FuncKey::PulleyHostCall(_) => {
+ unreachable!("we don't compile artifacts for Pulley host calls")
+ }
+ }
+}
+
+/// Get just the output indices of the functions that can participate in
+/// inlining from our unlinked outputs.
+fn inlining_functions<'a>(
+ outputs: &'a PrimaryMap<OutputIndex, Option<CompileOutput<'_>>>,
+) -> impl Iterator<Item = OutputIndex> + 'a {
+ outputs.iter().filter_map(|(index, output)| {
+ if is_inlining_function(output.as_ref().unwrap().key) {
+ Some(index)
+ } else {
+ None
+ }
+ })
+}
+
fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutput>) -> Result<()> {
let compiler = engine.try_compiler()?;
let mut builtins = HashSet::new();
diff --git a/crates/wasmtime/src/compile/call_graph.rs b/crates/wasmtime/src/compile/call_graph.rs
deleted file mode 100644
index 4ce62245d4..0000000000
--- a/crates/wasmtime/src/compile/call_graph.rs
+++ /dev/null
@@ -1,105 +0,0 @@
-//! Construction of the call-graph, for the purposes of inlining.
-//!
-//! These call graphs are not necessarily complete or accurate, and Wasmtime's
-//! soundness does not rely on those properties. First off, we do not attempt to
-//! understand indirect calls, which at their worst must force any call analysis
-//! give up and say "the callee could be absolutely any function". More
-//! interestingly, these call graphs are only used for scheduling bottom-up
-//! inlining, so the worst that inaccurate information can do is cause us to
-//! miss inlining opportunities or lose potential parallelism in our
-//! schedule. For best results, however, every direct call that is potentially
-//! inlinable should be reported when constructing these call graphs.
-
-use super::*;
-use core::{
- fmt::{self, Debug},
- ops::Range,
-};
-use wasmtime_environ::{EntityRef, SecondaryMap};
-
-/// A call graph reified into a densely packed and quickly accessible
-/// representation.
-///
-/// In a call graph, nodes are functions, and an edge `f --> g` means that the
-/// function `f` calls the function `g`.
-pub struct CallGraph<Node>
-where
- Node: EntityRef + Debug,
-{
- /// A map from each node to the subslice of `self.edge_elems` that are its
- /// edges.
- edges: SecondaryMap<Node, Range<u32>>,
-
- /// Densely packed edge elements for `self.edges`.
- edge_elems: Vec<Node>,
-}
-
-impl<Node> Debug for CallGraph<Node>
-where
- Node: EntityRef + Debug,
-{
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- struct Edges<'a, Node: EntityRef + Debug>(&'a CallGraph<Node>);
-
- impl<'a, Node: EntityRef + Debug> Debug for Edges<'a, Node> {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- f.debug_map()
- .entries(self.0.nodes().map(|n| (n, self.0.edges(n))))
- .finish()
- }
- }
-
- f.debug_struct("CallGraph")
- .field("edges", &Edges(self))
- .finish()
- }
-}
-
-impl<Node> CallGraph<Node>
-where
- Node: EntityRef + Debug,
-{
- /// Construct a new call graph.
- ///
- /// `funcs` should be an iterator over all function nodes in this call
- /// graph's translation unit.
- ///
- /// The `get_calls` function should yield (by pushing onto the given `Vec`)
- /// all of the callee function nodes that the given caller function node
- /// calls.
- pub fn new(
- funcs: impl IntoIterator<Item = Node>,
- mut get_calls: impl FnMut(Node, &mut Vec<Node>) -> Result<()>,
- ) -> Result<Self> {
- let funcs = funcs.into_iter();
-
- let (min, max) = funcs.size_hint();
- let capacity = max.unwrap_or_else(|| 2 * min);
- let mut edges = SecondaryMap::with_capacity(capacity);
- let mut edge_elems = vec![];
-
- let mut calls = vec![];
- for caller in funcs {
- debug_assert!(calls.is_empty());
- get_calls(caller, &mut calls)?;
-
- debug_assert_eq!(edges[caller], Range::default());
- edges[caller] = extend_with_range(&mut edge_elems, calls.drain(..));
- }
-
- Ok(CallGraph { edges, edge_elems })
- }
-
- /// Get the function nodes in this call graph.
- pub fn nodes(&self) -> impl ExactSizeIterator<Item = Node> {
- self.edges.keys()
- }
-
- /// Get the callee function nodes that the given caller function node calls.
- pub fn edges(&self, node: Node) -> &[Node] {
- let Range { start, end } = self.edges[node].clone();
- let start = usize::try_from(start).unwrap();
- let end = usize::try_from(end).unwrap();
- &self.edge_elems[start..end]
- }
-}
diff --git a/crates/wasmtime/src/compile/stratify.rs b/crates/wasmtime/src/compile/stratify.rs
index 888f6635f8..028d985482 100644
--- a/crates/wasmtime/src/compile/stratify.rs
+++ b/crates/wasmtime/src/compile/stratify.rs
@@ -50,13 +50,12 @@
//! dependencies each component has, and a component is ready for inclusion in a
//! layer once its unprocessed-dependencies count reaches zero.
-use super::{
- call_graph::CallGraph,
- scc::{Scc, StronglyConnectedComponents},
- *,
-};
+use super::*;
use std::{fmt::Debug, ops::Range};
-use wasmtime_environ::{EntityRef, SecondaryMap};
+use wasmtime_environ::{
+ EntityRef, SecondaryMap,
+ graphs::{Graph, Scc, StronglyConnectedComponents},
+};
/// A stratified call graph; essentially a parallel-execution plan for bottom-up
/// inlining.
@@ -90,15 +89,15 @@ impl<Node: Debug> Debug for Strata<Node> {
impl<Node> Strata<Node> {
/// Stratify the given call graph, yielding a `Strata` parallel-execution
/// plan.
- pub fn new(nodes: impl IntoIterator<Item = Node>, call_graph: &CallGraph<Node>) -> Self
+ pub fn new<G>(call_graph: &G) -> Self
where
Node: EntityRef + Debug,
+ G: Debug + Graph<Node>,
{
log::trace!("Stratifying {call_graph:#?}");
- let components =
- StronglyConnectedComponents::new(nodes, |node| call_graph.edges(node).iter().copied());
- let evaporation = components.evaporation(|node| call_graph.edges(node).iter().copied());
+ let components = StronglyConnectedComponents::new(call_graph);
+ let evaporation = components.evaporation(call_graph);
// A map from each component to the count of how many call-graph
// dependencies to other components it has that have not been fulfilled
@@ -122,14 +121,16 @@ impl<Node> Strata<Node> {
// fixed point, similarly to GC tracing and ref-count decrementing.
let mut layers: Vec<Range<u32>> = vec![];
- let mut layer_elems: Vec<Node> = Vec::with_capacity(call_graph.nodes().len());
+ let (min, max) = call_graph.nodes().size_hint();
+ let cap = max.unwrap_or(min);
+ let mut layer_elems: Vec<Node> = Vec::with_capacity(cap);
let mut current_layer: Vec<Scc> = components
.keys()
.filter(|scc| unfulfilled_deps_count[*scc] == 0)
.collect();
debug_assert!(
- !current_layer.is_empty() || call_graph.nodes().len() == 0,
+ !current_layer.is_empty() || call_graph.nodes().next().is_none(),
"the first layer can only be empty when the call graph itself is empty"
);
@@ -187,11 +188,13 @@ impl<Node> Strata<Node> {
#[cfg(test)]
mod tests {
use super::*;
+ use wasmtime_environ::graphs::Graph;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
struct Function(u32);
wasmtime_environ::entity_impl!(Function);
+ #[derive(Debug)]
struct Functions {
calls: SecondaryMap<Function, Vec<Function>>,
}
@@ -205,6 +208,26 @@ mod tests {
}
}
+ impl Graph<Function> for Functions {
+ type NodesIter<'a>
+ = wasmtime_environ::Keys<Function>
+ where
+ Self: 'a;
+
+ fn nodes(&self) -> Self::NodesIter<'_> {
+ self.calls.keys()
+ }
+
+ type SuccessorsIter<'a>
+ = core::iter::Copied<core::slice::Iter<'a, Function>>
+ where
+ Self: 'a;
+
+ fn successors(&self, f: Function) -> Self::SuccessorsIter<'_> {
+ self.calls[f].iter().copied()
+ }
+ }
+
impl Functions {
fn define_func(&mut self, f: u32) -> &mut Self {
let f = Function::from_u32(f);
@@ -235,12 +258,7 @@ mod tests {
}
fn stratify(&self) -> Strata<Function> {
- let call_graph = CallGraph::new(self.calls.keys(), |f, calls| {
- calls.extend_from_slice(&self.calls[f]);
- Ok(())
- })
- .unwrap();
- Strata::<Function>::new(self.calls.keys(), &call_graph)
+ Strata::new(self)
}
fn assert_stratification(&self, mut expected: Vec<Vec<u32>>) {