A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Make SCCs and graph traversals reusable (#12590)
Nick Fitzgerald59d23a356484
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 6e538cdde3..0e55cba034 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -21,5 +21,6 @@ extern crate std;pub mod alloc;pub mod error;pub mod math;+pub mod non_max;pub mod slab;pub mod undo;diff --git a/crates/core/src/non_max.rs b/crates/core/src/non_max.rsnew file mode 100644index 0000000000..51e1d33413--- /dev/null+++ b/crates/core/src/non_max.rs@@ -0,0 +1,94 @@+//! Integer newtypes that are guaranteed not to be their maximum value.++macro_rules! define_non_max {+ ( $(+ $( #[ $attr:meta ] )*+ pub struct $non_max:ident($non_zero:ty) : $prim:ty;+ )* ) => {+ $(+ /// A+ #[doc = concat!("`", stringify!($prim), "`")]+ /// value that is known not to be+ #[doc = concat!("`", stringify!($prim), "::MAX`.")]+ ///+ /// This enables some memory layout optimizations. For example,+ #[doc = concat!("`Option<", stringify!($prim), ">`")]+ /// is the same size as+ #[doc = concat!("`", stringify!($prim), "`:")]+ ///+ /// ```+ /// # use wasmtime_internal_core::non_max::NonMaxU32;+ /// assert_eq!(+ /// core::mem::size_of::<u32>(),+ /// core::mem::size_of::<Option<NonMaxU32>>(),+ /// );+ /// ```+ #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]+ pub struct $non_max($non_zero);++ impl Default for $non_max {+ fn default() -> Self {+ Self::new(0).unwrap()+ }+ }++ impl core::fmt::Debug for $non_max {+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {+ f.debug_tuple(stringify!($non_max))+ .field(&self.get())+ .finish()+ }+ }++ impl $non_max {+ /// Construct a new+ #[doc = concat!("`", stringify!($prim), "`")]+ /// value.+ ///+ /// Returns `None` when given+ #[doc = concat!("`", stringify!($prim), "::MAX`.")]+ #[inline]+ pub fn new(x: $prim) -> Option<Self> {+ if x == <$prim>::MAX {+ None+ } else {+ // Safety: `x != $prim::MAX`.+ Some(unsafe { Self::new_unchecked(x) })+ }+ }++ /// Unsafely construct a new+ #[doc = concat!("`", stringify!($prim), "`")]+ /// value.+ ///+ /// # Safety+ ///+ /// The given value must not be+ #[doc = concat!("`", stringify!($prim), "::MAX`.")]+ #[inline]+ pub unsafe fn new_unchecked(x: $prim) -> Self {+ debug_assert_ne!(x, <$prim>::MAX);+ // Safety: We know that `x+1` is non-zero because it will not+ // overflow because `x` is not `$prim::MAX`.+ Self(unsafe { <$non_zero>::new_unchecked(x + 1) })+ }++ /// Get the underlying+ #[doc = concat!("`", stringify!($prim), "`")]+ /// value.+ #[inline]+ pub fn get(&self) -> $prim {+ self.0.get() - 1+ }+ }+ )*+ };+}++define_non_max! {+ pub struct NonMaxU8(core::num::NonZeroU8) : u8;+ pub struct NonMaxU16(core::num::NonZeroU16) : u16;+ pub struct NonMaxU32(core::num::NonZeroU32) : u32;+ pub struct NonMaxU64(core::num::NonZeroU64) : u64;+ pub struct NonMaxUsize(core::num::NonZeroUsize) : usize;+}diff --git a/crates/environ/src/graphs.rs b/crates/environ/src/graphs.rsnew file mode 100644index 0000000000..5c9be856d5--- /dev/null+++ b/crates/environ/src/graphs.rs@@ -0,0 +1,134 @@+//! Generic graph traits, traversals, and data structures for use in Wasmtime.++mod dfs;+mod entity_graph;+mod scc;+use core::{fmt, iter};++pub use dfs::*;+pub use entity_graph::*;+pub use scc::*;++use crate::prelude::*;++/// A trait for any kind of graph data structure.+pub trait Graph<Node> {+ /// The iterator type returned by `Nodes::nodes`.+ type NodesIter<'a>: Iterator<Item = Node>+ where+ Self: 'a;++ /// Iterate over the nodes in this graph.+ fn nodes(&self) -> Self::NodesIter<'_>;++ /// The iterator type returned by `Successors::successors`.+ type SuccessorsIter<'a>: Iterator<Item = Node>+ where+ Self: 'a;++ /// Iterate over the successors of the given `node`.+ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_>;++ // Provided Methods.++ /// Like `Iterator::by_ref` but for `Graph`.+ fn by_ref(&self) -> &Self {+ self+ }++ /// Use the given predicate to filter out certain nodes from the graph.+ fn filter_nodes<F>(self, predicate: F) -> FilterNodes<Self, F>+ where+ Self: Sized,+ F: Fn(&Node) -> bool,+ {+ FilterNodes {+ graph: self,+ predicate,+ }+ }+}++impl<T, Node> Graph<Node> for &'_ T+where+ T: ?Sized + Graph<Node>,+{+ type NodesIter<'a>+ = T::NodesIter<'a>+ where+ Self: 'a;++ fn nodes(&self) -> Self::NodesIter<'_> {+ (*self).nodes()+ }++ type SuccessorsIter<'a>+ = T::SuccessorsIter<'a>+ where+ Self: 'a;++ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_> {+ (*self).successors(node)+ }+}++/// A graph whose nodes are being filtered by the predicate `F`.+///+/// Created by the `Graph::filter_nodes` trait method.+pub struct FilterNodes<G, F> {+ graph: G,+ predicate: F,+}++impl<G, F> fmt::Debug for FilterNodes<G, F>+where+ G: fmt::Debug,+{+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_struct("Filter")+ .field("graph", &self.graph)+ .field("predicate", &"..")+ .finish()+ }+}++impl<G, F, Node> Graph<Node> for FilterNodes<G, F>+where+ G: Graph<Node>,+ F: Fn(&Node) -> bool,+{+ type NodesIter<'a>+ = iter::Filter<G::NodesIter<'a>, &'a F>+ where+ Self: 'a;++ fn nodes(&self) -> Self::NodesIter<'_> {+ self.graph.nodes().filter(&self.predicate)+ }++ type SuccessorsIter<'a>+ = iter::Filter<G::SuccessorsIter<'a>, &'a F>+ where+ Self: 'a;++ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_> {+ self.graph.successors(node).filter(&self.predicate)+ }+}++/// Extend `dest` with `items` and return the range of indices in `dest` where+/// they ended up.+fn extend_with_range<T>(+ dest: &mut Vec<T>,+ items: impl IntoIterator<Item = T>,+) -> core::ops::Range<u32> {+ let start = dest.len();+ let start = u32::try_from(start).unwrap();++ dest.extend(items);++ let end = dest.len();+ let end = u32::try_from(end).unwrap();++ start..end+}diff --git a/crates/environ/src/graphs/dfs.rs b/crates/environ/src/graphs/dfs.rsnew file mode 100644index 0000000000..6265c7f314--- /dev/null+++ b/crates/environ/src/graphs/dfs.rs@@ -0,0 +1,91 @@+use super::*;++/// An iterative depth-first traversal.+pub struct Dfs<Node> {+ stack: Vec<DfsEvent<Node>>,+}++impl<Node> Default for Dfs<Node> {+ fn default() -> Self {+ Self {+ stack: Default::default(),+ }+ }+}++impl<Node> Dfs<Node> {+ /// Create a new DFS traversal, starting at the given roots.+ pub fn new(roots: impl IntoIterator<Item = Node>) -> Self {+ let mut dfs = Self::default();+ dfs.add_roots(roots);+ dfs+ }++ /// Add a single new root to this traversal, to be visited immediately.+ pub fn add_root(&mut self, root: Node) {+ self.stack.push(DfsEvent::Pre(root));+ }++ /// Add multiple new roots to this traversal, to be visited immediately.+ pub fn add_roots(&mut self, roots: impl IntoIterator<Item = Node>) {+ self.stack+ .extend(roots.into_iter().map(|v| DfsEvent::Pre(v)));+ }+}++/// An event during a DFS traversal.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub enum DfsEvent<Node> {+ /// The first time seeing this node.+ Pre(Node),++ /// After having just visited the given edge.+ AfterEdge(Node, Node),++ /// Finished visiting this node and all of its successors.+ Post(Node),+}++impl<Node> Dfs<Node>+where+ Node: Copy,+{+ /// Pump the traversal, yielding the next `DfsEvent`.+ ///+ /// Returns `None` when the traversal is complete.+ pub fn next<G>(&mut self, graph: G, seen: impl Fn(Node) -> bool) -> Option<DfsEvent<Node>>+ where+ G: Graph<Node>,+ {+ loop {+ let event = self.stack.pop()?;++ if let DfsEvent::Pre(node) = event {+ if seen(node) {+ continue;+ }++ let successors = graph.successors(node);++ let (min, max) = successors.size_hint();+ let estimated_successors_len = max.unwrap_or_else(|| 2 * min);+ self.stack.reserve(+ // We push an after-edge and pre event for each successor.+ 2 * estimated_successors_len+ // And we push one post event for this node.+ + 1,+ );++ self.stack.push(DfsEvent::Post(node));+ for succ in successors {+ self.stack.push(DfsEvent::AfterEdge(node, succ));+ if !seen(succ) {+ self.stack.push(DfsEvent::Pre(succ));+ }+ }+ }++ return Some(event);+ }+ }+}diff --git a/crates/environ/src/graphs/entity_graph.rs b/crates/environ/src/graphs/entity_graph.rsnew file mode 100644index 0000000000..659d439441--- /dev/null+++ b/crates/environ/src/graphs/entity_graph.rs@@ -0,0 +1,102 @@+use super::*;+use crate::{EntityRef, SecondaryMap, prelude::*};+use core::{+ fmt::{self, Debug},+ iter,+ ops::Range,+};++/// A graph of `EntityRef` nodes reified into a densely packed representation.+pub struct EntityGraph<Node>+where+ Node: EntityRef,+{+ /// A map from each node to the subslice of `self.edge_elems` that are its+ /// edges.+ edges: SecondaryMap<Node, Range<u32>>,++ /// Densely packed edge elements for `self.edges`.+ edge_elems: Vec<Node>,+}++impl<Node> Debug for EntityGraph<Node>+where+ Node: EntityRef + Debug,+{+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ struct Edges<'a, Node: EntityRef + Debug>(&'a EntityGraph<Node>);++ impl<'a, Node: EntityRef + Debug> Debug for Edges<'a, Node> {+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {+ f.debug_map()+ .entries(+ self.0+ .nodes()+ .map(|n| (n, self.0.successors(n).collect::<Box<[_]>>())),+ )+ .finish()+ }+ }++ f.debug_struct("Graph")+ .field("edges", &Edges(self))+ .finish()+ }+}++impl<Node> EntityGraph<Node>+where+ Node: EntityRef + Debug,+{+ /// Construct a new, concrete `EntityGraph`.+ pub fn new<E>(+ nodes: impl IntoIterator<Item = Node>,+ mut successors: impl FnMut(Node, &mut Vec<Node>) -> Result<(), E>,+ ) -> Result<Self, E> {+ let nodes = nodes.into_iter();++ let (min, max) = nodes.size_hint();+ let capacity = max.unwrap_or_else(|| 2 * min);++ let mut edges = SecondaryMap::with_capacity(capacity);+ let mut edge_elems = vec![];++ let mut succs = vec![];+ for v in nodes {+ debug_assert!(succs.is_empty());+ successors(v, &mut succs)?;++ debug_assert_eq!(edges[v], Range::default());+ edges[v] = extend_with_range(&mut edge_elems, succs.drain(..));+ }++ Ok(EntityGraph { edges, edge_elems })+ }+}++impl<Node> Graph<Node> for EntityGraph<Node>+where+ Node: EntityRef,+{+ type NodesIter<'a>+ = cranelift_entity::Keys<Node>+ where+ Self: 'a;++ #[inline]+ fn nodes(&self) -> Self::NodesIter<'_> {+ self.edges.keys()+ }++ type SuccessorsIter<'a>+ = iter::Copied<core::slice::Iter<'a, Node>>+ where+ Self: 'a;++ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_> {+ let Range { start, end } = self.edges[node].clone();+ let start = usize::try_from(start).unwrap();+ let end = usize::try_from(end).unwrap();+ self.edge_elems[start..end].iter().copied()+ }+}diff --git a/crates/wasmtime/src/compile/scc.rs b/crates/environ/src/graphs/scc.rssimilarity index 84%rename from crates/wasmtime/src/compile/scc.rsrename to crates/environ/src/graphs/scc.rsindex 025f2e3f53..37072db83b 100644--- a/crates/wasmtime/src/compile/scc.rs+++ b/crates/environ/src/graphs/scc.rs@@ -12,20 +12,23 @@//! [Tarjan's algorithm]: https://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithmuse super::*;-use std::{- collections::BTreeSet,+use crate::{+ EntityRef, EntitySet, PrimaryMap, SecondaryMap, non_max::NonMaxU32,+ packed_option::PackedOption, prelude::*,+};+use alloc::collections::BTreeSet;+use core::{+ cmp,fmt::{self, Debug},hash::Hash,+ iter,ops::Range,};-use wasmtime_environ::{- EntityRef, EntitySet, PrimaryMap, SecondaryMap, packed_option::PackedOption,-};/// A strongly-connected component.#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct Scc(u32);-wasmtime_environ::entity_impl!(Scc);+crate::entity_impl!(Scc);/// The set of strongly-connected components for a graph of `Node`s.pub struct StronglyConnectedComponents<Node>@@ -64,16 +67,14 @@ whereimpl<Node> StronglyConnectedComponents<Node>where- Node: EntityRef + std::fmt::Debug,+ Node: EntityRef + Debug,{/// Find the strongly-connected for the given graph.- pub fn new<I, F, S>(nodes: I, successors: F) -> Self+ pub fn new<G>(graph: G) -> Selfwhere- I: IntoIterator<Item = Node>,- F: Fn(Node) -> S,- S: Iterator<Item = Node>,+ G: Graph<Node>,{- let nodes = nodes.into_iter();+ let nodes = graph.nodes();// The resulting components and their nodes.let mut component_nodes = vec![];@@ -97,7 +98,7 @@ wherelet mut dfs = Dfs::new(nodes);while let Some(event) = dfs.next(- &successors,+ &graph,// We have seen the node before if we have assigned it a DFS index.|node| indices[node].is_some(),) {@@ -134,10 +135,8 @@ where// part of the same SCC as this node, so propagate its// lowlink to this node's lowlink.if on_stack.contains(succ) {- lowlinks[node] = Some(std::cmp::min(- lowlinks[node].unwrap(),- lowlinks[succ].unwrap(),- ));+ lowlinks[node] =+ Some(cmp::min(lowlinks[node].unwrap(), lowlinks[succ].unwrap()));}}@@ -153,7 +152,7 @@ wherelet mut done = false;components.push(extend_with_range(&mut component_nodes,- std::iter::from_fn(|| {+ iter::from_fn(|| {if done {return None;}@@ -236,7 +235,7 @@ where/// In the following diagram, the solid lines are the input graph and the/// dotted lines show its condensation:///- /// ```ignore+ /// ```text/// .............../// : :/// : ,-----. :@@ -302,10 +301,9 @@ where/// question "given that I've finished processing calls in `scc({e,f,...})`/// for inlining, which other SCCs are now potentially ready for/// processing?".- pub fn evaporation<F, S>(&self, successors: F) -> Evaporation+ pub fn evaporation<G>(&self, graph: G) -> Evaporationwhere- F: Fn(Node) -> S,- S: Iterator<Item = Node>,+ G: Graph<Node>,{log::trace!("Creating the evaporation of {self:#?}");@@ -328,7 +326,7 @@ wherelet mut reverse_edge_set = BTreeSet::<(Scc, Scc)>::new();for (from_component, nodes) in self.iter() {for node in nodes {- for succ in successors(*node) {+ for succ in graph.successors(*node) {let to_component = node_to_component[succ].unwrap();if to_component == from_component {continue;@@ -408,120 +406,13 @@ impl Evaporation {}}-/// An iterative depth-first traversal.-struct Dfs<Node> {- stack: Vec<DfsEvent<Node>>,-}--impl<Node> Dfs<Node> {- fn new(roots: impl IntoIterator<Item = Node>) -> Self {- Self {- stack: roots.into_iter().map(|v| DfsEvent::Pre(v)).collect(),- }- }-}--#[derive(Clone, Copy, Debug, PartialEq, Eq)]-enum DfsEvent<Node> {- /// The first time seeing this node.- Pre(Node),-- /// After having just visited the given edge.- AfterEdge(Node, Node),-- /// Finished visiting this node and all of its successors.- Post(Node),-}--impl<Node> Dfs<Node>-where- Node: Copy + std::fmt::Debug,-{- /// Pump the traversal, yielding the next `DfsEvent`.- fn next<S>(- &mut self,- successors: impl Fn(Node) -> S,- seen: impl Fn(Node) -> bool,- ) -> Option<DfsEvent<Node>>- where- S: Iterator<Item = Node>,- {- loop {- let event = self.stack.pop()?;-- if let DfsEvent::Pre(node) = event {- if seen(node) {- continue;- }-- let successors = successors(node);-- let (min, max) = successors.size_hint();- let estimated_successors_len = max.unwrap_or_else(|| 2 * min);- self.stack.reserve(- // We push an after-edge and pre event for each successor.- 2 * estimated_successors_len- // And we push one post event for this node.- + 1,- );-- self.stack.push(DfsEvent::Post(node));- for succ in successors {- self.stack.push(DfsEvent::AfterEdge(node, succ));- if !seen(succ) {- self.stack.push(DfsEvent::Pre(succ));- }- }- }-- return Some(event);- }- }-}--mod non_max {- use std::num::NonZeroU32;-- #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]- pub struct NonMaxU32(NonZeroU32);-- impl Default for NonMaxU32 {- fn default() -> Self {- Self::new(0).unwrap()- }- }-- impl core::fmt::Debug for NonMaxU32 {- fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {- f.debug_tuple("NonMaxU32").field(&self.get()).finish()- }- }-- impl NonMaxU32 {- pub fn new(x: u32) -> Option<Self> {- if x == u32::MAX {- None- } else {- // Safety: We know that `x+1` is non-zero because it will not- // overflow because `x` is not `u32::MAX`.- Some(Self(unsafe { NonZeroU32::new_unchecked(x + 1) }))- }- }-- pub fn get(&self) -> u32 {- self.0.get() - 1- }- }-}-use non_max::*;-#[cfg(test)]mod tests {use super::*;#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct Node(u32);- wasmtime_environ::entity_impl!(Node);+ crate::entity_impl!(Node);#[derive(Debug)]struct Graph {@@ -543,14 +434,30 @@ mod tests {self.edges[Node::from_u32(node)].extend(edges.into_iter().map(|v| Node::from_u32(v)));self}+ }++ impl super::Graph<Node> for Graph {+ type NodesIter<'a>+ = cranelift_entity::Keys<Node>+ where+ Self: 'a;++ fn nodes(&self) -> Self::NodesIter<'_> {+ self.edges.keys()+ }++ type SuccessorsIter<'a>+ = core::iter::Copied<core::slice::Iter<'a, Node>>+ where+ Self: 'a;- fn edges(&self, node: Node) -> impl Iterator<Item = Node> {+ fn successors(&self, node: Node) -> Self::SuccessorsIter<'_> {self.edges[node].iter().copied()}}fn components(graph: &Graph) -> Vec<Vec<u32>> {- let components = StronglyConnectedComponents::new(graph.edges.keys(), |v| graph.edges(v));+ let components = StronglyConnectedComponents::new(graph);components.values().map(|vs| vs.iter().map(|v| v.as_u32()).collect::<Vec<_>>())diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rsindex f7bd814eae..2ff579a09b 100644--- a/crates/environ/src/lib.rs+++ b/crates/environ/src/lib.rs@@ -17,6 +17,7 @@ extern crate std;extern crate alloc;pub mod collections;+pub mod graphs;pub mod prelude;mod address_map;@@ -90,7 +91,7 @@ pub use wasmtime_core::error;#[cfg(feature = "anyhow")]pub use self::error::ToWasmtimeResult;-pub use wasmtime_core::{alloc::PanicOnOom, undo::Undo};+pub use wasmtime_core::{alloc::PanicOnOom, non_max, undo::Undo};// Only for use with `bindgen!`-generated code.#[doc(hidden)]diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rsindex 658d182fe7..a4985f3219 100644--- a/crates/wasmtime/src/compile.rs+++ b/crates/wasmtime/src/compile.rs@@ -27,19 +27,16 @@ use crate::hash_map::HashMap;use crate::hash_set::HashSet;use crate::prelude::*;use std::{any::Any, borrow::Cow, collections::BTreeMap, mem, ops::Range};--use call_graph::CallGraph;use wasmtime_environ::{Abi, CompiledFunctionBody, CompiledFunctionsTable, CompiledFunctionsTableBuilder,CompiledModuleInfo, Compiler, DefinedFuncIndex, FilePos, FinishedObject, FuncKey,FunctionBodyData, InliningCompiler, IntraModuleInlining, ModuleEnvironment, ModuleTranslation,ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap, StaticModuleIndex, Tunables,+ graphs::{EntityGraph, Graph as _},};#[cfg(feature = "component-model")]use wasmtime_environ::{WasmChecksum, component::Translator};-mod call_graph;-mod scc;mod stratify;mod code_builder;@@ -598,61 +595,11 @@ the use case.compiler: &dyn Compiler,inlining_compiler: &dyn InliningCompiler,) -> Result<Vec<CompileOutput<'a>>, Error> {- /// The index of a function (of any kind: Wasm function, trampoline, or- /// etc...) in our list of unlinked outputs.- #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]- struct OutputIndex(u32);- wasmtime_environ::entity_impl!(OutputIndex);-// Our list of unlinked outputs.let mut outputs = PrimaryMap::<OutputIndex, Option<CompileOutput<'_>>>::from(engine.run_maybe_parallel(self.inputs, |f| f(compiler).map(Some))?,);- /// Whether a function (as described by the given `FuncKey`) can- /// participate in inlining or not (either as a candidate for being- /// inlined into a caller or having a callee inlined into a callsite- /// within itself).- fn is_inlining_function(key: FuncKey) -> bool {- match key {- // Wasm functions can both be inlined into other functions and- // have other functions inlined into them.- FuncKey::DefinedWasmFunction(..) => true,-- // Intrinsics can be inlined into other functions.- #[cfg(feature = "component-model")]- FuncKey::UnsafeIntrinsic(..) => true,-- // Trampolines cannot participate in inlining since our- // unwinding and exceptions infrastructure relies on them being- // in their own call frames.- FuncKey::ArrayToWasmTrampoline(..)- | FuncKey::WasmToArrayTrampoline(..)- | FuncKey::WasmToBuiltinTrampoline(..)- | FuncKey::PatchableToBuiltinTrampoline(..) => false,- #[cfg(feature = "component-model")]- FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,-- FuncKey::PulleyHostCall(_) => {- unreachable!("we don't compile artifacts for Pulley host calls")- }- }- }-- /// Get just the output indices of the functions that can participate in- /// inlining from our unlinked outputs.- fn inlining_functions<'a>(- outputs: &'a PrimaryMap<OutputIndex, Option<CompileOutput<'_>>>,- ) -> impl Iterator<Item = OutputIndex> + 'a {- outputs.iter().filter_map(|(index, output)| {- if is_inlining_function(output.as_ref().unwrap().key) {- Some(index)- } else {- None- }- })- }-// A map from a `FuncKey` to its index in our unlinked outputs.//// We will generally just be working with `OutputIndex`es, but@@ -672,13 +619,11 @@ the use case.// We only inline Wasm functions, not trampolines, because we rely on// trampolines being in their own stack frame when we save the entry and// exit SP, FP, and PC for backtraces in trampolines.- let call_graph = CallGraph::<OutputIndex>::new(inlining_functions(&outputs), {+ let call_graph = EntityGraph::<OutputIndex>::new(inlining_functions(&outputs), {let mut func_keys = IndexSet::default();let outputs = &outputs;let key_to_output = &key_to_output;move |output_index, calls| {- debug_assert!(calls.is_empty());-let output = outputs[output_index].as_ref().unwrap();debug_assert!(is_inlining_function(output.key));@@ -688,18 +633,20 @@ the use case.// Translate each of those to keys to output indices, which is// what we actually need.+ debug_assert!(calls.is_empty());calls.extend(func_keys.iter().copied()- .filter_map(|key| key_to_output.get(&key)),+ .filter_map(|key| key_to_output.get(&key).copied()),);log::trace!("call graph edges for {output_index:?} = {:?}: {calls:?}",output.key);- Ok(())++ crate::error::Ok(())}})?;@@ -708,8 +655,10 @@ the use case.// (because they either do not call each other or are part of a// mutual-recursion cycle; either way we won't inline members of the// same layer into each other).- let strata =- stratify::Strata::<OutputIndex>::new(inlining_functions(&outputs), &call_graph);+ let strata = stratify::Strata::<OutputIndex>::new(&call_graph.filter_nodes(|f| {+ let key = outputs[*f].as_ref().unwrap().key;+ is_inlining_function(key)+ }));let mut layer_outputs = vec![];for layer in strata.layers() {// Temporarily take this layer's outputs out of our unlinked outputs@@ -915,6 +864,56 @@ the use case.}}+/// The index of a function (of any kind: Wasm function, trampoline, or+/// etc...) in our list of unlinked outputs.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]+struct OutputIndex(u32);+wasmtime_environ::entity_impl!(OutputIndex);++/// Whether a function (as described by the given `FuncKey`) can+/// participate in inlining or not (either as a candidate for being+/// inlined into a caller or having a callee inlined into a callsite+/// within itself).+fn is_inlining_function(key: FuncKey) -> bool {+ match key {+ // Wasm functions can both be inlined into other functions and+ // have other functions inlined into them.+ FuncKey::DefinedWasmFunction(..) => true,++ // Intrinsics can be inlined into other functions.+ #[cfg(feature = "component-model")]+ FuncKey::UnsafeIntrinsic(..) => true,++ // Trampolines cannot participate in inlining since our+ // unwinding and exceptions infrastructure relies on them being+ // in their own call frames.+ FuncKey::ArrayToWasmTrampoline(..)+ | FuncKey::WasmToArrayTrampoline(..)+ | FuncKey::WasmToBuiltinTrampoline(..)+ | FuncKey::PatchableToBuiltinTrampoline(..) => false,+ #[cfg(feature = "component-model")]+ FuncKey::ComponentTrampoline(..) | FuncKey::ResourceDropTrampoline => false,++ FuncKey::PulleyHostCall(_) => {+ unreachable!("we don't compile artifacts for Pulley host calls")+ }+ }+}++/// Get just the output indices of the functions that can participate in+/// inlining from our unlinked outputs.+fn inlining_functions<'a>(+ outputs: &'a PrimaryMap<OutputIndex, Option<CompileOutput<'_>>>,+) -> impl Iterator<Item = OutputIndex> + 'a {+ outputs.iter().filter_map(|(index, output)| {+ if is_inlining_function(output.as_ref().unwrap().key) {+ Some(index)+ } else {+ None+ }+ })+}+fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutput>) -> Result<()> {let compiler = engine.try_compiler()?;let mut builtins = HashSet::new();diff --git a/crates/wasmtime/src/compile/call_graph.rs b/crates/wasmtime/src/compile/call_graph.rsdeleted file mode 100644index 4ce62245d4..0000000000--- a/crates/wasmtime/src/compile/call_graph.rs+++ /dev/null@@ -1,105 +0,0 @@-//! Construction of the call-graph, for the purposes of inlining.-//!-//! These call graphs are not necessarily complete or accurate, and Wasmtime's-//! soundness does not rely on those properties. First off, we do not attempt to-//! understand indirect calls, which at their worst must force any call analysis-//! give up and say "the callee could be absolutely any function". More-//! interestingly, these call graphs are only used for scheduling bottom-up-//! inlining, so the worst that inaccurate information can do is cause us to-//! miss inlining opportunities or lose potential parallelism in our-//! schedule. For best results, however, every direct call that is potentially-//! inlinable should be reported when constructing these call graphs.--use super::*;-use core::{- fmt::{self, Debug},- ops::Range,-};-use wasmtime_environ::{EntityRef, SecondaryMap};--/// A call graph reified into a densely packed and quickly accessible-/// representation.-///-/// In a call graph, nodes are functions, and an edge `f --> g` means that the-/// function `f` calls the function `g`.-pub struct CallGraph<Node>-where- Node: EntityRef + Debug,-{- /// A map from each node to the subslice of `self.edge_elems` that are its- /// edges.- edges: SecondaryMap<Node, Range<u32>>,-- /// Densely packed edge elements for `self.edges`.- edge_elems: Vec<Node>,-}--impl<Node> Debug for CallGraph<Node>-where- Node: EntityRef + Debug,-{- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- struct Edges<'a, Node: EntityRef + Debug>(&'a CallGraph<Node>);-- impl<'a, Node: EntityRef + Debug> Debug for Edges<'a, Node> {- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- f.debug_map()- .entries(self.0.nodes().map(|n| (n, self.0.edges(n))))- .finish()- }- }-- f.debug_struct("CallGraph")- .field("edges", &Edges(self))- .finish()- }-}--impl<Node> CallGraph<Node>-where- Node: EntityRef + Debug,-{- /// Construct a new call graph.- ///- /// `funcs` should be an iterator over all function nodes in this call- /// graph's translation unit.- ///- /// The `get_calls` function should yield (by pushing onto the given `Vec`)- /// all of the callee function nodes that the given caller function node- /// calls.- pub fn new(- funcs: impl IntoIterator<Item = Node>,- mut get_calls: impl FnMut(Node, &mut Vec<Node>) -> Result<()>,- ) -> Result<Self> {- let funcs = funcs.into_iter();-- let (min, max) = funcs.size_hint();- let capacity = max.unwrap_or_else(|| 2 * min);- let mut edges = SecondaryMap::with_capacity(capacity);- let mut edge_elems = vec![];-- let mut calls = vec![];- for caller in funcs {- debug_assert!(calls.is_empty());- get_calls(caller, &mut calls)?;-- debug_assert_eq!(edges[caller], Range::default());- edges[caller] = extend_with_range(&mut edge_elems, calls.drain(..));- }-- Ok(CallGraph { edges, edge_elems })- }-- /// Get the function nodes in this call graph.- pub fn nodes(&self) -> impl ExactSizeIterator<Item = Node> {- self.edges.keys()- }-- /// Get the callee function nodes that the given caller function node calls.- pub fn edges(&self, node: Node) -> &[Node] {- let Range { start, end } = self.edges[node].clone();- let start = usize::try_from(start).unwrap();- let end = usize::try_from(end).unwrap();- &self.edge_elems[start..end]- }-}diff --git a/crates/wasmtime/src/compile/stratify.rs b/crates/wasmtime/src/compile/stratify.rsindex 888f6635f8..028d985482 100644--- a/crates/wasmtime/src/compile/stratify.rs+++ b/crates/wasmtime/src/compile/stratify.rs@@ -50,13 +50,12 @@//! dependencies each component has, and a component is ready for inclusion in a//! layer once its unprocessed-dependencies count reaches zero.-use super::{- call_graph::CallGraph,- scc::{Scc, StronglyConnectedComponents},- *,-};+use super::*;use std::{fmt::Debug, ops::Range};-use wasmtime_environ::{EntityRef, SecondaryMap};+use wasmtime_environ::{+ EntityRef, SecondaryMap,+ graphs::{Graph, Scc, StronglyConnectedComponents},+};/// A stratified call graph; essentially a parallel-execution plan for bottom-up/// inlining.@@ -90,15 +89,15 @@ impl<Node: Debug> Debug for Strata<Node> {impl<Node> Strata<Node> {/// Stratify the given call graph, yielding a `Strata` parallel-execution/// plan.- pub fn new(nodes: impl IntoIterator<Item = Node>, call_graph: &CallGraph<Node>) -> Self+ pub fn new<G>(call_graph: &G) -> SelfwhereNode: EntityRef + Debug,+ G: Debug + Graph<Node>,{log::trace!("Stratifying {call_graph:#?}");- let components =- StronglyConnectedComponents::new(nodes, |node| call_graph.edges(node).iter().copied());- let evaporation = components.evaporation(|node| call_graph.edges(node).iter().copied());+ let components = StronglyConnectedComponents::new(call_graph);+ let evaporation = components.evaporation(call_graph);// A map from each component to the count of how many call-graph// dependencies to other components it has that have not been fulfilled@@ -122,14 +121,16 @@ impl<Node> Strata<Node> {// fixed point, similarly to GC tracing and ref-count decrementing.let mut layers: Vec<Range<u32>> = vec![];- let mut layer_elems: Vec<Node> = Vec::with_capacity(call_graph.nodes().len());+ let (min, max) = call_graph.nodes().size_hint();+ let cap = max.unwrap_or(min);+ let mut layer_elems: Vec<Node> = Vec::with_capacity(cap);let mut current_layer: Vec<Scc> = components.keys().filter(|scc| unfulfilled_deps_count[*scc] == 0).collect();debug_assert!(- !current_layer.is_empty() || call_graph.nodes().len() == 0,+ !current_layer.is_empty() || call_graph.nodes().next().is_none(),"the first layer can only be empty when the call graph itself is empty");@@ -187,11 +188,13 @@ impl<Node> Strata<Node> {#[cfg(test)]mod tests {use super::*;+ use wasmtime_environ::graphs::Graph;#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]struct Function(u32);wasmtime_environ::entity_impl!(Function);+ #[derive(Debug)]struct Functions {calls: SecondaryMap<Function, Vec<Function>>,}@@ -205,6 +208,26 @@ mod tests {}}+ impl Graph<Function> for Functions {+ type NodesIter<'a>+ = wasmtime_environ::Keys<Function>+ where+ Self: 'a;++ fn nodes(&self) -> Self::NodesIter<'_> {+ self.calls.keys()+ }++ type SuccessorsIter<'a>+ = core::iter::Copied<core::slice::Iter<'a, Function>>+ where+ Self: 'a;++ fn successors(&self, f: Function) -> Self::SuccessorsIter<'_> {+ self.calls[f].iter().copied()+ }+ }+impl Functions {fn define_func(&mut self, f: u32) -> &mut Self {let f = Function::from_u32(f);@@ -235,12 +258,7 @@ mod tests {}fn stratify(&self) -> Strata<Function> {- let call_graph = CallGraph::new(self.calls.keys(), |f, calls| {- calls.extend_from_slice(&self.calls[f]);- Ok(())- })- .unwrap();- Strata::<Function>::new(self.calls.keys(), &call_graph)+ Strata::new(self)}fn assert_stratification(&self, mut expected: Vec<Vec<u32>>) {