A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Remove `AllCallFunc` (#11694)
Nick Fitzgerald5245e1f88d8f
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex 7bc5c7725d..15dc427f83 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -30,7 +30,7 @@ use std::sync::{Arc, Mutex};use wasmparser::{FuncValidatorAllocations, FunctionBody};use wasmtime_environ::obj::ELF_WASMTIME_EXCEPTIONS;use wasmtime_environ::{- AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,+ Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,DefinedFuncIndex, FlagValue, FuncKey, FunctionBodyData, FunctionLoc, HostCall,InliningCompiler, ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapSection,StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, VMOffsets,@@ -48,19 +48,6 @@ struct IncrementalCacheContext {num_cached: usize,}-/// ABI signature of functions that are generated here.-#[derive(Debug, Copy, Clone)]-#[cfg_attr(- not(feature = "component-model"),- expect(dead_code, reason = "only used with component model compiler")-)]-enum Abi {- /// The "wasm" ABI, or suitable to be a `wasm_call` field of a `VMFuncRef`.- Wasm,- /// The "array" ABI, or suitable to be an `array_call` field.- Array,-}-struct CompilerContext {func_translator: FuncTranslator,codegen_context: Context,@@ -246,7 +233,7 @@ impl wasmtime_environ::Compiler for Compiler {let module = &translation.module;let (module_index, def_func_index) = key.unwrap_defined_wasm_function();- debug_assert_eq!(translation.module_index, module_index);+ debug_assert_eq!(translation.module_index(), module_index);let func_index = module.func_index(def_func_index);let sig = translation.module.functions[func_index]@@ -359,7 +346,7 @@ impl wasmtime_environ::Compiler for Compiler {log::trace!("compiling array-to-wasm trampoline: {key:?} = {symbol:?}");let (module_index, def_func_index) = key.unwrap_array_to_wasm_trampoline();- debug_assert_eq!(translation.module_index, module_index);+ debug_assert_eq!(translation.module_index(), module_index);let func_index = translation.module.func_index(def_func_index);let sig = translation.module.functions[func_index]diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex fb6b6eb636..50df4c20eb 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -1,20 +1,16 @@//! Compilation support for the component model.-use crate::{- TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT,- compiler::{Abi, Compiler},-};+use crate::{TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT, compiler::Compiler};use anyhow::Result;use cranelift_codegen::ir::condcodes::IntCC;use cranelift_codegen::ir::{self, InstBuilder, MemFlags, Value};use cranelift_codegen::isa::{CallConv, TargetIsa};use cranelift_frontend::FunctionBuilder;-use wasmtime_environ::{CompiledFunctionBody, component::*};use wasmtime_environ::{- EntityRef, HostCall, ModuleInternedTypeIndex, PtrSize, TrapSentinel, Tunables, WasmFuncType,- WasmValType,+ Abi, CompiledFunctionBody, EntityRef, FuncKey, HostCall, ModuleInternedTypeIndex, PtrSize,+ TrapSentinel, Tunables, WasmFuncType, WasmValType, component::*,+ fact::PREPARE_CALL_FIXED_PARAMS,};-use wasmtime_environ::{FuncKey, fact::PREPARE_CALL_FIXED_PARAMS};struct TrampolineCompiler<'a> {compiler: &'a Compiler,@@ -1392,61 +1388,58 @@ impl ComponentCompiler for Compiler {component: &ComponentTranslation,types: &ComponentTypesBuilder,key: FuncKey,+ abi: Abi,tunables: &Tunables,_symbol: &str,- ) -> Result<AllCallFunc<CompiledFunctionBody>> {- let compile = |abi: Abi| -> Result<_> {- let mut compiler = self.function_compiler();- let mut c = TrampolineCompiler::new(- self,- &mut compiler,- &component.component,- types,- key.unwrap_component_trampoline(),- abi,- tunables,- );+ ) -> Result<CompiledFunctionBody> {+ let (abi2, trampoline_index) = key.unwrap_component_trampoline();+ debug_assert_eq!(abi, abi2);++ let mut compiler = self.function_compiler();+ let mut c = TrampolineCompiler::new(+ self,+ &mut compiler,+ &component.component,+ types,+ trampoline_index,+ abi,+ tunables,+ );- // If we are crossing the Wasm-to-native boundary, we need to save the- // exit FP and return address for stack walking purposes. However, we- // always debug assert that our vmctx is a component context, regardless- // whether we are actually crossing that boundary because it should- // always hold.- let vmctx = c.builder.block_params(c.block0)[0];- let pointer_type = self.isa.pointer_type();- self.debug_assert_vmctx_kind(- &mut c.builder,+ // If we are crossing the Wasm-to-native boundary, we need to save the+ // exit FP and return address for stack walking purposes. However, we+ // always debug assert that our vmctx is a component context, regardless+ // whether we are actually crossing that boundary because it should+ // always hold.+ let vmctx = c.builder.block_params(c.block0)[0];+ let pointer_type = self.isa.pointer_type();+ self.debug_assert_vmctx_kind(+ &mut c.builder,+ vmctx,+ wasmtime_environ::component::VMCOMPONENT_MAGIC,+ );+ if let Abi::Wasm = abi {+ let vm_store_context = c.builder.ins().load(+ pointer_type,+ MemFlags::trusted(),vmctx,- wasmtime_environ::component::VMCOMPONENT_MAGIC,+ i32::try_from(c.offsets.vm_store_context()).unwrap(),);- if let Abi::Wasm = abi {- let vm_store_context = c.builder.ins().load(- pointer_type,- MemFlags::trusted(),- vmctx,- i32::try_from(c.offsets.vm_store_context()).unwrap(),- );- super::save_last_wasm_exit_fp_and_pc(- &mut c.builder,- pointer_type,- &c.offsets.ptr,- vm_store_context,- );- }-- c.translate(&component.trampolines[key.unwrap_component_trampoline()]);- c.builder.finalize();- compiler.cx.abi = Some(abi);+ super::save_last_wasm_exit_fp_and_pc(+ &mut c.builder,+ pointer_type,+ &c.offsets.ptr,+ vm_store_context,+ );+ }- Ok(CompiledFunctionBody {- code: super::box_dyn_any_compiler_context(Some(compiler.cx)),- needs_gc_heap: false,- })- };+ c.translate(&component.trampolines[trampoline_index]);+ c.builder.finalize();+ compiler.cx.abi = Some(abi);- Ok(AllCallFunc {- wasm_call: compile(Abi::Wasm)?,- array_call: compile(Abi::Array)?,+ Ok(CompiledFunctionBody {+ code: super::box_dyn_any_compiler_context(Some(compiler.cx)),+ needs_gc_heap: false,})}}diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 877ecc14ad..1ba0a3c9b4 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -1290,7 +1290,7 @@ impl FuncEnvironment<'_> {.unwrap_module_type_index();let signature = self.get_or_create_interned_sig_ref(func, ty);- let key = FuncKey::DefinedWasmFunction(self.translation.module_index, def_func_index);+ let key = FuncKey::DefinedWasmFunction(self.translation.module_index(), def_func_index);let (namespace, index) = key.into_raw_parts();let name = ir::ExternalName::User(func.declare_imported_user_function(ir::UserExternalName { namespace, index }),diff --git a/crates/environ/src/compile/key.rs b/crates/environ/src/compile/key.rsindex 140bd08c77..58c43ecb4b 100644--- a/crates/environ/src/compile/key.rs+++ b/crates/environ/src/compile/key.rs@@ -6,10 +6,178 @@ use crate::component;use crate::{BuiltinFunctionIndex, DefinedFuncIndex, HostCall, ModuleInternedTypeIndex, StaticModuleIndex,};+use core::cmp;+use serde_derive::{Deserialize, Serialize};++/// The kind of a function that is being compiled, linked, or otherwise+/// referenced.+///+/// This is like a `FuncKey` but without any payload values.+#[repr(u32)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+pub enum FuncKeyKind {+ /// A Wasm-defined function.+ DefinedWasmFunction = FuncKey::new_kind(0b000),++ /// A trampoline from an array-caller to the given Wasm-callee.+ ArrayToWasmTrampoline = FuncKey::new_kind(0b001),++ /// A trampoline from a Wasm-caller to an array-callee of the given type.+ WasmToArrayTrampoline = FuncKey::new_kind(0b010),++ /// A trampoline from a Wasm-caller to the given builtin.+ WasmToBuiltinTrampoline = FuncKey::new_kind(0b011),++ /// A Pulley-specific host call.+ PulleyHostCall = FuncKey::new_kind(0b100),++ /// A Wasm-caller to component builtin trampoline.+ #[cfg(feature = "component-model")]+ ComponentTrampoline = FuncKey::new_kind(0b101),++ /// A Wasm-caller to array-callee `resource.drop` trampoline.+ #[cfg(feature = "component-model")]+ ResourceDropTrampoline = FuncKey::new_kind(0b110),+}++impl From<FuncKeyKind> for u32 {+ fn from(kind: FuncKeyKind) -> Self {+ kind as u32+ }+}++impl FuncKeyKind {+ /// Get this kind's raw representation.+ pub fn into_raw(self) -> u32 {+ self.into()+ }++ /// Construct a `FuncKind` from its raw representation.+ ///+ /// Panics when given invalid raw representations.+ pub fn from_raw(raw: u32) -> Self {+ match raw {+ x if x == Self::DefinedWasmFunction.into() => Self::DefinedWasmFunction,+ x if x == Self::ArrayToWasmTrampoline.into() => Self::ArrayToWasmTrampoline,+ x if x == Self::WasmToArrayTrampoline.into() => Self::WasmToArrayTrampoline,+ x if x == Self::WasmToBuiltinTrampoline.into() => Self::WasmToBuiltinTrampoline,+ x if x == Self::PulleyHostCall.into() => Self::PulleyHostCall,++ #[cfg(feature = "component-model")]+ x if x == Self::ComponentTrampoline.into() => Self::ComponentTrampoline,+ #[cfg(feature = "component-model")]+ x if x == Self::ResourceDropTrampoline.into() => Self::ResourceDropTrampoline,++ _ => panic!("invalid raw value passed to `FuncKind::from_raw`: {raw}"),+ }+ }+}++/// The namespace half of a `FuncKey`.+///+/// This is an opaque combination of the key's kind and module index, if any.+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+pub struct FuncKeyNamespace(u32);++impl From<FuncKeyNamespace> for u32 {+ fn from(ns: FuncKeyNamespace) -> Self {+ ns.0+ }+}++impl FuncKeyNamespace {+ /// Get this `FuncNamespace`'s raw representation.+ pub fn into_raw(self) -> u32 {+ self.0+ }++ /// Construct a `FuncNamespace` from its raw representation.+ ///+ /// Panics when given invalid raw representations.+ pub fn from_raw(raw: u32) -> Self {+ match FuncKeyKind::from_raw(raw & FuncKey::KIND_MASK) {+ FuncKeyKind::DefinedWasmFunction | FuncKeyKind::ArrayToWasmTrampoline => Self(raw),+ FuncKeyKind::WasmToArrayTrampoline+ | FuncKeyKind::WasmToBuiltinTrampoline+ | FuncKeyKind::PulleyHostCall => {+ assert_eq!(raw & FuncKey::MODULE_MASK, 0);+ Self(raw)+ }++ #[cfg(feature = "component-model")]+ FuncKeyKind::ComponentTrampoline => {+ let _ = Abi::from_raw(raw & FuncKey::MODULE_MASK);+ Self(raw)+ }++ #[cfg(feature = "component-model")]+ FuncKeyKind::ResourceDropTrampoline => {+ assert_eq!(raw & FuncKey::MODULE_MASK, 0);+ Self(raw)+ }+ }+ }++ /// Get this `FuncNamespace`'s kind.+ pub fn kind(&self) -> FuncKeyKind {+ let raw = self.0 & FuncKey::KIND_MASK;+ FuncKeyKind::from_raw(raw)+ }+}++/// The index half of a `FuncKey`.+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+pub struct FuncKeyIndex(u32);++impl From<FuncKeyIndex> for u32 {+ fn from(index: FuncKeyIndex) -> Self {+ index.0+ }+}++impl FuncKeyIndex {+ /// Get this index's raw representation.+ pub fn into_raw(self) -> u32 {+ self.0+ }++ /// Construct a `FuncKeyIndex` from its raw representation.+ ///+ /// Invalid raw representations will not be caught eagerly, but will cause+ /// panics when paired with a `FuncKeyNamespace` to create a whole+ /// `FuncKey`.+ pub fn from_raw(raw: u32) -> Self {+ FuncKeyIndex(raw)+ }+}++/// ABI signature of functions that are generated here.+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+pub enum Abi {+ /// The "wasm" ABI, or suitable to be a `wasm_call` field of a `VMFuncRef`.+ Wasm = 0,+ /// The "array" ABI, or suitable to be an `array_call` field.+ Array = 1,+}++#[cfg(feature = "component-model")]+impl Abi {+ fn from_raw(raw: u32) -> Self {+ match raw {+ x if x == Self::Wasm.into_raw() => Self::Wasm,+ x if x == Self::Array.into_raw() => Self::Array,+ _ => panic!("invalid raw representation passed to `Abi::from_raw`: {raw}"),+ }+ }++ fn into_raw(self) -> u32 {+ (self as u8).into()+ }+}/// A sortable, comparable function key for compilation output, call graph/// edges, and relocations.-#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]pub enum FuncKey {/// A Wasm-defined function.DefinedWasmFunction(StaticModuleIndex, DefinedFuncIndex),@@ -28,13 +196,29 @@ pub enum FuncKey {/// A Wasm-caller to component builtin trampoline.#[cfg(feature = "component-model")]- ComponentTrampoline(component::TrampolineIndex),+ ComponentTrampoline(Abi, component::TrampolineIndex),/// A Wasm-caller to array-callee `resource.drop` trampoline.#[cfg(feature = "component-model")]ResourceDropTrampoline,}+impl Ord for FuncKey {+ fn cmp(&self, other: &Self) -> cmp::Ordering {+ // Make sure to sort by our raw parts, because `CompiledFunctionsTable`+ // relies on this for its binary search tables.+ let raw_self = self.into_raw_parts();+ let raw_other = other.into_raw_parts();+ raw_self.cmp(&raw_other)+ }+}++impl PartialOrd for FuncKey {+ fn partial_cmp(&self, other: &Self) -> Option<cmp::Ordering> {+ Some(self.cmp(other))+ }+}+impl FuncKey {const KIND_BITS: u32 = 3;const KIND_OFFSET: u32 = 32 - Self::KIND_BITS;@@ -46,118 +230,142 @@ impl FuncKey {kind << Self::KIND_OFFSET}- const DEFINED_WASM_FUNCTION_KIND: u32 = Self::new_kind(0);- const ARRAY_TO_WASM_TRAMPOLINE_KIND: u32 = Self::new_kind(1);- const WASM_TO_ARRAY_TRAMPOLINE_KIND: u32 = Self::new_kind(2);- const WASM_TO_BUILTIN_TRAMPOLINE_KIND: u32 = Self::new_kind(3);- const PULLEY_HOST_CALL_KIND: u32 = Self::new_kind(4);+ /// Get this key's kind.+ pub fn kind(self) -> FuncKeyKind {+ self.namespace().kind()+ }- #[cfg(feature = "component-model")]- const COMPONENT_TRAMPOLINE_KIND: u32 = Self::new_kind(5);- #[cfg(feature = "component-model")]- const RESOURCE_DROP_TRAMPOLINE_KIND: u32 = Self::new_kind(6);+ /// Get this key's namespace.+ pub fn namespace(self) -> FuncKeyNamespace {+ self.into_parts().0+ }- /// Get the raw, underlying representation of this compilation key.- ///- /// The resulting values should only be used for (eventually) calling- /// `CompileKey::from_raw_parts`.- //- // NB: We use two `u32`s to exactly match- // `cranelift_codegen::ir::UserExternalName` and ensure that we can map- // one-to-one between that and `FuncKey`.- pub fn into_raw_parts(self) -> (u32, u32) {- match self {+ /// Get this key's index.+ pub fn index(self) -> FuncKeyIndex {+ self.into_parts().1+ }++ /// Split this key into its namespace and index halves.+ pub fn into_parts(self) -> (FuncKeyNamespace, FuncKeyIndex) {+ let (namespace, index) = match self {FuncKey::DefinedWasmFunction(module, def_func) => {assert_eq!(module.as_u32() & Self::KIND_MASK, 0);- let namespace = Self::DEFINED_WASM_FUNCTION_KIND | module.as_u32();+ let namespace = FuncKeyKind::DefinedWasmFunction.into_raw() | module.as_u32();let index = def_func.as_u32();(namespace, index)}FuncKey::ArrayToWasmTrampoline(module, def_func) => {assert_eq!(module.as_u32() & Self::KIND_MASK, 0);- let namespace = Self::ARRAY_TO_WASM_TRAMPOLINE_KIND | module.as_u32();+ let namespace = FuncKeyKind::ArrayToWasmTrampoline.into_raw() | module.as_u32();let index = def_func.as_u32();(namespace, index)}FuncKey::WasmToArrayTrampoline(ty) => {- let namespace = Self::WASM_TO_ARRAY_TRAMPOLINE_KIND;+ let namespace = FuncKeyKind::WasmToArrayTrampoline.into_raw();let index = ty.as_u32();(namespace, index)}FuncKey::WasmToBuiltinTrampoline(builtin) => {- let namespace = Self::WASM_TO_BUILTIN_TRAMPOLINE_KIND;+ let namespace = FuncKeyKind::WasmToBuiltinTrampoline.into_raw();let index = builtin.index();(namespace, index)}FuncKey::PulleyHostCall(host_call) => {- let namespace = Self::PULLEY_HOST_CALL_KIND;+ let namespace = FuncKeyKind::PulleyHostCall.into_raw();let index = host_call.index();(namespace, index)}#[cfg(feature = "component-model")]- FuncKey::ComponentTrampoline(trampoline) => {- let namespace = Self::COMPONENT_TRAMPOLINE_KIND;+ FuncKey::ComponentTrampoline(abi, trampoline) => {+ let abi = abi.into_raw();+ assert_eq!(abi & Self::KIND_MASK, 0);+ let namespace = FuncKeyKind::ComponentTrampoline.into_raw() | abi;let index = trampoline.as_u32();(namespace, index)}#[cfg(feature = "component-model")]FuncKey::ResourceDropTrampoline => {- let namespace = Self::RESOURCE_DROP_TRAMPOLINE_KIND;+ let namespace = FuncKeyKind::ResourceDropTrampoline.into_raw();let index = 0;(namespace, index)}- }+ };+ (FuncKeyNamespace(namespace), FuncKeyIndex(index))}- /// Create a compilation key from its raw, underlying representation.+ /// Create a key from its namespace and index parts.+ ///+ /// Should only be called with namespaces and indices that are ultimately+ /// derived from the same key. For example, if you attempt to pair an index+ /// and namespace that come from different keys, that may panic. If it+ /// happens not to panic, you'll end up with a valid key that names an+ /// arbitrary function in the given namespace, but that function probably+ /// does not actually exist in the compilation artifact.+ pub fn from_parts(namespace: FuncKeyNamespace, index: FuncKeyIndex) -> Self {+ Self::from_raw_parts(namespace.into_raw(), index.into_raw())+ }++ /// Get the raw, underlying `(namespace, index)` representation of this+ /// compilation key.+ ///+ /// The resulting values should only be used for (eventually) calling+ /// `FuncKey::from_raw_parts` or `FuncKey{Namespace,Index}::from_raw`.+ //+ // NB: We use two `u32`s to exactly match+ // `cranelift_codegen::ir::UserExternalName` and ensure that we can map+ // one-to-one between that and `FuncKey`.+ pub fn into_raw_parts(self) -> (u32, u32) {+ let (ns, index) = self.into_parts();+ (ns.into_raw(), index.into_raw())+ }++ /// Create a key from its raw, underlying representation.////// Should only be given the results of a previous call to- /// `CompileKey::into_raw_parts`.+ /// `FuncKey::into_raw_parts`.+ ///+ /// Panics when given invalid raw parts.pub fn from_raw_parts(a: u32, b: u32) -> Self {- match a & Self::KIND_MASK {- Self::DEFINED_WASM_FUNCTION_KIND => {+ match FuncKeyKind::from_raw(a & Self::KIND_MASK) {+ FuncKeyKind::DefinedWasmFunction => {let module = StaticModuleIndex::from_u32(a & Self::MODULE_MASK);let def_func = DefinedFuncIndex::from_u32(b);Self::DefinedWasmFunction(module, def_func)}- Self::ARRAY_TO_WASM_TRAMPOLINE_KIND => {+ FuncKeyKind::ArrayToWasmTrampoline => {let module = StaticModuleIndex::from_u32(a & Self::MODULE_MASK);let def_func = DefinedFuncIndex::from_u32(b);Self::ArrayToWasmTrampoline(module, def_func)}- Self::WASM_TO_ARRAY_TRAMPOLINE_KIND => {+ FuncKeyKind::WasmToArrayTrampoline => {assert_eq!(a & Self::MODULE_MASK, 0);let ty = ModuleInternedTypeIndex::from_u32(b);Self::WasmToArrayTrampoline(ty)}- Self::WASM_TO_BUILTIN_TRAMPOLINE_KIND => {+ FuncKeyKind::WasmToBuiltinTrampoline => {assert_eq!(a & Self::MODULE_MASK, 0);let builtin = BuiltinFunctionIndex::from_u32(b);Self::WasmToBuiltinTrampoline(builtin)}- Self::PULLEY_HOST_CALL_KIND => {+ FuncKeyKind::PulleyHostCall => {assert_eq!(a & Self::MODULE_MASK, 0);let host_call = HostCall::from_index(b);Self::PulleyHostCall(host_call)}#[cfg(feature = "component-model")]- Self::COMPONENT_TRAMPOLINE_KIND => {- assert_eq!(a & Self::MODULE_MASK, 0);+ FuncKeyKind::ComponentTrampoline => {+ let abi = Abi::from_raw(a & Self::MODULE_MASK);let trampoline = component::TrampolineIndex::from_u32(b);- Self::ComponentTrampoline(trampoline)+ Self::ComponentTrampoline(abi, trampoline)}#[cfg(feature = "component-model")]- Self::RESOURCE_DROP_TRAMPOLINE_KIND => {+ FuncKeyKind::ResourceDropTrampoline => {assert_eq!(a & Self::MODULE_MASK, 0);assert_eq!(b, 0);Self::ResourceDropTrampoline}-- k => panic!(- "bad raw parts given to `FuncKey::from_raw_parts` call: ({a}, {b}), kind would be {k}"- ),}}@@ -203,9 +411,9 @@ impl FuncKey {/// Unwrap a `FuncKey::ComponentTrampoline` or else panic.#[cfg(feature = "component-model")]- pub fn unwrap_component_trampoline(self) -> component::TrampolineIndex {+ pub fn unwrap_component_trampoline(self) -> (crate::Abi, component::TrampolineIndex) {match self {- Self::ComponentTrampoline(trampoline) => trampoline,+ Self::ComponentTrampoline(abi, trampoline) => (abi, trampoline),_ => panic!("`FuncKey::unwrap_component_trampoline` called on {self:?}"),}}diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rsindex 4fc468eadd..82f5c0eb50 100644--- a/crates/environ/src/compile/module_environ.rs+++ b/crates/environ/src/compile/module_environ.rs@@ -45,9 +45,6 @@ pub struct ModuleTranslation<'data> {/// Module information.pub module: Module,- /// This module's index.- pub module_index: StaticModuleIndex,-/// The input wasm binary.////// This can be useful, for example, when modules are parsed from a@@ -115,8 +112,7 @@ impl<'data> ModuleTranslation<'data> {/// Create a new translation for the module with the given index.pub fn new(module_index: StaticModuleIndex) -> Self {Self {- module_index,- module: Module::default(),+ module: Module::new(module_index),wasm: &[],function_body_inputs: PrimaryMap::default(),known_imported_functions: SecondaryMap::default(),@@ -139,6 +135,11 @@ impl<'data> ModuleTranslation<'data> {.as_ref().expect("module type information to be available")}++ /// Get this translation's module's index.+ pub fn module_index(&self) -> StaticModuleIndex {+ self.module.module_index+ }}/// Contains function data: byte code and its offset in the module.diff --git a/crates/environ/src/component/artifacts.rs b/crates/environ/src/component/artifacts.rsindex 096ea839be..2aabb21ef3 100644--- a/crates/environ/src/component/artifacts.rs+++ b/crates/environ/src/component/artifacts.rs@@ -26,43 +26,15 @@ pub struct CompiledComponentInfo {/// Type information calculated during translation about this component.pub component: Component,- /// Where lowered function trampolines are located within the `text`- /// section of `code_memory`.- ///- /// These are the- ///- /// 1. Wasm-call,- /// 2. array-call- ///- /// function pointers that end up in a `VMFuncRef` for each- /// lowering.- pub trampolines: PrimaryMap<TrampolineIndex, AllCallFunc<FunctionLoc>>,+ /// Where lowered function wasm-call trampolines that end up in a+ /// `VMFuncRef` are located within the `text` section of `code_memory`.+ pub wasm_call_trampolines: PrimaryMap<TrampolineIndex, FunctionLoc>,++ /// Where lowered function array-call trampolines that end up in a+ /// `VMFuncRef` are located within the `text` section of `code_memory`.+ pub array_call_trampolines: PrimaryMap<TrampolineIndex, FunctionLoc>,/// The location of the wasm-to-array trampoline for the `resource.drop`/// intrinsic.pub resource_drop_wasm_to_array_trampoline: Option<FunctionLoc>,}--/// A triple of related functions/trampolines variants with differing calling-/// conventions: `{wasm,array}_call`.-///-/// Generic so we can use this with either the `Box<dyn Any + Send>`s that-/// implementations of the compiler trait return or with `FunctionLoc`s inside-/// an object file, for example.-#[derive(Clone, Copy, Default, Serialize, Deserialize)]-pub struct AllCallFunc<T> {- /// The function exposing the Wasm calling convention.- pub wasm_call: T,- /// The function exposing the array calling convention.- pub array_call: T,-}--impl<T> AllCallFunc<T> {- /// Map an `AllCallFunc<T>` into an `AllCallFunc<U>`.- pub fn map<U>(self, mut f: impl FnMut(T) -> U) -> AllCallFunc<U> {- AllCallFunc {- wasm_call: f(self.wasm_call),- array_call: f(self.array_call),- }- }-}diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rsindex 1f0658bad9..ccfdab4566 100644--- a/crates/environ/src/component/compiler.rs+++ b/crates/environ/src/component/compiler.rs@@ -1,5 +1,5 @@-use crate::component::{AllCallFunc, ComponentTranslation, ComponentTypesBuilder};-use crate::{CompiledFunctionBody, FuncKey, Tunables};+use crate::component::{ComponentTranslation, ComponentTypesBuilder};+use crate::{Abi, CompiledFunctionBody, FuncKey, Tunables};use anyhow::Result;/// Compilation support necessary for components.@@ -15,7 +15,8 @@ pub trait ComponentCompiler: Send + Sync {component: &ComponentTranslation,types: &ComponentTypesBuilder,key: FuncKey,+ abi: Abi,tunables: &Tunables,symbol: &str,- ) -> Result<AllCallFunc<CompiledFunctionBody>>;+ ) -> Result<CompiledFunctionBody>;}diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rsindex 7d31c47c9e..8284b419ef 100644--- a/crates/environ/src/module.rs+++ b/crates/environ/src/module.rs@@ -289,8 +289,11 @@ impl TableSegmentElements {/// A translated WebAssembly module, excluding the function bodies and/// memory initializers.-#[derive(Default, Debug, Serialize, Deserialize)]+#[derive(Debug, Serialize, Deserialize)]pub struct Module {+ /// This module's index.+ pub module_index: StaticModuleIndex,+/// The name of this wasm module, often found in the wasm file.pub name: Option<String>,@@ -383,8 +386,33 @@ pub enum Initializer {impl Module {/// Allocates the module data structures.- pub fn new() -> Self {- Module::default()+ pub fn new(module_index: StaticModuleIndex) -> Self {+ Self {+ module_index,+ name: Default::default(),+ initializers: Default::default(),+ exports: Default::default(),+ start_func: Default::default(),+ table_initialization: Default::default(),+ memory_initialization: Default::default(),+ passive_elements: Default::default(),+ passive_elements_map: Default::default(),+ passive_data_map: Default::default(),+ types: Default::default(),+ num_imported_funcs: Default::default(),+ num_imported_tables: Default::default(),+ num_imported_memories: Default::default(),+ num_imported_globals: Default::default(),+ num_imported_tags: Default::default(),+ needs_gc_heap: Default::default(),+ num_escaped_funcs: Default::default(),+ functions: Default::default(),+ tables: Default::default(),+ memories: Default::default(),+ globals: Default::default(),+ global_initializers: Default::default(),+ tags: Default::default(),+ }}/// Convert a `DefinedFuncIndex` into a `FuncIndex`.@@ -587,10 +615,16 @@ impl Module {/// Returns an iterator over all of the defined function indices in this/// module.- pub fn defined_func_indices(&self) -> impl Iterator<Item = DefinedFuncIndex> + use<> {+ pub fn defined_func_indices(&self) -> impl ExactSizeIterator<Item = DefinedFuncIndex> + use<> {(0..self.functions.len() - self.num_imported_funcs).map(|i| DefinedFuncIndex::new(i))}+ /// Returns the number of functions defined by this module itself: all+ /// functions minus imported functions.+ pub fn num_defined_funcs(&self) -> usize {+ self.functions.len() - self.num_imported_funcs+ }+/// Returns the number of tables defined by this module itself: all tables/// minus imported tables.pub fn num_defined_tables(&self) -> usize {@@ -624,6 +658,7 @@ impl TypeTrace for Module {// NB: Do not `..` elide unmodified fields so that we get compile errors// when adding new fields that might need re-canonicalization.let Self {+ module_index: _,name: _,initializers: _,exports: _,@@ -674,6 +709,7 @@ impl TypeTrace for Module {// NB: Do not `..` elide unmodified fields so that we get compile errors// when adding new fields that might need re-canonicalization.let Self {+ module_index: _,name: _,initializers: _,exports: _,diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rsindex 229a469e19..87a3252c95 100644--- a/crates/wasmtime/src/compile.rs+++ b/crates/wasmtime/src/compile.rs@@ -35,14 +35,14 @@ use std::{};use call_graph::CallGraph;+#[cfg(feature = "component-model")]+use wasmtime_environ::component::Translator;use wasmtime_environ::{BuiltinFunctionIndex, CompiledFunctionBody, CompiledFunctionInfo, CompiledModuleInfo, Compiler,DefinedFuncIndex, FilePos, FinishedObject, FuncKey, FunctionBodyData, InliningCompiler,IntraModuleInlining, ModuleEnvironment, ModuleTranslation, ModuleTypes, ModuleTypesBuilder,ObjectKind, PrimaryMap, SecondaryMap, StaticModuleIndex, Tunables,};-#[cfg(feature = "component-model")]-use wasmtime_environ::{FunctionLoc, component::Translator};mod call_graph;mod scc;@@ -199,7 +199,8 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(let info = CompiledComponentInfo {component: component.component,- trampolines: compilation_artifacts.trampolines,+ wasm_call_trampolines: compilation_artifacts.wasm_call_trampolines,+ array_call_trampolines: compilation_artifacts.array_call_trampolines,resource_drop_wasm_to_array_trampoline: compilation_artifacts.resource_drop_trampoline,};let artifacts = ComponentArtifacts {@@ -216,58 +217,10 @@ pub(crate) fn build_component_artifacts<T: FinishedObject>(type CompileInput<'a> = Box<dyn FnOnce(&dyn Compiler) -> Result<CompileOutput<'a>> + Send + 'a>;-#[derive(Clone, Copy)]-enum CompiledFunction<T> {- Function(T),- #[cfg(feature = "component-model")]- AllCallFunc(wasmtime_environ::component::AllCallFunc<T>),-}--impl<T> CompiledFunction<T> {- fn as_function(&self) -> Option<&T> {- match self {- Self::Function(f) => Some(f),- #[cfg(feature = "component-model")]- Self::AllCallFunc(_) => None,- }- }-- fn as_function_mut(&mut self) -> Option<&mut T> {- match self {- Self::Function(f) => Some(f),- #[cfg(feature = "component-model")]- Self::AllCallFunc(_) => None,- }- }-- fn unwrap_function(self) -> T {- match self {- Self::Function(f) => f,- #[cfg(feature = "component-model")]- Self::AllCallFunc(_) => panic!("CompiledFunction::unwrap_function"),- }- }-- #[cfg(feature = "component-model")]- fn unwrap_all_call_func(self) -> wasmtime_environ::component::AllCallFunc<T> {- match self {- Self::AllCallFunc(f) => f,- Self::Function(_) => panic!("CompiledFunction::unwrap_all_call_func"),- }- }-}--#[cfg(feature = "component-model")]-impl<T> From<wasmtime_environ::component::AllCallFunc<T>> for CompiledFunction<T> {- fn from(f: wasmtime_environ::component::AllCallFunc<T>) -> Self {- Self::AllCallFunc(f)- }-}-struct CompileOutput<'a> {key: FuncKey,symbol: String,- function: CompiledFunction<CompiledFunctionBody>,+ function: CompiledFunctionBody,start_srcloc: FilePos,// Only present when `self.key` is a `FuncKey::DefinedWasmFunction(..)`.@@ -332,28 +285,36 @@ impl<'a> CompileInputs<'a> {),>,) -> Self {+ use wasmtime_environ::Abi;+let mut ret = CompileInputs { inputs: vec![] };ret.collect_inputs_in_translations(types.module_types_builder(), module_translations);let tunables = engine.tunables();for (idx, trampoline) in component.trampolines.iter() {- ret.push_input(move |compiler| {- let key = FuncKey::ComponentTrampoline(idx);- let symbol = trampoline.symbol_name();- Ok(CompileOutput {- key,- function: compiler+ for abi in [Abi::Wasm, Abi::Array] {+ ret.push_input(move |compiler| {+ let key = FuncKey::ComponentTrampoline(abi, idx);+ let mut symbol = trampoline.symbol_name();+ symbol.push_str(match abi {+ Abi::Wasm => "_wasm_call",+ Abi::Array => "_array_call",+ });+ let function = compiler.component_compiler()- .compile_trampoline(component, types, key, tunables, &symbol)- .with_context(|| format!("failed to compile {symbol}"))?- .into(),- symbol,- start_srcloc: FilePos::default(),- translation: None,- func_body: None,- })- });+ .compile_trampoline(component, types, key, abi, tunables, &symbol)+ .with_context(|| format!("failed to compile {symbol}"))?;+ Ok(CompileOutput {+ key,+ function,+ symbol,+ start_srcloc: FilePos::default(),+ translation: None,+ func_body: None,+ })+ });+ }}// If there are any resources defined within this component, the@@ -372,7 +333,7 @@ impl<'a> CompileInputs<'a> {.with_context(|| format!("failed to compile `{symbol}`"))?;Ok(CompileOutput {key,- function: CompiledFunction::Function(function),+ function,symbol,start_srcloc: FilePos::default(),translation: None,@@ -460,7 +421,7 @@ impl<'a> CompileInputs<'a> {Ok(CompileOutput {key,symbol,- function: CompiledFunction::Function(function),+ function,start_srcloc,translation: Some(translation),func_body: Some(func_body),@@ -477,13 +438,13 @@ impl<'a> CompileInputs<'a> {module.as_u32(),func_index.as_u32());- let trampoline = compiler+ let function = compiler.compile_array_to_wasm_trampoline(translation, types, key, &symbol).with_context(|| format!("failed to compile: {symbol}"))?;Ok(CompileOutput {key,symbol,- function: CompiledFunction::Function(trampoline),+ function,start_srcloc: FilePos::default(),translation: None,func_body: None,@@ -506,12 +467,12 @@ impl<'a> CompileInputs<'a> {"signatures[{}]::wasm_to_array_trampoline",trampoline_type_index.as_u32());- let trampoline = compiler+ let function = compiler.compile_wasm_to_array_trampoline(trampoline_func_ty, key, &symbol).with_context(|| format!("failed to compile: {symbol}"))?;Ok(CompileOutput {key,- function: CompiledFunction::Function(trampoline),+ function,symbol,start_srcloc: FilePos::default(),translation: None,@@ -553,27 +514,11 @@ the use case.// call graph computation and all that.engine.run_maybe_parallel::<_, _, Error, _>(self.inputs, |f| {let mut compiled = f(compiler)?;- match &mut compiled.function {- CompiledFunction::Function(f) => inlining_compiler.finish_compiling(- f,- compiled.func_body.take(),- &compiled.symbol,- )?,- #[cfg(feature = "component-model")]- CompiledFunction::AllCallFunc(f) => {- debug_assert!(compiled.func_body.is_none());- inlining_compiler.finish_compiling(- &mut f.array_call,- None,- &compiled.symbol,- )?;- inlining_compiler.finish_compiling(- &mut f.wasm_call,- None,- &compiled.symbol,- )?;- }- };+ inlining_compiler.finish_compiling(+ &mut compiled.function,+ compiled.func_body.take(),+ &compiled.symbol,+ )?;Ok(compiled)})?}@@ -661,17 +606,9 @@ the use case.let output = outputs[output_index].as_ref().unwrap();debug_assert!(matches!(output.key, FuncKey::DefinedWasmFunction(..)));- let func = match &output.function {- CompiledFunction::Function(f) => f,- #[cfg(feature = "component-model")]- CompiledFunction::AllCallFunc(_) => {- unreachable!("wasm functions are not all-call functions")- }- };-// Get this function's call graph edges as `FuncKey`s.func_keys.clear();- inlining_compiler.calls(func, &mut func_keys)?;+ inlining_compiler.calls(&output.function, &mut func_keys)?;// Translate each of those to keys to output indices, which is// what we actually need.@@ -716,11 +653,7 @@ the use case.let (caller_module, caller_def_func) =output.key.unwrap_defined_wasm_function();let caller_needs_gc_heap = caller_translation.module.needs_gc_heap;-- let caller = output- .function- .as_function_mut()- .expect("wasm functions are not all-call functions");+ let caller = &mut output.function;let mut caller_size = inlining_compiler.size(caller);@@ -737,10 +670,7 @@ the use case.debug_assert_eq!(callee_output.key, callee_key);- let callee = callee_output- .function- .as_function()- .expect("wasm functions are not all-call functions");+ let callee = &callee_output.function;let callee_size = inlining_compiler.size(callee);let callee_needs_gc_heap =@@ -777,19 +707,11 @@ the use case.// Fan out in parallel again and finish compiling each function.engine.run_maybe_parallel(outputs.into(), |output| {let mut output = output.unwrap();- match &mut output.function {- CompiledFunction::Function(f) => inlining_compiler.finish_compiling(- f,- output.func_body.take(),- &output.symbol,- )?,- #[cfg(feature = "component-model")]- CompiledFunction::AllCallFunc(f) => {- debug_assert!(output.func_body.is_none());- inlining_compiler.finish_compiling(&mut f.array_call, None, &output.symbol)?;- inlining_compiler.finish_compiling(&mut f.wasm_call, None, &output.symbol)?;- }- };+ inlining_compiler.finish_compiling(+ &mut output.function,+ output.func_body.take(),+ &output.symbol,+ )?;Ok(output)})}@@ -911,15 +833,15 @@ fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutpuBox::new(move |compiler: &dyn Compiler| {let key = FuncKey::WasmToBuiltinTrampoline(builtin);let symbol = format!("wasmtime_builtin_{}", builtin.name());- let mut trampoline = compiler+ let mut function = compiler.compile_wasm_to_builtin(key, &symbol).with_context(|| format!("failed to compile `{symbol}`"))?;if let Some(compiler) = compiler.inlining_compiler() {- compiler.finish_compiling(&mut trampoline, None, &symbol)?;+ compiler.finish_compiling(&mut function, None, &symbol)?;}Ok(CompileOutput {key,- function: CompiledFunction::Function(trampoline),+ function,symbol,start_srcloc: FilePos::default(),translation: None,@@ -929,12 +851,7 @@ fn compile_required_builtins(engine: &Engine, raw_outputs: &mut Vec<CompileOutpu};for output in raw_outputs.iter() {- let f = match &output.function {- CompiledFunction::Function(f) => f,- #[cfg(feature = "component-model")]- CompiledFunction::AllCallFunc(_) => continue,- };- for reloc in compiler.compiled_function_relocation_targets(&*f.code) {+ for reloc in compiler.compiled_function_relocation_targets(&*output.function.code) {if let FuncKey::WasmToBuiltinTrampoline(builtin) = reloc {if builtins.insert(builtin) {new_inputs.push(compile_builtin(builtin));@@ -956,49 +873,32 @@ impl UnlinkedCompileOutputs<'_> {/// Flatten all our functions into a single list and remember each of their/// indices within it.fn pre_link(self) -> PreLinkOutput {- // The order the functions end up within `compiled_funcs` is the order- // that they will be laid out in the ELF file, so try and group hot and- // cold functions together as best we can. However, because we bucket by- // kind, we shouldn't have any issues with, e.g., cold trampolines- // appearing in between hot Wasm functions.+ // We must ensure that `compiled_funcs` contains the function bodies+ // sorted by their `FuncKey`, as `CompiledFunctionsIndex` relies on that+ // property.+ //+ // Furthermore, note that, because the order functions end up in+ // `compiled_funcs` is the order they will ultimately be laid out inside+ // the object file, we will group all trampolines together, all defined+ // Wasm functions from the same module together, and etc... This is a+ // nice property, because it means that (a) cold functions, like builtin+ // trampolines, are not interspersed between hot Wasm functions, and (b)+ // Wasm functions that are likely to call each other (i.e. are in the+ // same module together) are grouped together.let mut compiled_funcs = vec![];+let mut indices = FunctionIndices::default();let mut needs_gc_heap = false;+ // NB: Iteration over this `BTreeMap` ensures that we uphold+ // `compiled_func`'s sorted property.for output in self.outputs.into_values() {- let index = match output.function {- CompiledFunction::Function(f) => {- needs_gc_heap |= f.needs_gc_heap;- let index = compiled_funcs.len();- compiled_funcs.push((output.symbol, f.code));- CompiledFunction::Function(index)- }- #[cfg(feature = "component-model")]- CompiledFunction::AllCallFunc(wasmtime_environ::component::AllCallFunc {- wasm_call,- array_call,- }) => {- needs_gc_heap |= array_call.needs_gc_heap;- let array_call_idx = compiled_funcs.len();- compiled_funcs.push((format!("{}_array_call", output.symbol), array_call.code));-- needs_gc_heap |= wasm_call.needs_gc_heap;- let wasm_call_idx = compiled_funcs.len();- compiled_funcs.push((format!("{}_wasm_call", output.symbol), wasm_call.code));-- CompiledFunction::AllCallFunc(wasmtime_environ::component::AllCallFunc {- array_call: array_call_idx,- wasm_call: wasm_call_idx,- })- }- };+ needs_gc_heap |= output.function.needs_gc_heap;- if let FuncKey::DefinedWasmFunction(module, _)- | FuncKey::ArrayToWasmTrampoline(module, _) = output.key- {- indices- .compiled_func_index_to_module- .insert(index.unwrap_function(), module);+ let index = compiled_funcs.len();+ compiled_funcs.push((output.symbol, output.function.code));++ if output.start_srcloc != FilePos::default() {indices.start_srclocs.insert(output.key, output.start_srcloc);@@ -1029,15 +929,11 @@ struct PreLinkOutput {#[derive(Default)]struct FunctionIndices {- // A reverse map from an index in `compiled_funcs` to the- // `StaticModuleIndex` for that function.- compiled_func_index_to_module: HashMap<usize, StaticModuleIndex>,-// A map of wasm functions and where they're located in the original file.start_srclocs: HashMap<FuncKey, FilePos>,- // The index of each compiled function.- indices: BTreeMap<FuncKey, CompiledFunction<usize>>,+ // The index of each compiled function in `compiled_funcs`.+ indices: BTreeMap<FuncKey, usize>,}impl FunctionIndices {@@ -1063,12 +959,9 @@ impl FunctionIndices {&mut obj,&compiled_funcs,&|_caller_index: usize, callee: FuncKey| {- self.indices- .get(&callee)- .and_then(|f| f.as_function().copied())- .unwrap_or_else(|| {- panic!("cannot resolve relocation! no index for callee {callee:?}")- })+ self.indices.get(&callee).copied().unwrap_or_else(|| {+ panic!("cannot resolve relocation! no index for callee {callee:?}")+ })},)?;@@ -1078,8 +971,7 @@ impl FunctionIndices {&mut obj,&translations,&|module, func| {- let i =- self.indices[&FuncKey::DefinedWasmFunction(module, func)].unwrap_function();+ let i = self.indices[&FuncKey::DefinedWasmFunction(module, func)];let (symbol, _) = symbol_ids_and_locs[i];let (_, compiled_func) = &compiled_funcs[i];(symbol, &**compiled_func)@@ -1095,9 +987,15 @@ impl FunctionIndices {>::new();#[cfg(feature = "component-model")]- let mut trampolines = PrimaryMap::<+ let mut wasm_call_trampolines = PrimaryMap::<+ wasmtime_environ::component::TrampolineIndex,+ wasmtime_environ::FunctionLoc,+ >::new();++ #[cfg(feature = "component-model")]+ let mut array_call_trampolines = PrimaryMap::<wasmtime_environ::component::TrampolineIndex,- wasmtime_environ::component::AllCallFunc<FunctionLoc>,+ wasmtime_environ::FunctionLoc,>::new();#[cfg(feature = "component-model")]@@ -1109,16 +1007,14 @@ impl FunctionIndices {// final linking and metadata-collection step.match *key {FuncKey::DefinedWasmFunction(module, def_func) => {- let index = index.unwrap_function();- let (_, wasm_func_loc) = symbol_ids_and_locs[index];+ let (_, wasm_func_loc) = symbol_ids_and_locs[*index];let start_srcloc = self.start_srclocs[key];let array_to_wasm_trampoline = self.indices.get(&FuncKey::ArrayToWasmTrampoline(module, def_func)).map(|index| {- let index = index.unwrap_function();- let (_, loc) = symbol_ids_and_locs[index];+ let (_, loc) = symbol_ids_and_locs[*index];loc});@@ -1152,21 +1048,25 @@ impl FunctionIndices {}#[cfg(feature = "component-model")]- FuncKey::ComponentTrampoline(trampoline) => {- let index = index.unwrap_all_call_func();- let loc = index.map(|i| {- let (_, loc) = symbol_ids_and_locs[i];- loc- });- debug_assert!(trampolines.get(trampoline).is_none());- let trampoline2 = trampolines.push(loc);- debug_assert_eq!(trampoline, trampoline2);+ FuncKey::ComponentTrampoline(abi, trampoline) => {+ let (_, loc) = symbol_ids_and_locs[*index];+ match abi {+ wasmtime_environ::Abi::Wasm => {+ debug_assert!(wasm_call_trampolines.get(trampoline).is_none());+ let trampoline2 = wasm_call_trampolines.push(loc);+ debug_assert_eq!(trampoline, trampoline2);+ }+ wasmtime_environ::Abi::Array => {+ debug_assert!(array_call_trampolines.get(trampoline).is_none());+ let trampoline2 = array_call_trampolines.push(loc);+ debug_assert_eq!(trampoline, trampoline2);+ }+ }}#[cfg(feature = "component-model")]FuncKey::ResourceDropTrampoline => {- let index = index.unwrap_function();- let (_, loc) = symbol_ids_and_locs[index];+ let (_, loc) = symbol_ids_and_locs[*index];resource_drop_trampoline = Some(loc);}}@@ -1209,7 +1109,7 @@ impl FunctionIndices {.map(|ty| {debug_assert_eq!(ty, types.trampoline_type(ty));let key = FuncKey::WasmToArrayTrampoline(ty);- let index = self.indices[&key].unwrap_function();+ let index = self.indices[&key];let (_, loc) = symbol_ids_and_locs[index];(ty, loc)})@@ -1222,7 +1122,9 @@ impl FunctionIndices {let artifacts = Artifacts {modules,#[cfg(feature = "component-model")]- trampolines,+ wasm_call_trampolines,+ #[cfg(feature = "component-model")]+ array_call_trampolines,#[cfg(feature = "component-model")]resource_drop_trampoline,};@@ -1237,10 +1139,11 @@ impl FunctionIndices {struct Artifacts {modules: PrimaryMap<StaticModuleIndex, CompiledModuleInfo>,#[cfg(feature = "component-model")]- trampolines: PrimaryMap<- wasmtime_environ::component::TrampolineIndex,- wasmtime_environ::component::AllCallFunc<wasmtime_environ::FunctionLoc>,- >,+ wasm_call_trampolines:+ PrimaryMap<wasmtime_environ::component::TrampolineIndex, wasmtime_environ::FunctionLoc>,+ #[cfg(feature = "component-model")]+ array_call_trampolines:+ PrimaryMap<wasmtime_environ::component::TrampolineIndex, wasmtime_environ::FunctionLoc>,#[cfg(feature = "component-model")]resource_drop_trampoline: Option<wasmtime_environ::FunctionLoc>,}@@ -1250,8 +1153,13 @@ impl Artifacts {/// resulting `CompiledModuleInfo`.fn unwrap_as_module_info(self) -> CompiledModuleInfo {assert_eq!(self.modules.len(), 1);+#[cfg(feature = "component-model")]- assert!(self.trampolines.is_empty());+ {+ assert!(self.wasm_call_trampolines.is_empty());+ assert!(self.array_call_trampolines.is_empty());+ }+self.modules.into_iter().next().unwrap().1}}diff --git a/crates/wasmtime/src/runtime/component/component.rs b/crates/wasmtime/src/runtime/component/component.rsindex 00f7acf72d..222dccdb63 100644--- a/crates/wasmtime/src/runtime/component/component.rs+++ b/crates/wasmtime/src/runtime/component/component.rs@@ -17,9 +17,9 @@ use core::ptr::NonNull;use std::path::Path;use wasmtime_environ::TypeTrace;use wasmtime_environ::component::{- AllCallFunc, CompiledComponentInfo, ComponentArtifacts, ComponentTypes, CoreDef, Export,- ExportIndex, GlobalInitializer, InstantiateModule, NameMapNoIntern, OptionsIndex,- StaticModuleIndex, TrampolineIndex, TypeComponentIndex, TypeFuncIndex, VMComponentOffsets,+ CompiledComponentInfo, ComponentArtifacts, ComponentTypes, CoreDef, Export, ExportIndex,+ GlobalInitializer, InstantiateModule, NameMapNoIntern, OptionsIndex, StaticModuleIndex,+ TrampolineIndex, TypeComponentIndex, TypeFuncIndex, VMComponentOffsets,};use wasmtime_environ::{FunctionLoc, HostPtr, ObjectKind, PrimaryMap};@@ -492,10 +492,8 @@ impl Component {}pub(crate) fn trampoline_ptrs(&self, index: TrampolineIndex) -> AllCallFuncPointers {- let AllCallFunc {- wasm_call,- array_call,- } = &self.inner.info.trampolines[index];+ let wasm_call = &self.inner.info.wasm_call_trampolines[index];+ let array_call = &self.inner.info.array_call_trampolines[index];AllCallFuncPointers {wasm_call: self.func(wasm_call).cast(),array_call: self.func(array_call).cast(),diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 8a3d897133..47fccb0684 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -110,6 +110,7 @@ use core::num::NonZeroU64;use core::ops::{Deref, DerefMut};use core::pin::Pin;use core::ptr::NonNull;+use wasmtime_environ::StaticModuleIndex;use wasmtime_environ::{DefinedGlobalIndex, DefinedTableIndex, EntityRef, PrimaryMap, TripleExt};mod context;@@ -715,7 +716,9 @@ impl<T> Store<T> {// single "default callee" for the entire `Store`. This is then used as// part of `Func::call` to guarantee that the `callee: *mut VMContext`// is never null.- let module = Arc::new(wasmtime_environ::Module::default());+ let module = Arc::new(wasmtime_environ::Module::new(StaticModuleIndex::from_u32(+ 0,+ )));let shim = ModuleRuntimeInfo::bare(module);let allocator = OnDemandInstanceAllocator::default();@@ -1666,7 +1669,7 @@ impl StoreOpaque {store: &mut StoreOpaque,limiter: Option<&mut StoreResourceLimiter<'_>>,) -> Result<GcStore> {- use wasmtime_environ::packed_option::ReservedValue;+ use wasmtime_environ::{StaticModuleIndex, packed_option::ReservedValue};let engine = store.engine();let mem_ty = engine.tunables().gc_heap_memory_type();@@ -1678,9 +1681,9 @@ impl StoreOpaque {// First, allocate the memory that will be our GC heap's storage.let mut request = InstanceAllocationRequest {id: InstanceId::reserved_value(),- runtime_info: &ModuleRuntimeInfo::bare(Arc::new(- wasmtime_environ::Module::default(),- )),+ runtime_info: &ModuleRuntimeInfo::bare(Arc::new(wasmtime_environ::Module::new(+ StaticModuleIndex::from_u32(0),+ ))),imports: vm::Imports::default(),store,limiter,diff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rsindex 27566a7570..9b91d7eadf 100644--- a/crates/wasmtime/src/runtime/trampoline/memory.rs+++ b/crates/wasmtime/src/runtime/trampoline/memory.rs@@ -10,14 +10,12 @@ use crate::runtime::vm::{use crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque, StoreResourceLimiter};use alloc::sync::Arc;use wasmtime_environ::{- DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, Module, Tunables, VMOffsets,+ DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, Module, StaticModuleIndex,+ Tunables, VMOffsets,};#[cfg(feature = "component-model")]-use wasmtime_environ::{- StaticModuleIndex,- component::{Component, VMComponentOffsets},-};+use wasmtime_environ::component::{Component, VMComponentOffsets};/// Create a "frankenstein" instance with a single memory.///@@ -30,7 +28,7 @@ pub async fn create_memory(memory_ty: &MemoryType,preallocation: Option<&SharedMemory>,) -> Result<InstanceId> {- let mut module = Module::new();+ let mut module = Module::new(StaticModuleIndex::from_u32(0));// Create a memory, though it will never be used for constructing a memory// with an allocator: instead the memories are either preallocated (i.e.,diff --git a/crates/wasmtime/src/runtime/trampoline/table.rs b/crates/wasmtime/src/runtime/trampoline/table.rsindex a0516f5bd7..745fa494a1 100644--- a/crates/wasmtime/src/runtime/trampoline/table.rs+++ b/crates/wasmtime/src/runtime/trampoline/table.rs@@ -3,6 +3,7 @@ use crate::prelude::*;use crate::runtime::vm::{Imports, ModuleRuntimeInfo, OnDemandInstanceAllocator};use crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque, StoreResourceLimiter};use alloc::sync::Arc;+use wasmtime_environ::StaticModuleIndex;use wasmtime_environ::{EntityIndex, Module, TypeTrace};pub async fn create_table(@@ -10,7 +11,7 @@ pub async fn create_table(limiter: Option<&mut StoreResourceLimiter<'_>>,table: &TableType,) -> Result<InstanceId> {- let mut module = Module::new();+ let mut module = Module::new(StaticModuleIndex::from_u32(0));let wasmtime_table = *table.wasmtime_table();diff --git a/crates/wasmtime/src/runtime/trampoline/tag.rs b/crates/wasmtime/src/runtime/trampoline/tag.rsindex 4c9e25fc05..aa4de83766 100644--- a/crates/wasmtime/src/runtime/trampoline/tag.rs+++ b/crates/wasmtime/src/runtime/trampoline/tag.rs@@ -5,11 +5,12 @@ use crate::runtime::vm::{self, Imports, ModuleRuntimeInfo, OnDemandInstanceAllocuse crate::store::{AllocateInstanceKind, InstanceId, StoreOpaque};use alloc::sync::Arc;use wasmtime_environ::EngineOrModuleTypeIndex;+use wasmtime_environ::StaticModuleIndex;use wasmtime_environ::Tag;use wasmtime_environ::{EntityIndex, Module, TypeTrace};pub fn create_tag(store: &mut StoreOpaque, ty: &TagType) -> Result<InstanceId> {- let mut module = Module::new();+ let mut module = Module::new(StaticModuleIndex::from_u32(0));let func_ty = ty.ty().clone().into_registered_type();let exn_ty = ExnType::from_tag_type(ty)?.registered_type().clone();diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rsindex 32c2c27533..a96c896ae1 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs@@ -545,7 +545,7 @@ pub enum VMStackState {mod tests {use core::mem::{offset_of, size_of};- use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};use super::*;@@ -558,7 +558,7 @@ mod tests {#[test]fn check_vm_stack_limits_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(offset_of!(VMStackLimits, stack_limit),@@ -572,7 +572,7 @@ mod tests {#[test]fn check_vm_common_stack_information_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMCommonStackInformation>(),@@ -603,7 +603,7 @@ mod tests {#[test]fn check_vm_array_offsets() {// Note that the type parameter has no influence on the size and offsets.- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMHostArray<()>>(),@@ -625,7 +625,7 @@ mod tests {#[test]fn check_vm_contobj_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(offset_of!(VMContObj, contref),@@ -643,7 +643,7 @@ mod tests {#[test]fn check_vm_contref_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(offset_of!(VMContRef, common_stack_information),@@ -681,7 +681,7 @@ mod tests {#[test]fn check_vm_stack_chain_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMStackChain>(),diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex a6cca3dd0d..24dc0bfd57 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -95,11 +95,11 @@ mod test_vmfunction_import {use super::VMFunctionImport;use core::mem::offset_of;use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmfunction_import_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMFunctionImport>(),@@ -165,11 +165,11 @@ mod test_vmtable {use core::mem::offset_of;use std::mem::size_of;use wasmtime_environ::component::{Component, VMComponentOffsets};- use wasmtime_environ::{HostPtr, Module, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmtable_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMTableImport>(),@@ -194,7 +194,7 @@ mod test_vmtable {// Because we use `VMTableImport` for recording tables used by components, we// want to make sure that the size calculations between `VMOffsets` and// `VMComponentOffsets` stay the same.- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let vm_offsets = VMOffsets::new(HostPtr, &module);let component = Component::default();let vm_component_offsets = VMComponentOffsets::new(HostPtr, &component);@@ -228,11 +228,11 @@ mod test_vmmemory_import {use super::VMMemoryImport;use core::mem::offset_of;use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmmemory_import_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMMemoryImport>(),@@ -302,11 +302,11 @@ mod test_vmglobal_import {use super::VMGlobalImport;use core::mem::offset_of;use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmglobal_import_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMGlobalImport>(),@@ -341,11 +341,11 @@ unsafe impl VmSafe for VMTagImport {}mod test_vmtag_import {use super::VMTagImport;use core::mem::{offset_of, size_of};- use wasmtime_environ::{HostPtr, Module, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmtag_import_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMTagImport>(),@@ -418,11 +418,11 @@ mod test_vmmemory_definition {use super::VMMemoryDefinition;use core::mem::offset_of;use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};#[test]fn check_vmmemory_definition_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMMemoryDefinition>(),@@ -465,11 +465,11 @@ mod test_vmtable_definition {use super::VMTableDefinition;use core::mem::offset_of;use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmtable_definition_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMTableDefinition>(),@@ -504,7 +504,7 @@ unsafe impl VmSafe for VMGlobalDefinition {}mod test_vmglobal_definition {use super::VMGlobalDefinition;use std::mem::{align_of, size_of};- use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};#[test]fn check_vmglobal_definition_alignment() {@@ -519,7 +519,7 @@ mod test_vmglobal_definition {#[test]fn check_vmglobal_definition_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMGlobalDefinition>(),@@ -529,7 +529,7 @@ mod test_vmglobal_definition {#[test]fn check_vmglobal_begins_aligned() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(offsets.vmctx_globals_begin() % 16, 0);}@@ -777,11 +777,11 @@ impl VMGlobalDefinition {mod test_vmshared_type_index {use super::VMSharedTypeIndex;use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, StaticModuleIndex, VMOffsets};#[test]fn check_vmshared_type_index() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMSharedTypeIndex>(),@@ -813,11 +813,11 @@ unsafe impl VmSafe for VMTagDefinition {}mod test_vmtag_definition {use super::VMTagDefinition;use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};#[test]fn check_vmtag_definition_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMTagDefinition>(),@@ -827,7 +827,7 @@ mod test_vmtag_definition {#[test]fn check_vmtag_begins_aligned() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(offsets.vmctx_tags_begin() % 16, 0);}@@ -973,11 +973,11 @@ mod test_vm_func_ref {use super::VMFuncRef;use core::mem::offset_of;use std::mem::size_of;- use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};#[test]fn check_vm_func_ref_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(size_of::<VMFuncRef>(),@@ -1274,11 +1274,11 @@ impl Default for VMStoreContext {mod test_vmstore_context {use super::{VMMemoryDefinition, VMStoreContext};use core::mem::offset_of;- use wasmtime_environ::{HostPtr, Module, PtrSize, VMOffsets};+ use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};#[test]fn field_offsets() {- let module = Module::new();+ let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);assert_eq!(offset_of!(VMStoreContext, stack_limit),diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rsindex 41fb19ff61..66279e0f2b 100644--- a/crates/winch/src/compiler.rs+++ b/crates/winch/src/compiler.rs@@ -120,7 +120,7 @@ impl wasmtime_environ::Compiler for Compiler {log::trace!("compiling function: {key:?} = {symbol:?}");let (module_index, def_func_index) = key.unwrap_defined_wasm_function();- debug_assert_eq!(module_index, translation.module_index);+ debug_assert_eq!(module_index, translation.module_index());let index = translation.module.func_index(def_func_index);let sig = translation.module.functions[index]@@ -384,17 +384,16 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {component: &wasmtime_environ::component::ComponentTranslation,types: &wasmtime_environ::component::ComponentTypesBuilder,key: FuncKey,+ abi: wasmtime_environ::Abi,tunables: &Tunables,symbol: &str,- ) -> Result<wasmtime_environ::component::AllCallFunc<CompiledFunctionBody>> {+ ) -> Result<CompiledFunctionBody> {let mut body = self.0.component_compiler()- .compile_trampoline(component, types, key, tunables, symbol)?;+ .compile_trampoline(component, types, key, abi, tunables, symbol)?;if let Some(c) = self.0.inlining_compiler() {- c.finish_compiling(&mut body.array_call, None, symbol)- .map_err(|e| CompileError::Codegen(e.to_string()))?;- c.finish_compiling(&mut body.wasm_call, None, symbol)+ c.finish_compiling(&mut body, None, symbol).map_err(|e| CompileError::Codegen(e.to_string()))?;}Ok(body)diff --git a/winch/codegen/src/codegen/env.rs b/winch/codegen/src/codegen/env.rsindex 9f1b5e9ef3..a0d1d85ee1 100644--- a/winch/codegen/src/codegen/env.rs+++ b/winch/codegen/src/codegen/env.rs@@ -361,7 +361,7 @@ impl<'a, 'translation, 'data, P: PtrSize> FuncEnv<'a, 'translation, 'data, P> {/// Creates a name to reference the wasm function `index` provided.pub fn name_wasm(&mut self, def_func: DefinedFuncIndex) -> UserExternalNameRef {- let key = FuncKey::DefinedWasmFunction(self.translation.module_index, def_func);+ let key = FuncKey::DefinedWasmFunction(self.translation.module_index(), def_func);let (namespace, index) = key.into_raw_parts();self.intern_name(UserExternalName { namespace, index })}