Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Further refactor `TrampolineCompiler` for the wasm ABI (#11939)

Alex Crichton093b520f40c8
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 4b0dfd38d6..c0edc72a98 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -36,9 +36,8 @@ use wasmtime_environ::{
Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,
DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder,
FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, HostCall, InliningCompiler,
- ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapSection,
- StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, WasmFuncType,
- WasmValType,
+ ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapSection, StaticModuleIndex,
+ TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, WasmFuncType, WasmValType,
};
use wasmtime_unwinder::ExceptionTableBuilder;
@@ -363,8 +362,7 @@ impl wasmtime_environ::Compiler for Compiler {
self.array_to_wasm_trampoline(
key,
FuncKey::DefinedWasmFunction(module_index, def_func_index),
- types,
- sig,
+ types[sig].unwrap_func(),
symbol,
self.isa.pointer_bytes().vmctx_store_context().into(),
wasmtime_environ::VMCONTEXT_MAGIC,
@@ -1231,19 +1229,16 @@ impl Compiler {
&self,
trampoline_key: FuncKey,
callee_key: FuncKey,
- types: &ModuleTypesBuilder,
- callee_sig: ModuleInternedTypeIndex,
+ callee_sig: &WasmFuncType,
symbol: &str,
vm_store_context_offset: u32,
expected_vmctx_magic: u32,
) -> Result<CompiledFunctionBody, CompileError> {
log::trace!("compiling array-to-wasm trampoline: {trampoline_key:?} = {symbol:?}");
- let wasm_func_ty = types[callee_sig].unwrap_func();
-
let isa = &*self.isa;
let pointer_type = isa.pointer_type();
- let wasm_call_sig = wasm_call_signature(isa, wasm_func_ty, &self.tunables);
+ let wasm_call_sig = wasm_call_signature(isa, callee_sig, &self.tunables);
let array_call_sig = array_call_signature(isa);
let mut compiler = self.function_compiler();
@@ -1261,7 +1256,7 @@ impl Compiler {
// First load the actual arguments out of the array.
let mut args = self.load_values_from_array(
- wasm_func_ty.params(),
+ callee_sig.params(),
&mut builder,
values_vec_ptr,
values_vec_len,
@@ -1340,7 +1335,7 @@ impl Compiler {
builder.switch_to_block(normal_return);
self.store_values_to_array(
&mut builder,
- wasm_func_ty.returns(),
+ callee_sig.returns(),
&normal_return_values,
values_vec_ptr,
values_vec_len,
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index 237bfd62be..054f864b4b 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -1,18 +1,14 @@
//! Compilation support for the component model.
-use crate::array_call_signature;
-use crate::{
- TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT, compiler::Compiler, wasm_call_signature,
-};
+use crate::{TRAP_ALWAYS, TRAP_CANNOT_ENTER, TRAP_INTERNAL_ASSERT, compiler::Compiler};
use anyhow::{Result, bail};
use cranelift_codegen::ir::condcodes::IntCC;
use cranelift_codegen::ir::{self, InstBuilder, MemFlags, Value};
use cranelift_codegen::isa::{CallConv, TargetIsa};
use cranelift_frontend::FunctionBuilder;
use wasmtime_environ::{
- Abi, CompiledFunctionBody, EntityRef, FuncKey, HostCall, ModuleInternedTypeIndex, PtrSize,
- TrapSentinel, Tunables, WasmFuncType, WasmValType, component::*,
- fact::PREPARE_CALL_FIXED_PARAMS,
+ Abi, CompiledFunctionBody, EntityRef, FuncKey, HostCall, PtrSize, TrapSentinel, Tunables,
+ WasmFuncType, WasmValType, component::*, fact::PREPARE_CALL_FIXED_PARAMS,
};
struct TrampolineCompiler<'a> {
@@ -22,9 +18,8 @@ struct TrampolineCompiler<'a> {
component: &'a Component,
types: &'a ComponentTypesBuilder,
offsets: VMComponentOffsets<u8>,
- abi: Abi,
block0: ir::Block,
- signature: ModuleInternedTypeIndex,
+ signature: &'a WasmFuncType,
tunables: &'a Tunables,
}
@@ -98,104 +93,19 @@ enum WasmArgs {
InRegistersUpTo(usize),
}
-/// Get a function's parameters regardless of the ABI in use.
-///
-/// This emits code to load the parameters from the array-call's ABI's values
-/// vector, if necessary.
-fn abi_params(
- compiler: &Compiler,
- builder: &mut FunctionBuilder<'_>,
- abi: Abi,
- types: &[WasmValType],
-) -> Vec<ir::Value> {
- let entry_block = builder
- .func
- .layout
- .entry_block()
- .expect("should have an entry block");
- let entry_block_params = builder.func.dfg.block_params(entry_block);
- match abi {
- // Wasm and native ABIs pass parameters as normal function parameters.
- Abi::Wasm => entry_block_params.to_vec(),
-
- // The array ABI passes a pointer/length as the 3rd/4th arguments and
- // those are used to load the actual Wasm parameters.
- Abi::Array => {
- let &[callee_vmctx, caller_vmctx, values_vec_ptr, values_vec_len] = entry_block_params
- else {
- unreachable!()
- };
- let mut results =
- compiler.load_values_from_array(types, builder, values_vec_ptr, values_vec_len);
- results.splice(0..0, [callee_vmctx, caller_vmctx]);
- results
- }
- }
-}
-
-/// Emit code to return the given result values, regardless of the ABI in use.
-fn abi_return(
- compiler: &Compiler,
- builder: &mut FunctionBuilder<'_>,
- abi: Abi,
- result_types: &[WasmValType],
- results: &[ir::Value],
-) {
- match abi {
- // The Wasm ABI returns values as usual.
- Abi::Wasm => {
- builder.ins().return_(results);
- }
-
- // The array ABI stores all results in the pointer/length passed
- // as arguments to this function, which contractually are required
- // to have enough space for the results.
- Abi::Array => {
- let entry_block = builder
- .func
- .layout
- .entry_block()
- .expect("should have an entry block");
- let &[_callee_vmctx, _caller_vmctx, values_vec_ptr, values_vec_len] =
- builder.func.dfg.block_params(entry_block)
- else {
- unreachable!()
- };
-
- compiler.store_values_to_array(
- builder,
- result_types,
- results,
- values_vec_ptr,
- values_vec_len,
- );
-
- // A "true" return value indicates a successful, non-trapping call.
- let true_value = builder.ins().iconst(ir::types::I8, 1);
- builder.ins().return_(&[true_value]);
- }
- }
-}
-
impl<'a> TrampolineCompiler<'a> {
fn new(
compiler: &'a Compiler,
func_compiler: &'a mut super::FunctionCompiler<'_>,
component: &'a Component,
types: &'a ComponentTypesBuilder,
- index: TrampolineIndex,
- abi: Abi,
+ signature: &'a WasmFuncType,
tunables: &'a Tunables,
) -> TrampolineCompiler<'a> {
let isa = &*compiler.isa;
- let signature = component.trampolines[index];
- let ty = types[signature].unwrap_func();
let func = ir::Function::with_name_signature(
ir::UserFuncName::user(0, 0),
- match abi {
- Abi::Wasm => crate::wasm_call_signature(isa, ty, &compiler.tunables),
- Abi::Array => crate::array_call_signature(isa),
- },
+ crate::wasm_call_signature(isa, signature, &compiler.tunables),
);
let (builder, block0) = func_compiler.builder(func);
TrampolineCompiler {
@@ -205,7 +115,6 @@ impl<'a> TrampolineCompiler<'a> {
component,
types,
offsets: VMComponentOffsets::new(isa.pointer_bytes(), component),
- abi,
block0,
signature,
tunables,
@@ -221,16 +130,7 @@ impl<'a> TrampolineCompiler<'a> {
to,
to64,
} => {
- match self.abi {
- Abi::Wasm => {
- self.translate_transcode(*op, *from, *from64, *to, *to64);
- }
- // Transcoders can only actually be called by Wasm, so let's assert
- // that here.
- Abi::Array => {
- self.builder.ins().trap(TRAP_INTERNAL_ASSERT);
- }
- }
+ self.translate_transcode(*op, *from, *from64, *to, *to64);
}
Trampoline::LowerImport {
index,
@@ -277,10 +177,7 @@ impl<'a> TrampolineCompiler<'a> {
}
Trampoline::ResourceNew { instance, ty } => {
// Currently this only supports resources represented by `i32`
- assert_eq!(
- self.types[self.signature].unwrap_func().params()[0],
- WasmValType::I32
- );
+ assert_eq!(self.signature.params()[0], WasmValType::I32);
self.translate_libcall(
host::resource_new32,
TrapSentinel::NegativeOne,
@@ -293,10 +190,7 @@ impl<'a> TrampolineCompiler<'a> {
}
Trampoline::ResourceRep { instance, ty } => {
// Currently this only supports resources represented by `i32`
- assert_eq!(
- self.types[self.signature].unwrap_func().returns()[0],
- WasmValType::I32
- );
+ assert_eq!(self.signature.returns()[0], WasmValType::I32);
self.translate_libcall(
host::resource_rep32,
TrapSentinel::NegativeOne,
@@ -783,11 +677,10 @@ impl<'a> TrampolineCompiler<'a> {
}
Trampoline::SyncStartCall { callback } => {
let pointer_type = self.isa.pointer_type();
- let wasm_func_ty = &self.types[self.signature].unwrap_func();
let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
&WasmFuncType::new(
Box::new([]),
- wasm_func_ty.returns().iter().copied().collect(),
+ self.signature.returns().iter().copied().collect(),
),
&mut self.builder,
&[],
@@ -997,28 +890,14 @@ impl<'a> TrampolineCompiler<'a> {
/// a stack-allocated array.
fn store_wasm_arguments(&mut self, args: &[Value]) -> (Value, Value) {
let pointer_type = self.isa.pointer_type();
- let wasm_func_ty = &self.types[self.signature].unwrap_func();
-
- match self.abi {
- // For the wasm ABI a stack needs to be allocated and these
- // arguments are stored onto the stack.
- Abi::Wasm => {
- let (ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
- wasm_func_ty,
- &mut self.builder,
- args,
- );
- let len = self.builder.ins().iconst(pointer_type, i64::from(len));
- (ptr, len)
- }
- // For the array ABI all arguments were already in a stack, so
- // forward along that pointer/len.
- Abi::Array => {
- let params = self.builder.func.dfg.block_params(self.block0);
- (params[2], params[3])
- }
- }
+ let (ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
+ self.signature,
+ &mut self.builder,
+ args,
+ );
+ let len = self.builder.ins().iconst(pointer_type, i64::from(len));
+ (ptr, len)
}
/// Convenience wrapper around `translate_hostcall` to enable type inference
@@ -1059,7 +938,6 @@ impl<'a> TrampolineCompiler<'a> {
extra_host_args: impl FnOnce(&mut Self, &mut Vec<ir::Value>),
) {
let pointer_type = self.isa.pointer_type();
- let wasm_func_ty = self.types[self.signature].unwrap_func();
// Load all parameters in an ABI-agnostic fashion, of which the
// `VMComponentContext` will be the first.
@@ -1091,7 +969,7 @@ impl<'a> TrampolineCompiler<'a> {
WasmArgs::InRegistersUpTo(n) => {
let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
&WasmFuncType::new(
- wasm_func_ty.params().iter().skip(n).copied().collect(),
+ self.signature.params().iter().skip(n).copied().collect(),
Box::new([]),
),
&mut self.builder,
@@ -1144,7 +1022,7 @@ impl<'a> TrampolineCompiler<'a> {
// accounts for the ABI of this function when storing results.
let result = self.builder.func.dfg.inst_results(call).get(0).copied();
let result_ty = result.map(|v| self.builder.func.dfg.value_type(v));
- let expected = wasm_func_ty.returns();
+ let expected = self.signature.returns();
match host_result.into() {
HostResult::Sentinel(TrapSentinel::NegativeOne) => {
assert_eq!(expected.len(), 1);
@@ -1176,7 +1054,7 @@ impl<'a> TrampolineCompiler<'a> {
let len = len.or(val_raw_len).unwrap();
self.raise_if_host_trapped(result.unwrap());
let results = self.compiler.load_values_from_array(
- wasm_func_ty.returns(),
+ self.signature.returns(),
&mut self.builder,
ptr,
len,
@@ -1354,11 +1232,7 @@ impl<'a> TrampolineCompiler<'a> {
i32::from(self.offsets.ptr.vm_func_ref_vmctx()),
);
- let sig = crate::wasm_call_signature(
- self.isa,
- &self.types[self.signature].unwrap_func(),
- &self.compiler.tunables,
- );
+ let sig = crate::wasm_call_signature(self.isa, self.signature, &self.compiler.tunables);
let sig_ref = self.builder.import_signature(sig);
// NB: note that the "caller" vmctx here is the caller of this
@@ -1461,23 +1335,17 @@ impl<'a> TrampolineCompiler<'a> {
)
}
+ /// Get a function's parameters regardless of the ABI in use.
+ ///
+ /// This emits code to load the parameters from the array-call's ABI's values
+ /// vector, if necessary.
fn abi_load_params(&mut self) -> Vec<ir::Value> {
- abi_params(
- self.compiler,
- &mut self.builder,
- self.abi,
- self.types[self.signature].unwrap_func().params(),
- )
+ self.builder.func.dfg.block_params(self.block0).to_vec()
}
+ /// Emit code to return the given result values, regardless of the ABI in use.
fn abi_store_results(&mut self, results: &[ir::Value]) {
- abi_return(
- self.compiler,
- &mut self.builder,
- self.abi,
- self.types[self.signature].unwrap_func().returns(),
- results,
- );
+ self.builder.ins().return_(results);
}
fn raise_if_host_trapped(&mut self, succeeded: ir::Value) {
@@ -1534,6 +1402,7 @@ impl ComponentCompiler for Compiler {
) -> Result<CompiledFunctionBody> {
let (abi2, trampoline_index) = key.unwrap_component_trampoline();
debug_assert_eq!(abi, abi2);
+ let sig = types[component.component.trampolines[trampoline_index]].unwrap_func();
match abi {
// Fall through to the trampoline compiler.
@@ -1547,8 +1416,7 @@ impl ComponentCompiler for Compiler {
return Ok(self.array_to_wasm_trampoline(
key,
FuncKey::ComponentTrampoline(Abi::Wasm, trampoline_index),
- types.module_types_builder(),
- component.component.trampolines[trampoline_index],
+ sig,
symbol,
offsets.vm_store_context(),
wasmtime_environ::component::VMCOMPONENT_MAGIC,
@@ -1562,8 +1430,7 @@ impl ComponentCompiler for Compiler {
&mut compiler,
&component.component,
types,
- trampoline_index,
- abi,
+ sig,
tunables,
);
@@ -1579,20 +1446,18 @@ impl ComponentCompiler for Compiler {
vmctx,
wasmtime_environ::component::VMCOMPONENT_MAGIC,
);
- if let Abi::Wasm = abi {
- let vm_store_context = c.builder.ins().load(
- pointer_type,
- MemFlags::trusted(),
- vmctx,
- i32::try_from(c.offsets.vm_store_context()).unwrap(),
- );
- super::save_last_wasm_exit_fp_and_pc(
- &mut c.builder,
- pointer_type,
- &c.offsets.ptr,
- vm_store_context,
- );
- }
+ let vm_store_context = c.builder.ins().load(
+ pointer_type,
+ MemFlags::trusted(),
+ vmctx,
+ i32::try_from(c.offsets.vm_store_context()).unwrap(),
+ );
+ super::save_last_wasm_exit_fp_and_pc(
+ &mut c.builder,
+ pointer_type,
+ &c.offsets.ptr,
+ vm_store_context,
+ );
c.translate(&component.trampolines[trampoline_index]);
c.builder.finalize();
@@ -1608,172 +1473,97 @@ impl ComponentCompiler for Compiler {
&self,
tunables: &Tunables,
component: &ComponentTranslation,
+ types: &ComponentTypesBuilder,
intrinsic: UnsafeIntrinsic,
abi: Abi,
- _symbol: &str,
+ symbol: &str,
) -> Result<CompiledFunctionBody> {
- let offsets = VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);
- let mut compiler = self.function_compiler();
- let cx = &mut compiler.cx;
- let ctx = &mut cx.codegen_context;
- let mut builder = FunctionBuilder::new(&mut ctx.func, cx.func_translator.context());
-
- // Initialize the function, its signature, and entry block.
- let init = |builder: &mut FunctionBuilder| -> Vec<ir::Value> {
- let wasm_func_ty = WasmFuncType::new(
- intrinsic.core_params().into(),
- intrinsic.core_results().into(),
- );
- builder.func.signature = match abi {
- Abi::Wasm => wasm_call_signature(&*self.isa, &wasm_func_ty, tunables),
- Abi::Array => array_call_signature(&*self.isa),
- };
-
- assert!(builder.func.layout.entry_block().is_none());
- let entry_block = builder.create_block();
- builder.append_block_params_for_function_params(entry_block);
- builder.seal_block(entry_block);
- builder.switch_to_block(entry_block);
- builder.ensure_inserted_block();
-
- abi_params(self, builder, abi, wasm_func_ty.params())
- };
+ let wasm_func_ty = WasmFuncType::new(
+ intrinsic.core_params().into(),
+ intrinsic.core_results().into(),
+ );
- // Emit code for a native-load intrinsic.
- let load = |builder: &mut FunctionBuilder| -> Result<()> {
- debug_assert_eq!(intrinsic.core_params(), &[WasmValType::I64]);
- debug_assert_eq!(intrinsic.core_results().len(), 1);
-
- let wasm_ty = intrinsic.core_results()[0];
- let clif_ty = unsafe_intrinsic_clif_results(intrinsic)[0];
-
- let [_callee_vmctx, _caller_vmctx, pointer] = *init(builder) else {
- unreachable!()
- };
-
- // Truncate the pointer, if necessary.
- debug_assert_eq!(builder.func.dfg.value_type(pointer), ir::types::I64);
- let pointer = match self.isa.pointer_bits() {
- 32 => builder.ins().ireduce(ir::types::I32, pointer),
- 64 => pointer,
- p => bail!("unsupported architecture: no support for {p}-bit pointers"),
- };
-
- // Do the load!
- let mut value = builder
- .ins()
- .load(clif_ty, ir::MemFlags::trusted(), pointer, 0);
-
- // Extend the value, if necessary. When implementing the
- // `u8-native-load` intrinsic, for example, we will load a Cranelift
- // value of type `i8` but we need to extend it to an `i32` because
- // Wasm doesn't have an `i8` core value type.
- let wasm_clif_ty = crate::value_type(&*self.isa, wasm_ty);
- if clif_ty != wasm_clif_ty {
- assert!(clif_ty.bytes() < wasm_clif_ty.bytes());
- // NB: all of our unsafe intrinsics for native loads are
- // unsigned, so we always zero-extend.
- value = builder.ins().uextend(wasm_clif_ty, value);
- }
-
- abi_return(self, builder, abi, &[wasm_ty], &[value]);
- Ok(())
- };
+ match abi {
+ // Fall through to the trampoline compiler.
+ Abi::Wasm => {}
- // Emit code for a native-store intrinsic.
- let store = |builder: &mut FunctionBuilder| -> Result<()> {
- debug_assert!(intrinsic.core_results().is_empty());
- debug_assert!(matches!(intrinsic.core_params(), [WasmValType::I64, _]));
-
- let wasm_ty = intrinsic.core_params()[1];
- let clif_ty = unsafe_intrinsic_clif_params(intrinsic)[1];
-
- let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *init(builder) else {
- unreachable!()
- };
-
- // Truncate the pointer, if necessary.
- debug_assert_eq!(builder.func.dfg.value_type(pointer), ir::types::I64);
- let pointer = match self.isa.pointer_bits() {
- 32 => builder.ins().ireduce(ir::types::I32, pointer),
- 64 => pointer,
- p => bail!("unsupported architecture: no support for {p}-bit pointers"),
- };
-
- // Truncate the value, if necessary. For example, with
- // `u8-native-store` we will be given an `i32` from Wasm (because
- // core Wasm does not have an 8-bit integer value type) and we need
- // to reduce that into an `i8`.
- let wasm_ty = crate::value_type(&*self.isa, wasm_ty);
- if clif_ty != wasm_ty {
- assert!(clif_ty.bytes() < wasm_ty.bytes());
- value = builder.ins().ireduce(clif_ty, value);
- }
-
- // Do the store!
- builder
- .ins()
- .store(ir::MemFlags::trusted(), value, pointer, 0);
+ // Implement the array-abi trampoline in terms of calling the
+ // wasm-abi trampoline.
+ Abi::Array => {
+ let offsets =
+ VMComponentOffsets::new(self.isa.pointer_bytes(), &component.component);
+ return Ok(self.array_to_wasm_trampoline(
+ FuncKey::UnsafeIntrinsic(abi, intrinsic),
+ FuncKey::UnsafeIntrinsic(Abi::Wasm, intrinsic),
+ &wasm_func_ty,
+ symbol,
+ offsets.vm_store_context(),
+ wasmtime_environ::component::VMCOMPONENT_MAGIC,
+ )?);
+ }
+ }
- abi_return(self, builder, abi, &[], &[]);
- Ok(())
- };
+ let mut compiler = self.function_compiler();
+ let mut c = TrampolineCompiler::new(
+ self,
+ &mut compiler,
+ &component.component,
+ &types,
+ &wasm_func_ty,
+ tunables,
+ );
match intrinsic {
- UnsafeIntrinsic::U8NativeLoad => load(&mut builder)?,
- UnsafeIntrinsic::U16NativeLoad => load(&mut builder)?,
- UnsafeIntrinsic::U32NativeLoad => load(&mut builder)?,
- UnsafeIntrinsic::U64NativeLoad => load(&mut builder)?,
- UnsafeIntrinsic::U8NativeStore => store(&mut builder)?,
- UnsafeIntrinsic::U16NativeStore => store(&mut builder)?,
- UnsafeIntrinsic::U32NativeStore => store(&mut builder)?,
- UnsafeIntrinsic::U64NativeStore => store(&mut builder)?,
+ UnsafeIntrinsic::U8NativeLoad
+ | UnsafeIntrinsic::U16NativeLoad
+ | UnsafeIntrinsic::U32NativeLoad
+ | UnsafeIntrinsic::U64NativeLoad => c.translate_load_intrinsic(intrinsic)?,
+ UnsafeIntrinsic::U8NativeStore
+ | UnsafeIntrinsic::U16NativeStore
+ | UnsafeIntrinsic::U32NativeStore
+ | UnsafeIntrinsic::U64NativeStore => c.translate_store_intrinsic(intrinsic)?,
UnsafeIntrinsic::StoreDataAddress => {
- let [callee_vmctx, _caller_vmctx] = *init(&mut builder) else {
+ let [callee_vmctx, _caller_vmctx] = *c.abi_load_params() else {
unreachable!()
};
let pointer_type = self.isa.pointer_type();
// Load the `*mut VMStoreContext` out of our vmctx.
- let store_ctx = builder.ins().load(
+ let store_ctx = c.builder.ins().load(
pointer_type,
ir::MemFlags::trusted()
.with_readonly()
.with_alias_region(Some(ir::AliasRegion::Vmctx))
.with_can_move(),
callee_vmctx,
- i32::try_from(offsets.vm_store_context()).unwrap(),
+ i32::try_from(c.offsets.vm_store_context()).unwrap(),
);
// Load the `*mut T` out of the `VMStoreContext`.
- let data_address = builder.ins().load(
+ let data_address = c.builder.ins().load(
pointer_type,
ir::MemFlags::trusted()
.with_readonly()
.with_alias_region(Some(ir::AliasRegion::Vmctx))
.with_can_move(),
store_ctx,
- i32::from(offsets.ptr.vmstore_context_store_data()),
+ i32::from(c.offsets.ptr.vmstore_context_store_data()),
);
// Zero-extend the address if we are on a 32-bit architecture.
let data_address = match pointer_type.bits() {
- 32 => builder.ins().uextend(ir::types::I64, data_address),
+ 32 => c.builder.ins().uextend(ir::types::I64, data_address),
64 => data_address,
p => bail!("unsupported architecture: no support for {p}-bit pointers"),
};
- abi_return(
- self,
- &mut builder,
- abi,
- intrinsic.core_results(),
- &[data_address],
- );
+ c.abi_store_results(&[data_address]);
}
}
- builder.finalize();
+ c.builder.finalize();
+ compiler.cx.abi = Some(abi);
+
Ok(CompiledFunctionBody {
code: super::box_dyn_any_compiler_context(Some(compiler.cx)),
needs_gc_heap: false,
@@ -1995,6 +1785,86 @@ impl TrampolineCompiler<'_> {
i32::from(self.offsets.ptr.vmmemory_definition_base()),
)
}
+
+ fn translate_load_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()> {
+ // Emit code for a native-load intrinsic.
+ debug_assert_eq!(intrinsic.core_params(), &[WasmValType::I64]);
+ debug_assert_eq!(intrinsic.core_results().len(), 1);
+
+ let wasm_ty = intrinsic.core_results()[0];
+ let clif_ty = unsafe_intrinsic_clif_results(intrinsic)[0];
+
+ let [_callee_vmctx, _caller_vmctx, pointer] = *self.abi_load_params() else {
+ unreachable!()
+ };
+
+ // Truncate the pointer, if necessary.
+ debug_assert_eq!(self.builder.func.dfg.value_type(pointer), ir::types::I64);
+ let pointer = match self.isa.pointer_bits() {
+ 32 => self.builder.ins().ireduce(ir::types::I32, pointer),
+ 64 => pointer,
+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),
+ };
+
+ // Do the load!
+ let mut value = self
+ .builder
+ .ins()
+ .load(clif_ty, ir::MemFlags::trusted(), pointer, 0);
+
+ // Extend the value, if necessary. When implementing the
+ // `u8-native-load` intrinsic, for example, we will load a Cranelift
+ // value of type `i8` but we need to extend it to an `i32` because
+ // Wasm doesn't have an `i8` core value type.
+ let wasm_clif_ty = crate::value_type(self.isa, wasm_ty);
+ if clif_ty != wasm_clif_ty {
+ assert!(clif_ty.bytes() < wasm_clif_ty.bytes());
+ // NB: all of our unsafe intrinsics for native loads are
+ // unsigned, so we always zero-extend.
+ value = self.builder.ins().uextend(wasm_clif_ty, value);
+ }
+
+ self.abi_store_results(&[value]);
+ Ok(())
+ }
+
+ fn translate_store_intrinsic(&mut self, intrinsic: UnsafeIntrinsic) -> Result<()> {
+ debug_assert!(intrinsic.core_results().is_empty());
+ debug_assert!(matches!(intrinsic.core_params(), [WasmValType::I64, _]));
+
+ let wasm_ty = intrinsic.core_params()[1];
+ let clif_ty = unsafe_intrinsic_clif_params(intrinsic)[1];
+
+ let [_callee_vmctx, _caller_vmctx, pointer, mut value] = *self.abi_load_params() else {
+ unreachable!()
+ };
+
+ // Truncate the pointer, if necessary.
+ debug_assert_eq!(self.builder.func.dfg.value_type(pointer), ir::types::I64);
+ let pointer = match self.isa.pointer_bits() {
+ 32 => self.builder.ins().ireduce(ir::types::I32, pointer),
+ 64 => pointer,
+ p => bail!("unsupported architecture: no support for {p}-bit pointers"),
+ };
+
+ // Truncate the value, if necessary. For example, with
+ // `u8-native-store` we will be given an `i32` from Wasm (because
+ // core Wasm does not have an 8-bit integer value type) and we need
+ // to reduce that into an `i8`.
+ let wasm_ty = crate::value_type(self.isa, wasm_ty);
+ if clif_ty != wasm_ty {
+ assert!(clif_ty.bytes() < wasm_ty.bytes());
+ value = self.builder.ins().ireduce(clif_ty, value);
+ }
+
+ // Do the store!
+ self.builder
+ .ins()
+ .store(ir::MemFlags::trusted(), value, pointer, 0);
+
+ self.abi_store_results(&[]);
+ Ok(())
+ }
}
/// Module with macro-generated contents that will return the signature and
diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs
index 08c089c66b..1c386108b6 100644
--- a/crates/environ/src/component/compiler.rs
+++ b/crates/environ/src/component/compiler.rs
@@ -25,6 +25,7 @@ pub trait ComponentCompiler: Send + Sync {
&self,
tunables: &Tunables,
component: &ComponentTranslation,
+ types: &ComponentTypesBuilder,
intrinsic: UnsafeIntrinsic,
abi: Abi,
symbol: &str,
diff --git a/crates/environ/src/key.rs b/crates/environ/src/key.rs
index f8b88cf7c7..27becc5991 100644
--- a/crates/environ/src/key.rs
+++ b/crates/environ/src/key.rs
@@ -341,6 +341,23 @@ impl FuncKey {
self.into_parts().1
}
+ /// Get ABI of the function that this key is definining.
+ pub fn abi(self) -> Abi {
+ match self {
+ FuncKey::DefinedWasmFunction(_, _) => Abi::Wasm,
+ FuncKey::ArrayToWasmTrampoline(_, _) => Abi::Array,
+ FuncKey::WasmToArrayTrampoline(_) => Abi::Wasm,
+ FuncKey::WasmToBuiltinTrampoline(_) => Abi::Wasm,
+ FuncKey::PulleyHostCall(_) => Abi::Wasm,
+ #[cfg(feature = "component-model")]
+ FuncKey::ComponentTrampoline(abi, _) => abi,
+ #[cfg(feature = "component-model")]
+ FuncKey::ResourceDropTrampoline => Abi::Wasm,
+ #[cfg(feature = "component-model")]
+ FuncKey::UnsafeIntrinsic(abi, _) => abi,
+ }
+ }
+
/// Get the raw, underlying `(namespace, index)` representation of this
/// compilation key.
///
diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs
index bdf885263a..5ebb978d07 100644
--- a/crates/wasmtime/src/compile.rs
+++ b/crates/wasmtime/src/compile.rs
@@ -32,7 +32,7 @@ use call_graph::CallGraph;
#[cfg(feature = "component-model")]
use wasmtime_environ::component::Translator;
use wasmtime_environ::{
- BuiltinFunctionIndex, CompiledFunctionBody, CompiledFunctionsTable,
+ Abi, BuiltinFunctionIndex, CompiledFunctionBody, CompiledFunctionsTable,
CompiledFunctionsTableBuilder, CompiledModuleInfo, Compiler, DefinedFuncIndex, FilePos,
FinishedObject, FuncKey, FunctionBodyData, InliningCompiler, IntraModuleInlining,
ModuleEnvironment, ModuleTranslation, ModuleTypes, ModuleTypesBuilder, ObjectKind, PrimaryMap,
@@ -312,7 +312,7 @@ impl<'a> CompileInputs<'a> {
key: FuncKey::UnsafeIntrinsic(abi, intrinsic),
function: compiler
.component_compiler()
- .compile_intrinsic(tunables, component, intrinsic, abi, &symbol)
+ .compile_intrinsic(tunables, component, types, intrinsic, abi, &symbol)
.with_context(|| format!("failed to compile `{symbol}`"))?,
symbol,
start_srcloc: FilePos::default(),
@@ -848,6 +848,16 @@ the use case.
return false;
}
+ // Skip inlining into array-abi functions which are entry
+ // trampolines into wasm. ABI-wise it's required that these have a
+ // single `try_call` into the module and it doesn't work if multiple
+ // get inlined or if the `try_call` goes away. Prevent all inlining
+ // to guarantee the structure of entry trampolines.
+ if caller_key.abi() == Abi::Array {
+ log::trace!(" --> not inlining: not inlining into array-abi caller");
+ return false;
+ }
+
// Consider whether this is an intra-module call.
//
// Inlining within a single core module has most often already been done
@@ -881,6 +891,7 @@ the use case.
}
}
}
+
_ => {}
}
diff --git a/crates/winch/src/compiler.rs b/crates/winch/src/compiler.rs
index 77f76dd742..906a987235 100644
--- a/crates/winch/src/compiler.rs
+++ b/crates/winch/src/compiler.rs
@@ -405,6 +405,7 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {
&self,
tunables: &Tunables,
component: &ComponentTranslation,
+ types: &wasmtime_environ::component::ComponentTypesBuilder,
intrinsic: wasmtime_environ::component::UnsafeIntrinsic,
abi: wasmtime_environ::Abi,
symbol: &str,
@@ -412,7 +413,7 @@ impl wasmtime_environ::component::ComponentCompiler for NoInlineCompiler {
let mut body = self
.0
.component_compiler()
- .compile_intrinsic(tunables, component, intrinsic, abi, symbol)?;
+ .compile_intrinsic(tunables, component, types, intrinsic, abi, symbol)?;
if let Some(c) = self.0.inlining_compiler() {
c.finish_compiling(&mut body, None, symbol)
.map_err(|e| CompileError::Codegen(e.to_string()))?;