Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Tag trampolines' `ValRaw` marshalling with alias regions (#13796)

Nick Fitzgerald8006e967dbed
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index d11da57d11..4d727a25a6 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -38,6 +38,7 @@ enum VmType {
EpochCounter,
BuiltinFunctionsArray,
ComponentBuiltinFunctionsArray,
+ HostValRaw,
}
/// A key that uniquely identifies an alias region across an entire compilation.
@@ -161,6 +162,7 @@ impl AliasRegionKey {
const BUILTIN_FUNCTIONS_KIND: u32 = Self::new_kind(0b11011);
const COMPONENT_BUILTIN_FUNCTIONS_KIND: u32 = Self::new_kind(0b11100);
const UNSAFE_INTRINSIC_MEMORY_KIND: u32 = Self::new_kind(0b11101);
+ const HOST_VAL_RAW_KIND: u32 = Self::new_kind(0b11110);
/// Encode this key into a raw `u32` suitable for use as an
/// `AliasRegionData::user_id`.
@@ -192,6 +194,7 @@ impl AliasRegionKey {
VmType::ComponentBuiltinFunctionsArray => {
Self::COMPONENT_BUILTIN_FUNCTIONS_KIND
}
+ VmType::HostValRaw => Self::HOST_VAL_RAW_KIND,
};
kind | (offset & Self::OFFSET_MASK)
}
@@ -1901,6 +1904,27 @@ where
i32::from(offset),
)
}
+
+ /// Load the `array_call` field of the `VMFuncRef` inlined in a
+ /// `VMArrayCallHostFuncContext`.
+ pub fn vmarray_call_host_func_context_array_call(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ host_func_ctx: ir::Value,
+ ) -> ir::Value {
+ let func_ref = self
+ .offsets
+ .get_ptr_size()
+ .vmarray_call_host_func_context_func_ref();
+ let field = self.offsets.get_ptr_size().vm_func_ref_array_call();
+ let region = self.vmfuncref_region(cursor.func, field.into());
+ cursor.ins().load(
+ self.pointer_type,
+ ir::MemFlagsData::trusted().with_alias_region(Some(region)),
+ host_func_ctx,
+ i32::from(func_ref) + i32::from(field),
+ )
+ }
}
/// `[VMSharedTypeIndex]`-related methods.
@@ -2047,6 +2071,19 @@ where
i32::try_from(offset).unwrap(),
)
}
+
+ /// Get the alias region for the `ValRaw` array used to marshal arguments
+ /// and results across the array calling convention used by various
+ /// trampolines.
+ pub fn host_val_raw_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
+ self.region(
+ func,
+ AliasRegionKey::Vm {
+ ty: VmType::HostValRaw,
+ offset: 0,
+ },
+ )
+ }
}
/// Unsafe intrinsic-related methods.
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index d857b4fced..bfebbd2255 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -41,7 +41,7 @@ use wasmtime_environ::{
Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,
DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameStateSlotBuilder,
FrameTableBuilder, FuncKey, FunctionBodyData, FunctionLoc, GetPtrSize, HostCall,
- InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder, PtrSize,
+ InliningCompiler, ModulePC, ModuleStartup, ModuleTranslation, ModuleTypesBuilder,
StackMapSection, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables,
WasmFuncType, WasmValType, prelude::*,
};
@@ -240,19 +240,18 @@ impl Compiler {
);
// Spill all wasm arguments to the stack in `ValRaw` slots.
- let (args_base, args_len) =
- self.allocate_stack_array_and_spill_args(wasm_func_ty, &mut builder, &args[2..]);
+ let (args_base, args_len) = self.allocate_stack_array_and_spill_args(
+ &mut alias_regions,
+ wasm_func_ty,
+ &mut builder,
+ &args[2..],
+ );
let args_len = builder.ins().iconst(pointer_type, i64::from(args_len));
// Load the actual callee out of the
// `VMArrayCallHostFuncContext::host_func`.
- let ptr_size = isa.pointer_bytes();
- let callee = builder.ins().load(
- pointer_type,
- MemFlagsData::trusted(),
- callee_vmctx,
- ptr_size.vmarray_call_host_func_context_func_ref() + ptr_size.vm_func_ref_array_call(),
- );
+ let callee = alias_regions
+ .vmarray_call_host_func_context_array_call(&mut builder.cursor(), callee_vmctx);
// Do an indirect call to the callee.
let callee_signature = builder.func.import_signature(array_call_sig);
@@ -284,8 +283,13 @@ impl Compiler {
self.raise_if_host_trapped(&mut builder, &mut alias_regions, caller_vmctx, succeeded);
// Return results from the array as native return values.
- let results =
- self.load_values_from_array(wasm_func_ty.results(), &mut builder, args_base, args_len);
+ let results = self.load_values_from_array(
+ &mut alias_regions,
+ wasm_func_ty.results(),
+ &mut builder,
+ args_base,
+ args_len,
+ );
builder.ins().return_(&results);
builder.finalize(self.isa().frontend_config());
@@ -1125,12 +1129,16 @@ impl Compiler {
///
/// The stack slot pointer is returned in addition to the size, in units of
/// `ValRaw`, of the stack slot.
- fn allocate_stack_array_and_spill_args(
+ fn allocate_stack_array_and_spill_args<O>(
&self,
+ alias_regions: &mut AliasRegions<O>,
ty: &WasmFuncType,
builder: &mut FunctionBuilder,
args: &[ir::Value],
- ) -> (Value, u32) {
+ ) -> (Value, u32)
+ where
+ O: GetPtrSize,
+ {
let isa = &*self.isa;
let pointer_type = isa.pointer_type();
@@ -1151,7 +1159,14 @@ impl Compiler {
let values_vec_len = builder
.ins()
.iconst(ir::types::I32, i64::from(values_vec_len));
- self.store_values_to_array(builder, ty.params(), args, values_vec_ptr, values_vec_len);
+ self.store_values_to_array(
+ alias_regions,
+ builder,
+ ty.params(),
+ args,
+ values_vec_ptr,
+ values_vec_len,
+ );
}
(values_vec_ptr, values_vec_len)
@@ -1163,25 +1178,33 @@ impl Compiler {
/// using the array calling convention, or used to store results to the
/// array when implementing a function that exposes the array calling
/// convention.
- fn store_values_to_array(
+ fn store_values_to_array<O>(
&self,
+ alias_regions: &mut AliasRegions<O>,
builder: &mut FunctionBuilder,
types: &[WasmValType],
values: &[Value],
values_vec_ptr: Value,
values_vec_capacity: Value,
- ) {
+ ) where
+ O: GetPtrSize,
+ {
debug_assert_eq!(types.len(), values.len());
self.debug_assert_enough_capacity_for_length(builder, types.len(), values_vec_capacity);
+ if values.is_empty() {
+ return;
+ }
// Note that loads and stores are unconditionally done in the
// little-endian format rather than the host's native-endianness,
// despite this load/store being unrelated to execution in Wasm itself.
// For more details on this see the `ValRaw` type in
// `wasmtime::runtime::vm`.
+ let region = alias_regions.host_val_raw_region(builder.func);
let flags = ir::MemFlagsData::new()
.with_notrap()
- .with_endianness(ir::Endianness::Little);
+ .with_endianness(ir::Endianness::Little)
+ .with_alias_region(Some(region));
let value_size = mem::size_of::<u128>();
for (i, val) in values.iter().copied().enumerate() {
@@ -1202,23 +1225,32 @@ impl Compiler {
/// are building exposes the array calling convention, or it can be used to
/// load results out of the array if the trampoline we are building calls a
/// function that uses the array calling convention.
- fn load_values_from_array(
+ fn load_values_from_array<O>(
&self,
+ alias_regions: &mut AliasRegions<O>,
types: &[WasmValType],
builder: &mut FunctionBuilder,
values_vec_ptr: Value,
values_vec_capacity: Value,
- ) -> Vec<ir::Value> {
+ ) -> Vec<ir::Value>
+ where
+ O: GetPtrSize,
+ {
let isa = &*self.isa;
let value_size = mem::size_of::<u128>();
self.debug_assert_enough_capacity_for_length(builder, types.len(), values_vec_capacity);
+ if types.is_empty() {
+ return Vec::new();
+ }
// Note that this is little-endian like `store_values_to_array` above,
// see notes there for more information.
+ let region = alias_regions.host_val_raw_region(builder.func);
let flags = MemFlagsData::new()
.with_notrap()
- .with_endianness(ir::Endianness::Little);
+ .with_endianness(ir::Endianness::Little)
+ .with_alias_region(Some(region));
let mut results = Vec::new();
for (i, ty) in types.iter().enumerate() {
@@ -1416,6 +1448,7 @@ impl Compiler {
// First load the actual arguments out of the array.
let mut args = self.load_values_from_array(
+ &mut alias_regions,
callee_sig.params(),
&mut builder,
values_vec_ptr,
@@ -1505,6 +1538,7 @@ impl Compiler {
// provided and return "true" for "returned successfully".
builder.switch_to_block(normal_return);
self.store_values_to_array(
+ &mut alias_regions,
&mut builder,
callee_sig.results(),
&normal_return_values,
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index 8a7cc2be67..8dba0992be 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -645,6 +645,7 @@ impl<'a> TrampolineCompiler<'a> {
Trampoline::SyncStartCall { callback } => {
let pointer_type = self.isa.pointer_type();
let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
+ &mut self.alias_regions,
&WasmFuncType::new([], self.signature.results().iter().copied()).panic_on_oom(),
&mut self.builder,
&[],
@@ -877,6 +878,7 @@ impl<'a> TrampolineCompiler<'a> {
let pointer_type = self.isa.pointer_type();
let (ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
+ &mut self.alias_regions,
self.signature,
&mut self.builder,
args,
@@ -953,6 +955,7 @@ impl<'a> TrampolineCompiler<'a> {
// A mixture of the above two.
WasmArgs::InRegistersUpTo(n) => {
let (values_vec_ptr, len) = self.compiler.allocate_stack_array_and_spill_args(
+ &mut self.alias_regions,
&WasmFuncType::new(self.signature.params().iter().skip(n).copied(), [])
.panic_on_oom(),
&mut self.builder,
@@ -1032,6 +1035,7 @@ impl<'a> TrampolineCompiler<'a> {
let len = len.or(val_raw_len).unwrap();
self.raise_if_host_trapped(result.unwrap());
let results = self.compiler.load_values_from_array(
+ &mut self.alias_regions,
self.signature.results(),
&mut self.builder,
ptr,
diff --git a/tests/disas/array-call-trampoline.wat b/tests/disas/array-call-trampoline.wat
new file mode 100644
index 0000000000..673642e702
--- /dev/null
+++ b/tests/disas/array-call-trampoline.wat
@@ -0,0 +1,47 @@
+;;! target = "x86_64"
+;;! test = "optimize"
+;;! filter = "array_to_wasm_trampoline"
+
+(module
+ (func (export "f") (param i32 i64) (result i32 i64)
+ (local.get 0)
+ (local.get 1)
+ )
+)
+;; function u268435456:0(i64 vmctx, i64, i64, i64) -> i8 system_v {
+;; region0 = 4026531840 "HostValRaw+0x0"
+;; region1 = 8 "VMContext+0x8"
+;; region2 = 134217800 "VMStoreContext+0x48"
+;; region3 = 134217792 "VMStoreContext+0x40"
+;; region4 = 134217808 "VMStoreContext+0x50"
+;; region5 = 134217864 "VMStoreContext+0x88"
+;; sig0 = (i64 vmctx, i64, i32, i64) -> i32, i64 tail
+;; fn0 = colocated u0:0 sig0
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; jump block1
+;;
+;; block1:
+;; v4 = load.i32 notrap little region0 v2
+;; v5 = load.i64 notrap little region0 v2+16
+;; v7 = get_frame_pointer.i64
+;; v6 = load.i64 notrap aligned readonly can_move region1 v0+8
+;; store notrap aligned region2 v7, v6+72
+;; v8 = get_stack_pointer.i64
+;; store notrap aligned region3 v8, v6+64
+;; v9 = get_exception_handler_address.i64 block1, 0
+;; store notrap aligned region4 v9, v6+80
+;; try_call fn0(v0, v1, v4, v5), sig0, block2(ret0, ret1), [ default: block3 ]
+;;
+;; block2(v10: i32, v11: i64):
+;; store notrap little region0 v10, v2
+;; store notrap little region0 v11, v2+16
+;; v12 = iconst.i8 1
+;; return v12 ; v12 = 1
+;;
+;; block3:
+;; v13 = iconst.i64 1
+;; store notrap aligned region5 v13, v6+136 ; v13 = 1
+;; v14 = iconst.i8 0
+;; return v14 ; v14 = 0
+;; }