Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

cranelift: stack-switching support (#11003)

Paul Osbornea631d20afa7a
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index f5dd244d8d..d3c7f9ec2f 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -101,6 +101,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
mut args: ArgsAccumulator,
) -> CodegenResult<(u32, Option<usize>)> {
let is_fastcall = call_conv == CallConv::WindowsFastcall;
+ let is_tail = call_conv == CallConv::Tail;
let mut next_gpr = 0;
let mut next_vreg = 0;
@@ -181,6 +182,11 @@ impl ABIMachineSpec for X64ABIMachineSpec {
// This is consistent with LLVM's behavior, and is needed for
// some uses of Cranelift (e.g., the rustc backend).
//
+ // - Otherwise, if the calling convention is Tail, we behave as in
+ // the previous case, even if `enable_llvm_abi_extensions` is not
+ // set in the flags: This is a custom calling convention defined
+ // by Cranelift, LLVM doesn't know about it.
+ //
// - Otherwise, both SysV and Fastcall specify behavior (use of
// vector register, a register pair, or passing by reference
// depending on the case), but for simplicity, we will just panic if
@@ -194,6 +200,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
if param.value_type.bits() > 64
&& !(param.value_type.is_vector() || param.value_type.is_float())
&& !flags.enable_llvm_abi_extensions()
+ && !is_tail
{
panic!(
"i128 args/return values not supported unless LLVM ABI extensions are enabled"
diff --git a/crates/c-api/src/val.rs b/crates/c-api/src/val.rs
index c9c4614743..44823c70f1 100644
--- a/crates/c-api/src/val.rs
+++ b/crates/c-api/src/val.rs
@@ -98,6 +98,7 @@ impl wasm_val_t {
Val::ExternRef(_) => crate::abort("creating a wasm_val_t from an externref"),
Val::ExnRef(_) => crate::abort("creating a wasm_val_t from an exnref"),
Val::V128(_) => crate::abort("creating a wasm_val_t from a v128"),
+ Val::ContRef(_) => crate::abort("creating a wasm_val_t from a contref"),
}
}
@@ -258,6 +259,7 @@ impl wasmtime_val_t {
v128: val.as_u128().to_le_bytes(),
},
},
+ Val::ContRef(_) => crate::abort("contrefs not yet supported in C API (#10248)"),
}
}
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 3871f47b92..877ecc14ad 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -1,4 +1,5 @@
mod gc;
+pub(crate) mod stack_switching;
use crate::compiler::Compiler;
use crate::translate::{
@@ -171,6 +172,16 @@ pub struct FuncEnvironment<'module_environment> {
/// always present even if this is a "leaf" function, as we have to call
/// into the host to trap when signal handlers are disabled.
pub(crate) stack_limit_at_function_entry: Option<ir::GlobalValue>,
+
+ /// Used by the stack switching feature. If set, we have a allocated a
+ /// slot on this function's stack to be used for the
+ /// current stack's `handler_list` field.
+ stack_switching_handler_list_buffer: Option<ir::StackSlot>,
+
+ /// Used by the stack switching feature. If set, we have a allocated a
+ /// slot on this function's stack to be used for the
+ /// current continuation's `values` field.
+ stack_switching_values_buffer: Option<ir::StackSlot>,
}
impl<'module_environment> FuncEnvironment<'module_environment> {
@@ -224,6 +235,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
translation,
stack_limit_at_function_entry: None,
+
+ stack_switching_handler_list_buffer: None,
+ stack_switching_values_buffer: None,
}
}
@@ -1958,8 +1972,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
return CheckIndirectCallTypeSignature::StaticTrap;
}
- WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support.
-
// Engine-indexed types don't show up until runtime and it's a Wasm
// validation error to perform a call through a non-function table,
// so these cases are dynamically not reachable.
@@ -1977,6 +1989,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
| WasmHeapType::Exn
| WasmHeapType::ConcreteExn(_)
| WasmHeapType::NoExn
+ | WasmHeapType::Cont
+ | WasmHeapType::ConcreteCont(_)
+ | WasmHeapType::NoCont
| WasmHeapType::None => {
unreachable!()
}
@@ -2267,7 +2282,9 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environm
let needs_stack_map = match wasm_ty.top() {
WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => true,
WasmHeapTopType::Func => false,
- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
+ // TODO(#10248) Once continuations can be stored on the GC heap, we
+ // will need stack maps for continuation objects.
+ WasmHeapTopType::Cont => false,
};
(ty, needs_stack_map)
}
@@ -2320,22 +2337,32 @@ impl FuncEnvironment<'_> {
let mut pos = builder.cursor();
let table = self.table(table_index);
let ty = table.ref_type.heap_type;
- let grow = if ty.is_vmgcref_type() {
- gc::builtins::table_grow_gc_ref(self, &mut pos.func)?
- } else {
- debug_assert_eq!(ty.top(), WasmHeapTopType::Func);
- self.builtin_functions.table_grow_func_ref(&mut pos.func)
- };
-
let (table_vmctx, defined_table_index) =
self.table_vmctx_and_defined_index(&mut pos, table_index);
-
let index_type = table.idx_type;
let delta = self.cast_index_to_i64(&mut pos, delta, index_type);
- let call_inst = pos
- .ins()
- .call(grow, &[table_vmctx, defined_table_index, delta, init_value]);
- let result = pos.func.dfg.first_result(call_inst);
+
+ let mut args: SmallVec<[_; 6]> = smallvec![table_vmctx, defined_table_index, delta];
+ let grow = match ty.top() {
+ WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {
+ args.push(init_value);
+ gc::builtins::table_grow_gc_ref(self, pos.func)?
+ }
+ WasmHeapTopType::Func => {
+ args.push(init_value);
+ self.builtin_functions.table_grow_func_ref(pos.func)
+ }
+ WasmHeapTopType::Cont => {
+ let (revision, contref) =
+ stack_switching::fatpointer::deconstruct(self, &mut pos, init_value);
+ args.extend_from_slice(&[contref, revision]);
+ stack_switching::builtins::table_grow_cont_obj(self, pos.func)?
+ }
+ };
+
+ let call_inst = pos.ins().call(grow, &args);
+ let result = builder.func.dfg.first_result(call_inst);
+
Ok(self.convert_pointer_to_index_type(builder.cursor(), result, index_type, false))
}
@@ -2367,7 +2394,15 @@ impl FuncEnvironment<'_> {
}
// Continuation types.
- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
+ WasmHeapTopType::Cont => {
+ let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index);
+ Ok(builder.ins().load(
+ stack_switching::fatpointer::fatpointer_type(self),
+ flags,
+ elem_addr,
+ 0,
+ ))
+ }
}
}
@@ -2415,7 +2450,11 @@ impl FuncEnvironment<'_> {
}
// Continuation types.
- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
+ WasmHeapTopType::Cont => {
+ let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index);
+ builder.ins().store(flags, value, elem_addr, 0);
+ Ok(())
+ }
}
}
@@ -2429,21 +2468,31 @@ impl FuncEnvironment<'_> {
) -> WasmResult<()> {
let mut pos = builder.cursor();
let table = self.table(table_index);
- let index_type = table.idx_type;
- let dst = self.cast_index_to_i64(&mut pos, dst, index_type);
- let len = self.cast_index_to_i64(&mut pos, len, index_type);
let ty = table.ref_type.heap_type;
- let libcall = if ty.is_vmgcref_type() {
- gc::builtins::table_fill_gc_ref(self, &mut pos.func)?
- } else {
- debug_assert_eq!(ty.top(), WasmHeapTopType::Func);
- self.builtin_functions.table_fill_func_ref(&mut pos.func)
- };
-
+ let dst = self.cast_index_to_i64(&mut pos, dst, table.idx_type);
+ let len = self.cast_index_to_i64(&mut pos, len, table.idx_type);
let (table_vmctx, table_index) = self.table_vmctx_and_defined_index(&mut pos, table_index);
- pos.ins()
- .call(libcall, &[table_vmctx, table_index, dst, val, len]);
+ let mut args: SmallVec<[_; 6]> = smallvec![table_vmctx, table_index, dst];
+ let libcall = match ty.top() {
+ WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
+ args.push(val);
+ gc::builtins::table_fill_gc_ref(self, &mut pos.func)?
+ }
+ WasmHeapTopType::Func => {
+ args.push(val);
+ self.builtin_functions.table_fill_func_ref(&mut pos.func)
+ }
+ WasmHeapTopType::Cont => {
+ let (revision, contref) =
+ stack_switching::fatpointer::deconstruct(self, &mut pos, val);
+ args.extend_from_slice(&[contref, revision]);
+ stack_switching::builtins::table_fill_cont_obj(self, &mut pos.func)?
+ }
+ };
+
+ args.push(len);
+ builder.ins().call(libcall, &args);
Ok(())
}
@@ -2795,7 +2844,10 @@ impl FuncEnvironment<'_> {
WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {
pos.ins().iconst(types::I32, 0)
}
- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
+ WasmHeapTopType::Cont => {
+ let zero = pos.ins().iconst(self.pointer_type(), 0);
+ stack_switching::fatpointer::construct(self, &mut pos, zero, zero)
+ }
})
}
@@ -2811,9 +2863,18 @@ impl FuncEnvironment<'_> {
return Ok(pos.ins().iconst(ir::types::I32, 0));
}
- let byte_is_null =
- pos.ins()
- .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0);
+ let byte_is_null = match ty.heap_type.top() {
+ WasmHeapTopType::Cont => {
+ let (_revision, contref) =
+ stack_switching::fatpointer::deconstruct(self, &mut pos, value);
+ pos.ins()
+ .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, contref, 0)
+ }
+ _ => pos
+ .ins()
+ .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0),
+ };
+
Ok(pos.ins().uextend(ir::types::I32, byte_is_null))
}
@@ -3556,6 +3617,118 @@ impl FuncEnvironment<'_> {
self.isa.triple().architecture == target_lexicon::Architecture::X86_64
}
+ pub fn translate_cont_bind(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ contobj: ir::Value,
+ args: &[ir::Value],
+ ) -> ir::Value {
+ stack_switching::instructions::translate_cont_bind(self, builder, contobj, args)
+ }
+
+ pub fn translate_cont_new(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ func: ir::Value,
+ arg_types: &[WasmValType],
+ return_types: &[WasmValType],
+ ) -> WasmResult<ir::Value> {
+ stack_switching::instructions::translate_cont_new(
+ self,
+ builder,
+ func,
+ arg_types,
+ return_types,
+ )
+ }
+
+ pub fn translate_resume(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ type_index: u32,
+ contobj: ir::Value,
+ resume_args: &[ir::Value],
+ resumetable: &[(u32, Option<ir::Block>)],
+ ) -> WasmResult<Vec<ir::Value>> {
+ stack_switching::instructions::translate_resume(
+ self,
+ builder,
+ type_index,
+ contobj,
+ resume_args,
+ resumetable,
+ )
+ }
+
+ pub fn translate_suspend(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ tag_index: u32,
+ suspend_args: &[ir::Value],
+ tag_return_types: &[ir::Type],
+ ) -> Vec<ir::Value> {
+ stack_switching::instructions::translate_suspend(
+ self,
+ builder,
+ tag_index,
+ suspend_args,
+ tag_return_types,
+ )
+ }
+
+ /// Translates switch instructions.
+ pub fn translate_switch(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ tag_index: u32,
+ contobj: ir::Value,
+ switch_args: &[ir::Value],
+ return_types: &[ir::Type],
+ ) -> WasmResult<Vec<ir::Value>> {
+ stack_switching::instructions::translate_switch(
+ self,
+ builder,
+ tag_index,
+ contobj,
+ switch_args,
+ return_types,
+ )
+ }
+
+ pub fn continuation_arguments(&self, index: TypeIndex) -> &[WasmValType] {
+ let idx = self.module.types[index].unwrap_module_type_index();
+ self.types[self.types[idx]
+ .unwrap_cont()
+ .clone()
+ .unwrap_module_type_index()]
+ .unwrap_func()
+ .params()
+ }
+
+ pub fn continuation_returns(&self, index: TypeIndex) -> &[WasmValType] {
+ let idx = self.module.types[index].unwrap_module_type_index();
+ self.types[self.types[idx]
+ .unwrap_cont()
+ .clone()
+ .unwrap_module_type_index()]
+ .unwrap_func()
+ .returns()
+ }
+
+ pub fn tag_params(&self, tag_index: TagIndex) -> &[WasmValType] {
+ let idx = self.module.tags[tag_index].signature;
+ self.types[idx.unwrap_module_type_index()]
+ .unwrap_func()
+ .params()
+ }
+
+ pub fn tag_returns(&self, tag_index: TagIndex) -> &[WasmValType] {
+ let idx = self.module.tags[tag_index].signature;
+ self.types[idx.unwrap_module_type_index()]
+ .unwrap_func()
+ .returns()
+ }
+
pub fn use_x86_blendv_for_relaxed_laneselect(&self, ty: Type) -> bool {
self.isa.has_x86_blendv_lowering(ty)
}
@@ -4138,17 +4311,3 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type {
IndexType::I64 => I64,
}
}
-
-/// TODO(10248) This is removed in the next stack switching PR. It stops the
-/// compiler from complaining about the stack switching libcalls being dead
-/// code.
-#[cfg(feature = "stack-switching")]
-#[allow(
- dead_code,
- reason = "Dummy function to suppress more dead code warnings"
-)]
-pub fn use_stack_switching_libcalls() {
- let _ = BuiltinFunctions::cont_new;
- let _ = BuiltinFunctions::table_grow_cont_obj;
- let _ = BuiltinFunctions::table_fill_cont_obj;
-}
diff --git a/crates/cranelift/src/func_environ/stack_switching/control_effect.rs b/crates/cranelift/src/func_environ/stack_switching/control_effect.rs
new file mode 100644
index 0000000000..558fdfb870
--- /dev/null
+++ b/crates/cranelift/src/func_environ/stack_switching/control_effect.rs
@@ -0,0 +1,65 @@
+use cranelift_codegen::ir;
+use cranelift_codegen::ir::InstBuilder;
+use cranelift_codegen::ir::types::{I32, I64};
+use cranelift_frontend::FunctionBuilder;
+
+/// Universal control effect. This structure encodes return signal,
+/// resume signal, suspension signal, and handler index into a
+/// u64 value. This instance is used at compile time. There is a runtime
+/// counterpart in `continuations/src/lib.rs`.
+/// We convert to and from u64 as follows: The low 32 bits of the u64 are the
+/// discriminant, the high 32 bits are the handler_index (if `Suspend`)
+#[derive(Clone, Copy)]
+pub struct ControlEffect(ir::Value);
+
+impl ControlEffect {
+ // Returns the discriminant
+ pub fn signal(&self, builder: &mut FunctionBuilder) -> ir::Value {
+ builder.ins().ushr_imm(self.0, 32)
+ }
+
+ pub fn from_u64(val: ir::Value) -> Self {
+ Self(val)
+ }
+
+ pub fn to_u64(&self) -> ir::Value {
+ self.0
+ }
+
+ pub fn encode_resume(builder: &mut FunctionBuilder) -> Self {
+ let discriminant = builder.ins().iconst(
+ I64,
+ i64::from(wasmtime_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT),
+ );
+ let val = builder.ins().ishl_imm(discriminant, 32);
+
+ Self(val)
+ }
+
+ pub fn encode_switch(builder: &mut FunctionBuilder) -> Self {
+ let discriminant = builder.ins().iconst(
+ I64,
+ i64::from(wasmtime_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT),
+ );
+ let val = builder.ins().ishl_imm(discriminant, 32);
+
+ Self(val)
+ }
+
+ pub fn encode_suspend(builder: &mut FunctionBuilder, handler_index: ir::Value) -> Self {
+ let discriminant = builder.ins().iconst(
+ I64,
+ i64::from(wasmtime_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT),
+ );
+ let val = builder.ins().ishl_imm(discriminant, 32);
+ let handler_index = builder.ins().uextend(I64, handler_index);
+ let val = builder.ins().bor(val, handler_index);
+
+ Self(val)
+ }
+
+ /// Returns the payload of the `Suspend` variant
+ pub fn handler_index(self, builder: &mut FunctionBuilder) -> ir::Value {
+ builder.ins().ireduce(I32, self.0)
+ }
+}
diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs
new file mode 100644
index 0000000000..e2fe365ace
--- /dev/null
+++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs
@@ -0,0 +1,55 @@
+use cranelift_codegen::ir;
+use cranelift_codegen::ir::InstBuilder;
+
+/// Returns the Cranelift type used to represent all of the following:
+/// - wasm values of type `(ref null $ct)` and `(ref $ct)`
+/// - equivalently: runtime values of type `Option<VMContObj>` and `VMContObj`
+/// Note that a `VMContObj` is a fat pointer consisting of a pointer to
+/// `VMContRef` and a pointer-sized revision counter. We represent this as 2 words
+/// (pointer and usize).
+pub fn fatpointer_type(env: &crate::func_environ::FuncEnvironment) -> ir::Type {
+ let ptr_bits = env.pointer_type().bits();
+ ir::Type::int((2 * ptr_bits).try_into().unwrap()).unwrap()
+}
+
+/// Turns a (possibly null) reference to a continuation object into a tuple
+/// (revision, contref_ptr). If `contobj` denotes a wasm null reference, the
+/// contref_ptr part will be a null pointer.
+pub(crate) fn deconstruct<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ pos: &mut cranelift_codegen::cursor::FuncCursor,
+ contobj: ir::Value,
+) -> (ir::Value, ir::Value) {
+ debug_assert_eq!(pos.func.dfg.value_type(contobj), fatpointer_type(env));
+ let ptr_ty = env.pointer_type();
+ let ptr_bits = ptr_ty.bits();
+
+ let contref = pos.ins().ireduce(ptr_ty, contobj);
+ let shifted = pos.ins().ushr_imm(contobj, i64::from(ptr_bits));
+ let revision_counter = pos.ins().ireduce(ptr_ty, shifted);
+
+ (revision_counter, contref)
+}
+
+/// Constructs a continuation object from a given contref and revision pointer.
+/// The contref_addr may be 0, to indicate that we want to build a wasm null reference.
+pub(crate) fn construct<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ pos: &mut cranelift_codegen::cursor::FuncCursor,
+ revision_counter: ir::Value,
+ contref_addr: ir::Value,
+) -> ir::Value {
+ let ptr_ty = env.pointer_type();
+ let ptr_bits = ptr_ty.bits();
+ let fat_ptr_ty = fatpointer_type(env);
+
+ debug_assert_eq!(pos.func.dfg.value_type(contref_addr), ptr_ty);
+ debug_assert_eq!(pos.func.dfg.value_type(revision_counter), ptr_ty);
+
+ let contref_addr = pos.ins().uextend(fat_ptr_ty, contref_addr);
+ let revision_counter = pos.ins().uextend(fat_ptr_ty, revision_counter);
+ let shifted_counter = pos.ins().ishl_imm(revision_counter, i64::from(ptr_bits));
+ let contobj = pos.ins().bor(shifted_counter, contref_addr);
+
+ contobj
+}
diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
new file mode 100644
index 0000000000..383da04f93
--- /dev/null
+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
@@ -0,0 +1,1917 @@
+use cranelift_codegen::ir::BlockArg;
+use itertools::{Either, Itertools};
+
+use cranelift_codegen::ir::condcodes::*;
+use cranelift_codegen::ir::types::*;
+use cranelift_codegen::ir::{self, MemFlags};
+use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData};
+use cranelift_frontend::FunctionBuilder;
+use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wasm_unsupported};
+
+fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult<u8> {
+ match (triple.architecture, triple.operating_system) {
+ (target_lexicon::Architecture::X86_64, target_lexicon::OperatingSystem::Linux) => Ok(24),
+ _ => Err(wasm_unsupported!(
+ "stack switching not supported on {triple}"
+ )),
+ }
+}
+
+use super::control_effect::ControlEffect;
+use super::fatpointer;
+
+/// This module contains compile-time counterparts to types defined elsewhere.
+pub(crate) mod stack_switching_helpers {
+ use core::marker::PhantomData;
+ use cranelift_codegen::ir;
+ use cranelift_codegen::ir::InstBuilder;
+ use cranelift_codegen::ir::condcodes::IntCC;
+ use cranelift_codegen::ir::types::*;
+ use cranelift_codegen::ir::{StackSlot, StackSlotKind::*};
+ use cranelift_frontend::FunctionBuilder;
+ use wasmtime_environ::PtrSize;
+
+ /// Provides information about the layout of a type when it is used as an
+ /// element in a host array. This is used for `VMHostArrayRef`.
+ pub(crate) trait VMHostArrayEntry {
+ /// Returns `(align, size)` in bytes.
+ fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(p: &P) -> (u8, u32);
+ }
+
+ impl VMHostArrayEntry for u128 {
+ fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(_p: &P) -> (u8, u32) {
+ (16, 16)
+ }
+ }
+
+ impl<T> VMHostArrayEntry for *mut T {
+ fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(p: &P) -> (u8, u32) {
+ (p.size(), p.size().into())
+ }
+ }
+
+ #[derive(Copy, Clone)]
+ pub struct VMContRef {
+ pub address: ir::Value,
+ }
+
+ #[derive(Copy, Clone)]
+ pub struct VMHostArrayRef<T> {
+ /// Address of the VMHostArray we are referencing
+ address: ir::Value,
+
+ /// The type parameter T is never used in the fields above. We still
+ /// want to have it for consistency with
+ /// `wasmtime_environ::Vector` and to use it in the associated
+ /// functions.
+ phantom: PhantomData<T>,
+ }
+
+ pub type VMPayloads = VMHostArrayRef<u128>;
+
+ // Actually a vector of *mut VMTagDefinition
+ pub type VMHandlerList = VMHostArrayRef<*mut u8>;
+
+ /// Compile-time representation of wasmtime_environ::VMStackChain,
+ /// consisting of two `ir::Value`s.
+ pub struct VMStackChain {
+ discriminant: ir::Value,
+ payload: ir::Value,
+ }
+
+ pub struct VMCommonStackInformation {
+ pub address: ir::Value,
+ }
+
+ /// Compile-time representation of `crate::runtime::vm::stack::VMContinuationStack`.
+ pub struct VMContinuationStack {
+ /// This is NOT the "top of stack" address of the stack itself. In line
+ /// with how the (runtime) `FiberStack` type works, this is a pointer to
+ /// the TOS address.
+ tos_ptr: ir::Value,
+ }
+
+ impl VMContRef {
+ pub fn new(address: ir::Value) -> VMContRef {
+ VMContRef { address }
+ }
+
+ pub fn args<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> VMPayloads {
+ let offset: i64 = env.offsets.ptr.vmcontref_args().into();
+ let address = builder.ins().iadd_imm(self.address, offset);
+ VMPayloads::new(address)
+ }
+
+ pub fn values<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> VMPayloads {
+ let offset: i64 = env.offsets.ptr.vmcontref_values().into();
+ let address = builder.ins().iadd_imm(self.address, offset);
+ VMPayloads::new(address)
+ }
+
+ pub fn common_stack_information<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> VMCommonStackInformation {
+ let offset: i64 = env.offsets.ptr.vmcontref_common_stack_information().into();
+ let address = builder.ins().iadd_imm(self.address, offset);
+ VMCommonStackInformation { address }
+ }
+
+ /// Stores the parent of this continuation, which may either be another
+ /// continuation or the initial stack. It is therefore represented as a
+ /// `VMStackChain` element.
+ pub fn set_parent_stack_chain<'a>(
+ &mut self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ new_stack_chain: &VMStackChain,
+ ) {
+ let offset = env.offsets.ptr.vmcontref_parent_chain().into();
+ new_stack_chain.store(env, builder, self.address, offset)
+ }
+
+ /// Loads the parent of this continuation, which may either be another
+ /// continuation or the initial stack. It is therefore represented as a
+ /// `VMStackChain` element.
+ pub fn get_parent_stack_chain<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> VMStackChain {
+ let offset = env.offsets.ptr.vmcontref_parent_chain().into();
+ VMStackChain::load(env, builder, self.address, offset, env.pointer_type())
+ }
+
+ pub fn set_last_ancestor<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ last_ancestor: ir::Value,
+ ) {
+ let offset: i32 = env.offsets.ptr.vmcontref_last_ancestor().into();
+ let mem_flags = ir::MemFlags::trusted();
+ builder
+ .ins()
+ .store(mem_flags, last_ancestor, self.address, offset);
+ }
+
+ pub fn get_last_ancestor<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let offset: i32 = env.offsets.ptr.vmcontref_last_ancestor().into();
+ let mem_flags = ir::MemFlags::trusted();
+ builder
+ .ins()
+ .load(env.pointer_type(), mem_flags, self.address, offset)
+ }
+
+ /// Gets the revision counter the a given continuation
+ /// reference.
+ pub fn get_revision<'a>(
+ &mut self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let mem_flags = ir::MemFlags::trusted();
+ let offset: i32 = env.offsets.ptr.vmcontref_revision().into();
+ let revision = builder.ins().load(I64, mem_flags, self.address, offset);
+ revision
+ }
+
+ /// Sets the revision counter on the given continuation
+ /// reference to `revision + 1`.
+
+ pub fn incr_revision<'a>(
+ &mut self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ revision: ir::Value,
+ ) -> ir::Value {
+ let mem_flags = ir::MemFlags::trusted();
+ let offset: i32 = env.offsets.ptr.vmcontref_revision().into();
+ let revision_plus1 = builder.ins().iadd_imm(revision, 1);
+ builder
+ .ins()
+ .store(mem_flags, revision_plus1, self.address, offset);
+ revision_plus1
+ }
+
+ pub fn get_fiber_stack<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> VMContinuationStack {
+ // The top of stack field is stored at offset 0 of the `FiberStack`.
+ let offset: i64 = env.offsets.ptr.vmcontref_stack().into();
+ let fiber_stack_top_of_stack_ptr = builder.ins().iadd_imm(self.address, offset);
+ VMContinuationStack::new(fiber_stack_top_of_stack_ptr)
+ }
+ }
+
+ impl<T: VMHostArrayEntry> VMHostArrayRef<T> {
+ pub(crate) fn new(address: ir::Value) -> Self {
+ Self {
+ address,
+ phantom: PhantomData::default(),
+ }
+ }
+
+ fn get(&self, builder: &mut FunctionBuilder, ty: ir::Type, offset: i32) -> ir::Value {
+ let mem_flags = ir::MemFlags::trusted();
+ builder.ins().load(ty, mem_flags, self.address, offset)
+ }
+
+ fn set<U>(&self, builder: &mut FunctionBuilder, offset: i32, value: ir::Value) {
+ debug_assert_eq!(
+ builder.func.dfg.value_type(value),
+ Type::int_with_byte_size(u16::try_from(core::mem::size_of::<U>()).unwrap())
+ .unwrap()
+ );
+ let mem_flags = ir::MemFlags::trusted();
+ builder.ins().store(mem_flags, value, self.address, offset);
+ }
+
+ pub fn get_data<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let offset = env.offsets.ptr.vmhostarray_data().into();
+ self.get(builder, env.pointer_type(), offset)
+ }
+
+ pub fn get_length<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ // Array length is stored as u32.
+ let offset = env.offsets.ptr.vmhostarray_length().into();
+ self.get(builder, I32, offset)
+ }
+
+ fn set_length<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ length: ir::Value,
+ ) {
+ // Array length is stored as u32.
+ let offset = env.offsets.ptr.vmhostarray_length().into();
+ self.set::<u32>(builder, offset, length);
+ }
+
+ fn set_capacity<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ capacity: ir::Value,
+ ) {
+ // Array capacity is stored as u32.
+ let offset = env.offsets.ptr.vmhostarray_capacity().into();
+ self.set::<u32>(builder, offset, capacity);
+ }
+
+ fn set_data<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ data: ir::Value,
+ ) {
+ debug_assert_eq!(builder.func.dfg.value_type(data), env.pointer_type());
+ let offset: i32 = env.offsets.ptr.vmhostarray_data().into();
+ let mem_flags = ir::MemFlags::trusted();
+ builder.ins().store(mem_flags, data, self.address, offset);
+ }
+
+ /// Returns pointer to next empty slot in data buffer and marks the
+ /// subsequent `arg_count` slots as occupied.
+ pub fn occupy_next_slots<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ arg_count: i32,
+ ) -> ir::Value {
+ let data = self.get_data(env, builder);
+ let original_length = self.get_length(env, builder);
+ let new_length = builder
+ .ins()
+ .iadd_imm(original_length, i64::from(arg_count));
+ self.set_length(env, builder, new_length);
+
+ let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);
+ let original_length = builder.ins().uextend(I64, original_length);
+ let byte_offset = builder
+ .ins()
+ .imul_imm(original_length, i64::from(entry_size));
+ builder.ins().iadd(data, byte_offset)
+ }
+
+ pub fn allocate_or_reuse_stack_slot<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ required_capacity: u32,
+ existing_slot: Option<StackSlot>,
+ ) -> StackSlot {
+ let (align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);
+ let required_size = required_capacity * entry_size;
+
+ match existing_slot {
+ Some(slot) if builder.func.sized_stack_slots[slot].size >= required_size => {
+ let slot_data = &builder.func.sized_stack_slots[slot];
+ debug_assert!(align <= slot_data.align_shift);
+ debug_assert_eq!(slot_data.kind, ExplicitSlot);
+ let existing_capacity = slot_data.size / entry_size;
+
+ let capacity_value = builder.ins().iconst(I32, i64::from(existing_capacity));
+ let existing_data = builder.ins().stack_addr(env.pointer_type(), slot, 0);
+
+ self.set_capacity(env, builder, capacity_value);
+ self.set_data(env, builder, existing_data);
+
+ slot
+ }
+ _ => {
+ let capacity_value = builder.ins().iconst(I32, i64::from(required_capacity));
+ let slot_size = ir::StackSlotData::new(
+ ir::StackSlotKind::ExplicitSlot,
+ required_size,
+ align,
+ );
+ let slot = builder.create_sized_stack_slot(slot_size);
+ let new_data = builder.ins().stack_addr(env.pointer_type(), slot, 0);
+
+ self.set_capacity(env, builder, capacity_value);
+ self.set_data(env, builder, new_data);
+
+ slot
+ }
+ }
+ }
+
+ /// Loads n entries from this Vector object, where n is the length of
+ /// `load_types`, which also gives the types of the values to load.
+ /// Loading starts at index 0 of the Vector object.
+ pub fn load_data_entries<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ load_types: &[ir::Type],
+ ) -> Vec<ir::Value> {
+ let memflags = ir::MemFlags::trusted();
+
+ let data_start_pointer = self.get_data(env, builder);
+ let mut values = vec![];
+ let mut offset = 0;
+ let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);
+ for valtype in load_types {
+ let val = builder
+ .ins()
+ .load(*valtype, memflags, data_start_pointer, offset);
+ values.push(val);
+ offset += i32::try_from(entry_size).unwrap();
+ }
+ values
+ }
+
+ /// Stores the given `values` in this Vector object, beginning at
+ /// index 0. This expects the Vector object to be empty (i.e., current
+ /// length is 0), and to be of sufficient capacity to store |`values`|
+ /// entries.
+ pub fn store_data_entries<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ values: &[ir::Value],
+ ) {
+ let store_count = builder
+ .ins()
+ .iconst(I32, i64::try_from(values.len()).unwrap());
+
+ let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);
+
+ debug_assert!(values.iter().all(|val| {
+ let ty = builder.func.dfg.value_type(*val);
+ let size = ty.bytes();
+ size <= entry_size
+ }));
+
+ let memflags = ir::MemFlags::trusted();
+
+ let data_start_pointer = self.get_data(env, builder);
+
+ let mut offset = 0;
+ for value in values {
+ builder
+ .ins()
+ .store(memflags, *value, data_start_pointer, offset);
+ offset += i32::try_from(entry_size).unwrap();
+ }
+
+ self.set_length(env, builder, store_count);
+ }
+
+ pub fn clear<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ discard_buffer: bool,
+ ) {
+ let zero32 = builder.ins().iconst(I32, 0);
+ self.set_length(env, builder, zero32);
+
+ if discard_buffer {
+ let zero32 = builder.ins().iconst(I32, 0);
+ self.set_capacity(env, builder, zero32);
+
+ let zero_ptr = builder.ins().iconst(env.pointer_type(), 0);
+ self.set_data(env, builder, zero_ptr);
+ }
+ }
+ }
+
+ impl VMStackChain {
+ /// Creates a `Self` corressponding to `VMStackChain::Continuation(contref)`.
+ pub fn from_continuation<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ contref: ir::Value,
+ ) -> VMStackChain {
+ debug_assert_eq!(
+ env.offsets.ptr.size_of_vmstack_chain(),
+ 2 * env.offsets.ptr.size()
+ );
+ let discriminant = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT;
+ let discriminant = builder
+ .ins()
+ .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap());
+ VMStackChain {
+ discriminant,
+ payload: contref,
+ }
+ }
+
+ /// Creates a `Self` corressponding to `VMStackChain::Absent`.
+ pub fn absent<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> VMStackChain {
+ debug_assert_eq!(
+ env.offsets.ptr.size_of_vmstack_chain(),
+ 2 * env.offsets.ptr.size()
+ );
+ let discriminant = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT;
+ let discriminant = builder
+ .ins()
+ .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap());
+ let zero_filler = builder.ins().iconst(env.pointer_type(), 0i64);
+ VMStackChain {
+ discriminant,
+ payload: zero_filler,
+ }
+ }
+
+ pub fn is_initial_stack<'a>(
+ &self,
+ _env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ builder.ins().icmp_imm(
+ IntCC::Equal,
+ self.discriminant,
+ i64::try_from(wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT).unwrap(),
+ )
+ }
+
+ /// Return the two raw `ir::Value`s that represent this VMStackChain.
+ pub fn to_raw_parts(&self) -> [ir::Value; 2] {
+ [self.discriminant, self.payload]
+ }
+
+ /// Construct a `Self` from two raw `ir::Value`s.
+ pub fn from_raw_parts(raw_data: [ir::Value; 2]) -> VMStackChain {
+ VMStackChain {
+ discriminant: raw_data[0],
+ payload: raw_data[1],
+ }
+ }
+
+ /// Load a `VMStackChain` object from the given address.
+ pub fn load<'a>(
+ _env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ pointer: ir::Value,
+ initial_offset: i32,
+ pointer_type: ir::Type,
+ ) -> VMStackChain {
+ let memflags = ir::MemFlags::trusted();
+ let mut offset = initial_offset;
+ let mut data = vec![];
+ for _ in 0..2 {
+ data.push(builder.ins().load(pointer_type, memflags, pointer, offset));
+ offset += i32::try_from(pointer_type.bytes()).unwrap();
+ }
+ let data = <[ir::Value; 2]>::try_from(data).unwrap();
+ Self::from_raw_parts(data)
+ }
+
+ /// Store this `VMStackChain` object at the given address.
+ pub fn store<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ target_pointer: ir::Value,
+ initial_offset: i32,
+ ) {
+ let memflags = ir::MemFlags::trusted();
+ let mut offset = initial_offset;
+ let data = self.to_raw_parts();
+
+ for value in data {
+ debug_assert_eq!(builder.func.dfg.value_type(value), env.pointer_type());
+ builder.ins().store(memflags, value, target_pointer, offset);
+ offset += i32::try_from(env.pointer_type().bytes()).unwrap();
+ }
+ }
+
+ /// Use this only if you've already checked that `self` corresponds to a `VMStackChain::Continuation`.
+ pub fn unchecked_get_continuation(&self) -> ir::Value {
+ self.payload
+ }
+
+ /// Must only be called if `self` represents a `InitialStack` or
+ /// `Continuation` variant. Returns a pointer to the associated
+ /// `CommonStackInformation` object.
+ pub fn get_common_stack_information<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ _builder: &mut FunctionBuilder,
+ ) -> VMCommonStackInformation {
+ // `self` corresponds to a VMStackChain::InitialStack or
+ // VMStackChain::Continuation.
+ // In both cases, the payload is a pointer.
+ let address = self.payload;
+
+ // `obj` is now a pointer to the beginning of either
+ // 1. A `VMContRef` struct (in the case of a
+ // VMStackChain::Continuation)
+ // 2. A CommonStackInformation struct (in the case of
+ // VMStackChain::InitialStack)
+ //
+ // Since a `VMContRef` starts with an (inlined) CommonStackInformation
+ // object at offset 0, we actually have in both cases that `ptr` is
+ // now the address of the beginning of a VMStackLimits object.
+ debug_assert_eq!(env.offsets.ptr.vmcontref_common_stack_information(), 0);
+ VMCommonStackInformation { address }
+ }
+ }
+
+ impl VMCommonStackInformation {
+ fn get_state_ptr<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let offset: i64 = env.offsets.ptr.vmcommon_stack_information_state().into();
+
+ builder.ins().iadd_imm(self.address, offset)
+ }
+
+ fn get_stack_limits_ptr<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let offset: i64 = env.offsets.ptr.vmcommon_stack_information_limits().into();
+
+ builder.ins().iadd_imm(self.address, offset)
+ }
+
+ fn load_state<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let mem_flags = ir::MemFlags::trusted();
+ let state_ptr = self.get_state_ptr(env, builder);
+
+ builder.ins().load(I32, mem_flags, state_ptr, 0)
+ }
+
+ fn set_state_no_payload<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ discriminant: u32,
+ ) {
+ let discriminant = builder.ins().iconst(I32, i64::from(discriminant));
+ let mem_flags = ir::MemFlags::trusted();
+ let state_ptr = self.get_state_ptr(env, builder);
+
+ builder.ins().store(mem_flags, discriminant, state_ptr, 0);
+ }
+
+ pub fn set_state_running<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) {
+ let discriminant = wasmtime_environ::STACK_STATE_RUNNING_DISCRIMINANT;
+ self.set_state_no_payload(env, builder, discriminant);
+ }
+
+ pub fn set_state_parent<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) {
+ let discriminant = wasmtime_environ::STACK_STATE_PARENT_DISCRIMINANT;
+ self.set_state_no_payload(env, builder, discriminant);
+ }
+
+ pub fn set_state_returned<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) {
+ let discriminant = wasmtime_environ::STACK_STATE_RETURNED_DISCRIMINANT;
+ self.set_state_no_payload(env, builder, discriminant);
+ }
+
+ pub fn set_state_suspended<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) {
+ let discriminant = wasmtime_environ::STACK_STATE_SUSPENDED_DISCRIMINANT;
+ self.set_state_no_payload(env, builder, discriminant);
+ }
+
+ /// Checks whether the `VMStackState` reflects that the stack has ever been
+ /// active (instead of just having been allocated, but never resumed).
+ pub fn was_invoked<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let actual_state = self.load_state(env, builder);
+ let allocated = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT;
+ builder
+ .ins()
+ .icmp_imm(IntCC::NotEqual, actual_state, i64::from(allocated))
+ }
+
+ pub fn get_handler_list<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> VMHandlerList {
+ let offset: i64 = env.offsets.ptr.vmcommon_stack_information_handlers().into();
+ let address = builder.ins().iadd_imm(self.address, offset);
+ VMHandlerList::new(address)
+ }
+
+ pub fn get_first_switch_handler_index<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ // Field first_switch_handler_index has type u32
+ let memflags = ir::MemFlags::trusted();
+ let offset: i32 = env
+ .offsets
+ .ptr
+ .vmcommon_stack_information_first_switch_handler_index()
+ .into();
+ builder.ins().load(I32, memflags, self.address, offset)
+ }
+
+ pub fn set_first_switch_handler_index<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ value: ir::Value,
+ ) {
+ // Field first_switch_handler_index has type u32
+ let memflags = ir::MemFlags::trusted();
+ let offset: i32 = env
+ .offsets
+ .ptr
+ .vmcommon_stack_information_first_switch_handler_index()
+ .into();
+ builder.ins().store(memflags, value, self.address, offset);
+ }
+
+ /// Sets `last_wasm_entry_sp` and `stack_limit` fields in
+ /// `VMRuntimelimits` using the values from the `VMStackLimits` of this
+ /// object.
+ pub fn write_limits_to_vmcontext<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ vmruntime_limits_ptr: ir::Value,
+ ) {
+ let stack_limits_ptr = self.get_stack_limits_ptr(env, builder);
+
+ let memflags = ir::MemFlags::trusted();
+
+ let mut copy_to_vm_runtime_limits = |our_offset, their_offset| {
+ let our_value = builder.ins().load(
+ env.pointer_type(),
+ memflags,
+ stack_limits_ptr,
+ i32::from(our_offset),
+ );
+ builder.ins().store(
+ memflags,
+ our_value,
+ vmruntime_limits_ptr,
+ i32::from(their_offset),
+ );
+ };
+
+ let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap();
+ let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit();
+ let last_wasm_entry_fp_offset = env.offsets.ptr.vmstack_limits_last_wasm_entry_fp();
+ copy_to_vm_runtime_limits(
+ stack_limit_offset,
+ pointer_size.vmstore_context_stack_limit(),
+ );
+ copy_to_vm_runtime_limits(
+ last_wasm_entry_fp_offset,
+ pointer_size.vmstore_context_last_wasm_entry_fp(),
+ );
+ }
+
+ /// Overwrites the `last_wasm_entry_fp` field of the `VMStackLimits`
+ /// object in the `VMStackLimits` of this object by loading the corresponding
+ /// field from the `VMRuntimeLimits`.
+ /// If `load_stack_limit` is true, we do the same for the `stack_limit`
+ /// field.
+ pub fn load_limits_from_vmcontext<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ vmruntime_limits_ptr: ir::Value,
+ load_stack_limit: bool,
+ ) {
+ let stack_limits_ptr = self.get_stack_limits_ptr(env, builder);
+
+ let memflags = ir::MemFlags::trusted();
+ let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap();
+
+ let mut copy = |runtime_limits_offset, stack_limits_offset| {
+ let from_vm_runtime_limits = builder.ins().load(
+ env.pointer_type(),
+ memflags,
+ vmruntime_limits_ptr,
+ runtime_limits_offset,
+ );
+ builder.ins().store(
+ memflags,
+ from_vm_runtime_limits,
+ stack_limits_ptr,
+ stack_limits_offset,
+ );
+ };
+
+ let last_wasm_entry_fp_offset = env.offsets.ptr.vmstack_limits_last_wasm_entry_fp();
+ copy(
+ pointer_size.vmstore_context_last_wasm_entry_fp(),
+ last_wasm_entry_fp_offset,
+ );
+
+ if load_stack_limit {
+ let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit();
+ copy(
+ pointer_size.vmstore_context_stack_limit(),
+ stack_limit_offset,
+ );
+ }
+ }
+ }
+
+ impl VMContinuationStack {
+ /// The parameter is NOT the "top of stack" address of the stack itself. In line
+ /// with how the (runtime) `FiberStack` type works, this is a pointer to
+ /// the TOS address.
+ pub fn new(tos_ptr: ir::Value) -> Self {
+ Self { tos_ptr }
+ }
+
+ fn load_top_of_stack<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let mem_flags = ir::MemFlags::trusted();
+ builder
+ .ins()
+ .load(env.pointer_type(), mem_flags, self.tos_ptr, 0)
+ }
+
+ /// Returns address of the control context stored in the stack memory,
+ /// as used by stack_switch instructions.
+ pub fn load_control_context<'a>(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let tos = self.load_top_of_stack(env, builder);
+ // Control context begins 24 bytes below top of stack (see unix.rs)
+ builder.ins().iadd_imm(tos, -0x18)
+ }
+ }
+}
+
+use helpers::VMStackChain;
+use stack_switching_helpers as helpers;
+
+/// Stores the given arguments in the appropriate `VMPayloads` object in the
+/// continuation. If the continuation was never invoked, use the `args` object.
+/// Otherwise, use the `values` object.
+pub(crate) fn vmcontref_store_payloads<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ values: &[ir::Value],
+ contref: ir::Value,
+) {
+ let count =
+ i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32");
+ if values.len() > 0 {
+ let use_args_block = builder.create_block();
+ let use_payloads_block = builder.create_block();
+ let store_data_block = builder.create_block();
+ builder.append_block_param(store_data_block, env.pointer_type());
+
+ let co = helpers::VMContRef::new(contref);
+ let csi = co.common_stack_information(env, builder);
+ let was_invoked = csi.was_invoked(env, builder);
+ builder
+ .ins()
+ .brif(was_invoked, use_payloads_block, &[], use_args_block, &[]);
+
+ {
+ builder.switch_to_block(use_args_block);
+ builder.seal_block(use_args_block);
+
+ let args = co.args(env, builder);
+ let ptr = args.occupy_next_slots(env, builder, count);
+
+ builder
+ .ins()
+ .jump(store_data_block, &[BlockArg::Value(ptr)]);
+ }
+
+ {
+ builder.switch_to_block(use_payloads_block);
+ builder.seal_block(use_payloads_block);
+
+ let payloads = co.values(env, builder);
+
+ // This also checks that the buffer is large enough to hold
+ // `values.len()` more elements.
+ let ptr = payloads.occupy_next_slots(env, builder, count);
+ builder
+ .ins()
+ .jump(store_data_block, &[BlockArg::Value(ptr)]);
+ }
+
+ {
+ builder.switch_to_block(store_data_block);
+ builder.seal_block(store_data_block);
+
+ let ptr = builder.block_params(store_data_block)[0];
+
+ // Store the values.
+ let memflags = ir::MemFlags::trusted();
+ let mut offset = 0;
+ for value in values {
+ builder.ins().store(memflags, *value, ptr, offset);
+ offset += i32::from(env.offsets.ptr.maximum_value_size());
+ }
+ }
+ }
+}
+
+pub(crate) fn tag_address<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ index: u32,
+) -> ir::Value {
+ let vmctx = env.vmctx_val(&mut builder.cursor());
+ let tag_index = wasmtime_environ::TagIndex::from_u32(index);
+ let pointer_type = env.pointer_type();
+ if let Some(def_index) = env.module.defined_tag_index(tag_index) {
+ let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap();
+ builder.ins().iadd_imm(vmctx, i64::from(offset))
+ } else {
+ let offset = i32::try_from(env.offsets.vmctx_vmtag_import_from(tag_index)).unwrap();
+ builder.ins().load(
+ pointer_type,
+ ir::MemFlags::trusted().with_readonly(),
+ vmctx,
+ ir::immediates::Offset32::new(offset),
+ )
+ }
+}
+
+/// Returns the stack chain saved in the given `VMContext`. Note that the
+/// head of the list is the actively running stack (initial stack or
+/// continuation).
+pub fn vmctx_load_stack_chain<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ vmctx: ir::Value,
+) -> VMStackChain {
+ let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into();
+
+ // First we need to get the `VMStoreContext`.
+ let vm_store_context_offset = env.offsets.ptr.vmctx_store_context();
+ let vm_store_context = builder.ins().load(
+ env.pointer_type(),
+ MemFlags::trusted(),
+ vmctx,
+ vm_store_context_offset,
+ );
+
+ VMStackChain::load(
+ env,
+ builder,
+ vm_store_context,
+ stack_chain_offset,
+ env.pointer_type(),
+ )
+}
+
+/// Stores the given stack chain saved in the `VMContext`, overwriting the
+/// exsiting one.
+pub fn vmctx_store_stack_chain<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ vmctx: ir::Value,
+ stack_chain: &VMStackChain,
+) {
+ let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into();
+
+ // First we need to get the `VMStoreContext`.
+ let vm_store_context_offset = env.offsets.ptr.vmctx_store_context();
+ let vm_store_context = builder.ins().load(
+ env.pointer_type(),
+ MemFlags::trusted(),
+ vmctx,
+ vm_store_context_offset,
+ );
+
+ stack_chain.store(env, builder, vm_store_context, stack_chain_offset)
+}
+
+/// Similar to `vmctx_store_stack_chain`, but instead of storing an arbitrary
+/// `VMStackChain`, stores VMStackChain::Continuation(contref)`.
+pub fn vmctx_set_active_continuation<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ vmctx: ir::Value,
+ contref: ir::Value,
+) {
+ let chain = VMStackChain::from_continuation(env, builder, contref);
+ vmctx_store_stack_chain(env, builder, vmctx, &chain)
+}
+
+pub fn vmctx_load_vm_runtime_limits_ptr<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ vmctx: ir::Value,
+) -> ir::Value {
+ let pointer_type = env.pointer_type();
+ let offset = i32::from(env.offsets.ptr.vmctx_store_context());
+
+ // The *pointer* to the VMRuntimeLimits does not change within the
+ // same function, allowing us to set the `read_only` flag.
+ let flags = ir::MemFlags::trusted().with_readonly();
+
+ builder.ins().load(pointer_type, flags, vmctx, offset)
+}
+
+/// This function generates code that searches for a handler for `tag_address`,
+/// which must be a `*mut VMTagDefinition`. The search walks up the chain of
+/// continuations beginning at `start`.
+///
+/// The flag `search_suspend_handlers` determines whether we search for a
+/// suspend or switch handler. Concretely, this influences which part of each
+/// handler list we will search.
+///
+/// We trap if no handler was found.
+///
+/// The returned values are:
+/// 1. The stack (continuation or initial stack, represented as a VMStackChain) in
+/// whose handler list we found the tag (i.e., the stack that performed the
+/// resume instruction that installed handler for the tag).
+/// 2. The continuation whose parent is the stack mentioned in 1.
+/// 3. The index of the handler in the handler list.
+///
+/// In pseudo-code, the generated code's behavior can be expressed as
+/// follows:
+///
+/// chain_link = start
+/// while !chain_link.is_initial_stack() {
+/// contref = chain_link.get_contref()
+/// parent_link = contref.parent
+/// parent_csi = parent_link.get_common_stack_information();
+/// handlers = parent_csi.handlers;
+/// (begin_range, end_range) = if search_suspend_handlers {
+/// (0, parent_csi.first_switch_handler_index)
+/// } else {
+/// (parent_csi.first_switch_handler_index, handlers.length)
+/// };
+/// for index in begin_range..end_range {
+/// if handlers[index] == tag_address {
+/// goto on_match(contref, index)
+/// }
+/// }
+/// chain_link = parent_link
+/// }
+/// trap(unhandled_tag)
+///
+/// on_match(conref : VMContRef, handler_index : u32)
+/// ... execution continues here here ...
+///
+fn search_handler<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ start: &helpers::VMStackChain,
+ tag_address: ir::Value,
+ search_suspend_handlers: bool,
+) -> (VMStackChain, ir::Value, ir::Value) {
+ let handle_link = builder.create_block();
+ let begin_search_handler_list = builder.create_block();
+ let try_index = builder.create_block();
+ let compare_tags = builder.create_block();
+ let on_match = builder.create_block();
+ let on_no_match = builder.create_block();
+ let block_args = start.to_raw_parts().map(|v| BlockArg::Value(v));
+
+ // Terminate previous block:
+ builder.ins().jump(handle_link, &block_args);
+
+ // Block handle_link
+ let chain_link = {
+ builder.append_block_param(handle_link, env.pointer_type());
+ builder.append_block_param(handle_link, env.pointer_type());
+ builder.switch_to_block(handle_link);
+
+ let raw_parts = builder.block_params(handle_link);
+ let chain_link = helpers::VMStackChain::from_raw_parts([raw_parts[0], raw_parts[1]]);
+ let is_initial_stack = chain_link.is_initial_stack(env, builder);
+ builder.ins().brif(
+ is_initial_stack,
+ on_no_match,
+ &[],
+ begin_search_handler_list,
+ &[],
+ );
+ chain_link
+ };
+
+ // Block begin_search_handler_list
+ let (contref, parent_link, handler_list_data_ptr, end_range) = {
+ builder.switch_to_block(begin_search_handler_list);
+ let contref = chain_link.unchecked_get_continuation();
+ let contref = helpers::VMContRef::new(contref);
+
+ let parent_link = contref.get_parent_stack_chain(env, builder);
+ let parent_csi = parent_link.get_common_stack_information(env, builder);
+
+ let handlers = parent_csi.get_handler_list(env, builder);
+ let handler_list_data_ptr = handlers.get_data(env, builder);
+
+ let first_switch_handler_index = parent_csi.get_first_switch_handler_index(env, builder);
+
+ // Note that these indices are inclusive-exclusive, i.e. [begin_range, end_range).
+ let (begin_range, end_range) = if search_suspend_handlers {
+ let zero = builder.ins().iconst(I32, 0);
+ (zero, first_switch_handler_index)
+ } else {
+ let length = handlers.get_length(env, builder);
+ (first_switch_handler_index, length)
+ };
+
+ builder
+ .ins()
+ .jump(try_index, &[BlockArg::Value(begin_range)]);
+
+ (contref, parent_link, handler_list_data_ptr, end_range)
+ };
+
+ // Block try_index
+ let index = {
+ builder.append_block_param(try_index, I32);
+ builder.switch_to_block(try_index);
+ let index = builder.block_params(try_index)[0];
+
+ let in_bounds = builder
+ .ins()
+ .icmp(IntCC::UnsignedLessThan, index, end_range);
+ let block_args = parent_link.to_raw_parts().map(|v| BlockArg::Value(v));
+ builder
+ .ins()
+ .brif(in_bounds, compare_tags, &[], handle_link, &block_args);
+ index
+ };
+
+ // Block compare_tags
+ {
+ builder.switch_to_block(compare_tags);
+
+ let base = handler_list_data_ptr;
+ let entry_size = env.pointer_type().bytes();
+ let offset = builder.ins().imul_imm(index, i64::from(entry_size));
+ let offset = builder.ins().uextend(I64, offset);
+ let entry_address = builder.ins().iadd(base, offset);
+
+ let memflags = ir::MemFlags::trusted();
+
+ let handled_tag = builder
+ .ins()
+ .load(env.pointer_type(), memflags, entry_address, 0);
+
+ let tags_match = builder.ins().icmp(IntCC::Equal, handled_tag, tag_address);
+ let incremented_index = builder.ins().iadd_imm(index, 1);
+ builder.ins().brif(
+ tags_match,
+ on_match,
+ &[],
+ try_index,
+ &[BlockArg::Value(incremented_index)],
+ );
+ }
+
+ // Block on_no_match
+ {
+ builder.switch_to_block(on_no_match);
+ builder.set_cold_block(on_no_match);
+ builder.ins().trap(crate::TRAP_UNHANDLED_TAG);
+ }
+
+ builder.seal_block(handle_link);
+ builder.seal_block(begin_search_handler_list);
+ builder.seal_block(try_index);
+ builder.seal_block(compare_tags);
+ builder.seal_block(on_match);
+ builder.seal_block(on_no_match);
+
+ // final block: on_match
+ builder.switch_to_block(on_match);
+
+ (parent_link, contref.address, index)
+}
+
+pub(crate) fn translate_cont_bind<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ contobj: ir::Value,
+ args: &[ir::Value],
+) -> ir::Value {
+ let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj);
+
+ // The typing rules for cont.bind allow a null reference to be passed to it.
+ builder.ins().trapz(contref, crate::TRAP_NULL_REFERENCE);
+
+ let mut vmcontref = helpers::VMContRef::new(contref);
+ let revision = vmcontref.get_revision(env, builder);
+ let evidence = builder.ins().icmp(IntCC::Equal, witness, revision);
+ builder
+ .ins()
+ .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);
+
+ vmcontref_store_payloads(env, builder, args, contref);
+
+ let revision = vmcontref.incr_revision(env, builder, revision);
+ let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref);
+ contobj
+}
+
+pub(crate) fn translate_cont_new<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ func: ir::Value,
+ arg_types: &[WasmValType],
+ return_types: &[WasmValType],
+) -> WasmResult<ir::Value> {
+ // The typing rules for cont.new allow a null reference to be passed to it.
+ builder.ins().trapz(func, crate::TRAP_NULL_REFERENCE);
+
+ let nargs = builder
+ .ins()
+ .iconst(I32, i64::try_from(arg_types.len()).unwrap());
+ let nreturns = builder
+ .ins()
+ .iconst(I32, i64::try_from(return_types.len()).unwrap());
+
+ let cont_new_func = super::builtins::cont_new(env, &mut builder.func)?;
+ let vmctx = env.vmctx_val(&mut builder.cursor());
+ let call_inst = builder
+ .ins()
+ .call(cont_new_func, &[vmctx, func, nargs, nreturns]);
+ let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap();
+
+ let tag = helpers::VMContRef::new(contref).get_revision(env, builder);
+ let contobj = fatpointer::construct(env, &mut builder.cursor(), tag, contref);
+ Ok(contobj)
+}
+
+pub(crate) fn translate_resume<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ type_index: u32,
+ resume_contobj: ir::Value,
+ resume_args: &[ir::Value],
+ resumetable: &[(u32, Option<ir::Block>)],
+) -> WasmResult<Vec<ir::Value>> {
+ // The resume instruction is the most involved instruction to
+ // compile as it is responsible for both continuation application
+ // and control tag dispatch.
+ //
+ // Here we translate a resume instruction into several basic
+ // blocks as follows:
+ //
+ // previous block
+ // |
+ // |
+ // resume_block
+ // / \
+ // / \
+ // | |
+ // return_block |
+ // suspend block
+ // |
+ // dispatch block
+ //
+ // * resume_block handles continuation arguments and performs
+ // actual stack switch. On ordinary return from resume, it jumps
+ // to the `return_block`, whereas on suspension it jumps to the
+ // `suspend_block`.
+ // * suspend_block is used on suspension, jumps onward to
+ // `dispatch_block`.
+ // * dispatch_block uses a jump table to dispatch to actual
+ // user-defined handler blocks, based on the handler index
+ // provided on suspension. Note that we do not jump to the
+ // handler blocks directly. Instead, each handler block has a
+ // corresponding premable block, which we jump to in order to
+ // reach a particular handler block. The preamble block prepares
+ // the arguments and continuation object to be passed to the
+ // actual handler block.
+ //
+ let resume_block = builder.create_block();
+ let return_block = builder.create_block();
+ let suspend_block = builder.create_block();
+ let dispatch_block = builder.create_block();
+
+ let vmctx = env.vmctx_val(&mut builder.cursor());
+
+ // Split the resumetable into suspend handlers (each represented by the tag
+ // index and handler block) and the switch handlers (represented just by the
+ // tag index). Note that we currently don't remove duplicate tags.
+ let (suspend_handlers, switch_tags): (Vec<(u32, Block)>, Vec<u32>) = resumetable
+ .iter()
+ .partition_map(|(tag_index, block_opt)| match block_opt {
+ Some(block) => Either::Left((*tag_index, *block)),
+ None => Either::Right(*tag_index),
+ });
+
+ // Technically, there is no need to have a dedicated resume block, we could
+ // just put all of its contents into the current block.
+ builder.ins().jump(resume_block, &[]);
+
+ // Resume block: actually resume the continuation chain ending at `resume_contref`.
+ let (resume_result, vm_runtime_limits_ptr, original_stack_chain, new_stack_chain) = {
+ builder.switch_to_block(resume_block);
+ builder.seal_block(resume_block);
+
+ let (witness, resume_contref) =
+ fatpointer::deconstruct(env, &mut builder.cursor(), resume_contobj);
+
+ // The typing rules for resume allow a null reference to be passed to it.
+ builder
+ .ins()
+ .trapz(resume_contref, crate::TRAP_NULL_REFERENCE);
+
+ let mut vmcontref = helpers::VMContRef::new(resume_contref);
+
+ let revision = vmcontref.get_revision(env, builder);
+ let evidence = builder.ins().icmp(IntCC::Equal, revision, witness);
+ builder
+ .ins()
+ .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);
+ let _next_revision = vmcontref.incr_revision(env, builder, revision);
+
+ if resume_args.len() > 0 {
+ // We store the arguments in the `VMContRef` to be resumed.
+ vmcontref_store_payloads(env, builder, resume_args, resume_contref);
+ }
+
+ // Splice together stack chains:
+ // Connect the end of the chain starting at `resume_contref` to the currently active chain.
+ let mut last_ancestor = helpers::VMContRef::new(vmcontref.get_last_ancestor(env, builder));
+
+ // Make the currently running continuation (if any) the parent of the one we are about to resume.
+ let original_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);
+ last_ancestor.set_parent_stack_chain(env, builder, &original_stack_chain);
+
+ // Just for consistency: `vmcontref` is about to get state Running, so let's zero out its last_ancestor field.
+ let zero = builder.ins().iconst(env.pointer_type(), 0);
+ vmcontref.set_last_ancestor(env, builder, zero);
+
+ // We mark `resume_contref` as the currently running one
+ vmctx_set_active_continuation(env, builder, vmctx, resume_contref);
+
+ // Note that the resume_contref libcall a few lines further below
+ // manipulates the stack limits as follows:
+ // 1. Copy stack_limit, last_wasm_entry_sp and last_wasm_exit* values from
+ // VMRuntimeLimits into the currently active continuation (i.e., the
+ // one that will become the parent of the to-be-resumed one)
+ //
+ // 2. Copy `stack_limit` and `last_wasm_entry_sp` in the
+ // `VMStackLimits` of `resume_contref` into the `VMRuntimeLimits`.
+ //
+ // See the comment on `wasmtime_environ::VMStackChain` for a
+ // description of the invariants that we maintain for the various stack
+ // limits.
+
+ // `resume_contref` is now active, and its parent is suspended.
+ let resume_contref = helpers::VMContRef::new(resume_contref);
+ let resume_csi = resume_contref.common_stack_information(env, builder);
+ let parent_csi = original_stack_chain.get_common_stack_information(env, builder);
+ resume_csi.set_state_running(env, builder);
+ parent_csi.set_state_parent(env, builder);
+
+ // We update the `VMStackLimits` of the parent of the continuation to be resumed
+ // as well as the `VMRuntimeLimits`.
+ // See the comment on `wasmtime_environ::VMStackChain` for a description
+ // of the invariants that we maintain for the various stack limits.
+ let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx);
+ parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true);
+ resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);
+
+ // Install handlers in (soon to be) parent's VMHandlerList:
+ // Let the i-th handler clause be (on $tag $block).
+ // Then the i-th entry of the VMHandlerList will be the address of $tag.
+ let handler_list = parent_csi.get_handler_list(env, builder);
+
+ if resumetable.len() > 0 {
+ // Total number of handlers (suspend and switch).
+ let handler_count = u32::try_from(resumetable.len()).unwrap();
+ // Populate the Array's data ptr with a pointer to a sufficiently
+ // large area on this stack.
+ env.stack_switching_handler_list_buffer =
+ Some(handler_list.allocate_or_reuse_stack_slot(
+ env,
+ builder,
+ handler_count,
+ env.stack_switching_handler_list_buffer,
+ ));
+
+ let suspend_handler_count = suspend_handlers.len();
+
+ // All handlers, represented by the indices of the tags they handle.
+ // All the suspend handlers come first, followed by all the switch handlers.
+ let all_handlers = suspend_handlers
+ .iter()
+ .map(|(tag_index, _block)| *tag_index)
+ .chain(switch_tags);
+
+ // Translate all tag indices to tag addresses (i.e., the corresponding *mut VMTagDefinition).
+ let all_tag_addresses: Vec<ir::Value> = all_handlers
+ .map(|tag_index| tag_address(env, builder, tag_index))
+ .collect();
+
+ // Store all tag addresess in the handler list.
+ handler_list.store_data_entries(env, builder, &all_tag_addresses);
+
+ // To enable distinguishing switch and suspend handlers when searching the handler list:
+ // Store at which index the switch handlers start.
+ let first_switch_handler_index = builder
+ .ins()
+ .iconst(I32, i64::try_from(suspend_handler_count).unwrap());
+ parent_csi.set_first_switch_handler_index(env, builder, first_switch_handler_index);
+ }
+
+ let resume_payload = ControlEffect::encode_resume(builder).to_u64();
+
+ // Note that the control context we use for switching is not the one in
+ // (the stack of) resume_contref, but in (the stack of) last_ancestor!
+ let fiber_stack = last_ancestor.get_fiber_stack(env, builder);
+ let control_context_ptr = fiber_stack.load_control_context(env, builder);
+
+ let result =
+ builder
+ .ins()
+ .stack_switch(control_context_ptr, control_context_ptr, resume_payload);
+
+ // At this point we know nothing about the continuation that just
+ // suspended or returned. In particular, it does not have to be what we
+ // called `resume_contref` earlier on. We must reload the information
+ // about the now active continuation from the VMContext.
+ let new_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);
+
+ // Now the parent contref (or initial stack) is active again
+ vmctx_store_stack_chain(env, builder, vmctx, &original_stack_chain);
+ parent_csi.set_state_running(env, builder);
+
+ // Just for consistency: Clear the handler list.
+ handler_list.clear(env, builder, true);
+ parent_csi.set_first_switch_handler_index(env, builder, zero);
+
+ // Extract the result and signal bit.
+ let result = ControlEffect::from_u64(result);
+ let signal = result.signal(builder);
+
+ // Jump to the return block if the result signal is 0, otherwise jump to
+ // the suspend block.
+ builder
+ .ins()
+ .brif(signal, suspend_block, &[], return_block, &[]);
+
+ (
+ result,
+ vm_runtime_limits_ptr,
+ original_stack_chain,
+ new_stack_chain,
+ )
+ };
+
+ // The suspend block: Only used when we suspended, not for returns.
+ // Here we extract the index of the handler to use.
+ let (handler_index, suspended_contref, suspended_contobj) = {
+ builder.switch_to_block(suspend_block);
+ builder.seal_block(suspend_block);
+
+ let suspended_continuation = new_stack_chain.unchecked_get_continuation();
+ let mut suspended_continuation = helpers::VMContRef::new(suspended_continuation);
+ let suspended_csi = suspended_continuation.common_stack_information(env, builder);
+
+ // Note that at the suspend site, we already
+ // 1. Set the state of suspended_continuation to Suspended
+ // 2. Set suspended_continuation.last_ancestor
+ // 3. Broke the continuation chain at suspended_continuation.last_ancestor
+
+ // We store parts of the VMRuntimeLimits into the continuation that just suspended.
+ suspended_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false);
+
+ // Afterwards (!), restore parts of the VMRuntimeLimits from the
+ // parent of the suspended continuation (which is now active).
+ let parent_csi = original_stack_chain.get_common_stack_information(env, builder);
+ parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);
+
+ // Extract the handler index
+ let handler_index = resume_result.handler_index(builder);
+
+ let revision = suspended_continuation.get_revision(env, builder);
+ let suspended_contobj = fatpointer::construct(
+ env,
+ &mut builder.cursor(),
+ revision,
+ suspended_continuation.address,
+ );
+
+ // We need to terminate this block before being allowed to switch to
+ // another one.
+ builder.ins().jump(dispatch_block, &[]);
+
+ (handler_index, suspended_continuation, suspended_contobj)
+ };
+
+ // For technical reasons, the jump table needs to have a default
+ // block. In our case, it should be unreachable, since the handler
+ // index we dispatch on should correspond to a an actual handler
+ // block in the jump table.
+ let jt_default_block = builder.create_block();
+ {
+ builder.switch_to_block(jt_default_block);
+ builder.set_cold_block(jt_default_block);
+
+ builder.ins().trap(crate::TRAP_UNREACHABLE);
+ }
+
+ // We create a preamble block for each of the actual handler blocks: It
+ // reads the necessary arguments and passes them to the actual handler
+ // block, together with the continuation object.
+ let target_preamble_blocks = {
+ let mut preamble_blocks = vec![];
+
+ for &(handle_tag, target_block) in &suspend_handlers {
+ let preamble_block = builder.create_block();
+ preamble_blocks.push(preamble_block);
+ builder.switch_to_block(preamble_block);
+
+ let param_types = env.tag_params(TagIndex::from_u32(handle_tag));
+ let param_types: Vec<ir::Type> = param_types
+ .iter()
+ .map(|wty| crate::value_type(env.isa(), *wty))
+ .collect();
+
+ let values = suspended_contref.values(env, builder);
+ let mut suspend_args: Vec<BlockArg> = values
+ .load_data_entries(env, builder, &param_types)
+ .into_iter()
+ .map(|v| BlockArg::Value(v))
+ .collect();
+
+ // At the suspend site, we store the suspend args in the the
+ // `values` buffer of the VMContRef that was active at the time that
+ // the suspend instruction was performed.
+ suspend_args.push(BlockArg::Value(suspended_contobj));
+
+ // We clear the suspend args. This is mostly for consistency. Note
+ // that we don't zero out the data buffer, we still need it for the
+
+ values.clear(env, builder, false);
+
+ builder.ins().jump(target_block, &suspend_args);
+ }
+
+ preamble_blocks
+ };
+
+ // Dispatch block. All it does is jump to the right premable block based on
+ // the handler index.
+ {
+ builder.switch_to_block(dispatch_block);
+ builder.seal_block(dispatch_block);
+
+ let default_bc = builder.func.dfg.block_call(jt_default_block, &[]);
+
+ let adapter_bcs: Vec<BlockCall> = target_preamble_blocks
+ .iter()
+ .map(|b| builder.func.dfg.block_call(*b, &[]))
+ .collect();
+
+ let jt_data = JumpTableData::new(default_bc, &adapter_bcs);
+ let jt = builder.create_jump_table(jt_data);
+
+ builder.ins().br_table(handler_index, jt);
+
+ for preamble_block in target_preamble_blocks {
+ builder.seal_block(preamble_block);
+ }
+ builder.seal_block(jt_default_block);
+ }
+
+ // Return block: Jumped to by resume block if continuation
+ // returned normally.
+ {
+ builder.switch_to_block(return_block);
+ builder.seal_block(return_block);
+
+ // If we got a return signal, a continuation must have been running.
+ let returned_contref = new_stack_chain.unchecked_get_continuation();
+ let returned_contref = helpers::VMContRef::new(returned_contref);
+
+ // Restore parts of the VMRuntimeLimits from the parent of the
+ // returned continuation (which is now active).
+ let parent_csi = original_stack_chain.get_common_stack_information(env, builder);
+ parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);
+
+ let returned_csi = returned_contref.common_stack_information(env, builder);
+ returned_csi.set_state_returned(env, builder);
+
+ // Load the values returned by the continuation.
+ let return_types: Vec<_> = env
+ .continuation_returns(TypeIndex::from_u32(type_index))
+ .iter()
+ .map(|ty| crate::value_type(env.isa(), *ty))
+ .collect();
+ let payloads = returned_contref.args(env, builder);
+ let return_values = payloads.load_data_entries(env, builder, &return_types);
+ payloads.clear(env, builder, true);
+
+ Ok(return_values)
+ }
+}
+
+pub(crate) fn translate_suspend<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ tag_index: u32,
+ suspend_args: &[ir::Value],
+ tag_return_types: &[ir::Type],
+) -> Vec<ir::Value> {
+ let tag_addr = tag_address(env, builder, tag_index);
+
+ let vmctx = env.vmctx_val(&mut builder.cursor());
+ let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);
+
+ let (_, end_of_chain_contref, handler_index) =
+ search_handler(env, builder, &active_stack_chain, tag_addr, true);
+
+ // If we get here, the search_handler logic succeeded (i.e., did not trap).
+ // Thus, there is at least one parent, so we are not on the initial stack.
+ // Can therefore extract continuation directly.
+ let active_contref = active_stack_chain.unchecked_get_continuation();
+ let active_contref = helpers::VMContRef::new(active_contref);
+ let mut end_of_chain_contref = helpers::VMContRef::new(end_of_chain_contref);
+
+ active_contref.set_last_ancestor(env, builder, end_of_chain_contref.address);
+
+ // In the active_contref's `values` buffer, stack-allocate enough room so that we can
+ // later store the following:
+ // 1. The suspend arguments
+ // 2. Afterwards, the tag return values
+ let values = active_contref.values(env, builder);
+ let required_capacity =
+ u32::try_from(std::cmp::max(suspend_args.len(), tag_return_types.len()))
+ .expect("Number of stack switching payloads should fit in u32");
+
+ if required_capacity > 0 {
+ env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot(
+ env,
+ builder,
+ required_capacity,
+ env.stack_switching_values_buffer,
+ ));
+ }
+
+ if suspend_args.len() > 0 {
+ values.store_data_entries(env, builder, suspend_args);
+ }
+
+ // Set current continuation to suspended and break up handler chain.
+ let active_contref_csi = active_contref.common_stack_information(env, builder);
+ active_contref_csi.set_state_suspended(env, builder);
+ let absent_chain_link = VMStackChain::absent(env, builder);
+ end_of_chain_contref.set_parent_stack_chain(env, builder, &absent_chain_link);
+
+ let suspend_payload = ControlEffect::encode_suspend(builder, handler_index).to_u64();
+
+ // Note that the control context we use for switching is the one
+ // at the end of the chain, not the one in active_contref!
+ // This also means that stack_switch saves the information about
+ // the current stack in the control context located in the stack
+ // of end_of_chain_contref.
+ let fiber_stack = end_of_chain_contref.get_fiber_stack(env, builder);
+ let control_context_ptr = fiber_stack.load_control_context(env, builder);
+
+ builder
+ .ins()
+ .stack_switch(control_context_ptr, control_context_ptr, suspend_payload);
+
+ // The return values of the suspend instruction are the tag return values, saved in the `args` buffer.
+ let values = active_contref.values(env, builder);
+ let return_values = values.load_data_entries(env, builder, tag_return_types);
+ // We effectively consume the values and discard the stack allocated buffer.
+ values.clear(env, builder, true);
+
+ return_values
+}
+
+pub(crate) fn translate_switch<'a>(
+ env: &mut crate::func_environ::FuncEnvironment<'a>,
+ builder: &mut FunctionBuilder,
+ tag_index: u32,
+ switchee_contobj: ir::Value,
+ switch_args: &[ir::Value],
+ return_types: &[ir::Type],
+) -> WasmResult<Vec<ir::Value>> {
+ let vmctx = env.vmctx_val(&mut builder.cursor());
+
+ // Check and increment revision on switchee continuation object (i.e., the
+ // one being switched to). Logically, the switchee continuation extends from
+ // `switchee_contref` to `switchee_contref.last_ancestor` (i.e., the end of
+ // the parent chain starting at `switchee_contref`).
+ let switchee_contref = {
+ let (witness, target_contref) =
+ fatpointer::deconstruct(env, &mut builder.cursor(), switchee_contobj);
+
+ // The typing rules for switch allow a null reference to be passed to it.
+ builder
+ .ins()
+ .trapz(target_contref, crate::TRAP_NULL_REFERENCE);
+
+ let mut target_contref = helpers::VMContRef::new(target_contref);
+
+ let revision = target_contref.get_revision(env, builder);
+ let evidence = builder.ins().icmp(IntCC::Equal, revision, witness);
+ builder
+ .ins()
+ .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);
+ let _next_revision = target_contref.incr_revision(env, builder, revision);
+ target_contref
+ };
+
+ // We create the "switcher continuation" (i.e., the one executing switch)
+ // from the current execution context: Logically, it extends from the
+ // continuation reference executing `switch` (subsequently called
+ // `switcher_contref`) to the immediate child (called
+ // `switcher_contref_last_ancestor`) of the stack with the corresponding
+ // handler (saved in `handler_stack_chain`).
+ let (
+ switcher_contref,
+ switcher_contobj,
+ switcher_contref_last_ancestor,
+ handler_stack_chain,
+ vm_runtime_limits_ptr,
+ ) = {
+ let tag_addr = tag_address(env, builder, tag_index);
+ let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);
+ let (handler_stack_chain, last_ancestor, _handler_index) =
+ search_handler(env, builder, &active_stack_chain, tag_addr, false);
+ let mut last_ancestor = helpers::VMContRef::new(last_ancestor);
+
+ // If we get here, the search_handler logic succeeded (i.e., did not trap).
+ // Thus, there is at least one parent, so we are not on the initial stack.
+ // Can therefore extract continuation directly.
+ let switcher_contref = active_stack_chain.unchecked_get_continuation();
+ let mut switcher_contref = helpers::VMContRef::new(switcher_contref);
+
+ switcher_contref.set_last_ancestor(env, builder, last_ancestor.address);
+
+ // In the switcher_contref's `values` buffer, stack-allocate enough room so that we can
+ // later store `tag_return_types.len()` when resuming the continuation.
+ let values = switcher_contref.values(env, builder);
+ let required_capacity = u32::try_from(return_types.len()).unwrap();
+ if required_capacity > 0 {
+ env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot(
+ env,
+ builder,
+ required_capacity,
+ env.stack_switching_values_buffer,
+ ));
+ }
+
+ let switcher_contref_csi = switcher_contref.common_stack_information(env, builder);
+ switcher_contref_csi.set_state_suspended(env, builder);
+ // We break off `switcher_contref` from the chain of active
+ // continuations, by separating the link between `last_ancestor` and its
+ // parent stack.
+ let absent = VMStackChain::absent(env, builder);
+ last_ancestor.set_parent_stack_chain(env, builder, &absent);
+
+ // Load current runtime limits from `VMContext` and store in the
+ // switcher continuation.
+ let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx);
+ switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false);
+
+ let revision = switcher_contref.get_revision(env, builder);
+ let new_contobj = fatpointer::construct(
+ env,
+ &mut builder.cursor(),
+ revision,
+ switcher_contref.address,
+ );
+
+ (
+ switcher_contref,
+ new_contobj,
+ last_ancestor,
+ handler_stack_chain,
+ vm_runtime_limits_ptr,
+ )
+ };
+
+ // Prepare switchee continuation:
+ // - Store "ordinary" switch arguments as well as the contobj just
+ // synthesized from the current context (i.e., `switcher_contobj`) in the
+ // switchee continuation's payload buffer.
+ // - Splice switchee's continuation chain with handler stack to form new
+ // overall chain of active continuations.
+ let (switchee_contref_csi, switchee_contref_last_ancestor) = {
+ let mut combined_payloads = switch_args.to_vec();
+ combined_payloads.push(switcher_contobj);
+ vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address);
+
+ let switchee_contref_csi = switchee_contref.common_stack_information(env, builder);
+ switchee_contref_csi.set_state_running(env, builder);
+
+ let switchee_contref_last_ancestor = switchee_contref.get_last_ancestor(env, builder);
+ let mut switchee_contref_last_ancestor =
+ helpers::VMContRef::new(switchee_contref_last_ancestor);
+
+ switchee_contref_last_ancestor.set_parent_stack_chain(env, builder, &handler_stack_chain);
+
+ (switchee_contref_csi, switchee_contref_last_ancestor)
+ };
+
+ // Update VMContext/Store: Update active continuation and `VMRuntimeLimits`.
+ {
+ vmctx_set_active_continuation(env, builder, vmctx, switchee_contref.address);
+
+ switchee_contref_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);
+ }
+
+ // Perform actual stack switch
+ {
+ let switcher_last_ancestor_fs =
+ switcher_contref_last_ancestor.get_fiber_stack(env, builder);
+ let switcher_last_ancestor_cc =
+ switcher_last_ancestor_fs.load_control_context(env, builder);
+
+ let switchee_last_ancestor_fs =
+ switchee_contref_last_ancestor.get_fiber_stack(env, builder);
+ let switchee_last_ancestor_cc =
+ switchee_last_ancestor_fs.load_control_context(env, builder);
+
+ // The stack switch involves the following control contexts (e.g., IP,
+ // SP, FP, ...):
+ // - `switchee_last_ancestor_cc` contains the information to continue
+ // execution in the switchee/target continuation.
+ // - `switcher_last_ancestor_cc` contains the information about how to
+ // continue execution once we suspend/return to the stack with the
+ // switch handler.
+ //
+ // In total, the following needs to happen:
+ // 1. Load control context at `switchee_last_ancestor_cc` to perform
+ // stack switch.
+ // 2. Move control context at `switcher_last_ancestor_cc` over to
+ // `switchee_last_ancestor_cc`.
+ // 3. Upon actual switch, save current control context at
+ // `switcher_last_ancestor_cc`.
+ //
+ // We implement this as follows:
+ // 1. We copy `switchee_last_ancestor_cc` to a temporary area on the
+ // stack (`tmp_control_context`).
+ // 2. We copy `switcher_last_ancestor_cc` over to
+ // `switchee_last_ancestor_cc`.
+ // 3. We invoke the stack switch instruction such that it reads from the
+ // temporary area, and writes to `switcher_last_ancestor_cc`.
+ //
+ // Note that the temporary area is only accessed once by the
+ // `stack_switch` instruction emitted later in this block, meaning that we
+ // don't have to worry about its lifetime.
+ //
+ // NOTE(frank-emrich) The implementation below results in one stack slot
+ // being created per switch instruction, even though multiple switch
+ // instructions in the same function could safely re-use the same stack
+ // slot. Thus, we could implement logic for sharing the stack slot by
+ // adding an appropriate field to `FuncEnvironment`.
+ //
+ // NOTE(frank-emrich) We could avoid the copying to a temporary area by
+ // making `stack_switch` do all of the necessary moving itself. However,
+ // that would be a rather ad-hoc change to how the instruction uses the
+ // two pointers given to it.
+
+ let cctx_size = control_context_size(env.isa().triple())?;
+ let slot_size = ir::StackSlotData::new(
+ ir::StackSlotKind::ExplicitSlot,
+ u32::from(cctx_size),
+ u8::try_from(env.pointer_type().bytes()).unwrap(),
+ );
+ let slot = builder.create_sized_stack_slot(slot_size);
+ let tmp_control_context = builder.ins().stack_addr(env.pointer_type(), slot, 0);
+
+ let flags = MemFlags::trusted();
+ let mut offset: i32 = 0;
+ while offset < i32::from(cctx_size) {
+ // switchee_last_ancestor_cc -> tmp control context
+ let tmp1 =
+ builder
+ .ins()
+ .load(env.pointer_type(), flags, switchee_last_ancestor_cc, offset);
+ builder
+ .ins()
+ .store(flags, tmp1, tmp_control_context, offset);
+
+ // switcher_last_ancestor_cc -> switchee_last_ancestor_cc
+ let tmp2 =
+ builder
+ .ins()
+ .load(env.pointer_type(), flags, switcher_last_ancestor_cc, offset);
+ builder
+ .ins()
+ .store(flags, tmp2, switchee_last_ancestor_cc, offset);
+
+ offset += i32::try_from(env.pointer_type().bytes()).unwrap();
+ }
+
+ let switch_payload = ControlEffect::encode_switch(builder).to_u64();
+
+ let _result = builder.ins().stack_switch(
+ switcher_last_ancestor_cc,
+ tmp_control_context,
+ switch_payload,
+ );
+ }
+
+ // After switching back to the original stack: Load return values, they are
+ // stored on the switcher continuation.
+ let return_values = {
+ let payloads = switcher_contref.values(env, builder);
+ let return_values = payloads.load_data_entries(env, builder, return_types);
+ // We consume the values and discard the buffer (allocated on this stack)
+ payloads.clear(env, builder, true);
+ return_values
+ };
+
+ Ok(return_values)
+}
diff --git a/crates/cranelift/src/func_environ/stack_switching/mod.rs b/crates/cranelift/src/func_environ/stack_switching/mod.rs
new file mode 100644
index 0000000000..25381f7279
--- /dev/null
+++ b/crates/cranelift/src/func_environ/stack_switching/mod.rs
@@ -0,0 +1,37 @@
+mod control_effect;
+pub(crate) mod fatpointer;
+pub(crate) mod instructions;
+
+pub(crate) mod builtins {
+ macro_rules! define_builtin_accessors {
+ ( $( $name:ident , )* ) => {
+ $(
+ #[inline]
+ pub fn $name(
+ func_env: &mut crate::func_environ::FuncEnvironment<'_>,
+ func: &mut crate::ir::Function,
+ ) -> wasmtime_environ::WasmResult<crate::ir::FuncRef> {
+ #[cfg(feature = "stack-switching")]
+ {
+ return Ok(func_env.builtin_functions.$name(func));
+ }
+
+ #[cfg(not(feature = "stack-switching"))]
+ {
+ let _ = (func, func_env);
+ return Err(wasmtime_environ::wasm_unsupported!(
+ "support for Wasm Stack Switching disabled at compile time because the `stack-switching` cargo \
+ feature was not enabled"
+ ));
+ }
+ }
+ )*
+ };
+ }
+
+ define_builtin_accessors! {
+ cont_new,
+ table_grow_cont_obj,
+ table_fill_cont_obj,
+ }
+}
diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs
index b25fc8921d..021c042ebd 100644
--- a/crates/cranelift/src/lib.rs
+++ b/crates/cranelift/src/lib.rs
@@ -213,10 +213,9 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type {
match wasm_ht.top() {
WasmHeapTopType::Func => pointer_type,
WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => ir::types::I32,
- WasmHeapTopType::Cont =>
- // TODO(10248) This is added in a follow-up PR
- {
- unimplemented!("codegen for stack switching types not implemented, yet")
+ WasmHeapTopType::Cont => {
+ // VMContObj is 2 * pointer_size (pointer + usize revision)
+ ir::Type::int((2 * pointer_type.bits()).try_into().unwrap()).unwrap()
}
}
}
@@ -376,6 +375,11 @@ impl BuiltinFunctionSignatures {
AbiParam::new(ir::types::I8)
}
+ #[cfg(feature = "stack-switching")]
+ fn size(&self) -> AbiParam {
+ AbiParam::new(self.pointer_type)
+ }
+
fn wasm_signature(&self, builtin: BuiltinFunctionIndex) -> Signature {
let mut _cur = 0;
macro_rules! iter {
diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs
index 359278a2df..1e15bec0c3 100644
--- a/crates/cranelift/src/translate/code_translator.rs
+++ b/crates/cranelift/src/translate/code_translator.rs
@@ -89,7 +89,7 @@ use cranelift_codegen::ir::{BlockArg, types::*};
use cranelift_codegen::packed_option::ReservedValue;
use cranelift_frontend::{FunctionBuilder, Variable};
use itertools::Itertools;
-use smallvec::SmallVec;
+use smallvec::{SmallVec, ToSmallVec};
use std::collections::{HashMap, hash_map};
use std::vec::Vec;
use wasmparser::{FuncValidator, MemArg, Operator, WasmModuleResources};
@@ -2942,35 +2942,101 @@ pub fn translate_operator(
// representation, so we don't actually need to do anything.
}
- Operator::ContNew { cont_type_index: _ } => {
- // TODO(10248) This is added in a follow-up PR
- return Err(wasmtime_environ::WasmError::Unsupported(
- "codegen for stack switching instructions not implemented, yet".to_string(),
- ));
+ Operator::ContNew { cont_type_index } => {
+ let cont_type_index = TypeIndex::from_u32(*cont_type_index);
+ let arg_types: SmallVec<[_; 8]> = environ
+ .continuation_arguments(cont_type_index)
+ .to_smallvec();
+ let result_types: SmallVec<[_; 8]> =
+ environ.continuation_returns(cont_type_index).to_smallvec();
+ let r = stack.pop1();
+ let contobj = environ.translate_cont_new(builder, r, &arg_types, &result_types)?;
+ stack.push1(contobj);
}
Operator::ContBind {
- argument_index: _,
- result_index: _,
+ argument_index,
+ result_index,
} => {
- // TODO(10248) This is added in a follow-up PR
- return Err(wasmtime_environ::WasmError::Unsupported(
- "codegen for stack switching instructions not implemented, yet".to_string(),
- ));
+ let src_types = environ.continuation_arguments(TypeIndex::from_u32(*argument_index));
+ let dst_arity = environ
+ .continuation_arguments(TypeIndex::from_u32(*result_index))
+ .len();
+ let arg_count = src_types.len() - dst_arity;
+
+ let arg_types = &src_types[0..arg_count];
+ for arg_type in arg_types {
+ // We can't bind GC objects using cont.bind at the moment: We
+ // don't have the necessary infrastructure to traverse the
+ // buffers used by cont.bind when looking for GC roots. Thus,
+ // this crude check ensures that these buffers can never contain
+ // GC roots to begin with.
+ if arg_type.is_vmgcref_type_and_not_i31() {
+ return Err(wasmtime_environ::WasmError::Unsupported(
+ "cont.bind does not support GC types at the moment".into(),
+ ));
+ }
+ }
+
+ let (original_contobj, args) = stack.peekn(arg_count + 1).split_last().unwrap();
+
+ let new_contobj = environ.translate_cont_bind(builder, *original_contobj, args);
+
+ stack.popn(arg_count + 1);
+ stack.push1(new_contobj);
}
- Operator::Suspend { tag_index: _ } => {
- // TODO(10248) This is added in a follow-up PR
- return Err(wasmtime_environ::WasmError::Unsupported(
- "codegen for stack switching instructions not implemented, yet".to_string(),
- ));
+ Operator::Suspend { tag_index } => {
+ let tag_index = TagIndex::from_u32(*tag_index);
+ let param_types = environ.tag_params(tag_index).to_vec();
+ let return_types: SmallVec<[_; 8]> = environ
+ .tag_returns(tag_index)
+ .iter()
+ .map(|ty| crate::value_type(environ.isa(), *ty))
+ .collect();
+
+ let params = stack.peekn(param_types.len());
+ let param_count = params.len();
+
+ let return_values =
+ environ.translate_suspend(builder, tag_index.as_u32(), params, &return_types);
+
+ stack.popn(param_count);
+ stack.pushn(&return_values);
}
Operator::Resume {
- cont_type_index: _,
- resume_table: _,
+ cont_type_index,
+ resume_table: wasm_resume_table,
} => {
- // TODO(10248) This is added in a follow-up PR
- return Err(wasmtime_environ::WasmError::Unsupported(
- "codegen for stack switching instructions not implemented, yet".to_string(),
- ));
+ // We translate the block indices in the wasm resume_table to actual Blocks.
+ let mut clif_resume_table = vec![];
+ for handle in &wasm_resume_table.handlers {
+ match handle {
+ wasmparser::Handle::OnLabel { tag, label } => {
+ let i = stack.control_stack.len() - 1 - (*label as usize);
+ let frame = &mut stack.control_stack[i];
+ // This is side-effecting!
+ frame.set_branched_to_exit();
+ clif_resume_table.push((*tag, Some(frame.br_destination())));
+ }
+ wasmparser::Handle::OnSwitch { tag } => {
+ clif_resume_table.push((*tag, None));
+ }
+ }
+ }
+
+ let cont_type_index = TypeIndex::from_u32(*cont_type_index);
+ let arity = environ.continuation_arguments(cont_type_index).len();
+ let (contobj, call_args) = stack.peekn(arity + 1).split_last().unwrap();
+
+ let cont_return_vals = environ.translate_resume(
+ builder,
+ cont_type_index.as_u32(),
+ *contobj,
+ call_args,
+ &clif_resume_table,
+ )?;
+
+ stack.popn(arity + 1); // arguments + continuation
+ stack.pushn(&cont_return_vals);
}
Operator::ResumeThrow {
cont_type_index: _,
@@ -2983,13 +3049,51 @@ pub fn translate_operator(
));
}
Operator::Switch {
- cont_type_index: _,
- tag_index: _,
+ cont_type_index,
+ tag_index,
} => {
- // TODO(10248) This is added in a follow-up PR
- return Err(wasmtime_environ::WasmError::Unsupported(
- "codegen for stack switching instructions not implemented, yet".to_string(),
- ));
+ // Arguments of the continuation we are going to switch to
+ let continuation_argument_types: SmallVec<[_; 8]> = environ
+ .continuation_arguments(TypeIndex::from_u32(*cont_type_index))
+ .to_smallvec();
+ // Arity includes the continuation argument
+ let arity = continuation_argument_types.len();
+ let (contobj, switch_args) = stack.peekn(arity).split_last().unwrap();
+
+ // Type of the continuation we are going to create by suspending the
+ // currently running stack
+ let current_continuation_type = continuation_argument_types.last().unwrap();
+ let current_continuation_type = current_continuation_type.unwrap_ref_type();
+
+ // Argument types of current_continuation_type. These will in turn
+ // be the types of the arguments we receive when someone switches
+ // back to this switch instruction
+ let current_continuation_arg_types: SmallVec<[_; 8]> =
+ match current_continuation_type.heap_type {
+ WasmHeapType::ConcreteCont(index) => {
+ let mti = index
+ .as_module_type_index()
+ .expect("Only supporting module type indices on switch for now");
+
+ environ
+ .continuation_arguments(TypeIndex::from_u32(mti.as_u32()))
+ .iter()
+ .map(|ty| crate::value_type(environ.isa(), *ty))
+ .collect()
+ }
+ _ => panic!("Invalid type on switch"),
+ };
+
+ let switch_return_values = environ.translate_switch(
+ builder,
+ *tag_index,
+ *contobj,
+ switch_args,
+ &current_continuation_arg_types,
+ )?;
+
+ stack.popn(arity);
+ stack.pushn(&switch_return_values)
}
Operator::GlobalAtomicGet { .. }
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index 299ed239e2..65f0225db0 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -233,12 +233,12 @@ macro_rules! foreach_builtin_function {
// denote the continuation being `None`, `init_contref`
// may be 0.
#[cfg(feature = "stack-switching")]
- table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: u64) -> pointer;
+ table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: size) -> pointer;
// `value_contref` and `value_revision` together encode
// the Option<VMContObj>, as in previous libcall.
#[cfg(feature = "stack-switching")]
- table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: u64, len: u64) -> bool;
+ table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: size, len: u64) -> bool;
// Return the instance ID for a given vmctx.
#[cfg(feature = "gc")]
diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs
index 92104ad181..d9e771b02f 100644
--- a/crates/environ/src/gc.rs
+++ b/crates/environ/src/gc.rs
@@ -40,15 +40,10 @@ pub const VM_GC_HEADER_TYPE_INDEX_OFFSET: u32 = 4;
/// Get the byte size of the given Wasm type when it is stored inside the GC
/// heap.
pub fn byte_size_of_wasm_ty_in_gc_heap(ty: &WasmStorageType) -> u32 {
- use crate::{WasmHeapType::*, WasmRefType};
match ty {
WasmStorageType::I8 => 1,
WasmStorageType::I16 => 2,
WasmStorageType::Val(ty) => match ty {
- WasmValType::Ref(WasmRefType {
- nullable: _,
- heap_type: ConcreteCont(_) | Cont,
- }) => unimplemented!("Stack switching feature not compatbile with GC, yet"),
WasmValType::I32 | WasmValType::F32 | WasmValType::Ref(_) => 4,
WasmValType::I64 | WasmValType::F64 => 8,
WasmValType::V128 => 16,
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 1b12ed9a61..71fb7b045d 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -161,6 +161,12 @@ pub trait PtrSize {
4
}
+ /// This is the size of the largest value type (i.e. a V128).
+ #[inline]
+ fn maximum_value_size(&self) -> u8 {
+ self.size_of_vmglobal_definition()
+ }
+
// Offsets within `VMStoreContext`
/// Return the offset of the `fuel_consumed` field of `VMStoreContext`
@@ -333,6 +339,28 @@ pub trait PtrSize {
.unwrap()
}
+ // Offsets within `VMContObj`
+
+ /// Return the offset of `VMContObj::contref`
+ fn vmcontobj_contref(&self) -> u8 {
+ 0
+ }
+
+ /// Return the offset of `VMContObj::revision`
+ fn vmcontobj_revision(&self) -> u8 {
+ self.size()
+ }
+
+ /// Return the size of `VMContObj`.
+ fn size_of_vmcontobj(&self) -> u8 {
+ u8::try_from(align(
+ u32::from(self.vmcontobj_revision())
+ + u32::try_from(core::mem::size_of::<usize>()).unwrap(),
+ u32::from(self.size()),
+ ))
+ .unwrap()
+ }
+
// Offsets within `VMContRef`
/// Return the offset of `VMContRef::common_stack_information`.
@@ -362,7 +390,7 @@ pub trait PtrSize {
/// Return the offset of `VMContRef::stack`.
fn vmcontref_stack(&self) -> u8 {
- self.vmcontref_revision() + 8
+ self.vmcontref_revision() + self.size()
}
/// Return the offset of `VMContRef::args`.
diff --git a/crates/fuzzing/src/generators/value.rs b/crates/fuzzing/src/generators/value.rs
index 697982a120..685c655400 100644
--- a/crates/fuzzing/src/generators/value.rs
+++ b/crates/fuzzing/src/generators/value.rs
@@ -18,6 +18,7 @@ pub enum DiffValue {
ExternRef { null: bool },
AnyRef { null: bool },
ExnRef { null: bool },
+ ContRef { null: bool },
}
impl DiffValue {
@@ -32,6 +33,7 @@ impl DiffValue {
DiffValue::ExternRef { .. } => DiffValueType::ExternRef,
DiffValue::AnyRef { .. } => DiffValueType::AnyRef,
DiffValue::ExnRef { .. } => DiffValueType::ExnRef,
+ DiffValue::ContRef { .. } => DiffValueType::ContRef,
}
}
@@ -189,6 +191,7 @@ impl DiffValue {
ExternRef => DiffValue::ExternRef { null: true },
AnyRef => DiffValue::AnyRef { null: true },
ExnRef => DiffValue::ExnRef { null: true },
+ ContRef => DiffValue::ContRef { null: true },
};
arbitrary::Result::Ok(val)
}
@@ -236,6 +239,7 @@ impl Hash for DiffValue {
DiffValue::FuncRef { null } => null.hash(state),
DiffValue::AnyRef { null } => null.hash(state),
DiffValue::ExnRef { null } => null.hash(state),
+ DiffValue::ContRef { null } => null.hash(state),
}
}
}
@@ -273,6 +277,7 @@ impl PartialEq for DiffValue {
(Self::ExternRef { null: a }, Self::ExternRef { null: b }) => a == b,
(Self::AnyRef { null: a }, Self::AnyRef { null: b }) => a == b,
(Self::ExnRef { null: a }, Self::ExnRef { null: b }) => a == b,
+ (Self::ContRef { null: a }, Self::ContRef { null: b }) => a == b,
_ => false,
}
}
@@ -291,6 +296,7 @@ pub enum DiffValueType {
ExternRef,
AnyRef,
ExnRef,
+ ContRef,
}
impl TryFrom<wasmtime::ValType> for DiffValueType {
@@ -310,6 +316,7 @@ impl TryFrom<wasmtime::ValType> for DiffValueType {
(true, HeapType::I31) => Ok(Self::AnyRef),
(true, HeapType::None) => Ok(Self::AnyRef),
(true, HeapType::Exn) => Ok(Self::ExnRef),
+ (true, HeapType::Cont) => Ok(Self::ContRef),
_ => Err("non-null reference types are not supported yet"),
},
}
diff --git a/crates/fuzzing/src/oracles/diff_spec.rs b/crates/fuzzing/src/oracles/diff_spec.rs
index c5b9edbafc..85fab4b5d7 100644
--- a/crates/fuzzing/src/oracles/diff_spec.rs
+++ b/crates/fuzzing/src/oracles/diff_spec.rs
@@ -107,7 +107,8 @@ impl From<&DiffValue> for SpecValue {
DiffValue::FuncRef { .. }
| DiffValue::ExternRef { .. }
| DiffValue::AnyRef { .. }
- | DiffValue::ExnRef { .. } => {
+ | DiffValue::ExnRef { .. }
+ | DiffValue::ContRef { .. } => {
unimplemented!()
}
}
diff --git a/crates/fuzzing/src/oracles/diff_v8.rs b/crates/fuzzing/src/oracles/diff_v8.rs
index 7e0ed12e21..eaf612a4be 100644
--- a/crates/fuzzing/src/oracles/diff_v8.rs
+++ b/crates/fuzzing/src/oracles/diff_v8.rs
@@ -191,6 +191,7 @@ impl DiffInstance for V8Instance {
DiffValue::V128(_) => return Ok(None),
DiffValue::AnyRef { .. } => unimplemented!(),
DiffValue::ExnRef { .. } => unimplemented!(),
+ DiffValue::ContRef { .. } => unimplemented!(),
});
}
// JS doesn't support v128 return values
@@ -316,6 +317,7 @@ fn get_diff_value(
DiffValueType::AnyRef => unimplemented!(),
DiffValueType::ExnRef => unimplemented!(),
DiffValueType::V128 => unreachable!(),
+ DiffValueType::ContRef => unimplemented!(),
}
}
diff --git a/crates/fuzzing/src/oracles/diff_wasmi.rs b/crates/fuzzing/src/oracles/diff_wasmi.rs
index c591c218cd..dbed7f2f27 100644
--- a/crates/fuzzing/src/oracles/diff_wasmi.rs
+++ b/crates/fuzzing/src/oracles/diff_wasmi.rs
@@ -195,6 +195,7 @@ impl From<&DiffValue> for wasmi::Val {
}
DiffValue::AnyRef { .. } => unimplemented!(),
DiffValue::ExnRef { .. } => unimplemented!(),
+ DiffValue::ContRef { .. } => unimplemented!(),
}
}
}
diff --git a/crates/fuzzing/src/oracles/diff_wasmtime.rs b/crates/fuzzing/src/oracles/diff_wasmtime.rs
index 53c54b73a2..f600005164 100644
--- a/crates/fuzzing/src/oracles/diff_wasmtime.rs
+++ b/crates/fuzzing/src/oracles/diff_wasmtime.rs
@@ -227,6 +227,10 @@ impl From<&DiffValue> for Val {
assert!(null);
Val::ExnRef(None)
}
+ DiffValue::ContRef { null } => {
+ assert!(null);
+ Val::ExnRef(None)
+ }
}
}
}
@@ -243,6 +247,7 @@ impl From<Val> for DiffValue {
Val::FuncRef(r) => DiffValue::FuncRef { null: r.is_none() },
Val::AnyRef(r) => DiffValue::AnyRef { null: r.is_none() },
Val::ExnRef(e) => DiffValue::ExnRef { null: e.is_none() },
+ Val::ContRef(c) => DiffValue::ContRef { null: c.is_none() },
}
}
}
diff --git a/crates/wasmtime/src/runtime/coredump.rs b/crates/wasmtime/src/runtime/coredump.rs
index 517c2fbb24..ac557b7f60 100644
--- a/crates/wasmtime/src/runtime/coredump.rs
+++ b/crates/wasmtime/src/runtime/coredump.rs
@@ -210,6 +210,12 @@ impl WasmCoreDump {
ty: wasm_encoder::AbstractHeapType::Exn,
})
}
+ Val::ContRef(_) => {
+ wasm_encoder::ConstExpr::ref_null(wasm_encoder::HeapType::Abstract {
+ shared: false,
+ ty: wasm_encoder::AbstractHeapType::Cont,
+ })
+ }
};
globals.global(
wasm_encoder::GlobalType {
diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs
index 11d7d8427b..88daa9c326 100644
--- a/crates/wasmtime/src/runtime/externals/global.rs
+++ b/crates/wasmtime/src/runtime/externals/global.rs
@@ -280,6 +280,16 @@ impl Global {
let new = new.as_ref();
definition.write_gc_ref(&mut store, new);
}
+ Val::ContRef(None) => {
+ // Allow null continuation references for globals - these are just placeholders
+ definition.write_gc_ref(&mut store, None);
+ }
+ Val::ContRef(Some(_)) => {
+ // TODO(#10248): Implement non-null global continuation reference handling
+ return Err(anyhow::anyhow!(
+ "setting non-null continuation references in globals not yet supported"
+ ));
+ }
}
}
Ok(())
diff --git a/crates/wasmtime/src/runtime/trampoline/global.rs b/crates/wasmtime/src/runtime/trampoline/global.rs
index 2fad1d81aa..5ef0a6ddf2 100644
--- a/crates/wasmtime/src/runtime/trampoline/global.rs
+++ b/crates/wasmtime/src/runtime/trampoline/global.rs
@@ -70,6 +70,16 @@ pub fn generate_global_export(
let new = new.as_ref();
global.write_gc_ref(&mut store, new);
}
+ Val::ContRef(None) => {
+ // Allow null continuation references for trampoline globals - these are just placeholders
+ global.write_gc_ref(&mut store, None);
+ }
+ Val::ContRef(Some(_)) => {
+ // TODO(#10248): Implement non-null trampoline continuation reference handling
+ return Err(anyhow::anyhow!(
+ "non-null continuation references in trampoline globals not yet supported"
+ ));
+ }
}
}
diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rs
index e572b2dfb1..15aa627b8b 100644
--- a/crates/wasmtime/src/runtime/values.rs
+++ b/crates/wasmtime/src/runtime/values.rs
@@ -8,6 +8,13 @@ use wasmtime_environ::WasmHeapTopType;
pub use crate::runtime::vm::ValRaw;
+/// A stub implementation for continuation references.
+///
+/// This is a placeholder until continuation objects are fully integrated
+/// with the GC system (see #10248).
+#[derive(Debug, Clone, Copy)]
+pub struct ContRef;
+
/// Possible runtime values that a WebAssembly module can either consume or
/// produce.
///
@@ -50,6 +57,12 @@ pub enum Val {
/// An exception reference.
ExnRef(Option<Rooted<ExnRef>>),
+
+ /// A continuation reference.
+ ///
+ /// Note: This is currently a stub implementation as continuation objects
+ /// are not yet fully integrated with the GC system. See #10248.
+ ContRef(Option<ContRef>),
}
macro_rules! accessors {
@@ -118,7 +131,7 @@ impl Val {
WasmHeapTopType::Extern => Val::ExternRef(None),
WasmHeapTopType::Any => Val::AnyRef(None),
WasmHeapTopType::Exn => Val::ExnRef(None),
- WasmHeapTopType::Cont => todo!(), // FIXME(#10248)
+ WasmHeapTopType::Cont => Val::ContRef(None),
}
}
@@ -177,6 +190,12 @@ impl Val {
Val::AnyRef(Some(a)) => ValType::Ref(RefType::new(false, a._ty(store)?)),
Val::ExnRef(None) => ValType::NULLEXNREF,
Val::ExnRef(Some(e)) => ValType::Ref(RefType::new(false, e._ty(store)?.into())),
+ Val::ContRef(_) => {
+ // TODO(#10248): Return proper continuation reference type when available
+ return Err(anyhow::anyhow!(
+ "continuation references not yet supported in embedder API"
+ ));
+ }
})
}
@@ -216,7 +235,8 @@ impl Val {
| (Val::FuncRef(_), _)
| (Val::ExternRef(_), _)
| (Val::AnyRef(_), _)
- | (Val::ExnRef(_), _) => false,
+ | (Val::ExnRef(_), _)
+ | (Val::ContRef(_), _) => false,
})
}
@@ -268,6 +288,12 @@ impl Val {
Some(f) => f.to_raw(store),
None => ptr::null_mut(),
})),
+ Val::ContRef(_) => {
+ // TODO(#10248): Implement proper continuation reference to_raw conversion
+ Err(anyhow::anyhow!(
+ "continuation references not yet supported in to_raw conversion"
+ ))
+ }
}
}
@@ -363,6 +389,7 @@ impl Val {
Val::AnyRef(a) => Some(Ref::Any(a)),
Val::ExnRef(e) => Some(Ref::Exn(e)),
Val::I32(_) | Val::I64(_) | Val::F32(_) | Val::F64(_) | Val::V128(_) => None,
+ Val::ContRef(_) => None, // TODO(#10248): Return proper Ref::Cont when available
}
}
@@ -509,6 +536,9 @@ impl Val {
// particular store, so they're always considered as "yes I came
// from that store",
Val::I32(_) | Val::I64(_) | Val::F32(_) | Val::F64(_) | Val::V128(_) => true,
+
+ // Continuation references are not yet associated with stores
+ Val::ContRef(_) => true, // TODO(#10248): Proper store association when implemented
}
}
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs
index 04982d3c49..5d6f525f8a 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs
@@ -276,6 +276,12 @@ impl VMArrayRef {
.gc_object_data(self.as_gc_ref())
.write_u32(offset, id.into_raw());
}
+ Val::ContRef(_) => {
+ // TODO(#10248): Implement array continuation reference element handling
+ return Err(anyhow::anyhow!(
+ "setting continuation references in array elements not yet supported"
+ ));
+ }
}
Ok(())
}
@@ -383,6 +389,12 @@ impl VMArrayRef {
.gc_object_data(self.as_gc_ref())
.write_u32(offset, id.into_raw());
}
+ Val::ContRef(_) => {
+ // TODO(#10248): Implement array continuation reference init handling
+ return Err(anyhow::anyhow!(
+ "initializing continuation references in array elements not yet supported"
+ ));
+ }
}
Ok(())
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs
index 364faf1c01..a8ecfea074 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs
@@ -233,6 +233,12 @@ impl VMStructRef {
.gc_object_data(self.as_gc_ref())
.write_u32(offset, id.into_raw());
}
+ Val::ContRef(_) => {
+ // TODO(#10248): Implement struct continuation reference field handling
+ return Err(anyhow::anyhow!(
+ "setting continuation references in struct fields not yet supported"
+ ));
+ }
}
Ok(())
}
@@ -386,6 +392,12 @@ pub(crate) fn initialize_field_impl(
.gc_object_data(gc_ref)
.write_u32(offset, id.into_raw());
}
+ Val::ContRef(_) => {
+ // TODO(#10248): Implement struct continuation reference field init handling
+ return Err(anyhow::anyhow!(
+ "initializing continuation references in struct fields not yet supported"
+ ));
+ }
}
Ok(())
}
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 3e4ca7c5d4..20b3e7154f 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -157,6 +157,7 @@ pub mod raw {
(@ty f64x2) => (f64x2);
(@ty bool) => (bool);
(@ty pointer) => (*mut u8);
+ (@ty size) => (usize);
}
wasmtime_environ::foreach_builtin_function!(libcall);
@@ -339,7 +340,7 @@ unsafe fn table_grow_cont_obj(
// The following two values together form the initial Option<VMContObj>.
// A None value is indicated by the pointer being null.
init_value_contref: *mut u8,
- init_value_revision: u64,
+ init_value_revision: usize,
) -> Result<Option<AllocationSize>> {
let defined_table_index = DefinedTableIndex::from_u32(defined_table_index);
let element = unsafe { VMContObj::from_raw_parts(init_value_contref, init_value_revision) };
@@ -416,7 +417,7 @@ unsafe fn table_fill_cont_obj(
table_index: u32,
dst: u64,
value_contref: *mut u8,
- value_revision: u64,
+ value_revision: usize,
len: u64,
) -> Result<()> {
let instance = store.instance_mut(instance);
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs
index 2a63d0baa3..32c2c27533 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs
@@ -39,19 +39,15 @@ pub use stack::*;
/// For performance reasons, the VMContRef at the bottom of this chain
/// (i.e., the one pointed to by the VMContObj) has a pointer to the
/// other end of the chain (i.e., its last ancestor).
-// FIXME(frank-emrich) Does this actually need to be 16-byte aligned any
-// more? Now that we use I128 on the Cranelift side (see
-// [wasmtime_cranelift::stack_switching::fatpointer::pointer_type]), it
-// should be fine to use the natural alignment of the type.
-#[repr(C, align(16))]
+#[repr(C)]
#[derive(Debug, Clone, Copy)]
pub struct VMContObj {
- pub revision: u64,
pub contref: NonNull<VMContRef>,
+ pub revision: usize,
}
impl VMContObj {
- pub fn new(contref: NonNull<VMContRef>, revision: u64) -> Self {
+ pub fn new(contref: NonNull<VMContRef>, revision: usize) -> Self {
Self { contref, revision }
}
@@ -63,7 +59,7 @@ impl VMContObj {
///
/// Behavior will be undefined if a pointer to data that is not a
/// VMContRef is provided.
- pub unsafe fn from_raw_parts(contref: *mut u8, revision: u64) -> Option<Self> {
+ pub unsafe fn from_raw_parts(contref: *mut u8, revision: usize) -> Option<Self> {
NonNull::new(contref.cast::<VMContRef>()).map(|contref| Self::new(contref, revision))
}
}
@@ -208,7 +204,7 @@ pub struct VMContRef {
pub last_ancestor: *mut VMContRef,
/// Revision counter.
- pub revision: u64,
+ pub revision: usize,
/// The underlying stack.
pub stack: VMContinuationStack,
@@ -627,6 +623,24 @@ mod tests {
);
}
+ #[test]
+ fn check_vm_contobj_offsets() {
+ let module = Module::new();
+ let offsets = VMOffsets::new(HostPtr, &module);
+ assert_eq!(
+ offset_of!(VMContObj, contref),
+ usize::from(offsets.ptr.vmcontobj_contref())
+ );
+ assert_eq!(
+ offset_of!(VMContObj, revision),
+ usize::from(offsets.ptr.vmcontobj_revision())
+ );
+ assert_eq!(
+ size_of::<VMContObj>(),
+ usize::from(offsets.ptr.size_of_vmcontobj())
+ )
+ }
+
#[test]
fn check_vm_contref_offsets() {
let module = Module::new();
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 818b089408..220e973082 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -1056,6 +1056,7 @@ macro_rules! define_builtin_array {
(@ty f64x2) => (f64x2);
(@ty bool) => (bool);
(@ty pointer) => (*mut u8);
+ (@ty size) => (usize);
(@ty vmctx) => (NonNull<VMContext>);
}
diff --git a/crates/wasmtime/src/runtime/wave/core.rs b/crates/wasmtime/src/runtime/wave/core.rs
index 338ce4b2d8..fd89da83d7 100644
--- a/crates/wasmtime/src/runtime/wave/core.rs
+++ b/crates/wasmtime/src/runtime/wave/core.rs
@@ -40,6 +40,7 @@ impl WasmValue for crate::Val {
Self::ExternRef(_) => WasmTypeKind::Unsupported,
Self::AnyRef(_) => WasmTypeKind::Unsupported,
Self::ExnRef(_) => WasmTypeKind::Unsupported,
+ Self::ContRef(_) => WasmTypeKind::Unsupported,
}
}
diff --git a/src/commands/run.rs b/src/commands/run.rs
index d42e573b4c..e3fd86d27b 100644
--- a/src/commands/run.rs
+++ b/src/commands/run.rs
@@ -746,6 +746,8 @@ impl RunCommand {
Val::AnyRef(Some(_)) => println!("<anyref>"),
Val::ExnRef(None) => println!("<null exnref>"),
Val::ExnRef(Some(_)) => println!("<exnref>"),
+ Val::ContRef(None) => println!("<null contref>"),
+ Val::ContRef(Some(_)) => println!("<contref>"),
}
}
diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.wat
new file mode 100644
index 0000000000..76afed2d11
--- /dev/null
+++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat
@@ -0,0 +1,311 @@
+;;! target = "x86_64-unknown-linux-gnu"
+;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y"
+;;! test = "optimize"
+
+(module
+ (type $ft (func))
+ (tag $t (param i32))
+ (type $ct (cont $ft))
+
+ (func $countdown
+ (local $i i32)
+ (local.set $i (i32.const 10))
+ (loop $loop
+ ;; suspend and pass countdown to our cosnumer
+ (suspend $t (local.get $i))
+ ;; decrement i; break if we're at 0
+ (local.tee $i (i32.sub (local.get $i) (i32.const 1)))
+ (br_if $loop)
+ )
+ )
+ (elem declare func $countdown)
+
+ (func (export "main")
+ (local $c (ref $ct))
+ (local.set $c (cont.new $ct (ref.func $countdown)))
+ (loop $loop
+ (block $on_gen (result i32 (ref $ct))
+ (resume $ct (on $t $on_gen) (local.get $c))
+ ;; no more data, return
+ (return)
+ )
+ ;; stack contains [i32 (ref $ct)]
+ (local.set $c)
+ (drop) ;; could print here
+ (br $loop)
+ )
+ )
+)
+
+;; function u0:0(i64 vmctx, i64) tail {
+;; ss0 = explicit_slot 16, align = 65536
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+16
+;; gv3 = vmctx
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64):
+;; @003c v3 = iconst.i32 10
+;; v61 = iconst.i64 120
+;; @0044 v30 = stack_addr.i64 ss0
+;; v59 = iconst.i64 16
+;; v60 = iconst.i64 0
+;; v57 = iconst.i64 80
+;; v56 = iconst.i64 -24
+;; v70 = iconst.i64 0x0002_0000_0000
+;; @0040 jump block2(v3) ; v3 = 10
+;;
+;; block2(v4: i32):
+;; @0044 v8 = load.i64 notrap aligned v0+8
+;; @0044 v9 = load.i64 notrap aligned v8+64
+;; @0044 v10 = load.i64 notrap aligned v8+72
+;; v65 = iconst.i64 1
+;; v64 = iconst.i64 24
+;; @003a v2 = iconst.i32 0
+;; @0044 jump block4(v9, v10, v4)
+;;
+;; block4(v11: i64, v12: i64, v52: i32):
+;; v73 = iconst.i64 1
+;; v74 = icmp eq v11, v73 ; v73 = 1
+;; @0044 trapnz v74, user22
+;; @0044 jump block5
+;;
+;; block5:
+;; @0044 v14 = load.i64 notrap aligned v12+48
+;; @0044 v15 = load.i64 notrap aligned v12+56
+;; v75 = iconst.i64 24
+;; v76 = iadd v15, v75 ; v75 = 24
+;; @0044 v17 = load.i64 notrap aligned v76+8
+;; @0044 v18 = load.i32 notrap aligned v15+40
+;; v77 = iconst.i32 0
+;; v67 = iconst.i32 3
+;; v66 = iconst.i64 48
+;; @0044 v6 = iadd.i64 v0, v66 ; v66 = 48
+;; v62 = iconst.i32 1
+;; @0044 jump block6(v77) ; v77 = 0
+;;
+;; block6(v20: i32):
+;; @0044 v21 = icmp ult v20, v18
+;; @0044 brif v21, block7, block4(v14, v15, v52)
+;;
+;; block7:
+;; v78 = iconst.i32 3
+;; v79 = ishl.i32 v20, v78 ; v78 = 3
+;; @0044 v23 = uextend.i64 v79
+;; @0044 v24 = iadd.i64 v17, v23
+;; @0044 v25 = load.i64 notrap aligned v24
+;; v80 = iadd.i64 v0, v66 ; v66 = 48
+;; v81 = icmp eq v25, v80
+;; v82 = iconst.i32 1
+;; v83 = iadd.i32 v20, v82 ; v82 = 1
+;; @0044 brif v81, block8, block6(v83)
+;;
+;; block8:
+;; @0044 store.i64 notrap aligned v12, v10+64
+;; v84 = iconst.i32 1
+;; v85 = iconst.i64 120
+;; v86 = iadd.i64 v10, v85 ; v85 = 120
+;; @0044 store notrap aligned v84, v86+4 ; v84 = 1
+;; @0044 store.i64 notrap aligned v30, v86+8
+;; @0044 store.i32 notrap aligned v4, v30
+;; @0044 store notrap aligned v84, v86 ; v84 = 1
+;; v87 = iconst.i32 3
+;; v88 = iconst.i64 16
+;; v89 = iadd.i64 v10, v88 ; v88 = 16
+;; @0044 store notrap aligned v87, v89 ; v87 = 3
+;; v90 = iconst.i64 0
+;; @0044 store notrap aligned v90, v12+48 ; v90 = 0
+;; @0044 store notrap aligned v90, v12+56 ; v90 = 0
+;; v91 = iconst.i64 80
+;; v92 = iadd.i64 v12, v91 ; v91 = 80
+;; @0044 v43 = load.i64 notrap aligned v92
+;; v93 = iconst.i64 -24
+;; v94 = iadd v43, v93 ; v93 = -24
+;; @0044 v40 = uextend.i64 v20
+;; v95 = iconst.i64 0x0002_0000_0000
+;; v96 = bor v40, v95 ; v95 = 0x0002_0000_0000
+;; @0044 v45 = stack_switch v94, v94, v96
+;; @0044 v47 = load.i64 notrap aligned v86+8
+;; v97 = iconst.i32 0
+;; @0044 store notrap aligned v97, v86 ; v97 = 0
+;; @0044 store notrap aligned v97, v86+4 ; v97 = 0
+;; @0044 store notrap aligned v90, v86+8 ; v90 = 0
+;; v98 = isub.i32 v52, v84 ; v84 = 1
+;; @004d brif v98, block2(v98), block10
+;;
+;; block10:
+;; @004f jump block3
+;;
+;; block3:
+;; @0050 jump block1
+;;
+;; block1:
+;; @0050 return
+;; }
+;;
+;; function u0:1(i64 vmctx, i64) tail {
+;; ss0 = explicit_slot 8, align = 256
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+16
+;; gv3 = vmctx
+;; sig0 = (i64 vmctx, i32) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
+;; fn0 = colocated u1610612736:7 sig0
+;; fn1 = colocated u1610612736:52 sig1
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64):
+;; @0056 v2 = iconst.i32 0
+;; @0056 v4 = call fn0(v0, v2) ; v2 = 0
+;; @0058 trapz v4, user16
+;; @0058 v8 = call fn1(v0, v4, v2, v2) ; v2 = 0, v2 = 0
+;; @0058 v9 = load.i64 notrap aligned v8+72
+;; @0058 v11 = uextend.i128 v9
+;; v119 = iconst.i64 64
+;; v121 = ishl v11, v119 ; v119 = 64
+;; @0058 v10 = uextend.i128 v8
+;; @0058 v13 = bor v121, v10
+;; v116 = iconst.i64 1
+;; @0062 v28 = iconst.i64 0
+;; @0062 v29 = iconst.i64 2
+;; @0062 v32 = iconst.i32 1
+;; v114 = iconst.i64 16
+;; @0062 v34 = iconst.i32 2
+;; v110 = iconst.i64 24
+;; @0062 v45 = stack_addr.i64 ss0
+;; v109 = iconst.i64 48
+;; @0062 v47 = iadd v0, v109 ; v109 = 48
+;; v107 = iconst.i64 80
+;; v106 = iconst.i64 -24
+;; v125 = iconst.i64 0x0001_0000_0000
+;; v108 = iconst.i64 32
+;; @005c jump block2(v13)
+;;
+;; block2(v16: i128):
+;; @0062 jump block5
+;;
+;; block5:
+;; @0062 v18 = ireduce.i64 v16
+;; @0062 trapz v18, user16
+;; @0062 v21 = load.i64 notrap aligned v18+72
+;; v128 = iconst.i64 64
+;; v129 = ushr.i128 v16, v128 ; v128 = 64
+;; @0062 v20 = ireduce.i64 v129
+;; @0062 v22 = icmp eq v21, v20
+;; @0062 trapz v22, user23
+;; v130 = iconst.i64 1
+;; v131 = iadd v21, v130 ; v130 = 1
+;; @0062 store notrap aligned v131, v18+72
+;; @0062 v24 = load.i64 notrap aligned v18+64
+;; @0062 v25 = load.i64 notrap aligned v0+8
+;; @0062 v26 = load.i64 notrap aligned v25+64
+;; @0062 v27 = load.i64 notrap aligned v25+72
+;; @0062 store notrap aligned v26, v24+48
+;; @0062 store notrap aligned v27, v24+56
+;; v132 = iconst.i64 0
+;; @0062 store notrap aligned v132, v18+64 ; v132 = 0
+;; @0062 v30 = load.i64 notrap aligned v0+8
+;; v133 = iconst.i64 2
+;; @0062 store notrap aligned v133, v30+64 ; v133 = 2
+;; @0062 store notrap aligned v18, v30+72
+;; v134 = iconst.i32 1
+;; v135 = iconst.i64 16
+;; v136 = iadd v18, v135 ; v135 = 16
+;; @0062 store notrap aligned v134, v136 ; v134 = 1
+;; v137 = iconst.i32 2
+;; v138 = iadd v27, v135 ; v135 = 16
+;; @0062 store notrap aligned v137, v138 ; v137 = 2
+;; @0062 v36 = load.i64 notrap aligned readonly v0+8
+;; @0062 v38 = load.i64 notrap aligned v36+56
+;; @0062 store notrap aligned v38, v27+8
+;; @0062 v39 = load.i64 notrap aligned v36+16
+;; @0062 store notrap aligned v39, v27
+;; @0062 v41 = load.i64 notrap aligned v18
+;; @0062 store notrap aligned v41, v36+16
+;; @0062 v42 = load.i64 notrap aligned v18+8
+;; @0062 store notrap aligned v42, v36+56
+;; v139 = iconst.i64 24
+;; v140 = iadd v27, v139 ; v139 = 24
+;; @0062 store notrap aligned v134, v140+4 ; v134 = 1
+;; @0062 store.i64 notrap aligned v45, v140+8
+;; v141 = iadd.i64 v0, v109 ; v109 = 48
+;; @0062 store notrap aligned v141, v45
+;; @0062 store notrap aligned v134, v140 ; v134 = 1
+;; @0062 store notrap aligned v134, v27+40 ; v134 = 1
+;; v142 = iconst.i64 80
+;; v143 = iadd v24, v142 ; v142 = 80
+;; @0062 v54 = load.i64 notrap aligned v143
+;; v144 = iconst.i64 -24
+;; v145 = iadd v54, v144 ; v144 = -24
+;; v146 = iconst.i64 0x0001_0000_0000
+;; @0062 v56 = stack_switch v145, v145, v146 ; v146 = 0x0001_0000_0000
+;; @0062 v57 = load.i64 notrap aligned v0+8
+;; @0062 v58 = load.i64 notrap aligned v57+64
+;; @0062 v59 = load.i64 notrap aligned v57+72
+;; @0062 store notrap aligned v26, v57+64
+;; @0062 store notrap aligned v27, v57+72
+;; @0062 store notrap aligned v134, v138 ; v134 = 1
+;; v147 = iconst.i32 0
+;; @0062 store notrap aligned v147, v140 ; v147 = 0
+;; @0062 store notrap aligned v147, v140+4 ; v147 = 0
+;; @0062 store notrap aligned v132, v140+8 ; v132 = 0
+;; @0062 store notrap aligned v132, v27+40 ; v132 = 0
+;; v148 = iconst.i64 32
+;; v149 = ushr v56, v148 ; v148 = 32
+;; @0062 brif v149, block7, block6
+;;
+;; block7:
+;; @0062 v69 = load.i64 notrap aligned v36+56
+;; @0062 store notrap aligned v69, v59+8
+;; @0062 v71 = load.i64 notrap aligned v27
+;; @0062 store notrap aligned v71, v36+16
+;; @0062 v72 = load.i64 notrap aligned v27+8
+;; @0062 store notrap aligned v72, v36+56
+;; @0062 v74 = load.i64 notrap aligned v59+72
+;; @0062 jump block8
+;;
+;; block9 cold:
+;; @0062 trap user11
+;;
+;; block10:
+;; v98 = iconst.i64 120
+;; @0062 v79 = iadd.i64 v59, v98 ; v98 = 120
+;; @0062 v80 = load.i64 notrap aligned v79+8
+;; @0062 v81 = load.i32 notrap aligned v80
+;; v154 = iconst.i32 0
+;; @0062 store notrap aligned v154, v79 ; v154 = 0
+;; @0062 jump block4
+;;
+;; block8:
+;; @0062 v73 = ireduce.i32 v56
+;; @0062 br_table v73, block9, [block10]
+;;
+;; block6:
+;; @0062 v84 = load.i64 notrap aligned v27
+;; @0062 store notrap aligned v84, v36+16
+;; @0062 v85 = load.i64 notrap aligned v27+8
+;; @0062 store notrap aligned v85, v36+56
+;; @0062 v87 = iconst.i32 4
+;; v150 = iconst.i64 16
+;; v151 = iadd.i64 v59, v150 ; v150 = 16
+;; @0062 store notrap aligned v87, v151 ; v87 = 4
+;; v94 = iconst.i64 104
+;; @0062 v89 = iadd.i64 v59, v94 ; v94 = 104
+;; @0062 v90 = load.i64 notrap aligned v89+8
+;; v152 = iconst.i32 0
+;; @0062 store notrap aligned v152, v89 ; v152 = 0
+;; @0062 store notrap aligned v152, v89+4 ; v152 = 0
+;; v153 = iconst.i64 0
+;; @0062 store notrap aligned v153, v89+8 ; v153 = 0
+;; @0068 return
+;;
+;; block4:
+;; @0062 v76 = uextend.i128 v74
+;; v155 = iconst.i64 64
+;; v156 = ishl v76, v155 ; v155 = 64
+;; @0062 v75 = uextend.i128 v59
+;; @0062 v78 = bor v156, v75
+;; @006d jump block2(v78)
+;; }
diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.wat
new file mode 100644
index 0000000000..bfe8a715b1
--- /dev/null
+++ b/tests/disas/stack-switching/resume-suspend.wat
@@ -0,0 +1,263 @@
+;;! target = "x86_64-unknown-linux-gnu"
+;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y"
+;;! test = "optimize"
+
+(module
+ (type $ft (func))
+ (tag $t (type $ft))
+ (type $ct (cont $ft))
+
+ (func $target (suspend $t))
+ (elem declare func $target)
+
+ (func (export "minimal_suspend")
+ (local $k (ref null $ct))
+ (local.set $k (cont.new $ct (ref.func $target)))
+ (block $h (result (ref null $ct))
+ (resume $ct (on $t $h) (local.get $k))
+ ;; continuation suspended back...
+ (ref.null $ct)
+ )
+ (drop)
+ )
+)
+
+;; function u0:0(i64 vmctx, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+16
+;; gv3 = vmctx
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64):
+;; @003b v5 = load.i64 notrap aligned v0+8
+;; @003b v6 = load.i64 notrap aligned v5+64
+;; @003b v7 = load.i64 notrap aligned v5+72
+;; v54 = iconst.i64 1
+;; v53 = iconst.i64 24
+;; @003b v16 = iconst.i32 0
+;; @003b jump block2(v6, v7)
+;;
+;; block2(v8: i64, v9: i64):
+;; v62 = iconst.i64 1
+;; v63 = icmp eq v8, v62 ; v62 = 1
+;; @003b trapnz v63, user22
+;; @003b jump block3
+;;
+;; block3:
+;; @003b v11 = load.i64 notrap aligned v9+48
+;; @003b v12 = load.i64 notrap aligned v9+56
+;; v64 = iconst.i64 24
+;; v65 = iadd v12, v64 ; v64 = 24
+;; @003b v14 = load.i64 notrap aligned v65+8
+;; @003b v15 = load.i32 notrap aligned v12+40
+;; v66 = iconst.i32 0
+;; v56 = iconst.i32 3
+;; v55 = iconst.i64 48
+;; @003b v3 = iadd.i64 v0, v55 ; v55 = 48
+;; v51 = iconst.i32 1
+;; @003b jump block4(v66) ; v66 = 0
+;;
+;; block4(v17: i32):
+;; @003b v18 = icmp ult v17, v15
+;; @003b brif v18, block5, block2(v11, v12)
+;;
+;; block5:
+;; v67 = iconst.i32 3
+;; v68 = ishl.i32 v17, v67 ; v67 = 3
+;; @003b v20 = uextend.i64 v68
+;; @003b v21 = iadd.i64 v14, v20
+;; @003b v22 = load.i64 notrap aligned v21
+;; v69 = iadd.i64 v0, v55 ; v55 = 48
+;; v70 = icmp eq v22, v69
+;; v71 = iconst.i32 1
+;; v72 = iadd.i32 v17, v71 ; v71 = 1
+;; @003b brif v70, block6, block4(v72)
+;;
+;; block6:
+;; @003b store.i64 notrap aligned v9, v7+64
+;; v73 = iconst.i32 3
+;; v48 = iconst.i64 16
+;; @003b v28 = iadd.i64 v7, v48 ; v48 = 16
+;; @003b store notrap aligned v73, v28 ; v73 = 3
+;; v49 = iconst.i64 0
+;; @003b store notrap aligned v49, v9+48 ; v49 = 0
+;; @003b store notrap aligned v49, v9+56 ; v49 = 0
+;; v46 = iconst.i64 80
+;; @003b v35 = iadd.i64 v9, v46 ; v46 = 80
+;; @003b v36 = load.i64 notrap aligned v35
+;; v45 = iconst.i64 -24
+;; @003b v37 = iadd v36, v45 ; v45 = -24
+;; @003b v33 = uextend.i64 v17
+;; v59 = iconst.i64 0x0002_0000_0000
+;; v60 = bor v33, v59 ; v59 = 0x0002_0000_0000
+;; @003b v38 = stack_switch v37, v37, v60
+;; v50 = iconst.i64 120
+;; @003b v25 = iadd.i64 v7, v50 ; v50 = 120
+;; @003b v40 = load.i64 notrap aligned v25+8
+;; v74 = iconst.i32 0
+;; @003b store notrap aligned v74, v25 ; v74 = 0
+;; @003b store notrap aligned v74, v25+4 ; v74 = 0
+;; @003b store notrap aligned v49, v25+8 ; v49 = 0
+;; @003d jump block1
+;;
+;; block1:
+;; @003d return
+;; }
+;;
+;; function u0:1(i64 vmctx, i64) tail {
+;; ss0 = explicit_slot 8, align = 256
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+16
+;; gv3 = vmctx
+;; sig0 = (i64 vmctx, i32) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
+;; fn0 = colocated u1610612736:7 sig0
+;; fn1 = colocated u1610612736:52 sig1
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64):
+;; @0043 v7 = iconst.i32 0
+;; @0043 v9 = call fn0(v0, v7) ; v7 = 0
+;; @0045 trapz v9, user16
+;; @0045 v13 = call fn1(v0, v9, v7, v7) ; v7 = 0, v7 = 0
+;; @0045 v14 = load.i64 notrap aligned v13+72
+;; @004e jump block3
+;;
+;; block3:
+;; @0045 v16 = uextend.i128 v14
+;; v130 = iconst.i64 64
+;; v134 = ishl v16, v130 ; v130 = 64
+;; v136 = ireduce.i64 v134
+;; v138 = bor v136, v13
+;; @004e trapz v138, user16
+;; @004e v24 = load.i64 notrap aligned v138+72
+;; @0045 v15 = uextend.i128 v13
+;; @0045 v18 = bor v134, v15
+;; v140 = ushr v18, v130 ; v130 = 64
+;; @004e v23 = ireduce.i64 v140
+;; @004e v25 = icmp eq v24, v23
+;; @004e trapz v25, user23
+;; v125 = iconst.i64 1
+;; @004e v26 = iadd v24, v125 ; v125 = 1
+;; @004e store notrap aligned v26, v138+72
+;; @004e v27 = load.i64 notrap aligned v138+64
+;; @004e v28 = load.i64 notrap aligned v0+8
+;; @004e v29 = load.i64 notrap aligned v28+64
+;; @004e v30 = load.i64 notrap aligned v28+72
+;; @004e store notrap aligned v29, v27+48
+;; @004e store notrap aligned v30, v27+56
+;; @0040 v2 = iconst.i64 0
+;; @004e store notrap aligned v2, v138+64 ; v2 = 0
+;; @004e v33 = load.i64 notrap aligned v0+8
+;; @004e v32 = iconst.i64 2
+;; @004e store notrap aligned v32, v33+64 ; v32 = 2
+;; @004e store notrap aligned v138, v33+72
+;; @004e v35 = iconst.i32 1
+;; v123 = iconst.i64 16
+;; @004e v36 = iadd v138, v123 ; v123 = 16
+;; @004e store notrap aligned v35, v36 ; v35 = 1
+;; @004e v37 = iconst.i32 2
+;; @004e v38 = iadd v30, v123 ; v123 = 16
+;; @004e store notrap aligned v37, v38 ; v37 = 2
+;; @004e v39 = load.i64 notrap aligned readonly v0+8
+;; @004e v41 = load.i64 notrap aligned v39+56
+;; @004e store notrap aligned v41, v30+8
+;; @004e v42 = load.i64 notrap aligned v39+16
+;; @004e store notrap aligned v42, v30
+;; @004e v44 = load.i64 notrap aligned v138
+;; @004e store notrap aligned v44, v39+16
+;; @004e v45 = load.i64 notrap aligned v138+8
+;; @004e store notrap aligned v45, v39+56
+;; v119 = iconst.i64 24
+;; @004e v46 = iadd v30, v119 ; v119 = 24
+;; @004e store notrap aligned v35, v46+4 ; v35 = 1
+;; @004e v48 = stack_addr.i64 ss0
+;; @004e store notrap aligned v48, v46+8
+;; v118 = iconst.i64 48
+;; @004e v50 = iadd.i64 v0, v118 ; v118 = 48
+;; @004e store notrap aligned v50, v48
+;; @004e store notrap aligned v35, v46 ; v35 = 1
+;; @004e store notrap aligned v35, v30+40 ; v35 = 1
+;; v116 = iconst.i64 80
+;; @004e v56 = iadd v27, v116 ; v116 = 80
+;; @004e v57 = load.i64 notrap aligned v56
+;; v115 = iconst.i64 -24
+;; @004e v58 = iadd v57, v115 ; v115 = -24
+;; v142 = iconst.i64 0x0001_0000_0000
+;; @004e v59 = stack_switch v58, v58, v142 ; v142 = 0x0001_0000_0000
+;; @004e v60 = load.i64 notrap aligned v0+8
+;; @004e v61 = load.i64 notrap aligned v60+64
+;; @004e v62 = load.i64 notrap aligned v60+72
+;; @004e store notrap aligned v29, v60+64
+;; @004e store notrap aligned v30, v60+72
+;; @004e store notrap aligned v35, v38 ; v35 = 1
+;; v145 = iconst.i32 0
+;; @004e store notrap aligned v145, v46 ; v145 = 0
+;; @004e store notrap aligned v145, v46+4 ; v145 = 0
+;; @004e store notrap aligned v2, v46+8 ; v2 = 0
+;; @004e store notrap aligned v2, v30+40 ; v2 = 0
+;; v117 = iconst.i64 32
+;; @004e v69 = ushr v59, v117 ; v117 = 32
+;; @004e brif v69, block5, block4
+;;
+;; block5:
+;; @004e v72 = load.i64 notrap aligned v39+56
+;; @004e store notrap aligned v72, v62+8
+;; @004e v74 = load.i64 notrap aligned v30
+;; @004e store notrap aligned v74, v39+16
+;; @004e v75 = load.i64 notrap aligned v30+8
+;; @004e store notrap aligned v75, v39+56
+;; @004e v77 = load.i64 notrap aligned v62+72
+;; @004e jump block6
+;;
+;; block7 cold:
+;; @004e trap user11
+;;
+;; block8:
+;; v107 = iconst.i64 120
+;; @004e v82 = iadd.i64 v62, v107 ; v107 = 120
+;; @004e v83 = load.i64 notrap aligned v82+8
+;; v153 = iconst.i32 0
+;; @004e store notrap aligned v153, v82 ; v153 = 0
+;; @004e v79 = uextend.i128 v77
+;; v154 = iconst.i64 64
+;; v155 = ishl v79, v154 ; v154 = 64
+;; @004e v78 = uextend.i128 v62
+;; @004e v81 = bor v155, v78
+;; @004e jump block2(v81)
+;;
+;; block6:
+;; @004e v76 = ireduce.i32 v59
+;; @004e br_table v76, block7, [block8]
+;;
+;; block4:
+;; @004e v86 = load.i64 notrap aligned v30
+;; @004e store notrap aligned v86, v39+16
+;; @004e v87 = load.i64 notrap aligned v30+8
+;; @004e store notrap aligned v87, v39+56
+;; @004e v89 = iconst.i32 4
+;; v146 = iconst.i64 16
+;; v147 = iadd.i64 v62, v146 ; v146 = 16
+;; @004e store notrap aligned v89, v147 ; v89 = 4
+;; v103 = iconst.i64 104
+;; @004e v91 = iadd.i64 v62, v103 ; v103 = 104
+;; @004e v92 = load.i64 notrap aligned v91+8
+;; v148 = iconst.i32 0
+;; @004e store notrap aligned v148, v91 ; v148 = 0
+;; @004e store notrap aligned v148, v91+4 ; v148 = 0
+;; v149 = iconst.i64 0
+;; @004e store notrap aligned v149, v91+8 ; v149 = 0
+;; v150 = uextend.i128 v149 ; v149 = 0
+;; v151 = iconst.i64 64
+;; v152 = ishl v150, v151 ; v151 = 64
+;; @0040 v6 = bor v152, v150
+;; @0056 jump block2(v6)
+;;
+;; block2(v19: i128):
+;; @0058 jump block1
+;;
+;; block1:
+;; @0058 return
+;; }
diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.wat
new file mode 100644
index 0000000000..75c0458dfb
--- /dev/null
+++ b/tests/disas/stack-switching/symmetric-switch.wat
@@ -0,0 +1,418 @@
+;;! target = "x86_64-unknown-linux-gnu"
+;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y"
+
+(module
+ (type $fta (func))
+ (type $cta (cont $fta))
+
+ (type $ftb (func (param (ref $cta))))
+ (type $ctb (cont $ftb))
+
+ (tag $yield)
+
+ (func $task_a (type $fta)
+ (cont.new $ctb (ref.func $task_b))
+ (switch $ctb $yield)
+ )
+
+ (func $task_b (type $ftb))
+
+ (elem declare func $task_a $task_b)
+
+ (func (export "entry")
+ (cont.new $cta (ref.func $task_a))
+ (resume $cta (on $yield switch))
+ )
+)
+
+;; function u0:0(i64 vmctx, i64) tail {
+;; ss0 = explicit_slot 24, align = 256
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+16
+;; gv3 = vmctx
+;; sig0 = (i64 vmctx, i32) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
+;; fn0 = colocated u1610612736:7 sig0
+;; fn1 = colocated u1610612736:52 sig1
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64):
+;; @003a v2 = iconst.i32 1
+;; @003a v4 = call fn0(v0, v2) ; v2 = 1
+;; @003c trapz v4, user16
+;; @003c v5 = iconst.i32 1
+;; @003c v6 = iconst.i32 0
+;; @003c v8 = call fn1(v0, v4, v5, v6) ; v5 = 1, v6 = 0
+;; @003c v9 = load.i64 notrap aligned v8+72
+;; @003c v10 = uextend.i128 v8
+;; @003c v11 = uextend.i128 v9
+;; v138 = iconst.i64 64
+;; v139 = uextend.i128 v138 ; v138 = 64
+;; @003c v12 = ishl v11, v139
+;; @003c v13 = bor v12, v10
+;; @003e v15 = ireduce.i64 v13
+;; v136 = iconst.i64 64
+;; v137 = uextend.i128 v136 ; v136 = 64
+;; @003e v16 = ushr v13, v137
+;; @003e v17 = ireduce.i64 v16
+;; @003e trapz v15, user16
+;; @003e v18 = load.i64 notrap aligned v15+72
+;; @003e v19 = icmp eq v18, v17
+;; @003e trapz v19, user23
+;; v135 = iconst.i64 1
+;; @003e v20 = iadd v18, v135 ; v135 = 1
+;; @003e store notrap aligned v20, v15+72
+;; v134 = iconst.i64 48
+;; @003e v22 = iadd v0, v134 ; v134 = 48
+;; @003e v23 = load.i64 notrap aligned v0+8
+;; @003e v24 = load.i64 notrap aligned v23+64
+;; @003e v25 = load.i64 notrap aligned v23+72
+;; @003e jump block2(v24, v25)
+;;
+;; block2(v26: i64, v27: i64):
+;; v133 = iconst.i64 1
+;; @003e v28 = icmp eq v26, v133 ; v133 = 1
+;; @003e trapnz v28, user22
+;; @003e jump block3
+;;
+;; block3:
+;; @003e v29 = load.i64 notrap aligned v27+48
+;; @003e v30 = load.i64 notrap aligned v27+56
+;; v132 = iconst.i64 24
+;; @003e v31 = iadd v30, v132 ; v132 = 24
+;; @003e v32 = load.i64 notrap aligned v31+8
+;; @003e v33 = load.i32 notrap aligned v30+40
+;; @003e v34 = load.i32 notrap aligned v31
+;; @003e jump block4(v33)
+;;
+;; block4(v35: i32):
+;; @003e v36 = icmp ult v35, v34
+;; @003e brif v36, block5, block2(v29, v30)
+;;
+;; block5:
+;; v131 = iconst.i32 8
+;; @003e v37 = imul.i32 v35, v131 ; v131 = 8
+;; @003e v38 = uextend.i64 v37
+;; @003e v39 = iadd.i64 v32, v38
+;; @003e v40 = load.i64 notrap aligned v39
+;; @003e v41 = icmp eq v40, v22
+;; v130 = iconst.i32 1
+;; @003e v42 = iadd.i32 v35, v130 ; v130 = 1
+;; @003e brif v41, block6, block4(v42)
+;;
+;; block6:
+;; @003e store.i64 notrap aligned v27, v25+64
+;; v129 = iconst.i64 120
+;; @003e v43 = iadd.i64 v25, v129 ; v129 = 120
+;; v128 = iconst.i64 0
+;; @003e v44 = iadd.i64 v25, v128 ; v128 = 0
+;; @003e v45 = iconst.i32 3
+;; v127 = iconst.i64 16
+;; @003e v46 = iadd v44, v127 ; v127 = 16
+;; @003e store notrap aligned v45, v46 ; v45 = 3
+;; @003e v47 = iconst.i64 0
+;; @003e v48 = iconst.i64 0
+;; @003e store notrap aligned v47, v27+48 ; v47 = 0
+;; @003e store notrap aligned v48, v27+56 ; v48 = 0
+;; @003e v49 = load.i64 notrap aligned readonly v0+8
+;; v126 = iconst.i64 0
+;; @003e v50 = iadd v44, v126 ; v126 = 0
+;; @003e v51 = load.i64 notrap aligned v49+56
+;; @003e store notrap aligned v51, v50+8
+;; @003e v52 = load.i64 notrap aligned v25+72
+;; @003e v53 = uextend.i128 v25
+;; @003e v54 = uextend.i128 v52
+;; v124 = iconst.i64 64
+;; v125 = uextend.i128 v124 ; v124 = 64
+;; @003e v55 = ishl v54, v125
+;; @003e v56 = bor v55, v53
+;; v123 = iconst.i64 0
+;; @003e v58 = iadd.i64 v15, v123 ; v123 = 0
+;; v122 = iconst.i64 16
+;; @003e v59 = iadd v58, v122 ; v122 = 16
+;; @003e v60 = load.i32 notrap aligned v59
+;; v121 = iconst.i32 0
+;; @003e v61 = icmp ne v60, v121 ; v121 = 0
+;; @003e brif v61, block9, block8
+;;
+;; block8:
+;; v120 = iconst.i64 104
+;; @003e v62 = iadd.i64 v15, v120 ; v120 = 104
+;; @003e v63 = load.i64 notrap aligned v62+8
+;; @003e v64 = load.i32 notrap aligned v62
+;; v119 = iconst.i32 1
+;; @003e v65 = iadd v64, v119 ; v119 = 1
+;; @003e store notrap aligned v65, v62
+;; @003e v66 = uextend.i64 v64
+;; v118 = iconst.i64 16
+;; @003e v67 = imul v66, v118 ; v118 = 16
+;; @003e v68 = iadd v63, v67
+;; @003e jump block10(v68)
+;;
+;; block9:
+;; v117 = iconst.i64 120
+;; @003e v69 = iadd.i64 v15, v117 ; v117 = 120
+;; @003e v70 = load.i64 notrap aligned v69+8
+;; @003e v71 = load.i32 notrap aligned v69
+;; v116 = iconst.i32 1
+;; @003e v72 = iadd v71, v116 ; v116 = 1
+;; @003e store notrap aligned v72, v69
+;; @003e v73 = uextend.i64 v71
+;; v115 = iconst.i64 16
+;; @003e v74 = imul v73, v115 ; v115 = 16
+;; @003e v75 = iadd v70, v74
+;; @003e jump block10(v75)
+;;
+;; block10(v57: i64):
+;; @003e store.i128 notrap aligned v56, v57
+;; v114 = iconst.i64 0
+;; @003e v76 = iadd.i64 v15, v114 ; v114 = 0
+;; @003e v77 = iconst.i32 1
+;; v113 = iconst.i64 16
+;; @003e v78 = iadd v76, v113 ; v113 = 16
+;; @003e store notrap aligned v77, v78 ; v77 = 1
+;; @003e v79 = load.i64 notrap aligned v15+64
+;; @003e store.i64 notrap aligned v29, v79+48
+;; @003e store.i64 notrap aligned v30, v79+56
+;; @003e v80 = iconst.i64 2
+;; @003e v81 = load.i64 notrap aligned v0+8
+;; @003e store notrap aligned v80, v81+64 ; v80 = 2
+;; @003e store.i64 notrap aligned v15, v81+72
+;; v112 = iconst.i64 0
+;; @003e v82 = iadd v76, v112 ; v112 = 0
+;; @003e v83 = load.i64 notrap aligned v82
+;; @003e store notrap aligned v83, v49+16
+;; @003e v84 = load.i64 notrap aligned v82+8
+;; @003e store notrap aligned v84, v49+56
+;; v111 = iconst.i64 80
+;; @003e v85 = iadd.i64 v27, v111 ; v111 = 80
+;; @003e v86 = load.i64 notrap aligned v85
+;; v110 = iconst.i64 -24
+;; @003e v87 = iadd v86, v110 ; v110 = -24
+;; v109 = iconst.i64 80
+;; @003e v88 = iadd v79, v109 ; v109 = 80
+;; @003e v89 = load.i64 notrap aligned v88
+;; v108 = iconst.i64 -24
+;; @003e v90 = iadd v89, v108 ; v108 = -24
+;; @003e v91 = stack_addr.i64 ss0
+;; @003e v92 = load.i64 notrap aligned v90
+;; @003e store notrap aligned v92, v91
+;; @003e v93 = load.i64 notrap aligned v87
+;; @003e store notrap aligned v93, v90
+;; @003e v94 = load.i64 notrap aligned v90+8
+;; @003e store notrap aligned v94, v91+8
+;; @003e v95 = load.i64 notrap aligned v87+8
+;; @003e store notrap aligned v95, v90+8
+;; @003e v96 = load.i64 notrap aligned v90+16
+;; @003e store notrap aligned v96, v91+16
+;; @003e v97 = load.i64 notrap aligned v87+16
+;; @003e store notrap aligned v97, v90+16
+;; @003e v98 = iconst.i64 3
+;; v107 = iconst.i64 32
+;; @003e v99 = ishl v98, v107 ; v98 = 3, v107 = 32
+;; @003e v100 = stack_switch v87, v91, v99
+;; v106 = iconst.i64 120
+;; @003e v101 = iadd.i64 v25, v106 ; v106 = 120
+;; @003e v102 = load.i64 notrap aligned v101+8
+;; @003e v103 = iconst.i32 0
+;; @003e store notrap aligned v103, v101 ; v103 = 0
+;; @003e v104 = iconst.i32 0
+;; @003e store notrap aligned v104, v101+4 ; v104 = 0
+;; @003e v105 = iconst.i64 0
+;; @003e store notrap aligned v105, v101+8 ; v105 = 0
+;; @0041 jump block1
+;;
+;; block1:
+;; @0041 return
+;; }
+;;
+;; function u0:1(i64 vmctx, i64, i128) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+16
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i128):
+;; @0044 jump block1
+;;
+;; block1:
+;; @0044 return
+;; }
+;;
+;; function u0:2(i64 vmctx, i64) tail {
+;; ss0 = explicit_slot 8, align = 256
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+16
+;; gv3 = vmctx
+;; sig0 = (i64 vmctx, i32) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
+;; fn0 = colocated u1610612736:7 sig0
+;; fn1 = colocated u1610612736:52 sig1
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64):
+;; @0047 v2 = iconst.i32 0
+;; @0047 v4 = call fn0(v0, v2) ; v2 = 0
+;; @0049 trapz v4, user16
+;; @0049 v5 = iconst.i32 0
+;; @0049 v6 = iconst.i32 0
+;; @0049 v8 = call fn1(v0, v4, v5, v6) ; v5 = 0, v6 = 0
+;; @0049 v9 = load.i64 notrap aligned v8+72
+;; @0049 v10 = uextend.i128 v8
+;; @0049 v11 = uextend.i128 v9
+;; v111 = iconst.i64 64
+;; v112 = uextend.i128 v111 ; v111 = 64
+;; @0049 v12 = ishl v11, v112
+;; @0049 v13 = bor v12, v10
+;; @004b jump block2
+;;
+;; block2:
+;; @004b v15 = ireduce.i64 v13
+;; v109 = iconst.i64 64
+;; v110 = uextend.i128 v109 ; v109 = 64
+;; @004b v16 = ushr.i128 v13, v110
+;; @004b v17 = ireduce.i64 v16
+;; @004b trapz v15, user16
+;; @004b v18 = load.i64 notrap aligned v15+72
+;; @004b v19 = icmp eq v18, v17
+;; @004b trapz v19, user23
+;; v108 = iconst.i64 1
+;; @004b v20 = iadd v18, v108 ; v108 = 1
+;; @004b store notrap aligned v20, v15+72
+;; @004b v21 = load.i64 notrap aligned v15+64
+;; @004b v22 = load.i64 notrap aligned v0+8
+;; @004b v23 = load.i64 notrap aligned v22+64
+;; @004b v24 = load.i64 notrap aligned v22+72
+;; @004b store notrap aligned v23, v21+48
+;; @004b store notrap aligned v24, v21+56
+;; @004b v25 = iconst.i64 0
+;; @004b store notrap aligned v25, v15+64 ; v25 = 0
+;; @004b v26 = iconst.i64 2
+;; @004b v27 = load.i64 notrap aligned v0+8
+;; @004b store notrap aligned v26, v27+64 ; v26 = 2
+;; @004b store notrap aligned v15, v27+72
+;; v107 = iconst.i64 0
+;; @004b v28 = iadd v15, v107 ; v107 = 0
+;; @004b v29 = iconst.i32 1
+;; v106 = iconst.i64 16
+;; @004b v30 = iadd v28, v106 ; v106 = 16
+;; @004b store notrap aligned v29, v30 ; v29 = 1
+;; @004b v31 = iconst.i32 2
+;; v105 = iconst.i64 16
+;; @004b v32 = iadd v24, v105 ; v105 = 16
+;; @004b store notrap aligned v31, v32 ; v31 = 2
+;; @004b v33 = load.i64 notrap aligned readonly v0+8
+;; v104 = iconst.i64 0
+;; @004b v34 = iadd v24, v104 ; v104 = 0
+;; @004b v35 = load.i64 notrap aligned v33+56
+;; @004b store notrap aligned v35, v34+8
+;; @004b v36 = load.i64 notrap aligned v33+16
+;; @004b store notrap aligned v36, v34
+;; v103 = iconst.i64 0
+;; @004b v37 = iadd v28, v103 ; v103 = 0
+;; @004b v38 = load.i64 notrap aligned v37
+;; @004b store notrap aligned v38, v33+16
+;; @004b v39 = load.i64 notrap aligned v37+8
+;; @004b store notrap aligned v39, v33+56
+;; v102 = iconst.i64 24
+;; @004b v40 = iadd v24, v102 ; v102 = 24
+;; @004b v41 = iconst.i32 1
+;; @004b v42 = stack_addr.i64 ss0
+;; @004b store notrap aligned v41, v40+4 ; v41 = 1
+;; @004b store notrap aligned v42, v40+8
+;; v101 = iconst.i64 48
+;; @004b v44 = iadd.i64 v0, v101 ; v101 = 48
+;; @004b v45 = iconst.i32 1
+;; @004b v46 = load.i64 notrap aligned v40+8
+;; @004b store notrap aligned v44, v46
+;; @004b store notrap aligned v45, v40 ; v45 = 1
+;; @004b v47 = iconst.i32 0
+;; @004b store notrap aligned v47, v24+40 ; v47 = 0
+;; @004b v48 = iconst.i64 1
+;; v100 = iconst.i64 32
+;; @004b v49 = ishl v48, v100 ; v48 = 1, v100 = 32
+;; v99 = iconst.i64 80
+;; @004b v50 = iadd v21, v99 ; v99 = 80
+;; @004b v51 = load.i64 notrap aligned v50
+;; v98 = iconst.i64 -24
+;; @004b v52 = iadd v51, v98 ; v98 = -24
+;; @004b v53 = stack_switch v52, v52, v49
+;; @004b v54 = load.i64 notrap aligned v0+8
+;; @004b v55 = load.i64 notrap aligned v54+64
+;; @004b v56 = load.i64 notrap aligned v54+72
+;; @004b v57 = load.i64 notrap aligned v0+8
+;; @004b store notrap aligned v23, v57+64
+;; @004b store notrap aligned v24, v57+72
+;; @004b v58 = iconst.i32 1
+;; v97 = iconst.i64 16
+;; @004b v59 = iadd v24, v97 ; v97 = 16
+;; @004b store notrap aligned v58, v59 ; v58 = 1
+;; @004b v60 = iconst.i32 0
+;; @004b store notrap aligned v60, v40 ; v60 = 0
+;; @004b v61 = iconst.i32 0
+;; @004b store notrap aligned v61, v40+4 ; v61 = 0
+;; @004b v62 = iconst.i64 0
+;; @004b store notrap aligned v62, v40+8 ; v62 = 0
+;; @004b store notrap aligned v25, v24+40 ; v25 = 0
+;; v96 = iconst.i64 32
+;; @004b v63 = ushr v53, v96 ; v96 = 32
+;; @004b brif v63, block4, block3
+;;
+;; block4:
+;; v95 = iconst.i64 0
+;; @004b v64 = iadd.i64 v56, v95 ; v95 = 0
+;; v94 = iconst.i64 0
+;; @004b v65 = iadd v64, v94 ; v94 = 0
+;; @004b v66 = load.i64 notrap aligned v33+56
+;; @004b store notrap aligned v66, v65+8
+;; v93 = iconst.i64 0
+;; @004b v67 = iadd.i64 v24, v93 ; v93 = 0
+;; @004b v68 = load.i64 notrap aligned v67
+;; @004b store notrap aligned v68, v33+16
+;; @004b v69 = load.i64 notrap aligned v67+8
+;; @004b store notrap aligned v69, v33+56
+;; @004b v70 = ireduce.i32 v53
+;; @004b v71 = load.i64 notrap aligned v56+72
+;; @004b v72 = uextend.i128 v56
+;; @004b v73 = uextend.i128 v71
+;; v91 = iconst.i64 64
+;; v92 = uextend.i128 v91 ; v91 = 64
+;; @004b v74 = ishl v73, v92
+;; @004b v75 = bor v74, v72
+;; @004b jump block5
+;;
+;; block6 cold:
+;; @004b trap user11
+;;
+;; block5:
+;; @004b br_table v70, block6, []
+;;
+;; block3:
+;; v90 = iconst.i64 0
+;; @004b v76 = iadd.i64 v24, v90 ; v90 = 0
+;; @004b v77 = load.i64 notrap aligned v76
+;; @004b store notrap aligned v77, v33+16
+;; @004b v78 = load.i64 notrap aligned v76+8
+;; @004b store notrap aligned v78, v33+56
+;; v89 = iconst.i64 0
+;; @004b v79 = iadd.i64 v56, v89 ; v89 = 0
+;; @004b v80 = iconst.i32 4
+;; v88 = iconst.i64 16
+;; @004b v81 = iadd v79, v88 ; v88 = 16
+;; @004b store notrap aligned v80, v81 ; v80 = 4
+;; v87 = iconst.i64 104
+;; @004b v82 = iadd.i64 v56, v87 ; v87 = 104
+;; @004b v83 = load.i64 notrap aligned v82+8
+;; @004b v84 = iconst.i32 0
+;; @004b store notrap aligned v84, v82 ; v84 = 0
+;; @004b v85 = iconst.i32 0
+;; @004b store notrap aligned v85, v82+4 ; v85 = 0
+;; @004b v86 = iconst.i64 0
+;; @004b store notrap aligned v86, v82+8 ; v86 = 0
+;; @0050 jump block1
+;;
+;; block1:
+;; @0050 return
+;; }