A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
cranelift: stack-switching support (#11003)
Paul Osbornea631d20afa7a
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rsindex f5dd244d8d..d3c7f9ec2f 100644--- a/cranelift/codegen/src/isa/x64/abi.rs+++ b/cranelift/codegen/src/isa/x64/abi.rs@@ -101,6 +101,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {mut args: ArgsAccumulator,) -> CodegenResult<(u32, Option<usize>)> {let is_fastcall = call_conv == CallConv::WindowsFastcall;+ let is_tail = call_conv == CallConv::Tail;let mut next_gpr = 0;let mut next_vreg = 0;@@ -181,6 +182,11 @@ impl ABIMachineSpec for X64ABIMachineSpec {// This is consistent with LLVM's behavior, and is needed for// some uses of Cranelift (e.g., the rustc backend).//+ // - Otherwise, if the calling convention is Tail, we behave as in+ // the previous case, even if `enable_llvm_abi_extensions` is not+ // set in the flags: This is a custom calling convention defined+ // by Cranelift, LLVM doesn't know about it.+ //// - Otherwise, both SysV and Fastcall specify behavior (use of// vector register, a register pair, or passing by reference// depending on the case), but for simplicity, we will just panic if@@ -194,6 +200,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {if param.value_type.bits() > 64&& !(param.value_type.is_vector() || param.value_type.is_float())&& !flags.enable_llvm_abi_extensions()+ && !is_tail{panic!("i128 args/return values not supported unless LLVM ABI extensions are enabled"diff --git a/crates/c-api/src/val.rs b/crates/c-api/src/val.rsindex c9c4614743..44823c70f1 100644--- a/crates/c-api/src/val.rs+++ b/crates/c-api/src/val.rs@@ -98,6 +98,7 @@ impl wasm_val_t {Val::ExternRef(_) => crate::abort("creating a wasm_val_t from an externref"),Val::ExnRef(_) => crate::abort("creating a wasm_val_t from an exnref"),Val::V128(_) => crate::abort("creating a wasm_val_t from a v128"),+ Val::ContRef(_) => crate::abort("creating a wasm_val_t from a contref"),}}@@ -258,6 +259,7 @@ impl wasmtime_val_t {v128: val.as_u128().to_le_bytes(),},},+ Val::ContRef(_) => crate::abort("contrefs not yet supported in C API (#10248)"),}}diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 3871f47b92..877ecc14ad 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -1,4 +1,5 @@mod gc;+pub(crate) mod stack_switching;use crate::compiler::Compiler;use crate::translate::{@@ -171,6 +172,16 @@ pub struct FuncEnvironment<'module_environment> {/// always present even if this is a "leaf" function, as we have to call/// into the host to trap when signal handlers are disabled.pub(crate) stack_limit_at_function_entry: Option<ir::GlobalValue>,++ /// Used by the stack switching feature. If set, we have a allocated a+ /// slot on this function's stack to be used for the+ /// current stack's `handler_list` field.+ stack_switching_handler_list_buffer: Option<ir::StackSlot>,++ /// Used by the stack switching feature. If set, we have a allocated a+ /// slot on this function's stack to be used for the+ /// current continuation's `values` field.+ stack_switching_values_buffer: Option<ir::StackSlot>,}impl<'module_environment> FuncEnvironment<'module_environment> {@@ -224,6 +235,9 @@ impl<'module_environment> FuncEnvironment<'module_environment> {translation,stack_limit_at_function_entry: None,++ stack_switching_handler_list_buffer: None,+ stack_switching_values_buffer: None,}}@@ -1958,8 +1972,6 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {return CheckIndirectCallTypeSignature::StaticTrap;}- WasmHeapType::Cont | WasmHeapType::ConcreteCont(_) | WasmHeapType::NoCont => todo!(), // FIXME: #10248 stack switching support.-// Engine-indexed types don't show up until runtime and it's a Wasm// validation error to perform a call through a non-function table,// so these cases are dynamically not reachable.@@ -1977,6 +1989,9 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {| WasmHeapType::Exn| WasmHeapType::ConcreteExn(_)| WasmHeapType::NoExn+ | WasmHeapType::Cont+ | WasmHeapType::ConcreteCont(_)+ | WasmHeapType::NoCont| WasmHeapType::None => {unreachable!()}@@ -2267,7 +2282,9 @@ impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environmlet needs_stack_map = match wasm_ty.top() {WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => true,WasmHeapTopType::Func => false,- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.+ // TODO(#10248) Once continuations can be stored on the GC heap, we+ // will need stack maps for continuation objects.+ WasmHeapTopType::Cont => false,};(ty, needs_stack_map)}@@ -2320,22 +2337,32 @@ impl FuncEnvironment<'_> {let mut pos = builder.cursor();let table = self.table(table_index);let ty = table.ref_type.heap_type;- let grow = if ty.is_vmgcref_type() {- gc::builtins::table_grow_gc_ref(self, &mut pos.func)?- } else {- debug_assert_eq!(ty.top(), WasmHeapTopType::Func);- self.builtin_functions.table_grow_func_ref(&mut pos.func)- };-let (table_vmctx, defined_table_index) =self.table_vmctx_and_defined_index(&mut pos, table_index);-let index_type = table.idx_type;let delta = self.cast_index_to_i64(&mut pos, delta, index_type);- let call_inst = pos- .ins()- .call(grow, &[table_vmctx, defined_table_index, delta, init_value]);- let result = pos.func.dfg.first_result(call_inst);++ let mut args: SmallVec<[_; 6]> = smallvec![table_vmctx, defined_table_index, delta];+ let grow = match ty.top() {+ WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {+ args.push(init_value);+ gc::builtins::table_grow_gc_ref(self, pos.func)?+ }+ WasmHeapTopType::Func => {+ args.push(init_value);+ self.builtin_functions.table_grow_func_ref(pos.func)+ }+ WasmHeapTopType::Cont => {+ let (revision, contref) =+ stack_switching::fatpointer::deconstruct(self, &mut pos, init_value);+ args.extend_from_slice(&[contref, revision]);+ stack_switching::builtins::table_grow_cont_obj(self, pos.func)?+ }+ };++ let call_inst = pos.ins().call(grow, &args);+ let result = builder.func.dfg.first_result(call_inst);+Ok(self.convert_pointer_to_index_type(builder.cursor(), result, index_type, false))}@@ -2367,7 +2394,15 @@ impl FuncEnvironment<'_> {}// Continuation types.- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.+ WasmHeapTopType::Cont => {+ let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index);+ Ok(builder.ins().load(+ stack_switching::fatpointer::fatpointer_type(self),+ flags,+ elem_addr,+ 0,+ ))+ }}}@@ -2415,7 +2450,11 @@ impl FuncEnvironment<'_> {}// Continuation types.- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.+ WasmHeapTopType::Cont => {+ let (elem_addr, flags) = table_data.prepare_table_addr(self, builder, index);+ builder.ins().store(flags, value, elem_addr, 0);+ Ok(())+ }}}@@ -2429,21 +2468,31 @@ impl FuncEnvironment<'_> {) -> WasmResult<()> {let mut pos = builder.cursor();let table = self.table(table_index);- let index_type = table.idx_type;- let dst = self.cast_index_to_i64(&mut pos, dst, index_type);- let len = self.cast_index_to_i64(&mut pos, len, index_type);let ty = table.ref_type.heap_type;- let libcall = if ty.is_vmgcref_type() {- gc::builtins::table_fill_gc_ref(self, &mut pos.func)?- } else {- debug_assert_eq!(ty.top(), WasmHeapTopType::Func);- self.builtin_functions.table_fill_func_ref(&mut pos.func)- };-+ let dst = self.cast_index_to_i64(&mut pos, dst, table.idx_type);+ let len = self.cast_index_to_i64(&mut pos, len, table.idx_type);let (table_vmctx, table_index) = self.table_vmctx_and_defined_index(&mut pos, table_index);- pos.ins()- .call(libcall, &[table_vmctx, table_index, dst, val, len]);+ let mut args: SmallVec<[_; 6]> = smallvec![table_vmctx, table_index, dst];+ let libcall = match ty.top() {+ WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {+ args.push(val);+ gc::builtins::table_fill_gc_ref(self, &mut pos.func)?+ }+ WasmHeapTopType::Func => {+ args.push(val);+ self.builtin_functions.table_fill_func_ref(&mut pos.func)+ }+ WasmHeapTopType::Cont => {+ let (revision, contref) =+ stack_switching::fatpointer::deconstruct(self, &mut pos, val);+ args.extend_from_slice(&[contref, revision]);+ stack_switching::builtins::table_fill_cont_obj(self, &mut pos.func)?+ }+ };++ args.push(len);+ builder.ins().call(libcall, &args);Ok(())}@@ -2795,7 +2844,10 @@ impl FuncEnvironment<'_> {WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => {pos.ins().iconst(types::I32, 0)}- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.+ WasmHeapTopType::Cont => {+ let zero = pos.ins().iconst(self.pointer_type(), 0);+ stack_switching::fatpointer::construct(self, &mut pos, zero, zero)+ }})}@@ -2811,9 +2863,18 @@ impl FuncEnvironment<'_> {return Ok(pos.ins().iconst(ir::types::I32, 0));}- let byte_is_null =- pos.ins()- .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0);+ let byte_is_null = match ty.heap_type.top() {+ WasmHeapTopType::Cont => {+ let (_revision, contref) =+ stack_switching::fatpointer::deconstruct(self, &mut pos, value);+ pos.ins()+ .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, contref, 0)+ }+ _ => pos+ .ins()+ .icmp_imm(cranelift_codegen::ir::condcodes::IntCC::Equal, value, 0),+ };+Ok(pos.ins().uextend(ir::types::I32, byte_is_null))}@@ -3556,6 +3617,118 @@ impl FuncEnvironment<'_> {self.isa.triple().architecture == target_lexicon::Architecture::X86_64}+ pub fn translate_cont_bind(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ contobj: ir::Value,+ args: &[ir::Value],+ ) -> ir::Value {+ stack_switching::instructions::translate_cont_bind(self, builder, contobj, args)+ }++ pub fn translate_cont_new(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ func: ir::Value,+ arg_types: &[WasmValType],+ return_types: &[WasmValType],+ ) -> WasmResult<ir::Value> {+ stack_switching::instructions::translate_cont_new(+ self,+ builder,+ func,+ arg_types,+ return_types,+ )+ }++ pub fn translate_resume(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ type_index: u32,+ contobj: ir::Value,+ resume_args: &[ir::Value],+ resumetable: &[(u32, Option<ir::Block>)],+ ) -> WasmResult<Vec<ir::Value>> {+ stack_switching::instructions::translate_resume(+ self,+ builder,+ type_index,+ contobj,+ resume_args,+ resumetable,+ )+ }++ pub fn translate_suspend(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ tag_index: u32,+ suspend_args: &[ir::Value],+ tag_return_types: &[ir::Type],+ ) -> Vec<ir::Value> {+ stack_switching::instructions::translate_suspend(+ self,+ builder,+ tag_index,+ suspend_args,+ tag_return_types,+ )+ }++ /// Translates switch instructions.+ pub fn translate_switch(+ &mut self,+ builder: &mut FunctionBuilder,+ tag_index: u32,+ contobj: ir::Value,+ switch_args: &[ir::Value],+ return_types: &[ir::Type],+ ) -> WasmResult<Vec<ir::Value>> {+ stack_switching::instructions::translate_switch(+ self,+ builder,+ tag_index,+ contobj,+ switch_args,+ return_types,+ )+ }++ pub fn continuation_arguments(&self, index: TypeIndex) -> &[WasmValType] {+ let idx = self.module.types[index].unwrap_module_type_index();+ self.types[self.types[idx]+ .unwrap_cont()+ .clone()+ .unwrap_module_type_index()]+ .unwrap_func()+ .params()+ }++ pub fn continuation_returns(&self, index: TypeIndex) -> &[WasmValType] {+ let idx = self.module.types[index].unwrap_module_type_index();+ self.types[self.types[idx]+ .unwrap_cont()+ .clone()+ .unwrap_module_type_index()]+ .unwrap_func()+ .returns()+ }++ pub fn tag_params(&self, tag_index: TagIndex) -> &[WasmValType] {+ let idx = self.module.tags[tag_index].signature;+ self.types[idx.unwrap_module_type_index()]+ .unwrap_func()+ .params()+ }++ pub fn tag_returns(&self, tag_index: TagIndex) -> &[WasmValType] {+ let idx = self.module.tags[tag_index].signature;+ self.types[idx.unwrap_module_type_index()]+ .unwrap_func()+ .returns()+ }+pub fn use_x86_blendv_for_relaxed_laneselect(&self, ty: Type) -> bool {self.isa.has_x86_blendv_lowering(ty)}@@ -4138,17 +4311,3 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type {IndexType::I64 => I64,}}--/// TODO(10248) This is removed in the next stack switching PR. It stops the-/// compiler from complaining about the stack switching libcalls being dead-/// code.-#[cfg(feature = "stack-switching")]-#[allow(- dead_code,- reason = "Dummy function to suppress more dead code warnings"-)]-pub fn use_stack_switching_libcalls() {- let _ = BuiltinFunctions::cont_new;- let _ = BuiltinFunctions::table_grow_cont_obj;- let _ = BuiltinFunctions::table_fill_cont_obj;-}diff --git a/crates/cranelift/src/func_environ/stack_switching/control_effect.rs b/crates/cranelift/src/func_environ/stack_switching/control_effect.rsnew file mode 100644index 0000000000..558fdfb870--- /dev/null+++ b/crates/cranelift/src/func_environ/stack_switching/control_effect.rs@@ -0,0 +1,65 @@+use cranelift_codegen::ir;+use cranelift_codegen::ir::InstBuilder;+use cranelift_codegen::ir::types::{I32, I64};+use cranelift_frontend::FunctionBuilder;++/// Universal control effect. This structure encodes return signal,+/// resume signal, suspension signal, and handler index into a+/// u64 value. This instance is used at compile time. There is a runtime+/// counterpart in `continuations/src/lib.rs`.+/// We convert to and from u64 as follows: The low 32 bits of the u64 are the+/// discriminant, the high 32 bits are the handler_index (if `Suspend`)+#[derive(Clone, Copy)]+pub struct ControlEffect(ir::Value);++impl ControlEffect {+ // Returns the discriminant+ pub fn signal(&self, builder: &mut FunctionBuilder) -> ir::Value {+ builder.ins().ushr_imm(self.0, 32)+ }++ pub fn from_u64(val: ir::Value) -> Self {+ Self(val)+ }++ pub fn to_u64(&self) -> ir::Value {+ self.0+ }++ pub fn encode_resume(builder: &mut FunctionBuilder) -> Self {+ let discriminant = builder.ins().iconst(+ I64,+ i64::from(wasmtime_environ::CONTROL_EFFECT_RESUME_DISCRIMINANT),+ );+ let val = builder.ins().ishl_imm(discriminant, 32);++ Self(val)+ }++ pub fn encode_switch(builder: &mut FunctionBuilder) -> Self {+ let discriminant = builder.ins().iconst(+ I64,+ i64::from(wasmtime_environ::CONTROL_EFFECT_SWITCH_DISCRIMINANT),+ );+ let val = builder.ins().ishl_imm(discriminant, 32);++ Self(val)+ }++ pub fn encode_suspend(builder: &mut FunctionBuilder, handler_index: ir::Value) -> Self {+ let discriminant = builder.ins().iconst(+ I64,+ i64::from(wasmtime_environ::CONTROL_EFFECT_SUSPEND_DISCRIMINANT),+ );+ let val = builder.ins().ishl_imm(discriminant, 32);+ let handler_index = builder.ins().uextend(I64, handler_index);+ let val = builder.ins().bor(val, handler_index);++ Self(val)+ }++ /// Returns the payload of the `Suspend` variant+ pub fn handler_index(self, builder: &mut FunctionBuilder) -> ir::Value {+ builder.ins().ireduce(I32, self.0)+ }+}diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rsnew file mode 100644index 0000000000..e2fe365ace--- /dev/null+++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs@@ -0,0 +1,55 @@+use cranelift_codegen::ir;+use cranelift_codegen::ir::InstBuilder;++/// Returns the Cranelift type used to represent all of the following:+/// - wasm values of type `(ref null $ct)` and `(ref $ct)`+/// - equivalently: runtime values of type `Option<VMContObj>` and `VMContObj`+/// Note that a `VMContObj` is a fat pointer consisting of a pointer to+/// `VMContRef` and a pointer-sized revision counter. We represent this as 2 words+/// (pointer and usize).+pub fn fatpointer_type(env: &crate::func_environ::FuncEnvironment) -> ir::Type {+ let ptr_bits = env.pointer_type().bits();+ ir::Type::int((2 * ptr_bits).try_into().unwrap()).unwrap()+}++/// Turns a (possibly null) reference to a continuation object into a tuple+/// (revision, contref_ptr). If `contobj` denotes a wasm null reference, the+/// contref_ptr part will be a null pointer.+pub(crate) fn deconstruct<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ pos: &mut cranelift_codegen::cursor::FuncCursor,+ contobj: ir::Value,+) -> (ir::Value, ir::Value) {+ debug_assert_eq!(pos.func.dfg.value_type(contobj), fatpointer_type(env));+ let ptr_ty = env.pointer_type();+ let ptr_bits = ptr_ty.bits();++ let contref = pos.ins().ireduce(ptr_ty, contobj);+ let shifted = pos.ins().ushr_imm(contobj, i64::from(ptr_bits));+ let revision_counter = pos.ins().ireduce(ptr_ty, shifted);++ (revision_counter, contref)+}++/// Constructs a continuation object from a given contref and revision pointer.+/// The contref_addr may be 0, to indicate that we want to build a wasm null reference.+pub(crate) fn construct<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ pos: &mut cranelift_codegen::cursor::FuncCursor,+ revision_counter: ir::Value,+ contref_addr: ir::Value,+) -> ir::Value {+ let ptr_ty = env.pointer_type();+ let ptr_bits = ptr_ty.bits();+ let fat_ptr_ty = fatpointer_type(env);++ debug_assert_eq!(pos.func.dfg.value_type(contref_addr), ptr_ty);+ debug_assert_eq!(pos.func.dfg.value_type(revision_counter), ptr_ty);++ let contref_addr = pos.ins().uextend(fat_ptr_ty, contref_addr);+ let revision_counter = pos.ins().uextend(fat_ptr_ty, revision_counter);+ let shifted_counter = pos.ins().ishl_imm(revision_counter, i64::from(ptr_bits));+ let contobj = pos.ins().bor(shifted_counter, contref_addr);++ contobj+}diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rsnew file mode 100644index 0000000000..383da04f93--- /dev/null+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs@@ -0,0 +1,1917 @@+use cranelift_codegen::ir::BlockArg;+use itertools::{Either, Itertools};++use cranelift_codegen::ir::condcodes::*;+use cranelift_codegen::ir::types::*;+use cranelift_codegen::ir::{self, MemFlags};+use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData};+use cranelift_frontend::FunctionBuilder;+use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wasm_unsupported};++fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult<u8> {+ match (triple.architecture, triple.operating_system) {+ (target_lexicon::Architecture::X86_64, target_lexicon::OperatingSystem::Linux) => Ok(24),+ _ => Err(wasm_unsupported!(+ "stack switching not supported on {triple}"+ )),+ }+}++use super::control_effect::ControlEffect;+use super::fatpointer;++/// This module contains compile-time counterparts to types defined elsewhere.+pub(crate) mod stack_switching_helpers {+ use core::marker::PhantomData;+ use cranelift_codegen::ir;+ use cranelift_codegen::ir::InstBuilder;+ use cranelift_codegen::ir::condcodes::IntCC;+ use cranelift_codegen::ir::types::*;+ use cranelift_codegen::ir::{StackSlot, StackSlotKind::*};+ use cranelift_frontend::FunctionBuilder;+ use wasmtime_environ::PtrSize;++ /// Provides information about the layout of a type when it is used as an+ /// element in a host array. This is used for `VMHostArrayRef`.+ pub(crate) trait VMHostArrayEntry {+ /// Returns `(align, size)` in bytes.+ fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(p: &P) -> (u8, u32);+ }++ impl VMHostArrayEntry for u128 {+ fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(_p: &P) -> (u8, u32) {+ (16, 16)+ }+ }++ impl<T> VMHostArrayEntry for *mut T {+ fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(p: &P) -> (u8, u32) {+ (p.size(), p.size().into())+ }+ }++ #[derive(Copy, Clone)]+ pub struct VMContRef {+ pub address: ir::Value,+ }++ #[derive(Copy, Clone)]+ pub struct VMHostArrayRef<T> {+ /// Address of the VMHostArray we are referencing+ address: ir::Value,++ /// The type parameter T is never used in the fields above. We still+ /// want to have it for consistency with+ /// `wasmtime_environ::Vector` and to use it in the associated+ /// functions.+ phantom: PhantomData<T>,+ }++ pub type VMPayloads = VMHostArrayRef<u128>;++ // Actually a vector of *mut VMTagDefinition+ pub type VMHandlerList = VMHostArrayRef<*mut u8>;++ /// Compile-time representation of wasmtime_environ::VMStackChain,+ /// consisting of two `ir::Value`s.+ pub struct VMStackChain {+ discriminant: ir::Value,+ payload: ir::Value,+ }++ pub struct VMCommonStackInformation {+ pub address: ir::Value,+ }++ /// Compile-time representation of `crate::runtime::vm::stack::VMContinuationStack`.+ pub struct VMContinuationStack {+ /// This is NOT the "top of stack" address of the stack itself. In line+ /// with how the (runtime) `FiberStack` type works, this is a pointer to+ /// the TOS address.+ tos_ptr: ir::Value,+ }++ impl VMContRef {+ pub fn new(address: ir::Value) -> VMContRef {+ VMContRef { address }+ }++ pub fn args<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> VMPayloads {+ let offset: i64 = env.offsets.ptr.vmcontref_args().into();+ let address = builder.ins().iadd_imm(self.address, offset);+ VMPayloads::new(address)+ }++ pub fn values<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> VMPayloads {+ let offset: i64 = env.offsets.ptr.vmcontref_values().into();+ let address = builder.ins().iadd_imm(self.address, offset);+ VMPayloads::new(address)+ }++ pub fn common_stack_information<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> VMCommonStackInformation {+ let offset: i64 = env.offsets.ptr.vmcontref_common_stack_information().into();+ let address = builder.ins().iadd_imm(self.address, offset);+ VMCommonStackInformation { address }+ }++ /// Stores the parent of this continuation, which may either be another+ /// continuation or the initial stack. It is therefore represented as a+ /// `VMStackChain` element.+ pub fn set_parent_stack_chain<'a>(+ &mut self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ new_stack_chain: &VMStackChain,+ ) {+ let offset = env.offsets.ptr.vmcontref_parent_chain().into();+ new_stack_chain.store(env, builder, self.address, offset)+ }++ /// Loads the parent of this continuation, which may either be another+ /// continuation or the initial stack. It is therefore represented as a+ /// `VMStackChain` element.+ pub fn get_parent_stack_chain<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> VMStackChain {+ let offset = env.offsets.ptr.vmcontref_parent_chain().into();+ VMStackChain::load(env, builder, self.address, offset, env.pointer_type())+ }++ pub fn set_last_ancestor<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ last_ancestor: ir::Value,+ ) {+ let offset: i32 = env.offsets.ptr.vmcontref_last_ancestor().into();+ let mem_flags = ir::MemFlags::trusted();+ builder+ .ins()+ .store(mem_flags, last_ancestor, self.address, offset);+ }++ pub fn get_last_ancestor<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let offset: i32 = env.offsets.ptr.vmcontref_last_ancestor().into();+ let mem_flags = ir::MemFlags::trusted();+ builder+ .ins()+ .load(env.pointer_type(), mem_flags, self.address, offset)+ }++ /// Gets the revision counter the a given continuation+ /// reference.+ pub fn get_revision<'a>(+ &mut self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let mem_flags = ir::MemFlags::trusted();+ let offset: i32 = env.offsets.ptr.vmcontref_revision().into();+ let revision = builder.ins().load(I64, mem_flags, self.address, offset);+ revision+ }++ /// Sets the revision counter on the given continuation+ /// reference to `revision + 1`.++ pub fn incr_revision<'a>(+ &mut self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ revision: ir::Value,+ ) -> ir::Value {+ let mem_flags = ir::MemFlags::trusted();+ let offset: i32 = env.offsets.ptr.vmcontref_revision().into();+ let revision_plus1 = builder.ins().iadd_imm(revision, 1);+ builder+ .ins()+ .store(mem_flags, revision_plus1, self.address, offset);+ revision_plus1+ }++ pub fn get_fiber_stack<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> VMContinuationStack {+ // The top of stack field is stored at offset 0 of the `FiberStack`.+ let offset: i64 = env.offsets.ptr.vmcontref_stack().into();+ let fiber_stack_top_of_stack_ptr = builder.ins().iadd_imm(self.address, offset);+ VMContinuationStack::new(fiber_stack_top_of_stack_ptr)+ }+ }++ impl<T: VMHostArrayEntry> VMHostArrayRef<T> {+ pub(crate) fn new(address: ir::Value) -> Self {+ Self {+ address,+ phantom: PhantomData::default(),+ }+ }++ fn get(&self, builder: &mut FunctionBuilder, ty: ir::Type, offset: i32) -> ir::Value {+ let mem_flags = ir::MemFlags::trusted();+ builder.ins().load(ty, mem_flags, self.address, offset)+ }++ fn set<U>(&self, builder: &mut FunctionBuilder, offset: i32, value: ir::Value) {+ debug_assert_eq!(+ builder.func.dfg.value_type(value),+ Type::int_with_byte_size(u16::try_from(core::mem::size_of::<U>()).unwrap())+ .unwrap()+ );+ let mem_flags = ir::MemFlags::trusted();+ builder.ins().store(mem_flags, value, self.address, offset);+ }++ pub fn get_data<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let offset = env.offsets.ptr.vmhostarray_data().into();+ self.get(builder, env.pointer_type(), offset)+ }++ pub fn get_length<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ // Array length is stored as u32.+ let offset = env.offsets.ptr.vmhostarray_length().into();+ self.get(builder, I32, offset)+ }++ fn set_length<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ length: ir::Value,+ ) {+ // Array length is stored as u32.+ let offset = env.offsets.ptr.vmhostarray_length().into();+ self.set::<u32>(builder, offset, length);+ }++ fn set_capacity<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ capacity: ir::Value,+ ) {+ // Array capacity is stored as u32.+ let offset = env.offsets.ptr.vmhostarray_capacity().into();+ self.set::<u32>(builder, offset, capacity);+ }++ fn set_data<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ data: ir::Value,+ ) {+ debug_assert_eq!(builder.func.dfg.value_type(data), env.pointer_type());+ let offset: i32 = env.offsets.ptr.vmhostarray_data().into();+ let mem_flags = ir::MemFlags::trusted();+ builder.ins().store(mem_flags, data, self.address, offset);+ }++ /// Returns pointer to next empty slot in data buffer and marks the+ /// subsequent `arg_count` slots as occupied.+ pub fn occupy_next_slots<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ arg_count: i32,+ ) -> ir::Value {+ let data = self.get_data(env, builder);+ let original_length = self.get_length(env, builder);+ let new_length = builder+ .ins()+ .iadd_imm(original_length, i64::from(arg_count));+ self.set_length(env, builder, new_length);++ let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);+ let original_length = builder.ins().uextend(I64, original_length);+ let byte_offset = builder+ .ins()+ .imul_imm(original_length, i64::from(entry_size));+ builder.ins().iadd(data, byte_offset)+ }++ pub fn allocate_or_reuse_stack_slot<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ required_capacity: u32,+ existing_slot: Option<StackSlot>,+ ) -> StackSlot {+ let (align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);+ let required_size = required_capacity * entry_size;++ match existing_slot {+ Some(slot) if builder.func.sized_stack_slots[slot].size >= required_size => {+ let slot_data = &builder.func.sized_stack_slots[slot];+ debug_assert!(align <= slot_data.align_shift);+ debug_assert_eq!(slot_data.kind, ExplicitSlot);+ let existing_capacity = slot_data.size / entry_size;++ let capacity_value = builder.ins().iconst(I32, i64::from(existing_capacity));+ let existing_data = builder.ins().stack_addr(env.pointer_type(), slot, 0);++ self.set_capacity(env, builder, capacity_value);+ self.set_data(env, builder, existing_data);++ slot+ }+ _ => {+ let capacity_value = builder.ins().iconst(I32, i64::from(required_capacity));+ let slot_size = ir::StackSlotData::new(+ ir::StackSlotKind::ExplicitSlot,+ required_size,+ align,+ );+ let slot = builder.create_sized_stack_slot(slot_size);+ let new_data = builder.ins().stack_addr(env.pointer_type(), slot, 0);++ self.set_capacity(env, builder, capacity_value);+ self.set_data(env, builder, new_data);++ slot+ }+ }+ }++ /// Loads n entries from this Vector object, where n is the length of+ /// `load_types`, which also gives the types of the values to load.+ /// Loading starts at index 0 of the Vector object.+ pub fn load_data_entries<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ load_types: &[ir::Type],+ ) -> Vec<ir::Value> {+ let memflags = ir::MemFlags::trusted();++ let data_start_pointer = self.get_data(env, builder);+ let mut values = vec![];+ let mut offset = 0;+ let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);+ for valtype in load_types {+ let val = builder+ .ins()+ .load(*valtype, memflags, data_start_pointer, offset);+ values.push(val);+ offset += i32::try_from(entry_size).unwrap();+ }+ values+ }++ /// Stores the given `values` in this Vector object, beginning at+ /// index 0. This expects the Vector object to be empty (i.e., current+ /// length is 0), and to be of sufficient capacity to store |`values`|+ /// entries.+ pub fn store_data_entries<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ values: &[ir::Value],+ ) {+ let store_count = builder+ .ins()+ .iconst(I32, i64::try_from(values.len()).unwrap());++ let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);++ debug_assert!(values.iter().all(|val| {+ let ty = builder.func.dfg.value_type(*val);+ let size = ty.bytes();+ size <= entry_size+ }));++ let memflags = ir::MemFlags::trusted();++ let data_start_pointer = self.get_data(env, builder);++ let mut offset = 0;+ for value in values {+ builder+ .ins()+ .store(memflags, *value, data_start_pointer, offset);+ offset += i32::try_from(entry_size).unwrap();+ }++ self.set_length(env, builder, store_count);+ }++ pub fn clear<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ discard_buffer: bool,+ ) {+ let zero32 = builder.ins().iconst(I32, 0);+ self.set_length(env, builder, zero32);++ if discard_buffer {+ let zero32 = builder.ins().iconst(I32, 0);+ self.set_capacity(env, builder, zero32);++ let zero_ptr = builder.ins().iconst(env.pointer_type(), 0);+ self.set_data(env, builder, zero_ptr);+ }+ }+ }++ impl VMStackChain {+ /// Creates a `Self` corressponding to `VMStackChain::Continuation(contref)`.+ pub fn from_continuation<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ contref: ir::Value,+ ) -> VMStackChain {+ debug_assert_eq!(+ env.offsets.ptr.size_of_vmstack_chain(),+ 2 * env.offsets.ptr.size()+ );+ let discriminant = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT;+ let discriminant = builder+ .ins()+ .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap());+ VMStackChain {+ discriminant,+ payload: contref,+ }+ }++ /// Creates a `Self` corressponding to `VMStackChain::Absent`.+ pub fn absent<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> VMStackChain {+ debug_assert_eq!(+ env.offsets.ptr.size_of_vmstack_chain(),+ 2 * env.offsets.ptr.size()+ );+ let discriminant = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT;+ let discriminant = builder+ .ins()+ .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap());+ let zero_filler = builder.ins().iconst(env.pointer_type(), 0i64);+ VMStackChain {+ discriminant,+ payload: zero_filler,+ }+ }++ pub fn is_initial_stack<'a>(+ &self,+ _env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ builder.ins().icmp_imm(+ IntCC::Equal,+ self.discriminant,+ i64::try_from(wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT).unwrap(),+ )+ }++ /// Return the two raw `ir::Value`s that represent this VMStackChain.+ pub fn to_raw_parts(&self) -> [ir::Value; 2] {+ [self.discriminant, self.payload]+ }++ /// Construct a `Self` from two raw `ir::Value`s.+ pub fn from_raw_parts(raw_data: [ir::Value; 2]) -> VMStackChain {+ VMStackChain {+ discriminant: raw_data[0],+ payload: raw_data[1],+ }+ }++ /// Load a `VMStackChain` object from the given address.+ pub fn load<'a>(+ _env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ pointer: ir::Value,+ initial_offset: i32,+ pointer_type: ir::Type,+ ) -> VMStackChain {+ let memflags = ir::MemFlags::trusted();+ let mut offset = initial_offset;+ let mut data = vec![];+ for _ in 0..2 {+ data.push(builder.ins().load(pointer_type, memflags, pointer, offset));+ offset += i32::try_from(pointer_type.bytes()).unwrap();+ }+ let data = <[ir::Value; 2]>::try_from(data).unwrap();+ Self::from_raw_parts(data)+ }++ /// Store this `VMStackChain` object at the given address.+ pub fn store<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ target_pointer: ir::Value,+ initial_offset: i32,+ ) {+ let memflags = ir::MemFlags::trusted();+ let mut offset = initial_offset;+ let data = self.to_raw_parts();++ for value in data {+ debug_assert_eq!(builder.func.dfg.value_type(value), env.pointer_type());+ builder.ins().store(memflags, value, target_pointer, offset);+ offset += i32::try_from(env.pointer_type().bytes()).unwrap();+ }+ }++ /// Use this only if you've already checked that `self` corresponds to a `VMStackChain::Continuation`.+ pub fn unchecked_get_continuation(&self) -> ir::Value {+ self.payload+ }++ /// Must only be called if `self` represents a `InitialStack` or+ /// `Continuation` variant. Returns a pointer to the associated+ /// `CommonStackInformation` object.+ pub fn get_common_stack_information<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ _builder: &mut FunctionBuilder,+ ) -> VMCommonStackInformation {+ // `self` corresponds to a VMStackChain::InitialStack or+ // VMStackChain::Continuation.+ // In both cases, the payload is a pointer.+ let address = self.payload;++ // `obj` is now a pointer to the beginning of either+ // 1. A `VMContRef` struct (in the case of a+ // VMStackChain::Continuation)+ // 2. A CommonStackInformation struct (in the case of+ // VMStackChain::InitialStack)+ //+ // Since a `VMContRef` starts with an (inlined) CommonStackInformation+ // object at offset 0, we actually have in both cases that `ptr` is+ // now the address of the beginning of a VMStackLimits object.+ debug_assert_eq!(env.offsets.ptr.vmcontref_common_stack_information(), 0);+ VMCommonStackInformation { address }+ }+ }++ impl VMCommonStackInformation {+ fn get_state_ptr<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let offset: i64 = env.offsets.ptr.vmcommon_stack_information_state().into();++ builder.ins().iadd_imm(self.address, offset)+ }++ fn get_stack_limits_ptr<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let offset: i64 = env.offsets.ptr.vmcommon_stack_information_limits().into();++ builder.ins().iadd_imm(self.address, offset)+ }++ fn load_state<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let mem_flags = ir::MemFlags::trusted();+ let state_ptr = self.get_state_ptr(env, builder);++ builder.ins().load(I32, mem_flags, state_ptr, 0)+ }++ fn set_state_no_payload<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ discriminant: u32,+ ) {+ let discriminant = builder.ins().iconst(I32, i64::from(discriminant));+ let mem_flags = ir::MemFlags::trusted();+ let state_ptr = self.get_state_ptr(env, builder);++ builder.ins().store(mem_flags, discriminant, state_ptr, 0);+ }++ pub fn set_state_running<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) {+ let discriminant = wasmtime_environ::STACK_STATE_RUNNING_DISCRIMINANT;+ self.set_state_no_payload(env, builder, discriminant);+ }++ pub fn set_state_parent<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) {+ let discriminant = wasmtime_environ::STACK_STATE_PARENT_DISCRIMINANT;+ self.set_state_no_payload(env, builder, discriminant);+ }++ pub fn set_state_returned<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) {+ let discriminant = wasmtime_environ::STACK_STATE_RETURNED_DISCRIMINANT;+ self.set_state_no_payload(env, builder, discriminant);+ }++ pub fn set_state_suspended<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) {+ let discriminant = wasmtime_environ::STACK_STATE_SUSPENDED_DISCRIMINANT;+ self.set_state_no_payload(env, builder, discriminant);+ }++ /// Checks whether the `VMStackState` reflects that the stack has ever been+ /// active (instead of just having been allocated, but never resumed).+ pub fn was_invoked<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let actual_state = self.load_state(env, builder);+ let allocated = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT;+ builder+ .ins()+ .icmp_imm(IntCC::NotEqual, actual_state, i64::from(allocated))+ }++ pub fn get_handler_list<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> VMHandlerList {+ let offset: i64 = env.offsets.ptr.vmcommon_stack_information_handlers().into();+ let address = builder.ins().iadd_imm(self.address, offset);+ VMHandlerList::new(address)+ }++ pub fn get_first_switch_handler_index<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ // Field first_switch_handler_index has type u32+ let memflags = ir::MemFlags::trusted();+ let offset: i32 = env+ .offsets+ .ptr+ .vmcommon_stack_information_first_switch_handler_index()+ .into();+ builder.ins().load(I32, memflags, self.address, offset)+ }++ pub fn set_first_switch_handler_index<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ value: ir::Value,+ ) {+ // Field first_switch_handler_index has type u32+ let memflags = ir::MemFlags::trusted();+ let offset: i32 = env+ .offsets+ .ptr+ .vmcommon_stack_information_first_switch_handler_index()+ .into();+ builder.ins().store(memflags, value, self.address, offset);+ }++ /// Sets `last_wasm_entry_sp` and `stack_limit` fields in+ /// `VMRuntimelimits` using the values from the `VMStackLimits` of this+ /// object.+ pub fn write_limits_to_vmcontext<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ vmruntime_limits_ptr: ir::Value,+ ) {+ let stack_limits_ptr = self.get_stack_limits_ptr(env, builder);++ let memflags = ir::MemFlags::trusted();++ let mut copy_to_vm_runtime_limits = |our_offset, their_offset| {+ let our_value = builder.ins().load(+ env.pointer_type(),+ memflags,+ stack_limits_ptr,+ i32::from(our_offset),+ );+ builder.ins().store(+ memflags,+ our_value,+ vmruntime_limits_ptr,+ i32::from(their_offset),+ );+ };++ let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap();+ let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit();+ let last_wasm_entry_fp_offset = env.offsets.ptr.vmstack_limits_last_wasm_entry_fp();+ copy_to_vm_runtime_limits(+ stack_limit_offset,+ pointer_size.vmstore_context_stack_limit(),+ );+ copy_to_vm_runtime_limits(+ last_wasm_entry_fp_offset,+ pointer_size.vmstore_context_last_wasm_entry_fp(),+ );+ }++ /// Overwrites the `last_wasm_entry_fp` field of the `VMStackLimits`+ /// object in the `VMStackLimits` of this object by loading the corresponding+ /// field from the `VMRuntimeLimits`.+ /// If `load_stack_limit` is true, we do the same for the `stack_limit`+ /// field.+ pub fn load_limits_from_vmcontext<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ vmruntime_limits_ptr: ir::Value,+ load_stack_limit: bool,+ ) {+ let stack_limits_ptr = self.get_stack_limits_ptr(env, builder);++ let memflags = ir::MemFlags::trusted();+ let pointer_size = u8::try_from(env.pointer_type().bytes()).unwrap();++ let mut copy = |runtime_limits_offset, stack_limits_offset| {+ let from_vm_runtime_limits = builder.ins().load(+ env.pointer_type(),+ memflags,+ vmruntime_limits_ptr,+ runtime_limits_offset,+ );+ builder.ins().store(+ memflags,+ from_vm_runtime_limits,+ stack_limits_ptr,+ stack_limits_offset,+ );+ };++ let last_wasm_entry_fp_offset = env.offsets.ptr.vmstack_limits_last_wasm_entry_fp();+ copy(+ pointer_size.vmstore_context_last_wasm_entry_fp(),+ last_wasm_entry_fp_offset,+ );++ if load_stack_limit {+ let stack_limit_offset = env.offsets.ptr.vmstack_limits_stack_limit();+ copy(+ pointer_size.vmstore_context_stack_limit(),+ stack_limit_offset,+ );+ }+ }+ }++ impl VMContinuationStack {+ /// The parameter is NOT the "top of stack" address of the stack itself. In line+ /// with how the (runtime) `FiberStack` type works, this is a pointer to+ /// the TOS address.+ pub fn new(tos_ptr: ir::Value) -> Self {+ Self { tos_ptr }+ }++ fn load_top_of_stack<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let mem_flags = ir::MemFlags::trusted();+ builder+ .ins()+ .load(env.pointer_type(), mem_flags, self.tos_ptr, 0)+ }++ /// Returns address of the control context stored in the stack memory,+ /// as used by stack_switch instructions.+ pub fn load_control_context<'a>(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let tos = self.load_top_of_stack(env, builder);+ // Control context begins 24 bytes below top of stack (see unix.rs)+ builder.ins().iadd_imm(tos, -0x18)+ }+ }+}++use helpers::VMStackChain;+use stack_switching_helpers as helpers;++/// Stores the given arguments in the appropriate `VMPayloads` object in the+/// continuation. If the continuation was never invoked, use the `args` object.+/// Otherwise, use the `values` object.+pub(crate) fn vmcontref_store_payloads<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ values: &[ir::Value],+ contref: ir::Value,+) {+ let count =+ i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32");+ if values.len() > 0 {+ let use_args_block = builder.create_block();+ let use_payloads_block = builder.create_block();+ let store_data_block = builder.create_block();+ builder.append_block_param(store_data_block, env.pointer_type());++ let co = helpers::VMContRef::new(contref);+ let csi = co.common_stack_information(env, builder);+ let was_invoked = csi.was_invoked(env, builder);+ builder+ .ins()+ .brif(was_invoked, use_payloads_block, &[], use_args_block, &[]);++ {+ builder.switch_to_block(use_args_block);+ builder.seal_block(use_args_block);++ let args = co.args(env, builder);+ let ptr = args.occupy_next_slots(env, builder, count);++ builder+ .ins()+ .jump(store_data_block, &[BlockArg::Value(ptr)]);+ }++ {+ builder.switch_to_block(use_payloads_block);+ builder.seal_block(use_payloads_block);++ let payloads = co.values(env, builder);++ // This also checks that the buffer is large enough to hold+ // `values.len()` more elements.+ let ptr = payloads.occupy_next_slots(env, builder, count);+ builder+ .ins()+ .jump(store_data_block, &[BlockArg::Value(ptr)]);+ }++ {+ builder.switch_to_block(store_data_block);+ builder.seal_block(store_data_block);++ let ptr = builder.block_params(store_data_block)[0];++ // Store the values.+ let memflags = ir::MemFlags::trusted();+ let mut offset = 0;+ for value in values {+ builder.ins().store(memflags, *value, ptr, offset);+ offset += i32::from(env.offsets.ptr.maximum_value_size());+ }+ }+ }+}++pub(crate) fn tag_address<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ index: u32,+) -> ir::Value {+ let vmctx = env.vmctx_val(&mut builder.cursor());+ let tag_index = wasmtime_environ::TagIndex::from_u32(index);+ let pointer_type = env.pointer_type();+ if let Some(def_index) = env.module.defined_tag_index(tag_index) {+ let offset = i32::try_from(env.offsets.vmctx_vmtag_definition(def_index)).unwrap();+ builder.ins().iadd_imm(vmctx, i64::from(offset))+ } else {+ let offset = i32::try_from(env.offsets.vmctx_vmtag_import_from(tag_index)).unwrap();+ builder.ins().load(+ pointer_type,+ ir::MemFlags::trusted().with_readonly(),+ vmctx,+ ir::immediates::Offset32::new(offset),+ )+ }+}++/// Returns the stack chain saved in the given `VMContext`. Note that the+/// head of the list is the actively running stack (initial stack or+/// continuation).+pub fn vmctx_load_stack_chain<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ vmctx: ir::Value,+) -> VMStackChain {+ let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into();++ // First we need to get the `VMStoreContext`.+ let vm_store_context_offset = env.offsets.ptr.vmctx_store_context();+ let vm_store_context = builder.ins().load(+ env.pointer_type(),+ MemFlags::trusted(),+ vmctx,+ vm_store_context_offset,+ );++ VMStackChain::load(+ env,+ builder,+ vm_store_context,+ stack_chain_offset,+ env.pointer_type(),+ )+}++/// Stores the given stack chain saved in the `VMContext`, overwriting the+/// exsiting one.+pub fn vmctx_store_stack_chain<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ vmctx: ir::Value,+ stack_chain: &VMStackChain,+) {+ let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into();++ // First we need to get the `VMStoreContext`.+ let vm_store_context_offset = env.offsets.ptr.vmctx_store_context();+ let vm_store_context = builder.ins().load(+ env.pointer_type(),+ MemFlags::trusted(),+ vmctx,+ vm_store_context_offset,+ );++ stack_chain.store(env, builder, vm_store_context, stack_chain_offset)+}++/// Similar to `vmctx_store_stack_chain`, but instead of storing an arbitrary+/// `VMStackChain`, stores VMStackChain::Continuation(contref)`.+pub fn vmctx_set_active_continuation<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ vmctx: ir::Value,+ contref: ir::Value,+) {+ let chain = VMStackChain::from_continuation(env, builder, contref);+ vmctx_store_stack_chain(env, builder, vmctx, &chain)+}++pub fn vmctx_load_vm_runtime_limits_ptr<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ vmctx: ir::Value,+) -> ir::Value {+ let pointer_type = env.pointer_type();+ let offset = i32::from(env.offsets.ptr.vmctx_store_context());++ // The *pointer* to the VMRuntimeLimits does not change within the+ // same function, allowing us to set the `read_only` flag.+ let flags = ir::MemFlags::trusted().with_readonly();++ builder.ins().load(pointer_type, flags, vmctx, offset)+}++/// This function generates code that searches for a handler for `tag_address`,+/// which must be a `*mut VMTagDefinition`. The search walks up the chain of+/// continuations beginning at `start`.+///+/// The flag `search_suspend_handlers` determines whether we search for a+/// suspend or switch handler. Concretely, this influences which part of each+/// handler list we will search.+///+/// We trap if no handler was found.+///+/// The returned values are:+/// 1. The stack (continuation or initial stack, represented as a VMStackChain) in+/// whose handler list we found the tag (i.e., the stack that performed the+/// resume instruction that installed handler for the tag).+/// 2. The continuation whose parent is the stack mentioned in 1.+/// 3. The index of the handler in the handler list.+///+/// In pseudo-code, the generated code's behavior can be expressed as+/// follows:+///+/// chain_link = start+/// while !chain_link.is_initial_stack() {+/// contref = chain_link.get_contref()+/// parent_link = contref.parent+/// parent_csi = parent_link.get_common_stack_information();+/// handlers = parent_csi.handlers;+/// (begin_range, end_range) = if search_suspend_handlers {+/// (0, parent_csi.first_switch_handler_index)+/// } else {+/// (parent_csi.first_switch_handler_index, handlers.length)+/// };+/// for index in begin_range..end_range {+/// if handlers[index] == tag_address {+/// goto on_match(contref, index)+/// }+/// }+/// chain_link = parent_link+/// }+/// trap(unhandled_tag)+///+/// on_match(conref : VMContRef, handler_index : u32)+/// ... execution continues here here ...+///+fn search_handler<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ start: &helpers::VMStackChain,+ tag_address: ir::Value,+ search_suspend_handlers: bool,+) -> (VMStackChain, ir::Value, ir::Value) {+ let handle_link = builder.create_block();+ let begin_search_handler_list = builder.create_block();+ let try_index = builder.create_block();+ let compare_tags = builder.create_block();+ let on_match = builder.create_block();+ let on_no_match = builder.create_block();+ let block_args = start.to_raw_parts().map(|v| BlockArg::Value(v));++ // Terminate previous block:+ builder.ins().jump(handle_link, &block_args);++ // Block handle_link+ let chain_link = {+ builder.append_block_param(handle_link, env.pointer_type());+ builder.append_block_param(handle_link, env.pointer_type());+ builder.switch_to_block(handle_link);++ let raw_parts = builder.block_params(handle_link);+ let chain_link = helpers::VMStackChain::from_raw_parts([raw_parts[0], raw_parts[1]]);+ let is_initial_stack = chain_link.is_initial_stack(env, builder);+ builder.ins().brif(+ is_initial_stack,+ on_no_match,+ &[],+ begin_search_handler_list,+ &[],+ );+ chain_link+ };++ // Block begin_search_handler_list+ let (contref, parent_link, handler_list_data_ptr, end_range) = {+ builder.switch_to_block(begin_search_handler_list);+ let contref = chain_link.unchecked_get_continuation();+ let contref = helpers::VMContRef::new(contref);++ let parent_link = contref.get_parent_stack_chain(env, builder);+ let parent_csi = parent_link.get_common_stack_information(env, builder);++ let handlers = parent_csi.get_handler_list(env, builder);+ let handler_list_data_ptr = handlers.get_data(env, builder);++ let first_switch_handler_index = parent_csi.get_first_switch_handler_index(env, builder);++ // Note that these indices are inclusive-exclusive, i.e. [begin_range, end_range).+ let (begin_range, end_range) = if search_suspend_handlers {+ let zero = builder.ins().iconst(I32, 0);+ (zero, first_switch_handler_index)+ } else {+ let length = handlers.get_length(env, builder);+ (first_switch_handler_index, length)+ };++ builder+ .ins()+ .jump(try_index, &[BlockArg::Value(begin_range)]);++ (contref, parent_link, handler_list_data_ptr, end_range)+ };++ // Block try_index+ let index = {+ builder.append_block_param(try_index, I32);+ builder.switch_to_block(try_index);+ let index = builder.block_params(try_index)[0];++ let in_bounds = builder+ .ins()+ .icmp(IntCC::UnsignedLessThan, index, end_range);+ let block_args = parent_link.to_raw_parts().map(|v| BlockArg::Value(v));+ builder+ .ins()+ .brif(in_bounds, compare_tags, &[], handle_link, &block_args);+ index+ };++ // Block compare_tags+ {+ builder.switch_to_block(compare_tags);++ let base = handler_list_data_ptr;+ let entry_size = env.pointer_type().bytes();+ let offset = builder.ins().imul_imm(index, i64::from(entry_size));+ let offset = builder.ins().uextend(I64, offset);+ let entry_address = builder.ins().iadd(base, offset);++ let memflags = ir::MemFlags::trusted();++ let handled_tag = builder+ .ins()+ .load(env.pointer_type(), memflags, entry_address, 0);++ let tags_match = builder.ins().icmp(IntCC::Equal, handled_tag, tag_address);+ let incremented_index = builder.ins().iadd_imm(index, 1);+ builder.ins().brif(+ tags_match,+ on_match,+ &[],+ try_index,+ &[BlockArg::Value(incremented_index)],+ );+ }++ // Block on_no_match+ {+ builder.switch_to_block(on_no_match);+ builder.set_cold_block(on_no_match);+ builder.ins().trap(crate::TRAP_UNHANDLED_TAG);+ }++ builder.seal_block(handle_link);+ builder.seal_block(begin_search_handler_list);+ builder.seal_block(try_index);+ builder.seal_block(compare_tags);+ builder.seal_block(on_match);+ builder.seal_block(on_no_match);++ // final block: on_match+ builder.switch_to_block(on_match);++ (parent_link, contref.address, index)+}++pub(crate) fn translate_cont_bind<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ contobj: ir::Value,+ args: &[ir::Value],+) -> ir::Value {+ let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj);++ // The typing rules for cont.bind allow a null reference to be passed to it.+ builder.ins().trapz(contref, crate::TRAP_NULL_REFERENCE);++ let mut vmcontref = helpers::VMContRef::new(contref);+ let revision = vmcontref.get_revision(env, builder);+ let evidence = builder.ins().icmp(IntCC::Equal, witness, revision);+ builder+ .ins()+ .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);++ vmcontref_store_payloads(env, builder, args, contref);++ let revision = vmcontref.incr_revision(env, builder, revision);+ let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref);+ contobj+}++pub(crate) fn translate_cont_new<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ func: ir::Value,+ arg_types: &[WasmValType],+ return_types: &[WasmValType],+) -> WasmResult<ir::Value> {+ // The typing rules for cont.new allow a null reference to be passed to it.+ builder.ins().trapz(func, crate::TRAP_NULL_REFERENCE);++ let nargs = builder+ .ins()+ .iconst(I32, i64::try_from(arg_types.len()).unwrap());+ let nreturns = builder+ .ins()+ .iconst(I32, i64::try_from(return_types.len()).unwrap());++ let cont_new_func = super::builtins::cont_new(env, &mut builder.func)?;+ let vmctx = env.vmctx_val(&mut builder.cursor());+ let call_inst = builder+ .ins()+ .call(cont_new_func, &[vmctx, func, nargs, nreturns]);+ let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap();++ let tag = helpers::VMContRef::new(contref).get_revision(env, builder);+ let contobj = fatpointer::construct(env, &mut builder.cursor(), tag, contref);+ Ok(contobj)+}++pub(crate) fn translate_resume<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ type_index: u32,+ resume_contobj: ir::Value,+ resume_args: &[ir::Value],+ resumetable: &[(u32, Option<ir::Block>)],+) -> WasmResult<Vec<ir::Value>> {+ // The resume instruction is the most involved instruction to+ // compile as it is responsible for both continuation application+ // and control tag dispatch.+ //+ // Here we translate a resume instruction into several basic+ // blocks as follows:+ //+ // previous block+ // |+ // |+ // resume_block+ // / \+ // / \+ // | |+ // return_block |+ // suspend block+ // |+ // dispatch block+ //+ // * resume_block handles continuation arguments and performs+ // actual stack switch. On ordinary return from resume, it jumps+ // to the `return_block`, whereas on suspension it jumps to the+ // `suspend_block`.+ // * suspend_block is used on suspension, jumps onward to+ // `dispatch_block`.+ // * dispatch_block uses a jump table to dispatch to actual+ // user-defined handler blocks, based on the handler index+ // provided on suspension. Note that we do not jump to the+ // handler blocks directly. Instead, each handler block has a+ // corresponding premable block, which we jump to in order to+ // reach a particular handler block. The preamble block prepares+ // the arguments and continuation object to be passed to the+ // actual handler block.+ //+ let resume_block = builder.create_block();+ let return_block = builder.create_block();+ let suspend_block = builder.create_block();+ let dispatch_block = builder.create_block();++ let vmctx = env.vmctx_val(&mut builder.cursor());++ // Split the resumetable into suspend handlers (each represented by the tag+ // index and handler block) and the switch handlers (represented just by the+ // tag index). Note that we currently don't remove duplicate tags.+ let (suspend_handlers, switch_tags): (Vec<(u32, Block)>, Vec<u32>) = resumetable+ .iter()+ .partition_map(|(tag_index, block_opt)| match block_opt {+ Some(block) => Either::Left((*tag_index, *block)),+ None => Either::Right(*tag_index),+ });++ // Technically, there is no need to have a dedicated resume block, we could+ // just put all of its contents into the current block.+ builder.ins().jump(resume_block, &[]);++ // Resume block: actually resume the continuation chain ending at `resume_contref`.+ let (resume_result, vm_runtime_limits_ptr, original_stack_chain, new_stack_chain) = {+ builder.switch_to_block(resume_block);+ builder.seal_block(resume_block);++ let (witness, resume_contref) =+ fatpointer::deconstruct(env, &mut builder.cursor(), resume_contobj);++ // The typing rules for resume allow a null reference to be passed to it.+ builder+ .ins()+ .trapz(resume_contref, crate::TRAP_NULL_REFERENCE);++ let mut vmcontref = helpers::VMContRef::new(resume_contref);++ let revision = vmcontref.get_revision(env, builder);+ let evidence = builder.ins().icmp(IntCC::Equal, revision, witness);+ builder+ .ins()+ .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);+ let _next_revision = vmcontref.incr_revision(env, builder, revision);++ if resume_args.len() > 0 {+ // We store the arguments in the `VMContRef` to be resumed.+ vmcontref_store_payloads(env, builder, resume_args, resume_contref);+ }++ // Splice together stack chains:+ // Connect the end of the chain starting at `resume_contref` to the currently active chain.+ let mut last_ancestor = helpers::VMContRef::new(vmcontref.get_last_ancestor(env, builder));++ // Make the currently running continuation (if any) the parent of the one we are about to resume.+ let original_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);+ last_ancestor.set_parent_stack_chain(env, builder, &original_stack_chain);++ // Just for consistency: `vmcontref` is about to get state Running, so let's zero out its last_ancestor field.+ let zero = builder.ins().iconst(env.pointer_type(), 0);+ vmcontref.set_last_ancestor(env, builder, zero);++ // We mark `resume_contref` as the currently running one+ vmctx_set_active_continuation(env, builder, vmctx, resume_contref);++ // Note that the resume_contref libcall a few lines further below+ // manipulates the stack limits as follows:+ // 1. Copy stack_limit, last_wasm_entry_sp and last_wasm_exit* values from+ // VMRuntimeLimits into the currently active continuation (i.e., the+ // one that will become the parent of the to-be-resumed one)+ //+ // 2. Copy `stack_limit` and `last_wasm_entry_sp` in the+ // `VMStackLimits` of `resume_contref` into the `VMRuntimeLimits`.+ //+ // See the comment on `wasmtime_environ::VMStackChain` for a+ // description of the invariants that we maintain for the various stack+ // limits.++ // `resume_contref` is now active, and its parent is suspended.+ let resume_contref = helpers::VMContRef::new(resume_contref);+ let resume_csi = resume_contref.common_stack_information(env, builder);+ let parent_csi = original_stack_chain.get_common_stack_information(env, builder);+ resume_csi.set_state_running(env, builder);+ parent_csi.set_state_parent(env, builder);++ // We update the `VMStackLimits` of the parent of the continuation to be resumed+ // as well as the `VMRuntimeLimits`.+ // See the comment on `wasmtime_environ::VMStackChain` for a description+ // of the invariants that we maintain for the various stack limits.+ let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx);+ parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true);+ resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);++ // Install handlers in (soon to be) parent's VMHandlerList:+ // Let the i-th handler clause be (on $tag $block).+ // Then the i-th entry of the VMHandlerList will be the address of $tag.+ let handler_list = parent_csi.get_handler_list(env, builder);++ if resumetable.len() > 0 {+ // Total number of handlers (suspend and switch).+ let handler_count = u32::try_from(resumetable.len()).unwrap();+ // Populate the Array's data ptr with a pointer to a sufficiently+ // large area on this stack.+ env.stack_switching_handler_list_buffer =+ Some(handler_list.allocate_or_reuse_stack_slot(+ env,+ builder,+ handler_count,+ env.stack_switching_handler_list_buffer,+ ));++ let suspend_handler_count = suspend_handlers.len();++ // All handlers, represented by the indices of the tags they handle.+ // All the suspend handlers come first, followed by all the switch handlers.+ let all_handlers = suspend_handlers+ .iter()+ .map(|(tag_index, _block)| *tag_index)+ .chain(switch_tags);++ // Translate all tag indices to tag addresses (i.e., the corresponding *mut VMTagDefinition).+ let all_tag_addresses: Vec<ir::Value> = all_handlers+ .map(|tag_index| tag_address(env, builder, tag_index))+ .collect();++ // Store all tag addresess in the handler list.+ handler_list.store_data_entries(env, builder, &all_tag_addresses);++ // To enable distinguishing switch and suspend handlers when searching the handler list:+ // Store at which index the switch handlers start.+ let first_switch_handler_index = builder+ .ins()+ .iconst(I32, i64::try_from(suspend_handler_count).unwrap());+ parent_csi.set_first_switch_handler_index(env, builder, first_switch_handler_index);+ }++ let resume_payload = ControlEffect::encode_resume(builder).to_u64();++ // Note that the control context we use for switching is not the one in+ // (the stack of) resume_contref, but in (the stack of) last_ancestor!+ let fiber_stack = last_ancestor.get_fiber_stack(env, builder);+ let control_context_ptr = fiber_stack.load_control_context(env, builder);++ let result =+ builder+ .ins()+ .stack_switch(control_context_ptr, control_context_ptr, resume_payload);++ // At this point we know nothing about the continuation that just+ // suspended or returned. In particular, it does not have to be what we+ // called `resume_contref` earlier on. We must reload the information+ // about the now active continuation from the VMContext.+ let new_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);++ // Now the parent contref (or initial stack) is active again+ vmctx_store_stack_chain(env, builder, vmctx, &original_stack_chain);+ parent_csi.set_state_running(env, builder);++ // Just for consistency: Clear the handler list.+ handler_list.clear(env, builder, true);+ parent_csi.set_first_switch_handler_index(env, builder, zero);++ // Extract the result and signal bit.+ let result = ControlEffect::from_u64(result);+ let signal = result.signal(builder);++ // Jump to the return block if the result signal is 0, otherwise jump to+ // the suspend block.+ builder+ .ins()+ .brif(signal, suspend_block, &[], return_block, &[]);++ (+ result,+ vm_runtime_limits_ptr,+ original_stack_chain,+ new_stack_chain,+ )+ };++ // The suspend block: Only used when we suspended, not for returns.+ // Here we extract the index of the handler to use.+ let (handler_index, suspended_contref, suspended_contobj) = {+ builder.switch_to_block(suspend_block);+ builder.seal_block(suspend_block);++ let suspended_continuation = new_stack_chain.unchecked_get_continuation();+ let mut suspended_continuation = helpers::VMContRef::new(suspended_continuation);+ let suspended_csi = suspended_continuation.common_stack_information(env, builder);++ // Note that at the suspend site, we already+ // 1. Set the state of suspended_continuation to Suspended+ // 2. Set suspended_continuation.last_ancestor+ // 3. Broke the continuation chain at suspended_continuation.last_ancestor++ // We store parts of the VMRuntimeLimits into the continuation that just suspended.+ suspended_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false);++ // Afterwards (!), restore parts of the VMRuntimeLimits from the+ // parent of the suspended continuation (which is now active).+ let parent_csi = original_stack_chain.get_common_stack_information(env, builder);+ parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);++ // Extract the handler index+ let handler_index = resume_result.handler_index(builder);++ let revision = suspended_continuation.get_revision(env, builder);+ let suspended_contobj = fatpointer::construct(+ env,+ &mut builder.cursor(),+ revision,+ suspended_continuation.address,+ );++ // We need to terminate this block before being allowed to switch to+ // another one.+ builder.ins().jump(dispatch_block, &[]);++ (handler_index, suspended_continuation, suspended_contobj)+ };++ // For technical reasons, the jump table needs to have a default+ // block. In our case, it should be unreachable, since the handler+ // index we dispatch on should correspond to a an actual handler+ // block in the jump table.+ let jt_default_block = builder.create_block();+ {+ builder.switch_to_block(jt_default_block);+ builder.set_cold_block(jt_default_block);++ builder.ins().trap(crate::TRAP_UNREACHABLE);+ }++ // We create a preamble block for each of the actual handler blocks: It+ // reads the necessary arguments and passes them to the actual handler+ // block, together with the continuation object.+ let target_preamble_blocks = {+ let mut preamble_blocks = vec![];++ for &(handle_tag, target_block) in &suspend_handlers {+ let preamble_block = builder.create_block();+ preamble_blocks.push(preamble_block);+ builder.switch_to_block(preamble_block);++ let param_types = env.tag_params(TagIndex::from_u32(handle_tag));+ let param_types: Vec<ir::Type> = param_types+ .iter()+ .map(|wty| crate::value_type(env.isa(), *wty))+ .collect();++ let values = suspended_contref.values(env, builder);+ let mut suspend_args: Vec<BlockArg> = values+ .load_data_entries(env, builder, ¶m_types)+ .into_iter()+ .map(|v| BlockArg::Value(v))+ .collect();++ // At the suspend site, we store the suspend args in the the+ // `values` buffer of the VMContRef that was active at the time that+ // the suspend instruction was performed.+ suspend_args.push(BlockArg::Value(suspended_contobj));++ // We clear the suspend args. This is mostly for consistency. Note+ // that we don't zero out the data buffer, we still need it for the++ values.clear(env, builder, false);++ builder.ins().jump(target_block, &suspend_args);+ }++ preamble_blocks+ };++ // Dispatch block. All it does is jump to the right premable block based on+ // the handler index.+ {+ builder.switch_to_block(dispatch_block);+ builder.seal_block(dispatch_block);++ let default_bc = builder.func.dfg.block_call(jt_default_block, &[]);++ let adapter_bcs: Vec<BlockCall> = target_preamble_blocks+ .iter()+ .map(|b| builder.func.dfg.block_call(*b, &[]))+ .collect();++ let jt_data = JumpTableData::new(default_bc, &adapter_bcs);+ let jt = builder.create_jump_table(jt_data);++ builder.ins().br_table(handler_index, jt);++ for preamble_block in target_preamble_blocks {+ builder.seal_block(preamble_block);+ }+ builder.seal_block(jt_default_block);+ }++ // Return block: Jumped to by resume block if continuation+ // returned normally.+ {+ builder.switch_to_block(return_block);+ builder.seal_block(return_block);++ // If we got a return signal, a continuation must have been running.+ let returned_contref = new_stack_chain.unchecked_get_continuation();+ let returned_contref = helpers::VMContRef::new(returned_contref);++ // Restore parts of the VMRuntimeLimits from the parent of the+ // returned continuation (which is now active).+ let parent_csi = original_stack_chain.get_common_stack_information(env, builder);+ parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);++ let returned_csi = returned_contref.common_stack_information(env, builder);+ returned_csi.set_state_returned(env, builder);++ // Load the values returned by the continuation.+ let return_types: Vec<_> = env+ .continuation_returns(TypeIndex::from_u32(type_index))+ .iter()+ .map(|ty| crate::value_type(env.isa(), *ty))+ .collect();+ let payloads = returned_contref.args(env, builder);+ let return_values = payloads.load_data_entries(env, builder, &return_types);+ payloads.clear(env, builder, true);++ Ok(return_values)+ }+}++pub(crate) fn translate_suspend<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ tag_index: u32,+ suspend_args: &[ir::Value],+ tag_return_types: &[ir::Type],+) -> Vec<ir::Value> {+ let tag_addr = tag_address(env, builder, tag_index);++ let vmctx = env.vmctx_val(&mut builder.cursor());+ let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);++ let (_, end_of_chain_contref, handler_index) =+ search_handler(env, builder, &active_stack_chain, tag_addr, true);++ // If we get here, the search_handler logic succeeded (i.e., did not trap).+ // Thus, there is at least one parent, so we are not on the initial stack.+ // Can therefore extract continuation directly.+ let active_contref = active_stack_chain.unchecked_get_continuation();+ let active_contref = helpers::VMContRef::new(active_contref);+ let mut end_of_chain_contref = helpers::VMContRef::new(end_of_chain_contref);++ active_contref.set_last_ancestor(env, builder, end_of_chain_contref.address);++ // In the active_contref's `values` buffer, stack-allocate enough room so that we can+ // later store the following:+ // 1. The suspend arguments+ // 2. Afterwards, the tag return values+ let values = active_contref.values(env, builder);+ let required_capacity =+ u32::try_from(std::cmp::max(suspend_args.len(), tag_return_types.len()))+ .expect("Number of stack switching payloads should fit in u32");++ if required_capacity > 0 {+ env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot(+ env,+ builder,+ required_capacity,+ env.stack_switching_values_buffer,+ ));+ }++ if suspend_args.len() > 0 {+ values.store_data_entries(env, builder, suspend_args);+ }++ // Set current continuation to suspended and break up handler chain.+ let active_contref_csi = active_contref.common_stack_information(env, builder);+ active_contref_csi.set_state_suspended(env, builder);+ let absent_chain_link = VMStackChain::absent(env, builder);+ end_of_chain_contref.set_parent_stack_chain(env, builder, &absent_chain_link);++ let suspend_payload = ControlEffect::encode_suspend(builder, handler_index).to_u64();++ // Note that the control context we use for switching is the one+ // at the end of the chain, not the one in active_contref!+ // This also means that stack_switch saves the information about+ // the current stack in the control context located in the stack+ // of end_of_chain_contref.+ let fiber_stack = end_of_chain_contref.get_fiber_stack(env, builder);+ let control_context_ptr = fiber_stack.load_control_context(env, builder);++ builder+ .ins()+ .stack_switch(control_context_ptr, control_context_ptr, suspend_payload);++ // The return values of the suspend instruction are the tag return values, saved in the `args` buffer.+ let values = active_contref.values(env, builder);+ let return_values = values.load_data_entries(env, builder, tag_return_types);+ // We effectively consume the values and discard the stack allocated buffer.+ values.clear(env, builder, true);++ return_values+}++pub(crate) fn translate_switch<'a>(+ env: &mut crate::func_environ::FuncEnvironment<'a>,+ builder: &mut FunctionBuilder,+ tag_index: u32,+ switchee_contobj: ir::Value,+ switch_args: &[ir::Value],+ return_types: &[ir::Type],+) -> WasmResult<Vec<ir::Value>> {+ let vmctx = env.vmctx_val(&mut builder.cursor());++ // Check and increment revision on switchee continuation object (i.e., the+ // one being switched to). Logically, the switchee continuation extends from+ // `switchee_contref` to `switchee_contref.last_ancestor` (i.e., the end of+ // the parent chain starting at `switchee_contref`).+ let switchee_contref = {+ let (witness, target_contref) =+ fatpointer::deconstruct(env, &mut builder.cursor(), switchee_contobj);++ // The typing rules for switch allow a null reference to be passed to it.+ builder+ .ins()+ .trapz(target_contref, crate::TRAP_NULL_REFERENCE);++ let mut target_contref = helpers::VMContRef::new(target_contref);++ let revision = target_contref.get_revision(env, builder);+ let evidence = builder.ins().icmp(IntCC::Equal, revision, witness);+ builder+ .ins()+ .trapz(evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);+ let _next_revision = target_contref.incr_revision(env, builder, revision);+ target_contref+ };++ // We create the "switcher continuation" (i.e., the one executing switch)+ // from the current execution context: Logically, it extends from the+ // continuation reference executing `switch` (subsequently called+ // `switcher_contref`) to the immediate child (called+ // `switcher_contref_last_ancestor`) of the stack with the corresponding+ // handler (saved in `handler_stack_chain`).+ let (+ switcher_contref,+ switcher_contobj,+ switcher_contref_last_ancestor,+ handler_stack_chain,+ vm_runtime_limits_ptr,+ ) = {+ let tag_addr = tag_address(env, builder, tag_index);+ let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);+ let (handler_stack_chain, last_ancestor, _handler_index) =+ search_handler(env, builder, &active_stack_chain, tag_addr, false);+ let mut last_ancestor = helpers::VMContRef::new(last_ancestor);++ // If we get here, the search_handler logic succeeded (i.e., did not trap).+ // Thus, there is at least one parent, so we are not on the initial stack.+ // Can therefore extract continuation directly.+ let switcher_contref = active_stack_chain.unchecked_get_continuation();+ let mut switcher_contref = helpers::VMContRef::new(switcher_contref);++ switcher_contref.set_last_ancestor(env, builder, last_ancestor.address);++ // In the switcher_contref's `values` buffer, stack-allocate enough room so that we can+ // later store `tag_return_types.len()` when resuming the continuation.+ let values = switcher_contref.values(env, builder);+ let required_capacity = u32::try_from(return_types.len()).unwrap();+ if required_capacity > 0 {+ env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot(+ env,+ builder,+ required_capacity,+ env.stack_switching_values_buffer,+ ));+ }++ let switcher_contref_csi = switcher_contref.common_stack_information(env, builder);+ switcher_contref_csi.set_state_suspended(env, builder);+ // We break off `switcher_contref` from the chain of active+ // continuations, by separating the link between `last_ancestor` and its+ // parent stack.+ let absent = VMStackChain::absent(env, builder);+ last_ancestor.set_parent_stack_chain(env, builder, &absent);++ // Load current runtime limits from `VMContext` and store in the+ // switcher continuation.+ let vm_runtime_limits_ptr = vmctx_load_vm_runtime_limits_ptr(env, builder, vmctx);+ switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false);++ let revision = switcher_contref.get_revision(env, builder);+ let new_contobj = fatpointer::construct(+ env,+ &mut builder.cursor(),+ revision,+ switcher_contref.address,+ );++ (+ switcher_contref,+ new_contobj,+ last_ancestor,+ handler_stack_chain,+ vm_runtime_limits_ptr,+ )+ };++ // Prepare switchee continuation:+ // - Store "ordinary" switch arguments as well as the contobj just+ // synthesized from the current context (i.e., `switcher_contobj`) in the+ // switchee continuation's payload buffer.+ // - Splice switchee's continuation chain with handler stack to form new+ // overall chain of active continuations.+ let (switchee_contref_csi, switchee_contref_last_ancestor) = {+ let mut combined_payloads = switch_args.to_vec();+ combined_payloads.push(switcher_contobj);+ vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address);++ let switchee_contref_csi = switchee_contref.common_stack_information(env, builder);+ switchee_contref_csi.set_state_running(env, builder);++ let switchee_contref_last_ancestor = switchee_contref.get_last_ancestor(env, builder);+ let mut switchee_contref_last_ancestor =+ helpers::VMContRef::new(switchee_contref_last_ancestor);++ switchee_contref_last_ancestor.set_parent_stack_chain(env, builder, &handler_stack_chain);++ (switchee_contref_csi, switchee_contref_last_ancestor)+ };++ // Update VMContext/Store: Update active continuation and `VMRuntimeLimits`.+ {+ vmctx_set_active_continuation(env, builder, vmctx, switchee_contref.address);++ switchee_contref_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);+ }++ // Perform actual stack switch+ {+ let switcher_last_ancestor_fs =+ switcher_contref_last_ancestor.get_fiber_stack(env, builder);+ let switcher_last_ancestor_cc =+ switcher_last_ancestor_fs.load_control_context(env, builder);++ let switchee_last_ancestor_fs =+ switchee_contref_last_ancestor.get_fiber_stack(env, builder);+ let switchee_last_ancestor_cc =+ switchee_last_ancestor_fs.load_control_context(env, builder);++ // The stack switch involves the following control contexts (e.g., IP,+ // SP, FP, ...):+ // - `switchee_last_ancestor_cc` contains the information to continue+ // execution in the switchee/target continuation.+ // - `switcher_last_ancestor_cc` contains the information about how to+ // continue execution once we suspend/return to the stack with the+ // switch handler.+ //+ // In total, the following needs to happen:+ // 1. Load control context at `switchee_last_ancestor_cc` to perform+ // stack switch.+ // 2. Move control context at `switcher_last_ancestor_cc` over to+ // `switchee_last_ancestor_cc`.+ // 3. Upon actual switch, save current control context at+ // `switcher_last_ancestor_cc`.+ //+ // We implement this as follows:+ // 1. We copy `switchee_last_ancestor_cc` to a temporary area on the+ // stack (`tmp_control_context`).+ // 2. We copy `switcher_last_ancestor_cc` over to+ // `switchee_last_ancestor_cc`.+ // 3. We invoke the stack switch instruction such that it reads from the+ // temporary area, and writes to `switcher_last_ancestor_cc`.+ //+ // Note that the temporary area is only accessed once by the+ // `stack_switch` instruction emitted later in this block, meaning that we+ // don't have to worry about its lifetime.+ //+ // NOTE(frank-emrich) The implementation below results in one stack slot+ // being created per switch instruction, even though multiple switch+ // instructions in the same function could safely re-use the same stack+ // slot. Thus, we could implement logic for sharing the stack slot by+ // adding an appropriate field to `FuncEnvironment`.+ //+ // NOTE(frank-emrich) We could avoid the copying to a temporary area by+ // making `stack_switch` do all of the necessary moving itself. However,+ // that would be a rather ad-hoc change to how the instruction uses the+ // two pointers given to it.++ let cctx_size = control_context_size(env.isa().triple())?;+ let slot_size = ir::StackSlotData::new(+ ir::StackSlotKind::ExplicitSlot,+ u32::from(cctx_size),+ u8::try_from(env.pointer_type().bytes()).unwrap(),+ );+ let slot = builder.create_sized_stack_slot(slot_size);+ let tmp_control_context = builder.ins().stack_addr(env.pointer_type(), slot, 0);++ let flags = MemFlags::trusted();+ let mut offset: i32 = 0;+ while offset < i32::from(cctx_size) {+ // switchee_last_ancestor_cc -> tmp control context+ let tmp1 =+ builder+ .ins()+ .load(env.pointer_type(), flags, switchee_last_ancestor_cc, offset);+ builder+ .ins()+ .store(flags, tmp1, tmp_control_context, offset);++ // switcher_last_ancestor_cc -> switchee_last_ancestor_cc+ let tmp2 =+ builder+ .ins()+ .load(env.pointer_type(), flags, switcher_last_ancestor_cc, offset);+ builder+ .ins()+ .store(flags, tmp2, switchee_last_ancestor_cc, offset);++ offset += i32::try_from(env.pointer_type().bytes()).unwrap();+ }++ let switch_payload = ControlEffect::encode_switch(builder).to_u64();++ let _result = builder.ins().stack_switch(+ switcher_last_ancestor_cc,+ tmp_control_context,+ switch_payload,+ );+ }++ // After switching back to the original stack: Load return values, they are+ // stored on the switcher continuation.+ let return_values = {+ let payloads = switcher_contref.values(env, builder);+ let return_values = payloads.load_data_entries(env, builder, return_types);+ // We consume the values and discard the buffer (allocated on this stack)+ payloads.clear(env, builder, true);+ return_values+ };++ Ok(return_values)+}diff --git a/crates/cranelift/src/func_environ/stack_switching/mod.rs b/crates/cranelift/src/func_environ/stack_switching/mod.rsnew file mode 100644index 0000000000..25381f7279--- /dev/null+++ b/crates/cranelift/src/func_environ/stack_switching/mod.rs@@ -0,0 +1,37 @@+mod control_effect;+pub(crate) mod fatpointer;+pub(crate) mod instructions;++pub(crate) mod builtins {+ macro_rules! define_builtin_accessors {+ ( $( $name:ident , )* ) => {+ $(+ #[inline]+ pub fn $name(+ func_env: &mut crate::func_environ::FuncEnvironment<'_>,+ func: &mut crate::ir::Function,+ ) -> wasmtime_environ::WasmResult<crate::ir::FuncRef> {+ #[cfg(feature = "stack-switching")]+ {+ return Ok(func_env.builtin_functions.$name(func));+ }++ #[cfg(not(feature = "stack-switching"))]+ {+ let _ = (func, func_env);+ return Err(wasmtime_environ::wasm_unsupported!(+ "support for Wasm Stack Switching disabled at compile time because the `stack-switching` cargo \+ feature was not enabled"+ ));+ }+ }+ )*+ };+ }++ define_builtin_accessors! {+ cont_new,+ table_grow_cont_obj,+ table_fill_cont_obj,+ }+}diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rsindex b25fc8921d..021c042ebd 100644--- a/crates/cranelift/src/lib.rs+++ b/crates/cranelift/src/lib.rs@@ -213,10 +213,9 @@ fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type {match wasm_ht.top() {WasmHeapTopType::Func => pointer_type,WasmHeapTopType::Any | WasmHeapTopType::Extern | WasmHeapTopType::Exn => ir::types::I32,- WasmHeapTopType::Cont =>- // TODO(10248) This is added in a follow-up PR- {- unimplemented!("codegen for stack switching types not implemented, yet")+ WasmHeapTopType::Cont => {+ // VMContObj is 2 * pointer_size (pointer + usize revision)+ ir::Type::int((2 * pointer_type.bits()).try_into().unwrap()).unwrap()}}}@@ -376,6 +375,11 @@ impl BuiltinFunctionSignatures {AbiParam::new(ir::types::I8)}+ #[cfg(feature = "stack-switching")]+ fn size(&self) -> AbiParam {+ AbiParam::new(self.pointer_type)+ }+fn wasm_signature(&self, builtin: BuiltinFunctionIndex) -> Signature {let mut _cur = 0;macro_rules! iter {diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rsindex 359278a2df..1e15bec0c3 100644--- a/crates/cranelift/src/translate/code_translator.rs+++ b/crates/cranelift/src/translate/code_translator.rs@@ -89,7 +89,7 @@ use cranelift_codegen::ir::{BlockArg, types::*};use cranelift_codegen::packed_option::ReservedValue;use cranelift_frontend::{FunctionBuilder, Variable};use itertools::Itertools;-use smallvec::SmallVec;+use smallvec::{SmallVec, ToSmallVec};use std::collections::{HashMap, hash_map};use std::vec::Vec;use wasmparser::{FuncValidator, MemArg, Operator, WasmModuleResources};@@ -2942,35 +2942,101 @@ pub fn translate_operator(// representation, so we don't actually need to do anything.}- Operator::ContNew { cont_type_index: _ } => {- // TODO(10248) This is added in a follow-up PR- return Err(wasmtime_environ::WasmError::Unsupported(- "codegen for stack switching instructions not implemented, yet".to_string(),- ));+ Operator::ContNew { cont_type_index } => {+ let cont_type_index = TypeIndex::from_u32(*cont_type_index);+ let arg_types: SmallVec<[_; 8]> = environ+ .continuation_arguments(cont_type_index)+ .to_smallvec();+ let result_types: SmallVec<[_; 8]> =+ environ.continuation_returns(cont_type_index).to_smallvec();+ let r = stack.pop1();+ let contobj = environ.translate_cont_new(builder, r, &arg_types, &result_types)?;+ stack.push1(contobj);}Operator::ContBind {- argument_index: _,- result_index: _,+ argument_index,+ result_index,} => {- // TODO(10248) This is added in a follow-up PR- return Err(wasmtime_environ::WasmError::Unsupported(- "codegen for stack switching instructions not implemented, yet".to_string(),- ));+ let src_types = environ.continuation_arguments(TypeIndex::from_u32(*argument_index));+ let dst_arity = environ+ .continuation_arguments(TypeIndex::from_u32(*result_index))+ .len();+ let arg_count = src_types.len() - dst_arity;++ let arg_types = &src_types[0..arg_count];+ for arg_type in arg_types {+ // We can't bind GC objects using cont.bind at the moment: We+ // don't have the necessary infrastructure to traverse the+ // buffers used by cont.bind when looking for GC roots. Thus,+ // this crude check ensures that these buffers can never contain+ // GC roots to begin with.+ if arg_type.is_vmgcref_type_and_not_i31() {+ return Err(wasmtime_environ::WasmError::Unsupported(+ "cont.bind does not support GC types at the moment".into(),+ ));+ }+ }++ let (original_contobj, args) = stack.peekn(arg_count + 1).split_last().unwrap();++ let new_contobj = environ.translate_cont_bind(builder, *original_contobj, args);++ stack.popn(arg_count + 1);+ stack.push1(new_contobj);}- Operator::Suspend { tag_index: _ } => {- // TODO(10248) This is added in a follow-up PR- return Err(wasmtime_environ::WasmError::Unsupported(- "codegen for stack switching instructions not implemented, yet".to_string(),- ));+ Operator::Suspend { tag_index } => {+ let tag_index = TagIndex::from_u32(*tag_index);+ let param_types = environ.tag_params(tag_index).to_vec();+ let return_types: SmallVec<[_; 8]> = environ+ .tag_returns(tag_index)+ .iter()+ .map(|ty| crate::value_type(environ.isa(), *ty))+ .collect();++ let params = stack.peekn(param_types.len());+ let param_count = params.len();++ let return_values =+ environ.translate_suspend(builder, tag_index.as_u32(), params, &return_types);++ stack.popn(param_count);+ stack.pushn(&return_values);}Operator::Resume {- cont_type_index: _,- resume_table: _,+ cont_type_index,+ resume_table: wasm_resume_table,} => {- // TODO(10248) This is added in a follow-up PR- return Err(wasmtime_environ::WasmError::Unsupported(- "codegen for stack switching instructions not implemented, yet".to_string(),- ));+ // We translate the block indices in the wasm resume_table to actual Blocks.+ let mut clif_resume_table = vec![];+ for handle in &wasm_resume_table.handlers {+ match handle {+ wasmparser::Handle::OnLabel { tag, label } => {+ let i = stack.control_stack.len() - 1 - (*label as usize);+ let frame = &mut stack.control_stack[i];+ // This is side-effecting!+ frame.set_branched_to_exit();+ clif_resume_table.push((*tag, Some(frame.br_destination())));+ }+ wasmparser::Handle::OnSwitch { tag } => {+ clif_resume_table.push((*tag, None));+ }+ }+ }++ let cont_type_index = TypeIndex::from_u32(*cont_type_index);+ let arity = environ.continuation_arguments(cont_type_index).len();+ let (contobj, call_args) = stack.peekn(arity + 1).split_last().unwrap();++ let cont_return_vals = environ.translate_resume(+ builder,+ cont_type_index.as_u32(),+ *contobj,+ call_args,+ &clif_resume_table,+ )?;++ stack.popn(arity + 1); // arguments + continuation+ stack.pushn(&cont_return_vals);}Operator::ResumeThrow {cont_type_index: _,@@ -2983,13 +3049,51 @@ pub fn translate_operator());}Operator::Switch {- cont_type_index: _,- tag_index: _,+ cont_type_index,+ tag_index,} => {- // TODO(10248) This is added in a follow-up PR- return Err(wasmtime_environ::WasmError::Unsupported(- "codegen for stack switching instructions not implemented, yet".to_string(),- ));+ // Arguments of the continuation we are going to switch to+ let continuation_argument_types: SmallVec<[_; 8]> = environ+ .continuation_arguments(TypeIndex::from_u32(*cont_type_index))+ .to_smallvec();+ // Arity includes the continuation argument+ let arity = continuation_argument_types.len();+ let (contobj, switch_args) = stack.peekn(arity).split_last().unwrap();++ // Type of the continuation we are going to create by suspending the+ // currently running stack+ let current_continuation_type = continuation_argument_types.last().unwrap();+ let current_continuation_type = current_continuation_type.unwrap_ref_type();++ // Argument types of current_continuation_type. These will in turn+ // be the types of the arguments we receive when someone switches+ // back to this switch instruction+ let current_continuation_arg_types: SmallVec<[_; 8]> =+ match current_continuation_type.heap_type {+ WasmHeapType::ConcreteCont(index) => {+ let mti = index+ .as_module_type_index()+ .expect("Only supporting module type indices on switch for now");++ environ+ .continuation_arguments(TypeIndex::from_u32(mti.as_u32()))+ .iter()+ .map(|ty| crate::value_type(environ.isa(), *ty))+ .collect()+ }+ _ => panic!("Invalid type on switch"),+ };++ let switch_return_values = environ.translate_switch(+ builder,+ *tag_index,+ *contobj,+ switch_args,+ ¤t_continuation_arg_types,+ )?;++ stack.popn(arity);+ stack.pushn(&switch_return_values)}Operator::GlobalAtomicGet { .. }diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rsindex 299ed239e2..65f0225db0 100644--- a/crates/environ/src/builtin.rs+++ b/crates/environ/src/builtin.rs@@ -233,12 +233,12 @@ macro_rules! foreach_builtin_function {// denote the continuation being `None`, `init_contref`// may be 0.#[cfg(feature = "stack-switching")]- table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: u64) -> pointer;+ table_grow_cont_obj(vmctx: vmctx, table: u32, delta: u64, init_contref: pointer, init_revision: size) -> pointer;// `value_contref` and `value_revision` together encode// the Option<VMContObj>, as in previous libcall.#[cfg(feature = "stack-switching")]- table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: u64, len: u64) -> bool;+ table_fill_cont_obj(vmctx: vmctx, table: u32, dst: u64, value_contref: pointer, value_revision: size, len: u64) -> bool;// Return the instance ID for a given vmctx.#[cfg(feature = "gc")]diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rsindex 92104ad181..d9e771b02f 100644--- a/crates/environ/src/gc.rs+++ b/crates/environ/src/gc.rs@@ -40,15 +40,10 @@ pub const VM_GC_HEADER_TYPE_INDEX_OFFSET: u32 = 4;/// Get the byte size of the given Wasm type when it is stored inside the GC/// heap.pub fn byte_size_of_wasm_ty_in_gc_heap(ty: &WasmStorageType) -> u32 {- use crate::{WasmHeapType::*, WasmRefType};match ty {WasmStorageType::I8 => 1,WasmStorageType::I16 => 2,WasmStorageType::Val(ty) => match ty {- WasmValType::Ref(WasmRefType {- nullable: _,- heap_type: ConcreteCont(_) | Cont,- }) => unimplemented!("Stack switching feature not compatbile with GC, yet"),WasmValType::I32 | WasmValType::F32 | WasmValType::Ref(_) => 4,WasmValType::I64 | WasmValType::F64 => 8,WasmValType::V128 => 16,diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex 1b12ed9a61..71fb7b045d 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -161,6 +161,12 @@ pub trait PtrSize {4}+ /// This is the size of the largest value type (i.e. a V128).+ #[inline]+ fn maximum_value_size(&self) -> u8 {+ self.size_of_vmglobal_definition()+ }+// Offsets within `VMStoreContext`/// Return the offset of the `fuel_consumed` field of `VMStoreContext`@@ -333,6 +339,28 @@ pub trait PtrSize {.unwrap()}+ // Offsets within `VMContObj`++ /// Return the offset of `VMContObj::contref`+ fn vmcontobj_contref(&self) -> u8 {+ 0+ }++ /// Return the offset of `VMContObj::revision`+ fn vmcontobj_revision(&self) -> u8 {+ self.size()+ }++ /// Return the size of `VMContObj`.+ fn size_of_vmcontobj(&self) -> u8 {+ u8::try_from(align(+ u32::from(self.vmcontobj_revision())+ + u32::try_from(core::mem::size_of::<usize>()).unwrap(),+ u32::from(self.size()),+ ))+ .unwrap()+ }+// Offsets within `VMContRef`/// Return the offset of `VMContRef::common_stack_information`.@@ -362,7 +390,7 @@ pub trait PtrSize {/// Return the offset of `VMContRef::stack`.fn vmcontref_stack(&self) -> u8 {- self.vmcontref_revision() + 8+ self.vmcontref_revision() + self.size()}/// Return the offset of `VMContRef::args`.diff --git a/crates/fuzzing/src/generators/value.rs b/crates/fuzzing/src/generators/value.rsindex 697982a120..685c655400 100644--- a/crates/fuzzing/src/generators/value.rs+++ b/crates/fuzzing/src/generators/value.rs@@ -18,6 +18,7 @@ pub enum DiffValue {ExternRef { null: bool },AnyRef { null: bool },ExnRef { null: bool },+ ContRef { null: bool },}impl DiffValue {@@ -32,6 +33,7 @@ impl DiffValue {DiffValue::ExternRef { .. } => DiffValueType::ExternRef,DiffValue::AnyRef { .. } => DiffValueType::AnyRef,DiffValue::ExnRef { .. } => DiffValueType::ExnRef,+ DiffValue::ContRef { .. } => DiffValueType::ContRef,}}@@ -189,6 +191,7 @@ impl DiffValue {ExternRef => DiffValue::ExternRef { null: true },AnyRef => DiffValue::AnyRef { null: true },ExnRef => DiffValue::ExnRef { null: true },+ ContRef => DiffValue::ContRef { null: true },};arbitrary::Result::Ok(val)}@@ -236,6 +239,7 @@ impl Hash for DiffValue {DiffValue::FuncRef { null } => null.hash(state),DiffValue::AnyRef { null } => null.hash(state),DiffValue::ExnRef { null } => null.hash(state),+ DiffValue::ContRef { null } => null.hash(state),}}}@@ -273,6 +277,7 @@ impl PartialEq for DiffValue {(Self::ExternRef { null: a }, Self::ExternRef { null: b }) => a == b,(Self::AnyRef { null: a }, Self::AnyRef { null: b }) => a == b,(Self::ExnRef { null: a }, Self::ExnRef { null: b }) => a == b,+ (Self::ContRef { null: a }, Self::ContRef { null: b }) => a == b,_ => false,}}@@ -291,6 +296,7 @@ pub enum DiffValueType {ExternRef,AnyRef,ExnRef,+ ContRef,}impl TryFrom<wasmtime::ValType> for DiffValueType {@@ -310,6 +316,7 @@ impl TryFrom<wasmtime::ValType> for DiffValueType {(true, HeapType::I31) => Ok(Self::AnyRef),(true, HeapType::None) => Ok(Self::AnyRef),(true, HeapType::Exn) => Ok(Self::ExnRef),+ (true, HeapType::Cont) => Ok(Self::ContRef),_ => Err("non-null reference types are not supported yet"),},}diff --git a/crates/fuzzing/src/oracles/diff_spec.rs b/crates/fuzzing/src/oracles/diff_spec.rsindex c5b9edbafc..85fab4b5d7 100644--- a/crates/fuzzing/src/oracles/diff_spec.rs+++ b/crates/fuzzing/src/oracles/diff_spec.rs@@ -107,7 +107,8 @@ impl From<&DiffValue> for SpecValue {DiffValue::FuncRef { .. }| DiffValue::ExternRef { .. }| DiffValue::AnyRef { .. }- | DiffValue::ExnRef { .. } => {+ | DiffValue::ExnRef { .. }+ | DiffValue::ContRef { .. } => {unimplemented!()}}diff --git a/crates/fuzzing/src/oracles/diff_v8.rs b/crates/fuzzing/src/oracles/diff_v8.rsindex 7e0ed12e21..eaf612a4be 100644--- a/crates/fuzzing/src/oracles/diff_v8.rs+++ b/crates/fuzzing/src/oracles/diff_v8.rs@@ -191,6 +191,7 @@ impl DiffInstance for V8Instance {DiffValue::V128(_) => return Ok(None),DiffValue::AnyRef { .. } => unimplemented!(),DiffValue::ExnRef { .. } => unimplemented!(),+ DiffValue::ContRef { .. } => unimplemented!(),});}// JS doesn't support v128 return values@@ -316,6 +317,7 @@ fn get_diff_value(DiffValueType::AnyRef => unimplemented!(),DiffValueType::ExnRef => unimplemented!(),DiffValueType::V128 => unreachable!(),+ DiffValueType::ContRef => unimplemented!(),}}diff --git a/crates/fuzzing/src/oracles/diff_wasmi.rs b/crates/fuzzing/src/oracles/diff_wasmi.rsindex c591c218cd..dbed7f2f27 100644--- a/crates/fuzzing/src/oracles/diff_wasmi.rs+++ b/crates/fuzzing/src/oracles/diff_wasmi.rs@@ -195,6 +195,7 @@ impl From<&DiffValue> for wasmi::Val {}DiffValue::AnyRef { .. } => unimplemented!(),DiffValue::ExnRef { .. } => unimplemented!(),+ DiffValue::ContRef { .. } => unimplemented!(),}}}diff --git a/crates/fuzzing/src/oracles/diff_wasmtime.rs b/crates/fuzzing/src/oracles/diff_wasmtime.rsindex 53c54b73a2..f600005164 100644--- a/crates/fuzzing/src/oracles/diff_wasmtime.rs+++ b/crates/fuzzing/src/oracles/diff_wasmtime.rs@@ -227,6 +227,10 @@ impl From<&DiffValue> for Val {assert!(null);Val::ExnRef(None)}+ DiffValue::ContRef { null } => {+ assert!(null);+ Val::ExnRef(None)+ }}}}@@ -243,6 +247,7 @@ impl From<Val> for DiffValue {Val::FuncRef(r) => DiffValue::FuncRef { null: r.is_none() },Val::AnyRef(r) => DiffValue::AnyRef { null: r.is_none() },Val::ExnRef(e) => DiffValue::ExnRef { null: e.is_none() },+ Val::ContRef(c) => DiffValue::ContRef { null: c.is_none() },}}}diff --git a/crates/wasmtime/src/runtime/coredump.rs b/crates/wasmtime/src/runtime/coredump.rsindex 517c2fbb24..ac557b7f60 100644--- a/crates/wasmtime/src/runtime/coredump.rs+++ b/crates/wasmtime/src/runtime/coredump.rs@@ -210,6 +210,12 @@ impl WasmCoreDump {ty: wasm_encoder::AbstractHeapType::Exn,})}+ Val::ContRef(_) => {+ wasm_encoder::ConstExpr::ref_null(wasm_encoder::HeapType::Abstract {+ shared: false,+ ty: wasm_encoder::AbstractHeapType::Cont,+ })+ }};globals.global(wasm_encoder::GlobalType {diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rsindex 11d7d8427b..88daa9c326 100644--- a/crates/wasmtime/src/runtime/externals/global.rs+++ b/crates/wasmtime/src/runtime/externals/global.rs@@ -280,6 +280,16 @@ impl Global {let new = new.as_ref();definition.write_gc_ref(&mut store, new);}+ Val::ContRef(None) => {+ // Allow null continuation references for globals - these are just placeholders+ definition.write_gc_ref(&mut store, None);+ }+ Val::ContRef(Some(_)) => {+ // TODO(#10248): Implement non-null global continuation reference handling+ return Err(anyhow::anyhow!(+ "setting non-null continuation references in globals not yet supported"+ ));+ }}}Ok(())diff --git a/crates/wasmtime/src/runtime/trampoline/global.rs b/crates/wasmtime/src/runtime/trampoline/global.rsindex 2fad1d81aa..5ef0a6ddf2 100644--- a/crates/wasmtime/src/runtime/trampoline/global.rs+++ b/crates/wasmtime/src/runtime/trampoline/global.rs@@ -70,6 +70,16 @@ pub fn generate_global_export(let new = new.as_ref();global.write_gc_ref(&mut store, new);}+ Val::ContRef(None) => {+ // Allow null continuation references for trampoline globals - these are just placeholders+ global.write_gc_ref(&mut store, None);+ }+ Val::ContRef(Some(_)) => {+ // TODO(#10248): Implement non-null trampoline continuation reference handling+ return Err(anyhow::anyhow!(+ "non-null continuation references in trampoline globals not yet supported"+ ));+ }}}diff --git a/crates/wasmtime/src/runtime/values.rs b/crates/wasmtime/src/runtime/values.rsindex e572b2dfb1..15aa627b8b 100644--- a/crates/wasmtime/src/runtime/values.rs+++ b/crates/wasmtime/src/runtime/values.rs@@ -8,6 +8,13 @@ use wasmtime_environ::WasmHeapTopType;pub use crate::runtime::vm::ValRaw;+/// A stub implementation for continuation references.+///+/// This is a placeholder until continuation objects are fully integrated+/// with the GC system (see #10248).+#[derive(Debug, Clone, Copy)]+pub struct ContRef;+/// Possible runtime values that a WebAssembly module can either consume or/// produce.///@@ -50,6 +57,12 @@ pub enum Val {/// An exception reference.ExnRef(Option<Rooted<ExnRef>>),++ /// A continuation reference.+ ///+ /// Note: This is currently a stub implementation as continuation objects+ /// are not yet fully integrated with the GC system. See #10248.+ ContRef(Option<ContRef>),}macro_rules! accessors {@@ -118,7 +131,7 @@ impl Val {WasmHeapTopType::Extern => Val::ExternRef(None),WasmHeapTopType::Any => Val::AnyRef(None),WasmHeapTopType::Exn => Val::ExnRef(None),- WasmHeapTopType::Cont => todo!(), // FIXME(#10248)+ WasmHeapTopType::Cont => Val::ContRef(None),}}@@ -177,6 +190,12 @@ impl Val {Val::AnyRef(Some(a)) => ValType::Ref(RefType::new(false, a._ty(store)?)),Val::ExnRef(None) => ValType::NULLEXNREF,Val::ExnRef(Some(e)) => ValType::Ref(RefType::new(false, e._ty(store)?.into())),+ Val::ContRef(_) => {+ // TODO(#10248): Return proper continuation reference type when available+ return Err(anyhow::anyhow!(+ "continuation references not yet supported in embedder API"+ ));+ }})}@@ -216,7 +235,8 @@ impl Val {| (Val::FuncRef(_), _)| (Val::ExternRef(_), _)| (Val::AnyRef(_), _)- | (Val::ExnRef(_), _) => false,+ | (Val::ExnRef(_), _)+ | (Val::ContRef(_), _) => false,})}@@ -268,6 +288,12 @@ impl Val {Some(f) => f.to_raw(store),None => ptr::null_mut(),})),+ Val::ContRef(_) => {+ // TODO(#10248): Implement proper continuation reference to_raw conversion+ Err(anyhow::anyhow!(+ "continuation references not yet supported in to_raw conversion"+ ))+ }}}@@ -363,6 +389,7 @@ impl Val {Val::AnyRef(a) => Some(Ref::Any(a)),Val::ExnRef(e) => Some(Ref::Exn(e)),Val::I32(_) | Val::I64(_) | Val::F32(_) | Val::F64(_) | Val::V128(_) => None,+ Val::ContRef(_) => None, // TODO(#10248): Return proper Ref::Cont when available}}@@ -509,6 +536,9 @@ impl Val {// particular store, so they're always considered as "yes I came// from that store",Val::I32(_) | Val::I64(_) | Val::F32(_) | Val::F64(_) | Val::V128(_) => true,++ // Continuation references are not yet associated with stores+ Val::ContRef(_) => true, // TODO(#10248): Proper store association when implemented}}}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rsindex 04982d3c49..5d6f525f8a 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs@@ -276,6 +276,12 @@ impl VMArrayRef {.gc_object_data(self.as_gc_ref()).write_u32(offset, id.into_raw());}+ Val::ContRef(_) => {+ // TODO(#10248): Implement array continuation reference element handling+ return Err(anyhow::anyhow!(+ "setting continuation references in array elements not yet supported"+ ));+ }}Ok(())}@@ -383,6 +389,12 @@ impl VMArrayRef {.gc_object_data(self.as_gc_ref()).write_u32(offset, id.into_raw());}+ Val::ContRef(_) => {+ // TODO(#10248): Implement array continuation reference init handling+ return Err(anyhow::anyhow!(+ "initializing continuation references in array elements not yet supported"+ ));+ }}Ok(())}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rsindex 364faf1c01..a8ecfea074 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs@@ -233,6 +233,12 @@ impl VMStructRef {.gc_object_data(self.as_gc_ref()).write_u32(offset, id.into_raw());}+ Val::ContRef(_) => {+ // TODO(#10248): Implement struct continuation reference field handling+ return Err(anyhow::anyhow!(+ "setting continuation references in struct fields not yet supported"+ ));+ }}Ok(())}@@ -386,6 +392,12 @@ pub(crate) fn initialize_field_impl(.gc_object_data(gc_ref).write_u32(offset, id.into_raw());}+ Val::ContRef(_) => {+ // TODO(#10248): Implement struct continuation reference field init handling+ return Err(anyhow::anyhow!(+ "initializing continuation references in struct fields not yet supported"+ ));+ }}Ok(())}diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex 3e4ca7c5d4..20b3e7154f 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -157,6 +157,7 @@ pub mod raw {(@ty f64x2) => (f64x2);(@ty bool) => (bool);(@ty pointer) => (*mut u8);+ (@ty size) => (usize);}wasmtime_environ::foreach_builtin_function!(libcall);@@ -339,7 +340,7 @@ unsafe fn table_grow_cont_obj(// The following two values together form the initial Option<VMContObj>.// A None value is indicated by the pointer being null.init_value_contref: *mut u8,- init_value_revision: u64,+ init_value_revision: usize,) -> Result<Option<AllocationSize>> {let defined_table_index = DefinedTableIndex::from_u32(defined_table_index);let element = unsafe { VMContObj::from_raw_parts(init_value_contref, init_value_revision) };@@ -416,7 +417,7 @@ unsafe fn table_fill_cont_obj(table_index: u32,dst: u64,value_contref: *mut u8,- value_revision: u64,+ value_revision: usize,len: u64,) -> Result<()> {let instance = store.instance_mut(instance);diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rsindex 2a63d0baa3..32c2c27533 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs@@ -39,19 +39,15 @@ pub use stack::*;/// For performance reasons, the VMContRef at the bottom of this chain/// (i.e., the one pointed to by the VMContObj) has a pointer to the/// other end of the chain (i.e., its last ancestor).-// FIXME(frank-emrich) Does this actually need to be 16-byte aligned any-// more? Now that we use I128 on the Cranelift side (see-// [wasmtime_cranelift::stack_switching::fatpointer::pointer_type]), it-// should be fine to use the natural alignment of the type.-#[repr(C, align(16))]+#[repr(C)]#[derive(Debug, Clone, Copy)]pub struct VMContObj {- pub revision: u64,pub contref: NonNull<VMContRef>,+ pub revision: usize,}impl VMContObj {- pub fn new(contref: NonNull<VMContRef>, revision: u64) -> Self {+ pub fn new(contref: NonNull<VMContRef>, revision: usize) -> Self {Self { contref, revision }}@@ -63,7 +59,7 @@ impl VMContObj {////// Behavior will be undefined if a pointer to data that is not a/// VMContRef is provided.- pub unsafe fn from_raw_parts(contref: *mut u8, revision: u64) -> Option<Self> {+ pub unsafe fn from_raw_parts(contref: *mut u8, revision: usize) -> Option<Self> {NonNull::new(contref.cast::<VMContRef>()).map(|contref| Self::new(contref, revision))}}@@ -208,7 +204,7 @@ pub struct VMContRef {pub last_ancestor: *mut VMContRef,/// Revision counter.- pub revision: u64,+ pub revision: usize,/// The underlying stack.pub stack: VMContinuationStack,@@ -627,6 +623,24 @@ mod tests {);}+ #[test]+ fn check_vm_contobj_offsets() {+ let module = Module::new();+ let offsets = VMOffsets::new(HostPtr, &module);+ assert_eq!(+ offset_of!(VMContObj, contref),+ usize::from(offsets.ptr.vmcontobj_contref())+ );+ assert_eq!(+ offset_of!(VMContObj, revision),+ usize::from(offsets.ptr.vmcontobj_revision())+ );+ assert_eq!(+ size_of::<VMContObj>(),+ usize::from(offsets.ptr.size_of_vmcontobj())+ )+ }+#[test]fn check_vm_contref_offsets() {let module = Module::new();diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex 818b089408..220e973082 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -1056,6 +1056,7 @@ macro_rules! define_builtin_array {(@ty f64x2) => (f64x2);(@ty bool) => (bool);(@ty pointer) => (*mut u8);+ (@ty size) => (usize);(@ty vmctx) => (NonNull<VMContext>);}diff --git a/crates/wasmtime/src/runtime/wave/core.rs b/crates/wasmtime/src/runtime/wave/core.rsindex 338ce4b2d8..fd89da83d7 100644--- a/crates/wasmtime/src/runtime/wave/core.rs+++ b/crates/wasmtime/src/runtime/wave/core.rs@@ -40,6 +40,7 @@ impl WasmValue for crate::Val {Self::ExternRef(_) => WasmTypeKind::Unsupported,Self::AnyRef(_) => WasmTypeKind::Unsupported,Self::ExnRef(_) => WasmTypeKind::Unsupported,+ Self::ContRef(_) => WasmTypeKind::Unsupported,}}diff --git a/src/commands/run.rs b/src/commands/run.rsindex d42e573b4c..e3fd86d27b 100644--- a/src/commands/run.rs+++ b/src/commands/run.rs@@ -746,6 +746,8 @@ impl RunCommand {Val::AnyRef(Some(_)) => println!("<anyref>"),Val::ExnRef(None) => println!("<null exnref>"),Val::ExnRef(Some(_)) => println!("<exnref>"),+ Val::ContRef(None) => println!("<null contref>"),+ Val::ContRef(Some(_)) => println!("<contref>"),}}diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.watnew file mode 100644index 0000000000..76afed2d11--- /dev/null+++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat@@ -0,0 +1,311 @@+;;! target = "x86_64-unknown-linux-gnu"+;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y"+;;! test = "optimize"++(module+ (type $ft (func))+ (tag $t (param i32))+ (type $ct (cont $ft))++ (func $countdown+ (local $i i32)+ (local.set $i (i32.const 10))+ (loop $loop+ ;; suspend and pass countdown to our cosnumer+ (suspend $t (local.get $i))+ ;; decrement i; break if we're at 0+ (local.tee $i (i32.sub (local.get $i) (i32.const 1)))+ (br_if $loop)+ )+ )+ (elem declare func $countdown)++ (func (export "main")+ (local $c (ref $ct))+ (local.set $c (cont.new $ct (ref.func $countdown)))+ (loop $loop+ (block $on_gen (result i32 (ref $ct))+ (resume $ct (on $t $on_gen) (local.get $c))+ ;; no more data, return+ (return)+ )+ ;; stack contains [i32 (ref $ct)]+ (local.set $c)+ (drop) ;; could print here+ (br $loop)+ )+ )+)++;; function u0:0(i64 vmctx, i64) tail {+;; ss0 = explicit_slot 16, align = 65536+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; gv3 = vmctx+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64):+;; @003c v3 = iconst.i32 10+;; v61 = iconst.i64 120+;; @0044 v30 = stack_addr.i64 ss0+;; v59 = iconst.i64 16+;; v60 = iconst.i64 0+;; v57 = iconst.i64 80+;; v56 = iconst.i64 -24+;; v70 = iconst.i64 0x0002_0000_0000+;; @0040 jump block2(v3) ; v3 = 10+;;+;; block2(v4: i32):+;; @0044 v8 = load.i64 notrap aligned v0+8+;; @0044 v9 = load.i64 notrap aligned v8+64+;; @0044 v10 = load.i64 notrap aligned v8+72+;; v65 = iconst.i64 1+;; v64 = iconst.i64 24+;; @003a v2 = iconst.i32 0+;; @0044 jump block4(v9, v10, v4)+;;+;; block4(v11: i64, v12: i64, v52: i32):+;; v73 = iconst.i64 1+;; v74 = icmp eq v11, v73 ; v73 = 1+;; @0044 trapnz v74, user22+;; @0044 jump block5+;;+;; block5:+;; @0044 v14 = load.i64 notrap aligned v12+48+;; @0044 v15 = load.i64 notrap aligned v12+56+;; v75 = iconst.i64 24+;; v76 = iadd v15, v75 ; v75 = 24+;; @0044 v17 = load.i64 notrap aligned v76+8+;; @0044 v18 = load.i32 notrap aligned v15+40+;; v77 = iconst.i32 0+;; v67 = iconst.i32 3+;; v66 = iconst.i64 48+;; @0044 v6 = iadd.i64 v0, v66 ; v66 = 48+;; v62 = iconst.i32 1+;; @0044 jump block6(v77) ; v77 = 0+;;+;; block6(v20: i32):+;; @0044 v21 = icmp ult v20, v18+;; @0044 brif v21, block7, block4(v14, v15, v52)+;;+;; block7:+;; v78 = iconst.i32 3+;; v79 = ishl.i32 v20, v78 ; v78 = 3+;; @0044 v23 = uextend.i64 v79+;; @0044 v24 = iadd.i64 v17, v23+;; @0044 v25 = load.i64 notrap aligned v24+;; v80 = iadd.i64 v0, v66 ; v66 = 48+;; v81 = icmp eq v25, v80+;; v82 = iconst.i32 1+;; v83 = iadd.i32 v20, v82 ; v82 = 1+;; @0044 brif v81, block8, block6(v83)+;;+;; block8:+;; @0044 store.i64 notrap aligned v12, v10+64+;; v84 = iconst.i32 1+;; v85 = iconst.i64 120+;; v86 = iadd.i64 v10, v85 ; v85 = 120+;; @0044 store notrap aligned v84, v86+4 ; v84 = 1+;; @0044 store.i64 notrap aligned v30, v86+8+;; @0044 store.i32 notrap aligned v4, v30+;; @0044 store notrap aligned v84, v86 ; v84 = 1+;; v87 = iconst.i32 3+;; v88 = iconst.i64 16+;; v89 = iadd.i64 v10, v88 ; v88 = 16+;; @0044 store notrap aligned v87, v89 ; v87 = 3+;; v90 = iconst.i64 0+;; @0044 store notrap aligned v90, v12+48 ; v90 = 0+;; @0044 store notrap aligned v90, v12+56 ; v90 = 0+;; v91 = iconst.i64 80+;; v92 = iadd.i64 v12, v91 ; v91 = 80+;; @0044 v43 = load.i64 notrap aligned v92+;; v93 = iconst.i64 -24+;; v94 = iadd v43, v93 ; v93 = -24+;; @0044 v40 = uextend.i64 v20+;; v95 = iconst.i64 0x0002_0000_0000+;; v96 = bor v40, v95 ; v95 = 0x0002_0000_0000+;; @0044 v45 = stack_switch v94, v94, v96+;; @0044 v47 = load.i64 notrap aligned v86+8+;; v97 = iconst.i32 0+;; @0044 store notrap aligned v97, v86 ; v97 = 0+;; @0044 store notrap aligned v97, v86+4 ; v97 = 0+;; @0044 store notrap aligned v90, v86+8 ; v90 = 0+;; v98 = isub.i32 v52, v84 ; v84 = 1+;; @004d brif v98, block2(v98), block10+;;+;; block10:+;; @004f jump block3+;;+;; block3:+;; @0050 jump block1+;;+;; block1:+;; @0050 return+;; }+;;+;; function u0:1(i64 vmctx, i64) tail {+;; ss0 = explicit_slot 8, align = 256+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; gv3 = vmctx+;; sig0 = (i64 vmctx, i32) -> i64 tail+;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail+;; fn0 = colocated u1610612736:7 sig0+;; fn1 = colocated u1610612736:52 sig1+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64):+;; @0056 v2 = iconst.i32 0+;; @0056 v4 = call fn0(v0, v2) ; v2 = 0+;; @0058 trapz v4, user16+;; @0058 v8 = call fn1(v0, v4, v2, v2) ; v2 = 0, v2 = 0+;; @0058 v9 = load.i64 notrap aligned v8+72+;; @0058 v11 = uextend.i128 v9+;; v119 = iconst.i64 64+;; v121 = ishl v11, v119 ; v119 = 64+;; @0058 v10 = uextend.i128 v8+;; @0058 v13 = bor v121, v10+;; v116 = iconst.i64 1+;; @0062 v28 = iconst.i64 0+;; @0062 v29 = iconst.i64 2+;; @0062 v32 = iconst.i32 1+;; v114 = iconst.i64 16+;; @0062 v34 = iconst.i32 2+;; v110 = iconst.i64 24+;; @0062 v45 = stack_addr.i64 ss0+;; v109 = iconst.i64 48+;; @0062 v47 = iadd v0, v109 ; v109 = 48+;; v107 = iconst.i64 80+;; v106 = iconst.i64 -24+;; v125 = iconst.i64 0x0001_0000_0000+;; v108 = iconst.i64 32+;; @005c jump block2(v13)+;;+;; block2(v16: i128):+;; @0062 jump block5+;;+;; block5:+;; @0062 v18 = ireduce.i64 v16+;; @0062 trapz v18, user16+;; @0062 v21 = load.i64 notrap aligned v18+72+;; v128 = iconst.i64 64+;; v129 = ushr.i128 v16, v128 ; v128 = 64+;; @0062 v20 = ireduce.i64 v129+;; @0062 v22 = icmp eq v21, v20+;; @0062 trapz v22, user23+;; v130 = iconst.i64 1+;; v131 = iadd v21, v130 ; v130 = 1+;; @0062 store notrap aligned v131, v18+72+;; @0062 v24 = load.i64 notrap aligned v18+64+;; @0062 v25 = load.i64 notrap aligned v0+8+;; @0062 v26 = load.i64 notrap aligned v25+64+;; @0062 v27 = load.i64 notrap aligned v25+72+;; @0062 store notrap aligned v26, v24+48+;; @0062 store notrap aligned v27, v24+56+;; v132 = iconst.i64 0+;; @0062 store notrap aligned v132, v18+64 ; v132 = 0+;; @0062 v30 = load.i64 notrap aligned v0+8+;; v133 = iconst.i64 2+;; @0062 store notrap aligned v133, v30+64 ; v133 = 2+;; @0062 store notrap aligned v18, v30+72+;; v134 = iconst.i32 1+;; v135 = iconst.i64 16+;; v136 = iadd v18, v135 ; v135 = 16+;; @0062 store notrap aligned v134, v136 ; v134 = 1+;; v137 = iconst.i32 2+;; v138 = iadd v27, v135 ; v135 = 16+;; @0062 store notrap aligned v137, v138 ; v137 = 2+;; @0062 v36 = load.i64 notrap aligned readonly v0+8+;; @0062 v38 = load.i64 notrap aligned v36+56+;; @0062 store notrap aligned v38, v27+8+;; @0062 v39 = load.i64 notrap aligned v36+16+;; @0062 store notrap aligned v39, v27+;; @0062 v41 = load.i64 notrap aligned v18+;; @0062 store notrap aligned v41, v36+16+;; @0062 v42 = load.i64 notrap aligned v18+8+;; @0062 store notrap aligned v42, v36+56+;; v139 = iconst.i64 24+;; v140 = iadd v27, v139 ; v139 = 24+;; @0062 store notrap aligned v134, v140+4 ; v134 = 1+;; @0062 store.i64 notrap aligned v45, v140+8+;; v141 = iadd.i64 v0, v109 ; v109 = 48+;; @0062 store notrap aligned v141, v45+;; @0062 store notrap aligned v134, v140 ; v134 = 1+;; @0062 store notrap aligned v134, v27+40 ; v134 = 1+;; v142 = iconst.i64 80+;; v143 = iadd v24, v142 ; v142 = 80+;; @0062 v54 = load.i64 notrap aligned v143+;; v144 = iconst.i64 -24+;; v145 = iadd v54, v144 ; v144 = -24+;; v146 = iconst.i64 0x0001_0000_0000+;; @0062 v56 = stack_switch v145, v145, v146 ; v146 = 0x0001_0000_0000+;; @0062 v57 = load.i64 notrap aligned v0+8+;; @0062 v58 = load.i64 notrap aligned v57+64+;; @0062 v59 = load.i64 notrap aligned v57+72+;; @0062 store notrap aligned v26, v57+64+;; @0062 store notrap aligned v27, v57+72+;; @0062 store notrap aligned v134, v138 ; v134 = 1+;; v147 = iconst.i32 0+;; @0062 store notrap aligned v147, v140 ; v147 = 0+;; @0062 store notrap aligned v147, v140+4 ; v147 = 0+;; @0062 store notrap aligned v132, v140+8 ; v132 = 0+;; @0062 store notrap aligned v132, v27+40 ; v132 = 0+;; v148 = iconst.i64 32+;; v149 = ushr v56, v148 ; v148 = 32+;; @0062 brif v149, block7, block6+;;+;; block7:+;; @0062 v69 = load.i64 notrap aligned v36+56+;; @0062 store notrap aligned v69, v59+8+;; @0062 v71 = load.i64 notrap aligned v27+;; @0062 store notrap aligned v71, v36+16+;; @0062 v72 = load.i64 notrap aligned v27+8+;; @0062 store notrap aligned v72, v36+56+;; @0062 v74 = load.i64 notrap aligned v59+72+;; @0062 jump block8+;;+;; block9 cold:+;; @0062 trap user11+;;+;; block10:+;; v98 = iconst.i64 120+;; @0062 v79 = iadd.i64 v59, v98 ; v98 = 120+;; @0062 v80 = load.i64 notrap aligned v79+8+;; @0062 v81 = load.i32 notrap aligned v80+;; v154 = iconst.i32 0+;; @0062 store notrap aligned v154, v79 ; v154 = 0+;; @0062 jump block4+;;+;; block8:+;; @0062 v73 = ireduce.i32 v56+;; @0062 br_table v73, block9, [block10]+;;+;; block6:+;; @0062 v84 = load.i64 notrap aligned v27+;; @0062 store notrap aligned v84, v36+16+;; @0062 v85 = load.i64 notrap aligned v27+8+;; @0062 store notrap aligned v85, v36+56+;; @0062 v87 = iconst.i32 4+;; v150 = iconst.i64 16+;; v151 = iadd.i64 v59, v150 ; v150 = 16+;; @0062 store notrap aligned v87, v151 ; v87 = 4+;; v94 = iconst.i64 104+;; @0062 v89 = iadd.i64 v59, v94 ; v94 = 104+;; @0062 v90 = load.i64 notrap aligned v89+8+;; v152 = iconst.i32 0+;; @0062 store notrap aligned v152, v89 ; v152 = 0+;; @0062 store notrap aligned v152, v89+4 ; v152 = 0+;; v153 = iconst.i64 0+;; @0062 store notrap aligned v153, v89+8 ; v153 = 0+;; @0068 return+;;+;; block4:+;; @0062 v76 = uextend.i128 v74+;; v155 = iconst.i64 64+;; v156 = ishl v76, v155 ; v155 = 64+;; @0062 v75 = uextend.i128 v59+;; @0062 v78 = bor v156, v75+;; @006d jump block2(v78)+;; }diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.watnew file mode 100644index 0000000000..bfe8a715b1--- /dev/null+++ b/tests/disas/stack-switching/resume-suspend.wat@@ -0,0 +1,263 @@+;;! target = "x86_64-unknown-linux-gnu"+;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y"+;;! test = "optimize"++(module+ (type $ft (func))+ (tag $t (type $ft))+ (type $ct (cont $ft))++ (func $target (suspend $t))+ (elem declare func $target)++ (func (export "minimal_suspend")+ (local $k (ref null $ct))+ (local.set $k (cont.new $ct (ref.func $target)))+ (block $h (result (ref null $ct))+ (resume $ct (on $t $h) (local.get $k))+ ;; continuation suspended back...+ (ref.null $ct)+ )+ (drop)+ )+)++;; function u0:0(i64 vmctx, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; gv3 = vmctx+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64):+;; @003b v5 = load.i64 notrap aligned v0+8+;; @003b v6 = load.i64 notrap aligned v5+64+;; @003b v7 = load.i64 notrap aligned v5+72+;; v54 = iconst.i64 1+;; v53 = iconst.i64 24+;; @003b v16 = iconst.i32 0+;; @003b jump block2(v6, v7)+;;+;; block2(v8: i64, v9: i64):+;; v62 = iconst.i64 1+;; v63 = icmp eq v8, v62 ; v62 = 1+;; @003b trapnz v63, user22+;; @003b jump block3+;;+;; block3:+;; @003b v11 = load.i64 notrap aligned v9+48+;; @003b v12 = load.i64 notrap aligned v9+56+;; v64 = iconst.i64 24+;; v65 = iadd v12, v64 ; v64 = 24+;; @003b v14 = load.i64 notrap aligned v65+8+;; @003b v15 = load.i32 notrap aligned v12+40+;; v66 = iconst.i32 0+;; v56 = iconst.i32 3+;; v55 = iconst.i64 48+;; @003b v3 = iadd.i64 v0, v55 ; v55 = 48+;; v51 = iconst.i32 1+;; @003b jump block4(v66) ; v66 = 0+;;+;; block4(v17: i32):+;; @003b v18 = icmp ult v17, v15+;; @003b brif v18, block5, block2(v11, v12)+;;+;; block5:+;; v67 = iconst.i32 3+;; v68 = ishl.i32 v17, v67 ; v67 = 3+;; @003b v20 = uextend.i64 v68+;; @003b v21 = iadd.i64 v14, v20+;; @003b v22 = load.i64 notrap aligned v21+;; v69 = iadd.i64 v0, v55 ; v55 = 48+;; v70 = icmp eq v22, v69+;; v71 = iconst.i32 1+;; v72 = iadd.i32 v17, v71 ; v71 = 1+;; @003b brif v70, block6, block4(v72)+;;+;; block6:+;; @003b store.i64 notrap aligned v9, v7+64+;; v73 = iconst.i32 3+;; v48 = iconst.i64 16+;; @003b v28 = iadd.i64 v7, v48 ; v48 = 16+;; @003b store notrap aligned v73, v28 ; v73 = 3+;; v49 = iconst.i64 0+;; @003b store notrap aligned v49, v9+48 ; v49 = 0+;; @003b store notrap aligned v49, v9+56 ; v49 = 0+;; v46 = iconst.i64 80+;; @003b v35 = iadd.i64 v9, v46 ; v46 = 80+;; @003b v36 = load.i64 notrap aligned v35+;; v45 = iconst.i64 -24+;; @003b v37 = iadd v36, v45 ; v45 = -24+;; @003b v33 = uextend.i64 v17+;; v59 = iconst.i64 0x0002_0000_0000+;; v60 = bor v33, v59 ; v59 = 0x0002_0000_0000+;; @003b v38 = stack_switch v37, v37, v60+;; v50 = iconst.i64 120+;; @003b v25 = iadd.i64 v7, v50 ; v50 = 120+;; @003b v40 = load.i64 notrap aligned v25+8+;; v74 = iconst.i32 0+;; @003b store notrap aligned v74, v25 ; v74 = 0+;; @003b store notrap aligned v74, v25+4 ; v74 = 0+;; @003b store notrap aligned v49, v25+8 ; v49 = 0+;; @003d jump block1+;;+;; block1:+;; @003d return+;; }+;;+;; function u0:1(i64 vmctx, i64) tail {+;; ss0 = explicit_slot 8, align = 256+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; gv3 = vmctx+;; sig0 = (i64 vmctx, i32) -> i64 tail+;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail+;; fn0 = colocated u1610612736:7 sig0+;; fn1 = colocated u1610612736:52 sig1+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64):+;; @0043 v7 = iconst.i32 0+;; @0043 v9 = call fn0(v0, v7) ; v7 = 0+;; @0045 trapz v9, user16+;; @0045 v13 = call fn1(v0, v9, v7, v7) ; v7 = 0, v7 = 0+;; @0045 v14 = load.i64 notrap aligned v13+72+;; @004e jump block3+;;+;; block3:+;; @0045 v16 = uextend.i128 v14+;; v130 = iconst.i64 64+;; v134 = ishl v16, v130 ; v130 = 64+;; v136 = ireduce.i64 v134+;; v138 = bor v136, v13+;; @004e trapz v138, user16+;; @004e v24 = load.i64 notrap aligned v138+72+;; @0045 v15 = uextend.i128 v13+;; @0045 v18 = bor v134, v15+;; v140 = ushr v18, v130 ; v130 = 64+;; @004e v23 = ireduce.i64 v140+;; @004e v25 = icmp eq v24, v23+;; @004e trapz v25, user23+;; v125 = iconst.i64 1+;; @004e v26 = iadd v24, v125 ; v125 = 1+;; @004e store notrap aligned v26, v138+72+;; @004e v27 = load.i64 notrap aligned v138+64+;; @004e v28 = load.i64 notrap aligned v0+8+;; @004e v29 = load.i64 notrap aligned v28+64+;; @004e v30 = load.i64 notrap aligned v28+72+;; @004e store notrap aligned v29, v27+48+;; @004e store notrap aligned v30, v27+56+;; @0040 v2 = iconst.i64 0+;; @004e store notrap aligned v2, v138+64 ; v2 = 0+;; @004e v33 = load.i64 notrap aligned v0+8+;; @004e v32 = iconst.i64 2+;; @004e store notrap aligned v32, v33+64 ; v32 = 2+;; @004e store notrap aligned v138, v33+72+;; @004e v35 = iconst.i32 1+;; v123 = iconst.i64 16+;; @004e v36 = iadd v138, v123 ; v123 = 16+;; @004e store notrap aligned v35, v36 ; v35 = 1+;; @004e v37 = iconst.i32 2+;; @004e v38 = iadd v30, v123 ; v123 = 16+;; @004e store notrap aligned v37, v38 ; v37 = 2+;; @004e v39 = load.i64 notrap aligned readonly v0+8+;; @004e v41 = load.i64 notrap aligned v39+56+;; @004e store notrap aligned v41, v30+8+;; @004e v42 = load.i64 notrap aligned v39+16+;; @004e store notrap aligned v42, v30+;; @004e v44 = load.i64 notrap aligned v138+;; @004e store notrap aligned v44, v39+16+;; @004e v45 = load.i64 notrap aligned v138+8+;; @004e store notrap aligned v45, v39+56+;; v119 = iconst.i64 24+;; @004e v46 = iadd v30, v119 ; v119 = 24+;; @004e store notrap aligned v35, v46+4 ; v35 = 1+;; @004e v48 = stack_addr.i64 ss0+;; @004e store notrap aligned v48, v46+8+;; v118 = iconst.i64 48+;; @004e v50 = iadd.i64 v0, v118 ; v118 = 48+;; @004e store notrap aligned v50, v48+;; @004e store notrap aligned v35, v46 ; v35 = 1+;; @004e store notrap aligned v35, v30+40 ; v35 = 1+;; v116 = iconst.i64 80+;; @004e v56 = iadd v27, v116 ; v116 = 80+;; @004e v57 = load.i64 notrap aligned v56+;; v115 = iconst.i64 -24+;; @004e v58 = iadd v57, v115 ; v115 = -24+;; v142 = iconst.i64 0x0001_0000_0000+;; @004e v59 = stack_switch v58, v58, v142 ; v142 = 0x0001_0000_0000+;; @004e v60 = load.i64 notrap aligned v0+8+;; @004e v61 = load.i64 notrap aligned v60+64+;; @004e v62 = load.i64 notrap aligned v60+72+;; @004e store notrap aligned v29, v60+64+;; @004e store notrap aligned v30, v60+72+;; @004e store notrap aligned v35, v38 ; v35 = 1+;; v145 = iconst.i32 0+;; @004e store notrap aligned v145, v46 ; v145 = 0+;; @004e store notrap aligned v145, v46+4 ; v145 = 0+;; @004e store notrap aligned v2, v46+8 ; v2 = 0+;; @004e store notrap aligned v2, v30+40 ; v2 = 0+;; v117 = iconst.i64 32+;; @004e v69 = ushr v59, v117 ; v117 = 32+;; @004e brif v69, block5, block4+;;+;; block5:+;; @004e v72 = load.i64 notrap aligned v39+56+;; @004e store notrap aligned v72, v62+8+;; @004e v74 = load.i64 notrap aligned v30+;; @004e store notrap aligned v74, v39+16+;; @004e v75 = load.i64 notrap aligned v30+8+;; @004e store notrap aligned v75, v39+56+;; @004e v77 = load.i64 notrap aligned v62+72+;; @004e jump block6+;;+;; block7 cold:+;; @004e trap user11+;;+;; block8:+;; v107 = iconst.i64 120+;; @004e v82 = iadd.i64 v62, v107 ; v107 = 120+;; @004e v83 = load.i64 notrap aligned v82+8+;; v153 = iconst.i32 0+;; @004e store notrap aligned v153, v82 ; v153 = 0+;; @004e v79 = uextend.i128 v77+;; v154 = iconst.i64 64+;; v155 = ishl v79, v154 ; v154 = 64+;; @004e v78 = uextend.i128 v62+;; @004e v81 = bor v155, v78+;; @004e jump block2(v81)+;;+;; block6:+;; @004e v76 = ireduce.i32 v59+;; @004e br_table v76, block7, [block8]+;;+;; block4:+;; @004e v86 = load.i64 notrap aligned v30+;; @004e store notrap aligned v86, v39+16+;; @004e v87 = load.i64 notrap aligned v30+8+;; @004e store notrap aligned v87, v39+56+;; @004e v89 = iconst.i32 4+;; v146 = iconst.i64 16+;; v147 = iadd.i64 v62, v146 ; v146 = 16+;; @004e store notrap aligned v89, v147 ; v89 = 4+;; v103 = iconst.i64 104+;; @004e v91 = iadd.i64 v62, v103 ; v103 = 104+;; @004e v92 = load.i64 notrap aligned v91+8+;; v148 = iconst.i32 0+;; @004e store notrap aligned v148, v91 ; v148 = 0+;; @004e store notrap aligned v148, v91+4 ; v148 = 0+;; v149 = iconst.i64 0+;; @004e store notrap aligned v149, v91+8 ; v149 = 0+;; v150 = uextend.i128 v149 ; v149 = 0+;; v151 = iconst.i64 64+;; v152 = ishl v150, v151 ; v151 = 64+;; @0040 v6 = bor v152, v150+;; @0056 jump block2(v6)+;;+;; block2(v19: i128):+;; @0058 jump block1+;;+;; block1:+;; @0058 return+;; }diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.watnew file mode 100644index 0000000000..75c0458dfb--- /dev/null+++ b/tests/disas/stack-switching/symmetric-switch.wat@@ -0,0 +1,418 @@+;;! target = "x86_64-unknown-linux-gnu"+;;! flags = "-W stack-switching=y -W exceptions=y -W function-references=y"++(module+ (type $fta (func))+ (type $cta (cont $fta))++ (type $ftb (func (param (ref $cta))))+ (type $ctb (cont $ftb))++ (tag $yield)++ (func $task_a (type $fta)+ (cont.new $ctb (ref.func $task_b))+ (switch $ctb $yield)+ )++ (func $task_b (type $ftb))++ (elem declare func $task_a $task_b)++ (func (export "entry")+ (cont.new $cta (ref.func $task_a))+ (resume $cta (on $yield switch))+ )+)++;; function u0:0(i64 vmctx, i64) tail {+;; ss0 = explicit_slot 24, align = 256+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; gv3 = vmctx+;; sig0 = (i64 vmctx, i32) -> i64 tail+;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail+;; fn0 = colocated u1610612736:7 sig0+;; fn1 = colocated u1610612736:52 sig1+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64):+;; @003a v2 = iconst.i32 1+;; @003a v4 = call fn0(v0, v2) ; v2 = 1+;; @003c trapz v4, user16+;; @003c v5 = iconst.i32 1+;; @003c v6 = iconst.i32 0+;; @003c v8 = call fn1(v0, v4, v5, v6) ; v5 = 1, v6 = 0+;; @003c v9 = load.i64 notrap aligned v8+72+;; @003c v10 = uextend.i128 v8+;; @003c v11 = uextend.i128 v9+;; v138 = iconst.i64 64+;; v139 = uextend.i128 v138 ; v138 = 64+;; @003c v12 = ishl v11, v139+;; @003c v13 = bor v12, v10+;; @003e v15 = ireduce.i64 v13+;; v136 = iconst.i64 64+;; v137 = uextend.i128 v136 ; v136 = 64+;; @003e v16 = ushr v13, v137+;; @003e v17 = ireduce.i64 v16+;; @003e trapz v15, user16+;; @003e v18 = load.i64 notrap aligned v15+72+;; @003e v19 = icmp eq v18, v17+;; @003e trapz v19, user23+;; v135 = iconst.i64 1+;; @003e v20 = iadd v18, v135 ; v135 = 1+;; @003e store notrap aligned v20, v15+72+;; v134 = iconst.i64 48+;; @003e v22 = iadd v0, v134 ; v134 = 48+;; @003e v23 = load.i64 notrap aligned v0+8+;; @003e v24 = load.i64 notrap aligned v23+64+;; @003e v25 = load.i64 notrap aligned v23+72+;; @003e jump block2(v24, v25)+;;+;; block2(v26: i64, v27: i64):+;; v133 = iconst.i64 1+;; @003e v28 = icmp eq v26, v133 ; v133 = 1+;; @003e trapnz v28, user22+;; @003e jump block3+;;+;; block3:+;; @003e v29 = load.i64 notrap aligned v27+48+;; @003e v30 = load.i64 notrap aligned v27+56+;; v132 = iconst.i64 24+;; @003e v31 = iadd v30, v132 ; v132 = 24+;; @003e v32 = load.i64 notrap aligned v31+8+;; @003e v33 = load.i32 notrap aligned v30+40+;; @003e v34 = load.i32 notrap aligned v31+;; @003e jump block4(v33)+;;+;; block4(v35: i32):+;; @003e v36 = icmp ult v35, v34+;; @003e brif v36, block5, block2(v29, v30)+;;+;; block5:+;; v131 = iconst.i32 8+;; @003e v37 = imul.i32 v35, v131 ; v131 = 8+;; @003e v38 = uextend.i64 v37+;; @003e v39 = iadd.i64 v32, v38+;; @003e v40 = load.i64 notrap aligned v39+;; @003e v41 = icmp eq v40, v22+;; v130 = iconst.i32 1+;; @003e v42 = iadd.i32 v35, v130 ; v130 = 1+;; @003e brif v41, block6, block4(v42)+;;+;; block6:+;; @003e store.i64 notrap aligned v27, v25+64+;; v129 = iconst.i64 120+;; @003e v43 = iadd.i64 v25, v129 ; v129 = 120+;; v128 = iconst.i64 0+;; @003e v44 = iadd.i64 v25, v128 ; v128 = 0+;; @003e v45 = iconst.i32 3+;; v127 = iconst.i64 16+;; @003e v46 = iadd v44, v127 ; v127 = 16+;; @003e store notrap aligned v45, v46 ; v45 = 3+;; @003e v47 = iconst.i64 0+;; @003e v48 = iconst.i64 0+;; @003e store notrap aligned v47, v27+48 ; v47 = 0+;; @003e store notrap aligned v48, v27+56 ; v48 = 0+;; @003e v49 = load.i64 notrap aligned readonly v0+8+;; v126 = iconst.i64 0+;; @003e v50 = iadd v44, v126 ; v126 = 0+;; @003e v51 = load.i64 notrap aligned v49+56+;; @003e store notrap aligned v51, v50+8+;; @003e v52 = load.i64 notrap aligned v25+72+;; @003e v53 = uextend.i128 v25+;; @003e v54 = uextend.i128 v52+;; v124 = iconst.i64 64+;; v125 = uextend.i128 v124 ; v124 = 64+;; @003e v55 = ishl v54, v125+;; @003e v56 = bor v55, v53+;; v123 = iconst.i64 0+;; @003e v58 = iadd.i64 v15, v123 ; v123 = 0+;; v122 = iconst.i64 16+;; @003e v59 = iadd v58, v122 ; v122 = 16+;; @003e v60 = load.i32 notrap aligned v59+;; v121 = iconst.i32 0+;; @003e v61 = icmp ne v60, v121 ; v121 = 0+;; @003e brif v61, block9, block8+;;+;; block8:+;; v120 = iconst.i64 104+;; @003e v62 = iadd.i64 v15, v120 ; v120 = 104+;; @003e v63 = load.i64 notrap aligned v62+8+;; @003e v64 = load.i32 notrap aligned v62+;; v119 = iconst.i32 1+;; @003e v65 = iadd v64, v119 ; v119 = 1+;; @003e store notrap aligned v65, v62+;; @003e v66 = uextend.i64 v64+;; v118 = iconst.i64 16+;; @003e v67 = imul v66, v118 ; v118 = 16+;; @003e v68 = iadd v63, v67+;; @003e jump block10(v68)+;;+;; block9:+;; v117 = iconst.i64 120+;; @003e v69 = iadd.i64 v15, v117 ; v117 = 120+;; @003e v70 = load.i64 notrap aligned v69+8+;; @003e v71 = load.i32 notrap aligned v69+;; v116 = iconst.i32 1+;; @003e v72 = iadd v71, v116 ; v116 = 1+;; @003e store notrap aligned v72, v69+;; @003e v73 = uextend.i64 v71+;; v115 = iconst.i64 16+;; @003e v74 = imul v73, v115 ; v115 = 16+;; @003e v75 = iadd v70, v74+;; @003e jump block10(v75)+;;+;; block10(v57: i64):+;; @003e store.i128 notrap aligned v56, v57+;; v114 = iconst.i64 0+;; @003e v76 = iadd.i64 v15, v114 ; v114 = 0+;; @003e v77 = iconst.i32 1+;; v113 = iconst.i64 16+;; @003e v78 = iadd v76, v113 ; v113 = 16+;; @003e store notrap aligned v77, v78 ; v77 = 1+;; @003e v79 = load.i64 notrap aligned v15+64+;; @003e store.i64 notrap aligned v29, v79+48+;; @003e store.i64 notrap aligned v30, v79+56+;; @003e v80 = iconst.i64 2+;; @003e v81 = load.i64 notrap aligned v0+8+;; @003e store notrap aligned v80, v81+64 ; v80 = 2+;; @003e store.i64 notrap aligned v15, v81+72+;; v112 = iconst.i64 0+;; @003e v82 = iadd v76, v112 ; v112 = 0+;; @003e v83 = load.i64 notrap aligned v82+;; @003e store notrap aligned v83, v49+16+;; @003e v84 = load.i64 notrap aligned v82+8+;; @003e store notrap aligned v84, v49+56+;; v111 = iconst.i64 80+;; @003e v85 = iadd.i64 v27, v111 ; v111 = 80+;; @003e v86 = load.i64 notrap aligned v85+;; v110 = iconst.i64 -24+;; @003e v87 = iadd v86, v110 ; v110 = -24+;; v109 = iconst.i64 80+;; @003e v88 = iadd v79, v109 ; v109 = 80+;; @003e v89 = load.i64 notrap aligned v88+;; v108 = iconst.i64 -24+;; @003e v90 = iadd v89, v108 ; v108 = -24+;; @003e v91 = stack_addr.i64 ss0+;; @003e v92 = load.i64 notrap aligned v90+;; @003e store notrap aligned v92, v91+;; @003e v93 = load.i64 notrap aligned v87+;; @003e store notrap aligned v93, v90+;; @003e v94 = load.i64 notrap aligned v90+8+;; @003e store notrap aligned v94, v91+8+;; @003e v95 = load.i64 notrap aligned v87+8+;; @003e store notrap aligned v95, v90+8+;; @003e v96 = load.i64 notrap aligned v90+16+;; @003e store notrap aligned v96, v91+16+;; @003e v97 = load.i64 notrap aligned v87+16+;; @003e store notrap aligned v97, v90+16+;; @003e v98 = iconst.i64 3+;; v107 = iconst.i64 32+;; @003e v99 = ishl v98, v107 ; v98 = 3, v107 = 32+;; @003e v100 = stack_switch v87, v91, v99+;; v106 = iconst.i64 120+;; @003e v101 = iadd.i64 v25, v106 ; v106 = 120+;; @003e v102 = load.i64 notrap aligned v101+8+;; @003e v103 = iconst.i32 0+;; @003e store notrap aligned v103, v101 ; v103 = 0+;; @003e v104 = iconst.i32 0+;; @003e store notrap aligned v104, v101+4 ; v104 = 0+;; @003e v105 = iconst.i64 0+;; @003e store notrap aligned v105, v101+8 ; v105 = 0+;; @0041 jump block1+;;+;; block1:+;; @0041 return+;; }+;;+;; function u0:1(i64 vmctx, i64, i128) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i128):+;; @0044 jump block1+;;+;; block1:+;; @0044 return+;; }+;;+;; function u0:2(i64 vmctx, i64) tail {+;; ss0 = explicit_slot 8, align = 256+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+16+;; gv3 = vmctx+;; sig0 = (i64 vmctx, i32) -> i64 tail+;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail+;; fn0 = colocated u1610612736:7 sig0+;; fn1 = colocated u1610612736:52 sig1+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64):+;; @0047 v2 = iconst.i32 0+;; @0047 v4 = call fn0(v0, v2) ; v2 = 0+;; @0049 trapz v4, user16+;; @0049 v5 = iconst.i32 0+;; @0049 v6 = iconst.i32 0+;; @0049 v8 = call fn1(v0, v4, v5, v6) ; v5 = 0, v6 = 0+;; @0049 v9 = load.i64 notrap aligned v8+72+;; @0049 v10 = uextend.i128 v8+;; @0049 v11 = uextend.i128 v9+;; v111 = iconst.i64 64+;; v112 = uextend.i128 v111 ; v111 = 64+;; @0049 v12 = ishl v11, v112+;; @0049 v13 = bor v12, v10+;; @004b jump block2+;;+;; block2:+;; @004b v15 = ireduce.i64 v13+;; v109 = iconst.i64 64+;; v110 = uextend.i128 v109 ; v109 = 64+;; @004b v16 = ushr.i128 v13, v110+;; @004b v17 = ireduce.i64 v16+;; @004b trapz v15, user16+;; @004b v18 = load.i64 notrap aligned v15+72+;; @004b v19 = icmp eq v18, v17+;; @004b trapz v19, user23+;; v108 = iconst.i64 1+;; @004b v20 = iadd v18, v108 ; v108 = 1+;; @004b store notrap aligned v20, v15+72+;; @004b v21 = load.i64 notrap aligned v15+64+;; @004b v22 = load.i64 notrap aligned v0+8+;; @004b v23 = load.i64 notrap aligned v22+64+;; @004b v24 = load.i64 notrap aligned v22+72+;; @004b store notrap aligned v23, v21+48+;; @004b store notrap aligned v24, v21+56+;; @004b v25 = iconst.i64 0+;; @004b store notrap aligned v25, v15+64 ; v25 = 0+;; @004b v26 = iconst.i64 2+;; @004b v27 = load.i64 notrap aligned v0+8+;; @004b store notrap aligned v26, v27+64 ; v26 = 2+;; @004b store notrap aligned v15, v27+72+;; v107 = iconst.i64 0+;; @004b v28 = iadd v15, v107 ; v107 = 0+;; @004b v29 = iconst.i32 1+;; v106 = iconst.i64 16+;; @004b v30 = iadd v28, v106 ; v106 = 16+;; @004b store notrap aligned v29, v30 ; v29 = 1+;; @004b v31 = iconst.i32 2+;; v105 = iconst.i64 16+;; @004b v32 = iadd v24, v105 ; v105 = 16+;; @004b store notrap aligned v31, v32 ; v31 = 2+;; @004b v33 = load.i64 notrap aligned readonly v0+8+;; v104 = iconst.i64 0+;; @004b v34 = iadd v24, v104 ; v104 = 0+;; @004b v35 = load.i64 notrap aligned v33+56+;; @004b store notrap aligned v35, v34+8+;; @004b v36 = load.i64 notrap aligned v33+16+;; @004b store notrap aligned v36, v34+;; v103 = iconst.i64 0+;; @004b v37 = iadd v28, v103 ; v103 = 0+;; @004b v38 = load.i64 notrap aligned v37+;; @004b store notrap aligned v38, v33+16+;; @004b v39 = load.i64 notrap aligned v37+8+;; @004b store notrap aligned v39, v33+56+;; v102 = iconst.i64 24+;; @004b v40 = iadd v24, v102 ; v102 = 24+;; @004b v41 = iconst.i32 1+;; @004b v42 = stack_addr.i64 ss0+;; @004b store notrap aligned v41, v40+4 ; v41 = 1+;; @004b store notrap aligned v42, v40+8+;; v101 = iconst.i64 48+;; @004b v44 = iadd.i64 v0, v101 ; v101 = 48+;; @004b v45 = iconst.i32 1+;; @004b v46 = load.i64 notrap aligned v40+8+;; @004b store notrap aligned v44, v46+;; @004b store notrap aligned v45, v40 ; v45 = 1+;; @004b v47 = iconst.i32 0+;; @004b store notrap aligned v47, v24+40 ; v47 = 0+;; @004b v48 = iconst.i64 1+;; v100 = iconst.i64 32+;; @004b v49 = ishl v48, v100 ; v48 = 1, v100 = 32+;; v99 = iconst.i64 80+;; @004b v50 = iadd v21, v99 ; v99 = 80+;; @004b v51 = load.i64 notrap aligned v50+;; v98 = iconst.i64 -24+;; @004b v52 = iadd v51, v98 ; v98 = -24+;; @004b v53 = stack_switch v52, v52, v49+;; @004b v54 = load.i64 notrap aligned v0+8+;; @004b v55 = load.i64 notrap aligned v54+64+;; @004b v56 = load.i64 notrap aligned v54+72+;; @004b v57 = load.i64 notrap aligned v0+8+;; @004b store notrap aligned v23, v57+64+;; @004b store notrap aligned v24, v57+72+;; @004b v58 = iconst.i32 1+;; v97 = iconst.i64 16+;; @004b v59 = iadd v24, v97 ; v97 = 16+;; @004b store notrap aligned v58, v59 ; v58 = 1+;; @004b v60 = iconst.i32 0+;; @004b store notrap aligned v60, v40 ; v60 = 0+;; @004b v61 = iconst.i32 0+;; @004b store notrap aligned v61, v40+4 ; v61 = 0+;; @004b v62 = iconst.i64 0+;; @004b store notrap aligned v62, v40+8 ; v62 = 0+;; @004b store notrap aligned v25, v24+40 ; v25 = 0+;; v96 = iconst.i64 32+;; @004b v63 = ushr v53, v96 ; v96 = 32+;; @004b brif v63, block4, block3+;;+;; block4:+;; v95 = iconst.i64 0+;; @004b v64 = iadd.i64 v56, v95 ; v95 = 0+;; v94 = iconst.i64 0+;; @004b v65 = iadd v64, v94 ; v94 = 0+;; @004b v66 = load.i64 notrap aligned v33+56+;; @004b store notrap aligned v66, v65+8+;; v93 = iconst.i64 0+;; @004b v67 = iadd.i64 v24, v93 ; v93 = 0+;; @004b v68 = load.i64 notrap aligned v67+;; @004b store notrap aligned v68, v33+16+;; @004b v69 = load.i64 notrap aligned v67+8+;; @004b store notrap aligned v69, v33+56+;; @004b v70 = ireduce.i32 v53+;; @004b v71 = load.i64 notrap aligned v56+72+;; @004b v72 = uextend.i128 v56+;; @004b v73 = uextend.i128 v71+;; v91 = iconst.i64 64+;; v92 = uextend.i128 v91 ; v91 = 64+;; @004b v74 = ishl v73, v92+;; @004b v75 = bor v74, v72+;; @004b jump block5+;;+;; block6 cold:+;; @004b trap user11+;;+;; block5:+;; @004b br_table v70, block6, []+;;+;; block3:+;; v90 = iconst.i64 0+;; @004b v76 = iadd.i64 v24, v90 ; v90 = 0+;; @004b v77 = load.i64 notrap aligned v76+;; @004b store notrap aligned v77, v33+16+;; @004b v78 = load.i64 notrap aligned v76+8+;; @004b store notrap aligned v78, v33+56+;; v89 = iconst.i64 0+;; @004b v79 = iadd.i64 v56, v89 ; v89 = 0+;; @004b v80 = iconst.i32 4+;; v88 = iconst.i64 16+;; @004b v81 = iadd v79, v88 ; v88 = 16+;; @004b store notrap aligned v80, v81 ; v80 = 4+;; v87 = iconst.i64 104+;; @004b v82 = iadd.i64 v56, v87 ; v87 = 104+;; @004b v83 = load.i64 notrap aligned v82+8+;; @004b v84 = iconst.i32 0+;; @004b store notrap aligned v84, v82 ; v84 = 0+;; @004b v85 = iconst.i32 0+;; @004b store notrap aligned v85, v82+4 ; v85 = 0+;; @004b v86 = iconst.i64 0+;; @004b store notrap aligned v86, v82+8 ; v86 = 0+;; @0050 jump block1+;;+;; block1:+;; @0050 return+;; }