A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
[stack-switching] GC interaction (#14140)
Daniel Hillerströmb68f140620f7
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rsindex eb7244e00b..fb8cba43e0 100644--- a/cranelift/codegen/src/isa/x64/inst/emit.rs+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs@@ -550,6 +550,8 @@ pub(crate) fn emit(in_payload0,out_payload0,} => {+ let stack_map = state.take_stack_map();+// Note that we do not emit anything for preserving and restoring// ordinary registers here: That's taken care of by regalloc for us,// since we marked this instruction as clobbering all registers.@@ -643,6 +645,9 @@ pub(crate) fn emit(asm::inst::jmpq_m::new(tmp1.to_reg()).emit(sink, info, state);sink.bind_label(resume, state.ctrl_plane_mut());+ if let Some(s) = stack_map {+ sink.push_user_stack_map(state, sink.cur_offset(), s);+ }}Inst::JmpKnown { dst } => uncond_jmp(sink, *dst),diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rsindex 1ee94966b3..e758ee087c 100644--- a/cranelift/codegen/src/isa/x64/inst/mod.rs+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs@@ -1467,7 +1467,9 @@ impl MachInst for Inst {fn is_safepoint(&self) -> bool {match self {- Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true,+ Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::StackSwitchBasic { .. } => {+ true+ }_ => false,}}diff --git a/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif b/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clifnew file mode 100644index 0000000000..dff66b8137--- /dev/null+++ b/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif@@ -0,0 +1,14 @@+test compile+set opt_level=speed+set stack_switch_model=basic+target x86_64++;; A stack switch is a GC safepoint. Its stack map is associated with the+;; resume PC saved in the outgoing control context.+function %switch_with_stack_map(i64, i64, i64) -> i64 {+ ss0 = explicit_slot 4++block0(v0: i64, v1: i64, v2: i64):+ v3 = stack_switch v0, v1, v2, stack_map=[i32 @ ss0+0]+ return v3+}diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex 3282f36ae2..bcab270214 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -80,6 +80,8 @@ enum VmType {BuiltinFunctionsArray,ComponentBuiltinFunctionsArray,HostValRaw,+ VMPayloads,+ VMRawContObj,}/// A key that uniquely identifies an alias region across an entire compilation.diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 52c86cdcea..6da9845a9e 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -46,6 +46,20 @@ use wasmtime_environ::{};use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};+/// Function-local stack slots backing a continuation's+/// `VMPayloads::values`.+///+/// The values and their GC-reference markers are logically one+/// payload descriptor but use distinct stack slots so that Cranelift+/// can assign them distinct alias regions. The marker slot is only+/// allocated when this function contains a stack switching site whose+/// payloads may contain GC references.+#[derive(Clone, Copy)]+pub(crate) struct VMPayloadStackSlots {+ pub(crate) values: ir::StackSlot,+ pub(crate) gc_ref_markers: Option<ir::StackSlot>,+}+#[derive(Copy, Clone, Debug)]pub(crate) enum Extension {Sign,@@ -216,10 +230,12 @@ pub struct FuncEnvironment<'module_environment> {/// current stack's `handler_list` field.stack_switching_handler_list_buffer: Option<ir::StackSlot>,- /// Used by the stack switching feature. If set, we have a allocated a- /// slot on this function's stack to be used for the- /// current continuation's `values` field.- stack_switching_values_buffer: Option<ir::StackSlot>,+ /// Used by the stack switching feature. If set, these are the stack slots+ /// backing the current continuation's `values` field.+ stack_switching_values_storage: Option<VMPayloadStackSlots>,++ /// Reusable storage for `get_interned_contref` builtin.+ stack_switching_cont_ref_result_storage: Option<ir::StackSlot>,/// The stack-slot used for exposing Wasm state via debug/// instrumentation, if any, and the builder containing its metadata.@@ -300,7 +316,8 @@ impl<'module_environment> FuncEnvironment<'module_environment> {stack_limit_at_function_entry: None,stack_switching_handler_list_buffer: None,- stack_switching_values_buffer: None,+ stack_switching_values_storage: None,+ stack_switching_cont_ref_result_storage: None,state_slot: None,next_srcloc: ir::SourceLoc::default(),@@ -313,6 +330,18 @@ impl<'module_environment> FuncEnvironment<'module_environment> {}}+ /// Returns the cached continuation-reference stack slot, creating it with+ /// `data` if necessary.+ pub(crate) fn get_or_create_contref_stack_slot(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ data: ir::StackSlotData,+ ) -> ir::StackSlot {+ *self+ .stack_switching_cont_ref_result_storage+ .get_or_insert_with(|| builder.create_sized_stack_slot(data))+ }+/// Consume the branch hint for the instruction at module-relative `offset`/// (i.e. `builder.srcloc().bits()`), if any. The lazy decoder only moves/// forward, making this O(n) over a function body.@@ -5295,8 +5324,9 @@ impl FuncEnvironment<'_> {builder: &mut FunctionBuilder<'_>,contobj: ir::Value,args: &[ir::Value],+ arg_types: &[WasmValType],) -> ir::Value {- stack_switching::instructions::translate_cont_bind(self, builder, contobj, args)+ stack_switching::instructions::translate_cont_bind(self, builder, contobj, args, arg_types)}pub fn translate_cont_new(@@ -5376,13 +5406,15 @@ impl FuncEnvironment<'_> {builder: &mut FunctionBuilder<'_>,tag_index: u32,suspend_args: &[ir::Value],- tag_return_types: &[ir::Type],+ suspend_arg_types: &[WasmValType],+ tag_return_types: &[WasmValType],) -> WasmResult<Vec<ir::Value>> {stack_switching::instructions::translate_suspend(self,builder,tag_index,suspend_args,+ suspend_arg_types,tag_return_types,)}@@ -5394,7 +5426,8 @@ impl FuncEnvironment<'_> {tag_index: u32,contobj: ir::Value,switch_args: &[ir::Value],- return_types: &[ir::Type],+ switch_arg_types: &[WasmValType],+ return_types: &[WasmValType],) -> WasmResult<Vec<ir::Value>> {stack_switching::instructions::translate_switch(self,@@ -5402,6 +5435,7 @@ impl FuncEnvironment<'_> {tag_index,contobj,switch_args,+ switch_arg_types,return_types,)}diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rsindex ca88d1bc7d..8603eca2a8 100644--- a/crates/cranelift/src/func_environ/gc.rs+++ b/crates/cranelift/src/func_environ/gc.rs@@ -2,7 +2,7 @@use crate::TRAP_ARRAY_OUT_OF_BOUNDS;use crate::bounds_checks::BoundsCheck;-use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment};+use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment, stack_switching::fatpointer};use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment, VmctxLoadChain,};@@ -18,7 +18,7 @@ use cranelift_frontend::FunctionBuilder;use smallvec::{SmallVec, smallvec};use wasmtime_environ::{Collector, GcArrayLayout, GcLayout, GcStructLayout, GcTypeLayouts, I31_DISCRIMINANT,- ModuleInternedTypeIndex, TagIndex, TypeIndex, VMGcKind, WasmCompositeInnerType,+ ModuleInternedTypeIndex, PtrSize, TagIndex, TypeIndex, VMGcKind, WasmCompositeInnerType,WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,wasm_unsupported,};@@ -377,10 +377,7 @@ pub fn read_field_at_addr(.call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]);builder.func.dfg.first_result(call_inst)}- WasmHeapTopType::Cont => {- // TODO(#10248) GC integration for stack switching- return stack_switching_unsupported();- }+ WasmHeapTopType::Cont => read_cont_ref_at_addr(func_env, builder, addr, flags)?,},},};@@ -431,6 +428,92 @@ pub fn intern_func_ref(Ok(builder.ins().ireduce(ir::types::I32, func_ref_id))}+fn intern_cont_ref(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ ref_type: WasmRefType,+ contobj: ir::Value,+) -> ir::Value {+ assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Cont);++ if ref_type.heap_type == WasmHeapType::NoCont {+ let zero = builder.ins().iconst(ir::types::I32, 0);++ if !ref_type.nullable {+ builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT);+ }++ return zero;+ }++ let (revision, contref) = fatpointer::deconstruct(func_env, &mut builder.cursor(), contobj);+ let vmctx = func_env.vmctx_val(&mut builder.cursor());+ let intern = func_env+ .builtin_functions+ .intern_contref_for_gc_heap(builder.func);+ let call = builder.ins().call(intern, &[vmctx, contref, revision]);+ let id = builder.func.dfg.first_result(call);+ builder.ins().ireduce(ir::types::I32, id)+}++fn read_cont_ref_at_addr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ addr: ir::Value,+ flags: ir::MemFlagsData,+) -> WasmResult<ir::Value> {+ let id = builder.ins().load(ir::types::I32, flags, addr, 0);++ // We either create or fetch an existing stack slot to hold the raw+ // continuation object, and pass its address as the out parameter to the+ // builtin.+ let pointer_type = func_env.pointer_type();+ let layout = func_env.offsets.ptr.vm_raw_cont_obj();+ let slot_size = u32::from(layout.size());+ let slot_align = u8::try_from(layout.align().trailing_zeros()).unwrap();+ let contref_offset = layout.contref();+ let revision_offset = layout.revision();++ let slot = func_env.get_or_create_contref_stack_slot(+ builder,+ ir::StackSlotData::new(ir::StackSlotKind::ExplicitSlot, slot_size, slot_align),+ );+ let out_result = builder.ins().stack_addr(pointer_type, slot, 0);++ let vmctx = func_env.vmctx_val(&mut builder.cursor());+ let get = func_env+ .builtin_functions+ .get_interned_contref(builder.func);+ builder.ins().call(get, &[vmctx, id, out_result]);++ let region = func_env.alias_regions.stack_slot_region(builder.func, slot);+ let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region));+ let contref = builder+ .ins()+ .load(pointer_type, flags, out_result, contref_offset);+ let revision = builder+ .ins()+ .load(pointer_type, flags, out_result, revision_offset);+ Ok(fatpointer::construct(+ func_env,+ &mut builder.cursor(),+ revision,+ contref,+ ))+}++fn write_cont_ref_at_addr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ ref_type: WasmRefType,+ flags: ir::MemFlagsData,+ field_addr: ir::Value,+ contobj: ir::Value,+) {+ let id = intern_cont_ref(func_env, builder, ref_type, contobj);+ builder.ins().store(flags, id, field_addr, 0);+}+fn write_func_ref_at_addr(func_env: &mut FuncEnvironment<'_>,builder: &mut FunctionBuilder<'_>,@@ -487,7 +570,9 @@ pub fn write_field_at_addr(func_env, builder, r, field_addr, new_val, flags,)?;}- WasmHeapTopType::Cont => return stack_switching_unsupported(),+ WasmHeapTopType::Cont => {+ write_cont_ref_at_addr(func_env, builder, r, flags, field_addr, new_val)+ }},WasmStorageType::Val(_) => {assert_eq!(@@ -527,6 +612,10 @@ fn default_value(}WasmValType::Ref(r) => {assert!(r.nullable);+ if r.heap_type.top() == WasmHeapTopType::Cont {+ let zero = cursor.ins().iconst(func_env.pointer_type(), 0);+ return fatpointer::construct(func_env, cursor, zero, zero);+ }let (ty, needs_stack_map) = func_env.reference_type(r.heap_type);// NB: The collector doesn't need to know about null references.diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rsindex 7cf5628d0b..ec9b47a9e8 100644--- a/crates/cranelift/src/func_environ/gc/copying.rs+++ b/crates/cranelift/src/func_environ/gc/copying.rs@@ -444,7 +444,9 @@ impl GcCompiler for CopyingCompiler {// store the reference directly.unbarriered_store_gc_ref(builder, r.heap_type, field_addr, val, flags)?;}- WasmHeapTopType::Cont => return super::stack_switching_unsupported(),+ WasmHeapTopType::Cont => {+ write_field_at_addr(func_env, builder, ty, field_addr, val)?+ }},WasmStorageType::I8 => {assert_eq!(builder.func.dfg.value_type(val), ir::types::I32);diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rsindex 05b92e7779..89cc0083b8 100644--- a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs+++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs@@ -34,7 +34,7 @@ pub(crate) fn deconstruct<'a>(/// Constructs a continuation object from a given contref and revision pointer./// The contref_addr may be 0, to indicate that we want to build a wasm null reference.pub(crate) fn construct<'a>(- env: &mut crate::func_environ::FuncEnvironment<'a>,+ env: &crate::func_environ::FuncEnvironment<'a>,pos: &mut cranelift_codegen::cursor::FuncCursor,revision_counter: ir::Value,contref_addr: ir::Value,diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rsindex 0310f62bd1..5a6dac28c5 100644--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs@@ -8,7 +8,10 @@ use cranelift_codegen::ir::{self, MemFlagsData};use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData};use cranelift_frontend::FunctionBuilder;use itertools::{Either, Itertools};-use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wasm_unsupported};+use wasmtime_environ::{+ PtrSize, TagIndex, TypeIndex, WasmHeapType, WasmRefType, WasmResult, WasmValType,+ wasm_unsupported,+};fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult<u8> {match (triple.architecture, triple.operating_system) {@@ -31,7 +34,7 @@ pub(crate) mod stack_switching_helpers {use cranelift_codegen::ir::types::*;use cranelift_codegen::ir::{StackSlot, StackSlotKind::*};use cranelift_frontend::FunctionBuilder;- use wasmtime_environ::PtrSize;+ use wasmtime_environ::{PtrSize, WasmValType};/// Provides information about the layout of a type when it is used as an/// element in a host array. This is used for `VMHostArrayRef`.@@ -69,7 +72,17 @@ pub(crate) mod stack_switching_helpers {phantom: PhantomData<T>,}- pub type VMPayloads = VMHostArrayRef<u128>;+ /// Compile-time reference to a runtime `VMPayloads` descriptor.+ ///+ /// This wrapper keeps the payload buffer and its GC-reference markers+ /// paired. Its core invariant is that `gc_ref_data`, when present, points+ /// to one marker byte for every slot in `buffer`; callers must update a+ /// slot's value and marker together.+ #[derive(Copy, Clone)]+ pub struct VMPayloads {+ address: ir::Value,+ buffer: VMHostArrayRef<u128>,+ }// Actually a vector of *mut VMTagDefinitionpub type VMHandlerList = VMHostArrayRef<*mut u8>;@@ -105,7 +118,7 @@ pub(crate) mod stack_switching_helpers {) -> VMPayloads {let offset: i64 = env.offsets.ptr.vm_cont_ref().args().into();let address = builder.ins().iadd_imm_s(self.address, offset);- VMPayloads::new(address)+ VMPayloads::new(env, address)}pub fn values<'a>(@@ -115,7 +128,7 @@ pub(crate) mod stack_switching_helpers {) -> VMPayloads {let offset: i64 = env.offsets.ptr.vm_cont_ref().values().into();let address = builder.ins().iadd_imm_s(self.address, offset);- VMPayloads::new(address)+ VMPayloads::new(env, address)}pub fn common_stack_information<'a>(@@ -238,6 +251,147 @@ pub(crate) mod stack_switching_helpers {}}+ impl VMPayloads {+ pub fn new(env: &mut crate::func_environ::FuncEnvironment<'_>, address: ir::Value) -> Self {+ debug_assert_eq!(env.offsets.ptr.vm_payloads().buffer(), 0);+ Self {+ address,+ buffer: VMHostArrayRef::new(address),+ }+ }++ pub fn get_gc_ref_data(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ env.alias_regions+ .vm_payloads()+ .gc_ref_data()+ .load(&mut builder.cursor(), self.address)+ }++ pub fn set_gc_ref_data(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ data: ir::Value,+ ) {+ env.alias_regions.vm_payloads().gc_ref_data().store(+ &mut builder.cursor(),+ self.address,+ data,+ );+ }++ /// Prepares stack storage for this payload buffer.+ ///+ /// Payload values and their GC-reference markers use distinct stack+ /// slots so that they have distinct alias regions. The descriptor's+ /// capacity and pointers always describe the storage selected at this+ /// particular instruction site. Both slots retain their identities and+ /// grow in place as later sites require more capacity.+ pub fn prepare_stack_storage(+ &self,+ env: &mut crate::func_environ::FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ capacity: u32,+ initial_types: &[WasmValType],+ needs_gc_ref_markers: bool,+ existing_slots: Option<crate::func_environ::VMPayloadStackSlots>,+ ) -> crate::func_environ::VMPayloadStackSlots {+ debug_assert!(capacity > 0);+ debug_assert!(initial_types.len() <= usize::try_from(capacity).unwrap());++ let (align, entry_size) =+ <u128 as VMHostArrayEntry>::vmhostarray_entry_layout(&env.offsets.ptr);+ let values_size = capacity+ .checked_mul(entry_size)+ .expect("stack switching values buffer size should fit in u32");++ let values_slot = match existing_slots.map(|slots| slots.values) {+ Some(slot) => {+ // Keep the slot identity stable: `stack_addr` instructions+ // emitted at earlier sites continue to refer to this slot+ // when a later site requires more storage.+ let slot_data = &mut builder.func.sized_stack_slots[slot];+ debug_assert!(align <= slot_data.align_shift);+ debug_assert_eq!(slot_data.kind, ExplicitSlot);+ slot_data.size = slot_data.size.max(values_size);+ slot+ }+ None => builder.create_sized_stack_slot(ir::StackSlotData::new(+ ir::StackSlotKind::ExplicitSlot,+ values_size,+ align,+ )),+ };++ let capacity_value = builder.ins().iconst(I32, i64::from(capacity));+ let values_data = builder.ins().stack_addr(env.pointer_type(), values_slot, 0);+ self.set_capacity(env, builder, capacity_value);+ self.set_data(env, builder, values_data);++ let mut gc_ref_markers = existing_slots.and_then(|slots| slots.gc_ref_markers);+ if needs_gc_ref_markers {+ let marker_slot = match gc_ref_markers {+ Some(slot) => {+ let slot_data = &mut builder.func.sized_stack_slots[slot];+ debug_assert_eq!(slot_data.kind, ExplicitSlot);+ slot_data.size = slot_data.size.max(capacity);+ slot+ }+ None => builder.create_sized_stack_slot(ir::StackSlotData::new(+ ir::StackSlotKind::ExplicitSlot,+ capacity,+ 0,+ )),+ };+ gc_ref_markers = Some(marker_slot);++ let marker_data = builder.ins().stack_addr(env.pointer_type(), marker_slot, 0);+ self.set_gc_ref_data(env, builder, marker_data);++ let region = env+ .alias_regions+ .stack_slot_region(builder.func, marker_slot);+ let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region));+ let zero = builder.ins().iconst(I8, 0);+ for offset in 0..capacity {+ builder+ .ins()+ .store(flags, zero, marker_data, i32::try_from(offset).unwrap());+ }+ for (offset, ty) in initial_types.iter().enumerate() {+ if ty.is_vmgcref_type_and_not_i31() {+ let marker = builder+ .ins()+ .iconst(I8, i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF));+ builder.ins().store(+ flags,+ marker,+ marker_data,+ i32::try_from(offset).unwrap(),+ );+ }+ }+ }++ crate::func_environ::VMPayloadStackSlots {+ values: values_slot,+ gc_ref_markers,+ }+ }+ }++ impl core::ops::Deref for VMPayloads {+ type Target = VMHostArrayRef<u128>;++ fn deref(&self) -> &Self::Target {+ &self.buffer+ }+ }+impl<T: VMHostArrayEntry> VMHostArrayRef<T> {pub(crate) fn new(address: ir::Value) -> Self {Self {@@ -315,7 +469,7 @@ pub(crate) mod stack_switching_helpers {env: &mut crate::func_environ::FuncEnvironment<'a>,builder: &mut FunctionBuilder,arg_count: i32,- ) -> ir::Value {+ ) -> (ir::Value, ir::Value) {let data = self.get_data(env, builder);let original_length = self.get_length(env, builder);let new_length = builder@@ -328,7 +482,7 @@ pub(crate) mod stack_switching_helpers {let byte_offset = builder.ins().imul_imm_s(original_length, i64::from(entry_size));- builder.ins().iadd(data, byte_offset)+ (builder.ins().iadd(data, byte_offset), original_length)}pub fn allocate_or_reuse_stack_slot<'a>(@@ -375,13 +529,13 @@ pub(crate) mod stack_switching_helpers {}/// Loads n entries from this Vector object, where n is the length of- /// `load_types`, which also gives the types of the values to load.+ /// `load_types`, which also gives the Wasm types of the values to load./// Loading starts at index 0 of the Vector object.pub fn load_data_entries<'a>(&self,env: &mut crate::func_environ::FuncEnvironment<'a>,builder: &mut FunctionBuilder,- load_types: &[ir::Type],+ load_types: &[WasmValType],) -> Vec<ir::Value> {let region = env.alias_regions@@ -392,10 +546,14 @@ pub(crate) mod stack_switching_helpers {let mut values = vec![];let mut offset = 0;let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);- for valtype in load_types {+ for wasm_ty in load_types {+ let valtype = crate::value_type(env.isa(), *wasm_ty);let val = builder.ins()- .load(*valtype, memflags, data_start_pointer, offset);+ .load(valtype, memflags, data_start_pointer, offset);+ if env.val_ty_needs_stack_map(*wasm_ty) {+ builder.declare_value_needs_stack_map(val);+ }values.push(val);offset += i32::try_from(entry_size).unwrap();}@@ -915,6 +1073,10 @@ pub(crate) mod stack_switching_helpers {use helpers::VMStackChain;use stack_switching_helpers as helpers;+fn types_need_gc_ref_markers(types: &[WasmValType]) -> bool {+ types.iter().any(|ty| ty.is_vmgcref_type_and_not_i31())+}+/// Stores the given arguments in the appropriate `VMPayloads` object in the/// continuation. If the continuation was never invoked, use the `args` object./// Otherwise, use the `values` object.@@ -922,8 +1084,11 @@ pub(crate) fn vmcontref_store_payloads<'a>(env: &mut crate::func_environ::FuncEnvironment<'a>,builder: &mut FunctionBuilder,values: &[ir::Value],+ types: &[WasmValType],contref: ir::Value,) {+ debug_assert_eq!(values.len(), types.len());+ let needs_gc_ref_markers = types_need_gc_ref_markers(types);let count =i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32");if values.len() > 0 {@@ -931,6 +1096,9 @@ pub(crate) fn vmcontref_store_payloads<'a>(let use_payloads_block = builder.create_block();let store_data_block = builder.create_block();builder.append_block_param(store_data_block, env.pointer_type());+ if needs_gc_ref_markers {+ builder.append_block_param(store_data_block, env.pointer_type());+ }let co = helpers::VMContRef::new(contref);let csi = co.common_stack_information(env, builder);@@ -944,11 +1112,15 @@ pub(crate) fn vmcontref_store_payloads<'a>(builder.seal_block(use_args_block);let args = co.args(env, builder);- let ptr = args.occupy_next_slots(env, builder, count);+ let (ptr, original_length) = args.occupy_next_slots(env, builder, count);+ let mut block_args = vec![BlockArg::Value(ptr)];+ if needs_gc_ref_markers {+ let gc_refs_ptr = args.get_gc_ref_data(env, builder);+ let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length);+ block_args.push(BlockArg::Value(gc_refs_ptr));+ }- builder- .ins()- .jump(store_data_block, &[BlockArg::Value(ptr)]);+ builder.ins().jump(store_data_block, &block_args);}{@@ -959,10 +1131,14 @@ pub(crate) fn vmcontref_store_payloads<'a>(// This also checks that the buffer is large enough to hold// `values.len()` more elements.- let ptr = payloads.occupy_next_slots(env, builder, count);- builder- .ins()- .jump(store_data_block, &[BlockArg::Value(ptr)]);+ let (ptr, original_length) = payloads.occupy_next_slots(env, builder, count);+ let mut block_args = vec![BlockArg::Value(ptr)];+ if needs_gc_ref_markers {+ let gc_refs_ptr = payloads.get_gc_ref_data(env, builder);+ let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length);+ block_args.push(BlockArg::Value(gc_refs_ptr));+ }+ builder.ins().jump(store_data_block, &block_args);}{@@ -970,6 +1146,8 @@ pub(crate) fn vmcontref_store_payloads<'a>(builder.seal_block(store_data_block);let ptr = builder.block_params(store_data_block)[0];+ let gc_refs_ptr =+ needs_gc_ref_markers.then(|| builder.block_params(store_data_block)[1]);// Store the values.let region = env@@ -977,8 +1155,22 @@ pub(crate) fn vmcontref_store_payloads<'a>(.continuation_stack_memory_region(builder.func);let memflags = ir::MemFlagsData::trusted().with_alias_region(Some(region));let mut offset = 0;- for value in values {+ for (index, (value, ty)) in values.iter().zip(types).enumerate() {builder.ins().store(memflags, *value, ptr, offset);+ if let Some(gc_refs_ptr) = gc_refs_ptr {+ let marker = if ty.is_vmgcref_type_and_not_i31() {+ i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF)+ } else {+ 0+ };+ let marker = builder.ins().iconst(I8, marker);+ builder.ins().store(+ memflags,+ marker,+ gc_refs_ptr,+ i32::try_from(index).unwrap(),+ );+ }offset += i32::from(env.offsets.ptr.maximum_value_size());}}@@ -1257,6 +1449,7 @@ pub(crate) fn translate_cont_bind<'a>(builder: &mut FunctionBuilder,contobj: ir::Value,args: &[ir::Value],+ arg_types: &[WasmValType],) -> ir::Value {let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj);@@ -1268,7 +1461,7 @@ pub(crate) fn translate_cont_bind<'a>(let evidence = builder.ins().icmp(IntCC::Equal, witness, revision);env.trapz(builder, evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);- vmcontref_store_payloads(env, builder, args, contref);+ vmcontref_store_payloads(env, builder, args, arg_types, contref);let revision = vmcontref.incr_revision(env, builder, revision);let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref);@@ -1291,12 +1484,16 @@ pub(crate) fn translate_cont_new<'a>(let nreturns = builder.ins().iconst(I32, i64::try_from(return_types.len()).unwrap());+ let gc_refs = builder.ins().iconst(+ I32,+ i64::from(u8::from(types_need_gc_ref_markers(arg_types))),+ );let cont_new_func = env.builtin_functions.cont_new(&mut builder.func);let vmctx = env.vmctx_val(&mut builder.cursor());let call_inst = builder.ins()- .call(cont_new_func, &[vmctx, func, nargs, nreturns]);+ .call(cont_new_func, &[vmctx, func, nargs, nreturns, gc_refs]);let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap();let tag = helpers::VMContRef::new(contref).get_revision(env, builder);@@ -1380,6 +1577,10 @@ fn translate_resume_impl<'a>(resume_payload: ResumePayload<'_>,resumetable: &[(u32, Option<ir::Block>)],) -> WasmResult<Vec<ir::Value>> {+ let resume_arg_types = env+ .continuation_arguments(TypeIndex::from_u32(type_index))+ .to_vec();+// The resume instruction is the most involved instruction to// compile as it is responsible for both continuation application// and control tag dispatch.@@ -1461,10 +1662,17 @@ fn translate_resume_impl<'a>(match resume_payload {ResumePayload::Values(resume_args) => {+ debug_assert_eq!(resume_args.len(), resume_arg_types.len());let _next_revision = vmcontref.incr_revision(env, builder, revision);if resume_args.len() > 0 {// We store the arguments in the `VMContRef` to be resumed.- vmcontref_store_payloads(env, builder, resume_args, resume_contref);+ vmcontref_store_payloads(+ env,+ builder,+ resume_args,+ &resume_arg_types,+ resume_contref,+ );}}ResumePayload::Throw { .. } | ResumePayload::ThrowRef(_) => {@@ -1752,11 +1960,7 @@ fn translate_resume_impl<'a>(preamble_blocks.push(preamble_block);builder.switch_to_block(preamble_block);- let param_types = env.tag_params(TagIndex::from_u32(handle_tag));- let param_types: Vec<ir::Type> = param_types- .iter()- .map(|wty| crate::value_type(env.isa(), *wty))- .collect();+ let param_types = env.tag_params(TagIndex::from_u32(handle_tag)).to_vec();let values = suspended_contref.values(env, builder);let mut suspend_args: Vec<ir::Value> =@@ -1821,11 +2025,9 @@ fn translate_resume_impl<'a>(returned_csi.set_state_returned(env, builder);// Load the values returned by the continuation.- let return_types: Vec<_> = env+ let return_types = env.continuation_returns(TypeIndex::from_u32(type_index))- .iter()- .map(|ty| crate::value_type(env.isa(), *ty))- .collect();+ .to_vec();let payloads = returned_contref.args(env, builder);let return_values = payloads.load_data_entries(env, builder, &return_types);payloads.clear(env, builder, true);@@ -1839,7 +2041,7 @@ fn load_resume_values_or_throw<'a>(builder: &mut FunctionBuilder,result: ControlEffect,continuation: helpers::VMContRef,- return_types: &[ir::Type],+ return_types: &[WasmValType],) -> WasmResult<Vec<ir::Value>> {let throw_block = builder.create_block();let values_block = builder.create_block();@@ -1854,7 +2056,11 @@ fn load_resume_values_or_throw<'a>(let values = continuation.values(env, builder);// Exception references use Wasmtime's compressed, 32-bit GC-reference// representation.- let exnref = values.load_data_entries(env, builder, &[I32])[0];+ let exnref_ty = WasmValType::Ref(WasmRefType {+ nullable: false,+ heap_type: WasmHeapType::Exn,+ });+ let exnref = values.load_data_entries(env, builder, &[exnref_ty])[0];values.clear(env, builder, true);gc::translate_exn_throw_ref(env, builder, exnref)?;@@ -1871,8 +2077,10 @@ pub(crate) fn translate_suspend<'a>(builder: &mut FunctionBuilder,tag_index: u32,suspend_args: &[ir::Value],- tag_return_types: &[ir::Type],+ suspend_arg_types: &[WasmValType],+ tag_return_types: &[WasmValType],) -> WasmResult<Vec<ir::Value>> {+ debug_assert_eq!(suspend_args.len(), suspend_arg_types.len());let tag_addr = tag_address(env, builder, tag_index);let vmctx = env.vmctx_val(&mut builder.cursor());@@ -1894,16 +2102,21 @@ pub(crate) fn translate_suspend<'a>(// return values including the possible exception reference.let values = active_contref.values(env, builder);let required_capacity = u32::try_from(std::cmp::max(- 1, // This account for the possible exception reference.+ 1, // This accounts for the possible exception reference.std::cmp::max(suspend_args.len(), tag_return_types.len()),)).expect("Number of stack switching payloads should fit in u32");- env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot(+ let needs_gc_ref_markers =+ types_need_gc_ref_markers(suspend_arg_types) || types_need_gc_ref_markers(tag_return_types);+ let existing_storage = env.stack_switching_values_storage;+ env.stack_switching_values_storage = Some(values.prepare_stack_storage(env,builder,required_capacity,- env.stack_switching_values_buffer,+ suspend_arg_types,+ needs_gc_ref_markers,+ existing_storage,));if suspend_args.len() > 0 {@@ -1933,13 +2146,20 @@ pub(crate) fn translate_suspend<'a>(// A normal resume supplies the tag's return values. `resume_throw_ref`// supplies an exception reference and throws it at this suspension point.- load_resume_values_or_throw(+ let return_values = load_resume_values_or_throw(env,builder,ControlEffect::from_u64(result),active_contref,tag_return_types,- )+ );++ if needs_gc_ref_markers {+ let zero = builder.ins().iconst(env.pointer_type(), 0);+ values.set_gc_ref_data(env, builder, zero);+ }++ return_values}pub(crate) fn translate_switch<'a>(@@ -1948,7 +2168,8 @@ pub(crate) fn translate_switch<'a>(tag_index: u32,switchee_contobj: ir::Value,switch_args: &[ir::Value],- return_types: &[ir::Type],+ switch_arg_types: &[WasmValType],+ return_types: &[WasmValType],) -> WasmResult<Vec<ir::Value>> {let vmctx = env.vmctx_val(&mut builder.cursor());@@ -1978,6 +2199,7 @@ pub(crate) fn translate_switch<'a>(// `switcher_contref`) to the immediate child (called// `switcher_contref_last_ancestor`) of the stack with the corresponding// handler (saved in `handler_stack_chain`).+ let return_values_need_gc_ref_markers = types_need_gc_ref_markers(return_types);let (switcher_contref,switcher_contobj,@@ -2004,11 +2226,17 @@ pub(crate) fn translate_switch<'a>(// reference.let values = switcher_contref.values(env, builder);let required_capacity = u32::try_from(std::cmp::max(1, return_types.len())).unwrap();- env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot(+ let existing_storage = env.stack_switching_values_storage;+ env.stack_switching_values_storage = Some(values.prepare_stack_storage(env,builder,required_capacity,- env.stack_switching_values_buffer,+ // No GC ref marking is required at this point. Upon+ // return to this site `vmcontref_store_payloads` will+ // store the markers, if needed.+ &[],+ return_values_need_gc_ref_markers,+ existing_storage,));let switcher_contref_csi = switcher_contref.common_stack_information(env, builder);@@ -2054,7 +2282,13 @@ pub(crate) fn translate_switch<'a>(let (switchee_contref_csi, switchee_contref_last_ancestor) = {let mut combined_payloads = switch_args.to_vec();combined_payloads.push(switcher_contobj);- vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address);+ vmcontref_store_payloads(+ env,+ builder,+ &combined_payloads,+ switch_arg_types,+ switchee_contref.address,+ );let switchee_contref_csi = switchee_contref.common_stack_information(env, builder);switchee_contref_csi.set_state_running(env, builder);@@ -2183,11 +2417,20 @@ pub(crate) fn translate_switch<'a>(// After switching back to the original stack, either load the values// supplied by an ordinary resume or throw the injected exception.- load_resume_values_or_throw(+ let return_values = load_resume_values_or_throw(env,builder,ControlEffect::from_u64(result),switcher_contref,return_types,- )+ );++ if return_values_need_gc_ref_markers {+ let zero = builder.ins().iconst(env.pointer_type(), 0);+ switcher_contref+ .values(env, builder)+ .set_gc_ref_data(env, builder, zero);+ }++ return_values}diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rsindex c0c37e96cf..4ed39e1cab 100644--- a/crates/cranelift/src/translate/code_translator.rs+++ b/crates/cranelift/src/translate/code_translator.rs@@ -3168,25 +3168,14 @@ pub fn translate_operator(let arg_count = src_types.len() - dst_arity;let arg_types = &src_types[0..arg_count];- for arg_type in arg_types {- // We can't bind GC objects using cont.bind at the moment: We- // don't have the necessary infrastructure to traverse the- // buffers used by cont.bind when looking for GC roots. Thus,- // this crude check ensures that these buffers can never contain- // GC roots to begin with.- if arg_type.is_vmgcref_type_and_not_i31() {- return Err(wasmtime_environ::WasmError::Unsupported(- "cont.bind does not support GC types at the moment".into(),- ));- }- }-let (original_contobj, args) =environ.stacks.peekn(arg_count + 1).split_last().unwrap();let original_contobj = *original_contobj;let args = args.to_vec();- let new_contobj = environ.translate_cont_bind(builder, original_contobj, &args);+ let arg_types = arg_types.to_vec();+ let new_contobj =+ environ.translate_cont_bind(builder, original_contobj, &args, &arg_types);environ.stacks.popn(arg_count + 1);environ.stacks.push1(new_contobj);@@ -3194,17 +3183,18 @@ pub fn translate_operator(Operator::Suspend { tag_index } => {let tag_index = TagIndex::from_u32(*tag_index);let param_types = environ.tag_params(tag_index).to_vec();- let return_types: SmallVec<[_; 8]> = environ- .tag_returns(tag_index)- .iter()- .map(|ty| crate::value_type(environ.isa(), *ty))- .collect();+ let return_types = environ.tag_returns(tag_index).to_vec();let params = environ.stacks.peekn(param_types.len()).to_vec();let param_count = params.len();- let return_values =- environ.translate_suspend(builder, tag_index.as_u32(), ¶ms, &return_types)?;+ let return_values = environ.translate_suspend(+ builder,+ tag_index.as_u32(),+ ¶ms,+ ¶m_types,+ &return_types,+ )?;environ.stacks.popn(param_count);environ.stacks.pushn(&return_values);@@ -3326,45 +3316,43 @@ pub fn translate_operator(tag_index,} => {// Arguments of the continuation we are going to switch to- let continuation_argument_types: SmallVec<[_; 8]> = environ+ let switch_arg_types: SmallVec<[_; 8]> = environ.continuation_arguments(TypeIndex::from_u32(*cont_type_index)).to_smallvec();// Arity includes the continuation argument- let arity = continuation_argument_types.len();+ let arity = switch_arg_types.len();let (contobj, switch_args) = environ.stacks.peekn(arity).split_last().unwrap();let contobj = *contobj;let switch_args = switch_args.to_vec();// Type of the continuation we are going to create by suspending the// currently running stack- let current_continuation_type = continuation_argument_types.last().unwrap();+ let current_continuation_type = switch_arg_types.last().unwrap();let current_continuation_type = current_continuation_type.unwrap_ref_type();// Argument types of current_continuation_type. These will in turn// be the types of the arguments we receive when someone switches// back to this switch instruction- let current_continuation_arg_types: SmallVec<[_; 8]> =- match current_continuation_type.heap_type {- WasmHeapType::ConcreteCont(index) => {- let mti = index- .as_module_type_index()- .expect("expected module-local type index");-- environ- .continuation_arguments_from_interned(mti)- .iter()- .map(|ty| crate::value_type(environ.isa(), *ty))- .collect()- }- _ => panic!("Invalid type on switch"),- };+ let return_types: SmallVec<[_; 8]> = match current_continuation_type.heap_type {+ WasmHeapType::ConcreteCont(index) => {+ let mti = index+ .as_module_type_index()+ .expect("expected module-local type index");++ environ+ .continuation_arguments_from_interned(mti)+ .to_smallvec()+ }+ _ => panic!("Invalid type on switch"),+ };let switch_return_values = environ.translate_switch(builder,*tag_index,contobj,&switch_args,- ¤t_continuation_arg_types,+ &switch_arg_types,+ &return_types,)?;environ.stacks.popn(arity);diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rsindex d59f3c16af..fc390ef509 100644--- a/crates/environ/src/builtin.rs+++ b/crates/environ/src/builtin.rs@@ -154,7 +154,7 @@ macro_rules! foreach_builtin_function {// Creates a new continuation from a funcref.#[cfg(feature = "stack-switching")]- cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer;+ cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32, gc_refs: u32) -> pointer;// Return the instance ID for a given vmctx.#[cfg(feature = "gc")]@@ -170,6 +170,23 @@ macro_rules! foreach_builtin_function {// Process a debug breakpoint.breakpoint(vmctx: vmctx) -> bool;++ // Intern a continuation reference into the GC heap's side table.+ #[cfg(all(feature = "gc", feature = "stack-switching"))]+ intern_contref_for_gc_heap(+ vmctx: vmctx,+ contref: pointer,+ revision: pointer+ ) -> u64;++ // Resolve an interned continuation-reference ID and write its+ // pointer and revision witness to `result`.+ #[cfg(all(feature = "gc", feature = "stack-switching"))]+ get_interned_contref(+ vmctx: vmctx,+ contref_id: u32,+ result: pointer+ ) -> bool;}};}@@ -387,6 +404,7 @@ impl BuiltinFunctionIndex {(@get ref_func pointer) => (return None);(@get table_get_lazy_init_func_ref pointer) => (return None);(@get intern_func_ref_for_gc_heap u64) => (return None);+ (@get intern_contref_for_gc_heap u64) => (return None);(@get is_subtype u32) => (return None);(@get ceil_f32 f32) => (return None);(@get ceil_f64 f64) => (return None);diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rsindex 8f43cbdd84..a0d1c9d7de 100644--- a/crates/environ/src/stack_switching.rs+++ b/crates/environ/src/stack_switching.rs@@ -33,6 +33,10 @@ pub const STACK_STATE_TRAPPED_DISCRIMINANT: u32 = 5;/// Discriminant of variant `Return` in/// `runtime::vm::ControlEffect`.pub const CONTROL_EFFECT_RETURN_DISCRIMINANT: u32 = 0;++/// Marker for a continuation payload slot that contains a GC-managed+/// reference and must be traced while the continuation is suspended.+pub const CONTINUATION_PAYLOAD_GC_REF: u8 = 1;/// Discriminant of variant `Resume` in/// `runtime::vm::ControlEffect`.pub const CONTROL_EFFECT_RESUME_DISCRIMINANT: u32 = 1;diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex 139309f46d..741bf7e23d 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -71,6 +71,7 @@ macro_rules! define_vm_type_offsets {(@size ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().size()) };(@size ($p:expr) VMStackLimits) => { u32::from(($p).vm_stack_limits().size()) };(@size ($p:expr) VMHostArray) => { u32::from(($p).vm_host_array().size()) };+ (@size ($p:expr) VMPayloads) => { u32::from(($p).vm_payloads().size()) };(@size ($p:expr) VMCommonStackInformation) => {u32::from(($p).vm_common_stack_information().size())};@@ -116,6 +117,7 @@ macro_rules! define_vm_type_offsets {(@align ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().align()) };(@align ($p:expr) VMStackLimits) => { u32::from(($p).vm_stack_limits().align()) };(@align ($p:expr) VMHostArray) => { u32::from(($p).vm_host_array().align()) };+ (@align ($p:expr) VMPayloads) => { u32::from(($p).vm_payloads().align()) };(@align ($p:expr) VMCommonStackInformation) => {u32::from(($p).vm_common_stack_information().align())};diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rsindex aca8cdcc37..ab7ac0675b 100644--- a/crates/environ/src/vmtypes.rs+++ b/crates/environ/src/vmtypes.rs@@ -602,6 +602,21 @@ macro_rules! for_each_vm_type {pub data: *mut u8,}+ /// Payload values exchanged with a continuation and the metadata+ /// needed to trace GC references among them.+ #[derive(Debug, Clone)]+ #[repr(C)]+ #[snake_name = vm_payloads]+ pub struct VMPayloads {+ /// The payload values themselves.+ #[aggregate]+ pub buffer: VMHostArray,++ /// One marker byte per buffer slot, indicating whether that+ /// slot contains a GC reference, or `None` when no slots do.+ pub gc_ref_data: Option<VmPtr<u8>>,+ }+/// The information saved for every stack, whether it is a/// continuation's or the initial stack's.#[derive(Debug, Clone)]@@ -651,12 +666,12 @@ macro_rules! for_each_vm_type {/// The arguments to, and return values of, the function passed/// to `cont.new`.#[aggregate]- pub args: VMHostArray,+ pub args: VMPayloads,/// The payloads passed to and from this continuation once it/// has been suspended.#[aggregate]- pub values: VMHostArray,+ pub values: VMPayloads,/// Tells the compiler that this structure has potential/// self-references, through `last_ancestor`.@@ -667,6 +682,19 @@ macro_rules! for_each_vm_type {pub _marker: PhantomPinned,}+ /// A slight variation of `VMContObj` which allows the+ /// `contref` to be instantiated to null,+ /// i.e. `Option::None`. This representation is used by+ /// the GC infrastructure to construct a canonical+ /// null-esque continuation object.+ #[cfg(all(feature = "gc", feature = "stack-switching"))]+ #[repr(C)]+ #[snake_name = vm_raw_cont_obj]+ pub struct VMRawContObj {+ pub contref: Option<VmPtr<u8>>,+ pub revision: usize,+ }+/// The deferred-reference-counting collector's JIT-accessible heap/// data.///diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rsindex 5c84a41807..09a63ba2cf 100644--- a/crates/wasmtime/src/runtime/store/gc.rs+++ b/crates/wasmtime/src/runtime/store/gc.rs@@ -836,22 +836,38 @@ impl StoreOpaque {#[cfg(feature = "stack-switching")]fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) {- use crate::vm::VMStackState;+ use crate::vm::{VMPayloads, VMStackState, ValRaw};++ unsafe fn trace_payload_roots(gc_roots_list: &mut GcRootsList, payloads: &VMPayloads) {+ let gc_ref_data = payloads.gc_ref_data;+ let payloads = &payloads.buffer;+ assert!(payloads.length <= payloads.capacity);+ let Some(gc_ref_data) = gc_ref_data else {+ return;+ };+ let gc_ref_data = gc_ref_data.as_ptr();++ for index in 0..usize::try_from(payloads.length).unwrap() {+ let marker = unsafe { gc_ref_data.add(index).read() };+ if marker == wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF {+ let slot = unsafe { payloads.data.cast::<ValRaw>().add(index).cast::<u32>() };+ unsafe {+ StoreOpaque::trace_wasm_stack_slot(gc_roots_list, slot);+ }+ }+ }+ }log::trace!("Begin trace GC roots :: continuations");for continuation in &self.continuations {let state = continuation.common_stack_information.state;- // FIXME(frank-emrich) In general, it is not enough to just trace- // through the stacks of continuations; we also need to look through- // their `cont.bind` arguments. However, we don't currently have- // enough RTTI information to check if any of the values in the- // buffers used by `cont.bind` are GC values. As a workaround, note- // that we currently disallow cont.bind-ing GC values altogether.- // This way, it is okay not to check them here.match state {VMStackState::Suspended => {+ unsafe {+ trace_payload_roots(gc_roots_list, &continuation.values);+ }Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| {Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame);core::ops::ControlFlow::Continue(())@@ -865,8 +881,11 @@ impl StoreOpaque {// either case things should be handled correctly when traversing// further along in the chain, nothing required at this point.}- VMStackState::Fresh | VMStackState::Returned | VMStackState::Trapped => {- // Fresh/terminal continuations have no live GC values on their stack.+ VMStackState::Fresh => unsafe {+ trace_payload_roots(gc_roots_list, &continuation.args);+ },+ VMStackState::Returned | VMStackState::Trapped => {+ // Terminal continuations have no live GC values.}}}diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex e7f0c94d93..10526a1499 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -137,7 +137,7 @@ pub use crate::runtime::vm::vmcontext::VMNullHeapData;pub use crate::runtime::vm::vmcontext::{VMArrayCallHostFuncContext, VMCommonStackInformation, VMContRef, VMContext, VMFuncRef,VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMHostArray,- VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMStackLimits, VMStoreContext,+ VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMPayloads, VMStackLimits, VMStoreContext,VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,};#[cfg(has_custom_sync)]diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rsindex 5b0aae7248..28bf087928 100644--- a/crates/wasmtime/src/runtime/vm/gc.rs+++ b/crates/wasmtime/src/runtime/vm/gc.rs@@ -8,6 +8,8 @@ mod disabled;#[cfg(not(feature = "gc"))]pub use disabled::*;+#[cfg(feature = "stack-switching")]+mod cont_ref;mod data;mod func_ref;mod gc_ref;@@ -15,6 +17,8 @@ mod gc_runtime;mod host_data;mod i31;+#[cfg(feature = "stack-switching")]+pub use cont_ref::*;pub use data::*;pub use func_ref::*;pub use gc_ref::*;@@ -54,6 +58,10 @@ pub struct GcStore {/// The function-references table for this GC heap.pub func_ref_table: FuncRefTable,+ /// The continuation references table for this GC heap.+ #[cfg(feature = "stack-switching")]+ pub cont_ref_table: ContRefTable,+/// The total allocated bytes recorded after the last GC collection./// `None` if no collection has been performed yet. Used by the/// grow-or-collect heuristic.@@ -96,6 +104,8 @@ impl GcStore {) -> Self {let host_data_table = ExternRefHostDataTable::default();let func_ref_table = FuncRefTable::default();+ #[cfg(feature = "stack-switching")]+ let cont_ref_table = ContRefTable::default();let _ = &gc_zeal_alloc_counter;@@ -104,6 +114,8 @@ impl GcStore {gc_heap,host_data_table,func_ref_table,+ #[cfg(feature = "stack-switching")]+ cont_ref_table,last_post_gc_allocated_bytes: None,#[cfg(gc_zeal)]gc_zeal_alloc_counter,diff --git a/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs b/crates/wasmtime/src/runtime/vm/gc/cont_ref.rsnew file mode 100644index 0000000000..48c01b4935--- /dev/null+++ b/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs@@ -0,0 +1,56 @@+//! Side table for continuation references stored in the GC heap.+//!+//! This follows the same idea as `VMFuncRef` and `externref`. We+//! cannot trust native addresses that come out of the GC heap.++use crate::{Result, bail_bug, hash_map::HashMap, vm::VMContObj};+use wasmtime_core::{+ alloc::PanicOnOom,+ slab::{Id, Slab},+};++/// Side table mapping the IDs stored in GC fields to continuation+/// objects.+///+/// Raw ID zero is reserved for a null continuation reference.+/// Non-null IDs are one greater than their underlying slab IDs.+#[derive(Default)]+pub struct ContRefTable {+ interned: HashMap<VMContObj, u32>,+ slab: Slab<VMContObj>,+}++impl ContRefTable {+ /// Intern a continuation object and return the ID to store in the+ /// GC heap. Null continuation references use the reserved ID+ /// zero.+ ///+ /// # Safety+ ///+ /// A non-null continuation's `VMContRef` pointer must remain valid for the+ /// duration of this table's lifetime.+ pub unsafe fn intern(&mut self, contobj: Option<VMContObj>) -> u32 {+ let Some(contobj) = contobj else {+ return 0;+ };++ *self.interned.entry(contobj).or_insert_with(|| {+ // TODO(dhil): Handle allocation failure here rather than panicking.+ let id = self.slab.alloc(contobj).panic_on_oom().into_raw();+ id.checked_add(1).unwrap()+ })+ }++ /// Resolve an ID loaded from the GC heap.+ pub fn get(&self, raw: u32) -> Result<Option<VMContObj>> {+ if raw == 0 {+ return Ok(None);+ }++ let id = Id::from_raw(raw - 1);+ match self.slab.get(id).copied() {+ Some(contobj) => Ok(Some(contobj)),+ None => bail_bug!("bad continuation-reference table ID"),+ }+ }+}diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex 6964e405d6..157cf8095b 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -59,7 +59,11 @@ use crate::prelude::*;use crate::runtime::store::{Asyncness, InstanceId, StoreOpaque};#[cfg(feature = "gc")]use crate::runtime::vm::VMGcRef;+#[cfg(all(feature = "gc", feature = "stack-switching"))]+use crate::runtime::vm::provenance::VmPtr;use crate::runtime::vm::{self, HostResultHasUnwindSentinel, VMStore, f32x4, f64x2, i8x16};+#[cfg(all(feature = "gc", feature = "stack-switching"))]+use crate::vm::vmcontext::VMRawContObj;use core::convert::Infallible;use core::ptr::NonNull;#[cfg(feature = "threads")]@@ -604,6 +608,53 @@ fn get_interned_func_ref(Ok(func_ref.map_or(core::ptr::null_mut(), |f| f.as_ptr().cast()))}+// Intern a continuation reference into the GC heap's side table.+//+// This libcall may not GC.+#[cfg(all(feature = "gc", feature = "stack-switching"))]+unsafe fn intern_contref_for_gc_heap(+ store: &mut dyn VMStore,+ _instance: InstanceId,+ contref: *mut u8,+ revision: *mut u8,+) -> Result<u32> {+ use crate::store::AutoAssertNoGc;++ let mut store = AutoAssertNoGc::new(store.store_opaque_mut());+ let contobj = unsafe { crate::vm::VMContObj::from_raw_parts(contref, revision.addr()) };+ let id = unsafe { store.require_gc_store_mut()?.cont_ref_table.intern(contobj) };+ Ok(id)+}++// Resolve a continuation reference ID loaded from the GC heap and+// write its 16 bytes value into caller provided storage `out_result`.+//+// This libcall may not GC.+#[cfg(all(feature = "gc", feature = "stack-switching"))]+unsafe fn get_interned_contref(+ store: &mut dyn VMStore,+ _instance: InstanceId,+ contref_id: u32,+ out_result: *mut u8,+) -> Result<()> {+ use crate::store::AutoAssertNoGc;++ let store = AutoAssertNoGc::new(store.store_opaque_mut());+ let contobj = store.unwrap_gc_store().cont_ref_table.get(contref_id)?;+ let raw = match contobj {+ Some(contobj) => VMRawContObj {+ contref: NonNull::new(contobj.contref.as_ptr().cast::<u8>()).map(VmPtr::from),+ revision: contobj.revision,+ },+ None => VMRawContObj {+ contref: None,+ revision: 0,+ },+ };+ unsafe { out_result.cast::<VMRawContObj>().write(raw) };+ Ok(())+}+#[cfg(feature = "gc")]fn is_subtype(store: &mut dyn VMStore,@@ -1119,9 +1170,16 @@ fn cont_new(func: *mut u8,param_count: u32,result_count: u32,+ gc_refs: u32,) -> Result<Option<AllocationSize>> {- let ans =- crate::vm::stack_switching::cont_new(store, instance, func, param_count, result_count)?;+ let ans = crate::vm::stack_switching::cont_new(+ store,+ instance,+ func,+ param_count,+ result_count,+ gc_refs != 0,+ )?;Ok(Some(AllocationSize(ans.cast::<u8>() as usize)))}diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rsindex 8a71d52e7c..06f61340d3 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs@@ -3,7 +3,7 @@mod stack;-use crate::vm::{VMCommonStackInformation, VMContRef, VMHostArray, VMStackLimits};+use crate::vm::{VMCommonStackInformation, VMContRef, VMHostArray, VMPayloads, VMStackLimits};use core::{marker::PhantomPinned, ptr::NonNull};pub use stack::*;@@ -51,7 +51,7 @@ pub const CONTROL_EFFECT_TRAP_ENCODING: u64 =/// (i.e., the one pointed to by the VMContObj) has a pointer to the/// other end of the chain (i.e., its last ancestor).#[repr(C)]-#[derive(Debug, Clone, Copy)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct VMContObj {pub contref: NonNull<VMContRef>,pub revision: usize,@@ -109,6 +109,27 @@ impl VMHostArray {data: core::ptr::null_mut(),}}++ /// Makes this array empty.+ pub fn clear(&mut self) {+ *self = Self::empty();+ }+}++impl VMPayloads {+ /// Creates an empty payload buffer with no GC metadata.+ pub fn empty() -> Self {+ Self {+ buffer: VMHostArray::empty(),+ gc_ref_data: None,+ }+ }++ /// Makes this payload buffer empty and invalidates its GC metadata.+ pub fn clear(&mut self) {+ self.buffer.clear();+ self.gc_ref_data = None;+ }}impl VMContRef {@@ -135,8 +156,8 @@ impl VMContRef {let parent_chain = VMStackChain::Absent;let last_ancestor = core::ptr::null_mut();let stack = VMContinuationStack::unallocated();- let args = VMHostArray::empty();- let values = VMHostArray::empty();+ let args = VMPayloads::empty();+ let values = VMPayloads::empty();let revision = 0;let _marker = PhantomPinned;@@ -180,6 +201,7 @@ pub fn cont_new(func: *mut u8,param_count: u32,result_count: u32,+ gc_refs: bool,) -> crate::Result<*mut VMContRef> {let instance = store.instance_mut(instance);let caller_vmctx = instance.vmctx();@@ -197,14 +219,14 @@ pub fn cont_new(// The initialization function will allocate the actual args/return value buffer and// update this object (if needed).- let contref_args_ptr = &mut contref.args as *mut VMHostArray;-+ let contref_args_ptr = &mut contref.args as *mut VMPayloads;contref.stack.initialize(func.cast::<crate::vm::VMFuncRef>(),caller_vmctx.as_ptr(),contref_args_ptr,param_count,result_count,+ gc_refs,)?;// Now that the initial stack pointer was set by the initializationdiff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rsindex fc24418c59..e8438a91d4 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs@@ -4,7 +4,7 @@use crate::Result;use core::ops::Range;-use crate::runtime::vm::VMHostArray;+use crate::runtime::vm::VMPayloads;use crate::runtime::vm::{VMContext, VMFuncRef};cfg_select! {@@ -95,7 +95,9 @@ impl VMContinuationStack {/// Initializes this stack, such that it will execute the function denoted/// by `func_ref`. `parameter_count` and `return_value_count` must be the/// corresponding number of parameters and return values of `func_ref`.- /// `args` must point to the `args` field of the `VMContRef` owning this pointer.+ /// `args` must point to the `args` field of the `VMContRef` owning this+ /// stack. When `gc_refs` is true, its `gc_ref_data` field receives the+ /// corresponding root-marker buffer.////// It will be updated by this function to correctly describe/// the buffer used by this function for its arguments and return values.@@ -103,9 +105,10 @@ impl VMContinuationStack {&self,func_ref: *const VMFuncRef,caller_vmctx: *mut VMContext,- args: *mut VMHostArray,+ args: *mut VMPayloads,parameter_count: u32,return_value_count: u32,+ gc_refs: bool,) -> Result<()> {self.0.initialize(func_ref,@@ -113,6 +116,7 @@ impl VMContinuationStack {args,parameter_count,return_value_count,+ gc_refs,)}}diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rsindex 5c2cf049e4..1e5781f17b 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs@@ -1,7 +1,7 @@use crate::Result;use core::ops::Range;-use crate::runtime::vm::VMHostArray;+use crate::runtime::vm::VMPayloads;use crate::runtime::vm::{VMContext, VMFuncRef};/// Making sure that this has the same size as the non-dummy version, to@@ -59,9 +59,10 @@ impl VMContinuationStack {&self,_func_ref: *const VMFuncRef,_caller_vmctx: *mut VMContext,- _args: *mut VMHostArray,+ _args: *mut VMPayloads,_parameter_count: u32,_return_value_count: u32,+ _gc_refs: bool,) -> Result<()> {Ok(())}diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rsindex 247495a599..580b104653 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs@@ -64,8 +64,7 @@ use std::ops::Range;use std::ptr;use crate::prelude::*;-use crate::runtime::vm::VMHostArray;-use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw};+use crate::runtime::vm::{VMContext, VMFuncRef, VMHostArray, VMPayloads, ValRaw, VmPtr};#[derive(Debug, PartialEq, Eq)]pub enum Allocator {@@ -194,11 +193,14 @@ impl VMContinuationStack {/// calls `fiber_start` with the following arguments:/// TOS, func_ref, caller_vmctx, args_ptr, args_capacity///- /// Note that at this point we also allocate the args buffer- /// (see picture at the top of this file).+ /// Note that at this point we also allocate persistent launch storage for+ /// the args buffer and, when `gc_refs` is true, its parallel+ /// GC-reference-marker buffer (see picture at the top of this file)./// We define `args_capacity` as the max of parameter and return value count.- /// Then the size s of the actual buffer size is calculated as follows:- /// s = size_of(ValRaw) * `args_capacity`,+ /// Their combined, 16-byte-aligned size `s` is calculated as follows when+ /// `gc_refs` is true:+ /// s = size_of(ValRaw) * `args_capacity`+ /// + align_up(`args_capacity`, 16),////// Note that this value is used below, and we may have s = 0.///@@ -222,15 +224,22 @@ impl VMContinuationStack {/// ---------------|-------------------------------------------------------/// -0x28 - s | func_ref/// -0x30 - s | caller_vmctx- /// -0x38 - s | args (of type *mut ArrayRef<ValRaw>)+ /// -0x38 - s | args (of type *mut VMHostArray)/// -0x40 - s | return_value_count+ ///+ /// The saved RSP points at `TOS - 0x40 - s`. On entry,+ /// `wasmtime_continuation_start` pops the four launch-record words, leaving+ /// RSP at `TOS - 0x20 - s`, immediately below the persistent buffers.+ /// Subsequent stack frames grow towards lower addresses and may reuse the+ /// consumed launch-record storage.pub fn initialize(&self,func_ref: *const VMFuncRef,caller_vmctx: *mut VMContext,- args: *mut VMHostArray,+ args: *mut VMPayloads,parameter_count: u32,return_value_count: u32,+ gc_refs: bool,) -> Result<()> {let tos = self.top;@@ -240,22 +249,50 @@ impl VMContinuationStack {target.write(value)};- let args_ref = &mut *args;+ let payloads = &mut *args;+ let args_ref = &mut payloads.buffer;let args_capacity = std::cmp::max(parameter_count, return_value_count);// The args object must currently be empty.debug_assert_eq!(args_ref.capacity, 0);debug_assert_eq!(args_ref.length, 0);- let total_control_size = usize::try_from(args_capacity)?+ let args_data_size = usize::try_from(args_capacity)?.checked_mul(std::mem::size_of::<ValRaw>())- .and_then(|s| s.checked_add(0x40)).ok_or_else(|| {format_err!("continuation function type with {args_capacity} args \overflows stack control data size calculation")})?;- let args_data_size = total_control_size - 0x40;+ // Keep the fixed startup data 16-byte aligned.+ let gc_refs_data_size = if cfg!(feature = "gc") && gc_refs {+ usize::try_from(args_capacity)?+ .checked_add(15)+ .map(|s| s & !15)+ .ok_or_else(|| {+ format_err!(+ "continuation function type with {args_capacity} args \+ overflows stack control data size calculation"+ )+ })?+ } else {+ 0+ };+ let dynamic_data_size =+ args_data_size+ .checked_add(gc_refs_data_size)+ .ok_or_else(|| {+ format_err!(+ "continuation function type with {args_capacity} args \+ overflows stack control data size calculation"+ )+ })?;+ let total_control_size = dynamic_data_size.checked_add(0x40).ok_or_else(|| {+ format_err!(+ "continuation function type with {args_capacity} args \+ overflows stack control data size calculation"+ )+ })?;// Ensure the control data (fixed header + args buffer) fits// within the usable stack space. For Mmap allocations,@@ -282,6 +319,17 @@ impl VMContinuationStack {args_ref.capacity = args_capacity;args_ref.data = args_data_ptr;+ if cfg!(feature = "gc") && gc_refs {+ let data = if args_capacity == 0 {+ ptr::null_mut()+ } else {+ tos.sub(0x20 + dynamic_data_size)+ };+ if args_capacity > 0 {+ data.write_bytes(0, usize::try_from(args_capacity)?);+ }+ payloads.gc_ref_data = NonNull::new(data).map(VmPtr::from);+ }let to_store = [// Data near top of stack:@@ -290,10 +338,16 @@ impl VMContinuationStack {(0x18, tos.sub(total_control_size).addr()),(0x20, usize::try_from(args_capacity)?),// Data after the args buffer:- (0x28 + args_data_size, func_ref.addr()),- (0x30 + args_data_size, caller_vmctx.addr()),- (0x38 + args_data_size, args.addr()),- (0x40 + args_data_size, usize::try_from(return_value_count)?),+ (0x28 + dynamic_data_size, func_ref.addr()),+ (0x30 + dynamic_data_size, caller_vmctx.addr()),+ (+ 0x38 + dynamic_data_size,+ (args_ref as *mut VMHostArray).addr(),+ ),+ (+ 0x40 + dynamic_data_size,+ usize::try_from(return_value_count)?,+ ),];for (offset, data) in to_store {diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rsindex bc4f60a7d6..69337194d2 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs@@ -34,6 +34,7 @@ pub fn wasmtime_continuation_start_address() -> *const () {// values in various registers when execution of wasmtime_continuation_start begins://// RSP: TOS - 0x40 - (16 * `args_capacity`)+// - align_up(`args_capacity`, 16)// RBP: TOS - 0x10#[unsafe(naked)]diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rsindex d2d530e2fb..199efb34ba 100644--- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs+++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs@@ -335,10 +335,15 @@ impl Backtrace {let mut continuations_iter = unsafe { chain.into_continuation_iter() }.peekable();while let Some(continuation_ptr) = continuations_iter.next() {- let parent_limits_ptr = stack_limits_iter- .next()- .expect("expected one more VMStackLimits than continuations");let continuation = unsafe { &*continuation_ptr };+ let Some(parent_limits_ptr) = stack_limits_iter.next() else {+ // A detached suspended continuation chain ends in `Absent`,+ // unlike a running chain which ends in an initial stack. The+ // currently suspended stack was already traced above, and the+ // last continuation has no parent stack left to trace.+ debug_assert_eq!(continuation.parent_chain, VMStackChain::Absent);+ break;+ };let parent_limits = unsafe { &*parent_limits_ptr };// The parent of `continuation` if present and not the last in the chain.diff --git a/tests/all/stack_switching.rs b/tests/all/stack_switching.rsindex dfc33714f8..3a075d4719 100644--- a/tests/all/stack_switching.rs+++ b/tests/all/stack_switching.rs@@ -339,6 +339,71 @@ fn inter_instance_suspend() -> Result<()> {Ok(())}+/// Regression test for https://github.com/bytecodealliance/wasmtime/issues/13750.+///+/// A GC reference returned through a continuation payload must retain+/// its stack-map metadata while it is live on the operand stack+/// across subsequent GC safepoints.+#[cfg_attr(any(asan, miri), ignore)]+#[test]+fn continuation_result_gc_ref_on_operand_stack() -> Result<()> {+ let mut config = Config::new();+ config+ .wasm_stack_switching(true)+ .wasm_exceptions(true)+ .wasm_function_references(true)+ .wasm_gc(true)+ .collector(Collector::Copying);++ #[cfg(gc_zeal)]+ {+ let _ = config.gc_zeal_alloc_counter(std::num::NonZeroU32::new(1));+ }++ let engine = Engine::new(&config)?;+ let module = Module::new(+ &engine,+ r#"+ (module+ (type $S (struct (field $x i32)))+ (type $A (array i64))+ (type $ft (func (result (ref $S))))+ (type $ct (cont $ft))++ (func $target (result (ref $S))+ (struct.new $S (i32.const 0x12345678)))+ (elem declare func $target)++ (func (export "run") (result i32)+ (local $k (ref null $ct))+ (local $i i32)+ (local.set $k (cont.new $ct (ref.func $target)))++ ;; Keep the continuation's GC-reference result on the operand+ ;; stack, rather than storing it in a GC-typed local.+ (resume $ct (local.get $k))++ ;; Force collections while that operand-stack value is the+ ;; only live reference to the struct.+ (local.set $i (i32.const 0))+ (loop $again+ (drop (array.new_default $A (i32.const 256)))+ (local.set $i (i32.add (local.get $i) (i32.const 1)))+ (br_if $again+ (i32.lt_u (local.get $i) (i32.const 2000))))++ (struct.get $S $x))+ )+ "#,+ )?;++ let mut store = Store::new(&engine, ());+ let instance = Instance::new(&mut store, &module, &[])?;+ let run = instance.get_typed_func::<(), i32>(&mut store, "run")?;+ assert_eq!(run.call(&mut store, ())?, 0x12345678);+ Ok(())+}+/// Tests interaction with host functions. Note that the interaction with host/// functions and traps is covered by the module `traps` further down.mod host {diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.watindex 2606be8d12..1dcad97b8d 100644--- a/tests/disas/stack-switching/resume-suspend-data-passing.wat+++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat@@ -61,7 +61,7 @@;; block0(v0: i64, v1: i64):;; @003c v3 = iconst.i32 10;; @0044 v7 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @0044 v31 = iconst.i64 136+;; @0044 v31 = iconst.i64 144;; @0044 v34 = stack_addr.i64 ss0;; @0044 v37 = iconst.i64 0;; @0044 v47 = iconst.i64 96@@ -118,8 +118,8 @@;; block8:;; @0044 store.i64 notrap aligned region7 v11, v9+80;; v91 = iconst.i32 1-;; v92 = iconst.i64 136-;; v93 = iadd.i64 v9, v92 ; v92 = 136+;; v92 = iconst.i64 144+;; v93 = iadd.i64 v9, v92 ; v92 = 144;; @0044 store notrap aligned region8 v91, v93+4 ; v91 = 1;; @0044 store.i64 notrap aligned region4 v34, v93+8;; @0044 store.i32 notrap aligned region6 v4, v34@@ -203,7 +203,7 @@;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8;; gv2 = load.i64 notrap aligned region1 gv1+24;; sig0 = (i64 vmctx, i32) -> i64 tail-;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail+;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail;; sig2 = (i64 vmctx) tail;; fn0 = colocated u805306368:6 sig0;; fn1 = colocated u805306368:42 sig1@@ -214,195 +214,195 @@;; @0056 v2 = iconst.i32 0;; @0056 v3 = call fn0(v0, v2) ; v2 = 0;; @0058 trapz v3, user16-;; @0058 v6 = call fn1(v0, v3, v2, v2) ; v2 = 0, v2 = 0-;; @0058 v7 = load.i64 notrap aligned region2 v6+88+;; @0058 v7 = call fn1(v0, v3, v2, v2, v2) ; v2 = 0, v2 = 0, v2 = 0+;; @0058 v8 = load.i64 notrap aligned region2 v7+88+;; @0058 v10 = uextend.i128 v8+;; @0058 v11 = iconst.i64 64+;; v139 = ishl v10, v11 ; v11 = 64;; @0058 v9 = uextend.i128 v7-;; @0058 v10 = iconst.i64 64-;; v138 = ishl v9, v10 ; v10 = 64-;; @0058 v8 = uextend.i128 v6-;; @0058 v13 = bor v138, v8-;; @0062 v22 = iconst.i64 1-;; @0062 v25 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @0062 v28 = iconst.i64 0-;; @0062 v29 = iconst.i64 2-;; @0062 v33 = iconst.i32 1-;; @0062 v34 = iconst.i32 2-;; @0062 v48 = iconst.i64 40-;; @0062 v51 = stack_addr.i64 ss0-;; @0062 v52 = iconst.i64 48-;; @0062 v53 = iadd v0, v52 ; v52 = 48-;; @0062 v60 = iconst.i64 96-;; @0062 v63 = iconst.i64 -24-;; v142 = iconst.i64 0x0001_0000_0000-;; @005c jump block2(v13)+;; @0058 v14 = bor v139, v9+;; @0062 v23 = iconst.i64 1+;; @0062 v26 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @0062 v29 = iconst.i64 0+;; @0062 v30 = iconst.i64 2+;; @0062 v34 = iconst.i32 1+;; @0062 v35 = iconst.i32 2+;; @0062 v49 = iconst.i64 40+;; @0062 v52 = stack_addr.i64 ss0+;; @0062 v53 = iconst.i64 48+;; @0062 v54 = iadd v0, v53 ; v53 = 48+;; @0062 v61 = iconst.i64 96+;; @0062 v64 = iconst.i64 -24+;; v143 = iconst.i64 0x0001_0000_0000+;; @005c jump block2(v14);;-;; block2(v14: i128):+;; block2(v15: i128):;; @0062 jump block5;;;; block5:-;; @0062 v15 = ireduce.i64 v14-;; @0062 trapz v15, user16-;; @0062 v20 = load.i64 notrap aligned region2 v15+88-;; v145 = iconst.i64 64-;; v146 = ushr.i128 v14, v145 ; v145 = 64-;; @0062 v19 = ireduce.i64 v146-;; @0062 v21 = icmp eq v20, v19-;; @0062 trapz v21, user23-;; v147 = iconst.i64 1-;; v148 = iadd v20, v147 ; v147 = 1-;; @0062 store notrap aligned region2 v148, v15+88-;; @0062 v24 = load.i64 notrap aligned region3 v15+80-;; @0062 v26 = load.i64 notrap aligned region4 v25+88-;; @0062 v27 = load.i64 notrap aligned region4 v25+96-;; @0062 store notrap aligned region5 v26, v24+64-;; @0062 store notrap aligned region5 v27, v24+72-;; v149 = iconst.i64 0-;; @0062 store notrap aligned region3 v149, v15+80 ; v149 = 0-;; v150 = iconst.i64 2-;; @0062 store notrap aligned region4 v150, v25+88 ; v150 = 2-;; @0062 store notrap aligned region4 v15, v25+96-;; v151 = iconst.i32 1-;; @0062 store notrap aligned region6 v151, v15+32 ; v151 = 1-;; v152 = iconst.i32 2-;; @0062 store notrap aligned region6 v152, v27+32 ; v152 = 2-;; @0062 v38 = load.i64 notrap aligned region7 v25+72-;; @0062 v39 = load.i64 notrap aligned region8 v25+64-;; @0062 v40 = load.i64 notrap aligned region9 v25+80-;; @0062 store notrap aligned region10 v38, v27+8-;; @0062 store notrap aligned region11 v39, v27+16-;; @0062 store notrap aligned region12 v40, v27+24-;; @0062 v41 = load.i64 notrap aligned region1 v25+24-;; @0062 store notrap aligned region13 v41, v27-;; @0062 v44 = load.i64 notrap aligned region13 v15-;; @0062 store notrap aligned region1 v44, v25+24-;; @0062 v45 = load.i64 notrap aligned region10 v15+8-;; @0062 store notrap aligned region7 v45, v25+72-;; @0062 v46 = load.i64 notrap aligned region11 v15+16-;; @0062 store notrap aligned region8 v46, v25+64-;; @0062 v47 = load.i64 notrap aligned region12 v15+24-;; @0062 store notrap aligned region9 v47, v25+80-;; v153 = iconst.i64 40-;; v154 = iadd v27, v153 ; v153 = 40-;; @0062 store notrap aligned region14 v151, v154+4 ; v151 = 1-;; @0062 store.i64 notrap aligned region15 v51, v154+8-;; v155 = iadd.i64 v0, v52 ; v52 = 48-;; @0062 store notrap aligned region16 v155, v51-;; @0062 store notrap aligned region17 v151, v154 ; v151 = 1-;; @0062 store notrap aligned region12 v151, v27+56 ; v151 = 1-;; v156 = iconst.i64 96-;; v157 = iadd v24, v156 ; v156 = 96-;; @0062 v62 = load.i64 notrap aligned region18 v157-;; v158 = iconst.i64 -24-;; v159 = iadd v62, v158 ; v158 = -24-;; v160 = iconst.i64 0x0001_0000_0000-;; @0062 v65 = stack_switch v159, v159, v160 ; v160 = 0x0001_0000_0000-;; @0062 v67 = load.i64 notrap aligned region4 v25+88-;; @0062 v68 = load.i64 notrap aligned region4 v25+96-;; @0062 store notrap aligned region4 v26, v25+88-;; @0062 store notrap aligned region4 v27, v25+96-;; @0062 store notrap aligned region6 v151, v27+32 ; v151 = 1-;; v161 = iconst.i32 0-;; @0062 store notrap aligned region17 v161, v154 ; v161 = 0-;; @0062 store notrap aligned region14 v161, v154+4 ; v161 = 0-;; @0062 store notrap aligned region15 v149, v154+8 ; v149 = 0-;; @0062 store notrap aligned region12 v149, v27+56 ; v149 = 0-;; @0062 brif v65, block9, block6+;; @0062 v16 = ireduce.i64 v15+;; @0062 trapz v16, user16+;; @0062 v21 = load.i64 notrap aligned region2 v16+88+;; v146 = iconst.i64 64+;; v147 = ushr.i128 v15, v146 ; v146 = 64+;; @0062 v20 = ireduce.i64 v147+;; @0062 v22 = icmp eq v21, v20+;; @0062 trapz v22, user23+;; v148 = iconst.i64 1+;; v149 = iadd v21, v148 ; v148 = 1+;; @0062 store notrap aligned region2 v149, v16+88+;; @0062 v25 = load.i64 notrap aligned region3 v16+80+;; @0062 v27 = load.i64 notrap aligned region4 v26+88+;; @0062 v28 = load.i64 notrap aligned region4 v26+96+;; @0062 store notrap aligned region5 v27, v25+64+;; @0062 store notrap aligned region5 v28, v25+72+;; v150 = iconst.i64 0+;; @0062 store notrap aligned region3 v150, v16+80 ; v150 = 0+;; v151 = iconst.i64 2+;; @0062 store notrap aligned region4 v151, v26+88 ; v151 = 2+;; @0062 store notrap aligned region4 v16, v26+96+;; v152 = iconst.i32 1+;; @0062 store notrap aligned region6 v152, v16+32 ; v152 = 1+;; v153 = iconst.i32 2+;; @0062 store notrap aligned region6 v153, v28+32 ; v153 = 2+;; @0062 v39 = load.i64 notrap aligned region7 v26+72+;; @0062 v40 = load.i64 notrap aligned region8 v26+64+;; @0062 v41 = load.i64 notrap aligned region9 v26+80+;; @0062 store notrap aligned region10 v39, v28+8+;; @0062 store notrap aligned region11 v40, v28+16+;; @0062 store notrap aligned region12 v41, v28+24+;; @0062 v42 = load.i64 notrap aligned region1 v26+24+;; @0062 store notrap aligned region13 v42, v28+;; @0062 v45 = load.i64 notrap aligned region13 v16+;; @0062 store notrap aligned region1 v45, v26+24+;; @0062 v46 = load.i64 notrap aligned region10 v16+8+;; @0062 store notrap aligned region7 v46, v26+72+;; @0062 v47 = load.i64 notrap aligned region11 v16+16+;; @0062 store notrap aligned region8 v47, v26+64+;; @0062 v48 = load.i64 notrap aligned region12 v16+24+;; @0062 store notrap aligned region9 v48, v26+80+;; v154 = iconst.i64 40+;; v155 = iadd v28, v154 ; v154 = 40+;; @0062 store notrap aligned region14 v152, v155+4 ; v152 = 1+;; @0062 store.i64 notrap aligned region15 v52, v155+8+;; v156 = iadd.i64 v0, v53 ; v53 = 48+;; @0062 store notrap aligned region16 v156, v52+;; @0062 store notrap aligned region17 v152, v155 ; v152 = 1+;; @0062 store notrap aligned region12 v152, v28+56 ; v152 = 1+;; v157 = iconst.i64 96+;; v158 = iadd v25, v157 ; v157 = 96+;; @0062 v63 = load.i64 notrap aligned region18 v158+;; v159 = iconst.i64 -24+;; v160 = iadd v63, v159 ; v159 = -24+;; v161 = iconst.i64 0x0001_0000_0000+;; @0062 v66 = stack_switch v160, v160, v161 ; v161 = 0x0001_0000_0000+;; @0062 v68 = load.i64 notrap aligned region4 v26+88+;; @0062 v69 = load.i64 notrap aligned region4 v26+96+;; @0062 store notrap aligned region4 v27, v26+88+;; @0062 store notrap aligned region4 v28, v26+96+;; @0062 store notrap aligned region6 v152, v28+32 ; v152 = 1+;; v162 = iconst.i32 0+;; @0062 store notrap aligned region17 v162, v155 ; v162 = 0+;; @0062 store notrap aligned region14 v162, v155+4 ; v162 = 0+;; @0062 store notrap aligned region15 v150, v155+8 ; v150 = 0+;; @0062 store notrap aligned region12 v150, v28+56 ; v150 = 0+;; @0062 brif v66, block9, block6;;;; block9:-;; @0062 v58 = iconst.i64 32-;; @0062 v75 = ushr.i64 v65, v58 ; v58 = 32-;; @0062 v76 = iconst.i64 4-;; @0062 v77 = icmp eq v75, v76 ; v76 = 4-;; @0062 brif v77, block8, block7+;; @0062 v59 = iconst.i64 32+;; @0062 v76 = ushr.i64 v66, v59 ; v59 = 32+;; @0062 v77 = iconst.i64 4+;; @0062 v78 = icmp eq v76, v77 ; v77 = 4+;; @0062 brif v78, block8, block7;;;; block8 cold:-;; @0062 v80 = iconst.i32 5-;; @0062 store notrap aligned region6 v80, v68+32 ; v80 = 5-;; @0062 v83 = load.i64 notrap aligned region13 v27-;; @0062 store notrap aligned region1 v83, v25+24-;; @0062 v84 = load.i64 notrap aligned region10 v27+8-;; @0062 store notrap aligned region7 v84, v25+72-;; @0062 v85 = load.i64 notrap aligned region11 v27+16-;; @0062 store notrap aligned region8 v85, v25+64-;; @0062 v86 = load.i64 notrap aligned region12 v27+24-;; @0062 store notrap aligned region9 v86, v25+80-;; v167 = iconst.i32 0-;; v168 = iconst.i64 120-;; v169 = iadd.i64 v68, v168 ; v168 = 120-;; @0062 store notrap aligned region17 v167, v169 ; v167 = 0-;; @0062 store notrap aligned region14 v167, v169+4 ; v167 = 0-;; v170 = iconst.i64 0-;; @0062 store notrap aligned region15 v170, v169+8 ; v170 = 0-;; v171 = iconst.i64 136-;; v172 = iadd.i64 v68, v171 ; v171 = 136-;; @0062 store notrap aligned region17 v167, v172 ; v167 = 0-;; @0062 store notrap aligned region14 v167, v172+4 ; v167 = 0-;; @0062 store notrap aligned region15 v170, v172+8 ; v170 = 0+;; @0062 v81 = iconst.i32 5+;; @0062 store notrap aligned region6 v81, v69+32 ; v81 = 5+;; @0062 v84 = load.i64 notrap aligned region13 v28+;; @0062 store notrap aligned region1 v84, v26+24+;; @0062 v85 = load.i64 notrap aligned region10 v28+8+;; @0062 store notrap aligned region7 v85, v26+72+;; @0062 v86 = load.i64 notrap aligned region11 v28+16+;; @0062 store notrap aligned region8 v86, v26+64+;; @0062 v87 = load.i64 notrap aligned region12 v28+24+;; @0062 store notrap aligned region9 v87, v26+80+;; v168 = iconst.i32 0+;; v169 = iconst.i64 120+;; v170 = iadd.i64 v69, v169 ; v169 = 120+;; @0062 store notrap aligned region17 v168, v170 ; v168 = 0+;; @0062 store notrap aligned region14 v168, v170+4 ; v168 = 0+;; v171 = iconst.i64 0+;; @0062 store notrap aligned region15 v171, v170+8 ; v171 = 0+;; v172 = iconst.i64 144+;; v173 = iadd.i64 v69, v172 ; v172 = 144+;; @0062 store notrap aligned region17 v168, v173 ; v168 = 0+;; @0062 store notrap aligned region14 v168, v173+4 ; v168 = 0+;; @0062 store notrap aligned region15 v171, v173+8 ; v171 = 0;; @0062 try_call fn2(v0), sig2, block11, [ context v0 ];;;; block11:;; @0062 trap user12;;;; block7:-;; @0062 v101 = load.i64 notrap aligned region7 v25+72-;; @0062 v102 = load.i64 notrap aligned region8 v25+64-;; @0062 v103 = load.i64 notrap aligned region9 v25+80-;; @0062 store notrap aligned region10 v101, v68+8-;; @0062 store notrap aligned region11 v102, v68+16-;; @0062 store notrap aligned region12 v103, v68+24-;; @0062 v106 = load.i64 notrap aligned region13 v27-;; @0062 store notrap aligned region1 v106, v25+24-;; @0062 v107 = load.i64 notrap aligned region10 v27+8-;; @0062 store notrap aligned region7 v107, v25+72-;; @0062 v108 = load.i64 notrap aligned region11 v27+16-;; @0062 store notrap aligned region8 v108, v25+64-;; @0062 v109 = load.i64 notrap aligned region12 v27+24-;; @0062 store notrap aligned region9 v109, v25+80-;; @0062 v111 = load.i64 notrap aligned region2 v68+88+;; @0062 v102 = load.i64 notrap aligned region7 v26+72+;; @0062 v103 = load.i64 notrap aligned region8 v26+64+;; @0062 v104 = load.i64 notrap aligned region9 v26+80+;; @0062 store notrap aligned region10 v102, v69+8+;; @0062 store notrap aligned region11 v103, v69+16+;; @0062 store notrap aligned region12 v104, v69+24+;; @0062 v107 = load.i64 notrap aligned region13 v28+;; @0062 store notrap aligned region1 v107, v26+24+;; @0062 v108 = load.i64 notrap aligned region10 v28+8+;; @0062 store notrap aligned region7 v108, v26+72+;; @0062 v109 = load.i64 notrap aligned region11 v28+16+;; @0062 store notrap aligned region8 v109, v26+64+;; @0062 v110 = load.i64 notrap aligned region12 v28+24+;; @0062 store notrap aligned region9 v110, v26+80+;; @0062 v112 = load.i64 notrap aligned region2 v69+88;; @0062 jump block10;;;; block12 cold:;; @0062 trap user12;;;; block13:-;; @0062 v118 = iconst.i64 136-;; @0062 v119 = iadd.i64 v68, v118 ; v118 = 136-;; @0062 v120 = load.i64 notrap aligned region15 v119+8-;; @0062 v121 = load.i32 notrap aligned region16 v120-;; v164 = iconst.i32 0-;; @0062 store notrap aligned region17 v164, v119 ; v164 = 0+;; @0062 v119 = iconst.i64 144+;; @0062 v120 = iadd.i64 v69, v119 ; v119 = 144+;; @0062 v121 = load.i64 notrap aligned region15 v120+8+;; @0062 v122 = load.i32 notrap aligned region16 v121+;; v165 = iconst.i32 0+;; @0062 store notrap aligned region17 v165, v120 ; v165 = 0;; @0062 jump block4;;;; block10:-;; @0062 v110 = ireduce.i32 v65-;; @0062 br_table v110, block12, [block13]+;; @0062 v111 = ireduce.i32 v66+;; @0062 br_table v111, block12, [block13];;;; block6:-;; @0062 v125 = load.i64 notrap aligned region13 v27-;; @0062 store notrap aligned region1 v125, v25+24-;; @0062 v126 = load.i64 notrap aligned region10 v27+8-;; @0062 store notrap aligned region7 v126, v25+72-;; @0062 v127 = load.i64 notrap aligned region11 v27+16-;; @0062 store notrap aligned region8 v127, v25+64-;; @0062 v128 = load.i64 notrap aligned region12 v27+24-;; @0062 store notrap aligned region9 v128, v25+80-;; @0062 v131 = iconst.i32 4-;; @0062 store notrap aligned region6 v131, v68+32 ; v131 = 4-;; @0062 v132 = iconst.i64 120-;; @0062 v133 = iadd.i64 v68, v132 ; v132 = 120-;; @0062 v134 = load.i64 notrap aligned region15 v133+8-;; v162 = iconst.i32 0-;; @0062 store notrap aligned region17 v162, v133 ; v162 = 0-;; @0062 store notrap aligned region14 v162, v133+4 ; v162 = 0-;; v163 = iconst.i64 0-;; @0062 store notrap aligned region15 v163, v133+8 ; v163 = 0+;; @0062 v126 = load.i64 notrap aligned region13 v28+;; @0062 store notrap aligned region1 v126, v26+24+;; @0062 v127 = load.i64 notrap aligned region10 v28+8+;; @0062 store notrap aligned region7 v127, v26+72+;; @0062 v128 = load.i64 notrap aligned region11 v28+16+;; @0062 store notrap aligned region8 v128, v26+64+;; @0062 v129 = load.i64 notrap aligned region12 v28+24+;; @0062 store notrap aligned region9 v129, v26+80+;; @0062 v132 = iconst.i32 4+;; @0062 store notrap aligned region6 v132, v69+32 ; v132 = 4+;; @0062 v133 = iconst.i64 120+;; @0062 v134 = iadd.i64 v69, v133 ; v133 = 120+;; @0062 v135 = load.i64 notrap aligned region15 v134+8+;; v163 = iconst.i32 0+;; @0062 store notrap aligned region17 v163, v134 ; v163 = 0+;; @0062 store notrap aligned region14 v163, v134+4 ; v163 = 0+;; v164 = iconst.i64 0+;; @0062 store notrap aligned region15 v164, v134+8 ; v164 = 0;; @0068 return;;;; block4:-;; @0062 v113 = uextend.i128 v111-;; v165 = iconst.i64 64-;; v166 = ishl v113, v165 ; v165 = 64-;; @0062 v112 = uextend.i128 v68-;; @0062 v117 = bor v166, v112-;; @006d jump block2(v117)+;; @0062 v114 = uextend.i128 v112+;; v166 = iconst.i64 64+;; v167 = ishl v114, v166 ; v166 = 64+;; @0062 v113 = uextend.i128 v69+;; @0062 v118 = bor v167, v113+;; @006d jump block2(v118);; }diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.watindex bff2ab520a..ff5c9192fb 100644--- a/tests/disas/stack-switching/resume-suspend.wat+++ b/tests/disas/stack-switching/resume-suspend.wat@@ -91,8 +91,8 @@;; block6:;; @003b store.i64 notrap aligned region7 v8, v6+80;; v82 = iconst.i32 1-;; @003b v28 = iconst.i64 136-;; @003b v29 = iadd.i64 v6, v28 ; v28 = 136+;; @003b v28 = iconst.i64 144+;; @003b v29 = iadd.i64 v6, v28 ; v28 = 144;; @003b store notrap aligned region8 v82, v29+4 ; v82 = 1;; @003b v31 = stack_addr.i64 ss0;; @003b store notrap aligned region4 v31, v29+8@@ -117,7 +117,7 @@;; @003b brif v51, block8, block9;;;; block8 cold:-;; v87 = iadd.i64 v6, v28 ; v28 = 136+;; v87 = iadd.i64 v6, v28 ; v28 = 144;; @003b v54 = load.i64 notrap aligned region4 v87+8;; @003b v55 = load.i32 notrap aligned region6 v54;; v88 = iconst.i32 0@@ -131,7 +131,7 @@;; @003b trap user12;;;; block9:-;; v84 = iadd.i64 v6, v28 ; v28 = 136+;; v84 = iadd.i64 v6, v28 ; v28 = 144;; @003b v61 = load.i64 notrap aligned region4 v84+8;; v85 = iconst.i32 0;; @003b store notrap aligned region11 v85, v84 ; v85 = 0@@ -169,7 +169,7 @@;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8;; gv2 = load.i64 notrap aligned region1 gv1+24;; sig0 = (i64 vmctx, i32) -> i64 tail-;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail+;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail;; sig2 = (i64 vmctx) tail;; fn0 = colocated u805306368:6 sig0;; fn1 = colocated u805306368:42 sig1@@ -180,186 +180,186 @@;; @0043 v9 = iconst.i32 0;; @0043 v10 = call fn0(v0, v9) ; v9 = 0;; @0045 trapz v10, user16-;; @0045 v13 = call fn1(v0, v10, v9, v9) ; v9 = 0, v9 = 0-;; @0045 v14 = load.i64 notrap aligned region2 v13+88+;; @0045 v14 = call fn1(v0, v10, v9, v9, v9) ; v9 = 0, v9 = 0, v9 = 0+;; @0045 v15 = load.i64 notrap aligned region2 v14+88;; @004e jump block3;;;; block3:-;; @0045 v16 = uextend.i128 v14+;; @0045 v17 = uextend.i128 v15;; @0040 v5 = iconst.i64 64-;; v153 = ishl v16, v5 ; v5 = 64-;; v155 = ireduce.i64 v153-;; v157 = bor v155, v13-;; @004e trapz v157, user16-;; @004e v26 = load.i64 notrap aligned region2 v157+88-;; @0045 v15 = uextend.i128 v13-;; @0045 v20 = bor v153, v15-;; v159 = ushr v20, v5 ; v5 = 64-;; @004e v25 = ireduce.i64 v159-;; @004e v27 = icmp eq v26, v25-;; @004e trapz v27, user23-;; @004e v28 = iconst.i64 1-;; @004e v29 = iadd v26, v28 ; v28 = 1-;; @004e store notrap aligned region2 v29, v157+88-;; @004e v30 = load.i64 notrap aligned region3 v157+80-;; @004e v31 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @004e v32 = load.i64 notrap aligned region4 v31+88-;; @004e v33 = load.i64 notrap aligned region4 v31+96-;; @004e store notrap aligned region5 v32, v30+64-;; @004e store notrap aligned region5 v33, v30+72+;; v154 = ishl v17, v5 ; v5 = 64+;; v156 = ireduce.i64 v154+;; v158 = bor v156, v14+;; @004e trapz v158, user16+;; @004e v27 = load.i64 notrap aligned region2 v158+88+;; @0045 v16 = uextend.i128 v14+;; @0045 v21 = bor v154, v16+;; v160 = ushr v21, v5 ; v5 = 64+;; @004e v26 = ireduce.i64 v160+;; @004e v28 = icmp eq v27, v26+;; @004e trapz v28, user23+;; @004e v29 = iconst.i64 1+;; @004e v30 = iadd v27, v29 ; v29 = 1+;; @004e store notrap aligned region2 v30, v158+88+;; @004e v31 = load.i64 notrap aligned region3 v158+80+;; @004e v32 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @004e v33 = load.i64 notrap aligned region4 v32+88+;; @004e v34 = load.i64 notrap aligned region4 v32+96+;; @004e store notrap aligned region5 v33, v31+64+;; @004e store notrap aligned region5 v34, v31+72;; @0040 v2 = iconst.i64 0-;; @004e store notrap aligned region3 v2, v157+80 ; v2 = 0-;; @004e v35 = iconst.i64 2-;; @004e store notrap aligned region4 v35, v31+88 ; v35 = 2-;; @004e store notrap aligned region4 v157, v31+96-;; @004e v39 = iconst.i32 1-;; @004e store notrap aligned region6 v39, v157+32 ; v39 = 1-;; @004e v40 = iconst.i32 2-;; @004e store notrap aligned region6 v40, v33+32 ; v40 = 2-;; @004e v44 = load.i64 notrap aligned region7 v31+72-;; @004e v45 = load.i64 notrap aligned region8 v31+64-;; @004e v46 = load.i64 notrap aligned region9 v31+80-;; @004e store notrap aligned region10 v44, v33+8-;; @004e store notrap aligned region11 v45, v33+16-;; @004e store notrap aligned region12 v46, v33+24-;; @004e v47 = load.i64 notrap aligned region1 v31+24-;; @004e store notrap aligned region13 v47, v33-;; @004e v50 = load.i64 notrap aligned region13 v157-;; @004e store notrap aligned region1 v50, v31+24-;; @004e v51 = load.i64 notrap aligned region10 v157+8-;; @004e store notrap aligned region7 v51, v31+72-;; @004e v52 = load.i64 notrap aligned region11 v157+16-;; @004e store notrap aligned region8 v52, v31+64-;; @004e v53 = load.i64 notrap aligned region12 v157+24-;; @004e store notrap aligned region9 v53, v31+80-;; @004e v54 = iconst.i64 40-;; @004e v55 = iadd v33, v54 ; v54 = 40-;; @004e store notrap aligned region14 v39, v55+4 ; v39 = 1-;; @004e v57 = stack_addr.i64 ss0-;; @004e store notrap aligned region15 v57, v55+8-;; @004e v58 = iconst.i64 48-;; @004e v59 = iadd.i64 v0, v58 ; v58 = 48-;; @004e store notrap aligned region16 v59, v57-;; @004e store notrap aligned region17 v39, v55 ; v39 = 1-;; @004e store notrap aligned region12 v39, v33+56 ; v39 = 1-;; @004e v66 = iconst.i64 96-;; @004e v67 = iadd v30, v66 ; v66 = 96-;; @004e v68 = load.i64 notrap aligned region18 v67-;; @004e v69 = iconst.i64 -24-;; @004e v70 = iadd v68, v69 ; v69 = -24-;; v161 = iconst.i64 0x0001_0000_0000-;; @004e v71 = stack_switch v70, v70, v161 ; v161 = 0x0001_0000_0000-;; @004e v73 = load.i64 notrap aligned region4 v31+88-;; @004e v74 = load.i64 notrap aligned region4 v31+96-;; @004e store notrap aligned region4 v32, v31+88-;; @004e store notrap aligned region4 v33, v31+96-;; @004e store notrap aligned region6 v39, v33+32 ; v39 = 1-;; v164 = iconst.i32 0-;; @004e store notrap aligned region17 v164, v55 ; v164 = 0-;; @004e store notrap aligned region14 v164, v55+4 ; v164 = 0-;; @004e store notrap aligned region15 v2, v55+8 ; v2 = 0-;; @004e store notrap aligned region12 v2, v33+56 ; v2 = 0-;; @004e brif v71, block7, block4+;; @004e store notrap aligned region3 v2, v158+80 ; v2 = 0+;; @004e v36 = iconst.i64 2+;; @004e store notrap aligned region4 v36, v32+88 ; v36 = 2+;; @004e store notrap aligned region4 v158, v32+96+;; @004e v40 = iconst.i32 1+;; @004e store notrap aligned region6 v40, v158+32 ; v40 = 1+;; @004e v41 = iconst.i32 2+;; @004e store notrap aligned region6 v41, v34+32 ; v41 = 2+;; @004e v45 = load.i64 notrap aligned region7 v32+72+;; @004e v46 = load.i64 notrap aligned region8 v32+64+;; @004e v47 = load.i64 notrap aligned region9 v32+80+;; @004e store notrap aligned region10 v45, v34+8+;; @004e store notrap aligned region11 v46, v34+16+;; @004e store notrap aligned region12 v47, v34+24+;; @004e v48 = load.i64 notrap aligned region1 v32+24+;; @004e store notrap aligned region13 v48, v34+;; @004e v51 = load.i64 notrap aligned region13 v158+;; @004e store notrap aligned region1 v51, v32+24+;; @004e v52 = load.i64 notrap aligned region10 v158+8+;; @004e store notrap aligned region7 v52, v32+72+;; @004e v53 = load.i64 notrap aligned region11 v158+16+;; @004e store notrap aligned region8 v53, v32+64+;; @004e v54 = load.i64 notrap aligned region12 v158+24+;; @004e store notrap aligned region9 v54, v32+80+;; @004e v55 = iconst.i64 40+;; @004e v56 = iadd v34, v55 ; v55 = 40+;; @004e store notrap aligned region14 v40, v56+4 ; v40 = 1+;; @004e v58 = stack_addr.i64 ss0+;; @004e store notrap aligned region15 v58, v56+8+;; @004e v59 = iconst.i64 48+;; @004e v60 = iadd.i64 v0, v59 ; v59 = 48+;; @004e store notrap aligned region16 v60, v58+;; @004e store notrap aligned region17 v40, v56 ; v40 = 1+;; @004e store notrap aligned region12 v40, v34+56 ; v40 = 1+;; @004e v67 = iconst.i64 96+;; @004e v68 = iadd v31, v67 ; v67 = 96+;; @004e v69 = load.i64 notrap aligned region18 v68+;; @004e v70 = iconst.i64 -24+;; @004e v71 = iadd v69, v70 ; v70 = -24+;; v162 = iconst.i64 0x0001_0000_0000+;; @004e v72 = stack_switch v71, v71, v162 ; v162 = 0x0001_0000_0000+;; @004e v74 = load.i64 notrap aligned region4 v32+88+;; @004e v75 = load.i64 notrap aligned region4 v32+96+;; @004e store notrap aligned region4 v33, v32+88+;; @004e store notrap aligned region4 v34, v32+96+;; @004e store notrap aligned region6 v40, v34+32 ; v40 = 1+;; v165 = iconst.i32 0+;; @004e store notrap aligned region17 v165, v56 ; v165 = 0+;; @004e store notrap aligned region14 v165, v56+4 ; v165 = 0+;; @004e store notrap aligned region15 v2, v56+8 ; v2 = 0+;; @004e store notrap aligned region12 v2, v34+56 ; v2 = 0+;; @004e brif v72, block7, block4;;;; block7:-;; @004e v64 = iconst.i64 32-;; @004e v81 = ushr.i64 v71, v64 ; v64 = 32-;; @004e v82 = iconst.i64 4-;; @004e v83 = icmp eq v81, v82 ; v82 = 4-;; @004e brif v83, block6, block5+;; @004e v65 = iconst.i64 32+;; @004e v82 = ushr.i64 v72, v65 ; v65 = 32+;; @004e v83 = iconst.i64 4+;; @004e v84 = icmp eq v82, v83 ; v83 = 4+;; @004e brif v84, block6, block5;;;; block6 cold:-;; @004e v86 = iconst.i32 5-;; @004e store notrap aligned region6 v86, v74+32 ; v86 = 5-;; @004e v89 = load.i64 notrap aligned region13 v33-;; @004e store notrap aligned region1 v89, v31+24-;; @004e v90 = load.i64 notrap aligned region10 v33+8-;; @004e store notrap aligned region7 v90, v31+72-;; @004e v91 = load.i64 notrap aligned region11 v33+16-;; @004e store notrap aligned region8 v91, v31+64-;; @004e v92 = load.i64 notrap aligned region12 v33+24-;; @004e store notrap aligned region9 v92, v31+80-;; v173 = iconst.i32 0-;; v174 = iconst.i64 120-;; v175 = iadd.i64 v74, v174 ; v174 = 120-;; @004e store notrap aligned region17 v173, v175 ; v173 = 0-;; @004e store notrap aligned region14 v173, v175+4 ; v173 = 0-;; v176 = iconst.i64 0-;; @004e store notrap aligned region15 v176, v175+8 ; v176 = 0-;; v177 = iconst.i64 136-;; v178 = iadd.i64 v74, v177 ; v177 = 136-;; @004e store notrap aligned region17 v173, v178 ; v173 = 0-;; @004e store notrap aligned region14 v173, v178+4 ; v173 = 0-;; @004e store notrap aligned region15 v176, v178+8 ; v176 = 0+;; @004e v87 = iconst.i32 5+;; @004e store notrap aligned region6 v87, v75+32 ; v87 = 5+;; @004e v90 = load.i64 notrap aligned region13 v34+;; @004e store notrap aligned region1 v90, v32+24+;; @004e v91 = load.i64 notrap aligned region10 v34+8+;; @004e store notrap aligned region7 v91, v32+72+;; @004e v92 = load.i64 notrap aligned region11 v34+16+;; @004e store notrap aligned region8 v92, v32+64+;; @004e v93 = load.i64 notrap aligned region12 v34+24+;; @004e store notrap aligned region9 v93, v32+80+;; v174 = iconst.i32 0+;; v175 = iconst.i64 120+;; v176 = iadd.i64 v75, v175 ; v175 = 120+;; @004e store notrap aligned region17 v174, v176 ; v174 = 0+;; @004e store notrap aligned region14 v174, v176+4 ; v174 = 0+;; v177 = iconst.i64 0+;; @004e store notrap aligned region15 v177, v176+8 ; v177 = 0+;; v178 = iconst.i64 144+;; v179 = iadd.i64 v75, v178 ; v178 = 144+;; @004e store notrap aligned region17 v174, v179 ; v174 = 0+;; @004e store notrap aligned region14 v174, v179+4 ; v174 = 0+;; @004e store notrap aligned region15 v177, v179+8 ; v177 = 0;; @004e try_call fn2(v0), sig2, block9, [ context v0 ];;;; block9:;; @004e trap user12;;;; block5:-;; @004e v107 = load.i64 notrap aligned region7 v31+72-;; @004e v108 = load.i64 notrap aligned region8 v31+64-;; @004e v109 = load.i64 notrap aligned region9 v31+80-;; @004e store notrap aligned region10 v107, v74+8-;; @004e store notrap aligned region11 v108, v74+16-;; @004e store notrap aligned region12 v109, v74+24-;; @004e v112 = load.i64 notrap aligned region13 v33-;; @004e store notrap aligned region1 v112, v31+24-;; @004e v113 = load.i64 notrap aligned region10 v33+8-;; @004e store notrap aligned region7 v113, v31+72-;; @004e v114 = load.i64 notrap aligned region11 v33+16-;; @004e store notrap aligned region8 v114, v31+64-;; @004e v115 = load.i64 notrap aligned region12 v33+24-;; @004e store notrap aligned region9 v115, v31+80-;; @004e v117 = load.i64 notrap aligned region2 v74+88+;; @004e v108 = load.i64 notrap aligned region7 v32+72+;; @004e v109 = load.i64 notrap aligned region8 v32+64+;; @004e v110 = load.i64 notrap aligned region9 v32+80+;; @004e store notrap aligned region10 v108, v75+8+;; @004e store notrap aligned region11 v109, v75+16+;; @004e store notrap aligned region12 v110, v75+24+;; @004e v113 = load.i64 notrap aligned region13 v34+;; @004e store notrap aligned region1 v113, v32+24+;; @004e v114 = load.i64 notrap aligned region10 v34+8+;; @004e store notrap aligned region7 v114, v32+72+;; @004e v115 = load.i64 notrap aligned region11 v34+16+;; @004e store notrap aligned region8 v115, v32+64+;; @004e v116 = load.i64 notrap aligned region12 v34+24+;; @004e store notrap aligned region9 v116, v32+80+;; @004e v118 = load.i64 notrap aligned region2 v75+88;; @004e jump block8;;;; block10 cold:;; @004e trap user12;;;; block11:-;; @004e v124 = iconst.i64 136-;; @004e v125 = iadd.i64 v74, v124 ; v124 = 136-;; @004e v126 = load.i64 notrap aligned region15 v125+8-;; v170 = iconst.i32 0-;; @004e store notrap aligned region17 v170, v125 ; v170 = 0-;; @004e v119 = uextend.i128 v117-;; v171 = iconst.i64 64-;; v172 = ishl v119, v171 ; v171 = 64-;; @004e v118 = uextend.i128 v74-;; @004e v123 = bor v172, v118-;; @004e jump block2(v123)+;; @004e v125 = iconst.i64 144+;; @004e v126 = iadd.i64 v75, v125 ; v125 = 144+;; @004e v127 = load.i64 notrap aligned region15 v126+8+;; v171 = iconst.i32 0+;; @004e store notrap aligned region17 v171, v126 ; v171 = 0+;; @004e v120 = uextend.i128 v118+;; v172 = iconst.i64 64+;; v173 = ishl v120, v172 ; v172 = 64+;; @004e v119 = uextend.i128 v75+;; @004e v124 = bor v173, v119+;; @004e jump block2(v124);;;; block8:-;; @004e v116 = ireduce.i32 v71-;; @004e br_table v116, block10, [block11]+;; @004e v117 = ireduce.i32 v72+;; @004e br_table v117, block10, [block11];;;; block4:-;; @004e v130 = load.i64 notrap aligned region13 v33-;; @004e store notrap aligned region1 v130, v31+24-;; @004e v131 = load.i64 notrap aligned region10 v33+8-;; @004e store notrap aligned region7 v131, v31+72-;; @004e v132 = load.i64 notrap aligned region11 v33+16-;; @004e store notrap aligned region8 v132, v31+64-;; @004e v133 = load.i64 notrap aligned region12 v33+24-;; @004e store notrap aligned region9 v133, v31+80-;; @004e v136 = iconst.i32 4-;; @004e store notrap aligned region6 v136, v74+32 ; v136 = 4-;; @004e v137 = iconst.i64 120-;; @004e v138 = iadd.i64 v74, v137 ; v137 = 120-;; @004e v139 = load.i64 notrap aligned region15 v138+8-;; v165 = iconst.i32 0-;; @004e store notrap aligned region17 v165, v138 ; v165 = 0-;; @004e store notrap aligned region14 v165, v138+4 ; v165 = 0-;; v166 = iconst.i64 0-;; @004e store notrap aligned region15 v166, v138+8 ; v166 = 0-;; v167 = uextend.i128 v166 ; v166 = 0-;; v168 = iconst.i64 64-;; v169 = ishl v167, v168 ; v168 = 64-;; @0040 v8 = bor v169, v167+;; @004e v131 = load.i64 notrap aligned region13 v34+;; @004e store notrap aligned region1 v131, v32+24+;; @004e v132 = load.i64 notrap aligned region10 v34+8+;; @004e store notrap aligned region7 v132, v32+72+;; @004e v133 = load.i64 notrap aligned region11 v34+16+;; @004e store notrap aligned region8 v133, v32+64+;; @004e v134 = load.i64 notrap aligned region12 v34+24+;; @004e store notrap aligned region9 v134, v32+80+;; @004e v137 = iconst.i32 4+;; @004e store notrap aligned region6 v137, v75+32 ; v137 = 4+;; @004e v138 = iconst.i64 120+;; @004e v139 = iadd.i64 v75, v138 ; v138 = 120+;; @004e v140 = load.i64 notrap aligned region15 v139+8+;; v166 = iconst.i32 0+;; @004e store notrap aligned region17 v166, v139 ; v166 = 0+;; @004e store notrap aligned region14 v166, v139+4 ; v166 = 0+;; v167 = iconst.i64 0+;; @004e store notrap aligned region15 v167, v139+8 ; v167 = 0+;; v168 = uextend.i128 v167 ; v167 = 0+;; v169 = iconst.i64 64+;; v170 = ishl v168, v169 ; v169 = 64+;; @0040 v8 = bor v170, v168;; @0056 jump block2(v8);;-;; block2(v150: i128):+;; block2(v151: i128):;; @0058 jump block1;;;; block1:diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.watindex 8cc5e60c08..2318571bda 100644--- a/tests/disas/stack-switching/symmetric-switch.wat+++ b/tests/disas/stack-switching/symmetric-switch.wat@@ -52,7 +52,7 @@;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8;; gv2 = load.i64 notrap aligned region1 gv1+24;; sig0 = (i64 vmctx, i32) -> i64 tail-;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail+;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail;; sig2 = (i64 vmctx, i32) -> i8 tail;; fn0 = colocated u805306368:6 sig0;; fn1 = colocated u805306368:42 sig1@@ -65,215 +65,216 @@;; @003c trapz v3, user16;; @003c v4 = iconst.i32 1;; @003c v5 = iconst.i32 0-;; @003c v6 = call fn1(v0, v3, v4, v5) ; v4 = 1, v5 = 0-;; @003c v7 = load.i64 notrap aligned region2 v6+88-;; @003c v8 = uextend.i128 v6+;; @003c v6 = iconst.i32 0+;; @003c v7 = call fn1(v0, v3, v4, v5, v6) ; v4 = 1, v5 = 0, v6 = 0+;; @003c v8 = load.i64 notrap aligned region2 v7+88;; @003c v9 = uextend.i128 v7-;; @003c v10 = iconst.i64 64-;; @003c v11 = uextend.i128 v10 ; v10 = 64-;; @003c v12 = ishl v9, v11-;; @003c v13 = bor v12, v8-;; @003e v14 = ireduce.i64 v13-;; @003e v15 = iconst.i64 64-;; @003e v16 = uextend.i128 v15 ; v15 = 64-;; @003e v17 = ushr v13, v16-;; @003e v18 = ireduce.i64 v17-;; @003e trapz v14, user16-;; @003e v19 = load.i64 notrap aligned region2 v14+88-;; @003e v20 = icmp eq v19, v18-;; @003e trapz v20, user23-;; @003e v21 = iconst.i64 1-;; @003e v22 = iadd v19, v21 ; v21 = 1-;; @003e store notrap aligned region2 v22, v14+88-;; @003e v23 = iconst.i64 48-;; @003e v24 = iadd v0, v23 ; v23 = 48-;; @003e v25 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @003e v26 = load.i64 notrap aligned region3 v25+88-;; @003e v27 = load.i64 notrap aligned region3 v25+96-;; @003e jump block2(v26, v27)+;; @003c v10 = uextend.i128 v8+;; @003c v11 = iconst.i64 64+;; @003c v12 = uextend.i128 v11 ; v11 = 64+;; @003c v13 = ishl v10, v12+;; @003c v14 = bor v13, v9+;; @003e v15 = ireduce.i64 v14+;; @003e v16 = iconst.i64 64+;; @003e v17 = uextend.i128 v16 ; v16 = 64+;; @003e v18 = ushr v14, v17+;; @003e v19 = ireduce.i64 v18+;; @003e trapz v15, user16+;; @003e v20 = load.i64 notrap aligned region2 v15+88+;; @003e v21 = icmp eq v20, v19+;; @003e trapz v21, user23+;; @003e v22 = iconst.i64 1+;; @003e v23 = iadd v20, v22 ; v22 = 1+;; @003e store notrap aligned region2 v23, v15+88+;; @003e v24 = iconst.i64 48+;; @003e v25 = iadd v0, v24 ; v24 = 48+;; @003e v26 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @003e v27 = load.i64 notrap aligned region3 v26+88+;; @003e v28 = load.i64 notrap aligned region3 v26+96+;; @003e jump block2(v27, v28);;-;; block2(v28: i64, v29: i64):-;; @003e v30 = iconst.i64 1-;; @003e v31 = icmp eq v28, v30 ; v30 = 1-;; @003e brif v31, block7, block3+;; block2(v29: i64, v30: i64):+;; @003e v31 = iconst.i64 1+;; @003e v32 = icmp eq v29, v31 ; v31 = 1+;; @003e brif v32, block7, block3;;;; block3:-;; @003e v32 = load.i64 notrap aligned region4 v29+64-;; @003e v33 = load.i64 notrap aligned region4 v29+72-;; @003e v34 = iconst.i64 40-;; @003e v35 = iadd v33, v34 ; v34 = 40-;; @003e v36 = load.i64 notrap aligned region5 v35+8-;; @003e v37 = load.i32 notrap aligned region6 v33+56-;; @003e v38 = load.i32 notrap aligned region7 v35-;; @003e jump block4(v37)+;; @003e v33 = load.i64 notrap aligned region4 v30+64+;; @003e v34 = load.i64 notrap aligned region4 v30+72+;; @003e v35 = iconst.i64 40+;; @003e v36 = iadd v34, v35 ; v35 = 40+;; @003e v37 = load.i64 notrap aligned region5 v36+8+;; @003e v38 = load.i32 notrap aligned region6 v34+56+;; @003e v39 = load.i32 notrap aligned region7 v36+;; @003e jump block4(v38);;-;; block4(v39: i32):-;; @003e v40 = icmp ult v39, v38-;; @003e brif v40, block5, block2(v32, v33)+;; block4(v40: i32):+;; @003e v41 = icmp ult v40, v39+;; @003e brif v41, block5, block2(v33, v34);;;; block5:-;; @003e v41 = iconst.i32 8-;; @003e v42 = imul.i32 v39, v41 ; v41 = 8-;; @003e v43 = uextend.i64 v42-;; @003e v44 = iadd.i64 v36, v43-;; @003e v45 = load.i64 notrap aligned region8 v44-;; @003e v46 = icmp eq v45, v24-;; @003e v47 = iconst.i32 1-;; @003e v48 = iadd.i32 v39, v47 ; v47 = 1-;; @003e brif v46, block6, block4(v48)+;; @003e v42 = iconst.i32 8+;; @003e v43 = imul.i32 v40, v42 ; v42 = 8+;; @003e v44 = uextend.i64 v43+;; @003e v45 = iadd.i64 v37, v44+;; @003e v46 = load.i64 notrap aligned region8 v45+;; @003e v47 = icmp eq v46, v25+;; @003e v48 = iconst.i32 1+;; @003e v49 = iadd.i32 v40, v48 ; v48 = 1+;; @003e brif v47, block6, block4(v49);;;; block7 cold:;; @003e trap user22;;;; block6:-;; @003e store.i64 notrap aligned region9 v29, v27+80-;; @003e v49 = iconst.i64 136-;; @003e v50 = iadd.i64 v27, v49 ; v49 = 136-;; @003e v51 = iconst.i32 1-;; @003e v52 = stack_addr.i64 ss0-;; @003e store notrap aligned region10 v51, v50+4 ; v51 = 1-;; @003e store notrap aligned region5 v52, v50+8-;; @003e v53 = iconst.i64 0-;; @003e v54 = iadd.i64 v27, v53 ; v53 = 0-;; @003e v55 = iconst.i32 3-;; @003e store notrap aligned region11 v55, v54+32 ; v55 = 3-;; @003e v56 = iconst.i64 0+;; @003e store.i64 notrap aligned region9 v30, v28+80+;; @003e v50 = iconst.i64 144+;; @003e v51 = iadd.i64 v28, v50 ; v50 = 144+;; @003e v52 = iconst.i32 1+;; @003e v53 = stack_addr.i64 ss0+;; @003e store notrap aligned region10 v52, v51+4 ; v52 = 1+;; @003e store notrap aligned region5 v53, v51+8+;; @003e v54 = iconst.i64 0+;; @003e v55 = iadd.i64 v28, v54 ; v54 = 0+;; @003e v56 = iconst.i32 3+;; @003e store notrap aligned region11 v56, v55+32 ; v56 = 3;; @003e v57 = iconst.i64 0-;; @003e store notrap aligned region4 v56, v29+64 ; v56 = 0-;; @003e store notrap aligned region4 v57, v29+72 ; v57 = 0-;; @003e v58 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @003e v59 = iconst.i64 0-;; @003e v60 = iadd v54, v59 ; v59 = 0-;; @003e v61 = load.i64 notrap aligned region12 v58+72-;; @003e v62 = load.i64 notrap aligned region13 v58+64-;; @003e v63 = load.i64 notrap aligned region14 v58+80-;; @003e store notrap aligned region15 v61, v60+8-;; @003e store notrap aligned region16 v62, v60+16-;; @003e store notrap aligned region6 v63, v60+24-;; @003e v64 = load.i64 notrap aligned region2 v27+88-;; @003e v65 = uextend.i128 v27-;; @003e v66 = uextend.i128 v64-;; @003e v67 = iconst.i64 64-;; @003e v68 = uextend.i128 v67 ; v67 = 64-;; @003e v69 = ishl v66, v68-;; @003e v70 = bor v69, v65-;; @003e v72 = iconst.i64 0-;; @003e v73 = iadd.i64 v14, v72 ; v72 = 0-;; @003e v74 = load.i32 notrap aligned region11 v73+32-;; @003e v75 = iconst.i32 0-;; @003e v76 = icmp ne v74, v75 ; v75 = 0-;; @003e brif v76, block9, block8+;; @003e v58 = iconst.i64 0+;; @003e store notrap aligned region4 v57, v30+64 ; v57 = 0+;; @003e store notrap aligned region4 v58, v30+72 ; v58 = 0+;; @003e v59 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @003e v60 = iconst.i64 0+;; @003e v61 = iadd v55, v60 ; v60 = 0+;; @003e v62 = load.i64 notrap aligned region12 v59+72+;; @003e v63 = load.i64 notrap aligned region13 v59+64+;; @003e v64 = load.i64 notrap aligned region14 v59+80+;; @003e store notrap aligned region15 v62, v61+8+;; @003e store notrap aligned region16 v63, v61+16+;; @003e store notrap aligned region6 v64, v61+24+;; @003e v65 = load.i64 notrap aligned region2 v28+88+;; @003e v66 = uextend.i128 v28+;; @003e v67 = uextend.i128 v65+;; @003e v68 = iconst.i64 64+;; @003e v69 = uextend.i128 v68 ; v68 = 64+;; @003e v70 = ishl v67, v69+;; @003e v71 = bor v70, v66+;; @003e v73 = iconst.i64 0+;; @003e v74 = iadd.i64 v15, v73 ; v73 = 0+;; @003e v75 = load.i32 notrap aligned region11 v74+32+;; @003e v76 = iconst.i32 0+;; @003e v77 = icmp ne v75, v76 ; v76 = 0+;; @003e brif v77, block9, block8;;;; block8:-;; @003e v77 = iconst.i64 120-;; @003e v78 = iadd.i64 v14, v77 ; v77 = 120-;; @003e v79 = load.i64 notrap aligned region5 v78+8-;; @003e v80 = load.i32 notrap aligned region7 v78-;; @003e v81 = iconst.i32 1-;; @003e v82 = iadd v80, v81 ; v81 = 1-;; @003e store notrap aligned region7 v82, v78-;; @003e v83 = uextend.i64 v80-;; @003e v84 = iconst.i64 16-;; @003e v85 = imul v83, v84 ; v84 = 16-;; @003e v86 = iadd v79, v85-;; @003e jump block10(v86)+;; @003e v78 = iconst.i64 120+;; @003e v79 = iadd.i64 v15, v78 ; v78 = 120+;; @003e v80 = load.i64 notrap aligned region5 v79+8+;; @003e v81 = load.i32 notrap aligned region7 v79+;; @003e v82 = iconst.i32 1+;; @003e v83 = iadd v81, v82 ; v82 = 1+;; @003e store notrap aligned region7 v83, v79+;; @003e v84 = uextend.i64 v81+;; @003e v85 = iconst.i64 16+;; @003e v86 = imul v84, v85 ; v85 = 16+;; @003e v87 = iadd v80, v86+;; @003e jump block10(v87);;;; block9:-;; @003e v87 = iconst.i64 136-;; @003e v88 = iadd.i64 v14, v87 ; v87 = 136-;; @003e v89 = load.i64 notrap aligned region5 v88+8-;; @003e v90 = load.i32 notrap aligned region7 v88-;; @003e v91 = iconst.i32 1-;; @003e v92 = iadd v90, v91 ; v91 = 1-;; @003e store notrap aligned region7 v92, v88-;; @003e v93 = uextend.i64 v90-;; @003e v94 = iconst.i64 16-;; @003e v95 = imul v93, v94 ; v94 = 16-;; @003e v96 = iadd v89, v95-;; @003e jump block10(v96)+;; @003e v88 = iconst.i64 144+;; @003e v89 = iadd.i64 v15, v88 ; v88 = 144+;; @003e v90 = load.i64 notrap aligned region5 v89+8+;; @003e v91 = load.i32 notrap aligned region7 v89+;; @003e v92 = iconst.i32 1+;; @003e v93 = iadd v91, v92 ; v92 = 1+;; @003e store notrap aligned region7 v93, v89+;; @003e v94 = uextend.i64 v91+;; @003e v95 = iconst.i64 16+;; @003e v96 = imul v94, v95 ; v95 = 16+;; @003e v97 = iadd v90, v96+;; @003e jump block10(v97);;-;; block10(v71: i64):-;; @003e store.i128 notrap aligned region8 v70, v71-;; @003e v97 = iconst.i64 0-;; @003e v98 = iadd.i64 v14, v97 ; v97 = 0-;; @003e v99 = iconst.i32 1-;; @003e store notrap aligned region11 v99, v98+32 ; v99 = 1-;; @003e v100 = load.i64 notrap aligned region9 v14+80-;; @003e store.i64 notrap aligned region4 v32, v100+64-;; @003e store.i64 notrap aligned region4 v33, v100+72-;; @003e v101 = iconst.i64 2-;; @003e v102 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @003e store notrap aligned region3 v101, v102+88 ; v101 = 2-;; @003e store.i64 notrap aligned region3 v14, v102+96-;; @003e v103 = iconst.i64 0-;; @003e v104 = iadd v98, v103 ; v103 = 0-;; @003e v105 = load.i64 notrap aligned region17 v104-;; @003e store notrap aligned region1 v105, v58+24-;; @003e v106 = load.i64 notrap aligned region15 v104+8-;; @003e store notrap aligned region12 v106, v58+72-;; @003e v107 = load.i64 notrap aligned region16 v104+16-;; @003e store notrap aligned region13 v107, v58+64-;; @003e v108 = load.i64 notrap aligned region6 v104+24-;; @003e store notrap aligned region14 v108, v58+80-;; @003e v109 = iconst.i64 96-;; @003e v110 = iadd.i64 v29, v109 ; v109 = 96-;; @003e v111 = load.i64 notrap aligned region18 v110-;; @003e v112 = iconst.i64 -24-;; @003e v113 = iadd v111, v112 ; v112 = -24-;; @003e v114 = iconst.i64 96-;; @003e v115 = iadd v100, v114 ; v114 = 96-;; @003e v116 = load.i64 notrap aligned region18 v115-;; @003e v117 = iconst.i64 -24-;; @003e v118 = iadd v116, v117 ; v117 = -24-;; @003e v119 = stack_addr.i64 ss1-;; @003e v120 = load.i64 notrap aligned region8 v118-;; @003e store notrap aligned region19 v120, v119-;; @003e v121 = load.i64 notrap aligned region8 v113-;; @003e store notrap aligned region8 v121, v118-;; @003e v122 = load.i64 notrap aligned region8 v118+8-;; @003e store notrap aligned region19 v122, v119+8-;; @003e v123 = load.i64 notrap aligned region8 v113+8-;; @003e store notrap aligned region8 v123, v118+8-;; @003e v124 = load.i64 notrap aligned region8 v118+16-;; @003e store notrap aligned region19 v124, v119+16-;; @003e v125 = load.i64 notrap aligned region8 v113+16-;; @003e store notrap aligned region8 v125, v118+16-;; @003e v126 = iconst.i64 3-;; @003e v127 = iconst.i64 32-;; @003e v128 = ishl v126, v127 ; v126 = 3, v127 = 32-;; @003e v129 = stack_switch v113, v119, v128-;; @003e v130 = iconst.i64 32-;; @003e v131 = ushr v129, v130 ; v130 = 32-;; @003e v132 = iconst.i64 5-;; @003e v133 = icmp eq v131, v132 ; v132 = 5-;; @003e brif v133, block11, block12+;; block10(v72: i64):+;; @003e store.i128 notrap aligned region8 v71, v72+;; @003e v98 = iconst.i64 0+;; @003e v99 = iadd.i64 v15, v98 ; v98 = 0+;; @003e v100 = iconst.i32 1+;; @003e store notrap aligned region11 v100, v99+32 ; v100 = 1+;; @003e v101 = load.i64 notrap aligned region9 v15+80+;; @003e store.i64 notrap aligned region4 v33, v101+64+;; @003e store.i64 notrap aligned region4 v34, v101+72+;; @003e v102 = iconst.i64 2+;; @003e v103 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @003e store notrap aligned region3 v102, v103+88 ; v102 = 2+;; @003e store.i64 notrap aligned region3 v15, v103+96+;; @003e v104 = iconst.i64 0+;; @003e v105 = iadd v99, v104 ; v104 = 0+;; @003e v106 = load.i64 notrap aligned region17 v105+;; @003e store notrap aligned region1 v106, v59+24+;; @003e v107 = load.i64 notrap aligned region15 v105+8+;; @003e store notrap aligned region12 v107, v59+72+;; @003e v108 = load.i64 notrap aligned region16 v105+16+;; @003e store notrap aligned region13 v108, v59+64+;; @003e v109 = load.i64 notrap aligned region6 v105+24+;; @003e store notrap aligned region14 v109, v59+80+;; @003e v110 = iconst.i64 96+;; @003e v111 = iadd.i64 v30, v110 ; v110 = 96+;; @003e v112 = load.i64 notrap aligned region18 v111+;; @003e v113 = iconst.i64 -24+;; @003e v114 = iadd v112, v113 ; v113 = -24+;; @003e v115 = iconst.i64 96+;; @003e v116 = iadd v101, v115 ; v115 = 96+;; @003e v117 = load.i64 notrap aligned region18 v116+;; @003e v118 = iconst.i64 -24+;; @003e v119 = iadd v117, v118 ; v118 = -24+;; @003e v120 = stack_addr.i64 ss1+;; @003e v121 = load.i64 notrap aligned region8 v119+;; @003e store notrap aligned region19 v121, v120+;; @003e v122 = load.i64 notrap aligned region8 v114+;; @003e store notrap aligned region8 v122, v119+;; @003e v123 = load.i64 notrap aligned region8 v119+8+;; @003e store notrap aligned region19 v123, v120+8+;; @003e v124 = load.i64 notrap aligned region8 v114+8+;; @003e store notrap aligned region8 v124, v119+8+;; @003e v125 = load.i64 notrap aligned region8 v119+16+;; @003e store notrap aligned region19 v125, v120+16+;; @003e v126 = load.i64 notrap aligned region8 v114+16+;; @003e store notrap aligned region8 v126, v119+16+;; @003e v127 = iconst.i64 3+;; @003e v128 = iconst.i64 32+;; @003e v129 = ishl v127, v128 ; v127 = 3, v128 = 32+;; @003e v130 = stack_switch v114, v120, v129+;; @003e v131 = iconst.i64 32+;; @003e v132 = ushr v130, v131 ; v131 = 32+;; @003e v133 = iconst.i64 5+;; @003e v134 = icmp eq v132, v133 ; v133 = 5+;; @003e brif v134, block11, block12;;;; block11 cold:-;; @003e v134 = iconst.i64 136-;; @003e v135 = iadd.i64 v27, v134 ; v134 = 136-;; @003e v136 = load.i64 notrap aligned region5 v135+8-;; @003e v137 = load.i32 notrap aligned region8 v136-;; @003e v138 = iconst.i32 0-;; @003e store notrap aligned region7 v138, v135 ; v138 = 0+;; @003e v135 = iconst.i64 144+;; @003e v136 = iadd.i64 v28, v135 ; v135 = 144+;; @003e v137 = load.i64 notrap aligned region5 v136+8+;; @003e v138 = load.i32 notrap aligned region8 v137;; @003e v139 = iconst.i32 0-;; @003e store notrap aligned region10 v139, v135+4 ; v139 = 0-;; @003e v140 = iconst.i64 0-;; @003e store notrap aligned region5 v140, v135+8 ; v140 = 0-;; @003e try_call fn2(v0, v137), sig2, block13, [ context v0 ]+;; @003e store notrap aligned region7 v139, v136 ; v139 = 0+;; @003e v140 = iconst.i32 0+;; @003e store notrap aligned region10 v140, v136+4 ; v140 = 0+;; @003e v141 = iconst.i64 0+;; @003e store notrap aligned region5 v141, v136+8 ; v141 = 0+;; @003e try_call fn2(v0, v138), sig2, block13, [ context v0 ];;;; block13:;; @003e trap user12;;;; block12:-;; @003e v141 = iconst.i64 136-;; @003e v142 = iadd.i64 v27, v141 ; v141 = 136-;; @003e v143 = load.i64 notrap aligned region5 v142+8-;; @003e v144 = iconst.i32 0-;; @003e store notrap aligned region7 v144, v142 ; v144 = 0+;; @003e v142 = iconst.i64 144+;; @003e v143 = iadd.i64 v28, v142 ; v142 = 144+;; @003e v144 = load.i64 notrap aligned region5 v143+8;; @003e v145 = iconst.i32 0-;; @003e store notrap aligned region10 v145, v142+4 ; v145 = 0-;; @003e v146 = iconst.i64 0-;; @003e store notrap aligned region5 v146, v142+8 ; v146 = 0+;; @003e store notrap aligned region7 v145, v143 ; v145 = 0+;; @003e v146 = iconst.i32 0+;; @003e store notrap aligned region10 v146, v143+4 ; v146 = 0+;; @003e v147 = iconst.i64 0+;; @003e store notrap aligned region5 v147, v143+8 ; v147 = 0;; @0041 jump block1;;;; block1:@@ -320,7 +321,7 @@;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8;; gv2 = load.i64 notrap aligned region1 gv1+24;; sig0 = (i64 vmctx, i32) -> i64 tail-;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail+;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail;; sig2 = (i64 vmctx) tail;; fn0 = colocated u805306368:6 sig0;; fn1 = colocated u805306368:42 sig1@@ -333,212 +334,213 @@;; @0049 trapz v3, user16;; @0049 v4 = iconst.i32 0;; @0049 v5 = iconst.i32 0-;; @0049 v6 = call fn1(v0, v3, v4, v5) ; v4 = 0, v5 = 0-;; @0049 v7 = load.i64 notrap aligned region2 v6+88-;; @0049 v8 = uextend.i128 v6+;; @0049 v6 = iconst.i32 0+;; @0049 v7 = call fn1(v0, v3, v4, v5, v6) ; v4 = 0, v5 = 0, v6 = 0+;; @0049 v8 = load.i64 notrap aligned region2 v7+88;; @0049 v9 = uextend.i128 v7-;; @0049 v10 = iconst.i64 64-;; @0049 v11 = uextend.i128 v10 ; v10 = 64-;; @0049 v12 = ishl v9, v11-;; @0049 v13 = bor v12, v8+;; @0049 v10 = uextend.i128 v8+;; @0049 v11 = iconst.i64 64+;; @0049 v12 = uextend.i128 v11 ; v11 = 64+;; @0049 v13 = ishl v10, v12+;; @0049 v14 = bor v13, v9;; @004b jump block2;;;; block2:-;; @004b v14 = ireduce.i64 v13-;; @004b v15 = iconst.i64 64-;; @004b v16 = uextend.i128 v15 ; v15 = 64-;; @004b v17 = ushr.i128 v13, v16-;; @004b v18 = ireduce.i64 v17-;; @004b trapz v14, user16-;; @004b v19 = load.i64 notrap aligned region2 v14+88-;; @004b v20 = icmp eq v19, v18-;; @004b trapz v20, user23-;; @004b v21 = iconst.i64 1-;; @004b v22 = iadd v19, v21 ; v21 = 1-;; @004b store notrap aligned region2 v22, v14+88-;; @004b v23 = load.i64 notrap aligned region3 v14+80-;; @004b v24 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @004b v25 = load.i64 notrap aligned region4 v24+88-;; @004b v26 = load.i64 notrap aligned region4 v24+96-;; @004b store notrap aligned region5 v25, v23+64-;; @004b store notrap aligned region5 v26, v23+72-;; @004b v27 = iconst.i64 0-;; @004b store notrap aligned region3 v27, v14+80 ; v27 = 0-;; @004b v28 = iconst.i64 2-;; @004b v29 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @004b store notrap aligned region4 v28, v29+88 ; v28 = 2-;; @004b store notrap aligned region4 v14, v29+96-;; @004b v30 = iconst.i64 0-;; @004b v31 = iadd v14, v30 ; v30 = 0-;; @004b v32 = iconst.i32 1-;; @004b store notrap aligned region6 v32, v31+32 ; v32 = 1-;; @004b v33 = iconst.i32 2-;; @004b store notrap aligned region6 v33, v26+32 ; v33 = 2-;; @004b v34 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @004b v35 = iconst.i64 0-;; @004b v36 = iadd v26, v35 ; v35 = 0-;; @004b v37 = load.i64 notrap aligned region7 v34+72-;; @004b v38 = load.i64 notrap aligned region8 v34+64-;; @004b v39 = load.i64 notrap aligned region9 v34+80-;; @004b store notrap aligned region10 v37, v36+8-;; @004b store notrap aligned region11 v38, v36+16-;; @004b store notrap aligned region12 v39, v36+24-;; @004b v40 = load.i64 notrap aligned region1 v34+24-;; @004b store notrap aligned region13 v40, v36-;; @004b v41 = iconst.i64 0-;; @004b v42 = iadd v31, v41 ; v41 = 0-;; @004b v43 = load.i64 notrap aligned region13 v42-;; @004b store notrap aligned region1 v43, v34+24-;; @004b v44 = load.i64 notrap aligned region10 v42+8-;; @004b store notrap aligned region7 v44, v34+72-;; @004b v45 = load.i64 notrap aligned region11 v42+16-;; @004b store notrap aligned region8 v45, v34+64-;; @004b v46 = load.i64 notrap aligned region12 v42+24-;; @004b store notrap aligned region9 v46, v34+80-;; @004b v47 = iconst.i64 40-;; @004b v48 = iadd v26, v47 ; v47 = 40-;; @004b v49 = iconst.i32 1-;; @004b v50 = stack_addr.i64 ss0-;; @004b store notrap aligned region14 v49, v48+4 ; v49 = 1-;; @004b store notrap aligned region15 v50, v48+8-;; @004b v51 = iconst.i64 48-;; @004b v52 = iadd.i64 v0, v51 ; v51 = 48-;; @004b v53 = iconst.i32 1-;; @004b v54 = load.i64 notrap aligned region15 v48+8-;; @004b store notrap aligned region16 v52, v54-;; @004b store notrap aligned region17 v53, v48 ; v53 = 1-;; @004b v55 = iconst.i32 0-;; @004b store notrap aligned region12 v55, v26+56 ; v55 = 0-;; @004b v56 = iconst.i64 1-;; @004b v57 = iconst.i64 32-;; @004b v58 = ishl v56, v57 ; v56 = 1, v57 = 32-;; @004b v59 = iconst.i64 96-;; @004b v60 = iadd v23, v59 ; v59 = 96-;; @004b v61 = load.i64 notrap aligned region18 v60-;; @004b v62 = iconst.i64 -24-;; @004b v63 = iadd v61, v62 ; v62 = -24-;; @004b v64 = stack_switch v63, v63, v58-;; @004b v65 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @004b v66 = load.i64 notrap aligned region4 v65+88-;; @004b v67 = load.i64 notrap aligned region4 v65+96-;; @004b v68 = load.i64 notrap aligned readonly can_move region0 v0+8-;; @004b store notrap aligned region4 v25, v68+88-;; @004b store notrap aligned region4 v26, v68+96-;; @004b v69 = iconst.i32 1-;; @004b store notrap aligned region6 v69, v26+32 ; v69 = 1-;; @004b v70 = iconst.i32 0-;; @004b store notrap aligned region17 v70, v48 ; v70 = 0+;; @004b v15 = ireduce.i64 v14+;; @004b v16 = iconst.i64 64+;; @004b v17 = uextend.i128 v16 ; v16 = 64+;; @004b v18 = ushr.i128 v14, v17+;; @004b v19 = ireduce.i64 v18+;; @004b trapz v15, user16+;; @004b v20 = load.i64 notrap aligned region2 v15+88+;; @004b v21 = icmp eq v20, v19+;; @004b trapz v21, user23+;; @004b v22 = iconst.i64 1+;; @004b v23 = iadd v20, v22 ; v22 = 1+;; @004b store notrap aligned region2 v23, v15+88+;; @004b v24 = load.i64 notrap aligned region3 v15+80+;; @004b v25 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @004b v26 = load.i64 notrap aligned region4 v25+88+;; @004b v27 = load.i64 notrap aligned region4 v25+96+;; @004b store notrap aligned region5 v26, v24+64+;; @004b store notrap aligned region5 v27, v24+72+;; @004b v28 = iconst.i64 0+;; @004b store notrap aligned region3 v28, v15+80 ; v28 = 0+;; @004b v29 = iconst.i64 2+;; @004b v30 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @004b store notrap aligned region4 v29, v30+88 ; v29 = 2+;; @004b store notrap aligned region4 v15, v30+96+;; @004b v31 = iconst.i64 0+;; @004b v32 = iadd v15, v31 ; v31 = 0+;; @004b v33 = iconst.i32 1+;; @004b store notrap aligned region6 v33, v32+32 ; v33 = 1+;; @004b v34 = iconst.i32 2+;; @004b store notrap aligned region6 v34, v27+32 ; v34 = 2+;; @004b v35 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @004b v36 = iconst.i64 0+;; @004b v37 = iadd v27, v36 ; v36 = 0+;; @004b v38 = load.i64 notrap aligned region7 v35+72+;; @004b v39 = load.i64 notrap aligned region8 v35+64+;; @004b v40 = load.i64 notrap aligned region9 v35+80+;; @004b store notrap aligned region10 v38, v37+8+;; @004b store notrap aligned region11 v39, v37+16+;; @004b store notrap aligned region12 v40, v37+24+;; @004b v41 = load.i64 notrap aligned region1 v35+24+;; @004b store notrap aligned region13 v41, v37+;; @004b v42 = iconst.i64 0+;; @004b v43 = iadd v32, v42 ; v42 = 0+;; @004b v44 = load.i64 notrap aligned region13 v43+;; @004b store notrap aligned region1 v44, v35+24+;; @004b v45 = load.i64 notrap aligned region10 v43+8+;; @004b store notrap aligned region7 v45, v35+72+;; @004b v46 = load.i64 notrap aligned region11 v43+16+;; @004b store notrap aligned region8 v46, v35+64+;; @004b v47 = load.i64 notrap aligned region12 v43+24+;; @004b store notrap aligned region9 v47, v35+80+;; @004b v48 = iconst.i64 40+;; @004b v49 = iadd v27, v48 ; v48 = 40+;; @004b v50 = iconst.i32 1+;; @004b v51 = stack_addr.i64 ss0+;; @004b store notrap aligned region14 v50, v49+4 ; v50 = 1+;; @004b store notrap aligned region15 v51, v49+8+;; @004b v52 = iconst.i64 48+;; @004b v53 = iadd.i64 v0, v52 ; v52 = 48+;; @004b v54 = iconst.i32 1+;; @004b v55 = load.i64 notrap aligned region15 v49+8+;; @004b store notrap aligned region16 v53, v55+;; @004b store notrap aligned region17 v54, v49 ; v54 = 1+;; @004b v56 = iconst.i32 0+;; @004b store notrap aligned region12 v56, v27+56 ; v56 = 0+;; @004b v57 = iconst.i64 1+;; @004b v58 = iconst.i64 32+;; @004b v59 = ishl v57, v58 ; v57 = 1, v58 = 32+;; @004b v60 = iconst.i64 96+;; @004b v61 = iadd v24, v60 ; v60 = 96+;; @004b v62 = load.i64 notrap aligned region18 v61+;; @004b v63 = iconst.i64 -24+;; @004b v64 = iadd v62, v63 ; v63 = -24+;; @004b v65 = stack_switch v64, v64, v59+;; @004b v66 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @004b v67 = load.i64 notrap aligned region4 v66+88+;; @004b v68 = load.i64 notrap aligned region4 v66+96+;; @004b v69 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @004b store notrap aligned region4 v26, v69+88+;; @004b store notrap aligned region4 v27, v69+96+;; @004b v70 = iconst.i32 1+;; @004b store notrap aligned region6 v70, v27+32 ; v70 = 1;; @004b v71 = iconst.i32 0-;; @004b store notrap aligned region14 v71, v48+4 ; v71 = 0-;; @004b v72 = iconst.i64 0-;; @004b store notrap aligned region15 v72, v48+8 ; v72 = 0-;; @004b store notrap aligned region12 v27, v26+56 ; v27 = 0-;; @004b brif v64, block6, block3+;; @004b store notrap aligned region17 v71, v49 ; v71 = 0+;; @004b v72 = iconst.i32 0+;; @004b store notrap aligned region14 v72, v49+4 ; v72 = 0+;; @004b v73 = iconst.i64 0+;; @004b store notrap aligned region15 v73, v49+8 ; v73 = 0+;; @004b store notrap aligned region12 v28, v27+56 ; v28 = 0+;; @004b brif v65, block6, block3;;;; block6:-;; @004b v73 = iconst.i64 32-;; @004b v74 = ushr.i64 v64, v73 ; v73 = 32-;; @004b v75 = iconst.i64 4-;; @004b v76 = icmp eq v74, v75 ; v75 = 4-;; @004b brif v76, block5, block4+;; @004b v74 = iconst.i64 32+;; @004b v75 = ushr.i64 v65, v74 ; v74 = 32+;; @004b v76 = iconst.i64 4+;; @004b v77 = icmp eq v75, v76 ; v76 = 4+;; @004b brif v77, block5, block4;;;; block5 cold:-;; @004b v77 = iconst.i64 0-;; @004b v78 = iadd.i64 v67, v77 ; v77 = 0-;; @004b v79 = iconst.i32 5-;; @004b store notrap aligned region6 v79, v78+32 ; v79 = 5-;; @004b v80 = iconst.i64 0-;; @004b v81 = iadd.i64 v26, v80 ; v80 = 0-;; @004b v82 = load.i64 notrap aligned region13 v81-;; @004b store notrap aligned region1 v82, v34+24-;; @004b v83 = load.i64 notrap aligned region10 v81+8-;; @004b store notrap aligned region7 v83, v34+72-;; @004b v84 = load.i64 notrap aligned region11 v81+16-;; @004b store notrap aligned region8 v84, v34+64-;; @004b v85 = load.i64 notrap aligned region12 v81+24-;; @004b store notrap aligned region9 v85, v34+80-;; @004b v86 = iconst.i64 120-;; @004b v87 = iadd.i64 v67, v86 ; v86 = 120-;; @004b v88 = iconst.i32 0-;; @004b store notrap aligned region17 v88, v87 ; v88 = 0+;; @004b v78 = iconst.i64 0+;; @004b v79 = iadd.i64 v68, v78 ; v78 = 0+;; @004b v80 = iconst.i32 5+;; @004b store notrap aligned region6 v80, v79+32 ; v80 = 5+;; @004b v81 = iconst.i64 0+;; @004b v82 = iadd.i64 v27, v81 ; v81 = 0+;; @004b v83 = load.i64 notrap aligned region13 v82+;; @004b store notrap aligned region1 v83, v35+24+;; @004b v84 = load.i64 notrap aligned region10 v82+8+;; @004b store notrap aligned region7 v84, v35+72+;; @004b v85 = load.i64 notrap aligned region11 v82+16+;; @004b store notrap aligned region8 v85, v35+64+;; @004b v86 = load.i64 notrap aligned region12 v82+24+;; @004b store notrap aligned region9 v86, v35+80+;; @004b v87 = iconst.i64 120+;; @004b v88 = iadd.i64 v68, v87 ; v87 = 120;; @004b v89 = iconst.i32 0-;; @004b store notrap aligned region14 v89, v87+4 ; v89 = 0-;; @004b v90 = iconst.i64 0-;; @004b store notrap aligned region15 v90, v87+8 ; v90 = 0-;; @004b v91 = iconst.i64 136-;; @004b v92 = iadd.i64 v67, v91 ; v91 = 136-;; @004b v93 = iconst.i32 0-;; @004b store notrap aligned region17 v93, v92 ; v93 = 0+;; @004b store notrap aligned region17 v89, v88 ; v89 = 0+;; @004b v90 = iconst.i32 0+;; @004b store notrap aligned region14 v90, v88+4 ; v90 = 0+;; @004b v91 = iconst.i64 0+;; @004b store notrap aligned region15 v91, v88+8 ; v91 = 0+;; @004b v92 = iconst.i64 144+;; @004b v93 = iadd.i64 v68, v92 ; v92 = 144;; @004b v94 = iconst.i32 0-;; @004b store notrap aligned region14 v94, v92+4 ; v94 = 0-;; @004b v95 = iconst.i64 0-;; @004b store notrap aligned region15 v95, v92+8 ; v95 = 0+;; @004b store notrap aligned region17 v94, v93 ; v94 = 0+;; @004b v95 = iconst.i32 0+;; @004b store notrap aligned region14 v95, v93+4 ; v95 = 0+;; @004b v96 = iconst.i64 0+;; @004b store notrap aligned region15 v96, v93+8 ; v96 = 0;; @004b try_call fn2(v0), sig2, block8, [ context v0 ];;;; block8:;; @004b trap user12;;;; block4:-;; @004b v96 = iconst.i64 0-;; @004b v97 = iadd.i64 v67, v96 ; v96 = 0-;; @004b v98 = iconst.i64 0-;; @004b v99 = iadd v97, v98 ; v98 = 0-;; @004b v100 = load.i64 notrap aligned region7 v34+72-;; @004b v101 = load.i64 notrap aligned region8 v34+64-;; @004b v102 = load.i64 notrap aligned region9 v34+80-;; @004b store notrap aligned region10 v100, v99+8-;; @004b store notrap aligned region11 v101, v99+16-;; @004b store notrap aligned region12 v102, v99+24-;; @004b v103 = iconst.i64 0-;; @004b v104 = iadd.i64 v26, v103 ; v103 = 0-;; @004b v105 = load.i64 notrap aligned region13 v104-;; @004b store notrap aligned region1 v105, v34+24-;; @004b v106 = load.i64 notrap aligned region10 v104+8-;; @004b store notrap aligned region7 v106, v34+72-;; @004b v107 = load.i64 notrap aligned region11 v104+16-;; @004b store notrap aligned region8 v107, v34+64-;; @004b v108 = load.i64 notrap aligned region12 v104+24-;; @004b store notrap aligned region9 v108, v34+80-;; @004b v109 = ireduce.i32 v64-;; @004b v110 = load.i64 notrap aligned region2 v67+88-;; @004b v111 = uextend.i128 v67-;; @004b v112 = uextend.i128 v110-;; @004b v113 = iconst.i64 64-;; @004b v114 = uextend.i128 v113 ; v113 = 64-;; @004b v115 = ishl v112, v114-;; @004b v116 = bor v115, v111+;; @004b v97 = iconst.i64 0+;; @004b v98 = iadd.i64 v68, v97 ; v97 = 0+;; @004b v99 = iconst.i64 0+;; @004b v100 = iadd v98, v99 ; v99 = 0+;; @004b v101 = load.i64 notrap aligned region7 v35+72+;; @004b v102 = load.i64 notrap aligned region8 v35+64+;; @004b v103 = load.i64 notrap aligned region9 v35+80+;; @004b store notrap aligned region10 v101, v100+8+;; @004b store notrap aligned region11 v102, v100+16+;; @004b store notrap aligned region12 v103, v100+24+;; @004b v104 = iconst.i64 0+;; @004b v105 = iadd.i64 v27, v104 ; v104 = 0+;; @004b v106 = load.i64 notrap aligned region13 v105+;; @004b store notrap aligned region1 v106, v35+24+;; @004b v107 = load.i64 notrap aligned region10 v105+8+;; @004b store notrap aligned region7 v107, v35+72+;; @004b v108 = load.i64 notrap aligned region11 v105+16+;; @004b store notrap aligned region8 v108, v35+64+;; @004b v109 = load.i64 notrap aligned region12 v105+24+;; @004b store notrap aligned region9 v109, v35+80+;; @004b v110 = ireduce.i32 v65+;; @004b v111 = load.i64 notrap aligned region2 v68+88+;; @004b v112 = uextend.i128 v68+;; @004b v113 = uextend.i128 v111+;; @004b v114 = iconst.i64 64+;; @004b v115 = uextend.i128 v114 ; v114 = 64+;; @004b v116 = ishl v113, v115+;; @004b v117 = bor v116, v112;; @004b jump block7;;;; block9 cold:;; @004b trap user12;;;; block7:-;; @004b br_table v109, block9, []+;; @004b br_table v110, block9, [];;;; block3:-;; @004b v117 = iconst.i64 0-;; @004b v118 = iadd.i64 v26, v117 ; v117 = 0-;; @004b v119 = load.i64 notrap aligned region13 v118-;; @004b store notrap aligned region1 v119, v34+24-;; @004b v120 = load.i64 notrap aligned region10 v118+8-;; @004b store notrap aligned region7 v120, v34+72-;; @004b v121 = load.i64 notrap aligned region11 v118+16-;; @004b store notrap aligned region8 v121, v34+64-;; @004b v122 = load.i64 notrap aligned region12 v118+24-;; @004b store notrap aligned region9 v122, v34+80-;; @004b v123 = iconst.i64 0-;; @004b v124 = iadd.i64 v67, v123 ; v123 = 0-;; @004b v125 = iconst.i32 4-;; @004b store notrap aligned region6 v125, v124+32 ; v125 = 4-;; @004b v126 = iconst.i64 120-;; @004b v127 = iadd.i64 v67, v126 ; v126 = 120-;; @004b v128 = load.i64 notrap aligned region15 v127+8-;; @004b v129 = iconst.i32 0-;; @004b store notrap aligned region17 v129, v127 ; v129 = 0+;; @004b v118 = iconst.i64 0+;; @004b v119 = iadd.i64 v27, v118 ; v118 = 0+;; @004b v120 = load.i64 notrap aligned region13 v119+;; @004b store notrap aligned region1 v120, v35+24+;; @004b v121 = load.i64 notrap aligned region10 v119+8+;; @004b store notrap aligned region7 v121, v35+72+;; @004b v122 = load.i64 notrap aligned region11 v119+16+;; @004b store notrap aligned region8 v122, v35+64+;; @004b v123 = load.i64 notrap aligned region12 v119+24+;; @004b store notrap aligned region9 v123, v35+80+;; @004b v124 = iconst.i64 0+;; @004b v125 = iadd.i64 v68, v124 ; v124 = 0+;; @004b v126 = iconst.i32 4+;; @004b store notrap aligned region6 v126, v125+32 ; v126 = 4+;; @004b v127 = iconst.i64 120+;; @004b v128 = iadd.i64 v68, v127 ; v127 = 120+;; @004b v129 = load.i64 notrap aligned region15 v128+8;; @004b v130 = iconst.i32 0-;; @004b store notrap aligned region14 v130, v127+4 ; v130 = 0-;; @004b v131 = iconst.i64 0-;; @004b store notrap aligned region15 v131, v127+8 ; v131 = 0+;; @004b store notrap aligned region17 v130, v128 ; v130 = 0+;; @004b v131 = iconst.i32 0+;; @004b store notrap aligned region14 v131, v128+4 ; v131 = 0+;; @004b v132 = iconst.i64 0+;; @004b store notrap aligned region15 v132, v128+8 ; v132 = 0;; @0050 jump block1;;;; block1:diff --git a/tests/misc_testsuite/stack-switching-and-gc.wast b/tests/misc_testsuite/stack-switching-and-gc.wastdeleted file mode 100644index 229f2fe16d..0000000000--- a/tests/misc_testsuite/stack-switching-and-gc.wast+++ /dev/null@@ -1,13 +0,0 @@-;;! stack_switching = true-;;! gc = true--(assert_invalid- (module- (type $f (func))- (type $c (cont $f))- (type $s (struct (field (ref null $c))))- (func (export "run")- (drop (struct.new_default $s))- )- )- "Stack switching feature not compatible with GC")diff --git a/tests/misc_testsuite/stack-switching/gc.wast b/tests/misc_testsuite/stack-switching/gc.wastnew file mode 100644index 0000000000..5cbe6af868--- /dev/null+++ b/tests/misc_testsuite/stack-switching/gc.wast@@ -0,0 +1,363 @@+;;! stack_switching = true+;;! gc = true+;;! function_references = true+;;! bulk_memory = true++;; A GC reference in a parent frame must remain live while a child+;; continuation is running.+(module+ (type $box (struct (field i32)))+ (type $f (func))+ (type $c (cont $f))+ (tag $suspend)++ (import "wasmtime" "gc" (func $gc))++ (func $child+ (call $gc)+ (suspend $suspend))+ (elem declare func $child)++ (func (export "parent-frame") (result i32)+ (local $box (ref null $box))+ (local.set $box (struct.new $box (i32.const 42)))++ (drop+ (block $on-suspend (result (ref $c))+ (resume $c+ (on $suspend $on-suspend)+ (cont.new $c (ref.func $child)))+ (unreachable)))++ (struct.get $box 0 (local.get $box)))+)++(assert_return (invoke "parent-frame") (i32.const 42))++;; Stack-map tracing follows the full parent chain, and execution can+;; subsequently re-enter every parent frame.+(module+ (type $box (struct (field i32)))+ (type $f (func (result i32)))+ (type $c (cont $f))+ (tag $suspend)++ (import "wasmtime" "gc" (func $gc))++ (func $child (result i32)+ (call $gc)+ (suspend $suspend)+ (i32.const 1))+ (elem declare func $child)++ (func $middle (result i32)+ (local $box (ref null $box))+ (local.set $box (struct.new $box (i32.const 20)))+ (i32.add+ (resume $c (cont.new $c (ref.func $child)))+ (struct.get $box 0 (local.get $box))))+ (elem declare func $middle)++ (func (export "parent-chain") (result i32)+ (local $box (ref null $box))+ (local $suspended (ref null $c))+ (local.set $box (struct.new $box (i32.const 100)))++ (local.set $suspended+ (block $on-suspend (result (ref $c))+ (resume $c+ (on $suspend $on-suspend)+ (cont.new $c (ref.func $middle)))+ (unreachable)))++ (i32.add+ (resume $c (local.get $suspended))+ (struct.get $box 0 (local.get $box))))+)++(assert_return (invoke "parent-chain") (i32.const 121))++;; A bound GC reference is retained only by a fresh continuation's+;; argument buffer when collection runs before the continuation is+;; first resumed.+(module+ (type $box (struct (field i32)))+ (type $with-arg-f (func (param (ref $box)) (result i32)))+ (type $without-arg-f (func (result i32)))+ (type $with-arg-c (cont $with-arg-f))+ (type $without-arg-c (cont $without-arg-f))++ (import "wasmtime" "gc" (func $gc))++ (func $read-box (param $box (ref $box)) (result i32)+ (struct.get $box 0 (local.get $box)))+ (elem declare func $read-box)++ (func (export "bound-argument") (result i32)+ (local $continuation (ref null $without-arg-c))++ (local.set $continuation+ (cont.bind $with-arg-c $without-arg-c+ (struct.new $box (i32.const 73))+ (cont.new $with-arg-c (ref.func $read-box))))++ (call $gc)+ (resume $without-arg-c+ (local.get $continuation)))+)++(assert_return (invoke "bound-argument") (i32.const 73))++;; The same root metadata is used for the `values` buffer after a+;; continuation has suspended.+(module+ (type $box (struct (field i32)))+ (type $initial-f (func (result i32)))+ (type $suspended-f (func (param (ref $box)) (result i32)))+ (type $initial-c (cont $initial-f))+ (type $suspended-c (cont $suspended-f))+ (tag $yield (result (ref $box)))++ (import "wasmtime" "gc" (func $gc))++ (func $suspend-then-read (result i32)+ (struct.get $box 0 (suspend $yield)))+ (elem declare func $suspend-then-read)++ (func (export "bound-suspended-argument") (result i32)+ (local $suspended (ref null $suspended-c))+ (local $ready (ref null $initial-c))++ (local.set $suspended+ (block $on-yield (result (ref $suspended-c))+ (resume $initial-c+ (on $yield $on-yield)+ (cont.new $initial-c (ref.func $suspend-then-read)))+ (unreachable)))++ (local.set $ready+ (cont.bind $suspended-c $initial-c+ (struct.new $box (i32.const 91))+ (local.get $suspended)))++ (call $gc)+ (resume $initial-c (ref.as_non_null (local.get $ready))))+)++(assert_return (invoke "bound-suspended-argument") (i32.const 91))++;; Struct and array fields store continuation references through a+;; store-local side table. Collection may run while the aggregate is+;; the only place that retains the continuation object.+(module+ (type $f (func (result i32)))+ (type $c (cont $f))+ (type $holder (struct (field (mut (ref null $c)))))+ (type $array (array (mut (ref null $c))))++ (import "wasmtime" "gc" (func $gc))++ (func $forty-two (result i32)+ (i32.const 42))+ (elem declare func $forty-two)++ (func (export "struct-initialize") (result i32)+ (local $holder (ref $holder))+ (local.set $holder+ (struct.new $holder+ (cont.new $c (ref.func $forty-two))))+ (call $gc)+ (resume $c+ (struct.get $holder 0 (local.get $holder))))++ (func (export "struct-update") (result i32)+ (local $holder (ref $holder))+ (local.set $holder (struct.new_default $holder))+ (struct.set $holder 0+ (local.get $holder)+ (cont.new $c (ref.func $forty-two)))+ (call $gc)+ (resume $c+ (struct.get $holder 0 (local.get $holder))))++ (func (export "array-initialize") (result i32)+ (local $array (ref $array))+ (local.set $array+ (array.new_fixed $array 1+ (cont.new $c (ref.func $forty-two))))+ (call $gc)+ (resume $c+ (array.get $array (local.get $array) (i32.const 0))))++ (func (export "array-update") (result i32)+ (local $array (ref $array))+ (local.set $array (array.new_default $array (i32.const 1)))+ (array.set $array+ (local.get $array)+ (i32.const 0)+ (cont.new $c (ref.func $forty-two)))+ (call $gc)+ (resume $c+ (array.get $array (local.get $array) (i32.const 0))))++ (func (export "array-fill-and-copy") (result i32)+ (local $source (ref $array))+ (local $destination (ref $array))+ (local.set $source (array.new_default $array (i32.const 1)))+ (local.set $destination (array.new_default $array (i32.const 1)))+ (array.fill $array+ (local.get $source)+ (i32.const 0)+ (cont.new $c (ref.func $forty-two))+ (i32.const 1))+ (array.copy $array $array+ (local.get $destination)+ (i32.const 0)+ (local.get $source)+ (i32.const 0)+ (i32.const 1))+ (call $gc)+ (resume $c+ (array.get $array (local.get $destination) (i32.const 0))))++ (func (export "null-defaults") (result i32)+ (local $holder (ref $holder))+ (local $array (ref $array))+ (local.set $holder (struct.new_default $holder))+ (local.set $array (array.new_default $array (i32.const 1)))+ (i32.add+ (ref.is_null (struct.get $holder 0 (local.get $holder)))+ (ref.is_null+ (array.get $array (local.get $array) (i32.const 0)))))++ ;; Loading an alias from a GC object must reproduce the interned+ ;; revision witness. Consuming one alias therefore invalidates all+ ;; other loads of the same field.+ (func (export "consume-struct-field-twice")+ (local $holder (ref $holder))+ (local.set $holder+ (struct.new $holder+ (cont.new $c (ref.func $forty-two))))+ (drop+ (resume $c+ (struct.get $holder 0 (local.get $holder))))+ (drop+ (resume $c+ (struct.get $holder 0 (local.get $holder)))))+)++(assert_return (invoke "struct-initialize") (i32.const 42))+(assert_return (invoke "struct-update") (i32.const 42))+(assert_return (invoke "array-initialize") (i32.const 42))+(assert_return (invoke "array-update") (i32.const 42))+(assert_return (invoke "array-fill-and-copy") (i32.const 42))+(assert_return (invoke "null-defaults") (i32.const 2))+(assert_trap+ (invoke "consume-struct-field-twice")+ "continuation already consumed")++;; A GC reference passed to a child continuation must remain live when+;; that child is unwound by `resume_throw`. The parent retains another+;; reference, catches the injected exception, and can still use the+;; object after a second collection.+(module+ (type $box (struct (field i32)))+ (type $child-f (func (param (ref $box)) (result i32)))+ (type $parent-f (func (result i32)))+ (type $child-c (cont $child-f))+ (type $parent-c (cont $parent-f))+ (tag $suspend)+ (tag $exception)++ (import "wasmtime" "gc" (func $gc))++ (func $child (param $box (ref $box)) (result i32)+ ;; At this collection, both the child and its parent have the box+ ;; in live stack slots. Keep the parameter live across the+ ;; suspension point.+ (call $gc)+ (suspend $suspend)+ (struct.get $box 0 (local.get $box)))+ (elem declare func $child)++ (func $parent (result i32)+ (local $box (ref $box))+ (local.set $box (struct.new $box (i32.const 101)))++ (block $caught+ (try_table (catch $exception $caught)+ (drop+ (resume $child-c+ (local.get $box)+ (cont.new $child-c (ref.func $child)))))+ (unreachable))++ ;; The child continuation is terminal, so only the parent's stack+ ;; slot retains the box at this collection.+ (call $gc)+ (struct.get $box 0 (local.get $box)))+ (elem declare func $parent)++ (func (export "gc-ref-survives-resume-throw") (result i32)+ (local $suspended (ref null $parent-c))++ (local.set $suspended+ (block $on-suspend (result (ref $parent-c))+ (resume $parent-c+ (on $suspend $on-suspend)+ (cont.new $parent-c (ref.func $parent)))+ (unreachable)))++ (resume_throw $parent-c $exception+ (local.get $suspended)))+)++(assert_return+ (invoke "gc-ref-survives-resume-throw")+ (i32.const 101))++;; A GC reference carried by an exception injected with `resume_throw`+;; must become a live stack root when the exception is caught inside+;; the resumed continuation.+(module+ (type $box (struct (field i32)))+ (type $f (func (result i32)))+ (type $c (cont $f))+ (tag $suspend)+ (tag $exception (param (ref $box)))++ (import "wasmtime" "gc" (func $gc))++ (func $catch-payload (result i32)+ (local $box (ref null $box))++ (local.set $box+ (block $caught (result (ref $box))+ (try_table (result (ref $box)) (catch $exception $caught)+ (suspend $suspend)+ (unreachable))))++ ;; The caught local is now the only reference to the exception+ ;; payload.+ (call $gc)+ (struct.get $box 0 (ref.as_non_null (local.get $box))))+ (elem declare func $catch-payload)++ (func (export "gc-ref-in-resume-throw-payload") (result i32)+ (local $suspended (ref null $c))++ (local.set $suspended+ (block $on-suspend (result (ref $c))+ (resume $c+ (on $suspend $on-suspend)+ (cont.new $c (ref.func $catch-payload)))+ (unreachable)))++ (resume_throw $c $exception+ (struct.new $box (i32.const 202))+ (local.get $suspended)))+)++(assert_return+ (invoke "gc-ref-in-resume-throw-payload")+ (i32.const 202))diff --git a/tests/misc_testsuite/stack-switching/issue13021.wast b/tests/misc_testsuite/stack-switching/issue13021.wastnew file mode 100644index 0000000000..949362e019--- /dev/null+++ b/tests/misc_testsuite/stack-switching/issue13021.wast@@ -0,0 +1,18 @@+;;! gc = true+;;! stack_switching = true++;; Regression test for issue https://github.com/bytecodealliance/wasmtime/issues/13021+(module+ (type $ft (func))+ (type $ct (cont $ft))+ (type $arr (array (mut (ref null $ct))))+ (func (export "boom")+ (local $a (ref $arr))+ (local.set $a (array.new_default $arr (i32.const 2)))+ (array.copy $arr $arr+ (local.get $a) (i32.const 0)+ (local.get $a) (i32.const 0)+ (i32.const 1))+ )+)+(assert_return (invoke "boom"))\ No newline at end of filediff --git a/tests/misc_testsuite/stack-switching/issue13022.wast b/tests/misc_testsuite/stack-switching/issue13022.wastnew file mode 100644index 0000000000..1ccf5a7c69--- /dev/null+++ b/tests/misc_testsuite/stack-switching/issue13022.wast@@ -0,0 +1,18 @@+;;! gc = true+;;! stack_switching = true++;; Regression test for https://github.com/bytecodealliance/wasmtime/issues/13022+(module+ (type $ft (func))+ (type $ct (cont $ft))+ (type $arr (array (mut (ref null $ct))))+ (func (export "boom")+ (local $a (ref $arr))+ (local.set $a (array.new_default $arr (i32.const 2)))+ (array.copy $arr $arr+ (local.get $a) (i32.const 0)+ (local.get $a) (i32.const 0)+ (i32.const 1))+ )+)+(assert_return (invoke "boom"))\ No newline at end of file