Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

[stack-switching] GC interaction (#14140)

Daniel Hillerströmb68f140620f7
diff --git a/cranelift/codegen/src/isa/x64/inst/emit.rs b/cranelift/codegen/src/isa/x64/inst/emit.rs
index eb7244e00b..fb8cba43e0 100644
--- a/cranelift/codegen/src/isa/x64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/x64/inst/emit.rs
@@ -550,6 +550,8 @@ pub(crate) fn emit(
in_payload0,
out_payload0,
} => {
+ let stack_map = state.take_stack_map();
+
// Note that we do not emit anything for preserving and restoring
// ordinary registers here: That's taken care of by regalloc for us,
// since we marked this instruction as clobbering all registers.
@@ -643,6 +645,9 @@ pub(crate) fn emit(
asm::inst::jmpq_m::new(tmp1.to_reg()).emit(sink, info, state);
sink.bind_label(resume, state.ctrl_plane_mut());
+ if let Some(s) = stack_map {
+ sink.push_user_stack_map(state, sink.cur_offset(), s);
+ }
}
Inst::JmpKnown { dst } => uncond_jmp(sink, *dst),
diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs
index 1ee94966b3..e758ee087c 100644
--- a/cranelift/codegen/src/isa/x64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/x64/inst/mod.rs
@@ -1467,7 +1467,9 @@ impl MachInst for Inst {
fn is_safepoint(&self) -> bool {
match self {
- Inst::CallKnown { .. } | Inst::CallUnknown { .. } => true,
+ Inst::CallKnown { .. } | Inst::CallUnknown { .. } | Inst::StackSwitchBasic { .. } => {
+ true
+ }
_ => false,
}
}
diff --git a/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif b/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif
new file mode 100644
index 0000000000..dff66b8137
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/stack_switch_stack_map.clif
@@ -0,0 +1,14 @@
+test compile
+set opt_level=speed
+set stack_switch_model=basic
+target x86_64
+
+;; A stack switch is a GC safepoint. Its stack map is associated with the
+;; resume PC saved in the outgoing control context.
+function %switch_with_stack_map(i64, i64, i64) -> i64 {
+ ss0 = explicit_slot 4
+
+block0(v0: i64, v1: i64, v2: i64):
+ v3 = stack_switch v0, v1, v2, stack_map=[i32 @ ss0+0]
+ return v3
+}
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index 3282f36ae2..bcab270214 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -80,6 +80,8 @@ enum VmType {
BuiltinFunctionsArray,
ComponentBuiltinFunctionsArray,
HostValRaw,
+ VMPayloads,
+ VMRawContObj,
}
/// A key that uniquely identifies an alias region across an entire compilation.
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 52c86cdcea..6da9845a9e 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -46,6 +46,20 @@ use wasmtime_environ::{
};
use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};
+/// Function-local stack slots backing a continuation's
+/// `VMPayloads::values`.
+///
+/// The values and their GC-reference markers are logically one
+/// payload descriptor but use distinct stack slots so that Cranelift
+/// can assign them distinct alias regions. The marker slot is only
+/// allocated when this function contains a stack switching site whose
+/// payloads may contain GC references.
+#[derive(Clone, Copy)]
+pub(crate) struct VMPayloadStackSlots {
+ pub(crate) values: ir::StackSlot,
+ pub(crate) gc_ref_markers: Option<ir::StackSlot>,
+}
+
#[derive(Copy, Clone, Debug)]
pub(crate) enum Extension {
Sign,
@@ -216,10 +230,12 @@ pub struct FuncEnvironment<'module_environment> {
/// current stack's `handler_list` field.
stack_switching_handler_list_buffer: Option<ir::StackSlot>,
- /// Used by the stack switching feature. If set, we have a allocated a
- /// slot on this function's stack to be used for the
- /// current continuation's `values` field.
- stack_switching_values_buffer: Option<ir::StackSlot>,
+ /// Used by the stack switching feature. If set, these are the stack slots
+ /// backing the current continuation's `values` field.
+ stack_switching_values_storage: Option<VMPayloadStackSlots>,
+
+ /// Reusable storage for `get_interned_contref` builtin.
+ stack_switching_cont_ref_result_storage: Option<ir::StackSlot>,
/// The stack-slot used for exposing Wasm state via debug
/// instrumentation, if any, and the builder containing its metadata.
@@ -300,7 +316,8 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
stack_limit_at_function_entry: None,
stack_switching_handler_list_buffer: None,
- stack_switching_values_buffer: None,
+ stack_switching_values_storage: None,
+ stack_switching_cont_ref_result_storage: None,
state_slot: None,
next_srcloc: ir::SourceLoc::default(),
@@ -313,6 +330,18 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
}
}
+ /// Returns the cached continuation-reference stack slot, creating it with
+ /// `data` if necessary.
+ pub(crate) fn get_or_create_contref_stack_slot(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ data: ir::StackSlotData,
+ ) -> ir::StackSlot {
+ *self
+ .stack_switching_cont_ref_result_storage
+ .get_or_insert_with(|| builder.create_sized_stack_slot(data))
+ }
+
/// Consume the branch hint for the instruction at module-relative `offset`
/// (i.e. `builder.srcloc().bits()`), if any. The lazy decoder only moves
/// forward, making this O(n) over a function body.
@@ -5295,8 +5324,9 @@ impl FuncEnvironment<'_> {
builder: &mut FunctionBuilder<'_>,
contobj: ir::Value,
args: &[ir::Value],
+ arg_types: &[WasmValType],
) -> ir::Value {
- stack_switching::instructions::translate_cont_bind(self, builder, contobj, args)
+ stack_switching::instructions::translate_cont_bind(self, builder, contobj, args, arg_types)
}
pub fn translate_cont_new(
@@ -5376,13 +5406,15 @@ impl FuncEnvironment<'_> {
builder: &mut FunctionBuilder<'_>,
tag_index: u32,
suspend_args: &[ir::Value],
- tag_return_types: &[ir::Type],
+ suspend_arg_types: &[WasmValType],
+ tag_return_types: &[WasmValType],
) -> WasmResult<Vec<ir::Value>> {
stack_switching::instructions::translate_suspend(
self,
builder,
tag_index,
suspend_args,
+ suspend_arg_types,
tag_return_types,
)
}
@@ -5394,7 +5426,8 @@ impl FuncEnvironment<'_> {
tag_index: u32,
contobj: ir::Value,
switch_args: &[ir::Value],
- return_types: &[ir::Type],
+ switch_arg_types: &[WasmValType],
+ return_types: &[WasmValType],
) -> WasmResult<Vec<ir::Value>> {
stack_switching::instructions::translate_switch(
self,
@@ -5402,6 +5435,7 @@ impl FuncEnvironment<'_> {
tag_index,
contobj,
switch_args,
+ switch_arg_types,
return_types,
)
}
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index ca88d1bc7d..8603eca2a8 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -2,7 +2,7 @@
use crate::TRAP_ARRAY_OUT_OF_BOUNDS;
use crate::bounds_checks::BoundsCheck;
-use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment};
+use crate::func_environ::{CheckedEntity, Extension, FuncEnvironment, stack_switching::fatpointer};
use crate::translate::{
Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment, VmctxLoadChain,
};
@@ -18,7 +18,7 @@ use cranelift_frontend::FunctionBuilder;
use smallvec::{SmallVec, smallvec};
use wasmtime_environ::{
Collector, GcArrayLayout, GcLayout, GcStructLayout, GcTypeLayouts, I31_DISCRIMINANT,
- ModuleInternedTypeIndex, TagIndex, TypeIndex, VMGcKind, WasmCompositeInnerType,
+ ModuleInternedTypeIndex, PtrSize, TagIndex, TypeIndex, VMGcKind, WasmCompositeInnerType,
WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,
wasm_unsupported,
};
@@ -377,10 +377,7 @@ pub fn read_field_at_addr(
.call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]);
builder.func.dfg.first_result(call_inst)
}
- WasmHeapTopType::Cont => {
- // TODO(#10248) GC integration for stack switching
- return stack_switching_unsupported();
- }
+ WasmHeapTopType::Cont => read_cont_ref_at_addr(func_env, builder, addr, flags)?,
},
},
};
@@ -431,6 +428,92 @@ pub fn intern_func_ref(
Ok(builder.ins().ireduce(ir::types::I32, func_ref_id))
}
+fn intern_cont_ref(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ ref_type: WasmRefType,
+ contobj: ir::Value,
+) -> ir::Value {
+ assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Cont);
+
+ if ref_type.heap_type == WasmHeapType::NoCont {
+ let zero = builder.ins().iconst(ir::types::I32, 0);
+
+ if !ref_type.nullable {
+ builder.ins().trapz(zero, TRAP_INTERNAL_ASSERT);
+ }
+
+ return zero;
+ }
+
+ let (revision, contref) = fatpointer::deconstruct(func_env, &mut builder.cursor(), contobj);
+ let vmctx = func_env.vmctx_val(&mut builder.cursor());
+ let intern = func_env
+ .builtin_functions
+ .intern_contref_for_gc_heap(builder.func);
+ let call = builder.ins().call(intern, &[vmctx, contref, revision]);
+ let id = builder.func.dfg.first_result(call);
+ builder.ins().ireduce(ir::types::I32, id)
+}
+
+fn read_cont_ref_at_addr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ addr: ir::Value,
+ flags: ir::MemFlagsData,
+) -> WasmResult<ir::Value> {
+ let id = builder.ins().load(ir::types::I32, flags, addr, 0);
+
+ // We either create or fetch an existing stack slot to hold the raw
+ // continuation object, and pass its address as the out parameter to the
+ // builtin.
+ let pointer_type = func_env.pointer_type();
+ let layout = func_env.offsets.ptr.vm_raw_cont_obj();
+ let slot_size = u32::from(layout.size());
+ let slot_align = u8::try_from(layout.align().trailing_zeros()).unwrap();
+ let contref_offset = layout.contref();
+ let revision_offset = layout.revision();
+
+ let slot = func_env.get_or_create_contref_stack_slot(
+ builder,
+ ir::StackSlotData::new(ir::StackSlotKind::ExplicitSlot, slot_size, slot_align),
+ );
+ let out_result = builder.ins().stack_addr(pointer_type, slot, 0);
+
+ let vmctx = func_env.vmctx_val(&mut builder.cursor());
+ let get = func_env
+ .builtin_functions
+ .get_interned_contref(builder.func);
+ builder.ins().call(get, &[vmctx, id, out_result]);
+
+ let region = func_env.alias_regions.stack_slot_region(builder.func, slot);
+ let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region));
+ let contref = builder
+ .ins()
+ .load(pointer_type, flags, out_result, contref_offset);
+ let revision = builder
+ .ins()
+ .load(pointer_type, flags, out_result, revision_offset);
+ Ok(fatpointer::construct(
+ func_env,
+ &mut builder.cursor(),
+ revision,
+ contref,
+ ))
+}
+
+fn write_cont_ref_at_addr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ ref_type: WasmRefType,
+ flags: ir::MemFlagsData,
+ field_addr: ir::Value,
+ contobj: ir::Value,
+) {
+ let id = intern_cont_ref(func_env, builder, ref_type, contobj);
+ builder.ins().store(flags, id, field_addr, 0);
+}
+
fn write_func_ref_at_addr(
func_env: &mut FuncEnvironment<'_>,
builder: &mut FunctionBuilder<'_>,
@@ -487,7 +570,9 @@ pub fn write_field_at_addr(
func_env, builder, r, field_addr, new_val, flags,
)?;
}
- WasmHeapTopType::Cont => return stack_switching_unsupported(),
+ WasmHeapTopType::Cont => {
+ write_cont_ref_at_addr(func_env, builder, r, flags, field_addr, new_val)
+ }
},
WasmStorageType::Val(_) => {
assert_eq!(
@@ -527,6 +612,10 @@ fn default_value(
}
WasmValType::Ref(r) => {
assert!(r.nullable);
+ if r.heap_type.top() == WasmHeapTopType::Cont {
+ let zero = cursor.ins().iconst(func_env.pointer_type(), 0);
+ return fatpointer::construct(func_env, cursor, zero, zero);
+ }
let (ty, needs_stack_map) = func_env.reference_type(r.heap_type);
// NB: The collector doesn't need to know about null references.
diff --git a/crates/cranelift/src/func_environ/gc/copying.rs b/crates/cranelift/src/func_environ/gc/copying.rs
index 7cf5628d0b..ec9b47a9e8 100644
--- a/crates/cranelift/src/func_environ/gc/copying.rs
+++ b/crates/cranelift/src/func_environ/gc/copying.rs
@@ -444,7 +444,9 @@ impl GcCompiler for CopyingCompiler {
// store the reference directly.
unbarriered_store_gc_ref(builder, r.heap_type, field_addr, val, flags)?;
}
- WasmHeapTopType::Cont => return super::stack_switching_unsupported(),
+ WasmHeapTopType::Cont => {
+ write_field_at_addr(func_env, builder, ty, field_addr, val)?
+ }
},
WasmStorageType::I8 => {
assert_eq!(builder.func.dfg.value_type(val), ir::types::I32);
diff --git a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs
index 05b92e7779..89cc0083b8 100644
--- a/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs
+++ b/crates/cranelift/src/func_environ/stack_switching/fatpointer.rs
@@ -34,7 +34,7 @@ pub(crate) fn deconstruct<'a>(
/// Constructs a continuation object from a given contref and revision pointer.
/// The contref_addr may be 0, to indicate that we want to build a wasm null reference.
pub(crate) fn construct<'a>(
- env: &mut crate::func_environ::FuncEnvironment<'a>,
+ env: &crate::func_environ::FuncEnvironment<'a>,
pos: &mut cranelift_codegen::cursor::FuncCursor,
revision_counter: ir::Value,
contref_addr: ir::Value,
diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
index 0310f62bd1..5a6dac28c5 100644
--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs
+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
@@ -8,7 +8,10 @@ use cranelift_codegen::ir::{self, MemFlagsData};
use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData};
use cranelift_frontend::FunctionBuilder;
use itertools::{Either, Itertools};
-use wasmtime_environ::{PtrSize, TagIndex, TypeIndex, WasmResult, WasmValType, wasm_unsupported};
+use wasmtime_environ::{
+ PtrSize, TagIndex, TypeIndex, WasmHeapType, WasmRefType, WasmResult, WasmValType,
+ wasm_unsupported,
+};
fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult<u8> {
match (triple.architecture, triple.operating_system) {
@@ -31,7 +34,7 @@ pub(crate) mod stack_switching_helpers {
use cranelift_codegen::ir::types::*;
use cranelift_codegen::ir::{StackSlot, StackSlotKind::*};
use cranelift_frontend::FunctionBuilder;
- use wasmtime_environ::PtrSize;
+ use wasmtime_environ::{PtrSize, WasmValType};
/// Provides information about the layout of a type when it is used as an
/// element in a host array. This is used for `VMHostArrayRef`.
@@ -69,7 +72,17 @@ pub(crate) mod stack_switching_helpers {
phantom: PhantomData<T>,
}
- pub type VMPayloads = VMHostArrayRef<u128>;
+ /// Compile-time reference to a runtime `VMPayloads` descriptor.
+ ///
+ /// This wrapper keeps the payload buffer and its GC-reference markers
+ /// paired. Its core invariant is that `gc_ref_data`, when present, points
+ /// to one marker byte for every slot in `buffer`; callers must update a
+ /// slot's value and marker together.
+ #[derive(Copy, Clone)]
+ pub struct VMPayloads {
+ address: ir::Value,
+ buffer: VMHostArrayRef<u128>,
+ }
// Actually a vector of *mut VMTagDefinition
pub type VMHandlerList = VMHostArrayRef<*mut u8>;
@@ -105,7 +118,7 @@ pub(crate) mod stack_switching_helpers {
) -> VMPayloads {
let offset: i64 = env.offsets.ptr.vm_cont_ref().args().into();
let address = builder.ins().iadd_imm_s(self.address, offset);
- VMPayloads::new(address)
+ VMPayloads::new(env, address)
}
pub fn values<'a>(
@@ -115,7 +128,7 @@ pub(crate) mod stack_switching_helpers {
) -> VMPayloads {
let offset: i64 = env.offsets.ptr.vm_cont_ref().values().into();
let address = builder.ins().iadd_imm_s(self.address, offset);
- VMPayloads::new(address)
+ VMPayloads::new(env, address)
}
pub fn common_stack_information<'a>(
@@ -238,6 +251,147 @@ pub(crate) mod stack_switching_helpers {
}
}
+ impl VMPayloads {
+ pub fn new(env: &mut crate::func_environ::FuncEnvironment<'_>, address: ir::Value) -> Self {
+ debug_assert_eq!(env.offsets.ptr.vm_payloads().buffer(), 0);
+ Self {
+ address,
+ buffer: VMHostArrayRef::new(address),
+ }
+ }
+
+ pub fn get_gc_ref_data(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ env.alias_regions
+ .vm_payloads()
+ .gc_ref_data()
+ .load(&mut builder.cursor(), self.address)
+ }
+
+ pub fn set_gc_ref_data(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ data: ir::Value,
+ ) {
+ env.alias_regions.vm_payloads().gc_ref_data().store(
+ &mut builder.cursor(),
+ self.address,
+ data,
+ );
+ }
+
+ /// Prepares stack storage for this payload buffer.
+ ///
+ /// Payload values and their GC-reference markers use distinct stack
+ /// slots so that they have distinct alias regions. The descriptor's
+ /// capacity and pointers always describe the storage selected at this
+ /// particular instruction site. Both slots retain their identities and
+ /// grow in place as later sites require more capacity.
+ pub fn prepare_stack_storage(
+ &self,
+ env: &mut crate::func_environ::FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ capacity: u32,
+ initial_types: &[WasmValType],
+ needs_gc_ref_markers: bool,
+ existing_slots: Option<crate::func_environ::VMPayloadStackSlots>,
+ ) -> crate::func_environ::VMPayloadStackSlots {
+ debug_assert!(capacity > 0);
+ debug_assert!(initial_types.len() <= usize::try_from(capacity).unwrap());
+
+ let (align, entry_size) =
+ <u128 as VMHostArrayEntry>::vmhostarray_entry_layout(&env.offsets.ptr);
+ let values_size = capacity
+ .checked_mul(entry_size)
+ .expect("stack switching values buffer size should fit in u32");
+
+ let values_slot = match existing_slots.map(|slots| slots.values) {
+ Some(slot) => {
+ // Keep the slot identity stable: `stack_addr` instructions
+ // emitted at earlier sites continue to refer to this slot
+ // when a later site requires more storage.
+ let slot_data = &mut builder.func.sized_stack_slots[slot];
+ debug_assert!(align <= slot_data.align_shift);
+ debug_assert_eq!(slot_data.kind, ExplicitSlot);
+ slot_data.size = slot_data.size.max(values_size);
+ slot
+ }
+ None => builder.create_sized_stack_slot(ir::StackSlotData::new(
+ ir::StackSlotKind::ExplicitSlot,
+ values_size,
+ align,
+ )),
+ };
+
+ let capacity_value = builder.ins().iconst(I32, i64::from(capacity));
+ let values_data = builder.ins().stack_addr(env.pointer_type(), values_slot, 0);
+ self.set_capacity(env, builder, capacity_value);
+ self.set_data(env, builder, values_data);
+
+ let mut gc_ref_markers = existing_slots.and_then(|slots| slots.gc_ref_markers);
+ if needs_gc_ref_markers {
+ let marker_slot = match gc_ref_markers {
+ Some(slot) => {
+ let slot_data = &mut builder.func.sized_stack_slots[slot];
+ debug_assert_eq!(slot_data.kind, ExplicitSlot);
+ slot_data.size = slot_data.size.max(capacity);
+ slot
+ }
+ None => builder.create_sized_stack_slot(ir::StackSlotData::new(
+ ir::StackSlotKind::ExplicitSlot,
+ capacity,
+ 0,
+ )),
+ };
+ gc_ref_markers = Some(marker_slot);
+
+ let marker_data = builder.ins().stack_addr(env.pointer_type(), marker_slot, 0);
+ self.set_gc_ref_data(env, builder, marker_data);
+
+ let region = env
+ .alias_regions
+ .stack_slot_region(builder.func, marker_slot);
+ let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region));
+ let zero = builder.ins().iconst(I8, 0);
+ for offset in 0..capacity {
+ builder
+ .ins()
+ .store(flags, zero, marker_data, i32::try_from(offset).unwrap());
+ }
+ for (offset, ty) in initial_types.iter().enumerate() {
+ if ty.is_vmgcref_type_and_not_i31() {
+ let marker = builder
+ .ins()
+ .iconst(I8, i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF));
+ builder.ins().store(
+ flags,
+ marker,
+ marker_data,
+ i32::try_from(offset).unwrap(),
+ );
+ }
+ }
+ }
+
+ crate::func_environ::VMPayloadStackSlots {
+ values: values_slot,
+ gc_ref_markers,
+ }
+ }
+ }
+
+ impl core::ops::Deref for VMPayloads {
+ type Target = VMHostArrayRef<u128>;
+
+ fn deref(&self) -> &Self::Target {
+ &self.buffer
+ }
+ }
+
impl<T: VMHostArrayEntry> VMHostArrayRef<T> {
pub(crate) fn new(address: ir::Value) -> Self {
Self {
@@ -315,7 +469,7 @@ pub(crate) mod stack_switching_helpers {
env: &mut crate::func_environ::FuncEnvironment<'a>,
builder: &mut FunctionBuilder,
arg_count: i32,
- ) -> ir::Value {
+ ) -> (ir::Value, ir::Value) {
let data = self.get_data(env, builder);
let original_length = self.get_length(env, builder);
let new_length = builder
@@ -328,7 +482,7 @@ pub(crate) mod stack_switching_helpers {
let byte_offset = builder
.ins()
.imul_imm_s(original_length, i64::from(entry_size));
- builder.ins().iadd(data, byte_offset)
+ (builder.ins().iadd(data, byte_offset), original_length)
}
pub fn allocate_or_reuse_stack_slot<'a>(
@@ -375,13 +529,13 @@ pub(crate) mod stack_switching_helpers {
}
/// Loads n entries from this Vector object, where n is the length of
- /// `load_types`, which also gives the types of the values to load.
+ /// `load_types`, which also gives the Wasm types of the values to load.
/// Loading starts at index 0 of the Vector object.
pub fn load_data_entries<'a>(
&self,
env: &mut crate::func_environ::FuncEnvironment<'a>,
builder: &mut FunctionBuilder,
- load_types: &[ir::Type],
+ load_types: &[WasmValType],
) -> Vec<ir::Value> {
let region = env
.alias_regions
@@ -392,10 +546,14 @@ pub(crate) mod stack_switching_helpers {
let mut values = vec![];
let mut offset = 0;
let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);
- for valtype in load_types {
+ for wasm_ty in load_types {
+ let valtype = crate::value_type(env.isa(), *wasm_ty);
let val = builder
.ins()
- .load(*valtype, memflags, data_start_pointer, offset);
+ .load(valtype, memflags, data_start_pointer, offset);
+ if env.val_ty_needs_stack_map(*wasm_ty) {
+ builder.declare_value_needs_stack_map(val);
+ }
values.push(val);
offset += i32::try_from(entry_size).unwrap();
}
@@ -915,6 +1073,10 @@ pub(crate) mod stack_switching_helpers {
use helpers::VMStackChain;
use stack_switching_helpers as helpers;
+fn types_need_gc_ref_markers(types: &[WasmValType]) -> bool {
+ types.iter().any(|ty| ty.is_vmgcref_type_and_not_i31())
+}
+
/// Stores the given arguments in the appropriate `VMPayloads` object in the
/// continuation. If the continuation was never invoked, use the `args` object.
/// Otherwise, use the `values` object.
@@ -922,8 +1084,11 @@ pub(crate) fn vmcontref_store_payloads<'a>(
env: &mut crate::func_environ::FuncEnvironment<'a>,
builder: &mut FunctionBuilder,
values: &[ir::Value],
+ types: &[WasmValType],
contref: ir::Value,
) {
+ debug_assert_eq!(values.len(), types.len());
+ let needs_gc_ref_markers = types_need_gc_ref_markers(types);
let count =
i32::try_from(values.len()).expect("Number of stack switching payloads should fit in i32");
if values.len() > 0 {
@@ -931,6 +1096,9 @@ pub(crate) fn vmcontref_store_payloads<'a>(
let use_payloads_block = builder.create_block();
let store_data_block = builder.create_block();
builder.append_block_param(store_data_block, env.pointer_type());
+ if needs_gc_ref_markers {
+ builder.append_block_param(store_data_block, env.pointer_type());
+ }
let co = helpers::VMContRef::new(contref);
let csi = co.common_stack_information(env, builder);
@@ -944,11 +1112,15 @@ pub(crate) fn vmcontref_store_payloads<'a>(
builder.seal_block(use_args_block);
let args = co.args(env, builder);
- let ptr = args.occupy_next_slots(env, builder, count);
+ let (ptr, original_length) = args.occupy_next_slots(env, builder, count);
+ let mut block_args = vec![BlockArg::Value(ptr)];
+ if needs_gc_ref_markers {
+ let gc_refs_ptr = args.get_gc_ref_data(env, builder);
+ let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length);
+ block_args.push(BlockArg::Value(gc_refs_ptr));
+ }
- builder
- .ins()
- .jump(store_data_block, &[BlockArg::Value(ptr)]);
+ builder.ins().jump(store_data_block, &block_args);
}
{
@@ -959,10 +1131,14 @@ pub(crate) fn vmcontref_store_payloads<'a>(
// This also checks that the buffer is large enough to hold
// `values.len()` more elements.
- let ptr = payloads.occupy_next_slots(env, builder, count);
- builder
- .ins()
- .jump(store_data_block, &[BlockArg::Value(ptr)]);
+ let (ptr, original_length) = payloads.occupy_next_slots(env, builder, count);
+ let mut block_args = vec![BlockArg::Value(ptr)];
+ if needs_gc_ref_markers {
+ let gc_refs_ptr = payloads.get_gc_ref_data(env, builder);
+ let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length);
+ block_args.push(BlockArg::Value(gc_refs_ptr));
+ }
+ builder.ins().jump(store_data_block, &block_args);
}
{
@@ -970,6 +1146,8 @@ pub(crate) fn vmcontref_store_payloads<'a>(
builder.seal_block(store_data_block);
let ptr = builder.block_params(store_data_block)[0];
+ let gc_refs_ptr =
+ needs_gc_ref_markers.then(|| builder.block_params(store_data_block)[1]);
// Store the values.
let region = env
@@ -977,8 +1155,22 @@ pub(crate) fn vmcontref_store_payloads<'a>(
.continuation_stack_memory_region(builder.func);
let memflags = ir::MemFlagsData::trusted().with_alias_region(Some(region));
let mut offset = 0;
- for value in values {
+ for (index, (value, ty)) in values.iter().zip(types).enumerate() {
builder.ins().store(memflags, *value, ptr, offset);
+ if let Some(gc_refs_ptr) = gc_refs_ptr {
+ let marker = if ty.is_vmgcref_type_and_not_i31() {
+ i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF)
+ } else {
+ 0
+ };
+ let marker = builder.ins().iconst(I8, marker);
+ builder.ins().store(
+ memflags,
+ marker,
+ gc_refs_ptr,
+ i32::try_from(index).unwrap(),
+ );
+ }
offset += i32::from(env.offsets.ptr.maximum_value_size());
}
}
@@ -1257,6 +1449,7 @@ pub(crate) fn translate_cont_bind<'a>(
builder: &mut FunctionBuilder,
contobj: ir::Value,
args: &[ir::Value],
+ arg_types: &[WasmValType],
) -> ir::Value {
let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj);
@@ -1268,7 +1461,7 @@ pub(crate) fn translate_cont_bind<'a>(
let evidence = builder.ins().icmp(IntCC::Equal, witness, revision);
env.trapz(builder, evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);
- vmcontref_store_payloads(env, builder, args, contref);
+ vmcontref_store_payloads(env, builder, args, arg_types, contref);
let revision = vmcontref.incr_revision(env, builder, revision);
let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref);
@@ -1291,12 +1484,16 @@ pub(crate) fn translate_cont_new<'a>(
let nreturns = builder
.ins()
.iconst(I32, i64::try_from(return_types.len()).unwrap());
+ let gc_refs = builder.ins().iconst(
+ I32,
+ i64::from(u8::from(types_need_gc_ref_markers(arg_types))),
+ );
let cont_new_func = env.builtin_functions.cont_new(&mut builder.func);
let vmctx = env.vmctx_val(&mut builder.cursor());
let call_inst = builder
.ins()
- .call(cont_new_func, &[vmctx, func, nargs, nreturns]);
+ .call(cont_new_func, &[vmctx, func, nargs, nreturns, gc_refs]);
let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap();
let tag = helpers::VMContRef::new(contref).get_revision(env, builder);
@@ -1380,6 +1577,10 @@ fn translate_resume_impl<'a>(
resume_payload: ResumePayload<'_>,
resumetable: &[(u32, Option<ir::Block>)],
) -> WasmResult<Vec<ir::Value>> {
+ let resume_arg_types = env
+ .continuation_arguments(TypeIndex::from_u32(type_index))
+ .to_vec();
+
// The resume instruction is the most involved instruction to
// compile as it is responsible for both continuation application
// and control tag dispatch.
@@ -1461,10 +1662,17 @@ fn translate_resume_impl<'a>(
match resume_payload {
ResumePayload::Values(resume_args) => {
+ debug_assert_eq!(resume_args.len(), resume_arg_types.len());
let _next_revision = vmcontref.incr_revision(env, builder, revision);
if resume_args.len() > 0 {
// We store the arguments in the `VMContRef` to be resumed.
- vmcontref_store_payloads(env, builder, resume_args, resume_contref);
+ vmcontref_store_payloads(
+ env,
+ builder,
+ resume_args,
+ &resume_arg_types,
+ resume_contref,
+ );
}
}
ResumePayload::Throw { .. } | ResumePayload::ThrowRef(_) => {
@@ -1752,11 +1960,7 @@ fn translate_resume_impl<'a>(
preamble_blocks.push(preamble_block);
builder.switch_to_block(preamble_block);
- let param_types = env.tag_params(TagIndex::from_u32(handle_tag));
- let param_types: Vec<ir::Type> = param_types
- .iter()
- .map(|wty| crate::value_type(env.isa(), *wty))
- .collect();
+ let param_types = env.tag_params(TagIndex::from_u32(handle_tag)).to_vec();
let values = suspended_contref.values(env, builder);
let mut suspend_args: Vec<ir::Value> =
@@ -1821,11 +2025,9 @@ fn translate_resume_impl<'a>(
returned_csi.set_state_returned(env, builder);
// Load the values returned by the continuation.
- let return_types: Vec<_> = env
+ let return_types = env
.continuation_returns(TypeIndex::from_u32(type_index))
- .iter()
- .map(|ty| crate::value_type(env.isa(), *ty))
- .collect();
+ .to_vec();
let payloads = returned_contref.args(env, builder);
let return_values = payloads.load_data_entries(env, builder, &return_types);
payloads.clear(env, builder, true);
@@ -1839,7 +2041,7 @@ fn load_resume_values_or_throw<'a>(
builder: &mut FunctionBuilder,
result: ControlEffect,
continuation: helpers::VMContRef,
- return_types: &[ir::Type],
+ return_types: &[WasmValType],
) -> WasmResult<Vec<ir::Value>> {
let throw_block = builder.create_block();
let values_block = builder.create_block();
@@ -1854,7 +2056,11 @@ fn load_resume_values_or_throw<'a>(
let values = continuation.values(env, builder);
// Exception references use Wasmtime's compressed, 32-bit GC-reference
// representation.
- let exnref = values.load_data_entries(env, builder, &[I32])[0];
+ let exnref_ty = WasmValType::Ref(WasmRefType {
+ nullable: false,
+ heap_type: WasmHeapType::Exn,
+ });
+ let exnref = values.load_data_entries(env, builder, &[exnref_ty])[0];
values.clear(env, builder, true);
gc::translate_exn_throw_ref(env, builder, exnref)?;
@@ -1871,8 +2077,10 @@ pub(crate) fn translate_suspend<'a>(
builder: &mut FunctionBuilder,
tag_index: u32,
suspend_args: &[ir::Value],
- tag_return_types: &[ir::Type],
+ suspend_arg_types: &[WasmValType],
+ tag_return_types: &[WasmValType],
) -> WasmResult<Vec<ir::Value>> {
+ debug_assert_eq!(suspend_args.len(), suspend_arg_types.len());
let tag_addr = tag_address(env, builder, tag_index);
let vmctx = env.vmctx_val(&mut builder.cursor());
@@ -1894,16 +2102,21 @@ pub(crate) fn translate_suspend<'a>(
// return values including the possible exception reference.
let values = active_contref.values(env, builder);
let required_capacity = u32::try_from(std::cmp::max(
- 1, // This account for the possible exception reference.
+ 1, // This accounts for the possible exception reference.
std::cmp::max(suspend_args.len(), tag_return_types.len()),
))
.expect("Number of stack switching payloads should fit in u32");
- env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot(
+ let needs_gc_ref_markers =
+ types_need_gc_ref_markers(suspend_arg_types) || types_need_gc_ref_markers(tag_return_types);
+ let existing_storage = env.stack_switching_values_storage;
+ env.stack_switching_values_storage = Some(values.prepare_stack_storage(
env,
builder,
required_capacity,
- env.stack_switching_values_buffer,
+ suspend_arg_types,
+ needs_gc_ref_markers,
+ existing_storage,
));
if suspend_args.len() > 0 {
@@ -1933,13 +2146,20 @@ pub(crate) fn translate_suspend<'a>(
// A normal resume supplies the tag's return values. `resume_throw_ref`
// supplies an exception reference and throws it at this suspension point.
- load_resume_values_or_throw(
+ let return_values = load_resume_values_or_throw(
env,
builder,
ControlEffect::from_u64(result),
active_contref,
tag_return_types,
- )
+ );
+
+ if needs_gc_ref_markers {
+ let zero = builder.ins().iconst(env.pointer_type(), 0);
+ values.set_gc_ref_data(env, builder, zero);
+ }
+
+ return_values
}
pub(crate) fn translate_switch<'a>(
@@ -1948,7 +2168,8 @@ pub(crate) fn translate_switch<'a>(
tag_index: u32,
switchee_contobj: ir::Value,
switch_args: &[ir::Value],
- return_types: &[ir::Type],
+ switch_arg_types: &[WasmValType],
+ return_types: &[WasmValType],
) -> WasmResult<Vec<ir::Value>> {
let vmctx = env.vmctx_val(&mut builder.cursor());
@@ -1978,6 +2199,7 @@ pub(crate) fn translate_switch<'a>(
// `switcher_contref`) to the immediate child (called
// `switcher_contref_last_ancestor`) of the stack with the corresponding
// handler (saved in `handler_stack_chain`).
+ let return_values_need_gc_ref_markers = types_need_gc_ref_markers(return_types);
let (
switcher_contref,
switcher_contobj,
@@ -2004,11 +2226,17 @@ pub(crate) fn translate_switch<'a>(
// reference.
let values = switcher_contref.values(env, builder);
let required_capacity = u32::try_from(std::cmp::max(1, return_types.len())).unwrap();
- env.stack_switching_values_buffer = Some(values.allocate_or_reuse_stack_slot(
+ let existing_storage = env.stack_switching_values_storage;
+ env.stack_switching_values_storage = Some(values.prepare_stack_storage(
env,
builder,
required_capacity,
- env.stack_switching_values_buffer,
+ // No GC ref marking is required at this point. Upon
+ // return to this site `vmcontref_store_payloads` will
+ // store the markers, if needed.
+ &[],
+ return_values_need_gc_ref_markers,
+ existing_storage,
));
let switcher_contref_csi = switcher_contref.common_stack_information(env, builder);
@@ -2054,7 +2282,13 @@ pub(crate) fn translate_switch<'a>(
let (switchee_contref_csi, switchee_contref_last_ancestor) = {
let mut combined_payloads = switch_args.to_vec();
combined_payloads.push(switcher_contobj);
- vmcontref_store_payloads(env, builder, &combined_payloads, switchee_contref.address);
+ vmcontref_store_payloads(
+ env,
+ builder,
+ &combined_payloads,
+ switch_arg_types,
+ switchee_contref.address,
+ );
let switchee_contref_csi = switchee_contref.common_stack_information(env, builder);
switchee_contref_csi.set_state_running(env, builder);
@@ -2183,11 +2417,20 @@ pub(crate) fn translate_switch<'a>(
// After switching back to the original stack, either load the values
// supplied by an ordinary resume or throw the injected exception.
- load_resume_values_or_throw(
+ let return_values = load_resume_values_or_throw(
env,
builder,
ControlEffect::from_u64(result),
switcher_contref,
return_types,
- )
+ );
+
+ if return_values_need_gc_ref_markers {
+ let zero = builder.ins().iconst(env.pointer_type(), 0);
+ switcher_contref
+ .values(env, builder)
+ .set_gc_ref_data(env, builder, zero);
+ }
+
+ return_values
}
diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs
index c0c37e96cf..4ed39e1cab 100644
--- a/crates/cranelift/src/translate/code_translator.rs
+++ b/crates/cranelift/src/translate/code_translator.rs
@@ -3168,25 +3168,14 @@ pub fn translate_operator(
let arg_count = src_types.len() - dst_arity;
let arg_types = &src_types[0..arg_count];
- for arg_type in arg_types {
- // We can't bind GC objects using cont.bind at the moment: We
- // don't have the necessary infrastructure to traverse the
- // buffers used by cont.bind when looking for GC roots. Thus,
- // this crude check ensures that these buffers can never contain
- // GC roots to begin with.
- if arg_type.is_vmgcref_type_and_not_i31() {
- return Err(wasmtime_environ::WasmError::Unsupported(
- "cont.bind does not support GC types at the moment".into(),
- ));
- }
- }
-
let (original_contobj, args) =
environ.stacks.peekn(arg_count + 1).split_last().unwrap();
let original_contobj = *original_contobj;
let args = args.to_vec();
- let new_contobj = environ.translate_cont_bind(builder, original_contobj, &args);
+ let arg_types = arg_types.to_vec();
+ let new_contobj =
+ environ.translate_cont_bind(builder, original_contobj, &args, &arg_types);
environ.stacks.popn(arg_count + 1);
environ.stacks.push1(new_contobj);
@@ -3194,17 +3183,18 @@ pub fn translate_operator(
Operator::Suspend { tag_index } => {
let tag_index = TagIndex::from_u32(*tag_index);
let param_types = environ.tag_params(tag_index).to_vec();
- let return_types: SmallVec<[_; 8]> = environ
- .tag_returns(tag_index)
- .iter()
- .map(|ty| crate::value_type(environ.isa(), *ty))
- .collect();
+ let return_types = environ.tag_returns(tag_index).to_vec();
let params = environ.stacks.peekn(param_types.len()).to_vec();
let param_count = params.len();
- let return_values =
- environ.translate_suspend(builder, tag_index.as_u32(), &params, &return_types)?;
+ let return_values = environ.translate_suspend(
+ builder,
+ tag_index.as_u32(),
+ &params,
+ &param_types,
+ &return_types,
+ )?;
environ.stacks.popn(param_count);
environ.stacks.pushn(&return_values);
@@ -3326,45 +3316,43 @@ pub fn translate_operator(
tag_index,
} => {
// Arguments of the continuation we are going to switch to
- let continuation_argument_types: SmallVec<[_; 8]> = environ
+ let switch_arg_types: SmallVec<[_; 8]> = environ
.continuation_arguments(TypeIndex::from_u32(*cont_type_index))
.to_smallvec();
// Arity includes the continuation argument
- let arity = continuation_argument_types.len();
+ let arity = switch_arg_types.len();
let (contobj, switch_args) = environ.stacks.peekn(arity).split_last().unwrap();
let contobj = *contobj;
let switch_args = switch_args.to_vec();
// Type of the continuation we are going to create by suspending the
// currently running stack
- let current_continuation_type = continuation_argument_types.last().unwrap();
+ let current_continuation_type = switch_arg_types.last().unwrap();
let current_continuation_type = current_continuation_type.unwrap_ref_type();
// Argument types of current_continuation_type. These will in turn
// be the types of the arguments we receive when someone switches
// back to this switch instruction
- let current_continuation_arg_types: SmallVec<[_; 8]> =
- match current_continuation_type.heap_type {
- WasmHeapType::ConcreteCont(index) => {
- let mti = index
- .as_module_type_index()
- .expect("expected module-local type index");
-
- environ
- .continuation_arguments_from_interned(mti)
- .iter()
- .map(|ty| crate::value_type(environ.isa(), *ty))
- .collect()
- }
- _ => panic!("Invalid type on switch"),
- };
+ let return_types: SmallVec<[_; 8]> = match current_continuation_type.heap_type {
+ WasmHeapType::ConcreteCont(index) => {
+ let mti = index
+ .as_module_type_index()
+ .expect("expected module-local type index");
+
+ environ
+ .continuation_arguments_from_interned(mti)
+ .to_smallvec()
+ }
+ _ => panic!("Invalid type on switch"),
+ };
let switch_return_values = environ.translate_switch(
builder,
*tag_index,
contobj,
&switch_args,
- &current_continuation_arg_types,
+ &switch_arg_types,
+ &return_types,
)?;
environ.stacks.popn(arity);
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index d59f3c16af..fc390ef509 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -154,7 +154,7 @@ macro_rules! foreach_builtin_function {
// Creates a new continuation from a funcref.
#[cfg(feature = "stack-switching")]
- cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32) -> pointer;
+ cont_new(vmctx: vmctx, r: pointer, param_count: u32, result_count: u32, gc_refs: u32) -> pointer;
// Return the instance ID for a given vmctx.
#[cfg(feature = "gc")]
@@ -170,6 +170,23 @@ macro_rules! foreach_builtin_function {
// Process a debug breakpoint.
breakpoint(vmctx: vmctx) -> bool;
+
+ // Intern a continuation reference into the GC heap's side table.
+ #[cfg(all(feature = "gc", feature = "stack-switching"))]
+ intern_contref_for_gc_heap(
+ vmctx: vmctx,
+ contref: pointer,
+ revision: pointer
+ ) -> u64;
+
+ // Resolve an interned continuation-reference ID and write its
+ // pointer and revision witness to `result`.
+ #[cfg(all(feature = "gc", feature = "stack-switching"))]
+ get_interned_contref(
+ vmctx: vmctx,
+ contref_id: u32,
+ result: pointer
+ ) -> bool;
}
};
}
@@ -387,6 +404,7 @@ impl BuiltinFunctionIndex {
(@get ref_func pointer) => (return None);
(@get table_get_lazy_init_func_ref pointer) => (return None);
(@get intern_func_ref_for_gc_heap u64) => (return None);
+ (@get intern_contref_for_gc_heap u64) => (return None);
(@get is_subtype u32) => (return None);
(@get ceil_f32 f32) => (return None);
(@get ceil_f64 f64) => (return None);
diff --git a/crates/environ/src/stack_switching.rs b/crates/environ/src/stack_switching.rs
index 8f43cbdd84..a0d1c9d7de 100644
--- a/crates/environ/src/stack_switching.rs
+++ b/crates/environ/src/stack_switching.rs
@@ -33,6 +33,10 @@ pub const STACK_STATE_TRAPPED_DISCRIMINANT: u32 = 5;
/// Discriminant of variant `Return` in
/// `runtime::vm::ControlEffect`.
pub const CONTROL_EFFECT_RETURN_DISCRIMINANT: u32 = 0;
+
+/// Marker for a continuation payload slot that contains a GC-managed
+/// reference and must be traced while the continuation is suspended.
+pub const CONTINUATION_PAYLOAD_GC_REF: u8 = 1;
/// Discriminant of variant `Resume` in
/// `runtime::vm::ControlEffect`.
pub const CONTROL_EFFECT_RESUME_DISCRIMINANT: u32 = 1;
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 139309f46d..741bf7e23d 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -71,6 +71,7 @@ macro_rules! define_vm_type_offsets {
(@size ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().size()) };
(@size ($p:expr) VMStackLimits) => { u32::from(($p).vm_stack_limits().size()) };
(@size ($p:expr) VMHostArray) => { u32::from(($p).vm_host_array().size()) };
+ (@size ($p:expr) VMPayloads) => { u32::from(($p).vm_payloads().size()) };
(@size ($p:expr) VMCommonStackInformation) => {
u32::from(($p).vm_common_stack_information().size())
};
@@ -116,6 +117,7 @@ macro_rules! define_vm_type_offsets {
(@align ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().align()) };
(@align ($p:expr) VMStackLimits) => { u32::from(($p).vm_stack_limits().align()) };
(@align ($p:expr) VMHostArray) => { u32::from(($p).vm_host_array().align()) };
+ (@align ($p:expr) VMPayloads) => { u32::from(($p).vm_payloads().align()) };
(@align ($p:expr) VMCommonStackInformation) => {
u32::from(($p).vm_common_stack_information().align())
};
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
index aca8cdcc37..ab7ac0675b 100644
--- a/crates/environ/src/vmtypes.rs
+++ b/crates/environ/src/vmtypes.rs
@@ -602,6 +602,21 @@ macro_rules! for_each_vm_type {
pub data: *mut u8,
}
+ /// Payload values exchanged with a continuation and the metadata
+ /// needed to trace GC references among them.
+ #[derive(Debug, Clone)]
+ #[repr(C)]
+ #[snake_name = vm_payloads]
+ pub struct VMPayloads {
+ /// The payload values themselves.
+ #[aggregate]
+ pub buffer: VMHostArray,
+
+ /// One marker byte per buffer slot, indicating whether that
+ /// slot contains a GC reference, or `None` when no slots do.
+ pub gc_ref_data: Option<VmPtr<u8>>,
+ }
+
/// The information saved for every stack, whether it is a
/// continuation's or the initial stack's.
#[derive(Debug, Clone)]
@@ -651,12 +666,12 @@ macro_rules! for_each_vm_type {
/// The arguments to, and return values of, the function passed
/// to `cont.new`.
#[aggregate]
- pub args: VMHostArray,
+ pub args: VMPayloads,
/// The payloads passed to and from this continuation once it
/// has been suspended.
#[aggregate]
- pub values: VMHostArray,
+ pub values: VMPayloads,
/// Tells the compiler that this structure has potential
/// self-references, through `last_ancestor`.
@@ -667,6 +682,19 @@ macro_rules! for_each_vm_type {
pub _marker: PhantomPinned,
}
+ /// A slight variation of `VMContObj` which allows the
+ /// `contref` to be instantiated to null,
+ /// i.e. `Option::None`. This representation is used by
+ /// the GC infrastructure to construct a canonical
+ /// null-esque continuation object.
+ #[cfg(all(feature = "gc", feature = "stack-switching"))]
+ #[repr(C)]
+ #[snake_name = vm_raw_cont_obj]
+ pub struct VMRawContObj {
+ pub contref: Option<VmPtr<u8>>,
+ pub revision: usize,
+ }
+
/// The deferred-reference-counting collector's JIT-accessible heap
/// data.
///
diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs
index 5c84a41807..09a63ba2cf 100644
--- a/crates/wasmtime/src/runtime/store/gc.rs
+++ b/crates/wasmtime/src/runtime/store/gc.rs
@@ -836,22 +836,38 @@ impl StoreOpaque {
#[cfg(feature = "stack-switching")]
fn trace_wasm_continuation_roots(&mut self, gc_roots_list: &mut GcRootsList) {
- use crate::vm::VMStackState;
+ use crate::vm::{VMPayloads, VMStackState, ValRaw};
+
+ unsafe fn trace_payload_roots(gc_roots_list: &mut GcRootsList, payloads: &VMPayloads) {
+ let gc_ref_data = payloads.gc_ref_data;
+ let payloads = &payloads.buffer;
+ assert!(payloads.length <= payloads.capacity);
+ let Some(gc_ref_data) = gc_ref_data else {
+ return;
+ };
+ let gc_ref_data = gc_ref_data.as_ptr();
+
+ for index in 0..usize::try_from(payloads.length).unwrap() {
+ let marker = unsafe { gc_ref_data.add(index).read() };
+ if marker == wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF {
+ let slot = unsafe { payloads.data.cast::<ValRaw>().add(index).cast::<u32>() };
+ unsafe {
+ StoreOpaque::trace_wasm_stack_slot(gc_roots_list, slot);
+ }
+ }
+ }
+ }
log::trace!("Begin trace GC roots :: continuations");
for continuation in &self.continuations {
let state = continuation.common_stack_information.state;
- // FIXME(frank-emrich) In general, it is not enough to just trace
- // through the stacks of continuations; we also need to look through
- // their `cont.bind` arguments. However, we don't currently have
- // enough RTTI information to check if any of the values in the
- // buffers used by `cont.bind` are GC values. As a workaround, note
- // that we currently disallow cont.bind-ing GC values altogether.
- // This way, it is okay not to check them here.
match state {
VMStackState::Suspended => {
+ unsafe {
+ trace_payload_roots(gc_roots_list, &continuation.values);
+ }
Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| {
Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame);
core::ops::ControlFlow::Continue(())
@@ -865,8 +881,11 @@ impl StoreOpaque {
// either case things should be handled correctly when traversing
// further along in the chain, nothing required at this point.
}
- VMStackState::Fresh | VMStackState::Returned | VMStackState::Trapped => {
- // Fresh/terminal continuations have no live GC values on their stack.
+ VMStackState::Fresh => unsafe {
+ trace_payload_roots(gc_roots_list, &continuation.args);
+ },
+ VMStackState::Returned | VMStackState::Trapped => {
+ // Terminal continuations have no live GC values.
}
}
}
diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs
index e7f0c94d93..10526a1499 100644
--- a/crates/wasmtime/src/runtime/vm.rs
+++ b/crates/wasmtime/src/runtime/vm.rs
@@ -137,7 +137,7 @@ pub use crate::runtime::vm::vmcontext::VMNullHeapData;
pub use crate::runtime::vm::vmcontext::{
VMArrayCallHostFuncContext, VMCommonStackInformation, VMContRef, VMContext, VMFuncRef,
VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMHostArray,
- VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMStackLimits, VMStoreContext,
+ VMMemoryDefinition, VMMemoryImport, VMOpaqueContext, VMPayloads, VMStackLimits, VMStoreContext,
VMTableImport, VMTagImport, VMWasmCallFunction, ValRaw,
};
#[cfg(has_custom_sync)]
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index 5b0aae7248..28bf087928 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -8,6 +8,8 @@ mod disabled;
#[cfg(not(feature = "gc"))]
pub use disabled::*;
+#[cfg(feature = "stack-switching")]
+mod cont_ref;
mod data;
mod func_ref;
mod gc_ref;
@@ -15,6 +17,8 @@ mod gc_runtime;
mod host_data;
mod i31;
+#[cfg(feature = "stack-switching")]
+pub use cont_ref::*;
pub use data::*;
pub use func_ref::*;
pub use gc_ref::*;
@@ -54,6 +58,10 @@ pub struct GcStore {
/// The function-references table for this GC heap.
pub func_ref_table: FuncRefTable,
+ /// The continuation references table for this GC heap.
+ #[cfg(feature = "stack-switching")]
+ pub cont_ref_table: ContRefTable,
+
/// The total allocated bytes recorded after the last GC collection.
/// `None` if no collection has been performed yet. Used by the
/// grow-or-collect heuristic.
@@ -96,6 +104,8 @@ impl GcStore {
) -> Self {
let host_data_table = ExternRefHostDataTable::default();
let func_ref_table = FuncRefTable::default();
+ #[cfg(feature = "stack-switching")]
+ let cont_ref_table = ContRefTable::default();
let _ = &gc_zeal_alloc_counter;
@@ -104,6 +114,8 @@ impl GcStore {
gc_heap,
host_data_table,
func_ref_table,
+ #[cfg(feature = "stack-switching")]
+ cont_ref_table,
last_post_gc_allocated_bytes: None,
#[cfg(gc_zeal)]
gc_zeal_alloc_counter,
diff --git a/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs b/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs
new file mode 100644
index 0000000000..48c01b4935
--- /dev/null
+++ b/crates/wasmtime/src/runtime/vm/gc/cont_ref.rs
@@ -0,0 +1,56 @@
+//! Side table for continuation references stored in the GC heap.
+//!
+//! This follows the same idea as `VMFuncRef` and `externref`. We
+//! cannot trust native addresses that come out of the GC heap.
+
+use crate::{Result, bail_bug, hash_map::HashMap, vm::VMContObj};
+use wasmtime_core::{
+ alloc::PanicOnOom,
+ slab::{Id, Slab},
+};
+
+/// Side table mapping the IDs stored in GC fields to continuation
+/// objects.
+///
+/// Raw ID zero is reserved for a null continuation reference.
+/// Non-null IDs are one greater than their underlying slab IDs.
+#[derive(Default)]
+pub struct ContRefTable {
+ interned: HashMap<VMContObj, u32>,
+ slab: Slab<VMContObj>,
+}
+
+impl ContRefTable {
+ /// Intern a continuation object and return the ID to store in the
+ /// GC heap. Null continuation references use the reserved ID
+ /// zero.
+ ///
+ /// # Safety
+ ///
+ /// A non-null continuation's `VMContRef` pointer must remain valid for the
+ /// duration of this table's lifetime.
+ pub unsafe fn intern(&mut self, contobj: Option<VMContObj>) -> u32 {
+ let Some(contobj) = contobj else {
+ return 0;
+ };
+
+ *self.interned.entry(contobj).or_insert_with(|| {
+ // TODO(dhil): Handle allocation failure here rather than panicking.
+ let id = self.slab.alloc(contobj).panic_on_oom().into_raw();
+ id.checked_add(1).unwrap()
+ })
+ }
+
+ /// Resolve an ID loaded from the GC heap.
+ pub fn get(&self, raw: u32) -> Result<Option<VMContObj>> {
+ if raw == 0 {
+ return Ok(None);
+ }
+
+ let id = Id::from_raw(raw - 1);
+ match self.slab.get(id).copied() {
+ Some(contobj) => Ok(Some(contobj)),
+ None => bail_bug!("bad continuation-reference table ID"),
+ }
+ }
+}
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 6964e405d6..157cf8095b 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -59,7 +59,11 @@ use crate::prelude::*;
use crate::runtime::store::{Asyncness, InstanceId, StoreOpaque};
#[cfg(feature = "gc")]
use crate::runtime::vm::VMGcRef;
+#[cfg(all(feature = "gc", feature = "stack-switching"))]
+use crate::runtime::vm::provenance::VmPtr;
use crate::runtime::vm::{self, HostResultHasUnwindSentinel, VMStore, f32x4, f64x2, i8x16};
+#[cfg(all(feature = "gc", feature = "stack-switching"))]
+use crate::vm::vmcontext::VMRawContObj;
use core::convert::Infallible;
use core::ptr::NonNull;
#[cfg(feature = "threads")]
@@ -604,6 +608,53 @@ fn get_interned_func_ref(
Ok(func_ref.map_or(core::ptr::null_mut(), |f| f.as_ptr().cast()))
}
+// Intern a continuation reference into the GC heap's side table.
+//
+// This libcall may not GC.
+#[cfg(all(feature = "gc", feature = "stack-switching"))]
+unsafe fn intern_contref_for_gc_heap(
+ store: &mut dyn VMStore,
+ _instance: InstanceId,
+ contref: *mut u8,
+ revision: *mut u8,
+) -> Result<u32> {
+ use crate::store::AutoAssertNoGc;
+
+ let mut store = AutoAssertNoGc::new(store.store_opaque_mut());
+ let contobj = unsafe { crate::vm::VMContObj::from_raw_parts(contref, revision.addr()) };
+ let id = unsafe { store.require_gc_store_mut()?.cont_ref_table.intern(contobj) };
+ Ok(id)
+}
+
+// Resolve a continuation reference ID loaded from the GC heap and
+// write its 16 bytes value into caller provided storage `out_result`.
+//
+// This libcall may not GC.
+#[cfg(all(feature = "gc", feature = "stack-switching"))]
+unsafe fn get_interned_contref(
+ store: &mut dyn VMStore,
+ _instance: InstanceId,
+ contref_id: u32,
+ out_result: *mut u8,
+) -> Result<()> {
+ use crate::store::AutoAssertNoGc;
+
+ let store = AutoAssertNoGc::new(store.store_opaque_mut());
+ let contobj = store.unwrap_gc_store().cont_ref_table.get(contref_id)?;
+ let raw = match contobj {
+ Some(contobj) => VMRawContObj {
+ contref: NonNull::new(contobj.contref.as_ptr().cast::<u8>()).map(VmPtr::from),
+ revision: contobj.revision,
+ },
+ None => VMRawContObj {
+ contref: None,
+ revision: 0,
+ },
+ };
+ unsafe { out_result.cast::<VMRawContObj>().write(raw) };
+ Ok(())
+}
+
#[cfg(feature = "gc")]
fn is_subtype(
store: &mut dyn VMStore,
@@ -1119,9 +1170,16 @@ fn cont_new(
func: *mut u8,
param_count: u32,
result_count: u32,
+ gc_refs: u32,
) -> Result<Option<AllocationSize>> {
- let ans =
- crate::vm::stack_switching::cont_new(store, instance, func, param_count, result_count)?;
+ let ans = crate::vm::stack_switching::cont_new(
+ store,
+ instance,
+ func,
+ param_count,
+ result_count,
+ gc_refs != 0,
+ )?;
Ok(Some(AllocationSize(ans.cast::<u8>() as usize)))
}
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs
index 8a71d52e7c..06f61340d3 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs
@@ -3,7 +3,7 @@
mod stack;
-use crate::vm::{VMCommonStackInformation, VMContRef, VMHostArray, VMStackLimits};
+use crate::vm::{VMCommonStackInformation, VMContRef, VMHostArray, VMPayloads, VMStackLimits};
use core::{marker::PhantomPinned, ptr::NonNull};
pub use stack::*;
@@ -51,7 +51,7 @@ pub const CONTROL_EFFECT_TRAP_ENCODING: u64 =
/// (i.e., the one pointed to by the VMContObj) has a pointer to the
/// other end of the chain (i.e., its last ancestor).
#[repr(C)]
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct VMContObj {
pub contref: NonNull<VMContRef>,
pub revision: usize,
@@ -109,6 +109,27 @@ impl VMHostArray {
data: core::ptr::null_mut(),
}
}
+
+ /// Makes this array empty.
+ pub fn clear(&mut self) {
+ *self = Self::empty();
+ }
+}
+
+impl VMPayloads {
+ /// Creates an empty payload buffer with no GC metadata.
+ pub fn empty() -> Self {
+ Self {
+ buffer: VMHostArray::empty(),
+ gc_ref_data: None,
+ }
+ }
+
+ /// Makes this payload buffer empty and invalidates its GC metadata.
+ pub fn clear(&mut self) {
+ self.buffer.clear();
+ self.gc_ref_data = None;
+ }
}
impl VMContRef {
@@ -135,8 +156,8 @@ impl VMContRef {
let parent_chain = VMStackChain::Absent;
let last_ancestor = core::ptr::null_mut();
let stack = VMContinuationStack::unallocated();
- let args = VMHostArray::empty();
- let values = VMHostArray::empty();
+ let args = VMPayloads::empty();
+ let values = VMPayloads::empty();
let revision = 0;
let _marker = PhantomPinned;
@@ -180,6 +201,7 @@ pub fn cont_new(
func: *mut u8,
param_count: u32,
result_count: u32,
+ gc_refs: bool,
) -> crate::Result<*mut VMContRef> {
let instance = store.instance_mut(instance);
let caller_vmctx = instance.vmctx();
@@ -197,14 +219,14 @@ pub fn cont_new(
// The initialization function will allocate the actual args/return value buffer and
// update this object (if needed).
- let contref_args_ptr = &mut contref.args as *mut VMHostArray;
-
+ let contref_args_ptr = &mut contref.args as *mut VMPayloads;
contref.stack.initialize(
func.cast::<crate::vm::VMFuncRef>(),
caller_vmctx.as_ptr(),
contref_args_ptr,
param_count,
result_count,
+ gc_refs,
)?;
// Now that the initial stack pointer was set by the initialization
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs
index fc24418c59..e8438a91d4 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack.rs
@@ -4,7 +4,7 @@
use crate::Result;
use core::ops::Range;
-use crate::runtime::vm::VMHostArray;
+use crate::runtime::vm::VMPayloads;
use crate::runtime::vm::{VMContext, VMFuncRef};
cfg_select! {
@@ -95,7 +95,9 @@ impl VMContinuationStack {
/// Initializes this stack, such that it will execute the function denoted
/// by `func_ref`. `parameter_count` and `return_value_count` must be the
/// corresponding number of parameters and return values of `func_ref`.
- /// `args` must point to the `args` field of the `VMContRef` owning this pointer.
+ /// `args` must point to the `args` field of the `VMContRef` owning this
+ /// stack. When `gc_refs` is true, its `gc_ref_data` field receives the
+ /// corresponding root-marker buffer.
///
/// It will be updated by this function to correctly describe
/// the buffer used by this function for its arguments and return values.
@@ -103,9 +105,10 @@ impl VMContinuationStack {
&self,
func_ref: *const VMFuncRef,
caller_vmctx: *mut VMContext,
- args: *mut VMHostArray,
+ args: *mut VMPayloads,
parameter_count: u32,
return_value_count: u32,
+ gc_refs: bool,
) -> Result<()> {
self.0.initialize(
func_ref,
@@ -113,6 +116,7 @@ impl VMContinuationStack {
args,
parameter_count,
return_value_count,
+ gc_refs,
)
}
}
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs
index 5c2cf049e4..1e5781f17b 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs
@@ -1,7 +1,7 @@
use crate::Result;
use core::ops::Range;
-use crate::runtime::vm::VMHostArray;
+use crate::runtime::vm::VMPayloads;
use crate::runtime::vm::{VMContext, VMFuncRef};
/// Making sure that this has the same size as the non-dummy version, to
@@ -59,9 +59,10 @@ impl VMContinuationStack {
&self,
_func_ref: *const VMFuncRef,
_caller_vmctx: *mut VMContext,
- _args: *mut VMHostArray,
+ _args: *mut VMPayloads,
_parameter_count: u32,
_return_value_count: u32,
+ _gc_refs: bool,
) -> Result<()> {
Ok(())
}
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs
index 247495a599..580b104653 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs
@@ -64,8 +64,7 @@ use std::ops::Range;
use std::ptr;
use crate::prelude::*;
-use crate::runtime::vm::VMHostArray;
-use crate::runtime::vm::{VMContext, VMFuncRef, ValRaw};
+use crate::runtime::vm::{VMContext, VMFuncRef, VMHostArray, VMPayloads, ValRaw, VmPtr};
#[derive(Debug, PartialEq, Eq)]
pub enum Allocator {
@@ -194,11 +193,14 @@ impl VMContinuationStack {
/// calls `fiber_start` with the following arguments:
/// TOS, func_ref, caller_vmctx, args_ptr, args_capacity
///
- /// Note that at this point we also allocate the args buffer
- /// (see picture at the top of this file).
+ /// Note that at this point we also allocate persistent launch storage for
+ /// the args buffer and, when `gc_refs` is true, its parallel
+ /// GC-reference-marker buffer (see picture at the top of this file).
/// We define `args_capacity` as the max of parameter and return value count.
- /// Then the size s of the actual buffer size is calculated as follows:
- /// s = size_of(ValRaw) * `args_capacity`,
+ /// Their combined, 16-byte-aligned size `s` is calculated as follows when
+ /// `gc_refs` is true:
+ /// s = size_of(ValRaw) * `args_capacity`
+ /// + align_up(`args_capacity`, 16),
///
/// Note that this value is used below, and we may have s = 0.
///
@@ -222,15 +224,22 @@ impl VMContinuationStack {
/// ---------------|-------------------------------------------------------
/// -0x28 - s | func_ref
/// -0x30 - s | caller_vmctx
- /// -0x38 - s | args (of type *mut ArrayRef<ValRaw>)
+ /// -0x38 - s | args (of type *mut VMHostArray)
/// -0x40 - s | return_value_count
+ ///
+ /// The saved RSP points at `TOS - 0x40 - s`. On entry,
+ /// `wasmtime_continuation_start` pops the four launch-record words, leaving
+ /// RSP at `TOS - 0x20 - s`, immediately below the persistent buffers.
+ /// Subsequent stack frames grow towards lower addresses and may reuse the
+ /// consumed launch-record storage.
pub fn initialize(
&self,
func_ref: *const VMFuncRef,
caller_vmctx: *mut VMContext,
- args: *mut VMHostArray,
+ args: *mut VMPayloads,
parameter_count: u32,
return_value_count: u32,
+ gc_refs: bool,
) -> Result<()> {
let tos = self.top;
@@ -240,22 +249,50 @@ impl VMContinuationStack {
target.write(value)
};
- let args_ref = &mut *args;
+ let payloads = &mut *args;
+ let args_ref = &mut payloads.buffer;
let args_capacity = std::cmp::max(parameter_count, return_value_count);
// The args object must currently be empty.
debug_assert_eq!(args_ref.capacity, 0);
debug_assert_eq!(args_ref.length, 0);
- let total_control_size = usize::try_from(args_capacity)?
+ let args_data_size = usize::try_from(args_capacity)?
.checked_mul(std::mem::size_of::<ValRaw>())
- .and_then(|s| s.checked_add(0x40))
.ok_or_else(|| {
format_err!(
"continuation function type with {args_capacity} args \
overflows stack control data size calculation"
)
})?;
- let args_data_size = total_control_size - 0x40;
+ // Keep the fixed startup data 16-byte aligned.
+ let gc_refs_data_size = if cfg!(feature = "gc") && gc_refs {
+ usize::try_from(args_capacity)?
+ .checked_add(15)
+ .map(|s| s & !15)
+ .ok_or_else(|| {
+ format_err!(
+ "continuation function type with {args_capacity} args \
+ overflows stack control data size calculation"
+ )
+ })?
+ } else {
+ 0
+ };
+ let dynamic_data_size =
+ args_data_size
+ .checked_add(gc_refs_data_size)
+ .ok_or_else(|| {
+ format_err!(
+ "continuation function type with {args_capacity} args \
+ overflows stack control data size calculation"
+ )
+ })?;
+ let total_control_size = dynamic_data_size.checked_add(0x40).ok_or_else(|| {
+ format_err!(
+ "continuation function type with {args_capacity} args \
+ overflows stack control data size calculation"
+ )
+ })?;
// Ensure the control data (fixed header + args buffer) fits
// within the usable stack space. For Mmap allocations,
@@ -282,6 +319,17 @@ impl VMContinuationStack {
args_ref.capacity = args_capacity;
args_ref.data = args_data_ptr;
+ if cfg!(feature = "gc") && gc_refs {
+ let data = if args_capacity == 0 {
+ ptr::null_mut()
+ } else {
+ tos.sub(0x20 + dynamic_data_size)
+ };
+ if args_capacity > 0 {
+ data.write_bytes(0, usize::try_from(args_capacity)?);
+ }
+ payloads.gc_ref_data = NonNull::new(data).map(VmPtr::from);
+ }
let to_store = [
// Data near top of stack:
@@ -290,10 +338,16 @@ impl VMContinuationStack {
(0x18, tos.sub(total_control_size).addr()),
(0x20, usize::try_from(args_capacity)?),
// Data after the args buffer:
- (0x28 + args_data_size, func_ref.addr()),
- (0x30 + args_data_size, caller_vmctx.addr()),
- (0x38 + args_data_size, args.addr()),
- (0x40 + args_data_size, usize::try_from(return_value_count)?),
+ (0x28 + dynamic_data_size, func_ref.addr()),
+ (0x30 + dynamic_data_size, caller_vmctx.addr()),
+ (
+ 0x38 + dynamic_data_size,
+ (args_ref as *mut VMHostArray).addr(),
+ ),
+ (
+ 0x40 + dynamic_data_size,
+ usize::try_from(return_value_count)?,
+ ),
];
for (offset, data) in to_store {
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs
index bc4f60a7d6..69337194d2 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix/x86_64.rs
@@ -34,6 +34,7 @@ pub fn wasmtime_continuation_start_address() -> *const () {
// values in various registers when execution of wasmtime_continuation_start begins:
//
// RSP: TOS - 0x40 - (16 * `args_capacity`)
+// - align_up(`args_capacity`, 16)
// RBP: TOS - 0x10
#[unsafe(naked)]
diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs
index d2d530e2fb..199efb34ba 100644
--- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs
+++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs
@@ -335,10 +335,15 @@ impl Backtrace {
let mut continuations_iter = unsafe { chain.into_continuation_iter() }.peekable();
while let Some(continuation_ptr) = continuations_iter.next() {
- let parent_limits_ptr = stack_limits_iter
- .next()
- .expect("expected one more VMStackLimits than continuations");
let continuation = unsafe { &*continuation_ptr };
+ let Some(parent_limits_ptr) = stack_limits_iter.next() else {
+ // A detached suspended continuation chain ends in `Absent`,
+ // unlike a running chain which ends in an initial stack. The
+ // currently suspended stack was already traced above, and the
+ // last continuation has no parent stack left to trace.
+ debug_assert_eq!(continuation.parent_chain, VMStackChain::Absent);
+ break;
+ };
let parent_limits = unsafe { &*parent_limits_ptr };
// The parent of `continuation` if present and not the last in the chain.
diff --git a/tests/all/stack_switching.rs b/tests/all/stack_switching.rs
index dfc33714f8..3a075d4719 100644
--- a/tests/all/stack_switching.rs
+++ b/tests/all/stack_switching.rs
@@ -339,6 +339,71 @@ fn inter_instance_suspend() -> Result<()> {
Ok(())
}
+/// Regression test for https://github.com/bytecodealliance/wasmtime/issues/13750.
+///
+/// A GC reference returned through a continuation payload must retain
+/// its stack-map metadata while it is live on the operand stack
+/// across subsequent GC safepoints.
+#[cfg_attr(any(asan, miri), ignore)]
+#[test]
+fn continuation_result_gc_ref_on_operand_stack() -> Result<()> {
+ let mut config = Config::new();
+ config
+ .wasm_stack_switching(true)
+ .wasm_exceptions(true)
+ .wasm_function_references(true)
+ .wasm_gc(true)
+ .collector(Collector::Copying);
+
+ #[cfg(gc_zeal)]
+ {
+ let _ = config.gc_zeal_alloc_counter(std::num::NonZeroU32::new(1));
+ }
+
+ let engine = Engine::new(&config)?;
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (type $S (struct (field $x i32)))
+ (type $A (array i64))
+ (type $ft (func (result (ref $S))))
+ (type $ct (cont $ft))
+
+ (func $target (result (ref $S))
+ (struct.new $S (i32.const 0x12345678)))
+ (elem declare func $target)
+
+ (func (export "run") (result i32)
+ (local $k (ref null $ct))
+ (local $i i32)
+ (local.set $k (cont.new $ct (ref.func $target)))
+
+ ;; Keep the continuation's GC-reference result on the operand
+ ;; stack, rather than storing it in a GC-typed local.
+ (resume $ct (local.get $k))
+
+ ;; Force collections while that operand-stack value is the
+ ;; only live reference to the struct.
+ (local.set $i (i32.const 0))
+ (loop $again
+ (drop (array.new_default $A (i32.const 256)))
+ (local.set $i (i32.add (local.get $i) (i32.const 1)))
+ (br_if $again
+ (i32.lt_u (local.get $i) (i32.const 2000))))
+
+ (struct.get $S $x))
+ )
+ "#,
+ )?;
+
+ let mut store = Store::new(&engine, ());
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let run = instance.get_typed_func::<(), i32>(&mut store, "run")?;
+ assert_eq!(run.call(&mut store, ())?, 0x12345678);
+ Ok(())
+}
+
/// Tests interaction with host functions. Note that the interaction with host
/// functions and traps is covered by the module `traps` further down.
mod host {
diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.wat
index 2606be8d12..1dcad97b8d 100644
--- a/tests/disas/stack-switching/resume-suspend-data-passing.wat
+++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat
@@ -61,7 +61,7 @@
;; block0(v0: i64, v1: i64):
;; @003c v3 = iconst.i32 10
;; @0044 v7 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @0044 v31 = iconst.i64 136
+;; @0044 v31 = iconst.i64 144
;; @0044 v34 = stack_addr.i64 ss0
;; @0044 v37 = iconst.i64 0
;; @0044 v47 = iconst.i64 96
@@ -118,8 +118,8 @@
;; block8:
;; @0044 store.i64 notrap aligned region7 v11, v9+80
;; v91 = iconst.i32 1
-;; v92 = iconst.i64 136
-;; v93 = iadd.i64 v9, v92 ; v92 = 136
+;; v92 = iconst.i64 144
+;; v93 = iadd.i64 v9, v92 ; v92 = 144
;; @0044 store notrap aligned region8 v91, v93+4 ; v91 = 1
;; @0044 store.i64 notrap aligned region4 v34, v93+8
;; @0044 store.i32 notrap aligned region6 v4, v34
@@ -203,7 +203,7 @@
;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
;; gv2 = load.i64 notrap aligned region1 gv1+24
;; sig0 = (i64 vmctx, i32) -> i64 tail
-;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail
;; sig2 = (i64 vmctx) tail
;; fn0 = colocated u805306368:6 sig0
;; fn1 = colocated u805306368:42 sig1
@@ -214,195 +214,195 @@
;; @0056 v2 = iconst.i32 0
;; @0056 v3 = call fn0(v0, v2) ; v2 = 0
;; @0058 trapz v3, user16
-;; @0058 v6 = call fn1(v0, v3, v2, v2) ; v2 = 0, v2 = 0
-;; @0058 v7 = load.i64 notrap aligned region2 v6+88
+;; @0058 v7 = call fn1(v0, v3, v2, v2, v2) ; v2 = 0, v2 = 0, v2 = 0
+;; @0058 v8 = load.i64 notrap aligned region2 v7+88
+;; @0058 v10 = uextend.i128 v8
+;; @0058 v11 = iconst.i64 64
+;; v139 = ishl v10, v11 ; v11 = 64
;; @0058 v9 = uextend.i128 v7
-;; @0058 v10 = iconst.i64 64
-;; v138 = ishl v9, v10 ; v10 = 64
-;; @0058 v8 = uextend.i128 v6
-;; @0058 v13 = bor v138, v8
-;; @0062 v22 = iconst.i64 1
-;; @0062 v25 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @0062 v28 = iconst.i64 0
-;; @0062 v29 = iconst.i64 2
-;; @0062 v33 = iconst.i32 1
-;; @0062 v34 = iconst.i32 2
-;; @0062 v48 = iconst.i64 40
-;; @0062 v51 = stack_addr.i64 ss0
-;; @0062 v52 = iconst.i64 48
-;; @0062 v53 = iadd v0, v52 ; v52 = 48
-;; @0062 v60 = iconst.i64 96
-;; @0062 v63 = iconst.i64 -24
-;; v142 = iconst.i64 0x0001_0000_0000
-;; @005c jump block2(v13)
+;; @0058 v14 = bor v139, v9
+;; @0062 v23 = iconst.i64 1
+;; @0062 v26 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @0062 v29 = iconst.i64 0
+;; @0062 v30 = iconst.i64 2
+;; @0062 v34 = iconst.i32 1
+;; @0062 v35 = iconst.i32 2
+;; @0062 v49 = iconst.i64 40
+;; @0062 v52 = stack_addr.i64 ss0
+;; @0062 v53 = iconst.i64 48
+;; @0062 v54 = iadd v0, v53 ; v53 = 48
+;; @0062 v61 = iconst.i64 96
+;; @0062 v64 = iconst.i64 -24
+;; v143 = iconst.i64 0x0001_0000_0000
+;; @005c jump block2(v14)
;;
-;; block2(v14: i128):
+;; block2(v15: i128):
;; @0062 jump block5
;;
;; block5:
-;; @0062 v15 = ireduce.i64 v14
-;; @0062 trapz v15, user16
-;; @0062 v20 = load.i64 notrap aligned region2 v15+88
-;; v145 = iconst.i64 64
-;; v146 = ushr.i128 v14, v145 ; v145 = 64
-;; @0062 v19 = ireduce.i64 v146
-;; @0062 v21 = icmp eq v20, v19
-;; @0062 trapz v21, user23
-;; v147 = iconst.i64 1
-;; v148 = iadd v20, v147 ; v147 = 1
-;; @0062 store notrap aligned region2 v148, v15+88
-;; @0062 v24 = load.i64 notrap aligned region3 v15+80
-;; @0062 v26 = load.i64 notrap aligned region4 v25+88
-;; @0062 v27 = load.i64 notrap aligned region4 v25+96
-;; @0062 store notrap aligned region5 v26, v24+64
-;; @0062 store notrap aligned region5 v27, v24+72
-;; v149 = iconst.i64 0
-;; @0062 store notrap aligned region3 v149, v15+80 ; v149 = 0
-;; v150 = iconst.i64 2
-;; @0062 store notrap aligned region4 v150, v25+88 ; v150 = 2
-;; @0062 store notrap aligned region4 v15, v25+96
-;; v151 = iconst.i32 1
-;; @0062 store notrap aligned region6 v151, v15+32 ; v151 = 1
-;; v152 = iconst.i32 2
-;; @0062 store notrap aligned region6 v152, v27+32 ; v152 = 2
-;; @0062 v38 = load.i64 notrap aligned region7 v25+72
-;; @0062 v39 = load.i64 notrap aligned region8 v25+64
-;; @0062 v40 = load.i64 notrap aligned region9 v25+80
-;; @0062 store notrap aligned region10 v38, v27+8
-;; @0062 store notrap aligned region11 v39, v27+16
-;; @0062 store notrap aligned region12 v40, v27+24
-;; @0062 v41 = load.i64 notrap aligned region1 v25+24
-;; @0062 store notrap aligned region13 v41, v27
-;; @0062 v44 = load.i64 notrap aligned region13 v15
-;; @0062 store notrap aligned region1 v44, v25+24
-;; @0062 v45 = load.i64 notrap aligned region10 v15+8
-;; @0062 store notrap aligned region7 v45, v25+72
-;; @0062 v46 = load.i64 notrap aligned region11 v15+16
-;; @0062 store notrap aligned region8 v46, v25+64
-;; @0062 v47 = load.i64 notrap aligned region12 v15+24
-;; @0062 store notrap aligned region9 v47, v25+80
-;; v153 = iconst.i64 40
-;; v154 = iadd v27, v153 ; v153 = 40
-;; @0062 store notrap aligned region14 v151, v154+4 ; v151 = 1
-;; @0062 store.i64 notrap aligned region15 v51, v154+8
-;; v155 = iadd.i64 v0, v52 ; v52 = 48
-;; @0062 store notrap aligned region16 v155, v51
-;; @0062 store notrap aligned region17 v151, v154 ; v151 = 1
-;; @0062 store notrap aligned region12 v151, v27+56 ; v151 = 1
-;; v156 = iconst.i64 96
-;; v157 = iadd v24, v156 ; v156 = 96
-;; @0062 v62 = load.i64 notrap aligned region18 v157
-;; v158 = iconst.i64 -24
-;; v159 = iadd v62, v158 ; v158 = -24
-;; v160 = iconst.i64 0x0001_0000_0000
-;; @0062 v65 = stack_switch v159, v159, v160 ; v160 = 0x0001_0000_0000
-;; @0062 v67 = load.i64 notrap aligned region4 v25+88
-;; @0062 v68 = load.i64 notrap aligned region4 v25+96
-;; @0062 store notrap aligned region4 v26, v25+88
-;; @0062 store notrap aligned region4 v27, v25+96
-;; @0062 store notrap aligned region6 v151, v27+32 ; v151 = 1
-;; v161 = iconst.i32 0
-;; @0062 store notrap aligned region17 v161, v154 ; v161 = 0
-;; @0062 store notrap aligned region14 v161, v154+4 ; v161 = 0
-;; @0062 store notrap aligned region15 v149, v154+8 ; v149 = 0
-;; @0062 store notrap aligned region12 v149, v27+56 ; v149 = 0
-;; @0062 brif v65, block9, block6
+;; @0062 v16 = ireduce.i64 v15
+;; @0062 trapz v16, user16
+;; @0062 v21 = load.i64 notrap aligned region2 v16+88
+;; v146 = iconst.i64 64
+;; v147 = ushr.i128 v15, v146 ; v146 = 64
+;; @0062 v20 = ireduce.i64 v147
+;; @0062 v22 = icmp eq v21, v20
+;; @0062 trapz v22, user23
+;; v148 = iconst.i64 1
+;; v149 = iadd v21, v148 ; v148 = 1
+;; @0062 store notrap aligned region2 v149, v16+88
+;; @0062 v25 = load.i64 notrap aligned region3 v16+80
+;; @0062 v27 = load.i64 notrap aligned region4 v26+88
+;; @0062 v28 = load.i64 notrap aligned region4 v26+96
+;; @0062 store notrap aligned region5 v27, v25+64
+;; @0062 store notrap aligned region5 v28, v25+72
+;; v150 = iconst.i64 0
+;; @0062 store notrap aligned region3 v150, v16+80 ; v150 = 0
+;; v151 = iconst.i64 2
+;; @0062 store notrap aligned region4 v151, v26+88 ; v151 = 2
+;; @0062 store notrap aligned region4 v16, v26+96
+;; v152 = iconst.i32 1
+;; @0062 store notrap aligned region6 v152, v16+32 ; v152 = 1
+;; v153 = iconst.i32 2
+;; @0062 store notrap aligned region6 v153, v28+32 ; v153 = 2
+;; @0062 v39 = load.i64 notrap aligned region7 v26+72
+;; @0062 v40 = load.i64 notrap aligned region8 v26+64
+;; @0062 v41 = load.i64 notrap aligned region9 v26+80
+;; @0062 store notrap aligned region10 v39, v28+8
+;; @0062 store notrap aligned region11 v40, v28+16
+;; @0062 store notrap aligned region12 v41, v28+24
+;; @0062 v42 = load.i64 notrap aligned region1 v26+24
+;; @0062 store notrap aligned region13 v42, v28
+;; @0062 v45 = load.i64 notrap aligned region13 v16
+;; @0062 store notrap aligned region1 v45, v26+24
+;; @0062 v46 = load.i64 notrap aligned region10 v16+8
+;; @0062 store notrap aligned region7 v46, v26+72
+;; @0062 v47 = load.i64 notrap aligned region11 v16+16
+;; @0062 store notrap aligned region8 v47, v26+64
+;; @0062 v48 = load.i64 notrap aligned region12 v16+24
+;; @0062 store notrap aligned region9 v48, v26+80
+;; v154 = iconst.i64 40
+;; v155 = iadd v28, v154 ; v154 = 40
+;; @0062 store notrap aligned region14 v152, v155+4 ; v152 = 1
+;; @0062 store.i64 notrap aligned region15 v52, v155+8
+;; v156 = iadd.i64 v0, v53 ; v53 = 48
+;; @0062 store notrap aligned region16 v156, v52
+;; @0062 store notrap aligned region17 v152, v155 ; v152 = 1
+;; @0062 store notrap aligned region12 v152, v28+56 ; v152 = 1
+;; v157 = iconst.i64 96
+;; v158 = iadd v25, v157 ; v157 = 96
+;; @0062 v63 = load.i64 notrap aligned region18 v158
+;; v159 = iconst.i64 -24
+;; v160 = iadd v63, v159 ; v159 = -24
+;; v161 = iconst.i64 0x0001_0000_0000
+;; @0062 v66 = stack_switch v160, v160, v161 ; v161 = 0x0001_0000_0000
+;; @0062 v68 = load.i64 notrap aligned region4 v26+88
+;; @0062 v69 = load.i64 notrap aligned region4 v26+96
+;; @0062 store notrap aligned region4 v27, v26+88
+;; @0062 store notrap aligned region4 v28, v26+96
+;; @0062 store notrap aligned region6 v152, v28+32 ; v152 = 1
+;; v162 = iconst.i32 0
+;; @0062 store notrap aligned region17 v162, v155 ; v162 = 0
+;; @0062 store notrap aligned region14 v162, v155+4 ; v162 = 0
+;; @0062 store notrap aligned region15 v150, v155+8 ; v150 = 0
+;; @0062 store notrap aligned region12 v150, v28+56 ; v150 = 0
+;; @0062 brif v66, block9, block6
;;
;; block9:
-;; @0062 v58 = iconst.i64 32
-;; @0062 v75 = ushr.i64 v65, v58 ; v58 = 32
-;; @0062 v76 = iconst.i64 4
-;; @0062 v77 = icmp eq v75, v76 ; v76 = 4
-;; @0062 brif v77, block8, block7
+;; @0062 v59 = iconst.i64 32
+;; @0062 v76 = ushr.i64 v66, v59 ; v59 = 32
+;; @0062 v77 = iconst.i64 4
+;; @0062 v78 = icmp eq v76, v77 ; v77 = 4
+;; @0062 brif v78, block8, block7
;;
;; block8 cold:
-;; @0062 v80 = iconst.i32 5
-;; @0062 store notrap aligned region6 v80, v68+32 ; v80 = 5
-;; @0062 v83 = load.i64 notrap aligned region13 v27
-;; @0062 store notrap aligned region1 v83, v25+24
-;; @0062 v84 = load.i64 notrap aligned region10 v27+8
-;; @0062 store notrap aligned region7 v84, v25+72
-;; @0062 v85 = load.i64 notrap aligned region11 v27+16
-;; @0062 store notrap aligned region8 v85, v25+64
-;; @0062 v86 = load.i64 notrap aligned region12 v27+24
-;; @0062 store notrap aligned region9 v86, v25+80
-;; v167 = iconst.i32 0
-;; v168 = iconst.i64 120
-;; v169 = iadd.i64 v68, v168 ; v168 = 120
-;; @0062 store notrap aligned region17 v167, v169 ; v167 = 0
-;; @0062 store notrap aligned region14 v167, v169+4 ; v167 = 0
-;; v170 = iconst.i64 0
-;; @0062 store notrap aligned region15 v170, v169+8 ; v170 = 0
-;; v171 = iconst.i64 136
-;; v172 = iadd.i64 v68, v171 ; v171 = 136
-;; @0062 store notrap aligned region17 v167, v172 ; v167 = 0
-;; @0062 store notrap aligned region14 v167, v172+4 ; v167 = 0
-;; @0062 store notrap aligned region15 v170, v172+8 ; v170 = 0
+;; @0062 v81 = iconst.i32 5
+;; @0062 store notrap aligned region6 v81, v69+32 ; v81 = 5
+;; @0062 v84 = load.i64 notrap aligned region13 v28
+;; @0062 store notrap aligned region1 v84, v26+24
+;; @0062 v85 = load.i64 notrap aligned region10 v28+8
+;; @0062 store notrap aligned region7 v85, v26+72
+;; @0062 v86 = load.i64 notrap aligned region11 v28+16
+;; @0062 store notrap aligned region8 v86, v26+64
+;; @0062 v87 = load.i64 notrap aligned region12 v28+24
+;; @0062 store notrap aligned region9 v87, v26+80
+;; v168 = iconst.i32 0
+;; v169 = iconst.i64 120
+;; v170 = iadd.i64 v69, v169 ; v169 = 120
+;; @0062 store notrap aligned region17 v168, v170 ; v168 = 0
+;; @0062 store notrap aligned region14 v168, v170+4 ; v168 = 0
+;; v171 = iconst.i64 0
+;; @0062 store notrap aligned region15 v171, v170+8 ; v171 = 0
+;; v172 = iconst.i64 144
+;; v173 = iadd.i64 v69, v172 ; v172 = 144
+;; @0062 store notrap aligned region17 v168, v173 ; v168 = 0
+;; @0062 store notrap aligned region14 v168, v173+4 ; v168 = 0
+;; @0062 store notrap aligned region15 v171, v173+8 ; v171 = 0
;; @0062 try_call fn2(v0), sig2, block11, [ context v0 ]
;;
;; block11:
;; @0062 trap user12
;;
;; block7:
-;; @0062 v101 = load.i64 notrap aligned region7 v25+72
-;; @0062 v102 = load.i64 notrap aligned region8 v25+64
-;; @0062 v103 = load.i64 notrap aligned region9 v25+80
-;; @0062 store notrap aligned region10 v101, v68+8
-;; @0062 store notrap aligned region11 v102, v68+16
-;; @0062 store notrap aligned region12 v103, v68+24
-;; @0062 v106 = load.i64 notrap aligned region13 v27
-;; @0062 store notrap aligned region1 v106, v25+24
-;; @0062 v107 = load.i64 notrap aligned region10 v27+8
-;; @0062 store notrap aligned region7 v107, v25+72
-;; @0062 v108 = load.i64 notrap aligned region11 v27+16
-;; @0062 store notrap aligned region8 v108, v25+64
-;; @0062 v109 = load.i64 notrap aligned region12 v27+24
-;; @0062 store notrap aligned region9 v109, v25+80
-;; @0062 v111 = load.i64 notrap aligned region2 v68+88
+;; @0062 v102 = load.i64 notrap aligned region7 v26+72
+;; @0062 v103 = load.i64 notrap aligned region8 v26+64
+;; @0062 v104 = load.i64 notrap aligned region9 v26+80
+;; @0062 store notrap aligned region10 v102, v69+8
+;; @0062 store notrap aligned region11 v103, v69+16
+;; @0062 store notrap aligned region12 v104, v69+24
+;; @0062 v107 = load.i64 notrap aligned region13 v28
+;; @0062 store notrap aligned region1 v107, v26+24
+;; @0062 v108 = load.i64 notrap aligned region10 v28+8
+;; @0062 store notrap aligned region7 v108, v26+72
+;; @0062 v109 = load.i64 notrap aligned region11 v28+16
+;; @0062 store notrap aligned region8 v109, v26+64
+;; @0062 v110 = load.i64 notrap aligned region12 v28+24
+;; @0062 store notrap aligned region9 v110, v26+80
+;; @0062 v112 = load.i64 notrap aligned region2 v69+88
;; @0062 jump block10
;;
;; block12 cold:
;; @0062 trap user12
;;
;; block13:
-;; @0062 v118 = iconst.i64 136
-;; @0062 v119 = iadd.i64 v68, v118 ; v118 = 136
-;; @0062 v120 = load.i64 notrap aligned region15 v119+8
-;; @0062 v121 = load.i32 notrap aligned region16 v120
-;; v164 = iconst.i32 0
-;; @0062 store notrap aligned region17 v164, v119 ; v164 = 0
+;; @0062 v119 = iconst.i64 144
+;; @0062 v120 = iadd.i64 v69, v119 ; v119 = 144
+;; @0062 v121 = load.i64 notrap aligned region15 v120+8
+;; @0062 v122 = load.i32 notrap aligned region16 v121
+;; v165 = iconst.i32 0
+;; @0062 store notrap aligned region17 v165, v120 ; v165 = 0
;; @0062 jump block4
;;
;; block10:
-;; @0062 v110 = ireduce.i32 v65
-;; @0062 br_table v110, block12, [block13]
+;; @0062 v111 = ireduce.i32 v66
+;; @0062 br_table v111, block12, [block13]
;;
;; block6:
-;; @0062 v125 = load.i64 notrap aligned region13 v27
-;; @0062 store notrap aligned region1 v125, v25+24
-;; @0062 v126 = load.i64 notrap aligned region10 v27+8
-;; @0062 store notrap aligned region7 v126, v25+72
-;; @0062 v127 = load.i64 notrap aligned region11 v27+16
-;; @0062 store notrap aligned region8 v127, v25+64
-;; @0062 v128 = load.i64 notrap aligned region12 v27+24
-;; @0062 store notrap aligned region9 v128, v25+80
-;; @0062 v131 = iconst.i32 4
-;; @0062 store notrap aligned region6 v131, v68+32 ; v131 = 4
-;; @0062 v132 = iconst.i64 120
-;; @0062 v133 = iadd.i64 v68, v132 ; v132 = 120
-;; @0062 v134 = load.i64 notrap aligned region15 v133+8
-;; v162 = iconst.i32 0
-;; @0062 store notrap aligned region17 v162, v133 ; v162 = 0
-;; @0062 store notrap aligned region14 v162, v133+4 ; v162 = 0
-;; v163 = iconst.i64 0
-;; @0062 store notrap aligned region15 v163, v133+8 ; v163 = 0
+;; @0062 v126 = load.i64 notrap aligned region13 v28
+;; @0062 store notrap aligned region1 v126, v26+24
+;; @0062 v127 = load.i64 notrap aligned region10 v28+8
+;; @0062 store notrap aligned region7 v127, v26+72
+;; @0062 v128 = load.i64 notrap aligned region11 v28+16
+;; @0062 store notrap aligned region8 v128, v26+64
+;; @0062 v129 = load.i64 notrap aligned region12 v28+24
+;; @0062 store notrap aligned region9 v129, v26+80
+;; @0062 v132 = iconst.i32 4
+;; @0062 store notrap aligned region6 v132, v69+32 ; v132 = 4
+;; @0062 v133 = iconst.i64 120
+;; @0062 v134 = iadd.i64 v69, v133 ; v133 = 120
+;; @0062 v135 = load.i64 notrap aligned region15 v134+8
+;; v163 = iconst.i32 0
+;; @0062 store notrap aligned region17 v163, v134 ; v163 = 0
+;; @0062 store notrap aligned region14 v163, v134+4 ; v163 = 0
+;; v164 = iconst.i64 0
+;; @0062 store notrap aligned region15 v164, v134+8 ; v164 = 0
;; @0068 return
;;
;; block4:
-;; @0062 v113 = uextend.i128 v111
-;; v165 = iconst.i64 64
-;; v166 = ishl v113, v165 ; v165 = 64
-;; @0062 v112 = uextend.i128 v68
-;; @0062 v117 = bor v166, v112
-;; @006d jump block2(v117)
+;; @0062 v114 = uextend.i128 v112
+;; v166 = iconst.i64 64
+;; v167 = ishl v114, v166 ; v166 = 64
+;; @0062 v113 = uextend.i128 v69
+;; @0062 v118 = bor v167, v113
+;; @006d jump block2(v118)
;; }
diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.wat
index bff2ab520a..ff5c9192fb 100644
--- a/tests/disas/stack-switching/resume-suspend.wat
+++ b/tests/disas/stack-switching/resume-suspend.wat
@@ -91,8 +91,8 @@
;; block6:
;; @003b store.i64 notrap aligned region7 v8, v6+80
;; v82 = iconst.i32 1
-;; @003b v28 = iconst.i64 136
-;; @003b v29 = iadd.i64 v6, v28 ; v28 = 136
+;; @003b v28 = iconst.i64 144
+;; @003b v29 = iadd.i64 v6, v28 ; v28 = 144
;; @003b store notrap aligned region8 v82, v29+4 ; v82 = 1
;; @003b v31 = stack_addr.i64 ss0
;; @003b store notrap aligned region4 v31, v29+8
@@ -117,7 +117,7 @@
;; @003b brif v51, block8, block9
;;
;; block8 cold:
-;; v87 = iadd.i64 v6, v28 ; v28 = 136
+;; v87 = iadd.i64 v6, v28 ; v28 = 144
;; @003b v54 = load.i64 notrap aligned region4 v87+8
;; @003b v55 = load.i32 notrap aligned region6 v54
;; v88 = iconst.i32 0
@@ -131,7 +131,7 @@
;; @003b trap user12
;;
;; block9:
-;; v84 = iadd.i64 v6, v28 ; v28 = 136
+;; v84 = iadd.i64 v6, v28 ; v28 = 144
;; @003b v61 = load.i64 notrap aligned region4 v84+8
;; v85 = iconst.i32 0
;; @003b store notrap aligned region11 v85, v84 ; v85 = 0
@@ -169,7 +169,7 @@
;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
;; gv2 = load.i64 notrap aligned region1 gv1+24
;; sig0 = (i64 vmctx, i32) -> i64 tail
-;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail
;; sig2 = (i64 vmctx) tail
;; fn0 = colocated u805306368:6 sig0
;; fn1 = colocated u805306368:42 sig1
@@ -180,186 +180,186 @@
;; @0043 v9 = iconst.i32 0
;; @0043 v10 = call fn0(v0, v9) ; v9 = 0
;; @0045 trapz v10, user16
-;; @0045 v13 = call fn1(v0, v10, v9, v9) ; v9 = 0, v9 = 0
-;; @0045 v14 = load.i64 notrap aligned region2 v13+88
+;; @0045 v14 = call fn1(v0, v10, v9, v9, v9) ; v9 = 0, v9 = 0, v9 = 0
+;; @0045 v15 = load.i64 notrap aligned region2 v14+88
;; @004e jump block3
;;
;; block3:
-;; @0045 v16 = uextend.i128 v14
+;; @0045 v17 = uextend.i128 v15
;; @0040 v5 = iconst.i64 64
-;; v153 = ishl v16, v5 ; v5 = 64
-;; v155 = ireduce.i64 v153
-;; v157 = bor v155, v13
-;; @004e trapz v157, user16
-;; @004e v26 = load.i64 notrap aligned region2 v157+88
-;; @0045 v15 = uextend.i128 v13
-;; @0045 v20 = bor v153, v15
-;; v159 = ushr v20, v5 ; v5 = 64
-;; @004e v25 = ireduce.i64 v159
-;; @004e v27 = icmp eq v26, v25
-;; @004e trapz v27, user23
-;; @004e v28 = iconst.i64 1
-;; @004e v29 = iadd v26, v28 ; v28 = 1
-;; @004e store notrap aligned region2 v29, v157+88
-;; @004e v30 = load.i64 notrap aligned region3 v157+80
-;; @004e v31 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @004e v32 = load.i64 notrap aligned region4 v31+88
-;; @004e v33 = load.i64 notrap aligned region4 v31+96
-;; @004e store notrap aligned region5 v32, v30+64
-;; @004e store notrap aligned region5 v33, v30+72
+;; v154 = ishl v17, v5 ; v5 = 64
+;; v156 = ireduce.i64 v154
+;; v158 = bor v156, v14
+;; @004e trapz v158, user16
+;; @004e v27 = load.i64 notrap aligned region2 v158+88
+;; @0045 v16 = uextend.i128 v14
+;; @0045 v21 = bor v154, v16
+;; v160 = ushr v21, v5 ; v5 = 64
+;; @004e v26 = ireduce.i64 v160
+;; @004e v28 = icmp eq v27, v26
+;; @004e trapz v28, user23
+;; @004e v29 = iconst.i64 1
+;; @004e v30 = iadd v27, v29 ; v29 = 1
+;; @004e store notrap aligned region2 v30, v158+88
+;; @004e v31 = load.i64 notrap aligned region3 v158+80
+;; @004e v32 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @004e v33 = load.i64 notrap aligned region4 v32+88
+;; @004e v34 = load.i64 notrap aligned region4 v32+96
+;; @004e store notrap aligned region5 v33, v31+64
+;; @004e store notrap aligned region5 v34, v31+72
;; @0040 v2 = iconst.i64 0
-;; @004e store notrap aligned region3 v2, v157+80 ; v2 = 0
-;; @004e v35 = iconst.i64 2
-;; @004e store notrap aligned region4 v35, v31+88 ; v35 = 2
-;; @004e store notrap aligned region4 v157, v31+96
-;; @004e v39 = iconst.i32 1
-;; @004e store notrap aligned region6 v39, v157+32 ; v39 = 1
-;; @004e v40 = iconst.i32 2
-;; @004e store notrap aligned region6 v40, v33+32 ; v40 = 2
-;; @004e v44 = load.i64 notrap aligned region7 v31+72
-;; @004e v45 = load.i64 notrap aligned region8 v31+64
-;; @004e v46 = load.i64 notrap aligned region9 v31+80
-;; @004e store notrap aligned region10 v44, v33+8
-;; @004e store notrap aligned region11 v45, v33+16
-;; @004e store notrap aligned region12 v46, v33+24
-;; @004e v47 = load.i64 notrap aligned region1 v31+24
-;; @004e store notrap aligned region13 v47, v33
-;; @004e v50 = load.i64 notrap aligned region13 v157
-;; @004e store notrap aligned region1 v50, v31+24
-;; @004e v51 = load.i64 notrap aligned region10 v157+8
-;; @004e store notrap aligned region7 v51, v31+72
-;; @004e v52 = load.i64 notrap aligned region11 v157+16
-;; @004e store notrap aligned region8 v52, v31+64
-;; @004e v53 = load.i64 notrap aligned region12 v157+24
-;; @004e store notrap aligned region9 v53, v31+80
-;; @004e v54 = iconst.i64 40
-;; @004e v55 = iadd v33, v54 ; v54 = 40
-;; @004e store notrap aligned region14 v39, v55+4 ; v39 = 1
-;; @004e v57 = stack_addr.i64 ss0
-;; @004e store notrap aligned region15 v57, v55+8
-;; @004e v58 = iconst.i64 48
-;; @004e v59 = iadd.i64 v0, v58 ; v58 = 48
-;; @004e store notrap aligned region16 v59, v57
-;; @004e store notrap aligned region17 v39, v55 ; v39 = 1
-;; @004e store notrap aligned region12 v39, v33+56 ; v39 = 1
-;; @004e v66 = iconst.i64 96
-;; @004e v67 = iadd v30, v66 ; v66 = 96
-;; @004e v68 = load.i64 notrap aligned region18 v67
-;; @004e v69 = iconst.i64 -24
-;; @004e v70 = iadd v68, v69 ; v69 = -24
-;; v161 = iconst.i64 0x0001_0000_0000
-;; @004e v71 = stack_switch v70, v70, v161 ; v161 = 0x0001_0000_0000
-;; @004e v73 = load.i64 notrap aligned region4 v31+88
-;; @004e v74 = load.i64 notrap aligned region4 v31+96
-;; @004e store notrap aligned region4 v32, v31+88
-;; @004e store notrap aligned region4 v33, v31+96
-;; @004e store notrap aligned region6 v39, v33+32 ; v39 = 1
-;; v164 = iconst.i32 0
-;; @004e store notrap aligned region17 v164, v55 ; v164 = 0
-;; @004e store notrap aligned region14 v164, v55+4 ; v164 = 0
-;; @004e store notrap aligned region15 v2, v55+8 ; v2 = 0
-;; @004e store notrap aligned region12 v2, v33+56 ; v2 = 0
-;; @004e brif v71, block7, block4
+;; @004e store notrap aligned region3 v2, v158+80 ; v2 = 0
+;; @004e v36 = iconst.i64 2
+;; @004e store notrap aligned region4 v36, v32+88 ; v36 = 2
+;; @004e store notrap aligned region4 v158, v32+96
+;; @004e v40 = iconst.i32 1
+;; @004e store notrap aligned region6 v40, v158+32 ; v40 = 1
+;; @004e v41 = iconst.i32 2
+;; @004e store notrap aligned region6 v41, v34+32 ; v41 = 2
+;; @004e v45 = load.i64 notrap aligned region7 v32+72
+;; @004e v46 = load.i64 notrap aligned region8 v32+64
+;; @004e v47 = load.i64 notrap aligned region9 v32+80
+;; @004e store notrap aligned region10 v45, v34+8
+;; @004e store notrap aligned region11 v46, v34+16
+;; @004e store notrap aligned region12 v47, v34+24
+;; @004e v48 = load.i64 notrap aligned region1 v32+24
+;; @004e store notrap aligned region13 v48, v34
+;; @004e v51 = load.i64 notrap aligned region13 v158
+;; @004e store notrap aligned region1 v51, v32+24
+;; @004e v52 = load.i64 notrap aligned region10 v158+8
+;; @004e store notrap aligned region7 v52, v32+72
+;; @004e v53 = load.i64 notrap aligned region11 v158+16
+;; @004e store notrap aligned region8 v53, v32+64
+;; @004e v54 = load.i64 notrap aligned region12 v158+24
+;; @004e store notrap aligned region9 v54, v32+80
+;; @004e v55 = iconst.i64 40
+;; @004e v56 = iadd v34, v55 ; v55 = 40
+;; @004e store notrap aligned region14 v40, v56+4 ; v40 = 1
+;; @004e v58 = stack_addr.i64 ss0
+;; @004e store notrap aligned region15 v58, v56+8
+;; @004e v59 = iconst.i64 48
+;; @004e v60 = iadd.i64 v0, v59 ; v59 = 48
+;; @004e store notrap aligned region16 v60, v58
+;; @004e store notrap aligned region17 v40, v56 ; v40 = 1
+;; @004e store notrap aligned region12 v40, v34+56 ; v40 = 1
+;; @004e v67 = iconst.i64 96
+;; @004e v68 = iadd v31, v67 ; v67 = 96
+;; @004e v69 = load.i64 notrap aligned region18 v68
+;; @004e v70 = iconst.i64 -24
+;; @004e v71 = iadd v69, v70 ; v70 = -24
+;; v162 = iconst.i64 0x0001_0000_0000
+;; @004e v72 = stack_switch v71, v71, v162 ; v162 = 0x0001_0000_0000
+;; @004e v74 = load.i64 notrap aligned region4 v32+88
+;; @004e v75 = load.i64 notrap aligned region4 v32+96
+;; @004e store notrap aligned region4 v33, v32+88
+;; @004e store notrap aligned region4 v34, v32+96
+;; @004e store notrap aligned region6 v40, v34+32 ; v40 = 1
+;; v165 = iconst.i32 0
+;; @004e store notrap aligned region17 v165, v56 ; v165 = 0
+;; @004e store notrap aligned region14 v165, v56+4 ; v165 = 0
+;; @004e store notrap aligned region15 v2, v56+8 ; v2 = 0
+;; @004e store notrap aligned region12 v2, v34+56 ; v2 = 0
+;; @004e brif v72, block7, block4
;;
;; block7:
-;; @004e v64 = iconst.i64 32
-;; @004e v81 = ushr.i64 v71, v64 ; v64 = 32
-;; @004e v82 = iconst.i64 4
-;; @004e v83 = icmp eq v81, v82 ; v82 = 4
-;; @004e brif v83, block6, block5
+;; @004e v65 = iconst.i64 32
+;; @004e v82 = ushr.i64 v72, v65 ; v65 = 32
+;; @004e v83 = iconst.i64 4
+;; @004e v84 = icmp eq v82, v83 ; v83 = 4
+;; @004e brif v84, block6, block5
;;
;; block6 cold:
-;; @004e v86 = iconst.i32 5
-;; @004e store notrap aligned region6 v86, v74+32 ; v86 = 5
-;; @004e v89 = load.i64 notrap aligned region13 v33
-;; @004e store notrap aligned region1 v89, v31+24
-;; @004e v90 = load.i64 notrap aligned region10 v33+8
-;; @004e store notrap aligned region7 v90, v31+72
-;; @004e v91 = load.i64 notrap aligned region11 v33+16
-;; @004e store notrap aligned region8 v91, v31+64
-;; @004e v92 = load.i64 notrap aligned region12 v33+24
-;; @004e store notrap aligned region9 v92, v31+80
-;; v173 = iconst.i32 0
-;; v174 = iconst.i64 120
-;; v175 = iadd.i64 v74, v174 ; v174 = 120
-;; @004e store notrap aligned region17 v173, v175 ; v173 = 0
-;; @004e store notrap aligned region14 v173, v175+4 ; v173 = 0
-;; v176 = iconst.i64 0
-;; @004e store notrap aligned region15 v176, v175+8 ; v176 = 0
-;; v177 = iconst.i64 136
-;; v178 = iadd.i64 v74, v177 ; v177 = 136
-;; @004e store notrap aligned region17 v173, v178 ; v173 = 0
-;; @004e store notrap aligned region14 v173, v178+4 ; v173 = 0
-;; @004e store notrap aligned region15 v176, v178+8 ; v176 = 0
+;; @004e v87 = iconst.i32 5
+;; @004e store notrap aligned region6 v87, v75+32 ; v87 = 5
+;; @004e v90 = load.i64 notrap aligned region13 v34
+;; @004e store notrap aligned region1 v90, v32+24
+;; @004e v91 = load.i64 notrap aligned region10 v34+8
+;; @004e store notrap aligned region7 v91, v32+72
+;; @004e v92 = load.i64 notrap aligned region11 v34+16
+;; @004e store notrap aligned region8 v92, v32+64
+;; @004e v93 = load.i64 notrap aligned region12 v34+24
+;; @004e store notrap aligned region9 v93, v32+80
+;; v174 = iconst.i32 0
+;; v175 = iconst.i64 120
+;; v176 = iadd.i64 v75, v175 ; v175 = 120
+;; @004e store notrap aligned region17 v174, v176 ; v174 = 0
+;; @004e store notrap aligned region14 v174, v176+4 ; v174 = 0
+;; v177 = iconst.i64 0
+;; @004e store notrap aligned region15 v177, v176+8 ; v177 = 0
+;; v178 = iconst.i64 144
+;; v179 = iadd.i64 v75, v178 ; v178 = 144
+;; @004e store notrap aligned region17 v174, v179 ; v174 = 0
+;; @004e store notrap aligned region14 v174, v179+4 ; v174 = 0
+;; @004e store notrap aligned region15 v177, v179+8 ; v177 = 0
;; @004e try_call fn2(v0), sig2, block9, [ context v0 ]
;;
;; block9:
;; @004e trap user12
;;
;; block5:
-;; @004e v107 = load.i64 notrap aligned region7 v31+72
-;; @004e v108 = load.i64 notrap aligned region8 v31+64
-;; @004e v109 = load.i64 notrap aligned region9 v31+80
-;; @004e store notrap aligned region10 v107, v74+8
-;; @004e store notrap aligned region11 v108, v74+16
-;; @004e store notrap aligned region12 v109, v74+24
-;; @004e v112 = load.i64 notrap aligned region13 v33
-;; @004e store notrap aligned region1 v112, v31+24
-;; @004e v113 = load.i64 notrap aligned region10 v33+8
-;; @004e store notrap aligned region7 v113, v31+72
-;; @004e v114 = load.i64 notrap aligned region11 v33+16
-;; @004e store notrap aligned region8 v114, v31+64
-;; @004e v115 = load.i64 notrap aligned region12 v33+24
-;; @004e store notrap aligned region9 v115, v31+80
-;; @004e v117 = load.i64 notrap aligned region2 v74+88
+;; @004e v108 = load.i64 notrap aligned region7 v32+72
+;; @004e v109 = load.i64 notrap aligned region8 v32+64
+;; @004e v110 = load.i64 notrap aligned region9 v32+80
+;; @004e store notrap aligned region10 v108, v75+8
+;; @004e store notrap aligned region11 v109, v75+16
+;; @004e store notrap aligned region12 v110, v75+24
+;; @004e v113 = load.i64 notrap aligned region13 v34
+;; @004e store notrap aligned region1 v113, v32+24
+;; @004e v114 = load.i64 notrap aligned region10 v34+8
+;; @004e store notrap aligned region7 v114, v32+72
+;; @004e v115 = load.i64 notrap aligned region11 v34+16
+;; @004e store notrap aligned region8 v115, v32+64
+;; @004e v116 = load.i64 notrap aligned region12 v34+24
+;; @004e store notrap aligned region9 v116, v32+80
+;; @004e v118 = load.i64 notrap aligned region2 v75+88
;; @004e jump block8
;;
;; block10 cold:
;; @004e trap user12
;;
;; block11:
-;; @004e v124 = iconst.i64 136
-;; @004e v125 = iadd.i64 v74, v124 ; v124 = 136
-;; @004e v126 = load.i64 notrap aligned region15 v125+8
-;; v170 = iconst.i32 0
-;; @004e store notrap aligned region17 v170, v125 ; v170 = 0
-;; @004e v119 = uextend.i128 v117
-;; v171 = iconst.i64 64
-;; v172 = ishl v119, v171 ; v171 = 64
-;; @004e v118 = uextend.i128 v74
-;; @004e v123 = bor v172, v118
-;; @004e jump block2(v123)
+;; @004e v125 = iconst.i64 144
+;; @004e v126 = iadd.i64 v75, v125 ; v125 = 144
+;; @004e v127 = load.i64 notrap aligned region15 v126+8
+;; v171 = iconst.i32 0
+;; @004e store notrap aligned region17 v171, v126 ; v171 = 0
+;; @004e v120 = uextend.i128 v118
+;; v172 = iconst.i64 64
+;; v173 = ishl v120, v172 ; v172 = 64
+;; @004e v119 = uextend.i128 v75
+;; @004e v124 = bor v173, v119
+;; @004e jump block2(v124)
;;
;; block8:
-;; @004e v116 = ireduce.i32 v71
-;; @004e br_table v116, block10, [block11]
+;; @004e v117 = ireduce.i32 v72
+;; @004e br_table v117, block10, [block11]
;;
;; block4:
-;; @004e v130 = load.i64 notrap aligned region13 v33
-;; @004e store notrap aligned region1 v130, v31+24
-;; @004e v131 = load.i64 notrap aligned region10 v33+8
-;; @004e store notrap aligned region7 v131, v31+72
-;; @004e v132 = load.i64 notrap aligned region11 v33+16
-;; @004e store notrap aligned region8 v132, v31+64
-;; @004e v133 = load.i64 notrap aligned region12 v33+24
-;; @004e store notrap aligned region9 v133, v31+80
-;; @004e v136 = iconst.i32 4
-;; @004e store notrap aligned region6 v136, v74+32 ; v136 = 4
-;; @004e v137 = iconst.i64 120
-;; @004e v138 = iadd.i64 v74, v137 ; v137 = 120
-;; @004e v139 = load.i64 notrap aligned region15 v138+8
-;; v165 = iconst.i32 0
-;; @004e store notrap aligned region17 v165, v138 ; v165 = 0
-;; @004e store notrap aligned region14 v165, v138+4 ; v165 = 0
-;; v166 = iconst.i64 0
-;; @004e store notrap aligned region15 v166, v138+8 ; v166 = 0
-;; v167 = uextend.i128 v166 ; v166 = 0
-;; v168 = iconst.i64 64
-;; v169 = ishl v167, v168 ; v168 = 64
-;; @0040 v8 = bor v169, v167
+;; @004e v131 = load.i64 notrap aligned region13 v34
+;; @004e store notrap aligned region1 v131, v32+24
+;; @004e v132 = load.i64 notrap aligned region10 v34+8
+;; @004e store notrap aligned region7 v132, v32+72
+;; @004e v133 = load.i64 notrap aligned region11 v34+16
+;; @004e store notrap aligned region8 v133, v32+64
+;; @004e v134 = load.i64 notrap aligned region12 v34+24
+;; @004e store notrap aligned region9 v134, v32+80
+;; @004e v137 = iconst.i32 4
+;; @004e store notrap aligned region6 v137, v75+32 ; v137 = 4
+;; @004e v138 = iconst.i64 120
+;; @004e v139 = iadd.i64 v75, v138 ; v138 = 120
+;; @004e v140 = load.i64 notrap aligned region15 v139+8
+;; v166 = iconst.i32 0
+;; @004e store notrap aligned region17 v166, v139 ; v166 = 0
+;; @004e store notrap aligned region14 v166, v139+4 ; v166 = 0
+;; v167 = iconst.i64 0
+;; @004e store notrap aligned region15 v167, v139+8 ; v167 = 0
+;; v168 = uextend.i128 v167 ; v167 = 0
+;; v169 = iconst.i64 64
+;; v170 = ishl v168, v169 ; v169 = 64
+;; @0040 v8 = bor v170, v168
;; @0056 jump block2(v8)
;;
-;; block2(v150: i128):
+;; block2(v151: i128):
;; @0058 jump block1
;;
;; block1:
diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.wat
index 8cc5e60c08..2318571bda 100644
--- a/tests/disas/stack-switching/symmetric-switch.wat
+++ b/tests/disas/stack-switching/symmetric-switch.wat
@@ -52,7 +52,7 @@
;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
;; gv2 = load.i64 notrap aligned region1 gv1+24
;; sig0 = (i64 vmctx, i32) -> i64 tail
-;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail
;; sig2 = (i64 vmctx, i32) -> i8 tail
;; fn0 = colocated u805306368:6 sig0
;; fn1 = colocated u805306368:42 sig1
@@ -65,215 +65,216 @@
;; @003c trapz v3, user16
;; @003c v4 = iconst.i32 1
;; @003c v5 = iconst.i32 0
-;; @003c v6 = call fn1(v0, v3, v4, v5) ; v4 = 1, v5 = 0
-;; @003c v7 = load.i64 notrap aligned region2 v6+88
-;; @003c v8 = uextend.i128 v6
+;; @003c v6 = iconst.i32 0
+;; @003c v7 = call fn1(v0, v3, v4, v5, v6) ; v4 = 1, v5 = 0, v6 = 0
+;; @003c v8 = load.i64 notrap aligned region2 v7+88
;; @003c v9 = uextend.i128 v7
-;; @003c v10 = iconst.i64 64
-;; @003c v11 = uextend.i128 v10 ; v10 = 64
-;; @003c v12 = ishl v9, v11
-;; @003c v13 = bor v12, v8
-;; @003e v14 = ireduce.i64 v13
-;; @003e v15 = iconst.i64 64
-;; @003e v16 = uextend.i128 v15 ; v15 = 64
-;; @003e v17 = ushr v13, v16
-;; @003e v18 = ireduce.i64 v17
-;; @003e trapz v14, user16
-;; @003e v19 = load.i64 notrap aligned region2 v14+88
-;; @003e v20 = icmp eq v19, v18
-;; @003e trapz v20, user23
-;; @003e v21 = iconst.i64 1
-;; @003e v22 = iadd v19, v21 ; v21 = 1
-;; @003e store notrap aligned region2 v22, v14+88
-;; @003e v23 = iconst.i64 48
-;; @003e v24 = iadd v0, v23 ; v23 = 48
-;; @003e v25 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @003e v26 = load.i64 notrap aligned region3 v25+88
-;; @003e v27 = load.i64 notrap aligned region3 v25+96
-;; @003e jump block2(v26, v27)
+;; @003c v10 = uextend.i128 v8
+;; @003c v11 = iconst.i64 64
+;; @003c v12 = uextend.i128 v11 ; v11 = 64
+;; @003c v13 = ishl v10, v12
+;; @003c v14 = bor v13, v9
+;; @003e v15 = ireduce.i64 v14
+;; @003e v16 = iconst.i64 64
+;; @003e v17 = uextend.i128 v16 ; v16 = 64
+;; @003e v18 = ushr v14, v17
+;; @003e v19 = ireduce.i64 v18
+;; @003e trapz v15, user16
+;; @003e v20 = load.i64 notrap aligned region2 v15+88
+;; @003e v21 = icmp eq v20, v19
+;; @003e trapz v21, user23
+;; @003e v22 = iconst.i64 1
+;; @003e v23 = iadd v20, v22 ; v22 = 1
+;; @003e store notrap aligned region2 v23, v15+88
+;; @003e v24 = iconst.i64 48
+;; @003e v25 = iadd v0, v24 ; v24 = 48
+;; @003e v26 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @003e v27 = load.i64 notrap aligned region3 v26+88
+;; @003e v28 = load.i64 notrap aligned region3 v26+96
+;; @003e jump block2(v27, v28)
;;
-;; block2(v28: i64, v29: i64):
-;; @003e v30 = iconst.i64 1
-;; @003e v31 = icmp eq v28, v30 ; v30 = 1
-;; @003e brif v31, block7, block3
+;; block2(v29: i64, v30: i64):
+;; @003e v31 = iconst.i64 1
+;; @003e v32 = icmp eq v29, v31 ; v31 = 1
+;; @003e brif v32, block7, block3
;;
;; block3:
-;; @003e v32 = load.i64 notrap aligned region4 v29+64
-;; @003e v33 = load.i64 notrap aligned region4 v29+72
-;; @003e v34 = iconst.i64 40
-;; @003e v35 = iadd v33, v34 ; v34 = 40
-;; @003e v36 = load.i64 notrap aligned region5 v35+8
-;; @003e v37 = load.i32 notrap aligned region6 v33+56
-;; @003e v38 = load.i32 notrap aligned region7 v35
-;; @003e jump block4(v37)
+;; @003e v33 = load.i64 notrap aligned region4 v30+64
+;; @003e v34 = load.i64 notrap aligned region4 v30+72
+;; @003e v35 = iconst.i64 40
+;; @003e v36 = iadd v34, v35 ; v35 = 40
+;; @003e v37 = load.i64 notrap aligned region5 v36+8
+;; @003e v38 = load.i32 notrap aligned region6 v34+56
+;; @003e v39 = load.i32 notrap aligned region7 v36
+;; @003e jump block4(v38)
;;
-;; block4(v39: i32):
-;; @003e v40 = icmp ult v39, v38
-;; @003e brif v40, block5, block2(v32, v33)
+;; block4(v40: i32):
+;; @003e v41 = icmp ult v40, v39
+;; @003e brif v41, block5, block2(v33, v34)
;;
;; block5:
-;; @003e v41 = iconst.i32 8
-;; @003e v42 = imul.i32 v39, v41 ; v41 = 8
-;; @003e v43 = uextend.i64 v42
-;; @003e v44 = iadd.i64 v36, v43
-;; @003e v45 = load.i64 notrap aligned region8 v44
-;; @003e v46 = icmp eq v45, v24
-;; @003e v47 = iconst.i32 1
-;; @003e v48 = iadd.i32 v39, v47 ; v47 = 1
-;; @003e brif v46, block6, block4(v48)
+;; @003e v42 = iconst.i32 8
+;; @003e v43 = imul.i32 v40, v42 ; v42 = 8
+;; @003e v44 = uextend.i64 v43
+;; @003e v45 = iadd.i64 v37, v44
+;; @003e v46 = load.i64 notrap aligned region8 v45
+;; @003e v47 = icmp eq v46, v25
+;; @003e v48 = iconst.i32 1
+;; @003e v49 = iadd.i32 v40, v48 ; v48 = 1
+;; @003e brif v47, block6, block4(v49)
;;
;; block7 cold:
;; @003e trap user22
;;
;; block6:
-;; @003e store.i64 notrap aligned region9 v29, v27+80
-;; @003e v49 = iconst.i64 136
-;; @003e v50 = iadd.i64 v27, v49 ; v49 = 136
-;; @003e v51 = iconst.i32 1
-;; @003e v52 = stack_addr.i64 ss0
-;; @003e store notrap aligned region10 v51, v50+4 ; v51 = 1
-;; @003e store notrap aligned region5 v52, v50+8
-;; @003e v53 = iconst.i64 0
-;; @003e v54 = iadd.i64 v27, v53 ; v53 = 0
-;; @003e v55 = iconst.i32 3
-;; @003e store notrap aligned region11 v55, v54+32 ; v55 = 3
-;; @003e v56 = iconst.i64 0
+;; @003e store.i64 notrap aligned region9 v30, v28+80
+;; @003e v50 = iconst.i64 144
+;; @003e v51 = iadd.i64 v28, v50 ; v50 = 144
+;; @003e v52 = iconst.i32 1
+;; @003e v53 = stack_addr.i64 ss0
+;; @003e store notrap aligned region10 v52, v51+4 ; v52 = 1
+;; @003e store notrap aligned region5 v53, v51+8
+;; @003e v54 = iconst.i64 0
+;; @003e v55 = iadd.i64 v28, v54 ; v54 = 0
+;; @003e v56 = iconst.i32 3
+;; @003e store notrap aligned region11 v56, v55+32 ; v56 = 3
;; @003e v57 = iconst.i64 0
-;; @003e store notrap aligned region4 v56, v29+64 ; v56 = 0
-;; @003e store notrap aligned region4 v57, v29+72 ; v57 = 0
-;; @003e v58 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @003e v59 = iconst.i64 0
-;; @003e v60 = iadd v54, v59 ; v59 = 0
-;; @003e v61 = load.i64 notrap aligned region12 v58+72
-;; @003e v62 = load.i64 notrap aligned region13 v58+64
-;; @003e v63 = load.i64 notrap aligned region14 v58+80
-;; @003e store notrap aligned region15 v61, v60+8
-;; @003e store notrap aligned region16 v62, v60+16
-;; @003e store notrap aligned region6 v63, v60+24
-;; @003e v64 = load.i64 notrap aligned region2 v27+88
-;; @003e v65 = uextend.i128 v27
-;; @003e v66 = uextend.i128 v64
-;; @003e v67 = iconst.i64 64
-;; @003e v68 = uextend.i128 v67 ; v67 = 64
-;; @003e v69 = ishl v66, v68
-;; @003e v70 = bor v69, v65
-;; @003e v72 = iconst.i64 0
-;; @003e v73 = iadd.i64 v14, v72 ; v72 = 0
-;; @003e v74 = load.i32 notrap aligned region11 v73+32
-;; @003e v75 = iconst.i32 0
-;; @003e v76 = icmp ne v74, v75 ; v75 = 0
-;; @003e brif v76, block9, block8
+;; @003e v58 = iconst.i64 0
+;; @003e store notrap aligned region4 v57, v30+64 ; v57 = 0
+;; @003e store notrap aligned region4 v58, v30+72 ; v58 = 0
+;; @003e v59 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @003e v60 = iconst.i64 0
+;; @003e v61 = iadd v55, v60 ; v60 = 0
+;; @003e v62 = load.i64 notrap aligned region12 v59+72
+;; @003e v63 = load.i64 notrap aligned region13 v59+64
+;; @003e v64 = load.i64 notrap aligned region14 v59+80
+;; @003e store notrap aligned region15 v62, v61+8
+;; @003e store notrap aligned region16 v63, v61+16
+;; @003e store notrap aligned region6 v64, v61+24
+;; @003e v65 = load.i64 notrap aligned region2 v28+88
+;; @003e v66 = uextend.i128 v28
+;; @003e v67 = uextend.i128 v65
+;; @003e v68 = iconst.i64 64
+;; @003e v69 = uextend.i128 v68 ; v68 = 64
+;; @003e v70 = ishl v67, v69
+;; @003e v71 = bor v70, v66
+;; @003e v73 = iconst.i64 0
+;; @003e v74 = iadd.i64 v15, v73 ; v73 = 0
+;; @003e v75 = load.i32 notrap aligned region11 v74+32
+;; @003e v76 = iconst.i32 0
+;; @003e v77 = icmp ne v75, v76 ; v76 = 0
+;; @003e brif v77, block9, block8
;;
;; block8:
-;; @003e v77 = iconst.i64 120
-;; @003e v78 = iadd.i64 v14, v77 ; v77 = 120
-;; @003e v79 = load.i64 notrap aligned region5 v78+8
-;; @003e v80 = load.i32 notrap aligned region7 v78
-;; @003e v81 = iconst.i32 1
-;; @003e v82 = iadd v80, v81 ; v81 = 1
-;; @003e store notrap aligned region7 v82, v78
-;; @003e v83 = uextend.i64 v80
-;; @003e v84 = iconst.i64 16
-;; @003e v85 = imul v83, v84 ; v84 = 16
-;; @003e v86 = iadd v79, v85
-;; @003e jump block10(v86)
+;; @003e v78 = iconst.i64 120
+;; @003e v79 = iadd.i64 v15, v78 ; v78 = 120
+;; @003e v80 = load.i64 notrap aligned region5 v79+8
+;; @003e v81 = load.i32 notrap aligned region7 v79
+;; @003e v82 = iconst.i32 1
+;; @003e v83 = iadd v81, v82 ; v82 = 1
+;; @003e store notrap aligned region7 v83, v79
+;; @003e v84 = uextend.i64 v81
+;; @003e v85 = iconst.i64 16
+;; @003e v86 = imul v84, v85 ; v85 = 16
+;; @003e v87 = iadd v80, v86
+;; @003e jump block10(v87)
;;
;; block9:
-;; @003e v87 = iconst.i64 136
-;; @003e v88 = iadd.i64 v14, v87 ; v87 = 136
-;; @003e v89 = load.i64 notrap aligned region5 v88+8
-;; @003e v90 = load.i32 notrap aligned region7 v88
-;; @003e v91 = iconst.i32 1
-;; @003e v92 = iadd v90, v91 ; v91 = 1
-;; @003e store notrap aligned region7 v92, v88
-;; @003e v93 = uextend.i64 v90
-;; @003e v94 = iconst.i64 16
-;; @003e v95 = imul v93, v94 ; v94 = 16
-;; @003e v96 = iadd v89, v95
-;; @003e jump block10(v96)
+;; @003e v88 = iconst.i64 144
+;; @003e v89 = iadd.i64 v15, v88 ; v88 = 144
+;; @003e v90 = load.i64 notrap aligned region5 v89+8
+;; @003e v91 = load.i32 notrap aligned region7 v89
+;; @003e v92 = iconst.i32 1
+;; @003e v93 = iadd v91, v92 ; v92 = 1
+;; @003e store notrap aligned region7 v93, v89
+;; @003e v94 = uextend.i64 v91
+;; @003e v95 = iconst.i64 16
+;; @003e v96 = imul v94, v95 ; v95 = 16
+;; @003e v97 = iadd v90, v96
+;; @003e jump block10(v97)
;;
-;; block10(v71: i64):
-;; @003e store.i128 notrap aligned region8 v70, v71
-;; @003e v97 = iconst.i64 0
-;; @003e v98 = iadd.i64 v14, v97 ; v97 = 0
-;; @003e v99 = iconst.i32 1
-;; @003e store notrap aligned region11 v99, v98+32 ; v99 = 1
-;; @003e v100 = load.i64 notrap aligned region9 v14+80
-;; @003e store.i64 notrap aligned region4 v32, v100+64
-;; @003e store.i64 notrap aligned region4 v33, v100+72
-;; @003e v101 = iconst.i64 2
-;; @003e v102 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @003e store notrap aligned region3 v101, v102+88 ; v101 = 2
-;; @003e store.i64 notrap aligned region3 v14, v102+96
-;; @003e v103 = iconst.i64 0
-;; @003e v104 = iadd v98, v103 ; v103 = 0
-;; @003e v105 = load.i64 notrap aligned region17 v104
-;; @003e store notrap aligned region1 v105, v58+24
-;; @003e v106 = load.i64 notrap aligned region15 v104+8
-;; @003e store notrap aligned region12 v106, v58+72
-;; @003e v107 = load.i64 notrap aligned region16 v104+16
-;; @003e store notrap aligned region13 v107, v58+64
-;; @003e v108 = load.i64 notrap aligned region6 v104+24
-;; @003e store notrap aligned region14 v108, v58+80
-;; @003e v109 = iconst.i64 96
-;; @003e v110 = iadd.i64 v29, v109 ; v109 = 96
-;; @003e v111 = load.i64 notrap aligned region18 v110
-;; @003e v112 = iconst.i64 -24
-;; @003e v113 = iadd v111, v112 ; v112 = -24
-;; @003e v114 = iconst.i64 96
-;; @003e v115 = iadd v100, v114 ; v114 = 96
-;; @003e v116 = load.i64 notrap aligned region18 v115
-;; @003e v117 = iconst.i64 -24
-;; @003e v118 = iadd v116, v117 ; v117 = -24
-;; @003e v119 = stack_addr.i64 ss1
-;; @003e v120 = load.i64 notrap aligned region8 v118
-;; @003e store notrap aligned region19 v120, v119
-;; @003e v121 = load.i64 notrap aligned region8 v113
-;; @003e store notrap aligned region8 v121, v118
-;; @003e v122 = load.i64 notrap aligned region8 v118+8
-;; @003e store notrap aligned region19 v122, v119+8
-;; @003e v123 = load.i64 notrap aligned region8 v113+8
-;; @003e store notrap aligned region8 v123, v118+8
-;; @003e v124 = load.i64 notrap aligned region8 v118+16
-;; @003e store notrap aligned region19 v124, v119+16
-;; @003e v125 = load.i64 notrap aligned region8 v113+16
-;; @003e store notrap aligned region8 v125, v118+16
-;; @003e v126 = iconst.i64 3
-;; @003e v127 = iconst.i64 32
-;; @003e v128 = ishl v126, v127 ; v126 = 3, v127 = 32
-;; @003e v129 = stack_switch v113, v119, v128
-;; @003e v130 = iconst.i64 32
-;; @003e v131 = ushr v129, v130 ; v130 = 32
-;; @003e v132 = iconst.i64 5
-;; @003e v133 = icmp eq v131, v132 ; v132 = 5
-;; @003e brif v133, block11, block12
+;; block10(v72: i64):
+;; @003e store.i128 notrap aligned region8 v71, v72
+;; @003e v98 = iconst.i64 0
+;; @003e v99 = iadd.i64 v15, v98 ; v98 = 0
+;; @003e v100 = iconst.i32 1
+;; @003e store notrap aligned region11 v100, v99+32 ; v100 = 1
+;; @003e v101 = load.i64 notrap aligned region9 v15+80
+;; @003e store.i64 notrap aligned region4 v33, v101+64
+;; @003e store.i64 notrap aligned region4 v34, v101+72
+;; @003e v102 = iconst.i64 2
+;; @003e v103 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @003e store notrap aligned region3 v102, v103+88 ; v102 = 2
+;; @003e store.i64 notrap aligned region3 v15, v103+96
+;; @003e v104 = iconst.i64 0
+;; @003e v105 = iadd v99, v104 ; v104 = 0
+;; @003e v106 = load.i64 notrap aligned region17 v105
+;; @003e store notrap aligned region1 v106, v59+24
+;; @003e v107 = load.i64 notrap aligned region15 v105+8
+;; @003e store notrap aligned region12 v107, v59+72
+;; @003e v108 = load.i64 notrap aligned region16 v105+16
+;; @003e store notrap aligned region13 v108, v59+64
+;; @003e v109 = load.i64 notrap aligned region6 v105+24
+;; @003e store notrap aligned region14 v109, v59+80
+;; @003e v110 = iconst.i64 96
+;; @003e v111 = iadd.i64 v30, v110 ; v110 = 96
+;; @003e v112 = load.i64 notrap aligned region18 v111
+;; @003e v113 = iconst.i64 -24
+;; @003e v114 = iadd v112, v113 ; v113 = -24
+;; @003e v115 = iconst.i64 96
+;; @003e v116 = iadd v101, v115 ; v115 = 96
+;; @003e v117 = load.i64 notrap aligned region18 v116
+;; @003e v118 = iconst.i64 -24
+;; @003e v119 = iadd v117, v118 ; v118 = -24
+;; @003e v120 = stack_addr.i64 ss1
+;; @003e v121 = load.i64 notrap aligned region8 v119
+;; @003e store notrap aligned region19 v121, v120
+;; @003e v122 = load.i64 notrap aligned region8 v114
+;; @003e store notrap aligned region8 v122, v119
+;; @003e v123 = load.i64 notrap aligned region8 v119+8
+;; @003e store notrap aligned region19 v123, v120+8
+;; @003e v124 = load.i64 notrap aligned region8 v114+8
+;; @003e store notrap aligned region8 v124, v119+8
+;; @003e v125 = load.i64 notrap aligned region8 v119+16
+;; @003e store notrap aligned region19 v125, v120+16
+;; @003e v126 = load.i64 notrap aligned region8 v114+16
+;; @003e store notrap aligned region8 v126, v119+16
+;; @003e v127 = iconst.i64 3
+;; @003e v128 = iconst.i64 32
+;; @003e v129 = ishl v127, v128 ; v127 = 3, v128 = 32
+;; @003e v130 = stack_switch v114, v120, v129
+;; @003e v131 = iconst.i64 32
+;; @003e v132 = ushr v130, v131 ; v131 = 32
+;; @003e v133 = iconst.i64 5
+;; @003e v134 = icmp eq v132, v133 ; v133 = 5
+;; @003e brif v134, block11, block12
;;
;; block11 cold:
-;; @003e v134 = iconst.i64 136
-;; @003e v135 = iadd.i64 v27, v134 ; v134 = 136
-;; @003e v136 = load.i64 notrap aligned region5 v135+8
-;; @003e v137 = load.i32 notrap aligned region8 v136
-;; @003e v138 = iconst.i32 0
-;; @003e store notrap aligned region7 v138, v135 ; v138 = 0
+;; @003e v135 = iconst.i64 144
+;; @003e v136 = iadd.i64 v28, v135 ; v135 = 144
+;; @003e v137 = load.i64 notrap aligned region5 v136+8
+;; @003e v138 = load.i32 notrap aligned region8 v137
;; @003e v139 = iconst.i32 0
-;; @003e store notrap aligned region10 v139, v135+4 ; v139 = 0
-;; @003e v140 = iconst.i64 0
-;; @003e store notrap aligned region5 v140, v135+8 ; v140 = 0
-;; @003e try_call fn2(v0, v137), sig2, block13, [ context v0 ]
+;; @003e store notrap aligned region7 v139, v136 ; v139 = 0
+;; @003e v140 = iconst.i32 0
+;; @003e store notrap aligned region10 v140, v136+4 ; v140 = 0
+;; @003e v141 = iconst.i64 0
+;; @003e store notrap aligned region5 v141, v136+8 ; v141 = 0
+;; @003e try_call fn2(v0, v138), sig2, block13, [ context v0 ]
;;
;; block13:
;; @003e trap user12
;;
;; block12:
-;; @003e v141 = iconst.i64 136
-;; @003e v142 = iadd.i64 v27, v141 ; v141 = 136
-;; @003e v143 = load.i64 notrap aligned region5 v142+8
-;; @003e v144 = iconst.i32 0
-;; @003e store notrap aligned region7 v144, v142 ; v144 = 0
+;; @003e v142 = iconst.i64 144
+;; @003e v143 = iadd.i64 v28, v142 ; v142 = 144
+;; @003e v144 = load.i64 notrap aligned region5 v143+8
;; @003e v145 = iconst.i32 0
-;; @003e store notrap aligned region10 v145, v142+4 ; v145 = 0
-;; @003e v146 = iconst.i64 0
-;; @003e store notrap aligned region5 v146, v142+8 ; v146 = 0
+;; @003e store notrap aligned region7 v145, v143 ; v145 = 0
+;; @003e v146 = iconst.i32 0
+;; @003e store notrap aligned region10 v146, v143+4 ; v146 = 0
+;; @003e v147 = iconst.i64 0
+;; @003e store notrap aligned region5 v147, v143+8 ; v147 = 0
;; @0041 jump block1
;;
;; block1:
@@ -320,7 +321,7 @@
;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
;; gv2 = load.i64 notrap aligned region1 gv1+24
;; sig0 = (i64 vmctx, i32) -> i64 tail
-;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
+;; sig1 = (i64 vmctx, i64, i32, i32, i32) -> i64 tail
;; sig2 = (i64 vmctx) tail
;; fn0 = colocated u805306368:6 sig0
;; fn1 = colocated u805306368:42 sig1
@@ -333,212 +334,213 @@
;; @0049 trapz v3, user16
;; @0049 v4 = iconst.i32 0
;; @0049 v5 = iconst.i32 0
-;; @0049 v6 = call fn1(v0, v3, v4, v5) ; v4 = 0, v5 = 0
-;; @0049 v7 = load.i64 notrap aligned region2 v6+88
-;; @0049 v8 = uextend.i128 v6
+;; @0049 v6 = iconst.i32 0
+;; @0049 v7 = call fn1(v0, v3, v4, v5, v6) ; v4 = 0, v5 = 0, v6 = 0
+;; @0049 v8 = load.i64 notrap aligned region2 v7+88
;; @0049 v9 = uextend.i128 v7
-;; @0049 v10 = iconst.i64 64
-;; @0049 v11 = uextend.i128 v10 ; v10 = 64
-;; @0049 v12 = ishl v9, v11
-;; @0049 v13 = bor v12, v8
+;; @0049 v10 = uextend.i128 v8
+;; @0049 v11 = iconst.i64 64
+;; @0049 v12 = uextend.i128 v11 ; v11 = 64
+;; @0049 v13 = ishl v10, v12
+;; @0049 v14 = bor v13, v9
;; @004b jump block2
;;
;; block2:
-;; @004b v14 = ireduce.i64 v13
-;; @004b v15 = iconst.i64 64
-;; @004b v16 = uextend.i128 v15 ; v15 = 64
-;; @004b v17 = ushr.i128 v13, v16
-;; @004b v18 = ireduce.i64 v17
-;; @004b trapz v14, user16
-;; @004b v19 = load.i64 notrap aligned region2 v14+88
-;; @004b v20 = icmp eq v19, v18
-;; @004b trapz v20, user23
-;; @004b v21 = iconst.i64 1
-;; @004b v22 = iadd v19, v21 ; v21 = 1
-;; @004b store notrap aligned region2 v22, v14+88
-;; @004b v23 = load.i64 notrap aligned region3 v14+80
-;; @004b v24 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @004b v25 = load.i64 notrap aligned region4 v24+88
-;; @004b v26 = load.i64 notrap aligned region4 v24+96
-;; @004b store notrap aligned region5 v25, v23+64
-;; @004b store notrap aligned region5 v26, v23+72
-;; @004b v27 = iconst.i64 0
-;; @004b store notrap aligned region3 v27, v14+80 ; v27 = 0
-;; @004b v28 = iconst.i64 2
-;; @004b v29 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @004b store notrap aligned region4 v28, v29+88 ; v28 = 2
-;; @004b store notrap aligned region4 v14, v29+96
-;; @004b v30 = iconst.i64 0
-;; @004b v31 = iadd v14, v30 ; v30 = 0
-;; @004b v32 = iconst.i32 1
-;; @004b store notrap aligned region6 v32, v31+32 ; v32 = 1
-;; @004b v33 = iconst.i32 2
-;; @004b store notrap aligned region6 v33, v26+32 ; v33 = 2
-;; @004b v34 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @004b v35 = iconst.i64 0
-;; @004b v36 = iadd v26, v35 ; v35 = 0
-;; @004b v37 = load.i64 notrap aligned region7 v34+72
-;; @004b v38 = load.i64 notrap aligned region8 v34+64
-;; @004b v39 = load.i64 notrap aligned region9 v34+80
-;; @004b store notrap aligned region10 v37, v36+8
-;; @004b store notrap aligned region11 v38, v36+16
-;; @004b store notrap aligned region12 v39, v36+24
-;; @004b v40 = load.i64 notrap aligned region1 v34+24
-;; @004b store notrap aligned region13 v40, v36
-;; @004b v41 = iconst.i64 0
-;; @004b v42 = iadd v31, v41 ; v41 = 0
-;; @004b v43 = load.i64 notrap aligned region13 v42
-;; @004b store notrap aligned region1 v43, v34+24
-;; @004b v44 = load.i64 notrap aligned region10 v42+8
-;; @004b store notrap aligned region7 v44, v34+72
-;; @004b v45 = load.i64 notrap aligned region11 v42+16
-;; @004b store notrap aligned region8 v45, v34+64
-;; @004b v46 = load.i64 notrap aligned region12 v42+24
-;; @004b store notrap aligned region9 v46, v34+80
-;; @004b v47 = iconst.i64 40
-;; @004b v48 = iadd v26, v47 ; v47 = 40
-;; @004b v49 = iconst.i32 1
-;; @004b v50 = stack_addr.i64 ss0
-;; @004b store notrap aligned region14 v49, v48+4 ; v49 = 1
-;; @004b store notrap aligned region15 v50, v48+8
-;; @004b v51 = iconst.i64 48
-;; @004b v52 = iadd.i64 v0, v51 ; v51 = 48
-;; @004b v53 = iconst.i32 1
-;; @004b v54 = load.i64 notrap aligned region15 v48+8
-;; @004b store notrap aligned region16 v52, v54
-;; @004b store notrap aligned region17 v53, v48 ; v53 = 1
-;; @004b v55 = iconst.i32 0
-;; @004b store notrap aligned region12 v55, v26+56 ; v55 = 0
-;; @004b v56 = iconst.i64 1
-;; @004b v57 = iconst.i64 32
-;; @004b v58 = ishl v56, v57 ; v56 = 1, v57 = 32
-;; @004b v59 = iconst.i64 96
-;; @004b v60 = iadd v23, v59 ; v59 = 96
-;; @004b v61 = load.i64 notrap aligned region18 v60
-;; @004b v62 = iconst.i64 -24
-;; @004b v63 = iadd v61, v62 ; v62 = -24
-;; @004b v64 = stack_switch v63, v63, v58
-;; @004b v65 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @004b v66 = load.i64 notrap aligned region4 v65+88
-;; @004b v67 = load.i64 notrap aligned region4 v65+96
-;; @004b v68 = load.i64 notrap aligned readonly can_move region0 v0+8
-;; @004b store notrap aligned region4 v25, v68+88
-;; @004b store notrap aligned region4 v26, v68+96
-;; @004b v69 = iconst.i32 1
-;; @004b store notrap aligned region6 v69, v26+32 ; v69 = 1
-;; @004b v70 = iconst.i32 0
-;; @004b store notrap aligned region17 v70, v48 ; v70 = 0
+;; @004b v15 = ireduce.i64 v14
+;; @004b v16 = iconst.i64 64
+;; @004b v17 = uextend.i128 v16 ; v16 = 64
+;; @004b v18 = ushr.i128 v14, v17
+;; @004b v19 = ireduce.i64 v18
+;; @004b trapz v15, user16
+;; @004b v20 = load.i64 notrap aligned region2 v15+88
+;; @004b v21 = icmp eq v20, v19
+;; @004b trapz v21, user23
+;; @004b v22 = iconst.i64 1
+;; @004b v23 = iadd v20, v22 ; v22 = 1
+;; @004b store notrap aligned region2 v23, v15+88
+;; @004b v24 = load.i64 notrap aligned region3 v15+80
+;; @004b v25 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @004b v26 = load.i64 notrap aligned region4 v25+88
+;; @004b v27 = load.i64 notrap aligned region4 v25+96
+;; @004b store notrap aligned region5 v26, v24+64
+;; @004b store notrap aligned region5 v27, v24+72
+;; @004b v28 = iconst.i64 0
+;; @004b store notrap aligned region3 v28, v15+80 ; v28 = 0
+;; @004b v29 = iconst.i64 2
+;; @004b v30 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @004b store notrap aligned region4 v29, v30+88 ; v29 = 2
+;; @004b store notrap aligned region4 v15, v30+96
+;; @004b v31 = iconst.i64 0
+;; @004b v32 = iadd v15, v31 ; v31 = 0
+;; @004b v33 = iconst.i32 1
+;; @004b store notrap aligned region6 v33, v32+32 ; v33 = 1
+;; @004b v34 = iconst.i32 2
+;; @004b store notrap aligned region6 v34, v27+32 ; v34 = 2
+;; @004b v35 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @004b v36 = iconst.i64 0
+;; @004b v37 = iadd v27, v36 ; v36 = 0
+;; @004b v38 = load.i64 notrap aligned region7 v35+72
+;; @004b v39 = load.i64 notrap aligned region8 v35+64
+;; @004b v40 = load.i64 notrap aligned region9 v35+80
+;; @004b store notrap aligned region10 v38, v37+8
+;; @004b store notrap aligned region11 v39, v37+16
+;; @004b store notrap aligned region12 v40, v37+24
+;; @004b v41 = load.i64 notrap aligned region1 v35+24
+;; @004b store notrap aligned region13 v41, v37
+;; @004b v42 = iconst.i64 0
+;; @004b v43 = iadd v32, v42 ; v42 = 0
+;; @004b v44 = load.i64 notrap aligned region13 v43
+;; @004b store notrap aligned region1 v44, v35+24
+;; @004b v45 = load.i64 notrap aligned region10 v43+8
+;; @004b store notrap aligned region7 v45, v35+72
+;; @004b v46 = load.i64 notrap aligned region11 v43+16
+;; @004b store notrap aligned region8 v46, v35+64
+;; @004b v47 = load.i64 notrap aligned region12 v43+24
+;; @004b store notrap aligned region9 v47, v35+80
+;; @004b v48 = iconst.i64 40
+;; @004b v49 = iadd v27, v48 ; v48 = 40
+;; @004b v50 = iconst.i32 1
+;; @004b v51 = stack_addr.i64 ss0
+;; @004b store notrap aligned region14 v50, v49+4 ; v50 = 1
+;; @004b store notrap aligned region15 v51, v49+8
+;; @004b v52 = iconst.i64 48
+;; @004b v53 = iadd.i64 v0, v52 ; v52 = 48
+;; @004b v54 = iconst.i32 1
+;; @004b v55 = load.i64 notrap aligned region15 v49+8
+;; @004b store notrap aligned region16 v53, v55
+;; @004b store notrap aligned region17 v54, v49 ; v54 = 1
+;; @004b v56 = iconst.i32 0
+;; @004b store notrap aligned region12 v56, v27+56 ; v56 = 0
+;; @004b v57 = iconst.i64 1
+;; @004b v58 = iconst.i64 32
+;; @004b v59 = ishl v57, v58 ; v57 = 1, v58 = 32
+;; @004b v60 = iconst.i64 96
+;; @004b v61 = iadd v24, v60 ; v60 = 96
+;; @004b v62 = load.i64 notrap aligned region18 v61
+;; @004b v63 = iconst.i64 -24
+;; @004b v64 = iadd v62, v63 ; v63 = -24
+;; @004b v65 = stack_switch v64, v64, v59
+;; @004b v66 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @004b v67 = load.i64 notrap aligned region4 v66+88
+;; @004b v68 = load.i64 notrap aligned region4 v66+96
+;; @004b v69 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @004b store notrap aligned region4 v26, v69+88
+;; @004b store notrap aligned region4 v27, v69+96
+;; @004b v70 = iconst.i32 1
+;; @004b store notrap aligned region6 v70, v27+32 ; v70 = 1
;; @004b v71 = iconst.i32 0
-;; @004b store notrap aligned region14 v71, v48+4 ; v71 = 0
-;; @004b v72 = iconst.i64 0
-;; @004b store notrap aligned region15 v72, v48+8 ; v72 = 0
-;; @004b store notrap aligned region12 v27, v26+56 ; v27 = 0
-;; @004b brif v64, block6, block3
+;; @004b store notrap aligned region17 v71, v49 ; v71 = 0
+;; @004b v72 = iconst.i32 0
+;; @004b store notrap aligned region14 v72, v49+4 ; v72 = 0
+;; @004b v73 = iconst.i64 0
+;; @004b store notrap aligned region15 v73, v49+8 ; v73 = 0
+;; @004b store notrap aligned region12 v28, v27+56 ; v28 = 0
+;; @004b brif v65, block6, block3
;;
;; block6:
-;; @004b v73 = iconst.i64 32
-;; @004b v74 = ushr.i64 v64, v73 ; v73 = 32
-;; @004b v75 = iconst.i64 4
-;; @004b v76 = icmp eq v74, v75 ; v75 = 4
-;; @004b brif v76, block5, block4
+;; @004b v74 = iconst.i64 32
+;; @004b v75 = ushr.i64 v65, v74 ; v74 = 32
+;; @004b v76 = iconst.i64 4
+;; @004b v77 = icmp eq v75, v76 ; v76 = 4
+;; @004b brif v77, block5, block4
;;
;; block5 cold:
-;; @004b v77 = iconst.i64 0
-;; @004b v78 = iadd.i64 v67, v77 ; v77 = 0
-;; @004b v79 = iconst.i32 5
-;; @004b store notrap aligned region6 v79, v78+32 ; v79 = 5
-;; @004b v80 = iconst.i64 0
-;; @004b v81 = iadd.i64 v26, v80 ; v80 = 0
-;; @004b v82 = load.i64 notrap aligned region13 v81
-;; @004b store notrap aligned region1 v82, v34+24
-;; @004b v83 = load.i64 notrap aligned region10 v81+8
-;; @004b store notrap aligned region7 v83, v34+72
-;; @004b v84 = load.i64 notrap aligned region11 v81+16
-;; @004b store notrap aligned region8 v84, v34+64
-;; @004b v85 = load.i64 notrap aligned region12 v81+24
-;; @004b store notrap aligned region9 v85, v34+80
-;; @004b v86 = iconst.i64 120
-;; @004b v87 = iadd.i64 v67, v86 ; v86 = 120
-;; @004b v88 = iconst.i32 0
-;; @004b store notrap aligned region17 v88, v87 ; v88 = 0
+;; @004b v78 = iconst.i64 0
+;; @004b v79 = iadd.i64 v68, v78 ; v78 = 0
+;; @004b v80 = iconst.i32 5
+;; @004b store notrap aligned region6 v80, v79+32 ; v80 = 5
+;; @004b v81 = iconst.i64 0
+;; @004b v82 = iadd.i64 v27, v81 ; v81 = 0
+;; @004b v83 = load.i64 notrap aligned region13 v82
+;; @004b store notrap aligned region1 v83, v35+24
+;; @004b v84 = load.i64 notrap aligned region10 v82+8
+;; @004b store notrap aligned region7 v84, v35+72
+;; @004b v85 = load.i64 notrap aligned region11 v82+16
+;; @004b store notrap aligned region8 v85, v35+64
+;; @004b v86 = load.i64 notrap aligned region12 v82+24
+;; @004b store notrap aligned region9 v86, v35+80
+;; @004b v87 = iconst.i64 120
+;; @004b v88 = iadd.i64 v68, v87 ; v87 = 120
;; @004b v89 = iconst.i32 0
-;; @004b store notrap aligned region14 v89, v87+4 ; v89 = 0
-;; @004b v90 = iconst.i64 0
-;; @004b store notrap aligned region15 v90, v87+8 ; v90 = 0
-;; @004b v91 = iconst.i64 136
-;; @004b v92 = iadd.i64 v67, v91 ; v91 = 136
-;; @004b v93 = iconst.i32 0
-;; @004b store notrap aligned region17 v93, v92 ; v93 = 0
+;; @004b store notrap aligned region17 v89, v88 ; v89 = 0
+;; @004b v90 = iconst.i32 0
+;; @004b store notrap aligned region14 v90, v88+4 ; v90 = 0
+;; @004b v91 = iconst.i64 0
+;; @004b store notrap aligned region15 v91, v88+8 ; v91 = 0
+;; @004b v92 = iconst.i64 144
+;; @004b v93 = iadd.i64 v68, v92 ; v92 = 144
;; @004b v94 = iconst.i32 0
-;; @004b store notrap aligned region14 v94, v92+4 ; v94 = 0
-;; @004b v95 = iconst.i64 0
-;; @004b store notrap aligned region15 v95, v92+8 ; v95 = 0
+;; @004b store notrap aligned region17 v94, v93 ; v94 = 0
+;; @004b v95 = iconst.i32 0
+;; @004b store notrap aligned region14 v95, v93+4 ; v95 = 0
+;; @004b v96 = iconst.i64 0
+;; @004b store notrap aligned region15 v96, v93+8 ; v96 = 0
;; @004b try_call fn2(v0), sig2, block8, [ context v0 ]
;;
;; block8:
;; @004b trap user12
;;
;; block4:
-;; @004b v96 = iconst.i64 0
-;; @004b v97 = iadd.i64 v67, v96 ; v96 = 0
-;; @004b v98 = iconst.i64 0
-;; @004b v99 = iadd v97, v98 ; v98 = 0
-;; @004b v100 = load.i64 notrap aligned region7 v34+72
-;; @004b v101 = load.i64 notrap aligned region8 v34+64
-;; @004b v102 = load.i64 notrap aligned region9 v34+80
-;; @004b store notrap aligned region10 v100, v99+8
-;; @004b store notrap aligned region11 v101, v99+16
-;; @004b store notrap aligned region12 v102, v99+24
-;; @004b v103 = iconst.i64 0
-;; @004b v104 = iadd.i64 v26, v103 ; v103 = 0
-;; @004b v105 = load.i64 notrap aligned region13 v104
-;; @004b store notrap aligned region1 v105, v34+24
-;; @004b v106 = load.i64 notrap aligned region10 v104+8
-;; @004b store notrap aligned region7 v106, v34+72
-;; @004b v107 = load.i64 notrap aligned region11 v104+16
-;; @004b store notrap aligned region8 v107, v34+64
-;; @004b v108 = load.i64 notrap aligned region12 v104+24
-;; @004b store notrap aligned region9 v108, v34+80
-;; @004b v109 = ireduce.i32 v64
-;; @004b v110 = load.i64 notrap aligned region2 v67+88
-;; @004b v111 = uextend.i128 v67
-;; @004b v112 = uextend.i128 v110
-;; @004b v113 = iconst.i64 64
-;; @004b v114 = uextend.i128 v113 ; v113 = 64
-;; @004b v115 = ishl v112, v114
-;; @004b v116 = bor v115, v111
+;; @004b v97 = iconst.i64 0
+;; @004b v98 = iadd.i64 v68, v97 ; v97 = 0
+;; @004b v99 = iconst.i64 0
+;; @004b v100 = iadd v98, v99 ; v99 = 0
+;; @004b v101 = load.i64 notrap aligned region7 v35+72
+;; @004b v102 = load.i64 notrap aligned region8 v35+64
+;; @004b v103 = load.i64 notrap aligned region9 v35+80
+;; @004b store notrap aligned region10 v101, v100+8
+;; @004b store notrap aligned region11 v102, v100+16
+;; @004b store notrap aligned region12 v103, v100+24
+;; @004b v104 = iconst.i64 0
+;; @004b v105 = iadd.i64 v27, v104 ; v104 = 0
+;; @004b v106 = load.i64 notrap aligned region13 v105
+;; @004b store notrap aligned region1 v106, v35+24
+;; @004b v107 = load.i64 notrap aligned region10 v105+8
+;; @004b store notrap aligned region7 v107, v35+72
+;; @004b v108 = load.i64 notrap aligned region11 v105+16
+;; @004b store notrap aligned region8 v108, v35+64
+;; @004b v109 = load.i64 notrap aligned region12 v105+24
+;; @004b store notrap aligned region9 v109, v35+80
+;; @004b v110 = ireduce.i32 v65
+;; @004b v111 = load.i64 notrap aligned region2 v68+88
+;; @004b v112 = uextend.i128 v68
+;; @004b v113 = uextend.i128 v111
+;; @004b v114 = iconst.i64 64
+;; @004b v115 = uextend.i128 v114 ; v114 = 64
+;; @004b v116 = ishl v113, v115
+;; @004b v117 = bor v116, v112
;; @004b jump block7
;;
;; block9 cold:
;; @004b trap user12
;;
;; block7:
-;; @004b br_table v109, block9, []
+;; @004b br_table v110, block9, []
;;
;; block3:
-;; @004b v117 = iconst.i64 0
-;; @004b v118 = iadd.i64 v26, v117 ; v117 = 0
-;; @004b v119 = load.i64 notrap aligned region13 v118
-;; @004b store notrap aligned region1 v119, v34+24
-;; @004b v120 = load.i64 notrap aligned region10 v118+8
-;; @004b store notrap aligned region7 v120, v34+72
-;; @004b v121 = load.i64 notrap aligned region11 v118+16
-;; @004b store notrap aligned region8 v121, v34+64
-;; @004b v122 = load.i64 notrap aligned region12 v118+24
-;; @004b store notrap aligned region9 v122, v34+80
-;; @004b v123 = iconst.i64 0
-;; @004b v124 = iadd.i64 v67, v123 ; v123 = 0
-;; @004b v125 = iconst.i32 4
-;; @004b store notrap aligned region6 v125, v124+32 ; v125 = 4
-;; @004b v126 = iconst.i64 120
-;; @004b v127 = iadd.i64 v67, v126 ; v126 = 120
-;; @004b v128 = load.i64 notrap aligned region15 v127+8
-;; @004b v129 = iconst.i32 0
-;; @004b store notrap aligned region17 v129, v127 ; v129 = 0
+;; @004b v118 = iconst.i64 0
+;; @004b v119 = iadd.i64 v27, v118 ; v118 = 0
+;; @004b v120 = load.i64 notrap aligned region13 v119
+;; @004b store notrap aligned region1 v120, v35+24
+;; @004b v121 = load.i64 notrap aligned region10 v119+8
+;; @004b store notrap aligned region7 v121, v35+72
+;; @004b v122 = load.i64 notrap aligned region11 v119+16
+;; @004b store notrap aligned region8 v122, v35+64
+;; @004b v123 = load.i64 notrap aligned region12 v119+24
+;; @004b store notrap aligned region9 v123, v35+80
+;; @004b v124 = iconst.i64 0
+;; @004b v125 = iadd.i64 v68, v124 ; v124 = 0
+;; @004b v126 = iconst.i32 4
+;; @004b store notrap aligned region6 v126, v125+32 ; v126 = 4
+;; @004b v127 = iconst.i64 120
+;; @004b v128 = iadd.i64 v68, v127 ; v127 = 120
+;; @004b v129 = load.i64 notrap aligned region15 v128+8
;; @004b v130 = iconst.i32 0
-;; @004b store notrap aligned region14 v130, v127+4 ; v130 = 0
-;; @004b v131 = iconst.i64 0
-;; @004b store notrap aligned region15 v131, v127+8 ; v131 = 0
+;; @004b store notrap aligned region17 v130, v128 ; v130 = 0
+;; @004b v131 = iconst.i32 0
+;; @004b store notrap aligned region14 v131, v128+4 ; v131 = 0
+;; @004b v132 = iconst.i64 0
+;; @004b store notrap aligned region15 v132, v128+8 ; v132 = 0
;; @0050 jump block1
;;
;; block1:
diff --git a/tests/misc_testsuite/stack-switching-and-gc.wast b/tests/misc_testsuite/stack-switching-and-gc.wast
deleted file mode 100644
index 229f2fe16d..0000000000
--- a/tests/misc_testsuite/stack-switching-and-gc.wast
+++ /dev/null
@@ -1,13 +0,0 @@
-;;! stack_switching = true
-;;! gc = true
-
-(assert_invalid
- (module
- (type $f (func))
- (type $c (cont $f))
- (type $s (struct (field (ref null $c))))
- (func (export "run")
- (drop (struct.new_default $s))
- )
- )
- "Stack switching feature not compatible with GC")
diff --git a/tests/misc_testsuite/stack-switching/gc.wast b/tests/misc_testsuite/stack-switching/gc.wast
new file mode 100644
index 0000000000..5cbe6af868
--- /dev/null
+++ b/tests/misc_testsuite/stack-switching/gc.wast
@@ -0,0 +1,363 @@
+;;! stack_switching = true
+;;! gc = true
+;;! function_references = true
+;;! bulk_memory = true
+
+;; A GC reference in a parent frame must remain live while a child
+;; continuation is running.
+(module
+ (type $box (struct (field i32)))
+ (type $f (func))
+ (type $c (cont $f))
+ (tag $suspend)
+
+ (import "wasmtime" "gc" (func $gc))
+
+ (func $child
+ (call $gc)
+ (suspend $suspend))
+ (elem declare func $child)
+
+ (func (export "parent-frame") (result i32)
+ (local $box (ref null $box))
+ (local.set $box (struct.new $box (i32.const 42)))
+
+ (drop
+ (block $on-suspend (result (ref $c))
+ (resume $c
+ (on $suspend $on-suspend)
+ (cont.new $c (ref.func $child)))
+ (unreachable)))
+
+ (struct.get $box 0 (local.get $box)))
+)
+
+(assert_return (invoke "parent-frame") (i32.const 42))
+
+;; Stack-map tracing follows the full parent chain, and execution can
+;; subsequently re-enter every parent frame.
+(module
+ (type $box (struct (field i32)))
+ (type $f (func (result i32)))
+ (type $c (cont $f))
+ (tag $suspend)
+
+ (import "wasmtime" "gc" (func $gc))
+
+ (func $child (result i32)
+ (call $gc)
+ (suspend $suspend)
+ (i32.const 1))
+ (elem declare func $child)
+
+ (func $middle (result i32)
+ (local $box (ref null $box))
+ (local.set $box (struct.new $box (i32.const 20)))
+ (i32.add
+ (resume $c (cont.new $c (ref.func $child)))
+ (struct.get $box 0 (local.get $box))))
+ (elem declare func $middle)
+
+ (func (export "parent-chain") (result i32)
+ (local $box (ref null $box))
+ (local $suspended (ref null $c))
+ (local.set $box (struct.new $box (i32.const 100)))
+
+ (local.set $suspended
+ (block $on-suspend (result (ref $c))
+ (resume $c
+ (on $suspend $on-suspend)
+ (cont.new $c (ref.func $middle)))
+ (unreachable)))
+
+ (i32.add
+ (resume $c (local.get $suspended))
+ (struct.get $box 0 (local.get $box))))
+)
+
+(assert_return (invoke "parent-chain") (i32.const 121))
+
+;; A bound GC reference is retained only by a fresh continuation's
+;; argument buffer when collection runs before the continuation is
+;; first resumed.
+(module
+ (type $box (struct (field i32)))
+ (type $with-arg-f (func (param (ref $box)) (result i32)))
+ (type $without-arg-f (func (result i32)))
+ (type $with-arg-c (cont $with-arg-f))
+ (type $without-arg-c (cont $without-arg-f))
+
+ (import "wasmtime" "gc" (func $gc))
+
+ (func $read-box (param $box (ref $box)) (result i32)
+ (struct.get $box 0 (local.get $box)))
+ (elem declare func $read-box)
+
+ (func (export "bound-argument") (result i32)
+ (local $continuation (ref null $without-arg-c))
+
+ (local.set $continuation
+ (cont.bind $with-arg-c $without-arg-c
+ (struct.new $box (i32.const 73))
+ (cont.new $with-arg-c (ref.func $read-box))))
+
+ (call $gc)
+ (resume $without-arg-c
+ (local.get $continuation)))
+)
+
+(assert_return (invoke "bound-argument") (i32.const 73))
+
+;; The same root metadata is used for the `values` buffer after a
+;; continuation has suspended.
+(module
+ (type $box (struct (field i32)))
+ (type $initial-f (func (result i32)))
+ (type $suspended-f (func (param (ref $box)) (result i32)))
+ (type $initial-c (cont $initial-f))
+ (type $suspended-c (cont $suspended-f))
+ (tag $yield (result (ref $box)))
+
+ (import "wasmtime" "gc" (func $gc))
+
+ (func $suspend-then-read (result i32)
+ (struct.get $box 0 (suspend $yield)))
+ (elem declare func $suspend-then-read)
+
+ (func (export "bound-suspended-argument") (result i32)
+ (local $suspended (ref null $suspended-c))
+ (local $ready (ref null $initial-c))
+
+ (local.set $suspended
+ (block $on-yield (result (ref $suspended-c))
+ (resume $initial-c
+ (on $yield $on-yield)
+ (cont.new $initial-c (ref.func $suspend-then-read)))
+ (unreachable)))
+
+ (local.set $ready
+ (cont.bind $suspended-c $initial-c
+ (struct.new $box (i32.const 91))
+ (local.get $suspended)))
+
+ (call $gc)
+ (resume $initial-c (ref.as_non_null (local.get $ready))))
+)
+
+(assert_return (invoke "bound-suspended-argument") (i32.const 91))
+
+;; Struct and array fields store continuation references through a
+;; store-local side table. Collection may run while the aggregate is
+;; the only place that retains the continuation object.
+(module
+ (type $f (func (result i32)))
+ (type $c (cont $f))
+ (type $holder (struct (field (mut (ref null $c)))))
+ (type $array (array (mut (ref null $c))))
+
+ (import "wasmtime" "gc" (func $gc))
+
+ (func $forty-two (result i32)
+ (i32.const 42))
+ (elem declare func $forty-two)
+
+ (func (export "struct-initialize") (result i32)
+ (local $holder (ref $holder))
+ (local.set $holder
+ (struct.new $holder
+ (cont.new $c (ref.func $forty-two))))
+ (call $gc)
+ (resume $c
+ (struct.get $holder 0 (local.get $holder))))
+
+ (func (export "struct-update") (result i32)
+ (local $holder (ref $holder))
+ (local.set $holder (struct.new_default $holder))
+ (struct.set $holder 0
+ (local.get $holder)
+ (cont.new $c (ref.func $forty-two)))
+ (call $gc)
+ (resume $c
+ (struct.get $holder 0 (local.get $holder))))
+
+ (func (export "array-initialize") (result i32)
+ (local $array (ref $array))
+ (local.set $array
+ (array.new_fixed $array 1
+ (cont.new $c (ref.func $forty-two))))
+ (call $gc)
+ (resume $c
+ (array.get $array (local.get $array) (i32.const 0))))
+
+ (func (export "array-update") (result i32)
+ (local $array (ref $array))
+ (local.set $array (array.new_default $array (i32.const 1)))
+ (array.set $array
+ (local.get $array)
+ (i32.const 0)
+ (cont.new $c (ref.func $forty-two)))
+ (call $gc)
+ (resume $c
+ (array.get $array (local.get $array) (i32.const 0))))
+
+ (func (export "array-fill-and-copy") (result i32)
+ (local $source (ref $array))
+ (local $destination (ref $array))
+ (local.set $source (array.new_default $array (i32.const 1)))
+ (local.set $destination (array.new_default $array (i32.const 1)))
+ (array.fill $array
+ (local.get $source)
+ (i32.const 0)
+ (cont.new $c (ref.func $forty-two))
+ (i32.const 1))
+ (array.copy $array $array
+ (local.get $destination)
+ (i32.const 0)
+ (local.get $source)
+ (i32.const 0)
+ (i32.const 1))
+ (call $gc)
+ (resume $c
+ (array.get $array (local.get $destination) (i32.const 0))))
+
+ (func (export "null-defaults") (result i32)
+ (local $holder (ref $holder))
+ (local $array (ref $array))
+ (local.set $holder (struct.new_default $holder))
+ (local.set $array (array.new_default $array (i32.const 1)))
+ (i32.add
+ (ref.is_null (struct.get $holder 0 (local.get $holder)))
+ (ref.is_null
+ (array.get $array (local.get $array) (i32.const 0)))))
+
+ ;; Loading an alias from a GC object must reproduce the interned
+ ;; revision witness. Consuming one alias therefore invalidates all
+ ;; other loads of the same field.
+ (func (export "consume-struct-field-twice")
+ (local $holder (ref $holder))
+ (local.set $holder
+ (struct.new $holder
+ (cont.new $c (ref.func $forty-two))))
+ (drop
+ (resume $c
+ (struct.get $holder 0 (local.get $holder))))
+ (drop
+ (resume $c
+ (struct.get $holder 0 (local.get $holder)))))
+)
+
+(assert_return (invoke "struct-initialize") (i32.const 42))
+(assert_return (invoke "struct-update") (i32.const 42))
+(assert_return (invoke "array-initialize") (i32.const 42))
+(assert_return (invoke "array-update") (i32.const 42))
+(assert_return (invoke "array-fill-and-copy") (i32.const 42))
+(assert_return (invoke "null-defaults") (i32.const 2))
+(assert_trap
+ (invoke "consume-struct-field-twice")
+ "continuation already consumed")
+
+;; A GC reference passed to a child continuation must remain live when
+;; that child is unwound by `resume_throw`. The parent retains another
+;; reference, catches the injected exception, and can still use the
+;; object after a second collection.
+(module
+ (type $box (struct (field i32)))
+ (type $child-f (func (param (ref $box)) (result i32)))
+ (type $parent-f (func (result i32)))
+ (type $child-c (cont $child-f))
+ (type $parent-c (cont $parent-f))
+ (tag $suspend)
+ (tag $exception)
+
+ (import "wasmtime" "gc" (func $gc))
+
+ (func $child (param $box (ref $box)) (result i32)
+ ;; At this collection, both the child and its parent have the box
+ ;; in live stack slots. Keep the parameter live across the
+ ;; suspension point.
+ (call $gc)
+ (suspend $suspend)
+ (struct.get $box 0 (local.get $box)))
+ (elem declare func $child)
+
+ (func $parent (result i32)
+ (local $box (ref $box))
+ (local.set $box (struct.new $box (i32.const 101)))
+
+ (block $caught
+ (try_table (catch $exception $caught)
+ (drop
+ (resume $child-c
+ (local.get $box)
+ (cont.new $child-c (ref.func $child)))))
+ (unreachable))
+
+ ;; The child continuation is terminal, so only the parent's stack
+ ;; slot retains the box at this collection.
+ (call $gc)
+ (struct.get $box 0 (local.get $box)))
+ (elem declare func $parent)
+
+ (func (export "gc-ref-survives-resume-throw") (result i32)
+ (local $suspended (ref null $parent-c))
+
+ (local.set $suspended
+ (block $on-suspend (result (ref $parent-c))
+ (resume $parent-c
+ (on $suspend $on-suspend)
+ (cont.new $parent-c (ref.func $parent)))
+ (unreachable)))
+
+ (resume_throw $parent-c $exception
+ (local.get $suspended)))
+)
+
+(assert_return
+ (invoke "gc-ref-survives-resume-throw")
+ (i32.const 101))
+
+;; A GC reference carried by an exception injected with `resume_throw`
+;; must become a live stack root when the exception is caught inside
+;; the resumed continuation.
+(module
+ (type $box (struct (field i32)))
+ (type $f (func (result i32)))
+ (type $c (cont $f))
+ (tag $suspend)
+ (tag $exception (param (ref $box)))
+
+ (import "wasmtime" "gc" (func $gc))
+
+ (func $catch-payload (result i32)
+ (local $box (ref null $box))
+
+ (local.set $box
+ (block $caught (result (ref $box))
+ (try_table (result (ref $box)) (catch $exception $caught)
+ (suspend $suspend)
+ (unreachable))))
+
+ ;; The caught local is now the only reference to the exception
+ ;; payload.
+ (call $gc)
+ (struct.get $box 0 (ref.as_non_null (local.get $box))))
+ (elem declare func $catch-payload)
+
+ (func (export "gc-ref-in-resume-throw-payload") (result i32)
+ (local $suspended (ref null $c))
+
+ (local.set $suspended
+ (block $on-suspend (result (ref $c))
+ (resume $c
+ (on $suspend $on-suspend)
+ (cont.new $c (ref.func $catch-payload)))
+ (unreachable)))
+
+ (resume_throw $c $exception
+ (struct.new $box (i32.const 202))
+ (local.get $suspended)))
+)
+
+(assert_return
+ (invoke "gc-ref-in-resume-throw-payload")
+ (i32.const 202))
diff --git a/tests/misc_testsuite/stack-switching/issue13021.wast b/tests/misc_testsuite/stack-switching/issue13021.wast
new file mode 100644
index 0000000000..949362e019
--- /dev/null
+++ b/tests/misc_testsuite/stack-switching/issue13021.wast
@@ -0,0 +1,18 @@
+;;! gc = true
+;;! stack_switching = true
+
+;; Regression test for issue https://github.com/bytecodealliance/wasmtime/issues/13021
+(module
+ (type $ft (func))
+ (type $ct (cont $ft))
+ (type $arr (array (mut (ref null $ct))))
+ (func (export "boom")
+ (local $a (ref $arr))
+ (local.set $a (array.new_default $arr (i32.const 2)))
+ (array.copy $arr $arr
+ (local.get $a) (i32.const 0)
+ (local.get $a) (i32.const 0)
+ (i32.const 1))
+ )
+)
+(assert_return (invoke "boom"))
\ No newline at end of file
diff --git a/tests/misc_testsuite/stack-switching/issue13022.wast b/tests/misc_testsuite/stack-switching/issue13022.wast
new file mode 100644
index 0000000000..1ccf5a7c69
--- /dev/null
+++ b/tests/misc_testsuite/stack-switching/issue13022.wast
@@ -0,0 +1,18 @@
+;;! gc = true
+;;! stack_switching = true
+
+;; Regression test for https://github.com/bytecodealliance/wasmtime/issues/13022
+(module
+ (type $ft (func))
+ (type $ct (cont $ft))
+ (type $arr (array (mut (ref null $ct))))
+ (func (export "boom")
+ (local $a (ref $arr))
+ (local.set $a (array.new_default $arr (i32.const 2)))
+ (array.copy $arr $arr
+ (local.get $a) (i32.const 0)
+ (local.get $a) (i32.const 0)
+ (i32.const 1))
+ )
+)
+(assert_return (invoke "boom"))
\ No newline at end of file