Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

98.8 KB
Raw Blame
use crate::func_environ::gc;
use crate::translate::set_block_params;
use crate::trap::TranslateTrap;
use cranelift_codegen::ir::BlockArg;
use cranelift_codegen::ir::condcodes::*;
use cranelift_codegen::ir::types::*;
use cranelift_codegen::ir::{self, MemFlagsData};
use cranelift_codegen::ir::{Block, BlockCall, InstBuilder, JumpTableData};
use cranelift_frontend::FunctionBuilder;
use itertools::{Either, Itertools};
use wasmtime_environ::{
    BuiltinFunctionIndex, PtrSize, TagIndex, TypeIndex, WasmHeapType, WasmRefType, WasmResult,
    WasmValType, wasm_unsupported,
};

fn control_context_size(triple: &target_lexicon::Triple) -> WasmResult<u8> {
    match (triple.architecture, triple.operating_system) {
        (target_lexicon::Architecture::X86_64, target_lexicon::OperatingSystem::Linux) => Ok(24),
        _ => Err(wasm_unsupported!(
            "stack switching not supported on {triple}"
        )),
    }
}

/// Emit a stack switch instruction. On ASan-enabled builds this
/// function also emits the fiber switch hooks.
fn emit_stack_switch<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    store_context_ptr: ir::Value,
    load_context_ptr: ir::Value,
    payload: ir::Value,
    asan_stack_information: impl FnOnce(
        &mut crate::func_environ::FuncEnvironment<'a>,
        &mut FunctionBuilder,
    ) -> (ir::Value, ir::Value),
) -> ir::Value {
    if !env.compiler.tunables().asan_stack_switching {
        return builder
            .ins()
            .stack_switch(store_context_ptr, load_context_ptr, payload);
    }

    // ASan-aware stack switching.
    // The `asan_target_csi` is provided as a function to lazily load
    // the necessary ASan bookkeeping.
    let (source_csi, target_csi) = asan_stack_information(env, builder);
    let pointer_type = env.pointer_type();
    let slot = env.get_or_create_asan_fake_stack_slot(builder);
    let fake_stack_save = builder.ins().stack_addr(pointer_type, slot, 0);
    let region = env.alias_regions.stack_slot_region(builder.func, slot);
    let flags = MemFlagsData::trusted().with_alias_region(Some(region));
    let null = builder.ins().iconst(pointer_type, 0);
    builder.ins().store(flags, null, fake_stack_save, 0);
    let vmctx = env.vmctx_val(&mut builder.cursor());

    let asan_start_switch_fiber = env.builtin_functions.load_builtin(
        builder.func,
        BuiltinFunctionIndex::asan_start_switch_fiber(),
    );
    env.call_builtin(
        builder,
        asan_start_switch_fiber,
        &[vmctx, fake_stack_save, source_csi, target_csi],
    );

    let result = builder
        .ins()
        .stack_switch(store_context_ptr, load_context_ptr, payload);

    let fake_stack = builder.ins().load(pointer_type, flags, fake_stack_save, 0);
    let asan_finish_switch_fiber = env.builtin_functions.load_builtin(
        builder.func,
        BuiltinFunctionIndex::asan_finish_switch_fiber(),
    );
    env.call_builtin(
        builder,
        asan_finish_switch_fiber,
        &[vmctx, fake_stack, source_csi],
    );

    result
}

use super::control_effect::ControlEffect;
use super::fatpointer;

/// This module contains compile-time counterparts to types defined elsewhere.
pub(crate) mod stack_switching_helpers {
    use core::marker::PhantomData;
    use cranelift_codegen::ir;
    use cranelift_codegen::ir::InstBuilder;
    use cranelift_codegen::ir::condcodes::IntCC;
    use cranelift_codegen::ir::types::*;
    use cranelift_codegen::ir::{StackSlot, StackSlotKind::*};
    use cranelift_frontend::FunctionBuilder;
    use wasmtime_environ::{PtrSize, WasmValType};

    /// Provides information about the layout of a type when it is used as an
    /// element in a host array. This is used for `VMHostArrayRef`.
    pub(crate) trait VMHostArrayEntry {
        /// Returns `(align_shift, size)`: the log2 of the alignment, as used by
        /// `StackSlotData`, and the size in bytes.
        fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(p: &P) -> (u8, u32);
    }

    impl VMHostArrayEntry for u128 {
        fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(_p: &P) -> (u8, u32) {
            (4, 16)
        }
    }

    impl<T> VMHostArrayEntry for *mut T {
        fn vmhostarray_entry_layout<P: wasmtime_environ::PtrSize>(p: &P) -> (u8, u32) {
            (
                u8::try_from(p.size().trailing_zeros()).unwrap(),
                p.size().into(),
            )
        }
    }

    #[derive(Copy, Clone)]
    pub struct VMContRef {
        pub address: ir::Value,
    }

    #[derive(Copy, Clone)]
    pub struct VMHostArrayRef<T> {
        /// Address of the VMHostArray we are referencing
        address: ir::Value,

        /// The type parameter T is never used in the fields above. We still
        /// want to have it for consistency with
        /// `wasmtime_environ::Vector` and to use it in the associated
        /// functions.
        phantom: PhantomData<T>,
    }

    /// Compile-time reference to a runtime `VMPayloads` descriptor.
    ///
    /// This wrapper keeps the payload buffer and its GC-reference markers
    /// paired. Its core invariant is that `gc_ref_data`, when present, points
    /// to one marker byte for every slot in `buffer`; callers must update a
    /// slot's value and marker together.
    #[derive(Copy, Clone)]
    pub struct VMPayloads {
        address: ir::Value,
        buffer: VMHostArrayRef<u128>,
    }

    // Actually a vector of *mut VMTagDefinition
    pub type VMHandlerList = VMHostArrayRef<*mut u8>;

    /// Compile-time representation of wasmtime_environ::VMStackChain,
    /// consisting of two `ir::Value`s.
    pub struct VMStackChain {
        discriminant: ir::Value,
        payload: ir::Value,
    }

    pub struct VMCommonStackInformation {
        pub address: ir::Value,
    }

    /// Compile-time representation of `crate::runtime::vm::stack::VMContinuationStack`.
    pub struct VMContinuationStack {
        /// This is NOT the "top of stack" address of the stack itself. In line
        /// with how the (runtime) `FiberStack` type works, this is a pointer to
        /// the TOS address.
        tos_ptr: ir::Value,
    }

    impl VMContRef {
        pub fn new(address: ir::Value) -> VMContRef {
            VMContRef { address }
        }

        pub fn args<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> VMPayloads {
            let offset: i64 = env.offsets.ptr.vm_cont_ref().args().into();
            let address = builder.ins().iadd_imm_s(self.address, offset);
            VMPayloads::new(env, address)
        }

        pub fn values<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> VMPayloads {
            let offset: i64 = env.offsets.ptr.vm_cont_ref().values().into();
            let address = builder.ins().iadd_imm_s(self.address, offset);
            VMPayloads::new(env, address)
        }

        pub fn common_stack_information<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> VMCommonStackInformation {
            let offset: i64 = env
                .offsets
                .ptr
                .vm_cont_ref()
                .common_stack_information()
                .into();
            let address = builder.ins().iadd_imm_s(self.address, offset);
            VMCommonStackInformation { address }
        }

        /// Stores the parent of this continuation, which may either be another
        /// continuation or the initial stack. It is therefore represented as a
        /// `VMStackChain` element.
        pub fn set_parent_stack_chain<'a>(
            &mut self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            new_stack_chain: &VMStackChain,
        ) {
            let offset = env.offsets.ptr.vm_cont_ref().parent_chain().into();
            let region = env
                .alias_regions
                .vm_cont_ref_parent_chain_region(builder.func);
            new_stack_chain.store(env, builder, self.address, offset, Some(region))
        }

        /// Loads the parent of this continuation, which may either be another
        /// continuation or the initial stack. It is therefore represented as a
        /// `VMStackChain` element.
        pub fn get_parent_stack_chain<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> VMStackChain {
            let offset = env.offsets.ptr.vm_cont_ref().parent_chain().into();
            let region = env
                .alias_regions
                .vm_cont_ref_parent_chain_region(builder.func);
            VMStackChain::load(
                env,
                builder,
                self.address,
                offset,
                env.pointer_type(),
                Some(region),
            )
        }

        pub fn set_last_ancestor<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            last_ancestor: ir::Value,
        ) {
            env.alias_regions.vm_cont_ref().last_ancestor().store(
                &mut builder.cursor(),
                self.address,
                last_ancestor,
            );
        }

        pub fn get_last_ancestor<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            env.alias_regions
                .vm_cont_ref()
                .last_ancestor()
                .load(&mut builder.cursor(), self.address)
        }

        /// Gets the revision counter the a given continuation
        /// reference.
        pub fn get_revision<'a>(
            &mut self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            env.alias_regions
                .vm_cont_ref()
                .revision()
                .load(&mut builder.cursor(), self.address)
        }

        /// Sets the revision counter on the given continuation
        /// reference to `revision + 1`.

        pub fn incr_revision<'a>(
            &mut self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            revision: ir::Value,
        ) -> ir::Value {
            let revision_plus1 = builder.ins().iadd_imm_s(revision, 1);
            env.alias_regions.vm_cont_ref().revision().store(
                &mut builder.cursor(),
                self.address,
                revision_plus1,
            );
            revision_plus1
        }

        pub fn get_fiber_stack<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> VMContinuationStack {
            // The top of stack field is stored at offset 0 of the `FiberStack`.
            let offset: i64 = env.offsets.ptr.vm_cont_ref().stack().into();
            let fiber_stack_top_of_stack_ptr = builder.ins().iadd_imm_s(self.address, offset);
            VMContinuationStack::new(fiber_stack_top_of_stack_ptr)
        }
    }

    impl VMPayloads {
        pub fn new(env: &mut crate::func_environ::FuncEnvironment<'_>, address: ir::Value) -> Self {
            debug_assert_eq!(env.offsets.ptr.vm_payloads().buffer(), 0);
            Self {
                address,
                buffer: VMHostArrayRef::new(address),
            }
        }

        pub fn get_gc_ref_data(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'_>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            env.alias_regions
                .vm_payloads()
                .gc_ref_data()
                .load(&mut builder.cursor(), self.address)
        }

        pub fn set_gc_ref_data(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'_>,
            builder: &mut FunctionBuilder,
            data: ir::Value,
        ) {
            env.alias_regions.vm_payloads().gc_ref_data().store(
                &mut builder.cursor(),
                self.address,
                data,
            );
        }

        /// Prepares stack storage for this payload buffer.
        ///
        /// Payload values and their GC-reference markers use distinct stack
        /// slots so that they have distinct alias regions. The descriptor's
        /// capacity and pointers always describe the storage selected at this
        /// particular instruction site. Both slots retain their identities and
        /// grow in place as later sites require more capacity.
        pub fn prepare_stack_storage(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'_>,
            builder: &mut FunctionBuilder,
            capacity: u32,
            initial_types: &[WasmValType],
            needs_gc_ref_markers: bool,
            existing_slots: Option<crate::func_environ::VMPayloadStackSlots>,
        ) -> crate::func_environ::VMPayloadStackSlots {
            debug_assert!(capacity > 0);
            debug_assert!(initial_types.len() <= usize::try_from(capacity).unwrap());

            let (align, entry_size) =
                <u128 as VMHostArrayEntry>::vmhostarray_entry_layout(&env.offsets.ptr);
            let values_size = capacity
                .checked_mul(entry_size)
                .expect("stack switching values buffer size should fit in u32");

            let values_slot = match existing_slots.map(|slots| slots.values) {
                Some(slot) => {
                    // Keep the slot identity stable: `stack_addr` instructions
                    // emitted at earlier sites continue to refer to this slot
                    // when a later site requires more storage.
                    let slot_data = &mut builder.func.sized_stack_slots[slot];
                    debug_assert!(align <= slot_data.align_shift);
                    debug_assert_eq!(slot_data.kind, ExplicitSlot);
                    slot_data.size = slot_data.size.max(values_size);
                    slot
                }
                None => builder.create_sized_stack_slot(ir::StackSlotData::new(
                    ir::StackSlotKind::ExplicitSlot,
                    values_size,
                    align,
                )),
            };

            let capacity_value = builder.ins().iconst(I32, i64::from(capacity));
            let values_data = builder.ins().stack_addr(env.pointer_type(), values_slot, 0);
            self.set_capacity(env, builder, capacity_value);
            self.set_data(env, builder, values_data);

            let mut gc_ref_markers = existing_slots.and_then(|slots| slots.gc_ref_markers);
            if needs_gc_ref_markers {
                let marker_slot = match gc_ref_markers {
                    Some(slot) => {
                        let slot_data = &mut builder.func.sized_stack_slots[slot];
                        debug_assert_eq!(slot_data.kind, ExplicitSlot);
                        slot_data.size = slot_data.size.max(capacity);
                        slot
                    }
                    None => builder.create_sized_stack_slot(ir::StackSlotData::new(
                        ir::StackSlotKind::ExplicitSlot,
                        capacity,
                        0,
                    )),
                };
                gc_ref_markers = Some(marker_slot);

                let marker_data = builder.ins().stack_addr(env.pointer_type(), marker_slot, 0);
                self.set_gc_ref_data(env, builder, marker_data);

                let region = env
                    .alias_regions
                    .stack_slot_region(builder.func, marker_slot);
                let flags = ir::MemFlagsData::trusted().with_alias_region(Some(region));
                let zero = builder.ins().iconst(I8, 0);
                for offset in 0..capacity {
                    builder
                        .ins()
                        .store(flags, zero, marker_data, i32::try_from(offset).unwrap());
                }
                for (offset, ty) in initial_types.iter().enumerate() {
                    if ty.is_vmgcref_type_and_not_i31() {
                        let marker = builder
                            .ins()
                            .iconst(I8, i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF));
                        builder.ins().store(
                            flags,
                            marker,
                            marker_data,
                            i32::try_from(offset).unwrap(),
                        );
                    }
                }
            }

            crate::func_environ::VMPayloadStackSlots {
                values: values_slot,
                gc_ref_markers,
            }
        }
    }

    impl core::ops::Deref for VMPayloads {
        type Target = VMHostArrayRef<u128>;

        fn deref(&self) -> &Self::Target {
            &self.buffer
        }
    }

    impl<T: VMHostArrayEntry> VMHostArrayRef<T> {
        pub(crate) fn new(address: ir::Value) -> Self {
            Self {
                address,
                phantom: PhantomData::default(),
            }
        }

        pub fn get_data<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            env.alias_regions
                .vm_host_array()
                .data()
                .load(&mut builder.cursor(), self.address)
        }

        pub fn get_length<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            env.alias_regions
                .vm_host_array()
                .length()
                .load(&mut builder.cursor(), self.address)
        }

        pub fn get_capacity<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            env.alias_regions
                .vm_host_array()
                .capacity()
                .load(&mut builder.cursor(), self.address)
        }

        fn set_length<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            length: ir::Value,
        ) {
            env.alias_regions.vm_host_array().length().store(
                &mut builder.cursor(),
                self.address,
                length,
            );
        }

        fn set_capacity<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            capacity: ir::Value,
        ) {
            env.alias_regions.vm_host_array().capacity().store(
                &mut builder.cursor(),
                self.address,
                capacity,
            );
        }

        fn set_data<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            data: ir::Value,
        ) {
            debug_assert_eq!(builder.func.dfg.value_type(data), env.pointer_type());
            env.alias_regions.vm_host_array().data().store(
                &mut builder.cursor(),
                self.address,
                data,
            );
        }

        /// Returns a pointer to the next empty slot in the data
        /// buffer and marks the subsequent `arg_count` slots as
        /// occupied.
        ///
        /// Traps with an internal assertion if the requested slots
        /// exceed the buffer's capacity. A well-typed Wasm program
        /// cannot violate this invariant. Therefore a violation
        /// indicates that the continuation was corrupted.
        pub fn occupy_next_slots<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            arg_count: u32,
        ) -> (ir::Value, ir::Value) {
            let data = self.get_data(env, builder);
            let original_length = self.get_length(env, builder);
            let capacity = self.get_capacity(env, builder);

            // Widen before adding so that a wrapped `u32` length cannot pass
            // the capacity check.
            let original_length = builder.ins().uextend(I64, original_length);
            let capacity = builder.ins().uextend(I64, capacity);
            let new_length = builder
                .ins()
                .iadd_imm_s(original_length, i64::from(arg_count));
            let in_bounds =
                builder
                    .ins()
                    .icmp(IntCC::UnsignedLessThanOrEqual, new_length, capacity);
            builder.ins().trapz(in_bounds, crate::TRAP_INTERNAL_ASSERT);

            let new_length_32 = builder.ins().ireduce(I32, new_length);
            self.set_length(env, builder, new_length_32);

            let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);
            let byte_offset = builder
                .ins()
                .imul_imm_s(original_length, i64::from(entry_size));
            (builder.ins().iadd(data, byte_offset), original_length)
        }

        pub fn allocate_or_reuse_stack_slot<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            required_capacity: u32,
            existing_slot: Option<StackSlot>,
        ) -> StackSlot {
            let (align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);
            let required_size = required_capacity * entry_size;

            match existing_slot {
                Some(slot) if builder.func.sized_stack_slots[slot].size >= required_size => {
                    let slot_data = &builder.func.sized_stack_slots[slot];
                    debug_assert!(align <= slot_data.align_shift);
                    debug_assert_eq!(slot_data.kind, ExplicitSlot);
                    let existing_capacity = slot_data.size / entry_size;

                    let capacity_value = builder.ins().iconst(I32, i64::from(existing_capacity));
                    let existing_data = builder.ins().stack_addr(env.pointer_type(), slot, 0);

                    self.set_capacity(env, builder, capacity_value);
                    self.set_data(env, builder, existing_data);

                    slot
                }
                _ => {
                    let capacity_value = builder.ins().iconst(I32, i64::from(required_capacity));
                    let slot_size = ir::StackSlotData::new(
                        ir::StackSlotKind::ExplicitSlot,
                        required_size,
                        align,
                    );
                    let slot = builder.create_sized_stack_slot(slot_size);
                    let new_data = builder.ins().stack_addr(env.pointer_type(), slot, 0);

                    self.set_capacity(env, builder, capacity_value);
                    self.set_data(env, builder, new_data);

                    slot
                }
            }
        }

        /// Loads n entries from this Vector object, where n is the length of
        /// `load_types`, which also gives the Wasm types of the values to load.
        /// Loading starts at index 0 of the Vector object.
        pub fn load_data_entries<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            load_types: &[WasmValType],
        ) -> Vec<ir::Value> {
            let region = env
                .alias_regions
                .continuation_stack_memory_region(builder.func);
            let memflags = ir::MemFlagsData::trusted().with_alias_region(Some(region));

            let data_start_pointer = self.get_data(env, builder);
            let mut values = vec![];
            let mut offset = 0;
            let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);
            for wasm_ty in load_types {
                let valtype = crate::value_type(env.isa(), *wasm_ty);
                let val = builder
                    .ins()
                    .load(valtype, memflags, data_start_pointer, offset);
                if env.val_ty_needs_stack_map(*wasm_ty) {
                    builder.declare_value_needs_stack_map(val);
                }
                values.push(val);
                offset += i32::try_from(entry_size).unwrap();
            }
            values
        }

        /// Stores the given `values` in this Vector object, beginning at
        /// index 0. This expects the Vector object to be empty (i.e., current
        /// length is 0), and to be of sufficient capacity to store |`values`|
        /// entries.
        pub fn store_data_entries<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            values: &[ir::Value],
        ) {
            let store_count = builder
                .ins()
                .iconst(I32, i64::try_from(values.len()).unwrap());

            let (_align, entry_size) = T::vmhostarray_entry_layout(&env.offsets.ptr);

            debug_assert!(values.iter().all(|val| {
                let ty = builder.func.dfg.value_type(*val);
                let size = ty.bytes();
                size <= entry_size
            }));

            let region = env
                .alias_regions
                .continuation_stack_memory_region(builder.func);
            let memflags = ir::MemFlagsData::trusted().with_alias_region(Some(region));

            let data_start_pointer = self.get_data(env, builder);

            let mut offset = 0;
            for value in values {
                builder
                    .ins()
                    .store(memflags, *value, data_start_pointer, offset);
                offset += i32::try_from(entry_size).unwrap();
            }

            self.set_length(env, builder, store_count);
        }

        pub fn clear<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            discard_buffer: bool,
        ) {
            let zero32 = builder.ins().iconst(I32, 0);
            self.set_length(env, builder, zero32);

            if discard_buffer {
                let zero32 = builder.ins().iconst(I32, 0);
                self.set_capacity(env, builder, zero32);

                let zero_ptr = builder.ins().iconst(env.pointer_type(), 0);
                self.set_data(env, builder, zero_ptr);
            }
        }
    }

    impl VMStackChain {
        /// Creates a `Self` corresponding to `VMStackChain::Continuation(contref)`.
        pub fn from_continuation<'a>(
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            contref: ir::Value,
        ) -> VMStackChain {
            debug_assert_eq!(
                env.offsets.ptr.size_of_vmstack_chain(),
                2 * env.offsets.ptr.size()
            );
            let discriminant = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT;
            let discriminant = builder
                .ins()
                .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap());
            VMStackChain {
                discriminant,
                payload: contref,
            }
        }

        /// Creates a `Self` corresponding to `VMStackChain::Absent`.
        pub fn absent<'a>(
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> VMStackChain {
            debug_assert_eq!(
                env.offsets.ptr.size_of_vmstack_chain(),
                2 * env.offsets.ptr.size()
            );
            let discriminant = wasmtime_environ::STACK_CHAIN_ABSENT_DISCRIMINANT;
            let discriminant = builder
                .ins()
                .iconst(env.pointer_type(), i64::try_from(discriminant).unwrap());
            let zero_filler = builder.ins().iconst(env.pointer_type(), 0i64);
            VMStackChain {
                discriminant,
                payload: zero_filler,
            }
        }

        pub fn is_initial_stack<'a>(
            &self,
            _env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            builder.ins().icmp_imm_s(
                IntCC::Equal,
                self.discriminant,
                i64::try_from(wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT).unwrap(),
            )
        }

        /// Return the two raw `ir::Value`s that represent this VMStackChain.
        pub fn to_raw_parts(&self) -> [ir::Value; 2] {
            [self.discriminant, self.payload]
        }

        /// Construct a `Self` from two raw `ir::Value`s.
        pub fn from_raw_parts(raw_data: [ir::Value; 2]) -> VMStackChain {
            VMStackChain {
                discriminant: raw_data[0],
                payload: raw_data[1],
            }
        }

        /// Load a `VMStackChain` object from the given address.
        ///
        /// The `region` is the alias region for the two-pointer field being
        /// loaded (e.g. the `VMStoreContext::stack_chain` region, or the
        /// `VMContRef` region for a continuation's parent chain).
        pub fn load<'a>(
            _env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            pointer: ir::Value,
            initial_offset: i32,
            pointer_type: ir::Type,
            region: Option<ir::AliasRegion>,
        ) -> VMStackChain {
            let memflags = ir::MemFlagsData::trusted().with_alias_region(region);
            let mut offset = initial_offset;
            let mut data = vec![];
            for _ in 0..2 {
                data.push(builder.ins().load(pointer_type, memflags, pointer, offset));
                offset += i32::try_from(pointer_type.bytes()).unwrap();
            }
            let data = <[ir::Value; 2]>::try_from(data).unwrap();
            Self::from_raw_parts(data)
        }

        /// Store this `VMStackChain` object at the given address.
        ///
        /// The `region` is the alias region for the two-pointer field being
        /// stored (e.g. the `VMStoreContext::stack_chain` region, or the
        /// `VMContRef` region for a continuation's parent chain).
        pub fn store<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            target_pointer: ir::Value,
            initial_offset: i32,
            region: Option<ir::AliasRegion>,
        ) {
            let memflags = ir::MemFlagsData::trusted().with_alias_region(region);
            let mut offset = initial_offset;
            let data = self.to_raw_parts();

            for value in data {
                debug_assert_eq!(builder.func.dfg.value_type(value), env.pointer_type());
                builder.ins().store(memflags, value, target_pointer, offset);
                offset += i32::try_from(env.pointer_type().bytes()).unwrap();
            }
        }

        /// Use this only if you've already checked that `self` corresponds to a `VMStackChain::Continuation`.
        pub fn unchecked_get_continuation(&self) -> ir::Value {
            self.payload
        }

        /// Must only be called if `self` represents a `InitialStack` or
        /// `Continuation` variant. Returns a pointer to the associated
        /// `CommonStackInformation` object.
        pub fn get_common_stack_information<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            _builder: &mut FunctionBuilder,
        ) -> VMCommonStackInformation {
            // `self` corresponds to a VMStackChain::InitialStack or
            // VMStackChain::Continuation.
            // In both cases, the payload is a pointer.
            let address = self.payload;

            // `obj` is now a pointer to the beginning of either
            // 1. A `VMContRef` struct (in the case of a
            // VMStackChain::Continuation)
            // 2. A CommonStackInformation struct (in the case of
            // VMStackChain::InitialStack)
            //
            // Since a `VMContRef` starts with an (inlined) CommonStackInformation
            // object at offset 0, we actually have in both cases that `ptr` is
            // now the address of the beginning of a VMStackLimits object.
            debug_assert_eq!(env.offsets.ptr.vm_cont_ref().common_stack_information(), 0);
            VMCommonStackInformation { address }
        }
    }

    impl VMCommonStackInformation {
        fn get_stack_limits_ptr<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            let offset: i64 = env
                .offsets
                .ptr
                .vm_common_stack_information()
                .limits()
                .into();

            builder.ins().iadd_imm_s(self.address, offset)
        }

        fn load_state<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            env.alias_regions
                .vm_common_stack_information()
                .state()
                .load(&mut builder.cursor(), self.address)
        }

        fn set_state_no_payload<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            discriminant: u32,
        ) {
            let discriminant = builder.ins().iconst(I32, i64::from(discriminant));
            env.alias_regions
                .vm_common_stack_information()
                .state()
                .store(&mut builder.cursor(), self.address, discriminant);
        }

        pub fn set_state_running<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) {
            let discriminant = wasmtime_environ::STACK_STATE_RUNNING_DISCRIMINANT;
            self.set_state_no_payload(env, builder, discriminant);
        }

        pub fn set_state_parent<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) {
            let discriminant = wasmtime_environ::STACK_STATE_PARENT_DISCRIMINANT;
            self.set_state_no_payload(env, builder, discriminant);
        }

        pub fn set_state_returned<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) {
            let discriminant = wasmtime_environ::STACK_STATE_RETURNED_DISCRIMINANT;
            self.set_state_no_payload(env, builder, discriminant);
        }

        pub fn set_state_suspended<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) {
            let discriminant = wasmtime_environ::STACK_STATE_SUSPENDED_DISCRIMINANT;
            self.set_state_no_payload(env, builder, discriminant);
        }

        pub fn set_state_trapped<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) {
            let discriminant = wasmtime_environ::STACK_STATE_TRAPPED_DISCRIMINANT;
            self.set_state_no_payload(env, builder, discriminant);
        }

        /// Checks whether the `VMStackState` reflects that the stack has ever been
        /// active (instead of just having been allocated, but never resumed).
        pub fn was_invoked<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            let actual_state = self.load_state(env, builder);
            let allocated = wasmtime_environ::STACK_STATE_FRESH_DISCRIMINANT;
            builder
                .ins()
                .icmp_imm_s(IntCC::NotEqual, actual_state, i64::from(allocated))
        }

        pub fn get_handler_list<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> VMHandlerList {
            let offset: i64 = env
                .offsets
                .ptr
                .vm_common_stack_information()
                .handlers()
                .into();
            let address = builder.ins().iadd_imm_s(self.address, offset);
            VMHandlerList::new(address)
        }

        pub fn get_first_switch_handler_index<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            env.alias_regions
                .vm_common_stack_information()
                .first_switch_handler_index()
                .load(&mut builder.cursor(), self.address)
        }

        pub fn set_first_switch_handler_index<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            value: ir::Value,
        ) {
            env.alias_regions
                .vm_common_stack_information()
                .first_switch_handler_index()
                .store(&mut builder.cursor(), self.address, value);
        }

        /// Restores the stack limit and Wasm entry handler fields in
        /// `VMStoreContext` from this object's `VMStackLimits`.
        pub fn write_limits_to_vmcontext<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            vmruntime_limits_ptr: ir::Value,
        ) {
            let stack_limits_ptr = self.get_stack_limits_ptr(env, builder);

            // The load side reads this continuation's inline `VMStackLimits`;
            // the store side targets the `VMStoreContext`.
            let stack_limit = env
                .alias_regions
                .vm_stack_limits()
                .stack_limit()
                .load(&mut builder.cursor(), stack_limits_ptr);
            env.alias_regions.vm_store_context().stack_limit().store(
                &mut builder.cursor(),
                vmruntime_limits_ptr,
                stack_limit,
            );

            let last_wasm_entry_fp = env
                .alias_regions
                .vm_stack_limits()
                .last_wasm_entry_fp()
                .load(&mut builder.cursor(), stack_limits_ptr);
            env.alias_regions
                .vm_store_context()
                .last_wasm_entry_fp()
                .store(
                    &mut builder.cursor(),
                    vmruntime_limits_ptr,
                    last_wasm_entry_fp,
                );

            let last_wasm_entry_sp = env
                .alias_regions
                .vm_stack_limits()
                .last_wasm_entry_sp()
                .load(&mut builder.cursor(), stack_limits_ptr);
            env.alias_regions
                .vm_store_context()
                .last_wasm_entry_sp()
                .store(
                    &mut builder.cursor(),
                    vmruntime_limits_ptr,
                    last_wasm_entry_sp,
                );

            let last_wasm_entry_trap_handler = env
                .alias_regions
                .vm_stack_limits()
                .last_wasm_entry_trap_handler()
                .load(&mut builder.cursor(), stack_limits_ptr);
            env.alias_regions
                .vm_store_context()
                .last_wasm_entry_trap_handler()
                .store(
                    &mut builder.cursor(),
                    vmruntime_limits_ptr,
                    last_wasm_entry_trap_handler,
                );
        }

        /// Saves the Wasm entry handler fields from `VMStoreContext` into this
        /// object's `VMStackLimits`.
        ///
        /// If `load_stack_limit` is true, we do the same for the `stack_limit`
        /// field.
        pub fn load_limits_from_vmcontext<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
            vmruntime_limits_ptr: ir::Value,
            load_stack_limit: bool,
        ) {
            let stack_limits_ptr = self.get_stack_limits_ptr(env, builder);

            // Loads read the `VMStoreContext`; stores write this
            // continuation's inline `VMStackLimits`.
            let last_wasm_entry_fp = env
                .alias_regions
                .vm_store_context()
                .last_wasm_entry_fp()
                .load(&mut builder.cursor(), vmruntime_limits_ptr);
            let last_wasm_entry_sp = env
                .alias_regions
                .vm_store_context()
                .last_wasm_entry_sp()
                .load(&mut builder.cursor(), vmruntime_limits_ptr);
            let last_wasm_entry_trap_handler = env
                .alias_regions
                .vm_store_context()
                .last_wasm_entry_trap_handler()
                .load(&mut builder.cursor(), vmruntime_limits_ptr);

            env.alias_regions
                .vm_stack_limits()
                .last_wasm_entry_fp()
                .store(&mut builder.cursor(), stack_limits_ptr, last_wasm_entry_fp);
            env.alias_regions
                .vm_stack_limits()
                .last_wasm_entry_sp()
                .store(&mut builder.cursor(), stack_limits_ptr, last_wasm_entry_sp);
            env.alias_regions
                .vm_stack_limits()
                .last_wasm_entry_trap_handler()
                .store(
                    &mut builder.cursor(),
                    stack_limits_ptr,
                    last_wasm_entry_trap_handler,
                );

            if load_stack_limit {
                // Load from the `VMStoreContext`...
                let stack_limit = env
                    .alias_regions
                    .vm_store_context()
                    .stack_limit()
                    .load(&mut builder.cursor(), vmruntime_limits_ptr);
                // ...and store to this continuation's inline `VMStackLimits`.
                env.alias_regions.vm_stack_limits().stack_limit().store(
                    &mut builder.cursor(),
                    stack_limits_ptr,
                    stack_limit,
                );
            }
        }
    }

    impl VMContinuationStack {
        /// The parameter is NOT the "top of stack" address of the stack itself. In line
        /// with how the (runtime) `FiberStack` type works, this is a pointer to
        /// the TOS address.
        pub fn new(tos_ptr: ir::Value) -> Self {
            Self { tos_ptr }
        }

        fn load_top_of_stack<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            let region = env
                .alias_regions
                .vm_cont_ref_top_of_stack_region(builder.func);
            let mem_flags = ir::MemFlagsData::trusted().with_alias_region(Some(region));
            builder
                .ins()
                .load(env.pointer_type(), mem_flags, self.tos_ptr, 0)
        }

        /// Returns address of the control context stored in the stack memory,
        /// as used by stack_switch instructions.
        pub fn load_control_context<'a>(
            &self,
            env: &mut crate::func_environ::FuncEnvironment<'a>,
            builder: &mut FunctionBuilder,
        ) -> ir::Value {
            let tos = self.load_top_of_stack(env, builder);
            // Control context begins 24 bytes below top of stack (see unix.rs)
            builder.ins().iadd_imm_s(tos, -0x18)
        }
    }
}

use helpers::VMStackChain;
use stack_switching_helpers as helpers;

fn types_need_gc_ref_markers(types: &[WasmValType]) -> bool {
    types.iter().any(|ty| ty.is_vmgcref_type_and_not_i31())
}

/// Stores the given arguments in the appropriate `VMPayloads` object in the
/// continuation. If the continuation was never invoked, use the `args` object.
/// Otherwise, use the `values` object.
pub(crate) fn vmcontref_store_payloads<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    values: &[ir::Value],
    types: &[WasmValType],
    contref: ir::Value,
) {
    debug_assert_eq!(values.len(), types.len());
    let needs_gc_ref_markers = types_need_gc_ref_markers(types);
    let count =
        u32::try_from(values.len()).expect("Number of stack switching payloads should fit in u32");
    if values.len() > 0 {
        let use_args_block = builder.create_block();
        let use_payloads_block = builder.create_block();
        let store_data_block = builder.create_block();
        builder.append_block_param(store_data_block, env.pointer_type());
        if needs_gc_ref_markers {
            builder.append_block_param(store_data_block, env.pointer_type());
        }

        let co = helpers::VMContRef::new(contref);
        let csi = co.common_stack_information(env, builder);
        let was_invoked = csi.was_invoked(env, builder);
        builder
            .ins()
            .brif(was_invoked, use_payloads_block, &[], use_args_block, &[]);

        {
            builder.switch_to_block(use_args_block);
            builder.seal_block(use_args_block);

            let args = co.args(env, builder);
            let (ptr, original_length) = args.occupy_next_slots(env, builder, count);
            let mut block_args = vec![BlockArg::Value(ptr)];
            if needs_gc_ref_markers {
                let gc_refs_ptr = args.get_gc_ref_data(env, builder);
                let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length);
                block_args.push(BlockArg::Value(gc_refs_ptr));
            }

            builder.ins().jump(store_data_block, &block_args);
        }

        {
            builder.switch_to_block(use_payloads_block);
            builder.seal_block(use_payloads_block);

            let payloads = co.values(env, builder);

            let (ptr, original_length) = payloads.occupy_next_slots(env, builder, count);
            let mut block_args = vec![BlockArg::Value(ptr)];
            if needs_gc_ref_markers {
                let gc_refs_ptr = payloads.get_gc_ref_data(env, builder);
                let gc_refs_ptr = builder.ins().iadd(gc_refs_ptr, original_length);
                block_args.push(BlockArg::Value(gc_refs_ptr));
            }
            builder.ins().jump(store_data_block, &block_args);
        }

        {
            builder.switch_to_block(store_data_block);
            builder.seal_block(store_data_block);

            let ptr = builder.block_params(store_data_block)[0];
            let gc_refs_ptr =
                needs_gc_ref_markers.then(|| builder.block_params(store_data_block)[1]);

            // Store the values.
            let region = env
                .alias_regions
                .continuation_stack_memory_region(builder.func);
            let memflags = ir::MemFlagsData::trusted().with_alias_region(Some(region));
            let mut offset = 0;
            for (index, (value, ty)) in values.iter().zip(types).enumerate() {
                builder.ins().store(memflags, *value, ptr, offset);
                if let Some(gc_refs_ptr) = gc_refs_ptr {
                    let marker = if ty.is_vmgcref_type_and_not_i31() {
                        i64::from(wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF)
                    } else {
                        0
                    };
                    let marker = builder.ins().iconst(I8, marker);
                    builder.ins().store(
                        memflags,
                        marker,
                        gc_refs_ptr,
                        i32::try_from(index).unwrap(),
                    );
                }
                offset += i32::from(env.offsets.ptr.maximum_value_size());
            }
        }
    }
}

pub(crate) fn tag_address<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    index: u32,
) -> ir::Value {
    let vmctx = env.vmctx_val(&mut builder.cursor());
    let tag_index = wasmtime_environ::TagIndex::from_u32(index);
    if let Some(def_index) = env.module.defined_tag_index(tag_index) {
        let offset = i32::try_from(env.offsets.tags().at(def_index)).unwrap();
        builder.ins().iadd_imm_s(vmctx, i64::from(offset))
    } else {
        let import_off = env.offsets.imported_tags().at(tag_index);
        env.alias_regions
            .vm_tag_import()
            .from()
            .relative_to(import_off)
            .load(&mut builder.cursor(), vmctx)
    }
}

/// Returns the stack chain saved in the given `VMContext`. Note that the
/// head of the list is the actively running stack (initial stack or
/// continuation).
pub fn vmctx_load_stack_chain<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    vmctx: ir::Value,
) -> VMStackChain {
    let stack_chain_offset = env.offsets.ptr.vm_store_context().stack_chain().into();

    // First we need to get the `VMStoreContext`.
    let vm_store_context = env
        .alias_regions
        .vmctx()
        .store_context()
        .load(&mut builder.cursor(), vmctx);

    let stack_chain_region = env
        .alias_regions
        .vmstore_context_stack_chain_region(builder.func);
    VMStackChain::load(
        env,
        builder,
        vm_store_context,
        stack_chain_offset,
        env.pointer_type(),
        Some(stack_chain_region),
    )
}

/// Stores the given stack chain saved in the `VMContext`, overwriting the
/// existing one.
pub fn vmctx_store_stack_chain<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    vmctx: ir::Value,
    stack_chain: &VMStackChain,
) {
    let stack_chain_offset = env.offsets.ptr.vm_store_context().stack_chain().into();

    // First we need to get the `VMStoreContext`.
    let vm_store_context = env
        .alias_regions
        .vmctx()
        .store_context()
        .load(&mut builder.cursor(), vmctx);

    let stack_chain_region = env
        .alias_regions
        .vmstore_context_stack_chain_region(builder.func);
    stack_chain.store(
        env,
        builder,
        vm_store_context,
        stack_chain_offset,
        Some(stack_chain_region),
    )
}

/// Similar to `vmctx_store_stack_chain`, but instead of storing an arbitrary
/// `VMStackChain`, stores VMStackChain::Continuation(contref)`.
pub fn vmctx_set_active_continuation<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    vmctx: ir::Value,
    contref: ir::Value,
) {
    let chain = VMStackChain::from_continuation(env, builder, contref);
    vmctx_store_stack_chain(env, builder, vmctx, &chain)
}

/// This function generates code that searches for a handler for `tag_address`,
/// which must be a `*mut VMTagDefinition`. The search walks up the chain of
/// continuations beginning at `start`.
///
/// The flag `search_suspend_handlers` determines whether we search for a
/// suspend or switch handler. Concretely, this influences which part of each
/// handler list we will search.
///
/// We trap if no handler was found.
///
/// The returned values are:
/// 1. The stack (continuation or initial stack, represented as a VMStackChain) in
///    whose handler list we found the tag (i.e., the stack that performed the
///    resume instruction that installed handler for the tag).
/// 2. The continuation whose parent is the stack mentioned in 1.
/// 3. The index of the handler in the handler list.
///
/// In pseudo-code, the generated code's behavior can be expressed as
/// follows:
///
/// chain_link = start
/// while !chain_link.is_initial_stack() {
///   contref = chain_link.get_contref()
///   parent_link = contref.parent
///   parent_csi = parent_link.get_common_stack_information();
///   handlers = parent_csi.handlers;
///   (begin_range, end_range) = if search_suspend_handlers {
///     (0, parent_csi.first_switch_handler_index)
///   } else {
///     (parent_csi.first_switch_handler_index, handlers.length)
///   };
///   for index in begin_range..end_range {
///     if handlers[index] == tag_address {
///       goto on_match(contref, index)
///     }
///   }
///   chain_link = parent_link
/// }
/// trap(unhandled_tag)
///
/// on_match(conref : VMContRef, handler_index : u32)
/// ... execution continues here here ...
///
fn search_handler<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    start: &helpers::VMStackChain,
    tag_address: ir::Value,
    search_suspend_handlers: bool,
) -> (VMStackChain, ir::Value, ir::Value) {
    let handle_link = builder.create_block();
    let begin_search_handler_list = builder.create_block();
    let try_index = builder.create_block();
    let compare_tags = builder.create_block();
    let on_match = builder.create_block();
    let on_no_match = builder.create_block();
    let block_args = start.to_raw_parts().map(|v| BlockArg::Value(v));

    // Terminate previous block:
    builder.ins().jump(handle_link, &block_args);

    // Block handle_link
    let chain_link = {
        builder.append_block_param(handle_link, env.pointer_type());
        builder.append_block_param(handle_link, env.pointer_type());
        builder.switch_to_block(handle_link);

        let raw_parts = builder.block_params(handle_link);
        let chain_link = helpers::VMStackChain::from_raw_parts([raw_parts[0], raw_parts[1]]);
        let is_initial_stack = chain_link.is_initial_stack(env, builder);
        builder.ins().brif(
            is_initial_stack,
            on_no_match,
            &[],
            begin_search_handler_list,
            &[],
        );
        chain_link
    };

    // Block begin_search_handler_list
    let (contref, parent_link, handler_list_data_ptr, end_range) = {
        builder.switch_to_block(begin_search_handler_list);
        let contref = chain_link.unchecked_get_continuation();
        let contref = helpers::VMContRef::new(contref);

        let parent_link = contref.get_parent_stack_chain(env, builder);
        let parent_csi = parent_link.get_common_stack_information(env, builder);

        let handlers = parent_csi.get_handler_list(env, builder);
        let handler_list_data_ptr = handlers.get_data(env, builder);

        let first_switch_handler_index = parent_csi.get_first_switch_handler_index(env, builder);

        // Note that these indices are inclusive-exclusive, i.e. [begin_range, end_range).
        let (begin_range, end_range) = if search_suspend_handlers {
            let zero = builder.ins().iconst(I32, 0);
            (zero, first_switch_handler_index)
        } else {
            let length = handlers.get_length(env, builder);
            (first_switch_handler_index, length)
        };

        builder
            .ins()
            .jump(try_index, &[BlockArg::Value(begin_range)]);

        (contref, parent_link, handler_list_data_ptr, end_range)
    };

    // Block try_index
    let index = {
        builder.append_block_param(try_index, I32);
        builder.switch_to_block(try_index);
        let index = builder.block_params(try_index)[0];

        let in_bounds = builder
            .ins()
            .icmp(IntCC::UnsignedLessThan, index, end_range);
        let block_args = parent_link.to_raw_parts().map(|v| BlockArg::Value(v));
        builder
            .ins()
            .brif(in_bounds, compare_tags, &[], handle_link, &block_args);
        index
    };

    // Block compare_tags
    {
        builder.switch_to_block(compare_tags);

        let base = handler_list_data_ptr;
        let entry_size = env.pointer_type().bytes();
        let offset = builder.ins().imul_imm_s(index, i64::from(entry_size));
        let offset = builder.ins().uextend(I64, offset);
        let entry_address = builder.ins().iadd(base, offset);

        let region = env
            .alias_regions
            .continuation_stack_memory_region(builder.func);
        let memflags = ir::MemFlagsData::trusted().with_alias_region(Some(region));

        let handled_tag = builder
            .ins()
            .load(env.pointer_type(), memflags, entry_address, 0);

        let tags_match = builder.ins().icmp(IntCC::Equal, handled_tag, tag_address);
        let incremented_index = builder.ins().iadd_imm_s(index, 1);
        builder.ins().brif(
            tags_match,
            on_match,
            &[],
            try_index,
            &[BlockArg::Value(incremented_index)],
        );
    }

    // Block on_no_match
    {
        builder.switch_to_block(on_no_match);
        builder.set_cold_block(on_no_match);
        env.trap(builder, crate::TRAP_UNHANDLED_TAG);
    }

    builder.seal_block(handle_link);
    builder.seal_block(begin_search_handler_list);
    builder.seal_block(try_index);
    builder.seal_block(compare_tags);
    builder.seal_block(on_match);
    builder.seal_block(on_no_match);

    // final block: on_match
    builder.switch_to_block(on_match);

    (parent_link, contref.address, index)
}

pub(crate) fn translate_cont_bind<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    contobj: ir::Value,
    args: &[ir::Value],
    arg_types: &[WasmValType],
) -> ir::Value {
    let (witness, contref) = fatpointer::deconstruct(env, &mut builder.cursor(), contobj);

    // The typing rules for cont.bind allow a null reference to be passed to it.
    env.trapz(builder, contref, crate::TRAP_NULL_REFERENCE);

    let mut vmcontref = helpers::VMContRef::new(contref);
    let revision = vmcontref.get_revision(env, builder);
    let evidence = builder.ins().icmp(IntCC::Equal, witness, revision);
    env.trapz(builder, evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);

    vmcontref_store_payloads(env, builder, args, arg_types, contref);

    let revision = vmcontref.incr_revision(env, builder, revision);
    let contobj = fatpointer::construct(env, &mut builder.cursor(), revision, contref);
    contobj
}

pub(crate) fn translate_cont_new<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    func: ir::Value,
    arg_types: &[WasmValType],
    return_types: &[WasmValType],
) -> WasmResult<ir::Value> {
    // The typing rules for cont.new allow a null reference to be passed to it.
    env.trapz(builder, func, crate::TRAP_NULL_REFERENCE);

    let nargs = builder
        .ins()
        .iconst(I32, i64::try_from(arg_types.len()).unwrap());
    let nreturns = builder
        .ins()
        .iconst(I32, i64::try_from(return_types.len()).unwrap());
    let gc_refs = builder.ins().iconst(
        I32,
        i64::from(u8::from(types_need_gc_ref_markers(arg_types))),
    );

    let cont_new_func = env.builtin_functions.cont_new(&mut builder.func);
    let vmctx = env.vmctx_val(&mut builder.cursor());
    let call_inst = env.call_builtin(
        builder,
        cont_new_func,
        &[vmctx, func, nargs, nreturns, gc_refs],
    );
    let contref = *builder.func.dfg.inst_results(call_inst).first().unwrap();

    let tag = helpers::VMContRef::new(contref).get_revision(env, builder);
    let contobj = fatpointer::construct(env, &mut builder.cursor(), tag, contref);
    Ok(contobj)
}

#[derive(Clone, Copy)]
enum ResumePayload<'a> {
    Values(&'a [ir::Value]),
    Throw {
        tag_index: TagIndex,
        args: &'a [ir::Value],
    },
    ThrowRef(ir::Value),
}

pub(crate) fn translate_resume<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    type_index: u32,
    resume_contobj: ir::Value,
    resume_args: &[ir::Value],
    resumetable: &[(u32, Option<ir::Block>)],
) -> WasmResult<Vec<ir::Value>> {
    translate_resume_impl(
        env,
        builder,
        type_index,
        resume_contobj,
        ResumePayload::Values(resume_args),
        resumetable,
    )
}

pub(crate) fn translate_resume_throw_ref<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    type_index: u32,
    exnref: ir::Value,
    resume_contobj: ir::Value,
    resumetable: &[(u32, Option<ir::Block>)],
) -> WasmResult<Vec<ir::Value>> {
    translate_resume_impl(
        env,
        builder,
        type_index,
        resume_contobj,
        ResumePayload::ThrowRef(exnref),
        resumetable,
    )
}

pub(crate) fn translate_resume_throw<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    type_index: u32,
    tag_index: TagIndex,
    exception_args: &[ir::Value],
    resume_contobj: ir::Value,
    resumetable: &[(u32, Option<ir::Block>)],
) -> WasmResult<Vec<ir::Value>> {
    translate_resume_impl(
        env,
        builder,
        type_index,
        resume_contobj,
        ResumePayload::Throw {
            tag_index,
            args: exception_args,
        },
        resumetable,
    )
}

fn translate_resume_impl<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    type_index: u32,
    resume_contobj: ir::Value,
    resume_payload: ResumePayload<'_>,
    resumetable: &[(u32, Option<ir::Block>)],
) -> WasmResult<Vec<ir::Value>> {
    let resume_arg_types = env
        .continuation_arguments(TypeIndex::from_u32(type_index))
        .to_vec();

    // The resume instruction is the most involved instruction to
    // compile as it is responsible for both continuation application
    // and control tag dispatch.
    //
    // Here we translate a resume instruction into several basic
    // blocks as follows:
    //
    //        previous block
    //              |
    //              |
    //        resume_block
    //       /      |      \
    //      /       |       \
    // return    suspend    trap
    //  block     block     block
    //              |
    //       dispatch block
    //
    // * resume_block handles continuation arguments and performs
    //   actual stack switch. On ordinary return from resume, it jumps
    //   to the `return_block`; on suspension it jumps to the
    //   `suspend_block`; and on a terminal trap it jumps to `trap_block`.
    // * suspend_block is used on suspension, jumps onward to
    //   `dispatch_block`.
    // * dispatch_block uses a jump table to dispatch to actual
    //   user-defined handler blocks, based on the handler index
    //   provided on suspension. Note that we do not jump to the
    //   handler blocks directly. Instead, each handler block has a
    //   corresponding preamble block, which we jump to in order to
    //   reach a particular handler block. The preamble block prepares
    //   the arguments and continuation object to be passed to the
    //   actual handler block.
    //
    let resume_block = builder.create_block();
    let return_block = builder.create_block();
    let suspend_block = builder.create_block();
    let trap_block = builder.create_block();
    let nonreturn_block = builder.create_block();
    let dispatch_block = builder.create_block();
    let throw_blocks = match resume_payload {
        ResumePayload::Throw { .. } | ResumePayload::ThrowRef(_) => {
            Some((builder.create_block(), builder.create_block()))
        }
        ResumePayload::Values(_) => None,
    };

    let vmctx = env.vmctx_val(&mut builder.cursor());

    // Split the resumetable into suspend handlers (each represented by the tag
    // index and handler block) and the switch handlers (represented just by the
    // tag index). Note that we currently don't remove duplicate tags.
    let (suspend_handlers, switch_tags): (Vec<(u32, Block)>, Vec<u32>) = resumetable
        .iter()
        .partition_map(|(tag_index, block_opt)| match block_opt {
            Some(block) => Either::Left((*tag_index, *block)),
            None => Either::Right(*tag_index),
        });

    // Technically, there is no need to have a dedicated resume block, we could
    // just put all of its contents into the current block.
    builder.ins().jump(resume_block, &[]);

    // Resume block: actually resume the continuation chain ending at `resume_contref`.
    let (resume_result, vm_runtime_limits_ptr, original_stack_chain, new_stack_chain) = {
        builder.switch_to_block(resume_block);
        builder.seal_block(resume_block);

        let (witness, resume_contref) =
            fatpointer::deconstruct(env, &mut builder.cursor(), resume_contobj);

        // The typing rules for resume allow a null reference to be passed to it.
        env.trapz(builder, resume_contref, crate::TRAP_NULL_REFERENCE);

        let mut vmcontref = helpers::VMContRef::new(resume_contref);

        let revision = vmcontref.get_revision(env, builder);
        let evidence = builder.ins().icmp(IntCC::Equal, revision, witness);
        env.trapz(builder, evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);

        match resume_payload {
            ResumePayload::Values(resume_args) => {
                debug_assert_eq!(resume_args.len(), resume_arg_types.len());
                let _next_revision = vmcontref.incr_revision(env, builder, revision);
                if resume_args.len() > 0 {
                    // We store the arguments in the `VMContRef` to be resumed.
                    vmcontref_store_payloads(
                        env,
                        builder,
                        resume_args,
                        &resume_arg_types,
                        resume_contref,
                    );
                }
            }
            ResumePayload::Throw { .. } | ResumePayload::ThrowRef(_) => {
                // Materializing an exception can allocate and fail. Do it
                // after validating the continuation, but before consuming it.
                let exnref = match resume_payload {
                    ResumePayload::Throw { tag_index, args } => {
                        gc::translate_exn_new(env, builder, tag_index, args)?
                    }
                    ResumePayload::ThrowRef(exnref) => {
                        // Validate both operands before consuming the
                        // continuation.
                        env.trapz(builder, exnref, crate::TRAP_NULL_REFERENCE);
                        exnref
                    }
                    ResumePayload::Values(_) => unreachable!(),
                };
                let csi = vmcontref.common_stack_information(env, builder);
                let was_invoked = csi.was_invoked(env, builder);
                let _next_revision = vmcontref.incr_revision(env, builder, revision);

                let (invoked_throw_block, fresh_throw_block) = throw_blocks.unwrap();
                builder.ins().brif(
                    was_invoked,
                    invoked_throw_block,
                    &[],
                    fresh_throw_block,
                    &[],
                );

                // A fresh continuation has not entered its Wasm function yet.
                // Throwing into its initial suspension point therefore
                // terminates it without executing any of its body.
                builder.switch_to_block(fresh_throw_block);
                builder.seal_block(fresh_throw_block);
                builder.set_cold_block(fresh_throw_block);
                csi.set_state_trapped(env, builder);
                vmcontref.args(env, builder).clear(env, builder, true);
                vmcontref.values(env, builder).clear(env, builder, true);
                gc::translate_exn_throw_ref(env, builder, exnref)?;

                builder.switch_to_block(invoked_throw_block);
                builder.seal_block(invoked_throw_block);
                let values = vmcontref.values(env, builder);
                values.store_data_entries(env, builder, &[exnref]);
            }
        }

        // Splice together stack chains:
        // Connect the end of the chain starting at `resume_contref` to the currently active chain.
        let mut last_ancestor = helpers::VMContRef::new(vmcontref.get_last_ancestor(env, builder));

        // Make the currently running continuation (if any) the parent of the one we are about to resume.
        let original_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);
        last_ancestor.set_parent_stack_chain(env, builder, &original_stack_chain);

        // Just for consistency: `vmcontref` is about to get state Running, so let's zero out its last_ancestor field.
        let zero = builder.ins().iconst(env.pointer_type(), 0);
        vmcontref.set_last_ancestor(env, builder, zero);

        // We mark `resume_contref` as the currently running one
        vmctx_set_active_continuation(env, builder, vmctx, resume_contref);

        // The code below manipulates the per-stack runtime state as follows:
        //
        // 1. Copy the stack limit and Wasm entry handler from
        // `VMStoreContext` into the currently active stack (i.e., the
        // one that will become the parent of the to-be-resumed one).
        //
        // 2. Copy the corresponding fields from the resumed
        // continuation into `VMStoreContext`.
        //
        // See the comment on `wasmtime_environ::VMStackChain` for a
        // description of the invariants that we maintain for the various stack
        // limits.

        // `resume_contref` is now active, and its parent is suspended.
        let resume_contref = helpers::VMContRef::new(resume_contref);
        let resume_csi = resume_contref.common_stack_information(env, builder);
        let parent_csi = original_stack_chain.get_common_stack_information(env, builder);
        resume_csi.set_state_running(env, builder);
        parent_csi.set_state_parent(env, builder);

        // We update the `VMStackLimits` of the parent of the continuation to be resumed
        // as well as the `VMRuntimeLimits`.
        // See the comment on `wasmtime_environ::VMStackChain` for a description
        // of the invariants that we maintain for the various stack limits.
        let vm_runtime_limits_ptr = env
            .alias_regions
            .vmctx()
            .store_context()
            .load(&mut builder.cursor(), vmctx);
        parent_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, true);
        resume_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);

        // Install handlers in (soon to be) parent's VMHandlerList:
        // Let the i-th handler clause be (on $tag $block).
        // Then the i-th entry of the VMHandlerList will be the address of $tag.
        let handler_list = parent_csi.get_handler_list(env, builder);

        if resumetable.len() > 0 {
            // Total number of handlers (suspend and switch).
            let handler_count = u32::try_from(resumetable.len()).unwrap();
            // Populate the Array's data ptr with a pointer to a sufficiently
            // large area on this stack.
            env.stack_switching.handler_list_buffer =
                Some(handler_list.allocate_or_reuse_stack_slot(
                    env,
                    builder,
                    handler_count,
                    env.stack_switching.handler_list_buffer,
                ));

            let suspend_handler_count = suspend_handlers.len();

            // All handlers, represented by the indices of the tags they handle.
            // All the suspend handlers come first, followed by all the switch handlers.
            let all_handlers = suspend_handlers
                .iter()
                .map(|(tag_index, _block)| *tag_index)
                .chain(switch_tags);

            // Translate all tag indices to tag addresses (i.e., the corresponding *mut VMTagDefinition).
            let all_tag_addresses: Vec<ir::Value> = all_handlers
                .map(|tag_index| tag_address(env, builder, tag_index))
                .collect();

            // Store all tag addresses in the handler list.
            handler_list.store_data_entries(env, builder, &all_tag_addresses);

            // To enable distinguishing switch and suspend handlers when searching the handler list:
            // Store at which index the switch handlers start.
            let first_switch_handler_index = builder
                .ins()
                .iconst(I32, i64::try_from(suspend_handler_count).unwrap());
            parent_csi.set_first_switch_handler_index(env, builder, first_switch_handler_index);
        }

        let resume_payload = match resume_payload {
            ResumePayload::Values(_) => ControlEffect::encode_resume(builder),
            ResumePayload::Throw { .. } | ResumePayload::ThrowRef(_) => {
                ControlEffect::encode_resume_throw(builder)
            }
        }
        .to_u64();

        // Note that the control context we use for switching is not the one in
        // (the stack of) resume_contref, but in (the stack of) last_ancestor!
        let fiber_stack = last_ancestor.get_fiber_stack(env, builder);
        let control_context_ptr = fiber_stack.load_control_context(env, builder);

        let result = emit_stack_switch(
            env,
            builder,
            control_context_ptr,
            control_context_ptr,
            resume_payload,
            |env, builder| {
                (
                    parent_csi.address,
                    last_ancestor.common_stack_information(env, builder).address,
                )
            },
        );

        // At this point we know nothing about the continuation that just
        // suspended or returned. In particular, it does not have to be what we
        // called `resume_contref` earlier on. We must reload the information
        // about the now active continuation from the VMContext.
        let new_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);

        // Now the parent contref (or initial stack) is active again
        vmctx_store_stack_chain(env, builder, vmctx, &original_stack_chain);
        parent_csi.set_state_running(env, builder);

        // Just for consistency: Clear the handler list.
        handler_list.clear(env, builder, true);
        parent_csi.set_first_switch_handler_index(env, builder, zero);

        // An ordinary return is encoded as zero, so make it the fast path with
        // a single branch. Only the nonreturn path needs to distinguish a
        // terminal trap from a suspension.
        // TODO(dhil): We may want to make suspension the fast path
        // instead, as I hypothesise suspensions occur more often than
        // normal returns in a stack-switching application.
        let result = ControlEffect::from_u64(result);
        builder
            .ins()
            .brif(result.to_u64(), nonreturn_block, &[], return_block, &[]);

        builder.switch_to_block(nonreturn_block);
        builder.seal_block(nonreturn_block);
        let is_trap = result.is_trap(builder);
        builder
            .ins()
            .brif(is_trap, trap_block, &[], suspend_block, &[]);

        (
            result,
            vm_runtime_limits_ptr,
            original_stack_chain,
            new_stack_chain,
        )
    };

    // A Wasm trap terminates the child continuation. The user code
    // cannot handle this trap, thus we restore the parent stack's
    // entry handler and retrap there. Its array trampoline will
    // return the failure through `fiber_start`, propagating the
    // terminal control effect until the nearest host/engine frame
    // delimiting this Wasm invocation is reached.
    {
        builder.switch_to_block(trap_block);
        builder.seal_block(trap_block);
        builder.set_cold_block(trap_block);

        let trapped_continuation = new_stack_chain.unchecked_get_continuation();
        let trapped_continuation = helpers::VMContRef::new(trapped_continuation);
        let trapped_csi = trapped_continuation.common_stack_information(env, builder);
        trapped_csi.set_state_trapped(env, builder);

        let parent_csi = original_stack_chain.get_common_stack_information(env, builder);
        parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);

        let args = trapped_continuation.args(env, builder);
        args.clear(env, builder, true);
        let values = trapped_continuation.values(env, builder);
        values.clear(env, builder, true);

        gc::translate_raise(env, builder)?;
    }

    // The suspend block: Only used when we suspended, not for returns.
    // Here we extract the index of the handler to use.
    let (handler_index, suspended_contref, suspended_contobj) = {
        builder.switch_to_block(suspend_block);
        builder.seal_block(suspend_block);
        env.on_catch_block_entry(builder);

        let suspended_continuation = new_stack_chain.unchecked_get_continuation();
        let mut suspended_continuation = helpers::VMContRef::new(suspended_continuation);
        let suspended_csi = suspended_continuation.common_stack_information(env, builder);

        // Note that at the suspend site, we already
        // 1. Set the state of suspended_continuation to Suspended
        // 2. Set suspended_continuation.last_ancestor
        // 3. Broke the continuation chain at suspended_continuation.last_ancestor

        // We store parts of the VMRuntimeLimits into the continuation that just suspended.
        suspended_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false);

        // Afterwards (!), restore parts of the VMRuntimeLimits from the
        // parent of the suspended continuation (which is now active).
        let parent_csi = original_stack_chain.get_common_stack_information(env, builder);
        parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);

        // Extract the handler index
        let handler_index = resume_result.handler_index(builder);

        let revision = suspended_continuation.get_revision(env, builder);
        let suspended_contobj = fatpointer::construct(
            env,
            &mut builder.cursor(),
            revision,
            suspended_continuation.address,
        );

        // We need to terminate this block before being allowed to switch to
        // another one.
        builder.ins().jump(dispatch_block, &[]);

        (handler_index, suspended_continuation, suspended_contobj)
    };

    // For technical reasons, the jump table needs to have a default
    // block. In our case, it should be unreachable, since the handler
    // index we dispatch on should correspond to an actual handler
    // block in the jump table.
    let jt_default_block = builder.create_block();
    {
        builder.switch_to_block(jt_default_block);
        builder.set_cold_block(jt_default_block);

        env.trap(builder, crate::TRAP_UNREACHABLE);
    }

    // We create a preamble block for each of the actual handler blocks: It
    // reads the necessary arguments and passes them to the actual handler
    // block, together with the continuation object.
    let target_preamble_blocks = {
        let mut preamble_blocks = vec![];

        for &(handle_tag, target_block) in &suspend_handlers {
            let preamble_block = builder.create_block();
            preamble_blocks.push(preamble_block);
            builder.switch_to_block(preamble_block);

            let param_types = env.tag_params(TagIndex::from_u32(handle_tag)).to_vec();

            let values = suspended_contref.values(env, builder);
            let mut suspend_args: Vec<ir::Value> =
                values.load_data_entries(env, builder, &param_types);

            // At the suspend site, we store the suspend args in the the
            // `values` buffer of the VMContRef that was active at the time that
            // the suspend instruction was performed.
            suspend_args.push(suspended_contobj);

            // We clear the suspend args. This is mostly for consistency. Note
            // that we don't zero out the data buffer, we still need it for the
            values.clear(env, builder, false);

            set_block_params(env, builder, target_block, &suspend_args);
            builder.ins().jump(target_block, &[]);
        }

        preamble_blocks
    };

    // Dispatch block. All it does is jump to the right preamble block based on
    // the handler index.
    {
        builder.switch_to_block(dispatch_block);
        builder.seal_block(dispatch_block);

        let default_bc = builder.func.dfg.block_call(jt_default_block, &[]);

        let adapter_bcs: Vec<BlockCall> = target_preamble_blocks
            .iter()
            .map(|b| builder.func.dfg.block_call(*b, &[]))
            .collect();

        let jt_data = JumpTableData::new(default_bc, &adapter_bcs);
        let jt = builder.create_jump_table(jt_data);

        builder.ins().br_table(handler_index, jt);

        for preamble_block in target_preamble_blocks {
            builder.seal_block(preamble_block);
        }
        builder.seal_block(jt_default_block);
    }

    // Return block: Jumped to by resume block if continuation
    // returned normally.
    {
        builder.switch_to_block(return_block);
        builder.seal_block(return_block);

        // If we got a return signal, a continuation must have been running.
        let returned_contref = new_stack_chain.unchecked_get_continuation();
        let returned_contref = helpers::VMContRef::new(returned_contref);

        // Restore parts of the VMRuntimeLimits from the parent of the
        // returned continuation (which is now active).
        let parent_csi = original_stack_chain.get_common_stack_information(env, builder);
        parent_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);

        let returned_csi = returned_contref.common_stack_information(env, builder);
        returned_csi.set_state_returned(env, builder);

        // Load the values returned by the continuation.
        let return_types = env
            .continuation_returns(TypeIndex::from_u32(type_index))
            .to_vec();
        let payloads = returned_contref.args(env, builder);
        let return_values = payloads.load_data_entries(env, builder, &return_types);
        payloads.clear(env, builder, true);

        Ok(return_values)
    }
}

fn load_resume_values_or_throw<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    result: ControlEffect,
    continuation: helpers::VMContRef,
    return_types: &[WasmValType],
) -> WasmResult<Vec<ir::Value>> {
    let throw_block = builder.create_block();
    let values_block = builder.create_block();
    let is_resume_throw = result.is_resume_throw(builder);
    builder
        .ins()
        .brif(is_resume_throw, throw_block, &[], values_block, &[]);

    builder.switch_to_block(throw_block);
    builder.seal_block(throw_block);
    builder.set_cold_block(throw_block);
    let values = continuation.values(env, builder);
    // Exception references use Wasmtime's compressed, 32-bit GC-reference
    // representation.
    let exnref_ty = WasmValType::Ref(WasmRefType {
        nullable: false,
        heap_type: WasmHeapType::Exn,
    });
    let exnref = values.load_data_entries(env, builder, &[exnref_ty])[0];
    values.clear(env, builder, true);
    gc::translate_exn_throw_ref(env, builder, exnref)?;

    builder.switch_to_block(values_block);
    builder.seal_block(values_block);
    let values = continuation.values(env, builder);
    let return_values = values.load_data_entries(env, builder, return_types);
    values.clear(env, builder, true);
    Ok(return_values)
}

pub(crate) fn translate_suspend<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    tag_index: u32,
    suspend_args: &[ir::Value],
    suspend_arg_types: &[WasmValType],
    tag_return_types: &[WasmValType],
) -> WasmResult<Vec<ir::Value>> {
    debug_assert_eq!(suspend_args.len(), suspend_arg_types.len());
    let tag_addr = tag_address(env, builder, tag_index);

    let vmctx = env.vmctx_val(&mut builder.cursor());
    let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);

    let (handler_stack_chain, end_of_chain_contref, handler_index) =
        search_handler(env, builder, &active_stack_chain, tag_addr, true);

    // If we get here, the search_handler logic succeeded (i.e., did not trap).
    // Thus, there is at least one parent, so we are not on the initial stack.
    // Can therefore extract continuation directly.
    let active_contref = active_stack_chain.unchecked_get_continuation();
    let active_contref = helpers::VMContRef::new(active_contref);
    let mut end_of_chain_contref = helpers::VMContRef::new(end_of_chain_contref);

    active_contref.set_last_ancestor(env, builder, end_of_chain_contref.address);

    // We stack allocate enough room for the suspend arguments and the
    // return values including the possible exception reference.
    let values = active_contref.values(env, builder);
    let required_capacity = u32::try_from(std::cmp::max(
        1, // This accounts for the possible exception reference.
        std::cmp::max(suspend_args.len(), tag_return_types.len()),
    ))
    .expect("Number of stack switching payloads should fit in u32");

    let needs_gc_ref_markers =
        types_need_gc_ref_markers(suspend_arg_types) || types_need_gc_ref_markers(tag_return_types);
    let existing_storage = env.stack_switching.values_storage;
    env.stack_switching.values_storage = Some(values.prepare_stack_storage(
        env,
        builder,
        required_capacity,
        suspend_arg_types,
        needs_gc_ref_markers,
        existing_storage,
    ));

    if suspend_args.len() > 0 {
        values.store_data_entries(env, builder, suspend_args);
    }

    // Set current continuation to suspended and break up handler chain.
    let active_contref_csi = active_contref.common_stack_information(env, builder);
    active_contref_csi.set_state_suspended(env, builder);
    let absent_chain_link = VMStackChain::absent(env, builder);
    end_of_chain_contref.set_parent_stack_chain(env, builder, &absent_chain_link);

    let suspend_payload = ControlEffect::encode_suspend(builder, handler_index).to_u64();

    // Note that the control context we use for switching is the one
    // at the end of the chain, not the one in active_contref!
    // This also means that stack_switch saves the information about
    // the current stack in the control context located in the stack
    // of end_of_chain_contref.
    let fiber_stack = end_of_chain_contref.get_fiber_stack(env, builder);
    let control_context_ptr = fiber_stack.load_control_context(env, builder);

    let result = emit_stack_switch(
        env,
        builder,
        control_context_ptr,
        control_context_ptr,
        suspend_payload,
        |env, builder| {
            (
                end_of_chain_contref
                    .common_stack_information(env, builder)
                    .address,
                handler_stack_chain
                    .get_common_stack_information(env, builder)
                    .address,
            )
        },
    );

    // A normal resume supplies the tag's return values. `resume_throw_ref`
    // supplies an exception reference and throws it at this suspension point.
    let return_values = load_resume_values_or_throw(
        env,
        builder,
        ControlEffect::from_u64(result),
        active_contref,
        tag_return_types,
    );

    if needs_gc_ref_markers {
        let zero = builder.ins().iconst(env.pointer_type(), 0);
        values.set_gc_ref_data(env, builder, zero);
    }

    return_values
}

pub(crate) fn translate_switch<'a>(
    env: &mut crate::func_environ::FuncEnvironment<'a>,
    builder: &mut FunctionBuilder,
    tag_index: u32,
    switchee_contobj: ir::Value,
    switch_args: &[ir::Value],
    switch_arg_types: &[WasmValType],
    return_types: &[WasmValType],
) -> WasmResult<Vec<ir::Value>> {
    let vmctx = env.vmctx_val(&mut builder.cursor());

    // Check and increment revision on switchee continuation object (i.e., the
    // one being switched to). Logically, the switchee continuation extends from
    // `switchee_contref` to `switchee_contref.last_ancestor` (i.e., the end of
    // the parent chain starting at `switchee_contref`).
    let switchee_contref = {
        let (witness, target_contref) =
            fatpointer::deconstruct(env, &mut builder.cursor(), switchee_contobj);

        // The typing rules for switch allow a null reference to be passed to it.
        env.trapz(builder, target_contref, crate::TRAP_NULL_REFERENCE);

        let mut target_contref = helpers::VMContRef::new(target_contref);

        let revision = target_contref.get_revision(env, builder);
        let evidence = builder.ins().icmp(IntCC::Equal, revision, witness);
        env.trapz(builder, evidence, crate::TRAP_CONTINUATION_ALREADY_CONSUMED);
        let _next_revision = target_contref.incr_revision(env, builder, revision);
        target_contref
    };

    // We create the "switcher continuation" (i.e., the one executing switch)
    // from the current execution context: Logically, it extends from the
    // continuation reference executing `switch` (subsequently called
    // `switcher_contref`) to the immediate child (called
    // `switcher_contref_last_ancestor`) of the stack with the corresponding
    // handler (saved in `handler_stack_chain`).
    let return_values_need_gc_ref_markers = types_need_gc_ref_markers(return_types);
    let (
        switcher_contref,
        switcher_contobj,
        switcher_contref_last_ancestor,
        handler_stack_chain,
        vm_runtime_limits_ptr,
    ) = {
        let tag_addr = tag_address(env, builder, tag_index);
        let active_stack_chain = vmctx_load_stack_chain(env, builder, vmctx);
        let (handler_stack_chain, last_ancestor, _handler_index) =
            search_handler(env, builder, &active_stack_chain, tag_addr, false);
        let mut last_ancestor = helpers::VMContRef::new(last_ancestor);

        // If we get here, the search_handler logic succeeded (i.e., did not trap).
        // Thus, there is at least one parent, so we are not on the initial stack.
        // Can therefore extract continuation directly.
        let switcher_contref = active_stack_chain.unchecked_get_continuation();
        let mut switcher_contref = helpers::VMContRef::new(switcher_contref);

        switcher_contref.set_last_ancestor(env, builder, last_ancestor.address);

        // We stack allocate enough room for the switch arguments and
        // the return values including the possible exception
        // reference.
        let values = switcher_contref.values(env, builder);
        let required_capacity = u32::try_from(std::cmp::max(1, return_types.len())).unwrap();
        let existing_storage = env.stack_switching.values_storage;
        env.stack_switching.values_storage = Some(values.prepare_stack_storage(
            env,
            builder,
            required_capacity,
            // No GC ref marking is required at this point. Upon
            // return to this site `vmcontref_store_payloads` will
            // store the markers, if needed.
            &[],
            return_values_need_gc_ref_markers,
            existing_storage,
        ));

        let switcher_contref_csi = switcher_contref.common_stack_information(env, builder);
        switcher_contref_csi.set_state_suspended(env, builder);
        // We break off `switcher_contref` from the chain of active
        // continuations, by separating the link between `last_ancestor` and its
        // parent stack.
        let absent = VMStackChain::absent(env, builder);
        last_ancestor.set_parent_stack_chain(env, builder, &absent);

        // Load current runtime limits from `VMContext` and store in the
        // switcher continuation.
        let vm_runtime_limits_ptr = env
            .alias_regions
            .vmctx()
            .store_context()
            .load(&mut builder.cursor(), vmctx);
        switcher_contref_csi.load_limits_from_vmcontext(env, builder, vm_runtime_limits_ptr, false);

        let revision = switcher_contref.get_revision(env, builder);
        let new_contobj = fatpointer::construct(
            env,
            &mut builder.cursor(),
            revision,
            switcher_contref.address,
        );

        (
            switcher_contref,
            new_contobj,
            last_ancestor,
            handler_stack_chain,
            vm_runtime_limits_ptr,
        )
    };

    // Prepare switchee continuation:
    // - Store "ordinary" switch arguments as well as the contobj just
    //   synthesized from the current context (i.e., `switcher_contobj`) in the
    //   switchee continuation's payload buffer.
    // - Splice switchee's continuation chain with handler stack to form new
    //   overall chain of active continuations.
    let (switchee_contref_csi, switchee_contref_last_ancestor) = {
        let mut combined_payloads = switch_args.to_vec();
        combined_payloads.push(switcher_contobj);
        vmcontref_store_payloads(
            env,
            builder,
            &combined_payloads,
            switch_arg_types,
            switchee_contref.address,
        );

        let switchee_contref_csi = switchee_contref.common_stack_information(env, builder);
        switchee_contref_csi.set_state_running(env, builder);

        let switchee_contref_last_ancestor = switchee_contref.get_last_ancestor(env, builder);
        let mut switchee_contref_last_ancestor =
            helpers::VMContRef::new(switchee_contref_last_ancestor);

        switchee_contref_last_ancestor.set_parent_stack_chain(env, builder, &handler_stack_chain);

        (switchee_contref_csi, switchee_contref_last_ancestor)
    };

    // Update VMContext/Store: Update active continuation and `VMRuntimeLimits`.
    {
        vmctx_set_active_continuation(env, builder, vmctx, switchee_contref.address);

        switchee_contref_csi.write_limits_to_vmcontext(env, builder, vm_runtime_limits_ptr);
    }

    // Perform actual stack switch
    let result = {
        let switcher_last_ancestor_fs =
            switcher_contref_last_ancestor.get_fiber_stack(env, builder);
        let switcher_last_ancestor_cc =
            switcher_last_ancestor_fs.load_control_context(env, builder);

        let switchee_last_ancestor_fs =
            switchee_contref_last_ancestor.get_fiber_stack(env, builder);
        let switchee_last_ancestor_cc =
            switchee_last_ancestor_fs.load_control_context(env, builder);

        // The stack switch involves the following control contexts (e.g., IP,
        // SP, FP, ...):
        // - `switchee_last_ancestor_cc` contains the information to continue
        //    execution in the switchee/target continuation.
        // - `switcher_last_ancestor_cc` contains the information about how to
        //    continue execution once we suspend/return to the stack with the
        //    switch handler.
        //
        // In total, the following needs to happen:
        // 1. Load control context at `switchee_last_ancestor_cc` to perform
        //    stack switch.
        // 2. Move control context at `switcher_last_ancestor_cc` over to
        //    `switchee_last_ancestor_cc`.
        // 3. Upon actual switch, save current control context at
        //    `switcher_last_ancestor_cc`.
        //
        // We implement this as follows:
        // 1. We copy `switchee_last_ancestor_cc` to a temporary area on the
        //    stack (`tmp_control_context`).
        // 2. We copy `switcher_last_ancestor_cc` over to
        //    `switchee_last_ancestor_cc`.
        // 3. We invoke the stack switch instruction such that it reads from the
        //    temporary area, and writes to `switcher_last_ancestor_cc`.
        //
        // Note that the temporary area is only accessed once by the
        // `stack_switch` instruction emitted later in this block, meaning that we
        // don't have to worry about its lifetime.
        //
        // NOTE(frank-emrich) The implementation below results in one stack slot
        // being created per switch instruction, even though multiple switch
        // instructions in the same function could safely re-use the same stack
        // slot. Thus, we could implement logic for sharing the stack slot by
        // adding an appropriate field to `FuncEnvironment`.
        //
        // NOTE(frank-emrich) We could avoid the copying to a temporary area by
        // making `stack_switch` do all of the necessary moving itself. However,
        // that would be a rather ad-hoc change to how the instruction uses the
        // two pointers given to it.

        let cctx_size = control_context_size(env.isa().triple())?;
        let slot_size = ir::StackSlotData::new(
            ir::StackSlotKind::ExplicitSlot,
            u32::from(cctx_size),
            u8::try_from(env.pointer_type().bytes().trailing_zeros()).unwrap(),
        );
        let slot = builder.create_sized_stack_slot(slot_size);
        let tmp_control_context = builder.ins().stack_addr(env.pointer_type(), slot, 0);

        // The `*_last_ancestor_cc` control contexts live on continuation
        // stacks, but the temporary lives in a current-frame stack slot, so the
        // former get the continuation-stack-memory region while the latter gets
        // its own stack-slot region.
        let region = env
            .alias_regions
            .continuation_stack_memory_region(builder.func);
        let cc_flags = MemFlagsData::trusted().with_alias_region(Some(region));
        let tmp_region = env.alias_regions.stack_slot_region(builder.func, slot);
        let tmp_flags = MemFlagsData::trusted().with_alias_region(Some(tmp_region));
        let mut offset: i32 = 0;
        while offset < i32::from(cctx_size) {
            // switchee_last_ancestor_cc -> tmp control context
            let tmp1 = builder.ins().load(
                env.pointer_type(),
                cc_flags,
                switchee_last_ancestor_cc,
                offset,
            );
            builder
                .ins()
                .store(tmp_flags, tmp1, tmp_control_context, offset);

            // switcher_last_ancestor_cc -> switchee_last_ancestor_cc
            let tmp2 = builder.ins().load(
                env.pointer_type(),
                cc_flags,
                switcher_last_ancestor_cc,
                offset,
            );
            builder
                .ins()
                .store(cc_flags, tmp2, switchee_last_ancestor_cc, offset);

            offset += i32::try_from(env.pointer_type().bytes()).unwrap();
        }

        let switch_payload = ControlEffect::encode_switch(builder).to_u64();

        emit_stack_switch(
            env,
            builder,
            switcher_last_ancestor_cc,
            tmp_control_context,
            switch_payload,
            |env, builder| {
                (
                    switcher_contref_last_ancestor
                        .common_stack_information(env, builder)
                        .address,
                    switchee_contref_last_ancestor
                        .common_stack_information(env, builder)
                        .address,
                )
            },
        )
    };

    // After switching back to the original stack, either load the values
    // supplied by an ordinary resume or throw the injected exception.
    let return_values = load_resume_values_or_throw(
        env,
        builder,
        ControlEffect::from_u64(result),
        switcher_contref,
        return_types,
    );

    if return_values_need_gc_ref_markers {
        let zero = builder.ins().iconst(env.pointer_type(), 0);
        switcher_contref
            .values(env, builder)
            .set_gc_ref_data(env, builder, zero);
    }

    return_values
}