Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Inline `{enter,exit}_sync_call` trampolines into sync-to-sync fused adapters (#13695)

Nick Fitzgerald946f7633b387
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index bb7638a361..b0bdc536d4 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -24,6 +24,7 @@ enum VmType {
VMDrcHeapData,
VMCopyingHeapData,
VMNullHeapData,
+ VMDeferredThread,
}
/// A key that uniquely identifies an alias region across an entire compilation.
@@ -117,6 +118,7 @@ impl AliasRegionKey {
const VM_DRC_HEAP_DATA_KIND: u32 = Self::new_kind(0b1100);
const VM_COPYING_HEAP_DATA_KIND: u32 = Self::new_kind(0b1101);
const VM_NULL_HEAP_DATA_KIND: u32 = Self::new_kind(0b1110);
+ const VM_DEFERRED_THREAD_KIND: u32 = Self::new_kind(0b1111);
/// Encode this key into a raw `u32` suitable for use as an
/// `AliasRegionData::user_id`.
@@ -133,6 +135,7 @@ impl AliasRegionKey {
VmType::VMDrcHeapData => Self::VM_DRC_HEAP_DATA_KIND,
VmType::VMCopyingHeapData => Self::VM_COPYING_HEAP_DATA_KIND,
VmType::VMNullHeapData => Self::VM_NULL_HEAP_DATA_KIND,
+ VmType::VMDeferredThread => Self::VM_DEFERRED_THREAD_KIND,
};
kind | (offset & Self::OFFSET_MASK)
}
@@ -1108,6 +1111,44 @@ where
)
}
+ /// Load the `VMStoreContext::current_thread` field (the JIT-visible
+ /// deferred-thread pointer; see `VMLazyThread`).
+ pub fn vmstore_context_current_thread(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmstore_ctx: ir::Value,
+ ) -> ir::Value {
+ self.vmstore_context_load(
+ cursor,
+ self.pointer_type,
+ ir::MemFlagsData::trusted(),
+ vmstore_ctx,
+ self.offsets
+ .get_ptr_size()
+ .vmstore_context_current_thread()
+ .into(),
+ )
+ }
+
+ /// Store the `VMStoreContext::current_thread` field.
+ pub fn store_vmstore_context_current_thread(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmstore_ctx: ir::Value,
+ new_thread: ir::Value,
+ ) {
+ self.vmstore_context_store(
+ cursor,
+ ir::MemFlagsData::trusted(),
+ vmstore_ctx,
+ self.offsets
+ .get_ptr_size()
+ .vmstore_context_current_thread()
+ .into(),
+ new_thread,
+ )
+ }
+
/// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).
///
/// The caller supplies the base flags because whether the base pointer is
@@ -1342,6 +1383,194 @@ where
}
}
+/// `VMDeferredThread`-related methods.
+impl<Offsets> AliasRegions<Offsets>
+where
+ Offsets: GetPtrSize,
+{
+ fn vmdeferred_thread_region(
+ &mut self,
+ func: &mut ir::Function,
+ offset: u32,
+ ) -> ir::AliasRegion {
+ self.region(
+ func,
+ AliasRegionKey::Vm {
+ ty: VmType::VMDeferredThread,
+ offset,
+ },
+ )
+ }
+
+ fn vmdeferred_thread_load(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ ty: ir::Type,
+ base_flags: ir::MemFlagsData,
+ vmdeferred_thread_ptr: ir::Value,
+ offset: u32,
+ ) -> ir::Value {
+ let region = self.vmdeferred_thread_region(cursor.func, offset);
+ cursor.ins().load(
+ ty,
+ base_flags.with_alias_region(Some(region)),
+ vmdeferred_thread_ptr,
+ i32::try_from(offset).unwrap(),
+ )
+ }
+
+ fn vmdeferred_thread_store(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ base_flags: ir::MemFlagsData,
+ vmdeferred_thread_ptr: ir::Value,
+ offset: u32,
+ val: ir::Value,
+ ) {
+ let region = self.vmdeferred_thread_region(cursor.func, offset);
+ cursor.ins().store(
+ base_flags.with_alias_region(Some(region)),
+ val,
+ vmdeferred_thread_ptr,
+ i32::try_from(offset).unwrap(),
+ );
+ }
+
+ /// Load `VMDeferredThread::parent` (the current thread this frame replaced).
+ pub fn vmdeferred_thread_parent(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmdeferred_thread_ptr: ir::Value,
+ ) -> ir::Value {
+ self.vmdeferred_thread_load(
+ cursor,
+ self.pointer_type,
+ ir::MemFlagsData::trusted(),
+ vmdeferred_thread_ptr,
+ self.offsets
+ .get_ptr_size()
+ .vmdeferred_thread_parent()
+ .into(),
+ )
+ }
+
+ /// Store `VMDeferredThread::parent`.
+ pub fn store_vmdeferred_thread_parent(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmdeferred_thread_ptr: ir::Value,
+ parent: ir::Value,
+ ) {
+ self.vmdeferred_thread_store(
+ cursor,
+ ir::MemFlagsData::trusted(),
+ vmdeferred_thread_ptr,
+ self.offsets
+ .get_ptr_size()
+ .vmdeferred_thread_parent()
+ .into(),
+ parent,
+ )
+ }
+
+ /// Store `VMDeferredThread::caller_instance`.
+ pub fn store_vmdeferred_thread_caller_instance(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmdeferred_thread_ptr: ir::Value,
+ caller_instance: ir::Value,
+ ) {
+ self.vmdeferred_thread_store(
+ cursor,
+ ir::MemFlagsData::trusted(),
+ vmdeferred_thread_ptr,
+ self.offsets
+ .get_ptr_size()
+ .vmdeferred_thread_caller_instance()
+ .into(),
+ caller_instance,
+ )
+ }
+
+ /// Store `VMDeferredThread::callee_async`.
+ pub fn store_vmdeferred_thread_callee_async(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmdeferred_thread_ptr: ir::Value,
+ callee_async: ir::Value,
+ ) {
+ self.vmdeferred_thread_store(
+ cursor,
+ ir::MemFlagsData::trusted(),
+ vmdeferred_thread_ptr,
+ self.offsets
+ .get_ptr_size()
+ .vmdeferred_thread_callee_async()
+ .into(),
+ callee_async,
+ )
+ }
+
+ /// Store `VMDeferredThread::callee_instance`.
+ pub fn store_vmdeferred_thread_callee_instance(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmdeferred_thread_ptr: ir::Value,
+ callee_instance: ir::Value,
+ ) {
+ self.vmdeferred_thread_store(
+ cursor,
+ ir::MemFlagsData::trusted(),
+ vmdeferred_thread_ptr,
+ self.offsets
+ .get_ptr_size()
+ .vmdeferred_thread_callee_instance()
+ .into(),
+ callee_instance,
+ )
+ }
+
+ /// Load `VMDeferredThread::saved_context[i]` (a saved `context.{get,set}`
+ /// slot).
+ pub fn vmdeferred_thread_saved_context(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmdeferred_thread_ptr: ir::Value,
+ i: u8,
+ ) -> ir::Value {
+ self.vmdeferred_thread_load(
+ cursor,
+ ir::types::I32,
+ ir::MemFlagsData::trusted(),
+ vmdeferred_thread_ptr,
+ self.offsets
+ .get_ptr_size()
+ .vmdeferred_thread_saved_context(i)
+ .into(),
+ )
+ }
+
+ /// Store `VMDeferredThread::saved_context[i]`.
+ pub fn store_vmdeferred_thread_saved_context(
+ &mut self,
+ cursor: &mut FuncCursor<'_>,
+ vmdeferred_thread_ptr: ir::Value,
+ i: u8,
+ val: ir::Value,
+ ) {
+ self.vmdeferred_thread_store(
+ cursor,
+ ir::MemFlagsData::trusted(),
+ vmdeferred_thread_ptr,
+ self.offsets
+ .get_ptr_size()
+ .vmdeferred_thread_saved_context(i)
+ .into(),
+ val,
+ )
+ }
+}
+
/// `VMMemoryDefinition`-related methods.
///
/// The `base` and `current_length` fields are reached either directly through
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index 793cd20664..bfabe07a02 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -1523,6 +1523,20 @@ impl Compiler {
// succeed". Note that register restoration is part of the `try_call`
// and handler implementation.
builder.switch_to_block(exceptional_return);
+ if cfg!(feature = "component-model") && self.tunables().concurrency_support {
+ // NB: A trap unwinds over any fused adapter frames without running
+ // their `exit-sync-call`s. That can leave
+ // `VMStoreContext::current_thread` pointing at an old, invalid
+ // `VMDeferredThread` inside an old, since-unwound stack
+ // frame. Therefore, we must reset `current_thread` to avoid
+ // potential use-after-free bugs.
+ let forced = builder.ins().iconst(pointer_type, 1);
+ alias_regions.store_vmstore_context_current_thread(
+ &mut builder.cursor(),
+ vm_store_ctx,
+ forced,
+ );
+ }
let false_return = builder.ins().iconst(ir::types::I8, 0);
builder.ins().return_(&[false_return]);
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 0a915a7b19..9dd47f3d7a 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -35,13 +35,14 @@ use wasmparser::{
use wasmtime_core::math::f64_cvt_to_int_bounds;
use wasmtime_environ::{
BuiltinFunctionIndex, ComponentPC, ConstExpr, ConstOp, DataIndex, DefinedFuncIndex,
- DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex,
+ DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex, FactInlineIntrinsic,
FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex,
- IndexType, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables, Module,
- ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PassiveElemIndex, PtrSize,
- RuntimeDataIndex, Table, TableIndex, TableInitialValue, TableSegment, TableSegmentElements,
- TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,
- WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,
+ IndexType, KnownFunc, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables,
+ Module, ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder,
+ NUM_COMPONENT_CONTEXT_SLOTS, PassiveElemIndex, PtrSize, RuntimeDataIndex, Table, TableIndex,
+ TableInitialValue, TableSegment, TableSegmentElements, TagIndex, Tunables, TypeConvert,
+ TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType,
+ WasmRefType, WasmResult, WasmStorageType, WasmValType,
};
use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};
@@ -166,6 +167,12 @@ pub struct FuncEnvironment<'module_environment> {
/// Heaps implementing WebAssembly linear memories.
heaps: PrimaryMap<Heap, HeapData>,
+ /// When translating a fused sync-to-sync adapter, the explicit stack slot
+ /// holding the `VMDeferredThread` pushed by the inline `enter-sync-call`
+ /// lowering, shared with the matching inline `exit-sync-call`. `None`
+ /// outside that window.
+ fact_sync_call_slot: Option<ir::StackSlot>,
+
/// Caches of signatures for builtin functions.
builtin_functions: BuiltinFunctions,
@@ -276,6 +283,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
gc_heap: None,
heaps: PrimaryMap::default(),
+ fact_sync_call_slot: None,
builtin_functions,
offsets,
tunables,
@@ -1523,12 +1531,12 @@ impl FuncEnvironment<'_> {
let signature = self.get_or_create_interned_sig_ref(func, ty);
let key = match self.translation.known_imported_functions[func_index] {
- Some(key @ FuncKey::DefinedWasmFunction(..)) => key,
+ Some(KnownFunc::FuncKey(key @ FuncKey::DefinedWasmFunction(..))) => key,
- Some(key @ FuncKey::UnsafeIntrinsic(..)) => key,
+ Some(KnownFunc::FuncKey(key @ FuncKey::UnsafeIntrinsic(..))) => key,
- Some(key) => {
- panic!("unexpected kind of known-import function: {key:?}")
+ Some(ref f) => {
+ panic!("unexpected kind of known-import function: {f:?}")
}
None => panic!(
@@ -1872,7 +1880,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// The import is always a compile-time builtin intrinsic. Make a
// direct call to that function (presumably it will eventually be
// inlined).
- Some(FuncKey::UnsafeIntrinsic(abi, intrinsic)) => {
+ Some(KnownFunc::FuncKey(FuncKey::UnsafeIntrinsic(abi, intrinsic))) => {
let callee = self
.env
.get_or_create_imported_func_ref(self.builder.func, callee_index);
@@ -1898,13 +1906,43 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
// function, so do a direct call to that function! (Although we take
// care to still pass its `funcref`'s `vmctx` as the callee `vmctx`
// in `real_call_args` and not the caller's.)
- Some(FuncKey::DefinedWasmFunction(..)) => {
+ Some(KnownFunc::FuncKey(FuncKey::DefinedWasmFunction(..))) => {
let callee = self
.env
.get_or_create_imported_func_ref(self.builder.func, callee_index);
Ok(self.direct_call_inst(callee, &real_call_args))
}
+ // The guest-to-guest sync fast path: these adapter intrinsics are
+ // lowered inline rather than called, but only when concurrency
+ // support is enabled (the deferred thread state only exists then)
+ // and this isn't a tail call (the deferred frame must outlive the
+ // call). Otherwise fall back to the indirect call, which is also
+ // the out-of-line slow path the inline `exit` branches to.
+ Some(KnownFunc::FactIntrinsic(intrinsic)) => {
+ if self.env.tunables.concurrency_support {
+ debug_assert!(!self.tail);
+ match intrinsic {
+ FactInlineIntrinsic::EnterSyncCall => {
+ return Ok(self.lower_fact_enter_sync_call(&real_call_args));
+ }
+ FactInlineIntrinsic::ExitSyncCall => {
+ return Ok(self.lower_fact_exit_sync_call(
+ callee_index,
+ sig_ref,
+ &real_call_args,
+ ));
+ }
+ }
+ }
+ let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(
+ &mut self.builder.cursor(),
+ vmctx,
+ callee_index,
+ );
+ Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))
+ }
+
Some(key) => panic!("unexpected kind of known-import function: {key:?}"),
// Unknown import function or this module is instantiated many times
@@ -1935,6 +1973,187 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
abi == wasmtime_environ::Abi::Wasm && !self.tail && !self.env.tunables.debug_guest
}
+ /// Inline lowering of a FACT adapter's `enter-sync-call` intrinsic: push a
+ /// `VMDeferredThread` onto an explicit stack slot and publish it as the
+ /// store's current thread, deferring the heavyweight task bookkeeping the
+ /// `enter_sync_call` libcall would otherwise do eagerly.
+ ///
+ /// `real_call_args` is `[callee_vmctx, caller_vmctx, caller_instance,
+ /// callee_async, callee_instance]`.
+ fn lower_fact_enter_sync_call(&mut self, real_call_args: &[ir::Value]) -> CallRets {
+ let ptr_ty = self.env.pointer_type();
+ let ptr = self.env.offsets.ptr;
+
+ // Allocate the on-stack `VMDeferredThread`.
+ let size = u32::from(ptr.size_of_vmdeferred_thread());
+ let align_shift = u8::try_from(ptr.size().trailing_zeros()).unwrap();
+ let slot = self
+ .builder
+ .func
+ .create_sized_stack_slot(ir::StackSlotData::new(
+ ir::StackSlotKind::ExplicitSlot,
+ size,
+ align_shift,
+ ));
+ let slot_addr = self.builder.ins().stack_addr(ptr_ty, slot, 0);
+
+ let vmstore = self.env.get_vmstore_context_ptr(self.builder);
+
+ // Link the previous current thread in as this frame's parent.
+ let parent = self
+ .env
+ .alias_regions
+ .vmstore_context_current_thread(&mut self.builder.cursor(), vmstore);
+ self.env.alias_regions.store_vmdeferred_thread_parent(
+ &mut self.builder.cursor(),
+ slot_addr,
+ parent,
+ );
+
+ // Record the deferred `enter_sync_call` arguments.
+ self.env
+ .alias_regions
+ .store_vmdeferred_thread_caller_instance(
+ &mut self.builder.cursor(),
+ slot_addr,
+ real_call_args[2],
+ );
+ self.env.alias_regions.store_vmdeferred_thread_callee_async(
+ &mut self.builder.cursor(),
+ slot_addr,
+ real_call_args[3],
+ );
+ self.env
+ .alias_regions
+ .store_vmdeferred_thread_callee_instance(
+ &mut self.builder.cursor(),
+ slot_addr,
+ real_call_args[4],
+ );
+
+ // Save the caller's context slots into the frame and reset the live
+ // values to 0 for the freshly-entered (deferred) thread.
+ for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
+ let saved = self
+ .env
+ .alias_regions
+ .vmstore_context_component_context_slot(
+ &mut self.builder.cursor(),
+ ir::types::I32,
+ vmstore,
+ i,
+ );
+ self.env
+ .alias_regions
+ .store_vmdeferred_thread_saved_context(
+ &mut self.builder.cursor(),
+ slot_addr,
+ i,
+ saved,
+ );
+ let zero = self.builder.ins().iconst(ir::types::I32, 0);
+ self.env
+ .alias_regions
+ .store_vmstore_context_component_context_slot(
+ &mut self.builder.cursor(),
+ vmstore,
+ i,
+ zero,
+ );
+ }
+
+ // Publish the deferred thread as the store's current thread.
+ self.env.alias_regions.store_vmstore_context_current_thread(
+ &mut self.builder.cursor(),
+ vmstore,
+ slot_addr,
+ );
+
+ debug_assert!(self.env.fact_sync_call_slot.is_none());
+ self.env.fact_sync_call_slot = Some(slot);
+ CallRets::new()
+ }
+
+ /// Inline lowering of a FACT adapter's `exit-sync-call` intrinsic, the
+ /// counterpart to `lower_fact_enter_sync_call`. If our deferred thread is
+ /// still current (nothing forced it) we pop it and restore the caller's
+ /// context inline; otherwise we fall back to the out-of-line
+ /// `exit_sync_call` libcall.
+ fn lower_fact_exit_sync_call(
+ &mut self,
+ callee_index: FuncIndex,
+ sig_ref: ir::SigRef,
+ real_call_args: &[ir::Value],
+ ) -> CallRets {
+ let ptr_ty = self.env.pointer_type();
+
+ let slot = self
+ .env
+ .fact_sync_call_slot
+ .take()
+ .expect("inline exit-sync-call without a matching enter-sync-call");
+ let slot_addr = self.builder.ins().stack_addr(ptr_ty, slot, 0);
+ let vmstore = self.env.get_vmstore_context_ptr(self.builder);
+ let cur = self
+ .env
+ .alias_regions
+ .vmstore_context_current_thread(&mut self.builder.cursor(), vmstore);
+ let is_fast = self.builder.ins().icmp(IntCC::Equal, cur, slot_addr);
+
+ let fast_block = self.builder.create_block();
+ let slow_block = self.builder.create_block();
+ let cont_block = self.builder.create_block();
+ self.builder
+ .ins()
+ .brif(is_fast, fast_block, &[], slow_block, &[]);
+ self.builder.seal_block(fast_block);
+ self.builder.seal_block(slow_block);
+
+ // Fast path: pop the deferred thread and restore the caller's context.
+ self.builder.switch_to_block(fast_block);
+ let parent = self
+ .env
+ .alias_regions
+ .vmdeferred_thread_parent(&mut self.builder.cursor(), slot_addr);
+ self.env.alias_regions.store_vmstore_context_current_thread(
+ &mut self.builder.cursor(),
+ vmstore,
+ parent,
+ );
+ for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
+ let saved = self.env.alias_regions.vmdeferred_thread_saved_context(
+ &mut self.builder.cursor(),
+ slot_addr,
+ i,
+ );
+ self.env
+ .alias_regions
+ .store_vmstore_context_component_context_slot(
+ &mut self.builder.cursor(),
+ vmstore,
+ i,
+ saved,
+ );
+ }
+ self.builder.ins().jump(cont_block, &[]);
+
+ // Slow path: the thread was promoted to a real one, so do the
+ // equivalent out-of-line teardown via the `exit_sync_call` libcall.
+ self.builder.switch_to_block(slow_block);
+ let vmctx = self.env.vmctx_val(&mut self.builder.cursor());
+ let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(
+ &mut self.builder.cursor(),
+ vmctx,
+ callee_index,
+ );
+ self.indirect_call_inst(sig_ref, func_addr, real_call_args);
+ self.builder.ins().jump(cont_block, &[]);
+
+ self.builder.seal_block(cont_block);
+ self.builder.switch_to_block(cont_block);
+ CallRets::new()
+ }
+
/// Do a Wasm-level indirect call through the given funcref table.
pub fn indirect_call(
mut self,
diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs
index 542181e55f..d60f35b641 100644
--- a/crates/environ/src/compile/module_environ.rs
+++ b/crates/environ/src/compile/module_environ.rs
@@ -37,6 +37,39 @@ pub struct ModuleEnvironment<'a, 'data> {
tunables: &'a Tunables,
}
+/// Identifies a FACT adapter-module import that the compiler lowers inline when
+/// translating the adapter function.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum FactInlineIntrinsic {
+ /// `enter-sync-call`: push a deferred component-model thread inline.
+ EnterSyncCall,
+ /// `exit-sync-call`: pop the deferred thread inline on the fast path, or
+ /// fall back to the out-of-line `exit-sync-call` libcall when the thread
+ /// was promoted.
+ ExitSyncCall,
+}
+
+/// TODO FITZGEN
+#[derive(Clone, Debug)]
+pub enum KnownFunc {
+ /// TODO FITZGEN
+ FuncKey(FuncKey),
+ /// TODO FITZGEN
+ FactIntrinsic(FactInlineIntrinsic),
+}
+
+impl From<FuncKey> for KnownFunc {
+ fn from(key: FuncKey) -> Self {
+ Self::FuncKey(key)
+ }
+}
+
+impl From<FactInlineIntrinsic> for KnownFunc {
+ fn from(intrinsic: FactInlineIntrinsic) -> Self {
+ Self::FactIntrinsic(intrinsic)
+ }
+}
+
/// The result of translating via `ModuleEnvironment`.
///
/// Function bodies are not yet translated, and data initializers have not yet
@@ -68,8 +101,9 @@ pub struct ModuleTranslation<'data> {
/// imports table into direct calls, when possible.
///
/// When filled in, this only ever contains
- /// `FuncKey::DefinedWasmFunction(..)`s and `FuncKey::Intrinsic(..)`s.
- pub known_imported_functions: SecondaryMap<FuncIndex, Option<FuncKey>>,
+ /// `FuncKey::DefinedWasmFunction(..)`s, `FuncKey::Intrinsic(..)`s, and
+ /// `FuncKey::FactInlineIntrinsic`s.
+ pub known_imported_functions: SecondaryMap<FuncIndex, Option<KnownFunc>>,
/// A list of type signatures which are considered exported from this
/// module, or those that can possibly be called. This list is sorted, and
diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rs
index 447ec4faa8..edd81b72a3 100644
--- a/crates/environ/src/component/translate.rs
+++ b/crates/environ/src/component/translate.rs
@@ -3,9 +3,9 @@ use crate::component::dfg::AbstractInstantiations;
use crate::component::*;
use crate::prelude::*;
use crate::{
- EngineOrModuleTypeIndex, EntityIndex, FuncKey, ModuleEnvironment, ModuleInternedTypeIndex,
- ModuleTranslation, ModuleTypesBuilder, PrimaryMap, ScopeVec, TagIndex, Tunables, TypeConvert,
- WasmHeapType, WasmResult, WasmValType,
+ EngineOrModuleTypeIndex, EntityIndex, FactInlineIntrinsic, FuncKey, ModuleEnvironment,
+ ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PrimaryMap, ScopeVec, TagIndex,
+ Tunables, TypeConvert, WasmHeapType, WasmResult, WasmValType,
};
use core::str::FromStr;
use cranelift_entity::SecondaryMap;
@@ -623,12 +623,21 @@ impl<'a, 'data> Translator<'a, 'data> {
// turn them into direct calls even though we probably
// wouldn't ever inline them, but it just doesn't seem worth
// the effort.
- CoreDef::Trampoline(_) => continue,
+ //
+ // That said, a couple of adapter trampolines are lowered
+ // inline during translation. We record these here so
+ // `FuncEnvironment` recognizes them. All other trampolines
+ // remain indirect calls.
+ CoreDef::Trampoline(index) => match translation.trampolines[*index] {
+ Trampoline::EnterSyncCall => FactInlineIntrinsic::EnterSyncCall.into(),
+ Trampoline::ExitSyncCall => FactInlineIntrinsic::ExitSyncCall.into(),
+ _ => continue,
+ },
// This import is a compile-time builtin intrinsic, we
// should inline its implementation during function
// translation.
- CoreDef::UnsafeIntrinsic(i) => FuncKey::UnsafeIntrinsic(Abi::Wasm, *i),
+ CoreDef::UnsafeIntrinsic(i) => FuncKey::UnsafeIntrinsic(Abi::Wasm, *i).into(),
// This imported function is an export from another
// instance, a perfect candidate for becoming an inlinable
@@ -660,7 +669,7 @@ impl<'a, 'data> Translator<'a, 'data> {
continue;
};
- FuncKey::DefinedWasmFunction(*arg_module, arg_module_def_func)
+ FuncKey::DefinedWasmFunction(*arg_module, arg_module_def_func).into()
}
};
diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rs
index 4539ad13ec..bf1236dd90 100644
--- a/crates/environ/src/fact/trampoline.rs
+++ b/crates/environ/src/fact/trampoline.rs
@@ -790,8 +790,12 @@ impl<'a, 'b> Compiler<'a, 'b> {
// Push a task onto the current task stack.
//
- // FIXME: Apply the optimizations described in #12311.
-
+ // Note that for sync-to-sync calls, we replace this call with
+ // inline code for lazy/deferred task creation during translation to
+ // CLIF. This avoids task creation and out-of-line calls in the
+ // adapter for most sync-to-sync calls, since most sync-to-sync
+ // calls do not do anything to force the task's creation
+ // (e.g. adjust backpressure).
self.instruction(I32Const(
i32::try_from(adapter.lower.instance.as_u32()).unwrap(),
));
@@ -893,7 +897,9 @@ impl<'a, 'b> Compiler<'a, 'b> {
// lowering below may call realloc which is in the context of the
// caller's task, not the callee.
//
- // FIXME: Apply the optimizations described in #12311.
+ // Note that for sync-to-sync calls, we will emit inline code during
+ // translation to CLIF to avoid actually calling out to a libcall when
+ // the deferred task's allocation was never forced.
if self.emit_resource_call || self.module.tunables.concurrency_support {
let exit_sync_call = self.module.import_exit_sync_call();
self.instruction(Call(exit_sync_call.as_u32()));
@@ -3601,12 +3607,12 @@ impl<'a, 'b> Compiler<'a, 'b> {
///
/// The `may_leave` flag is a boolean (0 or 1) so no masking is required.
fn trap_if_not_may_leave(&mut self, flags_global: GlobalIndex, trap: Trap) -> TempLocal {
+ self.instruction(Block(BlockType::Empty));
self.instruction(GlobalGet(flags_global.as_u32()));
// Save the flag's value (known to be `true` whenever the trap below is
// not taken) into a temporary for later restoration.
let saved = self.local_tee_new_tmp(ValType::I32);
- self.instruction(I32Eqz);
- self.instruction(If(BlockType::Empty));
+ self.instruction(BrIf(0));
self.trap(trap);
self.instruction(End);
saved
diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs
index c3173b3819..80ec061049 100644
--- a/crates/environ/src/module_artifacts.rs
+++ b/crates/environ/src/module_artifacts.rs
@@ -555,6 +555,8 @@ impl CompiledFunctionsTable {
| FuncKeyKind::PatchableToBuiltinTrampoline
| FuncKeyKind::ModuleStartup => false,
+ // `FactInlineIntrinsic` never names a compiled function, so it is
+ // never classified here; group it with the other intrinsics.
#[cfg(feature = "component-model")]
FuncKeyKind::ComponentTrampoline
| FuncKeyKind::ResourceDropTrampoline
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index df1cbc516f..244ab92e39 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -279,6 +279,46 @@ pub trait PtrSize {
base + i * slot_size
}
+ /// Return the offset of the `current_thread` field of `VMStoreContext`.
+ fn vmstore_context_current_thread(&self) -> u8 {
+ self.vmstore_context_component_context_slot(0) + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4
+ }
+
+ // Offsets within `VMDeferredThread`
+
+ /// Offset of `VMDeferredThread::parent`.
+ fn vmdeferred_thread_parent(&self) -> u8 {
+ 0
+ }
+
+ /// Offset of `VMDeferredThread::caller_instance`.
+ fn vmdeferred_thread_caller_instance(&self) -> u8 {
+ self.size()
+ }
+
+ /// Offset of `VMDeferredThread::callee_async`.
+ fn vmdeferred_thread_callee_async(&self) -> u8 {
+ self.vmdeferred_thread_caller_instance() + 4
+ }
+
+ /// Offset of `VMDeferredThread::callee_instance`.
+ fn vmdeferred_thread_callee_instance(&self) -> u8 {
+ self.vmdeferred_thread_callee_async() + 4
+ }
+
+ /// Offset of `VMDeferredThread::saved_context[i]`.
+ fn vmdeferred_thread_saved_context(&self, i: u8) -> u8 {
+ assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);
+ self.vmdeferred_thread_callee_instance() + 4 + i * 4
+ }
+
+ /// Return the size of `VMDeferredThread`, rounded up to pointer alignment.
+ fn size_of_vmdeferred_thread(&self) -> u8 {
+ let unaligned =
+ self.vmdeferred_thread_callee_instance() + 4 + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4;
+ align(u32::from(unaligned), u32::from(self.size())) as u8
+ }
+
// Offsets within `VMMemoryDefinition`
/// The offset of the `base` field.
diff --git a/crates/misc/component-async-tests/tests/scenario/round_trip.rs b/crates/misc/component-async-tests/tests/scenario/round_trip.rs
index a6ecec7d29..bb2c9fee8c 100644
--- a/crates/misc/component-async-tests/tests/scenario/round_trip.rs
+++ b/crates/misc/component-async-tests/tests/scenario/round_trip.rs
@@ -461,7 +461,7 @@ pub async fn test_round_trip(
.map(|(a, b)| (String::from(*a), String::from(*b)))
.collect::<Vec<_>>(),
tx,
- });
+ })?;
store.run_concurrent(async |_| rx.await).await??;
diff --git a/crates/wasi-tls/src/p3/host.rs b/crates/wasi-tls/src/p3/host.rs
index 857072116e..66c084f415 100644
--- a/crates/wasi-tls/src/p3/host.rs
+++ b/crates/wasi-tls/src/p3/host.rs
@@ -101,7 +101,7 @@ impl<T> bindings::tls::client::HostConnectorWithStore<T> for WasiTls {
.map_err(|e| Error::from(e));
_ = send_result_tx.send(combined_result);
Ok(())
- }));
+ }))?;
let result = ResultProducer::new(getter, send_result_rx);
Ok((
@@ -167,7 +167,7 @@ impl<T> bindings::tls::client::HostConnectorWithStore<T> for WasiTls {
.map_err(|e| Error::from(e));
_ = recv_result_tx.send(combined_result);
Ok(())
- }));
+ }))?;
let result = ResultProducer::new(getter, recv_result_rx);
Ok((
diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rs
index 4f6501056b..357ff565e3 100644
--- a/crates/wasmtime/src/runtime/component/concurrent.rs
+++ b/crates/wasmtime/src/runtime/component/concurrent.rs
@@ -65,7 +65,7 @@ use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
#[cfg(feature = "gc")]
use crate::vm::GcRootsList;
use crate::vm::component::{CallContext, ComponentInstance, InstanceState};
-use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMMemoryDefinition, VMStore};
+use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore};
use crate::{
AsContext, AsContextMut, FuncType, Result, StoreContext, StoreContextMut, ValRaw, ValType, bail,
};
@@ -241,7 +241,7 @@ where
/// Spawn a background task.
///
/// See [`Accessor::spawn`] for details.
- pub fn spawn(&mut self, task: impl AccessorTask<T, D>) -> JoinHandle
+ pub fn spawn(&mut self, task: impl AccessorTask<T, D>) -> Result<JoinHandle>
where
T: 'static,
{
@@ -517,7 +517,7 @@ where
/// Panics if called within a closure provided to the [`Accessor::with`]
/// function. This can only be called outside an active invocation of
/// [`Accessor::with`].
- pub fn spawn(&self, task: impl AccessorTask<T, D>) -> JoinHandle
+ pub fn spawn(&self, task: impl AccessorTask<T, D>) -> Result<JoinHandle>
where
T: 'static,
{
@@ -570,7 +570,7 @@ where
pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
self.with(|mut access| {
let store = access.as_context_mut().0;
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut_without_forcing_current_thread();
if state.interesting_tasks == 0 {
Poll::Ready(())
} else {
@@ -723,7 +723,7 @@ impl GuestCall {
/// instance to be called has backpressure enabled
fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
let instance = store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(self.thread.task)?
.instance;
let state = store.instance_state(instance).concurrent_state();
@@ -801,8 +801,7 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
store: &mut dyn VMStore,
future: impl Future<Output = Result<R>> + Send + 'static,
) -> Result<R> {
- let state = store.concurrent_state_mut();
- let task = state.current_host_thread()?;
+ let task = store.current_host_thread()?;
// Wrap the future in a closure which will take care of stashing the result
// in `GuestTask::result` and resuming this fiber when the host task
@@ -810,7 +809,7 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
let mut future = Box::pin(async move {
let result = future.await?;
tls::get(move |store| {
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let host_state = &mut state.get_mut(task)?.state;
assert!(matches!(host_state, HostTaskState::CalleeStarted));
*host_state = HostTaskState::CalleeFinished(Box::new(result));
@@ -844,7 +843,7 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
// then use `GuestThread::sync_call_set` to wait for the task to
// complete, suspending the current fiber until it does so.
Poll::Pending => {
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
state.push_future(future);
let caller = state.get_mut(task)?.caller;
@@ -860,12 +859,12 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
// Remove the `task` from the `sync_call_set` to ensure that when
// this function returns and the task is deleted that there are no
// more lingering references to this host task.
- Waitable::Host(task).join(store.concurrent_state_mut(), None)?;
+ Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
}
}
// Retrieve and return the result.
- let host_state = &mut store.concurrent_state_mut().get_mut(task)?.state;
+ let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
Ok(result) => *result,
@@ -886,7 +885,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
Some(pair) => pair,
None => bail_bug!("delivering non-present event"),
};
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let task = state.get_mut(call.thread.task)?;
let runtime_instance = task.instance;
let handle = waitable.map(|(_, v)| v).unwrap_or(0);
@@ -905,7 +904,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
store.enter_instance(runtime_instance);
let Some(callback) = store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(call.thread.task)?
.callback
.take()
@@ -916,7 +915,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
let code = callback(store, event, handle)?;
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(call.thread.task)?
.callback = Some(callback);
@@ -971,7 +970,7 @@ impl<T> Store<T> {
}
/// Convenience wrapper for [`StoreContextMut::spawn`].
- pub fn spawn(&mut self, task: impl AccessorTask<T, HasSelf<T>>) -> JoinHandle
+ pub fn spawn(&mut self, task: impl AccessorTask<T, HasSelf<T>>) -> Result<JoinHandle>
where
T: 'static,
{
@@ -997,7 +996,7 @@ impl<T> StoreContextMut<'_, T> {
.store_data_mut()
.components
.assert_instance_states_empty();
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut().unwrap();
assert!(
state.table.get_mut().is_empty(),
"non-empty table: {:?}",
@@ -1005,7 +1004,7 @@ impl<T> StoreContextMut<'_, T> {
);
assert!(state.high_priority.is_empty());
assert!(state.low_priority.is_empty());
- assert!(state.current_thread.is_none());
+ assert!(state.unforced_current_thread.is_none());
assert!(state.futures_mut().unwrap().is_empty());
assert!(state.global_error_context_ref_counts.is_empty());
}
@@ -1018,6 +1017,7 @@ impl<T> StoreContextMut<'_, T> {
pub fn concurrent_state_table_size(&mut self) -> usize {
self.0
.concurrent_state_mut()
+ .unwrap()
.table
.get_mut()
.iter_mut()
@@ -1033,7 +1033,7 @@ impl<T> StoreContextMut<'_, T> {
/// for this instance is run.
///
/// The returned [`JoinHandle`] may be used to cancel the task.
- pub fn spawn(mut self, task: impl AccessorTask<T>) -> JoinHandle
+ pub fn spawn(mut self, task: impl AccessorTask<T>) -> Result<JoinHandle>
where
T: 'static,
{
@@ -1047,7 +1047,7 @@ impl<T> StoreContextMut<'_, T> {
self,
accessor: Accessor<T, D>,
task: impl AccessorTask<T, D>,
- ) -> JoinHandle
+ ) -> Result<JoinHandle>
where
T: 'static,
D: HasData + ?Sized,
@@ -1057,9 +1057,9 @@ impl<T> StoreContextMut<'_, T> {
// iteration.
let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
self.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
- handle
+ Ok(handle)
}
/// Run the specified closure `fun` to completion as part of this store's
@@ -1231,7 +1231,12 @@ impl<T> StoreContextMut<'_, T> {
impl<'a, T> Drop for Reset<'a, T> {
fn drop(&mut self) {
if let Some(futures) = self.futures.take() {
- *self.store.0.concurrent_state_mut().futures.get_mut() = Some(futures);
+ *self
+ .store
+ .0
+ .concurrent_state_mut_already_forced_current_thread()
+ .futures
+ .get_mut() = Some(futures);
}
}
}
@@ -1240,7 +1245,7 @@ impl<T> StoreContextMut<'_, T> {
// Take `ConcurrentState::futures` out of the store so we can poll
// it while also safely giving any of the futures inside access to
// `self`.
- let futures = self.0.concurrent_state_mut().futures.get_mut().take();
+ let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
let mut reset = Reset {
store: self.as_context_mut(),
futures,
@@ -1283,7 +1288,7 @@ impl<T> StoreContextMut<'_, T> {
// Next, collect the next batch of work items to process, if
// any. This will be either all of the high-priority work
// items, or if there are none, a single low-priority work item.
- let state = reset.store.0.concurrent_state_mut();
+ let state = reset.store.0.concurrent_state_mut()?;
let mut ready = mem::take(&mut state.high_priority);
let mut low_priority = false;
if ready.is_empty() {
@@ -1434,7 +1439,7 @@ impl<T> StoreContextMut<'_, T> {
match item {
WorkItem::PushFuture(future) => {
self.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.futures_mut()?
.push(future.into_inner());
}
@@ -1443,7 +1448,7 @@ impl<T> StoreContextMut<'_, T> {
}
WorkItem::ResumeThread(_, thread) => {
if let GuestThreadState::Ready { fiber, .. } = mem::replace(
- &mut self.0.concurrent_state_mut().get_mut(thread.thread)?.state,
+ &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state,
GuestThreadState::Running,
) {
self.0.resume_fiber(fiber).await?;
@@ -1455,7 +1460,7 @@ impl<T> StoreContextMut<'_, T> {
if call.is_ready(self.0)? {
self.run_on_worker(WorkerItem::GuestCall(call)).await?;
} else {
- let state = self.0.concurrent_state_mut();
+ let state = self.0.concurrent_state_mut()?;
let task = state.get_mut(call.thread.task)?;
if !task.starting_sent {
task.starting_sent = true;
@@ -1490,12 +1495,12 @@ impl<T> StoreContextMut<'_, T> {
where
T: Send,
{
- let worker = if let Some(fiber) = self.0.concurrent_state_mut().worker.take() {
+ let worker = if let Some(fiber) = self.0.concurrent_state_mut()?.worker.take() {
fiber
} else {
fiber::make_fiber(self.0, move |store| {
loop {
- let Some(item) = store.concurrent_state_mut().worker_item.take() else {
+ let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
bail_bug!("worker_item not present when resuming fiber")
};
match item {
@@ -1508,7 +1513,7 @@ impl<T> StoreContextMut<'_, T> {
})?
};
- let worker_item = &mut self.0.concurrent_state_mut().worker_item;
+ let worker_item = &mut self.0.concurrent_state_mut()?.worker_item;
assert!(worker_item.is_none());
*worker_item = Some(item);
@@ -1544,10 +1549,10 @@ impl<T> StoreContextMut<'_, T> {
/// Tasks are yielded "youngest first" where the first item in the iterator
/// is the current task, and the last item in the iterator is the original
/// call.
- pub fn async_call_stack(&mut self) -> impl Iterator<Item = GuestTaskId> {
- let state = self.0.concurrent_state_mut();
- let mut cur = Some(state.current_thread);
- core::iter::from_fn(move || {
+ pub fn async_call_stack(&mut self) -> Result<impl Iterator<Item = GuestTaskId>> {
+ let mut cur = Some(self.0.current_thread()?);
+ let state = self.0.concurrent_state_mut()?;
+ Ok(core::iter::from_fn(move || {
while let Some(t) = cur {
cur = state.parent(t);
if let Some(thread) = t.guest() {
@@ -1556,17 +1561,132 @@ impl<T> StoreContextMut<'_, T> {
}
None
- })
+ }))
}
}
impl StoreOpaque {
+ /// Returns the currently-running thread, promoting any deferred lazy thread
+ /// into a fully-materialized `CurrentThread`.
+ pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
+ // Without concurrency support there is nothing to force.
+ if !self.concurrency_support() {
+ return Ok(CurrentThread::None);
+ }
+
+ // If the JIT-visible current thread isn't a deferred thread then
+ // `ConcurrentState` is already up to date.
+ if !self
+ .vm_store_context_mut()
+ .current_thread_mut()
+ .is_deferred()
+ {
+ return Ok(self
+ .concurrent_state_mut_already_forced_current_thread()
+ .unforced_current_thread);
+ }
+
+ // The component instance whose adapters pushed the deferred frames; all
+ // frames in a guest-to-guest, sync-to-sync call chain of fused adapters
+ // live within a single `wasmtime::component::Instance` (because
+ // cross-`wasmtime::component::Instance` calls don't go through fused
+ // adapters), and guest code only ever runs as a guest thread, so the
+ // chain's base thread is already materialized in `ConcurrentState` and
+ // we can get the `ComponentInstanceId` shared by the whole chain from
+ // here.
+ let state = self.concurrent_state_mut_without_forcing_current_thread();
+ let id = match state.unforced_current_thread.guest().copied() {
+ Some(thread) => state.get_mut(thread.task)?.instance.instance,
+ None => bail_bug!("deferred component-model thread with non-guest base"),
+ };
+
+ // Collect the deferred frames pushed inline by fused adapters, walking
+ // the `parent` chain from innermost to the base.
+ let mut frames = Vec::new();
+ let mut cur = *self.vm_store_context_mut().current_thread_mut();
+ while let Some(ptr) = cur.as_deferred() {
+ // SAFETY: `ptr` points at a `VMDeferredThread` living in a fused
+ // adapter's stack frame that is suspended below us on the stack
+ // (mid-call, waiting for this nested call to return), so the
+ // referent is still valid and exclusively ours to read.
+ let deferred = unsafe { ptr.as_non_null().as_ref() };
+ frames.push((
+ deferred.callee_async != 0,
+ deferred.callee_instance,
+ deferred.saved_context,
+ ));
+ cur = deferred.parent;
+ }
+
+ // Mark the current thread forced *before* replaying so that any
+ // reentrant `force_current_thread` call short-circuits via the
+ // non-deferred path above.
+ *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
+
+ // Save the current context, as we need to overwrite it while replaying
+ // below.
+ let current_context = *self.vm_store_context_mut().component_context_mut();
+
+ // Replay the deferred `enter_guest_sync_call`s outermost-first so that
+ // the resulting `ConcurrentState` matches what the non-deferred path
+ // would have otherwise produced.
+ for (callee_async, callee_instance, saved_context) in frames.into_iter().rev() {
+ // Restore the caller's context slots so that we save the correct
+ // values into the caller's thread, exactly as the non-deferred path
+ // would have on entry.
+ *self.vm_store_context_mut().component_context_mut() = saved_context;
+ let callee = RuntimeInstance {
+ instance: id,
+ index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
+ };
+ self.enter_guest_sync_call(None, callee_async, callee)?;
+ }
+
+ // Replaying done; restore the current context.
+ *self.vm_store_context_mut().component_context_mut() = current_context;
+
+ Ok(self
+ .concurrent_state_mut_without_forcing_current_thread()
+ .unforced_current_thread)
+ }
+
+ fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
+ match self.current_thread()?.guest() {
+ Some(id) => Ok(*id),
+ None => bail_bug!("current thread is not a guest thread"),
+ }
+ }
+
+ fn current_host_thread(&mut self) -> Result<TableId<HostTask>> {
+ match self.current_thread()?.host() {
+ Some(id) => Ok(id),
+ None => bail_bug!("current thread is not a host thread"),
+ }
+ }
+
+ /// Returns whether there's a pending cancellation on the current guest thread,
+ /// consuming the event if so.
+ fn take_pending_cancellation(&mut self) -> Result<bool> {
+ let thread = self.current_guest_thread()?;
+ let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
+ if let Some(Event::Cancelled) = task.event {
+ task.event.take();
+ return Ok(true);
+ }
+ Ok(false)
+ }
+
/// Push a `GuestTask` onto the task stack for either a sync-to-sync,
/// guest-to-guest call or a sync host-to-guest call.
///
/// This task will only be used for the purpose of handling calls to
/// intrinsic functions; both parameter lowering and result lifting are
/// assumed to be taken care of elsewhere.
+ ///
+ /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in
+ /// fused adapters, see `StoreOpaque::current_thread`, `VMDeferredThread`,
+ /// and `lower_fact_enter_sync_call`. Make sure all this stuff stays in
+ /// sync!
pub(crate) fn enter_guest_sync_call(
&mut self,
guest_caller: Option<RuntimeInstance>,
@@ -1578,8 +1698,8 @@ impl StoreOpaque {
return self.enter_call_not_concurrent();
}
- let state = self.concurrent_state_mut();
- let thread = state.current_thread;
+ let thread = self.current_thread()?;
+ let state = self.concurrent_state_mut()?;
let instance = if let Some(thread) = thread.guest() {
Some(state.get_mut(thread.task)?.instance)
} else {
@@ -1622,6 +1742,12 @@ impl StoreOpaque {
}
/// Pop a `GuestTask` previously pushed using `enter_sync_call`.
+ ///
+ /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in
+ /// fused adapters and then when the call returns we check to see if the
+ /// task's contruction was forced and if not avoid calling out of the JIT
+ /// code to this function. See `lower_fact_exit_sync_call`. Make sure all
+ /// this stuff stays in sync!
pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
if !self.concurrency_support() {
return Ok(self.exit_call_not_concurrent());
@@ -1630,7 +1756,7 @@ impl StoreOpaque {
Some(t) => *t,
None => bail_bug!("expected task when exiting"),
};
- let task = self.concurrent_state_mut().get_mut(thread.task)?;
+ let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
let instance = task.instance;
let caller = match &task.caller {
&Caller::Guest { thread } => thread.into(),
@@ -1658,8 +1784,8 @@ impl StoreOpaque {
self.enter_call_not_concurrent()?;
return Ok(None);
}
- let state = self.concurrent_state_mut();
- let caller = state.current_guest_thread()?;
+ let caller = self.current_guest_thread()?;
+ let state = self.concurrent_state_mut()?;
let task = state.push(HostTask::new(caller, HostTaskState::CalleeStarted))?;
log::trace!("new host task {task:?}");
self.set_thread(task)?;
@@ -1675,8 +1801,8 @@ impl StoreOpaque {
if !self.concurrency_support() {
return Ok(());
}
- let task = self.concurrent_state_mut().current_host_thread()?;
- let caller = self.concurrent_state_mut().get_mut(task)?.caller;
+ let task = self.current_host_thread()?;
+ let caller = self.concurrent_state_mut()?.get_mut(task)?.caller;
self.set_thread(caller)?;
Ok(())
}
@@ -1691,7 +1817,7 @@ impl StoreOpaque {
match task {
Some(task) => {
log::trace!("delete host task {task:?}");
- self.concurrent_state_mut().delete(task)?;
+ self.concurrent_state_mut()?.delete(task)?;
}
None => {
self.exit_call_not_concurrent();
@@ -1714,8 +1840,8 @@ impl StoreOpaque {
if !self.concurrency_support() {
return Ok(true);
}
- let state = self.concurrent_state_mut();
- let mut cur = Some(state.current_thread);
+ let mut cur = Some(self.current_thread()?);
+ let state = self.concurrent_state_mut()?;
while let Some(t) = cur {
if let Some(thread) = t.guest() {
let task = state.get_mut(thread.task)?;
@@ -1748,8 +1874,8 @@ impl StoreOpaque {
/// needed too.
fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
let thread = thread.into();
- let state = self.concurrent_state_mut();
- let old_thread = mem::replace(&mut state.current_thread, thread);
+ let state = self.concurrent_state_mut()?;
+ let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
// First thing to do after swapping threads is updating the context
// slots for this thread within the store. This restores the behavior of
@@ -1759,21 +1885,22 @@ impl StoreOpaque {
// this also leaves behind sentinel values to try to uncover bugs where
// this may be forgotten.
if let Some(old_thread) = old_thread.guest() {
- let old_context = self.vm_store_context().component_context;
- self.concurrent_state_mut()
+ let old_context = *self.vm_store_context_mut().component_context_mut();
+ self.concurrent_state_mut()?
.get_mut(old_thread.thread)?
.context = old_context;
}
if cfg!(debug_assertions) {
- self.vm_store_context_mut().component_context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
+ *self.vm_store_context_mut().component_context_mut() =
+ [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
}
if let Some(thread) = thread.guest() {
- let thread = self.concurrent_state_mut().get_mut(thread.thread)?;
+ let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
let context = thread.context;
if cfg!(debug_assertions) {
thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
}
- self.vm_store_context_mut().component_context = context;
+ *self.vm_store_context_mut().component_context_mut() = context;
}
// Each time we switch threads, we conservatively set `task_may_block`
@@ -1783,7 +1910,7 @@ impl StoreOpaque {
//
// Additionally if we're switching to a new thread, set its component
// instance's `task_may_block` according to where it left off.
- let state = self.concurrent_state_mut();
+ let state = self.concurrent_state_mut()?;
if let Some(old_thread) = old_thread.guest() {
let instance = state.get_mut(old_thread.task)?.instance.instance;
self.component_instance_mut(instance)
@@ -1794,16 +1921,23 @@ impl StoreOpaque {
self.set_task_may_block()?;
}
+ // Keep the JIT-visible current-thread pointer in sync.
+ *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
+ VMLazyThread::none()
+ } else {
+ VMLazyThread::forced()
+ };
+
Ok(old_thread)
}
/// Set the global variable representing whether the current task may block
/// prior to entering Wasm code.
fn set_task_may_block(&mut self) -> Result<()> {
- let state = self.concurrent_state_mut();
- let guest_thread = state.current_guest_thread()?;
+ let guest_thread = self.current_guest_thread()?;
+ let state = self.concurrent_state_mut()?;
let instance = state.get_mut(guest_thread.task)?.instance.instance;
- let may_block = self.concurrent_state_mut().may_block(guest_thread.task)?;
+ let may_block = self.concurrent_state_mut()?.may_block(guest_thread.task)?;
self.component_instance_mut(instance)
.set_task_may_block(may_block);
Ok(())
@@ -1813,8 +1947,8 @@ impl StoreOpaque {
if !self.concurrency_support() {
return Ok(());
}
- let state = self.concurrent_state_mut();
- let task = state.current_guest_thread()?.task;
+ let task = self.current_guest_thread()?.task;
+ let state = self.concurrent_state_mut()?;
let instance = state.get_mut(task)?.instance.instance;
let task_may_block = self.component_instance(instance).get_task_may_block();
@@ -1856,7 +1990,7 @@ impl StoreOpaque {
{
let call = GuestCall { thread, kind };
if call.is_ready(self)? {
- self.concurrent_state_mut()
+ self.concurrent_state_mut()?
.push_high_priority(WorkItem::GuestCall(instance.index, call));
} else {
self.instance_state(instance)
@@ -1892,14 +2026,14 @@ impl StoreOpaque {
/// Resume the specified fiber, giving it exclusive access to the specified
/// store.
async fn resume_fiber(&mut self, fiber: StoreFiber<'static>) -> Result<()> {
- let old_thread = self.concurrent_state_mut().current_thread;
+ let old_thread = self.current_thread()?;
log::trace!("resume_fiber: save current thread {old_thread:?}");
let fiber = fiber::resolve_or_release(self, fiber).await?;
self.set_thread(old_thread)?;
- let state = self.concurrent_state_mut();
+ let state = self.concurrent_state_mut()?;
if let Some(ot) = old_thread.guest() {
state.get_mut(ot.thread)?.state = GuestThreadState::Running;
@@ -1968,7 +2102,7 @@ impl StoreOpaque {
};
let old_guest_thread = if task.is_some() {
- self.concurrent_state_mut().current_thread
+ self.current_thread()?
} else {
CurrentThread::None
};
@@ -1993,12 +2127,12 @@ impl StoreOpaque {
}
) || old_guest_thread
.guest()
- .map(|thread| self.concurrent_state_mut().may_block(thread.task))
+ .map(|thread| self.concurrent_state_mut()?.may_block(thread.task))
.transpose()?
.unwrap_or(true)
);
- let suspend_reason = &mut self.concurrent_state_mut().suspend_reason;
+ let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
assert!(suspend_reason.is_none());
*suspend_reason = Some(reason);
@@ -2012,13 +2146,13 @@ impl StoreOpaque {
}
fn wait_for_event(&mut self, waitable: Waitable) -> Result<()> {
- let state = self.concurrent_state_mut();
+ let caller = self.current_guest_thread()?;
+ let state = self.concurrent_state_mut()?;
if waitable.common(state)?.set.is_some() {
bail!(Trap::WaitableSyncAndAsync);
}
- let caller = state.current_guest_thread()?;
let set = state.get_mut(caller.thread)?.sync_call_set;
waitable.join(state, Some(set))?;
self.suspend(SuspendReason::Waiting {
@@ -2026,7 +2160,7 @@ impl StoreOpaque {
thread: caller,
skip_may_block_check: false,
})?;
- let state = self.concurrent_state_mut();
+ let state = self.concurrent_state_mut()?;
waitable.join(state, None)
}
@@ -2057,7 +2191,7 @@ impl StoreOpaque {
runtime_instance: RuntimeInstance,
cleanup_task: CleanupTask,
) -> Result<()> {
- let state = self.concurrent_state_mut();
+ let state = self.concurrent_state_mut()?;
let thread_data = state.get_mut(guest_thread.thread)?;
let sync_call_set = thread_data.sync_call_set;
if let Some(guest_id) = thread_data.instance_rep {
@@ -2065,7 +2199,7 @@ impl StoreOpaque {
.thread_handle_table()
.guest_thread_remove(guest_id)?;
}
- let state = self.concurrent_state_mut();
+ let state = self.concurrent_state_mut()?;
// Clean up any pending subtasks in the sync_call_set
for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
@@ -2128,7 +2262,7 @@ impl StoreOpaque {
caller_instance: RuntimeInstance,
guest_task: TableId<GuestTask>,
) -> Result<()> {
- let concurrent_state = self.concurrent_state_mut();
+ let concurrent_state = self.concurrent_state_mut()?;
let task = concurrent_state.get_mut(guest_task)?;
assert!(!task.already_lowered_parameters());
// The task is in a `starting` state, meaning it hasn't run at
@@ -2179,7 +2313,7 @@ impl Instance {
set: Option<TableId<WaitableSet>>,
cancellable: bool,
) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let event = &mut state.get_mut(guest_task)?.event;
if let Some(ev) = event
@@ -2235,7 +2369,7 @@ impl Instance {
log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
let set = store
@@ -2249,7 +2383,7 @@ impl Instance {
Ok(match code {
callback_code::EXIT => {
log::trace!("implicit thread {guest_thread:?} completed");
- let task = store.concurrent_state_mut().get_mut(guest_thread.task)?;
+ let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
task.exited = true;
task.callback = None;
store.cleanup_thread(
@@ -2295,7 +2429,7 @@ impl Instance {
state.check_blocking_for(guest_thread.task)?;
let set = get_set(store, set)?;
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
if state.get_mut(guest_thread.task)?.event.is_some()
|| !state.get_mut(set)?.ready.is_empty()
@@ -2387,10 +2521,10 @@ impl Instance {
let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(guest_thread.thread)?
.state = GuestThreadState::Running;
- let task = store.concurrent_state_mut().get_mut(guest_thread.task)?;
+ let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
let lower = match task.lower_params.take() {
Some(l) => l,
None => bail_bug!("lower_params missing"),
@@ -2433,7 +2567,7 @@ impl Instance {
let callee_instance = store
.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(guest_thread.task)?
.instance;
@@ -2464,7 +2598,7 @@ impl Instance {
store.exit_instance(callee_instance)?;
store.set_thread(old_thread)?;
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
if let Some(t) = old_thread.guest() {
state.get_mut(t.thread)?.state = GuestThreadState::Running;
}
@@ -2515,7 +2649,7 @@ impl Instance {
let lift = {
store.exit_instance(callee_instance)?;
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
if !state.get_mut(guest_thread.task)?.result.is_none() {
bail_bug!("task has already produced a result");
}
@@ -2557,7 +2691,7 @@ impl Instance {
store.set_thread(old_thread)?;
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(guest_thread.task)?
.exited = true;
@@ -2569,7 +2703,7 @@ impl Instance {
store
.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.push_high_priority(WorkItem::GuestCall(
callee_instance.index,
GuestCall {
@@ -2653,8 +2787,8 @@ impl Instance {
let start = SendSyncPtr::new(start);
let return_ = SendSyncPtr::new(return_);
let token = StoreToken::new(store.as_context_mut());
- let state = store.0.concurrent_state_mut();
- let old_thread = state.current_guest_thread()?;
+ let old_thread = store.0.current_guest_thread()?;
+ let state = store.0.concurrent_state_mut()?;
debug_assert_eq!(
state.get_mut(old_thread.task)?.instance,
@@ -2705,8 +2839,9 @@ impl Instance {
)?;
}
dst.copy_from_slice(&src[..dst.len()]);
- let state = store.0.concurrent_state_mut();
- Waitable::Guest(state.current_guest_thread()?.task).set_event(
+ let task = store.0.current_guest_thread()?.task;
+ let state = store.0.concurrent_state_mut()?;
+ Waitable::Guest(task).set_event(
state,
Some(Event::Subtask {
status: Status::Started,
@@ -2749,8 +2884,8 @@ impl Instance {
// lifting/lowering has returned.
store.0.set_thread(prev)?;
- let state = store.0.concurrent_state_mut();
- let thread = state.current_guest_thread()?;
+ let thread = store.0.current_guest_thread()?;
+ let state = store.0.concurrent_state_mut()?;
if sync_caller {
state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
if let ResultInfo::Stack { result_count } = &result_info {
@@ -2840,8 +2975,8 @@ impl Instance {
) -> Result<u32> {
let token = StoreToken::new(store.as_context_mut());
let async_caller = storage.is_none();
- let state = store.0.concurrent_state_mut();
- let guest_thread = state.current_guest_thread()?;
+ let guest_thread = store.0.current_guest_thread()?;
+ let state = store.0.concurrent_state_mut()?;
let callee_async = state.get_mut(guest_thread.task)?.async_function;
let callee = SendSyncPtr::new(callee);
let param_count = usize::try_from(param_count)?;
@@ -2883,7 +3018,7 @@ impl Instance {
)?;
}
- let state = store.0.concurrent_state_mut();
+ let state = store.0.concurrent_state_mut()?;
// Use the caller's `GuestThread::sync_call_set` to register interest in
// the subtask...
@@ -2923,7 +3058,7 @@ impl Instance {
skip_may_block_check: async_caller || !callee_async,
})?;
- let state = store.0.concurrent_state_mut();
+ let state = store.0.concurrent_state_mut()?;
log::trace!("taking event for {:?}", guest_thread.task);
let event = guest_waitable.take_event(state)?;
@@ -2947,7 +3082,7 @@ impl Instance {
.subtask_insert_guest(guest_thread.task.rep())?;
store
.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(guest_thread.task)?
.common
.handle = Some(handle);
@@ -2959,18 +3094,22 @@ impl Instance {
}
};
- guest_waitable.join(store.0.concurrent_state_mut(), old_set)?;
+ guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
// Reset the current thread to point to the caller as it resumes control.
store.0.set_thread(caller)?;
- store.0.concurrent_state_mut().get_mut(caller.thread)?.state = GuestThreadState::Running;
+ store
+ .0
+ .concurrent_state_mut()?
+ .get_mut(caller.thread)?
+ .state = GuestThreadState::Running;
log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
if let Some(storage) = storage {
// The caller used a sync-lowered import to call an async-lifted
// export, in which case the result, if any, has been stashed in
// `GuestTask::sync_result`.
- let state = store.0.concurrent_state_mut();
+ let state = store.0.concurrent_state_mut()?;
let task = state.get_mut(guest_thread.task)?;
if let Some(result) = task.sync_result.take()? {
if let Some(result) = result {
@@ -3005,8 +3144,8 @@ impl Instance {
lower: impl FnOnce(StoreContextMut<T>, Option<R>) -> Result<()> + Send + 'static,
) -> Result<Option<u32>> {
let token = StoreToken::new(store.as_context_mut());
- let state = store.0.concurrent_state_mut();
- let task = state.current_host_thread()?;
+ let task = store.0.current_host_thread()?;
+ let state = store.0.concurrent_state_mut()?;
// Create an abortable future which hooks calls to poll and manages call
// context state for the future.
@@ -3067,7 +3206,7 @@ impl Instance {
};
lower(store.as_context_mut(), result)?;
- let state = store.0.concurrent_state_mut();
+ let state = store.0.concurrent_state_mut()?;
match &mut state.get_mut(task)?.state {
// The task is already flagged as finished because it was
// cancelled. No need to transition further.
@@ -3088,7 +3227,7 @@ impl Instance {
// there are host embedder frames on the stack is unsound.
tls::get(move |store| {
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
on_complete,
))));
@@ -3098,7 +3237,7 @@ impl Instance {
// Make this task visible to the guest and then record what it
// was made visible as.
- let state = store.0.concurrent_state_mut();
+ let state = store.0.concurrent_state_mut()?;
state.push_future(future);
let caller = state.get_mut(task)?.caller;
let instance = state.get_mut(caller.task)?.instance;
@@ -3107,7 +3246,7 @@ impl Instance {
.instance_state(instance)
.handle_table()
.subtask_insert_host(task.rep())?;
- store.0.concurrent_state_mut().get_mut(task)?.common.handle = Some(handle);
+ store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
// Restore the currently running thread to this host task's
@@ -3126,8 +3265,8 @@ impl Instance {
options: OptionsIndex,
storage: &[ValRaw],
) -> Result<()> {
- let state = store.concurrent_state_mut();
- let guest_thread = state.current_guest_thread()?;
+ let guest_thread = store.current_guest_thread()?;
+ let state = store.concurrent_state_mut()?;
let lift = state
.get_mut(guest_thread.task)?
.lift_result
@@ -3172,8 +3311,8 @@ impl Instance {
/// Implements the `task.cancel` intrinsic.
pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
- let state = store.concurrent_state_mut();
- let guest_thread = state.current_guest_thread()?;
+ let guest_thread = store.current_guest_thread()?;
+ let state = store.concurrent_state_mut()?;
let task = state.get_mut(guest_thread.task)?;
if !task.cancel_sent {
bail!(Trap::TaskCancelNotCancelled);
@@ -3210,10 +3349,10 @@ impl Instance {
status: Status,
) -> Result<()> {
store
- .component_resource_tables(Some(self))
+ .component_resource_tables(Some(self))?
.validate_scope_exit()?;
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let task = state.get_mut(guest_task)?;
if let Caller::Host { tx, .. } = &mut task.caller {
@@ -3234,7 +3373,7 @@ impl Instance {
store: &mut StoreOpaque,
caller_instance: RuntimeComponentInstanceIndex,
) -> Result<u32> {
- let set = store.concurrent_state_mut().push(WaitableSet::default())?;
+ let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
let handle = store
.instance_state(self.runtime_instance(caller_instance))
.handle_table()
@@ -3261,7 +3400,7 @@ impl Instance {
// set to avoid dropping any waiters in `WaitMode::Fiber(_)`, which
// would panic. See `drop-waitable-set-with-waiters.wast` for details.
if !store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(TableId::<WaitableSet>::new(rep))?
.waiting
.is_empty()
@@ -3270,7 +3409,7 @@ impl Instance {
}
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.delete(TableId::<WaitableSet>::new(rep))?;
Ok(())
@@ -3295,7 +3434,7 @@ impl Instance {
.handle_table()
.waitable_set_rep(set_handle)?;
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
if let Some(old) = waitable.common(state)?.set
&& state.get_mut(old)?.is_sync_call_set
{
@@ -3309,7 +3448,7 @@ impl Instance {
"waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
);
- waitable.join(store.concurrent_state_mut(), set)
+ waitable.join(store.concurrent_state_mut()?, set)
}
/// Implements the `subtask.drop` intrinsic.
@@ -3326,7 +3465,7 @@ impl Instance {
.handle_table()
.subtask_remove(task_id)?;
- let concurrent_state = store.concurrent_state_mut();
+ let concurrent_state = store.concurrent_state_mut()?;
let (waitable, delete) = if is_host {
let id = TableId::<HostTask>::new(rep);
let task = concurrent_state.get_mut(id)?;
@@ -3435,9 +3574,9 @@ impl Instance {
/// Implements the `thread.index` intrinsic.
pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
- let thread_id = store.concurrent_state_mut().current_guest_thread()?.thread;
+ let thread_id = store.current_guest_thread()?.thread;
match store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(thread_id)?
.instance_rep
{
@@ -3489,7 +3628,7 @@ impl Instance {
CleanupTask::Yes,
)?;
log::trace!("explicit thread {guest_thread:?} completed");
- let state = store.0.concurrent_state_mut();
+ let state = store.0.concurrent_state_mut()?;
if let Some(t) = old_thread.guest() {
state.get_mut(t.thread)?.state = GuestThreadState::Running;
}
@@ -3499,8 +3638,8 @@ impl Instance {
},
);
- let state = store.0.concurrent_state_mut();
- let current_thread = state.current_guest_thread()?;
+ let current_thread = store.0.current_guest_thread()?;
+ let state = store.0.concurrent_state_mut()?;
let parent_task = current_thread.task;
let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
@@ -3522,7 +3661,7 @@ impl Instance {
) -> Result<()> {
let thread_id =
GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
let thread = state.get_mut(guest_thread.thread)?;
@@ -3539,20 +3678,20 @@ impl Instance {
},
);
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.push_work_item(guest_call, high_priority);
}
GuestThreadState::Suspended(fiber) => {
log::trace!("resuming thread {thread_id:?} that was suspended");
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.push_work_item(WorkItem::ResumeFiber(fiber), high_priority);
}
GuestThreadState::Ready { fiber, cancellable } if allow_ready => {
log::trace!("resuming thread {thread_id:?} that was ready");
thread.state = GuestThreadState::Ready { fiber, cancellable };
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.promote_thread_work_item(guest_thread);
}
other => {
@@ -3574,7 +3713,7 @@ impl Instance {
.thread_handle_table()
.guest_thread_insert(thread_id.rep())?;
store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(thread_id)?
.instance_rep = Some(guest_id);
Ok(guest_id)
@@ -3590,9 +3729,9 @@ impl Instance {
yielding: bool,
to_thread: SuspensionTarget,
) -> Result<WaitResult> {
- let guest_thread = store.concurrent_state_mut().current_guest_thread()?;
+ let guest_thread = store.current_guest_thread()?;
if to_thread.is_none() {
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
if yielding {
// This is a `thread.yield` call
if !state.may_block(guest_thread.task)? {
@@ -3612,7 +3751,7 @@ impl Instance {
}
// There could be a pending cancellation from a previous uncancellable wait
- if cancellable && store.concurrent_state_mut().take_pending_cancellation()? {
+ if cancellable && store.take_pending_cancellation()? {
return Ok(WaitResult::Cancelled);
}
@@ -3647,7 +3786,7 @@ impl Instance {
store.suspend(reason)?;
- if cancellable && store.concurrent_state_mut().take_pending_cancellation()? {
+ if cancellable && store.take_pending_cancellation()? {
Ok(WaitResult::Cancelled)
} else {
Ok(WaitResult::Completed)
@@ -3662,11 +3801,11 @@ impl Instance {
check: WaitableCheck,
params: WaitableCheckParams,
) -> Result<u32> {
- let guest_thread = store.concurrent_state_mut().current_guest_thread()?;
+ let guest_thread = store.current_guest_thread()?;
log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let task = state.get_mut(guest_thread.task)?;
// If we're waiting, and there are no events immediately available,
@@ -3768,7 +3907,7 @@ impl Instance {
} else {
Waitable::Guest(TableId::<GuestTask>::new(rep))
};
- let concurrent_state = store.concurrent_state_mut();
+ let concurrent_state = store.concurrent_state_mut()?;
log::trace!("subtask_cancel {waitable:?} (handle {task_id})");
@@ -3848,7 +3987,7 @@ impl Instance {
};
concurrent_state.push_high_priority(item);
- let caller = concurrent_state.current_guest_thread()?;
+ let caller = store.current_guest_thread()?;
store.suspend(SuspendReason::Yielding {
thread: caller,
cancellable: false,
@@ -3863,8 +4002,8 @@ impl Instance {
{
// The thread is in a cancellable yield, so yield back
// to it.
- let caller = concurrent_state.current_guest_thread()?;
concurrent_state.promote_thread_work_item(thread);
+ let caller = store.current_guest_thread()?;
store.suspend(SuspendReason::Yielding {
thread: caller,
cancellable: false,
@@ -3877,7 +4016,7 @@ impl Instance {
// Guest tasks need to block if they have not yet returned or
// cancelled, even as a result of the event delivery above.
needs_block = !store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(guest_task)?
.returned_or_cancelled()
} else {
@@ -3901,7 +4040,7 @@ impl Instance {
// .. fall through to determine what event's in store for us.
}
- let event = waitable.take_event(store.concurrent_state_mut())?;
+ let event = waitable.take_event(store.concurrent_state_mut()?)?;
if let Some(Event::Subtask {
status: status @ (Status::Returned | Status::ReturnCancelled),
}) = event
@@ -5045,7 +5184,11 @@ impl From<TableId<HostTask>> for CurrentThread {
/// Represents the Component Model Async state of a store.
pub struct ConcurrentState {
/// The currently running thread, if any.
- current_thread: CurrentThread,
+ ///
+ /// Note that we lazily materialize threads on-demand and this field is not
+ /// necessarily up-to-date. The `StoreOpaque::current_thread` method should
+ /// be preferred over directly accessing this field.
+ unforced_current_thread: CurrentThread,
/// The set of pending host and background tasks, if any.
///
@@ -5107,7 +5250,7 @@ pub struct ConcurrentState {
impl Default for ConcurrentState {
fn default() -> Self {
Self {
- current_thread: CurrentThread::None,
+ unforced_current_thread: CurrentThread::None,
table: AlwaysMut::new(ResourceTable::new()),
futures: AlwaysMut::new(Some(FuturesUnordered::new())),
high_priority: Vec::new(),
@@ -5211,7 +5354,7 @@ impl ConcurrentState {
// These fields do not contain GC references.
worker_item: _,
- current_thread: _,
+ unforced_current_thread: _,
suspend_reason: _,
global_error_context_ref_counts: _,
interesting_tasks: _,
@@ -5363,18 +5506,6 @@ impl ConcurrentState {
}
}
- /// Returns whether there's a pending cancellation on the current guest thread,
- /// consuming the event if so.
- fn take_pending_cancellation(&mut self) -> Result<bool> {
- let thread = self.current_guest_thread()?;
- let task = self.get_mut(thread.task)?;
- if let Some(Event::Cancelled) = task.event {
- task.event.take();
- return Ok(true);
- }
- Ok(false)
- }
-
fn check_blocking_for(&mut self, task: TableId<GuestTask>) -> Result<()> {
if self.may_block(task)? {
Ok(())
@@ -5407,7 +5538,7 @@ impl ConcurrentState {
/// Used by `ResourceTables` to record the scope of a borrow to get undone
/// in the future.
pub fn current_call_context_scope_id(&self) -> Result<u32> {
- let (bits, is_host) = match self.current_thread {
+ let (bits, is_host) = match self.unforced_current_thread {
CurrentThread::Guest(id) => (id.task.rep(), false),
CurrentThread::Host(id) => (id.rep(), true),
CurrentThread::None => bail_bug!("current thread is not set"),
@@ -5416,20 +5547,6 @@ impl ConcurrentState {
Ok((bits << 1) | u32::from(is_host))
}
- fn current_guest_thread(&self) -> Result<QualifiedThreadId> {
- match self.current_thread.guest() {
- Some(id) => Ok(*id),
- None => bail_bug!("current thread is not a guest thread"),
- }
- }
-
- fn current_host_thread(&self) -> Result<TableId<HostTask>> {
- match self.current_thread.host() {
- Some(id) => Ok(id),
- None => bail_bug!("current thread is not a host thread"),
- }
- }
-
fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
match self.futures.get_mut().as_mut() {
Some(f) => Ok(f),
@@ -5575,7 +5692,7 @@ impl TaskId {
/// and can be resumed by other tasks for this component, so we mark the future as dropped
/// and delete the task when all threads are done.
pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
- let task = store.concurrent_state_mut().get_mut(self.task)?;
+ let task = store.concurrent_state_mut()?.get_mut(self.task)?;
let delete = if !task.already_lowered_parameters() {
store.cancel_guest_subtask_without_lowered_parameters(
self.runtime_instance,
@@ -5587,7 +5704,7 @@ impl TaskId {
task.ready_to_delete()
};
if delete {
- Waitable::Guest(self.task).delete_from(store.concurrent_state_mut())?
+ Waitable::Guest(self.task).delete_from(store.concurrent_state_mut()?)?
}
Ok(())
}
@@ -5627,12 +5744,12 @@ pub(crate) fn prepare_call<T, R>(
.map(SendSyncPtr::new);
let string_encoding = options.string_encoding;
let token = StoreToken::new(store.as_context_mut());
- let state = store.0.concurrent_state_mut();
+ let caller = store.0.current_thread()?;
+ let state = store.0.concurrent_state_mut()?;
let (tx, rx) = oneshot::channel();
let instance = handle.instance().runtime_instance(component_instance);
- let caller = state.current_thread;
let thread = GuestTask::new(
state,
Box::new(for_any_lower(move |store, params| {
diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs
index 7b017f724b..cb7a700edc 100644
--- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs
+++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs
@@ -417,7 +417,7 @@ impl<'a, T, B> Destination<'a, T, B> {
}
fn remaining_(&self, store: &mut StoreOpaque) -> Result<Option<usize>> {
- let transmit = store.concurrent_state_mut().get_mut(self.id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(self.id)?;
if let &ReadState::GuestReady { count, .. } = &transmit.read {
let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {
@@ -500,7 +500,7 @@ impl<D: 'static> DirectDestination<'_, D> {
.store
.as_context_mut()
.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(self.id)?;
let &ReadState::GuestReady {
@@ -551,7 +551,7 @@ impl<D: 'static> DirectDestination<'_, D> {
.store
.as_context_mut()
.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(self.id)?;
let ReadState::GuestReady {
@@ -890,7 +890,7 @@ impl<'a, T> Source<'a, T> {
buffer.move_from(*input, count);
} else {
let store = store.as_context_mut();
- let transmit = store.0.concurrent_state_mut().get_mut(self.id)?;
+ let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?;
let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
bail_bug!("expected ReadState::HostReady");
@@ -919,7 +919,7 @@ impl<'a, T> Source<'a, T> {
count.as_usize() - guest_offset.as_usize(),
)?;
- let transmit = store.0.concurrent_state_mut().get_mut(self.id)?;
+ let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?;
let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {
bail_bug!("expected ReadState::HostReady");
@@ -947,7 +947,7 @@ impl<'a, T> Source<'a, T> {
where
T: 'static,
{
- let transmit = store.concurrent_state_mut().get_mut(self.id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(self.id)?;
if let &WriteState::GuestReady { count, .. } = &transmit.write {
let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {
@@ -1010,7 +1010,7 @@ impl<D: 'static> DirectSource<'_, D> {
.store
.as_context_mut()
.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(self.id)?;
let &WriteState::GuestReady {
@@ -1061,7 +1061,7 @@ impl<D: 'static> DirectSource<'_, D> {
.store
.as_context_mut()
.0
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(self.id)?;
let WriteState::GuestReady {
@@ -1721,7 +1721,7 @@ impl<T> StreamReader<T> {
/// belong to the specified `store`.
pub fn try_into<V: 'static>(mut self, mut store: impl AsContextMut) -> Result<V, Self> {
let store = store.as_context_mut();
- let state = store.0.concurrent_state_mut();
+ let state = store.0.concurrent_state_mut_already_forced_current_thread();
let id = state.get_mut(self.id).unwrap().state;
if let WriteState::HostReady { try_into, .. } = &state.get_mut(id).unwrap().write {
match try_into(TypeId::of::<V>()) {
@@ -2313,7 +2313,7 @@ impl StoreOpaque {
let future = async move {
let stream_state = future.await?;
tls::get(|store| {
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let transmit = state.get_mut(id)?;
let ReadState::HostReady {
consume,
@@ -2343,7 +2343,8 @@ impl StoreOpaque {
})
};
- self.concurrent_state_mut().push_future(future.boxed());
+ self.concurrent_state_mut_already_forced_current_thread()
+ .push_future(future.boxed());
}
fn pipe_to_guest(
@@ -2355,7 +2356,7 @@ impl StoreOpaque {
let future = async move {
let stream_state = future.await?;
tls::get(|store| {
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let transmit = state.get_mut(id)?;
let WriteState::HostReady {
produce,
@@ -2387,12 +2388,13 @@ impl StoreOpaque {
})
};
- self.concurrent_state_mut().push_future(future.boxed());
+ self.concurrent_state_mut_already_forced_current_thread()
+ .push_future(future.boxed());
}
/// Drop the read end of a stream or future read from the host.
fn host_drop_reader(&mut self, id: TableId<TransmitHandle>, kind: TransmitKind) -> Result<()> {
- let state = self.concurrent_state_mut();
+ let state = self.concurrent_state_mut()?;
Waitable::Transmit(id).join(state, None)?;
let transmit_id = state.get_mut(id)?.state;
let transmit = state
@@ -2469,7 +2471,7 @@ impl StoreOpaque {
id: TableId<TransmitHandle>,
on_drop_open: Option<fn() -> Result<()>>,
) -> Result<()> {
- let state = self.concurrent_state_mut();
+ let state = self.concurrent_state_mut()?;
Waitable::Transmit(id).join(state, None)?;
let transmit_id = state.get_mut(id)?.state;
let transmit = state
@@ -2497,7 +2499,7 @@ impl StoreOpaque {
WriteState::Dropped => bail_bug!("write state is already dropped"),
}
- let transmit = self.concurrent_state_mut().get_mut(transmit_id)?;
+ let transmit = self.concurrent_state_mut()?.get_mut(transmit_id)?;
// If the existing read state is dropped, then there's nothing to read
// and we can keep it that way.
@@ -2519,7 +2521,7 @@ impl StoreOpaque {
// represent that a read must be performed
ReadState::GuestReady { ty, handle, .. } => {
// Ensure the final read of the guest is queued, with appropriate closure indicator
- self.concurrent_state_mut().update_event(
+ self.concurrent_state_mut()?.update_event(
read_handle.rep(),
match ty {
TransmitIndex::Future(ty) => Event::FutureRead {
@@ -2536,7 +2538,7 @@ impl StoreOpaque {
// If the read state is open, then there are no registered readers of the stream/future
ReadState::Open => {
- self.concurrent_state_mut().update_event(
+ self.concurrent_state_mut()?.update_event(
read_handle.rep(),
match on_drop_open {
Some(_) => Event::FutureRead {
@@ -2559,7 +2561,7 @@ impl StoreOpaque {
// this event.
ReadState::Dropped | ReadState::HostReady { .. } | ReadState::HostToHost { .. } => {
log::trace!("host_drop_writer delete {transmit_id:?}");
- self.concurrent_state_mut().delete_transmit(transmit_id)?;
+ self.concurrent_state_mut()?.delete_transmit(transmit_id)?;
}
}
Ok(())
@@ -2569,7 +2571,7 @@ impl StoreOpaque {
&mut self,
id: TableId<TransmitHandle>,
) -> Result<TransmitOrigin> {
- let state = self.concurrent_state_mut();
+ let state = self.concurrent_state_mut()?;
let state_id = state.get_mut(id)?.state;
Ok(state.get_mut(state_id)?.origin)
}
@@ -2585,7 +2587,7 @@ impl<T> StoreContextMut<'_, T> {
P::Item: func::Lower,
{
let token = StoreToken::new(self.as_context_mut());
- let state = self.0.concurrent_state_mut();
+ let state = self.0.concurrent_state_mut()?;
let (_, read) = state.new_transmit(TransmitOrigin::Host)?;
let producer = Arc::new(LockedState::new((Box::pin(producer), P::Buffer::default())));
let id = state.get_mut(read)?.state;
@@ -2601,7 +2603,7 @@ impl<T> StoreContextMut<'_, T> {
let (result, cancelled) = if buffer.remaining().is_empty() {
future::poll_fn(|cx| {
tls::get(|store| {
- let transmit = store.concurrent_state_mut().get_mut(id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;
let &WriteState::HostReady { cancel, .. } = &transmit.write else {
bail_bug!("expected WriteState::HostReady")
@@ -2632,7 +2634,7 @@ impl<T> StoreContextMut<'_, T> {
cancel,
);
- let transmit = store.concurrent_state_mut().get_mut(id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;
let host_offset = if let (
Some(host_buffer),
@@ -2683,7 +2685,7 @@ impl<T> StoreContextMut<'_, T> {
};
let (guest_offset, host_offset, count) = tls::get(|store| {
- let transmit = store.concurrent_state_mut().get_mut(id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;
let (count, host_offset) = match &transmit.read {
&ReadState::GuestReady { count, .. } => (count.as_u32(), 0),
&ReadState::HostToHost { limit, .. } => (1, limit),
@@ -2770,7 +2772,7 @@ impl<T> StoreContextMut<'_, T> {
consumer: C,
) -> Result<()> {
let token = StoreToken::new(self.as_context_mut());
- let state = self.0.concurrent_state_mut();
+ let state = self.0.concurrent_state_mut()?;
let id = state.get_mut(id)?.state;
let transmit = state.get_mut(id)?;
let consumer = Arc::new(LockedState::new(Box::pin(consumer)));
@@ -2784,7 +2786,7 @@ impl<T> StoreContextMut<'_, T> {
let (result, cancelled) = future::poll_fn(|cx| {
tls::get(|store| {
- let cancel = match &store.concurrent_state_mut().get_mut(id)?.read {
+ let cancel = match &store.concurrent_state_mut()?.get_mut(id)?.read {
&ReadState::HostReady { cancel, .. } => cancel,
ReadState::Open => false,
_ => bail_bug!("unexpected read state"),
@@ -2804,7 +2806,7 @@ impl<T> StoreContextMut<'_, T> {
cancel_waker,
cancel,
..
- } = &mut store.concurrent_state_mut().get_mut(id)?.read
+ } = &mut store.concurrent_state_mut()?.get_mut(id)?.read
{
if poll.is_pending() {
*cancel_waker = Some(cx.waker().clone());
@@ -2820,7 +2822,7 @@ impl<T> StoreContextMut<'_, T> {
.await?;
let (guest_offset, count) = tls::get(|store| {
- let transmit = store.concurrent_state_mut().get_mut(id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;
Ok((
match &transmit.read {
&ReadState::HostReady { guest_offset, .. } => guest_offset,
@@ -2855,7 +2857,7 @@ impl<T> StoreContextMut<'_, T> {
if let TransmitKind::Future = kind {
tls::get(|store| {
- store.concurrent_state_mut().get_mut(id)?.done = true;
+ store.concurrent_state_mut()?.get_mut(id)?.done = true;
crate::error::Ok(())
})?;
}
@@ -2930,7 +2932,7 @@ impl<T> StoreContextMut<'_, T> {
loop {
if tls::get(|store| {
crate::error::Ok(matches!(
- store.concurrent_state_mut().get_mut(id)?.read,
+ store.concurrent_state_mut()?.get_mut(id)?.read,
ReadState::Dropped
))
})? {
@@ -2948,7 +2950,7 @@ impl<T> StoreContextMut<'_, T> {
}
}
.map(move |result| {
- tls::get(|store| store.concurrent_state_mut().delete_transmit(id))?;
+ tls::get(|store| store.concurrent_state_mut()?.delete_transmit(id))?;
result
});
@@ -2970,7 +2972,7 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Write
kind: TransmitKind,
) -> Result<()> {
let (read, guest_offset) = tls::get(|store| {
- let transmit = store.concurrent_state_mut().get_mut(id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;
let guest_offset = if let &WriteState::HostReady { guest_offset, .. } = &transmit.write {
Some(guest_offset)
@@ -3003,7 +3005,7 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Write
if let TransmitKind::Future = kind {
tls::get(|store| {
- store.concurrent_state_mut().get_mut(id)?.done = true;
+ store.concurrent_state_mut()?.get_mut(id)?.done = true;
crate::error::Ok(())
})?;
}
@@ -3036,14 +3038,15 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Write
let (tx, rx) = oneshot::channel();
tls::get(move |store| {
store
- .concurrent_state_mut()
- .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
- move |store| {
+ .concurrent_state_mut()?
+ .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(
+ Box::new(move |store| {
_ = tx.send(accept(token.as_context_mut(store))?);
Ok(())
- },
- ))))
- });
+ }),
+ )));
+ crate::error::Ok(())
+ })?;
match rx.await {
Ok(r) => r,
Err(oneshot::Canceled) => bail_bug!("work cancelled"),
@@ -3060,7 +3063,7 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Write
tls::get(|store| {
let count = old_remaining - pair.with(|p| p.1.remaining().len())?;
- let transmit = store.concurrent_state_mut().get_mut(id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;
let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {
bail_bug!("expected WriteState::HostReady")
@@ -3110,7 +3113,7 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Write
}
tls::get(|store| {
- store.concurrent_state_mut().get_mut(id)?.read = match state {
+ store.concurrent_state_mut()?.get_mut(id)?.read = match state {
StreamResult::Dropped => ReadState::Dropped,
StreamResult::Completed | StreamResult::Cancelled => ReadState::HostToHost {
accept,
@@ -3141,7 +3144,7 @@ impl Instance {
cancel: bool,
) -> Result<ReturnCode> {
let mut future = consume();
- store.concurrent_state_mut().get_mut(transmit_id)?.read = ReadState::HostReady {
+ store.concurrent_state_mut()?.get_mut(transmit_id)?.read = ReadState::HostReady {
consume,
guest_offset,
cancel,
@@ -3155,7 +3158,7 @@ impl Instance {
Ok(match poll {
Poll::Ready(state) => {
- let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {
bail_bug!("expected ReadState::HostReady")
};
@@ -3183,7 +3186,7 @@ impl Instance {
cancel: bool,
) -> Result<ReturnCode> {
let mut future = produce();
- store.concurrent_state_mut().get_mut(transmit_id)?.write = WriteState::HostReady {
+ store.concurrent_state_mut()?.get_mut(transmit_id)?.write = WriteState::HostReady {
produce,
try_into,
guest_offset,
@@ -3198,7 +3201,7 @@ impl Instance {
Ok(match poll {
Poll::Ready(state) => {
- let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {
bail_bug!("expected WriteState::HostReady")
};
@@ -3503,7 +3506,7 @@ impl Instance {
*state = TransmitLocalState::Busy;
let transmit_handle = TableId::<TransmitHandle>::new(rep);
- let concurrent_state = store.0.concurrent_state_mut();
+ let concurrent_state = store.0.concurrent_state_mut()?;
let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;
let transmit = concurrent_state.get_mut(transmit_id)?;
log::trace!(
@@ -3612,7 +3615,7 @@ impl Instance {
Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?,
None => 0,
};
- let concurrent_state = store.0.concurrent_state_mut();
+ let concurrent_state = store.0.concurrent_state_mut()?;
if read_complete {
let total = if let Some(Event::StreamRead {
code: ReturnCode::Completed(old_total),
@@ -3757,8 +3760,8 @@ impl Instance {
*state = TransmitLocalState::Busy;
let transmit_handle = TableId::<TransmitHandle>::new(rep);
- let concurrent_state = store.0.concurrent_state_mut();
- let caller_thread = concurrent_state.current_guest_thread()?;
+ let caller_thread = store.0.current_guest_thread()?;
+ let concurrent_state = store.0.concurrent_state_mut()?;
let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;
let transmit = concurrent_state.get_mut(transmit_id)?;
log::trace!(
@@ -3850,7 +3853,7 @@ impl Instance {
Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?,
None => 0,
};
- let concurrent_state = store.0.concurrent_state_mut();
+ let concurrent_state = store.0.concurrent_state_mut()?;
if write_complete {
let total = if let Some(Event::StreamWrite {
@@ -3925,7 +3928,7 @@ impl Instance {
)?;
if let (TransmitIndex::Future(_), ReturnCode::Completed(_)) = (ty, code) {
- store.0.concurrent_state_mut().get_mut(transmit_id)?.done = true;
+ store.0.concurrent_state_mut()?.get_mut(transmit_id)?.done = true;
}
code
@@ -3967,7 +3970,7 @@ impl Instance {
) -> Result<ReturnCode> {
let waitable = Waitable::Transmit(handle);
store.wait_for_event(waitable)?;
- let event = waitable.take_event(store.concurrent_state_mut())?;
+ let event = waitable.take_event(store.concurrent_state_mut()?)?;
if let Some(event @ (Event::StreamWrite { code, .. } | Event::FutureWrite { code, .. })) =
event
{
@@ -3985,7 +3988,7 @@ impl Instance {
transmit_id: TableId<TransmitState>,
async_: bool,
) -> Result<ReturnCode> {
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let transmit = state.get_mut(transmit_id)?;
log::trace!(
"host_cancel_write state {transmit_id:?}; write state {:?} read state {:?}",
@@ -4021,7 +4024,7 @@ impl Instance {
ReturnCode::Blocked
} else {
let handle = store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(transmit_id)?
.write_handle;
self.wait_for_write(store, handle)?
@@ -4031,7 +4034,7 @@ impl Instance {
};
if !matches!(code, ReturnCode::Blocked) {
- let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
match &transmit.write {
WriteState::GuestReady { .. } => {
@@ -4054,7 +4057,7 @@ impl Instance {
) -> Result<ReturnCode> {
let waitable = Waitable::Transmit(handle);
store.wait_for_event(waitable)?;
- let event = waitable.take_event(store.concurrent_state_mut())?;
+ let event = waitable.take_event(store.concurrent_state_mut()?)?;
if let Some(event @ (Event::StreamRead { code, .. } | Event::FutureRead { code, .. })) =
event
{
@@ -4072,7 +4075,7 @@ impl Instance {
transmit_id: TableId<TransmitState>,
async_: bool,
) -> Result<ReturnCode> {
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let transmit = state.get_mut(transmit_id)?;
log::trace!(
"host_cancel_read state {transmit_id:?}; read state {:?} write state {:?}",
@@ -4108,7 +4111,7 @@ impl Instance {
ReturnCode::Blocked
} else {
let handle = store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.get_mut(transmit_id)?
.read_handle;
self.wait_for_read(store, handle)?
@@ -4118,7 +4121,7 @@ impl Instance {
};
if !matches!(code, ReturnCode::Blocked) {
- let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;
+ let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;
match &transmit.read {
ReadState::GuestReady { .. } => {
@@ -4164,7 +4167,7 @@ impl Instance {
}
TransmitLocalState::Busy => {}
}
- let transmit_id = store.concurrent_state_mut().get_mut(id)?.state;
+ let transmit_id = store.concurrent_state_mut()?.get_mut(id)?.state;
let code = self.cancel_write(store, transmit_id, async_)?;
if !matches!(code, ReturnCode::Blocked) {
let state =
@@ -4205,7 +4208,7 @@ impl Instance {
}
TransmitLocalState::Busy => {}
}
- let transmit_id = store.concurrent_state_mut().get_mut(id)?.state;
+ let transmit_id = store.concurrent_state_mut()?.get_mut(id)?.state;
let code = self.cancel_read(store, transmit_id, async_)?;
if !matches!(code, ReturnCode::Blocked) {
let state =
@@ -4257,7 +4260,7 @@ impl Instance {
// Create a new ErrorContext that is tracked along with other concurrent state
let err_ctx = ErrorContextState { debug_msg };
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let table_id = state.push(err_ctx)?;
let global_ref_count_idx =
TypeComponentGlobalErrorContextTableIndex::from_u32(table_id.rep());
@@ -4298,7 +4301,7 @@ impl Instance {
.table_for_error_context(ty)
.error_context_rep(err_ctx_handle)?;
- let state = store.0.concurrent_state_mut();
+ let state = store.0.concurrent_state_mut()?;
// Get the state associated with the error context
let ErrorContextState { debug_msg } =
state.get_mut(TableId::<ErrorContextState>::new(handle_table_id_rep))?;
@@ -4397,7 +4400,7 @@ impl Instance {
/// `TransmitIndex` belongs.
fn guest_new(self, store: &mut StoreOpaque, ty: TransmitIndex) -> Result<ResourcePair> {
let (write, read) = store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.new_transmit(TransmitOrigin::guest(self.id().instance(), ty))?;
let table = self.id().get_mut(store).table_for_transmit(ty);
@@ -4412,7 +4415,7 @@ impl Instance {
),
};
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
state.get_mut(read)?.common.handle = Some(read_handle);
state.get_mut(write)?.common.handle = Some(write_handle);
@@ -4437,7 +4440,7 @@ impl Instance {
let global_ref_count_idx = TypeComponentGlobalErrorContextTableIndex::from_u32(rep);
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let Some(GlobalErrorContextRefCount(global_ref_count)) = state
.global_error_context_ref_counts
.get_mut(&global_ref_count_idx)
@@ -4569,7 +4572,7 @@ impl Instance {
// as the new component has essentially created a new reference that will
// be dropped/handled independently
let global_ref_count = store
- .concurrent_state_mut()
+ .concurrent_state_mut()?
.global_error_context_ref_counts
.get_mut(&TypeComponentGlobalErrorContextTableIndex::from_u32(rep))
.context("global ref count present for existing (sub)component error context")?;
@@ -4881,7 +4884,7 @@ impl Waitable {
};
let transmit_handle = TableId::<TransmitHandle>::new(rep);
- let state = store.concurrent_state_mut();
+ let state = store.concurrent_state_mut()?;
let transmit_id = state.get_mut(transmit_handle)?.state;
let transmit = state.get_mut(transmit_id)?;
diff --git a/crates/wasmtime/src/runtime/component/func.rs b/crates/wasmtime/src/runtime/component/func.rs
index 423bb60ebb..a16fc33b6b 100644
--- a/crates/wasmtime/src/runtime/component/func.rs
+++ b/crates/wasmtime/src/runtime/component/func.rs
@@ -499,7 +499,7 @@ impl Func {
// as they're required to have been dropped by this point.
store
.0
- .component_resource_tables(Some(self.instance))
+ .component_resource_tables(Some(self.instance))?
.validate_scope_exit()?;
// SAFETY: We're relying on the correctness of the structure of
diff --git a/crates/wasmtime/src/runtime/component/func/options.rs b/crates/wasmtime/src/runtime/component/func/options.rs
index d02fc7e4db..51aab7e0d6 100644
--- a/crates/wasmtime/src/runtime/component/func/options.rs
+++ b/crates/wasmtime/src/runtime/component/func/options.rs
@@ -221,7 +221,7 @@ impl<'a, T: 'static> LowerContext<'a, T> {
ty: TypeResourceTableIndex,
rep: u32,
) -> Result<u32> {
- self.resource_tables().guest_resource_lower_own(rep, ty)
+ self.resource_tables()?.guest_resource_lower_own(rep, ty)
}
/// Lowers a `borrow` resource into the guest, converting the `rep` to a
@@ -242,19 +242,19 @@ impl<'a, T: 'static> LowerContext<'a, T> {
if self.instance().resource_owned_by_own_instance(ty) {
return Ok(rep);
}
- self.resource_tables().guest_resource_lower_borrow(rep, ty)
+ self.resource_tables()?.guest_resource_lower_borrow(rep, ty)
}
/// Lifts a host-owned `own` resource at the `idx` specified into the
/// representation of that resource.
pub fn host_resource_lift_own(&mut self, idx: HostResourceIndex) -> Result<u32> {
- self.resource_tables().host_resource_lift_own(idx)
+ self.resource_tables()?.host_resource_lift_own(idx)
}
/// Lifts a host-owned `borrow` resource at the `idx` specified into the
/// representation of that resource.
pub fn host_resource_lift_borrow(&mut self, idx: HostResourceIndex) -> Result<u32> {
- self.resource_tables().host_resource_lift_borrow(idx)
+ self.resource_tables()?.host_resource_lift_borrow(idx)
}
/// Lowers a resource into the host-owned table, returning the index it was
@@ -268,7 +268,7 @@ impl<'a, T: 'static> LowerContext<'a, T> {
dtor: Option<NonNull<VMFuncRef>>,
instance: Option<RuntimeInstance>,
) -> Result<HostResourceIndex> {
- self.resource_tables()
+ self.resource_tables()?
.host_resource_lower_own(rep, dtor, instance)
}
@@ -283,18 +283,18 @@ impl<'a, T: 'static> LowerContext<'a, T> {
InstanceType::new(self.instance())
}
- fn resource_tables(&mut self) -> HostResourceTables<'_> {
+ fn resource_tables(&mut self) -> Result<HostResourceTables<'_>> {
let (tables, data) = self
.store
.0
- .component_resource_tables_and_host_resource_data(Some(self.instance));
- HostResourceTables::from_parts(tables, data)
+ .component_resource_tables_and_host_resource_data(Some(self.instance))?;
+ Ok(HostResourceTables::from_parts(tables, data))
}
/// See [`HostResourceTables::validate_scope_exit`].
#[inline]
pub fn validate_scope_exit(&mut self) -> Result<()> {
- self.resource_tables().validate_scope_exit()
+ self.resource_tables()?.validate_scope_exit()
}
}
diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rs
index 262d181f98..13afe2c770 100644
--- a/crates/wasmtime/src/runtime/component/instance.rs
+++ b/crates/wasmtime/src/runtime/component/instance.rs
@@ -381,7 +381,7 @@ impl Instance {
rep: u32,
) -> Result<u32> {
store
- .component_resource_tables(Some(self))
+ .component_resource_tables(Some(self))?
.resource_new(TypedResource::Component { ty, rep })
}
@@ -394,7 +394,7 @@ impl Instance {
index: u32,
) -> Result<u32> {
store
- .component_resource_tables(Some(self))
+ .component_resource_tables(Some(self))?
.resource_rep(TypedResourceIndex::Component { ty, index })
}
@@ -406,7 +406,7 @@ impl Instance {
index: u32,
) -> Result<Option<u32>> {
store
- .component_resource_tables(Some(self))
+ .component_resource_tables(Some(self))?
.resource_drop(TypedResourceIndex::Component { ty, index })
}
@@ -417,7 +417,7 @@ impl Instance {
src: TypeResourceTableIndex,
dst: TypeResourceTableIndex,
) -> Result<u32> {
- let mut tables = store.component_resource_tables(Some(self));
+ let mut tables = store.component_resource_tables(Some(self))?;
let rep = tables.resource_lift_own(TypedResourceIndex::Component { ty: src, index })?;
tables.resource_lower_own(TypedResource::Component { ty: dst, rep })
}
@@ -430,7 +430,7 @@ impl Instance {
dst: TypeResourceTableIndex,
) -> Result<u32> {
let dst_owns_resource = self.id().get(store).resource_owned_by_own_instance(dst);
- let mut tables = store.component_resource_tables(Some(self));
+ let mut tables = store.component_resource_tables(Some(self))?;
let rep = tables.resource_lift_borrow(TypedResourceIndex::Component { ty: src, index })?;
// Implement `lower_borrow`'s special case here where if a borrow's
// resource type is owned by `dst` then the destination receives the
diff --git a/crates/wasmtime/src/runtime/component/resources/any.rs b/crates/wasmtime/src/runtime/component/resources/any.rs
index cc84cecb89..ecb69e4822 100644
--- a/crates/wasmtime/src/runtime/component/resources/any.rs
+++ b/crates/wasmtime/src/runtime/component/resources/any.rs
@@ -109,7 +109,7 @@ impl ResourceAny {
D: PartialEq + Send + Sync + Copy + 'static,
{
let store = store.as_context_mut();
- let mut tables = HostResourceTables::new_host(store.0);
+ let mut tables = HostResourceTables::new_host(store.0)?;
let ResourceAny { idx, ty, owned } = self;
let ty = T::typecheck(ty).ok_or_else(|| crate::format_err!("resource type mismatch"))?;
if owned {
@@ -189,7 +189,7 @@ impl ResourceAny {
//
// This could fail if the index is invalid or if this is removing an
// `Own` entry which is currently being borrowed.
- let pair = HostResourceTables::new_host(store.0).host_resource_drop(self.idx)?;
+ let pair = HostResourceTables::new_host(store.0)?.host_resource_drop(self.idx)?;
let (rep, slot) = match (pair, self.owned) {
(Some(pair), true) => pair,
diff --git a/crates/wasmtime/src/runtime/component/resources/host.rs b/crates/wasmtime/src/runtime/component/resources/host.rs
index e1d363c696..70f43f06f7 100644
--- a/crates/wasmtime/src/runtime/component/resources/host.rs
+++ b/crates/wasmtime/src/runtime/component/resources/host.rs
@@ -292,7 +292,7 @@ where
} = self;
let store = store.as_context_mut();
- let mut tables = HostResourceTables::new_host(store.0);
+ let mut tables = HostResourceTables::new_host(store.0)?;
let (idx, owned) = match state.get() {
ResourceState::Borrow => (tables.host_resource_lower_borrow(rep)?, false),
ResourceState::NotInTable => {
diff --git a/crates/wasmtime/src/runtime/component/resources/host_tables.rs b/crates/wasmtime/src/runtime/component/resources/host_tables.rs
index a8ba4809cb..4835e538e0 100644
--- a/crates/wasmtime/src/runtime/component/resources/host_tables.rs
+++ b/crates/wasmtime/src/runtime/component/resources/host_tables.rs
@@ -96,9 +96,9 @@ impl HostResourceIndex {
}
impl<'a> HostResourceTables<'a> {
- pub fn new_host(store: &'a mut StoreOpaque) -> HostResourceTables<'a> {
- let (tables, data) = store.component_resource_tables_and_host_resource_data(None);
- HostResourceTables::from_parts(tables, data)
+ pub fn new_host(store: &'a mut StoreOpaque) -> Result<HostResourceTables<'a>> {
+ let (tables, data) = store.component_resource_tables_and_host_resource_data(None)?;
+ Ok(HostResourceTables::from_parts(tables, data))
}
pub fn from_parts(
diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rs
index 7c1ace64c6..52e78130a4 100644
--- a/crates/wasmtime/src/runtime/component/store.rs
+++ b/crates/wasmtime/src/runtime/component/store.rs
@@ -136,7 +136,7 @@ impl ComponentStoreData {
let mut fibers = Vec::new();
let mut futures = Vec::new();
store
- .concurrent_state_mut()
+ .concurrent_state_mut_without_forcing_current_thread()
.take_fibers_and_futures(&mut fibers, &mut futures);
for mut fiber in fibers {
@@ -297,8 +297,13 @@ impl StoreOpaque {
&mut self.store_data_mut().components
}
- pub(crate) fn component_task_state_mut(&mut self) -> &mut ComponentTaskState {
- &mut self.component_data_mut().task_state
+ pub(crate) fn component_task_state_mut(&mut self) -> Result<&mut ComponentTaskState> {
+ // NB: Force the deferred lazy thread, if any, before handing out task
+ // state.
+ #[cfg(feature = "component-model-async")]
+ let _ = self.current_thread()?;
+
+ Ok(&mut self.component_data_mut().task_state)
}
pub(crate) fn push_component_instance(&mut self, instance: Instance) {
@@ -323,11 +328,34 @@ impl StoreOpaque {
}
#[cfg(feature = "component-model-async")]
- pub(crate) fn concurrent_state_mut(&mut self) -> &mut ConcurrentState {
+ pub(crate) fn concurrent_state_mut_without_forcing_current_thread(
+ &mut self,
+ ) -> &mut ConcurrentState {
debug_assert!(self.concurrency_support());
self.component_data_mut().task_state.concurrent_state_mut()
}
+ #[cfg(feature = "component-model-async")]
+ pub(crate) fn concurrent_state_mut_already_forced_current_thread(
+ &mut self,
+ ) -> &mut ConcurrentState {
+ debug_assert!(self.concurrency_support());
+ debug_assert!(
+ !self
+ .vm_store_context_mut()
+ .current_thread_mut()
+ .is_deferred()
+ );
+ self.concurrent_state_mut_without_forcing_current_thread()
+ }
+
+ #[cfg(feature = "component-model-async")]
+ pub(crate) fn concurrent_state_mut(&mut self) -> Result<&mut ConcurrentState> {
+ debug_assert!(self.concurrency_support());
+ self.current_thread()?;
+ Ok(self.component_data_mut().task_state.concurrent_state_mut())
+ }
+
#[inline]
#[cfg(feature = "component-model-async")]
pub(crate) fn concurrency_support(&self) -> bool {
@@ -362,18 +390,24 @@ impl StoreOpaque {
pub(crate) fn component_resource_tables(
&mut self,
instance: Option<Instance>,
- ) -> vm::component::ResourceTables<'_> {
- self.component_resource_tables_and_host_resource_data(instance)
- .0
+ ) -> Result<vm::component::ResourceTables<'_>> {
+ Ok(self
+ .component_resource_tables_and_host_resource_data(instance)?
+ .0)
}
pub(crate) fn component_resource_tables_and_host_resource_data(
&mut self,
instance: Option<Instance>,
- ) -> (
+ ) -> Result<(
vm::component::ResourceTables<'_>,
&mut crate::component::HostResourceData,
- ) {
+ )> {
+ // NB: Force the current lazy deferred thread, if any, before handing
+ // out resource tables.
+ #[cfg(feature = "component-model-async")]
+ let _ = self.current_thread()?;
+
let store_id = self.id();
let data = self.component_data_mut();
let guest = instance.map(|i| {
@@ -386,14 +420,14 @@ impl StoreOpaque {
.instance_states()
});
- (
+ Ok((
vm::component::ResourceTables {
host_table: &mut data.component_host_table,
task_state: &mut data.task_state,
guest,
},
&mut data.host_resource_data,
- )
+ ))
}
pub(crate) fn enter_call_not_concurrent(&mut self) -> Result<()> {
@@ -424,7 +458,10 @@ impl StoreOpaque {
#[cfg(feature = "component-model-async")]
fn concurrent_resource_table(&mut self) -> Option<&mut ResourceTable> {
if self.concurrency_support() {
- Some(self.concurrent_state_mut().table())
+ Some(
+ self.concurrent_state_mut_without_forcing_current_thread()
+ .table(),
+ )
} else {
None
}
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index bda01cd5c2..ac690ffd8e 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -2472,7 +2472,9 @@ unsafe impl<T> VMStore for StoreInner<T> {
}
#[cfg(feature = "component-model")]
- fn component_task_state_mut(&mut self) -> &mut crate::component::store::ComponentTaskState {
+ fn component_task_state_mut(
+ &mut self,
+ ) -> Result<&mut crate::component::store::ComponentTaskState> {
StoreOpaque::component_task_state_mut(self)
}
diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs
index e97b4259ad..7bea0110f7 100644
--- a/crates/wasmtime/src/runtime/vm.rs
+++ b/crates/wasmtime/src/runtime/vm.rs
@@ -126,6 +126,8 @@ pub use crate::runtime::vm::throw::*;
pub use crate::runtime::vm::traphandlers::*;
#[cfg(feature = "component-model")]
pub use crate::runtime::vm::vmcontext::VMArrayCallFunction;
+#[cfg(feature = "component-model-async")]
+pub use crate::runtime::vm::vmcontext::VMLazyThread;
pub use crate::runtime::vm::vmcontext::{
VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition,
VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext,
@@ -219,7 +221,9 @@ pub unsafe trait VMStore: 'static {
/// Metadata required for resources for the component model.
#[cfg(feature = "component-model")]
- fn component_task_state_mut(&mut self) -> &mut crate::component::store::ComponentTaskState;
+ fn component_task_state_mut(
+ &mut self,
+ ) -> Result<&mut crate::component::store::ComponentTaskState>;
#[cfg(feature = "component-model-async")]
fn component_async_store(
diff --git a/crates/wasmtime/src/runtime/vm/component/libcalls.rs b/crates/wasmtime/src/runtime/vm/component/libcalls.rs
index 094b5562c2..2484a09db1 100644
--- a/crates/wasmtime/src/runtime/vm/component/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/component/libcalls.rs
@@ -688,7 +688,7 @@ fn enter_sync_call(
fn exit_sync_call(store: &mut dyn VMStore, instance: Instance) -> Result<()> {
store
- .component_resource_tables(Some(instance))
+ .component_resource_tables(Some(instance))?
.validate_scope_exit()?;
store.exit_guest_sync_call()
}
diff --git a/crates/wasmtime/src/runtime/vm/provenance.rs b/crates/wasmtime/src/runtime/vm/provenance.rs
index 7453f17b23..7ea4787c8c 100644
--- a/crates/wasmtime/src/runtime/vm/provenance.rs
+++ b/crates/wasmtime/src/runtime/vm/provenance.rs
@@ -44,6 +44,8 @@
use crate::vm::SendSyncPtr;
use core::fmt;
use core::marker;
+use core::marker::PhantomData;
+use core::num::NonZero;
use core::num::NonZeroUsize;
use core::ptr::NonNull;
use core::sync::atomic::AtomicUsize;
@@ -117,8 +119,25 @@ impl<T> VmPtr<T> {
}
/// Similar to `{NonNull,core::ptr}::dangling()` but for `VmPtr`.
- pub fn dangling() -> Self {
- NonNull::dangling().into()
+ pub const fn dangling() -> Self {
+ let ptr = core::mem::align_of::<T>();
+ Self {
+ ptr: NonZeroUsize::new(ptr).unwrap(),
+ _marker: PhantomData,
+ }
+ }
+
+ /// Similar to `NonNull::cast` but for `VmPtr`.
+ pub const fn cast<U>(self) -> VmPtr<U> {
+ VmPtr {
+ ptr: self.ptr,
+ _marker: PhantomData,
+ }
+ }
+
+ /// Similar to `NonNull::addr` but for `VmPtr`.
+ pub fn addr(&self) -> NonZero<usize> {
+ self.ptr
}
}
@@ -131,6 +150,32 @@ impl<T> Clone for VmPtr<T> {
impl<T> Copy for VmPtr<T> {}
+impl<T> PartialEq for VmPtr<T> {
+ fn eq(&self, other: &Self) -> bool {
+ self.ptr == other.ptr
+ }
+}
+
+impl<T> Eq for VmPtr<T> {}
+
+impl<T> PartialOrd for VmPtr<T> {
+ fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
+ self.ptr.partial_cmp(&other.ptr)
+ }
+}
+
+impl<T> Ord for VmPtr<T> {
+ fn cmp(&self, other: &Self) -> core::cmp::Ordering {
+ self.ptr.cmp(&other.ptr)
+ }
+}
+
+impl<T> core::hash::Hash for VmPtr<T> {
+ fn hash<H: core::hash::Hasher>(&self, state: &mut H) {
+ self.ptr.hash(state);
+ }
+}
+
// Forward debugging to `SendSyncPtr<T>` which renders the address.
impl<T> fmt::Debug for VmPtr<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index 83844fb4ee..9c665e084b 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -1226,7 +1226,19 @@ pub struct VMStoreContext {
/// `VMOffsets` conditional.
///
/// This is saved/restored when threads are swapped in the component model.
- pub component_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
+ ///
+ /// NB: `UnsafeCell` because JIT code writes to the slots.
+ pub component_context: UnsafeCell<[u32; NUM_COMPONENT_CONTEXT_SLOTS]>,
+
+ /// JIT-visible current thread for the component model's sync-to-sync
+ /// adapter fast path.
+ ///
+ /// Like `component_context`, this is unconditionally present to keep
+ /// `VMOffsets` logic unconditional even though it is only used when
+ /// `component-model-async` is enabled.
+ ///
+ /// NB: `UnsafeCell` because JIT code writes to this field.
+ pub current_thread: UnsafeCell<VMLazyThread>,
}
impl VMStoreContext {
@@ -1280,6 +1292,16 @@ impl VMStoreContext {
0
}
}
+
+ #[cfg(feature = "component-model-async")]
+ pub(crate) fn component_context_mut(&mut self) -> &mut [u32; NUM_COMPONENT_CONTEXT_SLOTS] {
+ self.component_context.get_mut()
+ }
+
+ #[cfg(feature = "component-model-async")]
+ pub(crate) fn current_thread_mut(&mut self) -> &mut VMLazyThread {
+ self.current_thread.get_mut()
+ }
}
// The `VMStoreContext` type is a pod-type with no destructor, and we don't
@@ -1311,7 +1333,8 @@ impl Default for VMStoreContext {
stack_chain: UnsafeCell::new(VMStackChain::Absent),
async_guard_range: ptr::null_mut()..ptr::null_mut(),
store_data: VmPtr::dangling(),
- component_context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
+ component_context: UnsafeCell::new([0; NUM_COMPONENT_CONTEXT_SLOTS]),
+ current_thread: UnsafeCell::new(VMLazyThread::none()),
}
}
}
@@ -1386,19 +1409,177 @@ mod test_vmstore_context {
offset_of!(VMStoreContext, component_context),
usize::from(offsets.ptr.vmstore_context_component_context_slot(0))
);
+ assert_eq!(
+ offset_of!(VMStoreContext, current_thread),
+ usize::from(offsets.ptr.vmstore_context_current_thread())
+ );
// Make sure that the calculation for the size of a slot is also
// accurate.
let slot_width = offsets.ptr.vmstore_context_component_context_slot(1)
- offsets.ptr.vmstore_context_component_context_slot(0);
- let default = VMStoreContext::default();
+ let mut default = VMStoreContext::default();
assert_eq!(
- size_of_val(&default.component_context[0]),
+ size_of_val(&default.component_context.get_mut()[0]),
usize::from(slot_width)
);
}
}
+/// JIT-visible representation of the store's current thread for the component
+/// model, encoded as a single pointer-sized integer so that generated JIT code
+/// can load, store, and compare it with a handful of instructions.
+///
+/// This is the inline fast-path counterpart to the host-side `CurrentThread`: a
+/// fused sync-to-sync adapter records a lazy deferred thread here (a pointer to
+/// a `VMDeferredThread` on its own stack frame) instead of eagerly allocating a
+/// `GuestTask`/`GuestThread` in the host. Host code promotes the deferred
+/// thread into a real one only when it actually needs it; see
+/// `StoreOpaque::force_current_thread`.
+///
+/// This type is a bitpacked equivalent of the following logical `enum`:
+///
+/// ```ignore
+/// enum VMLazyThread {
+/// /// No thread.
+/// None,
+///
+/// /// The lazy thread was promoted and materialized; get it from
+/// /// `ConcurrentState::current_thread`.
+/// Forced,
+///
+/// /// The lazy thread has not been materialized, here is a pointer to the
+/// /// stack-allocated data needed to do force that promotion.
+/// Deferred(*mut VMDeferredThread),
+/// }
+/// ```
+//
+// Bitpacking details:
+//
+// * `None`: `0`
+//
+// * `Forced`: A non-zero value with its low-bit set.
+//
+// * `Deferred`: A non-zero value with its low-bit clear.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+#[repr(transparent)]
+pub struct VMLazyThread(Option<VmPtr<VMDeferredThread>>);
+
+impl VMLazyThread {
+ const _ASSERT_SIZE: () = assert!(
+ core::mem::size_of::<VMLazyThread>() == core::mem::size_of::<*mut VMDeferredThread>()
+ );
+ const _ASSERT_ALIGN: () = assert!(
+ core::mem::align_of::<VMLazyThread>() == core::mem::align_of::<*mut VMDeferredThread>()
+ );
+
+ const FORCED: VmPtr<VMDeferredThread> = VmPtr::<u8>::dangling().cast();
+
+ /// There is no current thread.
+ pub const fn none() -> Self {
+ Self(None)
+ }
+
+ /// A lazy thread that has already been promoted.
+ pub const fn forced() -> Self {
+ Self(Some(Self::FORCED))
+ }
+
+ /// A deferred thread referencing the given on-stack [`VMDeferredThread`].
+ pub fn deferred(ptr: NonNull<VMDeferredThread>) -> Self {
+ debug_assert_eq!(ptr.addr().get() & Self::FORCED.addr().get(), 0);
+ Self(Some(ptr.into()))
+ }
+
+ /// Returns `true` if there is no current thread.
+ pub fn is_none(self) -> bool {
+ self.0.is_none()
+ }
+
+ /// Returns `true` if a deferred thread has been forced/promoted.
+ pub fn is_forced(self) -> bool {
+ self.0.is_some_and(|p| p == Self::FORCED)
+ }
+
+ /// Returns `true` if this is a deferred thread (i.e. neither `None` nor
+ /// forced).
+ pub fn is_deferred(self) -> bool {
+ self.0.is_some_and(|p| p != Self::FORCED)
+ }
+
+ /// Returns the deferred [`VMDeferredThread`] pointer if this is a deferred
+ /// thread.
+ pub fn as_deferred(self) -> Option<VmPtr<VMDeferredThread>> {
+ self.0
+ .and_then(|p| if p == Self::FORCED { None } else { Some(p) })
+ }
+}
+
+/// A deferred component-model thread.
+///
+/// This is an on-stack record pushed by a fused sync-to-sync adapter's fast
+/// path to defer the work that the `enter_sync_call` libcall would otherwise do
+/// eagerly.
+///
+/// The adapter allocates one of these in its own stack frame, links the
+/// previous current-thread value to it via `parent`, and finally points
+/// `VMStoreContext::current_thread` at it. When host code actually needs the
+/// real thread, it walks the `parent` chain to materialize thread state (see
+/// `StoreOpaque::force_current_thread`).
+#[derive(Debug)]
+#[repr(C)]
+pub struct VMDeferredThread {
+ /// The previous value of `VMStoreContext::current_thread`.
+ pub parent: VMLazyThread,
+ /// The caller component instance (a deferred `enter_sync_call` argument).
+ pub caller_instance: u32,
+ /// Whether the callee is async-lifted (a deferred `enter_sync_call` arg).
+ pub callee_async: u32,
+ /// The callee component instance (a deferred `enter_sync_call` argument).
+ pub callee_instance: u32,
+ /// The caller thread's `context.{get,set}` slots, saved on entry and
+ /// restored on the fast-path exit (or recovered while forcing).
+ pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
+}
+
+#[cfg(test)]
+mod test_vmdeferred_thread {
+ use super::*;
+ use core::mem::offset_of;
+ use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
+
+ #[test]
+ fn deferred_thread_field_offsets() {
+ let module = Module::new(StaticModuleIndex::from_u32(0));
+ let offsets = VMOffsets::new(HostPtr, &module);
+ let ptr = offsets.ptr;
+ assert_eq!(
+ offset_of!(VMDeferredThread, parent),
+ usize::from(ptr.vmdeferred_thread_parent())
+ );
+ assert_eq!(
+ offset_of!(VMDeferredThread, caller_instance),
+ usize::from(ptr.vmdeferred_thread_caller_instance())
+ );
+ assert_eq!(
+ offset_of!(VMDeferredThread, callee_async),
+ usize::from(ptr.vmdeferred_thread_callee_async())
+ );
+ assert_eq!(
+ offset_of!(VMDeferredThread, callee_instance),
+ usize::from(ptr.vmdeferred_thread_callee_instance())
+ );
+ assert_eq!(
+ offset_of!(VMDeferredThread, saved_context),
+ usize::from(ptr.vmdeferred_thread_saved_context(0))
+ );
+ assert_eq!(
+ size_of::<VMDeferredThread>(),
+ usize::from(ptr.size_of_vmdeferred_thread())
+ );
+ }
+}
+
/// The VM "context", which is pointed to by the `vmctx` arg in Cranelift.
/// This has information about globals, memories, tables, and other runtime
/// state associated with the current instance.
diff --git a/tests/all/component_model.rs b/tests/all/component_model.rs
index b7e46fcc13..e407806805 100644
--- a/tests/all/component_model.rs
+++ b/tests/all/component_model.rs
@@ -23,6 +23,7 @@ mod nested;
mod post_return;
mod resources;
mod strings;
+mod sync_call_inline;
#[derive(Copy, Clone)]
enum ApiStyle {
diff --git a/tests/all/component_model/async.rs b/tests/all/component_model/async.rs
index 7df03ca8df..6b93b7b2e1 100644
--- a/tests/all/component_model/async.rs
+++ b/tests/all/component_model/async.rs
@@ -1001,7 +1001,7 @@ async fn async_call_stack() -> Result<()> {
linker.root().func_wrap(
"a",
|mut store: StoreContextMut<Option<GuestTaskId>>, (): ()| {
- let stack = store.async_call_stack().collect::<Vec<_>>();
+ let stack = store.async_call_stack()?.collect::<Vec<_>>();
assert_eq!(stack, [store.data().unwrap()]);
Ok(())
},
@@ -1046,7 +1046,7 @@ async fn async_call_stack() -> Result<()> {
linker.root().func_wrap(
"a",
|mut store: StoreContextMut<Option<GuestTaskId>>, (): ()| {
- let stack = store.async_call_stack().collect::<Vec<_>>();
+ let stack = store.async_call_stack()?.collect::<Vec<_>>();
assert_eq!(stack.len(), 2);
assert_eq!(stack.last(), store.data().as_ref());
Ok(())
diff --git a/tests/all/component_model/sync_call_inline.rs b/tests/all/component_model/sync_call_inline.rs
new file mode 100644
index 0000000000..b93a5b6cbf
--- /dev/null
+++ b/tests/all/component_model/sync_call_inline.rs
@@ -0,0 +1,324 @@
+//! Runtime tests for the guest-to-guest sync-call inline fast path.
+
+#![cfg(not(miri))]
+
+use wasmtime::component::*;
+use wasmtime::{Config, Engine, Result, Store, StoreContextMut};
+
+fn engine() -> Engine {
+ let mut config = Config::new();
+ config.wasm_component_model(true);
+ config.wasm_component_model_async(true);
+ Engine::new(&config).unwrap()
+}
+
+#[tokio::test]
+async fn host_call_forces_slow_path_preserves_context() -> Result<()> {
+ let component = r#"
+(component
+ (import "poke" (func $poke))
+
+ (component $Inner
+ (import "poke" (func $poke))
+ (core func $poke' (canon lower (func $poke)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "poke" (func $poke'))
+ (import "" "cget" (func $cget (result i32)))
+ (import "" "cset" (func $cset (param i32)))
+ (func (export "f'") (param i32) (result i32)
+ ;; Freshly entered deferred thread: context starts zeroed.
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x5678))
+ ;; Force the deferred thread via a guest->host call.
+ (call $poke')
+ ;; Our context survives the force.
+ (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 42))))
+ (core instance $m (instantiate $M (with "" (instance
+ (export "poke" (func $poke'))
+ (export "cget" (func $cget))
+ (export "cset" (func $cset))))))
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'"))))
+
+ (component $Outer
+ (import "f" (func $f (param "x" u32) (result u32)))
+ (core func $f' (canon lower (func $f)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (import "" "cget" (func $cget (result i32)))
+ (import "" "cset" (func $cset (param i32)))
+ (func (export "g'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x1234))
+ (local.set $r (call $f' (i32.const 1234)))
+ ;; Restored after the callee forced the slow exit path.
+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))
+ (local.get $r)))
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f'" (func $f'))
+ (export "cget" (func $cget))
+ (export "cset" (func $cset))))))
+ (func (export "g") (result u32)
+ (canon lift (core func $n "g'"))))
+
+ (instance $inner (instantiate $Inner (with "poke" (func $poke))))
+ (instance $outer (instantiate $Outer (with "f" (func $inner "f"))))
+ (export "g" (func $outer "g"))
+)
+ "#;
+
+ let engine = engine();
+ let component = Component::new(&engine, component)?;
+ let mut store = Store::new(&engine, 0u32);
+ let mut linker = Linker::new(&engine);
+ linker
+ .root()
+ .func_wrap("poke", |mut cx: StoreContextMut<u32>, (): ()| {
+ *cx.data_mut() += 1;
+ Ok(())
+ })?;
+ let instance = linker.instantiate_async(&mut store, &component).await?;
+ let g = instance.get_typed_func::<(), (u32,)>(&mut store, "g")?;
+
+ let (result,) = g.call_async(&mut store, ()).await?;
+ assert_eq!(result, 1276);
+ assert_eq!(*store.data(), 1, "host import should have been called once");
+ Ok(())
+}
+
+#[tokio::test]
+async fn nested_chain_host_force_preserves_all_contexts() -> Result<()> {
+ let component = r#"
+(component
+ (import "poke" (func $poke))
+
+ (component $Leaf
+ (import "poke" (func $poke))
+ (core func $poke' (canon lower (func $poke)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "poke" (func $poke'))
+ (import "" "cget" (func $cget (result i32)))
+ (import "" "cset" (func $cset (param i32)))
+ (func (export "leaf'") (param i32) (result i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x0c0ffee0))
+ (call $poke')
+ (if (i32.ne (call $cget) (i32.const 0x0c0ffee0)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 1))))
+ (core instance $m (instantiate $M (with "" (instance
+ (export "poke" (func $poke'))
+ (export "cget" (func $cget))
+ (export "cset" (func $cset))))))
+ (func (export "leaf") (param "x" u32) (result u32)
+ (canon lift (core func $m "leaf'"))))
+
+ (component $Mid
+ (import "leaf" (func $leaf (param "x" u32) (result u32)))
+ (core func $leaf' (canon lower (func $leaf)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))
+ (import "" "cget" (func $cget (result i32)))
+ (import "" "cset" (func $cset (param i32)))
+ (func (export "mid'") (param i32) (result i32) (local $r i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x0d00d100))
+ (local.set $r (call $leaf' (local.get 0)))
+ (if (i32.ne (call $cget) (i32.const 0x0d00d100)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 10))))
+ (core instance $m (instantiate $M (with "" (instance
+ (export "leaf'" (func $leaf'))
+ (export "cget" (func $cget))
+ (export "cset" (func $cset))))))
+ (func (export "mid") (param "x" u32) (result u32)
+ (canon lift (core func $m "mid'"))))
+
+ (component $Root
+ (import "mid" (func $mid (param "x" u32) (result u32)))
+ (core func $mid' (canon lower (func $mid)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "mid'" (func $mid' (param i32) (result i32)))
+ (import "" "cget" (func $cget (result i32)))
+ (import "" "cset" (func $cset (param i32)))
+ (func (export "root'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x0badf00d))
+ (local.set $r (call $mid' (i32.const 100)))
+ (if (i32.ne (call $cget) (i32.const 0x0badf00d)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 1000))))
+ (core instance $m (instantiate $M (with "" (instance
+ (export "mid'" (func $mid'))
+ (export "cget" (func $cget))
+ (export "cset" (func $cset))))))
+ (func (export "root") (result u32)
+ (canon lift (core func $m "root'"))))
+
+ (instance $leaf (instantiate $Leaf (with "poke" (func $poke))))
+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))
+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))
+ (export "root" (func $root "root"))
+)
+ "#;
+
+ let engine = engine();
+ let component = Component::new(&engine, component)?;
+ let mut store = Store::new(&engine, 0u32);
+ let mut linker = Linker::new(&engine);
+ linker
+ .root()
+ .func_wrap("poke", |mut cx: StoreContextMut<u32>, (): ()| {
+ *cx.data_mut() += 1;
+ Ok(())
+ })?;
+ let instance = linker.instantiate_async(&mut store, &component).await?;
+ let root = instance.get_typed_func::<(), (u32,)>(&mut store, "root")?;
+
+ let (result,) = root.call_async(&mut store, ()).await?;
+ assert_eq!(result, 1111);
+ assert_eq!(*store.data(), 1);
+ Ok(())
+}
+
+#[tokio::test]
+async fn repeated_calls_have_no_state_leak() -> Result<()> {
+ let component = r#"
+(component
+ (import "poke" (func $poke))
+
+ (component $Inner
+ (import "poke" (func $poke))
+ (core func $poke' (canon lower (func $poke)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "poke" (func $poke'))
+ (import "" "cget" (func $cget (result i32)))
+ (import "" "cset" (func $cset (param i32)))
+ (func (export "f'") (param i32) (result i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (local.get 0))
+ (call $poke')
+ (if (i32.ne (call $cget) (local.get 0)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 42))))
+ (core instance $m (instantiate $M (with "" (instance
+ (export "poke" (func $poke'))
+ (export "cget" (func $cget))
+ (export "cset" (func $cset))))))
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'"))))
+
+ (component $Outer
+ (import "f" (func $f (param "x" u32) (result u32)))
+ (core func $f' (canon lower (func $f)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (import "" "cget" (func $cget (result i32)))
+ (import "" "cset" (func $cset (param i32)))
+ (func (export "g'") (param i32) (result i32) (local $r i32)
+ (call $cset (i32.const 0x4321))
+ (local.set $r (call $f' (local.get 0)))
+ (if (i32.ne (call $cget) (i32.const 0x4321)) (then unreachable))
+ (local.get $r)))
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f'" (func $f'))
+ (export "cget" (func $cget))
+ (export "cset" (func $cset))))))
+ (func (export "g") (param "x" u32) (result u32)
+ (canon lift (core func $n "g'"))))
+
+ (instance $inner (instantiate $Inner (with "poke" (func $poke))))
+ (instance $outer (instantiate $Outer (with "f" (func $inner "f"))))
+ (export "g" (func $outer "g"))
+)
+ "#;
+
+ let engine = engine();
+ let component = Component::new(&engine, component)?;
+ let mut store = Store::new(&engine, 0u32);
+ let mut linker = Linker::new(&engine);
+ linker
+ .root()
+ .func_wrap("poke", |mut cx: StoreContextMut<u32>, (): ()| {
+ *cx.data_mut() += 1;
+ Ok(())
+ })?;
+ let instance = linker.instantiate_async(&mut store, &component).await?;
+ let g = instance.get_typed_func::<(u32,), (u32,)>(&mut store, "g")?;
+
+ for x in [7u32, 100, 0x10000, 1] {
+ let (result,) = g.call_async(&mut store, (x,)).await?;
+ assert_eq!(result, x + 42);
+ }
+ assert_eq!(
+ *store.data(),
+ 4,
+ "host import called once per top-level call"
+ );
+ Ok(())
+}
+
+#[tokio::test]
+async fn trap_then_instantiate_uses_freed_deferred_thread() -> Result<()> {
+ let trapping = r#"
+(component
+ (component $Leaf
+ (core module $M (func (export "leaf'") (param i32) (result i32) unreachable))
+ (core instance $m (instantiate $M))
+ (func (export "leaf") (param "x" u32) (result u32) (canon lift (core func $m "leaf'"))))
+ (component $Mid
+ (import "leaf" (func $leaf (param "x" u32) (result u32)))
+ (core func $leaf' (canon lower (func $leaf)))
+ (core module $M
+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))
+ (func (export "mid'") (param i32) (result i32) (call $leaf' (local.get 0))))
+ (core instance $m (instantiate $M (with "" (instance (export "leaf'" (func $leaf'))))))
+ (func (export "mid") (param "x" u32) (result u32) (canon lift (core func $m "mid'"))))
+ (component $Root
+ (import "mid" (func $mid (param "x" u32) (result u32)))
+ (core func $mid' (canon lower (func $mid)))
+ (core module $M
+ (import "" "mid'" (func $mid' (param i32) (result i32)))
+ (func (export "root'") (result i32) (call $mid' (i32.const 1))))
+ (core instance $m (instantiate $M (with "" (instance (export "mid'" (func $mid'))))))
+ (func (export "root") (result u32) (canon lift (core func $m "root'"))))
+ (instance $leaf (instantiate $Leaf))
+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))
+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))
+ (export "root" (func $root "root"))
+)
+ "#;
+
+ let other = r#"
+(component
+ (core module $m (func (export "x")))
+ (core instance (instantiate $m))
+)
+ "#;
+
+ let engine = engine();
+ let trapping = Component::new(&engine, trapping)?;
+ let other = Component::new(&engine, other)?;
+ let mut store = Store::new(&engine, 0u32);
+ let linker = Linker::new(&engine);
+
+ let instance = linker.instantiate_async(&mut store, &trapping).await?;
+ let root = instance.get_typed_func::<(), (u32,)>(&mut store, "root")?;
+ let err = root.call_async(&mut store, ()).await.unwrap_err();
+ assert!(
+ err.downcast_ref::<wasmtime::Trap>().is_some(),
+ "expected a trap, got: {err:?}"
+ );
+
+ let _ = linker.instantiate_async(&mut store, &other).await?;
+ Ok(())
+}
diff --git a/tests/disas/aarch64-entry-trampoline.wat b/tests/disas/aarch64-entry-trampoline.wat
index b80d9968c8..a38bd560b2 100644
--- a/tests/disas/aarch64-entry-trampoline.wat
+++ b/tests/disas/aarch64-entry-trampoline.wat
@@ -16,19 +16,22 @@
;; stp d12, d13, [sp, #-0x10]!
;; stp d10, d11, [sp, #-0x10]!
;; stp d8, d9, [sp, #-0x10]!
+;; sub sp, sp, #0x10
+;; mov x3, x1
;; mov x12, x29
-;; ldr x11, [x0, #8]
-;; str x12, [x11, #0x48]
-;; mov x12, sp
-;; str x12, [x11, #0x40]
-;; adr x13, #0x94
-;; str x13, [x11, #0x50]
+;; ldr x15, [x0, #8]
+;; str x12, [x15, #0x48]
+;; mov x13, sp
+;; str x13, [x15, #0x40]
+;; adr x14, #0xa0
+;; str x14, [x15, #0x50]
;; mov x2, x0
-;; mov x3, x1
+;; stur x15, [sp]
;; bl #0
-;; ├─╼ exception frame offset: SP = FP - 0x90
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x94
-;; 64: mov w0, #1
+;; ├─╼ exception frame offset: SP = FP - 0xa0
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xa0
+;; 6c: mov w0, #1
+;; add sp, sp, #0x10
;; ldp d8, d9, [sp], #0x10
;; ldp d10, d11, [sp], #0x10
;; ldp d12, d13, [sp], #0x10
@@ -40,15 +43,19 @@
;; ldp x27, x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; 94: mov w0, #0
-;; 98: ldp d8, d9, [sp], #0x10
-;; 9c: ldp d10, d11, [sp], #0x10
-;; a0: ldp d12, d13, [sp], #0x10
-;; a4: ldp d14, d15, [sp], #0x10
-;; a8: ldp x19, x20, [sp], #0x10
-;; ac: ldp x21, x22, [sp], #0x10
-;; b0: ldp x23, x24, [sp], #0x10
-;; b4: ldp x25, x26, [sp], #0x10
-;; b8: ldp x27, x28, [sp], #0x10
-;; bc: ldp x29, x30, [sp], #0x10
-;; c0: ret
+;; a0: mov x0, #1
+;; a4: ldur x15, [sp]
+;; a8: str x0, [x15, #0x88]
+;; ac: mov w0, #0
+;; b0: add sp, sp, #0x10
+;; b4: ldp d8, d9, [sp], #0x10
+;; b8: ldp d10, d11, [sp], #0x10
+;; bc: ldp d12, d13, [sp], #0x10
+;; c0: ldp d14, d15, [sp], #0x10
+;; c4: ldp x19, x20, [sp], #0x10
+;; c8: ldp x21, x22, [sp], #0x10
+;; cc: ldp x23, x24, [sp], #0x10
+;; d0: ldp x25, x26, [sp], #0x10
+;; d4: ldp x27, x28, [sp], #0x10
+;; d8: ldp x29, x30, [sp], #0x10
+;; dc: ret
diff --git a/tests/disas/component-model/direct-adapter-calls-inlining.wat b/tests/disas/component-model/direct-adapter-calls-inlining.wat
index dd020c1cea..9d10e2490b 100644
--- a/tests/disas/component-model/direct-adapter-calls-inlining.wat
+++ b/tests/disas/component-model/direct-adapter-calls-inlining.wat
@@ -60,9 +60,9 @@
;; region2 = 72 "VMContext+0x48"
;; region3 = 136 "VMContext+0x88"
;; region4 = 1610612736 "PublicGlobal"
-;; region5 = 104 "VMContext+0x68"
-;; region6 = 88 "VMContext+0x58"
-;; region7 = 112 "VMContext+0x70"
+;; region5 = 112 "VMContext+0x70"
+;; region6 = 104 "VMContext+0x68"
+;; region7 = 88 "VMContext+0x58"
;; gv0 = vmctx
;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
;; gv2 = load.i64 notrap aligned region1 gv1+24
@@ -92,25 +92,23 @@
;; @00ee v3 = load.i64 notrap aligned readonly can_move region2 v0+72
;; v9 = load.i64 notrap aligned readonly can_move region3 v3+136
;; v10 = load.i32 notrap aligned region4 v9
-;; v8 = iconst.i32 0
-;; v12 = icmp eq v10, v8 ; v8 = 0
-;; brif v12, block9, block10
+;; brif v10, block9, block10
;;
-;; block9:
-;; v16 = load.i64 notrap aligned readonly can_move region6 v3+88
-;; v15 = load.i64 notrap aligned readonly can_move region5 v3+104
-;; v14 = iconst.i32 23
-;; try_call_indirect v16(v15, v3, v14), sig1, block11, [ context v3, default: block8(exn0) ] ; v14 = 23
+;; block10:
+;; v24 = load.i64 notrap aligned readonly can_move region7 v3+88
+;; v23 = load.i64 notrap aligned readonly can_move region6 v3+104
+;; v22 = iconst.i32 23
+;; try_call_indirect v24(v23, v3, v22), sig1, block11, [ context v3, default: block8(exn0) ] ; v22 = 23
;;
;; block11:
;; trap user12
;;
-;; block10:
-;; v21 = load.i64 notrap aligned readonly can_move region7 v3+112
-;; v22 = load.i32 notrap aligned region4 v21
-;; v38 = iconst.i32 0
-;; store notrap aligned region4 v38, v21 ; v38 = 0
-;; store notrap aligned region4 v22, v21
+;; block9:
+;; v11 = load.i64 notrap aligned readonly can_move region5 v3+112
+;; v12 = load.i32 notrap aligned region4 v11
+;; v8 = iconst.i32 0
+;; store notrap aligned region4 v8, v11 ; v8 = 0
+;; store notrap aligned region4 v12, v11
;; jump block13
;;
;; block13:
@@ -120,8 +118,8 @@
;; jump block12
;;
;; block12:
-;; v39 = iconst.i32 0
-;; store notrap aligned region4 v39, v9 ; v39 = 0
+;; v35 = iconst.i32 0
+;; store notrap aligned region4 v35, v9 ; v35 = 0
;; store.i32 notrap aligned region4 v10, v9
;; jump block7
;;
@@ -129,8 +127,8 @@
;; jump block4
;;
;; block5:
-;; v18 = iconst.i32 49
-;; call_indirect.i64 sig1, v16(v15, v3, v18) ; v18 = 49
+;; v26 = iconst.i32 49
+;; call_indirect.i64 sig1, v24(v23, v3, v26) ; v26 = 49
;; trap user12
;;
;; block4:
@@ -143,6 +141,6 @@
;; @00f0 jump block1
;;
;; block1:
-;; v32 = iconst.i32 1276
-;; @00f0 return v32 ; v32 = 1276
+;; v29 = iconst.i32 1276
+;; @00f0 return v29 ; v29 = 1276
;; }
diff --git a/tests/disas/component-model/direct-adapter-calls-x64.wat b/tests/disas/component-model/direct-adapter-calls-x64.wat
index 8dd507783d..27552042cf 100644
--- a/tests/disas/component-model/direct-adapter-calls-x64.wat
+++ b/tests/disas/component-model/direct-adapter-calls-x64.wat
@@ -87,7 +87,7 @@
;; movq 0x18(%r10), %r10
;; addq $0x60, %r10
;; cmpq %rsp, %r10
-;; ja 0x13e
+;; ja 0x140
;; 79: subq $0x50, %rsp
;; movq %rbx, 0x20(%rsp)
;; movq %r12, 0x28(%rsp)
@@ -97,25 +97,33 @@
;; movq %rdi, (%rsp)
;; movq (%rsp), %rdi
;; movq 0x88(%rdi), %rcx
-;; movl (%rcx), %esi
+;; movl (%rcx), %eax
;; movq %rcx, 0x10(%rsp)
-;; testl %esi, %esi
-;; movq %rsi, 8(%rsp)
-;; je 0x10c
+;; testl %eax, %eax
+;; movq %rax, 8(%rsp)
+;; jne 0xd5
;; b9: movq (%rsp), %rdi
-;; movq 0x70(%rdi), %rax
+;; movq 0x58(%rdi), %rax
+;; movq 0x68(%rdi), %rdi
+;; movl $0x17, %edx
+;; movq (%rsp), %rsi
+;; callq *%rax
+;; ├─╼ exception frame offset: SP = FP - 0x50
+;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x12b
+;; jmp 0x129
+;; d5: movq (%rsp), %rsi
+;; movq 0x70(%rsi), %rax
;; movl (%rax), %ecx
;; movl $0, (%rax)
;; movl %ecx, (%rax)
-;; movq 0x48(%rdi), %rdi
-;; movq (%rsp), %rsi
+;; movq 0x48(%rsi), %rdi
;; callq 0
;; ├─╼ exception frame offset: SP = FP - 0x50
-;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x125
+;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x12b
;; movq 0x10(%rsp), %rcx
;; movl $0, (%rcx)
-;; movq 8(%rsp), %rsi
-;; movl %esi, (%rcx)
+;; movq 8(%rsp), %rdx
+;; movl %edx, (%rcx)
;; movq 0x20(%rsp), %rbx
;; movq 0x28(%rsp), %r12
;; movq 0x30(%rsp), %r13
@@ -125,20 +133,12 @@
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 10c: movq (%rsp), %rcx
-;; 110: movq 0x58(%rcx), %rax
-;; 114: movq 0x68(%rcx), %rdi
-;; 118: movl $0x17, %edx
-;; 11d: movq (%rsp), %rsi
-;; 121: callq *%rax
-;; ├─╼ exception frame offset: SP = FP - 0x50
-;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x125
-;; 123: ud2
-;; 125: movq (%rsp), %rcx
-;; 129: movq 0x58(%rcx), %rax
-;; 12d: movq 0x68(%rcx), %rdi
-;; 131: movl $0x31, %edx
-;; 136: movq (%rsp), %rsi
-;; 13a: callq *%rax
-;; 13c: ud2
+;; 124: jmp 0x12b
+;; 129: ud2
+;; 12b: movq (%rsp), %rsi
+;; 12f: movq 0x58(%rsi), %rax
+;; 133: movq 0x68(%rsi), %rdi
+;; 137: movl $0x31, %edx
+;; 13c: callq *%rax
;; 13e: ud2
+;; 140: ud2
diff --git a/tests/disas/component-model/direct-adapter-calls.wat b/tests/disas/component-model/direct-adapter-calls.wat
index 0825e243a0..ae4b07d4d9 100644
--- a/tests/disas/component-model/direct-adapter-calls.wat
+++ b/tests/disas/component-model/direct-adapter-calls.wat
@@ -119,49 +119,47 @@
;; @007b jump block3
;;
;; block4:
-;; @0080 v6 = load.i64 notrap aligned readonly can_move region2 v0+136
-;; @0080 v7 = load.i32 notrap aligned region3 v6
-;; @0075 v3 = iconst.i32 0
-;; @0084 v9 = icmp eq v7, v3 ; v3 = 0
-;; @0085 brif v9, block7, block8
+;; @0082 v6 = load.i64 notrap aligned readonly can_move region2 v0+136
+;; @0082 v7 = load.i32 notrap aligned region3 v6
+;; @0086 brif v7, block7, block8
;;
-;; block7:
-;; @0089 v13 = load.i64 notrap aligned readonly can_move region5 v0+88
-;; @0089 v12 = load.i64 notrap aligned readonly can_move region4 v0+104
-;; @0087 v11 = iconst.i32 23
-;; @0089 try_call_indirect v13(v12, v0, v11), sig0, block9, [ context v0, default: block6(exn0) ] ; v11 = 23
+;; block8:
+;; @008a v10 = load.i64 notrap aligned readonly can_move region5 v0+88
+;; @008a v9 = load.i64 notrap aligned readonly can_move region4 v0+104
+;; @0088 v8 = iconst.i32 23
+;; @008a try_call_indirect v10(v9, v0, v8), sig0, block9, [ context v0, default: block6(exn0) ] ; v8 = 23
;;
;; block9:
-;; @008b trap user12
+;; @008c trap user12
;;
-;; block8:
-;; @008d v14 = load.i64 notrap aligned readonly can_move region6 v0+112
-;; @008d v15 = load.i32 notrap aligned region3 v14
-;; v27 = iconst.i32 0
-;; @0093 store notrap aligned region3 v27, v14 ; v27 = 0
-;; @0099 store notrap aligned region3 v15, v14
-;; @009b v19 = load.i64 notrap aligned readonly can_move region7 v0+72
-;; @009b try_call fn0(v19, v0, v2), sig1, block10(ret0), [ context v0, default: block6(exn0) ]
-;;
-;; block10(v20: i32):
-;; v28 = iconst.i32 0
-;; @00a1 store notrap aligned region3 v28, v6 ; v28 = 0
-;; @00a7 store.i32 notrap aligned region3 v7, v6
-;; @00a9 jump block5
+;; block7:
+;; @008e v11 = load.i64 notrap aligned readonly can_move region6 v0+112
+;; @008e v12 = load.i32 notrap aligned region3 v11
+;; @0075 v3 = iconst.i32 0
+;; @0094 store notrap aligned region3 v3, v11 ; v3 = 0
+;; @009a store notrap aligned region3 v12, v11
+;; @009c v16 = load.i64 notrap aligned readonly can_move region7 v0+72
+;; @009c try_call fn0(v16, v0, v2), sig1, block10(ret0), [ context v0, default: block6(exn0) ]
+;;
+;; block10(v17: i32):
+;; v24 = iconst.i32 0
+;; @00a2 store notrap aligned region3 v24, v6 ; v24 = 0
+;; @00a8 store.i32 notrap aligned region3 v7, v6
+;; @00aa jump block5
;;
;; block5:
-;; @00aa jump block2
+;; @00ab jump block2
;;
;; block3:
-;; v29 = load.i64 notrap aligned readonly can_move region5 v0+88
-;; v30 = load.i64 notrap aligned readonly can_move region4 v0+104
-;; @00ad v24 = iconst.i32 49
-;; @00af call_indirect sig0, v29(v30, v0, v24) ; v24 = 49
-;; @00b1 trap user12
+;; v25 = load.i64 notrap aligned readonly can_move region5 v0+88
+;; v26 = load.i64 notrap aligned readonly can_move region4 v0+104
+;; @00ae v21 = iconst.i32 49
+;; @00b0 call_indirect sig0, v25(v26, v0, v21) ; v21 = 49
+;; @00b2 trap user12
;;
;; block2:
-;; @00b3 jump block1
+;; @00b4 jump block1
;;
;; block1:
-;; @00b3 return v20
+;; @00b4 return v17
;; }
diff --git a/tests/disas/component-model/sync-adapter-calls.wat b/tests/disas/component-model/sync-adapter-calls.wat
new file mode 100644
index 0000000000..f2a1058e82
--- /dev/null
+++ b/tests/disas/component-model/sync-adapter-calls.wat
@@ -0,0 +1,330 @@
+;;! target = "x86_64"
+;;! test = "optimize"
+;;! filter = "function"
+;;! flags = "-C inlining=y -Wconcurrency-support=y"
+
+(component
+ (component $A
+ (core module $M
+ (func (export "f'") (param i32) (result i32)
+ (i32.add (local.get 0) (i32.const 42))
+ )
+ )
+
+ (core instance $m (instantiate $M))
+
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'"))
+ )
+ )
+
+ (component $B
+ (import "f" (func $f (param "x" u32) (result u32)))
+
+ (core func $f' (canon lower (func $f)))
+
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (func (export "g'") (result i32)
+ (call $f' (i32.const 1234))
+ )
+ )
+
+ (core instance $n
+ (instantiate $N
+ (with "" (instance (export "f'" (func $f'))))
+ )
+ )
+
+ (func (export "g") (result u32)
+ (canon lift (core func $n "g'"))
+ )
+ )
+
+ (instance $a (instantiate $A))
+ (instance $b
+ (instantiate $B
+ (with "f" (func $a "f"))
+ )
+ )
+
+ (export "g" (func $b "g"))
+)
+;; function u0:0(i64 vmctx, i64, i32) -> i32 tail {
+;; region0 = 8 "VMContext+0x8"
+;; region1 = 268435480 "VMStoreContext+0x18"
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
+;; gv2 = load.i64 notrap aligned region1 gv1+24
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @003b jump block1
+;;
+;; block1:
+;; @0038 v3 = iconst.i32 42
+;; v4 = iadd.i32 v2, v3 ; v3 = 42
+;; @003b return v4
+;; }
+;;
+;; function u1:0(i64 vmctx, i64) -> i32 tail {
+;; ss0 = explicit_slot 32, align = 8
+;; region0 = 8 "VMContext+0x8"
+;; region1 = 268435480 "VMStoreContext+0x18"
+;; region2 = 72 "VMContext+0x48"
+;; region3 = 200 "VMContext+0xc8"
+;; region4 = 1610612736 "PublicGlobal"
+;; region5 = 224 "VMContext+0xe0"
+;; region6 = 136 "VMContext+0x88"
+;; region7 = 268435592 "VMStoreContext+0x88"
+;; region8 = 4026531840 "VMDeferredThread+0x0"
+;; region9 = 4026531848 "VMDeferredThread+0x8"
+;; region10 = 4026531852 "VMDeferredThread+0xc"
+;; region11 = 4026531856 "VMDeferredThread+0x10"
+;; region12 = 268435584 "VMStoreContext+0x80"
+;; region13 = 4026531860 "VMDeferredThread+0x14"
+;; region14 = 268435588 "VMStoreContext+0x84"
+;; region15 = 4026531864 "VMDeferredThread+0x18"
+;; region16 = 176 "VMContext+0xb0"
+;; region17 = 168 "VMContext+0xa8"
+;; region18 = 152 "VMContext+0x98"
+;; region19 = 104 "VMContext+0x68"
+;; region20 = 88 "VMContext+0x58"
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
+;; gv2 = load.i64 notrap aligned region1 gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move region0 gv3+8
+;; gv5 = load.i64 notrap aligned region1 gv4+24
+;; gv6 = vmctx
+;; gv7 = load.i64 notrap aligned readonly can_move region0 gv6+8
+;; gv8 = load.i64 notrap aligned region1 gv7+24
+;; sig0 = (i64 vmctx, i64, i32) -> i32 tail
+;; sig1 = (i64 vmctx, i64, i32) tail
+;; sig2 = (i64 vmctx, i64, i32, i32, i32) tail
+;; sig3 = (i64 vmctx, i64, i32) -> i32 tail
+;; sig4 = (i64 vmctx, i64) tail
+;; fn0 = colocated u2:0 sig0
+;; fn1 = colocated u0:0 sig3
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64):
+;; @00ee jump block2
+;;
+;; block2:
+;; jump block6
+;;
+;; block8(v5: i64):
+;; jump block5
+;;
+;; block6:
+;; @00ee v3 = load.i64 notrap aligned readonly can_move region2 v0+72
+;; v9 = load.i64 notrap aligned readonly can_move region3 v3+200
+;; v10 = load.i32 notrap aligned region4 v9
+;; brif v10, block9, block10
+;;
+;; block10:
+;; v53 = load.i64 notrap aligned readonly can_move region20 v3+88
+;; v52 = load.i64 notrap aligned readonly can_move region19 v3+104
+;; v51 = iconst.i32 23
+;; try_call_indirect v53(v52, v3, v51), sig1, block11, [ context v3, default: block8(exn0) ] ; v51 = 23
+;;
+;; block11:
+;; trap user12
+;;
+;; block9:
+;; v11 = load.i64 notrap aligned readonly can_move region5 v3+224
+;; v12 = load.i32 notrap aligned region4 v11
+;; v8 = iconst.i32 0
+;; store notrap aligned region4 v8, v11 ; v8 = 0
+;; v20 = load.i64 notrap aligned readonly can_move region0 v3+8
+;; v21 = load.i64 notrap aligned region7 v20+136
+;; v19 = stack_addr.i64 ss0
+;; store notrap aligned region8 v21, v19
+;; v15 = iconst.i32 2
+;; store notrap aligned region9 v15, v19+8 ; v15 = 2
+;; store notrap aligned region10 v8, v19+12 ; v8 = 0
+;; v17 = iconst.i32 1
+;; store notrap aligned region11 v17, v19+16 ; v17 = 1
+;; v22 = load.i32 notrap aligned region12 v20+128
+;; store notrap aligned region13 v22, v19+20
+;; store notrap aligned region12 v8, v20+128 ; v8 = 0
+;; v24 = load.i32 notrap aligned region14 v20+132
+;; store notrap aligned region15 v24, v19+24
+;; store notrap aligned region14 v8, v20+132 ; v8 = 0
+;; store notrap aligned region7 v19, v20+136
+;; v26 = load.i64 notrap aligned readonly can_move region16 v3+176
+;; v27 = load.i32 notrap aligned region4 v26
+;; store notrap aligned region4 v8, v26 ; v8 = 0
+;; store notrap aligned region4 v27, v26
+;; jump block17
+;;
+;; block17:
+;; jump block18
+;;
+;; block18:
+;; jump block12
+;;
+;; block12:
+;; jump block13
+;;
+;; block13:
+;; store.i64 notrap aligned region7 v21, v20+136
+;; store.i32 notrap aligned region12 v22, v20+128
+;; store.i32 notrap aligned region14 v24, v20+132
+;; jump block15
+;;
+;; block15:
+;; v61 = iconst.i32 0
+;; store notrap aligned region4 v61, v9 ; v61 = 0
+;; store.i32 notrap aligned region4 v10, v9
+;; store.i32 notrap aligned region4 v12, v11
+;; jump block7
+;;
+;; block7:
+;; jump block4
+;;
+;; block5:
+;; v62 = load.i64 notrap aligned readonly can_move region20 v3+88
+;; v63 = load.i64 notrap aligned readonly can_move region19 v3+104
+;; v48 = iconst.i32 49
+;; call_indirect sig1, v62(v63, v3, v48) ; v48 = 49
+;; trap user12
+;;
+;; block4:
+;; jump block3
+;;
+;; block3:
+;; jump block19
+;;
+;; block19:
+;; @00f0 jump block1
+;;
+;; block1:
+;; v54 = iconst.i32 1276
+;; @00f0 return v54 ; v54 = 1276
+;; }
+;;
+;; function u2:0(i64 vmctx, i64, i32) -> i32 tail {
+;; ss0 = explicit_slot 32, align = 8
+;; region0 = 8 "VMContext+0x8"
+;; region1 = 268435480 "VMStoreContext+0x18"
+;; region2 = 200 "VMContext+0xc8"
+;; region3 = 1610612736 "PublicGlobal"
+;; region4 = 104 "VMContext+0x68"
+;; region5 = 88 "VMContext+0x58"
+;; region6 = 224 "VMContext+0xe0"
+;; region7 = 136 "VMContext+0x88"
+;; region8 = 268435592 "VMStoreContext+0x88"
+;; region9 = 4026531840 "VMDeferredThread+0x0"
+;; region10 = 4026531848 "VMDeferredThread+0x8"
+;; region11 = 4026531852 "VMDeferredThread+0xc"
+;; region12 = 4026531856 "VMDeferredThread+0x10"
+;; region13 = 268435584 "VMStoreContext+0x80"
+;; region14 = 4026531860 "VMDeferredThread+0x14"
+;; region15 = 268435588 "VMStoreContext+0x84"
+;; region16 = 4026531864 "VMDeferredThread+0x18"
+;; region17 = 176 "VMContext+0xb0"
+;; region18 = 72 "VMContext+0x48"
+;; region19 = 168 "VMContext+0xa8"
+;; region20 = 152 "VMContext+0x98"
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
+;; gv2 = load.i64 notrap aligned region1 gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move region0 gv3+8
+;; gv5 = load.i64 notrap aligned region1 gv4+24
+;; sig0 = (i64 vmctx, i64, i32) tail
+;; sig1 = (i64 vmctx, i64, i32, i32, i32) tail
+;; sig2 = (i64 vmctx, i64, i32) -> i32 tail
+;; sig3 = (i64 vmctx, i64) tail
+;; fn0 = colocated u0:0 sig2
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @00cf jump block4
+;;
+;; block6(v4: i64):
+;; @00cf jump block3
+;;
+;; block4:
+;; @00d6 v6 = load.i64 notrap aligned readonly can_move region2 v0+200
+;; @00d6 v7 = load.i32 notrap aligned region3 v6
+;; @00da brif v7, block7, block8
+;;
+;; block8:
+;; @00de v10 = load.i64 notrap aligned readonly can_move region5 v0+88
+;; @00de v9 = load.i64 notrap aligned readonly can_move region4 v0+104
+;; @00dc v8 = iconst.i32 23
+;; @00de try_call_indirect v10(v9, v0, v8), sig0, block9, [ context v0, default: block6(exn0) ] ; v8 = 23
+;;
+;; block9:
+;; @00e0 trap user12
+;;
+;; block7:
+;; @00e2 v11 = load.i64 notrap aligned readonly can_move region6 v0+224
+;; @00e2 v12 = load.i32 notrap aligned region3 v11
+;; @00c9 v3 = iconst.i32 0
+;; @00e8 store notrap aligned region3 v3, v11 ; v3 = 0
+;; @00f0 v20 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @00f0 v21 = load.i64 notrap aligned region8 v20+136
+;; @00f0 v19 = stack_addr.i64 ss0
+;; @00f0 store notrap aligned region9 v21, v19
+;; @00ea v15 = iconst.i32 2
+;; @00f0 store notrap aligned region10 v15, v19+8 ; v15 = 2
+;; @00f0 store notrap aligned region11 v3, v19+12 ; v3 = 0
+;; @00ee v17 = iconst.i32 1
+;; @00f0 store notrap aligned region12 v17, v19+16 ; v17 = 1
+;; @00f0 v22 = load.i32 notrap aligned region13 v20+128
+;; @00f0 store notrap aligned region14 v22, v19+20
+;; @00f0 store notrap aligned region13 v3, v20+128 ; v3 = 0
+;; @00f0 v24 = load.i32 notrap aligned region15 v20+132
+;; @00f0 store notrap aligned region16 v24, v19+24
+;; @00f0 store notrap aligned region15 v3, v20+132 ; v3 = 0
+;; @00f0 store notrap aligned region8 v19, v20+136
+;; @00f2 v26 = load.i64 notrap aligned readonly can_move region17 v0+176
+;; @00f2 v27 = load.i32 notrap aligned region3 v26
+;; @00f8 store notrap aligned region3 v3, v26 ; v3 = 0
+;; @00fe store notrap aligned region3 v27, v26
+;; @0100 jump block15
+;;
+;; block15:
+;; jump block16
+;;
+;; block16:
+;; jump block10
+;;
+;; block10:
+;; @0104 jump block11
+;;
+;; block11:
+;; @0104 store.i64 notrap aligned region8 v21, v20+136
+;; @0104 store.i32 notrap aligned region13 v22, v20+128
+;; @0104 store.i32 notrap aligned region15 v24, v20+132
+;; @0104 jump block13
+;;
+;; block13:
+;; v55 = iconst.i32 0
+;; @0108 store notrap aligned region3 v55, v6 ; v55 = 0
+;; @010e store.i32 notrap aligned region3 v7, v6
+;; @0112 store.i32 notrap aligned region3 v12, v11
+;; @0114 jump block5
+;;
+;; block5:
+;; @0115 jump block2
+;;
+;; block3:
+;; v56 = load.i64 notrap aligned readonly can_move region5 v0+88
+;; v57 = load.i64 notrap aligned readonly can_move region4 v0+104
+;; @0118 v49 = iconst.i32 49
+;; @011a call_indirect sig0, v56(v57, v0, v49) ; v49 = 49
+;; @011c trap user12
+;;
+;; block2:
+;; @011e jump block1
+;;
+;; block1:
+;; v52 = iconst.i32 42
+;; v53 = iadd.i32 v2, v52 ; v52 = 42
+;; @011e return v53
+;; }
diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.wat
index 327084f2f5..d6343a619b 100644
--- a/tests/disas/debug-exceptions.wat
+++ b/tests/disas/debug-exceptions.wat
@@ -37,29 +37,29 @@
;; stur x2, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x35, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [62, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [69, 1, 0, 148]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [60, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [67, 1, 0, 148]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3d, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [58, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [65, 1, 0, 148]
;; mov w1, #0x2a
;; stur w1, [sp, #8]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3f, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [55, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [62, 1, 0, 148]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x40, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [54, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [61, 1, 0, 148]
;; stur w1, [sp, #8]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [52, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [59, 1, 0, 148]
;; ldur x2, [sp, #0x10]
-;; bl #0x4b4
+;; bl #0x4d0
;; 88: ldur x0, [sp, #0x10]
;; mov x19, x2
;; ldr x1, [x0, #0x20]
@@ -92,7 +92,7 @@
;; mov w11, #0
;; str w11, [x1, #0x14]
;; ldur x2, [sp, #0x10]
-;; bl #0x4ec
+;; bl #0x508
;; ├─╼ exception frame offset: SP = FP - 0xb0
;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x110
;; b #0x1d8
@@ -113,7 +113,7 @@
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x48, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [0, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [7, 1, 0, 148]
;; ldur x1, [sp, #0x10]
;; ldr x0, [x1, #0x38]
;; ldr x2, [x1, #0x48]
@@ -123,10 +123,10 @@
;; 164: ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [249, 0, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [0, 1, 0, 148]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4b, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [248, 0, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [255, 0, 0, 148]
;; add sp, sp, #0x20
;; ldp d8, d9, [sp], #0x10
;; ldp d10, d11, [sp], #0x10
@@ -147,7 +147,7 @@
;; 1b4: mov w5, #0x20
;; 1b8: mov w6, #0x10
;; 1bc: ldur x2, [sp, #0x10]
-;; 1c0: bl #0x3e0
+;; 1c0: bl #0x3fc
;; 1c4: ldur x0, [sp, #0x18]
;; 1c8: ldr x3, [x0, #0x20]
;; 1cc: add x1, x3, w2, uxtw
@@ -155,29 +155,29 @@
;; 1d4: b #0xec
;; 1d8: mov w3, #9
;; 1dc: ldur x2, [sp, #0x10]
-;; 1e0: bl #0x448
+;; 1e0: bl #0x464
;; 1e4: ldur x2, [sp, #0x10]
-;; 1e8: bl #0x480
+;; 1e8: bl #0x49c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
;; 1ec: udf #0xc11f
;; 1f0: mov w3, #0xfe
;; 1f4: ldur x2, [sp, #0x10]
-;; 1f8: bl #0x448
+;; 1f8: bl #0x464
;; 1fc: ldur x2, [sp, #0x10]
-;; 200: bl #0x480
+;; 200: bl #0x49c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
;; 204: udf #0xc11f
;; 208: mov w3, #0xfe
;; 20c: ldur x2, [sp, #0x10]
-;; 210: bl #0x448
+;; 210: bl #0x464
;; 214: ldur x2, [sp, #0x10]
-;; 218: bl #0x480
+;; 218: bl #0x49c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
;; 21c: udf #0xc11f
;; 220: stur x2, [sp, #0x10]
;; 224: mov w3, #0
-;; 228: bl #0x448
+;; 228: bl #0x464
;; 22c: ldur x2, [sp, #0x10]
-;; 230: bl #0x480
+;; 230: bl #0x49c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x34, slot at FP-0xb0, locals , stack
;; 234: udf #0xc11f
diff --git a/tests/disas/debug.wat b/tests/disas/debug.wat
index abbcc23c97..74f254de13 100644
--- a/tests/disas/debug.wat
+++ b/tests/disas/debug.wat
@@ -23,20 +23,20 @@
;; 29: movq %rdi, (%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x24, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack
-;; ╰─╼ breakpoint patch: wasm PC 0x24, patch bytes [232, 190, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x24, patch bytes [232, 212, 1, 0, 0]
;; movl %edx, 0x10(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x26, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10
-;; ╰─╼ breakpoint patch: wasm PC 0x26, patch bytes [232, 181, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x26, patch bytes [232, 203, 1, 0, 0]
;; movl %ecx, 0x14(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x28, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10, I32 @ slot+0x14
-;; ╰─╼ breakpoint patch: wasm PC 0x28, patch bytes [232, 172, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x28, patch bytes [232, 194, 1, 0, 0]
;; leal (%rdx, %rcx), %eax
;; movl %eax, 0x10(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x29, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10
-;; ╰─╼ breakpoint patch: wasm PC 0x29, patch bytes [232, 160, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x29, patch bytes [232, 182, 1, 0, 0]
;; movl %eax, 0x10(%rsp)
;; movq 0x20(%rsp), %r12
;; addq $0x30, %rsp
@@ -45,9 +45,9 @@
;; retq
;; 62: movq %rdi, %r12
;; 65: xorl %esi, %esi
-;; 67: callq 0x192
+;; 67: callq 0x1a8
;; 6c: movq %r12, %rdi
-;; 6f: callq 0x1c3
+;; 6f: callq 0x1d9
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x23, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack
;; 74: ud2
;;
@@ -60,20 +60,21 @@
;; movq %r13, 0x20(%rsp)
;; movq %r14, 0x28(%rsp)
;; movq %r15, 0x30(%rsp)
-;; movl (%rdx), %eax
+;; movl (%rdx), %r8d
;; movl 0x10(%rdx), %ecx
;; movq %rdx, (%rsp)
;; movq %rbp, %r9
-;; movq 8(%rdi), %r8
-;; movq %r9, 0x48(%r8)
+;; movq 8(%rdi), %r10
+;; movq %r9, 0x48(%r10)
;; movq %rsp, %r9
-;; movq %r9, 0x40(%r8)
-;; leaq 0x39(%rip), %r9
-;; movq %r9, 0x50(%r8)
-;; movq %rax, %rdx
+;; movq %r9, 0x40(%r10)
+;; leaq 0x3e(%rip), %r9
+;; movq %r9, 0x50(%r10)
+;; movq %r8, %rdx
+;; movq %r10, 8(%rsp)
;; callq 0
;; ├─╼ exception frame offset: SP = FP - 0x40
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xf2
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xf8
;; movq (%rsp), %rdx
;; movl %eax, (%rdx)
;; movl $1, %eax
@@ -86,16 +87,18 @@
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; f2: xorl %eax, %eax
-;; f4: movq 0x10(%rsp), %rbx
-;; f9: movq 0x18(%rsp), %r12
-;; fe: movq 0x20(%rsp), %r13
-;; 103: movq 0x28(%rsp), %r14
-;; 108: movq 0x30(%rsp), %r15
-;; 10d: addq $0x40, %rsp
-;; 111: movq %rbp, %rsp
-;; 114: popq %rbp
-;; 115: retq
+;; f8: movq 8(%rsp), %r10
+;; fd: movq $1, 0x88(%r10)
+;; 108: xorl %eax, %eax
+;; 10a: movq 0x10(%rsp), %rbx
+;; 10f: movq 0x18(%rsp), %r12
+;; 114: movq 0x20(%rsp), %r13
+;; 119: movq 0x28(%rsp), %r14
+;; 11e: movq 0x30(%rsp), %r15
+;; 123: addq $0x40, %rsp
+;; 127: movq %rbp, %rsp
+;; 12a: popq %rbp
+;; 12b: retq
;;
;; signatures[0]::wasm_to_array_trampoline:
;; pushq %rbp
@@ -120,19 +123,19 @@
;; callq *%rax
;; addq $1, 0x10(%r15)
;; testb %al, %al
-;; je 0x180
-;; 16a: movl (%rsp), %eax
+;; je 0x196
+;; 180: movl (%rsp), %eax
;; movq 0x20(%rsp), %rbx
;; movq 0x28(%rsp), %r15
;; addq $0x30, %rsp
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 180: movq 0x10(%rbx), %rax
-;; 184: movq 0x148(%rax), %rax
-;; 18b: movq %rbx, %rdi
-;; 18e: callq *%rax
-;; 190: ud2
+;; 196: movq 0x10(%rbx), %rax
+;; 19a: movq 0x148(%rax), %rax
+;; 1a1: movq %rbx, %rdi
+;; 1a4: callq *%rax
+;; 1a6: ud2
;;
;; wasmtime_builtin_trap:
;; pushq %rbp
@@ -209,8 +212,8 @@
;; movq 0x170(%r12), %r11
;; callq *%r11
;; testb %al, %al
-;; je 0x3c7
-;; 2f3: movq (%rsp), %rax
+;; je 0x3dd
+;; 309: movq (%rsp), %rax
;; movq 8(%rsp), %rcx
;; movq 0x10(%rsp), %rdx
;; movq 0x18(%rsp), %rsi
@@ -241,7 +244,7 @@
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 3c7: movq 0x148(%r12), %rax
-;; 3cf: movq %r13, %rdi
-;; 3d2: callq *%rax
-;; 3d4: ud2
+;; 3dd: movq 0x148(%r12), %rax
+;; 3e5: movq %r13, %rdi
+;; 3e8: callq *%rax
+;; 3ea: ud2
diff --git a/tests/disas/exceptions.wat b/tests/disas/exceptions.wat
index efee2b2931..a544f5701b 100644
--- a/tests/disas/exceptions.wat
+++ b/tests/disas/exceptions.wat
@@ -33,7 +33,7 @@
;; movq %rdi, %rbx
;; movq %rcx, %r13
;; movq %rdx, %r14
-;; callq 0x3eb
+;; callq 0x401
;; movq %rax, %r12
;; movq 0x20(%rbx), %rdx
;; movl (%rdx), %esi
@@ -62,7 +62,7 @@
;; movl $0, 0x14(%rdx)
;; movq %rbx, %rdi
;; movq %rbx, (%rsp)
-;; callq 0x418
+;; callq 0x42e
;; jmp 0xea
;; b6: movl $0x4000002, %esi
;; movq 0x28(%rbx), %rax
@@ -70,7 +70,7 @@
;; movl $0x30, %ecx
;; movl $0x10, %r8d
;; movq %rbx, %rdi
-;; callq 0x382
+;; callq 0x398
;; movq 8(%rbx), %rcx
;; movl %eax, %edx
;; addq 0x20(%rcx), %rdx
diff --git a/tests/disas/gc/issue-11753.wat b/tests/disas/gc/issue-11753.wat
index 43061982a8..bfcf95eeb8 100644
--- a/tests/disas/gc/issue-11753.wat
+++ b/tests/disas/gc/issue-11753.wat
@@ -94,7 +94,7 @@
;; f4: movl (%rax), %edx
;; f6: movl $0x20, %ecx
;; fb: movl $0x10, %r8d
-;; 101: callq 0x271
+;; 101: callq 0x286
;; 106: movq 8(%rsp), %r9
;; 10b: movq 8(%r9), %rcx
;; 10f: movl %eax, %esi
diff --git a/tests/disas/issue-12808.wat b/tests/disas/issue-12808.wat
index e638c097ec..37c57321f5 100644
--- a/tests/disas/issue-12808.wat
+++ b/tests/disas/issue-12808.wat
@@ -23,23 +23,23 @@
;; 21: movq %rdi, (%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x1f, slot at FP-0x30, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x1f, patch bytes [232, 233, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x1f, patch bytes [232, 254, 1, 0, 0]
;; movl $0, 8(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x21, slot at FP-0x30, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x21, patch bytes [232, 220, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x21, patch bytes [232, 241, 1, 0, 0]
;; movl $0, 0xc(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x23, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc
-;; ╰─╼ breakpoint patch: wasm PC 0x23, patch bytes [232, 207, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x23, patch bytes [232, 228, 1, 0, 0]
;; movl $0, 0x10(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x25, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc, I32 @ slot+0x10
-;; ╰─╼ breakpoint patch: wasm PC 0x25, patch bytes [232, 194, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x25, patch bytes [232, 215, 1, 0, 0]
;; movl $0, 0x14(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x27, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc, I32 @ slot+0x10, I32 @ slot+0x14
-;; ╰─╼ breakpoint patch: wasm PC 0x27, patch bytes [232, 181, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x27, patch bytes [232, 202, 1, 0, 0]
;; movq 0x30(%rdi), %rax
;; movq (%rax), %rcx
;; xorl %eax, %eax
@@ -48,7 +48,7 @@
;; movq %rax, %rcx
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x2b, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc
-;; ╰─╼ breakpoint patch: wasm PC 0x2b, patch bytes [232, 156, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x2b, patch bytes [232, 177, 1, 0, 0]
;; xorl %eax, %eax
;; movl $0, 8(%rsp)
;; movl %ecx, 0xc(%rsp)
@@ -59,8 +59,8 @@
;; retq
;; 93: movq %rdi, %r14
;; 96: xorl %esi, %esi
-;; 98: callq 0x1b5
+;; 98: callq 0x1ca
;; 9d: movq %r14, %rdi
-;; a0: callq 0x1e6
+;; a0: callq 0x1fb
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x1e, slot at FP-0x30, locals , stack
;; a5: ud2
diff --git a/tests/disas/pulley-entry-trampoline.wat b/tests/disas/pulley-entry-trampoline.wat
index 891981b48e..06f18fd4c9 100644
--- a/tests/disas/pulley-entry-trampoline.wat
+++ b/tests/disas/pulley-entry-trampoline.wat
@@ -5,20 +5,24 @@
(module (func (export "")))
;; wasm[0]::array_to_wasm_trampoline[0]:
-;; push_frame_save 128, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+;; push_frame_save 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
;; xmov_fp x12
-;; xload64le_o32 x11, x0, 8
-;; xstore64le_o32 x11, 72, x12
-;; xmov x12, sp
-;; xstore64le_o32 x11, 64, x12
-;; xpcadd x13, 0x1c // target = 0x40
-;; xstore64le_o32 x11, 80, x13
-;; call -0x33 // target = 0x0
-;; ├─╼ exception frame offset: SP = FP - 0x80
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x40
+;; xload64le_o32 x15, x0, 8
+;; xstore64le_o32 x15, 72, x12
+;; xmov x13, sp
+;; xstore64le_o32 x15, 64, x13
+;; xpcadd x14, 0x23 // target = 0x47
+;; xstore64le_o32 x15, 80, x14
+;; xstore64le_o32 sp, 0, x15
+;; call -0x3a // target = 0x0
+;; ├─╼ exception frame offset: SP = FP - 0x90
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x47
;; xone x0
-;; pop_frame_restore 128, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+;; pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
;; ret
-;; 40: xzero x0
-;; 42: pop_frame_restore 128, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
-;; 47: ret
+;; 47: xone x0
+;; 49: xload64le_o32 x15, sp, 0
+;; 50: xstore64le_o32 x15, 136, x0
+;; 57: xzero x0
+;; 59: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+;; 5e: ret
diff --git a/tests/disas/riscv64-entry-trampoline.wat b/tests/disas/riscv64-entry-trampoline.wat
index dea9c1e05d..620f6cab6c 100644
--- a/tests/disas/riscv64-entry-trampoline.wat
+++ b/tests/disas/riscv64-entry-trampoline.wat
@@ -9,96 +9,100 @@
;; sd ra, 8(sp)
;; sd s0, 0(sp)
;; mv s0, sp
-;; addi sp, sp, -0xc0
-;; sd s0, 0xb8(sp)
-;; sd s1, 0xb0(sp)
-;; sd s2, 0xa8(sp)
-;; sd s3, 0xa0(sp)
-;; sd s4, 0x98(sp)
-;; sd s5, 0x90(sp)
-;; sd s6, 0x88(sp)
-;; sd s7, 0x80(sp)
-;; sd s8, 0x78(sp)
-;; sd s9, 0x70(sp)
-;; sd s10, 0x68(sp)
-;; sd s11, 0x60(sp)
-;; fsd fs0, 0x58(sp)
-;; fsd fs2, 0x50(sp)
-;; fsd fs3, 0x48(sp)
-;; fsd fs4, 0x40(sp)
-;; fsd fs5, 0x38(sp)
-;; fsd fs6, 0x30(sp)
-;; fsd fs7, 0x28(sp)
-;; fsd fs8, 0x20(sp)
-;; fsd fs9, 0x18(sp)
-;; fsd fs10, 0x10(sp)
-;; fsd fs11, 8(sp)
-;; mv a3, s0
-;; ld a2, 8(a0)
-;; sd a3, 0x48(a2)
+;; addi sp, sp, -0xd0
+;; sd s0, 0xc8(sp)
+;; sd s1, 0xc0(sp)
+;; sd s2, 0xb8(sp)
+;; sd s3, 0xb0(sp)
+;; sd s4, 0xa8(sp)
+;; sd s5, 0xa0(sp)
+;; sd s6, 0x98(sp)
+;; sd s7, 0x90(sp)
+;; sd s8, 0x88(sp)
+;; sd s9, 0x80(sp)
+;; sd s10, 0x78(sp)
+;; sd s11, 0x70(sp)
+;; fsd fs0, 0x68(sp)
+;; fsd fs2, 0x60(sp)
+;; fsd fs3, 0x58(sp)
+;; fsd fs4, 0x50(sp)
+;; fsd fs5, 0x48(sp)
+;; fsd fs6, 0x40(sp)
+;; fsd fs7, 0x38(sp)
+;; fsd fs8, 0x30(sp)
+;; fsd fs9, 0x28(sp)
+;; fsd fs10, 0x20(sp)
+;; fsd fs11, 0x18(sp)
+;; mv a2, s0
+;; ld a5, 8(a0)
+;; sd a2, 0x48(a5)
;; mv a3, sp
-;; sd a3, 0x40(a2)
-;; auipc a3, 0
-;; addi a3, a3, 0x88
-;; sd a3, 0x50(a2)
+;; sd a3, 0x40(a5)
+;; auipc a4, 0
+;; addi a4, a4, 0x8c
+;; sd a4, 0x50(a5)
+;; sd a5, 0(sp)
;; auipc ra, 0
-;; jalr ra, ra, -0xb0
-;; ├─╼ exception frame offset: SP = FP - 0xc0
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x12c
+;; jalr ra, ra, -0xb4
+;; ├─╼ exception frame offset: SP = FP - 0xd0
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x130
;; addi a0, zero, 1
-;; ld s0, 0xb8(sp)
-;; ld s1, 0xb0(sp)
-;; ld s2, 0xa8(sp)
-;; ld s3, 0xa0(sp)
-;; ld s4, 0x98(sp)
-;; ld s5, 0x90(sp)
-;; ld s6, 0x88(sp)
-;; ld s7, 0x80(sp)
-;; ld s8, 0x78(sp)
-;; ld s9, 0x70(sp)
-;; ld s10, 0x68(sp)
-;; ld s11, 0x60(sp)
-;; fld fs0, 0x58(sp)
-;; fld fs2, 0x50(sp)
-;; fld fs3, 0x48(sp)
-;; fld fs4, 0x40(sp)
-;; fld fs5, 0x38(sp)
-;; fld fs6, 0x30(sp)
-;; fld fs7, 0x28(sp)
-;; fld fs8, 0x20(sp)
-;; fld fs9, 0x18(sp)
-;; fld fs10, 0x10(sp)
-;; fld fs11, 8(sp)
-;; addi sp, sp, 0xc0
+;; ld s0, 0xc8(sp)
+;; ld s1, 0xc0(sp)
+;; ld s2, 0xb8(sp)
+;; ld s3, 0xb0(sp)
+;; ld s4, 0xa8(sp)
+;; ld s5, 0xa0(sp)
+;; ld s6, 0x98(sp)
+;; ld s7, 0x90(sp)
+;; ld s8, 0x88(sp)
+;; ld s9, 0x80(sp)
+;; ld s10, 0x78(sp)
+;; ld s11, 0x70(sp)
+;; fld fs0, 0x68(sp)
+;; fld fs2, 0x60(sp)
+;; fld fs3, 0x58(sp)
+;; fld fs4, 0x50(sp)
+;; fld fs5, 0x48(sp)
+;; fld fs6, 0x40(sp)
+;; fld fs7, 0x38(sp)
+;; fld fs8, 0x30(sp)
+;; fld fs9, 0x28(sp)
+;; fld fs10, 0x20(sp)
+;; fld fs11, 0x18(sp)
+;; addi sp, sp, 0xd0
;; ld ra, 8(sp)
;; ld s0, 0(sp)
;; addi sp, sp, 0x10
;; ret
+;; addi a0, zero, 1
+;; ld a5, 0(sp)
+;; sd a0, 0x88(a5)
;; mv a0, zero
-;; ld s0, 0xb8(sp)
-;; ld s1, 0xb0(sp)
-;; ld s2, 0xa8(sp)
-;; ld s3, 0xa0(sp)
-;; ld s4, 0x98(sp)
-;; ld s5, 0x90(sp)
-;; ld s6, 0x88(sp)
-;; ld s7, 0x80(sp)
-;; ld s8, 0x78(sp)
-;; ld s9, 0x70(sp)
-;; ld s10, 0x68(sp)
-;; ld s11, 0x60(sp)
-;; fld fs0, 0x58(sp)
-;; fld fs2, 0x50(sp)
-;; fld fs3, 0x48(sp)
-;; fld fs4, 0x40(sp)
-;; fld fs5, 0x38(sp)
-;; fld fs6, 0x30(sp)
-;; fld fs7, 0x28(sp)
-;; fld fs8, 0x20(sp)
-;; fld fs9, 0x18(sp)
-;; fld fs10, 0x10(sp)
-;; fld fs11, 8(sp)
-;; addi sp, sp, 0xc0
+;; ld s0, 0xc8(sp)
+;; ld s1, 0xc0(sp)
+;; ld s2, 0xb8(sp)
+;; ld s3, 0xb0(sp)
+;; ld s4, 0xa8(sp)
+;; ld s5, 0xa0(sp)
+;; ld s6, 0x98(sp)
+;; ld s7, 0x90(sp)
+;; ld s8, 0x88(sp)
+;; ld s9, 0x80(sp)
+;; ld s10, 0x78(sp)
+;; ld s11, 0x70(sp)
+;; fld fs0, 0x68(sp)
+;; fld fs2, 0x60(sp)
+;; fld fs3, 0x58(sp)
+;; fld fs4, 0x50(sp)
+;; fld fs5, 0x48(sp)
+;; fld fs6, 0x40(sp)
+;; fld fs7, 0x38(sp)
+;; fld fs8, 0x30(sp)
+;; fld fs9, 0x28(sp)
+;; fld fs10, 0x20(sp)
+;; fld fs11, 0x18(sp)
+;; addi sp, sp, 0xd0
;; ld ra, 8(sp)
;; ld s0, 0(sp)
;; addi sp, sp, 0x10
diff --git a/tests/disas/s390x-entry-trampoline.wat b/tests/disas/s390x-entry-trampoline.wat
index 17dd61f3a0..1e0d78fc84 100644
--- a/tests/disas/s390x-entry-trampoline.wat
+++ b/tests/disas/s390x-entry-trampoline.wat
@@ -7,45 +7,48 @@
;; wasm[0]::array_to_wasm_trampoline[0]:
;; stmg %r6, %r15, 0x30(%r15)
;; lgr %r1, %r15
-;; aghi %r15, -0xe0
+;; aghi %r15, -0xe8
;; stg %r1, 0(%r15)
-;; std %f8, 0xa0(%r15)
-;; std %f9, 0xa8(%r15)
-;; std %f10, 0xb0(%r15)
-;; std %f11, 0xb8(%r15)
-;; std %f12, 0xc0(%r15)
-;; std %f13, 0xc8(%r15)
-;; std %f14, 0xd0(%r15)
-;; std %f15, 0xd8(%r15)
-;; lg %r5, 0(%r15)
-;; lg %r4, 8(%r2)
-;; stg %r5, 0x48(%r4)
-;; lgr %r5, %r15
-;; stg %r5, 0x40(%r4)
-;; larl %r5, 0xbe
-;; stg %r5, 0x50(%r4)
+;; std %f8, 0xa8(%r15)
+;; std %f9, 0xb0(%r15)
+;; std %f10, 0xb8(%r15)
+;; std %f11, 0xc0(%r15)
+;; std %f12, 0xc8(%r15)
+;; std %f13, 0xd0(%r15)
+;; std %f14, 0xd8(%r15)
+;; std %f15, 0xe0(%r15)
+;; lg %r4, 0(%r15)
+;; lg %r5, 8(%r2)
+;; stg %r4, 0x48(%r5)
+;; lgr %r4, %r15
+;; stg %r4, 0x40(%r5)
+;; larl %r4, 0xc4
+;; stg %r4, 0x50(%r5)
+;; stg %r5, 0xa0(%r15)
;; brasl %r14, 0
-;; ├─╼ exception frame offset: SP = FP - 0xe0
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xbe
+;; ├─╼ exception frame offset: SP = FP - 0xe8
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xc4
;; lhi %r2, 1
-;; ld %f8, 0xa0(%r15)
-;; ld %f9, 0xa8(%r15)
-;; ld %f10, 0xb0(%r15)
-;; ld %f11, 0xb8(%r15)
-;; ld %f12, 0xc0(%r15)
-;; ld %f13, 0xc8(%r15)
-;; ld %f14, 0xd0(%r15)
-;; ld %f15, 0xd8(%r15)
-;; lmg %r6, %r15, 0x110(%r15)
+;; ld %f8, 0xa8(%r15)
+;; ld %f9, 0xb0(%r15)
+;; ld %f10, 0xb8(%r15)
+;; ld %f11, 0xc0(%r15)
+;; ld %f12, 0xc8(%r15)
+;; ld %f13, 0xd0(%r15)
+;; ld %f14, 0xd8(%r15)
+;; ld %f15, 0xe0(%r15)
+;; lmg %r6, %r15, 0x118(%r15)
;; br %r14
+;; lg %r5, 0xa0(%r15)
+;; mvghi 0x88(%r5), 1
;; lhi %r2, 0
-;; ld %f8, 0xa0(%r15)
-;; ld %f9, 0xa8(%r15)
-;; ld %f10, 0xb0(%r15)
-;; ld %f11, 0xb8(%r15)
-;; ld %f12, 0xc0(%r15)
-;; ld %f13, 0xc8(%r15)
-;; ld %f14, 0xd0(%r15)
-;; ld %f15, 0xd8(%r15)
-;; lmg %r6, %r15, 0x110(%r15)
+;; ld %f8, 0xa8(%r15)
+;; ld %f9, 0xb0(%r15)
+;; ld %f10, 0xb8(%r15)
+;; ld %f11, 0xc0(%r15)
+;; ld %f12, 0xc8(%r15)
+;; ld %f13, 0xd0(%r15)
+;; ld %f14, 0xd8(%r15)
+;; ld %f15, 0xe0(%r15)
+;; lmg %r6, %r15, 0x118(%r15)
;; br %r14
diff --git a/tests/disas/startup-data-active.wat b/tests/disas/startup-data-active.wat
index 028708a745..449e32a58f 100644
--- a/tests/disas/startup-data-active.wat
+++ b/tests/disas/startup-data-active.wat
@@ -13,6 +13,7 @@
;; region1 = 268435528 "VMStoreContext+0x48"
;; region2 = 268435520 "VMStoreContext+0x40"
;; region3 = 268435536 "VMStoreContext+0x50"
+;; region4 = 268435592 "VMStoreContext+0x88"
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -34,8 +35,10 @@
;; return v8 ; v8 = 1
;;
;; block3:
-;; v9 = iconst.i8 0
-;; return v9 ; v9 = 0
+;; v9 = iconst.i64 1
+;; store notrap aligned region4 v9, v4+136 ; v9 = 1
+;; v10 = iconst.i8 0
+;; return v10 ; v10 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat
index 4262c9fdf8..4fbd0f4a68 100644
--- a/tests/disas/startup-elem-active.wat
+++ b/tests/disas/startup-elem-active.wat
@@ -17,6 +17,7 @@
;; region1 = 268435528 "VMStoreContext+0x48"
;; region2 = 268435520 "VMStoreContext+0x40"
;; region3 = 268435536 "VMStoreContext+0x50"
+;; region4 = 268435592 "VMStoreContext+0x88"
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -38,8 +39,10 @@
;; return v8 ; v8 = 1
;;
;; block3:
-;; v9 = iconst.i8 0
-;; return v9 ; v9 = 0
+;; v9 = iconst.i64 1
+;; store notrap aligned region4 v9, v4+136 ; v9 = 1
+;; v10 = iconst.i8 0
+;; return v10 ; v10 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-global.wat b/tests/disas/startup-global.wat
index a507e6d125..a345c52e4a 100644
--- a/tests/disas/startup-global.wat
+++ b/tests/disas/startup-global.wat
@@ -10,6 +10,7 @@
;; region1 = 268435528 "VMStoreContext+0x48"
;; region2 = 268435520 "VMStoreContext+0x40"
;; region3 = 268435536 "VMStoreContext+0x50"
+;; region4 = 268435592 "VMStoreContext+0x88"
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -31,8 +32,10 @@
;; return v8 ; v8 = 1
;;
;; block3:
-;; v9 = iconst.i8 0
-;; return v9 ; v9 = 0
+;; v9 = iconst.i64 1
+;; store notrap aligned region4 v9, v4+136 ; v9 = 1
+;; v10 = iconst.i8 0
+;; return v10 ; v10 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-passive-segment.wat b/tests/disas/startup-passive-segment.wat
index 9631e49735..1e87184808 100644
--- a/tests/disas/startup-passive-segment.wat
+++ b/tests/disas/startup-passive-segment.wat
@@ -11,6 +11,7 @@
;; region1 = 268435528 "VMStoreContext+0x48"
;; region2 = 268435520 "VMStoreContext+0x40"
;; region3 = 268435536 "VMStoreContext+0x50"
+;; region4 = 268435592 "VMStoreContext+0x88"
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -32,8 +33,10 @@
;; return v8 ; v8 = 1
;;
;; block3:
-;; v9 = iconst.i8 0
-;; return v9 ; v9 = 0
+;; v9 = iconst.i64 1
+;; store notrap aligned region4 v9, v4+136 ; v9 = 1
+;; v10 = iconst.i8 0
+;; return v10 ; v10 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-start.wat b/tests/disas/startup-start.wat
index 02a113fd61..520bdf2a64 100644
--- a/tests/disas/startup-start.wat
+++ b/tests/disas/startup-start.wat
@@ -11,6 +11,7 @@
;; region1 = 268435528 "VMStoreContext+0x48"
;; region2 = 268435520 "VMStoreContext+0x40"
;; region3 = 268435536 "VMStoreContext+0x50"
+;; region4 = 268435592 "VMStoreContext+0x88"
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -32,8 +33,10 @@
;; return v8 ; v8 = 1
;;
;; block3:
-;; v9 = iconst.i8 0
-;; return v9 ; v9 = 0
+;; v9 = iconst.i64 1
+;; store notrap aligned region4 v9, v4+136 ; v9 = 1
+;; v10 = iconst.i8 0
+;; return v10 ; v10 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat
index 51af0b0123..2ea44c7e48 100644
--- a/tests/disas/startup-table-initial-value.wat
+++ b/tests/disas/startup-table-initial-value.wat
@@ -11,6 +11,7 @@
;; region1 = 268435528 "VMStoreContext+0x48"
;; region2 = 268435520 "VMStoreContext+0x40"
;; region3 = 268435536 "VMStoreContext+0x50"
+;; region4 = 268435592 "VMStoreContext+0x88"
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -32,8 +33,10 @@
;; return v8 ; v8 = 1
;;
;; block3:
-;; v9 = iconst.i8 0
-;; return v9 ; v9 = 0
+;; v9 = iconst.i64 1
+;; store notrap aligned region4 v9, v4+136 ; v9 = 1
+;; v10 = iconst.i8 0
+;; return v10 ; v10 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/trunc.wat b/tests/disas/trunc.wat
index 165db1d1a6..250b13dcb0 100644
--- a/tests/disas/trunc.wat
+++ b/tests/disas/trunc.wat
@@ -24,7 +24,7 @@
;; jne 0x101
;; 39: movq %rbx, %rdi
;; movdqu (%rsp), %xmm0
-;; callq 0x244
+;; callq 0x259
;; movabsq $13830554455654793216, %rax
;; movq %rax, %xmm1
;; ucomisd %xmm0, %xmm1
@@ -55,27 +55,27 @@
;; retq
;; d3: movl $6, %esi
;; d8: movq %rbx, %rdi
-;; db: callq 0x271
+;; db: callq 0x286
;; e0: movq %rbx, %rdi
-;; e3: callq 0x2a2
+;; e3: callq 0x2b7
;; e8: ud2
;; ea: movl $6, %esi
;; ef: movq %rbx, %rdi
-;; f2: callq 0x271
+;; f2: callq 0x286
;; f7: movq %rbx, %rdi
-;; fa: callq 0x2a2
+;; fa: callq 0x2b7
;; ff: ud2
;; 101: movl $8, %esi
;; 106: movq %rbx, %rdi
-;; 109: callq 0x271
+;; 109: callq 0x286
;; 10e: movq %rbx, %rdi
-;; 111: callq 0x2a2
+;; 111: callq 0x2b7
;; 116: ud2
;; 118: xorl %esi, %esi
;; 11a: movq %rbx, %rdi
-;; 11d: callq 0x271
+;; 11d: callq 0x286
;; 122: movq %rbx, %rdi
-;; 125: callq 0x2a2
+;; 125: callq 0x2b7
;; 12a: ud2
;; 12c: ud2
;; 12e: ud2
diff --git a/tests/disas/trunc32.wat b/tests/disas/trunc32.wat
index 370a72b7d5..dc903bde99 100644
--- a/tests/disas/trunc32.wat
+++ b/tests/disas/trunc32.wat
@@ -26,7 +26,7 @@
;; jp 0xf6
;; jne 0xf6
;; 46: movq %rbx, %rdi
-;; callq 0x242
+;; callq 0x257
;; movabsq $13830554455654793216, %rax
;; movq %rax, %xmm1
;; ucomisd %xmm0, %xmm1
@@ -56,27 +56,27 @@
;; retq
;; c8: movl $6, %esi
;; cd: movq %rbx, %rdi
-;; d0: callq 0x26f
+;; d0: callq 0x284
;; d5: movq %rbx, %rdi
-;; d8: callq 0x2a0
+;; d8: callq 0x2b5
;; dd: ud2
;; df: movl $6, %esi
;; e4: movq %rbx, %rdi
-;; e7: callq 0x26f
+;; e7: callq 0x284
;; ec: movq %rbx, %rdi
-;; ef: callq 0x2a0
+;; ef: callq 0x2b5
;; f4: ud2
;; f6: movl $8, %esi
;; fb: movq %rbx, %rdi
-;; fe: callq 0x26f
+;; fe: callq 0x284
;; 103: movq %rbx, %rdi
-;; 106: callq 0x2a0
+;; 106: callq 0x2b5
;; 10b: ud2
;; 10d: xorl %esi, %esi
;; 10f: movq %rbx, %rdi
-;; 112: callq 0x26f
+;; 112: callq 0x284
;; 117: movq %rbx, %rdi
-;; 11a: callq 0x2a0
+;; 11a: callq 0x2b5
;; 11f: ud2
;; 121: ud2
;; 123: ud2
diff --git a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
index e6b86c04d5..6c26d05746 100644
--- a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
+++ b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
@@ -85,7 +85,7 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28]
-;; bl #0x3f0
+;; bl #0x408
;; e0: add x28, x28, #4
;; mov sp, x28
;; ldur x9, [x28, #0x14]
@@ -153,7 +153,7 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28, #0xc]
-;; bl #0x3f0
+;; bl #0x408
;; 1f0: add x28, x28, #0xc
;; mov sp, x28
;; add x28, x28, #4
diff --git a/tests/disas/winch/aarch64/call_indirect/local_arg.wat b/tests/disas/winch/aarch64/call_indirect/local_arg.wat
index 2c7311368e..1675b9bbd6 100644
--- a/tests/disas/winch/aarch64/call_indirect/local_arg.wat
+++ b/tests/disas/winch/aarch64/call_indirect/local_arg.wat
@@ -91,7 +91,7 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28]
-;; bl #0x408
+;; bl #0x440
;; 120: add x28, x28, #4
;; mov sp, x28
;; ldur x9, [x28, #0x14]
diff --git a/tests/disas/winch/x64/atomic/notify/notify.wat b/tests/disas/winch/x64/atomic/notify/notify.wat
index 9ebac8b87d..98d426f598 100644
--- a/tests/disas/winch/x64/atomic/notify/notify.wat
+++ b/tests/disas/winch/x64/atomic/notify/notify.wat
@@ -27,7 +27,7 @@
;; movl $0, %esi
;; movq 8(%rsp), %rdx
;; movl 4(%rsp), %ecx
-;; callq 0x16e
+;; callq 0x183
;; addq $4, %rsp
;; addq $0xc, %rsp
;; movq 8(%rsp), %r14
diff --git a/tests/disas/winch/x64/atomic/notify/notify_offset.wat b/tests/disas/winch/x64/atomic/notify/notify_offset.wat
index 6d3cf71f12..19e54fba98 100644
--- a/tests/disas/winch/x64/atomic/notify/notify_offset.wat
+++ b/tests/disas/winch/x64/atomic/notify/notify_offset.wat
@@ -29,7 +29,7 @@
;; movl $0, %esi
;; movq 8(%rsp), %rdx
;; movl 4(%rsp), %ecx
-;; callq 0x17d
+;; callq 0x192
;; addq $4, %rsp
;; addq $0xc, %rsp
;; movq 8(%rsp), %r14
diff --git a/tests/disas/winch/x64/atomic/wait/wait32.wat b/tests/disas/winch/x64/atomic/wait/wait32.wat
index 763669203e..3142b7dfc0 100644
--- a/tests/disas/winch/x64/atomic/wait/wait32.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait32.wat
@@ -30,7 +30,7 @@
;; movq 0x18(%rsp), %rdx
;; movl 0x14(%rsp), %ecx
;; movq 0xc(%rsp), %r8
-;; callq 0x17b
+;; callq 0x190
;; addq $0xc, %rsp
;; addq $0x14, %rsp
;; movq 8(%rsp), %r14
diff --git a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
index ce8835b2ba..97c31b9c88 100644
--- a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
@@ -35,7 +35,7 @@
;; movq 0x18(%rsp), %rdx
;; movl 0x14(%rsp), %ecx
;; movq 0xc(%rsp), %r8
-;; callq 0x18a
+;; callq 0x19f
;; addq $0xc, %rsp
;; addq $0x14, %rsp
;; movq 8(%rsp), %r14
diff --git a/tests/disas/winch/x64/atomic/wait/wait64.wat b/tests/disas/winch/x64/atomic/wait/wait64.wat
index 5fa37e700b..f9d488d6d5 100644
--- a/tests/disas/winch/x64/atomic/wait/wait64.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait64.wat
@@ -29,7 +29,7 @@
;; movq 0x18(%rsp), %rdx
;; movq 0x10(%rsp), %rcx
;; movq 8(%rsp), %r8
-;; callq 0x173
+;; callq 0x188
;; addq $8, %rsp
;; addq $0x18, %rsp
;; movq 8(%rsp), %r14
diff --git a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
index c09815702a..584073f39c 100644
--- a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
@@ -34,7 +34,7 @@
;; movq 0x18(%rsp), %rdx
;; movq 0x10(%rsp), %rcx
;; movq 8(%rsp), %r8
-;; callq 0x182
+;; callq 0x197
;; addq $8, %rsp
;; addq $0x18, %rsp
;; movq 8(%rsp), %r14
diff --git a/tests/disas/winch/x64/call_indirect/call_indirect.wat b/tests/disas/winch/x64/call_indirect/call_indirect.wat
index 5ecf92d14c..17fa2ad96a 100644
--- a/tests/disas/winch/x64/call_indirect/call_indirect.wat
+++ b/tests/disas/winch/x64/call_indirect/call_indirect.wat
@@ -76,7 +76,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x336
+;; callq 0x34b
;; addq $8, %rsp
;; addq $4, %rsp
;; movq 0x1c(%rsp), %r14
@@ -128,7 +128,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 4(%rsp), %edx
-;; callq 0x336
+;; callq 0x34b
;; addq $4, %rsp
;; addq $4, %rsp
;; movq 0x20(%rsp), %r14
diff --git a/tests/disas/winch/x64/call_indirect/local_arg.wat b/tests/disas/winch/x64/call_indirect/local_arg.wat
index 595446e979..10858f12f6 100644
--- a/tests/disas/winch/x64/call_indirect/local_arg.wat
+++ b/tests/disas/winch/x64/call_indirect/local_arg.wat
@@ -72,7 +72,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x31e
+;; callq 0x34a
;; addq $8, %rsp
;; addq $4, %rsp
;; movq 0x1c(%rsp), %r14
diff --git a/tests/disas/winch/x64/epoch/func.wat b/tests/disas/winch/x64/epoch/func.wat
index 9efcc83ad4..061edd761b 100644
--- a/tests/disas/winch/x64/epoch/func.wat
+++ b/tests/disas/winch/x64/epoch/func.wat
@@ -23,7 +23,7 @@
;; cmpq %rcx, %rdx
;; jb 0x54
;; 47: movq %r14, %rdi
-;; callq 0x134
+;; callq 0x14a
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
diff --git a/tests/disas/winch/x64/epoch/loop.wat b/tests/disas/winch/x64/epoch/loop.wat
index 23baa77a5d..3701d1ed48 100644
--- a/tests/disas/winch/x64/epoch/loop.wat
+++ b/tests/disas/winch/x64/epoch/loop.wat
@@ -25,7 +25,7 @@
;; cmpq %rcx, %rdx
;; jb 0x54
;; 47: movq %r14, %rdi
-;; callq 0x15e
+;; callq 0x174
;; movq 8(%rsp), %r14
;; movq 0x18(%r14), %rdx
;; movq (%rdx), %rdx
@@ -34,7 +34,7 @@
;; cmpq %rcx, %rdx
;; jb 0x79
;; 6c: movq %r14, %rdi
-;; callq 0x15e
+;; callq 0x174
;; movq 8(%rsp), %r14
;; jmp 0x54
;; 7e: addq $0x10, %rsp
diff --git a/tests/disas/winch/x64/fuel/call.wat b/tests/disas/winch/x64/fuel/call.wat
index 4e35be2aa3..6ca0934728 100644
--- a/tests/disas/winch/x64/fuel/call.wat
+++ b/tests/disas/winch/x64/fuel/call.wat
@@ -28,7 +28,7 @@
;; cmpq $0, %rcx
;; jl 0x5e
;; 51: movq %r14, %rdi
-;; callq 0x1ee
+;; callq 0x204
;; movq 8(%rsp), %r14
;; movq 8(%r14), %rax
;; movq (%rax), %r11
@@ -74,7 +74,7 @@
;; cmpq $0, %rcx
;; jl 0x10e
;; 101: movq %r14, %rdi
-;; callq 0x1ee
+;; callq 0x204
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
diff --git a/tests/disas/winch/x64/fuel/func.wat b/tests/disas/winch/x64/fuel/func.wat
index b239b13708..cefba99a79 100644
--- a/tests/disas/winch/x64/fuel/func.wat
+++ b/tests/disas/winch/x64/fuel/func.wat
@@ -24,7 +24,7 @@
;; cmpq $0, %rcx
;; jl 0x5e
;; 51: movq %r14, %rdi
-;; callq 0x13e
+;; callq 0x154
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
diff --git a/tests/disas/winch/x64/fuel/loop.wat b/tests/disas/winch/x64/fuel/loop.wat
index 6d3bd20c27..fcb24042b2 100644
--- a/tests/disas/winch/x64/fuel/loop.wat
+++ b/tests/disas/winch/x64/fuel/loop.wat
@@ -26,14 +26,14 @@
;; cmpq $0, %rcx
;; jl 0x5e
;; 51: movq %r14, %rdi
-;; callq 0x172
+;; callq 0x188
;; movq 8(%rsp), %r14
;; movq 8(%r14), %rcx
;; movq (%rcx), %rcx
;; cmpq $0, %rcx
;; jl 0x7c
;; 6f: movq %r14, %rdi
-;; callq 0x172
+;; callq 0x188
;; movq 8(%rsp), %r14
;; movq 8(%r14), %rax
;; movq (%rax), %r11
diff --git a/tests/disas/winch/x64/load/grow_load.wat b/tests/disas/winch/x64/load/grow_load.wat
index a11c2c6db3..500e3caa24 100644
--- a/tests/disas/winch/x64/load/grow_load.wat
+++ b/tests/disas/winch/x64/load/grow_load.wat
@@ -65,7 +65,7 @@
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
;; movl $0, %edx
-;; callq 0x2ec
+;; callq 0x30a
;; addq $0xc, %rsp
;; addq $4, %rsp
;; movq 0x58(%rsp), %r14
diff --git a/tests/disas/winch/x64/ref/func.wat b/tests/disas/winch/x64/ref/func.wat
index 0bcad41049..49f72b56b7 100644
--- a/tests/disas/winch/x64/ref/func.wat
+++ b/tests/disas/winch/x64/ref/func.wat
@@ -39,7 +39,7 @@
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x252
+;; callq 0x27d
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
diff --git a/tests/disas/winch/x64/table/fill.wat b/tests/disas/winch/x64/table/fill.wat
index e944e0e58c..b971e7a536 100644
--- a/tests/disas/winch/x64/table/fill.wat
+++ b/tests/disas/winch/x64/table/fill.wat
@@ -113,7 +113,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x51d
+;; callq 0x575
;; addq $0xc, %rsp
;; addq $4, %rsp
;; movq 0x28(%rsp), %r14
diff --git a/tests/disas/winch/x64/table/get.wat b/tests/disas/winch/x64/table/get.wat
index 12d90ebd88..20c466bc5e 100644
--- a/tests/disas/winch/x64/table/get.wat
+++ b/tests/disas/winch/x64/table/get.wat
@@ -65,7 +65,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x2e6
+;; callq 0x311
;; addq $0xc, %rsp
;; addq $4, %rsp
;; movq 0x18(%rsp), %r14
diff --git a/tests/disas/winch/x64/table/grow.wat b/tests/disas/winch/x64/table/grow.wat
index 0ae72c0112..b92e861112 100644
--- a/tests/disas/winch/x64/table/grow.wat
+++ b/tests/disas/winch/x64/table/grow.wat
@@ -34,7 +34,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl (%rsp), %edx
-;; callq 0x20f
+;; callq 0x224
;; addq $4, %rsp
;; movq 0x24(%rsp), %r14
;; movl (%rsp), %ecx
diff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.wat
index ab1e576d67..f50465b2e8 100644
--- a/tests/disas/winch/x64/table/init_copy_drop.wat
+++ b/tests/disas/winch/x64/table/init_copy_drop.wat
@@ -138,13 +138,13 @@
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x10bf
+;; callq 0x1153
;; movq 8(%rsp), %r14
;; pushq %rax
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x10ea
+;; callq 0x117e
;; addq $8, %rsp
;; movq 0x10(%rsp), %r14
;; movq %r14, %r11
@@ -190,17 +190,17 @@
;; jmp 0x1f4
;; 256: movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x1115
+;; callq 0x11a9
;; movq 8(%rsp), %r14
;; movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x10bf
+;; callq 0x1153
;; movq 8(%rsp), %r14
;; pushq %rax
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x10ea
+;; callq 0x117e
;; addq $8, %rsp
;; movq 0x10(%rsp), %r14
;; movq %r14, %r11
@@ -246,7 +246,7 @@
;; jmp 0x2ed
;; 34f: movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x1115
+;; callq 0x11a9
;; movq 8(%rsp), %r14
;; movl $5, %eax
;; movl $0xf, %ecx
@@ -305,7 +305,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1184
+;; callq 0x1218
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -390,7 +390,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1184
+;; callq 0x1218
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -475,7 +475,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1184
+;; callq 0x1218
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -560,7 +560,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1184
+;; callq 0x1218
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -645,7 +645,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x1184
+;; callq 0x1218
;; addq $8, %rsp
;; addq $8, %rsp
;; movq 0x28(%rsp), %r14
@@ -756,7 +756,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x1184
+;; callq 0x1218
;; addq $0xc, %rsp
;; addq $4, %rsp
;; movq 0x18(%rsp), %r14
diff --git a/tests/disas/winch/x64/table/set.wat b/tests/disas/winch/x64/table/set.wat
index 9dd27dedaa..562ef0da46 100644
--- a/tests/disas/winch/x64/table/set.wat
+++ b/tests/disas/winch/x64/table/set.wat
@@ -109,7 +109,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x4b7
+;; callq 0x4f9
;; addq $8, %rsp
;; addq $4, %rsp
;; movq 0x1c(%rsp), %r14
diff --git a/tests/disas/x64-entry-trampoline.wat b/tests/disas/x64-entry-trampoline.wat
index eebc2bd415..ce6de4a91b 100644
--- a/tests/disas/x64-entry-trampoline.wat
+++ b/tests/disas/x64-entry-trampoline.wat
@@ -7,39 +7,42 @@
;; wasm[0]::array_to_wasm_trampoline[0]:
;; pushq %rbp
;; movq %rsp, %rbp
-;; subq $0x30, %rsp
-;; movq %rbx, (%rsp)
-;; movq %r12, 8(%rsp)
-;; movq %r13, 0x10(%rsp)
-;; movq %r14, 0x18(%rsp)
-;; movq %r15, 0x20(%rsp)
+;; subq $0x40, %rsp
+;; movq %rbx, 0x10(%rsp)
+;; movq %r12, 0x18(%rsp)
+;; movq %r13, 0x20(%rsp)
+;; movq %r14, 0x28(%rsp)
+;; movq %r15, 0x30(%rsp)
;; movq %rbp, %rcx
-;; movq 8(%rdi), %rax
-;; movq %rcx, 0x48(%rax)
+;; movq 8(%rdi), %r8
+;; movq %rcx, 0x48(%r8)
;; movq %rsp, %rcx
-;; movq %rcx, 0x40(%rax)
-;; leaq 0x2f(%rip), %rcx
-;; movq %rcx, 0x50(%rax)
+;; movq %rcx, 0x40(%r8)
+;; leaq 0x34(%rip), %rcx
+;; movq %rcx, 0x50(%r8)
+;; movq %r8, (%rsp)
;; callq 0
-;; ├─╼ exception frame offset: SP = FP - 0x30
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x71
+;; ├─╼ exception frame offset: SP = FP - 0x40
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x77
;; movl $1, %eax
-;; movq (%rsp), %rbx
-;; movq 8(%rsp), %r12
-;; movq 0x10(%rsp), %r13
-;; movq 0x18(%rsp), %r14
-;; movq 0x20(%rsp), %r15
-;; addq $0x30, %rsp
+;; movq 0x10(%rsp), %rbx
+;; movq 0x18(%rsp), %r12
+;; movq 0x20(%rsp), %r13
+;; movq 0x28(%rsp), %r14
+;; movq 0x30(%rsp), %r15
+;; addq $0x40, %rsp
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 71: xorl %eax, %eax
-;; 73: movq (%rsp), %rbx
-;; 77: movq 8(%rsp), %r12
-;; 7c: movq 0x10(%rsp), %r13
-;; 81: movq 0x18(%rsp), %r14
-;; 86: movq 0x20(%rsp), %r15
-;; 8b: addq $0x30, %rsp
-;; 8f: movq %rbp, %rsp
-;; 92: popq %rbp
-;; 93: retq
+;; 77: movq (%rsp), %r8
+;; 7b: movq $1, 0x88(%r8)
+;; 86: xorl %eax, %eax
+;; 88: movq 0x10(%rsp), %rbx
+;; 8d: movq 0x18(%rsp), %r12
+;; 92: movq 0x20(%rsp), %r13
+;; 97: movq 0x28(%rsp), %r14
+;; 9c: movq 0x30(%rsp), %r15
+;; a1: addq $0x40, %rsp
+;; a5: movq %rbp, %rsp
+;; a8: popq %rbp
+;; a9: retq
diff --git a/tests/disas/x64/call-indirect-with-gc.wat b/tests/disas/x64/call-indirect-with-gc.wat
index aee03e606d..d053980e32 100644
--- a/tests/disas/x64/call-indirect-with-gc.wat
+++ b/tests/disas/x64/call-indirect-with-gc.wat
@@ -53,7 +53,7 @@
;; 8b: xorl %esi, %esi
;; 8d: movq %r9, %rbx
;; 90: movq %rbx, %rdi
-;; 93: callq 0x219
+;; 93: callq 0x22f
;; 98: jmp 0x59
;; 9d: ud2
;; 9f: ud2
diff --git a/tests/disas/x64/call-indirect-without-gc.wat b/tests/disas/x64/call-indirect-without-gc.wat
index ab06cfedcb..98c559ff2b 100644
--- a/tests/disas/x64/call-indirect-without-gc.wat
+++ b/tests/disas/x64/call-indirect-without-gc.wat
@@ -53,7 +53,7 @@
;; 8b: xorl %esi, %esi
;; 8d: movq %r9, %rbx
;; 90: movq %rbx, %rdi
-;; 93: callq 0x219
+;; 93: callq 0x22f
;; 98: jmp 0x59
;; 9d: ud2
;; 9f: ud2
diff --git a/tests/misc_testsuite/component-model/async/sync-call-context-slots.wast b/tests/misc_testsuite/component-model/async/sync-call-context-slots.wast
new file mode 100644
index 0000000000..54a41f1c23
--- /dev/null
+++ b/tests/misc_testsuite/component-model/async/sync-call-context-slots.wast
@@ -0,0 +1,152 @@
+;;! component_model_async = true
+;;! component_model_more_async_builtins = true
+;;! component_model_threading = true
+
+;; Like sync-call-context.wast, but drives *both* component-context slots (0 and
+;; 1) at once.
+
+;; Both slots, inline fast path.
+(component
+ (component $A
+ (core func $get0 (canon context.get i32 0))
+ (core func $set0 (canon context.set i32 0))
+ (core func $get1 (canon context.get i32 1))
+ (core func $set1 (canon context.set i32 1))
+ (core module $M
+ (import "" "get0" (func $get0 (result i32)))
+ (import "" "set0" (func $set0 (param i32)))
+ (import "" "get1" (func $get1 (result i32)))
+ (import "" "set1" (func $set1 (param i32)))
+ (func (export "f'") (param i32) (result i32)
+ ;; Fresh thread: both slots zero.
+ (if (i32.ne (call $get0) (i32.const 0)) (then unreachable))
+ (if (i32.ne (call $get1) (i32.const 0)) (then unreachable))
+ (call $set0 (i32.const 0xAAAA1111))
+ (call $set1 (i32.const 0xBBBB2222))
+ ;; The two slots must not alias each other.
+ (if (i32.ne (call $get0) (i32.const 0xAAAA1111)) (then unreachable))
+ (if (i32.ne (call $get1) (i32.const 0xBBBB2222)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 42))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "get0" (func $get0)) (export "set0" (func $set0))
+ (export "get1" (func $get1)) (export "set1" (func $set1))
+ ))))
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'")))
+ )
+
+ (component $B
+ (import "f" (func $f (param "x" u32) (result u32)))
+ (core func $f' (canon lower (func $f)))
+ (core func $get0 (canon context.get i32 0))
+ (core func $set0 (canon context.set i32 0))
+ (core func $get1 (canon context.get i32 1))
+ (core func $set1 (canon context.set i32 1))
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (import "" "get0" (func $get0 (result i32)))
+ (import "" "set0" (func $set0 (param i32)))
+ (import "" "get1" (func $get1 (result i32)))
+ (import "" "set1" (func $set1 (param i32)))
+ (func (export "g'") (result i32) (local $r i32)
+ (call $set0 (i32.const 0x11110000))
+ (call $set1 (i32.const 0x22220000))
+ (local.set $r (call $f' (i32.const 1234)))
+ ;; Both of our slots restored, independently and unswapped.
+ (if (i32.ne (call $get0) (i32.const 0x11110000)) (then unreachable))
+ (if (i32.ne (call $get1) (i32.const 0x22220000)) (then unreachable))
+ (local.get $r)
+ )
+ )
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f'" (func $f'))
+ (export "get0" (func $get0)) (export "set0" (func $set0))
+ (export "get1" (func $get1)) (export "set1" (func $set1))
+ ))))
+ (func (export "g") (result u32)
+ (canon lift (core func $n "g'")))
+ )
+
+ (instance $a (instantiate $A))
+ (instance $b (instantiate $B (with "f" (func $a "f"))))
+ (export "g" (func $b "g"))
+)
+(assert_return (invoke "g") (u32.const 1276))
+
+;; Both slots, forced slow, out-of-line path.
+(component
+ (component $A
+ (core func $get0 (canon context.get i32 0))
+ (core func $set0 (canon context.set i32 0))
+ (core func $get1 (canon context.get i32 1))
+ (core func $set1 (canon context.set i32 1))
+ (core func $bpinc (canon backpressure.inc))
+ (core func $bpdec (canon backpressure.dec))
+ (core module $M
+ (import "" "get0" (func $get0 (result i32)))
+ (import "" "set0" (func $set0 (param i32)))
+ (import "" "get1" (func $get1 (result i32)))
+ (import "" "set1" (func $set1 (param i32)))
+ (import "" "backpressure.inc" (func $bpinc))
+ (import "" "backpressure.dec" (func $bpdec))
+ (func (export "f'") (param i32) (result i32)
+ (if (i32.ne (call $get0) (i32.const 0)) (then unreachable))
+ (if (i32.ne (call $get1) (i32.const 0)) (then unreachable))
+ (call $set0 (i32.const 0xAAAA1111))
+ (call $set1 (i32.const 0xBBBB2222))
+ ;; Call `backptressure.{inc,dec}` to force lazy task creation.
+ (call $bpinc) (call $bpdec)
+ ;; Both slots survive the force.
+ (if (i32.ne (call $get0) (i32.const 0xAAAA1111)) (then unreachable))
+ (if (i32.ne (call $get1) (i32.const 0xBBBB2222)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 42))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "get0" (func $get0)) (export "set0" (func $set0))
+ (export "get1" (func $get1)) (export "set1" (func $set1))
+ (export "backpressure.inc" (func $bpinc))
+ (export "backpressure.dec" (func $bpdec))
+ ))))
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'")))
+ )
+
+ (component $B
+ (import "f" (func $f (param "x" u32) (result u32)))
+ (core func $f' (canon lower (func $f)))
+ (core func $get0 (canon context.get i32 0))
+ (core func $set0 (canon context.set i32 0))
+ (core func $get1 (canon context.get i32 1))
+ (core func $set1 (canon context.set i32 1))
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (import "" "get0" (func $get0 (result i32)))
+ (import "" "set0" (func $set0 (param i32)))
+ (import "" "get1" (func $get1 (result i32)))
+ (import "" "set1" (func $set1 (param i32)))
+ (func (export "g'") (result i32) (local $r i32)
+ (call $set0 (i32.const 0x11110000))
+ (call $set1 (i32.const 0x22220000))
+ (local.set $r (call $f' (i32.const 1234)))
+ (if (i32.ne (call $get0) (i32.const 0x11110000)) (then unreachable))
+ (if (i32.ne (call $get1) (i32.const 0x22220000)) (then unreachable))
+ (local.get $r)
+ )
+ )
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f'" (func $f'))
+ (export "get0" (func $get0)) (export "set0" (func $set0))
+ (export "get1" (func $get1)) (export "set1" (func $set1))
+ ))))
+ (func (export "g") (result u32)
+ (canon lift (core func $n "g'")))
+ )
+
+ (instance $a (instantiate $A))
+ (instance $b (instantiate $B (with "f" (func $a "f"))))
+ (export "g" (func $b "g"))
+)
+(assert_return (invoke "g") (u32.const 1276))
diff --git a/tests/misc_testsuite/component-model/async/sync-call-context-trap.wast b/tests/misc_testsuite/component-model/async/sync-call-context-trap.wast
new file mode 100644
index 0000000000..0e1957c448
--- /dev/null
+++ b/tests/misc_testsuite/component-model/async/sync-call-context-trap.wast
@@ -0,0 +1,47 @@
+;;! component_model_async = true
+;;! component_model_more_async_builtins = true
+
+;; Test a guest-to-guest sync call whose callee traps. This should exercise
+;; deferred task construction/teardown in the face of traps.
+(component
+ (component $A
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "f'") (param i32) (result i32)
+ (call $cset (i32.const 0x5678))
+ unreachable
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'")))
+ )
+
+ (component $B
+ (import "f" (func $f (param "x" u32) (result u32)))
+ (core func $f' (canon lower (func $f)))
+ (core func $cset (canon context.set i32 0))
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "g'") (result i32)
+ (call $cset (i32.const 0x1234))
+ (call $f' (i32.const 1234))
+ )
+ )
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f'" (func $f'))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "g") (result u32)
+ (canon lift (core func $n "g'")))
+ )
+
+ (instance $a (instantiate $A))
+ (instance $b (instantiate $B (with "f" (func $a "f"))))
+ (export "g" (func $b "g"))
+)
+(assert_trap (invoke "g") "wasm `unreachable` instruction executed")
diff --git a/tests/misc_testsuite/component-model/async/sync-call-context.wast b/tests/misc_testsuite/component-model/async/sync-call-context.wast
new file mode 100644
index 0000000000..79f7d50c72
--- /dev/null
+++ b/tests/misc_testsuite/component-model/async/sync-call-context.wast
@@ -0,0 +1,544 @@
+;;! component_model_async = true
+;;! component_model_more_async_builtins = true
+
+;; Tests for guest-to-guest, sync-to-sync calls and their inline fast paths.
+
+;; Inline fast path, no task forcing.
+(component
+ (component $A
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "f'") (param i32) (result i32)
+ ;; A is a freshly-entered (deferred) thread: its context starts at 0.
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x5678))
+ (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 42))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'")))
+ )
+
+ (component $B
+ (import "f" (func $f (param "x" u32) (result u32)))
+ (core func $f' (canon lower (func $f)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "g'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x1234))
+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))
+ (local.set $r (call $f' (i32.const 1234)))
+ ;; The callee's context mutation must NOT leak: ours is restored.
+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))
+ (local.get $r)
+ )
+ )
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f'" (func $f'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "g") (result u32)
+ (canon lift (core func $n "g'")))
+ )
+
+ (instance $a (instantiate $A))
+ (instance $b (instantiate $B (with "f" (func $a "f"))))
+ (export "g" (func $b "g"))
+)
+(assert_return (invoke "g") (u32.const 1276))
+
+;; Single guest-to-guest sync call, forced slow path.
+(component
+ (component $A
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core func $bpinc (canon backpressure.inc))
+ (core func $bpdec (canon backpressure.dec))
+ (core module $M
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (import "" "backpressure.inc" (func $bpinc))
+ (import "" "backpressure.dec" (func $bpdec))
+ (func (export "f'") (param i32) (result i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x5678))
+ ;; Force promotion of the deferred thread mid-call.
+ (call $bpinc)
+ (call $bpdec)
+ ;; Our context must survive the force.
+ (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 42))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ (export "backpressure.inc" (func $bpinc))
+ (export "backpressure.dec" (func $bpdec))
+ ))))
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'")))
+ )
+
+ (component $B
+ (import "f" (func $f (param "x" u32) (result u32)))
+ (core func $f' (canon lower (func $f)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "g'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x1234))
+ (local.set $r (call $f' (i32.const 1234)))
+ ;; Restored even though the callee forced the slow exit path.
+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))
+ (local.get $r)
+ )
+ )
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f'" (func $f'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "g") (result u32)
+ (canon lift (core func $n "g'")))
+ )
+
+ (instance $a (instantiate $A))
+ (instance $b (instantiate $B (with "f" (func $a "f"))))
+ (export "g" (func $b "g"))
+)
+(assert_return (invoke "g") (u32.const 1276))
+
+;; Nested A->B->C sync-call chain, fast path.
+(component
+ (component $Leaf
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "leaf'") (param i32) (result i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x99aabbcc))
+ (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 1))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "leaf") (param "x" u32) (result u32)
+ (canon lift (core func $m "leaf'")))
+ )
+
+ (component $Mid
+ (import "leaf" (func $leaf (param "x" u32) (result u32)))
+ (core func $leaf' (canon lower (func $leaf)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "mid'") (param i32) (result i32) (local $r i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x55667788))
+ (local.set $r (call $leaf' (local.get 0)))
+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 10))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "leaf'" (func $leaf'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "mid") (param "x" u32) (result u32)
+ (canon lift (core func $m "mid'")))
+ )
+
+ (component $Root
+ (import "mid" (func $mid (param "x" u32) (result u32)))
+ (core func $mid' (canon lower (func $mid)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "mid'" (func $mid' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "root'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x11223344))
+ (local.set $r (call $mid' (i32.const 100)))
+ (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 1000))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "mid'" (func $mid'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "root") (result u32)
+ (canon lift (core func $m "root'")))
+ )
+
+ (instance $leaf (instantiate $Leaf))
+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))
+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))
+ (export "root" (func $root "root"))
+)
+(assert_return (invoke "root") (u32.const 1111))
+
+;; Nested A->B->C call chain, forced at the deepest level.
+(component
+ (component $Leaf
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core func $bpinc (canon backpressure.inc))
+ (core func $bpdec (canon backpressure.dec))
+ (core module $M
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (import "" "backpressure.inc" (func $bpinc))
+ (import "" "backpressure.dec" (func $bpdec))
+ (func (export "leaf'") (param i32) (result i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x99aabbcc))
+ ;; Force promotion.
+ (call $bpinc)
+ (call $bpdec)
+ (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 1))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ (export "backpressure.inc" (func $bpinc))
+ (export "backpressure.dec" (func $bpdec))
+ ))))
+ (func (export "leaf") (param "x" u32) (result u32)
+ (canon lift (core func $m "leaf'")))
+ )
+
+ (component $Mid
+ (import "leaf" (func $leaf (param "x" u32) (result u32)))
+ (core func $leaf' (canon lower (func $leaf)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "mid'") (param i32) (result i32) (local $r i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x55667788))
+ (local.set $r (call $leaf' (local.get 0)))
+ ;; Restored after the (forced) nested call.
+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 10))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "leaf'" (func $leaf'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "mid") (param "x" u32) (result u32)
+ (canon lift (core func $m "mid'")))
+ )
+
+ (component $Root
+ (import "mid" (func $mid (param "x" u32) (result u32)))
+ (core func $mid' (canon lower (func $mid)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "mid'" (func $mid' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "root'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x11223344))
+ (local.set $r (call $mid' (i32.const 100)))
+ (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 1000))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "mid'" (func $mid'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "root") (result u32)
+ (canon lift (core func $m "root'")))
+ )
+
+ (instance $leaf (instantiate $Leaf))
+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))
+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))
+ (export "root" (func $root "root"))
+)
+(assert_return (invoke "root") (u32.const 1111))
+
+;; Forcing promotion at an intermediate level, then a deeper sync call.
+(component
+ (component $Leaf
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "leaf'") (param i32) (result i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x99aabbcc))
+ (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 1))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "leaf") (param "x" u32) (result u32)
+ (canon lift (core func $m "leaf'")))
+ )
+
+ (component $Mid
+ (import "leaf" (func $leaf (param "x" u32) (result u32)))
+ (core func $leaf' (canon lower (func $leaf)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core func $bpinc (canon backpressure.inc))
+ (core func $bpdec (canon backpressure.dec))
+ (core module $M
+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (import "" "backpressure.inc" (func $bpinc))
+ (import "" "backpressure.dec" (func $bpdec))
+ (func (export "mid'") (param i32) (result i32) (local $r i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x55667788))
+ ;; Force *before* descending into the leaf.
+ (call $bpinc)
+ (call $bpdec)
+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))
+ (local.set $r (call $leaf' (local.get 0)))
+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 10))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "leaf'" (func $leaf'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ (export "backpressure.inc" (func $bpinc))
+ (export "backpressure.dec" (func $bpdec))
+ ))))
+ (func (export "mid") (param "x" u32) (result u32)
+ (canon lift (core func $m "mid'")))
+ )
+
+ (component $Root
+ (import "mid" (func $mid (param "x" u32) (result u32)))
+ (core func $mid' (canon lower (func $mid)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "mid'" (func $mid' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "root'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x11223344))
+ (local.set $r (call $mid' (i32.const 100)))
+ (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 1000))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "mid'" (func $mid'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "root") (result u32)
+ (canon lift (core func $m "root'")))
+ )
+
+ (instance $leaf (instantiate $Leaf))
+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))
+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))
+ (export "root" (func $root "root"))
+)
+(assert_return (invoke "root") (u32.const 1111))
+
+;; Repeated sync calls from the same caller.
+(component
+ (component $A
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "f'") (param i32) (result i32)
+ ;; Each fresh entry must zero the context regardless of prior calls.
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.add (local.get 0) (i32.const 0x10000)))
+ (i32.add (local.get 0) (i32.const 42))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "f") (param "x" u32) (result u32)
+ (canon lift (core func $m "f'")))
+ )
+
+ (component $B
+ (import "f" (func $f (param "x" u32) (result u32)))
+ (core func $f' (canon lower (func $f)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $N
+ (import "" "f'" (func $f' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "g'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x1234))
+ (local.set $r (call $f' (i32.const 1)))
+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))
+ (local.set $r (i32.add (local.get $r) (call $f' (i32.const 2))))
+ ;; Still restored after the second call.
+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))
+ (local.get $r)
+ )
+ )
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f'" (func $f'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "g") (result u32)
+ (canon lift (core func $n "g'")))
+ )
+
+ (instance $a (instantiate $A))
+ (instance $b (instantiate $B (with "f" (func $a "f"))))
+ (export "g" (func $b "g"))
+)
+;; (1 + 42) + (2 + 42) = 87
+(assert_return (invoke "g") (u32.const 87))
+
+;; Nested chain forced at *two* levels (re-forcing).
+(component
+ (component $Leaf
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core func $bpinc (canon backpressure.inc))
+ (core func $bpdec (canon backpressure.dec))
+ (core module $M
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (import "" "backpressure.inc" (func $bpinc))
+ (import "" "backpressure.dec" (func $bpdec))
+ (func (export "leaf'") (param i32) (result i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x99aabbcc))
+ ;; Force promotion.
+ (call $bpinc) (call $bpdec)
+ (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable))
+ (i32.add (local.get 0) (i32.const 1))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ (export "backpressure.inc" (func $bpinc))
+ (export "backpressure.dec" (func $bpdec))
+ ))))
+ (func (export "leaf") (param "x" u32) (result u32)
+ (canon lift (core func $m "leaf'")))
+ )
+
+ (component $Mid
+ (import "leaf" (func $leaf (param "x" u32) (result u32)))
+ (core func $leaf' (canon lower (func $leaf)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core func $bpinc (canon backpressure.inc))
+ (core func $bpdec (canon backpressure.dec))
+ (core module $M
+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (import "" "backpressure.inc" (func $bpinc))
+ (import "" "backpressure.dec" (func $bpdec))
+ (func (export "mid'") (param i32) (result i32) (local $r i32)
+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))
+ (call $cset (i32.const 0x55667788))
+ ;; Force promotion.
+ (call $bpinc) (call $bpdec)
+ (local.set $r (call $leaf' (local.get 0)))
+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 10))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "leaf'" (func $leaf'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ (export "backpressure.inc" (func $bpinc))
+ (export "backpressure.dec" (func $bpdec))
+ ))))
+ (func (export "mid") (param "x" u32) (result u32)
+ (canon lift (core func $m "mid'")))
+ )
+
+ (component $Root
+ (import "mid" (func $mid (param "x" u32) (result u32)))
+ (core func $mid' (canon lower (func $mid)))
+ (core func $cget (canon context.get i32 0))
+ (core func $cset (canon context.set i32 0))
+ (core module $M
+ (import "" "mid'" (func $mid' (param i32) (result i32)))
+ (import "" "context.get" (func $cget (result i32)))
+ (import "" "context.set" (func $cset (param i32)))
+ (func (export "root'") (result i32) (local $r i32)
+ (call $cset (i32.const 0x11223344))
+ (local.set $r (call $mid' (i32.const 100)))
+ (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable))
+ (i32.add (local.get $r) (i32.const 1000))
+ )
+ )
+ (core instance $m (instantiate $M (with "" (instance
+ (export "mid'" (func $mid'))
+ (export "context.get" (func $cget))
+ (export "context.set" (func $cset))
+ ))))
+ (func (export "root") (result u32)
+ (canon lift (core func $m "root'")))
+ )
+
+ (instance $leaf (instantiate $Leaf))
+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))
+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))
+ (export "root" (func $root "root"))
+)
+(assert_return (invoke "root") (u32.const 1111))
diff --git a/tests/misc_testsuite/component-model/trap.wast b/tests/misc_testsuite/component-model/trap.wast
new file mode 100644
index 0000000000..9653efe031
--- /dev/null
+++ b/tests/misc_testsuite/component-model/trap.wast
@@ -0,0 +1,30 @@
+;;! component_model_async = true
+
+(component
+ (component $Inner
+ (core module $M
+ (func (export "f") (result i32) unreachable)
+ )
+ (core instance $m (instantiate $M))
+ (func (export "f") (result u32) (canon lift (core func $m "f")))
+ )
+
+ (component $Outer
+ (import "f" (func $f (result u32)))
+ (core func $f-lowered (canon lower (func $f)))
+ (core module $N
+ (import "" "f" (func $f (result i32)))
+ (func (export "g") (result i32) (call $f))
+ )
+ (core instance $n (instantiate $N (with "" (instance
+ (export "f" (func $f-lowered))
+ ))))
+ (func (export "g") (result u32) (canon lift (core func $n "g")))
+ )
+
+ (instance $inner (instantiate $Inner))
+ (instance $outer (instantiate $Outer (with "f" (func $inner "f"))))
+ (export "g" (func $outer "g"))
+)
+
+(assert_trap (invoke "g") "wasm `unreachable` instruction executed")