A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Inline `{enter,exit}_sync_call` trampolines into sync-to-sync fused adapters (#13695)
Nick Fitzgerald946f7633b387
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex bb7638a361..b0bdc536d4 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -24,6 +24,7 @@ enum VmType {VMDrcHeapData,VMCopyingHeapData,VMNullHeapData,+ VMDeferredThread,}/// A key that uniquely identifies an alias region across an entire compilation.@@ -117,6 +118,7 @@ impl AliasRegionKey {const VM_DRC_HEAP_DATA_KIND: u32 = Self::new_kind(0b1100);const VM_COPYING_HEAP_DATA_KIND: u32 = Self::new_kind(0b1101);const VM_NULL_HEAP_DATA_KIND: u32 = Self::new_kind(0b1110);+ const VM_DEFERRED_THREAD_KIND: u32 = Self::new_kind(0b1111);/// Encode this key into a raw `u32` suitable for use as an/// `AliasRegionData::user_id`.@@ -133,6 +135,7 @@ impl AliasRegionKey {VmType::VMDrcHeapData => Self::VM_DRC_HEAP_DATA_KIND,VmType::VMCopyingHeapData => Self::VM_COPYING_HEAP_DATA_KIND,VmType::VMNullHeapData => Self::VM_NULL_HEAP_DATA_KIND,+ VmType::VMDeferredThread => Self::VM_DEFERRED_THREAD_KIND,};kind | (offset & Self::OFFSET_MASK)}@@ -1108,6 +1111,44 @@ where)}+ /// Load the `VMStoreContext::current_thread` field (the JIT-visible+ /// deferred-thread pointer; see `VMLazyThread`).+ pub fn vmstore_context_current_thread(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmstore_ctx: ir::Value,+ ) -> ir::Value {+ self.vmstore_context_load(+ cursor,+ self.pointer_type,+ ir::MemFlagsData::trusted(),+ vmstore_ctx,+ self.offsets+ .get_ptr_size()+ .vmstore_context_current_thread()+ .into(),+ )+ }++ /// Store the `VMStoreContext::current_thread` field.+ pub fn store_vmstore_context_current_thread(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmstore_ctx: ir::Value,+ new_thread: ir::Value,+ ) {+ self.vmstore_context_store(+ cursor,+ ir::MemFlagsData::trusted(),+ vmstore_ctx,+ self.offsets+ .get_ptr_size()+ .vmstore_context_current_thread()+ .into(),+ new_thread,+ )+ }+/// Get a `Load` of the GC heap base pointer (`VMStoreContext::gc_heap.base`).////// The caller supplies the base flags because whether the base pointer is@@ -1342,6 +1383,194 @@ where}}+/// `VMDeferredThread`-related methods.+impl<Offsets> AliasRegions<Offsets>+where+ Offsets: GetPtrSize,+{+ fn vmdeferred_thread_region(+ &mut self,+ func: &mut ir::Function,+ offset: u32,+ ) -> ir::AliasRegion {+ self.region(+ func,+ AliasRegionKey::Vm {+ ty: VmType::VMDeferredThread,+ offset,+ },+ )+ }++ fn vmdeferred_thread_load(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ ty: ir::Type,+ base_flags: ir::MemFlagsData,+ vmdeferred_thread_ptr: ir::Value,+ offset: u32,+ ) -> ir::Value {+ let region = self.vmdeferred_thread_region(cursor.func, offset);+ cursor.ins().load(+ ty,+ base_flags.with_alias_region(Some(region)),+ vmdeferred_thread_ptr,+ i32::try_from(offset).unwrap(),+ )+ }++ fn vmdeferred_thread_store(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ base_flags: ir::MemFlagsData,+ vmdeferred_thread_ptr: ir::Value,+ offset: u32,+ val: ir::Value,+ ) {+ let region = self.vmdeferred_thread_region(cursor.func, offset);+ cursor.ins().store(+ base_flags.with_alias_region(Some(region)),+ val,+ vmdeferred_thread_ptr,+ i32::try_from(offset).unwrap(),+ );+ }++ /// Load `VMDeferredThread::parent` (the current thread this frame replaced).+ pub fn vmdeferred_thread_parent(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmdeferred_thread_ptr: ir::Value,+ ) -> ir::Value {+ self.vmdeferred_thread_load(+ cursor,+ self.pointer_type,+ ir::MemFlagsData::trusted(),+ vmdeferred_thread_ptr,+ self.offsets+ .get_ptr_size()+ .vmdeferred_thread_parent()+ .into(),+ )+ }++ /// Store `VMDeferredThread::parent`.+ pub fn store_vmdeferred_thread_parent(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmdeferred_thread_ptr: ir::Value,+ parent: ir::Value,+ ) {+ self.vmdeferred_thread_store(+ cursor,+ ir::MemFlagsData::trusted(),+ vmdeferred_thread_ptr,+ self.offsets+ .get_ptr_size()+ .vmdeferred_thread_parent()+ .into(),+ parent,+ )+ }++ /// Store `VMDeferredThread::caller_instance`.+ pub fn store_vmdeferred_thread_caller_instance(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmdeferred_thread_ptr: ir::Value,+ caller_instance: ir::Value,+ ) {+ self.vmdeferred_thread_store(+ cursor,+ ir::MemFlagsData::trusted(),+ vmdeferred_thread_ptr,+ self.offsets+ .get_ptr_size()+ .vmdeferred_thread_caller_instance()+ .into(),+ caller_instance,+ )+ }++ /// Store `VMDeferredThread::callee_async`.+ pub fn store_vmdeferred_thread_callee_async(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmdeferred_thread_ptr: ir::Value,+ callee_async: ir::Value,+ ) {+ self.vmdeferred_thread_store(+ cursor,+ ir::MemFlagsData::trusted(),+ vmdeferred_thread_ptr,+ self.offsets+ .get_ptr_size()+ .vmdeferred_thread_callee_async()+ .into(),+ callee_async,+ )+ }++ /// Store `VMDeferredThread::callee_instance`.+ pub fn store_vmdeferred_thread_callee_instance(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmdeferred_thread_ptr: ir::Value,+ callee_instance: ir::Value,+ ) {+ self.vmdeferred_thread_store(+ cursor,+ ir::MemFlagsData::trusted(),+ vmdeferred_thread_ptr,+ self.offsets+ .get_ptr_size()+ .vmdeferred_thread_callee_instance()+ .into(),+ callee_instance,+ )+ }++ /// Load `VMDeferredThread::saved_context[i]` (a saved `context.{get,set}`+ /// slot).+ pub fn vmdeferred_thread_saved_context(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmdeferred_thread_ptr: ir::Value,+ i: u8,+ ) -> ir::Value {+ self.vmdeferred_thread_load(+ cursor,+ ir::types::I32,+ ir::MemFlagsData::trusted(),+ vmdeferred_thread_ptr,+ self.offsets+ .get_ptr_size()+ .vmdeferred_thread_saved_context(i)+ .into(),+ )+ }++ /// Store `VMDeferredThread::saved_context[i]`.+ pub fn store_vmdeferred_thread_saved_context(+ &mut self,+ cursor: &mut FuncCursor<'_>,+ vmdeferred_thread_ptr: ir::Value,+ i: u8,+ val: ir::Value,+ ) {+ self.vmdeferred_thread_store(+ cursor,+ ir::MemFlagsData::trusted(),+ vmdeferred_thread_ptr,+ self.offsets+ .get_ptr_size()+ .vmdeferred_thread_saved_context(i)+ .into(),+ val,+ )+ }+}+/// `VMMemoryDefinition`-related methods.////// The `base` and `current_length` fields are reached either directly throughdiff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rsindex 793cd20664..bfabe07a02 100644--- a/crates/cranelift/src/compiler.rs+++ b/crates/cranelift/src/compiler.rs@@ -1523,6 +1523,20 @@ impl Compiler {// succeed". Note that register restoration is part of the `try_call`// and handler implementation.builder.switch_to_block(exceptional_return);+ if cfg!(feature = "component-model") && self.tunables().concurrency_support {+ // NB: A trap unwinds over any fused adapter frames without running+ // their `exit-sync-call`s. That can leave+ // `VMStoreContext::current_thread` pointing at an old, invalid+ // `VMDeferredThread` inside an old, since-unwound stack+ // frame. Therefore, we must reset `current_thread` to avoid+ // potential use-after-free bugs.+ let forced = builder.ins().iconst(pointer_type, 1);+ alias_regions.store_vmstore_context_current_thread(+ &mut builder.cursor(),+ vm_store_ctx,+ forced,+ );+ }let false_return = builder.ins().iconst(ir::types::I8, 0);builder.ins().return_(&[false_return]);diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 0a915a7b19..9dd47f3d7a 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -35,13 +35,14 @@ use wasmparser::{use wasmtime_core::math::f64_cvt_to_int_bounds;use wasmtime_environ::{BuiltinFunctionIndex, ComponentPC, ConstExpr, ConstOp, DataIndex, DefinedFuncIndex,- DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex,+ DefinedGlobalIndex, DefinedTableIndex, ElemIndex, EngineOrModuleTypeIndex, FactInlineIntrinsic,FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey, GlobalConstValue, GlobalIndex,- IndexType, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables, Module,- ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PassiveElemIndex, PtrSize,- RuntimeDataIndex, Table, TableIndex, TableInitialValue, TableSegment, TableSegmentElements,- TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,- WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,+ IndexType, KnownFunc, Memory, MemoryIndex, MemoryInit, MemorySegmentOffset, MemoryTunables,+ Module, ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder,+ NUM_COMPONENT_CONTEXT_SLOTS, PassiveElemIndex, PtrSize, RuntimeDataIndex, Table, TableIndex,+ TableInitialValue, TableSegment, TableSegmentElements, TagIndex, Tunables, TypeConvert,+ TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType,+ WasmRefType, WasmResult, WasmStorageType, WasmValType,};use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};@@ -166,6 +167,12 @@ pub struct FuncEnvironment<'module_environment> {/// Heaps implementing WebAssembly linear memories.heaps: PrimaryMap<Heap, HeapData>,+ /// When translating a fused sync-to-sync adapter, the explicit stack slot+ /// holding the `VMDeferredThread` pushed by the inline `enter-sync-call`+ /// lowering, shared with the matching inline `exit-sync-call`. `None`+ /// outside that window.+ fact_sync_call_slot: Option<ir::StackSlot>,+/// Caches of signatures for builtin functions.builtin_functions: BuiltinFunctions,@@ -276,6 +283,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> {gc_heap: None,heaps: PrimaryMap::default(),+ fact_sync_call_slot: None,builtin_functions,offsets,tunables,@@ -1523,12 +1531,12 @@ impl FuncEnvironment<'_> {let signature = self.get_or_create_interned_sig_ref(func, ty);let key = match self.translation.known_imported_functions[func_index] {- Some(key @ FuncKey::DefinedWasmFunction(..)) => key,+ Some(KnownFunc::FuncKey(key @ FuncKey::DefinedWasmFunction(..))) => key,- Some(key @ FuncKey::UnsafeIntrinsic(..)) => key,+ Some(KnownFunc::FuncKey(key @ FuncKey::UnsafeIntrinsic(..))) => key,- Some(key) => {- panic!("unexpected kind of known-import function: {key:?}")+ Some(ref f) => {+ panic!("unexpected kind of known-import function: {f:?}")}None => panic!(@@ -1872,7 +1880,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// The import is always a compile-time builtin intrinsic. Make a// direct call to that function (presumably it will eventually be// inlined).- Some(FuncKey::UnsafeIntrinsic(abi, intrinsic)) => {+ Some(KnownFunc::FuncKey(FuncKey::UnsafeIntrinsic(abi, intrinsic))) => {let callee = self.env.get_or_create_imported_func_ref(self.builder.func, callee_index);@@ -1898,13 +1906,43 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {// function, so do a direct call to that function! (Although we take// care to still pass its `funcref`'s `vmctx` as the callee `vmctx`// in `real_call_args` and not the caller's.)- Some(FuncKey::DefinedWasmFunction(..)) => {+ Some(KnownFunc::FuncKey(FuncKey::DefinedWasmFunction(..))) => {let callee = self.env.get_or_create_imported_func_ref(self.builder.func, callee_index);Ok(self.direct_call_inst(callee, &real_call_args))}+ // The guest-to-guest sync fast path: these adapter intrinsics are+ // lowered inline rather than called, but only when concurrency+ // support is enabled (the deferred thread state only exists then)+ // and this isn't a tail call (the deferred frame must outlive the+ // call). Otherwise fall back to the indirect call, which is also+ // the out-of-line slow path the inline `exit` branches to.+ Some(KnownFunc::FactIntrinsic(intrinsic)) => {+ if self.env.tunables.concurrency_support {+ debug_assert!(!self.tail);+ match intrinsic {+ FactInlineIntrinsic::EnterSyncCall => {+ return Ok(self.lower_fact_enter_sync_call(&real_call_args));+ }+ FactInlineIntrinsic::ExitSyncCall => {+ return Ok(self.lower_fact_exit_sync_call(+ callee_index,+ sig_ref,+ &real_call_args,+ ));+ }+ }+ }+ let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(+ &mut self.builder.cursor(),+ vmctx,+ callee_index,+ );+ Ok(self.indirect_call_inst(sig_ref, func_addr, &real_call_args))+ }+Some(key) => panic!("unexpected kind of known-import function: {key:?}"),// Unknown import function or this module is instantiated many times@@ -1935,6 +1973,187 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {abi == wasmtime_environ::Abi::Wasm && !self.tail && !self.env.tunables.debug_guest}+ /// Inline lowering of a FACT adapter's `enter-sync-call` intrinsic: push a+ /// `VMDeferredThread` onto an explicit stack slot and publish it as the+ /// store's current thread, deferring the heavyweight task bookkeeping the+ /// `enter_sync_call` libcall would otherwise do eagerly.+ ///+ /// `real_call_args` is `[callee_vmctx, caller_vmctx, caller_instance,+ /// callee_async, callee_instance]`.+ fn lower_fact_enter_sync_call(&mut self, real_call_args: &[ir::Value]) -> CallRets {+ let ptr_ty = self.env.pointer_type();+ let ptr = self.env.offsets.ptr;++ // Allocate the on-stack `VMDeferredThread`.+ let size = u32::from(ptr.size_of_vmdeferred_thread());+ let align_shift = u8::try_from(ptr.size().trailing_zeros()).unwrap();+ let slot = self+ .builder+ .func+ .create_sized_stack_slot(ir::StackSlotData::new(+ ir::StackSlotKind::ExplicitSlot,+ size,+ align_shift,+ ));+ let slot_addr = self.builder.ins().stack_addr(ptr_ty, slot, 0);++ let vmstore = self.env.get_vmstore_context_ptr(self.builder);++ // Link the previous current thread in as this frame's parent.+ let parent = self+ .env+ .alias_regions+ .vmstore_context_current_thread(&mut self.builder.cursor(), vmstore);+ self.env.alias_regions.store_vmdeferred_thread_parent(+ &mut self.builder.cursor(),+ slot_addr,+ parent,+ );++ // Record the deferred `enter_sync_call` arguments.+ self.env+ .alias_regions+ .store_vmdeferred_thread_caller_instance(+ &mut self.builder.cursor(),+ slot_addr,+ real_call_args[2],+ );+ self.env.alias_regions.store_vmdeferred_thread_callee_async(+ &mut self.builder.cursor(),+ slot_addr,+ real_call_args[3],+ );+ self.env+ .alias_regions+ .store_vmdeferred_thread_callee_instance(+ &mut self.builder.cursor(),+ slot_addr,+ real_call_args[4],+ );++ // Save the caller's context slots into the frame and reset the live+ // values to 0 for the freshly-entered (deferred) thread.+ for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {+ let saved = self+ .env+ .alias_regions+ .vmstore_context_component_context_slot(+ &mut self.builder.cursor(),+ ir::types::I32,+ vmstore,+ i,+ );+ self.env+ .alias_regions+ .store_vmdeferred_thread_saved_context(+ &mut self.builder.cursor(),+ slot_addr,+ i,+ saved,+ );+ let zero = self.builder.ins().iconst(ir::types::I32, 0);+ self.env+ .alias_regions+ .store_vmstore_context_component_context_slot(+ &mut self.builder.cursor(),+ vmstore,+ i,+ zero,+ );+ }++ // Publish the deferred thread as the store's current thread.+ self.env.alias_regions.store_vmstore_context_current_thread(+ &mut self.builder.cursor(),+ vmstore,+ slot_addr,+ );++ debug_assert!(self.env.fact_sync_call_slot.is_none());+ self.env.fact_sync_call_slot = Some(slot);+ CallRets::new()+ }++ /// Inline lowering of a FACT adapter's `exit-sync-call` intrinsic, the+ /// counterpart to `lower_fact_enter_sync_call`. If our deferred thread is+ /// still current (nothing forced it) we pop it and restore the caller's+ /// context inline; otherwise we fall back to the out-of-line+ /// `exit_sync_call` libcall.+ fn lower_fact_exit_sync_call(+ &mut self,+ callee_index: FuncIndex,+ sig_ref: ir::SigRef,+ real_call_args: &[ir::Value],+ ) -> CallRets {+ let ptr_ty = self.env.pointer_type();++ let slot = self+ .env+ .fact_sync_call_slot+ .take()+ .expect("inline exit-sync-call without a matching enter-sync-call");+ let slot_addr = self.builder.ins().stack_addr(ptr_ty, slot, 0);+ let vmstore = self.env.get_vmstore_context_ptr(self.builder);+ let cur = self+ .env+ .alias_regions+ .vmstore_context_current_thread(&mut self.builder.cursor(), vmstore);+ let is_fast = self.builder.ins().icmp(IntCC::Equal, cur, slot_addr);++ let fast_block = self.builder.create_block();+ let slow_block = self.builder.create_block();+ let cont_block = self.builder.create_block();+ self.builder+ .ins()+ .brif(is_fast, fast_block, &[], slow_block, &[]);+ self.builder.seal_block(fast_block);+ self.builder.seal_block(slow_block);++ // Fast path: pop the deferred thread and restore the caller's context.+ self.builder.switch_to_block(fast_block);+ let parent = self+ .env+ .alias_regions+ .vmdeferred_thread_parent(&mut self.builder.cursor(), slot_addr);+ self.env.alias_regions.store_vmstore_context_current_thread(+ &mut self.builder.cursor(),+ vmstore,+ parent,+ );+ for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {+ let saved = self.env.alias_regions.vmdeferred_thread_saved_context(+ &mut self.builder.cursor(),+ slot_addr,+ i,+ );+ self.env+ .alias_regions+ .store_vmstore_context_component_context_slot(+ &mut self.builder.cursor(),+ vmstore,+ i,+ saved,+ );+ }+ self.builder.ins().jump(cont_block, &[]);++ // Slow path: the thread was promoted to a real one, so do the+ // equivalent out-of-line teardown via the `exit_sync_call` libcall.+ self.builder.switch_to_block(slow_block);+ let vmctx = self.env.vmctx_val(&mut self.builder.cursor());+ let func_addr = self.env.alias_regions.vmctx_vmfunction_import_wasm_call(+ &mut self.builder.cursor(),+ vmctx,+ callee_index,+ );+ self.indirect_call_inst(sig_ref, func_addr, real_call_args);+ self.builder.ins().jump(cont_block, &[]);++ self.builder.seal_block(cont_block);+ self.builder.switch_to_block(cont_block);+ CallRets::new()+ }+/// Do a Wasm-level indirect call through the given funcref table.pub fn indirect_call(mut self,diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rsindex 542181e55f..d60f35b641 100644--- a/crates/environ/src/compile/module_environ.rs+++ b/crates/environ/src/compile/module_environ.rs@@ -37,6 +37,39 @@ pub struct ModuleEnvironment<'a, 'data> {tunables: &'a Tunables,}+/// Identifies a FACT adapter-module import that the compiler lowers inline when+/// translating the adapter function.+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]+pub enum FactInlineIntrinsic {+ /// `enter-sync-call`: push a deferred component-model thread inline.+ EnterSyncCall,+ /// `exit-sync-call`: pop the deferred thread inline on the fast path, or+ /// fall back to the out-of-line `exit-sync-call` libcall when the thread+ /// was promoted.+ ExitSyncCall,+}++/// TODO FITZGEN+#[derive(Clone, Debug)]+pub enum KnownFunc {+ /// TODO FITZGEN+ FuncKey(FuncKey),+ /// TODO FITZGEN+ FactIntrinsic(FactInlineIntrinsic),+}++impl From<FuncKey> for KnownFunc {+ fn from(key: FuncKey) -> Self {+ Self::FuncKey(key)+ }+}++impl From<FactInlineIntrinsic> for KnownFunc {+ fn from(intrinsic: FactInlineIntrinsic) -> Self {+ Self::FactIntrinsic(intrinsic)+ }+}+/// The result of translating via `ModuleEnvironment`.////// Function bodies are not yet translated, and data initializers have not yet@@ -68,8 +101,9 @@ pub struct ModuleTranslation<'data> {/// imports table into direct calls, when possible.////// When filled in, this only ever contains- /// `FuncKey::DefinedWasmFunction(..)`s and `FuncKey::Intrinsic(..)`s.- pub known_imported_functions: SecondaryMap<FuncIndex, Option<FuncKey>>,+ /// `FuncKey::DefinedWasmFunction(..)`s, `FuncKey::Intrinsic(..)`s, and+ /// `FuncKey::FactInlineIntrinsic`s.+ pub known_imported_functions: SecondaryMap<FuncIndex, Option<KnownFunc>>,/// A list of type signatures which are considered exported from this/// module, or those that can possibly be called. This list is sorted, anddiff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rsindex 447ec4faa8..edd81b72a3 100644--- a/crates/environ/src/component/translate.rs+++ b/crates/environ/src/component/translate.rs@@ -3,9 +3,9 @@ use crate::component::dfg::AbstractInstantiations;use crate::component::*;use crate::prelude::*;use crate::{- EngineOrModuleTypeIndex, EntityIndex, FuncKey, ModuleEnvironment, ModuleInternedTypeIndex,- ModuleTranslation, ModuleTypesBuilder, PrimaryMap, ScopeVec, TagIndex, Tunables, TypeConvert,- WasmHeapType, WasmResult, WasmValType,+ EngineOrModuleTypeIndex, EntityIndex, FactInlineIntrinsic, FuncKey, ModuleEnvironment,+ ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PrimaryMap, ScopeVec, TagIndex,+ Tunables, TypeConvert, WasmHeapType, WasmResult, WasmValType,};use core::str::FromStr;use cranelift_entity::SecondaryMap;@@ -623,12 +623,21 @@ impl<'a, 'data> Translator<'a, 'data> {// turn them into direct calls even though we probably// wouldn't ever inline them, but it just doesn't seem worth// the effort.- CoreDef::Trampoline(_) => continue,+ //+ // That said, a couple of adapter trampolines are lowered+ // inline during translation. We record these here so+ // `FuncEnvironment` recognizes them. All other trampolines+ // remain indirect calls.+ CoreDef::Trampoline(index) => match translation.trampolines[*index] {+ Trampoline::EnterSyncCall => FactInlineIntrinsic::EnterSyncCall.into(),+ Trampoline::ExitSyncCall => FactInlineIntrinsic::ExitSyncCall.into(),+ _ => continue,+ },// This import is a compile-time builtin intrinsic, we// should inline its implementation during function// translation.- CoreDef::UnsafeIntrinsic(i) => FuncKey::UnsafeIntrinsic(Abi::Wasm, *i),+ CoreDef::UnsafeIntrinsic(i) => FuncKey::UnsafeIntrinsic(Abi::Wasm, *i).into(),// This imported function is an export from another// instance, a perfect candidate for becoming an inlinable@@ -660,7 +669,7 @@ impl<'a, 'data> Translator<'a, 'data> {continue;};- FuncKey::DefinedWasmFunction(*arg_module, arg_module_def_func)+ FuncKey::DefinedWasmFunction(*arg_module, arg_module_def_func).into()}};diff --git a/crates/environ/src/fact/trampoline.rs b/crates/environ/src/fact/trampoline.rsindex 4539ad13ec..bf1236dd90 100644--- a/crates/environ/src/fact/trampoline.rs+++ b/crates/environ/src/fact/trampoline.rs@@ -790,8 +790,12 @@ impl<'a, 'b> Compiler<'a, 'b> {// Push a task onto the current task stack.//- // FIXME: Apply the optimizations described in #12311.-+ // Note that for sync-to-sync calls, we replace this call with+ // inline code for lazy/deferred task creation during translation to+ // CLIF. This avoids task creation and out-of-line calls in the+ // adapter for most sync-to-sync calls, since most sync-to-sync+ // calls do not do anything to force the task's creation+ // (e.g. adjust backpressure).self.instruction(I32Const(i32::try_from(adapter.lower.instance.as_u32()).unwrap(),));@@ -893,7 +897,9 @@ impl<'a, 'b> Compiler<'a, 'b> {// lowering below may call realloc which is in the context of the// caller's task, not the callee.//- // FIXME: Apply the optimizations described in #12311.+ // Note that for sync-to-sync calls, we will emit inline code during+ // translation to CLIF to avoid actually calling out to a libcall when+ // the deferred task's allocation was never forced.if self.emit_resource_call || self.module.tunables.concurrency_support {let exit_sync_call = self.module.import_exit_sync_call();self.instruction(Call(exit_sync_call.as_u32()));@@ -3601,12 +3607,12 @@ impl<'a, 'b> Compiler<'a, 'b> {////// The `may_leave` flag is a boolean (0 or 1) so no masking is required.fn trap_if_not_may_leave(&mut self, flags_global: GlobalIndex, trap: Trap) -> TempLocal {+ self.instruction(Block(BlockType::Empty));self.instruction(GlobalGet(flags_global.as_u32()));// Save the flag's value (known to be `true` whenever the trap below is// not taken) into a temporary for later restoration.let saved = self.local_tee_new_tmp(ValType::I32);- self.instruction(I32Eqz);- self.instruction(If(BlockType::Empty));+ self.instruction(BrIf(0));self.trap(trap);self.instruction(End);saveddiff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rsindex c3173b3819..80ec061049 100644--- a/crates/environ/src/module_artifacts.rs+++ b/crates/environ/src/module_artifacts.rs@@ -555,6 +555,8 @@ impl CompiledFunctionsTable {| FuncKeyKind::PatchableToBuiltinTrampoline| FuncKeyKind::ModuleStartup => false,+ // `FactInlineIntrinsic` never names a compiled function, so it is+ // never classified here; group it with the other intrinsics.#[cfg(feature = "component-model")]FuncKeyKind::ComponentTrampoline| FuncKeyKind::ResourceDropTrampolinediff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex df1cbc516f..244ab92e39 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -279,6 +279,46 @@ pub trait PtrSize {base + i * slot_size}+ /// Return the offset of the `current_thread` field of `VMStoreContext`.+ fn vmstore_context_current_thread(&self) -> u8 {+ self.vmstore_context_component_context_slot(0) + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4+ }++ // Offsets within `VMDeferredThread`++ /// Offset of `VMDeferredThread::parent`.+ fn vmdeferred_thread_parent(&self) -> u8 {+ 0+ }++ /// Offset of `VMDeferredThread::caller_instance`.+ fn vmdeferred_thread_caller_instance(&self) -> u8 {+ self.size()+ }++ /// Offset of `VMDeferredThread::callee_async`.+ fn vmdeferred_thread_callee_async(&self) -> u8 {+ self.vmdeferred_thread_caller_instance() + 4+ }++ /// Offset of `VMDeferredThread::callee_instance`.+ fn vmdeferred_thread_callee_instance(&self) -> u8 {+ self.vmdeferred_thread_callee_async() + 4+ }++ /// Offset of `VMDeferredThread::saved_context[i]`.+ fn vmdeferred_thread_saved_context(&self, i: u8) -> u8 {+ assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);+ self.vmdeferred_thread_callee_instance() + 4 + i * 4+ }++ /// Return the size of `VMDeferredThread`, rounded up to pointer alignment.+ fn size_of_vmdeferred_thread(&self) -> u8 {+ let unaligned =+ self.vmdeferred_thread_callee_instance() + 4 + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4;+ align(u32::from(unaligned), u32::from(self.size())) as u8+ }+// Offsets within `VMMemoryDefinition`/// The offset of the `base` field.diff --git a/crates/misc/component-async-tests/tests/scenario/round_trip.rs b/crates/misc/component-async-tests/tests/scenario/round_trip.rsindex a6ecec7d29..bb2c9fee8c 100644--- a/crates/misc/component-async-tests/tests/scenario/round_trip.rs+++ b/crates/misc/component-async-tests/tests/scenario/round_trip.rs@@ -461,7 +461,7 @@ pub async fn test_round_trip(.map(|(a, b)| (String::from(*a), String::from(*b))).collect::<Vec<_>>(),tx,- });+ })?;store.run_concurrent(async |_| rx.await).await??;diff --git a/crates/wasi-tls/src/p3/host.rs b/crates/wasi-tls/src/p3/host.rsindex 857072116e..66c084f415 100644--- a/crates/wasi-tls/src/p3/host.rs+++ b/crates/wasi-tls/src/p3/host.rs@@ -101,7 +101,7 @@ impl<T> bindings::tls::client::HostConnectorWithStore<T> for WasiTls {.map_err(|e| Error::from(e));_ = send_result_tx.send(combined_result);Ok(())- }));+ }))?;let result = ResultProducer::new(getter, send_result_rx);Ok((@@ -167,7 +167,7 @@ impl<T> bindings::tls::client::HostConnectorWithStore<T> for WasiTls {.map_err(|e| Error::from(e));_ = recv_result_tx.send(combined_result);Ok(())- }));+ }))?;let result = ResultProducer::new(getter, recv_result_rx);Ok((diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex 4f6501056b..357ff565e3 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -65,7 +65,7 @@ use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};#[cfg(feature = "gc")]use crate::vm::GcRootsList;use crate::vm::component::{CallContext, ComponentInstance, InstanceState};-use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMMemoryDefinition, VMStore};+use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore};use crate::{AsContext, AsContextMut, FuncType, Result, StoreContext, StoreContextMut, ValRaw, ValType, bail,};@@ -241,7 +241,7 @@ where/// Spawn a background task.////// See [`Accessor::spawn`] for details.- pub fn spawn(&mut self, task: impl AccessorTask<T, D>) -> JoinHandle+ pub fn spawn(&mut self, task: impl AccessorTask<T, D>) -> Result<JoinHandle>whereT: 'static,{@@ -517,7 +517,7 @@ where/// Panics if called within a closure provided to the [`Accessor::with`]/// function. This can only be called outside an active invocation of/// [`Accessor::with`].- pub fn spawn(&self, task: impl AccessorTask<T, D>) -> JoinHandle+ pub fn spawn(&self, task: impl AccessorTask<T, D>) -> Result<JoinHandle>whereT: 'static,{@@ -570,7 +570,7 @@ wherepub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {self.with(|mut access| {let store = access.as_context_mut().0;- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut_without_forcing_current_thread();if state.interesting_tasks == 0 {Poll::Ready(())} else {@@ -723,7 +723,7 @@ impl GuestCall {/// instance to be called has backpressure enabledfn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {let instance = store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(self.thread.task)?.instance;let state = store.instance_state(instance).concurrent_state();@@ -801,8 +801,7 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(store: &mut dyn VMStore,future: impl Future<Output = Result<R>> + Send + 'static,) -> Result<R> {- let state = store.concurrent_state_mut();- let task = state.current_host_thread()?;+ let task = store.current_host_thread()?;// Wrap the future in a closure which will take care of stashing the result// in `GuestTask::result` and resuming this fiber when the host task@@ -810,7 +809,7 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(let mut future = Box::pin(async move {let result = future.await?;tls::get(move |store| {- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let host_state = &mut state.get_mut(task)?.state;assert!(matches!(host_state, HostTaskState::CalleeStarted));*host_state = HostTaskState::CalleeFinished(Box::new(result));@@ -844,7 +843,7 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(// then use `GuestThread::sync_call_set` to wait for the task to// complete, suspending the current fiber until it does so.Poll::Pending => {- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;state.push_future(future);let caller = state.get_mut(task)?.caller;@@ -860,12 +859,12 @@ pub(crate) fn poll_and_block<R: Send + Sync + 'static>(// Remove the `task` from the `sync_call_set` to ensure that when// this function returns and the task is deleted that there are no// more lingering references to this host task.- Waitable::Host(task).join(store.concurrent_state_mut(), None)?;+ Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;}}// Retrieve and return the result.- let host_state = &mut store.concurrent_state_mut().get_mut(task)?.state;+ let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {Ok(result) => *result,@@ -886,7 +885,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {Some(pair) => pair,None => bail_bug!("delivering non-present event"),};- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let task = state.get_mut(call.thread.task)?;let runtime_instance = task.instance;let handle = waitable.map(|(_, v)| v).unwrap_or(0);@@ -905,7 +904,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {store.enter_instance(runtime_instance);let Some(callback) = store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(call.thread.task)?.callback.take()@@ -916,7 +915,7 @@ fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {let code = callback(store, event, handle)?;store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(call.thread.task)?.callback = Some(callback);@@ -971,7 +970,7 @@ impl<T> Store<T> {}/// Convenience wrapper for [`StoreContextMut::spawn`].- pub fn spawn(&mut self, task: impl AccessorTask<T, HasSelf<T>>) -> JoinHandle+ pub fn spawn(&mut self, task: impl AccessorTask<T, HasSelf<T>>) -> Result<JoinHandle>whereT: 'static,{@@ -997,7 +996,7 @@ impl<T> StoreContextMut<'_, T> {.store_data_mut().components.assert_instance_states_empty();- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut().unwrap();assert!(state.table.get_mut().is_empty(),"non-empty table: {:?}",@@ -1005,7 +1004,7 @@ impl<T> StoreContextMut<'_, T> {);assert!(state.high_priority.is_empty());assert!(state.low_priority.is_empty());- assert!(state.current_thread.is_none());+ assert!(state.unforced_current_thread.is_none());assert!(state.futures_mut().unwrap().is_empty());assert!(state.global_error_context_ref_counts.is_empty());}@@ -1018,6 +1017,7 @@ impl<T> StoreContextMut<'_, T> {pub fn concurrent_state_table_size(&mut self) -> usize {self.0.concurrent_state_mut()+ .unwrap().table.get_mut().iter_mut()@@ -1033,7 +1033,7 @@ impl<T> StoreContextMut<'_, T> {/// for this instance is run.////// The returned [`JoinHandle`] may be used to cancel the task.- pub fn spawn(mut self, task: impl AccessorTask<T>) -> JoinHandle+ pub fn spawn(mut self, task: impl AccessorTask<T>) -> Result<JoinHandle>whereT: 'static,{@@ -1047,7 +1047,7 @@ impl<T> StoreContextMut<'_, T> {self,accessor: Accessor<T, D>,task: impl AccessorTask<T, D>,- ) -> JoinHandle+ ) -> Result<JoinHandle>whereT: 'static,D: HasData + ?Sized,@@ -1057,9 +1057,9 @@ impl<T> StoreContextMut<'_, T> {// iteration.let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });self.0- .concurrent_state_mut()+ .concurrent_state_mut()?.push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));- handle+ Ok(handle)}/// Run the specified closure `fun` to completion as part of this store's@@ -1231,7 +1231,12 @@ impl<T> StoreContextMut<'_, T> {impl<'a, T> Drop for Reset<'a, T> {fn drop(&mut self) {if let Some(futures) = self.futures.take() {- *self.store.0.concurrent_state_mut().futures.get_mut() = Some(futures);+ *self+ .store+ .0+ .concurrent_state_mut_already_forced_current_thread()+ .futures+ .get_mut() = Some(futures);}}}@@ -1240,7 +1245,7 @@ impl<T> StoreContextMut<'_, T> {// Take `ConcurrentState::futures` out of the store so we can poll// it while also safely giving any of the futures inside access to// `self`.- let futures = self.0.concurrent_state_mut().futures.get_mut().take();+ let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();let mut reset = Reset {store: self.as_context_mut(),futures,@@ -1283,7 +1288,7 @@ impl<T> StoreContextMut<'_, T> {// Next, collect the next batch of work items to process, if// any. This will be either all of the high-priority work// items, or if there are none, a single low-priority work item.- let state = reset.store.0.concurrent_state_mut();+ let state = reset.store.0.concurrent_state_mut()?;let mut ready = mem::take(&mut state.high_priority);let mut low_priority = false;if ready.is_empty() {@@ -1434,7 +1439,7 @@ impl<T> StoreContextMut<'_, T> {match item {WorkItem::PushFuture(future) => {self.0- .concurrent_state_mut()+ .concurrent_state_mut()?.futures_mut()?.push(future.into_inner());}@@ -1443,7 +1448,7 @@ impl<T> StoreContextMut<'_, T> {}WorkItem::ResumeThread(_, thread) => {if let GuestThreadState::Ready { fiber, .. } = mem::replace(- &mut self.0.concurrent_state_mut().get_mut(thread.thread)?.state,+ &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state,GuestThreadState::Running,) {self.0.resume_fiber(fiber).await?;@@ -1455,7 +1460,7 @@ impl<T> StoreContextMut<'_, T> {if call.is_ready(self.0)? {self.run_on_worker(WorkerItem::GuestCall(call)).await?;} else {- let state = self.0.concurrent_state_mut();+ let state = self.0.concurrent_state_mut()?;let task = state.get_mut(call.thread.task)?;if !task.starting_sent {task.starting_sent = true;@@ -1490,12 +1495,12 @@ impl<T> StoreContextMut<'_, T> {whereT: Send,{- let worker = if let Some(fiber) = self.0.concurrent_state_mut().worker.take() {+ let worker = if let Some(fiber) = self.0.concurrent_state_mut()?.worker.take() {fiber} else {fiber::make_fiber(self.0, move |store| {loop {- let Some(item) = store.concurrent_state_mut().worker_item.take() else {+ let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {bail_bug!("worker_item not present when resuming fiber")};match item {@@ -1508,7 +1513,7 @@ impl<T> StoreContextMut<'_, T> {})?};- let worker_item = &mut self.0.concurrent_state_mut().worker_item;+ let worker_item = &mut self.0.concurrent_state_mut()?.worker_item;assert!(worker_item.is_none());*worker_item = Some(item);@@ -1544,10 +1549,10 @@ impl<T> StoreContextMut<'_, T> {/// Tasks are yielded "youngest first" where the first item in the iterator/// is the current task, and the last item in the iterator is the original/// call.- pub fn async_call_stack(&mut self) -> impl Iterator<Item = GuestTaskId> {- let state = self.0.concurrent_state_mut();- let mut cur = Some(state.current_thread);- core::iter::from_fn(move || {+ pub fn async_call_stack(&mut self) -> Result<impl Iterator<Item = GuestTaskId>> {+ let mut cur = Some(self.0.current_thread()?);+ let state = self.0.concurrent_state_mut()?;+ Ok(core::iter::from_fn(move || {while let Some(t) = cur {cur = state.parent(t);if let Some(thread) = t.guest() {@@ -1556,17 +1561,132 @@ impl<T> StoreContextMut<'_, T> {}None- })+ }))}}impl StoreOpaque {+ /// Returns the currently-running thread, promoting any deferred lazy thread+ /// into a fully-materialized `CurrentThread`.+ pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {+ // Without concurrency support there is nothing to force.+ if !self.concurrency_support() {+ return Ok(CurrentThread::None);+ }++ // If the JIT-visible current thread isn't a deferred thread then+ // `ConcurrentState` is already up to date.+ if !self+ .vm_store_context_mut()+ .current_thread_mut()+ .is_deferred()+ {+ return Ok(self+ .concurrent_state_mut_already_forced_current_thread()+ .unforced_current_thread);+ }++ // The component instance whose adapters pushed the deferred frames; all+ // frames in a guest-to-guest, sync-to-sync call chain of fused adapters+ // live within a single `wasmtime::component::Instance` (because+ // cross-`wasmtime::component::Instance` calls don't go through fused+ // adapters), and guest code only ever runs as a guest thread, so the+ // chain's base thread is already materialized in `ConcurrentState` and+ // we can get the `ComponentInstanceId` shared by the whole chain from+ // here.+ let state = self.concurrent_state_mut_without_forcing_current_thread();+ let id = match state.unforced_current_thread.guest().copied() {+ Some(thread) => state.get_mut(thread.task)?.instance.instance,+ None => bail_bug!("deferred component-model thread with non-guest base"),+ };++ // Collect the deferred frames pushed inline by fused adapters, walking+ // the `parent` chain from innermost to the base.+ let mut frames = Vec::new();+ let mut cur = *self.vm_store_context_mut().current_thread_mut();+ while let Some(ptr) = cur.as_deferred() {+ // SAFETY: `ptr` points at a `VMDeferredThread` living in a fused+ // adapter's stack frame that is suspended below us on the stack+ // (mid-call, waiting for this nested call to return), so the+ // referent is still valid and exclusively ours to read.+ let deferred = unsafe { ptr.as_non_null().as_ref() };+ frames.push((+ deferred.callee_async != 0,+ deferred.callee_instance,+ deferred.saved_context,+ ));+ cur = deferred.parent;+ }++ // Mark the current thread forced *before* replaying so that any+ // reentrant `force_current_thread` call short-circuits via the+ // non-deferred path above.+ *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();++ // Save the current context, as we need to overwrite it while replaying+ // below.+ let current_context = *self.vm_store_context_mut().component_context_mut();++ // Replay the deferred `enter_guest_sync_call`s outermost-first so that+ // the resulting `ConcurrentState` matches what the non-deferred path+ // would have otherwise produced.+ for (callee_async, callee_instance, saved_context) in frames.into_iter().rev() {+ // Restore the caller's context slots so that we save the correct+ // values into the caller's thread, exactly as the non-deferred path+ // would have on entry.+ *self.vm_store_context_mut().component_context_mut() = saved_context;+ let callee = RuntimeInstance {+ instance: id,+ index: RuntimeComponentInstanceIndex::from_u32(callee_instance),+ };+ self.enter_guest_sync_call(None, callee_async, callee)?;+ }++ // Replaying done; restore the current context.+ *self.vm_store_context_mut().component_context_mut() = current_context;++ Ok(self+ .concurrent_state_mut_without_forcing_current_thread()+ .unforced_current_thread)+ }++ fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {+ match self.current_thread()?.guest() {+ Some(id) => Ok(*id),+ None => bail_bug!("current thread is not a guest thread"),+ }+ }++ fn current_host_thread(&mut self) -> Result<TableId<HostTask>> {+ match self.current_thread()?.host() {+ Some(id) => Ok(id),+ None => bail_bug!("current thread is not a host thread"),+ }+ }++ /// Returns whether there's a pending cancellation on the current guest thread,+ /// consuming the event if so.+ fn take_pending_cancellation(&mut self) -> Result<bool> {+ let thread = self.current_guest_thread()?;+ let task = self.concurrent_state_mut()?.get_mut(thread.task)?;+ if let Some(Event::Cancelled) = task.event {+ task.event.take();+ return Ok(true);+ }+ Ok(false)+ }+/// Push a `GuestTask` onto the task stack for either a sync-to-sync,/// guest-to-guest call or a sync host-to-guest call.////// This task will only be used for the purpose of handling calls to/// intrinsic functions; both parameter lowering and result lifting are/// assumed to be taken care of elsewhere.+ ///+ /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in+ /// fused adapters, see `StoreOpaque::current_thread`, `VMDeferredThread`,+ /// and `lower_fact_enter_sync_call`. Make sure all this stuff stays in+ /// sync!pub(crate) fn enter_guest_sync_call(&mut self,guest_caller: Option<RuntimeInstance>,@@ -1578,8 +1698,8 @@ impl StoreOpaque {return self.enter_call_not_concurrent();}- let state = self.concurrent_state_mut();- let thread = state.current_thread;+ let thread = self.current_thread()?;+ let state = self.concurrent_state_mut()?;let instance = if let Some(thread) = thread.guest() {Some(state.get_mut(thread.task)?.instance)} else {@@ -1622,6 +1742,12 @@ impl StoreOpaque {}/// Pop a `GuestTask` previously pushed using `enter_sync_call`.+ ///+ /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in+ /// fused adapters and then when the call returns we check to see if the+ /// task's contruction was forced and if not avoid calling out of the JIT+ /// code to this function. See `lower_fact_exit_sync_call`. Make sure all+ /// this stuff stays in sync!pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {if !self.concurrency_support() {return Ok(self.exit_call_not_concurrent());@@ -1630,7 +1756,7 @@ impl StoreOpaque {Some(t) => *t,None => bail_bug!("expected task when exiting"),};- let task = self.concurrent_state_mut().get_mut(thread.task)?;+ let task = self.concurrent_state_mut()?.get_mut(thread.task)?;let instance = task.instance;let caller = match &task.caller {&Caller::Guest { thread } => thread.into(),@@ -1658,8 +1784,8 @@ impl StoreOpaque {self.enter_call_not_concurrent()?;return Ok(None);}- let state = self.concurrent_state_mut();- let caller = state.current_guest_thread()?;+ let caller = self.current_guest_thread()?;+ let state = self.concurrent_state_mut()?;let task = state.push(HostTask::new(caller, HostTaskState::CalleeStarted))?;log::trace!("new host task {task:?}");self.set_thread(task)?;@@ -1675,8 +1801,8 @@ impl StoreOpaque {if !self.concurrency_support() {return Ok(());}- let task = self.concurrent_state_mut().current_host_thread()?;- let caller = self.concurrent_state_mut().get_mut(task)?.caller;+ let task = self.current_host_thread()?;+ let caller = self.concurrent_state_mut()?.get_mut(task)?.caller;self.set_thread(caller)?;Ok(())}@@ -1691,7 +1817,7 @@ impl StoreOpaque {match task {Some(task) => {log::trace!("delete host task {task:?}");- self.concurrent_state_mut().delete(task)?;+ self.concurrent_state_mut()?.delete(task)?;}None => {self.exit_call_not_concurrent();@@ -1714,8 +1840,8 @@ impl StoreOpaque {if !self.concurrency_support() {return Ok(true);}- let state = self.concurrent_state_mut();- let mut cur = Some(state.current_thread);+ let mut cur = Some(self.current_thread()?);+ let state = self.concurrent_state_mut()?;while let Some(t) = cur {if let Some(thread) = t.guest() {let task = state.get_mut(thread.task)?;@@ -1748,8 +1874,8 @@ impl StoreOpaque {/// needed too.fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {let thread = thread.into();- let state = self.concurrent_state_mut();- let old_thread = mem::replace(&mut state.current_thread, thread);+ let state = self.concurrent_state_mut()?;+ let old_thread = mem::replace(&mut state.unforced_current_thread, thread);// First thing to do after swapping threads is updating the context// slots for this thread within the store. This restores the behavior of@@ -1759,21 +1885,22 @@ impl StoreOpaque {// this also leaves behind sentinel values to try to uncover bugs where// this may be forgotten.if let Some(old_thread) = old_thread.guest() {- let old_context = self.vm_store_context().component_context;- self.concurrent_state_mut()+ let old_context = *self.vm_store_context_mut().component_context_mut();+ self.concurrent_state_mut()?.get_mut(old_thread.thread)?.context = old_context;}if cfg!(debug_assertions) {- self.vm_store_context_mut().component_context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];+ *self.vm_store_context_mut().component_context_mut() =+ [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];}if let Some(thread) = thread.guest() {- let thread = self.concurrent_state_mut().get_mut(thread.thread)?;+ let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;let context = thread.context;if cfg!(debug_assertions) {thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];}- self.vm_store_context_mut().component_context = context;+ *self.vm_store_context_mut().component_context_mut() = context;}// Each time we switch threads, we conservatively set `task_may_block`@@ -1783,7 +1910,7 @@ impl StoreOpaque {//// Additionally if we're switching to a new thread, set its component// instance's `task_may_block` according to where it left off.- let state = self.concurrent_state_mut();+ let state = self.concurrent_state_mut()?;if let Some(old_thread) = old_thread.guest() {let instance = state.get_mut(old_thread.task)?.instance.instance;self.component_instance_mut(instance)@@ -1794,16 +1921,23 @@ impl StoreOpaque {self.set_task_may_block()?;}+ // Keep the JIT-visible current-thread pointer in sync.+ *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {+ VMLazyThread::none()+ } else {+ VMLazyThread::forced()+ };+Ok(old_thread)}/// Set the global variable representing whether the current task may block/// prior to entering Wasm code.fn set_task_may_block(&mut self) -> Result<()> {- let state = self.concurrent_state_mut();- let guest_thread = state.current_guest_thread()?;+ let guest_thread = self.current_guest_thread()?;+ let state = self.concurrent_state_mut()?;let instance = state.get_mut(guest_thread.task)?.instance.instance;- let may_block = self.concurrent_state_mut().may_block(guest_thread.task)?;+ let may_block = self.concurrent_state_mut()?.may_block(guest_thread.task)?;self.component_instance_mut(instance).set_task_may_block(may_block);Ok(())@@ -1813,8 +1947,8 @@ impl StoreOpaque {if !self.concurrency_support() {return Ok(());}- let state = self.concurrent_state_mut();- let task = state.current_guest_thread()?.task;+ let task = self.current_guest_thread()?.task;+ let state = self.concurrent_state_mut()?;let instance = state.get_mut(task)?.instance.instance;let task_may_block = self.component_instance(instance).get_task_may_block();@@ -1856,7 +1990,7 @@ impl StoreOpaque {{let call = GuestCall { thread, kind };if call.is_ready(self)? {- self.concurrent_state_mut()+ self.concurrent_state_mut()?.push_high_priority(WorkItem::GuestCall(instance.index, call));} else {self.instance_state(instance)@@ -1892,14 +2026,14 @@ impl StoreOpaque {/// Resume the specified fiber, giving it exclusive access to the specified/// store.async fn resume_fiber(&mut self, fiber: StoreFiber<'static>) -> Result<()> {- let old_thread = self.concurrent_state_mut().current_thread;+ let old_thread = self.current_thread()?;log::trace!("resume_fiber: save current thread {old_thread:?}");let fiber = fiber::resolve_or_release(self, fiber).await?;self.set_thread(old_thread)?;- let state = self.concurrent_state_mut();+ let state = self.concurrent_state_mut()?;if let Some(ot) = old_thread.guest() {state.get_mut(ot.thread)?.state = GuestThreadState::Running;@@ -1968,7 +2102,7 @@ impl StoreOpaque {};let old_guest_thread = if task.is_some() {- self.concurrent_state_mut().current_thread+ self.current_thread()?} else {CurrentThread::None};@@ -1993,12 +2127,12 @@ impl StoreOpaque {}) || old_guest_thread.guest()- .map(|thread| self.concurrent_state_mut().may_block(thread.task))+ .map(|thread| self.concurrent_state_mut()?.may_block(thread.task)).transpose()?.unwrap_or(true));- let suspend_reason = &mut self.concurrent_state_mut().suspend_reason;+ let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;assert!(suspend_reason.is_none());*suspend_reason = Some(reason);@@ -2012,13 +2146,13 @@ impl StoreOpaque {}fn wait_for_event(&mut self, waitable: Waitable) -> Result<()> {- let state = self.concurrent_state_mut();+ let caller = self.current_guest_thread()?;+ let state = self.concurrent_state_mut()?;if waitable.common(state)?.set.is_some() {bail!(Trap::WaitableSyncAndAsync);}- let caller = state.current_guest_thread()?;let set = state.get_mut(caller.thread)?.sync_call_set;waitable.join(state, Some(set))?;self.suspend(SuspendReason::Waiting {@@ -2026,7 +2160,7 @@ impl StoreOpaque {thread: caller,skip_may_block_check: false,})?;- let state = self.concurrent_state_mut();+ let state = self.concurrent_state_mut()?;waitable.join(state, None)}@@ -2057,7 +2191,7 @@ impl StoreOpaque {runtime_instance: RuntimeInstance,cleanup_task: CleanupTask,) -> Result<()> {- let state = self.concurrent_state_mut();+ let state = self.concurrent_state_mut()?;let thread_data = state.get_mut(guest_thread.thread)?;let sync_call_set = thread_data.sync_call_set;if let Some(guest_id) = thread_data.instance_rep {@@ -2065,7 +2199,7 @@ impl StoreOpaque {.thread_handle_table().guest_thread_remove(guest_id)?;}- let state = self.concurrent_state_mut();+ let state = self.concurrent_state_mut()?;// Clean up any pending subtasks in the sync_call_setfor waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {@@ -2128,7 +2262,7 @@ impl StoreOpaque {caller_instance: RuntimeInstance,guest_task: TableId<GuestTask>,) -> Result<()> {- let concurrent_state = self.concurrent_state_mut();+ let concurrent_state = self.concurrent_state_mut()?;let task = concurrent_state.get_mut(guest_task)?;assert!(!task.already_lowered_parameters());// The task is in a `starting` state, meaning it hasn't run at@@ -2179,7 +2313,7 @@ impl Instance {set: Option<TableId<WaitableSet>>,cancellable: bool,) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let event = &mut state.get_mut(guest_task)?.event;if let Some(ev) = event@@ -2235,7 +2369,7 @@ impl Instance {log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {let set = store@@ -2249,7 +2383,7 @@ impl Instance {Ok(match code {callback_code::EXIT => {log::trace!("implicit thread {guest_thread:?} completed");- let task = store.concurrent_state_mut().get_mut(guest_thread.task)?;+ let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;task.exited = true;task.callback = None;store.cleanup_thread(@@ -2295,7 +2429,7 @@ impl Instance {state.check_blocking_for(guest_thread.task)?;let set = get_set(store, set)?;- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;if state.get_mut(guest_thread.task)?.event.is_some()|| !state.get_mut(set)?.ready.is_empty()@@ -2387,10 +2521,10 @@ impl Instance {let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(guest_thread.thread)?.state = GuestThreadState::Running;- let task = store.concurrent_state_mut().get_mut(guest_thread.task)?;+ let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;let lower = match task.lower_params.take() {Some(l) => l,None => bail_bug!("lower_params missing"),@@ -2433,7 +2567,7 @@ impl Instance {let callee_instance = store.0- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(guest_thread.task)?.instance;@@ -2464,7 +2598,7 @@ impl Instance {store.exit_instance(callee_instance)?;store.set_thread(old_thread)?;- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;if let Some(t) = old_thread.guest() {state.get_mut(t.thread)?.state = GuestThreadState::Running;}@@ -2515,7 +2649,7 @@ impl Instance {let lift = {store.exit_instance(callee_instance)?;- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;if !state.get_mut(guest_thread.task)?.result.is_none() {bail_bug!("task has already produced a result");}@@ -2557,7 +2691,7 @@ impl Instance {store.set_thread(old_thread)?;store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(guest_thread.task)?.exited = true;@@ -2569,7 +2703,7 @@ impl Instance {store.0- .concurrent_state_mut()+ .concurrent_state_mut()?.push_high_priority(WorkItem::GuestCall(callee_instance.index,GuestCall {@@ -2653,8 +2787,8 @@ impl Instance {let start = SendSyncPtr::new(start);let return_ = SendSyncPtr::new(return_);let token = StoreToken::new(store.as_context_mut());- let state = store.0.concurrent_state_mut();- let old_thread = state.current_guest_thread()?;+ let old_thread = store.0.current_guest_thread()?;+ let state = store.0.concurrent_state_mut()?;debug_assert_eq!(state.get_mut(old_thread.task)?.instance,@@ -2705,8 +2839,9 @@ impl Instance {)?;}dst.copy_from_slice(&src[..dst.len()]);- let state = store.0.concurrent_state_mut();- Waitable::Guest(state.current_guest_thread()?.task).set_event(+ let task = store.0.current_guest_thread()?.task;+ let state = store.0.concurrent_state_mut()?;+ Waitable::Guest(task).set_event(state,Some(Event::Subtask {status: Status::Started,@@ -2749,8 +2884,8 @@ impl Instance {// lifting/lowering has returned.store.0.set_thread(prev)?;- let state = store.0.concurrent_state_mut();- let thread = state.current_guest_thread()?;+ let thread = store.0.current_guest_thread()?;+ let state = store.0.concurrent_state_mut()?;if sync_caller {state.get_mut(thread.task)?.sync_result = SyncResult::Produced(if let ResultInfo::Stack { result_count } = &result_info {@@ -2840,8 +2975,8 @@ impl Instance {) -> Result<u32> {let token = StoreToken::new(store.as_context_mut());let async_caller = storage.is_none();- let state = store.0.concurrent_state_mut();- let guest_thread = state.current_guest_thread()?;+ let guest_thread = store.0.current_guest_thread()?;+ let state = store.0.concurrent_state_mut()?;let callee_async = state.get_mut(guest_thread.task)?.async_function;let callee = SendSyncPtr::new(callee);let param_count = usize::try_from(param_count)?;@@ -2883,7 +3018,7 @@ impl Instance {)?;}- let state = store.0.concurrent_state_mut();+ let state = store.0.concurrent_state_mut()?;// Use the caller's `GuestThread::sync_call_set` to register interest in// the subtask...@@ -2923,7 +3058,7 @@ impl Instance {skip_may_block_check: async_caller || !callee_async,})?;- let state = store.0.concurrent_state_mut();+ let state = store.0.concurrent_state_mut()?;log::trace!("taking event for {:?}", guest_thread.task);let event = guest_waitable.take_event(state)?;@@ -2947,7 +3082,7 @@ impl Instance {.subtask_insert_guest(guest_thread.task.rep())?;store.0- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(guest_thread.task)?.common.handle = Some(handle);@@ -2959,18 +3094,22 @@ impl Instance {}};- guest_waitable.join(store.0.concurrent_state_mut(), old_set)?;+ guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;// Reset the current thread to point to the caller as it resumes control.store.0.set_thread(caller)?;- store.0.concurrent_state_mut().get_mut(caller.thread)?.state = GuestThreadState::Running;+ store+ .0+ .concurrent_state_mut()?+ .get_mut(caller.thread)?+ .state = GuestThreadState::Running;log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");if let Some(storage) = storage {// The caller used a sync-lowered import to call an async-lifted// export, in which case the result, if any, has been stashed in// `GuestTask::sync_result`.- let state = store.0.concurrent_state_mut();+ let state = store.0.concurrent_state_mut()?;let task = state.get_mut(guest_thread.task)?;if let Some(result) = task.sync_result.take()? {if let Some(result) = result {@@ -3005,8 +3144,8 @@ impl Instance {lower: impl FnOnce(StoreContextMut<T>, Option<R>) -> Result<()> + Send + 'static,) -> Result<Option<u32>> {let token = StoreToken::new(store.as_context_mut());- let state = store.0.concurrent_state_mut();- let task = state.current_host_thread()?;+ let task = store.0.current_host_thread()?;+ let state = store.0.concurrent_state_mut()?;// Create an abortable future which hooks calls to poll and manages call// context state for the future.@@ -3067,7 +3206,7 @@ impl Instance {};lower(store.as_context_mut(), result)?;- let state = store.0.concurrent_state_mut();+ let state = store.0.concurrent_state_mut()?;match &mut state.get_mut(task)?.state {// The task is already flagged as finished because it was// cancelled. No need to transition further.@@ -3088,7 +3227,7 @@ impl Instance {// there are host embedder frames on the stack is unsound.tls::get(move |store| {store- .concurrent_state_mut()+ .concurrent_state_mut()?.push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(on_complete,))));@@ -3098,7 +3237,7 @@ impl Instance {// Make this task visible to the guest and then record what it// was made visible as.- let state = store.0.concurrent_state_mut();+ let state = store.0.concurrent_state_mut()?;state.push_future(future);let caller = state.get_mut(task)?.caller;let instance = state.get_mut(caller.task)?.instance;@@ -3107,7 +3246,7 @@ impl Instance {.instance_state(instance).handle_table().subtask_insert_host(task.rep())?;- store.0.concurrent_state_mut().get_mut(task)?.common.handle = Some(handle);+ store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");// Restore the currently running thread to this host task's@@ -3126,8 +3265,8 @@ impl Instance {options: OptionsIndex,storage: &[ValRaw],) -> Result<()> {- let state = store.concurrent_state_mut();- let guest_thread = state.current_guest_thread()?;+ let guest_thread = store.current_guest_thread()?;+ let state = store.concurrent_state_mut()?;let lift = state.get_mut(guest_thread.task)?.lift_result@@ -3172,8 +3311,8 @@ impl Instance {/// Implements the `task.cancel` intrinsic.pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {- let state = store.concurrent_state_mut();- let guest_thread = state.current_guest_thread()?;+ let guest_thread = store.current_guest_thread()?;+ let state = store.concurrent_state_mut()?;let task = state.get_mut(guest_thread.task)?;if !task.cancel_sent {bail!(Trap::TaskCancelNotCancelled);@@ -3210,10 +3349,10 @@ impl Instance {status: Status,) -> Result<()> {store- .component_resource_tables(Some(self))+ .component_resource_tables(Some(self))?.validate_scope_exit()?;- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let task = state.get_mut(guest_task)?;if let Caller::Host { tx, .. } = &mut task.caller {@@ -3234,7 +3373,7 @@ impl Instance {store: &mut StoreOpaque,caller_instance: RuntimeComponentInstanceIndex,) -> Result<u32> {- let set = store.concurrent_state_mut().push(WaitableSet::default())?;+ let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;let handle = store.instance_state(self.runtime_instance(caller_instance)).handle_table()@@ -3261,7 +3400,7 @@ impl Instance {// set to avoid dropping any waiters in `WaitMode::Fiber(_)`, which// would panic. See `drop-waitable-set-with-waiters.wast` for details.if !store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(TableId::<WaitableSet>::new(rep))?.waiting.is_empty()@@ -3270,7 +3409,7 @@ impl Instance {}store- .concurrent_state_mut()+ .concurrent_state_mut()?.delete(TableId::<WaitableSet>::new(rep))?;Ok(())@@ -3295,7 +3434,7 @@ impl Instance {.handle_table().waitable_set_rep(set_handle)?;- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;if let Some(old) = waitable.common(state)?.set&& state.get_mut(old)?.is_sync_call_set{@@ -3309,7 +3448,7 @@ impl Instance {"waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",);- waitable.join(store.concurrent_state_mut(), set)+ waitable.join(store.concurrent_state_mut()?, set)}/// Implements the `subtask.drop` intrinsic.@@ -3326,7 +3465,7 @@ impl Instance {.handle_table().subtask_remove(task_id)?;- let concurrent_state = store.concurrent_state_mut();+ let concurrent_state = store.concurrent_state_mut()?;let (waitable, delete) = if is_host {let id = TableId::<HostTask>::new(rep);let task = concurrent_state.get_mut(id)?;@@ -3435,9 +3574,9 @@ impl Instance {/// Implements the `thread.index` intrinsic.pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {- let thread_id = store.concurrent_state_mut().current_guest_thread()?.thread;+ let thread_id = store.current_guest_thread()?.thread;match store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(thread_id)?.instance_rep{@@ -3489,7 +3628,7 @@ impl Instance {CleanupTask::Yes,)?;log::trace!("explicit thread {guest_thread:?} completed");- let state = store.0.concurrent_state_mut();+ let state = store.0.concurrent_state_mut()?;if let Some(t) = old_thread.guest() {state.get_mut(t.thread)?.state = GuestThreadState::Running;}@@ -3499,8 +3638,8 @@ impl Instance {},);- let state = store.0.concurrent_state_mut();- let current_thread = state.current_guest_thread()?;+ let current_thread = store.0.current_guest_thread()?;+ let state = store.0.concurrent_state_mut()?;let parent_task = current_thread.task;let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;@@ -3522,7 +3661,7 @@ impl Instance {) -> Result<()> {let thread_id =GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;let thread = state.get_mut(guest_thread.thread)?;@@ -3539,20 +3678,20 @@ impl Instance {},);store- .concurrent_state_mut()+ .concurrent_state_mut()?.push_work_item(guest_call, high_priority);}GuestThreadState::Suspended(fiber) => {log::trace!("resuming thread {thread_id:?} that was suspended");store- .concurrent_state_mut()+ .concurrent_state_mut()?.push_work_item(WorkItem::ResumeFiber(fiber), high_priority);}GuestThreadState::Ready { fiber, cancellable } if allow_ready => {log::trace!("resuming thread {thread_id:?} that was ready");thread.state = GuestThreadState::Ready { fiber, cancellable };store- .concurrent_state_mut()+ .concurrent_state_mut()?.promote_thread_work_item(guest_thread);}other => {@@ -3574,7 +3713,7 @@ impl Instance {.thread_handle_table().guest_thread_insert(thread_id.rep())?;store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(thread_id)?.instance_rep = Some(guest_id);Ok(guest_id)@@ -3590,9 +3729,9 @@ impl Instance {yielding: bool,to_thread: SuspensionTarget,) -> Result<WaitResult> {- let guest_thread = store.concurrent_state_mut().current_guest_thread()?;+ let guest_thread = store.current_guest_thread()?;if to_thread.is_none() {- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;if yielding {// This is a `thread.yield` callif !state.may_block(guest_thread.task)? {@@ -3612,7 +3751,7 @@ impl Instance {}// There could be a pending cancellation from a previous uncancellable wait- if cancellable && store.concurrent_state_mut().take_pending_cancellation()? {+ if cancellable && store.take_pending_cancellation()? {return Ok(WaitResult::Cancelled);}@@ -3647,7 +3786,7 @@ impl Instance {store.suspend(reason)?;- if cancellable && store.concurrent_state_mut().take_pending_cancellation()? {+ if cancellable && store.take_pending_cancellation()? {Ok(WaitResult::Cancelled)} else {Ok(WaitResult::Completed)@@ -3662,11 +3801,11 @@ impl Instance {check: WaitableCheck,params: WaitableCheckParams,) -> Result<u32> {- let guest_thread = store.concurrent_state_mut().current_guest_thread()?;+ let guest_thread = store.current_guest_thread()?;log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let task = state.get_mut(guest_thread.task)?;// If we're waiting, and there are no events immediately available,@@ -3768,7 +3907,7 @@ impl Instance {} else {Waitable::Guest(TableId::<GuestTask>::new(rep))};- let concurrent_state = store.concurrent_state_mut();+ let concurrent_state = store.concurrent_state_mut()?;log::trace!("subtask_cancel {waitable:?} (handle {task_id})");@@ -3848,7 +3987,7 @@ impl Instance {};concurrent_state.push_high_priority(item);- let caller = concurrent_state.current_guest_thread()?;+ let caller = store.current_guest_thread()?;store.suspend(SuspendReason::Yielding {thread: caller,cancellable: false,@@ -3863,8 +4002,8 @@ impl Instance {{// The thread is in a cancellable yield, so yield back// to it.- let caller = concurrent_state.current_guest_thread()?;concurrent_state.promote_thread_work_item(thread);+ let caller = store.current_guest_thread()?;store.suspend(SuspendReason::Yielding {thread: caller,cancellable: false,@@ -3877,7 +4016,7 @@ impl Instance {// Guest tasks need to block if they have not yet returned or// cancelled, even as a result of the event delivery above.needs_block = !store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(guest_task)?.returned_or_cancelled()} else {@@ -3901,7 +4040,7 @@ impl Instance {// .. fall through to determine what event's in store for us.}- let event = waitable.take_event(store.concurrent_state_mut())?;+ let event = waitable.take_event(store.concurrent_state_mut()?)?;if let Some(Event::Subtask {status: status @ (Status::Returned | Status::ReturnCancelled),}) = event@@ -5045,7 +5184,11 @@ impl From<TableId<HostTask>> for CurrentThread {/// Represents the Component Model Async state of a store.pub struct ConcurrentState {/// The currently running thread, if any.- current_thread: CurrentThread,+ ///+ /// Note that we lazily materialize threads on-demand and this field is not+ /// necessarily up-to-date. The `StoreOpaque::current_thread` method should+ /// be preferred over directly accessing this field.+ unforced_current_thread: CurrentThread,/// The set of pending host and background tasks, if any.///@@ -5107,7 +5250,7 @@ pub struct ConcurrentState {impl Default for ConcurrentState {fn default() -> Self {Self {- current_thread: CurrentThread::None,+ unforced_current_thread: CurrentThread::None,table: AlwaysMut::new(ResourceTable::new()),futures: AlwaysMut::new(Some(FuturesUnordered::new())),high_priority: Vec::new(),@@ -5211,7 +5354,7 @@ impl ConcurrentState {// These fields do not contain GC references.worker_item: _,- current_thread: _,+ unforced_current_thread: _,suspend_reason: _,global_error_context_ref_counts: _,interesting_tasks: _,@@ -5363,18 +5506,6 @@ impl ConcurrentState {}}- /// Returns whether there's a pending cancellation on the current guest thread,- /// consuming the event if so.- fn take_pending_cancellation(&mut self) -> Result<bool> {- let thread = self.current_guest_thread()?;- let task = self.get_mut(thread.task)?;- if let Some(Event::Cancelled) = task.event {- task.event.take();- return Ok(true);- }- Ok(false)- }-fn check_blocking_for(&mut self, task: TableId<GuestTask>) -> Result<()> {if self.may_block(task)? {Ok(())@@ -5407,7 +5538,7 @@ impl ConcurrentState {/// Used by `ResourceTables` to record the scope of a borrow to get undone/// in the future.pub fn current_call_context_scope_id(&self) -> Result<u32> {- let (bits, is_host) = match self.current_thread {+ let (bits, is_host) = match self.unforced_current_thread {CurrentThread::Guest(id) => (id.task.rep(), false),CurrentThread::Host(id) => (id.rep(), true),CurrentThread::None => bail_bug!("current thread is not set"),@@ -5416,20 +5547,6 @@ impl ConcurrentState {Ok((bits << 1) | u32::from(is_host))}- fn current_guest_thread(&self) -> Result<QualifiedThreadId> {- match self.current_thread.guest() {- Some(id) => Ok(*id),- None => bail_bug!("current thread is not a guest thread"),- }- }-- fn current_host_thread(&self) -> Result<TableId<HostTask>> {- match self.current_thread.host() {- Some(id) => Ok(id),- None => bail_bug!("current thread is not a host thread"),- }- }-fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {match self.futures.get_mut().as_mut() {Some(f) => Ok(f),@@ -5575,7 +5692,7 @@ impl TaskId {/// and can be resumed by other tasks for this component, so we mark the future as dropped/// and delete the task when all threads are done.pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {- let task = store.concurrent_state_mut().get_mut(self.task)?;+ let task = store.concurrent_state_mut()?.get_mut(self.task)?;let delete = if !task.already_lowered_parameters() {store.cancel_guest_subtask_without_lowered_parameters(self.runtime_instance,@@ -5587,7 +5704,7 @@ impl TaskId {task.ready_to_delete()};if delete {- Waitable::Guest(self.task).delete_from(store.concurrent_state_mut())?+ Waitable::Guest(self.task).delete_from(store.concurrent_state_mut()?)?}Ok(())}@@ -5627,12 +5744,12 @@ pub(crate) fn prepare_call<T, R>(.map(SendSyncPtr::new);let string_encoding = options.string_encoding;let token = StoreToken::new(store.as_context_mut());- let state = store.0.concurrent_state_mut();+ let caller = store.0.current_thread()?;+ let state = store.0.concurrent_state_mut()?;let (tx, rx) = oneshot::channel();let instance = handle.instance().runtime_instance(component_instance);- let caller = state.current_thread;let thread = GuestTask::new(state,Box::new(for_any_lower(move |store, params| {diff --git a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rsindex 7b017f724b..cb7a700edc 100644--- a/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs+++ b/crates/wasmtime/src/runtime/component/concurrent/futures_and_streams.rs@@ -417,7 +417,7 @@ impl<'a, T, B> Destination<'a, T, B> {}fn remaining_(&self, store: &mut StoreOpaque) -> Result<Option<usize>> {- let transmit = store.concurrent_state_mut().get_mut(self.id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(self.id)?;if let &ReadState::GuestReady { count, .. } = &transmit.read {let &WriteState::HostReady { guest_offset, .. } = &transmit.write else {@@ -500,7 +500,7 @@ impl<D: 'static> DirectDestination<'_, D> {.store.as_context_mut().0- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(self.id)?;let &ReadState::GuestReady {@@ -551,7 +551,7 @@ impl<D: 'static> DirectDestination<'_, D> {.store.as_context_mut().0- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(self.id)?;let ReadState::GuestReady {@@ -890,7 +890,7 @@ impl<'a, T> Source<'a, T> {buffer.move_from(*input, count);} else {let store = store.as_context_mut();- let transmit = store.0.concurrent_state_mut().get_mut(self.id)?;+ let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?;let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {bail_bug!("expected ReadState::HostReady");@@ -919,7 +919,7 @@ impl<'a, T> Source<'a, T> {count.as_usize() - guest_offset.as_usize(),)?;- let transmit = store.0.concurrent_state_mut().get_mut(self.id)?;+ let transmit = store.0.concurrent_state_mut()?.get_mut(self.id)?;let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {bail_bug!("expected ReadState::HostReady");@@ -947,7 +947,7 @@ impl<'a, T> Source<'a, T> {whereT: 'static,{- let transmit = store.concurrent_state_mut().get_mut(self.id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(self.id)?;if let &WriteState::GuestReady { count, .. } = &transmit.write {let &ReadState::HostReady { guest_offset, .. } = &transmit.read else {@@ -1010,7 +1010,7 @@ impl<D: 'static> DirectSource<'_, D> {.store.as_context_mut().0- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(self.id)?;let &WriteState::GuestReady {@@ -1061,7 +1061,7 @@ impl<D: 'static> DirectSource<'_, D> {.store.as_context_mut().0- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(self.id)?;let WriteState::GuestReady {@@ -1721,7 +1721,7 @@ impl<T> StreamReader<T> {/// belong to the specified `store`.pub fn try_into<V: 'static>(mut self, mut store: impl AsContextMut) -> Result<V, Self> {let store = store.as_context_mut();- let state = store.0.concurrent_state_mut();+ let state = store.0.concurrent_state_mut_already_forced_current_thread();let id = state.get_mut(self.id).unwrap().state;if let WriteState::HostReady { try_into, .. } = &state.get_mut(id).unwrap().write {match try_into(TypeId::of::<V>()) {@@ -2313,7 +2313,7 @@ impl StoreOpaque {let future = async move {let stream_state = future.await?;tls::get(|store| {- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let transmit = state.get_mut(id)?;let ReadState::HostReady {consume,@@ -2343,7 +2343,8 @@ impl StoreOpaque {})};- self.concurrent_state_mut().push_future(future.boxed());+ self.concurrent_state_mut_already_forced_current_thread()+ .push_future(future.boxed());}fn pipe_to_guest(@@ -2355,7 +2356,7 @@ impl StoreOpaque {let future = async move {let stream_state = future.await?;tls::get(|store| {- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let transmit = state.get_mut(id)?;let WriteState::HostReady {produce,@@ -2387,12 +2388,13 @@ impl StoreOpaque {})};- self.concurrent_state_mut().push_future(future.boxed());+ self.concurrent_state_mut_already_forced_current_thread()+ .push_future(future.boxed());}/// Drop the read end of a stream or future read from the host.fn host_drop_reader(&mut self, id: TableId<TransmitHandle>, kind: TransmitKind) -> Result<()> {- let state = self.concurrent_state_mut();+ let state = self.concurrent_state_mut()?;Waitable::Transmit(id).join(state, None)?;let transmit_id = state.get_mut(id)?.state;let transmit = state@@ -2469,7 +2471,7 @@ impl StoreOpaque {id: TableId<TransmitHandle>,on_drop_open: Option<fn() -> Result<()>>,) -> Result<()> {- let state = self.concurrent_state_mut();+ let state = self.concurrent_state_mut()?;Waitable::Transmit(id).join(state, None)?;let transmit_id = state.get_mut(id)?.state;let transmit = state@@ -2497,7 +2499,7 @@ impl StoreOpaque {WriteState::Dropped => bail_bug!("write state is already dropped"),}- let transmit = self.concurrent_state_mut().get_mut(transmit_id)?;+ let transmit = self.concurrent_state_mut()?.get_mut(transmit_id)?;// If the existing read state is dropped, then there's nothing to read// and we can keep it that way.@@ -2519,7 +2521,7 @@ impl StoreOpaque {// represent that a read must be performedReadState::GuestReady { ty, handle, .. } => {// Ensure the final read of the guest is queued, with appropriate closure indicator- self.concurrent_state_mut().update_event(+ self.concurrent_state_mut()?.update_event(read_handle.rep(),match ty {TransmitIndex::Future(ty) => Event::FutureRead {@@ -2536,7 +2538,7 @@ impl StoreOpaque {// If the read state is open, then there are no registered readers of the stream/futureReadState::Open => {- self.concurrent_state_mut().update_event(+ self.concurrent_state_mut()?.update_event(read_handle.rep(),match on_drop_open {Some(_) => Event::FutureRead {@@ -2559,7 +2561,7 @@ impl StoreOpaque {// this event.ReadState::Dropped | ReadState::HostReady { .. } | ReadState::HostToHost { .. } => {log::trace!("host_drop_writer delete {transmit_id:?}");- self.concurrent_state_mut().delete_transmit(transmit_id)?;+ self.concurrent_state_mut()?.delete_transmit(transmit_id)?;}}Ok(())@@ -2569,7 +2571,7 @@ impl StoreOpaque {&mut self,id: TableId<TransmitHandle>,) -> Result<TransmitOrigin> {- let state = self.concurrent_state_mut();+ let state = self.concurrent_state_mut()?;let state_id = state.get_mut(id)?.state;Ok(state.get_mut(state_id)?.origin)}@@ -2585,7 +2587,7 @@ impl<T> StoreContextMut<'_, T> {P::Item: func::Lower,{let token = StoreToken::new(self.as_context_mut());- let state = self.0.concurrent_state_mut();+ let state = self.0.concurrent_state_mut()?;let (_, read) = state.new_transmit(TransmitOrigin::Host)?;let producer = Arc::new(LockedState::new((Box::pin(producer), P::Buffer::default())));let id = state.get_mut(read)?.state;@@ -2601,7 +2603,7 @@ impl<T> StoreContextMut<'_, T> {let (result, cancelled) = if buffer.remaining().is_empty() {future::poll_fn(|cx| {tls::get(|store| {- let transmit = store.concurrent_state_mut().get_mut(id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;let &WriteState::HostReady { cancel, .. } = &transmit.write else {bail_bug!("expected WriteState::HostReady")@@ -2632,7 +2634,7 @@ impl<T> StoreContextMut<'_, T> {cancel,);- let transmit = store.concurrent_state_mut().get_mut(id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;let host_offset = if let (Some(host_buffer),@@ -2683,7 +2685,7 @@ impl<T> StoreContextMut<'_, T> {};let (guest_offset, host_offset, count) = tls::get(|store| {- let transmit = store.concurrent_state_mut().get_mut(id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;let (count, host_offset) = match &transmit.read {&ReadState::GuestReady { count, .. } => (count.as_u32(), 0),&ReadState::HostToHost { limit, .. } => (1, limit),@@ -2770,7 +2772,7 @@ impl<T> StoreContextMut<'_, T> {consumer: C,) -> Result<()> {let token = StoreToken::new(self.as_context_mut());- let state = self.0.concurrent_state_mut();+ let state = self.0.concurrent_state_mut()?;let id = state.get_mut(id)?.state;let transmit = state.get_mut(id)?;let consumer = Arc::new(LockedState::new(Box::pin(consumer)));@@ -2784,7 +2786,7 @@ impl<T> StoreContextMut<'_, T> {let (result, cancelled) = future::poll_fn(|cx| {tls::get(|store| {- let cancel = match &store.concurrent_state_mut().get_mut(id)?.read {+ let cancel = match &store.concurrent_state_mut()?.get_mut(id)?.read {&ReadState::HostReady { cancel, .. } => cancel,ReadState::Open => false,_ => bail_bug!("unexpected read state"),@@ -2804,7 +2806,7 @@ impl<T> StoreContextMut<'_, T> {cancel_waker,cancel,..- } = &mut store.concurrent_state_mut().get_mut(id)?.read+ } = &mut store.concurrent_state_mut()?.get_mut(id)?.read{if poll.is_pending() {*cancel_waker = Some(cx.waker().clone());@@ -2820,7 +2822,7 @@ impl<T> StoreContextMut<'_, T> {.await?;let (guest_offset, count) = tls::get(|store| {- let transmit = store.concurrent_state_mut().get_mut(id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;Ok((match &transmit.read {&ReadState::HostReady { guest_offset, .. } => guest_offset,@@ -2855,7 +2857,7 @@ impl<T> StoreContextMut<'_, T> {if let TransmitKind::Future = kind {tls::get(|store| {- store.concurrent_state_mut().get_mut(id)?.done = true;+ store.concurrent_state_mut()?.get_mut(id)?.done = true;crate::error::Ok(())})?;}@@ -2930,7 +2932,7 @@ impl<T> StoreContextMut<'_, T> {loop {if tls::get(|store| {crate::error::Ok(matches!(- store.concurrent_state_mut().get_mut(id)?.read,+ store.concurrent_state_mut()?.get_mut(id)?.read,ReadState::Dropped))})? {@@ -2948,7 +2950,7 @@ impl<T> StoreContextMut<'_, T> {}}.map(move |result| {- tls::get(|store| store.concurrent_state_mut().delete_transmit(id))?;+ tls::get(|store| store.concurrent_state_mut()?.delete_transmit(id))?;result});@@ -2970,7 +2972,7 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Writekind: TransmitKind,) -> Result<()> {let (read, guest_offset) = tls::get(|store| {- let transmit = store.concurrent_state_mut().get_mut(id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;let guest_offset = if let &WriteState::HostReady { guest_offset, .. } = &transmit.write {Some(guest_offset)@@ -3003,7 +3005,7 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Writeif let TransmitKind::Future = kind {tls::get(|store| {- store.concurrent_state_mut().get_mut(id)?.done = true;+ store.concurrent_state_mut()?.get_mut(id)?.done = true;crate::error::Ok(())})?;}@@ -3036,14 +3038,15 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Writelet (tx, rx) = oneshot::channel();tls::get(move |store| {store- .concurrent_state_mut()- .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(- move |store| {+ .concurrent_state_mut()?+ .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(+ Box::new(move |store| {_ = tx.send(accept(token.as_context_mut(store))?);Ok(())- },- ))))- });+ }),+ )));+ crate::error::Ok(())+ })?;match rx.await {Ok(r) => r,Err(oneshot::Canceled) => bail_bug!("work cancelled"),@@ -3060,7 +3063,7 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Writetls::get(|store| {let count = old_remaining - pair.with(|p| p.1.remaining().len())?;- let transmit = store.concurrent_state_mut().get_mut(id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(id)?;let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {bail_bug!("expected WriteState::HostReady")@@ -3110,7 +3113,7 @@ async fn write<D: 'static, P: Send + 'static, T: func::Lower + 'static, B: Write}tls::get(|store| {- store.concurrent_state_mut().get_mut(id)?.read = match state {+ store.concurrent_state_mut()?.get_mut(id)?.read = match state {StreamResult::Dropped => ReadState::Dropped,StreamResult::Completed | StreamResult::Cancelled => ReadState::HostToHost {accept,@@ -3141,7 +3144,7 @@ impl Instance {cancel: bool,) -> Result<ReturnCode> {let mut future = consume();- store.concurrent_state_mut().get_mut(transmit_id)?.read = ReadState::HostReady {+ store.concurrent_state_mut()?.get_mut(transmit_id)?.read = ReadState::HostReady {consume,guest_offset,cancel,@@ -3155,7 +3158,7 @@ impl Instance {Ok(match poll {Poll::Ready(state) => {- let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;let ReadState::HostReady { guest_offset, .. } = &mut transmit.read else {bail_bug!("expected ReadState::HostReady")};@@ -3183,7 +3186,7 @@ impl Instance {cancel: bool,) -> Result<ReturnCode> {let mut future = produce();- store.concurrent_state_mut().get_mut(transmit_id)?.write = WriteState::HostReady {+ store.concurrent_state_mut()?.get_mut(transmit_id)?.write = WriteState::HostReady {produce,try_into,guest_offset,@@ -3198,7 +3201,7 @@ impl Instance {Ok(match poll {Poll::Ready(state) => {- let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;let WriteState::HostReady { guest_offset, .. } = &mut transmit.write else {bail_bug!("expected WriteState::HostReady")};@@ -3503,7 +3506,7 @@ impl Instance {*state = TransmitLocalState::Busy;let transmit_handle = TableId::<TransmitHandle>::new(rep);- let concurrent_state = store.0.concurrent_state_mut();+ let concurrent_state = store.0.concurrent_state_mut()?;let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;let transmit = concurrent_state.get_mut(transmit_id)?;log::trace!(@@ -3612,7 +3615,7 @@ impl Instance {Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?,None => 0,};- let concurrent_state = store.0.concurrent_state_mut();+ let concurrent_state = store.0.concurrent_state_mut()?;if read_complete {let total = if let Some(Event::StreamRead {code: ReturnCode::Completed(old_total),@@ -3757,8 +3760,8 @@ impl Instance {*state = TransmitLocalState::Busy;let transmit_handle = TableId::<TransmitHandle>::new(rep);- let concurrent_state = store.0.concurrent_state_mut();- let caller_thread = concurrent_state.current_guest_thread()?;+ let caller_thread = store.0.current_guest_thread()?;+ let concurrent_state = store.0.concurrent_state_mut()?;let transmit_id = concurrent_state.get_mut(transmit_handle)?.state;let transmit = concurrent_state.get_mut(transmit_id)?;log::trace!(@@ -3850,7 +3853,7 @@ impl Instance {Some(ty) => usize::try_from(types.canonical_abi(ty).size32)?,None => 0,};- let concurrent_state = store.0.concurrent_state_mut();+ let concurrent_state = store.0.concurrent_state_mut()?;if write_complete {let total = if let Some(Event::StreamWrite {@@ -3925,7 +3928,7 @@ impl Instance {)?;if let (TransmitIndex::Future(_), ReturnCode::Completed(_)) = (ty, code) {- store.0.concurrent_state_mut().get_mut(transmit_id)?.done = true;+ store.0.concurrent_state_mut()?.get_mut(transmit_id)?.done = true;}code@@ -3967,7 +3970,7 @@ impl Instance {) -> Result<ReturnCode> {let waitable = Waitable::Transmit(handle);store.wait_for_event(waitable)?;- let event = waitable.take_event(store.concurrent_state_mut())?;+ let event = waitable.take_event(store.concurrent_state_mut()?)?;if let Some(event @ (Event::StreamWrite { code, .. } | Event::FutureWrite { code, .. })) =event{@@ -3985,7 +3988,7 @@ impl Instance {transmit_id: TableId<TransmitState>,async_: bool,) -> Result<ReturnCode> {- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let transmit = state.get_mut(transmit_id)?;log::trace!("host_cancel_write state {transmit_id:?}; write state {:?} read state {:?}",@@ -4021,7 +4024,7 @@ impl Instance {ReturnCode::Blocked} else {let handle = store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(transmit_id)?.write_handle;self.wait_for_write(store, handle)?@@ -4031,7 +4034,7 @@ impl Instance {};if !matches!(code, ReturnCode::Blocked) {- let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;match &transmit.write {WriteState::GuestReady { .. } => {@@ -4054,7 +4057,7 @@ impl Instance {) -> Result<ReturnCode> {let waitable = Waitable::Transmit(handle);store.wait_for_event(waitable)?;- let event = waitable.take_event(store.concurrent_state_mut())?;+ let event = waitable.take_event(store.concurrent_state_mut()?)?;if let Some(event @ (Event::StreamRead { code, .. } | Event::FutureRead { code, .. })) =event{@@ -4072,7 +4075,7 @@ impl Instance {transmit_id: TableId<TransmitState>,async_: bool,) -> Result<ReturnCode> {- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let transmit = state.get_mut(transmit_id)?;log::trace!("host_cancel_read state {transmit_id:?}; read state {:?} write state {:?}",@@ -4108,7 +4111,7 @@ impl Instance {ReturnCode::Blocked} else {let handle = store- .concurrent_state_mut()+ .concurrent_state_mut()?.get_mut(transmit_id)?.read_handle;self.wait_for_read(store, handle)?@@ -4118,7 +4121,7 @@ impl Instance {};if !matches!(code, ReturnCode::Blocked) {- let transmit = store.concurrent_state_mut().get_mut(transmit_id)?;+ let transmit = store.concurrent_state_mut()?.get_mut(transmit_id)?;match &transmit.read {ReadState::GuestReady { .. } => {@@ -4164,7 +4167,7 @@ impl Instance {}TransmitLocalState::Busy => {}}- let transmit_id = store.concurrent_state_mut().get_mut(id)?.state;+ let transmit_id = store.concurrent_state_mut()?.get_mut(id)?.state;let code = self.cancel_write(store, transmit_id, async_)?;if !matches!(code, ReturnCode::Blocked) {let state =@@ -4205,7 +4208,7 @@ impl Instance {}TransmitLocalState::Busy => {}}- let transmit_id = store.concurrent_state_mut().get_mut(id)?.state;+ let transmit_id = store.concurrent_state_mut()?.get_mut(id)?.state;let code = self.cancel_read(store, transmit_id, async_)?;if !matches!(code, ReturnCode::Blocked) {let state =@@ -4257,7 +4260,7 @@ impl Instance {// Create a new ErrorContext that is tracked along with other concurrent statelet err_ctx = ErrorContextState { debug_msg };- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let table_id = state.push(err_ctx)?;let global_ref_count_idx =TypeComponentGlobalErrorContextTableIndex::from_u32(table_id.rep());@@ -4298,7 +4301,7 @@ impl Instance {.table_for_error_context(ty).error_context_rep(err_ctx_handle)?;- let state = store.0.concurrent_state_mut();+ let state = store.0.concurrent_state_mut()?;// Get the state associated with the error contextlet ErrorContextState { debug_msg } =state.get_mut(TableId::<ErrorContextState>::new(handle_table_id_rep))?;@@ -4397,7 +4400,7 @@ impl Instance {/// `TransmitIndex` belongs.fn guest_new(self, store: &mut StoreOpaque, ty: TransmitIndex) -> Result<ResourcePair> {let (write, read) = store- .concurrent_state_mut()+ .concurrent_state_mut()?.new_transmit(TransmitOrigin::guest(self.id().instance(), ty))?;let table = self.id().get_mut(store).table_for_transmit(ty);@@ -4412,7 +4415,7 @@ impl Instance {),};- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;state.get_mut(read)?.common.handle = Some(read_handle);state.get_mut(write)?.common.handle = Some(write_handle);@@ -4437,7 +4440,7 @@ impl Instance {let global_ref_count_idx = TypeComponentGlobalErrorContextTableIndex::from_u32(rep);- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let Some(GlobalErrorContextRefCount(global_ref_count)) = state.global_error_context_ref_counts.get_mut(&global_ref_count_idx)@@ -4569,7 +4572,7 @@ impl Instance {// as the new component has essentially created a new reference that will// be dropped/handled independentlylet global_ref_count = store- .concurrent_state_mut()+ .concurrent_state_mut()?.global_error_context_ref_counts.get_mut(&TypeComponentGlobalErrorContextTableIndex::from_u32(rep)).context("global ref count present for existing (sub)component error context")?;@@ -4881,7 +4884,7 @@ impl Waitable {};let transmit_handle = TableId::<TransmitHandle>::new(rep);- let state = store.concurrent_state_mut();+ let state = store.concurrent_state_mut()?;let transmit_id = state.get_mut(transmit_handle)?.state;let transmit = state.get_mut(transmit_id)?;diff --git a/crates/wasmtime/src/runtime/component/func.rs b/crates/wasmtime/src/runtime/component/func.rsindex 423bb60ebb..a16fc33b6b 100644--- a/crates/wasmtime/src/runtime/component/func.rs+++ b/crates/wasmtime/src/runtime/component/func.rs@@ -499,7 +499,7 @@ impl Func {// as they're required to have been dropped by this point.store.0- .component_resource_tables(Some(self.instance))+ .component_resource_tables(Some(self.instance))?.validate_scope_exit()?;// SAFETY: We're relying on the correctness of the structure ofdiff --git a/crates/wasmtime/src/runtime/component/func/options.rs b/crates/wasmtime/src/runtime/component/func/options.rsindex d02fc7e4db..51aab7e0d6 100644--- a/crates/wasmtime/src/runtime/component/func/options.rs+++ b/crates/wasmtime/src/runtime/component/func/options.rs@@ -221,7 +221,7 @@ impl<'a, T: 'static> LowerContext<'a, T> {ty: TypeResourceTableIndex,rep: u32,) -> Result<u32> {- self.resource_tables().guest_resource_lower_own(rep, ty)+ self.resource_tables()?.guest_resource_lower_own(rep, ty)}/// Lowers a `borrow` resource into the guest, converting the `rep` to a@@ -242,19 +242,19 @@ impl<'a, T: 'static> LowerContext<'a, T> {if self.instance().resource_owned_by_own_instance(ty) {return Ok(rep);}- self.resource_tables().guest_resource_lower_borrow(rep, ty)+ self.resource_tables()?.guest_resource_lower_borrow(rep, ty)}/// Lifts a host-owned `own` resource at the `idx` specified into the/// representation of that resource.pub fn host_resource_lift_own(&mut self, idx: HostResourceIndex) -> Result<u32> {- self.resource_tables().host_resource_lift_own(idx)+ self.resource_tables()?.host_resource_lift_own(idx)}/// Lifts a host-owned `borrow` resource at the `idx` specified into the/// representation of that resource.pub fn host_resource_lift_borrow(&mut self, idx: HostResourceIndex) -> Result<u32> {- self.resource_tables().host_resource_lift_borrow(idx)+ self.resource_tables()?.host_resource_lift_borrow(idx)}/// Lowers a resource into the host-owned table, returning the index it was@@ -268,7 +268,7 @@ impl<'a, T: 'static> LowerContext<'a, T> {dtor: Option<NonNull<VMFuncRef>>,instance: Option<RuntimeInstance>,) -> Result<HostResourceIndex> {- self.resource_tables()+ self.resource_tables()?.host_resource_lower_own(rep, dtor, instance)}@@ -283,18 +283,18 @@ impl<'a, T: 'static> LowerContext<'a, T> {InstanceType::new(self.instance())}- fn resource_tables(&mut self) -> HostResourceTables<'_> {+ fn resource_tables(&mut self) -> Result<HostResourceTables<'_>> {let (tables, data) = self.store.0- .component_resource_tables_and_host_resource_data(Some(self.instance));- HostResourceTables::from_parts(tables, data)+ .component_resource_tables_and_host_resource_data(Some(self.instance))?;+ Ok(HostResourceTables::from_parts(tables, data))}/// See [`HostResourceTables::validate_scope_exit`].#[inline]pub fn validate_scope_exit(&mut self) -> Result<()> {- self.resource_tables().validate_scope_exit()+ self.resource_tables()?.validate_scope_exit()}}diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rsindex 262d181f98..13afe2c770 100644--- a/crates/wasmtime/src/runtime/component/instance.rs+++ b/crates/wasmtime/src/runtime/component/instance.rs@@ -381,7 +381,7 @@ impl Instance {rep: u32,) -> Result<u32> {store- .component_resource_tables(Some(self))+ .component_resource_tables(Some(self))?.resource_new(TypedResource::Component { ty, rep })}@@ -394,7 +394,7 @@ impl Instance {index: u32,) -> Result<u32> {store- .component_resource_tables(Some(self))+ .component_resource_tables(Some(self))?.resource_rep(TypedResourceIndex::Component { ty, index })}@@ -406,7 +406,7 @@ impl Instance {index: u32,) -> Result<Option<u32>> {store- .component_resource_tables(Some(self))+ .component_resource_tables(Some(self))?.resource_drop(TypedResourceIndex::Component { ty, index })}@@ -417,7 +417,7 @@ impl Instance {src: TypeResourceTableIndex,dst: TypeResourceTableIndex,) -> Result<u32> {- let mut tables = store.component_resource_tables(Some(self));+ let mut tables = store.component_resource_tables(Some(self))?;let rep = tables.resource_lift_own(TypedResourceIndex::Component { ty: src, index })?;tables.resource_lower_own(TypedResource::Component { ty: dst, rep })}@@ -430,7 +430,7 @@ impl Instance {dst: TypeResourceTableIndex,) -> Result<u32> {let dst_owns_resource = self.id().get(store).resource_owned_by_own_instance(dst);- let mut tables = store.component_resource_tables(Some(self));+ let mut tables = store.component_resource_tables(Some(self))?;let rep = tables.resource_lift_borrow(TypedResourceIndex::Component { ty: src, index })?;// Implement `lower_borrow`'s special case here where if a borrow's// resource type is owned by `dst` then the destination receives thediff --git a/crates/wasmtime/src/runtime/component/resources/any.rs b/crates/wasmtime/src/runtime/component/resources/any.rsindex cc84cecb89..ecb69e4822 100644--- a/crates/wasmtime/src/runtime/component/resources/any.rs+++ b/crates/wasmtime/src/runtime/component/resources/any.rs@@ -109,7 +109,7 @@ impl ResourceAny {D: PartialEq + Send + Sync + Copy + 'static,{let store = store.as_context_mut();- let mut tables = HostResourceTables::new_host(store.0);+ let mut tables = HostResourceTables::new_host(store.0)?;let ResourceAny { idx, ty, owned } = self;let ty = T::typecheck(ty).ok_or_else(|| crate::format_err!("resource type mismatch"))?;if owned {@@ -189,7 +189,7 @@ impl ResourceAny {//// This could fail if the index is invalid or if this is removing an// `Own` entry which is currently being borrowed.- let pair = HostResourceTables::new_host(store.0).host_resource_drop(self.idx)?;+ let pair = HostResourceTables::new_host(store.0)?.host_resource_drop(self.idx)?;let (rep, slot) = match (pair, self.owned) {(Some(pair), true) => pair,diff --git a/crates/wasmtime/src/runtime/component/resources/host.rs b/crates/wasmtime/src/runtime/component/resources/host.rsindex e1d363c696..70f43f06f7 100644--- a/crates/wasmtime/src/runtime/component/resources/host.rs+++ b/crates/wasmtime/src/runtime/component/resources/host.rs@@ -292,7 +292,7 @@ where} = self;let store = store.as_context_mut();- let mut tables = HostResourceTables::new_host(store.0);+ let mut tables = HostResourceTables::new_host(store.0)?;let (idx, owned) = match state.get() {ResourceState::Borrow => (tables.host_resource_lower_borrow(rep)?, false),ResourceState::NotInTable => {diff --git a/crates/wasmtime/src/runtime/component/resources/host_tables.rs b/crates/wasmtime/src/runtime/component/resources/host_tables.rsindex a8ba4809cb..4835e538e0 100644--- a/crates/wasmtime/src/runtime/component/resources/host_tables.rs+++ b/crates/wasmtime/src/runtime/component/resources/host_tables.rs@@ -96,9 +96,9 @@ impl HostResourceIndex {}impl<'a> HostResourceTables<'a> {- pub fn new_host(store: &'a mut StoreOpaque) -> HostResourceTables<'a> {- let (tables, data) = store.component_resource_tables_and_host_resource_data(None);- HostResourceTables::from_parts(tables, data)+ pub fn new_host(store: &'a mut StoreOpaque) -> Result<HostResourceTables<'a>> {+ let (tables, data) = store.component_resource_tables_and_host_resource_data(None)?;+ Ok(HostResourceTables::from_parts(tables, data))}pub fn from_parts(diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rsindex 7c1ace64c6..52e78130a4 100644--- a/crates/wasmtime/src/runtime/component/store.rs+++ b/crates/wasmtime/src/runtime/component/store.rs@@ -136,7 +136,7 @@ impl ComponentStoreData {let mut fibers = Vec::new();let mut futures = Vec::new();store- .concurrent_state_mut()+ .concurrent_state_mut_without_forcing_current_thread().take_fibers_and_futures(&mut fibers, &mut futures);for mut fiber in fibers {@@ -297,8 +297,13 @@ impl StoreOpaque {&mut self.store_data_mut().components}- pub(crate) fn component_task_state_mut(&mut self) -> &mut ComponentTaskState {- &mut self.component_data_mut().task_state+ pub(crate) fn component_task_state_mut(&mut self) -> Result<&mut ComponentTaskState> {+ // NB: Force the deferred lazy thread, if any, before handing out task+ // state.+ #[cfg(feature = "component-model-async")]+ let _ = self.current_thread()?;++ Ok(&mut self.component_data_mut().task_state)}pub(crate) fn push_component_instance(&mut self, instance: Instance) {@@ -323,11 +328,34 @@ impl StoreOpaque {}#[cfg(feature = "component-model-async")]- pub(crate) fn concurrent_state_mut(&mut self) -> &mut ConcurrentState {+ pub(crate) fn concurrent_state_mut_without_forcing_current_thread(+ &mut self,+ ) -> &mut ConcurrentState {debug_assert!(self.concurrency_support());self.component_data_mut().task_state.concurrent_state_mut()}+ #[cfg(feature = "component-model-async")]+ pub(crate) fn concurrent_state_mut_already_forced_current_thread(+ &mut self,+ ) -> &mut ConcurrentState {+ debug_assert!(self.concurrency_support());+ debug_assert!(+ !self+ .vm_store_context_mut()+ .current_thread_mut()+ .is_deferred()+ );+ self.concurrent_state_mut_without_forcing_current_thread()+ }++ #[cfg(feature = "component-model-async")]+ pub(crate) fn concurrent_state_mut(&mut self) -> Result<&mut ConcurrentState> {+ debug_assert!(self.concurrency_support());+ self.current_thread()?;+ Ok(self.component_data_mut().task_state.concurrent_state_mut())+ }+#[inline]#[cfg(feature = "component-model-async")]pub(crate) fn concurrency_support(&self) -> bool {@@ -362,18 +390,24 @@ impl StoreOpaque {pub(crate) fn component_resource_tables(&mut self,instance: Option<Instance>,- ) -> vm::component::ResourceTables<'_> {- self.component_resource_tables_and_host_resource_data(instance)- .0+ ) -> Result<vm::component::ResourceTables<'_>> {+ Ok(self+ .component_resource_tables_and_host_resource_data(instance)?+ .0)}pub(crate) fn component_resource_tables_and_host_resource_data(&mut self,instance: Option<Instance>,- ) -> (+ ) -> Result<(vm::component::ResourceTables<'_>,&mut crate::component::HostResourceData,- ) {+ )> {+ // NB: Force the current lazy deferred thread, if any, before handing+ // out resource tables.+ #[cfg(feature = "component-model-async")]+ let _ = self.current_thread()?;+let store_id = self.id();let data = self.component_data_mut();let guest = instance.map(|i| {@@ -386,14 +420,14 @@ impl StoreOpaque {.instance_states()});- (+ Ok((vm::component::ResourceTables {host_table: &mut data.component_host_table,task_state: &mut data.task_state,guest,},&mut data.host_resource_data,- )+ ))}pub(crate) fn enter_call_not_concurrent(&mut self) -> Result<()> {@@ -424,7 +458,10 @@ impl StoreOpaque {#[cfg(feature = "component-model-async")]fn concurrent_resource_table(&mut self) -> Option<&mut ResourceTable> {if self.concurrency_support() {- Some(self.concurrent_state_mut().table())+ Some(+ self.concurrent_state_mut_without_forcing_current_thread()+ .table(),+ )} else {None}diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex bda01cd5c2..ac690ffd8e 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -2472,7 +2472,9 @@ unsafe impl<T> VMStore for StoreInner<T> {}#[cfg(feature = "component-model")]- fn component_task_state_mut(&mut self) -> &mut crate::component::store::ComponentTaskState {+ fn component_task_state_mut(+ &mut self,+ ) -> Result<&mut crate::component::store::ComponentTaskState> {StoreOpaque::component_task_state_mut(self)}diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rsindex e97b4259ad..7bea0110f7 100644--- a/crates/wasmtime/src/runtime/vm.rs+++ b/crates/wasmtime/src/runtime/vm.rs@@ -126,6 +126,8 @@ pub use crate::runtime::vm::throw::*;pub use crate::runtime::vm::traphandlers::*;#[cfg(feature = "component-model")]pub use crate::runtime::vm::vmcontext::VMArrayCallFunction;+#[cfg(feature = "component-model-async")]+pub use crate::runtime::vm::vmcontext::VMLazyThread;pub use crate::runtime::vm::vmcontext::{VMArrayCallHostFuncContext, VMContext, VMFuncRef, VMFunctionImport, VMGlobalDefinition,VMGlobalImport, VMGlobalKind, VMMemoryDefinition, VMMemoryImport, VMOpaqueContext,@@ -219,7 +221,9 @@ pub unsafe trait VMStore: 'static {/// Metadata required for resources for the component model.#[cfg(feature = "component-model")]- fn component_task_state_mut(&mut self) -> &mut crate::component::store::ComponentTaskState;+ fn component_task_state_mut(+ &mut self,+ ) -> Result<&mut crate::component::store::ComponentTaskState>;#[cfg(feature = "component-model-async")]fn component_async_store(diff --git a/crates/wasmtime/src/runtime/vm/component/libcalls.rs b/crates/wasmtime/src/runtime/vm/component/libcalls.rsindex 094b5562c2..2484a09db1 100644--- a/crates/wasmtime/src/runtime/vm/component/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/component/libcalls.rs@@ -688,7 +688,7 @@ fn enter_sync_call(fn exit_sync_call(store: &mut dyn VMStore, instance: Instance) -> Result<()> {store- .component_resource_tables(Some(instance))+ .component_resource_tables(Some(instance))?.validate_scope_exit()?;store.exit_guest_sync_call()}diff --git a/crates/wasmtime/src/runtime/vm/provenance.rs b/crates/wasmtime/src/runtime/vm/provenance.rsindex 7453f17b23..7ea4787c8c 100644--- a/crates/wasmtime/src/runtime/vm/provenance.rs+++ b/crates/wasmtime/src/runtime/vm/provenance.rs@@ -44,6 +44,8 @@use crate::vm::SendSyncPtr;use core::fmt;use core::marker;+use core::marker::PhantomData;+use core::num::NonZero;use core::num::NonZeroUsize;use core::ptr::NonNull;use core::sync::atomic::AtomicUsize;@@ -117,8 +119,25 @@ impl<T> VmPtr<T> {}/// Similar to `{NonNull,core::ptr}::dangling()` but for `VmPtr`.- pub fn dangling() -> Self {- NonNull::dangling().into()+ pub const fn dangling() -> Self {+ let ptr = core::mem::align_of::<T>();+ Self {+ ptr: NonZeroUsize::new(ptr).unwrap(),+ _marker: PhantomData,+ }+ }++ /// Similar to `NonNull::cast` but for `VmPtr`.+ pub const fn cast<U>(self) -> VmPtr<U> {+ VmPtr {+ ptr: self.ptr,+ _marker: PhantomData,+ }+ }++ /// Similar to `NonNull::addr` but for `VmPtr`.+ pub fn addr(&self) -> NonZero<usize> {+ self.ptr}}@@ -131,6 +150,32 @@ impl<T> Clone for VmPtr<T> {impl<T> Copy for VmPtr<T> {}+impl<T> PartialEq for VmPtr<T> {+ fn eq(&self, other: &Self) -> bool {+ self.ptr == other.ptr+ }+}++impl<T> Eq for VmPtr<T> {}++impl<T> PartialOrd for VmPtr<T> {+ fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {+ self.ptr.partial_cmp(&other.ptr)+ }+}++impl<T> Ord for VmPtr<T> {+ fn cmp(&self, other: &Self) -> core::cmp::Ordering {+ self.ptr.cmp(&other.ptr)+ }+}++impl<T> core::hash::Hash for VmPtr<T> {+ fn hash<H: core::hash::Hasher>(&self, state: &mut H) {+ self.ptr.hash(state);+ }+}+// Forward debugging to `SendSyncPtr<T>` which renders the address.impl<T> fmt::Debug for VmPtr<T> {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex 83844fb4ee..9c665e084b 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -1226,7 +1226,19 @@ pub struct VMStoreContext {/// `VMOffsets` conditional.////// This is saved/restored when threads are swapped in the component model.- pub component_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],+ ///+ /// NB: `UnsafeCell` because JIT code writes to the slots.+ pub component_context: UnsafeCell<[u32; NUM_COMPONENT_CONTEXT_SLOTS]>,++ /// JIT-visible current thread for the component model's sync-to-sync+ /// adapter fast path.+ ///+ /// Like `component_context`, this is unconditionally present to keep+ /// `VMOffsets` logic unconditional even though it is only used when+ /// `component-model-async` is enabled.+ ///+ /// NB: `UnsafeCell` because JIT code writes to this field.+ pub current_thread: UnsafeCell<VMLazyThread>,}impl VMStoreContext {@@ -1280,6 +1292,16 @@ impl VMStoreContext {0}}++ #[cfg(feature = "component-model-async")]+ pub(crate) fn component_context_mut(&mut self) -> &mut [u32; NUM_COMPONENT_CONTEXT_SLOTS] {+ self.component_context.get_mut()+ }++ #[cfg(feature = "component-model-async")]+ pub(crate) fn current_thread_mut(&mut self) -> &mut VMLazyThread {+ self.current_thread.get_mut()+ }}// The `VMStoreContext` type is a pod-type with no destructor, and we don't@@ -1311,7 +1333,8 @@ impl Default for VMStoreContext {stack_chain: UnsafeCell::new(VMStackChain::Absent),async_guard_range: ptr::null_mut()..ptr::null_mut(),store_data: VmPtr::dangling(),- component_context: [0; NUM_COMPONENT_CONTEXT_SLOTS],+ component_context: UnsafeCell::new([0; NUM_COMPONENT_CONTEXT_SLOTS]),+ current_thread: UnsafeCell::new(VMLazyThread::none()),}}}@@ -1386,19 +1409,177 @@ mod test_vmstore_context {offset_of!(VMStoreContext, component_context),usize::from(offsets.ptr.vmstore_context_component_context_slot(0)));+ assert_eq!(+ offset_of!(VMStoreContext, current_thread),+ usize::from(offsets.ptr.vmstore_context_current_thread())+ );// Make sure that the calculation for the size of a slot is also// accurate.let slot_width = offsets.ptr.vmstore_context_component_context_slot(1)- offsets.ptr.vmstore_context_component_context_slot(0);- let default = VMStoreContext::default();+ let mut default = VMStoreContext::default();assert_eq!(- size_of_val(&default.component_context[0]),+ size_of_val(&default.component_context.get_mut()[0]),usize::from(slot_width));}}+/// JIT-visible representation of the store's current thread for the component+/// model, encoded as a single pointer-sized integer so that generated JIT code+/// can load, store, and compare it with a handful of instructions.+///+/// This is the inline fast-path counterpart to the host-side `CurrentThread`: a+/// fused sync-to-sync adapter records a lazy deferred thread here (a pointer to+/// a `VMDeferredThread` on its own stack frame) instead of eagerly allocating a+/// `GuestTask`/`GuestThread` in the host. Host code promotes the deferred+/// thread into a real one only when it actually needs it; see+/// `StoreOpaque::force_current_thread`.+///+/// This type is a bitpacked equivalent of the following logical `enum`:+///+/// ```ignore+/// enum VMLazyThread {+/// /// No thread.+/// None,+///+/// /// The lazy thread was promoted and materialized; get it from+/// /// `ConcurrentState::current_thread`.+/// Forced,+///+/// /// The lazy thread has not been materialized, here is a pointer to the+/// /// stack-allocated data needed to do force that promotion.+/// Deferred(*mut VMDeferredThread),+/// }+/// ```+//+// Bitpacking details:+//+// * `None`: `0`+//+// * `Forced`: A non-zero value with its low-bit set.+//+// * `Deferred`: A non-zero value with its low-bit clear.+#[derive(Debug, Copy, Clone, PartialEq, Eq)]+#[repr(transparent)]+pub struct VMLazyThread(Option<VmPtr<VMDeferredThread>>);++impl VMLazyThread {+ const _ASSERT_SIZE: () = assert!(+ core::mem::size_of::<VMLazyThread>() == core::mem::size_of::<*mut VMDeferredThread>()+ );+ const _ASSERT_ALIGN: () = assert!(+ core::mem::align_of::<VMLazyThread>() == core::mem::align_of::<*mut VMDeferredThread>()+ );++ const FORCED: VmPtr<VMDeferredThread> = VmPtr::<u8>::dangling().cast();++ /// There is no current thread.+ pub const fn none() -> Self {+ Self(None)+ }++ /// A lazy thread that has already been promoted.+ pub const fn forced() -> Self {+ Self(Some(Self::FORCED))+ }++ /// A deferred thread referencing the given on-stack [`VMDeferredThread`].+ pub fn deferred(ptr: NonNull<VMDeferredThread>) -> Self {+ debug_assert_eq!(ptr.addr().get() & Self::FORCED.addr().get(), 0);+ Self(Some(ptr.into()))+ }++ /// Returns `true` if there is no current thread.+ pub fn is_none(self) -> bool {+ self.0.is_none()+ }++ /// Returns `true` if a deferred thread has been forced/promoted.+ pub fn is_forced(self) -> bool {+ self.0.is_some_and(|p| p == Self::FORCED)+ }++ /// Returns `true` if this is a deferred thread (i.e. neither `None` nor+ /// forced).+ pub fn is_deferred(self) -> bool {+ self.0.is_some_and(|p| p != Self::FORCED)+ }++ /// Returns the deferred [`VMDeferredThread`] pointer if this is a deferred+ /// thread.+ pub fn as_deferred(self) -> Option<VmPtr<VMDeferredThread>> {+ self.0+ .and_then(|p| if p == Self::FORCED { None } else { Some(p) })+ }+}++/// A deferred component-model thread.+///+/// This is an on-stack record pushed by a fused sync-to-sync adapter's fast+/// path to defer the work that the `enter_sync_call` libcall would otherwise do+/// eagerly.+///+/// The adapter allocates one of these in its own stack frame, links the+/// previous current-thread value to it via `parent`, and finally points+/// `VMStoreContext::current_thread` at it. When host code actually needs the+/// real thread, it walks the `parent` chain to materialize thread state (see+/// `StoreOpaque::force_current_thread`).+#[derive(Debug)]+#[repr(C)]+pub struct VMDeferredThread {+ /// The previous value of `VMStoreContext::current_thread`.+ pub parent: VMLazyThread,+ /// The caller component instance (a deferred `enter_sync_call` argument).+ pub caller_instance: u32,+ /// Whether the callee is async-lifted (a deferred `enter_sync_call` arg).+ pub callee_async: u32,+ /// The callee component instance (a deferred `enter_sync_call` argument).+ pub callee_instance: u32,+ /// The caller thread's `context.{get,set}` slots, saved on entry and+ /// restored on the fast-path exit (or recovered while forcing).+ pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],+}++#[cfg(test)]+mod test_vmdeferred_thread {+ use super::*;+ use core::mem::offset_of;+ use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};++ #[test]+ fn deferred_thread_field_offsets() {+ let module = Module::new(StaticModuleIndex::from_u32(0));+ let offsets = VMOffsets::new(HostPtr, &module);+ let ptr = offsets.ptr;+ assert_eq!(+ offset_of!(VMDeferredThread, parent),+ usize::from(ptr.vmdeferred_thread_parent())+ );+ assert_eq!(+ offset_of!(VMDeferredThread, caller_instance),+ usize::from(ptr.vmdeferred_thread_caller_instance())+ );+ assert_eq!(+ offset_of!(VMDeferredThread, callee_async),+ usize::from(ptr.vmdeferred_thread_callee_async())+ );+ assert_eq!(+ offset_of!(VMDeferredThread, callee_instance),+ usize::from(ptr.vmdeferred_thread_callee_instance())+ );+ assert_eq!(+ offset_of!(VMDeferredThread, saved_context),+ usize::from(ptr.vmdeferred_thread_saved_context(0))+ );+ assert_eq!(+ size_of::<VMDeferredThread>(),+ usize::from(ptr.size_of_vmdeferred_thread())+ );+ }+}+/// The VM "context", which is pointed to by the `vmctx` arg in Cranelift./// This has information about globals, memories, tables, and other runtime/// state associated with the current instance.diff --git a/tests/all/component_model.rs b/tests/all/component_model.rsindex b7e46fcc13..e407806805 100644--- a/tests/all/component_model.rs+++ b/tests/all/component_model.rs@@ -23,6 +23,7 @@ mod nested;mod post_return;mod resources;mod strings;+mod sync_call_inline;#[derive(Copy, Clone)]enum ApiStyle {diff --git a/tests/all/component_model/async.rs b/tests/all/component_model/async.rsindex 7df03ca8df..6b93b7b2e1 100644--- a/tests/all/component_model/async.rs+++ b/tests/all/component_model/async.rs@@ -1001,7 +1001,7 @@ async fn async_call_stack() -> Result<()> {linker.root().func_wrap("a",|mut store: StoreContextMut<Option<GuestTaskId>>, (): ()| {- let stack = store.async_call_stack().collect::<Vec<_>>();+ let stack = store.async_call_stack()?.collect::<Vec<_>>();assert_eq!(stack, [store.data().unwrap()]);Ok(())},@@ -1046,7 +1046,7 @@ async fn async_call_stack() -> Result<()> {linker.root().func_wrap("a",|mut store: StoreContextMut<Option<GuestTaskId>>, (): ()| {- let stack = store.async_call_stack().collect::<Vec<_>>();+ let stack = store.async_call_stack()?.collect::<Vec<_>>();assert_eq!(stack.len(), 2);assert_eq!(stack.last(), store.data().as_ref());Ok(())diff --git a/tests/all/component_model/sync_call_inline.rs b/tests/all/component_model/sync_call_inline.rsnew file mode 100644index 0000000000..b93a5b6cbf--- /dev/null+++ b/tests/all/component_model/sync_call_inline.rs@@ -0,0 +1,324 @@+//! Runtime tests for the guest-to-guest sync-call inline fast path.++#![cfg(not(miri))]++use wasmtime::component::*;+use wasmtime::{Config, Engine, Result, Store, StoreContextMut};++fn engine() -> Engine {+ let mut config = Config::new();+ config.wasm_component_model(true);+ config.wasm_component_model_async(true);+ Engine::new(&config).unwrap()+}++#[tokio::test]+async fn host_call_forces_slow_path_preserves_context() -> Result<()> {+ let component = r#"+(component+ (import "poke" (func $poke))++ (component $Inner+ (import "poke" (func $poke))+ (core func $poke' (canon lower (func $poke)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "poke" (func $poke'))+ (import "" "cget" (func $cget (result i32)))+ (import "" "cset" (func $cset (param i32)))+ (func (export "f'") (param i32) (result i32)+ ;; Freshly entered deferred thread: context starts zeroed.+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x5678))+ ;; Force the deferred thread via a guest->host call.+ (call $poke')+ ;; Our context survives the force.+ (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable))+ (i32.add (local.get 0) (i32.const 42))))+ (core instance $m (instantiate $M (with "" (instance+ (export "poke" (func $poke'))+ (export "cget" (func $cget))+ (export "cset" (func $cset))))))+ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'"))))++ (component $Outer+ (import "f" (func $f (param "x" u32) (result u32)))+ (core func $f' (canon lower (func $f)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (import "" "cget" (func $cget (result i32)))+ (import "" "cset" (func $cset (param i32)))+ (func (export "g'") (result i32) (local $r i32)+ (call $cset (i32.const 0x1234))+ (local.set $r (call $f' (i32.const 1234)))+ ;; Restored after the callee forced the slow exit path.+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))+ (local.get $r)))+ (core instance $n (instantiate $N (with "" (instance+ (export "f'" (func $f'))+ (export "cget" (func $cget))+ (export "cset" (func $cset))))))+ (func (export "g") (result u32)+ (canon lift (core func $n "g'"))))++ (instance $inner (instantiate $Inner (with "poke" (func $poke))))+ (instance $outer (instantiate $Outer (with "f" (func $inner "f"))))+ (export "g" (func $outer "g"))+)+ "#;++ let engine = engine();+ let component = Component::new(&engine, component)?;+ let mut store = Store::new(&engine, 0u32);+ let mut linker = Linker::new(&engine);+ linker+ .root()+ .func_wrap("poke", |mut cx: StoreContextMut<u32>, (): ()| {+ *cx.data_mut() += 1;+ Ok(())+ })?;+ let instance = linker.instantiate_async(&mut store, &component).await?;+ let g = instance.get_typed_func::<(), (u32,)>(&mut store, "g")?;++ let (result,) = g.call_async(&mut store, ()).await?;+ assert_eq!(result, 1276);+ assert_eq!(*store.data(), 1, "host import should have been called once");+ Ok(())+}++#[tokio::test]+async fn nested_chain_host_force_preserves_all_contexts() -> Result<()> {+ let component = r#"+(component+ (import "poke" (func $poke))++ (component $Leaf+ (import "poke" (func $poke))+ (core func $poke' (canon lower (func $poke)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "poke" (func $poke'))+ (import "" "cget" (func $cget (result i32)))+ (import "" "cset" (func $cset (param i32)))+ (func (export "leaf'") (param i32) (result i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x0c0ffee0))+ (call $poke')+ (if (i32.ne (call $cget) (i32.const 0x0c0ffee0)) (then unreachable))+ (i32.add (local.get 0) (i32.const 1))))+ (core instance $m (instantiate $M (with "" (instance+ (export "poke" (func $poke'))+ (export "cget" (func $cget))+ (export "cset" (func $cset))))))+ (func (export "leaf") (param "x" u32) (result u32)+ (canon lift (core func $m "leaf'"))))++ (component $Mid+ (import "leaf" (func $leaf (param "x" u32) (result u32)))+ (core func $leaf' (canon lower (func $leaf)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))+ (import "" "cget" (func $cget (result i32)))+ (import "" "cset" (func $cset (param i32)))+ (func (export "mid'") (param i32) (result i32) (local $r i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x0d00d100))+ (local.set $r (call $leaf' (local.get 0)))+ (if (i32.ne (call $cget) (i32.const 0x0d00d100)) (then unreachable))+ (i32.add (local.get $r) (i32.const 10))))+ (core instance $m (instantiate $M (with "" (instance+ (export "leaf'" (func $leaf'))+ (export "cget" (func $cget))+ (export "cset" (func $cset))))))+ (func (export "mid") (param "x" u32) (result u32)+ (canon lift (core func $m "mid'"))))++ (component $Root+ (import "mid" (func $mid (param "x" u32) (result u32)))+ (core func $mid' (canon lower (func $mid)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "mid'" (func $mid' (param i32) (result i32)))+ (import "" "cget" (func $cget (result i32)))+ (import "" "cset" (func $cset (param i32)))+ (func (export "root'") (result i32) (local $r i32)+ (call $cset (i32.const 0x0badf00d))+ (local.set $r (call $mid' (i32.const 100)))+ (if (i32.ne (call $cget) (i32.const 0x0badf00d)) (then unreachable))+ (i32.add (local.get $r) (i32.const 1000))))+ (core instance $m (instantiate $M (with "" (instance+ (export "mid'" (func $mid'))+ (export "cget" (func $cget))+ (export "cset" (func $cset))))))+ (func (export "root") (result u32)+ (canon lift (core func $m "root'"))))++ (instance $leaf (instantiate $Leaf (with "poke" (func $poke))))+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))+ (export "root" (func $root "root"))+)+ "#;++ let engine = engine();+ let component = Component::new(&engine, component)?;+ let mut store = Store::new(&engine, 0u32);+ let mut linker = Linker::new(&engine);+ linker+ .root()+ .func_wrap("poke", |mut cx: StoreContextMut<u32>, (): ()| {+ *cx.data_mut() += 1;+ Ok(())+ })?;+ let instance = linker.instantiate_async(&mut store, &component).await?;+ let root = instance.get_typed_func::<(), (u32,)>(&mut store, "root")?;++ let (result,) = root.call_async(&mut store, ()).await?;+ assert_eq!(result, 1111);+ assert_eq!(*store.data(), 1);+ Ok(())+}++#[tokio::test]+async fn repeated_calls_have_no_state_leak() -> Result<()> {+ let component = r#"+(component+ (import "poke" (func $poke))++ (component $Inner+ (import "poke" (func $poke))+ (core func $poke' (canon lower (func $poke)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "poke" (func $poke'))+ (import "" "cget" (func $cget (result i32)))+ (import "" "cset" (func $cset (param i32)))+ (func (export "f'") (param i32) (result i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (local.get 0))+ (call $poke')+ (if (i32.ne (call $cget) (local.get 0)) (then unreachable))+ (i32.add (local.get 0) (i32.const 42))))+ (core instance $m (instantiate $M (with "" (instance+ (export "poke" (func $poke'))+ (export "cget" (func $cget))+ (export "cset" (func $cset))))))+ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'"))))++ (component $Outer+ (import "f" (func $f (param "x" u32) (result u32)))+ (core func $f' (canon lower (func $f)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (import "" "cget" (func $cget (result i32)))+ (import "" "cset" (func $cset (param i32)))+ (func (export "g'") (param i32) (result i32) (local $r i32)+ (call $cset (i32.const 0x4321))+ (local.set $r (call $f' (local.get 0)))+ (if (i32.ne (call $cget) (i32.const 0x4321)) (then unreachable))+ (local.get $r)))+ (core instance $n (instantiate $N (with "" (instance+ (export "f'" (func $f'))+ (export "cget" (func $cget))+ (export "cset" (func $cset))))))+ (func (export "g") (param "x" u32) (result u32)+ (canon lift (core func $n "g'"))))++ (instance $inner (instantiate $Inner (with "poke" (func $poke))))+ (instance $outer (instantiate $Outer (with "f" (func $inner "f"))))+ (export "g" (func $outer "g"))+)+ "#;++ let engine = engine();+ let component = Component::new(&engine, component)?;+ let mut store = Store::new(&engine, 0u32);+ let mut linker = Linker::new(&engine);+ linker+ .root()+ .func_wrap("poke", |mut cx: StoreContextMut<u32>, (): ()| {+ *cx.data_mut() += 1;+ Ok(())+ })?;+ let instance = linker.instantiate_async(&mut store, &component).await?;+ let g = instance.get_typed_func::<(u32,), (u32,)>(&mut store, "g")?;++ for x in [7u32, 100, 0x10000, 1] {+ let (result,) = g.call_async(&mut store, (x,)).await?;+ assert_eq!(result, x + 42);+ }+ assert_eq!(+ *store.data(),+ 4,+ "host import called once per top-level call"+ );+ Ok(())+}++#[tokio::test]+async fn trap_then_instantiate_uses_freed_deferred_thread() -> Result<()> {+ let trapping = r#"+(component+ (component $Leaf+ (core module $M (func (export "leaf'") (param i32) (result i32) unreachable))+ (core instance $m (instantiate $M))+ (func (export "leaf") (param "x" u32) (result u32) (canon lift (core func $m "leaf'"))))+ (component $Mid+ (import "leaf" (func $leaf (param "x" u32) (result u32)))+ (core func $leaf' (canon lower (func $leaf)))+ (core module $M+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))+ (func (export "mid'") (param i32) (result i32) (call $leaf' (local.get 0))))+ (core instance $m (instantiate $M (with "" (instance (export "leaf'" (func $leaf'))))))+ (func (export "mid") (param "x" u32) (result u32) (canon lift (core func $m "mid'"))))+ (component $Root+ (import "mid" (func $mid (param "x" u32) (result u32)))+ (core func $mid' (canon lower (func $mid)))+ (core module $M+ (import "" "mid'" (func $mid' (param i32) (result i32)))+ (func (export "root'") (result i32) (call $mid' (i32.const 1))))+ (core instance $m (instantiate $M (with "" (instance (export "mid'" (func $mid'))))))+ (func (export "root") (result u32) (canon lift (core func $m "root'"))))+ (instance $leaf (instantiate $Leaf))+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))+ (export "root" (func $root "root"))+)+ "#;++ let other = r#"+(component+ (core module $m (func (export "x")))+ (core instance (instantiate $m))+)+ "#;++ let engine = engine();+ let trapping = Component::new(&engine, trapping)?;+ let other = Component::new(&engine, other)?;+ let mut store = Store::new(&engine, 0u32);+ let linker = Linker::new(&engine);++ let instance = linker.instantiate_async(&mut store, &trapping).await?;+ let root = instance.get_typed_func::<(), (u32,)>(&mut store, "root")?;+ let err = root.call_async(&mut store, ()).await.unwrap_err();+ assert!(+ err.downcast_ref::<wasmtime::Trap>().is_some(),+ "expected a trap, got: {err:?}"+ );++ let _ = linker.instantiate_async(&mut store, &other).await?;+ Ok(())+}diff --git a/tests/disas/aarch64-entry-trampoline.wat b/tests/disas/aarch64-entry-trampoline.watindex b80d9968c8..a38bd560b2 100644--- a/tests/disas/aarch64-entry-trampoline.wat+++ b/tests/disas/aarch64-entry-trampoline.wat@@ -16,19 +16,22 @@;; stp d12, d13, [sp, #-0x10]!;; stp d10, d11, [sp, #-0x10]!;; stp d8, d9, [sp, #-0x10]!+;; sub sp, sp, #0x10+;; mov x3, x1;; mov x12, x29-;; ldr x11, [x0, #8]-;; str x12, [x11, #0x48]-;; mov x12, sp-;; str x12, [x11, #0x40]-;; adr x13, #0x94-;; str x13, [x11, #0x50]+;; ldr x15, [x0, #8]+;; str x12, [x15, #0x48]+;; mov x13, sp+;; str x13, [x15, #0x40]+;; adr x14, #0xa0+;; str x14, [x15, #0x50];; mov x2, x0-;; mov x3, x1+;; stur x15, [sp];; bl #0-;; ├─╼ exception frame offset: SP = FP - 0x90-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x94-;; 64: mov w0, #1+;; ├─╼ exception frame offset: SP = FP - 0xa0+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xa0+;; 6c: mov w0, #1+;; add sp, sp, #0x10;; ldp d8, d9, [sp], #0x10;; ldp d10, d11, [sp], #0x10;; ldp d12, d13, [sp], #0x10@@ -40,15 +43,19 @@;; ldp x27, x28, [sp], #0x10;; ldp x29, x30, [sp], #0x10;; ret-;; 94: mov w0, #0-;; 98: ldp d8, d9, [sp], #0x10-;; 9c: ldp d10, d11, [sp], #0x10-;; a0: ldp d12, d13, [sp], #0x10-;; a4: ldp d14, d15, [sp], #0x10-;; a8: ldp x19, x20, [sp], #0x10-;; ac: ldp x21, x22, [sp], #0x10-;; b0: ldp x23, x24, [sp], #0x10-;; b4: ldp x25, x26, [sp], #0x10-;; b8: ldp x27, x28, [sp], #0x10-;; bc: ldp x29, x30, [sp], #0x10-;; c0: ret+;; a0: mov x0, #1+;; a4: ldur x15, [sp]+;; a8: str x0, [x15, #0x88]+;; ac: mov w0, #0+;; b0: add sp, sp, #0x10+;; b4: ldp d8, d9, [sp], #0x10+;; b8: ldp d10, d11, [sp], #0x10+;; bc: ldp d12, d13, [sp], #0x10+;; c0: ldp d14, d15, [sp], #0x10+;; c4: ldp x19, x20, [sp], #0x10+;; c8: ldp x21, x22, [sp], #0x10+;; cc: ldp x23, x24, [sp], #0x10+;; d0: ldp x25, x26, [sp], #0x10+;; d4: ldp x27, x28, [sp], #0x10+;; d8: ldp x29, x30, [sp], #0x10+;; dc: retdiff --git a/tests/disas/component-model/direct-adapter-calls-inlining.wat b/tests/disas/component-model/direct-adapter-calls-inlining.watindex dd020c1cea..9d10e2490b 100644--- a/tests/disas/component-model/direct-adapter-calls-inlining.wat+++ b/tests/disas/component-model/direct-adapter-calls-inlining.wat@@ -60,9 +60,9 @@;; region2 = 72 "VMContext+0x48";; region3 = 136 "VMContext+0x88";; region4 = 1610612736 "PublicGlobal"-;; region5 = 104 "VMContext+0x68"-;; region6 = 88 "VMContext+0x58"-;; region7 = 112 "VMContext+0x70"+;; region5 = 112 "VMContext+0x70"+;; region6 = 104 "VMContext+0x68"+;; region7 = 88 "VMContext+0x58";; gv0 = vmctx;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8;; gv2 = load.i64 notrap aligned region1 gv1+24@@ -92,25 +92,23 @@;; @00ee v3 = load.i64 notrap aligned readonly can_move region2 v0+72;; v9 = load.i64 notrap aligned readonly can_move region3 v3+136;; v10 = load.i32 notrap aligned region4 v9-;; v8 = iconst.i32 0-;; v12 = icmp eq v10, v8 ; v8 = 0-;; brif v12, block9, block10+;; brif v10, block9, block10;;-;; block9:-;; v16 = load.i64 notrap aligned readonly can_move region6 v3+88-;; v15 = load.i64 notrap aligned readonly can_move region5 v3+104-;; v14 = iconst.i32 23-;; try_call_indirect v16(v15, v3, v14), sig1, block11, [ context v3, default: block8(exn0) ] ; v14 = 23+;; block10:+;; v24 = load.i64 notrap aligned readonly can_move region7 v3+88+;; v23 = load.i64 notrap aligned readonly can_move region6 v3+104+;; v22 = iconst.i32 23+;; try_call_indirect v24(v23, v3, v22), sig1, block11, [ context v3, default: block8(exn0) ] ; v22 = 23;;;; block11:;; trap user12;;-;; block10:-;; v21 = load.i64 notrap aligned readonly can_move region7 v3+112-;; v22 = load.i32 notrap aligned region4 v21-;; v38 = iconst.i32 0-;; store notrap aligned region4 v38, v21 ; v38 = 0-;; store notrap aligned region4 v22, v21+;; block9:+;; v11 = load.i64 notrap aligned readonly can_move region5 v3+112+;; v12 = load.i32 notrap aligned region4 v11+;; v8 = iconst.i32 0+;; store notrap aligned region4 v8, v11 ; v8 = 0+;; store notrap aligned region4 v12, v11;; jump block13;;;; block13:@@ -120,8 +118,8 @@;; jump block12;;;; block12:-;; v39 = iconst.i32 0-;; store notrap aligned region4 v39, v9 ; v39 = 0+;; v35 = iconst.i32 0+;; store notrap aligned region4 v35, v9 ; v35 = 0;; store.i32 notrap aligned region4 v10, v9;; jump block7;;@@ -129,8 +127,8 @@;; jump block4;;;; block5:-;; v18 = iconst.i32 49-;; call_indirect.i64 sig1, v16(v15, v3, v18) ; v18 = 49+;; v26 = iconst.i32 49+;; call_indirect.i64 sig1, v24(v23, v3, v26) ; v26 = 49;; trap user12;;;; block4:@@ -143,6 +141,6 @@;; @00f0 jump block1;;;; block1:-;; v32 = iconst.i32 1276-;; @00f0 return v32 ; v32 = 1276+;; v29 = iconst.i32 1276+;; @00f0 return v29 ; v29 = 1276;; }diff --git a/tests/disas/component-model/direct-adapter-calls-x64.wat b/tests/disas/component-model/direct-adapter-calls-x64.watindex 8dd507783d..27552042cf 100644--- a/tests/disas/component-model/direct-adapter-calls-x64.wat+++ b/tests/disas/component-model/direct-adapter-calls-x64.wat@@ -87,7 +87,7 @@;; movq 0x18(%r10), %r10;; addq $0x60, %r10;; cmpq %rsp, %r10-;; ja 0x13e+;; ja 0x140;; 79: subq $0x50, %rsp;; movq %rbx, 0x20(%rsp);; movq %r12, 0x28(%rsp)@@ -97,25 +97,33 @@;; movq %rdi, (%rsp);; movq (%rsp), %rdi;; movq 0x88(%rdi), %rcx-;; movl (%rcx), %esi+;; movl (%rcx), %eax;; movq %rcx, 0x10(%rsp)-;; testl %esi, %esi-;; movq %rsi, 8(%rsp)-;; je 0x10c+;; testl %eax, %eax+;; movq %rax, 8(%rsp)+;; jne 0xd5;; b9: movq (%rsp), %rdi-;; movq 0x70(%rdi), %rax+;; movq 0x58(%rdi), %rax+;; movq 0x68(%rdi), %rdi+;; movl $0x17, %edx+;; movq (%rsp), %rsi+;; callq *%rax+;; ├─╼ exception frame offset: SP = FP - 0x50+;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x12b+;; jmp 0x129+;; d5: movq (%rsp), %rsi+;; movq 0x70(%rsi), %rax;; movl (%rax), %ecx;; movl $0, (%rax);; movl %ecx, (%rax)-;; movq 0x48(%rdi), %rdi-;; movq (%rsp), %rsi+;; movq 0x48(%rsi), %rdi;; callq 0;; ├─╼ exception frame offset: SP = FP - 0x50-;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x125+;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x12b;; movq 0x10(%rsp), %rcx;; movl $0, (%rcx)-;; movq 8(%rsp), %rsi-;; movl %esi, (%rcx)+;; movq 8(%rsp), %rdx+;; movl %edx, (%rcx);; movq 0x20(%rsp), %rbx;; movq 0x28(%rsp), %r12;; movq 0x30(%rsp), %r13@@ -125,20 +133,12 @@;; movq %rbp, %rsp;; popq %rbp;; retq-;; 10c: movq (%rsp), %rcx-;; 110: movq 0x58(%rcx), %rax-;; 114: movq 0x68(%rcx), %rdi-;; 118: movl $0x17, %edx-;; 11d: movq (%rsp), %rsi-;; 121: callq *%rax-;; ├─╼ exception frame offset: SP = FP - 0x50-;; ╰─╼ exception handler: default handler, context at [SP+0x0], handler=0x125-;; 123: ud2-;; 125: movq (%rsp), %rcx-;; 129: movq 0x58(%rcx), %rax-;; 12d: movq 0x68(%rcx), %rdi-;; 131: movl $0x31, %edx-;; 136: movq (%rsp), %rsi-;; 13a: callq *%rax-;; 13c: ud2+;; 124: jmp 0x12b+;; 129: ud2+;; 12b: movq (%rsp), %rsi+;; 12f: movq 0x58(%rsi), %rax+;; 133: movq 0x68(%rsi), %rdi+;; 137: movl $0x31, %edx+;; 13c: callq *%rax;; 13e: ud2+;; 140: ud2diff --git a/tests/disas/component-model/direct-adapter-calls.wat b/tests/disas/component-model/direct-adapter-calls.watindex 0825e243a0..ae4b07d4d9 100644--- a/tests/disas/component-model/direct-adapter-calls.wat+++ b/tests/disas/component-model/direct-adapter-calls.wat@@ -119,49 +119,47 @@;; @007b jump block3;;;; block4:-;; @0080 v6 = load.i64 notrap aligned readonly can_move region2 v0+136-;; @0080 v7 = load.i32 notrap aligned region3 v6-;; @0075 v3 = iconst.i32 0-;; @0084 v9 = icmp eq v7, v3 ; v3 = 0-;; @0085 brif v9, block7, block8+;; @0082 v6 = load.i64 notrap aligned readonly can_move region2 v0+136+;; @0082 v7 = load.i32 notrap aligned region3 v6+;; @0086 brif v7, block7, block8;;-;; block7:-;; @0089 v13 = load.i64 notrap aligned readonly can_move region5 v0+88-;; @0089 v12 = load.i64 notrap aligned readonly can_move region4 v0+104-;; @0087 v11 = iconst.i32 23-;; @0089 try_call_indirect v13(v12, v0, v11), sig0, block9, [ context v0, default: block6(exn0) ] ; v11 = 23+;; block8:+;; @008a v10 = load.i64 notrap aligned readonly can_move region5 v0+88+;; @008a v9 = load.i64 notrap aligned readonly can_move region4 v0+104+;; @0088 v8 = iconst.i32 23+;; @008a try_call_indirect v10(v9, v0, v8), sig0, block9, [ context v0, default: block6(exn0) ] ; v8 = 23;;;; block9:-;; @008b trap user12+;; @008c trap user12;;-;; block8:-;; @008d v14 = load.i64 notrap aligned readonly can_move region6 v0+112-;; @008d v15 = load.i32 notrap aligned region3 v14-;; v27 = iconst.i32 0-;; @0093 store notrap aligned region3 v27, v14 ; v27 = 0-;; @0099 store notrap aligned region3 v15, v14-;; @009b v19 = load.i64 notrap aligned readonly can_move region7 v0+72-;; @009b try_call fn0(v19, v0, v2), sig1, block10(ret0), [ context v0, default: block6(exn0) ]-;;-;; block10(v20: i32):-;; v28 = iconst.i32 0-;; @00a1 store notrap aligned region3 v28, v6 ; v28 = 0-;; @00a7 store.i32 notrap aligned region3 v7, v6-;; @00a9 jump block5+;; block7:+;; @008e v11 = load.i64 notrap aligned readonly can_move region6 v0+112+;; @008e v12 = load.i32 notrap aligned region3 v11+;; @0075 v3 = iconst.i32 0+;; @0094 store notrap aligned region3 v3, v11 ; v3 = 0+;; @009a store notrap aligned region3 v12, v11+;; @009c v16 = load.i64 notrap aligned readonly can_move region7 v0+72+;; @009c try_call fn0(v16, v0, v2), sig1, block10(ret0), [ context v0, default: block6(exn0) ]+;;+;; block10(v17: i32):+;; v24 = iconst.i32 0+;; @00a2 store notrap aligned region3 v24, v6 ; v24 = 0+;; @00a8 store.i32 notrap aligned region3 v7, v6+;; @00aa jump block5;;;; block5:-;; @00aa jump block2+;; @00ab jump block2;;;; block3:-;; v29 = load.i64 notrap aligned readonly can_move region5 v0+88-;; v30 = load.i64 notrap aligned readonly can_move region4 v0+104-;; @00ad v24 = iconst.i32 49-;; @00af call_indirect sig0, v29(v30, v0, v24) ; v24 = 49-;; @00b1 trap user12+;; v25 = load.i64 notrap aligned readonly can_move region5 v0+88+;; v26 = load.i64 notrap aligned readonly can_move region4 v0+104+;; @00ae v21 = iconst.i32 49+;; @00b0 call_indirect sig0, v25(v26, v0, v21) ; v21 = 49+;; @00b2 trap user12;;;; block2:-;; @00b3 jump block1+;; @00b4 jump block1;;;; block1:-;; @00b3 return v20+;; @00b4 return v17;; }diff --git a/tests/disas/component-model/sync-adapter-calls.wat b/tests/disas/component-model/sync-adapter-calls.watnew file mode 100644index 0000000000..f2a1058e82--- /dev/null+++ b/tests/disas/component-model/sync-adapter-calls.wat@@ -0,0 +1,330 @@+;;! target = "x86_64"+;;! test = "optimize"+;;! filter = "function"+;;! flags = "-C inlining=y -Wconcurrency-support=y"++(component+ (component $A+ (core module $M+ (func (export "f'") (param i32) (result i32)+ (i32.add (local.get 0) (i32.const 42))+ )+ )++ (core instance $m (instantiate $M))++ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'"))+ )+ )++ (component $B+ (import "f" (func $f (param "x" u32) (result u32)))++ (core func $f' (canon lower (func $f)))++ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (func (export "g'") (result i32)+ (call $f' (i32.const 1234))+ )+ )++ (core instance $n+ (instantiate $N+ (with "" (instance (export "f'" (func $f'))))+ )+ )++ (func (export "g") (result u32)+ (canon lift (core func $n "g'"))+ )+ )++ (instance $a (instantiate $A))+ (instance $b+ (instantiate $B+ (with "f" (func $a "f"))+ )+ )++ (export "g" (func $b "g"))+)+;; function u0:0(i64 vmctx, i64, i32) -> i32 tail {+;; region0 = 8 "VMContext+0x8"+;; region1 = 268435480 "VMStoreContext+0x18"+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8+;; gv2 = load.i64 notrap aligned region1 gv1+24+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @003b jump block1+;;+;; block1:+;; @0038 v3 = iconst.i32 42+;; v4 = iadd.i32 v2, v3 ; v3 = 42+;; @003b return v4+;; }+;;+;; function u1:0(i64 vmctx, i64) -> i32 tail {+;; ss0 = explicit_slot 32, align = 8+;; region0 = 8 "VMContext+0x8"+;; region1 = 268435480 "VMStoreContext+0x18"+;; region2 = 72 "VMContext+0x48"+;; region3 = 200 "VMContext+0xc8"+;; region4 = 1610612736 "PublicGlobal"+;; region5 = 224 "VMContext+0xe0"+;; region6 = 136 "VMContext+0x88"+;; region7 = 268435592 "VMStoreContext+0x88"+;; region8 = 4026531840 "VMDeferredThread+0x0"+;; region9 = 4026531848 "VMDeferredThread+0x8"+;; region10 = 4026531852 "VMDeferredThread+0xc"+;; region11 = 4026531856 "VMDeferredThread+0x10"+;; region12 = 268435584 "VMStoreContext+0x80"+;; region13 = 4026531860 "VMDeferredThread+0x14"+;; region14 = 268435588 "VMStoreContext+0x84"+;; region15 = 4026531864 "VMDeferredThread+0x18"+;; region16 = 176 "VMContext+0xb0"+;; region17 = 168 "VMContext+0xa8"+;; region18 = 152 "VMContext+0x98"+;; region19 = 104 "VMContext+0x68"+;; region20 = 88 "VMContext+0x58"+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8+;; gv2 = load.i64 notrap aligned region1 gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move region0 gv3+8+;; gv5 = load.i64 notrap aligned region1 gv4+24+;; gv6 = vmctx+;; gv7 = load.i64 notrap aligned readonly can_move region0 gv6+8+;; gv8 = load.i64 notrap aligned region1 gv7+24+;; sig0 = (i64 vmctx, i64, i32) -> i32 tail+;; sig1 = (i64 vmctx, i64, i32) tail+;; sig2 = (i64 vmctx, i64, i32, i32, i32) tail+;; sig3 = (i64 vmctx, i64, i32) -> i32 tail+;; sig4 = (i64 vmctx, i64) tail+;; fn0 = colocated u2:0 sig0+;; fn1 = colocated u0:0 sig3+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64):+;; @00ee jump block2+;;+;; block2:+;; jump block6+;;+;; block8(v5: i64):+;; jump block5+;;+;; block6:+;; @00ee v3 = load.i64 notrap aligned readonly can_move region2 v0+72+;; v9 = load.i64 notrap aligned readonly can_move region3 v3+200+;; v10 = load.i32 notrap aligned region4 v9+;; brif v10, block9, block10+;;+;; block10:+;; v53 = load.i64 notrap aligned readonly can_move region20 v3+88+;; v52 = load.i64 notrap aligned readonly can_move region19 v3+104+;; v51 = iconst.i32 23+;; try_call_indirect v53(v52, v3, v51), sig1, block11, [ context v3, default: block8(exn0) ] ; v51 = 23+;;+;; block11:+;; trap user12+;;+;; block9:+;; v11 = load.i64 notrap aligned readonly can_move region5 v3+224+;; v12 = load.i32 notrap aligned region4 v11+;; v8 = iconst.i32 0+;; store notrap aligned region4 v8, v11 ; v8 = 0+;; v20 = load.i64 notrap aligned readonly can_move region0 v3+8+;; v21 = load.i64 notrap aligned region7 v20+136+;; v19 = stack_addr.i64 ss0+;; store notrap aligned region8 v21, v19+;; v15 = iconst.i32 2+;; store notrap aligned region9 v15, v19+8 ; v15 = 2+;; store notrap aligned region10 v8, v19+12 ; v8 = 0+;; v17 = iconst.i32 1+;; store notrap aligned region11 v17, v19+16 ; v17 = 1+;; v22 = load.i32 notrap aligned region12 v20+128+;; store notrap aligned region13 v22, v19+20+;; store notrap aligned region12 v8, v20+128 ; v8 = 0+;; v24 = load.i32 notrap aligned region14 v20+132+;; store notrap aligned region15 v24, v19+24+;; store notrap aligned region14 v8, v20+132 ; v8 = 0+;; store notrap aligned region7 v19, v20+136+;; v26 = load.i64 notrap aligned readonly can_move region16 v3+176+;; v27 = load.i32 notrap aligned region4 v26+;; store notrap aligned region4 v8, v26 ; v8 = 0+;; store notrap aligned region4 v27, v26+;; jump block17+;;+;; block17:+;; jump block18+;;+;; block18:+;; jump block12+;;+;; block12:+;; jump block13+;;+;; block13:+;; store.i64 notrap aligned region7 v21, v20+136+;; store.i32 notrap aligned region12 v22, v20+128+;; store.i32 notrap aligned region14 v24, v20+132+;; jump block15+;;+;; block15:+;; v61 = iconst.i32 0+;; store notrap aligned region4 v61, v9 ; v61 = 0+;; store.i32 notrap aligned region4 v10, v9+;; store.i32 notrap aligned region4 v12, v11+;; jump block7+;;+;; block7:+;; jump block4+;;+;; block5:+;; v62 = load.i64 notrap aligned readonly can_move region20 v3+88+;; v63 = load.i64 notrap aligned readonly can_move region19 v3+104+;; v48 = iconst.i32 49+;; call_indirect sig1, v62(v63, v3, v48) ; v48 = 49+;; trap user12+;;+;; block4:+;; jump block3+;;+;; block3:+;; jump block19+;;+;; block19:+;; @00f0 jump block1+;;+;; block1:+;; v54 = iconst.i32 1276+;; @00f0 return v54 ; v54 = 1276+;; }+;;+;; function u2:0(i64 vmctx, i64, i32) -> i32 tail {+;; ss0 = explicit_slot 32, align = 8+;; region0 = 8 "VMContext+0x8"+;; region1 = 268435480 "VMStoreContext+0x18"+;; region2 = 200 "VMContext+0xc8"+;; region3 = 1610612736 "PublicGlobal"+;; region4 = 104 "VMContext+0x68"+;; region5 = 88 "VMContext+0x58"+;; region6 = 224 "VMContext+0xe0"+;; region7 = 136 "VMContext+0x88"+;; region8 = 268435592 "VMStoreContext+0x88"+;; region9 = 4026531840 "VMDeferredThread+0x0"+;; region10 = 4026531848 "VMDeferredThread+0x8"+;; region11 = 4026531852 "VMDeferredThread+0xc"+;; region12 = 4026531856 "VMDeferredThread+0x10"+;; region13 = 268435584 "VMStoreContext+0x80"+;; region14 = 4026531860 "VMDeferredThread+0x14"+;; region15 = 268435588 "VMStoreContext+0x84"+;; region16 = 4026531864 "VMDeferredThread+0x18"+;; region17 = 176 "VMContext+0xb0"+;; region18 = 72 "VMContext+0x48"+;; region19 = 168 "VMContext+0xa8"+;; region20 = 152 "VMContext+0x98"+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8+;; gv2 = load.i64 notrap aligned region1 gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move region0 gv3+8+;; gv5 = load.i64 notrap aligned region1 gv4+24+;; sig0 = (i64 vmctx, i64, i32) tail+;; sig1 = (i64 vmctx, i64, i32, i32, i32) tail+;; sig2 = (i64 vmctx, i64, i32) -> i32 tail+;; sig3 = (i64 vmctx, i64) tail+;; fn0 = colocated u0:0 sig2+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @00cf jump block4+;;+;; block6(v4: i64):+;; @00cf jump block3+;;+;; block4:+;; @00d6 v6 = load.i64 notrap aligned readonly can_move region2 v0+200+;; @00d6 v7 = load.i32 notrap aligned region3 v6+;; @00da brif v7, block7, block8+;;+;; block8:+;; @00de v10 = load.i64 notrap aligned readonly can_move region5 v0+88+;; @00de v9 = load.i64 notrap aligned readonly can_move region4 v0+104+;; @00dc v8 = iconst.i32 23+;; @00de try_call_indirect v10(v9, v0, v8), sig0, block9, [ context v0, default: block6(exn0) ] ; v8 = 23+;;+;; block9:+;; @00e0 trap user12+;;+;; block7:+;; @00e2 v11 = load.i64 notrap aligned readonly can_move region6 v0+224+;; @00e2 v12 = load.i32 notrap aligned region3 v11+;; @00c9 v3 = iconst.i32 0+;; @00e8 store notrap aligned region3 v3, v11 ; v3 = 0+;; @00f0 v20 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @00f0 v21 = load.i64 notrap aligned region8 v20+136+;; @00f0 v19 = stack_addr.i64 ss0+;; @00f0 store notrap aligned region9 v21, v19+;; @00ea v15 = iconst.i32 2+;; @00f0 store notrap aligned region10 v15, v19+8 ; v15 = 2+;; @00f0 store notrap aligned region11 v3, v19+12 ; v3 = 0+;; @00ee v17 = iconst.i32 1+;; @00f0 store notrap aligned region12 v17, v19+16 ; v17 = 1+;; @00f0 v22 = load.i32 notrap aligned region13 v20+128+;; @00f0 store notrap aligned region14 v22, v19+20+;; @00f0 store notrap aligned region13 v3, v20+128 ; v3 = 0+;; @00f0 v24 = load.i32 notrap aligned region15 v20+132+;; @00f0 store notrap aligned region16 v24, v19+24+;; @00f0 store notrap aligned region15 v3, v20+132 ; v3 = 0+;; @00f0 store notrap aligned region8 v19, v20+136+;; @00f2 v26 = load.i64 notrap aligned readonly can_move region17 v0+176+;; @00f2 v27 = load.i32 notrap aligned region3 v26+;; @00f8 store notrap aligned region3 v3, v26 ; v3 = 0+;; @00fe store notrap aligned region3 v27, v26+;; @0100 jump block15+;;+;; block15:+;; jump block16+;;+;; block16:+;; jump block10+;;+;; block10:+;; @0104 jump block11+;;+;; block11:+;; @0104 store.i64 notrap aligned region8 v21, v20+136+;; @0104 store.i32 notrap aligned region13 v22, v20+128+;; @0104 store.i32 notrap aligned region15 v24, v20+132+;; @0104 jump block13+;;+;; block13:+;; v55 = iconst.i32 0+;; @0108 store notrap aligned region3 v55, v6 ; v55 = 0+;; @010e store.i32 notrap aligned region3 v7, v6+;; @0112 store.i32 notrap aligned region3 v12, v11+;; @0114 jump block5+;;+;; block5:+;; @0115 jump block2+;;+;; block3:+;; v56 = load.i64 notrap aligned readonly can_move region5 v0+88+;; v57 = load.i64 notrap aligned readonly can_move region4 v0+104+;; @0118 v49 = iconst.i32 49+;; @011a call_indirect sig0, v56(v57, v0, v49) ; v49 = 49+;; @011c trap user12+;;+;; block2:+;; @011e jump block1+;;+;; block1:+;; v52 = iconst.i32 42+;; v53 = iadd.i32 v2, v52 ; v52 = 42+;; @011e return v53+;; }diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.watindex 327084f2f5..d6343a619b 100644--- a/tests/disas/debug-exceptions.wat+++ b/tests/disas/debug-exceptions.wat@@ -37,29 +37,29 @@;; stur x2, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x35, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [62, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [69, 1, 0, 148];; ldur x0, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [60, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [67, 1, 0, 148];; ldur x0, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3d, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [58, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [65, 1, 0, 148];; mov w1, #0x2a;; stur w1, [sp, #8];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3f, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [55, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [62, 1, 0, 148];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x40, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [54, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [61, 1, 0, 148];; stur w1, [sp, #8];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [52, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [59, 1, 0, 148];; ldur x2, [sp, #0x10]-;; bl #0x4b4+;; bl #0x4d0;; 88: ldur x0, [sp, #0x10];; mov x19, x2;; ldr x1, [x0, #0x20]@@ -92,7 +92,7 @@;; mov w11, #0;; str w11, [x1, #0x14];; ldur x2, [sp, #0x10]-;; bl #0x4ec+;; bl #0x508;; ├─╼ exception frame offset: SP = FP - 0xb0;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x110;; b #0x1d8@@ -113,7 +113,7 @@;; ldur x0, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x48, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [0, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [7, 1, 0, 148];; ldur x1, [sp, #0x10];; ldr x0, [x1, #0x38];; ldr x2, [x1, #0x48]@@ -123,10 +123,10 @@;; 164: ldur x0, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [249, 0, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [0, 1, 0, 148];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4b, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [248, 0, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [255, 0, 0, 148];; add sp, sp, #0x20;; ldp d8, d9, [sp], #0x10;; ldp d10, d11, [sp], #0x10@@ -147,7 +147,7 @@;; 1b4: mov w5, #0x20;; 1b8: mov w6, #0x10;; 1bc: ldur x2, [sp, #0x10]-;; 1c0: bl #0x3e0+;; 1c0: bl #0x3fc;; 1c4: ldur x0, [sp, #0x18];; 1c8: ldr x3, [x0, #0x20];; 1cc: add x1, x3, w2, uxtw@@ -155,29 +155,29 @@;; 1d4: b #0xec;; 1d8: mov w3, #9;; 1dc: ldur x2, [sp, #0x10]-;; 1e0: bl #0x448+;; 1e0: bl #0x464;; 1e4: ldur x2, [sp, #0x10]-;; 1e8: bl #0x480+;; 1e8: bl #0x49c;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8;; 1ec: udf #0xc11f;; 1f0: mov w3, #0xfe;; 1f4: ldur x2, [sp, #0x10]-;; 1f8: bl #0x448+;; 1f8: bl #0x464;; 1fc: ldur x2, [sp, #0x10]-;; 200: bl #0x480+;; 200: bl #0x49c;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack;; 204: udf #0xc11f;; 208: mov w3, #0xfe;; 20c: ldur x2, [sp, #0x10]-;; 210: bl #0x448+;; 210: bl #0x464;; 214: ldur x2, [sp, #0x10]-;; 218: bl #0x480+;; 218: bl #0x49c;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack;; 21c: udf #0xc11f;; 220: stur x2, [sp, #0x10];; 224: mov w3, #0-;; 228: bl #0x448+;; 228: bl #0x464;; 22c: ldur x2, [sp, #0x10]-;; 230: bl #0x480+;; 230: bl #0x49c;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x34, slot at FP-0xb0, locals , stack;; 234: udf #0xc11fdiff --git a/tests/disas/debug.wat b/tests/disas/debug.watindex abbcc23c97..74f254de13 100644--- a/tests/disas/debug.wat+++ b/tests/disas/debug.wat@@ -23,20 +23,20 @@;; 29: movq %rdi, (%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x24, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack-;; ╰─╼ breakpoint patch: wasm PC 0x24, patch bytes [232, 190, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x24, patch bytes [232, 212, 1, 0, 0];; movl %edx, 0x10(%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x26, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10-;; ╰─╼ breakpoint patch: wasm PC 0x26, patch bytes [232, 181, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x26, patch bytes [232, 203, 1, 0, 0];; movl %ecx, 0x14(%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x28, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10, I32 @ slot+0x14-;; ╰─╼ breakpoint patch: wasm PC 0x28, patch bytes [232, 172, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x28, patch bytes [232, 194, 1, 0, 0];; leal (%rdx, %rcx), %eax;; movl %eax, 0x10(%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x29, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10-;; ╰─╼ breakpoint patch: wasm PC 0x29, patch bytes [232, 160, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x29, patch bytes [232, 182, 1, 0, 0];; movl %eax, 0x10(%rsp);; movq 0x20(%rsp), %r12;; addq $0x30, %rsp@@ -45,9 +45,9 @@;; retq;; 62: movq %rdi, %r12;; 65: xorl %esi, %esi-;; 67: callq 0x192+;; 67: callq 0x1a8;; 6c: movq %r12, %rdi-;; 6f: callq 0x1c3+;; 6f: callq 0x1d9;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x23, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack;; 74: ud2;;@@ -60,20 +60,21 @@;; movq %r13, 0x20(%rsp);; movq %r14, 0x28(%rsp);; movq %r15, 0x30(%rsp)-;; movl (%rdx), %eax+;; movl (%rdx), %r8d;; movl 0x10(%rdx), %ecx;; movq %rdx, (%rsp);; movq %rbp, %r9-;; movq 8(%rdi), %r8-;; movq %r9, 0x48(%r8)+;; movq 8(%rdi), %r10+;; movq %r9, 0x48(%r10);; movq %rsp, %r9-;; movq %r9, 0x40(%r8)-;; leaq 0x39(%rip), %r9-;; movq %r9, 0x50(%r8)-;; movq %rax, %rdx+;; movq %r9, 0x40(%r10)+;; leaq 0x3e(%rip), %r9+;; movq %r9, 0x50(%r10)+;; movq %r8, %rdx+;; movq %r10, 8(%rsp);; callq 0;; ├─╼ exception frame offset: SP = FP - 0x40-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xf2+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xf8;; movq (%rsp), %rdx;; movl %eax, (%rdx);; movl $1, %eax@@ -86,16 +87,18 @@;; movq %rbp, %rsp;; popq %rbp;; retq-;; f2: xorl %eax, %eax-;; f4: movq 0x10(%rsp), %rbx-;; f9: movq 0x18(%rsp), %r12-;; fe: movq 0x20(%rsp), %r13-;; 103: movq 0x28(%rsp), %r14-;; 108: movq 0x30(%rsp), %r15-;; 10d: addq $0x40, %rsp-;; 111: movq %rbp, %rsp-;; 114: popq %rbp-;; 115: retq+;; f8: movq 8(%rsp), %r10+;; fd: movq $1, 0x88(%r10)+;; 108: xorl %eax, %eax+;; 10a: movq 0x10(%rsp), %rbx+;; 10f: movq 0x18(%rsp), %r12+;; 114: movq 0x20(%rsp), %r13+;; 119: movq 0x28(%rsp), %r14+;; 11e: movq 0x30(%rsp), %r15+;; 123: addq $0x40, %rsp+;; 127: movq %rbp, %rsp+;; 12a: popq %rbp+;; 12b: retq;;;; signatures[0]::wasm_to_array_trampoline:;; pushq %rbp@@ -120,19 +123,19 @@;; callq *%rax;; addq $1, 0x10(%r15);; testb %al, %al-;; je 0x180-;; 16a: movl (%rsp), %eax+;; je 0x196+;; 180: movl (%rsp), %eax;; movq 0x20(%rsp), %rbx;; movq 0x28(%rsp), %r15;; addq $0x30, %rsp;; movq %rbp, %rsp;; popq %rbp;; retq-;; 180: movq 0x10(%rbx), %rax-;; 184: movq 0x148(%rax), %rax-;; 18b: movq %rbx, %rdi-;; 18e: callq *%rax-;; 190: ud2+;; 196: movq 0x10(%rbx), %rax+;; 19a: movq 0x148(%rax), %rax+;; 1a1: movq %rbx, %rdi+;; 1a4: callq *%rax+;; 1a6: ud2;;;; wasmtime_builtin_trap:;; pushq %rbp@@ -209,8 +212,8 @@;; movq 0x170(%r12), %r11;; callq *%r11;; testb %al, %al-;; je 0x3c7-;; 2f3: movq (%rsp), %rax+;; je 0x3dd+;; 309: movq (%rsp), %rax;; movq 8(%rsp), %rcx;; movq 0x10(%rsp), %rdx;; movq 0x18(%rsp), %rsi@@ -241,7 +244,7 @@;; movq %rbp, %rsp;; popq %rbp;; retq-;; 3c7: movq 0x148(%r12), %rax-;; 3cf: movq %r13, %rdi-;; 3d2: callq *%rax-;; 3d4: ud2+;; 3dd: movq 0x148(%r12), %rax+;; 3e5: movq %r13, %rdi+;; 3e8: callq *%rax+;; 3ea: ud2diff --git a/tests/disas/exceptions.wat b/tests/disas/exceptions.watindex efee2b2931..a544f5701b 100644--- a/tests/disas/exceptions.wat+++ b/tests/disas/exceptions.wat@@ -33,7 +33,7 @@;; movq %rdi, %rbx;; movq %rcx, %r13;; movq %rdx, %r14-;; callq 0x3eb+;; callq 0x401;; movq %rax, %r12;; movq 0x20(%rbx), %rdx;; movl (%rdx), %esi@@ -62,7 +62,7 @@;; movl $0, 0x14(%rdx);; movq %rbx, %rdi;; movq %rbx, (%rsp)-;; callq 0x418+;; callq 0x42e;; jmp 0xea;; b6: movl $0x4000002, %esi;; movq 0x28(%rbx), %rax@@ -70,7 +70,7 @@;; movl $0x30, %ecx;; movl $0x10, %r8d;; movq %rbx, %rdi-;; callq 0x382+;; callq 0x398;; movq 8(%rbx), %rcx;; movl %eax, %edx;; addq 0x20(%rcx), %rdxdiff --git a/tests/disas/gc/issue-11753.wat b/tests/disas/gc/issue-11753.watindex 43061982a8..bfcf95eeb8 100644--- a/tests/disas/gc/issue-11753.wat+++ b/tests/disas/gc/issue-11753.wat@@ -94,7 +94,7 @@;; f4: movl (%rax), %edx;; f6: movl $0x20, %ecx;; fb: movl $0x10, %r8d-;; 101: callq 0x271+;; 101: callq 0x286;; 106: movq 8(%rsp), %r9;; 10b: movq 8(%r9), %rcx;; 10f: movl %eax, %esidiff --git a/tests/disas/issue-12808.wat b/tests/disas/issue-12808.watindex e638c097ec..37c57321f5 100644--- a/tests/disas/issue-12808.wat+++ b/tests/disas/issue-12808.wat@@ -23,23 +23,23 @@;; 21: movq %rdi, (%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x1f, slot at FP-0x30, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x1f, patch bytes [232, 233, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x1f, patch bytes [232, 254, 1, 0, 0];; movl $0, 8(%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x21, slot at FP-0x30, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x21, patch bytes [232, 220, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x21, patch bytes [232, 241, 1, 0, 0];; movl $0, 0xc(%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x23, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc-;; ╰─╼ breakpoint patch: wasm PC 0x23, patch bytes [232, 207, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x23, patch bytes [232, 228, 1, 0, 0];; movl $0, 0x10(%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x25, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc, I32 @ slot+0x10-;; ╰─╼ breakpoint patch: wasm PC 0x25, patch bytes [232, 194, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x25, patch bytes [232, 215, 1, 0, 0];; movl $0, 0x14(%rsp);; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x27, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc, I32 @ slot+0x10, I32 @ slot+0x14-;; ╰─╼ breakpoint patch: wasm PC 0x27, patch bytes [232, 181, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x27, patch bytes [232, 202, 1, 0, 0];; movq 0x30(%rdi), %rax;; movq (%rax), %rcx;; xorl %eax, %eax@@ -48,7 +48,7 @@;; movq %rax, %rcx;; nopl (%rax, %rax);; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x2b, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc-;; ╰─╼ breakpoint patch: wasm PC 0x2b, patch bytes [232, 156, 1, 0, 0]+;; ╰─╼ breakpoint patch: wasm PC 0x2b, patch bytes [232, 177, 1, 0, 0];; xorl %eax, %eax;; movl $0, 8(%rsp);; movl %ecx, 0xc(%rsp)@@ -59,8 +59,8 @@;; retq;; 93: movq %rdi, %r14;; 96: xorl %esi, %esi-;; 98: callq 0x1b5+;; 98: callq 0x1ca;; 9d: movq %r14, %rdi-;; a0: callq 0x1e6+;; a0: callq 0x1fb;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x1e, slot at FP-0x30, locals , stack;; a5: ud2diff --git a/tests/disas/pulley-entry-trampoline.wat b/tests/disas/pulley-entry-trampoline.watindex 891981b48e..06f18fd4c9 100644--- a/tests/disas/pulley-entry-trampoline.wat+++ b/tests/disas/pulley-entry-trampoline.wat@@ -5,20 +5,24 @@(module (func (export "")));; wasm[0]::array_to_wasm_trampoline[0]:-;; push_frame_save 128, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0+;; push_frame_save 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0;; xmov_fp x12-;; xload64le_o32 x11, x0, 8-;; xstore64le_o32 x11, 72, x12-;; xmov x12, sp-;; xstore64le_o32 x11, 64, x12-;; xpcadd x13, 0x1c // target = 0x40-;; xstore64le_o32 x11, 80, x13-;; call -0x33 // target = 0x0-;; ├─╼ exception frame offset: SP = FP - 0x80-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x40+;; xload64le_o32 x15, x0, 8+;; xstore64le_o32 x15, 72, x12+;; xmov x13, sp+;; xstore64le_o32 x15, 64, x13+;; xpcadd x14, 0x23 // target = 0x47+;; xstore64le_o32 x15, 80, x14+;; xstore64le_o32 sp, 0, x15+;; call -0x3a // target = 0x0+;; ├─╼ exception frame offset: SP = FP - 0x90+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x47;; xone x0-;; pop_frame_restore 128, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0+;; pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0;; ret-;; 40: xzero x0-;; 42: pop_frame_restore 128, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0-;; 47: ret+;; 47: xone x0+;; 49: xload64le_o32 x15, sp, 0+;; 50: xstore64le_o32 x15, 136, x0+;; 57: xzero x0+;; 59: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0+;; 5e: retdiff --git a/tests/disas/riscv64-entry-trampoline.wat b/tests/disas/riscv64-entry-trampoline.watindex dea9c1e05d..620f6cab6c 100644--- a/tests/disas/riscv64-entry-trampoline.wat+++ b/tests/disas/riscv64-entry-trampoline.wat@@ -9,96 +9,100 @@;; sd ra, 8(sp);; sd s0, 0(sp);; mv s0, sp-;; addi sp, sp, -0xc0-;; sd s0, 0xb8(sp)-;; sd s1, 0xb0(sp)-;; sd s2, 0xa8(sp)-;; sd s3, 0xa0(sp)-;; sd s4, 0x98(sp)-;; sd s5, 0x90(sp)-;; sd s6, 0x88(sp)-;; sd s7, 0x80(sp)-;; sd s8, 0x78(sp)-;; sd s9, 0x70(sp)-;; sd s10, 0x68(sp)-;; sd s11, 0x60(sp)-;; fsd fs0, 0x58(sp)-;; fsd fs2, 0x50(sp)-;; fsd fs3, 0x48(sp)-;; fsd fs4, 0x40(sp)-;; fsd fs5, 0x38(sp)-;; fsd fs6, 0x30(sp)-;; fsd fs7, 0x28(sp)-;; fsd fs8, 0x20(sp)-;; fsd fs9, 0x18(sp)-;; fsd fs10, 0x10(sp)-;; fsd fs11, 8(sp)-;; mv a3, s0-;; ld a2, 8(a0)-;; sd a3, 0x48(a2)+;; addi sp, sp, -0xd0+;; sd s0, 0xc8(sp)+;; sd s1, 0xc0(sp)+;; sd s2, 0xb8(sp)+;; sd s3, 0xb0(sp)+;; sd s4, 0xa8(sp)+;; sd s5, 0xa0(sp)+;; sd s6, 0x98(sp)+;; sd s7, 0x90(sp)+;; sd s8, 0x88(sp)+;; sd s9, 0x80(sp)+;; sd s10, 0x78(sp)+;; sd s11, 0x70(sp)+;; fsd fs0, 0x68(sp)+;; fsd fs2, 0x60(sp)+;; fsd fs3, 0x58(sp)+;; fsd fs4, 0x50(sp)+;; fsd fs5, 0x48(sp)+;; fsd fs6, 0x40(sp)+;; fsd fs7, 0x38(sp)+;; fsd fs8, 0x30(sp)+;; fsd fs9, 0x28(sp)+;; fsd fs10, 0x20(sp)+;; fsd fs11, 0x18(sp)+;; mv a2, s0+;; ld a5, 8(a0)+;; sd a2, 0x48(a5);; mv a3, sp-;; sd a3, 0x40(a2)-;; auipc a3, 0-;; addi a3, a3, 0x88-;; sd a3, 0x50(a2)+;; sd a3, 0x40(a5)+;; auipc a4, 0+;; addi a4, a4, 0x8c+;; sd a4, 0x50(a5)+;; sd a5, 0(sp);; auipc ra, 0-;; jalr ra, ra, -0xb0-;; ├─╼ exception frame offset: SP = FP - 0xc0-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x12c+;; jalr ra, ra, -0xb4+;; ├─╼ exception frame offset: SP = FP - 0xd0+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x130;; addi a0, zero, 1-;; ld s0, 0xb8(sp)-;; ld s1, 0xb0(sp)-;; ld s2, 0xa8(sp)-;; ld s3, 0xa0(sp)-;; ld s4, 0x98(sp)-;; ld s5, 0x90(sp)-;; ld s6, 0x88(sp)-;; ld s7, 0x80(sp)-;; ld s8, 0x78(sp)-;; ld s9, 0x70(sp)-;; ld s10, 0x68(sp)-;; ld s11, 0x60(sp)-;; fld fs0, 0x58(sp)-;; fld fs2, 0x50(sp)-;; fld fs3, 0x48(sp)-;; fld fs4, 0x40(sp)-;; fld fs5, 0x38(sp)-;; fld fs6, 0x30(sp)-;; fld fs7, 0x28(sp)-;; fld fs8, 0x20(sp)-;; fld fs9, 0x18(sp)-;; fld fs10, 0x10(sp)-;; fld fs11, 8(sp)-;; addi sp, sp, 0xc0+;; ld s0, 0xc8(sp)+;; ld s1, 0xc0(sp)+;; ld s2, 0xb8(sp)+;; ld s3, 0xb0(sp)+;; ld s4, 0xa8(sp)+;; ld s5, 0xa0(sp)+;; ld s6, 0x98(sp)+;; ld s7, 0x90(sp)+;; ld s8, 0x88(sp)+;; ld s9, 0x80(sp)+;; ld s10, 0x78(sp)+;; ld s11, 0x70(sp)+;; fld fs0, 0x68(sp)+;; fld fs2, 0x60(sp)+;; fld fs3, 0x58(sp)+;; fld fs4, 0x50(sp)+;; fld fs5, 0x48(sp)+;; fld fs6, 0x40(sp)+;; fld fs7, 0x38(sp)+;; fld fs8, 0x30(sp)+;; fld fs9, 0x28(sp)+;; fld fs10, 0x20(sp)+;; fld fs11, 0x18(sp)+;; addi sp, sp, 0xd0;; ld ra, 8(sp);; ld s0, 0(sp);; addi sp, sp, 0x10;; ret+;; addi a0, zero, 1+;; ld a5, 0(sp)+;; sd a0, 0x88(a5);; mv a0, zero-;; ld s0, 0xb8(sp)-;; ld s1, 0xb0(sp)-;; ld s2, 0xa8(sp)-;; ld s3, 0xa0(sp)-;; ld s4, 0x98(sp)-;; ld s5, 0x90(sp)-;; ld s6, 0x88(sp)-;; ld s7, 0x80(sp)-;; ld s8, 0x78(sp)-;; ld s9, 0x70(sp)-;; ld s10, 0x68(sp)-;; ld s11, 0x60(sp)-;; fld fs0, 0x58(sp)-;; fld fs2, 0x50(sp)-;; fld fs3, 0x48(sp)-;; fld fs4, 0x40(sp)-;; fld fs5, 0x38(sp)-;; fld fs6, 0x30(sp)-;; fld fs7, 0x28(sp)-;; fld fs8, 0x20(sp)-;; fld fs9, 0x18(sp)-;; fld fs10, 0x10(sp)-;; fld fs11, 8(sp)-;; addi sp, sp, 0xc0+;; ld s0, 0xc8(sp)+;; ld s1, 0xc0(sp)+;; ld s2, 0xb8(sp)+;; ld s3, 0xb0(sp)+;; ld s4, 0xa8(sp)+;; ld s5, 0xa0(sp)+;; ld s6, 0x98(sp)+;; ld s7, 0x90(sp)+;; ld s8, 0x88(sp)+;; ld s9, 0x80(sp)+;; ld s10, 0x78(sp)+;; ld s11, 0x70(sp)+;; fld fs0, 0x68(sp)+;; fld fs2, 0x60(sp)+;; fld fs3, 0x58(sp)+;; fld fs4, 0x50(sp)+;; fld fs5, 0x48(sp)+;; fld fs6, 0x40(sp)+;; fld fs7, 0x38(sp)+;; fld fs8, 0x30(sp)+;; fld fs9, 0x28(sp)+;; fld fs10, 0x20(sp)+;; fld fs11, 0x18(sp)+;; addi sp, sp, 0xd0;; ld ra, 8(sp);; ld s0, 0(sp);; addi sp, sp, 0x10diff --git a/tests/disas/s390x-entry-trampoline.wat b/tests/disas/s390x-entry-trampoline.watindex 17dd61f3a0..1e0d78fc84 100644--- a/tests/disas/s390x-entry-trampoline.wat+++ b/tests/disas/s390x-entry-trampoline.wat@@ -7,45 +7,48 @@;; wasm[0]::array_to_wasm_trampoline[0]:;; stmg %r6, %r15, 0x30(%r15);; lgr %r1, %r15-;; aghi %r15, -0xe0+;; aghi %r15, -0xe8;; stg %r1, 0(%r15)-;; std %f8, 0xa0(%r15)-;; std %f9, 0xa8(%r15)-;; std %f10, 0xb0(%r15)-;; std %f11, 0xb8(%r15)-;; std %f12, 0xc0(%r15)-;; std %f13, 0xc8(%r15)-;; std %f14, 0xd0(%r15)-;; std %f15, 0xd8(%r15)-;; lg %r5, 0(%r15)-;; lg %r4, 8(%r2)-;; stg %r5, 0x48(%r4)-;; lgr %r5, %r15-;; stg %r5, 0x40(%r4)-;; larl %r5, 0xbe-;; stg %r5, 0x50(%r4)+;; std %f8, 0xa8(%r15)+;; std %f9, 0xb0(%r15)+;; std %f10, 0xb8(%r15)+;; std %f11, 0xc0(%r15)+;; std %f12, 0xc8(%r15)+;; std %f13, 0xd0(%r15)+;; std %f14, 0xd8(%r15)+;; std %f15, 0xe0(%r15)+;; lg %r4, 0(%r15)+;; lg %r5, 8(%r2)+;; stg %r4, 0x48(%r5)+;; lgr %r4, %r15+;; stg %r4, 0x40(%r5)+;; larl %r4, 0xc4+;; stg %r4, 0x50(%r5)+;; stg %r5, 0xa0(%r15);; brasl %r14, 0-;; ├─╼ exception frame offset: SP = FP - 0xe0-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xbe+;; ├─╼ exception frame offset: SP = FP - 0xe8+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xc4;; lhi %r2, 1-;; ld %f8, 0xa0(%r15)-;; ld %f9, 0xa8(%r15)-;; ld %f10, 0xb0(%r15)-;; ld %f11, 0xb8(%r15)-;; ld %f12, 0xc0(%r15)-;; ld %f13, 0xc8(%r15)-;; ld %f14, 0xd0(%r15)-;; ld %f15, 0xd8(%r15)-;; lmg %r6, %r15, 0x110(%r15)+;; ld %f8, 0xa8(%r15)+;; ld %f9, 0xb0(%r15)+;; ld %f10, 0xb8(%r15)+;; ld %f11, 0xc0(%r15)+;; ld %f12, 0xc8(%r15)+;; ld %f13, 0xd0(%r15)+;; ld %f14, 0xd8(%r15)+;; ld %f15, 0xe0(%r15)+;; lmg %r6, %r15, 0x118(%r15);; br %r14+;; lg %r5, 0xa0(%r15)+;; mvghi 0x88(%r5), 1;; lhi %r2, 0-;; ld %f8, 0xa0(%r15)-;; ld %f9, 0xa8(%r15)-;; ld %f10, 0xb0(%r15)-;; ld %f11, 0xb8(%r15)-;; ld %f12, 0xc0(%r15)-;; ld %f13, 0xc8(%r15)-;; ld %f14, 0xd0(%r15)-;; ld %f15, 0xd8(%r15)-;; lmg %r6, %r15, 0x110(%r15)+;; ld %f8, 0xa8(%r15)+;; ld %f9, 0xb0(%r15)+;; ld %f10, 0xb8(%r15)+;; ld %f11, 0xc0(%r15)+;; ld %f12, 0xc8(%r15)+;; ld %f13, 0xd0(%r15)+;; ld %f14, 0xd8(%r15)+;; ld %f15, 0xe0(%r15)+;; lmg %r6, %r15, 0x118(%r15);; br %r14diff --git a/tests/disas/startup-data-active.wat b/tests/disas/startup-data-active.watindex 028708a745..449e32a58f 100644--- a/tests/disas/startup-data-active.wat+++ b/tests/disas/startup-data-active.wat@@ -13,6 +13,7 @@;; region1 = 268435528 "VMStoreContext+0x48";; region2 = 268435520 "VMStoreContext+0x40";; region3 = 268435536 "VMStoreContext+0x50"+;; region4 = 268435592 "VMStoreContext+0x88";; sig0 = (i64 vmctx, i64) tail;; fn0 = colocated u2415919104:0 sig0;;@@ -34,8 +35,10 @@;; return v8 ; v8 = 1;;;; block3:-;; v9 = iconst.i8 0-;; return v9 ; v9 = 0+;; v9 = iconst.i64 1+;; store notrap aligned region4 v9, v4+136 ; v9 = 1+;; v10 = iconst.i8 0+;; return v10 ; v10 = 0;; };;;; function u2415919104:0(i64 vmctx, i64) tail {diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.watindex 4262c9fdf8..4fbd0f4a68 100644--- a/tests/disas/startup-elem-active.wat+++ b/tests/disas/startup-elem-active.wat@@ -17,6 +17,7 @@;; region1 = 268435528 "VMStoreContext+0x48";; region2 = 268435520 "VMStoreContext+0x40";; region3 = 268435536 "VMStoreContext+0x50"+;; region4 = 268435592 "VMStoreContext+0x88";; sig0 = (i64 vmctx, i64) tail;; fn0 = colocated u2415919104:0 sig0;;@@ -38,8 +39,10 @@;; return v8 ; v8 = 1;;;; block3:-;; v9 = iconst.i8 0-;; return v9 ; v9 = 0+;; v9 = iconst.i64 1+;; store notrap aligned region4 v9, v4+136 ; v9 = 1+;; v10 = iconst.i8 0+;; return v10 ; v10 = 0;; };;;; function u2415919104:0(i64 vmctx, i64) tail {diff --git a/tests/disas/startup-global.wat b/tests/disas/startup-global.watindex a507e6d125..a345c52e4a 100644--- a/tests/disas/startup-global.wat+++ b/tests/disas/startup-global.wat@@ -10,6 +10,7 @@;; region1 = 268435528 "VMStoreContext+0x48";; region2 = 268435520 "VMStoreContext+0x40";; region3 = 268435536 "VMStoreContext+0x50"+;; region4 = 268435592 "VMStoreContext+0x88";; sig0 = (i64 vmctx, i64) tail;; fn0 = colocated u2415919104:0 sig0;;@@ -31,8 +32,10 @@;; return v8 ; v8 = 1;;;; block3:-;; v9 = iconst.i8 0-;; return v9 ; v9 = 0+;; v9 = iconst.i64 1+;; store notrap aligned region4 v9, v4+136 ; v9 = 1+;; v10 = iconst.i8 0+;; return v10 ; v10 = 0;; };;;; function u2415919104:0(i64 vmctx, i64) tail {diff --git a/tests/disas/startup-passive-segment.wat b/tests/disas/startup-passive-segment.watindex 9631e49735..1e87184808 100644--- a/tests/disas/startup-passive-segment.wat+++ b/tests/disas/startup-passive-segment.wat@@ -11,6 +11,7 @@;; region1 = 268435528 "VMStoreContext+0x48";; region2 = 268435520 "VMStoreContext+0x40";; region3 = 268435536 "VMStoreContext+0x50"+;; region4 = 268435592 "VMStoreContext+0x88";; sig0 = (i64 vmctx, i64) tail;; fn0 = colocated u2415919104:0 sig0;;@@ -32,8 +33,10 @@;; return v8 ; v8 = 1;;;; block3:-;; v9 = iconst.i8 0-;; return v9 ; v9 = 0+;; v9 = iconst.i64 1+;; store notrap aligned region4 v9, v4+136 ; v9 = 1+;; v10 = iconst.i8 0+;; return v10 ; v10 = 0;; };;;; function u2415919104:0(i64 vmctx, i64) tail {diff --git a/tests/disas/startup-start.wat b/tests/disas/startup-start.watindex 02a113fd61..520bdf2a64 100644--- a/tests/disas/startup-start.wat+++ b/tests/disas/startup-start.wat@@ -11,6 +11,7 @@;; region1 = 268435528 "VMStoreContext+0x48";; region2 = 268435520 "VMStoreContext+0x40";; region3 = 268435536 "VMStoreContext+0x50"+;; region4 = 268435592 "VMStoreContext+0x88";; sig0 = (i64 vmctx, i64) tail;; fn0 = colocated u2415919104:0 sig0;;@@ -32,8 +33,10 @@;; return v8 ; v8 = 1;;;; block3:-;; v9 = iconst.i8 0-;; return v9 ; v9 = 0+;; v9 = iconst.i64 1+;; store notrap aligned region4 v9, v4+136 ; v9 = 1+;; v10 = iconst.i8 0+;; return v10 ; v10 = 0;; };;;; function u2415919104:0(i64 vmctx, i64) tail {diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.watindex 51af0b0123..2ea44c7e48 100644--- a/tests/disas/startup-table-initial-value.wat+++ b/tests/disas/startup-table-initial-value.wat@@ -11,6 +11,7 @@;; region1 = 268435528 "VMStoreContext+0x48";; region2 = 268435520 "VMStoreContext+0x40";; region3 = 268435536 "VMStoreContext+0x50"+;; region4 = 268435592 "VMStoreContext+0x88";; sig0 = (i64 vmctx, i64) tail;; fn0 = colocated u2415919104:0 sig0;;@@ -32,8 +33,10 @@;; return v8 ; v8 = 1;;;; block3:-;; v9 = iconst.i8 0-;; return v9 ; v9 = 0+;; v9 = iconst.i64 1+;; store notrap aligned region4 v9, v4+136 ; v9 = 1+;; v10 = iconst.i8 0+;; return v10 ; v10 = 0;; };;;; function u2415919104:0(i64 vmctx, i64) tail {diff --git a/tests/disas/trunc.wat b/tests/disas/trunc.watindex 165db1d1a6..250b13dcb0 100644--- a/tests/disas/trunc.wat+++ b/tests/disas/trunc.wat@@ -24,7 +24,7 @@;; jne 0x101;; 39: movq %rbx, %rdi;; movdqu (%rsp), %xmm0-;; callq 0x244+;; callq 0x259;; movabsq $13830554455654793216, %rax;; movq %rax, %xmm1;; ucomisd %xmm0, %xmm1@@ -55,27 +55,27 @@;; retq;; d3: movl $6, %esi;; d8: movq %rbx, %rdi-;; db: callq 0x271+;; db: callq 0x286;; e0: movq %rbx, %rdi-;; e3: callq 0x2a2+;; e3: callq 0x2b7;; e8: ud2;; ea: movl $6, %esi;; ef: movq %rbx, %rdi-;; f2: callq 0x271+;; f2: callq 0x286;; f7: movq %rbx, %rdi-;; fa: callq 0x2a2+;; fa: callq 0x2b7;; ff: ud2;; 101: movl $8, %esi;; 106: movq %rbx, %rdi-;; 109: callq 0x271+;; 109: callq 0x286;; 10e: movq %rbx, %rdi-;; 111: callq 0x2a2+;; 111: callq 0x2b7;; 116: ud2;; 118: xorl %esi, %esi;; 11a: movq %rbx, %rdi-;; 11d: callq 0x271+;; 11d: callq 0x286;; 122: movq %rbx, %rdi-;; 125: callq 0x2a2+;; 125: callq 0x2b7;; 12a: ud2;; 12c: ud2;; 12e: ud2diff --git a/tests/disas/trunc32.wat b/tests/disas/trunc32.watindex 370a72b7d5..dc903bde99 100644--- a/tests/disas/trunc32.wat+++ b/tests/disas/trunc32.wat@@ -26,7 +26,7 @@;; jp 0xf6;; jne 0xf6;; 46: movq %rbx, %rdi-;; callq 0x242+;; callq 0x257;; movabsq $13830554455654793216, %rax;; movq %rax, %xmm1;; ucomisd %xmm0, %xmm1@@ -56,27 +56,27 @@;; retq;; c8: movl $6, %esi;; cd: movq %rbx, %rdi-;; d0: callq 0x26f+;; d0: callq 0x284;; d5: movq %rbx, %rdi-;; d8: callq 0x2a0+;; d8: callq 0x2b5;; dd: ud2;; df: movl $6, %esi;; e4: movq %rbx, %rdi-;; e7: callq 0x26f+;; e7: callq 0x284;; ec: movq %rbx, %rdi-;; ef: callq 0x2a0+;; ef: callq 0x2b5;; f4: ud2;; f6: movl $8, %esi;; fb: movq %rbx, %rdi-;; fe: callq 0x26f+;; fe: callq 0x284;; 103: movq %rbx, %rdi-;; 106: callq 0x2a0+;; 106: callq 0x2b5;; 10b: ud2;; 10d: xorl %esi, %esi;; 10f: movq %rbx, %rdi-;; 112: callq 0x26f+;; 112: callq 0x284;; 117: movq %rbx, %rdi-;; 11a: callq 0x2a0+;; 11a: callq 0x2b5;; 11f: ud2;; 121: ud2;; 123: ud2diff --git a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat b/tests/disas/winch/aarch64/call_indirect/call_indirect.watindex e6b86c04d5..6c26d05746 100644--- a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat+++ b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat@@ -85,7 +85,7 @@;; mov x0, x9;; mov x1, #0;; ldur w2, [x28]-;; bl #0x3f0+;; bl #0x408;; e0: add x28, x28, #4;; mov sp, x28;; ldur x9, [x28, #0x14]@@ -153,7 +153,7 @@;; mov x0, x9;; mov x1, #0;; ldur w2, [x28, #0xc]-;; bl #0x3f0+;; bl #0x408;; 1f0: add x28, x28, #0xc;; mov sp, x28;; add x28, x28, #4diff --git a/tests/disas/winch/aarch64/call_indirect/local_arg.wat b/tests/disas/winch/aarch64/call_indirect/local_arg.watindex 2c7311368e..1675b9bbd6 100644--- a/tests/disas/winch/aarch64/call_indirect/local_arg.wat+++ b/tests/disas/winch/aarch64/call_indirect/local_arg.wat@@ -91,7 +91,7 @@;; mov x0, x9;; mov x1, #0;; ldur w2, [x28]-;; bl #0x408+;; bl #0x440;; 120: add x28, x28, #4;; mov sp, x28;; ldur x9, [x28, #0x14]diff --git a/tests/disas/winch/x64/atomic/notify/notify.wat b/tests/disas/winch/x64/atomic/notify/notify.watindex 9ebac8b87d..98d426f598 100644--- a/tests/disas/winch/x64/atomic/notify/notify.wat+++ b/tests/disas/winch/x64/atomic/notify/notify.wat@@ -27,7 +27,7 @@;; movl $0, %esi;; movq 8(%rsp), %rdx;; movl 4(%rsp), %ecx-;; callq 0x16e+;; callq 0x183;; addq $4, %rsp;; addq $0xc, %rsp;; movq 8(%rsp), %r14diff --git a/tests/disas/winch/x64/atomic/notify/notify_offset.wat b/tests/disas/winch/x64/atomic/notify/notify_offset.watindex 6d3cf71f12..19e54fba98 100644--- a/tests/disas/winch/x64/atomic/notify/notify_offset.wat+++ b/tests/disas/winch/x64/atomic/notify/notify_offset.wat@@ -29,7 +29,7 @@;; movl $0, %esi;; movq 8(%rsp), %rdx;; movl 4(%rsp), %ecx-;; callq 0x17d+;; callq 0x192;; addq $4, %rsp;; addq $0xc, %rsp;; movq 8(%rsp), %r14diff --git a/tests/disas/winch/x64/atomic/wait/wait32.wat b/tests/disas/winch/x64/atomic/wait/wait32.watindex 763669203e..3142b7dfc0 100644--- a/tests/disas/winch/x64/atomic/wait/wait32.wat+++ b/tests/disas/winch/x64/atomic/wait/wait32.wat@@ -30,7 +30,7 @@;; movq 0x18(%rsp), %rdx;; movl 0x14(%rsp), %ecx;; movq 0xc(%rsp), %r8-;; callq 0x17b+;; callq 0x190;; addq $0xc, %rsp;; addq $0x14, %rsp;; movq 8(%rsp), %r14diff --git a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat b/tests/disas/winch/x64/atomic/wait/wait32_offset.watindex ce8835b2ba..97c31b9c88 100644--- a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat+++ b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat@@ -35,7 +35,7 @@;; movq 0x18(%rsp), %rdx;; movl 0x14(%rsp), %ecx;; movq 0xc(%rsp), %r8-;; callq 0x18a+;; callq 0x19f;; addq $0xc, %rsp;; addq $0x14, %rsp;; movq 8(%rsp), %r14diff --git a/tests/disas/winch/x64/atomic/wait/wait64.wat b/tests/disas/winch/x64/atomic/wait/wait64.watindex 5fa37e700b..f9d488d6d5 100644--- a/tests/disas/winch/x64/atomic/wait/wait64.wat+++ b/tests/disas/winch/x64/atomic/wait/wait64.wat@@ -29,7 +29,7 @@;; movq 0x18(%rsp), %rdx;; movq 0x10(%rsp), %rcx;; movq 8(%rsp), %r8-;; callq 0x173+;; callq 0x188;; addq $8, %rsp;; addq $0x18, %rsp;; movq 8(%rsp), %r14diff --git a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat b/tests/disas/winch/x64/atomic/wait/wait64_offset.watindex c09815702a..584073f39c 100644--- a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat+++ b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat@@ -34,7 +34,7 @@;; movq 0x18(%rsp), %rdx;; movq 0x10(%rsp), %rcx;; movq 8(%rsp), %r8-;; callq 0x182+;; callq 0x197;; addq $8, %rsp;; addq $0x18, %rsp;; movq 8(%rsp), %r14diff --git a/tests/disas/winch/x64/call_indirect/call_indirect.wat b/tests/disas/winch/x64/call_indirect/call_indirect.watindex 5ecf92d14c..17fa2ad96a 100644--- a/tests/disas/winch/x64/call_indirect/call_indirect.wat+++ b/tests/disas/winch/x64/call_indirect/call_indirect.wat@@ -76,7 +76,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl 8(%rsp), %edx-;; callq 0x336+;; callq 0x34b;; addq $8, %rsp;; addq $4, %rsp;; movq 0x1c(%rsp), %r14@@ -128,7 +128,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl 4(%rsp), %edx-;; callq 0x336+;; callq 0x34b;; addq $4, %rsp;; addq $4, %rsp;; movq 0x20(%rsp), %r14diff --git a/tests/disas/winch/x64/call_indirect/local_arg.wat b/tests/disas/winch/x64/call_indirect/local_arg.watindex 595446e979..10858f12f6 100644--- a/tests/disas/winch/x64/call_indirect/local_arg.wat+++ b/tests/disas/winch/x64/call_indirect/local_arg.wat@@ -72,7 +72,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl 8(%rsp), %edx-;; callq 0x31e+;; callq 0x34a;; addq $8, %rsp;; addq $4, %rsp;; movq 0x1c(%rsp), %r14diff --git a/tests/disas/winch/x64/epoch/func.wat b/tests/disas/winch/x64/epoch/func.watindex 9efcc83ad4..061edd761b 100644--- a/tests/disas/winch/x64/epoch/func.wat+++ b/tests/disas/winch/x64/epoch/func.wat@@ -23,7 +23,7 @@;; cmpq %rcx, %rdx;; jb 0x54;; 47: movq %r14, %rdi-;; callq 0x134+;; callq 0x14a;; movq 8(%rsp), %r14;; addq $0x10, %rsp;; popq %rbpdiff --git a/tests/disas/winch/x64/epoch/loop.wat b/tests/disas/winch/x64/epoch/loop.watindex 23baa77a5d..3701d1ed48 100644--- a/tests/disas/winch/x64/epoch/loop.wat+++ b/tests/disas/winch/x64/epoch/loop.wat@@ -25,7 +25,7 @@;; cmpq %rcx, %rdx;; jb 0x54;; 47: movq %r14, %rdi-;; callq 0x15e+;; callq 0x174;; movq 8(%rsp), %r14;; movq 0x18(%r14), %rdx;; movq (%rdx), %rdx@@ -34,7 +34,7 @@;; cmpq %rcx, %rdx;; jb 0x79;; 6c: movq %r14, %rdi-;; callq 0x15e+;; callq 0x174;; movq 8(%rsp), %r14;; jmp 0x54;; 7e: addq $0x10, %rspdiff --git a/tests/disas/winch/x64/fuel/call.wat b/tests/disas/winch/x64/fuel/call.watindex 4e35be2aa3..6ca0934728 100644--- a/tests/disas/winch/x64/fuel/call.wat+++ b/tests/disas/winch/x64/fuel/call.wat@@ -28,7 +28,7 @@;; cmpq $0, %rcx;; jl 0x5e;; 51: movq %r14, %rdi-;; callq 0x1ee+;; callq 0x204;; movq 8(%rsp), %r14;; movq 8(%r14), %rax;; movq (%rax), %r11@@ -74,7 +74,7 @@;; cmpq $0, %rcx;; jl 0x10e;; 101: movq %r14, %rdi-;; callq 0x1ee+;; callq 0x204;; movq 8(%rsp), %r14;; addq $0x10, %rsp;; popq %rbpdiff --git a/tests/disas/winch/x64/fuel/func.wat b/tests/disas/winch/x64/fuel/func.watindex b239b13708..cefba99a79 100644--- a/tests/disas/winch/x64/fuel/func.wat+++ b/tests/disas/winch/x64/fuel/func.wat@@ -24,7 +24,7 @@;; cmpq $0, %rcx;; jl 0x5e;; 51: movq %r14, %rdi-;; callq 0x13e+;; callq 0x154;; movq 8(%rsp), %r14;; addq $0x10, %rsp;; popq %rbpdiff --git a/tests/disas/winch/x64/fuel/loop.wat b/tests/disas/winch/x64/fuel/loop.watindex 6d3bd20c27..fcb24042b2 100644--- a/tests/disas/winch/x64/fuel/loop.wat+++ b/tests/disas/winch/x64/fuel/loop.wat@@ -26,14 +26,14 @@;; cmpq $0, %rcx;; jl 0x5e;; 51: movq %r14, %rdi-;; callq 0x172+;; callq 0x188;; movq 8(%rsp), %r14;; movq 8(%r14), %rcx;; movq (%rcx), %rcx;; cmpq $0, %rcx;; jl 0x7c;; 6f: movq %r14, %rdi-;; callq 0x172+;; callq 0x188;; movq 8(%rsp), %r14;; movq 8(%r14), %rax;; movq (%rax), %r11diff --git a/tests/disas/winch/x64/load/grow_load.wat b/tests/disas/winch/x64/load/grow_load.watindex a11c2c6db3..500e3caa24 100644--- a/tests/disas/winch/x64/load/grow_load.wat+++ b/tests/disas/winch/x64/load/grow_load.wat@@ -65,7 +65,7 @@;; movq %r14, %rdi;; movl 0xc(%rsp), %esi;; movl $0, %edx-;; callq 0x2ec+;; callq 0x30a;; addq $0xc, %rsp;; addq $4, %rsp;; movq 0x58(%rsp), %r14diff --git a/tests/disas/winch/x64/ref/func.wat b/tests/disas/winch/x64/ref/func.watindex 0bcad41049..49f72b56b7 100644--- a/tests/disas/winch/x64/ref/func.wat+++ b/tests/disas/winch/x64/ref/func.wat@@ -39,7 +39,7 @@;; movq %rsi, (%rsp);; movq %r14, %rdi;; movl $0, %esi-;; callq 0x252+;; callq 0x27d;; movq 8(%rsp), %r14;; addq $0x10, %rsp;; popq %rbpdiff --git a/tests/disas/winch/x64/table/fill.wat b/tests/disas/winch/x64/table/fill.watindex e944e0e58c..b971e7a536 100644--- a/tests/disas/winch/x64/table/fill.wat+++ b/tests/disas/winch/x64/table/fill.wat@@ -113,7 +113,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl 0xc(%rsp), %edx-;; callq 0x51d+;; callq 0x575;; addq $0xc, %rsp;; addq $4, %rsp;; movq 0x28(%rsp), %r14diff --git a/tests/disas/winch/x64/table/get.wat b/tests/disas/winch/x64/table/get.watindex 12d90ebd88..20c466bc5e 100644--- a/tests/disas/winch/x64/table/get.wat+++ b/tests/disas/winch/x64/table/get.wat@@ -65,7 +65,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl 0xc(%rsp), %edx-;; callq 0x2e6+;; callq 0x311;; addq $0xc, %rsp;; addq $4, %rsp;; movq 0x18(%rsp), %r14diff --git a/tests/disas/winch/x64/table/grow.wat b/tests/disas/winch/x64/table/grow.watindex 0ae72c0112..b92e861112 100644--- a/tests/disas/winch/x64/table/grow.wat+++ b/tests/disas/winch/x64/table/grow.wat@@ -34,7 +34,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl (%rsp), %edx-;; callq 0x20f+;; callq 0x224;; addq $4, %rsp;; movq 0x24(%rsp), %r14;; movl (%rsp), %ecxdiff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.watindex ab1e576d67..f50465b2e8 100644--- a/tests/disas/winch/x64/table/init_copy_drop.wat+++ b/tests/disas/winch/x64/table/init_copy_drop.wat@@ -138,13 +138,13 @@;; movq %rsi, (%rsp);; movq %r14, %rdi;; movl $0, %esi-;; callq 0x10bf+;; callq 0x1153;; movq 8(%rsp), %r14;; pushq %rax;; subq $8, %rsp;; movq %r14, %rdi;; movl $0, %esi-;; callq 0x10ea+;; callq 0x117e;; addq $8, %rsp;; movq 0x10(%rsp), %r14;; movq %r14, %r11@@ -190,17 +190,17 @@;; jmp 0x1f4;; 256: movq %r14, %rdi;; movl $0, %esi-;; callq 0x1115+;; callq 0x11a9;; movq 8(%rsp), %r14;; movq %r14, %rdi;; movl $1, %esi-;; callq 0x10bf+;; callq 0x1153;; movq 8(%rsp), %r14;; pushq %rax;; subq $8, %rsp;; movq %r14, %rdi;; movl $1, %esi-;; callq 0x10ea+;; callq 0x117e;; addq $8, %rsp;; movq 0x10(%rsp), %r14;; movq %r14, %r11@@ -246,7 +246,7 @@;; jmp 0x2ed;; 34f: movq %r14, %rdi;; movl $1, %esi-;; callq 0x1115+;; callq 0x11a9;; movq 8(%rsp), %r14;; movl $5, %eax;; movl $0xf, %ecx@@ -305,7 +305,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x1184+;; callq 0x1218;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -390,7 +390,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x1184+;; callq 0x1218;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -475,7 +475,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x1184+;; callq 0x1218;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -560,7 +560,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x1184+;; callq 0x1218;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -645,7 +645,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movq 8(%rsp), %rdx-;; callq 0x1184+;; callq 0x1218;; addq $8, %rsp;; addq $8, %rsp;; movq 0x28(%rsp), %r14@@ -756,7 +756,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl 0xc(%rsp), %edx-;; callq 0x1184+;; callq 0x1218;; addq $0xc, %rsp;; addq $4, %rsp;; movq 0x18(%rsp), %r14diff --git a/tests/disas/winch/x64/table/set.wat b/tests/disas/winch/x64/table/set.watindex 9dd27dedaa..562ef0da46 100644--- a/tests/disas/winch/x64/table/set.wat+++ b/tests/disas/winch/x64/table/set.wat@@ -109,7 +109,7 @@;; movq %r14, %rdi;; movl $0, %esi;; movl 8(%rsp), %edx-;; callq 0x4b7+;; callq 0x4f9;; addq $8, %rsp;; addq $4, %rsp;; movq 0x1c(%rsp), %r14diff --git a/tests/disas/x64-entry-trampoline.wat b/tests/disas/x64-entry-trampoline.watindex eebc2bd415..ce6de4a91b 100644--- a/tests/disas/x64-entry-trampoline.wat+++ b/tests/disas/x64-entry-trampoline.wat@@ -7,39 +7,42 @@;; wasm[0]::array_to_wasm_trampoline[0]:;; pushq %rbp;; movq %rsp, %rbp-;; subq $0x30, %rsp-;; movq %rbx, (%rsp)-;; movq %r12, 8(%rsp)-;; movq %r13, 0x10(%rsp)-;; movq %r14, 0x18(%rsp)-;; movq %r15, 0x20(%rsp)+;; subq $0x40, %rsp+;; movq %rbx, 0x10(%rsp)+;; movq %r12, 0x18(%rsp)+;; movq %r13, 0x20(%rsp)+;; movq %r14, 0x28(%rsp)+;; movq %r15, 0x30(%rsp);; movq %rbp, %rcx-;; movq 8(%rdi), %rax-;; movq %rcx, 0x48(%rax)+;; movq 8(%rdi), %r8+;; movq %rcx, 0x48(%r8);; movq %rsp, %rcx-;; movq %rcx, 0x40(%rax)-;; leaq 0x2f(%rip), %rcx-;; movq %rcx, 0x50(%rax)+;; movq %rcx, 0x40(%r8)+;; leaq 0x34(%rip), %rcx+;; movq %rcx, 0x50(%r8)+;; movq %r8, (%rsp);; callq 0-;; ├─╼ exception frame offset: SP = FP - 0x30-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x71+;; ├─╼ exception frame offset: SP = FP - 0x40+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x77;; movl $1, %eax-;; movq (%rsp), %rbx-;; movq 8(%rsp), %r12-;; movq 0x10(%rsp), %r13-;; movq 0x18(%rsp), %r14-;; movq 0x20(%rsp), %r15-;; addq $0x30, %rsp+;; movq 0x10(%rsp), %rbx+;; movq 0x18(%rsp), %r12+;; movq 0x20(%rsp), %r13+;; movq 0x28(%rsp), %r14+;; movq 0x30(%rsp), %r15+;; addq $0x40, %rsp;; movq %rbp, %rsp;; popq %rbp;; retq-;; 71: xorl %eax, %eax-;; 73: movq (%rsp), %rbx-;; 77: movq 8(%rsp), %r12-;; 7c: movq 0x10(%rsp), %r13-;; 81: movq 0x18(%rsp), %r14-;; 86: movq 0x20(%rsp), %r15-;; 8b: addq $0x30, %rsp-;; 8f: movq %rbp, %rsp-;; 92: popq %rbp-;; 93: retq+;; 77: movq (%rsp), %r8+;; 7b: movq $1, 0x88(%r8)+;; 86: xorl %eax, %eax+;; 88: movq 0x10(%rsp), %rbx+;; 8d: movq 0x18(%rsp), %r12+;; 92: movq 0x20(%rsp), %r13+;; 97: movq 0x28(%rsp), %r14+;; 9c: movq 0x30(%rsp), %r15+;; a1: addq $0x40, %rsp+;; a5: movq %rbp, %rsp+;; a8: popq %rbp+;; a9: retqdiff --git a/tests/disas/x64/call-indirect-with-gc.wat b/tests/disas/x64/call-indirect-with-gc.watindex aee03e606d..d053980e32 100644--- a/tests/disas/x64/call-indirect-with-gc.wat+++ b/tests/disas/x64/call-indirect-with-gc.wat@@ -53,7 +53,7 @@;; 8b: xorl %esi, %esi;; 8d: movq %r9, %rbx;; 90: movq %rbx, %rdi-;; 93: callq 0x219+;; 93: callq 0x22f;; 98: jmp 0x59;; 9d: ud2;; 9f: ud2diff --git a/tests/disas/x64/call-indirect-without-gc.wat b/tests/disas/x64/call-indirect-without-gc.watindex ab06cfedcb..98c559ff2b 100644--- a/tests/disas/x64/call-indirect-without-gc.wat+++ b/tests/disas/x64/call-indirect-without-gc.wat@@ -53,7 +53,7 @@;; 8b: xorl %esi, %esi;; 8d: movq %r9, %rbx;; 90: movq %rbx, %rdi-;; 93: callq 0x219+;; 93: callq 0x22f;; 98: jmp 0x59;; 9d: ud2;; 9f: ud2diff --git a/tests/misc_testsuite/component-model/async/sync-call-context-slots.wast b/tests/misc_testsuite/component-model/async/sync-call-context-slots.wastnew file mode 100644index 0000000000..54a41f1c23--- /dev/null+++ b/tests/misc_testsuite/component-model/async/sync-call-context-slots.wast@@ -0,0 +1,152 @@+;;! component_model_async = true+;;! component_model_more_async_builtins = true+;;! component_model_threading = true++;; Like sync-call-context.wast, but drives *both* component-context slots (0 and+;; 1) at once.++;; Both slots, inline fast path.+(component+ (component $A+ (core func $get0 (canon context.get i32 0))+ (core func $set0 (canon context.set i32 0))+ (core func $get1 (canon context.get i32 1))+ (core func $set1 (canon context.set i32 1))+ (core module $M+ (import "" "get0" (func $get0 (result i32)))+ (import "" "set0" (func $set0 (param i32)))+ (import "" "get1" (func $get1 (result i32)))+ (import "" "set1" (func $set1 (param i32)))+ (func (export "f'") (param i32) (result i32)+ ;; Fresh thread: both slots zero.+ (if (i32.ne (call $get0) (i32.const 0)) (then unreachable))+ (if (i32.ne (call $get1) (i32.const 0)) (then unreachable))+ (call $set0 (i32.const 0xAAAA1111))+ (call $set1 (i32.const 0xBBBB2222))+ ;; The two slots must not alias each other.+ (if (i32.ne (call $get0) (i32.const 0xAAAA1111)) (then unreachable))+ (if (i32.ne (call $get1) (i32.const 0xBBBB2222)) (then unreachable))+ (i32.add (local.get 0) (i32.const 42))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "get0" (func $get0)) (export "set0" (func $set0))+ (export "get1" (func $get1)) (export "set1" (func $set1))+ ))))+ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'")))+ )++ (component $B+ (import "f" (func $f (param "x" u32) (result u32)))+ (core func $f' (canon lower (func $f)))+ (core func $get0 (canon context.get i32 0))+ (core func $set0 (canon context.set i32 0))+ (core func $get1 (canon context.get i32 1))+ (core func $set1 (canon context.set i32 1))+ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (import "" "get0" (func $get0 (result i32)))+ (import "" "set0" (func $set0 (param i32)))+ (import "" "get1" (func $get1 (result i32)))+ (import "" "set1" (func $set1 (param i32)))+ (func (export "g'") (result i32) (local $r i32)+ (call $set0 (i32.const 0x11110000))+ (call $set1 (i32.const 0x22220000))+ (local.set $r (call $f' (i32.const 1234)))+ ;; Both of our slots restored, independently and unswapped.+ (if (i32.ne (call $get0) (i32.const 0x11110000)) (then unreachable))+ (if (i32.ne (call $get1) (i32.const 0x22220000)) (then unreachable))+ (local.get $r)+ )+ )+ (core instance $n (instantiate $N (with "" (instance+ (export "f'" (func $f'))+ (export "get0" (func $get0)) (export "set0" (func $set0))+ (export "get1" (func $get1)) (export "set1" (func $set1))+ ))))+ (func (export "g") (result u32)+ (canon lift (core func $n "g'")))+ )++ (instance $a (instantiate $A))+ (instance $b (instantiate $B (with "f" (func $a "f"))))+ (export "g" (func $b "g"))+)+(assert_return (invoke "g") (u32.const 1276))++;; Both slots, forced slow, out-of-line path.+(component+ (component $A+ (core func $get0 (canon context.get i32 0))+ (core func $set0 (canon context.set i32 0))+ (core func $get1 (canon context.get i32 1))+ (core func $set1 (canon context.set i32 1))+ (core func $bpinc (canon backpressure.inc))+ (core func $bpdec (canon backpressure.dec))+ (core module $M+ (import "" "get0" (func $get0 (result i32)))+ (import "" "set0" (func $set0 (param i32)))+ (import "" "get1" (func $get1 (result i32)))+ (import "" "set1" (func $set1 (param i32)))+ (import "" "backpressure.inc" (func $bpinc))+ (import "" "backpressure.dec" (func $bpdec))+ (func (export "f'") (param i32) (result i32)+ (if (i32.ne (call $get0) (i32.const 0)) (then unreachable))+ (if (i32.ne (call $get1) (i32.const 0)) (then unreachable))+ (call $set0 (i32.const 0xAAAA1111))+ (call $set1 (i32.const 0xBBBB2222))+ ;; Call `backptressure.{inc,dec}` to force lazy task creation.+ (call $bpinc) (call $bpdec)+ ;; Both slots survive the force.+ (if (i32.ne (call $get0) (i32.const 0xAAAA1111)) (then unreachable))+ (if (i32.ne (call $get1) (i32.const 0xBBBB2222)) (then unreachable))+ (i32.add (local.get 0) (i32.const 42))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "get0" (func $get0)) (export "set0" (func $set0))+ (export "get1" (func $get1)) (export "set1" (func $set1))+ (export "backpressure.inc" (func $bpinc))+ (export "backpressure.dec" (func $bpdec))+ ))))+ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'")))+ )++ (component $B+ (import "f" (func $f (param "x" u32) (result u32)))+ (core func $f' (canon lower (func $f)))+ (core func $get0 (canon context.get i32 0))+ (core func $set0 (canon context.set i32 0))+ (core func $get1 (canon context.get i32 1))+ (core func $set1 (canon context.set i32 1))+ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (import "" "get0" (func $get0 (result i32)))+ (import "" "set0" (func $set0 (param i32)))+ (import "" "get1" (func $get1 (result i32)))+ (import "" "set1" (func $set1 (param i32)))+ (func (export "g'") (result i32) (local $r i32)+ (call $set0 (i32.const 0x11110000))+ (call $set1 (i32.const 0x22220000))+ (local.set $r (call $f' (i32.const 1234)))+ (if (i32.ne (call $get0) (i32.const 0x11110000)) (then unreachable))+ (if (i32.ne (call $get1) (i32.const 0x22220000)) (then unreachable))+ (local.get $r)+ )+ )+ (core instance $n (instantiate $N (with "" (instance+ (export "f'" (func $f'))+ (export "get0" (func $get0)) (export "set0" (func $set0))+ (export "get1" (func $get1)) (export "set1" (func $set1))+ ))))+ (func (export "g") (result u32)+ (canon lift (core func $n "g'")))+ )++ (instance $a (instantiate $A))+ (instance $b (instantiate $B (with "f" (func $a "f"))))+ (export "g" (func $b "g"))+)+(assert_return (invoke "g") (u32.const 1276))diff --git a/tests/misc_testsuite/component-model/async/sync-call-context-trap.wast b/tests/misc_testsuite/component-model/async/sync-call-context-trap.wastnew file mode 100644index 0000000000..0e1957c448--- /dev/null+++ b/tests/misc_testsuite/component-model/async/sync-call-context-trap.wast@@ -0,0 +1,47 @@+;;! component_model_async = true+;;! component_model_more_async_builtins = true++;; Test a guest-to-guest sync call whose callee traps. This should exercise+;; deferred task construction/teardown in the face of traps.+(component+ (component $A+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "context.set" (func $cset (param i32)))+ (func (export "f'") (param i32) (result i32)+ (call $cset (i32.const 0x5678))+ unreachable+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "context.set" (func $cset))+ ))))+ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'")))+ )++ (component $B+ (import "f" (func $f (param "x" u32) (result u32)))+ (core func $f' (canon lower (func $f)))+ (core func $cset (canon context.set i32 0))+ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "g'") (result i32)+ (call $cset (i32.const 0x1234))+ (call $f' (i32.const 1234))+ )+ )+ (core instance $n (instantiate $N (with "" (instance+ (export "f'" (func $f'))+ (export "context.set" (func $cset))+ ))))+ (func (export "g") (result u32)+ (canon lift (core func $n "g'")))+ )++ (instance $a (instantiate $A))+ (instance $b (instantiate $B (with "f" (func $a "f"))))+ (export "g" (func $b "g"))+)+(assert_trap (invoke "g") "wasm `unreachable` instruction executed")diff --git a/tests/misc_testsuite/component-model/async/sync-call-context.wast b/tests/misc_testsuite/component-model/async/sync-call-context.wastnew file mode 100644index 0000000000..79f7d50c72--- /dev/null+++ b/tests/misc_testsuite/component-model/async/sync-call-context.wast@@ -0,0 +1,544 @@+;;! component_model_async = true+;;! component_model_more_async_builtins = true++;; Tests for guest-to-guest, sync-to-sync calls and their inline fast paths.++;; Inline fast path, no task forcing.+(component+ (component $A+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "f'") (param i32) (result i32)+ ;; A is a freshly-entered (deferred) thread: its context starts at 0.+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x5678))+ (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable))+ (i32.add (local.get 0) (i32.const 42))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'")))+ )++ (component $B+ (import "f" (func $f (param "x" u32) (result u32)))+ (core func $f' (canon lower (func $f)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "g'") (result i32) (local $r i32)+ (call $cset (i32.const 0x1234))+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))+ (local.set $r (call $f' (i32.const 1234)))+ ;; The callee's context mutation must NOT leak: ours is restored.+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))+ (local.get $r)+ )+ )+ (core instance $n (instantiate $N (with "" (instance+ (export "f'" (func $f'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "g") (result u32)+ (canon lift (core func $n "g'")))+ )++ (instance $a (instantiate $A))+ (instance $b (instantiate $B (with "f" (func $a "f"))))+ (export "g" (func $b "g"))+)+(assert_return (invoke "g") (u32.const 1276))++;; Single guest-to-guest sync call, forced slow path.+(component+ (component $A+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core func $bpinc (canon backpressure.inc))+ (core func $bpdec (canon backpressure.dec))+ (core module $M+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (import "" "backpressure.inc" (func $bpinc))+ (import "" "backpressure.dec" (func $bpdec))+ (func (export "f'") (param i32) (result i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x5678))+ ;; Force promotion of the deferred thread mid-call.+ (call $bpinc)+ (call $bpdec)+ ;; Our context must survive the force.+ (if (i32.ne (call $cget) (i32.const 0x5678)) (then unreachable))+ (i32.add (local.get 0) (i32.const 42))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ (export "backpressure.inc" (func $bpinc))+ (export "backpressure.dec" (func $bpdec))+ ))))+ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'")))+ )++ (component $B+ (import "f" (func $f (param "x" u32) (result u32)))+ (core func $f' (canon lower (func $f)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "g'") (result i32) (local $r i32)+ (call $cset (i32.const 0x1234))+ (local.set $r (call $f' (i32.const 1234)))+ ;; Restored even though the callee forced the slow exit path.+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))+ (local.get $r)+ )+ )+ (core instance $n (instantiate $N (with "" (instance+ (export "f'" (func $f'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "g") (result u32)+ (canon lift (core func $n "g'")))+ )++ (instance $a (instantiate $A))+ (instance $b (instantiate $B (with "f" (func $a "f"))))+ (export "g" (func $b "g"))+)+(assert_return (invoke "g") (u32.const 1276))++;; Nested A->B->C sync-call chain, fast path.+(component+ (component $Leaf+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "leaf'") (param i32) (result i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x99aabbcc))+ (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable))+ (i32.add (local.get 0) (i32.const 1))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "leaf") (param "x" u32) (result u32)+ (canon lift (core func $m "leaf'")))+ )++ (component $Mid+ (import "leaf" (func $leaf (param "x" u32) (result u32)))+ (core func $leaf' (canon lower (func $leaf)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "mid'") (param i32) (result i32) (local $r i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x55667788))+ (local.set $r (call $leaf' (local.get 0)))+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))+ (i32.add (local.get $r) (i32.const 10))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "leaf'" (func $leaf'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "mid") (param "x" u32) (result u32)+ (canon lift (core func $m "mid'")))+ )++ (component $Root+ (import "mid" (func $mid (param "x" u32) (result u32)))+ (core func $mid' (canon lower (func $mid)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "mid'" (func $mid' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "root'") (result i32) (local $r i32)+ (call $cset (i32.const 0x11223344))+ (local.set $r (call $mid' (i32.const 100)))+ (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable))+ (i32.add (local.get $r) (i32.const 1000))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "mid'" (func $mid'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "root") (result u32)+ (canon lift (core func $m "root'")))+ )++ (instance $leaf (instantiate $Leaf))+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))+ (export "root" (func $root "root"))+)+(assert_return (invoke "root") (u32.const 1111))++;; Nested A->B->C call chain, forced at the deepest level.+(component+ (component $Leaf+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core func $bpinc (canon backpressure.inc))+ (core func $bpdec (canon backpressure.dec))+ (core module $M+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (import "" "backpressure.inc" (func $bpinc))+ (import "" "backpressure.dec" (func $bpdec))+ (func (export "leaf'") (param i32) (result i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x99aabbcc))+ ;; Force promotion.+ (call $bpinc)+ (call $bpdec)+ (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable))+ (i32.add (local.get 0) (i32.const 1))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ (export "backpressure.inc" (func $bpinc))+ (export "backpressure.dec" (func $bpdec))+ ))))+ (func (export "leaf") (param "x" u32) (result u32)+ (canon lift (core func $m "leaf'")))+ )++ (component $Mid+ (import "leaf" (func $leaf (param "x" u32) (result u32)))+ (core func $leaf' (canon lower (func $leaf)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "mid'") (param i32) (result i32) (local $r i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x55667788))+ (local.set $r (call $leaf' (local.get 0)))+ ;; Restored after the (forced) nested call.+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))+ (i32.add (local.get $r) (i32.const 10))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "leaf'" (func $leaf'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "mid") (param "x" u32) (result u32)+ (canon lift (core func $m "mid'")))+ )++ (component $Root+ (import "mid" (func $mid (param "x" u32) (result u32)))+ (core func $mid' (canon lower (func $mid)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "mid'" (func $mid' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "root'") (result i32) (local $r i32)+ (call $cset (i32.const 0x11223344))+ (local.set $r (call $mid' (i32.const 100)))+ (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable))+ (i32.add (local.get $r) (i32.const 1000))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "mid'" (func $mid'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "root") (result u32)+ (canon lift (core func $m "root'")))+ )++ (instance $leaf (instantiate $Leaf))+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))+ (export "root" (func $root "root"))+)+(assert_return (invoke "root") (u32.const 1111))++;; Forcing promotion at an intermediate level, then a deeper sync call.+(component+ (component $Leaf+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "leaf'") (param i32) (result i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x99aabbcc))+ (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable))+ (i32.add (local.get 0) (i32.const 1))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "leaf") (param "x" u32) (result u32)+ (canon lift (core func $m "leaf'")))+ )++ (component $Mid+ (import "leaf" (func $leaf (param "x" u32) (result u32)))+ (core func $leaf' (canon lower (func $leaf)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core func $bpinc (canon backpressure.inc))+ (core func $bpdec (canon backpressure.dec))+ (core module $M+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (import "" "backpressure.inc" (func $bpinc))+ (import "" "backpressure.dec" (func $bpdec))+ (func (export "mid'") (param i32) (result i32) (local $r i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x55667788))+ ;; Force *before* descending into the leaf.+ (call $bpinc)+ (call $bpdec)+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))+ (local.set $r (call $leaf' (local.get 0)))+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))+ (i32.add (local.get $r) (i32.const 10))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "leaf'" (func $leaf'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ (export "backpressure.inc" (func $bpinc))+ (export "backpressure.dec" (func $bpdec))+ ))))+ (func (export "mid") (param "x" u32) (result u32)+ (canon lift (core func $m "mid'")))+ )++ (component $Root+ (import "mid" (func $mid (param "x" u32) (result u32)))+ (core func $mid' (canon lower (func $mid)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "mid'" (func $mid' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "root'") (result i32) (local $r i32)+ (call $cset (i32.const 0x11223344))+ (local.set $r (call $mid' (i32.const 100)))+ (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable))+ (i32.add (local.get $r) (i32.const 1000))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "mid'" (func $mid'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "root") (result u32)+ (canon lift (core func $m "root'")))+ )++ (instance $leaf (instantiate $Leaf))+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))+ (export "root" (func $root "root"))+)+(assert_return (invoke "root") (u32.const 1111))++;; Repeated sync calls from the same caller.+(component+ (component $A+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "f'") (param i32) (result i32)+ ;; Each fresh entry must zero the context regardless of prior calls.+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.add (local.get 0) (i32.const 0x10000)))+ (i32.add (local.get 0) (i32.const 42))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "f") (param "x" u32) (result u32)+ (canon lift (core func $m "f'")))+ )++ (component $B+ (import "f" (func $f (param "x" u32) (result u32)))+ (core func $f' (canon lower (func $f)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $N+ (import "" "f'" (func $f' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "g'") (result i32) (local $r i32)+ (call $cset (i32.const 0x1234))+ (local.set $r (call $f' (i32.const 1)))+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))+ (local.set $r (i32.add (local.get $r) (call $f' (i32.const 2))))+ ;; Still restored after the second call.+ (if (i32.ne (call $cget) (i32.const 0x1234)) (then unreachable))+ (local.get $r)+ )+ )+ (core instance $n (instantiate $N (with "" (instance+ (export "f'" (func $f'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "g") (result u32)+ (canon lift (core func $n "g'")))+ )++ (instance $a (instantiate $A))+ (instance $b (instantiate $B (with "f" (func $a "f"))))+ (export "g" (func $b "g"))+)+;; (1 + 42) + (2 + 42) = 87+(assert_return (invoke "g") (u32.const 87))++;; Nested chain forced at *two* levels (re-forcing).+(component+ (component $Leaf+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core func $bpinc (canon backpressure.inc))+ (core func $bpdec (canon backpressure.dec))+ (core module $M+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (import "" "backpressure.inc" (func $bpinc))+ (import "" "backpressure.dec" (func $bpdec))+ (func (export "leaf'") (param i32) (result i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x99aabbcc))+ ;; Force promotion.+ (call $bpinc) (call $bpdec)+ (if (i32.ne (call $cget) (i32.const 0x99aabbcc)) (then unreachable))+ (i32.add (local.get 0) (i32.const 1))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ (export "backpressure.inc" (func $bpinc))+ (export "backpressure.dec" (func $bpdec))+ ))))+ (func (export "leaf") (param "x" u32) (result u32)+ (canon lift (core func $m "leaf'")))+ )++ (component $Mid+ (import "leaf" (func $leaf (param "x" u32) (result u32)))+ (core func $leaf' (canon lower (func $leaf)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core func $bpinc (canon backpressure.inc))+ (core func $bpdec (canon backpressure.dec))+ (core module $M+ (import "" "leaf'" (func $leaf' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (import "" "backpressure.inc" (func $bpinc))+ (import "" "backpressure.dec" (func $bpdec))+ (func (export "mid'") (param i32) (result i32) (local $r i32)+ (if (i32.ne (call $cget) (i32.const 0)) (then unreachable))+ (call $cset (i32.const 0x55667788))+ ;; Force promotion.+ (call $bpinc) (call $bpdec)+ (local.set $r (call $leaf' (local.get 0)))+ (if (i32.ne (call $cget) (i32.const 0x55667788)) (then unreachable))+ (i32.add (local.get $r) (i32.const 10))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "leaf'" (func $leaf'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ (export "backpressure.inc" (func $bpinc))+ (export "backpressure.dec" (func $bpdec))+ ))))+ (func (export "mid") (param "x" u32) (result u32)+ (canon lift (core func $m "mid'")))+ )++ (component $Root+ (import "mid" (func $mid (param "x" u32) (result u32)))+ (core func $mid' (canon lower (func $mid)))+ (core func $cget (canon context.get i32 0))+ (core func $cset (canon context.set i32 0))+ (core module $M+ (import "" "mid'" (func $mid' (param i32) (result i32)))+ (import "" "context.get" (func $cget (result i32)))+ (import "" "context.set" (func $cset (param i32)))+ (func (export "root'") (result i32) (local $r i32)+ (call $cset (i32.const 0x11223344))+ (local.set $r (call $mid' (i32.const 100)))+ (if (i32.ne (call $cget) (i32.const 0x11223344)) (then unreachable))+ (i32.add (local.get $r) (i32.const 1000))+ )+ )+ (core instance $m (instantiate $M (with "" (instance+ (export "mid'" (func $mid'))+ (export "context.get" (func $cget))+ (export "context.set" (func $cset))+ ))))+ (func (export "root") (result u32)+ (canon lift (core func $m "root'")))+ )++ (instance $leaf (instantiate $Leaf))+ (instance $mid (instantiate $Mid (with "leaf" (func $leaf "leaf"))))+ (instance $root (instantiate $Root (with "mid" (func $mid "mid"))))+ (export "root" (func $root "root"))+)+(assert_return (invoke "root") (u32.const 1111))diff --git a/tests/misc_testsuite/component-model/trap.wast b/tests/misc_testsuite/component-model/trap.wastnew file mode 100644index 0000000000..9653efe031--- /dev/null+++ b/tests/misc_testsuite/component-model/trap.wast@@ -0,0 +1,30 @@+;;! component_model_async = true++(component+ (component $Inner+ (core module $M+ (func (export "f") (result i32) unreachable)+ )+ (core instance $m (instantiate $M))+ (func (export "f") (result u32) (canon lift (core func $m "f")))+ )++ (component $Outer+ (import "f" (func $f (result u32)))+ (core func $f-lowered (canon lower (func $f)))+ (core module $N+ (import "" "f" (func $f (result i32)))+ (func (export "g") (result i32) (call $f))+ )+ (core instance $n (instantiate $N (with "" (instance+ (export "f" (func $f-lowered))+ ))))+ (func (export "g") (result u32) (canon lift (core func $n "g")))+ )++ (instance $inner (instantiate $Inner))+ (instance $outer (instantiate $Outer (with "f" (func $inner "f"))))+ (export "g" (func $outer "g"))+)++(assert_trap (invoke "g") "wasm `unreachable` instruction executed")