Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Do not force the `VMDeferredThread` for borrows (#15034)

Nick Fitzgerald0147096be6db
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 9935de01b1..3870a540ea 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -12,6 +12,7 @@
#![cfg_attr(docsrs, feature(doc_cfg))]
#![doc(test(attr(deny(warnings))))]
#![doc(test(attr(allow(dead_code, unused_variables, unused_mut))))]
+#![warn(clippy::cast_possible_truncation)]
extern crate alloc as std_alloc;
diff --git a/crates/core/src/truncate.rs b/crates/core/src/truncate.rs
index 9bbcc27ce1..ab2340dd02 100644
--- a/crates/core/src/truncate.rs
+++ b/crates/core/src/truncate.rs
@@ -1,11 +1,24 @@
//! Explicit methods to clearly indicate that truncation is desired when used.
+#![allow(
+ clippy::cast_possible_truncation,
+ reason = "methods for intentional truncation"
+)]
+
/// Explicitly truncate an `i32` to an `i16`.
+#[inline]
pub fn truncate_i32_to_i16(a: i32) -> i16 {
a as i16
}
/// Explicitly truncate an `i32` to an `i8`.
+#[inline]
pub fn truncate_i32_to_i8(a: i32) -> i8 {
a as i8
}
+
+/// Explicitly truncate a `u64` to an `u32`.
+#[inline]
+pub fn truncate_u64_to_u32(a: u64) -> u32 {
+ a as u32
+}
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index 3bca3dd4da..ffccdd1e37 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -43,8 +43,6 @@ enum VmType {
VMStoreContext,
VMMemoryDefinition,
VMTableDefinition,
- // NB: this is currently only referenced by macro-generated
- // `AliasRegions` helpers that are not all wired up to call sites yet.
#[allow(
dead_code,
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
@@ -86,6 +84,11 @@ enum VmType {
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
)]
VMAsanStackSupport,
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for all VM types via `for_each_vm_type!`"
+ )]
+ VMResourceIndex,
}
/// A key that uniquely identifies an alias region across an entire compilation.
diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs
index c754ff49de..3a861810a0 100644
--- a/crates/cranelift/src/compiler.rs
+++ b/crates/cranelift/src/compiler.rs
@@ -1632,6 +1632,17 @@ impl Compiler {
vm_store_ctx,
forced,
);
+
+ // Likewise, those frames held the only record of where their
+ // lazy lends started. Those lends never touched a `lend_count`, so
+ // discarding them all is exactly what unwinding the frames' exits
+ // would have done.
+ let zero = builder.ins().iconst(pointer_type, 0);
+ alias_regions.vm_store_context().lazy_lenders_len().store(
+ &mut builder.cursor(),
+ vm_store_ctx,
+ zero,
+ );
}
let false_return = builder.ins().iconst(ir::types::I8, 0);
builder.ins().return_(&[false_return]);
diff --git a/crates/cranelift/src/component_sync_call.rs b/crates/cranelift/src/component_sync_call.rs
index 0313255414..30ee841449 100644
--- a/crates/cranelift/src/component_sync_call.rs
+++ b/crates/cranelift/src/component_sync_call.rs
@@ -108,6 +108,18 @@ where
);
}
+ // Record where this frame's lazy lends will start, so that `exit` can
+ // discard them and the host can find exactly this frame's entries if it
+ // has to promote them into real lends.
+ let lazy_lenders_len = alias_regions
+ .vm_store_context()
+ .lazy_lenders_len()
+ .load(&mut builder.cursor(), vmstore);
+ alias_regions
+ .vm_deferred_thread()
+ .lazy_lenders_start()
+ .store(&mut builder.cursor(), slot_addr, lazy_lenders_len);
+
// Publish the deferred thread as the store's current thread.
alias_regions.vm_store_context().current_thread().store(
&mut builder.cursor(),
@@ -138,11 +150,12 @@ impl SlowExit {
/// the stack slot that `enter` returned.
///
/// If the deferred thread pushed by `enter` is still the store's current
-/// thread, meaning nothing ever forced it into a real one, then it is popped
-/// and the caller's context slots restored inline. Otherwise the host has state
-/// to tear down, which only it can do, so the caller must emit an out-of-line
-/// call to the `exit_sync_call` intrinsic: on return the builder is positioned
-/// in that slow path, which is finished with [`SlowExit::finish`].
+/// thread, meaning nothing ever forced it into a real one, then it is popped,
+/// its lazy lends discarded, and the caller's context slots restored inline.
+/// Otherwise the host has state to tear down, which only it can do, so the
+/// caller must emit an out-of-line call to the `exit_sync_call` intrinsic: on
+/// return the builder is positioned in that slow path, which is finished with
+/// [`SlowExit::finish`].
pub fn exit<O>(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<O>,
@@ -162,14 +175,14 @@ where
.vm_store_context()
.current_thread()
.load(&mut builder.cursor(), vmstore);
- let is_fast = builder.ins().icmp(IntCC::Equal, cur, slot_addr);
+ let is_current = builder.ins().icmp(IntCC::Equal, cur, slot_addr);
let fast_block = builder.create_block();
let slow_block = builder.create_block();
let cont_block = builder.create_block();
builder
.ins()
- .brif(is_fast, fast_block, &[], slow_block, &[]);
+ .brif(is_current, fast_block, &[], slow_block, &[]);
builder.seal_block(fast_block);
builder.seal_block(slow_block);
@@ -183,6 +196,20 @@ where
.vm_store_context()
.current_thread()
.store(&mut builder.cursor(), vmstore, parent);
+
+ // Discard the lazy lends this frame recorded. We can do this by simply
+ // resetting the `lazy_lenders_len`, and without any per-lend work, since
+ // the deferred thread and its lazy lends were never materialized.
+ let start = alias_regions
+ .vm_deferred_thread()
+ .lazy_lenders_start()
+ .load(&mut builder.cursor(), slot_addr);
+ alias_regions.vm_store_context().lazy_lenders_len().store(
+ &mut builder.cursor(),
+ vmstore,
+ start,
+ );
+
for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
let saved = alias_regions
.vm_deferred_thread()
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 7820cd796d..d96e49ed3f 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -16,6 +16,13 @@ use crate::{
/// defined by the component model's `context.{get,set}` intrinsics.
pub const NUM_COMPONENT_CONTEXT_SLOTS: usize = 2;
+/// Capacity of the `lazy_lenders` buffer in the `VMStoreContext`.
+///
+/// Fast paths usually lend only one or two handles, so this is small; a lend
+/// that does not fit forces the deferred thread and records in its
+/// `CallContext` instead.
+pub const LAZY_LENDERS_CAPACITY: usize = 8;
+
#[cfg(target_pointer_width = "32")]
fn cast_to_u32(sz: usize) -> u32 {
u32::try_from(sz).unwrap()
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
index 5885775922..038f769b37 100644
--- a/crates/environ/src/vmtypes.rs
+++ b/crates/environ/src/vmtypes.rs
@@ -505,6 +505,19 @@ macro_rules! for_each_vm_type {
///
/// NB: `UnsafeCell` because JIT code writes to this field.
pub current_thread: UnsafeCell<VMLazyThread>,
+
+ /// Resource lends made under unforced `VMDeferredThread`s and not
+ /// yet reflected in any handle's `lend_count`.
+ ///
+ /// Holds `LAZY_LENDERS_CAPACITY` entries.
+ #[readonly]
+ #[can_move]
+ pub lazy_lenders: VmPtr<VMResourceIndex>,
+
+ /// Number of occupied slots at the front of `lazy_lenders`.
+ ///
+ /// NB: `UnsafeCell` because JIT code writes to this field.
+ pub lazy_lenders_len: UnsafeCell<usize>,
}
/// JIT-visible representation of the store's current thread for the component
@@ -554,6 +567,22 @@ macro_rules! for_each_vm_type {
thread: Option<VmPtr<VMDeferredThread>>,
}
+ /// JIT-visible encoding of a component-model resource handle, as stored in
+ /// the `VMStoreContext::lazy_lenders` buffer.
+ ///
+ /// This is the raw version of `TypedResourceIndex` that is exposed to Wasm
+ /// code; use `VMResourceIndex::{new,get}` to convert.
+ #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+ #[repr(C)]
+ #[snake_name = vm_resource_index]
+ pub struct VMResourceIndex {
+ /// The raw `TypeResourceTableIndex`, or `u32::MAX` for a host resource.
+ pub(crate) ty: u32,
+
+ /// The handle's index within its table.
+ pub(crate) index: u32,
+ }
+
/// A deferred component-model thread.
///
/// This is an on-stack record pushed by a fused sync-to-sync adapter's fast
@@ -571,14 +600,25 @@ macro_rules! for_each_vm_type {
pub struct VMDeferredThread {
/// The previous value of `VMStoreContext::current_thread`.
pub parent: VMLazyThread,
+
/// Whether the callee is async-lifted (a deferred `enter_sync_call` arg).
pub callee_async: u32,
+
/// The callee component instance (a deferred `enter_sync_call` argument).
pub callee_instance: u32,
+
/// The caller thread's `context.{get,set}` slots, saved on entry and
/// restored on the fast-path exit (or recovered while forcing).
#[indexed]
pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
+
+ /// `VMStoreContext::lazy_lenders_len` as of this frame's push, so
+ /// that the lends this frame recorded are everything at or after it.
+ ///
+ /// Frames nest with the call stack, so that is always a contiguous
+ /// suffix by the time this frame exits, and exiting an unforced frame
+ /// discards it by resetting `lazy_lenders_len` back to this.
+ pub lazy_lenders_start: usize,
}
/// This type is used to save (and subsequently restore) a subset of
diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rs
index 960853a81f..bb20bb8139 100644
--- a/crates/wasmtime/src/runtime/component/concurrent.rs
+++ b/crates/wasmtime/src/runtime/component/concurrent.rs
@@ -52,6 +52,7 @@
use self::error_contexts::GlobalErrorContextRefCount;
use crate::component::func::{Func, call_post_return};
+use crate::component::store::ComponentInstanceId;
use crate::component::{
HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
};
@@ -88,6 +89,7 @@ use core::task::{Context, Poll, Waker};
use futures::channel::oneshot;
use futures::stream::{FuturesUnordered, StreamExt};
use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
+use smallvec::SmallVec;
use table::{TableDebug, TableId};
use wasmtime_environ::component::{
CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
@@ -1868,7 +1870,7 @@ impl StoreOpaque {
// Collect the deferred frames pushed inline by fused adapters, walking
// the `parent` chain from innermost to the base.
- let mut frames = Vec::new();
+ let mut frames = SmallVec::<[_; 8]>::new();
let mut cur = *self.vm_store_context_mut().current_thread_mut();
while let Some(ptr) = cur.as_deferred() {
// SAFETY: `ptr` points at a `VMDeferredThread` living in a fused
@@ -1880,6 +1882,7 @@ impl StoreOpaque {
deferred.callee_async != 0,
deferred.callee_instance,
deferred.saved_context,
+ deferred.lazy_lenders_start,
));
cur = deferred.parent;
}
@@ -1896,7 +1899,24 @@ impl StoreOpaque {
// Replay the deferred `enter_guest_sync_call`s outermost-first so that
// the resulting `ConcurrentState` matches what the non-deferred path
// would have otherwise produced.
- for (callee_async, callee_instance, saved_context) in frames.into_iter().rev() {
+ //
+ // A frame's own lazy lends run from its start to that of the frame
+ // pushed after it, or to the end of the buffer for the innermost.
+ let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
+ frames.reverse();
+ let lazy_lenders_base = match frames.first() {
+ Some((_, _, _, start)) => *start,
+ None => bail_bug!("forcing a deferred thread with no deferred frames"),
+ };
+ let lazy_lender_ends = frames
+ .iter()
+ .skip(1)
+ .map(|(_, _, _, start)| *start)
+ .chain([lazy_lenders_end])
+ .collect::<SmallVec<[_; 8]>>();
+ for ((callee_async, callee_instance, saved_context, start), lenders_end) in
+ frames.into_iter().zip(lazy_lender_ends)
+ {
// Restore the caller's context slots so that we save the correct
// values into the caller's thread, exactly as the non-deferred path
// would have on entry.
@@ -1906,8 +1926,20 @@ impl StoreOpaque {
index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
};
self.enter_guest_sync_call(callee_async, callee)?;
+
+ // Now that this frame has a real `GuestTask`, promote any lazy
+ // lends it recorded while deferred.
+ self.promote_current_lazy_lends(id, start, lenders_end)?;
}
+ // Every lazy-lend record was promoted, so the length resets back to
+ // where the outermost frame found it.
+ debug_assert_eq!(
+ *self.vm_store_context_mut().lazy_lenders_len_mut(),
+ lazy_lenders_end
+ );
+ *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
+
// Replaying done; restore the current context.
*self.vm_store_context_mut().component_context_mut() = current_context;
@@ -1916,6 +1948,33 @@ impl StoreOpaque {
.unforced_current_thread)
}
+ /// Promotes the `lazy_lenders` entries in `start..end` into real lends
+ /// owned by the current thread's `CallContext`.
+ ///
+ /// The task's `validate_scope_exit` then undoes them as if they had been
+ /// lent there all along.
+ fn promote_current_lazy_lends(
+ &mut self,
+ id: ComponentInstanceId,
+ start: usize,
+ end: usize,
+ ) -> Result<()> {
+ if start == end {
+ return Ok(());
+ }
+
+ let scope = match self
+ .concurrent_state_mut_without_forcing_current_thread()
+ .unforced_current_thread
+ .guest_task()
+ {
+ Some(task) => Scope::Id(task.rep()),
+ None => bail_bug!("replayed deferred frame is not a guest thread"),
+ };
+ let instance = Instance::from_wasmtime(self, id);
+ self.promote_lazy_lends(instance, scope, start, end)
+ }
+
fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
match self.current_thread()?.guest() {
Some(id) => Ok(*id),
diff --git a/crates/wasmtime/src/runtime/component/func/options.rs b/crates/wasmtime/src/runtime/component/func/options.rs
index e4d706685f..eda17e7e3a 100644
--- a/crates/wasmtime/src/runtime/component/func/options.rs
+++ b/crates/wasmtime/src/runtime/component/func/options.rs
@@ -377,6 +377,16 @@ impl<'a> LiftContext<'a> {
let store_id = store.id();
let hostcall_fuel = store.hostcall_fuel();
let current_scope = store.current_scope()?;
+
+ // Computing the scope forces any deferred thread, so host lifts never
+ // record lazy lends and always see a complete `lend_count`.
+ debug_assert!(
+ !store
+ .vm_store_context_mut()
+ .current_thread_mut()
+ .is_deferred()
+ );
+
// From `&mut StoreOpaque` provided the goal here is to project out
// three different disjoint fields owned by the store: memory,
// `CallContexts`, and `HandleTable`. There's no native API for that
diff --git a/crates/wasmtime/src/runtime/component/instance.rs b/crates/wasmtime/src/runtime/component/instance.rs
index 4713762b08..eb86939d2c 100644
--- a/crates/wasmtime/src/runtime/component/instance.rs
+++ b/crates/wasmtime/src/runtime/component/instance.rs
@@ -433,6 +433,9 @@ impl Instance {
!self.id().get(store).resource_owned_by_own_instance(dst),
"should have used `resource.transfer-borrow-to-owner` instead",
);
+ // No lazy-lend path here: the `resource_lower_borrow` below needs a
+ // materialized scope for its `borrow_count`, so the thread has to be
+ // forced anyway.
let mut tables = store.component_resource_tables(Some(self))?;
let rep = tables.resource_lift_borrow(TypedResourceIndex::Component { ty: src, index })?;
tables.resource_lower_borrow(TypedResource::Component { ty: dst, rep })
@@ -446,8 +449,8 @@ impl Instance {
index: u32,
src: TypeResourceTableIndex,
) -> Result<u32> {
- let mut tables = store.component_resource_tables(Some(self))?;
- tables.resource_lift_borrow(TypedResourceIndex::Component { ty: src, index })
+ store
+ .resource_lift_borrow_maybe_lazy(self, TypedResourceIndex::Component { ty: src, index })
}
pub(crate) fn lookup_vmdef(&self, store: &mut StoreOpaque, def: &CoreDef) -> vm::Export {
diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rs
index 7de6329eb4..81b5a279c6 100644
--- a/crates/wasmtime/src/runtime/component/store.rs
+++ b/crates/wasmtime/src/runtime/component/store.rs
@@ -1,12 +1,14 @@
use crate::prelude::*;
use crate::runtime::component::{HostResourceData, Instance};
use crate::runtime::vm;
+use crate::runtime::vm::VMResourceIndex;
use crate::runtime::vm::component::{
CallContext, ComponentInstance, HandleCounter, HandleTable, OwnedComponentInstance, Scope,
};
use crate::store::{StoreData, StoreId, StoreOpaque};
use crate::{AsContext, AsContextMut, Engine, Store, StoreContextMut, bail_bug};
use core::pin::Pin;
+use wasmtime_environ::LAZY_LENDERS_CAPACITY;
use wasmtime_environ::component::RuntimeComponentInstanceIndex;
use wasmtime_environ::prelude::TryPrimaryMap;
@@ -65,6 +67,12 @@ pub struct ComponentStoreData {
component_host_table: HandleTable,
host_resource_data: HostResourceData,
+ /// Backing storage for `VMStoreContext::lazy_lenders`; see
+ /// `ResourceTables::resource_lift_borrow_lazy`.
+ ///
+ /// Boxed and never resized, since JIT code holds a raw pointer to it.
+ lazy_lenders: Box<[VMResourceIndex; LAZY_LENDERS_CAPACITY]>,
+
/// Metadata/tasks/etc related to component-model-async and concurrency
/// support.
task_state: ComponentTaskState,
@@ -100,8 +108,8 @@ pub struct RuntimeInstance {
}
impl ComponentStoreData {
- pub fn new(engine: &Engine) -> ComponentStoreData {
- ComponentStoreData {
+ pub fn new(engine: &Engine) -> Result<ComponentStoreData> {
+ Ok(ComponentStoreData {
instances: Default::default(),
component_trapped: false,
has_components: false,
@@ -125,7 +133,62 @@ impl ComponentStoreData {
ComponentTaskState::NotConcurrent(Default::default())
},
hostcall_fuel: DEFAULT_HOSTCALL_FUEL,
- }
+ lazy_lenders: try_new::<Box<_>>([VMResourceIndex::EMPTY; LAZY_LENDERS_CAPACITY])?,
+ })
+ }
+
+ /// Builds the resource tables for `instance` with an explicit
+ /// `current_scope`.
+ ///
+ /// Also returns the host resource data and the `lazy_lenders` buffer, which
+ /// live alongside the tables and so can only be borrowed together with
+ /// them.
+ fn resource_tables(
+ &mut self,
+ store_id: StoreId,
+ instance: Option<Instance>,
+ current_scope: Option<vm::component::CurrentScope>,
+ ) -> (
+ vm::component::ResourceTables<'_>,
+ &mut HostResourceData,
+ &mut [VMResourceIndex],
+ ) {
+ let guest = instance.map(|i| {
+ let i = i.id();
+ i.assert_belongs_to(store_id);
+ self.instances[i.instance]
+ .as_mut()
+ .unwrap()
+ .get_mut()
+ .instance_states()
+ });
+ (
+ vm::component::ResourceTables {
+ host_table: &mut self.component_host_table,
+ handle_counter: &mut self.handle_counter,
+ task_state: &mut self.task_state,
+ guest,
+ current_scope,
+ },
+ &mut self.host_resource_data,
+ &mut self.lazy_lenders[..],
+ )
+ }
+
+ /// Points `vm_store_context` at this store's `lazy_lenders` buffer.
+ ///
+ /// # Safety
+ ///
+ /// `self` must outlive `vm_store_context` and must not move while it is in
+ /// use, since JIT code reads the buffer through the published pointer.
+ pub(crate) unsafe fn publish_lazy_lenders(
+ &mut self,
+ vm_store_context: &mut vm::VMStoreContext,
+ ) {
+ let lazy_lenders = core::ptr::NonNull::from(&mut *self.lazy_lenders);
+ // SAFETY: the buffer is boxed, so it stays put as long as `self` does,
+ // which the caller guarantees.
+ unsafe { vm_store_context.set_lazy_lenders(lazy_lenders) }
}
/// Hook used just before a `Store` is dropped to dispose of anything
@@ -460,29 +523,73 @@ impl StoreOpaque {
&mut crate::component::HostResourceData,
)> {
let current_scope = self.current_scope()?;
-
let store_id = self.id();
- let data = self.component_data_mut();
- let guest = instance.map(|i| {
- let i = i.id();
- i.assert_belongs_to(store_id);
- data.instances[i.instance]
- .as_mut()
- .unwrap()
- .get_mut()
- .instance_states()
- });
+ let (tables, host_resource_data, _) =
+ self.component_data_mut()
+ .resource_tables(store_id, instance, current_scope);
+ Ok((tables, host_resource_data))
+ }
- Ok((
- vm::component::ResourceTables {
- host_table: &mut data.component_host_table,
- handle_counter: &mut data.handle_counter,
- task_state: &mut data.task_state,
- guest,
- current_scope,
- },
- &mut data.host_resource_data,
- ))
+ /// Records a borrow of `index` within `instance` without forcing an
+ /// unforced `VMDeferredThread`, falling back to forcing it when that is
+ /// not possible.
+ ///
+ /// See `ResourceTables::resource_lift_borrow_lazy`.
+ pub(crate) fn resource_lift_borrow_maybe_lazy(
+ &mut self,
+ instance: Instance,
+ index: vm::component::TypedResourceIndex,
+ ) -> Result<u32> {
+ // NB: this check must happen before anything calls `current_scope()`,
+ // which is itself the call that forces the thread.
+ if self
+ .vm_store_context_mut()
+ .current_thread_mut()
+ .is_deferred()
+ {
+ let store_id = self.id();
+ let (vm_store_context, store_data) = self.vm_store_context_and_store_data_mut();
+ let (mut tables, _, lazy_lenders) =
+ store_data
+ .components
+ .resource_tables(store_id, Some(instance), None);
+ match tables.resource_lift_borrow_lazy(index, lazy_lenders, vm_store_context)? {
+ Some(rep) => return Ok(rep),
+ None => {
+ // Fallthrough: the buffer is full and nothing was lent.
+ }
+ }
+ }
+
+ // The path for forcing the `VMDeferredThread`, or using an
+ // already-forced one.
+ self.component_resource_tables(Some(instance))?
+ .resource_lift_borrow(index)
+ }
+
+ /// Promotes the lazy lends in `start..end` into real lends owned by
+ /// `scope`; see `ResourceTables::promote_lazy_lends`.
+ #[cfg_attr(
+ not(feature = "component-model-async"),
+ expect(
+ dead_code,
+ reason = "deferred threads are only created with `component-model-async`, \
+ so nothing promotes lazy lends without it"
+ )
+ )]
+ pub(crate) fn promote_lazy_lends(
+ &mut self,
+ instance: Instance,
+ scope: vm::component::Scope,
+ start: usize,
+ end: usize,
+ ) -> Result<()> {
+ debug_assert!(start <= end && end <= *self.vm_store_context_mut().lazy_lenders_len_mut());
+ let store_id = self.id();
+ let (mut tables, _, lazy_lenders) =
+ self.component_data_mut()
+ .resource_tables(store_id, Some(instance), None);
+ tables.promote_lazy_lends(scope, &lazy_lenders[start..end])
}
pub(crate) fn enter_call_not_concurrent(&mut self) -> Result<()> {
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index fca2e3a652..592eb0e2d6 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -719,7 +719,7 @@ impl<T> Store<T> {
/// memory allocation fails. See the `OutOfMemory` type's documentation for
/// details on Wasmtime's out-of-memory handling.
pub fn try_new(engine: &Engine, data: T) -> Result<Self> {
- let store_data = StoreData::new(engine);
+ let store_data = StoreData::new(engine)?;
log::trace!("creating new store {:?}", store_data.id());
let pkey = engine.allocator().next_available_pkey();
@@ -775,6 +775,18 @@ impl<T> Store<T> {
<NonNull<ManuallyDrop<T>>>::from(&mut inner.data_no_provenance).cast::<()>();
inner.inner.vm_store_context.store_data = store_data.into();
+ // SAFETY: `store_data` and the `VMStoreContext` beside it live in the
+ // same boxed `StoreInner`, so neither outlives nor moves out from under
+ // the other.
+ unsafe {
+ let StoreOpaque {
+ store_data,
+ vm_store_context,
+ ..
+ } = &mut inner.inner;
+ store_data.publish_to_vm_store_context(vm_store_context);
+ }
+
inner.traitobj = StorePtr(Some(NonNull::from(&mut *inner)));
// Wasmtime uses the callee argument to host functions to learn about
@@ -1585,6 +1597,17 @@ impl StoreOpaque {
(&mut self.store_data, &self.modules)
}
+ /// Borrows the `VMStoreContext` and the store's data at the same time.
+ ///
+ /// These are disjoint fields, so JIT-visible state can be reached alongside
+ /// the tables in `StoreData` without a whole-store borrow in between.
+ #[cfg(feature = "component-model")]
+ pub(crate) fn vm_store_context_and_store_data_mut(
+ &mut self,
+ ) -> (&mut VMStoreContext, &mut StoreData) {
+ (&mut self.vm_store_context, &mut self.store_data)
+ }
+
#[cfg(feature = "debug")]
pub(crate) fn breakpoints_and_registry_and_engine_mut(
&mut self,
diff --git a/crates/wasmtime/src/runtime/store/data.rs b/crates/wasmtime/src/runtime/store/data.rs
index 3ed9501972..1bb0b9822f 100644
--- a/crates/wasmtime/src/runtime/store/data.rs
+++ b/crates/wasmtime/src/runtime/store/data.rs
@@ -1,4 +1,5 @@
use crate::module::ModuleRegistry;
+use crate::prelude::*;
use crate::runtime::vm::{self, VMStore};
use crate::store::StoreOpaque;
use crate::{Engine, StoreContext, StoreContextMut};
@@ -21,14 +22,14 @@ pub struct StoreData {
}
impl StoreData {
- pub fn new(engine: &Engine) -> StoreData {
+ pub fn new(engine: &Engine) -> Result<StoreData> {
#[cfg(not(feature = "component-model"))]
let _ = engine;
- StoreData {
+ Ok(StoreData {
id: StoreId::allocate(),
#[cfg(feature = "component-model")]
- components: crate::component::ComponentStoreData::new(engine),
- }
+ components: crate::component::ComponentStoreData::new(engine)?,
+ })
}
pub fn id(&self) -> StoreId {
@@ -48,6 +49,26 @@ impl StoreData {
#[cfg(not(feature = "component-model"))]
let _ = allocator;
}
+
+ /// Publishes any JIT-visible buffers this data owns into
+ /// `vm_store_context`.
+ ///
+ /// # Safety
+ ///
+ /// `self` must outlive `vm_store_context` and must not move while it is in
+ /// use.
+ pub unsafe fn publish_to_vm_store_context(
+ &mut self,
+ vm_store_context: &mut vm::VMStoreContext,
+ ) {
+ // SAFETY: forwarded from this function's own contract.
+ #[cfg(feature = "component-model")]
+ unsafe {
+ self.components.publish_lazy_lenders(vm_store_context);
+ }
+ #[cfg(not(feature = "component-model"))]
+ let _ = vm_store_context;
+ }
}
// forward StoreOpaque => StoreData
diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs
index 05da5a9c0d..76707bb380 100644
--- a/crates/wasmtime/src/runtime/vm.rs
+++ b/crates/wasmtime/src/runtime/vm.rs
@@ -136,6 +136,8 @@ pub use crate::runtime::vm::vmcontext::VMDrcHeapData;
pub use crate::runtime::vm::vmcontext::VMLazyThread;
#[cfg(feature = "gc-null")]
pub use crate::runtime::vm::vmcontext::VMNullHeapData;
+#[cfg(feature = "component-model")]
+pub use crate::runtime::vm::vmcontext::VMResourceIndex;
pub use crate::runtime::vm::vmcontext::{
VMArrayCallHostFuncContext, VMCommonStackInformation, VMContRef, VMContext, VMFuncRef,
VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMGlobalKind, VMHostArray,
diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs
index 57629a19fa..e21b11421b 100644
--- a/crates/wasmtime/src/runtime/vm/component.rs
+++ b/crates/wasmtime/src/runtime/vm/component.rs
@@ -14,10 +14,11 @@ use crate::runtime::component::concurrent::ConcurrentInstanceState;
use crate::runtime::component::{ComponentInstanceId, RuntimeInstance};
use crate::runtime::vm::instance::{InstanceLayout, OwnedInstance, OwnedVMContext};
use crate::runtime::vm::vmcontext::VMFunctionBody;
+use crate::runtime::vm::vmcontext::VMLazyThread;
use crate::runtime::vm::{
HostResult, SendSyncPtr, UncaughtException, VMArrayCallFunction, VMFuncRef, VMGlobalDefinition,
- VMMemoryDefinition, VMOpaqueContext, VMStore, VMStoreRawPtr, VMTableImport, VMWasmCallFunction,
- ValRaw, VmPtr, VmSafe, catch_unwind_and_record_trap,
+ VMMemoryDefinition, VMOpaqueContext, VMResourceIndex, VMStore, VMStoreContext, VMStoreRawPtr,
+ VMTableImport, VMWasmCallFunction, ValRaw, VmPtr, VmSafe, catch_unwind_and_record_trap,
};
use crate::store::InstanceId;
use crate::{Func, vm};
@@ -30,7 +31,9 @@ use core::ptr::NonNull;
use wasmtime_environ::component::*;
use wasmtime_environ::error::OutOfMemory;
use wasmtime_environ::prelude::TryPrimaryMap;
-use wasmtime_environ::{HostPtr, PrimaryMap, PtrSize as _, VMSharedTypeIndex};
+use wasmtime_environ::{
+ HostPtr, LAZY_LENDERS_CAPACITY, PrimaryMap, PtrSize as _, VMSharedTypeIndex,
+};
#[allow(
clippy::cast_possible_truncation,
@@ -49,6 +52,70 @@ pub use self::resources::{
CallContext, CurrentScope, ResourceTables, Scope, TypedResource, TypedResourceIndex,
};
+impl VMResourceIndex {
+ /// The `ty` of a host resource, which is otherwise never a valid
+ /// `TypeResourceTableIndex`.
+ const HOST: u32 = u32::MAX;
+
+ /// A slot that holds no recorded lend.
+ pub(crate) const EMPTY: VMResourceIndex = VMResourceIndex {
+ ty: Self::HOST,
+ index: 0,
+ };
+
+ /// Converts `index` into its JIT-visible form.
+ pub(crate) fn new(index: TypedResourceIndex) -> VMResourceIndex {
+ match index {
+ TypedResourceIndex::Host(index) => VMResourceIndex {
+ ty: Self::HOST,
+ index,
+ },
+ TypedResourceIndex::Component { ty, index } => {
+ debug_assert_ne!(ty.as_u32(), Self::HOST);
+ VMResourceIndex {
+ ty: ty.as_u32(),
+ index,
+ }
+ }
+ }
+ }
+
+ /// Inverse of `new`.
+ pub(crate) fn get(&self) -> TypedResourceIndex {
+ match self.ty {
+ Self::HOST => TypedResourceIndex::Host(self.index),
+ ty => TypedResourceIndex::Component {
+ index: self.index,
+ ty: TypeResourceTableIndex::from_u32(ty),
+ },
+ }
+ }
+}
+
+impl VMStoreContext {
+ pub(crate) fn current_thread_mut(&mut self) -> &mut VMLazyThread {
+ self.current_thread.get_mut()
+ }
+
+ /// Points `lazy_lenders` at the store's buffer.
+ ///
+ /// # Safety
+ ///
+ /// `buf` must stay valid and must not move for as long as this
+ /// `VMStoreContext` is used: JIT code reads the pointer without
+ /// re-deriving it, and nothing here invalidates it.
+ pub(crate) unsafe fn set_lazy_lenders(
+ &mut self,
+ buf: NonNull<[VMResourceIndex; LAZY_LENDERS_CAPACITY]>,
+ ) {
+ self.lazy_lenders = buf.cast::<VMResourceIndex>().into();
+ }
+
+ pub(crate) fn lazy_lenders_len_mut(&mut self) -> &mut usize {
+ self.lazy_lenders_len.get_mut()
+ }
+}
+
/// Represents the state of a (sub-)component instance.
#[derive(Default)]
pub struct InstanceState {
diff --git a/crates/wasmtime/src/runtime/vm/component/handle_table.rs b/crates/wasmtime/src/runtime/vm/component/handle_table.rs
index 8fae63c855..491aad97f9 100644
--- a/crates/wasmtime/src/runtime/vm/component/handle_table.rs
+++ b/crates/wasmtime/src/runtime/vm/component/handle_table.rs
@@ -44,6 +44,14 @@ impl HandleCounter {
pub fn set_limit(&mut self, limit: usize) {
self.limit = limit;
}
+
+ /// Returns whether `n` more slots can be allocated without exceeding the
+ /// limit.
+ pub fn has_room(&self, n: usize) -> bool {
+ self.used
+ .checked_add(n)
+ .is_some_and(|used| used <= self.limit)
+ }
}
/// Represents the state of a stream or future handle from the perspective of a
@@ -301,6 +309,26 @@ impl HandleTable {
&mut self,
counter: &mut HandleCounter,
idx: TypedResourceIndex,
+ ) -> Result<u32> {
+ self.resource_lend_impl(idx, Some(counter))
+ }
+
+ /// Like `resource_lend`, but only validates the lend, leaving `lend_count`
+ /// and the store's handle count alone.
+ ///
+ /// For lends recorded in the `lazy_lenders` buffer, whose `lend_count`
+ /// increments and handle charges are deferred until the recording thread is
+ /// forced.
+ pub fn resource_lend_deferred(&mut self, idx: TypedResourceIndex) -> Result<u32> {
+ self.resource_lend_impl(idx, None)
+ }
+
+ /// Validates a lend of `idx`, additionally counting it when given the
+ /// store's `counter`.
+ fn resource_lend_impl(
+ &mut self,
+ idx: TypedResourceIndex,
+ counter: Option<&mut HandleCounter>,
) -> Result<u32> {
match self.get_mut(idx.raw_index())? {
Slot::ResourceOwn {
@@ -313,14 +341,16 @@ impl HandleTable {
..
} => {
let rep = resource.rep(&idx)?;
- let Some(new_lend_count) = lend_count.checked_add(1) else {
- bail!("cannot lend resource: too many active lends")
- };
- if counter.used >= counter.limit {
- bail!("cannot lend resource: store handle limit exceeded");
+ if let Some(counter) = counter {
+ let Some(new_lend_count) = lend_count.checked_add(1) else {
+ bail!("cannot lend resource: too many active lends")
+ };
+ if counter.used >= counter.limit {
+ bail!("cannot lend resource: store handle limit exceeded");
+ }
+ counter.used += 1;
+ *lend_count = new_lend_count;
}
- counter.used += 1;
- *lend_count = new_lend_count;
Ok(rep)
}
_ => bail!("index {} is not a resource", idx.raw_index()),
diff --git a/crates/wasmtime/src/runtime/vm/component/resources.rs b/crates/wasmtime/src/runtime/vm/component/resources.rs
index c76566f3bf..8dede50daa 100644
--- a/crates/wasmtime/src/runtime/vm/component/resources.rs
+++ b/crates/wasmtime/src/runtime/vm/component/resources.rs
@@ -24,6 +24,7 @@ use super::{HandleCounter, HandleTable, InstanceState, RemovedResource};
use crate::bail_bug;
use crate::component::store::ComponentTaskState;
use crate::prelude::*;
+use crate::runtime::vm::{VMResourceIndex, VMStoreContext};
use core::error::Error;
use core::fmt;
use core::mem;
@@ -369,6 +370,79 @@ impl ResourceTables<'_> {
Ok(rep)
}
+ /// Variant of `resource_lift_borrow` for an unforced `VMDeferredThread`,
+ /// which records the lend in the store's `lazy_lenders` buffer instead of a
+ /// `CallContext`.
+ ///
+ /// The handle's `lend_count` is left alone: forcing the thread performs the
+ /// real lend for each buffered entry (see `promote_lazy_lends`), and exiting
+ /// an unforced frame discards its entries by resetting `lazy_lenders_len`.
+ /// Everything that reads `lend_count` forces the thread first, so the count
+ /// is always complete by the time anything relies on it.
+ ///
+ /// Returns `Ok(None)` without lending anything if the buffer is full, in
+ /// which case the caller must force the thread and use
+ /// `resource_lift_borrow` instead.
+ pub fn resource_lift_borrow_lazy(
+ &mut self,
+ index: TypedResourceIndex,
+ lazy_lenders: &mut [VMResourceIndex],
+ vm_store_context: &mut VMStoreContext,
+ ) -> Result<Option<u32>> {
+ // A forced thread already has a `CallContext`, and anything recorded
+ // here would never be promoted into it.
+ debug_assert!(vm_store_context.current_thread_mut().is_deferred());
+
+ let lazy_lenders_len = vm_store_context.lazy_lenders_len_mut();
+ let Some(slot) = lazy_lenders.get_mut(*lazy_lenders_len) else {
+ return Ok(None);
+ };
+ let rep = self
+ .table_for_index(&index)
+ .0
+ .resource_lend_deferred(index)?;
+ *slot = VMResourceIndex::new(index);
+ *lazy_lenders_len += 1;
+ Ok(Some(rep))
+ }
+
+ /// Promotes lazy lends recorded by `resource_lift_borrow_lazy` into real
+ /// lends owned by `scope`'s `CallContext`, incrementing each handle's
+ /// `lend_count` and charging the store's handle count now that the thread
+ /// that made them has been forced.
+ ///
+ /// Either every lend is promoted or, on error, none is.
+ #[cfg_attr(
+ not(feature = "component-model-async"),
+ expect(
+ dead_code,
+ reason = "deferred threads are only created with `component-model-async`, so nothing promotes lazy lends without it"
+ )
+ )]
+ pub fn promote_lazy_lends(
+ &mut self,
+ scope: Scope,
+ lazy_lenders: &[VMResourceIndex],
+ ) -> Result<()> {
+ // Fail up front on anything that could otherwise fail partway through
+ // and leave some of these lends promoted but not the rest.
+ self.task_state
+ .call_context(scope)?
+ .lenders
+ .reserve(lazy_lenders.len())?;
+ if !self.handle_counter.has_room(lazy_lenders.len()) {
+ bail!("cannot lend resource: store handle limit exceeded");
+ }
+
+ for lender in lazy_lenders {
+ let index = lender.get();
+ let (table, counter) = self.table_for_index(&index);
+ table.resource_lend(counter, index)?;
+ self.task_state.call_context(scope)?.lenders.push(index)?;
+ }
+ Ok(())
+ }
+
/// Records a new `borrow` resource with the given representation within the
/// current call scope.
///
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index d29d86a419..198686ff88 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -1013,11 +1013,6 @@ impl VMStoreContext {
pub(crate) fn component_context_mut(&mut self) -> &mut [u32; NUM_COMPONENT_CONTEXT_SLOTS] {
self.component_context.get_mut()
}
-
- #[cfg(feature = "component-model-async")]
- pub(crate) fn current_thread_mut(&mut self) -> &mut VMLazyThread {
- self.current_thread.get_mut()
- }
}
// The `VMStoreContext` type is a pod-type with no destructor, and we don't
@@ -1051,6 +1046,8 @@ impl Default for VMStoreContext {
store_data: VmPtr::dangling(),
component_context: UnsafeCell::new([0; NUM_COMPONENT_CONTEXT_SLOTS]),
current_thread: UnsafeCell::new(VMLazyThread::none()),
+ lazy_lenders: VmPtr::dangling(),
+ lazy_lenders_len: UnsafeCell::new(0),
}
}
}
diff --git a/tests/disas/aarch64-entry-trampoline.wat b/tests/disas/aarch64-entry-trampoline.wat
index a38bd560b2..dc65f67719 100644
--- a/tests/disas/aarch64-entry-trampoline.wat
+++ b/tests/disas/aarch64-entry-trampoline.wat
@@ -17,20 +17,22 @@
;; stp d10, d11, [sp, #-0x10]!
;; stp d8, d9, [sp, #-0x10]!
;; sub sp, sp, #0x10
+;; mov x3, x0
+;; mov x13, x29
+;; mov x0, x3
+;; ldr x2, [x0, #8]
+;; str x13, [x2, #0x48]
+;; mov x14, sp
+;; str x14, [x2, #0x40]
+;; adr x15, #0xa8
+;; str x15, [x2, #0x50]
+;; stur x2, [sp]
+;; mov x2, x3
;; mov x3, x1
-;; mov x12, x29
-;; ldr x15, [x0, #8]
-;; str x12, [x15, #0x48]
-;; mov x13, sp
-;; str x13, [x15, #0x40]
-;; adr x14, #0xa0
-;; str x14, [x15, #0x50]
-;; mov x2, x0
-;; stur x15, [sp]
;; bl #0
;; ├─╼ exception frame offset: SP = FP - 0xa0
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xa0
-;; 6c: mov w0, #1
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0xa8
+;; 74: mov w0, #1
;; add sp, sp, #0x10
;; ldp d8, d9, [sp], #0x10
;; ldp d10, d11, [sp], #0x10
@@ -43,19 +45,21 @@
;; ldp x27, x28, [sp], #0x10
;; ldp x29, x30, [sp], #0x10
;; ret
-;; a0: mov x0, #1
-;; a4: ldur x15, [sp]
-;; a8: str x0, [x15, #0x88]
-;; ac: mov w0, #0
-;; b0: add sp, sp, #0x10
-;; b4: ldp d8, d9, [sp], #0x10
-;; b8: ldp d10, d11, [sp], #0x10
-;; bc: ldp d12, d13, [sp], #0x10
-;; c0: ldp d14, d15, [sp], #0x10
-;; c4: ldp x19, x20, [sp], #0x10
-;; c8: ldp x21, x22, [sp], #0x10
-;; cc: ldp x23, x24, [sp], #0x10
-;; d0: ldp x25, x26, [sp], #0x10
-;; d4: ldp x27, x28, [sp], #0x10
-;; d8: ldp x29, x30, [sp], #0x10
-;; dc: ret
+;; a8: mov x1, #1
+;; ac: ldur x2, [sp]
+;; b0: str x1, [x2, #0x88]
+;; b4: mov x1, #0
+;; b8: str x1, [x2, #0x98]
+;; bc: mov w0, #0
+;; c0: add sp, sp, #0x10
+;; c4: ldp d8, d9, [sp], #0x10
+;; c8: ldp d10, d11, [sp], #0x10
+;; cc: ldp d12, d13, [sp], #0x10
+;; d0: ldp d14, d15, [sp], #0x10
+;; d4: ldp x19, x20, [sp], #0x10
+;; d8: ldp x21, x22, [sp], #0x10
+;; dc: ldp x23, x24, [sp], #0x10
+;; e0: ldp x25, x26, [sp], #0x10
+;; e4: ldp x27, x28, [sp], #0x10
+;; e8: ldp x29, x30, [sp], #0x10
+;; ec: ret
diff --git a/tests/disas/array-call-trampoline.wat b/tests/disas/array-call-trampoline.wat
index 639b6eebbe..fbd866fc2d 100644
--- a/tests/disas/array-call-trampoline.wat
+++ b/tests/disas/array-call-trampoline.wat
@@ -15,6 +15,7 @@
;; region3 = 243 ""
;; region4 = 209 ""
;; region5 = 68 ""
+;; region6 = 30 ""
;; sig0 = (i64 vmctx, i64, i32, i64) -> i32, i64 tail
;; fn0 = colocated u0:0 sig0
;;
@@ -45,6 +46,8 @@
;; block3:
;; v15 = iconst.i64 1
;; store notrap aligned region5 v15, v8+136 ; v15 = 1
-;; v16 = iconst.i8 0
-;; return v16 ; v16 = 0
+;; v16 = iconst.i64 0
+;; store notrap aligned region6 v16, v8+152 ; v16 = 0
+;; v17 = iconst.i8 0
+;; return v17 ; v17 = 0
;; }
diff --git a/tests/disas/component-model/thread-transparency/opaque.wat b/tests/disas/component-model/thread-transparency/opaque.wat
index 7c29c91e50..edb6503257 100644
--- a/tests/disas/component-model/thread-transparency/opaque.wat
+++ b/tests/disas/component-model/thread-transparency/opaque.wat
@@ -62,7 +62,7 @@
(export "g" (func $b "g"))
)
;; function u1:0(i64 vmctx, i64) -> i32 tail {
-;; ss0 = explicit_slot 24, align = 8
+;; ss0 = explicit_slot 32, align = 8
;; region0 = 123 ""
;; region1 = 160 ""
;; region2 = 42 ""
@@ -76,8 +76,10 @@
;; region10 = 71 ""
;; region11 = 149 ""
;; region12 = 93 ""
-;; region13 = 132 ""
-;; region14 = 174 ""
+;; region13 = 30 ""
+;; region14 = 184 ""
+;; region15 = 132 ""
+;; region16 = 174 ""
;; gv0 = vmctx
;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
;; gv2 = load.i64 notrap aligned region1 gv1+24
@@ -125,15 +127,17 @@
;; v19 = load.i32 notrap aligned region11 v15+132
;; store notrap aligned region12 v19, v14+20
;; store notrap aligned region11 v8, v15+132 ; v8 = 0
+;; v21 = load.i64 notrap aligned region13 v15+152
+;; store notrap aligned region14 v21, v14+24
;; store notrap aligned region5 v14, v15+136
-;; v21 = load.i64 notrap aligned readonly can_move region3 v3+208
-;; v22 = load.i32 notrap aligned region13 v21
+;; v22 = load.i64 notrap aligned readonly can_move region3 v3+208
+;; v23 = load.i32 notrap aligned region15 v22
;; jump block16
;;
;; block16:
-;; v26 = load.i64 notrap aligned readonly can_move region2 v3+72
-;; v28 = load.i64 notrap aligned readonly can_move region0 v26+8
-;; v29 = load.i32 notrap aligned region9 v28+128
+;; v27 = load.i64 notrap aligned readonly can_move region2 v3+72
+;; v29 = load.i64 notrap aligned readonly can_move region0 v27+8
+;; v30 = load.i32 notrap aligned region9 v29+128
;; jump block17
;;
;; block17:
@@ -166,6 +170,6 @@
;;
;; block1:
;; @0120 v2 = iconst.i32 1234
-;; v46 = iadd.i32 v29, v2 ; v2 = 1234
-;; @0125 return v46
+;; v48 = iadd.i32 v30, v2 ; v2 = 1234
+;; @0125 return v48
;; }
diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.wat
index ace7585acf..3187a1574c 100644
--- a/tests/disas/debug-exceptions.wat
+++ b/tests/disas/debug-exceptions.wat
@@ -37,29 +37,29 @@
;; stur x2, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x35, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [65, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [69, 1, 0, 148]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [63, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [67, 1, 0, 148]
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3d, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [61, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [65, 1, 0, 148]
;; mov w1, #0x2a
;; stur w1, [sp, #8]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3f, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [58, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [62, 1, 0, 148]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x40, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [57, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [61, 1, 0, 148]
;; stur w1, [sp, #8]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [55, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [59, 1, 0, 148]
;; ldur x2, [sp, #0x10]
-;; bl #0x4c0
+;; bl #0x4d0
;; 88: ldur x0, [sp, #0x10]
;; mov x19, x2
;; ldr x0, [x0, #0x20]
@@ -90,7 +90,7 @@
;; mov w8, #0
;; str w8, [x1, #0x14]
;; ldur x2, [sp, #0x10]
-;; bl #0x4f8
+;; bl #0x508
;; ├─╼ exception frame offset: SP = FP - 0xb0
;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x108
;; b #0x1c8
@@ -109,7 +109,7 @@
;; ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x48, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [7, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [11, 1, 0, 148]
;; ldur x14, [sp, #0x10]
;; ldr x0, [x14, #0x38]
;; ldr x2, [x14, #0x48]
@@ -119,10 +119,10 @@
;; 154: ldur x0, [sp, #0x10]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [0, 1, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [4, 1, 0, 148]
;; nop
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4b, slot at FP-0xb0, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [255, 0, 0, 148]
+;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [3, 1, 0, 148]
;; add sp, sp, #0x20
;; ldp d8, d9, [sp], #0x10
;; ldp d10, d11, [sp], #0x10
@@ -143,7 +143,7 @@
;; 1a4: mov w5, #0x20
;; 1a8: mov w6, #0x10
;; 1ac: ldur x2, [sp, #0x10]
-;; 1b0: bl #0x3ec
+;; 1b0: bl #0x3fc
;; 1b4: ldur x4, [sp, #0x18]
;; 1b8: ldr x0, [x4, #0x20]
;; 1bc: add x1, x0, w2, uxtw
@@ -151,29 +151,29 @@
;; 1c4: b #0xe4
;; 1c8: mov w3, #9
;; 1cc: ldur x2, [sp, #0x10]
-;; 1d0: bl #0x454
+;; 1d0: bl #0x464
;; 1d4: ldur x2, [sp, #0x10]
-;; 1d8: bl #0x48c
+;; 1d8: bl #0x49c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8
;; 1dc: udf #0xc11f
;; 1e0: mov w3, #0xfe
;; 1e4: ldur x2, [sp, #0x10]
-;; 1e8: bl #0x454
+;; 1e8: bl #0x464
;; 1ec: ldur x2, [sp, #0x10]
-;; 1f0: bl #0x48c
+;; 1f0: bl #0x49c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
;; 1f4: udf #0xc11f
;; 1f8: mov w3, #0xfe
;; 1fc: ldur x2, [sp, #0x10]
-;; 200: bl #0x454
+;; 200: bl #0x464
;; 204: ldur x2, [sp, #0x10]
-;; 208: bl #0x48c
+;; 208: bl #0x49c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack
;; 20c: udf #0xc11f
;; 210: stur x2, [sp, #0x10]
;; 214: mov w3, #0
-;; 218: bl #0x454
+;; 218: bl #0x464
;; 21c: ldur x2, [sp, #0x10]
-;; 220: bl #0x48c
+;; 220: bl #0x49c
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x34, slot at FP-0xb0, locals , stack
;; 224: udf #0xc11f
diff --git a/tests/disas/debug.wat b/tests/disas/debug.wat
index fcb16f88f7..c7fa9bea6c 100644
--- a/tests/disas/debug.wat
+++ b/tests/disas/debug.wat
@@ -23,20 +23,20 @@
;; 29: movq %rdi, (%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x24, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack
-;; ╰─╼ breakpoint patch: wasm PC 0x24, patch bytes [232, 224, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x24, patch bytes [232, 235, 1, 0, 0]
;; movl %edx, 0x10(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x26, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10
-;; ╰─╼ breakpoint patch: wasm PC 0x26, patch bytes [232, 215, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x26, patch bytes [232, 226, 1, 0, 0]
;; movl %ecx, 0x14(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x28, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10, I32 @ slot+0x14
-;; ╰─╼ breakpoint patch: wasm PC 0x28, patch bytes [232, 206, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x28, patch bytes [232, 217, 1, 0, 0]
;; leal (%rdx, %rcx), %eax
;; movl %eax, 0x10(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x29, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack I32 @ slot+0x10
-;; ╰─╼ breakpoint patch: wasm PC 0x29, patch bytes [232, 194, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x29, patch bytes [232, 205, 1, 0, 0]
;; movl %eax, 0x10(%rsp)
;; movq 0x20(%rsp), %r12
;; addq $0x30, %rsp
@@ -45,9 +45,9 @@
;; retq
;; 62: movq %rdi, %r12
;; 65: xorl %esi, %esi
-;; 67: callq 0x1b4
+;; 67: callq 0x1bf
;; 6c: movq %r12, %rdi
-;; 6f: callq 0x1e5
+;; 6f: callq 0x1f0
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x23, slot at FP-0x30, locals I32 @ slot+0x8, I32 @ slot+0xc, stack
;; 74: ud2
;;
@@ -61,7 +61,7 @@
;; movq %r14, 0x28(%rsp)
;; movq %r15, 0x30(%rsp)
;; cmpq $2, %rcx
-;; jb 0x136
+;; jb 0x141
;; a1: movl (%rdx), %r8d
;; movl 0x10(%rdx), %ecx
;; movq %rdx, (%rsp)
@@ -91,17 +91,18 @@
;; retq
;; 102: movq 8(%rsp), %r10
;; 107: movq $1, 0x88(%r10)
-;; 112: xorl %eax, %eax
-;; 114: movq 0x10(%rsp), %rbx
-;; 119: movq 0x18(%rsp), %r12
-;; 11e: movq 0x20(%rsp), %r13
-;; 123: movq 0x28(%rsp), %r14
-;; 128: movq 0x30(%rsp), %r15
-;; 12d: addq $0x40, %rsp
-;; 131: movq %rbp, %rsp
-;; 134: popq %rbp
-;; 135: retq
-;; 136: ud2
+;; 112: movq $0, 0x98(%r10)
+;; 11d: xorl %eax, %eax
+;; 11f: movq 0x10(%rsp), %rbx
+;; 124: movq 0x18(%rsp), %r12
+;; 129: movq 0x20(%rsp), %r13
+;; 12e: movq 0x28(%rsp), %r14
+;; 133: movq 0x30(%rsp), %r15
+;; 138: addq $0x40, %rsp
+;; 13c: movq %rbp, %rsp
+;; 13f: popq %rbp
+;; 140: retq
+;; 141: ud2
;;
;; signatures[0]::wasm_to_array_trampoline:
;; pushq %rbp
@@ -126,19 +127,19 @@
;; callq *%rax
;; addq $1, 0x10(%r15)
;; testb %al, %al
-;; je 0x1a2
-;; 18c: movl (%rsp), %eax
+;; je 0x1ad
+;; 197: movl (%rsp), %eax
;; movq 0x20(%rsp), %rbx
;; movq 0x28(%rsp), %r15
;; addq $0x30, %rsp
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 1a2: movq 0x10(%rbx), %rax
-;; 1a6: movq 0x140(%rax), %rax
-;; 1ad: movq %rbx, %rdi
-;; 1b0: callq *%rax
-;; 1b2: ud2
+;; 1ad: movq 0x10(%rbx), %rax
+;; 1b1: movq 0x140(%rax), %rax
+;; 1b8: movq %rbx, %rdi
+;; 1bb: callq *%rax
+;; 1bd: ud2
;;
;; wasmtime_builtin_trap:
;; pushq %rbp
@@ -215,8 +216,8 @@
;; movq 0x168(%r12), %r11
;; callq *%r11
;; testb %al, %al
-;; je 0x3e9
-;; 315: movq (%rsp), %rax
+;; je 0x3f4
+;; 320: movq (%rsp), %rax
;; movq 8(%rsp), %rcx
;; movq 0x10(%rsp), %rdx
;; movq 0x18(%rsp), %rsi
@@ -247,7 +248,7 @@
;; movq %rbp, %rsp
;; popq %rbp
;; retq
-;; 3e9: movq 0x140(%r12), %rax
-;; 3f1: movq %r13, %rdi
-;; 3f4: callq *%rax
-;; 3f6: ud2
+;; 3f4: movq 0x140(%r12), %rax
+;; 3fc: movq %r13, %rdi
+;; 3ff: callq *%rax
+;; 401: ud2
diff --git a/tests/disas/elem-segment.wat b/tests/disas/elem-segment.wat
index c413322c41..b300f09019 100644
--- a/tests/disas/elem-segment.wat
+++ b/tests/disas/elem-segment.wat
@@ -16,6 +16,7 @@
;; region2 = 243 ""
;; region3 = 209 ""
;; region4 = 68 ""
+;; region5 = 30 ""
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -33,14 +34,16 @@
;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
;;
;; block2:
-;; v13 = iconst.i8 1
-;; return v13 ; v13 = 1
+;; v14 = iconst.i8 1
+;; return v14 ; v14 = 1
;;
;; block3:
;; v11 = iconst.i64 1
;; store notrap aligned region4 v11, v6+136 ; v11 = 1
-;; v12 = iconst.i8 0
-;; return v12 ; v12 = 0
+;; v4 = iconst.i64 0
+;; store notrap aligned region5 v4, v6+152 ; v4 = 0
+;; v13 = iconst.i8 0
+;; return v13 ; v13 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/exceptions.wat b/tests/disas/exceptions.wat
index eb4b275f10..6cc677d0e4 100644
--- a/tests/disas/exceptions.wat
+++ b/tests/disas/exceptions.wat
@@ -33,7 +33,7 @@
;; movq %rdi, %rbx
;; movq %rcx, %r13
;; movq %rdx, %r14
-;; callq 0x40d
+;; callq 0x418
;; movq %rax, %r12
;; movq 0x20(%rbx), %rdx
;; movl (%rdx), %esi
@@ -62,7 +62,7 @@
;; movl $0, 0x14(%rdx)
;; movq %rbx, %rdi
;; movq %rbx, (%rsp)
-;; callq 0x43a
+;; callq 0x445
;; jmp 0xea
;; b6: movl $0x4000002, %esi
;; movq 0x28(%rbx), %rax
@@ -70,7 +70,7 @@
;; movl $0x30, %ecx
;; movl $0x10, %r8d
;; movq %rbx, %rdi
-;; callq 0x3a4
+;; callq 0x3af
;; movq 8(%rbx), %rcx
;; movl %eax, %edx
;; addq 0x20(%rcx), %rdx
diff --git a/tests/disas/gc/issue-11753.wat b/tests/disas/gc/issue-11753.wat
index cacba1c6d6..c48217b142 100644
--- a/tests/disas/gc/issue-11753.wat
+++ b/tests/disas/gc/issue-11753.wat
@@ -94,7 +94,7 @@
;; f4: movl (%rax), %edx
;; f6: movl $0x20, %ecx
;; fb: movl $0x10, %r8d
-;; 101: callq 0x291
+;; 101: callq 0x29c
;; 106: movq 8(%rsp), %r9
;; 10b: movq 8(%r9), %rcx
;; 10f: movl %eax, %esi
diff --git a/tests/disas/issue-12808.wat b/tests/disas/issue-12808.wat
index 5823ce78dc..6c54a56a79 100644
--- a/tests/disas/issue-12808.wat
+++ b/tests/disas/issue-12808.wat
@@ -23,23 +23,23 @@
;; 21: movq %rdi, (%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x1f, slot at FP-0x30, locals , stack
-;; ╰─╼ breakpoint patch: wasm PC 0x1f, patch bytes [232, 10, 2, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x1f, patch bytes [232, 21, 2, 0, 0]
;; movl $0, 8(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x21, slot at FP-0x30, locals , stack I32 @ slot+0x8
-;; ╰─╼ breakpoint patch: wasm PC 0x21, patch bytes [232, 253, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x21, patch bytes [232, 8, 2, 0, 0]
;; movl $0, 0xc(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x23, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc
-;; ╰─╼ breakpoint patch: wasm PC 0x23, patch bytes [232, 240, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x23, patch bytes [232, 251, 1, 0, 0]
;; movl $0, 0x10(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x25, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc, I32 @ slot+0x10
-;; ╰─╼ breakpoint patch: wasm PC 0x25, patch bytes [232, 227, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x25, patch bytes [232, 238, 1, 0, 0]
;; movl $0, 0x14(%rsp)
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x27, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc, I32 @ slot+0x10, I32 @ slot+0x14
-;; ╰─╼ breakpoint patch: wasm PC 0x27, patch bytes [232, 214, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x27, patch bytes [232, 225, 1, 0, 0]
;; movq 0x30(%rdi), %rax
;; movq (%rax), %rcx
;; xorl %eax, %eax
@@ -48,7 +48,7 @@
;; movq %rax, %rcx
;; nopl (%rax, %rax)
;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x2b, slot at FP-0x30, locals , stack I32 @ slot+0x8, I32 @ slot+0xc
-;; ╰─╼ breakpoint patch: wasm PC 0x2b, patch bytes [232, 189, 1, 0, 0]
+;; ╰─╼ breakpoint patch: wasm PC 0x2b, patch bytes [232, 200, 1, 0, 0]
;; xorl %eax, %eax
;; movl $0, 8(%rsp)
;; movl %ecx, 0xc(%rsp)
@@ -59,8 +59,8 @@
;; retq
;; 93: movq %rdi, %r14
;; 96: xorl %esi, %esi
-;; 98: callq 0x1d6
+;; 98: callq 0x1e1
;; 9d: movq %r14, %rdi
-;; a0: callq 0x207
+;; a0: callq 0x212
;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x1e, slot at FP-0x30, locals , stack
;; a5: ud2
diff --git a/tests/disas/pulley-entry-trampoline.wat b/tests/disas/pulley-entry-trampoline.wat
index 06f18fd4c9..6550feea23 100644
--- a/tests/disas/pulley-entry-trampoline.wat
+++ b/tests/disas/pulley-entry-trampoline.wat
@@ -6,23 +6,26 @@
;; wasm[0]::array_to_wasm_trampoline[0]:
;; push_frame_save 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
-;; xmov_fp x12
-;; xload64le_o32 x15, x0, 8
-;; xstore64le_o32 x15, 72, x12
-;; xmov x13, sp
-;; xstore64le_o32 x15, 64, x13
-;; xpcadd x14, 0x23 // target = 0x47
-;; xstore64le_o32 x15, 80, x14
-;; xstore64le_o32 sp, 0, x15
-;; call -0x3a // target = 0x0
+;; xmov_fp x13
+;; xmov x2, x0
+;; xload64le_o32 x3, x2, 8
+;; xstore64le_o32 x3, 72, x13
+;; xmov x14, sp
+;; xstore64le_o32 x3, 64, x14
+;; xpcadd x15, 0x23 // target = 0x4a
+;; xstore64le_o32 x3, 80, x15
+;; xstore64le_o32 sp, 0, x3
+;; call -0x3d // target = 0x0
;; ├─╼ exception frame offset: SP = FP - 0x90
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x47
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x4a
;; xone x0
;; pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
;; ret
-;; 47: xone x0
-;; 49: xload64le_o32 x15, sp, 0
-;; 50: xstore64le_o32 x15, 136, x0
-;; 57: xzero x0
-;; 59: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
-;; 5e: ret
+;; 4a: xone x1
+;; 4c: xload64le_o32 x3, sp, 0
+;; 53: xstore64le_o32 x3, 136, x1
+;; 5a: xzero x1
+;; 5c: xstore64le_o32 x3, 152, x1
+;; 63: xzero x0
+;; 65: pop_frame_restore 144, x16, x17, x18, x19, x20, x21, x22, x23, x24, x25, x26, x27, x28, x29, sp, spilltmp0
+;; 6a: ret
diff --git a/tests/disas/riscv64-entry-trampoline.wat b/tests/disas/riscv64-entry-trampoline.wat
index 620f6cab6c..7394bd6846 100644
--- a/tests/disas/riscv64-entry-trampoline.wat
+++ b/tests/disas/riscv64-entry-trampoline.wat
@@ -33,7 +33,9 @@
;; fsd fs9, 0x28(sp)
;; fsd fs10, 0x20(sp)
;; fsd fs11, 0x18(sp)
+;; mv a6, a0
;; mv a2, s0
+;; mv a0, a6
;; ld a5, 8(a0)
;; sd a2, 0x48(a5)
;; mv a3, sp
@@ -43,9 +45,9 @@
;; sd a4, 0x50(a5)
;; sd a5, 0(sp)
;; auipc ra, 0
-;; jalr ra, ra, -0xb4
+;; jalr ra, ra, -0xbc
;; ├─╼ exception frame offset: SP = FP - 0xd0
-;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x130
+;; ╰─╼ exception handler: default handler, no dynamic context, handler=0x138
;; addi a0, zero, 1
;; ld s0, 0xc8(sp)
;; ld s1, 0xc0(sp)
@@ -75,9 +77,10 @@
;; ld s0, 0(sp)
;; addi sp, sp, 0x10
;; ret
-;; addi a0, zero, 1
+;; addi a1, zero, 1
;; ld a5, 0(sp)
-;; sd a0, 0x88(a5)
+;; sd a1, 0x88(a5)
+;; sd zero, 0x98(a5)
;; mv a0, zero
;; ld s0, 0xc8(sp)
;; ld s1, 0xc0(sp)
diff --git a/tests/disas/s390x-entry-trampoline.wat b/tests/disas/s390x-entry-trampoline.wat
index 1e0d78fc84..8c8e23c598 100644
--- a/tests/disas/s390x-entry-trampoline.wat
+++ b/tests/disas/s390x-entry-trampoline.wat
@@ -41,6 +41,7 @@
;; br %r14
;; lg %r5, 0xa0(%r15)
;; mvghi 0x88(%r5), 1
+;; mvghi 0x98(%r5), 0
;; lhi %r2, 0
;; ld %f8, 0xa8(%r15)
;; ld %f9, 0xb0(%r15)
diff --git a/tests/disas/startup-data-active.wat b/tests/disas/startup-data-active.wat
index 928793b0fe..688e828c42 100644
--- a/tests/disas/startup-data-active.wat
+++ b/tests/disas/startup-data-active.wat
@@ -14,6 +14,7 @@
;; region2 = 243 ""
;; region3 = 209 ""
;; region4 = 68 ""
+;; region5 = 30 ""
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -31,14 +32,16 @@
;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
;;
;; block2:
-;; v13 = iconst.i8 1
-;; return v13 ; v13 = 1
+;; v14 = iconst.i8 1
+;; return v14 ; v14 = 1
;;
;; block3:
;; v11 = iconst.i64 1
;; store notrap aligned region4 v11, v6+136 ; v11 = 1
-;; v12 = iconst.i8 0
-;; return v12 ; v12 = 0
+;; v4 = iconst.i64 0
+;; store notrap aligned region5 v4, v6+152 ; v4 = 0
+;; v13 = iconst.i8 0
+;; return v13 ; v13 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-elem-active.wat b/tests/disas/startup-elem-active.wat
index 39b1843b6f..dd127cf728 100644
--- a/tests/disas/startup-elem-active.wat
+++ b/tests/disas/startup-elem-active.wat
@@ -18,6 +18,7 @@
;; region2 = 243 ""
;; region3 = 209 ""
;; region4 = 68 ""
+;; region5 = 30 ""
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -35,14 +36,16 @@
;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
;;
;; block2:
-;; v13 = iconst.i8 1
-;; return v13 ; v13 = 1
+;; v14 = iconst.i8 1
+;; return v14 ; v14 = 1
;;
;; block3:
;; v11 = iconst.i64 1
;; store notrap aligned region4 v11, v6+136 ; v11 = 1
-;; v12 = iconst.i8 0
-;; return v12 ; v12 = 0
+;; v4 = iconst.i64 0
+;; store notrap aligned region5 v4, v6+152 ; v4 = 0
+;; v13 = iconst.i8 0
+;; return v13 ; v13 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-global.wat b/tests/disas/startup-global.wat
index 3e7085d9b2..e5d9f2e4e7 100644
--- a/tests/disas/startup-global.wat
+++ b/tests/disas/startup-global.wat
@@ -11,6 +11,7 @@
;; region2 = 243 ""
;; region3 = 209 ""
;; region4 = 68 ""
+;; region5 = 30 ""
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -28,14 +29,16 @@
;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
;;
;; block2:
-;; v13 = iconst.i8 1
-;; return v13 ; v13 = 1
+;; v14 = iconst.i8 1
+;; return v14 ; v14 = 1
;;
;; block3:
;; v11 = iconst.i64 1
;; store notrap aligned region4 v11, v6+136 ; v11 = 1
-;; v12 = iconst.i8 0
-;; return v12 ; v12 = 0
+;; v4 = iconst.i64 0
+;; store notrap aligned region5 v4, v6+152 ; v4 = 0
+;; v13 = iconst.i8 0
+;; return v13 ; v13 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-passive-segment.wat b/tests/disas/startup-passive-segment.wat
index 2f2c2561ec..24ed32006a 100644
--- a/tests/disas/startup-passive-segment.wat
+++ b/tests/disas/startup-passive-segment.wat
@@ -12,6 +12,7 @@
;; region2 = 243 ""
;; region3 = 209 ""
;; region4 = 68 ""
+;; region5 = 30 ""
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -29,14 +30,16 @@
;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
;;
;; block2:
-;; v13 = iconst.i8 1
-;; return v13 ; v13 = 1
+;; v14 = iconst.i8 1
+;; return v14 ; v14 = 1
;;
;; block3:
;; v11 = iconst.i64 1
;; store notrap aligned region4 v11, v6+136 ; v11 = 1
-;; v12 = iconst.i8 0
-;; return v12 ; v12 = 0
+;; v4 = iconst.i64 0
+;; store notrap aligned region5 v4, v6+152 ; v4 = 0
+;; v13 = iconst.i8 0
+;; return v13 ; v13 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-start.wat b/tests/disas/startup-start.wat
index d45ced7486..6d481757eb 100644
--- a/tests/disas/startup-start.wat
+++ b/tests/disas/startup-start.wat
@@ -12,6 +12,7 @@
;; region2 = 243 ""
;; region3 = 209 ""
;; region4 = 68 ""
+;; region5 = 30 ""
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -29,14 +30,16 @@
;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
;;
;; block2:
-;; v13 = iconst.i8 1
-;; return v13 ; v13 = 1
+;; v14 = iconst.i8 1
+;; return v14 ; v14 = 1
;;
;; block3:
;; v11 = iconst.i64 1
;; store notrap aligned region4 v11, v6+136 ; v11 = 1
-;; v12 = iconst.i8 0
-;; return v12 ; v12 = 0
+;; v4 = iconst.i64 0
+;; store notrap aligned region5 v4, v6+152 ; v4 = 0
+;; v13 = iconst.i8 0
+;; return v13 ; v13 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/startup-table-initial-value.wat b/tests/disas/startup-table-initial-value.wat
index fbc5cde0a9..4ace6d44f1 100644
--- a/tests/disas/startup-table-initial-value.wat
+++ b/tests/disas/startup-table-initial-value.wat
@@ -12,6 +12,7 @@
;; region2 = 243 ""
;; region3 = 209 ""
;; region4 = 68 ""
+;; region5 = 30 ""
;; sig0 = (i64 vmctx, i64) tail
;; fn0 = colocated u2415919104:0 sig0
;;
@@ -29,14 +30,16 @@
;; try_call fn0(v0, v1), sig0, block2, [ default: block3 ]
;;
;; block2:
-;; v13 = iconst.i8 1
-;; return v13 ; v13 = 1
+;; v14 = iconst.i8 1
+;; return v14 ; v14 = 1
;;
;; block3:
;; v11 = iconst.i64 1
;; store notrap aligned region4 v11, v6+136 ; v11 = 1
-;; v12 = iconst.i8 0
-;; return v12 ; v12 = 0
+;; v4 = iconst.i64 0
+;; store notrap aligned region5 v4, v6+152 ; v4 = 0
+;; v13 = iconst.i8 0
+;; return v13 ; v13 = 0
;; }
;;
;; function u2415919104:0(i64 vmctx, i64) tail {
diff --git a/tests/disas/trunc.wat b/tests/disas/trunc.wat
index aed4755d4f..968b7dbd6e 100644
--- a/tests/disas/trunc.wat
+++ b/tests/disas/trunc.wat
@@ -24,7 +24,7 @@
;; jne 0x101
;; 39: movq %rbx, %rdi
;; movdqu (%rsp), %xmm0
-;; callq 0x264
+;; callq 0x26f
;; movabsq $13830554455654793216, %rax
;; movq %rax, %xmm1
;; ucomisd %xmm0, %xmm1
@@ -55,27 +55,27 @@
;; retq
;; d3: movl $6, %esi
;; d8: movq %rbx, %rdi
-;; db: callq 0x291
+;; db: callq 0x29c
;; e0: movq %rbx, %rdi
-;; e3: callq 0x2c2
+;; e3: callq 0x2cd
;; e8: ud2
;; ea: movl $6, %esi
;; ef: movq %rbx, %rdi
-;; f2: callq 0x291
+;; f2: callq 0x29c
;; f7: movq %rbx, %rdi
-;; fa: callq 0x2c2
+;; fa: callq 0x2cd
;; ff: ud2
;; 101: movl $8, %esi
;; 106: movq %rbx, %rdi
-;; 109: callq 0x291
+;; 109: callq 0x29c
;; 10e: movq %rbx, %rdi
-;; 111: callq 0x2c2
+;; 111: callq 0x2cd
;; 116: ud2
;; 118: xorl %esi, %esi
;; 11a: movq %rbx, %rdi
-;; 11d: callq 0x291
+;; 11d: callq 0x29c
;; 122: movq %rbx, %rdi
-;; 125: callq 0x2c2
+;; 125: callq 0x2cd
;; 12a: ud2
;; 12c: ud2
;; 12e: ud2
diff --git a/tests/disas/trunc32.wat b/tests/disas/trunc32.wat
index 5d8db810db..09ad4f82a5 100644
--- a/tests/disas/trunc32.wat
+++ b/tests/disas/trunc32.wat
@@ -26,7 +26,7 @@
;; jp 0xf6
;; jne 0xf6
;; 46: movq %rbx, %rdi
-;; callq 0x262
+;; callq 0x26d
;; movabsq $13830554455654793216, %rax
;; movq %rax, %xmm1
;; ucomisd %xmm0, %xmm1
@@ -56,27 +56,27 @@
;; retq
;; c8: movl $6, %esi
;; cd: movq %rbx, %rdi
-;; d0: callq 0x28f
+;; d0: callq 0x29a
;; d5: movq %rbx, %rdi
-;; d8: callq 0x2c0
+;; d8: callq 0x2cb
;; dd: ud2
;; df: movl $6, %esi
;; e4: movq %rbx, %rdi
-;; e7: callq 0x28f
+;; e7: callq 0x29a
;; ec: movq %rbx, %rdi
-;; ef: callq 0x2c0
+;; ef: callq 0x2cb
;; f4: ud2
;; f6: movl $8, %esi
;; fb: movq %rbx, %rdi
-;; fe: callq 0x28f
+;; fe: callq 0x29a
;; 103: movq %rbx, %rdi
-;; 106: callq 0x2c0
+;; 106: callq 0x2cb
;; 10b: ud2
;; 10d: xorl %esi, %esi
;; 10f: movq %rbx, %rdi
-;; 112: callq 0x28f
+;; 112: callq 0x29a
;; 117: movq %rbx, %rdi
-;; 11a: callq 0x2c0
+;; 11a: callq 0x2cb
;; 11f: ud2
;; 121: ud2
;; 123: ud2
diff --git a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
index d285aa51f6..c1c654ba03 100644
--- a/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
+++ b/tests/disas/winch/aarch64/call_indirect/call_indirect.wat
@@ -85,7 +85,7 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28]
-;; bl #0x400
+;; bl #0x408
;; e0: add x28, x28, #4
;; mov sp, x28
;; ldur x9, [x28, #0x14]
@@ -151,7 +151,7 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28, #0xc]
-;; bl #0x400
+;; bl #0x408
;; 1e8: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #0x18]
diff --git a/tests/disas/winch/aarch64/call_indirect/local_arg.wat b/tests/disas/winch/aarch64/call_indirect/local_arg.wat
index 2b1fae177d..98dde19191 100644
--- a/tests/disas/winch/aarch64/call_indirect/local_arg.wat
+++ b/tests/disas/winch/aarch64/call_indirect/local_arg.wat
@@ -91,7 +91,7 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28]
-;; bl #0x440
+;; bl #0x450
;; 120: add x28, x28, #4
;; mov sp, x28
;; ldur x9, [x28, #0x14]
diff --git a/tests/disas/winch/aarch64/callee_pop/cleanup.wat b/tests/disas/winch/aarch64/callee_pop/cleanup.wat
index 9e7038da7d..0b9c11bd44 100644
--- a/tests/disas/winch/aarch64/callee_pop/cleanup.wat
+++ b/tests/disas/winch/aarch64/callee_pop/cleanup.wat
@@ -292,7 +292,7 @@
;; mov x0, x9
;; mov x1, #0
;; ldur w2, [x28, #0xc]
-;; bl #0xd04
+;; bl #0xd24
;; 43c: add x28, x28, #0x10
;; mov sp, x28
;; ldur x9, [x28, #0x68]
diff --git a/tests/disas/winch/aarch64/global-get-externref-drc.wat b/tests/disas/winch/aarch64/global-get-externref-drc.wat
index efb3afea01..a238d36af1 100644
--- a/tests/disas/winch/aarch64/global-get-externref-drc.wat
+++ b/tests/disas/winch/aarch64/global-get-externref-drc.wat
@@ -76,7 +76,7 @@
;; 108: sub x28, x28, #0xc
;; mov sp, x28
;; mov x0, x9
-;; bl #0x2a4
+;; bl #0x2ac
;; 118: add x28, x28, #0xc
;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12]
;; mov sp, x28
diff --git a/tests/disas/winch/x64/atomic/notify/notify.wat b/tests/disas/winch/x64/atomic/notify/notify.wat
index 60ebbcffd0..db40a9db22 100644
--- a/tests/disas/winch/x64/atomic/notify/notify.wat
+++ b/tests/disas/winch/x64/atomic/notify/notify.wat
@@ -28,7 +28,7 @@
;; movl $0, %esi
;; movq 8(%rsp), %rdx
;; movl 4(%rsp), %ecx
-;; callq 0x187
+;; callq 0x192
;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
diff --git a/tests/disas/winch/x64/atomic/notify/notify_offset.wat b/tests/disas/winch/x64/atomic/notify/notify_offset.wat
index 6bfa82fdcf..f6dfaf51c1 100644
--- a/tests/disas/winch/x64/atomic/notify/notify_offset.wat
+++ b/tests/disas/winch/x64/atomic/notify/notify_offset.wat
@@ -30,7 +30,7 @@
;; movl $0, %esi
;; movq 8(%rsp), %rdx
;; movl 4(%rsp), %ecx
-;; callq 0x196
+;; callq 0x1a1
;; addq $0x10, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
diff --git a/tests/disas/winch/x64/atomic/wait/wait32.wat b/tests/disas/winch/x64/atomic/wait/wait32.wat
index c13e8a212c..9f588836d0 100644
--- a/tests/disas/winch/x64/atomic/wait/wait32.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait32.wat
@@ -31,7 +31,7 @@
;; movq 0x18(%rsp), %rdx
;; movl 0x14(%rsp), %ecx
;; movq 0xc(%rsp), %r8
-;; callq 0x194
+;; callq 0x19f
;; addq $0x20, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
diff --git a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
index 073813049a..f266066ec9 100644
--- a/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait32_offset.wat
@@ -36,7 +36,7 @@
;; movq 0x18(%rsp), %rdx
;; movl 0x14(%rsp), %ecx
;; movq 0xc(%rsp), %r8
-;; callq 0x1a3
+;; callq 0x1ae
;; addq $0x20, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
diff --git a/tests/disas/winch/x64/atomic/wait/wait64.wat b/tests/disas/winch/x64/atomic/wait/wait64.wat
index 79f116ee12..13cadc18a9 100644
--- a/tests/disas/winch/x64/atomic/wait/wait64.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait64.wat
@@ -30,7 +30,7 @@
;; movq 0x18(%rsp), %rdx
;; movq 0x10(%rsp), %rcx
;; movq 8(%rsp), %r8
-;; callq 0x18c
+;; callq 0x197
;; addq $0x20, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
diff --git a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
index c8d7011da5..56a1d7acd6 100644
--- a/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
+++ b/tests/disas/winch/x64/atomic/wait/wait64_offset.wat
@@ -35,7 +35,7 @@
;; movq 0x18(%rsp), %rdx
;; movq 0x10(%rsp), %rcx
;; movq 8(%rsp), %r8
-;; callq 0x19b
+;; callq 0x1a6
;; addq $0x20, %rsp
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
diff --git a/tests/disas/winch/x64/call_indirect/call_indirect.wat b/tests/disas/winch/x64/call_indirect/call_indirect.wat
index 0af89e84c3..1308cb12a3 100644
--- a/tests/disas/winch/x64/call_indirect/call_indirect.wat
+++ b/tests/disas/winch/x64/call_indirect/call_indirect.wat
@@ -76,7 +76,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x33a
+;; callq 0x345
;; addq $0xc, %rsp
;; movq 0x1c(%rsp), %r14
;; jmp 0xdd
@@ -126,7 +126,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 4(%rsp), %edx
-;; callq 0x33a
+;; callq 0x345
;; addq $8, %rsp
;; movq 0x20(%rsp), %r14
;; jmp 0x1b4
diff --git a/tests/disas/winch/x64/call_indirect/local_arg.wat b/tests/disas/winch/x64/call_indirect/local_arg.wat
index aa960834c2..d9349c13f7 100644
--- a/tests/disas/winch/x64/call_indirect/local_arg.wat
+++ b/tests/disas/winch/x64/call_indirect/local_arg.wat
@@ -72,7 +72,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x347
+;; callq 0x35d
;; addq $0xc, %rsp
;; movq 0x1c(%rsp), %r14
;; jmp 0xfe
diff --git a/tests/disas/winch/x64/callee_pop/cleanup.wat b/tests/disas/winch/x64/callee_pop/cleanup.wat
index 1173a29162..fa7efcfb66 100644
--- a/tests/disas/winch/x64/callee_pop/cleanup.wat
+++ b/tests/disas/winch/x64/callee_pop/cleanup.wat
@@ -207,7 +207,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 4(%rsp), %edx
-;; callq 0xaa7
+;; callq 0xad3
;; addq $8, %rsp
;; movq 0x70(%rsp), %r14
;; jmp 0x307
diff --git a/tests/disas/winch/x64/epoch/func.wat b/tests/disas/winch/x64/epoch/func.wat
index 061edd761b..4eb2d54d0a 100644
--- a/tests/disas/winch/x64/epoch/func.wat
+++ b/tests/disas/winch/x64/epoch/func.wat
@@ -23,7 +23,7 @@
;; cmpq %rcx, %rdx
;; jb 0x54
;; 47: movq %r14, %rdi
-;; callq 0x14a
+;; callq 0x155
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
diff --git a/tests/disas/winch/x64/epoch/loop.wat b/tests/disas/winch/x64/epoch/loop.wat
index 3701d1ed48..3104bfdc36 100644
--- a/tests/disas/winch/x64/epoch/loop.wat
+++ b/tests/disas/winch/x64/epoch/loop.wat
@@ -25,7 +25,7 @@
;; cmpq %rcx, %rdx
;; jb 0x54
;; 47: movq %r14, %rdi
-;; callq 0x174
+;; callq 0x17f
;; movq 8(%rsp), %r14
;; movq 0x18(%r14), %rdx
;; movq (%rdx), %rdx
@@ -34,7 +34,7 @@
;; cmpq %rcx, %rdx
;; jb 0x79
;; 6c: movq %r14, %rdi
-;; callq 0x174
+;; callq 0x17f
;; movq 8(%rsp), %r14
;; jmp 0x54
;; 7e: addq $0x10, %rsp
diff --git a/tests/disas/winch/x64/fuel/call.wat b/tests/disas/winch/x64/fuel/call.wat
index 6ca0934728..bbdba40e66 100644
--- a/tests/disas/winch/x64/fuel/call.wat
+++ b/tests/disas/winch/x64/fuel/call.wat
@@ -28,7 +28,7 @@
;; cmpq $0, %rcx
;; jl 0x5e
;; 51: movq %r14, %rdi
-;; callq 0x204
+;; callq 0x20f
;; movq 8(%rsp), %r14
;; movq 8(%r14), %rax
;; movq (%rax), %r11
@@ -74,7 +74,7 @@
;; cmpq $0, %rcx
;; jl 0x10e
;; 101: movq %r14, %rdi
-;; callq 0x204
+;; callq 0x20f
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
diff --git a/tests/disas/winch/x64/fuel/func.wat b/tests/disas/winch/x64/fuel/func.wat
index cefba99a79..e5cb7ca47f 100644
--- a/tests/disas/winch/x64/fuel/func.wat
+++ b/tests/disas/winch/x64/fuel/func.wat
@@ -24,7 +24,7 @@
;; cmpq $0, %rcx
;; jl 0x5e
;; 51: movq %r14, %rdi
-;; callq 0x154
+;; callq 0x15f
;; movq 8(%rsp), %r14
;; addq $0x10, %rsp
;; popq %rbp
diff --git a/tests/disas/winch/x64/fuel/loop.wat b/tests/disas/winch/x64/fuel/loop.wat
index fcb24042b2..fd37cdce44 100644
--- a/tests/disas/winch/x64/fuel/loop.wat
+++ b/tests/disas/winch/x64/fuel/loop.wat
@@ -26,14 +26,14 @@
;; cmpq $0, %rcx
;; jl 0x5e
;; 51: movq %r14, %rdi
-;; callq 0x188
+;; callq 0x193
;; movq 8(%rsp), %r14
;; movq 8(%r14), %rcx
;; movq (%rcx), %rcx
;; cmpq $0, %rcx
;; jl 0x7c
;; 6f: movq %r14, %rdi
-;; callq 0x188
+;; callq 0x193
;; movq 8(%rsp), %r14
;; movq 8(%r14), %rax
;; movq (%rax), %r11
diff --git a/tests/disas/winch/x64/global-get-externref-drc.wat b/tests/disas/winch/x64/global-get-externref-drc.wat
index b1ef396271..62c8521fb1 100644
--- a/tests/disas/winch/x64/global-get-externref-drc.wat
+++ b/tests/disas/winch/x64/global-get-externref-drc.wat
@@ -62,7 +62,7 @@
;; jb 0xff
;; e4: subq $0xc, %rsp
;; movq %r14, %rdi
-;; callq 0x225
+;; callq 0x230
;; addq $0xc, %rsp
;; ╰─╼ stack_map: frame_size=32, frame_offsets=[12]
;; movq 0xc(%rsp), %r14
diff --git a/tests/disas/winch/x64/load/grow_load.wat b/tests/disas/winch/x64/load/grow_load.wat
index 3a186ecd41..f3abc05cc0 100644
--- a/tests/disas/winch/x64/load/grow_load.wat
+++ b/tests/disas/winch/x64/load/grow_load.wat
@@ -65,7 +65,7 @@
;; movq %r14, %rdi
;; movl 0xc(%rsp), %esi
;; movl $0, %edx
-;; callq 0x30e
+;; callq 0x319
;; addq $0x10, %rsp
;; movq 0x58(%rsp), %r14
;; movl %eax, %eax
diff --git a/tests/disas/winch/x64/table/fill.wat b/tests/disas/winch/x64/table/fill.wat
index 4084913c5e..fee990d447 100644
--- a/tests/disas/winch/x64/table/fill.wat
+++ b/tests/disas/winch/x64/table/fill.wat
@@ -113,7 +113,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x57a
+;; callq 0x5a6
;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
;; jmp 0x198
diff --git a/tests/disas/winch/x64/table/get.wat b/tests/disas/winch/x64/table/get.wat
index 4c1b76bbb6..aecfaa58c6 100644
--- a/tests/disas/winch/x64/table/get.wat
+++ b/tests/disas/winch/x64/table/get.wat
@@ -65,7 +65,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x315
+;; callq 0x32b
;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; jmp 0xfe
diff --git a/tests/disas/winch/x64/table/grow.wat b/tests/disas/winch/x64/table/grow.wat
index 1e8f64cc62..1b008c7b95 100644
--- a/tests/disas/winch/x64/table/grow.wat
+++ b/tests/disas/winch/x64/table/grow.wat
@@ -34,7 +34,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl (%rsp), %edx
-;; callq 0x22f
+;; callq 0x23a
;; addq $4, %rsp
;; movq 0x24(%rsp), %r14
;; movl (%rsp), %ecx
diff --git a/tests/disas/winch/x64/table/init_copy_drop.wat b/tests/disas/winch/x64/table/init_copy_drop.wat
index cf74d708f3..6c6531d40d 100644
--- a/tests/disas/winch/x64/table/init_copy_drop.wat
+++ b/tests/disas/winch/x64/table/init_copy_drop.wat
@@ -138,13 +138,13 @@
;; movq %rsi, (%rsp)
;; movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x115e
+;; callq 0x11ab
;; movq 8(%rsp), %r14
;; pushq %rax
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x1189
+;; callq 0x11d6
;; addq $8, %rsp
;; movq 0x10(%rsp), %r14
;; movq %r14, %r11
@@ -190,17 +190,17 @@
;; jmp 0x1f4
;; 256: movq %r14, %rdi
;; movl $0, %esi
-;; callq 0x11b4
+;; callq 0x1201
;; movq 8(%rsp), %r14
;; movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x115e
+;; callq 0x11ab
;; movq 8(%rsp), %r14
;; pushq %rax
;; subq $8, %rsp
;; movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x1189
+;; callq 0x11d6
;; addq $8, %rsp
;; movq 0x10(%rsp), %r14
;; movq %r14, %r11
@@ -246,7 +246,7 @@
;; jmp 0x2ed
;; 34f: movq %r14, %rdi
;; movl $1, %esi
-;; callq 0x11b4
+;; callq 0x1201
;; movq 8(%rsp), %r14
;; movl $5, %eax
;; movl $0xf, %ecx
@@ -305,7 +305,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x11f8
+;; callq 0x1245
;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
;; jmp 0x463
@@ -389,7 +389,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x11f8
+;; callq 0x1245
;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
;; jmp 0x5b7
@@ -473,7 +473,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x11f8
+;; callq 0x1245
;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
;; jmp 0x70b
@@ -557,7 +557,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x11f8
+;; callq 0x1245
;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
;; jmp 0x85f
@@ -641,7 +641,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movq 8(%rsp), %rdx
-;; callq 0x11f8
+;; callq 0x1245
;; addq $0x10, %rsp
;; movq 0x28(%rsp), %r14
;; jmp 0x9b3
@@ -751,7 +751,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 0xc(%rsp), %edx
-;; callq 0x11f8
+;; callq 0x1245
;; addq $0x10, %rsp
;; movq 0x18(%rsp), %r14
;; jmp 0xb24
diff --git a/tests/disas/winch/x64/table/set.wat b/tests/disas/winch/x64/table/set.wat
index 0ba26b3ae0..da4c75efc1 100644
--- a/tests/disas/winch/x64/table/set.wat
+++ b/tests/disas/winch/x64/table/set.wat
@@ -109,7 +109,7 @@
;; movq %r14, %rdi
;; movl $0, %esi
;; movl 8(%rsp), %edx
-;; callq 0x50a
+;; callq 0x52b
;; addq $0xc, %rsp
;; movq 0x1c(%rsp), %r14
;; jmp 0x1a2
diff --git a/tests/disas/x64-entry-trampoline.wat b/tests/disas/x64-entry-trampoline.wat
index ce6de4a91b..fe7c323552 100644
--- a/tests/disas/x64-entry-trampoline.wat
+++ b/tests/disas/x64-entry-trampoline.wat
@@ -36,13 +36,14 @@
;; retq
;; 77: movq (%rsp), %r8
;; 7b: movq $1, 0x88(%r8)
-;; 86: xorl %eax, %eax
-;; 88: movq 0x10(%rsp), %rbx
-;; 8d: movq 0x18(%rsp), %r12
-;; 92: movq 0x20(%rsp), %r13
-;; 97: movq 0x28(%rsp), %r14
-;; 9c: movq 0x30(%rsp), %r15
-;; a1: addq $0x40, %rsp
-;; a5: movq %rbp, %rsp
-;; a8: popq %rbp
-;; a9: retq
+;; 86: movq $0, 0x98(%r8)
+;; 91: xorl %eax, %eax
+;; 93: movq 0x10(%rsp), %rbx
+;; 98: movq 0x18(%rsp), %r12
+;; 9d: movq 0x20(%rsp), %r13
+;; a2: movq 0x28(%rsp), %r14
+;; a7: movq 0x30(%rsp), %r15
+;; ac: addq $0x40, %rsp
+;; b0: movq %rbp, %rsp
+;; b3: popq %rbp
+;; b4: retq
diff --git a/tests/disas/x64/call-indirect-with-gc.wat b/tests/disas/x64/call-indirect-with-gc.wat
index 9be1ac3559..dbbaba1db1 100644
--- a/tests/disas/x64/call-indirect-with-gc.wat
+++ b/tests/disas/x64/call-indirect-with-gc.wat
@@ -53,7 +53,7 @@
;; 8b: xorl %esi, %esi
;; 8d: movq %r9, %rbx
;; 90: movq %rbx, %rdi
-;; 93: callq 0x23b
+;; 93: callq 0x246
;; 98: jmp 0x59
;; 9d: ud2
;; 9f: ud2
diff --git a/tests/disas/x64/call-indirect-without-gc.wat b/tests/disas/x64/call-indirect-without-gc.wat
index ddcaa7a871..7fd4490a8e 100644
--- a/tests/disas/x64/call-indirect-without-gc.wat
+++ b/tests/disas/x64/call-indirect-without-gc.wat
@@ -53,7 +53,7 @@
;; 8b: xorl %esi, %esi
;; 8d: movq %r9, %rbx
;; 90: movq %rbx, %rdi
-;; 93: callq 0x23b
+;; 93: callq 0x246
;; 98: jmp 0x59
;; 9d: ud2
;; 9f: ud2
diff --git a/tests/misc_testsuite/component-model/lazy-lenders.wast b/tests/misc_testsuite/component-model/lazy-lenders.wast
new file mode 100644
index 0000000000..afa4e329d8
--- /dev/null
+++ b/tests/misc_testsuite/component-model/lazy-lenders.wast
@@ -0,0 +1,119 @@
+;;! multi_memory = true
+
+;; Exercise filling the store's `lazy_lenders` buffer, which should
+;; (transparently to the guest) trigger the `VMDeferredThread` to be forced.
+
+(component
+ (component $A
+ (type $t' (resource (rep i32)))
+ (export $t "t" (type $t'))
+
+ (core func $ctor (canon resource.new $t))
+
+ (core module $m
+ (memory (export "mem") 1)
+
+ ;; Bump allocator, keeping its cursor at address 0.
+ (func (export "realloc") (param i32 i32 i32 i32) (result i32)
+ (local $ret i32)
+ (local.set $ret (i32.load (i32.const 0)))
+ (if (i32.lt_u (local.get $ret) (i32.const 1024))
+ (then (local.set $ret (i32.const 1024))))
+ (i32.store (i32.const 0) (i32.add (local.get $ret) (local.get 3)))
+ (local.get $ret))
+
+ (func (export "consume") (param $ptr i32) (param $len i32) (result i32)
+ (local.get $len))
+ )
+ (core instance $i (instantiate $m))
+
+ (func (export "[constructor]t") (param "x" u32) (result (own $t))
+ (canon lift (core func $ctor)))
+ (func (export "consume") (param "xs" (list (borrow $t))) (result u32)
+ (canon lift (core func $i "consume")
+ (memory (core memory $i "mem"))
+ (realloc (core func $i "realloc"))))
+ )
+ (instance $a (instantiate $A))
+
+ (component $B
+ (import "a" (instance $i
+ (export "t" (type $t (sub resource)))
+ (export "[constructor]t" (func (param "x" u32) (result (own $t))))
+ (export "consume" (func (param "xs" (list (borrow $t))) (result u32)))
+ ))
+ (alias export $i "t" (type $t))
+ (alias export $i "[constructor]t" (func $ctor))
+ (alias export $i "consume" (func $consume))
+
+ (core module $mm
+ (memory (export "mem") 1)
+ (func (export "realloc") (param i32 i32 i32 i32) (result i32)
+ (local $ret i32)
+ (local.set $ret (i32.load (i32.const 0)))
+ (if (i32.lt_u (local.get $ret) (i32.const 1024))
+ (then (local.set $ret (i32.const 1024))))
+ (i32.store (i32.const 0) (i32.add (local.get $ret) (local.get 3)))
+ (local.get $ret))
+ )
+ (core instance $mi (instantiate $mm))
+
+ (core func $ctor' (canon lower (func $ctor)
+ (memory (core memory $mi "mem"))
+ (realloc (core func $mi "realloc"))))
+ (core func $consume' (canon lower (func $consume)
+ (memory (core memory $mi "mem"))
+ (realloc (core func $mi "realloc"))))
+ (core func $drop (canon resource.drop $t))
+
+ (core module $m
+ (import "" "ctor" (func $ctor (param i32) (result i32)))
+ (import "" "consume" (func $consume (param i32 i32) (result i32)))
+ (import "" "drop" (func $drop (param i32)))
+ (import "" "mem" (memory 1))
+
+ ;; Lend one handle `$n` times in a single call, then drop it. The drop
+ ;; only succeeds if every one of those lends was undone.
+ (func (export "run") (param $n i32) (result i32)
+ (local $h i32) (local $i i32) (local $v i32)
+ (local.set $h (call $ctor (i32.const 9)))
+ (block $done
+ (loop $loop
+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))
+ (i32.store
+ (i32.add (i32.const 128) (i32.mul (local.get $i) (i32.const 4)))
+ (local.get $h))
+ (local.set $i (i32.add (local.get $i) (i32.const 1)))
+ (br $loop)))
+ (local.set $v (call $consume (i32.const 128) (local.get $n)))
+ (call $drop (local.get $h))
+ (local.get $v))
+ )
+ (core instance $i2 (instantiate $m (with "" (instance
+ (export "ctor" (func $ctor'))
+ (export "consume" (func $consume'))
+ (export "drop" (func $drop))
+ (export "mem" (memory $mi "mem"))))))
+
+ (func (export "run") (param "n" u32) (result u32)
+ (canon lift (core func $i2 "run")))
+ )
+ (instance $b (instantiate $B (with "a" (instance $a))))
+ (export "run" (func $b "run"))
+)
+
+;; Within the buffer, so every lend is recorded lazily and discarded inline when
+;; the call returns.
+(assert_return (invoke "run" (u32.const 4)) (u32.const 4))
+
+;; Exactly fills the buffer without forcing.
+(assert_return (invoke "run" (u32.const 8)) (u32.const 8))
+
+;; Past the buffer's capacity, so the lift forces the thread partway through:
+;; the buffered lends are promoted into real ones and the rest are made
+;; directly, and every one of them must be undone by the time of the drop.
+(assert_return (invoke "run" (u32.const 9)) (u32.const 9))
+(assert_return (invoke "run" (u32.const 1000)) (u32.const 1000))
+
+;; Lazy and forced calls interleave without leaking lends between them.
+(assert_return (invoke "run" (u32.const 3)) (u32.const 3))