Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Inline to-owner borrow lends in sync-to-sync adapters (#15208)

Nick Fitzgerald3e0ee3fbfdd8
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index f5ea88a052..a709ba3d83 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -85,16 +85,9 @@ enum VmType {
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
)]
VMAsanStackSupport,
- #[allow(
- dead_code,
- reason = "generated uniformly for all VM types via `for_each_vm_type!`"
- )]
VMResourceIndex,
+ VMHandleTable,
VMHandleTableSlot,
- #[allow(
- dead_code,
- reason = "generated uniformly for all VM types via `for_each_vm_type!`"
- )]
VMTypedResource,
#[allow(
dead_code,
@@ -106,7 +99,6 @@ enum VmType {
reason = "generated uniformly for all VM types via `for_each_vm_type!`"
)]
VMTransmitLocalState,
- HandleTableSlotsArray,
}
/// A key that uniquely identifies an alias region across an entire compilation.
@@ -1561,16 +1553,6 @@ where
introducer
}
- /// Get the `Field` for an element of the out-of-line array that
- /// `VMStoreContext::handle_table_slots` points at.
- ///
- /// The returned `Field` is relative to the array's base pointer.
- pub fn handle_table_slots_array(&mut self) -> Field<'_, Offsets> {
- let ty = self.pointer_type;
- let flags = ir::MemFlagsData::trusted();
- Field::new(self, VmType::HandleTableSlotsArray, 0, flags, ty)
- }
-
/// Get the alias region shared by all memories that cross a module boundary
/// and whose definition we do not statically know.
pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {
diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rs
index 22839b5af7..22fe9322df 100644
--- a/crates/cranelift/src/compiler/component.rs
+++ b/crates/cranelift/src/compiler/component.rs
@@ -1262,11 +1262,11 @@ impl<'a> TrampolineCompiler<'a> {
.builder
.ins()
.iconst(ir::types::I32, i64::from(callee_instance.as_u32()));
- crate::component_sync_call::enter(
+ crate::func_environ::component_sync_call::enter(
&mut self.builder,
&mut self.alias_regions,
vmctx,
- crate::component_sync_call::EnterArgs {
+ crate::func_environ::component_sync_call::EnterArgs {
callee_async,
callee_instance,
},
@@ -1278,7 +1278,7 @@ impl<'a> TrampolineCompiler<'a> {
// Note that the helper here wants a core wasm vmctx, not a component
// one like `vmctx` is in this function.
let caller_vmctx = self.caller_vmctx();
- let slow = crate::component_sync_call::exit(
+ let slow = crate::func_environ::component_sync_call::exit(
&mut self.builder,
&mut self.alias_regions,
caller_vmctx,
@@ -1973,17 +1973,7 @@ impl TrampolineCompiler<'_> {
// Helper function to cast a core wasm input to a host pointer type
// which will go into the host libcall.
fn cast_to_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value {
- let pointer_type = self.isa.pointer_type();
- let host64 = pointer_type == ir::types::I64;
- if is64 == host64 {
- val
- } else if !is64 {
- assert!(host64);
- self.builder.ins().uextend(pointer_type, val)
- } else {
- assert!(!host64);
- self.builder.ins().ireduce(pointer_type, val)
- }
+ crate::cast_to_pointer(&mut self.builder, self.isa.pointer_type(), val, is64)
}
// Helper to cast a host pointer integer type to the destination type.
diff --git a/crates/cranelift/src/component_sync_call.rs b/crates/cranelift/src/component_sync_call.rs
deleted file mode 100644
index 30ee841449..0000000000
--- a/crates/cranelift/src/component_sync_call.rs
+++ /dev/null
@@ -1,230 +0,0 @@
-//! Shared translation of the component model's `{enter,exit}-sync-call`
-//! intrinsics.
-//!
-//! Two different places wrap a synchronous call into another component
-//! instance with these intrinsics: fused adapters, whose calls are translated
-//! in `func_environ.rs`, and the `resource.drop` trampoline's call to a
-//! resource destructor, translated in `compiler/component.rs`.
-//!
-//! Rather than call out to the host to do the task bookkeeping that entering a
-//! sync call entails, both push a `VMDeferredThread` into their own stack frame
-//! and publish it as the store's current thread. The host then only promotes it
-//! into a real `GuestTask` if it ever actually needs one; see
-//! `StoreOpaque::force_current_thread` and
-//! `StoreOpaque::force_deferred_current_thread`.
-//!
-//! The two callers differ only in where the arguments recorded by `enter` come
-//! from and in how `exit`'s out-of-line slow path is called. Everything else
-//! lives here: the `VMDeferredThread` layout and, in particular, the
-//! `context.{get,set}` slot save/zero/restore, which has to match what the host
-//! does for a non-deferred thread in `StoreOpaque::set_thread`.
-
-use crate::alias_region::AliasRegions;
-use cranelift_codegen::ir::condcodes::IntCC;
-use cranelift_codegen::ir::{self, InstBuilder};
-use cranelift_frontend::FunctionBuilder;
-use wasmtime_environ::{GetPtrSize, NUM_COMPONENT_CONTEXT_SLOTS, PtrSize};
-
-/// The `enter_sync_call` arguments recorded into the pushed
-/// `VMDeferredThread`, to be replayed by the host if it ever has to promote the
-/// deferred thread into a real one.
-pub struct EnterArgs {
- /// Whether the callee is async-lifted, as an `i32` boolean.
- pub callee_async: ir::Value,
- /// The component instance being called into.
- pub callee_instance: ir::Value,
-}
-
-/// Translates `enter-sync-call`: allocates a `VMDeferredThread` in the current
-/// function's frame, records `args` into it along with the caller's context
-/// slots, zeroes the live context slots for the callee, and publishes the frame
-/// as the store's current thread.
-///
-/// `vmctx` must be a core wasm `*mut VMContext` for this store. The returned
-/// stack slot must be handed to [`exit`] once the callee returns.
-pub fn enter<O>(
- builder: &mut FunctionBuilder<'_>,
- alias_regions: &mut AliasRegions<O>,
- vmctx: ir::Value,
- args: EnterArgs,
-) -> ir::StackSlot
-where
- O: GetPtrSize,
-{
- let pointer_type = alias_regions.pointer_type();
- let ptr = alias_regions.ptr_size();
-
- // Allocate the on-stack `VMDeferredThread`.
- let slot = builder.func.create_sized_stack_slot(ir::StackSlotData::new(
- ir::StackSlotKind::ExplicitSlot,
- u32::from(ptr.vm_deferred_thread().size()),
- u8::try_from(ptr.size().trailing_zeros()).unwrap(),
- ));
- let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);
- let vmstore = alias_regions
- .vmctx()
- .store_context()
- .load(&mut builder.cursor(), vmctx);
-
- // Link the previous current thread in as this frame's parent.
- let parent = alias_regions
- .vm_store_context()
- .current_thread()
- .load(&mut builder.cursor(), vmstore);
- alias_regions
- .vm_deferred_thread()
- .parent()
- .store(&mut builder.cursor(), slot_addr, parent);
-
- // Record the deferred `enter_sync_call` arguments.
- alias_regions.vm_deferred_thread().callee_async().store(
- &mut builder.cursor(),
- slot_addr,
- args.callee_async,
- );
- alias_regions.vm_deferred_thread().callee_instance().store(
- &mut builder.cursor(),
- slot_addr,
- args.callee_instance,
- );
-
- // Save the caller's context slots into the frame and reset the live values
- // to 0 for the freshly-entered (deferred) thread.
- for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
- let saved = alias_regions
- .vm_store_context()
- .component_context(i)
- .load(&mut builder.cursor(), vmstore);
- alias_regions.vm_deferred_thread().saved_context(i).store(
- &mut builder.cursor(),
- slot_addr,
- saved,
- );
- let zero = builder.ins().iconst(ir::types::I32, 0);
- alias_regions.vm_store_context().component_context(i).store(
- &mut builder.cursor(),
- vmstore,
- zero,
- );
- }
-
- // Record where this frame's lazy lends will start, so that `exit` can
- // discard them and the host can find exactly this frame's entries if it
- // has to promote them into real lends.
- let lazy_lenders_len = alias_regions
- .vm_store_context()
- .lazy_lenders_len()
- .load(&mut builder.cursor(), vmstore);
- alias_regions
- .vm_deferred_thread()
- .lazy_lenders_start()
- .store(&mut builder.cursor(), slot_addr, lazy_lenders_len);
-
- // Publish the deferred thread as the store's current thread.
- alias_regions.vm_store_context().current_thread().store(
- &mut builder.cursor(),
- vmstore,
- slot_addr,
- );
-
- slot
-}
-
-/// The unfinished slow path of an [`exit`], which the caller must fill in with
-/// an out-of-line call to the `exit_sync_call` intrinsic before finishing it
-/// with [`SlowExit::finish`].
-#[must_use = "the slow path of an exit-sync-call must be emitted and finished"]
-pub struct SlowExit {
- cont_block: ir::Block,
-}
-
-impl SlowExit {
- pub fn finish(self, builder: &mut FunctionBuilder<'_>) {
- builder.ins().jump(self.cont_block, &[]);
- builder.seal_block(self.cont_block);
- builder.switch_to_block(self.cont_block);
- }
-}
-
-/// Translates `exit-sync-call`, the counterpart to [`enter`], where `slot` is
-/// the stack slot that `enter` returned.
-///
-/// If the deferred thread pushed by `enter` is still the store's current
-/// thread, meaning nothing ever forced it into a real one, then it is popped,
-/// its lazy lends discarded, and the caller's context slots restored inline.
-/// Otherwise the host has state to tear down, which only it can do, so the
-/// caller must emit an out-of-line call to the `exit_sync_call` intrinsic: on
-/// return the builder is positioned in that slow path, which is finished with
-/// [`SlowExit::finish`].
-pub fn exit<O>(
- builder: &mut FunctionBuilder<'_>,
- alias_regions: &mut AliasRegions<O>,
- vmctx: ir::Value,
- slot: ir::StackSlot,
-) -> SlowExit
-where
- O: GetPtrSize,
-{
- let pointer_type = alias_regions.pointer_type();
- let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);
- let vmstore = alias_regions
- .vmctx()
- .store_context()
- .load(&mut builder.cursor(), vmctx);
- let cur = alias_regions
- .vm_store_context()
- .current_thread()
- .load(&mut builder.cursor(), vmstore);
- let is_current = builder.ins().icmp(IntCC::Equal, cur, slot_addr);
-
- let fast_block = builder.create_block();
- let slow_block = builder.create_block();
- let cont_block = builder.create_block();
- builder
- .ins()
- .brif(is_current, fast_block, &[], slow_block, &[]);
- builder.seal_block(fast_block);
- builder.seal_block(slow_block);
-
- // Fast path: pop the deferred thread and restore the caller's context.
- builder.switch_to_block(fast_block);
- let parent = alias_regions
- .vm_deferred_thread()
- .parent()
- .load(&mut builder.cursor(), slot_addr);
- alias_regions
- .vm_store_context()
- .current_thread()
- .store(&mut builder.cursor(), vmstore, parent);
-
- // Discard the lazy lends this frame recorded. We can do this by simply
- // resetting the `lazy_lenders_len`, and without any per-lend work, since
- // the deferred thread and its lazy lends were never materialized.
- let start = alias_regions
- .vm_deferred_thread()
- .lazy_lenders_start()
- .load(&mut builder.cursor(), slot_addr);
- alias_regions.vm_store_context().lazy_lenders_len().store(
- &mut builder.cursor(),
- vmstore,
- start,
- );
-
- for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
- let saved = alias_regions
- .vm_deferred_thread()
- .saved_context(i)
- .load(&mut builder.cursor(), slot_addr);
- alias_regions.vm_store_context().component_context(i).store(
- &mut builder.cursor(),
- vmstore,
- saved,
- );
- }
- builder.ins().jump(cont_block, &[]);
-
- // Slow path: the thread was promoted to a real one, so the caller does the
- // equivalent teardown out-of-line.
- builder.switch_to_block(slow_block);
- SlowExit { cont_block }
-}
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index a6a2d26854..aed69be2d4 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -1,3 +1,4 @@
+pub(crate) mod component_sync_call;
mod gc;
pub(crate) mod stack_switching;
@@ -268,10 +269,6 @@ pub struct FuncEnvironment<'module_environment> {
/// Alias regions for the enclosing component's offsets.
///
/// Present only when this module is being compiled as part of a component.
- #[allow(
- dead_code,
- reason = "read by the fused-adapter fast paths in a follow-up commit"
- )]
pub(crate) component_regions:
Option<AliasRegions<wasmtime_environ::component::VMComponentOffsets<u8>>>,
}
@@ -2302,6 +2299,19 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
&real_call_args,
)));
}
+
+ #[cfg(feature = "component-model")]
+ FactInlineIntrinsic::Component(intrinsic) => {
+ if let Some(rets) = self.lower_fact_component_intrinsic(
+ *intrinsic,
+ callee_index,
+ sig_ref,
+ &real_call_args,
+ ) {
+ return Ok(Reachability::Reachable(rets));
+ }
+ }
+
FactInlineIntrinsic::EnterSyncCall | FactInlineIntrinsic::ExitSyncCall => {}
}
let import_off = self.env.offsets.imported_functions().at(callee_index);
@@ -2364,11 +2374,11 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
/// callee_instance]`.
fn lower_fact_enter_sync_call(&mut self, real_call_args: &[ir::Value]) -> CallRets {
let vmctx = self.env.vmctx_val(&mut self.builder.cursor());
- let slot = crate::component_sync_call::enter(
+ let slot = component_sync_call::enter(
self.builder,
&mut self.env.alias_regions,
vmctx,
- crate::component_sync_call::EnterArgs {
+ component_sync_call::EnterArgs {
callee_async: real_call_args[2],
callee_instance: real_call_args[3],
},
@@ -2379,6 +2389,44 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
CallRets::new()
}
+ /// Inline lowering of a FACT adapter's component-model-specific
+ /// intrinsics.
+ ///
+ /// Returns `None` when the intrinsic must instead be called out-of-line.
+ #[cfg(feature = "component-model")]
+ fn lower_fact_component_intrinsic(
+ &mut self,
+ intrinsic: wasmtime_environ::component::FactComponentIntrinsic,
+ callee_index: FuncIndex,
+ sig_ref: ir::SigRef,
+ real_call_args: &[ir::Value],
+ ) -> Option<CallRets> {
+ use wasmtime_environ::component::FactComponentIntrinsic;
+ match intrinsic {
+ FactComponentIntrinsic::ResourceTransferBorrowToOwner { ty, instance } => {
+ // Needs an enclosing deferred frame to record into.
+ if !self.env.tunables.concurrency_support || self.env.fact_sync_call_slot.is_none()
+ {
+ return None;
+ }
+ debug_assert!(!self.tail);
+ let rep = component_sync_call::transfer_borrow_to_owner(
+ self.builder,
+ self.env,
+ self.srcloc,
+ ty,
+ instance,
+ callee_index,
+ sig_ref,
+ real_call_args,
+ );
+ let mut out = CallRets::new();
+ out.push(rep);
+ Some(out)
+ }
+ }
+ }
+
/// Inline lowering of a FACT adapter's `exit-sync-call` intrinsic, the
/// counterpart to `lower_fact_enter_sync_call`.
fn lower_fact_exit_sync_call(
@@ -2393,12 +2441,8 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
.take()
.expect("inline exit-sync-call without a matching enter-sync-call");
let vmctx = self.env.vmctx_val(&mut self.builder.cursor());
- let slow = crate::component_sync_call::exit(
- self.builder,
- &mut self.env.alias_regions,
- vmctx,
- slot,
- );
+ let slow =
+ component_sync_call::exit(self.builder, &mut self.env.alias_regions, vmctx, slot);
let import_off = self.env.offsets.imported_functions().at(callee_index);
let func_addr = self
diff --git a/crates/cranelift/src/func_environ/component_sync_call.rs b/crates/cranelift/src/func_environ/component_sync_call.rs
new file mode 100644
index 0000000000..d015c2c506
--- /dev/null
+++ b/crates/cranelift/src/func_environ/component_sync_call.rs
@@ -0,0 +1,540 @@
+//! Shared translation of the component model's `{enter,exit}-sync-call`
+//! intrinsics, and of the `resource.transfer-borrow-to-owner` that fused
+//! adapters make between them.
+//!
+//! Two different places wrap a synchronous call into another component
+//! instance with these intrinsics: fused adapters, whose calls are translated
+//! in `func_environ.rs`, and the `resource.drop` trampoline's call to a
+//! resource destructor, translated in `compiler/component.rs`.
+//!
+//! Rather than call out to the host to do the task bookkeeping that entering a
+//! sync call entails, both push a `VMDeferredThread` into their own stack frame
+//! and publish it as the store's current thread. The host then only promotes it
+//! into a real `GuestTask` if it ever actually needs one; see
+//! `StoreOpaque::force_current_thread` and
+//! `StoreOpaque::force_deferred_current_thread`.
+//!
+//! The two callers differ only in where the arguments recorded by `enter` come
+//! from and in how `exit`'s out-of-line slow path is called. Everything else
+//! lives here: the `VMDeferredThread` layout and, in particular, the
+//! `context.{get,set}` slot save/zero/restore, which has to match what the host
+//! does for a non-deferred thread in `StoreOpaque::set_thread`.
+//!
+//! Between `enter` and `exit`, a fused adapter lending a `borrow` to the
+//! resource's owning instance uses `transfer_borrow_to_owner`. Instead of
+//! creating a real lend, it records the handle in the store's lazy lenders,
+//! which `exit` discards wholesale and the host promotes into real lends only
+//! if it forces the deferred thread. Anything else, such as a promoted thread,
+//! a full lazy lenders array, or an invalid handle, falls back to the libcall.
+
+use super::{Call, FuncEnvironment};
+use crate::alias_region::AliasRegions;
+use crate::trap::TranslateTrap;
+use cranelift_codegen::ir::condcodes::IntCC;
+use cranelift_codegen::ir::{self, InstBuilder};
+use cranelift_frontend::FunctionBuilder;
+use wasmtime_environ::{
+ FuncIndex, GetPtrSize, LAZY_LENDERS_CAPACITY, NUM_COMPONENT_CONTEXT_SLOTS, PtrSize,
+ component::{RuntimeComponentInstanceIndex, TypeResourceTableIndex, VMComponentOffsets},
+};
+
+/// The `enter_sync_call` arguments recorded into the pushed
+/// `VMDeferredThread`, to be replayed by the host if it ever has to promote the
+/// deferred thread into a real one.
+pub struct EnterArgs {
+ /// Whether the callee is async-lifted, as an `i32` boolean.
+ pub callee_async: ir::Value,
+ /// The component instance being called into.
+ pub callee_instance: ir::Value,
+}
+
+/// Translates `enter-sync-call`: allocates a `VMDeferredThread` in the current
+/// function's frame, records `args` into it along with the caller's context
+/// slots, zeroes the live context slots for the callee, and publishes the frame
+/// as the store's current thread.
+///
+/// `vmctx` must be a core wasm `*mut VMContext` for this store. The returned
+/// stack slot must be handed to [`exit`] once the callee returns.
+pub fn enter<O>(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<O>,
+ vmctx: ir::Value,
+ args: EnterArgs,
+) -> ir::StackSlot
+where
+ O: GetPtrSize,
+{
+ let pointer_type = alias_regions.pointer_type();
+ let ptr = alias_regions.ptr_size();
+
+ // Allocate the on-stack `VMDeferredThread`.
+ let slot = builder.func.create_sized_stack_slot(ir::StackSlotData::new(
+ ir::StackSlotKind::ExplicitSlot,
+ u32::from(ptr.vm_deferred_thread().size()),
+ u8::try_from(ptr.size().trailing_zeros()).unwrap(),
+ ));
+ let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);
+ let vmstore = alias_regions
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx);
+
+ // Link the previous current thread in as this frame's parent.
+ let parent = alias_regions
+ .vm_store_context()
+ .current_thread()
+ .load(&mut builder.cursor(), vmstore);
+ alias_regions
+ .vm_deferred_thread()
+ .parent()
+ .store(&mut builder.cursor(), slot_addr, parent);
+
+ // Record the deferred `enter_sync_call` arguments.
+ alias_regions.vm_deferred_thread().callee_async().store(
+ &mut builder.cursor(),
+ slot_addr,
+ args.callee_async,
+ );
+ alias_regions.vm_deferred_thread().callee_instance().store(
+ &mut builder.cursor(),
+ slot_addr,
+ args.callee_instance,
+ );
+
+ // Save the caller's context slots into the frame and reset the live values
+ // to 0 for the freshly-entered (deferred) thread.
+ for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
+ let saved = alias_regions
+ .vm_store_context()
+ .component_context(i)
+ .load(&mut builder.cursor(), vmstore);
+ alias_regions.vm_deferred_thread().saved_context(i).store(
+ &mut builder.cursor(),
+ slot_addr,
+ saved,
+ );
+ let zero = builder.ins().iconst(ir::types::I32, 0);
+ alias_regions.vm_store_context().component_context(i).store(
+ &mut builder.cursor(),
+ vmstore,
+ zero,
+ );
+ }
+
+ // Record where this frame's lazy lends will start, so that `exit` can
+ // discard them and the host can find exactly this frame's entries if it
+ // has to promote them into real lends.
+ let lazy_lenders_len = alias_regions
+ .vm_store_context()
+ .lazy_lenders_len()
+ .load(&mut builder.cursor(), vmstore);
+ alias_regions
+ .vm_deferred_thread()
+ .lazy_lenders_start()
+ .store(&mut builder.cursor(), slot_addr, lazy_lenders_len);
+
+ // Publish the deferred thread as the store's current thread.
+ alias_regions.vm_store_context().current_thread().store(
+ &mut builder.cursor(),
+ vmstore,
+ slot_addr,
+ );
+
+ slot
+}
+
+/// The unfinished slow path of an [`exit`], which the caller must fill in with
+/// an out-of-line call to the `exit_sync_call` intrinsic before finishing it
+/// with [`SlowExit::finish`].
+#[must_use = "the slow path of an exit-sync-call must be emitted and finished"]
+pub struct SlowExit {
+ cont_block: ir::Block,
+}
+
+impl SlowExit {
+ pub fn finish(self, builder: &mut FunctionBuilder<'_>) {
+ builder.ins().jump(self.cont_block, &[]);
+ builder.seal_block(self.cont_block);
+ builder.switch_to_block(self.cont_block);
+ }
+}
+
+/// Translates `exit-sync-call`, the counterpart to [`enter`], where `slot` is
+/// the stack slot that `enter` returned.
+///
+/// If the deferred thread pushed by `enter` is still the store's current
+/// thread, meaning nothing ever forced it into a real one, then it is popped,
+/// its lazy lends discarded, and the caller's context slots restored inline.
+/// Otherwise the host has state to tear down, which only it can do, so the
+/// caller must emit an out-of-line call to the `exit_sync_call` intrinsic: on
+/// return the builder is positioned in that slow path, which is finished with
+/// [`SlowExit::finish`].
+pub fn exit<O>(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<O>,
+ vmctx: ir::Value,
+ slot: ir::StackSlot,
+) -> SlowExit
+where
+ O: GetPtrSize,
+{
+ let pointer_type = alias_regions.pointer_type();
+ let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);
+ let vmstore = alias_regions
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx);
+ let cur = alias_regions
+ .vm_store_context()
+ .current_thread()
+ .load(&mut builder.cursor(), vmstore);
+ let is_current = builder.ins().icmp(IntCC::Equal, cur, slot_addr);
+
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let cont_block = builder.create_block();
+ builder
+ .ins()
+ .brif(is_current, fast_block, &[], slow_block, &[]);
+ builder.seal_block(fast_block);
+ builder.seal_block(slow_block);
+
+ // Fast path: pop the deferred thread and restore the caller's context.
+ builder.switch_to_block(fast_block);
+ let parent = alias_regions
+ .vm_deferred_thread()
+ .parent()
+ .load(&mut builder.cursor(), slot_addr);
+ alias_regions
+ .vm_store_context()
+ .current_thread()
+ .store(&mut builder.cursor(), vmstore, parent);
+
+ // Discard the lazy lends this frame recorded. We can do this by simply
+ // resetting the `lazy_lenders_len`, and without any per-lend work, since
+ // the deferred thread and its lazy lends were never materialized.
+ let start = alias_regions
+ .vm_deferred_thread()
+ .lazy_lenders_start()
+ .load(&mut builder.cursor(), slot_addr);
+ alias_regions.vm_store_context().lazy_lenders_len().store(
+ &mut builder.cursor(),
+ vmstore,
+ start,
+ );
+
+ for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
+ let saved = alias_regions
+ .vm_deferred_thread()
+ .saved_context(i)
+ .load(&mut builder.cursor(), slot_addr);
+ alias_regions.vm_store_context().component_context(i).store(
+ &mut builder.cursor(),
+ vmstore,
+ saved,
+ );
+ }
+ builder.ins().jump(cont_block, &[]);
+
+ // Slow path: the thread was promoted to a real one, so the caller does the
+ // equivalent teardown out-of-line.
+ builder.switch_to_block(slow_block);
+ builder.set_cold_block(slow_block);
+ SlowExit { cont_block }
+}
+
+/// Inline `resource.transfer-borrow-to-owner` in a FACT adapter, returning the
+/// borrowed resource's `rep`.
+///
+/// This is the inline equivalent of
+/// `ResourceTables::resource_lift_borrow_lazy`, falling back to calling the
+/// imported intrinsic `callee_index` with `real_call_args` when the fast path
+/// does not apply. `real_call_args` is `[callee_vmctx, caller_vmctx,
+/// handle_index]`.
+pub fn transfer_borrow_to_owner(
+ builder: &mut FunctionBuilder<'_>,
+ env: &mut FuncEnvironment<'_>,
+ srcloc: ir::SourceLoc,
+ ty: TypeResourceTableIndex,
+ instance: RuntimeComponentInstanceIndex,
+ callee_index: FuncIndex,
+ sig_ref: ir::SigRef,
+ real_call_args: &[ir::Value],
+) -> ir::Value {
+ let slot = env
+ .fact_sync_call_slot
+ .expect("inline to-owner borrow without an enclosing enter-sync-call");
+ let vmctx = env.vmctx_val(&mut builder.cursor());
+ let vmcomponent = real_call_args[0];
+ let index = real_call_args[2];
+
+ let alias_regions = env
+ .component_regions
+ .as_mut()
+ .expect("an adapter's to-owner borrow requires the component's offsets");
+ let pointer_type = alias_regions.pointer_type();
+ let layout = HandleTableSlotLayout::new(alias_regions);
+ let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);
+ let vmstore = alias_regions
+ .vmctx()
+ .store_context()
+ .load(&mut builder.cursor(), vmctx);
+
+ let table_offset = alias_regions.offsets().handle_tables().at(instance);
+
+ let slow_block = builder.create_block();
+ let join_block = builder.create_block();
+ builder.append_block_param(join_block, ir::types::I32);
+
+ let is_current = is_current_thread(builder, alias_regions, vmstore, slot_addr);
+ branch_to_slow_path_unless(builder, is_current, slow_block);
+
+ let (len, has_room) = lazy_lenders_has_room(builder, alias_regions, vmstore);
+ branch_to_slow_path_unless(builder, has_room, slow_block);
+
+ let (slot_index, in_bounds) =
+ bounds_check_handle_table_slot(builder, alias_regions, vmcomponent, table_offset, index);
+ branch_to_slow_path_unless(builder, in_bounds, slow_block);
+
+ // The slot is in bounds, so it is safe to read; now check that it holds a
+ // resource of the expected type.
+ let slot_ptr = handle_table_slot_addr(
+ builder,
+ alias_regions,
+ vmcomponent,
+ table_offset,
+ slot_index,
+ layout.size,
+ );
+ let is_resource = handle_table_slot_is_resource(builder, alias_regions, &layout, slot_ptr);
+ branch_to_slow_path_unless(builder, is_resource, slow_block);
+
+ let has_type = handle_table_slot_has_type(builder, alias_regions, &layout, slot_ptr, ty);
+ branch_to_slow_path_unless(builder, has_type, slow_block);
+
+ let rep = alias_regions
+ .vm_typed_resource()
+ .rep()
+ .relative_to(layout.resource_offset)
+ .load(&mut builder.cursor(), slot_ptr);
+ push_lazy_lender(builder, alias_regions, vmstore, len, ty, index);
+ builder.ins().jump(join_block, &[rep.into()]);
+
+ builder.switch_to_block(slow_block);
+ builder.set_cold_block(slow_block);
+ builder.seal_block(slow_block);
+ let import_off = env.offsets.imported_functions().at(callee_index);
+ let func_addr = env
+ .alias_regions
+ .vm_function_import()
+ .wasm_call()
+ .relative_to(import_off)
+ .load(&mut builder.cursor(), vmctx);
+ let rets =
+ Call::new(builder, env, srcloc).indirect_call_inst(sig_ref, func_addr, real_call_args);
+ builder.ins().jump(join_block, &[rets[0].into()]);
+
+ builder.switch_to_block(join_block);
+ builder.seal_block(join_block);
+ builder.block_params(join_block)[0]
+}
+
+/// The layout of a `VMHandleTableSlot` that the inline resource fast paths
+/// rely on.
+struct HandleTableSlotLayout {
+ /// The size of a slot, in bytes.
+ size: i64,
+ /// The offset of the `VMTypedResource` within either resource variant.
+ resource_offset: u32,
+ /// The tag of the `resource_own` variant.
+ own_tag: u32,
+ /// The tag of the `resource_borrow` variant.
+ borrow_tag: u32,
+}
+
+impl HandleTableSlotLayout {
+ fn new(alias_regions: &AliasRegions<VMComponentOffsets<u8>>) -> Self {
+ let slot = alias_regions.ptr_size().vm_handle_table_slot();
+ debug_assert_eq!(
+ slot.resource_own().resource(),
+ slot.resource_borrow().resource(),
+ "both resource variants must put `resource` at the same offset, \
+ so that the fast path need not branch on the tag",
+ );
+ let own_tag = slot.resource_own().tag_value();
+ let borrow_tag = slot.resource_borrow().tag_value();
+ debug_assert_eq!(borrow_tag, own_tag + 1);
+ Self {
+ size: i64::from(slot.size()),
+ resource_offset: u32::from(slot.resource_own().resource()),
+ own_tag,
+ borrow_tag,
+ }
+ }
+}
+
+/// Whether the `VMDeferredThread` at `slot_addr` is the store's current thread.
+fn is_current_thread(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
+ vmstore: ir::Value,
+ slot_addr: ir::Value,
+) -> ir::Value {
+ let cur = alias_regions
+ .vm_store_context()
+ .current_thread()
+ .load(&mut builder.cursor(), vmstore);
+ builder.ins().icmp(IntCC::Equal, cur, slot_addr)
+}
+
+/// Returns the store's current `lazy_lenders_len` and whether there is room to
+/// push another lazy lend.
+fn lazy_lenders_has_room(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
+ vmstore: ir::Value,
+) -> (ir::Value, ir::Value) {
+ let pointer_type = alias_regions.pointer_type();
+ let len = alias_regions
+ .vm_store_context()
+ .lazy_lenders_len()
+ .load(&mut builder.cursor(), vmstore);
+ let capacity = i64::try_from(LAZY_LENDERS_CAPACITY).unwrap();
+ let cap = builder.ins().iconst(pointer_type, capacity);
+ let has_room = builder.ins().icmp(IntCC::UnsignedLessThan, len, cap);
+ (len, has_room)
+}
+
+/// Converts the 1-based handle `index` into a slot index for the handle table
+/// at `table_offset`, returning that slot index and whether it is in bounds.
+fn bounds_check_handle_table_slot(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
+ vmcomponent: ir::Value,
+ table_offset: u32,
+ index: ir::Value,
+) -> (ir::Value, ir::Value) {
+ let pointer_type = alias_regions.pointer_type();
+ // Handle 0 wraps to `u32::MAX` and fails the bounds check.
+ let slot_index = builder.ins().iadd_imm_s(index, -1);
+ let slot_index = crate::cast_to_pointer(builder, pointer_type, slot_index, false);
+ let slots_len = alias_regions
+ .vm_handle_table()
+ .current_slots()
+ .relative_to(table_offset)
+ .load(&mut builder.cursor(), vmcomponent);
+ let in_bounds = builder
+ .ins()
+ .icmp(IntCC::UnsignedLessThan, slot_index, slots_len);
+ (slot_index, in_bounds)
+}
+
+/// The address of slot `slot_index` in the handle table at `table_offset`,
+/// which must already be bounds checked.
+fn handle_table_slot_addr(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
+ vmcomponent: ir::Value,
+ table_offset: u32,
+ slot_index: ir::Value,
+ slot_size: i64,
+) -> ir::Value {
+ let slots = alias_regions
+ .vm_handle_table()
+ .base()
+ .relative_to(table_offset)
+ .load(&mut builder.cursor(), vmcomponent);
+ let slot_offset = builder.ins().imul_imm_u(slot_index, slot_size);
+ builder.ins().iadd(slots, slot_offset)
+}
+
+/// Continues in a new block if `cond` is true, otherwise branches to
+/// `slow_block`.
+fn branch_to_slow_path_unless(
+ builder: &mut FunctionBuilder<'_>,
+ cond: ir::Value,
+ slow_block: ir::Block,
+) {
+ let next_block = builder.create_block();
+ builder.ins().brif(cond, next_block, &[], slow_block, &[]);
+ builder.seal_block(next_block);
+ builder.switch_to_block(next_block);
+}
+
+/// Whether the handle table slot at `slot_ptr` holds an owned or borrowed
+/// resource.
+fn handle_table_slot_is_resource(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
+ layout: &HandleTableSlotLayout,
+ slot_ptr: ir::Value,
+) -> ir::Value {
+ // The own and borrow tags are adjacent, so one unsigned comparison checks
+ // for either.
+ let tag = alias_regions
+ .vm_handle_table_slot()
+ .tag()
+ .load(&mut builder.cursor(), slot_ptr);
+ let tag = builder.ins().iadd_imm_s(tag, -i64::from(layout.own_tag));
+ builder.ins().icmp_imm_u(
+ IntCC::UnsignedLessThanOrEqual,
+ tag,
+ i64::from(layout.borrow_tag - layout.own_tag),
+ )
+}
+
+/// Whether the resource in the handle table slot at `slot_ptr` has type `ty`.
+///
+/// The slot must already be known to hold a resource.
+fn handle_table_slot_has_type(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
+ layout: &HandleTableSlotLayout,
+ slot_ptr: ir::Value,
+ ty: TypeResourceTableIndex,
+) -> ir::Value {
+ let slot_ty = alias_regions
+ .vm_typed_resource()
+ .ty()
+ .relative_to(layout.resource_offset)
+ .load(&mut builder.cursor(), slot_ptr);
+ builder
+ .ins()
+ .icmp_imm_u(IntCC::Equal, slot_ty, i64::from(ty.as_u32()))
+}
+
+/// Records a lend of handle `index` at position `len` in the store's lazy
+/// lenders, for the host to promote if it forces the current thread.
+///
+/// The caller must have already checked that there is room.
+fn push_lazy_lender(
+ builder: &mut FunctionBuilder<'_>,
+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
+ vmstore: ir::Value,
+ len: ir::Value,
+ ty: TypeResourceTableIndex,
+ index: ir::Value,
+) {
+ let entry_size = i64::from(alias_regions.ptr_size().vm_resource_index().size());
+ let lazy_lenders = alias_regions
+ .vm_store_context()
+ .lazy_lenders()
+ .load(&mut builder.cursor(), vmstore);
+ let entry_offset = builder.ins().imul_imm_u(len, entry_size);
+ let entry_addr = builder.ins().iadd(lazy_lenders, entry_offset);
+ let entry_ty = builder.ins().iconst(ir::types::I32, i64::from(ty.as_u32()));
+ alias_regions
+ .vm_resource_index()
+ .ty()
+ .store(&mut builder.cursor(), entry_addr, entry_ty);
+ alias_regions
+ .vm_resource_index()
+ .index()
+ .store(&mut builder.cursor(), entry_addr, index);
+ let next_len = builder.ins().iadd_imm_s(len, 1);
+ alias_regions.vm_store_context().lazy_lenders_len().store(
+ &mut builder.cursor(),
+ vmstore,
+ next_len,
+ );
+}
diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs
index 1664aad6f9..837c9cbe79 100644
--- a/crates/cranelift/src/lib.rs
+++ b/crates/cranelift/src/lib.rs
@@ -22,6 +22,7 @@ use cranelift_codegen::{
settings,
};
use cranelift_entity::PrimaryMap;
+use cranelift_frontend::FunctionBuilder;
use target_lexicon::Architecture;
use wasmtime_environ::{
@@ -41,7 +42,6 @@ mod alias_region;
mod bounds_checks;
mod builder;
mod compiler;
-mod component_sync_call;
mod debug;
mod func_environ;
mod translate;
@@ -158,6 +158,26 @@ fn unbarriered_load_type_at_offset(
pos.ins().load(ir_ty, flags, base, offset)
}
+/// Casts `val`, an `i64` if `is64` and an `i32` otherwise, to `pointer_type`.
+fn cast_to_pointer(
+ builder: &mut FunctionBuilder<'_>,
+ pointer_type: ir::Type,
+ val: ir::Value,
+ val_is_64: bool,
+) -> ir::Value {
+ debug_assert!(pointer_type == ir::types::I32 || pointer_type == ir::types::I64);
+ let pointer_is_64 = pointer_type == ir::types::I64;
+ if val_is_64 == pointer_is_64 {
+ val
+ } else if !val_is_64 {
+ assert!(pointer_is_64);
+ builder.ins().uextend(pointer_type, val)
+ } else {
+ assert!(!pointer_is_64);
+ builder.ins().ireduce(pointer_type, val)
+ }
+}
+
/// Returns the corresponding cranelift type for the provided wasm type.
fn value_type(isa: &dyn TargetIsa, ty: WasmValType) -> ir::types::Type {
match ty {
diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rs
index 9955980a7e..e343b4dd93 100644
--- a/crates/environ/src/compile/module_environ.rs
+++ b/crates/environ/src/compile/module_environ.rs
@@ -45,12 +45,18 @@ pub struct ModuleEnvironment<'a, 'data> {
pub enum FactInlineIntrinsic {
/// `enter-sync-call`: push a deferred component-model thread inline.
EnterSyncCall,
+
/// `exit-sync-call`: pop the deferred thread inline on the fast path, or
/// fall back to the out-of-line `exit-sync-call` libcall when the thread
/// was promoted.
ExitSyncCall,
+
/// `trap`: raise the given trap.
Trap(Trap),
+
+ /// An intrinsic that only exists for components.
+ #[cfg(feature = "component-model")]
+ Component(crate::component::FactComponentIntrinsic),
}
/// A statically-known function import.
diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rs
index 2bfc28a361..facef8986f 100644
--- a/crates/environ/src/component/compiler.rs
+++ b/crates/environ/src/component/compiler.rs
@@ -1,6 +1,29 @@
-use crate::component::{ComponentTranslation, ComponentTypesBuilder, UnsafeIntrinsic};
+use crate::component::{
+ ComponentTranslation, ComponentTypesBuilder, RuntimeComponentInstanceIndex,
+ TypeResourceTableIndex, UnsafeIntrinsic,
+};
use crate::error::Result;
-use crate::{Abi, CompiledFunctionBody, FuncKey, Tunables};
+use crate::{Abi, CompiledFunctionBody, FactInlineIntrinsic, FuncKey, Tunables};
+
+/// The component-model-specific `FactInlineIntrinsic`s.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum FactComponentIntrinsic {
+ /// `resource.transfer-borrow-to-owner`: lazily lend a `borrow` to its
+ /// resource's owning instance, or fall back to the libcall.
+ ResourceTransferBorrowToOwner {
+ /// The resource type of the handle being lent.
+ ty: TypeResourceTableIndex,
+ /// The component instance owning `ty`'s handle table.
+ instance: RuntimeComponentInstanceIndex,
+ },
+}
+
+impl From<FactComponentIntrinsic> for FactInlineIntrinsic {
+ #[inline]
+ fn from(intrinsic: FactComponentIntrinsic) -> Self {
+ Self::Component(intrinsic)
+ }
+}
/// Compilation support necessary for components.
pub trait ComponentCompiler: Send + Sync {
diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rs
index a1b57c07d4..d5bf6f3844 100644
--- a/crates/environ/src/component/translate.rs
+++ b/crates/environ/src/component/translate.rs
@@ -705,6 +705,17 @@ impl<'a, 'data> Translator<'a, 'data> {
FactInlineIntrinsic::ExitSyncCall.into()
}
Trampoline::Trap(trap) => FactInlineIntrinsic::Trap(trap).into(),
+ Trampoline::ResourceTransferBorrowToOwner { ty } => {
+ let instance =
+ self.types.types()[ty].unwrap_concrete_instance();
+ FactInlineIntrinsic::from(
+ FactComponentIntrinsic::ResourceTransferBorrowToOwner {
+ ty,
+ instance,
+ },
+ )
+ .into()
+ }
_ => continue,
},
diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rs
index fbd415a899..80a6833291 100644
--- a/crates/environ/src/component/vmcomponent_offsets.rs
+++ b/crates/environ/src/component/vmcomponent_offsets.rs
@@ -50,7 +50,7 @@ pub struct VMComponentOffsets<P> {
// generated `compute_field_offsets` and read by the generated accessors of
// the same names.
may_leave: u32,
- handle_table_slots: u32,
+ handle_tables: u32,
trampoline_func_refs: u32,
intrinsic_func_refs: u32,
lowerings: u32,
@@ -166,7 +166,7 @@ impl<P: PtrSize> VMComponentOffsets<P> {
},
num_resources: component.num_resources,
may_leave: 0,
- handle_table_slots: 0,
+ handle_tables: 0,
trampoline_func_refs: 0,
intrinsic_func_refs: 0,
lowerings: 0,
diff --git a/crates/environ/src/vmctxtypes.rs b/crates/environ/src/vmctxtypes.rs
index 0407abcb52..1e1044a43c 100644
--- a/crates/environ/src/vmctxtypes.rs
+++ b/crates/environ/src/vmctxtypes.rs
@@ -308,9 +308,10 @@ macro_rules! for_each_vmctx_type {
// adapter reads to reach a handle's slot without calling
// into the host.
array {
- handle_table_slots
+ #[aggregate]
+ handle_tables
[num_runtime_component_instances; RuntimeComponentInstanceIndex]:
- VmPtr<VMHandleTableSlot>
+ VMHandleTable
}
array {
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
index 5c07e88651..590f191ab8 100644
--- a/crates/environ/src/vmtypes.rs
+++ b/crates/environ/src/vmtypes.rs
@@ -760,6 +760,22 @@ macro_rules! for_each_vm_type {
} = 9,
}
+ /// One `HandleTable`'s slots, as compiled code sees them.
+ ///
+ /// The length lets a fused adapter's inline to-owner lend bounds check a
+ /// guest-supplied handle. Must be rewritten whenever the buffer grows.
+ #[cfg(feature = "component-model")]
+ #[derive(Debug, Copy, Clone)]
+ #[repr(C)]
+ #[snake_name = vm_handle_table]
+ pub struct VMHandleTable {
+ /// Pointer to the slots.
+ pub base: VmPtr<VMHandleTableSlot>,
+
+ /// The number of slots in `base`.
+ pub current_slots: usize,
+ }
+
/// A deferred component-model thread.
///
/// This is an on-stack record pushed by a fused sync-to-sync adapter's fast
diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs
index 1f1e9f1bf0..c37f581552 100644
--- a/crates/wasmtime/src/runtime/vm/component.rs
+++ b/crates/wasmtime/src/runtime/vm/component.rs
@@ -16,7 +16,7 @@ use crate::runtime::vm::vmcontext::VMFunctionBody;
use crate::runtime::vm::vmcontext::VMLazyThread;
use crate::runtime::vm::{
HostResult, SendSyncPtr, UncaughtException, VMArrayCallFunction, VMFuncRef, VMGlobalDefinition,
- VMHandleTableSlot, VMMemoryDefinition, VMOpaqueContext, VMResourceIndex, VMScope, VMStore,
+ VMHandleTable, VMMemoryDefinition, VMOpaqueContext, VMResourceIndex, VMScope, VMStore,
VMStoreContext, VMStoreRawPtr, VMTableImport, VMTransmitLocalState, VMTypedResource,
VMWasmCallFunction, ValRaw, VmPtr, VmSafe, catch_unwind_and_record_trap,
};
@@ -636,7 +636,7 @@ impl ComponentInstance {
let (states, _types, published) = self.instance_states_and_published();
for (instance, state) in states.iter_mut() {
let i = usize::try_from(instance.as_u32()).unwrap();
- published[i] = state.handle_table().slots_ptr();
+ published[i] = state.handle_table().vmhandletable();
}
}
@@ -1174,19 +1174,17 @@ impl ComponentInstance {
) -> (
&mut TryPrimaryMap<RuntimeComponentInstanceIndex, InstanceState>,
&ComponentTypes,
- &mut [VmPtr<VMHandleTableSlot>],
+ &mut [VMHandleTable],
) {
let offset = self
.offsets
- .handle_table_slots()
+ .handle_tables()
.at(RuntimeComponentInstanceIndex::from_u32(0));
let len = usize::try_from(self.offsets.num_runtime_component_instances).unwrap();
// SAFETY: the array is part of this `VMComponentContext`'s allocation
// and holds exactly `len` entries.
let published = unsafe {
- let base = self
- .as_mut()
- .vmctx_plus_offset_raw::<VmPtr<VMHandleTableSlot>>(offset);
+ let base = self.as_mut().vmctx_plus_offset_raw::<VMHandleTable>(offset);
core::slice::from_raw_parts_mut(base.as_ptr(), len)
};
// safety: as in `instance_states`.
diff --git a/crates/wasmtime/src/runtime/vm/component/handle_table.rs b/crates/wasmtime/src/runtime/vm/component/handle_table.rs
index e1643048bc..78f902bbef 100644
--- a/crates/wasmtime/src/runtime/vm/component/handle_table.rs
+++ b/crates/wasmtime/src/runtime/vm/component/handle_table.rs
@@ -1,6 +1,8 @@
use super::{TransmitLocalStateKind, TypedResourceIndex};
use crate::prelude::TryVec;
-use crate::vm::{VMHandleTableSlot, VMScope, VMTransmitLocalState, VMTypedResource, VmPtr, VmSafe};
+use crate::vm::{
+ VMHandleTable, VMHandleTableSlot, VMScope, VMTransmitLocalState, VMTypedResource, VmPtr, VmSafe,
+};
use crate::{Result, bail};
use core::mem;
use core::ptr::NonNull;
@@ -10,6 +12,10 @@ use wasmtime_environ::component::{TypeFutureTableIndex, TypeStreamTableIndex};
// variant holds only `u32`s and `VmSafe` structs of them.
unsafe impl VmSafe for VMHandleTableSlot {}
+// SAFETY: `VMHandleTable` is generated with `#[repr(C)]` and holds only a
+// `VmPtr` and a `usize`.
+unsafe impl VmSafe for VMHandleTable {}
+
#[cfg(test)]
mod test_vm_handle_table_slot {
use wasmtime_environ::PtrSize;
@@ -116,10 +122,12 @@ impl Default for HandleTable {
impl HandleTable {
/// This table's slots, as compiled code reads them.
- pub fn slots_ptr(&mut self) -> VmPtr<VMHandleTableSlot> {
- // An empty `TryVec`'s pointer is dangling but well-aligned; a table
- // with no slots has none for compiled code to reach.
- VmPtr::from(NonNull::new(self.slots.as_mut_ptr()).unwrap())
+ pub fn vmhandletable(&mut self) -> VMHandleTable {
+ // An empty `TryVec`'s pointer dangles, but then `current_slots` is 0.
+ VMHandleTable {
+ base: VmPtr::from(NonNull::new(self.slots.as_mut_ptr()).unwrap()),
+ current_slots: self.slots.len(),
+ }
}
/// Returns whether or not this table is empty.
@@ -130,22 +138,21 @@ impl HandleTable {
}
/// Begins inserting into this table, which compiled code reads through
- /// `vm_ptr`, re-publishing the table's slots there whenever they move.
+ /// `vm_table`, re-publishing the table there whenever it grows.
///
/// # Safety
///
- /// `vm_ptr` must be the `VMComponentContext` entry through which compiled
- /// code reads this table's slots, or else compiled code may later read
- /// through a stale pointer.
+ /// `vm_table` must be the `VMComponentContext` entry through which
+ /// compiled code reads this table, or it may go stale.
pub unsafe fn insert<'a>(
&'a mut self,
counter: &'a mut HandleCounter,
- vm_ptr: &'a mut VmPtr<VMHandleTableSlot>,
+ vm_table: &'a mut VMHandleTable,
) -> HandleTableInsertion<'a> {
HandleTableInsertion {
table: self,
counter,
- vm_ptr: Some(vm_ptr),
+ vm_table: Some(vm_table),
}
}
@@ -153,8 +160,8 @@ impl HandleTable {
///
/// # Safety
///
- /// This table's slots must never be published to compiled code, or else
- /// compiled code may later read through a stale pointer.
+ /// This table must never be published to compiled code, or it may go
+ /// stale.
pub unsafe fn insert_unpublished<'a>(
&'a mut self,
counter: &'a mut HandleCounter,
@@ -162,7 +169,7 @@ impl HandleTable {
HandleTableInsertion {
table: self,
counter,
- vm_ptr: None,
+ vm_table: None,
}
}
@@ -571,7 +578,7 @@ pub struct HandleTableInsertion<'a> {
table: &'a mut HandleTable,
counter: &'a mut HandleCounter,
/// `None` when compiled code never reads `table`.
- vm_ptr: Option<&'a mut VmPtr<VMHandleTableSlot>>,
+ vm_table: Option<&'a mut VMHandleTable>,
}
impl HandleTableInsertion<'_> {
@@ -602,8 +609,8 @@ impl HandleTableInsertion<'_> {
next: next.checked_add(1).unwrap(),
})?;
self.counter.used += 1;
- if let Some(vm_ptr) = &mut self.vm_ptr {
- **vm_ptr = table.slots_ptr();
+ if let Some(vm_table) = &mut self.vm_table {
+ **vm_table = table.vmhandletable();
}
}
table.next = match mem::replace(&mut table.slots[next as usize], slot) {
diff --git a/crates/wasmtime/src/runtime/vm/component/resources.rs b/crates/wasmtime/src/runtime/vm/component/resources.rs
index 1bc0c86158..e2de31cac9 100644
--- a/crates/wasmtime/src/runtime/vm/component/resources.rs
+++ b/crates/wasmtime/src/runtime/vm/component/resources.rs
@@ -25,7 +25,7 @@ use crate::bail_bug;
use crate::component::store::ComponentTaskState;
use crate::prelude::*;
use crate::runtime::vm::{
- VMHandleTableSlot, VMResourceIndex, VMScope, VMStoreContext, VMTypedResource, VmPtr,
+ VMHandleTable, VMResourceIndex, VMScope, VMStoreContext, VMTypedResource,
};
use core::error::Error;
use core::fmt;
@@ -64,7 +64,7 @@ pub struct ResourceTables<'a> {
pub guest: Option<(
&'a mut TryPrimaryMap<RuntimeComponentInstanceIndex, InstanceState>,
&'a ComponentTypes,
- &'a mut [VmPtr<VMHandleTableSlot>],
+ &'a mut [VMHandleTable],
)>,
/// Runtime state for resources currently owned by the host.
diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rs
index 7ec5d88a75..1a864d950a 100644
--- a/tests/all/pooling_allocator.rs
+++ b/tests/all/pooling_allocator.rs
@@ -884,9 +884,9 @@ fn component_instance_size_limit() -> Result<()> {
match wasmtime::component::Component::new(&engine, "(component)") {
Ok(_) => panic!("should have hit limit"),
Err(e) => {
- e.assert_contains("instance allocation for this component requires 56 bytes");
+ e.assert_contains("instance allocation for this component requires 64 bytes");
e.assert_contains("which exceeds the configured maximum of 1 bytes");
- e.assert_contains("`VMComponentContext` used 56 bytes");
+ e.assert_contains("`VMComponentContext` used 64 bytes");
}
}
diff --git a/tests/disas/component-model/sync-adapter-lazy-lend.wat b/tests/disas/component-model/sync-adapter-lazy-lend.wat
new file mode 100644
index 0000000000..aa81e938b3
--- /dev/null
+++ b/tests/disas/component-model/sync-adapter-lazy-lend.wat
@@ -0,0 +1,227 @@
+;;! target = "x86_64"
+;;! test = "optimize"
+;;! filter = "wasm[2]--function"
+;;! flags = "-Wconcurrency-support=y"
+
+;; An adapter that passes a `borrow` to the instance owning the resource, with
+;; both halves of the lazy-lend fast path inline and no libcall on either.
+
+(component
+ (component $A
+ (type $t' (resource (rep i32)))
+ (export $t "t" (type $t'))
+
+ (core func $ctor (canon resource.new $t))
+
+ (core module $M
+ (func (export "get'") (param i32) (result i32)
+ (local.get 0)
+ )
+ )
+ (core instance $m (instantiate $M))
+
+ (func (export "[constructor]t") (param "x" u32) (result (own $t))
+ (canon lift (core func $ctor))
+ )
+ (func (export "get") (param "self" (borrow $t)) (result u32)
+ (canon lift (core func $m "get'"))
+ )
+ )
+ (instance $a (instantiate $A))
+
+ (component $B
+ (import "a" (instance $i
+ (export "t" (type $t (sub resource)))
+ (export "get" (func (param "self" (borrow $t)) (result u32)))
+ ))
+ (alias export $i "t" (type $t))
+ (alias export $i "get" (func $get))
+
+ (core func $get' (canon lower (func $get)))
+
+ (core module $M
+ (import "" "get" (func $get (param i32) (result i32)))
+ (func (export "run") (param i32) (result i32)
+ (call $get (local.get 0))
+ )
+ )
+ (core instance $m (instantiate $M
+ (with "" (instance (export "get" (func $get'))))))
+
+ (func (export "run") (param "x" u32) (result u32)
+ (canon lift (core func $m "run"))
+ )
+ )
+ (instance $b (instantiate $B (with "a" (instance $a))))
+ (export "run" (func $b "run"))
+)
+;; function u2:0(i64 vmctx, i64, i32) -> i32 tail {
+;; ss0 = explicit_slot 32, align = 8
+;; region0 = 123 ""
+;; region1 = 160 ""
+;; region2 = 57 ""
+;; region3 = 209 ""
+;; region4 = 42 ""
+;; region5 = 68 ""
+;; region6 = 237 ""
+;; region7 = 126 ""
+;; region8 = 221 ""
+;; region9 = 112 ""
+;; region10 = 71 ""
+;; region11 = 149 ""
+;; region12 = 93 ""
+;; region13 = 30 ""
+;; region14 = 184 ""
+;; region15 = 132 ""
+;; region16 = 108 ""
+;; region17 = 8 ""
+;; region18 = 218 ""
+;; region19 = 252 ""
+;; region20 = 13 ""
+;; region21 = 34 ""
+;; region22 = 195 ""
+;; region23 = 158 ""
+;; region24 = 174 ""
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
+;; gv2 = load.i64 notrap aligned region1 gv1+24
+;; sig0 = (i64 vmctx, i64) tail
+;; sig1 = (i64 vmctx, i64, i32, i32) tail
+;; sig2 = (i64 vmctx, i64, i32) -> i32 tail
+;; fn0 = colocated u0:0 sig2
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @00e7 jump block4
+;;
+;; block6(v4: i64):
+;; @00e7 jump block3
+;;
+;; block4:
+;; @00ee v6 = load.i64 notrap aligned readonly can_move region2 v0+264
+;; @00ee v7 = load.i32 notrap aligned region3 v6
+;; @00f2 trapz v7, user26
+;; @00f2 jump block7
+;;
+;; block7:
+;; @00fc v13 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @00fc v14 = load.i64 notrap aligned region5 v13+136
+;; @00fc v12 = stack_addr.i64 ss0
+;; @00fc store notrap aligned region6 v14, v12
+;; @00e1 v3 = iconst.i32 0
+;; @00fc store notrap aligned region7 v3, v12+8 ; v3 = 0
+;; @00fa v10 = iconst.i32 1
+;; @00fc store notrap aligned region8 v10, v12+12 ; v10 = 1
+;; @00fc v15 = load.i32 notrap aligned region9 v13+128
+;; @00fc store notrap aligned region10 v15, v12+16
+;; @00fc store notrap aligned region9 v3, v13+128 ; v3 = 0
+;; @00fc v17 = load.i32 notrap aligned region11 v13+132
+;; @00fc store notrap aligned region12 v17, v12+20
+;; @00fc store notrap aligned region11 v3, v13+132 ; v3 = 0
+;; @00fc v19 = load.i64 notrap aligned region13 v13+152
+;; @00fc store notrap aligned region14 v19, v12+24
+;; @00fc store notrap aligned region5 v12, v13+136
+;; @00fe v20 = load.i64 notrap aligned readonly can_move region2 v0+240
+;; @00fe v21 = load.i32 notrap aligned region15 v20
+;; @0104 store notrap aligned region15 v3, v20 ; v3 = 0
+;; @0108 jump block11
+;;
+;; block11:
+;; @0108 v31 = iconst.i64 8
+;; @0108 v32 = icmp.i64 ult v19, v31 ; v31 = 8
+;; @0108 brif v32, block12, block9
+;;
+;; block12:
+;; @00f4 v8 = load.i64 notrap aligned readonly can_move region4 v0+104
+;; @0108 v36 = load.i64 notrap aligned region16 v8+120
+;; @0108 v33 = iconst.i32 -1
+;; @0108 v34 = iadd.i32 v2, v33 ; v33 = -1
+;; @0108 v35 = uextend.i64 v34
+;; @0108 v37 = icmp ult v35, v36
+;; @0108 brif v37, block13, block9
+;;
+;; block13:
+;; @0108 v38 = load.i64 notrap aligned region17 v8+112
+;; @0108 v39 = iconst.i64 24
+;; @0108 v40 = imul.i64 v35, v39 ; v39 = 24
+;; @0108 v41 = iadd v38, v40
+;; @0108 v42 = load.i32 notrap aligned region18 v41
+;; @0108 v87 = iconst.i32 -1
+;; @0108 v88 = iadd v42, v87 ; v87 = -1
+;; @00fa v89 = iconst.i32 1
+;; @0108 v90 = icmp ule v88, v89 ; v89 = 1
+;; @0108 brif v90, block14, block9
+;;
+;; block14:
+;; @0108 v47 = load.i32 notrap aligned region19 v41+4
+;; @0108 v48 = iconst.i32 2
+;; @0108 v49 = icmp eq v47, v48 ; v48 = 2
+;; @0108 brif v49, block15, block9
+;;
+;; block15:
+;; @0108 v50 = load.i32 notrap aligned region20 v41+8
+;; @0108 v91 = iconst.i32 2
+;; @0108 v51 = load.i64 notrap aligned readonly can_move region21 v13+144
+;; v84 = iconst.i64 3
+;; v85 = ishl.i64 v19, v84 ; v84 = 3
+;; @0108 v54 = iadd v51, v85
+;; @0108 store notrap aligned region22 v91, v54 ; v91 = 2
+;; @0108 store.i32 notrap aligned region23 v2, v54+4
+;; @0108 v56 = iconst.i64 1
+;; @0108 v57 = iadd.i64 v19, v56 ; v56 = 1
+;; @0108 store notrap aligned region13 v57, v13+152
+;; @0108 jump block10(v50)
+;;
+;; block9 cold:
+;; @0108 v58 = load.i64 notrap aligned readonly can_move region24 v0+152
+;; @00f4 v92 = load.i64 notrap aligned readonly can_move region4 v0+104
+;; @0108 try_call_indirect v58(v92, v0, v2), sig2, block16(ret0), [ context v0, default: block6(exn0) ]
+;;
+;; block16(v59: i32):
+;; @0108 jump block10(v59)
+;;
+;; block10(v27: i32):
+;; @010c store.i32 notrap aligned region15 v21, v20
+;; @010e v63 = load.i64 notrap aligned readonly can_move region4 v0+72
+;; @010e try_call fn0(v63, v0, v27), sig2, block17(ret0), [ context v0, default: block6(exn0) ]
+;;
+;; block17(v64: i32):
+;; @0112 v68 = load.i64 notrap aligned region5 v13+136
+;; @0112 v69 = icmp eq v68, v12
+;; @0112 brif v69, block18, block19
+;;
+;; block18:
+;; @0112 v70 = load.i64 notrap aligned region6 v12
+;; @0112 store notrap aligned region5 v70, v13+136
+;; @0112 v71 = load.i64 notrap aligned region14 v12+24
+;; @0112 store notrap aligned region13 v71, v13+152
+;; @0112 v72 = load.i32 notrap aligned region10 v12+16
+;; @0112 store notrap aligned region9 v72, v13+128
+;; @0112 v73 = load.i32 notrap aligned region12 v12+20
+;; @0112 store notrap aligned region11 v73, v13+132
+;; @0112 jump block20
+;;
+;; block19 cold:
+;; @0112 v74 = load.i64 notrap aligned readonly can_move region24 v0+184
+;; @00f4 v93 = load.i64 notrap aligned readonly can_move region4 v0+104
+;; @0112 try_call_indirect v74(v93, v0), sig0, block21, [ context v0, default: block6(exn0) ]
+;;
+;; block21:
+;; @0112 jump block20
+;;
+;; block20:
+;; @011c store.i32 notrap aligned region3 v7, v6
+;; @011e jump block5
+;;
+;; block5:
+;; @011f jump block2
+;;
+;; block3:
+;; @0122 trap user52
+;;
+;; block2:
+;; @0126 jump block1
+;;
+;; block1:
+;; @0126 return v64
+;; }
diff --git a/tests/misc_testsuite/component-model/borrow-transfer-to-owner.wast b/tests/misc_testsuite/component-model/borrow-transfer-to-owner.wast
index de4b0809ef..b6518252c1 100644
--- a/tests/misc_testsuite/component-model/borrow-transfer-to-owner.wast
+++ b/tests/misc_testsuite/component-model/borrow-transfer-to-owner.wast
@@ -165,3 +165,166 @@
)
(assert_return (invoke "run" (u32.const 42)) (u32.const 1001))
+
+;; An out-of-bounds handle takes the libcall, keeping the host's diagnostic.
+;; Separate component since a trap locks the instance down.
+(component
+ (component $A
+ (type $t' (resource (rep i32)))
+ (export $t "t" (type $t'))
+ (core func $ctor (canon resource.new $t))
+ (core module $m
+ (func (export "[method]t.get") (param i32) (result i32)
+ (local.get 0))
+ )
+ (core instance $i (instantiate $m))
+ (func (export "[constructor]t") (param "x" u32) (result (own $t))
+ (canon lift (core func $ctor)))
+ (func (export "[method]t.get") (param "self" (borrow $t)) (result u32)
+ (canon lift (core func $i "[method]t.get")))
+ )
+ (instance $a (instantiate $A))
+
+ (component $B
+ (import "a" (instance $i
+ (export "t" (type $t (sub resource)))
+ (export "[method]t.get" (func (param "self" (borrow $t)) (result u32)))
+ ))
+ (alias export $i "t" (type $t))
+ (alias export $i "[method]t.get" (func $get))
+ (core func $get' (canon lower (func $get)))
+ (core module $m
+ (import "" "get" (func $get (param i32) (result i32)))
+ (func (export "lend-unknown") (result i32)
+ (call $get (i32.const 1234)))
+ )
+ (core instance $i2 (instantiate $m
+ (with "" (instance (export "get" (func $get'))))))
+ (func (export "lend-unknown") (result u32)
+ (canon lift (core func $i2 "lend-unknown")))
+ )
+ (instance $b (instantiate $B (with "a" (instance $a))))
+ (export "lend-unknown" (func $b "lend-unknown"))
+)
+
+(assert_trap (invoke "lend-unknown") "unknown handle index 1234")
+
+;; The `own` transfer forces the deferred thread, so the `borrow` after it must
+;; be tracked as a real lend rather than recorded lazily.
+(component
+ (component $A
+ (type $t' (resource (rep i32)))
+ (export $t "t" (type $t'))
+
+ (core func $ctor (canon resource.new $t))
+ (core func $drop (canon resource.drop $t))
+
+ (core module $m
+ (import "" "drop" (func $drop (param i32)))
+ ;; `o` is an owned handle to drop; `b` is a rep since we own `t`.
+ (func (export "take") (param $o i32) (param $b i32) (result i32)
+ (call $drop (local.get $o))
+ (local.get $b))
+ )
+ (core instance $i (instantiate $m
+ (with "" (instance (export "drop" (func $drop))))))
+
+ (func (export "[constructor]t") (param "x" u32) (result (own $t))
+ (canon lift (core func $ctor)))
+ (func (export "take") (param "o" (own $t)) (param "b" (borrow $t)) (result u32)
+ (canon lift (core func $i "take")))
+ )
+ (instance $a (instantiate $A))
+
+ (component $B
+ (import "a" (instance $i
+ (export "t" (type $t (sub resource)))
+ (export "[constructor]t" (func (param "x" u32) (result (own $t))))
+ (export "take" (func (param "o" (own $t)) (param "b" (borrow $t)) (result u32)))
+ ))
+ (alias export $i "t" (type $t))
+ (alias export $i "[constructor]t" (func $ctor))
+ (alias export $i "take" (func $take))
+
+ (core func $ctor' (canon lower (func $ctor)))
+ (core func $take' (canon lower (func $take)))
+ (core func $drop (canon resource.drop $t))
+
+ (core module $m
+ (import "" "ctor" (func $ctor (param i32) (result i32)))
+ (import "" "take" (func $take (param i32 i32) (result i32)))
+ (import "" "drop" (func $drop (param i32)))
+
+ (func (export "run") (result i32)
+ (local $given i32) (local $lent i32) (local $v i32)
+ (local.set $given (call $ctor (i32.const 1)))
+ (local.set $lent (call $ctor (i32.const 42)))
+ (local.set $v (call $take (local.get $given) (local.get $lent)))
+ ;; Only succeeds if the lend of `$lent` was undone on the way out.
+ (call $drop (local.get $lent))
+ (local.get $v))
+ )
+ (core instance $i2 (instantiate $m (with "" (instance
+ (export "ctor" (func $ctor'))
+ (export "take" (func $take'))
+ (export "drop" (func $drop))))))
+
+ (func (export "run") (result u32)
+ (canon lift (core func $i2 "run")))
+ )
+ (instance $b (instantiate $B (with "a" (instance $a))))
+ (export "lend-after-force" (func $b "run"))
+)
+
+(assert_return (invoke "lend-after-force") (u32.const 42))
+
+;; A wrong-typed handle takes the libcall, keeping the host's diagnostic.
+(component
+ (component $A
+ (type $t' (resource (rep i32)))
+ (type $u' (resource (rep i32)))
+ (export $t "t" (type $t'))
+ (export $u "u" (type $u'))
+ (core func $ctor-u (canon resource.new $u))
+ (core module $m
+ (func (export "get") (param i32) (result i32)
+ (local.get 0))
+ )
+ (core instance $i (instantiate $m))
+ (func (export "[constructor]u") (param "x" u32) (result (own $u))
+ (canon lift (core func $ctor-u)))
+ (func (export "get") (param "self" (borrow $t)) (result u32)
+ (canon lift (core func $i "get")))
+ )
+ (instance $a (instantiate $A))
+
+ (component $B
+ (import "a" (instance $i
+ (export "t" (type $t (sub resource)))
+ (export "u" (type $u (sub resource)))
+ (export "[constructor]u" (func (param "x" u32) (result (own $u))))
+ (export "get" (func (param "self" (borrow $t)) (result u32)))
+ ))
+ (alias export $i "[constructor]u" (func $ctor-u))
+ (alias export $i "get" (func $get))
+ (core func $ctor-u' (canon lower (func $ctor-u)))
+ (core func $get' (canon lower (func $get)))
+ (core module $m
+ (import "" "ctor-u" (func $ctor-u (param i32) (result i32)))
+ (import "" "get" (func $get (param i32) (result i32)))
+ ;; Lends a `u` handle where a `t` is expected.
+ (func (export "lend-wrong-type") (result i32)
+ (call $get (call $ctor-u (i32.const 7))))
+ )
+ (core instance $i2 (instantiate $m (with "" (instance
+ (export "ctor-u" (func $ctor-u'))
+ (export "get" (func $get'))))))
+ (func (export "lend-wrong-type") (result u32)
+ (canon lift (core func $i2 "lend-wrong-type")))
+ )
+ (instance $b (instantiate $B (with "a" (instance $a))))
+ (export "lend-wrong-type" (func $b "lend-wrong-type"))
+)
+
+(assert_trap (invoke "lend-wrong-type")
+ "handle index 1 used with the wrong type, expected guest-defined resource but found a different guest-defined resource")