A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Inline to-owner borrow lends in sync-to-sync adapters (#15208)
Nick Fitzgerald3e0ee3fbfdd8
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex f5ea88a052..a709ba3d83 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -85,16 +85,9 @@ enum VmType {reason = "generated uniformly for all VM types via `for_each_vm_type!`")]VMAsanStackSupport,- #[allow(- dead_code,- reason = "generated uniformly for all VM types via `for_each_vm_type!`"- )]VMResourceIndex,+ VMHandleTable,VMHandleTableSlot,- #[allow(- dead_code,- reason = "generated uniformly for all VM types via `for_each_vm_type!`"- )]VMTypedResource,#[allow(dead_code,@@ -106,7 +99,6 @@ enum VmType {reason = "generated uniformly for all VM types via `for_each_vm_type!`")]VMTransmitLocalState,- HandleTableSlotsArray,}/// A key that uniquely identifies an alias region across an entire compilation.@@ -1561,16 +1553,6 @@ whereintroducer}- /// Get the `Field` for an element of the out-of-line array that- /// `VMStoreContext::handle_table_slots` points at.- ///- /// The returned `Field` is relative to the array's base pointer.- pub fn handle_table_slots_array(&mut self) -> Field<'_, Offsets> {- let ty = self.pointer_type;- let flags = ir::MemFlagsData::trusted();- Field::new(self, VmType::HandleTableSlotsArray, 0, flags, ty)- }-/// Get the alias region shared by all memories that cross a module boundary/// and whose definition we do not statically know.pub fn public_memory_region(&mut self, func: &mut ir::Function) -> ir::AliasRegion {diff --git a/crates/cranelift/src/compiler/component.rs b/crates/cranelift/src/compiler/component.rsindex 22839b5af7..22fe9322df 100644--- a/crates/cranelift/src/compiler/component.rs+++ b/crates/cranelift/src/compiler/component.rs@@ -1262,11 +1262,11 @@ impl<'a> TrampolineCompiler<'a> {.builder.ins().iconst(ir::types::I32, i64::from(callee_instance.as_u32()));- crate::component_sync_call::enter(+ crate::func_environ::component_sync_call::enter(&mut self.builder,&mut self.alias_regions,vmctx,- crate::component_sync_call::EnterArgs {+ crate::func_environ::component_sync_call::EnterArgs {callee_async,callee_instance,},@@ -1278,7 +1278,7 @@ impl<'a> TrampolineCompiler<'a> {// Note that the helper here wants a core wasm vmctx, not a component// one like `vmctx` is in this function.let caller_vmctx = self.caller_vmctx();- let slow = crate::component_sync_call::exit(+ let slow = crate::func_environ::component_sync_call::exit(&mut self.builder,&mut self.alias_regions,caller_vmctx,@@ -1973,17 +1973,7 @@ impl TrampolineCompiler<'_> {// Helper function to cast a core wasm input to a host pointer type// which will go into the host libcall.fn cast_to_pointer(&mut self, val: ir::Value, is64: bool) -> ir::Value {- let pointer_type = self.isa.pointer_type();- let host64 = pointer_type == ir::types::I64;- if is64 == host64 {- val- } else if !is64 {- assert!(host64);- self.builder.ins().uextend(pointer_type, val)- } else {- assert!(!host64);- self.builder.ins().ireduce(pointer_type, val)- }+ crate::cast_to_pointer(&mut self.builder, self.isa.pointer_type(), val, is64)}// Helper to cast a host pointer integer type to the destination type.diff --git a/crates/cranelift/src/component_sync_call.rs b/crates/cranelift/src/component_sync_call.rsdeleted file mode 100644index 30ee841449..0000000000--- a/crates/cranelift/src/component_sync_call.rs+++ /dev/null@@ -1,230 +0,0 @@-//! Shared translation of the component model's `{enter,exit}-sync-call`-//! intrinsics.-//!-//! Two different places wrap a synchronous call into another component-//! instance with these intrinsics: fused adapters, whose calls are translated-//! in `func_environ.rs`, and the `resource.drop` trampoline's call to a-//! resource destructor, translated in `compiler/component.rs`.-//!-//! Rather than call out to the host to do the task bookkeeping that entering a-//! sync call entails, both push a `VMDeferredThread` into their own stack frame-//! and publish it as the store's current thread. The host then only promotes it-//! into a real `GuestTask` if it ever actually needs one; see-//! `StoreOpaque::force_current_thread` and-//! `StoreOpaque::force_deferred_current_thread`.-//!-//! The two callers differ only in where the arguments recorded by `enter` come-//! from and in how `exit`'s out-of-line slow path is called. Everything else-//! lives here: the `VMDeferredThread` layout and, in particular, the-//! `context.{get,set}` slot save/zero/restore, which has to match what the host-//! does for a non-deferred thread in `StoreOpaque::set_thread`.--use crate::alias_region::AliasRegions;-use cranelift_codegen::ir::condcodes::IntCC;-use cranelift_codegen::ir::{self, InstBuilder};-use cranelift_frontend::FunctionBuilder;-use wasmtime_environ::{GetPtrSize, NUM_COMPONENT_CONTEXT_SLOTS, PtrSize};--/// The `enter_sync_call` arguments recorded into the pushed-/// `VMDeferredThread`, to be replayed by the host if it ever has to promote the-/// deferred thread into a real one.-pub struct EnterArgs {- /// Whether the callee is async-lifted, as an `i32` boolean.- pub callee_async: ir::Value,- /// The component instance being called into.- pub callee_instance: ir::Value,-}--/// Translates `enter-sync-call`: allocates a `VMDeferredThread` in the current-/// function's frame, records `args` into it along with the caller's context-/// slots, zeroes the live context slots for the callee, and publishes the frame-/// as the store's current thread.-///-/// `vmctx` must be a core wasm `*mut VMContext` for this store. The returned-/// stack slot must be handed to [`exit`] once the callee returns.-pub fn enter<O>(- builder: &mut FunctionBuilder<'_>,- alias_regions: &mut AliasRegions<O>,- vmctx: ir::Value,- args: EnterArgs,-) -> ir::StackSlot-where- O: GetPtrSize,-{- let pointer_type = alias_regions.pointer_type();- let ptr = alias_regions.ptr_size();-- // Allocate the on-stack `VMDeferredThread`.- let slot = builder.func.create_sized_stack_slot(ir::StackSlotData::new(- ir::StackSlotKind::ExplicitSlot,- u32::from(ptr.vm_deferred_thread().size()),- u8::try_from(ptr.size().trailing_zeros()).unwrap(),- ));- let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);- let vmstore = alias_regions- .vmctx()- .store_context()- .load(&mut builder.cursor(), vmctx);-- // Link the previous current thread in as this frame's parent.- let parent = alias_regions- .vm_store_context()- .current_thread()- .load(&mut builder.cursor(), vmstore);- alias_regions- .vm_deferred_thread()- .parent()- .store(&mut builder.cursor(), slot_addr, parent);-- // Record the deferred `enter_sync_call` arguments.- alias_regions.vm_deferred_thread().callee_async().store(- &mut builder.cursor(),- slot_addr,- args.callee_async,- );- alias_regions.vm_deferred_thread().callee_instance().store(- &mut builder.cursor(),- slot_addr,- args.callee_instance,- );-- // Save the caller's context slots into the frame and reset the live values- // to 0 for the freshly-entered (deferred) thread.- for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {- let saved = alias_regions- .vm_store_context()- .component_context(i)- .load(&mut builder.cursor(), vmstore);- alias_regions.vm_deferred_thread().saved_context(i).store(- &mut builder.cursor(),- slot_addr,- saved,- );- let zero = builder.ins().iconst(ir::types::I32, 0);- alias_regions.vm_store_context().component_context(i).store(- &mut builder.cursor(),- vmstore,- zero,- );- }-- // Record where this frame's lazy lends will start, so that `exit` can- // discard them and the host can find exactly this frame's entries if it- // has to promote them into real lends.- let lazy_lenders_len = alias_regions- .vm_store_context()- .lazy_lenders_len()- .load(&mut builder.cursor(), vmstore);- alias_regions- .vm_deferred_thread()- .lazy_lenders_start()- .store(&mut builder.cursor(), slot_addr, lazy_lenders_len);-- // Publish the deferred thread as the store's current thread.- alias_regions.vm_store_context().current_thread().store(- &mut builder.cursor(),- vmstore,- slot_addr,- );-- slot-}--/// The unfinished slow path of an [`exit`], which the caller must fill in with-/// an out-of-line call to the `exit_sync_call` intrinsic before finishing it-/// with [`SlowExit::finish`].-#[must_use = "the slow path of an exit-sync-call must be emitted and finished"]-pub struct SlowExit {- cont_block: ir::Block,-}--impl SlowExit {- pub fn finish(self, builder: &mut FunctionBuilder<'_>) {- builder.ins().jump(self.cont_block, &[]);- builder.seal_block(self.cont_block);- builder.switch_to_block(self.cont_block);- }-}--/// Translates `exit-sync-call`, the counterpart to [`enter`], where `slot` is-/// the stack slot that `enter` returned.-///-/// If the deferred thread pushed by `enter` is still the store's current-/// thread, meaning nothing ever forced it into a real one, then it is popped,-/// its lazy lends discarded, and the caller's context slots restored inline.-/// Otherwise the host has state to tear down, which only it can do, so the-/// caller must emit an out-of-line call to the `exit_sync_call` intrinsic: on-/// return the builder is positioned in that slow path, which is finished with-/// [`SlowExit::finish`].-pub fn exit<O>(- builder: &mut FunctionBuilder<'_>,- alias_regions: &mut AliasRegions<O>,- vmctx: ir::Value,- slot: ir::StackSlot,-) -> SlowExit-where- O: GetPtrSize,-{- let pointer_type = alias_regions.pointer_type();- let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);- let vmstore = alias_regions- .vmctx()- .store_context()- .load(&mut builder.cursor(), vmctx);- let cur = alias_regions- .vm_store_context()- .current_thread()- .load(&mut builder.cursor(), vmstore);- let is_current = builder.ins().icmp(IntCC::Equal, cur, slot_addr);-- let fast_block = builder.create_block();- let slow_block = builder.create_block();- let cont_block = builder.create_block();- builder- .ins()- .brif(is_current, fast_block, &[], slow_block, &[]);- builder.seal_block(fast_block);- builder.seal_block(slow_block);-- // Fast path: pop the deferred thread and restore the caller's context.- builder.switch_to_block(fast_block);- let parent = alias_regions- .vm_deferred_thread()- .parent()- .load(&mut builder.cursor(), slot_addr);- alias_regions- .vm_store_context()- .current_thread()- .store(&mut builder.cursor(), vmstore, parent);-- // Discard the lazy lends this frame recorded. We can do this by simply- // resetting the `lazy_lenders_len`, and without any per-lend work, since- // the deferred thread and its lazy lends were never materialized.- let start = alias_regions- .vm_deferred_thread()- .lazy_lenders_start()- .load(&mut builder.cursor(), slot_addr);- alias_regions.vm_store_context().lazy_lenders_len().store(- &mut builder.cursor(),- vmstore,- start,- );-- for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {- let saved = alias_regions- .vm_deferred_thread()- .saved_context(i)- .load(&mut builder.cursor(), slot_addr);- alias_regions.vm_store_context().component_context(i).store(- &mut builder.cursor(),- vmstore,- saved,- );- }- builder.ins().jump(cont_block, &[]);-- // Slow path: the thread was promoted to a real one, so the caller does the- // equivalent teardown out-of-line.- builder.switch_to_block(slow_block);- SlowExit { cont_block }-}diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex a6a2d26854..aed69be2d4 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -1,3 +1,4 @@+pub(crate) mod component_sync_call;mod gc;pub(crate) mod stack_switching;@@ -268,10 +269,6 @@ pub struct FuncEnvironment<'module_environment> {/// Alias regions for the enclosing component's offsets.////// Present only when this module is being compiled as part of a component.- #[allow(- dead_code,- reason = "read by the fused-adapter fast paths in a follow-up commit"- )]pub(crate) component_regions:Option<AliasRegions<wasmtime_environ::component::VMComponentOffsets<u8>>>,}@@ -2302,6 +2299,19 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {&real_call_args,)));}++ #[cfg(feature = "component-model")]+ FactInlineIntrinsic::Component(intrinsic) => {+ if let Some(rets) = self.lower_fact_component_intrinsic(+ *intrinsic,+ callee_index,+ sig_ref,+ &real_call_args,+ ) {+ return Ok(Reachability::Reachable(rets));+ }+ }+FactInlineIntrinsic::EnterSyncCall | FactInlineIntrinsic::ExitSyncCall => {}}let import_off = self.env.offsets.imported_functions().at(callee_index);@@ -2364,11 +2374,11 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {/// callee_instance]`.fn lower_fact_enter_sync_call(&mut self, real_call_args: &[ir::Value]) -> CallRets {let vmctx = self.env.vmctx_val(&mut self.builder.cursor());- let slot = crate::component_sync_call::enter(+ let slot = component_sync_call::enter(self.builder,&mut self.env.alias_regions,vmctx,- crate::component_sync_call::EnterArgs {+ component_sync_call::EnterArgs {callee_async: real_call_args[2],callee_instance: real_call_args[3],},@@ -2379,6 +2389,44 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {CallRets::new()}+ /// Inline lowering of a FACT adapter's component-model-specific+ /// intrinsics.+ ///+ /// Returns `None` when the intrinsic must instead be called out-of-line.+ #[cfg(feature = "component-model")]+ fn lower_fact_component_intrinsic(+ &mut self,+ intrinsic: wasmtime_environ::component::FactComponentIntrinsic,+ callee_index: FuncIndex,+ sig_ref: ir::SigRef,+ real_call_args: &[ir::Value],+ ) -> Option<CallRets> {+ use wasmtime_environ::component::FactComponentIntrinsic;+ match intrinsic {+ FactComponentIntrinsic::ResourceTransferBorrowToOwner { ty, instance } => {+ // Needs an enclosing deferred frame to record into.+ if !self.env.tunables.concurrency_support || self.env.fact_sync_call_slot.is_none()+ {+ return None;+ }+ debug_assert!(!self.tail);+ let rep = component_sync_call::transfer_borrow_to_owner(+ self.builder,+ self.env,+ self.srcloc,+ ty,+ instance,+ callee_index,+ sig_ref,+ real_call_args,+ );+ let mut out = CallRets::new();+ out.push(rep);+ Some(out)+ }+ }+ }+/// Inline lowering of a FACT adapter's `exit-sync-call` intrinsic, the/// counterpart to `lower_fact_enter_sync_call`.fn lower_fact_exit_sync_call(@@ -2393,12 +2441,8 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {.take().expect("inline exit-sync-call without a matching enter-sync-call");let vmctx = self.env.vmctx_val(&mut self.builder.cursor());- let slow = crate::component_sync_call::exit(- self.builder,- &mut self.env.alias_regions,- vmctx,- slot,- );+ let slow =+ component_sync_call::exit(self.builder, &mut self.env.alias_regions, vmctx, slot);let import_off = self.env.offsets.imported_functions().at(callee_index);let func_addr = selfdiff --git a/crates/cranelift/src/func_environ/component_sync_call.rs b/crates/cranelift/src/func_environ/component_sync_call.rsnew file mode 100644index 0000000000..d015c2c506--- /dev/null+++ b/crates/cranelift/src/func_environ/component_sync_call.rs@@ -0,0 +1,540 @@+//! Shared translation of the component model's `{enter,exit}-sync-call`+//! intrinsics, and of the `resource.transfer-borrow-to-owner` that fused+//! adapters make between them.+//!+//! Two different places wrap a synchronous call into another component+//! instance with these intrinsics: fused adapters, whose calls are translated+//! in `func_environ.rs`, and the `resource.drop` trampoline's call to a+//! resource destructor, translated in `compiler/component.rs`.+//!+//! Rather than call out to the host to do the task bookkeeping that entering a+//! sync call entails, both push a `VMDeferredThread` into their own stack frame+//! and publish it as the store's current thread. The host then only promotes it+//! into a real `GuestTask` if it ever actually needs one; see+//! `StoreOpaque::force_current_thread` and+//! `StoreOpaque::force_deferred_current_thread`.+//!+//! The two callers differ only in where the arguments recorded by `enter` come+//! from and in how `exit`'s out-of-line slow path is called. Everything else+//! lives here: the `VMDeferredThread` layout and, in particular, the+//! `context.{get,set}` slot save/zero/restore, which has to match what the host+//! does for a non-deferred thread in `StoreOpaque::set_thread`.+//!+//! Between `enter` and `exit`, a fused adapter lending a `borrow` to the+//! resource's owning instance uses `transfer_borrow_to_owner`. Instead of+//! creating a real lend, it records the handle in the store's lazy lenders,+//! which `exit` discards wholesale and the host promotes into real lends only+//! if it forces the deferred thread. Anything else, such as a promoted thread,+//! a full lazy lenders array, or an invalid handle, falls back to the libcall.++use super::{Call, FuncEnvironment};+use crate::alias_region::AliasRegions;+use crate::trap::TranslateTrap;+use cranelift_codegen::ir::condcodes::IntCC;+use cranelift_codegen::ir::{self, InstBuilder};+use cranelift_frontend::FunctionBuilder;+use wasmtime_environ::{+ FuncIndex, GetPtrSize, LAZY_LENDERS_CAPACITY, NUM_COMPONENT_CONTEXT_SLOTS, PtrSize,+ component::{RuntimeComponentInstanceIndex, TypeResourceTableIndex, VMComponentOffsets},+};++/// The `enter_sync_call` arguments recorded into the pushed+/// `VMDeferredThread`, to be replayed by the host if it ever has to promote the+/// deferred thread into a real one.+pub struct EnterArgs {+ /// Whether the callee is async-lifted, as an `i32` boolean.+ pub callee_async: ir::Value,+ /// The component instance being called into.+ pub callee_instance: ir::Value,+}++/// Translates `enter-sync-call`: allocates a `VMDeferredThread` in the current+/// function's frame, records `args` into it along with the caller's context+/// slots, zeroes the live context slots for the callee, and publishes the frame+/// as the store's current thread.+///+/// `vmctx` must be a core wasm `*mut VMContext` for this store. The returned+/// stack slot must be handed to [`exit`] once the callee returns.+pub fn enter<O>(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<O>,+ vmctx: ir::Value,+ args: EnterArgs,+) -> ir::StackSlot+where+ O: GetPtrSize,+{+ let pointer_type = alias_regions.pointer_type();+ let ptr = alias_regions.ptr_size();++ // Allocate the on-stack `VMDeferredThread`.+ let slot = builder.func.create_sized_stack_slot(ir::StackSlotData::new(+ ir::StackSlotKind::ExplicitSlot,+ u32::from(ptr.vm_deferred_thread().size()),+ u8::try_from(ptr.size().trailing_zeros()).unwrap(),+ ));+ let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);+ let vmstore = alias_regions+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx);++ // Link the previous current thread in as this frame's parent.+ let parent = alias_regions+ .vm_store_context()+ .current_thread()+ .load(&mut builder.cursor(), vmstore);+ alias_regions+ .vm_deferred_thread()+ .parent()+ .store(&mut builder.cursor(), slot_addr, parent);++ // Record the deferred `enter_sync_call` arguments.+ alias_regions.vm_deferred_thread().callee_async().store(+ &mut builder.cursor(),+ slot_addr,+ args.callee_async,+ );+ alias_regions.vm_deferred_thread().callee_instance().store(+ &mut builder.cursor(),+ slot_addr,+ args.callee_instance,+ );++ // Save the caller's context slots into the frame and reset the live values+ // to 0 for the freshly-entered (deferred) thread.+ for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {+ let saved = alias_regions+ .vm_store_context()+ .component_context(i)+ .load(&mut builder.cursor(), vmstore);+ alias_regions.vm_deferred_thread().saved_context(i).store(+ &mut builder.cursor(),+ slot_addr,+ saved,+ );+ let zero = builder.ins().iconst(ir::types::I32, 0);+ alias_regions.vm_store_context().component_context(i).store(+ &mut builder.cursor(),+ vmstore,+ zero,+ );+ }++ // Record where this frame's lazy lends will start, so that `exit` can+ // discard them and the host can find exactly this frame's entries if it+ // has to promote them into real lends.+ let lazy_lenders_len = alias_regions+ .vm_store_context()+ .lazy_lenders_len()+ .load(&mut builder.cursor(), vmstore);+ alias_regions+ .vm_deferred_thread()+ .lazy_lenders_start()+ .store(&mut builder.cursor(), slot_addr, lazy_lenders_len);++ // Publish the deferred thread as the store's current thread.+ alias_regions.vm_store_context().current_thread().store(+ &mut builder.cursor(),+ vmstore,+ slot_addr,+ );++ slot+}++/// The unfinished slow path of an [`exit`], which the caller must fill in with+/// an out-of-line call to the `exit_sync_call` intrinsic before finishing it+/// with [`SlowExit::finish`].+#[must_use = "the slow path of an exit-sync-call must be emitted and finished"]+pub struct SlowExit {+ cont_block: ir::Block,+}++impl SlowExit {+ pub fn finish(self, builder: &mut FunctionBuilder<'_>) {+ builder.ins().jump(self.cont_block, &[]);+ builder.seal_block(self.cont_block);+ builder.switch_to_block(self.cont_block);+ }+}++/// Translates `exit-sync-call`, the counterpart to [`enter`], where `slot` is+/// the stack slot that `enter` returned.+///+/// If the deferred thread pushed by `enter` is still the store's current+/// thread, meaning nothing ever forced it into a real one, then it is popped,+/// its lazy lends discarded, and the caller's context slots restored inline.+/// Otherwise the host has state to tear down, which only it can do, so the+/// caller must emit an out-of-line call to the `exit_sync_call` intrinsic: on+/// return the builder is positioned in that slow path, which is finished with+/// [`SlowExit::finish`].+pub fn exit<O>(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<O>,+ vmctx: ir::Value,+ slot: ir::StackSlot,+) -> SlowExit+where+ O: GetPtrSize,+{+ let pointer_type = alias_regions.pointer_type();+ let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);+ let vmstore = alias_regions+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx);+ let cur = alias_regions+ .vm_store_context()+ .current_thread()+ .load(&mut builder.cursor(), vmstore);+ let is_current = builder.ins().icmp(IntCC::Equal, cur, slot_addr);++ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let cont_block = builder.create_block();+ builder+ .ins()+ .brif(is_current, fast_block, &[], slow_block, &[]);+ builder.seal_block(fast_block);+ builder.seal_block(slow_block);++ // Fast path: pop the deferred thread and restore the caller's context.+ builder.switch_to_block(fast_block);+ let parent = alias_regions+ .vm_deferred_thread()+ .parent()+ .load(&mut builder.cursor(), slot_addr);+ alias_regions+ .vm_store_context()+ .current_thread()+ .store(&mut builder.cursor(), vmstore, parent);++ // Discard the lazy lends this frame recorded. We can do this by simply+ // resetting the `lazy_lenders_len`, and without any per-lend work, since+ // the deferred thread and its lazy lends were never materialized.+ let start = alias_regions+ .vm_deferred_thread()+ .lazy_lenders_start()+ .load(&mut builder.cursor(), slot_addr);+ alias_regions.vm_store_context().lazy_lenders_len().store(+ &mut builder.cursor(),+ vmstore,+ start,+ );++ for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {+ let saved = alias_regions+ .vm_deferred_thread()+ .saved_context(i)+ .load(&mut builder.cursor(), slot_addr);+ alias_regions.vm_store_context().component_context(i).store(+ &mut builder.cursor(),+ vmstore,+ saved,+ );+ }+ builder.ins().jump(cont_block, &[]);++ // Slow path: the thread was promoted to a real one, so the caller does the+ // equivalent teardown out-of-line.+ builder.switch_to_block(slow_block);+ builder.set_cold_block(slow_block);+ SlowExit { cont_block }+}++/// Inline `resource.transfer-borrow-to-owner` in a FACT adapter, returning the+/// borrowed resource's `rep`.+///+/// This is the inline equivalent of+/// `ResourceTables::resource_lift_borrow_lazy`, falling back to calling the+/// imported intrinsic `callee_index` with `real_call_args` when the fast path+/// does not apply. `real_call_args` is `[callee_vmctx, caller_vmctx,+/// handle_index]`.+pub fn transfer_borrow_to_owner(+ builder: &mut FunctionBuilder<'_>,+ env: &mut FuncEnvironment<'_>,+ srcloc: ir::SourceLoc,+ ty: TypeResourceTableIndex,+ instance: RuntimeComponentInstanceIndex,+ callee_index: FuncIndex,+ sig_ref: ir::SigRef,+ real_call_args: &[ir::Value],+) -> ir::Value {+ let slot = env+ .fact_sync_call_slot+ .expect("inline to-owner borrow without an enclosing enter-sync-call");+ let vmctx = env.vmctx_val(&mut builder.cursor());+ let vmcomponent = real_call_args[0];+ let index = real_call_args[2];++ let alias_regions = env+ .component_regions+ .as_mut()+ .expect("an adapter's to-owner borrow requires the component's offsets");+ let pointer_type = alias_regions.pointer_type();+ let layout = HandleTableSlotLayout::new(alias_regions);+ let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);+ let vmstore = alias_regions+ .vmctx()+ .store_context()+ .load(&mut builder.cursor(), vmctx);++ let table_offset = alias_regions.offsets().handle_tables().at(instance);++ let slow_block = builder.create_block();+ let join_block = builder.create_block();+ builder.append_block_param(join_block, ir::types::I32);++ let is_current = is_current_thread(builder, alias_regions, vmstore, slot_addr);+ branch_to_slow_path_unless(builder, is_current, slow_block);++ let (len, has_room) = lazy_lenders_has_room(builder, alias_regions, vmstore);+ branch_to_slow_path_unless(builder, has_room, slow_block);++ let (slot_index, in_bounds) =+ bounds_check_handle_table_slot(builder, alias_regions, vmcomponent, table_offset, index);+ branch_to_slow_path_unless(builder, in_bounds, slow_block);++ // The slot is in bounds, so it is safe to read; now check that it holds a+ // resource of the expected type.+ let slot_ptr = handle_table_slot_addr(+ builder,+ alias_regions,+ vmcomponent,+ table_offset,+ slot_index,+ layout.size,+ );+ let is_resource = handle_table_slot_is_resource(builder, alias_regions, &layout, slot_ptr);+ branch_to_slow_path_unless(builder, is_resource, slow_block);++ let has_type = handle_table_slot_has_type(builder, alias_regions, &layout, slot_ptr, ty);+ branch_to_slow_path_unless(builder, has_type, slow_block);++ let rep = alias_regions+ .vm_typed_resource()+ .rep()+ .relative_to(layout.resource_offset)+ .load(&mut builder.cursor(), slot_ptr);+ push_lazy_lender(builder, alias_regions, vmstore, len, ty, index);+ builder.ins().jump(join_block, &[rep.into()]);++ builder.switch_to_block(slow_block);+ builder.set_cold_block(slow_block);+ builder.seal_block(slow_block);+ let import_off = env.offsets.imported_functions().at(callee_index);+ let func_addr = env+ .alias_regions+ .vm_function_import()+ .wasm_call()+ .relative_to(import_off)+ .load(&mut builder.cursor(), vmctx);+ let rets =+ Call::new(builder, env, srcloc).indirect_call_inst(sig_ref, func_addr, real_call_args);+ builder.ins().jump(join_block, &[rets[0].into()]);++ builder.switch_to_block(join_block);+ builder.seal_block(join_block);+ builder.block_params(join_block)[0]+}++/// The layout of a `VMHandleTableSlot` that the inline resource fast paths+/// rely on.+struct HandleTableSlotLayout {+ /// The size of a slot, in bytes.+ size: i64,+ /// The offset of the `VMTypedResource` within either resource variant.+ resource_offset: u32,+ /// The tag of the `resource_own` variant.+ own_tag: u32,+ /// The tag of the `resource_borrow` variant.+ borrow_tag: u32,+}++impl HandleTableSlotLayout {+ fn new(alias_regions: &AliasRegions<VMComponentOffsets<u8>>) -> Self {+ let slot = alias_regions.ptr_size().vm_handle_table_slot();+ debug_assert_eq!(+ slot.resource_own().resource(),+ slot.resource_borrow().resource(),+ "both resource variants must put `resource` at the same offset, \+ so that the fast path need not branch on the tag",+ );+ let own_tag = slot.resource_own().tag_value();+ let borrow_tag = slot.resource_borrow().tag_value();+ debug_assert_eq!(borrow_tag, own_tag + 1);+ Self {+ size: i64::from(slot.size()),+ resource_offset: u32::from(slot.resource_own().resource()),+ own_tag,+ borrow_tag,+ }+ }+}++/// Whether the `VMDeferredThread` at `slot_addr` is the store's current thread.+fn is_current_thread(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,+ vmstore: ir::Value,+ slot_addr: ir::Value,+) -> ir::Value {+ let cur = alias_regions+ .vm_store_context()+ .current_thread()+ .load(&mut builder.cursor(), vmstore);+ builder.ins().icmp(IntCC::Equal, cur, slot_addr)+}++/// Returns the store's current `lazy_lenders_len` and whether there is room to+/// push another lazy lend.+fn lazy_lenders_has_room(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,+ vmstore: ir::Value,+) -> (ir::Value, ir::Value) {+ let pointer_type = alias_regions.pointer_type();+ let len = alias_regions+ .vm_store_context()+ .lazy_lenders_len()+ .load(&mut builder.cursor(), vmstore);+ let capacity = i64::try_from(LAZY_LENDERS_CAPACITY).unwrap();+ let cap = builder.ins().iconst(pointer_type, capacity);+ let has_room = builder.ins().icmp(IntCC::UnsignedLessThan, len, cap);+ (len, has_room)+}++/// Converts the 1-based handle `index` into a slot index for the handle table+/// at `table_offset`, returning that slot index and whether it is in bounds.+fn bounds_check_handle_table_slot(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,+ vmcomponent: ir::Value,+ table_offset: u32,+ index: ir::Value,+) -> (ir::Value, ir::Value) {+ let pointer_type = alias_regions.pointer_type();+ // Handle 0 wraps to `u32::MAX` and fails the bounds check.+ let slot_index = builder.ins().iadd_imm_s(index, -1);+ let slot_index = crate::cast_to_pointer(builder, pointer_type, slot_index, false);+ let slots_len = alias_regions+ .vm_handle_table()+ .current_slots()+ .relative_to(table_offset)+ .load(&mut builder.cursor(), vmcomponent);+ let in_bounds = builder+ .ins()+ .icmp(IntCC::UnsignedLessThan, slot_index, slots_len);+ (slot_index, in_bounds)+}++/// The address of slot `slot_index` in the handle table at `table_offset`,+/// which must already be bounds checked.+fn handle_table_slot_addr(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,+ vmcomponent: ir::Value,+ table_offset: u32,+ slot_index: ir::Value,+ slot_size: i64,+) -> ir::Value {+ let slots = alias_regions+ .vm_handle_table()+ .base()+ .relative_to(table_offset)+ .load(&mut builder.cursor(), vmcomponent);+ let slot_offset = builder.ins().imul_imm_u(slot_index, slot_size);+ builder.ins().iadd(slots, slot_offset)+}++/// Continues in a new block if `cond` is true, otherwise branches to+/// `slow_block`.+fn branch_to_slow_path_unless(+ builder: &mut FunctionBuilder<'_>,+ cond: ir::Value,+ slow_block: ir::Block,+) {+ let next_block = builder.create_block();+ builder.ins().brif(cond, next_block, &[], slow_block, &[]);+ builder.seal_block(next_block);+ builder.switch_to_block(next_block);+}++/// Whether the handle table slot at `slot_ptr` holds an owned or borrowed+/// resource.+fn handle_table_slot_is_resource(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,+ layout: &HandleTableSlotLayout,+ slot_ptr: ir::Value,+) -> ir::Value {+ // The own and borrow tags are adjacent, so one unsigned comparison checks+ // for either.+ let tag = alias_regions+ .vm_handle_table_slot()+ .tag()+ .load(&mut builder.cursor(), slot_ptr);+ let tag = builder.ins().iadd_imm_s(tag, -i64::from(layout.own_tag));+ builder.ins().icmp_imm_u(+ IntCC::UnsignedLessThanOrEqual,+ tag,+ i64::from(layout.borrow_tag - layout.own_tag),+ )+}++/// Whether the resource in the handle table slot at `slot_ptr` has type `ty`.+///+/// The slot must already be known to hold a resource.+fn handle_table_slot_has_type(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,+ layout: &HandleTableSlotLayout,+ slot_ptr: ir::Value,+ ty: TypeResourceTableIndex,+) -> ir::Value {+ let slot_ty = alias_regions+ .vm_typed_resource()+ .ty()+ .relative_to(layout.resource_offset)+ .load(&mut builder.cursor(), slot_ptr);+ builder+ .ins()+ .icmp_imm_u(IntCC::Equal, slot_ty, i64::from(ty.as_u32()))+}++/// Records a lend of handle `index` at position `len` in the store's lazy+/// lenders, for the host to promote if it forces the current thread.+///+/// The caller must have already checked that there is room.+fn push_lazy_lender(+ builder: &mut FunctionBuilder<'_>,+ alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,+ vmstore: ir::Value,+ len: ir::Value,+ ty: TypeResourceTableIndex,+ index: ir::Value,+) {+ let entry_size = i64::from(alias_regions.ptr_size().vm_resource_index().size());+ let lazy_lenders = alias_regions+ .vm_store_context()+ .lazy_lenders()+ .load(&mut builder.cursor(), vmstore);+ let entry_offset = builder.ins().imul_imm_u(len, entry_size);+ let entry_addr = builder.ins().iadd(lazy_lenders, entry_offset);+ let entry_ty = builder.ins().iconst(ir::types::I32, i64::from(ty.as_u32()));+ alias_regions+ .vm_resource_index()+ .ty()+ .store(&mut builder.cursor(), entry_addr, entry_ty);+ alias_regions+ .vm_resource_index()+ .index()+ .store(&mut builder.cursor(), entry_addr, index);+ let next_len = builder.ins().iadd_imm_s(len, 1);+ alias_regions.vm_store_context().lazy_lenders_len().store(+ &mut builder.cursor(),+ vmstore,+ next_len,+ );+}diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rsindex 1664aad6f9..837c9cbe79 100644--- a/crates/cranelift/src/lib.rs+++ b/crates/cranelift/src/lib.rs@@ -22,6 +22,7 @@ use cranelift_codegen::{settings,};use cranelift_entity::PrimaryMap;+use cranelift_frontend::FunctionBuilder;use target_lexicon::Architecture;use wasmtime_environ::{@@ -41,7 +42,6 @@ mod alias_region;mod bounds_checks;mod builder;mod compiler;-mod component_sync_call;mod debug;mod func_environ;mod translate;@@ -158,6 +158,26 @@ fn unbarriered_load_type_at_offset(pos.ins().load(ir_ty, flags, base, offset)}+/// Casts `val`, an `i64` if `is64` and an `i32` otherwise, to `pointer_type`.+fn cast_to_pointer(+ builder: &mut FunctionBuilder<'_>,+ pointer_type: ir::Type,+ val: ir::Value,+ val_is_64: bool,+) -> ir::Value {+ debug_assert!(pointer_type == ir::types::I32 || pointer_type == ir::types::I64);+ let pointer_is_64 = pointer_type == ir::types::I64;+ if val_is_64 == pointer_is_64 {+ val+ } else if !val_is_64 {+ assert!(pointer_is_64);+ builder.ins().uextend(pointer_type, val)+ } else {+ assert!(!pointer_is_64);+ builder.ins().ireduce(pointer_type, val)+ }+}+/// Returns the corresponding cranelift type for the provided wasm type.fn value_type(isa: &dyn TargetIsa, ty: WasmValType) -> ir::types::Type {match ty {diff --git a/crates/environ/src/compile/module_environ.rs b/crates/environ/src/compile/module_environ.rsindex 9955980a7e..e343b4dd93 100644--- a/crates/environ/src/compile/module_environ.rs+++ b/crates/environ/src/compile/module_environ.rs@@ -45,12 +45,18 @@ pub struct ModuleEnvironment<'a, 'data> {pub enum FactInlineIntrinsic {/// `enter-sync-call`: push a deferred component-model thread inline.EnterSyncCall,+/// `exit-sync-call`: pop the deferred thread inline on the fast path, or/// fall back to the out-of-line `exit-sync-call` libcall when the thread/// was promoted.ExitSyncCall,+/// `trap`: raise the given trap.Trap(Trap),++ /// An intrinsic that only exists for components.+ #[cfg(feature = "component-model")]+ Component(crate::component::FactComponentIntrinsic),}/// A statically-known function import.diff --git a/crates/environ/src/component/compiler.rs b/crates/environ/src/component/compiler.rsindex 2bfc28a361..facef8986f 100644--- a/crates/environ/src/component/compiler.rs+++ b/crates/environ/src/component/compiler.rs@@ -1,6 +1,29 @@-use crate::component::{ComponentTranslation, ComponentTypesBuilder, UnsafeIntrinsic};+use crate::component::{+ ComponentTranslation, ComponentTypesBuilder, RuntimeComponentInstanceIndex,+ TypeResourceTableIndex, UnsafeIntrinsic,+};use crate::error::Result;-use crate::{Abi, CompiledFunctionBody, FuncKey, Tunables};+use crate::{Abi, CompiledFunctionBody, FactInlineIntrinsic, FuncKey, Tunables};++/// The component-model-specific `FactInlineIntrinsic`s.+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]+pub enum FactComponentIntrinsic {+ /// `resource.transfer-borrow-to-owner`: lazily lend a `borrow` to its+ /// resource's owning instance, or fall back to the libcall.+ ResourceTransferBorrowToOwner {+ /// The resource type of the handle being lent.+ ty: TypeResourceTableIndex,+ /// The component instance owning `ty`'s handle table.+ instance: RuntimeComponentInstanceIndex,+ },+}++impl From<FactComponentIntrinsic> for FactInlineIntrinsic {+ #[inline]+ fn from(intrinsic: FactComponentIntrinsic) -> Self {+ Self::Component(intrinsic)+ }+}/// Compilation support necessary for components.pub trait ComponentCompiler: Send + Sync {diff --git a/crates/environ/src/component/translate.rs b/crates/environ/src/component/translate.rsindex a1b57c07d4..d5bf6f3844 100644--- a/crates/environ/src/component/translate.rs+++ b/crates/environ/src/component/translate.rs@@ -705,6 +705,17 @@ impl<'a, 'data> Translator<'a, 'data> {FactInlineIntrinsic::ExitSyncCall.into()}Trampoline::Trap(trap) => FactInlineIntrinsic::Trap(trap).into(),+ Trampoline::ResourceTransferBorrowToOwner { ty } => {+ let instance =+ self.types.types()[ty].unwrap_concrete_instance();+ FactInlineIntrinsic::from(+ FactComponentIntrinsic::ResourceTransferBorrowToOwner {+ ty,+ instance,+ },+ )+ .into()+ }_ => continue,},diff --git a/crates/environ/src/component/vmcomponent_offsets.rs b/crates/environ/src/component/vmcomponent_offsets.rsindex fbd415a899..80a6833291 100644--- a/crates/environ/src/component/vmcomponent_offsets.rs+++ b/crates/environ/src/component/vmcomponent_offsets.rs@@ -50,7 +50,7 @@ pub struct VMComponentOffsets<P> {// generated `compute_field_offsets` and read by the generated accessors of// the same names.may_leave: u32,- handle_table_slots: u32,+ handle_tables: u32,trampoline_func_refs: u32,intrinsic_func_refs: u32,lowerings: u32,@@ -166,7 +166,7 @@ impl<P: PtrSize> VMComponentOffsets<P> {},num_resources: component.num_resources,may_leave: 0,- handle_table_slots: 0,+ handle_tables: 0,trampoline_func_refs: 0,intrinsic_func_refs: 0,lowerings: 0,diff --git a/crates/environ/src/vmctxtypes.rs b/crates/environ/src/vmctxtypes.rsindex 0407abcb52..1e1044a43c 100644--- a/crates/environ/src/vmctxtypes.rs+++ b/crates/environ/src/vmctxtypes.rs@@ -308,9 +308,10 @@ macro_rules! for_each_vmctx_type {// adapter reads to reach a handle's slot without calling// into the host.array {- handle_table_slots+ #[aggregate]+ handle_tables[num_runtime_component_instances; RuntimeComponentInstanceIndex]:- VmPtr<VMHandleTableSlot>+ VMHandleTable}array {diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rsindex 5c07e88651..590f191ab8 100644--- a/crates/environ/src/vmtypes.rs+++ b/crates/environ/src/vmtypes.rs@@ -760,6 +760,22 @@ macro_rules! for_each_vm_type {} = 9,}+ /// One `HandleTable`'s slots, as compiled code sees them.+ ///+ /// The length lets a fused adapter's inline to-owner lend bounds check a+ /// guest-supplied handle. Must be rewritten whenever the buffer grows.+ #[cfg(feature = "component-model")]+ #[derive(Debug, Copy, Clone)]+ #[repr(C)]+ #[snake_name = vm_handle_table]+ pub struct VMHandleTable {+ /// Pointer to the slots.+ pub base: VmPtr<VMHandleTableSlot>,++ /// The number of slots in `base`.+ pub current_slots: usize,+ }+/// A deferred component-model thread.////// This is an on-stack record pushed by a fused sync-to-sync adapter's fastdiff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rsindex 1f1e9f1bf0..c37f581552 100644--- a/crates/wasmtime/src/runtime/vm/component.rs+++ b/crates/wasmtime/src/runtime/vm/component.rs@@ -16,7 +16,7 @@ use crate::runtime::vm::vmcontext::VMFunctionBody;use crate::runtime::vm::vmcontext::VMLazyThread;use crate::runtime::vm::{HostResult, SendSyncPtr, UncaughtException, VMArrayCallFunction, VMFuncRef, VMGlobalDefinition,- VMHandleTableSlot, VMMemoryDefinition, VMOpaqueContext, VMResourceIndex, VMScope, VMStore,+ VMHandleTable, VMMemoryDefinition, VMOpaqueContext, VMResourceIndex, VMScope, VMStore,VMStoreContext, VMStoreRawPtr, VMTableImport, VMTransmitLocalState, VMTypedResource,VMWasmCallFunction, ValRaw, VmPtr, VmSafe, catch_unwind_and_record_trap,};@@ -636,7 +636,7 @@ impl ComponentInstance {let (states, _types, published) = self.instance_states_and_published();for (instance, state) in states.iter_mut() {let i = usize::try_from(instance.as_u32()).unwrap();- published[i] = state.handle_table().slots_ptr();+ published[i] = state.handle_table().vmhandletable();}}@@ -1174,19 +1174,17 @@ impl ComponentInstance {) -> (&mut TryPrimaryMap<RuntimeComponentInstanceIndex, InstanceState>,&ComponentTypes,- &mut [VmPtr<VMHandleTableSlot>],+ &mut [VMHandleTable],) {let offset = self.offsets- .handle_table_slots()+ .handle_tables().at(RuntimeComponentInstanceIndex::from_u32(0));let len = usize::try_from(self.offsets.num_runtime_component_instances).unwrap();// SAFETY: the array is part of this `VMComponentContext`'s allocation// and holds exactly `len` entries.let published = unsafe {- let base = self- .as_mut()- .vmctx_plus_offset_raw::<VmPtr<VMHandleTableSlot>>(offset);+ let base = self.as_mut().vmctx_plus_offset_raw::<VMHandleTable>(offset);core::slice::from_raw_parts_mut(base.as_ptr(), len)};// safety: as in `instance_states`.diff --git a/crates/wasmtime/src/runtime/vm/component/handle_table.rs b/crates/wasmtime/src/runtime/vm/component/handle_table.rsindex e1643048bc..78f902bbef 100644--- a/crates/wasmtime/src/runtime/vm/component/handle_table.rs+++ b/crates/wasmtime/src/runtime/vm/component/handle_table.rs@@ -1,6 +1,8 @@use super::{TransmitLocalStateKind, TypedResourceIndex};use crate::prelude::TryVec;-use crate::vm::{VMHandleTableSlot, VMScope, VMTransmitLocalState, VMTypedResource, VmPtr, VmSafe};+use crate::vm::{+ VMHandleTable, VMHandleTableSlot, VMScope, VMTransmitLocalState, VMTypedResource, VmPtr, VmSafe,+};use crate::{Result, bail};use core::mem;use core::ptr::NonNull;@@ -10,6 +12,10 @@ use wasmtime_environ::component::{TypeFutureTableIndex, TypeStreamTableIndex};// variant holds only `u32`s and `VmSafe` structs of them.unsafe impl VmSafe for VMHandleTableSlot {}+// SAFETY: `VMHandleTable` is generated with `#[repr(C)]` and holds only a+// `VmPtr` and a `usize`.+unsafe impl VmSafe for VMHandleTable {}+#[cfg(test)]mod test_vm_handle_table_slot {use wasmtime_environ::PtrSize;@@ -116,10 +122,12 @@ impl Default for HandleTable {impl HandleTable {/// This table's slots, as compiled code reads them.- pub fn slots_ptr(&mut self) -> VmPtr<VMHandleTableSlot> {- // An empty `TryVec`'s pointer is dangling but well-aligned; a table- // with no slots has none for compiled code to reach.- VmPtr::from(NonNull::new(self.slots.as_mut_ptr()).unwrap())+ pub fn vmhandletable(&mut self) -> VMHandleTable {+ // An empty `TryVec`'s pointer dangles, but then `current_slots` is 0.+ VMHandleTable {+ base: VmPtr::from(NonNull::new(self.slots.as_mut_ptr()).unwrap()),+ current_slots: self.slots.len(),+ }}/// Returns whether or not this table is empty.@@ -130,22 +138,21 @@ impl HandleTable {}/// Begins inserting into this table, which compiled code reads through- /// `vm_ptr`, re-publishing the table's slots there whenever they move.+ /// `vm_table`, re-publishing the table there whenever it grows.////// # Safety///- /// `vm_ptr` must be the `VMComponentContext` entry through which compiled- /// code reads this table's slots, or else compiled code may later read- /// through a stale pointer.+ /// `vm_table` must be the `VMComponentContext` entry through which+ /// compiled code reads this table, or it may go stale.pub unsafe fn insert<'a>(&'a mut self,counter: &'a mut HandleCounter,- vm_ptr: &'a mut VmPtr<VMHandleTableSlot>,+ vm_table: &'a mut VMHandleTable,) -> HandleTableInsertion<'a> {HandleTableInsertion {table: self,counter,- vm_ptr: Some(vm_ptr),+ vm_table: Some(vm_table),}}@@ -153,8 +160,8 @@ impl HandleTable {////// # Safety///- /// This table's slots must never be published to compiled code, or else- /// compiled code may later read through a stale pointer.+ /// This table must never be published to compiled code, or it may go+ /// stale.pub unsafe fn insert_unpublished<'a>(&'a mut self,counter: &'a mut HandleCounter,@@ -162,7 +169,7 @@ impl HandleTable {HandleTableInsertion {table: self,counter,- vm_ptr: None,+ vm_table: None,}}@@ -571,7 +578,7 @@ pub struct HandleTableInsertion<'a> {table: &'a mut HandleTable,counter: &'a mut HandleCounter,/// `None` when compiled code never reads `table`.- vm_ptr: Option<&'a mut VmPtr<VMHandleTableSlot>>,+ vm_table: Option<&'a mut VMHandleTable>,}impl HandleTableInsertion<'_> {@@ -602,8 +609,8 @@ impl HandleTableInsertion<'_> {next: next.checked_add(1).unwrap(),})?;self.counter.used += 1;- if let Some(vm_ptr) = &mut self.vm_ptr {- **vm_ptr = table.slots_ptr();+ if let Some(vm_table) = &mut self.vm_table {+ **vm_table = table.vmhandletable();}}table.next = match mem::replace(&mut table.slots[next as usize], slot) {diff --git a/crates/wasmtime/src/runtime/vm/component/resources.rs b/crates/wasmtime/src/runtime/vm/component/resources.rsindex 1bc0c86158..e2de31cac9 100644--- a/crates/wasmtime/src/runtime/vm/component/resources.rs+++ b/crates/wasmtime/src/runtime/vm/component/resources.rs@@ -25,7 +25,7 @@ use crate::bail_bug;use crate::component::store::ComponentTaskState;use crate::prelude::*;use crate::runtime::vm::{- VMHandleTableSlot, VMResourceIndex, VMScope, VMStoreContext, VMTypedResource, VmPtr,+ VMHandleTable, VMResourceIndex, VMScope, VMStoreContext, VMTypedResource,};use core::error::Error;use core::fmt;@@ -64,7 +64,7 @@ pub struct ResourceTables<'a> {pub guest: Option<(&'a mut TryPrimaryMap<RuntimeComponentInstanceIndex, InstanceState>,&'a ComponentTypes,- &'a mut [VmPtr<VMHandleTableSlot>],+ &'a mut [VMHandleTable],)>,/// Runtime state for resources currently owned by the host.diff --git a/tests/all/pooling_allocator.rs b/tests/all/pooling_allocator.rsindex 7ec5d88a75..1a864d950a 100644--- a/tests/all/pooling_allocator.rs+++ b/tests/all/pooling_allocator.rs@@ -884,9 +884,9 @@ fn component_instance_size_limit() -> Result<()> {match wasmtime::component::Component::new(&engine, "(component)") {Ok(_) => panic!("should have hit limit"),Err(e) => {- e.assert_contains("instance allocation for this component requires 56 bytes");+ e.assert_contains("instance allocation for this component requires 64 bytes");e.assert_contains("which exceeds the configured maximum of 1 bytes");- e.assert_contains("`VMComponentContext` used 56 bytes");+ e.assert_contains("`VMComponentContext` used 64 bytes");}}diff --git a/tests/disas/component-model/sync-adapter-lazy-lend.wat b/tests/disas/component-model/sync-adapter-lazy-lend.watnew file mode 100644index 0000000000..aa81e938b3--- /dev/null+++ b/tests/disas/component-model/sync-adapter-lazy-lend.wat@@ -0,0 +1,227 @@+;;! target = "x86_64"+;;! test = "optimize"+;;! filter = "wasm[2]--function"+;;! flags = "-Wconcurrency-support=y"++;; An adapter that passes a `borrow` to the instance owning the resource, with+;; both halves of the lazy-lend fast path inline and no libcall on either.++(component+ (component $A+ (type $t' (resource (rep i32)))+ (export $t "t" (type $t'))++ (core func $ctor (canon resource.new $t))++ (core module $M+ (func (export "get'") (param i32) (result i32)+ (local.get 0)+ )+ )+ (core instance $m (instantiate $M))++ (func (export "[constructor]t") (param "x" u32) (result (own $t))+ (canon lift (core func $ctor))+ )+ (func (export "get") (param "self" (borrow $t)) (result u32)+ (canon lift (core func $m "get'"))+ )+ )+ (instance $a (instantiate $A))++ (component $B+ (import "a" (instance $i+ (export "t" (type $t (sub resource)))+ (export "get" (func (param "self" (borrow $t)) (result u32)))+ ))+ (alias export $i "t" (type $t))+ (alias export $i "get" (func $get))++ (core func $get' (canon lower (func $get)))++ (core module $M+ (import "" "get" (func $get (param i32) (result i32)))+ (func (export "run") (param i32) (result i32)+ (call $get (local.get 0))+ )+ )+ (core instance $m (instantiate $M+ (with "" (instance (export "get" (func $get'))))))++ (func (export "run") (param "x" u32) (result u32)+ (canon lift (core func $m "run"))+ )+ )+ (instance $b (instantiate $B (with "a" (instance $a))))+ (export "run" (func $b "run"))+)+;; function u2:0(i64 vmctx, i64, i32) -> i32 tail {+;; ss0 = explicit_slot 32, align = 8+;; region0 = 123 ""+;; region1 = 160 ""+;; region2 = 57 ""+;; region3 = 209 ""+;; region4 = 42 ""+;; region5 = 68 ""+;; region6 = 237 ""+;; region7 = 126 ""+;; region8 = 221 ""+;; region9 = 112 ""+;; region10 = 71 ""+;; region11 = 149 ""+;; region12 = 93 ""+;; region13 = 30 ""+;; region14 = 184 ""+;; region15 = 132 ""+;; region16 = 108 ""+;; region17 = 8 ""+;; region18 = 218 ""+;; region19 = 252 ""+;; region20 = 13 ""+;; region21 = 34 ""+;; region22 = 195 ""+;; region23 = 158 ""+;; region24 = 174 ""+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8+;; gv2 = load.i64 notrap aligned region1 gv1+24+;; sig0 = (i64 vmctx, i64) tail+;; sig1 = (i64 vmctx, i64, i32, i32) tail+;; sig2 = (i64 vmctx, i64, i32) -> i32 tail+;; fn0 = colocated u0:0 sig2+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @00e7 jump block4+;;+;; block6(v4: i64):+;; @00e7 jump block3+;;+;; block4:+;; @00ee v6 = load.i64 notrap aligned readonly can_move region2 v0+264+;; @00ee v7 = load.i32 notrap aligned region3 v6+;; @00f2 trapz v7, user26+;; @00f2 jump block7+;;+;; block7:+;; @00fc v13 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @00fc v14 = load.i64 notrap aligned region5 v13+136+;; @00fc v12 = stack_addr.i64 ss0+;; @00fc store notrap aligned region6 v14, v12+;; @00e1 v3 = iconst.i32 0+;; @00fc store notrap aligned region7 v3, v12+8 ; v3 = 0+;; @00fa v10 = iconst.i32 1+;; @00fc store notrap aligned region8 v10, v12+12 ; v10 = 1+;; @00fc v15 = load.i32 notrap aligned region9 v13+128+;; @00fc store notrap aligned region10 v15, v12+16+;; @00fc store notrap aligned region9 v3, v13+128 ; v3 = 0+;; @00fc v17 = load.i32 notrap aligned region11 v13+132+;; @00fc store notrap aligned region12 v17, v12+20+;; @00fc store notrap aligned region11 v3, v13+132 ; v3 = 0+;; @00fc v19 = load.i64 notrap aligned region13 v13+152+;; @00fc store notrap aligned region14 v19, v12+24+;; @00fc store notrap aligned region5 v12, v13+136+;; @00fe v20 = load.i64 notrap aligned readonly can_move region2 v0+240+;; @00fe v21 = load.i32 notrap aligned region15 v20+;; @0104 store notrap aligned region15 v3, v20 ; v3 = 0+;; @0108 jump block11+;;+;; block11:+;; @0108 v31 = iconst.i64 8+;; @0108 v32 = icmp.i64 ult v19, v31 ; v31 = 8+;; @0108 brif v32, block12, block9+;;+;; block12:+;; @00f4 v8 = load.i64 notrap aligned readonly can_move region4 v0+104+;; @0108 v36 = load.i64 notrap aligned region16 v8+120+;; @0108 v33 = iconst.i32 -1+;; @0108 v34 = iadd.i32 v2, v33 ; v33 = -1+;; @0108 v35 = uextend.i64 v34+;; @0108 v37 = icmp ult v35, v36+;; @0108 brif v37, block13, block9+;;+;; block13:+;; @0108 v38 = load.i64 notrap aligned region17 v8+112+;; @0108 v39 = iconst.i64 24+;; @0108 v40 = imul.i64 v35, v39 ; v39 = 24+;; @0108 v41 = iadd v38, v40+;; @0108 v42 = load.i32 notrap aligned region18 v41+;; @0108 v87 = iconst.i32 -1+;; @0108 v88 = iadd v42, v87 ; v87 = -1+;; @00fa v89 = iconst.i32 1+;; @0108 v90 = icmp ule v88, v89 ; v89 = 1+;; @0108 brif v90, block14, block9+;;+;; block14:+;; @0108 v47 = load.i32 notrap aligned region19 v41+4+;; @0108 v48 = iconst.i32 2+;; @0108 v49 = icmp eq v47, v48 ; v48 = 2+;; @0108 brif v49, block15, block9+;;+;; block15:+;; @0108 v50 = load.i32 notrap aligned region20 v41+8+;; @0108 v91 = iconst.i32 2+;; @0108 v51 = load.i64 notrap aligned readonly can_move region21 v13+144+;; v84 = iconst.i64 3+;; v85 = ishl.i64 v19, v84 ; v84 = 3+;; @0108 v54 = iadd v51, v85+;; @0108 store notrap aligned region22 v91, v54 ; v91 = 2+;; @0108 store.i32 notrap aligned region23 v2, v54+4+;; @0108 v56 = iconst.i64 1+;; @0108 v57 = iadd.i64 v19, v56 ; v56 = 1+;; @0108 store notrap aligned region13 v57, v13+152+;; @0108 jump block10(v50)+;;+;; block9 cold:+;; @0108 v58 = load.i64 notrap aligned readonly can_move region24 v0+152+;; @00f4 v92 = load.i64 notrap aligned readonly can_move region4 v0+104+;; @0108 try_call_indirect v58(v92, v0, v2), sig2, block16(ret0), [ context v0, default: block6(exn0) ]+;;+;; block16(v59: i32):+;; @0108 jump block10(v59)+;;+;; block10(v27: i32):+;; @010c store.i32 notrap aligned region15 v21, v20+;; @010e v63 = load.i64 notrap aligned readonly can_move region4 v0+72+;; @010e try_call fn0(v63, v0, v27), sig2, block17(ret0), [ context v0, default: block6(exn0) ]+;;+;; block17(v64: i32):+;; @0112 v68 = load.i64 notrap aligned region5 v13+136+;; @0112 v69 = icmp eq v68, v12+;; @0112 brif v69, block18, block19+;;+;; block18:+;; @0112 v70 = load.i64 notrap aligned region6 v12+;; @0112 store notrap aligned region5 v70, v13+136+;; @0112 v71 = load.i64 notrap aligned region14 v12+24+;; @0112 store notrap aligned region13 v71, v13+152+;; @0112 v72 = load.i32 notrap aligned region10 v12+16+;; @0112 store notrap aligned region9 v72, v13+128+;; @0112 v73 = load.i32 notrap aligned region12 v12+20+;; @0112 store notrap aligned region11 v73, v13+132+;; @0112 jump block20+;;+;; block19 cold:+;; @0112 v74 = load.i64 notrap aligned readonly can_move region24 v0+184+;; @00f4 v93 = load.i64 notrap aligned readonly can_move region4 v0+104+;; @0112 try_call_indirect v74(v93, v0), sig0, block21, [ context v0, default: block6(exn0) ]+;;+;; block21:+;; @0112 jump block20+;;+;; block20:+;; @011c store.i32 notrap aligned region3 v7, v6+;; @011e jump block5+;;+;; block5:+;; @011f jump block2+;;+;; block3:+;; @0122 trap user52+;;+;; block2:+;; @0126 jump block1+;;+;; block1:+;; @0126 return v64+;; }diff --git a/tests/misc_testsuite/component-model/borrow-transfer-to-owner.wast b/tests/misc_testsuite/component-model/borrow-transfer-to-owner.wastindex de4b0809ef..b6518252c1 100644--- a/tests/misc_testsuite/component-model/borrow-transfer-to-owner.wast+++ b/tests/misc_testsuite/component-model/borrow-transfer-to-owner.wast@@ -165,3 +165,166 @@)(assert_return (invoke "run" (u32.const 42)) (u32.const 1001))++;; An out-of-bounds handle takes the libcall, keeping the host's diagnostic.+;; Separate component since a trap locks the instance down.+(component+ (component $A+ (type $t' (resource (rep i32)))+ (export $t "t" (type $t'))+ (core func $ctor (canon resource.new $t))+ (core module $m+ (func (export "[method]t.get") (param i32) (result i32)+ (local.get 0))+ )+ (core instance $i (instantiate $m))+ (func (export "[constructor]t") (param "x" u32) (result (own $t))+ (canon lift (core func $ctor)))+ (func (export "[method]t.get") (param "self" (borrow $t)) (result u32)+ (canon lift (core func $i "[method]t.get")))+ )+ (instance $a (instantiate $A))++ (component $B+ (import "a" (instance $i+ (export "t" (type $t (sub resource)))+ (export "[method]t.get" (func (param "self" (borrow $t)) (result u32)))+ ))+ (alias export $i "t" (type $t))+ (alias export $i "[method]t.get" (func $get))+ (core func $get' (canon lower (func $get)))+ (core module $m+ (import "" "get" (func $get (param i32) (result i32)))+ (func (export "lend-unknown") (result i32)+ (call $get (i32.const 1234)))+ )+ (core instance $i2 (instantiate $m+ (with "" (instance (export "get" (func $get'))))))+ (func (export "lend-unknown") (result u32)+ (canon lift (core func $i2 "lend-unknown")))+ )+ (instance $b (instantiate $B (with "a" (instance $a))))+ (export "lend-unknown" (func $b "lend-unknown"))+)++(assert_trap (invoke "lend-unknown") "unknown handle index 1234")++;; The `own` transfer forces the deferred thread, so the `borrow` after it must+;; be tracked as a real lend rather than recorded lazily.+(component+ (component $A+ (type $t' (resource (rep i32)))+ (export $t "t" (type $t'))++ (core func $ctor (canon resource.new $t))+ (core func $drop (canon resource.drop $t))++ (core module $m+ (import "" "drop" (func $drop (param i32)))+ ;; `o` is an owned handle to drop; `b` is a rep since we own `t`.+ (func (export "take") (param $o i32) (param $b i32) (result i32)+ (call $drop (local.get $o))+ (local.get $b))+ )+ (core instance $i (instantiate $m+ (with "" (instance (export "drop" (func $drop))))))++ (func (export "[constructor]t") (param "x" u32) (result (own $t))+ (canon lift (core func $ctor)))+ (func (export "take") (param "o" (own $t)) (param "b" (borrow $t)) (result u32)+ (canon lift (core func $i "take")))+ )+ (instance $a (instantiate $A))++ (component $B+ (import "a" (instance $i+ (export "t" (type $t (sub resource)))+ (export "[constructor]t" (func (param "x" u32) (result (own $t))))+ (export "take" (func (param "o" (own $t)) (param "b" (borrow $t)) (result u32)))+ ))+ (alias export $i "t" (type $t))+ (alias export $i "[constructor]t" (func $ctor))+ (alias export $i "take" (func $take))++ (core func $ctor' (canon lower (func $ctor)))+ (core func $take' (canon lower (func $take)))+ (core func $drop (canon resource.drop $t))++ (core module $m+ (import "" "ctor" (func $ctor (param i32) (result i32)))+ (import "" "take" (func $take (param i32 i32) (result i32)))+ (import "" "drop" (func $drop (param i32)))++ (func (export "run") (result i32)+ (local $given i32) (local $lent i32) (local $v i32)+ (local.set $given (call $ctor (i32.const 1)))+ (local.set $lent (call $ctor (i32.const 42)))+ (local.set $v (call $take (local.get $given) (local.get $lent)))+ ;; Only succeeds if the lend of `$lent` was undone on the way out.+ (call $drop (local.get $lent))+ (local.get $v))+ )+ (core instance $i2 (instantiate $m (with "" (instance+ (export "ctor" (func $ctor'))+ (export "take" (func $take'))+ (export "drop" (func $drop))))))++ (func (export "run") (result u32)+ (canon lift (core func $i2 "run")))+ )+ (instance $b (instantiate $B (with "a" (instance $a))))+ (export "lend-after-force" (func $b "run"))+)++(assert_return (invoke "lend-after-force") (u32.const 42))++;; A wrong-typed handle takes the libcall, keeping the host's diagnostic.+(component+ (component $A+ (type $t' (resource (rep i32)))+ (type $u' (resource (rep i32)))+ (export $t "t" (type $t'))+ (export $u "u" (type $u'))+ (core func $ctor-u (canon resource.new $u))+ (core module $m+ (func (export "get") (param i32) (result i32)+ (local.get 0))+ )+ (core instance $i (instantiate $m))+ (func (export "[constructor]u") (param "x" u32) (result (own $u))+ (canon lift (core func $ctor-u)))+ (func (export "get") (param "self" (borrow $t)) (result u32)+ (canon lift (core func $i "get")))+ )+ (instance $a (instantiate $A))++ (component $B+ (import "a" (instance $i+ (export "t" (type $t (sub resource)))+ (export "u" (type $u (sub resource)))+ (export "[constructor]u" (func (param "x" u32) (result (own $u))))+ (export "get" (func (param "self" (borrow $t)) (result u32)))+ ))+ (alias export $i "[constructor]u" (func $ctor-u))+ (alias export $i "get" (func $get))+ (core func $ctor-u' (canon lower (func $ctor-u)))+ (core func $get' (canon lower (func $get)))+ (core module $m+ (import "" "ctor-u" (func $ctor-u (param i32) (result i32)))+ (import "" "get" (func $get (param i32) (result i32)))+ ;; Lends a `u` handle where a `t` is expected.+ (func (export "lend-wrong-type") (result i32)+ (call $get (call $ctor-u (i32.const 7))))+ )+ (core instance $i2 (instantiate $m (with "" (instance+ (export "ctor-u" (func $ctor-u'))+ (export "get" (func $get'))))))+ (func (export "lend-wrong-type") (result u32)+ (canon lift (core func $i2 "lend-wrong-type")))+ )+ (instance $b (instantiate $B (with "a" (instance $a))))+ (export "lend-wrong-type" (func $b "lend-wrong-type"))+)++(assert_trap (invoke "lend-wrong-type")+ "handle index 1 used with the wrong type, expected guest-defined resource but found a different guest-defined resource")