A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
//! Shared translation of the component model's `{enter,exit}-sync-call`
//! intrinsics, and of the `resource.transfer-borrow-to-owner` that fused
//! adapters make between them.
//!
//! Two different places wrap a synchronous call into another component
//! instance with these intrinsics: fused adapters, whose calls are translated
//! in `func_environ.rs`, and the `resource.drop` trampoline's call to a
//! resource destructor, translated in `compiler/component.rs`.
//!
//! Rather than call out to the host to do the task bookkeeping that entering a
//! sync call entails, both push a `VMDeferredThread` into their own stack frame
//! and publish it as the store's current thread. The host then only promotes it
//! into a real `GuestTask` if it ever actually needs one; see
//! `StoreOpaque::force_current_thread` and
//! `StoreOpaque::force_deferred_current_thread`.
//!
//! The two callers differ only in where the arguments recorded by `enter` come
//! from and in how `exit`'s out-of-line slow path is called. Everything else
//! lives here: the `VMDeferredThread` layout and, in particular, the
//! `context.{get,set}` slot save/zero/restore, which has to match what the host
//! does for a non-deferred thread in `StoreOpaque::set_thread`.
//!
//! Between `enter` and `exit`, a fused adapter lending a `borrow` to the
//! resource's owning instance uses `transfer_borrow_to_owner`. Instead of
//! creating a real lend, it records the handle in the store's lazy lenders,
//! which `exit` discards wholesale and the host promotes into real lends only
//! if it forces the deferred thread. Anything else, such as a promoted thread,
//! a full lazy lenders array, or an invalid handle, falls back to the libcall.
use super::Call;
use crate::alias_region::AliasRegions;
use crate::trap::TranslateTrap;
use cranelift_codegen::ir::condcodes::IntCC;
use cranelift_codegen::ir::{self, InstBuilder};
use cranelift_frontend::FunctionBuilder;
use wasmtime_environ::{
FuncIndex, GetPtrSize, LAZY_LENDERS_CAPACITY, NUM_COMPONENT_CONTEXT_SLOTS, PtrSize,
component::{RuntimeComponentInstanceIndex, TypeResourceTableIndex, VMComponentOffsets},
};
/// The `enter_sync_call` arguments recorded into the pushed
/// `VMDeferredThread`, to be replayed by the host if it ever has to promote the
/// deferred thread into a real one.
pub struct EnterArgs {
/// Whether the callee is async-lifted, as an `i32` boolean.
pub callee_async: ir::Value,
/// The component instance being called into.
pub callee_instance: ir::Value,
}
/// Translates `enter-sync-call`: allocates a `VMDeferredThread` in the current
/// function's frame, records `args` into it along with the caller's context
/// slots, zeroes the live context slots for the callee, and publishes the frame
/// as the store's current thread.
///
/// `vmctx` must be a core wasm `*mut VMContext` for this store. The returned
/// stack slot must be handed to [`exit`] once the callee returns.
pub fn enter<O>(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<O>,
vmctx: ir::Value,
args: EnterArgs,
) -> ir::StackSlot
where
O: GetPtrSize,
{
let pointer_type = alias_regions.pointer_type();
let ptr = alias_regions.ptr_size();
// Allocate the on-stack `VMDeferredThread`.
let slot = builder.func.create_sized_stack_slot(ir::StackSlotData::new(
ir::StackSlotKind::ExplicitSlot,
u32::from(ptr.vm_deferred_thread().size()),
u8::try_from(ptr.size().trailing_zeros()).unwrap(),
));
let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);
let vmstore = alias_regions
.vmctx()
.store_context()
.load(&mut builder.cursor(), vmctx);
// Link the previous current thread in as this frame's parent.
let parent = alias_regions
.vm_store_context()
.current_thread()
.load(&mut builder.cursor(), vmstore);
alias_regions
.vm_deferred_thread()
.parent()
.store(&mut builder.cursor(), slot_addr, parent);
// Record the deferred `enter_sync_call` arguments.
alias_regions.vm_deferred_thread().callee_async().store(
&mut builder.cursor(),
slot_addr,
args.callee_async,
);
alias_regions.vm_deferred_thread().callee_instance().store(
&mut builder.cursor(),
slot_addr,
args.callee_instance,
);
// Save the caller's context slots into the frame and reset the live values
// to 0 for the freshly-entered (deferred) thread.
for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
let saved = alias_regions
.vm_store_context()
.component_context(i)
.load(&mut builder.cursor(), vmstore);
alias_regions.vm_deferred_thread().saved_context(i).store(
&mut builder.cursor(),
slot_addr,
saved,
);
let zero = builder.ins().iconst(ir::types::I32, 0);
alias_regions.vm_store_context().component_context(i).store(
&mut builder.cursor(),
vmstore,
zero,
);
}
// Record where this frame's lazy lends will start, so that `exit` can
// discard them and the host can find exactly this frame's entries if it
// has to promote them into real lends.
let lazy_lenders_len = alias_regions
.vm_store_context()
.lazy_lenders_len()
.load(&mut builder.cursor(), vmstore);
alias_regions
.vm_deferred_thread()
.lazy_lenders_start()
.store(&mut builder.cursor(), slot_addr, lazy_lenders_len);
// Publish the deferred thread as the store's current thread.
alias_regions.vm_store_context().current_thread().store(
&mut builder.cursor(),
vmstore,
slot_addr,
);
slot
}
/// The unfinished slow path of an [`exit`], which the caller must fill in with
/// an out-of-line call to the `exit_sync_call` intrinsic before finishing it
/// with [`SlowExit::finish`].
#[must_use = "the slow path of an exit-sync-call must be emitted and finished"]
pub struct SlowExit {
cont_block: ir::Block,
}
impl SlowExit {
pub fn finish(self, builder: &mut FunctionBuilder<'_>) {
builder.ins().jump(self.cont_block, &[]);
builder.seal_block(self.cont_block);
builder.switch_to_block(self.cont_block);
}
}
/// Translates `exit-sync-call`, the counterpart to [`enter`], where `slot` is
/// the stack slot that `enter` returned.
///
/// If the deferred thread pushed by `enter` is still the store's current
/// thread, meaning nothing ever forced it into a real one, then it is popped,
/// its lazy lends discarded, and the caller's context slots restored inline.
/// Otherwise the host has state to tear down, which only it can do, so the
/// caller must emit an out-of-line call to the `exit_sync_call` intrinsic: on
/// return the builder is positioned in that slow path, which is finished with
/// [`SlowExit::finish`].
pub fn exit<O>(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<O>,
vmctx: ir::Value,
slot: ir::StackSlot,
) -> SlowExit
where
O: GetPtrSize,
{
let pointer_type = alias_regions.pointer_type();
let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);
let vmstore = alias_regions
.vmctx()
.store_context()
.load(&mut builder.cursor(), vmctx);
let cur = alias_regions
.vm_store_context()
.current_thread()
.load(&mut builder.cursor(), vmstore);
let is_current = builder.ins().icmp(IntCC::Equal, cur, slot_addr);
let fast_block = builder.create_block();
let slow_block = builder.create_block();
let cont_block = builder.create_block();
builder
.ins()
.brif(is_current, fast_block, &[], slow_block, &[]);
builder.seal_block(fast_block);
builder.seal_block(slow_block);
// Fast path: pop the deferred thread and restore the caller's context.
builder.switch_to_block(fast_block);
let parent = alias_regions
.vm_deferred_thread()
.parent()
.load(&mut builder.cursor(), slot_addr);
alias_regions
.vm_store_context()
.current_thread()
.store(&mut builder.cursor(), vmstore, parent);
// Discard the lazy lends this frame recorded. We can do this by simply
// resetting the `lazy_lenders_len`, and without any per-lend work, since
// the deferred thread and its lazy lends were never materialized.
let start = alias_regions
.vm_deferred_thread()
.lazy_lenders_start()
.load(&mut builder.cursor(), slot_addr);
alias_regions.vm_store_context().lazy_lenders_len().store(
&mut builder.cursor(),
vmstore,
start,
);
for i in 0..u8::try_from(NUM_COMPONENT_CONTEXT_SLOTS).unwrap() {
let saved = alias_regions
.vm_deferred_thread()
.saved_context(i)
.load(&mut builder.cursor(), slot_addr);
alias_regions.vm_store_context().component_context(i).store(
&mut builder.cursor(),
vmstore,
saved,
);
}
builder.ins().jump(cont_block, &[]);
// Slow path: the thread was promoted to a real one, so the caller does the
// equivalent teardown out-of-line.
builder.switch_to_block(slow_block);
builder.set_cold_block(slow_block);
SlowExit { cont_block }
}
/// Inline `resource.transfer-borrow-to-owner` in a FACT adapter, returning the
/// borrowed resource's `rep`.
///
/// This is the inline equivalent of
/// `ResourceTables::resource_lift_borrow_lazy`, falling back to calling the
/// imported intrinsic `callee_index` with `real_call_args` when the fast path
/// does not apply. `real_call_args` is `[callee_vmctx, caller_vmctx,
/// handle_index]`.
pub(super) fn transfer_borrow_to_owner(
call: &mut Call<'_, '_, '_>,
ty: TypeResourceTableIndex,
instance: RuntimeComponentInstanceIndex,
callee_index: FuncIndex,
sig_ref: ir::SigRef,
real_call_args: &[ir::Value],
) -> ir::Value {
let builder = &mut *call.builder;
let env = &mut *call.env;
let slot = env
.fact_sync_call_slot
.expect("inline to-owner borrow without an enclosing enter-sync-call");
let vmctx = env.vmctx_val(&mut builder.cursor());
let vmcomponent = real_call_args[0];
let index = real_call_args[2];
let alias_regions = env
.component_regions
.as_mut()
.expect("an adapter's to-owner borrow requires the component's offsets");
let pointer_type = alias_regions.pointer_type();
let layout = HandleTableSlotLayout::new(alias_regions);
let slot_addr = builder.ins().stack_addr(pointer_type, slot, 0);
let vmstore = alias_regions
.vmctx()
.store_context()
.load(&mut builder.cursor(), vmctx);
let table_offset = alias_regions.offsets().handle_tables().at(instance);
let slow_block = builder.create_block();
let join_block = builder.create_block();
builder.append_block_param(join_block, ir::types::I32);
let is_current = is_current_thread(builder, alias_regions, vmstore, slot_addr);
branch_to_slow_path_unless(builder, is_current, slow_block);
let (len, has_room) = lazy_lenders_has_room(builder, alias_regions, vmstore);
branch_to_slow_path_unless(builder, has_room, slow_block);
let (slot_index, in_bounds) =
bounds_check_handle_table_slot(builder, alias_regions, vmcomponent, table_offset, index);
branch_to_slow_path_unless(builder, in_bounds, slow_block);
// The slot is in bounds, so it is safe to read; now check that it holds a
// resource of the expected type.
let slot_ptr = handle_table_slot_addr(
builder,
alias_regions,
vmcomponent,
table_offset,
slot_index,
layout.size,
);
let is_resource = handle_table_slot_is_resource(builder, alias_regions, &layout, slot_ptr);
branch_to_slow_path_unless(builder, is_resource, slow_block);
let has_type = handle_table_slot_has_type(builder, alias_regions, &layout, slot_ptr, ty);
branch_to_slow_path_unless(builder, has_type, slow_block);
let rep = alias_regions
.vm_typed_resource()
.rep()
.relative_to(layout.resource_offset)
.load(&mut builder.cursor(), slot_ptr);
push_lazy_lender(builder, alias_regions, vmstore, len, ty, index);
builder.ins().jump(join_block, &[rep.into()]);
builder.switch_to_block(slow_block);
builder.set_cold_block(slow_block);
builder.seal_block(slow_block);
let rets = call.imported_call_inst(callee_index, sig_ref, real_call_args);
let builder = &mut *call.builder;
builder.ins().jump(join_block, &[rets[0].into()]);
builder.switch_to_block(join_block);
builder.seal_block(join_block);
builder.block_params(join_block)[0]
}
/// The layout of a `VMHandleTableSlot` that the inline resource fast paths
/// rely on.
struct HandleTableSlotLayout {
/// The size of a slot, in bytes.
size: i64,
/// The offset of the `VMTypedResource` within either resource variant.
resource_offset: u32,
/// The tag of the `resource_own` variant.
own_tag: u32,
/// The tag of the `resource_borrow` variant.
borrow_tag: u32,
}
impl HandleTableSlotLayout {
fn new(alias_regions: &AliasRegions<VMComponentOffsets<u8>>) -> Self {
let slot = alias_regions.ptr_size().vm_handle_table_slot();
debug_assert_eq!(
slot.resource_own().resource(),
slot.resource_borrow().resource(),
"both resource variants must put `resource` at the same offset, \
so that the fast path need not branch on the tag",
);
let own_tag = slot.resource_own().tag_value();
let borrow_tag = slot.resource_borrow().tag_value();
debug_assert_eq!(borrow_tag, own_tag + 1);
Self {
size: i64::from(slot.size()),
resource_offset: u32::from(slot.resource_own().resource()),
own_tag,
borrow_tag,
}
}
}
/// Whether the `VMDeferredThread` at `slot_addr` is the store's current thread.
fn is_current_thread(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
vmstore: ir::Value,
slot_addr: ir::Value,
) -> ir::Value {
let cur = alias_regions
.vm_store_context()
.current_thread()
.load(&mut builder.cursor(), vmstore);
builder.ins().icmp(IntCC::Equal, cur, slot_addr)
}
/// Returns the store's current `lazy_lenders_len` and whether there is room to
/// push another lazy lend.
fn lazy_lenders_has_room(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
vmstore: ir::Value,
) -> (ir::Value, ir::Value) {
let pointer_type = alias_regions.pointer_type();
let len = alias_regions
.vm_store_context()
.lazy_lenders_len()
.load(&mut builder.cursor(), vmstore);
let capacity = i64::try_from(LAZY_LENDERS_CAPACITY).unwrap();
let cap = builder.ins().iconst(pointer_type, capacity);
let has_room = builder.ins().icmp(IntCC::UnsignedLessThan, len, cap);
(len, has_room)
}
/// Converts the 1-based handle `index` into a slot index for the handle table
/// at `table_offset`, returning that slot index and whether it is in bounds.
fn bounds_check_handle_table_slot(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
vmcomponent: ir::Value,
table_offset: u32,
index: ir::Value,
) -> (ir::Value, ir::Value) {
let pointer_type = alias_regions.pointer_type();
// Handle 0 wraps to `u32::MAX` and fails the bounds check.
let slot_index = builder.ins().iadd_imm_s(index, -1);
let slot_index = crate::cast_to_pointer(builder, pointer_type, slot_index, false);
let slots_len = alias_regions
.vm_handle_table()
.current_slots()
.relative_to(table_offset)
.load(&mut builder.cursor(), vmcomponent);
let in_bounds = builder
.ins()
.icmp(IntCC::UnsignedLessThan, slot_index, slots_len);
(slot_index, in_bounds)
}
/// The address of slot `slot_index` in the handle table at `table_offset`,
/// which must already be bounds checked.
fn handle_table_slot_addr(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
vmcomponent: ir::Value,
table_offset: u32,
slot_index: ir::Value,
slot_size: i64,
) -> ir::Value {
let slots = alias_regions
.vm_handle_table()
.base()
.relative_to(table_offset)
.load(&mut builder.cursor(), vmcomponent);
let slot_offset = builder.ins().imul_imm_u(slot_index, slot_size);
builder.ins().iadd(slots, slot_offset)
}
/// Continues in a new block if `cond` is true, otherwise branches to
/// `slow_block`.
fn branch_to_slow_path_unless(
builder: &mut FunctionBuilder<'_>,
cond: ir::Value,
slow_block: ir::Block,
) {
let next_block = builder.create_block();
builder.ins().brif(cond, next_block, &[], slow_block, &[]);
builder.seal_block(next_block);
builder.switch_to_block(next_block);
}
/// Whether the handle table slot at `slot_ptr` holds an owned or borrowed
/// resource.
fn handle_table_slot_is_resource(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
layout: &HandleTableSlotLayout,
slot_ptr: ir::Value,
) -> ir::Value {
// The own and borrow tags are adjacent, so one unsigned comparison checks
// for either.
let tag = alias_regions
.vm_handle_table_slot()
.tag()
.load(&mut builder.cursor(), slot_ptr);
let tag = builder.ins().iadd_imm_s(tag, -i64::from(layout.own_tag));
builder.ins().icmp_imm_u(
IntCC::UnsignedLessThanOrEqual,
tag,
i64::from(layout.borrow_tag - layout.own_tag),
)
}
/// Whether the resource in the handle table slot at `slot_ptr` has type `ty`.
///
/// The slot must already be known to hold a resource.
fn handle_table_slot_has_type(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
layout: &HandleTableSlotLayout,
slot_ptr: ir::Value,
ty: TypeResourceTableIndex,
) -> ir::Value {
let slot_ty = alias_regions
.vm_typed_resource()
.ty()
.relative_to(layout.resource_offset)
.load(&mut builder.cursor(), slot_ptr);
builder
.ins()
.icmp_imm_u(IntCC::Equal, slot_ty, i64::from(ty.as_u32()))
}
/// Records a lend of handle `index` at position `len` in the store's lazy
/// lenders, for the host to promote if it forces the current thread.
///
/// The caller must have already checked that there is room.
fn push_lazy_lender(
builder: &mut FunctionBuilder<'_>,
alias_regions: &mut AliasRegions<VMComponentOffsets<u8>>,
vmstore: ir::Value,
len: ir::Value,
ty: TypeResourceTableIndex,
index: ir::Value,
) {
let entry_size = i64::from(alias_regions.ptr_size().vm_resource_index().size());
let lazy_lenders = alias_regions
.vm_store_context()
.lazy_lenders()
.load(&mut builder.cursor(), vmstore);
let entry_offset = builder.ins().imul_imm_u(len, entry_size);
let entry_addr = builder.ins().iadd(lazy_lenders, entry_offset);
let entry_ty = builder.ins().iconst(ir::types::I32, i64::from(ty.as_u32()));
alias_regions
.vm_resource_index()
.ty()
.store(&mut builder.cursor(), entry_addr, entry_ty);
alias_regions
.vm_resource_index()
.index()
.store(&mut builder.cursor(), entry_addr, index);
let next_len = builder.ins().iadd_imm_s(len, 1);
alias_regions.vm_store_context().lazy_lenders_len().store(
&mut builder.cursor(),
vmstore,
next_len,
);
}