Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Move `VMStoreContext`, `VMDeferredThread`, and `VMLazyThread` into `f… (#14020)

Nick Fitzgerald2f06aadad23d
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rs
index 061041db9c..873f34b7a2 100644
--- a/crates/cranelift/src/alias_region.rs
+++ b/crates/cranelift/src/alias_region.rs
@@ -38,6 +38,11 @@ enum VmType {
VMCopyingHeapData,
VMNullHeapData,
VMDeferredThread,
+ #[allow(
+ dead_code,
+ reason = "generated uniformly for all VM types via `for_each_vm_type!`"
+ )]
+ VMLazyThread,
VMContRef,
ContinuationStackMemory,
VMFunctionImport,
@@ -185,6 +190,7 @@ impl AliasRegionKey {
const DATA_SEGMENT_KIND: u32 = Self::new_kind(0b100000);
const VM_GLOBAL_DEFINITION_KIND: u32 = Self::new_kind(0b100001);
const VM_TAG_DEFINITION_KIND: u32 = Self::new_kind(0b100010);
+ const VM_LAZY_THREAD_KIND: u32 = Self::new_kind(0b100011);
/// Encode this key into a raw `u32` suitable for use as an
/// `AliasRegionData::user_id`.
@@ -204,6 +210,7 @@ impl AliasRegionKey {
VmType::VMCopyingHeapData => Self::VM_COPYING_HEAP_DATA_KIND,
VmType::VMNullHeapData => Self::VM_NULL_HEAP_DATA_KIND,
VmType::VMDeferredThread => Self::VM_DEFERRED_THREAD_KIND,
+ VmType::VMLazyThread => Self::VM_LAZY_THREAD_KIND,
VmType::VMContRef => Self::VM_CONTREF_KIND,
VmType::ContinuationStackMemory => Self::CONTINUATION_STACK_MEMORY_KIND,
VmType::VMFunctionImport => Self::VM_FUNCTION_IMPORT_KIND,
@@ -538,12 +545,29 @@ impl<'a, Offsets> Field<'a, Offsets> {
uniformly but not exercised until a VM type uses those markers"
)]
macro_rules! define_vm_type_alias_region_helpers {
+ // `UnsafeCell<T>` is `repr(transparent)` and is accessed exactly as its `T`
+ // would be; delegate to the inner type.
+ (@field_ty $pt:expr, UnsafeCell < $inner:tt >) => {
+ define_vm_type_alias_region_helpers!(@field_ty $pt, $inner)
+ };
+
// Classify a field type to its Cranelift `ir::Type`, given `$pt` (the
// target pointer type as an `ir::Type`).
+ //
+ // Note that there is deliberately no arm for a composite field (a nested
+ // struct, an array, or a range): such a field has no single Cranelift type,
+ // and is marked `#[aggregate]` in `for_each_vm_type!` so that no accessor is
+ // generated for it at all.
(@field_ty $pt:expr, VmPtr < $g:ty >) => { $pt };
(@field_ty $pt:expr, Option < VmPtr < $g:ty >>) => { $pt };
(@field_ty $pt:expr, AtomicUsize) => { $pt };
(@field_ty $pt:expr, usize) => { $pt };
+ (@field_ty $pt:expr, i64) => { ir::types::I64 };
+ (@field_ty $pt:expr, u64) => { ir::types::I64 };
+ (@field_ty $pt:expr, u32) => { ir::types::I32 };
+ // `VMLazyThread` is a pointer-sized bitpacked integer; see its definition in
+ // `for_each_vm_type!`.
+ (@field_ty $pt:expr, VMLazyThread) => { $pt };
(@field_ty $pt:expr, [u8; 16]) => { ir::types::I8X16 };
(@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };
(@field_ty $pt:expr, DefinedTableIndex) => { ir::types::I32 };
@@ -632,25 +656,34 @@ macro_rules! define_vm_type_alias_region_helpers {
// >)?` within a repetition) is what lets each field's type reach the
// `@field_ty` classifier as raw tokens, so it can require `Option`s to
// specifically be `Option<VmPtr<_>>`.
- (@munch $Name:ident $snake:ident { $($groups:tt)* } [ $($pend:tt)* ]) => {
+ (@munch $Name:ident $snake:ident { $($groups:tt)* }) => {
define_vm_type_alias_region_helpers!(@emit $Name $snake { $($groups)* });
};
- // Stash a field attribute's bracket group (`[readonly]`, `[can_move]`,
- // `[doc = ...]`) as pending for the next field.
- (@munch $Name:ident $snake:ident { $($groups:tt)* } [ $($pend:tt)* ] # $fattr:tt $($rest:tt)*) => {
- define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* } [ $($pend)* $fattr ] $($rest)*);
+ // A field marked `#[aggregate]` is a composite (a nested struct or array)
+ // that is not accessed as a whole by Wasm code, but piecewise
+ // instead. Therefore, there is no single Cranelift type for such fields, so
+ // no accessors are generated for them; interior accesses are computed from
+ // the field's offset instead, which the generated `offsets::*` methods
+ // still provide.
+ (@munch $Name:ident $snake:ident { $($groups:tt)* }
+ $(#[doc = $fdoc:literal])* #[aggregate] $fvis:vis $fname:ident : $fty:ty , $($rest:tt)*
+ ) => {
+ define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* } $($rest)*);
};
- // Consume a field's visibility and name, then collect its type tokens.
- (@munch $Name:ident $snake:ident { $($groups:tt)* } [ $($pend:tt)* ] $fvis:vis $fname:ident : $($rest:tt)*) => {
- define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($pend)* ] [] $($rest)*);
+ // Consume one field's attributes, visibility, and name, then collect its
+ // type tokens.
+ (@munch $Name:ident $snake:ident { $($groups:tt)* }
+ $(# $fattr:tt)* $fvis:vis $fname:ident : $($rest:tt)*
+ ) => {
+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($fattr)* ] [] $($rest)*);
};
// Accumulate one field's type tokens up to its terminating comma, then
- // append the completed group and reset the pending attributes.
- (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($pend:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {
- define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fname [ $($pend)* ] [ $($fty)* ] } } [] $($rest)*);
+ // append the completed group.
+ (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {
+ define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fname [ $($fattr)* ] [ $($fty)* ] } } $($rest)*);
};
- (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($pend:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {
- define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($pend)* ] [ $($fty)* $tok ] $($rest)*);
+ (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {
+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($fattr)* ] [ $($fty)* $tok ] $($rest)*);
};
// Top-level entry: the list of `VM*` type definitions.
@@ -664,7 +697,7 @@ macro_rules! define_vm_type_alias_region_helpers {
}
)* ) => {
$(
- define_vm_type_alias_region_helpers!(@munch $Name $snake {} [] $($body)*);
+ define_vm_type_alias_region_helpers!(@munch $Name $snake {} $($body)*);
)*
};
}
@@ -1323,7 +1356,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_store_data()
+ .vm_store_context()
+ .store_data()
.into(),
)
}
@@ -1341,7 +1375,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_execution_version()
+ .vm_store_context()
+ .execution_version()
.into(),
)
}
@@ -1359,7 +1394,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_execution_version()
+ .vm_store_context()
+ .execution_version()
.into(),
new_version,
)
@@ -1378,7 +1414,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_fuel_consumed()
+ .vm_store_context()
+ .fuel_consumed()
.into(),
)
}
@@ -1396,7 +1433,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_fuel_consumed()
+ .vm_store_context()
+ .fuel_consumed()
.into(),
fuel_consumed,
)
@@ -1415,7 +1453,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_epoch_deadline()
+ .vm_store_context()
+ .epoch_deadline()
.into(),
)
}
@@ -1425,7 +1464,8 @@ where
let offset = self
.offsets
.get_ptr_size()
- .vmstore_context_stack_limit()
+ .vm_store_context()
+ .stack_limit()
.into();
let region = self.vmstore_context_region(func, offset);
Load {
@@ -1458,7 +1498,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_stack_limit()
+ .vm_store_context()
+ .stack_limit()
.into(),
stack_limit,
)
@@ -1478,7 +1519,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_current_thread()
+ .vm_store_context()
+ .current_thread()
.into(),
)
}
@@ -1496,7 +1538,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_current_thread()
+ .vm_store_context()
+ .current_thread()
.into(),
new_thread,
)
@@ -1514,7 +1557,8 @@ where
let offset = self
.offsets
.get_ptr_size()
- .vmstore_context_gc_heap_base()
+ .vm_store_context()
+ .gc_heap_base()
.into();
let region = self.vmstore_context_region(func, offset);
Load {
@@ -1543,7 +1587,8 @@ where
let offset = self
.offsets
.get_ptr_size()
- .vmstore_context_gc_heap_current_length()
+ .vm_store_context()
+ .gc_heap_current_length()
.into();
let region = self.vmstore_context_region(func, offset);
Load {
@@ -1576,7 +1621,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_last_wasm_entry_fp()
+ .vm_store_context()
+ .last_wasm_entry_fp()
.into(),
)
}
@@ -1594,7 +1640,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_last_wasm_entry_sp()
+ .vm_store_context()
+ .last_wasm_entry_sp()
.into(),
)
}
@@ -1612,7 +1659,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_last_wasm_entry_trap_handler()
+ .vm_store_context()
+ .last_wasm_entry_trap_handler()
.into(),
)
}
@@ -1630,7 +1678,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_last_wasm_entry_fp()
+ .vm_store_context()
+ .last_wasm_entry_fp()
.into(),
fp,
)
@@ -1649,7 +1698,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_last_wasm_entry_sp()
+ .vm_store_context()
+ .last_wasm_entry_sp()
.into(),
sp,
)
@@ -1668,7 +1718,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_last_wasm_entry_trap_handler()
+ .vm_store_context()
+ .last_wasm_entry_trap_handler()
.into(),
trap_handler,
)
@@ -1687,7 +1738,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_last_wasm_exit_trampoline_fp()
+ .vm_store_context()
+ .last_wasm_exit_trampoline_fp()
.into(),
fp,
)
@@ -1706,7 +1758,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_last_wasm_exit_pc()
+ .vm_store_context()
+ .last_wasm_exit_pc()
.into(),
pc,
)
@@ -1721,7 +1774,7 @@ where
&mut self,
func: &mut ir::Function,
) -> ir::AliasRegion {
- let offset = self.offsets.get_ptr_size().vmstore_context_stack_chain();
+ let offset = self.offsets.get_ptr_size().vm_store_context().stack_chain();
self.vmstore_context_region(func, offset.into())
}
@@ -1743,7 +1796,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_component_context_slot(slot)
+ .vm_store_context()
+ .component_context_slot(slot)
.into(),
)
}
@@ -1765,7 +1819,8 @@ where
vmstore_ctx,
self.offsets
.get_ptr_size()
- .vmstore_context_component_context_slot(slot)
+ .vm_store_context()
+ .component_context_slot(slot)
.into(),
val,
)
@@ -1838,7 +1893,8 @@ where
vmdeferred_thread_ptr,
self.offsets
.get_ptr_size()
- .vmdeferred_thread_parent()
+ .vm_deferred_thread()
+ .parent()
.into(),
)
}
@@ -1856,7 +1912,8 @@ where
vmdeferred_thread_ptr,
self.offsets
.get_ptr_size()
- .vmdeferred_thread_parent()
+ .vm_deferred_thread()
+ .parent()
.into(),
parent,
)
@@ -1875,7 +1932,8 @@ where
vmdeferred_thread_ptr,
self.offsets
.get_ptr_size()
- .vmdeferred_thread_caller_instance()
+ .vm_deferred_thread()
+ .caller_instance()
.into(),
caller_instance,
)
@@ -1894,7 +1952,8 @@ where
vmdeferred_thread_ptr,
self.offsets
.get_ptr_size()
- .vmdeferred_thread_callee_async()
+ .vm_deferred_thread()
+ .callee_async()
.into(),
callee_async,
)
@@ -1913,7 +1972,8 @@ where
vmdeferred_thread_ptr,
self.offsets
.get_ptr_size()
- .vmdeferred_thread_callee_instance()
+ .vm_deferred_thread()
+ .callee_instance()
.into(),
callee_instance,
)
@@ -1934,7 +1994,8 @@ where
vmdeferred_thread_ptr,
self.offsets
.get_ptr_size()
- .vmdeferred_thread_saved_context(i)
+ .vm_deferred_thread()
+ .saved_context_slot(i)
.into(),
)
}
@@ -1953,7 +2014,8 @@ where
vmdeferred_thread_ptr,
self.offsets
.get_ptr_size()
- .vmdeferred_thread_saved_context(i)
+ .vm_deferred_thread()
+ .saved_context_slot(i)
.into(),
val,
)
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 9da84e7e9d..ba84e26174 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -2049,7 +2049,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {
let ptr = self.env.offsets.ptr;
// Allocate the on-stack `VMDeferredThread`.
- let size = u32::from(ptr.size_of_vmdeferred_thread());
+ let size = u32::from(ptr.vm_deferred_thread().size());
let align_shift = u8::try_from(ptr.size().trailing_zeros()).unwrap();
let slot = self
.builder
diff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
index 722406cd40..6cdf9d168c 100644
--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs
+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs
@@ -1027,7 +1027,7 @@ pub fn vmctx_load_stack_chain<'a>(
builder: &mut FunctionBuilder,
vmctx: ir::Value,
) -> VMStackChain {
- let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into();
+ let stack_chain_offset = env.offsets.ptr.vm_store_context().stack_chain().into();
// First we need to get the `VMStoreContext`.
let vm_store_context = env
@@ -1055,7 +1055,7 @@ pub fn vmctx_store_stack_chain<'a>(
vmctx: ir::Value,
stack_chain: &VMStackChain,
) {
- let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into();
+ let stack_chain_offset = env.offsets.ptr.vm_store_context().stack_chain().into();
// First we need to get the `VMStoreContext`.
let vm_store_context = env
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index e746a0b627..79f7bda603 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -66,6 +66,10 @@ fn align(offset: u32, width: u32) -> u32 {
/// `VM*` type, with a method per field returning that field's offset, plus
/// `size` and `align` methods.
macro_rules! define_vm_type_offsets {
+ // `UnsafeCell<T>` is `repr(transparent)`, so it has exactly `T`'s layout;
+ // delegate to the inner type.
+ (@size ($p:expr) UnsafeCell < $inner:tt >) => { define_vm_type_offsets!(@size ($p) $inner) };
+
// Classify a field type to its size in bytes as a `u32`, given `$p` (the
// target pointer size as a `u8`). All `VmPtr<_>` and `Option<VmPtr<_>>`
// fields are pointer-sized; the `Defined*Index` types are `u32` entity
@@ -75,28 +79,58 @@ macro_rules! define_vm_type_offsets {
(@size ($p:expr) Option < VmPtr < $g:ty >>) => { u32::from($p) };
(@size ($p:expr) AtomicUsize) => { u32::from($p) };
(@size ($p:expr) usize) => { u32::from($p) };
+ (@size ($p:expr) i64) => { 8u32 };
+ (@size ($p:expr) u64) => { 8u32 };
+ (@size ($p:expr) u32) => { 4u32 };
(@size ($p:expr) [u8; 16]) => { 16u32 };
+ (@size ($p:expr) [u32; $n:expr]) => { 4u32 * u32::try_from($n).unwrap() };
(@size ($p:expr) VMSharedTypeIndex) => { u32::from(($p).size_of_vmshared_type_index()) };
(@size ($p:expr) DefinedTableIndex) => { 4u32 };
(@size ($p:expr) DefinedMemoryIndex) => { 4u32 };
(@size ($p:expr) DefinedTagIndex) => { 4u32 };
(@size ($p:expr) VMGlobalKind) => { 8u32 };
+ // `Range<T>` is a two-field `{ start: T, end: T }` struct.
+ (@size ($p:expr) Range < *mut u8 >) => { 2u32 * u32::from($p) };
+ // Nested `VM*` types recurse through their own generated offsets, keeping
+ // this macro the single source of truth for their layout too.
+ (@size ($p:expr) VMMemoryDefinition) => { u32::from(($p).vm_memory_definition().size()) };
+ (@size ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().size()) };
+ // `VMStackChain` is a `repr(usize, C)` enum, and is not itself defined by
+ // `for_each_vm_type!`.
+ (@size ($p:expr) VMStackChain) => { u32::from(($p).size_of_vmstack_chain()) };
+
+ // As with `@size` above, `UnsafeCell<T>` has exactly `T`'s alignment.
+ (@align ($p:expr) UnsafeCell < $inner:tt >) => { define_vm_type_offsets!(@align ($p) $inner) };
// Classify a field type to its alignment in bytes as a `u32`, given `$p`
// (the target pointer size as a `u8`).
+ //
+ // NB: 64-bit integers are assumed to be 8-aligned, which holds everywhere except
+ // `i686-unknown-linux-gnu`, and the pointer size alone can't tell those apart. Types
+ // with 64-bit fields must therefore put them first and force their own alignment with
+ // `#[repr(C, align(8))]`, as `VMStoreContext` does.
(@align ($p:expr) VmPtr < $g:ty >) => { u32::from($p) };
(@align ($p:expr) Option < VmPtr < $g:ty >>) => { u32::from($p) };
(@align ($p:expr) AtomicUsize) => { u32::from($p) };
(@align ($p:expr) usize) => { u32::from($p) };
+ (@align ($p:expr) i64) => { 8u32 };
+ (@align ($p:expr) u64) => { 8u32 };
+ (@align ($p:expr) u32) => { 4u32 };
(@align ($p:expr) [u8; 16]) => { 16u32 };
+ (@align ($p:expr) [u32; $n:expr]) => { 4u32 };
(@align ($p:expr) VMSharedTypeIndex) => { u32::from(($p).align_of_vmshared_type_index()) };
(@align ($p:expr) DefinedTableIndex) => { 4u32 };
(@align ($p:expr) DefinedMemoryIndex) => { 4u32 };
(@align ($p:expr) DefinedTagIndex) => { 4u32 };
(@align ($p:expr) VMGlobalKind) => { 4u32 };
+ (@align ($p:expr) Range < *mut u8 >) => { u32::from($p) };
+ (@align ($p:expr) VMMemoryDefinition) => { u32::from(($p).vm_memory_definition().align()) };
+ (@align ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().align()) };
+ (@align ($p:expr) VMStackChain) => { u32::from($p) };
// Classify a `#[repr(...)]` to the minimum alignment it forces, as a `u32`.
(@repr_align C) => { 1u32 };
+ (@repr_align transparent) => { 1u32 };
(@repr_align C, align($n:literal)) => {{ let align: u32 = $n; align }};
// Emit a `pub fn` per field returning that field's offset, computed by
@@ -181,12 +215,13 @@ macro_rules! define_vm_type_offsets {
}
}
};
- // Skip a field attribute (`#[doc = ...]`, `#[readonly]`, `#[can_move]`).
- (@impl $Name:ident $repr:tt { $($groups:tt)* } #[$($attr:tt)*] $($rest:tt)*) => {
- define_vm_type_offsets!(@impl $Name $repr { $($groups)* } $($rest)*);
- };
- // Consume a field's visibility and name, then collect its type tokens.
- (@impl $Name:ident $repr:tt { $($groups:tt)* } $fvis:vis $fname:ident : $($rest:tt)*) => {
+ // Consume one field's attributes, visibility, and name, then collect its
+ // type tokens. None of the field attributes (doc comments and the
+ // `#[aggregate]`/`#[readonly]`/`#[can_move]` markers) affect layout, so they
+ // are all discarded here.
+ (@impl $Name:ident $repr:tt { $($groups:tt)* }
+ $(#[$($attr:tt)*])* $fvis:vis $fname:ident : $($rest:tt)*
+ ) => {
define_vm_type_offsets!(@impl_ty $Name $repr { $($groups)* } $fname [] $($rest)*);
};
// Accumulate one field's type tokens up to its terminating comma, then
@@ -215,7 +250,7 @@ macro_rules! define_vm_type_offsets {
/// These types are namespaced within their own module so that they never
/// collide with the real definitions of the `VM*` types themselves.
pub mod offsets {
- use super::{align, PtrSize};
+ use super::{align, PtrSize, NUM_COMPONENT_CONTEXT_SLOTS};
$(
#[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")]
@@ -228,6 +263,46 @@ macro_rules! define_vm_type_offsets {
}
for_each_vm_type!(define_vm_type_offsets);
+/// The size, in bytes, of one `context.{get,set}` slot. These slots are `u32`s,
+/// both in `VMStoreContext::component_context` and in
+/// `VMDeferredThread::saved_context`.
+const COMPONENT_CONTEXT_SLOT_SIZE: u8 = 4;
+
+/// Offsets within a `VMStoreContext` that are not simply the offset of one of
+/// its fields, and so are not generated by `for_each_vm_type!`.
+impl<P: PtrSize> offsets::VMStoreContext<P> {
+ /// The offset of the `gc_heap.base` field within a `VMStoreContext`.
+ pub fn gc_heap_base(&self) -> u8 {
+ let offset = self.gc_heap() + self.0.vm_memory_definition().base();
+ debug_assert!(offset < self.last_wasm_exit_trampoline_fp());
+ offset
+ }
+
+ /// The offset of the `gc_heap.current_length` field within a
+ /// `VMStoreContext`.
+ pub fn gc_heap_current_length(&self) -> u8 {
+ let offset = self.gc_heap() + self.0.vm_memory_definition().current_length();
+ debug_assert!(offset < self.last_wasm_exit_trampoline_fp());
+ offset
+ }
+
+ /// The offset of the `component_context[i]` slot within a `VMStoreContext`.
+ pub fn component_context_slot(&self, i: u8) -> u8 {
+ assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);
+ self.component_context() + i * COMPONENT_CONTEXT_SLOT_SIZE
+ }
+}
+
+/// Offsets within a `VMDeferredThread` that are not simply the offset of one of
+/// its fields, and so are not generated by `for_each_vm_type!`.
+impl<P: PtrSize> offsets::VMDeferredThread<P> {
+ /// The offset of the `saved_context[i]` slot within a `VMDeferredThread`.
+ pub fn saved_context_slot(&self, i: u8) -> u8 {
+ assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);
+ self.saved_context() + i * COMPONENT_CONTEXT_SLOT_SIZE
+ }
+}
+
/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to [`PtrSize`] for
/// each `VM*` type, using the `#[snake_name = ...]` attribute for the method
/// name.
@@ -238,12 +313,9 @@ macro_rules! define_ptr_size_vm_type_accessors {
#[repr($($repr:tt)*)]
#[snake_name = $snake:ident]
$svis:vis struct $Name:ident {
- $(
- $(#[doc = $fdoc:literal])*
- $(#[readonly])?
- $(#[can_move])?
- $fvis:vis $fname:ident : $fty:tt $(< $fgen:ty >)? ,
- )*
+ // This macro only needs each type's name and snake name, so the
+ // body is captured raw rather than parsed into fields.
+ $($body:tt)*
}
)* ) => {
$(
@@ -350,143 +422,6 @@ pub trait PtrSize {
self.vm_global_definition().size()
}
- // Offsets within `VMStoreContext`
-
- /// Return the offset of the `fuel_consumed` field of `VMStoreContext`
- #[inline]
- fn vmstore_context_fuel_consumed(&self) -> u8 {
- 0
- }
-
- /// Return the offset of the `epoch_deadline` field of `VMStoreContext`
- #[inline]
- fn vmstore_context_epoch_deadline(&self) -> u8 {
- self.vmstore_context_fuel_consumed() + 8
- }
-
- /// Return the offset of the `execution_version` field of
- /// `VMStoreContext`
- #[inline]
- fn vmstore_context_execution_version(&self) -> u8 {
- self.vmstore_context_epoch_deadline() + 8
- }
-
- /// Return the offset of the `stack_limit` field of `VMStoreContext`
- #[inline]
- fn vmstore_context_stack_limit(&self) -> u8 {
- self.vmstore_context_execution_version() + 8
- }
-
- /// Return the offset of the `gc_heap` field of `VMStoreContext`.
- #[inline]
- fn vmstore_context_gc_heap(&self) -> u8 {
- self.vmstore_context_stack_limit() + self.size()
- }
-
- /// Return the offset of the `gc_heap.base` field within a `VMStoreContext`.
- fn vmstore_context_gc_heap_base(&self) -> u8 {
- let offset = self.vmstore_context_gc_heap() + self.vm_memory_definition().base();
- debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());
- offset
- }
-
- /// Return the offset of the `gc_heap.current_length` field within a `VMStoreContext`.
- fn vmstore_context_gc_heap_current_length(&self) -> u8 {
- let offset = self.vmstore_context_gc_heap() + self.vm_memory_definition().current_length();
- debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());
- offset
- }
-
- /// Return the offset of the `last_wasm_exit_trampoline_fp` field
- /// of `VMStoreContext`.
- fn vmstore_context_last_wasm_exit_trampoline_fp(&self) -> u8 {
- self.vmstore_context_gc_heap() + self.vm_memory_definition().size()
- }
-
- /// Return the offset of the `last_wasm_exit_pc` field of `VMStoreContext`.
- fn vmstore_context_last_wasm_exit_pc(&self) -> u8 {
- self.vmstore_context_last_wasm_exit_trampoline_fp() + self.size()
- }
-
- /// Return the offset of the `last_wasm_entry_sp` field of `VMStoreContext`.
- fn vmstore_context_last_wasm_entry_sp(&self) -> u8 {
- self.vmstore_context_last_wasm_exit_pc() + self.size()
- }
-
- /// Return the offset of the `last_wasm_entry_fp` field of `VMStoreContext`.
- fn vmstore_context_last_wasm_entry_fp(&self) -> u8 {
- self.vmstore_context_last_wasm_entry_sp() + self.size()
- }
-
- /// Return the offset of the `last_wasm_entry_trap_handler` field of `VMStoreContext`.
- fn vmstore_context_last_wasm_entry_trap_handler(&self) -> u8 {
- self.vmstore_context_last_wasm_entry_fp() + self.size()
- }
-
- /// Return the offset of the `stack_chain` field of `VMStoreContext`.
- fn vmstore_context_stack_chain(&self) -> u8 {
- self.vmstore_context_last_wasm_entry_trap_handler() + self.size()
- }
-
- /// Return the offset of the `stack_chain` field of `VMStoreContext`.
- fn vmstore_context_store_data(&self) -> u8 {
- self.vmstore_context_stack_chain() + self.size_of_vmstack_chain()
- }
-
- /// Return the offset of the `async_guard_range` field of `VMStoreContext`.
- fn vmstore_context_async_guard_range(&self) -> u8 {
- self.vmstore_context_store_data() + self.size()
- }
-
- /// Return the offset of the `component_context[i]` field of
- /// `VMStoreContext`.
- fn vmstore_context_component_context_slot(&self, i: u8) -> u8 {
- assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);
- let base = self.vmstore_context_async_guard_range() + 2 * self.size();
- let slot_size = 4;
- base + i * slot_size
- }
-
- /// Return the offset of the `current_thread` field of `VMStoreContext`.
- fn vmstore_context_current_thread(&self) -> u8 {
- self.vmstore_context_component_context_slot(0) + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4
- }
-
- // Offsets within `VMDeferredThread`
-
- /// Offset of `VMDeferredThread::parent`.
- fn vmdeferred_thread_parent(&self) -> u8 {
- 0
- }
-
- /// Offset of `VMDeferredThread::caller_instance`.
- fn vmdeferred_thread_caller_instance(&self) -> u8 {
- self.size()
- }
-
- /// Offset of `VMDeferredThread::callee_async`.
- fn vmdeferred_thread_callee_async(&self) -> u8 {
- self.vmdeferred_thread_caller_instance() + 4
- }
-
- /// Offset of `VMDeferredThread::callee_instance`.
- fn vmdeferred_thread_callee_instance(&self) -> u8 {
- self.vmdeferred_thread_callee_async() + 4
- }
-
- /// Offset of `VMDeferredThread::saved_context[i]`.
- fn vmdeferred_thread_saved_context(&self, i: u8) -> u8 {
- assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);
- self.vmdeferred_thread_callee_instance() + 4 + i * 4
- }
-
- /// Return the size of `VMDeferredThread`, rounded up to pointer alignment.
- fn size_of_vmdeferred_thread(&self) -> u8 {
- let unaligned =
- self.vmdeferred_thread_callee_instance() + 4 + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4;
- align(u32::from(unaligned), u32::from(self.size())) as u8
- }
-
/// Return the size of `*mut VMMemoryDefinition`.
#[inline]
fn size_of_vmmemory_pointer(&self) -> u8 {
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
index d2a1922b4b..867c1b8060 100644
--- a/crates/environ/src/vmtypes.rs
+++ b/crates/environ/src/vmtypes.rs
@@ -27,9 +27,18 @@
/// * A `#[snake_name = <ident>]` attribute giving the type's name in
/// `snake_case`, used to generate accessor method names.
///
-/// Each field may be preceded by doc-comment attributes and, optionally, the
-/// marker attributes `#[readonly]` and/or `#[can_move]` (in that order) which
-/// describe how Cranelift may treat loads and stores of that field.
+/// Each field may be preceded by doc-comment attributes and, optionally, these
+/// marker attributes, in this order:
+///
+/// * `#[aggregate]`: this field is a composite (a nested struct or array)
+/// rather than a single scalar. Compiled Wasm code accesses such a field's
+/// interior piecewise, so there is no one Cranelift type for the field as a
+/// whole and no alias-region accessor is generated for it. The field's offset
+/// is still generated, since that is what interior accesses are computed
+/// relative to.
+///
+/// * `#[readonly]` and/or `#[can_move]`: describe how Cranelift may treat loads
+/// and stores of that field.
#[macro_export]
macro_rules! for_each_vm_type {
($mac:ident) => {
@@ -228,6 +237,263 @@ macro_rules! for_each_vm_type {
/// The index of the tag in the containing `vmctx`.
pub index: DefinedTagIndex,
}
+
+ /// Structure that holds all mutable context that is shared across all instances
+ /// in a store, for example data related to fuel or epochs.
+ ///
+ /// `VMStoreContext`s are one-to-one with `wasmtime::Store`s, the same way that
+ /// `VMContext`s are one-to-one with `wasmtime::Instance`s. And the same way
+ /// that multiple `wasmtime::Instance`s may be associated with the same
+ /// `wasmtime::Store`, multiple `VMContext`s hold a pointer to the same
+ /// `VMStoreContext` when they are associated with the same `wasmtime::Store`.
+ #[derive(Debug)]
+ // NB: `align(8)` is forced rather than inferred because the i386
+ // System V ABI aligns 64-bit integers to 4 bytes, and `VMOffsets`
+ // can't tell that target apart from the ones that align them to 8,
+ // since it only knows the target's pointer width.
+ #[repr(C, align(8))]
+ #[snake_name = vm_store_context]
+ pub struct VMStoreContext {
+ // NB: 64-bit integer fields are located first with pointer-sized fields
+ // trailing afterwards. That makes the offsets in this structure easier to
+ // calculate on 32-bit platforms as we don't have to worry about the
+ // alignment of 64-bit integers.
+ //
+ /// Indicator of how much fuel has been consumed and is remaining to
+ /// WebAssembly.
+ ///
+ /// This field is typically negative and increments towards positive. Upon
+ /// turning positive a wasm trap will be generated. This field is only
+ /// modified if wasm is configured to consume fuel.
+ pub fuel_consumed: UnsafeCell<i64>,
+
+ /// Deadline epoch for interruption: if epoch-based interruption
+ /// is enabled and the global (per engine) epoch counter is
+ /// observed to reach or exceed this value, the guest code will
+ /// yield if running asynchronously.
+ pub epoch_deadline: UnsafeCell<u64>,
+
+ /// The "store version".
+ ///
+ /// This is used to test whether stack-frame handles referring to
+ /// suspended stack frames remain valid.
+ ///
+ /// The invariant that this upward-counting number must satisfy
+ /// is: the number must be incremented whenever execution starts
+ /// or resumes in the `Store` or when any stack is
+ /// dropped/freed. That way, if we take a reference to some
+ /// suspended stack frame and track the "version" at the time we
+ /// took that reference, if the version still matches, we can be
+ /// sure that nothing could have unwound the referenced Wasm
+ /// frame.
+ ///
+ /// This version number is incremented in exactly one place: the
+ /// Wasm-to-host trampolines, after return from host code. Note
+ /// that this captures both the normal "return into Wasm" case
+ /// (where Wasm frames can subsequently return normally and thus
+ /// invalidate frames), and the "trap/exception unwinds Wasm
+ /// frames" case, which is done internally via the `raise` libcall
+ /// invoked after the main hostcall returns an error, and after we
+ /// increment this version number.
+ ///
+ /// Note that this also handles the fiber/future-drop case because
+ /// because we *always* return into the trampoline to clean up;
+ /// that trampoline immediately raises an error and uses the
+ /// longjmp-like unwind within Cranelift frames to skip over all
+ /// the guest Wasm frames, but not before it increments the
+ /// store's execution version number.
+ ///
+ /// This field is in use only if guest debugging is enabled.
+ pub execution_version: u64,
+
+ /// Current stack limit of the wasm module.
+ ///
+ /// For more information see `crates/cranelift/src/lib.rs`.
+ pub stack_limit: UnsafeCell<usize>,
+
+ /// The `VMMemoryDefinition` for this store's GC heap.
+ #[aggregate]
+ pub gc_heap: UnsafeCell<VMMemoryDefinition>,
+
+ /// The value of the frame pointer register in the trampoline used
+ /// to call from Wasm to the host.
+ ///
+ /// Maintained by our Wasm-to-host trampoline, and cleared just
+ /// before calling into Wasm in `catch_traps`.
+ ///
+ /// This member is `0` when Wasm is actively running and has not called out
+ /// to the host.
+ ///
+ /// Used to find the start of a contiguous sequence of Wasm frames
+ /// when walking the stack. Note that we record the FP of the
+ /// *trampoline*'s frame, not the last Wasm frame, because we need
+ /// to know the SP (bottom of frame) of the last Wasm frame as
+ /// well in case we need to resume to an exception handler in that
+ /// frame. The FP of the last Wasm frame can be recovered by
+ /// loading the saved FP value at this FP address.
+ pub last_wasm_exit_trampoline_fp: UnsafeCell<usize>,
+
+ /// The last Wasm program counter before we called from Wasm to the host.
+ ///
+ /// Maintained by our Wasm-to-host trampoline, and cleared just before
+ /// calling into Wasm in `catch_traps`.
+ ///
+ /// This member is `0` when Wasm is actively running and has not called out
+ /// to the host.
+ ///
+ /// Used when walking a contiguous sequence of Wasm frames.
+ pub last_wasm_exit_pc: UnsafeCell<usize>,
+
+ /// The last host stack pointer before we called into Wasm from the host.
+ ///
+ /// Maintained by our host-to-Wasm trampoline. This member is `0` when Wasm
+ /// is not running, and it's set to nonzero once a host-to-wasm trampoline
+ /// is executed.
+ ///
+ /// When a host function is wrapped into a `wasmtime::Func`, and is then
+ /// called from the host, then this member is not changed meaning that the
+ /// previous activation in pointed to by `last_wasm_exit_trampoline_fp` is
+ /// still the last wasm set of frames on the stack.
+ ///
+ /// This field is saved/restored during fiber suspension/resumption
+ /// resumption as part of `CallThreadState::swap`.
+ ///
+ /// This field is used to find the end of a contiguous sequence of Wasm
+ /// frames when walking the stack. Additionally it's used when a trap is
+ /// raised as part of the set of parameters used to resume in the entry
+ /// trampoline's "catch" block.
+ pub last_wasm_entry_sp: UnsafeCell<usize>,
+
+ /// Same as `last_wasm_entry_sp`, but for the `fp` of the trampoline.
+ pub last_wasm_entry_fp: UnsafeCell<usize>,
+
+ /// The last trap handler from a host-to-wasm entry trampoline on the stack.
+ ///
+ /// This field is configured when the host calls into wasm by the trampoline
+ /// itself. It stores the `pc` of an exception handler suitable to handle
+ /// all traps (or uncaught exceptions).
+ pub last_wasm_entry_trap_handler: UnsafeCell<usize>,
+
+ /// Stack information used by stack switching instructions. See documentation
+ /// on `VMStackChain` for details.
+ #[aggregate]
+ pub stack_chain: UnsafeCell<VMStackChain>,
+
+ /// A pointer to the embedder's `T` inside a `Store<T>`, for use with the
+ /// `store-data-address` unsafe intrinsic.
+ pub store_data: VmPtr<()>,
+
+ /// The range, in addresses, of the guard page that is currently in use.
+ ///
+ /// This field is used when signal handlers are run to determine whether a
+ /// faulting address lies within the guard page of an async stack for
+ /// example. If this happens then the signal handler aborts with a stack
+ /// overflow message similar to what would happen had the stack overflow
+ /// happened on the main thread. This field is, by default a null..null
+ /// range indicating that no async guard is in use (aka no fiber). In such a
+ /// situation while this field is read it'll never classify a fault as an
+ /// guard page fault.
+ #[aggregate]
+ pub async_guard_range: Range<*mut u8>,
+
+ /// The `context.{get,set}` values for the current thread in the component
+ /// model. This is only used for `component-model-async` and slot[1] is only
+ /// used for `component-model-threading`. Despite the conditional use nature
+ /// this is unconditionally present as it avoids the need to make logic in
+ /// `VMOffsets` conditional.
+ ///
+ /// This is saved/restored when threads are swapped in the component model.
+ ///
+ /// NB: `UnsafeCell` because JIT code writes to the slots.
+ #[aggregate]
+ pub component_context: UnsafeCell<[u32; NUM_COMPONENT_CONTEXT_SLOTS]>,
+
+ /// JIT-visible current thread for the component model's sync-to-sync
+ /// adapter fast path.
+ ///
+ /// Like `component_context`, this is unconditionally present to keep
+ /// `VMOffsets` logic unconditional even though it is only used when
+ /// `component-model-async` is enabled.
+ ///
+ /// NB: `UnsafeCell` because JIT code writes to this field.
+ pub current_thread: UnsafeCell<VMLazyThread>,
+ }
+
+ /// JIT-visible representation of the store's current thread for the component
+ /// model, encoded as a single pointer-sized integer so that generated JIT code
+ /// can load, store, and compare it with a handful of instructions.
+ ///
+ /// This is the inline fast-path counterpart to the host-side `CurrentThread`: a
+ /// fused sync-to-sync adapter records a lazy deferred thread here (a pointer to
+ /// a `VMDeferredThread` on its own stack frame) instead of eagerly allocating a
+ /// `GuestTask`/`GuestThread` in the host. Host code promotes the deferred
+ /// thread into a real one only when it actually needs it; see
+ /// `StoreOpaque::force_current_thread`.
+ ///
+ /// This type is a bitpacked equivalent of the following logical `enum`:
+ ///
+ /// ```ignore
+ /// enum VMLazyThread {
+ /// /// No thread.
+ /// None,
+ ///
+ /// /// The lazy thread was promoted and materialized; get it from
+ /// /// `ConcurrentState::current_thread`.
+ /// Forced,
+ ///
+ /// /// The lazy thread has not been materialized, here is a pointer to the
+ /// /// stack-allocated data needed to do force that promotion.
+ /// Deferred(*mut VMDeferredThread),
+ /// }
+ /// ```
+ ///
+ /// Bitpacking details:
+ ///
+ /// * `None`: `0`
+ ///
+ /// * `Forced`: A non-zero value with its low-bit set.
+ ///
+ /// * `Deferred`: A non-zero value with its low-bit clear.
+ #[derive(Debug, Copy, Clone, PartialEq, Eq)]
+ #[repr(transparent)]
+ #[snake_name = vm_lazy_thread]
+ pub struct VMLazyThread {
+ /// The bitpacked thread representation described above.
+ ///
+ /// Private: use the `VMLazyThread::{none,forced,deferred}` constructors
+ /// and the `is_*`/`as_deferred` accessors instead of touching this
+ /// directly.
+ thread: Option<VmPtr<VMDeferredThread>>,
+ }
+
+ /// A deferred component-model thread.
+ ///
+ /// This is an on-stack record pushed by a fused sync-to-sync adapter's fast
+ /// path to defer the work that the `enter_sync_call` libcall would otherwise do
+ /// eagerly.
+ ///
+ /// The adapter allocates one of these in its own stack frame, links the
+ /// previous current-thread value to it via `parent`, and finally points
+ /// `VMStoreContext::current_thread` at it. When host code actually needs the
+ /// real thread, it walks the `parent` chain to materialize thread state (see
+ /// `StoreOpaque::force_current_thread`).
+ #[derive(Debug)]
+ #[repr(C)]
+ #[snake_name = vm_deferred_thread]
+ pub struct VMDeferredThread {
+ /// The previous value of `VMStoreContext::current_thread`.
+ pub parent: VMLazyThread,
+ /// The caller component instance (a deferred `enter_sync_call` argument).
+ pub caller_instance: u32,
+ /// Whether the callee is async-lifted (a deferred `enter_sync_call` arg).
+ pub callee_async: u32,
+ /// The callee component instance (a deferred `enter_sync_call` argument).
+ pub callee_instance: u32,
+ /// The caller thread's `context.{get,set}` slots, saved on entry and
+ /// restored on the fast-path exit (or recovered while forcing).
+ #[aggregate]
+ pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
+ }
}
};
}
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index df665b7472..9324674f1e 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -204,6 +204,7 @@ macro_rules! define_vm_types {
$svis:vis struct $Name:ident {
$(
$(#[doc = $fdoc:literal])*
+ $(#[aggregate])?
$(#[readonly])?
$(#[can_move])?
$fvis:vis $fname:ident : $fty:tt $(< $fgen:ty >)? ,
@@ -719,178 +720,6 @@ const _: () = {
)
};
-/// Structure that holds all mutable context that is shared across all instances
-/// in a store, for example data related to fuel or epochs.
-///
-/// `VMStoreContext`s are one-to-one with `wasmtime::Store`s, the same way that
-/// `VMContext`s are one-to-one with `wasmtime::Instance`s. And the same way
-/// that multiple `wasmtime::Instance`s may be associated with the same
-/// `wasmtime::Store`, multiple `VMContext`s hold a pointer to the same
-/// `VMStoreContext` when they are associated with the same `wasmtime::Store`.
-#[derive(Debug)]
-#[repr(C)]
-pub struct VMStoreContext {
- // NB: 64-bit integer fields are located first with pointer-sized fields
- // trailing afterwards. That makes the offsets in this structure easier to
- // calculate on 32-bit platforms as we don't have to worry about the
- // alignment of 64-bit integers.
- //
- /// Indicator of how much fuel has been consumed and is remaining to
- /// WebAssembly.
- ///
- /// This field is typically negative and increments towards positive. Upon
- /// turning positive a wasm trap will be generated. This field is only
- /// modified if wasm is configured to consume fuel.
- pub fuel_consumed: UnsafeCell<i64>,
-
- /// Deadline epoch for interruption: if epoch-based interruption
- /// is enabled and the global (per engine) epoch counter is
- /// observed to reach or exceed this value, the guest code will
- /// yield if running asynchronously.
- pub epoch_deadline: UnsafeCell<u64>,
-
- /// The "store version".
- ///
- /// This is used to test whether stack-frame handles referring to
- /// suspended stack frames remain valid.
- ///
- /// The invariant that this upward-counting number must satisfy
- /// is: the number must be incremented whenever execution starts
- /// or resumes in the `Store` or when any stack is
- /// dropped/freed. That way, if we take a reference to some
- /// suspended stack frame and track the "version" at the time we
- /// took that reference, if the version still matches, we can be
- /// sure that nothing could have unwound the referenced Wasm
- /// frame.
- ///
- /// This version number is incremented in exactly one place: the
- /// Wasm-to-host trampolines, after return from host code. Note
- /// that this captures both the normal "return into Wasm" case
- /// (where Wasm frames can subsequently return normally and thus
- /// invalidate frames), and the "trap/exception unwinds Wasm
- /// frames" case, which is done internally via the `raise` libcall
- /// invoked after the main hostcall returns an error, and after we
- /// increment this version number.
- ///
- /// Note that this also handles the fiber/future-drop case because
- /// because we *always* return into the trampoline to clean up;
- /// that trampoline immediately raises an error and uses the
- /// longjmp-like unwind within Cranelift frames to skip over all
- /// the guest Wasm frames, but not before it increments the
- /// store's execution version number.
- ///
- /// This field is in use only if guest debugging is enabled.
- pub execution_version: u64,
-
- /// Current stack limit of the wasm module.
- ///
- /// For more information see `crates/cranelift/src/lib.rs`.
- pub stack_limit: UnsafeCell<usize>,
-
- /// The `VMMemoryDefinition` for this store's GC heap.
- pub gc_heap: UnsafeCell<VMMemoryDefinition>,
-
- /// The value of the frame pointer register in the trampoline used
- /// to call from Wasm to the host.
- ///
- /// Maintained by our Wasm-to-host trampoline, and cleared just
- /// before calling into Wasm in `catch_traps`.
- ///
- /// This member is `0` when Wasm is actively running and has not called out
- /// to the host.
- ///
- /// Used to find the start of a contiguous sequence of Wasm frames
- /// when walking the stack. Note that we record the FP of the
- /// *trampoline*'s frame, not the last Wasm frame, because we need
- /// to know the SP (bottom of frame) of the last Wasm frame as
- /// well in case we need to resume to an exception handler in that
- /// frame. The FP of the last Wasm frame can be recovered by
- /// loading the saved FP value at this FP address.
- pub last_wasm_exit_trampoline_fp: UnsafeCell<usize>,
-
- /// The last Wasm program counter before we called from Wasm to the host.
- ///
- /// Maintained by our Wasm-to-host trampoline, and cleared just before
- /// calling into Wasm in `catch_traps`.
- ///
- /// This member is `0` when Wasm is actively running and has not called out
- /// to the host.
- ///
- /// Used when walking a contiguous sequence of Wasm frames.
- pub last_wasm_exit_pc: UnsafeCell<usize>,
-
- /// The last host stack pointer before we called into Wasm from the host.
- ///
- /// Maintained by our host-to-Wasm trampoline. This member is `0` when Wasm
- /// is not running, and it's set to nonzero once a host-to-wasm trampoline
- /// is executed.
- ///
- /// When a host function is wrapped into a `wasmtime::Func`, and is then
- /// called from the host, then this member is not changed meaning that the
- /// previous activation in pointed to by `last_wasm_exit_trampoline_fp` is
- /// still the last wasm set of frames on the stack.
- ///
- /// This field is saved/restored during fiber suspension/resumption
- /// resumption as part of `CallThreadState::swap`.
- ///
- /// This field is used to find the end of a contiguous sequence of Wasm
- /// frames when walking the stack. Additionally it's used when a trap is
- /// raised as part of the set of parameters used to resume in the entry
- /// trampoline's "catch" block.
- pub last_wasm_entry_sp: UnsafeCell<usize>,
-
- /// Same as `last_wasm_entry_sp`, but for the `fp` of the trampoline.
- pub last_wasm_entry_fp: UnsafeCell<usize>,
-
- /// The last trap handler from a host-to-wasm entry trampoline on the stack.
- ///
- /// This field is configured when the host calls into wasm by the trampoline
- /// itself. It stores the `pc` of an exception handler suitable to handle
- /// all traps (or uncaught exceptions).
- pub last_wasm_entry_trap_handler: UnsafeCell<usize>,
-
- /// Stack information used by stack switching instructions. See documentation
- /// on `VMStackChain` for details.
- pub stack_chain: UnsafeCell<VMStackChain>,
-
- /// A pointer to the embedder's `T` inside a `Store<T>`, for use with the
- /// `store-data-address` unsafe intrinsic.
- pub store_data: VmPtr<()>,
-
- /// The range, in addresses, of the guard page that is currently in use.
- ///
- /// This field is used when signal handlers are run to determine whether a
- /// faulting address lies within the guard page of an async stack for
- /// example. If this happens then the signal handler aborts with a stack
- /// overflow message similar to what would happen had the stack overflow
- /// happened on the main thread. This field is, by default a null..null
- /// range indicating that no async guard is in use (aka no fiber). In such a
- /// situation while this field is read it'll never classify a fault as an
- /// guard page fault.
- pub async_guard_range: Range<*mut u8>,
-
- /// The `context.{get,set}` values for the current thread in the component
- /// model. This is only used for `component-model-async` and slot[1] is only
- /// used for `component-model-threading`. Despite the conditional use nature
- /// this is unconditionally present as it avoids the need to make logic in
- /// `VMOffsets` conditional.
- ///
- /// This is saved/restored when threads are swapped in the component model.
- ///
- /// NB: `UnsafeCell` because JIT code writes to the slots.
- pub component_context: UnsafeCell<[u32; NUM_COMPONENT_CONTEXT_SLOTS]>,
-
- /// JIT-visible current thread for the component model's sync-to-sync
- /// adapter fast path.
- ///
- /// Like `component_context`, this is unconditionally present to keep
- /// `VMOffsets` logic unconditional even though it is only used when
- /// `component-model-async` is enabled.
- ///
- /// NB: `UnsafeCell` because JIT code writes to this field.
- pub current_thread: UnsafeCell<VMLazyThread>,
-}
-
impl VMStoreContext {
/// From the current saved trampoline FP, get the FP of the last
/// Wasm frame. If the current saved trampoline FP is null, return
@@ -995,79 +824,33 @@ mod test_vmstore_context {
use core::mem::offset_of;
use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
+ /// Check the `VMStoreContext` offsets that `for_each_vm_type!` does *not*
+ /// generate: the offsets reaching into the inlined `gc_heap`, and the
+ /// indexed `component_context` slot accessor.
+ ///
+ /// Every plain field offset, plus the size and alignment of the type, is
+ /// already checked by the generated `test_vm_type_layouts::vm_store_context`.
#[test]
- fn field_offsets() {
+ fn derived_field_offsets() {
let module = Module::new(StaticModuleIndex::from_u32(0));
let offsets = VMOffsets::new(HostPtr, &module);
- assert_eq!(
- offset_of!(VMStoreContext, stack_limit),
- usize::from(offsets.ptr.vmstore_context_stack_limit())
- );
- assert_eq!(
- offset_of!(VMStoreContext, fuel_consumed),
- usize::from(offsets.ptr.vmstore_context_fuel_consumed())
- );
- assert_eq!(
- offset_of!(VMStoreContext, epoch_deadline),
- usize::from(offsets.ptr.vmstore_context_epoch_deadline())
- );
- assert_eq!(
- offset_of!(VMStoreContext, execution_version),
- usize::from(offsets.ptr.vmstore_context_execution_version())
- );
- assert_eq!(
- offset_of!(VMStoreContext, gc_heap),
- usize::from(offsets.ptr.vmstore_context_gc_heap())
- );
assert_eq!(
offset_of!(VMStoreContext, gc_heap) + offset_of!(VMMemoryDefinition, base),
- usize::from(offsets.ptr.vmstore_context_gc_heap_base())
+ usize::from(offsets.ptr.vm_store_context().gc_heap_base())
);
assert_eq!(
offset_of!(VMStoreContext, gc_heap) + offset_of!(VMMemoryDefinition, current_length),
- usize::from(offsets.ptr.vmstore_context_gc_heap_current_length())
- );
- assert_eq!(
- offset_of!(VMStoreContext, last_wasm_exit_trampoline_fp),
- usize::from(offsets.ptr.vmstore_context_last_wasm_exit_trampoline_fp())
- );
- assert_eq!(
- offset_of!(VMStoreContext, last_wasm_exit_pc),
- usize::from(offsets.ptr.vmstore_context_last_wasm_exit_pc())
- );
- assert_eq!(
- offset_of!(VMStoreContext, last_wasm_entry_fp),
- usize::from(offsets.ptr.vmstore_context_last_wasm_entry_fp())
- );
- assert_eq!(
- offset_of!(VMStoreContext, last_wasm_entry_sp),
- usize::from(offsets.ptr.vmstore_context_last_wasm_entry_sp())
- );
- assert_eq!(
- offset_of!(VMStoreContext, last_wasm_entry_trap_handler),
- usize::from(offsets.ptr.vmstore_context_last_wasm_entry_trap_handler())
- );
- assert_eq!(
- offset_of!(VMStoreContext, stack_chain),
- usize::from(offsets.ptr.vmstore_context_stack_chain())
- );
- assert_eq!(
- offset_of!(VMStoreContext, store_data),
- usize::from(offsets.ptr.vmstore_context_store_data())
+ usize::from(offsets.ptr.vm_store_context().gc_heap_current_length())
);
assert_eq!(
offset_of!(VMStoreContext, component_context),
- usize::from(offsets.ptr.vmstore_context_component_context_slot(0))
- );
- assert_eq!(
- offset_of!(VMStoreContext, current_thread),
- usize::from(offsets.ptr.vmstore_context_current_thread())
+ usize::from(offsets.ptr.vm_store_context().component_context_slot(0))
);
// Make sure that the calculation for the size of a slot is also
// accurate.
- let slot_width = offsets.ptr.vmstore_context_component_context_slot(1)
- - offsets.ptr.vmstore_context_component_context_slot(0);
+ let slot_width = offsets.ptr.vm_store_context().component_context_slot(1)
+ - offsets.ptr.vm_store_context().component_context_slot(0);
let mut default = VMStoreContext::default();
assert_eq!(
size_of_val(&default.component_context.get_mut()[0]),
@@ -1076,45 +859,6 @@ mod test_vmstore_context {
}
}
-/// JIT-visible representation of the store's current thread for the component
-/// model, encoded as a single pointer-sized integer so that generated JIT code
-/// can load, store, and compare it with a handful of instructions.
-///
-/// This is the inline fast-path counterpart to the host-side `CurrentThread`: a
-/// fused sync-to-sync adapter records a lazy deferred thread here (a pointer to
-/// a `VMDeferredThread` on its own stack frame) instead of eagerly allocating a
-/// `GuestTask`/`GuestThread` in the host. Host code promotes the deferred
-/// thread into a real one only when it actually needs it; see
-/// `StoreOpaque::force_current_thread`.
-///
-/// This type is a bitpacked equivalent of the following logical `enum`:
-///
-/// ```ignore
-/// enum VMLazyThread {
-/// /// No thread.
-/// None,
-///
-/// /// The lazy thread was promoted and materialized; get it from
-/// /// `ConcurrentState::current_thread`.
-/// Forced,
-///
-/// /// The lazy thread has not been materialized, here is a pointer to the
-/// /// stack-allocated data needed to do force that promotion.
-/// Deferred(*mut VMDeferredThread),
-/// }
-/// ```
-//
-// Bitpacking details:
-//
-// * `None`: `0`
-//
-// * `Forced`: A non-zero value with its low-bit set.
-//
-// * `Deferred`: A non-zero value with its low-bit clear.
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
-#[repr(transparent)]
-pub struct VMLazyThread(Option<VmPtr<VMDeferredThread>>);
-
impl VMLazyThread {
const _ASSERT_SIZE: () = assert!(
core::mem::size_of::<VMLazyThread>() == core::mem::size_of::<*mut VMDeferredThread>()
@@ -1127,40 +871,44 @@ impl VMLazyThread {
/// There is no current thread.
pub const fn none() -> Self {
- Self(None)
+ Self { thread: None }
}
/// A lazy thread that has already been promoted.
pub const fn forced() -> Self {
- Self(Some(Self::FORCED))
+ Self {
+ thread: Some(Self::FORCED),
+ }
}
/// A deferred thread referencing the given on-stack [`VMDeferredThread`].
pub fn deferred(ptr: NonNull<VMDeferredThread>) -> Self {
debug_assert_eq!(ptr.addr().get() & Self::FORCED.addr().get(), 0);
- Self(Some(ptr.into()))
+ Self {
+ thread: Some(ptr.into()),
+ }
}
/// Returns `true` if there is no current thread.
pub fn is_none(self) -> bool {
- self.0.is_none()
+ self.thread.is_none()
}
/// Returns `true` if a deferred thread has been forced/promoted.
pub fn is_forced(self) -> bool {
- self.0.is_some_and(|p| p == Self::FORCED)
+ self.thread.is_some_and(|p| p == Self::FORCED)
}
/// Returns `true` if this is a deferred thread (i.e. neither `None` nor
/// forced).
pub fn is_deferred(self) -> bool {
- self.0.is_some_and(|p| p != Self::FORCED)
+ self.thread.is_some_and(|p| p != Self::FORCED)
}
/// Returns the deferred [`VMDeferredThread`] pointer if this is a deferred
/// thread.
pub fn as_deferred(self) -> Option<VmPtr<VMDeferredThread>> {
- self.0
+ self.thread
.and_then(|p| if p == Self::FORCED { None } else { Some(p) })
}
}
@@ -1172,73 +920,32 @@ mod test_vmlazy_thread {
#[test]
fn vmlazy_thread_forced() {
assert_eq!(
- VMLazyThread::forced().0.unwrap().addr().get(),
+ VMLazyThread::forced().thread.unwrap().addr().get(),
usize::try_from(wasmtime_environ::VM_LAZY_THREAD_FORCED).unwrap()
);
}
}
-/// A deferred component-model thread.
-///
-/// This is an on-stack record pushed by a fused sync-to-sync adapter's fast
-/// path to defer the work that the `enter_sync_call` libcall would otherwise do
-/// eagerly.
-///
-/// The adapter allocates one of these in its own stack frame, links the
-/// previous current-thread value to it via `parent`, and finally points
-/// `VMStoreContext::current_thread` at it. When host code actually needs the
-/// real thread, it walks the `parent` chain to materialize thread state (see
-/// `StoreOpaque::force_current_thread`).
-#[derive(Debug)]
-#[repr(C)]
-pub struct VMDeferredThread {
- /// The previous value of `VMStoreContext::current_thread`.
- pub parent: VMLazyThread,
- /// The caller component instance (a deferred `enter_sync_call` argument).
- pub caller_instance: u32,
- /// Whether the callee is async-lifted (a deferred `enter_sync_call` arg).
- pub callee_async: u32,
- /// The callee component instance (a deferred `enter_sync_call` argument).
- pub callee_instance: u32,
- /// The caller thread's `context.{get,set}` slots, saved on entry and
- /// restored on the fast-path exit (or recovered while forcing).
- pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
-}
-
#[cfg(test)]
mod test_vmdeferred_thread {
use super::*;
use core::mem::offset_of;
use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};
+ /// Check the indexed `saved_context` slot accessor, which
+ /// `for_each_vm_type!` does not generate.
+ ///
+ /// Every plain field offset, plus the size and alignment of the type, is
+ /// already checked by the generated
+ /// `test_vm_type_layouts::vm_deferred_thread`.
#[test]
- fn deferred_thread_field_offsets() {
+ fn deferred_thread_derived_field_offsets() {
let module = Module::new(StaticModuleIndex::from_u32(0));
let offsets = VMOffsets::new(HostPtr, &module);
let ptr = offsets.ptr;
- assert_eq!(
- offset_of!(VMDeferredThread, parent),
- usize::from(ptr.vmdeferred_thread_parent())
- );
- assert_eq!(
- offset_of!(VMDeferredThread, caller_instance),
- usize::from(ptr.vmdeferred_thread_caller_instance())
- );
- assert_eq!(
- offset_of!(VMDeferredThread, callee_async),
- usize::from(ptr.vmdeferred_thread_callee_async())
- );
- assert_eq!(
- offset_of!(VMDeferredThread, callee_instance),
- usize::from(ptr.vmdeferred_thread_callee_instance())
- );
assert_eq!(
offset_of!(VMDeferredThread, saved_context),
- usize::from(ptr.vmdeferred_thread_saved_context(0))
- );
- assert_eq!(
- size_of::<VMDeferredThread>(),
- usize::from(ptr.size_of_vmdeferred_thread())
+ usize::from(ptr.vm_deferred_thread().saved_context_slot(0))
);
}
}
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 95df897ec1..34d7d1e90e 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -2099,7 +2099,7 @@ where
/// `VMStoreContext`.
fn emit_load_fuel_consumed(&mut self, fuel_reg: Reg) -> Result<()> {
let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
- let fuel_offset = self.env.vmoffsets.ptr.vmstore_context_fuel_consumed();
+ let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed();
self.masm.load_ptr(
self.masm
.address_at_vmctx(u32::from(store_context_offset))?,
@@ -2178,7 +2178,7 @@ where
) -> Result<()> {
let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx_epoch_ptr();
let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
- let epoch_deadline_offset = self.env.vmoffsets.ptr.vmstore_context_epoch_deadline();
+ let epoch_deadline_offset = self.env.vmoffsets.ptr.vm_store_context().epoch_deadline();
// Load the current epoch value into `epoch_counter_var`.
self.masm.load_ptr(
@@ -2219,7 +2219,7 @@ where
}
let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();
- let fuel_offset = self.env.vmoffsets.ptr.vmstore_context_fuel_consumed();
+ let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed();
let limits_reg = self.context.any_gpr(self.masm)?;
// Load `VMStoreContext` into the `limits_reg` reg.
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index da439cd970..c5c882216c 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -198,7 +198,7 @@ impl Masm for MacroAssembler {
masm.load_ptr(
Address::offset(
scratch_stk_limit.inner(),
- ptr_size_u8.vmstore_context_stack_limit().into(),
+ ptr_size_u8.vm_store_context().stack_limit().into(),
),
scratch_stk_limit.writable(),
)?;
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index 2c2fd67d7c..c29ddb9541 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -136,7 +136,7 @@ impl Masm for MacroAssembler {
masm.load_ptr(
Address::offset(
scratch.inner(),
- ptr_size.vmstore_context_stack_limit().into(),
+ ptr_size.vm_store_context().stack_limit().into(),
),
scratch.writable(),
)?;