A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Move `VMStoreContext`, `VMDeferredThread`, and `VMLazyThread` into `f… (#14020)
Nick Fitzgerald2f06aadad23d
diff --git a/crates/cranelift/src/alias_region.rs b/crates/cranelift/src/alias_region.rsindex 061041db9c..873f34b7a2 100644--- a/crates/cranelift/src/alias_region.rs+++ b/crates/cranelift/src/alias_region.rs@@ -38,6 +38,11 @@ enum VmType {VMCopyingHeapData,VMNullHeapData,VMDeferredThread,+ #[allow(+ dead_code,+ reason = "generated uniformly for all VM types via `for_each_vm_type!`"+ )]+ VMLazyThread,VMContRef,ContinuationStackMemory,VMFunctionImport,@@ -185,6 +190,7 @@ impl AliasRegionKey {const DATA_SEGMENT_KIND: u32 = Self::new_kind(0b100000);const VM_GLOBAL_DEFINITION_KIND: u32 = Self::new_kind(0b100001);const VM_TAG_DEFINITION_KIND: u32 = Self::new_kind(0b100010);+ const VM_LAZY_THREAD_KIND: u32 = Self::new_kind(0b100011);/// Encode this key into a raw `u32` suitable for use as an/// `AliasRegionData::user_id`.@@ -204,6 +210,7 @@ impl AliasRegionKey {VmType::VMCopyingHeapData => Self::VM_COPYING_HEAP_DATA_KIND,VmType::VMNullHeapData => Self::VM_NULL_HEAP_DATA_KIND,VmType::VMDeferredThread => Self::VM_DEFERRED_THREAD_KIND,+ VmType::VMLazyThread => Self::VM_LAZY_THREAD_KIND,VmType::VMContRef => Self::VM_CONTREF_KIND,VmType::ContinuationStackMemory => Self::CONTINUATION_STACK_MEMORY_KIND,VmType::VMFunctionImport => Self::VM_FUNCTION_IMPORT_KIND,@@ -538,12 +545,29 @@ impl<'a, Offsets> Field<'a, Offsets> {uniformly but not exercised until a VM type uses those markers")]macro_rules! define_vm_type_alias_region_helpers {+ // `UnsafeCell<T>` is `repr(transparent)` and is accessed exactly as its `T`+ // would be; delegate to the inner type.+ (@field_ty $pt:expr, UnsafeCell < $inner:tt >) => {+ define_vm_type_alias_region_helpers!(@field_ty $pt, $inner)+ };+// Classify a field type to its Cranelift `ir::Type`, given `$pt` (the// target pointer type as an `ir::Type`).+ //+ // Note that there is deliberately no arm for a composite field (a nested+ // struct, an array, or a range): such a field has no single Cranelift type,+ // and is marked `#[aggregate]` in `for_each_vm_type!` so that no accessor is+ // generated for it at all.(@field_ty $pt:expr, VmPtr < $g:ty >) => { $pt };(@field_ty $pt:expr, Option < VmPtr < $g:ty >>) => { $pt };(@field_ty $pt:expr, AtomicUsize) => { $pt };(@field_ty $pt:expr, usize) => { $pt };+ (@field_ty $pt:expr, i64) => { ir::types::I64 };+ (@field_ty $pt:expr, u64) => { ir::types::I64 };+ (@field_ty $pt:expr, u32) => { ir::types::I32 };+ // `VMLazyThread` is a pointer-sized bitpacked integer; see its definition in+ // `for_each_vm_type!`.+ (@field_ty $pt:expr, VMLazyThread) => { $pt };(@field_ty $pt:expr, [u8; 16]) => { ir::types::I8X16 };(@field_ty $pt:expr, VMSharedTypeIndex) => { ir::types::I32 };(@field_ty $pt:expr, DefinedTableIndex) => { ir::types::I32 };@@ -632,25 +656,34 @@ macro_rules! define_vm_type_alias_region_helpers {// >)?` within a repetition) is what lets each field's type reach the// `@field_ty` classifier as raw tokens, so it can require `Option`s to// specifically be `Option<VmPtr<_>>`.- (@munch $Name:ident $snake:ident { $($groups:tt)* } [ $($pend:tt)* ]) => {+ (@munch $Name:ident $snake:ident { $($groups:tt)* }) => {define_vm_type_alias_region_helpers!(@emit $Name $snake { $($groups)* });};- // Stash a field attribute's bracket group (`[readonly]`, `[can_move]`,- // `[doc = ...]`) as pending for the next field.- (@munch $Name:ident $snake:ident { $($groups:tt)* } [ $($pend:tt)* ] # $fattr:tt $($rest:tt)*) => {- define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* } [ $($pend)* $fattr ] $($rest)*);+ // A field marked `#[aggregate]` is a composite (a nested struct or array)+ // that is not accessed as a whole by Wasm code, but piecewise+ // instead. Therefore, there is no single Cranelift type for such fields, so+ // no accessors are generated for them; interior accesses are computed from+ // the field's offset instead, which the generated `offsets::*` methods+ // still provide.+ (@munch $Name:ident $snake:ident { $($groups:tt)* }+ $(#[doc = $fdoc:literal])* #[aggregate] $fvis:vis $fname:ident : $fty:ty , $($rest:tt)*+ ) => {+ define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* } $($rest)*);};- // Consume a field's visibility and name, then collect its type tokens.- (@munch $Name:ident $snake:ident { $($groups:tt)* } [ $($pend:tt)* ] $fvis:vis $fname:ident : $($rest:tt)*) => {- define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($pend)* ] [] $($rest)*);+ // Consume one field's attributes, visibility, and name, then collect its+ // type tokens.+ (@munch $Name:ident $snake:ident { $($groups:tt)* }+ $(# $fattr:tt)* $fvis:vis $fname:ident : $($rest:tt)*+ ) => {+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($fattr)* ] [] $($rest)*);};// Accumulate one field's type tokens up to its terminating comma, then- // append the completed group and reset the pending attributes.- (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($pend:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {- define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fname [ $($pend)* ] [ $($fty)* ] } } [] $($rest)*);+ // append the completed group.+ (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] , $($rest:tt)*) => {+ define_vm_type_alias_region_helpers!(@munch $Name $snake { $($groups)* { $fname [ $($fattr)* ] [ $($fty)* ] } } $($rest)*);};- (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($pend:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {- define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($pend)* ] [ $($fty)* $tok ] $($rest)*);+ (@munch_ty $Name:ident $snake:ident { $($groups:tt)* } $fname:ident [ $($fattr:tt)* ] [ $($fty:tt)* ] $tok:tt $($rest:tt)*) => {+ define_vm_type_alias_region_helpers!(@munch_ty $Name $snake { $($groups)* } $fname [ $($fattr)* ] [ $($fty)* $tok ] $($rest)*);};// Top-level entry: the list of `VM*` type definitions.@@ -664,7 +697,7 @@ macro_rules! define_vm_type_alias_region_helpers {})* ) => {$(- define_vm_type_alias_region_helpers!(@munch $Name $snake {} [] $($body)*);+ define_vm_type_alias_region_helpers!(@munch $Name $snake {} $($body)*);)*};}@@ -1323,7 +1356,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_store_data()+ .vm_store_context()+ .store_data().into(),)}@@ -1341,7 +1375,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_execution_version()+ .vm_store_context()+ .execution_version().into(),)}@@ -1359,7 +1394,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_execution_version()+ .vm_store_context()+ .execution_version().into(),new_version,)@@ -1378,7 +1414,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_fuel_consumed()+ .vm_store_context()+ .fuel_consumed().into(),)}@@ -1396,7 +1433,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_fuel_consumed()+ .vm_store_context()+ .fuel_consumed().into(),fuel_consumed,)@@ -1415,7 +1453,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_epoch_deadline()+ .vm_store_context()+ .epoch_deadline().into(),)}@@ -1425,7 +1464,8 @@ wherelet offset = self.offsets.get_ptr_size()- .vmstore_context_stack_limit()+ .vm_store_context()+ .stack_limit().into();let region = self.vmstore_context_region(func, offset);Load {@@ -1458,7 +1498,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_stack_limit()+ .vm_store_context()+ .stack_limit().into(),stack_limit,)@@ -1478,7 +1519,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_current_thread()+ .vm_store_context()+ .current_thread().into(),)}@@ -1496,7 +1538,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_current_thread()+ .vm_store_context()+ .current_thread().into(),new_thread,)@@ -1514,7 +1557,8 @@ wherelet offset = self.offsets.get_ptr_size()- .vmstore_context_gc_heap_base()+ .vm_store_context()+ .gc_heap_base().into();let region = self.vmstore_context_region(func, offset);Load {@@ -1543,7 +1587,8 @@ wherelet offset = self.offsets.get_ptr_size()- .vmstore_context_gc_heap_current_length()+ .vm_store_context()+ .gc_heap_current_length().into();let region = self.vmstore_context_region(func, offset);Load {@@ -1576,7 +1621,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_last_wasm_entry_fp()+ .vm_store_context()+ .last_wasm_entry_fp().into(),)}@@ -1594,7 +1640,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_last_wasm_entry_sp()+ .vm_store_context()+ .last_wasm_entry_sp().into(),)}@@ -1612,7 +1659,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_last_wasm_entry_trap_handler()+ .vm_store_context()+ .last_wasm_entry_trap_handler().into(),)}@@ -1630,7 +1678,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_last_wasm_entry_fp()+ .vm_store_context()+ .last_wasm_entry_fp().into(),fp,)@@ -1649,7 +1698,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_last_wasm_entry_sp()+ .vm_store_context()+ .last_wasm_entry_sp().into(),sp,)@@ -1668,7 +1718,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_last_wasm_entry_trap_handler()+ .vm_store_context()+ .last_wasm_entry_trap_handler().into(),trap_handler,)@@ -1687,7 +1738,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_last_wasm_exit_trampoline_fp()+ .vm_store_context()+ .last_wasm_exit_trampoline_fp().into(),fp,)@@ -1706,7 +1758,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_last_wasm_exit_pc()+ .vm_store_context()+ .last_wasm_exit_pc().into(),pc,)@@ -1721,7 +1774,7 @@ where&mut self,func: &mut ir::Function,) -> ir::AliasRegion {- let offset = self.offsets.get_ptr_size().vmstore_context_stack_chain();+ let offset = self.offsets.get_ptr_size().vm_store_context().stack_chain();self.vmstore_context_region(func, offset.into())}@@ -1743,7 +1796,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_component_context_slot(slot)+ .vm_store_context()+ .component_context_slot(slot).into(),)}@@ -1765,7 +1819,8 @@ wherevmstore_ctx,self.offsets.get_ptr_size()- .vmstore_context_component_context_slot(slot)+ .vm_store_context()+ .component_context_slot(slot).into(),val,)@@ -1838,7 +1893,8 @@ wherevmdeferred_thread_ptr,self.offsets.get_ptr_size()- .vmdeferred_thread_parent()+ .vm_deferred_thread()+ .parent().into(),)}@@ -1856,7 +1912,8 @@ wherevmdeferred_thread_ptr,self.offsets.get_ptr_size()- .vmdeferred_thread_parent()+ .vm_deferred_thread()+ .parent().into(),parent,)@@ -1875,7 +1932,8 @@ wherevmdeferred_thread_ptr,self.offsets.get_ptr_size()- .vmdeferred_thread_caller_instance()+ .vm_deferred_thread()+ .caller_instance().into(),caller_instance,)@@ -1894,7 +1952,8 @@ wherevmdeferred_thread_ptr,self.offsets.get_ptr_size()- .vmdeferred_thread_callee_async()+ .vm_deferred_thread()+ .callee_async().into(),callee_async,)@@ -1913,7 +1972,8 @@ wherevmdeferred_thread_ptr,self.offsets.get_ptr_size()- .vmdeferred_thread_callee_instance()+ .vm_deferred_thread()+ .callee_instance().into(),callee_instance,)@@ -1934,7 +1994,8 @@ wherevmdeferred_thread_ptr,self.offsets.get_ptr_size()- .vmdeferred_thread_saved_context(i)+ .vm_deferred_thread()+ .saved_context_slot(i).into(),)}@@ -1953,7 +2014,8 @@ wherevmdeferred_thread_ptr,self.offsets.get_ptr_size()- .vmdeferred_thread_saved_context(i)+ .vm_deferred_thread()+ .saved_context_slot(i).into(),val,)diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 9da84e7e9d..ba84e26174 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -2049,7 +2049,7 @@ impl<'a, 'func, 'module_env> Call<'a, 'func, 'module_env> {let ptr = self.env.offsets.ptr;// Allocate the on-stack `VMDeferredThread`.- let size = u32::from(ptr.size_of_vmdeferred_thread());+ let size = u32::from(ptr.vm_deferred_thread().size());let align_shift = u8::try_from(ptr.size().trailing_zeros()).unwrap();let slot = self.builderdiff --git a/crates/cranelift/src/func_environ/stack_switching/instructions.rs b/crates/cranelift/src/func_environ/stack_switching/instructions.rsindex 722406cd40..6cdf9d168c 100644--- a/crates/cranelift/src/func_environ/stack_switching/instructions.rs+++ b/crates/cranelift/src/func_environ/stack_switching/instructions.rs@@ -1027,7 +1027,7 @@ pub fn vmctx_load_stack_chain<'a>(builder: &mut FunctionBuilder,vmctx: ir::Value,) -> VMStackChain {- let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into();+ let stack_chain_offset = env.offsets.ptr.vm_store_context().stack_chain().into();// First we need to get the `VMStoreContext`.let vm_store_context = env@@ -1055,7 +1055,7 @@ pub fn vmctx_store_stack_chain<'a>(vmctx: ir::Value,stack_chain: &VMStackChain,) {- let stack_chain_offset = env.offsets.ptr.vmstore_context_stack_chain().into();+ let stack_chain_offset = env.offsets.ptr.vm_store_context().stack_chain().into();// First we need to get the `VMStoreContext`.let vm_store_context = envdiff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex e746a0b627..79f7bda603 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -66,6 +66,10 @@ fn align(offset: u32, width: u32) -> u32 {/// `VM*` type, with a method per field returning that field's offset, plus/// `size` and `align` methods.macro_rules! define_vm_type_offsets {+ // `UnsafeCell<T>` is `repr(transparent)`, so it has exactly `T`'s layout;+ // delegate to the inner type.+ (@size ($p:expr) UnsafeCell < $inner:tt >) => { define_vm_type_offsets!(@size ($p) $inner) };+// Classify a field type to its size in bytes as a `u32`, given `$p` (the// target pointer size as a `u8`). All `VmPtr<_>` and `Option<VmPtr<_>>`// fields are pointer-sized; the `Defined*Index` types are `u32` entity@@ -75,28 +79,58 @@ macro_rules! define_vm_type_offsets {(@size ($p:expr) Option < VmPtr < $g:ty >>) => { u32::from($p) };(@size ($p:expr) AtomicUsize) => { u32::from($p) };(@size ($p:expr) usize) => { u32::from($p) };+ (@size ($p:expr) i64) => { 8u32 };+ (@size ($p:expr) u64) => { 8u32 };+ (@size ($p:expr) u32) => { 4u32 };(@size ($p:expr) [u8; 16]) => { 16u32 };+ (@size ($p:expr) [u32; $n:expr]) => { 4u32 * u32::try_from($n).unwrap() };(@size ($p:expr) VMSharedTypeIndex) => { u32::from(($p).size_of_vmshared_type_index()) };(@size ($p:expr) DefinedTableIndex) => { 4u32 };(@size ($p:expr) DefinedMemoryIndex) => { 4u32 };(@size ($p:expr) DefinedTagIndex) => { 4u32 };(@size ($p:expr) VMGlobalKind) => { 8u32 };+ // `Range<T>` is a two-field `{ start: T, end: T }` struct.+ (@size ($p:expr) Range < *mut u8 >) => { 2u32 * u32::from($p) };+ // Nested `VM*` types recurse through their own generated offsets, keeping+ // this macro the single source of truth for their layout too.+ (@size ($p:expr) VMMemoryDefinition) => { u32::from(($p).vm_memory_definition().size()) };+ (@size ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().size()) };+ // `VMStackChain` is a `repr(usize, C)` enum, and is not itself defined by+ // `for_each_vm_type!`.+ (@size ($p:expr) VMStackChain) => { u32::from(($p).size_of_vmstack_chain()) };++ // As with `@size` above, `UnsafeCell<T>` has exactly `T`'s alignment.+ (@align ($p:expr) UnsafeCell < $inner:tt >) => { define_vm_type_offsets!(@align ($p) $inner) };// Classify a field type to its alignment in bytes as a `u32`, given `$p`// (the target pointer size as a `u8`).+ //+ // NB: 64-bit integers are assumed to be 8-aligned, which holds everywhere except+ // `i686-unknown-linux-gnu`, and the pointer size alone can't tell those apart. Types+ // with 64-bit fields must therefore put them first and force their own alignment with+ // `#[repr(C, align(8))]`, as `VMStoreContext` does.(@align ($p:expr) VmPtr < $g:ty >) => { u32::from($p) };(@align ($p:expr) Option < VmPtr < $g:ty >>) => { u32::from($p) };(@align ($p:expr) AtomicUsize) => { u32::from($p) };(@align ($p:expr) usize) => { u32::from($p) };+ (@align ($p:expr) i64) => { 8u32 };+ (@align ($p:expr) u64) => { 8u32 };+ (@align ($p:expr) u32) => { 4u32 };(@align ($p:expr) [u8; 16]) => { 16u32 };+ (@align ($p:expr) [u32; $n:expr]) => { 4u32 };(@align ($p:expr) VMSharedTypeIndex) => { u32::from(($p).align_of_vmshared_type_index()) };(@align ($p:expr) DefinedTableIndex) => { 4u32 };(@align ($p:expr) DefinedMemoryIndex) => { 4u32 };(@align ($p:expr) DefinedTagIndex) => { 4u32 };(@align ($p:expr) VMGlobalKind) => { 4u32 };+ (@align ($p:expr) Range < *mut u8 >) => { u32::from($p) };+ (@align ($p:expr) VMMemoryDefinition) => { u32::from(($p).vm_memory_definition().align()) };+ (@align ($p:expr) VMLazyThread) => { u32::from(($p).vm_lazy_thread().align()) };+ (@align ($p:expr) VMStackChain) => { u32::from($p) };// Classify a `#[repr(...)]` to the minimum alignment it forces, as a `u32`.(@repr_align C) => { 1u32 };+ (@repr_align transparent) => { 1u32 };(@repr_align C, align($n:literal)) => {{ let align: u32 = $n; align }};// Emit a `pub fn` per field returning that field's offset, computed by@@ -181,12 +215,13 @@ macro_rules! define_vm_type_offsets {}}};- // Skip a field attribute (`#[doc = ...]`, `#[readonly]`, `#[can_move]`).- (@impl $Name:ident $repr:tt { $($groups:tt)* } #[$($attr:tt)*] $($rest:tt)*) => {- define_vm_type_offsets!(@impl $Name $repr { $($groups)* } $($rest)*);- };- // Consume a field's visibility and name, then collect its type tokens.- (@impl $Name:ident $repr:tt { $($groups:tt)* } $fvis:vis $fname:ident : $($rest:tt)*) => {+ // Consume one field's attributes, visibility, and name, then collect its+ // type tokens. None of the field attributes (doc comments and the+ // `#[aggregate]`/`#[readonly]`/`#[can_move]` markers) affect layout, so they+ // are all discarded here.+ (@impl $Name:ident $repr:tt { $($groups:tt)* }+ $(#[$($attr:tt)*])* $fvis:vis $fname:ident : $($rest:tt)*+ ) => {define_vm_type_offsets!(@impl_ty $Name $repr { $($groups)* } $fname [] $($rest)*);};// Accumulate one field's type tokens up to its terminating comma, then@@ -215,7 +250,7 @@ macro_rules! define_vm_type_offsets {/// These types are namespaced within their own module so that they never/// collide with the real definitions of the `VM*` types themselves.pub mod offsets {- use super::{align, PtrSize};+ use super::{align, PtrSize, NUM_COMPONENT_CONTEXT_SLOTS};$(#[doc = concat!("Offsets of fields within the `", stringify!($Name), "` type.")]@@ -228,6 +263,46 @@ macro_rules! define_vm_type_offsets {}for_each_vm_type!(define_vm_type_offsets);+/// The size, in bytes, of one `context.{get,set}` slot. These slots are `u32`s,+/// both in `VMStoreContext::component_context` and in+/// `VMDeferredThread::saved_context`.+const COMPONENT_CONTEXT_SLOT_SIZE: u8 = 4;++/// Offsets within a `VMStoreContext` that are not simply the offset of one of+/// its fields, and so are not generated by `for_each_vm_type!`.+impl<P: PtrSize> offsets::VMStoreContext<P> {+ /// The offset of the `gc_heap.base` field within a `VMStoreContext`.+ pub fn gc_heap_base(&self) -> u8 {+ let offset = self.gc_heap() + self.0.vm_memory_definition().base();+ debug_assert!(offset < self.last_wasm_exit_trampoline_fp());+ offset+ }++ /// The offset of the `gc_heap.current_length` field within a+ /// `VMStoreContext`.+ pub fn gc_heap_current_length(&self) -> u8 {+ let offset = self.gc_heap() + self.0.vm_memory_definition().current_length();+ debug_assert!(offset < self.last_wasm_exit_trampoline_fp());+ offset+ }++ /// The offset of the `component_context[i]` slot within a `VMStoreContext`.+ pub fn component_context_slot(&self, i: u8) -> u8 {+ assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);+ self.component_context() + i * COMPONENT_CONTEXT_SLOT_SIZE+ }+}++/// Offsets within a `VMDeferredThread` that are not simply the offset of one of+/// its fields, and so are not generated by `for_each_vm_type!`.+impl<P: PtrSize> offsets::VMDeferredThread<P> {+ /// The offset of the `saved_context[i]` slot within a `VMDeferredThread`.+ pub fn saved_context_slot(&self, i: u8) -> u8 {+ assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);+ self.saved_context() + i * COMPONENT_CONTEXT_SLOT_SIZE+ }+}+/// Add a `fn vm_foo(&self) -> offsets::VMFoo<&Self>` accessor to [`PtrSize`] for/// each `VM*` type, using the `#[snake_name = ...]` attribute for the method/// name.@@ -238,12 +313,9 @@ macro_rules! define_ptr_size_vm_type_accessors {#[repr($($repr:tt)*)]#[snake_name = $snake:ident]$svis:vis struct $Name:ident {- $(- $(#[doc = $fdoc:literal])*- $(#[readonly])?- $(#[can_move])?- $fvis:vis $fname:ident : $fty:tt $(< $fgen:ty >)? ,- )*+ // This macro only needs each type's name and snake name, so the+ // body is captured raw rather than parsed into fields.+ $($body:tt)*})* ) => {$(@@ -350,143 +422,6 @@ pub trait PtrSize {self.vm_global_definition().size()}- // Offsets within `VMStoreContext`-- /// Return the offset of the `fuel_consumed` field of `VMStoreContext`- #[inline]- fn vmstore_context_fuel_consumed(&self) -> u8 {- 0- }-- /// Return the offset of the `epoch_deadline` field of `VMStoreContext`- #[inline]- fn vmstore_context_epoch_deadline(&self) -> u8 {- self.vmstore_context_fuel_consumed() + 8- }-- /// Return the offset of the `execution_version` field of- /// `VMStoreContext`- #[inline]- fn vmstore_context_execution_version(&self) -> u8 {- self.vmstore_context_epoch_deadline() + 8- }-- /// Return the offset of the `stack_limit` field of `VMStoreContext`- #[inline]- fn vmstore_context_stack_limit(&self) -> u8 {- self.vmstore_context_execution_version() + 8- }-- /// Return the offset of the `gc_heap` field of `VMStoreContext`.- #[inline]- fn vmstore_context_gc_heap(&self) -> u8 {- self.vmstore_context_stack_limit() + self.size()- }-- /// Return the offset of the `gc_heap.base` field within a `VMStoreContext`.- fn vmstore_context_gc_heap_base(&self) -> u8 {- let offset = self.vmstore_context_gc_heap() + self.vm_memory_definition().base();- debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());- offset- }-- /// Return the offset of the `gc_heap.current_length` field within a `VMStoreContext`.- fn vmstore_context_gc_heap_current_length(&self) -> u8 {- let offset = self.vmstore_context_gc_heap() + self.vm_memory_definition().current_length();- debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());- offset- }-- /// Return the offset of the `last_wasm_exit_trampoline_fp` field- /// of `VMStoreContext`.- fn vmstore_context_last_wasm_exit_trampoline_fp(&self) -> u8 {- self.vmstore_context_gc_heap() + self.vm_memory_definition().size()- }-- /// Return the offset of the `last_wasm_exit_pc` field of `VMStoreContext`.- fn vmstore_context_last_wasm_exit_pc(&self) -> u8 {- self.vmstore_context_last_wasm_exit_trampoline_fp() + self.size()- }-- /// Return the offset of the `last_wasm_entry_sp` field of `VMStoreContext`.- fn vmstore_context_last_wasm_entry_sp(&self) -> u8 {- self.vmstore_context_last_wasm_exit_pc() + self.size()- }-- /// Return the offset of the `last_wasm_entry_fp` field of `VMStoreContext`.- fn vmstore_context_last_wasm_entry_fp(&self) -> u8 {- self.vmstore_context_last_wasm_entry_sp() + self.size()- }-- /// Return the offset of the `last_wasm_entry_trap_handler` field of `VMStoreContext`.- fn vmstore_context_last_wasm_entry_trap_handler(&self) -> u8 {- self.vmstore_context_last_wasm_entry_fp() + self.size()- }-- /// Return the offset of the `stack_chain` field of `VMStoreContext`.- fn vmstore_context_stack_chain(&self) -> u8 {- self.vmstore_context_last_wasm_entry_trap_handler() + self.size()- }-- /// Return the offset of the `stack_chain` field of `VMStoreContext`.- fn vmstore_context_store_data(&self) -> u8 {- self.vmstore_context_stack_chain() + self.size_of_vmstack_chain()- }-- /// Return the offset of the `async_guard_range` field of `VMStoreContext`.- fn vmstore_context_async_guard_range(&self) -> u8 {- self.vmstore_context_store_data() + self.size()- }-- /// Return the offset of the `component_context[i]` field of- /// `VMStoreContext`.- fn vmstore_context_component_context_slot(&self, i: u8) -> u8 {- assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);- let base = self.vmstore_context_async_guard_range() + 2 * self.size();- let slot_size = 4;- base + i * slot_size- }-- /// Return the offset of the `current_thread` field of `VMStoreContext`.- fn vmstore_context_current_thread(&self) -> u8 {- self.vmstore_context_component_context_slot(0) + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4- }-- // Offsets within `VMDeferredThread`-- /// Offset of `VMDeferredThread::parent`.- fn vmdeferred_thread_parent(&self) -> u8 {- 0- }-- /// Offset of `VMDeferredThread::caller_instance`.- fn vmdeferred_thread_caller_instance(&self) -> u8 {- self.size()- }-- /// Offset of `VMDeferredThread::callee_async`.- fn vmdeferred_thread_callee_async(&self) -> u8 {- self.vmdeferred_thread_caller_instance() + 4- }-- /// Offset of `VMDeferredThread::callee_instance`.- fn vmdeferred_thread_callee_instance(&self) -> u8 {- self.vmdeferred_thread_callee_async() + 4- }-- /// Offset of `VMDeferredThread::saved_context[i]`.- fn vmdeferred_thread_saved_context(&self, i: u8) -> u8 {- assert!(usize::from(i) < NUM_COMPONENT_CONTEXT_SLOTS);- self.vmdeferred_thread_callee_instance() + 4 + i * 4- }-- /// Return the size of `VMDeferredThread`, rounded up to pointer alignment.- fn size_of_vmdeferred_thread(&self) -> u8 {- let unaligned =- self.vmdeferred_thread_callee_instance() + 4 + (NUM_COMPONENT_CONTEXT_SLOTS as u8) * 4;- align(u32::from(unaligned), u32::from(self.size())) as u8- }-/// Return the size of `*mut VMMemoryDefinition`.#[inline]fn size_of_vmmemory_pointer(&self) -> u8 {diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rsindex d2a1922b4b..867c1b8060 100644--- a/crates/environ/src/vmtypes.rs+++ b/crates/environ/src/vmtypes.rs@@ -27,9 +27,18 @@/// * A `#[snake_name = <ident>]` attribute giving the type's name in/// `snake_case`, used to generate accessor method names.///-/// Each field may be preceded by doc-comment attributes and, optionally, the-/// marker attributes `#[readonly]` and/or `#[can_move]` (in that order) which-/// describe how Cranelift may treat loads and stores of that field.+/// Each field may be preceded by doc-comment attributes and, optionally, these+/// marker attributes, in this order:+///+/// * `#[aggregate]`: this field is a composite (a nested struct or array)+/// rather than a single scalar. Compiled Wasm code accesses such a field's+/// interior piecewise, so there is no one Cranelift type for the field as a+/// whole and no alias-region accessor is generated for it. The field's offset+/// is still generated, since that is what interior accesses are computed+/// relative to.+///+/// * `#[readonly]` and/or `#[can_move]`: describe how Cranelift may treat loads+/// and stores of that field.#[macro_export]macro_rules! for_each_vm_type {($mac:ident) => {@@ -228,6 +237,263 @@ macro_rules! for_each_vm_type {/// The index of the tag in the containing `vmctx`.pub index: DefinedTagIndex,}++ /// Structure that holds all mutable context that is shared across all instances+ /// in a store, for example data related to fuel or epochs.+ ///+ /// `VMStoreContext`s are one-to-one with `wasmtime::Store`s, the same way that+ /// `VMContext`s are one-to-one with `wasmtime::Instance`s. And the same way+ /// that multiple `wasmtime::Instance`s may be associated with the same+ /// `wasmtime::Store`, multiple `VMContext`s hold a pointer to the same+ /// `VMStoreContext` when they are associated with the same `wasmtime::Store`.+ #[derive(Debug)]+ // NB: `align(8)` is forced rather than inferred because the i386+ // System V ABI aligns 64-bit integers to 4 bytes, and `VMOffsets`+ // can't tell that target apart from the ones that align them to 8,+ // since it only knows the target's pointer width.+ #[repr(C, align(8))]+ #[snake_name = vm_store_context]+ pub struct VMStoreContext {+ // NB: 64-bit integer fields are located first with pointer-sized fields+ // trailing afterwards. That makes the offsets in this structure easier to+ // calculate on 32-bit platforms as we don't have to worry about the+ // alignment of 64-bit integers.+ //+ /// Indicator of how much fuel has been consumed and is remaining to+ /// WebAssembly.+ ///+ /// This field is typically negative and increments towards positive. Upon+ /// turning positive a wasm trap will be generated. This field is only+ /// modified if wasm is configured to consume fuel.+ pub fuel_consumed: UnsafeCell<i64>,++ /// Deadline epoch for interruption: if epoch-based interruption+ /// is enabled and the global (per engine) epoch counter is+ /// observed to reach or exceed this value, the guest code will+ /// yield if running asynchronously.+ pub epoch_deadline: UnsafeCell<u64>,++ /// The "store version".+ ///+ /// This is used to test whether stack-frame handles referring to+ /// suspended stack frames remain valid.+ ///+ /// The invariant that this upward-counting number must satisfy+ /// is: the number must be incremented whenever execution starts+ /// or resumes in the `Store` or when any stack is+ /// dropped/freed. That way, if we take a reference to some+ /// suspended stack frame and track the "version" at the time we+ /// took that reference, if the version still matches, we can be+ /// sure that nothing could have unwound the referenced Wasm+ /// frame.+ ///+ /// This version number is incremented in exactly one place: the+ /// Wasm-to-host trampolines, after return from host code. Note+ /// that this captures both the normal "return into Wasm" case+ /// (where Wasm frames can subsequently return normally and thus+ /// invalidate frames), and the "trap/exception unwinds Wasm+ /// frames" case, which is done internally via the `raise` libcall+ /// invoked after the main hostcall returns an error, and after we+ /// increment this version number.+ ///+ /// Note that this also handles the fiber/future-drop case because+ /// because we *always* return into the trampoline to clean up;+ /// that trampoline immediately raises an error and uses the+ /// longjmp-like unwind within Cranelift frames to skip over all+ /// the guest Wasm frames, but not before it increments the+ /// store's execution version number.+ ///+ /// This field is in use only if guest debugging is enabled.+ pub execution_version: u64,++ /// Current stack limit of the wasm module.+ ///+ /// For more information see `crates/cranelift/src/lib.rs`.+ pub stack_limit: UnsafeCell<usize>,++ /// The `VMMemoryDefinition` for this store's GC heap.+ #[aggregate]+ pub gc_heap: UnsafeCell<VMMemoryDefinition>,++ /// The value of the frame pointer register in the trampoline used+ /// to call from Wasm to the host.+ ///+ /// Maintained by our Wasm-to-host trampoline, and cleared just+ /// before calling into Wasm in `catch_traps`.+ ///+ /// This member is `0` when Wasm is actively running and has not called out+ /// to the host.+ ///+ /// Used to find the start of a contiguous sequence of Wasm frames+ /// when walking the stack. Note that we record the FP of the+ /// *trampoline*'s frame, not the last Wasm frame, because we need+ /// to know the SP (bottom of frame) of the last Wasm frame as+ /// well in case we need to resume to an exception handler in that+ /// frame. The FP of the last Wasm frame can be recovered by+ /// loading the saved FP value at this FP address.+ pub last_wasm_exit_trampoline_fp: UnsafeCell<usize>,++ /// The last Wasm program counter before we called from Wasm to the host.+ ///+ /// Maintained by our Wasm-to-host trampoline, and cleared just before+ /// calling into Wasm in `catch_traps`.+ ///+ /// This member is `0` when Wasm is actively running and has not called out+ /// to the host.+ ///+ /// Used when walking a contiguous sequence of Wasm frames.+ pub last_wasm_exit_pc: UnsafeCell<usize>,++ /// The last host stack pointer before we called into Wasm from the host.+ ///+ /// Maintained by our host-to-Wasm trampoline. This member is `0` when Wasm+ /// is not running, and it's set to nonzero once a host-to-wasm trampoline+ /// is executed.+ ///+ /// When a host function is wrapped into a `wasmtime::Func`, and is then+ /// called from the host, then this member is not changed meaning that the+ /// previous activation in pointed to by `last_wasm_exit_trampoline_fp` is+ /// still the last wasm set of frames on the stack.+ ///+ /// This field is saved/restored during fiber suspension/resumption+ /// resumption as part of `CallThreadState::swap`.+ ///+ /// This field is used to find the end of a contiguous sequence of Wasm+ /// frames when walking the stack. Additionally it's used when a trap is+ /// raised as part of the set of parameters used to resume in the entry+ /// trampoline's "catch" block.+ pub last_wasm_entry_sp: UnsafeCell<usize>,++ /// Same as `last_wasm_entry_sp`, but for the `fp` of the trampoline.+ pub last_wasm_entry_fp: UnsafeCell<usize>,++ /// The last trap handler from a host-to-wasm entry trampoline on the stack.+ ///+ /// This field is configured when the host calls into wasm by the trampoline+ /// itself. It stores the `pc` of an exception handler suitable to handle+ /// all traps (or uncaught exceptions).+ pub last_wasm_entry_trap_handler: UnsafeCell<usize>,++ /// Stack information used by stack switching instructions. See documentation+ /// on `VMStackChain` for details.+ #[aggregate]+ pub stack_chain: UnsafeCell<VMStackChain>,++ /// A pointer to the embedder's `T` inside a `Store<T>`, for use with the+ /// `store-data-address` unsafe intrinsic.+ pub store_data: VmPtr<()>,++ /// The range, in addresses, of the guard page that is currently in use.+ ///+ /// This field is used when signal handlers are run to determine whether a+ /// faulting address lies within the guard page of an async stack for+ /// example. If this happens then the signal handler aborts with a stack+ /// overflow message similar to what would happen had the stack overflow+ /// happened on the main thread. This field is, by default a null..null+ /// range indicating that no async guard is in use (aka no fiber). In such a+ /// situation while this field is read it'll never classify a fault as an+ /// guard page fault.+ #[aggregate]+ pub async_guard_range: Range<*mut u8>,++ /// The `context.{get,set}` values for the current thread in the component+ /// model. This is only used for `component-model-async` and slot[1] is only+ /// used for `component-model-threading`. Despite the conditional use nature+ /// this is unconditionally present as it avoids the need to make logic in+ /// `VMOffsets` conditional.+ ///+ /// This is saved/restored when threads are swapped in the component model.+ ///+ /// NB: `UnsafeCell` because JIT code writes to the slots.+ #[aggregate]+ pub component_context: UnsafeCell<[u32; NUM_COMPONENT_CONTEXT_SLOTS]>,++ /// JIT-visible current thread for the component model's sync-to-sync+ /// adapter fast path.+ ///+ /// Like `component_context`, this is unconditionally present to keep+ /// `VMOffsets` logic unconditional even though it is only used when+ /// `component-model-async` is enabled.+ ///+ /// NB: `UnsafeCell` because JIT code writes to this field.+ pub current_thread: UnsafeCell<VMLazyThread>,+ }++ /// JIT-visible representation of the store's current thread for the component+ /// model, encoded as a single pointer-sized integer so that generated JIT code+ /// can load, store, and compare it with a handful of instructions.+ ///+ /// This is the inline fast-path counterpart to the host-side `CurrentThread`: a+ /// fused sync-to-sync adapter records a lazy deferred thread here (a pointer to+ /// a `VMDeferredThread` on its own stack frame) instead of eagerly allocating a+ /// `GuestTask`/`GuestThread` in the host. Host code promotes the deferred+ /// thread into a real one only when it actually needs it; see+ /// `StoreOpaque::force_current_thread`.+ ///+ /// This type is a bitpacked equivalent of the following logical `enum`:+ ///+ /// ```ignore+ /// enum VMLazyThread {+ /// /// No thread.+ /// None,+ ///+ /// /// The lazy thread was promoted and materialized; get it from+ /// /// `ConcurrentState::current_thread`.+ /// Forced,+ ///+ /// /// The lazy thread has not been materialized, here is a pointer to the+ /// /// stack-allocated data needed to do force that promotion.+ /// Deferred(*mut VMDeferredThread),+ /// }+ /// ```+ ///+ /// Bitpacking details:+ ///+ /// * `None`: `0`+ ///+ /// * `Forced`: A non-zero value with its low-bit set.+ ///+ /// * `Deferred`: A non-zero value with its low-bit clear.+ #[derive(Debug, Copy, Clone, PartialEq, Eq)]+ #[repr(transparent)]+ #[snake_name = vm_lazy_thread]+ pub struct VMLazyThread {+ /// The bitpacked thread representation described above.+ ///+ /// Private: use the `VMLazyThread::{none,forced,deferred}` constructors+ /// and the `is_*`/`as_deferred` accessors instead of touching this+ /// directly.+ thread: Option<VmPtr<VMDeferredThread>>,+ }++ /// A deferred component-model thread.+ ///+ /// This is an on-stack record pushed by a fused sync-to-sync adapter's fast+ /// path to defer the work that the `enter_sync_call` libcall would otherwise do+ /// eagerly.+ ///+ /// The adapter allocates one of these in its own stack frame, links the+ /// previous current-thread value to it via `parent`, and finally points+ /// `VMStoreContext::current_thread` at it. When host code actually needs the+ /// real thread, it walks the `parent` chain to materialize thread state (see+ /// `StoreOpaque::force_current_thread`).+ #[derive(Debug)]+ #[repr(C)]+ #[snake_name = vm_deferred_thread]+ pub struct VMDeferredThread {+ /// The previous value of `VMStoreContext::current_thread`.+ pub parent: VMLazyThread,+ /// The caller component instance (a deferred `enter_sync_call` argument).+ pub caller_instance: u32,+ /// Whether the callee is async-lifted (a deferred `enter_sync_call` arg).+ pub callee_async: u32,+ /// The callee component instance (a deferred `enter_sync_call` argument).+ pub callee_instance: u32,+ /// The caller thread's `context.{get,set}` slots, saved on entry and+ /// restored on the fast-path exit (or recovered while forcing).+ #[aggregate]+ pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],+ }}};}diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rsindex df665b7472..9324674f1e 100644--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs@@ -204,6 +204,7 @@ macro_rules! define_vm_types {$svis:vis struct $Name:ident {$($(#[doc = $fdoc:literal])*+ $(#[aggregate])?$(#[readonly])?$(#[can_move])?$fvis:vis $fname:ident : $fty:tt $(< $fgen:ty >)? ,@@ -719,178 +720,6 @@ const _: () = {)};-/// Structure that holds all mutable context that is shared across all instances-/// in a store, for example data related to fuel or epochs.-///-/// `VMStoreContext`s are one-to-one with `wasmtime::Store`s, the same way that-/// `VMContext`s are one-to-one with `wasmtime::Instance`s. And the same way-/// that multiple `wasmtime::Instance`s may be associated with the same-/// `wasmtime::Store`, multiple `VMContext`s hold a pointer to the same-/// `VMStoreContext` when they are associated with the same `wasmtime::Store`.-#[derive(Debug)]-#[repr(C)]-pub struct VMStoreContext {- // NB: 64-bit integer fields are located first with pointer-sized fields- // trailing afterwards. That makes the offsets in this structure easier to- // calculate on 32-bit platforms as we don't have to worry about the- // alignment of 64-bit integers.- //- /// Indicator of how much fuel has been consumed and is remaining to- /// WebAssembly.- ///- /// This field is typically negative and increments towards positive. Upon- /// turning positive a wasm trap will be generated. This field is only- /// modified if wasm is configured to consume fuel.- pub fuel_consumed: UnsafeCell<i64>,-- /// Deadline epoch for interruption: if epoch-based interruption- /// is enabled and the global (per engine) epoch counter is- /// observed to reach or exceed this value, the guest code will- /// yield if running asynchronously.- pub epoch_deadline: UnsafeCell<u64>,-- /// The "store version".- ///- /// This is used to test whether stack-frame handles referring to- /// suspended stack frames remain valid.- ///- /// The invariant that this upward-counting number must satisfy- /// is: the number must be incremented whenever execution starts- /// or resumes in the `Store` or when any stack is- /// dropped/freed. That way, if we take a reference to some- /// suspended stack frame and track the "version" at the time we- /// took that reference, if the version still matches, we can be- /// sure that nothing could have unwound the referenced Wasm- /// frame.- ///- /// This version number is incremented in exactly one place: the- /// Wasm-to-host trampolines, after return from host code. Note- /// that this captures both the normal "return into Wasm" case- /// (where Wasm frames can subsequently return normally and thus- /// invalidate frames), and the "trap/exception unwinds Wasm- /// frames" case, which is done internally via the `raise` libcall- /// invoked after the main hostcall returns an error, and after we- /// increment this version number.- ///- /// Note that this also handles the fiber/future-drop case because- /// because we *always* return into the trampoline to clean up;- /// that trampoline immediately raises an error and uses the- /// longjmp-like unwind within Cranelift frames to skip over all- /// the guest Wasm frames, but not before it increments the- /// store's execution version number.- ///- /// This field is in use only if guest debugging is enabled.- pub execution_version: u64,-- /// Current stack limit of the wasm module.- ///- /// For more information see `crates/cranelift/src/lib.rs`.- pub stack_limit: UnsafeCell<usize>,-- /// The `VMMemoryDefinition` for this store's GC heap.- pub gc_heap: UnsafeCell<VMMemoryDefinition>,-- /// The value of the frame pointer register in the trampoline used- /// to call from Wasm to the host.- ///- /// Maintained by our Wasm-to-host trampoline, and cleared just- /// before calling into Wasm in `catch_traps`.- ///- /// This member is `0` when Wasm is actively running and has not called out- /// to the host.- ///- /// Used to find the start of a contiguous sequence of Wasm frames- /// when walking the stack. Note that we record the FP of the- /// *trampoline*'s frame, not the last Wasm frame, because we need- /// to know the SP (bottom of frame) of the last Wasm frame as- /// well in case we need to resume to an exception handler in that- /// frame. The FP of the last Wasm frame can be recovered by- /// loading the saved FP value at this FP address.- pub last_wasm_exit_trampoline_fp: UnsafeCell<usize>,-- /// The last Wasm program counter before we called from Wasm to the host.- ///- /// Maintained by our Wasm-to-host trampoline, and cleared just before- /// calling into Wasm in `catch_traps`.- ///- /// This member is `0` when Wasm is actively running and has not called out- /// to the host.- ///- /// Used when walking a contiguous sequence of Wasm frames.- pub last_wasm_exit_pc: UnsafeCell<usize>,-- /// The last host stack pointer before we called into Wasm from the host.- ///- /// Maintained by our host-to-Wasm trampoline. This member is `0` when Wasm- /// is not running, and it's set to nonzero once a host-to-wasm trampoline- /// is executed.- ///- /// When a host function is wrapped into a `wasmtime::Func`, and is then- /// called from the host, then this member is not changed meaning that the- /// previous activation in pointed to by `last_wasm_exit_trampoline_fp` is- /// still the last wasm set of frames on the stack.- ///- /// This field is saved/restored during fiber suspension/resumption- /// resumption as part of `CallThreadState::swap`.- ///- /// This field is used to find the end of a contiguous sequence of Wasm- /// frames when walking the stack. Additionally it's used when a trap is- /// raised as part of the set of parameters used to resume in the entry- /// trampoline's "catch" block.- pub last_wasm_entry_sp: UnsafeCell<usize>,-- /// Same as `last_wasm_entry_sp`, but for the `fp` of the trampoline.- pub last_wasm_entry_fp: UnsafeCell<usize>,-- /// The last trap handler from a host-to-wasm entry trampoline on the stack.- ///- /// This field is configured when the host calls into wasm by the trampoline- /// itself. It stores the `pc` of an exception handler suitable to handle- /// all traps (or uncaught exceptions).- pub last_wasm_entry_trap_handler: UnsafeCell<usize>,-- /// Stack information used by stack switching instructions. See documentation- /// on `VMStackChain` for details.- pub stack_chain: UnsafeCell<VMStackChain>,-- /// A pointer to the embedder's `T` inside a `Store<T>`, for use with the- /// `store-data-address` unsafe intrinsic.- pub store_data: VmPtr<()>,-- /// The range, in addresses, of the guard page that is currently in use.- ///- /// This field is used when signal handlers are run to determine whether a- /// faulting address lies within the guard page of an async stack for- /// example. If this happens then the signal handler aborts with a stack- /// overflow message similar to what would happen had the stack overflow- /// happened on the main thread. This field is, by default a null..null- /// range indicating that no async guard is in use (aka no fiber). In such a- /// situation while this field is read it'll never classify a fault as an- /// guard page fault.- pub async_guard_range: Range<*mut u8>,-- /// The `context.{get,set}` values for the current thread in the component- /// model. This is only used for `component-model-async` and slot[1] is only- /// used for `component-model-threading`. Despite the conditional use nature- /// this is unconditionally present as it avoids the need to make logic in- /// `VMOffsets` conditional.- ///- /// This is saved/restored when threads are swapped in the component model.- ///- /// NB: `UnsafeCell` because JIT code writes to the slots.- pub component_context: UnsafeCell<[u32; NUM_COMPONENT_CONTEXT_SLOTS]>,-- /// JIT-visible current thread for the component model's sync-to-sync- /// adapter fast path.- ///- /// Like `component_context`, this is unconditionally present to keep- /// `VMOffsets` logic unconditional even though it is only used when- /// `component-model-async` is enabled.- ///- /// NB: `UnsafeCell` because JIT code writes to this field.- pub current_thread: UnsafeCell<VMLazyThread>,-}-impl VMStoreContext {/// From the current saved trampoline FP, get the FP of the last/// Wasm frame. If the current saved trampoline FP is null, return@@ -995,79 +824,33 @@ mod test_vmstore_context {use core::mem::offset_of;use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};+ /// Check the `VMStoreContext` offsets that `for_each_vm_type!` does *not*+ /// generate: the offsets reaching into the inlined `gc_heap`, and the+ /// indexed `component_context` slot accessor.+ ///+ /// Every plain field offset, plus the size and alignment of the type, is+ /// already checked by the generated `test_vm_type_layouts::vm_store_context`.#[test]- fn field_offsets() {+ fn derived_field_offsets() {let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);- assert_eq!(- offset_of!(VMStoreContext, stack_limit),- usize::from(offsets.ptr.vmstore_context_stack_limit())- );- assert_eq!(- offset_of!(VMStoreContext, fuel_consumed),- usize::from(offsets.ptr.vmstore_context_fuel_consumed())- );- assert_eq!(- offset_of!(VMStoreContext, epoch_deadline),- usize::from(offsets.ptr.vmstore_context_epoch_deadline())- );- assert_eq!(- offset_of!(VMStoreContext, execution_version),- usize::from(offsets.ptr.vmstore_context_execution_version())- );- assert_eq!(- offset_of!(VMStoreContext, gc_heap),- usize::from(offsets.ptr.vmstore_context_gc_heap())- );assert_eq!(offset_of!(VMStoreContext, gc_heap) + offset_of!(VMMemoryDefinition, base),- usize::from(offsets.ptr.vmstore_context_gc_heap_base())+ usize::from(offsets.ptr.vm_store_context().gc_heap_base()));assert_eq!(offset_of!(VMStoreContext, gc_heap) + offset_of!(VMMemoryDefinition, current_length),- usize::from(offsets.ptr.vmstore_context_gc_heap_current_length())- );- assert_eq!(- offset_of!(VMStoreContext, last_wasm_exit_trampoline_fp),- usize::from(offsets.ptr.vmstore_context_last_wasm_exit_trampoline_fp())- );- assert_eq!(- offset_of!(VMStoreContext, last_wasm_exit_pc),- usize::from(offsets.ptr.vmstore_context_last_wasm_exit_pc())- );- assert_eq!(- offset_of!(VMStoreContext, last_wasm_entry_fp),- usize::from(offsets.ptr.vmstore_context_last_wasm_entry_fp())- );- assert_eq!(- offset_of!(VMStoreContext, last_wasm_entry_sp),- usize::from(offsets.ptr.vmstore_context_last_wasm_entry_sp())- );- assert_eq!(- offset_of!(VMStoreContext, last_wasm_entry_trap_handler),- usize::from(offsets.ptr.vmstore_context_last_wasm_entry_trap_handler())- );- assert_eq!(- offset_of!(VMStoreContext, stack_chain),- usize::from(offsets.ptr.vmstore_context_stack_chain())- );- assert_eq!(- offset_of!(VMStoreContext, store_data),- usize::from(offsets.ptr.vmstore_context_store_data())+ usize::from(offsets.ptr.vm_store_context().gc_heap_current_length()));assert_eq!(offset_of!(VMStoreContext, component_context),- usize::from(offsets.ptr.vmstore_context_component_context_slot(0))- );- assert_eq!(- offset_of!(VMStoreContext, current_thread),- usize::from(offsets.ptr.vmstore_context_current_thread())+ usize::from(offsets.ptr.vm_store_context().component_context_slot(0)));// Make sure that the calculation for the size of a slot is also// accurate.- let slot_width = offsets.ptr.vmstore_context_component_context_slot(1)- - offsets.ptr.vmstore_context_component_context_slot(0);+ let slot_width = offsets.ptr.vm_store_context().component_context_slot(1)+ - offsets.ptr.vm_store_context().component_context_slot(0);let mut default = VMStoreContext::default();assert_eq!(size_of_val(&default.component_context.get_mut()[0]),@@ -1076,45 +859,6 @@ mod test_vmstore_context {}}-/// JIT-visible representation of the store's current thread for the component-/// model, encoded as a single pointer-sized integer so that generated JIT code-/// can load, store, and compare it with a handful of instructions.-///-/// This is the inline fast-path counterpart to the host-side `CurrentThread`: a-/// fused sync-to-sync adapter records a lazy deferred thread here (a pointer to-/// a `VMDeferredThread` on its own stack frame) instead of eagerly allocating a-/// `GuestTask`/`GuestThread` in the host. Host code promotes the deferred-/// thread into a real one only when it actually needs it; see-/// `StoreOpaque::force_current_thread`.-///-/// This type is a bitpacked equivalent of the following logical `enum`:-///-/// ```ignore-/// enum VMLazyThread {-/// /// No thread.-/// None,-///-/// /// The lazy thread was promoted and materialized; get it from-/// /// `ConcurrentState::current_thread`.-/// Forced,-///-/// /// The lazy thread has not been materialized, here is a pointer to the-/// /// stack-allocated data needed to do force that promotion.-/// Deferred(*mut VMDeferredThread),-/// }-/// ```-//-// Bitpacking details:-//-// * `None`: `0`-//-// * `Forced`: A non-zero value with its low-bit set.-//-// * `Deferred`: A non-zero value with its low-bit clear.-#[derive(Debug, Copy, Clone, PartialEq, Eq)]-#[repr(transparent)]-pub struct VMLazyThread(Option<VmPtr<VMDeferredThread>>);-impl VMLazyThread {const _ASSERT_SIZE: () = assert!(core::mem::size_of::<VMLazyThread>() == core::mem::size_of::<*mut VMDeferredThread>()@@ -1127,40 +871,44 @@ impl VMLazyThread {/// There is no current thread.pub const fn none() -> Self {- Self(None)+ Self { thread: None }}/// A lazy thread that has already been promoted.pub const fn forced() -> Self {- Self(Some(Self::FORCED))+ Self {+ thread: Some(Self::FORCED),+ }}/// A deferred thread referencing the given on-stack [`VMDeferredThread`].pub fn deferred(ptr: NonNull<VMDeferredThread>) -> Self {debug_assert_eq!(ptr.addr().get() & Self::FORCED.addr().get(), 0);- Self(Some(ptr.into()))+ Self {+ thread: Some(ptr.into()),+ }}/// Returns `true` if there is no current thread.pub fn is_none(self) -> bool {- self.0.is_none()+ self.thread.is_none()}/// Returns `true` if a deferred thread has been forced/promoted.pub fn is_forced(self) -> bool {- self.0.is_some_and(|p| p == Self::FORCED)+ self.thread.is_some_and(|p| p == Self::FORCED)}/// Returns `true` if this is a deferred thread (i.e. neither `None` nor/// forced).pub fn is_deferred(self) -> bool {- self.0.is_some_and(|p| p != Self::FORCED)+ self.thread.is_some_and(|p| p != Self::FORCED)}/// Returns the deferred [`VMDeferredThread`] pointer if this is a deferred/// thread.pub fn as_deferred(self) -> Option<VmPtr<VMDeferredThread>> {- self.0+ self.thread.and_then(|p| if p == Self::FORCED { None } else { Some(p) })}}@@ -1172,73 +920,32 @@ mod test_vmlazy_thread {#[test]fn vmlazy_thread_forced() {assert_eq!(- VMLazyThread::forced().0.unwrap().addr().get(),+ VMLazyThread::forced().thread.unwrap().addr().get(),usize::try_from(wasmtime_environ::VM_LAZY_THREAD_FORCED).unwrap());}}-/// A deferred component-model thread.-///-/// This is an on-stack record pushed by a fused sync-to-sync adapter's fast-/// path to defer the work that the `enter_sync_call` libcall would otherwise do-/// eagerly.-///-/// The adapter allocates one of these in its own stack frame, links the-/// previous current-thread value to it via `parent`, and finally points-/// `VMStoreContext::current_thread` at it. When host code actually needs the-/// real thread, it walks the `parent` chain to materialize thread state (see-/// `StoreOpaque::force_current_thread`).-#[derive(Debug)]-#[repr(C)]-pub struct VMDeferredThread {- /// The previous value of `VMStoreContext::current_thread`.- pub parent: VMLazyThread,- /// The caller component instance (a deferred `enter_sync_call` argument).- pub caller_instance: u32,- /// Whether the callee is async-lifted (a deferred `enter_sync_call` arg).- pub callee_async: u32,- /// The callee component instance (a deferred `enter_sync_call` argument).- pub callee_instance: u32,- /// The caller thread's `context.{get,set}` slots, saved on entry and- /// restored on the fast-path exit (or recovered while forcing).- pub saved_context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],-}-#[cfg(test)]mod test_vmdeferred_thread {use super::*;use core::mem::offset_of;use wasmtime_environ::{HostPtr, Module, PtrSize, StaticModuleIndex, VMOffsets};+ /// Check the indexed `saved_context` slot accessor, which+ /// `for_each_vm_type!` does not generate.+ ///+ /// Every plain field offset, plus the size and alignment of the type, is+ /// already checked by the generated+ /// `test_vm_type_layouts::vm_deferred_thread`.#[test]- fn deferred_thread_field_offsets() {+ fn deferred_thread_derived_field_offsets() {let module = Module::new(StaticModuleIndex::from_u32(0));let offsets = VMOffsets::new(HostPtr, &module);let ptr = offsets.ptr;- assert_eq!(- offset_of!(VMDeferredThread, parent),- usize::from(ptr.vmdeferred_thread_parent())- );- assert_eq!(- offset_of!(VMDeferredThread, caller_instance),- usize::from(ptr.vmdeferred_thread_caller_instance())- );- assert_eq!(- offset_of!(VMDeferredThread, callee_async),- usize::from(ptr.vmdeferred_thread_callee_async())- );- assert_eq!(- offset_of!(VMDeferredThread, callee_instance),- usize::from(ptr.vmdeferred_thread_callee_instance())- );assert_eq!(offset_of!(VMDeferredThread, saved_context),- usize::from(ptr.vmdeferred_thread_saved_context(0))- );- assert_eq!(- size_of::<VMDeferredThread>(),- usize::from(ptr.size_of_vmdeferred_thread())+ usize::from(ptr.vm_deferred_thread().saved_context_slot(0)));}}diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex 95df897ec1..34d7d1e90e 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -2099,7 +2099,7 @@ where/// `VMStoreContext`.fn emit_load_fuel_consumed(&mut self, fuel_reg: Reg) -> Result<()> {let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();- let fuel_offset = self.env.vmoffsets.ptr.vmstore_context_fuel_consumed();+ let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed();self.masm.load_ptr(self.masm.address_at_vmctx(u32::from(store_context_offset))?,@@ -2178,7 +2178,7 @@ where) -> Result<()> {let epoch_ptr_offset = self.env.vmoffsets.ptr.vmctx_epoch_ptr();let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();- let epoch_deadline_offset = self.env.vmoffsets.ptr.vmstore_context_epoch_deadline();+ let epoch_deadline_offset = self.env.vmoffsets.ptr.vm_store_context().epoch_deadline();// Load the current epoch value into `epoch_counter_var`.self.masm.load_ptr(@@ -2219,7 +2219,7 @@ where}let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context();- let fuel_offset = self.env.vmoffsets.ptr.vmstore_context_fuel_consumed();+ let fuel_offset = self.env.vmoffsets.ptr.vm_store_context().fuel_consumed();let limits_reg = self.context.any_gpr(self.masm)?;// Load `VMStoreContext` into the `limits_reg` reg.diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex da439cd970..c5c882216c 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -198,7 +198,7 @@ impl Masm for MacroAssembler {masm.load_ptr(Address::offset(scratch_stk_limit.inner(),- ptr_size_u8.vmstore_context_stack_limit().into(),+ ptr_size_u8.vm_store_context().stack_limit().into(),),scratch_stk_limit.writable(),)?;diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rsindex 2c2fd67d7c..c29ddb9541 100644--- a/winch/codegen/src/isa/x64/masm.rs+++ b/winch/codegen/src/isa/x64/masm.rs@@ -136,7 +136,7 @@ impl Masm for MacroAssembler {masm.load_ptr(Address::offset(scratch.inner(),- ptr_size.vmstore_context_stack_limit().into(),+ ptr_size.vm_store_context().stack_limit().into(),),scratch.writable(),)?;