A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
[stack-switching] Port raw pointer uses to `VmPtr` (#14369)
Daniel Hillerströme5ed75cf93f6
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex 4c8dca3135..511436017b 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -48,7 +48,6 @@ macro_rules! define_vm_type_offsets {(@size ($p:expr) Option < VmPtr < $g:ty >>) => { u32::from($p) };(@size ($p:expr) AtomicUsize) => { u32::from($p) };(@size ($p:expr) usize) => { u32::from($p) };- (@size ($p:expr) * mut $g:ty) => { u32::from($p) };(@size ($p:expr) i64) => { 8u32 };(@size ($p:expr) u64) => { 8u32 };(@size ($p:expr) u32) => { 4u32 };@@ -100,7 +99,6 @@ macro_rules! define_vm_type_offsets {(@align ($p:expr) Option < VmPtr < $g:ty >>) => { u32::from($p) };(@align ($p:expr) AtomicUsize) => { u32::from($p) };(@align ($p:expr) usize) => { u32::from($p) };- (@align ($p:expr) * mut $g:ty) => { u32::from($p) };(@align ($p:expr) i64) => { 8u32 };(@align ($p:expr) u64) => { 8u32 };(@align ($p:expr) u32) => { 4u32 };diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rsindex 1321291e72..3a8e22edb2 100644--- a/crates/environ/src/vmtypes.rs+++ b/crates/environ/src/vmtypes.rs@@ -599,7 +599,7 @@ macro_rules! for_each_vm_type {/// The buffer itself, which lives on the continuation's stack/// rather than in this object.- pub data: *mut u8,+ pub data: Option<VmPtr<u8>>,}/// Payload values exchanged with a continuation and the metadata@@ -652,9 +652,9 @@ macro_rules! for_each_vm_type {#[aggregate]pub parent_chain: VMStackChain,- /// The end of this continuation's parent chain, used only while- /// it is `Suspended` or `Fresh`.- pub last_ancestor: *mut VMContRef,+ /// The end of this continuation's parent chain while it is+ /// `Suspended` or `Fresh`, and `None` while it is running.+ pub last_ancestor: Option<VmPtr<VMContRef>>,/// Revision counter.pub revision: usize,diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rsindex c8ab027e51..71b123368e 100644--- a/crates/wasmtime/src/runtime/func.rs+++ b/crates/wasmtime/src/runtime/func.rs@@ -3,7 +3,7 @@ use crate::prelude::*;use crate::runtime::vm::{self, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext,VMCommonStackInformation, VMContext, VMFuncRef, VMFunctionImport, VMOpaqueContext,- VMStoreContext,+ VMStoreContext, VmPtr,};use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreId, StoreOpaque};use crate::type_registry::RegisteredType;@@ -1586,6 +1586,10 @@ impl EntryStoreContext {unsafe {let vm_store_context = store.0.vm_store_context();+ let initial_stack_information = VmPtr::from(+ NonNull::new(initial_stack_information)+ .expect("expected non-null stack information"),+ );let new_stack_chain = VMStackChain::InitialStack(initial_stack_information);let stack_chain =mem::replace(&mut *vm_store_context.stack_chain.get(), new_stack_chain);diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rsindex 09a63ba2cf..72492ccbb7 100644--- a/crates/wasmtime/src/runtime/store/gc.rs+++ b/crates/wasmtime/src/runtime/store/gc.rs@@ -846,11 +846,15 @@ impl StoreOpaque {return;};let gc_ref_data = gc_ref_data.as_ptr();+ let data = payloads+ .data+ .expect("payload GC metadata requires an allocated payload buffer")+ .as_ptr();for index in 0..usize::try_from(payloads.length).unwrap() {let marker = unsafe { gc_ref_data.add(index).read() };if marker == wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF {- let slot = unsafe { payloads.data.cast::<ValRaw>().add(index).cast::<u32>() };+ let slot = unsafe { data.cast::<ValRaw>().add(index).cast::<u32>() };unsafe {StoreOpaque::trace_wasm_stack_slot(gc_roots_list, slot);}diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rsindex 06f61340d3..f1984a63a2 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs@@ -3,7 +3,9 @@mod stack;-use crate::vm::{VMCommonStackInformation, VMContRef, VMHostArray, VMPayloads, VMStackLimits};+use crate::vm::{+ VMCommonStackInformation, VMContRef, VMHostArray, VMPayloads, VMStackLimits, VmPtr,+};use core::{marker::PhantomPinned, ptr::NonNull};pub use stack::*;@@ -106,7 +108,7 @@ impl VMHostArray {Self {length: 0,capacity: 0,- data: core::ptr::null_mut(),+ data: None,}}@@ -154,7 +156,7 @@ impl VMContRef {first_switch_handler_index: 0,};let parent_chain = VMStackChain::Absent;- let last_ancestor = core::ptr::null_mut();+ let last_ancestor = None;let stack = VMContinuationStack::unallocated();let args = VMPayloads::empty();let values = VMPayloads::empty();@@ -215,7 +217,7 @@ pub fn cont_new(contref.parent_chain = VMStackChain::Absent;// The continuation is fresh, which is a special case of being suspended.// Thus we need to set the correct end of the continuation chain: itself.- contref.last_ancestor = contref;+ contref.last_ancestor = Some(VmPtr::from(&*contref));// The initialization function will allocate the actual args/return value buffer and// update this object (if needed).@@ -336,10 +338,10 @@ pub enum VMStackChain {/// does not have a parent. The `CommonStackInformation` that this/// variant points to is stored in the stack frame of/// `invoke_wasm_and_catch_traps`.- InitialStack(*mut VMCommonStackInformation) =+ InitialStack(VmPtr<VMCommonStackInformation>) =wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT,/// Represents a continuation's stack.- Continuation(*mut VMContRef) = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT,+ Continuation(VmPtr<VMContRef>) = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT,}impl VMStackChain {@@ -382,13 +384,14 @@ pub struct ContinuationIterator(VMStackChain);pub struct StackLimitsIterator(VMStackChain);impl Iterator for ContinuationIterator {- type Item = *mut VMContRef;+ type Item = NonNull<VMContRef>;fn next(&mut self) -> Option<Self::Item> {match self.0 {VMStackChain::Absent | VMStackChain::InitialStack(_) => None,VMStackChain::Continuation(ptr) => {- let continuation = unsafe { ptr.as_mut().unwrap() };+ let ptr = ptr.as_non_null();+ let continuation = unsafe { ptr.as_ref() };self.0 = continuation.parent_chain.clone();Some(ptr)}@@ -397,20 +400,26 @@ impl Iterator for ContinuationIterator {}impl Iterator for StackLimitsIterator {- type Item = *mut VMStackLimits;+ type Item = NonNull<VMStackLimits>;fn next(&mut self) -> Option<Self::Item> {match self.0 {VMStackChain::Absent => None,VMStackChain::InitialStack(csi) => {- let stack_limits = unsafe { &mut (*csi).limits } as *mut VMStackLimits;+ let csi = csi.as_ptr();+ let stack_limits =+ unsafe { NonNull::new_unchecked(core::ptr::addr_of_mut!((*csi).limits)) };self.0 = VMStackChain::Absent;Some(stack_limits)}VMStackChain::Continuation(ptr) => {- let continuation = unsafe { ptr.as_mut().unwrap() };- let stack_limits =- (&mut continuation.common_stack_information.limits) as *mut VMStackLimits;+ let ptr = ptr.as_ptr();+ let continuation = unsafe { &*ptr };+ let stack_limits = unsafe {+ NonNull::new_unchecked(core::ptr::addr_of_mut!(+ (*ptr).common_stack_information.limits+ ))+ };self.0 = continuation.parent_chain.clone();Some(stack_limits)}diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rsindex 1e5781f17b..3c08078902 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs@@ -2,14 +2,14 @@ use crate::Result;use core::ops::Range;use crate::runtime::vm::VMPayloads;-use crate::runtime::vm::{VMContext, VMFuncRef};+use crate::runtime::vm::{VMContext, VMFuncRef, VmPtr};/// Making sure that this has the same size as the non-dummy version, to/// make some tests happy.#[derive(Debug)]#[repr(C)]pub struct VMContinuationStack {- _top: *mut u8,+ _top: VmPtr<u8>,_len: usize,_match_size_on_unix: u8,}diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rsindex 580b104653..0f35a6cf80 100644--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs@@ -75,9 +75,10 @@ pub enum Allocator {#[derive(Debug)]#[repr(C)]pub struct VMContinuationStack {- // The top of the stack; for stacks allocated by the fiber implementation itself,- // the base address of the allocation will be `top.sub(len.unwrap())`- top: *mut u8,+ // The top of the stack. For stacks allocated by the fiber implementation+ // itself, the base address of the allocation will be `top.sub(len)`.+ // Zero-length, unallocated stacks use a non-null dangling pointer.+ top: VmPtr<u8>,// The length of the stacklen: usize,// allocation strategy@@ -112,7 +113,7 @@ impl VMContinuationStack {)?;Ok(Self {- top: mmap.cast::<u8>().add(mmap_len),+ top: VmPtr::from(NonNull::new_unchecked(mmap.cast::<u8>().add(mmap_len))),len: mmap_len,allocator: Allocator::Mmap,})@@ -121,14 +122,13 @@ impl VMContinuationStack {pub fn unallocated() -> Self {Self {- top: std::ptr::null_mut(),+ top: VmPtr::dangling(),len: 0,allocator: Allocator::Custom,}}pub fn is_unallocated(&self) -> bool {- debug_assert_eq!(self.len == 0, self.top == std::ptr::null_mut());self.len == 0}@@ -138,7 +138,10 @@ impl VMContinuationStack {len: usize,) -> io::Result<Self> {Ok(Self {- top: unsafe { base.add(len) },+ top: VmPtr::from(+ NonNull::new(unsafe { base.add(len) })+ .expect("a continuation stack's top-of-stack pointer must be non-null"),+ ),len,allocator: Allocator::Custom,})@@ -149,11 +152,12 @@ impl VMContinuationStack {}pub fn top(&self) -> Option<*mut u8> {- Some(self.top)+ Some(self.top.as_ptr())}pub fn range(&self) -> Option<Range<usize>> {- let base = unsafe { self.top.sub(self.len).addr() };+ let top = self.top.as_ptr();+ let base = unsafe { top.sub(self.len).addr() };Some(base..base + self.len)}@@ -161,7 +165,7 @@ impl VMContinuationStack {// See picture at top of this file:// RIP is stored 8 bytes below top of stack.unsafe {- let ptr = self.top.sub(8).cast::<usize>();+ let ptr = self.top.as_ptr().sub(8).cast::<usize>();*ptr}}@@ -170,7 +174,7 @@ impl VMContinuationStack {// See picture at top of this file:// RBP is stored 16 bytes below top of stack.unsafe {- let ptr = self.top.sub(16).cast::<usize>();+ let ptr = self.top.as_ptr().sub(16).cast::<usize>();*ptr}}@@ -179,7 +183,7 @@ impl VMContinuationStack {// See picture at top of this file:// RSP is stored 24 bytes below top of stack.unsafe {- let ptr = self.top.sub(24).cast::<usize>();+ let ptr = self.top.as_ptr().sub(24).cast::<usize>();*ptr}}@@ -241,7 +245,7 @@ impl VMContinuationStack {return_value_count: u32,gc_refs: bool,) -> Result<()> {- let tos = self.top;+ let tos = self.top.as_ptr();unsafe {let store = |tos_neg_offset, value| {@@ -318,7 +322,7 @@ impl VMContinuationStack {};args_ref.capacity = args_capacity;- args_ref.data = args_data_ptr;+ args_ref.data = NonNull::new(args_data_ptr).map(VmPtr::from);if cfg!(feature = "gc") && gc_refs {let data = if args_capacity == 0 {ptr::null_mut()@@ -364,7 +368,8 @@ impl Drop for VMContinuationStack {unsafe {match self.allocator {Allocator::Mmap => {- let ret = rustix::mm::munmap(self.top.sub(self.len) as _, self.len);+ let top = self.top.as_ptr();+ let ret = rustix::mm::munmap(top.sub(self.len) as _, self.len);debug_assert!(ret.is_ok());}Allocator::Custom => {} // It's the creator's responsibility to reclaim the memory.@@ -388,8 +393,12 @@ unsafe extern "C" fn fiber_start(let params_and_returns: NonNull<[ValRaw]> = if args.capacity == 0 {NonNull::from(&[])} else {+ let data = args+ .data+ .expect("non-empty continuation arguments require an allocated buffer")+ .as_ptr();std::slice::from_raw_parts_mut(- args.data.cast::<ValRaw>(),+ data.cast::<ValRaw>(),usize::try_from(args.capacity).unwrap(),).into()diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rsindex 199efb34ba..ce154cfed0 100644--- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs+++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs@@ -158,11 +158,7 @@ impl Backtrace {.last_wasm_entry_fp;unsafe {- // FIXME(frank-emrich) Casting from *const to *mut pointer is- // terrible, but we won't actually modify any of the continuations- // here.- let stack_chain =- VMStackChain::Continuation(continuation as *const VMContRef as *mut VMContRef);+ let stack_chain = VMStackChain::Continuation(continuation.into());if let ControlFlow::Break(()) = Self::trace_through_continuations(stack_chain,@@ -330,12 +326,12 @@ impl Backtrace {// pair continuations[i] with stack_limits[i + 1].let mut stack_limits_iter = unsafe { chain.clone().into_stack_limits_iter() };// Skip the first stack limits (current running stack, handled above).- let _current: Option<*mut VMStackLimits> = stack_limits_iter.next();+ let _current: Option<core::ptr::NonNull<VMStackLimits>> = stack_limits_iter.next();let mut continuations_iter = unsafe { chain.into_continuation_iter() }.peekable();while let Some(continuation_ptr) = continuations_iter.next() {- let continuation = unsafe { &*continuation_ptr };+ let continuation = unsafe { continuation_ptr.as_ref() };let Some(parent_limits_ptr) = stack_limits_iter.next() else {// A detached suspended continuation chain ends in `Absent`,// unlike a running chain which ends in an initial stack. The@@ -344,10 +340,10 @@ impl Backtrace {debug_assert_eq!(continuation.parent_chain, VMStackChain::Absent);break;};- let parent_limits = unsafe { &*parent_limits_ptr };+ let parent_limits = unsafe { parent_limits_ptr.as_ref() };// The parent of `continuation` if present and not the last in the chain.- let parent_continuation = continuations_iter.peek().map(|&c| unsafe { &*c });+ let parent_continuation = continuations_iter.peek().map(|c| unsafe { c.as_ref() });let fiber_stack = continuation.fiber_stack();let resume_pc = fiber_stack.control_context_instruction_pointer();