Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

[stack-switching] Port raw pointer uses to `VmPtr` (#14369)

Daniel Hillerströme5ed75cf93f6
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index 4c8dca3135..511436017b 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -48,7 +48,6 @@ macro_rules! define_vm_type_offsets {
(@size ($p:expr) Option < VmPtr < $g:ty >>) => { u32::from($p) };
(@size ($p:expr) AtomicUsize) => { u32::from($p) };
(@size ($p:expr) usize) => { u32::from($p) };
- (@size ($p:expr) * mut $g:ty) => { u32::from($p) };
(@size ($p:expr) i64) => { 8u32 };
(@size ($p:expr) u64) => { 8u32 };
(@size ($p:expr) u32) => { 4u32 };
@@ -100,7 +99,6 @@ macro_rules! define_vm_type_offsets {
(@align ($p:expr) Option < VmPtr < $g:ty >>) => { u32::from($p) };
(@align ($p:expr) AtomicUsize) => { u32::from($p) };
(@align ($p:expr) usize) => { u32::from($p) };
- (@align ($p:expr) * mut $g:ty) => { u32::from($p) };
(@align ($p:expr) i64) => { 8u32 };
(@align ($p:expr) u64) => { 8u32 };
(@align ($p:expr) u32) => { 4u32 };
diff --git a/crates/environ/src/vmtypes.rs b/crates/environ/src/vmtypes.rs
index 1321291e72..3a8e22edb2 100644
--- a/crates/environ/src/vmtypes.rs
+++ b/crates/environ/src/vmtypes.rs
@@ -599,7 +599,7 @@ macro_rules! for_each_vm_type {
/// The buffer itself, which lives on the continuation's stack
/// rather than in this object.
- pub data: *mut u8,
+ pub data: Option<VmPtr<u8>>,
}
/// Payload values exchanged with a continuation and the metadata
@@ -652,9 +652,9 @@ macro_rules! for_each_vm_type {
#[aggregate]
pub parent_chain: VMStackChain,
- /// The end of this continuation's parent chain, used only while
- /// it is `Suspended` or `Fresh`.
- pub last_ancestor: *mut VMContRef,
+ /// The end of this continuation's parent chain while it is
+ /// `Suspended` or `Fresh`, and `None` while it is running.
+ pub last_ancestor: Option<VmPtr<VMContRef>>,
/// Revision counter.
pub revision: usize,
diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs
index c8ab027e51..71b123368e 100644
--- a/crates/wasmtime/src/runtime/func.rs
+++ b/crates/wasmtime/src/runtime/func.rs
@@ -3,7 +3,7 @@ use crate::prelude::*;
use crate::runtime::vm::{
self, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext,
VMCommonStackInformation, VMContext, VMFuncRef, VMFunctionImport, VMOpaqueContext,
- VMStoreContext,
+ VMStoreContext, VmPtr,
};
use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreId, StoreOpaque};
use crate::type_registry::RegisteredType;
@@ -1586,6 +1586,10 @@ impl EntryStoreContext {
unsafe {
let vm_store_context = store.0.vm_store_context();
+ let initial_stack_information = VmPtr::from(
+ NonNull::new(initial_stack_information)
+ .expect("expected non-null stack information"),
+ );
let new_stack_chain = VMStackChain::InitialStack(initial_stack_information);
let stack_chain =
mem::replace(&mut *vm_store_context.stack_chain.get(), new_stack_chain);
diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs
index 09a63ba2cf..72492ccbb7 100644
--- a/crates/wasmtime/src/runtime/store/gc.rs
+++ b/crates/wasmtime/src/runtime/store/gc.rs
@@ -846,11 +846,15 @@ impl StoreOpaque {
return;
};
let gc_ref_data = gc_ref_data.as_ptr();
+ let data = payloads
+ .data
+ .expect("payload GC metadata requires an allocated payload buffer")
+ .as_ptr();
for index in 0..usize::try_from(payloads.length).unwrap() {
let marker = unsafe { gc_ref_data.add(index).read() };
if marker == wasmtime_environ::CONTINUATION_PAYLOAD_GC_REF {
- let slot = unsafe { payloads.data.cast::<ValRaw>().add(index).cast::<u32>() };
+ let slot = unsafe { data.cast::<ValRaw>().add(index).cast::<u32>() };
unsafe {
StoreOpaque::trace_wasm_stack_slot(gc_roots_list, slot);
}
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching.rs b/crates/wasmtime/src/runtime/vm/stack_switching.rs
index 06f61340d3..f1984a63a2 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching.rs
@@ -3,7 +3,9 @@
mod stack;
-use crate::vm::{VMCommonStackInformation, VMContRef, VMHostArray, VMPayloads, VMStackLimits};
+use crate::vm::{
+ VMCommonStackInformation, VMContRef, VMHostArray, VMPayloads, VMStackLimits, VmPtr,
+};
use core::{marker::PhantomPinned, ptr::NonNull};
pub use stack::*;
@@ -106,7 +108,7 @@ impl VMHostArray {
Self {
length: 0,
capacity: 0,
- data: core::ptr::null_mut(),
+ data: None,
}
}
@@ -154,7 +156,7 @@ impl VMContRef {
first_switch_handler_index: 0,
};
let parent_chain = VMStackChain::Absent;
- let last_ancestor = core::ptr::null_mut();
+ let last_ancestor = None;
let stack = VMContinuationStack::unallocated();
let args = VMPayloads::empty();
let values = VMPayloads::empty();
@@ -215,7 +217,7 @@ pub fn cont_new(
contref.parent_chain = VMStackChain::Absent;
// The continuation is fresh, which is a special case of being suspended.
// Thus we need to set the correct end of the continuation chain: itself.
- contref.last_ancestor = contref;
+ contref.last_ancestor = Some(VmPtr::from(&*contref));
// The initialization function will allocate the actual args/return value buffer and
// update this object (if needed).
@@ -336,10 +338,10 @@ pub enum VMStackChain {
/// does not have a parent. The `CommonStackInformation` that this
/// variant points to is stored in the stack frame of
/// `invoke_wasm_and_catch_traps`.
- InitialStack(*mut VMCommonStackInformation) =
+ InitialStack(VmPtr<VMCommonStackInformation>) =
wasmtime_environ::STACK_CHAIN_INITIAL_STACK_DISCRIMINANT,
/// Represents a continuation's stack.
- Continuation(*mut VMContRef) = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT,
+ Continuation(VmPtr<VMContRef>) = wasmtime_environ::STACK_CHAIN_CONTINUATION_DISCRIMINANT,
}
impl VMStackChain {
@@ -382,13 +384,14 @@ pub struct ContinuationIterator(VMStackChain);
pub struct StackLimitsIterator(VMStackChain);
impl Iterator for ContinuationIterator {
- type Item = *mut VMContRef;
+ type Item = NonNull<VMContRef>;
fn next(&mut self) -> Option<Self::Item> {
match self.0 {
VMStackChain::Absent | VMStackChain::InitialStack(_) => None,
VMStackChain::Continuation(ptr) => {
- let continuation = unsafe { ptr.as_mut().unwrap() };
+ let ptr = ptr.as_non_null();
+ let continuation = unsafe { ptr.as_ref() };
self.0 = continuation.parent_chain.clone();
Some(ptr)
}
@@ -397,20 +400,26 @@ impl Iterator for ContinuationIterator {
}
impl Iterator for StackLimitsIterator {
- type Item = *mut VMStackLimits;
+ type Item = NonNull<VMStackLimits>;
fn next(&mut self) -> Option<Self::Item> {
match self.0 {
VMStackChain::Absent => None,
VMStackChain::InitialStack(csi) => {
- let stack_limits = unsafe { &mut (*csi).limits } as *mut VMStackLimits;
+ let csi = csi.as_ptr();
+ let stack_limits =
+ unsafe { NonNull::new_unchecked(core::ptr::addr_of_mut!((*csi).limits)) };
self.0 = VMStackChain::Absent;
Some(stack_limits)
}
VMStackChain::Continuation(ptr) => {
- let continuation = unsafe { ptr.as_mut().unwrap() };
- let stack_limits =
- (&mut continuation.common_stack_information.limits) as *mut VMStackLimits;
+ let ptr = ptr.as_ptr();
+ let continuation = unsafe { &*ptr };
+ let stack_limits = unsafe {
+ NonNull::new_unchecked(core::ptr::addr_of_mut!(
+ (*ptr).common_stack_information.limits
+ ))
+ };
self.0 = continuation.parent_chain.clone();
Some(stack_limits)
}
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs
index 1e5781f17b..3c08078902 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/dummy.rs
@@ -2,14 +2,14 @@ use crate::Result;
use core::ops::Range;
use crate::runtime::vm::VMPayloads;
-use crate::runtime::vm::{VMContext, VMFuncRef};
+use crate::runtime::vm::{VMContext, VMFuncRef, VmPtr};
/// Making sure that this has the same size as the non-dummy version, to
/// make some tests happy.
#[derive(Debug)]
#[repr(C)]
pub struct VMContinuationStack {
- _top: *mut u8,
+ _top: VmPtr<u8>,
_len: usize,
_match_size_on_unix: u8,
}
diff --git a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs
index 580b104653..0f35a6cf80 100644
--- a/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs
+++ b/crates/wasmtime/src/runtime/vm/stack_switching/stack/unix.rs
@@ -75,9 +75,10 @@ pub enum Allocator {
#[derive(Debug)]
#[repr(C)]
pub struct VMContinuationStack {
- // The top of the stack; for stacks allocated by the fiber implementation itself,
- // the base address of the allocation will be `top.sub(len.unwrap())`
- top: *mut u8,
+ // The top of the stack. For stacks allocated by the fiber implementation
+ // itself, the base address of the allocation will be `top.sub(len)`.
+ // Zero-length, unallocated stacks use a non-null dangling pointer.
+ top: VmPtr<u8>,
// The length of the stack
len: usize,
// allocation strategy
@@ -112,7 +113,7 @@ impl VMContinuationStack {
)?;
Ok(Self {
- top: mmap.cast::<u8>().add(mmap_len),
+ top: VmPtr::from(NonNull::new_unchecked(mmap.cast::<u8>().add(mmap_len))),
len: mmap_len,
allocator: Allocator::Mmap,
})
@@ -121,14 +122,13 @@ impl VMContinuationStack {
pub fn unallocated() -> Self {
Self {
- top: std::ptr::null_mut(),
+ top: VmPtr::dangling(),
len: 0,
allocator: Allocator::Custom,
}
}
pub fn is_unallocated(&self) -> bool {
- debug_assert_eq!(self.len == 0, self.top == std::ptr::null_mut());
self.len == 0
}
@@ -138,7 +138,10 @@ impl VMContinuationStack {
len: usize,
) -> io::Result<Self> {
Ok(Self {
- top: unsafe { base.add(len) },
+ top: VmPtr::from(
+ NonNull::new(unsafe { base.add(len) })
+ .expect("a continuation stack's top-of-stack pointer must be non-null"),
+ ),
len,
allocator: Allocator::Custom,
})
@@ -149,11 +152,12 @@ impl VMContinuationStack {
}
pub fn top(&self) -> Option<*mut u8> {
- Some(self.top)
+ Some(self.top.as_ptr())
}
pub fn range(&self) -> Option<Range<usize>> {
- let base = unsafe { self.top.sub(self.len).addr() };
+ let top = self.top.as_ptr();
+ let base = unsafe { top.sub(self.len).addr() };
Some(base..base + self.len)
}
@@ -161,7 +165,7 @@ impl VMContinuationStack {
// See picture at top of this file:
// RIP is stored 8 bytes below top of stack.
unsafe {
- let ptr = self.top.sub(8).cast::<usize>();
+ let ptr = self.top.as_ptr().sub(8).cast::<usize>();
*ptr
}
}
@@ -170,7 +174,7 @@ impl VMContinuationStack {
// See picture at top of this file:
// RBP is stored 16 bytes below top of stack.
unsafe {
- let ptr = self.top.sub(16).cast::<usize>();
+ let ptr = self.top.as_ptr().sub(16).cast::<usize>();
*ptr
}
}
@@ -179,7 +183,7 @@ impl VMContinuationStack {
// See picture at top of this file:
// RSP is stored 24 bytes below top of stack.
unsafe {
- let ptr = self.top.sub(24).cast::<usize>();
+ let ptr = self.top.as_ptr().sub(24).cast::<usize>();
*ptr
}
}
@@ -241,7 +245,7 @@ impl VMContinuationStack {
return_value_count: u32,
gc_refs: bool,
) -> Result<()> {
- let tos = self.top;
+ let tos = self.top.as_ptr();
unsafe {
let store = |tos_neg_offset, value| {
@@ -318,7 +322,7 @@ impl VMContinuationStack {
};
args_ref.capacity = args_capacity;
- args_ref.data = args_data_ptr;
+ args_ref.data = NonNull::new(args_data_ptr).map(VmPtr::from);
if cfg!(feature = "gc") && gc_refs {
let data = if args_capacity == 0 {
ptr::null_mut()
@@ -364,7 +368,8 @@ impl Drop for VMContinuationStack {
unsafe {
match self.allocator {
Allocator::Mmap => {
- let ret = rustix::mm::munmap(self.top.sub(self.len) as _, self.len);
+ let top = self.top.as_ptr();
+ let ret = rustix::mm::munmap(top.sub(self.len) as _, self.len);
debug_assert!(ret.is_ok());
}
Allocator::Custom => {} // It's the creator's responsibility to reclaim the memory.
@@ -388,8 +393,12 @@ unsafe extern "C" fn fiber_start(
let params_and_returns: NonNull<[ValRaw]> = if args.capacity == 0 {
NonNull::from(&[])
} else {
+ let data = args
+ .data
+ .expect("non-empty continuation arguments require an allocated buffer")
+ .as_ptr();
std::slice::from_raw_parts_mut(
- args.data.cast::<ValRaw>(),
+ data.cast::<ValRaw>(),
usize::try_from(args.capacity).unwrap(),
)
.into()
diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs
index 199efb34ba..ce154cfed0 100644
--- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs
+++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs
@@ -158,11 +158,7 @@ impl Backtrace {
.last_wasm_entry_fp;
unsafe {
- // FIXME(frank-emrich) Casting from *const to *mut pointer is
- // terrible, but we won't actually modify any of the continuations
- // here.
- let stack_chain =
- VMStackChain::Continuation(continuation as *const VMContRef as *mut VMContRef);
+ let stack_chain = VMStackChain::Continuation(continuation.into());
if let ControlFlow::Break(()) = Self::trace_through_continuations(
stack_chain,
@@ -330,12 +326,12 @@ impl Backtrace {
// pair continuations[i] with stack_limits[i + 1].
let mut stack_limits_iter = unsafe { chain.clone().into_stack_limits_iter() };
// Skip the first stack limits (current running stack, handled above).
- let _current: Option<*mut VMStackLimits> = stack_limits_iter.next();
+ let _current: Option<core::ptr::NonNull<VMStackLimits>> = stack_limits_iter.next();
let mut continuations_iter = unsafe { chain.into_continuation_iter() }.peekable();
while let Some(continuation_ptr) = continuations_iter.next() {
- let continuation = unsafe { &*continuation_ptr };
+ let continuation = unsafe { continuation_ptr.as_ref() };
let Some(parent_limits_ptr) = stack_limits_iter.next() else {
// A detached suspended continuation chain ends in `Absent`,
// unlike a running chain which ends in an initial stack. The
@@ -344,10 +340,10 @@ impl Backtrace {
debug_assert_eq!(continuation.parent_chain, VMStackChain::Absent);
break;
};
- let parent_limits = unsafe { &*parent_limits_ptr };
+ let parent_limits = unsafe { parent_limits_ptr.as_ref() };
// The parent of `continuation` if present and not the last in the chain.
- let parent_continuation = continuations_iter.peek().map(|&c| unsafe { &*c });
+ let parent_continuation = continuations_iter.peek().map(|c| unsafe { c.as_ref() });
let fiber_stack = continuation.fiber_stack();
let resume_pc = fiber_stack.control_context_instruction_pointer();