Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

wasmtime-fiber: add raw fibers with snapshots. (#15187)

Chris Fallinb95f045e4eb8
diff --git a/crates/core/src/alloc/vec.rs b/crates/core/src/alloc/vec.rs
index 1c65c4c2cd..4d4319bbd1 100644
--- a/crates/core/src/alloc/vec.rs
+++ b/crates/core/src/alloc/vec.rs
@@ -158,6 +158,20 @@ impl<T> TryVec<T> {
Ok(())
}
+ /// Same as [`std::vec::Vec::extend_from_slice`] but returns an error on
+ /// allocation failure, in which case `self` is unchanged.
+ ///
+ /// This is limited to `Copy` elements, which can be copied without
+ /// allocating, so that it is a single copy of the slice.
+ pub fn extend_from_slice(&mut self, other: &[T]) -> Result<(), OutOfMemory>
+ where
+ T: Copy,
+ {
+ self.reserve(other.len())?;
+ self.inner.extend_from_slice(other);
+ Ok(())
+ }
+
/// Same as [`std::vec::Vec::pop`].
pub fn pop(&mut self) -> Option<T> {
self.inner.pop()
@@ -483,6 +497,17 @@ mod tests {
use super::TryVec;
use crate::error::OutOfMemory;
+ #[test]
+ fn test_extend_from_slice() -> Result<(), OutOfMemory> {
+ let mut vec = TryVec::new();
+ vec.extend_from_slice(&[])?;
+ assert!(vec.is_empty());
+ vec.extend_from_slice(&[1, 2])?;
+ vec.extend_from_slice(&[3])?;
+ assert_eq!(*vec, [1, 2, 3]);
+ Ok(())
+ }
+
#[test]
fn test_into_boxed_slice() -> Result<(), OutOfMemory> {
assert_eq!(*TryVec::<i32>::new().into_boxed_slice()?, []);
diff --git a/crates/fiber/src/lib.rs b/crates/fiber/src/lib.rs
index 1b7e753a99..df683c1994 100644
--- a/crates/fiber/src/lib.rs
+++ b/crates/fiber/src/lib.rs
@@ -18,6 +18,9 @@ use core::marker::PhantomData;
use core::ops::Range;
use wasmtime_environ::error::Error;
+mod raw;
+pub use raw::{RawFiber, RawFiberEntry, RawFiberSnapshot, RawFiberState};
+
cfg_select! {
not(feature = "std") => {
mod nostd;
@@ -27,6 +30,8 @@ cfg_select! {
miri => {
mod miri;
use miri as imp;
+ // Unused by Miri's fibers, but raw fibers refer to it.
+ mod stackswitch;
}
windows => {
mod windows;
diff --git a/crates/fiber/src/raw.rs b/crates/fiber/src/raw.rs
new file mode 100644
index 0000000000..6ce8c2af72
--- /dev/null
+++ b/crates/fiber/src/raw.rs
@@ -0,0 +1,588 @@
+//! Raw fibers: fibers whose entry point and suspension points are machine code
+//! supplied by the caller rather than a Rust closure.
+//!
+//! See [`RawFiber`] for what raw fibers can do and the contract their code
+//! must uphold.
+
+use crate::FiberStack;
+use core::sync::atomic::Ordering;
+use wasmtime_environ::collections::TryVec;
+use wasmtime_environ::error::{Error, Result, bail, ensure};
+
+cfg_select! {
+ all(windows, feature = "std", not(miri)) => {
+ mod windows;
+ use windows as sys;
+ }
+ _ => {
+ mod stackswitch;
+ use stackswitch as sys;
+ }
+}
+
+/// The entry point of a raw fiber.
+///
+/// This is called on the fiber's stack with the `arg` given to
+/// [`RawFiber::new`] and the fiber's [`RawFiber::switch_arg`]. It must never
+/// return.
+pub type RawFiberEntry = unsafe extern "C" fn(arg: *mut u8, switch_arg: *mut u8);
+
+/// The lifecycle of a [`RawFiber`].
+///
+/// A raw fiber is only ever observed while stopped; it runs only for the
+/// duration of a call to [`RawFiber::resume`].
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum RawFiberState {
+ /// The fiber has not yet been resumed; resuming it calls its entry point.
+ Initial,
+ /// The fiber suspended itself and may be resumed.
+ Suspended,
+ /// The fiber suspended for the last time, as recorded by
+ /// [`RawFiber::finish`]. It has no continuation, so it may not be resumed
+ /// or snapshotted unless an earlier snapshot is restored first.
+ Terminal,
+}
+
+/// A fiber running caller-supplied machine code.
+///
+/// Raw fibers provide one key capability that ordinary fibers, which
+/// wrap a Rust closure, do not: the ability to snapshot and
+/// resume. (This capability is useful for Wasmtime features like
+/// record/replay with reversible execution.)
+///
+/// A raw fiber requires an entry point that is an `extern "C"`
+/// function with a particular signature (two opaque pointer args),
+/// and that *never returns*. Unlike ordinary fibers, this entry-point
+/// function is the first and only frame on the fiber stack when it
+/// starts executing.
+///
+/// A raw fiber's state -- stack and SP -- can be snapshotted and
+/// restored, potentially multiple times ("multi-shot
+/// continuations"). The safety of this is fairly subtle and has the
+/// following requirements:
+///
+/// - No Rust-compiled frame must exist anywhere on the fiber stack
+/// (because multiple returns from a function call in Rust are
+/// UB). This implies, in practice, either handwritten assembly or
+/// machine code that the user otherwise completely controls
+/// (e.g. Cranelift-compiled code in Wasmtime).
+/// - Any live pointers that may be in registers or on the stack
+/// (i.e., live anywhere in the executing code) continue to be live
+/// when execution is rewound to an earlier snapshot. In practice,
+/// it is safest to have "completely invariant pointers": e.g., VM
+/// data structures or memories/heaps that stay at the same address
+/// for the lifetime of a fiber.
+/// - The raw fiber entry point must never return. Instead, it can implement
+/// termination by yielding one final time, having set some state so that
+/// the rest of the runtime will then know to drop the raw fiber (thus freeing
+/// its stack).
+pub struct RawFiber {
+ stack: FiberStack,
+ state: RawFiberState,
+ id: RawFiberId,
+ sys: sys::Fiber,
+}
+
+/// A copy of a stopped [`RawFiber`]'s stack and lifecycle state.
+pub struct RawFiberSnapshot {
+ fiber: RawFiberId,
+ sp: usize,
+ bytes: TryVec<u8>,
+ state: RawFiberState,
+}
+
+/// A process-unique identifier of a [`RawFiber`], by which snapshots are
+/// matched to the fiber they were taken from.
+///
+/// IDs must never be reused, since a reused ID would let a snapshot be
+/// restored onto a different fiber. Where 64-bit atomics are available, IDs
+/// are 64 bits wide, so they never run out in practice; elsewhere, allocating
+/// an ID panics once a 32-bit counter is exhausted rather than wrapping.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+struct RawFiberId(u64);
+
+impl RawFiberId {
+ fn new() -> Self {
+ cfg_select! {
+ target_has_atomic = "64" => {
+ use core::sync::atomic::AtomicU64;
+ static NEXT_ID: AtomicU64 = AtomicU64::new(0);
+ RawFiberId(NEXT_ID.fetch_add(1, Ordering::Relaxed))
+ }
+ _ => {
+ use core::sync::atomic::AtomicU32;
+ static NEXT_ID: AtomicU32 = AtomicU32::new(0);
+ let mut id = NEXT_ID.load(Ordering::Relaxed);
+ loop {
+ let next = id.checked_add(1).expect("raw fiber IDs exhausted");
+ match NEXT_ID.compare_exchange_weak(
+ id,
+ next,
+ Ordering::Relaxed,
+ Ordering::Relaxed,
+ ) {
+ Ok(_) => break RawFiberId(u64::from(id)),
+ Err(actual) => id = actual,
+ }
+ }
+ }
+ }
+ }
+}
+
+impl RawFiber {
+ /// Whether raw fibers are supported on this platform and configuration.
+ ///
+ /// Raw fibers require one of the stack-switching routines in this crate
+ /// (not an embedder's `custom` one), or Windows fibers on x86_64 or
+ /// aarch64. They are not supported under Miri or with AddressSanitizer,
+ /// whose fiber-switch handshakes would require Rust code at every
+ /// suspension point, nor with hardware-enforced shadow stacks, which
+ /// snapshots do not restore.
+ pub fn is_supported() -> bool {
+ sys::supported() && !cfg!(asan)
+ }
+
+ /// Creates a new raw fiber that calls `entry(arg, switch_arg)` when first
+ /// resumed.
+ ///
+ /// On error the stack is handed back to the caller.
+ ///
+ /// # Safety
+ ///
+ /// `entry` must uphold the contract of [`RawFiberEntry`] and suspend only
+ /// through [`RawFiber::switch_routine`], passing [`RawFiber::switch_arg`].
+ /// `arg` must remain valid for as long as the fiber may run.
+ pub unsafe fn new(
+ mut stack: FiberStack,
+ entry: RawFiberEntry,
+ arg: *mut u8,
+ ) -> Result<Self, (Error, FiberStack)> {
+ if !Self::is_supported() {
+ return Err((
+ wasmtime_environ::error::format_err!(
+ "raw fibers are not supported on this platform"
+ ),
+ stack,
+ ));
+ }
+ // SAFETY: the stack is owned by this fiber, and `entry` and `arg`
+ // are valid per this function's contract.
+ let sys = match unsafe { sys::Fiber::new(&mut stack, entry, arg) } {
+ Ok(sys) => sys,
+ Err(e) => return Err((e, stack)),
+ };
+ Ok(RawFiber {
+ stack,
+ state: RawFiberState::Initial,
+ id: RawFiberId::new(),
+ sys,
+ })
+ }
+
+ /// The assembly routine which code on a raw fiber calls, with the
+ /// C calling convention and [`RawFiber::switch_arg`] as its only
+ /// argument, to suspend itself. It returns when the fiber is next
+ /// resumed.
+ pub fn switch_routine() -> unsafe extern "C" fn(*mut u8) {
+ sys::switch_routine()
+ }
+
+ /// The argument which code on this fiber passes to
+ /// [`RawFiber::switch_routine`]. This is the top of the fiber's stack as
+ /// reported by [`FiberStack::top`].
+ pub fn switch_arg(&self) -> *mut u8 {
+ self.stack.top().unwrap()
+ }
+
+ /// The current lifecycle state of this fiber.
+ pub fn state(&self) -> RawFiberState {
+ self.state
+ }
+
+ /// The stack this fiber runs on.
+ pub fn stack(&self) -> &FiberStack {
+ &self.stack
+ }
+
+ /// Destroys this fiber, returning its stack. Nothing on the fiber's stack
+ /// is run or unwound.
+ pub fn into_stack(self) -> FiberStack {
+ let mut stack = self.stack;
+ self.sys.destroy(&mut stack);
+ stack
+ }
+
+ /// Runs this fiber until it next suspends itself.
+ ///
+ /// Returns an error, without running anything, if the fiber is terminal.
+ ///
+ /// # Safety
+ ///
+ /// Everything the fiber's code requires must be valid for the duration of
+ /// the call. In particular after a [`RawFiber::restore`], every pointer the
+ /// restored continuation can dereference must refer to live storage.
+ pub unsafe fn resume(&mut self) -> Result<()> {
+ ensure!(
+ self.state != RawFiberState::Terminal,
+ "cannot resume a terminal raw fiber"
+ );
+ let top = self.switch_arg();
+ // SAFETY: the fiber is stopped and everything it requires is valid
+ // per this function's contract.
+ unsafe { self.sys.resume(top) };
+ self.state = RawFiberState::Suspended;
+ Ok(())
+ }
+
+ /// Records that this fiber's most recent suspension was its last one.
+ ///
+ /// The fiber's owner learns this through its own protocol with the code
+ /// running on the fiber.
+ pub fn finish(&mut self) -> Result<()> {
+ ensure!(
+ self.state == RawFiberState::Suspended,
+ "only a suspended raw fiber can finish"
+ );
+ self.state = RawFiberState::Terminal;
+ Ok(())
+ }
+
+ fn saved_sp_slot(&self) -> *mut usize {
+ // See the layout diagrams in `unix.rs` and `raw/windows.rs`: the
+ // second slot below the top holds the stack pointer to resume the
+ // fiber at.
+ self.switch_arg().cast::<usize>().wrapping_sub(2)
+ }
+
+ /// The stack pointer at which this fiber will resume.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the fiber is terminal: it will not resume, so it has no
+ /// continuation to inspect.
+ fn saved_sp(&self) -> usize {
+ assert_ne!(
+ self.state,
+ RawFiberState::Terminal,
+ "a terminal raw fiber has no saved stack pointer"
+ );
+ // SAFETY: the reserved slot is initialized when the fiber is created
+ // and maintained by the switch routine.
+ unsafe { self.saved_sp_slot().read() }
+ }
+
+ /// Copies this stopped fiber's stack and lifecycle state.
+ ///
+ /// Returns an error if the fiber is terminal: it has no continuation to
+ /// copy.
+ pub fn snapshot(&self) -> Result<RawFiberSnapshot> {
+ ensure!(
+ self.state != RawFiberState::Terminal,
+ "cannot snapshot a terminal raw fiber"
+ );
+ let top = self.switch_arg() as usize;
+ let sp = self.saved_sp();
+ let range = self.stack.range();
+ ensure!(
+ range.is_none_or(|r| r.start <= sp) && sp < top,
+ "raw fiber has an invalid saved stack pointer"
+ );
+ let len = top - sp;
+ let mut bytes = TryVec::new();
+ // SAFETY: `sp..top` is within this fiber's stack, which is not
+ // running because `self` is borrowed here.
+ bytes.extend_from_slice(unsafe { core::slice::from_raw_parts(sp as *const u8, len) })?;
+ Ok(RawFiberSnapshot {
+ fiber: self.id,
+ sp,
+ bytes,
+ state: self.state,
+ })
+ }
+
+ /// Restores a snapshot previously taken from this fiber, replacing its
+ /// current continuation and lifecycle state. The next [`RawFiber::resume`]
+ /// continues from the snapshot's suspension point (or the entry point, for
+ /// a snapshot of an initial fiber), returning to that resume's caller.
+ ///
+ /// Snapshots taken from a different fiber are rejected without modifying
+ /// this fiber.
+ ///
+ /// # Safety
+ ///
+ /// The snapshot must have been taken at a point eligible for restoration,
+ /// as described in the [`RawFiber`] documentation.
+ pub unsafe fn restore(&mut self, snapshot: &RawFiberSnapshot) -> Result<()> {
+ if snapshot.fiber != self.id {
+ bail!("raw fiber snapshot belongs to a different fiber");
+ }
+ // SAFETY: the snapshot came from this fiber's stack, so the range is
+ // in bounds. It includes the saved stack pointer slot, which this
+ // overwrites with `snapshot.sp`, and the stack is not running.
+ unsafe {
+ core::ptr::copy_nonoverlapping(
+ snapshot.bytes.as_ptr(),
+ snapshot.sp as *mut u8,
+ snapshot.bytes.len(),
+ );
+ }
+ self.state = snapshot.state;
+ debug_assert_eq!(self.saved_sp(), snapshot.sp);
+ Ok(())
+ }
+
+ /// The return address and frame pointer of the code that called the
+ /// switch routine, for walking a suspended fiber's frames.
+ ///
+ /// Returns `None` if the fiber has not suspended or if this is not
+ /// implemented for the current architecture.
+ ///
+ /// # Panics
+ ///
+ /// Panics if the fiber is terminal: it will not resume, so it has no
+ /// continuation to inspect.
+ pub fn suspended_frame(&self) -> Option<(usize, usize)> {
+ assert_ne!(
+ self.state,
+ RawFiberState::Terminal,
+ "a terminal raw fiber has no suspended frame"
+ );
+ let (fp, pc) = sys::SUSPENDED_FRAME?;
+ if self.state == RawFiberState::Initial {
+ return None;
+ }
+ let sp = self.saved_sp() as *const usize;
+ // SAFETY: a suspended fiber's saved stack pointer addresses the
+ // registers spilled by the switch routine.
+ unsafe { Some((sp.add(pc).read(), sp.add(fp).read())) }
+ }
+}
+
+impl RawFiberSnapshot {
+ /// The lifecycle state the fiber will have once this is restored.
+ pub fn state(&self) -> RawFiberState {
+ self.state
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use core::cell::Cell;
+
+ /// State shared between a test and the code on its raw fiber.
+ #[derive(Default)]
+ struct Shared {
+ observed: Cell<u64>,
+ finished: Cell<bool>,
+ }
+
+ /// Yields the values 1, 2, 3 and then finishes. The counter is local to
+ /// the fiber, so a restored snapshot rewinds it.
+ unsafe extern "C" fn count_to_three(arg: *mut u8, switch_arg: *mut u8) {
+ // SAFETY: tests pass a `Shared` that outlives the fiber's use.
+ let shared = unsafe { &*arg.cast::<Shared>() };
+ let switch = RawFiber::switch_routine();
+ let mut count = 0;
+ while count < 3 {
+ count += 1;
+ shared.observed.set(count);
+ // SAFETY: this is the switch routine for this fiber.
+ unsafe { switch(switch_arg) };
+ }
+ shared.finished.set(true);
+ // SAFETY: as above.
+ unsafe { switch(switch_arg) };
+ // Resuming a terminal fiber is rejected by `RawFiber::resume`.
+ std::process::abort();
+ }
+
+ fn fiber(shared: &Shared) -> Option<RawFiber> {
+ if !RawFiber::is_supported() {
+ return None;
+ }
+ let stack = FiberStack::new(64 * 1024, false).unwrap();
+ let arg = core::ptr::from_ref(shared).cast_mut().cast();
+ Some(
+ unsafe { RawFiber::new(stack, count_to_three, arg) }
+ .map_err(|e| e.0)
+ .unwrap(),
+ )
+ }
+
+ /// Resumes `fiber` from a host stack frame `depth` calls deeper than the
+ /// caller, so that each resume has a different host continuation.
+ #[inline(never)]
+ fn resume_at_depth(fiber: &mut RawFiber, depth: usize) {
+ if depth == 0 {
+ unsafe { fiber.resume().unwrap() };
+ } else {
+ resume_at_depth(fiber, depth - 1);
+ core::hint::black_box(depth);
+ }
+ }
+
+ #[test]
+ fn lifecycle() {
+ let shared = Shared::default();
+ let Some(mut fiber) = fiber(&shared) else {
+ return;
+ };
+ assert_eq!(fiber.state(), RawFiberState::Initial);
+ for i in 1..=3 {
+ resume_at_depth(&mut fiber, i as usize);
+ assert_eq!(shared.observed.get(), i);
+ assert_eq!(fiber.state(), RawFiberState::Suspended);
+ }
+ unsafe { fiber.resume().unwrap() };
+ assert!(shared.finished.get());
+ fiber.finish().unwrap();
+ assert_eq!(fiber.state(), RawFiberState::Terminal);
+ assert!(unsafe { fiber.resume() }.is_err());
+ assert!(fiber.finish().is_err());
+ assert!(fiber.snapshot().is_err());
+ // A terminal fiber is destroyed without being resumed.
+ drop(fiber.into_stack());
+ }
+
+ #[test]
+ fn rewind() {
+ let shared = Shared::default();
+ let Some(mut fiber) = fiber(&shared) else {
+ return;
+ };
+ let initial = fiber.snapshot().unwrap();
+ unsafe { fiber.resume().unwrap() };
+ let first = fiber.snapshot().unwrap();
+ assert_eq!(first.state(), RawFiberState::Suspended);
+
+ // Repeatedly rewind to the first yield, resuming from different host
+ // frames each time.
+ for depth in 0..3 {
+ unsafe { fiber.resume().unwrap() };
+ assert_eq!(shared.observed.get(), 2);
+ unsafe { fiber.restore(&first).unwrap() };
+ resume_at_depth(&mut fiber, depth);
+ assert_eq!(shared.observed.get(), 2);
+ unsafe { fiber.restore(&first).unwrap() };
+ }
+
+ // Run to completion, then rewind after the final yield.
+ for _ in 0..3 {
+ unsafe { fiber.resume().unwrap() };
+ }
+ assert!(shared.finished.get());
+ fiber.finish().unwrap();
+ unsafe { fiber.restore(&first).unwrap() };
+ assert_eq!(fiber.state(), RawFiberState::Suspended);
+ unsafe { fiber.resume().unwrap() };
+ assert_eq!(shared.observed.get(), 2);
+
+ // Restart from the very beginning.
+ unsafe { fiber.restore(&initial).unwrap() };
+ assert_eq!(fiber.state(), RawFiberState::Initial);
+ unsafe { fiber.resume().unwrap() };
+ assert_eq!(shared.observed.get(), 1);
+ }
+
+ #[test]
+ fn snapshots_belong_to_one_fiber() {
+ let a = Shared::default();
+ let b = Shared::default();
+ let (Some(mut fa), Some(mut fb)) = (fiber(&a), fiber(&b)) else {
+ return;
+ };
+ unsafe { fa.resume().unwrap() };
+ let snapshot = fa.snapshot().unwrap();
+ assert!(unsafe { fb.restore(&snapshot) }.is_err());
+ assert_eq!(fb.state(), RawFiberState::Initial);
+
+ // Destroying a fiber leaves its outstanding snapshots inert, even if a
+ // new fiber reuses the same stack memory.
+ unsafe { fa.resume().unwrap() };
+ let stack = fa.into_stack();
+ let arg = core::ptr::from_ref(&a).cast_mut().cast();
+ let mut fc = unsafe { RawFiber::new(stack, count_to_three, arg) }
+ .map_err(|e| e.0)
+ .unwrap();
+ assert!(unsafe { fc.restore(&snapshot) }.is_err());
+ unsafe { fc.resume().unwrap() };
+ assert_eq!(a.observed.get(), 1);
+ drop(snapshot);
+ unsafe { fb.resume().unwrap() };
+ assert_eq!(b.observed.get(), 1);
+ }
+
+ #[test]
+ fn resume_on_another_thread() {
+ let shared = std::sync::Arc::new(SharedSync::default());
+ if !RawFiber::is_supported() {
+ return;
+ }
+ unsafe extern "C" fn entry(arg: *mut u8, switch_arg: *mut u8) {
+ let shared = unsafe { &*arg.cast::<SharedSync>() };
+ let switch = RawFiber::switch_routine();
+ loop {
+ shared.0.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
+ unsafe { switch(switch_arg) };
+ }
+ }
+ #[derive(Default)]
+ struct SharedSync(core::sync::atomic::AtomicUsize);
+
+ let stack = FiberStack::new(64 * 1024, false).unwrap();
+ let arg = std::sync::Arc::as_ptr(&shared).cast_mut().cast();
+ let mut fiber = unsafe { RawFiber::new(stack, entry, arg) }
+ .map_err(|e| e.0)
+ .unwrap();
+ unsafe { fiber.resume().unwrap() };
+ let snapshot = fiber.snapshot().unwrap();
+ let mut fiber = std::thread::spawn(move || {
+ unsafe { fiber.resume().unwrap() };
+ fiber
+ })
+ .join()
+ .unwrap();
+ unsafe { fiber.restore(&snapshot).unwrap() };
+ unsafe { fiber.resume().unwrap() };
+ assert_eq!(shared.0.load(core::sync::atomic::Ordering::Relaxed), 3);
+ }
+
+ #[test]
+ fn terminal_fiber_has_no_suspended_frame() {
+ let shared = Shared::default();
+ let Some(mut fiber) = fiber(&shared) else {
+ return;
+ };
+ for _ in 0..4 {
+ unsafe { fiber.resume().unwrap() };
+ }
+ fiber.finish().unwrap();
+ let result = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| {
+ fiber.suspended_frame();
+ }));
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn suspended_frame_points_into_entry() {
+ let shared = Shared::default();
+ let Some(mut fiber) = fiber(&shared) else {
+ return;
+ };
+ assert!(fiber.suspended_frame().is_none());
+ unsafe { fiber.resume().unwrap() };
+ if let Some((pc, fp)) = fiber.suspended_frame() {
+ // Without frame pointers in this Rust entry, `fp` may still be
+ // the initial one: the top of the stack.
+ let range = fiber.stack().range().unwrap();
+ assert!(
+ range.start <= fp && fp <= range.end,
+ "{fp:#x} vs {range:x?}"
+ );
+ let entry = count_to_three as *const () as usize;
+ assert!(pc > entry && pc < entry + 4096, "{pc:#x} vs {entry:#x}");
+ }
+ }
+}
diff --git a/crates/fiber/src/raw/stackswitch.rs b/crates/fiber/src/raw/stackswitch.rs
new file mode 100644
index 0000000000..45e98d4bcb
--- /dev/null
+++ b/crates/fiber/src/raw/stackswitch.rs
@@ -0,0 +1,66 @@
+//! Raw fibers on this crate's stack-switching routines, which keep the
+//! suspended context on the fiber's stack in the reserved-slot layout
+//! documented in `unix.rs`.
+
+use super::RawFiberEntry;
+use crate::stackswitch::{
+ RAW_FIBERS, wasmtime_fiber_init, wasmtime_fiber_switch, wasmtime_fiber_switch_,
+};
+use wasmtime_environ::error::{Result, format_err};
+
+pub fn supported() -> bool {
+ RAW_FIBERS
+}
+
+pub fn switch_routine() -> unsafe extern "C" fn(*mut u8) {
+ wasmtime_fiber_switch_
+}
+
+// Indices, in words from the saved stack pointer, of the frame
+// pointer and return address that the switch routine spills. Keep
+// in sync with `wasmtime_fiber_switch_` in `stackswitch/*.rs`.
+pub const SUSPENDED_FRAME: Option<(usize, usize)> = if cfg!(target_arch = "x86_64") {
+ Some((5, 6))
+} else if cfg!(target_arch = "aarch64") {
+ Some((18, 19))
+} else {
+ None
+};
+
+pub struct Fiber;
+
+impl Fiber {
+ pub unsafe fn new(
+ stack: &mut crate::FiberStack,
+ entry: RawFiberEntry,
+ arg: *mut u8,
+ ) -> Result<Self> {
+ let top = stack
+ .top()
+ .ok_or_else(|| format_err!("raw fibers require a known stack top"))?;
+ // SAFETY: the stack is owned by the raw fiber and large
+ // enough for the initial frame, which the existing
+ // closure-based fibers rely on too. The fiber-start
+ // trampoline only forwards both arguments to `entry`.
+ unsafe {
+ wasmtime_fiber_init(
+ top,
+ core::mem::transmute::<RawFiberEntry, extern "C" fn(*mut u8, *mut u8) -> *mut u8>(
+ entry,
+ ),
+ arg,
+ );
+ }
+ Ok(Fiber)
+ }
+
+ pub unsafe fn resume(&mut self, top: *mut u8) {
+ // SAFETY: the saved stack pointer of a stopped fiber is in
+ // the reserved slot. The switch stores this host
+ // continuation in the same slot, so the fiber always
+ // suspends back to here.
+ unsafe { wasmtime_fiber_switch(top) }
+ }
+
+ pub fn destroy(self, _stack: &mut crate::FiberStack) {}
+}
diff --git a/crates/fiber/src/raw/windows.rs b/crates/fiber/src/raw/windows.rs
new file mode 100644
index 0000000000..32beeabd50
--- /dev/null
+++ b/crates/fiber/src/raw/windows.rs
@@ -0,0 +1,196 @@
+//! Raw fibers on top of Windows fibers.
+//!
+//! `SwitchToFiber` saves a fiber's stack pointer and nonvolatile registers in
+//! the fiber's undocumented, heap-allocated `FIBER` structure rather than on
+//! its stack, so the stack bytes alone do not capture where to resume a
+//! suspended fiber. Instead every suspension goes through one canonical frame:
+//! the suspend routine spills all nonvolatile registers onto the fiber's
+//! stack, stores the stack pointer in a reserved slot, and calls
+//! `SwitchToFiber` with a fixed stack pointer near the top of the stack. What
+//! Windows saves is thus the same at every suspension, and when
+//! `SwitchToFiber` returns the suspend routine takes everything else from the
+//! slot and the stack. A raw fiber's snapshot therefore captures it completely,
+//! just as on other platforms, while Windows keeps managing everything else
+//! about the fiber: its stack bounds, fiber-local storage, and so on.
+//!
+//! The start routine suspends through the same frame once before calling the
+//! entry point, so a new raw fiber is a suspended fiber whose continuation
+//! calls its entry point.
+//!
+//! The top of a raw fiber's stack, where `top` is its `switch_arg`, is:
+//!
+//! ```text
+//! frames of `fiber_start` and Windows' own fiber start (fixed)
+//! top + 8 return address into `fiber_start`
+//! top frame pointer of `fiber_start`
+//! top - 0x08 host fiber to suspend to, stored by each resume
+//! top - 0x10 stack pointer to resume this fiber at
+//! frame from which the suspend routine calls `SwitchToFiber`
+//! top - SCRATCH start of the entry point's stack
+//! ```
+//!
+//! A raw fiber's `FiberStack` reports its range as the bottom of the stack's
+//! reservation up to `top`, which is also what it reports as the stack's top.
+
+use crate::RawFiberEntry;
+use crate::windows::{
+ FIBER_FLAG_FLOAT_SWITCH, delete_fiber, set_stack_guarantee, with_current_fiber,
+};
+use std::ffi::c_void;
+use std::io;
+use std::ptr;
+use wasmtime_environ::prelude::*;
+use windows_sys::Win32::System::Threading::*;
+
+cfg_select! {
+ target_arch = "x86_64" => {
+ mod x86_64;
+ use x86_64 as asm;
+ }
+ target_arch = "aarch64" => {
+ mod aarch64;
+ use aarch64 as asm;
+ }
+ _ => {
+ mod asm {
+ pub const SUPPORTED_ARCH: bool = false;
+ pub const SUSPENDED_FRAME: Option<(usize, usize)> = None;
+
+ pub unsafe extern "C" fn wasmtime_fiber_start(
+ _entry: crate::RawFiberEntry,
+ _arg: *mut u8,
+ _host: *mut core::ffi::c_void,
+ _top: *mut *mut u8,
+ ) -> ! {
+ unreachable!()
+ }
+
+ pub unsafe extern "C" fn wasmtime_fiber_suspend(_top: *mut u8) {
+ unreachable!()
+ }
+ }
+ }
+}
+
+pub(crate) use asm::SUSPENDED_FRAME;
+
+/// A raw fiber's underlying Windows fiber, which owns its stack's memory.
+pub(crate) struct Fiber(*mut c_void);
+
+// SAFETY: a Windows fiber which is not running may be switched to from any
+// thread, and `Fiber` only does so from `&mut self`.
+unsafe impl Send for Fiber {}
+unsafe impl Sync for Fiber {}
+
+/// What `fiber_start` passes to `wasmtime_fiber_start`, and what it reports
+/// back.
+struct Start {
+ entry: RawFiberEntry,
+ arg: *mut u8,
+ host: *mut c_void,
+ top: *mut u8,
+ bottom: usize,
+}
+
+/// Whether raw fibers are supported on this platform and in this process.
+pub(crate) fn supported() -> bool {
+ asm::SUPPORTED_ARCH && !user_shadow_stacks()
+}
+
+/// Whether hardware-enforced shadow stacks are enabled for this process.
+/// Restoring a snapshot changes the return addresses on a fiber's stack but
+/// not on its shadow stack, so a restored fiber would fault on return.
+fn user_shadow_stacks() -> bool {
+ // `PROCESS_MITIGATION_USER_SHADOW_STACK_POLICY` is a `DWORD` of flags.
+ const ENABLE_USER_SHADOW_STACK: u32 = 1 << 0;
+ let mut flags = 0u32;
+ // SAFETY: `flags` has the size passed. On versions of Windows without
+ // shadow stacks this fails and leaves `flags` zero.
+ unsafe {
+ GetProcessMitigationPolicy(
+ GetCurrentProcess(),
+ ProcessUserShadowStackPolicy,
+ (&raw mut flags).cast(),
+ size_of::<u32>(),
+ );
+ }
+ flags & ENABLE_USER_SHADOW_STACK != 0
+}
+
+pub(crate) fn switch_routine() -> unsafe extern "C" fn(*mut u8) {
+ asm::wasmtime_fiber_suspend
+}
+
+unsafe extern "system" fn fiber_start(data: *mut c_void) {
+ unsafe {
+ set_stack_guarantee();
+ let start = data.cast::<Start>();
+ let mut high = 0;
+ GetCurrentThreadStackLimits(&raw mut (*start).bottom, &mut high);
+ asm::wasmtime_fiber_start(
+ (*start).entry,
+ (*start).arg,
+ (*start).host,
+ &raw mut (*start).top,
+ )
+ }
+}
+
+impl Fiber {
+ /// Creates a Windows fiber for `stack` and runs it until its initial
+ /// suspension, after which `stack` reports the fiber's stack bounds.
+ pub(crate) unsafe fn new(
+ stack: &mut crate::FiberStack,
+ entry: RawFiberEntry,
+ arg: *mut u8,
+ ) -> Result<Self> {
+ let mut start = Start {
+ entry,
+ arg,
+ host: ptr::null_mut(),
+ top: ptr::null_mut(),
+ bottom: 0,
+ };
+ let start = &raw mut start;
+ unsafe {
+ let fiber = CreateFiberEx(
+ 0,
+ stack.0.size,
+ FIBER_FLAG_FLOAT_SWITCH,
+ Some(fiber_start),
+ start.cast(),
+ );
+ if fiber.is_null() {
+ return Err(io::Error::last_os_error().into());
+ }
+ with_current_fiber(|host| {
+ (*start).host = host;
+ SwitchToFiber(fiber);
+ });
+ stack.0.raw_range = Some((*start).bottom..(*start).top as usize);
+ Ok(Fiber(fiber))
+ }
+ }
+
+ /// Runs the fiber, whose stack top is `top`, until it next suspends.
+ pub(crate) unsafe fn resume(&mut self, top: *mut u8) {
+ unsafe {
+ with_current_fiber(|host| {
+ top.cast::<*mut c_void>().sub(1).write(host);
+ SwitchToFiber(self.0);
+ });
+ }
+ }
+
+ /// Deletes the fiber, after which `stack` may be used for another one.
+ pub(crate) fn destroy(self, stack: &mut crate::FiberStack) {
+ stack.0.raw_range = None;
+ }
+}
+
+impl Drop for Fiber {
+ fn drop(&mut self) {
+ // SAFETY: a raw fiber only runs during `resume`.
+ unsafe { delete_fiber(self.0) }
+ }
+}
diff --git a/crates/fiber/src/raw/windows/aarch64.rs b/crates/fiber/src/raw/windows/aarch64.rs
new file mode 100644
index 0000000000..e300a049b2
--- /dev/null
+++ b/crates/fiber/src/raw/windows/aarch64.rs
@@ -0,0 +1,147 @@
+// A WORD OF CAUTION
+//
+// This file needs to be kept in sync with itself and with the layout
+// documented in `raw/windows.rs`, and `SUSPENDED_FRAME` with the frame that
+// `wasmtime_fiber_suspend` spills. It closely follows `x86_64.rs`.
+
+use core::arch::naked_asm;
+use windows_sys::Win32::System::Threading::SwitchToFiber;
+
+pub const SUPPORTED_ARCH: bool = true;
+
+/// The indices, in words from a suspended fiber's saved stack pointer, of the
+/// frame pointer and return address that `wasmtime_fiber_suspend` spills.
+pub const SUSPENDED_FRAME: Option<(usize, usize)> = Some((0, 1));
+
+/// Suspends the raw fiber whose stack top is `top`, returning when it is next
+/// resumed.
+#[unsafe(naked)]
+pub unsafe extern "C" fn wasmtime_fiber_suspend(top: *mut u8 /* x0 */) {
+ naked_asm!(
+ "
+ .seh_proc {suspend}
+
+ // Spill all nonvolatile registers, ending with a frame record.
+ stp d8, d9, [sp, -16]!
+ .seh_save_fregp_x d8, 16
+ stp d10, d11, [sp, -16]!
+ .seh_save_fregp_x d10, 16
+ stp d12, d13, [sp, -16]!
+ .seh_save_fregp_x d12, 16
+ stp d14, d15, [sp, -16]!
+ .seh_save_fregp_x d14, 16
+ stp x19, x20, [sp, -16]!
+ .seh_save_regp_x x19, 16
+ stp x21, x22, [sp, -16]!
+ .seh_save_regp_x x21, 16
+ stp x23, x24, [sp, -16]!
+ .seh_save_regp_x x23, 16
+ stp x25, x26, [sp, -16]!
+ .seh_save_regp_x x25, 16
+ stp x27, x28, [sp, -16]!
+ .seh_save_regp_x x27, 16
+ stp x29, x30, [sp, -16]!
+ .seh_save_fplr_x 16
+ // Unwind through the frame pointer while on the canonical frame.
+ mov x29, sp
+ .seh_set_fp
+ .seh_endprologue
+
+ // Store where to resume, then switch to the host fiber from the
+ // canonical frame.
+ mov x9, sp
+ str x9, [x0, -0x10]
+ ldr x9, [x0, -0x08]
+ sub sp, x0, 0x10
+ mov x0, x9
+ bl {switch_to_fiber}
+
+ // Resumed. Only the stack pointer is meaningful here: Windows restored
+ // the other registers from whichever suspension last saved them, which
+ // need not be the one resumed after a snapshot was restored. Take the
+ // registers spilled above from the stack pointer in the slot.
+ ldr x29, [sp]
+ .seh_startepilogue
+ mov sp, x29
+ .seh_set_fp
+ ldp x29, x30, [sp], 16
+ .seh_save_fplr_x 16
+ ldp x27, x28, [sp], 16
+ .seh_save_regp_x x27, 16
+ ldp x25, x26, [sp], 16
+ .seh_save_regp_x x25, 16
+ ldp x23, x24, [sp], 16
+ .seh_save_regp_x x23, 16
+ ldp x21, x22, [sp], 16
+ .seh_save_regp_x x21, 16
+ ldp x19, x20, [sp], 16
+ .seh_save_regp_x x19, 16
+ ldp d14, d15, [sp], 16
+ .seh_save_fregp_x d14, 16
+ ldp d12, d13, [sp], 16
+ .seh_save_fregp_x d12, 16
+ ldp d10, d11, [sp], 16
+ .seh_save_fregp_x d10, 16
+ ldp d8, d9, [sp], 16
+ .seh_save_fregp_x d8, 16
+ .seh_endepilogue
+ ret
+ .seh_endproc
+ ",
+ suspend = sym wasmtime_fiber_suspend,
+ switch_to_fiber = sym SwitchToFiber,
+ );
+}
+
+/// Sets up the top of the stack as documented in `raw/windows.rs`, reports `top`
+/// through `top_out`, suspends, and then calls `entry(arg, top)` when first
+/// resumed.
+///
+/// This never returns, so it does not preserve its caller's nonvolatile
+/// registers; the unwind information only describes how to find the caller's
+/// frame.
+#[unsafe(naked)]
+pub unsafe extern "C" fn wasmtime_fiber_start(
+ entry: crate::RawFiberEntry, // x0
+ arg: *mut u8, // x1
+ host: *mut core::ffi::c_void, // x2
+ top_out: *mut *mut u8, // x3
+) -> ! {
+ naked_asm!(
+ "
+ .seh_proc {start}
+ stp x29, x30, [sp, -16]!
+ .seh_save_fplr_x 16
+ mov x29, sp
+ .seh_set_fp
+ .seh_endprologue
+
+ // The frame pointer is `top`.
+ str x2, [x29, -0x08]
+ str x29, [x3]
+ mov x19, x0
+ mov x20, x1
+
+ // Move the stack pointer below the reserved slots and the canonical
+ // frame, touching the page in between first so that Windows grows the
+ // stack one page at a time.
+ sub x9, x29, 0x1000
+ ldr xzr, [x9]
+ sub sp, x9, 0x10
+
+ // The initial suspension.
+ mov x0, x29
+ bl {suspend}
+
+ // Resumed for the first time, or from a snapshot of the initial
+ // suspension. Call the entry point, which never returns.
+ mov x0, x20
+ mov x1, x29
+ blr x19
+ brk 0xf1b3
+ .seh_endproc
+ ",
+ start = sym wasmtime_fiber_start,
+ suspend = sym wasmtime_fiber_suspend,
+ );
+}
diff --git a/crates/fiber/src/raw/windows/x86_64.rs b/crates/fiber/src/raw/windows/x86_64.rs
new file mode 100644
index 0000000000..3028a99428
--- /dev/null
+++ b/crates/fiber/src/raw/windows/x86_64.rs
@@ -0,0 +1,159 @@
+// A WORD OF CAUTION
+//
+// This file needs to be kept in sync with itself and with the layout
+// documented in `raw/windows.rs`, and `SUSPENDED_FRAME` with the frame that
+// `wasmtime_fiber_suspend` spills.
+
+use core::arch::naked_asm;
+use windows_sys::Win32::System::Threading::SwitchToFiber;
+
+pub const SUPPORTED_ARCH: bool = true;
+
+/// The indices, in words from a suspended fiber's saved stack pointer, of the
+/// frame pointer and return address that `wasmtime_fiber_suspend` spills.
+pub const SUSPENDED_FRAME: Option<(usize, usize)> = Some((28, 29));
+
+/// Suspends the raw fiber whose stack top is `top`, returning when it is next
+/// resumed.
+#[unsafe(naked)]
+pub unsafe extern "C" fn wasmtime_fiber_suspend(top: *mut u8 /* rcx */) {
+ naked_asm!(
+ "
+ .seh_proc {suspend}
+
+ // Spill all nonvolatile registers.
+ push rbp
+ .seh_pushreg rbp
+ push rbx
+ .seh_pushreg rbx
+ push rdi
+ .seh_pushreg rdi
+ push rsi
+ .seh_pushreg rsi
+ push r12
+ .seh_pushreg r12
+ push r13
+ .seh_pushreg r13
+ push r14
+ .seh_pushreg r14
+ push r15
+ .seh_pushreg r15
+ sub rsp, 0xa8
+ .seh_stackalloc 0xa8
+ movaps [rsp + 0x00], xmm6
+ .seh_savexmm xmm6, 0x00
+ movaps [rsp + 0x10], xmm7
+ .seh_savexmm xmm7, 0x10
+ movaps [rsp + 0x20], xmm8
+ .seh_savexmm xmm8, 0x20
+ movaps [rsp + 0x30], xmm9
+ .seh_savexmm xmm9, 0x30
+ movaps [rsp + 0x40], xmm10
+ .seh_savexmm xmm10, 0x40
+ movaps [rsp + 0x50], xmm11
+ .seh_savexmm xmm11, 0x50
+ movaps [rsp + 0x60], xmm12
+ .seh_savexmm xmm12, 0x60
+ movaps [rsp + 0x70], xmm13
+ .seh_savexmm xmm13, 0x70
+ movaps [rsp + 0x80], xmm14
+ .seh_savexmm xmm14, 0x80
+ movaps [rsp + 0x90], xmm15
+ .seh_savexmm xmm15, 0x90
+ // Unwind through the frame pointer while on the canonical frame.
+ mov rbp, rsp
+ .seh_setframe rbp, 0
+ .seh_endprologue
+
+ // Store where to resume, then switch to the host fiber from the
+ // canonical frame, leaving shadow space above it.
+ mov -0x10[rcx], rsp
+ mov rax, rcx
+ mov rcx, -0x08[rax]
+ lea rsp, -0x30[rax]
+ call {switch_to_fiber}
+
+ // Resumed. Only the stack pointer is meaningful here: Windows restored
+ // the other registers from whichever suspension last saved them, which
+ // need not be the one resumed after a snapshot was restored. Take the
+ // registers spilled above from the stack pointer in the slot.
+ mov rbp, 0x20[rsp]
+ movaps xmm6, [rbp + 0x00]
+ movaps xmm7, [rbp + 0x10]
+ movaps xmm8, [rbp + 0x20]
+ movaps xmm9, [rbp + 0x30]
+ movaps xmm10, [rbp + 0x40]
+ movaps xmm11, [rbp + 0x50]
+ movaps xmm12, [rbp + 0x60]
+ movaps xmm13, [rbp + 0x70]
+ movaps xmm14, [rbp + 0x80]
+ movaps xmm15, [rbp + 0x90]
+ lea rsp, [rbp + 0xa8]
+ pop r15
+ pop r14
+ pop r13
+ pop r12
+ pop rsi
+ pop rdi
+ pop rbx
+ pop rbp
+ ret
+ .seh_endproc
+ ",
+ suspend = sym wasmtime_fiber_suspend,
+ switch_to_fiber = sym SwitchToFiber,
+ );
+}
+
+/// Sets up the top of the stack as documented in `raw/windows.rs`, reports `top`
+/// through `top_out`, suspends, and then calls `entry(arg, top)` when first
+/// resumed.
+///
+/// This never returns, so it does not preserve its caller's nonvolatile
+/// registers; the unwind information only describes how to find the caller's
+/// frame.
+#[unsafe(naked)]
+pub unsafe extern "C" fn wasmtime_fiber_start(
+ entry: crate::RawFiberEntry, // rcx
+ arg: *mut u8, // rdx
+ host: *mut core::ffi::c_void, // r8
+ top_out: *mut *mut u8, // r9
+) -> ! {
+ naked_asm!(
+ "
+ .seh_proc {start}
+ push rbp
+ .seh_pushreg rbp
+ mov rbp, rsp
+ .seh_setframe rbp, 0
+ .seh_endprologue
+
+ // The frame pointer is `top`, which is 16-byte aligned since the
+ // stack was 8 bytes off alignment at entry.
+ mov -0x08[rbp], r8
+ mov [r9], rbp
+ mov r12, rcx
+ mov r13, rdx
+
+ // Move the stack pointer below the reserved slots and the canonical
+ // frame, touching the page in between first so that Windows grows the
+ // stack one page at a time.
+ mov rax, -0x1000[rbp]
+ lea rsp, -0x1020[rbp]
+
+ // The initial suspension.
+ mov rcx, rbp
+ call {suspend}
+
+ // Resumed for the first time, or from a snapshot of the initial
+ // suspension. Call the entry point, which never returns.
+ mov rcx, r13
+ mov rdx, rbp
+ call r12
+ ud2
+ .seh_endproc
+ ",
+ start = sym wasmtime_fiber_start,
+ suspend = sym wasmtime_fiber_suspend,
+ );
+}
diff --git a/crates/fiber/src/stackswitch.rs b/crates/fiber/src/stackswitch.rs
index 224fac18df..b117ecd694 100644
--- a/crates/fiber/src/stackswitch.rs
+++ b/crates/fiber/src/stackswitch.rs
@@ -5,6 +5,11 @@
// accessed via the `extern "C"` declarations below that.
cfg_select! {
+ // Miri uses another fiber implementation, but raw fibers still refer to
+ // the routines here.
+ all(feature = "std", miri) => {
+ pub(crate) use unsupported::*;
+ }
target_arch = "aarch64" => {
mod aarch64;
pub(crate) use supported::*;
@@ -47,7 +52,6 @@ cfg_select! {
}
feature = "custom" => {
mod custom;
- pub(crate) use supported::*;
pub(crate) use custom::*;
}
_ => {
@@ -68,6 +72,10 @@ cfg_select! {
)]
mod supported {
pub const SUPPORTED_ARCH: bool = true;
+
+ /// Whether raw fibers can use the routines here. They require the
+ /// reserved-slot stack layout documented in `unix.rs`.
+ pub const RAW_FIBERS: bool = true;
}
/// Helper module reexported in the fallback case above when the current host
@@ -79,6 +87,7 @@ mod supported {
)]
mod unsupported {
pub const SUPPORTED_ARCH: bool = false;
+ pub const RAW_FIBERS: bool = false;
pub(crate) unsafe fn wasmtime_fiber_init(
_top_of_stack: *mut u8,
@@ -91,4 +100,8 @@ mod unsupported {
pub(crate) unsafe fn wasmtime_fiber_switch(_top_of_stack: *mut u8) {
unreachable!();
}
+
+ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(_top_of_stack: *mut u8) {
+ unreachable!();
+ }
}
diff --git a/crates/fiber/src/stackswitch/aarch64.rs b/crates/fiber/src/stackswitch/aarch64.rs
index 85f4a8b517..9ffdd06f3e 100644
--- a/crates/fiber/src/stackswitch/aarch64.rs
+++ b/crates/fiber/src/stackswitch/aarch64.rs
@@ -16,7 +16,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
}
#[unsafe(naked)]
-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) {
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) {
naked_asm!(
"
.cfi_startproc
diff --git a/crates/fiber/src/stackswitch/arm.rs b/crates/fiber/src/stackswitch/arm.rs
index d145178a73..b462d8530b 100644
--- a/crates/fiber/src/stackswitch/arm.rs
+++ b/crates/fiber/src/stackswitch/arm.rs
@@ -16,7 +16,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
}
#[unsafe(naked)]
-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* r0 */) {
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* r0 */) {
naked_asm!(
"
// Save callee-saved registers
diff --git a/crates/fiber/src/stackswitch/custom.rs b/crates/fiber/src/stackswitch/custom.rs
index faf0d0aad0..21f2e45c1c 100644
--- a/crates/fiber/src/stackswitch/custom.rs
+++ b/crates/fiber/src/stackswitch/custom.rs
@@ -20,3 +20,13 @@ unsafe extern "C" {
pub(crate) fn wasmtime_fiber_switch(top_of_stack: *mut u8);
}
+
+pub(crate) const SUPPORTED_ARCH: bool = true;
+
+// The embedder's routines need not use the stack layout that raw fiber
+// snapshots depend on.
+pub(crate) const RAW_FIBERS: bool = false;
+
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(_top_of_stack: *mut u8) {
+ unreachable!()
+}
diff --git a/crates/fiber/src/stackswitch/loongarch64.rs b/crates/fiber/src/stackswitch/loongarch64.rs
index 9493f77f5d..2bb11dc725 100644
--- a/crates/fiber/src/stackswitch/loongarch64.rs
+++ b/crates/fiber/src/stackswitch/loongarch64.rs
@@ -13,7 +13,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
}
#[unsafe(naked)]
-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {
naked_asm!(
"
// Save all callee-saved registers on the stack since we're
diff --git a/crates/fiber/src/stackswitch/riscv32imac.rs b/crates/fiber/src/stackswitch/riscv32imac.rs
index 78cc90fba9..55a3445d3f 100644
--- a/crates/fiber/src/stackswitch/riscv32imac.rs
+++ b/crates/fiber/src/stackswitch/riscv32imac.rs
@@ -16,7 +16,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
}
#[unsafe(naked)]
-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {
naked_asm!(
"
// See https://github.com/rust-lang/rust/issues/80608.
diff --git a/crates/fiber/src/stackswitch/riscv64.rs b/crates/fiber/src/stackswitch/riscv64.rs
index b55ac71cc7..e05f0e3062 100644
--- a/crates/fiber/src/stackswitch/riscv64.rs
+++ b/crates/fiber/src/stackswitch/riscv64.rs
@@ -13,7 +13,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
}
#[unsafe(naked)]
-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {
naked_asm!(
"
// See https://github.com/rust-lang/rust/issues/80608.
diff --git a/crates/fiber/src/stackswitch/s390x.rs b/crates/fiber/src/stackswitch/s390x.rs
index 52bbdbd2cc..e657989966 100644
--- a/crates/fiber/src/stackswitch/s390x.rs
+++ b/crates/fiber/src/stackswitch/s390x.rs
@@ -13,7 +13,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
}
#[unsafe(naked)]
-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) {
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) {
naked_asm!(
"
// Save all callee-saved registers on the stack since we're assuming
diff --git a/crates/fiber/src/stackswitch/x86.rs b/crates/fiber/src/stackswitch/x86.rs
index 1a09240dd4..5f32bacee9 100644
--- a/crates/fiber/src/stackswitch/x86.rs
+++ b/crates/fiber/src/stackswitch/x86.rs
@@ -18,7 +18,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
}
#[unsafe(naked)]
-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8) {
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8) {
naked_asm!(
"
// Load our stack-to-use
diff --git a/crates/fiber/src/stackswitch/x86_64.rs b/crates/fiber/src/stackswitch/x86_64.rs
index 5cea3ac715..e0536dbe62 100644
--- a/crates/fiber/src/stackswitch/x86_64.rs
+++ b/crates/fiber/src/stackswitch/x86_64.rs
@@ -13,7 +13,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {
}
#[unsafe(naked)]
-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* rdi */) {
+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* rdi */) {
naked_asm!(
"
// We're switching to arbitrary code somewhere else, so pessimistically
diff --git a/crates/fiber/src/windows.rs b/crates/fiber/src/windows.rs
index f9bc319cc5..f8f303d0f3 100644
--- a/crates/fiber/src/windows.rs
+++ b/crates/fiber/src/windows.rs
@@ -13,14 +13,22 @@ use windows_sys::Win32::System::Threading::*;
pub type Error = io::Error;
#[derive(Debug)]
-pub struct FiberStack(usize);
+pub struct FiberStack {
+ pub(crate) size: usize,
+ // Windows allocates a fiber's stack when creating the fiber, so its
+ // location is only known while a raw fiber owns it. See `raw/windows.rs`.
+ pub(crate) raw_range: Option<Range<usize>>,
+}
impl FiberStack {
pub fn new(size: usize, zeroed: bool) -> io::Result<Self> {
// We don't support fiber stack zeroing on windows.
let _ = zeroed;
- Ok(Self(size))
+ Ok(Self {
+ size,
+ raw_range: None,
+ })
}
pub unsafe fn from_raw_parts(
@@ -40,11 +48,11 @@ impl FiberStack {
}
pub fn top(&self) -> Option<*mut u8> {
- None
+ Some(self.raw_range.as_ref()?.end as *mut u8)
}
pub fn range(&self) -> Option<Range<usize>> {
- None
+ self.raw_range.clone()
}
pub fn guard_range(&self) -> Option<Range<*mut u8>> {
@@ -67,17 +75,15 @@ struct StartState {
result_location: Cell<*const u8>,
}
-const FIBER_FLAG_FLOAT_SWITCH: u32 = 1;
+pub(crate) const FIBER_FLAG_FLOAT_SWITCH: u32 = 1;
unsafe extern "C" {
#[wasmtime_versioned_export_macros::versioned_link]
fn wasmtime_fiber_get_current() -> *mut c_void;
}
-unsafe extern "system" fn fiber_start<F, A, B, C>(data: *mut c_void)
-where
- F: FnOnce(A, &mut super::Suspend<A, B, C>) -> C,
-{
+/// Sets the current fiber's stack guarantee.
+pub(crate) unsafe fn set_stack_guarantee() {
unsafe {
// Set the stack guarantee to be consistent with what Rust expects for threads
// This value is taken from:
@@ -85,6 +91,71 @@ where
if SetThreadStackGuarantee(&mut 0x5000) == 0 {
panic!("failed to set fiber stack guarantee");
}
+ }
+}
+
+/// Calls `f` with the current fiber, first converting the current thread into
+/// a fiber, and back again afterwards, if it is not one already.
+pub(crate) unsafe fn with_current_fiber(f: impl FnOnce(*mut c_void)) {
+ unsafe {
+ let is_fiber = IsThreadAFiber() != 0;
+ let parent_fiber = if is_fiber {
+ wasmtime_fiber_get_current()
+ } else {
+ // Newer Rust versions use fiber local storage to register an internal hook that
+ // calls thread locals' destructors on thread exit.
+ // This has a limitation: the hook only runs in a regular thread (not in a fiber).
+ // We convert back into a thread once execution returns to this function,
+ // but we must also ensure that the hook is registered before converting into a fiber.
+ // Otherwise, a different fiber could be the first to register the hook,
+ // causing the hook to be called (and skipped) prematurely when that fiber is deleted.
+ struct Guard;
+
+ impl Drop for Guard {
+ fn drop(&mut self) {}
+ }
+ assert!(needs_drop::<Guard>());
+ thread_local!(static GUARD: Guard = Guard);
+ GUARD.with(|_g| {});
+ ConvertThreadToFiber(ptr::null_mut())
+ };
+ assert!(
+ !parent_fiber.is_null(),
+ "failed to make current thread a fiber"
+ );
+ f(parent_fiber);
+ if !is_fiber {
+ let res = ConvertFiberToThread();
+ assert!(res != 0, "failed to convert main thread back");
+ }
+ }
+}
+
+/// Deletes a fiber which is not running.
+pub(crate) unsafe fn delete_fiber(fiber: *mut c_void) {
+ unsafe {
+ let is_fiber = IsThreadAFiber() != 0;
+ if !is_fiber {
+ // DeleteFiber runs FLS destructors. Make those destructors
+ // observe a fiber so Rust does not run thread-local cleanup
+ // for the live thread.
+ let fiber = ConvertThreadToFiber(ptr::null_mut());
+ assert!(!fiber.is_null(), "failed to make current thread a fiber");
+ }
+ DeleteFiber(fiber);
+ if !is_fiber {
+ let res = ConvertFiberToThread();
+ assert!(res != 0, "failed to convert main thread back");
+ }
+ }
+}
+
+unsafe extern "system" fn fiber_start<F, A, B, C>(data: *mut c_void)
+where
+ F: FnOnce(A, &mut super::Suspend<A, B, C>) -> C,
+{
+ unsafe {
+ set_stack_guarantee();
let state = data.cast::<StartState>();
let func = Box::from_raw((*state).initial_closure.get().cast::<F>());
@@ -116,7 +187,7 @@ impl Fiber {
let fiber = CreateFiberEx(
0,
- stack.0,
+ stack.size,
FIBER_FLAG_FLOAT_SWITCH,
Some(fiber_start::<F, A, B, C>),
&*state as *const StartState as *mut _,
@@ -133,42 +204,15 @@ impl Fiber {
pub(crate) fn resume<A, B, C>(&self, _stack: &FiberStack, result: &Cell<RunResult<A, B, C>>) {
unsafe {
- let is_fiber = IsThreadAFiber() != 0;
- let parent_fiber = if is_fiber {
- wasmtime_fiber_get_current()
- } else {
- // Newer Rust versions use fiber local storage to register an internal hook that
- // calls thread locals' destructors on thread exit.
- // This has a limitation: the hook only runs in a regular thread (not in a fiber).
- // We convert back into a thread once execution returns to this function,
- // but we must also ensure that the hook is registered before converting into a fiber.
- // Otherwise, a different fiber could be the first to register the hook,
- // causing the hook to be called (and skipped) prematurely when that fiber is deleted.
- struct Guard;
-
- impl Drop for Guard {
- fn drop(&mut self) {}
- }
- assert!(needs_drop::<Guard>());
- thread_local!(static GUARD: Guard = Guard);
- GUARD.with(|_g| {});
- ConvertThreadToFiber(ptr::null_mut())
- };
- assert!(
- !parent_fiber.is_null(),
- "failed to make current thread a fiber"
- );
- self.state
- .result_location
- .set(result as *const _ as *const _);
- self.state.parent.set(parent_fiber);
- SwitchToFiber(self.fiber);
- self.state.parent.set(ptr::null_mut());
- self.state.result_location.set(ptr::null());
- if !is_fiber {
- let res = ConvertFiberToThread();
- assert!(res != 0, "failed to convert main thread back");
- }
+ with_current_fiber(|parent_fiber| {
+ self.state
+ .result_location
+ .set(result as *const _ as *const _);
+ self.state.parent.set(parent_fiber);
+ SwitchToFiber(self.fiber);
+ self.state.parent.set(ptr::null_mut());
+ self.state.result_location.set(ptr::null());
+ });
}
}
@@ -178,19 +222,7 @@ impl Fiber {
impl Drop for Fiber {
fn drop(&mut self) {
unsafe {
- let is_fiber = IsThreadAFiber() != 0;
- if !is_fiber {
- // DeleteFiber runs FLS destructors. Make those destructors
- // observe a fiber so Rust does not run thread-local cleanup
- // for the live thread.
- let fiber = ConvertThreadToFiber(ptr::null_mut());
- assert!(!fiber.is_null(), "failed to make current thread a fiber");
- }
- DeleteFiber(self.fiber);
- if !is_fiber {
- let res = ConvertFiberToThread();
- assert!(res != 0, "failed to convert main thread back");
- }
+ delete_fiber(self.fiber);
}
}
}