A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
wasmtime-fiber: add raw fibers with snapshots. (#15187)
Chris Fallinb95f045e4eb8
diff --git a/crates/core/src/alloc/vec.rs b/crates/core/src/alloc/vec.rsindex 1c65c4c2cd..4d4319bbd1 100644--- a/crates/core/src/alloc/vec.rs+++ b/crates/core/src/alloc/vec.rs@@ -158,6 +158,20 @@ impl<T> TryVec<T> {Ok(())}+ /// Same as [`std::vec::Vec::extend_from_slice`] but returns an error on+ /// allocation failure, in which case `self` is unchanged.+ ///+ /// This is limited to `Copy` elements, which can be copied without+ /// allocating, so that it is a single copy of the slice.+ pub fn extend_from_slice(&mut self, other: &[T]) -> Result<(), OutOfMemory>+ where+ T: Copy,+ {+ self.reserve(other.len())?;+ self.inner.extend_from_slice(other);+ Ok(())+ }+/// Same as [`std::vec::Vec::pop`].pub fn pop(&mut self) -> Option<T> {self.inner.pop()@@ -483,6 +497,17 @@ mod tests {use super::TryVec;use crate::error::OutOfMemory;+ #[test]+ fn test_extend_from_slice() -> Result<(), OutOfMemory> {+ let mut vec = TryVec::new();+ vec.extend_from_slice(&[])?;+ assert!(vec.is_empty());+ vec.extend_from_slice(&[1, 2])?;+ vec.extend_from_slice(&[3])?;+ assert_eq!(*vec, [1, 2, 3]);+ Ok(())+ }+#[test]fn test_into_boxed_slice() -> Result<(), OutOfMemory> {assert_eq!(*TryVec::<i32>::new().into_boxed_slice()?, []);diff --git a/crates/fiber/src/lib.rs b/crates/fiber/src/lib.rsindex 1b7e753a99..df683c1994 100644--- a/crates/fiber/src/lib.rs+++ b/crates/fiber/src/lib.rs@@ -18,6 +18,9 @@ use core::marker::PhantomData;use core::ops::Range;use wasmtime_environ::error::Error;+mod raw;+pub use raw::{RawFiber, RawFiberEntry, RawFiberSnapshot, RawFiberState};+cfg_select! {not(feature = "std") => {mod nostd;@@ -27,6 +30,8 @@ cfg_select! {miri => {mod miri;use miri as imp;+ // Unused by Miri's fibers, but raw fibers refer to it.+ mod stackswitch;}windows => {mod windows;diff --git a/crates/fiber/src/raw.rs b/crates/fiber/src/raw.rsnew file mode 100644index 0000000000..6ce8c2af72--- /dev/null+++ b/crates/fiber/src/raw.rs@@ -0,0 +1,588 @@+//! Raw fibers: fibers whose entry point and suspension points are machine code+//! supplied by the caller rather than a Rust closure.+//!+//! See [`RawFiber`] for what raw fibers can do and the contract their code+//! must uphold.++use crate::FiberStack;+use core::sync::atomic::Ordering;+use wasmtime_environ::collections::TryVec;+use wasmtime_environ::error::{Error, Result, bail, ensure};++cfg_select! {+ all(windows, feature = "std", not(miri)) => {+ mod windows;+ use windows as sys;+ }+ _ => {+ mod stackswitch;+ use stackswitch as sys;+ }+}++/// The entry point of a raw fiber.+///+/// This is called on the fiber's stack with the `arg` given to+/// [`RawFiber::new`] and the fiber's [`RawFiber::switch_arg`]. It must never+/// return.+pub type RawFiberEntry = unsafe extern "C" fn(arg: *mut u8, switch_arg: *mut u8);++/// The lifecycle of a [`RawFiber`].+///+/// A raw fiber is only ever observed while stopped; it runs only for the+/// duration of a call to [`RawFiber::resume`].+#[derive(Copy, Clone, Debug, PartialEq, Eq)]+pub enum RawFiberState {+ /// The fiber has not yet been resumed; resuming it calls its entry point.+ Initial,+ /// The fiber suspended itself and may be resumed.+ Suspended,+ /// The fiber suspended for the last time, as recorded by+ /// [`RawFiber::finish`]. It has no continuation, so it may not be resumed+ /// or snapshotted unless an earlier snapshot is restored first.+ Terminal,+}++/// A fiber running caller-supplied machine code.+///+/// Raw fibers provide one key capability that ordinary fibers, which+/// wrap a Rust closure, do not: the ability to snapshot and+/// resume. (This capability is useful for Wasmtime features like+/// record/replay with reversible execution.)+///+/// A raw fiber requires an entry point that is an `extern "C"`+/// function with a particular signature (two opaque pointer args),+/// and that *never returns*. Unlike ordinary fibers, this entry-point+/// function is the first and only frame on the fiber stack when it+/// starts executing.+///+/// A raw fiber's state -- stack and SP -- can be snapshotted and+/// restored, potentially multiple times ("multi-shot+/// continuations"). The safety of this is fairly subtle and has the+/// following requirements:+///+/// - No Rust-compiled frame must exist anywhere on the fiber stack+/// (because multiple returns from a function call in Rust are+/// UB). This implies, in practice, either handwritten assembly or+/// machine code that the user otherwise completely controls+/// (e.g. Cranelift-compiled code in Wasmtime).+/// - Any live pointers that may be in registers or on the stack+/// (i.e., live anywhere in the executing code) continue to be live+/// when execution is rewound to an earlier snapshot. In practice,+/// it is safest to have "completely invariant pointers": e.g., VM+/// data structures or memories/heaps that stay at the same address+/// for the lifetime of a fiber.+/// - The raw fiber entry point must never return. Instead, it can implement+/// termination by yielding one final time, having set some state so that+/// the rest of the runtime will then know to drop the raw fiber (thus freeing+/// its stack).+pub struct RawFiber {+ stack: FiberStack,+ state: RawFiberState,+ id: RawFiberId,+ sys: sys::Fiber,+}++/// A copy of a stopped [`RawFiber`]'s stack and lifecycle state.+pub struct RawFiberSnapshot {+ fiber: RawFiberId,+ sp: usize,+ bytes: TryVec<u8>,+ state: RawFiberState,+}++/// A process-unique identifier of a [`RawFiber`], by which snapshots are+/// matched to the fiber they were taken from.+///+/// IDs must never be reused, since a reused ID would let a snapshot be+/// restored onto a different fiber. Where 64-bit atomics are available, IDs+/// are 64 bits wide, so they never run out in practice; elsewhere, allocating+/// an ID panics once a 32-bit counter is exhausted rather than wrapping.+#[derive(Copy, Clone, Debug, PartialEq, Eq)]+struct RawFiberId(u64);++impl RawFiberId {+ fn new() -> Self {+ cfg_select! {+ target_has_atomic = "64" => {+ use core::sync::atomic::AtomicU64;+ static NEXT_ID: AtomicU64 = AtomicU64::new(0);+ RawFiberId(NEXT_ID.fetch_add(1, Ordering::Relaxed))+ }+ _ => {+ use core::sync::atomic::AtomicU32;+ static NEXT_ID: AtomicU32 = AtomicU32::new(0);+ let mut id = NEXT_ID.load(Ordering::Relaxed);+ loop {+ let next = id.checked_add(1).expect("raw fiber IDs exhausted");+ match NEXT_ID.compare_exchange_weak(+ id,+ next,+ Ordering::Relaxed,+ Ordering::Relaxed,+ ) {+ Ok(_) => break RawFiberId(u64::from(id)),+ Err(actual) => id = actual,+ }+ }+ }+ }+ }+}++impl RawFiber {+ /// Whether raw fibers are supported on this platform and configuration.+ ///+ /// Raw fibers require one of the stack-switching routines in this crate+ /// (not an embedder's `custom` one), or Windows fibers on x86_64 or+ /// aarch64. They are not supported under Miri or with AddressSanitizer,+ /// whose fiber-switch handshakes would require Rust code at every+ /// suspension point, nor with hardware-enforced shadow stacks, which+ /// snapshots do not restore.+ pub fn is_supported() -> bool {+ sys::supported() && !cfg!(asan)+ }++ /// Creates a new raw fiber that calls `entry(arg, switch_arg)` when first+ /// resumed.+ ///+ /// On error the stack is handed back to the caller.+ ///+ /// # Safety+ ///+ /// `entry` must uphold the contract of [`RawFiberEntry`] and suspend only+ /// through [`RawFiber::switch_routine`], passing [`RawFiber::switch_arg`].+ /// `arg` must remain valid for as long as the fiber may run.+ pub unsafe fn new(+ mut stack: FiberStack,+ entry: RawFiberEntry,+ arg: *mut u8,+ ) -> Result<Self, (Error, FiberStack)> {+ if !Self::is_supported() {+ return Err((+ wasmtime_environ::error::format_err!(+ "raw fibers are not supported on this platform"+ ),+ stack,+ ));+ }+ // SAFETY: the stack is owned by this fiber, and `entry` and `arg`+ // are valid per this function's contract.+ let sys = match unsafe { sys::Fiber::new(&mut stack, entry, arg) } {+ Ok(sys) => sys,+ Err(e) => return Err((e, stack)),+ };+ Ok(RawFiber {+ stack,+ state: RawFiberState::Initial,+ id: RawFiberId::new(),+ sys,+ })+ }++ /// The assembly routine which code on a raw fiber calls, with the+ /// C calling convention and [`RawFiber::switch_arg`] as its only+ /// argument, to suspend itself. It returns when the fiber is next+ /// resumed.+ pub fn switch_routine() -> unsafe extern "C" fn(*mut u8) {+ sys::switch_routine()+ }++ /// The argument which code on this fiber passes to+ /// [`RawFiber::switch_routine`]. This is the top of the fiber's stack as+ /// reported by [`FiberStack::top`].+ pub fn switch_arg(&self) -> *mut u8 {+ self.stack.top().unwrap()+ }++ /// The current lifecycle state of this fiber.+ pub fn state(&self) -> RawFiberState {+ self.state+ }++ /// The stack this fiber runs on.+ pub fn stack(&self) -> &FiberStack {+ &self.stack+ }++ /// Destroys this fiber, returning its stack. Nothing on the fiber's stack+ /// is run or unwound.+ pub fn into_stack(self) -> FiberStack {+ let mut stack = self.stack;+ self.sys.destroy(&mut stack);+ stack+ }++ /// Runs this fiber until it next suspends itself.+ ///+ /// Returns an error, without running anything, if the fiber is terminal.+ ///+ /// # Safety+ ///+ /// Everything the fiber's code requires must be valid for the duration of+ /// the call. In particular after a [`RawFiber::restore`], every pointer the+ /// restored continuation can dereference must refer to live storage.+ pub unsafe fn resume(&mut self) -> Result<()> {+ ensure!(+ self.state != RawFiberState::Terminal,+ "cannot resume a terminal raw fiber"+ );+ let top = self.switch_arg();+ // SAFETY: the fiber is stopped and everything it requires is valid+ // per this function's contract.+ unsafe { self.sys.resume(top) };+ self.state = RawFiberState::Suspended;+ Ok(())+ }++ /// Records that this fiber's most recent suspension was its last one.+ ///+ /// The fiber's owner learns this through its own protocol with the code+ /// running on the fiber.+ pub fn finish(&mut self) -> Result<()> {+ ensure!(+ self.state == RawFiberState::Suspended,+ "only a suspended raw fiber can finish"+ );+ self.state = RawFiberState::Terminal;+ Ok(())+ }++ fn saved_sp_slot(&self) -> *mut usize {+ // See the layout diagrams in `unix.rs` and `raw/windows.rs`: the+ // second slot below the top holds the stack pointer to resume the+ // fiber at.+ self.switch_arg().cast::<usize>().wrapping_sub(2)+ }++ /// The stack pointer at which this fiber will resume.+ ///+ /// # Panics+ ///+ /// Panics if the fiber is terminal: it will not resume, so it has no+ /// continuation to inspect.+ fn saved_sp(&self) -> usize {+ assert_ne!(+ self.state,+ RawFiberState::Terminal,+ "a terminal raw fiber has no saved stack pointer"+ );+ // SAFETY: the reserved slot is initialized when the fiber is created+ // and maintained by the switch routine.+ unsafe { self.saved_sp_slot().read() }+ }++ /// Copies this stopped fiber's stack and lifecycle state.+ ///+ /// Returns an error if the fiber is terminal: it has no continuation to+ /// copy.+ pub fn snapshot(&self) -> Result<RawFiberSnapshot> {+ ensure!(+ self.state != RawFiberState::Terminal,+ "cannot snapshot a terminal raw fiber"+ );+ let top = self.switch_arg() as usize;+ let sp = self.saved_sp();+ let range = self.stack.range();+ ensure!(+ range.is_none_or(|r| r.start <= sp) && sp < top,+ "raw fiber has an invalid saved stack pointer"+ );+ let len = top - sp;+ let mut bytes = TryVec::new();+ // SAFETY: `sp..top` is within this fiber's stack, which is not+ // running because `self` is borrowed here.+ bytes.extend_from_slice(unsafe { core::slice::from_raw_parts(sp as *const u8, len) })?;+ Ok(RawFiberSnapshot {+ fiber: self.id,+ sp,+ bytes,+ state: self.state,+ })+ }++ /// Restores a snapshot previously taken from this fiber, replacing its+ /// current continuation and lifecycle state. The next [`RawFiber::resume`]+ /// continues from the snapshot's suspension point (or the entry point, for+ /// a snapshot of an initial fiber), returning to that resume's caller.+ ///+ /// Snapshots taken from a different fiber are rejected without modifying+ /// this fiber.+ ///+ /// # Safety+ ///+ /// The snapshot must have been taken at a point eligible for restoration,+ /// as described in the [`RawFiber`] documentation.+ pub unsafe fn restore(&mut self, snapshot: &RawFiberSnapshot) -> Result<()> {+ if snapshot.fiber != self.id {+ bail!("raw fiber snapshot belongs to a different fiber");+ }+ // SAFETY: the snapshot came from this fiber's stack, so the range is+ // in bounds. It includes the saved stack pointer slot, which this+ // overwrites with `snapshot.sp`, and the stack is not running.+ unsafe {+ core::ptr::copy_nonoverlapping(+ snapshot.bytes.as_ptr(),+ snapshot.sp as *mut u8,+ snapshot.bytes.len(),+ );+ }+ self.state = snapshot.state;+ debug_assert_eq!(self.saved_sp(), snapshot.sp);+ Ok(())+ }++ /// The return address and frame pointer of the code that called the+ /// switch routine, for walking a suspended fiber's frames.+ ///+ /// Returns `None` if the fiber has not suspended or if this is not+ /// implemented for the current architecture.+ ///+ /// # Panics+ ///+ /// Panics if the fiber is terminal: it will not resume, so it has no+ /// continuation to inspect.+ pub fn suspended_frame(&self) -> Option<(usize, usize)> {+ assert_ne!(+ self.state,+ RawFiberState::Terminal,+ "a terminal raw fiber has no suspended frame"+ );+ let (fp, pc) = sys::SUSPENDED_FRAME?;+ if self.state == RawFiberState::Initial {+ return None;+ }+ let sp = self.saved_sp() as *const usize;+ // SAFETY: a suspended fiber's saved stack pointer addresses the+ // registers spilled by the switch routine.+ unsafe { Some((sp.add(pc).read(), sp.add(fp).read())) }+ }+}++impl RawFiberSnapshot {+ /// The lifecycle state the fiber will have once this is restored.+ pub fn state(&self) -> RawFiberState {+ self.state+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use core::cell::Cell;++ /// State shared between a test and the code on its raw fiber.+ #[derive(Default)]+ struct Shared {+ observed: Cell<u64>,+ finished: Cell<bool>,+ }++ /// Yields the values 1, 2, 3 and then finishes. The counter is local to+ /// the fiber, so a restored snapshot rewinds it.+ unsafe extern "C" fn count_to_three(arg: *mut u8, switch_arg: *mut u8) {+ // SAFETY: tests pass a `Shared` that outlives the fiber's use.+ let shared = unsafe { &*arg.cast::<Shared>() };+ let switch = RawFiber::switch_routine();+ let mut count = 0;+ while count < 3 {+ count += 1;+ shared.observed.set(count);+ // SAFETY: this is the switch routine for this fiber.+ unsafe { switch(switch_arg) };+ }+ shared.finished.set(true);+ // SAFETY: as above.+ unsafe { switch(switch_arg) };+ // Resuming a terminal fiber is rejected by `RawFiber::resume`.+ std::process::abort();+ }++ fn fiber(shared: &Shared) -> Option<RawFiber> {+ if !RawFiber::is_supported() {+ return None;+ }+ let stack = FiberStack::new(64 * 1024, false).unwrap();+ let arg = core::ptr::from_ref(shared).cast_mut().cast();+ Some(+ unsafe { RawFiber::new(stack, count_to_three, arg) }+ .map_err(|e| e.0)+ .unwrap(),+ )+ }++ /// Resumes `fiber` from a host stack frame `depth` calls deeper than the+ /// caller, so that each resume has a different host continuation.+ #[inline(never)]+ fn resume_at_depth(fiber: &mut RawFiber, depth: usize) {+ if depth == 0 {+ unsafe { fiber.resume().unwrap() };+ } else {+ resume_at_depth(fiber, depth - 1);+ core::hint::black_box(depth);+ }+ }++ #[test]+ fn lifecycle() {+ let shared = Shared::default();+ let Some(mut fiber) = fiber(&shared) else {+ return;+ };+ assert_eq!(fiber.state(), RawFiberState::Initial);+ for i in 1..=3 {+ resume_at_depth(&mut fiber, i as usize);+ assert_eq!(shared.observed.get(), i);+ assert_eq!(fiber.state(), RawFiberState::Suspended);+ }+ unsafe { fiber.resume().unwrap() };+ assert!(shared.finished.get());+ fiber.finish().unwrap();+ assert_eq!(fiber.state(), RawFiberState::Terminal);+ assert!(unsafe { fiber.resume() }.is_err());+ assert!(fiber.finish().is_err());+ assert!(fiber.snapshot().is_err());+ // A terminal fiber is destroyed without being resumed.+ drop(fiber.into_stack());+ }++ #[test]+ fn rewind() {+ let shared = Shared::default();+ let Some(mut fiber) = fiber(&shared) else {+ return;+ };+ let initial = fiber.snapshot().unwrap();+ unsafe { fiber.resume().unwrap() };+ let first = fiber.snapshot().unwrap();+ assert_eq!(first.state(), RawFiberState::Suspended);++ // Repeatedly rewind to the first yield, resuming from different host+ // frames each time.+ for depth in 0..3 {+ unsafe { fiber.resume().unwrap() };+ assert_eq!(shared.observed.get(), 2);+ unsafe { fiber.restore(&first).unwrap() };+ resume_at_depth(&mut fiber, depth);+ assert_eq!(shared.observed.get(), 2);+ unsafe { fiber.restore(&first).unwrap() };+ }++ // Run to completion, then rewind after the final yield.+ for _ in 0..3 {+ unsafe { fiber.resume().unwrap() };+ }+ assert!(shared.finished.get());+ fiber.finish().unwrap();+ unsafe { fiber.restore(&first).unwrap() };+ assert_eq!(fiber.state(), RawFiberState::Suspended);+ unsafe { fiber.resume().unwrap() };+ assert_eq!(shared.observed.get(), 2);++ // Restart from the very beginning.+ unsafe { fiber.restore(&initial).unwrap() };+ assert_eq!(fiber.state(), RawFiberState::Initial);+ unsafe { fiber.resume().unwrap() };+ assert_eq!(shared.observed.get(), 1);+ }++ #[test]+ fn snapshots_belong_to_one_fiber() {+ let a = Shared::default();+ let b = Shared::default();+ let (Some(mut fa), Some(mut fb)) = (fiber(&a), fiber(&b)) else {+ return;+ };+ unsafe { fa.resume().unwrap() };+ let snapshot = fa.snapshot().unwrap();+ assert!(unsafe { fb.restore(&snapshot) }.is_err());+ assert_eq!(fb.state(), RawFiberState::Initial);++ // Destroying a fiber leaves its outstanding snapshots inert, even if a+ // new fiber reuses the same stack memory.+ unsafe { fa.resume().unwrap() };+ let stack = fa.into_stack();+ let arg = core::ptr::from_ref(&a).cast_mut().cast();+ let mut fc = unsafe { RawFiber::new(stack, count_to_three, arg) }+ .map_err(|e| e.0)+ .unwrap();+ assert!(unsafe { fc.restore(&snapshot) }.is_err());+ unsafe { fc.resume().unwrap() };+ assert_eq!(a.observed.get(), 1);+ drop(snapshot);+ unsafe { fb.resume().unwrap() };+ assert_eq!(b.observed.get(), 1);+ }++ #[test]+ fn resume_on_another_thread() {+ let shared = std::sync::Arc::new(SharedSync::default());+ if !RawFiber::is_supported() {+ return;+ }+ unsafe extern "C" fn entry(arg: *mut u8, switch_arg: *mut u8) {+ let shared = unsafe { &*arg.cast::<SharedSync>() };+ let switch = RawFiber::switch_routine();+ loop {+ shared.0.fetch_add(1, core::sync::atomic::Ordering::Relaxed);+ unsafe { switch(switch_arg) };+ }+ }+ #[derive(Default)]+ struct SharedSync(core::sync::atomic::AtomicUsize);++ let stack = FiberStack::new(64 * 1024, false).unwrap();+ let arg = std::sync::Arc::as_ptr(&shared).cast_mut().cast();+ let mut fiber = unsafe { RawFiber::new(stack, entry, arg) }+ .map_err(|e| e.0)+ .unwrap();+ unsafe { fiber.resume().unwrap() };+ let snapshot = fiber.snapshot().unwrap();+ let mut fiber = std::thread::spawn(move || {+ unsafe { fiber.resume().unwrap() };+ fiber+ })+ .join()+ .unwrap();+ unsafe { fiber.restore(&snapshot).unwrap() };+ unsafe { fiber.resume().unwrap() };+ assert_eq!(shared.0.load(core::sync::atomic::Ordering::Relaxed), 3);+ }++ #[test]+ fn terminal_fiber_has_no_suspended_frame() {+ let shared = Shared::default();+ let Some(mut fiber) = fiber(&shared) else {+ return;+ };+ for _ in 0..4 {+ unsafe { fiber.resume().unwrap() };+ }+ fiber.finish().unwrap();+ let result = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| {+ fiber.suspended_frame();+ }));+ assert!(result.is_err());+ }++ #[test]+ fn suspended_frame_points_into_entry() {+ let shared = Shared::default();+ let Some(mut fiber) = fiber(&shared) else {+ return;+ };+ assert!(fiber.suspended_frame().is_none());+ unsafe { fiber.resume().unwrap() };+ if let Some((pc, fp)) = fiber.suspended_frame() {+ // Without frame pointers in this Rust entry, `fp` may still be+ // the initial one: the top of the stack.+ let range = fiber.stack().range().unwrap();+ assert!(+ range.start <= fp && fp <= range.end,+ "{fp:#x} vs {range:x?}"+ );+ let entry = count_to_three as *const () as usize;+ assert!(pc > entry && pc < entry + 4096, "{pc:#x} vs {entry:#x}");+ }+ }+}diff --git a/crates/fiber/src/raw/stackswitch.rs b/crates/fiber/src/raw/stackswitch.rsnew file mode 100644index 0000000000..45e98d4bcb--- /dev/null+++ b/crates/fiber/src/raw/stackswitch.rs@@ -0,0 +1,66 @@+//! Raw fibers on this crate's stack-switching routines, which keep the+//! suspended context on the fiber's stack in the reserved-slot layout+//! documented in `unix.rs`.++use super::RawFiberEntry;+use crate::stackswitch::{+ RAW_FIBERS, wasmtime_fiber_init, wasmtime_fiber_switch, wasmtime_fiber_switch_,+};+use wasmtime_environ::error::{Result, format_err};++pub fn supported() -> bool {+ RAW_FIBERS+}++pub fn switch_routine() -> unsafe extern "C" fn(*mut u8) {+ wasmtime_fiber_switch_+}++// Indices, in words from the saved stack pointer, of the frame+// pointer and return address that the switch routine spills. Keep+// in sync with `wasmtime_fiber_switch_` in `stackswitch/*.rs`.+pub const SUSPENDED_FRAME: Option<(usize, usize)> = if cfg!(target_arch = "x86_64") {+ Some((5, 6))+} else if cfg!(target_arch = "aarch64") {+ Some((18, 19))+} else {+ None+};++pub struct Fiber;++impl Fiber {+ pub unsafe fn new(+ stack: &mut crate::FiberStack,+ entry: RawFiberEntry,+ arg: *mut u8,+ ) -> Result<Self> {+ let top = stack+ .top()+ .ok_or_else(|| format_err!("raw fibers require a known stack top"))?;+ // SAFETY: the stack is owned by the raw fiber and large+ // enough for the initial frame, which the existing+ // closure-based fibers rely on too. The fiber-start+ // trampoline only forwards both arguments to `entry`.+ unsafe {+ wasmtime_fiber_init(+ top,+ core::mem::transmute::<RawFiberEntry, extern "C" fn(*mut u8, *mut u8) -> *mut u8>(+ entry,+ ),+ arg,+ );+ }+ Ok(Fiber)+ }++ pub unsafe fn resume(&mut self, top: *mut u8) {+ // SAFETY: the saved stack pointer of a stopped fiber is in+ // the reserved slot. The switch stores this host+ // continuation in the same slot, so the fiber always+ // suspends back to here.+ unsafe { wasmtime_fiber_switch(top) }+ }++ pub fn destroy(self, _stack: &mut crate::FiberStack) {}+}diff --git a/crates/fiber/src/raw/windows.rs b/crates/fiber/src/raw/windows.rsnew file mode 100644index 0000000000..32beeabd50--- /dev/null+++ b/crates/fiber/src/raw/windows.rs@@ -0,0 +1,196 @@+//! Raw fibers on top of Windows fibers.+//!+//! `SwitchToFiber` saves a fiber's stack pointer and nonvolatile registers in+//! the fiber's undocumented, heap-allocated `FIBER` structure rather than on+//! its stack, so the stack bytes alone do not capture where to resume a+//! suspended fiber. Instead every suspension goes through one canonical frame:+//! the suspend routine spills all nonvolatile registers onto the fiber's+//! stack, stores the stack pointer in a reserved slot, and calls+//! `SwitchToFiber` with a fixed stack pointer near the top of the stack. What+//! Windows saves is thus the same at every suspension, and when+//! `SwitchToFiber` returns the suspend routine takes everything else from the+//! slot and the stack. A raw fiber's snapshot therefore captures it completely,+//! just as on other platforms, while Windows keeps managing everything else+//! about the fiber: its stack bounds, fiber-local storage, and so on.+//!+//! The start routine suspends through the same frame once before calling the+//! entry point, so a new raw fiber is a suspended fiber whose continuation+//! calls its entry point.+//!+//! The top of a raw fiber's stack, where `top` is its `switch_arg`, is:+//!+//! ```text+//! frames of `fiber_start` and Windows' own fiber start (fixed)+//! top + 8 return address into `fiber_start`+//! top frame pointer of `fiber_start`+//! top - 0x08 host fiber to suspend to, stored by each resume+//! top - 0x10 stack pointer to resume this fiber at+//! frame from which the suspend routine calls `SwitchToFiber`+//! top - SCRATCH start of the entry point's stack+//! ```+//!+//! A raw fiber's `FiberStack` reports its range as the bottom of the stack's+//! reservation up to `top`, which is also what it reports as the stack's top.++use crate::RawFiberEntry;+use crate::windows::{+ FIBER_FLAG_FLOAT_SWITCH, delete_fiber, set_stack_guarantee, with_current_fiber,+};+use std::ffi::c_void;+use std::io;+use std::ptr;+use wasmtime_environ::prelude::*;+use windows_sys::Win32::System::Threading::*;++cfg_select! {+ target_arch = "x86_64" => {+ mod x86_64;+ use x86_64 as asm;+ }+ target_arch = "aarch64" => {+ mod aarch64;+ use aarch64 as asm;+ }+ _ => {+ mod asm {+ pub const SUPPORTED_ARCH: bool = false;+ pub const SUSPENDED_FRAME: Option<(usize, usize)> = None;++ pub unsafe extern "C" fn wasmtime_fiber_start(+ _entry: crate::RawFiberEntry,+ _arg: *mut u8,+ _host: *mut core::ffi::c_void,+ _top: *mut *mut u8,+ ) -> ! {+ unreachable!()+ }++ pub unsafe extern "C" fn wasmtime_fiber_suspend(_top: *mut u8) {+ unreachable!()+ }+ }+ }+}++pub(crate) use asm::SUSPENDED_FRAME;++/// A raw fiber's underlying Windows fiber, which owns its stack's memory.+pub(crate) struct Fiber(*mut c_void);++// SAFETY: a Windows fiber which is not running may be switched to from any+// thread, and `Fiber` only does so from `&mut self`.+unsafe impl Send for Fiber {}+unsafe impl Sync for Fiber {}++/// What `fiber_start` passes to `wasmtime_fiber_start`, and what it reports+/// back.+struct Start {+ entry: RawFiberEntry,+ arg: *mut u8,+ host: *mut c_void,+ top: *mut u8,+ bottom: usize,+}++/// Whether raw fibers are supported on this platform and in this process.+pub(crate) fn supported() -> bool {+ asm::SUPPORTED_ARCH && !user_shadow_stacks()+}++/// Whether hardware-enforced shadow stacks are enabled for this process.+/// Restoring a snapshot changes the return addresses on a fiber's stack but+/// not on its shadow stack, so a restored fiber would fault on return.+fn user_shadow_stacks() -> bool {+ // `PROCESS_MITIGATION_USER_SHADOW_STACK_POLICY` is a `DWORD` of flags.+ const ENABLE_USER_SHADOW_STACK: u32 = 1 << 0;+ let mut flags = 0u32;+ // SAFETY: `flags` has the size passed. On versions of Windows without+ // shadow stacks this fails and leaves `flags` zero.+ unsafe {+ GetProcessMitigationPolicy(+ GetCurrentProcess(),+ ProcessUserShadowStackPolicy,+ (&raw mut flags).cast(),+ size_of::<u32>(),+ );+ }+ flags & ENABLE_USER_SHADOW_STACK != 0+}++pub(crate) fn switch_routine() -> unsafe extern "C" fn(*mut u8) {+ asm::wasmtime_fiber_suspend+}++unsafe extern "system" fn fiber_start(data: *mut c_void) {+ unsafe {+ set_stack_guarantee();+ let start = data.cast::<Start>();+ let mut high = 0;+ GetCurrentThreadStackLimits(&raw mut (*start).bottom, &mut high);+ asm::wasmtime_fiber_start(+ (*start).entry,+ (*start).arg,+ (*start).host,+ &raw mut (*start).top,+ )+ }+}++impl Fiber {+ /// Creates a Windows fiber for `stack` and runs it until its initial+ /// suspension, after which `stack` reports the fiber's stack bounds.+ pub(crate) unsafe fn new(+ stack: &mut crate::FiberStack,+ entry: RawFiberEntry,+ arg: *mut u8,+ ) -> Result<Self> {+ let mut start = Start {+ entry,+ arg,+ host: ptr::null_mut(),+ top: ptr::null_mut(),+ bottom: 0,+ };+ let start = &raw mut start;+ unsafe {+ let fiber = CreateFiberEx(+ 0,+ stack.0.size,+ FIBER_FLAG_FLOAT_SWITCH,+ Some(fiber_start),+ start.cast(),+ );+ if fiber.is_null() {+ return Err(io::Error::last_os_error().into());+ }+ with_current_fiber(|host| {+ (*start).host = host;+ SwitchToFiber(fiber);+ });+ stack.0.raw_range = Some((*start).bottom..(*start).top as usize);+ Ok(Fiber(fiber))+ }+ }++ /// Runs the fiber, whose stack top is `top`, until it next suspends.+ pub(crate) unsafe fn resume(&mut self, top: *mut u8) {+ unsafe {+ with_current_fiber(|host| {+ top.cast::<*mut c_void>().sub(1).write(host);+ SwitchToFiber(self.0);+ });+ }+ }++ /// Deletes the fiber, after which `stack` may be used for another one.+ pub(crate) fn destroy(self, stack: &mut crate::FiberStack) {+ stack.0.raw_range = None;+ }+}++impl Drop for Fiber {+ fn drop(&mut self) {+ // SAFETY: a raw fiber only runs during `resume`.+ unsafe { delete_fiber(self.0) }+ }+}diff --git a/crates/fiber/src/raw/windows/aarch64.rs b/crates/fiber/src/raw/windows/aarch64.rsnew file mode 100644index 0000000000..e300a049b2--- /dev/null+++ b/crates/fiber/src/raw/windows/aarch64.rs@@ -0,0 +1,147 @@+// A WORD OF CAUTION+//+// This file needs to be kept in sync with itself and with the layout+// documented in `raw/windows.rs`, and `SUSPENDED_FRAME` with the frame that+// `wasmtime_fiber_suspend` spills. It closely follows `x86_64.rs`.++use core::arch::naked_asm;+use windows_sys::Win32::System::Threading::SwitchToFiber;++pub const SUPPORTED_ARCH: bool = true;++/// The indices, in words from a suspended fiber's saved stack pointer, of the+/// frame pointer and return address that `wasmtime_fiber_suspend` spills.+pub const SUSPENDED_FRAME: Option<(usize, usize)> = Some((0, 1));++/// Suspends the raw fiber whose stack top is `top`, returning when it is next+/// resumed.+#[unsafe(naked)]+pub unsafe extern "C" fn wasmtime_fiber_suspend(top: *mut u8 /* x0 */) {+ naked_asm!(+ "+ .seh_proc {suspend}++ // Spill all nonvolatile registers, ending with a frame record.+ stp d8, d9, [sp, -16]!+ .seh_save_fregp_x d8, 16+ stp d10, d11, [sp, -16]!+ .seh_save_fregp_x d10, 16+ stp d12, d13, [sp, -16]!+ .seh_save_fregp_x d12, 16+ stp d14, d15, [sp, -16]!+ .seh_save_fregp_x d14, 16+ stp x19, x20, [sp, -16]!+ .seh_save_regp_x x19, 16+ stp x21, x22, [sp, -16]!+ .seh_save_regp_x x21, 16+ stp x23, x24, [sp, -16]!+ .seh_save_regp_x x23, 16+ stp x25, x26, [sp, -16]!+ .seh_save_regp_x x25, 16+ stp x27, x28, [sp, -16]!+ .seh_save_regp_x x27, 16+ stp x29, x30, [sp, -16]!+ .seh_save_fplr_x 16+ // Unwind through the frame pointer while on the canonical frame.+ mov x29, sp+ .seh_set_fp+ .seh_endprologue++ // Store where to resume, then switch to the host fiber from the+ // canonical frame.+ mov x9, sp+ str x9, [x0, -0x10]+ ldr x9, [x0, -0x08]+ sub sp, x0, 0x10+ mov x0, x9+ bl {switch_to_fiber}++ // Resumed. Only the stack pointer is meaningful here: Windows restored+ // the other registers from whichever suspension last saved them, which+ // need not be the one resumed after a snapshot was restored. Take the+ // registers spilled above from the stack pointer in the slot.+ ldr x29, [sp]+ .seh_startepilogue+ mov sp, x29+ .seh_set_fp+ ldp x29, x30, [sp], 16+ .seh_save_fplr_x 16+ ldp x27, x28, [sp], 16+ .seh_save_regp_x x27, 16+ ldp x25, x26, [sp], 16+ .seh_save_regp_x x25, 16+ ldp x23, x24, [sp], 16+ .seh_save_regp_x x23, 16+ ldp x21, x22, [sp], 16+ .seh_save_regp_x x21, 16+ ldp x19, x20, [sp], 16+ .seh_save_regp_x x19, 16+ ldp d14, d15, [sp], 16+ .seh_save_fregp_x d14, 16+ ldp d12, d13, [sp], 16+ .seh_save_fregp_x d12, 16+ ldp d10, d11, [sp], 16+ .seh_save_fregp_x d10, 16+ ldp d8, d9, [sp], 16+ .seh_save_fregp_x d8, 16+ .seh_endepilogue+ ret+ .seh_endproc+ ",+ suspend = sym wasmtime_fiber_suspend,+ switch_to_fiber = sym SwitchToFiber,+ );+}++/// Sets up the top of the stack as documented in `raw/windows.rs`, reports `top`+/// through `top_out`, suspends, and then calls `entry(arg, top)` when first+/// resumed.+///+/// This never returns, so it does not preserve its caller's nonvolatile+/// registers; the unwind information only describes how to find the caller's+/// frame.+#[unsafe(naked)]+pub unsafe extern "C" fn wasmtime_fiber_start(+ entry: crate::RawFiberEntry, // x0+ arg: *mut u8, // x1+ host: *mut core::ffi::c_void, // x2+ top_out: *mut *mut u8, // x3+) -> ! {+ naked_asm!(+ "+ .seh_proc {start}+ stp x29, x30, [sp, -16]!+ .seh_save_fplr_x 16+ mov x29, sp+ .seh_set_fp+ .seh_endprologue++ // The frame pointer is `top`.+ str x2, [x29, -0x08]+ str x29, [x3]+ mov x19, x0+ mov x20, x1++ // Move the stack pointer below the reserved slots and the canonical+ // frame, touching the page in between first so that Windows grows the+ // stack one page at a time.+ sub x9, x29, 0x1000+ ldr xzr, [x9]+ sub sp, x9, 0x10++ // The initial suspension.+ mov x0, x29+ bl {suspend}++ // Resumed for the first time, or from a snapshot of the initial+ // suspension. Call the entry point, which never returns.+ mov x0, x20+ mov x1, x29+ blr x19+ brk 0xf1b3+ .seh_endproc+ ",+ start = sym wasmtime_fiber_start,+ suspend = sym wasmtime_fiber_suspend,+ );+}diff --git a/crates/fiber/src/raw/windows/x86_64.rs b/crates/fiber/src/raw/windows/x86_64.rsnew file mode 100644index 0000000000..3028a99428--- /dev/null+++ b/crates/fiber/src/raw/windows/x86_64.rs@@ -0,0 +1,159 @@+// A WORD OF CAUTION+//+// This file needs to be kept in sync with itself and with the layout+// documented in `raw/windows.rs`, and `SUSPENDED_FRAME` with the frame that+// `wasmtime_fiber_suspend` spills.++use core::arch::naked_asm;+use windows_sys::Win32::System::Threading::SwitchToFiber;++pub const SUPPORTED_ARCH: bool = true;++/// The indices, in words from a suspended fiber's saved stack pointer, of the+/// frame pointer and return address that `wasmtime_fiber_suspend` spills.+pub const SUSPENDED_FRAME: Option<(usize, usize)> = Some((28, 29));++/// Suspends the raw fiber whose stack top is `top`, returning when it is next+/// resumed.+#[unsafe(naked)]+pub unsafe extern "C" fn wasmtime_fiber_suspend(top: *mut u8 /* rcx */) {+ naked_asm!(+ "+ .seh_proc {suspend}++ // Spill all nonvolatile registers.+ push rbp+ .seh_pushreg rbp+ push rbx+ .seh_pushreg rbx+ push rdi+ .seh_pushreg rdi+ push rsi+ .seh_pushreg rsi+ push r12+ .seh_pushreg r12+ push r13+ .seh_pushreg r13+ push r14+ .seh_pushreg r14+ push r15+ .seh_pushreg r15+ sub rsp, 0xa8+ .seh_stackalloc 0xa8+ movaps [rsp + 0x00], xmm6+ .seh_savexmm xmm6, 0x00+ movaps [rsp + 0x10], xmm7+ .seh_savexmm xmm7, 0x10+ movaps [rsp + 0x20], xmm8+ .seh_savexmm xmm8, 0x20+ movaps [rsp + 0x30], xmm9+ .seh_savexmm xmm9, 0x30+ movaps [rsp + 0x40], xmm10+ .seh_savexmm xmm10, 0x40+ movaps [rsp + 0x50], xmm11+ .seh_savexmm xmm11, 0x50+ movaps [rsp + 0x60], xmm12+ .seh_savexmm xmm12, 0x60+ movaps [rsp + 0x70], xmm13+ .seh_savexmm xmm13, 0x70+ movaps [rsp + 0x80], xmm14+ .seh_savexmm xmm14, 0x80+ movaps [rsp + 0x90], xmm15+ .seh_savexmm xmm15, 0x90+ // Unwind through the frame pointer while on the canonical frame.+ mov rbp, rsp+ .seh_setframe rbp, 0+ .seh_endprologue++ // Store where to resume, then switch to the host fiber from the+ // canonical frame, leaving shadow space above it.+ mov -0x10[rcx], rsp+ mov rax, rcx+ mov rcx, -0x08[rax]+ lea rsp, -0x30[rax]+ call {switch_to_fiber}++ // Resumed. Only the stack pointer is meaningful here: Windows restored+ // the other registers from whichever suspension last saved them, which+ // need not be the one resumed after a snapshot was restored. Take the+ // registers spilled above from the stack pointer in the slot.+ mov rbp, 0x20[rsp]+ movaps xmm6, [rbp + 0x00]+ movaps xmm7, [rbp + 0x10]+ movaps xmm8, [rbp + 0x20]+ movaps xmm9, [rbp + 0x30]+ movaps xmm10, [rbp + 0x40]+ movaps xmm11, [rbp + 0x50]+ movaps xmm12, [rbp + 0x60]+ movaps xmm13, [rbp + 0x70]+ movaps xmm14, [rbp + 0x80]+ movaps xmm15, [rbp + 0x90]+ lea rsp, [rbp + 0xa8]+ pop r15+ pop r14+ pop r13+ pop r12+ pop rsi+ pop rdi+ pop rbx+ pop rbp+ ret+ .seh_endproc+ ",+ suspend = sym wasmtime_fiber_suspend,+ switch_to_fiber = sym SwitchToFiber,+ );+}++/// Sets up the top of the stack as documented in `raw/windows.rs`, reports `top`+/// through `top_out`, suspends, and then calls `entry(arg, top)` when first+/// resumed.+///+/// This never returns, so it does not preserve its caller's nonvolatile+/// registers; the unwind information only describes how to find the caller's+/// frame.+#[unsafe(naked)]+pub unsafe extern "C" fn wasmtime_fiber_start(+ entry: crate::RawFiberEntry, // rcx+ arg: *mut u8, // rdx+ host: *mut core::ffi::c_void, // r8+ top_out: *mut *mut u8, // r9+) -> ! {+ naked_asm!(+ "+ .seh_proc {start}+ push rbp+ .seh_pushreg rbp+ mov rbp, rsp+ .seh_setframe rbp, 0+ .seh_endprologue++ // The frame pointer is `top`, which is 16-byte aligned since the+ // stack was 8 bytes off alignment at entry.+ mov -0x08[rbp], r8+ mov [r9], rbp+ mov r12, rcx+ mov r13, rdx++ // Move the stack pointer below the reserved slots and the canonical+ // frame, touching the page in between first so that Windows grows the+ // stack one page at a time.+ mov rax, -0x1000[rbp]+ lea rsp, -0x1020[rbp]++ // The initial suspension.+ mov rcx, rbp+ call {suspend}++ // Resumed for the first time, or from a snapshot of the initial+ // suspension. Call the entry point, which never returns.+ mov rcx, r13+ mov rdx, rbp+ call r12+ ud2+ .seh_endproc+ ",+ start = sym wasmtime_fiber_start,+ suspend = sym wasmtime_fiber_suspend,+ );+}diff --git a/crates/fiber/src/stackswitch.rs b/crates/fiber/src/stackswitch.rsindex 224fac18df..b117ecd694 100644--- a/crates/fiber/src/stackswitch.rs+++ b/crates/fiber/src/stackswitch.rs@@ -5,6 +5,11 @@// accessed via the `extern "C"` declarations below that.cfg_select! {+ // Miri uses another fiber implementation, but raw fibers still refer to+ // the routines here.+ all(feature = "std", miri) => {+ pub(crate) use unsupported::*;+ }target_arch = "aarch64" => {mod aarch64;pub(crate) use supported::*;@@ -47,7 +52,6 @@ cfg_select! {}feature = "custom" => {mod custom;- pub(crate) use supported::*;pub(crate) use custom::*;}_ => {@@ -68,6 +72,10 @@ cfg_select! {)]mod supported {pub const SUPPORTED_ARCH: bool = true;++ /// Whether raw fibers can use the routines here. They require the+ /// reserved-slot stack layout documented in `unix.rs`.+ pub const RAW_FIBERS: bool = true;}/// Helper module reexported in the fallback case above when the current host@@ -79,6 +87,7 @@ mod supported {)]mod unsupported {pub const SUPPORTED_ARCH: bool = false;+ pub const RAW_FIBERS: bool = false;pub(crate) unsafe fn wasmtime_fiber_init(_top_of_stack: *mut u8,@@ -91,4 +100,8 @@ mod unsupported {pub(crate) unsafe fn wasmtime_fiber_switch(_top_of_stack: *mut u8) {unreachable!();}++ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(_top_of_stack: *mut u8) {+ unreachable!();+ }}diff --git a/crates/fiber/src/stackswitch/aarch64.rs b/crates/fiber/src/stackswitch/aarch64.rsindex 85f4a8b517..9ffdd06f3e 100644--- a/crates/fiber/src/stackswitch/aarch64.rs+++ b/crates/fiber/src/stackswitch/aarch64.rs@@ -16,7 +16,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {}#[unsafe(naked)]-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) {+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) {naked_asm!(".cfi_startprocdiff --git a/crates/fiber/src/stackswitch/arm.rs b/crates/fiber/src/stackswitch/arm.rsindex d145178a73..b462d8530b 100644--- a/crates/fiber/src/stackswitch/arm.rs+++ b/crates/fiber/src/stackswitch/arm.rs@@ -16,7 +16,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {}#[unsafe(naked)]-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* r0 */) {+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* r0 */) {naked_asm!("// Save callee-saved registersdiff --git a/crates/fiber/src/stackswitch/custom.rs b/crates/fiber/src/stackswitch/custom.rsindex faf0d0aad0..21f2e45c1c 100644--- a/crates/fiber/src/stackswitch/custom.rs+++ b/crates/fiber/src/stackswitch/custom.rs@@ -20,3 +20,13 @@ unsafe extern "C" {pub(crate) fn wasmtime_fiber_switch(top_of_stack: *mut u8);}++pub(crate) const SUPPORTED_ARCH: bool = true;++// The embedder's routines need not use the stack layout that raw fiber+// snapshots depend on.+pub(crate) const RAW_FIBERS: bool = false;++pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(_top_of_stack: *mut u8) {+ unreachable!()+}diff --git a/crates/fiber/src/stackswitch/loongarch64.rs b/crates/fiber/src/stackswitch/loongarch64.rsindex 9493f77f5d..2bb11dc725 100644--- a/crates/fiber/src/stackswitch/loongarch64.rs+++ b/crates/fiber/src/stackswitch/loongarch64.rs@@ -13,7 +13,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {}#[unsafe(naked)]-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {naked_asm!("// Save all callee-saved registers on the stack since we'rediff --git a/crates/fiber/src/stackswitch/riscv32imac.rs b/crates/fiber/src/stackswitch/riscv32imac.rsindex 78cc90fba9..55a3445d3f 100644--- a/crates/fiber/src/stackswitch/riscv32imac.rs+++ b/crates/fiber/src/stackswitch/riscv32imac.rs@@ -16,7 +16,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {}#[unsafe(naked)]-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {naked_asm!("// See https://github.com/rust-lang/rust/issues/80608.diff --git a/crates/fiber/src/stackswitch/riscv64.rs b/crates/fiber/src/stackswitch/riscv64.rsindex b55ac71cc7..e05f0e3062 100644--- a/crates/fiber/src/stackswitch/riscv64.rs+++ b/crates/fiber/src/stackswitch/riscv64.rs@@ -13,7 +13,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {}#[unsafe(naked)]-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* a0 */) {naked_asm!("// See https://github.com/rust-lang/rust/issues/80608.diff --git a/crates/fiber/src/stackswitch/s390x.rs b/crates/fiber/src/stackswitch/s390x.rsindex 52bbdbd2cc..e657989966 100644--- a/crates/fiber/src/stackswitch/s390x.rs+++ b/crates/fiber/src/stackswitch/s390x.rs@@ -13,7 +13,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {}#[unsafe(naked)]-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) {+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* x0 */) {naked_asm!("// Save all callee-saved registers on the stack since we're assumingdiff --git a/crates/fiber/src/stackswitch/x86.rs b/crates/fiber/src/stackswitch/x86.rsindex 1a09240dd4..5f32bacee9 100644--- a/crates/fiber/src/stackswitch/x86.rs+++ b/crates/fiber/src/stackswitch/x86.rs@@ -18,7 +18,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {}#[unsafe(naked)]-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8) {+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8) {naked_asm!("// Load our stack-to-usediff --git a/crates/fiber/src/stackswitch/x86_64.rs b/crates/fiber/src/stackswitch/x86_64.rsindex 5cea3ac715..e0536dbe62 100644--- a/crates/fiber/src/stackswitch/x86_64.rs+++ b/crates/fiber/src/stackswitch/x86_64.rs@@ -13,7 +13,7 @@ pub(crate) unsafe extern "C" fn wasmtime_fiber_switch(top_of_stack: *mut u8) {}#[unsafe(naked)]-unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* rdi */) {+pub(crate) unsafe extern "C" fn wasmtime_fiber_switch_(top_of_stack: *mut u8 /* rdi */) {naked_asm!("// We're switching to arbitrary code somewhere else, so pessimisticallydiff --git a/crates/fiber/src/windows.rs b/crates/fiber/src/windows.rsindex f9bc319cc5..f8f303d0f3 100644--- a/crates/fiber/src/windows.rs+++ b/crates/fiber/src/windows.rs@@ -13,14 +13,22 @@ use windows_sys::Win32::System::Threading::*;pub type Error = io::Error;#[derive(Debug)]-pub struct FiberStack(usize);+pub struct FiberStack {+ pub(crate) size: usize,+ // Windows allocates a fiber's stack when creating the fiber, so its+ // location is only known while a raw fiber owns it. See `raw/windows.rs`.+ pub(crate) raw_range: Option<Range<usize>>,+}impl FiberStack {pub fn new(size: usize, zeroed: bool) -> io::Result<Self> {// We don't support fiber stack zeroing on windows.let _ = zeroed;- Ok(Self(size))+ Ok(Self {+ size,+ raw_range: None,+ })}pub unsafe fn from_raw_parts(@@ -40,11 +48,11 @@ impl FiberStack {}pub fn top(&self) -> Option<*mut u8> {- None+ Some(self.raw_range.as_ref()?.end as *mut u8)}pub fn range(&self) -> Option<Range<usize>> {- None+ self.raw_range.clone()}pub fn guard_range(&self) -> Option<Range<*mut u8>> {@@ -67,17 +75,15 @@ struct StartState {result_location: Cell<*const u8>,}-const FIBER_FLAG_FLOAT_SWITCH: u32 = 1;+pub(crate) const FIBER_FLAG_FLOAT_SWITCH: u32 = 1;unsafe extern "C" {#[wasmtime_versioned_export_macros::versioned_link]fn wasmtime_fiber_get_current() -> *mut c_void;}-unsafe extern "system" fn fiber_start<F, A, B, C>(data: *mut c_void)-where- F: FnOnce(A, &mut super::Suspend<A, B, C>) -> C,-{+/// Sets the current fiber's stack guarantee.+pub(crate) unsafe fn set_stack_guarantee() {unsafe {// Set the stack guarantee to be consistent with what Rust expects for threads// This value is taken from:@@ -85,6 +91,71 @@ whereif SetThreadStackGuarantee(&mut 0x5000) == 0 {panic!("failed to set fiber stack guarantee");}+ }+}++/// Calls `f` with the current fiber, first converting the current thread into+/// a fiber, and back again afterwards, if it is not one already.+pub(crate) unsafe fn with_current_fiber(f: impl FnOnce(*mut c_void)) {+ unsafe {+ let is_fiber = IsThreadAFiber() != 0;+ let parent_fiber = if is_fiber {+ wasmtime_fiber_get_current()+ } else {+ // Newer Rust versions use fiber local storage to register an internal hook that+ // calls thread locals' destructors on thread exit.+ // This has a limitation: the hook only runs in a regular thread (not in a fiber).+ // We convert back into a thread once execution returns to this function,+ // but we must also ensure that the hook is registered before converting into a fiber.+ // Otherwise, a different fiber could be the first to register the hook,+ // causing the hook to be called (and skipped) prematurely when that fiber is deleted.+ struct Guard;++ impl Drop for Guard {+ fn drop(&mut self) {}+ }+ assert!(needs_drop::<Guard>());+ thread_local!(static GUARD: Guard = Guard);+ GUARD.with(|_g| {});+ ConvertThreadToFiber(ptr::null_mut())+ };+ assert!(+ !parent_fiber.is_null(),+ "failed to make current thread a fiber"+ );+ f(parent_fiber);+ if !is_fiber {+ let res = ConvertFiberToThread();+ assert!(res != 0, "failed to convert main thread back");+ }+ }+}++/// Deletes a fiber which is not running.+pub(crate) unsafe fn delete_fiber(fiber: *mut c_void) {+ unsafe {+ let is_fiber = IsThreadAFiber() != 0;+ if !is_fiber {+ // DeleteFiber runs FLS destructors. Make those destructors+ // observe a fiber so Rust does not run thread-local cleanup+ // for the live thread.+ let fiber = ConvertThreadToFiber(ptr::null_mut());+ assert!(!fiber.is_null(), "failed to make current thread a fiber");+ }+ DeleteFiber(fiber);+ if !is_fiber {+ let res = ConvertFiberToThread();+ assert!(res != 0, "failed to convert main thread back");+ }+ }+}++unsafe extern "system" fn fiber_start<F, A, B, C>(data: *mut c_void)+where+ F: FnOnce(A, &mut super::Suspend<A, B, C>) -> C,+{+ unsafe {+ set_stack_guarantee();let state = data.cast::<StartState>();let func = Box::from_raw((*state).initial_closure.get().cast::<F>());@@ -116,7 +187,7 @@ impl Fiber {let fiber = CreateFiberEx(0,- stack.0,+ stack.size,FIBER_FLAG_FLOAT_SWITCH,Some(fiber_start::<F, A, B, C>),&*state as *const StartState as *mut _,@@ -133,42 +204,15 @@ impl Fiber {pub(crate) fn resume<A, B, C>(&self, _stack: &FiberStack, result: &Cell<RunResult<A, B, C>>) {unsafe {- let is_fiber = IsThreadAFiber() != 0;- let parent_fiber = if is_fiber {- wasmtime_fiber_get_current()- } else {- // Newer Rust versions use fiber local storage to register an internal hook that- // calls thread locals' destructors on thread exit.- // This has a limitation: the hook only runs in a regular thread (not in a fiber).- // We convert back into a thread once execution returns to this function,- // but we must also ensure that the hook is registered before converting into a fiber.- // Otherwise, a different fiber could be the first to register the hook,- // causing the hook to be called (and skipped) prematurely when that fiber is deleted.- struct Guard;-- impl Drop for Guard {- fn drop(&mut self) {}- }- assert!(needs_drop::<Guard>());- thread_local!(static GUARD: Guard = Guard);- GUARD.with(|_g| {});- ConvertThreadToFiber(ptr::null_mut())- };- assert!(- !parent_fiber.is_null(),- "failed to make current thread a fiber"- );- self.state- .result_location- .set(result as *const _ as *const _);- self.state.parent.set(parent_fiber);- SwitchToFiber(self.fiber);- self.state.parent.set(ptr::null_mut());- self.state.result_location.set(ptr::null());- if !is_fiber {- let res = ConvertFiberToThread();- assert!(res != 0, "failed to convert main thread back");- }+ with_current_fiber(|parent_fiber| {+ self.state+ .result_location+ .set(result as *const _ as *const _);+ self.state.parent.set(parent_fiber);+ SwitchToFiber(self.fiber);+ self.state.parent.set(ptr::null_mut());+ self.state.result_location.set(ptr::null());+ });}}@@ -178,19 +222,7 @@ impl Fiber {impl Drop for Fiber {fn drop(&mut self) {unsafe {- let is_fiber = IsThreadAFiber() != 0;- if !is_fiber {- // DeleteFiber runs FLS destructors. Make those destructors- // observe a fiber so Rust does not run thread-local cleanup- // for the live thread.- let fiber = ConvertThreadToFiber(ptr::null_mut());- assert!(!fiber.is_null(), "failed to make current thread a fiber");- }- DeleteFiber(self.fiber);- if !is_fiber {- let res = ConvertFiberToThread();- assert!(res != 0, "failed to convert main thread back");- }+ delete_fiber(self.fiber);}}}