A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Trace GC roots on component-model fiber stacks (#13693)
Nick Fitzgerald7f2faa6fbca6
diff --git a/crates/wasmtime/src/runtime/component/concurrent.rs b/crates/wasmtime/src/runtime/component/concurrent.rsindex 5ec4de15bb..4f6501056b 100644--- a/crates/wasmtime/src/runtime/component/concurrent.rs+++ b/crates/wasmtime/src/runtime/component/concurrent.rs@@ -58,8 +58,12 @@ use crate::component::{};use crate::fiber::{self, StoreFiber, StoreFiberYield};use crate::hash_set::HashSet;+#[cfg(feature = "gc")]+use crate::module::ModuleRegistry;use crate::prelude::*;use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};+#[cfg(feature = "gc")]+use crate::vm::GcRootsList;use crate::vm::component::{CallContext, ComponentInstance, InstanceState};use crate::vm::{AlwaysMut, SendSyncPtr, VMFuncRef, VMMemoryDefinition, VMStore};use crate::{@@ -91,6 +95,8 @@ use wasmtime_environ::component::{};use wasmtime_environ::packed_option::ReservedValue;use wasmtime_environ::{NUM_COMPONENT_CONTEXT_SLOTS, Trap};+#[cfg(feature = "gc")]+use wasmtime_unwinder::Unwind;pub use abort::JoinHandle;pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};@@ -5185,6 +5191,73 @@ impl ConcurrentState {}}+ #[cfg(feature = "gc")]+ pub(crate) fn trace_fiber_roots(+ &mut self,+ modules: &ModuleRegistry,+ unwind: &dyn Unwind,+ gc_roots_list: &mut GcRootsList,+ ) {+ let ConcurrentState {+ table,+ worker,+ high_priority,+ low_priority,++ // TODO(cm-gc): This field contains `ValRaw`s, but they are never GC+ // references because the component model doesn't support GC yet. We+ // will need to trace these somehow when it does.+ futures: _,++ // These fields do not contain GC references.+ worker_item: _,+ current_thread: _,+ suspend_reason: _,+ global_error_context_ref_counts: _,+ interesting_tasks: _,+ interesting_tasks_empty_waker: _,+ } = self;++ for entry in table.get_mut().iter_mut() {+ if let Some(set) = entry.downcast_mut::<WaitableSet>() {+ for mode in set.waiting.values_mut() {+ if let WaitMode::Fiber(fiber) = mode {+ fiber.trace_gc_roots(modules, unwind, gc_roots_list);+ }+ }+ } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {+ if let GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber, .. } =+ &mut thread.state+ {+ fiber.trace_gc_roots(modules, unwind, gc_roots_list);+ }+ }+ }++ if let Some(fiber) = worker {+ fiber.trace_gc_roots(modules, unwind, gc_roots_list);+ }++ let mut handle_item = |item: &mut WorkItem| match item {+ WorkItem::ResumeFiber(fiber) => {+ fiber.trace_gc_roots(modules, unwind, gc_roots_list);+ }+ WorkItem::PushFuture(_future) => {+ // TODO(cm-gc): once futures can contain GC roots, we will need+ // to trace them.+ }+ WorkItem::ResumeThread(..) | WorkItem::GuestCall(..) | WorkItem::WorkerFunction(..) => {+ }+ };++ for item in high_priority {+ handle_item(item);+ }+ for item in low_priority {+ handle_item(item);+ }+ }+fn push<V: Send + Sync + 'static>(&mut self,value: V,diff --git a/crates/wasmtime/src/runtime/component/store.rs b/crates/wasmtime/src/runtime/component/store.rsindex 46d7fe0bfa..7c1ace64c6 100644--- a/crates/wasmtime/src/runtime/component/store.rs+++ b/crates/wasmtime/src/runtime/component/store.rs@@ -165,6 +165,11 @@ impl ComponentStoreData {allocator.decrement_component_instance_count();}}++ #[cfg(all(feature = "component-model-async", feature = "gc"))]+ pub fn task_state_mut(&mut self) -> &mut ComponentTaskState {+ &mut self.task_state+ }}/// A type used to represent an allocated `ComponentInstance` located within adiff --git a/crates/wasmtime/src/runtime/fiber.rs b/crates/wasmtime/src/runtime/fiber.rsindex 779763b26a..50f086f7d4 100644--- a/crates/wasmtime/src/runtime/fiber.rs+++ b/crates/wasmtime/src/runtime/fiber.rs@@ -9,6 +9,13 @@ use core::pin::Pin;use core::ptr::{self, NonNull};use core::task::{Context, Poll};use wasmtime_fiber::{Fiber, FiberStack, Suspend};+#[cfg(all(feature = "component-model-async", feature = "gc"))]+use wasmtime_unwinder::Unwind;++#[cfg(all(feature = "component-model-async", feature = "gc"))]+use super::module::ModuleRegistry;+#[cfg(all(feature = "component-model-async", feature = "gc"))]+use super::vm::GcRootsList;type WasmtimeResume = Result<NonNull<Context<'static>>>;type WasmtimeYield = StoreFiberYield;@@ -453,6 +460,21 @@ impl<'a> StoreFiber<'a> {}}}++ #[cfg(all(feature = "component-model-async", feature = "gc"))]+ pub(crate) fn trace_gc_roots(+ &mut self,+ modules: &ModuleRegistry,+ unwind: &dyn Unwind,+ gc_roots_list: &mut GcRootsList,+ ) {+ if let Some(state) = &mut self.state {+ state+ .get_mut()+ .tls+ .trace_gc_roots(modules, unwind, gc_roots_list);+ }+ }}// Note that this implementation will panic if the fiber is in-progress, whichdiff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rsindex 63fa437b82..e1004ff7b6 100644--- a/crates/wasmtime/src/runtime/store/gc.rs+++ b/crates/wasmtime/src/runtime/store/gc.rs@@ -1,6 +1,7 @@//! GC-related methods for stores.use crate::error::Context;+use crate::module::ModuleRegistry;use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter, yield_now,};@@ -631,7 +632,11 @@ impl StoreOpaque {log::trace!("End trace GC roots")}- fn trace_wasm_stack_frame(&self, gc_roots_list: &mut GcRootsList, frame: Frame) {+ pub(crate) fn trace_wasm_stack_frame(+ modules: &ModuleRegistry,+ gc_roots_list: &mut GcRootsList,+ frame: Frame,+ ) {let pc = frame.pc();debug_assert!(pc != 0, "we should always get a valid PC for Wasm frames");@@ -641,8 +646,7 @@ impl StoreOpaque {"we should always get a valid frame pointer for Wasm frames");- let (store_code, offset) = self- .modules()+ let (store_code, offset) = modules.store_code_by_pc(pc).expect("should have store code for Wasm frame");let offset = u32::try_from(offset).unwrap();@@ -659,7 +663,7 @@ impl StoreOpaque {let sp = unsafe { stack_map.sp(fp) };for stack_slot in unsafe { stack_map.live_gc_refs(sp) } {unsafe {- self.trace_wasm_stack_slot(gc_roots_list, stack_slot);+ Self::trace_wasm_stack_slot(gc_roots_list, stack_slot);}}}@@ -668,13 +672,13 @@ impl StoreOpaque {if let Some(frame_table) = store_code.code_memory().frame_table() {for stack_slot in crate::debug::gc_refs_in_frame(frame_table, offset, fp) {unsafe {- self.trace_wasm_stack_slot(gc_roots_list, stack_slot);+ Self::trace_wasm_stack_slot(gc_roots_list, stack_slot);}}}}- unsafe fn trace_wasm_stack_slot(&self, gc_roots_list: &mut GcRootsList, stack_slot: *mut u32) {+ unsafe fn trace_wasm_stack_slot(gc_roots_list: &mut GcRootsList, stack_slot: *mut u32) {let raw: u32 = unsafe { core::ptr::read(stack_slot) };log::trace!("Stack slot @ {stack_slot:p} = {raw:#x}");@@ -691,10 +695,25 @@ impl StoreOpaque {log::trace!("Begin trace GC roots :: Wasm stack");Backtrace::trace(self, |frame| {- self.trace_wasm_stack_frame(gc_roots_list, frame);+ Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame);core::ops::ControlFlow::Continue(())});+ #[cfg(feature = "component-model-async")]+ if self.concurrency_support() {+ let unwind = self.unwinder();+ let StoreOpaque {+ modules,+ store_data,+ ..+ } = self;+ store_data+ .components+ .task_state_mut()+ .concurrent_state_mut()+ .trace_fiber_roots(modules, unwind, gc_roots_list);+ }+log::trace!("End trace GC roots :: Wasm stack");}@@ -717,7 +736,7 @@ impl StoreOpaque {match state {VMStackState::Suspended => {Backtrace::trace_suspended_continuation(self, continuation.deref(), |frame| {- self.trace_wasm_stack_frame(gc_roots_list, frame);+ Self::trace_wasm_stack_frame(self.modules(), gc_roots_list, frame);core::ops::ControlFlow::Continue(())});}diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rsindex 457754539e..acbe5c356a 100644--- a/crates/wasmtime/src/runtime/vm/traphandlers.rs+++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs@@ -1088,6 +1088,11 @@ fn entry_trap_handler(vm_store_context: &VMStoreContext) -> Handler {/// thread's current list pointed to by TLS is youngest-to-oldest links, while a/// suspended fiber stores oldest-to-youngest links.pub(crate) mod tls {+ #[cfg(all(feature = "component-model-async", feature = "gc"))]+ use crate::module::ModuleRegistry;+ #[cfg(all(feature = "component-model-async", feature = "gc"))]+ use crate::store::StoreOpaque;+use super::CallThreadState;pub use raw::Ptr;@@ -1164,6 +1169,8 @@ pub(crate) mod tls {}pub use raw::initialize as tls_eager_initialize;+ #[cfg(all(feature = "component-model-async", feature = "gc"))]+ use wasmtime_unwinder::Unwind;/// Opaque state used to persist the state of the `CallThreadState`/// activations associated with a fiber stack that's used as part of an@@ -1269,6 +1276,32 @@ pub(crate) mod tls {let p = raw::get() as usize;assert!(!range.contains(&p));}++ #[cfg(all(feature = "component-model-async", feature = "gc"))]+ pub(crate) fn trace_gc_roots(+ &mut self,+ modules: &ModuleRegistry,+ unwind: &dyn Unwind,+ gc_roots_list: &mut crate::vm::GcRootsList,+ ) {+ let mut ptr = self.state;+ unsafe {+ while let Some(state) = ptr.as_ref() {+ let _ = wasmtime_unwinder::visit_frames::<()>(+ unwind,+ state.old_last_wasm_exit_pc(),+ state.old_last_wasm_exit_fp(),+ state.old_last_wasm_entry_fp(),+ |frame| {+ StoreOpaque::trace_wasm_stack_frame(modules, gc_roots_list, frame);+ core::ops::ControlFlow::Continue(())+ },+ );++ ptr = state.prev.get();+ }+ }+ }}/// Opaque state used to help control TLS state across stack switches fordiff --git a/crates/wast/src/wast.rs b/crates/wast/src/wast.rsindex d2dcb42c35..6cd40bfc06 100644--- a/crates/wast/src/wast.rs+++ b/crates/wast/src/wast.rs@@ -267,6 +267,10 @@ impl WastContext {Ok(())},)?;+ i.func_wrap("gc", |mut store, (): ()| {+ store.as_context_mut().gc(None)?;+ Ok(())+ })?;}Ok(())}diff --git a/tests/misc_testsuite/gc/component-async-suspended-fiber-roots.wast b/tests/misc_testsuite/gc/component-async-suspended-fiber-roots.wastnew file mode 100644index 0000000000..218a88b748--- /dev/null+++ b/tests/misc_testsuite/gc/component-async-suspended-fiber-roots.wast@@ -0,0 +1,145 @@+;;! component_model_async = true+;;! component_model_async_stackful = true+;;! gc = true++;; Regression test for https://github.com/bytecodealliance/wasmtime/issues/13676+;;+;; A stackful component-model-async guest task ("holder") allocates a GC+;; reference and holds it in a local across a suspension point. While that fiber+;; is suspended, a sibling task triggers a garbage collection. If we don't trace+;; GC roots in suspended fibers, then we will get GC heap corruption.++(component+ (import "wasmtime" (instance $wasmtime+ (export "gc" (func))+ ))++ ;; The driver subtask: yields control back to the holder (so the holder can+ ;; suspend), then churns the GC heap.+ (component $Driver+ (import "wasmtime" (instance $wasmtime+ (export "gc" (func))+ ))++ (core func $gc (canon lower (func $wasmtime "gc")))+ (core func $yield (canon thread.yield))++ (core module $m+ (import "" "gc" (func $gc))+ (import "" "yield" (func $yield (result i32)))++ (type $s (struct (field i32)))++ (func (export "drive")+ ;; Hand control back to the holder so it can reach `waitable-set.wait`+ ;; and suspend with its GC reference live on its fiber stack.+ (drop (call $yield))++ ;; The holder is now suspended. Collect: this must trace the holder's+ ;; suspended fiber, otherwise its struct is freed here.+ (call $gc)++ ;; Allocate a batch of same-shaped garbage. Under deferred+ ;; reference counting this reuses the just-freed slot (if the bug is+ ;; present) and overwrites it with a different value.+ (call $churn)++ ;; Collect again for good measure (helps relocating collectors expose a+ ;; stale, un-updated pointer).+ (call $gc)+ )++ (func $churn+ (local $i i32)+ (local.set $i (i32.const 128))+ (loop $l+ (drop (struct.new $s (i32.const 0x5678)))+ (local.set $i (i32.sub (local.get $i) (i32.const 1)))+ (br_if $l (local.get $i))+ )+ )+ )+ (core instance $i (instantiate $m+ (with "" (instance+ (export "gc" (func $gc))+ (export "yield" (func $yield))+ ))+ ))++ (func (export "drive") async (canon lift (core func $i "drive")))+ )+ (instance $driver (instantiate $Driver (with "wasmtime" (instance $wasmtime))))++ ;; The holder task: allocates a GC reference, starts the driver, and waits for+ ;; it -- suspending with the reference live -- then validates the reference+ ;; survived.+ (component $Holder+ (import "drive" (func $drive async))++ (core module $libc (memory (export "memory") 1))+ (core instance $libc (instantiate $libc))++ (core func $drive (canon lower (func $drive) async))+ (core func $ws-new (canon waitable-set.new))+ (core func $w-join (canon waitable.join))+ (core func $ws-wait (canon waitable-set.wait (memory $libc "memory")))+ (core func $subtask-drop (canon subtask.drop))++ (core module $m+ (import "" "drive" (func $drive (result i32)))+ (import "" "ws-new" (func $ws-new (result i32)))+ (import "" "w-join" (func $w-join (param i32 i32)))+ (import "" "ws-wait" (func $ws-wait (param i32 i32) (result i32)))+ (import "" "subtask-drop" (func $subtask-drop (param i32)))++ (type $s (struct (field i32)))++ (func (export "run")+ (local $s (ref $s))+ (local $ret i32)+ (local $task i32)+ (local $set i32)++ ;; The reference we want to survive the collection that happens while we+ ;; are suspended below.+ (local.set $s (struct.new $s (i32.const 0x1234)))++ ;; Start the driver subtask. It will `thread.yield` almost immediately,+ ;; handing control back here, so it must not have already RETURNED (2).+ (local.set $ret (call $drive))+ (if (i32.eq (i32.and (local.get $ret) (i32.const 0xf)) (i32.const 2))+ (then unreachable))+ (local.set $task (i32.shr_u (local.get $ret) (i32.const 4)))++ ;; Wait for the driver to finish. This suspends our fiber with `$s` live+ ;; across the call, which is exactly the situation the GC must handle.+ (local.set $set (call $ws-new))+ (call $w-join (local.get $task) (local.get $set))+ (drop (call $ws-wait (local.get $set) (i32.const 0)))++ ;; Resumed: the driver ran two collections while we were suspended. If+ ;; our fiber roots were traced, `$s` still reads 0x1234.+ (if (i32.ne (struct.get $s 0 (local.get $s)) (i32.const 0x1234))+ (then unreachable))++ (call $subtask-drop (local.get $task))+ )+ )+ (core instance $i (instantiate $m+ (with "" (instance+ (export "drive" (func $drive))+ (export "ws-new" (func $ws-new))+ (export "w-join" (func $w-join))+ (export "ws-wait" (func $ws-wait))+ (export "subtask-drop" (func $subtask-drop))+ ))+ ))++ (func (export "run") async (canon lift (core func $i "run")))+ )+ (instance $holder (instantiate $Holder (with "drive" (func $driver "drive"))))++ (export "run" (func $holder "run"))+)++(assert_return (invoke "run"))