Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Perform multiple collection increments in the copying collector (#13491)

Nick Fitzgerald2a4a19bfc4f8
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
index 685f506327..49c126686d 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
@@ -951,7 +951,7 @@ unsafe impl GcHeap for CopyingHeap {
roots: Some(roots),
host_data_table,
heap: self,
- phase: CopyingCollectionPhase::Collect,
+ phase: CopyingCollectionPhase::ProcessRoots,
})
}
@@ -1009,6 +1009,8 @@ unsafe impl GcHeap for CopyingHeap {
}
}
+const OBJECTS_PER_INCREMENT: usize = 16_384;
+
struct CopyingCollection<'a> {
roots: Option<GcRootsIter<'a>>,
host_data_table: &'a mut ExternRefHostDataTable,
@@ -1017,7 +1019,9 @@ struct CopyingCollection<'a> {
}
enum CopyingCollectionPhase {
- Collect,
+ ProcessRoots,
+ ProcessWorklist,
+ SweepExternRefs,
Done,
}
@@ -1040,17 +1044,56 @@ impl CopyingCollection<'_> {
Ok(())
}
- /// Scan all grey objects until the worklist is empty.
- fn process_worklist(&mut self) -> Result<()> {
- log::trace!("Begin processing worklist");
+ /// Handle the `ProcessRoots` phase: flip semi-spaces, initialize the
+ /// worklist, forward all roots, then transition to `ProcessWorklist`.
+ fn begin_collection(&mut self) -> Result<GcProgress> {
+ log::trace!("Begin copying collection: processing roots");
+
+ assert!(self.heap.active_space_start <= self.heap.bump_ptr());
+ assert!(self.heap.bump_ptr() <= self.heap.active_space_end());
+ assert!(self.heap.idle_space_start <= self.heap.idle_space_end);
+ assert!(
+ self.heap.active_space_end() <= self.heap.idle_space_start
+ || self.heap.idle_space_end <= self.heap.active_space_start
+ );
+
+ self.heap.flip();
+ self.heap.initialize_worklist();
+
+ self.process_roots()?;
+
+ self.phase = CopyingCollectionPhase::ProcessWorklist;
+ Ok(GcProgress::Continue)
+ }
+
+ /// Scan up to `OBJECTS_PER_INCREMENT` grey objects from the worklist.
+ ///
+ /// When the worklist is fully drained, transitions to `SweepExternRefs`.
+ /// Always returns `GcProgress::Continue` so the caller yields after each
+ /// increment regardless of whether the worklist still has items.
+ fn process_worklist_increment(&mut self) -> Result<GcProgress> {
+ log::trace!("Begin processing worklist increment");
let trace_infos = mem::take(&mut self.heap.trace_infos);
- while let Some(gc_ref) = self.heap.worklist_pop()? {
- debug_assert!(self.heap.is_in_active_space(gc_ref.heap_index()?.get()));
- self.heap.scan(&gc_ref, &trace_infos)?;
- }
+ let mut count = 0;
+ let has_more = loop {
+ if count >= OBJECTS_PER_INCREMENT {
+ break true;
+ }
+ match self.heap.worklist_pop()? {
+ Some(gc_ref) => {
+ debug_assert!(self.heap.is_in_active_space(gc_ref.heap_index()?.get()));
+ self.heap.scan(&gc_ref, &trace_infos)?;
+ count += 1;
+ }
+ None => break false,
+ }
+ };
self.heap.trace_infos = trace_infos;
- log::trace!("End processing worklist");
- Ok(())
+ if !has_more {
+ self.phase = CopyingCollectionPhase::SweepExternRefs;
+ }
+ log::trace!("End processing worklist increment (has_more={has_more})");
+ Ok(GcProgress::Continue)
}
/// Clean up dead externrefs by iterating the idle semi-space's externref
@@ -1077,57 +1120,47 @@ impl CopyingCollection<'_> {
log::trace!("End sweeping `externref`s");
Ok(())
}
+
+ /// Handle the `SweepExternRefs` phase: sweep dead externrefs, resize
+ /// semi-spaces, and transition to `Done`.
+ fn finish_collection(&mut self) -> Result<GcProgress> {
+ log::trace!("Copying collection: sweeping externrefs");
+
+ assert!(self.heap.active_space_start <= self.heap.bump_ptr());
+ assert!(self.heap.bump_ptr() <= self.heap.active_space_end());
+ assert!(self.heap.idle_space_start <= self.heap.idle_space_end);
+ assert!(
+ self.heap.active_space_end() <= self.heap.idle_space_start
+ || self.heap.idle_space_end <= self.heap.active_space_start
+ );
+
+ self.sweep_extern_refs()?;
+ self.heap.resize_semi_spaces();
+
+ debug_assert_eq!(
+ self.heap.active_space_end() - self.heap.active_space_start,
+ self.heap.idle_space_end - self.heap.idle_space_start,
+ "the active and idle spaces should be the same size"
+ );
+
+ if cfg!(gc_zeal) {
+ let idle_start = usize::try_from(self.heap.idle_space_start).unwrap();
+ let idle_end = usize::try_from(self.heap.idle_space_end).unwrap();
+ self.heap.heap_slice_mut()[idle_start..idle_end].fill(POISON);
+ }
+
+ log::trace!("End copying collection");
+ self.phase = CopyingCollectionPhase::Done;
+ Ok(GcProgress::Complete)
+ }
}
impl GarbageCollection<'_> for CopyingCollection<'_> {
fn collect_increment(&mut self) -> Result<GcProgress> {
match self.phase {
- CopyingCollectionPhase::Collect => {
- log::trace!("Begin copying collection");
-
- assert!(self.heap.active_space_start <= self.heap.bump_ptr());
- assert!(self.heap.bump_ptr() <= self.heap.active_space_end());
- assert!(self.heap.idle_space_start <= self.heap.idle_space_end);
- assert!(
- self.heap.active_space_end() <= self.heap.idle_space_start
- || self.heap.idle_space_end <= self.heap.active_space_start
- );
-
- // Flip the semi-spaces.
- self.heap.flip();
- self.heap.initialize_worklist();
-
- self.process_roots()?;
- self.process_worklist()?;
-
- assert!(self.heap.active_space_start <= self.heap.bump_ptr());
- assert!(self.heap.bump_ptr() <= self.heap.active_space_end());
- assert!(self.heap.idle_space_start <= self.heap.idle_space_end);
- assert!(
- self.heap.active_space_end() <= self.heap.idle_space_start
- || self.heap.idle_space_end <= self.heap.active_space_start
- );
-
- self.sweep_extern_refs()?;
- self.heap.resize_semi_spaces();
-
- debug_assert_eq!(
- self.heap.active_space_end() - self.heap.active_space_start,
- self.heap.idle_space_end - self.heap.idle_space_start,
- "the active and idle spaces should be the same size"
- );
-
- // Poison the idle space so stale accesses are detectable.
- if cfg!(gc_zeal) {
- let idle_start = usize::try_from(self.heap.idle_space_start).unwrap();
- let idle_end = usize::try_from(self.heap.idle_space_end).unwrap();
- self.heap.heap_slice_mut()[idle_start..idle_end].fill(POISON);
- }
-
- log::trace!("End copying collection");
- self.phase = CopyingCollectionPhase::Done;
- Ok(GcProgress::Complete)
- }
+ CopyingCollectionPhase::ProcessRoots => self.begin_collection(),
+ CopyingCollectionPhase::ProcessWorklist => self.process_worklist_increment(),
+ CopyingCollectionPhase::SweepExternRefs => self.finish_collection(),
CopyingCollectionPhase::Done => Ok(GcProgress::Complete),
}
}
diff --git a/tests/all/gc.rs b/tests/all/gc.rs
index b5d5886351..7301fe6bf2 100644
--- a/tests/all/gc.rs
+++ b/tests/all/gc.rs
@@ -1,4 +1,4 @@
-use super::{gc_store, ref_types_module};
+use super::{async_functions::CountPending, gc_store, ref_types_module};
use std::num::NonZeroU32;
use std::sync::Arc;
use std::sync::atomic::Ordering;
@@ -3445,3 +3445,68 @@ fn miri_gc_smoke_test() -> Result<()> {
Ok(())
}
+
+#[tokio::test]
+#[cfg_attr(miri, ignore)]
+async fn copying_collector_async_gc_yields() -> Result<()> {
+ let _ = env_logger::try_init();
+
+ if crate::no_hog_memory() {
+ return Ok(());
+ }
+
+ let mut config = Config::new();
+ config.wasm_gc(true);
+ config.wasm_function_references(true);
+ config.collector(Collector::Copying);
+
+ let engine = Engine::new(&config)?;
+
+ const NUM_OBJECTS: usize = 100_000;
+ const OBJECTS_PER_INCREMENT: usize = 16_384;
+
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (type $node (struct (field i32) (field (ref null $node))))
+ (global $list (export "list") (mut (ref null $node)) (ref.null $node))
+
+ (func (export "build") (param $n i32)
+ (local $i i32)
+ (local.set $i (i32.const 0))
+ (block $done
+ (loop $loop
+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))
+ (global.set $list
+ (struct.new $node
+ (local.get $i)
+ (global.get $list)
+ )
+ )
+ (local.set $i (i32.add (local.get $i) (i32.const 1)))
+ (br $loop)
+ )
+ )
+ )
+ )
+ "#,
+ )?;
+
+ let mut store = Store::new(&engine, ());
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let build = instance.get_typed_func::<i32, ()>(&mut store, "build")?;
+ build.call(&mut store, NUM_OBJECTS as i32)?;
+
+ let gc_future = store.gc_async(None);
+ let (result, pending_count) = CountPending::new(Box::pin(gc_future)).await;
+ result?;
+
+ let min_expected = NUM_OBJECTS / OBJECTS_PER_INCREMENT;
+ assert!(
+ pending_count >= min_expected,
+ "expected at least {min_expected} yields but got {pending_count}",
+ );
+
+ Ok(())
+}