A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Perform multiple collection increments in the copying collector (#13491)
Nick Fitzgerald2a4a19bfc4f8
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rsindex 685f506327..49c126686d 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs@@ -951,7 +951,7 @@ unsafe impl GcHeap for CopyingHeap {roots: Some(roots),host_data_table,heap: self,- phase: CopyingCollectionPhase::Collect,+ phase: CopyingCollectionPhase::ProcessRoots,})}@@ -1009,6 +1009,8 @@ unsafe impl GcHeap for CopyingHeap {}}+const OBJECTS_PER_INCREMENT: usize = 16_384;+struct CopyingCollection<'a> {roots: Option<GcRootsIter<'a>>,host_data_table: &'a mut ExternRefHostDataTable,@@ -1017,7 +1019,9 @@ struct CopyingCollection<'a> {}enum CopyingCollectionPhase {- Collect,+ ProcessRoots,+ ProcessWorklist,+ SweepExternRefs,Done,}@@ -1040,17 +1044,56 @@ impl CopyingCollection<'_> {Ok(())}- /// Scan all grey objects until the worklist is empty.- fn process_worklist(&mut self) -> Result<()> {- log::trace!("Begin processing worklist");+ /// Handle the `ProcessRoots` phase: flip semi-spaces, initialize the+ /// worklist, forward all roots, then transition to `ProcessWorklist`.+ fn begin_collection(&mut self) -> Result<GcProgress> {+ log::trace!("Begin copying collection: processing roots");++ assert!(self.heap.active_space_start <= self.heap.bump_ptr());+ assert!(self.heap.bump_ptr() <= self.heap.active_space_end());+ assert!(self.heap.idle_space_start <= self.heap.idle_space_end);+ assert!(+ self.heap.active_space_end() <= self.heap.idle_space_start+ || self.heap.idle_space_end <= self.heap.active_space_start+ );++ self.heap.flip();+ self.heap.initialize_worklist();++ self.process_roots()?;++ self.phase = CopyingCollectionPhase::ProcessWorklist;+ Ok(GcProgress::Continue)+ }++ /// Scan up to `OBJECTS_PER_INCREMENT` grey objects from the worklist.+ ///+ /// When the worklist is fully drained, transitions to `SweepExternRefs`.+ /// Always returns `GcProgress::Continue` so the caller yields after each+ /// increment regardless of whether the worklist still has items.+ fn process_worklist_increment(&mut self) -> Result<GcProgress> {+ log::trace!("Begin processing worklist increment");let trace_infos = mem::take(&mut self.heap.trace_infos);- while let Some(gc_ref) = self.heap.worklist_pop()? {- debug_assert!(self.heap.is_in_active_space(gc_ref.heap_index()?.get()));- self.heap.scan(&gc_ref, &trace_infos)?;- }+ let mut count = 0;+ let has_more = loop {+ if count >= OBJECTS_PER_INCREMENT {+ break true;+ }+ match self.heap.worklist_pop()? {+ Some(gc_ref) => {+ debug_assert!(self.heap.is_in_active_space(gc_ref.heap_index()?.get()));+ self.heap.scan(&gc_ref, &trace_infos)?;+ count += 1;+ }+ None => break false,+ }+ };self.heap.trace_infos = trace_infos;- log::trace!("End processing worklist");- Ok(())+ if !has_more {+ self.phase = CopyingCollectionPhase::SweepExternRefs;+ }+ log::trace!("End processing worklist increment (has_more={has_more})");+ Ok(GcProgress::Continue)}/// Clean up dead externrefs by iterating the idle semi-space's externref@@ -1077,57 +1120,47 @@ impl CopyingCollection<'_> {log::trace!("End sweeping `externref`s");Ok(())}++ /// Handle the `SweepExternRefs` phase: sweep dead externrefs, resize+ /// semi-spaces, and transition to `Done`.+ fn finish_collection(&mut self) -> Result<GcProgress> {+ log::trace!("Copying collection: sweeping externrefs");++ assert!(self.heap.active_space_start <= self.heap.bump_ptr());+ assert!(self.heap.bump_ptr() <= self.heap.active_space_end());+ assert!(self.heap.idle_space_start <= self.heap.idle_space_end);+ assert!(+ self.heap.active_space_end() <= self.heap.idle_space_start+ || self.heap.idle_space_end <= self.heap.active_space_start+ );++ self.sweep_extern_refs()?;+ self.heap.resize_semi_spaces();++ debug_assert_eq!(+ self.heap.active_space_end() - self.heap.active_space_start,+ self.heap.idle_space_end - self.heap.idle_space_start,+ "the active and idle spaces should be the same size"+ );++ if cfg!(gc_zeal) {+ let idle_start = usize::try_from(self.heap.idle_space_start).unwrap();+ let idle_end = usize::try_from(self.heap.idle_space_end).unwrap();+ self.heap.heap_slice_mut()[idle_start..idle_end].fill(POISON);+ }++ log::trace!("End copying collection");+ self.phase = CopyingCollectionPhase::Done;+ Ok(GcProgress::Complete)+ }}impl GarbageCollection<'_> for CopyingCollection<'_> {fn collect_increment(&mut self) -> Result<GcProgress> {match self.phase {- CopyingCollectionPhase::Collect => {- log::trace!("Begin copying collection");-- assert!(self.heap.active_space_start <= self.heap.bump_ptr());- assert!(self.heap.bump_ptr() <= self.heap.active_space_end());- assert!(self.heap.idle_space_start <= self.heap.idle_space_end);- assert!(- self.heap.active_space_end() <= self.heap.idle_space_start- || self.heap.idle_space_end <= self.heap.active_space_start- );-- // Flip the semi-spaces.- self.heap.flip();- self.heap.initialize_worklist();-- self.process_roots()?;- self.process_worklist()?;-- assert!(self.heap.active_space_start <= self.heap.bump_ptr());- assert!(self.heap.bump_ptr() <= self.heap.active_space_end());- assert!(self.heap.idle_space_start <= self.heap.idle_space_end);- assert!(- self.heap.active_space_end() <= self.heap.idle_space_start- || self.heap.idle_space_end <= self.heap.active_space_start- );-- self.sweep_extern_refs()?;- self.heap.resize_semi_spaces();-- debug_assert_eq!(- self.heap.active_space_end() - self.heap.active_space_start,- self.heap.idle_space_end - self.heap.idle_space_start,- "the active and idle spaces should be the same size"- );-- // Poison the idle space so stale accesses are detectable.- if cfg!(gc_zeal) {- let idle_start = usize::try_from(self.heap.idle_space_start).unwrap();- let idle_end = usize::try_from(self.heap.idle_space_end).unwrap();- self.heap.heap_slice_mut()[idle_start..idle_end].fill(POISON);- }-- log::trace!("End copying collection");- self.phase = CopyingCollectionPhase::Done;- Ok(GcProgress::Complete)- }+ CopyingCollectionPhase::ProcessRoots => self.begin_collection(),+ CopyingCollectionPhase::ProcessWorklist => self.process_worklist_increment(),+ CopyingCollectionPhase::SweepExternRefs => self.finish_collection(),CopyingCollectionPhase::Done => Ok(GcProgress::Complete),}}diff --git a/tests/all/gc.rs b/tests/all/gc.rsindex b5d5886351..7301fe6bf2 100644--- a/tests/all/gc.rs+++ b/tests/all/gc.rs@@ -1,4 +1,4 @@-use super::{gc_store, ref_types_module};+use super::{async_functions::CountPending, gc_store, ref_types_module};use std::num::NonZeroU32;use std::sync::Arc;use std::sync::atomic::Ordering;@@ -3445,3 +3445,68 @@ fn miri_gc_smoke_test() -> Result<()> {Ok(())}++#[tokio::test]+#[cfg_attr(miri, ignore)]+async fn copying_collector_async_gc_yields() -> Result<()> {+ let _ = env_logger::try_init();++ if crate::no_hog_memory() {+ return Ok(());+ }++ let mut config = Config::new();+ config.wasm_gc(true);+ config.wasm_function_references(true);+ config.collector(Collector::Copying);++ let engine = Engine::new(&config)?;++ const NUM_OBJECTS: usize = 100_000;+ const OBJECTS_PER_INCREMENT: usize = 16_384;++ let module = Module::new(+ &engine,+ r#"+ (module+ (type $node (struct (field i32) (field (ref null $node))))+ (global $list (export "list") (mut (ref null $node)) (ref.null $node))++ (func (export "build") (param $n i32)+ (local $i i32)+ (local.set $i (i32.const 0))+ (block $done+ (loop $loop+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))+ (global.set $list+ (struct.new $node+ (local.get $i)+ (global.get $list)+ )+ )+ (local.set $i (i32.add (local.get $i) (i32.const 1)))+ (br $loop)+ )+ )+ )+ )+ "#,+ )?;++ let mut store = Store::new(&engine, ());+ let instance = Instance::new(&mut store, &module, &[])?;+ let build = instance.get_typed_func::<i32, ()>(&mut store, "build")?;+ build.call(&mut store, NUM_OBJECTS as i32)?;++ let gc_future = store.gc_async(None);+ let (result, pending_count) = CountPending::new(Box::pin(gc_future)).await;+ result?;++ let min_expected = NUM_OBJECTS / OBJECTS_PER_INCREMENT;+ assert!(+ pending_count >= min_expected,+ "expected at least {min_expected} yields but got {pending_count}",+ );++ Ok(())+}