A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Defer DRC drops in host-side `write_gc_ref` (#13842)
Alex Crichton66f269153d05
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rsindex e62a4dedee..47aeeafa41 100644--- a/crates/wasmtime/src/runtime/vm/gc.rs+++ b/crates/wasmtime/src/runtime/vm/gc.rs@@ -203,8 +203,7 @@ impl GcStore {// (that is, they are both either null or `i31ref`s) then we can skip// the GC barrier.if Self::needs_write_barrier(destination, source) {- self.gc_heap- .write_gc_ref(&mut self.host_data_table, destination, source)?;+ self.gc_heap.write_gc_ref(destination, source)?;} else {*destination = source.map(|s| s.copy_i31());}@@ -214,7 +213,7 @@ impl GcStore {/// Drop the given GC reference, performing drop barriers as necessary.pub fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {if !gc_ref.is_i31() {- self.gc_heap.drop_gc_ref(&mut self.host_data_table, gc_ref);+ self.gc_heap.drop_gc_ref(gc_ref);}}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rsindex 575dc6f3f0..f90d03adcb 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs@@ -760,7 +760,6 @@ unsafe impl GcHeap for CopyingHeap {fn write_gc_ref(&mut self,- _host_data_table: &mut ExternRefHostDataTable,destination: &mut Option<VMGcRef>,source: Option<&VMGcRef>,) -> Result<()> {diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rsindex 61be8250cd..6bed2e510d 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs@@ -275,23 +275,46 @@ impl DrcHeap {Ok(())}- /// Decrement the ref count for the associated object.+ /// Decrements the reference count of `gc_ref`, if applicable (e.g. not an+ /// i31).///- /// If the ref count reached zero, then deallocate the object and remove its- /// associated entry from the `host_data_table` if necessary.- ///- /// This uses an explicit stack, rather than recursion, for the scenario- /// where dropping one object means that the ref count for another object- /// that it referenced reaches zero.- fn dec_ref_and_maybe_dealloc(- &mut self,- host_data_table: &mut ExternRefHostDataTable,- gc_ref: &VMGcRef,- ) -> Result<()> {+ /// If the reference count reaches 0 then this will enqueue the reference to+ /// get deallocated at a later time within+ /// `self.tracing_allocs.dec_ref_stack`.+ fn dec_ref_and_maybe_enqueue_dealloc(&mut self, gc_ref: &VMGcRef) -> Result<()> {if gc_ref.is_i31() {return Ok(());}+ let drc_header = self.index_mut(drc_ref(gc_ref))?;+ log::trace!(+ "decrement {gc_ref:#p} ref count -> {}",+ drc_header.ref_count - 1+ );+ if drc_header.dec_ref() {+ // The `process_dec_ref_stack` loop below always starts out with a+ // decrement, so reset the reference count back to 1 so that knows+ // it has an exclusive copy.+ drc_header.ref_count = 1;+ match &mut self.tracing_allocs {+ Some(allocs) => allocs.dec_ref_stack.push(gc_ref.unchecked_copy()),+ None => bail_bug!("expected allocations to be present"),+ }+ }+ Ok(())+ }++ /// Process all elements on `self.tracing_allocs.dec_ref_stack` and+ /// deallocate them.+ ///+ /// This will `dec_ref` all elements on the stack and recursively deallocate+ /// any that have reached a reference count of 0. Note that this is modeled+ /// with an explicit stack rather than a recursive function to ensure that+ /// this cannot overflow the host stack.+ fn process_dec_ref_stack(+ &mut self,+ host_data_table: &mut ExternRefHostDataTable,+ ) -> Result<()> {let allocs = match self.tracing_allocs.take() {Some(allocs) => allocs,None => bail_bug!("allocs missing during tracing"),@@ -305,12 +328,12 @@ impl DrcHeap {let large_array_stack = &mut allocs.large_array_dec_ref_stack;let to_dealloc = &mut allocs.to_dealloc;- debug_assert!(stack.is_empty());+ // Assert that our temporary stacks are all empty, but note that+ // `dec_ref_stack`, our `stack` local variable, may not be empty as it+ // might have dangling references inserted by `write_gc_ref`.debug_assert!(large_array_stack.is_empty());debug_assert!(to_dealloc.is_empty());- stack.push(gc_ref.unchecked_copy());-while !stack.is_empty() || !large_array_stack.is_empty() {while let Some(gc_ref) = stack.pop() {debug_assert!(!gc_ref.is_i31());@@ -673,7 +696,15 @@ impl DrcHeap {}self.vmctx_data.decrement_current_over_approximated_stack_roots_len();- self.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref)?;+ self.dec_ref_and_maybe_enqueue_dealloc(&gc_ref)?;+ }++ // If some references have reached a 0 reference count then now's the+ // time to clear them all out.+ if let Some(allocs) = &self.tracing_allocs+ && !allocs.dec_ref_stack.is_empty()+ {+ self.process_dec_ref_stack(host_data_table)?;}self.vmctx_data@@ -905,11 +936,12 @@ unsafe impl GcHeap for DrcHeap {*allocated_bytes = 0;trace_infos.clear();- debug_assert!(tracing_allocs.as_ref().is_some_and(|allocs| {- allocs.dec_ref_stack.is_empty()- && allocs.large_array_dec_ref_stack.is_empty()- && allocs.to_dealloc.is_empty()- }));+ debug_assert!(tracing_allocs.is_some());+ if let Some(allocs) = tracing_allocs {+ allocs.dec_ref_stack.clear();+ debug_assert!(allocs.large_array_dec_ref_stack.is_empty());+ debug_assert!(allocs.to_dealloc.is_empty());+ }memory.take().unwrap()}@@ -952,7 +984,6 @@ unsafe impl GcHeap for DrcHeap {fn write_gc_ref(&mut self,- host_data_table: &mut ExternRefHostDataTable,destination: &mut Option<VMGcRef>,source: Option<&VMGcRef>,) -> Result<()> {@@ -961,10 +992,11 @@ unsafe impl GcHeap for DrcHeap {self.inc_ref(src)?;}- // Decrement the ref count of the value being overwritten and, if- // necessary, deallocate the GC object.+ // Decrement the ref count of the value being overwritten. If the+ // reference count reaches 0 then re-increment it back to one and queue+ // this up to get deallocated later on during a GC cycle.if let Some(dest) = destination {- self.dec_ref_and_maybe_dealloc(host_data_table, dest)?;+ self.dec_ref_and_maybe_enqueue_dealloc(dest)?;}// Do the actual write.diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rsindex 9ab201e61b..7b93e6eb98 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs@@ -240,7 +240,6 @@ unsafe impl GcHeap for NullHeap {fn write_gc_ref(&mut self,- _host_data_table: &mut ExternRefHostDataTable,destination: &mut Option<VMGcRef>,source: Option<&VMGcRef>,) -> Result<()> {diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rsindex 5cbfa4200c..e69ca8d9b1 100644--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs@@ -169,7 +169,7 @@ pub unsafe trait GcHeap: 'static + Send + Sync {/// failures such as panics or incorrect results.////// The given `gc_ref` should not be used again.- fn drop_gc_ref(&mut self, host_data_table: &mut ExternRefHostDataTable, gc_ref: VMGcRef) {+ fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {let mut dest = Some(gc_ref);// Similar to `clone_gc_ref` not being fallible this method,@@ -178,7 +178,7 @@ pub unsafe trait GcHeap: 'static + Send + Sync {// corrupted at this point it'll get a ltitle more corrupted from this// operation, but that's the tradeoff we're making ignoring the error// here.- if let Err(e) = self.write_gc_ref(host_data_table, &mut dest, None) {+ if let Err(e) = self.write_gc_ref(&mut dest, None) {if cfg!(debug_assertions) {panic!("heap corruption detected: {e}");}@@ -199,7 +199,6 @@ pub unsafe trait GcHeap: 'static + Send + Sync {/// but may result in general failures such as panics or incorrect results.fn write_gc_ref(&mut self,- host_data_table: &mut ExternRefHostDataTable,destination: &mut Option<VMGcRef>,source: Option<&VMGcRef>,) -> Result<()>;