Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Defer DRC drops in host-side `write_gc_ref` (#13842)

Alex Crichton66f269153d05
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index e62a4dedee..47aeeafa41 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -203,8 +203,7 @@ impl GcStore {
// (that is, they are both either null or `i31ref`s) then we can skip
// the GC barrier.
if Self::needs_write_barrier(destination, source) {
- self.gc_heap
- .write_gc_ref(&mut self.host_data_table, destination, source)?;
+ self.gc_heap.write_gc_ref(destination, source)?;
} else {
*destination = source.map(|s| s.copy_i31());
}
@@ -214,7 +213,7 @@ impl GcStore {
/// Drop the given GC reference, performing drop barriers as necessary.
pub fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {
if !gc_ref.is_i31() {
- self.gc_heap.drop_gc_ref(&mut self.host_data_table, gc_ref);
+ self.gc_heap.drop_gc_ref(gc_ref);
}
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
index 575dc6f3f0..f90d03adcb 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
@@ -760,7 +760,6 @@ unsafe impl GcHeap for CopyingHeap {
fn write_gc_ref(
&mut self,
- _host_data_table: &mut ExternRefHostDataTable,
destination: &mut Option<VMGcRef>,
source: Option<&VMGcRef>,
) -> Result<()> {
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index 61be8250cd..6bed2e510d 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -275,23 +275,46 @@ impl DrcHeap {
Ok(())
}
- /// Decrement the ref count for the associated object.
+ /// Decrements the reference count of `gc_ref`, if applicable (e.g. not an
+ /// i31).
///
- /// If the ref count reached zero, then deallocate the object and remove its
- /// associated entry from the `host_data_table` if necessary.
- ///
- /// This uses an explicit stack, rather than recursion, for the scenario
- /// where dropping one object means that the ref count for another object
- /// that it referenced reaches zero.
- fn dec_ref_and_maybe_dealloc(
- &mut self,
- host_data_table: &mut ExternRefHostDataTable,
- gc_ref: &VMGcRef,
- ) -> Result<()> {
+ /// If the reference count reaches 0 then this will enqueue the reference to
+ /// get deallocated at a later time within
+ /// `self.tracing_allocs.dec_ref_stack`.
+ fn dec_ref_and_maybe_enqueue_dealloc(&mut self, gc_ref: &VMGcRef) -> Result<()> {
if gc_ref.is_i31() {
return Ok(());
}
+ let drc_header = self.index_mut(drc_ref(gc_ref))?;
+ log::trace!(
+ "decrement {gc_ref:#p} ref count -> {}",
+ drc_header.ref_count - 1
+ );
+ if drc_header.dec_ref() {
+ // The `process_dec_ref_stack` loop below always starts out with a
+ // decrement, so reset the reference count back to 1 so that knows
+ // it has an exclusive copy.
+ drc_header.ref_count = 1;
+ match &mut self.tracing_allocs {
+ Some(allocs) => allocs.dec_ref_stack.push(gc_ref.unchecked_copy()),
+ None => bail_bug!("expected allocations to be present"),
+ }
+ }
+ Ok(())
+ }
+
+ /// Process all elements on `self.tracing_allocs.dec_ref_stack` and
+ /// deallocate them.
+ ///
+ /// This will `dec_ref` all elements on the stack and recursively deallocate
+ /// any that have reached a reference count of 0. Note that this is modeled
+ /// with an explicit stack rather than a recursive function to ensure that
+ /// this cannot overflow the host stack.
+ fn process_dec_ref_stack(
+ &mut self,
+ host_data_table: &mut ExternRefHostDataTable,
+ ) -> Result<()> {
let allocs = match self.tracing_allocs.take() {
Some(allocs) => allocs,
None => bail_bug!("allocs missing during tracing"),
@@ -305,12 +328,12 @@ impl DrcHeap {
let large_array_stack = &mut allocs.large_array_dec_ref_stack;
let to_dealloc = &mut allocs.to_dealloc;
- debug_assert!(stack.is_empty());
+ // Assert that our temporary stacks are all empty, but note that
+ // `dec_ref_stack`, our `stack` local variable, may not be empty as it
+ // might have dangling references inserted by `write_gc_ref`.
debug_assert!(large_array_stack.is_empty());
debug_assert!(to_dealloc.is_empty());
- stack.push(gc_ref.unchecked_copy());
-
while !stack.is_empty() || !large_array_stack.is_empty() {
while let Some(gc_ref) = stack.pop() {
debug_assert!(!gc_ref.is_i31());
@@ -673,7 +696,15 @@ impl DrcHeap {
}
self.vmctx_data
.decrement_current_over_approximated_stack_roots_len();
- self.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref)?;
+ self.dec_ref_and_maybe_enqueue_dealloc(&gc_ref)?;
+ }
+
+ // If some references have reached a 0 reference count then now's the
+ // time to clear them all out.
+ if let Some(allocs) = &self.tracing_allocs
+ && !allocs.dec_ref_stack.is_empty()
+ {
+ self.process_dec_ref_stack(host_data_table)?;
}
self.vmctx_data
@@ -905,11 +936,12 @@ unsafe impl GcHeap for DrcHeap {
*allocated_bytes = 0;
trace_infos.clear();
- debug_assert!(tracing_allocs.as_ref().is_some_and(|allocs| {
- allocs.dec_ref_stack.is_empty()
- && allocs.large_array_dec_ref_stack.is_empty()
- && allocs.to_dealloc.is_empty()
- }));
+ debug_assert!(tracing_allocs.is_some());
+ if let Some(allocs) = tracing_allocs {
+ allocs.dec_ref_stack.clear();
+ debug_assert!(allocs.large_array_dec_ref_stack.is_empty());
+ debug_assert!(allocs.to_dealloc.is_empty());
+ }
memory.take().unwrap()
}
@@ -952,7 +984,6 @@ unsafe impl GcHeap for DrcHeap {
fn write_gc_ref(
&mut self,
- host_data_table: &mut ExternRefHostDataTable,
destination: &mut Option<VMGcRef>,
source: Option<&VMGcRef>,
) -> Result<()> {
@@ -961,10 +992,11 @@ unsafe impl GcHeap for DrcHeap {
self.inc_ref(src)?;
}
- // Decrement the ref count of the value being overwritten and, if
- // necessary, deallocate the GC object.
+ // Decrement the ref count of the value being overwritten. If the
+ // reference count reaches 0 then re-increment it back to one and queue
+ // this up to get deallocated later on during a GC cycle.
if let Some(dest) = destination {
- self.dec_ref_and_maybe_dealloc(host_data_table, dest)?;
+ self.dec_ref_and_maybe_enqueue_dealloc(dest)?;
}
// Do the actual write.
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
index 9ab201e61b..7b93e6eb98 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
@@ -240,7 +240,6 @@ unsafe impl GcHeap for NullHeap {
fn write_gc_ref(
&mut self,
- _host_data_table: &mut ExternRefHostDataTable,
destination: &mut Option<VMGcRef>,
source: Option<&VMGcRef>,
) -> Result<()> {
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
index 5cbfa4200c..e69ca8d9b1 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
@@ -169,7 +169,7 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// failures such as panics or incorrect results.
///
/// The given `gc_ref` should not be used again.
- fn drop_gc_ref(&mut self, host_data_table: &mut ExternRefHostDataTable, gc_ref: VMGcRef) {
+ fn drop_gc_ref(&mut self, gc_ref: VMGcRef) {
let mut dest = Some(gc_ref);
// Similar to `clone_gc_ref` not being fallible this method,
@@ -178,7 +178,7 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
// corrupted at this point it'll get a ltitle more corrupted from this
// operation, but that's the tradeoff we're making ignoring the error
// here.
- if let Err(e) = self.write_gc_ref(host_data_table, &mut dest, None) {
+ if let Err(e) = self.write_gc_ref(&mut dest, None) {
if cfg!(debug_assertions) {
panic!("heap corruption detected: {e}");
}
@@ -199,7 +199,6 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// but may result in general failures such as panics or incorrect results.
fn write_gc_ref(
&mut self,
- host_data_table: &mut ExternRefHostDataTable,
destination: &mut Option<VMGcRef>,
source: Option<&VMGcRef>,
) -> Result<()>;