Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

GC: replace ManuallyRooted with OwnedRooted. (#11514)

Chris Fallin9826719ae049
diff --git a/crates/c-api/include/wasmtime/val.h b/crates/c-api/include/wasmtime/val.h
index bdac5b9c76..4c1af27a5c 100644
--- a/crates/c-api/include/wasmtime/val.h
+++ b/crates/c-api/include/wasmtime/val.h
@@ -32,6 +32,10 @@ extern "C" {
* Anyref values are required to be explicitly unrooted via
* #wasmtime_anyref_unroot to enable them to be garbage-collected.
*
+ * If you do not unroot the value, *even if you free the corresponding
+ * Store*, there will be some memory leaked, because GC roots use a
+ * separate allocation to track liveness.
+ *
* Null anyref values are represented by this structure and can be tested and
* created with the `wasmtime_anyref_is_null` and `wasmtime_anyref_set_null`
* functions.
@@ -44,6 +48,8 @@ typedef struct wasmtime_anyref {
uint32_t __private1;
/// Internal to Wasmtime.
uint32_t __private2;
+ /// Internal to Wasmtime.
+ void *__private3;
} wasmtime_anyref_t;
/// \brief Helper function to initialize the `ref` provided to a null anyref
@@ -179,6 +185,8 @@ typedef struct wasmtime_externref {
uint32_t __private1;
/// Internal to Wasmtime.
uint32_t __private2;
+ /// Internal to Wasmtime.
+ void *__private3;
} wasmtime_externref_t;
/// \brief Helper function to initialize the `ref` provided to a null externref
@@ -214,6 +222,10 @@ static inline bool wasmtime_externref_is_null(const wasmtime_externref_t *ref) {
* in the future with #wasmtime_externref_unroot. If `false` is returned then
* the host wasn't able to create more GC values at this time. Performing a GC
* may free up enough space to try again.
+ *
+ * If you do not unroot the value, *even if you free the corresponding
+ * Store*, there will be some memory leaked, because GC roots use a
+ * separate allocation to track liveness.
*/
WASM_API_EXTERN bool wasmtime_externref_new(wasmtime_context_t *context,
void *data,
@@ -418,7 +430,10 @@ typedef union wasmtime_val_raw {
// Assert that the shape of this type is as expected since it needs to match
// Rust.
static inline void __wasmtime_val_assertions() {
- static_assert(sizeof(wasmtime_valunion_t) == 16, "should be 16-bytes large");
+ static_assert(sizeof(wasmtime_valunion_t) >= 16 &&
+ sizeof(wasmtime_valunion_t) <= 24,
+ "should be 16 bytes plus a pointer large (plus alignment on "
+ "some platforms)");
static_assert(__alignof(wasmtime_valunion_t) == 8,
"should be 8-byte aligned");
static_assert(sizeof(wasmtime_val_raw_t) == 16, "should be 16 bytes large");
@@ -437,6 +452,10 @@ static inline void __wasmtime_val_assertions() {
* consume a #wasmtime_val_t do not take ownership, but APIs that return
* #wasmtime_val_t require that #wasmtime_val_unroot is called to clean up
* any possible GC roots in the value.
+ *
+ * If you do not unroot the value, *even if you free the corresponding
+ * Store*, there will be some memory leaked, because GC roots use a
+ * separate allocation to track liveness.
*/
typedef struct wasmtime_val {
/// Discriminant of which field of #of is valid.
diff --git a/crates/c-api/src/ref.rs b/crates/c-api/src/ref.rs
index ff2ee34bf3..e3a5b52d65 100644
--- a/crates/c-api/src/ref.rs
+++ b/crates/c-api/src/ref.rs
@@ -1,6 +1,6 @@
use crate::{WasmtimeStoreContextMut, abort};
use std::{mem::MaybeUninit, num::NonZeroU64, os::raw::c_void, ptr};
-use wasmtime::{AnyRef, ExternRef, I31, ManuallyRooted, Ref, RootScope, Val};
+use wasmtime::{AnyRef, ExternRef, I31, OwnedRooted, Ref, RootScope, Val};
/// `*mut wasm_ref_t` is a reference type (`externref` or `funcref`), as seen by
/// the C API. Because we do not have a uniform representation for `funcref`s
@@ -188,7 +188,7 @@ pub extern "C" fn wasm_foreign_new(_store: &crate::wasm_store_t) -> Box<wasm_for
/// this is dispatched internally. Null anyref values are represented with a
/// `store_id` of zero, and otherwise the `rooted` field is valid.
///
-/// Note that this relies on the Wasmtime definition of `ManuallyRooted` to have
+/// Note that this relies on the Wasmtime definition of `OwnedRooted` to have
/// a 64-bit store_id first.
macro_rules! ref_wrapper {
($wasmtime:ident => $c:ident) => {
@@ -196,33 +196,50 @@ macro_rules! ref_wrapper {
store_id: u64,
a: u32,
b: u32,
+ c: *const (),
}
impl $c {
- pub unsafe fn as_wasmtime(&self) -> Option<ManuallyRooted<$wasmtime>> {
+ pub unsafe fn as_wasmtime(&self) -> Option<OwnedRooted<$wasmtime>> {
let store_id = NonZeroU64::new(self.store_id)?;
- Some(ManuallyRooted::from_raw_parts_for_c_api(
- store_id, self.a, self.b,
+ Some(OwnedRooted::from_borrowed_raw_parts_for_c_api(
+ store_id, self.a, self.b, self.c,
+ ))
+ }
+
+ pub unsafe fn from_wasmtime(self) -> Option<OwnedRooted<$wasmtime>> {
+ let store_id = NonZeroU64::new(self.store_id)?;
+ Some(OwnedRooted::from_owned_raw_parts_for_c_api(
+ store_id, self.a, self.b, self.c,
))
}
}
- impl From<Option<ManuallyRooted<$wasmtime>>> for $c {
- fn from(rooted: Option<ManuallyRooted<$wasmtime>>) -> $c {
+ impl From<Option<OwnedRooted<$wasmtime>>> for $c {
+ fn from(rooted: Option<OwnedRooted<$wasmtime>>) -> $c {
let mut ret = $c {
store_id: 0,
a: 0,
b: 0,
+ c: core::ptr::null(),
};
if let Some(rooted) = rooted {
- let (store_id, a, b) = rooted.into_parts_for_c_api();
+ let (store_id, a, b, c) = rooted.into_parts_for_c_api();
ret.store_id = store_id.get();
ret.a = a;
ret.b = b;
+ ret.c = c;
}
ret
}
}
+
+ // SAFETY: The `*const ()` comes from (and is converted back
+ // into) an `Arc<()>`, and is only accessed as such, so this
+ // type is both Send and Sync. These constraints are necessary
+ // in the async machinery in this crate.
+ unsafe impl Send for $c {}
+ unsafe impl Sync for $c {}
};
}
@@ -231,21 +248,21 @@ ref_wrapper!(ExternRef => wasmtime_externref_t);
#[unsafe(no_mangle)]
pub unsafe extern "C" fn wasmtime_anyref_clone(
- cx: WasmtimeStoreContextMut<'_>,
+ _cx: WasmtimeStoreContextMut<'_>,
anyref: Option<&wasmtime_anyref_t>,
out: &mut MaybeUninit<wasmtime_anyref_t>,
) {
- let anyref = anyref.and_then(|a| a.as_wasmtime()).map(|a| a.clone(cx));
+ let anyref = anyref.and_then(|a| a.as_wasmtime());
crate::initialize(out, anyref.into());
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn wasmtime_anyref_unroot(
- cx: WasmtimeStoreContextMut<'_>,
+ _cx: WasmtimeStoreContextMut<'_>,
val: Option<&mut MaybeUninit<wasmtime_anyref_t>>,
) {
- if let Some(val) = val.and_then(|v| v.assume_init_read().as_wasmtime()) {
- val.unroot(cx);
+ if let Some(val) = val.and_then(|v| v.assume_init_read().from_wasmtime()) {
+ drop(val);
}
}
@@ -266,8 +283,8 @@ pub unsafe extern "C" fn wasmtime_anyref_from_raw(
val: &mut MaybeUninit<wasmtime_anyref_t>,
) {
let mut scope = RootScope::new(cx);
- let anyref = AnyRef::from_raw(&mut scope, raw)
- .map(|a| a.to_manually_rooted(&mut scope).expect("in scope"));
+ let anyref =
+ AnyRef::from_raw(&mut scope, raw).map(|a| a.to_owned_rooted(&mut scope).expect("in scope"));
crate::initialize(val, anyref.into());
}
@@ -279,7 +296,7 @@ pub extern "C" fn wasmtime_anyref_from_i31(
) {
let mut scope = RootScope::new(cx);
let anyref = AnyRef::from_i31(&mut scope, I31::wrapping_u32(val));
- let anyref = anyref.to_manually_rooted(&mut scope).expect("in scope");
+ let anyref = anyref.to_owned_rooted(&mut scope).expect("in scope");
crate::initialize(out, Some(anyref).into())
}
@@ -290,10 +307,10 @@ pub unsafe extern "C" fn wasmtime_anyref_i31_get_u(
dst: &mut MaybeUninit<u32>,
) -> bool {
match anyref.and_then(|a| a.as_wasmtime()) {
- Some(anyref) if anyref.is_i31(&cx).expect("ManuallyRooted always in scope") => {
+ Some(anyref) if anyref.is_i31(&cx).expect("OwnedRooted always in scope") => {
let val = anyref
.unwrap_i31(&cx)
- .expect("ManuallyRooted always in scope")
+ .expect("OwnedRooted always in scope")
.get_u32();
crate::initialize(dst, val);
true
@@ -309,10 +326,10 @@ pub unsafe extern "C" fn wasmtime_anyref_i31_get_s(
dst: &mut MaybeUninit<i32>,
) -> bool {
match anyref.and_then(|a| a.as_wasmtime()) {
- Some(anyref) if anyref.is_i31(&cx).expect("ManuallyRooted always in scope") => {
+ Some(anyref) if anyref.is_i31(&cx).expect("OwnedRooted always in scope") => {
let val = anyref
.unwrap_i31(&cx)
- .expect("ManuallyRooted always in scope")
+ .expect("OwnedRooted always in scope")
.get_i32();
crate::initialize(dst, val);
true
@@ -333,7 +350,7 @@ pub extern "C" fn wasmtime_externref_new(
Ok(e) => e,
Err(_) => return false,
};
- let e = e.to_manually_rooted(&mut scope).expect("in scope");
+ let e = e.to_owned_rooted(&mut scope).expect("in scope");
crate::initialize(out, Some(e).into());
true
}
@@ -354,21 +371,21 @@ pub unsafe extern "C" fn wasmtime_externref_data(
#[unsafe(no_mangle)]
pub unsafe extern "C" fn wasmtime_externref_clone(
- cx: WasmtimeStoreContextMut<'_>,
+ _cx: WasmtimeStoreContextMut<'_>,
externref: Option<&wasmtime_externref_t>,
out: &mut MaybeUninit<wasmtime_externref_t>,
) {
- let externref = externref.and_then(|e| e.as_wasmtime()).map(|e| e.clone(cx));
+ let externref = externref.and_then(|e| e.as_wasmtime());
crate::initialize(out, externref.into());
}
#[unsafe(no_mangle)]
pub unsafe extern "C" fn wasmtime_externref_unroot(
- cx: WasmtimeStoreContextMut<'_>,
+ _cx: WasmtimeStoreContextMut<'_>,
val: Option<&mut MaybeUninit<wasmtime_externref_t>>,
) {
- if let Some(val) = val.and_then(|v| v.assume_init_read().as_wasmtime()) {
- val.unroot(cx);
+ if let Some(val) = val.and_then(|v| v.assume_init_read().from_wasmtime()) {
+ drop(val);
}
}
@@ -390,6 +407,6 @@ pub unsafe extern "C" fn wasmtime_externref_from_raw(
) {
let mut scope = RootScope::new(cx);
let rooted = ExternRef::from_raw(&mut scope, raw)
- .map(|e| e.to_manually_rooted(&mut scope).expect("in scope"));
+ .map(|e| e.to_owned_rooted(&mut scope).expect("in scope"));
crate::initialize(val, rooted.into());
}
diff --git a/crates/c-api/src/val.rs b/crates/c-api/src/val.rs
index efc53cf529..c9c4614743 100644
--- a/crates/c-api/src/val.rs
+++ b/crates/c-api/src/val.rs
@@ -152,7 +152,8 @@ pub union wasmtime_val_union {
}
const _: () = {
- assert!(std::mem::size_of::<wasmtime_val_union>() == 16);
+ // This is forced to 24 or 20 bytes by `anyref` and `externref`.
+ assert!(std::mem::size_of::<wasmtime_val_union>() <= 24);
assert!(std::mem::align_of::<wasmtime_val_union>() == std::mem::align_of::<u64>());
};
@@ -235,15 +236,13 @@ impl wasmtime_val_t {
Val::AnyRef(a) => wasmtime_val_t {
kind: crate::WASMTIME_ANYREF,
of: wasmtime_val_union {
- anyref: ManuallyDrop::new(a.and_then(|a| a.to_manually_rooted(cx).ok()).into()),
+ anyref: ManuallyDrop::new(a.and_then(|a| a.to_owned_rooted(cx).ok()).into()),
},
},
Val::ExternRef(e) => wasmtime_val_t {
kind: crate::WASMTIME_EXTERNREF,
of: wasmtime_val_union {
- externref: ManuallyDrop::new(
- e.and_then(|e| e.to_manually_rooted(cx).ok()).into(),
- ),
+ externref: ManuallyDrop::new(e.and_then(|e| e.to_owned_rooted(cx).ok()).into()),
},
},
Val::FuncRef(func) => wasmtime_val_t {
@@ -298,19 +297,19 @@ impl wasmtime_val_t {
#[unsafe(no_mangle)]
pub unsafe extern "C" fn wasmtime_val_unroot(
- cx: WasmtimeStoreContextMut<'_>,
+ _cx: WasmtimeStoreContextMut<'_>,
val: &mut MaybeUninit<wasmtime_val_t>,
) {
let val = val.assume_init_read();
match val.kind {
crate::WASMTIME_ANYREF => {
if let Some(val) = ManuallyDrop::into_inner(val.of.anyref).as_wasmtime() {
- val.unroot(cx);
+ drop(val);
}
}
crate::WASMTIME_EXTERNREF => {
if let Some(val) = ManuallyDrop::into_inner(val.of.externref).as_wasmtime() {
- val.unroot(cx);
+ drop(val);
}
}
_ => {}
diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs
index 400f2186dd..ba8df37b1f 100644
--- a/crates/wasmtime/src/runtime/func.rs
+++ b/crates/wasmtime/src/runtime/func.rs
@@ -589,7 +589,7 @@ impl Func {
/// | `Option<NoneRef>` | `nullref` aka `(ref null none)` |
/// | `NoneRef` | `(ref none)` |
///
- /// Note that anywhere a `Rooted<T>` appears, a `ManuallyRooted<T>` may also
+ /// Note that anywhere a `Rooted<T>` appears, a `OwnedRooted<T>` may also
/// be used.
///
/// Any of the Rust types can be returned from the closure as well, in
@@ -1346,7 +1346,7 @@ impl Func {
/// | `v128` | `V128` on `x86-64` and `aarch64` only |
///
/// (Note that this mapping is the same as that of [`Func::wrap`], and that
- /// anywhere a `Rooted<T>` appears, a `ManuallyRooted<T>` may also appear).
+ /// anywhere a `Rooted<T>` appears, a `OwnedRooted<T>` may also appear).
///
/// Note that once the [`TypedFunc`] return value is acquired you'll use either
/// [`TypedFunc::call`] or [`TypedFunc::call_async`] as necessary to actually invoke
diff --git a/crates/wasmtime/src/runtime/gc.rs b/crates/wasmtime/src/runtime/gc.rs
index 9c050110ac..00dfaa1601 100644
--- a/crates/wasmtime/src/runtime/gc.rs
+++ b/crates/wasmtime/src/runtime/gc.rs
@@ -28,7 +28,7 @@ impl<T> GcRef for T where T: GcRefImpl {}
/// A trait implemented for GC references that are guaranteed to be rooted:
///
/// * [`Rooted<T>`][crate::Rooted]
-/// * [`ManuallyRooted<T>`][crate::ManuallyRooted]
+/// * [`OwnedRooted<T>`][crate::OwnedRooted]
///
/// You can use this to abstract over the different kinds of rooted GC
/// references. Note that `Deref<Target = T>` is a supertrait for
diff --git a/crates/wasmtime/src/runtime/gc/disabled/anyref.rs b/crates/wasmtime/src/runtime/gc/disabled/anyref.rs
index e93951c2ce..94915fd224 100644
--- a/crates/wasmtime/src/runtime/gc/disabled/anyref.rs
+++ b/crates/wasmtime/src/runtime/gc/disabled/anyref.rs
@@ -1,6 +1,6 @@
use crate::runtime::vm::VMGcRef;
use crate::{
- ArrayRef, AsContext, AsContextMut, EqRef, GcRefImpl, HeapType, I31, ManuallyRooted, Result,
+ ArrayRef, AsContext, AsContextMut, EqRef, GcRefImpl, HeapType, I31, OwnedRooted, Result,
Rooted, StructRef,
store::{AutoAssertNoGc, StoreOpaque},
};
@@ -16,9 +16,9 @@ impl From<Rooted<EqRef>> for Rooted<AnyRef> {
}
}
-impl From<ManuallyRooted<EqRef>> for ManuallyRooted<AnyRef> {
+impl From<OwnedRooted<EqRef>> for OwnedRooted<AnyRef> {
#[inline]
- fn from(s: ManuallyRooted<EqRef>) -> Self {
+ fn from(s: OwnedRooted<EqRef>) -> Self {
match s.inner {}
}
}
@@ -30,9 +30,9 @@ impl From<Rooted<StructRef>> for Rooted<AnyRef> {
}
}
-impl From<ManuallyRooted<StructRef>> for ManuallyRooted<AnyRef> {
+impl From<OwnedRooted<StructRef>> for OwnedRooted<AnyRef> {
#[inline]
- fn from(s: ManuallyRooted<StructRef>) -> Self {
+ fn from(s: OwnedRooted<StructRef>) -> Self {
match s.inner {}
}
}
@@ -44,9 +44,9 @@ impl From<Rooted<ArrayRef>> for Rooted<AnyRef> {
}
}
-impl From<ManuallyRooted<ArrayRef>> for ManuallyRooted<AnyRef> {
+impl From<OwnedRooted<ArrayRef>> for OwnedRooted<AnyRef> {
#[inline]
- fn from(s: ManuallyRooted<ArrayRef>) -> Self {
+ fn from(s: OwnedRooted<ArrayRef>) -> Self {
match s.inner {}
}
}
diff --git a/crates/wasmtime/src/runtime/gc/disabled/eqref.rs b/crates/wasmtime/src/runtime/gc/disabled/eqref.rs
index b248e290ad..151e482b09 100644
--- a/crates/wasmtime/src/runtime/gc/disabled/eqref.rs
+++ b/crates/wasmtime/src/runtime/gc/disabled/eqref.rs
@@ -1,5 +1,5 @@
use crate::{
- ArrayRef, AsContext, GcRefImpl, HeapType, I31, ManuallyRooted, Result, Rooted, StructRef,
+ ArrayRef, AsContext, GcRefImpl, HeapType, I31, OwnedRooted, Result, Rooted, StructRef,
store::StoreOpaque,
};
@@ -14,9 +14,9 @@ impl From<Rooted<StructRef>> for Rooted<EqRef> {
}
}
-impl From<ManuallyRooted<StructRef>> for ManuallyRooted<EqRef> {
+impl From<OwnedRooted<StructRef>> for OwnedRooted<EqRef> {
#[inline]
- fn from(s: ManuallyRooted<StructRef>) -> Self {
+ fn from(s: OwnedRooted<StructRef>) -> Self {
match s.inner {}
}
}
@@ -28,9 +28,9 @@ impl From<Rooted<ArrayRef>> for Rooted<EqRef> {
}
}
-impl From<ManuallyRooted<ArrayRef>> for ManuallyRooted<EqRef> {
+impl From<OwnedRooted<ArrayRef>> for OwnedRooted<EqRef> {
#[inline]
- fn from(s: ManuallyRooted<ArrayRef>) -> Self {
+ fn from(s: OwnedRooted<ArrayRef>) -> Self {
match s.inner {}
}
}
diff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
index 7f1dddc19c..fcbe9e0fb0 100644
--- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
@@ -99,7 +99,7 @@ impl<T: GcRef> Rooted<T> {
match self.inner {}
}
- pub fn to_manually_rooted(&self, _store: impl AsContextMut) -> Result<ManuallyRooted<T>> {
+ pub fn to_owned_rooted(&self, _store: impl AsContextMut) -> Result<OwnedRooted<T>> {
match self.inner {}
}
@@ -157,7 +157,7 @@ where
/// This type has been disabled because the `gc` cargo feature was not enabled
/// at compile time.
-pub struct ManuallyRooted<T>
+pub struct OwnedRooted<T>
where
T: GcRef,
{
@@ -165,13 +165,13 @@ where
_phantom: marker::PhantomData<T>,
}
-impl<T: GcRef> Debug for ManuallyRooted<T> {
+impl<T: GcRef> Debug for OwnedRooted<T> {
fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self.inner {}
}
}
-impl<T: GcRef> Deref for ManuallyRooted<T> {
+impl<T: GcRef> Deref for OwnedRooted<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
@@ -179,7 +179,7 @@ impl<T: GcRef> Deref for ManuallyRooted<T> {
}
}
-impl<T> ManuallyRooted<T>
+impl<T> OwnedRooted<T>
where
T: GcRef,
{
@@ -187,10 +187,6 @@ where
match self.inner {}
}
- pub fn unroot(self, _store: impl AsContextMut) {
- match self.inner {}
- }
-
pub fn to_rooted(&self, _context: impl AsContextMut) -> Rooted<T> {
match self.inner {}
}
@@ -200,7 +196,7 @@ where
}
}
-impl<T: GcRef> RootedGcRefImpl<T> for ManuallyRooted<T> {
+impl<T: GcRef> RootedGcRefImpl<T> for OwnedRooted<T> {
fn assert_unreachable<U>(&self) -> U {
match self.inner {}
}
diff --git a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
index bcbfc02567..561cb6fbef 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
@@ -5,7 +5,7 @@ use crate::prelude::*;
use crate::runtime::vm::VMGcRef;
use crate::{
ArrayRef, ArrayType, AsContext, AsContextMut, EqRef, GcRefImpl, GcRootIndex, HeapType, I31,
- ManuallyRooted, RefType, Result, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy,
+ OwnedRooted, RefType, Result, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy,
store::{AutoAssertNoGc, StoreOpaque},
};
use core::mem;
@@ -23,7 +23,7 @@ use wasmtime_environ::VMGcKind;
/// `0x12345678` into a reference, pretend it is a valid `anyref`, and trick the
/// host into dereferencing it and segfaulting or worse.
///
-/// Note that you can also use `Rooted<AnyRef>` and `ManuallyRooted<AnyRef>` as
+/// Note that you can also use `Rooted<AnyRef>` and `OwnedRooted<AnyRef>` as
/// a type parameter with [`Func::typed`][crate::Func::typed]- and
/// [`Func::wrap`][crate::Func::wrap]-style APIs.
///
@@ -103,9 +103,9 @@ impl From<Rooted<EqRef>> for Rooted<AnyRef> {
}
}
-impl From<ManuallyRooted<EqRef>> for ManuallyRooted<AnyRef> {
+impl From<OwnedRooted<EqRef>> for OwnedRooted<AnyRef> {
#[inline]
- fn from(e: ManuallyRooted<EqRef>) -> Self {
+ fn from(e: OwnedRooted<EqRef>) -> Self {
e.to_anyref()
}
}
@@ -117,9 +117,9 @@ impl From<Rooted<StructRef>> for Rooted<AnyRef> {
}
}
-impl From<ManuallyRooted<StructRef>> for ManuallyRooted<AnyRef> {
+impl From<OwnedRooted<StructRef>> for OwnedRooted<AnyRef> {
#[inline]
- fn from(s: ManuallyRooted<StructRef>) -> Self {
+ fn from(s: OwnedRooted<StructRef>) -> Self {
s.to_anyref()
}
}
@@ -131,9 +131,9 @@ impl From<Rooted<ArrayRef>> for Rooted<AnyRef> {
}
}
-impl From<ManuallyRooted<ArrayRef>> for ManuallyRooted<AnyRef> {
+impl From<OwnedRooted<ArrayRef>> for OwnedRooted<AnyRef> {
#[inline]
- fn from(s: ManuallyRooted<ArrayRef>) -> Self {
+ fn from(s: OwnedRooted<ArrayRef>) -> Self {
s.to_anyref()
}
}
@@ -754,7 +754,7 @@ unsafe impl WasmTy for Option<Rooted<AnyRef>> {
}
}
-unsafe impl WasmTy for ManuallyRooted<AnyRef> {
+unsafe impl WasmTy for OwnedRooted<AnyRef> {
#[inline]
fn valtype() -> ValType {
ValType::Ref(RefType::new(false, HeapType::Any))
@@ -784,7 +784,7 @@ unsafe impl WasmTy for ManuallyRooted<AnyRef> {
}
}
-unsafe impl WasmTy for Option<ManuallyRooted<AnyRef>> {
+unsafe impl WasmTy for Option<OwnedRooted<AnyRef>> {
#[inline]
fn valtype() -> ValType {
ValType::ANYREF
@@ -821,11 +821,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<AnyRef>> {
}
fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
- <ManuallyRooted<AnyRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
+ <OwnedRooted<AnyRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
}
unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
- <ManuallyRooted<AnyRef>>::wasm_ty_option_load(
+ <OwnedRooted<AnyRef>>::wasm_ty_option_load(
store,
ptr.get_anyref(),
AnyRef::from_cloned_gc_ref,
diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
index f8d3a0ee7a..78db88eaef 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
@@ -6,7 +6,7 @@ use crate::vm::{self, VMArrayRef, VMGcHeader};
use crate::{AnyRef, FieldType};
use crate::{
ArrayType, AsContext, AsContextMut, EqRef, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType,
- ManuallyRooted, RefType, Rooted, Val, ValRaw, ValType, WasmTy,
+ OwnedRooted, RefType, Rooted, Val, ValRaw, ValType, WasmTy,
prelude::*,
store::{AutoAssertNoGc, StoreContextMut, StoreOpaque},
};
@@ -108,7 +108,7 @@ impl ArrayRefPre {
/// `0x12345678` into a reference, pretend it is a valid `arrayref`, and trick
/// the host into dereferencing it and segfaulting or worse.
///
-/// Note that you can also use `Rooted<ArrayRef>` and `ManuallyRooted<ArrayRef>`
+/// Note that you can also use `Rooted<ArrayRef>` and `OwnedRooted<ArrayRef>`
/// as a type parameter with [`Func::typed`][crate::Func::typed]- and
/// [`Func::wrap`][crate::Func::wrap]-style APIs.
///
@@ -213,16 +213,16 @@ impl Rooted<ArrayRef> {
}
}
-impl ManuallyRooted<ArrayRef> {
+impl OwnedRooted<ArrayRef> {
/// Upcast this `arrayref` into an `anyref`.
#[inline]
- pub fn to_anyref(self) -> ManuallyRooted<AnyRef> {
+ pub fn to_anyref(self) -> OwnedRooted<AnyRef> {
self.unchecked_cast()
}
/// Upcast this `arrayref` into an `eqref`.
#[inline]
- pub fn to_eqref(self) -> ManuallyRooted<EqRef> {
+ pub fn to_eqref(self) -> OwnedRooted<EqRef> {
self.unchecked_cast()
}
}
@@ -923,7 +923,7 @@ unsafe impl WasmTy for Option<Rooted<ArrayRef>> {
}
}
-unsafe impl WasmTy for ManuallyRooted<ArrayRef> {
+unsafe impl WasmTy for OwnedRooted<ArrayRef> {
#[inline]
fn valtype() -> ValType {
ValType::Ref(RefType::new(false, HeapType::Array))
@@ -975,7 +975,7 @@ unsafe impl WasmTy for ManuallyRooted<ArrayRef> {
}
}
-unsafe impl WasmTy for Option<ManuallyRooted<ArrayRef>> {
+unsafe impl WasmTy for Option<OwnedRooted<ArrayRef>> {
#[inline]
fn valtype() -> ValType {
ValType::ARRAYREF
@@ -995,9 +995,7 @@ unsafe impl WasmTy for Option<ManuallyRooted<ArrayRef>> {
ty: &HeapType,
) -> Result<()> {
match self {
- Some(s) => {
- ManuallyRooted::<ArrayRef>::dynamic_concrete_type_check(s, store, nullable, ty)
- }
+ Some(s) => OwnedRooted::<ArrayRef>::dynamic_concrete_type_check(s, store, nullable, ty),
None => {
ensure!(
nullable,
@@ -1014,11 +1012,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<ArrayRef>> {
}
fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
- <ManuallyRooted<ArrayRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
+ <OwnedRooted<ArrayRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
}
unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
- <ManuallyRooted<ArrayRef>>::wasm_ty_option_load(
+ <OwnedRooted<ArrayRef>>::wasm_ty_option_load(
store,
ptr.get_anyref(),
ArrayRef::from_cloned_gc_ref,
diff --git a/crates/wasmtime/src/runtime/gc/enabled/eqref.rs b/crates/wasmtime/src/runtime/gc/enabled/eqref.rs
index 7b6d27a16b..de88a26536 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/eqref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/eqref.rs
@@ -1,7 +1,7 @@
//! Working with GC `eqref`s.
use crate::{
- AnyRef, ArrayRef, ArrayType, AsContext, GcRefImpl, GcRootIndex, HeapType, I31, ManuallyRooted,
+ AnyRef, ArrayRef, ArrayType, AsContext, GcRefImpl, GcRootIndex, HeapType, I31, OwnedRooted,
RefType, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy,
prelude::*,
runtime::vm::VMGcRef,
@@ -25,7 +25,7 @@ use wasmtime_environ::VMGcKind;
/// `0x12345678` into a reference, pretend it is a valid `eqref`, and trick the
/// host into dereferencing it and segfaulting or worse.
///
-/// Note that you can also use `Rooted<EqRef>` and `ManuallyRooted<EqRef>` as a
+/// Note that you can also use `Rooted<EqRef>` and `OwnedRooted<EqRef>` as a
/// type parameter with [`Func::typed`][crate::Func::typed]- and
/// [`Func::wrap`][crate::Func::wrap]-style APIs.
///
@@ -99,9 +99,9 @@ impl From<Rooted<StructRef>> for Rooted<EqRef> {
}
}
-impl From<ManuallyRooted<StructRef>> for ManuallyRooted<EqRef> {
+impl From<OwnedRooted<StructRef>> for OwnedRooted<EqRef> {
#[inline]
- fn from(s: ManuallyRooted<StructRef>) -> Self {
+ fn from(s: OwnedRooted<StructRef>) -> Self {
s.to_eqref()
}
}
@@ -113,9 +113,9 @@ impl From<Rooted<ArrayRef>> for Rooted<EqRef> {
}
}
-impl From<ManuallyRooted<ArrayRef>> for ManuallyRooted<EqRef> {
+impl From<OwnedRooted<ArrayRef>> for OwnedRooted<EqRef> {
#[inline]
- fn from(s: ManuallyRooted<ArrayRef>) -> Self {
+ fn from(s: OwnedRooted<ArrayRef>) -> Self {
s.to_eqref()
}
}
@@ -145,10 +145,10 @@ impl Rooted<EqRef> {
}
}
-impl ManuallyRooted<EqRef> {
+impl OwnedRooted<EqRef> {
/// Upcast this `eqref` into an `anyref`.
#[inline]
- pub fn to_anyref(self) -> ManuallyRooted<AnyRef> {
+ pub fn to_anyref(self) -> OwnedRooted<AnyRef> {
self.unchecked_cast()
}
}
@@ -516,7 +516,7 @@ unsafe impl WasmTy for Option<Rooted<EqRef>> {
}
}
-unsafe impl WasmTy for ManuallyRooted<EqRef> {
+unsafe impl WasmTy for OwnedRooted<EqRef> {
#[inline]
fn valtype() -> ValType {
ValType::Ref(RefType::new(false, HeapType::Eq))
@@ -546,7 +546,7 @@ unsafe impl WasmTy for ManuallyRooted<EqRef> {
}
}
-unsafe impl WasmTy for Option<ManuallyRooted<EqRef>> {
+unsafe impl WasmTy for Option<OwnedRooted<EqRef>> {
#[inline]
fn valtype() -> ValType {
ValType::EQREF
@@ -566,7 +566,7 @@ unsafe impl WasmTy for Option<ManuallyRooted<EqRef>> {
ty: &HeapType,
) -> Result<()> {
match self {
- Some(s) => ManuallyRooted::<EqRef>::dynamic_concrete_type_check(s, store, nullable, ty),
+ Some(s) => OwnedRooted::<EqRef>::dynamic_concrete_type_check(s, store, nullable, ty),
None => {
ensure!(
nullable,
@@ -583,11 +583,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<EqRef>> {
}
fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
- <ManuallyRooted<EqRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
+ <OwnedRooted<EqRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
}
unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
- <ManuallyRooted<EqRef>>::wasm_ty_option_load(
+ <OwnedRooted<EqRef>>::wasm_ty_option_load(
store,
ptr.get_anyref(),
EqRef::from_cloned_gc_ref,
diff --git a/crates/wasmtime/src/runtime/gc/enabled/exnref.rs b/crates/wasmtime/src/runtime/gc/enabled/exnref.rs
index 03ba35ad3b..5ada1f06d2 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/exnref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/exnref.rs
@@ -4,7 +4,7 @@ use crate::runtime::vm::{VMGcRef, VMStore};
use crate::store::{StoreId, StoreResourceLimiter};
use crate::vm::{self, VMExnRef, VMGcHeader};
use crate::{
- AsContext, AsContextMut, GcRefImpl, GcRootIndex, HeapType, ManuallyRooted, RefType, Result,
+ AsContext, AsContextMut, GcRefImpl, GcRootIndex, HeapType, OwnedRooted, RefType, Result,
Rooted, Val, ValRaw, ValType, WasmTy,
store::{AutoAssertNoGc, StoreOpaque},
};
@@ -101,7 +101,7 @@ impl ExnRefPre {
/// thrown exception in WebAssembly with a `catch_ref` clause of a
/// `try_table`, or by allocating via the host API.
///
-/// Note that you can also use `Rooted<ExnRef>` and `ManuallyRooted<ExnRef>` as
+/// Note that you can also use `Rooted<ExnRef>` and `OwnedRooted<ExnRef>` as
/// a type parameter with [`Func::typed`][crate::Func::typed]- and
/// [`Func::wrap`][crate::Func::wrap]-style APIs.
#[derive(Debug)]
@@ -695,7 +695,7 @@ unsafe impl WasmTy for Option<Rooted<ExnRef>> {
}
}
-unsafe impl WasmTy for ManuallyRooted<ExnRef> {
+unsafe impl WasmTy for OwnedRooted<ExnRef> {
#[inline]
fn valtype() -> ValType {
ValType::Ref(RefType::new(false, HeapType::Exn))
@@ -725,7 +725,7 @@ unsafe impl WasmTy for ManuallyRooted<ExnRef> {
}
}
-unsafe impl WasmTy for Option<ManuallyRooted<ExnRef>> {
+unsafe impl WasmTy for Option<OwnedRooted<ExnRef>> {
#[inline]
fn valtype() -> ValType {
ValType::EXNREF
@@ -762,11 +762,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<ExnRef>> {
}
fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
- <ManuallyRooted<ExnRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
+ <OwnedRooted<ExnRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
}
unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
- <ManuallyRooted<ExnRef>>::wasm_ty_option_load(
+ <OwnedRooted<ExnRef>>::wasm_ty_option_load(
store,
ptr.get_anyref(),
ExnRef::from_cloned_gc_ref,
diff --git a/crates/wasmtime/src/runtime/gc/enabled/externref.rs b/crates/wasmtime/src/runtime/gc/enabled/externref.rs
index 8564843b84..1d2195de48 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/externref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/externref.rs
@@ -4,7 +4,7 @@ use super::{AnyRef, RootedGcRefImpl};
use crate::prelude::*;
use crate::runtime::vm::{self, VMGcRef, VMStore};
use crate::{
- AsContextMut, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType, ManuallyRooted, RefType,
+ AsContextMut, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType, OwnedRooted, RefType,
Result, Rooted, StoreContext, StoreContextMut, ValRaw, ValType, WasmTy,
store::{AutoAssertNoGc, StoreOpaque, StoreResourceLimiter},
};
@@ -31,7 +31,7 @@ use core::mem::MaybeUninit;
/// the host into dereferencing it and segfaulting or worse.
///
/// Note that you can also use `Rooted<ExternRef>` and
-/// `ManuallyRooted<ExternRef>` as a type parameter with
+/// `OwnedRooted<ExternRef>` as a type parameter with
/// [`Func::typed`][crate::Func::typed]- and
/// [`Func::wrap`][crate::Func::wrap]-style APIs.
///
@@ -140,10 +140,10 @@ impl ExternRef {
///
/// The resulting value is automatically unrooted when the given `context`'s
/// scope is exited. If you need to hold the reference past the `context`'s
- /// scope, convert the result into a
- /// [`ManuallyRooted<T>`][crate::ManuallyRooted]. See the documentation for
+ /// scope, convert the result into an
+ /// [`OwnedRooted<T>`][crate::OwnedRooted]. See the documentation for
/// [`Rooted<T>`][crate::Rooted] and
- /// [`ManuallyRooted<T>`][crate::ManuallyRooted] for more details.
+ /// [`OwnedRooted<T>`][crate::OwnedRooted] for more details.
///
/// # Automatic Garbage Collection
///
@@ -223,10 +223,10 @@ impl ExternRef {
///
/// The resulting value is automatically unrooted when the given `context`'s
/// scope is exited. If you need to hold the reference past the `context`'s
- /// scope, convert the result into a
- /// [`ManuallyRooted<T>`][crate::ManuallyRooted]. See the documentation for
+ /// scope, convert the result into an
+ /// [`OwnedRooted<T>`][crate::OwnedRooted]. See the documentation for
/// [`Rooted<T>`][crate::Rooted] and
- /// [`ManuallyRooted<T>`][crate::ManuallyRooted] for more details.
+ /// [`OwnedRooted<T>`][crate::OwnedRooted] for more details.
///
/// # Automatic Garbage Collection
///
@@ -646,7 +646,7 @@ unsafe impl WasmTy for Option<Rooted<ExternRef>> {
}
}
-unsafe impl WasmTy for ManuallyRooted<ExternRef> {
+unsafe impl WasmTy for OwnedRooted<ExternRef> {
#[inline]
fn valtype() -> ValType {
ValType::Ref(RefType::new(false, HeapType::Extern))
@@ -676,7 +676,7 @@ unsafe impl WasmTy for ManuallyRooted<ExternRef> {
}
}
-unsafe impl WasmTy for Option<ManuallyRooted<ExternRef>> {
+unsafe impl WasmTy for Option<OwnedRooted<ExternRef>> {
#[inline]
fn valtype() -> ValType {
ValType::EXTERNREF
@@ -699,11 +699,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<ExternRef>> {
}
fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
- <ManuallyRooted<ExternRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::externref)
+ <OwnedRooted<ExternRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::externref)
}
unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
- <ManuallyRooted<ExternRef>>::wasm_ty_option_load(
+ <OwnedRooted<ExternRef>>::wasm_ty_option_load(
store,
ptr.get_externref(),
ExternRef::from_cloned_gc_ref,
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index 8ea6d49796..7e06c85d4d 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -27,24 +27,70 @@
//! not a burden for users. Additionally, the API's types should be `Sync`
//! and `Send` so that they work well with async Rust.
//!
-//! For example, goals (3) and (4) are in conflict when we think about how to
-//! support (2). Ideally, for ergonomics, a root would automatically unroot
-//! itself when dropped. But in the general case that requires holding a
-//! reference to the store's root set, and that root set needs to be held
-//! simultaneously by all GC roots, and they each need to mutate the set to
-//! unroot themselves. That implies `Rc<RefCell<...>>` or `Arc<Mutex<...>>`! The
-//! former makes the store and GC root types not `Send` and not `Sync`. The
-//! latter imposes synchronization and locking overhead. So we instead make GC
-//! roots indirect and require passing in a store context explicitly to unroot
-//! in the general case. This trades worse ergonomics for better performance and
-//! support for moving GC.
+//! The two main design axes that trade off the above goals are:
+//!
+//! - Where the GC reference itself is held. A root object could
+//! directly hold the underlying GC reference (offset into the GC
+//! storage area), which would allow more efficient dereferencing
+//! and access to the referred-to GC object. However, goal (2)
+//! requires that the GC is able to update references when objects
+//! move during a GC. Thus, such "direct roots" would need to be
+//! registered somehow in a global root registry, and would need to
+//! unregister themselves when dropped.
+//!
+//! The alternative is to hold some indirect kind of reference to a
+//! GC reference, with the latter stored directly in the `Store` so
+//! the GC can update it freely. This adds one pointer-chasing hop
+//! to accesses, but works much more nicely with ownership
+//! semantics. Logically, the `Store` "owns" the actual pointers;
+//! and rooting types own the slots that they are stored in.
+//!
+//! For the above reasons, all of our rooting types below use
+//! indirection. This avoids the need for an unsafe
+//! intrusive-linked-list for global registration, or a shared
+//! reference to a mutex-protected registry, or some other
+//! error-prone technique.
+//!
+//! - How unrooting occurs. Ideally, a rooting type would implement
+//! the Rust `Drop` trait and unroot itself when the Rust value is
+//! dropped. However, because the rooting state is held in the
+//! `Store`, this direct approach would imply keeping a shared,
+//! mutex-protected handle to the registry in every rooting
+//! object. This would add synchronization overhead to the common
+//! case, and in general would be a bad tradeoff.
+//!
+//! However, there are several other approaches:
+//!
+//! - The user could use an RAII wrapper that *does* own the `Store`,
+//! and defines a "scope" in which roots are created and then
+//! bulk-unrooted at the close of the scope.
+//! - The rooting type could hold a shared reference to some state
+//! *other* than the full registry, and update a flag in that
+//! state indicating it has been dropped; the `Store` could then
+//! later observe that flag and remove the root. This would have
+//! some allocation cost, but the shared state would be
+//! independent of the `Store` and specific to each root, so would
+//! impose no synchronization overhead between different roots or
+//! the GC itself.
+//! - The rooting type could provide a fully manual `unroot` method,
+//! allowing the user to make use of their own knowledge of their
+//! application's lifetimes and semantics and remove roots when
+//! appropriate.
+//!
+//! We provide an implementation of the first two of these
+//! strategies below in `Rooted` and `OwnedRooted`. The last, fully
+//! manual, approach is too difficult to use correctly (it cannot
+//! implement Rust's `Drop` trait, but there is no way in Rust to
+//! enforce that a value must be consumed rather than dropped) so it
+//! is not implemented.
//!
//! ## Two Flavors of Rooting API
//!
-//! Okay, with that out of the way, this module provides two flavors of rooting
-//! API. One for the common, scoped lifetime case, and another for the rare case
-//! where we really need a GC root with an arbitrary, non-LIFO/non-scoped
-//! lifetime:
+//! Okay, with that out of the way, this module provides two flavors
+//! of rooting API. One for the common, scoped lifetime case, and one that
+//! carries ownership until dropped, and can work as an RAII handle
+//! that interacts well with Rust ownership semantics (but at a minor
+//! performance cost):
//!
//! 1. `RootScope` and `Rooted<T>`: These are used for temporarily rooting GC
//! objects for the duration of a scope. The internal implementation takes
@@ -64,27 +110,40 @@
//! This supports the common use case for rooting and provides good
//! ergonomics.
//!
-//! 2. `ManuallyRooted<T>`: This is the fully general rooting API used for
-//! holding onto non-LIFO GC roots with arbitrary lifetimes. However, users
-//! must manually unroot them. Failure to manually unroot a
-//! `ManuallyRooted<T>` before it is dropped will result in the GC object
-//! (and everything it transitively references) leaking for the duration of
-//! the `Store`'s lifetime.
+//! 2. `OwnedRooted<T>`: This is a root that manages rooting and
+//! unrooting automatically with its lifetime as a Rust value. In
+//! other words, the continued existence of the Rust value ensures
+//! the rooting of the underlying GC reference; and when the Rust
+//! value is dropped, the underlying GC reference is no longer
+//! rooted.
+//!
+//! Internally, this root holds a shared reference to a
+//! *root-specific* bit of state that is also tracked and observed
+//! by the `Store`. This means that there is minor memory
+//! allocation overhead (an `Arc<()>`) for each such root; this
+//! memory is shared over all clones of this root. The rooted GC
+//! reference is *logically* unrooted as soon as the last clone of
+//! this root is dropped. Internally the root may still exist until
+//! the next GC, or "root trim" when another `OwnedRooted` is
+//! created, but that is unobservable externally, and will not
+//! result in any additional GC object lifetime because it is
+//! always cleaned up before a gc.
//!
//! This type is roughly similar to SpiderMonkey's [`PersistentRooted<T>`],
-//! although they avoid the manual-unrooting with internal mutation and
-//! shared references. (Our constraints mean we can't do those things, as
-//! mentioned explained above.)
+//! although they register roots on a per-thread `JSContext`, avoiding
+//! mutation costs in a way that is not viable for Wasmtime (which needs
+//! `Store`s to be `Send`).
//!
//! [`PersistentRooted<T>`]: http://devdoc.net/web/developer.mozilla.org/en-US/docs/Mozilla/Projects/SpiderMonkey/JSAPI_reference/JS::PersistentRooted.html
//!
-//! At the end of the day, both `Rooted<T>` and `ManuallyRooted<T>` are just
-//! tagged indices into the store's `RootSet`. This indirection allows working
-//! with Rust's borrowing discipline (we use `&mut Store` to represent mutable
-//! access to the GC heap) while still allowing rooted references to be moved
-//! around without tying up the whole store in borrows. Additionally, and
-//! crucially, this indirection allows us to update the *actual* GC pointers in
-//! the `RootSet` and support moving GCs (again, as mentioned above).
+//! At the end of the day, all of the above root types are just tagged
+//! indices into the store's `RootSet`. This indirection allows
+//! working with Rust's borrowing discipline (we use `&mut Store` to
+//! represent mutable access to the GC heap) while still allowing
+//! rooted references to be moved around without tying up the whole
+//! store in borrows. Additionally, and crucially, this indirection
+//! allows us to update the *actual* GC pointers in the `RootSet` and
+//! support moving GCs (again, as mentioned above).
//!
//! ## Unrooted References
//!
@@ -108,10 +167,11 @@ use crate::{
store::{AsStoreOpaque, AutoAssertNoGc, StoreId, StoreOpaque},
};
use crate::{ValRaw, prelude::*};
+use alloc::sync::{Arc, Weak};
use core::any;
use core::marker;
use core::mem::{self, MaybeUninit};
-use core::num::NonZeroU64;
+use core::num::{NonZeroU64, NonZeroUsize};
use core::{
fmt::{self, Debug},
hash::{Hash, Hasher},
@@ -182,7 +242,7 @@ pub(crate) use sealed::*;
/// The index of a GC root inside a particular store's GC root set.
///
-/// Can be either a LIFO- or manually-rooted object, depending on the
+/// Can be either a LIFO- or owned-rooted object, depending on the
/// `PackedIndex`.
///
/// Every `T` such that `T: GcRef` must be a newtype over this `GcRootIndex`.
@@ -237,8 +297,8 @@ impl GcRootIndex {
} else {
None
}
- } else if let Some(id) = self.index.as_manual() {
- let gc_ref = store.gc_roots().manually_rooted.get(id);
+ } else if let Some(id) = self.index.as_owned() {
+ let gc_ref = store.gc_roots().owned_rooted.get(id);
debug_assert!(gc_ref.is_some());
gc_ref
} else {
@@ -271,7 +331,7 @@ impl GcRootIndex {
/// ```ignore
/// enum {
/// Lifo(usize),
-/// Manual(SlabId),
+/// Owned(SlabId),
/// }
/// ```
///
@@ -285,8 +345,8 @@ impl Debug for PackedIndex {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if let Some(index) = self.as_lifo() {
f.debug_tuple("PackedIndex::Lifo").field(&index).finish()
- } else if let Some(id) = self.as_manual() {
- f.debug_tuple("PackedIndex::Manual").field(&id).finish()
+ } else if let Some(id) = self.as_owned() {
+ f.debug_tuple("PackedIndex::Owned").field(&id).finish()
} else {
unreachable!()
}
@@ -296,7 +356,7 @@ impl Debug for PackedIndex {
impl PackedIndex {
const DISCRIMINANT_MASK: u32 = 0b1 << 31;
const LIFO_DISCRIMINANT: u32 = 0b0 << 31;
- const MANUAL_DISCRIMINANT: u32 = 0b1 << 31;
+ const OWNED_DISCRIMINANT: u32 = 0b1 << 31;
const PAYLOAD_MASK: u32 = !Self::DISCRIMINANT_MASK;
fn new_lifo(index: usize) -> PackedIndex {
@@ -305,17 +365,17 @@ impl PackedIndex {
let packed = PackedIndex(Self::LIFO_DISCRIMINANT | index32);
debug_assert!(packed.is_lifo());
debug_assert_eq!(packed.as_lifo(), Some(index));
- debug_assert!(!packed.is_manual());
- debug_assert!(packed.as_manual().is_none());
+ debug_assert!(!packed.is_owned());
+ debug_assert!(packed.as_owned().is_none());
packed
}
- fn new_manual(id: SlabId) -> PackedIndex {
+ fn new_owned(id: SlabId) -> PackedIndex {
let raw = id.into_raw();
assert_eq!(raw & Self::DISCRIMINANT_MASK, 0);
- let packed = PackedIndex(Self::MANUAL_DISCRIMINANT | raw);
- debug_assert!(packed.is_manual());
- debug_assert_eq!(packed.as_manual(), Some(id));
+ let packed = PackedIndex(Self::OWNED_DISCRIMINANT | raw);
+ debug_assert!(packed.is_owned());
+ debug_assert_eq!(packed.as_owned(), Some(id));
debug_assert!(!packed.is_lifo());
debug_assert!(packed.as_lifo().is_none());
packed
@@ -329,8 +389,8 @@ impl PackedIndex {
self.discriminant() == Self::LIFO_DISCRIMINANT
}
- fn is_manual(&self) -> bool {
- self.discriminant() == Self::MANUAL_DISCRIMINANT
+ fn is_owned(&self) -> bool {
+ self.discriminant() == Self::OWNED_DISCRIMINANT
}
fn payload(&self) -> u32 {
@@ -345,8 +405,8 @@ impl PackedIndex {
}
}
- fn as_manual(&self) -> Option<SlabId> {
- if self.is_manual() {
+ fn as_owned(&self) -> Option<SlabId> {
+ if self.is_owned() {
Some(SlabId::from_raw(self.payload()))
} else {
None
@@ -357,9 +417,19 @@ impl PackedIndex {
/// The set of all embedder-API GC roots in a single store/heap.
#[derive(Debug, Default)]
pub(crate) struct RootSet {
- /// GC roots with arbitrary lifetime that are manually rooted and unrooted,
- /// for use with `ManuallyRooted<T>`.
- manually_rooted: Slab<VMGcRef>,
+ /// GC roots with arbitrary lifetime that are unrooted when
+ /// liveness flags are cleared (seen during a trimming pass), for
+ /// use with `OwnedRooted<T>`.
+ owned_rooted: Slab<VMGcRef>,
+
+ /// List of liveness flags and corresponding `SlabId`s into the
+ /// `owned_rooted` slab.
+ liveness_flags: Vec<(Weak<()>, SlabId)>,
+
+ /// High-water mark for liveness flag trimming. We use this to
+ /// ensure we have amortized constant-time behavior on adding
+ /// roots. See note below on `trim_liveness_flags()`.
+ liveness_trim_high_water: Option<NonZeroUsize>,
/// Strictly LIFO-ordered GC roots, for use with `RootScope` and
/// `Rooted<T>`.
@@ -386,13 +456,13 @@ impl RootSet {
}
log::trace!("End trace user LIFO roots");
- log::trace!("Begin trace user manual roots");
- for (_id, root) in self.manually_rooted.iter_mut() {
+ log::trace!("Begin trace user owned roots");
+ for (_id, root) in self.owned_rooted.iter_mut() {
unsafe {
- gc_roots_list.add_root(root.into(), "user manual root");
+ gc_roots_list.add_root(root.into(), "user owned root");
}
}
- log::trace!("End trace user manual roots");
+ log::trace!("End trace user owned roots");
}
/// Enter a LIFO rooting scope.
@@ -466,6 +536,181 @@ impl RootSet {
index,
}
}
+
+ /// Trim any stale (dropped) owned roots.
+ ///
+ /// `OwnedRooted` is implemented in a way that avoids the need to
+ /// have or keep a reference to the store (and thus this struct)
+ /// during its drop operation: to allow it to be independent, it
+ /// holds a shared reference to some other memory and sets that
+ /// "liveness flag" appropriately, then we later observe dead
+ /// liveness flags during a periodic scan and actually deallocate
+ /// the roots. We use an `Arc<()>` for this: it permits cheap
+ /// cloning, and it has minimal memory overhead. We hold a weak
+ /// ref in a list alongside the actual `GcRootIndex`, and we free
+ /// that index in the slab of owned roots when we observe that
+ /// only our weak ref remains.
+ ///
+ /// This trim step is logically separate from a full GC, though it
+ /// would not be very productive to do a GC without doing a
+ /// root-trim first: the root-trim should be quite a lot cheaper,
+ /// and it will allow for more garbage to exist.
+ ///
+ /// There is, additionally, nothing stopping us from doing trims
+ /// more often than just before each GC, and there are reasons
+ /// this could be a good idea: for example, a user program that
+ /// creates and removes many roots (perhaps as it accesses the GC
+ /// object graph) but ultimately is dealing with a static graph,
+ /// without further allocation, may need the "root set" to be GC'd
+ /// independently from the actual heap. Thus, we could trim before
+ /// adding a new root to ensure we don't grow that unboundedly (or
+ /// force an otherwise unneeded GC).
+ ///
+ /// The first case, just before GC, wants a "full trim": there's
+ /// no reason not to unroot as much as possible before we do the
+ /// expensive work of tracing the whole heap.
+ ///
+ /// On the other hand, the second case, adding a new root, wants a
+ /// kind of trim that is amortized constant time. Consider: if we
+ /// have some threshold for the trim, say N roots, and the user
+ /// program continually adds and removes one root such that it
+ /// goes just over the threshold, we might scan all N liveness
+ /// flags for each step, resulting in quadratic behavior overall.
+ ///
+ /// Thus, we implement a "high water mark" algorithm to guard
+ /// against this latter case: on the add-a-new-root case, we trim
+ /// only if the list is longer than the high water mark, and we
+ /// set the high water mark each time based on the after-trim
+ /// size. See below for details on this algorithm.
+ ///
+ /// `eager` chooses whether we eagerly trim roots or pre-filter
+ /// using the high-water mark.
+ ///
+ /// # Growth Algorithm
+ ///
+ /// We want to balance two factors: we must ensure that the
+ /// algorithmic complexity of creating a new root is amortized
+ /// O(1), and we must ensure that repeated creation and deletion
+ /// of roots without any GC must result in a root-set that has a
+ /// size linear in the actual live-root-set size. Stated formally:
+ ///
+ /// 1. Root creation must be O(1), amortized
+ /// 2. liveness_flags.len() must be O(|max live owned roots|),
+ /// i.e., must not grow to larger than a constant multiple of
+ /// the maximum working-root-set size of the program.
+ ///
+ /// Note that a naive exponential-growth-of-threshold algorithm,
+ /// where we trim when the root set reaches 1, 2, 4, 8, 16, 32,
+ /// ... elements, provides the first but *not* the second
+ /// property. A workload that has a constant live root set but
+ /// creates and drops roots constantly (say, as it's traversing a
+ /// static graph and moving "fingers" through it) will cause the
+ /// `liveness_flags` array to grow unboundedly.
+ ///
+ /// Instead, it turns out that we can achieve both of these goals
+ /// with a simple rule: we trim when the root list length reaches
+ /// a high-water mark; and then *after* trimming, we set the
+ /// high-water mark equal to the resulting live-root count
+ /// multiplied by a factor (e.g., 2).
+ ///
+ /// ## Proof
+ ///
+ /// - Root creation is O(1)
+ ///
+ /// Assume a sequence of root creation and drop events (and no
+ /// GCs, with a static GC graph, in the worst case -- only roots
+ /// are changing). We want to show that after N root creations,
+ /// we have incurred only only O(N) cost scanning the
+ /// `liveness_flags` list over the whole sequence.
+ ///
+ /// Assume a default high-water mark of D (e.g., 8) at
+ /// initialization with an empty root list.
+ ///
+ /// Consider "epochs" in the sequence split by trim events where
+ /// we scan the root list. Proceed by induction over epochs to
+ /// show: after each epoch, we will have scanned at most 2N
+ /// roots after N root creations.
+ ///
+ /// (These epochs don't exist in the algorithm: this is just a
+ /// mechanism to analyze the behavior.)
+ ///
+ /// Base case: after the first epoch, with D root creations, we
+ /// will scan D roots.
+ ///
+ /// Induction step: we have created N roots and scanned at most
+ /// 2N roots. After previous scan, L roots are still live; then
+ /// we set the high-water mark for next scan at 2L. The list
+ /// already has L, so after another L root creations, the epoch
+ /// ends. We will then incur a scan cost of 2L (the full
+ /// list). At that point we have thus seen N + L root creations,
+ /// with 2N + 2L scan cost; the invariant holds.
+ ///
+ /// (It's counter-intuitive that *not* raising the high-water
+ /// mark exponentially can still result in a constant amortized
+ /// cost! One intuition to understand this is that each root
+ /// that remains alive after a scan pushes the next high-water
+ /// mark up by one, so requires a new root creation to "pay for"
+ /// its next scan. So any given root may be scanned many times,
+ /// but each such root ensures other root creations happen to
+ /// maintain the amortized cost.)
+ ///
+ /// - `liveness_flags.len()` is always O(|max live roots|)
+ ///
+ /// Before the first trim, we have between 0 and D live roots,
+ /// which is O(1) (`D` is a compile-time constant).
+ ///
+ /// Just after a trim, the `liveness_flags` list has only live
+ /// roots, and the max live-root count is at least the count at
+ /// this time, so the property holds.
+ ///
+ /// The instantaneous maximum number of live roots is greater
+ /// than or equal to the maximum number of live roots observed
+ /// during a trim. (The trim is just some point in time, and the
+ /// max at some point in time is at most the overall max.)
+ ///
+ /// The high-water mark is set at 2 * `liveness_flags.len()`
+ /// after a trim, i.e., the number of live roots at that
+ /// time. We trim when we reach the high-water mark. So the
+ /// length of the array cannot exceed 2 *
+ /// `liveness_flags.len()`, which is less than or equal to the
+ /// overall max. So transitively, the list length at any time is
+ /// always O(|max live roots|).
+ ///
+ /// We thus have tight bounds (deterministic, not randomized) for
+ /// all possible sequences of root creation/dropping, ensuring
+ /// robustness.
+ pub(crate) fn trim_liveness_flags(&mut self, gc_store: &mut GcStore, eager: bool) {
+ const DEFAULT_HIGH_WATER: usize = 8;
+ const GROWTH_FACTOR: usize = 2;
+ let high_water_mark = self
+ .liveness_trim_high_water
+ .map(|x| x.get())
+ .unwrap_or(DEFAULT_HIGH_WATER);
+ if !eager && self.liveness_flags.len() < high_water_mark {
+ return;
+ }
+
+ self.liveness_flags.retain(|(flag, index)| {
+ if flag.strong_count() == 0 {
+ // No more `OwnedRooted` instances are holding onto
+ // this; dealloc the index and drop our Weak.
+ let gc_ref = self.owned_rooted.dealloc(*index);
+ gc_store.drop_gc_ref(gc_ref);
+ // Don't retain in the list.
+ false
+ } else {
+ // Retain in the list.
+ true
+ }
+ });
+
+ let post_trim_len = self.liveness_flags.len();
+ let high_water_mark = core::cmp::max(
+ DEFAULT_HIGH_WATER,
+ post_trim_len.saturating_mul(GROWTH_FACTOR),
+ );
+ self.liveness_trim_high_water = Some(NonZeroUsize::new(high_water_mark).unwrap());
+ }
}
/// A scoped, rooted reference to a garbage-collected `T`.
@@ -552,21 +797,25 @@ impl RootSet {
/// # }
/// ```
///
-/// # Differences Between `Rooted<T>` and `ManuallyRooted<T>`
+/// # Differences Between `Rooted<T>` and `OwnedRooted<T>`
///
-/// While `Rooted<T>` is automatically unrooted when its scope is exited, this
-/// means that `Rooted<T>` is only valid for strictly last-in-first-out (LIFO,
-/// aka stack order) lifetimes. This is in contrast to
-/// [`ManuallyRooted<T>`][crate::ManuallyRooted], which supports rooting GC
-/// objects for arbitrary lifetimes, but requires manual unrooting.
+/// While `Rooted<T>` is automatically unrooted when its scope is
+/// exited, this means that `Rooted<T>` is only valid for strictly
+/// last-in-first-out (LIFO, aka stack order) lifetimes. This is in
+/// contrast to [`OwnedRooted<T>`][crate::OwnedRooted], which supports
+/// rooting GC objects for arbitrary lifetimes.
///
-/// | Type | Supported Lifetimes | Unrooting |
-/// |----------------------------------------------|-----------------------------|-----------|
-/// | [`Rooted<T>`][crate::Rooted] | Strictly LIFO / stack order | Automatic |
-/// | [`ManuallyRooted<T>`][crate::ManuallyRooted] | Arbitrary | Manual |
+/// | Type | Supported Lifetimes | Unrooting | Cost |
+/// |----------------------------------------------|-----------------------------|-----------|----------------------------------|
+/// | [`Rooted<T>`][crate::Rooted] | Strictly LIFO / stack order | Automatic | very low (LIFO array) |
+/// | [`OwnedRooted<T>`][crate::OwnedRooted] | Arbitrary | Automatic | medium (separate `Arc` to track) |
///
-/// `Rooted<T>` should suffice for most use cases, and provides better
-/// ergonomics, but `ManuallyRooted<T>` exists as a fully-general escape hatch.
+/// `Rooted<T>` should suffice for most use cases, and provides decent
+/// ergonomics. In cases where LIFO scopes are difficult to reason
+/// about, e.g. heap-managed data structures, or when they may cause
+/// erroneous behavior, e.g. in errors that are propagated up the call
+/// stack, `OwnedRooted<T>` provides very safe ergonomics but at a
+/// small dynamic cost for the separate tracking allocation.
///
/// # Scopes
///
@@ -574,8 +823,7 @@ impl RootSet {
///
/// 1. A [`Store`][crate::Store] is the outermost rooting scope. Creating a
/// `Root<T>` directly inside a `Store` permanently roots the underlying
-/// object, similar to dropping a
-/// [`ManuallyRooted<T>`][crate::ManuallyRooted] without unrooting it.
+/// object.
///
/// 2. A [`Caller`][crate::Caller] provides a rooting scope for the duration of
/// a call from Wasm into a host function. Any objects rooted in a `Caller`
@@ -752,7 +1000,7 @@ impl<T: GcRef> Rooted<T> {
self.inner.comes_from_same_store(store)
}
- /// Create a [`ManuallyRooted<T>`][crate::ManuallyRooted] holding onto the
+ /// Create an [`OwnedRooted<T>`][crate::OwnedRooted] holding onto the
/// same GC object as `self`.
///
/// Returns `None` if `self` is used outside of its scope and has therefore
@@ -772,7 +1020,7 @@ impl<T: GcRef> Rooted<T> {
/// # fn _foo() -> Result<()> {
/// let mut store = Store::<()>::default();
///
- /// let y: ManuallyRooted<_> = {
+ /// let y: OwnedRooted<_> = {
/// // Create a nested rooting scope.
/// let mut scope = RootScope::new(&mut store);
///
@@ -780,8 +1028,8 @@ impl<T: GcRef> Rooted<T> {
/// let x: Rooted<_> = ExternRef::new(&mut scope, "hello!")?;
///
/// // Extend `x`'s rooting past its scope's lifetime by converting it
- /// // to a `ManuallyRooted`.
- /// x.to_manually_rooted(&mut scope)?
+ /// // to an `OwnedRooted`.
+ /// x.to_owned_rooted(&mut scope)?
/// };
///
/// // Now we can still access the reference outside the scope it was
@@ -789,20 +1037,17 @@ impl<T: GcRef> Rooted<T> {
/// let data = y.data(&store)?.expect("should have host data");
/// let data = data.downcast_ref::<&str>().expect("host data should be str");
/// assert_eq!(*data, "hello!");
- ///
- /// // But we have to manually unroot `y`.
- /// y.unroot(&mut store);
/// # Ok(())
/// # }
/// ```
- pub fn to_manually_rooted(&self, mut store: impl AsContextMut) -> Result<ManuallyRooted<T>> {
- self._to_manually_rooted(store.as_context_mut().0)
+ pub fn to_owned_rooted(&self, mut store: impl AsContextMut) -> Result<OwnedRooted<T>> {
+ self._to_owned_rooted(store.as_context_mut().0)
}
- pub(crate) fn _to_manually_rooted(&self, store: &mut StoreOpaque) -> Result<ManuallyRooted<T>> {
+ pub(crate) fn _to_owned_rooted(&self, store: &mut StoreOpaque) -> Result<OwnedRooted<T>> {
let mut store = AutoAssertNoGc::new(store);
let gc_ref = self.try_clone_gc_ref(&mut store)?;
- Ok(ManuallyRooted::new(&mut store, gc_ref))
+ Ok(OwnedRooted::new(&mut store, gc_ref))
}
/// Are these two `Rooted<T>`s the same GC root?
@@ -831,7 +1076,7 @@ impl<T: GcRef> Rooted<T> {
///
/// // `c` is a different GC root, in a different scope, even though it
/// // is rooting the same object.
- /// let c = a.to_manually_rooted(&mut scope)?.into_rooted(&mut scope);
+ /// let c = a.to_owned_rooted(&mut scope)?.to_rooted(&mut scope);
/// assert!(!Rooted::rooted_eq(a, c));
/// }
///
@@ -858,7 +1103,7 @@ impl<T: GcRef> Rooted<T> {
/// because the scope it was rooted within has been exited.
///
/// Because this method takes any `impl RootedGcRef<T>` arguments, it can be
- /// used to compare, for example, a `Rooted<T>` and a `ManuallyRooted<T>`.
+ /// used to compare, for example, a `Rooted<T>` and a `OwnedRooted<T>`.
///
/// # Panics
///
@@ -882,7 +1127,7 @@ impl<T: GcRef> Rooted<T> {
///
/// // `c` is a different GC root, in a different scope, but still
/// // rooting the same object.
- /// let c = a.to_manually_rooted(&mut scope)?.into_rooted(&mut scope);
+ /// let c = a.to_owned_rooted(&mut scope)?.to_rooted(&mut scope);
/// assert!(!Rooted::ref_eq(&scope, &a, &c)?);
/// }
///
@@ -891,12 +1136,10 @@ impl<T: GcRef> Rooted<T> {
/// // `a` and `x` are rooting different objects.
/// assert!(!Rooted::ref_eq(&store, &a, &x)?);
///
- /// // You can also compare `Rooted<T>`s and `ManuallyRooted<T>`s with this
+ /// // You can also compare `Rooted<T>`s and `OwnedRooted<T>`s with this
/// // function.
- /// let d = a.to_manually_rooted(&mut store)?;
+ /// let d = a.to_owned_rooted(&mut store)?;
/// assert!(Rooted::ref_eq(&store, &a, &d)?);
- ///
- /// d.unroot(&mut store);
/// # Ok(())
/// # }
/// ```
@@ -1277,18 +1520,23 @@ where
}
}
-/// A rooted reference to a garbage-collected `T` with arbitrary lifetime.
+/// A rooted reference to a garbage-collected `T` with automatic lifetime.
///
-/// A `ManuallyRooted<T>` is a strong handle to a garbage-collected `T`,
+/// An `OwnedRooted<T>` is a strong handle to a garbage-collected `T`,
/// preventing its referent (and anything else transitively referenced) from
-/// being collected by the GC until [`unroot`][crate::ManuallyRooted::unroot] is
+/// being collected by the GC until [`unroot`][crate::OwnedRooted::unroot] is
/// explicitly called.
///
-/// The primary way to create a `ManuallyRooted<T>` is to promote a temporary
-/// `Rooted<T>` into a `ManuallyRooted<T>` via its
-/// [`to_manually_rooted`][crate::Rooted::to_manually_rooted] method.
+/// An `OwnedRooted<T>` keeps its rooted GC object alive at least
+/// until the `OwnedRooted<T>` itself is dropped. The
+/// "de-registration" of the root is automatic and is triggered (in a
+/// deferred way) by the drop of this type.
///
-/// `ManuallyRooted<T>` dereferences to its underlying `T`, allowing you to call
+/// The primary way to create an `OwnedRooted<T>` is to promote a temporary
+/// `Rooted<T>` into an `OwnedRooted<T>` via its
+/// [`to_owned_rooted`][crate::Rooted::to_owned_rooted] method.
+///
+/// `OwnedRooted<T>` dereferences to its underlying `T`, allowing you to call
/// `T`'s methods.
///
/// # Example
@@ -1296,108 +1544,60 @@ where
/// ```
/// # use wasmtime::*;
/// # fn _foo() -> Result<()> {
-/// let mut store = Store::<Option<ManuallyRooted<ExternRef>>>::default();
+/// let mut store = Store::<Option<OwnedRooted<ExternRef>>>::default();
///
-/// // Create our `ManuallyRooted` in a nested scope to avoid rooting it for
+/// // Create our `OwnedRooted` in a nested scope to avoid rooting it for
/// // the duration of the store's lifetime.
/// let x = {
/// let mut scope = RootScope::new(&mut store);
/// let x = ExternRef::new(&mut scope, 1234)?;
-/// x.to_manually_rooted(&mut scope)?
+/// x.to_owned_rooted(&mut scope)?
/// };
///
/// // Place `x` into our store.
/// *store.data_mut() = Some(x);
///
/// // Do a bunch stuff that may or may not access, replace, or take `x`...
-///
-/// // At any time, in any arbitrary scope, we can remove `x` from the store
-/// // and unroot it:
-/// if let Some(x) = store.data_mut().take() {
-/// x.unroot(&mut store);
-/// }
/// # Ok(())
/// # }
/// ```
-///
-/// # Differences Between `ManuallyRooted<T>` and `Rooted<T>`
-///
-/// While `ManuallyRooted<T>` can have arbitrary lifetimes, it requires manual
-/// unrooting. This is in contrast to [`Rooted<T>`][crate::Rooted] which is
-/// restricted to strictly last-in-first-out (LIFO, aka stack order) lifetimes,
-/// but comes with automatic unrooting.
-///
-/// | Type | Supported Lifetimes | Unrooting |
-/// |----------------------------------------------|-----------------------------|-----------|
-/// | [`Rooted<T>`][crate::Rooted] | Strictly LIFO / stack order | Automatic |
-/// | [`ManuallyRooted<T>`][crate::ManuallyRooted] | Arbitrary | Manual |
-///
-/// `Rooted<T>` should suffice for most use cases, and provides better
-/// ergonomics, but `ManuallyRooted<T>` exists as a fully-general escape hatch.
-///
-/// # Manual Unrooting
-///
-/// Failure to explicitly call [`unroot`][crate::ManuallyRooted::unroot] (or
-/// another method that consumes `self` and unroots the reference, such as
-/// [`into_rooted`][crate::ManuallyRooted::into_rooted]) will leak the
-/// underlying GC object, preventing it from being garbage collected until its
-/// owning [`Store`][crate::Store] is dropped. That means all of the following
-/// will result in permanently rooting the underlying GC object:
-///
-/// * Implicitly dropping a `ManuallyRooted<T>`:
-///
-/// ```no_run
-/// # use wasmtime::*;
-/// # let get_manually_rooted = || -> ManuallyRooted<ExternRef> { todo!() };
-/// {
-/// let perma_root: ManuallyRooted<_> = get_manually_rooted();
-///
-/// // `perma_root` is implicitly dropped at the end of its scope,
-/// // permanently rooting/leaking its referent.
-/// }
-/// ```
-///
-/// * Explicitly dropping a `ManuallyRooted<T>`: `drop(my_manually_rooted)`.
-///
-/// * Forgetting a `ManuallyRooted<T>`: `std::mem::forget(my_manually_rooted)`.
-///
-/// * Inserting a `ManuallyRooted<T>` into a `std::sync::Arc` or `std::rc::Rc`
-/// cycle.
-///
-/// * Etc...
-///
-/// Wasmtime does *not* assert that a `ManuallyRooted<T>` is unrooted on `Drop`,
-/// or otherwise raise a panic, log a warning, or etc... on failure to manually
-/// unroot. Sometimes leaking is intentional and desirable, particularly when
-/// dealing with short-lived [`Store`][crate::Store]s where unrooting would just
-/// be busy work since the whole store is about to be dropped.
-#[repr(transparent)] // NB: the C API relies on this
-pub struct ManuallyRooted<T>
+pub struct OwnedRooted<T>
where
T: GcRef,
{
inner: GcRootIndex,
+ liveness_flag: Arc<()>,
_phantom: marker::PhantomData<T>,
}
const _: () = {
use crate::{AnyRef, ExternRef};
- // NB: these match the C API which should also be updated if this changes
- assert!(mem::size_of::<ManuallyRooted<AnyRef>>() == 16);
- assert!(mem::align_of::<ManuallyRooted<AnyRef>>() == mem::align_of::<u64>());
- assert!(mem::size_of::<ManuallyRooted<ExternRef>>() == 16);
- assert!(mem::align_of::<ManuallyRooted<ExternRef>>() == mem::align_of::<u64>());
+ // NB: these match the C API which should also be updated if this changes.
+ //
+ // The size is really "16 + pointer + alignment", which is either
+ // 20 bytes on some 32-bit platforms or 24 bytes on other 32-bit
+ // platforms (e.g., riscv32, which adds an extra 4 bytes of
+ // padding) and 64-bit platforms.
+ assert!(
+ mem::size_of::<OwnedRooted<AnyRef>>() >= 16 && mem::size_of::<OwnedRooted<AnyRef>>() <= 24
+ );
+ assert!(mem::align_of::<OwnedRooted<AnyRef>>() == mem::align_of::<u64>());
+ assert!(
+ mem::size_of::<OwnedRooted<ExternRef>>() >= 16
+ && mem::size_of::<OwnedRooted<ExternRef>>() <= 24
+ );
+ assert!(mem::align_of::<OwnedRooted<ExternRef>>() == mem::align_of::<u64>());
};
-impl<T: GcRef> Debug for ManuallyRooted<T> {
+impl<T: GcRef> Debug for OwnedRooted<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- let name = format!("ManuallyRooted<{}>", any::type_name::<T>());
+ let name = format!("OwnedRooted<{}>", any::type_name::<T>());
f.debug_struct(&name).field("inner", &self.inner).finish()
}
}
-impl<T: GcRef> Deref for ManuallyRooted<T> {
+impl<T: GcRef> Deref for OwnedRooted<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
@@ -1405,11 +1605,21 @@ impl<T: GcRef> Deref for ManuallyRooted<T> {
}
}
-impl<T> ManuallyRooted<T>
+impl<T: GcRef> Clone for OwnedRooted<T> {
+ fn clone(&self) -> Self {
+ OwnedRooted {
+ inner: self.inner,
+ liveness_flag: self.liveness_flag.clone(),
+ _phantom: marker::PhantomData,
+ }
+ }
+}
+
+impl<T> OwnedRooted<T>
where
T: GcRef,
{
- /// Construct a new manually-rooted GC root.
+ /// Construct a new owned GC root.
///
/// `gc_ref` should belong to `store`'s heap; failure to uphold this is
/// memory safe but will result in general failures down the line such as
@@ -1419,101 +1629,49 @@ where
/// that `T` represents. Failure to uphold this invariant is memory safe but
/// will result in general incorrectness such as panics and wrong results.
pub(crate) fn new(store: &mut AutoAssertNoGc<'_>, gc_ref: VMGcRef) -> Self {
- let id = store.gc_roots_mut().manually_rooted.alloc(gc_ref);
- ManuallyRooted {
+ // We always have the opportunity to trim and unregister stale
+ // owned roots whenever we have a mut borrow to the store. We
+ // take the opportunity to do so here to avoid tying growth of
+ // the root-set to the GC frequency -- it is much cheaper to
+ // eagerly trim these roots. Note that the trimming keeps a
+ // "high water mark" that grows exponentially, so we have
+ // amortized constant time even though an individual trim
+ // takes time linear in the number of roots.
+ store.trim_gc_liveness_flags(false);
+
+ let roots = store.gc_roots_mut();
+ let id = roots.owned_rooted.alloc(gc_ref);
+ let liveness_flag = Arc::new(());
+ roots
+ .liveness_flags
+ .push((Arc::downgrade(&liveness_flag), id));
+ OwnedRooted {
inner: GcRootIndex {
store_id: store.id(),
generation: 0,
- index: PackedIndex::new_manual(id),
+ index: PackedIndex::new_owned(id),
},
+ liveness_flag,
_phantom: marker::PhantomData,
}
}
#[inline]
pub(crate) fn comes_from_same_store(&self, store: &StoreOpaque) -> bool {
- debug_assert!(self.inner.index.is_manual());
+ debug_assert!(self.inner.index.is_owned());
self.inner.comes_from_same_store(store)
}
- /// Clone this `ManuallyRooted`.
- ///
- /// Does not consume or unroot `self`: both `self` and the new
- /// `ManuallyRooted` return value will need to be manually unrooted.
- ///
- /// # Panics
- ///
- /// Panics if `self` is not associated with the given `store`.
- ///
- /// # Example
- ///
- /// ```
- /// # use wasmtime::*;
- /// # fn _foo() -> Result<()> {
- /// let mut store = Store::<Vec<ManuallyRooted<ExternRef>>>::default();
- ///
- /// // Create our `ManuallyRooted` in a nested scope to avoid rooting it for
- /// // the duration of the store's lifetime.
- /// let x = {
- /// let mut scope = RootScope::new(&mut store);
- /// let x = ExternRef::new(&mut scope, 1234)?;
- /// x.to_manually_rooted(&mut scope)?
- /// };
- ///
- /// // Push five clones of `x` into our store.
- /// for _ in 0..5 {
- /// let x_clone = x.clone(&mut store);
- /// store.data_mut().push(x_clone);
- /// }
- /// # Ok(())
- /// # }
- /// ```
- pub fn clone(&self, mut store: impl AsContextMut) -> Self {
- self._clone(store.as_context_mut().0)
- }
-
- pub(crate) fn _clone(&self, store: &mut StoreOpaque) -> Self {
- let mut store = AutoAssertNoGc::new(store);
- let gc_ref = self
- .clone_gc_ref(&mut store)
- .expect("ManuallyRooted always has a gc ref");
- Self::new(&mut store, gc_ref)
- }
-
- /// Unroot this GC object.
- ///
- /// Failure to call this method will result in the GC object, and anything
- /// it transitively references, being kept alive (aka "leaking") for the
- /// entirety of the store's lifetime.
- ///
- /// See the type-level docs for example usage.
- pub fn unroot(self, mut store: impl AsContextMut) {
- self._unroot(store.as_context_mut().0)
- }
-
- pub(crate) fn _unroot(self, store: &mut StoreOpaque) {
- assert!(
- self.comes_from_same_store(store),
- "object used with wrong store"
- );
-
- let mut store = AutoAssertNoGc::new(store);
- let id = self.inner.index.as_manual().unwrap();
- let roots = store.gc_roots_mut();
- let gc_ref = roots.manually_rooted.dealloc(id);
- store.unwrap_gc_store_mut().drop_gc_ref(gc_ref);
- }
-
- /// Clone this `ManuallyRooted<T>` into a `Rooted<T>`.
+ /// Clone this `OwnedRooted<T>` into a `Rooted<T>`.
///
- /// This operation does not consume or unroot this `ManuallyRooted<T>`.
+ /// This operation does not consume or unroot this `OwnedRooted<T>`.
///
/// The underlying GC object is re-rooted in the given context's scope. The
/// resulting `Rooted<T>` is only valid during the given context's
/// scope. See the [`Rooted<T>`][crate::Rooted] documentation for more
/// details on rooting scopes.
///
- /// This operation does not consume or unroot this `ManuallyRooted<T>`.
+ /// This operation does not consume or unroot this `OwnedRooted<T>`.
///
/// # Panics
///
@@ -1528,24 +1686,19 @@ where
///
/// let root1: Rooted<_>;
///
- /// let manual = {
+ /// let owned = {
/// let mut scope = RootScope::new(&mut store);
/// root1 = ExternRef::new(&mut scope, 1234)?;
- /// root1.to_manually_rooted(&mut scope)?
+ /// root1.to_owned_rooted(&mut scope)?
/// };
///
/// // `root1` is no longer accessible because it was unrooted when `scope`
/// // was dropped.
/// assert!(root1.data(&store).is_err());
///
- /// // But we can re-root `manual` into this scope.
- /// let root2 = manual.to_rooted(&mut store);
+ /// // But we can re-root `owned` into this scope.
+ /// let root2 = owned.to_rooted(&mut store);
/// assert!(root2.data(&store).is_ok());
- ///
- /// // And we also still have access to `manual` and we still have to
- /// // manually unroot it.
- /// assert!(manual.data(&store).is_ok());
- /// manual.unroot(&mut store);
/// # Ok(())
/// # }
/// ```
@@ -1563,61 +1716,6 @@ where
Rooted::new(&mut store, gc_ref)
}
- /// Convert this `ManuallyRooted<T>` into a `Rooted<T>`.
- ///
- /// The underlying GC object is re-rooted in the given context's scope. The
- /// resulting `Rooted<T>` is only valid during the given context's
- /// scope. See the [`Rooted<T>`][crate::Rooted] documentation for more
- /// details on rooting scopes.
- ///
- /// This operation consumes and unroots this `ManuallyRooted<T>`.
- ///
- /// # Panics
- ///
- /// Panics if this object is not associate with the given context's store.
- ///
- /// # Example
- ///
- /// ```
- /// # use wasmtime::*;
- /// # fn _foo() -> Result<()> {
- /// let mut store = Store::<()>::default();
- ///
- /// let root1: Rooted<_>;
- ///
- /// let manual = {
- /// let mut scope = RootScope::new(&mut store);
- /// root1 = ExternRef::new(&mut scope, 1234)?;
- /// root1.to_manually_rooted(&mut scope)?
- /// };
- ///
- /// // `root1` is no longer accessible because it was unrooted when `scope`
- /// // was dropped.
- /// assert!(root1.data(&store).is_err());
- ///
- /// // But we can re-root `manual` into this scope.
- /// let root2 = manual.into_rooted(&mut store);
- /// assert!(root2.data(&store).is_ok());
- ///
- /// // `manual` was consumed by the `into_rooted` call, and we no longer
- /// // have access to it, nor need to manually unroot it.
- /// # Ok(())
- /// # }
- /// ```
- pub fn into_rooted(self, mut context: impl AsContextMut) -> Rooted<T> {
- self._into_rooted(context.as_context_mut().0)
- }
-
- pub(crate) fn _into_rooted(self, store: &mut StoreOpaque) -> Rooted<T> {
- assert!(
- self.comes_from_same_store(store),
- "object used with wrong store"
- );
- let rooted = self._to_rooted(store);
- self._unroot(store);
- rooted
- }
-
/// Are these two GC roots referencing the same underlying GC object?
///
/// This function will return `true` even when `a` and `b` are different GC
@@ -1625,7 +1723,7 @@ where
/// are rooting the same underlying GC object.
///
/// Because this method takes any `impl RootedGcRef<T>` arguments, it can be
- /// used to compare, for example, a `Rooted<T>` and a `ManuallyRooted<T>`.
+ /// used to compare, for example, a `Rooted<T>` and an `OwnedRooted<T>`.
///
/// # Panics
///
@@ -1645,28 +1743,23 @@ where
/// {
/// let mut scope = RootScope::new(&mut store);
///
- /// a = ExternRef::new(&mut scope, "hello")?.to_manually_rooted(&mut scope)?;
- /// b = a.clone(&mut scope);
+ /// a = ExternRef::new(&mut scope, "hello")?.to_owned_rooted(&mut scope)?;
+ /// b = a.clone();
///
/// // `a` and `b` are rooting the same object.
- /// assert!(ManuallyRooted::ref_eq(&scope, &a, &b)?);
+ /// assert!(OwnedRooted::ref_eq(&scope, &a, &b)?);
///
/// // `c` is a different GC root, is in a different scope, and is a
- /// // `Rooted<T>` instead of a `ManuallyRooted<T>`, but is still rooting
+ /// // `Rooted<T>` instead of a `OwnedRooted<T>`, but is still rooting
/// // the same object.
/// let c = a.to_rooted(&mut scope);
- /// assert!(ManuallyRooted::ref_eq(&scope, &a, &c)?);
+ /// assert!(OwnedRooted::ref_eq(&scope, &a, &c)?);
///
- /// x = ExternRef::new(&mut scope, "goodbye")?.to_manually_rooted(&mut scope)?;
+ /// x = ExternRef::new(&mut scope, "goodbye")?.to_owned_rooted(&mut scope)?;
///
/// // `a` and `x` are rooting different objects.
- /// assert!(!ManuallyRooted::ref_eq(&scope, &a, &x)?);
+ /// assert!(!OwnedRooted::ref_eq(&scope, &a, &x)?);
/// }
- ///
- /// // Unroot our manually-rooted GC references.
- /// a.unroot(&mut store);
- /// b.unroot(&mut store);
- /// x.unroot(&mut store);
/// # Ok(())
/// # }
/// ```
@@ -1684,7 +1777,7 @@ where
/// root and *not* the underlying GC reference. That means that two
/// different rootings of the same object will hash to different values
/// (modulo hash collisions). If this is undesirable, use the
- /// [`ref_hash`][crate::ManuallyRooted::ref_hash] method instead.
+ /// [`ref_hash`][crate::OwnedRooted::ref_hash] method instead.
pub fn rooted_hash<H>(&self, state: &mut H)
where
H: Hasher,
@@ -1705,47 +1798,25 @@ where
{
let gc_ref = self
.get_gc_ref(store.as_context().0)
- .expect("ManuallyRooted's get_gc_ref is infallible");
+ .expect("OwnedRooted's get_gc_ref is infallible");
gc_ref.hash(state);
}
- #[doc(hidden)]
- pub fn into_parts_for_c_api(self) -> (NonZeroU64, u32, u32) {
- (
- self.inner.store_id.as_raw(),
- self.inner.generation,
- self.inner.index.0,
- )
- }
-
- #[doc(hidden)]
- pub unsafe fn from_raw_parts_for_c_api(a: NonZeroU64, b: u32, c: u32) -> ManuallyRooted<T> {
- ManuallyRooted {
- inner: GcRootIndex {
- store_id: StoreId::from_raw(a),
- generation: b,
- index: PackedIndex(c),
- },
- _phantom: marker::PhantomData,
- }
- }
-
- /// Cast `self` to a `ManuallyRooted<U>`.
+ /// Cast `self` to an `OwnedRooted<U>`.
///
/// It is the caller's responsibility to ensure that `self` is actually a
/// `U`. Failure to uphold this invariant will be memory safe but will
/// result in general incorrectness such as panics and wrong results.
- pub(crate) fn unchecked_cast<U: GcRef>(self) -> ManuallyRooted<U> {
- let u = ManuallyRooted {
+ pub(crate) fn unchecked_cast<U: GcRef>(self) -> OwnedRooted<U> {
+ OwnedRooted {
inner: self.inner,
+ liveness_flag: self.liveness_flag,
_phantom: core::marker::PhantomData,
- };
- core::mem::forget(self);
- u
+ }
}
/// Common implementation of the `WasmTy::store` trait method for all
- /// `ManuallyRooted<T>`s.
+ /// `OwnedRooted<T>`s.
pub(super) fn wasm_ty_store(
self,
store: &mut AutoAssertNoGc<'_>,
@@ -1759,7 +1830,7 @@ where
}
/// Common implementation of the `WasmTy::load` trait method for all
- /// `ManuallyRooted<T>`s.
+ /// `OwnedRooted<T>`s.
pub(super) fn wasm_ty_load(
store: &mut AutoAssertNoGc<'_>,
raw_gc_ref: u32,
@@ -1770,14 +1841,12 @@ where
let gc_ref = store.clone_gc_ref(&gc_ref);
RootSet::with_lifo_scope(store, |store| {
let rooted = from_cloned_gc_ref(store, gc_ref);
- rooted
- ._to_manually_rooted(store)
- .expect("rooted is in scope")
+ rooted._to_owned_rooted(store).expect("rooted is in scope")
})
}
/// Common implementation of the `WasmTy::store` trait method for all
- /// `Option<ManuallyRooted<T>>`s.
+ /// `Option<OwnedRooted<T>>`s.
pub(super) fn wasm_ty_option_store(
me: Option<Self>,
store: &mut AutoAssertNoGc<'_>,
@@ -1794,7 +1863,7 @@ where
}
/// Common implementation of the `WasmTy::load` trait method for all
- /// `Option<ManuallyRooted<T>>`s.
+ /// `Option<OwnedRooted<T>>`s.
pub(super) fn wasm_ty_option_load(
store: &mut AutoAssertNoGc<'_>,
raw_gc_ref: u32,
@@ -1804,24 +1873,77 @@ where
let gc_ref = store.clone_gc_ref(&gc_ref);
RootSet::with_lifo_scope(store, |store| {
let rooted = from_cloned_gc_ref(store, gc_ref);
- Some(
- rooted
- ._to_manually_rooted(store)
- .expect("rooted is in scope"),
- )
+ Some(rooted._to_owned_rooted(store).expect("rooted is in scope"))
})
}
+
+ #[doc(hidden)]
+ pub fn into_parts_for_c_api(self) -> (NonZeroU64, u32, u32, *const ()) {
+ (
+ self.inner.store_id.as_raw(),
+ self.inner.generation,
+ self.inner.index.0,
+ Arc::into_raw(self.liveness_flag),
+ )
+ }
+
+ #[doc(hidden)]
+ pub unsafe fn from_borrowed_raw_parts_for_c_api(
+ a: NonZeroU64,
+ b: u32,
+ c: u32,
+ d: *const (),
+ ) -> OwnedRooted<T> {
+ // We are given a *borrow* of the Arc. This is a little
+ // sketchy because `Arc::from_raw()` takes *ownership* of the
+ // passed-in pointer, so we need to clone then forget that
+ // original.
+ let liveness_flag = {
+ let original = unsafe { Arc::from_raw(d) };
+ let clone = original.clone();
+ core::mem::forget(original);
+ clone
+ };
+ OwnedRooted {
+ inner: GcRootIndex {
+ store_id: StoreId::from_raw(a),
+ generation: b,
+ index: PackedIndex(c),
+ },
+ liveness_flag,
+ _phantom: marker::PhantomData,
+ }
+ }
+
+ #[doc(hidden)]
+ pub unsafe fn from_owned_raw_parts_for_c_api(
+ a: NonZeroU64,
+ b: u32,
+ c: u32,
+ d: *const (),
+ ) -> OwnedRooted<T> {
+ let liveness_flag = unsafe { Arc::from_raw(d) };
+ OwnedRooted {
+ inner: GcRootIndex {
+ store_id: StoreId::from_raw(a),
+ generation: b,
+ index: PackedIndex(c),
+ },
+ liveness_flag,
+ _phantom: marker::PhantomData,
+ }
+ }
}
-impl<T: GcRef> RootedGcRefImpl<T> for ManuallyRooted<T> {
+impl<T: GcRef> RootedGcRefImpl<T> for OwnedRooted<T> {
fn get_gc_ref<'a>(&self, store: &'a StoreOpaque) -> Option<&'a VMGcRef> {
assert!(
self.comes_from_same_store(store),
"object used with wrong store"
);
- let id = self.inner.index.as_manual().unwrap();
- store.gc_roots().manually_rooted.get(id)
+ let id = self.inner.index.as_owned().unwrap();
+ store.gc_roots().owned_rooted.get(id)
}
}
@@ -1836,6 +1958,6 @@ mod tests {
// Try to keep tabs on the size of these things. Don't want them growing
// unintentionally.
assert_eq!(std::mem::size_of::<Rooted<ExternRef>>(), 16);
- assert_eq!(std::mem::size_of::<ManuallyRooted<ExternRef>>(), 16);
+ assert!(std::mem::size_of::<OwnedRooted<ExternRef>>() <= 24);
}
}
diff --git a/crates/wasmtime/src/runtime/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/gc/enabled/structref.rs
index 234cf0baf4..b348338e61 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/structref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/structref.rs
@@ -6,7 +6,7 @@ use crate::vm::{self, VMGcHeader, VMStore, VMStructRef};
use crate::{AnyRef, FieldType};
use crate::{
AsContext, AsContextMut, EqRef, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType,
- ManuallyRooted, RefType, Rooted, StructType, Val, ValRaw, ValType, WasmTy,
+ OwnedRooted, RefType, Rooted, StructType, Val, ValRaw, ValType, WasmTy,
prelude::*,
store::{AutoAssertNoGc, StoreContextMut, StoreOpaque, StoreResourceLimiter},
};
@@ -103,7 +103,7 @@ impl StructRefPre {
/// the host into dereferencing it and segfaulting or worse.
///
/// Note that you can also use `Rooted<StructRef>` and
-/// `ManuallyRooted<StructRef>` as a type parameter with
+/// `OwnedRooted<StructRef>` as a type parameter with
/// [`Func::typed`][crate::Func::typed]- and
/// [`Func::wrap`][crate::Func::wrap]-style APIs.
///
@@ -185,16 +185,16 @@ impl Rooted<StructRef> {
}
}
-impl ManuallyRooted<StructRef> {
+impl OwnedRooted<StructRef> {
/// Upcast this `structref` into an `anyref`.
#[inline]
- pub fn to_anyref(self) -> ManuallyRooted<AnyRef> {
+ pub fn to_anyref(self) -> OwnedRooted<AnyRef> {
self.unchecked_cast()
}
/// Upcast this `structref` into an `eqref`.
#[inline]
- pub fn to_eqref(self) -> ManuallyRooted<EqRef> {
+ pub fn to_eqref(self) -> OwnedRooted<EqRef> {
self.unchecked_cast()
}
}
@@ -723,7 +723,7 @@ unsafe impl WasmTy for Option<Rooted<StructRef>> {
}
}
-unsafe impl WasmTy for ManuallyRooted<StructRef> {
+unsafe impl WasmTy for OwnedRooted<StructRef> {
#[inline]
fn valtype() -> ValType {
ValType::Ref(RefType::new(false, HeapType::Struct))
@@ -775,7 +775,7 @@ unsafe impl WasmTy for ManuallyRooted<StructRef> {
}
}
-unsafe impl WasmTy for Option<ManuallyRooted<StructRef>> {
+unsafe impl WasmTy for Option<OwnedRooted<StructRef>> {
#[inline]
fn valtype() -> ValType {
ValType::STRUCTREF
@@ -796,7 +796,7 @@ unsafe impl WasmTy for Option<ManuallyRooted<StructRef>> {
) -> Result<()> {
match self {
Some(s) => {
- ManuallyRooted::<StructRef>::dynamic_concrete_type_check(s, store, nullable, ty)
+ OwnedRooted::<StructRef>::dynamic_concrete_type_check(s, store, nullable, ty)
}
None => {
ensure!(
@@ -814,11 +814,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<StructRef>> {
}
fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {
- <ManuallyRooted<StructRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
+ <OwnedRooted<StructRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)
}
unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {
- <ManuallyRooted<StructRef>>::wasm_ty_option_load(
+ <OwnedRooted<StructRef>>::wasm_ty_option_load(
store,
ptr.get_anyref(),
StructRef::from_cloned_gc_ref,
diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs
index 5be8a7d717..70f60f444f 100644
--- a/crates/wasmtime/src/runtime/store/gc.rs
+++ b/crates/wasmtime/src/runtime/store/gc.rs
@@ -22,6 +22,7 @@ impl StoreOpaque {
bytes_needed: Option<u64>,
) -> Option<VMGcRef> {
let mut scope = crate::OpaqueRootScope::new(self);
+ scope.trim_gc_liveness_flags(true);
let store_id = scope.id();
let root = root.map(|r| scope.gc_roots_mut().push_lifo_root(store_id, r));
@@ -36,6 +37,16 @@ impl StoreOpaque {
})
}
+ // This lives on the Store because it must simultaneously borrow
+ // `gc_store` and `gc_roots`, and is invoked from other modules to
+ // which we do not want to expose the raw fields for piecewise
+ // borrows.
+ pub(crate) fn trim_gc_liveness_flags(&mut self, eager: bool) {
+ if let Some(gc_store) = self.gc_store.as_mut() {
+ self.gc_roots.trim_liveness_flags(gc_store, eager);
+ }
+ }
+
async fn grow_or_collect_gc_heap(
&mut self,
limiter: Option<&mut StoreResourceLimiter<'_>>,
diff --git a/tests/all/func.rs b/tests/all/func.rs
index 2c6209a5ff..98e39f8602 100644
--- a/tests/all/func.rs
+++ b/tests/all/func.rs
@@ -401,18 +401,18 @@ fn func_constructors() {
loop {}
});
Func::wrap(&mut store, || -> Option<Rooted<ExternRef>> { None });
- Func::wrap(&mut store, || -> ManuallyRooted<ExternRef> {
+ Func::wrap(&mut store, || -> OwnedRooted<ExternRef> {
loop {}
});
- Func::wrap(&mut store, || -> Option<ManuallyRooted<ExternRef>> { None });
+ Func::wrap(&mut store, || -> Option<OwnedRooted<ExternRef>> { None });
Func::wrap(&mut store, || -> Rooted<AnyRef> {
loop {}
});
Func::wrap(&mut store, || -> Option<Rooted<AnyRef>> { None });
- Func::wrap(&mut store, || -> ManuallyRooted<AnyRef> {
+ Func::wrap(&mut store, || -> OwnedRooted<AnyRef> {
loop {}
});
- Func::wrap(&mut store, || -> Option<ManuallyRooted<AnyRef>> { None });
+ Func::wrap(&mut store, || -> Option<OwnedRooted<AnyRef>> { None });
Func::wrap(&mut store, || -> I31 {
loop {}
});
@@ -455,23 +455,22 @@ fn func_constructors() {
Func::wrap(&mut store, || -> Result<Option<Rooted<ExternRef>>> {
loop {}
});
- Func::wrap(&mut store, || -> Result<ManuallyRooted<ExternRef>> {
+ Func::wrap(&mut store, || -> Result<OwnedRooted<ExternRef>> {
+ loop {}
+ });
+ Func::wrap(&mut store, || -> Result<Option<OwnedRooted<ExternRef>>> {
loop {}
});
- Func::wrap(
- &mut store,
- || -> Result<Option<ManuallyRooted<ExternRef>>> { loop {} },
- );
Func::wrap(&mut store, || -> Result<Rooted<AnyRef>> {
loop {}
});
Func::wrap(&mut store, || -> Result<Option<Rooted<AnyRef>>> {
loop {}
});
- Func::wrap(&mut store, || -> Result<ManuallyRooted<AnyRef>> {
+ Func::wrap(&mut store, || -> Result<OwnedRooted<AnyRef>> {
loop {}
});
- Func::wrap(&mut store, || -> Result<Option<ManuallyRooted<AnyRef>>> {
+ Func::wrap(&mut store, || -> Result<Option<OwnedRooted<AnyRef>>> {
loop {}
});
Func::wrap(&mut store, || -> Result<I31> {
@@ -601,9 +600,9 @@ fn signatures_match() {
_: i64,
_: i32,
_: Option<Rooted<ExternRef>>,
- _: Option<ManuallyRooted<ExternRef>>,
+ _: Option<OwnedRooted<ExternRef>>,
_: Option<Rooted<AnyRef>>,
- _: Option<ManuallyRooted<AnyRef>>,
+ _: Option<OwnedRooted<AnyRef>>,
_: Option<Func>|
-> f64 { loop {} },
);
@@ -698,10 +697,10 @@ fn import_works() -> Result<()> {
d: f32,
e: f64,
f: Option<Rooted<ExternRef>>,
- g: Option<ManuallyRooted<ExternRef>>,
+ g: Option<OwnedRooted<ExternRef>>,
h: Option<Func>,
i: Option<Rooted<AnyRef>>,
- j: Option<ManuallyRooted<AnyRef>>,
+ j: Option<OwnedRooted<AnyRef>>,
k: Option<I31>|
-> Result<()> {
assert_eq!(a, 100);
@@ -820,15 +819,15 @@ fn get_from_wrapper() {
loop {}
});
assert!(f.typed::<(), Option<Rooted<ExternRef>>>(&store).is_ok());
- let f = Func::wrap(&mut store, || -> ManuallyRooted<ExternRef> {
+ let f = Func::wrap(&mut store, || -> OwnedRooted<ExternRef> {
loop {}
});
- assert!(f.typed::<(), ManuallyRooted<ExternRef>>(&store).is_ok());
- let f = Func::wrap(&mut store, || -> Option<ManuallyRooted<ExternRef>> {
+ assert!(f.typed::<(), OwnedRooted<ExternRef>>(&store).is_ok());
+ let f = Func::wrap(&mut store, || -> Option<OwnedRooted<ExternRef>> {
loop {}
});
assert!(
- f.typed::<(), Option<ManuallyRooted<ExternRef>>>(&store)
+ f.typed::<(), Option<OwnedRooted<ExternRef>>>(&store)
.is_ok()
);
let f = Func::wrap(&mut store, || -> Rooted<AnyRef> {
@@ -839,17 +838,14 @@ fn get_from_wrapper() {
loop {}
});
assert!(f.typed::<(), Option<Rooted<AnyRef>>>(&store).is_ok());
- let f = Func::wrap(&mut store, || -> ManuallyRooted<AnyRef> {
+ let f = Func::wrap(&mut store, || -> OwnedRooted<AnyRef> {
loop {}
});
- assert!(f.typed::<(), ManuallyRooted<AnyRef>>(&store).is_ok());
- let f = Func::wrap(&mut store, || -> Option<ManuallyRooted<AnyRef>> {
+ assert!(f.typed::<(), OwnedRooted<AnyRef>>(&store).is_ok());
+ let f = Func::wrap(&mut store, || -> Option<OwnedRooted<AnyRef>> {
loop {}
});
- assert!(
- f.typed::<(), Option<ManuallyRooted<AnyRef>>>(&store)
- .is_ok()
- );
+ assert!(f.typed::<(), Option<OwnedRooted<AnyRef>>>(&store).is_ok());
let f = Func::wrap(&mut store, || -> I31 {
loop {}
});
@@ -882,24 +878,21 @@ fn get_from_wrapper() {
assert!(f.typed::<Rooted<ExternRef>, ()>(&store).is_ok());
let f = Func::wrap(&mut store, |_: Option<Rooted<ExternRef>>| {});
assert!(f.typed::<Option<Rooted<ExternRef>>, ()>(&store).is_ok());
- let f = Func::wrap(&mut store, |_: ManuallyRooted<ExternRef>| {});
- assert!(f.typed::<ManuallyRooted<ExternRef>, ()>(&store).is_ok());
- let f = Func::wrap(&mut store, |_: Option<ManuallyRooted<ExternRef>>| {});
+ let f = Func::wrap(&mut store, |_: OwnedRooted<ExternRef>| {});
+ assert!(f.typed::<OwnedRooted<ExternRef>, ()>(&store).is_ok());
+ let f = Func::wrap(&mut store, |_: Option<OwnedRooted<ExternRef>>| {});
assert!(
- f.typed::<Option<ManuallyRooted<ExternRef>>, ()>(&store)
+ f.typed::<Option<OwnedRooted<ExternRef>>, ()>(&store)
.is_ok()
);
let f = Func::wrap(&mut store, |_: Rooted<AnyRef>| {});
assert!(f.typed::<Rooted<AnyRef>, ()>(&store).is_ok());
let f = Func::wrap(&mut store, |_: Option<Rooted<AnyRef>>| {});
assert!(f.typed::<Option<Rooted<AnyRef>>, ()>(&store).is_ok());
- let f = Func::wrap(&mut store, |_: ManuallyRooted<AnyRef>| {});
- assert!(f.typed::<ManuallyRooted<AnyRef>, ()>(&store).is_ok());
- let f = Func::wrap(&mut store, |_: Option<ManuallyRooted<AnyRef>>| {});
- assert!(
- f.typed::<Option<ManuallyRooted<AnyRef>>, ()>(&store)
- .is_ok()
- );
+ let f = Func::wrap(&mut store, |_: OwnedRooted<AnyRef>| {});
+ assert!(f.typed::<OwnedRooted<AnyRef>, ()>(&store).is_ok());
+ let f = Func::wrap(&mut store, |_: Option<OwnedRooted<AnyRef>>| {});
+ assert!(f.typed::<Option<OwnedRooted<AnyRef>>, ()>(&store).is_ok());
let f = Func::wrap(&mut store, |_: I31| {});
assert!(f.typed::<I31, ()>(&store).is_ok());
let f = Func::wrap(&mut store, |_: Option<I31>| {});
diff --git a/tests/all/gc.rs b/tests/all/gc.rs
index 546f814e22..e1eb55b55a 100644
--- a/tests/all/gc.rs
+++ b/tests/all/gc.rs
@@ -747,19 +747,19 @@ fn rooted_gets_collected_after_scope_exit() -> Result<()> {
}
#[test]
-fn manually_rooted_gets_collected_after_unrooting() -> Result<()> {
+fn owned_rooted_gets_collected_after_unrooting() -> Result<()> {
let mut store = Store::<()>::default();
let flag = Arc::new(AtomicBool::new(false));
let externref = {
let mut scope = RootScope::new(&mut store);
- ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?.to_manually_rooted(&mut scope)?
+ ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?.to_owned_rooted(&mut scope)?
};
store.gc(None);
assert!(!flag.load(SeqCst), "not dropped when still rooted");
- externref.unroot(&mut store);
+ drop(externref);
store.gc(None);
assert!(flag.load(SeqCst), "dropped after being unrooted");
@@ -1495,3 +1495,75 @@ fn drc_gc_inbetween_host_calls() -> Result<()> {
Ok(())
}
+
+#[test]
+fn owned_rooted() -> Result<()> {
+ let _ = env_logger::try_init();
+
+ let mut config = Config::new();
+ config.wasm_function_references(true);
+ config.wasm_gc(true);
+ config.collector(Collector::DeferredReferenceCounting);
+
+ let engine = Engine::new(&config)?;
+
+ let mut store = Store::new(&engine, ());
+ let inner_dropped = Arc::new(AtomicBool::new(false));
+ let r = {
+ let mut scope = RootScope::new(&mut store);
+ let r = ExternRef::new(&mut scope, SetFlagOnDrop(inner_dropped.clone()))?;
+ r.to_owned_rooted(&mut scope)?
+ };
+ assert!(!inner_dropped.load(SeqCst));
+ store.gc(None);
+ assert!(!inner_dropped.load(SeqCst));
+ let r2 = r.clone();
+ store.gc(None);
+ assert!(!inner_dropped.load(SeqCst));
+ drop(r);
+ store.gc(None);
+ assert!(!inner_dropped.load(SeqCst));
+ drop(r2);
+ store.gc(None);
+ assert!(inner_dropped.load(SeqCst));
+
+ Ok(())
+}
+
+#[test]
+fn owned_rooted_lots_of_root_creation() -> Result<()> {
+ let mut config = Config::new();
+ config.wasm_function_references(true);
+ config.wasm_gc(true);
+ config.collector(Collector::DeferredReferenceCounting);
+
+ let engine = Engine::new(&config)?;
+
+ let mut store = Store::new(&engine, ());
+ let inner_dropped = Arc::new(AtomicBool::new(false));
+ let r = {
+ let mut scope = RootScope::new(&mut store);
+ let r = ExternRef::new(&mut scope, SetFlagOnDrop(inner_dropped.clone()))?;
+ r.to_owned_rooted(&mut scope)?
+ };
+ assert!(!inner_dropped.load(SeqCst));
+ store.gc(None);
+
+ for _ in 0..100_000 {
+ let mut scope = RootScope::new(&mut store);
+ // Go through a LIFO root then back to an owned root to create
+ // a distinct root.
+ let r2 = r.to_rooted(&mut scope);
+ let r3 = r2.to_owned_rooted(&mut scope);
+ drop(r3);
+ }
+
+ store.gc(None);
+ assert!(!inner_dropped.load(SeqCst));
+
+ drop(r);
+ store.gc(None);
+ assert!(inner_dropped.load(SeqCst));
+
+ Ok(())
+}