A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
GC: replace ManuallyRooted with OwnedRooted. (#11514)
Chris Fallin9826719ae049
diff --git a/crates/c-api/include/wasmtime/val.h b/crates/c-api/include/wasmtime/val.hindex bdac5b9c76..4c1af27a5c 100644--- a/crates/c-api/include/wasmtime/val.h+++ b/crates/c-api/include/wasmtime/val.h@@ -32,6 +32,10 @@ extern "C" {* Anyref values are required to be explicitly unrooted via* #wasmtime_anyref_unroot to enable them to be garbage-collected.*+ * If you do not unroot the value, *even if you free the corresponding+ * Store*, there will be some memory leaked, because GC roots use a+ * separate allocation to track liveness.+ ** Null anyref values are represented by this structure and can be tested and* created with the `wasmtime_anyref_is_null` and `wasmtime_anyref_set_null`* functions.@@ -44,6 +48,8 @@ typedef struct wasmtime_anyref {uint32_t __private1;/// Internal to Wasmtime.uint32_t __private2;+ /// Internal to Wasmtime.+ void *__private3;} wasmtime_anyref_t;/// \brief Helper function to initialize the `ref` provided to a null anyref@@ -179,6 +185,8 @@ typedef struct wasmtime_externref {uint32_t __private1;/// Internal to Wasmtime.uint32_t __private2;+ /// Internal to Wasmtime.+ void *__private3;} wasmtime_externref_t;/// \brief Helper function to initialize the `ref` provided to a null externref@@ -214,6 +222,10 @@ static inline bool wasmtime_externref_is_null(const wasmtime_externref_t *ref) {* in the future with #wasmtime_externref_unroot. If `false` is returned then* the host wasn't able to create more GC values at this time. Performing a GC* may free up enough space to try again.+ *+ * If you do not unroot the value, *even if you free the corresponding+ * Store*, there will be some memory leaked, because GC roots use a+ * separate allocation to track liveness.*/WASM_API_EXTERN bool wasmtime_externref_new(wasmtime_context_t *context,void *data,@@ -418,7 +430,10 @@ typedef union wasmtime_val_raw {// Assert that the shape of this type is as expected since it needs to match// Rust.static inline void __wasmtime_val_assertions() {- static_assert(sizeof(wasmtime_valunion_t) == 16, "should be 16-bytes large");+ static_assert(sizeof(wasmtime_valunion_t) >= 16 &&+ sizeof(wasmtime_valunion_t) <= 24,+ "should be 16 bytes plus a pointer large (plus alignment on "+ "some platforms)");static_assert(__alignof(wasmtime_valunion_t) == 8,"should be 8-byte aligned");static_assert(sizeof(wasmtime_val_raw_t) == 16, "should be 16 bytes large");@@ -437,6 +452,10 @@ static inline void __wasmtime_val_assertions() {* consume a #wasmtime_val_t do not take ownership, but APIs that return* #wasmtime_val_t require that #wasmtime_val_unroot is called to clean up* any possible GC roots in the value.+ *+ * If you do not unroot the value, *even if you free the corresponding+ * Store*, there will be some memory leaked, because GC roots use a+ * separate allocation to track liveness.*/typedef struct wasmtime_val {/// Discriminant of which field of #of is valid.diff --git a/crates/c-api/src/ref.rs b/crates/c-api/src/ref.rsindex ff2ee34bf3..e3a5b52d65 100644--- a/crates/c-api/src/ref.rs+++ b/crates/c-api/src/ref.rs@@ -1,6 +1,6 @@use crate::{WasmtimeStoreContextMut, abort};use std::{mem::MaybeUninit, num::NonZeroU64, os::raw::c_void, ptr};-use wasmtime::{AnyRef, ExternRef, I31, ManuallyRooted, Ref, RootScope, Val};+use wasmtime::{AnyRef, ExternRef, I31, OwnedRooted, Ref, RootScope, Val};/// `*mut wasm_ref_t` is a reference type (`externref` or `funcref`), as seen by/// the C API. Because we do not have a uniform representation for `funcref`s@@ -188,7 +188,7 @@ pub extern "C" fn wasm_foreign_new(_store: &crate::wasm_store_t) -> Box<wasm_for/// this is dispatched internally. Null anyref values are represented with a/// `store_id` of zero, and otherwise the `rooted` field is valid.///-/// Note that this relies on the Wasmtime definition of `ManuallyRooted` to have+/// Note that this relies on the Wasmtime definition of `OwnedRooted` to have/// a 64-bit store_id first.macro_rules! ref_wrapper {($wasmtime:ident => $c:ident) => {@@ -196,33 +196,50 @@ macro_rules! ref_wrapper {store_id: u64,a: u32,b: u32,+ c: *const (),}impl $c {- pub unsafe fn as_wasmtime(&self) -> Option<ManuallyRooted<$wasmtime>> {+ pub unsafe fn as_wasmtime(&self) -> Option<OwnedRooted<$wasmtime>> {let store_id = NonZeroU64::new(self.store_id)?;- Some(ManuallyRooted::from_raw_parts_for_c_api(- store_id, self.a, self.b,+ Some(OwnedRooted::from_borrowed_raw_parts_for_c_api(+ store_id, self.a, self.b, self.c,+ ))+ }++ pub unsafe fn from_wasmtime(self) -> Option<OwnedRooted<$wasmtime>> {+ let store_id = NonZeroU64::new(self.store_id)?;+ Some(OwnedRooted::from_owned_raw_parts_for_c_api(+ store_id, self.a, self.b, self.c,))}}- impl From<Option<ManuallyRooted<$wasmtime>>> for $c {- fn from(rooted: Option<ManuallyRooted<$wasmtime>>) -> $c {+ impl From<Option<OwnedRooted<$wasmtime>>> for $c {+ fn from(rooted: Option<OwnedRooted<$wasmtime>>) -> $c {let mut ret = $c {store_id: 0,a: 0,b: 0,+ c: core::ptr::null(),};if let Some(rooted) = rooted {- let (store_id, a, b) = rooted.into_parts_for_c_api();+ let (store_id, a, b, c) = rooted.into_parts_for_c_api();ret.store_id = store_id.get();ret.a = a;ret.b = b;+ ret.c = c;}ret}}++ // SAFETY: The `*const ()` comes from (and is converted back+ // into) an `Arc<()>`, and is only accessed as such, so this+ // type is both Send and Sync. These constraints are necessary+ // in the async machinery in this crate.+ unsafe impl Send for $c {}+ unsafe impl Sync for $c {}};}@@ -231,21 +248,21 @@ ref_wrapper!(ExternRef => wasmtime_externref_t);#[unsafe(no_mangle)]pub unsafe extern "C" fn wasmtime_anyref_clone(- cx: WasmtimeStoreContextMut<'_>,+ _cx: WasmtimeStoreContextMut<'_>,anyref: Option<&wasmtime_anyref_t>,out: &mut MaybeUninit<wasmtime_anyref_t>,) {- let anyref = anyref.and_then(|a| a.as_wasmtime()).map(|a| a.clone(cx));+ let anyref = anyref.and_then(|a| a.as_wasmtime());crate::initialize(out, anyref.into());}#[unsafe(no_mangle)]pub unsafe extern "C" fn wasmtime_anyref_unroot(- cx: WasmtimeStoreContextMut<'_>,+ _cx: WasmtimeStoreContextMut<'_>,val: Option<&mut MaybeUninit<wasmtime_anyref_t>>,) {- if let Some(val) = val.and_then(|v| v.assume_init_read().as_wasmtime()) {- val.unroot(cx);+ if let Some(val) = val.and_then(|v| v.assume_init_read().from_wasmtime()) {+ drop(val);}}@@ -266,8 +283,8 @@ pub unsafe extern "C" fn wasmtime_anyref_from_raw(val: &mut MaybeUninit<wasmtime_anyref_t>,) {let mut scope = RootScope::new(cx);- let anyref = AnyRef::from_raw(&mut scope, raw)- .map(|a| a.to_manually_rooted(&mut scope).expect("in scope"));+ let anyref =+ AnyRef::from_raw(&mut scope, raw).map(|a| a.to_owned_rooted(&mut scope).expect("in scope"));crate::initialize(val, anyref.into());}@@ -279,7 +296,7 @@ pub extern "C" fn wasmtime_anyref_from_i31() {let mut scope = RootScope::new(cx);let anyref = AnyRef::from_i31(&mut scope, I31::wrapping_u32(val));- let anyref = anyref.to_manually_rooted(&mut scope).expect("in scope");+ let anyref = anyref.to_owned_rooted(&mut scope).expect("in scope");crate::initialize(out, Some(anyref).into())}@@ -290,10 +307,10 @@ pub unsafe extern "C" fn wasmtime_anyref_i31_get_u(dst: &mut MaybeUninit<u32>,) -> bool {match anyref.and_then(|a| a.as_wasmtime()) {- Some(anyref) if anyref.is_i31(&cx).expect("ManuallyRooted always in scope") => {+ Some(anyref) if anyref.is_i31(&cx).expect("OwnedRooted always in scope") => {let val = anyref.unwrap_i31(&cx)- .expect("ManuallyRooted always in scope")+ .expect("OwnedRooted always in scope").get_u32();crate::initialize(dst, val);true@@ -309,10 +326,10 @@ pub unsafe extern "C" fn wasmtime_anyref_i31_get_s(dst: &mut MaybeUninit<i32>,) -> bool {match anyref.and_then(|a| a.as_wasmtime()) {- Some(anyref) if anyref.is_i31(&cx).expect("ManuallyRooted always in scope") => {+ Some(anyref) if anyref.is_i31(&cx).expect("OwnedRooted always in scope") => {let val = anyref.unwrap_i31(&cx)- .expect("ManuallyRooted always in scope")+ .expect("OwnedRooted always in scope").get_i32();crate::initialize(dst, val);true@@ -333,7 +350,7 @@ pub extern "C" fn wasmtime_externref_new(Ok(e) => e,Err(_) => return false,};- let e = e.to_manually_rooted(&mut scope).expect("in scope");+ let e = e.to_owned_rooted(&mut scope).expect("in scope");crate::initialize(out, Some(e).into());true}@@ -354,21 +371,21 @@ pub unsafe extern "C" fn wasmtime_externref_data(#[unsafe(no_mangle)]pub unsafe extern "C" fn wasmtime_externref_clone(- cx: WasmtimeStoreContextMut<'_>,+ _cx: WasmtimeStoreContextMut<'_>,externref: Option<&wasmtime_externref_t>,out: &mut MaybeUninit<wasmtime_externref_t>,) {- let externref = externref.and_then(|e| e.as_wasmtime()).map(|e| e.clone(cx));+ let externref = externref.and_then(|e| e.as_wasmtime());crate::initialize(out, externref.into());}#[unsafe(no_mangle)]pub unsafe extern "C" fn wasmtime_externref_unroot(- cx: WasmtimeStoreContextMut<'_>,+ _cx: WasmtimeStoreContextMut<'_>,val: Option<&mut MaybeUninit<wasmtime_externref_t>>,) {- if let Some(val) = val.and_then(|v| v.assume_init_read().as_wasmtime()) {- val.unroot(cx);+ if let Some(val) = val.and_then(|v| v.assume_init_read().from_wasmtime()) {+ drop(val);}}@@ -390,6 +407,6 @@ pub unsafe extern "C" fn wasmtime_externref_from_raw() {let mut scope = RootScope::new(cx);let rooted = ExternRef::from_raw(&mut scope, raw)- .map(|e| e.to_manually_rooted(&mut scope).expect("in scope"));+ .map(|e| e.to_owned_rooted(&mut scope).expect("in scope"));crate::initialize(val, rooted.into());}diff --git a/crates/c-api/src/val.rs b/crates/c-api/src/val.rsindex efc53cf529..c9c4614743 100644--- a/crates/c-api/src/val.rs+++ b/crates/c-api/src/val.rs@@ -152,7 +152,8 @@ pub union wasmtime_val_union {}const _: () = {- assert!(std::mem::size_of::<wasmtime_val_union>() == 16);+ // This is forced to 24 or 20 bytes by `anyref` and `externref`.+ assert!(std::mem::size_of::<wasmtime_val_union>() <= 24);assert!(std::mem::align_of::<wasmtime_val_union>() == std::mem::align_of::<u64>());};@@ -235,15 +236,13 @@ impl wasmtime_val_t {Val::AnyRef(a) => wasmtime_val_t {kind: crate::WASMTIME_ANYREF,of: wasmtime_val_union {- anyref: ManuallyDrop::new(a.and_then(|a| a.to_manually_rooted(cx).ok()).into()),+ anyref: ManuallyDrop::new(a.and_then(|a| a.to_owned_rooted(cx).ok()).into()),},},Val::ExternRef(e) => wasmtime_val_t {kind: crate::WASMTIME_EXTERNREF,of: wasmtime_val_union {- externref: ManuallyDrop::new(- e.and_then(|e| e.to_manually_rooted(cx).ok()).into(),- ),+ externref: ManuallyDrop::new(e.and_then(|e| e.to_owned_rooted(cx).ok()).into()),},},Val::FuncRef(func) => wasmtime_val_t {@@ -298,19 +297,19 @@ impl wasmtime_val_t {#[unsafe(no_mangle)]pub unsafe extern "C" fn wasmtime_val_unroot(- cx: WasmtimeStoreContextMut<'_>,+ _cx: WasmtimeStoreContextMut<'_>,val: &mut MaybeUninit<wasmtime_val_t>,) {let val = val.assume_init_read();match val.kind {crate::WASMTIME_ANYREF => {if let Some(val) = ManuallyDrop::into_inner(val.of.anyref).as_wasmtime() {- val.unroot(cx);+ drop(val);}}crate::WASMTIME_EXTERNREF => {if let Some(val) = ManuallyDrop::into_inner(val.of.externref).as_wasmtime() {- val.unroot(cx);+ drop(val);}}_ => {}diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rsindex 400f2186dd..ba8df37b1f 100644--- a/crates/wasmtime/src/runtime/func.rs+++ b/crates/wasmtime/src/runtime/func.rs@@ -589,7 +589,7 @@ impl Func {/// | `Option<NoneRef>` | `nullref` aka `(ref null none)` |/// | `NoneRef` | `(ref none)` |///- /// Note that anywhere a `Rooted<T>` appears, a `ManuallyRooted<T>` may also+ /// Note that anywhere a `Rooted<T>` appears, a `OwnedRooted<T>` may also/// be used.////// Any of the Rust types can be returned from the closure as well, in@@ -1346,7 +1346,7 @@ impl Func {/// | `v128` | `V128` on `x86-64` and `aarch64` only |////// (Note that this mapping is the same as that of [`Func::wrap`], and that- /// anywhere a `Rooted<T>` appears, a `ManuallyRooted<T>` may also appear).+ /// anywhere a `Rooted<T>` appears, a `OwnedRooted<T>` may also appear).////// Note that once the [`TypedFunc`] return value is acquired you'll use either/// [`TypedFunc::call`] or [`TypedFunc::call_async`] as necessary to actually invokediff --git a/crates/wasmtime/src/runtime/gc.rs b/crates/wasmtime/src/runtime/gc.rsindex 9c050110ac..00dfaa1601 100644--- a/crates/wasmtime/src/runtime/gc.rs+++ b/crates/wasmtime/src/runtime/gc.rs@@ -28,7 +28,7 @@ impl<T> GcRef for T where T: GcRefImpl {}/// A trait implemented for GC references that are guaranteed to be rooted:////// * [`Rooted<T>`][crate::Rooted]-/// * [`ManuallyRooted<T>`][crate::ManuallyRooted]+/// * [`OwnedRooted<T>`][crate::OwnedRooted]////// You can use this to abstract over the different kinds of rooted GC/// references. Note that `Deref<Target = T>` is a supertrait fordiff --git a/crates/wasmtime/src/runtime/gc/disabled/anyref.rs b/crates/wasmtime/src/runtime/gc/disabled/anyref.rsindex e93951c2ce..94915fd224 100644--- a/crates/wasmtime/src/runtime/gc/disabled/anyref.rs+++ b/crates/wasmtime/src/runtime/gc/disabled/anyref.rs@@ -1,6 +1,6 @@use crate::runtime::vm::VMGcRef;use crate::{- ArrayRef, AsContext, AsContextMut, EqRef, GcRefImpl, HeapType, I31, ManuallyRooted, Result,+ ArrayRef, AsContext, AsContextMut, EqRef, GcRefImpl, HeapType, I31, OwnedRooted, Result,Rooted, StructRef,store::{AutoAssertNoGc, StoreOpaque},};@@ -16,9 +16,9 @@ impl From<Rooted<EqRef>> for Rooted<AnyRef> {}}-impl From<ManuallyRooted<EqRef>> for ManuallyRooted<AnyRef> {+impl From<OwnedRooted<EqRef>> for OwnedRooted<AnyRef> {#[inline]- fn from(s: ManuallyRooted<EqRef>) -> Self {+ fn from(s: OwnedRooted<EqRef>) -> Self {match s.inner {}}}@@ -30,9 +30,9 @@ impl From<Rooted<StructRef>> for Rooted<AnyRef> {}}-impl From<ManuallyRooted<StructRef>> for ManuallyRooted<AnyRef> {+impl From<OwnedRooted<StructRef>> for OwnedRooted<AnyRef> {#[inline]- fn from(s: ManuallyRooted<StructRef>) -> Self {+ fn from(s: OwnedRooted<StructRef>) -> Self {match s.inner {}}}@@ -44,9 +44,9 @@ impl From<Rooted<ArrayRef>> for Rooted<AnyRef> {}}-impl From<ManuallyRooted<ArrayRef>> for ManuallyRooted<AnyRef> {+impl From<OwnedRooted<ArrayRef>> for OwnedRooted<AnyRef> {#[inline]- fn from(s: ManuallyRooted<ArrayRef>) -> Self {+ fn from(s: OwnedRooted<ArrayRef>) -> Self {match s.inner {}}}diff --git a/crates/wasmtime/src/runtime/gc/disabled/eqref.rs b/crates/wasmtime/src/runtime/gc/disabled/eqref.rsindex b248e290ad..151e482b09 100644--- a/crates/wasmtime/src/runtime/gc/disabled/eqref.rs+++ b/crates/wasmtime/src/runtime/gc/disabled/eqref.rs@@ -1,5 +1,5 @@use crate::{- ArrayRef, AsContext, GcRefImpl, HeapType, I31, ManuallyRooted, Result, Rooted, StructRef,+ ArrayRef, AsContext, GcRefImpl, HeapType, I31, OwnedRooted, Result, Rooted, StructRef,store::StoreOpaque,};@@ -14,9 +14,9 @@ impl From<Rooted<StructRef>> for Rooted<EqRef> {}}-impl From<ManuallyRooted<StructRef>> for ManuallyRooted<EqRef> {+impl From<OwnedRooted<StructRef>> for OwnedRooted<EqRef> {#[inline]- fn from(s: ManuallyRooted<StructRef>) -> Self {+ fn from(s: OwnedRooted<StructRef>) -> Self {match s.inner {}}}@@ -28,9 +28,9 @@ impl From<Rooted<ArrayRef>> for Rooted<EqRef> {}}-impl From<ManuallyRooted<ArrayRef>> for ManuallyRooted<EqRef> {+impl From<OwnedRooted<ArrayRef>> for OwnedRooted<EqRef> {#[inline]- fn from(s: ManuallyRooted<ArrayRef>) -> Self {+ fn from(s: OwnedRooted<ArrayRef>) -> Self {match s.inner {}}}diff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rsindex 7f1dddc19c..fcbe9e0fb0 100644--- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs+++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs@@ -99,7 +99,7 @@ impl<T: GcRef> Rooted<T> {match self.inner {}}- pub fn to_manually_rooted(&self, _store: impl AsContextMut) -> Result<ManuallyRooted<T>> {+ pub fn to_owned_rooted(&self, _store: impl AsContextMut) -> Result<OwnedRooted<T>> {match self.inner {}}@@ -157,7 +157,7 @@ where/// This type has been disabled because the `gc` cargo feature was not enabled/// at compile time.-pub struct ManuallyRooted<T>+pub struct OwnedRooted<T>whereT: GcRef,{@@ -165,13 +165,13 @@ where_phantom: marker::PhantomData<T>,}-impl<T: GcRef> Debug for ManuallyRooted<T> {+impl<T: GcRef> Debug for OwnedRooted<T> {fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result {match self.inner {}}}-impl<T: GcRef> Deref for ManuallyRooted<T> {+impl<T: GcRef> Deref for OwnedRooted<T> {type Target = T;fn deref(&self) -> &Self::Target {@@ -179,7 +179,7 @@ impl<T: GcRef> Deref for ManuallyRooted<T> {}}-impl<T> ManuallyRooted<T>+impl<T> OwnedRooted<T>whereT: GcRef,{@@ -187,10 +187,6 @@ wherematch self.inner {}}- pub fn unroot(self, _store: impl AsContextMut) {- match self.inner {}- }-pub fn to_rooted(&self, _context: impl AsContextMut) -> Rooted<T> {match self.inner {}}@@ -200,7 +196,7 @@ where}}-impl<T: GcRef> RootedGcRefImpl<T> for ManuallyRooted<T> {+impl<T: GcRef> RootedGcRefImpl<T> for OwnedRooted<T> {fn assert_unreachable<U>(&self) -> U {match self.inner {}}diff --git a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs b/crates/wasmtime/src/runtime/gc/enabled/anyref.rsindex bcbfc02567..561cb6fbef 100644--- a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs@@ -5,7 +5,7 @@ use crate::prelude::*;use crate::runtime::vm::VMGcRef;use crate::{ArrayRef, ArrayType, AsContext, AsContextMut, EqRef, GcRefImpl, GcRootIndex, HeapType, I31,- ManuallyRooted, RefType, Result, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy,+ OwnedRooted, RefType, Result, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy,store::{AutoAssertNoGc, StoreOpaque},};use core::mem;@@ -23,7 +23,7 @@ use wasmtime_environ::VMGcKind;/// `0x12345678` into a reference, pretend it is a valid `anyref`, and trick the/// host into dereferencing it and segfaulting or worse.///-/// Note that you can also use `Rooted<AnyRef>` and `ManuallyRooted<AnyRef>` as+/// Note that you can also use `Rooted<AnyRef>` and `OwnedRooted<AnyRef>` as/// a type parameter with [`Func::typed`][crate::Func::typed]- and/// [`Func::wrap`][crate::Func::wrap]-style APIs.///@@ -103,9 +103,9 @@ impl From<Rooted<EqRef>> for Rooted<AnyRef> {}}-impl From<ManuallyRooted<EqRef>> for ManuallyRooted<AnyRef> {+impl From<OwnedRooted<EqRef>> for OwnedRooted<AnyRef> {#[inline]- fn from(e: ManuallyRooted<EqRef>) -> Self {+ fn from(e: OwnedRooted<EqRef>) -> Self {e.to_anyref()}}@@ -117,9 +117,9 @@ impl From<Rooted<StructRef>> for Rooted<AnyRef> {}}-impl From<ManuallyRooted<StructRef>> for ManuallyRooted<AnyRef> {+impl From<OwnedRooted<StructRef>> for OwnedRooted<AnyRef> {#[inline]- fn from(s: ManuallyRooted<StructRef>) -> Self {+ fn from(s: OwnedRooted<StructRef>) -> Self {s.to_anyref()}}@@ -131,9 +131,9 @@ impl From<Rooted<ArrayRef>> for Rooted<AnyRef> {}}-impl From<ManuallyRooted<ArrayRef>> for ManuallyRooted<AnyRef> {+impl From<OwnedRooted<ArrayRef>> for OwnedRooted<AnyRef> {#[inline]- fn from(s: ManuallyRooted<ArrayRef>) -> Self {+ fn from(s: OwnedRooted<ArrayRef>) -> Self {s.to_anyref()}}@@ -754,7 +754,7 @@ unsafe impl WasmTy for Option<Rooted<AnyRef>> {}}-unsafe impl WasmTy for ManuallyRooted<AnyRef> {+unsafe impl WasmTy for OwnedRooted<AnyRef> {#[inline]fn valtype() -> ValType {ValType::Ref(RefType::new(false, HeapType::Any))@@ -784,7 +784,7 @@ unsafe impl WasmTy for ManuallyRooted<AnyRef> {}}-unsafe impl WasmTy for Option<ManuallyRooted<AnyRef>> {+unsafe impl WasmTy for Option<OwnedRooted<AnyRef>> {#[inline]fn valtype() -> ValType {ValType::ANYREF@@ -821,11 +821,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<AnyRef>> {}fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {- <ManuallyRooted<AnyRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)+ <OwnedRooted<AnyRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)}unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {- <ManuallyRooted<AnyRef>>::wasm_ty_option_load(+ <OwnedRooted<AnyRef>>::wasm_ty_option_load(store,ptr.get_anyref(),AnyRef::from_cloned_gc_ref,diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rsindex f8d3a0ee7a..78db88eaef 100644--- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs@@ -6,7 +6,7 @@ use crate::vm::{self, VMArrayRef, VMGcHeader};use crate::{AnyRef, FieldType};use crate::{ArrayType, AsContext, AsContextMut, EqRef, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType,- ManuallyRooted, RefType, Rooted, Val, ValRaw, ValType, WasmTy,+ OwnedRooted, RefType, Rooted, Val, ValRaw, ValType, WasmTy,prelude::*,store::{AutoAssertNoGc, StoreContextMut, StoreOpaque},};@@ -108,7 +108,7 @@ impl ArrayRefPre {/// `0x12345678` into a reference, pretend it is a valid `arrayref`, and trick/// the host into dereferencing it and segfaulting or worse.///-/// Note that you can also use `Rooted<ArrayRef>` and `ManuallyRooted<ArrayRef>`+/// Note that you can also use `Rooted<ArrayRef>` and `OwnedRooted<ArrayRef>`/// as a type parameter with [`Func::typed`][crate::Func::typed]- and/// [`Func::wrap`][crate::Func::wrap]-style APIs.///@@ -213,16 +213,16 @@ impl Rooted<ArrayRef> {}}-impl ManuallyRooted<ArrayRef> {+impl OwnedRooted<ArrayRef> {/// Upcast this `arrayref` into an `anyref`.#[inline]- pub fn to_anyref(self) -> ManuallyRooted<AnyRef> {+ pub fn to_anyref(self) -> OwnedRooted<AnyRef> {self.unchecked_cast()}/// Upcast this `arrayref` into an `eqref`.#[inline]- pub fn to_eqref(self) -> ManuallyRooted<EqRef> {+ pub fn to_eqref(self) -> OwnedRooted<EqRef> {self.unchecked_cast()}}@@ -923,7 +923,7 @@ unsafe impl WasmTy for Option<Rooted<ArrayRef>> {}}-unsafe impl WasmTy for ManuallyRooted<ArrayRef> {+unsafe impl WasmTy for OwnedRooted<ArrayRef> {#[inline]fn valtype() -> ValType {ValType::Ref(RefType::new(false, HeapType::Array))@@ -975,7 +975,7 @@ unsafe impl WasmTy for ManuallyRooted<ArrayRef> {}}-unsafe impl WasmTy for Option<ManuallyRooted<ArrayRef>> {+unsafe impl WasmTy for Option<OwnedRooted<ArrayRef>> {#[inline]fn valtype() -> ValType {ValType::ARRAYREF@@ -995,9 +995,7 @@ unsafe impl WasmTy for Option<ManuallyRooted<ArrayRef>> {ty: &HeapType,) -> Result<()> {match self {- Some(s) => {- ManuallyRooted::<ArrayRef>::dynamic_concrete_type_check(s, store, nullable, ty)- }+ Some(s) => OwnedRooted::<ArrayRef>::dynamic_concrete_type_check(s, store, nullable, ty),None => {ensure!(nullable,@@ -1014,11 +1012,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<ArrayRef>> {}fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {- <ManuallyRooted<ArrayRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)+ <OwnedRooted<ArrayRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)}unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {- <ManuallyRooted<ArrayRef>>::wasm_ty_option_load(+ <OwnedRooted<ArrayRef>>::wasm_ty_option_load(store,ptr.get_anyref(),ArrayRef::from_cloned_gc_ref,diff --git a/crates/wasmtime/src/runtime/gc/enabled/eqref.rs b/crates/wasmtime/src/runtime/gc/enabled/eqref.rsindex 7b6d27a16b..de88a26536 100644--- a/crates/wasmtime/src/runtime/gc/enabled/eqref.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/eqref.rs@@ -1,7 +1,7 @@//! Working with GC `eqref`s.use crate::{- AnyRef, ArrayRef, ArrayType, AsContext, GcRefImpl, GcRootIndex, HeapType, I31, ManuallyRooted,+ AnyRef, ArrayRef, ArrayType, AsContext, GcRefImpl, GcRootIndex, HeapType, I31, OwnedRooted,RefType, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy,prelude::*,runtime::vm::VMGcRef,@@ -25,7 +25,7 @@ use wasmtime_environ::VMGcKind;/// `0x12345678` into a reference, pretend it is a valid `eqref`, and trick the/// host into dereferencing it and segfaulting or worse.///-/// Note that you can also use `Rooted<EqRef>` and `ManuallyRooted<EqRef>` as a+/// Note that you can also use `Rooted<EqRef>` and `OwnedRooted<EqRef>` as a/// type parameter with [`Func::typed`][crate::Func::typed]- and/// [`Func::wrap`][crate::Func::wrap]-style APIs.///@@ -99,9 +99,9 @@ impl From<Rooted<StructRef>> for Rooted<EqRef> {}}-impl From<ManuallyRooted<StructRef>> for ManuallyRooted<EqRef> {+impl From<OwnedRooted<StructRef>> for OwnedRooted<EqRef> {#[inline]- fn from(s: ManuallyRooted<StructRef>) -> Self {+ fn from(s: OwnedRooted<StructRef>) -> Self {s.to_eqref()}}@@ -113,9 +113,9 @@ impl From<Rooted<ArrayRef>> for Rooted<EqRef> {}}-impl From<ManuallyRooted<ArrayRef>> for ManuallyRooted<EqRef> {+impl From<OwnedRooted<ArrayRef>> for OwnedRooted<EqRef> {#[inline]- fn from(s: ManuallyRooted<ArrayRef>) -> Self {+ fn from(s: OwnedRooted<ArrayRef>) -> Self {s.to_eqref()}}@@ -145,10 +145,10 @@ impl Rooted<EqRef> {}}-impl ManuallyRooted<EqRef> {+impl OwnedRooted<EqRef> {/// Upcast this `eqref` into an `anyref`.#[inline]- pub fn to_anyref(self) -> ManuallyRooted<AnyRef> {+ pub fn to_anyref(self) -> OwnedRooted<AnyRef> {self.unchecked_cast()}}@@ -516,7 +516,7 @@ unsafe impl WasmTy for Option<Rooted<EqRef>> {}}-unsafe impl WasmTy for ManuallyRooted<EqRef> {+unsafe impl WasmTy for OwnedRooted<EqRef> {#[inline]fn valtype() -> ValType {ValType::Ref(RefType::new(false, HeapType::Eq))@@ -546,7 +546,7 @@ unsafe impl WasmTy for ManuallyRooted<EqRef> {}}-unsafe impl WasmTy for Option<ManuallyRooted<EqRef>> {+unsafe impl WasmTy for Option<OwnedRooted<EqRef>> {#[inline]fn valtype() -> ValType {ValType::EQREF@@ -566,7 +566,7 @@ unsafe impl WasmTy for Option<ManuallyRooted<EqRef>> {ty: &HeapType,) -> Result<()> {match self {- Some(s) => ManuallyRooted::<EqRef>::dynamic_concrete_type_check(s, store, nullable, ty),+ Some(s) => OwnedRooted::<EqRef>::dynamic_concrete_type_check(s, store, nullable, ty),None => {ensure!(nullable,@@ -583,11 +583,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<EqRef>> {}fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {- <ManuallyRooted<EqRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)+ <OwnedRooted<EqRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)}unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {- <ManuallyRooted<EqRef>>::wasm_ty_option_load(+ <OwnedRooted<EqRef>>::wasm_ty_option_load(store,ptr.get_anyref(),EqRef::from_cloned_gc_ref,diff --git a/crates/wasmtime/src/runtime/gc/enabled/exnref.rs b/crates/wasmtime/src/runtime/gc/enabled/exnref.rsindex 03ba35ad3b..5ada1f06d2 100644--- a/crates/wasmtime/src/runtime/gc/enabled/exnref.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/exnref.rs@@ -4,7 +4,7 @@ use crate::runtime::vm::{VMGcRef, VMStore};use crate::store::{StoreId, StoreResourceLimiter};use crate::vm::{self, VMExnRef, VMGcHeader};use crate::{- AsContext, AsContextMut, GcRefImpl, GcRootIndex, HeapType, ManuallyRooted, RefType, Result,+ AsContext, AsContextMut, GcRefImpl, GcRootIndex, HeapType, OwnedRooted, RefType, Result,Rooted, Val, ValRaw, ValType, WasmTy,store::{AutoAssertNoGc, StoreOpaque},};@@ -101,7 +101,7 @@ impl ExnRefPre {/// thrown exception in WebAssembly with a `catch_ref` clause of a/// `try_table`, or by allocating via the host API.///-/// Note that you can also use `Rooted<ExnRef>` and `ManuallyRooted<ExnRef>` as+/// Note that you can also use `Rooted<ExnRef>` and `OwnedRooted<ExnRef>` as/// a type parameter with [`Func::typed`][crate::Func::typed]- and/// [`Func::wrap`][crate::Func::wrap]-style APIs.#[derive(Debug)]@@ -695,7 +695,7 @@ unsafe impl WasmTy for Option<Rooted<ExnRef>> {}}-unsafe impl WasmTy for ManuallyRooted<ExnRef> {+unsafe impl WasmTy for OwnedRooted<ExnRef> {#[inline]fn valtype() -> ValType {ValType::Ref(RefType::new(false, HeapType::Exn))@@ -725,7 +725,7 @@ unsafe impl WasmTy for ManuallyRooted<ExnRef> {}}-unsafe impl WasmTy for Option<ManuallyRooted<ExnRef>> {+unsafe impl WasmTy for Option<OwnedRooted<ExnRef>> {#[inline]fn valtype() -> ValType {ValType::EXNREF@@ -762,11 +762,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<ExnRef>> {}fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {- <ManuallyRooted<ExnRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)+ <OwnedRooted<ExnRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)}unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {- <ManuallyRooted<ExnRef>>::wasm_ty_option_load(+ <OwnedRooted<ExnRef>>::wasm_ty_option_load(store,ptr.get_anyref(),ExnRef::from_cloned_gc_ref,diff --git a/crates/wasmtime/src/runtime/gc/enabled/externref.rs b/crates/wasmtime/src/runtime/gc/enabled/externref.rsindex 8564843b84..1d2195de48 100644--- a/crates/wasmtime/src/runtime/gc/enabled/externref.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/externref.rs@@ -4,7 +4,7 @@ use super::{AnyRef, RootedGcRefImpl};use crate::prelude::*;use crate::runtime::vm::{self, VMGcRef, VMStore};use crate::{- AsContextMut, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType, ManuallyRooted, RefType,+ AsContextMut, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType, OwnedRooted, RefType,Result, Rooted, StoreContext, StoreContextMut, ValRaw, ValType, WasmTy,store::{AutoAssertNoGc, StoreOpaque, StoreResourceLimiter},};@@ -31,7 +31,7 @@ use core::mem::MaybeUninit;/// the host into dereferencing it and segfaulting or worse.////// Note that you can also use `Rooted<ExternRef>` and-/// `ManuallyRooted<ExternRef>` as a type parameter with+/// `OwnedRooted<ExternRef>` as a type parameter with/// [`Func::typed`][crate::Func::typed]- and/// [`Func::wrap`][crate::Func::wrap]-style APIs.///@@ -140,10 +140,10 @@ impl ExternRef {////// The resulting value is automatically unrooted when the given `context`'s/// scope is exited. If you need to hold the reference past the `context`'s- /// scope, convert the result into a- /// [`ManuallyRooted<T>`][crate::ManuallyRooted]. See the documentation for+ /// scope, convert the result into an+ /// [`OwnedRooted<T>`][crate::OwnedRooted]. See the documentation for/// [`Rooted<T>`][crate::Rooted] and- /// [`ManuallyRooted<T>`][crate::ManuallyRooted] for more details.+ /// [`OwnedRooted<T>`][crate::OwnedRooted] for more details.////// # Automatic Garbage Collection///@@ -223,10 +223,10 @@ impl ExternRef {////// The resulting value is automatically unrooted when the given `context`'s/// scope is exited. If you need to hold the reference past the `context`'s- /// scope, convert the result into a- /// [`ManuallyRooted<T>`][crate::ManuallyRooted]. See the documentation for+ /// scope, convert the result into an+ /// [`OwnedRooted<T>`][crate::OwnedRooted]. See the documentation for/// [`Rooted<T>`][crate::Rooted] and- /// [`ManuallyRooted<T>`][crate::ManuallyRooted] for more details.+ /// [`OwnedRooted<T>`][crate::OwnedRooted] for more details.////// # Automatic Garbage Collection///@@ -646,7 +646,7 @@ unsafe impl WasmTy for Option<Rooted<ExternRef>> {}}-unsafe impl WasmTy for ManuallyRooted<ExternRef> {+unsafe impl WasmTy for OwnedRooted<ExternRef> {#[inline]fn valtype() -> ValType {ValType::Ref(RefType::new(false, HeapType::Extern))@@ -676,7 +676,7 @@ unsafe impl WasmTy for ManuallyRooted<ExternRef> {}}-unsafe impl WasmTy for Option<ManuallyRooted<ExternRef>> {+unsafe impl WasmTy for Option<OwnedRooted<ExternRef>> {#[inline]fn valtype() -> ValType {ValType::EXTERNREF@@ -699,11 +699,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<ExternRef>> {}fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {- <ManuallyRooted<ExternRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::externref)+ <OwnedRooted<ExternRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::externref)}unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {- <ManuallyRooted<ExternRef>>::wasm_ty_option_load(+ <OwnedRooted<ExternRef>>::wasm_ty_option_load(store,ptr.get_externref(),ExternRef::from_cloned_gc_ref,diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rsindex 8ea6d49796..7e06c85d4d 100644--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs@@ -27,24 +27,70 @@//! not a burden for users. Additionally, the API's types should be `Sync`//! and `Send` so that they work well with async Rust.//!-//! For example, goals (3) and (4) are in conflict when we think about how to-//! support (2). Ideally, for ergonomics, a root would automatically unroot-//! itself when dropped. But in the general case that requires holding a-//! reference to the store's root set, and that root set needs to be held-//! simultaneously by all GC roots, and they each need to mutate the set to-//! unroot themselves. That implies `Rc<RefCell<...>>` or `Arc<Mutex<...>>`! The-//! former makes the store and GC root types not `Send` and not `Sync`. The-//! latter imposes synchronization and locking overhead. So we instead make GC-//! roots indirect and require passing in a store context explicitly to unroot-//! in the general case. This trades worse ergonomics for better performance and-//! support for moving GC.+//! The two main design axes that trade off the above goals are:+//!+//! - Where the GC reference itself is held. A root object could+//! directly hold the underlying GC reference (offset into the GC+//! storage area), which would allow more efficient dereferencing+//! and access to the referred-to GC object. However, goal (2)+//! requires that the GC is able to update references when objects+//! move during a GC. Thus, such "direct roots" would need to be+//! registered somehow in a global root registry, and would need to+//! unregister themselves when dropped.+//!+//! The alternative is to hold some indirect kind of reference to a+//! GC reference, with the latter stored directly in the `Store` so+//! the GC can update it freely. This adds one pointer-chasing hop+//! to accesses, but works much more nicely with ownership+//! semantics. Logically, the `Store` "owns" the actual pointers;+//! and rooting types own the slots that they are stored in.+//!+//! For the above reasons, all of our rooting types below use+//! indirection. This avoids the need for an unsafe+//! intrusive-linked-list for global registration, or a shared+//! reference to a mutex-protected registry, or some other+//! error-prone technique.+//!+//! - How unrooting occurs. Ideally, a rooting type would implement+//! the Rust `Drop` trait and unroot itself when the Rust value is+//! dropped. However, because the rooting state is held in the+//! `Store`, this direct approach would imply keeping a shared,+//! mutex-protected handle to the registry in every rooting+//! object. This would add synchronization overhead to the common+//! case, and in general would be a bad tradeoff.+//!+//! However, there are several other approaches:+//!+//! - The user could use an RAII wrapper that *does* own the `Store`,+//! and defines a "scope" in which roots are created and then+//! bulk-unrooted at the close of the scope.+//! - The rooting type could hold a shared reference to some state+//! *other* than the full registry, and update a flag in that+//! state indicating it has been dropped; the `Store` could then+//! later observe that flag and remove the root. This would have+//! some allocation cost, but the shared state would be+//! independent of the `Store` and specific to each root, so would+//! impose no synchronization overhead between different roots or+//! the GC itself.+//! - The rooting type could provide a fully manual `unroot` method,+//! allowing the user to make use of their own knowledge of their+//! application's lifetimes and semantics and remove roots when+//! appropriate.+//!+//! We provide an implementation of the first two of these+//! strategies below in `Rooted` and `OwnedRooted`. The last, fully+//! manual, approach is too difficult to use correctly (it cannot+//! implement Rust's `Drop` trait, but there is no way in Rust to+//! enforce that a value must be consumed rather than dropped) so it+//! is not implemented.//!//! ## Two Flavors of Rooting API//!-//! Okay, with that out of the way, this module provides two flavors of rooting-//! API. One for the common, scoped lifetime case, and another for the rare case-//! where we really need a GC root with an arbitrary, non-LIFO/non-scoped-//! lifetime:+//! Okay, with that out of the way, this module provides two flavors+//! of rooting API. One for the common, scoped lifetime case, and one that+//! carries ownership until dropped, and can work as an RAII handle+//! that interacts well with Rust ownership semantics (but at a minor+//! performance cost)://!//! 1. `RootScope` and `Rooted<T>`: These are used for temporarily rooting GC//! objects for the duration of a scope. The internal implementation takes@@ -64,27 +110,40 @@//! This supports the common use case for rooting and provides good//! ergonomics.//!-//! 2. `ManuallyRooted<T>`: This is the fully general rooting API used for-//! holding onto non-LIFO GC roots with arbitrary lifetimes. However, users-//! must manually unroot them. Failure to manually unroot a-//! `ManuallyRooted<T>` before it is dropped will result in the GC object-//! (and everything it transitively references) leaking for the duration of-//! the `Store`'s lifetime.+//! 2. `OwnedRooted<T>`: This is a root that manages rooting and+//! unrooting automatically with its lifetime as a Rust value. In+//! other words, the continued existence of the Rust value ensures+//! the rooting of the underlying GC reference; and when the Rust+//! value is dropped, the underlying GC reference is no longer+//! rooted.+//!+//! Internally, this root holds a shared reference to a+//! *root-specific* bit of state that is also tracked and observed+//! by the `Store`. This means that there is minor memory+//! allocation overhead (an `Arc<()>`) for each such root; this+//! memory is shared over all clones of this root. The rooted GC+//! reference is *logically* unrooted as soon as the last clone of+//! this root is dropped. Internally the root may still exist until+//! the next GC, or "root trim" when another `OwnedRooted` is+//! created, but that is unobservable externally, and will not+//! result in any additional GC object lifetime because it is+//! always cleaned up before a gc.//!//! This type is roughly similar to SpiderMonkey's [`PersistentRooted<T>`],-//! although they avoid the manual-unrooting with internal mutation and-//! shared references. (Our constraints mean we can't do those things, as-//! mentioned explained above.)+//! although they register roots on a per-thread `JSContext`, avoiding+//! mutation costs in a way that is not viable for Wasmtime (which needs+//! `Store`s to be `Send`).//!//! [`PersistentRooted<T>`]: http://devdoc.net/web/developer.mozilla.org/en-US/docs/Mozilla/Projects/SpiderMonkey/JSAPI_reference/JS::PersistentRooted.html//!-//! At the end of the day, both `Rooted<T>` and `ManuallyRooted<T>` are just-//! tagged indices into the store's `RootSet`. This indirection allows working-//! with Rust's borrowing discipline (we use `&mut Store` to represent mutable-//! access to the GC heap) while still allowing rooted references to be moved-//! around without tying up the whole store in borrows. Additionally, and-//! crucially, this indirection allows us to update the *actual* GC pointers in-//! the `RootSet` and support moving GCs (again, as mentioned above).+//! At the end of the day, all of the above root types are just tagged+//! indices into the store's `RootSet`. This indirection allows+//! working with Rust's borrowing discipline (we use `&mut Store` to+//! represent mutable access to the GC heap) while still allowing+//! rooted references to be moved around without tying up the whole+//! store in borrows. Additionally, and crucially, this indirection+//! allows us to update the *actual* GC pointers in the `RootSet` and+//! support moving GCs (again, as mentioned above).//!//! ## Unrooted References//!@@ -108,10 +167,11 @@ use crate::{store::{AsStoreOpaque, AutoAssertNoGc, StoreId, StoreOpaque},};use crate::{ValRaw, prelude::*};+use alloc::sync::{Arc, Weak};use core::any;use core::marker;use core::mem::{self, MaybeUninit};-use core::num::NonZeroU64;+use core::num::{NonZeroU64, NonZeroUsize};use core::{fmt::{self, Debug},hash::{Hash, Hasher},@@ -182,7 +242,7 @@ pub(crate) use sealed::*;/// The index of a GC root inside a particular store's GC root set.///-/// Can be either a LIFO- or manually-rooted object, depending on the+/// Can be either a LIFO- or owned-rooted object, depending on the/// `PackedIndex`.////// Every `T` such that `T: GcRef` must be a newtype over this `GcRootIndex`.@@ -237,8 +297,8 @@ impl GcRootIndex {} else {None}- } else if let Some(id) = self.index.as_manual() {- let gc_ref = store.gc_roots().manually_rooted.get(id);+ } else if let Some(id) = self.index.as_owned() {+ let gc_ref = store.gc_roots().owned_rooted.get(id);debug_assert!(gc_ref.is_some());gc_ref} else {@@ -271,7 +331,7 @@ impl GcRootIndex {/// ```ignore/// enum {/// Lifo(usize),-/// Manual(SlabId),+/// Owned(SlabId),/// }/// ```///@@ -285,8 +345,8 @@ impl Debug for PackedIndex {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {if let Some(index) = self.as_lifo() {f.debug_tuple("PackedIndex::Lifo").field(&index).finish()- } else if let Some(id) = self.as_manual() {- f.debug_tuple("PackedIndex::Manual").field(&id).finish()+ } else if let Some(id) = self.as_owned() {+ f.debug_tuple("PackedIndex::Owned").field(&id).finish()} else {unreachable!()}@@ -296,7 +356,7 @@ impl Debug for PackedIndex {impl PackedIndex {const DISCRIMINANT_MASK: u32 = 0b1 << 31;const LIFO_DISCRIMINANT: u32 = 0b0 << 31;- const MANUAL_DISCRIMINANT: u32 = 0b1 << 31;+ const OWNED_DISCRIMINANT: u32 = 0b1 << 31;const PAYLOAD_MASK: u32 = !Self::DISCRIMINANT_MASK;fn new_lifo(index: usize) -> PackedIndex {@@ -305,17 +365,17 @@ impl PackedIndex {let packed = PackedIndex(Self::LIFO_DISCRIMINANT | index32);debug_assert!(packed.is_lifo());debug_assert_eq!(packed.as_lifo(), Some(index));- debug_assert!(!packed.is_manual());- debug_assert!(packed.as_manual().is_none());+ debug_assert!(!packed.is_owned());+ debug_assert!(packed.as_owned().is_none());packed}- fn new_manual(id: SlabId) -> PackedIndex {+ fn new_owned(id: SlabId) -> PackedIndex {let raw = id.into_raw();assert_eq!(raw & Self::DISCRIMINANT_MASK, 0);- let packed = PackedIndex(Self::MANUAL_DISCRIMINANT | raw);- debug_assert!(packed.is_manual());- debug_assert_eq!(packed.as_manual(), Some(id));+ let packed = PackedIndex(Self::OWNED_DISCRIMINANT | raw);+ debug_assert!(packed.is_owned());+ debug_assert_eq!(packed.as_owned(), Some(id));debug_assert!(!packed.is_lifo());debug_assert!(packed.as_lifo().is_none());packed@@ -329,8 +389,8 @@ impl PackedIndex {self.discriminant() == Self::LIFO_DISCRIMINANT}- fn is_manual(&self) -> bool {- self.discriminant() == Self::MANUAL_DISCRIMINANT+ fn is_owned(&self) -> bool {+ self.discriminant() == Self::OWNED_DISCRIMINANT}fn payload(&self) -> u32 {@@ -345,8 +405,8 @@ impl PackedIndex {}}- fn as_manual(&self) -> Option<SlabId> {- if self.is_manual() {+ fn as_owned(&self) -> Option<SlabId> {+ if self.is_owned() {Some(SlabId::from_raw(self.payload()))} else {None@@ -357,9 +417,19 @@ impl PackedIndex {/// The set of all embedder-API GC roots in a single store/heap.#[derive(Debug, Default)]pub(crate) struct RootSet {- /// GC roots with arbitrary lifetime that are manually rooted and unrooted,- /// for use with `ManuallyRooted<T>`.- manually_rooted: Slab<VMGcRef>,+ /// GC roots with arbitrary lifetime that are unrooted when+ /// liveness flags are cleared (seen during a trimming pass), for+ /// use with `OwnedRooted<T>`.+ owned_rooted: Slab<VMGcRef>,++ /// List of liveness flags and corresponding `SlabId`s into the+ /// `owned_rooted` slab.+ liveness_flags: Vec<(Weak<()>, SlabId)>,++ /// High-water mark for liveness flag trimming. We use this to+ /// ensure we have amortized constant-time behavior on adding+ /// roots. See note below on `trim_liveness_flags()`.+ liveness_trim_high_water: Option<NonZeroUsize>,/// Strictly LIFO-ordered GC roots, for use with `RootScope` and/// `Rooted<T>`.@@ -386,13 +456,13 @@ impl RootSet {}log::trace!("End trace user LIFO roots");- log::trace!("Begin trace user manual roots");- for (_id, root) in self.manually_rooted.iter_mut() {+ log::trace!("Begin trace user owned roots");+ for (_id, root) in self.owned_rooted.iter_mut() {unsafe {- gc_roots_list.add_root(root.into(), "user manual root");+ gc_roots_list.add_root(root.into(), "user owned root");}}- log::trace!("End trace user manual roots");+ log::trace!("End trace user owned roots");}/// Enter a LIFO rooting scope.@@ -466,6 +536,181 @@ impl RootSet {index,}}++ /// Trim any stale (dropped) owned roots.+ ///+ /// `OwnedRooted` is implemented in a way that avoids the need to+ /// have or keep a reference to the store (and thus this struct)+ /// during its drop operation: to allow it to be independent, it+ /// holds a shared reference to some other memory and sets that+ /// "liveness flag" appropriately, then we later observe dead+ /// liveness flags during a periodic scan and actually deallocate+ /// the roots. We use an `Arc<()>` for this: it permits cheap+ /// cloning, and it has minimal memory overhead. We hold a weak+ /// ref in a list alongside the actual `GcRootIndex`, and we free+ /// that index in the slab of owned roots when we observe that+ /// only our weak ref remains.+ ///+ /// This trim step is logically separate from a full GC, though it+ /// would not be very productive to do a GC without doing a+ /// root-trim first: the root-trim should be quite a lot cheaper,+ /// and it will allow for more garbage to exist.+ ///+ /// There is, additionally, nothing stopping us from doing trims+ /// more often than just before each GC, and there are reasons+ /// this could be a good idea: for example, a user program that+ /// creates and removes many roots (perhaps as it accesses the GC+ /// object graph) but ultimately is dealing with a static graph,+ /// without further allocation, may need the "root set" to be GC'd+ /// independently from the actual heap. Thus, we could trim before+ /// adding a new root to ensure we don't grow that unboundedly (or+ /// force an otherwise unneeded GC).+ ///+ /// The first case, just before GC, wants a "full trim": there's+ /// no reason not to unroot as much as possible before we do the+ /// expensive work of tracing the whole heap.+ ///+ /// On the other hand, the second case, adding a new root, wants a+ /// kind of trim that is amortized constant time. Consider: if we+ /// have some threshold for the trim, say N roots, and the user+ /// program continually adds and removes one root such that it+ /// goes just over the threshold, we might scan all N liveness+ /// flags for each step, resulting in quadratic behavior overall.+ ///+ /// Thus, we implement a "high water mark" algorithm to guard+ /// against this latter case: on the add-a-new-root case, we trim+ /// only if the list is longer than the high water mark, and we+ /// set the high water mark each time based on the after-trim+ /// size. See below for details on this algorithm.+ ///+ /// `eager` chooses whether we eagerly trim roots or pre-filter+ /// using the high-water mark.+ ///+ /// # Growth Algorithm+ ///+ /// We want to balance two factors: we must ensure that the+ /// algorithmic complexity of creating a new root is amortized+ /// O(1), and we must ensure that repeated creation and deletion+ /// of roots without any GC must result in a root-set that has a+ /// size linear in the actual live-root-set size. Stated formally:+ ///+ /// 1. Root creation must be O(1), amortized+ /// 2. liveness_flags.len() must be O(|max live owned roots|),+ /// i.e., must not grow to larger than a constant multiple of+ /// the maximum working-root-set size of the program.+ ///+ /// Note that a naive exponential-growth-of-threshold algorithm,+ /// where we trim when the root set reaches 1, 2, 4, 8, 16, 32,+ /// ... elements, provides the first but *not* the second+ /// property. A workload that has a constant live root set but+ /// creates and drops roots constantly (say, as it's traversing a+ /// static graph and moving "fingers" through it) will cause the+ /// `liveness_flags` array to grow unboundedly.+ ///+ /// Instead, it turns out that we can achieve both of these goals+ /// with a simple rule: we trim when the root list length reaches+ /// a high-water mark; and then *after* trimming, we set the+ /// high-water mark equal to the resulting live-root count+ /// multiplied by a factor (e.g., 2).+ ///+ /// ## Proof+ ///+ /// - Root creation is O(1)+ ///+ /// Assume a sequence of root creation and drop events (and no+ /// GCs, with a static GC graph, in the worst case -- only roots+ /// are changing). We want to show that after N root creations,+ /// we have incurred only only O(N) cost scanning the+ /// `liveness_flags` list over the whole sequence.+ ///+ /// Assume a default high-water mark of D (e.g., 8) at+ /// initialization with an empty root list.+ ///+ /// Consider "epochs" in the sequence split by trim events where+ /// we scan the root list. Proceed by induction over epochs to+ /// show: after each epoch, we will have scanned at most 2N+ /// roots after N root creations.+ ///+ /// (These epochs don't exist in the algorithm: this is just a+ /// mechanism to analyze the behavior.)+ ///+ /// Base case: after the first epoch, with D root creations, we+ /// will scan D roots.+ ///+ /// Induction step: we have created N roots and scanned at most+ /// 2N roots. After previous scan, L roots are still live; then+ /// we set the high-water mark for next scan at 2L. The list+ /// already has L, so after another L root creations, the epoch+ /// ends. We will then incur a scan cost of 2L (the full+ /// list). At that point we have thus seen N + L root creations,+ /// with 2N + 2L scan cost; the invariant holds.+ ///+ /// (It's counter-intuitive that *not* raising the high-water+ /// mark exponentially can still result in a constant amortized+ /// cost! One intuition to understand this is that each root+ /// that remains alive after a scan pushes the next high-water+ /// mark up by one, so requires a new root creation to "pay for"+ /// its next scan. So any given root may be scanned many times,+ /// but each such root ensures other root creations happen to+ /// maintain the amortized cost.)+ ///+ /// - `liveness_flags.len()` is always O(|max live roots|)+ ///+ /// Before the first trim, we have between 0 and D live roots,+ /// which is O(1) (`D` is a compile-time constant).+ ///+ /// Just after a trim, the `liveness_flags` list has only live+ /// roots, and the max live-root count is at least the count at+ /// this time, so the property holds.+ ///+ /// The instantaneous maximum number of live roots is greater+ /// than or equal to the maximum number of live roots observed+ /// during a trim. (The trim is just some point in time, and the+ /// max at some point in time is at most the overall max.)+ ///+ /// The high-water mark is set at 2 * `liveness_flags.len()`+ /// after a trim, i.e., the number of live roots at that+ /// time. We trim when we reach the high-water mark. So the+ /// length of the array cannot exceed 2 *+ /// `liveness_flags.len()`, which is less than or equal to the+ /// overall max. So transitively, the list length at any time is+ /// always O(|max live roots|).+ ///+ /// We thus have tight bounds (deterministic, not randomized) for+ /// all possible sequences of root creation/dropping, ensuring+ /// robustness.+ pub(crate) fn trim_liveness_flags(&mut self, gc_store: &mut GcStore, eager: bool) {+ const DEFAULT_HIGH_WATER: usize = 8;+ const GROWTH_FACTOR: usize = 2;+ let high_water_mark = self+ .liveness_trim_high_water+ .map(|x| x.get())+ .unwrap_or(DEFAULT_HIGH_WATER);+ if !eager && self.liveness_flags.len() < high_water_mark {+ return;+ }++ self.liveness_flags.retain(|(flag, index)| {+ if flag.strong_count() == 0 {+ // No more `OwnedRooted` instances are holding onto+ // this; dealloc the index and drop our Weak.+ let gc_ref = self.owned_rooted.dealloc(*index);+ gc_store.drop_gc_ref(gc_ref);+ // Don't retain in the list.+ false+ } else {+ // Retain in the list.+ true+ }+ });++ let post_trim_len = self.liveness_flags.len();+ let high_water_mark = core::cmp::max(+ DEFAULT_HIGH_WATER,+ post_trim_len.saturating_mul(GROWTH_FACTOR),+ );+ self.liveness_trim_high_water = Some(NonZeroUsize::new(high_water_mark).unwrap());+ }}/// A scoped, rooted reference to a garbage-collected `T`.@@ -552,21 +797,25 @@ impl RootSet {/// # }/// ```///-/// # Differences Between `Rooted<T>` and `ManuallyRooted<T>`+/// # Differences Between `Rooted<T>` and `OwnedRooted<T>`///-/// While `Rooted<T>` is automatically unrooted when its scope is exited, this-/// means that `Rooted<T>` is only valid for strictly last-in-first-out (LIFO,-/// aka stack order) lifetimes. This is in contrast to-/// [`ManuallyRooted<T>`][crate::ManuallyRooted], which supports rooting GC-/// objects for arbitrary lifetimes, but requires manual unrooting.+/// While `Rooted<T>` is automatically unrooted when its scope is+/// exited, this means that `Rooted<T>` is only valid for strictly+/// last-in-first-out (LIFO, aka stack order) lifetimes. This is in+/// contrast to [`OwnedRooted<T>`][crate::OwnedRooted], which supports+/// rooting GC objects for arbitrary lifetimes.///-/// | Type | Supported Lifetimes | Unrooting |-/// |----------------------------------------------|-----------------------------|-----------|-/// | [`Rooted<T>`][crate::Rooted] | Strictly LIFO / stack order | Automatic |-/// | [`ManuallyRooted<T>`][crate::ManuallyRooted] | Arbitrary | Manual |+/// | Type | Supported Lifetimes | Unrooting | Cost |+/// |----------------------------------------------|-----------------------------|-----------|----------------------------------|+/// | [`Rooted<T>`][crate::Rooted] | Strictly LIFO / stack order | Automatic | very low (LIFO array) |+/// | [`OwnedRooted<T>`][crate::OwnedRooted] | Arbitrary | Automatic | medium (separate `Arc` to track) |///-/// `Rooted<T>` should suffice for most use cases, and provides better-/// ergonomics, but `ManuallyRooted<T>` exists as a fully-general escape hatch.+/// `Rooted<T>` should suffice for most use cases, and provides decent+/// ergonomics. In cases where LIFO scopes are difficult to reason+/// about, e.g. heap-managed data structures, or when they may cause+/// erroneous behavior, e.g. in errors that are propagated up the call+/// stack, `OwnedRooted<T>` provides very safe ergonomics but at a+/// small dynamic cost for the separate tracking allocation.////// # Scopes///@@ -574,8 +823,7 @@ impl RootSet {////// 1. A [`Store`][crate::Store] is the outermost rooting scope. Creating a/// `Root<T>` directly inside a `Store` permanently roots the underlying-/// object, similar to dropping a-/// [`ManuallyRooted<T>`][crate::ManuallyRooted] without unrooting it.+/// object.////// 2. A [`Caller`][crate::Caller] provides a rooting scope for the duration of/// a call from Wasm into a host function. Any objects rooted in a `Caller`@@ -752,7 +1000,7 @@ impl<T: GcRef> Rooted<T> {self.inner.comes_from_same_store(store)}- /// Create a [`ManuallyRooted<T>`][crate::ManuallyRooted] holding onto the+ /// Create an [`OwnedRooted<T>`][crate::OwnedRooted] holding onto the/// same GC object as `self`.////// Returns `None` if `self` is used outside of its scope and has therefore@@ -772,7 +1020,7 @@ impl<T: GcRef> Rooted<T> {/// # fn _foo() -> Result<()> {/// let mut store = Store::<()>::default();///- /// let y: ManuallyRooted<_> = {+ /// let y: OwnedRooted<_> = {/// // Create a nested rooting scope./// let mut scope = RootScope::new(&mut store);///@@ -780,8 +1028,8 @@ impl<T: GcRef> Rooted<T> {/// let x: Rooted<_> = ExternRef::new(&mut scope, "hello!")?;////// // Extend `x`'s rooting past its scope's lifetime by converting it- /// // to a `ManuallyRooted`.- /// x.to_manually_rooted(&mut scope)?+ /// // to an `OwnedRooted`.+ /// x.to_owned_rooted(&mut scope)?/// };////// // Now we can still access the reference outside the scope it was@@ -789,20 +1037,17 @@ impl<T: GcRef> Rooted<T> {/// let data = y.data(&store)?.expect("should have host data");/// let data = data.downcast_ref::<&str>().expect("host data should be str");/// assert_eq!(*data, "hello!");- ///- /// // But we have to manually unroot `y`.- /// y.unroot(&mut store);/// # Ok(())/// # }/// ```- pub fn to_manually_rooted(&self, mut store: impl AsContextMut) -> Result<ManuallyRooted<T>> {- self._to_manually_rooted(store.as_context_mut().0)+ pub fn to_owned_rooted(&self, mut store: impl AsContextMut) -> Result<OwnedRooted<T>> {+ self._to_owned_rooted(store.as_context_mut().0)}- pub(crate) fn _to_manually_rooted(&self, store: &mut StoreOpaque) -> Result<ManuallyRooted<T>> {+ pub(crate) fn _to_owned_rooted(&self, store: &mut StoreOpaque) -> Result<OwnedRooted<T>> {let mut store = AutoAssertNoGc::new(store);let gc_ref = self.try_clone_gc_ref(&mut store)?;- Ok(ManuallyRooted::new(&mut store, gc_ref))+ Ok(OwnedRooted::new(&mut store, gc_ref))}/// Are these two `Rooted<T>`s the same GC root?@@ -831,7 +1076,7 @@ impl<T: GcRef> Rooted<T> {////// // `c` is a different GC root, in a different scope, even though it/// // is rooting the same object.- /// let c = a.to_manually_rooted(&mut scope)?.into_rooted(&mut scope);+ /// let c = a.to_owned_rooted(&mut scope)?.to_rooted(&mut scope);/// assert!(!Rooted::rooted_eq(a, c));/// }///@@ -858,7 +1103,7 @@ impl<T: GcRef> Rooted<T> {/// because the scope it was rooted within has been exited.////// Because this method takes any `impl RootedGcRef<T>` arguments, it can be- /// used to compare, for example, a `Rooted<T>` and a `ManuallyRooted<T>`.+ /// used to compare, for example, a `Rooted<T>` and a `OwnedRooted<T>`.////// # Panics///@@ -882,7 +1127,7 @@ impl<T: GcRef> Rooted<T> {////// // `c` is a different GC root, in a different scope, but still/// // rooting the same object.- /// let c = a.to_manually_rooted(&mut scope)?.into_rooted(&mut scope);+ /// let c = a.to_owned_rooted(&mut scope)?.to_rooted(&mut scope);/// assert!(!Rooted::ref_eq(&scope, &a, &c)?);/// }///@@ -891,12 +1136,10 @@ impl<T: GcRef> Rooted<T> {/// // `a` and `x` are rooting different objects./// assert!(!Rooted::ref_eq(&store, &a, &x)?);///- /// // You can also compare `Rooted<T>`s and `ManuallyRooted<T>`s with this+ /// // You can also compare `Rooted<T>`s and `OwnedRooted<T>`s with this/// // function.- /// let d = a.to_manually_rooted(&mut store)?;+ /// let d = a.to_owned_rooted(&mut store)?;/// assert!(Rooted::ref_eq(&store, &a, &d)?);- ///- /// d.unroot(&mut store);/// # Ok(())/// # }/// ```@@ -1277,18 +1520,23 @@ where}}-/// A rooted reference to a garbage-collected `T` with arbitrary lifetime.+/// A rooted reference to a garbage-collected `T` with automatic lifetime.///-/// A `ManuallyRooted<T>` is a strong handle to a garbage-collected `T`,+/// An `OwnedRooted<T>` is a strong handle to a garbage-collected `T`,/// preventing its referent (and anything else transitively referenced) from-/// being collected by the GC until [`unroot`][crate::ManuallyRooted::unroot] is+/// being collected by the GC until [`unroot`][crate::OwnedRooted::unroot] is/// explicitly called.///-/// The primary way to create a `ManuallyRooted<T>` is to promote a temporary-/// `Rooted<T>` into a `ManuallyRooted<T>` via its-/// [`to_manually_rooted`][crate::Rooted::to_manually_rooted] method.+/// An `OwnedRooted<T>` keeps its rooted GC object alive at least+/// until the `OwnedRooted<T>` itself is dropped. The+/// "de-registration" of the root is automatic and is triggered (in a+/// deferred way) by the drop of this type.///-/// `ManuallyRooted<T>` dereferences to its underlying `T`, allowing you to call+/// The primary way to create an `OwnedRooted<T>` is to promote a temporary+/// `Rooted<T>` into an `OwnedRooted<T>` via its+/// [`to_owned_rooted`][crate::Rooted::to_owned_rooted] method.+///+/// `OwnedRooted<T>` dereferences to its underlying `T`, allowing you to call/// `T`'s methods.////// # Example@@ -1296,108 +1544,60 @@ where/// ```/// # use wasmtime::*;/// # fn _foo() -> Result<()> {-/// let mut store = Store::<Option<ManuallyRooted<ExternRef>>>::default();+/// let mut store = Store::<Option<OwnedRooted<ExternRef>>>::default();///-/// // Create our `ManuallyRooted` in a nested scope to avoid rooting it for+/// // Create our `OwnedRooted` in a nested scope to avoid rooting it for/// // the duration of the store's lifetime./// let x = {/// let mut scope = RootScope::new(&mut store);/// let x = ExternRef::new(&mut scope, 1234)?;-/// x.to_manually_rooted(&mut scope)?+/// x.to_owned_rooted(&mut scope)?/// };////// // Place `x` into our store./// *store.data_mut() = Some(x);////// // Do a bunch stuff that may or may not access, replace, or take `x`...-///-/// // At any time, in any arbitrary scope, we can remove `x` from the store-/// // and unroot it:-/// if let Some(x) = store.data_mut().take() {-/// x.unroot(&mut store);-/// }/// # Ok(())/// # }/// ```-///-/// # Differences Between `ManuallyRooted<T>` and `Rooted<T>`-///-/// While `ManuallyRooted<T>` can have arbitrary lifetimes, it requires manual-/// unrooting. This is in contrast to [`Rooted<T>`][crate::Rooted] which is-/// restricted to strictly last-in-first-out (LIFO, aka stack order) lifetimes,-/// but comes with automatic unrooting.-///-/// | Type | Supported Lifetimes | Unrooting |-/// |----------------------------------------------|-----------------------------|-----------|-/// | [`Rooted<T>`][crate::Rooted] | Strictly LIFO / stack order | Automatic |-/// | [`ManuallyRooted<T>`][crate::ManuallyRooted] | Arbitrary | Manual |-///-/// `Rooted<T>` should suffice for most use cases, and provides better-/// ergonomics, but `ManuallyRooted<T>` exists as a fully-general escape hatch.-///-/// # Manual Unrooting-///-/// Failure to explicitly call [`unroot`][crate::ManuallyRooted::unroot] (or-/// another method that consumes `self` and unroots the reference, such as-/// [`into_rooted`][crate::ManuallyRooted::into_rooted]) will leak the-/// underlying GC object, preventing it from being garbage collected until its-/// owning [`Store`][crate::Store] is dropped. That means all of the following-/// will result in permanently rooting the underlying GC object:-///-/// * Implicitly dropping a `ManuallyRooted<T>`:-///-/// ```no_run-/// # use wasmtime::*;-/// # let get_manually_rooted = || -> ManuallyRooted<ExternRef> { todo!() };-/// {-/// let perma_root: ManuallyRooted<_> = get_manually_rooted();-///-/// // `perma_root` is implicitly dropped at the end of its scope,-/// // permanently rooting/leaking its referent.-/// }-/// ```-///-/// * Explicitly dropping a `ManuallyRooted<T>`: `drop(my_manually_rooted)`.-///-/// * Forgetting a `ManuallyRooted<T>`: `std::mem::forget(my_manually_rooted)`.-///-/// * Inserting a `ManuallyRooted<T>` into a `std::sync::Arc` or `std::rc::Rc`-/// cycle.-///-/// * Etc...-///-/// Wasmtime does *not* assert that a `ManuallyRooted<T>` is unrooted on `Drop`,-/// or otherwise raise a panic, log a warning, or etc... on failure to manually-/// unroot. Sometimes leaking is intentional and desirable, particularly when-/// dealing with short-lived [`Store`][crate::Store]s where unrooting would just-/// be busy work since the whole store is about to be dropped.-#[repr(transparent)] // NB: the C API relies on this-pub struct ManuallyRooted<T>+pub struct OwnedRooted<T>whereT: GcRef,{inner: GcRootIndex,+ liveness_flag: Arc<()>,_phantom: marker::PhantomData<T>,}const _: () = {use crate::{AnyRef, ExternRef};- // NB: these match the C API which should also be updated if this changes- assert!(mem::size_of::<ManuallyRooted<AnyRef>>() == 16);- assert!(mem::align_of::<ManuallyRooted<AnyRef>>() == mem::align_of::<u64>());- assert!(mem::size_of::<ManuallyRooted<ExternRef>>() == 16);- assert!(mem::align_of::<ManuallyRooted<ExternRef>>() == mem::align_of::<u64>());+ // NB: these match the C API which should also be updated if this changes.+ //+ // The size is really "16 + pointer + alignment", which is either+ // 20 bytes on some 32-bit platforms or 24 bytes on other 32-bit+ // platforms (e.g., riscv32, which adds an extra 4 bytes of+ // padding) and 64-bit platforms.+ assert!(+ mem::size_of::<OwnedRooted<AnyRef>>() >= 16 && mem::size_of::<OwnedRooted<AnyRef>>() <= 24+ );+ assert!(mem::align_of::<OwnedRooted<AnyRef>>() == mem::align_of::<u64>());+ assert!(+ mem::size_of::<OwnedRooted<ExternRef>>() >= 16+ && mem::size_of::<OwnedRooted<ExternRef>>() <= 24+ );+ assert!(mem::align_of::<OwnedRooted<ExternRef>>() == mem::align_of::<u64>());};-impl<T: GcRef> Debug for ManuallyRooted<T> {+impl<T: GcRef> Debug for OwnedRooted<T> {fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {- let name = format!("ManuallyRooted<{}>", any::type_name::<T>());+ let name = format!("OwnedRooted<{}>", any::type_name::<T>());f.debug_struct(&name).field("inner", &self.inner).finish()}}-impl<T: GcRef> Deref for ManuallyRooted<T> {+impl<T: GcRef> Deref for OwnedRooted<T> {type Target = T;fn deref(&self) -> &Self::Target {@@ -1405,11 +1605,21 @@ impl<T: GcRef> Deref for ManuallyRooted<T> {}}-impl<T> ManuallyRooted<T>+impl<T: GcRef> Clone for OwnedRooted<T> {+ fn clone(&self) -> Self {+ OwnedRooted {+ inner: self.inner,+ liveness_flag: self.liveness_flag.clone(),+ _phantom: marker::PhantomData,+ }+ }+}++impl<T> OwnedRooted<T>whereT: GcRef,{- /// Construct a new manually-rooted GC root.+ /// Construct a new owned GC root.////// `gc_ref` should belong to `store`'s heap; failure to uphold this is/// memory safe but will result in general failures down the line such as@@ -1419,101 +1629,49 @@ where/// that `T` represents. Failure to uphold this invariant is memory safe but/// will result in general incorrectness such as panics and wrong results.pub(crate) fn new(store: &mut AutoAssertNoGc<'_>, gc_ref: VMGcRef) -> Self {- let id = store.gc_roots_mut().manually_rooted.alloc(gc_ref);- ManuallyRooted {+ // We always have the opportunity to trim and unregister stale+ // owned roots whenever we have a mut borrow to the store. We+ // take the opportunity to do so here to avoid tying growth of+ // the root-set to the GC frequency -- it is much cheaper to+ // eagerly trim these roots. Note that the trimming keeps a+ // "high water mark" that grows exponentially, so we have+ // amortized constant time even though an individual trim+ // takes time linear in the number of roots.+ store.trim_gc_liveness_flags(false);++ let roots = store.gc_roots_mut();+ let id = roots.owned_rooted.alloc(gc_ref);+ let liveness_flag = Arc::new(());+ roots+ .liveness_flags+ .push((Arc::downgrade(&liveness_flag), id));+ OwnedRooted {inner: GcRootIndex {store_id: store.id(),generation: 0,- index: PackedIndex::new_manual(id),+ index: PackedIndex::new_owned(id),},+ liveness_flag,_phantom: marker::PhantomData,}}#[inline]pub(crate) fn comes_from_same_store(&self, store: &StoreOpaque) -> bool {- debug_assert!(self.inner.index.is_manual());+ debug_assert!(self.inner.index.is_owned());self.inner.comes_from_same_store(store)}- /// Clone this `ManuallyRooted`.- ///- /// Does not consume or unroot `self`: both `self` and the new- /// `ManuallyRooted` return value will need to be manually unrooted.- ///- /// # Panics- ///- /// Panics if `self` is not associated with the given `store`.- ///- /// # Example- ///- /// ```- /// # use wasmtime::*;- /// # fn _foo() -> Result<()> {- /// let mut store = Store::<Vec<ManuallyRooted<ExternRef>>>::default();- ///- /// // Create our `ManuallyRooted` in a nested scope to avoid rooting it for- /// // the duration of the store's lifetime.- /// let x = {- /// let mut scope = RootScope::new(&mut store);- /// let x = ExternRef::new(&mut scope, 1234)?;- /// x.to_manually_rooted(&mut scope)?- /// };- ///- /// // Push five clones of `x` into our store.- /// for _ in 0..5 {- /// let x_clone = x.clone(&mut store);- /// store.data_mut().push(x_clone);- /// }- /// # Ok(())- /// # }- /// ```- pub fn clone(&self, mut store: impl AsContextMut) -> Self {- self._clone(store.as_context_mut().0)- }-- pub(crate) fn _clone(&self, store: &mut StoreOpaque) -> Self {- let mut store = AutoAssertNoGc::new(store);- let gc_ref = self- .clone_gc_ref(&mut store)- .expect("ManuallyRooted always has a gc ref");- Self::new(&mut store, gc_ref)- }-- /// Unroot this GC object.- ///- /// Failure to call this method will result in the GC object, and anything- /// it transitively references, being kept alive (aka "leaking") for the- /// entirety of the store's lifetime.- ///- /// See the type-level docs for example usage.- pub fn unroot(self, mut store: impl AsContextMut) {- self._unroot(store.as_context_mut().0)- }-- pub(crate) fn _unroot(self, store: &mut StoreOpaque) {- assert!(- self.comes_from_same_store(store),- "object used with wrong store"- );-- let mut store = AutoAssertNoGc::new(store);- let id = self.inner.index.as_manual().unwrap();- let roots = store.gc_roots_mut();- let gc_ref = roots.manually_rooted.dealloc(id);- store.unwrap_gc_store_mut().drop_gc_ref(gc_ref);- }-- /// Clone this `ManuallyRooted<T>` into a `Rooted<T>`.+ /// Clone this `OwnedRooted<T>` into a `Rooted<T>`.///- /// This operation does not consume or unroot this `ManuallyRooted<T>`.+ /// This operation does not consume or unroot this `OwnedRooted<T>`.////// The underlying GC object is re-rooted in the given context's scope. The/// resulting `Rooted<T>` is only valid during the given context's/// scope. See the [`Rooted<T>`][crate::Rooted] documentation for more/// details on rooting scopes.///- /// This operation does not consume or unroot this `ManuallyRooted<T>`.+ /// This operation does not consume or unroot this `OwnedRooted<T>`.////// # Panics///@@ -1528,24 +1686,19 @@ where////// let root1: Rooted<_>;///- /// let manual = {+ /// let owned = {/// let mut scope = RootScope::new(&mut store);/// root1 = ExternRef::new(&mut scope, 1234)?;- /// root1.to_manually_rooted(&mut scope)?+ /// root1.to_owned_rooted(&mut scope)?/// };////// // `root1` is no longer accessible because it was unrooted when `scope`/// // was dropped./// assert!(root1.data(&store).is_err());///- /// // But we can re-root `manual` into this scope.- /// let root2 = manual.to_rooted(&mut store);+ /// // But we can re-root `owned` into this scope.+ /// let root2 = owned.to_rooted(&mut store);/// assert!(root2.data(&store).is_ok());- ///- /// // And we also still have access to `manual` and we still have to- /// // manually unroot it.- /// assert!(manual.data(&store).is_ok());- /// manual.unroot(&mut store);/// # Ok(())/// # }/// ```@@ -1563,61 +1716,6 @@ whereRooted::new(&mut store, gc_ref)}- /// Convert this `ManuallyRooted<T>` into a `Rooted<T>`.- ///- /// The underlying GC object is re-rooted in the given context's scope. The- /// resulting `Rooted<T>` is only valid during the given context's- /// scope. See the [`Rooted<T>`][crate::Rooted] documentation for more- /// details on rooting scopes.- ///- /// This operation consumes and unroots this `ManuallyRooted<T>`.- ///- /// # Panics- ///- /// Panics if this object is not associate with the given context's store.- ///- /// # Example- ///- /// ```- /// # use wasmtime::*;- /// # fn _foo() -> Result<()> {- /// let mut store = Store::<()>::default();- ///- /// let root1: Rooted<_>;- ///- /// let manual = {- /// let mut scope = RootScope::new(&mut store);- /// root1 = ExternRef::new(&mut scope, 1234)?;- /// root1.to_manually_rooted(&mut scope)?- /// };- ///- /// // `root1` is no longer accessible because it was unrooted when `scope`- /// // was dropped.- /// assert!(root1.data(&store).is_err());- ///- /// // But we can re-root `manual` into this scope.- /// let root2 = manual.into_rooted(&mut store);- /// assert!(root2.data(&store).is_ok());- ///- /// // `manual` was consumed by the `into_rooted` call, and we no longer- /// // have access to it, nor need to manually unroot it.- /// # Ok(())- /// # }- /// ```- pub fn into_rooted(self, mut context: impl AsContextMut) -> Rooted<T> {- self._into_rooted(context.as_context_mut().0)- }-- pub(crate) fn _into_rooted(self, store: &mut StoreOpaque) -> Rooted<T> {- assert!(- self.comes_from_same_store(store),- "object used with wrong store"- );- let rooted = self._to_rooted(store);- self._unroot(store);- rooted- }-/// Are these two GC roots referencing the same underlying GC object?////// This function will return `true` even when `a` and `b` are different GC@@ -1625,7 +1723,7 @@ where/// are rooting the same underlying GC object.////// Because this method takes any `impl RootedGcRef<T>` arguments, it can be- /// used to compare, for example, a `Rooted<T>` and a `ManuallyRooted<T>`.+ /// used to compare, for example, a `Rooted<T>` and an `OwnedRooted<T>`.////// # Panics///@@ -1645,28 +1743,23 @@ where/// {/// let mut scope = RootScope::new(&mut store);///- /// a = ExternRef::new(&mut scope, "hello")?.to_manually_rooted(&mut scope)?;- /// b = a.clone(&mut scope);+ /// a = ExternRef::new(&mut scope, "hello")?.to_owned_rooted(&mut scope)?;+ /// b = a.clone();////// // `a` and `b` are rooting the same object.- /// assert!(ManuallyRooted::ref_eq(&scope, &a, &b)?);+ /// assert!(OwnedRooted::ref_eq(&scope, &a, &b)?);////// // `c` is a different GC root, is in a different scope, and is a- /// // `Rooted<T>` instead of a `ManuallyRooted<T>`, but is still rooting+ /// // `Rooted<T>` instead of a `OwnedRooted<T>`, but is still rooting/// // the same object./// let c = a.to_rooted(&mut scope);- /// assert!(ManuallyRooted::ref_eq(&scope, &a, &c)?);+ /// assert!(OwnedRooted::ref_eq(&scope, &a, &c)?);///- /// x = ExternRef::new(&mut scope, "goodbye")?.to_manually_rooted(&mut scope)?;+ /// x = ExternRef::new(&mut scope, "goodbye")?.to_owned_rooted(&mut scope)?;////// // `a` and `x` are rooting different objects.- /// assert!(!ManuallyRooted::ref_eq(&scope, &a, &x)?);+ /// assert!(!OwnedRooted::ref_eq(&scope, &a, &x)?);/// }- ///- /// // Unroot our manually-rooted GC references.- /// a.unroot(&mut store);- /// b.unroot(&mut store);- /// x.unroot(&mut store);/// # Ok(())/// # }/// ```@@ -1684,7 +1777,7 @@ where/// root and *not* the underlying GC reference. That means that two/// different rootings of the same object will hash to different values/// (modulo hash collisions). If this is undesirable, use the- /// [`ref_hash`][crate::ManuallyRooted::ref_hash] method instead.+ /// [`ref_hash`][crate::OwnedRooted::ref_hash] method instead.pub fn rooted_hash<H>(&self, state: &mut H)whereH: Hasher,@@ -1705,47 +1798,25 @@ where{let gc_ref = self.get_gc_ref(store.as_context().0)- .expect("ManuallyRooted's get_gc_ref is infallible");+ .expect("OwnedRooted's get_gc_ref is infallible");gc_ref.hash(state);}- #[doc(hidden)]- pub fn into_parts_for_c_api(self) -> (NonZeroU64, u32, u32) {- (- self.inner.store_id.as_raw(),- self.inner.generation,- self.inner.index.0,- )- }-- #[doc(hidden)]- pub unsafe fn from_raw_parts_for_c_api(a: NonZeroU64, b: u32, c: u32) -> ManuallyRooted<T> {- ManuallyRooted {- inner: GcRootIndex {- store_id: StoreId::from_raw(a),- generation: b,- index: PackedIndex(c),- },- _phantom: marker::PhantomData,- }- }-- /// Cast `self` to a `ManuallyRooted<U>`.+ /// Cast `self` to an `OwnedRooted<U>`.////// It is the caller's responsibility to ensure that `self` is actually a/// `U`. Failure to uphold this invariant will be memory safe but will/// result in general incorrectness such as panics and wrong results.- pub(crate) fn unchecked_cast<U: GcRef>(self) -> ManuallyRooted<U> {- let u = ManuallyRooted {+ pub(crate) fn unchecked_cast<U: GcRef>(self) -> OwnedRooted<U> {+ OwnedRooted {inner: self.inner,+ liveness_flag: self.liveness_flag,_phantom: core::marker::PhantomData,- };- core::mem::forget(self);- u+ }}/// Common implementation of the `WasmTy::store` trait method for all- /// `ManuallyRooted<T>`s.+ /// `OwnedRooted<T>`s.pub(super) fn wasm_ty_store(self,store: &mut AutoAssertNoGc<'_>,@@ -1759,7 +1830,7 @@ where}/// Common implementation of the `WasmTy::load` trait method for all- /// `ManuallyRooted<T>`s.+ /// `OwnedRooted<T>`s.pub(super) fn wasm_ty_load(store: &mut AutoAssertNoGc<'_>,raw_gc_ref: u32,@@ -1770,14 +1841,12 @@ wherelet gc_ref = store.clone_gc_ref(&gc_ref);RootSet::with_lifo_scope(store, |store| {let rooted = from_cloned_gc_ref(store, gc_ref);- rooted- ._to_manually_rooted(store)- .expect("rooted is in scope")+ rooted._to_owned_rooted(store).expect("rooted is in scope")})}/// Common implementation of the `WasmTy::store` trait method for all- /// `Option<ManuallyRooted<T>>`s.+ /// `Option<OwnedRooted<T>>`s.pub(super) fn wasm_ty_option_store(me: Option<Self>,store: &mut AutoAssertNoGc<'_>,@@ -1794,7 +1863,7 @@ where}/// Common implementation of the `WasmTy::load` trait method for all- /// `Option<ManuallyRooted<T>>`s.+ /// `Option<OwnedRooted<T>>`s.pub(super) fn wasm_ty_option_load(store: &mut AutoAssertNoGc<'_>,raw_gc_ref: u32,@@ -1804,24 +1873,77 @@ wherelet gc_ref = store.clone_gc_ref(&gc_ref);RootSet::with_lifo_scope(store, |store| {let rooted = from_cloned_gc_ref(store, gc_ref);- Some(- rooted- ._to_manually_rooted(store)- .expect("rooted is in scope"),- )+ Some(rooted._to_owned_rooted(store).expect("rooted is in scope"))})}++ #[doc(hidden)]+ pub fn into_parts_for_c_api(self) -> (NonZeroU64, u32, u32, *const ()) {+ (+ self.inner.store_id.as_raw(),+ self.inner.generation,+ self.inner.index.0,+ Arc::into_raw(self.liveness_flag),+ )+ }++ #[doc(hidden)]+ pub unsafe fn from_borrowed_raw_parts_for_c_api(+ a: NonZeroU64,+ b: u32,+ c: u32,+ d: *const (),+ ) -> OwnedRooted<T> {+ // We are given a *borrow* of the Arc. This is a little+ // sketchy because `Arc::from_raw()` takes *ownership* of the+ // passed-in pointer, so we need to clone then forget that+ // original.+ let liveness_flag = {+ let original = unsafe { Arc::from_raw(d) };+ let clone = original.clone();+ core::mem::forget(original);+ clone+ };+ OwnedRooted {+ inner: GcRootIndex {+ store_id: StoreId::from_raw(a),+ generation: b,+ index: PackedIndex(c),+ },+ liveness_flag,+ _phantom: marker::PhantomData,+ }+ }++ #[doc(hidden)]+ pub unsafe fn from_owned_raw_parts_for_c_api(+ a: NonZeroU64,+ b: u32,+ c: u32,+ d: *const (),+ ) -> OwnedRooted<T> {+ let liveness_flag = unsafe { Arc::from_raw(d) };+ OwnedRooted {+ inner: GcRootIndex {+ store_id: StoreId::from_raw(a),+ generation: b,+ index: PackedIndex(c),+ },+ liveness_flag,+ _phantom: marker::PhantomData,+ }+ }}-impl<T: GcRef> RootedGcRefImpl<T> for ManuallyRooted<T> {+impl<T: GcRef> RootedGcRefImpl<T> for OwnedRooted<T> {fn get_gc_ref<'a>(&self, store: &'a StoreOpaque) -> Option<&'a VMGcRef> {assert!(self.comes_from_same_store(store),"object used with wrong store");- let id = self.inner.index.as_manual().unwrap();- store.gc_roots().manually_rooted.get(id)+ let id = self.inner.index.as_owned().unwrap();+ store.gc_roots().owned_rooted.get(id)}}@@ -1836,6 +1958,6 @@ mod tests {// Try to keep tabs on the size of these things. Don't want them growing// unintentionally.assert_eq!(std::mem::size_of::<Rooted<ExternRef>>(), 16);- assert_eq!(std::mem::size_of::<ManuallyRooted<ExternRef>>(), 16);+ assert!(std::mem::size_of::<OwnedRooted<ExternRef>>() <= 24);}}diff --git a/crates/wasmtime/src/runtime/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/gc/enabled/structref.rsindex 234cf0baf4..b348338e61 100644--- a/crates/wasmtime/src/runtime/gc/enabled/structref.rs+++ b/crates/wasmtime/src/runtime/gc/enabled/structref.rs@@ -6,7 +6,7 @@ use crate::vm::{self, VMGcHeader, VMStore, VMStructRef};use crate::{AnyRef, FieldType};use crate::{AsContext, AsContextMut, EqRef, GcHeapOutOfMemory, GcRefImpl, GcRootIndex, HeapType,- ManuallyRooted, RefType, Rooted, StructType, Val, ValRaw, ValType, WasmTy,+ OwnedRooted, RefType, Rooted, StructType, Val, ValRaw, ValType, WasmTy,prelude::*,store::{AutoAssertNoGc, StoreContextMut, StoreOpaque, StoreResourceLimiter},};@@ -103,7 +103,7 @@ impl StructRefPre {/// the host into dereferencing it and segfaulting or worse.////// Note that you can also use `Rooted<StructRef>` and-/// `ManuallyRooted<StructRef>` as a type parameter with+/// `OwnedRooted<StructRef>` as a type parameter with/// [`Func::typed`][crate::Func::typed]- and/// [`Func::wrap`][crate::Func::wrap]-style APIs.///@@ -185,16 +185,16 @@ impl Rooted<StructRef> {}}-impl ManuallyRooted<StructRef> {+impl OwnedRooted<StructRef> {/// Upcast this `structref` into an `anyref`.#[inline]- pub fn to_anyref(self) -> ManuallyRooted<AnyRef> {+ pub fn to_anyref(self) -> OwnedRooted<AnyRef> {self.unchecked_cast()}/// Upcast this `structref` into an `eqref`.#[inline]- pub fn to_eqref(self) -> ManuallyRooted<EqRef> {+ pub fn to_eqref(self) -> OwnedRooted<EqRef> {self.unchecked_cast()}}@@ -723,7 +723,7 @@ unsafe impl WasmTy for Option<Rooted<StructRef>> {}}-unsafe impl WasmTy for ManuallyRooted<StructRef> {+unsafe impl WasmTy for OwnedRooted<StructRef> {#[inline]fn valtype() -> ValType {ValType::Ref(RefType::new(false, HeapType::Struct))@@ -775,7 +775,7 @@ unsafe impl WasmTy for ManuallyRooted<StructRef> {}}-unsafe impl WasmTy for Option<ManuallyRooted<StructRef>> {+unsafe impl WasmTy for Option<OwnedRooted<StructRef>> {#[inline]fn valtype() -> ValType {ValType::STRUCTREF@@ -796,7 +796,7 @@ unsafe impl WasmTy for Option<ManuallyRooted<StructRef>> {) -> Result<()> {match self {Some(s) => {- ManuallyRooted::<StructRef>::dynamic_concrete_type_check(s, store, nullable, ty)+ OwnedRooted::<StructRef>::dynamic_concrete_type_check(s, store, nullable, ty)}None => {ensure!(@@ -814,11 +814,11 @@ unsafe impl WasmTy for Option<ManuallyRooted<StructRef>> {}fn store(self, store: &mut AutoAssertNoGc<'_>, ptr: &mut MaybeUninit<ValRaw>) -> Result<()> {- <ManuallyRooted<StructRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)+ <OwnedRooted<StructRef>>::wasm_ty_option_store(self, store, ptr, ValRaw::anyref)}unsafe fn load(store: &mut AutoAssertNoGc<'_>, ptr: &ValRaw) -> Self {- <ManuallyRooted<StructRef>>::wasm_ty_option_load(+ <OwnedRooted<StructRef>>::wasm_ty_option_load(store,ptr.get_anyref(),StructRef::from_cloned_gc_ref,diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rsindex 5be8a7d717..70f60f444f 100644--- a/crates/wasmtime/src/runtime/store/gc.rs+++ b/crates/wasmtime/src/runtime/store/gc.rs@@ -22,6 +22,7 @@ impl StoreOpaque {bytes_needed: Option<u64>,) -> Option<VMGcRef> {let mut scope = crate::OpaqueRootScope::new(self);+ scope.trim_gc_liveness_flags(true);let store_id = scope.id();let root = root.map(|r| scope.gc_roots_mut().push_lifo_root(store_id, r));@@ -36,6 +37,16 @@ impl StoreOpaque {})}+ // This lives on the Store because it must simultaneously borrow+ // `gc_store` and `gc_roots`, and is invoked from other modules to+ // which we do not want to expose the raw fields for piecewise+ // borrows.+ pub(crate) fn trim_gc_liveness_flags(&mut self, eager: bool) {+ if let Some(gc_store) = self.gc_store.as_mut() {+ self.gc_roots.trim_liveness_flags(gc_store, eager);+ }+ }+async fn grow_or_collect_gc_heap(&mut self,limiter: Option<&mut StoreResourceLimiter<'_>>,diff --git a/tests/all/func.rs b/tests/all/func.rsindex 2c6209a5ff..98e39f8602 100644--- a/tests/all/func.rs+++ b/tests/all/func.rs@@ -401,18 +401,18 @@ fn func_constructors() {loop {}});Func::wrap(&mut store, || -> Option<Rooted<ExternRef>> { None });- Func::wrap(&mut store, || -> ManuallyRooted<ExternRef> {+ Func::wrap(&mut store, || -> OwnedRooted<ExternRef> {loop {}});- Func::wrap(&mut store, || -> Option<ManuallyRooted<ExternRef>> { None });+ Func::wrap(&mut store, || -> Option<OwnedRooted<ExternRef>> { None });Func::wrap(&mut store, || -> Rooted<AnyRef> {loop {}});Func::wrap(&mut store, || -> Option<Rooted<AnyRef>> { None });- Func::wrap(&mut store, || -> ManuallyRooted<AnyRef> {+ Func::wrap(&mut store, || -> OwnedRooted<AnyRef> {loop {}});- Func::wrap(&mut store, || -> Option<ManuallyRooted<AnyRef>> { None });+ Func::wrap(&mut store, || -> Option<OwnedRooted<AnyRef>> { None });Func::wrap(&mut store, || -> I31 {loop {}});@@ -455,23 +455,22 @@ fn func_constructors() {Func::wrap(&mut store, || -> Result<Option<Rooted<ExternRef>>> {loop {}});- Func::wrap(&mut store, || -> Result<ManuallyRooted<ExternRef>> {+ Func::wrap(&mut store, || -> Result<OwnedRooted<ExternRef>> {+ loop {}+ });+ Func::wrap(&mut store, || -> Result<Option<OwnedRooted<ExternRef>>> {loop {}});- Func::wrap(- &mut store,- || -> Result<Option<ManuallyRooted<ExternRef>>> { loop {} },- );Func::wrap(&mut store, || -> Result<Rooted<AnyRef>> {loop {}});Func::wrap(&mut store, || -> Result<Option<Rooted<AnyRef>>> {loop {}});- Func::wrap(&mut store, || -> Result<ManuallyRooted<AnyRef>> {+ Func::wrap(&mut store, || -> Result<OwnedRooted<AnyRef>> {loop {}});- Func::wrap(&mut store, || -> Result<Option<ManuallyRooted<AnyRef>>> {+ Func::wrap(&mut store, || -> Result<Option<OwnedRooted<AnyRef>>> {loop {}});Func::wrap(&mut store, || -> Result<I31> {@@ -601,9 +600,9 @@ fn signatures_match() {_: i64,_: i32,_: Option<Rooted<ExternRef>>,- _: Option<ManuallyRooted<ExternRef>>,+ _: Option<OwnedRooted<ExternRef>>,_: Option<Rooted<AnyRef>>,- _: Option<ManuallyRooted<AnyRef>>,+ _: Option<OwnedRooted<AnyRef>>,_: Option<Func>|-> f64 { loop {} },);@@ -698,10 +697,10 @@ fn import_works() -> Result<()> {d: f32,e: f64,f: Option<Rooted<ExternRef>>,- g: Option<ManuallyRooted<ExternRef>>,+ g: Option<OwnedRooted<ExternRef>>,h: Option<Func>,i: Option<Rooted<AnyRef>>,- j: Option<ManuallyRooted<AnyRef>>,+ j: Option<OwnedRooted<AnyRef>>,k: Option<I31>|-> Result<()> {assert_eq!(a, 100);@@ -820,15 +819,15 @@ fn get_from_wrapper() {loop {}});assert!(f.typed::<(), Option<Rooted<ExternRef>>>(&store).is_ok());- let f = Func::wrap(&mut store, || -> ManuallyRooted<ExternRef> {+ let f = Func::wrap(&mut store, || -> OwnedRooted<ExternRef> {loop {}});- assert!(f.typed::<(), ManuallyRooted<ExternRef>>(&store).is_ok());- let f = Func::wrap(&mut store, || -> Option<ManuallyRooted<ExternRef>> {+ assert!(f.typed::<(), OwnedRooted<ExternRef>>(&store).is_ok());+ let f = Func::wrap(&mut store, || -> Option<OwnedRooted<ExternRef>> {loop {}});assert!(- f.typed::<(), Option<ManuallyRooted<ExternRef>>>(&store)+ f.typed::<(), Option<OwnedRooted<ExternRef>>>(&store).is_ok());let f = Func::wrap(&mut store, || -> Rooted<AnyRef> {@@ -839,17 +838,14 @@ fn get_from_wrapper() {loop {}});assert!(f.typed::<(), Option<Rooted<AnyRef>>>(&store).is_ok());- let f = Func::wrap(&mut store, || -> ManuallyRooted<AnyRef> {+ let f = Func::wrap(&mut store, || -> OwnedRooted<AnyRef> {loop {}});- assert!(f.typed::<(), ManuallyRooted<AnyRef>>(&store).is_ok());- let f = Func::wrap(&mut store, || -> Option<ManuallyRooted<AnyRef>> {+ assert!(f.typed::<(), OwnedRooted<AnyRef>>(&store).is_ok());+ let f = Func::wrap(&mut store, || -> Option<OwnedRooted<AnyRef>> {loop {}});- assert!(- f.typed::<(), Option<ManuallyRooted<AnyRef>>>(&store)- .is_ok()- );+ assert!(f.typed::<(), Option<OwnedRooted<AnyRef>>>(&store).is_ok());let f = Func::wrap(&mut store, || -> I31 {loop {}});@@ -882,24 +878,21 @@ fn get_from_wrapper() {assert!(f.typed::<Rooted<ExternRef>, ()>(&store).is_ok());let f = Func::wrap(&mut store, |_: Option<Rooted<ExternRef>>| {});assert!(f.typed::<Option<Rooted<ExternRef>>, ()>(&store).is_ok());- let f = Func::wrap(&mut store, |_: ManuallyRooted<ExternRef>| {});- assert!(f.typed::<ManuallyRooted<ExternRef>, ()>(&store).is_ok());- let f = Func::wrap(&mut store, |_: Option<ManuallyRooted<ExternRef>>| {});+ let f = Func::wrap(&mut store, |_: OwnedRooted<ExternRef>| {});+ assert!(f.typed::<OwnedRooted<ExternRef>, ()>(&store).is_ok());+ let f = Func::wrap(&mut store, |_: Option<OwnedRooted<ExternRef>>| {});assert!(- f.typed::<Option<ManuallyRooted<ExternRef>>, ()>(&store)+ f.typed::<Option<OwnedRooted<ExternRef>>, ()>(&store).is_ok());let f = Func::wrap(&mut store, |_: Rooted<AnyRef>| {});assert!(f.typed::<Rooted<AnyRef>, ()>(&store).is_ok());let f = Func::wrap(&mut store, |_: Option<Rooted<AnyRef>>| {});assert!(f.typed::<Option<Rooted<AnyRef>>, ()>(&store).is_ok());- let f = Func::wrap(&mut store, |_: ManuallyRooted<AnyRef>| {});- assert!(f.typed::<ManuallyRooted<AnyRef>, ()>(&store).is_ok());- let f = Func::wrap(&mut store, |_: Option<ManuallyRooted<AnyRef>>| {});- assert!(- f.typed::<Option<ManuallyRooted<AnyRef>>, ()>(&store)- .is_ok()- );+ let f = Func::wrap(&mut store, |_: OwnedRooted<AnyRef>| {});+ assert!(f.typed::<OwnedRooted<AnyRef>, ()>(&store).is_ok());+ let f = Func::wrap(&mut store, |_: Option<OwnedRooted<AnyRef>>| {});+ assert!(f.typed::<Option<OwnedRooted<AnyRef>>, ()>(&store).is_ok());let f = Func::wrap(&mut store, |_: I31| {});assert!(f.typed::<I31, ()>(&store).is_ok());let f = Func::wrap(&mut store, |_: Option<I31>| {});diff --git a/tests/all/gc.rs b/tests/all/gc.rsindex 546f814e22..e1eb55b55a 100644--- a/tests/all/gc.rs+++ b/tests/all/gc.rs@@ -747,19 +747,19 @@ fn rooted_gets_collected_after_scope_exit() -> Result<()> {}#[test]-fn manually_rooted_gets_collected_after_unrooting() -> Result<()> {+fn owned_rooted_gets_collected_after_unrooting() -> Result<()> {let mut store = Store::<()>::default();let flag = Arc::new(AtomicBool::new(false));let externref = {let mut scope = RootScope::new(&mut store);- ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?.to_manually_rooted(&mut scope)?+ ExternRef::new(&mut scope, SetFlagOnDrop(flag.clone()))?.to_owned_rooted(&mut scope)?};store.gc(None);assert!(!flag.load(SeqCst), "not dropped when still rooted");- externref.unroot(&mut store);+ drop(externref);store.gc(None);assert!(flag.load(SeqCst), "dropped after being unrooted");@@ -1495,3 +1495,75 @@ fn drc_gc_inbetween_host_calls() -> Result<()> {Ok(())}++#[test]+fn owned_rooted() -> Result<()> {+ let _ = env_logger::try_init();++ let mut config = Config::new();+ config.wasm_function_references(true);+ config.wasm_gc(true);+ config.collector(Collector::DeferredReferenceCounting);++ let engine = Engine::new(&config)?;++ let mut store = Store::new(&engine, ());+ let inner_dropped = Arc::new(AtomicBool::new(false));+ let r = {+ let mut scope = RootScope::new(&mut store);+ let r = ExternRef::new(&mut scope, SetFlagOnDrop(inner_dropped.clone()))?;+ r.to_owned_rooted(&mut scope)?+ };+ assert!(!inner_dropped.load(SeqCst));+ store.gc(None);+ assert!(!inner_dropped.load(SeqCst));+ let r2 = r.clone();+ store.gc(None);+ assert!(!inner_dropped.load(SeqCst));+ drop(r);+ store.gc(None);+ assert!(!inner_dropped.load(SeqCst));+ drop(r2);+ store.gc(None);+ assert!(inner_dropped.load(SeqCst));++ Ok(())+}++#[test]+fn owned_rooted_lots_of_root_creation() -> Result<()> {+ let mut config = Config::new();+ config.wasm_function_references(true);+ config.wasm_gc(true);+ config.collector(Collector::DeferredReferenceCounting);++ let engine = Engine::new(&config)?;++ let mut store = Store::new(&engine, ());+ let inner_dropped = Arc::new(AtomicBool::new(false));+ let r = {+ let mut scope = RootScope::new(&mut store);+ let r = ExternRef::new(&mut scope, SetFlagOnDrop(inner_dropped.clone()))?;+ r.to_owned_rooted(&mut scope)?+ };+ assert!(!inner_dropped.load(SeqCst));+ store.gc(None);++ for _ in 0..100_000 {+ let mut scope = RootScope::new(&mut store);+ // Go through a LIFO root then back to an owned root to create+ // a distinct root.+ let r2 = r.to_rooted(&mut scope);+ let r3 = r2.to_owned_rooted(&mut scope);+ drop(r3);+ }++ store.gc(None);+ assert!(!inner_dropped.load(SeqCst));++ drop(r);+ store.gc(None);+ assert!(inner_dropped.load(SeqCst));++ Ok(())+}