Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Always use fallible accesses of the GC heap (#13320)

Alex Crichtonbf3eda7a9cc1
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 46ab22b02b..09cd072bdd 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -24,4 +24,5 @@ pub mod math;
pub mod mpk;
pub mod non_max;
pub mod slab;
+pub mod truncate;
pub mod undo;
diff --git a/crates/core/src/truncate.rs b/crates/core/src/truncate.rs
new file mode 100644
index 0000000000..9bbcc27ce1
--- /dev/null
+++ b/crates/core/src/truncate.rs
@@ -0,0 +1,11 @@
+//! Explicit methods to clearly indicate that truncation is desired when used.
+
+/// Explicitly truncate an `i32` to an `i16`.
+pub fn truncate_i32_to_i16(a: i32) -> i16 {
+ a as i16
+}
+
+/// Explicitly truncate an `i32` to an `i8`.
+pub fn truncate_i32_to_i8(a: i32) -> i8 {
+ a as i8
+}
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index 2e4385a7e6..8de8f857e7 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -391,10 +391,12 @@ impl BuiltinFunctionIndex {
// The final epoch represents a trap
(@get new_epoch u64) => (TrapSentinel::NegativeOne);
+ // Failure here indicates GC heap corruption.
+ (@get get_interned_func_ref pointer) => (TrapSentinel::NegativeOne);
+
// These libcalls can't trap
(@get ref_func pointer) => (return None);
(@get table_get_lazy_init_func_ref pointer) => (return None);
- (@get get_interned_func_ref pointer) => (return None);
(@get intern_func_ref_for_gc_heap u64) => (return None);
(@get is_subtype u32) => (return None);
(@get ceil_f32 f32) => (return None);
diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs
index c9fae3127c..6b4a486510 100644
--- a/crates/wasmtime/src/runtime/externals/global.rs
+++ b/crates/wasmtime/src/runtime/externals/global.rs
@@ -270,7 +270,7 @@ impl Global {
Some(e) => Some(e.try_gc_ref(&store)?.unchecked_copy()),
};
let new = new.as_ref();
- definition.write_gc_ref(&mut store, new);
+ definition.write_gc_ref(&mut store, new)?;
}
Val::AnyRef(a) => {
let new = match a {
@@ -278,7 +278,7 @@ impl Global {
Some(a) => Some(a.try_gc_ref(&store)?.unchecked_copy()),
};
let new = new.as_ref();
- definition.write_gc_ref(&mut store, new);
+ definition.write_gc_ref(&mut store, new)?;
}
Val::ExnRef(e) => {
let new = match e {
@@ -286,11 +286,11 @@ impl Global {
Some(e) => Some(e.try_gc_ref(&store)?.unchecked_copy()),
};
let new = new.as_ref();
- definition.write_gc_ref(&mut store, new);
+ definition.write_gc_ref(&mut store, new)?;
}
Val::ContRef(None) => {
// Allow null continuation references for globals - these are just placeholders
- definition.write_gc_ref(&mut store, None);
+ definition.write_gc_ref(&mut store, None)?;
}
Val::ContRef(Some(_)) => {
// TODO(#10248): Implement non-null global continuation reference handling
diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs
index a27b811e7a..22914fbdda 100644
--- a/crates/wasmtime/src/runtime/func.rs
+++ b/crates/wasmtime/src/runtime/func.rs
@@ -2157,11 +2157,11 @@ impl<T> Caller<'_, T> {
///
/// Same as [`Store::gc_async`](crate::Store::gc_async).
#[cfg(all(feature = "async", feature = "gc"))]
- pub async fn gc_async(&mut self, why: Option<&crate::GcHeapOutOfMemory<()>>)
+ pub async fn gc_async(&mut self, why: Option<&crate::GcHeapOutOfMemory<()>>) -> Result<()>
where
T: Send + 'static,
{
- self.store.gc_async(why).await;
+ self.store.gc_async(why).await
}
/// Returns the remaining fuel in the store.
diff --git a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
index e337303943..d03ba277f1 100644
--- a/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/disabled/rooting.rs
@@ -118,6 +118,10 @@ impl<T: GcRef> Rooted<T> {
pub(crate) fn try_gc_ref<'a>(&self, _store: &'a StoreOpaque) -> Result<&'a VMGcRef> {
match self.inner {}
}
+
+ pub(crate) fn try_clone_gc_ref(&self, _: &mut AutoAssertNoGc<'_>) -> Result<VMGcRef> {
+ match self.inner {}
+ }
}
/// This type has been disabled because the `gc` cargo feature was not enabled
diff --git a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
index 497e595397..8a811d0aae 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/anyref.rs
@@ -1,11 +1,11 @@
//! Implementation of `anyref` in Wasmtime.
-use super::{ExternRef, RootedGcRefImpl};
+use super::ExternRef;
use crate::prelude::*;
use crate::runtime::vm::VMGcRef;
use crate::{
ArrayRef, ArrayType, AsContext, AsContextMut, EqRef, GcRefImpl, GcRootIndex, HeapType, I31,
- OwnedRooted, RefType, Result, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy,
+ OwnedRooted, RefType, Result, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy, bail_bug,
store::{AutoAssertNoGc, StoreOpaque},
};
use core::mem;
@@ -298,11 +298,13 @@ impl AnyRef {
|| store
.unwrap_gc_store()
.header(&gc_ref)
+ .unwrap()
.kind()
.matches(VMGcKind::AnyRef)
|| store
.unwrap_gc_store()
.header(&gc_ref)
+ .unwrap()
.kind()
.matches(VMGcKind::ExternRef)
);
@@ -336,7 +338,9 @@ impl AnyRef {
let raw = if gc_ref.is_i31() {
gc_ref.as_raw_non_zero_u32()
} else {
- store.require_gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref)
+ store
+ .require_gc_store_mut()?
+ .expose_gc_ref_to_wasm(gc_ref)?
};
Ok(raw.get())
}
@@ -360,7 +364,7 @@ impl AnyRef {
return Ok(HeapType::I31);
}
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
if header.kind().matches(VMGcKind::ExternRef) {
return Ok(HeapType::Any);
@@ -368,21 +372,25 @@ impl AnyRef {
debug_assert!(header.kind().matches(VMGcKind::AnyRef));
debug_assert!(header.kind().matches(VMGcKind::EqRef));
+ let ty = match header.ty() {
+ Some(ty) => ty,
+ None => bail_bug!("ty should be present"),
+ };
if header.kind().matches(VMGcKind::StructRef) {
return Ok(HeapType::ConcreteStruct(
- StructType::from_shared_type_index(store.engine(), header.ty().unwrap()),
+ StructType::from_shared_type_index(store.engine(), ty),
));
}
if header.kind().matches(VMGcKind::ArrayRef) {
return Ok(HeapType::ConcreteArray(ArrayType::from_shared_type_index(
store.engine(),
- header.ty().unwrap(),
+ ty,
)));
}
- unreachable!("no other kinds of `anyref`s")
+ bail_bug!("no other kinds of `anyref`s")
}
/// Does this `anyref` match the given type?
@@ -436,7 +444,7 @@ impl AnyRef {
Ok(gc_ref.is_i31()
|| store
.require_gc_store()?
- .kind(gc_ref)
+ .kind(gc_ref)?
.matches(VMGcKind::EqRef))
}
@@ -564,7 +572,7 @@ impl AnyRef {
Ok(!gc_ref.is_i31()
&& store
.require_gc_store()?
- .kind(gc_ref)
+ .kind(gc_ref)?
.matches(VMGcKind::StructRef))
}
@@ -632,7 +640,7 @@ impl AnyRef {
Ok(!gc_ref.is_i31()
&& store
.require_gc_store()?
- .kind(gc_ref)
+ .kind(gc_ref)?
.matches(VMGcKind::ArrayRef))
}
diff --git a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
index fb8b2d8126..f7565aa9c3 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/arrayref.rs
@@ -393,7 +393,7 @@ impl ArrayRef {
"attempted to use a `ArrayRefPre` with the wrong store"
);
- let len = u32::try_from(elems.len()).unwrap();
+ let len = u32::try_from(elems.len())?;
// Allocate the array.
let arrayref = store
@@ -416,7 +416,7 @@ impl ArrayRef {
match (|| {
let elem_ty = allocator.ty.element_type();
for (i, elem) in elems.enumerate() {
- let i = u32::try_from(i).unwrap();
+ let i = u32::try_from(i)?;
debug_assert!(i < len);
arrayref.initialize_elem(&mut store, allocator.layout(), &elem_ty, i, *elem)?;
}
@@ -424,7 +424,9 @@ impl ArrayRef {
})() {
Ok(()) => Ok(Rooted::new(&mut store, arrayref.into())),
Err(e) => {
- store.require_gc_store_mut()?.dealloc_uninit_array(arrayref);
+ store
+ .require_gc_store_mut()?
+ .dealloc_uninit_array(arrayref)?;
Err(e)
}
}
@@ -613,11 +615,11 @@ impl ArrayRef {
assert!(self.comes_from_same_store(store));
let gc_ref = self.inner.try_gc_ref(store)?;
debug_assert!({
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
header.kind().matches(VMGcKind::ArrayRef)
});
let arrayref = gc_ref.as_arrayref_unchecked();
- Ok(arrayref.len(store))
+ arrayref.len(store)
}
/// Get the values of this array's elements.
@@ -647,7 +649,7 @@ impl ArrayRef {
let store = AutoAssertNoGc::new(store);
let gc_ref = self.inner.try_gc_ref(&store)?;
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
debug_assert!(header.kind().matches(VMGcKind::ArrayRef));
let len = self._len(&store)?;
@@ -677,7 +679,7 @@ impl ArrayRef {
return None;
}
self.index += 1;
- Some(self.arrayref._get(&mut self.store, i).unwrap())
+ self.arrayref._get(&mut self.store, i).ok()
}
#[inline]
@@ -700,7 +702,7 @@ impl ArrayRef {
fn header<'a>(&self, store: &'a AutoAssertNoGc<'_>) -> Result<&'a VMGcHeader> {
assert!(self.comes_from_same_store(&store));
let gc_ref = self.inner.try_gc_ref(store)?;
- Ok(store.require_gc_store()?.header(gc_ref))
+ Ok(store.require_gc_store()?.header(gc_ref)?)
}
fn arrayref<'a>(&self, store: &'a AutoAssertNoGc<'_>) -> Result<&'a VMArrayRef> {
@@ -755,12 +757,12 @@ impl ArrayRef {
let arrayref = self.arrayref(store)?.unchecked_copy();
let field_ty = self.field_ty(store)?;
let layout = self.layout(store)?;
- let len = arrayref.len(store);
+ let len = arrayref.len(store)?;
ensure!(
index < len,
"index out of bounds: the length is {len} but the index is {index}"
);
- Ok(arrayref.read_elem(store, &layout, field_ty.element_type(), index))
+ arrayref.read_elem(store, &layout, field_ty.element_type(), index)
}
/// Set this array's `index`th element.
@@ -811,7 +813,7 @@ impl ArrayRef {
let layout = self.layout(&store)?;
let arrayref = self.arrayref(&store)?.unchecked_copy();
- let len = arrayref.len(&store);
+ let len = arrayref.len(&store)?;
ensure!(
index < len,
"index out of bounds: the length is {len} but the index is {index}"
@@ -822,7 +824,7 @@ impl ArrayRef {
pub(crate) fn type_index(&self, store: &StoreOpaque) -> Result<VMSharedTypeIndex> {
let gc_ref = self.inner.try_gc_ref(store)?;
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
debug_assert!(header.kind().matches(VMGcKind::ArrayRef));
Ok(header.ty().expect("arrayrefs should have concrete types"))
}
diff --git a/crates/wasmtime/src/runtime/gc/enabled/eqref.rs b/crates/wasmtime/src/runtime/gc/enabled/eqref.rs
index eba879ecc4..0d95bc763d 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/eqref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/eqref.rs
@@ -2,7 +2,7 @@
#![cfg(feature = "gc")]
use crate::{
AnyRef, ArrayRef, ArrayType, AsContext, AsContextMut, GcRefImpl, GcRootIndex, HeapType, I31,
- OwnedRooted, RefType, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy,
+ OwnedRooted, RefType, Rooted, StructRef, StructType, ValRaw, ValType, WasmTy, bail_bug,
prelude::*,
runtime::vm::VMGcRef,
store::{AutoAssertNoGc, StoreOpaque},
@@ -168,6 +168,7 @@ impl EqRef {
|| store
.unwrap_gc_store()
.header(&gc_ref)
+ .unwrap()
.kind()
.matches(VMGcKind::EqRef)
);
@@ -198,22 +199,26 @@ impl EqRef {
return Ok(HeapType::I31);
}
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
+ let ty = match header.ty() {
+ Some(ty) => ty,
+ None => bail_bug!("ty should be present"),
+ };
if header.kind().matches(VMGcKind::StructRef) {
return Ok(HeapType::ConcreteStruct(
- StructType::from_shared_type_index(store.engine(), header.ty().unwrap()),
+ StructType::from_shared_type_index(store.engine(), ty),
));
}
if header.kind().matches(VMGcKind::ArrayRef) {
return Ok(HeapType::ConcreteArray(ArrayType::from_shared_type_index(
store.engine(),
- header.ty().unwrap(),
+ ty,
)));
}
- unreachable!("no other kinds of `eqref`s")
+ bail_bug!("no other kinds of `eqref`s")
}
/// Does this `eqref` match the given type?
@@ -350,7 +355,7 @@ impl EqRef {
Ok(!gc_ref.is_i31()
&& store
.require_gc_store()?
- .kind(gc_ref)
+ .kind(gc_ref)?
.matches(VMGcKind::StructRef))
}
@@ -418,7 +423,7 @@ impl EqRef {
Ok(!gc_ref.is_i31()
&& store
.require_gc_store()?
- .kind(gc_ref)
+ .kind(gc_ref)?
.matches(VMGcKind::ArrayRef))
}
diff --git a/crates/wasmtime/src/runtime/gc/enabled/exnref.rs b/crates/wasmtime/src/runtime/gc/enabled/exnref.rs
index 4374492cf7..a66bec616f 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/exnref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/exnref.rs
@@ -6,8 +6,8 @@ use crate::store::{Asyncness, StoreId, StoreResourceLimiter};
use crate::vm::VMStore;
use crate::vm::{self, VMExnRef, VMGcHeader};
use crate::{
- AsContext, AsContextMut, GcRefImpl, GcRootIndex, HeapType, OwnedRooted, RefType, Result,
- Rooted, Val, ValRaw, ValType, WasmTy,
+ AsContext, AsContextMut, GcRefImpl, GcRootIndex, HeapType, OwnedRooted, RefType, Rooted, Val,
+ ValRaw, ValType, WasmTy,
store::{AutoAssertNoGc, StoreOpaque},
};
use crate::{ExnType, FieldType, GcHeapOutOfMemory, StoreContextMut, Tag, prelude::*};
@@ -360,7 +360,7 @@ impl ExnRef {
})() {
Ok(()) => Ok(Rooted::new(&mut store, exnref.into())),
Err(e) => {
- store.require_gc_store_mut()?.dealloc_uninit_exn(exnref);
+ store.require_gc_store_mut()?.dealloc_uninit_exn(exnref)?;
Err(e)
}
}
@@ -368,7 +368,7 @@ impl ExnRef {
pub(crate) fn type_index(&self, store: &StoreOpaque) -> Result<VMSharedTypeIndex> {
let gc_ref = self.inner.try_gc_ref(store)?;
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
debug_assert!(header.kind().matches(VMGcKind::ExnRef));
Ok(header.ty().expect("exnrefs should have concrete types"))
}
@@ -386,8 +386,8 @@ impl ExnRef {
debug_assert!(
store
.unwrap_gc_store()
- .header(&gc_ref)
- .kind()
+ .kind(&gc_ref)
+ .unwrap()
.matches(VMGcKind::ExnRef)
);
Rooted::new(store, gc_ref)
@@ -420,7 +420,9 @@ impl ExnRef {
let raw = if gc_ref.is_i31() {
gc_ref.as_raw_non_zero_u32()
} else {
- store.require_gc_store_mut()?.expose_gc_ref_to_wasm(gc_ref)
+ store
+ .require_gc_store_mut()?
+ .expose_gc_ref_to_wasm(gc_ref)?
};
Ok(raw.get())
}
@@ -500,7 +502,7 @@ impl ExnRef {
let store = AutoAssertNoGc::new(store);
let gc_ref = self.inner.try_gc_ref(&store)?;
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
debug_assert!(header.kind().matches(VMGcKind::ExnRef));
let index = header.ty().expect("exnrefs should have concrete types");
@@ -532,7 +534,7 @@ impl ExnRef {
return None;
}
self.index += 1;
- Some(self.exnref._field(&mut self.store, i).unwrap())
+ self.exnref._field(&mut self.store, i).ok()
}
#[inline]
@@ -553,7 +555,7 @@ impl ExnRef {
fn header<'a>(&self, store: &'a AutoAssertNoGc<'_>) -> Result<&'a VMGcHeader> {
assert!(self.comes_from_same_store(&store));
let gc_ref = self.inner.try_gc_ref(store)?;
- Ok(store.require_gc_store()?.header(gc_ref))
+ Ok(store.require_gc_store()?.header(gc_ref)?)
}
fn exnref<'a>(&self, store: &'a AutoAssertNoGc<'_>) -> Result<&'a VMExnRef> {
@@ -608,7 +610,7 @@ impl ExnRef {
let exnref = self.exnref(store)?.unchecked_copy();
let field_ty = self.field_ty(store, index)?;
let layout = self.layout(store)?;
- Ok(exnref.read_field(store, &layout, field_ty.element_type(), index))
+ exnref.read_field(store, &layout, field_ty.element_type(), index)
}
/// Get this exception object's associated tag.
diff --git a/crates/wasmtime/src/runtime/gc/enabled/externref.rs b/crates/wasmtime/src/runtime/gc/enabled/externref.rs
index 17788ca4e2..c56378ede1 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/externref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/externref.rs
@@ -1,6 +1,6 @@
//! Implementation of `externref` in Wasmtime.
-use super::{AnyRef, RootedGcRefImpl};
+use super::AnyRef;
use crate::prelude::*;
use crate::runtime::vm::{self, VMGcRef};
use crate::store::Asyncness;
@@ -471,7 +471,7 @@ impl ExternRef {
}
let gc_store = store.require_gc_store()?;
if let Some(externref) = gc_ref.as_externref(&*gc_store.gc_heap) {
- Ok(Some(gc_store.externref_host_data(externref)))
+ Ok(Some(gc_store.externref_host_data(externref)?))
} else {
Ok(None)
}
@@ -525,7 +525,7 @@ impl ExternRef {
}
let gc_store = store.require_gc_store_mut()?;
if let Some(externref) = gc_ref.as_externref(&*gc_store.gc_heap) {
- Ok(Some(gc_store.externref_host_data_mut(externref)))
+ Ok(Some(gc_store.externref_host_data_mut(externref)?))
} else {
Ok(None)
}
@@ -593,7 +593,7 @@ impl ExternRef {
pub(crate) fn _to_raw(&self, store: &mut AutoAssertNoGc) -> Result<u32> {
let gc_ref = self.inner.try_clone_gc_ref(store)?;
- let raw = store.unwrap_gc_store_mut().expose_gc_ref_to_wasm(gc_ref);
+ let raw = store.unwrap_gc_store_mut().expose_gc_ref_to_wasm(gc_ref)?;
Ok(raw.get())
}
}
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index 87364fdea3..afaebbf681 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -1213,7 +1213,7 @@ impl<T: GcRef> Rooted<T> {
let gc_ref = self.inner.try_clone_gc_ref(store)?;
let raw = match store.optional_gc_store_mut() {
- Some(s) => s.expose_gc_ref_to_wasm(gc_ref),
+ Some(s) => s.expose_gc_ref_to_wasm(gc_ref)?,
None => {
// NB: do not force the allocation of a GC heap just because the
// program is using `i31ref`s.
@@ -1281,6 +1281,10 @@ impl<T: GcRef> Rooted<T> {
pub(crate) fn try_gc_ref<'a>(&self, store: &'a StoreOpaque) -> Result<&'a VMGcRef> {
<Self as RootedGcRefImpl<_>>::try_gc_ref(self, store)
}
+
+ pub(crate) fn try_clone_gc_ref(&self, store: &mut AutoAssertNoGc<'_>) -> Result<VMGcRef> {
+ <Self as RootedGcRefImpl<_>>::try_clone_gc_ref(self, store)
+ }
}
/// Nested rooting scopes.
@@ -1833,7 +1837,7 @@ where
let gc_ref = self.try_clone_gc_ref(store)?;
let raw = match store.optional_gc_store_mut() {
- Some(s) => s.expose_gc_ref_to_wasm(gc_ref),
+ Some(s) => s.expose_gc_ref_to_wasm(gc_ref)?,
None => {
debug_assert!(gc_ref.is_i31());
gc_ref.as_raw_non_zero_u32()
diff --git a/crates/wasmtime/src/runtime/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/gc/enabled/structref.rs
index 74294788cc..379c57165a 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/structref.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/structref.rs
@@ -363,7 +363,7 @@ impl StructRef {
Err(e) => {
store
.require_gc_store_mut()?
- .dealloc_uninit_struct(structref);
+ .dealloc_uninit_struct(structref)?;
Err(e)
}
}
@@ -453,7 +453,7 @@ impl StructRef {
let store = AutoAssertNoGc::new(store);
let gc_ref = self.inner.try_gc_ref(&store)?;
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
debug_assert!(header.kind().matches(VMGcKind::StructRef));
let index = header.ty().expect("structrefs should have concrete types");
@@ -485,7 +485,7 @@ impl StructRef {
return None;
}
self.index += 1;
- Some(self.structref._field(&mut self.store, i).unwrap())
+ self.structref._field(&mut self.store, i).ok()
}
#[inline]
@@ -506,7 +506,7 @@ impl StructRef {
fn header<'a>(&self, store: &'a AutoAssertNoGc<'_>) -> Result<&'a VMGcHeader> {
assert!(self.comes_from_same_store(&store));
let gc_ref = self.inner.try_gc_ref(store)?;
- Ok(store.require_gc_store()?.header(gc_ref))
+ Ok(store.require_gc_store()?.header(gc_ref)?)
}
fn structref<'a>(&self, store: &'a AutoAssertNoGc<'_>) -> Result<&'a VMStructRef> {
@@ -564,7 +564,7 @@ impl StructRef {
let structref = self.structref(store)?.unchecked_copy();
let field_ty = self.field_ty(store, index)?;
let layout = self.layout(store)?;
- Ok(structref.read_field(store, &layout, field_ty.element_type(), index))
+ structref.read_field(store, &layout, field_ty.element_type(), index)
}
/// Set this struct's `index`th field.
@@ -617,7 +617,7 @@ impl StructRef {
pub(crate) fn type_index(&self, store: &StoreOpaque) -> Result<VMSharedTypeIndex> {
let gc_ref = self.inner.try_gc_ref(store)?;
- let header = store.require_gc_store()?.header(gc_ref);
+ let header = store.require_gc_store()?.header(gc_ref)?;
debug_assert!(header.kind().matches(VMGcKind::StructRef));
Ok(header.ty().expect("structrefs should have concrete types"))
}
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index 72af9ad1b5..c099fc2d37 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1969,18 +1969,23 @@ impl StoreOpaque {
&mut self,
dest: &mut MaybeUninit<Option<VMGcRef>>,
gc_ref: Option<&VMGcRef>,
- ) {
+ ) -> Result<()> {
if GcStore::needs_init_barrier(gc_ref) {
self.unwrap_gc_store_mut().init_gc_ref(dest, gc_ref)
} else {
dest.write(gc_ref.map(|r| r.copy_i31()));
+ Ok(())
}
}
/// Helper function execute a write barrier when placing `gc_ref` in `dest`.
///
/// This avoids allocating `GcStore` where possible.
- pub(crate) fn write_gc_ref(&mut self, dest: &mut Option<VMGcRef>, gc_ref: Option<&VMGcRef>) {
+ pub(crate) fn write_gc_ref(
+ &mut self,
+ dest: &mut Option<VMGcRef>,
+ gc_ref: Option<&VMGcRef>,
+ ) -> Result<()> {
GcStore::write_gc_ref_optional_store(self.optional_gc_store_mut(), dest, gc_ref)
}
@@ -2407,6 +2412,11 @@ at https://bytecodealliance.org/security.
// otherwise fall back to the runtime-agnostic code.
yield_now().await
}
+
+ #[cfg(not(feature = "gc"))]
+ pub(crate) fn require_gc_store_mut(&mut self) -> Result<&mut GcStore> {
+ bail!("GC is disabled")
+ }
}
#[cfg(any(feature = "async", feature = "gc"))]
diff --git a/crates/wasmtime/src/runtime/store/async_.rs b/crates/wasmtime/src/runtime/store/async_.rs
index d28a3fd19f..09a98b24c5 100644
--- a/crates/wasmtime/src/runtime/store/async_.rs
+++ b/crates/wasmtime/src/runtime/store/async_.rs
@@ -97,7 +97,7 @@ impl<T> Store<T> {
///
/// This method is only available when the `gc` Cargo feature is enabled.
#[cfg(feature = "gc")]
- pub async fn gc_async(&mut self, why: Option<&crate::GcHeapOutOfMemory<()>>)
+ pub async fn gc_async(&mut self, why: Option<&crate::GcHeapOutOfMemory<()>>) -> Result<()>
where
T: Send,
{
@@ -140,7 +140,7 @@ impl<'a, T> StoreContextMut<'a, T> {
///
/// This method is only available when the `gc` Cargo feature is enabled.
#[cfg(feature = "gc")]
- pub async fn gc_async(&mut self, why: Option<&crate::GcHeapOutOfMemory<()>>)
+ pub async fn gc_async(&mut self, why: Option<&crate::GcHeapOutOfMemory<()>>) -> Result<()>
where
T: Send + 'static,
{
@@ -152,7 +152,8 @@ impl<'a, T> StoreContextMut<'a, T> {
why.map(|e| e.bytes_needed()),
crate::store::Asyncness::Yes,
)
- .await;
+ .await?;
+ Ok(())
}
/// Configures epoch-deadline expiration to yield to the async
diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs
index 9b8eec02b4..320824211e 100644
--- a/crates/wasmtime/src/runtime/store/gc.rs
+++ b/crates/wasmtime/src/runtime/store/gc.rs
@@ -1,5 +1,6 @@
//! GC-related methods for stores.
+use crate::error::Context;
use crate::store::{
Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter, yield_now,
};
@@ -7,8 +8,8 @@ use crate::type_registry::RegisteredType;
use crate::vm::{self, Backtrace, Frame, GcRootsList, GcStore, SendSyncPtr, VMGcRef};
use crate::{
ExnRef, GcHeapOutOfMemory, Result, Rooted, Store, StoreContextMut, ThrownException, bail,
- format_err,
};
+use core::fmt;
use core::mem::ManuallyDrop;
use core::num::NonZeroU32;
use core::ops::{Deref, DerefMut};
@@ -110,7 +111,7 @@ impl<'a, T> StoreContextMut<'a, T> {
None,
why.map(|e| e.bytes_needed()),
Asyncness::No,
- ));
+ ))?;
Ok(())
}
@@ -133,6 +134,17 @@ impl<'a, T> StoreContextMut<'a, T> {
}
}
+#[derive(Debug)]
+struct GcHeapGrowthFailed;
+
+impl fmt::Display for GcHeapGrowthFailed {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str("GC heap growth failed")
+ }
+}
+
+impl core::error::Error for GcHeapGrowthFailed {}
+
impl StoreOpaque {
/// Perform any growth or GC needed to allocate `bytes_needed` bytes.
///
@@ -150,7 +162,7 @@ impl StoreOpaque {
root: Option<VMGcRef>,
bytes_needed: Option<u64>,
asyncness: Asyncness,
- ) -> Option<VMGcRef> {
+ ) -> Result<Option<VMGcRef>> {
let mut scope = crate::OpaqueRootScope::new(self);
scope.trim_gc_liveness_flags(true);
let store_id = scope.id();
@@ -158,15 +170,15 @@ impl StoreOpaque {
scope
.collect_and_maybe_grow_gc_heap(limiter, bytes_needed, asyncness)
- .await;
+ .await?;
- root.map(|r| {
+ Ok(root.map(|r| {
let r = r
.get_gc_ref(&scope)
.expect("still in scope")
.unchecked_copy();
scope.clone_gc_ref(&r)
- })
+ }))
}
// This lives on the Store because it must simultaneously borrow
@@ -186,18 +198,25 @@ impl StoreOpaque {
limiter: Option<&mut StoreResourceLimiter<'_>>,
bytes_needed: Option<u64>,
asyncness: Asyncness,
- ) {
+ ) -> Result<()> {
log::trace!("collect_and_maybe_grow_gc_heap(bytes_needed = {bytes_needed:#x?})");
- self.do_gc(asyncness).await;
+ self.do_gc(asyncness).await?;
if let Some(n) = bytes_needed
- && n > u64::try_from(self.gc_heap_capacity())
- .unwrap()
- .saturating_sub(self.gc_store.as_ref().map_or(0, |gc| {
+ && n > u64::try_from(self.gc_heap_capacity())?.saturating_sub(
+ self.gc_store.as_ref().map_or(0, |gc| {
u64::try_from(gc.last_post_gc_allocated_bytes.unwrap_or(0)).unwrap()
- }))
+ }),
+ )
{
- let _ = self.grow_gc_heap(limiter, n, asyncness).await;
+ if let Err(e) = self.grow_gc_heap(limiter, n, asyncness).await {
+ if e.is::<GcHeapGrowthFailed>() {
+ log::trace!("ignoring GC heap growth failure: {e}");
+ } else {
+ return Err(e);
+ }
+ }
}
+ Ok(())
}
/// Attempt to grow the GC heap by `bytes_needed` bytes.
@@ -222,7 +241,7 @@ impl StoreOpaque {
.as_ref()
.map_or(false, |gc| gc.gc_heap.needs_gc_before_next_growth())
{
- self.do_gc(asyncness).await;
+ self.do_gc(asyncness).await?;
debug_assert!(
!self
.gc_store
@@ -238,7 +257,7 @@ impl StoreOpaque {
// grow it, then replace it.
let mut heap = TakenGcHeap::new(self);
- let current_size_in_bytes = u64::try_from(heap.memory.byte_size()).unwrap();
+ let current_size_in_bytes = u64::try_from(heap.memory.byte_size())?;
let current_size_in_pages = current_size_in_bytes / page_size;
// Aim to double the heap size, amortizing the cost of growth.
@@ -273,12 +292,13 @@ impl StoreOpaque {
unsafe {
heap.memory
.grow(delta_pages_for_alloc, limiter)
- .await?
- .ok_or_else(|| format_err!("failed to grow GC heap"))?;
+ .await
+ .context(GcHeapGrowthFailed)?
+ .ok_or(GcHeapGrowthFailed)?;
}
*heap.store.vm_store_context.gc_heap.get_mut() = heap.memory.vmmemory();
- let new_size_in_bytes = u64::try_from(heap.memory.byte_size()).unwrap();
+ let new_size_in_bytes = u64::try_from(heap.memory.byte_size())?;
assert!(new_size_in_bytes > current_size_in_bytes);
heap.delta_bytes_grown = new_size_in_bytes - current_size_in_bytes;
let delta_bytes_for_alloc = delta_pages_for_alloc.checked_mul(page_size).unwrap();
@@ -383,7 +403,7 @@ impl StoreOpaque {
);
store
.gc(limiter.as_deref_mut(), None, None, asyncness)
- .await;
+ .await?;
match alloc_func(&mut store, value) {
Ok(x) => Ok(x),
@@ -413,7 +433,7 @@ impl StoreOpaque {
.await
{
log::trace!("growing GC heap failed: {e}");
- store.gc(limiter, None, None, asyncness).await;
+ store.gc(limiter, None, None, asyncness).await?;
}
alloc_func(&mut store, value)
@@ -431,11 +451,13 @@ impl StoreOpaque {
/// the TLS call state; that must be done separately.
///
/// GC barriers are not required by the caller of this function.
- pub(crate) fn set_pending_exception(&mut self, exnref: &VMGcRef) -> ThrownException {
+ pub(crate) fn set_pending_exception(&mut self, exnref: &VMGcRef) -> crate::Error {
debug_assert!(exnref.is_exnref(&*self.unwrap_gc_store_mut().gc_heap));
let gc_store = self.gc_store.as_mut().unwrap();
- gc_store.write_gc_ref(&mut self.pending_exception, Some(exnref));
- ThrownException
+ match gc_store.write_gc_ref(&mut self.pending_exception, Some(exnref)) {
+ Ok(()) => ThrownException.into(),
+ Err(e) => e,
+ }
}
/// Takes the pending exception from this store, if any, and exposes it to
@@ -444,7 +466,7 @@ impl StoreOpaque {
let exnref = self.pending_exception.take()?;
let gc_store = self.unwrap_gc_store_mut();
debug_assert!(exnref.is_exnref(&*gc_store.gc_heap));
- Some(gc_store.expose_gc_ref_to_wasm(exnref))
+ Some(gc_store.expose_gc_ref_to_wasm(exnref).unwrap())
}
/// Takes the pending exception of the store, yielding ownership of its
@@ -464,12 +486,14 @@ impl StoreOpaque {
let pending_exnref = self.pending_exception.as_ref()?.unchecked_copy();
debug_assert!(pending_exnref.is_exnref(&*self.unwrap_gc_store_mut().gc_heap));
let mut store = AutoAssertNoGc::new(self);
- Some(
- pending_exnref
- .into_exnref_unchecked()
- .tag(&mut store)
- .expect("cannot read tag"),
- )
+
+ // Note that if the GC heap is corrupt this will return an error, and in
+ // such as situation we return `None` here pretending that there's no
+ // pending exception. This defers the GC heap corruption to get detected
+ // later. This method is primarily called right now to determine if
+ // there's a handler for an exception, and by returning `None` here this
+ // turns into just any old embedder error.
+ pending_exnref.into_exnref_unchecked().tag(&mut store).ok()
}
/// Get an owned rooted reference to the pending exception,
@@ -493,7 +517,7 @@ impl StoreOpaque {
/// exception pointer.
fn throw_impl<R>(&mut self, exception: Rooted<ExnRef>) -> Result<R> {
let exception = exception.try_gc_ref(self)?.unchecked_copy();
- Err(self.set_pending_exception(&exception).into())
+ Err(self.set_pending_exception(&exception))
}
/// Helper method to require that a `GcStore` was previously allocated for
@@ -535,10 +559,10 @@ impl StoreOpaque {
}
}
- async fn do_gc(&mut self, asyncness: Asyncness) {
+ async fn do_gc(&mut self, asyncness: Asyncness) -> Result<()> {
// If the GC heap hasn't been initialized, there is nothing to collect.
if self.gc_store.is_none() {
- return;
+ return Ok(());
}
log::trace!("============ Begin GC ===========");
@@ -558,13 +582,14 @@ impl StoreOpaque {
// fall back to the runtime-agnostic code.
yield_now,
)
- .await;
+ .await?;
// Restore the GC roots for the next GC.
roots.clear();
self.gc_roots_list = roots;
log::trace!("============ End GC ===========");
+ Ok(())
}
async fn trace_roots(&mut self, gc_roots_list: &mut GcRootsList, asyncness: Asyncness) {
diff --git a/crates/wasmtime/src/runtime/trampoline/global.rs b/crates/wasmtime/src/runtime/trampoline/global.rs
index 0670de188a..b1ac7376c6 100644
--- a/crates/wasmtime/src/runtime/trampoline/global.rs
+++ b/crates/wasmtime/src/runtime/trampoline/global.rs
@@ -52,7 +52,7 @@ pub fn generate_global_export(
Some(x) => Some(x.try_gc_ref(&store)?.unchecked_copy()),
};
let new = new.as_ref();
- global.write_gc_ref(&mut store, new);
+ global.write_gc_ref(&mut store, new)?;
}
Val::AnyRef(a) => {
let new = match a {
@@ -60,7 +60,7 @@ pub fn generate_global_export(
Some(a) => Some(a.try_gc_ref(&store)?.unchecked_copy()),
};
let new = new.as_ref();
- global.write_gc_ref(&mut store, new);
+ global.write_gc_ref(&mut store, new)?;
}
Val::ExnRef(e) => {
let new = match e {
@@ -68,11 +68,11 @@ pub fn generate_global_export(
Some(e) => Some(e.try_gc_ref(&store)?.unchecked_copy()),
};
let new = new.as_ref();
- global.write_gc_ref(&mut store, new);
+ global.write_gc_ref(&mut store, new)?;
}
Val::ContRef(None) => {
// Allow null continuation references for trampoline globals - these are just placeholders
- global.write_gc_ref(&mut store, None);
+ global.write_gc_ref(&mut store, None)?;
}
Val::ContRef(Some(_)) => {
// TODO(#10248): Implement non-null trampoline continuation reference handling
diff --git a/crates/wasmtime/src/runtime/vm/gc.rs b/crates/wasmtime/src/runtime/vm/gc.rs
index a4501f675b..e62a4dedee 100644
--- a/crates/wasmtime/src/runtime/vm/gc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc.rs
@@ -8,12 +8,14 @@ mod disabled;
#[cfg(not(feature = "gc"))]
pub use disabled::*;
+mod data;
mod func_ref;
mod gc_ref;
mod gc_runtime;
mod host_data;
mod i31;
+pub use data::*;
pub use func_ref::*;
pub use gc_ref::*;
pub use gc_runtime::*;
@@ -107,24 +109,25 @@ impl GcStore {
asyncness: Asyncness,
roots: GcRootsIter<'_>,
yield_fn: impl AsyncFn(),
- ) {
+ ) -> Result<()> {
let collection = self.gc_heap.gc(roots, &mut self.host_data_table);
- collect_async(collection, asyncness, yield_fn).await;
+ collect_async(collection, asyncness, yield_fn).await?;
self.last_post_gc_allocated_bytes = Some({
let size = self.gc_heap.allocated_bytes();
log::trace!("After collection, GC heap's allocated bytes = {size:#x} bytes");
size
});
+ Ok(())
}
/// Get the kind of the given GC reference.
- pub fn kind(&self, gc_ref: &VMGcRef) -> VMGcKind {
+ pub fn kind(&self, gc_ref: &VMGcRef) -> Result<VMGcKind> {
debug_assert!(!gc_ref.is_i31());
- self.header(gc_ref).kind()
+ Ok(self.header(gc_ref)?.kind())
}
/// Get the header of the given GC reference.
- pub fn header(&self, gc_ref: &VMGcRef) -> &VMGcHeader {
+ pub fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader> {
debug_assert!(!gc_ref.is_i31());
self.gc_heap.header(gc_ref)
}
@@ -144,11 +147,11 @@ impl GcStore {
&mut self,
destination: &mut MaybeUninit<Option<VMGcRef>>,
source: Option<&VMGcRef>,
- ) {
+ ) -> Result<()> {
// Initialize the destination to `None`, at which point the regular GC
// write barrier is safe to reuse.
let destination = destination.write(None);
- self.write_gc_ref(destination, source);
+ self.write_gc_ref(destination, source)
}
/// Dynamically tests whether a `init_gc_ref` is needed to write `gc_ref`
@@ -180,26 +183,32 @@ impl GcStore {
store: Option<&mut Self>,
dest: &mut Option<VMGcRef>,
gc_ref: Option<&VMGcRef>,
- ) {
+ ) -> Result<()> {
if Self::needs_write_barrier(dest, gc_ref) {
store.unwrap().write_gc_ref(dest, gc_ref)
} else {
*dest = gc_ref.map(|r| r.copy_i31());
+ Ok(())
}
}
/// Write the `source` GC reference into the `destination` slot, performing
/// write barriers as necessary.
- pub fn write_gc_ref(&mut self, destination: &mut Option<VMGcRef>, source: Option<&VMGcRef>) {
+ pub fn write_gc_ref(
+ &mut self,
+ destination: &mut Option<VMGcRef>,
+ source: Option<&VMGcRef>,
+ ) -> Result<()> {
// If neither the source nor destination actually point to a GC object
// (that is, they are both either null or `i31ref`s) then we can skip
// the GC barrier.
if Self::needs_write_barrier(destination, source) {
self.gc_heap
- .write_gc_ref(&mut self.host_data_table, destination, source);
+ .write_gc_ref(&mut self.host_data_table, destination, source)?;
} else {
*destination = source.map(|s| s.copy_i31());
}
+ Ok(())
}
/// Drop the given GC reference, performing drop barriers as necessary.
@@ -214,13 +223,13 @@ impl GcStore {
/// Returns the raw representation of this GC ref, ready to be passed to
/// Wasm.
#[must_use]
- pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) -> NonZeroU32 {
+ pub fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) -> Result<NonZeroU32> {
let raw = gc_ref.as_raw_non_zero_u32();
if !gc_ref.is_i31() {
log::trace!("exposing GC ref to Wasm: {gc_ref:p}");
- self.gc_heap.expose_gc_ref_to_wasm(gc_ref);
+ self.gc_heap.expose_gc_ref_to_wasm(gc_ref)?;
}
- raw
+ Ok(raw)
}
/// Allocate a new `externref`.
@@ -241,7 +250,7 @@ impl GcStore {
let host_data_id = self.host_data_table.alloc(value);
match self.gc_heap.alloc_externref(host_data_id)? {
Ok(x) => Ok(Ok(x)),
- Err(n) => Ok(Err((self.host_data_table.dealloc(host_data_id), n))),
+ Err(n) => Ok(Err((self.host_data_table.dealloc(host_data_id)?, n))),
}
}
@@ -250,8 +259,8 @@ impl GcStore {
/// Passing invalid `VMExternRef`s (eg garbage values or `externref`s
/// associated with a different heap is memory safe but will lead to general
/// incorrectness such as panics and wrong results.
- pub fn externref_host_data(&self, externref: &VMExternRef) -> &(dyn Any + Send + Sync) {
- let host_data_id = self.gc_heap.externref_host_data(externref);
+ pub fn externref_host_data(&self, externref: &VMExternRef) -> Result<&(dyn Any + Send + Sync)> {
+ let host_data_id = self.gc_heap.externref_host_data(externref)?;
self.host_data_table.get(host_data_id)
}
@@ -263,8 +272,8 @@ impl GcStore {
pub fn externref_host_data_mut(
&mut self,
externref: &VMExternRef,
- ) -> &mut (dyn Any + Send + Sync) {
- let host_data_id = self.gc_heap.externref_host_data(externref);
+ ) -> Result<&mut (dyn Any + Send + Sync)> {
+ let host_data_id = self.gc_heap.externref_host_data(externref)?;
self.host_data_table.get_mut(host_data_id)
}
@@ -313,14 +322,14 @@ impl GcStore {
}
/// Deallocate an uninitialized struct.
- pub fn dealloc_uninit_struct(&mut self, structref: VMStructRef) {
+ pub fn dealloc_uninit_struct(&mut self, structref: VMStructRef) -> Result<()> {
self.gc_heap.dealloc_uninit_struct_or_exn(structref.into())
}
/// Get the data for the given object reference.
///
/// Panics when the structref and its size is out of the GC heap bounds.
- pub fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> &mut VMGcObjectData {
+ pub fn gc_object_data(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcObjectData> {
self.gc_heap.gc_object_data_mut(gc_ref)
}
@@ -340,12 +349,12 @@ impl GcStore {
}
/// Deallocate an uninitialized array.
- pub fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef) {
- self.gc_heap.dealloc_uninit_array(arrayref);
+ pub fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef) -> Result<()> {
+ self.gc_heap.dealloc_uninit_array(arrayref)
}
/// Get the length of the given array.
- pub fn array_len(&self, arrayref: &VMArrayRef) -> u32 {
+ pub fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32> {
self.gc_heap.array_len(arrayref)
}
@@ -367,8 +376,8 @@ impl GcStore {
}
/// Deallocate an uninitialized exception object.
- pub fn dealloc_uninit_exn(&mut self, exnref: VMExnRef) {
- self.gc_heap.dealloc_uninit_struct_or_exn(exnref.into());
+ pub fn dealloc_uninit_exn(&mut self, exnref: VMExnRef) -> Result<()> {
+ self.gc_heap.dealloc_uninit_struct_or_exn(exnref.into())
}
#[cfg(feature = "gc")]
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs b/crates/wasmtime/src/runtime/vm/gc/data.rs
similarity index 65%
rename from crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
rename to crates/wasmtime/src/runtime/vm/gc/data.rs
index 21af992351..56a1223993 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/data.rs
@@ -1,7 +1,8 @@
//! Unchecked methods for working with the data inside GC objects.
-use crate::V128;
+use crate::{Result, V128, bail_bug};
use core::mem;
+use core::ops::Range;
/// A plain-old-data type that can be stored in a `ValType` or a `StorageType`.
pub trait PodValType<const SIZE: usize>: Copy {
@@ -72,9 +73,9 @@ macro_rules! impl_pod_methods {
#[doc = stringify!($T)]
/// ` field this object.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
#[inline]
- pub fn $read(&self, offset: u32) -> $T
+ pub fn $read(&self, offset: u32) -> Result<$T>
{
self.read_pod::<{ mem::size_of::<$T>() }, $T>(offset)
}
@@ -83,11 +84,11 @@ macro_rules! impl_pod_methods {
#[doc = stringify!($T)]
/// ` into this object.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
#[inline]
- pub fn $write(&mut self, offset: u32, val: $T)
+ pub fn $write(&mut self, offset: u32, val: $T) -> Result<()>
{
- self.write_pod::<{ mem::size_of::<$T>() }, $T>(offset, val);
+ self.write_pod::<{ mem::size_of::<$T>() }, $T>(offset, val)
}
)*
};
@@ -122,44 +123,46 @@ impl VMGcObjectData {
/// Read a POD field out of this object.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
///
/// Don't generally use this method, use `read_u8`, `read_i64`,
/// etc... instead.
#[inline]
- fn read_pod<const N: usize, T>(&self, offset: u32) -> T
+ fn read_pod<const N: usize, T>(&self, offset: u32) -> Result<T>
where
T: PodValType<N>,
{
assert_eq!(N, mem::size_of::<T>());
- let offset = usize::try_from(offset).unwrap();
- let end = offset.checked_add(N).unwrap();
- let bytes = self.data.get(offset..end).expect("out of bounds field");
- T::read_le(bytes.as_array().unwrap())
+ let offset = usize::try_from(offset)?;
+ let bytes = match self.data.get(offset..).and_then(|s| s.first_chunk()) {
+ Some(data) => data,
+ None => bail_bug!("out of bounds field"),
+ };
+ Ok(T::read_le(bytes))
}
/// Read a POD field out of this object.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
///
/// Don't generally use this method, use `write_u8`, `write_i64`,
/// etc... instead.
#[inline]
- fn write_pod<const N: usize, T>(&mut self, offset: u32, val: T)
+ fn write_pod<const N: usize, T>(&mut self, offset: u32, val: T) -> Result<()>
where
T: PodValType<N>,
{
assert_eq!(N, mem::size_of::<T>());
- let offset = usize::try_from(offset).unwrap();
- let end = offset.checked_add(N).unwrap();
- let into = match self.data.get_mut(offset..end) {
+ let offset = usize::try_from(offset)?;
+ let into = match self
+ .data
+ .get_mut(offset..)
+ .and_then(|s| s.first_chunk_mut())
+ {
Some(into) => into,
- None => panic!(
- "out of bounds field! field range = {offset:#x}..{end:#x}; object len = {:#x}",
- self.data.as_mut().len(),
- ),
+ None => bail_bug!("out of bounds field"),
};
- val.write_le(into.as_mut_array().unwrap());
+ Ok(val.write_le(into))
}
/// Get a slice of this object's data.
@@ -167,13 +170,15 @@ impl VMGcObjectData {
/// Note that GC data is always stored in little-endian order, and this
/// method does not do any conversions to/from host endianness for you.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
#[inline]
- pub fn slice(&self, offset: u32, len: u32) -> &[u8] {
- let start = usize::try_from(offset).unwrap();
- let len = usize::try_from(len).unwrap();
- let end = start.checked_add(len).unwrap();
- self.data.get(start..end).expect("out of bounds slice")
+ pub fn slice(&self, offset: u32, len: u32) -> Result<&[u8]> {
+ let start = usize::try_from(offset)?;
+ let len = usize::try_from(len)?;
+ match self.data.get(start..).and_then(|s| s.get(..len)) {
+ Some(slice) => Ok(slice),
+ None => bail_bug!("out of bounds slice"),
+ }
}
/// Get a mutable slice of this object's data.
@@ -181,13 +186,15 @@ impl VMGcObjectData {
/// Note that GC data is always stored in little-endian order, and this
/// method does not do any conversions to/from host endianness for you.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
#[inline]
- pub fn slice_mut(&mut self, offset: u32, len: u32) -> &mut [u8] {
- let start = usize::try_from(offset).unwrap();
- let len = usize::try_from(len).unwrap();
- let end = start.checked_add(len).unwrap();
- self.data.get_mut(start..end).expect("out of bounds slice")
+ pub fn slice_mut(&mut self, offset: u32, len: u32) -> Result<&mut [u8]> {
+ let start = usize::try_from(offset)?;
+ let len = usize::try_from(len)?;
+ match self.data.get_mut(start..).and_then(|s| s.get_mut(..len)) {
+ Some(slice) => Ok(slice),
+ None => bail_bug!("out of bounds slice"),
+ }
}
/// Copy the given slice into this object's data at the given offset.
@@ -195,13 +202,20 @@ impl VMGcObjectData {
/// Note that GC data is always stored in little-endian order, and this
/// method does not do any conversions to/from host endianness for you.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
#[inline]
- pub fn copy_from_slice(&mut self, offset: u32, src: &[u8]) {
- let offset = usize::try_from(offset).unwrap();
- let end = offset.checked_add(src.len()).unwrap();
- let into = self.data.get_mut(offset..end).expect("out of bounds copy");
+ pub fn copy_from_slice(&mut self, offset: u32, src: &[u8]) -> Result<()> {
+ let offset = usize::try_from(offset)?;
+ let into = match self
+ .data
+ .get_mut(offset..)
+ .and_then(|s| s.get_mut(..src.len()))
+ {
+ Some(into) => into,
+ None => bail_bug!("out of bounds copy"),
+ };
into.copy_from_slice(src);
+ Ok(())
}
/// Copy within this this object's data.
@@ -209,16 +223,23 @@ impl VMGcObjectData {
/// Note that GC data is always stored in little-endian order, and this
/// method does not do any conversions to/from host endianness for you.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
#[inline]
- pub fn copy_within<R>(&mut self, src: R, dest: u32)
- where
- R: core::ops::RangeBounds<u32>,
- {
- let start = src.start_bound().map(|s| usize::try_from(*s).unwrap());
- let end = src.end_bound().map(|e| usize::try_from(*e).unwrap());
- let dest = usize::try_from(dest).unwrap();
- self.data.copy_within((start, end), dest);
+ pub fn copy_within(&mut self, src: Range<u32>, dest: u32) -> Result<()> {
+ let start = usize::try_from(src.start)?;
+ let end = usize::try_from(src.end)?;
+ let dest = usize::try_from(dest)?;
+ if self.data.get(start..end).is_none()
+ || self
+ .data
+ .get(dest..)
+ .and_then(|s| s.get(..src.len()))
+ .is_none()
+ {
+ bail_bug!("invalid copy range");
+ }
+ self.data.copy_within(start..end, dest);
+ Ok(())
}
impl_pod_methods! {
diff --git a/crates/wasmtime/src/runtime/vm/gc/disabled.rs b/crates/wasmtime/src/runtime/vm/gc/disabled.rs
index 53717b31b7..4435d76796 100644
--- a/crates/wasmtime/src/runtime/vm/gc/disabled.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/disabled.rs
@@ -10,11 +10,6 @@ pub enum VMArrayRef {}
pub enum VMExnRef {}
-pub struct VMGcObjectData {
- _inner: VMStructRef,
- _phantom: core::marker::PhantomData<[u8]>,
-}
-
impl VMGcRef {
pub fn into_structref_unchecked(self) -> VMStructRef {
unreachable!()
@@ -36,15 +31,3 @@ impl From<VMExnRef> for VMGcRef {
match e {}
}
}
-
-impl<'a> From<&'a [u8]> for &'a VMGcObjectData {
- fn from(_: &'a [u8]) -> Self {
- unreachable!()
- }
-}
-
-impl<'a> From<&'a mut [u8]> for &'a mut VMGcObjectData {
- fn from(_: &'a mut [u8]) -> Self {
- unreachable!()
- }
-}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled.rs b/crates/wasmtime/src/runtime/vm/gc/enabled.rs
index 2eb3d29969..478cd85fec 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled.rs
@@ -1,7 +1,6 @@
//! Implementation of garbage collection and GC types in Wasmtime.
mod arrayref;
-mod data;
mod exnref;
mod externref;
#[cfg(feature = "gc-drc")]
@@ -11,7 +10,6 @@ mod structref;
mod trace_info;
pub use arrayref::*;
-pub use data::*;
pub use exnref::*;
pub use externref::*;
pub use structref::*;
@@ -30,20 +28,3 @@ pub use null::*;
mod copying;
#[cfg(feature = "gc-copying")]
pub use copying::*;
-
-// Explicit methods to clearly indicate that truncation is desired when used.
-#[expect(
- clippy::cast_possible_truncation,
- reason = "that's the purpose of this method"
-)]
-fn truncate_i32_to_i16(a: i32) -> i16 {
- a as i16
-}
-
-#[expect(
- clippy::cast_possible_truncation,
- reason = "that's the purpose of this method"
-)]
-fn truncate_i32_to_i8(a: i32) -> i8 {
- a as i8
-}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs
index 2cd708b3c0..f7354bd1c3 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs
@@ -1,10 +1,8 @@
-use super::{truncate_i32_to_i8, truncate_i32_to_i16};
use crate::{
- AnyRef, ExnRef, ExternRef, Func, HeapType, RootedGcRefImpl, StorageType, Val, ValType,
+ StorageType, Val,
prelude::*,
runtime::vm::{GcHeap, GcStore, VMGcRef},
store::{AutoAssertNoGc, StoreOpaque},
- vm::{FuncRefTableId, SendSyncPtr},
};
use core::fmt;
use wasmtime_environ::{GcArrayLayout, VMGcKind};
@@ -42,8 +40,10 @@ impl VMGcRef {
return false;
}
- let header = gc_heap.header(&self);
- header.kind().matches(VMGcKind::ArrayRef)
+ match gc_heap.header(&self) {
+ Ok(header) => header.kind().matches(VMGcKind::ArrayRef),
+ Err(_) => false,
+ }
}
/// Create a new `VMArrayRef` from the given `gc_ref`.
@@ -124,7 +124,7 @@ impl VMArrayRef {
}
/// Get the length of this array.
- pub fn len(&self, store: &StoreOpaque) -> u32 {
+ pub fn len(&self, store: &StoreOpaque) -> Result<u32> {
store.unwrap_gc_store().array_len(self)
}
@@ -144,44 +144,9 @@ impl VMArrayRef {
layout: &GcArrayLayout,
ty: &StorageType,
index: u32,
- ) -> Val {
+ ) -> Result<Val> {
let offset = layout.elem_offset(index).unwrap();
- let data = store.unwrap_gc_store_mut().gc_object_data(self.as_gc_ref());
- match ty {
- StorageType::I8 => Val::I32(data.read_u8(offset).into()),
- StorageType::I16 => Val::I32(data.read_u16(offset).into()),
- StorageType::ValType(ValType::I32) => Val::I32(data.read_i32(offset)),
- StorageType::ValType(ValType::I64) => Val::I64(data.read_i64(offset)),
- StorageType::ValType(ValType::F32) => Val::F32(data.read_u32(offset)),
- StorageType::ValType(ValType::F64) => Val::F64(data.read_u64(offset)),
- StorageType::ValType(ValType::V128) => Val::V128(data.read_v128(offset)),
- StorageType::ValType(ValType::Ref(r)) => match r.heap_type().top() {
- HeapType::Extern => {
- let raw = data.read_u32(offset);
- Val::ExternRef(ExternRef::_from_raw(store, raw))
- }
- HeapType::Any => {
- let raw = data.read_u32(offset);
- Val::AnyRef(AnyRef::_from_raw(store, raw))
- }
- HeapType::Exn => {
- let raw = data.read_u32(offset);
- Val::ExnRef(ExnRef::_from_raw(store, raw))
- }
- HeapType::Func => {
- let func_ref_id = data.read_u32(offset);
- let func_ref_id = FuncRefTableId::from_raw(func_ref_id);
- let func_ref = store
- .unwrap_gc_store()
- .func_ref_table
- .get_untyped(func_ref_id);
- Val::FuncRef(unsafe {
- func_ref.map(|p| Func::from_vm_func_ref(store.id(), p.as_non_null()))
- })
- }
- otherwise => unreachable!("not a top type: {otherwise:?}"),
- },
- }
+ self.as_gc_ref().read_val(store, ty, offset)
}
/// Write the given value into this array at the given offset.
@@ -206,84 +171,7 @@ impl VMArrayRef {
debug_assert!(val._matches_ty(&store, &ty.unpack())?);
let offset = layout.elem_offset(index).unwrap();
- let data = store.unwrap_gc_store_mut().gc_object_data(self.as_gc_ref());
- match val {
- Val::I32(i) if ty.is_i8() => data.write_i8(offset, truncate_i32_to_i8(i)),
- Val::I32(i) if ty.is_i16() => data.write_i16(offset, truncate_i32_to_i16(i)),
- Val::I32(i) => data.write_i32(offset, i),
- Val::I64(i) => data.write_i64(offset, i),
- Val::F32(f) => data.write_u32(offset, f),
- Val::F64(f) => data.write_u64(offset, f),
- Val::V128(v) => data.write_v128(offset, v),
-
- // For GC-managed references, we need to take care to run the
- // appropriate barriers, even when we are writing null references
- // into the array.
- //
- // POD-read the old value into a local copy, run the GC write
- // barrier on that local copy, and then POD-write the updated
- // value back into the array. This avoids transmuting the inner
- // data, which would probably be fine, but this approach is
- // Obviously Correct and should get us by for now. If LLVM isn't
- // able to elide some of these unnecessary copies, and this
- // method is ever hot enough, we can always come back and clean
- // it up in the future.
- Val::ExternRef(e) => {
- let raw = data.read_u32(offset);
- let mut gc_ref = VMGcRef::from_raw_u32(raw);
- let e = match e {
- Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()),
- None => None,
- };
- let store = store.require_gc_store_mut()?;
- store.write_gc_ref(&mut gc_ref, e.as_ref());
- let data = store.gc_object_data(self.as_gc_ref());
- data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()));
- }
- Val::AnyRef(a) => {
- let raw = data.read_u32(offset);
- let mut gc_ref = VMGcRef::from_raw_u32(raw);
- let a = match a {
- Some(a) => Some(a.try_gc_ref(store)?.unchecked_copy()),
- None => None,
- };
- let store = store.require_gc_store_mut()?;
- store.write_gc_ref(&mut gc_ref, a.as_ref());
- let data = store.gc_object_data(self.as_gc_ref());
- data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()));
- }
- Val::ExnRef(e) => {
- let raw = data.read_u32(offset);
- let mut gc_ref = VMGcRef::from_raw_u32(raw);
- let e = match e {
- Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()),
- None => None,
- };
- let store = store.require_gc_store_mut()?;
- store.write_gc_ref(&mut gc_ref, e.as_ref());
- let data = store.gc_object_data(self.as_gc_ref());
- data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()));
- }
-
- Val::FuncRef(f) => {
- let func_ref = match f {
- Some(f) => Some(SendSyncPtr::new(f.vm_func_ref(store))),
- None => None,
- };
- let store = store.require_gc_store_mut()?;
- let id = unsafe { store.func_ref_table.intern(func_ref) };
- store
- .gc_object_data(self.as_gc_ref())
- .write_u32(offset, id.into_raw());
- }
- Val::ContRef(_) => {
- // TODO(#10248): Implement array continuation reference element handling
- return Err(crate::format_err!(
- "setting continuation references in array elements not yet supported"
- ));
- }
- }
- Ok(())
+ self.as_gc_ref().write_val(store, ty, offset, val)
}
/// Initialize an element in this arrayref that is currently uninitialized.
@@ -320,82 +208,6 @@ impl VMArrayRef {
) -> Result<()> {
debug_assert!(val._matches_ty(&store, &ty.unpack())?);
let offset = layout.elem_offset(index).unwrap();
- let gcstore = store.require_gc_store_mut()?;
- match val {
- Val::I32(i) if ty.is_i8() => gcstore
- .gc_object_data(self.as_gc_ref())
- .write_i8(offset, truncate_i32_to_i8(i)),
- Val::I32(i) if ty.is_i16() => gcstore
- .gc_object_data(self.as_gc_ref())
- .write_i16(offset, truncate_i32_to_i16(i)),
- Val::I32(i) => gcstore
- .gc_object_data(self.as_gc_ref())
- .write_i32(offset, i),
- Val::I64(i) => gcstore
- .gc_object_data(self.as_gc_ref())
- .write_i64(offset, i),
- Val::F32(f) => gcstore
- .gc_object_data(self.as_gc_ref())
- .write_u32(offset, f),
- Val::F64(f) => gcstore
- .gc_object_data(self.as_gc_ref())
- .write_u64(offset, f),
- Val::V128(v) => gcstore
- .gc_object_data(self.as_gc_ref())
- .write_v128(offset, v),
-
- // NB: We don't need to do a write barrier when initializing a
- // field, because there is nothing being overwritten. Therefore, we
- // just the clone barrier.
- Val::ExternRef(x) => {
- let x = match x {
- None => 0,
- Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
- };
- store
- .require_gc_store_mut()?
- .gc_object_data(self.as_gc_ref())
- .write_u32(offset, x);
- }
- Val::AnyRef(x) => {
- let x = match x {
- None => 0,
- Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
- };
- store
- .require_gc_store_mut()?
- .gc_object_data(self.as_gc_ref())
- .write_u32(offset, x);
- }
- Val::ExnRef(x) => {
- let x = match x {
- None => 0,
- Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
- };
- store
- .require_gc_store_mut()?
- .gc_object_data(self.as_gc_ref())
- .write_u32(offset, x);
- }
-
- Val::FuncRef(f) => {
- let func_ref = match f {
- Some(f) => Some(SendSyncPtr::new(f.vm_func_ref(store))),
- None => None,
- };
- let gcstore = store.require_gc_store_mut()?;
- let id = unsafe { gcstore.func_ref_table.intern(func_ref) };
- gcstore
- .gc_object_data(self.as_gc_ref())
- .write_u32(offset, id.into_raw());
- }
- Val::ContRef(_) => {
- // TODO(#10248): Implement array continuation reference init handling
- return Err(crate::format_err!(
- "initializing continuation references in array elements not yet supported"
- ));
- }
- }
- Ok(())
+ self.as_gc_ref().initialize_val(store, ty, offset, val)
}
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
index cbd623c240..a9781d724f 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
@@ -17,7 +17,7 @@ use crate::runtime::vm::{
GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMExternRef, VMGcHeader,
VMGcRef, VMMemoryDefinition,
};
-use crate::{Engine, prelude::*};
+use crate::{Engine, bail_bug, prelude::*};
use core::{
alloc::Layout, any::Any, mem, num::NonZeroU32, ptr::NonNull, sync::atomic::AtomicUsize,
};
@@ -108,21 +108,19 @@ unsafe impl GcHeapObject for VMCopyingHeaderAndForwardingRef {
impl VMCopyingHeaderAndForwardingRef {
/// Get the forwarding reference for this object, if it has been copied
/// during the current collection.
- fn forwarding_ref(&self) -> Option<VMGcRef> {
+ fn forwarding_ref(&self) -> Result<Option<VMGcRef>> {
debug_assert!(
self.header.object_size()
>= u32::try_from(mem::size_of::<VMCopyingHeaderAndForwardingRef>()).unwrap()
);
- if self.header.copied() {
- Some(
- self.forwarding_ref
- .as_ref()
- .expect("should always have a forwarding ref if the copied bit is set")
- .unchecked_copy(),
- )
+ Ok(if self.header.copied() {
+ match self.forwarding_ref.as_ref() {
+ Some(r) => Some(r.unchecked_copy()),
+ None => bail_bug!("should always have a forwarding ref if the copied bit is set"),
+ }
} else {
None
- }
+ })
}
/// Set the forwarding reference for this object and mark it as copied.
@@ -449,7 +447,7 @@ impl CopyingHeap {
/// Pop the next item off the worklist, or return `None` if the worklist is
/// empty.
- fn worklist_pop(&mut self) -> Option<VMGcRef> {
+ fn worklist_pop(&mut self) -> Result<Option<VMGcRef>> {
debug_assert!(
self.is_in_active_space(self.worklist_ptr)
|| self.worklist_ptr == self.active_space_end()
@@ -460,158 +458,199 @@ impl CopyingHeap {
debug_assert!(self.worklist_ptr <= self.bump_ptr());
if self.worklist_ptr == self.bump_ptr() {
- return None;
+ return Ok(None);
}
let result = self.worklist_ptr;
- let result = NonZeroU32::new(result).unwrap();
- let result = VMGcRef::from_heap_index(result).unwrap();
+ let result = match NonZeroU32::new(result) {
+ Some(result) => result,
+ None => bail_bug!("worklist ptr is zero"),
+ };
+ let result = match VMGcRef::from_heap_index(result) {
+ Some(result) => result,
+ None => bail_bug!("corrupt worklist_ptr"),
+ };
- let obj_size = self.index(copying_ref(&result)).object_size();
+ let obj_size = self.index(copying_ref(&result))?.object_size();
self.worklist_ptr += obj_size;
debug_assert!(self.worklist_ptr <= self.bump_ptr());
- Some(result)
+ Ok(Some(result))
}
/// Insert `gc_ref`, which points to a grey object, into the worklist.
- fn worklist_insert(&self, gc_ref: &VMGcRef) {
+ fn worklist_insert(&self, gc_ref: &VMGcRef) -> Result<()> {
// This is a no-op: insertion happens implicitly in `copy` when
// allocating space for the copied object and advancing the
// `bump_ptr`. But we still define and call this method just for the
// debug assertions.
if !cfg!(debug_assertions) {
- return;
+ return Ok(());
}
- let index = gc_ref.as_heap_index().unwrap().get();
+ let index = gc_ref.heap_index()?.get();
debug_assert!(self.is_in_active_space(index));
- let obj_size = self.index(copying_ref(gc_ref)).object_size();
+ let obj_size = self.index(copying_ref(gc_ref))?.object_size();
debug_assert_eq!(index + obj_size, self.bump_ptr());
debug_assert!(self.worklist_ptr <= index);
+ Ok(())
}
/// Get-or-create the location of this idle-space ref in the new active
/// semi-space.
- fn forward(&mut self, from_ref: &VMGcRef) -> VMGcRef {
+ fn forward(&mut self, from_ref: &VMGcRef) -> Result<VMGcRef> {
debug_assert!(!from_ref.is_i31());
- debug_assert!(self.is_in_idle_space(from_ref.as_heap_index().unwrap().get()));
+ debug_assert!(self.is_in_idle_space(from_ref.heap_index()?.get()));
if let Some(to_ref) = self
- .index(header_and_forwarding_ref(from_ref))
- .forwarding_ref()
+ .index(header_and_forwarding_ref(from_ref))?
+ .forwarding_ref()?
{
- return to_ref;
+ return Ok(to_ref);
}
self.copy(from_ref)
}
/// Copy this idle-space ref into the new active semi-space and return its
/// new location.
- fn copy(&mut self, from_ref: &VMGcRef) -> VMGcRef {
+ fn copy(&mut self, from_ref: &VMGcRef) -> Result<VMGcRef> {
debug_assert!(!from_ref.is_i31());
- let from_index = from_ref.as_heap_index().unwrap().get();
+ let from_index = from_ref.heap_index()?.get();
debug_assert!(self.is_in_idle_space(from_index));
- debug_assert!(!self.index(copying_ref(from_ref)).copied());
-
- let size = self.index(copying_ref(from_ref)).object_size();
- let to_index = self.allocate(size).expect(
- "there should always be enough room in the active semi-space for objects that \
- survived collection, since the active space is the same size as the idle space",
- );
+ debug_assert!(!self.index(copying_ref(from_ref))?.copied());
+
+ let size = self.index(copying_ref(from_ref))?.object_size();
+ let to_index = match self.allocate(size) {
+ Some(i) => i,
+ None => {
+ bail_bug!(
+ "\
+there should always be enough room in the active semi-space for objects that \
+survived collection, since the active space is the same size as the idle space",
+ )
+ }
+ };
debug_assert!(self.is_in_active_space(to_index));
- let to_ref =
- VMGcRef::from_heap_index(NonZeroU32::new(to_index).unwrap()).expect("valid heap index");
+ let to_ref = match NonZeroU32::new(to_index).and_then(VMGcRef::from_heap_index) {
+ Some(r) => r,
+ None => bail_bug!("invalid to_index"),
+ };
// Copy the object bytes.
- let from_start = usize::try_from(from_index).unwrap();
- let to_start = usize::try_from(to_index).unwrap();
- let size_usize = usize::try_from(size).unwrap();
- self.heap_slice_mut()
- .copy_within(from_start..from_start + size_usize, to_start);
+ let from_start = usize::try_from(from_index)?;
+ let to_start = usize::try_from(to_index)?;
+ let size_usize = usize::try_from(size)?;
+ let heap = self.heap_slice_mut();
+ if heap
+ .get(from_start..)
+ .and_then(|s| s.get(..size_usize))
+ .is_none()
+ || heap
+ .get(to_start..)
+ .and_then(|s| s.get(..size_usize))
+ .is_none()
+ {
+ bail_bug!("corrupt gc heap");
+ }
+ heap.copy_within(from_start..from_start + size_usize, to_start);
// Set the forwarding ref in the old (idle-space) object.
- self.index_mut(header_and_forwarding_ref(from_ref))
+ self.index_mut(header_and_forwarding_ref(from_ref))?
.set_forwarding_ref(to_ref.unchecked_copy());
// If this is an externref, insert it into the active externref list.
if self
- .index(copying_ref(&to_ref))
+ .index(copying_ref(&to_ref))?
.header
.kind()
.matches(VMGcKind::ExternRef)
{
let old_head = self.active_extern_ref_set_head.take();
- self.index_mut::<VMCopyingExternRef>(to_ref.as_typed_unchecked())
+ self.index_mut::<VMCopyingExternRef>(to_ref.as_typed_unchecked())?
.next_extern_ref = old_head;
self.active_extern_ref_set_head =
Some(to_ref.unchecked_copy().into_externref_unchecked());
}
- self.worklist_insert(&to_ref);
- to_ref
+ self.worklist_insert(&to_ref)?;
+ Ok(to_ref)
}
/// Trace a grey object's outgoing edges, copying their referents into the
/// new semi-space if necessary, and updating the object's references with
/// their forwarded locations in the new semi-space.
- fn scan(&mut self, gc_ref: &VMGcRef, trace_infos: &TraceInfos) {
+ fn scan(&mut self, gc_ref: &VMGcRef, trace_infos: &TraceInfos) -> Result<()> {
debug_assert!(!gc_ref.is_i31());
- let index = gc_ref.as_heap_index().unwrap().get();
+ let index = gc_ref.heap_index()?.get();
debug_assert!(self.is_in_active_space(index));
- let ty = self.index(copying_ref(gc_ref)).header.ty();
+ let ty = self.index(copying_ref(gc_ref))?.header.ty();
// `externref`s have no GC edges.
let Some(ty) = ty else {
- return;
+ return Ok(());
};
- let object_start = usize::try_from(index).unwrap();
+ let object_start = usize::try_from(index)?;
match trace_infos.trace_info(&ty) {
TraceInfo::Struct { gc_ref_offsets } => {
for &offset in gc_ref_offsets {
- self.scan_field(object_start, offset);
+ self.scan_field(object_start, offset)?;
}
}
TraceInfo::Array { gc_ref_elems } => {
if *gc_ref_elems {
let array_ref = gc_ref.as_arrayref_unchecked();
- let len = self.array_len(array_ref);
+ let len = self.array_len(array_ref)?;
for i in 0..len {
- let elem_offset = GC_REF_ARRAY_ELEMS_OFFSET
- + i * u32::try_from(mem::size_of::<u32>()).unwrap();
- self.scan_field(object_start, elem_offset);
+ let elem_offset = match i
+ .checked_mul(u32::try_from(mem::size_of::<u32>())?)
+ .and_then(|i| i.checked_add(GC_REF_ARRAY_ELEMS_OFFSET))
+ {
+ Some(i) => i,
+ None => bail_bug!("array element offset overflow"),
+ };
+ self.scan_field(object_start, elem_offset)?;
}
}
}
}
+ Ok(())
}
#[inline]
- fn scan_field(&mut self, object_start: usize, offset: u32) {
- let offset = usize::try_from(offset).unwrap();
- let field_start = object_start + offset;
+ fn scan_field(&mut self, object_start: usize, offset: u32) -> Result<()> {
+ let offset = usize::try_from(offset)?;
+ let field_start = match object_start.checked_add(offset) {
+ Some(start) => start,
+ None => bail_bug!("field offset overflow"),
+ };
let field_end = field_start + mem::size_of::<u32>();
- let raw: [u8; 4] = self.heap_slice()[field_start..field_end]
- .try_into()
- .unwrap();
- let raw = u32::from_le_bytes(raw);
+ let raw = match self
+ .heap_slice()
+ .get(field_start..)
+ .and_then(|s| s.first_chunk())
+ {
+ Some(raw) => u32::from_le_bytes(*raw),
+ None => bail_bug!("corrupt gc heap"),
+ };
if let Some(child) = VMGcRef::from_raw_u32(raw)
&& !child.is_i31()
{
- debug_assert!(self.is_in_idle_space(child.as_heap_index().unwrap().get()));
- let new_ref = self.forward(&child);
- debug_assert!(self.is_in_active_space(new_ref.as_heap_index().unwrap().get()));
+ debug_assert!(self.is_in_idle_space(child.heap_index()?.get()));
+ let new_ref = self.forward(&child)?;
+ debug_assert!(self.is_in_active_space(new_ref.heap_index()?.get()));
// Write the new reference back.
let new_raw = new_ref.as_raw_u32().to_le_bytes();
self.heap_slice_mut()[field_start..field_end].copy_from_slice(&new_raw);
}
+
+ Ok(())
}
}
@@ -697,15 +736,17 @@ unsafe impl GcHeap for CopyingHeap {
_host_data_table: &mut ExternRefHostDataTable,
destination: &mut Option<VMGcRef>,
source: Option<&VMGcRef>,
- ) {
+ ) -> Result<()> {
// The copying collector doesn't use reference counting; writes are
// simple overwrites.
*destination = source.map(|s| s.unchecked_copy());
+ Ok(())
}
- fn expose_gc_ref_to_wasm(&mut self, _gc_ref: VMGcRef) {
+ fn expose_gc_ref_to_wasm(&mut self, _gc_ref: VMGcRef) -> Result<()> {
// The copying collector doesn't need any special handling when exposing
// a GC ref to Wasm. There is no over-approximated-stack-roots list.
+ Ok(())
}
fn alloc_externref(
@@ -717,14 +758,14 @@ unsafe impl GcHeap for CopyingHeap {
let size = (size + align - 1) & !(align - 1);
let gc_ref = match self.alloc_raw(
VMGcHeader::externref(),
- Layout::from_size_align(size, align).unwrap(),
+ Layout::from_size_align(size, align)?,
)? {
Err(n) => return Ok(Err(n)),
Ok(gc_ref) => gc_ref,
};
// Take the old list head before borrowing self mutably through index_mut.
let old_head = self.active_extern_ref_set_head.take();
- let externref_obj = self.index_mut::<VMCopyingExternRef>(gc_ref.as_typed_unchecked());
+ let externref_obj = self.index_mut::<VMCopyingExternRef>(gc_ref.as_typed_unchecked())?;
externref_obj.host_data = host_data;
externref_obj.next_extern_ref = old_head;
let externref = gc_ref.into_externref_unchecked();
@@ -732,13 +773,13 @@ unsafe impl GcHeap for CopyingHeap {
Ok(Ok(externref))
}
- fn externref_host_data(&self, externref: &VMExternRef) -> ExternRefHostDataId {
+ fn externref_host_data(&self, externref: &VMExternRef) -> Result<ExternRefHostDataId> {
let typed_ref = externref_to_copying(externref);
- self.index(typed_ref).host_data
+ Ok(self.index(typed_ref)?.host_data)
}
- fn header(&self, gc_ref: &VMGcRef) -> &VMGcHeader {
- let header: &VMGcHeader = self.index(gc_ref.as_typed_unchecked());
+ fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader> {
+ let header: &VMGcHeader = self.index(gc_ref.as_typed_unchecked())?;
debug_assert!(
VMGcKind::try_from_u32(header.kind().as_u32()).is_some(),
@@ -746,11 +787,11 @@ unsafe impl GcHeap for CopyingHeap {
header.kind().as_u32(),
);
- header
+ Ok(header)
}
- fn header_mut(&mut self, gc_ref: &VMGcRef) -> &mut VMGcHeader {
- let header: &mut VMGcHeader = self.index_mut(gc_ref.as_typed_unchecked());
+ fn header_mut(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcHeader> {
+ let header: &mut VMGcHeader = self.index_mut(gc_ref.as_typed_unchecked())?;
debug_assert!(
VMGcKind::try_from_u32(header.kind().as_u32()).is_some(),
@@ -758,11 +799,13 @@ unsafe impl GcHeap for CopyingHeap {
header.kind().as_u32(),
);
- header
+ Ok(header)
}
- fn object_size(&self, gc_ref: &VMGcRef) -> usize {
- usize::try_from(self.index(copying_ref(gc_ref)).object_size()).unwrap()
+ fn object_size(&self, gc_ref: &VMGcRef) -> Result<usize> {
+ Ok(usize::try_from(
+ self.index(copying_ref(gc_ref))?.object_size(),
+ )?)
}
fn alloc_raw(&mut self, header: VMGcHeader, layout: Layout) -> Result<Result<VMGcRef, u64>> {
@@ -811,7 +854,7 @@ unsafe impl GcHeap for CopyingHeap {
// Assert that the newly-allocated memory is still filled with the
// poison pattern.
if cfg!(gc_zeal) {
- let start = usize::try_from(gc_ref.as_heap_index().unwrap().get()).unwrap();
+ let start = usize::try_from(gc_ref.heap_index()?.get()).unwrap();
let slice = &self.heap_slice()[start..][..layout.size()];
gc_assert!(
slice.iter().all(|&b| b == POISON),
@@ -821,7 +864,7 @@ unsafe impl GcHeap for CopyingHeap {
}
let object_size = size;
- *self.index_mut(copying_ref(&gc_ref)) = VMCopyingHeader {
+ *self.index_mut(copying_ref(&gc_ref))? = VMCopyingHeader {
header,
object_size,
};
@@ -847,9 +890,10 @@ unsafe impl GcHeap for CopyingHeap {
Ok(Ok(gc_ref))
}
- fn dealloc_uninit_struct_or_exn(&mut self, _gcref: VMGcRef) {
+ fn dealloc_uninit_struct_or_exn(&mut self, _gcref: VMGcRef) -> Result<()> {
// The copying collector doesn't support individual deallocation; memory
// is reclaimed during collection.
+ Ok(())
}
fn alloc_uninit_array(
@@ -870,21 +914,23 @@ unsafe impl GcHeap for CopyingHeap {
Ok(gc_ref) => gc_ref,
};
- self.index_mut(gc_ref.as_typed_unchecked::<VMCopyingArrayHeader>())
+ self.index_mut(gc_ref.as_typed_unchecked::<VMCopyingArrayHeader>())?
.length = length;
Ok(Ok(gc_ref.into_arrayref_unchecked()))
}
- fn dealloc_uninit_array(&mut self, _arrayref: VMArrayRef) {
+ fn dealloc_uninit_array(&mut self, _arrayref: VMArrayRef) -> Result<()> {
// The copying collector doesn't support individual deallocation; memory
// is reclaimed during collection.
+ Ok(())
}
- fn array_len(&self, arrayref: &VMArrayRef) -> u32 {
+ fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32> {
debug_assert!(arrayref.as_gc_ref().is_typed::<VMCopyingArrayHeader>(self));
- self.index::<VMCopyingArrayHeader>(arrayref.as_gc_ref().as_typed_unchecked())
- .length
+ Ok(self
+ .index::<VMCopyingArrayHeader>(arrayref.as_gc_ref().as_typed_unchecked())?
+ .length)
}
fn allocated_bytes(&self) -> usize {
@@ -973,67 +1019,64 @@ enum CopyingCollectionPhase {
impl CopyingCollection<'_> {
/// Forward all GC roots from the idle space to the active space.
- fn process_roots(&mut self) {
+ fn process_roots(&mut self) -> Result<()> {
log::trace!("Begin processing GC roots");
let roots = self.roots.take().unwrap();
for mut root in roots {
- let gc_ref = root.get();
+ let gc_ref = root.get()?;
if gc_ref.is_i31() {
continue;
}
- let old_index = gc_ref.as_heap_index().unwrap().get();
+ let old_index = gc_ref.heap_index()?.get();
debug_assert!(self.heap.is_in_idle_space(old_index));
- let new_ref = self.heap.forward(&gc_ref);
+ let new_ref = self.heap.forward(&gc_ref)?;
root.set(new_ref);
}
log::trace!("End processing GC roots");
+ Ok(())
}
/// Scan all grey objects until the worklist is empty.
- fn process_worklist(&mut self) {
+ fn process_worklist(&mut self) -> Result<()> {
log::trace!("Begin processing worklist");
let trace_infos = mem::take(&mut self.heap.trace_infos);
- while let Some(gc_ref) = self.heap.worklist_pop() {
- debug_assert!(
- self.heap
- .is_in_active_space(gc_ref.as_heap_index().unwrap().get())
- );
- self.heap.scan(&gc_ref, &trace_infos);
+ while let Some(gc_ref) = self.heap.worklist_pop()? {
+ debug_assert!(self.heap.is_in_active_space(gc_ref.heap_index()?.get()));
+ self.heap.scan(&gc_ref, &trace_infos)?;
}
self.heap.trace_infos = trace_infos;
log::trace!("End processing worklist");
+ Ok(())
}
/// Clean up dead externrefs by iterating the idle semi-space's externref
/// linked list and deallocating host data for any that were not forwarded.
- fn sweep_extern_refs(&mut self) {
+ fn sweep_extern_refs(&mut self) -> Result<()> {
log::trace!("Begin sweeping `externref`s");
let mut link = self.heap.idle_extern_ref_set_head.take();
while let Some(externref) = link {
let gc_ref = externref.as_gc_ref();
- debug_assert!(
- self.heap
- .is_in_idle_space(gc_ref.as_heap_index().unwrap().get())
- );
- let header = self.heap.index(copying_ref(gc_ref));
+ debug_assert!(self.heap.is_in_idle_space(gc_ref.heap_index()?.get()));
+ let header = self.heap.index(copying_ref(gc_ref))?;
if !header.copied() {
let typed: &TypedGcRef<VMCopyingExternRef> = gc_ref.as_typed_unchecked();
- let host_data_id = self.heap.index(typed).host_data;
- self.host_data_table.dealloc(host_data_id);
+ let host_data_id = self.heap.index(typed)?.host_data;
+ self.host_data_table.dealloc(host_data_id)?;
}
link = self
.heap
- .index::<VMCopyingExternRef>(gc_ref.as_typed_unchecked())
+ .index::<VMCopyingExternRef>(gc_ref.as_typed_unchecked())?
.next_extern_ref
.as_ref()
.map(|e| e.unchecked_copy());
}
log::trace!("End sweeping `externref`s");
+ Ok(())
}
}
impl GarbageCollection<'_> for CopyingCollection<'_> {
- fn collect_increment(&mut self) -> GcProgress {
+ fn collect_increment(&mut self) -> Result<GcProgress> {
match self.phase {
CopyingCollectionPhase::Collect => {
log::trace!("Begin copying collection");
@@ -1050,8 +1093,8 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {
self.heap.flip();
self.heap.initialize_worklist();
- self.process_roots();
- self.process_worklist();
+ self.process_roots()?;
+ self.process_worklist()?;
assert!(self.heap.active_space_start <= self.heap.bump_ptr());
assert!(self.heap.bump_ptr() <= self.heap.active_space_end());
@@ -1061,7 +1104,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {
|| self.heap.idle_space_end <= self.heap.active_space_start
);
- self.sweep_extern_refs();
+ self.sweep_extern_refs()?;
self.heap.resize_semi_spaces();
debug_assert_eq!(
@@ -1079,9 +1122,9 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {
log::trace!("End copying collection");
self.phase = CopyingCollectionPhase::Done;
- GcProgress::Complete
+ Ok(GcProgress::Complete)
}
- CopyingCollectionPhase::Done => GcProgress::Complete,
+ CopyingCollectionPhase::Done => Ok(GcProgress::Complete),
}
}
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
index 63de033b2a..2d39177413 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/drc.rs
@@ -44,17 +44,17 @@
//! Examination of Deferred Reference Counting and Cycle Detection* by Quinane:
//! <https://openresearch-repository.anu.edu.au/bitstream/1885/42030/2/hon-thesis.pdf>
+use super::VMArrayRef;
use super::free_list::FreeList;
use super::trace_info::{TraceInfo, TraceInfos};
-use super::{VMArrayRef, VMGcObjectData};
use crate::hash_set::HashSet;
use crate::runtime::vm::{
ExternRefHostDataId, ExternRefHostDataTable, GarbageCollection, GcHeap, GcHeapObject,
GcProgress, GcRootsIter, GcRuntime, SendSyncUnsafeCell, TypedGcRef, VMExternRef, VMGcHeader,
- VMGcRef,
+ VMGcObjectData, VMGcRef,
};
use crate::vm::VMMemoryDefinition;
-use crate::{Engine, prelude::*};
+use crate::{Engine, bail_bug, prelude::*};
use core::sync::atomic::AtomicUsize;
use core::{
alloc::Layout,
@@ -63,6 +63,7 @@ use core::{
ops::{Deref, DerefMut, Range},
ptr::NonNull,
};
+use wasmtime_core::undo::Undo;
use wasmtime_environ::drc::{ARRAY_LENGTH_OFFSET, DrcTypeLayouts};
use wasmtime_environ::{
GcArrayLayout, GcStructLayout, GcTypeLayouts, POISON, VMGcKind, VMSharedTypeIndex, gc_assert,
@@ -191,29 +192,30 @@ struct DrcHeap {
/// A free list describing which ranges of the heap are available for use.
free_list: Option<FreeList>,
+ /// Allocations used during tracing, temporarily removed from `self` for
+ /// easier borrow-checker management.
+ tracing_allocs: Option<TracingAllocs>,
+
+ /// Running total of bytes currently allocated (live objects) in this heap.
+ allocated_bytes: usize,
+}
+
+struct TracingAllocs {
/// An explicit stack to avoid recursion when deallocating one object needs
/// to dec-ref another object, which can then be deallocated and dec-refs
/// yet another object, etc...
///
/// We store this stack here to reuse the storage and avoid repeated
/// allocations.
- ///
- /// Note that the `Option` is perhaps technically unnecessary (we could
- /// remove the `Option` and, when we take the stack out of `self`, leave
- /// behind an empty vec instead of `None`) but we keep it because it will
- /// help us catch unexpected re-entry, similar to how a `RefCell` would.
- dec_ref_stack: Option<Vec<VMGcRef>>,
+ dec_ref_stack: Vec<VMGcRef>,
/// An explicit stack for arrays that are too large to push all their
/// elements onto `dec_ref_stack` at once. Each entry is an array GC
/// reference and the range of element indices remaining to process.
- large_array_dec_ref_stack: Option<Vec<(VMGcRef, Range<u32>)>>,
+ large_array_dec_ref_stack: Vec<(VMGcRef, Range<u32>)>,
/// A batched set of GC refs to deallocate all at once.
- to_dealloc: Option<Vec<VMGcRef>>,
-
- /// Running total of bytes currently allocated (live objects) in this heap.
- allocated_bytes: usize,
+ to_dealloc: Vec<VMGcRef>,
}
impl DrcHeap {
@@ -227,43 +229,50 @@ impl DrcHeap {
memory: None,
vmmemory: None,
free_list: None,
- dec_ref_stack: Some(Vec::with_capacity(1)),
- large_array_dec_ref_stack: Some(Vec::with_capacity(1)),
- to_dealloc: Some(Vec::with_capacity(1)),
+ tracing_allocs: Some(TracingAllocs {
+ dec_ref_stack: Vec::with_capacity(1),
+ large_array_dec_ref_stack: Vec::with_capacity(1),
+ to_dealloc: Vec::with_capacity(1),
+ }),
allocated_bytes: 0,
})
}
- fn dealloc(&mut self, gc_ref: VMGcRef) {
+ fn dealloc(&mut self, gc_ref: VMGcRef) -> Result<()> {
let drc_ref = drc_ref(&gc_ref);
- let size = self.index(drc_ref).object_size;
- let alloc_size = FreeList::aligned_size(size).unwrap();
- let index = gc_ref.as_heap_index().unwrap();
+ let size = self.index(drc_ref)?.object_size;
+ let alloc_size = match FreeList::aligned_size(size) {
+ Some(size) => size,
+ None => bail_bug!("aligned size overflow"),
+ };
+ let index = gc_ref.heap_index()?;
// Poison the freed memory so that any stale access is detectable.
if cfg!(gc_zeal) {
- let index = usize::try_from(index.get()).unwrap();
- let alloc_size = usize::try_from(alloc_size).unwrap();
+ let index = usize::try_from(index.get())?;
+ let alloc_size = usize::try_from(alloc_size)?;
self.heap_slice_mut()[index..][..alloc_size].fill(POISON);
}
- self.allocated_bytes -= usize::try_from(alloc_size).unwrap();
+ self.allocated_bytes -= usize::try_from(alloc_size)?;
self.free_list
.as_mut()
.unwrap()
.dealloc_fast(index, alloc_size);
+ Ok(())
}
/// Increment the ref count for the associated object.
- fn inc_ref(&mut self, gc_ref: &VMGcRef) {
+ fn inc_ref(&mut self, gc_ref: &VMGcRef) -> Result<()> {
if gc_ref.is_i31() {
- return;
+ return Ok(());
}
let drc_ref = drc_ref(gc_ref);
- let header = self.index_mut(&drc_ref);
+ let header = self.index_mut(&drc_ref)?;
header.inc_ref();
log::trace!("increment {:#p} ref count -> {}", *gc_ref, header.ref_count);
+ Ok(())
}
/// Decrement the ref count for the associated object.
@@ -278,18 +287,26 @@ impl DrcHeap {
&mut self,
host_data_table: &mut ExternRefHostDataTable,
gc_ref: &VMGcRef,
- ) {
+ ) -> Result<()> {
if gc_ref.is_i31() {
- return;
+ return Ok(());
}
- let mut stack = self.dec_ref_stack.take().unwrap();
- debug_assert!(stack.is_empty());
+ let allocs = match self.tracing_allocs.take() {
+ Some(allocs) => allocs,
+ None => bail_bug!("allocs missing during tracing"),
+ };
+ let mut undo = Undo::new((self, allocs), |(this, allocs)| {
+ debug_assert!(this.tracing_allocs.is_none());
+ this.tracing_allocs = Some(allocs);
+ });
+ let (this, allocs) = &mut *undo;
+ let stack = &mut allocs.dec_ref_stack;
+ let large_array_stack = &mut allocs.large_array_dec_ref_stack;
+ let to_dealloc = &mut allocs.to_dealloc;
- let mut large_array_stack = self.large_array_dec_ref_stack.take().unwrap();
+ debug_assert!(stack.is_empty());
debug_assert!(large_array_stack.is_empty());
-
- let mut to_dealloc = self.to_dealloc.take().unwrap();
debug_assert!(to_dealloc.is_empty());
stack.push(gc_ref.unchecked_copy());
@@ -299,7 +316,7 @@ impl DrcHeap {
debug_assert!(!gc_ref.is_i31());
// Read the DRC header once to get ref_count, type, and object_size.
- let drc_header = self.index_mut(drc_ref(&gc_ref));
+ let drc_header = this.index_mut(drc_ref(&gc_ref))?;
log::trace!(
"decrement {:#p} ref count -> {}",
gc_ref,
@@ -315,33 +332,33 @@ impl DrcHeap {
// Trace: enqueue child GC refs for dec-ref'ing.
if let Some(ty) = ty {
- match self.trace_infos.trace_info(&ty) {
+ match this.trace_infos.trace_info(&ty) {
TraceInfo::Struct { gc_ref_offsets } => {
stack.reserve(gc_ref_offsets.len());
- let data = self.gc_object_data(&gc_ref);
+ let data = this.gc_object_data(&gc_ref)?;
for offset in gc_ref_offsets {
- Self::trace_offset(&mut stack, data, *offset);
+ Self::trace_offset(stack, data, *offset)?;
}
}
TraceInfo::Array { gc_ref_elems: true } => {
- let len = self.array_len(gc_ref.as_arrayref_unchecked());
- let len_usize = usize::try_from(len).unwrap();
+ let len = this.array_len(gc_ref.as_arrayref_unchecked())?;
+ let len_usize = usize::try_from(len)?;
if stack.len() + len_usize <= MAX_ARRAY_STACK_DEPTH {
- let data = self.gc_object_data(&gc_ref);
+ let data = this.gc_object_data(&gc_ref)?;
stack.reserve(len_usize);
for i in 0..len {
- Self::trace_array_elem(&mut stack, data, i);
+ Self::trace_array_elem(stack, data, i)?;
}
} else {
// Only push the first `n` elements onto the
// stack; process the rest via the
// `large_array_stack`.
let n = MAX_ARRAY_STACK_DEPTH.saturating_sub(stack.len());
- let n = u32::try_from(n).unwrap();
- let data = self.gc_object_data(&gc_ref);
+ let n = u32::try_from(n)?;
+ let data = this.gc_object_data(&gc_ref)?;
for i in 0..n {
- Self::trace_array_elem(&mut stack, data, i);
+ Self::trace_array_elem(stack, data, i)?;
}
large_array_stack.push((gc_ref.unchecked_copy(), n..len));
@@ -361,9 +378,12 @@ impl DrcHeap {
// data, and `ty` is `None` only for `externref`s, so we skip
// this for `struct` and `array` objects entirely.
debug_assert!(drc_header.header.kind().matches(VMGcKind::ExternRef));
- let externref = gc_ref.as_typed::<VMDrcExternRef>(self).unwrap();
- let host_data_id = self.index(externref).host_data;
- host_data_table.dealloc(host_data_id);
+ let externref = match gc_ref.as_typed::<VMDrcExternRef>(*this) {
+ Some(r) => r,
+ None => bail_bug!("expected externref"),
+ };
+ let host_data_id = this.index(externref)?.host_data;
+ host_data_table.dealloc(host_data_id)?;
}
to_dealloc.push(gc_ref);
@@ -371,9 +391,9 @@ impl DrcHeap {
if let Some((gc_ref, mut elems)) = large_array_stack.pop() {
// Add the next chunk of array elements onto the stack.
- let data = self.gc_object_data(&gc_ref);
+ let data = this.gc_object_data(&gc_ref)?;
for i in elems.by_ref().take(MAX_ARRAY_STACK_DEPTH) {
- Self::trace_array_elem(&mut stack, data, i);
+ Self::trace_array_elem(stack, data, i)?;
}
// If we are done processing this array, then enqueue it for
@@ -391,37 +411,31 @@ impl DrcHeap {
// Deallocate the dead objects and return their memory blocks to the
// free list.
for gc_ref in to_dealloc.drain(..) {
- self.dealloc(gc_ref);
+ this.dealloc(gc_ref)?;
}
debug_assert!(stack.is_empty());
- debug_assert!(self.dec_ref_stack.is_none());
- self.dec_ref_stack = Some(stack);
-
debug_assert!(large_array_stack.is_empty());
- debug_assert!(self.large_array_dec_ref_stack.is_none());
- self.large_array_dec_ref_stack = Some(large_array_stack);
-
debug_assert!(to_dealloc.is_empty());
- debug_assert!(self.to_dealloc.is_none());
- self.to_dealloc = Some(to_dealloc);
+
+ Ok(())
}
#[inline]
- fn trace_array_elem(stack: &mut Vec<VMGcRef>, data: &VMGcObjectData, i: u32) {
- let elem_offset =
- GC_REF_ARRAY_ELEMS_OFFSET + i * u32::try_from(mem::size_of::<u32>()).unwrap();
+ fn trace_array_elem(stack: &mut Vec<VMGcRef>, data: &VMGcObjectData, i: u32) -> Result<()> {
+ let elem_offset = GC_REF_ARRAY_ELEMS_OFFSET + i * u32::try_from(mem::size_of::<u32>())?;
Self::trace_offset(stack, data, elem_offset)
}
#[inline]
- fn trace_offset(stack: &mut Vec<VMGcRef>, data: &VMGcObjectData, offset: u32) {
- let raw = data.read_u32(offset);
+ fn trace_offset(stack: &mut Vec<VMGcRef>, data: &VMGcObjectData, offset: u32) -> Result<()> {
+ let raw = data.read_u32(offset)?;
if let Some(gc_ref) = VMGcRef::from_raw_u32(raw)
&& !gc_ref.is_i31()
{
stack.push(gc_ref);
}
+ Ok(())
}
/// Iterate over the over-approximated-stack-roots list.
@@ -430,23 +444,26 @@ impl DrcHeap {
core::iter::from_fn(move || {
let r = link.as_ref()?.unchecked_copy();
- link = self.index(drc_ref(&r)).next_over_approximated_stack_root();
+ link = self
+ .index(drc_ref(&r))
+ .ok()?
+ .next_over_approximated_stack_root();
Some(r)
})
}
/// Assert the integrity of the over-approximated stack roots list.
- fn assert_over_approximated_stack_roots_integrity(&self) {
+ fn assert_over_approximated_stack_roots_integrity(&self) -> Result<()> {
if !cfg!(gc_zeal) {
- return;
+ return Ok(());
}
let mut visited = HashSet::new();
for gc_ref in self.iter_over_approximated_stack_roots() {
- let idx = gc_ref.as_heap_index().unwrap().get();
+ let idx = gc_ref.heap_index()?.get();
// Each entry must have a valid `VMGcKind`.
- let header = self.header(&gc_ref);
+ let header = self.header(&gc_ref)?;
let kind = header.kind().as_u32();
assert!(
VMGcKind::try_from_u32(kind).is_some(),
@@ -454,7 +471,7 @@ impl DrcHeap {
);
// Each entry must have its in-list bit set.
- let drc_header = self.index(drc_ref(&gc_ref));
+ let drc_header = self.index(drc_ref(&gc_ref))?;
assert!(
drc_header.is_in_over_approximated_stack_roots(),
"over-approx list: entry at heap index {idx} does not have in-list bit set",
@@ -478,6 +495,7 @@ impl DrcHeap {
visited.len(),
"over-approx list: tracked size does not match actual size",
);
+ Ok(())
}
/// Assert that every free block in the free list is filled with the poison
@@ -499,7 +517,7 @@ impl DrcHeap {
}
}
- fn trace(&mut self, roots: &mut GcRootsIter<'_>) {
+ fn trace(&mut self, roots: &mut GcRootsIter<'_>) -> Result<()> {
// The `over_approx_set` is used for `debug_assert!`s checking that
// every reference we read out from the stack via stack maps is actually
// in the table. If that weren't true, than either we forgot to insert a
@@ -520,7 +538,7 @@ impl DrcHeap {
continue;
}
- let gc_ref = root.get();
+ let gc_ref = root.get()?;
if gc_ref.is_i31() {
continue;
@@ -535,21 +553,22 @@ impl DrcHeap {
but {gc_ref:#p} is not in the set",
);
debug_assert!(
- self.index(drc_ref(&gc_ref))
+ self.index(drc_ref(&gc_ref))?
.is_in_over_approximated_stack_roots(),
"every on-stack gc ref inside a Wasm frame should have \
its in-the-over-approximated-stack-roots-list bit set",
);
debug_assert_ne!(
- self.index_mut(drc_ref(&gc_ref)).ref_count,
+ self.index_mut(drc_ref(&gc_ref))?.ref_count,
0,
"{gc_ref:#p} is on the Wasm stack and therefore should be held \
alive by the over-approximated-stack-roots set; should have \
nonzero ref count",
);
- self.index_mut(drc_ref(&gc_ref)).set_marked();
+ self.index_mut(drc_ref(&gc_ref))?.set_marked();
}
+ Ok(())
}
#[inline(never)]
@@ -571,7 +590,7 @@ impl DrcHeap {
/// Sweep the bump allocation table after we've discovered our precise stack
/// roots.
- fn sweep(&mut self, host_data_table: &mut ExternRefHostDataTable) {
+ fn sweep(&mut self, host_data_table: &mut ExternRefHostDataTable) -> Result<()> {
if log::log_enabled!(log::Level::Trace) {
Self::log_gc_ref_set(
"over-approximated-stack-roots set before sweeping",
@@ -620,7 +639,7 @@ impl DrcHeap {
while let Some(gc_ref) = next {
log::trace!("sweeping gc ref: {gc_ref:#p}");
- let header = self.index_mut(drc_ref(&gc_ref));
+ let header = self.index_mut(drc_ref(&gc_ref))?;
debug_assert!(header.is_in_over_approximated_stack_roots());
if header.clear_marked() {
@@ -649,12 +668,12 @@ impl DrcHeap {
match &prev {
None => self.vmctx_data.set_over_approximated_stack_roots(prev_next),
Some(prev) => self
- .index_mut(drc_ref(prev))
+ .index_mut(drc_ref(prev))?
.set_next_over_approximated_stack_root(prev_next),
}
self.vmctx_data
.decrement_current_over_approximated_stack_roots_len();
- self.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref);
+ self.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref)?;
}
self.vmctx_data
@@ -670,6 +689,8 @@ impl DrcHeap {
self.iter_over_approximated_stack_roots(),
);
}
+
+ Ok(())
}
}
@@ -870,9 +891,7 @@ unsafe impl GcHeap for DrcHeap {
no_gc_count,
vmctx_data,
free_list,
- dec_ref_stack,
- large_array_dec_ref_stack,
- to_dealloc,
+ tracing_allocs,
memory,
vmmemory,
allocated_bytes,
@@ -885,13 +904,12 @@ unsafe impl GcHeap for DrcHeap {
*vmmemory = None;
*allocated_bytes = 0;
trace_infos.clear();
- debug_assert!(dec_ref_stack.as_ref().is_some_and(|s| s.is_empty()));
- debug_assert!(
- large_array_dec_ref_stack
- .as_ref()
- .is_some_and(|s| s.is_empty())
- );
- debug_assert!(to_dealloc.as_ref().is_some_and(|d| d.is_empty()));
+
+ debug_assert!(tracing_allocs.as_ref().is_some_and(|allocs| {
+ allocs.dec_ref_stack.is_empty()
+ && allocs.large_array_dec_ref_stack.is_empty()
+ && allocs.to_dealloc.is_empty()
+ }));
memory.take().unwrap()
}
@@ -917,7 +935,18 @@ unsafe impl GcHeap for DrcHeap {
}
fn clone_gc_ref(&mut self, gc_ref: &VMGcRef) -> VMGcRef {
- self.inc_ref(gc_ref);
+ // If incrementing the reference count fails then that means that the GC
+ // heap is corrupted. Plumbing this result all throughout Wasmtime has
+ // quite large implications which aren't necessarily worth the tradeoff.
+ // This is the only collector where this is a fallible operation, for
+ // example. For now catch this in debug mode but otherwise just leave
+ // the corruption to get detected later. This corrupted reference will
+ // trigger an error later on instead.
+ if let Err(e) = self.inc_ref(gc_ref) {
+ if cfg!(debug_assertions) {
+ panic!("gc heap corrupted: {e}");
+ }
+ }
gc_ref.unchecked_copy()
}
@@ -926,27 +955,28 @@ unsafe impl GcHeap for DrcHeap {
host_data_table: &mut ExternRefHostDataTable,
destination: &mut Option<VMGcRef>,
source: Option<&VMGcRef>,
- ) {
+ ) -> Result<()> {
// Increment the ref count of the object being written into the slot.
if let Some(src) = source {
- self.inc_ref(src);
+ self.inc_ref(src)?;
}
// Decrement the ref count of the value being overwritten and, if
// necessary, deallocate the GC object.
if let Some(dest) = destination {
- self.dec_ref_and_maybe_dealloc(host_data_table, dest);
+ self.dec_ref_and_maybe_dealloc(host_data_table, dest)?;
}
// Do the actual write.
*destination = source.map(|s| s.unchecked_copy());
+ Ok(())
}
- fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) {
+ fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) -> Result<()> {
// Read the current list head before borrowing through index_mut.
let next = self.vmctx_data.over_approximated_stack_roots();
- let header = self.index_mut(drc_ref(&gc_ref));
+ let header = self.index_mut(drc_ref(&gc_ref))?;
if header.is_in_over_approximated_stack_roots() {
// Already in the over-approximated-stack-roots list. Decrement the
// object's ref count because the OASR list can't hold multiple
@@ -957,7 +987,7 @@ unsafe impl GcHeap for DrcHeap {
"should not have reached refcount == 0 because the OASR list \
is holding a reference"
);
- return;
+ return Ok(());
}
// Push this object onto the head of the over-approximated-stack-roots
@@ -968,6 +998,7 @@ unsafe impl GcHeap for DrcHeap {
.set_over_approximated_stack_roots(Some(gc_ref));
self.vmctx_data
.increment_current_over_approximated_stack_roots_len();
+ Ok(())
}
fn alloc_externref(
@@ -979,18 +1010,18 @@ unsafe impl GcHeap for DrcHeap {
Err(n) => return Ok(Err(n)),
Ok(gc_ref) => gc_ref,
};
- self.index_mut::<VMDrcExternRef>(gc_ref.as_typed_unchecked())
+ self.index_mut::<VMDrcExternRef>(gc_ref.as_typed_unchecked())?
.host_data = host_data;
Ok(Ok(gc_ref.into_externref_unchecked()))
}
- fn externref_host_data(&self, externref: &VMExternRef) -> ExternRefHostDataId {
+ fn externref_host_data(&self, externref: &VMExternRef) -> Result<ExternRefHostDataId> {
let typed_ref = externref_to_drc(externref);
- self.index(typed_ref).host_data
+ Ok(self.index(typed_ref)?.host_data)
}
- fn header(&self, gc_ref: &VMGcRef) -> &VMGcHeader {
- let header: &VMGcHeader = self.index(gc_ref.as_typed_unchecked());
+ fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader> {
+ let header: &VMGcHeader = self.index(gc_ref.as_typed_unchecked())?;
debug_assert!(
VMGcKind::try_from_u32(header.kind().as_u32()).is_some(),
@@ -998,11 +1029,11 @@ unsafe impl GcHeap for DrcHeap {
header.kind().as_u32(),
);
- header
+ Ok(header)
}
- fn header_mut(&mut self, gc_ref: &VMGcRef) -> &mut VMGcHeader {
- let header: &mut VMGcHeader = self.index_mut(gc_ref.as_typed_unchecked());
+ fn header_mut(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcHeader> {
+ let header: &mut VMGcHeader = self.index_mut(gc_ref.as_typed_unchecked())?;
debug_assert!(
VMGcKind::try_from_u32(header.kind().as_u32()).is_some(),
@@ -1010,11 +1041,11 @@ unsafe impl GcHeap for DrcHeap {
header.kind().as_u32(),
);
- header
+ Ok(header)
}
- fn object_size(&self, gc_ref: &VMGcRef) -> usize {
- self.index(drc_ref(gc_ref)).object_size()
+ fn object_size(&self, gc_ref: &VMGcRef) -> Result<usize> {
+ Ok(self.index(drc_ref(gc_ref))?.object_size())
}
fn alloc_raw(&mut self, header: VMGcHeader, layout: Layout) -> Result<Result<VMGcRef, u64>> {
@@ -1043,17 +1074,20 @@ unsafe impl GcHeap for DrcHeap {
.ok_or_else(|| format_err!("allocation size too large"))?;
let gc_ref = match self.free_list.as_mut().unwrap().alloc_fast(alloc_size) {
- None => return Ok(Err(u64::try_from(layout.size()).unwrap())),
- Some(index) => VMGcRef::from_heap_index(index).unwrap_or_else(|| {
- panic!("invalid GC heap index returned from free list alloc: {index:#x}")
- }),
+ None => return Ok(Err(u64::try_from(layout.size())?)),
+ Some(index) => match VMGcRef::from_heap_index(index) {
+ Some(r) => r,
+ None => {
+ bail_bug!("invalid GC heap index returned from free list alloc: {index:#x}")
+ }
+ },
};
// Assert that the newly-allocated memory is still filled with the
// poison pattern, and hasn't been corrupted since deallocation (or
// initial heap creation).
if cfg!(gc_zeal) {
- let start = usize::try_from(gc_ref.as_heap_index().unwrap().get()).unwrap();
+ let start = usize::try_from(gc_ref.heap_index()?.get())?;
let slice = &self.heap_slice()[start..][..layout.size()];
gc_assert!(
slice.iter().all(|&b| b == POISON),
@@ -1062,13 +1096,13 @@ unsafe impl GcHeap for DrcHeap {
);
}
- *self.index_mut(drc_ref(&gc_ref)) = VMDrcHeader {
+ *self.index_mut(drc_ref(&gc_ref))? = VMDrcHeader {
header,
ref_count: 1,
next_over_approximated_stack_root: None,
object_size,
};
- self.allocated_bytes += usize::try_from(alloc_size).unwrap();
+ self.allocated_bytes += usize::try_from(alloc_size)?;
log::trace!("new object: increment {gc_ref:#p} ref count -> 1");
Ok(Ok(gc_ref))
}
@@ -1092,8 +1126,8 @@ unsafe impl GcHeap for DrcHeap {
Ok(Ok(gc_ref))
}
- fn dealloc_uninit_struct_or_exn(&mut self, gcref: VMGcRef) {
- self.dealloc(gcref);
+ fn dealloc_uninit_struct_or_exn(&mut self, gcref: VMGcRef) -> Result<()> {
+ self.dealloc(gcref)
}
fn alloc_uninit_array(
@@ -1113,20 +1147,21 @@ unsafe impl GcHeap for DrcHeap {
Ok(gc_ref) => gc_ref,
};
- self.index_mut(gc_ref.as_typed_unchecked::<VMDrcArrayHeader>())
+ self.index_mut(gc_ref.as_typed_unchecked::<VMDrcArrayHeader>())?
.length = length;
Ok(Ok(gc_ref.into_arrayref_unchecked()))
}
- fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef) {
+ fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef) -> Result<()> {
self.dealloc(arrayref.into())
}
- fn array_len(&self, arrayref: &VMArrayRef) -> u32 {
+ fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32> {
debug_assert!(arrayref.as_gc_ref().is_typed::<VMDrcArrayHeader>(self));
- self.index::<VMDrcArrayHeader>(arrayref.as_gc_ref().as_typed_unchecked())
- .length
+ Ok(self
+ .index::<VMDrcArrayHeader>(arrayref.as_gc_ref().as_typed_unchecked())?
+ .length)
}
fn allocated_bytes(&self) -> usize {
@@ -1206,19 +1241,19 @@ enum DrcCollectionPhase {
}
impl<'a> GarbageCollection<'a> for DrcCollection<'a> {
- fn collect_increment(&mut self) -> GcProgress {
+ fn collect_increment(&mut self) -> Result<GcProgress> {
match self.phase {
DrcCollectionPhase::Trace => {
#[cfg(feature = "std")]
let start = std::time::Instant::now();
log::debug!("Begin DRC trace");
- self.heap.assert_over_approximated_stack_roots_integrity();
+ self.heap.assert_over_approximated_stack_roots_integrity()?;
self.heap.assert_free_blocks_are_poisoned();
- self.heap.trace(&mut self.roots);
+ self.heap.trace(&mut self.roots)?;
- self.heap.assert_over_approximated_stack_roots_integrity();
+ self.heap.assert_over_approximated_stack_roots_integrity()?;
self.heap.assert_free_blocks_are_poisoned();
log::debug!("End DRC trace");
@@ -1226,19 +1261,19 @@ impl<'a> GarbageCollection<'a> for DrcCollection<'a> {
log::debug!(" -> {:.3} seconds", start.elapsed().as_secs_f64());
self.phase = DrcCollectionPhase::Sweep;
- GcProgress::Continue
+ Ok(GcProgress::Continue)
}
DrcCollectionPhase::Sweep => {
#[cfg(feature = "std")]
let start = std::time::Instant::now();
log::debug!("Begin DRC sweep");
- self.heap.assert_over_approximated_stack_roots_integrity();
+ self.heap.assert_over_approximated_stack_roots_integrity()?;
self.heap.assert_free_blocks_are_poisoned();
- self.heap.sweep(self.host_data_table);
+ self.heap.sweep(self.host_data_table)?;
- self.heap.assert_over_approximated_stack_roots_integrity();
+ self.heap.assert_over_approximated_stack_roots_integrity()?;
self.heap.assert_free_blocks_are_poisoned();
log::debug!("End DRC sweep");
@@ -1246,9 +1281,9 @@ impl<'a> GarbageCollection<'a> for DrcCollection<'a> {
log::debug!(" -> {:.3} seconds", start.elapsed().as_secs_f64());
self.phase = DrcCollectionPhase::Done;
- GcProgress::Complete
+ Ok(GcProgress::Complete)
}
- DrcCollectionPhase::Done => GcProgress::Complete,
+ DrcCollectionPhase::Done => Ok(GcProgress::Complete),
}
}
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/exnref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/exnref.rs
index 122663fc0a..d84f0187fe 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/exnref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/exnref.rs
@@ -1,4 +1,3 @@
-use super::structref::{initialize_field_impl, read_field_impl};
use crate::{
StorageType, Val,
prelude::*,
@@ -41,8 +40,10 @@ impl VMGcRef {
return false;
}
- let header = gc_heap.header(&self);
- header.kind().matches(VMGcKind::ExnRef)
+ match gc_heap.header(&self) {
+ Ok(header) => header.kind().matches(VMGcKind::ExnRef),
+ Err(_) => false,
+ }
}
/// Create a new `VMExnRef` from the given `gc_ref`.
@@ -147,9 +148,9 @@ impl VMExnRef {
layout: &GcStructLayout,
ty: &StorageType,
field: usize,
- ) -> Val {
+ ) -> Result<Val> {
let offset = layout.fields[field].offset;
- read_field_impl(self.as_gc_ref(), store, ty, offset)
+ self.as_gc_ref().read_val(store, ty, offset)
}
/// Initialize a field in this exnref that is currently uninitialized.
@@ -178,7 +179,7 @@ impl VMExnRef {
) -> Result<()> {
debug_assert!(val._matches_ty(&store, &ty.unpack())?);
let offset = layout.fields[field].offset;
- initialize_field_impl(self.as_gc_ref(), store, ty, offset, val)
+ self.as_gc_ref().initialize_val(store, ty, offset, val)
}
/// Initialize the tag referenced by this exception object.
@@ -193,11 +194,11 @@ impl VMExnRef {
let tag_offset = layouts.exception_tag_defined_offset();
let store = store.require_gc_store_mut()?;
store
- .gc_object_data(&self.0)
- .write_u32(instance_offset, instance.as_u32());
+ .gc_object_data(&self.0)?
+ .write_u32(instance_offset, instance.as_u32())?;
store
- .gc_object_data(&self.0)
- .write_u32(tag_offset, tag.as_u32());
+ .gc_object_data(&self.0)?
+ .write_u32(tag_offset, tag.as_u32())?;
Ok(())
}
@@ -208,11 +209,11 @@ impl VMExnRef {
let tag_offset = layouts.exception_tag_defined_offset();
let instance = store
.require_gc_store_mut()?
- .gc_object_data(&self.0)
- .read_u32(instance_offset);
+ .gc_object_data(&self.0)?
+ .read_u32(instance_offset)?;
let instance = InstanceId::from_u32(instance);
let store = store.require_gc_store_mut()?;
- let tag = store.gc_object_data(&self.0).read_u32(tag_offset);
+ let tag = store.gc_object_data(&self.0)?.read_u32(tag_offset)?;
let tag = DefinedTagIndex::from_u32(tag);
Ok((instance, tag))
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/externref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/externref.rs
index e0b684e366..f349928177 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/externref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/externref.rs
@@ -1,3 +1,4 @@
+use crate::Result;
use crate::runtime::vm::{GcHeap, GcStore, VMGcRef};
use core::fmt;
use wasmtime_environ::VMGcKind;
@@ -29,14 +30,23 @@ impl From<VMExternRef> for VMGcRef {
}
impl VMGcRef {
+ /// Is this `VMGcRef` pointing to an `extern`?
+ pub fn is_externref(&self, gc_heap: &(impl GcHeap + ?Sized)) -> bool {
+ if self.is_i31() {
+ return false;
+ }
+
+ match gc_heap.header(&self) {
+ Ok(header) => header.kind() == VMGcKind::ExternRef,
+ Err(_) => false,
+ }
+ }
+
/// Create a new `VMExternRef` from the given `gc_ref`.
///
/// If this is not GC reference to an `externref`, `Err(self)` is returned.
pub fn into_externref(self, gc_heap: &impl GcHeap) -> Result<VMExternRef, VMGcRef> {
- if self.is_i31() {
- return Err(self);
- }
- if gc_heap.header(&self).kind() == VMGcKind::ExternRef {
+ if self.is_externref(gc_heap) {
Ok(VMExternRef(self))
} else {
Err(self)
@@ -59,10 +69,7 @@ impl VMGcRef {
/// Get this GC reference as an `externref` reference, if it actually is an
/// `externref` reference.
pub fn as_externref(&self, gc_heap: &(impl GcHeap + ?Sized)) -> Option<&VMExternRef> {
- if self.is_i31() {
- return None;
- }
- if gc_heap.header(&self).kind() == VMGcKind::ExternRef {
+ if self.is_externref(gc_heap) {
let ptr = self as *const VMGcRef;
let ret = unsafe { &*ptr.cast() };
assert!(matches!(ret, VMExternRef(VMGcRef { .. })));
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
index f8da85eeff..8d31030751 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/null.rs
@@ -159,7 +159,7 @@ impl NullHeap {
let len = self.memory.as_ref().unwrap().byte_size();
let len = u32::try_from(len).unwrap_or(u32::MAX);
if end_of_object > len {
- return Ok(Err(u64::try_from(layout.size()).unwrap()));
+ return Ok(Err(u64::try_from(layout.size())?));
}
// Update the bump pointer, write the header, and return the GC ref.
@@ -170,7 +170,7 @@ impl NullHeap {
debug_assert_eq!(header.reserved_u26(), 0);
header.set_reserved_u26(size);
- *self.header_mut(&gc_ref) = header;
+ *self.header_mut(&gc_ref)? = header;
Ok(Ok(gc_ref))
}
@@ -243,12 +243,14 @@ unsafe impl GcHeap for NullHeap {
_host_data_table: &mut ExternRefHostDataTable,
destination: &mut Option<VMGcRef>,
source: Option<&VMGcRef>,
- ) {
+ ) -> Result<()> {
*destination = source.map(|s| s.unchecked_copy());
+ Ok(())
}
- fn expose_gc_ref_to_wasm(&mut self, _gc_ref: VMGcRef) {
+ fn expose_gc_ref_to_wasm(&mut self, _gc_ref: VMGcRef) -> Result<()> {
// Don't need to do anything special here.
+ Ok(())
}
fn alloc_externref(
@@ -259,26 +261,26 @@ unsafe impl GcHeap for NullHeap {
Ok(r) => r,
Err(bytes_needed) => return Ok(Err(bytes_needed)),
};
- self.index_mut::<VMNullExternRef>(gc_ref.as_typed_unchecked())
+ self.index_mut::<VMNullExternRef>(gc_ref.as_typed_unchecked())?
.host_data = host_data;
Ok(Ok(gc_ref.into_externref_unchecked()))
}
- fn externref_host_data(&self, externref: &VMExternRef) -> ExternRefHostDataId {
+ fn externref_host_data(&self, externref: &VMExternRef) -> Result<ExternRefHostDataId> {
let typed_ref = VMNullExternRef::typed_ref(self, externref);
- self.index(typed_ref).host_data
+ Ok(self.index(typed_ref)?.host_data)
}
- fn object_size(&self, gc_ref: &VMGcRef) -> usize {
- let size = self.header(gc_ref).reserved_u26();
- usize::try_from(size).unwrap()
+ fn object_size(&self, gc_ref: &VMGcRef) -> Result<usize> {
+ let size = self.header(gc_ref)?.reserved_u26();
+ Ok(usize::try_from(size)?)
}
- fn header(&self, gc_ref: &VMGcRef) -> &VMGcHeader {
+ fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader> {
self.index(gc_ref.as_typed_unchecked())
}
- fn header_mut(&mut self, gc_ref: &VMGcRef) -> &mut VMGcHeader {
+ fn header_mut(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcHeader> {
self.index_mut(gc_ref.as_typed_unchecked())
}
@@ -299,7 +301,9 @@ unsafe impl GcHeap for NullHeap {
self.alloc(VMGcHeader::from_kind_and_index(kind, ty), layout.layout())
}
- fn dealloc_uninit_struct_or_exn(&mut self, _struct_ref: VMGcRef) {}
+ fn dealloc_uninit_struct_or_exn(&mut self, _struct_ref: VMGcRef) -> Result<()> {
+ Ok(())
+ }
fn alloc_uninit_array(
&mut self,
@@ -310,24 +314,25 @@ unsafe impl GcHeap for NullHeap {
let layout = layout
.layout(length)
.ok_or_else(|| format_err!("allocation size too large"))?;
- self.alloc(
+ let gc_ref = match self.alloc(
VMGcHeader::from_kind_and_index(VMGcKind::ArrayRef, ty),
layout,
- )
- .map(|r| {
- r.map(|r| {
- self.index_mut::<VMNullArrayHeader>(r.as_typed_unchecked())
- .length = length;
- r.into_arrayref_unchecked()
- })
- })
+ )? {
+ Ok(r) => r,
+ Err(bytes_needed) => return Ok(Err(bytes_needed)),
+ };
+ self.index_mut::<VMNullArrayHeader>(gc_ref.as_typed_unchecked())?
+ .length = length;
+ Ok(Ok(gc_ref.into_arrayref_unchecked()))
}
- fn dealloc_uninit_array(&mut self, _array_ref: VMArrayRef) {}
+ fn dealloc_uninit_array(&mut self, _array_ref: VMArrayRef) -> Result<()> {
+ Ok(())
+ }
- fn array_len(&self, arrayref: &VMArrayRef) -> u32 {
+ fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32> {
let arrayref = VMNullArrayHeader::typed_ref(self, arrayref);
- self.index(arrayref).length
+ Ok(self.index(arrayref)?.length)
}
fn allocated_bytes(&self) -> usize {
@@ -357,8 +362,8 @@ unsafe impl GcHeap for NullHeap {
struct NullCollection {}
impl<'a> GarbageCollection<'a> for NullCollection {
- fn collect_increment(&mut self) -> GcProgress {
- GcProgress::Complete
+ fn collect_increment(&mut self) -> Result<GcProgress> {
+ Ok(GcProgress::Complete)
}
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs
index 74333e5683..299b138580 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/structref.rs
@@ -1,10 +1,8 @@
-use super::{truncate_i32_to_i8, truncate_i32_to_i16};
use crate::{
- AnyRef, ExnRef, ExternRef, Func, HeapType, RootedGcRefImpl, StorageType, Val, ValType,
+ StorageType, Val,
prelude::*,
runtime::vm::{GcHeap, GcStore, VMGcRef},
store::AutoAssertNoGc,
- vm::{FuncRefTableId, SendSyncPtr},
};
use core::fmt;
use wasmtime_environ::{GcStructLayout, VMGcKind};
@@ -42,8 +40,10 @@ impl VMGcRef {
return false;
}
- let header = gc_heap.header(&self);
- header.kind().matches(VMGcKind::StructRef)
+ match gc_heap.header(&self) {
+ Ok(header) => header.kind().matches(VMGcKind::StructRef),
+ Err(_) => false,
+ }
}
/// Create a new `VMStructRef` from the given `gc_ref`.
@@ -138,9 +138,9 @@ impl VMStructRef {
layout: &GcStructLayout,
ty: &StorageType,
field: usize,
- ) -> Val {
+ ) -> Result<Val> {
let offset = layout.fields[field].offset;
- read_field_impl(self.as_gc_ref(), store, ty, offset)
+ self.as_gc_ref().read_val(store, ty, offset)
}
/// Write the given value into this struct at the given offset.
@@ -165,82 +165,7 @@ impl VMStructRef {
debug_assert!(val._matches_ty(&store, &ty.unpack())?);
let offset = layout.fields[field].offset;
- let gcstore = store.require_gc_store_mut()?;
- let data = gcstore.gc_object_data(self.as_gc_ref());
- match val {
- Val::I32(i) if ty.is_i8() => data.write_i8(offset, truncate_i32_to_i8(i)),
- Val::I32(i) if ty.is_i16() => data.write_i16(offset, truncate_i32_to_i16(i)),
- Val::I32(i) => data.write_i32(offset, i),
- Val::I64(i) => data.write_i64(offset, i),
- Val::F32(f) => data.write_u32(offset, f),
- Val::F64(f) => data.write_u64(offset, f),
- Val::V128(v) => data.write_v128(offset, v),
-
- // For GC-managed references, we need to take care to run the
- // appropriate barriers, even when we are writing null references
- // into the struct.
- //
- // POD-read the old value into a local copy, run the GC write
- // barrier on that local copy, and then POD-write the updated
- // value back into the struct. This avoids transmuting the inner
- // data, which would probably be fine, but this approach is
- // Obviously Correct and should get us by for now. If LLVM isn't
- // able to elide some of these unnecessary copies, and this
- // method is ever hot enough, we can always come back and clean
- // it up in the future.
- Val::ExternRef(e) => {
- let raw = data.read_u32(offset);
- let mut gc_ref = VMGcRef::from_raw_u32(raw);
- let e = match e {
- Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()),
- None => None,
- };
- let store = store.require_gc_store_mut()?;
- store.write_gc_ref(&mut gc_ref, e.as_ref());
- let data = store.gc_object_data(self.as_gc_ref());
- data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()));
- }
- Val::AnyRef(a) => {
- let raw = data.read_u32(offset);
- let mut gc_ref = VMGcRef::from_raw_u32(raw);
- let a = match a {
- Some(a) => Some(a.try_gc_ref(store)?.unchecked_copy()),
- None => None,
- };
- let store = store.require_gc_store_mut()?;
- store.write_gc_ref(&mut gc_ref, a.as_ref());
- let data = store.gc_object_data(self.as_gc_ref());
- data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()));
- }
- Val::ExnRef(e) => {
- let raw = data.read_u32(offset);
- let mut gc_ref = VMGcRef::from_raw_u32(raw);
- let e = match e {
- Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()),
- None => None,
- };
- let store = store.require_gc_store_mut()?;
- store.write_gc_ref(&mut gc_ref, e.as_ref());
- let data = store.gc_object_data(self.as_gc_ref());
- data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()));
- }
-
- Val::FuncRef(f) => {
- let f = f.map(|f| SendSyncPtr::new(f.vm_func_ref(store)));
- let gcstore = store.require_gc_store_mut()?;
- let id = unsafe { gcstore.func_ref_table.intern(f) };
- gcstore
- .gc_object_data(self.as_gc_ref())
- .write_u32(offset, id.into_raw());
- }
- Val::ContRef(_) => {
- // TODO(#10248): Implement struct continuation reference field handling
- return Err(crate::format_err!(
- "setting continuation references in struct fields not yet supported"
- ));
- }
- }
- Ok(())
+ self.as_gc_ref().write_val(store, ty, offset, val)
}
/// Initialize a field in this structref that is currently uninitialized.
@@ -277,127 +202,6 @@ impl VMStructRef {
) -> Result<()> {
debug_assert!(val._matches_ty(&store, &ty.unpack())?);
let offset = layout.fields[field].offset;
- initialize_field_impl(self.as_gc_ref(), store, ty, offset, val)
- }
-}
-
-/// Read a field from a GC object at a given offset.
-///
-/// This factored-out function allows a shared implementation for both
-/// structs (this module) and exception objects.
-pub(crate) fn read_field_impl(
- gc_ref: &VMGcRef,
- store: &mut AutoAssertNoGc,
- ty: &StorageType,
- offset: u32,
-) -> Val {
- let data = store.unwrap_gc_store_mut().gc_object_data(gc_ref);
- match ty {
- StorageType::I8 => Val::I32(data.read_u8(offset).into()),
- StorageType::I16 => Val::I32(data.read_u16(offset).into()),
- StorageType::ValType(ValType::I32) => Val::I32(data.read_i32(offset)),
- StorageType::ValType(ValType::I64) => Val::I64(data.read_i64(offset)),
- StorageType::ValType(ValType::F32) => Val::F32(data.read_u32(offset)),
- StorageType::ValType(ValType::F64) => Val::F64(data.read_u64(offset)),
- StorageType::ValType(ValType::V128) => Val::V128(data.read_v128(offset)),
- StorageType::ValType(ValType::Ref(r)) => match r.heap_type().top() {
- HeapType::Extern => {
- let raw = data.read_u32(offset);
- Val::ExternRef(ExternRef::_from_raw(store, raw))
- }
- HeapType::Any => {
- let raw = data.read_u32(offset);
- Val::AnyRef(AnyRef::_from_raw(store, raw))
- }
- HeapType::Exn => {
- let raw = data.read_u32(offset);
- Val::ExnRef(ExnRef::_from_raw(store, raw))
- }
- HeapType::Func => {
- let func_ref_id = data.read_u32(offset);
- let func_ref_id = FuncRefTableId::from_raw(func_ref_id);
- let func_ref = store
- .unwrap_gc_store()
- .func_ref_table
- .get_untyped(func_ref_id);
- Val::FuncRef(unsafe {
- func_ref.map(|p| Func::from_vm_func_ref(store.id(), p.as_non_null()))
- })
- }
- otherwise => unreachable!("not a top type: {otherwise:?}"),
- },
- }
-}
-
-pub(crate) fn initialize_field_impl(
- gc_ref: &VMGcRef,
- store: &mut AutoAssertNoGc,
- ty: &StorageType,
- offset: u32,
- val: Val,
-) -> Result<()> {
- let gcstore = store.require_gc_store_mut()?;
- match val {
- Val::I32(i) if ty.is_i8() => gcstore
- .gc_object_data(gc_ref)
- .write_i8(offset, truncate_i32_to_i8(i)),
- Val::I32(i) if ty.is_i16() => gcstore
- .gc_object_data(gc_ref)
- .write_i16(offset, truncate_i32_to_i16(i)),
- Val::I32(i) => gcstore.gc_object_data(gc_ref).write_i32(offset, i),
- Val::I64(i) => gcstore.gc_object_data(gc_ref).write_i64(offset, i),
- Val::F32(f) => gcstore.gc_object_data(gc_ref).write_u32(offset, f),
- Val::F64(f) => gcstore.gc_object_data(gc_ref).write_u64(offset, f),
- Val::V128(v) => gcstore.gc_object_data(gc_ref).write_v128(offset, v),
-
- // NB: We don't need to do a write barrier when initializing a
- // field, because there is nothing being overwritten. Therefore, we
- // just the clone barrier.
- Val::ExternRef(x) => {
- let x = match x {
- None => 0,
- Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
- };
- store
- .require_gc_store_mut()?
- .gc_object_data(gc_ref)
- .write_u32(offset, x);
- }
- Val::AnyRef(x) => {
- let x = match x {
- None => 0,
- Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
- };
- store
- .require_gc_store_mut()?
- .gc_object_data(gc_ref)
- .write_u32(offset, x);
- }
- Val::ExnRef(x) => {
- let x = match x {
- None => 0,
- Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
- };
- store
- .require_gc_store_mut()?
- .gc_object_data(gc_ref)
- .write_u32(offset, x);
- }
-
- Val::FuncRef(f) => {
- let f = f.map(|f| SendSyncPtr::new(f.vm_func_ref(store)));
- let gcstore = store.require_gc_store_mut()?;
- let id = unsafe { gcstore.func_ref_table.intern(f) };
- gcstore
- .gc_object_data(gc_ref)
- .write_u32(offset, id.into_raw());
- }
- Val::ContRef(_) => {
- // TODO(#10248): Implement struct continuation reference field init handling
- return Err(crate::format_err!(
- "initializing continuation references in struct fields not yet supported"
- ));
- }
+ self.as_gc_ref().initialize_val(store, ty, offset, val)
}
- Ok(())
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/func_ref.rs b/crates/wasmtime/src/runtime/vm/gc/func_ref.rs
index 4635a5d015..dab931d546 100644
--- a/crates/wasmtime/src/runtime/vm/gc/func_ref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/func_ref.rs
@@ -9,6 +9,7 @@
//! `funcref` we got from inside the GC heap.
use crate::{
+ Result, bail_bug,
hash_map::HashMap,
type_registry::TypeRegistry,
vm::{SendSyncPtr, VMFuncRef},
@@ -67,8 +68,10 @@ impl FuncRefTable {
types: &TypeRegistry,
id: FuncRefTableId,
expected_ty: VMSharedTypeIndex,
- ) -> Option<SendSyncPtr<VMFuncRef>> {
- let f = self.slab.get(id.0).copied().expect("bad FuncRefTableId");
+ ) -> Result<Option<SendSyncPtr<VMFuncRef>>> {
+ let Some(f) = self.slab.get(id.0).copied() else {
+ bail_bug!("bad FuncRefTableId")
+ };
if let Some(f) = f {
// The safety contract for `intern` ensures that deref'ing `f` is safe.
@@ -84,7 +87,7 @@ impl FuncRefTable {
assert!(types.is_subtype(actual_ty, expected_ty));
}
- f
+ Ok(f)
}
/// Get the `VMFuncRef` associated with the given ID, without checking the
@@ -93,7 +96,10 @@ impl FuncRefTable {
/// Prefer `get_typed`. This method is only suitable for getting a
/// `VMFuncRef` as an untyped `funcref` function reference, and never as a
/// typed `(ref $some_func_type)` function reference.
- pub fn get_untyped(&self, id: FuncRefTableId) -> Option<SendSyncPtr<VMFuncRef>> {
- self.slab.get(id.0).copied().expect("bad FuncRefTableId")
+ pub fn get_untyped(&self, id: FuncRefTableId) -> Result<Option<SendSyncPtr<VMFuncRef>>> {
+ match self.slab.get(id.0).copied() {
+ Some(f) => Ok(f),
+ None => bail_bug!("bad FuncRefTableId"),
+ }
}
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_ref.rs b/crates/wasmtime/src/runtime/vm/gc/gc_ref.rs
index 54f45dc588..b6106ec742 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_ref.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_ref.rs
@@ -1,8 +1,12 @@
-use crate::prelude::*;
-use crate::runtime::vm::{GcHeap, GcStore, I31};
+use crate::runtime::vm::{FuncRefTableId, GcHeap, GcStore, I31, SendSyncPtr};
+use crate::store::AutoAssertNoGc;
+use crate::{
+ AnyRef, ExnRef, ExternRef, Func, HeapType, Result, StorageType, Val, ValType, bail_bug,
+};
use core::fmt;
use core::marker;
use core::num::NonZeroU32;
+use wasmtime_core::truncate::{truncate_i32_to_i8, truncate_i32_to_i16};
use wasmtime_environ::packed_option::ReservedValue;
use wasmtime_environ::{VMGcKind, VMSharedTypeIndex};
@@ -245,9 +249,13 @@ impl VMGcRef {
/// Get this GC reference as a u32 index into its GC heap.
///
- /// Returns `None` for `i31ref`s.
- pub fn as_heap_index(&self) -> Option<NonZeroU32> {
- if self.is_i31() { None } else { Some(self.0) }
+ /// Panics when called for `i31ref`s.
+ pub fn heap_index(&self) -> Result<NonZeroU32> {
+ if self.is_i31() {
+ bail_bug!("attempted to get heap index of i31")
+ }
+
+ Ok(self.0)
}
/// Get this GC reference as a raw, non-zero u32 value, regardless whether
@@ -273,7 +281,7 @@ impl VMGcRef {
if self.is_i31() {
return Err(self);
}
- if T::is(gc_heap.header(&self)) {
+ if self.is_typed::<T>(gc_heap) {
Ok(TypedGcRef {
gc_ref: self,
_phantom: marker::PhantomData,
@@ -308,7 +316,10 @@ impl VMGcRef {
if self.is_i31() {
return false;
}
- T::is(gc_heap.header(&self))
+ match gc_heap.header(&self) {
+ Ok(header) => T::is(header),
+ Err(_) => false,
+ }
}
/// Borrow `self` as a typed GC reference, checking that `self` actually is
@@ -321,7 +332,7 @@ impl VMGcRef {
if self.is_i31() {
return None;
}
- if T::is(gc_heap.header(&self)) {
+ if self.is_typed::<T>(gc_heap) {
let ptr = self as *const VMGcRef;
let ret = unsafe { &*ptr.cast() };
assert!(matches!(
@@ -368,7 +379,7 @@ impl VMGcRef {
if self.is_i31() {
None
} else {
- Some(gc_heap.header(self))
+ gc_heap.header(self).ok()
}
}
@@ -412,6 +423,204 @@ impl VMGcRef {
.gc_header(gc_heap)
.map_or(false, |h| h.kind().matches(VMGcKind::AnyRef))
}
+
+ pub fn read_val(
+ &self,
+ store: &mut AutoAssertNoGc,
+ ty: &StorageType,
+ offset: u32,
+ ) -> Result<Val> {
+ let data = store.require_gc_store_mut()?.gc_object_data(self)?;
+ Ok(match ty {
+ StorageType::I8 => Val::I32(data.read_u8(offset)?.into()),
+ StorageType::I16 => Val::I32(data.read_u16(offset)?.into()),
+ StorageType::ValType(ValType::I32) => Val::I32(data.read_i32(offset)?),
+ StorageType::ValType(ValType::I64) => Val::I64(data.read_i64(offset)?),
+ StorageType::ValType(ValType::F32) => Val::F32(data.read_u32(offset)?),
+ StorageType::ValType(ValType::F64) => Val::F64(data.read_u64(offset)?),
+ StorageType::ValType(ValType::V128) => Val::V128(data.read_v128(offset)?),
+ StorageType::ValType(ValType::Ref(r)) => match r.heap_type().top() {
+ HeapType::Extern => {
+ let raw = data.read_u32(offset)?;
+ Val::ExternRef(ExternRef::_from_raw(store, raw))
+ }
+ HeapType::Any => {
+ let raw = data.read_u32(offset)?;
+ Val::AnyRef(AnyRef::_from_raw(store, raw))
+ }
+ HeapType::Exn => {
+ let raw = data.read_u32(offset)?;
+ Val::ExnRef(ExnRef::_from_raw(store, raw))
+ }
+ HeapType::Func => {
+ let func_ref_id = data.read_u32(offset)?;
+ let func_ref_id = FuncRefTableId::from_raw(func_ref_id);
+ let func_ref = store
+ .unwrap_gc_store()
+ .func_ref_table
+ .get_untyped(func_ref_id)?;
+ Val::FuncRef(unsafe {
+ func_ref.map(|p| Func::from_vm_func_ref(store.id(), p.as_non_null()))
+ })
+ }
+ otherwise => bail_bug!("not a top type: {otherwise:?}"),
+ },
+ })
+ }
+
+ pub fn initialize_val(
+ &self,
+ store: &mut AutoAssertNoGc,
+ ty: &StorageType,
+ offset: u32,
+ val: Val,
+ ) -> Result<()> {
+ let gcstore = store.require_gc_store_mut()?;
+ match val {
+ Val::I32(i) if ty.is_i8() => gcstore
+ .gc_object_data(self)?
+ .write_i8(offset, truncate_i32_to_i8(i)),
+ Val::I32(i) if ty.is_i16() => gcstore
+ .gc_object_data(self)?
+ .write_i16(offset, truncate_i32_to_i16(i)),
+ Val::I32(i) => gcstore.gc_object_data(self)?.write_i32(offset, i),
+ Val::I64(i) => gcstore.gc_object_data(self)?.write_i64(offset, i),
+ Val::F32(f) => gcstore.gc_object_data(self)?.write_u32(offset, f),
+ Val::F64(f) => gcstore.gc_object_data(self)?.write_u64(offset, f),
+ Val::V128(v) => gcstore.gc_object_data(self)?.write_v128(offset, v),
+
+ // NB: We don't need to do a write barrier when initializing a
+ // field, because there is nothing being overwritten. Therefore, we
+ // just the clone barrier.
+ Val::ExternRef(x) => {
+ let x = match x {
+ None => 0,
+ Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
+ };
+ store
+ .require_gc_store_mut()?
+ .gc_object_data(self)?
+ .write_u32(offset, x)
+ }
+ Val::AnyRef(x) => {
+ let x = match x {
+ None => 0,
+ Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
+ };
+ store
+ .require_gc_store_mut()?
+ .gc_object_data(self)?
+ .write_u32(offset, x)
+ }
+ Val::ExnRef(x) => {
+ let x = match x {
+ None => 0,
+ Some(x) => x.try_clone_gc_ref(store)?.as_raw_u32(),
+ };
+ store
+ .require_gc_store_mut()?
+ .gc_object_data(self)?
+ .write_u32(offset, x)
+ }
+
+ Val::FuncRef(f) => {
+ let f = f.map(|f| SendSyncPtr::new(f.vm_func_ref(store)));
+ let gcstore = store.require_gc_store_mut()?;
+ let id = unsafe { gcstore.func_ref_table.intern(f) };
+ gcstore
+ .gc_object_data(self)?
+ .write_u32(offset, id.into_raw())
+ }
+ Val::ContRef(_) => {
+ // TODO(#10248): Implement struct continuation reference field init handling
+ bail_bug!(
+ "initializing continuation references in struct fields not yet supported"
+ );
+ }
+ }
+ }
+
+ pub fn write_val(
+ &self,
+ store: &mut AutoAssertNoGc,
+ ty: &StorageType,
+ offset: u32,
+ val: Val,
+ ) -> Result<()> {
+ let gcstore = store.require_gc_store_mut()?;
+ let data = gcstore.gc_object_data(self)?;
+ match val {
+ Val::I32(i) if ty.is_i8() => data.write_i8(offset, truncate_i32_to_i8(i)),
+ Val::I32(i) if ty.is_i16() => data.write_i16(offset, truncate_i32_to_i16(i)),
+ Val::I32(i) => data.write_i32(offset, i),
+ Val::I64(i) => data.write_i64(offset, i),
+ Val::F32(f) => data.write_u32(offset, f),
+ Val::F64(f) => data.write_u64(offset, f),
+ Val::V128(v) => data.write_v128(offset, v),
+
+ // For GC-managed references, we need to take care to run the
+ // appropriate barriers, even when we are writing null references
+ // into the struct.
+ //
+ // POD-read the old value into a local copy, run the GC write
+ // barrier on that local copy, and then POD-write the updated
+ // value back into the struct. This avoids transmuting the inner
+ // data, which would probably be fine, but this approach is
+ // Obviously Correct and should get us by for now. If LLVM isn't
+ // able to elide some of these unnecessary copies, and this
+ // method is ever hot enough, we can always come back and clean
+ // it up in the future.
+ Val::ExternRef(e) => {
+ let raw = data.read_u32(offset)?;
+ let mut gc_ref = VMGcRef::from_raw_u32(raw);
+ let e = match e {
+ Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()),
+ None => None,
+ };
+ let store = store.require_gc_store_mut()?;
+ store.write_gc_ref(&mut gc_ref, e.as_ref())?;
+ let data = store.gc_object_data(self)?;
+ data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()))
+ }
+ Val::AnyRef(a) => {
+ let raw = data.read_u32(offset)?;
+ let mut gc_ref = VMGcRef::from_raw_u32(raw);
+ let a = match a {
+ Some(a) => Some(a.try_gc_ref(store)?.unchecked_copy()),
+ None => None,
+ };
+ let store = store.require_gc_store_mut()?;
+ store.write_gc_ref(&mut gc_ref, a.as_ref())?;
+ let data = store.gc_object_data(self)?;
+ data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()))
+ }
+ Val::ExnRef(e) => {
+ let raw = data.read_u32(offset)?;
+ let mut gc_ref = VMGcRef::from_raw_u32(raw);
+ let e = match e {
+ Some(e) => Some(e.try_gc_ref(store)?.unchecked_copy()),
+ None => None,
+ };
+ let store = store.require_gc_store_mut()?;
+ store.write_gc_ref(&mut gc_ref, e.as_ref())?;
+ let data = store.gc_object_data(self)?;
+ data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()))
+ }
+
+ Val::FuncRef(f) => {
+ let f = f.map(|f| SendSyncPtr::new(f.vm_func_ref(store)));
+ let gcstore = store.require_gc_store_mut()?;
+ let id = unsafe { gcstore.func_ref_table.intern(f) };
+ gcstore
+ .gc_object_data(self)?
+ .write_u32(offset, id.into_raw())
+ }
+ Val::ContRef(_) => {
+ // TODO(#10248): Implement struct continuation reference field handling
+ bail_bug!("setting continuation references in struct fields not yet supported");
+ }
+ }
+ }
}
/// A trait implemented by all objects allocated inside a GC heap.
diff --git a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
index 6bad5b45cd..5cbfa4200c 100644
--- a/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/gc_runtime.rs
@@ -1,5 +1,6 @@
//! Traits for abstracting over our different garbage collectors.
+use crate::bail_bug;
use crate::prelude::*;
use crate::runtime::vm::{
ExternRefHostDataId, ExternRefHostDataTable, GcHeapObject, SendSyncPtr, TypedGcRef, VMArrayRef,
@@ -170,7 +171,18 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// The given `gc_ref` should not be used again.
fn drop_gc_ref(&mut self, host_data_table: &mut ExternRefHostDataTable, gc_ref: VMGcRef) {
let mut dest = Some(gc_ref);
- self.write_gc_ref(host_data_table, &mut dest, None);
+
+ // Similar to `clone_gc_ref` not being fallible this method,
+ // `drop_gc_ref`, is also not fallible as it's seen as too painful to
+ // propagate this result. The consequence is that if the GC heap is
+ // corrupted at this point it'll get a ltitle more corrupted from this
+ // operation, but that's the tradeoff we're making ignoring the error
+ // here.
+ if let Err(e) = self.write_gc_ref(host_data_table, &mut dest, None) {
+ if cfg!(debug_assertions) {
+ panic!("heap corruption detected: {e}");
+ }
+ }
}
/// Write barrier called every time the runtime overwrites a GC reference.
@@ -190,7 +202,7 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
host_data_table: &mut ExternRefHostDataTable,
destination: &mut Option<VMGcRef>,
source: Option<&VMGcRef>,
- );
+ ) -> Result<()>;
/// Read barrier called whenever a GC reference is passed from the runtime
/// to Wasm: an argument to a host-to-Wasm call, or a return from a
@@ -199,7 +211,7 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// Callers should pass a valid `VMGcRef` that belongs to the given
/// heap. Failure to do so is memory safe, but may result in general
/// failures such as panics or incorrect results.
- fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef);
+ fn expose_gc_ref_to_wasm(&mut self, gc_ref: VMGcRef) -> Result<()>;
////////////////////////////////////////////////////////////////////////////
// `externref` Methods
@@ -230,23 +242,23 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// Callers should pass a valid `externref` that belongs to the given
/// heap. Failure to do so is memory safe, but may result in general
/// failures such as panics or incorrect results.
- fn externref_host_data(&self, externref: &VMExternRef) -> ExternRefHostDataId;
+ fn externref_host_data(&self, externref: &VMExternRef) -> Result<ExternRefHostDataId>;
////////////////////////////////////////////////////////////////////////////
// Struct, array, and general GC object methods
/// Get the header of the object that `gc_ref` points to.
- fn header(&self, gc_ref: &VMGcRef) -> &VMGcHeader;
+ fn header(&self, gc_ref: &VMGcRef) -> Result<&VMGcHeader>;
/// Get the header of the object that `gc_ref` points to.
- fn header_mut(&mut self, gc_ref: &VMGcRef) -> &mut VMGcHeader;
+ fn header_mut(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcHeader>;
/// Get the size (in bytes) of the object referenced by `gc_ref`.
///
- /// # Panics
+ /// # Errors
///
- /// Panics on out of bounds or if the `gc_ref` is an `i31ref`.
- fn object_size(&self, gc_ref: &VMGcRef) -> usize;
+ /// Returns an error on out of bounds or if the `gc_ref` is an `i31ref`.
+ fn object_size(&self, gc_ref: &VMGcRef) -> Result<usize>;
/// Allocate a raw, uninitialized GC-managed object with the given header
/// and layout.
@@ -314,7 +326,7 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// that the struct's allocation can be eagerly reclaimed, and so that the
/// collector doesn't attempt to treat any of the uninitialized fields as
/// valid GC references, or something like that.
- fn dealloc_uninit_struct_or_exn(&mut self, structref: VMGcRef);
+ fn dealloc_uninit_struct_or_exn(&mut self, structref: VMGcRef) -> Result<()>;
/// * `Ok(Ok(_))`: The allocation was successful.
///
@@ -340,16 +352,16 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// that the array's allocation can be eagerly reclaimed, and so that the
/// collector doesn't attempt to treat any of the uninitialized fields as
/// valid GC references, or something like that.
- fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef);
+ fn dealloc_uninit_array(&mut self, arrayref: VMArrayRef) -> Result<()>;
/// Get the length of the given array.
///
- /// Panics on out-of-bounds accesses.
+ /// Returns an error on out-of-bounds accesses.
///
/// The given `arrayref` should be valid and of the given size. Failure to
/// do so is memory safe, but may result in general failures such as panics
/// or incorrect results.
- fn array_len(&self, arrayref: &VMArrayRef) -> u32;
+ fn array_len(&self, arrayref: &VMArrayRef) -> Result<u32>;
////////////////////////////////////////////////////////////////////////////
// Garbage Collection Methods
@@ -458,78 +470,103 @@ pub unsafe trait GcHeap: 'static + Send + Sync {
/// Index into this heap and get a shared reference to the `T` that `gc_ref`
/// points to.
///
- /// # Panics
+ /// # Errors
///
- /// Panics on out of bounds or if the `gc_ref` is an `i31ref`.
+ /// Returns an error on out of bounds or if the `gc_ref` is an `i31ref`.
#[inline]
- fn index<T>(&self, gc_ref: &TypedGcRef<T>) -> &T
+ fn index<T>(&self, gc_ref: &TypedGcRef<T>) -> Result<&T>
where
Self: Sized,
T: GcHeapObject,
{
assert!(!mem::needs_drop::<T>());
let gc_ref = gc_ref.as_untyped();
- let start = gc_ref.as_heap_index().unwrap().get();
- let start = usize::try_from(start).unwrap();
+ let start = gc_ref.heap_index()?.get();
+ let start = usize::try_from(start)?;
let len = mem::size_of::<T>();
- let slice = &self.heap_slice()[start..][..len];
- unsafe { &*(slice.as_ptr().cast::<T>()) }
+ let slice = match self.heap_slice().get(start..).and_then(|s| s.get(..len)) {
+ Some(slice) => slice,
+ None => bail_bug!("gc object out-of-bounds"),
+ };
+ if slice.as_ptr().addr() % mem::align_of::<T>() != 0 {
+ bail_bug!("gc object and/or heap misaligned");
+ }
+ Ok(unsafe { &*(slice.as_ptr().cast::<T>()) })
}
/// Index into this heap and get an exclusive reference to the `T` that
/// `gc_ref` points to.
///
- /// # Panics
+ /// # Errors
///
- /// Panics on out of bounds or if the `gc_ref` is an `i31ref`.
+ /// Returns an error on out of bounds or if the `gc_ref` is an `i31ref`.
#[inline]
- fn index_mut<T>(&mut self, gc_ref: &TypedGcRef<T>) -> &mut T
+ fn index_mut<T>(&mut self, gc_ref: &TypedGcRef<T>) -> Result<&mut T>
where
Self: Sized,
T: GcHeapObject,
{
assert!(!mem::needs_drop::<T>());
let gc_ref = gc_ref.as_untyped();
- let start = gc_ref.as_heap_index().unwrap().get();
- let start = usize::try_from(start).unwrap();
+ let start = gc_ref.heap_index()?.get();
+ let start = usize::try_from(start)?;
let len = mem::size_of::<T>();
- let slice = &mut self.heap_slice_mut()[start..][..len];
- unsafe { &mut *(slice.as_mut_ptr().cast::<T>()) }
+ let slice = match self
+ .heap_slice_mut()
+ .get_mut(start..)
+ .and_then(|s| s.get_mut(..len))
+ {
+ Some(slice) => slice,
+ None => bail_bug!("gc object out-of-bounds"),
+ };
+ assert!(slice.as_ptr().addr() % mem::align_of::<T>() == 0);
+ Ok(unsafe { &mut *(slice.as_mut_ptr().cast::<T>()) })
}
/// Get the range of bytes that the given object occupies in the heap.
///
- /// # Panics
+ /// # Errors
///
- /// Panics on out of bounds or if the `gc_ref` is an `i31ref`.
- fn object_range(&self, gc_ref: &VMGcRef) -> Range<usize> {
- let start = gc_ref.as_heap_index().unwrap().get();
- let start = usize::try_from(start).unwrap();
- let size = self.object_size(gc_ref);
- let end = start.checked_add(size).unwrap();
- start..end
+ /// Returns an error on out of bounds or if the `gc_ref` is an `i31ref`.
+ fn object_range(&self, gc_ref: &VMGcRef) -> Result<Range<usize>> {
+ let start = gc_ref.heap_index()?.get();
+ let start = usize::try_from(start)?;
+ let size = self.object_size(gc_ref)?;
+ let end = match start.checked_add(size) {
+ Some(end) => end,
+ None => bail_bug!("object size overflow"),
+ };
+ Ok(start..end)
}
/// Get a mutable borrow of the given object's data.
///
- /// # Panics
+ /// # Errors
///
- /// Panics on out-of-bounds accesses or if the `gc_ref` is an `i31ref`.
- fn gc_object_data(&self, gc_ref: &VMGcRef) -> &VMGcObjectData {
- let range = self.object_range(gc_ref);
- let data = &self.heap_slice()[range];
- data.into()
+ /// Returns an error on out-of-bounds accesses or if the `gc_ref` is an
+ /// `i31ref`.
+ fn gc_object_data(&self, gc_ref: &VMGcRef) -> Result<&VMGcObjectData> {
+ let range = self.object_range(gc_ref)?;
+ let data = match self.heap_slice().get(range) {
+ Some(data) => data,
+ None => bail_bug!("gc object out of bounds"),
+ };
+ Ok(data.into())
}
/// Get a mutable borrow of the given object's data.
///
- /// # Panics
+ /// # Errors
///
- /// Panics on out-of-bounds accesses or if the `gc_ref` is an `i31ref`.
- fn gc_object_data_mut(&mut self, gc_ref: &VMGcRef) -> &mut VMGcObjectData {
- let range = self.object_range(gc_ref);
- let data = &mut self.heap_slice_mut()[range];
- data.into()
+ /// Returns an error on out-of-bounds accesses or if the `gc_ref` is an
+ /// `i31ref`.
+ fn gc_object_data_mut(&mut self, gc_ref: &VMGcRef) -> Result<&mut VMGcObjectData> {
+ let range = self.object_range(gc_ref)?;
+ let data = match self.heap_slice_mut().get_mut(range) {
+ Some(data) => data,
+ None => bail_bug!("gc object out of bounds"),
+ };
+ Ok(data.into())
}
}
@@ -692,16 +729,22 @@ impl GcRoot<'_> {
///
/// Does NOT run GC barriers.
#[inline]
- pub fn get(&self) -> VMGcRef {
+ pub fn get(&self) -> Result<VMGcRef> {
match self.raw {
- RawGcRoot::VMGcRef(ptr) => unsafe { ptr::read(ptr.as_ptr()) },
+ RawGcRoot::VMGcRef(ptr) => Ok(unsafe { ptr::read(ptr.as_ptr()) }),
RawGcRoot::Stack(ptr) => unsafe {
let raw: u32 = ptr::read(ptr.as_ptr());
- VMGcRef::from_raw_u32(raw).expect("non-null")
+ match VMGcRef::from_raw_u32(raw) {
+ Some(r) => Ok(r),
+ None => bail_bug!("stacked contained null gcref"),
+ }
},
RawGcRoot::ValRaw(ptr) => unsafe {
let val: ValRaw = ptr::read(ptr.as_ptr());
- val.get_vmgcref().expect("non-null")
+ match val.get_vmgcref() {
+ Some(r) => Ok(r),
+ None => bail_bug!("val contained null gcref"),
+ }
},
}
}
@@ -751,17 +794,17 @@ pub trait GarbageCollection<'a>: Send + Sync {
///
/// The mutator does *not* run in between increments. This method exists
/// solely to allow cooperative yielding
- fn collect_increment(&mut self) -> GcProgress;
+ fn collect_increment(&mut self) -> Result<GcProgress>;
/// Run this GC process to completion.
///
/// Keeps calling `collect_increment` in a loop until the GC process is
/// complete.
- fn collect(&mut self) {
+ fn collect(&mut self) -> Result<()> {
loop {
- match self.collect_increment() {
+ match self.collect_increment()? {
GcProgress::Continue => continue,
- GcProgress::Complete => return,
+ GcProgress::Complete => return Ok(()),
}
}
}
@@ -781,21 +824,21 @@ pub async fn collect_async<'a>(
mut collection: Box<dyn GarbageCollection<'a> + 'a>,
asyncness: Asyncness,
yield_fn: impl AsyncFn(),
-) {
+) -> Result<()> {
#[cfg(not(feature = "async"))]
{
_ = yield_fn;
}
loop {
- match collection.collect_increment() {
+ match collection.collect_increment()? {
GcProgress::Continue => {
if asyncness != Asyncness::No {
#[cfg(feature = "async")]
yield_fn().await
}
}
- GcProgress::Complete => return,
+ GcProgress::Complete => return Ok(()),
}
}
}
diff --git a/crates/wasmtime/src/runtime/vm/gc/host_data.rs b/crates/wasmtime/src/runtime/vm/gc/host_data.rs
index 570cf6b15a..ae097ed691 100644
--- a/crates/wasmtime/src/runtime/vm/gc/host_data.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/host_data.rs
@@ -13,6 +13,7 @@
//! less catastrophic than doing an indirect call to an attacker-controlled
//! function pointer.
+use crate::bail_bug;
use crate::prelude::*;
use core::any::Any;
use wasmtime_core::{
@@ -50,21 +51,29 @@ impl ExternRefHostDataTable {
}
/// Deallocate an `externref` host data value.
- pub fn dealloc(&mut self, id: ExternRefHostDataId) -> Box<dyn Any + Send + Sync> {
+ pub fn dealloc(&mut self, id: ExternRefHostDataId) -> Result<Box<dyn Any + Send + Sync>> {
+ // Verify this exists before deleting it
+ self.get(id)?;
log::trace!("deallocated externref host data: {id:?}");
- self.slab.dealloc(id.0)
+ Ok(self.slab.dealloc(id.0))
}
/// Get a shared borrow of the host data associated with the given ID.
- pub fn get(&self, id: ExternRefHostDataId) -> &(dyn Any + Send + Sync) {
- let data: &Box<dyn Any + Send + Sync> = self.slab.get(id.0).unwrap();
- deref_box(data)
+ pub fn get(&self, id: ExternRefHostDataId) -> Result<&(dyn Any + Send + Sync)> {
+ let data: &Box<dyn Any + Send + Sync> = match self.slab.get(id.0) {
+ Some(data) => data,
+ None => bail_bug!("invalid `ExternRefHostDataId`"),
+ };
+ Ok(deref_box(data))
}
/// Get a mutable borrow of the host data associated with the given ID.
- pub fn get_mut(&mut self, id: ExternRefHostDataId) -> &mut (dyn Any + Send + Sync) {
- let data: &mut Box<dyn Any + Send + Sync> = self.slab.get_mut(id.0).unwrap();
- deref_box_mut(data)
+ pub fn get_mut(&mut self, id: ExternRefHostDataId) -> Result<&mut (dyn Any + Send + Sync)> {
+ let data: &mut Box<dyn Any + Send + Sync> = match self.slab.get_mut(id.0) {
+ Some(data) => data,
+ None => bail_bug!("invalid `ExternRefHostDataId`"),
+ };
+ Ok(deref_box_mut(data))
}
}
@@ -78,9 +87,12 @@ mod tests {
let x = 42_u32;
let id = table.alloc(Box::new(x));
- assert!(table.get(id).is::<u32>());
- assert_eq!(*table.get(id).downcast_ref::<u32>().unwrap(), 42);
- assert!(table.get_mut(id).is::<u32>());
- assert_eq!(*table.get_mut(id).downcast_ref::<u32>().unwrap(), 42);
+ assert!(table.get(id).unwrap().is::<u32>());
+ assert_eq!(*table.get(id).unwrap().downcast_ref::<u32>().unwrap(), 42);
+ assert!(table.get_mut(id).unwrap().is::<u32>());
+ assert_eq!(
+ *table.get_mut(id).unwrap().downcast_ref::<u32>().unwrap(),
+ 42
+ );
}
}
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index e51ac3944e..06fdf0aef1 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -421,7 +421,7 @@ fn drop_gc_ref(store: &mut dyn VMStore, _instance: InstanceId, gc_ref: u32) {
fn force_gc(store: &mut dyn VMStore, _instance: InstanceId) -> Result<()> {
let store = store.store_opaque_mut();
block_on!(store, async |store, asyncness| {
- store.gc(None, None, None, asyncness).await;
+ store.gc(None, None, None, asyncness).await?;
Ok::<(), Error>(())
})??;
Ok(())
@@ -504,7 +504,7 @@ fn gc_alloc_raw(
let opaque = store.store_opaque_mut();
if let Some(gc_store) = opaque.try_gc_store_mut() {
if let Ok(gc_ref) = gc_store.alloc_raw(header, layout)? {
- let raw = gc_store.expose_gc_ref_to_wasm(gc_ref);
+ let raw = gc_store.expose_gc_ref_to_wasm(gc_ref)?;
return Ok(raw);
}
}
@@ -520,8 +520,7 @@ fn gc_alloc_raw(
})
.await?;
- let raw = store.unwrap_gc_store_mut().expose_gc_ref_to_wasm(gc_ref);
- Ok(raw)
+ store.unwrap_gc_store_mut().expose_gc_ref_to_wasm(gc_ref)
})?
}
@@ -562,7 +561,7 @@ fn get_interned_func_ref(
instance: InstanceId,
func_ref_id: u32,
module_interned_type_index: u32,
-) -> *mut u8 {
+) -> Result<*mut u8> {
use super::FuncRefTableId;
use crate::store::AutoAssertNoGc;
use wasmtime_environ::{ModuleInternedTypeIndex, packed_option::ReservedValue};
@@ -576,7 +575,7 @@ fn get_interned_func_ref(
store
.unwrap_gc_store()
.func_ref_table
- .get_untyped(func_ref_id)
+ .get_untyped(func_ref_id)?
} else {
let types = store.engine().signatures();
let engine_ty = store
@@ -585,10 +584,10 @@ fn get_interned_func_ref(
store
.unwrap_gc_store()
.func_ref_table
- .get_typed(types, func_ref_id, engine_ty)
+ .get_typed(types, func_ref_id, engine_ty)?
};
- func_ref.map_or(core::ptr::null_mut(), |f| f.as_ptr().cast())
+ Ok(func_ref.map_or(core::ptr::null_mut(), |f| f.as_ptr().cast()))
}
#[cfg(feature = "gc")]
@@ -600,10 +599,7 @@ fn array_new_elem(
src: u32,
len: u32,
) -> Result<core::num::NonZeroU32> {
- use crate::{
- ArrayRef, ArrayRefPre, ArrayType, OpaqueRootScope, RootedGcRefImpl, Val,
- store::AutoAssertNoGc,
- };
+ use crate::{ArrayRef, ArrayRefPre, ArrayType, OpaqueRootScope, Val, store::AutoAssertNoGc};
use wasmtime_environ::ModuleInternedTypeIndex;
// Convert indices to their typed forms.
@@ -655,8 +651,7 @@ fn array_new_elem(
let mut store = AutoAssertNoGc::new(&mut store);
let gc_ref = array.try_clone_gc_ref(&mut store)?;
- let raw = store.unwrap_gc_store_mut().expose_gc_ref_to_wasm(gc_ref);
- Ok(raw)
+ store.unwrap_gc_store_mut().expose_gc_ref_to_wasm(gc_ref)
})?
}
@@ -1274,7 +1269,7 @@ fn get_instance_id(_store: &mut dyn VMStore, instance: InstanceId) -> u32 {
#[cfg(feature = "gc")]
fn throw_ref(store: &mut dyn VMStore, _instance: InstanceId, exnref: u32) -> Result<()> {
let exnref = VMGcRef::from_raw_u32(exnref).ok_or_else(|| Trap::NullReference)?;
- Err(store.set_pending_exception(&exnref).into())
+ Err(store.set_pending_exception(&exnref))
}
fn breakpoint(store: &mut dyn VMStore, _instance: InstanceId) -> Result<()> {
diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs
index eb5f46d15d..86f43c8deb 100644
--- a/crates/wasmtime/src/runtime/vm/table.rs
+++ b/crates/wasmtime/src/runtime/vm/table.rs
@@ -568,12 +568,12 @@ impl Table {
dst: u64,
val: Option<&VMGcRef>,
len: u64,
- ) -> Result<(), Trap> {
+ ) -> Result<()> {
let range = self.validate_fill(dst, len)?;
// Clone the init GC reference into each table slot.
for slot in &mut self.gc_refs_mut()[range] {
- GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), slot, val);
+ GcStore::write_gc_ref_optional_store(gc_store.as_deref_mut(), slot, val)?;
}
Ok(())
@@ -774,14 +774,14 @@ impl Table {
store: Option<&mut GcStore>,
index: u64,
elem: Option<&VMGcRef>,
- ) -> Result<(), Trap> {
+ ) -> Result<()> {
let trap = Trap::TableOutOfBounds;
let index: usize = index.try_into().map_err(|_| trap)?;
GcStore::write_gc_ref_optional_store(
store,
self.gc_refs_mut().get_mut(index).ok_or(trap)?,
elem,
- );
+ )?;
Ok(())
}
diff --git a/crates/wasmtime/src/runtime/vm/throw.rs b/crates/wasmtime/src/runtime/vm/throw.rs
index 643613f8bd..a2b24f9da2 100644
--- a/crates/wasmtime/src/runtime/vm/throw.rs
+++ b/crates/wasmtime/src/runtime/vm/throw.rs
@@ -1,9 +1,12 @@
//! Exception-throw logic for Wasm exceptions.
-use super::{VMContext, VMStore};
-use crate::{store::AutoAssertNoGc, vm::Instance};
+use super::VMContext;
+use crate::{
+ store::{InstanceId, StoreOpaque},
+ vm::Instance,
+};
use core::ptr::NonNull;
-use wasmtime_environ::TagIndex;
+use wasmtime_environ::{DefinedTagIndex, TagIndex};
use wasmtime_unwinder::{Frame, Handler};
/// Compute the target of the pending exception on the store.
@@ -12,14 +15,11 @@ use wasmtime_unwinder::{Frame, Handler};
///
/// The stored last-exit state in `store` either must be valid, or
/// must have a zeroed exit FP if no Wasm is on the stack.
-pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {
- let mut nogc = AutoAssertNoGc::new(store.store_opaque_mut());
-
- // Get the tag identity relative to the store.
-
- let (throwing_tag_instance_id, throwing_tag_defined_tag_index) = nogc
- .pending_exception_tag_and_instance()
- .expect("Only invoked when an exception is pending");
+pub unsafe fn compute_handler(
+ store: &mut StoreOpaque,
+ throwing_tag_instance_id: InstanceId,
+ throwing_tag_defined_tag_index: DefinedTagIndex,
+) -> Option<Handler> {
log::trace!(
"throwing: tag defined in instance {throwing_tag_instance_id:?} defined-tag {throwing_tag_defined_tag_index:?}"
);
@@ -27,9 +27,9 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {
// Get the state needed for a stack walk.
let (exit_pc, exit_fp, entry_fp) = unsafe {
(
- *nogc.vm_store_context().last_wasm_exit_pc.get(),
- nogc.vm_store_context().last_wasm_exit_fp(),
- *nogc.vm_store_context().last_wasm_entry_fp.get(),
+ *store.vm_store_context().last_wasm_exit_pc.get(),
+ store.vm_store_context().last_wasm_exit_fp(),
+ *store.vm_store_context().last_wasm_entry_fp.get(),
)
};
@@ -50,7 +50,7 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {
frame.fp(),
frame.pc()
);
- let (module, rel_pc) = nogc.modules().module_and_code_by_pc(frame.pc())?;
+ let (module, rel_pc) = store.modules().module_and_code_by_pc(frame.pc())?;
let et = module.module().exception_table();
let (frame_offset, handlers) = et.lookup_pc(u32::try_from(rel_pc).unwrap());
let fp_to_sp = frame_offset.map(|frame_offset| -isize::try_from(frame_offset).unwrap());
@@ -88,7 +88,7 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {
// other active borrows, and internally the borrow is scoped
// to this one block.
let (handler_tag_instance, handler_tag_index) = unsafe {
- let store_id = nogc.id();
+ let store_id = store.id();
let instance = Instance::from_vmctx(frame_vmctx);
let tag = instance
.as_ref()
@@ -116,7 +116,7 @@ pub unsafe fn compute_handler(store: &mut dyn VMStore) -> Option<Handler> {
}
None
};
- let unwinder = nogc.unwinder();
+ let unwinder = store.unwinder();
let action = unsafe { Handler::find(unwinder, handler_lookup, exit_pc, exit_fp, entry_fp) };
log::trace!("throw action: {action:?}");
action
diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs
index 694f9d14d3..01539f8bcc 100644
--- a/crates/wasmtime/src/runtime/vm/traphandlers.rs
+++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs
@@ -244,7 +244,11 @@ where
// generate the return value of this function. This is the
// conditionally, below, passed to `catch_unwind`.
let f = move || match f(store) {
- Ok(ret) => (ret.into_abi(), None),
+ Ok(ret) => {
+ let abi = ret.into_abi();
+ debug_assert!(abi != T::SENTINEL);
+ (abi, None)
+ }
Err(reason) => (T::SENTINEL, Some(UnwindReason::from(reason))),
};
@@ -280,7 +284,7 @@ where
/// `into_abi` function.
pub unsafe trait HostResultHasUnwindSentinel {
/// The Cranelift-understood ABI of this value (should not be `Self`).
- type Abi: Copy;
+ type Abi: Copy + PartialEq;
/// A value that indicates that an unwind should happen and is tested for in
/// Cranelift-generated code.
@@ -340,6 +344,14 @@ unsafe impl HostResultHasUnwindSentinel for bool {
}
}
+unsafe impl HostResultHasUnwindSentinel for *mut u8 {
+ type Abi = *mut u8;
+ const SENTINEL: Self::Abi = ptr::without_provenance_mut(usize::MAX);
+ fn into_abi(self) -> Self::Abi {
+ self
+ }
+}
+
/// A helper structure to schlep from this module to the
/// `crate::trap::from_runtime_box` function.
///
@@ -820,8 +832,8 @@ impl CallThreadState {
// required by its stack-walking logic.
#[cfg(feature = "gc")]
if err.is::<ThrownException>()
- && store.has_pending_exception()
- && let Some(catch) = unsafe { compute_handler(store) }
+ && let Some((instance, tag)) = store.pending_exception_tag_and_instance()
+ && let Some(catch) = unsafe { compute_handler(store, instance, tag) }
{
handler = catch;
// Take the pending exception at this time and use it as
diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs
index eedf3c7d9b..ab097b978e 100644
--- a/crates/wasmtime/src/runtime/vm/vmcontext.rs
+++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs
@@ -4,6 +4,7 @@
mod vm_host_func_context;
pub use self::vm_host_func_context::VMArrayCallHostFuncContext;
+use crate::bail_bug;
use crate::prelude::*;
use crate::runtime::vm::{InterpreterRef, VMGcRef, VmPtr, VmSafe, f32x4, f64x2, i8x16};
use crate::store::StoreOpaque;
@@ -620,17 +621,17 @@ impl VMGlobalDefinition {
WasmValType::Ref(r) => match r.heap_type.top() {
WasmHeapTopType::Extern => {
let r = VMGcRef::from_raw_u32(raw.get_externref());
- global.init_gc_ref(store, r.as_ref())
+ global.init_gc_ref(store, r.as_ref())?
}
WasmHeapTopType::Any => {
let r = VMGcRef::from_raw_u32(raw.get_anyref());
- global.init_gc_ref(store, r.as_ref())
+ global.init_gc_ref(store, r.as_ref())?
}
WasmHeapTopType::Func => *global.as_func_ref_mut() = raw.get_funcref().cast(),
WasmHeapTopType::Cont => *global.as_func_ref_mut() = raw.get_funcref().cast(), // TODO(#10248): temporary hack.
WasmHeapTopType::Exn => {
let r = VMGcRef::from_raw_u32(raw.get_exnref());
- global.init_gc_ref(store, r.as_ref())
+ global.init_gc_ref(store, r.as_ref())?
}
},
}
@@ -673,7 +674,7 @@ impl VMGlobalDefinition {
}
}),
WasmHeapTopType::Func => ValRaw::funcref(self.as_func_ref().cast()),
- WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
+ WasmHeapTopType::Cont => bail_bug!("unimplemented"), // FIXME: #10248 stack switching support.
},
})
}
@@ -804,7 +805,11 @@ impl VMGlobalDefinition {
}
/// Initialize a global to the given GC reference.
- pub unsafe fn init_gc_ref(&mut self, store: &mut StoreOpaque, gc_ref: Option<&VMGcRef>) {
+ pub unsafe fn init_gc_ref(
+ &mut self,
+ store: &mut StoreOpaque,
+ gc_ref: Option<&VMGcRef>,
+ ) -> Result<()> {
let dest = unsafe {
&mut *(self
.storage
@@ -817,7 +822,11 @@ impl VMGlobalDefinition {
}
/// Write a GC reference into this global value.
- pub unsafe fn write_gc_ref(&mut self, store: &mut StoreOpaque, gc_ref: Option<&VMGcRef>) {
+ pub unsafe fn write_gc_ref(
+ &mut self,
+ store: &mut StoreOpaque,
+ gc_ref: Option<&VMGcRef>,
+ ) -> Result<()> {
let dest = unsafe { &mut *(self.storage.as_mut().as_mut_ptr().cast::<Option<VMGcRef>>()) };
store.write_gc_ref(dest, gc_ref)
}
diff --git a/tests/all/missing_async.rs b/tests/all/missing_async.rs
index 722ed13fac..878458f5fc 100644
--- a/tests/all/missing_async.rs
+++ b/tests/all/missing_async.rs
@@ -207,7 +207,7 @@ async fn async_disallows_typed_func_call() -> Result<()> {
async fn async_disallows_gc() -> Result<()> {
let mut store = async_limiter_store();
assert!(store.gc(None).is_err());
- store.gc_async(None).await;
+ store.gc_async(None).await?;
Ok(())
}