Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Cache GC subtype checks per-store (#13860)

Alex Crichton8516b80337e6
diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rs
index 67b4964987..0e418455b3 100644
--- a/crates/wasmtime/src/runtime/store/gc.rs
+++ b/crates/wasmtime/src/runtime/store/gc.rs
@@ -2,14 +2,15 @@
use crate::RootSet;
use crate::error::{Context, ensure};
+use crate::hash_map::HashMap;
use crate::module::ModuleRegistry;
use crate::store::{
Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter, yield_now,
};
use crate::type_registry::RegisteredType;
use crate::vm::{
- self, Backtrace, Frame, GcRootsList, GcStore, InstanceAllocationRequest, SendSyncPtr,
- StoreGcHostAllocTypes, TraceInfo, VMGcRef,
+ self, Backtrace, Frame, GcRootsList, GcStore, InstanceAllocationRequest, NopHasher,
+ SendSyncPtr, StoreGcHostAllocTypes, TraceInfo, VMGcRef,
};
use crate::{
ExnRef, GcHeapOutOfMemory, Result, Rooted, Store, StoreContextMut, ThrownException, bail,
@@ -42,6 +43,27 @@ pub(crate) struct StoreGcData {
/// refinement of `VMGcRef`, but rooting APIs right now make it difficult to
/// work with that directly so this is stored as `VMGcRef` instead.
pending_exception: Option<VMGcRef>,
+
+ /// A store-local cache of engine-level subtype checks.
+ ///
+ /// Dynamic subtype checks (e.g. for `ref.cast` instructions) consult the
+ /// engine's type registry, which is protected by a read-write lock.
+ /// Testing locally shows that parallel readers take a significant
+ /// performance hit when using this lock (as measured with `wasmtime
+ /// serve`). Thus this cache is intended to serve as a fast path where the
+ /// lock need not be hit at all.
+ ///
+ /// Note that the size of this cache is currently bounded to a fixed size.
+ /// Once this cache is full then all future consultations which aren't in
+ /// the cache go upstream to the engine itself (slow). In the future
+ /// this'll either be removed entirely (see #13484) or will have some sort
+ /// of LRU-like behavior.
+ ///
+ /// FIXME(#13484) this field is a temporary workaround for a "true
+ /// solution" where supertypes are stored inline in a compiled-code-visible
+ /// location which means that a libcall isn't needed at all (nor
+ /// synchronization) to determine subtype/supertype relationships.
+ subtype_check_cache: HashMap<u64, bool, NopHasher>,
}
impl<T> Store<T> {
@@ -970,6 +992,36 @@ impl StoreOpaque {
.gc_roots
.exit_lifo_scope(self.gc_store.as_mut(), scope);
}
+
+ /// Is type `sub` a subtype of `sup`?
+ ///
+ /// Equivalent to `self.engine().signatures().is_subtype(sub, sup)` but
+ /// caches results store-locally to avoid contention on the engine's type
+ /// registry lock. See the documentation of the `subtype_check_cache` field
+ /// for details.
+ pub(crate) fn is_subtype_cached(
+ &mut self,
+ sub: wasmtime_environ::VMSharedTypeIndex,
+ sup: wasmtime_environ::VMSharedTypeIndex,
+ ) -> bool {
+ const MAX_SIZE: usize = 1 << 16; // 64k entries
+
+ let key = (u64::from(sub.as_u32()) << 32) | u64::from(sup.as_u32());
+ let engine_answer = || self.engine.signatures().is_subtype(sub, sup);
+ if self.gc_data.subtype_check_cache.len() < MAX_SIZE {
+ *self
+ .gc_data
+ .subtype_check_cache
+ .entry(key)
+ .or_insert_with(engine_answer)
+ } else {
+ self.gc_data
+ .subtype_check_cache
+ .get(&key)
+ .copied()
+ .unwrap_or_else(engine_answer)
+ }
+ }
}
/// RAII type to temporarily disable the GC zeal allocation counter.
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled.rs b/crates/wasmtime/src/runtime/vm/gc/enabled.rs
index fc8bebb58a..c5800046da 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled.rs
@@ -28,3 +28,57 @@ pub use null::*;
mod copying;
#[cfg(feature = "gc-copying")]
pub use copying::*;
+
+/// A hasher that doesn't hash, for use in the trace-info hash map, where we are
+/// just using scalar keys and aren't overly concerned with collision-based DoS.
+#[derive(Default)]
+pub struct NopHasher(u64);
+
+impl core::hash::BuildHasher for NopHasher {
+ type Hasher = Self;
+
+ #[inline]
+ fn build_hasher(&self) -> Self::Hasher {
+ NopHasher::default()
+ }
+}
+
+impl core::hash::Hasher for NopHasher {
+ #[inline]
+ fn finish(&self) -> u64 {
+ self.0
+ }
+
+ #[inline]
+ fn write(&mut self, bytes: &[u8]) {
+ let mut hash = self.0.to_ne_bytes();
+ let n = hash.len().min(bytes.len());
+ hash[..n].copy_from_slice(bytes);
+ self.0 = u64::from_ne_bytes(hash);
+ }
+
+ #[inline]
+ fn write_u8(&mut self, i: u8) {
+ self.write_u64(i.into());
+ }
+
+ #[inline]
+ fn write_u16(&mut self, i: u16) {
+ self.write_u64(i.into())
+ }
+
+ #[inline]
+ fn write_u32(&mut self, i: u32) {
+ self.write_u64(i.into())
+ }
+
+ #[inline]
+ fn write_u64(&mut self, i: u64) {
+ self.0 = i;
+ }
+
+ #[inline]
+ fn write_usize(&mut self, i: usize) {
+ self.write_u64(i.try_into().unwrap());
+ }
+}
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs
index 0fd20c6bd3..ae45b0cc06 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs
@@ -6,64 +6,9 @@
use crate::hash_map::{Entry, HashMap};
use crate::module::RegisteredModuleId;
-use crate::vm::{GcStoreTraceState, TraceInfo};
-use core::hash::BuildHasher;
+use crate::vm::{GcStoreTraceState, NopHasher, TraceInfo};
use wasmtime_environ::{ModuleInternedTypeIndex, VMSharedTypeIndex};
-/// A hasher that doesn't hash, for use in the trace-info hash map, where we are
-/// just using scalar keys and aren't overly concerned with collision-based DoS.
-#[derive(Default)]
-struct NopHasher(u64);
-
-impl BuildHasher for NopHasher {
- type Hasher = Self;
-
- #[inline]
- fn build_hasher(&self) -> Self::Hasher {
- NopHasher::default()
- }
-}
-
-impl core::hash::Hasher for NopHasher {
- #[inline]
- fn finish(&self) -> u64 {
- self.0
- }
-
- #[inline]
- fn write(&mut self, bytes: &[u8]) {
- let mut hash = self.0.to_ne_bytes();
- let n = hash.len().min(bytes.len());
- hash[..n].copy_from_slice(bytes);
- self.0 = u64::from_ne_bytes(hash);
- }
-
- #[inline]
- fn write_u8(&mut self, i: u8) {
- self.write_u64(i.into());
- }
-
- #[inline]
- fn write_u16(&mut self, i: u16) {
- self.write_u64(i.into())
- }
-
- #[inline]
- fn write_u32(&mut self, i: u32) {
- self.write_u64(i.into())
- }
-
- #[inline]
- fn write_u64(&mut self, i: u64) {
- self.0 = i;
- }
-
- #[inline]
- fn write_usize(&mut self, i: usize) {
- self.write_u64(i.try_into().unwrap());
- }
-}
-
#[derive(Clone, Copy)]
enum TraceInfoLoc {
HostType(VMSharedTypeIndex),
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index f8641c87d6..6964e405d6 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -615,11 +615,10 @@ fn is_subtype(
let actual = VMSharedTypeIndex::from_u32(actual_engine_type);
let expected = VMSharedTypeIndex::from_u32(expected_engine_type);
-
- let is_subtype: bool = store.engine().signatures().is_subtype(actual, expected);
+ let is_subtype = store.is_subtype_cached(actual, expected);
log::trace!("is_subtype(actual={actual:?}, expected={expected:?}) -> {is_subtype}",);
- is_subtype as u32
+ u32::from(is_subtype)
}
// Implementation of `memory.atomic.notify` for locally defined memories.