A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cache GC subtype checks per-store (#13860)
Alex Crichton8516b80337e6
diff --git a/crates/wasmtime/src/runtime/store/gc.rs b/crates/wasmtime/src/runtime/store/gc.rsindex 67b4964987..0e418455b3 100644--- a/crates/wasmtime/src/runtime/store/gc.rs+++ b/crates/wasmtime/src/runtime/store/gc.rs@@ -2,14 +2,15 @@use crate::RootSet;use crate::error::{Context, ensure};+use crate::hash_map::HashMap;use crate::module::ModuleRegistry;use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter, yield_now,};use crate::type_registry::RegisteredType;use crate::vm::{- self, Backtrace, Frame, GcRootsList, GcStore, InstanceAllocationRequest, SendSyncPtr,- StoreGcHostAllocTypes, TraceInfo, VMGcRef,+ self, Backtrace, Frame, GcRootsList, GcStore, InstanceAllocationRequest, NopHasher,+ SendSyncPtr, StoreGcHostAllocTypes, TraceInfo, VMGcRef,};use crate::{ExnRef, GcHeapOutOfMemory, Result, Rooted, Store, StoreContextMut, ThrownException, bail,@@ -42,6 +43,27 @@ pub(crate) struct StoreGcData {/// refinement of `VMGcRef`, but rooting APIs right now make it difficult to/// work with that directly so this is stored as `VMGcRef` instead.pending_exception: Option<VMGcRef>,++ /// A store-local cache of engine-level subtype checks.+ ///+ /// Dynamic subtype checks (e.g. for `ref.cast` instructions) consult the+ /// engine's type registry, which is protected by a read-write lock.+ /// Testing locally shows that parallel readers take a significant+ /// performance hit when using this lock (as measured with `wasmtime+ /// serve`). Thus this cache is intended to serve as a fast path where the+ /// lock need not be hit at all.+ ///+ /// Note that the size of this cache is currently bounded to a fixed size.+ /// Once this cache is full then all future consultations which aren't in+ /// the cache go upstream to the engine itself (slow). In the future+ /// this'll either be removed entirely (see #13484) or will have some sort+ /// of LRU-like behavior.+ ///+ /// FIXME(#13484) this field is a temporary workaround for a "true+ /// solution" where supertypes are stored inline in a compiled-code-visible+ /// location which means that a libcall isn't needed at all (nor+ /// synchronization) to determine subtype/supertype relationships.+ subtype_check_cache: HashMap<u64, bool, NopHasher>,}impl<T> Store<T> {@@ -970,6 +992,36 @@ impl StoreOpaque {.gc_roots.exit_lifo_scope(self.gc_store.as_mut(), scope);}++ /// Is type `sub` a subtype of `sup`?+ ///+ /// Equivalent to `self.engine().signatures().is_subtype(sub, sup)` but+ /// caches results store-locally to avoid contention on the engine's type+ /// registry lock. See the documentation of the `subtype_check_cache` field+ /// for details.+ pub(crate) fn is_subtype_cached(+ &mut self,+ sub: wasmtime_environ::VMSharedTypeIndex,+ sup: wasmtime_environ::VMSharedTypeIndex,+ ) -> bool {+ const MAX_SIZE: usize = 1 << 16; // 64k entries++ let key = (u64::from(sub.as_u32()) << 32) | u64::from(sup.as_u32());+ let engine_answer = || self.engine.signatures().is_subtype(sub, sup);+ if self.gc_data.subtype_check_cache.len() < MAX_SIZE {+ *self+ .gc_data+ .subtype_check_cache+ .entry(key)+ .or_insert_with(engine_answer)+ } else {+ self.gc_data+ .subtype_check_cache+ .get(&key)+ .copied()+ .unwrap_or_else(engine_answer)+ }+ }}/// RAII type to temporarily disable the GC zeal allocation counter.diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled.rs b/crates/wasmtime/src/runtime/vm/gc/enabled.rsindex fc8bebb58a..c5800046da 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled.rs@@ -28,3 +28,57 @@ pub use null::*;mod copying;#[cfg(feature = "gc-copying")]pub use copying::*;++/// A hasher that doesn't hash, for use in the trace-info hash map, where we are+/// just using scalar keys and aren't overly concerned with collision-based DoS.+#[derive(Default)]+pub struct NopHasher(u64);++impl core::hash::BuildHasher for NopHasher {+ type Hasher = Self;++ #[inline]+ fn build_hasher(&self) -> Self::Hasher {+ NopHasher::default()+ }+}++impl core::hash::Hasher for NopHasher {+ #[inline]+ fn finish(&self) -> u64 {+ self.0+ }++ #[inline]+ fn write(&mut self, bytes: &[u8]) {+ let mut hash = self.0.to_ne_bytes();+ let n = hash.len().min(bytes.len());+ hash[..n].copy_from_slice(bytes);+ self.0 = u64::from_ne_bytes(hash);+ }++ #[inline]+ fn write_u8(&mut self, i: u8) {+ self.write_u64(i.into());+ }++ #[inline]+ fn write_u16(&mut self, i: u16) {+ self.write_u64(i.into())+ }++ #[inline]+ fn write_u32(&mut self, i: u32) {+ self.write_u64(i.into())+ }++ #[inline]+ fn write_u64(&mut self, i: u64) {+ self.0 = i;+ }++ #[inline]+ fn write_usize(&mut self, i: usize) {+ self.write_u64(i.try_into().unwrap());+ }+}diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rsindex 0fd20c6bd3..ae45b0cc06 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/trace_infos.rs@@ -6,64 +6,9 @@use crate::hash_map::{Entry, HashMap};use crate::module::RegisteredModuleId;-use crate::vm::{GcStoreTraceState, TraceInfo};-use core::hash::BuildHasher;+use crate::vm::{GcStoreTraceState, NopHasher, TraceInfo};use wasmtime_environ::{ModuleInternedTypeIndex, VMSharedTypeIndex};-/// A hasher that doesn't hash, for use in the trace-info hash map, where we are-/// just using scalar keys and aren't overly concerned with collision-based DoS.-#[derive(Default)]-struct NopHasher(u64);--impl BuildHasher for NopHasher {- type Hasher = Self;-- #[inline]- fn build_hasher(&self) -> Self::Hasher {- NopHasher::default()- }-}--impl core::hash::Hasher for NopHasher {- #[inline]- fn finish(&self) -> u64 {- self.0- }-- #[inline]- fn write(&mut self, bytes: &[u8]) {- let mut hash = self.0.to_ne_bytes();- let n = hash.len().min(bytes.len());- hash[..n].copy_from_slice(bytes);- self.0 = u64::from_ne_bytes(hash);- }-- #[inline]- fn write_u8(&mut self, i: u8) {- self.write_u64(i.into());- }-- #[inline]- fn write_u16(&mut self, i: u16) {- self.write_u64(i.into())- }-- #[inline]- fn write_u32(&mut self, i: u32) {- self.write_u64(i.into())- }-- #[inline]- fn write_u64(&mut self, i: u64) {- self.0 = i;- }-- #[inline]- fn write_usize(&mut self, i: usize) {- self.write_u64(i.try_into().unwrap());- }-}-#[derive(Clone, Copy)]enum TraceInfoLoc {HostType(VMSharedTypeIndex),diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex f8641c87d6..6964e405d6 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -615,11 +615,10 @@ fn is_subtype(let actual = VMSharedTypeIndex::from_u32(actual_engine_type);let expected = VMSharedTypeIndex::from_u32(expected_engine_type);-- let is_subtype: bool = store.engine().signatures().is_subtype(actual, expected);+ let is_subtype = store.is_subtype_cached(actual, expected);log::trace!("is_subtype(actual={actual:?}, expected={expected:?}) -> {is_subtype}",);- is_subtype as u32+ u32::from(is_subtype)}// Implementation of `memory.atomic.notify` for locally defined memories.