A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Add separate GC heap tunables (#13080)
Nick Fitzgerald13b62708fcb0
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rsindex 35da8409b0..2354ccfe8f 100644--- a/crates/cli-flags/src/lib.rs+++ b/crates/cli-flags/src/lib.rs@@ -67,6 +67,19 @@ wasmtime_option_group! {/// Size, in bytes, of guard pages for linear memories.pub memory_guard_size: Option<u64>,+ /// Do not allow the GC heap to move in the host process's address+ /// space.+ pub gc_heap_may_move: Option<bool>,++ /// Initial virtual memory allocation size for the GC heap.+ pub gc_heap_reservation: Option<u64>,++ /// Bytes to reserve at the end of the GC heap for growth into.+ pub gc_heap_reservation_for_growth: Option<u64>,++ /// Size, in bytes, of guard pages for the GC heap.+ pub gc_heap_guard_size: Option<u64>,+/// Indicates whether an unmapped region of memory is placed before all/// linear memories.pub guard_before_linear_memory: Option<bool>,@@ -891,6 +904,19 @@ impl CommonOptions {if let Some(enable) = self.opts.guard_before_linear_memory {config.guard_before_linear_memory(enable);}++ if let Some(size) = self.opts.gc_heap_reservation {+ config.gc_heap_reservation(size);+ }+ if let Some(enable) = self.opts.gc_heap_may_move {+ config.gc_heap_may_move(enable);+ }+ if let Some(size) = self.opts.gc_heap_guard_size {+ config.gc_heap_guard_size(size);+ }+ if let Some(size) = self.opts.gc_heap_reservation_for_growth {+ config.gc_heap_reservation_for_growth(size);+ }if let Some(enable) = self.opts.table_lazy_init {config.table_lazy_init(enable);}diff --git a/crates/cranelift/src/bounds_checks.rs b/crates/cranelift/src/bounds_checks.rsindex 2743dd861d..310549224c 100644--- a/crates/cranelift/src/bounds_checks.rs+++ b/crates/cranelift/src/bounds_checks.rs@@ -184,16 +184,17 @@ fn bounds_check_field_access(env.heap_access_spectre_mitigation() && clif_memory_traps_enabled;let host_page_size_log2 = env.target_config().page_size_align_log2;+ let memory_tunables = heap.memory_tunables(env.tunables());let can_use_virtual_memory = heap.memory- .can_use_virtual_memory(env.tunables(), host_page_size_log2)+ .can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2)&& clif_memory_traps_enabled;let can_elide_bounds_check = heap.memory- .can_elide_bounds_check(env.tunables(), host_page_size_log2)+ .can_elide_bounds_check(&memory_tunables, host_page_size_log2)&& clif_memory_traps_enabled;- let memory_guard_size = env.tunables().memory_guard_size;- let memory_reservation = env.tunables().memory_reservation;+ let memory_guard_size = memory_tunables.guard_size();+ let memory_reservation = memory_tunables.reservation();let offset_and_size = offset_plus_size(offset, access_size);let statically_in_bounds = statically_in_bounds(&builder.func, heap, index, offset_and_size);@@ -348,7 +349,7 @@ fn bounds_check_field_access(// factor in the guard pages here.if can_use_virtual_memory&& heap.memory.minimum_byte_size().unwrap_or(u64::MAX) <= memory_reservation- && !heap.memory.memory_may_move(env.tunables())+ && !heap.memory.memory_may_move(&memory_tunables)&& memory_reservation >= offset_and_size{let adjusted_bound = memory_reservation.checked_sub(offset_and_size).unwrap();diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 1bde277a33..83b8538415 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -4,8 +4,8 @@ pub(crate) mod stack_switching;use crate::BuiltinFunctionSignatures;use crate::compiler::Compiler;use crate::translate::{- FuncTranslationStacks, GlobalVariable, Heap, HeapData, StructFieldsVec, TableData, TableSize,- TargetEnvironment,+ FuncTranslationStacks, GlobalVariable, Heap, HeapData, MemoryKind, StructFieldsVec, TableData,+ TableSize, TargetEnvironment,};use crate::trap::TranslateTrap;use cranelift_codegen::cursor::FuncCursor;@@ -28,10 +28,10 @@ use wasmtime_core::math::f64_cvt_to_int_bounds;use wasmtime_environ::{BuiltinFunctionIndex, ComponentPC, DataIndex, DefinedFuncIndex, ElemIndex,EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey,- GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, Module, ModuleInternedTypeIndex,- ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex, TagIndex, Tunables,- TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType,- WasmHeapType, WasmRefType, WasmResult, WasmValType,+ GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module,+ ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex,+ TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,+ WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmValType,};use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};@@ -1566,6 +1566,7 @@ impl FuncEnvironment<'_> {self.heaps.push(HeapData {base,bound,+ kind: MemoryKind::LinearMemory,memory,})}@@ -1578,9 +1579,10 @@ impl FuncEnvironment<'_> {offset: i32,) -> ir::GlobalValue {let pointer_type = self.pointer_type();+ let memory_tunables = MemoryTunables::new(self.tunables, MemoryKind::LinearMemory);let mut flags = ir::MemFlags::trusted().with_can_move();- if !memory.memory_may_move(self.tunables) {+ if !memory.memory_may_move(&memory_tunables) {flags.set_readonly();}diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rsindex 73ef862997..d2d42715a3 100644--- a/crates/cranelift/src/func_environ/gc/enabled.rs+++ b/crates/cranelift/src/func_environ/gc/enabled.rs@@ -1,7 +1,7 @@use super::{ArrayInit, GcCompiler};use crate::bounds_checks::BoundsCheck;use crate::func_environ::{Extension, FuncEnvironment};-use crate::translate::{Heap, HeapData, StructFieldsVec, TargetEnvironment};+use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment};use crate::trap::TranslateTrap;use crate::{Reachability, TRAP_INTERNAL_ASSERT};use cranelift_codegen::ir::immediates::Offset32;@@ -1449,10 +1449,12 @@ impl FuncEnvironment<'_> {let offset = self.offsets.ptr.vmstore_context_gc_heap_base();let mut flags = ir::MemFlags::trusted();+ let memory_tunables =+ wasmtime_environ::MemoryTunables::new(self.tunables, MemoryKind::GcHeap);if !self.tunables.gc_heap_memory_type()- .memory_may_move(self.tunables)+ .memory_may_move(&memory_tunables){flags.set_readonly();flags.set_can_move();@@ -1512,6 +1514,7 @@ impl FuncEnvironment<'_> {base,bound,memory,+ kind: MemoryKind::GcHeap,});self.gc_heap = Some(heap);heapdiff --git a/crates/cranelift/src/translate/heap.rs b/crates/cranelift/src/translate/heap.rsindex 32992dd191..459afd32d0 100644--- a/crates/cranelift/src/translate/heap.rs+++ b/crates/cranelift/src/translate/heap.rs@@ -4,6 +4,8 @@ use cranelift_codegen::ir::{self, GlobalValue, Type};use cranelift_entity::entity_impl;use wasmtime_environ::{IndexType, Memory};+pub use wasmtime_environ::MemoryKind;+/// An opaque reference to a [`HeapData`][crate::HeapData].////// While the order is stable, it is arbitrary.@@ -45,6 +47,9 @@ pub struct HeapData {/// The type of wasm memory that this heap is operating on.pub memory: Memory,++ /// Whether this is a linear memory or a GC heap.+ pub kind: MemoryKind,}impl HeapData {@@ -54,4 +59,12 @@ impl HeapData {IndexType::I64 => ir::types::I64,}}++ /// Get the [`MemoryTunables`] for this heap based on its [`MemoryKind`].+ pub fn memory_tunables<'a>(+ &self,+ tunables: &'a wasmtime_environ::Tunables,+ ) -> wasmtime_environ::MemoryTunables<'a> {+ wasmtime_environ::MemoryTunables::new(tunables, self.kind)+ }}diff --git a/crates/cranelift/src/translate/mod.rs b/crates/cranelift/src/translate/mod.rsindex 7effeb759b..f890c1e4a7 100644--- a/crates/cranelift/src/translate/mod.rs+++ b/crates/cranelift/src/translate/mod.rs@@ -20,7 +20,7 @@ mod translation_utils;pub use self::environ::{GlobalVariable, StructFieldsVec, TargetEnvironment};pub use self::func_translator::FuncTranslator;-pub use self::heap::{Heap, HeapData};+pub use self::heap::{Heap, HeapData, MemoryKind};pub use self::stack::FuncTranslationStacks;pub use self::table::{TableData, TableSize};pub use self::translation_utils::*;diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rsindex 898029a63d..354021e0dc 100644--- a/crates/environ/src/tunables.rs+++ b/crates/environ/src/tunables.rs@@ -160,6 +160,29 @@ define_tunables! {/// forced and the counter is reset. Only effective when/// `cfg(gc_zeal)` is enabled.pub gc_zeal_alloc_counter: Option<NonZeroU32>,++ /// Initial size, in bytes, to be allocated for GC heaps.+ ///+ /// This is the same as `memory_reservation` but for GC heaps.+ pub gc_heap_reservation: u64,++ /// The size, in bytes, of the guard page region for GC heaps.+ ///+ /// This is the same as `memory_guard_size` but for GC heaps.+ pub gc_heap_guard_size: u64,++ /// The size, in bytes, to allocate at the end of a relocated GC heap+ /// for growth.+ ///+ /// This is the same as `memory_reservation_for_growth` but for GC+ /// heaps.+ pub gc_heap_reservation_for_growth: u64,++ /// Whether or not GC heaps are allowed to be reallocated after initial+ /// allocation at runtime.+ ///+ /// This is the same as `memory_may_move` but for GC heaps.+ pub gc_heap_may_move: bool,}pub struct ConfigTunables {@@ -200,6 +223,7 @@ impl Tunables {if target.is_pulley() {ret.signals_based_traps = false;ret.memory_guard_size = 0;+ ret.gc_heap_guard_size = 0;}Ok(ret)}@@ -239,6 +263,10 @@ impl Tunables {concurrency_support: true,recording: false,gc_zeal_alloc_counter: None,+ gc_heap_reservation: 0,+ gc_heap_guard_size: 0,+ gc_heap_reservation_for_growth: 0,+ gc_heap_may_move: true,}}@@ -253,6 +281,12 @@ impl Tunables {memory_reservation_for_growth: 1 << 20, // 1MBsignals_based_traps: true,+ // GC heaps on 32-bit: conservative defaults similar to linear+ // memories.+ gc_heap_reservation: 10 * (1 << 20),+ gc_heap_guard_size: 0x1_0000,+ gc_heap_reservation_for_growth: 1 << 20, // 1MB+..Tunables::default_miri()}}@@ -278,6 +312,12 @@ impl Tunables {// to avoid memory movement.memory_reservation_for_growth: 2 << 30, // 2GB+ // GC heaps on 64-bit: use 4GiB reservation and 32MiB guard pages+ // to enable bounds check elision, matching linear memory defaults.+ gc_heap_reservation: 1 << 32,+ gc_heap_guard_size: 32 << 20,+ gc_heap_reservation_for_growth: 2 << 30, // 2GB+signals_based_traps: true,..Tunables::default_miri()}@@ -298,6 +338,73 @@ impl Tunables {}}+/// Whether a heap is backing a linear memory or a GC heap.+///+/// This is used by [`MemoryTunables`] to select between the memory tunables and+/// the GC heap tunables.+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]+pub enum MemoryKind {+ /// A WebAssembly linear memory.+ LinearMemory,+ /// A GC heap for garbage-collected objects.+ GcHeap,+}++/// A view into a [`Tunables`] that selects the appropriate linear-memory or+/// GC-heap flavor of each tunable based on a [`MemoryKind`].+pub struct MemoryTunables<'a> {+ tunables: &'a Tunables,+ kind: MemoryKind,+}++impl<'a> MemoryTunables<'a> {+ /// Create a new `MemoryTunables` view.+ pub fn new(tunables: &'a Tunables, kind: MemoryKind) -> Self {+ Self { tunables, kind }+ }++ /// The virtual memory reservation for this kind of memory.+ pub fn reservation(&self) -> u64 {+ match self.kind {+ MemoryKind::LinearMemory => self.tunables.memory_reservation,+ MemoryKind::GcHeap => self.tunables.gc_heap_reservation,+ }+ }++ /// The size of the guard page region for this kind of memory.+ pub fn guard_size(&self) -> u64 {+ match self.kind {+ MemoryKind::LinearMemory => self.tunables.memory_guard_size,+ MemoryKind::GcHeap => self.tunables.gc_heap_guard_size,+ }+ }++ /// Extra virtual memory to reserve beyond the initially mapped pages for+ /// this kind of memory.+ pub fn reservation_for_growth(&self) -> u64 {+ match self.kind {+ MemoryKind::LinearMemory => self.tunables.memory_reservation_for_growth,+ MemoryKind::GcHeap => self.tunables.gc_heap_reservation_for_growth,+ }+ }++ /// Whether this kind of memory's base pointer may be relocated at runtime.+ pub fn may_move(&self) -> bool {+ match self.kind {+ MemoryKind::LinearMemory => self.tunables.memory_may_move,+ MemoryKind::GcHeap => self.tunables.gc_heap_may_move,+ }+ }++ /// Get the underlying tunables.+ ///+ /// This is ONLY for accessing tunable fields that DO NOT come in a+ /// linear-memory flavor and a GC-heap flavor.+ pub fn tunables(&self) -> &'a Tunables {+ self.tunables+ }+}+/// The garbage collector implementation to use.#[derive(Clone, Copy, Hash, Serialize, Deserialize, Debug, PartialEq, Eq)]pub enum Collector {diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rsindex e582bc7c5d..2847be79cd 100644--- a/crates/environ/src/types.rs+++ b/crates/environ/src/types.rs@@ -1,6 +1,6 @@use crate::{- PanicOnOom as _, Tunables, WasmResult, collections::TryCow, error::OutOfMemory, prelude::*,- wasm_unsupported,+ MemoryTunables, PanicOnOom as _, Tunables, WasmResult, collections::TryCow, error::OutOfMemory,+ prelude::*, wasm_unsupported,};use alloc::boxed::Box;use core::{fmt, ops::Range};@@ -2266,26 +2266,30 @@ impl Memory {/// Returns whether this memory is a candidate for bounds check elision/// given the configuration and host page size.///- /// This function determines whether the given compilation configuration and- /// hos enables possible bounds check elision for this memory. Bounds checks+ /// This function determines whether the given compilation configuration+ /// enables possible bounds check elision for this memory. Bounds checks/// can only be elided if [`Memory::can_use_virtual_memory`] returns `true`/// for example but there are additionally requirements on the index size of- /// this memory and the memory reservation in `tunables`.+ /// this memory and the memory reservation in the tunables.////// Currently the only case that supports bounds check elision is when all/// of these apply:////// * When [`Memory::can_use_virtual_memory`] returns `true`./// * This is a 32-bit linear memory (e.g. not 64-bit)- /// * `tunables.memory_reservation` is in excess of 4GiB+ /// * The reservation + guard size is in excess of 4GiB////// In this situation all computable addresses fall within the reserved/// space (modulo static offsets factoring in guard pages) so bounds checks/// may be elidable.- pub fn can_elide_bounds_check(&self, tunables: &Tunables, host_page_size_log2: u8) -> bool {- self.can_use_virtual_memory(tunables, host_page_size_log2)+ pub fn can_elide_bounds_check(+ &self,+ memory_tunables: &MemoryTunables<'_>,+ host_page_size_log2: u8,+ ) -> bool {+ self.can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2)&& self.idx_type == IndexType::I32- && tunables.memory_reservation + tunables.memory_guard_size >= (1 << 32)+ && memory_tunables.reservation() + memory_tunables.guard_size() >= (1 << 32)}/// Returns the static size of this heap in bytes at runtime, if available.@@ -2303,7 +2307,7 @@ impl Memory {/// When this function returns `false` then it means that after the initial/// allocation the base pointer is constant for the entire lifetime of a/// memory. This can enable compiler optimizations, for example.- pub fn memory_may_move(&self, tunables: &Tunables) -> bool {+ pub fn memory_may_move(&self, memory_tunables: &MemoryTunables<'_>) -> bool {// Shared memories cannot ever relocate their base pointer so the// settings configured in the engine must be appropriate for them ahead// of time.@@ -2313,7 +2317,7 @@ impl Memory {// If movement is disallowed in engine configuration, then the answer is// "no".- if !tunables.memory_may_move {+ if !memory_tunables.may_move() {return false;}@@ -2326,7 +2330,7 @@ impl Memory {// If the maximum size of this memory is above the threshold of the// initial memory reservation then the memory may move.let max = self.maximum_byte_size().unwrap_or(u64::MAX);- max > tunables.memory_reservation+ max > memory_tunables.reservation()}/// Tests whether this memory type is allowed to grow up to `size` bytes.diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rsindex 0180136142..1e80f47825 100644--- a/crates/fuzzing/src/generators/config.rs+++ b/crates/fuzzing/src/generators/config.rs@@ -240,6 +240,11 @@ impl Config {pooling.total_stacks = pooling.total_stacks.max(limits::TOTAL_STACKS);}+ // Re-enforce internal consistency after all the adjustments above+ // (e.g. memory_reservation may have been bumped without a+ // corresponding bump to gc_heap_reservation).+ self.wasmtime.make_internally_consistent();+// Return the test configuration that this fuzz configuration represents// which is used afterwards to test if the `test` here is expected to// fail or not.@@ -408,23 +413,37 @@ impl Config {memory_config.configure(&mut cfg);};- // If malloc-based memory is going to be used, which requires these four- // options set to specific values (and Pulley auto-sets two of them)- // then be sure to cap `memory_reservation_for_growth` at a smaller- // value than the default. For malloc-based memory reservation beyond- // the end of memory isn't captured by `StoreLimiter` so we need to be- // sure it's small enough to not blow OOM limits while fuzzing.- if ((cfg.opts.signals_based_traps == Some(true) && cfg.opts.memory_guard_size == Some(0))- || self.wasmtime.compiler_strategy == CompilerStrategy::CraneliftPulley)- && cfg.opts.memory_reservation == Some(0)- && cfg.opts.memory_init_cow == Some(false)- {- let growth = &mut cfg.opts.memory_reservation_for_growth;- let max = 1 << 20;- *growth = match *growth {- Some(n) => Some(n.min(max)),- None => Some(max),- };+ // If malloc-based memory is going to be used, which requires these+ // options set to specific values (and Pulley auto-sets some of them)+ // then be sure to cap `memory_reservation_for_growth` and+ // `gc_heap_reservation_for_growth` at a smaller value than the+ // default. For malloc-based memory/heaps, reservation beyond the end+ // isn't captured by `StoreLimiter` so we need to be sure it's small+ // enough to not blow OOM limits while fuzzing.+ let is_pulley = self.wasmtime.compiler_strategy == CompilerStrategy::CraneliftPulley;+ let signals_traps = cfg.opts.signals_based_traps == Some(true);+ if signals_traps || is_pulley {+ if ((signals_traps && cfg.opts.memory_guard_size == Some(0)) || is_pulley)+ && cfg.opts.memory_reservation == Some(0)+ && cfg.opts.memory_init_cow == Some(false)+ {+ let growth = &mut cfg.opts.memory_reservation_for_growth;+ let max = 1 << 20;+ *growth = match *growth {+ Some(n) => Some(n.min(max)),+ None => Some(max),+ };+ }+ if ((signals_traps && cfg.opts.gc_heap_guard_size == Some(0)) || is_pulley)+ && cfg.opts.gc_heap_reservation == Some(0)+ {+ let growth = &mut cfg.opts.gc_heap_reservation_for_growth;+ let max = 1 << 20;+ *growth = match *growth {+ Some(n) => Some(n.min(max)),+ None => Some(max),+ };+ }}log::debug!("creating wasmtime config with CLI options:\n{cfg}");@@ -798,7 +817,19 @@ impl WasmtimeConfig {/// be considered a "TODO" to go implement more stuff in Wasmtime to accept/// these sorts of configurations. For now though it's intended to reflect/// the current state of the engine's development.- fn make_internally_consistent(&mut self) {+ pub(crate) fn make_internally_consistent(&mut self) {+ // When using the pooling allocator, GC heap tunables must match memory+ // tunables.+ if let InstanceAllocationStrategy::Pooling(_) = &self.strategy {+ self.memory_config.gc_heap_reservation = self.memory_config.memory_reservation;+ self.memory_config.gc_heap_guard_size = self.memory_config.memory_guard_size;+ self.memory_config.gc_heap_reservation_for_growth =+ self.memory_config.memory_reservation_for_growth;+ // memory_may_move is not in MemoryConfig, but gc_heap_may_move+ // must not conflict. Set it to None so the default matches.+ self.memory_config.gc_heap_may_move = None;+ }+if !self.signals_based_traps {let cfg = &mut self.memory_config;// Spectre-based heap mitigations require signal handlers so@@ -807,8 +838,8 @@ impl WasmtimeConfig {cfg.cranelift_enable_heap_access_spectre_mitigations = None;// With configuration settings that match the use of malloc for- // linear memories cap the `memory_reservation_for_growth` value- // to something reasonable to avoid OOM in fuzzing.+ // linear memories and GC heaps, cap the reservation-for-growth+ // values to something reasonable to avoid OOM in fuzzing.if !cfg.memory_init_cow&& cfg.memory_guard_size == Some(0)&& cfg.memory_reservation == Some(0)@@ -820,6 +851,17 @@ impl WasmtimeConfig {cfg.memory_reservation_for_growth = Some(min);}}++ // Similarly cap gc_heap_reservation_for_growth when signals-based+ // traps are disabled and the GC heap uses malloc-style allocation.+ if cfg.gc_heap_guard_size == Some(0) && cfg.gc_heap_reservation == Some(0) {+ let min = 10 << 20; // 10 MiB+ if let Some(val) = &mut cfg.gc_heap_reservation_for_growth {+ *val = (*val).min(min);+ } else {+ cfg.gc_heap_reservation_for_growth = Some(min);+ }+ }}}}diff --git a/crates/fuzzing/src/generators/memory.rs b/crates/fuzzing/src/generators/memory.rsindex 405832bffa..ea36ad3011 100644--- a/crates/fuzzing/src/generators/memory.rs+++ b/crates/fuzzing/src/generators/memory.rs@@ -124,6 +124,10 @@ pub struct MemoryConfig {pub guard_before_linear_memory: bool,pub cranelift_enable_heap_access_spectre_mitigations: Option<bool>,pub memory_init_cow: bool,+ pub gc_heap_reservation: Option<u64>,+ pub gc_heap_guard_size: Option<u64>,+ pub gc_heap_reservation_for_growth: Option<u64>,+ pub gc_heap_may_move: Option<bool>,}impl<'a> Arbitrary<'a> for MemoryConfig {@@ -142,6 +146,12 @@ impl<'a> Arbitrary<'a> for MemoryConfig {guard_before_linear_memory: u.arbitrary()?,cranelift_enable_heap_access_spectre_mitigations: u.arbitrary()?,memory_init_cow: u.arbitrary()?,++ // GC heap tunables, same ranges as corresponding memory tunables.+ gc_heap_reservation: interesting_virtual_memory_size(u, 33)?,+ gc_heap_guard_size: interesting_virtual_memory_size(u, 32)?,+ gc_heap_reservation_for_growth: interesting_virtual_memory_size(u, 30)?,+ gc_heap_may_move: u.arbitrary()?,})}}@@ -181,6 +191,10 @@ impl MemoryConfig {cfg.opts.memory_reservation_for_growth = self.memory_reservation_for_growth;cfg.opts.guard_before_linear_memory = Some(self.guard_before_linear_memory);cfg.opts.memory_init_cow = Some(self.memory_init_cow);+ cfg.opts.gc_heap_reservation = self.gc_heap_reservation;+ cfg.opts.gc_heap_guard_size = self.gc_heap_guard_size;+ cfg.opts.gc_heap_reservation_for_growth = self.gc_heap_reservation_for_growth;+ cfg.opts.gc_heap_may_move = self.gc_heap_may_move;if let Some(enable) = self.cranelift_enable_heap_access_spectre_mitigations {cfg.codegen.cranelift.push((diff --git a/crates/fuzzing/src/oracles/component_api.rs b/crates/fuzzing/src/oracles/component_api.rsindex 940529418b..6bbcd25eb3 100644--- a/crates/fuzzing/src/oracles/component_api.rs+++ b/crates/fuzzing/src/oracles/component_api.rs@@ -211,6 +211,11 @@ fn store<T>(input: &mut Unstructured<'_>, val: T) -> arbitrary::Result<Store<T>>10 << 20,);+ // Re-enforce internal consistency after the adjustments above (e.g.+ // memory_reservation may have been bumped without a corresponding bump+ // to gc_heap_reservation).+ config.wasmtime.make_internally_consistent();+let engine = Engine::new(config.to_wasmtime()diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rsindex 323d9b7ce0..a51bb7f2a7 100644--- a/crates/wasmtime/src/config.rs+++ b/crates/wasmtime/src/config.rs@@ -50,6 +50,12 @@ pub enum InstanceAllocationStrategy {/// A pool of resources is created in advance and module instantiation reuses resources/// from the pool. Resources are returned to the pool when the `Store` referencing the instance/// is dropped.+ ///+ /// When GC is enabled, the pooling allocator requires that the GC heap+ /// configuration matches the linear memory configuration (i.e.,+ /// `gc_heap_reservation` must equal `memory_reservation`, etc.). By+ /// default, if no `gc_heap_*` tunables are explicitly configured, they+ /// automatically inherit the `memory_*` values.#[cfg(feature = "pooling-allocator")]Pooling(PoolingAllocationConfig),}@@ -1947,6 +1953,75 @@ impl Config {self}+ /// Configures the initial size, in bytes, to be allocated for GC heaps.+ ///+ /// This is similar to [`Config::memory_reservation`] but applies to the GC+ /// heap rather than to linear memories. See that method for more details+ /// on what "reservation" means and the implications of this setting.+ ///+ /// ## Default+ ///+ /// If none of the `gc_heap_*` tunables are explicitly configured, they+ /// default to the same values as their `memory_*` counterparts. Otherwise,+ /// the default value for this property depends on the host platform: for+ /// 64-bit platforms this defaults to 4GiB, and for 32-bit platforms this+ /// defaults to 10MiB.+ pub fn gc_heap_reservation(&mut self, bytes: u64) -> &mut Self {+ self.tunables.gc_heap_reservation = Some(bytes);+ self+ }++ /// Configures the size, in bytes, of the guard page region for GC heaps.+ ///+ /// This is similar to [`Config::memory_guard_size`] but applies to the GC+ /// heap rather than to linear memories. See that method for more details on+ /// what guard pages are and the implications of this setting.+ ///+ /// ## Default+ ///+ /// If none of the `gc_heap_*` tunables are explicitly configured, they+ /// default to the same values as their `memory_*` counterparts. Otherwise,+ /// the default value for this property is 32MiB on 64-bit platforms and+ /// 64KiB on 32-bit platforms.+ pub fn gc_heap_guard_size(&mut self, bytes: u64) -> &mut Self {+ self.tunables.gc_heap_guard_size = Some(bytes);+ self+ }++ /// Configures the size, in bytes, of the extra virtual memory space+ /// reserved after a GC heap is relocated.+ ///+ /// This is similar to [`Config::memory_reservation_for_growth`] but applies+ /// to the GC heap rather than to linear memories. See that method for more+ /// details.+ ///+ /// ## Default+ ///+ /// If none of the `gc_heap_*` tunables are explicitly configured, they+ /// default to the same values as their `memory_*` counterparts. Otherwise,+ /// for 64-bit platforms this defaults to 2GiB, and for 32-bit platforms+ /// this defaults to 1MiB.+ pub fn gc_heap_reservation_for_growth(&mut self, bytes: u64) -> &mut Self {+ self.tunables.gc_heap_reservation_for_growth = Some(bytes);+ self+ }++ /// Indicates whether GC heaps are allowed to be reallocated after initial+ /// allocation at runtime.+ ///+ /// This is similar to [`Config::memory_may_move`] but applies to the GC+ /// heap rather than to linear memories. See that method for more details.+ ///+ /// ## Default+ ///+ /// If none of the `gc_heap_*` tunables are explicitly configured, they+ /// default to the same values as their `memory_*` counterparts. Otherwise,+ /// the default value for this option is `true`.+ pub fn gc_heap_may_move(&mut self, enable: bool) -> &mut Self {+ self.tunables.gc_heap_may_move = Some(enable);+ self+ }+/// Indicates whether a guard region is present before allocations of/// linear memory.///@@ -2383,6 +2458,15 @@ impl Config {target_lexicon::Triple::host()}+ /// Returns `true` if any of the `gc_heap_*` tunables have been explicitly+ /// configured.+ fn any_gc_heap_tunables_configured(&self) -> bool {+ self.tunables.gc_heap_reservation.is_some()+ || self.tunables.gc_heap_guard_size.is_some()+ || self.tunables.gc_heap_reservation_for_growth.is_some()+ || self.tunables.gc_heap_may_move.is_some()+ }+pub(crate) fn validate(&self) -> Result<(Tunables, WasmFeatures)> {let features = self.features();@@ -2456,6 +2540,7 @@ impl Config {if !cfg!(has_native_signals) {tunables.signals_based_traps = cfg!(has_native_signals);tunables.memory_guard_size = 0;+ tunables.gc_heap_guard_size = 0;}// When virtual memory is not available use slightly different@@ -2465,6 +2550,8 @@ impl Config {tunables.memory_reservation = 0;tunables.memory_reservation_for_growth = 1 << 20; // 1MBtunables.memory_init_cow = false;+ tunables.gc_heap_reservation = 0;+ tunables.gc_heap_reservation_for_growth = 1 << 20; // 1MB}}@@ -2479,6 +2566,16 @@ impl Config {self.tunables.configure(&mut tunables);+ // If no GC heap tunables are explicitly configured, copy the memory+ // tunables' configured values so that GC heaps default to the same+ // configuration as linear memories.+ if !self.any_gc_heap_tunables_configured() {+ tunables.gc_heap_reservation = tunables.memory_reservation;+ tunables.gc_heap_guard_size = tunables.memory_guard_size;+ tunables.gc_heap_reservation_for_growth = tunables.memory_reservation_for_growth;+ tunables.gc_heap_may_move = tunables.memory_may_move;+ }+// If we're going to compile with winch, we must use the winch calling convention.#[cfg(any(feature = "cranelift", feature = "winch"))]{@@ -2534,6 +2631,50 @@ impl Config {)}+ // If the pooling allocator is used and GC is enabled, check that+ // memories and the GC heap are configured identically, since the+ // pooling allocator can't support differently-configured heaps.+ #[cfg(feature = "pooling-allocator")]+ if matches!(+ &self.allocation_strategy,+ InstanceAllocationStrategy::Pooling(_)+ ) && tunables.collector.is_some()+ {+ if tunables.memory_reservation != tunables.gc_heap_reservation {+ bail!(+ "when using the pooling allocator with GC, `memory_reservation` ({}) \+ and `gc_heap_reservation` ({}) must be the same",+ tunables.memory_reservation,+ tunables.gc_heap_reservation,+ );+ }+ if tunables.memory_guard_size != tunables.gc_heap_guard_size {+ bail!(+ "when using the pooling allocator with GC, `memory_guard_size` ({}) \+ and `gc_heap_guard_size` ({}) must be the same",+ tunables.memory_guard_size,+ tunables.gc_heap_guard_size,+ );+ }+ if tunables.memory_reservation_for_growth != tunables.gc_heap_reservation_for_growth {+ bail!(+ "when using the pooling allocator with GC, \+ `memory_reservation_for_growth` ({}) and \+ `gc_heap_reservation_for_growth` ({}) must be the same",+ tunables.memory_reservation_for_growth,+ tunables.gc_heap_reservation_for_growth,+ );+ }+ if tunables.memory_may_move != tunables.gc_heap_may_move {+ bail!(+ "when using the pooling allocator with GC, `memory_may_move` ({}) \+ and `gc_heap_may_move` ({}) must be the same",+ tunables.memory_may_move,+ tunables.gc_heap_may_move,+ );+ }+ }+Ok((tunables, features))}diff --git a/crates/wasmtime/src/engine/serialization.rs b/crates/wasmtime/src/engine/serialization.rsindex 0028902864..6684ba9a1f 100644--- a/crates/wasmtime/src/engine/serialization.rs+++ b/crates/wasmtime/src/engine/serialization.rs@@ -330,6 +330,13 @@ impl Metadata<'_> {// This is a runtime GC debugging setting, doesn't affect compilation.gc_zeal_alloc_counter: _,++ gc_heap_reservation,+ gc_heap_guard_size,+ gc_heap_may_move,++ // This doesn't affect compilation, it's just a runtime setting.+ gc_heap_reservation_for_growth: _,} = self.tunables;Self::check_collector(collector, other.collector)?;@@ -406,6 +413,17 @@ impl Metadata<'_> {)?;Self::check_bool(recording, other.recording, "RR recording support")?;Self::check_intra_module_inlining(inlining_intra_module, other.inlining_intra_module)?;+ Self::check_int(+ gc_heap_reservation,+ other.gc_heap_reservation,+ "GC heap reservation",+ )?;+ Self::check_int(+ gc_heap_guard_size,+ other.gc_heap_guard_size,+ "GC heap guard size",+ )?;+ Self::check_bool(gc_heap_may_move, other.gc_heap_may_move, "GC heap may move")?;Ok(())}diff --git a/crates/wasmtime/src/runtime/memory.rs b/crates/wasmtime/src/runtime/memory.rsindex 7b027bddc6..81cb628896 100644--- a/crates/wasmtime/src/runtime/memory.rs+++ b/crates/wasmtime/src/runtime/memory.rs@@ -1067,10 +1067,13 @@ mod tests {let store = store.as_context();let tunables = store.engine().tunables();assert_eq!(tunables.memory_guard_size, 0);- assert!(- !mem.wasmtime_ty(store.0)- .can_elide_bounds_check(tunables, 12)- );+ assert!(!mem.wasmtime_ty(store.0).can_elide_bounds_check(+ &wasmtime_environ::MemoryTunables::new(+ tunables,+ wasmtime_environ::MemoryKind::LinearMemory,+ ),+ 12,+ ));}#[test]diff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rsindex 3ef0c4be3e..704bc60d0f 100644--- a/crates/wasmtime/src/runtime/trampoline/memory.rs+++ b/crates/wasmtime/src/runtime/trampoline/memory.rs@@ -12,8 +12,8 @@ use alloc::sync::Arc;use core::future::Future;use core::pin::Pin;use wasmtime_environ::{- DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, Module, StaticModuleIndex,- Tunables, VMOffsets,+ DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, MemoryTunables, Module,+ StaticModuleIndex, VMOffsets,};#[cfg(feature = "component-model")]@@ -105,18 +105,18 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy {fn new_memory(&self,ty: &wasmtime_environ::Memory,- tunables: &Tunables,+ memory_tunables: &MemoryTunables<'_>,minimum: usize,maximum: Option<usize>,) -> Result<Box<dyn RuntimeLinearMemory>> {- let reserved_size_in_bytes = Some(tunables.memory_reservation.try_into().unwrap());+ let reserved_size_in_bytes = Some(memory_tunables.reservation().try_into().unwrap());self.0.new_memory(MemoryType::from_wasmtime_memory(ty),minimum,maximum,reserved_size_in_bytes,- usize::try_from(tunables.memory_guard_size).unwrap(),+ usize::try_from(memory_tunables.guard_size()).unwrap(),).map(|mem| Box::new(LinearMemoryProxy { mem }) as Box<dyn RuntimeLinearMemory>).map_err(|e| format_err!(e))diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rsindex 987e9c944a..15dbbaae5d 100644--- a/crates/wasmtime/src/runtime/vm/cow.rs+++ b/crates/wasmtime/src/runtime/vm/cow.rs@@ -12,7 +12,7 @@ use alloc::sync::Arc;use core::fmt;use core::ops::Range;use wasmtime_environ::prelude::TryPrimaryMap;-use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, Module, Tunables};+use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, MemoryTunables, Module};/// Backing images for memories in a module.///@@ -416,7 +416,7 @@ impl MemoryImageSlot {initial_size_bytes: usize,maybe_image: Option<&Arc<MemoryImage>>,ty: &wasmtime_environ::Memory,- tunables: &Tunables,+ memory_tunables: &MemoryTunables<'_>,) -> Result<()> {assert!(!self.dirty);assert!(@@ -460,8 +460,8 @@ impl MemoryImageSlot {// using dynamic memory without any guard pages.let host_page_size_log2 = u8::try_from(host_page_size().ilog2()).unwrap();if initial_size_bytes_page_aligned < self.accessible- && (tunables.memory_guard_size > 0- || ty.can_use_virtual_memory(tunables, host_page_size_log2))+ && (memory_tunables.guard_size() > 0+ || ty.can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2)){self.set_protection(initial_size_bytes_page_aligned..self.accessible, false)?;self.accessible = initial_size_bytes_page_aligned;@@ -715,7 +715,7 @@ mod test {use crate::runtime::vm::sys::vm::decommit_pages;use crate::runtime::vm::{HostAlignedByteCount, MmapVec, host_page_size};use std::sync::Arc;- use wasmtime_environ::{IndexType, Limits, Memory};+ use wasmtime_environ::{IndexType, Limits, Memory, MemoryKind, Tunables};fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryImage> {// offset must be a multiple of the page size.@@ -804,7 +804,14 @@ mod test {MemoryImageSlot::create(mmap.zero_offset(), HostAlignedByteCount::ZERO, 4 << 20);assert!(!memfd.is_dirty());// instantiate with 64 KiB initial size- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();+ memfd+ .instantiate(+ 64 << 10,+ None,+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ )+ .unwrap();assert!(memfd.is_dirty());// We should be able to access this 64 KiB (try both ends) and@@ -831,7 +838,14 @@ mod test {}).unwrap();assert!(!memfd.is_dirty());- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();+ memfd+ .instantiate(+ 64 << 10,+ None,+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ )+ .unwrap();let slice = unsafe { mmap.slice(0..65536) };assert_eq!(0, slice[1024]);}@@ -853,7 +867,12 @@ mod test {let image = Arc::new(create_memfd_with_data(page_size, &[1, 2, 3, 4]).unwrap());// Instantiate with this imagememfd- .instantiate(64 << 10, Some(&image), &ty, &tunables)+ .instantiate(+ 64 << 10,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();assert!(memfd.has_image());@@ -871,7 +890,12 @@ mod test {}).unwrap();memfd- .instantiate(64 << 10, Some(&image), &ty, &tunables)+ .instantiate(+ 64 << 10,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();let slice = unsafe { mmap.slice(0..65536) };assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);@@ -882,7 +906,14 @@ mod test {decommit_pages(ptr, len).unwrap()}).unwrap();- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();+ memfd+ .instantiate(+ 64 << 10,+ None,+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ )+ .unwrap();assert!(!memfd.has_image());let slice = unsafe { mmap.slice(0..65536) };assert_eq!(&[0, 0, 0, 0], &slice[page_size..][..4]);@@ -894,7 +925,12 @@ mod test {}).unwrap();memfd- .instantiate(64 << 10, Some(&image), &ty, &tunables)+ .instantiate(+ 64 << 10,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();let slice = unsafe { mmap.slice(0..65536) };assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);@@ -907,7 +943,12 @@ mod test {}).unwrap();memfd- .instantiate(128 << 10, Some(&image2), &ty, &tunables)+ .instantiate(+ 128 << 10,+ Some(&image2),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();let slice = unsafe { mmap.slice(0..65536) };assert_eq!(&[10, 11, 12, 13], &slice[page_size..][..4]);@@ -920,7 +961,12 @@ mod test {}).unwrap();memfd- .instantiate(64 << 10, Some(&image), &ty, &tunables)+ .instantiate(+ 64 << 10,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();let slice = unsafe { mmap.slice(0..65536) };assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);@@ -945,7 +991,12 @@ mod test {for amt_to_memset in [0, page_size, page_size * 10, 1 << 20, 10 << 20] {let amt_to_memset = HostAlignedByteCount::new(amt_to_memset).unwrap();memfd- .instantiate(64 << 10, Some(&image), &ty, &tunables)+ .instantiate(+ 64 << 10,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();assert!(memfd.has_image());@@ -972,7 +1023,14 @@ mod test {// Test without an imagefor amt_to_memset in [0, page_size, page_size * 10, 1 << 20, 10 << 20] {let amt_to_memset = HostAlignedByteCount::new(amt_to_memset).unwrap();- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();+ memfd+ .instantiate(+ 64 << 10,+ None,+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ )+ .unwrap();unsafe {with_slice_mut(&mmap, 0..64 << 10, |slice| {@@ -1010,7 +1068,12 @@ mod test {// Instantiate the image and test that memory remains accessible after// it's cleared.memfd- .instantiate(initial, Some(&image), &ty, &tunables)+ .instantiate(+ initial,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();assert!(memfd.has_image());@@ -1033,7 +1096,12 @@ mod test {// Re-instantiate make sure it preserves memory. Grow a bit and set data// beyond the initial size.memfd- .instantiate(initial, Some(&image), &ty, &tunables)+ .instantiate(+ initial,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);@@ -1059,7 +1127,12 @@ mod test {// Instantiate again, and again memory beyond the initial size should// still be accessible. Grow into it again and make sure it works.memfd- .instantiate(initial, Some(&image), &ty, &tunables)+ .instantiate(+ initial,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();assert_eq!(&[0, 0], &slice[initial..initial + 2]);memfd.set_heap_limit(initial * 2).unwrap();@@ -1079,7 +1152,14 @@ mod test {.unwrap();// Reset the image to none and double-check everything is back to zero- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();+ memfd+ .instantiate(+ 64 << 10,+ None,+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ )+ .unwrap();assert!(!memfd.has_image());assert_eq!(&[0, 0, 0, 0], &slice[page_size..][..4]);assert_eq!(&[0, 0], &slice[initial..initial + 2]);@@ -1109,7 +1189,12 @@ mod test {let image = Arc::new(create_memfd_with_data(3 * page_size, &data).unwrap());memfd- .instantiate(mmap_len, Some(&image), &ty, &tunables)+ .instantiate(+ mmap_len,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();let keep_resident = HostAlignedByteCount::new(mmap_len).unwrap();@@ -1140,7 +1225,12 @@ mod test {// Re-instantiate, but then wipe the image entirely by keeping// nothing resident.memfd- .instantiate(mmap_len, Some(&image), &ty, &tunables)+ .instantiate(+ mmap_len,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();memfd.clear_and_remain_ready(pagemap, HostAlignedByteCount::ZERO, |ptr, len| {@@ -1150,7 +1240,12 @@ mod test {// Next re-instantiate a final time to get used for the next test.memfd- .instantiate(mmap_len, Some(&image), &ty, &tunables)+ .instantiate(+ mmap_len,+ Some(&image),+ &ty,+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),+ ).unwrap();};diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rsindex b9350e1afe..fdb3ef5e30 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs@@ -383,7 +383,7 @@ impl PoolingInstanceAllocator {tables: TablePool::new(config)?,live_tables: AtomicUsize::new(0),#[cfg(feature = "gc")]- gc_heaps: GcHeapPool::new(config)?,+ gc_heaps: GcHeapPool::new(config, tunables)?,#[cfg(feature = "gc")]live_gc_heaps: AtomicUsize::new(0),#[cfg(feature = "async")]diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rsindex 685c8aba29..87091a6393 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rs@@ -4,6 +4,7 @@ use crate::runtime::vm::{GcHeap, GcRuntime, PoolingInstanceAllocatorConfig, Resuuse crate::vm::{Memory, MemoryAllocationIndex};use crate::{Engine, prelude::*};use std::sync::Mutex;+use wasmtime_environ::Tunables;enum HeapSlot {/// The is available for use, and we may or may not have lazily allocated@@ -57,7 +58,26 @@ impl std::fmt::Debug for GcHeapPool {impl GcHeapPool {/// Create a new `GcHeapPool` with the given configuration.- pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {+ pub fn new(config: &PoolingInstanceAllocatorConfig, tunables: &Tunables) -> Result<Self> {+ // When using the pooling allocator, GC heaps and linear memories share+ // the same underlying memory pool, so they must be configured the same.+ debug_assert_eq!(+ tunables.memory_reservation, tunables.gc_heap_reservation,+ "pooling allocator requires memory_reservation == gc_heap_reservation"+ );+ debug_assert_eq!(+ tunables.memory_guard_size, tunables.gc_heap_guard_size,+ "pooling allocator requires memory_guard_size == gc_heap_guard_size"+ );+ debug_assert_eq!(+ tunables.memory_reservation_for_growth, tunables.gc_heap_reservation_for_growth,+ "pooling allocator requires memory_reservation_for_growth == gc_heap_reservation_for_growth"+ );+ debug_assert_eq!(+ tunables.memory_may_move, tunables.gc_heap_may_move,+ "pooling allocator requires memory_may_move == gc_heap_may_move"+ );+let index_allocator = SimpleIndexAllocator::new(config.limits.total_gc_heaps);let max_gc_heaps = usize::try_from(config.limits.total_gc_heaps).unwrap();diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rsindex ed19a02e83..aaa26ef5f5 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs@@ -67,7 +67,7 @@ use crate::{use std::mem;use std::sync::atomic::{AtomicUsize, Ordering};use std::sync::{Arc, Mutex};-use wasmtime_environ::{DefinedMemoryIndex, Module, Tunables};+use wasmtime_environ::{DefinedMemoryIndex, MemoryKind, MemoryTunables, Module, Tunables};/// A set of allocator slots.///@@ -354,6 +354,7 @@ impl MemoryPool {memory_index: Option<DefinedMemoryIndex>,) -> Result<(MemoryAllocationIndex, Memory)> {let tunables = request.store.engine().tunables();+ let memory_tunables = MemoryTunables::new(tunables, MemoryKind::LinearMemory);let stripe_index = if let Some(pkey) = request.store.get_pkey() {pkey.as_stripe()} else {@@ -391,7 +392,7 @@ impl MemoryPool {// should be returned as an error through `validate_memory_plans`// but double-check here to be sure.assert!(- tunables.memory_reservation + tunables.memory_guard_size+ memory_tunables.reservation() + memory_tunables.guard_size()<= u64::try_from(self.layout.bytes_to_next_stripe_slot().byte_count()).unwrap());@@ -421,11 +422,11 @@ impl MemoryPool {// mmap that would leave an open space for someone// else to come in and map something.let initial_size = usize::try_from(initial_size).unwrap();- slot.instantiate(initial_size, image, ty, tunables)?;+ slot.instantiate(initial_size, image, ty, &memory_tunables)?;let memory = Memory::new_static(ty,- tunables,+ &memory_tunables,MemoryBase::Mmap(base),base_capacity.byte_count(),slot,diff --git a/crates/wasmtime/src/runtime/vm/memory.rs b/crates/wasmtime/src/runtime/vm/memory.rsindex eb525fdf73..2416be7e81 100644--- a/crates/wasmtime/src/runtime/vm/memory.rs+++ b/crates/wasmtime/src/runtime/vm/memory.rs@@ -83,7 +83,7 @@ use crate::runtime::vm::{HostAlignedByteCount, MmapOffset};use crate::runtime::vm::{MemoryImage, MemoryImageSlot, SendSyncPtr};use alloc::sync::Arc;use core::{ops::Range, ptr::NonNull};-use wasmtime_environ::Tunables;+use wasmtime_environ::{MemoryKind, MemoryTunables};#[cfg(feature = "threads")]use wasmtime_environ::Trap;@@ -117,7 +117,7 @@ pub trait RuntimeMemoryCreator: Send + Sync {fn new_memory(&self,ty: &wasmtime_environ::Memory,- tunables: &Tunables,+ memory_tunables: &MemoryTunables<'_>,minimum: usize,maximum: Option<usize>,) -> Result<Box<dyn RuntimeLinearMemory>>;@@ -131,25 +131,25 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator {fn new_memory(&self,ty: &wasmtime_environ::Memory,- tunables: &Tunables,+ memory_tunables: &MemoryTunables<'_>,minimum: usize,maximum: Option<usize>,) -> Result<Box<dyn RuntimeLinearMemory>> {#[cfg(has_virtual_memory)]- if tunables.signals_based_traps- || tunables.memory_guard_size > 0- || tunables.memory_reservation > 0- || tunables.memory_init_cow+ if memory_tunables.tunables().signals_based_traps+ || memory_tunables.guard_size() > 0+ || memory_tunables.reservation() > 0+ || memory_tunables.tunables().memory_init_cow{return Ok(- try_new::<Box<_>>(MmapMemory::new(ty, tunables, minimum, maximum)?)?+ try_new::<Box<_>>(MmapMemory::new(ty, memory_tunables, minimum, maximum)?)?as Box<dyn RuntimeLinearMemory>,);}let _ = maximum;Ok(- try_new::<Box<_>>(MallocMemory::new(ty, tunables, minimum)?)?+ try_new::<Box<_>>(MallocMemory::new(ty, memory_tunables, minimum)?)?as Box<dyn RuntimeLinearMemory>,)}@@ -247,9 +247,10 @@ impl Memory {) -> Result<Self> {let (minimum, maximum) = Self::limit_new(ty, limiter).await?;let tunables = engine.tunables();- let allocation = creator.new_memory(ty, tunables, minimum, maximum)?;+ let memory_tunables = MemoryTunables::new(tunables, MemoryKind::LinearMemory);+ let allocation = creator.new_memory(ty, &memory_tunables, minimum, maximum)?;- let memory = LocalMemory::new(ty, tunables, allocation, memory_image)?;+ let memory = LocalMemory::new(ty, &memory_tunables, allocation, memory_image)?;Ok(if ty.shared {Memory::Shared(SharedMemory::wrap(engine, ty, memory)?)} else {@@ -261,7 +262,7 @@ impl Memory {#[cfg(feature = "pooling-allocator")]pub async fn new_static(ty: &wasmtime_environ::Memory,- tunables: &Tunables,+ memory_tunables: &MemoryTunables<'_>,base: MemoryBase,base_capacity: usize,memory_image: MemoryImageSlot,@@ -275,7 +276,7 @@ impl Memory {// `LocalMemory` structure created, notably we already have// `memory_image` and regardless of configuration settings this memory// can't move its base pointer since it's a fixed allocation.- let mut memory = LocalMemory::new(ty, tunables, allocation, None)?;+ let mut memory = LocalMemory::new(ty, memory_tunables, allocation, None)?;assert!(memory.memory_image.is_none());memory.memory_image = Some(memory_image);memory.memory_may_move = false;@@ -546,7 +547,7 @@ pub struct LocalMemory {impl LocalMemory {pub fn new(ty: &wasmtime_environ::Memory,- tunables: &Tunables,+ memory_tunables: &MemoryTunables<'_>,alloc: Box<dyn RuntimeLinearMemory>,memory_image: Option<&Arc<MemoryImage>>,) -> Result<LocalMemory> {@@ -570,7 +571,7 @@ impl LocalMemory {let mut slot =MemoryImageSlot::create(mmap_base, byte_size, alloc.byte_capacity());- slot.instantiate(alloc.byte_size(), Some(image), ty, tunables)?;+ slot.instantiate(alloc.byte_size(), Some(image), ty, memory_tunables)?;Some(slot)} else {None@@ -583,10 +584,10 @@ impl LocalMemory {Ok(LocalMemory {ty: *ty,alloc,- memory_may_move: ty.memory_may_move(tunables),+ memory_may_move: ty.memory_may_move(memory_tunables),memory_image,- memory_guard_size: tunables.memory_guard_size.try_into().unwrap(),- memory_reservation: tunables.memory_reservation.try_into().unwrap(),+ memory_guard_size: memory_tunables.guard_size().try_into().unwrap(),+ memory_reservation: memory_tunables.reservation().try_into().unwrap(),})}diff --git a/crates/wasmtime/src/runtime/vm/memory/malloc.rs b/crates/wasmtime/src/runtime/vm/memory/malloc.rsindex 7f543b174e..f07a5995d5 100644--- a/crates/wasmtime/src/runtime/vm/memory/malloc.rs+++ b/crates/wasmtime/src/runtime/vm/memory/malloc.rs@@ -9,7 +9,7 @@ use crate::runtime::vm::SendSyncPtr;use crate::runtime::vm::memory::{MemoryBase, RuntimeLinearMemory};use core::mem;use core::ptr::NonNull;-use wasmtime_environ::Tunables;+use wasmtime_environ::MemoryTunables;#[repr(C, align(16))]#[derive(Copy, Clone)]@@ -25,21 +25,21 @@ pub struct MallocMemory {impl MallocMemory {pub fn new(_ty: &wasmtime_environ::Memory,- tunables: &Tunables,+ memory_tunables: &MemoryTunables<'_>,minimum: usize,) -> Result<Self> {- if tunables.memory_guard_size > 0 {+ if memory_tunables.guard_size() > 0 {bail!("malloc memory is only compatible if guard pages aren't used");}- if tunables.memory_reservation > 0 {+ if memory_tunables.reservation() > 0 {bail!("malloc memory is only compatible with no ahead-of-time memory reservation");}- if tunables.memory_init_cow {+ if memory_tunables.tunables().memory_init_cow {bail!("malloc memory cannot be used with CoW images");}let initial_allocation_byte_size = minimum- .checked_add(tunables.memory_reservation_for_growth.try_into()?)+ .checked_add(memory_tunables.reservation_for_growth().try_into()?).context("memory allocation size too large")?;let initial_allocation_len = byte_size_to_element_len(initial_allocation_byte_size);@@ -50,7 +50,7 @@ impl MallocMemory {format!("failed to allocate {initial_allocation_byte_size:#x} \bytes ({minimum:#x} minimum + {:#x} memory_reservation_for_growth)",- tunables.memory_reservation_for_growth,+ memory_tunables.reservation_for_growth(),)})?;@@ -145,6 +145,7 @@ fn grow_storage_to(storage: &mut Vec<Align16>, new_len: usize) {#[cfg(test)]mod tests {use super::*;+ use wasmtime_environ::{MemoryKind, Tunables};// This is currently required by the constructor but otherwise ignored in// the creation of a `MallocMemory`, so just have a single one used in@@ -168,7 +169,9 @@ mod tests {#[test]fn simple() {- let mut memory = MallocMemory::new(&TY, &tunables(), 10).unwrap();+ let t = tunables();+ let mt = MemoryTunables::new(&t, MemoryKind::LinearMemory);+ let mut memory = MallocMemory::new(&TY, &mt, 10).unwrap();assert_eq!(memory.storage.len(), 1);assert_valid(&memory);@@ -191,15 +194,16 @@ mod tests {#[test]fn reservation_not_initialized() {- let tunables = Tunables {+ let t = Tunables {memory_reservation_for_growth: 1 << 20,..tunables()};- let mut memory = MallocMemory::new(&TY, &tunables, 10).unwrap();+ let mt = MemoryTunables::new(&t, MemoryKind::LinearMemory);+ let mut memory = MallocMemory::new(&TY, &mt, 10).unwrap();assert_eq!(memory.storage.len(), 1);assert_eq!(memory.storage.capacity(),- (tunables.memory_reservation_for_growth / 16) as usize + 1,+ (t.memory_reservation_for_growth / 16) as usize + 1,);assert_valid(&memory);@@ -207,7 +211,7 @@ mod tests {assert_eq!(memory.storage.len(), 7);assert_eq!(memory.storage.capacity(),- (tunables.memory_reservation_for_growth / 16) as usize + 1,+ (t.memory_reservation_for_growth / 16) as usize + 1,);assert_valid(&memory);}diff --git a/crates/wasmtime/src/runtime/vm/memory/mmap.rs b/crates/wasmtime/src/runtime/vm/memory/mmap.rsindex e632e62310..8d58e9e1d1 100644--- a/crates/wasmtime/src/runtime/vm/memory/mmap.rs+++ b/crates/wasmtime/src/runtime/vm/memory/mmap.rs@@ -5,7 +5,7 @@ use crate::prelude::*;use crate::runtime::vm::memory::RuntimeLinearMemory;use crate::runtime::vm::{HostAlignedByteCount, Mmap, mmap::AlignedLength};use alloc::sync::Arc;-use wasmtime_environ::Tunables;+use wasmtime_environ::MemoryTunables;use super::MemoryBase;@@ -50,7 +50,7 @@ impl MmapMemory {/// number of wasm pages.pub fn new(ty: &wasmtime_environ::Memory,- tunables: &Tunables,+ memory_tunables: &MemoryTunables<'_>,minimum: usize,maximum: Option<usize>,) -> Result<Self> {@@ -60,9 +60,9 @@ impl MmapMemory {//// Also be sure to round up to the host page size for this value.let offset_guard_bytes =- HostAlignedByteCount::new_rounded_up_u64(tunables.memory_guard_size)+ HostAlignedByteCount::new_rounded_up_u64(memory_tunables.guard_size()).context("tunable.memory_guard_size overflows")?;- let pre_guard_bytes = if tunables.guard_before_linear_memory {+ let pre_guard_bytes = if memory_tunables.tunables().guard_before_linear_memory {offset_guard_bytes} else {HostAlignedByteCount::ZERO@@ -78,8 +78,8 @@ impl MmapMemory {// to reserve any extra for growth.//// If the minimum size doesn't fit within this linear memory.- let mut alloc_bytes = tunables.memory_reservation;- let mut extra_to_reserve_on_growth = tunables.memory_reservation_for_growth;+ let mut alloc_bytes = memory_tunables.reservation();+ let mut extra_to_reserve_on_growth = memory_tunables.reservation_for_growth();let minimum_u64 = u64::try_from(minimum).unwrap();if minimum_u64 <= alloc_bytes {if let Ok(max) = ty.maximum_byte_size() {diff --git a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rsindex 51c9248beb..679984c250 100644--- a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs+++ b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs@@ -33,13 +33,21 @@ impl SharedMemory {/// Construct a new [`SharedMemory`].pub fn new(engine: &Engine, ty: &wasmtime_environ::Memory) -> Result<Self> {let tunables = engine.tunables();+ let memory_tunables = wasmtime_environ::MemoryTunables::new(+ tunables,+ wasmtime_environ::MemoryKind::LinearMemory,+ );// Note that without a limiter being passed to `limit_new` this// `assert_ready` should never panic.let (minimum_bytes, maximum_bytes) = vm::assert_ready(Memory::limit_new(ty, None))?;- let mmap_memory = MmapMemory::new(ty, tunables, minimum_bytes, maximum_bytes)?;+ let mmap_memory = MmapMemory::new(ty, &memory_tunables, minimum_bytes, maximum_bytes)?;let boxed: Box<dyn crate::runtime::vm::RuntimeLinearMemory> =try_new::<Box<_>>(mmap_memory)?;- Self::wrap(engine, ty, LocalMemory::new(ty, tunables, boxed, None)?)+ Self::wrap(+ engine,+ ty,+ LocalMemory::new(ty, &memory_tunables, boxed, None)?,+ )}/// Wrap an existing [Memory] with the locking provided by a [SharedMemory].diff --git a/tests/all/gc.rs b/tests/all/gc.rsindex 8ca6311706..7e27a6915b 100644--- a/tests/all/gc.rs+++ b/tests/all/gc.rs@@ -1355,6 +1355,10 @@ fn gc_heap_oom() -> Result<()> {config.memory_reservation_for_growth(0);config.memory_guard_size(0);config.memory_may_move(false);+ config.gc_heap_reservation(heap_size);+ config.gc_heap_reservation_for_growth(0);+ config.gc_heap_guard_size(0);+ config.gc_heap_may_move(false);if pooling {let mut pooling = crate::small_pool_config();@@ -1753,3 +1757,141 @@ fn gc_heap_does_not_grow_unboundedly() -> Result<()> {Ok(())}++#[test]+#[cfg_attr(miri, ignore)]+fn pooling_gc_different_configs_rejected() -> Result<()> {+ let _ = env_logger::try_init();++ let mut config = Config::new();+ config.wasm_function_references(true);+ config.wasm_gc(true);+ config.collector(Collector::Null);+ config.allocation_strategy(crate::small_pool_config());++ // Set GC heap reservation to a different value than memory reservation.+ config.gc_heap_reservation(0);++ assert!(Engine::new(&config).is_err());++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn memory_guard_pages_but_no_gc_heap_guard_pages() -> Result<()> {+ if std::mem::size_of::<usize>() < std::mem::size_of::<u64>()+ || std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok()+ {+ return Ok(());+ }++ let _ = env_logger::try_init();++ let mut config = Config::new();+ config.wasm_function_references(true);+ config.wasm_gc(true);+ config.collector(Collector::Null);++ // Memories get large guard pages (bounds checks elided for i32), but the+ // GC heap does not (bounds checks required).+ config.memory_reservation(1 << 32);+ config.memory_guard_size(1 << 32);+ config.gc_heap_reservation(0);+ config.gc_heap_guard_size(0);+ config.gc_heap_reservation_for_growth(1 << 20);++ let engine = Engine::new(&config)?;++ let module = Module::new(+ &engine,+ r#"+ (module+ (type $ty (struct (field (mut f32))))+ (func (export "roundtrip") (param (ref null $ty)) (result f32)+ (struct.get $ty 0 (local.get 0))+ )+ (func (export "alloc") (result (ref $ty))+ (struct.new $ty (f32.const 3.14))+ )+ )+ "#,+ )?;++ let mut store = Store::new(&engine, ());+ let instance = Instance::new(&mut store, &module, &[])?;++ let alloc = instance.get_typed_func::<(), Rooted<StructRef>>(&mut store, "alloc")?;+ let roundtrip =+ instance.get_typed_func::<(Option<Rooted<StructRef>>,), f32>(&mut store, "roundtrip")?;++ let s = alloc.call(&mut store, ())?;+ let val = roundtrip.call(&mut store, (Some(s),))?;+ assert_eq!(val, 3.14_f32);++ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn gc_heap_guard_pages_but_no_memory_guard_pages() -> Result<()> {+ if std::mem::size_of::<usize>() < std::mem::size_of::<u64>()+ || std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok()+ {+ return Ok(());+ }++ let _ = env_logger::try_init();++ let mut config = Config::new();+ config.wasm_function_references(true);+ config.wasm_gc(true);+ config.collector(Collector::Null);++ // GC heap gets large guard pages (bounds checks elided), but memories do+ // not (bounds checks required for memories).+ config.gc_heap_reservation(1 << 32);+ config.gc_heap_guard_size(1 << 32);+ config.memory_reservation(0);+ config.memory_guard_size(0);+ config.memory_reservation_for_growth(1 << 20);++ let engine = Engine::new(&config)?;++ let module = Module::new(+ &engine,+ r#"+ (module+ (type $ty (struct (field (mut f32))))+ (memory (export "memory") 1)+ (func (export "roundtrip") (param (ref null $ty)) (result f32)+ (struct.get $ty 0 (local.get 0))+ )+ (func (export "alloc") (result (ref $ty))+ (struct.new $ty (f32.const 2.72))+ )+ (func (export "load") (param i32) (result i32)+ (i32.load (local.get 0))+ )+ )+ "#,+ )?;++ let mut store = Store::new(&engine, ());+ let instance = Instance::new(&mut store, &module, &[])?;++ let alloc = instance.get_typed_func::<(), Rooted<StructRef>>(&mut store, "alloc")?;+ let roundtrip =+ instance.get_typed_func::<(Option<Rooted<StructRef>>,), f32>(&mut store, "roundtrip")?;+ let load = instance.get_typed_func::<(i32,), i32>(&mut store, "load")?;++ let s = alloc.call(&mut store, ())?;+ let val = roundtrip.call(&mut store, (Some(s),))?;+ assert_eq!(val, 2.72_f32);++ // Also exercise the memory access to make sure it works with bounds checks.+ let result = load.call(&mut store, (0,))?;+ assert_eq!(result, 0);++ Ok(())+}diff --git a/tests/disas/gc/null/struct-get-guard-pages.wat b/tests/disas/gc/null/struct-get-guard-pages.watnew file mode 100644index 0000000000..57d473dc84--- /dev/null+++ b/tests/disas/gc/null/struct-get-guard-pages.wat@@ -0,0 +1,126 @@+;;! target = "x86_64"+;;! flags = "-W function-references,gc -C collector=null -O gc-heap-reservation=0x100000000 -O gc-heap-guard-size=0x100000000"+;;! test = "optimize"++(module+ (type $ty (struct (field (mut f32))+ (field (mut i8))+ (field (mut anyref))))++ (func (param (ref null $ty)) (result f32)+ (struct.get $ty 0 (local.get 0))+ )++ (func (param (ref null $ty)) (result i32)+ (struct.get_s $ty 1 (local.get 0))+ )++ (func (param (ref null $ty)) (result i32)+ (struct.get_u $ty 1 (local.get 0))+ )++ (func (param (ref null $ty)) (result anyref)+ (struct.get $ty 2 (local.get 0))+ )+)+;; function u0:0(i64 vmctx, i64, i32) -> f32 tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8+;; gv5 = load.i64 notrap aligned readonly can_move gv4+32+;; gv6 = load.i64 notrap aligned gv4+40+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @0033 trapz v2, user15+;; @0033 v10 = load.i64 notrap aligned readonly can_move v0+8+;; @0033 v5 = load.i64 notrap aligned readonly can_move v10+32+;; @0033 v4 = uextend.i64 v2+;; @0033 v6 = iadd v5, v4+;; @0033 v7 = iconst.i64 8+;; @0033 v8 = iadd v6, v7 ; v7 = 8+;; @0033 v9 = load.f32 notrap aligned little v8+;; @0037 jump block1+;;+;; block1:+;; @0037 return v9+;; }+;;+;; function u0:1(i64 vmctx, i64, i32) -> i32 tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8+;; gv5 = load.i64 notrap aligned readonly can_move gv4+32+;; gv6 = load.i64 notrap aligned gv4+40+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @003c trapz v2, user15+;; @003c v11 = load.i64 notrap aligned readonly can_move v0+8+;; @003c v5 = load.i64 notrap aligned readonly can_move v11+32+;; @003c v4 = uextend.i64 v2+;; @003c v6 = iadd v5, v4+;; @003c v7 = iconst.i64 12+;; @003c v8 = iadd v6, v7 ; v7 = 12+;; @003c v9 = load.i8 notrap aligned little v8+;; @0040 jump block1+;;+;; block1:+;; @003c v10 = sextend.i32 v9+;; @0040 return v10+;; }+;;+;; function u0:2(i64 vmctx, i64, i32) -> i32 tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8+;; gv5 = load.i64 notrap aligned readonly can_move gv4+32+;; gv6 = load.i64 notrap aligned gv4+40+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @0045 trapz v2, user15+;; @0045 v11 = load.i64 notrap aligned readonly can_move v0+8+;; @0045 v5 = load.i64 notrap aligned readonly can_move v11+32+;; @0045 v4 = uextend.i64 v2+;; @0045 v6 = iadd v5, v4+;; @0045 v7 = iconst.i64 12+;; @0045 v8 = iadd v6, v7 ; v7 = 12+;; @0045 v9 = load.i8 notrap aligned little v8+;; @0049 jump block1+;;+;; block1:+;; @0045 v10 = uextend.i32 v9+;; @0049 return v10+;; }+;;+;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8+;; gv5 = load.i64 notrap aligned readonly can_move gv4+32+;; gv6 = load.i64 notrap aligned gv4+40+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @004e trapz v2, user15+;; @004e v10 = load.i64 notrap aligned readonly can_move v0+8+;; @004e v5 = load.i64 notrap aligned readonly can_move v10+32+;; @004e v4 = uextend.i64 v2+;; @004e v6 = iadd v5, v4+;; @004e v7 = iconst.i64 16+;; @004e v8 = iadd v6, v7 ; v7 = 16+;; @004e v9 = load.i32 notrap aligned little v8+;; @0052 jump block1+;;+;; block1:+;; @0052 return v9+;; }diff --git a/tests/disas/gc/null/struct-get-no-guard-pages.wat b/tests/disas/gc/null/struct-get-no-guard-pages.watnew file mode 100644index 0000000000..bb64d884c8--- /dev/null+++ b/tests/disas/gc/null/struct-get-no-guard-pages.wat@@ -0,0 +1,150 @@+;;! target = "x86_64"+;;! flags = "-W function-references,gc -C collector=null -O gc-heap-reservation=0 -O gc-heap-guard-size=0"+;;! test = "optimize"++(module+ (type $ty (struct (field (mut f32))+ (field (mut i8))+ (field (mut anyref))))++ (func (param (ref null $ty)) (result f32)+ (struct.get $ty 0 (local.get 0))+ )++ (func (param (ref null $ty)) (result i32)+ (struct.get_s $ty 1 (local.get 0))+ )++ (func (param (ref null $ty)) (result i32)+ (struct.get_u $ty 1 (local.get 0))+ )++ (func (param (ref null $ty)) (result anyref)+ (struct.get $ty 2 (local.get 0))+ )+)+;; function u0:0(i64 vmctx, i64, i32) -> f32 tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8+;; gv5 = load.i64 notrap aligned gv4+32+;; gv6 = load.i64 notrap aligned gv4+40+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @0033 trapz v2, user15+;; @0033 v4 = uextend.i64 v2+;; @0033 v5 = iconst.i64 24+;; @0033 v6 = uadd_overflow_trap v4, v5, user1 ; v5 = 24+;; @0033 v18 = load.i64 notrap aligned readonly can_move v0+8+;; @0033 v7 = load.i64 notrap aligned v18+40+;; @0033 v9 = load.i64 notrap aligned v18+32+;; @0033 v8 = icmp ugt v6, v7+;; @0033 v11 = iconst.i64 0+;; @0033 v10 = iadd v9, v4+;; @0033 v12 = select_spectre_guard v8, v11, v10 ; v11 = 0+;; @0033 v13 = iconst.i64 8+;; @0033 v14 = iadd v12, v13 ; v13 = 8+;; @0033 v15 = load.f32 notrap aligned little v14+;; @0037 jump block1+;;+;; block1:+;; @0037 return v15+;; }+;;+;; function u0:1(i64 vmctx, i64, i32) -> i32 tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8+;; gv5 = load.i64 notrap aligned gv4+32+;; gv6 = load.i64 notrap aligned gv4+40+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @003c trapz v2, user15+;; @003c v4 = uextend.i64 v2+;; @003c v5 = iconst.i64 24+;; @003c v6 = uadd_overflow_trap v4, v5, user1 ; v5 = 24+;; @003c v19 = load.i64 notrap aligned readonly can_move v0+8+;; @003c v7 = load.i64 notrap aligned v19+40+;; @003c v9 = load.i64 notrap aligned v19+32+;; @003c v8 = icmp ugt v6, v7+;; @003c v11 = iconst.i64 0+;; @003c v10 = iadd v9, v4+;; @003c v12 = select_spectre_guard v8, v11, v10 ; v11 = 0+;; @003c v13 = iconst.i64 12+;; @003c v14 = iadd v12, v13 ; v13 = 12+;; @003c v15 = load.i8 notrap aligned little v14+;; @0040 jump block1+;;+;; block1:+;; @003c v16 = sextend.i32 v15+;; @0040 return v16+;; }+;;+;; function u0:2(i64 vmctx, i64, i32) -> i32 tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8+;; gv5 = load.i64 notrap aligned gv4+32+;; gv6 = load.i64 notrap aligned gv4+40+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @0045 trapz v2, user15+;; @0045 v4 = uextend.i64 v2+;; @0045 v5 = iconst.i64 24+;; @0045 v6 = uadd_overflow_trap v4, v5, user1 ; v5 = 24+;; @0045 v19 = load.i64 notrap aligned readonly can_move v0+8+;; @0045 v7 = load.i64 notrap aligned v19+40+;; @0045 v9 = load.i64 notrap aligned v19+32+;; @0045 v8 = icmp ugt v6, v7+;; @0045 v11 = iconst.i64 0+;; @0045 v10 = iadd v9, v4+;; @0045 v12 = select_spectre_guard v8, v11, v10 ; v11 = 0+;; @0045 v13 = iconst.i64 12+;; @0045 v14 = iadd v12, v13 ; v13 = 12+;; @0045 v15 = load.i8 notrap aligned little v14+;; @0049 jump block1+;;+;; block1:+;; @0045 v16 = uextend.i32 v15+;; @0049 return v16+;; }+;;+;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8+;; gv5 = load.i64 notrap aligned gv4+32+;; gv6 = load.i64 notrap aligned gv4+40+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @004e trapz v2, user15+;; @004e v4 = uextend.i64 v2+;; @004e v5 = iconst.i64 24+;; @004e v6 = uadd_overflow_trap v4, v5, user1 ; v5 = 24+;; @004e v18 = load.i64 notrap aligned readonly can_move v0+8+;; @004e v7 = load.i64 notrap aligned v18+40+;; @004e v9 = load.i64 notrap aligned v18+32+;; @004e v8 = icmp ugt v6, v7+;; @004e v11 = iconst.i64 0+;; @004e v10 = iadd v9, v4+;; @004e v12 = select_spectre_guard v8, v11, v10 ; v11 = 0+;; @004e v13 = iconst.i64 16+;; @004e v14 = iadd v12, v13 ; v13 = 16+;; @004e v15 = load.i32 notrap aligned little v14+;; @0052 jump block1+;;+;; block1:+;; @0052 return v15+;; }diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex 732eea6b43..69a2baf76b 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -24,8 +24,8 @@ use wasmparser::{};use wasmtime_cranelift::{TRAP_BAD_SIGNATURE, TRAP_HEAP_MISALIGNED, TRAP_TABLE_OUT_OF_BOUNDS};use wasmtime_environ::{- FUNCREF_MASK, GlobalIndex, MemoryIndex, PtrSize, TableIndex, Tunables, TypeIndex, WasmHeapType,- WasmValType,+ FUNCREF_MASK, GlobalIndex, MemoryIndex, MemoryKind, MemoryTunables, PtrSize, TableIndex,+ Tunables, TypeIndex, WasmHeapType, WasmValType,};mod context;@@ -690,9 +690,10 @@ where)?;let offset_with_access_size = add_offset_and_access_size(offset, access_size);+ let memory_tunables = MemoryTunables::new(self.tunables, MemoryKind::LinearMemory);let can_elide_bounds_check = heap.memory- .can_elide_bounds_check(self.tunables, self.env.page_size_log2);+ .can_elide_bounds_check(&memory_tunables, self.env.page_size_log2);let addr = if offset_with_access_size > heap.memory.maximum_byte_size().unwrap_or(u64::MAX)|| (!self.tunables.memory_may_move