Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Add separate GC heap tunables (#13080)

Nick Fitzgerald13b62708fcb0
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs
index 35da8409b0..2354ccfe8f 100644
--- a/crates/cli-flags/src/lib.rs
+++ b/crates/cli-flags/src/lib.rs
@@ -67,6 +67,19 @@ wasmtime_option_group! {
/// Size, in bytes, of guard pages for linear memories.
pub memory_guard_size: Option<u64>,
+ /// Do not allow the GC heap to move in the host process's address
+ /// space.
+ pub gc_heap_may_move: Option<bool>,
+
+ /// Initial virtual memory allocation size for the GC heap.
+ pub gc_heap_reservation: Option<u64>,
+
+ /// Bytes to reserve at the end of the GC heap for growth into.
+ pub gc_heap_reservation_for_growth: Option<u64>,
+
+ /// Size, in bytes, of guard pages for the GC heap.
+ pub gc_heap_guard_size: Option<u64>,
+
/// Indicates whether an unmapped region of memory is placed before all
/// linear memories.
pub guard_before_linear_memory: Option<bool>,
@@ -891,6 +904,19 @@ impl CommonOptions {
if let Some(enable) = self.opts.guard_before_linear_memory {
config.guard_before_linear_memory(enable);
}
+
+ if let Some(size) = self.opts.gc_heap_reservation {
+ config.gc_heap_reservation(size);
+ }
+ if let Some(enable) = self.opts.gc_heap_may_move {
+ config.gc_heap_may_move(enable);
+ }
+ if let Some(size) = self.opts.gc_heap_guard_size {
+ config.gc_heap_guard_size(size);
+ }
+ if let Some(size) = self.opts.gc_heap_reservation_for_growth {
+ config.gc_heap_reservation_for_growth(size);
+ }
if let Some(enable) = self.opts.table_lazy_init {
config.table_lazy_init(enable);
}
diff --git a/crates/cranelift/src/bounds_checks.rs b/crates/cranelift/src/bounds_checks.rs
index 2743dd861d..310549224c 100644
--- a/crates/cranelift/src/bounds_checks.rs
+++ b/crates/cranelift/src/bounds_checks.rs
@@ -184,16 +184,17 @@ fn bounds_check_field_access(
env.heap_access_spectre_mitigation() && clif_memory_traps_enabled;
let host_page_size_log2 = env.target_config().page_size_align_log2;
+ let memory_tunables = heap.memory_tunables(env.tunables());
let can_use_virtual_memory = heap
.memory
- .can_use_virtual_memory(env.tunables(), host_page_size_log2)
+ .can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2)
&& clif_memory_traps_enabled;
let can_elide_bounds_check = heap
.memory
- .can_elide_bounds_check(env.tunables(), host_page_size_log2)
+ .can_elide_bounds_check(&memory_tunables, host_page_size_log2)
&& clif_memory_traps_enabled;
- let memory_guard_size = env.tunables().memory_guard_size;
- let memory_reservation = env.tunables().memory_reservation;
+ let memory_guard_size = memory_tunables.guard_size();
+ let memory_reservation = memory_tunables.reservation();
let offset_and_size = offset_plus_size(offset, access_size);
let statically_in_bounds = statically_in_bounds(&builder.func, heap, index, offset_and_size);
@@ -348,7 +349,7 @@ fn bounds_check_field_access(
// factor in the guard pages here.
if can_use_virtual_memory
&& heap.memory.minimum_byte_size().unwrap_or(u64::MAX) <= memory_reservation
- && !heap.memory.memory_may_move(env.tunables())
+ && !heap.memory.memory_may_move(&memory_tunables)
&& memory_reservation >= offset_and_size
{
let adjusted_bound = memory_reservation.checked_sub(offset_and_size).unwrap();
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 1bde277a33..83b8538415 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -4,8 +4,8 @@ pub(crate) mod stack_switching;
use crate::BuiltinFunctionSignatures;
use crate::compiler::Compiler;
use crate::translate::{
- FuncTranslationStacks, GlobalVariable, Heap, HeapData, StructFieldsVec, TableData, TableSize,
- TargetEnvironment,
+ FuncTranslationStacks, GlobalVariable, Heap, HeapData, MemoryKind, StructFieldsVec, TableData,
+ TableSize, TargetEnvironment,
};
use crate::trap::TranslateTrap;
use cranelift_codegen::cursor::FuncCursor;
@@ -28,10 +28,10 @@ use wasmtime_core::math::f64_cvt_to_int_bounds;
use wasmtime_environ::{
BuiltinFunctionIndex, ComponentPC, DataIndex, DefinedFuncIndex, ElemIndex,
EngineOrModuleTypeIndex, FrameStateSlotBuilder, FrameValType, FuncIndex, FuncKey,
- GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, Module, ModuleInternedTypeIndex,
- ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex, TagIndex, Tunables,
- TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType,
- WasmHeapType, WasmRefType, WasmResult, WasmValType,
+ GlobalConstValue, GlobalIndex, IndexType, Memory, MemoryIndex, MemoryTunables, Module,
+ ModuleInternedTypeIndex, ModuleTranslation, ModuleTypesBuilder, PtrSize, Table, TableIndex,
+ TagIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, WasmCompositeInnerType, WasmFuncType,
+ WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmValType,
};
use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK};
@@ -1566,6 +1566,7 @@ impl FuncEnvironment<'_> {
self.heaps.push(HeapData {
base,
bound,
+ kind: MemoryKind::LinearMemory,
memory,
})
}
@@ -1578,9 +1579,10 @@ impl FuncEnvironment<'_> {
offset: i32,
) -> ir::GlobalValue {
let pointer_type = self.pointer_type();
+ let memory_tunables = MemoryTunables::new(self.tunables, MemoryKind::LinearMemory);
let mut flags = ir::MemFlags::trusted().with_can_move();
- if !memory.memory_may_move(self.tunables) {
+ if !memory.memory_may_move(&memory_tunables) {
flags.set_readonly();
}
diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs
index 73ef862997..d2d42715a3 100644
--- a/crates/cranelift/src/func_environ/gc/enabled.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled.rs
@@ -1,7 +1,7 @@
use super::{ArrayInit, GcCompiler};
use crate::bounds_checks::BoundsCheck;
use crate::func_environ::{Extension, FuncEnvironment};
-use crate::translate::{Heap, HeapData, StructFieldsVec, TargetEnvironment};
+use crate::translate::{Heap, HeapData, MemoryKind, StructFieldsVec, TargetEnvironment};
use crate::trap::TranslateTrap;
use crate::{Reachability, TRAP_INTERNAL_ASSERT};
use cranelift_codegen::ir::immediates::Offset32;
@@ -1449,10 +1449,12 @@ impl FuncEnvironment<'_> {
let offset = self.offsets.ptr.vmstore_context_gc_heap_base();
let mut flags = ir::MemFlags::trusted();
+ let memory_tunables =
+ wasmtime_environ::MemoryTunables::new(self.tunables, MemoryKind::GcHeap);
if !self
.tunables
.gc_heap_memory_type()
- .memory_may_move(self.tunables)
+ .memory_may_move(&memory_tunables)
{
flags.set_readonly();
flags.set_can_move();
@@ -1512,6 +1514,7 @@ impl FuncEnvironment<'_> {
base,
bound,
memory,
+ kind: MemoryKind::GcHeap,
});
self.gc_heap = Some(heap);
heap
diff --git a/crates/cranelift/src/translate/heap.rs b/crates/cranelift/src/translate/heap.rs
index 32992dd191..459afd32d0 100644
--- a/crates/cranelift/src/translate/heap.rs
+++ b/crates/cranelift/src/translate/heap.rs
@@ -4,6 +4,8 @@ use cranelift_codegen::ir::{self, GlobalValue, Type};
use cranelift_entity::entity_impl;
use wasmtime_environ::{IndexType, Memory};
+pub use wasmtime_environ::MemoryKind;
+
/// An opaque reference to a [`HeapData`][crate::HeapData].
///
/// While the order is stable, it is arbitrary.
@@ -45,6 +47,9 @@ pub struct HeapData {
/// The type of wasm memory that this heap is operating on.
pub memory: Memory,
+
+ /// Whether this is a linear memory or a GC heap.
+ pub kind: MemoryKind,
}
impl HeapData {
@@ -54,4 +59,12 @@ impl HeapData {
IndexType::I64 => ir::types::I64,
}
}
+
+ /// Get the [`MemoryTunables`] for this heap based on its [`MemoryKind`].
+ pub fn memory_tunables<'a>(
+ &self,
+ tunables: &'a wasmtime_environ::Tunables,
+ ) -> wasmtime_environ::MemoryTunables<'a> {
+ wasmtime_environ::MemoryTunables::new(tunables, self.kind)
+ }
}
diff --git a/crates/cranelift/src/translate/mod.rs b/crates/cranelift/src/translate/mod.rs
index 7effeb759b..f890c1e4a7 100644
--- a/crates/cranelift/src/translate/mod.rs
+++ b/crates/cranelift/src/translate/mod.rs
@@ -20,7 +20,7 @@ mod translation_utils;
pub use self::environ::{GlobalVariable, StructFieldsVec, TargetEnvironment};
pub use self::func_translator::FuncTranslator;
-pub use self::heap::{Heap, HeapData};
+pub use self::heap::{Heap, HeapData, MemoryKind};
pub use self::stack::FuncTranslationStacks;
pub use self::table::{TableData, TableSize};
pub use self::translation_utils::*;
diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs
index 898029a63d..354021e0dc 100644
--- a/crates/environ/src/tunables.rs
+++ b/crates/environ/src/tunables.rs
@@ -160,6 +160,29 @@ define_tunables! {
/// forced and the counter is reset. Only effective when
/// `cfg(gc_zeal)` is enabled.
pub gc_zeal_alloc_counter: Option<NonZeroU32>,
+
+ /// Initial size, in bytes, to be allocated for GC heaps.
+ ///
+ /// This is the same as `memory_reservation` but for GC heaps.
+ pub gc_heap_reservation: u64,
+
+ /// The size, in bytes, of the guard page region for GC heaps.
+ ///
+ /// This is the same as `memory_guard_size` but for GC heaps.
+ pub gc_heap_guard_size: u64,
+
+ /// The size, in bytes, to allocate at the end of a relocated GC heap
+ /// for growth.
+ ///
+ /// This is the same as `memory_reservation_for_growth` but for GC
+ /// heaps.
+ pub gc_heap_reservation_for_growth: u64,
+
+ /// Whether or not GC heaps are allowed to be reallocated after initial
+ /// allocation at runtime.
+ ///
+ /// This is the same as `memory_may_move` but for GC heaps.
+ pub gc_heap_may_move: bool,
}
pub struct ConfigTunables {
@@ -200,6 +223,7 @@ impl Tunables {
if target.is_pulley() {
ret.signals_based_traps = false;
ret.memory_guard_size = 0;
+ ret.gc_heap_guard_size = 0;
}
Ok(ret)
}
@@ -239,6 +263,10 @@ impl Tunables {
concurrency_support: true,
recording: false,
gc_zeal_alloc_counter: None,
+ gc_heap_reservation: 0,
+ gc_heap_guard_size: 0,
+ gc_heap_reservation_for_growth: 0,
+ gc_heap_may_move: true,
}
}
@@ -253,6 +281,12 @@ impl Tunables {
memory_reservation_for_growth: 1 << 20, // 1MB
signals_based_traps: true,
+ // GC heaps on 32-bit: conservative defaults similar to linear
+ // memories.
+ gc_heap_reservation: 10 * (1 << 20),
+ gc_heap_guard_size: 0x1_0000,
+ gc_heap_reservation_for_growth: 1 << 20, // 1MB
+
..Tunables::default_miri()
}
}
@@ -278,6 +312,12 @@ impl Tunables {
// to avoid memory movement.
memory_reservation_for_growth: 2 << 30, // 2GB
+ // GC heaps on 64-bit: use 4GiB reservation and 32MiB guard pages
+ // to enable bounds check elision, matching linear memory defaults.
+ gc_heap_reservation: 1 << 32,
+ gc_heap_guard_size: 32 << 20,
+ gc_heap_reservation_for_growth: 2 << 30, // 2GB
+
signals_based_traps: true,
..Tunables::default_miri()
}
@@ -298,6 +338,73 @@ impl Tunables {
}
}
+/// Whether a heap is backing a linear memory or a GC heap.
+///
+/// This is used by [`MemoryTunables`] to select between the memory tunables and
+/// the GC heap tunables.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum MemoryKind {
+ /// A WebAssembly linear memory.
+ LinearMemory,
+ /// A GC heap for garbage-collected objects.
+ GcHeap,
+}
+
+/// A view into a [`Tunables`] that selects the appropriate linear-memory or
+/// GC-heap flavor of each tunable based on a [`MemoryKind`].
+pub struct MemoryTunables<'a> {
+ tunables: &'a Tunables,
+ kind: MemoryKind,
+}
+
+impl<'a> MemoryTunables<'a> {
+ /// Create a new `MemoryTunables` view.
+ pub fn new(tunables: &'a Tunables, kind: MemoryKind) -> Self {
+ Self { tunables, kind }
+ }
+
+ /// The virtual memory reservation for this kind of memory.
+ pub fn reservation(&self) -> u64 {
+ match self.kind {
+ MemoryKind::LinearMemory => self.tunables.memory_reservation,
+ MemoryKind::GcHeap => self.tunables.gc_heap_reservation,
+ }
+ }
+
+ /// The size of the guard page region for this kind of memory.
+ pub fn guard_size(&self) -> u64 {
+ match self.kind {
+ MemoryKind::LinearMemory => self.tunables.memory_guard_size,
+ MemoryKind::GcHeap => self.tunables.gc_heap_guard_size,
+ }
+ }
+
+ /// Extra virtual memory to reserve beyond the initially mapped pages for
+ /// this kind of memory.
+ pub fn reservation_for_growth(&self) -> u64 {
+ match self.kind {
+ MemoryKind::LinearMemory => self.tunables.memory_reservation_for_growth,
+ MemoryKind::GcHeap => self.tunables.gc_heap_reservation_for_growth,
+ }
+ }
+
+ /// Whether this kind of memory's base pointer may be relocated at runtime.
+ pub fn may_move(&self) -> bool {
+ match self.kind {
+ MemoryKind::LinearMemory => self.tunables.memory_may_move,
+ MemoryKind::GcHeap => self.tunables.gc_heap_may_move,
+ }
+ }
+
+ /// Get the underlying tunables.
+ ///
+ /// This is ONLY for accessing tunable fields that DO NOT come in a
+ /// linear-memory flavor and a GC-heap flavor.
+ pub fn tunables(&self) -> &'a Tunables {
+ self.tunables
+ }
+}
+
/// The garbage collector implementation to use.
#[derive(Clone, Copy, Hash, Serialize, Deserialize, Debug, PartialEq, Eq)]
pub enum Collector {
diff --git a/crates/environ/src/types.rs b/crates/environ/src/types.rs
index e582bc7c5d..2847be79cd 100644
--- a/crates/environ/src/types.rs
+++ b/crates/environ/src/types.rs
@@ -1,6 +1,6 @@
use crate::{
- PanicOnOom as _, Tunables, WasmResult, collections::TryCow, error::OutOfMemory, prelude::*,
- wasm_unsupported,
+ MemoryTunables, PanicOnOom as _, Tunables, WasmResult, collections::TryCow, error::OutOfMemory,
+ prelude::*, wasm_unsupported,
};
use alloc::boxed::Box;
use core::{fmt, ops::Range};
@@ -2266,26 +2266,30 @@ impl Memory {
/// Returns whether this memory is a candidate for bounds check elision
/// given the configuration and host page size.
///
- /// This function determines whether the given compilation configuration and
- /// hos enables possible bounds check elision for this memory. Bounds checks
+ /// This function determines whether the given compilation configuration
+ /// enables possible bounds check elision for this memory. Bounds checks
/// can only be elided if [`Memory::can_use_virtual_memory`] returns `true`
/// for example but there are additionally requirements on the index size of
- /// this memory and the memory reservation in `tunables`.
+ /// this memory and the memory reservation in the tunables.
///
/// Currently the only case that supports bounds check elision is when all
/// of these apply:
///
/// * When [`Memory::can_use_virtual_memory`] returns `true`.
/// * This is a 32-bit linear memory (e.g. not 64-bit)
- /// * `tunables.memory_reservation` is in excess of 4GiB
+ /// * The reservation + guard size is in excess of 4GiB
///
/// In this situation all computable addresses fall within the reserved
/// space (modulo static offsets factoring in guard pages) so bounds checks
/// may be elidable.
- pub fn can_elide_bounds_check(&self, tunables: &Tunables, host_page_size_log2: u8) -> bool {
- self.can_use_virtual_memory(tunables, host_page_size_log2)
+ pub fn can_elide_bounds_check(
+ &self,
+ memory_tunables: &MemoryTunables<'_>,
+ host_page_size_log2: u8,
+ ) -> bool {
+ self.can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2)
&& self.idx_type == IndexType::I32
- && tunables.memory_reservation + tunables.memory_guard_size >= (1 << 32)
+ && memory_tunables.reservation() + memory_tunables.guard_size() >= (1 << 32)
}
/// Returns the static size of this heap in bytes at runtime, if available.
@@ -2303,7 +2307,7 @@ impl Memory {
/// When this function returns `false` then it means that after the initial
/// allocation the base pointer is constant for the entire lifetime of a
/// memory. This can enable compiler optimizations, for example.
- pub fn memory_may_move(&self, tunables: &Tunables) -> bool {
+ pub fn memory_may_move(&self, memory_tunables: &MemoryTunables<'_>) -> bool {
// Shared memories cannot ever relocate their base pointer so the
// settings configured in the engine must be appropriate for them ahead
// of time.
@@ -2313,7 +2317,7 @@ impl Memory {
// If movement is disallowed in engine configuration, then the answer is
// "no".
- if !tunables.memory_may_move {
+ if !memory_tunables.may_move() {
return false;
}
@@ -2326,7 +2330,7 @@ impl Memory {
// If the maximum size of this memory is above the threshold of the
// initial memory reservation then the memory may move.
let max = self.maximum_byte_size().unwrap_or(u64::MAX);
- max > tunables.memory_reservation
+ max > memory_tunables.reservation()
}
/// Tests whether this memory type is allowed to grow up to `size` bytes.
diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs
index 0180136142..1e80f47825 100644
--- a/crates/fuzzing/src/generators/config.rs
+++ b/crates/fuzzing/src/generators/config.rs
@@ -240,6 +240,11 @@ impl Config {
pooling.total_stacks = pooling.total_stacks.max(limits::TOTAL_STACKS);
}
+ // Re-enforce internal consistency after all the adjustments above
+ // (e.g. memory_reservation may have been bumped without a
+ // corresponding bump to gc_heap_reservation).
+ self.wasmtime.make_internally_consistent();
+
// Return the test configuration that this fuzz configuration represents
// which is used afterwards to test if the `test` here is expected to
// fail or not.
@@ -408,23 +413,37 @@ impl Config {
memory_config.configure(&mut cfg);
};
- // If malloc-based memory is going to be used, which requires these four
- // options set to specific values (and Pulley auto-sets two of them)
- // then be sure to cap `memory_reservation_for_growth` at a smaller
- // value than the default. For malloc-based memory reservation beyond
- // the end of memory isn't captured by `StoreLimiter` so we need to be
- // sure it's small enough to not blow OOM limits while fuzzing.
- if ((cfg.opts.signals_based_traps == Some(true) && cfg.opts.memory_guard_size == Some(0))
- || self.wasmtime.compiler_strategy == CompilerStrategy::CraneliftPulley)
- && cfg.opts.memory_reservation == Some(0)
- && cfg.opts.memory_init_cow == Some(false)
- {
- let growth = &mut cfg.opts.memory_reservation_for_growth;
- let max = 1 << 20;
- *growth = match *growth {
- Some(n) => Some(n.min(max)),
- None => Some(max),
- };
+ // If malloc-based memory is going to be used, which requires these
+ // options set to specific values (and Pulley auto-sets some of them)
+ // then be sure to cap `memory_reservation_for_growth` and
+ // `gc_heap_reservation_for_growth` at a smaller value than the
+ // default. For malloc-based memory/heaps, reservation beyond the end
+ // isn't captured by `StoreLimiter` so we need to be sure it's small
+ // enough to not blow OOM limits while fuzzing.
+ let is_pulley = self.wasmtime.compiler_strategy == CompilerStrategy::CraneliftPulley;
+ let signals_traps = cfg.opts.signals_based_traps == Some(true);
+ if signals_traps || is_pulley {
+ if ((signals_traps && cfg.opts.memory_guard_size == Some(0)) || is_pulley)
+ && cfg.opts.memory_reservation == Some(0)
+ && cfg.opts.memory_init_cow == Some(false)
+ {
+ let growth = &mut cfg.opts.memory_reservation_for_growth;
+ let max = 1 << 20;
+ *growth = match *growth {
+ Some(n) => Some(n.min(max)),
+ None => Some(max),
+ };
+ }
+ if ((signals_traps && cfg.opts.gc_heap_guard_size == Some(0)) || is_pulley)
+ && cfg.opts.gc_heap_reservation == Some(0)
+ {
+ let growth = &mut cfg.opts.gc_heap_reservation_for_growth;
+ let max = 1 << 20;
+ *growth = match *growth {
+ Some(n) => Some(n.min(max)),
+ None => Some(max),
+ };
+ }
}
log::debug!("creating wasmtime config with CLI options:\n{cfg}");
@@ -798,7 +817,19 @@ impl WasmtimeConfig {
/// be considered a "TODO" to go implement more stuff in Wasmtime to accept
/// these sorts of configurations. For now though it's intended to reflect
/// the current state of the engine's development.
- fn make_internally_consistent(&mut self) {
+ pub(crate) fn make_internally_consistent(&mut self) {
+ // When using the pooling allocator, GC heap tunables must match memory
+ // tunables.
+ if let InstanceAllocationStrategy::Pooling(_) = &self.strategy {
+ self.memory_config.gc_heap_reservation = self.memory_config.memory_reservation;
+ self.memory_config.gc_heap_guard_size = self.memory_config.memory_guard_size;
+ self.memory_config.gc_heap_reservation_for_growth =
+ self.memory_config.memory_reservation_for_growth;
+ // memory_may_move is not in MemoryConfig, but gc_heap_may_move
+ // must not conflict. Set it to None so the default matches.
+ self.memory_config.gc_heap_may_move = None;
+ }
+
if !self.signals_based_traps {
let cfg = &mut self.memory_config;
// Spectre-based heap mitigations require signal handlers so
@@ -807,8 +838,8 @@ impl WasmtimeConfig {
cfg.cranelift_enable_heap_access_spectre_mitigations = None;
// With configuration settings that match the use of malloc for
- // linear memories cap the `memory_reservation_for_growth` value
- // to something reasonable to avoid OOM in fuzzing.
+ // linear memories and GC heaps, cap the reservation-for-growth
+ // values to something reasonable to avoid OOM in fuzzing.
if !cfg.memory_init_cow
&& cfg.memory_guard_size == Some(0)
&& cfg.memory_reservation == Some(0)
@@ -820,6 +851,17 @@ impl WasmtimeConfig {
cfg.memory_reservation_for_growth = Some(min);
}
}
+
+ // Similarly cap gc_heap_reservation_for_growth when signals-based
+ // traps are disabled and the GC heap uses malloc-style allocation.
+ if cfg.gc_heap_guard_size == Some(0) && cfg.gc_heap_reservation == Some(0) {
+ let min = 10 << 20; // 10 MiB
+ if let Some(val) = &mut cfg.gc_heap_reservation_for_growth {
+ *val = (*val).min(min);
+ } else {
+ cfg.gc_heap_reservation_for_growth = Some(min);
+ }
+ }
}
}
}
diff --git a/crates/fuzzing/src/generators/memory.rs b/crates/fuzzing/src/generators/memory.rs
index 405832bffa..ea36ad3011 100644
--- a/crates/fuzzing/src/generators/memory.rs
+++ b/crates/fuzzing/src/generators/memory.rs
@@ -124,6 +124,10 @@ pub struct MemoryConfig {
pub guard_before_linear_memory: bool,
pub cranelift_enable_heap_access_spectre_mitigations: Option<bool>,
pub memory_init_cow: bool,
+ pub gc_heap_reservation: Option<u64>,
+ pub gc_heap_guard_size: Option<u64>,
+ pub gc_heap_reservation_for_growth: Option<u64>,
+ pub gc_heap_may_move: Option<bool>,
}
impl<'a> Arbitrary<'a> for MemoryConfig {
@@ -142,6 +146,12 @@ impl<'a> Arbitrary<'a> for MemoryConfig {
guard_before_linear_memory: u.arbitrary()?,
cranelift_enable_heap_access_spectre_mitigations: u.arbitrary()?,
memory_init_cow: u.arbitrary()?,
+
+ // GC heap tunables, same ranges as corresponding memory tunables.
+ gc_heap_reservation: interesting_virtual_memory_size(u, 33)?,
+ gc_heap_guard_size: interesting_virtual_memory_size(u, 32)?,
+ gc_heap_reservation_for_growth: interesting_virtual_memory_size(u, 30)?,
+ gc_heap_may_move: u.arbitrary()?,
})
}
}
@@ -181,6 +191,10 @@ impl MemoryConfig {
cfg.opts.memory_reservation_for_growth = self.memory_reservation_for_growth;
cfg.opts.guard_before_linear_memory = Some(self.guard_before_linear_memory);
cfg.opts.memory_init_cow = Some(self.memory_init_cow);
+ cfg.opts.gc_heap_reservation = self.gc_heap_reservation;
+ cfg.opts.gc_heap_guard_size = self.gc_heap_guard_size;
+ cfg.opts.gc_heap_reservation_for_growth = self.gc_heap_reservation_for_growth;
+ cfg.opts.gc_heap_may_move = self.gc_heap_may_move;
if let Some(enable) = self.cranelift_enable_heap_access_spectre_mitigations {
cfg.codegen.cranelift.push((
diff --git a/crates/fuzzing/src/oracles/component_api.rs b/crates/fuzzing/src/oracles/component_api.rs
index 940529418b..6bbcd25eb3 100644
--- a/crates/fuzzing/src/oracles/component_api.rs
+++ b/crates/fuzzing/src/oracles/component_api.rs
@@ -211,6 +211,11 @@ fn store<T>(input: &mut Unstructured<'_>, val: T) -> arbitrary::Result<Store<T>>
10 << 20,
);
+ // Re-enforce internal consistency after the adjustments above (e.g.
+ // memory_reservation may have been bumped without a corresponding bump
+ // to gc_heap_reservation).
+ config.wasmtime.make_internally_consistent();
+
let engine = Engine::new(
config
.to_wasmtime()
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index 323d9b7ce0..a51bb7f2a7 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -50,6 +50,12 @@ pub enum InstanceAllocationStrategy {
/// A pool of resources is created in advance and module instantiation reuses resources
/// from the pool. Resources are returned to the pool when the `Store` referencing the instance
/// is dropped.
+ ///
+ /// When GC is enabled, the pooling allocator requires that the GC heap
+ /// configuration matches the linear memory configuration (i.e.,
+ /// `gc_heap_reservation` must equal `memory_reservation`, etc.). By
+ /// default, if no `gc_heap_*` tunables are explicitly configured, they
+ /// automatically inherit the `memory_*` values.
#[cfg(feature = "pooling-allocator")]
Pooling(PoolingAllocationConfig),
}
@@ -1947,6 +1953,75 @@ impl Config {
self
}
+ /// Configures the initial size, in bytes, to be allocated for GC heaps.
+ ///
+ /// This is similar to [`Config::memory_reservation`] but applies to the GC
+ /// heap rather than to linear memories. See that method for more details
+ /// on what "reservation" means and the implications of this setting.
+ ///
+ /// ## Default
+ ///
+ /// If none of the `gc_heap_*` tunables are explicitly configured, they
+ /// default to the same values as their `memory_*` counterparts. Otherwise,
+ /// the default value for this property depends on the host platform: for
+ /// 64-bit platforms this defaults to 4GiB, and for 32-bit platforms this
+ /// defaults to 10MiB.
+ pub fn gc_heap_reservation(&mut self, bytes: u64) -> &mut Self {
+ self.tunables.gc_heap_reservation = Some(bytes);
+ self
+ }
+
+ /// Configures the size, in bytes, of the guard page region for GC heaps.
+ ///
+ /// This is similar to [`Config::memory_guard_size`] but applies to the GC
+ /// heap rather than to linear memories. See that method for more details on
+ /// what guard pages are and the implications of this setting.
+ ///
+ /// ## Default
+ ///
+ /// If none of the `gc_heap_*` tunables are explicitly configured, they
+ /// default to the same values as their `memory_*` counterparts. Otherwise,
+ /// the default value for this property is 32MiB on 64-bit platforms and
+ /// 64KiB on 32-bit platforms.
+ pub fn gc_heap_guard_size(&mut self, bytes: u64) -> &mut Self {
+ self.tunables.gc_heap_guard_size = Some(bytes);
+ self
+ }
+
+ /// Configures the size, in bytes, of the extra virtual memory space
+ /// reserved after a GC heap is relocated.
+ ///
+ /// This is similar to [`Config::memory_reservation_for_growth`] but applies
+ /// to the GC heap rather than to linear memories. See that method for more
+ /// details.
+ ///
+ /// ## Default
+ ///
+ /// If none of the `gc_heap_*` tunables are explicitly configured, they
+ /// default to the same values as their `memory_*` counterparts. Otherwise,
+ /// for 64-bit platforms this defaults to 2GiB, and for 32-bit platforms
+ /// this defaults to 1MiB.
+ pub fn gc_heap_reservation_for_growth(&mut self, bytes: u64) -> &mut Self {
+ self.tunables.gc_heap_reservation_for_growth = Some(bytes);
+ self
+ }
+
+ /// Indicates whether GC heaps are allowed to be reallocated after initial
+ /// allocation at runtime.
+ ///
+ /// This is similar to [`Config::memory_may_move`] but applies to the GC
+ /// heap rather than to linear memories. See that method for more details.
+ ///
+ /// ## Default
+ ///
+ /// If none of the `gc_heap_*` tunables are explicitly configured, they
+ /// default to the same values as their `memory_*` counterparts. Otherwise,
+ /// the default value for this option is `true`.
+ pub fn gc_heap_may_move(&mut self, enable: bool) -> &mut Self {
+ self.tunables.gc_heap_may_move = Some(enable);
+ self
+ }
+
/// Indicates whether a guard region is present before allocations of
/// linear memory.
///
@@ -2383,6 +2458,15 @@ impl Config {
target_lexicon::Triple::host()
}
+ /// Returns `true` if any of the `gc_heap_*` tunables have been explicitly
+ /// configured.
+ fn any_gc_heap_tunables_configured(&self) -> bool {
+ self.tunables.gc_heap_reservation.is_some()
+ || self.tunables.gc_heap_guard_size.is_some()
+ || self.tunables.gc_heap_reservation_for_growth.is_some()
+ || self.tunables.gc_heap_may_move.is_some()
+ }
+
pub(crate) fn validate(&self) -> Result<(Tunables, WasmFeatures)> {
let features = self.features();
@@ -2456,6 +2540,7 @@ impl Config {
if !cfg!(has_native_signals) {
tunables.signals_based_traps = cfg!(has_native_signals);
tunables.memory_guard_size = 0;
+ tunables.gc_heap_guard_size = 0;
}
// When virtual memory is not available use slightly different
@@ -2465,6 +2550,8 @@ impl Config {
tunables.memory_reservation = 0;
tunables.memory_reservation_for_growth = 1 << 20; // 1MB
tunables.memory_init_cow = false;
+ tunables.gc_heap_reservation = 0;
+ tunables.gc_heap_reservation_for_growth = 1 << 20; // 1MB
}
}
@@ -2479,6 +2566,16 @@ impl Config {
self.tunables.configure(&mut tunables);
+ // If no GC heap tunables are explicitly configured, copy the memory
+ // tunables' configured values so that GC heaps default to the same
+ // configuration as linear memories.
+ if !self.any_gc_heap_tunables_configured() {
+ tunables.gc_heap_reservation = tunables.memory_reservation;
+ tunables.gc_heap_guard_size = tunables.memory_guard_size;
+ tunables.gc_heap_reservation_for_growth = tunables.memory_reservation_for_growth;
+ tunables.gc_heap_may_move = tunables.memory_may_move;
+ }
+
// If we're going to compile with winch, we must use the winch calling convention.
#[cfg(any(feature = "cranelift", feature = "winch"))]
{
@@ -2534,6 +2631,50 @@ impl Config {
)
}
+ // If the pooling allocator is used and GC is enabled, check that
+ // memories and the GC heap are configured identically, since the
+ // pooling allocator can't support differently-configured heaps.
+ #[cfg(feature = "pooling-allocator")]
+ if matches!(
+ &self.allocation_strategy,
+ InstanceAllocationStrategy::Pooling(_)
+ ) && tunables.collector.is_some()
+ {
+ if tunables.memory_reservation != tunables.gc_heap_reservation {
+ bail!(
+ "when using the pooling allocator with GC, `memory_reservation` ({}) \
+ and `gc_heap_reservation` ({}) must be the same",
+ tunables.memory_reservation,
+ tunables.gc_heap_reservation,
+ );
+ }
+ if tunables.memory_guard_size != tunables.gc_heap_guard_size {
+ bail!(
+ "when using the pooling allocator with GC, `memory_guard_size` ({}) \
+ and `gc_heap_guard_size` ({}) must be the same",
+ tunables.memory_guard_size,
+ tunables.gc_heap_guard_size,
+ );
+ }
+ if tunables.memory_reservation_for_growth != tunables.gc_heap_reservation_for_growth {
+ bail!(
+ "when using the pooling allocator with GC, \
+ `memory_reservation_for_growth` ({}) and \
+ `gc_heap_reservation_for_growth` ({}) must be the same",
+ tunables.memory_reservation_for_growth,
+ tunables.gc_heap_reservation_for_growth,
+ );
+ }
+ if tunables.memory_may_move != tunables.gc_heap_may_move {
+ bail!(
+ "when using the pooling allocator with GC, `memory_may_move` ({}) \
+ and `gc_heap_may_move` ({}) must be the same",
+ tunables.memory_may_move,
+ tunables.gc_heap_may_move,
+ );
+ }
+ }
+
Ok((tunables, features))
}
diff --git a/crates/wasmtime/src/engine/serialization.rs b/crates/wasmtime/src/engine/serialization.rs
index 0028902864..6684ba9a1f 100644
--- a/crates/wasmtime/src/engine/serialization.rs
+++ b/crates/wasmtime/src/engine/serialization.rs
@@ -330,6 +330,13 @@ impl Metadata<'_> {
// This is a runtime GC debugging setting, doesn't affect compilation.
gc_zeal_alloc_counter: _,
+
+ gc_heap_reservation,
+ gc_heap_guard_size,
+ gc_heap_may_move,
+
+ // This doesn't affect compilation, it's just a runtime setting.
+ gc_heap_reservation_for_growth: _,
} = self.tunables;
Self::check_collector(collector, other.collector)?;
@@ -406,6 +413,17 @@ impl Metadata<'_> {
)?;
Self::check_bool(recording, other.recording, "RR recording support")?;
Self::check_intra_module_inlining(inlining_intra_module, other.inlining_intra_module)?;
+ Self::check_int(
+ gc_heap_reservation,
+ other.gc_heap_reservation,
+ "GC heap reservation",
+ )?;
+ Self::check_int(
+ gc_heap_guard_size,
+ other.gc_heap_guard_size,
+ "GC heap guard size",
+ )?;
+ Self::check_bool(gc_heap_may_move, other.gc_heap_may_move, "GC heap may move")?;
Ok(())
}
diff --git a/crates/wasmtime/src/runtime/memory.rs b/crates/wasmtime/src/runtime/memory.rs
index 7b027bddc6..81cb628896 100644
--- a/crates/wasmtime/src/runtime/memory.rs
+++ b/crates/wasmtime/src/runtime/memory.rs
@@ -1067,10 +1067,13 @@ mod tests {
let store = store.as_context();
let tunables = store.engine().tunables();
assert_eq!(tunables.memory_guard_size, 0);
- assert!(
- !mem.wasmtime_ty(store.0)
- .can_elide_bounds_check(tunables, 12)
- );
+ assert!(!mem.wasmtime_ty(store.0).can_elide_bounds_check(
+ &wasmtime_environ::MemoryTunables::new(
+ tunables,
+ wasmtime_environ::MemoryKind::LinearMemory,
+ ),
+ 12,
+ ));
}
#[test]
diff --git a/crates/wasmtime/src/runtime/trampoline/memory.rs b/crates/wasmtime/src/runtime/trampoline/memory.rs
index 3ef0c4be3e..704bc60d0f 100644
--- a/crates/wasmtime/src/runtime/trampoline/memory.rs
+++ b/crates/wasmtime/src/runtime/trampoline/memory.rs
@@ -12,8 +12,8 @@ use alloc::sync::Arc;
use core::future::Future;
use core::pin::Pin;
use wasmtime_environ::{
- DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, Module, StaticModuleIndex,
- Tunables, VMOffsets,
+ DefinedMemoryIndex, DefinedTableIndex, EntityIndex, HostPtr, MemoryTunables, Module,
+ StaticModuleIndex, VMOffsets,
};
#[cfg(feature = "component-model")]
@@ -105,18 +105,18 @@ impl RuntimeMemoryCreator for MemoryCreatorProxy {
fn new_memory(
&self,
ty: &wasmtime_environ::Memory,
- tunables: &Tunables,
+ memory_tunables: &MemoryTunables<'_>,
minimum: usize,
maximum: Option<usize>,
) -> Result<Box<dyn RuntimeLinearMemory>> {
- let reserved_size_in_bytes = Some(tunables.memory_reservation.try_into().unwrap());
+ let reserved_size_in_bytes = Some(memory_tunables.reservation().try_into().unwrap());
self.0
.new_memory(
MemoryType::from_wasmtime_memory(ty),
minimum,
maximum,
reserved_size_in_bytes,
- usize::try_from(tunables.memory_guard_size).unwrap(),
+ usize::try_from(memory_tunables.guard_size()).unwrap(),
)
.map(|mem| Box::new(LinearMemoryProxy { mem }) as Box<dyn RuntimeLinearMemory>)
.map_err(|e| format_err!(e))
diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rs
index 987e9c944a..15dbbaae5d 100644
--- a/crates/wasmtime/src/runtime/vm/cow.rs
+++ b/crates/wasmtime/src/runtime/vm/cow.rs
@@ -12,7 +12,7 @@ use alloc::sync::Arc;
use core::fmt;
use core::ops::Range;
use wasmtime_environ::prelude::TryPrimaryMap;
-use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, Module, Tunables};
+use wasmtime_environ::{DefinedMemoryIndex, MemoryInitialization, MemoryTunables, Module};
/// Backing images for memories in a module.
///
@@ -416,7 +416,7 @@ impl MemoryImageSlot {
initial_size_bytes: usize,
maybe_image: Option<&Arc<MemoryImage>>,
ty: &wasmtime_environ::Memory,
- tunables: &Tunables,
+ memory_tunables: &MemoryTunables<'_>,
) -> Result<()> {
assert!(!self.dirty);
assert!(
@@ -460,8 +460,8 @@ impl MemoryImageSlot {
// using dynamic memory without any guard pages.
let host_page_size_log2 = u8::try_from(host_page_size().ilog2()).unwrap();
if initial_size_bytes_page_aligned < self.accessible
- && (tunables.memory_guard_size > 0
- || ty.can_use_virtual_memory(tunables, host_page_size_log2))
+ && (memory_tunables.guard_size() > 0
+ || ty.can_use_virtual_memory(memory_tunables.tunables(), host_page_size_log2))
{
self.set_protection(initial_size_bytes_page_aligned..self.accessible, false)?;
self.accessible = initial_size_bytes_page_aligned;
@@ -715,7 +715,7 @@ mod test {
use crate::runtime::vm::sys::vm::decommit_pages;
use crate::runtime::vm::{HostAlignedByteCount, MmapVec, host_page_size};
use std::sync::Arc;
- use wasmtime_environ::{IndexType, Limits, Memory};
+ use wasmtime_environ::{IndexType, Limits, Memory, MemoryKind, Tunables};
fn create_memfd_with_data(offset: usize, data: &[u8]) -> Result<MemoryImage> {
// offset must be a multiple of the page size.
@@ -804,7 +804,14 @@ mod test {
MemoryImageSlot::create(mmap.zero_offset(), HostAlignedByteCount::ZERO, 4 << 20);
assert!(!memfd.is_dirty());
// instantiate with 64 KiB initial size
- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();
+ memfd
+ .instantiate(
+ 64 << 10,
+ None,
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
+ .unwrap();
assert!(memfd.is_dirty());
// We should be able to access this 64 KiB (try both ends) and
@@ -831,7 +838,14 @@ mod test {
})
.unwrap();
assert!(!memfd.is_dirty());
- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();
+ memfd
+ .instantiate(
+ 64 << 10,
+ None,
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
+ .unwrap();
let slice = unsafe { mmap.slice(0..65536) };
assert_eq!(0, slice[1024]);
}
@@ -853,7 +867,12 @@ mod test {
let image = Arc::new(create_memfd_with_data(page_size, &[1, 2, 3, 4]).unwrap());
// Instantiate with this image
memfd
- .instantiate(64 << 10, Some(&image), &ty, &tunables)
+ .instantiate(
+ 64 << 10,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
assert!(memfd.has_image());
@@ -871,7 +890,12 @@ mod test {
})
.unwrap();
memfd
- .instantiate(64 << 10, Some(&image), &ty, &tunables)
+ .instantiate(
+ 64 << 10,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
let slice = unsafe { mmap.slice(0..65536) };
assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
@@ -882,7 +906,14 @@ mod test {
decommit_pages(ptr, len).unwrap()
})
.unwrap();
- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();
+ memfd
+ .instantiate(
+ 64 << 10,
+ None,
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
+ .unwrap();
assert!(!memfd.has_image());
let slice = unsafe { mmap.slice(0..65536) };
assert_eq!(&[0, 0, 0, 0], &slice[page_size..][..4]);
@@ -894,7 +925,12 @@ mod test {
})
.unwrap();
memfd
- .instantiate(64 << 10, Some(&image), &ty, &tunables)
+ .instantiate(
+ 64 << 10,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
let slice = unsafe { mmap.slice(0..65536) };
assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
@@ -907,7 +943,12 @@ mod test {
})
.unwrap();
memfd
- .instantiate(128 << 10, Some(&image2), &ty, &tunables)
+ .instantiate(
+ 128 << 10,
+ Some(&image2),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
let slice = unsafe { mmap.slice(0..65536) };
assert_eq!(&[10, 11, 12, 13], &slice[page_size..][..4]);
@@ -920,7 +961,12 @@ mod test {
})
.unwrap();
memfd
- .instantiate(64 << 10, Some(&image), &ty, &tunables)
+ .instantiate(
+ 64 << 10,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
let slice = unsafe { mmap.slice(0..65536) };
assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
@@ -945,7 +991,12 @@ mod test {
for amt_to_memset in [0, page_size, page_size * 10, 1 << 20, 10 << 20] {
let amt_to_memset = HostAlignedByteCount::new(amt_to_memset).unwrap();
memfd
- .instantiate(64 << 10, Some(&image), &ty, &tunables)
+ .instantiate(
+ 64 << 10,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
assert!(memfd.has_image());
@@ -972,7 +1023,14 @@ mod test {
// Test without an image
for amt_to_memset in [0, page_size, page_size * 10, 1 << 20, 10 << 20] {
let amt_to_memset = HostAlignedByteCount::new(amt_to_memset).unwrap();
- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();
+ memfd
+ .instantiate(
+ 64 << 10,
+ None,
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
+ .unwrap();
unsafe {
with_slice_mut(&mmap, 0..64 << 10, |slice| {
@@ -1010,7 +1068,12 @@ mod test {
// Instantiate the image and test that memory remains accessible after
// it's cleared.
memfd
- .instantiate(initial, Some(&image), &ty, &tunables)
+ .instantiate(
+ initial,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
assert!(memfd.has_image());
@@ -1033,7 +1096,12 @@ mod test {
// Re-instantiate make sure it preserves memory. Grow a bit and set data
// beyond the initial size.
memfd
- .instantiate(initial, Some(&image), &ty, &tunables)
+ .instantiate(
+ initial,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
@@ -1059,7 +1127,12 @@ mod test {
// Instantiate again, and again memory beyond the initial size should
// still be accessible. Grow into it again and make sure it works.
memfd
- .instantiate(initial, Some(&image), &ty, &tunables)
+ .instantiate(
+ initial,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
assert_eq!(&[0, 0], &slice[initial..initial + 2]);
memfd.set_heap_limit(initial * 2).unwrap();
@@ -1079,7 +1152,14 @@ mod test {
.unwrap();
// Reset the image to none and double-check everything is back to zero
- memfd.instantiate(64 << 10, None, &ty, &tunables).unwrap();
+ memfd
+ .instantiate(
+ 64 << 10,
+ None,
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
+ .unwrap();
assert!(!memfd.has_image());
assert_eq!(&[0, 0, 0, 0], &slice[page_size..][..4]);
assert_eq!(&[0, 0], &slice[initial..initial + 2]);
@@ -1109,7 +1189,12 @@ mod test {
let image = Arc::new(create_memfd_with_data(3 * page_size, &data).unwrap());
memfd
- .instantiate(mmap_len, Some(&image), &ty, &tunables)
+ .instantiate(
+ mmap_len,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
let keep_resident = HostAlignedByteCount::new(mmap_len).unwrap();
@@ -1140,7 +1225,12 @@ mod test {
// Re-instantiate, but then wipe the image entirely by keeping
// nothing resident.
memfd
- .instantiate(mmap_len, Some(&image), &ty, &tunables)
+ .instantiate(
+ mmap_len,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
memfd
.clear_and_remain_ready(pagemap, HostAlignedByteCount::ZERO, |ptr, len| {
@@ -1150,7 +1240,12 @@ mod test {
// Next re-instantiate a final time to get used for the next test.
memfd
- .instantiate(mmap_len, Some(&image), &ty, &tunables)
+ .instantiate(
+ mmap_len,
+ Some(&image),
+ &ty,
+ &MemoryTunables::new(&tunables, MemoryKind::LinearMemory),
+ )
.unwrap();
};
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs
index b9350e1afe..fdb3ef5e30 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling.rs
@@ -383,7 +383,7 @@ impl PoolingInstanceAllocator {
tables: TablePool::new(config)?,
live_tables: AtomicUsize::new(0),
#[cfg(feature = "gc")]
- gc_heaps: GcHeapPool::new(config)?,
+ gc_heaps: GcHeapPool::new(config, tunables)?,
#[cfg(feature = "gc")]
live_gc_heaps: AtomicUsize::new(0),
#[cfg(feature = "async")]
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rs
index 685c8aba29..87091a6393 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/gc_heap_pool.rs
@@ -4,6 +4,7 @@ use crate::runtime::vm::{GcHeap, GcRuntime, PoolingInstanceAllocatorConfig, Resu
use crate::vm::{Memory, MemoryAllocationIndex};
use crate::{Engine, prelude::*};
use std::sync::Mutex;
+use wasmtime_environ::Tunables;
enum HeapSlot {
/// The is available for use, and we may or may not have lazily allocated
@@ -57,7 +58,26 @@ impl std::fmt::Debug for GcHeapPool {
impl GcHeapPool {
/// Create a new `GcHeapPool` with the given configuration.
- pub fn new(config: &PoolingInstanceAllocatorConfig) -> Result<Self> {
+ pub fn new(config: &PoolingInstanceAllocatorConfig, tunables: &Tunables) -> Result<Self> {
+ // When using the pooling allocator, GC heaps and linear memories share
+ // the same underlying memory pool, so they must be configured the same.
+ debug_assert_eq!(
+ tunables.memory_reservation, tunables.gc_heap_reservation,
+ "pooling allocator requires memory_reservation == gc_heap_reservation"
+ );
+ debug_assert_eq!(
+ tunables.memory_guard_size, tunables.gc_heap_guard_size,
+ "pooling allocator requires memory_guard_size == gc_heap_guard_size"
+ );
+ debug_assert_eq!(
+ tunables.memory_reservation_for_growth, tunables.gc_heap_reservation_for_growth,
+ "pooling allocator requires memory_reservation_for_growth == gc_heap_reservation_for_growth"
+ );
+ debug_assert_eq!(
+ tunables.memory_may_move, tunables.gc_heap_may_move,
+ "pooling allocator requires memory_may_move == gc_heap_may_move"
+ );
+
let index_allocator = SimpleIndexAllocator::new(config.limits.total_gc_heaps);
let max_gc_heaps = usize::try_from(config.limits.total_gc_heaps).unwrap();
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
index ed19a02e83..aaa26ef5f5 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/memory_pool.rs
@@ -67,7 +67,7 @@ use crate::{
use std::mem;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
-use wasmtime_environ::{DefinedMemoryIndex, Module, Tunables};
+use wasmtime_environ::{DefinedMemoryIndex, MemoryKind, MemoryTunables, Module, Tunables};
/// A set of allocator slots.
///
@@ -354,6 +354,7 @@ impl MemoryPool {
memory_index: Option<DefinedMemoryIndex>,
) -> Result<(MemoryAllocationIndex, Memory)> {
let tunables = request.store.engine().tunables();
+ let memory_tunables = MemoryTunables::new(tunables, MemoryKind::LinearMemory);
let stripe_index = if let Some(pkey) = request.store.get_pkey() {
pkey.as_stripe()
} else {
@@ -391,7 +392,7 @@ impl MemoryPool {
// should be returned as an error through `validate_memory_plans`
// but double-check here to be sure.
assert!(
- tunables.memory_reservation + tunables.memory_guard_size
+ memory_tunables.reservation() + memory_tunables.guard_size()
<= u64::try_from(self.layout.bytes_to_next_stripe_slot().byte_count()).unwrap()
);
@@ -421,11 +422,11 @@ impl MemoryPool {
// mmap that would leave an open space for someone
// else to come in and map something.
let initial_size = usize::try_from(initial_size).unwrap();
- slot.instantiate(initial_size, image, ty, tunables)?;
+ slot.instantiate(initial_size, image, ty, &memory_tunables)?;
let memory = Memory::new_static(
ty,
- tunables,
+ &memory_tunables,
MemoryBase::Mmap(base),
base_capacity.byte_count(),
slot,
diff --git a/crates/wasmtime/src/runtime/vm/memory.rs b/crates/wasmtime/src/runtime/vm/memory.rs
index eb525fdf73..2416be7e81 100644
--- a/crates/wasmtime/src/runtime/vm/memory.rs
+++ b/crates/wasmtime/src/runtime/vm/memory.rs
@@ -83,7 +83,7 @@ use crate::runtime::vm::{HostAlignedByteCount, MmapOffset};
use crate::runtime::vm::{MemoryImage, MemoryImageSlot, SendSyncPtr};
use alloc::sync::Arc;
use core::{ops::Range, ptr::NonNull};
-use wasmtime_environ::Tunables;
+use wasmtime_environ::{MemoryKind, MemoryTunables};
#[cfg(feature = "threads")]
use wasmtime_environ::Trap;
@@ -117,7 +117,7 @@ pub trait RuntimeMemoryCreator: Send + Sync {
fn new_memory(
&self,
ty: &wasmtime_environ::Memory,
- tunables: &Tunables,
+ memory_tunables: &MemoryTunables<'_>,
minimum: usize,
maximum: Option<usize>,
) -> Result<Box<dyn RuntimeLinearMemory>>;
@@ -131,25 +131,25 @@ impl RuntimeMemoryCreator for DefaultMemoryCreator {
fn new_memory(
&self,
ty: &wasmtime_environ::Memory,
- tunables: &Tunables,
+ memory_tunables: &MemoryTunables<'_>,
minimum: usize,
maximum: Option<usize>,
) -> Result<Box<dyn RuntimeLinearMemory>> {
#[cfg(has_virtual_memory)]
- if tunables.signals_based_traps
- || tunables.memory_guard_size > 0
- || tunables.memory_reservation > 0
- || tunables.memory_init_cow
+ if memory_tunables.tunables().signals_based_traps
+ || memory_tunables.guard_size() > 0
+ || memory_tunables.reservation() > 0
+ || memory_tunables.tunables().memory_init_cow
{
return Ok(
- try_new::<Box<_>>(MmapMemory::new(ty, tunables, minimum, maximum)?)?
+ try_new::<Box<_>>(MmapMemory::new(ty, memory_tunables, minimum, maximum)?)?
as Box<dyn RuntimeLinearMemory>,
);
}
let _ = maximum;
Ok(
- try_new::<Box<_>>(MallocMemory::new(ty, tunables, minimum)?)?
+ try_new::<Box<_>>(MallocMemory::new(ty, memory_tunables, minimum)?)?
as Box<dyn RuntimeLinearMemory>,
)
}
@@ -247,9 +247,10 @@ impl Memory {
) -> Result<Self> {
let (minimum, maximum) = Self::limit_new(ty, limiter).await?;
let tunables = engine.tunables();
- let allocation = creator.new_memory(ty, tunables, minimum, maximum)?;
+ let memory_tunables = MemoryTunables::new(tunables, MemoryKind::LinearMemory);
+ let allocation = creator.new_memory(ty, &memory_tunables, minimum, maximum)?;
- let memory = LocalMemory::new(ty, tunables, allocation, memory_image)?;
+ let memory = LocalMemory::new(ty, &memory_tunables, allocation, memory_image)?;
Ok(if ty.shared {
Memory::Shared(SharedMemory::wrap(engine, ty, memory)?)
} else {
@@ -261,7 +262,7 @@ impl Memory {
#[cfg(feature = "pooling-allocator")]
pub async fn new_static(
ty: &wasmtime_environ::Memory,
- tunables: &Tunables,
+ memory_tunables: &MemoryTunables<'_>,
base: MemoryBase,
base_capacity: usize,
memory_image: MemoryImageSlot,
@@ -275,7 +276,7 @@ impl Memory {
// `LocalMemory` structure created, notably we already have
// `memory_image` and regardless of configuration settings this memory
// can't move its base pointer since it's a fixed allocation.
- let mut memory = LocalMemory::new(ty, tunables, allocation, None)?;
+ let mut memory = LocalMemory::new(ty, memory_tunables, allocation, None)?;
assert!(memory.memory_image.is_none());
memory.memory_image = Some(memory_image);
memory.memory_may_move = false;
@@ -546,7 +547,7 @@ pub struct LocalMemory {
impl LocalMemory {
pub fn new(
ty: &wasmtime_environ::Memory,
- tunables: &Tunables,
+ memory_tunables: &MemoryTunables<'_>,
alloc: Box<dyn RuntimeLinearMemory>,
memory_image: Option<&Arc<MemoryImage>>,
) -> Result<LocalMemory> {
@@ -570,7 +571,7 @@ impl LocalMemory {
let mut slot =
MemoryImageSlot::create(mmap_base, byte_size, alloc.byte_capacity());
- slot.instantiate(alloc.byte_size(), Some(image), ty, tunables)?;
+ slot.instantiate(alloc.byte_size(), Some(image), ty, memory_tunables)?;
Some(slot)
} else {
None
@@ -583,10 +584,10 @@ impl LocalMemory {
Ok(LocalMemory {
ty: *ty,
alloc,
- memory_may_move: ty.memory_may_move(tunables),
+ memory_may_move: ty.memory_may_move(memory_tunables),
memory_image,
- memory_guard_size: tunables.memory_guard_size.try_into().unwrap(),
- memory_reservation: tunables.memory_reservation.try_into().unwrap(),
+ memory_guard_size: memory_tunables.guard_size().try_into().unwrap(),
+ memory_reservation: memory_tunables.reservation().try_into().unwrap(),
})
}
diff --git a/crates/wasmtime/src/runtime/vm/memory/malloc.rs b/crates/wasmtime/src/runtime/vm/memory/malloc.rs
index 7f543b174e..f07a5995d5 100644
--- a/crates/wasmtime/src/runtime/vm/memory/malloc.rs
+++ b/crates/wasmtime/src/runtime/vm/memory/malloc.rs
@@ -9,7 +9,7 @@ use crate::runtime::vm::SendSyncPtr;
use crate::runtime::vm::memory::{MemoryBase, RuntimeLinearMemory};
use core::mem;
use core::ptr::NonNull;
-use wasmtime_environ::Tunables;
+use wasmtime_environ::MemoryTunables;
#[repr(C, align(16))]
#[derive(Copy, Clone)]
@@ -25,21 +25,21 @@ pub struct MallocMemory {
impl MallocMemory {
pub fn new(
_ty: &wasmtime_environ::Memory,
- tunables: &Tunables,
+ memory_tunables: &MemoryTunables<'_>,
minimum: usize,
) -> Result<Self> {
- if tunables.memory_guard_size > 0 {
+ if memory_tunables.guard_size() > 0 {
bail!("malloc memory is only compatible if guard pages aren't used");
}
- if tunables.memory_reservation > 0 {
+ if memory_tunables.reservation() > 0 {
bail!("malloc memory is only compatible with no ahead-of-time memory reservation");
}
- if tunables.memory_init_cow {
+ if memory_tunables.tunables().memory_init_cow {
bail!("malloc memory cannot be used with CoW images");
}
let initial_allocation_byte_size = minimum
- .checked_add(tunables.memory_reservation_for_growth.try_into()?)
+ .checked_add(memory_tunables.reservation_for_growth().try_into()?)
.context("memory allocation size too large")?;
let initial_allocation_len = byte_size_to_element_len(initial_allocation_byte_size);
@@ -50,7 +50,7 @@ impl MallocMemory {
format!(
"failed to allocate {initial_allocation_byte_size:#x} \
bytes ({minimum:#x} minimum + {:#x} memory_reservation_for_growth)",
- tunables.memory_reservation_for_growth,
+ memory_tunables.reservation_for_growth(),
)
})?;
@@ -145,6 +145,7 @@ fn grow_storage_to(storage: &mut Vec<Align16>, new_len: usize) {
#[cfg(test)]
mod tests {
use super::*;
+ use wasmtime_environ::{MemoryKind, Tunables};
// This is currently required by the constructor but otherwise ignored in
// the creation of a `MallocMemory`, so just have a single one used in
@@ -168,7 +169,9 @@ mod tests {
#[test]
fn simple() {
- let mut memory = MallocMemory::new(&TY, &tunables(), 10).unwrap();
+ let t = tunables();
+ let mt = MemoryTunables::new(&t, MemoryKind::LinearMemory);
+ let mut memory = MallocMemory::new(&TY, &mt, 10).unwrap();
assert_eq!(memory.storage.len(), 1);
assert_valid(&memory);
@@ -191,15 +194,16 @@ mod tests {
#[test]
fn reservation_not_initialized() {
- let tunables = Tunables {
+ let t = Tunables {
memory_reservation_for_growth: 1 << 20,
..tunables()
};
- let mut memory = MallocMemory::new(&TY, &tunables, 10).unwrap();
+ let mt = MemoryTunables::new(&t, MemoryKind::LinearMemory);
+ let mut memory = MallocMemory::new(&TY, &mt, 10).unwrap();
assert_eq!(memory.storage.len(), 1);
assert_eq!(
memory.storage.capacity(),
- (tunables.memory_reservation_for_growth / 16) as usize + 1,
+ (t.memory_reservation_for_growth / 16) as usize + 1,
);
assert_valid(&memory);
@@ -207,7 +211,7 @@ mod tests {
assert_eq!(memory.storage.len(), 7);
assert_eq!(
memory.storage.capacity(),
- (tunables.memory_reservation_for_growth / 16) as usize + 1,
+ (t.memory_reservation_for_growth / 16) as usize + 1,
);
assert_valid(&memory);
}
diff --git a/crates/wasmtime/src/runtime/vm/memory/mmap.rs b/crates/wasmtime/src/runtime/vm/memory/mmap.rs
index e632e62310..8d58e9e1d1 100644
--- a/crates/wasmtime/src/runtime/vm/memory/mmap.rs
+++ b/crates/wasmtime/src/runtime/vm/memory/mmap.rs
@@ -5,7 +5,7 @@ use crate::prelude::*;
use crate::runtime::vm::memory::RuntimeLinearMemory;
use crate::runtime::vm::{HostAlignedByteCount, Mmap, mmap::AlignedLength};
use alloc::sync::Arc;
-use wasmtime_environ::Tunables;
+use wasmtime_environ::MemoryTunables;
use super::MemoryBase;
@@ -50,7 +50,7 @@ impl MmapMemory {
/// number of wasm pages.
pub fn new(
ty: &wasmtime_environ::Memory,
- tunables: &Tunables,
+ memory_tunables: &MemoryTunables<'_>,
minimum: usize,
maximum: Option<usize>,
) -> Result<Self> {
@@ -60,9 +60,9 @@ impl MmapMemory {
//
// Also be sure to round up to the host page size for this value.
let offset_guard_bytes =
- HostAlignedByteCount::new_rounded_up_u64(tunables.memory_guard_size)
+ HostAlignedByteCount::new_rounded_up_u64(memory_tunables.guard_size())
.context("tunable.memory_guard_size overflows")?;
- let pre_guard_bytes = if tunables.guard_before_linear_memory {
+ let pre_guard_bytes = if memory_tunables.tunables().guard_before_linear_memory {
offset_guard_bytes
} else {
HostAlignedByteCount::ZERO
@@ -78,8 +78,8 @@ impl MmapMemory {
// to reserve any extra for growth.
//
// If the minimum size doesn't fit within this linear memory.
- let mut alloc_bytes = tunables.memory_reservation;
- let mut extra_to_reserve_on_growth = tunables.memory_reservation_for_growth;
+ let mut alloc_bytes = memory_tunables.reservation();
+ let mut extra_to_reserve_on_growth = memory_tunables.reservation_for_growth();
let minimum_u64 = u64::try_from(minimum).unwrap();
if minimum_u64 <= alloc_bytes {
if let Ok(max) = ty.maximum_byte_size() {
diff --git a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs
index 51c9248beb..679984c250 100644
--- a/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs
+++ b/crates/wasmtime/src/runtime/vm/memory/shared_memory.rs
@@ -33,13 +33,21 @@ impl SharedMemory {
/// Construct a new [`SharedMemory`].
pub fn new(engine: &Engine, ty: &wasmtime_environ::Memory) -> Result<Self> {
let tunables = engine.tunables();
+ let memory_tunables = wasmtime_environ::MemoryTunables::new(
+ tunables,
+ wasmtime_environ::MemoryKind::LinearMemory,
+ );
// Note that without a limiter being passed to `limit_new` this
// `assert_ready` should never panic.
let (minimum_bytes, maximum_bytes) = vm::assert_ready(Memory::limit_new(ty, None))?;
- let mmap_memory = MmapMemory::new(ty, tunables, minimum_bytes, maximum_bytes)?;
+ let mmap_memory = MmapMemory::new(ty, &memory_tunables, minimum_bytes, maximum_bytes)?;
let boxed: Box<dyn crate::runtime::vm::RuntimeLinearMemory> =
try_new::<Box<_>>(mmap_memory)?;
- Self::wrap(engine, ty, LocalMemory::new(ty, tunables, boxed, None)?)
+ Self::wrap(
+ engine,
+ ty,
+ LocalMemory::new(ty, &memory_tunables, boxed, None)?,
+ )
}
/// Wrap an existing [Memory] with the locking provided by a [SharedMemory].
diff --git a/tests/all/gc.rs b/tests/all/gc.rs
index 8ca6311706..7e27a6915b 100644
--- a/tests/all/gc.rs
+++ b/tests/all/gc.rs
@@ -1355,6 +1355,10 @@ fn gc_heap_oom() -> Result<()> {
config.memory_reservation_for_growth(0);
config.memory_guard_size(0);
config.memory_may_move(false);
+ config.gc_heap_reservation(heap_size);
+ config.gc_heap_reservation_for_growth(0);
+ config.gc_heap_guard_size(0);
+ config.gc_heap_may_move(false);
if pooling {
let mut pooling = crate::small_pool_config();
@@ -1753,3 +1757,141 @@ fn gc_heap_does_not_grow_unboundedly() -> Result<()> {
Ok(())
}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn pooling_gc_different_configs_rejected() -> Result<()> {
+ let _ = env_logger::try_init();
+
+ let mut config = Config::new();
+ config.wasm_function_references(true);
+ config.wasm_gc(true);
+ config.collector(Collector::Null);
+ config.allocation_strategy(crate::small_pool_config());
+
+ // Set GC heap reservation to a different value than memory reservation.
+ config.gc_heap_reservation(0);
+
+ assert!(Engine::new(&config).is_err());
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn memory_guard_pages_but_no_gc_heap_guard_pages() -> Result<()> {
+ if std::mem::size_of::<usize>() < std::mem::size_of::<u64>()
+ || std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok()
+ {
+ return Ok(());
+ }
+
+ let _ = env_logger::try_init();
+
+ let mut config = Config::new();
+ config.wasm_function_references(true);
+ config.wasm_gc(true);
+ config.collector(Collector::Null);
+
+ // Memories get large guard pages (bounds checks elided for i32), but the
+ // GC heap does not (bounds checks required).
+ config.memory_reservation(1 << 32);
+ config.memory_guard_size(1 << 32);
+ config.gc_heap_reservation(0);
+ config.gc_heap_guard_size(0);
+ config.gc_heap_reservation_for_growth(1 << 20);
+
+ let engine = Engine::new(&config)?;
+
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (type $ty (struct (field (mut f32))))
+ (func (export "roundtrip") (param (ref null $ty)) (result f32)
+ (struct.get $ty 0 (local.get 0))
+ )
+ (func (export "alloc") (result (ref $ty))
+ (struct.new $ty (f32.const 3.14))
+ )
+ )
+ "#,
+ )?;
+
+ let mut store = Store::new(&engine, ());
+ let instance = Instance::new(&mut store, &module, &[])?;
+
+ let alloc = instance.get_typed_func::<(), Rooted<StructRef>>(&mut store, "alloc")?;
+ let roundtrip =
+ instance.get_typed_func::<(Option<Rooted<StructRef>>,), f32>(&mut store, "roundtrip")?;
+
+ let s = alloc.call(&mut store, ())?;
+ let val = roundtrip.call(&mut store, (Some(s),))?;
+ assert_eq!(val, 3.14_f32);
+
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn gc_heap_guard_pages_but_no_memory_guard_pages() -> Result<()> {
+ if std::mem::size_of::<usize>() < std::mem::size_of::<u64>()
+ || std::env::var("WASMTIME_TEST_NO_HOG_MEMORY").is_ok()
+ {
+ return Ok(());
+ }
+
+ let _ = env_logger::try_init();
+
+ let mut config = Config::new();
+ config.wasm_function_references(true);
+ config.wasm_gc(true);
+ config.collector(Collector::Null);
+
+ // GC heap gets large guard pages (bounds checks elided), but memories do
+ // not (bounds checks required for memories).
+ config.gc_heap_reservation(1 << 32);
+ config.gc_heap_guard_size(1 << 32);
+ config.memory_reservation(0);
+ config.memory_guard_size(0);
+ config.memory_reservation_for_growth(1 << 20);
+
+ let engine = Engine::new(&config)?;
+
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (type $ty (struct (field (mut f32))))
+ (memory (export "memory") 1)
+ (func (export "roundtrip") (param (ref null $ty)) (result f32)
+ (struct.get $ty 0 (local.get 0))
+ )
+ (func (export "alloc") (result (ref $ty))
+ (struct.new $ty (f32.const 2.72))
+ )
+ (func (export "load") (param i32) (result i32)
+ (i32.load (local.get 0))
+ )
+ )
+ "#,
+ )?;
+
+ let mut store = Store::new(&engine, ());
+ let instance = Instance::new(&mut store, &module, &[])?;
+
+ let alloc = instance.get_typed_func::<(), Rooted<StructRef>>(&mut store, "alloc")?;
+ let roundtrip =
+ instance.get_typed_func::<(Option<Rooted<StructRef>>,), f32>(&mut store, "roundtrip")?;
+ let load = instance.get_typed_func::<(i32,), i32>(&mut store, "load")?;
+
+ let s = alloc.call(&mut store, ())?;
+ let val = roundtrip.call(&mut store, (Some(s),))?;
+ assert_eq!(val, 2.72_f32);
+
+ // Also exercise the memory access to make sure it works with bounds checks.
+ let result = load.call(&mut store, (0,))?;
+ assert_eq!(result, 0);
+
+ Ok(())
+}
diff --git a/tests/disas/gc/null/struct-get-guard-pages.wat b/tests/disas/gc/null/struct-get-guard-pages.wat
new file mode 100644
index 0000000000..57d473dc84
--- /dev/null
+++ b/tests/disas/gc/null/struct-get-guard-pages.wat
@@ -0,0 +1,126 @@
+;;! target = "x86_64"
+;;! flags = "-W function-references,gc -C collector=null -O gc-heap-reservation=0x100000000 -O gc-heap-guard-size=0x100000000"
+;;! test = "optimize"
+
+(module
+ (type $ty (struct (field (mut f32))
+ (field (mut i8))
+ (field (mut anyref))))
+
+ (func (param (ref null $ty)) (result f32)
+ (struct.get $ty 0 (local.get 0))
+ )
+
+ (func (param (ref null $ty)) (result i32)
+ (struct.get_s $ty 1 (local.get 0))
+ )
+
+ (func (param (ref null $ty)) (result i32)
+ (struct.get_u $ty 1 (local.get 0))
+ )
+
+ (func (param (ref null $ty)) (result anyref)
+ (struct.get $ty 2 (local.get 0))
+ )
+)
+;; function u0:0(i64 vmctx, i64, i32) -> f32 tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
+;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
+;; gv6 = load.i64 notrap aligned gv4+40
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @0033 trapz v2, user15
+;; @0033 v10 = load.i64 notrap aligned readonly can_move v0+8
+;; @0033 v5 = load.i64 notrap aligned readonly can_move v10+32
+;; @0033 v4 = uextend.i64 v2
+;; @0033 v6 = iadd v5, v4
+;; @0033 v7 = iconst.i64 8
+;; @0033 v8 = iadd v6, v7 ; v7 = 8
+;; @0033 v9 = load.f32 notrap aligned little v8
+;; @0037 jump block1
+;;
+;; block1:
+;; @0037 return v9
+;; }
+;;
+;; function u0:1(i64 vmctx, i64, i32) -> i32 tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
+;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
+;; gv6 = load.i64 notrap aligned gv4+40
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @003c trapz v2, user15
+;; @003c v11 = load.i64 notrap aligned readonly can_move v0+8
+;; @003c v5 = load.i64 notrap aligned readonly can_move v11+32
+;; @003c v4 = uextend.i64 v2
+;; @003c v6 = iadd v5, v4
+;; @003c v7 = iconst.i64 12
+;; @003c v8 = iadd v6, v7 ; v7 = 12
+;; @003c v9 = load.i8 notrap aligned little v8
+;; @0040 jump block1
+;;
+;; block1:
+;; @003c v10 = sextend.i32 v9
+;; @0040 return v10
+;; }
+;;
+;; function u0:2(i64 vmctx, i64, i32) -> i32 tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
+;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
+;; gv6 = load.i64 notrap aligned gv4+40
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @0045 trapz v2, user15
+;; @0045 v11 = load.i64 notrap aligned readonly can_move v0+8
+;; @0045 v5 = load.i64 notrap aligned readonly can_move v11+32
+;; @0045 v4 = uextend.i64 v2
+;; @0045 v6 = iadd v5, v4
+;; @0045 v7 = iconst.i64 12
+;; @0045 v8 = iadd v6, v7 ; v7 = 12
+;; @0045 v9 = load.i8 notrap aligned little v8
+;; @0049 jump block1
+;;
+;; block1:
+;; @0045 v10 = uextend.i32 v9
+;; @0049 return v10
+;; }
+;;
+;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
+;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
+;; gv6 = load.i64 notrap aligned gv4+40
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @004e trapz v2, user15
+;; @004e v10 = load.i64 notrap aligned readonly can_move v0+8
+;; @004e v5 = load.i64 notrap aligned readonly can_move v10+32
+;; @004e v4 = uextend.i64 v2
+;; @004e v6 = iadd v5, v4
+;; @004e v7 = iconst.i64 16
+;; @004e v8 = iadd v6, v7 ; v7 = 16
+;; @004e v9 = load.i32 notrap aligned little v8
+;; @0052 jump block1
+;;
+;; block1:
+;; @0052 return v9
+;; }
diff --git a/tests/disas/gc/null/struct-get-no-guard-pages.wat b/tests/disas/gc/null/struct-get-no-guard-pages.wat
new file mode 100644
index 0000000000..bb64d884c8
--- /dev/null
+++ b/tests/disas/gc/null/struct-get-no-guard-pages.wat
@@ -0,0 +1,150 @@
+;;! target = "x86_64"
+;;! flags = "-W function-references,gc -C collector=null -O gc-heap-reservation=0 -O gc-heap-guard-size=0"
+;;! test = "optimize"
+
+(module
+ (type $ty (struct (field (mut f32))
+ (field (mut i8))
+ (field (mut anyref))))
+
+ (func (param (ref null $ty)) (result f32)
+ (struct.get $ty 0 (local.get 0))
+ )
+
+ (func (param (ref null $ty)) (result i32)
+ (struct.get_s $ty 1 (local.get 0))
+ )
+
+ (func (param (ref null $ty)) (result i32)
+ (struct.get_u $ty 1 (local.get 0))
+ )
+
+ (func (param (ref null $ty)) (result anyref)
+ (struct.get $ty 2 (local.get 0))
+ )
+)
+;; function u0:0(i64 vmctx, i64, i32) -> f32 tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
+;; gv5 = load.i64 notrap aligned gv4+32
+;; gv6 = load.i64 notrap aligned gv4+40
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @0033 trapz v2, user15
+;; @0033 v4 = uextend.i64 v2
+;; @0033 v5 = iconst.i64 24
+;; @0033 v6 = uadd_overflow_trap v4, v5, user1 ; v5 = 24
+;; @0033 v18 = load.i64 notrap aligned readonly can_move v0+8
+;; @0033 v7 = load.i64 notrap aligned v18+40
+;; @0033 v9 = load.i64 notrap aligned v18+32
+;; @0033 v8 = icmp ugt v6, v7
+;; @0033 v11 = iconst.i64 0
+;; @0033 v10 = iadd v9, v4
+;; @0033 v12 = select_spectre_guard v8, v11, v10 ; v11 = 0
+;; @0033 v13 = iconst.i64 8
+;; @0033 v14 = iadd v12, v13 ; v13 = 8
+;; @0033 v15 = load.f32 notrap aligned little v14
+;; @0037 jump block1
+;;
+;; block1:
+;; @0037 return v15
+;; }
+;;
+;; function u0:1(i64 vmctx, i64, i32) -> i32 tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
+;; gv5 = load.i64 notrap aligned gv4+32
+;; gv6 = load.i64 notrap aligned gv4+40
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @003c trapz v2, user15
+;; @003c v4 = uextend.i64 v2
+;; @003c v5 = iconst.i64 24
+;; @003c v6 = uadd_overflow_trap v4, v5, user1 ; v5 = 24
+;; @003c v19 = load.i64 notrap aligned readonly can_move v0+8
+;; @003c v7 = load.i64 notrap aligned v19+40
+;; @003c v9 = load.i64 notrap aligned v19+32
+;; @003c v8 = icmp ugt v6, v7
+;; @003c v11 = iconst.i64 0
+;; @003c v10 = iadd v9, v4
+;; @003c v12 = select_spectre_guard v8, v11, v10 ; v11 = 0
+;; @003c v13 = iconst.i64 12
+;; @003c v14 = iadd v12, v13 ; v13 = 12
+;; @003c v15 = load.i8 notrap aligned little v14
+;; @0040 jump block1
+;;
+;; block1:
+;; @003c v16 = sextend.i32 v15
+;; @0040 return v16
+;; }
+;;
+;; function u0:2(i64 vmctx, i64, i32) -> i32 tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
+;; gv5 = load.i64 notrap aligned gv4+32
+;; gv6 = load.i64 notrap aligned gv4+40
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @0045 trapz v2, user15
+;; @0045 v4 = uextend.i64 v2
+;; @0045 v5 = iconst.i64 24
+;; @0045 v6 = uadd_overflow_trap v4, v5, user1 ; v5 = 24
+;; @0045 v19 = load.i64 notrap aligned readonly can_move v0+8
+;; @0045 v7 = load.i64 notrap aligned v19+40
+;; @0045 v9 = load.i64 notrap aligned v19+32
+;; @0045 v8 = icmp ugt v6, v7
+;; @0045 v11 = iconst.i64 0
+;; @0045 v10 = iadd v9, v4
+;; @0045 v12 = select_spectre_guard v8, v11, v10 ; v11 = 0
+;; @0045 v13 = iconst.i64 12
+;; @0045 v14 = iadd v12, v13 ; v13 = 12
+;; @0045 v15 = load.i8 notrap aligned little v14
+;; @0049 jump block1
+;;
+;; block1:
+;; @0045 v16 = uextend.i32 v15
+;; @0049 return v16
+;; }
+;;
+;; function u0:3(i64 vmctx, i64, i32) -> i32 tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned readonly can_move gv3+8
+;; gv5 = load.i64 notrap aligned gv4+32
+;; gv6 = load.i64 notrap aligned gv4+40
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @004e trapz v2, user15
+;; @004e v4 = uextend.i64 v2
+;; @004e v5 = iconst.i64 24
+;; @004e v6 = uadd_overflow_trap v4, v5, user1 ; v5 = 24
+;; @004e v18 = load.i64 notrap aligned readonly can_move v0+8
+;; @004e v7 = load.i64 notrap aligned v18+40
+;; @004e v9 = load.i64 notrap aligned v18+32
+;; @004e v8 = icmp ugt v6, v7
+;; @004e v11 = iconst.i64 0
+;; @004e v10 = iadd v9, v4
+;; @004e v12 = select_spectre_guard v8, v11, v10 ; v11 = 0
+;; @004e v13 = iconst.i64 16
+;; @004e v14 = iadd v12, v13 ; v13 = 16
+;; @004e v15 = load.i32 notrap aligned little v14
+;; @0052 jump block1
+;;
+;; block1:
+;; @0052 return v15
+;; }
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index 732eea6b43..69a2baf76b 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -24,8 +24,8 @@ use wasmparser::{
};
use wasmtime_cranelift::{TRAP_BAD_SIGNATURE, TRAP_HEAP_MISALIGNED, TRAP_TABLE_OUT_OF_BOUNDS};
use wasmtime_environ::{
- FUNCREF_MASK, GlobalIndex, MemoryIndex, PtrSize, TableIndex, Tunables, TypeIndex, WasmHeapType,
- WasmValType,
+ FUNCREF_MASK, GlobalIndex, MemoryIndex, MemoryKind, MemoryTunables, PtrSize, TableIndex,
+ Tunables, TypeIndex, WasmHeapType, WasmValType,
};
mod context;
@@ -690,9 +690,10 @@ where
)?;
let offset_with_access_size = add_offset_and_access_size(offset, access_size);
+ let memory_tunables = MemoryTunables::new(self.tunables, MemoryKind::LinearMemory);
let can_elide_bounds_check = heap
.memory
- .can_elide_bounds_check(self.tunables, self.env.page_size_log2);
+ .can_elide_bounds_check(&memory_tunables, self.env.page_size_log2);
let addr = if offset_with_access_size > heap.memory.maximum_byte_size().unwrap_or(u64::MAX)
|| (!self.tunables.memory_may_move