A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Always use mmap in cranelift-jit (#13180)
bjorn30c3a69f18df3
diff --git a/cranelift/filetests/Cargo.toml b/cranelift/filetests/Cargo.tomlindex 058e8ffccc..92118fd4de 100644--- a/cranelift/filetests/Cargo.toml+++ b/cranelift/filetests/Cargo.toml@@ -19,7 +19,7 @@ cranelift-frontend = { workspace = true }cranelift-interpreter = { workspace = true }cranelift-native = { workspace = true }cranelift-reader = { workspace = true }-cranelift-jit = { workspace = true, features = ["selinux-fix", "wasmtime-unwinder"] }+cranelift-jit = { workspace = true, features = ["wasmtime-unwinder"] }cranelift-module = { workspace = true }cranelift-control = { workspace = true }wasmtime-unwinder = { workspace = true, features = ["cranelift"] }diff --git a/cranelift/jit/Cargo.toml b/cranelift/jit/Cargo.tomlindex c7a4d7f53f..b75c7c0134 100644--- a/cranelift/jit/Cargo.toml+++ b/cranelift/jit/Cargo.toml@@ -24,7 +24,7 @@ anyhow = { workspace = true }region = "3.0.2"libc = { workspace = true }target-lexicon = { workspace = true }-memmap2 = { version = "0.2.1", optional = true }+memmap2 = "0.2.1"log = { workspace = true }wasmtime-jit-icache-coherence = { workspace = true }@@ -37,7 +37,6 @@ features = [][features]-selinux-fix = ['memmap2']default = []wasmtime-unwinder = ["dep:wasmtime-unwinder"]diff --git a/cranelift/jit/src/memory/system.rs b/cranelift/jit/src/memory/system.rsindex bd845aef4a..e6c5a4d2c3 100644--- a/cranelift/jit/src/memory/system.rs+++ b/cranelift/jit/src/memory/system.rs@@ -1,21 +1,15 @@-use cranelift_module::{ModuleError, ModuleResult};--#[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]-use memmap2::MmapMut;--#[cfg(not(any(feature = "selinux-fix", windows)))]-use std::alloc;use std::io;use std::mem;use std::ptr;+use cranelift_module::{ModuleError, ModuleResult};+use memmap2::MmapMut;+use super::{BranchProtection, JITMemoryKind, JITMemoryProvider};/// A simple struct consisting of a pointer and length.struct PtrLen {- #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]map: Option<MmapMut>,-ptr: *mut u8,len: usize,}@@ -24,9 +18,7 @@ impl PtrLen {/// Create a new empty `PtrLen`.fn new() -> Self {Self {- #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]map: None,-ptr: ptr::null_mut(),len: 0,}@@ -34,7 +26,6 @@ impl PtrLen {/// Create a new `PtrLen` pointing to at least `size` bytes of memory,/// suitably sized and aligned for memory protection.- #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]fn with_size(size: usize) -> io::Result<Self> {let alloc_size = region::page::ceil(size as *const ()) as usize;MmapMut::map_anon(alloc_size).map(|mut mmap| {@@ -47,70 +38,8 @@ impl PtrLen {}})}-- #[cfg(all(not(target_os = "windows"), not(feature = "selinux-fix")))]- fn with_size(size: usize) -> io::Result<Self> {- assert_ne!(size, 0);- let page_size = region::page::size();- let alloc_size = region::page::ceil(size as *const ()) as usize;- let layout = alloc::Layout::from_size_align(alloc_size, page_size).unwrap();- // Safety: We assert that the size is non-zero above.- let ptr = unsafe { alloc::alloc(layout) };-- if !ptr.is_null() {- Ok(Self {- ptr,- len: alloc_size,- })- } else {- Err(io::Error::from(io::ErrorKind::OutOfMemory))- }- }-- #[cfg(target_os = "windows")]- fn with_size(size: usize) -> io::Result<Self> {- use windows_sys::Win32::System::Memory::{- MEM_COMMIT, MEM_RESERVE, PAGE_READWRITE, VirtualAlloc,- };-- // VirtualAlloc always rounds up to the next multiple of the page size- let ptr = unsafe {- VirtualAlloc(- ptr::null_mut(),- size,- MEM_COMMIT | MEM_RESERVE,- PAGE_READWRITE,- )- };- if !ptr.is_null() {- Ok(Self {- ptr: ptr as *mut u8,- len: region::page::ceil(size as *const ()) as usize,- })- } else {- Err(io::Error::last_os_error())- }- }}-// `MMapMut` from `cfg(feature = "selinux-fix")` already deallocates properly.-#[cfg(all(not(target_os = "windows"), not(feature = "selinux-fix")))]-impl Drop for PtrLen {- fn drop(&mut self) {- if !self.ptr.is_null() {- let page_size = region::page::size();- let layout = alloc::Layout::from_size_align(self.len, page_size).unwrap();- unsafe {- region::protect(self.ptr, self.len, region::Protection::READ_WRITE)- .expect("unable to unprotect memory");- alloc::dealloc(self.ptr, layout)- }- }- }-}--// TODO: add a `Drop` impl for `cfg(target_os = "windows")`-/// JIT memory manager. This manages pages of suitably aligned and/// accessible memory. Memory will be leaked by default to have/// function pointers remain valid for the remainder of the@@ -201,13 +130,9 @@ impl Memory {/// Iterates non protected memory allocations that are of not zero bytes in size.fn non_protected_allocations_iter(&self) -> impl Iterator<Item = &PtrLen> {- let iter = self.allocations[self.already_protected..].iter();-- #[cfg(all(not(target_os = "windows"), feature = "selinux-fix"))]- return iter.filter(|&PtrLen { map, len, .. }| *len != 0 && map.is_some());-- #[cfg(any(target_os = "windows", not(feature = "selinux-fix")))]- return iter.filter(|&PtrLen { len, .. }| *len != 0);+ self.allocations[self.already_protected..]+ .iter()+ .filter(|&PtrLen { map, len, .. }| *len != 0 && map.is_some())}/// Frees all allocated memory regions that would be leaked otherwise.diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rsindex 7e48f0d8eb..77eb94a845 100644--- a/crates/wasmtime/src/runtime/func.rs+++ b/crates/wasmtime/src/runtime/func.rs@@ -1064,6 +1064,10 @@ impl Func {self.vm_func_ref(store).as_ptr().cast()}+ pub(crate) fn to_val_raw(&self, store: &mut StoreOpaque) -> ValRaw {+ ValRaw::funcref(self.to_raw_(store))+ }+/// Invokes this function with the `params` given, returning the results/// asynchronously.///diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rsindex 77e869dc8a..4730125d33 100644--- a/crates/wasmtime/src/runtime/vm/instance.rs+++ b/crates/wasmtime/src/runtime/vm/instance.rs@@ -13,7 +13,7 @@ use crate::runtime::vm::vmcontext::{VMTableDefinition, VMTableImport, VMTagDefinition, VMTagImport,};use crate::runtime::vm::{- GcStore, HostResult, Imports, ModuleRuntimeInfo, SendSyncPtr, VMGcRef, VMGlobalKind, VMStore,+ GcStore, HostResult, Imports, ModuleRuntimeInfo, SendSyncPtr, VMGlobalKind, VMStore,VMStoreRawPtr, VmPtr, VmSafe, WasmFault, catch_unwind_and_record_trap,};use crate::store::{@@ -133,7 +133,7 @@ pub struct Instance {//// TODO(#12621): This should be a `TrySecondaryMap<PassiveElemIndex, _>`// but that type is currently footgun-y / isn't actually OOM-safe yet.- passive_elements: TryVec<PassiveElementSegment>,+ passive_elements: TryVec<Option<(NeedsGcRooting, TryVec<ValRaw>)>>,/// Stores the dropped passive data segments in this instantiation by index./// If the index is present in the set, the segment has been dropped.@@ -225,8 +225,8 @@ impl Instance {gc_roots: &mut crate::vm::GcRootsList,) {for segment in self.passive_elements_mut().iter_mut() {- if segment.needs_gc_rooting() == NeedsGcRooting::Yes {- for e in segment.elements() {+ if let Some((wasmtime_environ::NeedsGcRooting::Yes, elems)) = segment {+ for e in elems {let Some(root) = e.as_vmgc_ref_ptr() else {continue;};@@ -912,12 +912,16 @@ impl Instance {return &[];};- self.passive_elements[passive.index()].elements()+ let Some((_, seg)) = &self.passive_elements[passive.index()] else {+ return &[];+ };++ &**seg}pub(crate) fn passive_elements_mut(self: Pin<&mut Self>,- ) -> Pin<&mut TryVec<PassiveElementSegment>> {+ ) -> Pin<&mut TryVec<Option<(NeedsGcRooting, TryVec<ValRaw>)>>> {// SAFETY: Not moving data out of `self`.Pin::new(&mut unsafe { self.get_unchecked_mut() }.passive_elements)}@@ -992,7 +996,6 @@ impl Instance {/// Drop an element.pub(crate) fn elem_drop(self: Pin<&mut Self>,- gc_store: Option<&mut GcStore>,elem_index: ElemIndex,) -> Result<(), OutOfMemory> {// https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-elem-drop@@ -1007,7 +1010,7 @@ impl Instance {return Ok(());};- self.passive_elements_mut()[passive_index.index()].clear(gc_store);+ self.passive_elements_mut()[passive_index.index()] = None;Ok(())}@@ -1919,77 +1922,3 @@ impl<T: InstanceLayout> Drop for OwnedInstance<T> {}}}--#[derive(Debug)]-pub(crate) struct PassiveElementSegment {- needs_gc_rooting: NeedsGcRooting,- elements: TryVec<ValRaw>,-}--impl PassiveElementSegment {- /// Create a new passive element segment with the given capacity.- pub(crate) fn new(- needs_gc_rooting: NeedsGcRooting,- capacity: usize,- ) -> Result<Self, OutOfMemory> {- Ok(Self {- needs_gc_rooting,- elements: TryVec::with_capacity(capacity)?,- })- }-- /// Push a value onto this passive element segment.- ///- /// NB: Does not type check the value, relies on callers to ensure the value- /// is of the correct type (generally, due to validation).- pub(crate) fn push(&mut self, store: &mut StoreOpaque, val: Val) -> Result<()> {- let val = {- let mut store = AutoAssertNoGc::new(store);- val.to_raw_(&mut store)?- };- let val = self.clone_gc_ref(store, val);- self.elements.push(val)?;- Ok(())- }-- fn clone_gc_ref(&mut self, store: &mut StoreOpaque, val: ValRaw) -> ValRaw {- if let NeedsGcRooting::Yes = self.needs_gc_rooting {- let gc_ref = val.get_anyref();- if let Some(gc_ref) = VMGcRef::from_raw_u32(gc_ref) {- if let Some(gc_store) = store.optional_gc_store_mut() {- return ValRaw::anyref(gc_store.clone_gc_ref(&gc_ref).as_raw_u32());- }- }- }- val- }-- /// Clear this segment's elements.- pub(crate) fn clear(&mut self, mut gc_store: Option<&mut GcStore>) {- for val in mem::take(&mut self.elements) {- self.drop_gc_ref(&mut gc_store, val);- }- }-- fn drop_gc_ref(&mut self, gc_store: &mut Option<&mut GcStore>, val: ValRaw) {- if let NeedsGcRooting::Yes = self.needs_gc_rooting {- let gc_ref = val.get_anyref();- if let Some(gc_ref) = VMGcRef::from_raw_u32(gc_ref) {- if let Some(gc_store) = gc_store.as_deref_mut() {- let _ = gc_store.drop_gc_ref(gc_ref);- }- }- }- }-- /// Whether this segment needs GC rooting and tracing.- #[cfg(feature = "gc")]- pub(crate) fn needs_gc_rooting(&self) -> NeedsGcRooting {- self.needs_gc_rooting- }-- /// The elements of this segment.- pub(crate) fn elements(&self) -> &[ValRaw] {- &self.elements- }-}diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator.rs b/crates/wasmtime/src/runtime/vm/instance/allocator.rsindex e3ea77d1a8..ba607f6f8d 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator.rs@@ -6,8 +6,7 @@ use crate::runtime::vm::memory::Memory;use crate::runtime::vm::mpk::ProtectionKey;use crate::runtime::vm::table::Table;use crate::runtime::vm::{CompiledModuleId, ModuleRuntimeInfo};-use crate::store::{Asyncness, InstanceId, StoreOpaque, StoreResourceLimiter};-use crate::vm::instance::PassiveElementSegment;+use crate::store::{Asyncness, AutoAssertNoGc, InstanceId, StoreOpaque, StoreResourceLimiter};use crate::{OpaqueRootScope, Val};use core::future::Future;use core::pin::Pin;@@ -860,34 +859,40 @@ async fn initialize_passive_elements(for (idx, segment) in &module.passive_elements {match segment {TableSegmentElements::Functions(func_indices) => {- let mut segment =- PassiveElementSegment::new(NeedsGcRooting::No, func_indices.len())?;+ let mut vals = TryVec::with_capacity(func_indices.len())?;for func_idx in func_indices {let (instance, registry) =store.instance_and_module_registry_mut(context.instance);// SAFETY: `store_id` is for the store that owns this instance.let func = unsafe { instance.get_exported_func(registry, store_id, *func_idx) };- segment.push(store, func.into())?;+ vals.push(func.to_val_raw(store))?;}let instance = store.instance_mut(context.instance);debug_assert_eq!(instance.passive_elements.len(), idx.index());- instance.passive_elements_mut().push(segment)?;+ instance+ .passive_elements_mut()+ .push(Some((NeedsGcRooting::No, vals)))?;}TableSegmentElements::Expressions {needs_gc_rooting,exprs,} => {- let mut segment = PassiveElementSegment::new(*needs_gc_rooting, exprs.len())?;+ let mut vals = TryVec::with_capacity(exprs.len())?;for expr in exprs {let mut store = OpaqueRootScope::new(&mut *store);+let val = const_evaluator.eval(&mut store, limiter.as_deref_mut(), context, expr).await?;- segment.push(&mut store, *val)?;++ let mut store = AutoAssertNoGc::new(&mut store);+ vals.push(val.to_raw_(&mut store)?)?;}let instance = store.instance_mut(context.instance);debug_assert_eq!(instance.passive_elements.len(), idx.index());- instance.passive_elements_mut().push(segment)?;+ instance+ .passive_elements_mut()+ .push(Some((*needs_gc_rooting, vals)))?;}}}diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex 54f0374991..013b651cba 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -528,8 +528,7 @@ fn table_init(// Implementation of `elem.drop`.fn elem_drop(store: &mut dyn VMStore, instance: InstanceId, elem_index: u32) -> Result<()> {let elem_index = ElemIndex::from_u32(elem_index);- let (gc_store, instance) = store.optional_gc_store_and_instance_mut(instance);- instance.elem_drop(gc_store, elem_index)?;+ store.instance_mut(instance).elem_drop(elem_index)?;Ok(())}diff --git a/tests/all/gc.rs b/tests/all/gc.rsindex 0e289cb5c5..7204f897f6 100644--- a/tests/all/gc.rs+++ b/tests/all/gc.rs@@ -1705,67 +1705,6 @@ fn select_gc_ref_stack_map() -> Result<()> {Ok(())}-#[test]-#[cfg_attr(miri, ignore)]-fn dropped_passive_elems_do_not_leak() -> Result<()> {- let _ = env_logger::try_init();-- let mut config = Config::new();- config.wasm_function_references(true);- config.wasm_gc(true);- config.collector(Collector::DeferredReferenceCounting);-- let engine = Engine::new(&config)?;-- let module = Module::new(- &engine,- r#"- (module- (type $a (array (mut externref)))- (type $aa (array (ref $a)))-- (elem $e (ref $a) (array.new_default $a (i32.const 1)))-- (func (export "run") (param $x externref)- (local $l (ref null $aa))-- ;; Create an array from the passive element segment.- (local.set $l (array.new_elem $aa $e (i32.const 0) (i32.const 1)))-- ;; Put our externref into the passive element segment's array:- ;; `l[0][0] = x`.- (array.set $a- (array.get $aa (local.get $l) (i32.const 0))- (i32.const 0)- (local.get $x))-- ;; Drop the passive element segment. This should make `x`- ;; unreachable upon return, since we aren't keeping our- ;; array live nor the passive element segment.- (elem.drop $e)- )- )- "#,- )?;-- let mut store = Store::new(&engine, ());- let dropped = Arc::new(AtomicBool::new(false));-- {- let mut store = RootScope::new(&mut store);- let e = ExternRef::new(&mut store, SetFlagOnDrop(dropped.clone()))?;-- let instance = Instance::new(&mut store, &module, &[])?;- let run = instance.get_typed_func::<Rooted<ExternRef>, ()>(&mut store, "run")?;- run.call(&mut store, e)?;- }-- store.gc(None)?;- assert!(dropped.load(SeqCst));-- Ok(())-}-#[test]#[cfg_attr(miri, ignore)]fn gc_heap_does_not_grow_unboundedly() -> Result<()> {diff --git a/tests/misc_testsuite/gc/issue-13066.wast b/tests/misc_testsuite/gc/issue-13066.wastdeleted file mode 100644index a8d8cda7f3..0000000000--- a/tests/misc_testsuite/gc/issue-13066.wast+++ /dev/null@@ -1,31 +0,0 @@-;;! gc = true-;;! bulk_memory = true--(module- (import "wasmtime" "gc" (func $gc))- (type $s (struct (field i32)))- (table $t 10 anyref)-- ;; Passive element segment with a GC struct- (elem $e anyref (struct.new $s (i32.const 42)))-- ;; Copy passive element into table- (func (export "init")- (table.init $t $e (i32.const 0) (i32.const 0) (i32.const 1))- )-- ;; Read the struct from the table and return its field value- (func (export "get_field") (result i32)- (struct.get $s 0- (ref.cast (ref $s)- (table.get $t (i32.const 0))- )- )- )-- (export "gc" (func $gc))-)--(assert_return (invoke "gc"))-(assert_return (invoke "init"))-(assert_return (invoke "get_field") (i32.const 42))