A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Push batched decommit into `sys` module (#13827)
Alex Crichton2c9ffe80b0f0
diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rsindex 6f8129f47c..6cc53eff00 100644--- a/crates/wasmtime/src/runtime/vm/cow.rs+++ b/crates/wasmtime/src/runtime/vm/cow.rs@@ -715,7 +715,7 @@ impl MemoryImageSlot {mod test {use super::*;use crate::runtime::vm::mmap::{AlignedLength, Mmap};- use crate::runtime::vm::sys::vm::decommit_pages;+ use crate::runtime::vm::sys::vm::{decommit_pages, iovec};use crate::runtime::vm::{HostAlignedByteCount, MmapVec, host_page_size};use std::sync::Arc;use wasmtime_environ::{IndexType, Limits, Memory, MemoryKind, Tunables};@@ -755,6 +755,16 @@ mod test {}}+ fn decommit(base: *mut u8, len: usize) {+ unsafe {+ decommit_pages(&[iovec {+ iov_base: base.cast(),+ iov_len: len,+ }])+ .unwrap();+ }+ }+fn dummy_memory() -> Memory {Memory {idx_type: IndexType::I32,@@ -836,9 +846,7 @@ mod test {// instantiate again; we should see zeroes, even as the// reuse-anon-mmap-opt kicks inmemfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();assert!(!memfd.is_dirty());memfd@@ -888,9 +896,7 @@ mod test {// Clear and re-instantiate same imagememfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();memfd.instantiate(@@ -905,9 +911,7 @@ mod test {// Clear and re-instantiate no imagememfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();memfd.instantiate(@@ -923,9 +927,7 @@ mod test {// Clear and re-instantiate image againmemfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();memfd.instantiate(@@ -941,9 +943,7 @@ mod test {// Create another image with different data.let image2 = Arc::new(create_memfd_with_data(page_size, &[10, 11, 12, 13]).unwrap());memfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();memfd.instantiate(@@ -959,9 +959,7 @@ mod test {// Instantiate the original image again; we should notice it's// a different image and not reuse the mappings.memfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();memfd.instantiate(@@ -1016,9 +1014,7 @@ mod test {};memfd- .clear_and_remain_ready(None, amt_to_memset, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, amt_to_memset, decommit).unwrap();}}@@ -1044,9 +1040,7 @@ mod test {});}memfd- .clear_and_remain_ready(None, amt_to_memset, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, amt_to_memset, decommit).unwrap();}}@@ -1089,9 +1083,7 @@ mod test {}memfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();let slice = unsafe { mmap.slice(0..(64 << 10) + page_size) };assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);@@ -1119,9 +1111,7 @@ mod test {}memfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();// Test that memory is still accessible, but it's been reset@@ -1149,9 +1139,7 @@ mod test {}memfd- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit).unwrap();// Reset the image to none and double-check everything is back to zero@@ -1204,9 +1192,7 @@ mod test {let assert_pristine_after_reset = |memfd: &mut MemoryImageSlot| unsafe {// Wipe the image, keeping some bytes resident.memfd- .clear_and_remain_ready(pagemap, keep_resident, |ptr, len| {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(pagemap, keep_resident, decommit).unwrap();// Double check that the contents of memory are as expected after@@ -1236,9 +1222,7 @@ mod test {).unwrap();memfd- .clear_and_remain_ready(pagemap, HostAlignedByteCount::ZERO, |ptr, len| {- decommit_pages(ptr, len).unwrap()- })+ .clear_and_remain_ready(pagemap, HostAlignedByteCount::ZERO, decommit).unwrap();// Next re-instantiate a final time to get used for the next test.diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/decommit_queue.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/decommit_queue.rsindex 96dcda3f22..8c27a79815 100644--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/decommit_queue.rs+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/decommit_queue.rs@@ -9,6 +9,7 @@//! immediately get flushed every time we push onto it.use super::PoolingInstanceAllocator;+use crate::vm::sys::vm::{decommit_pages, iovec};use crate::vm::{MemoryAllocationIndex, MemoryImageSlot, Table, TableAllocationIndex};use smallvec::SmallVec;use std::io;@@ -16,17 +17,6 @@ use std::io;#[cfg(feature = "async")]use wasmtime_fiber::FiberStack;-#[cfg(unix)]-#[expect(non_camel_case_types, reason = "matching libc naming")]-type iovec = libc::iovec;--#[cfg(not(unix))]-#[expect(non_camel_case_types, reason = "matching libc naming")]-struct iovec {- iov_base: *mut libc::c_void,- iov_len: libc::size_t,-}-#[repr(transparent)]struct IoVec(iovec);@@ -158,11 +148,17 @@ impl DecommitQueue {/// Returns if any decommit call failed.fn decommit_all_raw(&mut self) -> io::Result<()> {- for iovec in self.raw.drain(..) {- unsafe {- crate::vm::sys::vm::decommit_pages(iovec.0.iov_base.cast(), iovec.0.iov_len)?;- }+ let iov: &[IoVec] = self.raw.as_slice();+ // SAFETY: `IoVec` is `repr(transparent)` over `iovec` so it should be+ // safe to reinterpret the slice as the same type.+ let iov = unsafe { &*(iov as *const [IoVec] as *const [iovec]) };++ // SAFETY: the safety of this function depends on the caller of `push_*`+ // functions, otherwise this is just forwarding its safety requirements.+ unsafe {+ decommit_pages(iov)?;}+ self.raw.clear();Ok(())}diff --git a/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs b/crates/wasmtime/src/runtime/vm/sys/custom/vm.rsindex 23a98c6e7b..687af2477c 100644--- a/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs+++ b/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs@@ -34,18 +34,28 @@ pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> Result<()> {Ok(())}+#[allow(non_camel_case_types)] // matching C conventions+#[cfg(feature = "pooling-allocator")]+pub struct iovec {+ pub iov_base: *mut u8,+ pub iov_len: usize,+}+#[cfg(feature = "pooling-allocator")]pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> Result<()> {- if len == 0 {- return Ok(());- }+ for iov in iov {+ if iov.iov_len == 0 {+ continue;+ }- unsafe {- cvt(capi::wasmtime_mmap_remap(- addr,- len,- capi::PROT_READ | capi::PROT_WRITE,- ))+ unsafe {+ cvt(capi::wasmtime_mmap_remap(+ iov.iov_addr,+ iov.iov_len,+ capi::PROT_READ | capi::PROT_WRITE,+ ))?;+ }+ Ok(())}}diff --git a/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs b/crates/wasmtime/src/runtime/vm/sys/miri/vm.rsindex 9a800573d0..00446aedf4 100644--- a/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs+++ b/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs@@ -34,9 +34,17 @@ pub unsafe fn commit_pages(ptr: *mut u8, len: usize) -> io::Result<()> {Ok(())}-pub unsafe fn decommit_pages(ptr: *mut u8, len: usize) -> io::Result<()> {- unsafe {- std::ptr::write_bytes(ptr, 0, len);+#[allow(non_camel_case_types)] // matching C conventions+pub struct iovec {+ pub iov_base: *mut u8,+ pub iov_len: usize,+}++pub unsafe fn decommit_pages(iov: &[iovec]) -> io::Result<()> {+ for iov in iov {+ unsafe {+ std::ptr::write_bytes(iov.iov_base, 0, iov.iov_len);+ }}Ok(())}diff --git a/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs b/crates/wasmtime/src/runtime/vm/sys/unix/vm.rsindex 8ffac0d796..9f501cd27c 100644--- a/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs+++ b/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs@@ -35,6 +35,9 @@ pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()>Ok(())}+#[cfg(feature = "pooling-allocator")]+pub use libc::iovec;+#[cfg(feature = "pooling-allocator")]pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> io::Result<()> {// Pages are always READ | WRITE so there's nothing that needs to be done@@ -43,34 +46,35 @@ pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> io::Result<()> {}#[cfg(feature = "pooling-allocator")]-pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> io::Result<()> {- if len == 0 {- return Ok(());- }+pub unsafe fn decommit_pages(iov: &[iovec]) -> io::Result<()> {+ for iov in iov {+ if iov.iov_len == 0 {+ continue;+ }- unsafe {- cfg_if::cfg_if! {- if #[cfg(target_os = "linux")] {- use rustix::mm::{madvise, Advice};-- // On Linux, this is enough to cause the kernel to initialize- // the pages to 0 on next access- madvise(addr as _, len, Advice::LinuxDontNeed)?;- } else {- // By creating a new mapping at the same location, this will- // discard the mapping for the pages in the given range.- // The new mapping will be to the CoW zero page, so this- // effectively zeroes the pages.- mmap_anonymous(- addr as _,- len,- ProtFlags::READ | ProtFlags::WRITE,- MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,- )?;+ unsafe {+ cfg_if::cfg_if! {+ if #[cfg(target_os = "linux")] {+ use rustix::mm::{madvise, Advice};++ // On Linux, this is enough to cause the kernel to initialize+ // the pages to 0 on next access+ madvise(iov.iov_base, iov.iov_len, Advice::LinuxDontNeed)?;+ } else {+ // By creating a new mapping at the same location, this will+ // discard the mapping for the pages in the given range.+ // The new mapping will be to the CoW zero page, so this+ // effectively zeroes the pages.+ mmap_anonymous(+ iov.iov_base,+ iov.iov_len,+ ProtFlags::READ | ProtFlags::WRITE,+ MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,+ )?;+ }}}}-Ok(())}diff --git a/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs b/crates/wasmtime/src/runtime/vm/sys/windows/vm.rsindex eb367c9029..32e6f8ecbc 100644--- a/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs+++ b/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs@@ -39,9 +39,21 @@ pub unsafe fn commit_pages(addr: *mut u8, len: usize) -> io::Result<()> {unsafe { expose_existing_mapping(addr, len) }}+#[allow(non_camel_case_types)] // matching C conventions#[cfg(feature = "pooling-allocator")]-pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> io::Result<()> {- unsafe { erase_existing_mapping(addr, len) }+pub struct iovec {+ pub iov_base: *mut u8,+ pub iov_len: usize,+}++#[cfg(feature = "pooling-allocator")]+pub unsafe fn decommit_pages(iov: &[iovec]) -> io::Result<()> {+ for iov in iov {+ unsafe {+ erase_existing_mapping(iov.iov_base, iov.iov_len)?;+ }+ }+ Ok(())}pub fn get_page_size() -> usize {