Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Push batched decommit into `sys` module (#13827)

Alex Crichton2c9ffe80b0f0
diff --git a/crates/wasmtime/src/runtime/vm/cow.rs b/crates/wasmtime/src/runtime/vm/cow.rs
index 6f8129f47c..6cc53eff00 100644
--- a/crates/wasmtime/src/runtime/vm/cow.rs
+++ b/crates/wasmtime/src/runtime/vm/cow.rs
@@ -715,7 +715,7 @@ impl MemoryImageSlot {
mod test {
use super::*;
use crate::runtime::vm::mmap::{AlignedLength, Mmap};
- use crate::runtime::vm::sys::vm::decommit_pages;
+ use crate::runtime::vm::sys::vm::{decommit_pages, iovec};
use crate::runtime::vm::{HostAlignedByteCount, MmapVec, host_page_size};
use std::sync::Arc;
use wasmtime_environ::{IndexType, Limits, Memory, MemoryKind, Tunables};
@@ -755,6 +755,16 @@ mod test {
}
}
+ fn decommit(base: *mut u8, len: usize) {
+ unsafe {
+ decommit_pages(&[iovec {
+ iov_base: base.cast(),
+ iov_len: len,
+ }])
+ .unwrap();
+ }
+ }
+
fn dummy_memory() -> Memory {
Memory {
idx_type: IndexType::I32,
@@ -836,9 +846,7 @@ mod test {
// instantiate again; we should see zeroes, even as the
// reuse-anon-mmap-opt kicks in
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
assert!(!memfd.is_dirty());
memfd
@@ -888,9 +896,7 @@ mod test {
// Clear and re-instantiate same image
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
memfd
.instantiate(
@@ -905,9 +911,7 @@ mod test {
// Clear and re-instantiate no image
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
memfd
.instantiate(
@@ -923,9 +927,7 @@ mod test {
// Clear and re-instantiate image again
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
memfd
.instantiate(
@@ -941,9 +943,7 @@ mod test {
// Create another image with different data.
let image2 = Arc::new(create_memfd_with_data(page_size, &[10, 11, 12, 13]).unwrap());
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
memfd
.instantiate(
@@ -959,9 +959,7 @@ mod test {
// Instantiate the original image again; we should notice it's
// a different image and not reuse the mappings.
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
memfd
.instantiate(
@@ -1016,9 +1014,7 @@ mod test {
};
memfd
- .clear_and_remain_ready(None, amt_to_memset, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, amt_to_memset, decommit)
.unwrap();
}
}
@@ -1044,9 +1040,7 @@ mod test {
});
}
memfd
- .clear_and_remain_ready(None, amt_to_memset, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, amt_to_memset, decommit)
.unwrap();
}
}
@@ -1089,9 +1083,7 @@ mod test {
}
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
let slice = unsafe { mmap.slice(0..(64 << 10) + page_size) };
assert_eq!(&[1, 2, 3, 4], &slice[page_size..][..4]);
@@ -1119,9 +1111,7 @@ mod test {
}
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
// Test that memory is still accessible, but it's been reset
@@ -1149,9 +1139,7 @@ mod test {
}
memfd
- .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, |ptr, len| unsafe {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(None, HostAlignedByteCount::ZERO, decommit)
.unwrap();
// Reset the image to none and double-check everything is back to zero
@@ -1204,9 +1192,7 @@ mod test {
let assert_pristine_after_reset = |memfd: &mut MemoryImageSlot| unsafe {
// Wipe the image, keeping some bytes resident.
memfd
- .clear_and_remain_ready(pagemap, keep_resident, |ptr, len| {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(pagemap, keep_resident, decommit)
.unwrap();
// Double check that the contents of memory are as expected after
@@ -1236,9 +1222,7 @@ mod test {
)
.unwrap();
memfd
- .clear_and_remain_ready(pagemap, HostAlignedByteCount::ZERO, |ptr, len| {
- decommit_pages(ptr, len).unwrap()
- })
+ .clear_and_remain_ready(pagemap, HostAlignedByteCount::ZERO, decommit)
.unwrap();
// Next re-instantiate a final time to get used for the next test.
diff --git a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/decommit_queue.rs b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/decommit_queue.rs
index 96dcda3f22..8c27a79815 100644
--- a/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/decommit_queue.rs
+++ b/crates/wasmtime/src/runtime/vm/instance/allocator/pooling/decommit_queue.rs
@@ -9,6 +9,7 @@
//! immediately get flushed every time we push onto it.
use super::PoolingInstanceAllocator;
+use crate::vm::sys::vm::{decommit_pages, iovec};
use crate::vm::{MemoryAllocationIndex, MemoryImageSlot, Table, TableAllocationIndex};
use smallvec::SmallVec;
use std::io;
@@ -16,17 +17,6 @@ use std::io;
#[cfg(feature = "async")]
use wasmtime_fiber::FiberStack;
-#[cfg(unix)]
-#[expect(non_camel_case_types, reason = "matching libc naming")]
-type iovec = libc::iovec;
-
-#[cfg(not(unix))]
-#[expect(non_camel_case_types, reason = "matching libc naming")]
-struct iovec {
- iov_base: *mut libc::c_void,
- iov_len: libc::size_t,
-}
-
#[repr(transparent)]
struct IoVec(iovec);
@@ -158,11 +148,17 @@ impl DecommitQueue {
/// Returns if any decommit call failed.
fn decommit_all_raw(&mut self) -> io::Result<()> {
- for iovec in self.raw.drain(..) {
- unsafe {
- crate::vm::sys::vm::decommit_pages(iovec.0.iov_base.cast(), iovec.0.iov_len)?;
- }
+ let iov: &[IoVec] = self.raw.as_slice();
+ // SAFETY: `IoVec` is `repr(transparent)` over `iovec` so it should be
+ // safe to reinterpret the slice as the same type.
+ let iov = unsafe { &*(iov as *const [IoVec] as *const [iovec]) };
+
+ // SAFETY: the safety of this function depends on the caller of `push_*`
+ // functions, otherwise this is just forwarding its safety requirements.
+ unsafe {
+ decommit_pages(iov)?;
}
+ self.raw.clear();
Ok(())
}
diff --git a/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs b/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs
index 23a98c6e7b..687af2477c 100644
--- a/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/custom/vm.rs
@@ -34,18 +34,28 @@ pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> Result<()> {
Ok(())
}
+#[allow(non_camel_case_types)] // matching C conventions
+#[cfg(feature = "pooling-allocator")]
+pub struct iovec {
+ pub iov_base: *mut u8,
+ pub iov_len: usize,
+}
+
#[cfg(feature = "pooling-allocator")]
pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> Result<()> {
- if len == 0 {
- return Ok(());
- }
+ for iov in iov {
+ if iov.iov_len == 0 {
+ continue;
+ }
- unsafe {
- cvt(capi::wasmtime_mmap_remap(
- addr,
- len,
- capi::PROT_READ | capi::PROT_WRITE,
- ))
+ unsafe {
+ cvt(capi::wasmtime_mmap_remap(
+ iov.iov_addr,
+ iov.iov_len,
+ capi::PROT_READ | capi::PROT_WRITE,
+ ))?;
+ }
+ Ok(())
}
}
diff --git a/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs b/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs
index 9a800573d0..00446aedf4 100644
--- a/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/miri/vm.rs
@@ -34,9 +34,17 @@ pub unsafe fn commit_pages(ptr: *mut u8, len: usize) -> io::Result<()> {
Ok(())
}
-pub unsafe fn decommit_pages(ptr: *mut u8, len: usize) -> io::Result<()> {
- unsafe {
- std::ptr::write_bytes(ptr, 0, len);
+#[allow(non_camel_case_types)] // matching C conventions
+pub struct iovec {
+ pub iov_base: *mut u8,
+ pub iov_len: usize,
+}
+
+pub unsafe fn decommit_pages(iov: &[iovec]) -> io::Result<()> {
+ for iov in iov {
+ unsafe {
+ std::ptr::write_bytes(iov.iov_base, 0, iov.iov_len);
+ }
}
Ok(())
}
diff --git a/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs b/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs
index 8ffac0d796..9f501cd27c 100644
--- a/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/unix/vm.rs
@@ -35,6 +35,9 @@ pub unsafe fn erase_existing_mapping(ptr: *mut u8, len: usize) -> io::Result<()>
Ok(())
}
+#[cfg(feature = "pooling-allocator")]
+pub use libc::iovec;
+
#[cfg(feature = "pooling-allocator")]
pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> io::Result<()> {
// Pages are always READ | WRITE so there's nothing that needs to be done
@@ -43,34 +46,35 @@ pub unsafe fn commit_pages(_addr: *mut u8, _len: usize) -> io::Result<()> {
}
#[cfg(feature = "pooling-allocator")]
-pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> io::Result<()> {
- if len == 0 {
- return Ok(());
- }
+pub unsafe fn decommit_pages(iov: &[iovec]) -> io::Result<()> {
+ for iov in iov {
+ if iov.iov_len == 0 {
+ continue;
+ }
- unsafe {
- cfg_if::cfg_if! {
- if #[cfg(target_os = "linux")] {
- use rustix::mm::{madvise, Advice};
-
- // On Linux, this is enough to cause the kernel to initialize
- // the pages to 0 on next access
- madvise(addr as _, len, Advice::LinuxDontNeed)?;
- } else {
- // By creating a new mapping at the same location, this will
- // discard the mapping for the pages in the given range.
- // The new mapping will be to the CoW zero page, so this
- // effectively zeroes the pages.
- mmap_anonymous(
- addr as _,
- len,
- ProtFlags::READ | ProtFlags::WRITE,
- MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
- )?;
+ unsafe {
+ cfg_if::cfg_if! {
+ if #[cfg(target_os = "linux")] {
+ use rustix::mm::{madvise, Advice};
+
+ // On Linux, this is enough to cause the kernel to initialize
+ // the pages to 0 on next access
+ madvise(iov.iov_base, iov.iov_len, Advice::LinuxDontNeed)?;
+ } else {
+ // By creating a new mapping at the same location, this will
+ // discard the mapping for the pages in the given range.
+ // The new mapping will be to the CoW zero page, so this
+ // effectively zeroes the pages.
+ mmap_anonymous(
+ iov.iov_base,
+ iov.iov_len,
+ ProtFlags::READ | ProtFlags::WRITE,
+ MapFlags::PRIVATE | super::mmap::MMAP_NORESERVE_FLAG | MapFlags::FIXED,
+ )?;
+ }
}
}
}
-
Ok(())
}
diff --git a/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs b/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs
index eb367c9029..32e6f8ecbc 100644
--- a/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs
+++ b/crates/wasmtime/src/runtime/vm/sys/windows/vm.rs
@@ -39,9 +39,21 @@ pub unsafe fn commit_pages(addr: *mut u8, len: usize) -> io::Result<()> {
unsafe { expose_existing_mapping(addr, len) }
}
+#[allow(non_camel_case_types)] // matching C conventions
#[cfg(feature = "pooling-allocator")]
-pub unsafe fn decommit_pages(addr: *mut u8, len: usize) -> io::Result<()> {
- unsafe { erase_existing_mapping(addr, len) }
+pub struct iovec {
+ pub iov_base: *mut u8,
+ pub iov_len: usize,
+}
+
+#[cfg(feature = "pooling-allocator")]
+pub unsafe fn decommit_pages(iov: &[iovec]) -> io::Result<()> {
+ for iov in iov {
+ unsafe {
+ erase_existing_mapping(iov.iov_base, iov.iov_len)?;
+ }
+ }
+ Ok(())
}
pub fn get_page_size() -> usize {