Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Optimize the implementation of `memory.fill` (#13367)

Alex Crichton40c9406a2ae9
diff --git a/crates/cranelift/src/bounds_checks.rs b/crates/cranelift/src/bounds_checks.rs
index 310549224c..ab9bfc3dc9 100644
--- a/crates/cranelift/src/bounds_checks.rs
+++ b/crates/cranelift/src/bounds_checks.rs
@@ -618,7 +618,15 @@ fn explicit_check_oob_condition_and_compute_addr(
/// It is the caller's responsibility to ensure that any necessary bounds and
/// overflow checks are emitted, and that the resulting address is never used
/// unless they succeed.
-fn compute_addr(
+///
+/// Arguments are:
+///
+/// * `pos` - where to generate instructions
+/// * `heap` - the memory being accessed
+/// * `addr_ty` - the host's pointer type
+/// * `index` - the raw wasm index, of type `addr_ty`
+/// * `offset` - a static offset to add to `index`
+pub fn compute_addr(
pos: &mut FuncCursor,
heap: &HeapData,
addr_ty: ir::Type,
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 70b344efc8..201a47d856 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -771,6 +771,25 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
}
}
+ /// Cast the wasm pointer `val`, with type `index_type`, to a host pointer
+ /// type.
+ ///
+ /// Does not perform any in-bounds checks, so `val` must already be
+ /// validated to be in-bounds.
+ fn unchecked_cast_wasm_addr_to_native_addr(
+ &self,
+ pos: &mut FuncCursor<'_>,
+ val: ir::Value,
+ index_type: IndexType,
+ ) -> ir::Value {
+ match (self.pointer_type(), index_type) {
+ (I32, IndexType::I32) | (I64, IndexType::I64) => val,
+ (I32, IndexType::I64) => pos.ins().ireduce(I32, val),
+ (I64, IndexType::I32) => pos.ins().uextend(I64, val),
+ _ => unreachable!(),
+ }
+ }
+
/// Convert the target pointer-sized integer `val` into the memory/table's index type.
///
/// For memory, `val` is holding a memory length (or the `-1` `memory.grow`-failed sentinel).
@@ -3328,16 +3347,16 @@ impl FuncEnvironment<'_> {
))
}
- pub fn translate_memory_size(
- &mut self,
- mut pos: FuncCursor<'_>,
- index: MemoryIndex,
- ) -> WasmResult<ir::Value> {
+ /// Loads the size, in bytes, of the memory `index` specified.
+ ///
+ /// Returns the `ir::Value`, typed as a pointer-width integer, that is the
+ /// size in bytes.
+ fn memory_size_in_bytes(&mut self, pos: &mut FuncCursor<'_>, index: MemoryIndex) -> ir::Value {
let pointer_type = self.pointer_type();
let vmctx = self.vmctx(&mut pos.func);
let is_shared = self.module.memories[index].shared;
let base = pos.ins().global_value(pointer_type, vmctx);
- let current_length_in_bytes = match self.module.defined_memory_index(index) {
+ match self.module.defined_memory_index(index) {
Some(def_index) => {
if is_shared {
let offset =
@@ -3395,7 +3414,15 @@ impl FuncEnvironment<'_> {
)
}
}
- };
+ }
+ }
+
+ pub fn translate_memory_size(
+ &mut self,
+ mut pos: FuncCursor<'_>,
+ index: MemoryIndex,
+ ) -> WasmResult<ir::Value> {
+ let current_length_in_bytes = self.memory_size_in_bytes(&mut pos, index);
let page_size_log2 = i64::from(self.module.memories[index].page_size_log2);
let current_length_in_pages = pos.ins().ushr_imm(current_length_in_bytes, page_size_log2);
@@ -3454,20 +3481,64 @@ impl FuncEnvironment<'_> {
dst: ir::Value,
val: ir::Value,
len: ir::Value,
- ) -> WasmResult<()> {
+ ) {
+ let pointer_type = self.pointer_type();
+ let idx_type = self.memory(memory_index).idx_type;
+
+ // Load the memory size, as `pointer_type`, which is the size of this
+ // memory currently in
+ // bytes.
+ let size_in_bytes = self.memory_size_in_bytes(&mut builder.cursor(), memory_index);
+
+ // Compute the end address of this fill, casted to the `I64` type.
+ //
+ // Note that addition can't overflow after extending 32-bits to
+ // 64-bits, so no need to check for overflow in the 32-bit index case.
+ let end64 = match idx_type {
+ IndexType::I32 => {
+ let dst64 = builder.ins().uextend(I64, dst);
+ let len64 = builder.ins().uextend(I64, len);
+ builder.ins().iadd(dst64, len64)
+ }
+ IndexType::I64 => {
+ self.uadd_overflow_trap(builder, dst, len, ir::TrapCode::HEAP_OUT_OF_BOUNDS)
+ }
+ };
+
+ // Cast the host-pointer width to a 64-bit bit width.
+ let size_in_bytes64 = match pointer_type {
+ I32 => builder.ins().uextend(I64, size_in_bytes),
+ I64 => size_in_bytes,
+ _ => unreachable!(),
+ };
+
+ // This is the actual bounds check that verifies that this `fill`
+ // operation is in-bounds. Once control flow gets past here we know
+ // that nothing can overflow and everything is in-bounds.
+ let inbounds = builder
+ .ins()
+ .icmp(IntCC::UnsignedLessThanOrEqual, end64, size_in_bytes64);
+ self.trapz(builder, inbounds, ir::TrapCode::HEAP_OUT_OF_BOUNDS);
+
+ let memory_fill = self.builtin_functions.memory_fill(&mut builder.func);
let mut pos = builder.cursor();
- let memory_fill = self.builtin_functions.memory_fill(&mut pos.func);
- let dst = self.cast_index_to_i64(&mut pos, dst, self.memory(memory_index).idx_type);
- let len = self.cast_index_to_i64(&mut pos, len, self.memory(memory_index).idx_type);
- let (memory_vmctx, defined_memory_index) =
- self.memory_vmctx_and_defined_index(&mut pos, memory_index);
+ let vmctx = self.vmctx_val(&mut pos);
- pos.ins().call(
- memory_fill,
- &[memory_vmctx, defined_memory_index, dst, val, len],
- );
+ // Compute the actual raw heap address that the `memset` is writing to.
+ let heap = self.get_or_create_heap(pos.func, memory_index);
+ let heap = self.heaps()[heap].clone();
+ let dst_ptr = self.unchecked_cast_wasm_addr_to_native_addr(&mut pos, dst, idx_type);
+ let raw_heap_addr =
+ crate::bounds_checks::compute_addr(&mut pos, &heap, pointer_type, dst_ptr, 0);
- Ok(())
+ // Fit the `len` value to `pointer_type`. Note that at this point it's
+ // guaranteed inbounds so there's no loss in precision.
+ let len_ptr =
+ self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, idx_type);
+
+ builder
+ .ins()
+ .call(memory_fill, &[vmctx, raw_heap_addr, val, len_ptr]);
}
pub fn translate_memory_init(
diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs
index aa0797de67..23cc213bfd 100644
--- a/crates/cranelift/src/lib.rs
+++ b/crates/cranelift/src/lib.rs
@@ -385,7 +385,6 @@ impl BuiltinFunctionSignatures {
AbiParam::new(ir::types::I8)
}
- #[cfg(feature = "stack-switching")]
fn size(&self) -> AbiParam {
AbiParam::new(self.pointer_type)
}
diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs
index aa64e77bba..1d845bcb49 100644
--- a/crates/cranelift/src/translate/code_translator.rs
+++ b/crates/cranelift/src/translate/code_translator.rs
@@ -1638,7 +1638,7 @@ pub fn translate_operator(
let len = environ.stacks.pop1();
let val = environ.stacks.pop1();
let dest = environ.stacks.pop1();
- environ.translate_memory_fill(builder, mem, dest, val, len)?;
+ environ.translate_memory_fill(builder, mem, dest, val, len);
}
Operator::MemoryInit { data_index, mem } => {
let mem = MemoryIndex::from_u32(*mem);
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index 015797ffd6..f0cee56a8b 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -15,7 +15,7 @@ macro_rules! foreach_builtin_function {
// Returns an index for wasm's `memory.copy`
memory_copy(vmctx: vmctx, dst_index: u32, dst: u64, src_index: u32, src: u64, len: u64) -> bool;
// Returns an index for wasm's `memory.fill` instruction.
- memory_fill(vmctx: vmctx, memory: u32, dst: u64, val: u32, len: u64) -> bool;
+ memory_fill(vmctx: vmctx, dst: pointer, val: u32, len: size);
// Returns an index for wasm's `memory.init` instruction.
memory_init(vmctx: vmctx, memory: u32, data: u32, dst: u64, src: u32, len: u32) -> bool;
// Returns a value for wasm's `ref.func` instruction.
diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs
index 3be7d97009..e827e8adf2 100644
--- a/crates/wasmtime/src/runtime/vm/instance.rs
+++ b/crates/wasmtime/src/runtime/vm/instance.rs
@@ -1079,35 +1079,6 @@ impl Instance {
}
}
- /// Perform the `memory.fill` operation on a locally defined memory.
- ///
- /// # Errors
- ///
- /// Returns a `Trap` error if the memory range is out of bounds.
- pub(crate) fn memory_fill(
- self: Pin<&mut Self>,
- memory_index: DefinedMemoryIndex,
- dst: u64,
- val: u8,
- len: u64,
- ) -> Result<(), Trap> {
- let memory_index = self.env_module().memory_index(memory_index);
- let memory = self.get_memory(memory_index);
- let dst = self.validate_inbounds(memory.current_length(), dst, len)?;
- let len = usize::try_from(len).unwrap();
-
- // Bounds and casts are checked above, by this point we know that
- // everything is safe.
- unsafe {
- let dst = memory.base.as_ptr().add(dst);
- // FIXME audit whether this is safe in the presence of shared memory
- // (https://github.com/bytecodealliance/wasmtime/issues/4203).
- ptr::write_bytes(dst, val, len);
- }
-
- Ok(())
- }
-
/// Get the internal storage range of a particular Wasm data segment.
pub(crate) fn wasm_data_range(&self, index: DataIndex) -> Range<u32> {
match self.env_module().passive_data_map.get(&index) {
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index e7829d34ec..50c7673dd8 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -551,20 +551,23 @@ fn memory_copy(
.memory_copy(dst_index, dst, src_index, src, len)
}
-// Implementation of `memory.fill` for locally defined memories.
-fn memory_fill(
- store: &mut dyn VMStore,
- instance: InstanceId,
- memory_index: u32,
- dst: u64,
+unsafe fn memory_fill(
+ _store: &mut dyn VMStore,
+ _instance: InstanceId,
+ dst: *mut u8,
val: u32,
- len: u64,
-) -> Result<(), Trap> {
- let memory_index = DefinedMemoryIndex::from_u32(memory_index);
- #[expect(clippy::cast_possible_truncation, reason = "known to truncate here")]
- store
- .instance_mut(instance)
- .memory_fill(memory_index, dst, val as u8, len)
+ len: usize,
+) {
+ // FIXME(#4203): this is known to not be sound in the presence of shared
+ // memories.
+ unsafe {
+ #[expect(
+ clippy::cast_possible_truncation,
+ reason = "the libcall intentionally takes the raw 32-bit value, \
+ and semantically that's intentionally truncated"
+ )]
+ dst.write_bytes(val as u8, len);
+ }
}
// Implementation of `memory.init`.
diff --git a/tests/disas/memory_fill_host32.wat b/tests/disas/memory_fill_host32.wat
new file mode 100644
index 0000000000..61ec4847ad
--- /dev/null
+++ b/tests/disas/memory_fill_host32.wat
@@ -0,0 +1,172 @@
+;;! target = 'pulley32'
+;;! test = 'optimize'
+;;! flags = '-Wcustom-page-sizes'
+
+(module
+ (memory $m32 1)
+ (memory $m64 i64 1)
+
+ (memory $m32p1 65536 (pagesize 1))
+ (memory $m64p1 i64 65536 (pagesize 1))
+
+ (func $fill32 (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m32
+ )
+
+ (func $fill64 (param i64 i64)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m64
+ )
+
+ (func $fill32p1 (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m32p1
+ )
+
+ (func $fill64p1 (param i64 i64)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m64p1
+ )
+
+ (memory $empty 0 0)
+
+ (func $fill-empty (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $empty
+ )
+)
+;; function u0:0(i32 vmctx, i32, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+48
+;; gv2 = load.i32 notrap aligned can_move gv0+44
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:5 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i32, v3: i32):
+;; @003c v6 = load.i32 notrap aligned v0+48
+;; @003c v7 = uextend.i64 v2
+;; @003c v8 = uextend.i64 v3
+;; @003c v9 = iadd v7, v8
+;; @003c v10 = uextend.i64 v6
+;; @003c v11 = icmp ule v9, v10
+;; @003c trapz v11, heap_oob
+;; @003c v13 = load.i32 notrap aligned can_move v0+44
+;; @003c v14 = iadd v13, v2
+;; @0038 v4 = iconst.i32 0
+;; @003c call fn0(v0, v14, v4, v3) ; v4 = 0
+;; @003f jump block1
+;;
+;; block1:
+;; @003f return
+;; }
+;;
+;; function u0:1(i32 vmctx, i32, i64, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+56
+;; gv2 = load.i32 notrap aligned can_move gv0+52
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:5 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i64, v3: i64):
+;; @0048 v6 = load.i32 notrap aligned v0+56
+;; @0048 v7 = uadd_overflow_trap v2, v3, heap_oob
+;; @0048 v8 = uextend.i64 v6
+;; @0048 v9 = icmp ule v7, v8
+;; @0048 trapz v9, heap_oob
+;; @0048 v12 = load.i32 notrap aligned can_move v0+52
+;; @0048 v11 = ireduce.i32 v2
+;; @0048 v13 = iadd v12, v11
+;; @0044 v4 = iconst.i32 0
+;; @0048 v14 = ireduce.i32 v3
+;; @0048 call fn0(v0, v13, v4, v14) ; v4 = 0
+;; @004b jump block1
+;;
+;; block1:
+;; @004b return
+;; }
+;;
+;; function u0:2(i32 vmctx, i32, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+64
+;; gv2 = load.i32 notrap aligned can_move gv0+60
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:5 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i32, v3: i32):
+;; @0054 v6 = load.i32 notrap aligned v0+64
+;; @0054 v7 = uextend.i64 v2
+;; @0054 v8 = uextend.i64 v3
+;; @0054 v9 = iadd v7, v8
+;; @0054 v10 = uextend.i64 v6
+;; @0054 v11 = icmp ule v9, v10
+;; @0054 trapz v11, heap_oob
+;; @0054 v13 = load.i32 notrap aligned can_move v0+60
+;; @0054 v14 = iadd v13, v2
+;; @0050 v4 = iconst.i32 0
+;; @0054 call fn0(v0, v14, v4, v3) ; v4 = 0
+;; @0057 jump block1
+;;
+;; block1:
+;; @0057 return
+;; }
+;;
+;; function u0:3(i32 vmctx, i32, i64, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+72
+;; gv2 = load.i32 notrap aligned can_move gv0+68
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:5 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i64, v3: i64):
+;; @0060 v6 = load.i32 notrap aligned v0+72
+;; @0060 v7 = uadd_overflow_trap v2, v3, heap_oob
+;; @0060 v8 = uextend.i64 v6
+;; @0060 v9 = icmp ule v7, v8
+;; @0060 trapz v9, heap_oob
+;; @0060 v12 = load.i32 notrap aligned can_move v0+68
+;; @0060 v11 = ireduce.i32 v2
+;; @0060 v13 = iadd v12, v11
+;; @005c v4 = iconst.i32 0
+;; @0060 v14 = ireduce.i32 v3
+;; @0060 call fn0(v0, v13, v4, v14) ; v4 = 0
+;; @0063 jump block1
+;;
+;; block1:
+;; @0063 return
+;; }
+;;
+;; function u0:4(i32 vmctx, i32, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+80
+;; gv2 = load.i32 notrap aligned readonly can_move gv0+76
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:5 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i32, v3: i32):
+;; @006c v6 = load.i32 notrap aligned v0+80
+;; @006c v7 = uextend.i64 v2
+;; @006c v8 = uextend.i64 v3
+;; @006c v9 = iadd v7, v8
+;; @006c v10 = uextend.i64 v6
+;; @006c v11 = icmp ule v9, v10
+;; @006c trapz v11, heap_oob
+;; @006c v13 = load.i32 notrap aligned readonly can_move v0+76
+;; @006c v14 = iadd v13, v2
+;; @0068 v4 = iconst.i32 0
+;; @006c call fn0(v0, v14, v4, v3) ; v4 = 0
+;; @006f jump block1
+;;
+;; block1:
+;; @006f return
+;; }
diff --git a/tests/disas/memory_fill_host64.wat b/tests/disas/memory_fill_host64.wat
new file mode 100644
index 0000000000..af8650e35d
--- /dev/null
+++ b/tests/disas/memory_fill_host64.wat
@@ -0,0 +1,180 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+;;! flags = '-Wcustom-page-sizes'
+
+(module
+ (memory $m32 1)
+ (func $fill32 (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m32
+ )
+
+ (memory $m64 i64 1)
+ (func $fill64 (param i64 i64)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m64
+ )
+
+ (memory $m32p1 65536 (pagesize 1))
+ (func $fill32p1 (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m32p1
+ )
+
+ (memory $m64p1 i64 65536 (pagesize 1))
+ (func $fill64p1 (param i64 i64)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m64p1
+ )
+
+ (memory $empty 0 0)
+ (func $fill-empty (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $empty
+ )
+)
+;; function u0:0(i64 vmctx, i64, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+96
+;; gv5 = load.i64 notrap aligned readonly can_move gv3+88
+;; sig0 = (i64 vmctx, i64, i32, i64) tail
+;; fn0 = colocated u805306368:5 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32, v3: i32):
+;; @003c v6 = load.i64 notrap aligned v0+96
+;; @003c v7 = uextend.i64 v2
+;; @003c v8 = uextend.i64 v3
+;; @003c v9 = iadd v7, v8
+;; @003c v10 = icmp ule v9, v6
+;; @003c trapz v10, heap_oob
+;; @003c v13 = load.i64 notrap aligned readonly can_move v0+88
+;; @003c v14 = iadd v13, v7
+;; @0038 v4 = iconst.i32 0
+;; @003c call fn0(v0, v14, v4, v8) ; v4 = 0
+;; @003f jump block1
+;;
+;; block1:
+;; @003f return
+;; }
+;;
+;; function u0:1(i64 vmctx, i64, i64, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+112
+;; gv5 = load.i64 notrap aligned can_move gv3+104
+;; sig0 = (i64 vmctx, i64, i32, i64) tail
+;; fn0 = colocated u805306368:5 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; @0048 v6 = load.i64 notrap aligned v0+112
+;; @0048 v7 = uadd_overflow_trap v2, v3, heap_oob
+;; @0048 v8 = icmp ule v7, v6
+;; @0048 trapz v8, heap_oob
+;; @0048 v10 = load.i64 notrap aligned can_move v0+104
+;; @0048 v11 = iadd v10, v2
+;; @0044 v4 = iconst.i32 0
+;; @0048 call fn0(v0, v11, v4, v3) ; v4 = 0
+;; @004b jump block1
+;;
+;; block1:
+;; @004b return
+;; }
+;;
+;; function u0:2(i64 vmctx, i64, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+128
+;; gv5 = load.i64 notrap aligned readonly can_move gv3+120
+;; sig0 = (i64 vmctx, i64, i32, i64) tail
+;; fn0 = colocated u805306368:5 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32, v3: i32):
+;; @0054 v6 = load.i64 notrap aligned v0+128
+;; @0054 v7 = uextend.i64 v2
+;; @0054 v8 = uextend.i64 v3
+;; @0054 v9 = iadd v7, v8
+;; @0054 v10 = icmp ule v9, v6
+;; @0054 trapz v10, heap_oob
+;; @0054 v13 = load.i64 notrap aligned readonly can_move v0+120
+;; @0054 v14 = iadd v13, v7
+;; @0050 v4 = iconst.i32 0
+;; @0054 call fn0(v0, v14, v4, v8) ; v4 = 0
+;; @0057 jump block1
+;;
+;; block1:
+;; @0057 return
+;; }
+;;
+;; function u0:3(i64 vmctx, i64, i64, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+144
+;; gv5 = load.i64 notrap aligned can_move gv3+136
+;; sig0 = (i64 vmctx, i64, i32, i64) tail
+;; fn0 = colocated u805306368:5 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
+;; @0060 v6 = load.i64 notrap aligned v0+144
+;; @0060 v7 = uadd_overflow_trap v2, v3, heap_oob
+;; @0060 v8 = icmp ule v7, v6
+;; @0060 trapz v8, heap_oob
+;; @0060 v10 = load.i64 notrap aligned can_move v0+136
+;; @0060 v11 = iadd v10, v2
+;; @005c v4 = iconst.i32 0
+;; @0060 call fn0(v0, v11, v4, v3) ; v4 = 0
+;; @0063 jump block1
+;;
+;; block1:
+;; @0063 return
+;; }
+;;
+;; function u0:4(i64 vmctx, i64, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+160
+;; gv5 = load.i64 notrap aligned readonly can_move gv3+152
+;; sig0 = (i64 vmctx, i64, i32, i64) tail
+;; fn0 = colocated u805306368:5 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32, v3: i32):
+;; @006c v6 = load.i64 notrap aligned v0+160
+;; @006c v7 = uextend.i64 v2
+;; @006c v8 = uextend.i64 v3
+;; @006c v9 = iadd v7, v8
+;; @006c v10 = icmp ule v9, v6
+;; @006c trapz v10, heap_oob
+;; @006c v13 = load.i64 notrap aligned readonly can_move v0+152
+;; @006c v14 = iadd v13, v7
+;; @0068 v4 = iconst.i32 0
+;; @006c call fn0(v0, v14, v4, v8) ; v4 = 0
+;; @006f jump block1
+;;
+;; block1:
+;; @006f return
+;; }
diff --git a/tests/misc_testsuite/memory_fill.wast b/tests/misc_testsuite/memory_fill.wast
new file mode 100644
index 0000000000..64f1b76810
--- /dev/null
+++ b/tests/misc_testsuite/memory_fill.wast
@@ -0,0 +1,97 @@
+;;! memory64 = true
+;;! bulk_memory = true
+;;! multi_memory = true
+;;! custom_page_sizes = true
+
+(module
+ (memory $m32 1)
+ (func (export "fill32") (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m32
+ )
+
+ (memory $m64 i64 1)
+ (func (export "fill64") (param i64 i64)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m64
+ )
+
+ (memory $m32p1 65536 (pagesize 1))
+ (func (export "fill32p1") (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m32p1
+ )
+
+ (memory $m64p1 i64 65536 (pagesize 1))
+ (func (export "fill64p1") (param i64 i64)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $m64p1
+ )
+
+ (memory $empty 0 0)
+ (func (export "fill-empty") (param i32 i32)
+ local.get 0
+ i32.const 0
+ local.get 1
+ memory.fill $empty
+ )
+)
+
+(assert_return (invoke "fill32" (i32.const 0) (i32.const 16)))
+(assert_return (invoke "fill32" (i32.const 0) (i32.const 0)))
+(assert_return (invoke "fill32" (i32.const 65536) (i32.const 0)))
+(assert_trap (invoke "fill32" (i32.const 65536) (i32.const 1)) "out of bounds")
+(assert_return (invoke "fill32" (i32.const 65535) (i32.const 1)))
+(assert_trap (invoke "fill32" (i32.const 65535) (i32.const 2)) "out of bounds")
+(assert_trap (invoke "fill32" (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "fill32" (i32.const 1) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "fill32" (i32.const -1) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "fill32" (i32.const -1) (i32.const 1)) "out of bounds")
+
+(assert_return (invoke "fill64" (i64.const 0) (i64.const 16)))
+(assert_return (invoke "fill64" (i64.const 0) (i64.const 0)))
+(assert_return (invoke "fill64" (i64.const 65536) (i64.const 0)))
+(assert_trap (invoke "fill64" (i64.const 65536) (i64.const 1)) "out of bounds")
+(assert_return (invoke "fill64" (i64.const 65535) (i64.const 1)))
+(assert_trap (invoke "fill64" (i64.const 65535) (i64.const 2)) "out of bounds")
+(assert_trap (invoke "fill64" (i64.const 0) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "fill64" (i64.const 1) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "fill64" (i64.const -1) (i64.const 0)) "out of bounds")
+(assert_trap (invoke "fill64" (i64.const -1) (i64.const 1)) "out of bounds")
+
+(assert_return (invoke "fill32p1" (i32.const 0) (i32.const 16)))
+(assert_return (invoke "fill32p1" (i32.const 0) (i32.const 0)))
+(assert_return (invoke "fill32p1" (i32.const 65536) (i32.const 0)))
+(assert_trap (invoke "fill32p1" (i32.const 65536) (i32.const 1)) "out of bounds")
+(assert_return (invoke "fill32p1" (i32.const 65535) (i32.const 1)))
+(assert_trap (invoke "fill32p1" (i32.const 65535) (i32.const 2)) "out of bounds")
+(assert_trap (invoke "fill32p1" (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "fill32p1" (i32.const 1) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "fill32p1" (i32.const -1) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "fill32p1" (i32.const -1) (i32.const 1)) "out of bounds")
+
+(assert_return (invoke "fill64p1" (i64.const 0) (i64.const 16)))
+(assert_return (invoke "fill64p1" (i64.const 0) (i64.const 0)))
+(assert_return (invoke "fill64p1" (i64.const 65536) (i64.const 0)))
+(assert_trap (invoke "fill64p1" (i64.const 65536) (i64.const 1)) "out of bounds")
+(assert_return (invoke "fill64p1" (i64.const 65535) (i64.const 1)))
+(assert_trap (invoke "fill64p1" (i64.const 0) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "fill64p1" (i64.const 1) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "fill64p1" (i64.const 65535) (i64.const 2)) "out of bounds")
+(assert_trap (invoke "fill64p1" (i64.const -1) (i64.const 0)) "out of bounds")
+(assert_trap (invoke "fill64p1" (i64.const -1) (i64.const 1)) "out of bounds")
+
+(assert_return (invoke "fill-empty" (i32.const 0) (i32.const 0)))
+(assert_trap (invoke "fill-empty" (i32.const 0) (i32.const 1)) "out of bounds")
+(assert_trap (invoke "fill-empty" (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "fill-empty" (i32.const 1) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "fill-empty" (i32.const -1) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "fill-empty" (i32.const -1) (i32.const 1)) "out of bounds")
diff --git a/winch/codegen/src/codegen/bounds.rs b/winch/codegen/src/codegen/bounds.rs
index 00cd6281d6..1c7d151ed9 100644
--- a/winch/codegen/src/codegen/bounds.rs
+++ b/winch/codegen/src/codegen/bounds.rs
@@ -7,7 +7,7 @@ use crate::{
abi::vmctx,
codegen::{CodeGenContext, Emission},
isa::reg::{Reg, writable},
- masm::{Imm, IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm, TrapCode},
+ masm::{IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm, TrapCode},
stack::TypedReg,
};
use wasmtime_environ::WasmValType;
@@ -140,7 +140,7 @@ pub(crate) fn ensure_index_and_offset<M: MacroAssembler>(
masm.checked_uadd(
writable!(index.as_typed_reg().into()),
index.as_typed_reg().into(),
- Imm::i64(offset as i64),
+ RegImm::i64(offset as i64),
heap_ty_size,
TrapCode::HEAP_OUT_OF_BOUNDS,
)?;
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index a42776c3b4..ec4c460225 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -829,7 +829,7 @@ where
self.masm.checked_uadd(
writable!(index_offset_and_access_size),
index_offset_and_access_size,
- Imm::i64(offset_with_access_size as i64),
+ RegImm::i64(offset_with_access_size as i64),
ptr_size,
TrapCode::HEAP_OUT_OF_BOUNDS,
)?;
@@ -1096,9 +1096,7 @@ where
}
/// Retrieves the size of the memory, pushing the result to the value stack.
- pub fn emit_compute_memory_size(&mut self, heap_data: &HeapData) -> Result<()> {
- let size_reg = self.context.any_gpr(self.masm)?;
-
+ fn load_memory_length(&mut self, heap_data: &HeapData, size_reg: Reg) -> Result<()> {
self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {
let base = if let Some(offset) = heap_data.import_from {
masm.load_ptr(masm.address_at_vmctx(offset)?, scratch.writable())?;
@@ -1110,6 +1108,14 @@ where
let size_addr = masm.address_at_reg(base, heap_data.current_length_offset)?;
masm.load_ptr(size_addr, writable!(size_reg))
})?;
+ Ok(())
+ }
+
+ /// Retrieves the size of the memory, pushing the result to the value stack.
+ pub fn emit_compute_memory_size(&mut self, heap_data: &HeapData) -> Result<()> {
+ let size_reg = self.context.any_gpr(self.masm)?;
+ self.load_memory_length(heap_data, size_reg)?;
+
// Emit a shift to get the size in pages rather than in bytes.
let dst = TypedReg::new(heap_data.index_type(), size_reg);
let pow = heap_data.memory.page_size_log2;
@@ -1124,6 +1130,101 @@ where
Ok(())
}
+ /// Emit the `memory.fill` operation.
+ pub fn emit_memory_fill(&mut self, mem: MemoryIndex) -> Result<()> {
+ let heap = self.env.resolve_heap(mem);
+ let ptr_size: OperandSize = self.env.ptr_type().try_into()?;
+ let idx_size: OperandSize = heap.index_type().try_into()?;
+
+ // The wasm stack at this point is `[dst, val, len]`.
+ let len = self.context.pop_to_reg(self.masm, None)?;
+ let val = self.context.pop_to_reg(self.masm, None)?;
+ let dst = self.context.pop_to_reg(self.masm, None)?;
+
+ // Compute `end = dst + len` trapping on overflow. For an `i32` index
+ // type the operands are zero-extended to 64-bit so overflow is
+ // impossible.
+ let end = self.context.any_gpr(self.masm)?;
+ match idx_size {
+ OperandSize::S32 => {
+ self.masm
+ .extend(writable!(end), dst.reg, Extend::<Zero>::I64Extend32.into())?;
+ self.masm.add_uextend(
+ writable!(end),
+ end,
+ len.reg,
+ OperandSize::S32,
+ OperandSize::S64,
+ )?;
+ }
+ OperandSize::S64 => {
+ self.masm
+ .mov(writable!(end), dst.reg.into(), OperandSize::S64)?;
+ self.masm.checked_uadd(
+ writable!(end),
+ end,
+ len.reg.into(),
+ OperandSize::S64,
+ TrapCode::HEAP_OUT_OF_BOUNDS,
+ )?;
+ }
+ _ => unreachable!(),
+ }
+
+ // Load the current size in bytes of the memory.
+ let size_in_bytes = self.context.any_gpr(self.masm)?;
+ self.load_memory_length(&heap, size_in_bytes)?;
+
+ // Trap if `end > size_in_bytes`.
+ assert!(ptr_size == OperandSize::S64);
+ self.masm.cmp(end, size_in_bytes.into(), ptr_size)?;
+ self.masm
+ .trapif(IntCmpKind::GtU, TrapCode::HEAP_OUT_OF_BOUNDS)?;
+ self.context.free_reg(end);
+ self.context.free_reg(size_in_bytes);
+
+ // Compute `dst_ptr = memory_base + dst`.
+ let dst_ptr = self.context.any_gpr(self.masm)?;
+ bounds::load_heap_addr_unchecked(
+ self.masm,
+ &heap,
+ Index::from_typed_reg(dst),
+ ImmOffset::from_u32(0),
+ dst_ptr,
+ ptr_size,
+ )?;
+ self.context.free_reg(dst.reg);
+
+ // The libcall takes the length as a host-pointer-sized integer, so
+ // zero-extend if the wasm index type is smaller.
+ let len_reg = len.reg;
+ if idx_size == OperandSize::S32 && ptr_size == OperandSize::S64 {
+ self.masm.extend(
+ writable!(len_reg),
+ len_reg,
+ Extend::<Zero>::I64Extend32.into(),
+ )?;
+ }
+
+ // Set up the call arguments: `[dst_ptr, val, len]`.
+ self.context
+ .stack
+ .push(TypedReg::new(self.env.ptr_type(), dst_ptr).into());
+ self.context.stack.push(val.into());
+ self.context
+ .stack
+ .push(TypedReg::new(self.env.ptr_type(), len_reg).into());
+
+ let builtin = self.env.builtins.memory_fill::<M::ABI>()?;
+ FnCall::emit::<M>(
+ &mut self.env,
+ self.masm,
+ &mut self.context,
+ Callee::Builtin(builtin),
+ )?;
+ Ok(())
+ }
+
/// Checks if fuel consumption is enabled and emits a series of instructions
/// that check the current fuel usage by performing a zero-comparison with
/// the number of units stored in `VMStoreContext`.
@@ -1509,7 +1610,7 @@ where
self.masm.checked_uadd(
writable!(addr.reg),
addr.reg,
- Imm::i64(arg.offset as i64),
+ RegImm::i64(arg.offset as i64),
OperandSize::S64,
TrapCode::HEAP_OUT_OF_BOUNDS,
)?;
@@ -1560,7 +1661,7 @@ where
self.masm.checked_uadd(
writable!(addr.reg),
addr.reg,
- Imm::i64(arg.offset as i64),
+ RegImm::i64(arg.offset as i64),
OperandSize::S64,
TrapCode::HEAP_OUT_OF_BOUNDS,
)?;
diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs
index 15458b147a..4cb8c45167 100644
--- a/winch/codegen/src/isa/aarch64/masm.rs
+++ b/winch/codegen/src/isa/aarch64/masm.rs
@@ -17,7 +17,7 @@ use crate::{
reg::{Reg, WritableReg, writable},
},
masm::{
- CalleeKind, DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm,
+ CalleeKind, DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch,
Imm as I, IntCmpKind, IntScratch, LoadKind, MacroAssembler as Masm, MulWideKind,
OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, RoundingMode, SPOffset, Scratch,
ScratchType, ShiftKind, SplatKind, StackSlot, StoreKind, TRUSTED_FLAGS, TrapCode,
@@ -589,7 +589,7 @@ impl Masm for MacroAssembler {
&mut self,
dst: WritableReg,
lhs: Reg,
- rhs: Imm,
+ rhs: RegImm,
size: OperandSize,
trap: TrapCode,
) -> Result<()> {
@@ -600,14 +600,21 @@ impl Masm for MacroAssembler {
// NB: we don't use `Self::add_ir` since we explicitly
// want to emit the add variant which sets overflow
// flags.
- let imm = rhs.unwrap_as_u64();
- match Imm12::maybe_from_u64(imm) {
- Some(imm12) => masm.asm.adds_ir(imm12, lhs, dst, size),
- None => {
- masm.with_scratch::<IntScratch, _>(|masm, scratch| {
- masm.asm.mov_ir(scratch.writable(), rhs, rhs.size());
- masm.asm.adds_rrr(scratch.inner(), lhs, dst, size);
- });
+ match rhs {
+ RegImm::Reg(rm) => {
+ masm.asm.adds_rrr(rm, lhs, dst, size);
+ }
+ RegImm::Imm(rhs) => {
+ let imm = rhs.unwrap_as_u64();
+ match Imm12::maybe_from_u64(imm) {
+ Some(imm12) => masm.asm.adds_ir(imm12, lhs, dst, size),
+ None => {
+ masm.with_scratch::<IntScratch, _>(|masm, scratch| {
+ masm.asm.mov_ir(scratch.writable(), rhs, rhs.size());
+ masm.asm.adds_rrr(scratch.inner(), lhs, dst, size);
+ });
+ }
+ }
}
}
masm.asm.trapif(Cond::Hs, trap);
diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs
index 1b66952871..9fea5d7876 100644
--- a/winch/codegen/src/isa/x64/masm.rs
+++ b/winch/codegen/src/isa/x64/masm.rs
@@ -6,13 +6,13 @@ use super::{
regs::{self, rbp, rsp, scratch_fpr_bitset, scratch_gpr_bitset},
};
use crate::masm::{
- DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm, Imm as I,
- IntCmpKind, IntScratch, LaneSelector, LoadKind, MacroAssembler as Masm, MulWideKind,
- OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, RoundingMode, Scratch, ScratchType,
- ShiftKind, SplatKind, StoreKind, TRUSTED_FLAGS, TrapCode, TruncKind, UNTRUSTED_FLAGS,
- V128AbsKind, V128AddKind, V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind,
- V128MaxKind, V128MinKind, V128MulKind, V128NarrowKind, V128NegKind, V128SubKind, V128TruncKind,
- VectorCompareKind, VectorEqualityKind, Zero,
+ DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm as I, IntCmpKind,
+ IntScratch, LaneSelector, LoadKind, MacroAssembler as Masm, MulWideKind, OperandSize, RegImm,
+ RemKind, ReplaceLaneKind, RmwOp, RoundingMode, Scratch, ScratchType, ShiftKind, SplatKind,
+ StoreKind, TRUSTED_FLAGS, TrapCode, TruncKind, UNTRUSTED_FLAGS, V128AbsKind, V128AddKind,
+ V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind, V128MaxKind, V128MinKind,
+ V128MulKind, V128NarrowKind, V128NegKind, V128SubKind, V128TruncKind, VectorCompareKind,
+ VectorEqualityKind, Zero,
};
use crate::{
Result,
@@ -513,11 +513,11 @@ impl Masm for MacroAssembler {
&mut self,
dst: WritableReg,
lhs: Reg,
- rhs: Imm,
+ rhs: RegImm,
size: OperandSize,
trap: TrapCode,
) -> Result<()> {
- self.add(dst, lhs, RegImm::Imm(rhs), size)?;
+ self.add(dst, lhs, rhs, size)?;
self.asm.trapif(CC::B, trap);
Ok(())
}
diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs
index 566c1b0d58..992a3b4449 100644
--- a/winch/codegen/src/masm.rs
+++ b/winch/codegen/src/masm.rs
@@ -1665,7 +1665,7 @@ pub(crate) trait MacroAssembler {
&mut self,
dst: WritableReg,
lhs: Reg,
- rhs: Imm,
+ rhs: RegImm,
size: OperandSize,
trap: TrapCode,
) -> Result<()>;
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index 7033769d6b..138c10d51a 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -1916,14 +1916,7 @@ where
}
fn visit_memory_fill(&mut self, mem: u32) -> Self::Output {
- let at = self.context.stack.ensure_index_at(3)?;
- let mem = MemoryIndex::from_u32(mem);
-
- let builtin = self.env.builtins.memory_fill::<M::ABI>()?;
- let builtin = self.prepare_builtin_defined_memory_arg(mem, at, builtin)?;
-
- FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?;
- self.context.pop_and_free(self.masm)
+ self.emit_memory_fill(MemoryIndex::from_u32(mem))
}
fn visit_memory_size(&mut self, mem: u32) -> Self::Output {