A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Optimize the implementation of `memory.fill` (#13367)
Alex Crichton40c9406a2ae9
diff --git a/crates/cranelift/src/bounds_checks.rs b/crates/cranelift/src/bounds_checks.rsindex 310549224c..ab9bfc3dc9 100644--- a/crates/cranelift/src/bounds_checks.rs+++ b/crates/cranelift/src/bounds_checks.rs@@ -618,7 +618,15 @@ fn explicit_check_oob_condition_and_compute_addr(/// It is the caller's responsibility to ensure that any necessary bounds and/// overflow checks are emitted, and that the resulting address is never used/// unless they succeed.-fn compute_addr(+///+/// Arguments are:+///+/// * `pos` - where to generate instructions+/// * `heap` - the memory being accessed+/// * `addr_ty` - the host's pointer type+/// * `index` - the raw wasm index, of type `addr_ty`+/// * `offset` - a static offset to add to `index`+pub fn compute_addr(pos: &mut FuncCursor,heap: &HeapData,addr_ty: ir::Type,diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 70b344efc8..201a47d856 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -771,6 +771,25 @@ impl<'module_environment> FuncEnvironment<'module_environment> {}}+ /// Cast the wasm pointer `val`, with type `index_type`, to a host pointer+ /// type.+ ///+ /// Does not perform any in-bounds checks, so `val` must already be+ /// validated to be in-bounds.+ fn unchecked_cast_wasm_addr_to_native_addr(+ &self,+ pos: &mut FuncCursor<'_>,+ val: ir::Value,+ index_type: IndexType,+ ) -> ir::Value {+ match (self.pointer_type(), index_type) {+ (I32, IndexType::I32) | (I64, IndexType::I64) => val,+ (I32, IndexType::I64) => pos.ins().ireduce(I32, val),+ (I64, IndexType::I32) => pos.ins().uextend(I64, val),+ _ => unreachable!(),+ }+ }+/// Convert the target pointer-sized integer `val` into the memory/table's index type.////// For memory, `val` is holding a memory length (or the `-1` `memory.grow`-failed sentinel).@@ -3328,16 +3347,16 @@ impl FuncEnvironment<'_> {))}- pub fn translate_memory_size(- &mut self,- mut pos: FuncCursor<'_>,- index: MemoryIndex,- ) -> WasmResult<ir::Value> {+ /// Loads the size, in bytes, of the memory `index` specified.+ ///+ /// Returns the `ir::Value`, typed as a pointer-width integer, that is the+ /// size in bytes.+ fn memory_size_in_bytes(&mut self, pos: &mut FuncCursor<'_>, index: MemoryIndex) -> ir::Value {let pointer_type = self.pointer_type();let vmctx = self.vmctx(&mut pos.func);let is_shared = self.module.memories[index].shared;let base = pos.ins().global_value(pointer_type, vmctx);- let current_length_in_bytes = match self.module.defined_memory_index(index) {+ match self.module.defined_memory_index(index) {Some(def_index) => {if is_shared {let offset =@@ -3395,7 +3414,15 @@ impl FuncEnvironment<'_> {)}}- };+ }+ }++ pub fn translate_memory_size(+ &mut self,+ mut pos: FuncCursor<'_>,+ index: MemoryIndex,+ ) -> WasmResult<ir::Value> {+ let current_length_in_bytes = self.memory_size_in_bytes(&mut pos, index);let page_size_log2 = i64::from(self.module.memories[index].page_size_log2);let current_length_in_pages = pos.ins().ushr_imm(current_length_in_bytes, page_size_log2);@@ -3454,20 +3481,64 @@ impl FuncEnvironment<'_> {dst: ir::Value,val: ir::Value,len: ir::Value,- ) -> WasmResult<()> {+ ) {+ let pointer_type = self.pointer_type();+ let idx_type = self.memory(memory_index).idx_type;++ // Load the memory size, as `pointer_type`, which is the size of this+ // memory currently in+ // bytes.+ let size_in_bytes = self.memory_size_in_bytes(&mut builder.cursor(), memory_index);++ // Compute the end address of this fill, casted to the `I64` type.+ //+ // Note that addition can't overflow after extending 32-bits to+ // 64-bits, so no need to check for overflow in the 32-bit index case.+ let end64 = match idx_type {+ IndexType::I32 => {+ let dst64 = builder.ins().uextend(I64, dst);+ let len64 = builder.ins().uextend(I64, len);+ builder.ins().iadd(dst64, len64)+ }+ IndexType::I64 => {+ self.uadd_overflow_trap(builder, dst, len, ir::TrapCode::HEAP_OUT_OF_BOUNDS)+ }+ };++ // Cast the host-pointer width to a 64-bit bit width.+ let size_in_bytes64 = match pointer_type {+ I32 => builder.ins().uextend(I64, size_in_bytes),+ I64 => size_in_bytes,+ _ => unreachable!(),+ };++ // This is the actual bounds check that verifies that this `fill`+ // operation is in-bounds. Once control flow gets past here we know+ // that nothing can overflow and everything is in-bounds.+ let inbounds = builder+ .ins()+ .icmp(IntCC::UnsignedLessThanOrEqual, end64, size_in_bytes64);+ self.trapz(builder, inbounds, ir::TrapCode::HEAP_OUT_OF_BOUNDS);++ let memory_fill = self.builtin_functions.memory_fill(&mut builder.func);let mut pos = builder.cursor();- let memory_fill = self.builtin_functions.memory_fill(&mut pos.func);- let dst = self.cast_index_to_i64(&mut pos, dst, self.memory(memory_index).idx_type);- let len = self.cast_index_to_i64(&mut pos, len, self.memory(memory_index).idx_type);- let (memory_vmctx, defined_memory_index) =- self.memory_vmctx_and_defined_index(&mut pos, memory_index);+ let vmctx = self.vmctx_val(&mut pos);- pos.ins().call(- memory_fill,- &[memory_vmctx, defined_memory_index, dst, val, len],- );+ // Compute the actual raw heap address that the `memset` is writing to.+ let heap = self.get_or_create_heap(pos.func, memory_index);+ let heap = self.heaps()[heap].clone();+ let dst_ptr = self.unchecked_cast_wasm_addr_to_native_addr(&mut pos, dst, idx_type);+ let raw_heap_addr =+ crate::bounds_checks::compute_addr(&mut pos, &heap, pointer_type, dst_ptr, 0);- Ok(())+ // Fit the `len` value to `pointer_type`. Note that at this point it's+ // guaranteed inbounds so there's no loss in precision.+ let len_ptr =+ self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, idx_type);++ builder+ .ins()+ .call(memory_fill, &[vmctx, raw_heap_addr, val, len_ptr]);}pub fn translate_memory_init(diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rsindex aa0797de67..23cc213bfd 100644--- a/crates/cranelift/src/lib.rs+++ b/crates/cranelift/src/lib.rs@@ -385,7 +385,6 @@ impl BuiltinFunctionSignatures {AbiParam::new(ir::types::I8)}- #[cfg(feature = "stack-switching")]fn size(&self) -> AbiParam {AbiParam::new(self.pointer_type)}diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rsindex aa64e77bba..1d845bcb49 100644--- a/crates/cranelift/src/translate/code_translator.rs+++ b/crates/cranelift/src/translate/code_translator.rs@@ -1638,7 +1638,7 @@ pub fn translate_operator(let len = environ.stacks.pop1();let val = environ.stacks.pop1();let dest = environ.stacks.pop1();- environ.translate_memory_fill(builder, mem, dest, val, len)?;+ environ.translate_memory_fill(builder, mem, dest, val, len);}Operator::MemoryInit { data_index, mem } => {let mem = MemoryIndex::from_u32(*mem);diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rsindex 015797ffd6..f0cee56a8b 100644--- a/crates/environ/src/builtin.rs+++ b/crates/environ/src/builtin.rs@@ -15,7 +15,7 @@ macro_rules! foreach_builtin_function {// Returns an index for wasm's `memory.copy`memory_copy(vmctx: vmctx, dst_index: u32, dst: u64, src_index: u32, src: u64, len: u64) -> bool;// Returns an index for wasm's `memory.fill` instruction.- memory_fill(vmctx: vmctx, memory: u32, dst: u64, val: u32, len: u64) -> bool;+ memory_fill(vmctx: vmctx, dst: pointer, val: u32, len: size);// Returns an index for wasm's `memory.init` instruction.memory_init(vmctx: vmctx, memory: u32, data: u32, dst: u64, src: u32, len: u32) -> bool;// Returns a value for wasm's `ref.func` instruction.diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rsindex 3be7d97009..e827e8adf2 100644--- a/crates/wasmtime/src/runtime/vm/instance.rs+++ b/crates/wasmtime/src/runtime/vm/instance.rs@@ -1079,35 +1079,6 @@ impl Instance {}}- /// Perform the `memory.fill` operation on a locally defined memory.- ///- /// # Errors- ///- /// Returns a `Trap` error if the memory range is out of bounds.- pub(crate) fn memory_fill(- self: Pin<&mut Self>,- memory_index: DefinedMemoryIndex,- dst: u64,- val: u8,- len: u64,- ) -> Result<(), Trap> {- let memory_index = self.env_module().memory_index(memory_index);- let memory = self.get_memory(memory_index);- let dst = self.validate_inbounds(memory.current_length(), dst, len)?;- let len = usize::try_from(len).unwrap();-- // Bounds and casts are checked above, by this point we know that- // everything is safe.- unsafe {- let dst = memory.base.as_ptr().add(dst);- // FIXME audit whether this is safe in the presence of shared memory- // (https://github.com/bytecodealliance/wasmtime/issues/4203).- ptr::write_bytes(dst, val, len);- }-- Ok(())- }-/// Get the internal storage range of a particular Wasm data segment.pub(crate) fn wasm_data_range(&self, index: DataIndex) -> Range<u32> {match self.env_module().passive_data_map.get(&index) {diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex e7829d34ec..50c7673dd8 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -551,20 +551,23 @@ fn memory_copy(.memory_copy(dst_index, dst, src_index, src, len)}-// Implementation of `memory.fill` for locally defined memories.-fn memory_fill(- store: &mut dyn VMStore,- instance: InstanceId,- memory_index: u32,- dst: u64,+unsafe fn memory_fill(+ _store: &mut dyn VMStore,+ _instance: InstanceId,+ dst: *mut u8,val: u32,- len: u64,-) -> Result<(), Trap> {- let memory_index = DefinedMemoryIndex::from_u32(memory_index);- #[expect(clippy::cast_possible_truncation, reason = "known to truncate here")]- store- .instance_mut(instance)- .memory_fill(memory_index, dst, val as u8, len)+ len: usize,+) {+ // FIXME(#4203): this is known to not be sound in the presence of shared+ // memories.+ unsafe {+ #[expect(+ clippy::cast_possible_truncation,+ reason = "the libcall intentionally takes the raw 32-bit value, \+ and semantically that's intentionally truncated"+ )]+ dst.write_bytes(val as u8, len);+ }}// Implementation of `memory.init`.diff --git a/tests/disas/memory_fill_host32.wat b/tests/disas/memory_fill_host32.watnew file mode 100644index 0000000000..61ec4847ad--- /dev/null+++ b/tests/disas/memory_fill_host32.wat@@ -0,0 +1,172 @@+;;! target = 'pulley32'+;;! test = 'optimize'+;;! flags = '-Wcustom-page-sizes'++(module+ (memory $m32 1)+ (memory $m64 i64 1)++ (memory $m32p1 65536 (pagesize 1))+ (memory $m64p1 i64 65536 (pagesize 1))++ (func $fill32 (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m32+ )++ (func $fill64 (param i64 i64)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m64+ )++ (func $fill32p1 (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m32p1+ )++ (func $fill64p1 (param i64 i64)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m64p1+ )++ (memory $empty 0 0)++ (func $fill-empty (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $empty+ )+)+;; function u0:0(i32 vmctx, i32, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+48+;; gv2 = load.i32 notrap aligned can_move gv0+44+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:5 sig0+;;+;; block0(v0: i32, v1: i32, v2: i32, v3: i32):+;; @003c v6 = load.i32 notrap aligned v0+48+;; @003c v7 = uextend.i64 v2+;; @003c v8 = uextend.i64 v3+;; @003c v9 = iadd v7, v8+;; @003c v10 = uextend.i64 v6+;; @003c v11 = icmp ule v9, v10+;; @003c trapz v11, heap_oob+;; @003c v13 = load.i32 notrap aligned can_move v0+44+;; @003c v14 = iadd v13, v2+;; @0038 v4 = iconst.i32 0+;; @003c call fn0(v0, v14, v4, v3) ; v4 = 0+;; @003f jump block1+;;+;; block1:+;; @003f return+;; }+;;+;; function u0:1(i32 vmctx, i32, i64, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+56+;; gv2 = load.i32 notrap aligned can_move gv0+52+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:5 sig0+;;+;; block0(v0: i32, v1: i32, v2: i64, v3: i64):+;; @0048 v6 = load.i32 notrap aligned v0+56+;; @0048 v7 = uadd_overflow_trap v2, v3, heap_oob+;; @0048 v8 = uextend.i64 v6+;; @0048 v9 = icmp ule v7, v8+;; @0048 trapz v9, heap_oob+;; @0048 v12 = load.i32 notrap aligned can_move v0+52+;; @0048 v11 = ireduce.i32 v2+;; @0048 v13 = iadd v12, v11+;; @0044 v4 = iconst.i32 0+;; @0048 v14 = ireduce.i32 v3+;; @0048 call fn0(v0, v13, v4, v14) ; v4 = 0+;; @004b jump block1+;;+;; block1:+;; @004b return+;; }+;;+;; function u0:2(i32 vmctx, i32, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+64+;; gv2 = load.i32 notrap aligned can_move gv0+60+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:5 sig0+;;+;; block0(v0: i32, v1: i32, v2: i32, v3: i32):+;; @0054 v6 = load.i32 notrap aligned v0+64+;; @0054 v7 = uextend.i64 v2+;; @0054 v8 = uextend.i64 v3+;; @0054 v9 = iadd v7, v8+;; @0054 v10 = uextend.i64 v6+;; @0054 v11 = icmp ule v9, v10+;; @0054 trapz v11, heap_oob+;; @0054 v13 = load.i32 notrap aligned can_move v0+60+;; @0054 v14 = iadd v13, v2+;; @0050 v4 = iconst.i32 0+;; @0054 call fn0(v0, v14, v4, v3) ; v4 = 0+;; @0057 jump block1+;;+;; block1:+;; @0057 return+;; }+;;+;; function u0:3(i32 vmctx, i32, i64, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+72+;; gv2 = load.i32 notrap aligned can_move gv0+68+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:5 sig0+;;+;; block0(v0: i32, v1: i32, v2: i64, v3: i64):+;; @0060 v6 = load.i32 notrap aligned v0+72+;; @0060 v7 = uadd_overflow_trap v2, v3, heap_oob+;; @0060 v8 = uextend.i64 v6+;; @0060 v9 = icmp ule v7, v8+;; @0060 trapz v9, heap_oob+;; @0060 v12 = load.i32 notrap aligned can_move v0+68+;; @0060 v11 = ireduce.i32 v2+;; @0060 v13 = iadd v12, v11+;; @005c v4 = iconst.i32 0+;; @0060 v14 = ireduce.i32 v3+;; @0060 call fn0(v0, v13, v4, v14) ; v4 = 0+;; @0063 jump block1+;;+;; block1:+;; @0063 return+;; }+;;+;; function u0:4(i32 vmctx, i32, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+80+;; gv2 = load.i32 notrap aligned readonly can_move gv0+76+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:5 sig0+;;+;; block0(v0: i32, v1: i32, v2: i32, v3: i32):+;; @006c v6 = load.i32 notrap aligned v0+80+;; @006c v7 = uextend.i64 v2+;; @006c v8 = uextend.i64 v3+;; @006c v9 = iadd v7, v8+;; @006c v10 = uextend.i64 v6+;; @006c v11 = icmp ule v9, v10+;; @006c trapz v11, heap_oob+;; @006c v13 = load.i32 notrap aligned readonly can_move v0+76+;; @006c v14 = iadd v13, v2+;; @0068 v4 = iconst.i32 0+;; @006c call fn0(v0, v14, v4, v3) ; v4 = 0+;; @006f jump block1+;;+;; block1:+;; @006f return+;; }diff --git a/tests/disas/memory_fill_host64.wat b/tests/disas/memory_fill_host64.watnew file mode 100644index 0000000000..af8650e35d--- /dev/null+++ b/tests/disas/memory_fill_host64.wat@@ -0,0 +1,180 @@+;;! target = 'x86_64'+;;! test = 'optimize'+;;! flags = '-Wcustom-page-sizes'++(module+ (memory $m32 1)+ (func $fill32 (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m32+ )++ (memory $m64 i64 1)+ (func $fill64 (param i64 i64)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m64+ )++ (memory $m32p1 65536 (pagesize 1))+ (func $fill32p1 (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m32p1+ )++ (memory $m64p1 i64 65536 (pagesize 1))+ (func $fill64p1 (param i64 i64)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m64p1+ )++ (memory $empty 0 0)+ (func $fill-empty (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $empty+ )+)+;; function u0:0(i64 vmctx, i64, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+96+;; gv5 = load.i64 notrap aligned readonly can_move gv3+88+;; sig0 = (i64 vmctx, i64, i32, i64) tail+;; fn0 = colocated u805306368:5 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32, v3: i32):+;; @003c v6 = load.i64 notrap aligned v0+96+;; @003c v7 = uextend.i64 v2+;; @003c v8 = uextend.i64 v3+;; @003c v9 = iadd v7, v8+;; @003c v10 = icmp ule v9, v6+;; @003c trapz v10, heap_oob+;; @003c v13 = load.i64 notrap aligned readonly can_move v0+88+;; @003c v14 = iadd v13, v7+;; @0038 v4 = iconst.i32 0+;; @003c call fn0(v0, v14, v4, v8) ; v4 = 0+;; @003f jump block1+;;+;; block1:+;; @003f return+;; }+;;+;; function u0:1(i64 vmctx, i64, i64, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+112+;; gv5 = load.i64 notrap aligned can_move gv3+104+;; sig0 = (i64 vmctx, i64, i32, i64) tail+;; fn0 = colocated u805306368:5 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; @0048 v6 = load.i64 notrap aligned v0+112+;; @0048 v7 = uadd_overflow_trap v2, v3, heap_oob+;; @0048 v8 = icmp ule v7, v6+;; @0048 trapz v8, heap_oob+;; @0048 v10 = load.i64 notrap aligned can_move v0+104+;; @0048 v11 = iadd v10, v2+;; @0044 v4 = iconst.i32 0+;; @0048 call fn0(v0, v11, v4, v3) ; v4 = 0+;; @004b jump block1+;;+;; block1:+;; @004b return+;; }+;;+;; function u0:2(i64 vmctx, i64, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+128+;; gv5 = load.i64 notrap aligned readonly can_move gv3+120+;; sig0 = (i64 vmctx, i64, i32, i64) tail+;; fn0 = colocated u805306368:5 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32, v3: i32):+;; @0054 v6 = load.i64 notrap aligned v0+128+;; @0054 v7 = uextend.i64 v2+;; @0054 v8 = uextend.i64 v3+;; @0054 v9 = iadd v7, v8+;; @0054 v10 = icmp ule v9, v6+;; @0054 trapz v10, heap_oob+;; @0054 v13 = load.i64 notrap aligned readonly can_move v0+120+;; @0054 v14 = iadd v13, v7+;; @0050 v4 = iconst.i32 0+;; @0054 call fn0(v0, v14, v4, v8) ; v4 = 0+;; @0057 jump block1+;;+;; block1:+;; @0057 return+;; }+;;+;; function u0:3(i64 vmctx, i64, i64, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+144+;; gv5 = load.i64 notrap aligned can_move gv3+136+;; sig0 = (i64 vmctx, i64, i32, i64) tail+;; fn0 = colocated u805306368:5 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64):+;; @0060 v6 = load.i64 notrap aligned v0+144+;; @0060 v7 = uadd_overflow_trap v2, v3, heap_oob+;; @0060 v8 = icmp ule v7, v6+;; @0060 trapz v8, heap_oob+;; @0060 v10 = load.i64 notrap aligned can_move v0+136+;; @0060 v11 = iadd v10, v2+;; @005c v4 = iconst.i32 0+;; @0060 call fn0(v0, v11, v4, v3) ; v4 = 0+;; @0063 jump block1+;;+;; block1:+;; @0063 return+;; }+;;+;; function u0:4(i64 vmctx, i64, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+160+;; gv5 = load.i64 notrap aligned readonly can_move gv3+152+;; sig0 = (i64 vmctx, i64, i32, i64) tail+;; fn0 = colocated u805306368:5 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32, v3: i32):+;; @006c v6 = load.i64 notrap aligned v0+160+;; @006c v7 = uextend.i64 v2+;; @006c v8 = uextend.i64 v3+;; @006c v9 = iadd v7, v8+;; @006c v10 = icmp ule v9, v6+;; @006c trapz v10, heap_oob+;; @006c v13 = load.i64 notrap aligned readonly can_move v0+152+;; @006c v14 = iadd v13, v7+;; @0068 v4 = iconst.i32 0+;; @006c call fn0(v0, v14, v4, v8) ; v4 = 0+;; @006f jump block1+;;+;; block1:+;; @006f return+;; }diff --git a/tests/misc_testsuite/memory_fill.wast b/tests/misc_testsuite/memory_fill.wastnew file mode 100644index 0000000000..64f1b76810--- /dev/null+++ b/tests/misc_testsuite/memory_fill.wast@@ -0,0 +1,97 @@+;;! memory64 = true+;;! bulk_memory = true+;;! multi_memory = true+;;! custom_page_sizes = true++(module+ (memory $m32 1)+ (func (export "fill32") (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m32+ )++ (memory $m64 i64 1)+ (func (export "fill64") (param i64 i64)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m64+ )++ (memory $m32p1 65536 (pagesize 1))+ (func (export "fill32p1") (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m32p1+ )++ (memory $m64p1 i64 65536 (pagesize 1))+ (func (export "fill64p1") (param i64 i64)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $m64p1+ )++ (memory $empty 0 0)+ (func (export "fill-empty") (param i32 i32)+ local.get 0+ i32.const 0+ local.get 1+ memory.fill $empty+ )+)++(assert_return (invoke "fill32" (i32.const 0) (i32.const 16)))+(assert_return (invoke "fill32" (i32.const 0) (i32.const 0)))+(assert_return (invoke "fill32" (i32.const 65536) (i32.const 0)))+(assert_trap (invoke "fill32" (i32.const 65536) (i32.const 1)) "out of bounds")+(assert_return (invoke "fill32" (i32.const 65535) (i32.const 1)))+(assert_trap (invoke "fill32" (i32.const 65535) (i32.const 2)) "out of bounds")+(assert_trap (invoke "fill32" (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "fill32" (i32.const 1) (i32.const -1)) "out of bounds")+(assert_trap (invoke "fill32" (i32.const -1) (i32.const 0)) "out of bounds")+(assert_trap (invoke "fill32" (i32.const -1) (i32.const 1)) "out of bounds")++(assert_return (invoke "fill64" (i64.const 0) (i64.const 16)))+(assert_return (invoke "fill64" (i64.const 0) (i64.const 0)))+(assert_return (invoke "fill64" (i64.const 65536) (i64.const 0)))+(assert_trap (invoke "fill64" (i64.const 65536) (i64.const 1)) "out of bounds")+(assert_return (invoke "fill64" (i64.const 65535) (i64.const 1)))+(assert_trap (invoke "fill64" (i64.const 65535) (i64.const 2)) "out of bounds")+(assert_trap (invoke "fill64" (i64.const 0) (i64.const -1)) "out of bounds")+(assert_trap (invoke "fill64" (i64.const 1) (i64.const -1)) "out of bounds")+(assert_trap (invoke "fill64" (i64.const -1) (i64.const 0)) "out of bounds")+(assert_trap (invoke "fill64" (i64.const -1) (i64.const 1)) "out of bounds")++(assert_return (invoke "fill32p1" (i32.const 0) (i32.const 16)))+(assert_return (invoke "fill32p1" (i32.const 0) (i32.const 0)))+(assert_return (invoke "fill32p1" (i32.const 65536) (i32.const 0)))+(assert_trap (invoke "fill32p1" (i32.const 65536) (i32.const 1)) "out of bounds")+(assert_return (invoke "fill32p1" (i32.const 65535) (i32.const 1)))+(assert_trap (invoke "fill32p1" (i32.const 65535) (i32.const 2)) "out of bounds")+(assert_trap (invoke "fill32p1" (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "fill32p1" (i32.const 1) (i32.const -1)) "out of bounds")+(assert_trap (invoke "fill32p1" (i32.const -1) (i32.const 0)) "out of bounds")+(assert_trap (invoke "fill32p1" (i32.const -1) (i32.const 1)) "out of bounds")++(assert_return (invoke "fill64p1" (i64.const 0) (i64.const 16)))+(assert_return (invoke "fill64p1" (i64.const 0) (i64.const 0)))+(assert_return (invoke "fill64p1" (i64.const 65536) (i64.const 0)))+(assert_trap (invoke "fill64p1" (i64.const 65536) (i64.const 1)) "out of bounds")+(assert_return (invoke "fill64p1" (i64.const 65535) (i64.const 1)))+(assert_trap (invoke "fill64p1" (i64.const 0) (i64.const -1)) "out of bounds")+(assert_trap (invoke "fill64p1" (i64.const 1) (i64.const -1)) "out of bounds")+(assert_trap (invoke "fill64p1" (i64.const 65535) (i64.const 2)) "out of bounds")+(assert_trap (invoke "fill64p1" (i64.const -1) (i64.const 0)) "out of bounds")+(assert_trap (invoke "fill64p1" (i64.const -1) (i64.const 1)) "out of bounds")++(assert_return (invoke "fill-empty" (i32.const 0) (i32.const 0)))+(assert_trap (invoke "fill-empty" (i32.const 0) (i32.const 1)) "out of bounds")+(assert_trap (invoke "fill-empty" (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "fill-empty" (i32.const 1) (i32.const -1)) "out of bounds")+(assert_trap (invoke "fill-empty" (i32.const -1) (i32.const 0)) "out of bounds")+(assert_trap (invoke "fill-empty" (i32.const -1) (i32.const 1)) "out of bounds")diff --git a/winch/codegen/src/codegen/bounds.rs b/winch/codegen/src/codegen/bounds.rsindex 00cd6281d6..1c7d151ed9 100644--- a/winch/codegen/src/codegen/bounds.rs+++ b/winch/codegen/src/codegen/bounds.rs@@ -7,7 +7,7 @@ use crate::{abi::vmctx,codegen::{CodeGenContext, Emission},isa::reg::{Reg, writable},- masm::{Imm, IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm, TrapCode},+ masm::{IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm, TrapCode},stack::TypedReg,};use wasmtime_environ::WasmValType;@@ -140,7 +140,7 @@ pub(crate) fn ensure_index_and_offset<M: MacroAssembler>(masm.checked_uadd(writable!(index.as_typed_reg().into()),index.as_typed_reg().into(),- Imm::i64(offset as i64),+ RegImm::i64(offset as i64),heap_ty_size,TrapCode::HEAP_OUT_OF_BOUNDS,)?;diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex a42776c3b4..ec4c460225 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -829,7 +829,7 @@ whereself.masm.checked_uadd(writable!(index_offset_and_access_size),index_offset_and_access_size,- Imm::i64(offset_with_access_size as i64),+ RegImm::i64(offset_with_access_size as i64),ptr_size,TrapCode::HEAP_OUT_OF_BOUNDS,)?;@@ -1096,9 +1096,7 @@ where}/// Retrieves the size of the memory, pushing the result to the value stack.- pub fn emit_compute_memory_size(&mut self, heap_data: &HeapData) -> Result<()> {- let size_reg = self.context.any_gpr(self.masm)?;-+ fn load_memory_length(&mut self, heap_data: &HeapData, size_reg: Reg) -> Result<()> {self.masm.with_scratch::<IntScratch, _>(|masm, scratch| {let base = if let Some(offset) = heap_data.import_from {masm.load_ptr(masm.address_at_vmctx(offset)?, scratch.writable())?;@@ -1110,6 +1108,14 @@ wherelet size_addr = masm.address_at_reg(base, heap_data.current_length_offset)?;masm.load_ptr(size_addr, writable!(size_reg))})?;+ Ok(())+ }++ /// Retrieves the size of the memory, pushing the result to the value stack.+ pub fn emit_compute_memory_size(&mut self, heap_data: &HeapData) -> Result<()> {+ let size_reg = self.context.any_gpr(self.masm)?;+ self.load_memory_length(heap_data, size_reg)?;+// Emit a shift to get the size in pages rather than in bytes.let dst = TypedReg::new(heap_data.index_type(), size_reg);let pow = heap_data.memory.page_size_log2;@@ -1124,6 +1130,101 @@ whereOk(())}+ /// Emit the `memory.fill` operation.+ pub fn emit_memory_fill(&mut self, mem: MemoryIndex) -> Result<()> {+ let heap = self.env.resolve_heap(mem);+ let ptr_size: OperandSize = self.env.ptr_type().try_into()?;+ let idx_size: OperandSize = heap.index_type().try_into()?;++ // The wasm stack at this point is `[dst, val, len]`.+ let len = self.context.pop_to_reg(self.masm, None)?;+ let val = self.context.pop_to_reg(self.masm, None)?;+ let dst = self.context.pop_to_reg(self.masm, None)?;++ // Compute `end = dst + len` trapping on overflow. For an `i32` index+ // type the operands are zero-extended to 64-bit so overflow is+ // impossible.+ let end = self.context.any_gpr(self.masm)?;+ match idx_size {+ OperandSize::S32 => {+ self.masm+ .extend(writable!(end), dst.reg, Extend::<Zero>::I64Extend32.into())?;+ self.masm.add_uextend(+ writable!(end),+ end,+ len.reg,+ OperandSize::S32,+ OperandSize::S64,+ )?;+ }+ OperandSize::S64 => {+ self.masm+ .mov(writable!(end), dst.reg.into(), OperandSize::S64)?;+ self.masm.checked_uadd(+ writable!(end),+ end,+ len.reg.into(),+ OperandSize::S64,+ TrapCode::HEAP_OUT_OF_BOUNDS,+ )?;+ }+ _ => unreachable!(),+ }++ // Load the current size in bytes of the memory.+ let size_in_bytes = self.context.any_gpr(self.masm)?;+ self.load_memory_length(&heap, size_in_bytes)?;++ // Trap if `end > size_in_bytes`.+ assert!(ptr_size == OperandSize::S64);+ self.masm.cmp(end, size_in_bytes.into(), ptr_size)?;+ self.masm+ .trapif(IntCmpKind::GtU, TrapCode::HEAP_OUT_OF_BOUNDS)?;+ self.context.free_reg(end);+ self.context.free_reg(size_in_bytes);++ // Compute `dst_ptr = memory_base + dst`.+ let dst_ptr = self.context.any_gpr(self.masm)?;+ bounds::load_heap_addr_unchecked(+ self.masm,+ &heap,+ Index::from_typed_reg(dst),+ ImmOffset::from_u32(0),+ dst_ptr,+ ptr_size,+ )?;+ self.context.free_reg(dst.reg);++ // The libcall takes the length as a host-pointer-sized integer, so+ // zero-extend if the wasm index type is smaller.+ let len_reg = len.reg;+ if idx_size == OperandSize::S32 && ptr_size == OperandSize::S64 {+ self.masm.extend(+ writable!(len_reg),+ len_reg,+ Extend::<Zero>::I64Extend32.into(),+ )?;+ }++ // Set up the call arguments: `[dst_ptr, val, len]`.+ self.context+ .stack+ .push(TypedReg::new(self.env.ptr_type(), dst_ptr).into());+ self.context.stack.push(val.into());+ self.context+ .stack+ .push(TypedReg::new(self.env.ptr_type(), len_reg).into());++ let builtin = self.env.builtins.memory_fill::<M::ABI>()?;+ FnCall::emit::<M>(+ &mut self.env,+ self.masm,+ &mut self.context,+ Callee::Builtin(builtin),+ )?;+ Ok(())+ }+/// Checks if fuel consumption is enabled and emits a series of instructions/// that check the current fuel usage by performing a zero-comparison with/// the number of units stored in `VMStoreContext`.@@ -1509,7 +1610,7 @@ whereself.masm.checked_uadd(writable!(addr.reg),addr.reg,- Imm::i64(arg.offset as i64),+ RegImm::i64(arg.offset as i64),OperandSize::S64,TrapCode::HEAP_OUT_OF_BOUNDS,)?;@@ -1560,7 +1661,7 @@ whereself.masm.checked_uadd(writable!(addr.reg),addr.reg,- Imm::i64(arg.offset as i64),+ RegImm::i64(arg.offset as i64),OperandSize::S64,TrapCode::HEAP_OUT_OF_BOUNDS,)?;diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rsindex 15458b147a..4cb8c45167 100644--- a/winch/codegen/src/isa/aarch64/masm.rs+++ b/winch/codegen/src/isa/aarch64/masm.rs@@ -17,7 +17,7 @@ use crate::{reg::{Reg, WritableReg, writable},},masm::{- CalleeKind, DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm,+ CalleeKind, DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch,Imm as I, IntCmpKind, IntScratch, LoadKind, MacroAssembler as Masm, MulWideKind,OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, RoundingMode, SPOffset, Scratch,ScratchType, ShiftKind, SplatKind, StackSlot, StoreKind, TRUSTED_FLAGS, TrapCode,@@ -589,7 +589,7 @@ impl Masm for MacroAssembler {&mut self,dst: WritableReg,lhs: Reg,- rhs: Imm,+ rhs: RegImm,size: OperandSize,trap: TrapCode,) -> Result<()> {@@ -600,14 +600,21 @@ impl Masm for MacroAssembler {// NB: we don't use `Self::add_ir` since we explicitly// want to emit the add variant which sets overflow// flags.- let imm = rhs.unwrap_as_u64();- match Imm12::maybe_from_u64(imm) {- Some(imm12) => masm.asm.adds_ir(imm12, lhs, dst, size),- None => {- masm.with_scratch::<IntScratch, _>(|masm, scratch| {- masm.asm.mov_ir(scratch.writable(), rhs, rhs.size());- masm.asm.adds_rrr(scratch.inner(), lhs, dst, size);- });+ match rhs {+ RegImm::Reg(rm) => {+ masm.asm.adds_rrr(rm, lhs, dst, size);+ }+ RegImm::Imm(rhs) => {+ let imm = rhs.unwrap_as_u64();+ match Imm12::maybe_from_u64(imm) {+ Some(imm12) => masm.asm.adds_ir(imm12, lhs, dst, size),+ None => {+ masm.with_scratch::<IntScratch, _>(|masm, scratch| {+ masm.asm.mov_ir(scratch.writable(), rhs, rhs.size());+ masm.asm.adds_rrr(scratch.inner(), lhs, dst, size);+ });+ }+ }}}masm.asm.trapif(Cond::Hs, trap);diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rsindex 1b66952871..9fea5d7876 100644--- a/winch/codegen/src/isa/x64/masm.rs+++ b/winch/codegen/src/isa/x64/masm.rs@@ -6,13 +6,13 @@ use super::{regs::{self, rbp, rsp, scratch_fpr_bitset, scratch_gpr_bitset},};use crate::masm::{- DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm, Imm as I,- IntCmpKind, IntScratch, LaneSelector, LoadKind, MacroAssembler as Masm, MulWideKind,- OperandSize, RegImm, RemKind, ReplaceLaneKind, RmwOp, RoundingMode, Scratch, ScratchType,- ShiftKind, SplatKind, StoreKind, TRUSTED_FLAGS, TrapCode, TruncKind, UNTRUSTED_FLAGS,- V128AbsKind, V128AddKind, V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind,- V128MaxKind, V128MinKind, V128MulKind, V128NarrowKind, V128NegKind, V128SubKind, V128TruncKind,- VectorCompareKind, VectorEqualityKind, Zero,+ DivKind, Extend, ExtendKind, ExtractLaneKind, FloatCmpKind, FloatScratch, Imm as I, IntCmpKind,+ IntScratch, LaneSelector, LoadKind, MacroAssembler as Masm, MulWideKind, OperandSize, RegImm,+ RemKind, ReplaceLaneKind, RmwOp, RoundingMode, Scratch, ScratchType, ShiftKind, SplatKind,+ StoreKind, TRUSTED_FLAGS, TrapCode, TruncKind, UNTRUSTED_FLAGS, V128AbsKind, V128AddKind,+ V128ConvertKind, V128ExtAddKind, V128ExtMulKind, V128ExtendKind, V128MaxKind, V128MinKind,+ V128MulKind, V128NarrowKind, V128NegKind, V128SubKind, V128TruncKind, VectorCompareKind,+ VectorEqualityKind, Zero,};use crate::{Result,@@ -513,11 +513,11 @@ impl Masm for MacroAssembler {&mut self,dst: WritableReg,lhs: Reg,- rhs: Imm,+ rhs: RegImm,size: OperandSize,trap: TrapCode,) -> Result<()> {- self.add(dst, lhs, RegImm::Imm(rhs), size)?;+ self.add(dst, lhs, rhs, size)?;self.asm.trapif(CC::B, trap);Ok(())}diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rsindex 566c1b0d58..992a3b4449 100644--- a/winch/codegen/src/masm.rs+++ b/winch/codegen/src/masm.rs@@ -1665,7 +1665,7 @@ pub(crate) trait MacroAssembler {&mut self,dst: WritableReg,lhs: Reg,- rhs: Imm,+ rhs: RegImm,size: OperandSize,trap: TrapCode,) -> Result<()>;diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rsindex 7033769d6b..138c10d51a 100644--- a/winch/codegen/src/visitor.rs+++ b/winch/codegen/src/visitor.rs@@ -1916,14 +1916,7 @@ where}fn visit_memory_fill(&mut self, mem: u32) -> Self::Output {- let at = self.context.stack.ensure_index_at(3)?;- let mem = MemoryIndex::from_u32(mem);-- let builtin = self.env.builtins.memory_fill::<M::ABI>()?;- let builtin = self.prepare_builtin_defined_memory_arg(mem, at, builtin)?;-- FnCall::emit::<M>(&mut self.env, self.masm, &mut self.context, builtin)?;- self.context.pop_and_free(self.masm)+ self.emit_memory_fill(MemoryIndex::from_u32(mem))}fn visit_memory_size(&mut self, mem: u32) -> Self::Output {