A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Optimize `memory.copy` implementation (#13368)
Alex Crichton16b5f97a4223
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 201a47d856..37d2076d2b 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -3439,69 +3439,44 @@ impl FuncEnvironment<'_> {))}- pub fn translate_memory_copy(- &mut self,- builder: &mut FunctionBuilder<'_>,- src_index: MemoryIndex,- dst_index: MemoryIndex,- dst: ir::Value,- src: ir::Value,- len: ir::Value,- ) -> WasmResult<()> {- let mut pos = builder.cursor();- let vmctx = self.vmctx_val(&mut pos);-- let memory_copy = self.builtin_functions.memory_copy(&mut pos.func);- let dst = self.cast_index_to_i64(&mut pos, dst, self.memory(dst_index).idx_type);- let src = self.cast_index_to_i64(&mut pos, src, self.memory(src_index).idx_type);- // The length is 32-bit if either memory is 32-bit, but if they're both- // 64-bit then it's 64-bit. Our intrinsic takes a 64-bit length for- // compatibility across all memories, so make sure that it's cast- // correctly here (this is a bit special so no generic helper unlike for- // `dst`/`src` above)- let len = if index_type_to_ir_type(self.memory(dst_index).idx_type) == I64- && index_type_to_ir_type(self.memory(src_index).idx_type) == I64- {- len- } else {- pos.ins().uextend(I64, len)- };- let src_index = pos.ins().iconst(I32, i64::from(src_index.as_u32()));- let dst_index = pos.ins().iconst(I32, i64::from(dst_index.as_u32()));- pos.ins()- .call(memory_copy, &[vmctx, dst_index, dst, src_index, src, len]);-- Ok(())- }-- pub fn translate_memory_fill(+ /// Performs a bounds check and raises a trap if `ptr+len` is out-of-bounds+ /// for `len`.+ ///+ /// Returns the raw host-native heap address of `ptr`.+ fn bounds_check_for_memory_intrinsic(&mut self,builder: &mut FunctionBuilder<'_>,memory_index: MemoryIndex,- dst: ir::Value,- val: ir::Value,+ ptr: ir::Value,len: ir::Value,- ) {+ ) -> ir::Value {let pointer_type = self.pointer_type();let idx_type = self.memory(memory_index).idx_type;+ let idx_clif_type = match idx_type {+ IndexType::I32 => I32,+ IndexType::I64 => I64,+ };+ debug_assert_eq!(builder.func.dfg.value_type(ptr), idx_clif_type);+ debug_assert_eq!(builder.func.dfg.value_type(len), idx_clif_type);+// Load the memory size, as `pointer_type`, which is the size of this// memory currently in// bytes.let size_in_bytes = self.memory_size_in_bytes(&mut builder.cursor(), memory_index);- // Compute the end address of this fill, casted to the `I64` type.+ // Compute the end address of this operation, casted to the `I64` type.//// Note that addition can't overflow after extending 32-bits to// 64-bits, so no need to check for overflow in the 32-bit index case.let end64 = match idx_type {IndexType::I32 => {- let dst64 = builder.ins().uextend(I64, dst);+ let ptr64 = builder.ins().uextend(I64, ptr);let len64 = builder.ins().uextend(I64, len);- builder.ins().iadd(dst64, len64)+ builder.ins().iadd(ptr64, len64)}IndexType::I64 => {- self.uadd_overflow_trap(builder, dst, len, ir::TrapCode::HEAP_OUT_OF_BOUNDS)+ self.uadd_overflow_trap(builder, ptr, len, ir::TrapCode::HEAP_OUT_OF_BOUNDS)}};@@ -3512,24 +3487,90 @@ impl FuncEnvironment<'_> {_ => unreachable!(),};- // This is the actual bounds check that verifies that this `fill`- // operation is in-bounds. Once control flow gets past here we know- // that nothing can overflow and everything is in-bounds.+ // This is the actual bounds check that verifies that this operation is+ // in-bounds. Once control flow gets past here we know that nothing can+ // overflow and everything is in-bounds.let inbounds = builder.ins().icmp(IntCC::UnsignedLessThanOrEqual, end64, size_in_bytes64);self.trapz(builder, inbounds, ir::TrapCode::HEAP_OUT_OF_BOUNDS);+ // Compute the actual raw heap address to return+ let mut pos = builder.cursor();+ let heap = self.get_or_create_heap(pos.func, memory_index);+ let heap = self.heaps()[heap].clone();+ let ptr = self.unchecked_cast_wasm_addr_to_native_addr(&mut pos, ptr, idx_type);+ crate::bounds_checks::compute_addr(&mut pos, &heap, pointer_type, ptr, 0)+ }++ pub fn translate_memory_copy(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ src_index: MemoryIndex,+ dst_index: MemoryIndex,+ dst: ir::Value,+ src: ir::Value,+ len: ir::Value,+ ) -> WasmResult<()> {+ let src_idx_ty = self.memory(src_index).idx_type;+ let dst_idx_ty = self.memory(dst_index).idx_type;++ // The length is 32-bit if either memory is 32-bit, but if they're both+ // 64-bit then it's 64-bit.+ let len_idx_ty = match (src_idx_ty, dst_idx_ty) {+ (IndexType::I32, _) | (_, IndexType::I32) => IndexType::I32,+ (IndexType::I64, IndexType::I64) => IndexType::I64,+ };++ let mut pos = builder.cursor();+ let vmctx = self.vmctx_val(&mut pos);+ let memory_copy = self.builtin_functions.memory_copy(&mut pos.func);++ let src_len = if src_idx_ty == len_idx_ty {+ len+ } else {+ assert_eq!(src_idx_ty, IndexType::I64);+ builder.ins().uextend(I64, len)+ };+ let dst_len = if dst_idx_ty == len_idx_ty {+ len+ } else {+ assert_eq!(dst_idx_ty, IndexType::I64);+ builder.ins().uextend(I64, len)+ };++ // Perform a bounds check for the src/dst memories and compute the raw+ // heap addresses at the same time.+ let dst_raw_addr = self.bounds_check_for_memory_intrinsic(builder, dst_index, dst, dst_len);+ let src_raw_addr = self.bounds_check_for_memory_intrinsic(builder, src_index, src, src_len);++ // Fit the `len` value to `pointer_type`. Note that at this point it's+ // guaranteed inbounds so there's no loss in precision.+ let len_ptr =+ self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, len_idx_ty);++ builder+ .ins()+ .call(memory_copy, &[vmctx, dst_raw_addr, src_raw_addr, len_ptr]);++ Ok(())+ }++ pub fn translate_memory_fill(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ memory_index: MemoryIndex,+ dst: ir::Value,+ val: ir::Value,+ len: ir::Value,+ ) {+ let idx_type = self.memory(memory_index).idx_type;let memory_fill = self.builtin_functions.memory_fill(&mut builder.func);let mut pos = builder.cursor();let vmctx = self.vmctx_val(&mut pos);- // Compute the actual raw heap address that the `memset` is writing to.- let heap = self.get_or_create_heap(pos.func, memory_index);- let heap = self.heaps()[heap].clone();- let dst_ptr = self.unchecked_cast_wasm_addr_to_native_addr(&mut pos, dst, idx_type);- let raw_heap_addr =- crate::bounds_checks::compute_addr(&mut pos, &heap, pointer_type, dst_ptr, 0);+ // Bounds check `dst+len` and convert it to a raw heap address.+ let raw_heap_addr = self.bounds_check_for_memory_intrinsic(builder, memory_index, dst, len);// Fit the `len` value to `pointer_type`. Note that at this point it's// guaranteed inbounds so there's no loss in precision.diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rsindex f0cee56a8b..8d4906657c 100644--- a/crates/environ/src/builtin.rs+++ b/crates/environ/src/builtin.rs@@ -13,7 +13,7 @@ macro_rules! foreach_builtin_function {// Returns an index for wasm's `elem.drop`.elem_drop(vmctx: vmctx, elem: u32) -> bool;// Returns an index for wasm's `memory.copy`- memory_copy(vmctx: vmctx, dst_index: u32, dst: u64, src_index: u32, src: u64, len: u64) -> bool;+ memory_copy(vmctx: vmctx, dst: pointer, src: pointer, len: size);// Returns an index for wasm's `memory.fill` instruction.memory_fill(vmctx: vmctx, dst: pointer, val: u32, len: size);// Returns an index for wasm's `memory.init` instruction.diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rsindex e827e8adf2..d9ec7c14b6 100644--- a/crates/wasmtime/src/runtime/vm/instance.rs+++ b/crates/wasmtime/src/runtime/vm/instance.rs@@ -1030,42 +1030,6 @@ impl Instance {}}- /// Do a `memory.copy`- ///- /// # Errors- ///- /// Returns a `Trap` error when the source or destination ranges are out of- /// bounds.- pub(crate) fn memory_copy(- self: Pin<&mut Self>,- dst_index: MemoryIndex,- dst: u64,- src_index: MemoryIndex,- src: u64,- len: u64,- ) -> Result<(), Trap> {- // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy-- let src_mem = self.get_memory(src_index);- let dst_mem = self.get_memory(dst_index);-- let src = self.validate_inbounds(src_mem.current_length(), src, len)?;- let dst = self.validate_inbounds(dst_mem.current_length(), dst, len)?;- let len = usize::try_from(len).unwrap();-- // Bounds and casts are checked above, by this point we know that- // everything is safe.- unsafe {- let dst = dst_mem.base.as_ptr().add(dst);- let src = src_mem.base.as_ptr().add(src);- // FIXME audit whether this is safe in the presence of shared memory- // (https://github.com/bytecodealliance/wasmtime/issues/4203).- ptr::copy(src, dst, len);- }-- Ok(())- }-fn validate_inbounds(&self, max: usize, ptr: u64, len: u64) -> Result<usize, Trap> {let oob = || Trap::MemoryOutOfBounds;let end = ptrdiff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rsindex 50c7673dd8..ff75e49f6e 100644--- a/crates/wasmtime/src/runtime/vm/libcalls.rs+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs@@ -535,20 +535,17 @@ fn elem_drop(store: &mut dyn VMStore, instance: InstanceId, elem_index: u32) ->}// Implementation of `memory.copy`.-fn memory_copy(- store: &mut dyn VMStore,- instance: InstanceId,- dst_index: u32,- dst: u64,- src_index: u32,- src: u64,- len: u64,-) -> Result<(), Trap> {- let src_index = MemoryIndex::from_u32(src_index);- let dst_index = MemoryIndex::from_u32(dst_index);- store- .instance_mut(instance)- .memory_copy(dst_index, dst, src_index, src, len)+unsafe fn memory_copy(+ _store: &mut dyn VMStore,+ _instance: InstanceId,+ dst: *mut u8,+ src: *mut u8,+ len: usize,+) {+ let src = src.cast_const();+ // FIXME(#4203): this is known to not be sound in the presence of shared+ // memories.+ unsafe { src.copy_to(dst, len) }}unsafe fn memory_fill(diff --git a/tests/disas/memory_copy_host32.wat b/tests/disas/memory_copy_host32.watnew file mode 100644index 0000000000..ef6002493b--- /dev/null+++ b/tests/disas/memory_copy_host32.wat@@ -0,0 +1,244 @@+;;! target = 'pulley32'+;;! test = 'optimize'++(module+ (memory $m32_a 1)+ (memory $m32_b 1)+ (memory $m64_a i64 1)+ (memory $m64_b i64 1)++ (func $m32_to_same (param i32 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_a $m32_a+ )++ (func $m32_to_m32 (param i32 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_b $m32_a+ )++ (func $m32_to_m64 (param i64 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_a $m32_a+ )++ (func $m64_to_same (param i64 i64 i64)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_a $m64_a+ )++ (func $m64_to_m64 (param i64 i64 i64)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_b $m64_a+ )++ (func $m64_to_m32 (param i32 i64 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_a $m64_a+ )+)+;; function u0:0(i32 vmctx, i32, i32, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+44+;; gv2 = load.i32 notrap aligned can_move gv0+40+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:4 sig0+;;+;; block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32):+;; @0042 v7 = load.i32 notrap aligned v0+44+;; @0042 v8 = uextend.i64 v2+;; @0042 v9 = uextend.i64 v4+;; @0042 v10 = iadd v8, v9+;; @0042 v11 = uextend.i64 v7+;; @0042 v12 = icmp ule v10, v11+;; @0042 trapz v12, heap_oob+;; @0042 v13 = load.i32 notrap aligned can_move v0+40+;; @0042 v17 = uextend.i64 v3+;; @0042 v19 = iadd v17, v9+;; @0042 v21 = icmp ule v19, v11+;; @0042 trapz v21, heap_oob+;; @0042 v14 = iadd v13, v2+;; @0042 v23 = iadd v13, v3+;; @0042 call fn0(v0, v14, v23, v4)+;; @0046 jump block1+;;+;; block1:+;; @0046 return+;; }+;;+;; function u0:1(i32 vmctx, i32, i32, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+44+;; gv2 = load.i32 notrap aligned can_move gv0+40+;; gv3 = load.i32 notrap aligned gv0+52+;; gv4 = load.i32 notrap aligned can_move gv0+48+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:4 sig0+;;+;; block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32):+;; @004f v7 = load.i32 notrap aligned v0+52+;; @004f v8 = uextend.i64 v2+;; @004f v9 = uextend.i64 v4+;; @004f v10 = iadd v8, v9+;; @004f v11 = uextend.i64 v7+;; @004f v12 = icmp ule v10, v11+;; @004f trapz v12, heap_oob+;; @004f v13 = load.i32 notrap aligned can_move v0+48+;; @004f v16 = load.i32 notrap aligned v0+44+;; @004f v17 = uextend.i64 v3+;; @004f v19 = iadd v17, v9+;; @004f v20 = uextend.i64 v16+;; @004f v21 = icmp ule v19, v20+;; @004f trapz v21, heap_oob+;; @004f v22 = load.i32 notrap aligned can_move v0+40+;; @004f v14 = iadd v13, v2+;; @004f v23 = iadd v22, v3+;; @004f call fn0(v0, v14, v23, v4)+;; @0053 jump block1+;;+;; block1:+;; @0053 return+;; }+;;+;; function u0:2(i32 vmctx, i32, i64, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+44+;; gv2 = load.i32 notrap aligned can_move gv0+40+;; gv3 = load.i32 notrap aligned gv0+60+;; gv4 = load.i32 notrap aligned can_move gv0+56+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:4 sig0+;;+;; block0(v0: i32, v1: i32, v2: i64, v3: i32, v4: i32):+;; @005c v8 = load.i32 notrap aligned v0+60+;; @005c v6 = uextend.i64 v4+;; @005c v9 = uadd_overflow_trap v2, v6, heap_oob+;; @005c v10 = uextend.i64 v8+;; @005c v11 = icmp ule v9, v10+;; @005c trapz v11, heap_oob+;; @005c v13 = load.i32 notrap aligned can_move v0+56+;; @005c v16 = load.i32 notrap aligned v0+44+;; @005c v17 = uextend.i64 v3+;; @005c v19 = iadd v17, v6+;; @005c v20 = uextend.i64 v16+;; @005c v21 = icmp ule v19, v20+;; @005c trapz v21, heap_oob+;; @005c v22 = load.i32 notrap aligned can_move v0+40+;; @005c v12 = ireduce.i32 v2+;; @005c v14 = iadd v13, v12+;; @005c v23 = iadd v22, v3+;; @005c call fn0(v0, v14, v23, v4)+;; @0060 jump block1+;;+;; block1:+;; @0060 return+;; }+;;+;; function u0:3(i32 vmctx, i32, i64, i64, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+60+;; gv2 = load.i32 notrap aligned can_move gv0+56+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:4 sig0+;;+;; block0(v0: i32, v1: i32, v2: i64, v3: i64, v4: i64):+;; @0069 v7 = load.i32 notrap aligned v0+60+;; @0069 v8 = uadd_overflow_trap v2, v4, heap_oob+;; @0069 v9 = uextend.i64 v7+;; @0069 v10 = icmp ule v8, v9+;; @0069 trapz v10, heap_oob+;; @0069 v12 = load.i32 notrap aligned can_move v0+56+;; @0069 v16 = uadd_overflow_trap v3, v4, heap_oob+;; @0069 v18 = icmp ule v16, v9+;; @0069 trapz v18, heap_oob+;; @0069 v11 = ireduce.i32 v2+;; @0069 v13 = iadd v12, v11+;; @0069 v19 = ireduce.i32 v3+;; @0069 v21 = iadd v12, v19+;; @0069 v22 = ireduce.i32 v4+;; @0069 call fn0(v0, v13, v21, v22)+;; @006d jump block1+;;+;; block1:+;; @006d return+;; }+;;+;; function u0:4(i32 vmctx, i32, i64, i64, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+60+;; gv2 = load.i32 notrap aligned can_move gv0+56+;; gv3 = load.i32 notrap aligned gv0+68+;; gv4 = load.i32 notrap aligned can_move gv0+64+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:4 sig0+;;+;; block0(v0: i32, v1: i32, v2: i64, v3: i64, v4: i64):+;; @0076 v7 = load.i32 notrap aligned v0+68+;; @0076 v8 = uadd_overflow_trap v2, v4, heap_oob+;; @0076 v9 = uextend.i64 v7+;; @0076 v10 = icmp ule v8, v9+;; @0076 trapz v10, heap_oob+;; @0076 v12 = load.i32 notrap aligned can_move v0+64+;; @0076 v15 = load.i32 notrap aligned v0+60+;; @0076 v16 = uadd_overflow_trap v3, v4, heap_oob+;; @0076 v17 = uextend.i64 v15+;; @0076 v18 = icmp ule v16, v17+;; @0076 trapz v18, heap_oob+;; @0076 v20 = load.i32 notrap aligned can_move v0+56+;; @0076 v11 = ireduce.i32 v2+;; @0076 v13 = iadd v12, v11+;; @0076 v19 = ireduce.i32 v3+;; @0076 v21 = iadd v20, v19+;; @0076 v22 = ireduce.i32 v4+;; @0076 call fn0(v0, v13, v21, v22)+;; @007a jump block1+;;+;; block1:+;; @007a return+;; }+;;+;; function u0:5(i32 vmctx, i32, i32, i64, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i32 notrap aligned gv0+60+;; gv2 = load.i32 notrap aligned can_move gv0+56+;; gv3 = load.i32 notrap aligned gv0+44+;; gv4 = load.i32 notrap aligned can_move gv0+40+;; sig0 = (i32 vmctx, i32, i32, i32) tail+;; fn0 = colocated u805306368:4 sig0+;;+;; block0(v0: i32, v1: i32, v2: i32, v3: i64, v4: i32):+;; @0083 v8 = load.i32 notrap aligned v0+44+;; @0083 v9 = uextend.i64 v2+;; @0083 v6 = uextend.i64 v4+;; @0083 v11 = iadd v9, v6+;; @0083 v12 = uextend.i64 v8+;; @0083 v13 = icmp ule v11, v12+;; @0083 trapz v13, heap_oob+;; @0083 v14 = load.i32 notrap aligned can_move v0+40+;; @0083 v17 = load.i32 notrap aligned v0+60+;; @0083 v18 = uadd_overflow_trap v3, v6, heap_oob+;; @0083 v19 = uextend.i64 v17+;; @0083 v20 = icmp ule v18, v19+;; @0083 trapz v20, heap_oob+;; @0083 v22 = load.i32 notrap aligned can_move v0+56+;; @0083 v15 = iadd v14, v2+;; @0083 v21 = ireduce.i32 v3+;; @0083 v23 = iadd v22, v21+;; @0083 call fn0(v0, v15, v23, v4)+;; @0087 jump block1+;;+;; block1:+;; @0087 return+;; }diff --git a/tests/disas/memory_copy_host64.wat b/tests/disas/memory_copy_host64.watnew file mode 100644index 0000000000..9b4611fb5f--- /dev/null+++ b/tests/disas/memory_copy_host64.wat@@ -0,0 +1,250 @@+;;! target = 'x86_64'+;;! test = 'optimize'++(module+ (memory $m32_a 1)+ (memory $m32_b 1)+ (memory $m64_a i64 1)+ (memory $m64_b i64 1)++ (func $m32_to_same (param i32 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_a $m32_a+ )++ (func $m32_to_m32 (param i32 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_b $m32_a+ )++ (func $m32_to_m64 (param i64 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_a $m32_a+ )++ (func $m64_to_same (param i64 i64 i64)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_a $m64_a+ )++ (func $m64_to_m64 (param i64 i64 i64)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_b $m64_a+ )++ (func $m64_to_m32 (param i32 i64 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_a $m64_a+ )+)+;; function u0:0(i64 vmctx, i64, i32, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+88+;; gv5 = load.i64 notrap aligned readonly can_move gv3+80+;; sig0 = (i64 vmctx, i64, i64, i64) tail+;; fn0 = colocated u805306368:4 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):+;; @0042 v7 = load.i64 notrap aligned v0+88+;; @0042 v8 = uextend.i64 v2+;; @0042 v9 = uextend.i64 v4+;; @0042 v10 = iadd v8, v9+;; @0042 v11 = icmp ule v10, v7+;; @0042 trapz v11, heap_oob+;; @0042 v17 = uextend.i64 v3+;; @0042 v19 = iadd v17, v9+;; @0042 v20 = icmp ule v19, v7+;; @0042 trapz v20, heap_oob+;; @0042 v13 = load.i64 notrap aligned readonly can_move v0+80+;; @0042 v14 = iadd v13, v8+;; @0042 v23 = iadd v13, v17+;; @0042 call fn0(v0, v14, v23, v9)+;; @0046 jump block1+;;+;; block1:+;; @0046 return+;; }+;;+;; function u0:1(i64 vmctx, i64, i32, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+88+;; gv5 = load.i64 notrap aligned readonly can_move gv3+80+;; gv6 = load.i64 notrap aligned gv3+104+;; gv7 = load.i64 notrap aligned readonly can_move gv3+96+;; sig0 = (i64 vmctx, i64, i64, i64) tail+;; fn0 = colocated u805306368:4 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):+;; @004f v7 = load.i64 notrap aligned v0+104+;; @004f v8 = uextend.i64 v2+;; @004f v9 = uextend.i64 v4+;; @004f v10 = iadd v8, v9+;; @004f v11 = icmp ule v10, v7+;; @004f trapz v11, heap_oob+;; @004f v16 = load.i64 notrap aligned v0+88+;; @004f v17 = uextend.i64 v3+;; @004f v19 = iadd v17, v9+;; @004f v20 = icmp ule v19, v16+;; @004f trapz v20, heap_oob+;; @004f v13 = load.i64 notrap aligned readonly can_move v0+96+;; @004f v14 = iadd v13, v8+;; @004f v22 = load.i64 notrap aligned readonly can_move v0+80+;; @004f v23 = iadd v22, v17+;; @004f call fn0(v0, v14, v23, v9)+;; @0053 jump block1+;;+;; block1:+;; @0053 return+;; }+;;+;; function u0:2(i64 vmctx, i64, i64, i32, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+88+;; gv5 = load.i64 notrap aligned readonly can_move gv3+80+;; gv6 = load.i64 notrap aligned gv3+120+;; gv7 = load.i64 notrap aligned can_move gv3+112+;; sig0 = (i64 vmctx, i64, i64, i64) tail+;; fn0 = colocated u805306368:4 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i32, v4: i32):+;; @005c v8 = load.i64 notrap aligned v0+120+;; @005c v6 = uextend.i64 v4+;; @005c v9 = uadd_overflow_trap v2, v6, heap_oob+;; @005c v10 = icmp ule v9, v8+;; @005c trapz v10, heap_oob+;; @005c v11 = load.i64 notrap aligned can_move v0+112+;; @005c v14 = load.i64 notrap aligned v0+88+;; @005c v15 = uextend.i64 v3+;; @005c v17 = iadd v15, v6+;; @005c v18 = icmp ule v17, v14+;; @005c trapz v18, heap_oob+;; @005c v12 = iadd v11, v2+;; @005c v20 = load.i64 notrap aligned readonly can_move v0+80+;; @005c v21 = iadd v20, v15+;; @005c call fn0(v0, v12, v21, v6)+;; @0060 jump block1+;;+;; block1:+;; @0060 return+;; }+;;+;; function u0:3(i64 vmctx, i64, i64, i64, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+120+;; gv5 = load.i64 notrap aligned can_move gv3+112+;; sig0 = (i64 vmctx, i64, i64, i64) tail+;; fn0 = colocated u805306368:4 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64):+;; @0069 v7 = load.i64 notrap aligned v0+120+;; @0069 v8 = uadd_overflow_trap v2, v4, heap_oob+;; @0069 v9 = icmp ule v8, v7+;; @0069 trapz v9, heap_oob+;; @0069 v10 = load.i64 notrap aligned can_move v0+112+;; @0069 v14 = uadd_overflow_trap v3, v4, heap_oob+;; @0069 v15 = icmp ule v14, v7+;; @0069 trapz v15, heap_oob+;; @0069 v11 = iadd v10, v2+;; @0069 v17 = iadd v10, v3+;; @0069 call fn0(v0, v11, v17, v4)+;; @006d jump block1+;;+;; block1:+;; @006d return+;; }+;;+;; function u0:4(i64 vmctx, i64, i64, i64, i64) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+120+;; gv5 = load.i64 notrap aligned can_move gv3+112+;; gv6 = load.i64 notrap aligned gv3+136+;; gv7 = load.i64 notrap aligned can_move gv3+128+;; sig0 = (i64 vmctx, i64, i64, i64) tail+;; fn0 = colocated u805306368:4 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64):+;; @0076 v7 = load.i64 notrap aligned v0+136+;; @0076 v8 = uadd_overflow_trap v2, v4, heap_oob+;; @0076 v9 = icmp ule v8, v7+;; @0076 trapz v9, heap_oob+;; @0076 v10 = load.i64 notrap aligned can_move v0+128+;; @0076 v13 = load.i64 notrap aligned v0+120+;; @0076 v14 = uadd_overflow_trap v3, v4, heap_oob+;; @0076 v15 = icmp ule v14, v13+;; @0076 trapz v15, heap_oob+;; @0076 v16 = load.i64 notrap aligned can_move v0+112+;; @0076 v11 = iadd v10, v2+;; @0076 v17 = iadd v16, v3+;; @0076 call fn0(v0, v11, v17, v4)+;; @007a jump block1+;;+;; block1:+;; @007a return+;; }+;;+;; function u0:5(i64 vmctx, i64, i32, i64, i32) tail {+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly gv0+8+;; gv2 = load.i64 notrap aligned gv1+24+;; gv3 = vmctx+;; gv4 = load.i64 notrap aligned gv3+120+;; gv5 = load.i64 notrap aligned can_move gv3+112+;; gv6 = load.i64 notrap aligned gv3+88+;; gv7 = load.i64 notrap aligned readonly can_move gv3+80+;; sig0 = (i64 vmctx, i64, i64, i64) tail+;; fn0 = colocated u805306368:4 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32, v3: i64, v4: i32):+;; @0083 v8 = load.i64 notrap aligned v0+88+;; @0083 v9 = uextend.i64 v2+;; @0083 v6 = uextend.i64 v4+;; @0083 v11 = iadd v9, v6+;; @0083 v12 = icmp ule v11, v8+;; @0083 trapz v12, heap_oob+;; @0083 v17 = load.i64 notrap aligned v0+120+;; @0083 v18 = uadd_overflow_trap v3, v6, heap_oob+;; @0083 v19 = icmp ule v18, v17+;; @0083 trapz v19, heap_oob+;; @0083 v20 = load.i64 notrap aligned can_move v0+112+;; @0083 v14 = load.i64 notrap aligned readonly can_move v0+80+;; @0083 v15 = iadd v14, v9+;; @0083 v21 = iadd v20, v3+;; @0083 call fn0(v0, v15, v21, v6)+;; @0087 jump block1+;;+;; block1:+;; @0087 return+;; }diff --git a/tests/disas/memory_fill_host32.wat b/tests/disas/memory_fill_host32.watindex 61ec4847ad..89b68cf39a 100644--- a/tests/disas/memory_fill_host32.wat+++ b/tests/disas/memory_fill_host32.wat@@ -54,13 +54,13 @@;; fn0 = colocated u805306368:5 sig0;;;; block0(v0: i32, v1: i32, v2: i32, v3: i32):-;; @003c v6 = load.i32 notrap aligned v0+48-;; @003c v7 = uextend.i64 v2-;; @003c v8 = uextend.i64 v3-;; @003c v9 = iadd v7, v8-;; @003c v10 = uextend.i64 v6-;; @003c v11 = icmp ule v9, v10-;; @003c trapz v11, heap_oob+;; @003c v7 = load.i32 notrap aligned v0+48+;; @003c v8 = uextend.i64 v2+;; @003c v9 = uextend.i64 v3+;; @003c v10 = iadd v8, v9+;; @003c v11 = uextend.i64 v7+;; @003c v12 = icmp ule v10, v11+;; @003c trapz v12, heap_oob;; @003c v13 = load.i32 notrap aligned can_move v0+44;; @003c v14 = iadd v13, v2;; @0038 v4 = iconst.i32 0@@ -79,11 +79,11 @@;; fn0 = colocated u805306368:5 sig0;;;; block0(v0: i32, v1: i32, v2: i64, v3: i64):-;; @0048 v6 = load.i32 notrap aligned v0+56-;; @0048 v7 = uadd_overflow_trap v2, v3, heap_oob-;; @0048 v8 = uextend.i64 v6-;; @0048 v9 = icmp ule v7, v8-;; @0048 trapz v9, heap_oob+;; @0048 v7 = load.i32 notrap aligned v0+56+;; @0048 v8 = uadd_overflow_trap v2, v3, heap_oob+;; @0048 v9 = uextend.i64 v7+;; @0048 v10 = icmp ule v8, v9+;; @0048 trapz v10, heap_oob;; @0048 v12 = load.i32 notrap aligned can_move v0+52;; @0048 v11 = ireduce.i32 v2;; @0048 v13 = iadd v12, v11@@ -104,13 +104,13 @@;; fn0 = colocated u805306368:5 sig0;;;; block0(v0: i32, v1: i32, v2: i32, v3: i32):-;; @0054 v6 = load.i32 notrap aligned v0+64-;; @0054 v7 = uextend.i64 v2-;; @0054 v8 = uextend.i64 v3-;; @0054 v9 = iadd v7, v8-;; @0054 v10 = uextend.i64 v6-;; @0054 v11 = icmp ule v9, v10-;; @0054 trapz v11, heap_oob+;; @0054 v7 = load.i32 notrap aligned v0+64+;; @0054 v8 = uextend.i64 v2+;; @0054 v9 = uextend.i64 v3+;; @0054 v10 = iadd v8, v9+;; @0054 v11 = uextend.i64 v7+;; @0054 v12 = icmp ule v10, v11+;; @0054 trapz v12, heap_oob;; @0054 v13 = load.i32 notrap aligned can_move v0+60;; @0054 v14 = iadd v13, v2;; @0050 v4 = iconst.i32 0@@ -129,11 +129,11 @@;; fn0 = colocated u805306368:5 sig0;;;; block0(v0: i32, v1: i32, v2: i64, v3: i64):-;; @0060 v6 = load.i32 notrap aligned v0+72-;; @0060 v7 = uadd_overflow_trap v2, v3, heap_oob-;; @0060 v8 = uextend.i64 v6-;; @0060 v9 = icmp ule v7, v8-;; @0060 trapz v9, heap_oob+;; @0060 v7 = load.i32 notrap aligned v0+72+;; @0060 v8 = uadd_overflow_trap v2, v3, heap_oob+;; @0060 v9 = uextend.i64 v7+;; @0060 v10 = icmp ule v8, v9+;; @0060 trapz v10, heap_oob;; @0060 v12 = load.i32 notrap aligned can_move v0+68;; @0060 v11 = ireduce.i32 v2;; @0060 v13 = iadd v12, v11@@ -154,13 +154,13 @@;; fn0 = colocated u805306368:5 sig0;;;; block0(v0: i32, v1: i32, v2: i32, v3: i32):-;; @006c v6 = load.i32 notrap aligned v0+80-;; @006c v7 = uextend.i64 v2-;; @006c v8 = uextend.i64 v3-;; @006c v9 = iadd v7, v8-;; @006c v10 = uextend.i64 v6-;; @006c v11 = icmp ule v9, v10-;; @006c trapz v11, heap_oob+;; @006c v7 = load.i32 notrap aligned v0+80+;; @006c v8 = uextend.i64 v2+;; @006c v9 = uextend.i64 v3+;; @006c v10 = iadd v8, v9+;; @006c v11 = uextend.i64 v7+;; @006c v12 = icmp ule v10, v11+;; @006c trapz v12, heap_oob;; @006c v13 = load.i32 notrap aligned readonly can_move v0+76;; @006c v14 = iadd v13, v2;; @0068 v4 = iconst.i32 0diff --git a/tests/disas/memory_fill_host64.wat b/tests/disas/memory_fill_host64.watindex af8650e35d..5f4c497ecc 100644--- a/tests/disas/memory_fill_host64.wat+++ b/tests/disas/memory_fill_host64.wat@@ -55,16 +55,16 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32):-;; @003c v6 = load.i64 notrap aligned v0+96-;; @003c v7 = uextend.i64 v2-;; @003c v8 = uextend.i64 v3-;; @003c v9 = iadd v7, v8-;; @003c v10 = icmp ule v9, v6-;; @003c trapz v10, heap_oob+;; @003c v7 = load.i64 notrap aligned v0+96+;; @003c v8 = uextend.i64 v2+;; @003c v9 = uextend.i64 v3+;; @003c v10 = iadd v8, v9+;; @003c v11 = icmp ule v10, v7+;; @003c trapz v11, heap_oob;; @003c v13 = load.i64 notrap aligned readonly can_move v0+88-;; @003c v14 = iadd v13, v7+;; @003c v14 = iadd v13, v8;; @0038 v4 = iconst.i32 0-;; @003c call fn0(v0, v14, v4, v8) ; v4 = 0+;; @003c call fn0(v0, v14, v4, v9) ; v4 = 0;; @003f jump block1;;;; block1:@@ -83,10 +83,10 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i64, v3: i64):-;; @0048 v6 = load.i64 notrap aligned v0+112-;; @0048 v7 = uadd_overflow_trap v2, v3, heap_oob-;; @0048 v8 = icmp ule v7, v6-;; @0048 trapz v8, heap_oob+;; @0048 v7 = load.i64 notrap aligned v0+112+;; @0048 v8 = uadd_overflow_trap v2, v3, heap_oob+;; @0048 v9 = icmp ule v8, v7+;; @0048 trapz v9, heap_oob;; @0048 v10 = load.i64 notrap aligned can_move v0+104;; @0048 v11 = iadd v10, v2;; @0044 v4 = iconst.i32 0@@ -109,16 +109,16 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32):-;; @0054 v6 = load.i64 notrap aligned v0+128-;; @0054 v7 = uextend.i64 v2-;; @0054 v8 = uextend.i64 v3-;; @0054 v9 = iadd v7, v8-;; @0054 v10 = icmp ule v9, v6-;; @0054 trapz v10, heap_oob+;; @0054 v7 = load.i64 notrap aligned v0+128+;; @0054 v8 = uextend.i64 v2+;; @0054 v9 = uextend.i64 v3+;; @0054 v10 = iadd v8, v9+;; @0054 v11 = icmp ule v10, v7+;; @0054 trapz v11, heap_oob;; @0054 v13 = load.i64 notrap aligned readonly can_move v0+120-;; @0054 v14 = iadd v13, v7+;; @0054 v14 = iadd v13, v8;; @0050 v4 = iconst.i32 0-;; @0054 call fn0(v0, v14, v4, v8) ; v4 = 0+;; @0054 call fn0(v0, v14, v4, v9) ; v4 = 0;; @0057 jump block1;;;; block1:@@ -137,10 +137,10 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i64, v3: i64):-;; @0060 v6 = load.i64 notrap aligned v0+144-;; @0060 v7 = uadd_overflow_trap v2, v3, heap_oob-;; @0060 v8 = icmp ule v7, v6-;; @0060 trapz v8, heap_oob+;; @0060 v7 = load.i64 notrap aligned v0+144+;; @0060 v8 = uadd_overflow_trap v2, v3, heap_oob+;; @0060 v9 = icmp ule v8, v7+;; @0060 trapz v9, heap_oob;; @0060 v10 = load.i64 notrap aligned can_move v0+136;; @0060 v11 = iadd v10, v2;; @005c v4 = iconst.i32 0@@ -163,16 +163,16 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32):-;; @006c v6 = load.i64 notrap aligned v0+160-;; @006c v7 = uextend.i64 v2-;; @006c v8 = uextend.i64 v3-;; @006c v9 = iadd v7, v8-;; @006c v10 = icmp ule v9, v6-;; @006c trapz v10, heap_oob+;; @006c v7 = load.i64 notrap aligned v0+160+;; @006c v8 = uextend.i64 v2+;; @006c v9 = uextend.i64 v3+;; @006c v10 = iadd v8, v9+;; @006c v11 = icmp ule v10, v7+;; @006c trapz v11, heap_oob;; @006c v13 = load.i64 notrap aligned readonly can_move v0+152-;; @006c v14 = iadd v13, v7+;; @006c v14 = iadd v13, v8;; @0068 v4 = iconst.i32 0-;; @006c call fn0(v0, v14, v4, v8) ; v4 = 0+;; @006c call fn0(v0, v14, v4, v9) ; v4 = 0;; @006f jump block1;;;; block1:diff --git a/tests/misc_testsuite/memory-copy.wast b/tests/misc_testsuite/memory-copy.wastindex 86633f6bc4..4b18d7f5d8 100644--- a/tests/misc_testsuite/memory-copy.wast+++ b/tests/misc_testsuite/memory-copy.wast@@ -1,4 +1,6 @@;;! bulk_memory = true+;;! multi_memory = true+;;! memory64 = true(module(memory 1 1)@@ -124,3 +126,149 @@(assert_return(invoke "is olleh?" (i32.const 2000))(i32.const 1))++;; test trapping boundary behavior+(module+ (memory $m32_a 1)+ (memory $m32_b 1)+ (memory $m64_a i64 1)+ (memory $m64_b i64 1)++ (func (export "m32_to_same") (param i32 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_a $m32_a+ )++ (func (export "m32_to_m32") (param i32 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_b $m32_a+ )++ (func (export "m32_to_m64") (param i64 i32 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_a $m32_a+ )++ (func (export "m64_to_same") (param i64 i64 i64)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_a $m64_a+ )++ (func (export "m64_to_m64") (param i64 i64 i64)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m64_b $m64_a+ )++ (func (export "m64_to_m32") (param i32 i64 i32)+ local.get 0+ local.get 1+ local.get 2+ memory.copy $m32_a $m64_a+ )+)++(assert_return (invoke "m32_to_same" (i32.const 0) (i32.const 0) (i32.const 0)))+(assert_return (invoke "m32_to_same" (i32.const 0) (i32.const 0) (i32.const 65536)))+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const 0) (i32.const 65537)) "out of bounds")+(assert_return (invoke "m32_to_same" (i32.const 100) (i32.const 200) (i32.const 65336)))+(assert_return (invoke "m32_to_same" (i32.const 200) (i32.const 100) (i32.const 65336)))+(assert_trap (invoke "m32_to_same" (i32.const 201) (i32.const 100) (i32.const 65336)) "out of bounds")+(assert_return (invoke "m32_to_same" (i32.const 200) (i32.const 101) (i32.const 65336)))+(assert_trap (invoke "m32_to_same" (i32.const 200) (i32.const 100) (i32.const 65337)) "out of bounds")+(assert_trap (invoke "m32_to_same" (i32.const -1) (i32.const 0) (i32.const 0)) "out of bounds")+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const -1) (i32.const 0)) "out of bounds")+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_same" (i32.const 100) (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const 100) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_same" (i32.const -1) (i32.const 0) (i32.const 100)) "out of bounds")+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const -1) (i32.const 100)) "out of bounds")++(assert_return (invoke "m32_to_m32" (i32.const 0) (i32.const 0) (i32.const 0)))+(assert_return (invoke "m32_to_m32" (i32.const 0) (i32.const 0) (i32.const 65536)))+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const 0) (i32.const 65537)) "out of bounds")+(assert_return (invoke "m32_to_m32" (i32.const 100) (i32.const 200) (i32.const 65336)))+(assert_return (invoke "m32_to_m32" (i32.const 200) (i32.const 100) (i32.const 65336)))+(assert_trap (invoke "m32_to_m32" (i32.const 201) (i32.const 100) (i32.const 65336)) "out of bounds")+(assert_return (invoke "m32_to_m32" (i32.const 200) (i32.const 101) (i32.const 65336)))+(assert_trap (invoke "m32_to_m32" (i32.const 200) (i32.const 100) (i32.const 65337)) "out of bounds")+(assert_trap (invoke "m32_to_m32" (i32.const -1) (i32.const 0) (i32.const 0)) "out of bounds")+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const -1) (i32.const 0)) "out of bounds")+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_m32" (i32.const 100) (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const 100) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_m32" (i32.const -1) (i32.const 0) (i32.const 100)) "out of bounds")+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const -1) (i32.const 100)) "out of bounds")++(assert_return (invoke "m32_to_m64" (i64.const 0) (i32.const 0) (i32.const 0)))+(assert_return (invoke "m32_to_m64" (i64.const 0) (i32.const 0) (i32.const 65536)))+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const 0) (i32.const 65537)) "out of bounds")+(assert_return (invoke "m32_to_m64" (i64.const 100) (i32.const 200) (i32.const 65336)))+(assert_return (invoke "m32_to_m64" (i64.const 200) (i32.const 100) (i32.const 65336)))+(assert_trap (invoke "m32_to_m64" (i64.const 201) (i32.const 100) (i32.const 65336)) "out of bounds")+(assert_return (invoke "m32_to_m64" (i64.const 200) (i32.const 101) (i32.const 65336)))+(assert_trap (invoke "m32_to_m64" (i64.const 200) (i32.const 100) (i32.const 65337)) "out of bounds")+(assert_trap (invoke "m32_to_m64" (i64.const -1) (i32.const 0) (i32.const 0)) "out of bounds")+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const -1) (i32.const 0)) "out of bounds")+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_m64" (i64.const 100) (i32.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const 100) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m32_to_m64" (i64.const -1) (i32.const 0) (i32.const 100)) "out of bounds")+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const -1) (i32.const 100)) "out of bounds")++(assert_return (invoke "m64_to_same" (i64.const 0) (i64.const 0) (i64.const 0)))+(assert_return (invoke "m64_to_same" (i64.const 0) (i64.const 0) (i64.const 65536)))+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const 0) (i64.const 65537)) "out of bounds")+(assert_return (invoke "m64_to_same" (i64.const 100) (i64.const 200) (i64.const 65336)))+(assert_return (invoke "m64_to_same" (i64.const 200) (i64.const 100) (i64.const 65336)))+(assert_trap (invoke "m64_to_same" (i64.const 201) (i64.const 100) (i64.const 65336)) "out of bounds")+(assert_return (invoke "m64_to_same" (i64.const 200) (i64.const 101) (i64.const 65336)))+(assert_trap (invoke "m64_to_same" (i64.const 200) (i64.const 100) (i64.const 65337)) "out of bounds")+(assert_trap (invoke "m64_to_same" (i64.const -1) (i64.const 0) (i64.const 0)) "out of bounds")+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const -1) (i64.const 0)) "out of bounds")+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const 0) (i64.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_same" (i64.const 100) (i64.const 0) (i64.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const 100) (i64.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_same" (i64.const -1) (i64.const 0) (i64.const 100)) "out of bounds")+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const -1) (i64.const 100)) "out of bounds")++(assert_return (invoke "m64_to_m64" (i64.const 0) (i64.const 0) (i64.const 0)))+(assert_return (invoke "m64_to_m64" (i64.const 0) (i64.const 0) (i64.const 65536)))+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const 0) (i64.const 65537)) "out of bounds")+(assert_return (invoke "m64_to_m64" (i64.const 100) (i64.const 200) (i64.const 65336)))+(assert_return (invoke "m64_to_m64" (i64.const 200) (i64.const 100) (i64.const 65336)))+(assert_trap (invoke "m64_to_m64" (i64.const 201) (i64.const 100) (i64.const 65336)) "out of bounds")+(assert_return (invoke "m64_to_m64" (i64.const 200) (i64.const 101) (i64.const 65336)))+(assert_trap (invoke "m64_to_m64" (i64.const 200) (i64.const 100) (i64.const 65337)) "out of bounds")+(assert_trap (invoke "m64_to_m64" (i64.const -1) (i64.const 0) (i64.const 0)) "out of bounds")+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const -1) (i64.const 0)) "out of bounds")+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const 0) (i64.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_m64" (i64.const 100) (i64.const 0) (i64.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const 100) (i64.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_m64" (i64.const -1) (i64.const 0) (i64.const 100)) "out of bounds")+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const -1) (i64.const 100)) "out of bounds")++(assert_return (invoke "m64_to_m32" (i32.const 0) (i64.const 0) (i32.const 0)))+(assert_return (invoke "m64_to_m32" (i32.const 0) (i64.const 0) (i32.const 65536)))+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const 0) (i32.const 65537)) "out of bounds")+(assert_return (invoke "m64_to_m32" (i32.const 100) (i64.const 200) (i32.const 65336)))+(assert_return (invoke "m64_to_m32" (i32.const 200) (i64.const 100) (i32.const 65336)))+(assert_trap (invoke "m64_to_m32" (i32.const 201) (i64.const 100) (i32.const 65336)) "out of bounds")+(assert_return (invoke "m64_to_m32" (i32.const 200) (i64.const 101) (i32.const 65336)))+(assert_trap (invoke "m64_to_m32" (i32.const 200) (i64.const 100) (i32.const 65337)) "out of bounds")+(assert_trap (invoke "m64_to_m32" (i32.const -1) (i64.const 0) (i32.const 0)) "out of bounds")+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const -1) (i32.const 0)) "out of bounds")+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_m32" (i32.const 100) (i64.const 0) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const 100) (i32.const -1)) "out of bounds")+(assert_trap (invoke "m64_to_m32" (i32.const -1) (i64.const 0) (i32.const 100)) "out of bounds")+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const -1) (i32.const 100)) "out of bounds")diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rsindex ec4c460225..f0a34b163c 100644--- a/winch/codegen/src/codegen/mod.rs+++ b/winch/codegen/src/codegen/mod.rs@@ -1130,40 +1130,39 @@ whereOk(())}- /// Emit the `memory.fill` operation.- pub fn emit_memory_fill(&mut self, mem: MemoryIndex) -> Result<()> {- let heap = self.env.resolve_heap(mem);+ /// Emit a bounds check for `ptr+len` and put the native address for this+ /// wasm address into `dst`.+ fn emit_bounds_check_and_compute_addr(+ &mut self,+ heap: &HeapData,+ dst: Reg,+ ptr: Reg,+ len: Reg,+ ) -> Result<()> {let ptr_size: OperandSize = self.env.ptr_type().try_into()?;let idx_size: OperandSize = heap.index_type().try_into()?;-- // The wasm stack at this point is `[dst, val, len]`.- let len = self.context.pop_to_reg(self.masm, None)?;- let val = self.context.pop_to_reg(self.masm, None)?;- let dst = self.context.pop_to_reg(self.masm, None)?;-- // Compute `end = dst + len` trapping on overflow. For an `i32` index+ // Compute `dst = ptr + len` trapping on overflow. For an `i32` index// type the operands are zero-extended to 64-bit so overflow is// impossible.- let end = self.context.any_gpr(self.masm)?;match idx_size {OperandSize::S32 => {self.masm- .extend(writable!(end), dst.reg, Extend::<Zero>::I64Extend32.into())?;+ .extend(writable!(dst), ptr, Extend::<Zero>::I64Extend32.into())?;self.masm.add_uextend(- writable!(end),- end,- len.reg,+ writable!(dst),+ dst,+ len,OperandSize::S32,OperandSize::S64,)?;}OperandSize::S64 => {self.masm- .mov(writable!(end), dst.reg.into(), OperandSize::S64)?;+ .mov(writable!(dst), ptr.into(), OperandSize::S64)?;self.masm.checked_uadd(- writable!(end),- end,- len.reg.into(),+ writable!(dst),+ dst,+ len.into(),OperandSize::S64,TrapCode::HEAP_OUT_OF_BOUNDS,)?;@@ -1171,29 +1170,94 @@ where_ => unreachable!(),}- // Load the current size in bytes of the memory.+ // Load the current size in bytes of the memory, and trap if+ // `dst > size_in_bytes`.let size_in_bytes = self.context.any_gpr(self.masm)?;self.load_memory_length(&heap, size_in_bytes)?;-- // Trap if `end > size_in_bytes`.assert!(ptr_size == OperandSize::S64);- self.masm.cmp(end, size_in_bytes.into(), ptr_size)?;+ self.masm.cmp(dst, size_in_bytes.into(), ptr_size)?;self.masm.trapif(IntCmpKind::GtU, TrapCode::HEAP_OUT_OF_BOUNDS)?;- self.context.free_reg(end);self.context.free_reg(size_in_bytes);- // Compute `dst_ptr = memory_base + dst`.- let dst_ptr = self.context.any_gpr(self.masm)?;+ // Compute `dst = memory_base + ptr`.bounds::load_heap_addr_unchecked(self.masm,&heap,- Index::from_typed_reg(dst),+ Index::from_typed_reg(TypedReg::new(heap.index_type(), ptr)),ImmOffset::from_u32(0),- dst_ptr,+ dst,ptr_size,)?;- self.context.free_reg(dst.reg);+ Ok(())+ }++ /// Emit the `memory.copy` operation.+ pub fn emit_memory_copy(&mut self, dst_mem: MemoryIndex, src_mem: MemoryIndex) -> Result<()> {+ let dst_heap = self.env.resolve_heap(dst_mem);+ let src_heap = self.env.resolve_heap(src_mem);+ let dst_idx_size: OperandSize = dst_heap.index_type().try_into()?;+ let src_idx_size: OperandSize = src_heap.index_type().try_into()?;++ let len = self.context.pop_to_reg(self.masm, None)?;+ let src = self.context.pop_to_reg(self.masm, None)?;+ let dst = self.context.pop_to_reg(self.masm, None)?;++ // For 32-bit linear memories go ahead and make sure `len` is zero+ // extended within its register ensuring that the full 64-bits of the+ // register are defined. This assists in situations like cross-memory+ // copies where one memory is 32-bit and one is 64-bit and the same+ // register can be used for the length in both bounds checks below.+ if dst_idx_size == OperandSize::S32 || src_idx_size == OperandSize::S32 {+ self.masm.extend(+ writable!(len.reg),+ len.reg,+ Extend::<Zero>::I64Extend32.into(),+ )?;+ }++ let dst_raw_addr = self.context.any_gpr(self.masm)?;+ self.emit_bounds_check_and_compute_addr(&dst_heap, dst_raw_addr, dst.reg, len.reg)?;+ self.context.free_reg(dst);++ let src_raw_addr = self.context.any_gpr(self.masm)?;+ self.emit_bounds_check_and_compute_addr(&src_heap, src_raw_addr, src.reg, len.reg)?;+ self.context.free_reg(src);++ self.context+ .stack+ .push(TypedReg::new(self.env.ptr_type(), dst_raw_addr).into());+ self.context+ .stack+ .push(TypedReg::new(self.env.ptr_type(), src_raw_addr).into());+ self.context+ .stack+ .push(TypedReg::new(self.env.ptr_type(), len.reg).into());++ let builtin = self.env.builtins.memory_copy::<M::ABI>()?;+ FnCall::emit::<M>(+ &mut self.env,+ self.masm,+ &mut self.context,+ Callee::Builtin(builtin),+ )?;+ Ok(())+ }++ /// Emit the `memory.fill` operation.+ pub fn emit_memory_fill(&mut self, mem: MemoryIndex) -> Result<()> {+ let heap = self.env.resolve_heap(mem);+ let ptr_size: OperandSize = self.env.ptr_type().try_into()?;+ let idx_size: OperandSize = heap.index_type().try_into()?;++ // The wasm stack at this point is `[dst, val, len]`.+ let len = self.context.pop_to_reg(self.masm, None)?;+ let val = self.context.pop_to_reg(self.masm, None)?;+ let dst = self.context.pop_to_reg(self.masm, None)?;++ let raw_addr = self.context.any_gpr(self.masm)?;+ self.emit_bounds_check_and_compute_addr(&heap, raw_addr, dst.reg, len.reg)?;+ self.context.free_reg(dst);// The libcall takes the length as a host-pointer-sized integer, so// zero-extend if the wasm index type is smaller.@@ -1209,7 +1273,7 @@ where// Set up the call arguments: `[dst_ptr, val, len]`.self.context.stack- .push(TypedReg::new(self.env.ptr_type(), dst_ptr).into());+ .push(TypedReg::new(self.env.ptr_type(), raw_addr).into());self.context.stack.push(val.into());self.context.stackdiff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rsindex 138c10d51a..f908379b5e 100644--- a/winch/codegen/src/visitor.rs+++ b/winch/codegen/src/visitor.rs@@ -1891,28 +1891,10 @@ where}fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output {- // At this point, the stack is expected to contain:- // [ dst_offset, src_offset, len ]- // The following code inserts the missing params, so that stack contains:- // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ]- // Which is the order expected by the builtin function.- let _ = self.context.stack.ensure_index_at(3)?;- let at = self.context.stack.ensure_index_at(2)?;- self.context.stack.insert_many(at, &[src_mem.try_into()?]);-- // One element was inserted above, so instead of 3, we use 4.- let at = self.context.stack.ensure_index_at(4)?;- self.context.stack.insert_many(at, &[dst_mem.try_into()?]);-- let builtin = self.env.builtins.memory_copy::<M::ABI>()?;-- FnCall::emit::<M>(- &mut self.env,- self.masm,- &mut self.context,- Callee::Builtin(builtin),- )?;- self.context.pop_and_free(self.masm)+ self.emit_memory_copy(+ MemoryIndex::from_u32(dst_mem),+ MemoryIndex::from_u32(src_mem),+ )}fn visit_memory_fill(&mut self, mem: u32) -> Self::Output {