Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Optimize `memory.copy` implementation (#13368)

Alex Crichton16b5f97a4223
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 201a47d856..37d2076d2b 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -3439,69 +3439,44 @@ impl FuncEnvironment<'_> {
))
}
- pub fn translate_memory_copy(
- &mut self,
- builder: &mut FunctionBuilder<'_>,
- src_index: MemoryIndex,
- dst_index: MemoryIndex,
- dst: ir::Value,
- src: ir::Value,
- len: ir::Value,
- ) -> WasmResult<()> {
- let mut pos = builder.cursor();
- let vmctx = self.vmctx_val(&mut pos);
-
- let memory_copy = self.builtin_functions.memory_copy(&mut pos.func);
- let dst = self.cast_index_to_i64(&mut pos, dst, self.memory(dst_index).idx_type);
- let src = self.cast_index_to_i64(&mut pos, src, self.memory(src_index).idx_type);
- // The length is 32-bit if either memory is 32-bit, but if they're both
- // 64-bit then it's 64-bit. Our intrinsic takes a 64-bit length for
- // compatibility across all memories, so make sure that it's cast
- // correctly here (this is a bit special so no generic helper unlike for
- // `dst`/`src` above)
- let len = if index_type_to_ir_type(self.memory(dst_index).idx_type) == I64
- && index_type_to_ir_type(self.memory(src_index).idx_type) == I64
- {
- len
- } else {
- pos.ins().uextend(I64, len)
- };
- let src_index = pos.ins().iconst(I32, i64::from(src_index.as_u32()));
- let dst_index = pos.ins().iconst(I32, i64::from(dst_index.as_u32()));
- pos.ins()
- .call(memory_copy, &[vmctx, dst_index, dst, src_index, src, len]);
-
- Ok(())
- }
-
- pub fn translate_memory_fill(
+ /// Performs a bounds check and raises a trap if `ptr+len` is out-of-bounds
+ /// for `len`.
+ ///
+ /// Returns the raw host-native heap address of `ptr`.
+ fn bounds_check_for_memory_intrinsic(
&mut self,
builder: &mut FunctionBuilder<'_>,
memory_index: MemoryIndex,
- dst: ir::Value,
- val: ir::Value,
+ ptr: ir::Value,
len: ir::Value,
- ) {
+ ) -> ir::Value {
let pointer_type = self.pointer_type();
let idx_type = self.memory(memory_index).idx_type;
+ let idx_clif_type = match idx_type {
+ IndexType::I32 => I32,
+ IndexType::I64 => I64,
+ };
+ debug_assert_eq!(builder.func.dfg.value_type(ptr), idx_clif_type);
+ debug_assert_eq!(builder.func.dfg.value_type(len), idx_clif_type);
+
// Load the memory size, as `pointer_type`, which is the size of this
// memory currently in
// bytes.
let size_in_bytes = self.memory_size_in_bytes(&mut builder.cursor(), memory_index);
- // Compute the end address of this fill, casted to the `I64` type.
+ // Compute the end address of this operation, casted to the `I64` type.
//
// Note that addition can't overflow after extending 32-bits to
// 64-bits, so no need to check for overflow in the 32-bit index case.
let end64 = match idx_type {
IndexType::I32 => {
- let dst64 = builder.ins().uextend(I64, dst);
+ let ptr64 = builder.ins().uextend(I64, ptr);
let len64 = builder.ins().uextend(I64, len);
- builder.ins().iadd(dst64, len64)
+ builder.ins().iadd(ptr64, len64)
}
IndexType::I64 => {
- self.uadd_overflow_trap(builder, dst, len, ir::TrapCode::HEAP_OUT_OF_BOUNDS)
+ self.uadd_overflow_trap(builder, ptr, len, ir::TrapCode::HEAP_OUT_OF_BOUNDS)
}
};
@@ -3512,24 +3487,90 @@ impl FuncEnvironment<'_> {
_ => unreachable!(),
};
- // This is the actual bounds check that verifies that this `fill`
- // operation is in-bounds. Once control flow gets past here we know
- // that nothing can overflow and everything is in-bounds.
+ // This is the actual bounds check that verifies that this operation is
+ // in-bounds. Once control flow gets past here we know that nothing can
+ // overflow and everything is in-bounds.
let inbounds = builder
.ins()
.icmp(IntCC::UnsignedLessThanOrEqual, end64, size_in_bytes64);
self.trapz(builder, inbounds, ir::TrapCode::HEAP_OUT_OF_BOUNDS);
+ // Compute the actual raw heap address to return
+ let mut pos = builder.cursor();
+ let heap = self.get_or_create_heap(pos.func, memory_index);
+ let heap = self.heaps()[heap].clone();
+ let ptr = self.unchecked_cast_wasm_addr_to_native_addr(&mut pos, ptr, idx_type);
+ crate::bounds_checks::compute_addr(&mut pos, &heap, pointer_type, ptr, 0)
+ }
+
+ pub fn translate_memory_copy(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ src_index: MemoryIndex,
+ dst_index: MemoryIndex,
+ dst: ir::Value,
+ src: ir::Value,
+ len: ir::Value,
+ ) -> WasmResult<()> {
+ let src_idx_ty = self.memory(src_index).idx_type;
+ let dst_idx_ty = self.memory(dst_index).idx_type;
+
+ // The length is 32-bit if either memory is 32-bit, but if they're both
+ // 64-bit then it's 64-bit.
+ let len_idx_ty = match (src_idx_ty, dst_idx_ty) {
+ (IndexType::I32, _) | (_, IndexType::I32) => IndexType::I32,
+ (IndexType::I64, IndexType::I64) => IndexType::I64,
+ };
+
+ let mut pos = builder.cursor();
+ let vmctx = self.vmctx_val(&mut pos);
+ let memory_copy = self.builtin_functions.memory_copy(&mut pos.func);
+
+ let src_len = if src_idx_ty == len_idx_ty {
+ len
+ } else {
+ assert_eq!(src_idx_ty, IndexType::I64);
+ builder.ins().uextend(I64, len)
+ };
+ let dst_len = if dst_idx_ty == len_idx_ty {
+ len
+ } else {
+ assert_eq!(dst_idx_ty, IndexType::I64);
+ builder.ins().uextend(I64, len)
+ };
+
+ // Perform a bounds check for the src/dst memories and compute the raw
+ // heap addresses at the same time.
+ let dst_raw_addr = self.bounds_check_for_memory_intrinsic(builder, dst_index, dst, dst_len);
+ let src_raw_addr = self.bounds_check_for_memory_intrinsic(builder, src_index, src, src_len);
+
+ // Fit the `len` value to `pointer_type`. Note that at this point it's
+ // guaranteed inbounds so there's no loss in precision.
+ let len_ptr =
+ self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, len_idx_ty);
+
+ builder
+ .ins()
+ .call(memory_copy, &[vmctx, dst_raw_addr, src_raw_addr, len_ptr]);
+
+ Ok(())
+ }
+
+ pub fn translate_memory_fill(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ memory_index: MemoryIndex,
+ dst: ir::Value,
+ val: ir::Value,
+ len: ir::Value,
+ ) {
+ let idx_type = self.memory(memory_index).idx_type;
let memory_fill = self.builtin_functions.memory_fill(&mut builder.func);
let mut pos = builder.cursor();
let vmctx = self.vmctx_val(&mut pos);
- // Compute the actual raw heap address that the `memset` is writing to.
- let heap = self.get_or_create_heap(pos.func, memory_index);
- let heap = self.heaps()[heap].clone();
- let dst_ptr = self.unchecked_cast_wasm_addr_to_native_addr(&mut pos, dst, idx_type);
- let raw_heap_addr =
- crate::bounds_checks::compute_addr(&mut pos, &heap, pointer_type, dst_ptr, 0);
+ // Bounds check `dst+len` and convert it to a raw heap address.
+ let raw_heap_addr = self.bounds_check_for_memory_intrinsic(builder, memory_index, dst, len);
// Fit the `len` value to `pointer_type`. Note that at this point it's
// guaranteed inbounds so there's no loss in precision.
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index f0cee56a8b..8d4906657c 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -13,7 +13,7 @@ macro_rules! foreach_builtin_function {
// Returns an index for wasm's `elem.drop`.
elem_drop(vmctx: vmctx, elem: u32) -> bool;
// Returns an index for wasm's `memory.copy`
- memory_copy(vmctx: vmctx, dst_index: u32, dst: u64, src_index: u32, src: u64, len: u64) -> bool;
+ memory_copy(vmctx: vmctx, dst: pointer, src: pointer, len: size);
// Returns an index for wasm's `memory.fill` instruction.
memory_fill(vmctx: vmctx, dst: pointer, val: u32, len: size);
// Returns an index for wasm's `memory.init` instruction.
diff --git a/crates/wasmtime/src/runtime/vm/instance.rs b/crates/wasmtime/src/runtime/vm/instance.rs
index e827e8adf2..d9ec7c14b6 100644
--- a/crates/wasmtime/src/runtime/vm/instance.rs
+++ b/crates/wasmtime/src/runtime/vm/instance.rs
@@ -1030,42 +1030,6 @@ impl Instance {
}
}
- /// Do a `memory.copy`
- ///
- /// # Errors
- ///
- /// Returns a `Trap` error when the source or destination ranges are out of
- /// bounds.
- pub(crate) fn memory_copy(
- self: Pin<&mut Self>,
- dst_index: MemoryIndex,
- dst: u64,
- src_index: MemoryIndex,
- src: u64,
- len: u64,
- ) -> Result<(), Trap> {
- // https://webassembly.github.io/reference-types/core/exec/instructions.html#exec-memory-copy
-
- let src_mem = self.get_memory(src_index);
- let dst_mem = self.get_memory(dst_index);
-
- let src = self.validate_inbounds(src_mem.current_length(), src, len)?;
- let dst = self.validate_inbounds(dst_mem.current_length(), dst, len)?;
- let len = usize::try_from(len).unwrap();
-
- // Bounds and casts are checked above, by this point we know that
- // everything is safe.
- unsafe {
- let dst = dst_mem.base.as_ptr().add(dst);
- let src = src_mem.base.as_ptr().add(src);
- // FIXME audit whether this is safe in the presence of shared memory
- // (https://github.com/bytecodealliance/wasmtime/issues/4203).
- ptr::copy(src, dst, len);
- }
-
- Ok(())
- }
-
fn validate_inbounds(&self, max: usize, ptr: u64, len: u64) -> Result<usize, Trap> {
let oob = || Trap::MemoryOutOfBounds;
let end = ptr
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 50c7673dd8..ff75e49f6e 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -535,20 +535,17 @@ fn elem_drop(store: &mut dyn VMStore, instance: InstanceId, elem_index: u32) ->
}
// Implementation of `memory.copy`.
-fn memory_copy(
- store: &mut dyn VMStore,
- instance: InstanceId,
- dst_index: u32,
- dst: u64,
- src_index: u32,
- src: u64,
- len: u64,
-) -> Result<(), Trap> {
- let src_index = MemoryIndex::from_u32(src_index);
- let dst_index = MemoryIndex::from_u32(dst_index);
- store
- .instance_mut(instance)
- .memory_copy(dst_index, dst, src_index, src, len)
+unsafe fn memory_copy(
+ _store: &mut dyn VMStore,
+ _instance: InstanceId,
+ dst: *mut u8,
+ src: *mut u8,
+ len: usize,
+) {
+ let src = src.cast_const();
+ // FIXME(#4203): this is known to not be sound in the presence of shared
+ // memories.
+ unsafe { src.copy_to(dst, len) }
}
unsafe fn memory_fill(
diff --git a/tests/disas/memory_copy_host32.wat b/tests/disas/memory_copy_host32.wat
new file mode 100644
index 0000000000..ef6002493b
--- /dev/null
+++ b/tests/disas/memory_copy_host32.wat
@@ -0,0 +1,244 @@
+;;! target = 'pulley32'
+;;! test = 'optimize'
+
+(module
+ (memory $m32_a 1)
+ (memory $m32_b 1)
+ (memory $m64_a i64 1)
+ (memory $m64_b i64 1)
+
+ (func $m32_to_same (param i32 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_a $m32_a
+ )
+
+ (func $m32_to_m32 (param i32 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_b $m32_a
+ )
+
+ (func $m32_to_m64 (param i64 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_a $m32_a
+ )
+
+ (func $m64_to_same (param i64 i64 i64)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_a $m64_a
+ )
+
+ (func $m64_to_m64 (param i64 i64 i64)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_b $m64_a
+ )
+
+ (func $m64_to_m32 (param i32 i64 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_a $m64_a
+ )
+)
+;; function u0:0(i32 vmctx, i32, i32, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+44
+;; gv2 = load.i32 notrap aligned can_move gv0+40
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:4 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32):
+;; @0042 v7 = load.i32 notrap aligned v0+44
+;; @0042 v8 = uextend.i64 v2
+;; @0042 v9 = uextend.i64 v4
+;; @0042 v10 = iadd v8, v9
+;; @0042 v11 = uextend.i64 v7
+;; @0042 v12 = icmp ule v10, v11
+;; @0042 trapz v12, heap_oob
+;; @0042 v13 = load.i32 notrap aligned can_move v0+40
+;; @0042 v17 = uextend.i64 v3
+;; @0042 v19 = iadd v17, v9
+;; @0042 v21 = icmp ule v19, v11
+;; @0042 trapz v21, heap_oob
+;; @0042 v14 = iadd v13, v2
+;; @0042 v23 = iadd v13, v3
+;; @0042 call fn0(v0, v14, v23, v4)
+;; @0046 jump block1
+;;
+;; block1:
+;; @0046 return
+;; }
+;;
+;; function u0:1(i32 vmctx, i32, i32, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+44
+;; gv2 = load.i32 notrap aligned can_move gv0+40
+;; gv3 = load.i32 notrap aligned gv0+52
+;; gv4 = load.i32 notrap aligned can_move gv0+48
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:4 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i32, v3: i32, v4: i32):
+;; @004f v7 = load.i32 notrap aligned v0+52
+;; @004f v8 = uextend.i64 v2
+;; @004f v9 = uextend.i64 v4
+;; @004f v10 = iadd v8, v9
+;; @004f v11 = uextend.i64 v7
+;; @004f v12 = icmp ule v10, v11
+;; @004f trapz v12, heap_oob
+;; @004f v13 = load.i32 notrap aligned can_move v0+48
+;; @004f v16 = load.i32 notrap aligned v0+44
+;; @004f v17 = uextend.i64 v3
+;; @004f v19 = iadd v17, v9
+;; @004f v20 = uextend.i64 v16
+;; @004f v21 = icmp ule v19, v20
+;; @004f trapz v21, heap_oob
+;; @004f v22 = load.i32 notrap aligned can_move v0+40
+;; @004f v14 = iadd v13, v2
+;; @004f v23 = iadd v22, v3
+;; @004f call fn0(v0, v14, v23, v4)
+;; @0053 jump block1
+;;
+;; block1:
+;; @0053 return
+;; }
+;;
+;; function u0:2(i32 vmctx, i32, i64, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+44
+;; gv2 = load.i32 notrap aligned can_move gv0+40
+;; gv3 = load.i32 notrap aligned gv0+60
+;; gv4 = load.i32 notrap aligned can_move gv0+56
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:4 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i64, v3: i32, v4: i32):
+;; @005c v8 = load.i32 notrap aligned v0+60
+;; @005c v6 = uextend.i64 v4
+;; @005c v9 = uadd_overflow_trap v2, v6, heap_oob
+;; @005c v10 = uextend.i64 v8
+;; @005c v11 = icmp ule v9, v10
+;; @005c trapz v11, heap_oob
+;; @005c v13 = load.i32 notrap aligned can_move v0+56
+;; @005c v16 = load.i32 notrap aligned v0+44
+;; @005c v17 = uextend.i64 v3
+;; @005c v19 = iadd v17, v6
+;; @005c v20 = uextend.i64 v16
+;; @005c v21 = icmp ule v19, v20
+;; @005c trapz v21, heap_oob
+;; @005c v22 = load.i32 notrap aligned can_move v0+40
+;; @005c v12 = ireduce.i32 v2
+;; @005c v14 = iadd v13, v12
+;; @005c v23 = iadd v22, v3
+;; @005c call fn0(v0, v14, v23, v4)
+;; @0060 jump block1
+;;
+;; block1:
+;; @0060 return
+;; }
+;;
+;; function u0:3(i32 vmctx, i32, i64, i64, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+60
+;; gv2 = load.i32 notrap aligned can_move gv0+56
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:4 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i64, v3: i64, v4: i64):
+;; @0069 v7 = load.i32 notrap aligned v0+60
+;; @0069 v8 = uadd_overflow_trap v2, v4, heap_oob
+;; @0069 v9 = uextend.i64 v7
+;; @0069 v10 = icmp ule v8, v9
+;; @0069 trapz v10, heap_oob
+;; @0069 v12 = load.i32 notrap aligned can_move v0+56
+;; @0069 v16 = uadd_overflow_trap v3, v4, heap_oob
+;; @0069 v18 = icmp ule v16, v9
+;; @0069 trapz v18, heap_oob
+;; @0069 v11 = ireduce.i32 v2
+;; @0069 v13 = iadd v12, v11
+;; @0069 v19 = ireduce.i32 v3
+;; @0069 v21 = iadd v12, v19
+;; @0069 v22 = ireduce.i32 v4
+;; @0069 call fn0(v0, v13, v21, v22)
+;; @006d jump block1
+;;
+;; block1:
+;; @006d return
+;; }
+;;
+;; function u0:4(i32 vmctx, i32, i64, i64, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+60
+;; gv2 = load.i32 notrap aligned can_move gv0+56
+;; gv3 = load.i32 notrap aligned gv0+68
+;; gv4 = load.i32 notrap aligned can_move gv0+64
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:4 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i64, v3: i64, v4: i64):
+;; @0076 v7 = load.i32 notrap aligned v0+68
+;; @0076 v8 = uadd_overflow_trap v2, v4, heap_oob
+;; @0076 v9 = uextend.i64 v7
+;; @0076 v10 = icmp ule v8, v9
+;; @0076 trapz v10, heap_oob
+;; @0076 v12 = load.i32 notrap aligned can_move v0+64
+;; @0076 v15 = load.i32 notrap aligned v0+60
+;; @0076 v16 = uadd_overflow_trap v3, v4, heap_oob
+;; @0076 v17 = uextend.i64 v15
+;; @0076 v18 = icmp ule v16, v17
+;; @0076 trapz v18, heap_oob
+;; @0076 v20 = load.i32 notrap aligned can_move v0+56
+;; @0076 v11 = ireduce.i32 v2
+;; @0076 v13 = iadd v12, v11
+;; @0076 v19 = ireduce.i32 v3
+;; @0076 v21 = iadd v20, v19
+;; @0076 v22 = ireduce.i32 v4
+;; @0076 call fn0(v0, v13, v21, v22)
+;; @007a jump block1
+;;
+;; block1:
+;; @007a return
+;; }
+;;
+;; function u0:5(i32 vmctx, i32, i32, i64, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i32 notrap aligned gv0+60
+;; gv2 = load.i32 notrap aligned can_move gv0+56
+;; gv3 = load.i32 notrap aligned gv0+44
+;; gv4 = load.i32 notrap aligned can_move gv0+40
+;; sig0 = (i32 vmctx, i32, i32, i32) tail
+;; fn0 = colocated u805306368:4 sig0
+;;
+;; block0(v0: i32, v1: i32, v2: i32, v3: i64, v4: i32):
+;; @0083 v8 = load.i32 notrap aligned v0+44
+;; @0083 v9 = uextend.i64 v2
+;; @0083 v6 = uextend.i64 v4
+;; @0083 v11 = iadd v9, v6
+;; @0083 v12 = uextend.i64 v8
+;; @0083 v13 = icmp ule v11, v12
+;; @0083 trapz v13, heap_oob
+;; @0083 v14 = load.i32 notrap aligned can_move v0+40
+;; @0083 v17 = load.i32 notrap aligned v0+60
+;; @0083 v18 = uadd_overflow_trap v3, v6, heap_oob
+;; @0083 v19 = uextend.i64 v17
+;; @0083 v20 = icmp ule v18, v19
+;; @0083 trapz v20, heap_oob
+;; @0083 v22 = load.i32 notrap aligned can_move v0+56
+;; @0083 v15 = iadd v14, v2
+;; @0083 v21 = ireduce.i32 v3
+;; @0083 v23 = iadd v22, v21
+;; @0083 call fn0(v0, v15, v23, v4)
+;; @0087 jump block1
+;;
+;; block1:
+;; @0087 return
+;; }
diff --git a/tests/disas/memory_copy_host64.wat b/tests/disas/memory_copy_host64.wat
new file mode 100644
index 0000000000..9b4611fb5f
--- /dev/null
+++ b/tests/disas/memory_copy_host64.wat
@@ -0,0 +1,250 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+
+(module
+ (memory $m32_a 1)
+ (memory $m32_b 1)
+ (memory $m64_a i64 1)
+ (memory $m64_b i64 1)
+
+ (func $m32_to_same (param i32 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_a $m32_a
+ )
+
+ (func $m32_to_m32 (param i32 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_b $m32_a
+ )
+
+ (func $m32_to_m64 (param i64 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_a $m32_a
+ )
+
+ (func $m64_to_same (param i64 i64 i64)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_a $m64_a
+ )
+
+ (func $m64_to_m64 (param i64 i64 i64)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_b $m64_a
+ )
+
+ (func $m64_to_m32 (param i32 i64 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_a $m64_a
+ )
+)
+;; function u0:0(i64 vmctx, i64, i32, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+88
+;; gv5 = load.i64 notrap aligned readonly can_move gv3+80
+;; sig0 = (i64 vmctx, i64, i64, i64) tail
+;; fn0 = colocated u805306368:4 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):
+;; @0042 v7 = load.i64 notrap aligned v0+88
+;; @0042 v8 = uextend.i64 v2
+;; @0042 v9 = uextend.i64 v4
+;; @0042 v10 = iadd v8, v9
+;; @0042 v11 = icmp ule v10, v7
+;; @0042 trapz v11, heap_oob
+;; @0042 v17 = uextend.i64 v3
+;; @0042 v19 = iadd v17, v9
+;; @0042 v20 = icmp ule v19, v7
+;; @0042 trapz v20, heap_oob
+;; @0042 v13 = load.i64 notrap aligned readonly can_move v0+80
+;; @0042 v14 = iadd v13, v8
+;; @0042 v23 = iadd v13, v17
+;; @0042 call fn0(v0, v14, v23, v9)
+;; @0046 jump block1
+;;
+;; block1:
+;; @0046 return
+;; }
+;;
+;; function u0:1(i64 vmctx, i64, i32, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+88
+;; gv5 = load.i64 notrap aligned readonly can_move gv3+80
+;; gv6 = load.i64 notrap aligned gv3+104
+;; gv7 = load.i64 notrap aligned readonly can_move gv3+96
+;; sig0 = (i64 vmctx, i64, i64, i64) tail
+;; fn0 = colocated u805306368:4 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):
+;; @004f v7 = load.i64 notrap aligned v0+104
+;; @004f v8 = uextend.i64 v2
+;; @004f v9 = uextend.i64 v4
+;; @004f v10 = iadd v8, v9
+;; @004f v11 = icmp ule v10, v7
+;; @004f trapz v11, heap_oob
+;; @004f v16 = load.i64 notrap aligned v0+88
+;; @004f v17 = uextend.i64 v3
+;; @004f v19 = iadd v17, v9
+;; @004f v20 = icmp ule v19, v16
+;; @004f trapz v20, heap_oob
+;; @004f v13 = load.i64 notrap aligned readonly can_move v0+96
+;; @004f v14 = iadd v13, v8
+;; @004f v22 = load.i64 notrap aligned readonly can_move v0+80
+;; @004f v23 = iadd v22, v17
+;; @004f call fn0(v0, v14, v23, v9)
+;; @0053 jump block1
+;;
+;; block1:
+;; @0053 return
+;; }
+;;
+;; function u0:2(i64 vmctx, i64, i64, i32, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+88
+;; gv5 = load.i64 notrap aligned readonly can_move gv3+80
+;; gv6 = load.i64 notrap aligned gv3+120
+;; gv7 = load.i64 notrap aligned can_move gv3+112
+;; sig0 = (i64 vmctx, i64, i64, i64) tail
+;; fn0 = colocated u805306368:4 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i32, v4: i32):
+;; @005c v8 = load.i64 notrap aligned v0+120
+;; @005c v6 = uextend.i64 v4
+;; @005c v9 = uadd_overflow_trap v2, v6, heap_oob
+;; @005c v10 = icmp ule v9, v8
+;; @005c trapz v10, heap_oob
+;; @005c v11 = load.i64 notrap aligned can_move v0+112
+;; @005c v14 = load.i64 notrap aligned v0+88
+;; @005c v15 = uextend.i64 v3
+;; @005c v17 = iadd v15, v6
+;; @005c v18 = icmp ule v17, v14
+;; @005c trapz v18, heap_oob
+;; @005c v12 = iadd v11, v2
+;; @005c v20 = load.i64 notrap aligned readonly can_move v0+80
+;; @005c v21 = iadd v20, v15
+;; @005c call fn0(v0, v12, v21, v6)
+;; @0060 jump block1
+;;
+;; block1:
+;; @0060 return
+;; }
+;;
+;; function u0:3(i64 vmctx, i64, i64, i64, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+120
+;; gv5 = load.i64 notrap aligned can_move gv3+112
+;; sig0 = (i64 vmctx, i64, i64, i64) tail
+;; fn0 = colocated u805306368:4 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64):
+;; @0069 v7 = load.i64 notrap aligned v0+120
+;; @0069 v8 = uadd_overflow_trap v2, v4, heap_oob
+;; @0069 v9 = icmp ule v8, v7
+;; @0069 trapz v9, heap_oob
+;; @0069 v10 = load.i64 notrap aligned can_move v0+112
+;; @0069 v14 = uadd_overflow_trap v3, v4, heap_oob
+;; @0069 v15 = icmp ule v14, v7
+;; @0069 trapz v15, heap_oob
+;; @0069 v11 = iadd v10, v2
+;; @0069 v17 = iadd v10, v3
+;; @0069 call fn0(v0, v11, v17, v4)
+;; @006d jump block1
+;;
+;; block1:
+;; @006d return
+;; }
+;;
+;; function u0:4(i64 vmctx, i64, i64, i64, i64) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+120
+;; gv5 = load.i64 notrap aligned can_move gv3+112
+;; gv6 = load.i64 notrap aligned gv3+136
+;; gv7 = load.i64 notrap aligned can_move gv3+128
+;; sig0 = (i64 vmctx, i64, i64, i64) tail
+;; fn0 = colocated u805306368:4 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64):
+;; @0076 v7 = load.i64 notrap aligned v0+136
+;; @0076 v8 = uadd_overflow_trap v2, v4, heap_oob
+;; @0076 v9 = icmp ule v8, v7
+;; @0076 trapz v9, heap_oob
+;; @0076 v10 = load.i64 notrap aligned can_move v0+128
+;; @0076 v13 = load.i64 notrap aligned v0+120
+;; @0076 v14 = uadd_overflow_trap v3, v4, heap_oob
+;; @0076 v15 = icmp ule v14, v13
+;; @0076 trapz v15, heap_oob
+;; @0076 v16 = load.i64 notrap aligned can_move v0+112
+;; @0076 v11 = iadd v10, v2
+;; @0076 v17 = iadd v16, v3
+;; @0076 call fn0(v0, v11, v17, v4)
+;; @007a jump block1
+;;
+;; block1:
+;; @007a return
+;; }
+;;
+;; function u0:5(i64 vmctx, i64, i32, i64, i32) tail {
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly gv0+8
+;; gv2 = load.i64 notrap aligned gv1+24
+;; gv3 = vmctx
+;; gv4 = load.i64 notrap aligned gv3+120
+;; gv5 = load.i64 notrap aligned can_move gv3+112
+;; gv6 = load.i64 notrap aligned gv3+88
+;; gv7 = load.i64 notrap aligned readonly can_move gv3+80
+;; sig0 = (i64 vmctx, i64, i64, i64) tail
+;; fn0 = colocated u805306368:4 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32, v3: i64, v4: i32):
+;; @0083 v8 = load.i64 notrap aligned v0+88
+;; @0083 v9 = uextend.i64 v2
+;; @0083 v6 = uextend.i64 v4
+;; @0083 v11 = iadd v9, v6
+;; @0083 v12 = icmp ule v11, v8
+;; @0083 trapz v12, heap_oob
+;; @0083 v17 = load.i64 notrap aligned v0+120
+;; @0083 v18 = uadd_overflow_trap v3, v6, heap_oob
+;; @0083 v19 = icmp ule v18, v17
+;; @0083 trapz v19, heap_oob
+;; @0083 v20 = load.i64 notrap aligned can_move v0+112
+;; @0083 v14 = load.i64 notrap aligned readonly can_move v0+80
+;; @0083 v15 = iadd v14, v9
+;; @0083 v21 = iadd v20, v3
+;; @0083 call fn0(v0, v15, v21, v6)
+;; @0087 jump block1
+;;
+;; block1:
+;; @0087 return
+;; }
diff --git a/tests/disas/memory_fill_host32.wat b/tests/disas/memory_fill_host32.wat
index 61ec4847ad..89b68cf39a 100644
--- a/tests/disas/memory_fill_host32.wat
+++ b/tests/disas/memory_fill_host32.wat
@@ -54,13 +54,13 @@
;; fn0 = colocated u805306368:5 sig0
;;
;; block0(v0: i32, v1: i32, v2: i32, v3: i32):
-;; @003c v6 = load.i32 notrap aligned v0+48
-;; @003c v7 = uextend.i64 v2
-;; @003c v8 = uextend.i64 v3
-;; @003c v9 = iadd v7, v8
-;; @003c v10 = uextend.i64 v6
-;; @003c v11 = icmp ule v9, v10
-;; @003c trapz v11, heap_oob
+;; @003c v7 = load.i32 notrap aligned v0+48
+;; @003c v8 = uextend.i64 v2
+;; @003c v9 = uextend.i64 v3
+;; @003c v10 = iadd v8, v9
+;; @003c v11 = uextend.i64 v7
+;; @003c v12 = icmp ule v10, v11
+;; @003c trapz v12, heap_oob
;; @003c v13 = load.i32 notrap aligned can_move v0+44
;; @003c v14 = iadd v13, v2
;; @0038 v4 = iconst.i32 0
@@ -79,11 +79,11 @@
;; fn0 = colocated u805306368:5 sig0
;;
;; block0(v0: i32, v1: i32, v2: i64, v3: i64):
-;; @0048 v6 = load.i32 notrap aligned v0+56
-;; @0048 v7 = uadd_overflow_trap v2, v3, heap_oob
-;; @0048 v8 = uextend.i64 v6
-;; @0048 v9 = icmp ule v7, v8
-;; @0048 trapz v9, heap_oob
+;; @0048 v7 = load.i32 notrap aligned v0+56
+;; @0048 v8 = uadd_overflow_trap v2, v3, heap_oob
+;; @0048 v9 = uextend.i64 v7
+;; @0048 v10 = icmp ule v8, v9
+;; @0048 trapz v10, heap_oob
;; @0048 v12 = load.i32 notrap aligned can_move v0+52
;; @0048 v11 = ireduce.i32 v2
;; @0048 v13 = iadd v12, v11
@@ -104,13 +104,13 @@
;; fn0 = colocated u805306368:5 sig0
;;
;; block0(v0: i32, v1: i32, v2: i32, v3: i32):
-;; @0054 v6 = load.i32 notrap aligned v0+64
-;; @0054 v7 = uextend.i64 v2
-;; @0054 v8 = uextend.i64 v3
-;; @0054 v9 = iadd v7, v8
-;; @0054 v10 = uextend.i64 v6
-;; @0054 v11 = icmp ule v9, v10
-;; @0054 trapz v11, heap_oob
+;; @0054 v7 = load.i32 notrap aligned v0+64
+;; @0054 v8 = uextend.i64 v2
+;; @0054 v9 = uextend.i64 v3
+;; @0054 v10 = iadd v8, v9
+;; @0054 v11 = uextend.i64 v7
+;; @0054 v12 = icmp ule v10, v11
+;; @0054 trapz v12, heap_oob
;; @0054 v13 = load.i32 notrap aligned can_move v0+60
;; @0054 v14 = iadd v13, v2
;; @0050 v4 = iconst.i32 0
@@ -129,11 +129,11 @@
;; fn0 = colocated u805306368:5 sig0
;;
;; block0(v0: i32, v1: i32, v2: i64, v3: i64):
-;; @0060 v6 = load.i32 notrap aligned v0+72
-;; @0060 v7 = uadd_overflow_trap v2, v3, heap_oob
-;; @0060 v8 = uextend.i64 v6
-;; @0060 v9 = icmp ule v7, v8
-;; @0060 trapz v9, heap_oob
+;; @0060 v7 = load.i32 notrap aligned v0+72
+;; @0060 v8 = uadd_overflow_trap v2, v3, heap_oob
+;; @0060 v9 = uextend.i64 v7
+;; @0060 v10 = icmp ule v8, v9
+;; @0060 trapz v10, heap_oob
;; @0060 v12 = load.i32 notrap aligned can_move v0+68
;; @0060 v11 = ireduce.i32 v2
;; @0060 v13 = iadd v12, v11
@@ -154,13 +154,13 @@
;; fn0 = colocated u805306368:5 sig0
;;
;; block0(v0: i32, v1: i32, v2: i32, v3: i32):
-;; @006c v6 = load.i32 notrap aligned v0+80
-;; @006c v7 = uextend.i64 v2
-;; @006c v8 = uextend.i64 v3
-;; @006c v9 = iadd v7, v8
-;; @006c v10 = uextend.i64 v6
-;; @006c v11 = icmp ule v9, v10
-;; @006c trapz v11, heap_oob
+;; @006c v7 = load.i32 notrap aligned v0+80
+;; @006c v8 = uextend.i64 v2
+;; @006c v9 = uextend.i64 v3
+;; @006c v10 = iadd v8, v9
+;; @006c v11 = uextend.i64 v7
+;; @006c v12 = icmp ule v10, v11
+;; @006c trapz v12, heap_oob
;; @006c v13 = load.i32 notrap aligned readonly can_move v0+76
;; @006c v14 = iadd v13, v2
;; @0068 v4 = iconst.i32 0
diff --git a/tests/disas/memory_fill_host64.wat b/tests/disas/memory_fill_host64.wat
index af8650e35d..5f4c497ecc 100644
--- a/tests/disas/memory_fill_host64.wat
+++ b/tests/disas/memory_fill_host64.wat
@@ -55,16 +55,16 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32):
-;; @003c v6 = load.i64 notrap aligned v0+96
-;; @003c v7 = uextend.i64 v2
-;; @003c v8 = uextend.i64 v3
-;; @003c v9 = iadd v7, v8
-;; @003c v10 = icmp ule v9, v6
-;; @003c trapz v10, heap_oob
+;; @003c v7 = load.i64 notrap aligned v0+96
+;; @003c v8 = uextend.i64 v2
+;; @003c v9 = uextend.i64 v3
+;; @003c v10 = iadd v8, v9
+;; @003c v11 = icmp ule v10, v7
+;; @003c trapz v11, heap_oob
;; @003c v13 = load.i64 notrap aligned readonly can_move v0+88
-;; @003c v14 = iadd v13, v7
+;; @003c v14 = iadd v13, v8
;; @0038 v4 = iconst.i32 0
-;; @003c call fn0(v0, v14, v4, v8) ; v4 = 0
+;; @003c call fn0(v0, v14, v4, v9) ; v4 = 0
;; @003f jump block1
;;
;; block1:
@@ -83,10 +83,10 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
-;; @0048 v6 = load.i64 notrap aligned v0+112
-;; @0048 v7 = uadd_overflow_trap v2, v3, heap_oob
-;; @0048 v8 = icmp ule v7, v6
-;; @0048 trapz v8, heap_oob
+;; @0048 v7 = load.i64 notrap aligned v0+112
+;; @0048 v8 = uadd_overflow_trap v2, v3, heap_oob
+;; @0048 v9 = icmp ule v8, v7
+;; @0048 trapz v9, heap_oob
;; @0048 v10 = load.i64 notrap aligned can_move v0+104
;; @0048 v11 = iadd v10, v2
;; @0044 v4 = iconst.i32 0
@@ -109,16 +109,16 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32):
-;; @0054 v6 = load.i64 notrap aligned v0+128
-;; @0054 v7 = uextend.i64 v2
-;; @0054 v8 = uextend.i64 v3
-;; @0054 v9 = iadd v7, v8
-;; @0054 v10 = icmp ule v9, v6
-;; @0054 trapz v10, heap_oob
+;; @0054 v7 = load.i64 notrap aligned v0+128
+;; @0054 v8 = uextend.i64 v2
+;; @0054 v9 = uextend.i64 v3
+;; @0054 v10 = iadd v8, v9
+;; @0054 v11 = icmp ule v10, v7
+;; @0054 trapz v11, heap_oob
;; @0054 v13 = load.i64 notrap aligned readonly can_move v0+120
-;; @0054 v14 = iadd v13, v7
+;; @0054 v14 = iadd v13, v8
;; @0050 v4 = iconst.i32 0
-;; @0054 call fn0(v0, v14, v4, v8) ; v4 = 0
+;; @0054 call fn0(v0, v14, v4, v9) ; v4 = 0
;; @0057 jump block1
;;
;; block1:
@@ -137,10 +137,10 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i64, v3: i64):
-;; @0060 v6 = load.i64 notrap aligned v0+144
-;; @0060 v7 = uadd_overflow_trap v2, v3, heap_oob
-;; @0060 v8 = icmp ule v7, v6
-;; @0060 trapz v8, heap_oob
+;; @0060 v7 = load.i64 notrap aligned v0+144
+;; @0060 v8 = uadd_overflow_trap v2, v3, heap_oob
+;; @0060 v9 = icmp ule v8, v7
+;; @0060 trapz v9, heap_oob
;; @0060 v10 = load.i64 notrap aligned can_move v0+136
;; @0060 v11 = iadd v10, v2
;; @005c v4 = iconst.i32 0
@@ -163,16 +163,16 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32):
-;; @006c v6 = load.i64 notrap aligned v0+160
-;; @006c v7 = uextend.i64 v2
-;; @006c v8 = uextend.i64 v3
-;; @006c v9 = iadd v7, v8
-;; @006c v10 = icmp ule v9, v6
-;; @006c trapz v10, heap_oob
+;; @006c v7 = load.i64 notrap aligned v0+160
+;; @006c v8 = uextend.i64 v2
+;; @006c v9 = uextend.i64 v3
+;; @006c v10 = iadd v8, v9
+;; @006c v11 = icmp ule v10, v7
+;; @006c trapz v11, heap_oob
;; @006c v13 = load.i64 notrap aligned readonly can_move v0+152
-;; @006c v14 = iadd v13, v7
+;; @006c v14 = iadd v13, v8
;; @0068 v4 = iconst.i32 0
-;; @006c call fn0(v0, v14, v4, v8) ; v4 = 0
+;; @006c call fn0(v0, v14, v4, v9) ; v4 = 0
;; @006f jump block1
;;
;; block1:
diff --git a/tests/misc_testsuite/memory-copy.wast b/tests/misc_testsuite/memory-copy.wast
index 86633f6bc4..4b18d7f5d8 100644
--- a/tests/misc_testsuite/memory-copy.wast
+++ b/tests/misc_testsuite/memory-copy.wast
@@ -1,4 +1,6 @@
;;! bulk_memory = true
+;;! multi_memory = true
+;;! memory64 = true
(module
(memory 1 1)
@@ -124,3 +126,149 @@
(assert_return
(invoke "is olleh?" (i32.const 2000))
(i32.const 1))
+
+;; test trapping boundary behavior
+(module
+ (memory $m32_a 1)
+ (memory $m32_b 1)
+ (memory $m64_a i64 1)
+ (memory $m64_b i64 1)
+
+ (func (export "m32_to_same") (param i32 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_a $m32_a
+ )
+
+ (func (export "m32_to_m32") (param i32 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_b $m32_a
+ )
+
+ (func (export "m32_to_m64") (param i64 i32 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_a $m32_a
+ )
+
+ (func (export "m64_to_same") (param i64 i64 i64)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_a $m64_a
+ )
+
+ (func (export "m64_to_m64") (param i64 i64 i64)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m64_b $m64_a
+ )
+
+ (func (export "m64_to_m32") (param i32 i64 i32)
+ local.get 0
+ local.get 1
+ local.get 2
+ memory.copy $m32_a $m64_a
+ )
+)
+
+(assert_return (invoke "m32_to_same" (i32.const 0) (i32.const 0) (i32.const 0)))
+(assert_return (invoke "m32_to_same" (i32.const 0) (i32.const 0) (i32.const 65536)))
+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const 0) (i32.const 65537)) "out of bounds")
+(assert_return (invoke "m32_to_same" (i32.const 100) (i32.const 200) (i32.const 65336)))
+(assert_return (invoke "m32_to_same" (i32.const 200) (i32.const 100) (i32.const 65336)))
+(assert_trap (invoke "m32_to_same" (i32.const 201) (i32.const 100) (i32.const 65336)) "out of bounds")
+(assert_return (invoke "m32_to_same" (i32.const 200) (i32.const 101) (i32.const 65336)))
+(assert_trap (invoke "m32_to_same" (i32.const 200) (i32.const 100) (i32.const 65337)) "out of bounds")
+(assert_trap (invoke "m32_to_same" (i32.const -1) (i32.const 0) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const -1) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_same" (i32.const 100) (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const 100) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_same" (i32.const -1) (i32.const 0) (i32.const 100)) "out of bounds")
+(assert_trap (invoke "m32_to_same" (i32.const 0) (i32.const -1) (i32.const 100)) "out of bounds")
+
+(assert_return (invoke "m32_to_m32" (i32.const 0) (i32.const 0) (i32.const 0)))
+(assert_return (invoke "m32_to_m32" (i32.const 0) (i32.const 0) (i32.const 65536)))
+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const 0) (i32.const 65537)) "out of bounds")
+(assert_return (invoke "m32_to_m32" (i32.const 100) (i32.const 200) (i32.const 65336)))
+(assert_return (invoke "m32_to_m32" (i32.const 200) (i32.const 100) (i32.const 65336)))
+(assert_trap (invoke "m32_to_m32" (i32.const 201) (i32.const 100) (i32.const 65336)) "out of bounds")
+(assert_return (invoke "m32_to_m32" (i32.const 200) (i32.const 101) (i32.const 65336)))
+(assert_trap (invoke "m32_to_m32" (i32.const 200) (i32.const 100) (i32.const 65337)) "out of bounds")
+(assert_trap (invoke "m32_to_m32" (i32.const -1) (i32.const 0) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const -1) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_m32" (i32.const 100) (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const 100) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_m32" (i32.const -1) (i32.const 0) (i32.const 100)) "out of bounds")
+(assert_trap (invoke "m32_to_m32" (i32.const 0) (i32.const -1) (i32.const 100)) "out of bounds")
+
+(assert_return (invoke "m32_to_m64" (i64.const 0) (i32.const 0) (i32.const 0)))
+(assert_return (invoke "m32_to_m64" (i64.const 0) (i32.const 0) (i32.const 65536)))
+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const 0) (i32.const 65537)) "out of bounds")
+(assert_return (invoke "m32_to_m64" (i64.const 100) (i32.const 200) (i32.const 65336)))
+(assert_return (invoke "m32_to_m64" (i64.const 200) (i32.const 100) (i32.const 65336)))
+(assert_trap (invoke "m32_to_m64" (i64.const 201) (i32.const 100) (i32.const 65336)) "out of bounds")
+(assert_return (invoke "m32_to_m64" (i64.const 200) (i32.const 101) (i32.const 65336)))
+(assert_trap (invoke "m32_to_m64" (i64.const 200) (i32.const 100) (i32.const 65337)) "out of bounds")
+(assert_trap (invoke "m32_to_m64" (i64.const -1) (i32.const 0) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const -1) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_m64" (i64.const 100) (i32.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const 100) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m32_to_m64" (i64.const -1) (i32.const 0) (i32.const 100)) "out of bounds")
+(assert_trap (invoke "m32_to_m64" (i64.const 0) (i32.const -1) (i32.const 100)) "out of bounds")
+
+(assert_return (invoke "m64_to_same" (i64.const 0) (i64.const 0) (i64.const 0)))
+(assert_return (invoke "m64_to_same" (i64.const 0) (i64.const 0) (i64.const 65536)))
+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const 0) (i64.const 65537)) "out of bounds")
+(assert_return (invoke "m64_to_same" (i64.const 100) (i64.const 200) (i64.const 65336)))
+(assert_return (invoke "m64_to_same" (i64.const 200) (i64.const 100) (i64.const 65336)))
+(assert_trap (invoke "m64_to_same" (i64.const 201) (i64.const 100) (i64.const 65336)) "out of bounds")
+(assert_return (invoke "m64_to_same" (i64.const 200) (i64.const 101) (i64.const 65336)))
+(assert_trap (invoke "m64_to_same" (i64.const 200) (i64.const 100) (i64.const 65337)) "out of bounds")
+(assert_trap (invoke "m64_to_same" (i64.const -1) (i64.const 0) (i64.const 0)) "out of bounds")
+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const -1) (i64.const 0)) "out of bounds")
+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const 0) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_same" (i64.const 100) (i64.const 0) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const 100) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_same" (i64.const -1) (i64.const 0) (i64.const 100)) "out of bounds")
+(assert_trap (invoke "m64_to_same" (i64.const 0) (i64.const -1) (i64.const 100)) "out of bounds")
+
+(assert_return (invoke "m64_to_m64" (i64.const 0) (i64.const 0) (i64.const 0)))
+(assert_return (invoke "m64_to_m64" (i64.const 0) (i64.const 0) (i64.const 65536)))
+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const 0) (i64.const 65537)) "out of bounds")
+(assert_return (invoke "m64_to_m64" (i64.const 100) (i64.const 200) (i64.const 65336)))
+(assert_return (invoke "m64_to_m64" (i64.const 200) (i64.const 100) (i64.const 65336)))
+(assert_trap (invoke "m64_to_m64" (i64.const 201) (i64.const 100) (i64.const 65336)) "out of bounds")
+(assert_return (invoke "m64_to_m64" (i64.const 200) (i64.const 101) (i64.const 65336)))
+(assert_trap (invoke "m64_to_m64" (i64.const 200) (i64.const 100) (i64.const 65337)) "out of bounds")
+(assert_trap (invoke "m64_to_m64" (i64.const -1) (i64.const 0) (i64.const 0)) "out of bounds")
+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const -1) (i64.const 0)) "out of bounds")
+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const 0) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_m64" (i64.const 100) (i64.const 0) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const 100) (i64.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_m64" (i64.const -1) (i64.const 0) (i64.const 100)) "out of bounds")
+(assert_trap (invoke "m64_to_m64" (i64.const 0) (i64.const -1) (i64.const 100)) "out of bounds")
+
+(assert_return (invoke "m64_to_m32" (i32.const 0) (i64.const 0) (i32.const 0)))
+(assert_return (invoke "m64_to_m32" (i32.const 0) (i64.const 0) (i32.const 65536)))
+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const 0) (i32.const 65537)) "out of bounds")
+(assert_return (invoke "m64_to_m32" (i32.const 100) (i64.const 200) (i32.const 65336)))
+(assert_return (invoke "m64_to_m32" (i32.const 200) (i64.const 100) (i32.const 65336)))
+(assert_trap (invoke "m64_to_m32" (i32.const 201) (i64.const 100) (i32.const 65336)) "out of bounds")
+(assert_return (invoke "m64_to_m32" (i32.const 200) (i64.const 101) (i32.const 65336)))
+(assert_trap (invoke "m64_to_m32" (i32.const 200) (i64.const 100) (i32.const 65337)) "out of bounds")
+(assert_trap (invoke "m64_to_m32" (i32.const -1) (i64.const 0) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const -1) (i32.const 0)) "out of bounds")
+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_m32" (i32.const 100) (i64.const 0) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const 100) (i32.const -1)) "out of bounds")
+(assert_trap (invoke "m64_to_m32" (i32.const -1) (i64.const 0) (i32.const 100)) "out of bounds")
+(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const -1) (i32.const 100)) "out of bounds")
diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs
index ec4c460225..f0a34b163c 100644
--- a/winch/codegen/src/codegen/mod.rs
+++ b/winch/codegen/src/codegen/mod.rs
@@ -1130,40 +1130,39 @@ where
Ok(())
}
- /// Emit the `memory.fill` operation.
- pub fn emit_memory_fill(&mut self, mem: MemoryIndex) -> Result<()> {
- let heap = self.env.resolve_heap(mem);
+ /// Emit a bounds check for `ptr+len` and put the native address for this
+ /// wasm address into `dst`.
+ fn emit_bounds_check_and_compute_addr(
+ &mut self,
+ heap: &HeapData,
+ dst: Reg,
+ ptr: Reg,
+ len: Reg,
+ ) -> Result<()> {
let ptr_size: OperandSize = self.env.ptr_type().try_into()?;
let idx_size: OperandSize = heap.index_type().try_into()?;
-
- // The wasm stack at this point is `[dst, val, len]`.
- let len = self.context.pop_to_reg(self.masm, None)?;
- let val = self.context.pop_to_reg(self.masm, None)?;
- let dst = self.context.pop_to_reg(self.masm, None)?;
-
- // Compute `end = dst + len` trapping on overflow. For an `i32` index
+ // Compute `dst = ptr + len` trapping on overflow. For an `i32` index
// type the operands are zero-extended to 64-bit so overflow is
// impossible.
- let end = self.context.any_gpr(self.masm)?;
match idx_size {
OperandSize::S32 => {
self.masm
- .extend(writable!(end), dst.reg, Extend::<Zero>::I64Extend32.into())?;
+ .extend(writable!(dst), ptr, Extend::<Zero>::I64Extend32.into())?;
self.masm.add_uextend(
- writable!(end),
- end,
- len.reg,
+ writable!(dst),
+ dst,
+ len,
OperandSize::S32,
OperandSize::S64,
)?;
}
OperandSize::S64 => {
self.masm
- .mov(writable!(end), dst.reg.into(), OperandSize::S64)?;
+ .mov(writable!(dst), ptr.into(), OperandSize::S64)?;
self.masm.checked_uadd(
- writable!(end),
- end,
- len.reg.into(),
+ writable!(dst),
+ dst,
+ len.into(),
OperandSize::S64,
TrapCode::HEAP_OUT_OF_BOUNDS,
)?;
@@ -1171,29 +1170,94 @@ where
_ => unreachable!(),
}
- // Load the current size in bytes of the memory.
+ // Load the current size in bytes of the memory, and trap if
+ // `dst > size_in_bytes`.
let size_in_bytes = self.context.any_gpr(self.masm)?;
self.load_memory_length(&heap, size_in_bytes)?;
-
- // Trap if `end > size_in_bytes`.
assert!(ptr_size == OperandSize::S64);
- self.masm.cmp(end, size_in_bytes.into(), ptr_size)?;
+ self.masm.cmp(dst, size_in_bytes.into(), ptr_size)?;
self.masm
.trapif(IntCmpKind::GtU, TrapCode::HEAP_OUT_OF_BOUNDS)?;
- self.context.free_reg(end);
self.context.free_reg(size_in_bytes);
- // Compute `dst_ptr = memory_base + dst`.
- let dst_ptr = self.context.any_gpr(self.masm)?;
+ // Compute `dst = memory_base + ptr`.
bounds::load_heap_addr_unchecked(
self.masm,
&heap,
- Index::from_typed_reg(dst),
+ Index::from_typed_reg(TypedReg::new(heap.index_type(), ptr)),
ImmOffset::from_u32(0),
- dst_ptr,
+ dst,
ptr_size,
)?;
- self.context.free_reg(dst.reg);
+ Ok(())
+ }
+
+ /// Emit the `memory.copy` operation.
+ pub fn emit_memory_copy(&mut self, dst_mem: MemoryIndex, src_mem: MemoryIndex) -> Result<()> {
+ let dst_heap = self.env.resolve_heap(dst_mem);
+ let src_heap = self.env.resolve_heap(src_mem);
+ let dst_idx_size: OperandSize = dst_heap.index_type().try_into()?;
+ let src_idx_size: OperandSize = src_heap.index_type().try_into()?;
+
+ let len = self.context.pop_to_reg(self.masm, None)?;
+ let src = self.context.pop_to_reg(self.masm, None)?;
+ let dst = self.context.pop_to_reg(self.masm, None)?;
+
+ // For 32-bit linear memories go ahead and make sure `len` is zero
+ // extended within its register ensuring that the full 64-bits of the
+ // register are defined. This assists in situations like cross-memory
+ // copies where one memory is 32-bit and one is 64-bit and the same
+ // register can be used for the length in both bounds checks below.
+ if dst_idx_size == OperandSize::S32 || src_idx_size == OperandSize::S32 {
+ self.masm.extend(
+ writable!(len.reg),
+ len.reg,
+ Extend::<Zero>::I64Extend32.into(),
+ )?;
+ }
+
+ let dst_raw_addr = self.context.any_gpr(self.masm)?;
+ self.emit_bounds_check_and_compute_addr(&dst_heap, dst_raw_addr, dst.reg, len.reg)?;
+ self.context.free_reg(dst);
+
+ let src_raw_addr = self.context.any_gpr(self.masm)?;
+ self.emit_bounds_check_and_compute_addr(&src_heap, src_raw_addr, src.reg, len.reg)?;
+ self.context.free_reg(src);
+
+ self.context
+ .stack
+ .push(TypedReg::new(self.env.ptr_type(), dst_raw_addr).into());
+ self.context
+ .stack
+ .push(TypedReg::new(self.env.ptr_type(), src_raw_addr).into());
+ self.context
+ .stack
+ .push(TypedReg::new(self.env.ptr_type(), len.reg).into());
+
+ let builtin = self.env.builtins.memory_copy::<M::ABI>()?;
+ FnCall::emit::<M>(
+ &mut self.env,
+ self.masm,
+ &mut self.context,
+ Callee::Builtin(builtin),
+ )?;
+ Ok(())
+ }
+
+ /// Emit the `memory.fill` operation.
+ pub fn emit_memory_fill(&mut self, mem: MemoryIndex) -> Result<()> {
+ let heap = self.env.resolve_heap(mem);
+ let ptr_size: OperandSize = self.env.ptr_type().try_into()?;
+ let idx_size: OperandSize = heap.index_type().try_into()?;
+
+ // The wasm stack at this point is `[dst, val, len]`.
+ let len = self.context.pop_to_reg(self.masm, None)?;
+ let val = self.context.pop_to_reg(self.masm, None)?;
+ let dst = self.context.pop_to_reg(self.masm, None)?;
+
+ let raw_addr = self.context.any_gpr(self.masm)?;
+ self.emit_bounds_check_and_compute_addr(&heap, raw_addr, dst.reg, len.reg)?;
+ self.context.free_reg(dst);
// The libcall takes the length as a host-pointer-sized integer, so
// zero-extend if the wasm index type is smaller.
@@ -1209,7 +1273,7 @@ where
// Set up the call arguments: `[dst_ptr, val, len]`.
self.context
.stack
- .push(TypedReg::new(self.env.ptr_type(), dst_ptr).into());
+ .push(TypedReg::new(self.env.ptr_type(), raw_addr).into());
self.context.stack.push(val.into());
self.context
.stack
diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs
index 138c10d51a..f908379b5e 100644
--- a/winch/codegen/src/visitor.rs
+++ b/winch/codegen/src/visitor.rs
@@ -1891,28 +1891,10 @@ where
}
fn visit_memory_copy(&mut self, dst_mem: u32, src_mem: u32) -> Self::Output {
- // At this point, the stack is expected to contain:
- // [ dst_offset, src_offset, len ]
- // The following code inserts the missing params, so that stack contains:
- // [ vmctx, dst_mem, dst_offset, src_mem, src_offset, len ]
- // Which is the order expected by the builtin function.
- let _ = self.context.stack.ensure_index_at(3)?;
- let at = self.context.stack.ensure_index_at(2)?;
- self.context.stack.insert_many(at, &[src_mem.try_into()?]);
-
- // One element was inserted above, so instead of 3, we use 4.
- let at = self.context.stack.ensure_index_at(4)?;
- self.context.stack.insert_many(at, &[dst_mem.try_into()?]);
-
- let builtin = self.env.builtins.memory_copy::<M::ABI>()?;
-
- FnCall::emit::<M>(
- &mut self.env,
- self.masm,
- &mut self.context,
- Callee::Builtin(builtin),
- )?;
- self.context.pop_and_free(self.masm)
+ self.emit_memory_copy(
+ MemoryIndex::from_u32(dst_mem),
+ MemoryIndex::from_u32(src_mem),
+ )
}
fn visit_memory_fill(&mut self, mem: u32) -> Self::Output {