Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Add separate libcalls for array.copy w/ vs. w/out GC ref elems (#13312)

Nick Fitzgerald4013d783297f
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index d9acc09612..19499bac9b 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -2802,7 +2802,7 @@ impl FuncEnvironment<'_> {
pub fn translate_array_copy(
&mut self,
builder: &mut FunctionBuilder,
- _dst_array_type_index: TypeIndex,
+ dst_array_type_index: TypeIndex,
dst_array: ir::Value,
dst_index: ir::Value,
_src_array_type_index: TypeIndex,
@@ -2810,12 +2810,38 @@ impl FuncEnvironment<'_> {
src_index: ir::Value,
len: ir::Value,
) -> WasmResult<()> {
- let libcall = gc::builtins::array_copy(self, builder.func)?;
+ let interned_type_index =
+ self.module.types[dst_array_type_index].unwrap_module_type_index();
+ let array_ty = self.types.unwrap_array(interned_type_index)?;
+ let elem_ty = array_ty.0.element_type;
+
let vmctx = self.vmctx_val(&mut builder.cursor());
- builder.ins().call(
- libcall,
- &[vmctx, dst_array, dst_index, src_array, src_index, len],
- );
+
+ if elem_ty.is_vmgcref_type_and_not_i31() {
+ let libcall = gc::builtins::array_copy_gc_ref_elems(self, builder.func)?;
+ builder.ins().call(
+ libcall,
+ &[vmctx, dst_array, dst_index, src_array, src_index, len],
+ );
+ } else {
+ let libcall = gc::builtins::array_copy_non_gc_ref_elems(self, builder.func)?;
+ let interned_type_index = builder
+ .ins()
+ .iconst(I32, i64::from(interned_type_index.as_u32()));
+ builder.ins().call(
+ libcall,
+ &[
+ vmctx,
+ interned_type_index,
+ dst_array,
+ dst_index,
+ src_array,
+ src_index,
+ len,
+ ],
+ );
+ }
+
Ok(())
}
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index c3c6c2b0b0..3ddc7a5156 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -207,7 +207,8 @@ pub mod builtins {
table_fill_gc_ref,
array_new_data,
array_new_elem,
- array_copy,
+ array_copy_gc_ref_elems,
+ array_copy_non_gc_ref_elems,
array_init_data,
array_init_elem,
}
diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs
index c1f11c252d..015797ffd6 100644
--- a/crates/environ/src/builtin.rs
+++ b/crates/environ/src/builtin.rs
@@ -145,9 +145,10 @@ macro_rules! foreach_builtin_function {
len: u32
) -> u32;
- // Builtin implementation of the `array.copy` instruction.
+ // Builtin implementation of `array.copy` for arrays whose
+ // elements are GC references.
#[cfg(feature = "gc")]
- array_copy(
+ array_copy_gc_ref_elems(
vmctx: vmctx,
dst_array: u32,
dst_index: u32,
@@ -156,6 +157,19 @@ macro_rules! foreach_builtin_function {
len: u32
) -> bool;
+ // Builtin implementation of `array.copy` for arrays whose
+ // elements are not GC references.
+ #[cfg(feature = "gc")]
+ array_copy_non_gc_ref_elems(
+ vmctx: vmctx,
+ array_interned_type_index: u32,
+ dst_array: u32,
+ dst_index: u32,
+ src_array: u32,
+ src_index: u32,
+ len: u32
+ ) -> bool;
+
// Builtin implementation of the `array.init_data` instruction.
#[cfg(feature = "gc")]
array_init_data(
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
index 21c7ad74af..21af992351 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/data.rs
@@ -204,6 +204,23 @@ impl VMGcObjectData {
into.copy_from_slice(src);
}
+ /// Copy within this this object's data.
+ ///
+ /// Note that GC data is always stored in little-endian order, and this
+ /// method does not do any conversions to/from host endianness for you.
+ ///
+ /// Panics on out-of-bounds accesses.
+ #[inline]
+ pub fn copy_within<R>(&mut self, src: R, dest: u32)
+ where
+ R: core::ops::RangeBounds<u32>,
+ {
+ let start = src.start_bound().map(|s| usize::try_from(*s).unwrap());
+ let end = src.end_bound().map(|e| usize::try_from(*e).unwrap());
+ let dest = usize::try_from(dest).unwrap();
+ self.data.copy_within((start, end), dest);
+ }
+
impl_pod_methods! {
u8, read_u8, write_u8;
u16, read_u16, write_u16;
diff --git a/crates/wasmtime/src/runtime/vm/libcalls.rs b/crates/wasmtime/src/runtime/vm/libcalls.rs
index 00d83582e8..5c4e4b2fff 100644
--- a/crates/wasmtime/src/runtime/vm/libcalls.rs
+++ b/crates/wasmtime/src/runtime/vm/libcalls.rs
@@ -1103,12 +1103,8 @@ fn array_init_elem(
Ok(())
}
-// TODO: Specialize this libcall for only non-GC array elements, so we never
-// have to do GC barriers and their associated indirect calls through the `dyn
-// GcHeap`. Instead, implement those copies inline in Wasm code. Then, use bulk
-// `memcpy`-style APIs to do the actual copies here.
#[cfg(feature = "gc")]
-fn array_copy(
+fn array_copy_gc_ref_elems(
store: &mut dyn VMStore,
_instance: InstanceId,
dst_array: u32,
@@ -1134,6 +1130,8 @@ fn array_copy(
let src_array = store.unwrap_gc_store_mut().clone_gc_ref(&src_array);
let src_array = ArrayRef::from_cloned_gc_ref(&mut store, src_array);
+ debug_assert!(dst_array.layout(&store).unwrap().elems_are_gc_refs);
+
// Bounds check the destination array's elements.
let dst_array_len = dst_array._len(&store)?;
if dst.checked_add(len).ok_or_else(|| Trap::ArrayOutOfBounds)? > dst_array_len {
@@ -1168,6 +1166,91 @@ fn array_copy(
Ok(())
}
+#[cfg(feature = "gc")]
+fn array_copy_non_gc_ref_elems(
+ store: &mut dyn VMStore,
+ instance_id: InstanceId,
+ array_type_index: u32,
+ dst_array: u32,
+ dst: u32,
+ src_array: u32,
+ src: u32,
+ len: u32,
+) -> Result<()> {
+ use wasmtime_environ::ModuleInternedTypeIndex;
+
+ log::trace!(
+ "array.copy non-gc-refs(dst_array={dst_array:#x}, dst_index={dst}, src_array={src_array:#x}, src_index={src}, len={len})",
+ );
+
+ let array_type_index = ModuleInternedTypeIndex::from_u32(array_type_index);
+
+ let same_array = dst_array == src_array;
+
+ // Null checks and conversion to `VMArrayRef`.
+ let dst_gc_ref = VMGcRef::from_raw_u32(dst_array).ok_or_else(|| Trap::NullReference)?;
+ let dst_arr = dst_gc_ref
+ .into_arrayref(&*store.unwrap_gc_store().gc_heap)
+ .expect("gc ref should be an array");
+ let src_gc_ref = VMGcRef::from_raw_u32(src_array).ok_or_else(|| Trap::NullReference)?;
+ let src_arr = src_gc_ref
+ .into_arrayref(&*store.unwrap_gc_store().gc_heap)
+ .expect("gc ref should be an array");
+
+ // Bounds check the destination array's elements.
+ let dst_len = dst_arr.len(store.store_opaque());
+ let dst_end = dst.checked_add(len).ok_or_else(|| Trap::ArrayOutOfBounds)?;
+ if dst_end > dst_len {
+ return Err(Trap::ArrayOutOfBounds.into());
+ }
+
+ // Bounds check the source array's elements.
+ let src_len = src_arr.len(store.store_opaque());
+ let src_end = src.checked_add(len).ok_or_else(|| Trap::ArrayOutOfBounds)?;
+ if src_end > src_len {
+ return Err(Trap::ArrayOutOfBounds.into());
+ }
+
+ // Get the array layout to compute byte offsets.
+ let instance = store.instance(instance_id);
+ let shared_ty = instance.engine_type_index(array_type_index);
+ let gc_layout = store
+ .engine()
+ .signatures()
+ .layout(shared_ty)
+ .expect("array types have GC layouts");
+ let array_layout = gc_layout.unwrap_array();
+ debug_assert!(!array_layout.elems_are_gc_refs);
+
+ let byte_len = len
+ .checked_mul(array_layout.elem_size)
+ .expect("already checked bounds");
+
+ let src_byte_start = array_layout.elem_offset(src).unwrap();
+ let dst_byte_start = array_layout.elem_offset(dst).unwrap();
+
+ // Use `core::ptr::copy` to do a bulk copy of the element data.
+ let gc_store = store.unwrap_gc_store_mut();
+ if same_array {
+ // Same array: potentially overlapping.
+ let src_byte_end = src_byte_start
+ .checked_add(byte_len)
+ .expect("already checked bounds");
+ gc_store
+ .gc_object_data(dst_arr.as_gc_ref())
+ .copy_within(src_byte_start..src_byte_end, dst_byte_start);
+ } else {
+ // Different arrays: non-overlapping.
+ let (src_data, dst_data) =
+ gc_store.gc_object_data_pair(src_arr.as_gc_ref(), dst_arr.as_gc_ref());
+ let src_slice = src_data.slice(src_byte_start, byte_len);
+ let dst_slice = dst_data.slice_mut(dst_byte_start, byte_len);
+ dst_slice.copy_from_slice(src_slice);
+ }
+
+ Ok(())
+}
+
#[cfg(feature = "gc")]
fn is_subtype(
store: &mut dyn VMStore,
diff --git a/tests/disas/component-may-leave-without-signals-based-traps.wat b/tests/disas/component-may-leave-without-signals-based-traps.wat
index e67ea17b34..20574ffedc 100644
--- a/tests/disas/component-may-leave-without-signals-based-traps.wat
+++ b/tests/disas/component-may-leave-without-signals-based-traps.wat
@@ -44,7 +44,7 @@
;; retq
;; 129: movq %rbx, %rsi
;; 12c: movq 0x10(%rsi), %rax
-;; 130: movq 0x198(%rax), %rax
+;; 130: movq 0x1a0(%rax), %rax
;; 137: movq %rbx, %rdi
;; 13a: callq *%rax
;; 13c: ud2
diff --git a/tests/disas/debug.wat b/tests/disas/debug.wat
index 2a8663ea0a..146965ebf8 100644
--- a/tests/disas/debug.wat
+++ b/tests/disas/debug.wat
@@ -128,7 +128,7 @@
;; popq %rbp
;; retq
;; 17a: movq 0x10(%rbx), %rax
-;; 17e: movq 0x198(%rax), %rax
+;; 17e: movq 0x1a0(%rax), %rax
;; 185: movq %rbx, %rdi
;; 188: callq *%rax
;; 18a: ud2
@@ -143,7 +143,7 @@
;; movq 8(%r10), %r11
;; movq %r11, 0x38(%r9)
;; movq 0x10(%rdi), %r11
-;; movq 0x190(%r11), %r11
+;; movq 0x198(%r11), %r11
;; movzbq %sil, %rsi
;; callq *%r11
;; movq %rbp, %rsp
@@ -160,7 +160,7 @@
;; movq 8(%r9), %r9
;; movq %r9, 0x38(%r8)
;; movq 0x10(%rdi), %r9
-;; movq 0x198(%r9), %r9
+;; movq 0x1a0(%r9), %r9
;; callq *%r9
;; movq %rbp, %rsp
;; popq %rbp
@@ -203,7 +203,7 @@
;; movq 8(%r11), %rax
;; movq %rax, 0x38(%r10)
;; movq 0x10(%rdi), %rax
-;; movq 0x1c8(%rax), %rcx
+;; movq 0x1d0(%rax), %rcx
;; movq %rdi, %rbx
;; callq *%rcx
;; testb %al, %al
@@ -239,7 +239,7 @@
;; popq %rbp
;; retq
;; 3af: movq 0x10(%rbx), %rax
-;; 3b3: movq 0x198(%rax), %rax
+;; 3b3: movq 0x1a0(%rax), %rax
;; 3ba: movq %rbx, %rdi
;; 3bd: callq *%rax
;; 3bf: ud2
diff --git a/tests/disas/gc/copying/br-on-cast-fail.wat b/tests/disas/gc/copying/br-on-cast-fail.wat
index 42f74ff490..870ebdd429 100644
--- a/tests/disas/gc/copying/br-on-cast-fail.wat
+++ b/tests/disas/gc/copying/br-on-cast-fail.wat
@@ -26,7 +26,7 @@
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
;; sig1 = (i64 vmctx, i64) tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/copying/br-on-cast.wat b/tests/disas/gc/copying/br-on-cast.wat
index e28bfeb3aa..749a37889a 100644
--- a/tests/disas/gc/copying/br-on-cast.wat
+++ b/tests/disas/gc/copying/br-on-cast.wat
@@ -26,7 +26,7 @@
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
;; sig1 = (i64 vmctx, i64) tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/copying/call-indirect-and-subtyping.wat b/tests/disas/gc/copying/call-indirect-and-subtyping.wat
index 2c18137d12..ae94992428 100644
--- a/tests/disas/gc/copying/call-indirect-and-subtyping.wat
+++ b/tests/disas/gc/copying/call-indirect-and-subtyping.wat
@@ -25,7 +25,7 @@
;; sig1 = (i64 vmctx, i32, i64) -> i64 tail
;; sig2 = (i64 vmctx, i32, i32) -> i32 tail
;; fn0 = colocated u805306368:9 sig1
-;; fn1 = colocated u805306368:35 sig2
+;; fn1 = colocated u805306368:36 sig2
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/copying/ref-cast.wat b/tests/disas/gc/copying/ref-cast.wat
index b773ad68ff..03df9934f1 100644
--- a/tests/disas/gc/copying/ref-cast.wat
+++ b/tests/disas/gc/copying/ref-cast.wat
@@ -17,7 +17,7 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/copying/ref-test-concrete-func-type.wat b/tests/disas/gc/copying/ref-test-concrete-func-type.wat
index 32fbcd1ab0..a90fec1307 100644
--- a/tests/disas/gc/copying/ref-test-concrete-func-type.wat
+++ b/tests/disas/gc/copying/ref-test-concrete-func-type.wat
@@ -13,7 +13,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i64):
diff --git a/tests/disas/gc/copying/ref-test-concrete-type.wat b/tests/disas/gc/copying/ref-test-concrete-type.wat
index c5d3d74300..b61febcda0 100644
--- a/tests/disas/gc/copying/ref-test-concrete-type.wat
+++ b/tests/disas/gc/copying/ref-test-concrete-type.wat
@@ -16,7 +16,7 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/drc/br-on-cast-fail.wat b/tests/disas/gc/drc/br-on-cast-fail.wat
index 3507a5f0e6..f0f893c70e 100644
--- a/tests/disas/gc/drc/br-on-cast-fail.wat
+++ b/tests/disas/gc/drc/br-on-cast-fail.wat
@@ -27,7 +27,7 @@
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
;; sig1 = (i64 vmctx, i64) tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/drc/br-on-cast.wat b/tests/disas/gc/drc/br-on-cast.wat
index b227f41284..3257ae627c 100644
--- a/tests/disas/gc/drc/br-on-cast.wat
+++ b/tests/disas/gc/drc/br-on-cast.wat
@@ -27,7 +27,7 @@
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
;; sig1 = (i64 vmctx, i64) tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/drc/call-indirect-and-subtyping.wat b/tests/disas/gc/drc/call-indirect-and-subtyping.wat
index cb02276312..e7fccc198d 100644
--- a/tests/disas/gc/drc/call-indirect-and-subtyping.wat
+++ b/tests/disas/gc/drc/call-indirect-and-subtyping.wat
@@ -26,7 +26,7 @@
;; sig1 = (i64 vmctx, i32, i64) -> i64 tail
;; sig2 = (i64 vmctx, i32, i32) -> i32 tail
;; fn0 = colocated u805306368:9 sig1
-;; fn1 = colocated u805306368:35 sig2
+;; fn1 = colocated u805306368:36 sig2
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/drc/ref-cast.wat b/tests/disas/gc/drc/ref-cast.wat
index 4f53a5cf13..7958e65935 100644
--- a/tests/disas/gc/drc/ref-cast.wat
+++ b/tests/disas/gc/drc/ref-cast.wat
@@ -18,7 +18,7 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/drc/ref-test-concrete-func-type.wat b/tests/disas/gc/drc/ref-test-concrete-func-type.wat
index a1f5d90904..d03258a7e4 100644
--- a/tests/disas/gc/drc/ref-test-concrete-func-type.wat
+++ b/tests/disas/gc/drc/ref-test-concrete-func-type.wat
@@ -14,7 +14,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i64):
diff --git a/tests/disas/gc/drc/ref-test-concrete-type.wat b/tests/disas/gc/drc/ref-test-concrete-type.wat
index 0ea03f50a4..8498db7253 100644
--- a/tests/disas/gc/drc/ref-test-concrete-type.wat
+++ b/tests/disas/gc/drc/ref-test-concrete-type.wat
@@ -17,7 +17,7 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/null/br-on-cast-fail.wat b/tests/disas/gc/null/br-on-cast-fail.wat
index 7f91e96cac..6392b1e12a 100644
--- a/tests/disas/gc/null/br-on-cast-fail.wat
+++ b/tests/disas/gc/null/br-on-cast-fail.wat
@@ -27,7 +27,7 @@
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
;; sig1 = (i64 vmctx, i64) tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/null/br-on-cast.wat b/tests/disas/gc/null/br-on-cast.wat
index 2ba7dab8c7..70ef4b9335 100644
--- a/tests/disas/gc/null/br-on-cast.wat
+++ b/tests/disas/gc/null/br-on-cast.wat
@@ -27,7 +27,7 @@
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
;; sig1 = (i64 vmctx, i64) tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/null/call-indirect-and-subtyping.wat b/tests/disas/gc/null/call-indirect-and-subtyping.wat
index 822ab92b7b..f6114c0441 100644
--- a/tests/disas/gc/null/call-indirect-and-subtyping.wat
+++ b/tests/disas/gc/null/call-indirect-and-subtyping.wat
@@ -26,7 +26,7 @@
;; sig1 = (i64 vmctx, i32, i64) -> i64 tail
;; sig2 = (i64 vmctx, i32, i32) -> i32 tail
;; fn0 = colocated u805306368:9 sig1
-;; fn1 = colocated u805306368:35 sig2
+;; fn1 = colocated u805306368:36 sig2
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/null/ref-cast.wat b/tests/disas/gc/null/ref-cast.wat
index 81fd44c6a5..cb4afa15a3 100644
--- a/tests/disas/gc/null/ref-cast.wat
+++ b/tests/disas/gc/null/ref-cast.wat
@@ -18,7 +18,7 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/gc/null/ref-test-concrete-func-type.wat b/tests/disas/gc/null/ref-test-concrete-func-type.wat
index 1911938a53..7b24dd1003 100644
--- a/tests/disas/gc/null/ref-test-concrete-func-type.wat
+++ b/tests/disas/gc/null/ref-test-concrete-func-type.wat
@@ -14,7 +14,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i64):
diff --git a/tests/disas/gc/null/ref-test-concrete-type.wat b/tests/disas/gc/null/ref-test-concrete-type.wat
index 87126b5942..698199c1ba 100644
--- a/tests/disas/gc/null/ref-test-concrete-type.wat
+++ b/tests/disas/gc/null/ref-test-concrete-type.wat
@@ -17,7 +17,7 @@
;; gv5 = load.i64 notrap aligned readonly can_move gv4+32
;; gv6 = load.i64 notrap aligned gv4+40
;; sig0 = (i64 vmctx, i32, i32) -> i32 tail
-;; fn0 = colocated u805306368:35 sig0
+;; fn0 = colocated u805306368:36 sig0
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32):
diff --git a/tests/disas/riscv64-component-builtins-asm.wat b/tests/disas/riscv64-component-builtins-asm.wat
index e56763251f..5d93705b9a 100644
--- a/tests/disas/riscv64-component-builtins-asm.wat
+++ b/tests/disas/riscv64-component-builtins-asm.wat
@@ -53,7 +53,7 @@
;; ret
;; mv a1, s4
;; ld a0, 0x10(a1)
-;; ld a2, 0x198(a0)
+;; ld a2, 0x1a0(a0)
;; mv a0, a1
;; jalr a2
;; .byte 0x00, 0x00, 0x00, 0x00
diff --git a/tests/disas/riscv64-component-builtins.wat b/tests/disas/riscv64-component-builtins.wat
index 4940688c43..67ee547433 100644
--- a/tests/disas/riscv64-component-builtins.wat
+++ b/tests/disas/riscv64-component-builtins.wat
@@ -34,7 +34,7 @@
;;
;; block1 cold:
;; v15 = load.i64 notrap aligned readonly v1+16
-;; v16 = load.i64 notrap aligned readonly v15+408
+;; v16 = load.i64 notrap aligned readonly v15+416
;; call_indirect sig1, v16(v1)
;; trap user1
;;
diff --git a/tests/disas/stack-switching/resume-suspend-data-passing.wat b/tests/disas/stack-switching/resume-suspend-data-passing.wat
index fa1472c2e4..436a9de699 100644
--- a/tests/disas/stack-switching/resume-suspend-data-passing.wat
+++ b/tests/disas/stack-switching/resume-suspend-data-passing.wat
@@ -153,7 +153,7 @@
;; sig0 = (i64 vmctx, i32) -> i64 tail
;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
;; fn0 = colocated u805306368:7 sig0
-;; fn1 = colocated u805306368:52 sig1
+;; fn1 = colocated u805306368:53 sig1
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64):
diff --git a/tests/disas/stack-switching/resume-suspend.wat b/tests/disas/stack-switching/resume-suspend.wat
index 8e84fc3e0c..8701a16710 100644
--- a/tests/disas/stack-switching/resume-suspend.wat
+++ b/tests/disas/stack-switching/resume-suspend.wat
@@ -114,7 +114,7 @@
;; sig0 = (i64 vmctx, i32) -> i64 tail
;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
;; fn0 = colocated u805306368:7 sig0
-;; fn1 = colocated u805306368:52 sig1
+;; fn1 = colocated u805306368:53 sig1
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64):
diff --git a/tests/disas/stack-switching/symmetric-switch.wat b/tests/disas/stack-switching/symmetric-switch.wat
index c1138303a5..dbbcfe0ef2 100644
--- a/tests/disas/stack-switching/symmetric-switch.wat
+++ b/tests/disas/stack-switching/symmetric-switch.wat
@@ -34,7 +34,7 @@
;; sig0 = (i64 vmctx, i32) -> i64 tail
;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
;; fn0 = colocated u805306368:7 sig0
-;; fn1 = colocated u805306368:52 sig1
+;; fn1 = colocated u805306368:53 sig1
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64):
@@ -249,7 +249,7 @@
;; sig0 = (i64 vmctx, i32) -> i64 tail
;; sig1 = (i64 vmctx, i64, i32, i32) -> i64 tail
;; fn0 = colocated u805306368:7 sig0
-;; fn1 = colocated u805306368:52 sig1
+;; fn1 = colocated u805306368:53 sig1
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64):
diff --git a/tests/disas/x64-simd-round-without-sse41.wat b/tests/disas/x64-simd-round-without-sse41.wat
index 259945d697..def5b2b8b5 100644
--- a/tests/disas/x64-simd-round-without-sse41.wat
+++ b/tests/disas/x64-simd-round-without-sse41.wat
@@ -17,7 +17,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, f32) -> f32 tail
-;; fn0 = colocated u805306368:38 sig0
+;; fn0 = colocated u805306368:39 sig0
;; const0 = 0x00000000000000000000000000000000
;; stack_limit = gv2
;;
@@ -49,7 +49,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, f32) -> f32 tail
-;; fn0 = colocated u805306368:40 sig0
+;; fn0 = colocated u805306368:41 sig0
;; const0 = 0x00000000000000000000000000000000
;; stack_limit = gv2
;;
@@ -81,7 +81,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, f32) -> f32 tail
-;; fn0 = colocated u805306368:42 sig0
+;; fn0 = colocated u805306368:43 sig0
;; const0 = 0x00000000000000000000000000000000
;; stack_limit = gv2
;;
@@ -113,7 +113,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, f32) -> f32 tail
-;; fn0 = colocated u805306368:44 sig0
+;; fn0 = colocated u805306368:45 sig0
;; const0 = 0x00000000000000000000000000000000
;; stack_limit = gv2
;;
@@ -145,7 +145,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, f64) -> f64 tail
-;; fn0 = colocated u805306368:39 sig0
+;; fn0 = colocated u805306368:40 sig0
;; const0 = 0x00000000000000000000000000000000
;; stack_limit = gv2
;;
@@ -171,7 +171,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, f64) -> f64 tail
-;; fn0 = colocated u805306368:41 sig0
+;; fn0 = colocated u805306368:42 sig0
;; const0 = 0x00000000000000000000000000000000
;; stack_limit = gv2
;;
@@ -197,7 +197,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, f64) -> f64 tail
-;; fn0 = colocated u805306368:43 sig0
+;; fn0 = colocated u805306368:44 sig0
;; const0 = 0x00000000000000000000000000000000
;; stack_limit = gv2
;;
@@ -223,7 +223,7 @@
;; gv2 = load.i64 notrap aligned gv1+24
;; gv3 = vmctx
;; sig0 = (i64 vmctx, f64) -> f64 tail
-;; fn0 = colocated u805306368:45 sig0
+;; fn0 = colocated u805306368:46 sig0
;; const0 = 0x00000000000000000000000000000000
;; stack_limit = gv2
;;
diff --git a/tests/misc_testsuite/gc/array-copy-non-gc-refs.wast b/tests/misc_testsuite/gc/array-copy-non-gc-refs.wast
new file mode 100644
index 0000000000..966a83a848
--- /dev/null
+++ b/tests/misc_testsuite/gc/array-copy-non-gc-refs.wast
@@ -0,0 +1,250 @@
+;;! gc = true
+
+;; Basic copy of i32 elements between two arrays.
+(module
+ (type $arr (array (mut i32)))
+
+ (func (export "basic-copy") (result i32)
+ (local $src (ref $arr))
+ (local $dst (ref $arr))
+
+ ;; Source: [10, 20, 30, 40, 50]
+ (local.set $src (array.new_fixed $arr 5
+ (i32.const 10)
+ (i32.const 20)
+ (i32.const 30)
+ (i32.const 40)
+ (i32.const 50)
+ ))
+
+ ;; Destination: [0, 0, 0, 0, 0]
+ (local.set $dst (array.new $arr (i32.const 0) (i32.const 5)))
+
+ ;; Copy src[1..4] to dst[0..3]
+ (array.copy $arr $arr
+ (local.get $dst) (i32.const 0)
+ (local.get $src) (i32.const 1)
+ (i32.const 3)
+ )
+
+ ;; dst should be [20, 30, 40, 0, 0]
+ (i32.add
+ (i32.add
+ (array.get $arr (local.get $dst) (i32.const 0))
+ (array.get $arr (local.get $dst) (i32.const 1))
+ )
+ (i32.add
+ (array.get $arr (local.get $dst) (i32.const 2))
+ (array.get $arr (local.get $dst) (i32.const 3))
+ )
+ )
+ )
+)
+
+;; 20 + 30 + 40 + 0 = 90
+(assert_return (invoke "basic-copy") (i32.const 90))
+
+;; Overlapping copy within same array: forward direction (src > dst).
+(module
+ (type $arr (array (mut i32)))
+
+ (func (export "overlap-forward") (result i32)
+ (local $a (ref $arr))
+
+ ;; [10, 20, 30, 40, 50]
+ (local.set $a (array.new_fixed $arr 5
+ (i32.const 10)
+ (i32.const 20)
+ (i32.const 30)
+ (i32.const 40)
+ (i32.const 50)
+ ))
+
+ ;; Copy a[2..5] to a[0..3] (src_index=2 > dst_index=0)
+ (array.copy $arr $arr
+ (local.get $a) (i32.const 0)
+ (local.get $a) (i32.const 2)
+ (i32.const 3)
+ )
+
+ ;; a should be [30, 40, 50, 40, 50]
+ (i32.add
+ (i32.add
+ (array.get $arr (local.get $a) (i32.const 0))
+ (array.get $arr (local.get $a) (i32.const 1))
+ )
+ (i32.add
+ (array.get $arr (local.get $a) (i32.const 2))
+ (array.get $arr (local.get $a) (i32.const 3))
+ )
+ )
+ )
+)
+
+;; 30 + 40 + 50 + 40 = 160
+(assert_return (invoke "overlap-forward") (i32.const 160))
+
+;; Overlapping copy within same array: backward direction (dst > src).
+(module
+ (type $arr (array (mut i32)))
+
+ (func (export "overlap-backward") (result i32)
+ (local $a (ref $arr))
+
+ ;; [10, 20, 30, 40, 50]
+ (local.set $a (array.new_fixed $arr 5
+ (i32.const 10)
+ (i32.const 20)
+ (i32.const 30)
+ (i32.const 40)
+ (i32.const 50)
+ ))
+
+ ;; Copy a[0..3] to a[2..5] (src_index=0 < dst_index=2)
+ (array.copy $arr $arr
+ (local.get $a) (i32.const 2)
+ (local.get $a) (i32.const 0)
+ (i32.const 3)
+ )
+
+ ;; a should be [10, 20, 10, 20, 30]
+ (i32.add
+ (i32.add
+ (array.get $arr (local.get $a) (i32.const 0))
+ (array.get $arr (local.get $a) (i32.const 1))
+ )
+ (i32.add
+ (array.get $arr (local.get $a) (i32.const 2))
+ (array.get $arr (local.get $a) (i32.const 3))
+ )
+ )
+ )
+)
+
+;; 10 + 20 + 10 + 20 = 60
+(assert_return (invoke "overlap-backward") (i32.const 60))
+
+;; Zero-length copy is a no-op.
+(module
+ (type $arr (array (mut i32)))
+
+ (func (export "zero-length") (result i32)
+ (local $a (ref $arr))
+ (local.set $a (array.new_fixed $arr 3
+ (i32.const 1)
+ (i32.const 2)
+ (i32.const 3)
+ ))
+ ;; Copy length 0 at array boundary -- should not trap.
+ (array.copy $arr $arr
+ (local.get $a) (i32.const 3)
+ (local.get $a) (i32.const 3)
+ (i32.const 0)
+ )
+ (array.get $arr (local.get $a) (i32.const 2))
+ )
+)
+
+(assert_return (invoke "zero-length") (i32.const 3))
+
+;; Copy of i8 (packed) elements.
+(module
+ (type $arr (array (mut i8)))
+
+ (func (export "i8-copy") (result i32)
+ (local $src (ref $arr))
+ (local $dst (ref $arr))
+ (local.set $src (array.new_fixed $arr 3
+ (i32.const 100)
+ (i32.const 200)
+ (i32.const 42)
+ ))
+ (local.set $dst (array.new $arr (i32.const 0) (i32.const 3)))
+ (array.copy $arr $arr
+ (local.get $dst) (i32.const 0)
+ (local.get $src) (i32.const 0)
+ (i32.const 3)
+ )
+ (i32.add
+ (array.get_u $arr (local.get $dst) (i32.const 0))
+ (array.get_u $arr (local.get $dst) (i32.const 2))
+ )
+ )
+)
+
+;; 100 + 42 = 142
+(assert_return (invoke "i8-copy") (i32.const 142))
+
+;; Out-of-bounds destination traps.
+(module
+ (type $arr (array (mut i32)))
+
+ (func (export "oob-dst")
+ (local $src (ref $arr))
+ (local $dst (ref $arr))
+ (local.set $src (array.new $arr (i32.const 0) (i32.const 5)))
+ (local.set $dst (array.new $arr (i32.const 0) (i32.const 3)))
+ ;; dst has length 3, copying 4 elements starting at 0 is out of bounds.
+ (array.copy $arr $arr
+ (local.get $dst) (i32.const 0)
+ (local.get $src) (i32.const 0)
+ (i32.const 4)
+ )
+ )
+)
+
+(assert_trap (invoke "oob-dst") "out of bounds array access")
+
+;; Out-of-bounds source traps.
+(module
+ (type $arr (array (mut i32)))
+
+ (func (export "oob-src")
+ (local $src (ref $arr))
+ (local $dst (ref $arr))
+ (local.set $src (array.new $arr (i32.const 0) (i32.const 3)))
+ (local.set $dst (array.new $arr (i32.const 0) (i32.const 5)))
+ ;; src has length 3, copying 4 elements starting at 0 is out of bounds.
+ (array.copy $arr $arr
+ (local.get $dst) (i32.const 0)
+ (local.get $src) (i32.const 0)
+ (i32.const 4)
+ )
+ )
+)
+
+(assert_trap (invoke "oob-src") "out of bounds array access")
+
+;; Null destination traps.
+(module
+ (type $arr (array (mut i32)))
+
+ (func (export "null-dst")
+ (local $src (ref $arr))
+ (local.set $src (array.new $arr (i32.const 0) (i32.const 3)))
+ (array.copy $arr $arr
+ (ref.null $arr) (i32.const 0)
+ (local.get $src) (i32.const 0)
+ (i32.const 1)
+ )
+ )
+)
+
+(assert_trap (invoke "null-dst") "null reference")
+
+;; Null source traps.
+(module
+ (type $arr (array (mut i32)))
+
+ (func (export "null-src")
+ (local $dst (ref $arr))
+ (local.set $dst (array.new $arr (i32.const 0) (i32.const 3)))
+ (array.copy $arr $arr
+ (local.get $dst) (i32.const 0)
+ (ref.null $arr) (i32.const 0)
+ (i32.const 1)
+ )
+ )
+)
+
+(assert_trap (invoke "null-src") "null reference")