Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Inline the copying collector's bump allocation in Wasm code (#13323)

Nick Fitzgeraldfd7947780385
diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs
index 0ed775f7f3..0a65d1c8a8 100644
--- a/crates/cranelift/src/func_environ/gc/enabled.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled.rs
@@ -105,7 +105,7 @@ fn unbarriered_store_gc_ref(
Ok(())
}
-/// Emit CLIF to call the `gc_raw_alloc` libcall.
+/// Emit CLIF to call the `gc_alloc_raw` libcall.
#[cfg(any(feature = "gc-drc", feature = "gc-copying"))]
fn emit_gc_raw_alloc(
func_env: &mut FuncEnvironment<'_>,
diff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/enabled/copying.rs
index 1c82ddea03..336f2df487 100644
--- a/crates/cranelift/src/func_environ/gc/enabled/copying.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled/copying.rs
@@ -1,7 +1,8 @@
//! Compiler for the copying (semi-space/Cheney) collector.
//!
-//! Allocation is performed via the `gc_alloc_raw` libcall (not inlined) for
-//! now. Read and write barriers are unnecessary (e.g. no reference counting and
+//! Allocation is performed inline with a bump pointer when possible, falling
+//! back to the `gc_alloc_raw` libcall when the active semi-space is full.
+//! Read and write barriers are unnecessary (e.g. no reference counting and
//! no concurrent mutation during collection) but we do need stack maps so the
//! collector can find and update roots when it relocates objects.
@@ -11,11 +12,13 @@ use crate::func_environ::FuncEnvironment;
use crate::translate::TargetEnvironment;
use cranelift_codegen::ir::{self, InstBuilder};
use cranelift_frontend::FunctionBuilder;
-use smallvec::SmallVec;
-use wasmtime_environ::copying::{EXCEPTION_TAG_DEFINED_OFFSET, EXCEPTION_TAG_INSTANCE_OFFSET};
+use wasmtime_environ::copying::{
+ ALIGN, EXCEPTION_TAG_DEFINED_OFFSET, EXCEPTION_TAG_INSTANCE_OFFSET,
+};
use wasmtime_environ::{
- GcTypeLayouts, TypeIndex, VMGcKind, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult,
- WasmStorageType, WasmValType, copying::CopyingTypeLayouts,
+ GcTypeLayouts, ModuleInternedTypeIndex, PtrSize, TypeIndex, VMGcKind, WasmHeapTopType,
+ WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,
+ copying::CopyingTypeLayouts,
};
#[derive(Default)]
@@ -24,6 +27,188 @@ pub struct CopyingCompiler {
}
impl CopyingCompiler {
+ /// Load the pointer to the `VMCopyingHeapData` from vmctx.
+ fn load_vmcopying_heap_data_ptr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ ) -> ir::Value {
+ let pointer_type = func_env.pointer_type();
+ let vmctx = func_env.vmctx_val(&mut builder.cursor());
+ builder.ins().load(
+ pointer_type,
+ ir::MemFlags::trusted().with_readonly().with_can_move(),
+ vmctx,
+ i32::from(func_env.offsets.ptr.vmctx_gc_heap_data()),
+ )
+ }
+
+ /// Load the current bump pointer and active-space end from a `*mut
+ /// VMCopyingHeapData`.
+ ///
+ /// Returns `(bump_ptr, active_space_end)` as `i32` values.
+ fn load_bump_state(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ ptr_to_heap_data: ir::Value,
+ ) -> (ir::Value, ir::Value) {
+ let bump_ptr = builder.ins().load(
+ ir::types::I32,
+ ir::MemFlags::trusted().with_can_move(),
+ ptr_to_heap_data,
+ i32::from(func_env.offsets.ptr.vmcopying_heap_data_bump_ptr()),
+ );
+ let active_space_end = builder.ins().load(
+ ir::types::I32,
+ ir::MemFlags::trusted().with_readonly().with_can_move(),
+ ptr_to_heap_data,
+ i32::from(func_env.offsets.ptr.vmcopying_heap_data_active_space_end()),
+ );
+ (bump_ptr, active_space_end)
+ }
+
+ /// Round `size` (an `i32`) up to `ALIGN`, returning the result as an `i64`.
+ ///
+ /// Uses `i64` arithmetic so that overflow produces a value larger than any
+ /// valid heap index, which sends us to the slow allocation path instead of
+ /// wrapping around.
+ fn aligned_size(builder: &mut FunctionBuilder, size: ir::Value) -> ir::Value {
+ let size_64 = builder.ins().uextend(ir::types::I64, size);
+ let align_mask = builder.ins().iconst(ir::types::I64, i64::from(ALIGN - 1));
+ let inv_align_mask = builder.ins().iconst(ir::types::I64, !i64::from(ALIGN - 1));
+ let size_plus_mask = builder.ins().iadd(size_64, align_mask);
+ builder.ins().band(size_plus_mask, inv_align_mask)
+ }
+
+ /// Emit inline bump allocation, falling back to `gc_alloc_raw` on failure.
+ ///
+ /// `size` must be an `i32` value >= `size_of(VMCopyingHeader)`.
+ ///
+ /// Returns `(gc_ref, raw_ptr_to_object)` where `gc_ref` is the `i32` GC
+ /// heap index and `raw_ptr_to_object` is a native pointer into the GC heap.
+ fn emit_inline_alloc(
+ &mut self,
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder,
+ kind: VMGcKind,
+ ty: ModuleInternedTypeIndex,
+ size: ir::Value,
+ ) -> (ir::Value, ir::Value) {
+ debug_assert_ne!(kind, VMGcKind::ExternRef);
+ debug_assert!(!ty.is_reserved_value());
+ assert_eq!(builder.func.dfg.value_type(size), ir::types::I32);
+
+ let pointer_type = func_env.pointer_type();
+ let current_block = builder.current_block().unwrap();
+ let fast_block = builder.create_block();
+ let slow_block = builder.create_block();
+ let merge_block = builder.create_block();
+
+ builder.ensure_inserted_block();
+ builder.insert_block_after(fast_block, current_block);
+ builder.insert_block_after(slow_block, fast_block);
+ builder.insert_block_after(merge_block, slow_block);
+
+ let ptr_to_heap_data = Self::load_vmcopying_heap_data_ptr(func_env, builder);
+ let (bump_ptr, active_space_end) =
+ Self::load_bump_state(func_env, builder, ptr_to_heap_data);
+ let aligned_size_64 = Self::aligned_size(builder, size);
+
+ // Compute `end_of_object = bump_ptr + aligned_size` (in i64) and check
+ // whether it fits within the active semi-space.
+ let bump_ptr_64 = builder.ins().uextend(ir::types::I64, bump_ptr);
+ let end_64 = builder.ins().iadd(bump_ptr_64, aligned_size_64);
+ let active_space_end_64 = builder.ins().uextend(ir::types::I64, active_space_end);
+ let fits = builder.ins().icmp(
+ ir::condcodes::IntCC::UnsignedLessThanOrEqual,
+ end_64,
+ active_space_end_64,
+ );
+ builder.ins().brif(fits, fast_block, &[], slow_block, &[]);
+
+ // Slow path: when there isn't enough room in the bump region, call the
+ // `gc_alloc_raw` libcall, which will collect or grow the GC heap as
+ // necessary.
+ {
+ builder.switch_to_block(slow_block);
+ builder.seal_block(slow_block);
+ builder.set_cold_block(slow_block);
+ let gc_ref = emit_gc_raw_alloc(func_env, builder, kind, ty, size, ALIGN);
+ let base = func_env.get_gc_heap_base(builder);
+ let heap_offset = uextend_i32_to_pointer_type(builder, pointer_type, gc_ref);
+ let obj_ptr = builder.ins().iadd(base, heap_offset);
+ builder
+ .ins()
+ .jump(merge_block, &[gc_ref.into(), obj_ptr.into()]);
+ }
+
+ // Fast path: there is capacity for the requested object in the bump
+ // region, so finish the allocation inline, update our bump pointer,
+ // etc...
+ {
+ builder.switch_to_block(fast_block);
+ builder.seal_block(fast_block);
+
+ // The old bump_ptr is the start of the new object.
+ let gc_ref = bump_ptr;
+
+ // Update the bump pointer.
+ let end_of_object = builder.ins().ireduce(ir::types::I32, end_64);
+ builder.ins().store(
+ ir::MemFlags::trusted().with_alias_region(Some(ir::AliasRegion::Vmctx)),
+ end_of_object,
+ ptr_to_heap_data,
+ i32::from(func_env.offsets.ptr.vmcopying_heap_data_bump_ptr()),
+ );
+
+ // Compute the raw pointer to the new object.
+ let base = func_env.get_gc_heap_base(builder);
+ let heap_offset = uextend_i32_to_pointer_type(builder, pointer_type, gc_ref);
+ let obj_ptr = builder.ins().iadd(base, heap_offset);
+
+ // Write `VMGcHeader::kind`.
+ let kind_val = builder
+ .ins()
+ .iconst(ir::types::I32, i64::from(kind.as_u32()));
+ builder.ins().store(
+ ir::MemFlags::trusted(),
+ kind_val,
+ obj_ptr,
+ i32::try_from(wasmtime_environ::VM_GC_HEADER_KIND_OFFSET).unwrap(),
+ );
+
+ // Write `VMGcHeader::type_index`.
+ let shared_ty = func_env.module_interned_to_shared_ty(&mut builder.cursor(), ty);
+ builder.ins().store(
+ ir::MemFlags::trusted(),
+ shared_ty,
+ obj_ptr,
+ i32::try_from(wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET).unwrap(),
+ );
+
+ // Write `VMCopyingHeader::object_size`.
+ builder.ins().istore32(
+ ir::MemFlags::trusted(),
+ aligned_size_64,
+ obj_ptr,
+ i32::try_from(wasmtime_environ::VM_GC_HEADER_SIZE).unwrap(),
+ );
+
+ builder
+ .ins()
+ .jump(merge_block, &[gc_ref.into(), obj_ptr.into()]);
+ }
+
+ // Merge block: takes the GC ref and the raw pointer to the GC object as
+ // block parameters.
+ builder.switch_to_block(merge_block);
+ let gc_ref = builder.append_block_param(merge_block, ir::types::I32);
+ let ptr_to_object = builder.append_block_param(merge_block, pointer_type);
+ builder.seal_block(merge_block);
+ builder.declare_value_needs_stack_map(gc_ref);
+
+ (gc_ref, ptr_to_object)
+ }
+
fn init_field(
&mut self,
func_env: &mut FuncEnvironment<'_>,
@@ -85,28 +270,20 @@ impl GcCompiler for CopyingCompiler {
let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset();
let array_layout = func_env.array_layout(interned_type_index).clone();
let base_size = array_layout.base_size;
- let align = array_layout.align;
let len_to_elems_delta = base_size.checked_sub(len_offset).unwrap();
// First, compute the array's total size.
let len = init.len(&mut builder.cursor());
let size = emit_array_size(func_env, builder, &array_layout, len);
- // Allocate via libcall.
- let array_ref = emit_gc_raw_alloc(
+ // Allocate inline (with fallback to libcall).
+ let (array_ref, object_addr) = self.emit_inline_alloc(
func_env,
builder,
VMGcKind::ArrayRef,
interned_type_index,
size,
- align,
);
-
- // Write the array's length.
- let base = func_env.get_gc_heap_base(builder);
- let extended_array_ref =
- uextend_i32_to_pointer_type(builder, func_env.pointer_type(), array_ref);
- let object_addr = builder.ins().iadd(base, extended_array_ref);
let len_addr = builder.ins().iadd_imm(object_addr, i64::from(len_offset));
let len = init.len(&mut builder.cursor());
builder
@@ -142,26 +319,18 @@ impl GcCompiler for CopyingCompiler {
let struct_layout = func_env.struct_or_exn_layout(interned_type_index);
let struct_size = struct_layout.size;
- let struct_align = struct_layout.align;
- let field_offsets: SmallVec<[_; 8]> = struct_layout.fields.iter().copied().collect();
- assert_eq!(field_vals.len(), field_offsets.len());
let struct_size_val = builder.ins().iconst(ir::types::I32, i64::from(struct_size));
- let struct_ref = emit_gc_raw_alloc(
+ let (struct_ref, raw_ptr_to_struct) = self.emit_inline_alloc(
func_env,
builder,
VMGcKind::StructRef,
interned_type_index,
struct_size_val,
- struct_align,
);
// Initialize fields.
- let base = func_env.get_gc_heap_base(builder);
- let extended_struct_ref =
- uextend_i32_to_pointer_type(builder, func_env.pointer_type(), struct_ref);
- let raw_ptr_to_struct = builder.ins().iadd(base, extended_struct_ref);
initialize_struct_fields(
func_env,
builder,
@@ -191,26 +360,18 @@ impl GcCompiler for CopyingCompiler {
let exn_layout = func_env.struct_or_exn_layout(interned_type_index);
let exn_size = exn_layout.size;
- let exn_align = exn_layout.align;
- let field_offsets: SmallVec<[_; 8]> = exn_layout.fields.iter().copied().collect();
- assert_eq!(field_vals.len(), field_offsets.len());
let exn_size_val = builder.ins().iconst(ir::types::I32, i64::from(exn_size));
- let exn_ref = emit_gc_raw_alloc(
+ let (exn_ref, raw_ptr_to_exn) = self.emit_inline_alloc(
func_env,
builder,
VMGcKind::ExnRef,
interned_type_index,
exn_size_val,
- exn_align,
);
// Initialize fields.
- let base = func_env.get_gc_heap_base(builder);
- let extended_exn_ref =
- uextend_i32_to_pointer_type(builder, func_env.pointer_type(), exn_ref);
- let raw_ptr_to_exn = builder.ins().iadd(base, extended_exn_ref);
initialize_struct_fields(
func_env,
builder,
diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs
index d73193ab56..5e5b9e7df4 100644
--- a/crates/environ/src/vmoffsets.rs
+++ b/crates/environ/src/vmoffsets.rs
@@ -480,6 +480,31 @@ pub trait PtrSize {
self.vmctx_epoch_ptr() + self.size()
}
+ /// Return the offset of the `bump_ptr` field within `VMCopyingHeapData`.
+ #[inline]
+ fn vmcopying_heap_data_bump_ptr(&self) -> u8 {
+ 0
+ }
+
+ /// Return the offset of the `active_space_end` field within
+ /// `VMCopyingHeapData`.
+ #[inline]
+ fn vmcopying_heap_data_active_space_end(&self) -> u8 {
+ 4
+ }
+
+ /// Return the size of `VMCopyingHeapData`.
+ #[inline]
+ fn size_of_vmcopying_heap_data(&self) -> u8 {
+ 8
+ }
+
+ /// Return the alignment of `VMCopyingHeapData`.
+ #[inline]
+ fn align_of_vmcopying_heap_data(&self) -> u8 {
+ 4
+ }
+
/// The offset of the `type_ids` array pointer.
#[inline]
fn vmctx_type_ids_array(&self) -> u8 {
diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
index a5d70100bc..48f5591605 100644
--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs
@@ -171,6 +171,18 @@ unsafe impl GcHeapObject for VMCopyingExternRef {
}
}
+/// JIT-accessible bump-allocation state for the copying collector.
+///
+/// NB: Layout is defined by constants in `wasmtime_environ::copying`. Keep in
+/// sync!
+#[repr(C)]
+struct VMCopyingHeapData {
+ /// Current bump pointer (index into the GC heap).
+ bump_ptr: u32,
+ /// End of the active semi-space.
+ active_space_end: u32,
+}
+
/// Get a typed reference to a copying-collector object from a raw `VMGcRef`.
fn copying_ref(gc_ref: &VMGcRef) -> &TypedGcRef<VMCopyingHeader> {
debug_assert!(!gc_ref.is_i31());
@@ -207,17 +219,15 @@ struct CopyingHeap {
/// memory is taken and updated when the memory is replaced.
vmmemory: Option<VMMemoryDefinition>,
- /// The bump "pointer" (really an index) for allocating new objects.
+ /// JIT-accessible allocation state: bump pointer and active-space end.
///
- /// This is always within the active semi-space.
- bump_ptr: u32,
+ /// NB: The bump pointer is written to by compiled Wasm code via the pointer
+ /// returned by `vmctx_gc_heap_data`.
+ vmctx_data: VMCopyingHeapData,
/// The start of the active semi-space.
active_space_start: u32,
- /// The end of the active semi-space.
- active_space_end: u32,
-
/// The start of the idle semi-space.
idle_space_start: u32,
@@ -254,9 +264,11 @@ impl CopyingHeap {
no_gc_count: 0,
memory: None,
vmmemory: None,
- bump_ptr: 0,
+ vmctx_data: VMCopyingHeapData {
+ bump_ptr: 0,
+ active_space_end: 0,
+ },
active_space_start: 0,
- active_space_end: 0,
idle_space_start: 0,
idle_space_end: 0,
worklist_ptr: 0,
@@ -265,6 +277,22 @@ impl CopyingHeap {
})
}
+ fn bump_ptr(&self) -> u32 {
+ self.vmctx_data.bump_ptr
+ }
+
+ fn set_bump_ptr(&mut self, val: u32) {
+ self.vmctx_data.bump_ptr = val;
+ }
+
+ fn active_space_end(&self) -> u32 {
+ self.vmctx_data.active_space_end
+ }
+
+ fn set_active_space_end(&mut self, val: u32) {
+ self.vmctx_data.active_space_end = val;
+ }
+
fn capacity(&self) -> u32 {
let len = self.vmmemory.as_ref().unwrap().current_length();
let len = u32::try_from(len).unwrap_or(u32::MAX);
@@ -277,10 +305,10 @@ impl CopyingHeap {
/// Initialize the semi-spaces for a heap of the given capacity.
fn initialize_semi_spaces(&mut self) {
debug_assert_eq!(self.active_space_start, 0);
- debug_assert_eq!(self.active_space_end, 0);
+ debug_assert_eq!(self.active_space_end(), 0);
debug_assert_eq!(self.idle_space_start, 0);
debug_assert_eq!(self.idle_space_end, 0);
- debug_assert_eq!(self.bump_ptr, 0);
+ debug_assert_eq!(self.bump_ptr(), 0);
self.resize_semi_spaces();
self.reset_bump_ptr();
@@ -288,7 +316,7 @@ impl CopyingHeap {
fn resize_semi_spaces(&mut self) {
debug_assert_eq!(
- self.active_space_end - self.active_space_start,
+ self.active_space_end() - self.active_space_start,
self.idle_space_end - self.idle_space_start,
"the active and idle spaces should be the same size"
);
@@ -304,16 +332,16 @@ impl CopyingHeap {
let capacity = self.capacity();
let halfway = capacity / 2;
- debug_assert!(self.bump_ptr <= halfway);
+ debug_assert!(self.bump_ptr() <= halfway);
debug_assert!(self.idle_space_start <= halfway);
- debug_assert!(self.active_space_end <= halfway);
+ debug_assert!(self.active_space_end() <= halfway);
- self.active_space_end = halfway;
+ self.set_active_space_end(halfway);
self.idle_space_start = halfway;
self.idle_space_end = capacity;
debug_assert_eq!(
- self.active_space_end - self.active_space_start,
+ self.active_space_end() - self.active_space_start,
self.idle_space_end - self.idle_space_start,
"the active and idle spaces should be the same size"
);
@@ -329,11 +357,12 @@ impl CopyingHeap {
// skip the first `ALIGN` bytes. This ensures that the active and idle
// spaces are always equally sized, which is required to guarantee that
// evacuating objects from one to the other will succeed.
- self.bump_ptr = self.active_space_start;
- if self.active_space_end - self.active_space_start >= ALIGN {
- self.bump_ptr += ALIGN;
+ let mut bp = self.active_space_start;
+ if self.active_space_end() - self.active_space_start >= ALIGN {
+ bp += ALIGN;
}
- debug_assert!(self.bump_ptr.is_multiple_of(ALIGN));
+ self.set_bump_ptr(bp);
+ debug_assert!(self.bump_ptr().is_multiple_of(ALIGN));
}
/// Allocate `size` bytes from the active semi-space bump pointer.
@@ -341,27 +370,27 @@ impl CopyingHeap {
/// Returns `None` if there isn't enough room.
fn allocate(&mut self, size: u32) -> Option<u32> {
debug_assert!(size.is_multiple_of(ALIGN));
- debug_assert!(self.bump_ptr.is_multiple_of(ALIGN));
- debug_assert!(self.bump_ptr >= self.active_space_start);
- debug_assert!(self.bump_ptr <= self.active_space_end);
+ debug_assert!(self.bump_ptr().is_multiple_of(ALIGN));
+ debug_assert!(self.bump_ptr() >= self.active_space_start);
+ debug_assert!(self.bump_ptr() <= self.active_space_end());
- let result = self.bump_ptr;
+ let result = self.bump_ptr();
let new_bump_ptr = result.checked_add(size)?;
- if new_bump_ptr > self.active_space_end {
+ if new_bump_ptr > self.active_space_end() {
return None;
}
- self.bump_ptr = new_bump_ptr;
- debug_assert!(self.bump_ptr.is_multiple_of(ALIGN));
- debug_assert!(self.bump_ptr >= self.active_space_start);
- debug_assert!(self.bump_ptr <= self.active_space_end);
+ self.set_bump_ptr(new_bump_ptr);
+ debug_assert!(self.bump_ptr().is_multiple_of(ALIGN));
+ debug_assert!(self.bump_ptr() >= self.active_space_start);
+ debug_assert!(self.bump_ptr() <= self.active_space_end());
Some(result)
}
/// Check whether an index is within the active semi-space.
fn is_in_active_space(&self, index: u32) -> bool {
- index >= self.active_space_start && index < self.active_space_end
+ index >= self.active_space_start && index < self.active_space_end()
}
/// Check whether an index is within the idle semi-space.
@@ -372,13 +401,15 @@ impl CopyingHeap {
/// Swap the active and idle semi-spaces.
fn flip(&mut self) {
debug_assert_eq!(
- self.active_space_end - self.active_space_start,
+ self.active_space_end() - self.active_space_start,
self.idle_space_end - self.idle_space_start,
"the active and idle spaces should be the same size"
);
mem::swap(&mut self.active_space_start, &mut self.idle_space_start);
- mem::swap(&mut self.active_space_end, &mut self.idle_space_end);
+ let new_active_space_end = self.idle_space_end;
+ self.idle_space_end = self.active_space_end();
+ self.set_active_space_end(new_active_space_end);
self.reset_bump_ptr();
// The active idle list becomes the old active list; the active list is
@@ -388,7 +419,7 @@ impl CopyingHeap {
/// Initialize the worklist at the start of a collection.
fn initialize_worklist(&mut self) {
- self.worklist_ptr = self.bump_ptr;
+ self.worklist_ptr = self.bump_ptr();
}
/// Pop the next item off the worklist, or return `None` if the worklist is
@@ -396,14 +427,14 @@ impl CopyingHeap {
fn worklist_pop(&mut self) -> Option<VMGcRef> {
debug_assert!(
self.is_in_active_space(self.worklist_ptr)
- || self.worklist_ptr == self.active_space_end
+ || self.worklist_ptr == self.active_space_end()
);
debug_assert!(
- self.is_in_active_space(self.bump_ptr) || self.bump_ptr == self.active_space_end
+ self.is_in_active_space(self.bump_ptr()) || self.bump_ptr() == self.active_space_end()
);
- debug_assert!(self.worklist_ptr <= self.bump_ptr);
+ debug_assert!(self.worklist_ptr <= self.bump_ptr());
- if self.worklist_ptr == self.bump_ptr {
+ if self.worklist_ptr == self.bump_ptr() {
return None;
}
@@ -414,7 +445,7 @@ impl CopyingHeap {
let obj_size = self.index(copying_ref(&result)).object_size();
self.worklist_ptr += obj_size;
- debug_assert!(self.worklist_ptr <= self.bump_ptr);
+ debug_assert!(self.worklist_ptr <= self.bump_ptr());
Some(result)
}
@@ -432,7 +463,7 @@ impl CopyingHeap {
let index = gc_ref.as_heap_index().unwrap().get();
debug_assert!(self.is_in_active_space(index));
let obj_size = self.index(copying_ref(gc_ref)).object_size();
- debug_assert_eq!(index + obj_size, self.bump_ptr);
+ debug_assert_eq!(index + obj_size, self.bump_ptr());
debug_assert!(self.worklist_ptr <= index);
}
@@ -580,13 +611,15 @@ unsafe impl GcHeap for CopyingHeap {
fn detach(&mut self) -> crate::vm::Memory {
assert!(self.is_attached());
+ self.set_bump_ptr(0);
+ self.set_active_space_end(0);
+
let CopyingHeap {
no_gc_count,
memory,
vmmemory,
- bump_ptr,
+ vmctx_data: _,
active_space_start,
- active_space_end,
idle_space_start,
idle_space_end,
worklist_ptr,
@@ -600,9 +633,7 @@ unsafe impl GcHeap for CopyingHeap {
*no_gc_count = 0;
*vmmemory = None;
- *bump_ptr = 0;
*active_space_start = 0;
- *active_space_end = 0;
*idle_space_start = 0;
*idle_space_end = 0;
*worklist_ptr = 0;
@@ -720,7 +751,11 @@ unsafe impl GcHeap for CopyingHeap {
);
debug_assert!(layout.size() >= core::mem::size_of::<VMCopyingHeader>());
- debug_assert_eq!(self.bump_ptr % ALIGN, 0, "bump_ptr is not aligned to ALIGN");
+ debug_assert_eq!(
+ self.bump_ptr() % ALIGN,
+ 0,
+ "bump_ptr is not aligned to ALIGN"
+ );
debug_assert_eq!(header.reserved_u26(), 0);
// We must have trace info for every GC type that we allocate in this
@@ -830,7 +865,7 @@ unsafe impl GcHeap for CopyingHeap {
}
fn allocated_bytes(&self) -> usize {
- usize::try_from(self.bump_ptr - self.active_space_start).unwrap()
+ usize::try_from(self.bump_ptr() - self.active_space_start).unwrap()
}
fn gc<'a>(
@@ -848,9 +883,8 @@ unsafe impl GcHeap for CopyingHeap {
}
unsafe fn vmctx_gc_heap_data(&self) -> NonNull<u8> {
- // The copying collector doesn't currently have vmctx GC heap
- // data. Return a dangling pointer.
- NonNull::dangling()
+ let ptr: *const VMCopyingHeapData = &self.vmctx_data;
+ NonNull::new(ptr as *mut u8).unwrap()
}
fn take_memory(&mut self) -> crate::vm::Memory {
@@ -874,7 +908,7 @@ unsafe impl GcHeap for CopyingHeap {
// If the heap was previously empty, reinitialize the semi-spaces from
// scratch.
- if self.active_space_end == 0 && self.idle_space_end == 0 {
+ if self.active_space_end() == 0 && self.idle_space_end == 0 {
self.initialize_semi_spaces();
} else {
// Otherwise the memory was grown: try to resize the semi-spaces
@@ -982,11 +1016,11 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {
CopyingCollectionPhase::Collect => {
log::trace!("Begin copying collection");
- assert!(self.heap.active_space_start <= self.heap.bump_ptr);
- assert!(self.heap.bump_ptr <= self.heap.active_space_end);
+ assert!(self.heap.active_space_start <= self.heap.bump_ptr());
+ assert!(self.heap.bump_ptr() <= self.heap.active_space_end());
assert!(self.heap.idle_space_start <= self.heap.idle_space_end);
assert!(
- self.heap.active_space_end <= self.heap.idle_space_start
+ self.heap.active_space_end() <= self.heap.idle_space_start
|| self.heap.idle_space_end <= self.heap.active_space_start
);
@@ -997,11 +1031,11 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {
self.process_roots();
self.process_worklist();
- assert!(self.heap.active_space_start <= self.heap.bump_ptr);
- assert!(self.heap.bump_ptr <= self.heap.active_space_end);
+ assert!(self.heap.active_space_start <= self.heap.bump_ptr());
+ assert!(self.heap.bump_ptr() <= self.heap.active_space_end());
assert!(self.heap.idle_space_start <= self.heap.idle_space_end);
assert!(
- self.heap.active_space_end <= self.heap.idle_space_start
+ self.heap.active_space_end() <= self.heap.idle_space_start
|| self.heap.idle_space_end <= self.heap.active_space_start
);
@@ -1009,7 +1043,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {
self.heap.resize_semi_spaces();
debug_assert_eq!(
- self.heap.active_space_end - self.heap.active_space_start,
+ self.heap.active_space_end() - self.heap.active_space_start,
self.heap.idle_space_end - self.heap.idle_space_start,
"the active and idle spaces should be the same size"
);
@@ -1033,6 +1067,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {
#[cfg(test)]
mod tests {
use super::*;
+ use wasmtime_environ::{HostPtr, PtrSize};
#[test]
fn vm_copying_header_size_align() {
@@ -1083,4 +1118,36 @@ mod tests {
<= wasmtime_environ::copying::MIN_OBJECT_SIZE as usize,
);
}
+
+ #[test]
+ fn vm_copying_heap_data_bump_ptr_offset() {
+ assert_eq!(
+ HostPtr.vmcopying_heap_data_bump_ptr() as usize,
+ core::mem::offset_of!(VMCopyingHeapData, bump_ptr),
+ );
+ }
+
+ #[test]
+ fn vm_copying_heap_data_active_space_end_offset() {
+ assert_eq!(
+ HostPtr.vmcopying_heap_data_active_space_end() as usize,
+ core::mem::offset_of!(VMCopyingHeapData, active_space_end),
+ );
+ }
+
+ #[test]
+ fn vm_copying_heap_data_size() {
+ assert_eq!(
+ HostPtr.size_of_vmcopying_heap_data() as usize,
+ core::mem::size_of::<VMCopyingHeapData>(),
+ );
+ }
+
+ #[test]
+ fn vm_copying_heap_data_align() {
+ assert_eq!(
+ HostPtr.align_of_vmcopying_heap_data() as usize,
+ core::mem::align_of::<VMCopyingHeapData>(),
+ );
+ }
}
diff --git a/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat b/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat
index d73275c023..00693251b6 100644
--- a/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat
+++ b/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat
@@ -23,40 +23,70 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):
-;; v44 = stack_addr.i64 ss2
-;; store notrap v2, v44
-;; v43 = stack_addr.i64 ss1
-;; store notrap v3, v43
-;; v42 = stack_addr.i64 ss0
-;; store notrap v4, v42
-;; @0025 v14 = iconst.i32 -1476395008
-;; @0025 v16 = load.i64 notrap aligned readonly can_move v0+40
-;; @0025 v17 = load.i32 notrap aligned readonly can_move v16
-;; v56 = iconst.i32 32
-;; @0025 v18 = iconst.i32 16
-;; @0025 v19 = call fn0(v0, v14, v17, v56, v18), stack_map=[i32 @ ss2+0, i32 @ ss1+0, i32 @ ss0+0] ; v14 = -1476395008, v56 = 32, v18 = 16
+;; v69 = stack_addr.i64 ss2
+;; store notrap v2, v69
+;; v68 = stack_addr.i64 ss1
+;; store notrap v3, v68
+;; v67 = stack_addr.i64 ss0
+;; store notrap v4, v67
+;; @0025 v14 = load.i64 notrap aligned readonly can_move v0+32
+;; @0025 v15 = load.i32 notrap aligned can_move v14
+;; @0025 v22 = uextend.i64 v15
+;; v65 = iconst.i64 32
+;; @0025 v23 = iadd v22, v65 ; v65 = 32
+;; @0025 v16 = load.i32 notrap aligned readonly can_move v14+4
+;; @0025 v24 = uextend.i64 v16
+;; @0025 v25 = icmp ule v23, v24
+;; @0025 brif v25, block2, block3
+;;
+;; block2:
+;; v189 = iconst.i32 32
+;; v95 = iadd.i32 v15, v189 ; v189 = 32
+;; @0025 store notrap aligned vmctx v95, v14
+;; v190 = iconst.i32 -1476395008
+;; v191 = load.i64 notrap aligned readonly can_move v0+8
+;; v192 = load.i64 notrap aligned readonly can_move v191+32
+;; @0025 v39 = iadd v192, v22
+;; @0025 store notrap aligned v190, v39 ; v190 = -1476395008
+;; v193 = load.i64 notrap aligned readonly can_move v0+40
+;; v194 = load.i32 notrap aligned readonly can_move v193
+;; @0025 store notrap aligned v194, v39+4
+;; v195 = iconst.i64 32
+;; @0025 istore32 notrap aligned v195, v39+8 ; v195 = 32
+;; @0025 jump block4(v15, v39)
+;;
+;; block3 cold:
+;; @0025 v27 = iconst.i32 -1476395008
+;; @0025 v29 = load.i64 notrap aligned readonly can_move v0+40
+;; @0025 v30 = load.i32 notrap aligned readonly can_move v29
+;; v81 = iconst.i32 32
+;; @0025 v31 = iconst.i32 16
+;; @0025 v32 = call fn0(v0, v27, v30, v81, v31), stack_map=[i32 @ ss2+0, i32 @ ss1+0, i32 @ ss0+0] ; v27 = -1476395008, v81 = 32, v31 = 16
+;; @0025 v61 = load.i64 notrap aligned readonly can_move v0+8
+;; @0025 v33 = load.i64 notrap aligned readonly can_move v61+32
+;; @0025 v34 = uextend.i64 v32
+;; @0025 v35 = iadd v33, v34
+;; @0025 jump block4(v32, v35)
+;;
+;; block4(v44: i32, v45: i64):
;; @0025 v6 = iconst.i32 3
-;; @0025 v38 = load.i64 notrap aligned readonly can_move v0+8
-;; @0025 v20 = load.i64 notrap aligned readonly can_move v38+32
-;; @0025 v21 = uextend.i64 v19
-;; @0025 v22 = iadd v20, v21
-;; v37 = iconst.i64 16
-;; @0025 v23 = iadd v22, v37 ; v37 = 16
-;; @0025 store notrap aligned v6, v23 ; v6 = 3
-;; v33 = load.i32 notrap v44
-;; v58 = iconst.i64 20
-;; v63 = iadd v22, v58 ; v58 = 20
-;; @0025 store notrap aligned little v33, v63
-;; v32 = load.i32 notrap v43
-;; v66 = iconst.i64 24
-;; v71 = iadd v22, v66 ; v66 = 24
-;; @0025 store notrap aligned little v32, v71
-;; v31 = load.i32 notrap v42
-;; v87 = iconst.i64 28
-;; v92 = iadd v22, v87 ; v87 = 28
-;; @0025 store notrap aligned little v31, v92
-;; @0029 jump block1
+;; v60 = iconst.i64 16
+;; @0025 v46 = iadd v45, v60 ; v60 = 16
+;; @0025 store notrap aligned v6, v46 ; v6 = 3
+;; v56 = load.i32 notrap v69
+;; v98 = iconst.i64 20
+;; v103 = iadd v45, v98 ; v98 = 20
+;; @0025 store notrap aligned little v56, v103
+;; v55 = load.i32 notrap v68
+;; v106 = iconst.i64 24
+;; v111 = iadd v45, v106 ; v106 = 24
+;; @0025 store notrap aligned little v55, v111
+;; v54 = load.i32 notrap v67
+;; v127 = iconst.i64 28
+;; v132 = iadd v45, v127 ; v127 = 28
+;; @0025 store notrap aligned little v54, v132
+;; @0029 jump block1(v44)
;;
-;; block1:
-;; @0029 return v19
+;; block1(v5: i32):
+;; @0029 return v5
;; }
diff --git a/tests/disas/gc/copying/array-new-fixed.wat b/tests/disas/gc/copying/array-new-fixed.wat
index 726abb4204..3fbbe70777 100644
--- a/tests/disas/gc/copying/array-new-fixed.wat
+++ b/tests/disas/gc/copying/array-new-fixed.wat
@@ -20,31 +20,61 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64):
-;; @0025 v14 = iconst.i32 -1476395008
-;; @0025 v16 = load.i64 notrap aligned readonly can_move v0+40
-;; @0025 v17 = load.i32 notrap aligned readonly can_move v16
-;; v45 = iconst.i32 48
-;; @0025 v18 = iconst.i32 16
-;; @0025 v19 = call fn0(v0, v14, v17, v45, v18) ; v14 = -1476395008, v45 = 48, v18 = 16
+;; @0025 v14 = load.i64 notrap aligned readonly can_move v0+32
+;; @0025 v15 = load.i32 notrap aligned can_move v14
+;; @0025 v22 = uextend.i64 v15
+;; v71 = iconst.i64 48
+;; @0025 v23 = iadd v22, v71 ; v71 = 48
+;; @0025 v16 = load.i32 notrap aligned readonly can_move v14+4
+;; @0025 v24 = uextend.i64 v16
+;; @0025 v25 = icmp ule v23, v24
+;; @0025 brif v25, block2, block3
+;;
+;; block2:
+;; v154 = iconst.i32 48
+;; v85 = iadd.i32 v15, v154 ; v154 = 48
+;; @0025 store notrap aligned vmctx v85, v14
+;; v155 = iconst.i32 -1476395008
+;; v156 = load.i64 notrap aligned readonly can_move v0+8
+;; v157 = load.i64 notrap aligned readonly can_move v156+32
+;; @0025 v39 = iadd v157, v22
+;; @0025 store notrap aligned v155, v39 ; v155 = -1476395008
+;; v158 = load.i64 notrap aligned readonly can_move v0+40
+;; v159 = load.i32 notrap aligned readonly can_move v158
+;; @0025 store notrap aligned v159, v39+4
+;; v160 = iconst.i64 48
+;; @0025 istore32 notrap aligned v160, v39+8 ; v160 = 48
+;; @0025 jump block4(v15, v39)
+;;
+;; block3 cold:
+;; @0025 v27 = iconst.i32 -1476395008
+;; @0025 v29 = load.i64 notrap aligned readonly can_move v0+40
+;; @0025 v30 = load.i32 notrap aligned readonly can_move v29
+;; v70 = iconst.i32 48
+;; @0025 v31 = iconst.i32 16
+;; @0025 v32 = call fn0(v0, v27, v30, v70, v31) ; v27 = -1476395008, v70 = 48, v31 = 16
+;; @0025 v55 = load.i64 notrap aligned readonly can_move v0+8
+;; @0025 v33 = load.i64 notrap aligned readonly can_move v55+32
+;; @0025 v34 = uextend.i64 v32
+;; @0025 v35 = iadd v33, v34
+;; @0025 jump block4(v32, v35)
+;;
+;; block4(v44: i32, v45: i64):
;; @0025 v6 = iconst.i32 3
-;; @0025 v32 = load.i64 notrap aligned readonly can_move v0+8
-;; @0025 v20 = load.i64 notrap aligned readonly can_move v32+32
-;; @0025 v21 = uextend.i64 v19
-;; @0025 v22 = iadd v20, v21
-;; v31 = iconst.i64 16
-;; @0025 v23 = iadd v22, v31 ; v31 = 16
-;; @0025 store notrap aligned v6, v23 ; v6 = 3
-;; v37 = iconst.i64 24
-;; v51 = iadd v22, v37 ; v37 = 24
-;; @0025 store notrap aligned little v2, v51
-;; v34 = iconst.i64 32
-;; v58 = iadd v22, v34 ; v34 = 32
-;; @0025 store notrap aligned little v3, v58
-;; v73 = iconst.i64 40
-;; v78 = iadd v22, v73 ; v73 = 40
-;; @0025 store notrap aligned little v4, v78
-;; @0029 jump block1
+;; v54 = iconst.i64 16
+;; @0025 v46 = iadd v45, v54 ; v54 = 16
+;; @0025 store notrap aligned v6, v46 ; v6 = 3
+;; v62 = iconst.i64 24
+;; v92 = iadd v45, v62 ; v62 = 24
+;; @0025 store.i64 notrap aligned little v2, v92
+;; v59 = iconst.i64 32
+;; v99 = iadd v45, v59 ; v59 = 32
+;; @0025 store.i64 notrap aligned little v3, v99
+;; v114 = iconst.i64 40
+;; v119 = iadd v45, v114 ; v114 = 40
+;; @0025 store.i64 notrap aligned little v4, v119
+;; @0029 jump block1(v44)
;;
-;; block1:
-;; @0029 return v19
+;; block1(v5: i32):
+;; @0029 return v5
;; }
diff --git a/tests/disas/gc/copying/array-new.wat b/tests/disas/gc/copying/array-new.wat
index 8f5539acc1..82be522a68 100644
--- a/tests/disas/gc/copying/array-new.wat
+++ b/tests/disas/gc/copying/array-new.wat
@@ -21,47 +21,83 @@
;;
;; block0(v0: i64, v1: i64, v2: i64, v3: i32):
;; @0022 v6 = uextend.i64 v3
-;; v37 = iconst.i64 3
-;; v38 = ishl v6, v37 ; v37 = 3
-;; v35 = iconst.i64 32
-;; @0022 v8 = ushr v38, v35 ; v35 = 32
+;; v62 = iconst.i64 3
+;; v63 = ishl v6, v62 ; v62 = 3
+;; v60 = iconst.i64 32
+;; @0022 v8 = ushr v63, v60 ; v60 = 32
;; @0022 trapnz v8, user17
;; @0022 v5 = iconst.i32 24
-;; v44 = iconst.i32 3
-;; v45 = ishl v3, v44 ; v44 = 3
-;; @0022 v10 = uadd_overflow_trap v5, v45, user17 ; v5 = 24
-;; @0022 v12 = iconst.i32 -1476395008
-;; @0022 v14 = load.i64 notrap aligned readonly can_move v0+40
-;; @0022 v15 = load.i32 notrap aligned readonly can_move v14
-;; @0022 v16 = iconst.i32 16
-;; @0022 v17 = call fn0(v0, v12, v15, v10, v16) ; v12 = -1476395008, v16 = 16
-;; @0022 v33 = load.i64 notrap aligned readonly can_move v0+8
-;; @0022 v18 = load.i64 notrap aligned readonly can_move v33+32
-;; @0022 v19 = uextend.i64 v17
-;; @0022 v20 = iadd v18, v19
-;; v32 = iconst.i64 16
-;; @0022 v21 = iadd v20, v32 ; v32 = 16
-;; @0022 store notrap aligned v3, v21
-;; v49 = iconst.i64 24
-;; v54 = iadd v20, v49 ; v49 = 24
-;; @0022 v27 = uextend.i64 v10
-;; @0022 v28 = iadd v20, v27
-;; v36 = iconst.i64 8
-;; @0022 jump block2(v54)
+;; v69 = iconst.i32 3
+;; v70 = ishl v3, v69 ; v69 = 3
+;; @0022 v10 = uadd_overflow_trap v5, v70, user17 ; v5 = 24
+;; @0022 v12 = load.i64 notrap aligned readonly can_move v0+32
+;; @0022 v13 = load.i32 notrap aligned can_move v12
+;; @0022 v20 = uextend.i64 v13
+;; @0022 v15 = uextend.i64 v10
+;; @0022 v16 = iconst.i64 15
+;; @0022 v18 = iadd v15, v16 ; v16 = 15
+;; @0022 v17 = iconst.i64 -16
+;; @0022 v19 = band v18, v17 ; v17 = -16
+;; @0022 v21 = iadd v20, v19
+;; @0022 v14 = load.i32 notrap aligned readonly can_move v12+4
+;; @0022 v22 = uextend.i64 v14
+;; @0022 v23 = icmp ule v21, v22
+;; @0022 brif v23, block2, block3
;;
-;; block2(v29: i64):
-;; @0022 v30 = icmp eq v29, v28
-;; @0022 brif v30, block4, block3
+;; block2:
+;; v78 = iconst.i32 15
+;; v79 = iadd.i32 v10, v78 ; v78 = 15
+;; v82 = iconst.i32 -16
+;; v83 = band v79, v82 ; v82 = -16
+;; v85 = iadd.i32 v13, v83
+;; @0022 store notrap aligned vmctx v85, v12
+;; v98 = iconst.i32 -1476395008
+;; v99 = load.i64 notrap aligned readonly can_move v0+8
+;; v100 = load.i64 notrap aligned readonly can_move v99+32
+;; @0022 v37 = iadd v100, v20
+;; @0022 store notrap aligned v98, v37 ; v98 = -1476395008
+;; v101 = load.i64 notrap aligned readonly can_move v0+40
+;; v102 = load.i32 notrap aligned readonly can_move v101
+;; @0022 store notrap aligned v102, v37+4
+;; v103 = band.i64 v18, v17 ; v17 = -16
+;; @0022 istore32 notrap aligned v103, v37+8
+;; @0022 jump block4(v13, v37)
;;
-;; block3:
-;; @0022 store.i64 notrap aligned little v2, v29
-;; v59 = iconst.i64 8
-;; v60 = iadd.i64 v29, v59 ; v59 = 8
-;; @0022 jump block2(v60)
+;; block3 cold:
+;; @0022 v25 = iconst.i32 -1476395008
+;; @0022 v27 = load.i64 notrap aligned readonly can_move v0+40
+;; @0022 v28 = load.i32 notrap aligned readonly can_move v27
+;; @0022 v29 = iconst.i32 16
+;; @0022 v30 = call fn0(v0, v25, v28, v10, v29) ; v25 = -1476395008, v29 = 16
+;; @0022 v56 = load.i64 notrap aligned readonly can_move v0+8
+;; @0022 v31 = load.i64 notrap aligned readonly can_move v56+32
+;; @0022 v32 = uextend.i64 v30
+;; @0022 v33 = iadd v31, v32
+;; @0022 jump block4(v30, v33)
;;
-;; block4:
-;; @0025 jump block1
+;; block4(v42: i32, v43: i64):
+;; v55 = iconst.i64 16
+;; @0022 v44 = iadd v43, v55 ; v55 = 16
+;; @0022 store.i32 notrap aligned v3, v44
+;; v88 = iconst.i64 24
+;; v93 = iadd v43, v88 ; v88 = 24
+;; @0022 v51 = iadd v43, v15
+;; v61 = iconst.i64 8
+;; @0022 jump block5(v93)
;;
-;; block1:
-;; @0025 return v17
+;; block5(v52: i64):
+;; @0022 v53 = icmp eq v52, v51
+;; @0022 brif v53, block7, block6
+;;
+;; block6:
+;; @0022 store.i64 notrap aligned little v2, v52
+;; v104 = iconst.i64 8
+;; v105 = iadd.i64 v52, v104 ; v104 = 8
+;; @0022 jump block5(v105)
+;;
+;; block7:
+;; @0025 jump block1(v42)
+;;
+;; block1(v4: i32):
+;; @0025 return v4
;; }
diff --git a/tests/disas/gc/copying/funcref-in-gc-heap-new.wat b/tests/disas/gc/copying/funcref-in-gc-heap-new.wat
index 08b5e03b19..c2fb4c5a75 100644
--- a/tests/disas/gc/copying/funcref-in-gc-heap-new.wat
+++ b/tests/disas/gc/copying/funcref-in-gc-heap-new.wat
@@ -23,26 +23,56 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i64):
-;; @0020 v6 = iconst.i32 -1342177280
-;; @0020 v8 = load.i64 notrap aligned readonly can_move v0+40
-;; @0020 v9 = load.i32 notrap aligned readonly can_move v8
+;; @0020 v6 = load.i64 notrap aligned readonly can_move v0+32
+;; @0020 v7 = load.i32 notrap aligned can_move v6
+;; @0020 v14 = uextend.i64 v7
+;; v50 = iconst.i64 32
+;; @0020 v15 = iadd v14, v50 ; v50 = 32
+;; @0020 v8 = load.i32 notrap aligned readonly can_move v6+4
+;; @0020 v16 = uextend.i64 v8
+;; @0020 v17 = icmp ule v15, v16
+;; @0020 brif v17, block2, block3
+;;
+;; block2:
+;; v66 = iconst.i32 32
+;; v64 = iadd.i32 v7, v66 ; v66 = 32
+;; @0020 store notrap aligned vmctx v64, v6
+;; v67 = iconst.i32 -1342177280
+;; v68 = load.i64 notrap aligned readonly can_move v0+8
+;; v69 = load.i64 notrap aligned readonly can_move v68+32
+;; @0020 v31 = iadd v69, v14
+;; @0020 store notrap aligned v67, v31 ; v67 = -1342177280
+;; v70 = load.i64 notrap aligned readonly can_move v0+40
+;; v71 = load.i32 notrap aligned readonly can_move v70
+;; @0020 store notrap aligned v71, v31+4
+;; v72 = iconst.i64 32
+;; @0020 istore32 notrap aligned v72, v31+8 ; v72 = 32
+;; @0020 jump block4(v7, v31)
+;;
+;; block3 cold:
+;; @0020 v19 = iconst.i32 -1342177280
+;; @0020 v21 = load.i64 notrap aligned readonly can_move v0+40
+;; @0020 v22 = load.i32 notrap aligned readonly can_move v21
;; @0020 v4 = iconst.i32 32
-;; @0020 v10 = iconst.i32 16
-;; @0020 v11 = call fn0(v0, v6, v9, v4, v10) ; v6 = -1342177280, v4 = 32, v10 = 16
-;; v26 = stack_addr.i64 ss0
-;; store notrap v11, v26
-;; @0020 v17 = call fn1(v0, v2), stack_map=[i32 @ ss0+0]
-;; @0020 v18 = ireduce.i32 v17
-;; @0020 v24 = load.i64 notrap aligned readonly can_move v0+8
-;; @0020 v12 = load.i64 notrap aligned readonly can_move v24+32
-;; @0020 v13 = uextend.i64 v11
-;; @0020 v14 = iadd v12, v13
-;; v22 = iconst.i64 16
-;; @0020 v15 = iadd v14, v22 ; v22 = 16
-;; @0020 store notrap aligned little v18, v15
-;; v19 = load.i32 notrap v26
+;; @0020 v23 = iconst.i32 16
+;; @0020 v24 = call fn0(v0, v19, v22, v4, v23) ; v19 = -1342177280, v4 = 32, v23 = 16
+;; @0020 v46 = load.i64 notrap aligned readonly can_move v0+8
+;; @0020 v25 = load.i64 notrap aligned readonly can_move v46+32
+;; @0020 v26 = uextend.i64 v24
+;; @0020 v27 = iadd v25, v26
+;; @0020 jump block4(v24, v27)
+;;
+;; block4(v36: i32, v37: i64):
+;; v45 = stack_addr.i64 ss0
+;; store notrap v36, v45
+;; @0020 v40 = call fn1(v0, v2), stack_map=[i32 @ ss0+0]
+;; @0020 v41 = ireduce.i32 v40
+;; v44 = iconst.i64 16
+;; @0020 v38 = iadd v37, v44 ; v44 = 16
+;; @0020 store notrap aligned little v41, v38
+;; v42 = load.i32 notrap v45
;; @0023 jump block1
;;
;; block1:
-;; @0023 return v19
+;; @0023 return v42
;; }
diff --git a/tests/disas/gc/copying/struct-new-default.wat b/tests/disas/gc/copying/struct-new-default.wat
index 5d0cf53096..f2c6f96afe 100644
--- a/tests/disas/gc/copying/struct-new-default.wat
+++ b/tests/disas/gc/copying/struct-new-default.wat
@@ -22,29 +22,59 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64):
-;; @0021 v8 = iconst.i32 -1342177280
-;; @0021 v10 = load.i64 notrap aligned readonly can_move v0+40
-;; @0021 v11 = load.i32 notrap aligned readonly can_move v10
+;; @0021 v8 = load.i64 notrap aligned readonly can_move v0+32
+;; @0021 v9 = load.i32 notrap aligned can_move v8
+;; @0021 v16 = uextend.i64 v9
+;; v50 = iconst.i64 32
+;; @0021 v17 = iadd v16, v50 ; v50 = 32
+;; @0021 v10 = load.i32 notrap aligned readonly can_move v8+4
+;; @0021 v18 = uextend.i64 v10
+;; @0021 v19 = icmp ule v17, v18
+;; @0021 brif v19, block2, block3
+;;
+;; block2:
+;; v66 = iconst.i32 32
+;; v64 = iadd.i32 v9, v66 ; v66 = 32
+;; @0021 store notrap aligned vmctx v64, v8
+;; v67 = iconst.i32 -1342177280
+;; v68 = load.i64 notrap aligned readonly can_move v0+8
+;; v69 = load.i64 notrap aligned readonly can_move v68+32
+;; @0021 v33 = iadd v69, v16
+;; @0021 store notrap aligned v67, v33 ; v67 = -1342177280
+;; v70 = load.i64 notrap aligned readonly can_move v0+40
+;; v71 = load.i32 notrap aligned readonly can_move v70
+;; @0021 store notrap aligned v71, v33+4
+;; v72 = iconst.i64 32
+;; @0021 istore32 notrap aligned v72, v33+8 ; v72 = 32
+;; @0021 jump block4(v9, v33)
+;;
+;; block3 cold:
+;; @0021 v21 = iconst.i32 -1342177280
+;; @0021 v23 = load.i64 notrap aligned readonly can_move v0+40
+;; @0021 v24 = load.i32 notrap aligned readonly can_move v23
;; @0021 v6 = iconst.i32 32
-;; @0021 v12 = iconst.i32 16
-;; @0021 v13 = call fn0(v0, v8, v11, v6, v12) ; v8 = -1342177280, v6 = 32, v12 = 16
+;; @0021 v25 = iconst.i32 16
+;; @0021 v26 = call fn0(v0, v21, v24, v6, v25) ; v21 = -1342177280, v6 = 32, v25 = 16
+;; @0021 v46 = load.i64 notrap aligned readonly can_move v0+8
+;; @0021 v27 = load.i64 notrap aligned readonly can_move v46+32
+;; @0021 v28 = uextend.i64 v26
+;; @0021 v29 = iadd v27, v28
+;; @0021 jump block4(v26, v29)
+;;
+;; block4(v38: i32, v39: i64):
;; @0021 v3 = f32const 0.0
-;; @0021 v23 = load.i64 notrap aligned readonly can_move v0+8
-;; @0021 v14 = load.i64 notrap aligned readonly can_move v23+32
-;; @0021 v15 = uextend.i64 v13
-;; @0021 v16 = iadd v14, v15
-;; v22 = iconst.i64 16
-;; @0021 v17 = iadd v16, v22 ; v22 = 16
-;; @0021 store notrap aligned little v3, v17 ; v3 = 0.0
+;; v45 = iconst.i64 16
+;; @0021 v40 = iadd v39, v45 ; v45 = 16
+;; @0021 store notrap aligned little v3, v40 ; v3 = 0.0
;; @0021 v4 = iconst.i32 0
-;; v21 = iconst.i64 20
-;; @0021 v18 = iadd v16, v21 ; v21 = 20
-;; @0021 istore8 notrap aligned little v4, v18 ; v4 = 0
-;; v20 = iconst.i64 24
-;; @0021 v19 = iadd v16, v20 ; v20 = 24
-;; @0021 store notrap aligned little v4, v19 ; v4 = 0
-;; @0024 jump block1
+;; v44 = iconst.i64 20
+;; @0021 v41 = iadd v39, v44 ; v44 = 20
+;; @0021 istore8 notrap aligned little v4, v41 ; v4 = 0
+;; v43 = iconst.i64 24
+;; @0021 v42 = iadd v39, v43 ; v43 = 24
+;; @0021 store notrap aligned little v4, v42 ; v4 = 0
+;; @0024 jump block1(v38)
;;
-;; block1:
-;; @0024 return v13
+;; block1(v2: i32):
+;; @0024 return v2
;; }
diff --git a/tests/disas/gc/copying/struct-new.wat b/tests/disas/gc/copying/struct-new.wat
index ee3b8c7a8d..0eaaab4919 100644
--- a/tests/disas/gc/copying/struct-new.wat
+++ b/tests/disas/gc/copying/struct-new.wat
@@ -23,30 +23,60 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: f32, v3: i32, v4: i32):
-;; v27 = stack_addr.i64 ss0
-;; store notrap v4, v27
-;; @002a v8 = iconst.i32 -1342177280
-;; @002a v10 = load.i64 notrap aligned readonly can_move v0+40
-;; @002a v11 = load.i32 notrap aligned readonly can_move v10
+;; v52 = stack_addr.i64 ss0
+;; store notrap v4, v52
+;; @002a v8 = load.i64 notrap aligned readonly can_move v0+32
+;; @002a v9 = load.i32 notrap aligned can_move v8
+;; @002a v16 = uextend.i64 v9
+;; v53 = iconst.i64 32
+;; @002a v17 = iadd v16, v53 ; v53 = 32
+;; @002a v10 = load.i32 notrap aligned readonly can_move v8+4
+;; @002a v18 = uextend.i64 v10
+;; @002a v19 = icmp ule v17, v18
+;; @002a brif v19, block2, block3
+;;
+;; block2:
+;; v69 = iconst.i32 32
+;; v67 = iadd.i32 v9, v69 ; v69 = 32
+;; @002a store notrap aligned vmctx v67, v8
+;; v70 = iconst.i32 -1342177280
+;; v71 = load.i64 notrap aligned readonly can_move v0+8
+;; v72 = load.i64 notrap aligned readonly can_move v71+32
+;; @002a v33 = iadd v72, v16
+;; @002a store notrap aligned v70, v33 ; v70 = -1342177280
+;; v73 = load.i64 notrap aligned readonly can_move v0+40
+;; v74 = load.i32 notrap aligned readonly can_move v73
+;; @002a store notrap aligned v74, v33+4
+;; v75 = iconst.i64 32
+;; @002a istore32 notrap aligned v75, v33+8 ; v75 = 32
+;; @002a jump block4(v9, v33)
+;;
+;; block3 cold:
+;; @002a v21 = iconst.i32 -1342177280
+;; @002a v23 = load.i64 notrap aligned readonly can_move v0+40
+;; @002a v24 = load.i32 notrap aligned readonly can_move v23
;; @002a v6 = iconst.i32 32
-;; @002a v12 = iconst.i32 16
-;; @002a v13 = call fn0(v0, v8, v11, v6, v12), stack_map=[i32 @ ss0+0] ; v8 = -1342177280, v6 = 32, v12 = 16
-;; @002a v25 = load.i64 notrap aligned readonly can_move v0+8
-;; @002a v14 = load.i64 notrap aligned readonly can_move v25+32
-;; @002a v15 = uextend.i64 v13
-;; @002a v16 = iadd v14, v15
-;; v24 = iconst.i64 16
-;; @002a v17 = iadd v16, v24 ; v24 = 16
-;; @002a store notrap aligned little v2, v17
-;; v23 = iconst.i64 20
-;; @002a v18 = iadd v16, v23 ; v23 = 20
-;; @002a istore8 notrap aligned little v3, v18
-;; v20 = load.i32 notrap v27
-;; v22 = iconst.i64 24
-;; @002a v19 = iadd v16, v22 ; v22 = 24
-;; @002a store notrap aligned little v20, v19
-;; @002d jump block1
+;; @002a v25 = iconst.i32 16
+;; @002a v26 = call fn0(v0, v21, v24, v6, v25), stack_map=[i32 @ ss0+0] ; v21 = -1342177280, v6 = 32, v25 = 16
+;; @002a v48 = load.i64 notrap aligned readonly can_move v0+8
+;; @002a v27 = load.i64 notrap aligned readonly can_move v48+32
+;; @002a v28 = uextend.i64 v26
+;; @002a v29 = iadd v27, v28
+;; @002a jump block4(v26, v29)
+;;
+;; block4(v38: i32, v39: i64):
+;; v47 = iconst.i64 16
+;; @002a v40 = iadd v39, v47 ; v47 = 16
+;; @002a store.f32 notrap aligned little v2, v40
+;; v46 = iconst.i64 20
+;; @002a v41 = iadd v39, v46 ; v46 = 20
+;; @002a istore8.i32 notrap aligned little v3, v41
+;; v43 = load.i32 notrap v52
+;; v45 = iconst.i64 24
+;; @002a v42 = iadd v39, v45 ; v45 = 24
+;; @002a store notrap aligned little v43, v42
+;; @002d jump block1(v38)
;;
-;; block1:
-;; @002d return v13
+;; block1(v5: i32):
+;; @002d return v5
;; }