A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Inline the copying collector's bump allocation in Wasm code (#13323)
Nick Fitzgeraldfd7947780385
diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rsindex 0ed775f7f3..0a65d1c8a8 100644--- a/crates/cranelift/src/func_environ/gc/enabled.rs+++ b/crates/cranelift/src/func_environ/gc/enabled.rs@@ -105,7 +105,7 @@ fn unbarriered_store_gc_ref(Ok(())}-/// Emit CLIF to call the `gc_raw_alloc` libcall.+/// Emit CLIF to call the `gc_alloc_raw` libcall.#[cfg(any(feature = "gc-drc", feature = "gc-copying"))]fn emit_gc_raw_alloc(func_env: &mut FuncEnvironment<'_>,diff --git a/crates/cranelift/src/func_environ/gc/enabled/copying.rs b/crates/cranelift/src/func_environ/gc/enabled/copying.rsindex 1c82ddea03..336f2df487 100644--- a/crates/cranelift/src/func_environ/gc/enabled/copying.rs+++ b/crates/cranelift/src/func_environ/gc/enabled/copying.rs@@ -1,7 +1,8 @@//! Compiler for the copying (semi-space/Cheney) collector.//!-//! Allocation is performed via the `gc_alloc_raw` libcall (not inlined) for-//! now. Read and write barriers are unnecessary (e.g. no reference counting and+//! Allocation is performed inline with a bump pointer when possible, falling+//! back to the `gc_alloc_raw` libcall when the active semi-space is full.+//! Read and write barriers are unnecessary (e.g. no reference counting and//! no concurrent mutation during collection) but we do need stack maps so the//! collector can find and update roots when it relocates objects.@@ -11,11 +12,13 @@ use crate::func_environ::FuncEnvironment;use crate::translate::TargetEnvironment;use cranelift_codegen::ir::{self, InstBuilder};use cranelift_frontend::FunctionBuilder;-use smallvec::SmallVec;-use wasmtime_environ::copying::{EXCEPTION_TAG_DEFINED_OFFSET, EXCEPTION_TAG_INSTANCE_OFFSET};+use wasmtime_environ::copying::{+ ALIGN, EXCEPTION_TAG_DEFINED_OFFSET, EXCEPTION_TAG_INSTANCE_OFFSET,+};use wasmtime_environ::{- GcTypeLayouts, TypeIndex, VMGcKind, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult,- WasmStorageType, WasmValType, copying::CopyingTypeLayouts,+ GcTypeLayouts, ModuleInternedTypeIndex, PtrSize, TypeIndex, VMGcKind, WasmHeapTopType,+ WasmHeapType, WasmRefType, WasmResult, WasmStorageType, WasmValType,+ copying::CopyingTypeLayouts,};#[derive(Default)]@@ -24,6 +27,188 @@ pub struct CopyingCompiler {}impl CopyingCompiler {+ /// Load the pointer to the `VMCopyingHeapData` from vmctx.+ fn load_vmcopying_heap_data_ptr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ ) -> ir::Value {+ let pointer_type = func_env.pointer_type();+ let vmctx = func_env.vmctx_val(&mut builder.cursor());+ builder.ins().load(+ pointer_type,+ ir::MemFlags::trusted().with_readonly().with_can_move(),+ vmctx,+ i32::from(func_env.offsets.ptr.vmctx_gc_heap_data()),+ )+ }++ /// Load the current bump pointer and active-space end from a `*mut+ /// VMCopyingHeapData`.+ ///+ /// Returns `(bump_ptr, active_space_end)` as `i32` values.+ fn load_bump_state(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ ptr_to_heap_data: ir::Value,+ ) -> (ir::Value, ir::Value) {+ let bump_ptr = builder.ins().load(+ ir::types::I32,+ ir::MemFlags::trusted().with_can_move(),+ ptr_to_heap_data,+ i32::from(func_env.offsets.ptr.vmcopying_heap_data_bump_ptr()),+ );+ let active_space_end = builder.ins().load(+ ir::types::I32,+ ir::MemFlags::trusted().with_readonly().with_can_move(),+ ptr_to_heap_data,+ i32::from(func_env.offsets.ptr.vmcopying_heap_data_active_space_end()),+ );+ (bump_ptr, active_space_end)+ }++ /// Round `size` (an `i32`) up to `ALIGN`, returning the result as an `i64`.+ ///+ /// Uses `i64` arithmetic so that overflow produces a value larger than any+ /// valid heap index, which sends us to the slow allocation path instead of+ /// wrapping around.+ fn aligned_size(builder: &mut FunctionBuilder, size: ir::Value) -> ir::Value {+ let size_64 = builder.ins().uextend(ir::types::I64, size);+ let align_mask = builder.ins().iconst(ir::types::I64, i64::from(ALIGN - 1));+ let inv_align_mask = builder.ins().iconst(ir::types::I64, !i64::from(ALIGN - 1));+ let size_plus_mask = builder.ins().iadd(size_64, align_mask);+ builder.ins().band(size_plus_mask, inv_align_mask)+ }++ /// Emit inline bump allocation, falling back to `gc_alloc_raw` on failure.+ ///+ /// `size` must be an `i32` value >= `size_of(VMCopyingHeader)`.+ ///+ /// Returns `(gc_ref, raw_ptr_to_object)` where `gc_ref` is the `i32` GC+ /// heap index and `raw_ptr_to_object` is a native pointer into the GC heap.+ fn emit_inline_alloc(+ &mut self,+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder,+ kind: VMGcKind,+ ty: ModuleInternedTypeIndex,+ size: ir::Value,+ ) -> (ir::Value, ir::Value) {+ debug_assert_ne!(kind, VMGcKind::ExternRef);+ debug_assert!(!ty.is_reserved_value());+ assert_eq!(builder.func.dfg.value_type(size), ir::types::I32);++ let pointer_type = func_env.pointer_type();+ let current_block = builder.current_block().unwrap();+ let fast_block = builder.create_block();+ let slow_block = builder.create_block();+ let merge_block = builder.create_block();++ builder.ensure_inserted_block();+ builder.insert_block_after(fast_block, current_block);+ builder.insert_block_after(slow_block, fast_block);+ builder.insert_block_after(merge_block, slow_block);++ let ptr_to_heap_data = Self::load_vmcopying_heap_data_ptr(func_env, builder);+ let (bump_ptr, active_space_end) =+ Self::load_bump_state(func_env, builder, ptr_to_heap_data);+ let aligned_size_64 = Self::aligned_size(builder, size);++ // Compute `end_of_object = bump_ptr + aligned_size` (in i64) and check+ // whether it fits within the active semi-space.+ let bump_ptr_64 = builder.ins().uextend(ir::types::I64, bump_ptr);+ let end_64 = builder.ins().iadd(bump_ptr_64, aligned_size_64);+ let active_space_end_64 = builder.ins().uextend(ir::types::I64, active_space_end);+ let fits = builder.ins().icmp(+ ir::condcodes::IntCC::UnsignedLessThanOrEqual,+ end_64,+ active_space_end_64,+ );+ builder.ins().brif(fits, fast_block, &[], slow_block, &[]);++ // Slow path: when there isn't enough room in the bump region, call the+ // `gc_alloc_raw` libcall, which will collect or grow the GC heap as+ // necessary.+ {+ builder.switch_to_block(slow_block);+ builder.seal_block(slow_block);+ builder.set_cold_block(slow_block);+ let gc_ref = emit_gc_raw_alloc(func_env, builder, kind, ty, size, ALIGN);+ let base = func_env.get_gc_heap_base(builder);+ let heap_offset = uextend_i32_to_pointer_type(builder, pointer_type, gc_ref);+ let obj_ptr = builder.ins().iadd(base, heap_offset);+ builder+ .ins()+ .jump(merge_block, &[gc_ref.into(), obj_ptr.into()]);+ }++ // Fast path: there is capacity for the requested object in the bump+ // region, so finish the allocation inline, update our bump pointer,+ // etc...+ {+ builder.switch_to_block(fast_block);+ builder.seal_block(fast_block);++ // The old bump_ptr is the start of the new object.+ let gc_ref = bump_ptr;++ // Update the bump pointer.+ let end_of_object = builder.ins().ireduce(ir::types::I32, end_64);+ builder.ins().store(+ ir::MemFlags::trusted().with_alias_region(Some(ir::AliasRegion::Vmctx)),+ end_of_object,+ ptr_to_heap_data,+ i32::from(func_env.offsets.ptr.vmcopying_heap_data_bump_ptr()),+ );++ // Compute the raw pointer to the new object.+ let base = func_env.get_gc_heap_base(builder);+ let heap_offset = uextend_i32_to_pointer_type(builder, pointer_type, gc_ref);+ let obj_ptr = builder.ins().iadd(base, heap_offset);++ // Write `VMGcHeader::kind`.+ let kind_val = builder+ .ins()+ .iconst(ir::types::I32, i64::from(kind.as_u32()));+ builder.ins().store(+ ir::MemFlags::trusted(),+ kind_val,+ obj_ptr,+ i32::try_from(wasmtime_environ::VM_GC_HEADER_KIND_OFFSET).unwrap(),+ );++ // Write `VMGcHeader::type_index`.+ let shared_ty = func_env.module_interned_to_shared_ty(&mut builder.cursor(), ty);+ builder.ins().store(+ ir::MemFlags::trusted(),+ shared_ty,+ obj_ptr,+ i32::try_from(wasmtime_environ::VM_GC_HEADER_TYPE_INDEX_OFFSET).unwrap(),+ );++ // Write `VMCopyingHeader::object_size`.+ builder.ins().istore32(+ ir::MemFlags::trusted(),+ aligned_size_64,+ obj_ptr,+ i32::try_from(wasmtime_environ::VM_GC_HEADER_SIZE).unwrap(),+ );++ builder+ .ins()+ .jump(merge_block, &[gc_ref.into(), obj_ptr.into()]);+ }++ // Merge block: takes the GC ref and the raw pointer to the GC object as+ // block parameters.+ builder.switch_to_block(merge_block);+ let gc_ref = builder.append_block_param(merge_block, ir::types::I32);+ let ptr_to_object = builder.append_block_param(merge_block, pointer_type);+ builder.seal_block(merge_block);+ builder.declare_value_needs_stack_map(gc_ref);++ (gc_ref, ptr_to_object)+ }+fn init_field(&mut self,func_env: &mut FuncEnvironment<'_>,@@ -85,28 +270,20 @@ impl GcCompiler for CopyingCompiler {let len_offset = gc_compiler(func_env)?.layouts().array_length_field_offset();let array_layout = func_env.array_layout(interned_type_index).clone();let base_size = array_layout.base_size;- let align = array_layout.align;let len_to_elems_delta = base_size.checked_sub(len_offset).unwrap();// First, compute the array's total size.let len = init.len(&mut builder.cursor());let size = emit_array_size(func_env, builder, &array_layout, len);- // Allocate via libcall.- let array_ref = emit_gc_raw_alloc(+ // Allocate inline (with fallback to libcall).+ let (array_ref, object_addr) = self.emit_inline_alloc(func_env,builder,VMGcKind::ArrayRef,interned_type_index,size,- align,);-- // Write the array's length.- let base = func_env.get_gc_heap_base(builder);- let extended_array_ref =- uextend_i32_to_pointer_type(builder, func_env.pointer_type(), array_ref);- let object_addr = builder.ins().iadd(base, extended_array_ref);let len_addr = builder.ins().iadd_imm(object_addr, i64::from(len_offset));let len = init.len(&mut builder.cursor());builder@@ -142,26 +319,18 @@ impl GcCompiler for CopyingCompiler {let struct_layout = func_env.struct_or_exn_layout(interned_type_index);let struct_size = struct_layout.size;- let struct_align = struct_layout.align;- let field_offsets: SmallVec<[_; 8]> = struct_layout.fields.iter().copied().collect();- assert_eq!(field_vals.len(), field_offsets.len());let struct_size_val = builder.ins().iconst(ir::types::I32, i64::from(struct_size));- let struct_ref = emit_gc_raw_alloc(+ let (struct_ref, raw_ptr_to_struct) = self.emit_inline_alloc(func_env,builder,VMGcKind::StructRef,interned_type_index,struct_size_val,- struct_align,);// Initialize fields.- let base = func_env.get_gc_heap_base(builder);- let extended_struct_ref =- uextend_i32_to_pointer_type(builder, func_env.pointer_type(), struct_ref);- let raw_ptr_to_struct = builder.ins().iadd(base, extended_struct_ref);initialize_struct_fields(func_env,builder,@@ -191,26 +360,18 @@ impl GcCompiler for CopyingCompiler {let exn_layout = func_env.struct_or_exn_layout(interned_type_index);let exn_size = exn_layout.size;- let exn_align = exn_layout.align;- let field_offsets: SmallVec<[_; 8]> = exn_layout.fields.iter().copied().collect();- assert_eq!(field_vals.len(), field_offsets.len());let exn_size_val = builder.ins().iconst(ir::types::I32, i64::from(exn_size));- let exn_ref = emit_gc_raw_alloc(+ let (exn_ref, raw_ptr_to_exn) = self.emit_inline_alloc(func_env,builder,VMGcKind::ExnRef,interned_type_index,exn_size_val,- exn_align,);// Initialize fields.- let base = func_env.get_gc_heap_base(builder);- let extended_exn_ref =- uextend_i32_to_pointer_type(builder, func_env.pointer_type(), exn_ref);- let raw_ptr_to_exn = builder.ins().iadd(base, extended_exn_ref);initialize_struct_fields(func_env,builder,diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rsindex d73193ab56..5e5b9e7df4 100644--- a/crates/environ/src/vmoffsets.rs+++ b/crates/environ/src/vmoffsets.rs@@ -480,6 +480,31 @@ pub trait PtrSize {self.vmctx_epoch_ptr() + self.size()}+ /// Return the offset of the `bump_ptr` field within `VMCopyingHeapData`.+ #[inline]+ fn vmcopying_heap_data_bump_ptr(&self) -> u8 {+ 0+ }++ /// Return the offset of the `active_space_end` field within+ /// `VMCopyingHeapData`.+ #[inline]+ fn vmcopying_heap_data_active_space_end(&self) -> u8 {+ 4+ }++ /// Return the size of `VMCopyingHeapData`.+ #[inline]+ fn size_of_vmcopying_heap_data(&self) -> u8 {+ 8+ }++ /// Return the alignment of `VMCopyingHeapData`.+ #[inline]+ fn align_of_vmcopying_heap_data(&self) -> u8 {+ 4+ }+/// The offset of the `type_ids` array pointer.#[inline]fn vmctx_type_ids_array(&self) -> u8 {diff --git a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rsindex a5d70100bc..48f5591605 100644--- a/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs+++ b/crates/wasmtime/src/runtime/vm/gc/enabled/copying.rs@@ -171,6 +171,18 @@ unsafe impl GcHeapObject for VMCopyingExternRef {}}+/// JIT-accessible bump-allocation state for the copying collector.+///+/// NB: Layout is defined by constants in `wasmtime_environ::copying`. Keep in+/// sync!+#[repr(C)]+struct VMCopyingHeapData {+ /// Current bump pointer (index into the GC heap).+ bump_ptr: u32,+ /// End of the active semi-space.+ active_space_end: u32,+}+/// Get a typed reference to a copying-collector object from a raw `VMGcRef`.fn copying_ref(gc_ref: &VMGcRef) -> &TypedGcRef<VMCopyingHeader> {debug_assert!(!gc_ref.is_i31());@@ -207,17 +219,15 @@ struct CopyingHeap {/// memory is taken and updated when the memory is replaced.vmmemory: Option<VMMemoryDefinition>,- /// The bump "pointer" (really an index) for allocating new objects.+ /// JIT-accessible allocation state: bump pointer and active-space end.///- /// This is always within the active semi-space.- bump_ptr: u32,+ /// NB: The bump pointer is written to by compiled Wasm code via the pointer+ /// returned by `vmctx_gc_heap_data`.+ vmctx_data: VMCopyingHeapData,/// The start of the active semi-space.active_space_start: u32,- /// The end of the active semi-space.- active_space_end: u32,-/// The start of the idle semi-space.idle_space_start: u32,@@ -254,9 +264,11 @@ impl CopyingHeap {no_gc_count: 0,memory: None,vmmemory: None,- bump_ptr: 0,+ vmctx_data: VMCopyingHeapData {+ bump_ptr: 0,+ active_space_end: 0,+ },active_space_start: 0,- active_space_end: 0,idle_space_start: 0,idle_space_end: 0,worklist_ptr: 0,@@ -265,6 +277,22 @@ impl CopyingHeap {})}+ fn bump_ptr(&self) -> u32 {+ self.vmctx_data.bump_ptr+ }++ fn set_bump_ptr(&mut self, val: u32) {+ self.vmctx_data.bump_ptr = val;+ }++ fn active_space_end(&self) -> u32 {+ self.vmctx_data.active_space_end+ }++ fn set_active_space_end(&mut self, val: u32) {+ self.vmctx_data.active_space_end = val;+ }+fn capacity(&self) -> u32 {let len = self.vmmemory.as_ref().unwrap().current_length();let len = u32::try_from(len).unwrap_or(u32::MAX);@@ -277,10 +305,10 @@ impl CopyingHeap {/// Initialize the semi-spaces for a heap of the given capacity.fn initialize_semi_spaces(&mut self) {debug_assert_eq!(self.active_space_start, 0);- debug_assert_eq!(self.active_space_end, 0);+ debug_assert_eq!(self.active_space_end(), 0);debug_assert_eq!(self.idle_space_start, 0);debug_assert_eq!(self.idle_space_end, 0);- debug_assert_eq!(self.bump_ptr, 0);+ debug_assert_eq!(self.bump_ptr(), 0);self.resize_semi_spaces();self.reset_bump_ptr();@@ -288,7 +316,7 @@ impl CopyingHeap {fn resize_semi_spaces(&mut self) {debug_assert_eq!(- self.active_space_end - self.active_space_start,+ self.active_space_end() - self.active_space_start,self.idle_space_end - self.idle_space_start,"the active and idle spaces should be the same size");@@ -304,16 +332,16 @@ impl CopyingHeap {let capacity = self.capacity();let halfway = capacity / 2;- debug_assert!(self.bump_ptr <= halfway);+ debug_assert!(self.bump_ptr() <= halfway);debug_assert!(self.idle_space_start <= halfway);- debug_assert!(self.active_space_end <= halfway);+ debug_assert!(self.active_space_end() <= halfway);- self.active_space_end = halfway;+ self.set_active_space_end(halfway);self.idle_space_start = halfway;self.idle_space_end = capacity;debug_assert_eq!(- self.active_space_end - self.active_space_start,+ self.active_space_end() - self.active_space_start,self.idle_space_end - self.idle_space_start,"the active and idle spaces should be the same size");@@ -329,11 +357,12 @@ impl CopyingHeap {// skip the first `ALIGN` bytes. This ensures that the active and idle// spaces are always equally sized, which is required to guarantee that// evacuating objects from one to the other will succeed.- self.bump_ptr = self.active_space_start;- if self.active_space_end - self.active_space_start >= ALIGN {- self.bump_ptr += ALIGN;+ let mut bp = self.active_space_start;+ if self.active_space_end() - self.active_space_start >= ALIGN {+ bp += ALIGN;}- debug_assert!(self.bump_ptr.is_multiple_of(ALIGN));+ self.set_bump_ptr(bp);+ debug_assert!(self.bump_ptr().is_multiple_of(ALIGN));}/// Allocate `size` bytes from the active semi-space bump pointer.@@ -341,27 +370,27 @@ impl CopyingHeap {/// Returns `None` if there isn't enough room.fn allocate(&mut self, size: u32) -> Option<u32> {debug_assert!(size.is_multiple_of(ALIGN));- debug_assert!(self.bump_ptr.is_multiple_of(ALIGN));- debug_assert!(self.bump_ptr >= self.active_space_start);- debug_assert!(self.bump_ptr <= self.active_space_end);+ debug_assert!(self.bump_ptr().is_multiple_of(ALIGN));+ debug_assert!(self.bump_ptr() >= self.active_space_start);+ debug_assert!(self.bump_ptr() <= self.active_space_end());- let result = self.bump_ptr;+ let result = self.bump_ptr();let new_bump_ptr = result.checked_add(size)?;- if new_bump_ptr > self.active_space_end {+ if new_bump_ptr > self.active_space_end() {return None;}- self.bump_ptr = new_bump_ptr;- debug_assert!(self.bump_ptr.is_multiple_of(ALIGN));- debug_assert!(self.bump_ptr >= self.active_space_start);- debug_assert!(self.bump_ptr <= self.active_space_end);+ self.set_bump_ptr(new_bump_ptr);+ debug_assert!(self.bump_ptr().is_multiple_of(ALIGN));+ debug_assert!(self.bump_ptr() >= self.active_space_start);+ debug_assert!(self.bump_ptr() <= self.active_space_end());Some(result)}/// Check whether an index is within the active semi-space.fn is_in_active_space(&self, index: u32) -> bool {- index >= self.active_space_start && index < self.active_space_end+ index >= self.active_space_start && index < self.active_space_end()}/// Check whether an index is within the idle semi-space.@@ -372,13 +401,15 @@ impl CopyingHeap {/// Swap the active and idle semi-spaces.fn flip(&mut self) {debug_assert_eq!(- self.active_space_end - self.active_space_start,+ self.active_space_end() - self.active_space_start,self.idle_space_end - self.idle_space_start,"the active and idle spaces should be the same size");mem::swap(&mut self.active_space_start, &mut self.idle_space_start);- mem::swap(&mut self.active_space_end, &mut self.idle_space_end);+ let new_active_space_end = self.idle_space_end;+ self.idle_space_end = self.active_space_end();+ self.set_active_space_end(new_active_space_end);self.reset_bump_ptr();// The active idle list becomes the old active list; the active list is@@ -388,7 +419,7 @@ impl CopyingHeap {/// Initialize the worklist at the start of a collection.fn initialize_worklist(&mut self) {- self.worklist_ptr = self.bump_ptr;+ self.worklist_ptr = self.bump_ptr();}/// Pop the next item off the worklist, or return `None` if the worklist is@@ -396,14 +427,14 @@ impl CopyingHeap {fn worklist_pop(&mut self) -> Option<VMGcRef> {debug_assert!(self.is_in_active_space(self.worklist_ptr)- || self.worklist_ptr == self.active_space_end+ || self.worklist_ptr == self.active_space_end());debug_assert!(- self.is_in_active_space(self.bump_ptr) || self.bump_ptr == self.active_space_end+ self.is_in_active_space(self.bump_ptr()) || self.bump_ptr() == self.active_space_end());- debug_assert!(self.worklist_ptr <= self.bump_ptr);+ debug_assert!(self.worklist_ptr <= self.bump_ptr());- if self.worklist_ptr == self.bump_ptr {+ if self.worklist_ptr == self.bump_ptr() {return None;}@@ -414,7 +445,7 @@ impl CopyingHeap {let obj_size = self.index(copying_ref(&result)).object_size();self.worklist_ptr += obj_size;- debug_assert!(self.worklist_ptr <= self.bump_ptr);+ debug_assert!(self.worklist_ptr <= self.bump_ptr());Some(result)}@@ -432,7 +463,7 @@ impl CopyingHeap {let index = gc_ref.as_heap_index().unwrap().get();debug_assert!(self.is_in_active_space(index));let obj_size = self.index(copying_ref(gc_ref)).object_size();- debug_assert_eq!(index + obj_size, self.bump_ptr);+ debug_assert_eq!(index + obj_size, self.bump_ptr());debug_assert!(self.worklist_ptr <= index);}@@ -580,13 +611,15 @@ unsafe impl GcHeap for CopyingHeap {fn detach(&mut self) -> crate::vm::Memory {assert!(self.is_attached());+ self.set_bump_ptr(0);+ self.set_active_space_end(0);+let CopyingHeap {no_gc_count,memory,vmmemory,- bump_ptr,+ vmctx_data: _,active_space_start,- active_space_end,idle_space_start,idle_space_end,worklist_ptr,@@ -600,9 +633,7 @@ unsafe impl GcHeap for CopyingHeap {*no_gc_count = 0;*vmmemory = None;- *bump_ptr = 0;*active_space_start = 0;- *active_space_end = 0;*idle_space_start = 0;*idle_space_end = 0;*worklist_ptr = 0;@@ -720,7 +751,11 @@ unsafe impl GcHeap for CopyingHeap {);debug_assert!(layout.size() >= core::mem::size_of::<VMCopyingHeader>());- debug_assert_eq!(self.bump_ptr % ALIGN, 0, "bump_ptr is not aligned to ALIGN");+ debug_assert_eq!(+ self.bump_ptr() % ALIGN,+ 0,+ "bump_ptr is not aligned to ALIGN"+ );debug_assert_eq!(header.reserved_u26(), 0);// We must have trace info for every GC type that we allocate in this@@ -830,7 +865,7 @@ unsafe impl GcHeap for CopyingHeap {}fn allocated_bytes(&self) -> usize {- usize::try_from(self.bump_ptr - self.active_space_start).unwrap()+ usize::try_from(self.bump_ptr() - self.active_space_start).unwrap()}fn gc<'a>(@@ -848,9 +883,8 @@ unsafe impl GcHeap for CopyingHeap {}unsafe fn vmctx_gc_heap_data(&self) -> NonNull<u8> {- // The copying collector doesn't currently have vmctx GC heap- // data. Return a dangling pointer.- NonNull::dangling()+ let ptr: *const VMCopyingHeapData = &self.vmctx_data;+ NonNull::new(ptr as *mut u8).unwrap()}fn take_memory(&mut self) -> crate::vm::Memory {@@ -874,7 +908,7 @@ unsafe impl GcHeap for CopyingHeap {// If the heap was previously empty, reinitialize the semi-spaces from// scratch.- if self.active_space_end == 0 && self.idle_space_end == 0 {+ if self.active_space_end() == 0 && self.idle_space_end == 0 {self.initialize_semi_spaces();} else {// Otherwise the memory was grown: try to resize the semi-spaces@@ -982,11 +1016,11 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {CopyingCollectionPhase::Collect => {log::trace!("Begin copying collection");- assert!(self.heap.active_space_start <= self.heap.bump_ptr);- assert!(self.heap.bump_ptr <= self.heap.active_space_end);+ assert!(self.heap.active_space_start <= self.heap.bump_ptr());+ assert!(self.heap.bump_ptr() <= self.heap.active_space_end());assert!(self.heap.idle_space_start <= self.heap.idle_space_end);assert!(- self.heap.active_space_end <= self.heap.idle_space_start+ self.heap.active_space_end() <= self.heap.idle_space_start|| self.heap.idle_space_end <= self.heap.active_space_start);@@ -997,11 +1031,11 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {self.process_roots();self.process_worklist();- assert!(self.heap.active_space_start <= self.heap.bump_ptr);- assert!(self.heap.bump_ptr <= self.heap.active_space_end);+ assert!(self.heap.active_space_start <= self.heap.bump_ptr());+ assert!(self.heap.bump_ptr() <= self.heap.active_space_end());assert!(self.heap.idle_space_start <= self.heap.idle_space_end);assert!(- self.heap.active_space_end <= self.heap.idle_space_start+ self.heap.active_space_end() <= self.heap.idle_space_start|| self.heap.idle_space_end <= self.heap.active_space_start);@@ -1009,7 +1043,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {self.heap.resize_semi_spaces();debug_assert_eq!(- self.heap.active_space_end - self.heap.active_space_start,+ self.heap.active_space_end() - self.heap.active_space_start,self.heap.idle_space_end - self.heap.idle_space_start,"the active and idle spaces should be the same size");@@ -1033,6 +1067,7 @@ impl GarbageCollection<'_> for CopyingCollection<'_> {#[cfg(test)]mod tests {use super::*;+ use wasmtime_environ::{HostPtr, PtrSize};#[test]fn vm_copying_header_size_align() {@@ -1083,4 +1118,36 @@ mod tests {<= wasmtime_environ::copying::MIN_OBJECT_SIZE as usize,);}++ #[test]+ fn vm_copying_heap_data_bump_ptr_offset() {+ assert_eq!(+ HostPtr.vmcopying_heap_data_bump_ptr() as usize,+ core::mem::offset_of!(VMCopyingHeapData, bump_ptr),+ );+ }++ #[test]+ fn vm_copying_heap_data_active_space_end_offset() {+ assert_eq!(+ HostPtr.vmcopying_heap_data_active_space_end() as usize,+ core::mem::offset_of!(VMCopyingHeapData, active_space_end),+ );+ }++ #[test]+ fn vm_copying_heap_data_size() {+ assert_eq!(+ HostPtr.size_of_vmcopying_heap_data() as usize,+ core::mem::size_of::<VMCopyingHeapData>(),+ );+ }++ #[test]+ fn vm_copying_heap_data_align() {+ assert_eq!(+ HostPtr.align_of_vmcopying_heap_data() as usize,+ core::mem::align_of::<VMCopyingHeapData>(),+ );+ }}diff --git a/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat b/tests/disas/gc/copying/array-new-fixed-of-gc-refs.watindex d73275c023..00693251b6 100644--- a/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat+++ b/tests/disas/gc/copying/array-new-fixed-of-gc-refs.wat@@ -23,40 +23,70 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32):-;; v44 = stack_addr.i64 ss2-;; store notrap v2, v44-;; v43 = stack_addr.i64 ss1-;; store notrap v3, v43-;; v42 = stack_addr.i64 ss0-;; store notrap v4, v42-;; @0025 v14 = iconst.i32 -1476395008-;; @0025 v16 = load.i64 notrap aligned readonly can_move v0+40-;; @0025 v17 = load.i32 notrap aligned readonly can_move v16-;; v56 = iconst.i32 32-;; @0025 v18 = iconst.i32 16-;; @0025 v19 = call fn0(v0, v14, v17, v56, v18), stack_map=[i32 @ ss2+0, i32 @ ss1+0, i32 @ ss0+0] ; v14 = -1476395008, v56 = 32, v18 = 16+;; v69 = stack_addr.i64 ss2+;; store notrap v2, v69+;; v68 = stack_addr.i64 ss1+;; store notrap v3, v68+;; v67 = stack_addr.i64 ss0+;; store notrap v4, v67+;; @0025 v14 = load.i64 notrap aligned readonly can_move v0+32+;; @0025 v15 = load.i32 notrap aligned can_move v14+;; @0025 v22 = uextend.i64 v15+;; v65 = iconst.i64 32+;; @0025 v23 = iadd v22, v65 ; v65 = 32+;; @0025 v16 = load.i32 notrap aligned readonly can_move v14+4+;; @0025 v24 = uextend.i64 v16+;; @0025 v25 = icmp ule v23, v24+;; @0025 brif v25, block2, block3+;;+;; block2:+;; v189 = iconst.i32 32+;; v95 = iadd.i32 v15, v189 ; v189 = 32+;; @0025 store notrap aligned vmctx v95, v14+;; v190 = iconst.i32 -1476395008+;; v191 = load.i64 notrap aligned readonly can_move v0+8+;; v192 = load.i64 notrap aligned readonly can_move v191+32+;; @0025 v39 = iadd v192, v22+;; @0025 store notrap aligned v190, v39 ; v190 = -1476395008+;; v193 = load.i64 notrap aligned readonly can_move v0+40+;; v194 = load.i32 notrap aligned readonly can_move v193+;; @0025 store notrap aligned v194, v39+4+;; v195 = iconst.i64 32+;; @0025 istore32 notrap aligned v195, v39+8 ; v195 = 32+;; @0025 jump block4(v15, v39)+;;+;; block3 cold:+;; @0025 v27 = iconst.i32 -1476395008+;; @0025 v29 = load.i64 notrap aligned readonly can_move v0+40+;; @0025 v30 = load.i32 notrap aligned readonly can_move v29+;; v81 = iconst.i32 32+;; @0025 v31 = iconst.i32 16+;; @0025 v32 = call fn0(v0, v27, v30, v81, v31), stack_map=[i32 @ ss2+0, i32 @ ss1+0, i32 @ ss0+0] ; v27 = -1476395008, v81 = 32, v31 = 16+;; @0025 v61 = load.i64 notrap aligned readonly can_move v0+8+;; @0025 v33 = load.i64 notrap aligned readonly can_move v61+32+;; @0025 v34 = uextend.i64 v32+;; @0025 v35 = iadd v33, v34+;; @0025 jump block4(v32, v35)+;;+;; block4(v44: i32, v45: i64):;; @0025 v6 = iconst.i32 3-;; @0025 v38 = load.i64 notrap aligned readonly can_move v0+8-;; @0025 v20 = load.i64 notrap aligned readonly can_move v38+32-;; @0025 v21 = uextend.i64 v19-;; @0025 v22 = iadd v20, v21-;; v37 = iconst.i64 16-;; @0025 v23 = iadd v22, v37 ; v37 = 16-;; @0025 store notrap aligned v6, v23 ; v6 = 3-;; v33 = load.i32 notrap v44-;; v58 = iconst.i64 20-;; v63 = iadd v22, v58 ; v58 = 20-;; @0025 store notrap aligned little v33, v63-;; v32 = load.i32 notrap v43-;; v66 = iconst.i64 24-;; v71 = iadd v22, v66 ; v66 = 24-;; @0025 store notrap aligned little v32, v71-;; v31 = load.i32 notrap v42-;; v87 = iconst.i64 28-;; v92 = iadd v22, v87 ; v87 = 28-;; @0025 store notrap aligned little v31, v92-;; @0029 jump block1+;; v60 = iconst.i64 16+;; @0025 v46 = iadd v45, v60 ; v60 = 16+;; @0025 store notrap aligned v6, v46 ; v6 = 3+;; v56 = load.i32 notrap v69+;; v98 = iconst.i64 20+;; v103 = iadd v45, v98 ; v98 = 20+;; @0025 store notrap aligned little v56, v103+;; v55 = load.i32 notrap v68+;; v106 = iconst.i64 24+;; v111 = iadd v45, v106 ; v106 = 24+;; @0025 store notrap aligned little v55, v111+;; v54 = load.i32 notrap v67+;; v127 = iconst.i64 28+;; v132 = iadd v45, v127 ; v127 = 28+;; @0025 store notrap aligned little v54, v132+;; @0029 jump block1(v44);;-;; block1:-;; @0029 return v19+;; block1(v5: i32):+;; @0029 return v5;; }diff --git a/tests/disas/gc/copying/array-new-fixed.wat b/tests/disas/gc/copying/array-new-fixed.watindex 726abb4204..3fbbe70777 100644--- a/tests/disas/gc/copying/array-new-fixed.wat+++ b/tests/disas/gc/copying/array-new-fixed.wat@@ -20,31 +20,61 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i64, v3: i64, v4: i64):-;; @0025 v14 = iconst.i32 -1476395008-;; @0025 v16 = load.i64 notrap aligned readonly can_move v0+40-;; @0025 v17 = load.i32 notrap aligned readonly can_move v16-;; v45 = iconst.i32 48-;; @0025 v18 = iconst.i32 16-;; @0025 v19 = call fn0(v0, v14, v17, v45, v18) ; v14 = -1476395008, v45 = 48, v18 = 16+;; @0025 v14 = load.i64 notrap aligned readonly can_move v0+32+;; @0025 v15 = load.i32 notrap aligned can_move v14+;; @0025 v22 = uextend.i64 v15+;; v71 = iconst.i64 48+;; @0025 v23 = iadd v22, v71 ; v71 = 48+;; @0025 v16 = load.i32 notrap aligned readonly can_move v14+4+;; @0025 v24 = uextend.i64 v16+;; @0025 v25 = icmp ule v23, v24+;; @0025 brif v25, block2, block3+;;+;; block2:+;; v154 = iconst.i32 48+;; v85 = iadd.i32 v15, v154 ; v154 = 48+;; @0025 store notrap aligned vmctx v85, v14+;; v155 = iconst.i32 -1476395008+;; v156 = load.i64 notrap aligned readonly can_move v0+8+;; v157 = load.i64 notrap aligned readonly can_move v156+32+;; @0025 v39 = iadd v157, v22+;; @0025 store notrap aligned v155, v39 ; v155 = -1476395008+;; v158 = load.i64 notrap aligned readonly can_move v0+40+;; v159 = load.i32 notrap aligned readonly can_move v158+;; @0025 store notrap aligned v159, v39+4+;; v160 = iconst.i64 48+;; @0025 istore32 notrap aligned v160, v39+8 ; v160 = 48+;; @0025 jump block4(v15, v39)+;;+;; block3 cold:+;; @0025 v27 = iconst.i32 -1476395008+;; @0025 v29 = load.i64 notrap aligned readonly can_move v0+40+;; @0025 v30 = load.i32 notrap aligned readonly can_move v29+;; v70 = iconst.i32 48+;; @0025 v31 = iconst.i32 16+;; @0025 v32 = call fn0(v0, v27, v30, v70, v31) ; v27 = -1476395008, v70 = 48, v31 = 16+;; @0025 v55 = load.i64 notrap aligned readonly can_move v0+8+;; @0025 v33 = load.i64 notrap aligned readonly can_move v55+32+;; @0025 v34 = uextend.i64 v32+;; @0025 v35 = iadd v33, v34+;; @0025 jump block4(v32, v35)+;;+;; block4(v44: i32, v45: i64):;; @0025 v6 = iconst.i32 3-;; @0025 v32 = load.i64 notrap aligned readonly can_move v0+8-;; @0025 v20 = load.i64 notrap aligned readonly can_move v32+32-;; @0025 v21 = uextend.i64 v19-;; @0025 v22 = iadd v20, v21-;; v31 = iconst.i64 16-;; @0025 v23 = iadd v22, v31 ; v31 = 16-;; @0025 store notrap aligned v6, v23 ; v6 = 3-;; v37 = iconst.i64 24-;; v51 = iadd v22, v37 ; v37 = 24-;; @0025 store notrap aligned little v2, v51-;; v34 = iconst.i64 32-;; v58 = iadd v22, v34 ; v34 = 32-;; @0025 store notrap aligned little v3, v58-;; v73 = iconst.i64 40-;; v78 = iadd v22, v73 ; v73 = 40-;; @0025 store notrap aligned little v4, v78-;; @0029 jump block1+;; v54 = iconst.i64 16+;; @0025 v46 = iadd v45, v54 ; v54 = 16+;; @0025 store notrap aligned v6, v46 ; v6 = 3+;; v62 = iconst.i64 24+;; v92 = iadd v45, v62 ; v62 = 24+;; @0025 store.i64 notrap aligned little v2, v92+;; v59 = iconst.i64 32+;; v99 = iadd v45, v59 ; v59 = 32+;; @0025 store.i64 notrap aligned little v3, v99+;; v114 = iconst.i64 40+;; v119 = iadd v45, v114 ; v114 = 40+;; @0025 store.i64 notrap aligned little v4, v119+;; @0029 jump block1(v44);;-;; block1:-;; @0029 return v19+;; block1(v5: i32):+;; @0029 return v5;; }diff --git a/tests/disas/gc/copying/array-new.wat b/tests/disas/gc/copying/array-new.watindex 8f5539acc1..82be522a68 100644--- a/tests/disas/gc/copying/array-new.wat+++ b/tests/disas/gc/copying/array-new.wat@@ -21,47 +21,83 @@;;;; block0(v0: i64, v1: i64, v2: i64, v3: i32):;; @0022 v6 = uextend.i64 v3-;; v37 = iconst.i64 3-;; v38 = ishl v6, v37 ; v37 = 3-;; v35 = iconst.i64 32-;; @0022 v8 = ushr v38, v35 ; v35 = 32+;; v62 = iconst.i64 3+;; v63 = ishl v6, v62 ; v62 = 3+;; v60 = iconst.i64 32+;; @0022 v8 = ushr v63, v60 ; v60 = 32;; @0022 trapnz v8, user17;; @0022 v5 = iconst.i32 24-;; v44 = iconst.i32 3-;; v45 = ishl v3, v44 ; v44 = 3-;; @0022 v10 = uadd_overflow_trap v5, v45, user17 ; v5 = 24-;; @0022 v12 = iconst.i32 -1476395008-;; @0022 v14 = load.i64 notrap aligned readonly can_move v0+40-;; @0022 v15 = load.i32 notrap aligned readonly can_move v14-;; @0022 v16 = iconst.i32 16-;; @0022 v17 = call fn0(v0, v12, v15, v10, v16) ; v12 = -1476395008, v16 = 16-;; @0022 v33 = load.i64 notrap aligned readonly can_move v0+8-;; @0022 v18 = load.i64 notrap aligned readonly can_move v33+32-;; @0022 v19 = uextend.i64 v17-;; @0022 v20 = iadd v18, v19-;; v32 = iconst.i64 16-;; @0022 v21 = iadd v20, v32 ; v32 = 16-;; @0022 store notrap aligned v3, v21-;; v49 = iconst.i64 24-;; v54 = iadd v20, v49 ; v49 = 24-;; @0022 v27 = uextend.i64 v10-;; @0022 v28 = iadd v20, v27-;; v36 = iconst.i64 8-;; @0022 jump block2(v54)+;; v69 = iconst.i32 3+;; v70 = ishl v3, v69 ; v69 = 3+;; @0022 v10 = uadd_overflow_trap v5, v70, user17 ; v5 = 24+;; @0022 v12 = load.i64 notrap aligned readonly can_move v0+32+;; @0022 v13 = load.i32 notrap aligned can_move v12+;; @0022 v20 = uextend.i64 v13+;; @0022 v15 = uextend.i64 v10+;; @0022 v16 = iconst.i64 15+;; @0022 v18 = iadd v15, v16 ; v16 = 15+;; @0022 v17 = iconst.i64 -16+;; @0022 v19 = band v18, v17 ; v17 = -16+;; @0022 v21 = iadd v20, v19+;; @0022 v14 = load.i32 notrap aligned readonly can_move v12+4+;; @0022 v22 = uextend.i64 v14+;; @0022 v23 = icmp ule v21, v22+;; @0022 brif v23, block2, block3;;-;; block2(v29: i64):-;; @0022 v30 = icmp eq v29, v28-;; @0022 brif v30, block4, block3+;; block2:+;; v78 = iconst.i32 15+;; v79 = iadd.i32 v10, v78 ; v78 = 15+;; v82 = iconst.i32 -16+;; v83 = band v79, v82 ; v82 = -16+;; v85 = iadd.i32 v13, v83+;; @0022 store notrap aligned vmctx v85, v12+;; v98 = iconst.i32 -1476395008+;; v99 = load.i64 notrap aligned readonly can_move v0+8+;; v100 = load.i64 notrap aligned readonly can_move v99+32+;; @0022 v37 = iadd v100, v20+;; @0022 store notrap aligned v98, v37 ; v98 = -1476395008+;; v101 = load.i64 notrap aligned readonly can_move v0+40+;; v102 = load.i32 notrap aligned readonly can_move v101+;; @0022 store notrap aligned v102, v37+4+;; v103 = band.i64 v18, v17 ; v17 = -16+;; @0022 istore32 notrap aligned v103, v37+8+;; @0022 jump block4(v13, v37);;-;; block3:-;; @0022 store.i64 notrap aligned little v2, v29-;; v59 = iconst.i64 8-;; v60 = iadd.i64 v29, v59 ; v59 = 8-;; @0022 jump block2(v60)+;; block3 cold:+;; @0022 v25 = iconst.i32 -1476395008+;; @0022 v27 = load.i64 notrap aligned readonly can_move v0+40+;; @0022 v28 = load.i32 notrap aligned readonly can_move v27+;; @0022 v29 = iconst.i32 16+;; @0022 v30 = call fn0(v0, v25, v28, v10, v29) ; v25 = -1476395008, v29 = 16+;; @0022 v56 = load.i64 notrap aligned readonly can_move v0+8+;; @0022 v31 = load.i64 notrap aligned readonly can_move v56+32+;; @0022 v32 = uextend.i64 v30+;; @0022 v33 = iadd v31, v32+;; @0022 jump block4(v30, v33);;-;; block4:-;; @0025 jump block1+;; block4(v42: i32, v43: i64):+;; v55 = iconst.i64 16+;; @0022 v44 = iadd v43, v55 ; v55 = 16+;; @0022 store.i32 notrap aligned v3, v44+;; v88 = iconst.i64 24+;; v93 = iadd v43, v88 ; v88 = 24+;; @0022 v51 = iadd v43, v15+;; v61 = iconst.i64 8+;; @0022 jump block5(v93);;-;; block1:-;; @0025 return v17+;; block5(v52: i64):+;; @0022 v53 = icmp eq v52, v51+;; @0022 brif v53, block7, block6+;;+;; block6:+;; @0022 store.i64 notrap aligned little v2, v52+;; v104 = iconst.i64 8+;; v105 = iadd.i64 v52, v104 ; v104 = 8+;; @0022 jump block5(v105)+;;+;; block7:+;; @0025 jump block1(v42)+;;+;; block1(v4: i32):+;; @0025 return v4;; }diff --git a/tests/disas/gc/copying/funcref-in-gc-heap-new.wat b/tests/disas/gc/copying/funcref-in-gc-heap-new.watindex 08b5e03b19..c2fb4c5a75 100644--- a/tests/disas/gc/copying/funcref-in-gc-heap-new.wat+++ b/tests/disas/gc/copying/funcref-in-gc-heap-new.wat@@ -23,26 +23,56 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i64):-;; @0020 v6 = iconst.i32 -1342177280-;; @0020 v8 = load.i64 notrap aligned readonly can_move v0+40-;; @0020 v9 = load.i32 notrap aligned readonly can_move v8+;; @0020 v6 = load.i64 notrap aligned readonly can_move v0+32+;; @0020 v7 = load.i32 notrap aligned can_move v6+;; @0020 v14 = uextend.i64 v7+;; v50 = iconst.i64 32+;; @0020 v15 = iadd v14, v50 ; v50 = 32+;; @0020 v8 = load.i32 notrap aligned readonly can_move v6+4+;; @0020 v16 = uextend.i64 v8+;; @0020 v17 = icmp ule v15, v16+;; @0020 brif v17, block2, block3+;;+;; block2:+;; v66 = iconst.i32 32+;; v64 = iadd.i32 v7, v66 ; v66 = 32+;; @0020 store notrap aligned vmctx v64, v6+;; v67 = iconst.i32 -1342177280+;; v68 = load.i64 notrap aligned readonly can_move v0+8+;; v69 = load.i64 notrap aligned readonly can_move v68+32+;; @0020 v31 = iadd v69, v14+;; @0020 store notrap aligned v67, v31 ; v67 = -1342177280+;; v70 = load.i64 notrap aligned readonly can_move v0+40+;; v71 = load.i32 notrap aligned readonly can_move v70+;; @0020 store notrap aligned v71, v31+4+;; v72 = iconst.i64 32+;; @0020 istore32 notrap aligned v72, v31+8 ; v72 = 32+;; @0020 jump block4(v7, v31)+;;+;; block3 cold:+;; @0020 v19 = iconst.i32 -1342177280+;; @0020 v21 = load.i64 notrap aligned readonly can_move v0+40+;; @0020 v22 = load.i32 notrap aligned readonly can_move v21;; @0020 v4 = iconst.i32 32-;; @0020 v10 = iconst.i32 16-;; @0020 v11 = call fn0(v0, v6, v9, v4, v10) ; v6 = -1342177280, v4 = 32, v10 = 16-;; v26 = stack_addr.i64 ss0-;; store notrap v11, v26-;; @0020 v17 = call fn1(v0, v2), stack_map=[i32 @ ss0+0]-;; @0020 v18 = ireduce.i32 v17-;; @0020 v24 = load.i64 notrap aligned readonly can_move v0+8-;; @0020 v12 = load.i64 notrap aligned readonly can_move v24+32-;; @0020 v13 = uextend.i64 v11-;; @0020 v14 = iadd v12, v13-;; v22 = iconst.i64 16-;; @0020 v15 = iadd v14, v22 ; v22 = 16-;; @0020 store notrap aligned little v18, v15-;; v19 = load.i32 notrap v26+;; @0020 v23 = iconst.i32 16+;; @0020 v24 = call fn0(v0, v19, v22, v4, v23) ; v19 = -1342177280, v4 = 32, v23 = 16+;; @0020 v46 = load.i64 notrap aligned readonly can_move v0+8+;; @0020 v25 = load.i64 notrap aligned readonly can_move v46+32+;; @0020 v26 = uextend.i64 v24+;; @0020 v27 = iadd v25, v26+;; @0020 jump block4(v24, v27)+;;+;; block4(v36: i32, v37: i64):+;; v45 = stack_addr.i64 ss0+;; store notrap v36, v45+;; @0020 v40 = call fn1(v0, v2), stack_map=[i32 @ ss0+0]+;; @0020 v41 = ireduce.i32 v40+;; v44 = iconst.i64 16+;; @0020 v38 = iadd v37, v44 ; v44 = 16+;; @0020 store notrap aligned little v41, v38+;; v42 = load.i32 notrap v45;; @0023 jump block1;;;; block1:-;; @0023 return v19+;; @0023 return v42;; }diff --git a/tests/disas/gc/copying/struct-new-default.wat b/tests/disas/gc/copying/struct-new-default.watindex 5d0cf53096..f2c6f96afe 100644--- a/tests/disas/gc/copying/struct-new-default.wat+++ b/tests/disas/gc/copying/struct-new-default.wat@@ -22,29 +22,59 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64):-;; @0021 v8 = iconst.i32 -1342177280-;; @0021 v10 = load.i64 notrap aligned readonly can_move v0+40-;; @0021 v11 = load.i32 notrap aligned readonly can_move v10+;; @0021 v8 = load.i64 notrap aligned readonly can_move v0+32+;; @0021 v9 = load.i32 notrap aligned can_move v8+;; @0021 v16 = uextend.i64 v9+;; v50 = iconst.i64 32+;; @0021 v17 = iadd v16, v50 ; v50 = 32+;; @0021 v10 = load.i32 notrap aligned readonly can_move v8+4+;; @0021 v18 = uextend.i64 v10+;; @0021 v19 = icmp ule v17, v18+;; @0021 brif v19, block2, block3+;;+;; block2:+;; v66 = iconst.i32 32+;; v64 = iadd.i32 v9, v66 ; v66 = 32+;; @0021 store notrap aligned vmctx v64, v8+;; v67 = iconst.i32 -1342177280+;; v68 = load.i64 notrap aligned readonly can_move v0+8+;; v69 = load.i64 notrap aligned readonly can_move v68+32+;; @0021 v33 = iadd v69, v16+;; @0021 store notrap aligned v67, v33 ; v67 = -1342177280+;; v70 = load.i64 notrap aligned readonly can_move v0+40+;; v71 = load.i32 notrap aligned readonly can_move v70+;; @0021 store notrap aligned v71, v33+4+;; v72 = iconst.i64 32+;; @0021 istore32 notrap aligned v72, v33+8 ; v72 = 32+;; @0021 jump block4(v9, v33)+;;+;; block3 cold:+;; @0021 v21 = iconst.i32 -1342177280+;; @0021 v23 = load.i64 notrap aligned readonly can_move v0+40+;; @0021 v24 = load.i32 notrap aligned readonly can_move v23;; @0021 v6 = iconst.i32 32-;; @0021 v12 = iconst.i32 16-;; @0021 v13 = call fn0(v0, v8, v11, v6, v12) ; v8 = -1342177280, v6 = 32, v12 = 16+;; @0021 v25 = iconst.i32 16+;; @0021 v26 = call fn0(v0, v21, v24, v6, v25) ; v21 = -1342177280, v6 = 32, v25 = 16+;; @0021 v46 = load.i64 notrap aligned readonly can_move v0+8+;; @0021 v27 = load.i64 notrap aligned readonly can_move v46+32+;; @0021 v28 = uextend.i64 v26+;; @0021 v29 = iadd v27, v28+;; @0021 jump block4(v26, v29)+;;+;; block4(v38: i32, v39: i64):;; @0021 v3 = f32const 0.0-;; @0021 v23 = load.i64 notrap aligned readonly can_move v0+8-;; @0021 v14 = load.i64 notrap aligned readonly can_move v23+32-;; @0021 v15 = uextend.i64 v13-;; @0021 v16 = iadd v14, v15-;; v22 = iconst.i64 16-;; @0021 v17 = iadd v16, v22 ; v22 = 16-;; @0021 store notrap aligned little v3, v17 ; v3 = 0.0+;; v45 = iconst.i64 16+;; @0021 v40 = iadd v39, v45 ; v45 = 16+;; @0021 store notrap aligned little v3, v40 ; v3 = 0.0;; @0021 v4 = iconst.i32 0-;; v21 = iconst.i64 20-;; @0021 v18 = iadd v16, v21 ; v21 = 20-;; @0021 istore8 notrap aligned little v4, v18 ; v4 = 0-;; v20 = iconst.i64 24-;; @0021 v19 = iadd v16, v20 ; v20 = 24-;; @0021 store notrap aligned little v4, v19 ; v4 = 0-;; @0024 jump block1+;; v44 = iconst.i64 20+;; @0021 v41 = iadd v39, v44 ; v44 = 20+;; @0021 istore8 notrap aligned little v4, v41 ; v4 = 0+;; v43 = iconst.i64 24+;; @0021 v42 = iadd v39, v43 ; v43 = 24+;; @0021 store notrap aligned little v4, v42 ; v4 = 0+;; @0024 jump block1(v38);;-;; block1:-;; @0024 return v13+;; block1(v2: i32):+;; @0024 return v2;; }diff --git a/tests/disas/gc/copying/struct-new.wat b/tests/disas/gc/copying/struct-new.watindex ee3b8c7a8d..0eaaab4919 100644--- a/tests/disas/gc/copying/struct-new.wat+++ b/tests/disas/gc/copying/struct-new.wat@@ -23,30 +23,60 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: f32, v3: i32, v4: i32):-;; v27 = stack_addr.i64 ss0-;; store notrap v4, v27-;; @002a v8 = iconst.i32 -1342177280-;; @002a v10 = load.i64 notrap aligned readonly can_move v0+40-;; @002a v11 = load.i32 notrap aligned readonly can_move v10+;; v52 = stack_addr.i64 ss0+;; store notrap v4, v52+;; @002a v8 = load.i64 notrap aligned readonly can_move v0+32+;; @002a v9 = load.i32 notrap aligned can_move v8+;; @002a v16 = uextend.i64 v9+;; v53 = iconst.i64 32+;; @002a v17 = iadd v16, v53 ; v53 = 32+;; @002a v10 = load.i32 notrap aligned readonly can_move v8+4+;; @002a v18 = uextend.i64 v10+;; @002a v19 = icmp ule v17, v18+;; @002a brif v19, block2, block3+;;+;; block2:+;; v69 = iconst.i32 32+;; v67 = iadd.i32 v9, v69 ; v69 = 32+;; @002a store notrap aligned vmctx v67, v8+;; v70 = iconst.i32 -1342177280+;; v71 = load.i64 notrap aligned readonly can_move v0+8+;; v72 = load.i64 notrap aligned readonly can_move v71+32+;; @002a v33 = iadd v72, v16+;; @002a store notrap aligned v70, v33 ; v70 = -1342177280+;; v73 = load.i64 notrap aligned readonly can_move v0+40+;; v74 = load.i32 notrap aligned readonly can_move v73+;; @002a store notrap aligned v74, v33+4+;; v75 = iconst.i64 32+;; @002a istore32 notrap aligned v75, v33+8 ; v75 = 32+;; @002a jump block4(v9, v33)+;;+;; block3 cold:+;; @002a v21 = iconst.i32 -1342177280+;; @002a v23 = load.i64 notrap aligned readonly can_move v0+40+;; @002a v24 = load.i32 notrap aligned readonly can_move v23;; @002a v6 = iconst.i32 32-;; @002a v12 = iconst.i32 16-;; @002a v13 = call fn0(v0, v8, v11, v6, v12), stack_map=[i32 @ ss0+0] ; v8 = -1342177280, v6 = 32, v12 = 16-;; @002a v25 = load.i64 notrap aligned readonly can_move v0+8-;; @002a v14 = load.i64 notrap aligned readonly can_move v25+32-;; @002a v15 = uextend.i64 v13-;; @002a v16 = iadd v14, v15-;; v24 = iconst.i64 16-;; @002a v17 = iadd v16, v24 ; v24 = 16-;; @002a store notrap aligned little v2, v17-;; v23 = iconst.i64 20-;; @002a v18 = iadd v16, v23 ; v23 = 20-;; @002a istore8 notrap aligned little v3, v18-;; v20 = load.i32 notrap v27-;; v22 = iconst.i64 24-;; @002a v19 = iadd v16, v22 ; v22 = 24-;; @002a store notrap aligned little v20, v19-;; @002d jump block1+;; @002a v25 = iconst.i32 16+;; @002a v26 = call fn0(v0, v21, v24, v6, v25), stack_map=[i32 @ ss0+0] ; v21 = -1342177280, v6 = 32, v25 = 16+;; @002a v48 = load.i64 notrap aligned readonly can_move v0+8+;; @002a v27 = load.i64 notrap aligned readonly can_move v48+32+;; @002a v28 = uextend.i64 v26+;; @002a v29 = iadd v27, v28+;; @002a jump block4(v26, v29)+;;+;; block4(v38: i32, v39: i64):+;; v47 = iconst.i64 16+;; @002a v40 = iadd v39, v47 ; v47 = 16+;; @002a store.f32 notrap aligned little v2, v40+;; v46 = iconst.i64 20+;; @002a v41 = iadd v39, v46 ; v46 = 20+;; @002a istore8.i32 notrap aligned little v3, v41+;; v43 = load.i32 notrap v52+;; v45 = iconst.i64 24+;; @002a v42 = iadd v39, v45 ; v45 = 24+;; @002a store notrap aligned little v43, v42+;; @002d jump block1(v38);;-;; block1:-;; @002d return v13+;; block1(v5: i32):+;; @002d return v5;; }