A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Remove preemption points in bulk operations (#14045)
Alex Crichton3ebfbe5af492
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 1e47b8c70d..8b65df4341 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -663,85 +663,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {builder.switch_to_block(continuation_block);}- /// Manually insert a fuel check, as opposed to what already happens around- /// normal loops headers and function entries.- ///- /// This can be used for expensive opcodes, such as `array.copy`, where the- /// operation's runtime is a function of the runtime state.- fn manual_fuel_check(&mut self, builder: &mut FunctionBuilder<'_>, fuel_to_consume: ir::Value) {- self.fuel_increment_var(builder);-- let fuel = builder.use_var(self.fuel_var);- let fuel = builder.ins().iadd(fuel, fuel_to_consume);- builder.def_var(self.fuel_var, fuel);-- self.fuel_check(builder);- }-- /// Consumes `units * cost_per_unit` fuel, saturating the charge at- /// `i64::MAX` so that an oversized unsigned operand cannot wrap around and- /// add fuel instead.- fn consume_variable_fuel(- &mut self,- builder: &mut FunctionBuilder<'_>,- units: ir::Value,- cost_per_unit: u8,- ) {- let may_exceed_i64_max = match builder.func.dfg.value_type(units) {- I32 => false,- I64 => true,- ty => unreachable!("unsupported variable fuel unit type: {ty}"),- };- self.consume_variable_fuel_impl(builder, units, cost_per_unit, may_exceed_i64_max);- }-- /// Like [`Self::consume_variable_fuel`], but for a unit count whose product- /// with `cost_per_unit` is statically known to fit in an `i64`.- fn consume_bounded_variable_fuel(- &mut self,- builder: &mut FunctionBuilder<'_>,- units: ir::Value,- cost_per_unit: u8,- ) {- self.consume_variable_fuel_impl(builder, units, cost_per_unit, false);- }-- fn consume_variable_fuel_impl(- &mut self,- builder: &mut FunctionBuilder<'_>,- units: ir::Value,- cost_per_unit: u8,- may_exceed_i64_max: bool,- ) {- if !self.tunables.consume_fuel || cost_per_unit == 0 {- return;- }-- let units = match builder.func.dfg.value_type(units) {- I32 => builder.ins().uextend(I64, units),- I64 => units,- ty => unreachable!("unsupported variable fuel unit type: {ty}"),- };- let fuel = if cost_per_unit == 1 {- units- } else {- builder.ins().imul_imm_s(units, i64::from(cost_per_unit))- };- let fuel = if may_exceed_i64_max {- let max = builder.ins().iconst(I64, i64::MAX);- let max_units = builder- .ins()- .iconst(I64, i64::MAX / i64::from(cost_per_unit));- let saturate = builder- .ins()- .icmp(IntCC::UnsignedGreaterThan, units, max_units);- builder.ins().select(saturate, max, fuel)- } else {- fuel- };- self.manual_fuel_check(builder, fuel);- }-fn epoch_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) {debug_assert!(self.epoch_deadline_var.is_reserved_value());self.epoch_deadline_var = builder.declare_var(ir::types::I64);@@ -2684,17 +2605,19 @@ impl FuncEnvironment<'_> {delta: ir::Value,init_value: ir::Value,) -> WasmResult<ir::Value> {+ let cost = self+ .tunables+ .operator_cost+ .variable()+ .table_grow_per_element;+ self.pre_translate_bulk_op(builder, delta, cost)?;+let mut pos = builder.cursor();let table = self.table(table_index);let (table_vmctx, defined_table_index) =self.table_vmctx_and_defined_index(&mut pos, table_index);let index_type = table.idx_type;let delta64 = self.cast_index_to_i64(&mut pos, delta, index_type);- let cost = self- .tunables- .operator_cost- .variable()- .table_grow_per_element;// Call out to the host to perform the actual growth of the underlying// table. This will initialize table slots as all null. Afterwards the@@ -2740,7 +2663,6 @@ impl FuncEnvironment<'_> {// A failed attempt performs no initialization loop, but still charge// for the requested growth so repeated failures are not free.builder.switch_to_block(failed_block);- self.consume_variable_fuel(builder, delta, cost);builder.ins().jump(done_block, &[]);builder.switch_to_block(fill_block);@@ -2748,12 +2670,11 @@ impl FuncEnvironment<'_> {builder,CheckedEntity::Table {table: table_index,- initialized: true,+ initialized: false,},result_idx,init_value,delta,- cost,)?;builder.ins().jump(done_block, &[]);@@ -2907,6 +2828,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_fill_per_element;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_fill(builder,CheckedEntity::Table {@@ -2916,7 +2838,6 @@ impl FuncEnvironment<'_> {dst,val,len,- cost,)}@@ -3053,8 +2974,7 @@ impl FuncEnvironment<'_> {elem: ir::Value,len: ir::Value,) -> WasmResult<ir::Value> {- let cost = self.tunables.operator_cost.variable().array_new_per_element;- gc::translate_array_new(self, builder, array_type_index, elem, len, cost)+ gc::translate_array_new(self, builder, array_type_index, elem, len)}pub fn translate_array_new_default(@@ -3063,12 +2983,7 @@ impl FuncEnvironment<'_> {array_type_index: TypeIndex,len: ir::Value,) -> WasmResult<ir::Value> {- let cost = self- .tunables- .operator_cost- .variable()- .array_new_default_per_element;- gc::translate_array_new_default(self, builder, array_type_index, len, cost)+ gc::translate_array_new_default(self, builder, array_type_index, len)}pub fn translate_array_new_fixed(@@ -3151,6 +3066,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().array_copy_per_element;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_copy(builder,CheckedEntity::Array {@@ -3166,7 +3082,6 @@ impl FuncEnvironment<'_> {dst_index,src_index,len,- cost,)}@@ -3185,6 +3100,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().array_fill_per_element;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_fill(builder,CheckedEntity::Array {@@ -3195,7 +3111,6 @@ impl FuncEnvironment<'_> {index,value,len,- cost,)}@@ -3216,6 +3131,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().array_init_data_per_element;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_copy(builder,CheckedEntity::Array {@@ -3230,7 +3146,6 @@ impl FuncEnvironment<'_> {dst_index,data_offset,len,- cost,)}@@ -3250,6 +3165,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().array_init_elem_per_element;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_copy(builder,CheckedEntity::Array {@@ -3261,7 +3177,6 @@ impl FuncEnvironment<'_> {dst_index,elem_offset,len,- cost,)}@@ -3634,7 +3549,7 @@ impl FuncEnvironment<'_> {let index_type = self.memory(index).idx_type;let cost = self.tunables.operator_cost.variable().memory_grow_per_page;- self.consume_variable_fuel(builder, val, cost);+ self.pre_translate_bulk_op(builder, val, cost)?;let mut pos = builder.cursor();let val = self.cast_index_to_i64(&mut pos, val, index_type);let call_inst = pos@@ -3733,7 +3648,8 @@ impl FuncEnvironment<'_> {len: ir::Value,) -> WasmResult<()> {let cost = self.tunables.operator_cost.variable().memory_copy_per_byte;- self.translate_entity_copy(builder, dst_index, src_index, dst, src, len, cost)+ self.pre_translate_bulk_op(builder, len, cost)?;+ self.translate_entity_copy(builder, dst_index, src_index, dst, src, len)}/// Perform a raw bulk-memory-like libcall.@@ -3741,7 +3657,11 @@ impl FuncEnvironment<'_> {/// The main purpose of this helper is to handle situations when fuel and/// epochs are enabled to break up the copy into a loop of chunks with/// preemption checks between them.- fn raw_bulk_memory_operation(&mut self, builder: &mut FunctionBuilder<'_>, mut op: BulkOp) {+ fn raw_bulk_memory_operation(+ &mut self,+ builder: &mut FunctionBuilder<'_>,+ op: BulkOp,+ ) -> WasmResult<()> {// Fast path: a copy whose byte length is a small compile-time constant is// expanded inline (see `emit_inline_memcpy`), skipping the libcall's fixed// per-call cost (a wasm/host transition and an indirect call) that@@ -3758,259 +3678,33 @@ impl FuncEnvironment<'_> {const_len: Some(bytes),src_entity,dst_entity,- fuel,..} = op{if bytes <= INLINE_COPY_MAX_BYTES {- if self.tunables.consume_fuel {- let units = bytes / u64::from(fuel.bytes_per_unit);- self.fuel_consumed +=- i64::try_from(units * u64::from(fuel.cost_per_unit)).unwrap();- }let src_region = self.bulk_copy_alias_region(builder.func, src_entity);let dst_region = self.bulk_copy_alias_region(builder.func, dst_entity);self.emit_inline_memcpy(builder, dst, src, bytes, src_region, dst_region);- return;+ return Ok(());}}- // Very scientifically chosen. Or, more seriously, this is just an- // arbitrary number for now. 100k copies of this size locally takes half- // a second, so seems like a reasonably large chunk size to not hit perf- // too much by chunking but also enable time slicing.- const UNINTERRUPTABLE_CHUNK_SIZE: i64 = 128 << 20;-let mut pos = builder.cursor();let vmctx = self.vmctx_val(&mut pos);- let pointer_type = self.pointer_type();// Performs a raw call to the actual libcall, as dictated by the- // provided `op`. This inserts configured epoch/fuel checks before the- // call.- let raw_call =- |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &BulkOp| {- if env.tunables.epoch_interruption {- env.epoch_check(builder);- }- let fuel = op.fuel();- if env.tunables.consume_fuel && fuel.cost_per_unit != 0 {- let byte_len = op.len();- debug_assert!(fuel.bytes_per_unit.is_power_of_two());- let units = if fuel.bytes_per_unit == 1 {- byte_len- } else {- builder- .ins()- .ushr_imm_u(byte_len, i64::from(fuel.bytes_per_unit.trailing_zeros()))- };- // With fuel enabled all calls emitted below are limited to- // `UNINTERRUPTABLE_CHUNK_SIZE`, so this multiplication- // cannot exceed `i64::MAX` even at the maximum `u8` rate.- env.consume_bounded_variable_fuel(builder, units, fuel.cost_per_unit);- }- match *op {- BulkOp::MemoryCopy { dst, src, len, .. } => {- let memory_copy = env.builtin_functions.memory_copy(&mut builder.func);- builder.ins().call(memory_copy, &[vmctx, dst, src, len]);- }- BulkOp::MemoryFill { dst, val, len, .. } => {- let memory_fill = env.builtin_functions.memory_fill(&mut builder.func);- builder.ins().call(memory_fill, &[vmctx, dst, val, len]);- }- }- };-- // If epochs and fuel are disabled, then just call the libcall and- // return. No need for the loops below.- if !self.tunables.epoch_interruption && !self.tunables.consume_fuel {- raw_call(self, builder, &op);- return;- }-- // If fuel is enabled, first take all the pending fuel and flush it to- // our internal variable. This is necessary to avoid picking up all- // pending fuel on each turn of the loop below.- if self.tunables.consume_fuel {- self.fuel_increment_var(builder);- }-- let current_block = builder.current_block().unwrap();- let chunk_block = builder.create_block();- let last_chunk_block = builder.create_block();-- builder.ensure_inserted_block();- builder.insert_block_after(chunk_block, current_block);- builder.insert_block_after(last_chunk_block, chunk_block);-- let chunk = builder- .ins()- .iconst(pointer_type, UNINTERRUPTABLE_CHUNK_SIZE);-- // For `memcpy` when chunking this up we might need to do a backwards- // copy or a forwards copy. Determine that here and jump to the- // backwards copy if needed.- let backwards_block = if let BulkOp::MemoryCopy { dst, src, .. } = op {- let forwards = builder.ins().icmp(IntCC::UnsignedGreaterThan, src, dst);- let forwards_block = builder.create_block();- let backwards_block = builder.create_block();- builder- .ins()- .brif(forwards, forwards_block, &[], backwards_block, &[]);- builder.switch_to_block(forwards_block);- builder.seal_block(forwards_block);- Some((backwards_block, op.clone()))- } else {- None- };-- // Helper closure to test if the length in `op` is larger than `chunk`,- // and if so do a single chunk. Else this goes to the final block with- // the final operation.- let has_chunk_branch =- |builder: &mut FunctionBuilder<'_>, op: &_, chunk_block, last_chunk_block| {- let len = match *op {- BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len,- };- let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk);- match *op {- BulkOp::MemoryCopy { dst, src, len, .. } => {- builder.ins().brif(- has_chunk,- chunk_block,- &[dst.into(), src.into(), len.into()],- last_chunk_block,- &[dst.into(), src.into(), len.into()],- );- }- BulkOp::MemoryFill { dst, len, .. } => {- builder.ins().brif(- has_chunk,- chunk_block,- &[dst.into(), len.into()],- last_chunk_block,- &[dst.into(), len.into()],- );- }- }- };-- let append_block_params = |builder: &mut FunctionBuilder<'_>, block, op: &mut _| match op {- BulkOp::MemoryCopy { dst, src, len, .. } => {- *dst = builder.append_block_param(block, pointer_type);- *src = builder.append_block_param(block, pointer_type);- *len = builder.append_block_param(block, pointer_type);- }- BulkOp::MemoryFill { dst, len, .. } => {- *dst = builder.append_block_param(block, pointer_type);- *len = builder.append_block_param(block, pointer_type);- }- };-- // Forwards copy: dispatch to the per-chunk loop or the final iteration- // if there's no chunks.- has_chunk_branch(builder, &op, chunk_block, last_chunk_block);-- // Forwards copy: In the block with per-chunk copies, each operation- // performs `chunk` length of bytes and then decrements the current- // length by `chunk`. Afterwards a condition tests if we do another- // chunk or break out for the final chunk.- builder.switch_to_block(chunk_block);- append_block_params(builder, chunk_block, &mut op);- let op_len = match &mut op {- BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len,- };- let remaining_len = *op_len;- *op_len = chunk;- raw_call(self, builder, &op);- match &mut op {+ // provided `op`.+ match op {BulkOp::MemoryCopy { dst, src, len, .. } => {- *dst = builder.ins().iadd(*dst, chunk);- *src = builder.ins().iadd(*src, chunk);- *len = builder.ins().isub(remaining_len, chunk);+ let memory_copy = self.builtin_functions.memory_copy(&mut builder.func);+ builder.ins().call(memory_copy, &[vmctx, dst, src, len]);}- BulkOp::MemoryFill { len, dst, .. } => {- *dst = builder.ins().iadd(*dst, chunk);- *len = builder.ins().isub(remaining_len, chunk);+ BulkOp::MemoryFill { dst, val, len } => {+ let memory_fill = self.builtin_functions.memory_fill(&mut builder.func);+ builder.ins().call(memory_fill, &[vmctx, dst, val, len]);}- };- has_chunk_branch(builder, &op, chunk_block, last_chunk_block);- builder.seal_block(chunk_block);-- // Backwards copy: similar to the above but with adjustments on where- // increments/decrements happen. Notably:- //- // * Initial src/end are the final byte address- // * Each chunk starts out by decrementing src/end as opposed to above- // where the increment happens at the end.- // * The final block performs the final decrement before jumping to the- // shared `last_chunk_block` between the forwards/backwards paths.- if let Some((backwards_block, mut op)) = backwards_block {- // Setup `dst=dst+len` and `src=src+len`, then see if we have a- // chunk.- builder.switch_to_block(backwards_block);- builder.seal_block(backwards_block);- let backwards_chunk_block = builder.create_block();- let backwards_last_chunk_block = builder.create_block();- let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {- unreachable!()- };- *dst = builder.ins().iadd(*dst, *len);- *src = builder.ins().iadd(*src, *len);- has_chunk_branch(- builder,- &op,- backwards_chunk_block,- backwards_last_chunk_block,- );-- // Execute the per-chunk backwards copy, adjusting pointers before- // the copy itself.- builder.switch_to_block(backwards_chunk_block);- append_block_params(builder, backwards_chunk_block, &mut op);- let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {- unreachable!()- };- let remaining_len = *len;- *len = chunk;- *dst = builder.ins().isub(*dst, chunk);- *src = builder.ins().isub(*src, chunk);- raw_call(self, builder, &op);- let BulkOp::MemoryCopy { len, .. } = &mut op else {- unreachable!()- };- *len = builder.ins().isub(remaining_len, chunk);- has_chunk_branch(- builder,- &op,- backwards_chunk_block,- backwards_last_chunk_block,- );- builder.seal_block(backwards_chunk_block);-- // Final backwards chunk: adjust the dst/src to be their true base- // pointers and then delegate to `last_chunk_block` for the actual- // memcpy.- builder.switch_to_block(backwards_last_chunk_block);- builder.seal_block(backwards_last_chunk_block);- append_block_params(builder, backwards_last_chunk_block, &mut op);- let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {- unreachable!()- };- *dst = builder.ins().isub(*dst, *len);- *src = builder.ins().isub(*src, *len);- builder.ins().jump(- last_chunk_block,- &[(*dst).into(), (*src).into(), (*len).into()],- );}-- // In the final block we know that the length of the operation is less- // than `chunk`.- builder.switch_to_block(last_chunk_block);- builder.seal_block(last_chunk_block);- append_block_params(builder, last_chunk_block, &mut op);- raw_call(self, builder, &op);+ Ok(())}/// Emits a generic "fill" of `entity` from `dst` for `len` elements,@@ -4021,6 +3715,9 @@ impl FuncEnvironment<'_> {/// `dst` and `len` values must be typed appropriately for `entity`. This/// will perform a bounds-check before actually executing the operation and/// then afterwards will perform the operation.+ ///+ /// Callers must invoke `pre_translate_bulk_op` before calling this method+ /// to properly account for fuel/epoch checks for this bulk operation.fn translate_entity_fill(&mut self,builder: &mut FunctionBuilder<'_>,@@ -4028,7 +3725,6 @@ impl FuncEnvironment<'_> {dst: ir::Value,val: ir::Value,len: ir::Value,- cost_per_unit: u8,) -> WasmResult<()> {let entity = entity.into();let idx_type = entity.index_type(self);@@ -4042,37 +3738,22 @@ impl FuncEnvironment<'_> {self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, idx_type);match entity {- CheckedEntity::Memory(_) => {- self.raw_bulk_memory_operation(- builder,- BulkOp::MemoryFill {- dst: raw_dst_addr,- val,- len: len_ptr,- fuel: BulkFuel {- cost_per_unit,- bytes_per_unit: 1,- },- },- );- }- CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => {- self.emit_raw_array_or_table_fill(- builder,- entity,- raw_dst_addr,+ CheckedEntity::Memory(_) => self.raw_bulk_memory_operation(+ builder,+ BulkOp::MemoryFill {+ dst: raw_dst_addr,val,- len_ptr,- cost_per_unit,- )?;+ len: len_ptr,+ },+ ),+ CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => {+ self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr)}// Not allowed to be written to in wasm.CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {unreachable!()}}-- Ok(())}/// Performs a manual element-by-element fill of `entity`, starting at@@ -4096,7 +3777,6 @@ impl FuncEnvironment<'_> {dst_elem_addr: ir::Value,value: ir::Value,copy_len: ir::Value,- cost_per_element: u8,) -> WasmResult<()> {let pointer_ty = self.pointer_type();@@ -4116,19 +3796,14 @@ impl FuncEnvironment<'_> {if entity.allows_memset(self)&& let Some(value) = self.fill_value_as_memset(builder, elem_ty, value){- self.raw_bulk_memory_operation(+ return self.raw_bulk_memory_operation(builder,BulkOp::MemoryFill {dst: dst_elem_addr,val: value,len: copy_byte_len,- fuel: BulkFuel {- cost_per_unit: cost_per_element,- bytes_per_unit: u8::try_from(elem_size).unwrap(),- },},);- return Ok(());}// Funcref values are intern'd when stored on the GC heap, and the@@ -4169,12 +3844,6 @@ impl FuncEnvironment<'_> {builder.insert_block_after(loop_block, current_block);builder.insert_block_after(continue_block, loop_block);- // Before entering the loop below flush our fuel counters to ensure- // that previous instructions' fuel isn't counted once-per-iteration.- if self.tunables.consume_fuel {- self.fuel_increment_var(builder);- }-// Current block: test to see if this is actually an empty copy. If it// is then skip over the entire loop, otherwise enter the loop and// perform the first ieration.@@ -4192,11 +3861,6 @@ impl FuncEnvironment<'_> {// by the element size, then see if we turn again or exit.builder.switch_to_block(loop_block);let elem_addr = builder.append_block_param(loop_block, pointer_ty);- // Consume the configured cost for this element before writing it.- if self.tunables.consume_fuel {- self.fuel_consumed += i64::from(cost_per_element);- }- self.translate_loop_header(builder)?;match entity {CheckedEntity::Table { table, initialized } => {assert!(!is_pre_interned_funcref);@@ -4312,7 +3976,8 @@ impl FuncEnvironment<'_> {len: ir::Value,) -> WasmResult<()> {let cost = self.tunables.operator_cost.variable().memory_fill_per_byte;- self.translate_entity_fill(builder, memory_index, dst, val, len, cost)+ self.pre_translate_bulk_op(builder, len, cost)?;+ self.translate_entity_fill(builder, memory_index, dst, val, len)}pub fn translate_memory_init(@@ -4326,6 +3991,7 @@ impl FuncEnvironment<'_> {) -> WasmResult<()> {let seg_index = DataIndex::from_u32(seg_index);let cost = self.tunables.operator_cost.variable().memory_init_per_byte;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_copy(builder,memory_index,@@ -4336,7 +4002,6 @@ impl FuncEnvironment<'_> {dst,src,len,- cost,)}@@ -4379,6 +4044,9 @@ impl FuncEnvironment<'_> {/// of the copy. Both `dst` and `src` have types appropriate to index their/// respective entities, and `len` has a type that's the smaller of the two/// index types.+ ///+ /// Callers must invoke `pre_translate_bulk_op` before calling this method+ /// to properly account for fuel/epoch checks for this bulk operation.fn translate_entity_copy(&mut self,builder: &mut FunctionBuilder<'_>,@@ -4387,7 +4055,6 @@ impl FuncEnvironment<'_> {dst: ir::Value,src: ir::Value,len: ir::Value,- cost_per_unit: u8,) -> WasmResult<()> {let dst_entity = dst_entity.into();let src_entity = src_entity.into();@@ -4449,13 +4116,8 @@ impl FuncEnvironment<'_> {const_len: const_count,src_entity,dst_entity,- fuel: BulkFuel {- cost_per_unit,- bytes_per_unit: 1,- },},- );- Ok(())+ )}// Tables/arrays are sometimes a memcpy, sometimes a per-element@@ -4470,7 +4132,6 @@ impl FuncEnvironment<'_> {len_ptr,src,const_count,- cost_per_unit,),// Cannot copy into a data or element segment in wasm.@@ -4686,6 +4347,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_copy_per_element;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_copy(builder,CheckedEntity::Table {@@ -4699,7 +4361,6 @@ impl FuncEnvironment<'_> {dst,src,len,- cost,)}@@ -4729,7 +4390,6 @@ impl FuncEnvironment<'_> {copy_len: ir::Value,src_index: ir::Value,const_count: Option<u64>,- cost_per_element: u8,) -> WasmResult<()> {let pointer_type = self.pointer_type();assert_eq!(builder.func.dfg.value_type(dst_elem_addr), pointer_type);@@ -4808,7 +4468,7 @@ impl FuncEnvironment<'_> {// parameters (or expand it inline; see `raw_bulk_memory_operation`).if !type_forbids_memcpy && dst_element_size == src_element_size {let const_len = const_count.and_then(|c| c.checked_mul(u64::from(dst_element_size)));- self.raw_bulk_memory_operation(+ return self.raw_bulk_memory_operation(builder,BulkOp::MemoryCopy {dst: dst_elem_addr,@@ -4817,13 +4477,8 @@ impl FuncEnvironment<'_> {const_len,src_entity,dst_entity,- fuel: BulkFuel {- cost_per_unit: cost_per_element,- bytes_per_unit: u8::try_from(dst_element_size).unwrap(),- },},);- return Ok(());}// For other copies, this is a per-element loop. Use the helper to@@ -4837,7 +4492,6 @@ impl FuncEnvironment<'_> {src_elem_addr,copy_len,src_index,- cost_per_element,&|this, builder, dst, src, src_index| {let write_ty = dst_entity.storage_type(this);let val = match src_entity {@@ -5044,7 +4698,6 @@ impl FuncEnvironment<'_> {src_elem_addr: ir::Value,copy_len: ir::Value,src_index: ir::Value,- cost_per_element: u8,copy_one: &dyn Fn(&mut Self,&mut FunctionBuilder<'_>,@@ -5096,13 +4749,6 @@ impl FuncEnvironment<'_> {builder.insert_block_after(backwards_block, forward_block);builder.insert_block_after(done_block, backwards_block);- // Update our local fuel counter, if enabled, before entering the loops- // below. This zeros out `self.fuel_consumed` so we don't consume- // previous fuel on each iteration of the loop.- if self.tunables.consume_fuel {- self.fuel_increment_var(builder);- }-// Terminate `current_block` by testing to see if we're copying any// elements at all.builder@@ -5192,11 +4838,6 @@ impl FuncEnvironment<'_> {let src_cur = builder.append_block_param(forward_block, self.pointer_type());let src_index = builder.append_block_param(forward_block, src_index_ty);let forward_keepalives = append_keepalive_params(builder, forward_block);- // Consume the configured cost for this element before copying it.- if self.tunables.consume_fuel {- self.fuel_consumed += i64::from(cost_per_element);- }- self.translate_loop_header(builder)?;copy_one(self, builder, dst_cur, src_cur, src_index)?;let dst_next = builder.ins()@@ -5222,10 +4863,6 @@ impl FuncEnvironment<'_> {let src_cur = builder.append_block_param(backwards_block, self.pointer_type());let src_index = builder.append_block_param(backwards_block, src_index_ty);let backward_keepalives = append_keepalive_params(builder, backwards_block);- if self.tunables.consume_fuel {- self.fuel_consumed += i64::from(cost_per_element);- }- self.translate_loop_header(builder)?;let dst_cur = {let size = builder.ins()@@ -5281,6 +4918,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_init_per_element;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_copy(builder,CheckedEntity::Table {@@ -5291,7 +4929,6 @@ impl FuncEnvironment<'_> {dst,src,len,- cost,)}@@ -5355,6 +4992,78 @@ impl FuncEnvironment<'_> {Ok(builder.ins().ireduce(ir::types::I32, ret))}+ /// Translation prefix before bulk operations such as `memory.copy`.+ ///+ /// Takes a dynamic value `units` for the size of the operation as well as+ /// a `cost_per_unit` configured for this operation. If fuel is enabled+ /// this fuel will be consumed, and if epochs are enabled then an epoch+ /// check happens. If neither epochs nor fuel are enabled this is a noop.+ fn pre_translate_bulk_op(+ &mut self,+ builder: &mut FunctionBuilder,+ units: ir::Value,+ cost_per_unit: u8,+ ) -> WasmResult<()> {+ let const_units =+ Self::value_as_const_int(builder, units).map(|c| i64::try_from(c).unwrap_or(i64::MAX));++ if self.tunables.consume_fuel && cost_per_unit > 0 {+ match const_units {+ // Fold constant costs directly into internal state.+ Some(units) => {+ self.fuel_consumed = self+ .fuel_consumed+ .saturating_add(units.saturating_mul(i64::from(cost_per_unit)))+ }++ None => {+ // Note that fuel is always a 64-bit counter.+ //+ // Also note that the cost is clamped to `i64::MAX` to+ // prevent fuel counter overflows since `cost` is otherwise+ // an untrusted value.+ let units_clamped64 = match builder.func.dfg.value_type(units) {+ ir::types::I32 => {+ let units64 = builder.ins().uextend(ir::types::I64, units);+ builder.ins().imul_imm_u(units64, i64::from(cost_per_unit))+ }+ ir::types::I64 => {+ let fuel = builder.ins().imul_imm_u(units, i64::from(cost_per_unit));+ let max = builder.ins().iconst(ir::types::I64, i64::MAX);+ let max_units = builder+ .ins()+ .iconst(I64, i64::MAX / i64::from(cost_per_unit));+ let saturate =+ builder+ .ins()+ .icmp(IntCC::UnsignedGreaterThan, units, max_units);+ builder.ins().select(saturate, max, fuel)+ }+ _ => unreachable!(),+ };+ self.fuel_increment_var(builder);+ let fuel = builder.use_var(self.fuel_var);+ let fuel = builder.ins().iadd(fuel, units_clamped64);+ builder.def_var(self.fuel_var, fuel);+ }+ }+ }++ // Skip explicit fuel/epoch checks for operations which are+ // subjectively, and statically, considered cheap.+ const SMALL_BULK_OP_COST: i64 = 128;+ if let Some(units) = const_units+ && let Some(cost) = units.checked_mul(i64::from(cost_per_unit))+ && cost <= SMALL_BULK_OP_COST+ {+ return Ok(());+ }++ // This isn't a loop header but for fuel/epoch purposes it's the same+ // thing.+ self.translate_loop_header(builder)+ }+pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {// Additionally if enabled check how much fuel we have remaining to see// if we've run out by this point.@@ -6242,6 +5951,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_fill_per_element;+ self.pre_translate_bulk_op(builder, len, cost)?;self.translate_entity_fill(builder,CheckedEntity::Table {@@ -6251,7 +5961,6 @@ impl FuncEnvironment<'_> {dst,val,len,- cost,)}@@ -6287,7 +5996,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_init_per_element;- self.consume_variable_fuel(builder, segment_len, cost);+ self.pre_translate_bulk_op(builder, segment_len, cost)?;// Re-use the `table.set` translation for making this a simple function// to define. That re-executes the bounds check which is a bit@@ -6360,7 +6069,8 @@ impl FuncEnvironment<'_> {let len = self.load_runtime_data_length(builder, data);let start = builder.ins().iconst(I32, 0);let cost = self.tunables.operator_cost.variable().memory_init_per_byte;- self.translate_entity_copy(builder, memory, data, offset, start, len, cost)?;+ self.pre_translate_bulk_op(builder, len, cost)?;+ self.translate_entity_copy(builder, memory, data, offset, start, len)?;// Finalize control-flow for the `MemorySegmentOffset::Static` case// above.@@ -6530,7 +6240,6 @@ enum BulkOp {const_len: Option<u64>,src_entity: CheckedEntity,dst_entity: CheckedEntity,- fuel: BulkFuel,},/// A `memory.fill` operation, setting all bytes of `dst` to `val`.@@ -6543,31 +6252,9 @@ enum BulkOp {dst: ir::Value,val: ir::Value,len: ir::Value,- fuel: BulkFuel,},}-impl BulkOp {- fn len(&self) -> ir::Value {- match self {- BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => *len,- }- }-- fn fuel(&self) -> BulkFuel {- match self {- BulkOp::MemoryCopy { fuel, .. } | BulkOp::MemoryFill { fuel, .. } => *fuel,- }- }-}--#[derive(Copy, Clone)]-struct BulkFuel {- cost_per_unit: u8,- /// Number of copied bytes represented by one billable unit.- bytes_per_unit: u8,-}-/// A list of entities which can participate in various kinds of bulk operations/// in wasm.///diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rsindex 183b0d1d1c..ac8275f027 100644--- a/crates/cranelift/src/func_environ/gc.rs+++ b/crates/cranelift/src/func_environ/gc.rs@@ -801,9 +801,15 @@ pub fn translate_array_new(array_type_index: TypeIndex,elem: ir::Value,len: ir::Value,- cost_per_element: u8,) -> WasmResult<ir::Value> {log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})");+ let cost = func_env+ .tunables+ .operator_cost+ .variable()+ .array_new_per_element;+ func_env.pre_translate_bulk_op(builder, len, cost)?;+let result =gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;let zero = builder.ins().iconst(ir::types::I32, 0);@@ -818,7 +824,6 @@ pub fn translate_array_new(zero,elem,len,- cost_per_element,)?;log::trace!("translate_array_new(..) -> {result:?}");Ok(result)@@ -829,9 +834,14 @@ pub fn translate_array_new_default(builder: &mut FunctionBuilder,array_type_index: TypeIndex,len: ir::Value,- cost_per_element: u8,) -> WasmResult<ir::Value> {log::trace!("translate_array_new_default({array_type_index:?}, {len:?})");+ let cost = func_env+ .tunables+ .operator_cost+ .variable()+ .array_new_default_per_element;+ func_env.pre_translate_bulk_op(builder, len, cost)?;let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index();let array_ty = func_env.types.unwrap_array(interned_ty)?;@@ -850,7 +860,6 @@ pub fn translate_array_new_default(zero,elem,len,- cost_per_element,)?;Ok(result)}@@ -1754,8 +1763,10 @@ pub fn translate_array_new_entity(entity: CheckedEntity,entity_offset: ir::Value,len: ir::Value,- cost_per_element: u8,+ cost_per_unit: u8,) -> WasmResult<ir::Value> {+ env.pre_translate_bulk_op(builder, len, cost_per_unit)?;+// Before actually allocating this array first do a bounds-check on the// passive entity itself.let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index();@@ -1774,7 +1785,6 @@ pub fn translate_array_new_entity(dst,entity_offset,len,- cost_per_element,)?;Ok(array)diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rsindex 98090ecfc4..fd411683ef 100644--- a/crates/wasmtime/src/config.rs+++ b/crates/wasmtime/src/config.rs@@ -690,10 +690,6 @@ impl Config {/// signal handler), then we can ensure that all async code will/// yield to the executor within a bounded time.///- /// The deadline check cannot be avoided by malicious wasm code. It is safe- /// to use epoch deadlines to limit the execution time of untrusted- /// code.- ////// The [`Store`](crate::Store) tracks the deadline, and controls/// what happens when the deadline is reached during/// execution. Several behaviors are possible:@@ -739,6 +735,27 @@ impl Config {/// computation and have the desired effect of cancelling a blocking/// operation when a timeout expires.///+ /// ## Limitations with malicious guests+ ///+ /// Epochs are designed to handle malicious WebAssembly guests -- the+ /// deadline check cannot be avoided by WebAssembly code. It is safe to use+ /// epoch deadlines to limit the execution time of untrusted code.+ ///+ /// Note, though, that a current limitation to this is that+ /// bulk-data-transfer instructions, such as `memory.copy`, only check the+ /// epoch once at the start of the operation. These operations can take a+ /// variable amount of time to complete based on how many bytes are being+ /// copied. This means that the maximal time slice a guest might take is+ /// the maximum of the epoch interval and the largest+ /// memory-copy-style-instruction executed. The size of a copy is bounded+ /// on the size of linear memory or GC heap size. In the limit, however, a+ /// guest using a 64-bit linear memory with a 128GiB size could issue a+ /// 128GiB `memory.copy` which would have no preemption within the+ /// instruction itself. Hosts which need strict time limits for guests right+ /// now are recommended to ensure that the store's allocated heap size+ /// (linear memory + GC heap) are bounded with a+ /// [`ResourceLimiter`](crate::ResourceLimiter).+ ////// ## When to use fuel vs. epochs////// In general, epoch-based interruption results in fasterdiff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rsindex 9cc49ea836..a0816548b8 100644--- a/crates/wasmtime/src/runtime/store.rs+++ b/crates/wasmtime/src/runtime/store.rs@@ -1041,6 +1041,10 @@ impl<T> Store<T> {/// The `interval` parameter indicates how much fuel should be/// consumed between yields of an async future. When fuel runs out wasm will trap.///+ /// For limitations related to consumption of fuel and when yield points are+ /// injected, see the discussion in+ /// [`Config::epoch_interruption`](crate::Config::epoch_interruption).+ ////// # Error////// This method will error if fuel is not enabled or `interval` isdiff --git a/tests/all/epoch_interruption.rs b/tests/all/epoch_interruption.rsindex 6a1c1ff7bb..1d2b4664f3 100644--- a/tests/all/epoch_interruption.rs+++ b/tests/all/epoch_interruption.rs@@ -496,3 +496,130 @@ async fn drop_future_on_epoch_yield(config: &mut Config) -> Result<()> {assert_eq!(true, alive_flag.load(Ordering::Acquire));Ok(())}++#[test]+fn memory_grow_in_epoch_callback() -> Result<()> {+ let mut config = Config::new();+ config.epoch_interruption(true);+ config.memory_reservation(0);+ config.memory_reservation_for_growth(0);+ config.memory_guard_size(0);+ config.memory_may_move(true);+ config.memory_init_cow(false);+ let engine = Engine::new(&config)?;+ let module = Module::new(+ &engine,+ r#"+ (module+ (memory (export "mem") 1)+ (func (export "go")+ (memory.fill (i32.const 0) (i32.const 0x41) (i32.const 65536))))+ "#,+ )?;++ let mut store: Store<Option<Memory>> = Store::new(&engine, None);+ store.set_epoch_deadline(1);+ store.epoch_deadline_callback(move |mut cx| {+ if let Some(mem) = *cx.data() {+ mem.grow(&mut cx, 5)?;+ }+ Ok(UpdateDeadline::Continue(0))+ });++ let instance = Instance::new(&mut store, &module, &[])?;+ let mem = instance.get_memory(&mut store, "mem").unwrap();+ *store.data_mut() = Some(mem);+ engine.increment_epoch();++ instance+ .get_typed_func::<(), ()>(&mut store, "go")?+ .call(&mut store, ())?;++ let data = mem.data(&store);+ assert_eq!(data[0], 0x41);+ assert_eq!(data[65535], 0x41);+ Ok(())+}++#[test]+fn table_grow_in_epoch_callback() -> Result<()> {+ let mut config = Config::new();+ config.epoch_interruption(true);+ let engine = Engine::new(&config)?;+ let module = Module::new(+ &engine,+ r#"+ (module+ (table $t (export "t") 1 funcref)+ (func (export "go")+ (table.fill $t (i32.const 0) (ref.null func) (i32.const 1))))+ "#,+ )?;++ let mut store: Store<Option<Table>> = Store::new(&engine, None);+ store.set_epoch_deadline(1);+ store.epoch_deadline_callback(move |mut cx| {+ if let Some(t) = *cx.data() {+ t.grow(&mut cx, 5, Ref::Func(None))?;+ }+ Ok(UpdateDeadline::Continue(0))+ });++ let instance = Instance::new(&mut store, &module, &[])?;+ let t = instance.get_table(&mut store, "t").unwrap();+ *store.data_mut() = Some(t);+ engine.increment_epoch();++ instance+ .get_typed_func::<(), ()>(&mut store, "go")?+ .call(&mut store, ())?;+ Ok(())+}++#[test]+fn gc_during_epoch_callback() -> Result<()> {+ let mut config = Config::new();+ config.epoch_interruption(true);+ let engine = Engine::new(&config)?;+ let module = Module::new(+ &engine,+ r#"+ (module+ (type $box (struct (field i32)))+ (type $arr (array (mut (ref null $box))))+ (global $sink (mut (ref null $arr)) (ref.null $arr))+ (func $mk (param $n i32) (result (ref $arr))+ (array.new_default $arr (local.get $n)))+ (func (export "run") (param $n i32) (result i32)+ (local $i i32)+ (block $done+ (loop $l+ (br_if $done (i32.ge_u (local.get $i) (i32.const 40)))+ (array.fill $arr (call $mk (local.get $n)) (i32.const 0)+ (struct.new $box (i32.const 7)) (local.get $n))+ (global.set $sink (call $mk (i32.const 8)))+ (local.set $i (i32.add (local.get $i) (i32.const 1)))+ (br $l)+ )+ )+ (i32.mul (local.get $n) (i32.const 7))))+ "#,+ )?;++ let mut store = Store::new(&engine, ());+ store.set_epoch_deadline(1);+ store.epoch_deadline_callback(|mut caller| {+ caller.gc(None)?;+ Ok(UpdateDeadline::Continue(0))+ });+ engine.increment_epoch();++ let instance = Instance::new(&mut store, &module, &[])?;+ let run = instance.get_typed_func::<u32, u32>(&mut store, "run")?;+ let n = 200;+ for i in 0..5 {+ let got = run.call(&mut store, n)?;+ assert_eq!(got, 7 * n, "iteration {i} read back {got}");+ }+ Ok(())+}diff --git a/tests/all/fuel.rs b/tests/all/fuel.rsindex 445a6c1cbd..1192fdc46d 100644--- a/tests/all/fuel.rs+++ b/tests/all/fuel.rs@@ -966,6 +966,89 @@ fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> {// i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2.let error = grow.call(&mut store, i64::MAX).unwrap_err();assert_eq!(error.downcast::<Trap>().unwrap(), Trap::OutOfFuel);+ Ok(())+}++#[test]+#[cfg_attr(miri, ignore)]+fn huge_table64_grow_cannot_mint_fuel() -> Result<()> {+ huge_table64_grow_cannot_mint_fuel_impl(+ r#"+ (module+ (table $t i64 0 0x10000 (ref null func))+ (func (export "run") (param $delta i64)+ (loop $l+ (drop (table.grow $t (ref.null func) (local.get $delta)))+ (br $l))))+ "#,+ )+}++#[test]+#[cfg_attr(miri, ignore)]+fn huge_table64_grow_cannot_mint_fuel_const() -> Result<()> {+ huge_table64_grow_cannot_mint_fuel_impl(+ r#"+ (module+ (table $t i64 0 0x10000 (ref null func))+ (func (export "run") (param $delta i64)+ (loop $l+ (drop (table.grow $t (ref.null func) (i64.const -500)))+ (br $l))))+ "#,+ )+}++fn huge_table64_grow_cannot_mint_fuel_impl(wat: &str) -> Result<()> {+ let mut config = Config::new();+ config.consume_fuel(true);+ let engine = Engine::new(&config)?;+ let module = Module::new(&engine, wat)?;++ let mut store = Store::new(&engine, ());+ store.set_fuel(100_000)?;+ let instance = Instance::new(&mut store, &module, &[])?;+ let run = instance.get_typed_func::<i64, ()>(&mut store, "run")?;++ let trap = run.call(&mut store, -500).unwrap_err().downcast::<Trap>()?;+ assert_eq!(trap, Trap::OutOfFuel);+ assert_eq!(store.get_fuel()?, 0);+ Ok(())+}+#[test]+#[cfg_attr(miri, ignore)]+fn fuel_around_table_grow() -> Result<()> {+ let mut config = Config::new();+ config.consume_fuel(true);+ let engine = Engine::new(&config)?;+ let module = Module::new(+ &engine,+ r#"+ (module+ (type $ft (func))+ (func $f (type $ft))+ (table $t 1 10000000 (ref $ft) (ref.func $f))+ (func (export "grow") (result i32)+ (table.grow $t (ref.func $f) (i32.const 9999999)))+ (func (export "call") (param i32)+ (call_indirect $t (type $ft) (local.get 0))))+ "#,+ )?;++ let mut store = Store::new(&engine, ());+ store.set_fuel(2)?;+ let instance = Instance::new(&mut store, &module, &[])?;+ let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?;+ let trap = grow.call(&mut store, ()).unwrap_err().downcast::<Trap>()?;+ assert_eq!(trap, Trap::OutOfFuel);++ store.set_fuel(u64::MAX)?;+ let call = instance.get_typed_func::<i32, ()>(&mut store, "call")?;+ let trap = call+ .call(&mut store, 9999999)+ .unwrap_err()+ .downcast::<Trap>()?;+ assert_eq!(trap, Trap::TableOutOfBounds);Ok(())}diff --git a/tests/all/gc.rs b/tests/all/gc.rsindex ac381877f7..ff32f09fc4 100644--- a/tests/all/gc.rs+++ b/tests/all/gc.rs@@ -3892,3 +3892,107 @@ fn winch_ref_params_and_results_across_gc() -> Result<()> {}Ok(())}++#[test]+#[cfg_attr(miri, ignore)]+fn array_fill_i64_gc_during_epoch() -> Result<()> {+ gc_during_epoch(+ r#"+ (module+ (type $arr (array (mut i64)))+ (type $box (struct (field i32)))+ (func (export "run") (param $n i32) (result i32)+ (local $a (ref null $arr)) (local $i i32) (local $s i32)+ (local.set $a (array.new_default $arr (local.get $n)))+ ;; Keep the collector busy so it has something to move.+ (drop (struct.new $box (i32.const 1)))+ (array.fill $arr (local.get $a) (i32.const 0) (i64.const 7) (local.get $n))+ (block $done (loop $l+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))+ (local.set $s (i32.add (local.get $s)+ (i32.wrap_i64 (array.get $arr (local.get $a) (local.get $i)))))+ (local.set $i (i32.add (local.get $i) (i32.const 1)))+ (br $l)))+ (local.get $s)))+ "#,+ )+}++#[test]+#[cfg_attr(miri, ignore)]+fn array_new_gc_during_epoch() -> Result<()> {+ gc_during_epoch(+ r#"+ (module+ (type $box (struct (field i32)))+ (type $arr (array (mut (ref null $box))))+ (func (export "run") (param $n i32) (result i32)+ (local $a (ref null $arr)) (local $i i32) (local $s i32)+ (local.set $a (array.new $arr (struct.new $box (i32.const 7)) (local.get $n)))+ (block $done (loop $l+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))+ (local.set $s (i32.add (local.get $s)+ (struct.get $box 0 (ref.as_non_null+ (array.get $arr (local.get $a) (local.get $i))))))+ (local.set $i (i32.add (local.get $i) (i32.const 1)))+ (br $l)))+ (local.get $s)))+ "#,+ )+}++#[test]+#[cfg_attr(miri, ignore)]+fn array_copy_gc_during_epoch() -> Result<()> {+ gc_during_epoch(+ r#"+ (module+ (type $box (struct (field i32)))+ (type $arr (array (mut (ref null $box))))+ (func (export "run") (param $n i32) (result i32)+ (local $a (ref null $arr)) (local $b (ref null $arr))+ (local $i i32) (local $s i32)+ (local.set $a (array.new_default $arr (local.get $n)))+ (local.set $b (array.new_default $arr (local.get $n)))+ (array.fill $arr (local.get $b) (i32.const 0)+ (struct.new $box (i32.const 7)) (local.get $n))+ (array.copy $arr $arr (local.get $a) (i32.const 0)+ (local.get $b) (i32.const 0) (local.get $n))+ (block $done (loop $l+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))+ (local.set $s (i32.add (local.get $s)+ (struct.get $box 0 (ref.as_non_null+ (array.get $arr (local.get $a) (local.get $i))))))+ (local.set $i (i32.add (local.get $i) (i32.const 1)))+ (br $l)))+ (local.get $s)))+ "#,+ )+}++fn gc_during_epoch(wat: &str) -> Result<()> {+ let mut config = Config::new();+ config.epoch_interruption(true);+ let engine = Engine::new(&config)?;+ let module = Module::new(&engine, wat)?;++ let mut store = Store::new(&engine, ());+ store.set_epoch_deadline(1);+ store.epoch_deadline_callback(|mut caller| {+ caller.gc(None)?;+ Ok(UpdateDeadline::Continue(0))+ });+ engine.increment_epoch();++ let instance = Instance::new(&mut store, &module, &[])?;+ let f = instance.get_typed_func::<u32, u32>(&mut store, "run")?;++ let n = 100;+ for i in 0..5 {+ match f.call(&mut store, n) {+ Ok(got) => assert_eq!(got, 7 * n, "iteration {i} read back {got}"),+ Err(e) => panic!("iteration {i} failed: {e:?}"),+ }+ }+ Ok(())+}diff --git a/tests/disas/gc/array-copy-with-fuel.wat b/tests/disas/gc/array-copy-with-fuel.watindex aeab1b17eb..8371c0eed2 100644--- a/tests/disas/gc/array-copy-with-fuel.wat+++ b/tests/disas/gc/array-copy-with-fuel.wat@@ -28,10 +28,10 @@;; stack_limit = gv2;;;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32):-;; v181 = stack_addr.i64 ss0-;; store notrap aligned region6 v2, v181-;; v182 = stack_addr.i64 ss1-;; store notrap aligned region7 v4, v182+;; v162 = stack_addr.i64 ss0+;; store notrap aligned region6 v2, v162+;; v163 = stack_addr.i64 ss1+;; store notrap aligned region7 v4, v163;; @0020 v7 = load.i64 notrap aligned readonly can_move region0 v0+8;; @0020 v8 = load.i64 notrap aligned region2 v7;; @0020 v9 = iconst.i64 1@@ -41,131 +41,108 @@;; @0020 brif v12, block2, block3(v10);;;; block2:-;; v191 = iadd.i64 v8, v9 ; v9 = 1-;; @0020 store notrap aligned region2 v191, v7+;; v172 = iadd.i64 v8, v9 ; v9 = 1+;; @0020 store notrap aligned region2 v172, v7;; @0020 v14 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0];; @0020 v16 = load.i64 notrap aligned region2 v7;; @0020 jump block3(v16);;-;; block3(v89: i64):-;; v180 = load.i32 notrap aligned region6 v181-;; @002b trapz v180, user16-;; @002b v24 = load.i64 notrap aligned readonly can_move region3 v7+32-;; @002b v22 = uextend.i64 v180-;; @002b v25 = iadd v24, v22-;; @002b v26 = iconst.i64 16-;; @002b v27 = iadd v25, v26 ; v26 = 16-;; @002b v28 = load.i32 user2 readonly region5 v27-;; @002b v30 = uextend.i64 v3-;; @002b v31 = uextend.i64 v6-;; @002b v34 = iadd v30, v31-;; @002b v29 = uextend.i64 v28-;; @002b v35 = icmp ugt v34, v29-;; @002b trapnz v35, user17-;; v174 = load.i32 notrap aligned region7 v182-;; @002b trapz v174, user16-;; @002b v46 = uextend.i64 v174-;; @002b v49 = iadd v24, v46-;; @002b v51 = iadd v49, v26 ; v26 = 16-;; @002b v52 = load.i32 user2 readonly region5 v51-;; @002b v54 = uextend.i64 v5-;; @002b v58 = iadd v54, v31-;; @002b v53 = uextend.i64 v52-;; @002b v59 = icmp ugt v58, v53-;; @002b trapnz v59, user17-;; @002b v78 = load.i64 notrap aligned region4 v7+40-;; @002b v40 = iconst.i64 20-;; @002b v41 = iadd v25, v40 ; v40 = 20-;; v184 = iconst.i64 2-;; v185 = ishl v30, v184 ; v184 = 2-;; @002b v45 = iadd v41, v185-;; v189 = ishl v31, v184 ; v184 = 2-;; @002b v80 = uadd_overflow_trap v45, v189, user2-;; @002b v79 = iadd v24, v78-;; @002b v81 = icmp ugt v80, v79-;; @002b trapnz v81, user2-;; @002b v65 = iadd v49, v40 ; v40 = 20-;; v187 = ishl v54, v184 ; v184 = 2-;; @002b v69 = iadd v65, v187-;; @002b v87 = uadd_overflow_trap v69, v189, user2-;; @002b v88 = icmp ugt v87, v79-;; @002b trapnz v88, user2-;; @002b v90 = iconst.i64 6-;; @002b v91 = iadd v89, v90 ; v90 = 6-;; @002b brif.i32 v6, block4, block7(v91)+;; block3(v25: i64):+;; @002b v26 = iconst.i64 6+;; @002b v27 = iadd v25, v26 ; v26 = 6+;; @002b v22 = uextend.i64 v6+;; @002b v28 = iadd v27, v22+;; v173 = iconst.i64 0+;; v174 = icmp sge v28, v173 ; v173 = 0+;; @002b brif v174, block4, block5(v28);;;; block4:-;; v162 = load.i32 notrap aligned region6 v181-;; v164 = load.i32 notrap aligned region7 v182-;; @002b v92 = icmp.i64 ult v45, v69-;; v192 = iadd.i64 v89, v90 ; v90 = 6-;; @002b v97 = iadd.i64 v45, v189-;; @002b v98 = iadd.i64 v69, v189-;; @002b v100 = iadd.i32 v5, v6-;; @002b v43 = iconst.i64 4-;; @002b v141 = iconst.i32 1-;; @002b brif v92, block5(v45, v69, v5, v162, v164, v192), block6(v97, v98, v100, v162, v164, v192)+;; @002b store.i64 notrap aligned region2 v28, v7+;; @002b v32 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0]+;; @002b v34 = load.i64 notrap aligned region2 v7+;; @002b jump block5(v34);;-;; block5(v101: i64, v102: i64, v103: i32, v104: i32, v105: i32, v106: i64):-;; store notrap aligned region6 v104, v181-;; store notrap aligned region7 v105, v182-;; v202 = iconst.i64 1-;; v203 = iadd v106, v202 ; v202 = 1-;; v204 = iconst.i64 0-;; v205 = icmp sge v203, v204 ; v204 = 0-;; @002b brif v205, block8, block9(v203)+;; block5(v139: i64):+;; v161 = load.i32 notrap aligned region6 v162+;; @002b trapz v161, user16+;; @002b v37 = load.i64 notrap aligned readonly can_move region3 v7+32+;; @002b v35 = uextend.i64 v161+;; @002b v38 = iadd v37, v35+;; @002b v39 = iconst.i64 16+;; @002b v40 = iadd v38, v39 ; v39 = 16+;; @002b v41 = load.i32 user2 readonly region5 v40+;; @002b v43 = uextend.i64 v3+;; @002b v47 = iadd v43, v22+;; @002b v42 = uextend.i64 v41+;; @002b v48 = icmp ugt v47, v42+;; @002b trapnz v48, user17+;; v155 = load.i32 notrap aligned region7 v163+;; @002b trapz v155, user16+;; @002b v59 = uextend.i64 v155+;; @002b v62 = iadd v37, v59+;; @002b v64 = iadd v62, v39 ; v39 = 16+;; @002b v65 = load.i32 user2 readonly region5 v64+;; @002b v67 = uextend.i64 v5+;; @002b v71 = iadd v67, v22+;; @002b v66 = uextend.i64 v65+;; @002b v72 = icmp ugt v71, v66+;; @002b trapnz v72, user17+;; @002b v91 = load.i64 notrap aligned region4 v7+40+;; @002b v53 = iconst.i64 20+;; @002b v54 = iadd v38, v53 ; v53 = 20+;; v165 = iconst.i64 2+;; v166 = ishl v43, v165 ; v165 = 2+;; @002b v58 = iadd v54, v166+;; v170 = ishl.i64 v22, v165 ; v165 = 2+;; @002b v93 = uadd_overflow_trap v58, v170, user2+;; @002b v92 = iadd v37, v91+;; @002b v94 = icmp ugt v93, v92+;; @002b trapnz v94, user2+;; @002b v78 = iadd v62, v53 ; v53 = 20+;; v168 = ishl v67, v165 ; v165 = 2+;; @002b v82 = iadd v78, v168+;; @002b v100 = uadd_overflow_trap v82, v170, user2+;; @002b v101 = icmp ugt v100, v92+;; @002b trapnz v101, user2+;; @002b brif.i32 v6, block6, block9;;-;; block6(v123: i64, v124: i64, v125: i32, v126: i32, v127: i32, v128: i64):-;; store notrap aligned region7 v126, v182-;; store notrap aligned region6 v127, v181-;; v193 = iconst.i64 1-;; v194 = iadd v128, v193 ; v193 = 1-;; v195 = iconst.i64 0-;; v196 = icmp sge v194, v195 ; v195 = 0-;; @002b brif v196, block10, block11(v194)+;; block6:+;; v143 = load.i32 notrap aligned region6 v162+;; v145 = load.i32 notrap aligned region7 v163+;; @002b v102 = icmp.i64 ult v58, v82+;; @002b v107 = iadd.i64 v58, v170+;; @002b v108 = iadd.i64 v82, v170+;; @002b v110 = iadd.i32 v5, v6+;; @002b v56 = iconst.i64 4+;; @002b v133 = iconst.i32 1+;; @002b brif v102, block7(v58, v82, v5), block8(v107, v108, v110);;-;; block7(v148: i64):-;; @002f jump block1-;;-;; block8:-;; @002b store.i64 notrap aligned region2 v203, v7-;; @002b v112 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0]-;; @002b v114 = load.i64 notrap aligned region2 v7-;; @002b jump block9(v114)+;; block7(v111: i64, v112: i64, v113: i32):+;; @002b v116 = load.i32 user2 little region5 v112+;; @002b store user2 little region5 v116, v111+;; v180 = iconst.i64 4+;; v181 = iadd v112, v180 ; v180 = 4+;; @002b v123 = icmp eq v181, v108+;; v182 = iadd v111, v180 ; v180 = 4+;; v183 = iconst.i32 1+;; v184 = iadd v113, v183 ; v183 = 1+;; @002b brif v123, block9, block7(v182, v181, v184);;-;; block9(v145: i64):-;; @002b v115 = load.i32 user2 little region5 v102-;; @002b store user2 little region5 v115, v101-;; v150 = load.i32 notrap aligned region6 v181-;; v152 = load.i32 notrap aligned region7 v182-;; v206 = iconst.i64 4-;; v207 = iadd.i64 v102, v206 ; v206 = 4-;; @002b v122 = icmp eq v207, v98-;; v208 = iadd.i64 v101, v206 ; v206 = 4-;; v209 = iconst.i32 1-;; v210 = iadd.i32 v103, v209 ; v209 = 1-;; @002b brif v122, block7(v145), block5(v208, v207, v210, v150, v152, v145)+;; block8(v124: i64, v125: i64, v126: i32):+;; v175 = iconst.i64 4+;; v176 = isub v125, v175 ; v175 = 4+;; @002b v135 = load.i32 user2 little region5 v176+;; v177 = isub v124, v175 ; v175 = 4+;; @002b store user2 little region5 v135, v177+;; @002b v136 = icmp eq v176, v82+;; v178 = iconst.i32 1+;; v179 = isub v126, v178 ; v178 = 1+;; @002b brif v136, block9, block8(v177, v176, v179);;-;; block10:-;; @002b store.i64 notrap aligned region2 v194, v7-;; @002b v134 = call fn0(v0), stack_map=[i32 @ ss1+0, i32 @ ss0+0]-;; @002b v136 = load.i64 notrap aligned region2 v7-;; @002b jump block11(v136)-;;-;; block11(v146: i64):-;; v197 = iconst.i64 4-;; v198 = isub.i64 v124, v197 ; v197 = 4-;; @002b v143 = load.i32 user2 little region5 v198-;; v199 = isub.i64 v123, v197 ; v197 = 4-;; @002b store user2 little region5 v143, v199-;; v156 = load.i32 notrap aligned region7 v182-;; v158 = load.i32 notrap aligned region6 v181-;; @002b v144 = icmp eq v198, v69-;; v200 = iconst.i32 1-;; v201 = isub.i32 v125, v200 ; v200 = 1-;; @002b brif v144, block7(v146), block6(v199, v198, v201, v156, v158, v146)+;; block9:+;; @002f jump block1;;;; block1:-;; @002f store.i64 notrap aligned region2 v148, v7+;; @002f store.i64 notrap aligned region2 v139, v7;; @002f return;; }diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.watindex f712972b87..d6113471a1 100644--- a/tests/disas/memory-copy-epochs.wat+++ b/tests/disas/memory-copy-epochs.wat@@ -37,101 +37,35 @@;; @001e v10 = call fn0(v0);; @001e jump block2(v10);;-;; block2(v59: i64):-;; @0025 v14 = load.i64 notrap aligned region6 v0+64-;; @0025 v15 = uextend.i64 v2-;; @0025 v16 = uextend.i64 v4-;; @0025 v19 = iadd v15, v16-;; @0025 v20 = icmp ugt v19, v14-;; @0025 trapnz v20, heap_oob-;; @0025 v27 = uextend.i64 v3-;; @0025 v31 = iadd v27, v16-;; @0025 v32 = icmp ugt v31, v14-;; @0025 trapnz v32, heap_oob-;; @0025 v21 = load.i64 notrap aligned readonly can_move region5 v0+56-;; @0025 v37 = iadd v21, v27-;; @0025 v25 = iadd v21, v15-;; @0025 v40 = icmp ugt v37, v25-;; @0025 brif v40, block6, block7-;;-;; block4(v42: i64, v43: i64, v44: i64, v47: i64):-;; @0025 v46 = load.i64 notrap aligned region3 v5-;; @0025 v48 = icmp uge v46, v47-;; @0025 brif v48, block9, block8(v47)-;;-;; block5(v86: i64, v87: i64, v88: i64, v92: i64):-;; @0025 v91 = load.i64 notrap aligned region3 v5-;; @0025 v94 = icmp uge v91, v92-;; @0025 brif v94, block17, block16-;;-;; block6:-;; v108 = iconst.i64 0x0800_0000-;; v109 = icmp.i64 ugt v16, v108 ; v108 = 0x0800_0000-;; @0025 brif v109, block4(v25, v37, v16, v59), block5(v25, v37, v16, v59)-;;-;; block9 cold:-;; @0025 v50 = load.i64 notrap aligned region4 v7+8-;; @0025 v51 = icmp.i64 uge v46, v50-;; @0025 brif v51, block10, block8(v50)-;;-;; block10 cold:-;; @0025 v52 = call fn0(v0)-;; @0025 jump block8(v52)-;;-;; block8(v60: i64):-;; @0025 call fn1(v0, v42, v43, v108) ; v108 = 0x0800_0000-;; @0025 v55 = isub.i64 v44, v108 ; v108 = 0x0800_0000-;; @0025 v56 = icmp ugt v55, v108 ; v108 = 0x0800_0000-;; @0025 v53 = iadd.i64 v42, v108 ; v108 = 0x0800_0000-;; @0025 v54 = iadd.i64 v43, v108 ; v108 = 0x0800_0000-;; @0025 brif v56, block4(v53, v54, v55, v60), block5(v53, v54, v55, v60)-;;-;; block7:-;; @0025 v39 = iconst.i64 0x0800_0000-;; @0025 v63 = icmp.i64 ugt v16, v39 ; v39 = 0x0800_0000-;; @0025 v61 = iadd.i64 v25, v16-;; @0025 v62 = iadd.i64 v37, v16-;; @0025 brif v63, block11(v61, v62, v16, v59), block12(v61, v62, v16, v59)-;;-;; block11(v64: i64, v65: i64, v66: i64, v71: i64):-;; @0025 v70 = load.i64 notrap aligned region3 v5-;; @0025 v72 = icmp uge v70, v71-;; @0025 brif v72, block14, block13(v71)-;;-;; block14 cold:-;; @0025 v74 = load.i64 notrap aligned region4 v7+8-;; @0025 v75 = icmp.i64 uge v70, v74-;; @0025 brif v75, block15, block13(v74)-;;-;; block15 cold:-;; @0025 v76 = call fn0(v0)-;; @0025 jump block13(v76)-;;-;; block13(v80: i64):-;; v103 = iconst.i64 0x0800_0000-;; v104 = isub.i64 v64, v103 ; v103 = 0x0800_0000-;; v105 = isub.i64 v65, v103 ; v103 = 0x0800_0000-;; @0025 call fn1(v0, v104, v105, v103) ; v103 = 0x0800_0000-;; v106 = isub.i64 v66, v103 ; v103 = 0x0800_0000-;; v107 = icmp ugt v106, v103 ; v103 = 0x0800_0000-;; @0025 brif v107, block11(v104, v105, v106, v80), block12(v104, v105, v106, v80)-;;-;; block12(v81: i64, v82: i64, v83: i64, v93: i64):-;; @0025 v84 = isub v81, v83-;; @0025 v85 = isub v82, v83-;; @0025 jump block5(v84, v85, v83, v93)-;;-;; block17 cold:-;; @0025 v96 = load.i64 notrap aligned region4 v7+8-;; @0025 v97 = icmp.i64 uge v91, v96-;; @0025 brif v97, block18, block16-;;-;; block18 cold:-;; @0025 v98 = call fn0(v0)-;; @0025 jump block16-;;-;; block16:-;; @0025 call fn1(v0, v86, v87, v88)+;; block2(v16: i64):+;; @0025 v15 = load.i64 notrap aligned region3 v5+;; @0025 v17 = icmp uge v15, v16+;; @0025 brif v17, block5, block4+;;+;; block5 cold:+;; @0025 v19 = load.i64 notrap aligned region4 v7+8+;; @0025 v20 = icmp.i64 uge v15, v19+;; @0025 brif v20, block6, block4+;;+;; block6 cold:+;; @0025 v21 = call fn0(v0)+;; @0025 jump block4+;;+;; block4:+;; @0025 v22 = load.i64 notrap aligned region6 v0+64+;; @0025 v23 = uextend.i64 v2+;; @0025 v24 = uextend.i64 v4+;; @0025 v27 = iadd v23, v24+;; @0025 v28 = icmp ugt v27, v22+;; @0025 trapnz v28, heap_oob+;; @0025 v35 = uextend.i64 v3+;; @0025 v39 = iadd v35, v24+;; @0025 v40 = icmp ugt v39, v22+;; @0025 trapnz v40, heap_oob+;; @0025 v29 = load.i64 notrap aligned readonly can_move region5 v0+56+;; @0025 v33 = iadd v29, v23+;; @0025 v45 = iadd v29, v35+;; @0025 call fn1(v0, v33, v45, v24);; @0029 jump block1;;;; block1:diff --git a/tests/disas/memory-copy-fuel-const-len.wat b/tests/disas/memory-copy-fuel-const-len.watnew file mode 100644index 0000000000..aee7af900c--- /dev/null+++ b/tests/disas/memory-copy-fuel-const-len.wat@@ -0,0 +1,184 @@+;;! target = 'x86_64'+;;! test = 'optimize'+;;! flags = '-Wfuel=100'++(module+ (memory 1)+ (func $copy_16 (param i32 i32)+ (memory.copy (local.get 0) (local.get 1) (i32.const 16))+ )+ (func $fill_128 (param i32)+ (memory.fill (local.get 0) (i32.const 0) (i32.const 128))+ )+ (func $fill_4096 (param i32)+ (memory.fill (local.get 0) (i32.const 0) (i32.const 4096))+ )+)+;; function u0:0(i64 vmctx, i64, i32, i32) tail {+;; region0 = 8 "VMContext+0x8"+;; region1 = 67108888 "VMStoreContext+0x18"+;; region2 = 67108864 "VMStoreContext+0x0"+;; region3 = 603979776 "VMMemoryDefinition+0x0"+;; region4 = 603979784 "VMMemoryDefinition+0x8"+;; region5 = 201326592 "DefinedMemory(StaticModuleIndex(0), DefinedMemoryIndex(0))"+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8+;; gv2 = load.i64 notrap aligned region1 gv1+24+;; sig0 = (i64 vmctx) -> i8 tail+;; fn0 = colocated u805306368:12 sig0+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32, v3: i32):+;; @0023 v4 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @0023 v5 = load.i64 notrap aligned region2 v4+;; @0023 v6 = iconst.i64 1+;; @0023 v7 = iadd v5, v6 ; v6 = 1+;; @0023 v8 = iconst.i64 0+;; @0023 v9 = icmp sge v7, v8 ; v8 = 0+;; @0023 brif v9, block2, block3(v7)+;;+;; block2:+;; v58 = iadd.i64 v5, v6 ; v6 = 1+;; @0023 store notrap aligned region2 v58, v4+;; @0023 v11 = call fn0(v0)+;; @0023 v13 = load.i64 notrap aligned region2 v4+;; @0023 jump block3(v13)+;;+;; block3(v43: i64):+;; @002a v17 = load.i64 notrap aligned region4 v0+64+;; @002a v18 = uextend.i64 v2+;; v47 = iconst.i64 16+;; @002a v22 = iadd v18, v47 ; v47 = 16+;; @002a v23 = icmp ugt v22, v17+;; @002a trapnz v23, heap_oob+;; @002a v30 = uextend.i64 v3+;; @002a v34 = iadd v30, v47 ; v47 = 16+;; @002a v35 = icmp ugt v34, v17+;; @002a trapnz v35, heap_oob+;; @002a v24 = load.i64 notrap aligned readonly can_move region3 v0+56+;; @002a v40 = iadd v24, v30+;; @002a v42 = load.i8x16 notrap aligned little region5 v40+;; @002a v28 = iadd v24, v18+;; @002a store notrap aligned little region5 v42, v28+;; @002e jump block1+;;+;; block1:+;; @002e v44 = iconst.i64 20+;; @002e v45 = iadd.i64 v43, v44 ; v44 = 20+;; @002e store notrap aligned region2 v45, v4+;; @002e return+;; }+;;+;; function u0:1(i64 vmctx, i64, i32) tail {+;; region0 = 8 "VMContext+0x8"+;; region1 = 67108888 "VMStoreContext+0x18"+;; region2 = 67108864 "VMStoreContext+0x0"+;; region3 = 603979776 "VMMemoryDefinition+0x0"+;; region4 = 603979784 "VMMemoryDefinition+0x8"+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8+;; gv2 = load.i64 notrap aligned region1 gv1+24+;; sig0 = (i64 vmctx) -> i8 tail+;; sig1 = (i64 vmctx, i64, i32, i64) tail+;; fn0 = colocated u805306368:12 sig0+;; fn1 = colocated u805306368:2 sig1+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @0030 v3 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @0030 v4 = load.i64 notrap aligned region2 v3+;; @0030 v5 = iconst.i64 1+;; @0030 v6 = iadd v4, v5 ; v5 = 1+;; @0030 v7 = iconst.i64 0+;; @0030 v8 = icmp sge v6, v7 ; v7 = 0+;; @0030 brif v8, block2, block3(v6)+;;+;; block2:+;; v43 = iadd.i64 v4, v5 ; v5 = 1+;; @0030 store notrap aligned region2 v43, v3+;; @0030 v10 = call fn0(v0)+;; @0030 v12 = load.i64 notrap aligned region2 v3+;; @0030 jump block3(v12)+;;+;; block3(v29: i64):+;; @0038 v16 = load.i64 notrap aligned region4 v0+64+;; @0038 v17 = uextend.i64 v2+;; v33 = iconst.i64 128+;; @0038 v21 = iadd v17, v33 ; v33 = 128+;; @0038 v22 = icmp ugt v21, v16+;; @0038 trapnz v22, heap_oob+;; @0038 v23 = load.i64 notrap aligned readonly can_move region3 v0+56+;; @0038 v27 = iadd v23, v17+;; @0033 v14 = iconst.i32 0+;; @0038 call fn1(v0, v27, v14, v33) ; v14 = 0, v33 = 128+;; @003b jump block1+;;+;; block1:+;; @003b v30 = iconst.i64 132+;; @003b v31 = iadd.i64 v29, v30 ; v30 = 132+;; @003b store notrap aligned region2 v31, v3+;; @003b return+;; }+;;+;; function u0:2(i64 vmctx, i64, i32) tail {+;; region0 = 8 "VMContext+0x8"+;; region1 = 67108888 "VMStoreContext+0x18"+;; region2 = 67108864 "VMStoreContext+0x0"+;; region3 = 603979776 "VMMemoryDefinition+0x0"+;; region4 = 603979784 "VMMemoryDefinition+0x8"+;; gv0 = vmctx+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8+;; gv2 = load.i64 notrap aligned region1 gv1+24+;; sig0 = (i64 vmctx) -> i8 tail+;; sig1 = (i64 vmctx, i64, i32, i64) tail+;; fn0 = colocated u805306368:12 sig0+;; fn1 = colocated u805306368:2 sig1+;; stack_limit = gv2+;;+;; block0(v0: i64, v1: i64, v2: i32):+;; @003d v3 = load.i64 notrap aligned readonly can_move region0 v0+8+;; @003d v4 = load.i64 notrap aligned region2 v3+;; @003d v5 = iconst.i64 1+;; @003d v6 = iadd v4, v5 ; v5 = 1+;; @003d v7 = iconst.i64 0+;; @003d v8 = icmp sge v6, v7 ; v7 = 0+;; @003d brif v8, block2, block3(v6)+;;+;; block2:+;; v50 = iadd.i64 v4, v5 ; v5 = 1+;; @003d store notrap aligned region2 v50, v3+;; @003d v10 = call fn0(v0)+;; @003d v12 = load.i64 notrap aligned region2 v3+;; @003d jump block3(v12)+;;+;; block3(v16: i64):+;; @0045 v17 = iconst.i64 4100+;; @0045 v18 = iadd v16, v17 ; v17 = 4100+;; v51 = iconst.i64 0+;; v52 = icmp sge v18, v51 ; v51 = 0+;; @0045 brif v52, block4, block5(v18)+;;+;; block4:+;; v53 = iadd.i64 v16, v17 ; v17 = 4100+;; @0045 store notrap aligned region2 v53, v3+;; @0045 v22 = call fn0(v0)+;; @0045 v24 = load.i64 notrap aligned region2 v3+;; @0045 jump block5(v24)+;;+;; block5(v39: i64):+;; @0045 v25 = load.i64 notrap aligned region4 v0+64+;; @0045 v26 = uextend.i64 v2+;; v40 = iconst.i64 4096+;; @0045 v30 = iadd v26, v40 ; v40 = 4096+;; @0045 v31 = icmp ugt v30, v25+;; @0045 trapnz v31, heap_oob+;; @0045 v32 = load.i64 notrap aligned readonly can_move region3 v0+56+;; @0045 v36 = iadd v32, v26+;; @0040 v14 = iconst.i32 0+;; @0045 call fn1(v0, v36, v14, v40) ; v14 = 0, v40 = 4096+;; @0048 jump block1+;;+;; block1:+;; @0048 store.i64 notrap aligned region2 v39, v3+;; @0048 return+;; }diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.watindex 45e72b7043..29104ad297 100644--- a/tests/disas/memory-copy-fuel.wat+++ b/tests/disas/memory-copy-fuel.wat@@ -33,110 +33,44 @@;; @001e brif v10, block2, block3(v8);;;; block2:-;; v106 = iadd.i64 v6, v7 ; v7 = 1-;; @001e store notrap aligned region2 v106, v5+;; v61 = iadd.i64 v6, v7 ; v7 = 1+;; @001e store notrap aligned region2 v61, v5;; @001e v12 = call fn0(v0);; @001e v14 = load.i64 notrap aligned region2 v5;; @001e jump block3(v14);;-;; block3(v43: i64):-;; @0025 v18 = load.i64 notrap aligned region4 v0+64-;; @0025 v19 = uextend.i64 v2-;; @0025 v20 = uextend.i64 v4-;; @0025 v23 = iadd v19, v20-;; @0025 v24 = icmp ugt v23, v18-;; @0025 trapnz v24, heap_oob-;; @0025 v31 = uextend.i64 v3-;; @0025 v35 = iadd v31, v20-;; @0025 v36 = icmp ugt v35, v18-;; @0025 trapnz v36, heap_oob-;; @0025 v25 = load.i64 notrap aligned readonly can_move region3 v0+56-;; @0025 v41 = iadd v25, v31-;; @0025 v29 = iadd v25, v19-;; @0025 v47 = icmp ugt v41, v29-;; @0025 brif v47, block6, block7-;;-;; block4(v49: i64, v50: i64, v51: i64, v52: i64):-;; @0025 v53 = iadd v52, v116 ; v116 = 0x0800_0000-;; v120 = iconst.i64 0-;; v121 = icmp sge v53, v120 ; v120 = 0-;; @0025 brif v121, block8, block9(v53)-;;-;; block5(v89: i64, v90: i64, v91: i64, v92: i64):-;; @0025 v94 = iadd v92, v91-;; v123 = iconst.i64 0-;; v124 = icmp sge v94, v123 ; v123 = 0-;; @0025 brif v124, block14, block15(v94)-;;-;; block6:-;; v116 = iconst.i64 0x0800_0000-;; v117 = icmp.i64 ugt v20, v116 ; v116 = 0x0800_0000-;; v118 = iconst.i64 4-;; v119 = iadd.i64 v43, v118 ; v118 = 4-;; @0025 brif v117, block4(v29, v41, v20, v119), block5(v29, v41, v20, v119)-;;-;; block8:-;; v122 = iadd.i64 v52, v116 ; v116 = 0x0800_0000-;; @0025 store notrap aligned region2 v122, v5-;; @0025 v57 = call fn0(v0)-;; @0025 v59 = load.i64 notrap aligned region2 v5-;; @0025 jump block9(v59)-;;-;; block9(v64: i64):-;; @0025 call fn1(v0, v49, v50, v116) ; v116 = 0x0800_0000-;; @0025 v62 = isub.i64 v51, v116 ; v116 = 0x0800_0000-;; @0025 v63 = icmp ugt v62, v116 ; v116 = 0x0800_0000-;; @0025 v60 = iadd.i64 v49, v116 ; v116 = 0x0800_0000-;; @0025 v61 = iadd.i64 v50, v116 ; v116 = 0x0800_0000-;; @0025 brif v63, block4(v60, v61, v62, v64), block5(v60, v61, v62, v64)-;;-;; block7:-;; @0025 v46 = iconst.i64 0x0800_0000-;; @0025 v67 = icmp.i64 ugt v20, v46 ; v46 = 0x0800_0000-;; @0025 v65 = iadd.i64 v29, v20-;; @0025 v66 = iadd.i64 v41, v20-;; @0025 v44 = iconst.i64 4-;; @0025 v45 = iadd.i64 v43, v44 ; v44 = 4-;; @0025 brif v67, block10(v65, v66, v20, v45), block11(v65, v66, v20, v45)-;;-;; block10(v68: i64, v69: i64, v70: i64, v73: i64):-;; v107 = iconst.i64 0x0800_0000-;; v108 = iadd v73, v107 ; v107 = 0x0800_0000-;; v109 = iconst.i64 0-;; v110 = icmp sge v108, v109 ; v109 = 0-;; @0025 brif v110, block12, block13(v108)-;;-;; block12:-;; @0025 store.i64 notrap aligned region2 v108, v5-;; @0025 v78 = call fn0(v0)-;; @0025 v80 = load.i64 notrap aligned region2 v5-;; @0025 jump block13(v80)-;;-;; block13(v83: i64):-;; v111 = iconst.i64 0x0800_0000-;; v112 = isub.i64 v68, v111 ; v111 = 0x0800_0000-;; v113 = isub.i64 v69, v111 ; v111 = 0x0800_0000-;; @0025 call fn1(v0, v112, v113, v111) ; v111 = 0x0800_0000-;; v114 = isub.i64 v70, v111 ; v111 = 0x0800_0000-;; v115 = icmp ugt v114, v111 ; v111 = 0x0800_0000-;; @0025 brif v115, block10(v112, v113, v114, v83), block11(v112, v113, v114, v83)-;;-;; block11(v84: i64, v85: i64, v86: i64, v93: i64):-;; @0025 v87 = isub v84, v86-;; @0025 v88 = isub v85, v86-;; @0025 jump block5(v87, v88, v86, v93)-;;-;; block14:-;; @0025 store.i64 notrap aligned region2 v94, v5-;; @0025 v98 = call fn0(v0)-;; @0025 v100 = load.i64 notrap aligned region2 v5-;; @0025 jump block15(v100)-;;-;; block15(v102: i64):-;; @0025 call fn1(v0, v89, v90, v91)+;; block3(v21: i64):+;; @0025 v22 = iconst.i64 4+;; @0025 v23 = iadd v21, v22 ; v22 = 4+;; @0025 v18 = uextend.i64 v4+;; @0025 v24 = iadd v23, v18+;; v62 = iconst.i64 0+;; v63 = icmp sge v24, v62 ; v62 = 0+;; @0025 brif v63, block4, block5(v24)+;;+;; block4:+;; @0025 store.i64 notrap aligned region2 v24, v5+;; @0025 v28 = call fn0(v0)+;; @0025 v30 = load.i64 notrap aligned region2 v5+;; @0025 jump block5(v30)+;;+;; block5(v57: i64):+;; @0025 v31 = load.i64 notrap aligned region4 v0+64+;; @0025 v32 = uextend.i64 v2+;; @0025 v36 = iadd v32, v18+;; @0025 v37 = icmp ugt v36, v31+;; @0025 trapnz v37, heap_oob+;; @0025 v44 = uextend.i64 v3+;; @0025 v48 = iadd v44, v18+;; @0025 v49 = icmp ugt v48, v31+;; @0025 trapnz v49, heap_oob+;; @0025 v38 = load.i64 notrap aligned readonly can_move region3 v0+56+;; @0025 v42 = iadd v38, v32+;; @0025 v54 = iadd v38, v44+;; @0025 call fn1(v0, v42, v54, v18);; @0029 jump block1;;;; block1:-;; @0029 store.i64 notrap aligned region2 v102, v5+;; @0029 store.i64 notrap aligned region2 v57, v5;; @0029 return;; }