Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Remove preemption points in bulk operations (#14045)

Alex Crichton3ebfbe5af492
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 1e47b8c70d..8b65df4341 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -663,85 +663,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
builder.switch_to_block(continuation_block);
}
- /// Manually insert a fuel check, as opposed to what already happens around
- /// normal loops headers and function entries.
- ///
- /// This can be used for expensive opcodes, such as `array.copy`, where the
- /// operation's runtime is a function of the runtime state.
- fn manual_fuel_check(&mut self, builder: &mut FunctionBuilder<'_>, fuel_to_consume: ir::Value) {
- self.fuel_increment_var(builder);
-
- let fuel = builder.use_var(self.fuel_var);
- let fuel = builder.ins().iadd(fuel, fuel_to_consume);
- builder.def_var(self.fuel_var, fuel);
-
- self.fuel_check(builder);
- }
-
- /// Consumes `units * cost_per_unit` fuel, saturating the charge at
- /// `i64::MAX` so that an oversized unsigned operand cannot wrap around and
- /// add fuel instead.
- fn consume_variable_fuel(
- &mut self,
- builder: &mut FunctionBuilder<'_>,
- units: ir::Value,
- cost_per_unit: u8,
- ) {
- let may_exceed_i64_max = match builder.func.dfg.value_type(units) {
- I32 => false,
- I64 => true,
- ty => unreachable!("unsupported variable fuel unit type: {ty}"),
- };
- self.consume_variable_fuel_impl(builder, units, cost_per_unit, may_exceed_i64_max);
- }
-
- /// Like [`Self::consume_variable_fuel`], but for a unit count whose product
- /// with `cost_per_unit` is statically known to fit in an `i64`.
- fn consume_bounded_variable_fuel(
- &mut self,
- builder: &mut FunctionBuilder<'_>,
- units: ir::Value,
- cost_per_unit: u8,
- ) {
- self.consume_variable_fuel_impl(builder, units, cost_per_unit, false);
- }
-
- fn consume_variable_fuel_impl(
- &mut self,
- builder: &mut FunctionBuilder<'_>,
- units: ir::Value,
- cost_per_unit: u8,
- may_exceed_i64_max: bool,
- ) {
- if !self.tunables.consume_fuel || cost_per_unit == 0 {
- return;
- }
-
- let units = match builder.func.dfg.value_type(units) {
- I32 => builder.ins().uextend(I64, units),
- I64 => units,
- ty => unreachable!("unsupported variable fuel unit type: {ty}"),
- };
- let fuel = if cost_per_unit == 1 {
- units
- } else {
- builder.ins().imul_imm_s(units, i64::from(cost_per_unit))
- };
- let fuel = if may_exceed_i64_max {
- let max = builder.ins().iconst(I64, i64::MAX);
- let max_units = builder
- .ins()
- .iconst(I64, i64::MAX / i64::from(cost_per_unit));
- let saturate = builder
- .ins()
- .icmp(IntCC::UnsignedGreaterThan, units, max_units);
- builder.ins().select(saturate, max, fuel)
- } else {
- fuel
- };
- self.manual_fuel_check(builder, fuel);
- }
-
fn epoch_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) {
debug_assert!(self.epoch_deadline_var.is_reserved_value());
self.epoch_deadline_var = builder.declare_var(ir::types::I64);
@@ -2684,17 +2605,19 @@ impl FuncEnvironment<'_> {
delta: ir::Value,
init_value: ir::Value,
) -> WasmResult<ir::Value> {
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .table_grow_per_element;
+ self.pre_translate_bulk_op(builder, delta, cost)?;
+
let mut pos = builder.cursor();
let table = self.table(table_index);
let (table_vmctx, defined_table_index) =
self.table_vmctx_and_defined_index(&mut pos, table_index);
let index_type = table.idx_type;
let delta64 = self.cast_index_to_i64(&mut pos, delta, index_type);
- let cost = self
- .tunables
- .operator_cost
- .variable()
- .table_grow_per_element;
// Call out to the host to perform the actual growth of the underlying
// table. This will initialize table slots as all null. Afterwards the
@@ -2740,7 +2663,6 @@ impl FuncEnvironment<'_> {
// A failed attempt performs no initialization loop, but still charge
// for the requested growth so repeated failures are not free.
builder.switch_to_block(failed_block);
- self.consume_variable_fuel(builder, delta, cost);
builder.ins().jump(done_block, &[]);
builder.switch_to_block(fill_block);
@@ -2748,12 +2670,11 @@ impl FuncEnvironment<'_> {
builder,
CheckedEntity::Table {
table: table_index,
- initialized: true,
+ initialized: false,
},
result_idx,
init_value,
delta,
- cost,
)?;
builder.ins().jump(done_block, &[]);
@@ -2907,6 +2828,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_fill_per_element;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_fill(
builder,
CheckedEntity::Table {
@@ -2916,7 +2838,6 @@ impl FuncEnvironment<'_> {
dst,
val,
len,
- cost,
)
}
@@ -3053,8 +2974,7 @@ impl FuncEnvironment<'_> {
elem: ir::Value,
len: ir::Value,
) -> WasmResult<ir::Value> {
- let cost = self.tunables.operator_cost.variable().array_new_per_element;
- gc::translate_array_new(self, builder, array_type_index, elem, len, cost)
+ gc::translate_array_new(self, builder, array_type_index, elem, len)
}
pub fn translate_array_new_default(
@@ -3063,12 +2983,7 @@ impl FuncEnvironment<'_> {
array_type_index: TypeIndex,
len: ir::Value,
) -> WasmResult<ir::Value> {
- let cost = self
- .tunables
- .operator_cost
- .variable()
- .array_new_default_per_element;
- gc::translate_array_new_default(self, builder, array_type_index, len, cost)
+ gc::translate_array_new_default(self, builder, array_type_index, len)
}
pub fn translate_array_new_fixed(
@@ -3151,6 +3066,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.array_copy_per_element;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3166,7 +3082,6 @@ impl FuncEnvironment<'_> {
dst_index,
src_index,
len,
- cost,
)
}
@@ -3185,6 +3100,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.array_fill_per_element;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_fill(
builder,
CheckedEntity::Array {
@@ -3195,7 +3111,6 @@ impl FuncEnvironment<'_> {
index,
value,
len,
- cost,
)
}
@@ -3216,6 +3131,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.array_init_data_per_element;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3230,7 +3146,6 @@ impl FuncEnvironment<'_> {
dst_index,
data_offset,
len,
- cost,
)
}
@@ -3250,6 +3165,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.array_init_elem_per_element;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3261,7 +3177,6 @@ impl FuncEnvironment<'_> {
dst_index,
elem_offset,
len,
- cost,
)
}
@@ -3634,7 +3549,7 @@ impl FuncEnvironment<'_> {
let index_type = self.memory(index).idx_type;
let cost = self.tunables.operator_cost.variable().memory_grow_per_page;
- self.consume_variable_fuel(builder, val, cost);
+ self.pre_translate_bulk_op(builder, val, cost)?;
let mut pos = builder.cursor();
let val = self.cast_index_to_i64(&mut pos, val, index_type);
let call_inst = pos
@@ -3733,7 +3648,8 @@ impl FuncEnvironment<'_> {
len: ir::Value,
) -> WasmResult<()> {
let cost = self.tunables.operator_cost.variable().memory_copy_per_byte;
- self.translate_entity_copy(builder, dst_index, src_index, dst, src, len, cost)
+ self.pre_translate_bulk_op(builder, len, cost)?;
+ self.translate_entity_copy(builder, dst_index, src_index, dst, src, len)
}
/// Perform a raw bulk-memory-like libcall.
@@ -3741,7 +3657,11 @@ impl FuncEnvironment<'_> {
/// The main purpose of this helper is to handle situations when fuel and
/// epochs are enabled to break up the copy into a loop of chunks with
/// preemption checks between them.
- fn raw_bulk_memory_operation(&mut self, builder: &mut FunctionBuilder<'_>, mut op: BulkOp) {
+ fn raw_bulk_memory_operation(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ op: BulkOp,
+ ) -> WasmResult<()> {
// Fast path: a copy whose byte length is a small compile-time constant is
// expanded inline (see `emit_inline_memcpy`), skipping the libcall's fixed
// per-call cost (a wasm/host transition and an indirect call) that
@@ -3758,259 +3678,33 @@ impl FuncEnvironment<'_> {
const_len: Some(bytes),
src_entity,
dst_entity,
- fuel,
..
} = op
{
if bytes <= INLINE_COPY_MAX_BYTES {
- if self.tunables.consume_fuel {
- let units = bytes / u64::from(fuel.bytes_per_unit);
- self.fuel_consumed +=
- i64::try_from(units * u64::from(fuel.cost_per_unit)).unwrap();
- }
let src_region = self.bulk_copy_alias_region(builder.func, src_entity);
let dst_region = self.bulk_copy_alias_region(builder.func, dst_entity);
self.emit_inline_memcpy(builder, dst, src, bytes, src_region, dst_region);
- return;
+ return Ok(());
}
}
- // Very scientifically chosen. Or, more seriously, this is just an
- // arbitrary number for now. 100k copies of this size locally takes half
- // a second, so seems like a reasonably large chunk size to not hit perf
- // too much by chunking but also enable time slicing.
- const UNINTERRUPTABLE_CHUNK_SIZE: i64 = 128 << 20;
-
let mut pos = builder.cursor();
let vmctx = self.vmctx_val(&mut pos);
- let pointer_type = self.pointer_type();
// Performs a raw call to the actual libcall, as dictated by the
- // provided `op`. This inserts configured epoch/fuel checks before the
- // call.
- let raw_call =
- |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &BulkOp| {
- if env.tunables.epoch_interruption {
- env.epoch_check(builder);
- }
- let fuel = op.fuel();
- if env.tunables.consume_fuel && fuel.cost_per_unit != 0 {
- let byte_len = op.len();
- debug_assert!(fuel.bytes_per_unit.is_power_of_two());
- let units = if fuel.bytes_per_unit == 1 {
- byte_len
- } else {
- builder
- .ins()
- .ushr_imm_u(byte_len, i64::from(fuel.bytes_per_unit.trailing_zeros()))
- };
- // With fuel enabled all calls emitted below are limited to
- // `UNINTERRUPTABLE_CHUNK_SIZE`, so this multiplication
- // cannot exceed `i64::MAX` even at the maximum `u8` rate.
- env.consume_bounded_variable_fuel(builder, units, fuel.cost_per_unit);
- }
- match *op {
- BulkOp::MemoryCopy { dst, src, len, .. } => {
- let memory_copy = env.builtin_functions.memory_copy(&mut builder.func);
- builder.ins().call(memory_copy, &[vmctx, dst, src, len]);
- }
- BulkOp::MemoryFill { dst, val, len, .. } => {
- let memory_fill = env.builtin_functions.memory_fill(&mut builder.func);
- builder.ins().call(memory_fill, &[vmctx, dst, val, len]);
- }
- }
- };
-
- // If epochs and fuel are disabled, then just call the libcall and
- // return. No need for the loops below.
- if !self.tunables.epoch_interruption && !self.tunables.consume_fuel {
- raw_call(self, builder, &op);
- return;
- }
-
- // If fuel is enabled, first take all the pending fuel and flush it to
- // our internal variable. This is necessary to avoid picking up all
- // pending fuel on each turn of the loop below.
- if self.tunables.consume_fuel {
- self.fuel_increment_var(builder);
- }
-
- let current_block = builder.current_block().unwrap();
- let chunk_block = builder.create_block();
- let last_chunk_block = builder.create_block();
-
- builder.ensure_inserted_block();
- builder.insert_block_after(chunk_block, current_block);
- builder.insert_block_after(last_chunk_block, chunk_block);
-
- let chunk = builder
- .ins()
- .iconst(pointer_type, UNINTERRUPTABLE_CHUNK_SIZE);
-
- // For `memcpy` when chunking this up we might need to do a backwards
- // copy or a forwards copy. Determine that here and jump to the
- // backwards copy if needed.
- let backwards_block = if let BulkOp::MemoryCopy { dst, src, .. } = op {
- let forwards = builder.ins().icmp(IntCC::UnsignedGreaterThan, src, dst);
- let forwards_block = builder.create_block();
- let backwards_block = builder.create_block();
- builder
- .ins()
- .brif(forwards, forwards_block, &[], backwards_block, &[]);
- builder.switch_to_block(forwards_block);
- builder.seal_block(forwards_block);
- Some((backwards_block, op.clone()))
- } else {
- None
- };
-
- // Helper closure to test if the length in `op` is larger than `chunk`,
- // and if so do a single chunk. Else this goes to the final block with
- // the final operation.
- let has_chunk_branch =
- |builder: &mut FunctionBuilder<'_>, op: &_, chunk_block, last_chunk_block| {
- let len = match *op {
- BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len,
- };
- let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk);
- match *op {
- BulkOp::MemoryCopy { dst, src, len, .. } => {
- builder.ins().brif(
- has_chunk,
- chunk_block,
- &[dst.into(), src.into(), len.into()],
- last_chunk_block,
- &[dst.into(), src.into(), len.into()],
- );
- }
- BulkOp::MemoryFill { dst, len, .. } => {
- builder.ins().brif(
- has_chunk,
- chunk_block,
- &[dst.into(), len.into()],
- last_chunk_block,
- &[dst.into(), len.into()],
- );
- }
- }
- };
-
- let append_block_params = |builder: &mut FunctionBuilder<'_>, block, op: &mut _| match op {
- BulkOp::MemoryCopy { dst, src, len, .. } => {
- *dst = builder.append_block_param(block, pointer_type);
- *src = builder.append_block_param(block, pointer_type);
- *len = builder.append_block_param(block, pointer_type);
- }
- BulkOp::MemoryFill { dst, len, .. } => {
- *dst = builder.append_block_param(block, pointer_type);
- *len = builder.append_block_param(block, pointer_type);
- }
- };
-
- // Forwards copy: dispatch to the per-chunk loop or the final iteration
- // if there's no chunks.
- has_chunk_branch(builder, &op, chunk_block, last_chunk_block);
-
- // Forwards copy: In the block with per-chunk copies, each operation
- // performs `chunk` length of bytes and then decrements the current
- // length by `chunk`. Afterwards a condition tests if we do another
- // chunk or break out for the final chunk.
- builder.switch_to_block(chunk_block);
- append_block_params(builder, chunk_block, &mut op);
- let op_len = match &mut op {
- BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len,
- };
- let remaining_len = *op_len;
- *op_len = chunk;
- raw_call(self, builder, &op);
- match &mut op {
+ // provided `op`.
+ match op {
BulkOp::MemoryCopy { dst, src, len, .. } => {
- *dst = builder.ins().iadd(*dst, chunk);
- *src = builder.ins().iadd(*src, chunk);
- *len = builder.ins().isub(remaining_len, chunk);
+ let memory_copy = self.builtin_functions.memory_copy(&mut builder.func);
+ builder.ins().call(memory_copy, &[vmctx, dst, src, len]);
}
- BulkOp::MemoryFill { len, dst, .. } => {
- *dst = builder.ins().iadd(*dst, chunk);
- *len = builder.ins().isub(remaining_len, chunk);
+ BulkOp::MemoryFill { dst, val, len } => {
+ let memory_fill = self.builtin_functions.memory_fill(&mut builder.func);
+ builder.ins().call(memory_fill, &[vmctx, dst, val, len]);
}
- };
- has_chunk_branch(builder, &op, chunk_block, last_chunk_block);
- builder.seal_block(chunk_block);
-
- // Backwards copy: similar to the above but with adjustments on where
- // increments/decrements happen. Notably:
- //
- // * Initial src/end are the final byte address
- // * Each chunk starts out by decrementing src/end as opposed to above
- // where the increment happens at the end.
- // * The final block performs the final decrement before jumping to the
- // shared `last_chunk_block` between the forwards/backwards paths.
- if let Some((backwards_block, mut op)) = backwards_block {
- // Setup `dst=dst+len` and `src=src+len`, then see if we have a
- // chunk.
- builder.switch_to_block(backwards_block);
- builder.seal_block(backwards_block);
- let backwards_chunk_block = builder.create_block();
- let backwards_last_chunk_block = builder.create_block();
- let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {
- unreachable!()
- };
- *dst = builder.ins().iadd(*dst, *len);
- *src = builder.ins().iadd(*src, *len);
- has_chunk_branch(
- builder,
- &op,
- backwards_chunk_block,
- backwards_last_chunk_block,
- );
-
- // Execute the per-chunk backwards copy, adjusting pointers before
- // the copy itself.
- builder.switch_to_block(backwards_chunk_block);
- append_block_params(builder, backwards_chunk_block, &mut op);
- let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {
- unreachable!()
- };
- let remaining_len = *len;
- *len = chunk;
- *dst = builder.ins().isub(*dst, chunk);
- *src = builder.ins().isub(*src, chunk);
- raw_call(self, builder, &op);
- let BulkOp::MemoryCopy { len, .. } = &mut op else {
- unreachable!()
- };
- *len = builder.ins().isub(remaining_len, chunk);
- has_chunk_branch(
- builder,
- &op,
- backwards_chunk_block,
- backwards_last_chunk_block,
- );
- builder.seal_block(backwards_chunk_block);
-
- // Final backwards chunk: adjust the dst/src to be their true base
- // pointers and then delegate to `last_chunk_block` for the actual
- // memcpy.
- builder.switch_to_block(backwards_last_chunk_block);
- builder.seal_block(backwards_last_chunk_block);
- append_block_params(builder, backwards_last_chunk_block, &mut op);
- let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {
- unreachable!()
- };
- *dst = builder.ins().isub(*dst, *len);
- *src = builder.ins().isub(*src, *len);
- builder.ins().jump(
- last_chunk_block,
- &[(*dst).into(), (*src).into(), (*len).into()],
- );
}
-
- // In the final block we know that the length of the operation is less
- // than `chunk`.
- builder.switch_to_block(last_chunk_block);
- builder.seal_block(last_chunk_block);
- append_block_params(builder, last_chunk_block, &mut op);
- raw_call(self, builder, &op);
+ Ok(())
}
/// Emits a generic "fill" of `entity` from `dst` for `len` elements,
@@ -4021,6 +3715,9 @@ impl FuncEnvironment<'_> {
/// `dst` and `len` values must be typed appropriately for `entity`. This
/// will perform a bounds-check before actually executing the operation and
/// then afterwards will perform the operation.
+ ///
+ /// Callers must invoke `pre_translate_bulk_op` before calling this method
+ /// to properly account for fuel/epoch checks for this bulk operation.
fn translate_entity_fill(
&mut self,
builder: &mut FunctionBuilder<'_>,
@@ -4028,7 +3725,6 @@ impl FuncEnvironment<'_> {
dst: ir::Value,
val: ir::Value,
len: ir::Value,
- cost_per_unit: u8,
) -> WasmResult<()> {
let entity = entity.into();
let idx_type = entity.index_type(self);
@@ -4042,37 +3738,22 @@ impl FuncEnvironment<'_> {
self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, idx_type);
match entity {
- CheckedEntity::Memory(_) => {
- self.raw_bulk_memory_operation(
- builder,
- BulkOp::MemoryFill {
- dst: raw_dst_addr,
- val,
- len: len_ptr,
- fuel: BulkFuel {
- cost_per_unit,
- bytes_per_unit: 1,
- },
- },
- );
- }
- CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => {
- self.emit_raw_array_or_table_fill(
- builder,
- entity,
- raw_dst_addr,
+ CheckedEntity::Memory(_) => self.raw_bulk_memory_operation(
+ builder,
+ BulkOp::MemoryFill {
+ dst: raw_dst_addr,
val,
- len_ptr,
- cost_per_unit,
- )?;
+ len: len_ptr,
+ },
+ ),
+ CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => {
+ self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr)
}
// Not allowed to be written to in wasm.
CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {
unreachable!()
}
}
-
- Ok(())
}
/// Performs a manual element-by-element fill of `entity`, starting at
@@ -4096,7 +3777,6 @@ impl FuncEnvironment<'_> {
dst_elem_addr: ir::Value,
value: ir::Value,
copy_len: ir::Value,
- cost_per_element: u8,
) -> WasmResult<()> {
let pointer_ty = self.pointer_type();
@@ -4116,19 +3796,14 @@ impl FuncEnvironment<'_> {
if entity.allows_memset(self)
&& let Some(value) = self.fill_value_as_memset(builder, elem_ty, value)
{
- self.raw_bulk_memory_operation(
+ return self.raw_bulk_memory_operation(
builder,
BulkOp::MemoryFill {
dst: dst_elem_addr,
val: value,
len: copy_byte_len,
- fuel: BulkFuel {
- cost_per_unit: cost_per_element,
- bytes_per_unit: u8::try_from(elem_size).unwrap(),
- },
},
);
- return Ok(());
}
// Funcref values are intern'd when stored on the GC heap, and the
@@ -4169,12 +3844,6 @@ impl FuncEnvironment<'_> {
builder.insert_block_after(loop_block, current_block);
builder.insert_block_after(continue_block, loop_block);
- // Before entering the loop below flush our fuel counters to ensure
- // that previous instructions' fuel isn't counted once-per-iteration.
- if self.tunables.consume_fuel {
- self.fuel_increment_var(builder);
- }
-
// Current block: test to see if this is actually an empty copy. If it
// is then skip over the entire loop, otherwise enter the loop and
// perform the first ieration.
@@ -4192,11 +3861,6 @@ impl FuncEnvironment<'_> {
// by the element size, then see if we turn again or exit.
builder.switch_to_block(loop_block);
let elem_addr = builder.append_block_param(loop_block, pointer_ty);
- // Consume the configured cost for this element before writing it.
- if self.tunables.consume_fuel {
- self.fuel_consumed += i64::from(cost_per_element);
- }
- self.translate_loop_header(builder)?;
match entity {
CheckedEntity::Table { table, initialized } => {
assert!(!is_pre_interned_funcref);
@@ -4312,7 +3976,8 @@ impl FuncEnvironment<'_> {
len: ir::Value,
) -> WasmResult<()> {
let cost = self.tunables.operator_cost.variable().memory_fill_per_byte;
- self.translate_entity_fill(builder, memory_index, dst, val, len, cost)
+ self.pre_translate_bulk_op(builder, len, cost)?;
+ self.translate_entity_fill(builder, memory_index, dst, val, len)
}
pub fn translate_memory_init(
@@ -4326,6 +3991,7 @@ impl FuncEnvironment<'_> {
) -> WasmResult<()> {
let seg_index = DataIndex::from_u32(seg_index);
let cost = self.tunables.operator_cost.variable().memory_init_per_byte;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_copy(
builder,
memory_index,
@@ -4336,7 +4002,6 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
- cost,
)
}
@@ -4379,6 +4044,9 @@ impl FuncEnvironment<'_> {
/// of the copy. Both `dst` and `src` have types appropriate to index their
/// respective entities, and `len` has a type that's the smaller of the two
/// index types.
+ ///
+ /// Callers must invoke `pre_translate_bulk_op` before calling this method
+ /// to properly account for fuel/epoch checks for this bulk operation.
fn translate_entity_copy(
&mut self,
builder: &mut FunctionBuilder<'_>,
@@ -4387,7 +4055,6 @@ impl FuncEnvironment<'_> {
dst: ir::Value,
src: ir::Value,
len: ir::Value,
- cost_per_unit: u8,
) -> WasmResult<()> {
let dst_entity = dst_entity.into();
let src_entity = src_entity.into();
@@ -4449,13 +4116,8 @@ impl FuncEnvironment<'_> {
const_len: const_count,
src_entity,
dst_entity,
- fuel: BulkFuel {
- cost_per_unit,
- bytes_per_unit: 1,
- },
},
- );
- Ok(())
+ )
}
// Tables/arrays are sometimes a memcpy, sometimes a per-element
@@ -4470,7 +4132,6 @@ impl FuncEnvironment<'_> {
len_ptr,
src,
const_count,
- cost_per_unit,
),
// Cannot copy into a data or element segment in wasm.
@@ -4686,6 +4347,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_copy_per_element;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_copy(
builder,
CheckedEntity::Table {
@@ -4699,7 +4361,6 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
- cost,
)
}
@@ -4729,7 +4390,6 @@ impl FuncEnvironment<'_> {
copy_len: ir::Value,
src_index: ir::Value,
const_count: Option<u64>,
- cost_per_element: u8,
) -> WasmResult<()> {
let pointer_type = self.pointer_type();
assert_eq!(builder.func.dfg.value_type(dst_elem_addr), pointer_type);
@@ -4808,7 +4468,7 @@ impl FuncEnvironment<'_> {
// parameters (or expand it inline; see `raw_bulk_memory_operation`).
if !type_forbids_memcpy && dst_element_size == src_element_size {
let const_len = const_count.and_then(|c| c.checked_mul(u64::from(dst_element_size)));
- self.raw_bulk_memory_operation(
+ return self.raw_bulk_memory_operation(
builder,
BulkOp::MemoryCopy {
dst: dst_elem_addr,
@@ -4817,13 +4477,8 @@ impl FuncEnvironment<'_> {
const_len,
src_entity,
dst_entity,
- fuel: BulkFuel {
- cost_per_unit: cost_per_element,
- bytes_per_unit: u8::try_from(dst_element_size).unwrap(),
- },
},
);
- return Ok(());
}
// For other copies, this is a per-element loop. Use the helper to
@@ -4837,7 +4492,6 @@ impl FuncEnvironment<'_> {
src_elem_addr,
copy_len,
src_index,
- cost_per_element,
&|this, builder, dst, src, src_index| {
let write_ty = dst_entity.storage_type(this);
let val = match src_entity {
@@ -5044,7 +4698,6 @@ impl FuncEnvironment<'_> {
src_elem_addr: ir::Value,
copy_len: ir::Value,
src_index: ir::Value,
- cost_per_element: u8,
copy_one: &dyn Fn(
&mut Self,
&mut FunctionBuilder<'_>,
@@ -5096,13 +4749,6 @@ impl FuncEnvironment<'_> {
builder.insert_block_after(backwards_block, forward_block);
builder.insert_block_after(done_block, backwards_block);
- // Update our local fuel counter, if enabled, before entering the loops
- // below. This zeros out `self.fuel_consumed` so we don't consume
- // previous fuel on each iteration of the loop.
- if self.tunables.consume_fuel {
- self.fuel_increment_var(builder);
- }
-
// Terminate `current_block` by testing to see if we're copying any
// elements at all.
builder
@@ -5192,11 +4838,6 @@ impl FuncEnvironment<'_> {
let src_cur = builder.append_block_param(forward_block, self.pointer_type());
let src_index = builder.append_block_param(forward_block, src_index_ty);
let forward_keepalives = append_keepalive_params(builder, forward_block);
- // Consume the configured cost for this element before copying it.
- if self.tunables.consume_fuel {
- self.fuel_consumed += i64::from(cost_per_element);
- }
- self.translate_loop_header(builder)?;
copy_one(self, builder, dst_cur, src_cur, src_index)?;
let dst_next = builder
.ins()
@@ -5222,10 +4863,6 @@ impl FuncEnvironment<'_> {
let src_cur = builder.append_block_param(backwards_block, self.pointer_type());
let src_index = builder.append_block_param(backwards_block, src_index_ty);
let backward_keepalives = append_keepalive_params(builder, backwards_block);
- if self.tunables.consume_fuel {
- self.fuel_consumed += i64::from(cost_per_element);
- }
- self.translate_loop_header(builder)?;
let dst_cur = {
let size = builder
.ins()
@@ -5281,6 +4918,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_init_per_element;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_copy(
builder,
CheckedEntity::Table {
@@ -5291,7 +4929,6 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
- cost,
)
}
@@ -5355,6 +4992,78 @@ impl FuncEnvironment<'_> {
Ok(builder.ins().ireduce(ir::types::I32, ret))
}
+ /// Translation prefix before bulk operations such as `memory.copy`.
+ ///
+ /// Takes a dynamic value `units` for the size of the operation as well as
+ /// a `cost_per_unit` configured for this operation. If fuel is enabled
+ /// this fuel will be consumed, and if epochs are enabled then an epoch
+ /// check happens. If neither epochs nor fuel are enabled this is a noop.
+ fn pre_translate_bulk_op(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ units: ir::Value,
+ cost_per_unit: u8,
+ ) -> WasmResult<()> {
+ let const_units =
+ Self::value_as_const_int(builder, units).map(|c| i64::try_from(c).unwrap_or(i64::MAX));
+
+ if self.tunables.consume_fuel && cost_per_unit > 0 {
+ match const_units {
+ // Fold constant costs directly into internal state.
+ Some(units) => {
+ self.fuel_consumed = self
+ .fuel_consumed
+ .saturating_add(units.saturating_mul(i64::from(cost_per_unit)))
+ }
+
+ None => {
+ // Note that fuel is always a 64-bit counter.
+ //
+ // Also note that the cost is clamped to `i64::MAX` to
+ // prevent fuel counter overflows since `cost` is otherwise
+ // an untrusted value.
+ let units_clamped64 = match builder.func.dfg.value_type(units) {
+ ir::types::I32 => {
+ let units64 = builder.ins().uextend(ir::types::I64, units);
+ builder.ins().imul_imm_u(units64, i64::from(cost_per_unit))
+ }
+ ir::types::I64 => {
+ let fuel = builder.ins().imul_imm_u(units, i64::from(cost_per_unit));
+ let max = builder.ins().iconst(ir::types::I64, i64::MAX);
+ let max_units = builder
+ .ins()
+ .iconst(I64, i64::MAX / i64::from(cost_per_unit));
+ let saturate =
+ builder
+ .ins()
+ .icmp(IntCC::UnsignedGreaterThan, units, max_units);
+ builder.ins().select(saturate, max, fuel)
+ }
+ _ => unreachable!(),
+ };
+ self.fuel_increment_var(builder);
+ let fuel = builder.use_var(self.fuel_var);
+ let fuel = builder.ins().iadd(fuel, units_clamped64);
+ builder.def_var(self.fuel_var, fuel);
+ }
+ }
+ }
+
+ // Skip explicit fuel/epoch checks for operations which are
+ // subjectively, and statically, considered cheap.
+ const SMALL_BULK_OP_COST: i64 = 128;
+ if let Some(units) = const_units
+ && let Some(cost) = units.checked_mul(i64::from(cost_per_unit))
+ && cost <= SMALL_BULK_OP_COST
+ {
+ return Ok(());
+ }
+
+ // This isn't a loop header but for fuel/epoch purposes it's the same
+ // thing.
+ self.translate_loop_header(builder)
+ }
+
pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {
// Additionally if enabled check how much fuel we have remaining to see
// if we've run out by this point.
@@ -6242,6 +5951,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_fill_per_element;
+ self.pre_translate_bulk_op(builder, len, cost)?;
self.translate_entity_fill(
builder,
CheckedEntity::Table {
@@ -6251,7 +5961,6 @@ impl FuncEnvironment<'_> {
dst,
val,
len,
- cost,
)
}
@@ -6287,7 +5996,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_init_per_element;
- self.consume_variable_fuel(builder, segment_len, cost);
+ self.pre_translate_bulk_op(builder, segment_len, cost)?;
// Re-use the `table.set` translation for making this a simple function
// to define. That re-executes the bounds check which is a bit
@@ -6360,7 +6069,8 @@ impl FuncEnvironment<'_> {
let len = self.load_runtime_data_length(builder, data);
let start = builder.ins().iconst(I32, 0);
let cost = self.tunables.operator_cost.variable().memory_init_per_byte;
- self.translate_entity_copy(builder, memory, data, offset, start, len, cost)?;
+ self.pre_translate_bulk_op(builder, len, cost)?;
+ self.translate_entity_copy(builder, memory, data, offset, start, len)?;
// Finalize control-flow for the `MemorySegmentOffset::Static` case
// above.
@@ -6530,7 +6240,6 @@ enum BulkOp {
const_len: Option<u64>,
src_entity: CheckedEntity,
dst_entity: CheckedEntity,
- fuel: BulkFuel,
},
/// A `memory.fill` operation, setting all bytes of `dst` to `val`.
@@ -6543,31 +6252,9 @@ enum BulkOp {
dst: ir::Value,
val: ir::Value,
len: ir::Value,
- fuel: BulkFuel,
},
}
-impl BulkOp {
- fn len(&self) -> ir::Value {
- match self {
- BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => *len,
- }
- }
-
- fn fuel(&self) -> BulkFuel {
- match self {
- BulkOp::MemoryCopy { fuel, .. } | BulkOp::MemoryFill { fuel, .. } => *fuel,
- }
- }
-}
-
-#[derive(Copy, Clone)]
-struct BulkFuel {
- cost_per_unit: u8,
- /// Number of copied bytes represented by one billable unit.
- bytes_per_unit: u8,
-}
-
/// A list of entities which can participate in various kinds of bulk operations
/// in wasm.
///
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index 183b0d1d1c..ac8275f027 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -801,9 +801,15 @@ pub fn translate_array_new(
array_type_index: TypeIndex,
elem: ir::Value,
len: ir::Value,
- cost_per_element: u8,
) -> WasmResult<ir::Value> {
log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})");
+ let cost = func_env
+ .tunables
+ .operator_cost
+ .variable()
+ .array_new_per_element;
+ func_env.pre_translate_bulk_op(builder, len, cost)?;
+
let result =
gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;
let zero = builder.ins().iconst(ir::types::I32, 0);
@@ -818,7 +824,6 @@ pub fn translate_array_new(
zero,
elem,
len,
- cost_per_element,
)?;
log::trace!("translate_array_new(..) -> {result:?}");
Ok(result)
@@ -829,9 +834,14 @@ pub fn translate_array_new_default(
builder: &mut FunctionBuilder,
array_type_index: TypeIndex,
len: ir::Value,
- cost_per_element: u8,
) -> WasmResult<ir::Value> {
log::trace!("translate_array_new_default({array_type_index:?}, {len:?})");
+ let cost = func_env
+ .tunables
+ .operator_cost
+ .variable()
+ .array_new_default_per_element;
+ func_env.pre_translate_bulk_op(builder, len, cost)?;
let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index();
let array_ty = func_env.types.unwrap_array(interned_ty)?;
@@ -850,7 +860,6 @@ pub fn translate_array_new_default(
zero,
elem,
len,
- cost_per_element,
)?;
Ok(result)
}
@@ -1754,8 +1763,10 @@ pub fn translate_array_new_entity(
entity: CheckedEntity,
entity_offset: ir::Value,
len: ir::Value,
- cost_per_element: u8,
+ cost_per_unit: u8,
) -> WasmResult<ir::Value> {
+ env.pre_translate_bulk_op(builder, len, cost_per_unit)?;
+
// Before actually allocating this array first do a bounds-check on the
// passive entity itself.
let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index();
@@ -1774,7 +1785,6 @@ pub fn translate_array_new_entity(
dst,
entity_offset,
len,
- cost_per_element,
)?;
Ok(array)
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index 98090ecfc4..fd411683ef 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -690,10 +690,6 @@ impl Config {
/// signal handler), then we can ensure that all async code will
/// yield to the executor within a bounded time.
///
- /// The deadline check cannot be avoided by malicious wasm code. It is safe
- /// to use epoch deadlines to limit the execution time of untrusted
- /// code.
- ///
/// The [`Store`](crate::Store) tracks the deadline, and controls
/// what happens when the deadline is reached during
/// execution. Several behaviors are possible:
@@ -739,6 +735,27 @@ impl Config {
/// computation and have the desired effect of cancelling a blocking
/// operation when a timeout expires.
///
+ /// ## Limitations with malicious guests
+ ///
+ /// Epochs are designed to handle malicious WebAssembly guests -- the
+ /// deadline check cannot be avoided by WebAssembly code. It is safe to use
+ /// epoch deadlines to limit the execution time of untrusted code.
+ ///
+ /// Note, though, that a current limitation to this is that
+ /// bulk-data-transfer instructions, such as `memory.copy`, only check the
+ /// epoch once at the start of the operation. These operations can take a
+ /// variable amount of time to complete based on how many bytes are being
+ /// copied. This means that the maximal time slice a guest might take is
+ /// the maximum of the epoch interval and the largest
+ /// memory-copy-style-instruction executed. The size of a copy is bounded
+ /// on the size of linear memory or GC heap size. In the limit, however, a
+ /// guest using a 64-bit linear memory with a 128GiB size could issue a
+ /// 128GiB `memory.copy` which would have no preemption within the
+ /// instruction itself. Hosts which need strict time limits for guests right
+ /// now are recommended to ensure that the store's allocated heap size
+ /// (linear memory + GC heap) are bounded with a
+ /// [`ResourceLimiter`](crate::ResourceLimiter).
+ ///
/// ## When to use fuel vs. epochs
///
/// In general, epoch-based interruption results in faster
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index 9cc49ea836..a0816548b8 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1041,6 +1041,10 @@ impl<T> Store<T> {
/// The `interval` parameter indicates how much fuel should be
/// consumed between yields of an async future. When fuel runs out wasm will trap.
///
+ /// For limitations related to consumption of fuel and when yield points are
+ /// injected, see the discussion in
+ /// [`Config::epoch_interruption`](crate::Config::epoch_interruption).
+ ///
/// # Error
///
/// This method will error if fuel is not enabled or `interval` is
diff --git a/tests/all/epoch_interruption.rs b/tests/all/epoch_interruption.rs
index 6a1c1ff7bb..1d2b4664f3 100644
--- a/tests/all/epoch_interruption.rs
+++ b/tests/all/epoch_interruption.rs
@@ -496,3 +496,130 @@ async fn drop_future_on_epoch_yield(config: &mut Config) -> Result<()> {
assert_eq!(true, alive_flag.load(Ordering::Acquire));
Ok(())
}
+
+#[test]
+fn memory_grow_in_epoch_callback() -> Result<()> {
+ let mut config = Config::new();
+ config.epoch_interruption(true);
+ config.memory_reservation(0);
+ config.memory_reservation_for_growth(0);
+ config.memory_guard_size(0);
+ config.memory_may_move(true);
+ config.memory_init_cow(false);
+ let engine = Engine::new(&config)?;
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (memory (export "mem") 1)
+ (func (export "go")
+ (memory.fill (i32.const 0) (i32.const 0x41) (i32.const 65536))))
+ "#,
+ )?;
+
+ let mut store: Store<Option<Memory>> = Store::new(&engine, None);
+ store.set_epoch_deadline(1);
+ store.epoch_deadline_callback(move |mut cx| {
+ if let Some(mem) = *cx.data() {
+ mem.grow(&mut cx, 5)?;
+ }
+ Ok(UpdateDeadline::Continue(0))
+ });
+
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let mem = instance.get_memory(&mut store, "mem").unwrap();
+ *store.data_mut() = Some(mem);
+ engine.increment_epoch();
+
+ instance
+ .get_typed_func::<(), ()>(&mut store, "go")?
+ .call(&mut store, ())?;
+
+ let data = mem.data(&store);
+ assert_eq!(data[0], 0x41);
+ assert_eq!(data[65535], 0x41);
+ Ok(())
+}
+
+#[test]
+fn table_grow_in_epoch_callback() -> Result<()> {
+ let mut config = Config::new();
+ config.epoch_interruption(true);
+ let engine = Engine::new(&config)?;
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (table $t (export "t") 1 funcref)
+ (func (export "go")
+ (table.fill $t (i32.const 0) (ref.null func) (i32.const 1))))
+ "#,
+ )?;
+
+ let mut store: Store<Option<Table>> = Store::new(&engine, None);
+ store.set_epoch_deadline(1);
+ store.epoch_deadline_callback(move |mut cx| {
+ if let Some(t) = *cx.data() {
+ t.grow(&mut cx, 5, Ref::Func(None))?;
+ }
+ Ok(UpdateDeadline::Continue(0))
+ });
+
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let t = instance.get_table(&mut store, "t").unwrap();
+ *store.data_mut() = Some(t);
+ engine.increment_epoch();
+
+ instance
+ .get_typed_func::<(), ()>(&mut store, "go")?
+ .call(&mut store, ())?;
+ Ok(())
+}
+
+#[test]
+fn gc_during_epoch_callback() -> Result<()> {
+ let mut config = Config::new();
+ config.epoch_interruption(true);
+ let engine = Engine::new(&config)?;
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (type $box (struct (field i32)))
+ (type $arr (array (mut (ref null $box))))
+ (global $sink (mut (ref null $arr)) (ref.null $arr))
+ (func $mk (param $n i32) (result (ref $arr))
+ (array.new_default $arr (local.get $n)))
+ (func (export "run") (param $n i32) (result i32)
+ (local $i i32)
+ (block $done
+ (loop $l
+ (br_if $done (i32.ge_u (local.get $i) (i32.const 40)))
+ (array.fill $arr (call $mk (local.get $n)) (i32.const 0)
+ (struct.new $box (i32.const 7)) (local.get $n))
+ (global.set $sink (call $mk (i32.const 8)))
+ (local.set $i (i32.add (local.get $i) (i32.const 1)))
+ (br $l)
+ )
+ )
+ (i32.mul (local.get $n) (i32.const 7))))
+ "#,
+ )?;
+
+ let mut store = Store::new(&engine, ());
+ store.set_epoch_deadline(1);
+ store.epoch_deadline_callback(|mut caller| {
+ caller.gc(None)?;
+ Ok(UpdateDeadline::Continue(0))
+ });
+ engine.increment_epoch();
+
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let run = instance.get_typed_func::<u32, u32>(&mut store, "run")?;
+ let n = 200;
+ for i in 0..5 {
+ let got = run.call(&mut store, n)?;
+ assert_eq!(got, 7 * n, "iteration {i} read back {got}");
+ }
+ Ok(())
+}
diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs
index 445a6c1cbd..1192fdc46d 100644
--- a/tests/all/fuel.rs
+++ b/tests/all/fuel.rs
@@ -966,6 +966,89 @@ fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> {
// i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2.
let error = grow.call(&mut store, i64::MAX).unwrap_err();
assert_eq!(error.downcast::<Trap>().unwrap(), Trap::OutOfFuel);
+ Ok(())
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn huge_table64_grow_cannot_mint_fuel() -> Result<()> {
+ huge_table64_grow_cannot_mint_fuel_impl(
+ r#"
+ (module
+ (table $t i64 0 0x10000 (ref null func))
+ (func (export "run") (param $delta i64)
+ (loop $l
+ (drop (table.grow $t (ref.null func) (local.get $delta)))
+ (br $l))))
+ "#,
+ )
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn huge_table64_grow_cannot_mint_fuel_const() -> Result<()> {
+ huge_table64_grow_cannot_mint_fuel_impl(
+ r#"
+ (module
+ (table $t i64 0 0x10000 (ref null func))
+ (func (export "run") (param $delta i64)
+ (loop $l
+ (drop (table.grow $t (ref.null func) (i64.const -500)))
+ (br $l))))
+ "#,
+ )
+}
+
+fn huge_table64_grow_cannot_mint_fuel_impl(wat: &str) -> Result<()> {
+ let mut config = Config::new();
+ config.consume_fuel(true);
+ let engine = Engine::new(&config)?;
+ let module = Module::new(&engine, wat)?;
+
+ let mut store = Store::new(&engine, ());
+ store.set_fuel(100_000)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let run = instance.get_typed_func::<i64, ()>(&mut store, "run")?;
+
+ let trap = run.call(&mut store, -500).unwrap_err().downcast::<Trap>()?;
+ assert_eq!(trap, Trap::OutOfFuel);
+ assert_eq!(store.get_fuel()?, 0);
+ Ok(())
+}
+#[test]
+#[cfg_attr(miri, ignore)]
+fn fuel_around_table_grow() -> Result<()> {
+ let mut config = Config::new();
+ config.consume_fuel(true);
+ let engine = Engine::new(&config)?;
+ let module = Module::new(
+ &engine,
+ r#"
+ (module
+ (type $ft (func))
+ (func $f (type $ft))
+ (table $t 1 10000000 (ref $ft) (ref.func $f))
+ (func (export "grow") (result i32)
+ (table.grow $t (ref.func $f) (i32.const 9999999)))
+ (func (export "call") (param i32)
+ (call_indirect $t (type $ft) (local.get 0))))
+ "#,
+ )?;
+
+ let mut store = Store::new(&engine, ());
+ store.set_fuel(2)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?;
+ let trap = grow.call(&mut store, ()).unwrap_err().downcast::<Trap>()?;
+ assert_eq!(trap, Trap::OutOfFuel);
+
+ store.set_fuel(u64::MAX)?;
+ let call = instance.get_typed_func::<i32, ()>(&mut store, "call")?;
+ let trap = call
+ .call(&mut store, 9999999)
+ .unwrap_err()
+ .downcast::<Trap>()?;
+ assert_eq!(trap, Trap::TableOutOfBounds);
Ok(())
}
diff --git a/tests/all/gc.rs b/tests/all/gc.rs
index ac381877f7..ff32f09fc4 100644
--- a/tests/all/gc.rs
+++ b/tests/all/gc.rs
@@ -3892,3 +3892,107 @@ fn winch_ref_params_and_results_across_gc() -> Result<()> {
}
Ok(())
}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn array_fill_i64_gc_during_epoch() -> Result<()> {
+ gc_during_epoch(
+ r#"
+ (module
+ (type $arr (array (mut i64)))
+ (type $box (struct (field i32)))
+ (func (export "run") (param $n i32) (result i32)
+ (local $a (ref null $arr)) (local $i i32) (local $s i32)
+ (local.set $a (array.new_default $arr (local.get $n)))
+ ;; Keep the collector busy so it has something to move.
+ (drop (struct.new $box (i32.const 1)))
+ (array.fill $arr (local.get $a) (i32.const 0) (i64.const 7) (local.get $n))
+ (block $done (loop $l
+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))
+ (local.set $s (i32.add (local.get $s)
+ (i32.wrap_i64 (array.get $arr (local.get $a) (local.get $i)))))
+ (local.set $i (i32.add (local.get $i) (i32.const 1)))
+ (br $l)))
+ (local.get $s)))
+ "#,
+ )
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn array_new_gc_during_epoch() -> Result<()> {
+ gc_during_epoch(
+ r#"
+ (module
+ (type $box (struct (field i32)))
+ (type $arr (array (mut (ref null $box))))
+ (func (export "run") (param $n i32) (result i32)
+ (local $a (ref null $arr)) (local $i i32) (local $s i32)
+ (local.set $a (array.new $arr (struct.new $box (i32.const 7)) (local.get $n)))
+ (block $done (loop $l
+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))
+ (local.set $s (i32.add (local.get $s)
+ (struct.get $box 0 (ref.as_non_null
+ (array.get $arr (local.get $a) (local.get $i))))))
+ (local.set $i (i32.add (local.get $i) (i32.const 1)))
+ (br $l)))
+ (local.get $s)))
+ "#,
+ )
+}
+
+#[test]
+#[cfg_attr(miri, ignore)]
+fn array_copy_gc_during_epoch() -> Result<()> {
+ gc_during_epoch(
+ r#"
+ (module
+ (type $box (struct (field i32)))
+ (type $arr (array (mut (ref null $box))))
+ (func (export "run") (param $n i32) (result i32)
+ (local $a (ref null $arr)) (local $b (ref null $arr))
+ (local $i i32) (local $s i32)
+ (local.set $a (array.new_default $arr (local.get $n)))
+ (local.set $b (array.new_default $arr (local.get $n)))
+ (array.fill $arr (local.get $b) (i32.const 0)
+ (struct.new $box (i32.const 7)) (local.get $n))
+ (array.copy $arr $arr (local.get $a) (i32.const 0)
+ (local.get $b) (i32.const 0) (local.get $n))
+ (block $done (loop $l
+ (br_if $done (i32.ge_u (local.get $i) (local.get $n)))
+ (local.set $s (i32.add (local.get $s)
+ (struct.get $box 0 (ref.as_non_null
+ (array.get $arr (local.get $a) (local.get $i))))))
+ (local.set $i (i32.add (local.get $i) (i32.const 1)))
+ (br $l)))
+ (local.get $s)))
+ "#,
+ )
+}
+
+fn gc_during_epoch(wat: &str) -> Result<()> {
+ let mut config = Config::new();
+ config.epoch_interruption(true);
+ let engine = Engine::new(&config)?;
+ let module = Module::new(&engine, wat)?;
+
+ let mut store = Store::new(&engine, ());
+ store.set_epoch_deadline(1);
+ store.epoch_deadline_callback(|mut caller| {
+ caller.gc(None)?;
+ Ok(UpdateDeadline::Continue(0))
+ });
+ engine.increment_epoch();
+
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let f = instance.get_typed_func::<u32, u32>(&mut store, "run")?;
+
+ let n = 100;
+ for i in 0..5 {
+ match f.call(&mut store, n) {
+ Ok(got) => assert_eq!(got, 7 * n, "iteration {i} read back {got}"),
+ Err(e) => panic!("iteration {i} failed: {e:?}"),
+ }
+ }
+ Ok(())
+}
diff --git a/tests/disas/gc/array-copy-with-fuel.wat b/tests/disas/gc/array-copy-with-fuel.wat
index aeab1b17eb..8371c0eed2 100644
--- a/tests/disas/gc/array-copy-with-fuel.wat
+++ b/tests/disas/gc/array-copy-with-fuel.wat
@@ -28,10 +28,10 @@
;; stack_limit = gv2
;;
;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32):
-;; v181 = stack_addr.i64 ss0
-;; store notrap aligned region6 v2, v181
-;; v182 = stack_addr.i64 ss1
-;; store notrap aligned region7 v4, v182
+;; v162 = stack_addr.i64 ss0
+;; store notrap aligned region6 v2, v162
+;; v163 = stack_addr.i64 ss1
+;; store notrap aligned region7 v4, v163
;; @0020 v7 = load.i64 notrap aligned readonly can_move region0 v0+8
;; @0020 v8 = load.i64 notrap aligned region2 v7
;; @0020 v9 = iconst.i64 1
@@ -41,131 +41,108 @@
;; @0020 brif v12, block2, block3(v10)
;;
;; block2:
-;; v191 = iadd.i64 v8, v9 ; v9 = 1
-;; @0020 store notrap aligned region2 v191, v7
+;; v172 = iadd.i64 v8, v9 ; v9 = 1
+;; @0020 store notrap aligned region2 v172, v7
;; @0020 v14 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0]
;; @0020 v16 = load.i64 notrap aligned region2 v7
;; @0020 jump block3(v16)
;;
-;; block3(v89: i64):
-;; v180 = load.i32 notrap aligned region6 v181
-;; @002b trapz v180, user16
-;; @002b v24 = load.i64 notrap aligned readonly can_move region3 v7+32
-;; @002b v22 = uextend.i64 v180
-;; @002b v25 = iadd v24, v22
-;; @002b v26 = iconst.i64 16
-;; @002b v27 = iadd v25, v26 ; v26 = 16
-;; @002b v28 = load.i32 user2 readonly region5 v27
-;; @002b v30 = uextend.i64 v3
-;; @002b v31 = uextend.i64 v6
-;; @002b v34 = iadd v30, v31
-;; @002b v29 = uextend.i64 v28
-;; @002b v35 = icmp ugt v34, v29
-;; @002b trapnz v35, user17
-;; v174 = load.i32 notrap aligned region7 v182
-;; @002b trapz v174, user16
-;; @002b v46 = uextend.i64 v174
-;; @002b v49 = iadd v24, v46
-;; @002b v51 = iadd v49, v26 ; v26 = 16
-;; @002b v52 = load.i32 user2 readonly region5 v51
-;; @002b v54 = uextend.i64 v5
-;; @002b v58 = iadd v54, v31
-;; @002b v53 = uextend.i64 v52
-;; @002b v59 = icmp ugt v58, v53
-;; @002b trapnz v59, user17
-;; @002b v78 = load.i64 notrap aligned region4 v7+40
-;; @002b v40 = iconst.i64 20
-;; @002b v41 = iadd v25, v40 ; v40 = 20
-;; v184 = iconst.i64 2
-;; v185 = ishl v30, v184 ; v184 = 2
-;; @002b v45 = iadd v41, v185
-;; v189 = ishl v31, v184 ; v184 = 2
-;; @002b v80 = uadd_overflow_trap v45, v189, user2
-;; @002b v79 = iadd v24, v78
-;; @002b v81 = icmp ugt v80, v79
-;; @002b trapnz v81, user2
-;; @002b v65 = iadd v49, v40 ; v40 = 20
-;; v187 = ishl v54, v184 ; v184 = 2
-;; @002b v69 = iadd v65, v187
-;; @002b v87 = uadd_overflow_trap v69, v189, user2
-;; @002b v88 = icmp ugt v87, v79
-;; @002b trapnz v88, user2
-;; @002b v90 = iconst.i64 6
-;; @002b v91 = iadd v89, v90 ; v90 = 6
-;; @002b brif.i32 v6, block4, block7(v91)
+;; block3(v25: i64):
+;; @002b v26 = iconst.i64 6
+;; @002b v27 = iadd v25, v26 ; v26 = 6
+;; @002b v22 = uextend.i64 v6
+;; @002b v28 = iadd v27, v22
+;; v173 = iconst.i64 0
+;; v174 = icmp sge v28, v173 ; v173 = 0
+;; @002b brif v174, block4, block5(v28)
;;
;; block4:
-;; v162 = load.i32 notrap aligned region6 v181
-;; v164 = load.i32 notrap aligned region7 v182
-;; @002b v92 = icmp.i64 ult v45, v69
-;; v192 = iadd.i64 v89, v90 ; v90 = 6
-;; @002b v97 = iadd.i64 v45, v189
-;; @002b v98 = iadd.i64 v69, v189
-;; @002b v100 = iadd.i32 v5, v6
-;; @002b v43 = iconst.i64 4
-;; @002b v141 = iconst.i32 1
-;; @002b brif v92, block5(v45, v69, v5, v162, v164, v192), block6(v97, v98, v100, v162, v164, v192)
+;; @002b store.i64 notrap aligned region2 v28, v7
+;; @002b v32 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0]
+;; @002b v34 = load.i64 notrap aligned region2 v7
+;; @002b jump block5(v34)
;;
-;; block5(v101: i64, v102: i64, v103: i32, v104: i32, v105: i32, v106: i64):
-;; store notrap aligned region6 v104, v181
-;; store notrap aligned region7 v105, v182
-;; v202 = iconst.i64 1
-;; v203 = iadd v106, v202 ; v202 = 1
-;; v204 = iconst.i64 0
-;; v205 = icmp sge v203, v204 ; v204 = 0
-;; @002b brif v205, block8, block9(v203)
+;; block5(v139: i64):
+;; v161 = load.i32 notrap aligned region6 v162
+;; @002b trapz v161, user16
+;; @002b v37 = load.i64 notrap aligned readonly can_move region3 v7+32
+;; @002b v35 = uextend.i64 v161
+;; @002b v38 = iadd v37, v35
+;; @002b v39 = iconst.i64 16
+;; @002b v40 = iadd v38, v39 ; v39 = 16
+;; @002b v41 = load.i32 user2 readonly region5 v40
+;; @002b v43 = uextend.i64 v3
+;; @002b v47 = iadd v43, v22
+;; @002b v42 = uextend.i64 v41
+;; @002b v48 = icmp ugt v47, v42
+;; @002b trapnz v48, user17
+;; v155 = load.i32 notrap aligned region7 v163
+;; @002b trapz v155, user16
+;; @002b v59 = uextend.i64 v155
+;; @002b v62 = iadd v37, v59
+;; @002b v64 = iadd v62, v39 ; v39 = 16
+;; @002b v65 = load.i32 user2 readonly region5 v64
+;; @002b v67 = uextend.i64 v5
+;; @002b v71 = iadd v67, v22
+;; @002b v66 = uextend.i64 v65
+;; @002b v72 = icmp ugt v71, v66
+;; @002b trapnz v72, user17
+;; @002b v91 = load.i64 notrap aligned region4 v7+40
+;; @002b v53 = iconst.i64 20
+;; @002b v54 = iadd v38, v53 ; v53 = 20
+;; v165 = iconst.i64 2
+;; v166 = ishl v43, v165 ; v165 = 2
+;; @002b v58 = iadd v54, v166
+;; v170 = ishl.i64 v22, v165 ; v165 = 2
+;; @002b v93 = uadd_overflow_trap v58, v170, user2
+;; @002b v92 = iadd v37, v91
+;; @002b v94 = icmp ugt v93, v92
+;; @002b trapnz v94, user2
+;; @002b v78 = iadd v62, v53 ; v53 = 20
+;; v168 = ishl v67, v165 ; v165 = 2
+;; @002b v82 = iadd v78, v168
+;; @002b v100 = uadd_overflow_trap v82, v170, user2
+;; @002b v101 = icmp ugt v100, v92
+;; @002b trapnz v101, user2
+;; @002b brif.i32 v6, block6, block9
;;
-;; block6(v123: i64, v124: i64, v125: i32, v126: i32, v127: i32, v128: i64):
-;; store notrap aligned region7 v126, v182
-;; store notrap aligned region6 v127, v181
-;; v193 = iconst.i64 1
-;; v194 = iadd v128, v193 ; v193 = 1
-;; v195 = iconst.i64 0
-;; v196 = icmp sge v194, v195 ; v195 = 0
-;; @002b brif v196, block10, block11(v194)
+;; block6:
+;; v143 = load.i32 notrap aligned region6 v162
+;; v145 = load.i32 notrap aligned region7 v163
+;; @002b v102 = icmp.i64 ult v58, v82
+;; @002b v107 = iadd.i64 v58, v170
+;; @002b v108 = iadd.i64 v82, v170
+;; @002b v110 = iadd.i32 v5, v6
+;; @002b v56 = iconst.i64 4
+;; @002b v133 = iconst.i32 1
+;; @002b brif v102, block7(v58, v82, v5), block8(v107, v108, v110)
;;
-;; block7(v148: i64):
-;; @002f jump block1
-;;
-;; block8:
-;; @002b store.i64 notrap aligned region2 v203, v7
-;; @002b v112 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0]
-;; @002b v114 = load.i64 notrap aligned region2 v7
-;; @002b jump block9(v114)
+;; block7(v111: i64, v112: i64, v113: i32):
+;; @002b v116 = load.i32 user2 little region5 v112
+;; @002b store user2 little region5 v116, v111
+;; v180 = iconst.i64 4
+;; v181 = iadd v112, v180 ; v180 = 4
+;; @002b v123 = icmp eq v181, v108
+;; v182 = iadd v111, v180 ; v180 = 4
+;; v183 = iconst.i32 1
+;; v184 = iadd v113, v183 ; v183 = 1
+;; @002b brif v123, block9, block7(v182, v181, v184)
;;
-;; block9(v145: i64):
-;; @002b v115 = load.i32 user2 little region5 v102
-;; @002b store user2 little region5 v115, v101
-;; v150 = load.i32 notrap aligned region6 v181
-;; v152 = load.i32 notrap aligned region7 v182
-;; v206 = iconst.i64 4
-;; v207 = iadd.i64 v102, v206 ; v206 = 4
-;; @002b v122 = icmp eq v207, v98
-;; v208 = iadd.i64 v101, v206 ; v206 = 4
-;; v209 = iconst.i32 1
-;; v210 = iadd.i32 v103, v209 ; v209 = 1
-;; @002b brif v122, block7(v145), block5(v208, v207, v210, v150, v152, v145)
+;; block8(v124: i64, v125: i64, v126: i32):
+;; v175 = iconst.i64 4
+;; v176 = isub v125, v175 ; v175 = 4
+;; @002b v135 = load.i32 user2 little region5 v176
+;; v177 = isub v124, v175 ; v175 = 4
+;; @002b store user2 little region5 v135, v177
+;; @002b v136 = icmp eq v176, v82
+;; v178 = iconst.i32 1
+;; v179 = isub v126, v178 ; v178 = 1
+;; @002b brif v136, block9, block8(v177, v176, v179)
;;
-;; block10:
-;; @002b store.i64 notrap aligned region2 v194, v7
-;; @002b v134 = call fn0(v0), stack_map=[i32 @ ss1+0, i32 @ ss0+0]
-;; @002b v136 = load.i64 notrap aligned region2 v7
-;; @002b jump block11(v136)
-;;
-;; block11(v146: i64):
-;; v197 = iconst.i64 4
-;; v198 = isub.i64 v124, v197 ; v197 = 4
-;; @002b v143 = load.i32 user2 little region5 v198
-;; v199 = isub.i64 v123, v197 ; v197 = 4
-;; @002b store user2 little region5 v143, v199
-;; v156 = load.i32 notrap aligned region7 v182
-;; v158 = load.i32 notrap aligned region6 v181
-;; @002b v144 = icmp eq v198, v69
-;; v200 = iconst.i32 1
-;; v201 = isub.i32 v125, v200 ; v200 = 1
-;; @002b brif v144, block7(v146), block6(v199, v198, v201, v156, v158, v146)
+;; block9:
+;; @002f jump block1
;;
;; block1:
-;; @002f store.i64 notrap aligned region2 v148, v7
+;; @002f store.i64 notrap aligned region2 v139, v7
;; @002f return
;; }
diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.wat
index f712972b87..d6113471a1 100644
--- a/tests/disas/memory-copy-epochs.wat
+++ b/tests/disas/memory-copy-epochs.wat
@@ -37,101 +37,35 @@
;; @001e v10 = call fn0(v0)
;; @001e jump block2(v10)
;;
-;; block2(v59: i64):
-;; @0025 v14 = load.i64 notrap aligned region6 v0+64
-;; @0025 v15 = uextend.i64 v2
-;; @0025 v16 = uextend.i64 v4
-;; @0025 v19 = iadd v15, v16
-;; @0025 v20 = icmp ugt v19, v14
-;; @0025 trapnz v20, heap_oob
-;; @0025 v27 = uextend.i64 v3
-;; @0025 v31 = iadd v27, v16
-;; @0025 v32 = icmp ugt v31, v14
-;; @0025 trapnz v32, heap_oob
-;; @0025 v21 = load.i64 notrap aligned readonly can_move region5 v0+56
-;; @0025 v37 = iadd v21, v27
-;; @0025 v25 = iadd v21, v15
-;; @0025 v40 = icmp ugt v37, v25
-;; @0025 brif v40, block6, block7
-;;
-;; block4(v42: i64, v43: i64, v44: i64, v47: i64):
-;; @0025 v46 = load.i64 notrap aligned region3 v5
-;; @0025 v48 = icmp uge v46, v47
-;; @0025 brif v48, block9, block8(v47)
-;;
-;; block5(v86: i64, v87: i64, v88: i64, v92: i64):
-;; @0025 v91 = load.i64 notrap aligned region3 v5
-;; @0025 v94 = icmp uge v91, v92
-;; @0025 brif v94, block17, block16
-;;
-;; block6:
-;; v108 = iconst.i64 0x0800_0000
-;; v109 = icmp.i64 ugt v16, v108 ; v108 = 0x0800_0000
-;; @0025 brif v109, block4(v25, v37, v16, v59), block5(v25, v37, v16, v59)
-;;
-;; block9 cold:
-;; @0025 v50 = load.i64 notrap aligned region4 v7+8
-;; @0025 v51 = icmp.i64 uge v46, v50
-;; @0025 brif v51, block10, block8(v50)
-;;
-;; block10 cold:
-;; @0025 v52 = call fn0(v0)
-;; @0025 jump block8(v52)
-;;
-;; block8(v60: i64):
-;; @0025 call fn1(v0, v42, v43, v108) ; v108 = 0x0800_0000
-;; @0025 v55 = isub.i64 v44, v108 ; v108 = 0x0800_0000
-;; @0025 v56 = icmp ugt v55, v108 ; v108 = 0x0800_0000
-;; @0025 v53 = iadd.i64 v42, v108 ; v108 = 0x0800_0000
-;; @0025 v54 = iadd.i64 v43, v108 ; v108 = 0x0800_0000
-;; @0025 brif v56, block4(v53, v54, v55, v60), block5(v53, v54, v55, v60)
-;;
-;; block7:
-;; @0025 v39 = iconst.i64 0x0800_0000
-;; @0025 v63 = icmp.i64 ugt v16, v39 ; v39 = 0x0800_0000
-;; @0025 v61 = iadd.i64 v25, v16
-;; @0025 v62 = iadd.i64 v37, v16
-;; @0025 brif v63, block11(v61, v62, v16, v59), block12(v61, v62, v16, v59)
-;;
-;; block11(v64: i64, v65: i64, v66: i64, v71: i64):
-;; @0025 v70 = load.i64 notrap aligned region3 v5
-;; @0025 v72 = icmp uge v70, v71
-;; @0025 brif v72, block14, block13(v71)
-;;
-;; block14 cold:
-;; @0025 v74 = load.i64 notrap aligned region4 v7+8
-;; @0025 v75 = icmp.i64 uge v70, v74
-;; @0025 brif v75, block15, block13(v74)
-;;
-;; block15 cold:
-;; @0025 v76 = call fn0(v0)
-;; @0025 jump block13(v76)
-;;
-;; block13(v80: i64):
-;; v103 = iconst.i64 0x0800_0000
-;; v104 = isub.i64 v64, v103 ; v103 = 0x0800_0000
-;; v105 = isub.i64 v65, v103 ; v103 = 0x0800_0000
-;; @0025 call fn1(v0, v104, v105, v103) ; v103 = 0x0800_0000
-;; v106 = isub.i64 v66, v103 ; v103 = 0x0800_0000
-;; v107 = icmp ugt v106, v103 ; v103 = 0x0800_0000
-;; @0025 brif v107, block11(v104, v105, v106, v80), block12(v104, v105, v106, v80)
-;;
-;; block12(v81: i64, v82: i64, v83: i64, v93: i64):
-;; @0025 v84 = isub v81, v83
-;; @0025 v85 = isub v82, v83
-;; @0025 jump block5(v84, v85, v83, v93)
-;;
-;; block17 cold:
-;; @0025 v96 = load.i64 notrap aligned region4 v7+8
-;; @0025 v97 = icmp.i64 uge v91, v96
-;; @0025 brif v97, block18, block16
-;;
-;; block18 cold:
-;; @0025 v98 = call fn0(v0)
-;; @0025 jump block16
-;;
-;; block16:
-;; @0025 call fn1(v0, v86, v87, v88)
+;; block2(v16: i64):
+;; @0025 v15 = load.i64 notrap aligned region3 v5
+;; @0025 v17 = icmp uge v15, v16
+;; @0025 brif v17, block5, block4
+;;
+;; block5 cold:
+;; @0025 v19 = load.i64 notrap aligned region4 v7+8
+;; @0025 v20 = icmp.i64 uge v15, v19
+;; @0025 brif v20, block6, block4
+;;
+;; block6 cold:
+;; @0025 v21 = call fn0(v0)
+;; @0025 jump block4
+;;
+;; block4:
+;; @0025 v22 = load.i64 notrap aligned region6 v0+64
+;; @0025 v23 = uextend.i64 v2
+;; @0025 v24 = uextend.i64 v4
+;; @0025 v27 = iadd v23, v24
+;; @0025 v28 = icmp ugt v27, v22
+;; @0025 trapnz v28, heap_oob
+;; @0025 v35 = uextend.i64 v3
+;; @0025 v39 = iadd v35, v24
+;; @0025 v40 = icmp ugt v39, v22
+;; @0025 trapnz v40, heap_oob
+;; @0025 v29 = load.i64 notrap aligned readonly can_move region5 v0+56
+;; @0025 v33 = iadd v29, v23
+;; @0025 v45 = iadd v29, v35
+;; @0025 call fn1(v0, v33, v45, v24)
;; @0029 jump block1
;;
;; block1:
diff --git a/tests/disas/memory-copy-fuel-const-len.wat b/tests/disas/memory-copy-fuel-const-len.wat
new file mode 100644
index 0000000000..aee7af900c
--- /dev/null
+++ b/tests/disas/memory-copy-fuel-const-len.wat
@@ -0,0 +1,184 @@
+;;! target = 'x86_64'
+;;! test = 'optimize'
+;;! flags = '-Wfuel=100'
+
+(module
+ (memory 1)
+ (func $copy_16 (param i32 i32)
+ (memory.copy (local.get 0) (local.get 1) (i32.const 16))
+ )
+ (func $fill_128 (param i32)
+ (memory.fill (local.get 0) (i32.const 0) (i32.const 128))
+ )
+ (func $fill_4096 (param i32)
+ (memory.fill (local.get 0) (i32.const 0) (i32.const 4096))
+ )
+)
+;; function u0:0(i64 vmctx, i64, i32, i32) tail {
+;; region0 = 8 "VMContext+0x8"
+;; region1 = 67108888 "VMStoreContext+0x18"
+;; region2 = 67108864 "VMStoreContext+0x0"
+;; region3 = 603979776 "VMMemoryDefinition+0x0"
+;; region4 = 603979784 "VMMemoryDefinition+0x8"
+;; region5 = 201326592 "DefinedMemory(StaticModuleIndex(0), DefinedMemoryIndex(0))"
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
+;; gv2 = load.i64 notrap aligned region1 gv1+24
+;; sig0 = (i64 vmctx) -> i8 tail
+;; fn0 = colocated u805306368:12 sig0
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32, v3: i32):
+;; @0023 v4 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @0023 v5 = load.i64 notrap aligned region2 v4
+;; @0023 v6 = iconst.i64 1
+;; @0023 v7 = iadd v5, v6 ; v6 = 1
+;; @0023 v8 = iconst.i64 0
+;; @0023 v9 = icmp sge v7, v8 ; v8 = 0
+;; @0023 brif v9, block2, block3(v7)
+;;
+;; block2:
+;; v58 = iadd.i64 v5, v6 ; v6 = 1
+;; @0023 store notrap aligned region2 v58, v4
+;; @0023 v11 = call fn0(v0)
+;; @0023 v13 = load.i64 notrap aligned region2 v4
+;; @0023 jump block3(v13)
+;;
+;; block3(v43: i64):
+;; @002a v17 = load.i64 notrap aligned region4 v0+64
+;; @002a v18 = uextend.i64 v2
+;; v47 = iconst.i64 16
+;; @002a v22 = iadd v18, v47 ; v47 = 16
+;; @002a v23 = icmp ugt v22, v17
+;; @002a trapnz v23, heap_oob
+;; @002a v30 = uextend.i64 v3
+;; @002a v34 = iadd v30, v47 ; v47 = 16
+;; @002a v35 = icmp ugt v34, v17
+;; @002a trapnz v35, heap_oob
+;; @002a v24 = load.i64 notrap aligned readonly can_move region3 v0+56
+;; @002a v40 = iadd v24, v30
+;; @002a v42 = load.i8x16 notrap aligned little region5 v40
+;; @002a v28 = iadd v24, v18
+;; @002a store notrap aligned little region5 v42, v28
+;; @002e jump block1
+;;
+;; block1:
+;; @002e v44 = iconst.i64 20
+;; @002e v45 = iadd.i64 v43, v44 ; v44 = 20
+;; @002e store notrap aligned region2 v45, v4
+;; @002e return
+;; }
+;;
+;; function u0:1(i64 vmctx, i64, i32) tail {
+;; region0 = 8 "VMContext+0x8"
+;; region1 = 67108888 "VMStoreContext+0x18"
+;; region2 = 67108864 "VMStoreContext+0x0"
+;; region3 = 603979776 "VMMemoryDefinition+0x0"
+;; region4 = 603979784 "VMMemoryDefinition+0x8"
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
+;; gv2 = load.i64 notrap aligned region1 gv1+24
+;; sig0 = (i64 vmctx) -> i8 tail
+;; sig1 = (i64 vmctx, i64, i32, i64) tail
+;; fn0 = colocated u805306368:12 sig0
+;; fn1 = colocated u805306368:2 sig1
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @0030 v3 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @0030 v4 = load.i64 notrap aligned region2 v3
+;; @0030 v5 = iconst.i64 1
+;; @0030 v6 = iadd v4, v5 ; v5 = 1
+;; @0030 v7 = iconst.i64 0
+;; @0030 v8 = icmp sge v6, v7 ; v7 = 0
+;; @0030 brif v8, block2, block3(v6)
+;;
+;; block2:
+;; v43 = iadd.i64 v4, v5 ; v5 = 1
+;; @0030 store notrap aligned region2 v43, v3
+;; @0030 v10 = call fn0(v0)
+;; @0030 v12 = load.i64 notrap aligned region2 v3
+;; @0030 jump block3(v12)
+;;
+;; block3(v29: i64):
+;; @0038 v16 = load.i64 notrap aligned region4 v0+64
+;; @0038 v17 = uextend.i64 v2
+;; v33 = iconst.i64 128
+;; @0038 v21 = iadd v17, v33 ; v33 = 128
+;; @0038 v22 = icmp ugt v21, v16
+;; @0038 trapnz v22, heap_oob
+;; @0038 v23 = load.i64 notrap aligned readonly can_move region3 v0+56
+;; @0038 v27 = iadd v23, v17
+;; @0033 v14 = iconst.i32 0
+;; @0038 call fn1(v0, v27, v14, v33) ; v14 = 0, v33 = 128
+;; @003b jump block1
+;;
+;; block1:
+;; @003b v30 = iconst.i64 132
+;; @003b v31 = iadd.i64 v29, v30 ; v30 = 132
+;; @003b store notrap aligned region2 v31, v3
+;; @003b return
+;; }
+;;
+;; function u0:2(i64 vmctx, i64, i32) tail {
+;; region0 = 8 "VMContext+0x8"
+;; region1 = 67108888 "VMStoreContext+0x18"
+;; region2 = 67108864 "VMStoreContext+0x0"
+;; region3 = 603979776 "VMMemoryDefinition+0x0"
+;; region4 = 603979784 "VMMemoryDefinition+0x8"
+;; gv0 = vmctx
+;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8
+;; gv2 = load.i64 notrap aligned region1 gv1+24
+;; sig0 = (i64 vmctx) -> i8 tail
+;; sig1 = (i64 vmctx, i64, i32, i64) tail
+;; fn0 = colocated u805306368:12 sig0
+;; fn1 = colocated u805306368:2 sig1
+;; stack_limit = gv2
+;;
+;; block0(v0: i64, v1: i64, v2: i32):
+;; @003d v3 = load.i64 notrap aligned readonly can_move region0 v0+8
+;; @003d v4 = load.i64 notrap aligned region2 v3
+;; @003d v5 = iconst.i64 1
+;; @003d v6 = iadd v4, v5 ; v5 = 1
+;; @003d v7 = iconst.i64 0
+;; @003d v8 = icmp sge v6, v7 ; v7 = 0
+;; @003d brif v8, block2, block3(v6)
+;;
+;; block2:
+;; v50 = iadd.i64 v4, v5 ; v5 = 1
+;; @003d store notrap aligned region2 v50, v3
+;; @003d v10 = call fn0(v0)
+;; @003d v12 = load.i64 notrap aligned region2 v3
+;; @003d jump block3(v12)
+;;
+;; block3(v16: i64):
+;; @0045 v17 = iconst.i64 4100
+;; @0045 v18 = iadd v16, v17 ; v17 = 4100
+;; v51 = iconst.i64 0
+;; v52 = icmp sge v18, v51 ; v51 = 0
+;; @0045 brif v52, block4, block5(v18)
+;;
+;; block4:
+;; v53 = iadd.i64 v16, v17 ; v17 = 4100
+;; @0045 store notrap aligned region2 v53, v3
+;; @0045 v22 = call fn0(v0)
+;; @0045 v24 = load.i64 notrap aligned region2 v3
+;; @0045 jump block5(v24)
+;;
+;; block5(v39: i64):
+;; @0045 v25 = load.i64 notrap aligned region4 v0+64
+;; @0045 v26 = uextend.i64 v2
+;; v40 = iconst.i64 4096
+;; @0045 v30 = iadd v26, v40 ; v40 = 4096
+;; @0045 v31 = icmp ugt v30, v25
+;; @0045 trapnz v31, heap_oob
+;; @0045 v32 = load.i64 notrap aligned readonly can_move region3 v0+56
+;; @0045 v36 = iadd v32, v26
+;; @0040 v14 = iconst.i32 0
+;; @0045 call fn1(v0, v36, v14, v40) ; v14 = 0, v40 = 4096
+;; @0048 jump block1
+;;
+;; block1:
+;; @0048 store.i64 notrap aligned region2 v39, v3
+;; @0048 return
+;; }
diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.wat
index 45e72b7043..29104ad297 100644
--- a/tests/disas/memory-copy-fuel.wat
+++ b/tests/disas/memory-copy-fuel.wat
@@ -33,110 +33,44 @@
;; @001e brif v10, block2, block3(v8)
;;
;; block2:
-;; v106 = iadd.i64 v6, v7 ; v7 = 1
-;; @001e store notrap aligned region2 v106, v5
+;; v61 = iadd.i64 v6, v7 ; v7 = 1
+;; @001e store notrap aligned region2 v61, v5
;; @001e v12 = call fn0(v0)
;; @001e v14 = load.i64 notrap aligned region2 v5
;; @001e jump block3(v14)
;;
-;; block3(v43: i64):
-;; @0025 v18 = load.i64 notrap aligned region4 v0+64
-;; @0025 v19 = uextend.i64 v2
-;; @0025 v20 = uextend.i64 v4
-;; @0025 v23 = iadd v19, v20
-;; @0025 v24 = icmp ugt v23, v18
-;; @0025 trapnz v24, heap_oob
-;; @0025 v31 = uextend.i64 v3
-;; @0025 v35 = iadd v31, v20
-;; @0025 v36 = icmp ugt v35, v18
-;; @0025 trapnz v36, heap_oob
-;; @0025 v25 = load.i64 notrap aligned readonly can_move region3 v0+56
-;; @0025 v41 = iadd v25, v31
-;; @0025 v29 = iadd v25, v19
-;; @0025 v47 = icmp ugt v41, v29
-;; @0025 brif v47, block6, block7
-;;
-;; block4(v49: i64, v50: i64, v51: i64, v52: i64):
-;; @0025 v53 = iadd v52, v116 ; v116 = 0x0800_0000
-;; v120 = iconst.i64 0
-;; v121 = icmp sge v53, v120 ; v120 = 0
-;; @0025 brif v121, block8, block9(v53)
-;;
-;; block5(v89: i64, v90: i64, v91: i64, v92: i64):
-;; @0025 v94 = iadd v92, v91
-;; v123 = iconst.i64 0
-;; v124 = icmp sge v94, v123 ; v123 = 0
-;; @0025 brif v124, block14, block15(v94)
-;;
-;; block6:
-;; v116 = iconst.i64 0x0800_0000
-;; v117 = icmp.i64 ugt v20, v116 ; v116 = 0x0800_0000
-;; v118 = iconst.i64 4
-;; v119 = iadd.i64 v43, v118 ; v118 = 4
-;; @0025 brif v117, block4(v29, v41, v20, v119), block5(v29, v41, v20, v119)
-;;
-;; block8:
-;; v122 = iadd.i64 v52, v116 ; v116 = 0x0800_0000
-;; @0025 store notrap aligned region2 v122, v5
-;; @0025 v57 = call fn0(v0)
-;; @0025 v59 = load.i64 notrap aligned region2 v5
-;; @0025 jump block9(v59)
-;;
-;; block9(v64: i64):
-;; @0025 call fn1(v0, v49, v50, v116) ; v116 = 0x0800_0000
-;; @0025 v62 = isub.i64 v51, v116 ; v116 = 0x0800_0000
-;; @0025 v63 = icmp ugt v62, v116 ; v116 = 0x0800_0000
-;; @0025 v60 = iadd.i64 v49, v116 ; v116 = 0x0800_0000
-;; @0025 v61 = iadd.i64 v50, v116 ; v116 = 0x0800_0000
-;; @0025 brif v63, block4(v60, v61, v62, v64), block5(v60, v61, v62, v64)
-;;
-;; block7:
-;; @0025 v46 = iconst.i64 0x0800_0000
-;; @0025 v67 = icmp.i64 ugt v20, v46 ; v46 = 0x0800_0000
-;; @0025 v65 = iadd.i64 v29, v20
-;; @0025 v66 = iadd.i64 v41, v20
-;; @0025 v44 = iconst.i64 4
-;; @0025 v45 = iadd.i64 v43, v44 ; v44 = 4
-;; @0025 brif v67, block10(v65, v66, v20, v45), block11(v65, v66, v20, v45)
-;;
-;; block10(v68: i64, v69: i64, v70: i64, v73: i64):
-;; v107 = iconst.i64 0x0800_0000
-;; v108 = iadd v73, v107 ; v107 = 0x0800_0000
-;; v109 = iconst.i64 0
-;; v110 = icmp sge v108, v109 ; v109 = 0
-;; @0025 brif v110, block12, block13(v108)
-;;
-;; block12:
-;; @0025 store.i64 notrap aligned region2 v108, v5
-;; @0025 v78 = call fn0(v0)
-;; @0025 v80 = load.i64 notrap aligned region2 v5
-;; @0025 jump block13(v80)
-;;
-;; block13(v83: i64):
-;; v111 = iconst.i64 0x0800_0000
-;; v112 = isub.i64 v68, v111 ; v111 = 0x0800_0000
-;; v113 = isub.i64 v69, v111 ; v111 = 0x0800_0000
-;; @0025 call fn1(v0, v112, v113, v111) ; v111 = 0x0800_0000
-;; v114 = isub.i64 v70, v111 ; v111 = 0x0800_0000
-;; v115 = icmp ugt v114, v111 ; v111 = 0x0800_0000
-;; @0025 brif v115, block10(v112, v113, v114, v83), block11(v112, v113, v114, v83)
-;;
-;; block11(v84: i64, v85: i64, v86: i64, v93: i64):
-;; @0025 v87 = isub v84, v86
-;; @0025 v88 = isub v85, v86
-;; @0025 jump block5(v87, v88, v86, v93)
-;;
-;; block14:
-;; @0025 store.i64 notrap aligned region2 v94, v5
-;; @0025 v98 = call fn0(v0)
-;; @0025 v100 = load.i64 notrap aligned region2 v5
-;; @0025 jump block15(v100)
-;;
-;; block15(v102: i64):
-;; @0025 call fn1(v0, v89, v90, v91)
+;; block3(v21: i64):
+;; @0025 v22 = iconst.i64 4
+;; @0025 v23 = iadd v21, v22 ; v22 = 4
+;; @0025 v18 = uextend.i64 v4
+;; @0025 v24 = iadd v23, v18
+;; v62 = iconst.i64 0
+;; v63 = icmp sge v24, v62 ; v62 = 0
+;; @0025 brif v63, block4, block5(v24)
+;;
+;; block4:
+;; @0025 store.i64 notrap aligned region2 v24, v5
+;; @0025 v28 = call fn0(v0)
+;; @0025 v30 = load.i64 notrap aligned region2 v5
+;; @0025 jump block5(v30)
+;;
+;; block5(v57: i64):
+;; @0025 v31 = load.i64 notrap aligned region4 v0+64
+;; @0025 v32 = uextend.i64 v2
+;; @0025 v36 = iadd v32, v18
+;; @0025 v37 = icmp ugt v36, v31
+;; @0025 trapnz v37, heap_oob
+;; @0025 v44 = uextend.i64 v3
+;; @0025 v48 = iadd v44, v18
+;; @0025 v49 = icmp ugt v48, v31
+;; @0025 trapnz v49, heap_oob
+;; @0025 v38 = load.i64 notrap aligned readonly can_move region3 v0+56
+;; @0025 v42 = iadd v38, v32
+;; @0025 v54 = iadd v38, v44
+;; @0025 call fn1(v0, v42, v54, v18)
;; @0029 jump block1
;;
;; block1:
-;; @0029 store.i64 notrap aligned region2 v102, v5
+;; @0029 store.i64 notrap aligned region2 v57, v5
;; @0029 return
;; }