Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

`fuel`: Defer fuel consumption for bulk operations (#14213)

Adam Bratschi-Kaye4ca0d0d060cd
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 9ecc2964a6..a617dc06c1 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -2607,7 +2607,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_grow_per_element;
- self.pre_translate_bulk_op(builder, delta, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, delta, cost);
let mut pos = builder.cursor();
let table = self.table(table_index);
@@ -2638,12 +2638,10 @@ impl FuncEnvironment<'_> {
// Conditionally call that on growth success, and otherwise fall through
// to continue to yield -1 for this growth operation.
let current_block = builder.current_block().unwrap();
- let failed_block = builder.create_block();
let fill_block = builder.create_block();
let done_block = builder.create_block();
- builder.insert_block_after(failed_block, current_block);
- builder.insert_block_after(fill_block, failed_block);
+ builder.insert_block_after(fill_block, current_block);
builder.insert_block_after(done_block, fill_block);
// Commit the operator's flat cost before branching so translating the
@@ -2653,14 +2651,7 @@ impl FuncEnvironment<'_> {
}
let failure = builder.ins().iconst(index_type_to_ir_type(index_type), -1);
let failed = builder.ins().icmp(IntCC::Equal, result_idx, failure);
- builder
- .ins()
- .brif(failed, failed_block, &[], fill_block, &[]);
-
- // A failed attempt performs no initialization loop, but still charge
- // for the requested growth so repeated failures are not free.
- builder.switch_to_block(failed_block);
- builder.ins().jump(done_block, &[]);
+ builder.ins().brif(failed, done_block, &[], fill_block, &[]);
builder.switch_to_block(fill_block);
self.translate_entity_fill(
@@ -2673,11 +2664,12 @@ impl FuncEnvironment<'_> {
init_value,
delta,
)?;
+ self.post_translate_bulk_op(builder, fuel)?;
+
builder.ins().jump(done_block, &[]);
builder.switch_to_block(done_block);
- builder.seal_block(failed_block);
builder.seal_block(fill_block);
builder.seal_block(done_block);
@@ -2825,7 +2817,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_fill_per_element;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_fill(
builder,
CheckedEntity::Table {
@@ -2835,7 +2827,8 @@ impl FuncEnvironment<'_> {
dst,
val,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
pub fn translate_ref_i31(
@@ -3063,7 +3056,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.array_copy_per_element;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3079,7 +3072,8 @@ impl FuncEnvironment<'_> {
dst_index,
src_index,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
pub fn translate_array_fill(
@@ -3097,7 +3091,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.array_fill_per_element;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_fill(
builder,
CheckedEntity::Array {
@@ -3108,7 +3102,8 @@ impl FuncEnvironment<'_> {
index,
value,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
pub fn translate_array_init_data(
@@ -3128,7 +3123,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.array_init_data_per_element;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3143,7 +3138,8 @@ impl FuncEnvironment<'_> {
dst_index,
data_offset,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
pub fn translate_array_init_elem(
@@ -3162,7 +3158,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.array_init_elem_per_element;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3174,7 +3170,8 @@ impl FuncEnvironment<'_> {
dst_index,
elem_offset,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
pub fn translate_array_len(
@@ -3563,7 +3560,7 @@ impl FuncEnvironment<'_> {
let index_type = self.memory(index).idx_type;
let cost = self.tunables.operator_cost.variable().memory_grow_per_page;
- self.pre_translate_bulk_op(builder, val, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, val, cost);
let mut pos = builder.cursor();
let val = self.cast_index_to_i64(&mut pos, val, index_type);
let call_inst = pos
@@ -3575,12 +3572,42 @@ impl FuncEnvironment<'_> {
0 => true,
_ => unreachable!("only page sizes 2**0 and 2**16 are currently valid"),
};
- Ok(self.convert_pointer_to_index_type(
+ let grow_result = self.convert_pointer_to_index_type(
builder.cursor(),
result,
index_type,
single_byte_pages,
- ))
+ );
+
+ // Consume fuel and do a fuel/epoch check only when the grow actually
+ // succeeded.
+ if fuel.is_some() {
+ let current_block = builder.current_block().unwrap();
+ let success_block = builder.create_block();
+ let done_block = builder.create_block();
+ builder.insert_block_after(success_block, current_block);
+ builder.insert_block_after(done_block, success_block);
+
+ // Flush outstanding fuel before branching.
+ if self.tunables.consume_fuel {
+ self.fuel_increment_var(builder);
+ }
+ let failure = builder.ins().iconst(index_type_to_ir_type(index_type), -1);
+ let failed = builder.ins().icmp(IntCC::Equal, grow_result, failure);
+ builder
+ .ins()
+ .brif(failed, done_block, &[], success_block, &[]);
+
+ builder.switch_to_block(success_block);
+ self.post_translate_bulk_op(builder, fuel)?;
+ builder.ins().jump(done_block, &[]);
+
+ builder.switch_to_block(done_block);
+ builder.seal_block(success_block);
+ builder.seal_block(done_block);
+ }
+
+ Ok(grow_result)
}
/// Loads the size, in bytes, of the memory `index` specified.
@@ -3668,8 +3695,9 @@ impl FuncEnvironment<'_> {
len: ir::Value,
) -> WasmResult<()> {
let cost = self.tunables.operator_cost.variable().memory_copy_per_byte;
- self.pre_translate_bulk_op(builder, len, cost)?;
- self.translate_entity_copy(builder, dst_index, src_index, dst, src, len)
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
+ self.translate_entity_copy(builder, dst_index, src_index, dst, src, len)?;
+ self.post_translate_bulk_op(builder, fuel)
}
/// Perform a raw bulk-memory-like libcall.
@@ -3996,8 +4024,9 @@ impl FuncEnvironment<'_> {
len: ir::Value,
) -> WasmResult<()> {
let cost = self.tunables.operator_cost.variable().memory_fill_per_byte;
- self.pre_translate_bulk_op(builder, len, cost)?;
- self.translate_entity_fill(builder, memory_index, dst, val, len)
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
+ self.translate_entity_fill(builder, memory_index, dst, val, len)?;
+ self.post_translate_bulk_op(builder, fuel)
}
pub fn translate_memory_init(
@@ -4011,7 +4040,7 @@ impl FuncEnvironment<'_> {
) -> WasmResult<()> {
let seg_index = DataIndex::from_u32(seg_index);
let cost = self.tunables.operator_cost.variable().memory_init_per_byte;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_copy(
builder,
memory_index,
@@ -4022,7 +4051,8 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
pub fn translate_data_drop(&mut self, mut pos: FuncCursor, seg_index: u32) -> WasmResult<()> {
@@ -4369,7 +4399,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_copy_per_element;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_copy(
builder,
CheckedEntity::Table {
@@ -4383,7 +4413,8 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
/// Emits a copy between two WebAssembly table or array entities.
@@ -4940,7 +4971,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_init_per_element;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_copy(
builder,
CheckedEntity::Table {
@@ -4951,7 +4982,8 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
pub fn translate_elem_drop(&mut self, mut pos: FuncCursor, elem_index: u32) -> WasmResult<()> {
@@ -5016,41 +5048,102 @@ impl FuncEnvironment<'_> {
/// Translation prefix before bulk operations such as `memory.copy`.
///
- /// Takes a dynamic value `units` for the size of the operation as well as
- /// a `cost_per_unit` configured for this operation. If fuel is enabled
- /// this fuel will be consumed, and if epochs are enabled then an epoch
- /// check happens. If neither epochs nor fuel are enabled this is a noop.
+ /// Takes a dynamic value `units` for the size of the operation as well as a
+ /// `cost_per_unit` configured for this operation.
+ ///
+ /// If fuel or epochs are enabled, a return value of `Some` indicates that a
+ /// fuel/epoch check should be performed if the operation succeeded. If
+ /// fuel is enabled, the return value also indicates how much fuel should be
+ /// consumed if the operation succeeds. The return value should be used by
+ /// `post_translate_bulk_op` to consume the fuel _after_ the operation has
+ /// succeeded to prevent turning OOB traps or grow failures into out-of-fuel
+ /// traps.
+ ///
+ /// For constant small operations the fuel is consumed here (if needed), and
+ /// the returned value is `None`.
+ ///
+ /// If neither fuel nor epochs are enabled this is a noop.
fn pre_translate_bulk_op(
&mut self,
builder: &mut FunctionBuilder,
units: ir::Value,
cost_per_unit: u8,
- ) -> WasmResult<()> {
+ ) -> Option<DeferredBulkOp> {
let const_units =
Self::value_as_const_int(builder, units).map(|c| i64::try_from(c).unwrap_or(i64::MAX));
- if self.tunables.consume_fuel && cost_per_unit > 0 {
- match const_units {
- // Fold constant costs directly into internal state.
- Some(units) => {
+ // Skip explicit fuel/epoch checks for operations which are
+ // subjectively, and statically, considered cheap and consume the fuel
+ // now instead of waiting to see if the operation succeeds.
+ const SMALL_BULK_OP_COST: i64 = 128;
+ if let Some(units) = const_units
+ && let Some(cost) = units.checked_mul(i64::from(cost_per_unit))
+ && cost <= SMALL_BULK_OP_COST
+ {
+ if self.tunables.consume_fuel && cost_per_unit > 0 {
+ self.fuel_consumed = self.fuel_consumed.saturating_add(cost);
+ }
+ return None;
+ }
+
+ if (self.tunables.consume_fuel || self.tunables.epoch_interruption) && cost_per_unit > 0 {
+ Some(DeferredBulkOp {
+ units: match const_units {
+ Some(const_units) => DeferredBulkUnits::Const(const_units),
+ None => DeferredBulkUnits::Runtime(units),
+ },
+ cost_per_unit,
+ })
+ } else {
+ None
+ }
+ }
+
+ /// Emitted after a bulk operation has completed successfully.
+ ///
+ /// First consumes the size-proportional fuel deferred by
+ /// [`Self::pre_translate_bulk_op`] and then performs the fuel/epoch check
+ /// that was likewise deferred from before the operation, retroactively
+ /// discovering whether the operation exhausted the fuel budget.
+ ///
+ /// Note: the fuel charge is emitted as runtime code if the units are
+ /// dynamic, but for constant units only `self.fuel_consumed` is updated.
+ /// The trailing fuel/epoch check flushes `self.fuel_consumed` before it
+ /// runs, so on return `self.fuel_consumed` is zero.
+ fn post_translate_bulk_op(
+ &mut self,
+ builder: &mut FunctionBuilder,
+ deferred: Option<DeferredBulkOp>,
+ ) -> WasmResult<()> {
+ // Charge the deferred size-proportional fuel now that the operation is
+ // known to have succeeded.
+ let Some(DeferredBulkOp {
+ units,
+ cost_per_unit,
+ }) = deferred
+ else {
+ return Ok(());
+ };
+ debug_assert!(
+ (self.tunables.consume_fuel || self.tunables.epoch_interruption) && cost_per_unit > 0
+ );
+ if self.tunables.consume_fuel {
+ match units {
+ DeferredBulkUnits::Const(units) => {
self.fuel_consumed = self
.fuel_consumed
.saturating_add(units.saturating_mul(i64::from(cost_per_unit)))
}
-
- None => {
- // Note that fuel is always a 64-bit counter.
- //
- // Also note that the cost is clamped to `i64::MAX` to
- // prevent fuel counter overflows since `cost` is otherwise
- // an untrusted value.
- let units_clamped64 = match builder.func.dfg.value_type(units) {
+ DeferredBulkUnits::Runtime(units) => {
+ self.fuel_increment_var(builder);
+ let fuel_var = builder.use_var(self.fuel_var);
+ let variable = match builder.func.dfg.value_type(units) {
ir::types::I32 => {
let units64 = builder.ins().uextend(ir::types::I64, units);
builder.ins().imul_imm_u(units64, i64::from(cost_per_unit))
}
ir::types::I64 => {
- let fuel = builder.ins().imul_imm_u(units, i64::from(cost_per_unit));
+ let product = builder.ins().imul_imm_u(units, i64::from(cost_per_unit));
let max = builder.ins().iconst(ir::types::I64, i64::MAX);
let max_units = builder
.ins()
@@ -5059,34 +5152,25 @@ impl FuncEnvironment<'_> {
builder
.ins()
.icmp(IntCC::UnsignedGreaterThan, units, max_units);
- builder.ins().select(saturate, max, fuel)
+ builder.ins().select(saturate, max, product)
}
_ => unreachable!(),
};
- self.fuel_increment_var(builder);
- let fuel = builder.use_var(self.fuel_var);
- let fuel = builder.ins().iadd(fuel, units_clamped64);
- builder.def_var(self.fuel_var, fuel);
+ let updated = builder.ins().iadd(fuel_var, variable);
+ builder.def_var(self.fuel_var, updated);
}
}
}
- // Skip explicit fuel/epoch checks for operations which are
- // subjectively, and statically, considered cheap.
- const SMALL_BULK_OP_COST: i64 = 128;
- if let Some(units) = const_units
- && let Some(cost) = units.checked_mul(i64::from(cost_per_unit))
- && cost <= SMALL_BULK_OP_COST
- {
- return Ok(());
- }
+ // Perform the fuel/epoch check that was deferred from before the
+ // operation. This isn't a loop header but for fuel/epoch purposes it's
+ // the same thing.
+ self.translate_loop_header(builder);
- // This isn't a loop header but for fuel/epoch purposes it's the same
- // thing.
- self.translate_loop_header(builder)
+ Ok(())
}
- pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {
+ pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) {
// Additionally if enabled check how much fuel we have remaining to see
// if we've run out by this point.
if self.tunables.consume_fuel {
@@ -5098,8 +5182,6 @@ impl FuncEnvironment<'_> {
if self.tunables.epoch_interruption {
self.epoch_check(builder);
}
-
- Ok(())
}
pub fn before_translate_operator(
@@ -5973,7 +6055,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_fill_per_element;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_fill(
builder,
CheckedEntity::Table {
@@ -5983,7 +6065,8 @@ impl FuncEnvironment<'_> {
dst,
val,
len,
- )
+ )?;
+ self.post_translate_bulk_op(builder, fuel)
}
/// Executes initialization for an active element segment in a module.
@@ -6018,7 +6101,7 @@ impl FuncEnvironment<'_> {
.operator_cost
.variable()
.table_init_per_element;
- self.pre_translate_bulk_op(builder, segment_len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, segment_len, cost);
// Re-use the `table.set` translation for making this a simple function
// to define. That re-executes the bounds check which is a bit
@@ -6039,7 +6122,7 @@ impl FuncEnvironment<'_> {
}
}
}
- Ok(())
+ self.post_translate_bulk_op(builder, fuel)
}
/// Peform initialization of an active data segment in a module.
@@ -6091,8 +6174,9 @@ impl FuncEnvironment<'_> {
let len = self.load_runtime_data_length(builder, data);
let start = builder.ins().iconst(I32, 0);
let cost = self.tunables.operator_cost.variable().memory_init_per_byte;
- self.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = self.pre_translate_bulk_op(builder, len, cost);
self.translate_entity_copy(builder, memory, data, offset, start, len)?;
+ self.post_translate_bulk_op(builder, fuel)?;
// Finalize control-flow for the `MemorySegmentOffset::Static` case
// above.
@@ -6244,6 +6328,32 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type {
}
}
+/// Deferred bookkeeping for a "large" bulk operation, produced by
+/// [`FuncEnvironment::pre_translate_bulk_op`] and consumed by
+/// [`FuncEnvironment::post_translate_bulk_op`] once the operation has completed
+/// successfully.
+///
+/// Its presence means a fuel/epoch check must be emitted _after_ the operation
+/// rather than before it. Deferring the check means a bulk op that traps or
+/// fails is not retroactively billed as running out of fuel; instead a
+/// successful op that exhausts the budget is discovered by the check that
+/// immediately follows it. The [`DeferredBulkOpCheck::Fuel`] variant
+/// additionally carries the size-proportional fuel to charge on success.
+#[must_use = "DeferredBulkOpCheck must be passed to post_translate_bulk_op"]
+struct DeferredBulkOp {
+ /// The number of units (bytes/elements/pages) operated on.
+ units: DeferredBulkUnits,
+ /// The per-unit fuel cost.
+ cost_per_unit: u8,
+}
+
+enum DeferredBulkUnits {
+ /// A statically-known unit count, already clamped to `i64`.
+ Const(i64),
+ /// A runtime unit count held in an `ir::Value`.
+ Runtime(ir::Value),
+}
+
/// Operations to [`FuncEnvironment::raw_bulk_memory_operation`].
#[derive(Clone)]
enum BulkOp {
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index 8987cd9a25..9177b11d59 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -808,7 +808,7 @@ pub fn translate_array_new(
.operator_cost
.variable()
.array_new_per_element;
- func_env.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = func_env.pre_translate_bulk_op(builder, len, cost);
let result =
gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;
@@ -825,6 +825,7 @@ pub fn translate_array_new(
elem,
len,
)?;
+ func_env.post_translate_bulk_op(builder, fuel)?;
log::trace!("translate_array_new(..) -> {result:?}");
Ok(result)
}
@@ -841,7 +842,7 @@ pub fn translate_array_new_default(
.operator_cost
.variable()
.array_new_default_per_element;
- func_env.pre_translate_bulk_op(builder, len, cost)?;
+ let fuel = func_env.pre_translate_bulk_op(builder, len, cost);
let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index();
let array_ty = func_env.types.unwrap_array(interned_ty)?;
@@ -861,6 +862,7 @@ pub fn translate_array_new_default(
elem,
len,
)?;
+ func_env.post_translate_bulk_op(builder, fuel)?;
Ok(result)
}
@@ -1769,7 +1771,7 @@ pub fn translate_array_new_entity(
len: ir::Value,
cost_per_unit: u8,
) -> WasmResult<ir::Value> {
- env.pre_translate_bulk_op(builder, len, cost_per_unit)?;
+ let fuel = env.pre_translate_bulk_op(builder, len, cost_per_unit);
// Before actually allocating this array first do a bounds-check on the
// passive entity itself.
@@ -1791,5 +1793,6 @@ pub fn translate_array_new_entity(
len,
)?;
+ env.post_translate_bulk_op(builder, fuel)?;
Ok(array)
}
diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rs
index 6a1f0dac2a..e86340ec2e 100644
--- a/crates/cranelift/src/translate/code_translator.rs
+++ b/crates/cranelift/src/translate/code_translator.rs
@@ -285,7 +285,7 @@ pub fn translate_operator(
environ.stacks.popn(params.len());
builder.switch_to_block(loop_body);
push_block_params(environ, builder, loop_body);
- environ.translate_loop_header(builder)?;
+ environ.translate_loop_header(builder);
}
Operator::If { blockty } => {
// Read the hint before `environ` is borrowed mutably below.
diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs
index 70ea869d64..8e0c6f03ab 100644
--- a/crates/environ/src/tunables.rs
+++ b/crates/environ/src/tunables.rs
@@ -601,8 +601,7 @@ impl VariableOperatorCost {
memory_copy_per_byte: 1,
memory_fill_per_byte: 1,
memory_init_per_byte: 1,
- // `memory.grow` did not previously have a dynamic fuel charge.
- memory_grow_per_page: 0,
+ memory_grow_per_page: 1,
table_copy_per_element: 1,
table_fill_per_element: 1,
table_init_per_element: 1,
diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rs
index 011e5a544c..fceb5fc55c 100644
--- a/crates/wasmtime/src/runtime/vm/table.rs
+++ b/crates/wasmtime/src/runtime/vm/table.rs
@@ -632,7 +632,10 @@ impl Table {
if delta == 0 {
return Ok(Some(old_size));
}
- let delta = usize::try_from(delta).map_err(|_| Trap::TableOutOfBounds)?;
+
+ // Clamp to maximum usize, the system won't be able to allocate this
+ // anyway and we'll go to the normal failure path.
+ let delta = usize::try_from(delta).unwrap_or(usize::MAX);
let new_size = match old_size.checked_add(delta) {
Some(s) => s,
diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs
index f6a06a21f0..cee3e8332e 100644
--- a/tests/all/fuel.rs
+++ b/tests/all/fuel.rs
@@ -829,48 +829,6 @@ fn custom_variable_operator_cost(config: &mut Config) -> Result<()> {
Ok(())
}
-#[wasmtime_test(wasm_features(reference_types), strategies(not(Winch)))]
-#[cfg_attr(miri, ignore)]
-fn variable_operator_cost_failed_growth(config: &mut Config) -> Result<()> {
- config.consume_fuel(true);
- let mut op_cost = OperatorCost {
- I32Const: 0,
- RefNull: 0,
- MemoryGrow: 0,
- TableGrow: 0,
- ..Default::default()
- };
- op_cost.variable.memory_grow_per_page = 7;
- op_cost.variable.table_grow_per_element = 19;
- config.operator_cost(op_cost.clone());
-
- let engine = Engine::new(config)?;
- let module = Module::new(
- &engine,
- r#"(module
- (memory 1 1)
- (table 0 0 funcref)
- (func (export "main")
- ;; these exceed the max limits so they fail
- i32.const 5 memory.grow drop
- ref.null func i32.const 5 table.grow drop)
- )"#,
- )?;
- let mut store = Store::new(&engine, ());
- store.set_fuel(1_000)?;
- let instance = Instance::new(&mut store, &module, &[])?;
- let main = instance.get_typed_func::<(), ()>(&mut store, "main")?;
-
- let initial_fuel = store.get_fuel()?;
- main.call(&mut store, ())?;
- let cost_of_execution = 5 * u64::from(op_cost.variable.memory_grow_per_page)
- + 5 * u64::from(op_cost.variable.table_grow_per_element)
- + 1;
- assert_eq!(store.get_fuel()?, initial_fuel - cost_of_execution);
-
- Ok(())
-}
-
#[wasmtime_test(wasm_features(bulk_memory), strategies(not(Winch)))]
#[cfg_attr(miri, ignore)]
fn variable_operator_cost_follows_bounds_check(config: &mut Config) -> Result<()> {
@@ -911,64 +869,6 @@ fn variable_operator_cost_follows_bounds_check(config: &mut Config) -> Result<()
Ok(())
}
-#[wasmtime_test(wasm_features(memory64), strategies(not(Winch)))]
-#[cfg_attr(miri, ignore)]
-fn memory64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> {
- config.consume_fuel(true);
- let mut operator_cost = OperatorCost::default();
- operator_cost.variable.memory_grow_per_page = 2;
- config.operator_cost(operator_cost);
-
- let engine = Engine::new(config)?;
- let module = Module::new(
- &engine,
- r#"(module
- (memory i64 0 0)
- (func (export "grow") (param i64) (result i64)
- local.get 0 memory.grow)
- )"#,
- )?;
- let mut store = Store::new(&engine, ());
- store.set_fuel(10_000)?;
- let instance = Instance::new(&mut store, &module, &[])?;
- let grow = instance.get_typed_func::<i64, i64>(&mut store, "grow")?;
-
- // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2.
- let error = grow.call(&mut store, i64::MAX).unwrap_err();
- assert_eq!(error.downcast::<Trap>().unwrap(), Trap::OutOfFuel);
-
- Ok(())
-}
-
-#[wasmtime_test(wasm_features(memory64, reference_types), strategies(not(Winch)))]
-#[cfg_attr(miri, ignore)]
-#[cfg(target_pointer_width = "64")]
-fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> {
- config.consume_fuel(true);
- let mut operator_cost = OperatorCost::default();
- operator_cost.variable.table_grow_per_element = 2;
- config.operator_cost(operator_cost);
-
- let engine = Engine::new(config)?;
- let module = Module::new(
- &engine,
- r#"(module
- (table i64 0 0 funcref)
- (func (export "grow") (param i64) (result i64)
- ref.null func local.get 0 table.grow)
- )"#,
- )?;
- let mut store = Store::new(&engine, ());
- store.set_fuel(10_000)?;
- let instance = Instance::new(&mut store, &module, &[])?;
- let grow = instance.get_typed_func::<i64, i64>(&mut store, "grow")?;
-
- // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2.
- let error = grow.call(&mut store, i64::MAX).unwrap_err();
- assert_eq!(error.downcast::<Trap>().unwrap(), Trap::OutOfFuel);
- Ok(())
-}
-
#[test]
#[cfg_attr(miri, ignore)]
fn huge_table64_grow_cannot_mint_fuel() -> Result<()> {
@@ -1040,16 +940,15 @@ fn fuel_around_table_grow() -> Result<()> {
store.set_fuel(2)?;
let instance = Instance::new(&mut store, &module, &[])?;
let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?;
+ // The fuel check for a large bulk operation runs after the operation
+ // completes so this call still traps with "out of fuel", but the grow
+ // itself took effect.
let trap = grow.call(&mut store, ()).unwrap_err().downcast::<Trap>()?;
assert_eq!(trap, Trap::OutOfFuel);
store.set_fuel(u64::MAX)?;
let call = instance.get_typed_func::<i32, ()>(&mut store, "call")?;
- let trap = call
- .call(&mut store, 9999999)
- .unwrap_err()
- .downcast::<Trap>()?;
- assert_eq!(trap, Trap::TableOutOfBounds);
+ call.call(&mut store, 9999999)?;
Ok(())
}
diff --git a/tests/all/fuel.wast b/tests/all/fuel.wast
index f918eb94ac..1a3f2af3e3 100644
--- a/tests/all/fuel.wast
+++ b/tests/all/fuel.wast
@@ -207,6 +207,9 @@
)
(start $f)))
+;; A small constant size (size * cost <= 128, the `SMALL_BULK_OP_COST`
+;; threshold) charges the size-proportional "variable" fuel up front, before
+;; the op runs.
(assert_fuel 107
(module
(memory 1)
@@ -263,17 +266,6 @@
)
(start $f)))
-(assert_fuel 106
- (module
- (table 0 funcref)
- (func $f
- ref.null func
- i32.const 100
- table.grow
- drop
- )
- (start $f)))
-
(assert_fuel 27
(module
(table 20 funcref)
@@ -385,3 +377,493 @@
drop
)
(start $f)))
+
+;; A large constant size (> 128) instead defers that variable fuel until
+;; after the op has run.
+(assert_fuel 207
+ (module
+ (memory 1)
+ (func $f
+ i32.const 0
+ i32.const 0
+ i32.const 200
+ memory.copy
+ )
+ (start $f)))
+
+(assert_fuel 207
+ (module
+ (memory 1)
+ (func $f
+ i32.const 0
+ i32.const 0
+ i32.const 200
+ memory.fill
+ )
+ (start $f)))
+
+(assert_fuel 207
+ (module
+ (memory 1)
+ (func $f
+ i32.const 0
+ i32.const 0
+ i32.const 200
+ memory.init $d
+ )
+ (start $f)
+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))
+
+(assert_fuel 207
+ (module
+ (table 200 funcref)
+ (func $f
+ i32.const 0
+ i32.const 0
+ i32.const 200
+ table.copy
+ )
+ (start $f)))
+
+(assert_fuel 207
+ (module
+ (table 200 funcref)
+ (func $f
+ i32.const 0
+ ref.null func
+ i32.const 200
+ table.fill
+ )
+ (start $f)))
+
+(assert_fuel 207
+ (module
+ (table 200 funcref)
+ (func $f
+ i32.const 0
+ i32.const 0
+ i32.const 200
+ table.init $e
+ )
+ (start $f)
+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
+
+(assert_fuel 411
+ (module
+ (type $a (array (mut i8)))
+ (global $a (ref $a) (array.new_default $a (i32.const 200)))
+ (func $f
+ global.get $a
+ i32.const 0
+ global.get $a
+ i32.const 0
+ i32.const 200
+ array.copy $a $a
+ )
+ (start $f)))
+
+(assert_fuel 410
+ (module
+ (type $a (array (mut i8)))
+ (global $a (ref $a) (array.new_default $a (i32.const 200)))
+ (func $f
+ global.get $a
+ i32.const 0
+ i32.const 0
+ i32.const 200
+ array.fill $a
+ )
+ (start $f)))
+
+(assert_fuel 206
+ (module
+ (type $a (array (mut i8)))
+ (func $f
+ i32.const 0
+ i32.const 200
+ array.new_data $a $d
+ drop
+ )
+ (start $f)
+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))
+
+(assert_fuel 410
+ (module
+ (type $a (array (mut i8)))
+ (global $a (ref $a) (array.new_default $a (i32.const 200)))
+ (func $f
+ global.get $a
+ i32.const 0
+ i32.const 0
+ i32.const 200
+ array.init_data $a $d
+ )
+ (start $f)
+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))
+
+(assert_fuel 206
+ (module
+ (type $a (array (mut funcref)))
+ (func $f
+ i32.const 0
+ i32.const 200
+ array.new_elem $a $e
+ drop
+ )
+ (start $f)
+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
+
+(assert_fuel 410
+ (module
+ (type $a (array (mut funcref)))
+ (global $a (ref $a) (array.new_default $a (i32.const 200)))
+ (func $f
+ global.get $a
+ i32.const 0
+ i32.const 0
+ i32.const 200
+ array.init_elem $a $e
+ )
+ (start $f)
+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
+
+(assert_fuel 205
+ (module
+ (type $a (array (mut funcref)))
+ (func $f
+ i32.const 200
+ array.new_default $a
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 206
+ (module
+ (type $a (array (mut funcref)))
+ (func $f
+ ref.null func
+ i32.const 200
+ array.new $a
+ drop
+ )
+ (start $f)))
+
+;; A dynamic (runtime) size likewise defers the variable fuel until after
+;; the op has run, whatever its magnitude.
+(assert_fuel 57
+ (module
+ (memory 1)
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ i32.const 0
+ i32.const 0
+ global.get $n
+ memory.copy
+ )
+ (start $f)))
+
+(assert_fuel 57
+ (module
+ (memory 1)
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ i32.const 0
+ i32.const 0
+ global.get $n
+ memory.fill
+ )
+ (start $f)))
+
+(assert_fuel 57
+ (module
+ (memory 1)
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ i32.const 0
+ i32.const 0
+ global.get $n
+ memory.init $d
+ )
+ (start $f)
+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))
+
+(assert_fuel 57
+ (module
+ (table 50 funcref)
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ i32.const 0
+ i32.const 0
+ global.get $n
+ table.copy
+ )
+ (start $f)))
+
+(assert_fuel 57
+ (module
+ (table 50 funcref)
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ i32.const 0
+ ref.null func
+ global.get $n
+ table.fill
+ )
+ (start $f)))
+
+(assert_fuel 57
+ (module
+ (table 50 funcref)
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ i32.const 0
+ i32.const 0
+ global.get $n
+ table.init $e
+ )
+ (start $f)
+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
+
+(assert_fuel 111
+ (module
+ (type $a (array (mut i8)))
+ (global $a (ref $a) (array.new_default $a (i32.const 50)))
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ global.get $a
+ i32.const 0
+ global.get $a
+ i32.const 0
+ global.get $n
+ array.copy $a $a
+ )
+ (start $f)))
+
+(assert_fuel 110
+ (module
+ (type $a (array (mut i8)))
+ (global $a (ref $a) (array.new_default $a (i32.const 50)))
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ global.get $a
+ i32.const 0
+ i32.const 0
+ global.get $n
+ array.fill $a
+ )
+ (start $f)))
+
+(assert_fuel 56
+ (module
+ (type $a (array (mut i8)))
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ i32.const 0
+ global.get $n
+ array.new_data $a $d
+ drop
+ )
+ (start $f)
+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))
+
+(assert_fuel 110
+ (module
+ (type $a (array (mut i8)))
+ (global $a (ref $a) (array.new_default $a (i32.const 50)))
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ global.get $a
+ i32.const 0
+ i32.const 0
+ global.get $n
+ array.init_data $a $d
+ )
+ (start $f)
+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))
+
+(assert_fuel 56
+ (module
+ (type $a (array (mut funcref)))
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ i32.const 0
+ global.get $n
+ array.new_elem $a $e
+ drop
+ )
+ (start $f)
+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
+
+(assert_fuel 110
+ (module
+ (type $a (array (mut funcref)))
+ (global $a (ref $a) (array.new_default $a (i32.const 50)))
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ global.get $a
+ i32.const 0
+ i32.const 0
+ global.get $n
+ array.init_elem $a $e
+ )
+ (start $f)
+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))
+
+(assert_fuel 55
+ (module
+ (type $a (array (mut funcref)))
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ global.get $n
+ array.new_default $a
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 56
+ (module
+ (type $a (array (mut funcref)))
+ (global $n (mut i32) (i32.const 50))
+ (func $f
+ ref.null func
+ global.get $n
+ array.new $a
+ drop
+ )
+ (start $f)))
+
+;; Growing a memory or table charges a small constant size's variable fuel up
+;; front, but defers a large or dynamic size's variable fuel until the grow has
+;; succeeded.
+;;
+;; The following tests cover the 12 cases of table/memory x small/large/dynamic
+;; size x success/failure.
+(assert_fuel 106
+ (module
+ (table 0 funcref)
+ (func $f
+ ref.null func
+ i32.const 100
+ table.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 106
+ (module
+ (table 0 0 funcref)
+ (func $f
+ ref.null func
+ i32.const 100
+ table.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 206
+ (module
+ (table 0 funcref)
+ (func $f
+ ref.null func
+ i32.const 200
+ table.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 6
+ (module
+ (table 0 0 funcref)
+ (func $f
+ ref.null func
+ i32.const 200
+ table.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 56
+ (module
+ (table 0 funcref)
+ (global $d (mut i32) (i32.const 50))
+ (func $f
+ ref.null func
+ global.get $d
+ table.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 6
+ (module
+ (table 0 0 funcref)
+ (global $d (mut i32) (i32.const 50))
+ (func $f
+ ref.null func
+ global.get $d
+ table.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 105
+ (module
+ (memory 1)
+ (func $f
+ i32.const 100
+ memory.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 105
+ (module
+ (memory 0 0)
+ (func $f
+ i32.const 100
+ memory.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 205
+ (module
+ (memory 1)
+ (func $f
+ i32.const 200
+ memory.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 5
+ (module
+ (memory 0 0)
+ (func $f
+ i32.const 200
+ memory.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 55
+ (module
+ (memory 1)
+ (global $d (mut i32) (i32.const 50))
+ (func $f
+ global.get $d
+ memory.grow
+ drop
+ )
+ (start $f)))
+
+(assert_fuel 5
+ (module
+ (memory 0 0)
+ (global $d (mut i32) (i32.const 50))
+ (func $f
+ global.get $d
+ memory.grow
+ drop
+ )
+ (start $f)))
diff --git a/tests/disas/gc/array-copy-with-fuel.wat b/tests/disas/gc/array-copy-with-fuel.wat
index 8371c0eed2..63ab28b9a6 100644
--- a/tests/disas/gc/array-copy-with-fuel.wat
+++ b/tests/disas/gc/array-copy-with-fuel.wat
@@ -47,102 +47,102 @@
;; @0020 v16 = load.i64 notrap aligned region2 v7
;; @0020 jump block3(v16)
;;
-;; block3(v25: i64):
-;; @002b v26 = iconst.i64 6
-;; @002b v27 = iadd v25, v26 ; v26 = 6
-;; @002b v22 = uextend.i64 v6
-;; @002b v28 = iadd v27, v22
-;; v173 = iconst.i64 0
-;; v174 = icmp sge v28, v173 ; v173 = 0
-;; @002b brif v174, block4, block5(v28)
-;;
-;; block4:
-;; @002b store.i64 notrap aligned region2 v28, v7
-;; @002b v32 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0]
-;; @002b v34 = load.i64 notrap aligned region2 v7
-;; @002b jump block5(v34)
-;;
-;; block5(v139: i64):
+;; block3(v125: i64):
;; v161 = load.i32 notrap aligned region6 v162
;; @002b trapz v161, user16
-;; @002b v37 = load.i64 notrap aligned readonly can_move region3 v7+32
-;; @002b v35 = uextend.i64 v161
-;; @002b v38 = iadd v37, v35
-;; @002b v39 = iconst.i64 16
-;; @002b v40 = iadd v38, v39 ; v39 = 16
-;; @002b v41 = load.i32 user2 readonly region5 v40
-;; @002b v43 = uextend.i64 v3
-;; @002b v47 = iadd v43, v22
-;; @002b v42 = uextend.i64 v41
-;; @002b v48 = icmp ugt v47, v42
-;; @002b trapnz v48, user17
+;; @002b v24 = load.i64 notrap aligned readonly can_move region3 v7+32
+;; @002b v22 = uextend.i64 v161
+;; @002b v25 = iadd v24, v22
+;; @002b v26 = iconst.i64 16
+;; @002b v27 = iadd v25, v26 ; v26 = 16
+;; @002b v28 = load.i32 user2 readonly region5 v27
+;; @002b v30 = uextend.i64 v3
+;; @002b v31 = uextend.i64 v6
+;; @002b v34 = iadd v30, v31
+;; @002b v29 = uextend.i64 v28
+;; @002b v35 = icmp ugt v34, v29
+;; @002b trapnz v35, user17
;; v155 = load.i32 notrap aligned region7 v163
;; @002b trapz v155, user16
-;; @002b v59 = uextend.i64 v155
-;; @002b v62 = iadd v37, v59
-;; @002b v64 = iadd v62, v39 ; v39 = 16
-;; @002b v65 = load.i32 user2 readonly region5 v64
-;; @002b v67 = uextend.i64 v5
-;; @002b v71 = iadd v67, v22
-;; @002b v66 = uextend.i64 v65
-;; @002b v72 = icmp ugt v71, v66
-;; @002b trapnz v72, user17
-;; @002b v91 = load.i64 notrap aligned region4 v7+40
-;; @002b v53 = iconst.i64 20
-;; @002b v54 = iadd v38, v53 ; v53 = 20
+;; @002b v46 = uextend.i64 v155
+;; @002b v49 = iadd v24, v46
+;; @002b v51 = iadd v49, v26 ; v26 = 16
+;; @002b v52 = load.i32 user2 readonly region5 v51
+;; @002b v54 = uextend.i64 v5
+;; @002b v58 = iadd v54, v31
+;; @002b v53 = uextend.i64 v52
+;; @002b v59 = icmp ugt v58, v53
+;; @002b trapnz v59, user17
+;; @002b v78 = load.i64 notrap aligned region4 v7+40
+;; @002b v40 = iconst.i64 20
+;; @002b v41 = iadd v25, v40 ; v40 = 20
;; v165 = iconst.i64 2
-;; v166 = ishl v43, v165 ; v165 = 2
-;; @002b v58 = iadd v54, v166
-;; v170 = ishl.i64 v22, v165 ; v165 = 2
-;; @002b v93 = uadd_overflow_trap v58, v170, user2
-;; @002b v92 = iadd v37, v91
-;; @002b v94 = icmp ugt v93, v92
-;; @002b trapnz v94, user2
-;; @002b v78 = iadd v62, v53 ; v53 = 20
-;; v168 = ishl v67, v165 ; v165 = 2
-;; @002b v82 = iadd v78, v168
-;; @002b v100 = uadd_overflow_trap v82, v170, user2
-;; @002b v101 = icmp ugt v100, v92
-;; @002b trapnz v101, user2
-;; @002b brif.i32 v6, block6, block9
+;; v166 = ishl v30, v165 ; v165 = 2
+;; @002b v45 = iadd v41, v166
+;; v170 = ishl v31, v165 ; v165 = 2
+;; @002b v80 = uadd_overflow_trap v45, v170, user2
+;; @002b v79 = iadd v24, v78
+;; @002b v81 = icmp ugt v80, v79
+;; @002b trapnz v81, user2
+;; @002b v65 = iadd v49, v40 ; v40 = 20
+;; v168 = ishl v54, v165 ; v165 = 2
+;; @002b v69 = iadd v65, v168
+;; @002b v87 = uadd_overflow_trap v69, v170, user2
+;; @002b v88 = icmp ugt v87, v79
+;; @002b trapnz v88, user2
+;; @002b brif.i32 v6, block4, block7
;;
-;; block6:
+;; block4:
;; v143 = load.i32 notrap aligned region6 v162
;; v145 = load.i32 notrap aligned region7 v163
-;; @002b v102 = icmp.i64 ult v58, v82
-;; @002b v107 = iadd.i64 v58, v170
-;; @002b v108 = iadd.i64 v82, v170
-;; @002b v110 = iadd.i32 v5, v6
-;; @002b v56 = iconst.i64 4
-;; @002b v133 = iconst.i32 1
-;; @002b brif v102, block7(v58, v82, v5), block8(v107, v108, v110)
+;; @002b v89 = icmp.i64 ult v45, v69
+;; @002b v94 = iadd.i64 v45, v170
+;; @002b v95 = iadd.i64 v69, v170
+;; @002b v97 = iadd.i32 v5, v6
+;; @002b v43 = iconst.i64 4
+;; @002b v120 = iconst.i32 1
+;; @002b brif v89, block5(v45, v69, v5), block6(v94, v95, v97)
+;;
+;; block5(v98: i64, v99: i64, v100: i32):
+;; @002b v103 = load.i32 user2 little region5 v99
+;; @002b store user2 little region5 v103, v98
+;; v178 = iconst.i64 4
+;; v179 = iadd v99, v178 ; v178 = 4
+;; @002b v110 = icmp eq v179, v95
+;; v180 = iadd v98, v178 ; v178 = 4
+;; v181 = iconst.i32 1
+;; v182 = iadd v100, v181 ; v181 = 1
+;; @002b brif v110, block7, block5(v180, v179, v182)
+;;
+;; block6(v111: i64, v112: i64, v113: i32):
+;; v173 = iconst.i64 4
+;; v174 = isub v112, v173 ; v173 = 4
+;; @002b v122 = load.i32 user2 little region5 v174
+;; v175 = isub v111, v173 ; v173 = 4
+;; @002b store user2 little region5 v122, v175
+;; @002b v123 = icmp eq v174, v69
+;; v176 = iconst.i32 1
+;; v177 = isub v113, v176 ; v176 = 1
+;; @002b brif v123, block7, block6(v175, v174, v177)
;;
-;; block7(v111: i64, v112: i64, v113: i32):
-;; @002b v116 = load.i32 user2 little region5 v112
-;; @002b store user2 little region5 v116, v111
-;; v180 = iconst.i64 4
-;; v181 = iadd v112, v180 ; v180 = 4
-;; @002b v123 = icmp eq v181, v108
-;; v182 = iadd v111, v180 ; v180 = 4
-;; v183 = iconst.i32 1
-;; v184 = iadd v113, v183 ; v183 = 1
-;; @002b brif v123, block9, block7(v182, v181, v184)
+;; block7:
+;; @002b v128 = iconst.i64 6
+;; @002b v129 = iadd.i64 v125, v128 ; v128 = 6
+;; @002b v133 = iadd v129, v31
+;; v183 = iconst.i64 0
+;; v184 = icmp sge v133, v183 ; v183 = 0
+;; @002b brif v184, block8, block9(v133)
;;
-;; block8(v124: i64, v125: i64, v126: i32):
-;; v175 = iconst.i64 4
-;; v176 = isub v125, v175 ; v175 = 4
-;; @002b v135 = load.i32 user2 little region5 v176
-;; v177 = isub v124, v175 ; v175 = 4
-;; @002b store user2 little region5 v135, v177
-;; @002b v136 = icmp eq v176, v82
-;; v178 = iconst.i32 1
-;; v179 = isub v126, v178 ; v178 = 1
-;; @002b brif v136, block9, block8(v177, v176, v179)
+;; block8:
+;; @002b store.i64 notrap aligned region2 v133, v7
+;; @002b v137 = call fn0(v0)
+;; @002b v139 = load.i64 notrap aligned region2 v7
+;; @002b jump block9(v139)
;;
-;; block9:
+;; block9(v141: i64):
;; @002f jump block1
;;
;; block1:
-;; @002f store.i64 notrap aligned region2 v139, v7
+;; @002f store.i64 notrap aligned region2 v141, v7
;; @002f return
;; }
diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.wat
index d6113471a1..81e78d5fbf 100644
--- a/tests/disas/memory-copy-epochs.wat
+++ b/tests/disas/memory-copy-epochs.wat
@@ -37,35 +37,35 @@
;; @001e v10 = call fn0(v0)
;; @001e jump block2(v10)
;;
-;; block2(v16: i64):
-;; @0025 v15 = load.i64 notrap aligned region3 v5
-;; @0025 v17 = icmp uge v15, v16
-;; @0025 brif v17, block5, block4
+;; block2(v41: i64):
+;; @0025 v14 = load.i64 notrap aligned region6 v0+64
+;; @0025 v15 = uextend.i64 v2
+;; @0025 v16 = uextend.i64 v4
+;; @0025 v19 = iadd v15, v16
+;; @0025 v20 = icmp ugt v19, v14
+;; @0025 trapnz v20, heap_oob
+;; @0025 v27 = uextend.i64 v3
+;; @0025 v31 = iadd v27, v16
+;; @0025 v32 = icmp ugt v31, v14
+;; @0025 trapnz v32, heap_oob
+;; @0025 v21 = load.i64 notrap aligned readonly can_move region5 v0+56
+;; @0025 v25 = iadd v21, v15
+;; @0025 v37 = iadd v21, v27
+;; @0025 call fn1(v0, v25, v37, v16)
+;; @0025 v40 = load.i64 notrap aligned region3 v5
+;; @0025 v42 = icmp uge v40, v41
+;; @0025 brif v42, block5, block4
;;
;; block5 cold:
-;; @0025 v19 = load.i64 notrap aligned region4 v7+8
-;; @0025 v20 = icmp.i64 uge v15, v19
-;; @0025 brif v20, block6, block4
+;; @0025 v44 = load.i64 notrap aligned region4 v7+8
+;; @0025 v45 = icmp.i64 uge v40, v44
+;; @0025 brif v45, block6, block4
;;
;; block6 cold:
-;; @0025 v21 = call fn0(v0)
+;; @0025 v46 = call fn0(v0)
;; @0025 jump block4
;;
;; block4:
-;; @0025 v22 = load.i64 notrap aligned region6 v0+64
-;; @0025 v23 = uextend.i64 v2
-;; @0025 v24 = uextend.i64 v4
-;; @0025 v27 = iadd v23, v24
-;; @0025 v28 = icmp ugt v27, v22
-;; @0025 trapnz v28, heap_oob
-;; @0025 v35 = uextend.i64 v3
-;; @0025 v39 = iadd v35, v24
-;; @0025 v40 = icmp ugt v39, v22
-;; @0025 trapnz v40, heap_oob
-;; @0025 v29 = load.i64 notrap aligned readonly can_move region5 v0+56
-;; @0025 v33 = iadd v29, v23
-;; @0025 v45 = iadd v29, v35
-;; @0025 call fn1(v0, v33, v45, v24)
;; @0029 jump block1
;;
;; block1:
diff --git a/tests/disas/memory-copy-fuel-const-len.wat b/tests/disas/memory-copy-fuel-const-len.wat
index aee7af900c..c8cea81fb2 100644
--- a/tests/disas/memory-copy-fuel-const-len.wat
+++ b/tests/disas/memory-copy-fuel-const-len.wat
@@ -151,31 +151,31 @@
;; @003d v12 = load.i64 notrap aligned region2 v3
;; @003d jump block3(v12)
;;
-;; block3(v16: i64):
-;; @0045 v17 = iconst.i64 4100
-;; @0045 v18 = iadd v16, v17 ; v17 = 4100
+;; block3(v29: i64):
+;; @0045 v16 = load.i64 notrap aligned region4 v0+64
+;; @0045 v17 = uextend.i64 v2
+;; v40 = iconst.i64 4096
+;; @0045 v21 = iadd v17, v40 ; v40 = 4096
+;; @0045 v22 = icmp ugt v21, v16
+;; @0045 trapnz v22, heap_oob
+;; @0045 v23 = load.i64 notrap aligned readonly can_move region3 v0+56
+;; @0045 v27 = iadd v23, v17
+;; @0040 v14 = iconst.i32 0
+;; @0045 call fn1(v0, v27, v14, v40) ; v14 = 0, v40 = 4096
+;; @0045 v30 = iconst.i64 4100
+;; @0045 v31 = iadd v29, v30 ; v30 = 4100
;; v51 = iconst.i64 0
-;; v52 = icmp sge v18, v51 ; v51 = 0
-;; @0045 brif v52, block4, block5(v18)
+;; v52 = icmp sge v31, v51 ; v51 = 0
+;; @0045 brif v52, block4, block5(v31)
;;
;; block4:
-;; v53 = iadd.i64 v16, v17 ; v17 = 4100
+;; v53 = iadd.i64 v29, v30 ; v30 = 4100
;; @0045 store notrap aligned region2 v53, v3
-;; @0045 v22 = call fn0(v0)
-;; @0045 v24 = load.i64 notrap aligned region2 v3
-;; @0045 jump block5(v24)
+;; @0045 v35 = call fn0(v0)
+;; @0045 v37 = load.i64 notrap aligned region2 v3
+;; @0045 jump block5(v37)
;;
;; block5(v39: i64):
-;; @0045 v25 = load.i64 notrap aligned region4 v0+64
-;; @0045 v26 = uextend.i64 v2
-;; v40 = iconst.i64 4096
-;; @0045 v30 = iadd v26, v40 ; v40 = 4096
-;; @0045 v31 = icmp ugt v30, v25
-;; @0045 trapnz v31, heap_oob
-;; @0045 v32 = load.i64 notrap aligned readonly can_move region3 v0+56
-;; @0045 v36 = iadd v32, v26
-;; @0040 v14 = iconst.i32 0
-;; @0045 call fn1(v0, v36, v14, v40) ; v14 = 0, v40 = 4096
;; @0048 jump block1
;;
;; block1:
diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.wat
index 29104ad297..624d063e22 100644
--- a/tests/disas/memory-copy-fuel.wat
+++ b/tests/disas/memory-copy-fuel.wat
@@ -39,35 +39,35 @@
;; @001e v14 = load.i64 notrap aligned region2 v5
;; @001e jump block3(v14)
;;
-;; block3(v21: i64):
-;; @0025 v22 = iconst.i64 4
-;; @0025 v23 = iadd v21, v22 ; v22 = 4
-;; @0025 v18 = uextend.i64 v4
-;; @0025 v24 = iadd v23, v18
+;; block3(v43: i64):
+;; @0025 v18 = load.i64 notrap aligned region4 v0+64
+;; @0025 v19 = uextend.i64 v2
+;; @0025 v20 = uextend.i64 v4
+;; @0025 v23 = iadd v19, v20
+;; @0025 v24 = icmp ugt v23, v18
+;; @0025 trapnz v24, heap_oob
+;; @0025 v31 = uextend.i64 v3
+;; @0025 v35 = iadd v31, v20
+;; @0025 v36 = icmp ugt v35, v18
+;; @0025 trapnz v36, heap_oob
+;; @0025 v25 = load.i64 notrap aligned readonly can_move region3 v0+56
+;; @0025 v29 = iadd v25, v19
+;; @0025 v41 = iadd v25, v31
+;; @0025 call fn1(v0, v29, v41, v20)
+;; @0025 v44 = iconst.i64 4
+;; @0025 v45 = iadd v43, v44 ; v44 = 4
+;; @0025 v49 = iadd v45, v20
;; v62 = iconst.i64 0
-;; v63 = icmp sge v24, v62 ; v62 = 0
-;; @0025 brif v63, block4, block5(v24)
+;; v63 = icmp sge v49, v62 ; v62 = 0
+;; @0025 brif v63, block4, block5(v49)
;;
;; block4:
-;; @0025 store.i64 notrap aligned region2 v24, v5
-;; @0025 v28 = call fn0(v0)
-;; @0025 v30 = load.i64 notrap aligned region2 v5
-;; @0025 jump block5(v30)
+;; @0025 store.i64 notrap aligned region2 v49, v5
+;; @0025 v53 = call fn0(v0)
+;; @0025 v55 = load.i64 notrap aligned region2 v5
+;; @0025 jump block5(v55)
;;
;; block5(v57: i64):
-;; @0025 v31 = load.i64 notrap aligned region4 v0+64
-;; @0025 v32 = uextend.i64 v2
-;; @0025 v36 = iadd v32, v18
-;; @0025 v37 = icmp ugt v36, v31
-;; @0025 trapnz v37, heap_oob
-;; @0025 v44 = uextend.i64 v3
-;; @0025 v48 = iadd v44, v18
-;; @0025 v49 = icmp ugt v48, v31
-;; @0025 trapnz v49, heap_oob
-;; @0025 v38 = load.i64 notrap aligned readonly can_move region3 v0+56
-;; @0025 v42 = iadd v38, v32
-;; @0025 v54 = iadd v38, v44
-;; @0025 call fn1(v0, v42, v54, v18)
;; @0029 jump block1
;;
;; block1:
diff --git a/tests/misc_testsuite/memory64/table64-grow-huge-delta.wast b/tests/misc_testsuite/memory64/table64-grow-huge-delta.wast
new file mode 100644
index 0000000000..66887f622b
--- /dev/null
+++ b/tests/misc_testsuite/memory64/table64-grow-huge-delta.wast
@@ -0,0 +1,15 @@
+;;! memory64 = true
+;;! reference_types = true
+
+;; Verify that table.grow returns -1 even if the delta overflows the host
+;; pointer width.
+(module
+ (table $t i64 0 0x10000 funcref)
+ (func (export "grow") (param $delta i64) (result i64)
+ (table.grow $t (ref.null func) (local.get $delta))))
+
+;; Delta just past 2**32: fits in a 64-bit `usize` but not a 32-bit one.
+(assert_return (invoke "grow" (i64.const 0x1_0000_0000)) (i64.const -1))
+
+;; The maximum possible 2**64-1 delta.
+(assert_return (invoke "grow" (i64.const -1)) (i64.const -1))