A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
`fuel`: Defer fuel consumption for bulk operations (#14213)
Adam Bratschi-Kaye4ca0d0d060cd
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 9ecc2964a6..a617dc06c1 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -2607,7 +2607,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_grow_per_element;- self.pre_translate_bulk_op(builder, delta, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, delta, cost);let mut pos = builder.cursor();let table = self.table(table_index);@@ -2638,12 +2638,10 @@ impl FuncEnvironment<'_> {// Conditionally call that on growth success, and otherwise fall through// to continue to yield -1 for this growth operation.let current_block = builder.current_block().unwrap();- let failed_block = builder.create_block();let fill_block = builder.create_block();let done_block = builder.create_block();- builder.insert_block_after(failed_block, current_block);- builder.insert_block_after(fill_block, failed_block);+ builder.insert_block_after(fill_block, current_block);builder.insert_block_after(done_block, fill_block);// Commit the operator's flat cost before branching so translating the@@ -2653,14 +2651,7 @@ impl FuncEnvironment<'_> {}let failure = builder.ins().iconst(index_type_to_ir_type(index_type), -1);let failed = builder.ins().icmp(IntCC::Equal, result_idx, failure);- builder- .ins()- .brif(failed, failed_block, &[], fill_block, &[]);-- // A failed attempt performs no initialization loop, but still charge- // for the requested growth so repeated failures are not free.- builder.switch_to_block(failed_block);- builder.ins().jump(done_block, &[]);+ builder.ins().brif(failed, done_block, &[], fill_block, &[]);builder.switch_to_block(fill_block);self.translate_entity_fill(@@ -2673,11 +2664,12 @@ impl FuncEnvironment<'_> {init_value,delta,)?;+ self.post_translate_bulk_op(builder, fuel)?;+builder.ins().jump(done_block, &[]);builder.switch_to_block(done_block);- builder.seal_block(failed_block);builder.seal_block(fill_block);builder.seal_block(done_block);@@ -2825,7 +2817,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_fill_per_element;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_fill(builder,CheckedEntity::Table {@@ -2835,7 +2827,8 @@ impl FuncEnvironment<'_> {dst,val,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}pub fn translate_ref_i31(@@ -3063,7 +3056,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().array_copy_per_element;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_copy(builder,CheckedEntity::Array {@@ -3079,7 +3072,8 @@ impl FuncEnvironment<'_> {dst_index,src_index,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}pub fn translate_array_fill(@@ -3097,7 +3091,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().array_fill_per_element;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_fill(builder,CheckedEntity::Array {@@ -3108,7 +3102,8 @@ impl FuncEnvironment<'_> {index,value,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}pub fn translate_array_init_data(@@ -3128,7 +3123,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().array_init_data_per_element;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_copy(builder,CheckedEntity::Array {@@ -3143,7 +3138,8 @@ impl FuncEnvironment<'_> {dst_index,data_offset,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}pub fn translate_array_init_elem(@@ -3162,7 +3158,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().array_init_elem_per_element;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_copy(builder,CheckedEntity::Array {@@ -3174,7 +3170,8 @@ impl FuncEnvironment<'_> {dst_index,elem_offset,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}pub fn translate_array_len(@@ -3563,7 +3560,7 @@ impl FuncEnvironment<'_> {let index_type = self.memory(index).idx_type;let cost = self.tunables.operator_cost.variable().memory_grow_per_page;- self.pre_translate_bulk_op(builder, val, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, val, cost);let mut pos = builder.cursor();let val = self.cast_index_to_i64(&mut pos, val, index_type);let call_inst = pos@@ -3575,12 +3572,42 @@ impl FuncEnvironment<'_> {0 => true,_ => unreachable!("only page sizes 2**0 and 2**16 are currently valid"),};- Ok(self.convert_pointer_to_index_type(+ let grow_result = self.convert_pointer_to_index_type(builder.cursor(),result,index_type,single_byte_pages,- ))+ );++ // Consume fuel and do a fuel/epoch check only when the grow actually+ // succeeded.+ if fuel.is_some() {+ let current_block = builder.current_block().unwrap();+ let success_block = builder.create_block();+ let done_block = builder.create_block();+ builder.insert_block_after(success_block, current_block);+ builder.insert_block_after(done_block, success_block);++ // Flush outstanding fuel before branching.+ if self.tunables.consume_fuel {+ self.fuel_increment_var(builder);+ }+ let failure = builder.ins().iconst(index_type_to_ir_type(index_type), -1);+ let failed = builder.ins().icmp(IntCC::Equal, grow_result, failure);+ builder+ .ins()+ .brif(failed, done_block, &[], success_block, &[]);++ builder.switch_to_block(success_block);+ self.post_translate_bulk_op(builder, fuel)?;+ builder.ins().jump(done_block, &[]);++ builder.switch_to_block(done_block);+ builder.seal_block(success_block);+ builder.seal_block(done_block);+ }++ Ok(grow_result)}/// Loads the size, in bytes, of the memory `index` specified.@@ -3668,8 +3695,9 @@ impl FuncEnvironment<'_> {len: ir::Value,) -> WasmResult<()> {let cost = self.tunables.operator_cost.variable().memory_copy_per_byte;- self.pre_translate_bulk_op(builder, len, cost)?;- self.translate_entity_copy(builder, dst_index, src_index, dst, src, len)+ let fuel = self.pre_translate_bulk_op(builder, len, cost);+ self.translate_entity_copy(builder, dst_index, src_index, dst, src, len)?;+ self.post_translate_bulk_op(builder, fuel)}/// Perform a raw bulk-memory-like libcall.@@ -3996,8 +4024,9 @@ impl FuncEnvironment<'_> {len: ir::Value,) -> WasmResult<()> {let cost = self.tunables.operator_cost.variable().memory_fill_per_byte;- self.pre_translate_bulk_op(builder, len, cost)?;- self.translate_entity_fill(builder, memory_index, dst, val, len)+ let fuel = self.pre_translate_bulk_op(builder, len, cost);+ self.translate_entity_fill(builder, memory_index, dst, val, len)?;+ self.post_translate_bulk_op(builder, fuel)}pub fn translate_memory_init(@@ -4011,7 +4040,7 @@ impl FuncEnvironment<'_> {) -> WasmResult<()> {let seg_index = DataIndex::from_u32(seg_index);let cost = self.tunables.operator_cost.variable().memory_init_per_byte;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_copy(builder,memory_index,@@ -4022,7 +4051,8 @@ impl FuncEnvironment<'_> {dst,src,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}pub fn translate_data_drop(&mut self, mut pos: FuncCursor, seg_index: u32) -> WasmResult<()> {@@ -4369,7 +4399,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_copy_per_element;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_copy(builder,CheckedEntity::Table {@@ -4383,7 +4413,8 @@ impl FuncEnvironment<'_> {dst,src,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}/// Emits a copy between two WebAssembly table or array entities.@@ -4940,7 +4971,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_init_per_element;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_copy(builder,CheckedEntity::Table {@@ -4951,7 +4982,8 @@ impl FuncEnvironment<'_> {dst,src,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}pub fn translate_elem_drop(&mut self, mut pos: FuncCursor, elem_index: u32) -> WasmResult<()> {@@ -5016,41 +5048,102 @@ impl FuncEnvironment<'_> {/// Translation prefix before bulk operations such as `memory.copy`.///- /// Takes a dynamic value `units` for the size of the operation as well as- /// a `cost_per_unit` configured for this operation. If fuel is enabled- /// this fuel will be consumed, and if epochs are enabled then an epoch- /// check happens. If neither epochs nor fuel are enabled this is a noop.+ /// Takes a dynamic value `units` for the size of the operation as well as a+ /// `cost_per_unit` configured for this operation.+ ///+ /// If fuel or epochs are enabled, a return value of `Some` indicates that a+ /// fuel/epoch check should be performed if the operation succeeded. If+ /// fuel is enabled, the return value also indicates how much fuel should be+ /// consumed if the operation succeeds. The return value should be used by+ /// `post_translate_bulk_op` to consume the fuel _after_ the operation has+ /// succeeded to prevent turning OOB traps or grow failures into out-of-fuel+ /// traps.+ ///+ /// For constant small operations the fuel is consumed here (if needed), and+ /// the returned value is `None`.+ ///+ /// If neither fuel nor epochs are enabled this is a noop.fn pre_translate_bulk_op(&mut self,builder: &mut FunctionBuilder,units: ir::Value,cost_per_unit: u8,- ) -> WasmResult<()> {+ ) -> Option<DeferredBulkOp> {let const_units =Self::value_as_const_int(builder, units).map(|c| i64::try_from(c).unwrap_or(i64::MAX));- if self.tunables.consume_fuel && cost_per_unit > 0 {- match const_units {- // Fold constant costs directly into internal state.- Some(units) => {+ // Skip explicit fuel/epoch checks for operations which are+ // subjectively, and statically, considered cheap and consume the fuel+ // now instead of waiting to see if the operation succeeds.+ const SMALL_BULK_OP_COST: i64 = 128;+ if let Some(units) = const_units+ && let Some(cost) = units.checked_mul(i64::from(cost_per_unit))+ && cost <= SMALL_BULK_OP_COST+ {+ if self.tunables.consume_fuel && cost_per_unit > 0 {+ self.fuel_consumed = self.fuel_consumed.saturating_add(cost);+ }+ return None;+ }++ if (self.tunables.consume_fuel || self.tunables.epoch_interruption) && cost_per_unit > 0 {+ Some(DeferredBulkOp {+ units: match const_units {+ Some(const_units) => DeferredBulkUnits::Const(const_units),+ None => DeferredBulkUnits::Runtime(units),+ },+ cost_per_unit,+ })+ } else {+ None+ }+ }++ /// Emitted after a bulk operation has completed successfully.+ ///+ /// First consumes the size-proportional fuel deferred by+ /// [`Self::pre_translate_bulk_op`] and then performs the fuel/epoch check+ /// that was likewise deferred from before the operation, retroactively+ /// discovering whether the operation exhausted the fuel budget.+ ///+ /// Note: the fuel charge is emitted as runtime code if the units are+ /// dynamic, but for constant units only `self.fuel_consumed` is updated.+ /// The trailing fuel/epoch check flushes `self.fuel_consumed` before it+ /// runs, so on return `self.fuel_consumed` is zero.+ fn post_translate_bulk_op(+ &mut self,+ builder: &mut FunctionBuilder,+ deferred: Option<DeferredBulkOp>,+ ) -> WasmResult<()> {+ // Charge the deferred size-proportional fuel now that the operation is+ // known to have succeeded.+ let Some(DeferredBulkOp {+ units,+ cost_per_unit,+ }) = deferred+ else {+ return Ok(());+ };+ debug_assert!(+ (self.tunables.consume_fuel || self.tunables.epoch_interruption) && cost_per_unit > 0+ );+ if self.tunables.consume_fuel {+ match units {+ DeferredBulkUnits::Const(units) => {self.fuel_consumed = self.fuel_consumed.saturating_add(units.saturating_mul(i64::from(cost_per_unit)))}-- None => {- // Note that fuel is always a 64-bit counter.- //- // Also note that the cost is clamped to `i64::MAX` to- // prevent fuel counter overflows since `cost` is otherwise- // an untrusted value.- let units_clamped64 = match builder.func.dfg.value_type(units) {+ DeferredBulkUnits::Runtime(units) => {+ self.fuel_increment_var(builder);+ let fuel_var = builder.use_var(self.fuel_var);+ let variable = match builder.func.dfg.value_type(units) {ir::types::I32 => {let units64 = builder.ins().uextend(ir::types::I64, units);builder.ins().imul_imm_u(units64, i64::from(cost_per_unit))}ir::types::I64 => {- let fuel = builder.ins().imul_imm_u(units, i64::from(cost_per_unit));+ let product = builder.ins().imul_imm_u(units, i64::from(cost_per_unit));let max = builder.ins().iconst(ir::types::I64, i64::MAX);let max_units = builder.ins()@@ -5059,34 +5152,25 @@ impl FuncEnvironment<'_> {builder.ins().icmp(IntCC::UnsignedGreaterThan, units, max_units);- builder.ins().select(saturate, max, fuel)+ builder.ins().select(saturate, max, product)}_ => unreachable!(),};- self.fuel_increment_var(builder);- let fuel = builder.use_var(self.fuel_var);- let fuel = builder.ins().iadd(fuel, units_clamped64);- builder.def_var(self.fuel_var, fuel);+ let updated = builder.ins().iadd(fuel_var, variable);+ builder.def_var(self.fuel_var, updated);}}}- // Skip explicit fuel/epoch checks for operations which are- // subjectively, and statically, considered cheap.- const SMALL_BULK_OP_COST: i64 = 128;- if let Some(units) = const_units- && let Some(cost) = units.checked_mul(i64::from(cost_per_unit))- && cost <= SMALL_BULK_OP_COST- {- return Ok(());- }+ // Perform the fuel/epoch check that was deferred from before the+ // operation. This isn't a loop header but for fuel/epoch purposes it's+ // the same thing.+ self.translate_loop_header(builder);- // This isn't a loop header but for fuel/epoch purposes it's the same- // thing.- self.translate_loop_header(builder)+ Ok(())}- pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> {+ pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) {// Additionally if enabled check how much fuel we have remaining to see// if we've run out by this point.if self.tunables.consume_fuel {@@ -5098,8 +5182,6 @@ impl FuncEnvironment<'_> {if self.tunables.epoch_interruption {self.epoch_check(builder);}-- Ok(())}pub fn before_translate_operator(@@ -5973,7 +6055,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_fill_per_element;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_fill(builder,CheckedEntity::Table {@@ -5983,7 +6065,8 @@ impl FuncEnvironment<'_> {dst,val,len,- )+ )?;+ self.post_translate_bulk_op(builder, fuel)}/// Executes initialization for an active element segment in a module.@@ -6018,7 +6101,7 @@ impl FuncEnvironment<'_> {.operator_cost.variable().table_init_per_element;- self.pre_translate_bulk_op(builder, segment_len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, segment_len, cost);// Re-use the `table.set` translation for making this a simple function// to define. That re-executes the bounds check which is a bit@@ -6039,7 +6122,7 @@ impl FuncEnvironment<'_> {}}}- Ok(())+ self.post_translate_bulk_op(builder, fuel)}/// Peform initialization of an active data segment in a module.@@ -6091,8 +6174,9 @@ impl FuncEnvironment<'_> {let len = self.load_runtime_data_length(builder, data);let start = builder.ins().iconst(I32, 0);let cost = self.tunables.operator_cost.variable().memory_init_per_byte;- self.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = self.pre_translate_bulk_op(builder, len, cost);self.translate_entity_copy(builder, memory, data, offset, start, len)?;+ self.post_translate_bulk_op(builder, fuel)?;// Finalize control-flow for the `MemorySegmentOffset::Static` case// above.@@ -6244,6 +6328,32 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type {}}+/// Deferred bookkeeping for a "large" bulk operation, produced by+/// [`FuncEnvironment::pre_translate_bulk_op`] and consumed by+/// [`FuncEnvironment::post_translate_bulk_op`] once the operation has completed+/// successfully.+///+/// Its presence means a fuel/epoch check must be emitted _after_ the operation+/// rather than before it. Deferring the check means a bulk op that traps or+/// fails is not retroactively billed as running out of fuel; instead a+/// successful op that exhausts the budget is discovered by the check that+/// immediately follows it. The [`DeferredBulkOpCheck::Fuel`] variant+/// additionally carries the size-proportional fuel to charge on success.+#[must_use = "DeferredBulkOpCheck must be passed to post_translate_bulk_op"]+struct DeferredBulkOp {+ /// The number of units (bytes/elements/pages) operated on.+ units: DeferredBulkUnits,+ /// The per-unit fuel cost.+ cost_per_unit: u8,+}++enum DeferredBulkUnits {+ /// A statically-known unit count, already clamped to `i64`.+ Const(i64),+ /// A runtime unit count held in an `ir::Value`.+ Runtime(ir::Value),+}+/// Operations to [`FuncEnvironment::raw_bulk_memory_operation`].#[derive(Clone)]enum BulkOp {diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rsindex 8987cd9a25..9177b11d59 100644--- a/crates/cranelift/src/func_environ/gc.rs+++ b/crates/cranelift/src/func_environ/gc.rs@@ -808,7 +808,7 @@ pub fn translate_array_new(.operator_cost.variable().array_new_per_element;- func_env.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = func_env.pre_translate_bulk_op(builder, len, cost);let result =gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?;@@ -825,6 +825,7 @@ pub fn translate_array_new(elem,len,)?;+ func_env.post_translate_bulk_op(builder, fuel)?;log::trace!("translate_array_new(..) -> {result:?}");Ok(result)}@@ -841,7 +842,7 @@ pub fn translate_array_new_default(.operator_cost.variable().array_new_default_per_element;- func_env.pre_translate_bulk_op(builder, len, cost)?;+ let fuel = func_env.pre_translate_bulk_op(builder, len, cost);let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index();let array_ty = func_env.types.unwrap_array(interned_ty)?;@@ -861,6 +862,7 @@ pub fn translate_array_new_default(elem,len,)?;+ func_env.post_translate_bulk_op(builder, fuel)?;Ok(result)}@@ -1769,7 +1771,7 @@ pub fn translate_array_new_entity(len: ir::Value,cost_per_unit: u8,) -> WasmResult<ir::Value> {- env.pre_translate_bulk_op(builder, len, cost_per_unit)?;+ let fuel = env.pre_translate_bulk_op(builder, len, cost_per_unit);// Before actually allocating this array first do a bounds-check on the// passive entity itself.@@ -1791,5 +1793,6 @@ pub fn translate_array_new_entity(len,)?;+ env.post_translate_bulk_op(builder, fuel)?;Ok(array)}diff --git a/crates/cranelift/src/translate/code_translator.rs b/crates/cranelift/src/translate/code_translator.rsindex 6a1f0dac2a..e86340ec2e 100644--- a/crates/cranelift/src/translate/code_translator.rs+++ b/crates/cranelift/src/translate/code_translator.rs@@ -285,7 +285,7 @@ pub fn translate_operator(environ.stacks.popn(params.len());builder.switch_to_block(loop_body);push_block_params(environ, builder, loop_body);- environ.translate_loop_header(builder)?;+ environ.translate_loop_header(builder);}Operator::If { blockty } => {// Read the hint before `environ` is borrowed mutably below.diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rsindex 70ea869d64..8e0c6f03ab 100644--- a/crates/environ/src/tunables.rs+++ b/crates/environ/src/tunables.rs@@ -601,8 +601,7 @@ impl VariableOperatorCost {memory_copy_per_byte: 1,memory_fill_per_byte: 1,memory_init_per_byte: 1,- // `memory.grow` did not previously have a dynamic fuel charge.- memory_grow_per_page: 0,+ memory_grow_per_page: 1,table_copy_per_element: 1,table_fill_per_element: 1,table_init_per_element: 1,diff --git a/crates/wasmtime/src/runtime/vm/table.rs b/crates/wasmtime/src/runtime/vm/table.rsindex 011e5a544c..fceb5fc55c 100644--- a/crates/wasmtime/src/runtime/vm/table.rs+++ b/crates/wasmtime/src/runtime/vm/table.rs@@ -632,7 +632,10 @@ impl Table {if delta == 0 {return Ok(Some(old_size));}- let delta = usize::try_from(delta).map_err(|_| Trap::TableOutOfBounds)?;++ // Clamp to maximum usize, the system won't be able to allocate this+ // anyway and we'll go to the normal failure path.+ let delta = usize::try_from(delta).unwrap_or(usize::MAX);let new_size = match old_size.checked_add(delta) {Some(s) => s,diff --git a/tests/all/fuel.rs b/tests/all/fuel.rsindex f6a06a21f0..cee3e8332e 100644--- a/tests/all/fuel.rs+++ b/tests/all/fuel.rs@@ -829,48 +829,6 @@ fn custom_variable_operator_cost(config: &mut Config) -> Result<()> {Ok(())}-#[wasmtime_test(wasm_features(reference_types), strategies(not(Winch)))]-#[cfg_attr(miri, ignore)]-fn variable_operator_cost_failed_growth(config: &mut Config) -> Result<()> {- config.consume_fuel(true);- let mut op_cost = OperatorCost {- I32Const: 0,- RefNull: 0,- MemoryGrow: 0,- TableGrow: 0,- ..Default::default()- };- op_cost.variable.memory_grow_per_page = 7;- op_cost.variable.table_grow_per_element = 19;- config.operator_cost(op_cost.clone());-- let engine = Engine::new(config)?;- let module = Module::new(- &engine,- r#"(module- (memory 1 1)- (table 0 0 funcref)- (func (export "main")- ;; these exceed the max limits so they fail- i32.const 5 memory.grow drop- ref.null func i32.const 5 table.grow drop)- )"#,- )?;- let mut store = Store::new(&engine, ());- store.set_fuel(1_000)?;- let instance = Instance::new(&mut store, &module, &[])?;- let main = instance.get_typed_func::<(), ()>(&mut store, "main")?;-- let initial_fuel = store.get_fuel()?;- main.call(&mut store, ())?;- let cost_of_execution = 5 * u64::from(op_cost.variable.memory_grow_per_page)- + 5 * u64::from(op_cost.variable.table_grow_per_element)- + 1;- assert_eq!(store.get_fuel()?, initial_fuel - cost_of_execution);-- Ok(())-}-#[wasmtime_test(wasm_features(bulk_memory), strategies(not(Winch)))]#[cfg_attr(miri, ignore)]fn variable_operator_cost_follows_bounds_check(config: &mut Config) -> Result<()> {@@ -911,64 +869,6 @@ fn variable_operator_cost_follows_bounds_check(config: &mut Config) -> Result<()Ok(())}-#[wasmtime_test(wasm_features(memory64), strategies(not(Winch)))]-#[cfg_attr(miri, ignore)]-fn memory64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> {- config.consume_fuel(true);- let mut operator_cost = OperatorCost::default();- operator_cost.variable.memory_grow_per_page = 2;- config.operator_cost(operator_cost);-- let engine = Engine::new(config)?;- let module = Module::new(- &engine,- r#"(module- (memory i64 0 0)- (func (export "grow") (param i64) (result i64)- local.get 0 memory.grow)- )"#,- )?;- let mut store = Store::new(&engine, ());- store.set_fuel(10_000)?;- let instance = Instance::new(&mut store, &module, &[])?;- let grow = instance.get_typed_func::<i64, i64>(&mut store, "grow")?;-- // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2.- let error = grow.call(&mut store, i64::MAX).unwrap_err();- assert_eq!(error.downcast::<Trap>().unwrap(), Trap::OutOfFuel);-- Ok(())-}--#[wasmtime_test(wasm_features(memory64, reference_types), strategies(not(Winch)))]-#[cfg_attr(miri, ignore)]-#[cfg(target_pointer_width = "64")]-fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> {- config.consume_fuel(true);- let mut operator_cost = OperatorCost::default();- operator_cost.variable.table_grow_per_element = 2;- config.operator_cost(operator_cost);-- let engine = Engine::new(config)?;- let module = Module::new(- &engine,- r#"(module- (table i64 0 0 funcref)- (func (export "grow") (param i64) (result i64)- ref.null func local.get 0 table.grow)- )"#,- )?;- let mut store = Store::new(&engine, ());- store.set_fuel(10_000)?;- let instance = Instance::new(&mut store, &module, &[])?;- let grow = instance.get_typed_func::<i64, i64>(&mut store, "grow")?;-- // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2.- let error = grow.call(&mut store, i64::MAX).unwrap_err();- assert_eq!(error.downcast::<Trap>().unwrap(), Trap::OutOfFuel);- Ok(())-}-#[test]#[cfg_attr(miri, ignore)]fn huge_table64_grow_cannot_mint_fuel() -> Result<()> {@@ -1040,16 +940,15 @@ fn fuel_around_table_grow() -> Result<()> {store.set_fuel(2)?;let instance = Instance::new(&mut store, &module, &[])?;let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?;+ // The fuel check for a large bulk operation runs after the operation+ // completes so this call still traps with "out of fuel", but the grow+ // itself took effect.let trap = grow.call(&mut store, ()).unwrap_err().downcast::<Trap>()?;assert_eq!(trap, Trap::OutOfFuel);store.set_fuel(u64::MAX)?;let call = instance.get_typed_func::<i32, ()>(&mut store, "call")?;- let trap = call- .call(&mut store, 9999999)- .unwrap_err()- .downcast::<Trap>()?;- assert_eq!(trap, Trap::TableOutOfBounds);+ call.call(&mut store, 9999999)?;Ok(())}diff --git a/tests/all/fuel.wast b/tests/all/fuel.wastindex f918eb94ac..1a3f2af3e3 100644--- a/tests/all/fuel.wast+++ b/tests/all/fuel.wast@@ -207,6 +207,9 @@)(start $f)))+;; A small constant size (size * cost <= 128, the `SMALL_BULK_OP_COST`+;; threshold) charges the size-proportional "variable" fuel up front, before+;; the op runs.(assert_fuel 107(module(memory 1)@@ -263,17 +266,6 @@)(start $f)))-(assert_fuel 106- (module- (table 0 funcref)- (func $f- ref.null func- i32.const 100- table.grow- drop- )- (start $f)))-(assert_fuel 27(module(table 20 funcref)@@ -385,3 +377,493 @@drop)(start $f)))++;; A large constant size (> 128) instead defers that variable fuel until+;; after the op has run.+(assert_fuel 207+ (module+ (memory 1)+ (func $f+ i32.const 0+ i32.const 0+ i32.const 200+ memory.copy+ )+ (start $f)))++(assert_fuel 207+ (module+ (memory 1)+ (func $f+ i32.const 0+ i32.const 0+ i32.const 200+ memory.fill+ )+ (start $f)))++(assert_fuel 207+ (module+ (memory 1)+ (func $f+ i32.const 0+ i32.const 0+ i32.const 200+ memory.init $d+ )+ (start $f)+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))++(assert_fuel 207+ (module+ (table 200 funcref)+ (func $f+ i32.const 0+ i32.const 0+ i32.const 200+ table.copy+ )+ (start $f)))++(assert_fuel 207+ (module+ (table 200 funcref)+ (func $f+ i32.const 0+ ref.null func+ i32.const 200+ table.fill+ )+ (start $f)))++(assert_fuel 207+ (module+ (table 200 funcref)+ (func $f+ i32.const 0+ i32.const 0+ i32.const 200+ table.init $e+ )+ (start $f)+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))++(assert_fuel 411+ (module+ (type $a (array (mut i8)))+ (global $a (ref $a) (array.new_default $a (i32.const 200)))+ (func $f+ global.get $a+ i32.const 0+ global.get $a+ i32.const 0+ i32.const 200+ array.copy $a $a+ )+ (start $f)))++(assert_fuel 410+ (module+ (type $a (array (mut i8)))+ (global $a (ref $a) (array.new_default $a (i32.const 200)))+ (func $f+ global.get $a+ i32.const 0+ i32.const 0+ i32.const 200+ array.fill $a+ )+ (start $f)))++(assert_fuel 206+ (module+ (type $a (array (mut i8)))+ (func $f+ i32.const 0+ i32.const 200+ array.new_data $a $d+ drop+ )+ (start $f)+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))++(assert_fuel 410+ (module+ (type $a (array (mut i8)))+ (global $a (ref $a) (array.new_default $a (i32.const 200)))+ (func $f+ global.get $a+ i32.const 0+ i32.const 0+ i32.const 200+ array.init_data $a $d+ )+ (start $f)+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))++(assert_fuel 206+ (module+ (type $a (array (mut funcref)))+ (func $f+ i32.const 0+ i32.const 200+ array.new_elem $a $e+ drop+ )+ (start $f)+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))++(assert_fuel 410+ (module+ (type $a (array (mut funcref)))+ (global $a (ref $a) (array.new_default $a (i32.const 200)))+ (func $f+ global.get $a+ i32.const 0+ i32.const 0+ i32.const 200+ array.init_elem $a $e+ )+ (start $f)+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))++(assert_fuel 205+ (module+ (type $a (array (mut funcref)))+ (func $f+ i32.const 200+ array.new_default $a+ drop+ )+ (start $f)))++(assert_fuel 206+ (module+ (type $a (array (mut funcref)))+ (func $f+ ref.null func+ i32.const 200+ array.new $a+ drop+ )+ (start $f)))++;; A dynamic (runtime) size likewise defers the variable fuel until after+;; the op has run, whatever its magnitude.+(assert_fuel 57+ (module+ (memory 1)+ (global $n (mut i32) (i32.const 50))+ (func $f+ i32.const 0+ i32.const 0+ global.get $n+ memory.copy+ )+ (start $f)))++(assert_fuel 57+ (module+ (memory 1)+ (global $n (mut i32) (i32.const 50))+ (func $f+ i32.const 0+ i32.const 0+ global.get $n+ memory.fill+ )+ (start $f)))++(assert_fuel 57+ (module+ (memory 1)+ (global $n (mut i32) (i32.const 50))+ (func $f+ i32.const 0+ i32.const 0+ global.get $n+ memory.init $d+ )+ (start $f)+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))++(assert_fuel 57+ (module+ (table 50 funcref)+ (global $n (mut i32) (i32.const 50))+ (func $f+ i32.const 0+ i32.const 0+ global.get $n+ table.copy+ )+ (start $f)))++(assert_fuel 57+ (module+ (table 50 funcref)+ (global $n (mut i32) (i32.const 50))+ (func $f+ i32.const 0+ ref.null func+ global.get $n+ table.fill+ )+ (start $f)))++(assert_fuel 57+ (module+ (table 50 funcref)+ (global $n (mut i32) (i32.const 50))+ (func $f+ i32.const 0+ i32.const 0+ global.get $n+ table.init $e+ )+ (start $f)+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))++(assert_fuel 111+ (module+ (type $a (array (mut i8)))+ (global $a (ref $a) (array.new_default $a (i32.const 50)))+ (global $n (mut i32) (i32.const 50))+ (func $f+ global.get $a+ i32.const 0+ global.get $a+ i32.const 0+ global.get $n+ array.copy $a $a+ )+ (start $f)))++(assert_fuel 110+ (module+ (type $a (array (mut i8)))+ (global $a (ref $a) (array.new_default $a (i32.const 50)))+ (global $n (mut i32) (i32.const 50))+ (func $f+ global.get $a+ i32.const 0+ i32.const 0+ global.get $n+ array.fill $a+ )+ (start $f)))++(assert_fuel 56+ (module+ (type $a (array (mut i8)))+ (global $n (mut i32) (i32.const 50))+ (func $f+ i32.const 0+ global.get $n+ array.new_data $a $d+ drop+ )+ (start $f)+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))++(assert_fuel 110+ (module+ (type $a (array (mut i8)))+ (global $a (ref $a) (array.new_default $a (i32.const 50)))+ (global $n (mut i32) (i32.const 50))+ (func $f+ global.get $a+ i32.const 0+ i32.const 0+ global.get $n+ array.init_data $a $d+ )+ (start $f)+ (data $d "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")))++(assert_fuel 56+ (module+ (type $a (array (mut funcref)))+ (global $n (mut i32) (i32.const 50))+ (func $f+ i32.const 0+ global.get $n+ array.new_elem $a $e+ drop+ )+ (start $f)+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))++(assert_fuel 110+ (module+ (type $a (array (mut funcref)))+ (global $a (ref $a) (array.new_default $a (i32.const 50)))+ (global $n (mut i32) (i32.const 50))+ (func $f+ global.get $a+ i32.const 0+ i32.const 0+ global.get $n+ array.init_elem $a $e+ )+ (start $f)+ (elem $e func $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f $f)))++(assert_fuel 55+ (module+ (type $a (array (mut funcref)))+ (global $n (mut i32) (i32.const 50))+ (func $f+ global.get $n+ array.new_default $a+ drop+ )+ (start $f)))++(assert_fuel 56+ (module+ (type $a (array (mut funcref)))+ (global $n (mut i32) (i32.const 50))+ (func $f+ ref.null func+ global.get $n+ array.new $a+ drop+ )+ (start $f)))++;; Growing a memory or table charges a small constant size's variable fuel up+;; front, but defers a large or dynamic size's variable fuel until the grow has+;; succeeded.+;;+;; The following tests cover the 12 cases of table/memory x small/large/dynamic+;; size x success/failure.+(assert_fuel 106+ (module+ (table 0 funcref)+ (func $f+ ref.null func+ i32.const 100+ table.grow+ drop+ )+ (start $f)))++(assert_fuel 106+ (module+ (table 0 0 funcref)+ (func $f+ ref.null func+ i32.const 100+ table.grow+ drop+ )+ (start $f)))++(assert_fuel 206+ (module+ (table 0 funcref)+ (func $f+ ref.null func+ i32.const 200+ table.grow+ drop+ )+ (start $f)))++(assert_fuel 6+ (module+ (table 0 0 funcref)+ (func $f+ ref.null func+ i32.const 200+ table.grow+ drop+ )+ (start $f)))++(assert_fuel 56+ (module+ (table 0 funcref)+ (global $d (mut i32) (i32.const 50))+ (func $f+ ref.null func+ global.get $d+ table.grow+ drop+ )+ (start $f)))++(assert_fuel 6+ (module+ (table 0 0 funcref)+ (global $d (mut i32) (i32.const 50))+ (func $f+ ref.null func+ global.get $d+ table.grow+ drop+ )+ (start $f)))++(assert_fuel 105+ (module+ (memory 1)+ (func $f+ i32.const 100+ memory.grow+ drop+ )+ (start $f)))++(assert_fuel 105+ (module+ (memory 0 0)+ (func $f+ i32.const 100+ memory.grow+ drop+ )+ (start $f)))++(assert_fuel 205+ (module+ (memory 1)+ (func $f+ i32.const 200+ memory.grow+ drop+ )+ (start $f)))++(assert_fuel 5+ (module+ (memory 0 0)+ (func $f+ i32.const 200+ memory.grow+ drop+ )+ (start $f)))++(assert_fuel 55+ (module+ (memory 1)+ (global $d (mut i32) (i32.const 50))+ (func $f+ global.get $d+ memory.grow+ drop+ )+ (start $f)))++(assert_fuel 5+ (module+ (memory 0 0)+ (global $d (mut i32) (i32.const 50))+ (func $f+ global.get $d+ memory.grow+ drop+ )+ (start $f)))diff --git a/tests/disas/gc/array-copy-with-fuel.wat b/tests/disas/gc/array-copy-with-fuel.watindex 8371c0eed2..63ab28b9a6 100644--- a/tests/disas/gc/array-copy-with-fuel.wat+++ b/tests/disas/gc/array-copy-with-fuel.wat@@ -47,102 +47,102 @@;; @0020 v16 = load.i64 notrap aligned region2 v7;; @0020 jump block3(v16);;-;; block3(v25: i64):-;; @002b v26 = iconst.i64 6-;; @002b v27 = iadd v25, v26 ; v26 = 6-;; @002b v22 = uextend.i64 v6-;; @002b v28 = iadd v27, v22-;; v173 = iconst.i64 0-;; v174 = icmp sge v28, v173 ; v173 = 0-;; @002b brif v174, block4, block5(v28)-;;-;; block4:-;; @002b store.i64 notrap aligned region2 v28, v7-;; @002b v32 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0]-;; @002b v34 = load.i64 notrap aligned region2 v7-;; @002b jump block5(v34)-;;-;; block5(v139: i64):+;; block3(v125: i64):;; v161 = load.i32 notrap aligned region6 v162;; @002b trapz v161, user16-;; @002b v37 = load.i64 notrap aligned readonly can_move region3 v7+32-;; @002b v35 = uextend.i64 v161-;; @002b v38 = iadd v37, v35-;; @002b v39 = iconst.i64 16-;; @002b v40 = iadd v38, v39 ; v39 = 16-;; @002b v41 = load.i32 user2 readonly region5 v40-;; @002b v43 = uextend.i64 v3-;; @002b v47 = iadd v43, v22-;; @002b v42 = uextend.i64 v41-;; @002b v48 = icmp ugt v47, v42-;; @002b trapnz v48, user17+;; @002b v24 = load.i64 notrap aligned readonly can_move region3 v7+32+;; @002b v22 = uextend.i64 v161+;; @002b v25 = iadd v24, v22+;; @002b v26 = iconst.i64 16+;; @002b v27 = iadd v25, v26 ; v26 = 16+;; @002b v28 = load.i32 user2 readonly region5 v27+;; @002b v30 = uextend.i64 v3+;; @002b v31 = uextend.i64 v6+;; @002b v34 = iadd v30, v31+;; @002b v29 = uextend.i64 v28+;; @002b v35 = icmp ugt v34, v29+;; @002b trapnz v35, user17;; v155 = load.i32 notrap aligned region7 v163;; @002b trapz v155, user16-;; @002b v59 = uextend.i64 v155-;; @002b v62 = iadd v37, v59-;; @002b v64 = iadd v62, v39 ; v39 = 16-;; @002b v65 = load.i32 user2 readonly region5 v64-;; @002b v67 = uextend.i64 v5-;; @002b v71 = iadd v67, v22-;; @002b v66 = uextend.i64 v65-;; @002b v72 = icmp ugt v71, v66-;; @002b trapnz v72, user17-;; @002b v91 = load.i64 notrap aligned region4 v7+40-;; @002b v53 = iconst.i64 20-;; @002b v54 = iadd v38, v53 ; v53 = 20+;; @002b v46 = uextend.i64 v155+;; @002b v49 = iadd v24, v46+;; @002b v51 = iadd v49, v26 ; v26 = 16+;; @002b v52 = load.i32 user2 readonly region5 v51+;; @002b v54 = uextend.i64 v5+;; @002b v58 = iadd v54, v31+;; @002b v53 = uextend.i64 v52+;; @002b v59 = icmp ugt v58, v53+;; @002b trapnz v59, user17+;; @002b v78 = load.i64 notrap aligned region4 v7+40+;; @002b v40 = iconst.i64 20+;; @002b v41 = iadd v25, v40 ; v40 = 20;; v165 = iconst.i64 2-;; v166 = ishl v43, v165 ; v165 = 2-;; @002b v58 = iadd v54, v166-;; v170 = ishl.i64 v22, v165 ; v165 = 2-;; @002b v93 = uadd_overflow_trap v58, v170, user2-;; @002b v92 = iadd v37, v91-;; @002b v94 = icmp ugt v93, v92-;; @002b trapnz v94, user2-;; @002b v78 = iadd v62, v53 ; v53 = 20-;; v168 = ishl v67, v165 ; v165 = 2-;; @002b v82 = iadd v78, v168-;; @002b v100 = uadd_overflow_trap v82, v170, user2-;; @002b v101 = icmp ugt v100, v92-;; @002b trapnz v101, user2-;; @002b brif.i32 v6, block6, block9+;; v166 = ishl v30, v165 ; v165 = 2+;; @002b v45 = iadd v41, v166+;; v170 = ishl v31, v165 ; v165 = 2+;; @002b v80 = uadd_overflow_trap v45, v170, user2+;; @002b v79 = iadd v24, v78+;; @002b v81 = icmp ugt v80, v79+;; @002b trapnz v81, user2+;; @002b v65 = iadd v49, v40 ; v40 = 20+;; v168 = ishl v54, v165 ; v165 = 2+;; @002b v69 = iadd v65, v168+;; @002b v87 = uadd_overflow_trap v69, v170, user2+;; @002b v88 = icmp ugt v87, v79+;; @002b trapnz v88, user2+;; @002b brif.i32 v6, block4, block7;;-;; block6:+;; block4:;; v143 = load.i32 notrap aligned region6 v162;; v145 = load.i32 notrap aligned region7 v163-;; @002b v102 = icmp.i64 ult v58, v82-;; @002b v107 = iadd.i64 v58, v170-;; @002b v108 = iadd.i64 v82, v170-;; @002b v110 = iadd.i32 v5, v6-;; @002b v56 = iconst.i64 4-;; @002b v133 = iconst.i32 1-;; @002b brif v102, block7(v58, v82, v5), block8(v107, v108, v110)+;; @002b v89 = icmp.i64 ult v45, v69+;; @002b v94 = iadd.i64 v45, v170+;; @002b v95 = iadd.i64 v69, v170+;; @002b v97 = iadd.i32 v5, v6+;; @002b v43 = iconst.i64 4+;; @002b v120 = iconst.i32 1+;; @002b brif v89, block5(v45, v69, v5), block6(v94, v95, v97)+;;+;; block5(v98: i64, v99: i64, v100: i32):+;; @002b v103 = load.i32 user2 little region5 v99+;; @002b store user2 little region5 v103, v98+;; v178 = iconst.i64 4+;; v179 = iadd v99, v178 ; v178 = 4+;; @002b v110 = icmp eq v179, v95+;; v180 = iadd v98, v178 ; v178 = 4+;; v181 = iconst.i32 1+;; v182 = iadd v100, v181 ; v181 = 1+;; @002b brif v110, block7, block5(v180, v179, v182)+;;+;; block6(v111: i64, v112: i64, v113: i32):+;; v173 = iconst.i64 4+;; v174 = isub v112, v173 ; v173 = 4+;; @002b v122 = load.i32 user2 little region5 v174+;; v175 = isub v111, v173 ; v173 = 4+;; @002b store user2 little region5 v122, v175+;; @002b v123 = icmp eq v174, v69+;; v176 = iconst.i32 1+;; v177 = isub v113, v176 ; v176 = 1+;; @002b brif v123, block7, block6(v175, v174, v177);;-;; block7(v111: i64, v112: i64, v113: i32):-;; @002b v116 = load.i32 user2 little region5 v112-;; @002b store user2 little region5 v116, v111-;; v180 = iconst.i64 4-;; v181 = iadd v112, v180 ; v180 = 4-;; @002b v123 = icmp eq v181, v108-;; v182 = iadd v111, v180 ; v180 = 4-;; v183 = iconst.i32 1-;; v184 = iadd v113, v183 ; v183 = 1-;; @002b brif v123, block9, block7(v182, v181, v184)+;; block7:+;; @002b v128 = iconst.i64 6+;; @002b v129 = iadd.i64 v125, v128 ; v128 = 6+;; @002b v133 = iadd v129, v31+;; v183 = iconst.i64 0+;; v184 = icmp sge v133, v183 ; v183 = 0+;; @002b brif v184, block8, block9(v133);;-;; block8(v124: i64, v125: i64, v126: i32):-;; v175 = iconst.i64 4-;; v176 = isub v125, v175 ; v175 = 4-;; @002b v135 = load.i32 user2 little region5 v176-;; v177 = isub v124, v175 ; v175 = 4-;; @002b store user2 little region5 v135, v177-;; @002b v136 = icmp eq v176, v82-;; v178 = iconst.i32 1-;; v179 = isub v126, v178 ; v178 = 1-;; @002b brif v136, block9, block8(v177, v176, v179)+;; block8:+;; @002b store.i64 notrap aligned region2 v133, v7+;; @002b v137 = call fn0(v0)+;; @002b v139 = load.i64 notrap aligned region2 v7+;; @002b jump block9(v139);;-;; block9:+;; block9(v141: i64):;; @002f jump block1;;;; block1:-;; @002f store.i64 notrap aligned region2 v139, v7+;; @002f store.i64 notrap aligned region2 v141, v7;; @002f return;; }diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.watindex d6113471a1..81e78d5fbf 100644--- a/tests/disas/memory-copy-epochs.wat+++ b/tests/disas/memory-copy-epochs.wat@@ -37,35 +37,35 @@;; @001e v10 = call fn0(v0);; @001e jump block2(v10);;-;; block2(v16: i64):-;; @0025 v15 = load.i64 notrap aligned region3 v5-;; @0025 v17 = icmp uge v15, v16-;; @0025 brif v17, block5, block4+;; block2(v41: i64):+;; @0025 v14 = load.i64 notrap aligned region6 v0+64+;; @0025 v15 = uextend.i64 v2+;; @0025 v16 = uextend.i64 v4+;; @0025 v19 = iadd v15, v16+;; @0025 v20 = icmp ugt v19, v14+;; @0025 trapnz v20, heap_oob+;; @0025 v27 = uextend.i64 v3+;; @0025 v31 = iadd v27, v16+;; @0025 v32 = icmp ugt v31, v14+;; @0025 trapnz v32, heap_oob+;; @0025 v21 = load.i64 notrap aligned readonly can_move region5 v0+56+;; @0025 v25 = iadd v21, v15+;; @0025 v37 = iadd v21, v27+;; @0025 call fn1(v0, v25, v37, v16)+;; @0025 v40 = load.i64 notrap aligned region3 v5+;; @0025 v42 = icmp uge v40, v41+;; @0025 brif v42, block5, block4;;;; block5 cold:-;; @0025 v19 = load.i64 notrap aligned region4 v7+8-;; @0025 v20 = icmp.i64 uge v15, v19-;; @0025 brif v20, block6, block4+;; @0025 v44 = load.i64 notrap aligned region4 v7+8+;; @0025 v45 = icmp.i64 uge v40, v44+;; @0025 brif v45, block6, block4;;;; block6 cold:-;; @0025 v21 = call fn0(v0)+;; @0025 v46 = call fn0(v0);; @0025 jump block4;;;; block4:-;; @0025 v22 = load.i64 notrap aligned region6 v0+64-;; @0025 v23 = uextend.i64 v2-;; @0025 v24 = uextend.i64 v4-;; @0025 v27 = iadd v23, v24-;; @0025 v28 = icmp ugt v27, v22-;; @0025 trapnz v28, heap_oob-;; @0025 v35 = uextend.i64 v3-;; @0025 v39 = iadd v35, v24-;; @0025 v40 = icmp ugt v39, v22-;; @0025 trapnz v40, heap_oob-;; @0025 v29 = load.i64 notrap aligned readonly can_move region5 v0+56-;; @0025 v33 = iadd v29, v23-;; @0025 v45 = iadd v29, v35-;; @0025 call fn1(v0, v33, v45, v24);; @0029 jump block1;;;; block1:diff --git a/tests/disas/memory-copy-fuel-const-len.wat b/tests/disas/memory-copy-fuel-const-len.watindex aee7af900c..c8cea81fb2 100644--- a/tests/disas/memory-copy-fuel-const-len.wat+++ b/tests/disas/memory-copy-fuel-const-len.wat@@ -151,31 +151,31 @@;; @003d v12 = load.i64 notrap aligned region2 v3;; @003d jump block3(v12);;-;; block3(v16: i64):-;; @0045 v17 = iconst.i64 4100-;; @0045 v18 = iadd v16, v17 ; v17 = 4100+;; block3(v29: i64):+;; @0045 v16 = load.i64 notrap aligned region4 v0+64+;; @0045 v17 = uextend.i64 v2+;; v40 = iconst.i64 4096+;; @0045 v21 = iadd v17, v40 ; v40 = 4096+;; @0045 v22 = icmp ugt v21, v16+;; @0045 trapnz v22, heap_oob+;; @0045 v23 = load.i64 notrap aligned readonly can_move region3 v0+56+;; @0045 v27 = iadd v23, v17+;; @0040 v14 = iconst.i32 0+;; @0045 call fn1(v0, v27, v14, v40) ; v14 = 0, v40 = 4096+;; @0045 v30 = iconst.i64 4100+;; @0045 v31 = iadd v29, v30 ; v30 = 4100;; v51 = iconst.i64 0-;; v52 = icmp sge v18, v51 ; v51 = 0-;; @0045 brif v52, block4, block5(v18)+;; v52 = icmp sge v31, v51 ; v51 = 0+;; @0045 brif v52, block4, block5(v31);;;; block4:-;; v53 = iadd.i64 v16, v17 ; v17 = 4100+;; v53 = iadd.i64 v29, v30 ; v30 = 4100;; @0045 store notrap aligned region2 v53, v3-;; @0045 v22 = call fn0(v0)-;; @0045 v24 = load.i64 notrap aligned region2 v3-;; @0045 jump block5(v24)+;; @0045 v35 = call fn0(v0)+;; @0045 v37 = load.i64 notrap aligned region2 v3+;; @0045 jump block5(v37);;;; block5(v39: i64):-;; @0045 v25 = load.i64 notrap aligned region4 v0+64-;; @0045 v26 = uextend.i64 v2-;; v40 = iconst.i64 4096-;; @0045 v30 = iadd v26, v40 ; v40 = 4096-;; @0045 v31 = icmp ugt v30, v25-;; @0045 trapnz v31, heap_oob-;; @0045 v32 = load.i64 notrap aligned readonly can_move region3 v0+56-;; @0045 v36 = iadd v32, v26-;; @0040 v14 = iconst.i32 0-;; @0045 call fn1(v0, v36, v14, v40) ; v14 = 0, v40 = 4096;; @0048 jump block1;;;; block1:diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.watindex 29104ad297..624d063e22 100644--- a/tests/disas/memory-copy-fuel.wat+++ b/tests/disas/memory-copy-fuel.wat@@ -39,35 +39,35 @@;; @001e v14 = load.i64 notrap aligned region2 v5;; @001e jump block3(v14);;-;; block3(v21: i64):-;; @0025 v22 = iconst.i64 4-;; @0025 v23 = iadd v21, v22 ; v22 = 4-;; @0025 v18 = uextend.i64 v4-;; @0025 v24 = iadd v23, v18+;; block3(v43: i64):+;; @0025 v18 = load.i64 notrap aligned region4 v0+64+;; @0025 v19 = uextend.i64 v2+;; @0025 v20 = uextend.i64 v4+;; @0025 v23 = iadd v19, v20+;; @0025 v24 = icmp ugt v23, v18+;; @0025 trapnz v24, heap_oob+;; @0025 v31 = uextend.i64 v3+;; @0025 v35 = iadd v31, v20+;; @0025 v36 = icmp ugt v35, v18+;; @0025 trapnz v36, heap_oob+;; @0025 v25 = load.i64 notrap aligned readonly can_move region3 v0+56+;; @0025 v29 = iadd v25, v19+;; @0025 v41 = iadd v25, v31+;; @0025 call fn1(v0, v29, v41, v20)+;; @0025 v44 = iconst.i64 4+;; @0025 v45 = iadd v43, v44 ; v44 = 4+;; @0025 v49 = iadd v45, v20;; v62 = iconst.i64 0-;; v63 = icmp sge v24, v62 ; v62 = 0-;; @0025 brif v63, block4, block5(v24)+;; v63 = icmp sge v49, v62 ; v62 = 0+;; @0025 brif v63, block4, block5(v49);;;; block4:-;; @0025 store.i64 notrap aligned region2 v24, v5-;; @0025 v28 = call fn0(v0)-;; @0025 v30 = load.i64 notrap aligned region2 v5-;; @0025 jump block5(v30)+;; @0025 store.i64 notrap aligned region2 v49, v5+;; @0025 v53 = call fn0(v0)+;; @0025 v55 = load.i64 notrap aligned region2 v5+;; @0025 jump block5(v55);;;; block5(v57: i64):-;; @0025 v31 = load.i64 notrap aligned region4 v0+64-;; @0025 v32 = uextend.i64 v2-;; @0025 v36 = iadd v32, v18-;; @0025 v37 = icmp ugt v36, v31-;; @0025 trapnz v37, heap_oob-;; @0025 v44 = uextend.i64 v3-;; @0025 v48 = iadd v44, v18-;; @0025 v49 = icmp ugt v48, v31-;; @0025 trapnz v49, heap_oob-;; @0025 v38 = load.i64 notrap aligned readonly can_move region3 v0+56-;; @0025 v42 = iadd v38, v32-;; @0025 v54 = iadd v38, v44-;; @0025 call fn1(v0, v42, v54, v18);; @0029 jump block1;;;; block1:diff --git a/tests/misc_testsuite/memory64/table64-grow-huge-delta.wast b/tests/misc_testsuite/memory64/table64-grow-huge-delta.wastnew file mode 100644index 0000000000..66887f622b--- /dev/null+++ b/tests/misc_testsuite/memory64/table64-grow-huge-delta.wast@@ -0,0 +1,15 @@+;;! memory64 = true+;;! reference_types = true++;; Verify that table.grow returns -1 even if the delta overflows the host+;; pointer width.+(module+ (table $t i64 0 0x10000 funcref)+ (func (export "grow") (param $delta i64) (result i64)+ (table.grow $t (ref.null func) (local.get $delta))))++;; Delta just past 2**32: fits in a 64-bit `usize` but not a 32-bit one.+(assert_return (invoke "grow" (i64.const 0x1_0000_0000)) (i64.const -1))++;; The maximum possible 2**64-1 delta.+(assert_return (invoke "grow" (i64.const -1)) (i64.const -1))