Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Add configurable costs for variable-work operators (#13931) (#13932)

Demilade Sonuga3a85d38bbea4
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index d1a6b753ad..9da84e7e9d 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -678,6 +678,70 @@ impl<'module_environment> FuncEnvironment<'module_environment> {
self.fuel_check(builder);
}
+ /// Consumes `units * cost_per_unit` fuel, saturating the charge at
+ /// `i64::MAX` so that an oversized unsigned operand cannot wrap around and
+ /// add fuel instead.
+ fn consume_variable_fuel(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ units: ir::Value,
+ cost_per_unit: u8,
+ ) {
+ let may_exceed_i64_max = match builder.func.dfg.value_type(units) {
+ I32 => false,
+ I64 => true,
+ ty => unreachable!("unsupported variable fuel unit type: {ty}"),
+ };
+ self.consume_variable_fuel_impl(builder, units, cost_per_unit, may_exceed_i64_max);
+ }
+
+ /// Like [`Self::consume_variable_fuel`], but for a unit count whose product
+ /// with `cost_per_unit` is statically known to fit in an `i64`.
+ fn consume_bounded_variable_fuel(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ units: ir::Value,
+ cost_per_unit: u8,
+ ) {
+ self.consume_variable_fuel_impl(builder, units, cost_per_unit, false);
+ }
+
+ fn consume_variable_fuel_impl(
+ &mut self,
+ builder: &mut FunctionBuilder<'_>,
+ units: ir::Value,
+ cost_per_unit: u8,
+ may_exceed_i64_max: bool,
+ ) {
+ if !self.tunables.consume_fuel || cost_per_unit == 0 {
+ return;
+ }
+
+ let units = match builder.func.dfg.value_type(units) {
+ I32 => builder.ins().uextend(I64, units),
+ I64 => units,
+ ty => unreachable!("unsupported variable fuel unit type: {ty}"),
+ };
+ let fuel = if cost_per_unit == 1 {
+ units
+ } else {
+ builder.ins().imul_imm_s(units, i64::from(cost_per_unit))
+ };
+ let fuel = if may_exceed_i64_max {
+ let max = builder.ins().iconst(I64, i64::MAX);
+ let max_units = builder
+ .ins()
+ .iconst(I64, i64::MAX / i64::from(cost_per_unit));
+ let saturate = builder
+ .ins()
+ .icmp(IntCC::UnsignedGreaterThan, units, max_units);
+ builder.ins().select(saturate, max, fuel)
+ } else {
+ fuel
+ };
+ self.manual_fuel_check(builder, fuel);
+ }
+
fn epoch_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) {
debug_assert!(self.epoch_deadline_var.is_reserved_value());
self.epoch_deadline_var = builder.declare_var(ir::types::I64);
@@ -2626,6 +2690,11 @@ impl FuncEnvironment<'_> {
self.table_vmctx_and_defined_index(&mut pos, table_index);
let index_type = table.idx_type;
let delta64 = self.cast_index_to_i64(&mut pos, delta, index_type);
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .table_grow_per_element;
// Call out to the host to perform the actual growth of the underlying
// table. This will initialize table slots as all null. Afterwards the
@@ -2649,22 +2718,48 @@ impl FuncEnvironment<'_> {
// Conditionally call that on growth success, and otherwise fall through
// to continue to yield -1 for this growth operation.
let current_block = builder.current_block().unwrap();
+ let failed_block = builder.create_block();
let fill_block = builder.create_block();
let done_block = builder.create_block();
- builder.insert_block_after(fill_block, current_block);
+ builder.insert_block_after(failed_block, current_block);
+ builder.insert_block_after(fill_block, failed_block);
builder.insert_block_after(done_block, fill_block);
+ // Commit the operator's flat cost before branching so translating the
+ // failed path cannot consume fuel that also belongs to the success path.
+ if self.tunables.consume_fuel {
+ self.fuel_increment_var(builder);
+ }
let failure = builder.ins().iconst(index_type_to_ir_type(index_type), -1);
let failed = builder.ins().icmp(IntCC::Equal, result_idx, failure);
- builder.ins().brif(failed, done_block, &[], fill_block, &[]);
+ builder
+ .ins()
+ .brif(failed, failed_block, &[], fill_block, &[]);
+
+ // A failed attempt performs no initialization loop, but still charge
+ // for the requested growth so repeated failures are not free.
+ builder.switch_to_block(failed_block);
+ self.consume_variable_fuel(builder, delta, cost);
+ builder.ins().jump(done_block, &[]);
builder.switch_to_block(fill_block);
- self.translate_table_fill(builder, table_index, result_idx, init_value, delta)?;
+ self.translate_entity_fill(
+ builder,
+ CheckedEntity::Table {
+ table: table_index,
+ initialized: true,
+ },
+ result_idx,
+ init_value,
+ delta,
+ cost,
+ )?;
builder.ins().jump(done_block, &[]);
builder.switch_to_block(done_block);
+ builder.seal_block(failed_block);
builder.seal_block(fill_block);
builder.seal_block(done_block);
@@ -2807,6 +2902,11 @@ impl FuncEnvironment<'_> {
val: ir::Value,
len: ir::Value,
) -> WasmResult<()> {
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .table_fill_per_element;
self.translate_entity_fill(
builder,
CheckedEntity::Table {
@@ -2816,6 +2916,7 @@ impl FuncEnvironment<'_> {
dst,
val,
len,
+ cost,
)
}
@@ -2952,7 +3053,8 @@ impl FuncEnvironment<'_> {
elem: ir::Value,
len: ir::Value,
) -> WasmResult<ir::Value> {
- gc::translate_array_new(self, builder, array_type_index, elem, len)
+ let cost = self.tunables.operator_cost.variable().array_new_per_element;
+ gc::translate_array_new(self, builder, array_type_index, elem, len, cost)
}
pub fn translate_array_new_default(
@@ -2961,7 +3063,12 @@ impl FuncEnvironment<'_> {
array_type_index: TypeIndex,
len: ir::Value,
) -> WasmResult<ir::Value> {
- gc::translate_array_new_default(self, builder, array_type_index, len)
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .array_new_default_per_element;
+ gc::translate_array_new_default(self, builder, array_type_index, len, cost)
}
pub fn translate_array_new_fixed(
@@ -2983,6 +3090,11 @@ impl FuncEnvironment<'_> {
) -> WasmResult<ir::Value> {
let interned_type_index = self.module.types[array_type_index].unwrap_module_type_index();
let array_layout = self.array_layout(interned_type_index)?.clone();
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .array_new_data_per_element;
gc::translate_array_new_entity(
self,
builder,
@@ -2993,6 +3105,7 @@ impl FuncEnvironment<'_> {
},
data_offset,
len,
+ cost,
)
}
@@ -3004,6 +3117,11 @@ impl FuncEnvironment<'_> {
elem_offset: ir::Value,
len: ir::Value,
) -> WasmResult<ir::Value> {
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .array_new_elem_per_element;
gc::translate_array_new_entity(
self,
builder,
@@ -3011,6 +3129,7 @@ impl FuncEnvironment<'_> {
CheckedEntity::Elem(elem_index),
elem_offset,
len,
+ cost,
)
}
@@ -3027,6 +3146,11 @@ impl FuncEnvironment<'_> {
) -> WasmResult<()> {
let dst_ty = self.module.types[dst_array_type_index].unwrap_module_type_index();
let src_ty = self.module.types[src_array_type_index].unwrap_module_type_index();
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .array_copy_per_element;
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3042,6 +3166,7 @@ impl FuncEnvironment<'_> {
dst_index,
src_index,
len,
+ cost,
)
}
@@ -3055,6 +3180,11 @@ impl FuncEnvironment<'_> {
len: ir::Value,
) -> WasmResult<()> {
let ty = self.module.types[array_type_index].unwrap_module_type_index();
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .array_fill_per_element;
self.translate_entity_fill(
builder,
CheckedEntity::Array {
@@ -3065,6 +3195,7 @@ impl FuncEnvironment<'_> {
index,
value,
len,
+ cost,
)
}
@@ -3080,6 +3211,11 @@ impl FuncEnvironment<'_> {
) -> WasmResult<()> {
let ty = self.module.types[array_type_index].unwrap_module_type_index();
let array_layout = self.array_layout(ty)?.clone();
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .array_init_data_per_element;
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3094,6 +3230,7 @@ impl FuncEnvironment<'_> {
dst_index,
data_offset,
len,
+ cost,
)
}
@@ -3108,6 +3245,11 @@ impl FuncEnvironment<'_> {
len: ir::Value,
) -> WasmResult<()> {
let ty = self.module.types[array_type_index].unwrap_module_type_index();
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .array_init_elem_per_element;
self.translate_entity_copy(
builder,
CheckedEntity::Array {
@@ -3119,6 +3261,7 @@ impl FuncEnvironment<'_> {
dst_index,
elem_offset,
len,
+ cost,
)
}
@@ -3490,6 +3633,9 @@ impl FuncEnvironment<'_> {
self.memory_vmctx_and_defined_index(&mut pos, index);
let index_type = self.memory(index).idx_type;
+ let cost = self.tunables.operator_cost.variable().memory_grow_per_page;
+ self.consume_variable_fuel(builder, val, cost);
+ let mut pos = builder.cursor();
let val = self.cast_index_to_i64(&mut pos, val, index_type);
let call_inst = pos
.ins()
@@ -3586,7 +3732,8 @@ impl FuncEnvironment<'_> {
src: ir::Value,
len: ir::Value,
) -> WasmResult<()> {
- self.translate_entity_copy(builder, dst_index, src_index, dst, src, len)
+ let cost = self.tunables.operator_cost.variable().memory_copy_per_byte;
+ self.translate_entity_copy(builder, dst_index, src_index, dst, src, len, cost)
}
/// Perform a raw bulk-memory-like libcall.
@@ -3611,12 +3758,15 @@ impl FuncEnvironment<'_> {
const_len: Some(bytes),
src_entity,
dst_entity,
+ fuel,
..
} = op
{
if bytes <= INLINE_COPY_MAX_BYTES {
if self.tunables.consume_fuel {
- self.fuel_consumed += bytes as i64;
+ let units = bytes / u64::from(fuel.bytes_per_unit);
+ self.fuel_consumed +=
+ i64::try_from(units * u64::from(fuel.cost_per_unit)).unwrap();
}
let src_region = self.bulk_copy_alias_region(builder.func, src_entity);
let dst_region = self.bulk_copy_alias_region(builder.func, dst_entity);
@@ -3636,36 +3786,35 @@ impl FuncEnvironment<'_> {
let pointer_type = self.pointer_type();
// Performs a raw call to the actual libcall, as dictated by the
- // provided `op`. This unconditionally inserts epoch/fuel checks for all
- // calls.
+ // provided `op`. This inserts configured epoch/fuel checks before the
+ // call.
let raw_call =
- |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &_| {
+ |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &BulkOp| {
if env.tunables.epoch_interruption {
env.epoch_check(builder);
}
+ let fuel = op.fuel();
+ if env.tunables.consume_fuel && fuel.cost_per_unit != 0 {
+ let byte_len = op.len();
+ debug_assert!(fuel.bytes_per_unit.is_power_of_two());
+ let units = if fuel.bytes_per_unit == 1 {
+ byte_len
+ } else {
+ builder
+ .ins()
+ .ushr_imm_u(byte_len, i64::from(fuel.bytes_per_unit.trailing_zeros()))
+ };
+ // With fuel enabled all calls emitted below are limited to
+ // `UNINTERRUPTABLE_CHUNK_SIZE`, so this multiplication
+ // cannot exceed `i64::MAX` even at the maximum `u8` rate.
+ env.consume_bounded_variable_fuel(builder, units, fuel.cost_per_unit);
+ }
match *op {
BulkOp::MemoryCopy { dst, src, len, .. } => {
- if env.tunables.consume_fuel {
- // Note that fuel is always a 64-bit counter.
- let fuel_consumed = match env.pointer_type() {
- ir::types::I32 => builder.ins().uextend(ir::types::I64, len),
- ir::types::I64 => len,
- _ => unreachable!(),
- };
- env.manual_fuel_check(builder, fuel_consumed);
- }
let memory_copy = env.builtin_functions.memory_copy(&mut builder.func);
builder.ins().call(memory_copy, &[vmctx, dst, src, len]);
}
- BulkOp::MemoryFill { dst, val, len } => {
- if env.tunables.consume_fuel {
- let fuel_consumed = match env.pointer_type() {
- ir::types::I32 => builder.ins().uextend(ir::types::I64, len),
- ir::types::I64 => len,
- _ => unreachable!(),
- };
- env.manual_fuel_check(builder, fuel_consumed);
- }
+ BulkOp::MemoryFill { dst, val, len, .. } => {
let memory_fill = env.builtin_functions.memory_fill(&mut builder.func);
builder.ins().call(memory_fill, &[vmctx, dst, val, len]);
}
@@ -3879,6 +4028,7 @@ impl FuncEnvironment<'_> {
dst: ir::Value,
val: ir::Value,
len: ir::Value,
+ cost_per_unit: u8,
) -> WasmResult<()> {
let entity = entity.into();
let idx_type = entity.index_type(self);
@@ -3899,11 +4049,22 @@ impl FuncEnvironment<'_> {
dst: raw_dst_addr,
val,
len: len_ptr,
+ fuel: BulkFuel {
+ cost_per_unit,
+ bytes_per_unit: 1,
+ },
},
);
}
CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => {
- self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr)?;
+ self.emit_raw_array_or_table_fill(
+ builder,
+ entity,
+ raw_dst_addr,
+ val,
+ len_ptr,
+ cost_per_unit,
+ )?;
}
// Not allowed to be written to in wasm.
CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {
@@ -3935,6 +4096,7 @@ impl FuncEnvironment<'_> {
dst_elem_addr: ir::Value,
value: ir::Value,
copy_len: ir::Value,
+ cost_per_element: u8,
) -> WasmResult<()> {
let pointer_ty = self.pointer_type();
@@ -3960,6 +4122,10 @@ impl FuncEnvironment<'_> {
dst: dst_elem_addr,
val: value,
len: copy_byte_len,
+ fuel: BulkFuel {
+ cost_per_unit: cost_per_element,
+ bytes_per_unit: u8::try_from(elem_size).unwrap(),
+ },
},
);
return Ok(());
@@ -4026,9 +4192,9 @@ impl FuncEnvironment<'_> {
// by the element size, then see if we turn again or exit.
builder.switch_to_block(loop_block);
let elem_addr = builder.append_block_param(loop_block, pointer_ty);
- // Consume one unit of fuel per loop iteration.
+ // Consume the configured cost for this element before writing it.
if self.tunables.consume_fuel {
- self.fuel_consumed += 1;
+ self.fuel_consumed += i64::from(cost_per_element);
}
self.translate_loop_header(builder)?;
match entity {
@@ -4145,7 +4311,8 @@ impl FuncEnvironment<'_> {
val: ir::Value,
len: ir::Value,
) -> WasmResult<()> {
- self.translate_entity_fill(builder, memory_index, dst, val, len)
+ let cost = self.tunables.operator_cost.variable().memory_fill_per_byte;
+ self.translate_entity_fill(builder, memory_index, dst, val, len, cost)
}
pub fn translate_memory_init(
@@ -4158,6 +4325,7 @@ impl FuncEnvironment<'_> {
len: ir::Value,
) -> WasmResult<()> {
let seg_index = DataIndex::from_u32(seg_index);
+ let cost = self.tunables.operator_cost.variable().memory_init_per_byte;
self.translate_entity_copy(
builder,
memory_index,
@@ -4168,6 +4336,7 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
+ cost,
)
}
@@ -4218,6 +4387,7 @@ impl FuncEnvironment<'_> {
dst: ir::Value,
src: ir::Value,
len: ir::Value,
+ cost_per_unit: u8,
) -> WasmResult<()> {
let dst_entity = dst_entity.into();
let src_entity = src_entity.into();
@@ -4279,6 +4449,10 @@ impl FuncEnvironment<'_> {
const_len: const_count,
src_entity,
dst_entity,
+ fuel: BulkFuel {
+ cost_per_unit,
+ bytes_per_unit: 1,
+ },
},
);
Ok(())
@@ -4296,6 +4470,7 @@ impl FuncEnvironment<'_> {
len_ptr,
src,
const_count,
+ cost_per_unit,
),
// Cannot copy into a data or element segment in wasm.
@@ -4506,6 +4681,11 @@ impl FuncEnvironment<'_> {
src: ir::Value,
len: ir::Value,
) -> WasmResult<()> {
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .table_copy_per_element;
self.translate_entity_copy(
builder,
CheckedEntity::Table {
@@ -4519,6 +4699,7 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
+ cost,
)
}
@@ -4548,6 +4729,7 @@ impl FuncEnvironment<'_> {
copy_len: ir::Value,
src_index: ir::Value,
const_count: Option<u64>,
+ cost_per_element: u8,
) -> WasmResult<()> {
let pointer_type = self.pointer_type();
assert_eq!(builder.func.dfg.value_type(dst_elem_addr), pointer_type);
@@ -4635,6 +4817,10 @@ impl FuncEnvironment<'_> {
const_len,
src_entity,
dst_entity,
+ fuel: BulkFuel {
+ cost_per_unit: cost_per_element,
+ bytes_per_unit: u8::try_from(dst_element_size).unwrap(),
+ },
},
);
return Ok(());
@@ -4651,6 +4837,7 @@ impl FuncEnvironment<'_> {
src_elem_addr,
copy_len,
src_index,
+ cost_per_element,
&|this, builder, dst, src, src_index| {
let write_ty = dst_entity.storage_type(this);
let val = match src_entity {
@@ -4857,6 +5044,7 @@ impl FuncEnvironment<'_> {
src_elem_addr: ir::Value,
copy_len: ir::Value,
src_index: ir::Value,
+ cost_per_element: u8,
copy_one: &dyn Fn(
&mut Self,
&mut FunctionBuilder<'_>,
@@ -5004,9 +5192,9 @@ impl FuncEnvironment<'_> {
let src_cur = builder.append_block_param(forward_block, self.pointer_type());
let src_index = builder.append_block_param(forward_block, src_index_ty);
let forward_keepalives = append_keepalive_params(builder, forward_block);
- // Consume a single unit of fuel on each iteration of the loop.
+ // Consume the configured cost for this element before copying it.
if self.tunables.consume_fuel {
- self.fuel_consumed += 1;
+ self.fuel_consumed += i64::from(cost_per_element);
}
self.translate_loop_header(builder)?;
copy_one(self, builder, dst_cur, src_cur, src_index)?;
@@ -5035,7 +5223,7 @@ impl FuncEnvironment<'_> {
let src_index = builder.append_block_param(backwards_block, src_index_ty);
let backward_keepalives = append_keepalive_params(builder, backwards_block);
if self.tunables.consume_fuel {
- self.fuel_consumed += 1;
+ self.fuel_consumed += i64::from(cost_per_element);
}
self.translate_loop_header(builder)?;
let dst_cur = {
@@ -5088,6 +5276,11 @@ impl FuncEnvironment<'_> {
src: ir::Value,
len: ir::Value,
) -> WasmResult<()> {
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .table_init_per_element;
self.translate_entity_copy(
builder,
CheckedEntity::Table {
@@ -5098,6 +5291,7 @@ impl FuncEnvironment<'_> {
dst,
src,
len,
+ cost,
)
}
@@ -5996,6 +6190,11 @@ impl FuncEnvironment<'_> {
index_type_to_ir_type(ty.idx_type),
ty.limits.min.cast_signed(),
);
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .table_fill_per_element;
self.translate_entity_fill(
builder,
CheckedEntity::Table {
@@ -6005,6 +6204,7 @@ impl FuncEnvironment<'_> {
dst,
val,
len,
+ cost,
)
}
@@ -6035,6 +6235,12 @@ impl FuncEnvironment<'_> {
offset,
segment_len,
)?;
+ let cost = self
+ .tunables
+ .operator_cost
+ .variable()
+ .table_init_per_element;
+ self.consume_variable_fuel(builder, segment_len, cost);
// Re-use the `table.set` translation for making this a simple function
// to define. That re-executes the bounds check which is a bit
@@ -6106,7 +6312,8 @@ impl FuncEnvironment<'_> {
// Model the initialization here as a `memory.init`.
let len = self.load_runtime_data_length(builder, data);
let start = builder.ins().iconst(I32, 0);
- self.translate_entity_copy(builder, memory, data, offset, start, len)?;
+ let cost = self.tunables.operator_cost.variable().memory_init_per_byte;
+ self.translate_entity_copy(builder, memory, data, offset, start, len, cost)?;
// Finalize control-flow for the `MemorySegmentOffset::Static` case
// above.
@@ -6276,6 +6483,7 @@ enum BulkOp {
const_len: Option<u64>,
src_entity: CheckedEntity,
dst_entity: CheckedEntity,
+ fuel: BulkFuel,
},
/// A `memory.fill` operation, setting all bytes of `dst` to `val`.
@@ -6288,9 +6496,31 @@ enum BulkOp {
dst: ir::Value,
val: ir::Value,
len: ir::Value,
+ fuel: BulkFuel,
},
}
+impl BulkOp {
+ fn len(&self) -> ir::Value {
+ match self {
+ BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => *len,
+ }
+ }
+
+ fn fuel(&self) -> BulkFuel {
+ match self {
+ BulkOp::MemoryCopy { fuel, .. } | BulkOp::MemoryFill { fuel, .. } => *fuel,
+ }
+ }
+}
+
+#[derive(Copy, Clone)]
+struct BulkFuel {
+ cost_per_unit: u8,
+ /// Number of copied bytes represented by one billable unit.
+ bytes_per_unit: u8,
+}
+
/// A list of entities which can participate in various kinds of bulk operations
/// in wasm.
///
diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs
index e52e80a358..ce984ce7ae 100644
--- a/crates/cranelift/src/func_environ/gc.rs
+++ b/crates/cranelift/src/func_environ/gc.rs
@@ -771,6 +771,7 @@ pub fn translate_array_new(
array_type_index: TypeIndex,
elem: ir::Value,
len: ir::Value,
+ cost_per_element: u8,
) -> WasmResult<ir::Value> {
log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})");
let result =
@@ -787,6 +788,7 @@ pub fn translate_array_new(
zero,
elem,
len,
+ cost_per_element,
)?;
log::trace!("translate_array_new(..) -> {result:?}");
Ok(result)
@@ -797,6 +799,7 @@ pub fn translate_array_new_default(
builder: &mut FunctionBuilder,
array_type_index: TypeIndex,
len: ir::Value,
+ cost_per_element: u8,
) -> WasmResult<ir::Value> {
log::trace!("translate_array_new_default({array_type_index:?}, {len:?})");
@@ -817,6 +820,7 @@ pub fn translate_array_new_default(
zero,
elem,
len,
+ cost_per_element,
)?;
Ok(result)
}
@@ -1720,6 +1724,7 @@ pub fn translate_array_new_entity(
entity: CheckedEntity,
entity_offset: ir::Value,
len: ir::Value,
+ cost_per_element: u8,
) -> WasmResult<ir::Value> {
// Before actually allocating this array first do a bounds-check on the
// passive entity itself.
@@ -1739,6 +1744,7 @@ pub fn translate_array_new_entity(
dst,
entity_offset,
len,
+ cost_per_element,
)?;
Ok(array)
diff --git a/crates/environ/src/tunables.rs b/crates/environ/src/tunables.rs
index 37efb27a0c..70ea869d64 100644
--- a/crates/environ/src/tunables.rs
+++ b/crates/environ/src/tunables.rs
@@ -540,6 +540,89 @@ impl OperatorCostStrategy {
OperatorCostStrategy::Default => default_operator_cost(op),
}
}
+
+ /// Get the costs of work whose size is only known at runtime.
+ pub fn variable(&self) -> &VariableOperatorCost {
+ match self {
+ OperatorCostStrategy::Table(cost) => &cost.variable,
+ OperatorCostStrategy::Default => &DEFAULT_VARIABLE_OPERATOR_COST,
+ }
+ }
+}
+
+const DEFAULT_VARIABLE_OPERATOR_COST: VariableOperatorCost = VariableOperatorCost::new();
+
+/// Fuel costs for operators whose work is proportional to a runtime operand.
+///
+/// These costs are charged in addition to the corresponding flat cost in
+/// [`OperatorCost`].
+#[derive(Clone, Hash, Serialize, Deserialize, Debug, PartialEq, Eq)]
+pub struct VariableOperatorCost {
+ /// Cost per byte copied by `memory.copy`.
+ pub memory_copy_per_byte: u8,
+ /// Cost per byte written by `memory.fill`.
+ pub memory_fill_per_byte: u8,
+ /// Cost per byte copied by `memory.init`.
+ pub memory_init_per_byte: u8,
+ /// Cost per page requested by `memory.grow`.
+ pub memory_grow_per_page: u8,
+
+ /// Cost per element copied by `table.copy`.
+ pub table_copy_per_element: u8,
+ /// Cost per element written by `table.fill`.
+ pub table_fill_per_element: u8,
+ /// Cost per element copied by `table.init`.
+ pub table_init_per_element: u8,
+ /// Cost per element requested by `table.grow`.
+ pub table_grow_per_element: u8,
+
+ /// Cost per element copied by `array.copy`.
+ pub array_copy_per_element: u8,
+ /// Cost per element written by `array.fill`.
+ pub array_fill_per_element: u8,
+ /// Cost per element initialized by `array.new_data`.
+ pub array_new_data_per_element: u8,
+ /// Cost per element initialized by `array.init_data`.
+ pub array_init_data_per_element: u8,
+ /// Cost per element initialized by `array.new_elem`.
+ pub array_new_elem_per_element: u8,
+ /// Cost per element initialized by `array.init_elem`.
+ pub array_init_elem_per_element: u8,
+ /// Cost per element initialized by `array.new_default`.
+ pub array_new_default_per_element: u8,
+ /// Cost per element initialized by `array.new`.
+ pub array_new_per_element: u8,
+}
+
+impl VariableOperatorCost {
+ /// Creates the default variable-cost table.
+ pub const fn new() -> Self {
+ Self {
+ memory_copy_per_byte: 1,
+ memory_fill_per_byte: 1,
+ memory_init_per_byte: 1,
+ // `memory.grow` did not previously have a dynamic fuel charge.
+ memory_grow_per_page: 0,
+ table_copy_per_element: 1,
+ table_fill_per_element: 1,
+ table_init_per_element: 1,
+ table_grow_per_element: 1,
+ array_copy_per_element: 1,
+ array_fill_per_element: 1,
+ array_new_data_per_element: 1,
+ array_init_data_per_element: 1,
+ array_new_elem_per_element: 1,
+ array_init_elem_per_element: 1,
+ array_new_default_per_element: 1,
+ array_new_per_element: 1,
+ }
+ }
+}
+
+impl Default for VariableOperatorCost {
+ fn default() -> Self {
+ Self::new()
+ }
}
const fn default_operator_cost(op: &Operator) -> i64 {
@@ -608,6 +691,8 @@ macro_rules! define_operator_cost {
$(
pub $op: u8,
)*
+ /// Costs for work whose size is only known at runtime.
+ pub variable: VariableOperatorCost,
}
impl OperatorCost {
@@ -629,6 +714,7 @@ macro_rules! define_operator_cost {
$(
$op: default_cost!($op),
)*
+ variable: VariableOperatorCost::new(),
}
}
}
diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs
index 0b0cdcc21b..1ceb91a4ac 100644
--- a/crates/wasmtime/src/config.rs
+++ b/crates/wasmtime/src/config.rs
@@ -647,6 +647,10 @@ impl Config {
/// Configures the fuel cost of each WebAssembly operator.
///
+ /// In addition to each operator's flat cost, [`OperatorCost::variable`]
+ /// configures per-byte, per-element, and per-page costs for operators whose
+ /// work depends on a runtime operand.
+ ///
/// This is only relevant when [`Config::consume_fuel`] is enabled.
pub fn operator_cost(&mut self, cost: OperatorCost) -> &mut Self {
self.tunables.operator_cost = Some(OperatorCostStrategy::table(cost));
diff --git a/crates/wasmtime/src/lib.rs b/crates/wasmtime/src/lib.rs
index 175fd37032..f3f0a3fb90 100644
--- a/crates/wasmtime/src/lib.rs
+++ b/crates/wasmtime/src/lib.rs
@@ -509,11 +509,11 @@ mod sync_nostd;
#[cfg(not(feature = "std"))]
use sync_nostd as sync;
-pub use wasmtime_environ::OperatorCost;
pub use wasmtime_environ::ToWasmtimeResult;
#[doc(inline)]
pub use wasmtime_environ::error;
pub use wasmtime_environ::{FuncIndex, StaticModuleIndex};
+pub use wasmtime_environ::{OperatorCost, VariableOperatorCost};
// Only for use in `bindgen!`-generated code.
#[doc(hidden)]
diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs
index 8a9c0d41ef..445a6c1cbd 100644
--- a/tests/all/fuel.rs
+++ b/tests/all/fuel.rs
@@ -693,3 +693,279 @@ fn custom_operator_cost(config: &mut Config) -> Result<()> {
Ok(())
}
+
+#[wasmtime_test(wasm_features(bulk_memory, reference_types, gc, function_references))]
+#[cfg_attr(miri, ignore)]
+fn custom_variable_operator_cost(config: &mut Config) -> Result<()> {
+ config.consume_fuel(true);
+
+ let mut op_cost = OperatorCost {
+ I32Const: 0,
+ I64Const: 0,
+ RefNull: 0,
+ LocalGet: 0,
+ LocalSet: 0,
+ MemoryCopy: 59,
+ MemoryFill: 0,
+ MemoryInit: 0,
+ MemoryGrow: 61,
+ TableCopy: 0,
+ TableFill: 67,
+ TableInit: 0,
+ TableGrow: 0,
+ ArrayCopy: 73,
+ ArrayFill: 0,
+ ArrayNewData: 71,
+ ArrayInitData: 0,
+ ArrayNewElem: 0,
+ ArrayInitElem: 0,
+ ArrayNewDefault: 0,
+ ArrayNew: 0,
+ ..Default::default()
+ };
+ op_cost.variable = VariableOperatorCost {
+ memory_copy_per_byte: 2,
+ memory_fill_per_byte: 3,
+ memory_init_per_byte: 5,
+ memory_grow_per_page: 7,
+ table_copy_per_element: 11,
+ table_fill_per_element: 13,
+ table_init_per_element: 17,
+ table_grow_per_element: 19,
+ array_copy_per_element: 23,
+ array_fill_per_element: 29,
+ array_new_data_per_element: 31,
+ array_init_data_per_element: 37,
+ array_new_elem_per_element: 41,
+ array_init_elem_per_element: 43,
+ array_new_default_per_element: 47,
+ array_new_per_element: 53,
+ };
+ config.operator_cost(op_cost.clone());
+
+ let wasm = wat::parse_str(
+ r#"(module
+ (type $i64_array (array (mut i64)))
+ (type $i32_array (array (mut i32)))
+ (type $ref_array (array (mut funcref)))
+ (memory 1 6)
+ (table 5 10 funcref)
+ (data $data "abcdefghijklmnopqrstuvwxyz")
+ (func $f)
+ (elem $elem func $f $f $f $f $f)
+
+ (func (export "main")
+ (local $i64_values (ref null $i64_array))
+ (local $i32_values (ref null $i32_array))
+ (local $ref_values (ref null $ref_array))
+
+ i32.const 0 i32.const 0 i32.const 5 memory.copy
+ i32.const 0 i32.const 0 i32.const 5 memory.fill
+ i32.const 0 i32.const 0 i32.const 5 memory.init $data
+ i32.const 5 memory.grow drop
+
+ i32.const 0 i32.const 0 i32.const 5 table.copy
+ i32.const 0 ref.null func i32.const 5 table.fill
+ i32.const 0 i32.const 0 i32.const 5 table.init $elem
+ ref.null func i32.const 5 table.grow drop
+
+ i64.const 0 i32.const 5 array.new $i64_array
+ local.set $i64_values
+ i32.const 5 array.new_default $i64_array drop
+ i32.const 0 i32.const 5 array.new_data $i32_array $data
+ local.set $i32_values
+ i32.const 0 i32.const 5 array.new_elem $ref_array $elem
+ local.set $ref_values
+
+ local.get $i64_values i32.const 0
+ local.get $i64_values i32.const 0
+ i32.const 5 array.copy $i64_array $i64_array
+ local.get $i64_values i32.const 0 i64.const 0 i32.const 5
+ array.fill $i64_array
+ local.get $i32_values i32.const 0 i32.const 0 i32.const 5
+ array.init_data $i32_array $data
+ local.get $ref_values i32.const 0 i32.const 0 i32.const 5
+ array.init_elem $ref_array $elem)
+ )"#,
+ )?;
+ let engine = Engine::new(config)?;
+ let module = Module::new(&engine, wasm)?;
+ let mut store = Store::new(&engine, ());
+ store.set_fuel(10_000)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let main = instance.get_typed_func::<(), ()>(&mut store, "main")?;
+ let initial_fuel = store.get_fuel()?;
+ main.call(&mut store, ())?;
+ let consumed = initial_fuel - store.get_fuel()?;
+
+ const UNITS: u64 = 5;
+ let variable = &op_cost.variable;
+ let base_cost = u64::from(op_cost.MemoryCopy)
+ + u64::from(op_cost.MemoryGrow)
+ + u64::from(op_cost.TableFill)
+ + u64::from(op_cost.ArrayCopy)
+ + u64::from(op_cost.ArrayNewData);
+ let cost_of_execution = base_cost
+ + UNITS
+ * (u64::from(variable.memory_copy_per_byte)
+ + u64::from(variable.memory_fill_per_byte)
+ + u64::from(variable.memory_init_per_byte)
+ + u64::from(variable.memory_grow_per_page)
+ + u64::from(variable.table_copy_per_element)
+ + u64::from(variable.table_fill_per_element)
+ + u64::from(variable.table_init_per_element)
+ + u64::from(variable.table_grow_per_element)
+ + u64::from(variable.array_copy_per_element)
+ + u64::from(variable.array_fill_per_element)
+ + u64::from(variable.array_new_data_per_element)
+ + u64::from(variable.array_init_data_per_element)
+ + u64::from(variable.array_new_elem_per_element)
+ + u64::from(variable.array_init_elem_per_element)
+ + u64::from(variable.array_new_default_per_element)
+ + u64::from(variable.array_new_per_element))
+ + 1;
+ assert_eq!(consumed, cost_of_execution);
+
+ Ok(())
+}
+
+#[wasmtime_test(wasm_features(reference_types), strategies(not(Winch)))]
+#[cfg_attr(miri, ignore)]
+fn variable_operator_cost_failed_growth(config: &mut Config) -> Result<()> {
+ config.consume_fuel(true);
+ let mut op_cost = OperatorCost {
+ I32Const: 0,
+ RefNull: 0,
+ MemoryGrow: 0,
+ TableGrow: 0,
+ ..Default::default()
+ };
+ op_cost.variable.memory_grow_per_page = 7;
+ op_cost.variable.table_grow_per_element = 19;
+ config.operator_cost(op_cost.clone());
+
+ let engine = Engine::new(config)?;
+ let module = Module::new(
+ &engine,
+ r#"(module
+ (memory 1 1)
+ (table 0 0 funcref)
+ (func (export "main")
+ ;; these exceed the max limits so they fail
+ i32.const 5 memory.grow drop
+ ref.null func i32.const 5 table.grow drop)
+ )"#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ store.set_fuel(1_000)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let main = instance.get_typed_func::<(), ()>(&mut store, "main")?;
+
+ let initial_fuel = store.get_fuel()?;
+ main.call(&mut store, ())?;
+ let cost_of_execution = 5 * u64::from(op_cost.variable.memory_grow_per_page)
+ + 5 * u64::from(op_cost.variable.table_grow_per_element)
+ + 1;
+ assert_eq!(store.get_fuel()?, initial_fuel - cost_of_execution);
+
+ Ok(())
+}
+
+#[wasmtime_test(wasm_features(bulk_memory), strategies(not(Winch)))]
+#[cfg_attr(miri, ignore)]
+fn variable_operator_cost_follows_bounds_check(config: &mut Config) -> Result<()> {
+ config.consume_fuel(true);
+ let op_cost = OperatorCost {
+ I32Const: 0,
+ MemoryFill: 0,
+ variable: VariableOperatorCost {
+ memory_fill_per_byte: 7,
+ ..Default::default()
+ },
+ ..Default::default()
+ };
+ config.operator_cost(op_cost);
+
+ let engine = Engine::new(config)?;
+ let module = Module::new(
+ &engine,
+ r#"(module
+ (memory 1)
+ (func (export "main")
+ ;; out of bounds fill
+ i32.const 65535 i32.const 0 i32.const 5 memory.fill)
+ )"#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ store.set_fuel(1_000)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let main = instance.get_typed_func::<(), ()>(&mut store, "main")?;
+
+ let initial_fuel = store.get_fuel()?;
+ let error = main.call(&mut store, ()).unwrap_err();
+ assert_eq!(error.downcast::<Trap>().unwrap(), Trap::MemoryOutOfBounds);
+ // The operation traps during bounds validation, before its five-byte
+ // variable charge or the pending function-entry unit is flushed.
+ assert_eq!(store.get_fuel()?, initial_fuel);
+
+ Ok(())
+}
+
+#[wasmtime_test(wasm_features(memory64), strategies(not(Winch)))]
+#[cfg_attr(miri, ignore)]
+fn memory64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> {
+ config.consume_fuel(true);
+ let mut operator_cost = OperatorCost::default();
+ operator_cost.variable.memory_grow_per_page = 2;
+ config.operator_cost(operator_cost);
+
+ let engine = Engine::new(config)?;
+ let module = Module::new(
+ &engine,
+ r#"(module
+ (memory i64 0 0)
+ (func (export "grow") (param i64) (result i64)
+ local.get 0 memory.grow)
+ )"#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ store.set_fuel(10_000)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let grow = instance.get_typed_func::<i64, i64>(&mut store, "grow")?;
+
+ // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2.
+ let error = grow.call(&mut store, i64::MAX).unwrap_err();
+ assert_eq!(error.downcast::<Trap>().unwrap(), Trap::OutOfFuel);
+
+ Ok(())
+}
+
+#[wasmtime_test(wasm_features(memory64, reference_types), strategies(not(Winch)))]
+#[cfg_attr(miri, ignore)]
+#[cfg(target_pointer_width = "64")]
+fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> {
+ config.consume_fuel(true);
+ let mut operator_cost = OperatorCost::default();
+ operator_cost.variable.table_grow_per_element = 2;
+ config.operator_cost(operator_cost);
+
+ let engine = Engine::new(config)?;
+ let module = Module::new(
+ &engine,
+ r#"(module
+ (table i64 0 0 funcref)
+ (func (export "grow") (param i64) (result i64)
+ ref.null func local.get 0 table.grow)
+ )"#,
+ )?;
+ let mut store = Store::new(&engine, ());
+ store.set_fuel(10_000)?;
+ let instance = Instance::new(&mut store, &module, &[])?;
+ let grow = instance.get_typed_func::<i64, i64>(&mut store, "grow")?;
+
+ // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2.
+ let error = grow.call(&mut store, i64::MAX).unwrap_err();
+ assert_eq!(error.downcast::<Trap>().unwrap(), Trap::OutOfFuel);
+
+ Ok(())
+}