A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Intern functions once in array-fill ops (#13524)
Alex Crichton7e56e24991fc
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex 50a7452e9f..64a0977335 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -3832,41 +3832,8 @@ impl FuncEnvironment<'_> {},);}- CheckedEntity::Table { table, initialized } => {- self.emit_raw_array_or_table_fill(- builder,- entity,- raw_dst_addr,- val,- len_ptr,- &|env, builder, addr, value| {- env.emit_table_set(- builder,- table,- addr,- ir::MemFlagsData::trusted(),- value,- initialized,- )- },- )?;- }- CheckedEntity::Array { initialized, .. } => {- let elem_ty = entity.storage_type(self);- self.emit_raw_array_or_table_fill(- builder,- entity,- raw_dst_addr,- val,- len_ptr,- &|env, builder, addr, value| {- if initialized {- gc::write_field_at_addr(env, builder, elem_ty, addr, value)- } else {- gc::init_field_at_addr(env, builder, elem_ty, addr, value)- }- },- )?;+ CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => {+ self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr)?;}// Not allowed to be written to in wasm.CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {@@ -3898,12 +3865,6 @@ impl FuncEnvironment<'_> {dst_elem_addr: ir::Value,value: ir::Value,copy_len: ir::Value,- emit_elem_write: &dyn Fn(- &mut Self,- &mut FunctionBuilder<'_>,- ir::Value,- ir::Value,- ) -> WasmResult<()>,) -> WasmResult<()> {let pointer_ty = self.pointer_type();@@ -3934,6 +3895,18 @@ impl FuncEnvironment<'_> {return Ok(());}+ // Funcref values are intern'd when stored on the GC heap, and the+ // interning can be an expensive operation. Do it once up-front instead+ // of each iteration in this case.+ let (is_pre_interned_funcref, value) = match (entity, elem_ty) {+ (CheckedEntity::Array { .. }, WasmStorageType::Val(WasmValType::Ref(r)))+ if r.heap_type.top() == WasmHeapTopType::Func =>+ {+ (true, gc::intern_func_ref(self, builder, r, value)?)+ }+ _ => (false, value),+ };+// Loop to fill the elements, emitting the equivalent of the following// pseudo-CLIF://@@ -3988,7 +3961,34 @@ impl FuncEnvironment<'_> {self.fuel_consumed += 1;}self.translate_loop_header(builder)?;- emit_elem_write(self, builder, elem_addr, value)?;+ match entity {+ CheckedEntity::Table { table, initialized } => {+ assert!(!is_pre_interned_funcref);+ self.emit_table_set(+ builder,+ table,+ elem_addr,+ ir::MemFlagsData::trusted(),+ value,+ initialized,+ )?+ }+ CheckedEntity::Array { initialized, .. } => {+ if is_pre_interned_funcref {+ builder.ins().store(+ ir::MemFlagsData::trusted().with_endianness(Endianness::Little),+ value,+ elem_addr,+ 0,+ );+ } else if initialized {+ gc::write_field_at_addr(self, builder, elem_ty, elem_addr, value)?+ } else {+ gc::init_field_at_addr(self, builder, elem_ty, elem_addr, value)?+ }+ }+ _ => unreachable!(),+ }let next_elem_addr = builder.ins().iadd_imm(elem_addr, i64::from(elem_size));let done = builder.ins().icmp(IntCC::Equal, next_elem_addr, end_addr);builder.ins().brif(diff --git a/crates/cranelift/src/func_environ/gc/disabled.rs b/crates/cranelift/src/func_environ/gc/disabled.rsindex 7956439b57..e659ff6f74 100644--- a/crates/cranelift/src/func_environ/gc/disabled.rs+++ b/crates/cranelift/src/func_environ/gc/disabled.rs@@ -218,3 +218,12 @@ impl FuncEnvironment<'_> {disabled()}}++pub fn intern_func_ref(+ _func_env: &mut FuncEnvironment<'_>,+ _builder: &mut FunctionBuilder<'_>,+ _ref_type: WasmRefType,+ _func_ref: ir::Value,+) -> WasmResult<ir::Value> {+ disabled()+}diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rsindex e3563305c6..d509f5d4f5 100644--- a/crates/cranelift/src/func_environ/gc/enabled.rs+++ b/crates/cranelift/src/func_environ/gc/enabled.rs@@ -277,14 +277,12 @@ pub fn read_field_at_addr(Ok(value)}-fn write_func_ref_at_addr(+pub fn intern_func_ref(func_env: &mut FuncEnvironment<'_>,builder: &mut FunctionBuilder<'_>,ref_type: WasmRefType,- flags: ir::MemFlagsData,- field_addr: ir::Value,func_ref: ir::Value,-) -> WasmResult<()> {+) -> WasmResult<ir::Value> {assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Func);let vmctx = func_env.vmctx_val(&mut builder.cursor());@@ -313,7 +311,20 @@ fn write_func_ref_at_addr(.ins().call(intern_func_ref_for_gc_heap, &[vmctx, func_ref]);let func_ref_id = builder.func.dfg.first_result(call_inst);- let func_ref_id = builder.ins().ireduce(ir::types::I32, func_ref_id);+ Ok(builder.ins().ireduce(ir::types::I32, func_ref_id))+}++fn write_func_ref_at_addr(+ func_env: &mut FuncEnvironment<'_>,+ builder: &mut FunctionBuilder<'_>,+ ref_type: WasmRefType,+ flags: ir::MemFlagsData,+ field_addr: ir::Value,+ func_ref: ir::Value,+) -> WasmResult<()> {+ // Convert the raw `funcref` into a `FuncRefTableId` for use in the+ // GC heap.+ let func_ref_id = intern_func_ref(func_env, builder, ref_type, func_ref)?;// Store the id in the field.builder.ins().store(flags, func_ref_id, field_addr, 0);diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/enabled/drc.rsindex d59c3dc6b2..b0fdceb1f2 100644--- a/crates/cranelift/src/func_environ/gc/enabled/drc.rs+++ b/crates/cranelift/src/func_environ/gc/enabled/drc.rs@@ -11,8 +11,8 @@ use cranelift_frontend::FunctionBuilder;use smallvec::SmallVec;use wasmtime_environ::drc::{EXCEPTION_TAG_DEFINED_OFFSET, EXCEPTION_TAG_INSTANCE_OFFSET};use wasmtime_environ::{- GcTypeLayouts, PtrSize, TypeIndex, VMGcKind, WasmHeapTopType, WasmHeapType, WasmRefType,- WasmResult, WasmStorageType, WasmValType, drc::DrcTypeLayouts,+ GcTypeLayouts, PtrSize, TypeIndex, VMGcKind, WasmHeapType, WasmRefType, WasmResult,+ WasmStorageType, WasmValType, drc::DrcTypeLayouts,};// The minimum over-approximated stack roots list size for which we will trigger@@ -968,34 +968,16 @@ impl GcCompiler for DrcCompiler {field_addr: ir::Value,val: ir::Value,) -> WasmResult<()> {- // Data inside GC objects is always little endian.- let flags = GC_MEMFLAGS.with_endianness(ir::Endianness::Little);-- match ty {- WasmStorageType::Val(WasmValType::Ref(r)) => match r.heap_type.top() {- WasmHeapTopType::Func => {- write_func_ref_at_addr(func_env, builder, r, flags, field_addr, val)?- }- WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {- self.translate_init_gc_reference(func_env, builder, r, field_addr, val, flags)?- }- WasmHeapTopType::Cont => return super::stack_switching_unsupported(),- },- WasmStorageType::I8 => {- assert_eq!(builder.func.dfg.value_type(val), ir::types::I32);- builder.ins().istore8(flags, val, field_addr, 0);- }- WasmStorageType::I16 => {- assert_eq!(builder.func.dfg.value_type(val), ir::types::I32);- builder.ins().istore16(flags, val, field_addr, 0);- }- WasmStorageType::Val(_) => {- let size_of_access = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&ty);- assert_eq!(builder.func.dfg.value_type(val).bytes(), size_of_access);- builder.ins().store(flags, val, field_addr, 0);- }+ if let WasmStorageType::Val(WasmValType::Ref(r)) = ty+ && r.is_vmgcref_type_and_not_i31()+ {+ // Data inside GC objects is always little endian.+ let flags = GC_MEMFLAGS.with_endianness(ir::Endianness::Little);+ return self.translate_init_gc_reference(func_env, builder, r, field_addr, val, flags);}+ write_field_at_addr(func_env, builder, ty, field_addr, val)?;+Ok(())}}diff --git a/tests/disas/array-fill-funcref.wat b/tests/disas/array-fill-funcref.watindex 026b754942..fb22fd5abc 100644--- a/tests/disas/array-fill-funcref.wat+++ b/tests/disas/array-fill-funcref.wat@@ -58,19 +58,19 @@;; @003b v29 = iadd v7, v28;; @003b v31 = icmp ugt v30, v29;; @003b trapnz v31, user2+;; @003b v33 = call fn0(v0, v4);; v54 = iconst.i64 0-;; @003b v33 = icmp eq v14, v54 ; v54 = 0+;; @003b v36 = icmp eq v14, v54 ; v54 = 0+;; @003b v34 = ireduce.i32 v33;; v47 = iconst.i64 4-;; @003b v32 = iadd v24, v59-;; @003b brif v33, block3, block2(v24)+;; @003b v35 = iadd v24, v59+;; @003b brif v36, block3, block2(v24);;-;; block2(v34: i64):-;; @003b v36 = call fn0(v0, v4)-;; @003b v37 = ireduce.i32 v36-;; @003b store user2 little v37, v34+;; block2(v37: i64):+;; @003b store.i32 notrap aligned little v34, v37;; v61 = iconst.i64 4-;; v62 = iadd v34, v61 ; v61 = 4-;; @003b v39 = icmp eq v62, v32+;; v62 = iadd v37, v61 ; v61 = 4+;; @003b v39 = icmp eq v62, v35;; @003b brif v39, block3, block2(v62);;;; block3:@@ -119,20 +119,19 @@;; @0049 v31 = icmp ugt v30, v29;; @0049 trapnz v31, user2;; @0045 v5 = iconst.i64 0-;; @0049 v33 = icmp eq v14, v5 ; v5 = 0+;; @0049 v33 = call fn0(v0, v5) ; v5 = 0+;; @0049 v36 = icmp eq v14, v5 ; v5 = 0+;; @0049 v34 = ireduce.i32 v33;; v47 = iconst.i64 4-;; @0049 v32 = iadd v24, v58-;; @0049 brif v33, block3, block2(v24)-;;-;; block2(v34: i64):-;; v60 = iconst.i64 0-;; @0049 v36 = call fn0(v0, v60) ; v60 = 0-;; @0049 v37 = ireduce.i32 v36-;; @0049 store user2 little v37, v34-;; v61 = iconst.i64 4-;; v62 = iadd v34, v61 ; v61 = 4-;; @0049 v39 = icmp eq v62, v32-;; @0049 brif v39, block3, block2(v62)+;; @0049 v35 = iadd v24, v58+;; @0049 brif v36, block3, block2(v24)+;;+;; block2(v37: i64):+;; @0049 store.i32 notrap aligned little v34, v37+;; v60 = iconst.i64 4+;; v61 = iadd v37, v60 ; v60 = 4+;; @0049 v39 = icmp eq v61, v35+;; @0049 brif v39, block3, block2(v61);;;; block3:;; @004c jump block1@@ -187,19 +186,19 @@;; @0057 v31 = iadd v9, v30;; @0057 v33 = icmp ugt v32, v31;; @0057 trapnz v33, user2+;; @0057 v35 = call fn1(v0, v7);; v63 = iconst.i64 0-;; @0057 v35 = icmp eq v16, v63 ; v63 = 0+;; @0057 v38 = icmp eq v16, v63 ; v63 = 0+;; @0057 v36 = ireduce.i32 v35;; v52 = iconst.i64 4-;; @0057 v34 = iadd v26, v68-;; @0057 brif v35, block3, block2(v26)+;; @0057 v37 = iadd v26, v68+;; @0057 brif v38, block3, block2(v26);;-;; block2(v36: i64):-;; @0057 v38 = call fn1(v0, v7)-;; @0057 v39 = ireduce.i32 v38-;; @0057 store user2 little v39, v36+;; block2(v39: i64):+;; @0057 store.i32 notrap aligned little v36, v39;; v70 = iconst.i64 4-;; v71 = iadd v36, v70 ; v70 = 4-;; @0057 v41 = icmp eq v71, v34+;; v71 = iadd v39, v70 ; v70 = 4+;; @0057 v41 = icmp eq v71, v37;; @0057 brif v41, block3, block2(v71);;;; block3:diff --git a/tests/disas/gc/array-new-default-funcref.wat b/tests/disas/gc/array-new-default-funcref.watindex 45901103e3..1856a42b0f 100644--- a/tests/disas/gc/array-new-default-funcref.wat+++ b/tests/disas/gc/array-new-default-funcref.wat@@ -106,20 +106,19 @@;; @001f v71 = icmp ugt v70, v69;; @001f trapnz v71, user2;; @001f v44 = iconst.i64 0-;; @001f v73 = icmp.i64 eq v5, v44 ; v44 = 0+;; @001f v73 = call fn1(v0, v44), stack_map=[i32 @ ss0+0] ; v44 = 0+;; @001f v76 = icmp.i64 eq v5, v44 ; v44 = 0+;; @001f v74 = ireduce.i32 v73;; v109 = iconst.i64 4-;; @001f v72 = iadd v61, v111-;; @001f brif v73, block6, block5(v61)+;; @001f v75 = iadd v61, v111+;; @001f brif v76, block6, block5(v61);;-;; block5(v74: i64):-;; v156 = iconst.i64 0-;; @001f v76 = call fn1(v0, v156), stack_map=[i32 @ ss0+0] ; v156 = 0-;; @001f v77 = ireduce.i32 v76-;; @001f store user2 little v77, v74-;; v157 = iconst.i64 4-;; v158 = iadd v74, v157 ; v157 = 4-;; @001f v79 = icmp eq v158, v72-;; @001f brif v79, block6, block5(v158)+;; block5(v77: i64):+;; @001f store.i32 notrap aligned little v74, v77+;; v156 = iconst.i64 4+;; v157 = iadd v77, v156 ; v156 = 4+;; @001f v79 = icmp eq v157, v75+;; @001f brif v79, block6, block5(v157);;;; block6:;; v80 = load.i32 notrap v103