Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Intern functions once in array-fill ops (#13524)

Alex Crichton7e56e24991fc
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index 50a7452e9f..64a0977335 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -3832,41 +3832,8 @@ impl FuncEnvironment<'_> {
},
);
}
- CheckedEntity::Table { table, initialized } => {
- self.emit_raw_array_or_table_fill(
- builder,
- entity,
- raw_dst_addr,
- val,
- len_ptr,
- &|env, builder, addr, value| {
- env.emit_table_set(
- builder,
- table,
- addr,
- ir::MemFlagsData::trusted(),
- value,
- initialized,
- )
- },
- )?;
- }
- CheckedEntity::Array { initialized, .. } => {
- let elem_ty = entity.storage_type(self);
- self.emit_raw_array_or_table_fill(
- builder,
- entity,
- raw_dst_addr,
- val,
- len_ptr,
- &|env, builder, addr, value| {
- if initialized {
- gc::write_field_at_addr(env, builder, elem_ty, addr, value)
- } else {
- gc::init_field_at_addr(env, builder, elem_ty, addr, value)
- }
- },
- )?;
+ CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => {
+ self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr)?;
}
// Not allowed to be written to in wasm.
CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => {
@@ -3898,12 +3865,6 @@ impl FuncEnvironment<'_> {
dst_elem_addr: ir::Value,
value: ir::Value,
copy_len: ir::Value,
- emit_elem_write: &dyn Fn(
- &mut Self,
- &mut FunctionBuilder<'_>,
- ir::Value,
- ir::Value,
- ) -> WasmResult<()>,
) -> WasmResult<()> {
let pointer_ty = self.pointer_type();
@@ -3934,6 +3895,18 @@ impl FuncEnvironment<'_> {
return Ok(());
}
+ // Funcref values are intern'd when stored on the GC heap, and the
+ // interning can be an expensive operation. Do it once up-front instead
+ // of each iteration in this case.
+ let (is_pre_interned_funcref, value) = match (entity, elem_ty) {
+ (CheckedEntity::Array { .. }, WasmStorageType::Val(WasmValType::Ref(r)))
+ if r.heap_type.top() == WasmHeapTopType::Func =>
+ {
+ (true, gc::intern_func_ref(self, builder, r, value)?)
+ }
+ _ => (false, value),
+ };
+
// Loop to fill the elements, emitting the equivalent of the following
// pseudo-CLIF:
//
@@ -3988,7 +3961,34 @@ impl FuncEnvironment<'_> {
self.fuel_consumed += 1;
}
self.translate_loop_header(builder)?;
- emit_elem_write(self, builder, elem_addr, value)?;
+ match entity {
+ CheckedEntity::Table { table, initialized } => {
+ assert!(!is_pre_interned_funcref);
+ self.emit_table_set(
+ builder,
+ table,
+ elem_addr,
+ ir::MemFlagsData::trusted(),
+ value,
+ initialized,
+ )?
+ }
+ CheckedEntity::Array { initialized, .. } => {
+ if is_pre_interned_funcref {
+ builder.ins().store(
+ ir::MemFlagsData::trusted().with_endianness(Endianness::Little),
+ value,
+ elem_addr,
+ 0,
+ );
+ } else if initialized {
+ gc::write_field_at_addr(self, builder, elem_ty, elem_addr, value)?
+ } else {
+ gc::init_field_at_addr(self, builder, elem_ty, elem_addr, value)?
+ }
+ }
+ _ => unreachable!(),
+ }
let next_elem_addr = builder.ins().iadd_imm(elem_addr, i64::from(elem_size));
let done = builder.ins().icmp(IntCC::Equal, next_elem_addr, end_addr);
builder.ins().brif(
diff --git a/crates/cranelift/src/func_environ/gc/disabled.rs b/crates/cranelift/src/func_environ/gc/disabled.rs
index 7956439b57..e659ff6f74 100644
--- a/crates/cranelift/src/func_environ/gc/disabled.rs
+++ b/crates/cranelift/src/func_environ/gc/disabled.rs
@@ -218,3 +218,12 @@ impl FuncEnvironment<'_> {
disabled()
}
}
+
+pub fn intern_func_ref(
+ _func_env: &mut FuncEnvironment<'_>,
+ _builder: &mut FunctionBuilder<'_>,
+ _ref_type: WasmRefType,
+ _func_ref: ir::Value,
+) -> WasmResult<ir::Value> {
+ disabled()
+}
diff --git a/crates/cranelift/src/func_environ/gc/enabled.rs b/crates/cranelift/src/func_environ/gc/enabled.rs
index e3563305c6..d509f5d4f5 100644
--- a/crates/cranelift/src/func_environ/gc/enabled.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled.rs
@@ -277,14 +277,12 @@ pub fn read_field_at_addr(
Ok(value)
}
-fn write_func_ref_at_addr(
+pub fn intern_func_ref(
func_env: &mut FuncEnvironment<'_>,
builder: &mut FunctionBuilder<'_>,
ref_type: WasmRefType,
- flags: ir::MemFlagsData,
- field_addr: ir::Value,
func_ref: ir::Value,
-) -> WasmResult<()> {
+) -> WasmResult<ir::Value> {
assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Func);
let vmctx = func_env.vmctx_val(&mut builder.cursor());
@@ -313,7 +311,20 @@ fn write_func_ref_at_addr(
.ins()
.call(intern_func_ref_for_gc_heap, &[vmctx, func_ref]);
let func_ref_id = builder.func.dfg.first_result(call_inst);
- let func_ref_id = builder.ins().ireduce(ir::types::I32, func_ref_id);
+ Ok(builder.ins().ireduce(ir::types::I32, func_ref_id))
+}
+
+fn write_func_ref_at_addr(
+ func_env: &mut FuncEnvironment<'_>,
+ builder: &mut FunctionBuilder<'_>,
+ ref_type: WasmRefType,
+ flags: ir::MemFlagsData,
+ field_addr: ir::Value,
+ func_ref: ir::Value,
+) -> WasmResult<()> {
+ // Convert the raw `funcref` into a `FuncRefTableId` for use in the
+ // GC heap.
+ let func_ref_id = intern_func_ref(func_env, builder, ref_type, func_ref)?;
// Store the id in the field.
builder.ins().store(flags, func_ref_id, field_addr, 0);
diff --git a/crates/cranelift/src/func_environ/gc/enabled/drc.rs b/crates/cranelift/src/func_environ/gc/enabled/drc.rs
index d59c3dc6b2..b0fdceb1f2 100644
--- a/crates/cranelift/src/func_environ/gc/enabled/drc.rs
+++ b/crates/cranelift/src/func_environ/gc/enabled/drc.rs
@@ -11,8 +11,8 @@ use cranelift_frontend::FunctionBuilder;
use smallvec::SmallVec;
use wasmtime_environ::drc::{EXCEPTION_TAG_DEFINED_OFFSET, EXCEPTION_TAG_INSTANCE_OFFSET};
use wasmtime_environ::{
- GcTypeLayouts, PtrSize, TypeIndex, VMGcKind, WasmHeapTopType, WasmHeapType, WasmRefType,
- WasmResult, WasmStorageType, WasmValType, drc::DrcTypeLayouts,
+ GcTypeLayouts, PtrSize, TypeIndex, VMGcKind, WasmHeapType, WasmRefType, WasmResult,
+ WasmStorageType, WasmValType, drc::DrcTypeLayouts,
};
// The minimum over-approximated stack roots list size for which we will trigger
@@ -968,34 +968,16 @@ impl GcCompiler for DrcCompiler {
field_addr: ir::Value,
val: ir::Value,
) -> WasmResult<()> {
- // Data inside GC objects is always little endian.
- let flags = GC_MEMFLAGS.with_endianness(ir::Endianness::Little);
-
- match ty {
- WasmStorageType::Val(WasmValType::Ref(r)) => match r.heap_type.top() {
- WasmHeapTopType::Func => {
- write_func_ref_at_addr(func_env, builder, r, flags, field_addr, val)?
- }
- WasmHeapTopType::Extern | WasmHeapTopType::Any | WasmHeapTopType::Exn => {
- self.translate_init_gc_reference(func_env, builder, r, field_addr, val, flags)?
- }
- WasmHeapTopType::Cont => return super::stack_switching_unsupported(),
- },
- WasmStorageType::I8 => {
- assert_eq!(builder.func.dfg.value_type(val), ir::types::I32);
- builder.ins().istore8(flags, val, field_addr, 0);
- }
- WasmStorageType::I16 => {
- assert_eq!(builder.func.dfg.value_type(val), ir::types::I32);
- builder.ins().istore16(flags, val, field_addr, 0);
- }
- WasmStorageType::Val(_) => {
- let size_of_access = wasmtime_environ::byte_size_of_wasm_ty_in_gc_heap(&ty);
- assert_eq!(builder.func.dfg.value_type(val).bytes(), size_of_access);
- builder.ins().store(flags, val, field_addr, 0);
- }
+ if let WasmStorageType::Val(WasmValType::Ref(r)) = ty
+ && r.is_vmgcref_type_and_not_i31()
+ {
+ // Data inside GC objects is always little endian.
+ let flags = GC_MEMFLAGS.with_endianness(ir::Endianness::Little);
+ return self.translate_init_gc_reference(func_env, builder, r, field_addr, val, flags);
}
+ write_field_at_addr(func_env, builder, ty, field_addr, val)?;
+
Ok(())
}
}
diff --git a/tests/disas/array-fill-funcref.wat b/tests/disas/array-fill-funcref.wat
index 026b754942..fb22fd5abc 100644
--- a/tests/disas/array-fill-funcref.wat
+++ b/tests/disas/array-fill-funcref.wat
@@ -58,19 +58,19 @@
;; @003b v29 = iadd v7, v28
;; @003b v31 = icmp ugt v30, v29
;; @003b trapnz v31, user2
+;; @003b v33 = call fn0(v0, v4)
;; v54 = iconst.i64 0
-;; @003b v33 = icmp eq v14, v54 ; v54 = 0
+;; @003b v36 = icmp eq v14, v54 ; v54 = 0
+;; @003b v34 = ireduce.i32 v33
;; v47 = iconst.i64 4
-;; @003b v32 = iadd v24, v59
-;; @003b brif v33, block3, block2(v24)
+;; @003b v35 = iadd v24, v59
+;; @003b brif v36, block3, block2(v24)
;;
-;; block2(v34: i64):
-;; @003b v36 = call fn0(v0, v4)
-;; @003b v37 = ireduce.i32 v36
-;; @003b store user2 little v37, v34
+;; block2(v37: i64):
+;; @003b store.i32 notrap aligned little v34, v37
;; v61 = iconst.i64 4
-;; v62 = iadd v34, v61 ; v61 = 4
-;; @003b v39 = icmp eq v62, v32
+;; v62 = iadd v37, v61 ; v61 = 4
+;; @003b v39 = icmp eq v62, v35
;; @003b brif v39, block3, block2(v62)
;;
;; block3:
@@ -119,20 +119,19 @@
;; @0049 v31 = icmp ugt v30, v29
;; @0049 trapnz v31, user2
;; @0045 v5 = iconst.i64 0
-;; @0049 v33 = icmp eq v14, v5 ; v5 = 0
+;; @0049 v33 = call fn0(v0, v5) ; v5 = 0
+;; @0049 v36 = icmp eq v14, v5 ; v5 = 0
+;; @0049 v34 = ireduce.i32 v33
;; v47 = iconst.i64 4
-;; @0049 v32 = iadd v24, v58
-;; @0049 brif v33, block3, block2(v24)
-;;
-;; block2(v34: i64):
-;; v60 = iconst.i64 0
-;; @0049 v36 = call fn0(v0, v60) ; v60 = 0
-;; @0049 v37 = ireduce.i32 v36
-;; @0049 store user2 little v37, v34
-;; v61 = iconst.i64 4
-;; v62 = iadd v34, v61 ; v61 = 4
-;; @0049 v39 = icmp eq v62, v32
-;; @0049 brif v39, block3, block2(v62)
+;; @0049 v35 = iadd v24, v58
+;; @0049 brif v36, block3, block2(v24)
+;;
+;; block2(v37: i64):
+;; @0049 store.i32 notrap aligned little v34, v37
+;; v60 = iconst.i64 4
+;; v61 = iadd v37, v60 ; v60 = 4
+;; @0049 v39 = icmp eq v61, v35
+;; @0049 brif v39, block3, block2(v61)
;;
;; block3:
;; @004c jump block1
@@ -187,19 +186,19 @@
;; @0057 v31 = iadd v9, v30
;; @0057 v33 = icmp ugt v32, v31
;; @0057 trapnz v33, user2
+;; @0057 v35 = call fn1(v0, v7)
;; v63 = iconst.i64 0
-;; @0057 v35 = icmp eq v16, v63 ; v63 = 0
+;; @0057 v38 = icmp eq v16, v63 ; v63 = 0
+;; @0057 v36 = ireduce.i32 v35
;; v52 = iconst.i64 4
-;; @0057 v34 = iadd v26, v68
-;; @0057 brif v35, block3, block2(v26)
+;; @0057 v37 = iadd v26, v68
+;; @0057 brif v38, block3, block2(v26)
;;
-;; block2(v36: i64):
-;; @0057 v38 = call fn1(v0, v7)
-;; @0057 v39 = ireduce.i32 v38
-;; @0057 store user2 little v39, v36
+;; block2(v39: i64):
+;; @0057 store.i32 notrap aligned little v36, v39
;; v70 = iconst.i64 4
-;; v71 = iadd v36, v70 ; v70 = 4
-;; @0057 v41 = icmp eq v71, v34
+;; v71 = iadd v39, v70 ; v70 = 4
+;; @0057 v41 = icmp eq v71, v37
;; @0057 brif v41, block3, block2(v71)
;;
;; block3:
diff --git a/tests/disas/gc/array-new-default-funcref.wat b/tests/disas/gc/array-new-default-funcref.wat
index 45901103e3..1856a42b0f 100644
--- a/tests/disas/gc/array-new-default-funcref.wat
+++ b/tests/disas/gc/array-new-default-funcref.wat
@@ -106,20 +106,19 @@
;; @001f v71 = icmp ugt v70, v69
;; @001f trapnz v71, user2
;; @001f v44 = iconst.i64 0
-;; @001f v73 = icmp.i64 eq v5, v44 ; v44 = 0
+;; @001f v73 = call fn1(v0, v44), stack_map=[i32 @ ss0+0] ; v44 = 0
+;; @001f v76 = icmp.i64 eq v5, v44 ; v44 = 0
+;; @001f v74 = ireduce.i32 v73
;; v109 = iconst.i64 4
-;; @001f v72 = iadd v61, v111
-;; @001f brif v73, block6, block5(v61)
+;; @001f v75 = iadd v61, v111
+;; @001f brif v76, block6, block5(v61)
;;
-;; block5(v74: i64):
-;; v156 = iconst.i64 0
-;; @001f v76 = call fn1(v0, v156), stack_map=[i32 @ ss0+0] ; v156 = 0
-;; @001f v77 = ireduce.i32 v76
-;; @001f store user2 little v77, v74
-;; v157 = iconst.i64 4
-;; v158 = iadd v74, v157 ; v157 = 4
-;; @001f v79 = icmp eq v158, v72
-;; @001f brif v79, block6, block5(v158)
+;; block5(v77: i64):
+;; @001f store.i32 notrap aligned little v74, v77
+;; v156 = iconst.i64 4
+;; v157 = iadd v77, v156 ; v156 = 4
+;; @001f v79 = icmp eq v157, v75
+;; @001f brif v79, block6, block5(v157)
;;
;; block6:
;; v80 = load.i32 notrap v103