Skip to content

A lightweight WebAssembly runtime that is fast, secure, and standards-compliant

Fix large, backwards, `memory.copy` with fuel/epochs (#13577)

Alex Crichton40ac36360f57
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs
index daa1007dd9..8888be113b 100644
--- a/crates/cranelift/src/func_environ.rs
+++ b/crates/cranelift/src/func_environ.rs
@@ -3686,36 +3686,53 @@ impl FuncEnvironment<'_> {
.ins()
.iconst(pointer_type, UNINTERRUPTABLE_CHUNK_SIZE);
+ // For `memcpy` when chunking this up we might need to do a backwards
+ // copy or a forwards copy. Determine that here and jump to the
+ // backwards copy if needed.
+ let backwards_block = if let BulkOp::MemoryCopy { dst, src, .. } = op {
+ let forwards = builder.ins().icmp(IntCC::UnsignedGreaterThan, src, dst);
+ let forwards_block = builder.create_block();
+ let backwards_block = builder.create_block();
+ builder
+ .ins()
+ .brif(forwards, forwards_block, &[], backwards_block, &[]);
+ builder.switch_to_block(forwards_block);
+ builder.seal_block(forwards_block);
+ Some((backwards_block, op.clone()))
+ } else {
+ None
+ };
+
// Helper closure to test if the length in `op` is larger than `chunk`,
// and if so do a single chunk. Else this goes to the final block with
// the final operation.
- let has_chunk_branch = |builder: &mut FunctionBuilder<'_>, op: &_| {
- let len = match *op {
- BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len,
- };
- let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk);
- match *op {
- BulkOp::MemoryCopy { dst, src, len, .. } => {
- builder.ins().brif(
- has_chunk,
- chunk_block,
- &[dst.into(), src.into(), len.into()],
- last_chunk_block,
- &[dst.into(), src.into(), len.into()],
- );
- }
- BulkOp::MemoryFill { dst, len, .. } => {
- builder.ins().brif(
- has_chunk,
- chunk_block,
- &[dst.into(), len.into()],
- last_chunk_block,
- &[dst.into(), len.into()],
- );
+ let has_chunk_branch =
+ |builder: &mut FunctionBuilder<'_>, op: &_, chunk_block, last_chunk_block| {
+ let len = match *op {
+ BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len,
+ };
+ let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk);
+ match *op {
+ BulkOp::MemoryCopy { dst, src, len, .. } => {
+ builder.ins().brif(
+ has_chunk,
+ chunk_block,
+ &[dst.into(), src.into(), len.into()],
+ last_chunk_block,
+ &[dst.into(), src.into(), len.into()],
+ );
+ }
+ BulkOp::MemoryFill { dst, len, .. } => {
+ builder.ins().brif(
+ has_chunk,
+ chunk_block,
+ &[dst.into(), len.into()],
+ last_chunk_block,
+ &[dst.into(), len.into()],
+ );
+ }
}
- }
- };
- has_chunk_branch(builder, &op);
+ };
let append_block_params = |builder: &mut FunctionBuilder<'_>, block, op: &mut _| match op {
BulkOp::MemoryCopy { dst, src, len, .. } => {
@@ -3729,10 +3746,14 @@ impl FuncEnvironment<'_> {
}
};
- // In the block with per-chunk copies, each operation performs `chunk`
- // length of bytes and then decrements the current length by `chunk`.
- // Afterwards a condition tests if we do another chunk or break out for
- // the final chunk.
+ // Forwards copy: dispatch to the per-chunk loop or the final iteration
+ // if there's no chunks.
+ has_chunk_branch(builder, &op, chunk_block, last_chunk_block);
+
+ // Forwards copy: In the block with per-chunk copies, each operation
+ // performs `chunk` length of bytes and then decrements the current
+ // length by `chunk`. Afterwards a condition tests if we do another
+ // chunk or break out for the final chunk.
builder.switch_to_block(chunk_block);
append_block_params(builder, chunk_block, &mut op);
let op_len = match &mut op {
@@ -3752,17 +3773,83 @@ impl FuncEnvironment<'_> {
*len = builder.ins().isub(remaining_len, chunk);
}
};
- has_chunk_branch(builder, &op);
+ has_chunk_branch(builder, &op, chunk_block, last_chunk_block);
+ builder.seal_block(chunk_block);
+
+ // Backwards copy: similar to the above but with adjustments on where
+ // increments/decrements happen. Notably:
+ //
+ // * Initial src/end are the final byte address
+ // * Each chunk starts out by decrementing src/end as opposed to above
+ // where the increment happens at the end.
+ // * The final block performs the final decrement before jumping to the
+ // shared `last_chunk_block` between the forwards/backwards paths.
+ if let Some((backwards_block, mut op)) = backwards_block {
+ // Setup `dst=dst+len` and `src=src+len`, then see if we have a
+ // chunk.
+ builder.switch_to_block(backwards_block);
+ builder.seal_block(backwards_block);
+ let backwards_chunk_block = builder.create_block();
+ let backwards_last_chunk_block = builder.create_block();
+ let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {
+ unreachable!()
+ };
+ *dst = builder.ins().iadd(*dst, *len);
+ *src = builder.ins().iadd(*src, *len);
+ has_chunk_branch(
+ builder,
+ &op,
+ backwards_chunk_block,
+ backwards_last_chunk_block,
+ );
+
+ // Execute the per-chunk backwards copy, adjusting pointers before
+ // the copy itself.
+ builder.switch_to_block(backwards_chunk_block);
+ append_block_params(builder, backwards_chunk_block, &mut op);
+ let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {
+ unreachable!()
+ };
+ let remaining_len = *len;
+ *len = chunk;
+ *dst = builder.ins().isub(*dst, chunk);
+ *src = builder.ins().isub(*src, chunk);
+ raw_call(self, builder, &op);
+ let BulkOp::MemoryCopy { len, .. } = &mut op else {
+ unreachable!()
+ };
+ *len = builder.ins().isub(remaining_len, chunk);
+ has_chunk_branch(
+ builder,
+ &op,
+ backwards_chunk_block,
+ backwards_last_chunk_block,
+ );
+ builder.seal_block(backwards_chunk_block);
+
+ // Final backwards chunk: adjust the dst/src to be their true base
+ // pointers and then delegate to `last_chunk_block` for the actual
+ // memcpy.
+ builder.switch_to_block(backwards_last_chunk_block);
+ builder.seal_block(backwards_last_chunk_block);
+ append_block_params(builder, backwards_last_chunk_block, &mut op);
+ let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {
+ unreachable!()
+ };
+ *dst = builder.ins().isub(*dst, *len);
+ *src = builder.ins().isub(*src, *len);
+ builder.ins().jump(
+ last_chunk_block,
+ &[(*dst).into(), (*src).into(), (*len).into()],
+ );
+ }
// In the final block we know that the length of the operation is less
- // than `chunk`. This could still be sizable, though, so a final
- // fuel/epoch check is inserted.
+ // than `chunk`.
builder.switch_to_block(last_chunk_block);
+ builder.seal_block(last_chunk_block);
append_block_params(builder, last_chunk_block, &mut op);
raw_call(self, builder, &op);
-
- builder.seal_block(chunk_block);
- builder.seal_block(last_chunk_block);
}
/// Emits a generic "fill" of `entity` from `dst` for `len` elements,
@@ -6134,6 +6221,7 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type {
}
/// Operations to [`FuncEnvironment::raw_bulk_memory_operation`].
+#[derive(Clone)]
enum BulkOp {
/// A `memory.copy` operation, copying memory from `src` to `dst`.
///
diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.wat
index b0fc7703a5..709377b718 100644
--- a/tests/disas/memory-copy-epochs.wat
+++ b/tests/disas/memory-copy-epochs.wat
@@ -34,7 +34,7 @@
;; @001e v12 = call fn0(v0)
;; @001e jump block2(v12)
;;
-;; block2(v76: i64):
+;; block2(v65: i64):
;; @0025 v17 = load.i64 notrap aligned v0+64
;; @0025 v18 = uextend.i64 v2
;; @0025 v19 = uextend.i64 v4
@@ -45,52 +45,90 @@
;; @0025 v35 = iadd v31, v19
;; @0025 v36 = icmp ugt v35, v17
;; @0025 trapnz v36, heap_oob
-;; @0025 v44 = iconst.i64 0x0800_0000
-;; @0025 v45 = icmp ugt v19, v44 ; v44 = 0x0800_0000
;; @0025 v24 = load.i64 notrap aligned readonly can_move v0+56
-;; @0025 v28 = iadd v24, v18
;; @0025 v41 = iadd v24, v31
-;; @0025 brif v45, block4(v28, v41, v19, v76), block5(v28, v41, v19, v76)
-;;
-;; block4(v46: i64, v47: i64, v48: i64, v51: i64):
-;; @0025 v50 = load.i64 notrap aligned v6
-;; @0025 v52 = icmp uge v50, v51
-;; @0025 brif v52, block7, block6(v51)
-;;
-;; block5(v62: i64, v63: i64, v64: i64, v67: i64):
-;; @0025 v66 = load.i64 notrap aligned v6
-;; @0025 v68 = icmp uge v66, v67
-;; @0025 brif v68, block10, block9
-;;
-;; block7 cold:
-;; @0025 v54 = load.i64 notrap aligned v8+8
-;; @0025 v55 = icmp.i64 uge v50, v54
-;; @0025 brif v55, block8, block6(v54)
-;;
-;; block8 cold:
-;; @0025 v57 = call fn0(v0)
-;; @0025 jump block6(v57)
-;;
-;; block6(v77: i64):
-;; v87 = iconst.i64 0x0800_0000
-;; @0025 call fn1(v0, v46, v47, v87) ; v87 = 0x0800_0000
-;; v88 = isub.i64 v48, v87 ; v87 = 0x0800_0000
-;; v89 = icmp ugt v88, v87 ; v87 = 0x0800_0000
-;; v90 = iadd.i64 v46, v87 ; v87 = 0x0800_0000
-;; v91 = iadd.i64 v47, v87 ; v87 = 0x0800_0000
-;; @0025 brif v89, block4(v90, v91, v88, v77), block5(v90, v91, v88, v77)
+;; @0025 v28 = iadd v24, v18
+;; @0025 v45 = icmp ugt v41, v28
+;; @0025 brif v45, block6, block7
+;;
+;; block4(v47: i64, v48: i64, v49: i64, v52: i64):
+;; @0025 v51 = load.i64 notrap aligned v6
+;; @0025 v53 = icmp uge v51, v52
+;; @0025 brif v53, block9, block8(v52)
+;;
+;; block5(v93: i64, v94: i64, v95: i64, v99: i64):
+;; @0025 v98 = load.i64 notrap aligned v6
+;; @0025 v101 = icmp uge v98, v99
+;; @0025 brif v101, block17, block16
+;;
+;; block6:
+;; v122 = iconst.i64 0x0800_0000
+;; v123 = icmp.i64 ugt v19, v122 ; v122 = 0x0800_0000
+;; @0025 brif v123, block4(v28, v41, v19, v65), block5(v28, v41, v19, v65)
+;;
+;; block9 cold:
+;; @0025 v55 = load.i64 notrap aligned v8+8
+;; @0025 v56 = icmp.i64 uge v51, v55
+;; @0025 brif v56, block10, block8(v55)
;;
;; block10 cold:
-;; @0025 v70 = load.i64 notrap aligned v8+8
-;; @0025 v71 = icmp.i64 uge v66, v70
-;; @0025 brif v71, block11, block9
+;; @0025 v58 = call fn0(v0)
+;; @0025 jump block8(v58)
+;;
+;; block8(v66: i64):
+;; @0025 call fn1(v0, v47, v48, v122) ; v122 = 0x0800_0000
+;; @0025 v61 = isub.i64 v49, v122 ; v122 = 0x0800_0000
+;; @0025 v62 = icmp ugt v61, v122 ; v122 = 0x0800_0000
+;; @0025 v59 = iadd.i64 v47, v122 ; v122 = 0x0800_0000
+;; @0025 v60 = iadd.i64 v48, v122 ; v122 = 0x0800_0000
+;; @0025 brif v62, block4(v59, v60, v61, v66), block5(v59, v60, v61, v66)
+;;
+;; block7:
+;; @0025 v44 = iconst.i64 0x0800_0000
+;; @0025 v69 = icmp.i64 ugt v19, v44 ; v44 = 0x0800_0000
+;; @0025 v67 = iadd.i64 v28, v19
+;; @0025 v68 = iadd.i64 v41, v19
+;; @0025 brif v69, block11(v67, v68, v19, v65), block12(v67, v68, v19, v65)
+;;
+;; block11(v70: i64, v71: i64, v72: i64, v77: i64):
+;; @0025 v76 = load.i64 notrap aligned v6
+;; @0025 v78 = icmp uge v76, v77
+;; @0025 brif v78, block14, block13(v77)
+;;
+;; block14 cold:
+;; @0025 v80 = load.i64 notrap aligned v8+8
+;; @0025 v81 = icmp.i64 uge v76, v80
+;; @0025 brif v81, block15, block13(v80)
+;;
+;; block15 cold:
+;; @0025 v83 = call fn0(v0)
+;; @0025 jump block13(v83)
+;;
+;; block13(v87: i64):
+;; v117 = iconst.i64 0x0800_0000
+;; v118 = isub.i64 v70, v117 ; v117 = 0x0800_0000
+;; v119 = isub.i64 v71, v117 ; v117 = 0x0800_0000
+;; @0025 call fn1(v0, v118, v119, v117) ; v117 = 0x0800_0000
+;; v120 = isub.i64 v72, v117 ; v117 = 0x0800_0000
+;; v121 = icmp ugt v120, v117 ; v117 = 0x0800_0000
+;; @0025 brif v121, block11(v118, v119, v120, v87), block12(v118, v119, v120, v87)
+;;
+;; block12(v88: i64, v89: i64, v90: i64, v100: i64):
+;; @0025 v91 = isub v88, v90
+;; @0025 v92 = isub v89, v90
+;; @0025 jump block5(v91, v92, v90, v100)
+;;
+;; block17 cold:
+;; @0025 v103 = load.i64 notrap aligned v8+8
+;; @0025 v104 = icmp.i64 uge v98, v103
+;; @0025 brif v104, block18, block16
;;
-;; block11 cold:
-;; @0025 v73 = call fn0(v0)
-;; @0025 jump block9
+;; block18 cold:
+;; @0025 v106 = call fn0(v0)
+;; @0025 jump block16
;;
-;; block9:
-;; @0025 call fn1(v0, v62, v63, v64)
+;; block16:
+;; @0025 call fn1(v0, v93, v94, v95)
;; @0029 jump block1
;;
;; block1:
diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.wat
index 5eb9dcd2e4..ffc092b097 100644
--- a/tests/disas/memory-copy-fuel.wat
+++ b/tests/disas/memory-copy-fuel.wat
@@ -32,8 +32,8 @@
;; @001e brif v10, block2, block3(v8)
;;
;; block2:
-;; v96 = iadd.i64 v6, v7 ; v7 = 1
-;; @001e store notrap aligned v96, v5
+;; v125 = iadd.i64 v6, v7 ; v7 = 1
+;; @001e store notrap aligned v125, v5
;; @001e v13 = call fn0(v0)
;; @001e v15 = load.i64 notrap aligned v5
;; @001e jump block3(v15)
@@ -49,54 +49,93 @@
;; @0025 v38 = iadd v34, v22
;; @0025 v39 = icmp ugt v38, v20
;; @0025 trapnz v39, heap_oob
-;; @0025 v50 = iconst.i64 0x0800_0000
-;; @0025 v51 = icmp ugt v22, v50 ; v50 = 0x0800_0000
;; @0025 v27 = load.i64 notrap aligned readonly can_move v0+56
-;; @0025 v31 = iadd v27, v21
;; @0025 v44 = iadd v27, v34
-;; @0025 v48 = iconst.i64 4
-;; @0025 v49 = iadd v47, v48 ; v48 = 4
-;; @0025 brif v51, block4(v31, v44, v22, v49), block5(v31, v44, v22, v49)
-;;
-;; block4(v52: i64, v53: i64, v54: i64, v55: i64):
-;; v97 = iconst.i64 0x0800_0000
-;; v98 = iadd v55, v97 ; v97 = 0x0800_0000
-;; v99 = iconst.i64 0
-;; v100 = icmp sge v98, v99 ; v99 = 0
-;; @0025 brif v100, block6, block7(v98)
-;;
-;; block5(v68: i64, v69: i64, v70: i64, v71: i64):
-;; @0025 v72 = iadd v71, v70
-;; v106 = iconst.i64 0
-;; v107 = icmp sge v72, v106 ; v106 = 0
-;; @0025 brif v107, block8, block9(v72)
+;; @0025 v31 = iadd v27, v21
+;; @0025 v51 = icmp ugt v44, v31
+;; @0025 brif v51, block6, block7
+;;
+;; block4(v53: i64, v54: i64, v55: i64, v56: i64):
+;; @0025 v57 = iadd v56, v135 ; v135 = 0x0800_0000
+;; v139 = iconst.i64 0
+;; v140 = icmp sge v57, v139 ; v139 = 0
+;; @0025 brif v140, block8, block9(v57)
+;;
+;; block5(v95: i64, v96: i64, v97: i64, v98: i64):
+;; @0025 v100 = iadd v98, v97
+;; v142 = iconst.i64 0
+;; v143 = icmp sge v100, v142 ; v142 = 0
+;; @0025 brif v143, block14, block15(v100)
;;
;; block6:
-;; @0025 store.i64 notrap aligned v98, v5
-;; @0025 v61 = call fn0(v0)
-;; @0025 v63 = load.i64 notrap aligned v5
-;; @0025 jump block7(v63)
-;;
-;; block7(v80: i64):
-;; v101 = iconst.i64 0x0800_0000
-;; @0025 call fn1(v0, v52, v53, v101) ; v101 = 0x0800_0000
-;; v102 = isub.i64 v54, v101 ; v101 = 0x0800_0000
-;; v103 = icmp ugt v102, v101 ; v101 = 0x0800_0000
-;; v104 = iadd.i64 v52, v101 ; v101 = 0x0800_0000
-;; v105 = iadd.i64 v53, v101 ; v101 = 0x0800_0000
-;; @0025 brif v103, block4(v104, v105, v102, v80), block5(v104, v105, v102, v80)
+;; v135 = iconst.i64 0x0800_0000
+;; v136 = icmp.i64 ugt v22, v135 ; v135 = 0x0800_0000
+;; v137 = iconst.i64 4
+;; v138 = iadd.i64 v47, v137 ; v137 = 4
+;; @0025 brif v136, block4(v31, v44, v22, v138), block5(v31, v44, v22, v138)
;;
;; block8:
-;; @0025 store.i64 notrap aligned v72, v5
-;; @0025 v77 = call fn0(v0)
-;; @0025 v79 = load.i64 notrap aligned v5
-;; @0025 jump block9(v79)
+;; v141 = iadd.i64 v56, v135 ; v135 = 0x0800_0000
+;; @0025 store notrap aligned v141, v5
+;; @0025 v62 = call fn0(v0)
+;; @0025 v64 = load.i64 notrap aligned v5
+;; @0025 jump block9(v64)
+;;
+;; block9(v69: i64):
+;; @0025 call fn1(v0, v53, v54, v135) ; v135 = 0x0800_0000
+;; @0025 v67 = isub.i64 v55, v135 ; v135 = 0x0800_0000
+;; @0025 v68 = icmp ugt v67, v135 ; v135 = 0x0800_0000
+;; @0025 v65 = iadd.i64 v53, v135 ; v135 = 0x0800_0000
+;; @0025 v66 = iadd.i64 v54, v135 ; v135 = 0x0800_0000
+;; @0025 brif v68, block4(v65, v66, v67, v69), block5(v65, v66, v67, v69)
+;;
+;; block7:
+;; @0025 v50 = iconst.i64 0x0800_0000
+;; @0025 v72 = icmp.i64 ugt v22, v50 ; v50 = 0x0800_0000
+;; @0025 v70 = iadd.i64 v31, v22
+;; @0025 v71 = iadd.i64 v44, v22
+;; @0025 v48 = iconst.i64 4
+;; @0025 v49 = iadd.i64 v47, v48 ; v48 = 4
+;; @0025 brif v72, block10(v70, v71, v22, v49), block11(v70, v71, v22, v49)
+;;
+;; block10(v73: i64, v74: i64, v75: i64, v78: i64):
+;; v126 = iconst.i64 0x0800_0000
+;; v127 = iadd v78, v126 ; v126 = 0x0800_0000
+;; v128 = iconst.i64 0
+;; v129 = icmp sge v127, v128 ; v128 = 0
+;; @0025 brif v129, block12, block13(v127)
+;;
+;; block12:
+;; @0025 store.i64 notrap aligned v127, v5
+;; @0025 v84 = call fn0(v0)
+;; @0025 v86 = load.i64 notrap aligned v5
+;; @0025 jump block13(v86)
+;;
+;; block13(v89: i64):
+;; v130 = iconst.i64 0x0800_0000
+;; v131 = isub.i64 v73, v130 ; v130 = 0x0800_0000
+;; v132 = isub.i64 v74, v130 ; v130 = 0x0800_0000
+;; @0025 call fn1(v0, v131, v132, v130) ; v130 = 0x0800_0000
+;; v133 = isub.i64 v75, v130 ; v130 = 0x0800_0000
+;; v134 = icmp ugt v133, v130 ; v130 = 0x0800_0000
+;; @0025 brif v134, block10(v131, v132, v133, v89), block11(v131, v132, v133, v89)
+;;
+;; block11(v90: i64, v91: i64, v92: i64, v99: i64):
+;; @0025 v93 = isub v90, v92
+;; @0025 v94 = isub v91, v92
+;; @0025 jump block5(v93, v94, v92, v99)
+;;
+;; block14:
+;; @0025 store.i64 notrap aligned v100, v5
+;; @0025 v105 = call fn0(v0)
+;; @0025 v107 = load.i64 notrap aligned v5
+;; @0025 jump block15(v107)
;;
-;; block9(v82: i64):
-;; @0025 call fn1(v0, v68, v69, v70)
+;; block15(v109: i64):
+;; @0025 call fn1(v0, v95, v96, v97)
;; @0029 jump block1
;;
;; block1:
-;; @0029 store.i64 notrap aligned v82, v5
+;; @0029 store.i64 notrap aligned v109, v5
;; @0029 return
;; }
diff --git a/tests/misc_testsuite/memory-copy.wast b/tests/misc_testsuite/memory-copy.wast
index 4b18d7f5d8..200abf763f 100644
--- a/tests/misc_testsuite/memory-copy.wast
+++ b/tests/misc_testsuite/memory-copy.wast
@@ -1,6 +1,7 @@
;;! bulk_memory = true
;;! multi_memory = true
;;! memory64 = true
+;;! hogs_memory = true
(module
(memory 1 1)
@@ -272,3 +273,17 @@
(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const 100) (i32.const -1)) "out of bounds")
(assert_trap (invoke "m64_to_m32" (i32.const -1) (i64.const 0) (i32.const 100)) "out of bounds")
(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const -1) (i32.const 100)) "out of bounds")
+
+;; Copying a big chunk up one byte should do a backwards copy, not a forwards
+;; copy, even if this is chunked up in to separate slices for fuel/epochs/etc.
+(module
+ (memory (export "mem") 3200 3200) ;; 200 MiB
+ (func (export "run") (result i32)
+ (i32.store8 (i32.const 134217727) (i32.const 0xAA))
+ (i32.store8 (i32.const 134217728) (i32.const 0xBB))
+ (memory.copy (i32.const 1) (i32.const 0) (i32.const 134217744))
+ (i32.load8_u (i32.const 134217729))
+ )
+)
+
+(assert_return (invoke "run") (i32.const 0xBB))