A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Fix large, backwards, `memory.copy` with fuel/epochs (#13577)
Alex Crichton40ac36360f57
diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rsindex daa1007dd9..8888be113b 100644--- a/crates/cranelift/src/func_environ.rs+++ b/crates/cranelift/src/func_environ.rs@@ -3686,36 +3686,53 @@ impl FuncEnvironment<'_> {.ins().iconst(pointer_type, UNINTERRUPTABLE_CHUNK_SIZE);+ // For `memcpy` when chunking this up we might need to do a backwards+ // copy or a forwards copy. Determine that here and jump to the+ // backwards copy if needed.+ let backwards_block = if let BulkOp::MemoryCopy { dst, src, .. } = op {+ let forwards = builder.ins().icmp(IntCC::UnsignedGreaterThan, src, dst);+ let forwards_block = builder.create_block();+ let backwards_block = builder.create_block();+ builder+ .ins()+ .brif(forwards, forwards_block, &[], backwards_block, &[]);+ builder.switch_to_block(forwards_block);+ builder.seal_block(forwards_block);+ Some((backwards_block, op.clone()))+ } else {+ None+ };+// Helper closure to test if the length in `op` is larger than `chunk`,// and if so do a single chunk. Else this goes to the final block with// the final operation.- let has_chunk_branch = |builder: &mut FunctionBuilder<'_>, op: &_| {- let len = match *op {- BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len,- };- let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk);- match *op {- BulkOp::MemoryCopy { dst, src, len, .. } => {- builder.ins().brif(- has_chunk,- chunk_block,- &[dst.into(), src.into(), len.into()],- last_chunk_block,- &[dst.into(), src.into(), len.into()],- );- }- BulkOp::MemoryFill { dst, len, .. } => {- builder.ins().brif(- has_chunk,- chunk_block,- &[dst.into(), len.into()],- last_chunk_block,- &[dst.into(), len.into()],- );+ let has_chunk_branch =+ |builder: &mut FunctionBuilder<'_>, op: &_, chunk_block, last_chunk_block| {+ let len = match *op {+ BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len,+ };+ let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk);+ match *op {+ BulkOp::MemoryCopy { dst, src, len, .. } => {+ builder.ins().brif(+ has_chunk,+ chunk_block,+ &[dst.into(), src.into(), len.into()],+ last_chunk_block,+ &[dst.into(), src.into(), len.into()],+ );+ }+ BulkOp::MemoryFill { dst, len, .. } => {+ builder.ins().brif(+ has_chunk,+ chunk_block,+ &[dst.into(), len.into()],+ last_chunk_block,+ &[dst.into(), len.into()],+ );+ }}- }- };- has_chunk_branch(builder, &op);+ };let append_block_params = |builder: &mut FunctionBuilder<'_>, block, op: &mut _| match op {BulkOp::MemoryCopy { dst, src, len, .. } => {@@ -3729,10 +3746,14 @@ impl FuncEnvironment<'_> {}};- // In the block with per-chunk copies, each operation performs `chunk`- // length of bytes and then decrements the current length by `chunk`.- // Afterwards a condition tests if we do another chunk or break out for- // the final chunk.+ // Forwards copy: dispatch to the per-chunk loop or the final iteration+ // if there's no chunks.+ has_chunk_branch(builder, &op, chunk_block, last_chunk_block);++ // Forwards copy: In the block with per-chunk copies, each operation+ // performs `chunk` length of bytes and then decrements the current+ // length by `chunk`. Afterwards a condition tests if we do another+ // chunk or break out for the final chunk.builder.switch_to_block(chunk_block);append_block_params(builder, chunk_block, &mut op);let op_len = match &mut op {@@ -3752,17 +3773,83 @@ impl FuncEnvironment<'_> {*len = builder.ins().isub(remaining_len, chunk);}};- has_chunk_branch(builder, &op);+ has_chunk_branch(builder, &op, chunk_block, last_chunk_block);+ builder.seal_block(chunk_block);++ // Backwards copy: similar to the above but with adjustments on where+ // increments/decrements happen. Notably:+ //+ // * Initial src/end are the final byte address+ // * Each chunk starts out by decrementing src/end as opposed to above+ // where the increment happens at the end.+ // * The final block performs the final decrement before jumping to the+ // shared `last_chunk_block` between the forwards/backwards paths.+ if let Some((backwards_block, mut op)) = backwards_block {+ // Setup `dst=dst+len` and `src=src+len`, then see if we have a+ // chunk.+ builder.switch_to_block(backwards_block);+ builder.seal_block(backwards_block);+ let backwards_chunk_block = builder.create_block();+ let backwards_last_chunk_block = builder.create_block();+ let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {+ unreachable!()+ };+ *dst = builder.ins().iadd(*dst, *len);+ *src = builder.ins().iadd(*src, *len);+ has_chunk_branch(+ builder,+ &op,+ backwards_chunk_block,+ backwards_last_chunk_block,+ );++ // Execute the per-chunk backwards copy, adjusting pointers before+ // the copy itself.+ builder.switch_to_block(backwards_chunk_block);+ append_block_params(builder, backwards_chunk_block, &mut op);+ let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {+ unreachable!()+ };+ let remaining_len = *len;+ *len = chunk;+ *dst = builder.ins().isub(*dst, chunk);+ *src = builder.ins().isub(*src, chunk);+ raw_call(self, builder, &op);+ let BulkOp::MemoryCopy { len, .. } = &mut op else {+ unreachable!()+ };+ *len = builder.ins().isub(remaining_len, chunk);+ has_chunk_branch(+ builder,+ &op,+ backwards_chunk_block,+ backwards_last_chunk_block,+ );+ builder.seal_block(backwards_chunk_block);++ // Final backwards chunk: adjust the dst/src to be their true base+ // pointers and then delegate to `last_chunk_block` for the actual+ // memcpy.+ builder.switch_to_block(backwards_last_chunk_block);+ builder.seal_block(backwards_last_chunk_block);+ append_block_params(builder, backwards_last_chunk_block, &mut op);+ let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else {+ unreachable!()+ };+ *dst = builder.ins().isub(*dst, *len);+ *src = builder.ins().isub(*src, *len);+ builder.ins().jump(+ last_chunk_block,+ &[(*dst).into(), (*src).into(), (*len).into()],+ );+ }// In the final block we know that the length of the operation is less- // than `chunk`. This could still be sizable, though, so a final- // fuel/epoch check is inserted.+ // than `chunk`.builder.switch_to_block(last_chunk_block);+ builder.seal_block(last_chunk_block);append_block_params(builder, last_chunk_block, &mut op);raw_call(self, builder, &op);-- builder.seal_block(chunk_block);- builder.seal_block(last_chunk_block);}/// Emits a generic "fill" of `entity` from `dst` for `len` elements,@@ -6134,6 +6221,7 @@ fn index_type_to_ir_type(index_type: IndexType) -> ir::Type {}/// Operations to [`FuncEnvironment::raw_bulk_memory_operation`].+#[derive(Clone)]enum BulkOp {/// A `memory.copy` operation, copying memory from `src` to `dst`.///diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.watindex b0fc7703a5..709377b718 100644--- a/tests/disas/memory-copy-epochs.wat+++ b/tests/disas/memory-copy-epochs.wat@@ -34,7 +34,7 @@;; @001e v12 = call fn0(v0);; @001e jump block2(v12);;-;; block2(v76: i64):+;; block2(v65: i64):;; @0025 v17 = load.i64 notrap aligned v0+64;; @0025 v18 = uextend.i64 v2;; @0025 v19 = uextend.i64 v4@@ -45,52 +45,90 @@;; @0025 v35 = iadd v31, v19;; @0025 v36 = icmp ugt v35, v17;; @0025 trapnz v36, heap_oob-;; @0025 v44 = iconst.i64 0x0800_0000-;; @0025 v45 = icmp ugt v19, v44 ; v44 = 0x0800_0000;; @0025 v24 = load.i64 notrap aligned readonly can_move v0+56-;; @0025 v28 = iadd v24, v18;; @0025 v41 = iadd v24, v31-;; @0025 brif v45, block4(v28, v41, v19, v76), block5(v28, v41, v19, v76)-;;-;; block4(v46: i64, v47: i64, v48: i64, v51: i64):-;; @0025 v50 = load.i64 notrap aligned v6-;; @0025 v52 = icmp uge v50, v51-;; @0025 brif v52, block7, block6(v51)-;;-;; block5(v62: i64, v63: i64, v64: i64, v67: i64):-;; @0025 v66 = load.i64 notrap aligned v6-;; @0025 v68 = icmp uge v66, v67-;; @0025 brif v68, block10, block9-;;-;; block7 cold:-;; @0025 v54 = load.i64 notrap aligned v8+8-;; @0025 v55 = icmp.i64 uge v50, v54-;; @0025 brif v55, block8, block6(v54)-;;-;; block8 cold:-;; @0025 v57 = call fn0(v0)-;; @0025 jump block6(v57)-;;-;; block6(v77: i64):-;; v87 = iconst.i64 0x0800_0000-;; @0025 call fn1(v0, v46, v47, v87) ; v87 = 0x0800_0000-;; v88 = isub.i64 v48, v87 ; v87 = 0x0800_0000-;; v89 = icmp ugt v88, v87 ; v87 = 0x0800_0000-;; v90 = iadd.i64 v46, v87 ; v87 = 0x0800_0000-;; v91 = iadd.i64 v47, v87 ; v87 = 0x0800_0000-;; @0025 brif v89, block4(v90, v91, v88, v77), block5(v90, v91, v88, v77)+;; @0025 v28 = iadd v24, v18+;; @0025 v45 = icmp ugt v41, v28+;; @0025 brif v45, block6, block7+;;+;; block4(v47: i64, v48: i64, v49: i64, v52: i64):+;; @0025 v51 = load.i64 notrap aligned v6+;; @0025 v53 = icmp uge v51, v52+;; @0025 brif v53, block9, block8(v52)+;;+;; block5(v93: i64, v94: i64, v95: i64, v99: i64):+;; @0025 v98 = load.i64 notrap aligned v6+;; @0025 v101 = icmp uge v98, v99+;; @0025 brif v101, block17, block16+;;+;; block6:+;; v122 = iconst.i64 0x0800_0000+;; v123 = icmp.i64 ugt v19, v122 ; v122 = 0x0800_0000+;; @0025 brif v123, block4(v28, v41, v19, v65), block5(v28, v41, v19, v65)+;;+;; block9 cold:+;; @0025 v55 = load.i64 notrap aligned v8+8+;; @0025 v56 = icmp.i64 uge v51, v55+;; @0025 brif v56, block10, block8(v55);;;; block10 cold:-;; @0025 v70 = load.i64 notrap aligned v8+8-;; @0025 v71 = icmp.i64 uge v66, v70-;; @0025 brif v71, block11, block9+;; @0025 v58 = call fn0(v0)+;; @0025 jump block8(v58)+;;+;; block8(v66: i64):+;; @0025 call fn1(v0, v47, v48, v122) ; v122 = 0x0800_0000+;; @0025 v61 = isub.i64 v49, v122 ; v122 = 0x0800_0000+;; @0025 v62 = icmp ugt v61, v122 ; v122 = 0x0800_0000+;; @0025 v59 = iadd.i64 v47, v122 ; v122 = 0x0800_0000+;; @0025 v60 = iadd.i64 v48, v122 ; v122 = 0x0800_0000+;; @0025 brif v62, block4(v59, v60, v61, v66), block5(v59, v60, v61, v66)+;;+;; block7:+;; @0025 v44 = iconst.i64 0x0800_0000+;; @0025 v69 = icmp.i64 ugt v19, v44 ; v44 = 0x0800_0000+;; @0025 v67 = iadd.i64 v28, v19+;; @0025 v68 = iadd.i64 v41, v19+;; @0025 brif v69, block11(v67, v68, v19, v65), block12(v67, v68, v19, v65)+;;+;; block11(v70: i64, v71: i64, v72: i64, v77: i64):+;; @0025 v76 = load.i64 notrap aligned v6+;; @0025 v78 = icmp uge v76, v77+;; @0025 brif v78, block14, block13(v77)+;;+;; block14 cold:+;; @0025 v80 = load.i64 notrap aligned v8+8+;; @0025 v81 = icmp.i64 uge v76, v80+;; @0025 brif v81, block15, block13(v80)+;;+;; block15 cold:+;; @0025 v83 = call fn0(v0)+;; @0025 jump block13(v83)+;;+;; block13(v87: i64):+;; v117 = iconst.i64 0x0800_0000+;; v118 = isub.i64 v70, v117 ; v117 = 0x0800_0000+;; v119 = isub.i64 v71, v117 ; v117 = 0x0800_0000+;; @0025 call fn1(v0, v118, v119, v117) ; v117 = 0x0800_0000+;; v120 = isub.i64 v72, v117 ; v117 = 0x0800_0000+;; v121 = icmp ugt v120, v117 ; v117 = 0x0800_0000+;; @0025 brif v121, block11(v118, v119, v120, v87), block12(v118, v119, v120, v87)+;;+;; block12(v88: i64, v89: i64, v90: i64, v100: i64):+;; @0025 v91 = isub v88, v90+;; @0025 v92 = isub v89, v90+;; @0025 jump block5(v91, v92, v90, v100)+;;+;; block17 cold:+;; @0025 v103 = load.i64 notrap aligned v8+8+;; @0025 v104 = icmp.i64 uge v98, v103+;; @0025 brif v104, block18, block16;;-;; block11 cold:-;; @0025 v73 = call fn0(v0)-;; @0025 jump block9+;; block18 cold:+;; @0025 v106 = call fn0(v0)+;; @0025 jump block16;;-;; block9:-;; @0025 call fn1(v0, v62, v63, v64)+;; block16:+;; @0025 call fn1(v0, v93, v94, v95);; @0029 jump block1;;;; block1:diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.watindex 5eb9dcd2e4..ffc092b097 100644--- a/tests/disas/memory-copy-fuel.wat+++ b/tests/disas/memory-copy-fuel.wat@@ -32,8 +32,8 @@;; @001e brif v10, block2, block3(v8);;;; block2:-;; v96 = iadd.i64 v6, v7 ; v7 = 1-;; @001e store notrap aligned v96, v5+;; v125 = iadd.i64 v6, v7 ; v7 = 1+;; @001e store notrap aligned v125, v5;; @001e v13 = call fn0(v0);; @001e v15 = load.i64 notrap aligned v5;; @001e jump block3(v15)@@ -49,54 +49,93 @@;; @0025 v38 = iadd v34, v22;; @0025 v39 = icmp ugt v38, v20;; @0025 trapnz v39, heap_oob-;; @0025 v50 = iconst.i64 0x0800_0000-;; @0025 v51 = icmp ugt v22, v50 ; v50 = 0x0800_0000;; @0025 v27 = load.i64 notrap aligned readonly can_move v0+56-;; @0025 v31 = iadd v27, v21;; @0025 v44 = iadd v27, v34-;; @0025 v48 = iconst.i64 4-;; @0025 v49 = iadd v47, v48 ; v48 = 4-;; @0025 brif v51, block4(v31, v44, v22, v49), block5(v31, v44, v22, v49)-;;-;; block4(v52: i64, v53: i64, v54: i64, v55: i64):-;; v97 = iconst.i64 0x0800_0000-;; v98 = iadd v55, v97 ; v97 = 0x0800_0000-;; v99 = iconst.i64 0-;; v100 = icmp sge v98, v99 ; v99 = 0-;; @0025 brif v100, block6, block7(v98)-;;-;; block5(v68: i64, v69: i64, v70: i64, v71: i64):-;; @0025 v72 = iadd v71, v70-;; v106 = iconst.i64 0-;; v107 = icmp sge v72, v106 ; v106 = 0-;; @0025 brif v107, block8, block9(v72)+;; @0025 v31 = iadd v27, v21+;; @0025 v51 = icmp ugt v44, v31+;; @0025 brif v51, block6, block7+;;+;; block4(v53: i64, v54: i64, v55: i64, v56: i64):+;; @0025 v57 = iadd v56, v135 ; v135 = 0x0800_0000+;; v139 = iconst.i64 0+;; v140 = icmp sge v57, v139 ; v139 = 0+;; @0025 brif v140, block8, block9(v57)+;;+;; block5(v95: i64, v96: i64, v97: i64, v98: i64):+;; @0025 v100 = iadd v98, v97+;; v142 = iconst.i64 0+;; v143 = icmp sge v100, v142 ; v142 = 0+;; @0025 brif v143, block14, block15(v100);;;; block6:-;; @0025 store.i64 notrap aligned v98, v5-;; @0025 v61 = call fn0(v0)-;; @0025 v63 = load.i64 notrap aligned v5-;; @0025 jump block7(v63)-;;-;; block7(v80: i64):-;; v101 = iconst.i64 0x0800_0000-;; @0025 call fn1(v0, v52, v53, v101) ; v101 = 0x0800_0000-;; v102 = isub.i64 v54, v101 ; v101 = 0x0800_0000-;; v103 = icmp ugt v102, v101 ; v101 = 0x0800_0000-;; v104 = iadd.i64 v52, v101 ; v101 = 0x0800_0000-;; v105 = iadd.i64 v53, v101 ; v101 = 0x0800_0000-;; @0025 brif v103, block4(v104, v105, v102, v80), block5(v104, v105, v102, v80)+;; v135 = iconst.i64 0x0800_0000+;; v136 = icmp.i64 ugt v22, v135 ; v135 = 0x0800_0000+;; v137 = iconst.i64 4+;; v138 = iadd.i64 v47, v137 ; v137 = 4+;; @0025 brif v136, block4(v31, v44, v22, v138), block5(v31, v44, v22, v138);;;; block8:-;; @0025 store.i64 notrap aligned v72, v5-;; @0025 v77 = call fn0(v0)-;; @0025 v79 = load.i64 notrap aligned v5-;; @0025 jump block9(v79)+;; v141 = iadd.i64 v56, v135 ; v135 = 0x0800_0000+;; @0025 store notrap aligned v141, v5+;; @0025 v62 = call fn0(v0)+;; @0025 v64 = load.i64 notrap aligned v5+;; @0025 jump block9(v64)+;;+;; block9(v69: i64):+;; @0025 call fn1(v0, v53, v54, v135) ; v135 = 0x0800_0000+;; @0025 v67 = isub.i64 v55, v135 ; v135 = 0x0800_0000+;; @0025 v68 = icmp ugt v67, v135 ; v135 = 0x0800_0000+;; @0025 v65 = iadd.i64 v53, v135 ; v135 = 0x0800_0000+;; @0025 v66 = iadd.i64 v54, v135 ; v135 = 0x0800_0000+;; @0025 brif v68, block4(v65, v66, v67, v69), block5(v65, v66, v67, v69)+;;+;; block7:+;; @0025 v50 = iconst.i64 0x0800_0000+;; @0025 v72 = icmp.i64 ugt v22, v50 ; v50 = 0x0800_0000+;; @0025 v70 = iadd.i64 v31, v22+;; @0025 v71 = iadd.i64 v44, v22+;; @0025 v48 = iconst.i64 4+;; @0025 v49 = iadd.i64 v47, v48 ; v48 = 4+;; @0025 brif v72, block10(v70, v71, v22, v49), block11(v70, v71, v22, v49)+;;+;; block10(v73: i64, v74: i64, v75: i64, v78: i64):+;; v126 = iconst.i64 0x0800_0000+;; v127 = iadd v78, v126 ; v126 = 0x0800_0000+;; v128 = iconst.i64 0+;; v129 = icmp sge v127, v128 ; v128 = 0+;; @0025 brif v129, block12, block13(v127)+;;+;; block12:+;; @0025 store.i64 notrap aligned v127, v5+;; @0025 v84 = call fn0(v0)+;; @0025 v86 = load.i64 notrap aligned v5+;; @0025 jump block13(v86)+;;+;; block13(v89: i64):+;; v130 = iconst.i64 0x0800_0000+;; v131 = isub.i64 v73, v130 ; v130 = 0x0800_0000+;; v132 = isub.i64 v74, v130 ; v130 = 0x0800_0000+;; @0025 call fn1(v0, v131, v132, v130) ; v130 = 0x0800_0000+;; v133 = isub.i64 v75, v130 ; v130 = 0x0800_0000+;; v134 = icmp ugt v133, v130 ; v130 = 0x0800_0000+;; @0025 brif v134, block10(v131, v132, v133, v89), block11(v131, v132, v133, v89)+;;+;; block11(v90: i64, v91: i64, v92: i64, v99: i64):+;; @0025 v93 = isub v90, v92+;; @0025 v94 = isub v91, v92+;; @0025 jump block5(v93, v94, v92, v99)+;;+;; block14:+;; @0025 store.i64 notrap aligned v100, v5+;; @0025 v105 = call fn0(v0)+;; @0025 v107 = load.i64 notrap aligned v5+;; @0025 jump block15(v107);;-;; block9(v82: i64):-;; @0025 call fn1(v0, v68, v69, v70)+;; block15(v109: i64):+;; @0025 call fn1(v0, v95, v96, v97);; @0029 jump block1;;;; block1:-;; @0029 store.i64 notrap aligned v82, v5+;; @0029 store.i64 notrap aligned v109, v5;; @0029 return;; }diff --git a/tests/misc_testsuite/memory-copy.wast b/tests/misc_testsuite/memory-copy.wastindex 4b18d7f5d8..200abf763f 100644--- a/tests/misc_testsuite/memory-copy.wast+++ b/tests/misc_testsuite/memory-copy.wast@@ -1,6 +1,7 @@;;! bulk_memory = true;;! multi_memory = true;;! memory64 = true+;;! hogs_memory = true(module(memory 1 1)@@ -272,3 +273,17 @@(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const 100) (i32.const -1)) "out of bounds")(assert_trap (invoke "m64_to_m32" (i32.const -1) (i64.const 0) (i32.const 100)) "out of bounds")(assert_trap (invoke "m64_to_m32" (i32.const 0) (i64.const -1) (i32.const 100)) "out of bounds")++;; Copying a big chunk up one byte should do a backwards copy, not a forwards+;; copy, even if this is chunked up in to separate slices for fuel/epochs/etc.+(module+ (memory (export "mem") 3200 3200) ;; 200 MiB+ (func (export "run") (result i32)+ (i32.store8 (i32.const 134217727) (i32.const 0xAA))+ (i32.store8 (i32.const 134217728) (i32.const 0xBB))+ (memory.copy (i32.const 1) (i32.const 0) (i32.const 134217744))+ (i32.load8_u (i32.const 134217729))+ )+)++(assert_return (invoke "run") (i32.const 0xBB))