A lightweight WebAssembly runtime that is fast, secure, and standards-compliant
Cranelift: implicitly consider trapping blocks as "cold" during lowering (#13689)
Nick Fitzgeraldd95a3ec9e5a6
diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rsindex ee1d2debae..a194246ad8 100644--- a/cranelift/codegen/src/ir/function.rs+++ b/cranelift/codegen/src/ir/function.rs@@ -473,6 +473,20 @@ impl Function {pub fn import_function(&mut self, data: ExtFuncData) -> FuncRef {self.stencil.dfg.ext_funcs.push(data)}++ /// Is the given block marked `cold` or otherwise effectively `cold` in+ /// practice?+ pub fn is_effectively_cold(&self, block: Block) -> bool {+ if self.layout.is_cold(block) {+ return true;+ }++ // Blocks that unconditionally trap are effectively+ // also cold.+ self.layout+ .last_inst(block)+ .is_some_and(|inst| self.dfg.insts[inst].opcode() == ir::Opcode::Trap)+ }}/// Wrapper type capable of displaying a `Function`.diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rsindex 215b204eee..f19720e6c2 100644--- a/cranelift/codegen/src/machinst/blockorder.rs+++ b/cranelift/codegen/src/machinst/blockorder.rs@@ -257,7 +257,7 @@ impl BlockLoweringOrder {lowered_succ_indices.extend(block_succs[range].iter().map(|lb| lb_to_bindex[lb]));- if f.layout.is_cold(block) {+ if f.is_effectively_cold(block) {cold_blocks.insert(bindex);}@@ -282,7 +282,7 @@ impl BlockLoweringOrder {// Edges inherit indirect branch and cold block metadata from their// successor.- if f.layout.is_cold(succ) {+ if f.is_effectively_cold(succ) {cold_blocks.insert(bindex);}diff --git a/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clif b/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clifnew file mode 100644index 0000000000..a4d1863992--- /dev/null+++ b/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clif@@ -0,0 +1,40 @@+test compile precise-output+target pulley64++function %f(i8, i64) -> i64 {+block0(v0: i8, v1: i64):+ brif v0, block1, block2++;; This block is implicitly cold because it ends in a trap; it should come last+;; in the lowered block order.+block1:+ v2 = iconst.i32 42+ store notrap aligned v2, v1+ trap user123++block2:+ v3 = iconst.i64 0+ return v3+}++; VCode:+; block0:+; zext8 x5, x0+; br_if32 x5, label2; jump label1+; block1:+; xzero x0+; ret+; block2:+; xconst8 x7, 42+; xstore32 x1+0, x7 // flags = notrap aligned+; trap // trap=TrapCode(123)+;+; Disassembled:+; zext8 x5, x0+; br_if32 x5, 0x9 // target = 0xc+; xzero x0+; ret+; xconst8 x7, 42+; xstore32le_o32 x1, 0, x7+; trap+diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.watindex c903d304a5..327084f2f5 100644--- a/tests/disas/debug-exceptions.wat+++ b/tests/disas/debug-exceptions.wat@@ -31,35 +31,35 @@;; stur x0, [sp, #0x18];; mov x0, sp;; cmp x0, x1-;; b.lo #0x21c+;; b.lo #0x220;; 48: stur x2, [sp];; mov x0, x2;; stur x2, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x35, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [61, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [62, 1, 0, 148];; ldur x0, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [59, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [60, 1, 0, 148];; ldur x0, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3d, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [57, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [58, 1, 0, 148];; mov w1, #0x2a;; stur w1, [sp, #8];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3f, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [54, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [55, 1, 0, 148];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x40, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [53, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [54, 1, 0, 148];; stur w1, [sp, #8];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [51, 1, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [52, 1, 0, 148];; ldur x2, [sp, #0x10]-;; bl #0x4b0+;; bl #0x4b4;; 88: ldur x0, [sp, #0x10];; mov x19, x2;; ldr x1, [x0, #0x20]@@ -68,7 +68,7 @@;; add x2, x2, #0x20;; ldr w4, [x1, #4];; cmp x2, x4-;; b.hi #0x1b4+;; b.hi #0x1a0;; ac: ldur x0, [sp, #0x18];; add w4, w3, #0x20;; str w4, [x1]@@ -92,47 +92,41 @@;; mov w11, #0;; str w11, [x1, #0x14];; ldur x2, [sp, #0x10]-;; bl #0x4e8+;; bl #0x4ec;; ├─╼ exception frame offset: SP = FP - 0xb0-;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x124-;; 10c: mov w3, #9-;; ldur x2, [sp, #0x10]-;; bl #0x444-;; 118: ldur x2, [sp, #0x10]-;; bl #0x47c-;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; 120: udf #0xc11f-;; mov w1, w0+;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x110+;; b #0x1d8+;; 110: mov w1, w0;; mov x2, #0x20;; adds x15, x1, x2;; cset x1, hs;; tst w1, #0xff-;; b.ne #0x204-;; 13c: ldur x2, [sp, #0x18]+;; b.ne #0x208+;; 128: ldur x2, [sp, #0x18];; ldr x1, [x2, #0x28];; cmp x15, x1-;; b.hi #0x1ec-;; 14c: ldr x1, [x2, #0x20]+;; b.hi #0x1f0+;; 138: ldr x1, [x2, #0x20];; add x1, x1, #0x18;; ldr w0, [x1, w0, uxtw];; stur w0, [sp, #8];; ldur x0, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x48, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [250, 0, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [0, 1, 0, 148];; ldur x1, [sp, #0x10];; ldr x0, [x1, #0x38];; ldr x2, [x1, #0x48];; ldur x3, [sp, #0x10];; blr x0;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; 178: ldur x0, [sp, #0x10]+;; 164: ldur x0, [sp, #0x10];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8-;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [243, 0, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [249, 0, 0, 148];; nop;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4b, slot at FP-0xb0, locals , stack-;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [242, 0, 0, 148]+;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [248, 0, 0, 148];; add sp, sp, #0x20;; ldp d8, d9, [sp], #0x10;; ldp d10, d11, [sp], #0x10@@ -145,38 +139,45 @@;; ldp x27, x28, [sp], #0x10;; ldp x29, x30, [sp], #0x10;; ret-;; 1b4: mov w3, #2-;; 1b8: movk w3, #0x400, lsl #16-;; 1bc: ldur x0, [sp, #0x10]-;; 1c0: ldr x4, [x0, #0x28]-;; 1c4: ldr w4, [x4, #8]-;; 1c8: mov w5, #0x20-;; 1cc: mov w6, #0x10-;; 1d0: ldur x2, [sp, #0x10]-;; 1d4: bl #0x3dc-;; 1d8: ldur x0, [sp, #0x18]-;; 1dc: ldr x3, [x0, #0x20]-;; 1e0: add x1, x3, w2, uxtw-;; 1e4: mov x3, x2-;; 1e8: b #0xec-;; 1ec: mov w3, #0xfe-;; 1f0: ldur x2, [sp, #0x10]-;; 1f4: bl #0x444-;; 1f8: ldur x2, [sp, #0x10]-;; 1fc: bl #0x47c+;; 1a0: mov w3, #2+;; 1a4: movk w3, #0x400, lsl #16+;; 1a8: ldur x0, [sp, #0x10]+;; 1ac: ldr x4, [x0, #0x28]+;; 1b0: ldr w4, [x4, #8]+;; 1b4: mov w5, #0x20+;; 1b8: mov w6, #0x10+;; 1bc: ldur x2, [sp, #0x10]+;; 1c0: bl #0x3e0+;; 1c4: ldur x0, [sp, #0x18]+;; 1c8: ldr x3, [x0, #0x20]+;; 1cc: add x1, x3, w2, uxtw+;; 1d0: mov x3, x2+;; 1d4: b #0xec+;; 1d8: mov w3, #9+;; 1dc: ldur x2, [sp, #0x10]+;; 1e0: bl #0x448+;; 1e4: ldur x2, [sp, #0x10]+;; 1e8: bl #0x480+;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8+;; 1ec: udf #0xc11f+;; 1f0: mov w3, #0xfe+;; 1f4: ldur x2, [sp, #0x10]+;; 1f8: bl #0x448+;; 1fc: ldur x2, [sp, #0x10]+;; 200: bl #0x480;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack-;; 200: udf #0xc11f-;; 204: mov w3, #0xfe-;; 208: ldur x2, [sp, #0x10]-;; 20c: bl #0x444-;; 210: ldur x2, [sp, #0x10]-;; 214: bl #0x47c+;; 204: udf #0xc11f+;; 208: mov w3, #0xfe+;; 20c: ldur x2, [sp, #0x10]+;; 210: bl #0x448+;; 214: ldur x2, [sp, #0x10]+;; 218: bl #0x480;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack-;; 218: udf #0xc11f-;; 21c: stur x2, [sp, #0x10]-;; 220: mov w3, #0-;; 224: bl #0x444-;; 228: ldur x2, [sp, #0x10]-;; 22c: bl #0x47c+;; 21c: udf #0xc11f+;; 220: stur x2, [sp, #0x10]+;; 224: mov w3, #0+;; 228: bl #0x448+;; 22c: ldur x2, [sp, #0x10]+;; 230: bl #0x480;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x34, slot at FP-0xb0, locals , stack-;; 230: udf #0xc11f+;; 234: udf #0xc11fdiff --git a/tests/disas/exceptions.wat b/tests/disas/exceptions.watindex 8af24433e7..efee2b2931 100644--- a/tests/disas/exceptions.wat+++ b/tests/disas/exceptions.wat@@ -23,7 +23,7 @@;; movq 0x18(%r10), %r10;; addq $0x50, %r10;; cmpq %rsp, %r10-;; ja 0xe7+;; ja 0xec;; 19: subq $0x40, %rsp;; movq %rbx, 0x10(%rsp);; movq %r12, 0x18(%rsp)@@ -41,7 +41,7 @@;; leaq 0x30(%rcx), %rax;; movl 4(%rdx), %edi;; cmpq %rdi, %rax-;; ja 0xb3+;; ja 0xb6;; 5f: leal 0x30(%rsi), %eax;; movl %eax, (%rdx);; movq 8(%rbx), %rax@@ -63,8 +63,8 @@;; movq %rbx, %rdi;; movq %rbx, (%rsp);; callq 0x418-;; ud2-;; movl $0x4000002, %esi+;; jmp 0xea+;; b6: movl $0x4000002, %esi;; movq 0x28(%rbx), %rax;; movl 0xc(%rax), %edx;; movl $0x30, %ecx@@ -77,7 +77,8 @@;; movq %rax, %rsi;; movq %r14, %rdi;; jmp 0x8e-;; e7: ud2+;; ea: ud2+;; ud2;;;; wasm[0]::function[1]::catch:;; pushq %rbp